# www.robotstxt.org/ # Allow all crawlers User-agent: * Allow: / # Disallow crawling of specific folders (uncomment if needed) # Disallow: /admin/ # Crawl-delay for all bots (optional, in seconds) # Crawl-delay: 10 # Block specific bots (if needed) # User-agent: BadBot # Disallow: /