#/robots.txt # Limit access to robots User-agent: * # directed to all robots Crawl-Delay: 30 Disallow: /public