# robots.txt file tells web crawlers not to look in certain directories User-agent: * Disallow: /cgi-bin Disallow: /bin