# ******************************************************* # /robot.txt - Controls indexing of pages by web robots # ******************************************************* # # Place a single User-agent identifier, followed by any number of Disallow statements. # Each one prevents indexing of this folder (or file), and any folders below this folder. # NOTE: this file and its content is case sensitive! #User-agent: * # applies to all robots #Disallow: /fileorfolder # all files and folders with this name #Disallow: /thisfolder/ # disallow a folder with this name #Disallow: /thisfolder/thisfile.htm # disallow just this file User-agent: * Disallow: / # This line deals with Yahoo's bots who are hitting this site very hard # It forces the bots to time out for a number of seconds before crwaling the site #User-agent: Slurp #Crawl-delay: 60000