# robots.txt # User-agent: * # applies to all robots Disallow: / # disallow indexing of all pages #Disallow: /images # this folder #User-agent: Google # allow Google #Disallow: # To prevent all robots from visiting the entire site # and indexing spiders and robots from indexing files. # # Set the operating systems permissions, # so only the default document is visible and # all other directories cannot be browsed.