#This code applies to every spider. User-agent: * #Stay out of these directories Disallow: /cgi-bin/ #don't spider any files in this directory. Disallow: /utility/ #don't spider any files in this directory. Disallow: /utilities/ #don't spider any files in this directory.