User-agent: Googlebot
# disallow all files with ? in url

     User-Agent: *
Allow: /sitemap.html

          # All robots will spider the domain
User-agent: *
Disallow: /cgi-bin
Disallow: /etc
Disallow: /shared-cgi-bin
Disallow: /raw.logs
Disallow: /test1.htm
Disallow: /photo-directory.html
Disallow: /~site/Scripts_statcounter