# robots.txt all crawlers all pages except cgi-bin
User-agent: *
Disallow: 
Disallow: /cgi-bin/