# address any search engine robot/spider

User-agent: *


# block indexing of whole directories

Disallow: /cgi-bin/
Disallow: /media/
Disallow: /images/
Disallow: /redirect/
Disallow: /otherstuff/

# block indexing of specific files

Disallow: /gallery.html
Disallow: /theory_general.css