# addresses all robots by using wild card * # User-agent: * # # list folders robots are not allowed to index # Disallow: /cgi-bin/ Disallow: /album/ Disallow: /john/ Disallow: /john/album/ # # list specific files robots are not allowed to index # Disallow: /*.mpg$ Disallow: /*.css$ Disallow: /*.jpg$ Disallow: /*.gif$