############################### # # User-agent: * # # list folders robots are not allowed to index # Disallow: /site/images/ Disallow: /site/includes/ Disallow: /site/errmsg/ Disallow: /site/administration/ Disallow: /Email/ Disallow: /sandx/ Disallow: /BB_media/ Disallow: /old_site/ Disallow: /site_1/ Disallow: /Course/DOM_1/ Disallow: /course/int_1/ # # list specific files robots are not allowed to index # # Disallow: /directory/foo.html # Disallow: /folder2/whatever.html # # ###############################