# /robots.txt for http://www.murdoch.edu.au/ #last modified, June 3rd 2009 De Stanton # comments to cwis@www.murdoch.edu.au # # use # to suggest html files be skipped on a per file basis # User-agent: bad_robot Disallow: /test/apple/file1.html Allow: /cwisad Allow: /about Disallow: / User-agent: webcrawler User-agent: excite Allow: / Disallow: /test Disallow: /dev User-agent: * Disallow: /test Disallow: /dev Disallow:/goto/stats Disallow: /stats Disallow: /server/stats Disallow: /cgi-bin/tt-check Disallow: /index/feedback Disallow: /index/pageinfo Disallow: /index/policies/pageinfo Disallow: /cwisad Disallow: /wnew