# robots.txt for http://oepg-fakt.itp.tuwien.ac.at/ User-agent: * Disallow: /download # These are usually binary files (.zip) Disallow: /copy # this is just a copy of an old version Disallow: /counter # internal counter info - not for robots Disallow: /oldfakt # the intermedia page should not be roboted any more Disallow: /xpoll # internal xpoll info - not for robots Disallow: /save # yet another copy Disallow: /help # just internal info Disallow: /newstuff # quasi-temporary directory with newly downloaded files