# http://www.it-audit.de/robots.txt

User-agent: crawl-it
Disallow: web.archive.org
Disallow: /cgi-bin/
Disallow: /stats/
Disallow: /images/
Disallow: /assets/
Disallow: /logs/
Disallow: /extras/
Disallow: /special/
Disallow: /diverse/
Disallow: /korr*
Disallow: /preview*

User-agent: *
Disallow: /cgi-bin/
Disallow: /stats/
Disallow: /images/
Disallow: /logs/
Disallow: /extras/
Disallow: /korr*
Disallow: /preview*
Disallow: /special/
Disallow: /diverse/