# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism

# User-agent: webcrawler
# Disallow:

# User-agent: lycra
# Disallow: /

User-agent: *
Disallow: /eshop
Disallow: /intern
Disallow: /intra
Disallow: /testa
Disallow: /reliadb
Disallow: /tmp
Disallow: /logs
Disallow: /upload
Disallow: /uploaded
Disallow: /timereg
Disallow: /vf2001
Disallow: /reguma
Disallow: /hviding_utgatt

# Mrk:
# Wildcards are _not_ supported: instead of 'Disallow: /tmp/*' just say 'Disallow: /tmp'
# You shouldn't put more than one path on a Disallow line
# Disallow: /help disallows both /help.html and /help/index.html, whereas 
# Disallow: /help/ would disallow /help/index.html but allow /help.html. 


# tips til html-dokumentene
# <META NAME="ROBOTS" CONTENT="NOINDEX">
# <META NAME="ROBOTS" CONTENT="NOFOLLOW">