# if you want to add own robot rules, do it BEFORE the final rule matching *

# We allow search.fsfe.org
User-agent: yacybot
Allow: /
Crawl-delay: 3

# We disallow all other bots
User-agent: *
Disallow: /
Crawl-delay: 20

# This has to match script url + cfg.url_prefix_action - it
# saves lots of search engine load and traffic by disallowing crawlers
# to request action related URLs.
#
# NOTE - in order to make this have any effect, you have to set
#        url_prefix_action to "action", cf. HelpOnConfiguration

Disallow: /action/