User-agent: *
Disallow: /*/print$
# Don't allow indexing of search results
Disallow: /search?
# We only allow indexing of the licence-finder landing page
Disallow: /licence-finder/
Allow: /licence-finder
Disallow: /apply-for-a-licence/
# Don't allow indexing of user needs pages
Disallow: /info/*
Sitemap: https://www.gov.uk/sitemap.xml
Crawl-delay: 0.5
# https://ahrefs.com/robot/ crawls the site frequently
User-agent: AhrefsBot
Crawl-delay: 10
# https://www.deepcrawl.com/bot/ makes lots of requests. Ideally
# we'd slow it down rather than blocking it but it doesn't mention
# whether or not it supports crawl-delay.
User-agent: deepcrawl
Disallow: /