Crawl-delay: 10
 
# START WPFORMS BLOCK
# ---------------------------
User-agent: *
Disallow: /wp-content/uploads/wpforms/
# ---------------------------
# END WPFORMS BLOCK
Crawl-delay: 10
 
# START CRAWL-TRAP FIX
# ---------------------------
# /find-a-collaborator/ has a taxonomy filter that generates a near-infinite
# number of crawlable URL combinations (?tax_areas-of-interest=a,b,c,...).
# Confirmed Bingbot traffic was found crawling these combinations, which is
# contributing to server overload / 504s on that page. This blocks the
# filtered variants while still allowing the base listing page to be indexed.
User-agent: *
Disallow: /find-a-collaborator/?*
# ---------------------------
# END CRAWL-TRAP FIX
 
# START YOAST BLOCK
# ---------------------------
User-agent: *
Disallow:
 
Sitemap: https://www.foldnet.uk/sitemap_index.xml
# ---------------------------
# END YOAST BLOCK
 
# --- Optional: block named AI-training crawlers ---
# Separate from the crawl-trap fix above (that blocks ALL bots from the
# problem URLs). If you also want this site opted out of AI-training
# crawling more broadly, uncomment the block(s) you want. Has no effect on
# scrapers that ignore robots.txt entirely (see the traffic report for why
# rate-limiting / a WAF is the tool for those).
#
# User-agent: GPTBot
# Disallow: /
#
# User-agent: ChatGPT-User
# Disallow: /
#
# User-agent: Google-Extended
# Disallow: /
#
# User-agent: CCBot
# Disallow: /
#
# User-agent: Bytespider
# Disallow: /
#
# User-agent: PerplexityBot
# Disallow: /
#
# User-agent: ClaudeBot
# Disallow: /
#
# User-agent: anthropic-ai
# Disallow: /