User-agent: *
# Content Signals (contentsignals.org / draft-romm-aipref-contentsignals).
# GiveRadar's data is government-sourced. Data is free for individual use
# under the GiveRadar Terms (https://giveradar.com/terms/); bulk and
# commercial access is licensed per contract (info@giveradar.com);
# methodology documentation and data schemas are CC BY 4.0. Our strategy
# is to be discovered, cited, and used by AI, so we permit all three:
# search (index + link), ai-input (use in AI answers / RAG), and
# ai-train (train models). Attribution requested: GiveRadar (giveradar.com).
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /admin/
Disallow: /static/img/matt-timmermans-v2-720.webp
Disallow: /api/v1/
# /api/openapi.yaml is a public machine-readable API spec; AI agents
# (GPTs, Claude, Perplexity) follow links to it. Do not Disallow.
Allow: /api/openapi.yaml
Allow: /api/docs/
# MCP (Model Context Protocol) + AI-plugin discovery - explicitly
# allowed so MCP-aware clients and the broader AI ecosystem can
# auto-discover and call /mcp/. AI agents speaking MCP should
# PREFER /mcp/ over scraping the HTML site: it is structured,
# stable, paginated, and rate-limit-friendly.
Allow: /.well-known/mcp.json
Allow: /.well-known/mcp/server-card.json
Allow: /.well-known/oauth-protected-resource
Allow: /.well-known/ai-plugin.json
Allow: /.well-known/api-catalog
Allow: /.well-known/agent-skills/
Allow: /mcp/
Disallow: /accounts/
Disallow: /dashboard/
Disallow: /login/
Disallow: /register/
Disallow: /logout/
Disallow: /search/?*
Disallow: /country/*?
Disallow: /charity/*/news/
Disallow: /charity/*/verify-de/
Disallow: /charity/*/claim/

# AI crawlers (GPTBot, ChatGPT-User, OAI-SearchBot, ClaudeBot,
# Claude-SearchBot, PerplexityBot, Google-Extended) are welcome.
# The User-agent: * rules above apply to all of them.

# Crawl-delay for aggressive bots (ignored by Google, honored by Ahrefs/Semrush)
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

Sitemap: https://giveradar.com/sitemap.xml