# As a condition of accessing this website, you agree to abide by the following
# content signals:

# (a)  If a Content-Signal = yes, you may collect content for the corresponding
#      use.
# (b)  If a Content-Signal = no, you may not collect content for the
#      corresponding use.
# (c)  If the website operator does not include a Content-Signal for a
#      corresponding use, the website operator neither grants nor restricts
#      permission via Content-Signal with respect to the corresponding use.

# The content signals and their meanings are:

# search:   building a search index and providing search results (e.g., returning
#           hyperlinks and short excerpts from your website's contents). Search does not
#           include providing AI-generated search summaries.
# ai-input: inputting content into one or more AI models (e.g., retrieval
#           augmented generation, grounding, or other real-time taking of content for
#           generative AI search answers).
# ai-train: training or fine-tuning AI models.

# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT
# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.

# BEGIN Cloudflare Managed content

User-agent: *
Content-Signal: search=yes,ai-train=no
Allow: /

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CloudflareBrowserRenderingCrawler
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# END Cloudflare Managed Content

# ============================================================
# Ubuntu Guard — robots.txt
# Last updated: 2026-05-05
# ============================================================

# --- AI Crawlers (explicitly welcome) ---
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Amazonbot
User-agent: Google-Extended
User-agent: Gemini-crawler
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: FacebookBot
User-agent: Applebot
User-agent: Bytespider
User-agent: Diffbot
User-agent: CCBot
User-agent: cohere-ai
User-agent: AI2Bot
User-agent: Timpibot
Allow: /
Disallow: /api/
Disallow: /inc/
Disallow: /sql/
Disallow: /data/
Disallow: /scamnet/api/
Disallow: /clickguard/api/
Disallow: /clickguard/admin/
Disallow: /clickguard/logs/
Disallow: /goldphish/php/
Disallow: /goldphish/data/
Disallow: /scamnet/login.html
Disallow: /scamnet/dash.html
Disallow: /scamnet/report.html
Disallow: /old_pass/
Disallow: /.vscode/
Disallow: /.claude/
Disallow: /cdn-cgi/
Disallow: /cyber-toolkit/blog/
# --- CheckURLink V2 (deployed at /checkurlink/) ---
# Public pages (homepage, privacy, terms) are crawlable; the rest is blocked.
# /api returns JSON — never useful in search results.
# /app, /data, /log are also blocked at the server level via .htaccess; these
# entries are defence-in-depth so crawlers don't waste budget probing them.
# /r/ is the user-generated permalink resolver — each id is a per-scan snapshot
# that expires after 90 days, so indexing them would just feed Google a stream
# of stale 410s. Sharing on WhatsApp / email still works either way.
Disallow: /checkurlink/api/
Disallow: /checkurlink/app/
Disallow: /checkurlink/data/
Disallow: /checkurlink/log/
Disallow: /checkurlink/r/

# --- All other crawlers (Google, Bing, etc.) ---
User-agent: *
Allow: /
Disallow: /api/
Disallow: /inc/
Disallow: /sql/
Disallow: /data/
Disallow: /scamnet/api/
Disallow: /clickguard/api/
Disallow: /clickguard/admin/
Disallow: /clickguard/logs/
Disallow: /goldphish/php/
Disallow: /goldphish/data/
Disallow: /scamnet/login.html
Disallow: /scamnet/dash.html
Disallow: /scamnet/report.html
Disallow: /old_pass/
Disallow: /.vscode/
Disallow: /.claude/
Disallow: /cdn-cgi/
Disallow: /cyber-toolkit/blog/
# CheckURLink V2 (see comment block above for rationale)
Disallow: /checkurlink/api/
Disallow: /checkurlink/app/
Disallow: /checkurlink/data/
Disallow: /checkurlink/log/
Disallow: /checkurlink/r/

Sitemap: https://www.ubuntuguard.co.za/sitemap.xml