Robots.txt Audit Claude Skill

#!/usr/bin/env python3
"""Fetch and parse a site's robots.txt, then probe its sitemap and resource references into a JSON inventory.

Usage:
    python3 collect_robots.py https://example.com [--output robots_inventory.json]
    python3 collect_robots.py https://example.com/robots.txt --robots [--output robots_inventory.json]

It fetches /robots.txt (without auto-following redirects, so a redirected or
HTML-served file is visible), records HTTP status, content type, byte size, gzip,
and a BOM flag, then parses it into user-agent groups, Sitemap: directives,
comments, unknown/unsupported directives, and per-line syntax errors. It probes
every declared Sitemap: URL for status, and fetches the homepage to extract
same-host CSS/JS/image resources and test each one against the '*' group's rules
so resources blocked from crawling can be flagged.

Output JSON schema:
{
  "site": "https://example.com",
  "robots_url": "https://example.com/robots.txt",
  "fetch": {"found": true, "status": 200, "content_type": "text/plain",
            "bytes": 1234, "gzip": false, "has_bom": false,
            "redirected": false, "final_url": "...", "looks_like_html": false},
  "raw": "<full robots.txt text>",
  "groups": [{"agents": ["*"], "disallow": ["/admin"], "allow": ["/admin/public"],
              "crawl_delay": null, "other": []}],
  "sitemaps": [{"url": "...", "absolute": true, "status": 200}],
  "comments": 3,
  "unknown_directives": [{"line": 12, "field": "noindex", "value": "/x"}],
  "line_errors": [{"line": 4, "text": "...", "error": "missing colon"}],
  "has_star_group": true,
  "sitewide_block_agents": ["*"],
  "resources": [{"url": "...", "type": "css", "path": "/a.css", "blocked": false}]
}

Uses only the Python standard library.
"""

import argparse
import gzip
import io
import json
import sys
import urllib.error
import urllib.parse
import urllib.request
from html.parser import HTMLParser

USER_AGENT = "RobotsAuditBot/1.0 (+claude-skill)"
TIMEOUT = 15
KNOWN_FIELDS = {"user-agent", "disallow", "allow", "sitemap", "crawl-delay", "host", "clean-param"}


class ResourceParser(HTMLParser):
    """Extract same-host CSS / JS / image resource URLs from a page."""

    def __init__(self, base_url):
        super().__init__(convert_charrefs=True)
        self.base_url = base_url
        self.resources = []

    def _add(self, kind, href):
        if href:
            self.resources.append((kind, urllib.parse.urljoin(self.base_url, href.strip())))

    def handle_starttag(self, tag, attrs):
        a = dict(attrs)
        if tag == "link" and "stylesheet" in (a.get("rel") or "").lower():
            self._add("css", a.get("href"))
        elif tag == "script" and a.get("src"):
            self._add("js", a.get("src"))
        elif tag == "img" and a.get("src"):
            self._add("img", a.get("src"))


def norm_host(host):
    host = host.lower()
    return host[4:] if host.startswith("www.") else host


def raw_get(url, max_bytes=10_000_000):
    """Plain GET (auto-follows redirects). Returns (status, final_url, raw_bytes, headers)."""
    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
    try:
        with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
            return resp.status, resp.geturl(), resp.read(max_bytes), resp.headers
    except urllib.error.HTTPError as e:
        return e.code, url, b"", (e.headers if e.headers else None)
    except Exception:
        return 0, url, b"", None


def fetch_no_follow(url, max_bytes=2_000_000):
    """Fetch without following redirects. Returns (status, location, raw_bytes, headers)."""

    class NoRedirect(urllib.request.HTTPRedirectHandler):
        def redirect_request(self, req, fp, code, msg, headers, newurl):
            return None

    opener = urllib.request.build_opener(NoRedirect)
    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
    try:
        with opener.open(req, timeout=TIMEOUT) as resp:
            return resp.status, None, resp.read(max_bytes), resp.headers
    except urllib.error.HTTPError as e:
        loc = e.headers.get("Location") if e.headers else None
        return e.code, loc, b"", (e.headers if e.headers else None)
    except Exception:
        return 0, None, b"", None


def decode_body(raw, url, headers):
    """Return (text, is_gzip), handling gzip by header, .gz extension, or magic bytes."""
    is_gzip = url.lower().endswith(".gz")
    if headers is not None and "gzip" in (headers.get("Content-Encoding", "") or "").lower():
        is_gzip = True
    if raw[:2] == b"\x1f\x8b":
        is_gzip = True
    if is_gzip:
        try:
            raw = gzip.GzipFile(fileobj=io.BytesIO(raw)).read()
        except Exception:
            pass
    return raw.decode("utf-8", errors="replace"), is_gzip


def parse_robots(text):
    """Parse robots.txt into groups, sitemaps, comments, unknown directives, and line errors."""
    groups, sitemaps, comments = [], [], 0
    unknown, errors = [], []
    current, last_directive = None, False
    for i, raw_line in enumerate(text.splitlines(), 1):
        stripped = raw_line.strip()
        if not stripped:
            continue
        if stripped.startswith("#"):
            comments += 1
            continue
        line = raw_line.split("#", 1)[0].strip()
        if not line:
            comments += 1
            continue
        if ":" not in line:
            errors.append({"line": i, "text": raw_line, "error": "missing colon"})
            continue
        field, _, value = line.partition(":")
        field, value = field.strip().lower(), value.strip()
        if field == "sitemap":
            if value:
                sitemaps.append(value)
            else:
                errors.append({"line": i, "text": raw_line, "error": "empty Sitemap value"})
        elif field == "user-agent":
            if current is None or last_directive:
                current = {"agents": [], "disallow": [], "allow": [],
                           "crawl_delay": None, "other": []}
                groups.append(current)
            if value:
                current["agents"].append(value)
            else:
                errors.append({"line": i, "text": raw_line, "error": "empty User-agent value"})
            last_directive = False
        elif field in ("disallow", "allow"):
            if current is None:
                errors.append({"line": i, "text": raw_line,
                               "error": f"{field} before any User-agent"})
            else:
                current[field].append(value)
                last_directive = True
        elif field == "crawl-delay":
            if current is not None:
                current["crawl_delay"] = value
                last_directive = True
        elif field in KNOWN_FIELDS:
            if current is not None:
                current["other"].append({"field": field, "value": value})
                last_directive = True
        else:
            unknown.append({"line": i, "field": field, "value": value})
    return {"groups": groups, "sitemaps": sitemaps, "comments": comments,
            "unknown": unknown, "errors": errors}


def robots_disallowed(path, groups):
    """True if the '*' user-agent group disallows the path (longest-match wins)."""
    star = [g for g in groups if "*" in g["agents"]]
    if not star:
        return False
    best_len, decision = -1, False
    for g in star:
        for rule, allow in [(d, False) for d in g["disallow"]] + [(a, True) for a in g["allow"]]:
            if rule == "":
                continue
            if path.startswith(rule) and len(rule) > best_len:
                best_len, decision = len(rule), allow
    return decision is False and best_len >= 0


def is_sitewide_block(group):
    """A group blocks the whole site if it disallows '/' and has no allow that re-opens root."""
    if "/" not in group["disallow"]:
        return False
    return not any(a == "/" for a in group["allow"])


def main():
    ap = argparse.ArgumentParser(description="Fetch, parse, and probe a site's robots.txt.")
    ap.add_argument("source", help="Site root URL, or a robots.txt URL with --robots")
    ap.add_argument("--robots", action="store_true", help="Treat source as a robots.txt URL directly")
    ap.add_argument("--no-resources", action="store_true",
                    help="skip homepage resource extraction / blocked-resource check")
    ap.add_argument("--output", default="robots_inventory.json")
    args = ap.parse_args()

    if args.robots:
        pu = urllib.parse.urlparse(args.source)
        site_root = f"{pu.scheme}://{pu.netloc}"
        robots_url = args.source
    else:
        site_root = args.source.rstrip("/")
        robots_url = site_root + "/robots.txt"

    status, location, raw, headers = fetch_no_follow(robots_url)
    redirected = 300 <= status < 400
    final_url = urllib.parse.urljoin(robots_url, location) if (redirected and location) else robots_url
    content_type = (headers.get("Content-Type", "") if headers else "") or ""
    has_bom = raw[:3] == b"\xef\xbb\xbf"
    text, is_gzip = decode_body(raw, robots_url, headers)
    if has_bom:
        text = text.lstrip("")
    looks_like_html = "<html" in text[:2000].lower() or "<!doctype html" in text[:2000].lower()

    fetch = {
        "found": status == 200, "status": status, "content_type": content_type,
        "bytes": len(raw), "gzip": is_gzip, "has_bom": has_bom,
        "redirected": redirected, "final_url": final_url, "looks_like_html": looks_like_html,
    }
    print(f"robots.txt: [{status}] {robots_url} ({len(raw)} bytes)", file=sys.stderr)

    parsed = parse_robots(text) if (status == 200 and not looks_like_html) else {
        "groups": [], "sitemaps": [], "comments": 0, "unknown": [], "errors": []}

    # Probe declared sitemaps.
    site_host = norm_host(urllib.parse.urlparse(site_root).netloc)
    sitemaps = []
    for sm in parsed["sitemaps"]:
        absolute = bool(urllib.parse.urlparse(sm).scheme)
        entry = {"url": sm, "absolute": absolute, "status": None}
        if absolute:
            st, _f, _r, _h = raw_get(sm, max_bytes=2_000_000)
            entry["status"] = st
            print(f"  Sitemap [{st}] {sm}", file=sys.stderr)
        sitemaps.append(entry)

    groups = parsed["groups"]
    sitewide = [a for g in groups if is_sitewide_block(g) for a in g["agents"]]

    # Extract and test homepage resources.
    resources = []
    if not args.no_resources and status == 200 and not looks_like_html:
        hp_status, _f, hp_raw, hp_headers = raw_get(site_root, max_bytes=3_000_000)
        if hp_status == 200 and hp_raw:
            body, _g = decode_body(hp_raw, site_root, hp_headers)
            parser = ResourceParser(site_root)
            try:
                parser.feed(body)
            except Exception:
                pass
            seen = set()
            for kind, url in parser.resources:
                if url in seen:
                    continue
                seen.add(url)
                pu = urllib.parse.urlparse(url)
                if pu.netloc and norm_host(pu.netloc) != site_host:
                    continue  # only test same-host resources
                path = pu.path or "/"
                resources.append({"url": url, "type": kind, "path": path,
                                  "blocked": robots_disallowed(path, groups)})
            print(f"  Resources extracted: {len(resources)}", file=sys.stderr)

    out = {
        "site": site_root,
        "robots_url": robots_url,
        "fetch": fetch,
        "raw": text,
        "groups": groups,
        "sitemaps": sitemaps,
        "comments": parsed["comments"],
        "unknown_directives": parsed["unknown"],
        "line_errors": parsed["errors"],
        "has_star_group": any("*" in g["agents"] for g in groups),
        "sitewide_block_agents": sitewide,
        "resources": resources,
    }
    with open(args.output, "w", encoding="utf-8") as f:
        json.dump(out, f, indent=2, ensure_ascii=False)
    print(f"Wrote {len(groups)} groups, {len(sitemaps)} sitemaps, "
          f"{len(resources)} resources to {args.output}", file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Audit a robots.txt inventory JSON (from collect_robots.py) and emit findings.

Usage:
    python3 audit_robots.py robots_inventory.json [--output audit_report.json]

Checks implemented (see references/audit-checks.md):
- robots_5xx: robots.txt returns 5xx (Google treats this as "disallow all") (high)
- robots_redirect: robots.txt redirects instead of returning the file directly (high)
- robots_served_as_html: robots.txt returns an HTML page, not plain text (high)
- sitewide_block: a User-agent group disallows the entire site with Disallow: / (high)
- syntax_error: malformed lines (missing colon, directive before User-agent, etc.) (high)
- file_too_large: robots.txt exceeds 500 KiB (Google ignores content past it) (high)
- blocked_resource: a CSS/JS/image needed to render the page is disallowed (medium)
- no_sitemap_directive: no Sitemap: directive present (medium)
- sitemap_unreachable: a declared Sitemap: URL is non-200 or not absolute (medium)
- no_star_group: no default User-agent: * group (medium)
- unsupported_directive: noindex/nofollow/crawl-delay etc. that Google ignores (medium/low)
- robots_404: robots.txt missing entirely (low — means "allow all", but worth a real file)
- duplicate_star_group: more than one User-agent: * group (low)
- has_bom: a UTF-8 BOM precedes the file (low)
- empty_disallow_confusion: Disallow: with no value (allows everything — often a mistake) (low)

Uses only the Python standard library.
"""

import argparse
import json
import sys

MAX_BYTES = 500 * 1024  # Google ignores robots.txt content past 500 KiB
RENDER_BLOCKING_TYPES = {"css", "js"}
# Directives that are not part of the robots.txt standard / are ignored by Google.
UNSUPPORTED_FIELDS = {"noindex", "nofollow", "crawl-delay", "host", "request-rate", "clean-param"}


def main():
    ap = argparse.ArgumentParser(description="Run robots.txt audit checks.")
    ap.add_argument("inventory", help="robots_inventory.json from collect_robots.py")
    ap.add_argument("--output", default="audit_report.json")
    args = ap.parse_args()

    with open(args.inventory, encoding="utf-8") as f:
        data = json.load(f)

    fetch = data.get("fetch", {})
    groups = data.get("groups", [])
    sitemaps = data.get("sitemaps", [])
    resources = data.get("resources", [])

    findings = {
        "robots_5xx": [], "robots_redirect": [], "robots_served_as_html": [],
        "sitewide_block": [], "syntax_error": [], "file_too_large": [],
        "blocked_resource": [], "no_sitemap_directive": [], "sitemap_unreachable": [],
        "no_star_group": [], "unsupported_directive": [],
        "robots_404": [], "duplicate_star_group": [], "has_bom": [],
        "empty_disallow_confusion": [],
    }

    status = fetch.get("status")
    if status and 500 <= status < 600:
        findings["robots_5xx"].append({"status": status, "url": data.get("robots_url")})
    if fetch.get("redirected"):
        findings["robots_redirect"].append(
            {"status": status, "final_url": fetch.get("final_url")})
    if fetch.get("looks_like_html"):
        findings["robots_served_as_html"].append(
            {"content_type": fetch.get("content_type")})
    if status == 404 or (status == 0):
        findings["robots_404"].append({"status": status, "url": data.get("robots_url")})

    if fetch.get("bytes", 0) > MAX_BYTES:
        findings["file_too_large"].append({"bytes": fetch.get("bytes"), "limit": MAX_BYTES})
    if fetch.get("has_bom"):
        findings["has_bom"].append({"note": "UTF-8 BOM precedes the file"})

    for agent in data.get("sitewide_block_agents", []):
        findings["sitewide_block"].append({"agent": agent, "rule": "Disallow: /"})

    for err in data.get("line_errors", []):
        findings["syntax_error"].append(err)

    # Resource blocking.
    for r in resources:
        if r.get("blocked") and r.get("type") in RENDER_BLOCKING_TYPES:
            findings["blocked_resource"].append(
                {"url": r["url"], "type": r["type"], "path": r["path"]})

    # Sitemap directives.
    if not sitemaps:
        findings["no_sitemap_directive"].append({"note": "no Sitemap: directive in robots.txt"})
    for sm in sitemaps:
        if not sm.get("absolute"):
            findings["sitemap_unreachable"].append(
                {"url": sm["url"], "issue": "not an absolute URL"})
        elif sm.get("status") not in (200, None):
            findings["sitemap_unreachable"].append(
                {"url": sm["url"], "status": sm.get("status")})

    # Group structure.
    star_groups = [g for g in groups if "*" in g.get("agents", [])]
    if groups and not star_groups:
        findings["no_star_group"].append(
            {"note": "no default User-agent: * group; unlisted crawlers get no rules"})
    if len(star_groups) > 1:
        findings["duplicate_star_group"].append({"count": len(star_groups)})

    # Unsupported directives + empty Disallow.
    for u in data.get("unknown_directives", []):
        if u.get("field") in UNSUPPORTED_FIELDS:
            findings["unsupported_directive"].append(u)
    for g in groups:
        if g.get("crawl_delay") is not None:
            findings["unsupported_directive"].append(
                {"field": "crawl-delay", "value": g["crawl_delay"], "agents": g["agents"]})
        for o in g.get("other", []):
            if o.get("field") in UNSUPPORTED_FIELDS:
                findings["unsupported_directive"].append({**o, "agents": g["agents"]})
        # An explicit empty Disallow allows everything; flag only when it coexists with
        # real Disallow rules in the same group (a likely mistake / no-op leftover).
        if "" in g.get("disallow", []) and any(d for d in g.get("disallow", [])):
            findings["empty_disallow_confusion"].append({"agents": g["agents"]})

    severity = {
        "robots_5xx": "high", "robots_redirect": "high", "robots_served_as_html": "high",
        "sitewide_block": "high", "syntax_error": "high", "file_too_large": "high",
        "blocked_resource": "medium", "no_sitemap_directive": "medium",
        "sitemap_unreachable": "medium", "no_star_group": "medium",
        "unsupported_directive": "medium",
        "robots_404": "low", "duplicate_star_group": "low", "has_bom": "low",
        "empty_disallow_confusion": "low",
    }
    summary = {
        "robots_status": status,
        "groups": len(groups),
        "sitemaps_declared": len(sitemaps),
        "resources_checked": len(resources),
        "issues_by_check": {k: len(v) for k, v in findings.items()},
        "issues_by_severity": {
            level: sum(len(findings[k]) for k, s in severity.items() if s == level)
            for level in ("high", "medium", "low")
        },
    }
    report = {"summary": summary, "severity": severity, "findings": findings}

    with open(args.output, "w", encoding="utf-8") as f:
        json.dump(report, f, indent=2, ensure_ascii=False)
    print(json.dumps(summary, indent=2))
    print(f"Full report written to {args.output}", file=sys.stderr)


if __name__ == "__main__":
    main()