Site Architecture Audit Claude Skill

#!/usr/bin/env python3
"""Crawl a website and gather architecture signals into a JSON inventory.

Usage:
    python3 crawl_architecture.py https://example.com [--max-pages 500] [--max-depth 10] [--output architecture_inventory.json]

Gathers:
- Host/protocol variant probe (http/https x www/non-www): status + redirect target of each
- robots.txt: status, parsed user-agent groups (Disallow/Allow), Sitemap: directives, raw text
- XML sitemaps: every <loc>, following sitemap index files; total URL count
- Per crawled page: requested URL, final URL, status, redirect chain, click depth,
  scheme/host/path/query, path-segment count, canonical, meta robots, title,
  and the raw resolved internal link targets (for URL-form consistency checks)

Output JSON schema:
{
  "site": "https://example.com",
  "canonical_host": "https://example.com",
  "host_variants": {"<variant>": {"status": 301, "final_url": "..."}},
  "robots_txt": {"found": true, "status": 200, "sitemaps": [...],
                 "rules": [{"agent": "*", "disallow": [...], "allow": [...]}], "raw": "..."},
  "sitemap": {"found": true, "sources": [...], "url_count": 120, "urls": [...], "errors": [...]},
  "pages_crawled": 100,
  "pages": {
    "<requested url>": {
      "status": 200, "final_url": "...", "redirected": false, "redirect_chain": [],
      "depth": 0, "scheme": "https", "host": "example.com", "path": "/x", "query": "",
      "segments": 1, "canonical": "...|null", "meta_robots": "...|null", "title": "...|null",
      "links": ["<raw resolved internal target>", ...]
    }
  }
}

Uses only the Python standard library.
"""

import argparse
import json
import sys
import urllib.error
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from collections import deque
from html.parser import HTMLParser

USER_AGENT = "SiteArchitectureAuditBot/1.0 (+claude-skill)"
TIMEOUT = 15
MAX_SITEMAPS = 50


class PageParser(HTMLParser):
    """Extract title, canonical, meta robots, and same-document link hrefs."""

    def __init__(self, base_url):
        super().__init__(convert_charrefs=True)
        self.base_url = base_url
        self.title = None
        self.canonical = None
        self.meta_robots = None
        self.links = []
        self._in_title = False
        self._title_parts = []

    def handle_starttag(self, tag, attrs):
        a = dict(attrs)
        if tag == "title":
            self._in_title = True
            self._title_parts = []
        elif tag == "link" and (a.get("rel") or "").lower() == "canonical" and a.get("href"):
            self.canonical = urllib.parse.urljoin(self.base_url, a["href"].strip())
        elif tag == "meta" and (a.get("name") or "").lower() == "robots":
            self.meta_robots = a.get("content")
        elif tag == "a" and a.get("href"):
            href = a["href"].strip()
            if href.startswith(("mailto:", "tel:", "javascript:", "#")):
                return
            target = urllib.parse.urljoin(self.base_url, href)
            self.links.append(urllib.parse.urldefrag(target)[0])

    def handle_endtag(self, tag):
        if tag == "title" and self._in_title:
            self._in_title = False
            self.title = " ".join("".join(self._title_parts).split()) or None

    def handle_data(self, data):
        if self._in_title:
            self._title_parts.append(data)


def norm_host(host):
    host = host.lower()
    return host[4:] if host.startswith("www.") else host


def fetch(url, follow=True):
    """Return (status, final_url, html_or_none, redirect_chain).

    redirect_chain is a list of {"from": url, "status": code, "to": location}.
    """
    chain = []

    class ChainRedirect(urllib.request.HTTPRedirectHandler):
        def redirect_request(self, req, fp, code, msg, headers, newurl):
            chain.append({"from": req.full_url, "status": code, "to": newurl})
            if not follow:
                return None
            return super().redirect_request(req, fp, code, msg, headers, newurl)

    opener = urllib.request.build_opener(ChainRedirect)
    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
    try:
        with opener.open(req, timeout=TIMEOUT) as resp:
            ctype = resp.headers.get("Content-Type", "")
            body = None
            if "html" in ctype or "xml" in ctype:
                body = resp.read(2_000_000).decode("utf-8", errors="replace")
            return resp.status, resp.geturl(), body, chain
    except urllib.error.HTTPError as e:
        if not follow and e.code in range(300, 400):
            loc = e.headers.get("Location")
            final = urllib.parse.urljoin(url, loc) if loc else url
            return e.code, final, None, chain
        return e.code, url, None, chain
    except Exception:
        return 0, url, None, chain  # 0 = network/connection error


def probe_host_variants(start):
    """Probe http/https x www/non-www; return dict and the chosen canonical host."""
    parsed = urllib.parse.urlparse(start)
    bare = norm_host(parsed.netloc)
    variants = {}
    for scheme in ("https", "http"):
        for host in (bare, "www." + bare):
            root = f"{scheme}://{host}/"
            status, final_url, _body, _chain = fetch(root, follow=False)
            variants[f"{scheme}://{host}"] = {"status": status, "final_url": final_url}

    # Canonical host = the 200 variant that the others redirect toward, preferring https.
    live_200 = [v for v, info in variants.items()
                if info["status"] == 200]
    canonical_host = None
    for pref in (f"https://{bare}", f"https://www.{bare}",
                 f"http://{bare}", f"http://www.{bare}"):
        if pref in variants and variants[pref]["status"] == 200:
            canonical_host = pref
            break
    if canonical_host is None and live_200:
        canonical_host = live_200[0]
    if canonical_host is None:
        canonical_host = f"{parsed.scheme}://{parsed.netloc}"
    return variants, canonical_host


def parse_robots(text):
    """Minimal robots.txt parse: groups of agents with allow/disallow, plus sitemaps.

    A run of consecutive User-agent lines shares one group. The first allow/disallow
    after a User-agent line closes the run, so the next User-agent starts a new group.
    """
    groups = []
    sitemaps = []
    current = None
    last_was_directive = False
    for raw_line in text.splitlines():
        line = raw_line.split("#", 1)[0].strip()
        if not line or ":" not in line:
            continue
        field, _, value = line.partition(":")
        field = field.strip().lower()
        value = value.strip()
        if field == "sitemap":
            if value:
                sitemaps.append(value)
            continue
        if field == "user-agent":
            if current is None or last_was_directive:
                current = {"agents": [], "disallow": [], "allow": []}
                groups.append(current)
            current["agents"].append(value)
            last_was_directive = False
        elif field in ("disallow", "allow") and current is not None:
            current[field].append(value)
            last_was_directive = True
    flat = []
    for g in groups:
        for agent in g["agents"]:
            flat.append({"agent": agent, "disallow": list(g["disallow"]),
                         "allow": list(g["allow"])})
    return flat, sitemaps


def fetch_robots(canonical_host):
    url = canonical_host.rstrip("/") + "/robots.txt"
    status, _final, body, _chain = fetch(url, follow=True)
    if status != 200 or body is None:
        # Try a plain fetch in case content-type was not html/xml
        try:
            req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
            with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
                status = resp.status
                body = resp.read(1_000_000).decode("utf-8", errors="replace")
        except urllib.error.HTTPError as e:
            return {"found": False, "status": e.code, "sitemaps": [], "rules": [], "raw": ""}
        except Exception:
            return {"found": False, "status": status, "sitemaps": [], "rules": [], "raw": ""}
    rules, sitemaps = parse_robots(body or "")
    return {"found": status == 200, "status": status, "sitemaps": sitemaps,
            "rules": rules, "raw": (body or "")[:20000]}


def collect_sitemap_urls(seed_sitemaps, canonical_host):
    """Follow sitemap index files and gather all page <loc> URLs."""
    SM_NS = "{http://www.sitemaps.org/schemas/sitemap/0.9}"
    seeds = list(seed_sitemaps)
    if not seeds:
        seeds = [canonical_host.rstrip("/") + "/sitemap.xml"]
    seen = set()
    queue = deque(seeds)
    page_urls = []
    sources = []
    errors = []
    found_any = False
    while queue and len(seen) < MAX_SITEMAPS:
        sm_url = queue.popleft()
        if sm_url in seen:
            continue
        seen.add(sm_url)
        status, _final, body, _chain = fetch(sm_url, follow=True)
        if status != 200 or not body:
            # retry plain
            try:
                req = urllib.request.Request(sm_url, headers={"User-Agent": USER_AGENT})
                with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
                    status = resp.status
                    body = resp.read(5_000_000).decode("utf-8", errors="replace")
            except Exception:
                errors.append({"sitemap": sm_url, "error": f"status {status}"})
                continue
        try:
            root = ET.fromstring(body)
        except ET.ParseError as e:
            errors.append({"sitemap": sm_url, "error": f"parse error: {e}"})
            continue
        found_any = True
        sources.append(sm_url)
        tag = root.tag.lower()
        if tag.endswith("sitemapindex"):
            for loc in root.iter(SM_NS + "loc"):
                if loc.text:
                    queue.append(loc.text.strip())
        else:  # urlset
            for loc in root.iter(SM_NS + "loc"):
                if loc.text:
                    page_urls.append(loc.text.strip())
    # Deduplicate while preserving order
    seen_u = set()
    deduped = []
    for u in page_urls:
        if u not in seen_u:
            seen_u.add(u)
            deduped.append(u)
    return {"found": found_any, "sources": sources, "url_count": len(deduped),
            "urls": deduped, "errors": errors}


def url_parts(url):
    p = urllib.parse.urlparse(url)
    path = p.path or "/"
    segments = [s for s in path.split("/") if s]
    return {"scheme": p.scheme, "host": p.netloc.lower(), "path": path,
            "query": p.query, "segments": len(segments)}


def main():
    ap = argparse.ArgumentParser(description="Crawl a site and gather architecture signals.")
    ap.add_argument("start_url")
    ap.add_argument("--max-pages", type=int, default=500)
    ap.add_argument("--max-depth", type=int, default=10)
    ap.add_argument("--output", default="architecture_inventory.json")
    args = ap.parse_args()

    start = args.start_url.rstrip("/")
    host = norm_host(urllib.parse.urlparse(start).netloc)

    print("Probing host/protocol variants...", file=sys.stderr)
    host_variants, canonical_host = probe_host_variants(start)
    print(f"Canonical host: {canonical_host}", file=sys.stderr)

    print("Fetching robots.txt...", file=sys.stderr)
    robots = fetch_robots(canonical_host)

    print("Collecting sitemap URLs...", file=sys.stderr)
    sitemap = collect_sitemap_urls(robots.get("sitemaps", []), canonical_host)
    print(f"Sitemap URLs found: {sitemap['url_count']}", file=sys.stderr)

    print("Crawling pages...", file=sys.stderr)
    pages = {}
    queue = deque([(start, 0)])
    seen = {start}
    while queue and len(pages) < args.max_pages:
        url, depth = queue.popleft()
        status, final_url, html, chain = fetch(url, follow=True)
        parts = url_parts(final_url if status < 400 else url)
        page = {
            "status": status,
            "final_url": final_url,
            "redirected": bool(chain),
            "redirect_chain": chain,
            "depth": depth,
            "scheme": parts["scheme"],
            "host": parts["host"],
            "path": parts["path"],
            "query": parts["query"],
            "segments": parts["segments"],
            "canonical": None,
            "meta_robots": None,
            "title": None,
            "links": [],
        }
        print(f"[{status}] depth={depth} {url}", file=sys.stderr)
        if html:
            parser = PageParser(final_url)
            try:
                parser.feed(html)
            except Exception:
                pass
            page["canonical"] = parser.canonical
            page["meta_robots"] = parser.meta_robots
            page["title"] = parser.title
            internal = []
            for target in parser.links:
                if not target.startswith(("http://", "https://")):
                    continue
                if norm_host(urllib.parse.urlparse(target).netloc) == host:
                    internal.append(target)
                    if target not in seen and depth + 1 <= args.max_depth:
                        seen.add(target)
                        queue.append((target, depth + 1))
            page["links"] = internal
        pages[url] = page

    inventory = {
        "site": start,
        "canonical_host": canonical_host,
        "host_variants": host_variants,
        "robots_txt": robots,
        "sitemap": sitemap,
        "pages_crawled": len(pages),
        "pages": pages,
    }
    with open(args.output, "w", encoding="utf-8") as f:
        json.dump(inventory, f, indent=2, ensure_ascii=False)
    print(f"Wrote {len(pages)} pages to {args.output}", file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Analyze an architecture inventory JSON (from crawl_architecture.py) and emit audit findings.

Usage:
    python3 audit_architecture.py architecture_inventory.json [--output audit_report.json]
        [--max-url-depth 4] [--max-click-depth 3] [--tracking-params utm_source,gclid,fbclid]

Checks implemented (see references/audit-checks.md):
- multiple_host_variants: more than one 200 host/protocol variant (high)
- robots_blocks_content: Disallow rules matching linked/indexable pages (high)
- directive_conflicts: disallow+noindex, noindex+canonical-elsewhere, canonical to bad target (high)
- sitemap_non_indexable: sitemap URLs that are non-200/redirect/noindex/non-canonical (high)
- sitemap_missing_pages: indexable pages absent from the sitemap (medium)
- deep_pages / depth_distribution: click depth issues (medium)
- url_quality: uppercase/underscore/space/over-long/over-deep/id-only URLs (medium)
- inconsistent_url_forms: mixed trailing slash, http/https, www in internal links (medium)
- parameter_urls: tracking/session/sort params in internal links (medium)
- missing_robots / missing_sitemap (medium)
- missing_self_canonical (low)
- sitemap_not_in_robots (low)
- thin_directories (low)
- sitemap_over_limit (low)

Uses only the Python standard library.
"""

import argparse
import json
import re
import sys
import urllib.parse
from collections import Counter, defaultdict

SITEMAP_URL_LIMIT = 50000
TRACKING_DEFAULT = ["utm_source", "utm_medium", "utm_campaign", "utm_term",
                    "utm_content", "gclid", "fbclid", "mc_cid", "mc_eid"]
SESSION_PARAMS = ["sid", "sessionid", "phpsessid", "jsessionid", "aspsessionid"]
ID_SLUG = re.compile(r"^(\d+|[0-9a-f]{12,}|[0-9a-f-]{20,})$", re.I)


def norm_host(host):
    host = host.lower()
    return host[4:] if host.startswith("www.") else host


def normalize_url(url):
    """Canonical comparison key: lowercase host, strip default ports and fragment, keep path/query."""
    p = urllib.parse.urlparse(url)
    netloc = p.netloc.lower()
    netloc = re.sub(r":(80|443)$", "", netloc)
    path = p.path or "/"
    return urllib.parse.urlunparse((p.scheme, netloc, path, "", p.query, ""))


def strip_slash(url):
    p = urllib.parse.urlparse(url)
    path = p.path.rstrip("/") or "/"
    return urllib.parse.urlunparse((p.scheme, p.netloc.lower(), path, "", p.query, ""))


def robots_match(path, rules, agent="*"):
    """Return True if path is Disallowed for the given agent (longest-match, Allow wins on ties)."""
    applicable = [r for r in rules if r["agent"] == agent]
    if not applicable:
        applicable = [r for r in rules if r["agent"] == "*"]
    if not applicable:
        return False
    disallow, allow = [], []
    for r in applicable:
        disallow.extend(r["disallow"])
        allow.extend(r["allow"])

    def pattern_len(pattern):
        if pattern == "":
            return None  # empty Disallow means allow everything
        regex = re.escape(pattern).replace(r"\*", ".*")
        if regex.endswith(r"\$"):
            regex = regex[:-2] + "$"
        m = re.match(regex, path)
        return len(pattern) if m else -1

    best_dis = max([pattern_len(p) for p in disallow if pattern_len(p) is not None] or [-1])
    best_allow = max([pattern_len(p) for p in allow if pattern_len(p) is not None] or [-1])
    if best_dis < 0:
        return False
    return best_dis > best_allow


def is_noindex(meta_robots):
    return "noindex" in (meta_robots or "").lower()


def main():
    ap = argparse.ArgumentParser(description="Run architecture audit checks on an inventory.")
    ap.add_argument("inventory")
    ap.add_argument("--output", default="audit_report.json")
    ap.add_argument("--max-url-depth", type=int, default=4)
    ap.add_argument("--max-click-depth", type=int, default=3)
    ap.add_argument("--tracking-params", default="")
    args = ap.parse_args()

    with open(args.inventory, encoding="utf-8") as f:
        inv = json.load(f)

    pages = inv["pages"]
    host_variants = inv.get("host_variants", {})
    robots = inv.get("robots_txt", {})
    sitemap = inv.get("sitemap", {})
    rules = robots.get("rules", [])
    tracking = set(TRACKING_DEFAULT + [t.strip() for t in args.tracking_params.split(",") if t.strip()])

    findings = {
        "multiple_host_variants": [], "robots_blocks_content": [], "directive_conflicts": [],
        "sitemap_non_indexable": [], "sitemap_missing_pages": [],
        "deep_pages": [], "url_quality": [], "inconsistent_url_forms": [],
        "parameter_urls": [], "missing_robots": [], "missing_sitemap": [],
        "missing_self_canonical": [], "sitemap_not_in_robots": [],
        "thin_directories": [], "sitemap_over_limit": [],
    }

    # --- 1. Host/protocol variants -------------------------------------------------
    live_200 = [v for v, info in host_variants.items() if info.get("status") == 200]
    if len(live_200) > 1:
        findings["multiple_host_variants"].append({
            "variants_200": live_200,
            "canonical_host": inv.get("canonical_host"),
            "detail": "More than one host/protocol variant returns 200; redirect all but the canonical host.",
        })

    # --- robots.txt / sitemap presence --------------------------------------------
    if not robots.get("found"):
        findings["missing_robots"].append(
            {"status": robots.get("status"),
             "detail": "robots.txt missing or non-200; serve a valid robots.txt and declare the sitemap."})
    if not sitemap.get("found"):
        findings["missing_sitemap"].append(
            {"detail": "No XML sitemap found at /sitemap.xml or via robots.txt."})
    elif not robots.get("sitemaps"):
        findings["sitemap_not_in_robots"].append(
            {"detail": "A sitemap exists but is not referenced with a Sitemap: line in robots.txt."})

    if sitemap.get("url_count", 0) > SITEMAP_URL_LIMIT and len(sitemap.get("sources", [])) <= 1:
        findings["sitemap_over_limit"].append(
            {"url_count": sitemap["url_count"],
             "detail": f"Single sitemap has {sitemap['url_count']} URLs (limit {SITEMAP_URL_LIMIT}); split with a sitemap index."})

    # --- Build lookups for indexability -------------------------------------------
    # A crawled page is "indexable" if 200, not noindex, and self-canonical (or no canonical).
    indexable_keys = {}     # normalized self-URL -> requested url
    page_by_norm = {}       # normalized requested url -> page
    canonical_target = {}   # normalized requested url -> normalized canonical
    status_by_norm = {}
    noindex_by_norm = {}
    redirected_by_norm = {}

    for url, page in pages.items():
        nkey = normalize_url(url)
        page_by_norm[nkey] = page
        status_by_norm[nkey] = page["status"]
        noindex_by_norm[nkey] = is_noindex(page.get("meta_robots"))
        redirected_by_norm[nkey] = page.get("redirected", False)
        canon = page.get("canonical")
        if canon:
            canonical_target[nkey] = normalize_url(canon)
        self_canon = (not canon) or strip_slash(canon) == strip_slash(url)
        if page["status"] == 200 and not noindex_by_norm[nkey] and self_canon:
            indexable_keys[nkey] = url

    # --- 2. robots.txt blocks indexable/linked content ----------------------------
    for nkey, url in indexable_keys.items():
        path = urllib.parse.urlparse(url).path or "/"
        if robots_match(path, rules):
            findings["robots_blocks_content"].append(
                {"page": url, "detail": "Disallowed in robots.txt but linked and otherwise indexable."})

    # --- 3. Directive conflicts ----------------------------------------------------
    for url, page in pages.items():
        nkey = normalize_url(url)
        path = urllib.parse.urlparse(url).path or "/"
        noidx = noindex_by_norm[nkey]
        canon = page.get("canonical")

        if noidx and robots_match(path, rules):
            findings["directive_conflicts"].append(
                {"page": url, "conflict": "disallow + noindex",
                 "detail": "Page is Disallowed in robots.txt, so the noindex can never be seen."})

        if noidx and canon and strip_slash(canon) != strip_slash(url):
            findings["directive_conflicts"].append(
                {"page": url, "conflict": "noindex + canonical elsewhere",
                 "detail": f"noindex contradicts canonical -> {canon}."})

        if canon:
            ckey = normalize_url(canon)
            if ckey in status_by_norm:
                if status_by_norm[ckey] >= 400 or status_by_norm[ckey] == 0:
                    findings["directive_conflicts"].append(
                        {"page": url, "conflict": "canonical to broken URL",
                         "detail": f"canonical -> {canon} returns {status_by_norm[ckey]}."})
                elif redirected_by_norm.get(ckey):
                    findings["directive_conflicts"].append(
                        {"page": url, "conflict": "canonical to redirect",
                         "detail": f"canonical -> {canon} is itself a redirect."})
                elif noindex_by_norm.get(ckey) and ckey != nkey:
                    findings["directive_conflicts"].append(
                        {"page": url, "conflict": "canonical to noindex",
                         "detail": f"canonical -> {canon} is noindex."})

    # --- 4 & 5. Sitemap parity -----------------------------------------------------
    sitemap_urls = sitemap.get("urls", [])
    sitemap_norm = {normalize_url(u): u for u in sitemap_urls}
    for nkey, original in sitemap_norm.items():
        page = page_by_norm.get(nkey)
        if page is None:
            continue  # not crawled; cannot assess
        problem = None
        if page["status"] >= 400 or page["status"] == 0:
            problem = f"returns {page['status']}"
        elif page.get("redirected"):
            problem = f"redirects to {page.get('final_url')}"
        elif is_noindex(page.get("meta_robots")):
            problem = "is noindex"
        else:
            canon = page.get("canonical")
            if canon and strip_slash(canon) != strip_slash(original):
                problem = f"canonical points to {canon}"
        if problem:
            findings["sitemap_non_indexable"].append({"sitemap_url": original, "problem": problem})

    sitemap_slash_set = {strip_slash(u) for u in sitemap_urls}
    if sitemap.get("found"):
        for nkey, url in indexable_keys.items():
            if strip_slash(url) not in sitemap_slash_set:
                findings["sitemap_missing_pages"].append({"page": url})

    # --- 6. Click depth ------------------------------------------------------------
    depth_hist = Counter()
    beyond = 0
    total = 0
    for url, page in pages.items():
        if page["status"] != 200:
            continue
        total += 1
        d = page["depth"]
        depth_hist[d] += 1
        if d > args.max_click_depth:
            beyond += 1
            findings["deep_pages"].append({"page": url, "depth": d})
    depth_distribution = {
        "histogram": dict(sorted(depth_hist.items())),
        "beyond_threshold": beyond,
        "share_beyond_threshold": round(beyond / total, 3) if total else 0,
        "threshold": args.max_click_depth,
    }

    # --- 7. URL quality ------------------------------------------------------------
    for url, page in pages.items():
        if page["status"] >= 400 or page["status"] == 0:
            continue
        p = urllib.parse.urlparse(url)
        path = p.path or "/"
        problems = []
        if re.search(r"[A-Z]", path):
            problems.append("uppercase")
        if "_" in path:
            problems.append("underscore")
        if "%20" in url or " " in path or re.search(r"%[0-9A-Fa-f]{2}", path):
            problems.append("space/encoded char")
        if len(url) > 115:
            problems.append(f"long ({len(url)} chars)")
        if page["segments"] > args.max_url_depth:
            problems.append(f"deep path ({page['segments']} segments)")
        last_seg = [s for s in path.split("/") if s]
        if last_seg and ID_SLUG.match(last_seg[-1]):
            problems.append("ID-only slug")
        if problems:
            findings["url_quality"].append({"page": url, "problems": problems})

    # --- 8. Inconsistent URL forms (across internal links) ------------------------
    link_forms = []
    for page in pages.values():
        link_forms.extend(page.get("links", []))
    slash_paths = defaultdict(set)  # path without slash -> set of "slash"/"noslash"
    schemes = set()
    www_forms = set()
    for target in link_forms:
        tp = urllib.parse.urlparse(target)
        schemes.add(tp.scheme)
        www_forms.add(tp.netloc.lower().startswith("www."))
        base = tp.path.rstrip("/")
        if tp.path.endswith("/") and tp.path != "/":
            slash_paths[base].add("slash")
        elif tp.path and tp.path != "/":
            slash_paths[base].add("noslash")
    mixed_slash = [b for b, forms in slash_paths.items() if len(forms) > 1]
    if mixed_slash:
        findings["inconsistent_url_forms"].append(
            {"type": "mixed trailing slash", "examples": mixed_slash[:10],
             "count": len(mixed_slash)})
    if "http" in schemes and "https" in schemes:
        findings["inconsistent_url_forms"].append(
            {"type": "mixed http/https in internal links"})
    if len(www_forms) > 1:
        findings["inconsistent_url_forms"].append(
            {"type": "mixed www / non-www in internal links"})

    # --- 9. Parameter URLs ---------------------------------------------------------
    seen_param = set()
    for target in link_forms:
        tp = urllib.parse.urlparse(target)
        if not tp.query:
            continue
        keys = {k.lower() for k, _ in urllib.parse.parse_qsl(tp.query)}
        cat = None
        if keys & tracking:
            cat = "tracking parameter in internal link"
        elif keys & set(SESSION_PARAMS):
            cat = "session id in URL"
        else:
            cat = "functional parameter (sort/filter/pagination)"
        key = (cat, urllib.parse.urlunparse(tp._replace(query="")), tuple(sorted(keys)))
        if key in seen_param:
            continue
        seen_param.add(key)
        findings["parameter_urls"].append(
            {"url": target, "category": cat, "params": sorted(keys)})

    # --- 12. Missing self-canonical ------------------------------------------------
    for url, page in pages.items():
        if page["status"] == 200 and not is_noindex(page.get("meta_robots")) and not page.get("canonical"):
            findings["missing_self_canonical"].append({"page": url})

    # --- 14. Directory taxonomy ----------------------------------------------------
    section_counts = Counter()
    for url, page in pages.items():
        if page["status"] != 200:
            continue
        segs = [s for s in urllib.parse.urlparse(url).path.split("/") if s]
        section = "/" + segs[0] if segs else "/"
        section_counts[section] += 1
    thin = [{"section": s, "pages": c} for s, c in section_counts.items()
            if c == 1 and s != "/"]
    if thin:
        findings["thin_directories"] = thin

    severity = {
        "multiple_host_variants": "high", "robots_blocks_content": "high",
        "directive_conflicts": "high", "sitemap_non_indexable": "high",
        "sitemap_missing_pages": "medium", "deep_pages": "medium",
        "url_quality": "medium", "inconsistent_url_forms": "medium",
        "parameter_urls": "medium", "missing_robots": "medium", "missing_sitemap": "medium",
        "missing_self_canonical": "low", "sitemap_not_in_robots": "low",
        "thin_directories": "low", "sitemap_over_limit": "low",
    }

    summary = {
        "pages_crawled": inv.get("pages_crawled"),
        "canonical_host": inv.get("canonical_host"),
        "robots_found": robots.get("found"),
        "sitemap_found": sitemap.get("found"),
        "sitemap_url_count": sitemap.get("url_count"),
        "depth_distribution": depth_distribution,
        "section_counts": dict(section_counts.most_common()),
        "issues_by_check": {k: len(v) for k, v in findings.items()},
        "issues_by_severity": {
            level: sum(len(findings[k]) for k, s in severity.items() if s == level)
            for level in ("high", "medium", "low")
        },
    }
    report = {"summary": summary, "severity": severity, "findings": findings,
              "host_variants": host_variants}

    with open(args.output, "w", encoding="utf-8") as f:
        json.dump(report, f, indent=2, ensure_ascii=False)
    print(json.dumps(summary, indent=2))
    print(f"Full report written to {args.output}", file=sys.stderr)


if __name__ == "__main__":
    main()