Heading Structure Audit Claude Skill

#!/usr/bin/env python3
"""Crawl a website (or scan local HTML files) and build a heading inventory as JSON.

Usage:
    python3 extract_headings.py https://example.com [--max-pages 500] [--output heading_inventory.json]
    python3 extract_headings.py ./site-folder --local [--output heading_inventory.json]
    python3 extract_headings.py urls.txt --url-list [--output heading_inventory.json]

Output JSON schema:
{
  "site": "https://example.com",
  "pages": {
    "<url>": {
      "status": 200,
      "title": "<title text or null>",
      "headings": [ {"level": 1, "text": "..."}, {"level": 2, "text": "..."}, ... ]
    }
  }
}

Headings are captured in document order with their level (1–6) and trimmed text.

Uses only the Python standard library.
"""

import argparse
import json
import os
import sys
import urllib.error
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from collections import deque
from html.parser import HTMLParser

USER_AGENT = "HeadingAuditBot/1.0 (+claude-skill)"
TIMEOUT = 15
HEADING_TAGS = {"h1", "h2", "h3", "h4", "h5", "h6"}


class HeadingParser(HTMLParser):
    """Extract the page title, headings in document order, and same-host links."""

    def __init__(self, base_url):
        super().__init__(convert_charrefs=True)
        self.base_url = base_url
        self.title = None
        self.headings = []
        self.links = []
        self._in_title = False
        self._title_parts = []
        self._heading_level = None
        self._heading_parts = []

    def handle_starttag(self, tag, attrs):
        a = dict(attrs)
        if tag == "title":
            self._in_title = True
            self._title_parts = []
        elif tag in HEADING_TAGS and self._heading_level is None:
            self._heading_level = int(tag[1])
            self._heading_parts = []
        elif tag == "a" and a.get("href"):
            href = a["href"].strip()
            if not href.startswith(("mailto:", "tel:", "javascript:", "#")):
                target = urllib.parse.urljoin(self.base_url, href)
                self.links.append(urllib.parse.urldefrag(target)[0])

    def handle_endtag(self, tag):
        if tag == "title" and self._in_title:
            self._in_title = False
            self.title = " ".join("".join(self._title_parts).split()) or None
        elif tag in HEADING_TAGS and self._heading_level == int(tag[1]):
            text = " ".join("".join(self._heading_parts).split())
            self.headings.append({"level": self._heading_level, "text": text})
            self._heading_level = None

    def handle_data(self, data):
        if self._in_title:
            self._title_parts.append(data)
        if self._heading_level is not None:
            self._heading_parts.append(data)


def fetch(url):
    """Return (status, final_url, html_or_none)."""
    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
    try:
        with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
            ctype = resp.headers.get("Content-Type", "")
            body = None
            if "html" in ctype or "xml" in ctype:
                body = resp.read(2_000_000).decode("utf-8", errors="replace")
            return resp.status, resp.geturl(), body
    except urllib.error.HTTPError as e:
        return e.code, url, None
    except Exception:
        return 0, url, None


def norm_host(url):
    host = urllib.parse.urlparse(url).netloc.lower()
    return host[4:] if host.startswith("www.") else host


def pages_from_sitemap(site):
    status, _final, body = fetch(urllib.parse.urljoin(site, "/sitemap.xml"))
    if status != 200 or not body:
        return []
    urls = []
    try:
        root = ET.fromstring(body)
        for loc in root.iter("{http://www.sitemaps.org/schemas/sitemap/0.9}loc"):
            if loc.text:
                urls.append(loc.text.strip())
    except ET.ParseError:
        return []
    return urls


def parse_page(url, html):
    parser = HeadingParser(url)
    try:
        parser.feed(html)
    except Exception:
        pass
    return {"title": parser.title, "headings": parser.headings}, parser.links


def extract_from_cache(cache_path):
    """Pure: turn a shared page cache (from fetch_pages.py) into a heading inventory.

    No network — parses the cached HTML for exactly the fields this audit needs.
    """
    with open(cache_path, encoding="utf-8") as f:
        cache = json.load(f)
    pages = {}
    for url, page in cache.get("pages", {}).items():
        html = page.get("html")
        status = page.get("status", 0)
        if not html:
            pages[url] = {"status": status, "title": None, "headings": []}
            continue
        info, _links = parse_page(page.get("final_url") or url, html)
        info["status"] = status
        pages[url] = info
    return cache.get("site", cache_path), pages


def crawl(site, max_pages):
    start = site.rstrip("/")
    host = norm_host(start)
    seeds = pages_from_sitemap(start) or [start]
    queue = deque(seeds[:max_pages])
    seen = set(queue)
    pages = {}
    while queue and len(pages) < max_pages:
        url = queue.popleft()
        status, final_url, html = fetch(url)
        print(f"[{status}] {url}", file=sys.stderr)
        if html is None:
            pages[url] = {"status": status, "title": None, "headings": []}
            continue
        info, links = parse_page(final_url, html)
        info["status"] = status
        pages[url] = info
        for link in links:
            if norm_host(link) == host and link not in seen:
                seen.add(link)
                queue.append(link)
    return pages


def scan_local(folder):
    pages = {}
    for root, _dirs, files in os.walk(folder):
        for name in files:
            if name.lower().endswith((".html", ".htm")):
                path = os.path.join(root, name)
                with open(path, encoding="utf-8", errors="replace") as f:
                    html = f.read()
                info, _links = parse_page("file://" + os.path.abspath(path), html)
                info["status"] = 200
                pages[os.path.relpath(path, folder)] = info
    return pages


def scan_url_list(path, max_pages):
    pages = {}
    with open(path, encoding="utf-8") as f:
        urls = [line.strip() for line in f if line.strip() and not line.startswith("#")]
    for url in urls[:max_pages]:
        status, final_url, html = fetch(url)
        print(f"[{status}] {url}", file=sys.stderr)
        if html is None:
            pages[url] = {"status": status, "title": None, "headings": []}
            continue
        info, _links = parse_page(final_url, html)
        info["status"] = status
        pages[url] = info
    return pages


def main():
    ap = argparse.ArgumentParser(description="Build a heading inventory for a site.")
    ap.add_argument("source", nargs="?",
                    help="Site root URL, local folder (--local), or URL list file (--url-list)")
    ap.add_argument("--from-cache",
                    help="Extract from a shared page cache produced by fetch_pages.py (no network)")
    ap.add_argument("--max-pages", type=int, default=500)
    ap.add_argument("--output", default="heading_inventory.json")
    ap.add_argument("--local", action="store_true", help="Treat source as a local folder of HTML files")
    ap.add_argument("--url-list", action="store_true", help="Treat source as a text file of URLs")
    args = ap.parse_args()

    if args.from_cache:
        site, pages = extract_from_cache(args.from_cache)
    elif not args.source:
        ap.error("provide a source, or --from-cache page_cache.json")
    elif args.local:
        site, pages = args.source, scan_local(args.source)
    elif args.url_list:
        site, pages = args.source, scan_url_list(args.source, args.max_pages)
    else:
        site, pages = args.source, crawl(args.source, args.max_pages)

    with open(args.output, "w", encoding="utf-8") as f:
        json.dump({"site": site, "pages": pages}, f, indent=2, ensure_ascii=False)
    total = sum(len(p.get("headings", [])) for p in pages.values())
    print(f"Wrote {len(pages)} pages ({total} headings) to {args.output}", file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Audit a heading inventory JSON (from extract_headings.py) and emit audit findings.

Usage:
    python3 audit_headings.py heading_inventory.json [--output audit_report.json]
        [--long-heading 70]

Checks implemented (see references/audit-checks.md):
- missing_h1: page has no <h1> (high)
- no_headings: page has no headings at all (high)
- multiple_h1: page has more than one <h1> (medium)
- skipped_level: heading depth jumps down by more than one (e.g. h2 -> h4) (medium)
- empty_heading: a heading element with no text (medium)
- first_not_h1: the first heading on the page is not an <h1> (low)
- long_heading: heading longer than --long-heading characters (low)
- duplicate_heading: identical heading text repeated on one page (low)
- title_h1_mismatch: <h1> and <title> share no significant words (informational)

Uses only the Python standard library.
"""

import argparse
import json
import re
import sys
from collections import Counter

STOPWORDS = {
    "a", "an", "and", "are", "as", "at", "be", "by", "for", "from", "in",
    "is", "it", "of", "on", "or", "the", "to", "with", "your", "you", "our",
}


def words(text):
    return [w for w in re.findall(r"[a-z0-9']+", (text or "").lower()) if w not in STOPWORDS]


def main():
    ap = argparse.ArgumentParser(description="Run heading-structure audit checks.")
    ap.add_argument("inventory", help="heading_inventory.json from extract_headings.py")
    ap.add_argument("--output", default="audit_report.json")
    ap.add_argument("--long-heading", type=int, default=70,
                    help="flag headings longer than this many characters (default 70)")
    args = ap.parse_args()

    with open(args.inventory, encoding="utf-8") as f:
        data = json.load(f)
    pages = data.get("pages", {})

    findings = {
        "missing_h1": [], "no_headings": [], "multiple_h1": [],
        "skipped_level": [], "empty_heading": [], "first_not_h1": [],
        "long_heading": [], "duplicate_heading": [], "title_h1_mismatch": [],
    }

    pages_analyzed = 0
    for url, page in pages.items():
        if page.get("status", 200) != 200:
            continue
        pages_analyzed += 1
        headings = page.get("headings", [])
        levels = [h["level"] for h in headings]
        h1s = [h for h in headings if h["level"] == 1]

        if not headings:
            findings["no_headings"].append({"page": url})
            continue

        if not h1s:
            findings["missing_h1"].append({"page": url})
        if len(h1s) > 1:
            findings["multiple_h1"].append(
                {"page": url, "count": len(h1s), "values": [h["text"] for h in h1s]})

        if headings[0]["level"] != 1:
            findings["first_not_h1"].append(
                {"page": url, "first_level": headings[0]["level"], "first_text": headings[0]["text"]})

        # Skipped levels: a descent of more than one between consecutive headings.
        prev = None
        for h in headings:
            if prev is not None and h["level"] > prev + 1:
                findings["skipped_level"].append(
                    {"page": url, "from": f"h{prev}", "to": f"h{h['level']}", "text": h["text"]})
            prev = h["level"]

        # Empty headings
        for h in headings:
            if not h["text"].strip():
                findings["empty_heading"].append({"page": url, "level": f"h{h['level']}"})

        # Long headings
        for h in headings:
            if len(h["text"]) > args.long_heading:
                findings["long_heading"].append(
                    {"page": url, "level": f"h{h['level']}", "text": h["text"], "length": len(h["text"])})

        # Duplicate heading text on the same page
        counts = Counter(h["text"].strip().lower() for h in headings if h["text"].strip())
        for text, c in counts.items():
            if c > 1:
                findings["duplicate_heading"].append({"page": url, "text": text, "count": c})

        # Title / H1 topical mismatch (informational)
        title = page.get("title")
        if h1s and title:
            h1_words, t_words = set(words(h1s[0]["text"])), set(words(title))
            if h1_words and t_words and not (h1_words & t_words):
                findings["title_h1_mismatch"].append(
                    {"page": url, "h1": h1s[0]["text"], "title": title})

    severity = {
        "missing_h1": "high", "no_headings": "high",
        "multiple_h1": "medium", "skipped_level": "medium", "empty_heading": "medium",
        "first_not_h1": "low", "long_heading": "low", "duplicate_heading": "low",
        "title_h1_mismatch": "info",
    }
    summary = {
        "pages_analyzed": pages_analyzed,
        "headings_found": sum(len(p.get("headings", [])) for p in pages.values()),
        "issues_by_check": {k: len(v) for k, v in findings.items()},
        "issues_by_severity": {
            level: sum(len(findings[k]) for k, s in severity.items() if s == level)
            for level in ("high", "medium", "low", "info")
        },
    }
    report = {"summary": summary, "severity": severity, "findings": findings}

    with open(args.output, "w", encoding="utf-8") as f:
        json.dump(report, f, indent=2, ensure_ascii=False)
    print(json.dumps(summary, indent=2))
    print(f"Full report written to {args.output}", file=sys.stderr)


if __name__ == "__main__":
    main()