Meta Data Audit Claude Skill

#!/usr/bin/env python3
"""Build a title/meta description inventory as JSON.

Preferred: extract from a shared page cache produced by the shared fetch stage
(`fetch_pages.py`), so the site is crawled once and every audit reuses it:

    python3 extract_metadata.py --from-cache page_cache.json [--output metadata_inventory.json]

Standalone (fetches on its own — fine for a single-area run):
    python3 extract_metadata.py https://example.com [--max-pages 500] [--output metadata_inventory.json]
    python3 extract_metadata.py ./site-folder --local [--output metadata_inventory.json]
    python3 extract_metadata.py urls.txt --url-list [--output metadata_inventory.json]

Output JSON schema:
{
  "site": "https://example.com",
  "pages": {
    "<url>": {
      "status": 200,
      "titles": ["..."],            # every <title> found, in order
      "descriptions": ["..."],      # every <meta name="description"> found, in order
      "h1": "<first h1 text or null>",
      "canonical": "<url or null>",
      "meta_robots": "<content or null>",
      "og_title": "<content or null>",
      "og_description": "<content or null>"
    }
  }
}

Uses only the Python standard library.
"""

import argparse
import json
import os
import sys
import urllib.error
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from collections import deque
from html.parser import HTMLParser

USER_AGENT = "MetaDataAuditBot/1.0 (+claude-skill)"
TIMEOUT = 15


class MetadataParser(HTMLParser):
    """Extract titles, meta descriptions, first H1, canonical, robots, OG tags, and links."""

    def __init__(self, base_url):
        super().__init__(convert_charrefs=True)
        self.base_url = base_url
        self.titles = []
        self.descriptions = []
        self.h1 = None
        self.canonical = None
        self.meta_robots = None
        self.og_title = None
        self.og_description = None
        self.links = []
        self._in_title = False
        self._title_parts = []
        self._in_h1 = False
        self._h1_parts = []
        self._h1_done = False

    def handle_starttag(self, tag, attrs):
        a = dict(attrs)
        if tag == "title":
            self._in_title = True
            self._title_parts = []
        elif tag == "h1" and not self._h1_done:
            self._in_h1 = True
            self._h1_parts = []
        elif tag == "meta":
            name = (a.get("name") or "").lower()
            prop = (a.get("property") or "").lower()
            content = a.get("content")
            if name == "description":
                self.descriptions.append((content or "").strip())
            elif name == "robots":
                self.meta_robots = content
            elif prop == "og:title" and self.og_title is None:
                self.og_title = (content or "").strip() or None
            elif prop == "og:description" and self.og_description is None:
                self.og_description = (content or "").strip() or None
        elif tag == "link" and (a.get("rel") or "").lower() == "canonical" and a.get("href"):
            self.canonical = urllib.parse.urljoin(self.base_url, a["href"])
        elif tag == "a" and a.get("href"):
            href = a["href"].strip()
            if not href.startswith(("mailto:", "tel:", "javascript:", "#")):
                target = urllib.parse.urljoin(self.base_url, href)
                self.links.append(urllib.parse.urldefrag(target)[0])

    def handle_endtag(self, tag):
        if tag == "title" and self._in_title:
            self._in_title = False
            self.titles.append(" ".join("".join(self._title_parts).split()))
        elif tag == "h1" and self._in_h1:
            self._in_h1 = False
            self._h1_done = True
            self.h1 = " ".join("".join(self._h1_parts).split()) or None

    def handle_data(self, data):
        if self._in_title:
            self._title_parts.append(data)
        if self._in_h1:
            self._h1_parts.append(data)


def fetch(url):
    """Return (status, final_url, html_or_none)."""
    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
    try:
        with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
            ctype = resp.headers.get("Content-Type", "")
            body = None
            if "html" in ctype or "xml" in ctype:
                body = resp.read(2_000_000).decode("utf-8", errors="replace")
            return resp.status, resp.geturl(), body
    except urllib.error.HTTPError as e:
        return e.code, url, None
    except Exception:
        return 0, url, None  # 0 = network/connection error


def norm_host(url):
    host = urllib.parse.urlparse(url).netloc.lower()
    return host[4:] if host.startswith("www.") else host


def pages_from_sitemap(site):
    status, _final, body = fetch(urllib.parse.urljoin(site, "/sitemap.xml"))
    if status != 200 or not body:
        return []
    urls = []
    try:
        root = ET.fromstring(body)
        for loc in root.iter("{http://www.sitemaps.org/schemas/sitemap/0.9}loc"):
            if loc.text:
                urls.append(loc.text.strip())
    except ET.ParseError:
        return []
    return urls


def parse_page(url, html):
    parser = MetadataParser(url)
    try:
        parser.feed(html)
    except Exception:
        pass
    info = {
        "titles": parser.titles,
        "descriptions": parser.descriptions,
        "h1": parser.h1,
        "canonical": parser.canonical,
        "meta_robots": parser.meta_robots,
        "og_title": parser.og_title,
        "og_description": parser.og_description,
    }
    return info, parser.links


def empty_record(status):
    return {"status": status, "titles": [], "descriptions": [], "h1": None,
            "canonical": None, "meta_robots": None,
            "og_title": None, "og_description": None}


def extract_from_cache(cache_path):
    """Pure: turn a shared page cache (from fetch_pages.py) into a metadata inventory.

    Makes no network calls — parses the cached HTML for exactly the fields this
    audit needs. This is the half of the old collector that stays after fetching
    is factored out into the shared fetch stage.
    """
    with open(cache_path, encoding="utf-8") as f:
        cache = json.load(f)
    pages = {}
    for url, page in cache.get("pages", {}).items():
        html = page.get("html")
        status = page.get("status", 0)
        if not html:
            pages[url] = empty_record(status)
            continue
        info, _links = parse_page(page.get("final_url") or url, html)
        info["status"] = status
        pages[url] = info
    return cache.get("site", cache_path), pages


def crawl(site, max_pages):
    start = site.rstrip("/")
    host = norm_host(start)
    seeds = pages_from_sitemap(start) or [start]
    queue = deque(seeds[:max_pages])
    seen = set(queue)
    pages = {}
    while queue and len(pages) < max_pages:
        url = queue.popleft()
        status, final_url, html = fetch(url)
        print(f"[{status}] {url}", file=sys.stderr)
        if html is None:
            pages[url] = empty_record(status)
            continue
        info, links = parse_page(final_url, html)
        info["status"] = status
        pages[url] = info
        for link in links:
            if norm_host(link) == host and link not in seen:
                seen.add(link)
                queue.append(link)
    return pages


def scan_local(folder):
    pages = {}
    for root, _dirs, files in os.walk(folder):
        for name in files:
            if name.lower().endswith((".html", ".htm")):
                path = os.path.join(root, name)
                with open(path, encoding="utf-8", errors="replace") as f:
                    html = f.read()
                info, _links = parse_page("file://" + os.path.abspath(path), html)
                info["status"] = 200
                pages[os.path.relpath(path, folder)] = info
    return pages


def scan_url_list(path, max_pages):
    pages = {}
    with open(path, encoding="utf-8") as f:
        urls = [line.strip() for line in f if line.strip() and not line.startswith("#")]
    for url in urls[:max_pages]:
        status, final_url, html = fetch(url)
        print(f"[{status}] {url}", file=sys.stderr)
        if html is None:
            pages[url] = empty_record(status)
            continue
        info, _links = parse_page(final_url, html)
        info["status"] = status
        pages[url] = info
    return pages


def main():
    ap = argparse.ArgumentParser(description="Build a title/meta description inventory for a site.")
    ap.add_argument("source", nargs="?",
                    help="Site root URL, local folder (--local), or URL list file (--url-list)")
    ap.add_argument("--from-cache",
                    help="Extract from a shared page cache produced by fetch_pages.py (no network)")
    ap.add_argument("--max-pages", type=int, default=500)
    ap.add_argument("--output", default="metadata_inventory.json")
    ap.add_argument("--local", action="store_true", help="Treat source as a local folder of HTML files")
    ap.add_argument("--url-list", action="store_true", help="Treat source as a text file of URLs")
    args = ap.parse_args()

    if args.from_cache:
        site, pages = extract_from_cache(args.from_cache)
    elif not args.source:
        ap.error("provide a source, or --from-cache page_cache.json")
    elif args.local:
        site, pages = args.source, scan_local(args.source)
    elif args.url_list:
        site, pages = args.source, scan_url_list(args.source, args.max_pages)
    else:
        site, pages = args.source, crawl(args.source, args.max_pages)

    with open(args.output, "w", encoding="utf-8") as f:
        json.dump({"site": site, "pages": pages}, f, indent=2, ensure_ascii=False)
    titles = sum(len(p["titles"]) for p in pages.values())
    descs = sum(len(p["descriptions"]) for p in pages.values())
    print(f"Wrote {len(pages)} pages ({titles} titles, {descs} descriptions) to {args.output}",
          file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Audit a metadata inventory JSON (from extract_metadata.py) and emit audit findings.

Usage:
    python3 audit_metadata.py metadata_inventory.json [--output audit_report.json]
        [--brand-suffix "| Acme Co"] [--include-noindex]

Checks implemented (see references/audit-checks.md):
- missing_title: no or empty <title> (high)
- missing_description: no or empty meta description (high)
- duplicate_titles: identical title on 2+ indexable pages (high)
- duplicate_descriptions: identical description on 2+ indexable pages (high)
- multiple_titles / multiple_descriptions: more than one tag on a page (medium)
- title_too_long (> 60 chars) / title_too_short (< 30 chars) (medium)
- description_too_long (> 160 chars) / description_too_short (< 70 chars) (medium)
- generic_title: boilerplate or brand-only titles (medium)
- description_duplicates_title: description equals or contains only the title (low)
- keyword_stuffing: same significant word 3+ times in a title (low)
- title_h1_mismatch: title and H1 share no significant words (informational)

Uses only the Python standard library.
"""

import argparse
import json
import re
import sys
from collections import Counter, defaultdict

TITLE_MAX = 60
TITLE_MIN = 30
DESC_MAX = 160
DESC_MIN = 70

GENERIC_TITLES = {
    "home", "homepage", "index", "untitled", "untitled page", "new page",
    "default", "page", "welcome", "blog", "about", "about us", "contact",
    "contact us", "products", "services",
}
STOPWORDS = {
    "a", "an", "and", "are", "as", "at", "be", "by", "for", "from", "in",
    "is", "it", "of", "on", "or", "the", "to", "with", "your", "you", "our",
}


def words(text):
    return [w for w in re.findall(r"[a-z0-9']+", text.lower()) if w not in STOPWORDS]


def strip_brand(text, brand_suffix):
    if not brand_suffix:
        return text.strip()
    stripped = text.strip()
    brand = brand_suffix.strip()
    if stripped.lower().endswith(brand.lower()):
        stripped = stripped[: len(stripped) - len(brand)].rstrip(" |-–:·")
    return stripped


def is_noindex(page):
    return "noindex" in (page.get("meta_robots") or "").lower()


def main():
    ap = argparse.ArgumentParser(description="Run title/meta description audit checks.")
    ap.add_argument("inventory", help="metadata_inventory.json from extract_metadata.py")
    ap.add_argument("--output", default="audit_report.json")
    ap.add_argument("--brand-suffix", default="",
                    help='brand pattern appended to titles, e.g. "| Acme Co"')
    ap.add_argument("--include-noindex", action="store_true",
                    help="include noindexed pages in duplicate checks")
    args = ap.parse_args()

    with open(args.inventory, encoding="utf-8") as f:
        data = json.load(f)
    pages = data.get("pages", {})

    findings = {
        "missing_title": [], "missing_description": [],
        "duplicate_titles": [], "duplicate_descriptions": [],
        "multiple_titles": [], "multiple_descriptions": [],
        "title_too_long": [], "title_too_short": [],
        "description_too_long": [], "description_too_short": [],
        "generic_title": [],
        "description_duplicates_title": [], "keyword_stuffing": [],
        "title_h1_mismatch": [],
    }

    title_groups = defaultdict(list)
    desc_groups = defaultdict(list)
    bands = {"title": Counter(), "description": Counter()}

    for url, page in pages.items():
        if page.get("status", 200) != 200:
            continue
        titles = [t for t in page.get("titles", [])]
        descs = [d for d in page.get("descriptions", [])]
        title = next((t for t in titles if t.strip()), "")
        desc = next((d for d in descs if d.strip()), "")
        noindex = is_noindex(page)

        # Missing / multiple
        if not title:
            findings["missing_title"].append({"page": url, "h1": page.get("h1")})
            bands["title"]["missing"] += 1
        if not desc:
            findings["missing_description"].append({"page": url, "h1": page.get("h1")})
            bands["description"]["missing"] += 1
        if len([t for t in titles if t.strip()]) > 1:
            findings["multiple_titles"].append({"page": url, "count": len(titles), "values": titles})
        if len([d for d in descs if d.strip()]) > 1:
            findings["multiple_descriptions"].append({"page": url, "count": len(descs), "values": descs})

        # Title checks
        if title:
            n = len(title)
            entry = {"page": url, "title": title, "length": n}
            if n > TITLE_MAX:
                findings["title_too_long"].append(entry)
                bands["title"]["too_long"] += 1
            elif n < TITLE_MIN:
                findings["title_too_short"].append(entry)
                bands["title"]["too_short"] += 1
            else:
                bands["title"]["good"] += 1

            core = strip_brand(title, args.brand_suffix)
            if core.lower() in GENERIC_TITLES or not core:
                findings["generic_title"].append({"page": url, "title": title})

            counts = Counter(words(core))
            stuffed = [w for w, c in counts.items() if c >= 3 and len(w) > 2]
            if stuffed:
                findings["keyword_stuffing"].append(
                    {"page": url, "title": title, "repeated": stuffed})

            if not noindex or args.include_noindex:
                key = " ".join(core.lower().split())
                if key:
                    title_groups[key].append(url)

            h1 = page.get("h1")
            if h1:
                t_words, h_words = set(words(core)), set(words(h1))
                if t_words and h_words and not (t_words & h_words):
                    findings["title_h1_mismatch"].append(
                        {"page": url, "title": title, "h1": h1})

        # Description checks
        if desc:
            n = len(desc)
            entry = {"page": url, "description": desc, "length": n}
            if n > DESC_MAX:
                findings["description_too_long"].append(entry)
                bands["description"]["too_long"] += 1
            elif n < DESC_MIN:
                findings["description_too_short"].append(entry)
                bands["description"]["too_short"] += 1
            else:
                bands["description"]["good"] += 1

            if title and desc.lower().strip(" .") == title.lower().strip(" ."):
                findings["description_duplicates_title"].append(
                    {"page": url, "value": desc})

            if not noindex or args.include_noindex:
                key = " ".join(desc.lower().split())
                desc_groups[key].append(url)

    for value, urls in title_groups.items():
        if len(urls) > 1:
            findings["duplicate_titles"].append({"title": value, "pages": sorted(urls)})
    for value, urls in desc_groups.items():
        if len(urls) > 1:
            findings["duplicate_descriptions"].append({"description": value, "pages": sorted(urls)})

    severity = {
        "missing_title": "high", "missing_description": "high",
        "duplicate_titles": "high", "duplicate_descriptions": "high",
        "multiple_titles": "medium", "multiple_descriptions": "medium",
        "title_too_long": "medium", "title_too_short": "medium",
        "description_too_long": "medium", "description_too_short": "medium",
        "generic_title": "medium",
        "description_duplicates_title": "low", "keyword_stuffing": "low",
        "title_h1_mismatch": "info",
    }
    summary = {
        "pages_analyzed": sum(1 for p in pages.values() if p.get("status", 200) == 200),
        "titles_found": sum(len(p.get("titles", [])) for p in pages.values()),
        "descriptions_found": sum(len(p.get("descriptions", [])) for p in pages.values()),
        "length_bands": {k: dict(v) for k, v in bands.items()},
        "issues_by_check": {k: len(v) for k, v in findings.items()},
        "issues_by_severity": {
            level: sum(len(findings[k]) for k, s in severity.items() if s == level)
            for level in ("high", "medium", "low", "info")
        },
    }
    report = {"summary": summary, "severity": severity, "findings": findings}

    with open(args.output, "w", encoding="utf-8") as f:
        json.dump(report, f, indent=2, ensure_ascii=False)
    print(json.dumps(summary, indent=2))
    print(f"Full report written to {args.output}", file=sys.stderr)


if __name__ == "__main__":
    main()