Internal Link Audit Claude Skill

#!/usr/bin/env python3
"""Crawl a website and build an internal link graph as JSON.

Usage:
    python3 crawl_links.py https://example.com [--max-pages 500] [--max-depth 5] [--output link_graph.json]

Output JSON schema:
{
  "site": "https://example.com",
  "pages": {
    "<url>": {
      "status": 200,
      "depth": 1,
      "title": "...",
      "canonical": "<url or null>",
      "meta_robots": "<content or null>",
      "links": [
        {"target": "<url>", "anchor": "...", "rel": "nofollow|null", "location": "nav|footer|content"}
      ]
    }
  },
  "statuses": {"<url>": 200}   # status of every linked URL, including non-crawled targets
}

Uses only the Python standard library.
"""

import argparse
import json
import sys
import urllib.error
import urllib.parse
import urllib.request
from collections import deque
from html.parser import HTMLParser

USER_AGENT = "InternalLinkAuditBot/1.0 (+claude-skill)"
TIMEOUT = 15


class LinkParser(HTMLParser):
    def __init__(self, base_url):
        super().__init__()
        self.base_url = base_url
        self.links = []
        self.title = None
        self.canonical = None
        self.meta_robots = None
        self._in_title = False
        self._current = None  # link being captured
        self._anchor_parts = []
        self._context = []  # nav/footer nesting

    def handle_starttag(self, tag, attrs):
        a = dict(attrs)
        if tag in ("nav", "footer"):
            self._context.append(tag)
        elif tag == "title":
            self._in_title = True
        elif tag == "link" and a.get("rel", "").lower() == "canonical" and a.get("href"):
            self.canonical = urllib.parse.urljoin(self.base_url, a["href"])
        elif tag == "meta" and a.get("name", "").lower() == "robots":
            self.meta_robots = a.get("content")
        elif tag == "a" and a.get("href"):
            href = a["href"].strip()
            if href.startswith(("mailto:", "tel:", "javascript:", "#")):
                return
            target = urllib.parse.urljoin(self.base_url, href)
            target = urllib.parse.urldefrag(target)[0]
            location = self._context[-1] if self._context else "content"
            self._current = {
                "target": target,
                "anchor": "",
                "rel": a.get("rel") or None,
                "location": location,
            }
            self._anchor_parts = []
        elif tag == "img" and self._current is not None and a.get("alt"):
            self._anchor_parts.append(a["alt"])

    def handle_endtag(self, tag):
        if tag in ("nav", "footer") and self._context and self._context[-1] == tag:
            self._context.pop()
        elif tag == "title":
            self._in_title = False
        elif tag == "a" and self._current is not None:
            self._current["anchor"] = " ".join(" ".join(self._anchor_parts).split())
            self.links.append(self._current)
            self._current = None

    def handle_data(self, data):
        if self._in_title:
            self.title = (self.title or "") + data.strip()
        if self._current is not None:
            self._anchor_parts.append(data)


def fetch(url):
    """Return (status, final_url, html_or_none)."""
    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
    try:
        with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
            ctype = resp.headers.get("Content-Type", "")
            body = None
            if "text/html" in ctype:
                body = resp.read(2_000_000).decode("utf-8", errors="replace")
            return resp.status, resp.geturl(), body
    except urllib.error.HTTPError as e:
        return e.code, url, None
    except Exception:
        return 0, url, None  # 0 = network/connection error


def same_host(url, host):
    return urllib.parse.urlparse(url).netloc.lower().lstrip("www.") == host


def main():
    ap = argparse.ArgumentParser(description="Crawl a site and build an internal link graph.")
    ap.add_argument("start_url")
    ap.add_argument("--max-pages", type=int, default=500)
    ap.add_argument("--max-depth", type=int, default=5)
    ap.add_argument("--output", default="link_graph.json")
    args = ap.parse_args()

    start = args.start_url.rstrip("/")
    host = urllib.parse.urlparse(start).netloc.lower().lstrip("www.")
    pages, statuses = {}, {}
    queue = deque([(start, 0)])
    seen = {start}

    while queue and len(pages) < args.max_pages:
        url, depth = queue.popleft()
        status, final_url, html = fetch(url)
        statuses[url] = status
        print(f"[{status}] depth={depth} {url}", file=sys.stderr)

        page = {"status": status, "depth": depth, "title": None,
                "canonical": None, "meta_robots": None, "links": []}
        if html:
            parser = LinkParser(final_url)
            try:
                parser.feed(html)
            except Exception:
                pass
            page.update(title=parser.title, canonical=parser.canonical,
                        meta_robots=parser.meta_robots)
            for link in parser.links:
                if not same_host(link["target"], host):
                    continue
                page["links"].append(link)
                t = link["target"]
                if t not in seen and depth + 1 <= args.max_depth:
                    seen.add(t)
                    queue.append((t, depth + 1))
        pages[url] = page

    # Resolve statuses for linked-but-uncrawled targets
    linked = {l["target"] for p in pages.values() for l in p["links"]}
    for t in sorted(linked - set(statuses)):
        status, _, _ = fetch(t)
        statuses[t] = status
        print(f"[{status}] (status-check) {t}", file=sys.stderr)

    with open(args.output, "w") as f:
        json.dump({"site": start, "pages": pages, "statuses": statuses}, f, indent=2)
    print(f"Wrote {len(pages)} pages, {len(statuses)} URL statuses to {args.output}", file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Analyze an internal link graph JSON (from crawl_links.py) and emit audit findings.

Usage:
    python3 analyze_graph.py link_graph.json [--output audit_report.json]

Checks implemented (see references/audit-checks.md):
- broken_links: internal links to 4xx/5xx targets (high)
- orphan_pages: crawled pages with zero inbound internal links (high)
- redirected_links: internal links to 3xx targets (medium)
- deep_pages: pages with click depth > 3 (medium)
- generic_anchors: weak anchor text (medium)
- noncanonical_targets: links to pages whose canonical differs or are noindexed (medium)
- excessive_outlinks: pages with > 150 outgoing internal links (low)
- nofollow_internal: internal links with rel nofollow/sponsored/ugc (low)

Uses only the Python standard library.
"""

import argparse
import json
import sys
from collections import Counter

GENERIC_ANCHORS = {
    "click here", "here", "read more", "learn more", "more",
    "this page", "link", "this", "continue", "view", "see more", "",
}
MAX_DEPTH = 3
MAX_OUTLINKS = 150


def main():
    ap = argparse.ArgumentParser(description="Run internal link audit checks on a link graph.")
    ap.add_argument("graph", help="link_graph.json from crawl_links.py")
    ap.add_argument("--output", default="audit_report.json")
    args = ap.parse_args()

    with open(args.graph) as f:
        data = json.load(f)
    pages = data["pages"]
    statuses = data.get("statuses", {})

    inbound = Counter()
    for src, page in pages.items():
        for link in page["links"]:
            if link["target"] != src:
                inbound[link["target"]] += 1

    findings = {
        "broken_links": [], "orphan_pages": [], "redirected_links": [],
        "deep_pages": [], "generic_anchors": [], "noncanonical_targets": [],
        "excessive_outlinks": [], "nofollow_internal": [],
    }

    home_depth0 = min((p["depth"] for p in pages.values()), default=0)

    for src, page in pages.items():
        if len(page["links"]) > MAX_OUTLINKS:
            findings["excessive_outlinks"].append(
                {"page": src, "outgoing_links": len(page["links"])})
        if page["depth"] - home_depth0 > MAX_DEPTH:
            findings["deep_pages"].append({"page": src, "depth": page["depth"]})

        for link in page["links"]:
            target, anchor = link["target"], link["anchor"]
            status = statuses.get(target)
            entry = {"source": src, "target": target, "anchor": anchor}

            if status is not None:
                if status >= 400 or status == 0:
                    findings["broken_links"].append({**entry, "status": status})
                elif 300 <= status < 400:
                    findings["redirected_links"].append({**entry, "status": status})

            if anchor.lower().strip() in GENERIC_ANCHORS:
                findings["generic_anchors"].append(entry)

            rel = (link.get("rel") or "").lower()
            if any(r in rel for r in ("nofollow", "sponsored", "ugc")):
                findings["nofollow_internal"].append({**entry, "rel": link["rel"]})

            tpage = pages.get(target)
            if tpage:
                robots = (tpage.get("meta_robots") or "").lower()
                canonical = tpage.get("canonical")
                if "noindex" in robots:
                    findings["noncanonical_targets"].append({**entry, "reason": "noindex"})
                elif canonical and canonical.rstrip("/") != target.rstrip("/"):
                    findings["noncanonical_targets"].append(
                        {**entry, "reason": "canonical points elsewhere",
                         "canonical": canonical})

    for url, page in pages.items():
        if inbound[url] == 0 and page["depth"] > home_depth0:
            findings["orphan_pages"].append({"page": url})

    severity = {
        "broken_links": "high", "orphan_pages": "high",
        "redirected_links": "medium", "deep_pages": "medium",
        "generic_anchors": "medium", "noncanonical_targets": "medium",
        "excessive_outlinks": "low", "nofollow_internal": "low",
    }
    summary = {
        "pages_analyzed": len(pages),
        "total_internal_links": sum(len(p["links"]) for p in pages.values()),
        "issues_by_check": {k: len(v) for k, v in findings.items()},
        "issues_by_severity": {
            level: sum(len(findings[k]) for k, s in severity.items() if s == level)
            for level in ("high", "medium", "low")
        },
    }
    report = {"summary": summary, "severity": severity, "findings": findings,
              "inbound_link_counts": dict(inbound)}

    with open(args.output, "w") as f:
        json.dump(report, f, indent=2)
    print(json.dumps(summary, indent=2))
    print(f"Full report written to {args.output}", file=sys.stderr)


if __name__ == "__main__":
    main()