External Link Audit Claude Skill

#!/usr/bin/env python3
"""Crawl a website and build an external (outbound) link inventory as JSON.

Usage:
    python3 extract_external_links.py https://example.com [--max-pages 500] [--max-depth 5] [--output external_links.json]

Output JSON schema:
{
  "site": "https://example.com",
  "pages_crawled": 42,
  "links": [
    {
      "source": "<page url>",
      "target": "<external url>",
      "domain": "<target host>",
      "anchor": "...",
      "rel": "nofollow sponsored|null",
      "target_attr": "_blank|null",
      "location": "nav|footer|content"
    }
  ]
}

Uses only the Python standard library.
"""

import argparse
import json
import sys
import urllib.error
import urllib.parse
import urllib.request
from collections import deque
from html.parser import HTMLParser

USER_AGENT = "ExternalLinkAuditBot/1.0 (+claude-skill)"
TIMEOUT = 15


class LinkParser(HTMLParser):
    def __init__(self, base_url):
        super().__init__()
        self.base_url = base_url
        self.links = []
        self._current = None  # link being captured
        self._anchor_parts = []
        self._context = []  # nav/footer nesting

    def handle_starttag(self, tag, attrs):
        a = dict(attrs)
        if tag in ("nav", "footer"):
            self._context.append(tag)
        elif tag == "a" and a.get("href"):
            href = a["href"].strip()
            if href.startswith(("mailto:", "tel:", "javascript:", "#")):
                return
            target = urllib.parse.urljoin(self.base_url, href)
            target = urllib.parse.urldefrag(target)[0]
            location = self._context[-1] if self._context else "content"
            self._current = {
                "target": target,
                "anchor": "",
                "rel": a.get("rel") or None,
                "target_attr": a.get("target") or None,
                "location": location,
            }
            self._anchor_parts = []
        elif tag == "img" and self._current is not None and a.get("alt"):
            self._anchor_parts.append(a["alt"])

    def handle_endtag(self, tag):
        if tag in ("nav", "footer") and self._context and self._context[-1] == tag:
            self._context.pop()
        elif tag == "a" and self._current is not None:
            self._current["anchor"] = " ".join(" ".join(self._anchor_parts).split())
            self.links.append(self._current)
            self._current = None

    def handle_data(self, data):
        if self._current is not None:
            self._anchor_parts.append(data)


def fetch(url):
    """Return (status, final_url, html_or_none)."""
    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
    try:
        with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
            ctype = resp.headers.get("Content-Type", "")
            body = None
            if "text/html" in ctype:
                body = resp.read(2_000_000).decode("utf-8", errors="replace")
            return resp.status, resp.geturl(), body
    except urllib.error.HTTPError as e:
        return e.code, url, None
    except Exception:
        return 0, url, None  # 0 = network/connection error


def norm_host(url):
    host = urllib.parse.urlparse(url).netloc.lower()
    return host[4:] if host.startswith("www.") else host


def external_in(source_url, final_url, html, host):
    """Return (external_link_records, internal_targets) parsed from one page."""
    parser = LinkParser(final_url)
    try:
        parser.feed(html)
    except Exception:
        pass
    external, internal = [], []
    for link in parser.links:
        target = link["target"]
        if not target.startswith(("http://", "https://")):
            continue
        if norm_host(target) == host:
            internal.append(target)
            continue
        external.append({"source": source_url, "domain": norm_host(target), **link})
    return external, internal


def extract_from_cache(cache_path):
    """Pure: collect external links from a shared page cache (from fetch_pages.py).

    No network — the link checking lives in check_external_links.py, which still
    probes the external URLs.
    """
    with open(cache_path, encoding="utf-8") as f:
        cache = json.load(f)
    pages = cache.get("pages", {})
    site = cache.get("site", cache_path)
    host = norm_host(site) if site.startswith(("http://", "https://")) else ""
    if not host:
        for page in pages.values():
            if page.get("final_url", "").startswith(("http://", "https://")):
                host = norm_host(page["final_url"])
                break
    external_links, crawled = [], 0
    for url, page in pages.items():
        html = page.get("html")
        if not html:
            continue
        crawled += 1
        out, _internal = external_in(url, page.get("final_url") or url, html, host)
        external_links.extend(out)
    return site, crawled, external_links


def main():
    ap = argparse.ArgumentParser(description="Crawl a site and collect external links.")
    ap.add_argument("start_url", nargs="?")
    ap.add_argument("--from-cache",
                    help="Extract from a shared page cache produced by fetch_pages.py (no network)")
    ap.add_argument("--max-pages", type=int, default=500)
    ap.add_argument("--max-depth", type=int, default=5)
    ap.add_argument("--output", default="external_links.json")
    args = ap.parse_args()

    if args.from_cache:
        start, pages_crawled, external_links = extract_from_cache(args.from_cache)
    elif not args.start_url:
        ap.error("provide a start_url, or --from-cache page_cache.json")
    else:
        start = args.start_url.rstrip("/")
        host = norm_host(start)
        external_links = []
        crawled = set()
        queue = deque([(start, 0)])
        seen = {start}

        while queue and len(crawled) < args.max_pages:
            url, depth = queue.popleft()
            status, final_url, html = fetch(url)
            crawled.add(url)
            print(f"[{status}] depth={depth} {url}", file=sys.stderr)

            if not html:
                continue
            out, internal = external_in(url, final_url, html, host)
            external_links.extend(out)
            for target in internal:
                if target not in seen and depth + 1 <= args.max_depth:
                    seen.add(target)
                    queue.append((target, depth + 1))
        pages_crawled = len(crawled)

    with open(args.output, "w") as f:
        json.dump({"site": start, "pages_crawled": pages_crawled,
                   "links": external_links}, f, indent=2)
    print(f"Wrote {len(external_links)} external links from {pages_crawled} pages to {args.output}",
          file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Check external link targets and run audit checks on an inventory from extract_external_links.py.

Usage:
    python3 check_external_links.py external_links.json [--output audit_report.json]
        [--sponsored-domains amzn.to,partner.example] [--skip-status-checks]

Checks implemented (see references/audit-checks.md):
- broken_links: external links to 4xx/5xx or unresolvable targets (high)
- unflagged_sponsored: links to known affiliate/paid domains missing rel sponsored/nofollow (high)
- insecure_links: http:// external links (medium)
- redirected_links: external links to 3xx targets (medium)
- unsafe_blank: target="_blank" without rel noopener/noreferrer (medium)
- generic_anchors: weak anchor text (low)
- excessive_external: pages with > 100 external links (low)
- sitewide_links: same nav/footer external link on 5+ pages (low)

Uses only the Python standard library.
"""

import argparse
import json
import sys
import urllib.error
import urllib.request
from collections import Counter, defaultdict

USER_AGENT = "ExternalLinkAuditBot/1.0 (+claude-skill)"
TIMEOUT = 15

GENERIC_ANCHORS = {
    "click here", "here", "read more", "learn more", "more",
    "this page", "link", "this", "source", "website", "see more", "",
}
MAX_EXTERNAL_LINKS = 100
SITEWIDE_MIN_PAGES = 5
AFFILIATE_PARAMS = ("tag=", "ref=", "aff_id=", "affid=", "affiliate=", "utm_medium=affiliate")


def check_status(url):
    """Return final HTTP status without following redirects (HEAD, GET fallback)."""

    class NoRedirect(urllib.request.HTTPRedirectHandler):
        def redirect_request(self, *args, **kwargs):
            return None

    opener = urllib.request.build_opener(NoRedirect)
    for method in ("HEAD", "GET"):
        req = urllib.request.Request(url, method=method,
                                     headers={"User-Agent": USER_AGENT})
        try:
            with opener.open(req, timeout=TIMEOUT) as resp:
                return resp.status
        except urllib.error.HTTPError as e:
            if e.code in (405, 501) and method == "HEAD":
                continue  # retry with GET
            return e.code
        except Exception:
            return 0  # 0 = DNS/network/connection error
    return 0


def main():
    ap = argparse.ArgumentParser(description="Run external link audit checks.")
    ap.add_argument("inventory", help="external_links.json from extract_external_links.py")
    ap.add_argument("--output", default="audit_report.json")
    ap.add_argument("--sponsored-domains", default="",
                    help="comma-separated affiliate/paid domains requiring rel=sponsored")
    ap.add_argument("--skip-status-checks", action="store_true",
                    help="skip network requests; run only static checks")
    args = ap.parse_args()

    with open(args.inventory) as f:
        data = json.load(f)
    links = data["links"]
    sponsored_domains = {d.strip().lower() for d in args.sponsored_domains.split(",") if d.strip()}

    statuses = {}
    if not args.skip_status_checks:
        for target in sorted({l["target"] for l in links}):
            statuses[target] = check_status(target)
            print(f"[{statuses[target]}] {target}", file=sys.stderr)

    findings = {
        "broken_links": [], "unflagged_sponsored": [], "insecure_links": [],
        "redirected_links": [], "unsafe_blank": [], "generic_anchors": [],
        "excessive_external": [], "sitewide_links": [],
    }

    per_page = Counter()
    sitewide = defaultdict(set)  # nav/footer target -> source pages
    domains = Counter()
    domain_followed = Counter()

    for link in links:
        src, target, anchor = link["source"], link["target"], link["anchor"]
        rel = (link.get("rel") or "").lower()
        entry = {"source": src, "target": target, "anchor": anchor}

        per_page[src] += 1
        domains[link["domain"]] += 1
        qualified = any(r in rel for r in ("nofollow", "sponsored", "ugc"))
        if not qualified:
            domain_followed[link["domain"]] += 1
        if link["location"] in ("nav", "footer"):
            sitewide[target].add(src)

        status = statuses.get(target)
        if status is not None:
            if status >= 400 or status == 0:
                findings["broken_links"].append({**entry, "status": status})
            elif 300 <= status < 400:
                findings["redirected_links"].append({**entry, "status": status})

        if target.startswith("http://"):
            findings["insecure_links"].append(entry)

        is_affiliate = (link["domain"] in sponsored_domains
                        or any(p in target for p in AFFILIATE_PARAMS))
        if is_affiliate and "sponsored" not in rel and "nofollow" not in rel:
            findings["unflagged_sponsored"].append({**entry, "rel": link.get("rel")})

        if (link.get("target_attr") == "_blank"
                and "noopener" not in rel and "noreferrer" not in rel):
            findings["unsafe_blank"].append({**entry, "rel": link.get("rel")})

        if anchor.lower().strip() in GENERIC_ANCHORS:
            findings["generic_anchors"].append(entry)

    for page, count in per_page.items():
        if count > MAX_EXTERNAL_LINKS:
            findings["excessive_external"].append({"page": page, "external_links": count})

    for target, sources in sitewide.items():
        if len(sources) >= SITEWIDE_MIN_PAGES:
            findings["sitewide_links"].append({"target": target, "pages": len(sources)})

    severity = {
        "broken_links": "high", "unflagged_sponsored": "high",
        "insecure_links": "medium", "redirected_links": "medium",
        "unsafe_blank": "medium",
        "generic_anchors": "low", "excessive_external": "low", "sitewide_links": "low",
    }
    summary = {
        "pages_crawled": data.get("pages_crawled"),
        "external_links": len(links),
        "unique_domains": len(domains),
        "issues_by_check": {k: len(v) for k, v in findings.items()},
        "issues_by_severity": {
            level: sum(len(findings[k]) for k, s in severity.items() if s == level)
            for level in ("high", "medium", "low")
        },
    }
    domain_overview = [
        {"domain": d, "links": n, "followed": domain_followed[d],
         "qualified": n - domain_followed[d]}
        for d, n in domains.most_common()
    ]
    report = {"summary": summary, "severity": severity, "findings": findings,
              "domain_overview": domain_overview, "statuses": statuses}

    with open(args.output, "w") as f:
        json.dump(report, f, indent=2)
    print(json.dumps(summary, indent=2))
    print(f"Full report written to {args.output}", file=sys.stderr)


if __name__ == "__main__":
    main()