#!/usr/bin/env python3
"""PDF tag-tree editor (file-remediation Stage 2, Engine E2 library).

Library used by remediate_pdf.py. Two data sources feed judgment repairs:
  - The ODL JSON manifest (load_odl_json): flat depth-first content nodes with
    real text + bounding boxes, bottom-up PDF coords converted to top-down.
  - The pikepdf StructTreeRoot tag tree (walk_figures/walk_elems): the actual
    editable structure elements.

# ponytail: cross-tool index alignment is a documented assumption, not a proven
# invariant beyond Figures. Verified live (2026-07-20): the depth-first order of
# /Figure nodes in ODL's JSON output matches the depth-first order of /Figure
# struct elements in ODL's own tagged-pdf output for the reference fixture. This
# module extrapolates the same one-pass-tool assumption to all node types
# (headings, paragraphs) for load_odl_json <-> walk_elems(pdf, None) index
# alignment, used only by the bbox-matched heading-override path in
# remediate_pdf.py. Index-keyed overrides (the primary path) never rely on this
# assumption — they index directly into walk_elems(pdf, None) / walk_figures(pdf).
"""

from __future__ import annotations

import argparse
import json
import re
import sys
from collections import Counter
from pathlib import Path

import pikepdf

SCRIPTS_DIR = Path(__file__).resolve().parent
if str(SCRIPTS_DIR) not in sys.path:
    sys.path.insert(0, str(SCRIPTS_DIR))

import mcid_text

_H_RE = re.compile(r"^/H(\d+)$")


# ── ODL JSON manifest ───────────────────────────────────────────────────────

def load_odl_json(path, page_heights=None):
    """Parse an opendataloader-pdf JSON manifest into a flat depth-first node
    list. Each entry: {type, pdfua_tag, page, bbox, content}. bbox is
    top-down [x0,y0,x1,y1] (converted from ODL's bottom-up PDF coords using
    page_heights={page_num: height} if given, else 792 (letter) default)."""
    data = json.loads(Path(path).read_text(encoding="utf-8"))
    flat = []

    def _convert(bbox, page):
        if not bbox:
            return None
        h = (page_heights or {}).get(page, 792)
        x0, y0, x1, y1 = bbox
        return [x0, h - y1, x1, h - y0]

    def _flatten(nodes):
        for n in nodes or []:
            page = n.get("page number")
            flat.append({
                "type": n.get("type"),
                "pdfua_tag": n.get("pdfua_tag"),
                "page": page,
                "bbox": _convert(n.get("bounding box"), page),
                "content": n.get("content"),
            })
            _flatten(n.get("kids"))

    _flatten(data.get("kids"))
    return flat


# ── StructTreeRoot walk ──────────────────────────────────────────────────────

def _children(node):
    """Direct StructElem children of a tag-tree node's /K entry."""
    kids = node.get("/K") if hasattr(node, "get") else None
    if kids is None:
        return []
    if isinstance(kids, pikepdf.Array):
        return [k for k in kids if hasattr(k, "get") and "/S" in k]
    if hasattr(kids, "get") and "/S" in kids:
        return [kids]
    return []


def walk_elems(pdf, s_type):
    """DFS pre-order over the StructTreeRoot. s_type filters by /S value (e.g.
    "/Figure", "/P"). s_type=None returns every LEAF struct element (nodes
    with no nested struct-element children) — this excludes pure grouping
    containers like /Document/ /Sect so index-into-this-list stays aligned
    with ODL's flat JSON node list, which never emits a container entry
    either. Cycle-guarded via a visited-objgen set."""
    root = pdf.Root.get("/StructTreeRoot")
    if root is None:
        return []
    out, visited = [], set()
    stack = list(reversed(_children(root)))
    while stack:
        node = stack.pop()
        key = node.objgen if node.is_indirect else id(node)
        if key in visited:
            continue
        visited.add(key)
        kids = _children(node)
        if s_type is not None:
            if str(node.get("/S", "")) == s_type:
                out.append(node)
        elif not kids:  # leaf only
            out.append(node)
        stack.extend(reversed(kids))
    return out


def walk_figures(pdf):
    """DFS pre-order list of /Figure struct elements."""
    return walk_elems(pdf, "/Figure")


# ── Alt text ─────────────────────────────────────────────────────────────

def set_alts(pdf, alt_map):
    """Set /Alt on /Figure elements. alt_map: {figure_index (depth-first,
    0-based, walk_figures order): alt_text}. Returns {index: alt_text} for
    the entries actually applied (out-of-range indices are skipped)."""
    figures = walk_figures(pdf)
    applied = {}
    for k, v in (alt_map or {}).items():
        i = int(k)
        if 0 <= i < len(figures):
            figures[i]["/Alt"] = v
            applied[i] = v
    return applied


def match_figures(odl_images, manifest_figures):
    """Match an external manifest's figures (e.g. Claude-vision crops, each
    {page, bbox}) to this doc's odl_images ({page, bbox}, ODL order) by
    nearest bbox-center distance per page (greedy, no cross-page matches).
    Returns a list aligned to odl_images: matched manifest_figures index, or
    None if no same-page candidate exists."""
    def _center(bbox):
        x0, y0, x1, y1 = bbox
        return ((x0 + x1) / 2, (y0 + y1) / 2)

    used = set()
    result = []
    for odl_img in odl_images:
        page = odl_img.get("page")
        oc = _center(odl_img["bbox"])
        best_i, best_d = None, None
        for i, mf in enumerate(manifest_figures):
            if i in used or mf.get("page") != page:
                continue
            mc = _center(mf["bbox"])
            d = (mc[0] - oc[0]) ** 2 + (mc[1] - oc[1]) ** 2
            if best_d is None or d < best_d:
                best_d, best_i = d, i
        if best_i is not None:
            used.add(best_i)
        result.append(best_i)
    return result


# ── Headings ─────────────────────────────────────────────────────────────

def retag(pdf, elem, new_s):
    """Change a struct element's /S (the heading-promotion primitive)."""
    elem["/S"] = pikepdf.Name(new_s if new_s.startswith("/") else f"/{new_s}")


def normalize_heading_levels(pdf):
    """Walk headings in document order; clamp any level jump > +1 from the
    previous heading level (Hn > prev+1 -> prev+1). Never skips a level.
    Returns promotions: [{index, from, to}] in walk_elems(pdf, None) order."""
    nodes = walk_elems(pdf, None)
    promotions = []
    prev = 0
    for i, node in enumerate(nodes):
        m = _H_RE.match(str(node.get("/S", "")))
        if not m:
            continue
        cur = int(m.group(1))
        new = cur if cur <= prev + 1 else prev + 1
        if new != cur:
            retag(pdf, node, f"/H{new}")
            promotions.append({"index": i, "from": f"/H{cur}", "to": f"/H{new}"})
        prev = new
    return promotions


# ── Heading candidate detection (SCRIPT half of heading-promote) ───────────
#
# docs/plan-heading-promote.md. veraPDF checks structural VALIDITY, never
# whether headings exist -- a document can be 106/106 compliant while every
# element is tagged /P (confirmed live: chapter10-remediated.pdf, 854 /P,
# 0 /H). This module surfaces font-signature evidence per non-heading leaf
# struct element (rounded size, bold, fill rgb, base font name) and clusters
# occurrences whose signature differs from the document's BODY signature
# (the most common signature among small/non-bold/dark spans -- ordinary
# paragraph text). It NEVER decides a cluster is a heading or picks its
# level -- font size is evidence, not a verdict (a large word mid-sentence
# isn't a heading; a colored label might not be one either). Every candidate
# this module returns carries "level": None; an agent/human fills it in
# before promote_headings() is ever called (heading_promote.py Phase 2/3).
#
# Signature extraction goes through PyMuPDF (fitz) span metadata, not a raw
# content-stream /Tf read -- verified against a real remediated chapter file
# where EVERY span sets `/Tf ... 1` and encodes the actual 10.5pt/12pt/18pt
# sizes purely via the text matrix scale. fitz's span "size" already resolves
# that full transform into a real point size; a raw-content-stream /Tf
# operand alone would misreport every span in that file as size 1. Spans are
# correlated back to a struct element's mcids by exact normalized-text match
# (falling back to substring containment for the rare multi-span element) --
# never by position/bbox, so no reading-order assumption is required.

_BOLD_NAME_RE = re.compile(r"bold|black|heavy|semibold", re.IGNORECASE)
_HEADING_EXCLUDE_TAGS = {"/Figure", "/Link"}
_SMALL_SIZE_MAX = 13.0
_DARK_LUMINANCE_MAX = 0.5
_SPAN_BOLD_FLAG = 16  # PyMuPDF span "flags" bit 4


def _page_fitz_spans(fitz_page):
    """[(normalized_text, size_rounded, bold, rgb, font), ...] in PyMuPDF's
    natural reading order (`sort=True`) -- the font-signature evidence
    source for heading_candidates(). `rgb` is each channel 0-1, rounded.
    Never raises: an extraction failure yields []."""
    out = []
    try:
        d = fitz_page.get_text("dict", sort=True)
    except Exception:  # noqa: BLE001
        return out
    for block in d.get("blocks", []):
        for line in block.get("lines", []):
            for span in line.get("spans", []):
                text = re.sub(r"\s+", " ", span.get("text", "")).strip()
                if not text:
                    continue
                font = span.get("font", "") or ""
                bold = bool(span.get("flags", 0) & _SPAN_BOLD_FLAG) or bool(_BOLD_NAME_RE.search(font))
                color_int = int(span.get("color", 0) or 0)
                rgb = (
                    round(((color_int >> 16) & 255) / 255, 3),
                    round(((color_int >> 8) & 255) / 255, 3),
                    round((color_int & 255) / 255, 3),
                )
                out.append((text, round(span.get("size", 0.0)), bold, rgb, font))
    return out


def _page_signature_lookup(fitz_page):
    """({normalized_text: Counter[signature]}, [(text, size, bold, rgb,
    font), ...]) for `fitz_page` -- the dict is the fast exact-match path,
    the list backs the substring fallback in _match_signature()."""
    spans = _page_fitz_spans(fitz_page)
    by_text: dict = {}
    for text, size, bold, rgb, font in spans:
        by_text.setdefault(text, Counter())[(size, bold, rgb, font)] += 1
    return by_text, spans


def _match_signature(by_text, spans, text):
    """The signature for `text` (already whitespace-normalized): an exact
    span-text match wins outright; otherwise the longest span whose text
    contains (or is contained by) `text` -- covers the rare element whose
    mcids span more than one fitz span. None when no evidence at all can be
    found (never guessed from nothing)."""
    if text in by_text:
        return by_text[text].most_common(1)[0][0]
    best, best_len = None, -1
    for span_text, size, bold, rgb, font in spans:
        if span_text == text:
            continue
        if text in span_text or span_text in text:
            overlap = min(len(text), len(span_text))
            if overlap > best_len:
                best_len, best = overlap, (size, bold, rgb, font)
    return best


def _detect_body_signature(occurrences):
    """Most common signature among small (<~13pt), non-bold, dark spans --
    ordinary paragraph text. Falls back to the single most common signature
    overall when nothing matches that filter (e.g. an all-bold or all-large
    document), so a real body paragraph is never mistaken for a heading
    candidate purely because the small/non-bold/dark filter came up empty."""
    body_like, overall = Counter(), Counter()
    for o in occurrences:
        size, bold, rgb, _font = o["sig"]
        overall[o["sig"]] += 1
        luminance = 0.299 * rgb[0] + 0.587 * rgb[1] + 0.114 * rgb[2]
        if size < _SMALL_SIZE_MAX and not bold and luminance < _DARK_LUMINANCE_MAX:
            body_like[o["sig"]] += 1
    if body_like:
        return body_like.most_common(1)[0][0]
    return overall.most_common(1)[0][0]


def heading_candidates(pdf, path=None):
    """Detector half of heading-promotion (docs/plan-heading-promote.md
    Deliverable #1). Scans every non-heading, non-Figure/Link leaf struct
    element's text, resolves its dominant font signature via PyMuPDF span
    metadata (_page_signature_lookup / _match_signature), excludes the
    document's body-text signature (_detect_body_signature), and clusters
    the rest by exact signature into "cluster_id"s ordered largest-size-
    first. Returns one dict per surviving occurrence: {cluster_id, size,
    bold, rgb, font, text, page, mcids, current_tag, cluster_count, level}.
    "level" is always None -- this function never concludes a cluster IS a
    heading or assigns it a level; that is the agent's job (see
    heading_promote.py). `path` is the PDF's filesystem path (required to
    open it via fitz); falls back to `pdf.filename` when omitted. Returns
    [] (never raises) when there's no struct tree, or no path is resolvable
    while the struct tree is non-empty (no evidence source available)."""
    page_map = _figure_page_map(pdf)
    text_cache: dict = {}
    elems = []
    for elem in walk_elems(pdf, None):
        s = str(elem.get("/S", ""))
        if not s or s in _HEADING_EXCLUDE_TAGS or _H_RE.match(s):
            continue
        pg = elem.get("/Pg")
        if pg is None:
            continue
        page_no = page_map.get(_page_key(pg))
        if page_no is None:
            continue
        mcids = _leaf_mcids(elem.get("/K"))
        if not mcids:
            continue
        if page_no not in text_cache:
            text_cache[page_no] = mcid_text.page_mcid_text(pdf.pages[page_no - 1])
        texts = text_cache[page_no]
        text = " ".join(texts[m] for m in mcids if m in texts).strip()
        if not text:
            continue
        elems.append({"page": page_no, "mcids": mcids, "text": text, "current_tag": s})

    if not elems:
        return []

    resolved_path = path or getattr(pdf, "filename", None)
    if not resolved_path:
        return []

    try:
        import fitz  # PyMuPDF
        fitz_doc = fitz.open(str(resolved_path))
    except Exception:  # noqa: BLE001
        return []

    try:
        sig_lookup_cache: dict = {}
        occurrences = []
        for e in elems:
            page_no = e["page"]
            if page_no not in sig_lookup_cache:
                if 1 <= page_no <= fitz_doc.page_count:
                    sig_lookup_cache[page_no] = _page_signature_lookup(fitz_doc[page_no - 1])
                else:
                    sig_lookup_cache[page_no] = ({}, [])
            by_text, spans = sig_lookup_cache[page_no]
            sig = _match_signature(by_text, spans, e["text"])
            if sig is None:
                continue
            occurrences.append({**e, "sig": sig})
    finally:
        fitz_doc.close()

    if not occurrences:
        return []

    body_sig = _detect_body_signature(occurrences)
    candidates = [o for o in occurrences if o["sig"] != body_sig]
    if not candidates:
        return []

    distinct, seen = [], set()
    for o in candidates:
        if o["sig"] not in seen:
            seen.add(o["sig"])
            distinct.append(o["sig"])
    distinct.sort(key=lambda sig: (-sig[0], not sig[1]))
    cluster_id_by_sig = {sig: i for i, sig in enumerate(distinct)}
    counts = Counter(o["sig"] for o in candidates)

    out = []
    for o in candidates:
        size, bold, rgb, font = o["sig"]
        out.append({
            "cluster_id": cluster_id_by_sig[o["sig"]],
            "size": size, "bold": bold, "rgb": list(rgb), "font": font,
            "text": o["text"], "page": o["page"], "mcids": o["mcids"],
            "current_tag": o["current_tag"], "cluster_count": counts[o["sig"]],
            "level": None,
        })
    return out


def _leaf_index(pdf):
    """{(page_no, frozenset(mcids)): elem} for every leaf struct element
    with a resolvable page and at least one mcid -- promote_headings' lookup
    table. Built once per call: retagging /S never changes /Pg or /K, so this
    stays valid across every assignment processed in that call."""
    page_map = _figure_page_map(pdf)
    index = {}
    for elem in walk_elems(pdf, None):
        pg = elem.get("/Pg")
        if pg is None:
            continue
        page_no = page_map.get(_page_key(pg))
        if page_no is None:
            continue
        mcids = _leaf_mcids(elem.get("/K"))
        if not mcids:
            continue
        index[(page_no, frozenset(mcids))] = elem
    return index


def promote_headings(pdf, assignments):
    """Promoter half of heading-promotion (Deliverable #2). `assignments`:
    [{page, mcids, level}, ...] -- normally heading_candidates() occurrences
    with an agent-assigned level (1-6) merged back in. For each assignment,
    resolves the leaf struct element owning exactly those mcids on that page
    and retags it /Hn via retag(). A None/invalid/out-of-range level, or an
    element that can't be resolved, is skipped and reported in "unresolved"
    -- never guessed, never raised (defense in depth: heading_promote.py's
    --apply already filters to non-null levels before calling this, but this
    function enforces the same honesty rule on its own). Always finishes
    with normalize_heading_levels(pdf) (clamps any level jump the new
    headings introduced). Returns {"promoted": [...], "unresolved": [...]}."""
    index = _leaf_index(pdf)
    promoted, unresolved = [], []
    for a in assignments or []:
        page = a.get("page")
        mcids = a.get("mcids") or []
        level = a.get("level")
        if level is None:
            unresolved.append({"page": page, "mcids": mcids, "reason": "no level assigned"})
            continue
        try:
            lvl = int(level)
        except (TypeError, ValueError):
            unresolved.append({"page": page, "mcids": mcids, "reason": f"invalid level: {level!r}"})
            continue
        if not 1 <= lvl <= 6:
            unresolved.append({"page": page, "mcids": mcids, "reason": f"level out of range: {lvl}"})
            continue
        elem = index.get((page, frozenset(mcids)))
        if elem is None:
            unresolved.append({
                "page": page, "mcids": mcids,
                "reason": "could not resolve owning struct element",
            })
            continue
        from_tag = str(elem.get("/S", ""))
        retag(pdf, elem, f"/H{lvl}")
        promoted.append({"page": page, "mcids": mcids, "from": from_tag, "to": f"/H{lvl}"})
    normalize_heading_levels(pdf)
    return {"promoted": promoted, "unresolved": unresolved}


# ── Link annotations ─────────────────────────────────────────────────────

def tag_link_annotations(pdf):
    """Set /Contents on every Link annotation missing it. LibreOffice/ODL
    output a /Link struct elem + OBJR but no /Contents key -> veraPDF 7.18.x
    fail. URI-derived deterministic fallback ("Link: <uri>", or bare "Link"
    with no /A/URI); judgment can later supply a real link description.
    Returns the number of annotations fixed."""
    fixed = 0
    for page in pdf.pages:
        for annot in page.get("/Annots") or []:
            if str(annot.get("/Subtype", "")) != "/Link" or "/Contents" in annot:
                continue
            a = annot.get("/A")
            uri = str(a["/URI"]) if a is not None and "/URI" in a else None
            annot["/Contents"] = pikepdf.String(f"Link: {uri}" if uri else "Link")
            fixed += 1
    return fixed


def _raw_kids(node):
    """Every /K child of a struct-tree node, unfiltered (ints, MCR dicts,
    OBJR dicts, and StructElem dicts alike) -- unlike _children(), which
    keeps only nested StructElems."""
    if not hasattr(node, "get"):
        return []
    k = node.get("/K")
    if k is None:
        return []
    return list(k) if isinstance(k, pikepdf.Array) else [k]


def _node_key(node):
    try:
        return node.objgen if getattr(node, "is_indirect", False) else id(node)
    except Exception:  # noqa: BLE001
        return id(node)


def _collect_objr_annot_keys(root):
    """{annot.objgen} for every annotation already referenced by an /OBJR
    somewhere in the struct tree -- so wrap_link_annotations() never
    double-wraps a Link the source document already tagged correctly."""
    keys, visited = set(), set()

    def walk(node):
        key = _node_key(node)
        if key in visited:
            return
        visited.add(key)
        for item in _raw_kids(node):
            if not hasattr(item, "get"):
                continue
            if str(item.get("/Type", "")) == "/OBJR":
                obj = item.get("/Obj")
                if obj is not None and getattr(obj, "is_indirect", False):
                    keys.add(obj.objgen)
            elif "/S" in item:
                walk(item)

    for child in _children(root):
        walk(child)
    return keys


def _page_leaf_slots(root, page_obj):
    """[(leaf_node, parent_node), ...] in document order for every leaf
    struct element whose (inherited) /Pg resolves to page_obj -- mirrors
    the /Pg-inheritance walk in _remove_struct_kids()."""
    out, visited = [], set()

    def walk(node, parent, pg):
        key = _node_key(node)
        if key in visited:
            return
        visited.add(key)
        node_pg = node.get("/Pg", pg) if hasattr(node, "get") else pg
        kids = _children(node)
        if not kids:
            if _same_page(node_pg, page_obj):
                out.append((node, parent))
            return
        for kid in kids:
            walk(kid, node, node_pg)

    for child in _children(root):
        walk(child, root, None)
    return out


def _append_struct_kid(parent, elem):
    kids = parent.get("/K")
    if not isinstance(kids, pikepdf.Array):
        parent["/K"] = pikepdf.Array([kids] if kids is not None else [])
        kids = parent["/K"]
    kids.append(elem)


def _next_parent_tree_key(nums):
    if nums is None or len(nums) == 0:
        return 0
    return max(int(nums[i]) for i in range(0, len(nums), 2)) + 1


def _page_height(page):
    box = page.get("/MediaBox") or [0, 0, 612, 792]
    return float(box[3]) - float(box[1])


def _rect_overlaps(a, b):
    ax0, ay0, ax1, ay1 = a
    bx0, by0, bx1, by1 = b
    return ax0 < bx1 and bx0 < ax1 and ay0 < by1 and by0 < ay1


def _resolve_link_text(flat_nodes, page_1based, rect_td):
    parts = []
    for n in flat_nodes or []:
        if n.get("page") != page_1based or not n.get("bbox"):
            continue
        if _rect_overlaps(n["bbox"], rect_td):
            txt = (n.get("content") or "").strip()
            if txt:
                parts.append(txt)
    return " ".join(parts) if parts else None


def wrap_link_annotations(pdf, flat_nodes=None):
    """Wrap every /Link annotation not already reachable from a /Link
    struct element in a fresh /Link StructElem + /OBJR pair (ISO 32000-1
    14.8.4.4.2) -- the veraPDF 7.18.5 fix ("A Link annotation is an
    Artifact or is nested within null tag instead of Link"). Placement:
    appended to the struct element that already owns the surrounding
    content on that page (the last leaf struct element in document
    order), falling back to the page's top-level struct ancestor when the
    page has no leaves yet. Wires the annotation's /StructParent and the
    StructTreeRoot's /ParentTree so the reverse mapping resolves both
    ways -- a no-op (returns all-zero) when the doc has no
    /StructTreeRoot or no flat /ParentTree/Nums (nested number-tree
    /Kids are unsupported, same limitation as _page_parenttree_array).

    Where a wrapped annotation has no /Contents, best-effort resolves one
    from flat_nodes (the ODL JSON manifest -- real document text,
    bbox-matched against the annotation's /Rect) rather than inventing
    text (clause 7.18.1); a miss is left alone and counted, never
    fabricated. Returns {"wrapped", "contents_set", "contents_missing"}."""
    root = pdf.Root.get("/StructTreeRoot")
    if root is None:
        return {"wrapped": 0, "contents_set": 0, "contents_missing": 0}
    parent_tree = root.get("/ParentTree")
    nums = parent_tree.get("/Nums") if parent_tree is not None else None
    if nums is None:
        return {"wrapped": 0, "contents_set": 0, "contents_missing": 0}

    already_wrapped = _collect_objr_annot_keys(root)
    top_level = _children(root)
    next_key = _next_parent_tree_key(nums)

    wrapped = contents_set = contents_missing = 0
    for page_1based, page in enumerate(pdf.pages, start=1):
        annots = page.get("/Annots") or []
        link_annots = [a for a in annots if str(a.get("/Subtype", "")) == "/Link"]
        if not link_annots:
            continue
        slots = _page_leaf_slots(root, page.obj)
        target_parent = slots[-1][1] if slots else (top_level[0] if top_level else root)

        for annot in link_annots:
            if not getattr(annot, "is_indirect", False):
                continue  # ponytail: must be indirect to be OBJR-referenced
            if annot.objgen in already_wrapped:
                continue

            objr = pikepdf.Dictionary({
                "/Type": pikepdf.Name("/OBJR"), "/Obj": annot, "/Pg": page.obj,
            })
            link_elem = pdf.make_indirect(pikepdf.Dictionary({
                "/Type": pikepdf.Name("/StructElem"), "/S": pikepdf.Name("/Link"),
                "/Pg": page.obj, "/P": target_parent, "/K": objr,
            }))
            _append_struct_kid(target_parent, link_elem)

            annot["/StructParent"] = next_key
            nums.append(next_key)
            nums.append(link_elem)
            next_key += 1
            wrapped += 1

            if "/Contents" not in annot:
                rect = annot.get("/Rect")
                text = None
                if rect is not None and flat_nodes:
                    h = _page_height(page)
                    llx, lly, urx, ury = (float(v) for v in rect)
                    rect_td = [llx, h - ury, urx, h - lly]
                    text = _resolve_link_text(flat_nodes, page_1based, rect_td)
                if text:
                    annot["/Contents"] = pikepdf.String(text)
                    contents_set += 1
                else:
                    contents_missing += 1

    return {"wrapped": wrapped, "contents_set": contents_set, "contents_missing": contents_missing}


# ── Metadata (the verified 4-failure veraPDF fix) ───────────────────────────

def set_metadata(pdf, title, lang="en-US"):
    """pikepdf metadata pass: MarkInfo.Marked, ViewerPreferences.DisplayDocTitle,
    /Lang, docinfo Title, XMP dc:title + pdfuaid:part="1", /Tabs=/S per page.
    Verified live (2026-07-20): fixes the exact 4 veraPDF UA1 failures ODL's
    tagged output leaves behind -> COMPLIANT 106/106 on the reference fixture."""
    pdf.Root.MarkInfo = pikepdf.Dictionary(Marked=True)
    if "/ViewerPreferences" not in pdf.Root:
        pdf.Root.ViewerPreferences = pikepdf.Dictionary()
    pdf.Root.ViewerPreferences.DisplayDocTitle = True
    pdf.Root.Lang = lang
    with pdf.open_metadata() as meta:
        meta["dc:title"] = title
        meta["pdfuaid:part"] = "1"  # pikepdf XMP setter requires str, not int
    pdf.docinfo["/Title"] = title
    for page in pdf.pages:
        page.Tabs = pikepdf.Name.S


# ── Post-fix helpers (v7.63.1: STP 530 shakedown findings) ──────────────────

_MC_OPEN = {"BMC", "BDC"}
_PAINT_OPS = {"f", "f*", "F", "S", "s", "B", "B*", "b", "b*", "sh", "Do", "TJ", "Tj", "'", '"'}

_STANDARD_TYPES = {
    "Document", "Part", "Art", "Sect", "Div", "BlockQuote", "Caption", "TOC",
    "TOCI", "Index", "NonStruct", "Private", "P", "H", "H1", "H2", "H3", "H4",
    "H5", "H6", "L", "LI", "Lbl", "LBody", "Table", "TR", "TH", "TD", "THead",
    "TBody", "TFoot", "Span", "Quote", "Note", "Reference", "BibEntry", "Code",
    "Link", "Annot", "Ruby", "RB", "RT", "RP", "Warichu", "WT", "WP", "Figure",
    "Formula", "Form",
}


def ensure_role_map(pdf):
    """Map any non-standard structure type to /P via /RoleMap (veraPDF
    role-map rule). No-op when every type is standard. Returns mapped types."""
    root = pdf.Root.get("/StructTreeRoot")
    if root is None:
        return []
    nonstd = set()
    visited = set()

    def _walk(el):
        if not hasattr(el, "get"):
            return
        key = el.objgen if getattr(el, "is_indirect", False) else id(el)
        if key in visited:
            return
        visited.add(key)
        s = str(el.get("/S", ""))
        if s and s[1:] not in _STANDARD_TYPES:
            nonstd.add(s[1:])
        kids = el.get("/K")
        if kids is None:
            return
        for c in (kids if isinstance(kids, pikepdf.Array) else [kids]):
            _walk(c)

    for child in (root.get("/K") if isinstance(root.get("/K"), pikepdf.Array) else [root.get("/K")]):
        _walk(child)
    if not nonstd:
        return []
    rm = root.get("/RoleMap")
    if rm is None:
        root["/RoleMap"] = pikepdf.Dictionary()
        rm = root["/RoleMap"]
    mapped = []
    for t in sorted(nonstd):
        if f"/{t}" not in rm:
            rm[f"/{t}"] = pikepdf.Name("/P")  # ponytail: P is the safe default
            mapped.append(t)
    return mapped


def remove_circular_rolemap(pdf):
    """Delete every /RoleMap entry that is circular: a direct self-mapping
    (key == value, e.g. /Document -> /Document -- a standard structure type
    needs no RoleMap entry at all, so mapping it to itself is circular by
    definition) or a multi-hop cycle (A -> B -> A). veraPDF clause 7.1 "A
    circular mapping shall not exist" fails once per STRUCT ELEMENT
    INSTANCE that resolves through a circular chain, not once per RoleMap
    entry -- a handful of self-mapped entries can produce dozens of
    failedChecks on a heavily-linked document (verified live: 6 self-map
    entries -> 67 failedChecks on a document with many /Link elements).

    Walks each entry's mapping chain with a seen-set (per the caller's
    spec) and drops the entry whose value closes the loop; for a multi-hop
    cycle this removes every entry in the cycle (each cycle member is also
    walked as its own starting key), which is always safe -- a standard
    type needs no entry, and this never touches a legitimate
    non-standard-to-/P entry (e.g. the ones ensure_role_map() writes),
    since those chains terminate at "/P" (not itself a RoleMap key) rather
    than looping back. Deletes the whole /RoleMap when nothing is left in
    it. Returns the removed type names (no leading '/')."""
    root = pdf.Root.get("/StructTreeRoot")
    if root is None:
        return []
    rm = root.get("/RoleMap")
    if rm is None:
        return []

    to_remove = []
    for k in list(rm.keys()):
        seen = [k]
        cur = k
        while True:
            nxt = rm.get(cur)
            if nxt is None:
                break  # chain terminates at a type with no further mapping
            nxt = str(nxt)
            if nxt in seen:
                to_remove.append(cur)  # cur -> nxt is the edge that closes the loop
                break
            seen.append(nxt)
            cur = nxt

    to_remove = sorted(set(to_remove))
    for k in to_remove:
        if k in rm:
            del rm[k]
    if len(list(rm.keys())) == 0:
        del root["/RoleMap"]
    return [k[1:] if k.startswith("/") else k for k in to_remove]


def detect_unmarked_paint_pages(pdf):
    """{1-based page: count} of painting ops outside any marked content.
    These fail veraPDF 7.1 and need judgment: real figure vs decoration."""
    out = {}
    for pno, page in enumerate(pdf.pages):
        depth, count = 0, 0
        try:
            for instr in pikepdf.parse_content_stream(page):
                if isinstance(instr, pikepdf.ContentStreamInlineImage):
                    if depth == 0:
                        count += 1  # unmarked inline image is unmarked content too
                    continue
                op = str(instr.operator)
                if op in _MC_OPEN:
                    depth += 1
                elif op == "EMC":
                    depth = max(0, depth - 1)
                elif depth == 0 and op in _PAINT_OPS:
                    count += 1
        except Exception:
            out[pno + 1] = "parse-error"  # surfaced, not swallowed
            continue
        if count:
            out[pno + 1] = count
    return out


def _page_parenttree_array(pdf, page):
    sp = page.get("/StructParents")
    if sp is None:
        return None
    nums = pdf.Root.StructTreeRoot.get("/ParentTree", {}).get("/Nums")
    if nums is None:
        return None  # ponytail: nested /Kids number trees unsupported; flat only
    for i in range(0, len(nums), 2):
        if int(nums[i]) == int(sp):
            return nums[i + 1]
    return None


def wrap_page_vectors_as_figure(pdf, page_1based, alt):
    """Wrap every contiguous depth-0 unmarked-paint cluster on the page as
    tagged /Figure marked content. One struct elem with /Alt whose /K is the
    array of new MCIDs (one per cluster — a single MCID may not be reused
    across marked-content sequences). New MCIDs start above the page's max
    existing MCID (sparse-safe); ParentTree array is null-padded to match.
    Validates BEFORE mutating — no dangling struct elems on failure.
    Raises RuntimeError when the page has no flat ParentTree entry or no
    unmarked paint."""
    page = pdf.pages[page_1based - 1]
    arr = _page_parenttree_array(pdf, page)
    if arr is None:
        raise RuntimeError(f"page {page_1based}: no flat ParentTree entry")

    # Pass 1 (read-only): find clusters + max existing MCID.
    instrs = list(pikepdf.parse_content_stream(page))
    props_res = page.get("/Resources", {}).get("/Properties", {})
    max_mcid = -1
    clusters = []          # list of (start_idx, end_idx_exclusive)
    run_start, depth = None, 0
    run_has_paint = False

    def _mcid_of(operands):
        if len(operands) < 2:
            return None
        props = operands[1]
        if isinstance(props, pikepdf.Name):
            props = props_res.get(props)
        if hasattr(props, "get") and "/MCID" in props:
            return int(props.get("/MCID"))
        return None

    for i, instr in enumerate(instrs):
        op = str(instr.operator)
        if depth == 0 and op not in _MC_OPEN and op != "EMC":
            if run_start is None:
                run_start = i
            if op in _PAINT_OPS:
                run_has_paint = True
            continue
        if run_start is not None:
            if run_has_paint:
                clusters.append((run_start, i))
            run_start, run_has_paint = None, False
        if op == "BDC":
            m = _mcid_of(instr.operands)
            if m is not None:
                max_mcid = max(max_mcid, m)
        if op in _MC_OPEN:
            depth += 1
        elif op == "EMC":
            depth = max(0, depth - 1)
    if run_start is not None and run_has_paint:
        clusters.append((run_start, len(instrs)))
    if not clusters:
        raise RuntimeError(f"page {page_1based}: no unmarked paint to wrap")

    # Pass 2: mutate. MCIDs = max_existing+1 ... ; ParentTree null-padded.
    base = max(max_mcid + 1, len(arr))
    mcids = list(range(base, base + len(clusters)))
    doc_elem = pdf.Root.StructTreeRoot.K
    if isinstance(doc_elem, pikepdf.Array):
        doc_elem = doc_elem[0]
    fig = pdf.make_indirect(pikepdf.Dictionary({
        "/Type": pikepdf.Name("/StructElem"), "/S": pikepdf.Name("/Figure"),
        "/Pg": page.obj, "/Alt": pikepdf.String(alt), "/P": doc_elem,
        "/K": pikepdf.Array(mcids) if len(mcids) > 1 else mcids[0],
    }))
    kids = doc_elem.get("/K")
    if not isinstance(kids, pikepdf.Array):
        doc_elem["/K"] = pikepdf.Array([kids] if kids is not None else [])
        kids = doc_elem["/K"]
    kids.append(fig)
    null = pikepdf.Object.parse(b"null")
    while len(arr) < base:
        arr.append(null)  # sparse gap: no struct elem owns these MCIDs
    for _ in mcids:
        arr.append(fig)

    out, ci = [], 0
    cluster_bounds = dict((s, (e, m)) for (s, e), m in zip(clusters, mcids))
    i = 0
    while i < len(instrs):
        if i in cluster_bounds:
            end, mcid = cluster_bounds[i]
            out.append(pikepdf.ContentStreamInstruction(
                [pikepdf.Name("/Figure"), pikepdf.Dictionary({"/MCID": mcid})],
                pikepdf.Operator("BDC")))
            out.extend(instrs[i:end])
            out.append(pikepdf.ContentStreamInstruction([], pikepdf.Operator("EMC")))
            i = end
        else:
            out.append(instrs[i])
            i += 1
    page.Contents = pdf.make_stream(pikepdf.unparse_content_stream(out))
    return mcids


def _same_page(pg, page_obj):
    """True if struct-elem-inherited /Pg `pg` refers to `page_obj`."""
    if pg is None:
        return False
    try:
        return pg.objgen == page_obj.objgen
    except Exception:  # noqa: BLE001
        return pg is page_obj


def _demote_content_stream(pdf, page, mcid_set):
    """Rewrite `/Tag <</MCID n>> BDC ... EMC` to `/Artifact BMC ... EMC` for
    every n in mcid_set found on `page`. EMC operators are untouched — they
    carry no operands and always close the nearest open BDC/BMC regardless
    of kind, so no depth tracking is needed. Returns the subset of
    mcid_set actually found (and demoted); the page's content stream is
    left unmodified if none are found."""
    try:
        instrs = list(pikepdf.parse_content_stream(page))
    except Exception:  # noqa: BLE001
        return set()
    resources = page.get("/Resources") or {}
    props_res = resources.get("/Properties") if "/Properties" in resources else None

    def _mcid_of(operands):
        if len(operands) < 2:
            return None
        props = operands[1]
        if isinstance(props, pikepdf.Name):
            props = props_res.get(str(props)) if props_res is not None else None
        if hasattr(props, "get") and "/MCID" in props:
            try:
                return int(props["/MCID"])
            except (TypeError, ValueError):
                return None
        return None

    out, found = [], set()
    for instr in instrs:
        if isinstance(instr, pikepdf.ContentStreamInlineImage):
            out.append(instr)
            continue
        if str(instr.operator) == "BDC":
            mcid = _mcid_of(instr.operands)
            if mcid is not None and mcid in mcid_set:
                out.append(pikepdf.ContentStreamInstruction(
                    [pikepdf.Name("/Artifact")], pikepdf.Operator("BMC")))
                found.add(mcid)
                continue
        out.append(instr)

    if found:
        page.Contents = pdf.make_stream(pikepdf.unparse_content_stream(out))
    return found


def _remove_struct_kids(node, page_obj, mcid_set, pg=None, visited=None):
    """Recursively strip any /K entry on `node` (or its descendants) that
    resolves to an MCID in mcid_set on `page_obj` — a bare int or an MCR
    dict ({/MCID, no /S}), inheriting /Pg down the tree same as the reading-
    order walk. Removes the whole struct element from its parent's /K when
    doing so empties it (cascades up through now-empty grouping
    containers). Cycle-guarded via a seen-objgen set. Returns True if
    `node` itself ended up with no /K left (so the CALLER should also drop
    `node` from ITS parent's /K)."""
    if visited is None:
        visited = set()
    if not hasattr(node, "get"):
        return False
    try:
        key = node.objgen if getattr(node, "is_indirect", False) else id(node)
    except Exception:  # noqa: BLE001
        key = id(node)
    if key in visited:
        return False
    visited.add(key)

    pg = node.get("/Pg", pg)
    k = node.get("/K")
    if k is None:
        return False
    was_array = isinstance(k, pikepdf.Array)
    items = list(k) if was_array else [k]

    new_items, removed = [], False
    for item in items:
        if isinstance(item, int):
            if _same_page(pg, page_obj) and int(item) in mcid_set:
                removed = True
                continue
            new_items.append(item)
        elif hasattr(item, "get") and "/MCID" in item and "/S" not in item:
            item_pg = item.get("/Pg", pg)
            if _same_page(item_pg, page_obj) and int(item["/MCID"]) in mcid_set:
                removed = True
                continue
            new_items.append(item)
        elif hasattr(item, "get"):
            if _remove_struct_kids(item, page_obj, mcid_set, pg, visited):
                removed = True
                continue
            new_items.append(item)
        else:
            new_items.append(item)

    if not removed:
        return False
    if new_items:
        node["/K"] = pikepdf.Array(new_items) if was_array else new_items[0]
    else:
        del node["/K"]
    return "/K" not in node


def _null_parenttree_entries(pdf, page, mcid_set):
    arr = _page_parenttree_array(pdf, page)
    if arr is None:
        return
    null = pikepdf.Object.parse(b"null")
    for m in mcid_set:
        if 0 <= m < len(arr):
            arr[m] = null


def demote_mcids_to_artifact(pdf, targets):
    """Demote specific MCIDs to /Artifact — the leaked-artifact-status
    counterpart to artifact_page_unmarked (which wraps content that was
    never marked at all; this is the adjacent case, content that IS marked
    and must be demoted). `targets`: {page_1based: [mcid, ...]}.

    For each target MCID: rewrites the page's content stream so the
    marked-content sequence carrying it becomes /Artifact BMC..EMC instead
    of /Tag <</MCID n>> BDC..EMC, then removes the owning struct element
    from the tag tree and nulls its /ParentTree slot so it drops out of
    struct-tree (== screen-reader) reading order. A target MCID that
    doesn't exist on the page (already demoted, wrong page, typo) is a
    no-op for that entry, never a raise. Returns the count of marked-
    content sequences demoted."""
    root = pdf.Root.get("/StructTreeRoot")
    demoted = 0
    for page_key, mcids in (targets or {}).items():
        try:
            page_1based = int(page_key)
            page = pdf.pages[page_1based - 1]
        except (IndexError, TypeError, ValueError):
            continue
        mcid_set = {int(m) for m in (mcids or [])}
        if not mcid_set:
            continue
        found = _demote_content_stream(pdf, page, mcid_set)
        if not found:
            continue
        if root is not None:
            _remove_struct_kids(root, page.obj, found)
            _null_parenttree_entries(pdf, page, found)
        demoted += len(found)
    return demoted


def _leaf_mcids(k):
    """Flatten a struct element's /K value (bare int, /MCR-style dict, or an
    Array mixing either) into a plain list of MCIDs. Nested StructElem /OBJR
    entries (never present on a plain /Figure in practice) are ignored rather
    than raised on -- best-effort, matches the rest of this module's
    tolerance for atypical trees."""
    if k is None:
        return []
    items = k if isinstance(k, pikepdf.Array) else [k]
    out = []
    for item in items:
        if isinstance(item, int):
            out.append(int(item))
        elif hasattr(item, "get") and "/MCID" in item and "/S" not in item:
            try:
                out.append(int(item["/MCID"]))
            except (TypeError, ValueError):
                continue
    return out


def _page_key(page_obj):
    try:
        return page_obj.objgen if getattr(page_obj, "is_indirect", False) else id(page_obj)
    except Exception:  # noqa: BLE001
        return id(page_obj)


def _figure_page_map(pdf):
    """{page-object-key: 1-based page number} shared by demote_figures_to_artifact
    and list_figures/figures_image_info."""
    page_map = {}
    for pno, page in enumerate(pdf.pages):
        page_map[_page_key(page.obj)] = pno + 1
    return page_map


def demote_figures_to_artifact(pdf, figure_indices):
    """Generalizes the by-hand `{page: [mcid, ...]}` demote_mcids_to_artifact
    call: resolve each 0-based walk_figures() index to its /Pg (1-based page)
    and /K MCIDs, group by page, and demote them all in one
    demote_mcids_to_artifact call. Out-of-range indices, and figures with no
    resolvable page or no MCIDs, are skipped -- never raised. Returns the
    count of marked-content sequences demoted (same unit as
    demote_mcids_to_artifact)."""
    figures = walk_figures(pdf)
    page_map = _figure_page_map(pdf)

    targets: dict = {}
    for idx in figure_indices or []:
        try:
            i = int(idx)
        except (TypeError, ValueError):
            continue
        if not (0 <= i < len(figures)):
            continue
        fig = figures[i]
        pg = fig.get("/Pg")
        if pg is None:
            continue
        page_no = page_map.get(_page_key(pg))
        if page_no is None:
            continue
        mcids = _leaf_mcids(fig.get("/K"))
        if not mcids:
            continue
        targets.setdefault(page_no, []).extend(mcids)
    return demote_mcids_to_artifact(pdf, targets)


def artifact_page_unmarked(pdf, page_1based):
    """Wrap every depth-0 unmarked paint run on the page in /Artifact BMC..EMC.
    The decorative counterpart of wrap_page_vectors_as_figure — for slide
    backgrounds/chrome that should be hidden from AT, not described. Tagged
    content (inside BDC/EMC) is untouched by construction. Returns wrapped-run
    count (0 = nothing to do, page left unmodified)."""
    page = pdf.pages[page_1based - 1]
    try:
        instrs = list(pikepdf.parse_content_stream(page))
    except Exception as exc:
        raise RuntimeError(f"page {page_1based}: content stream parse failed: {exc}")
    out, run, depth, wrapped = [], [], 0, 0

    def _is_paint(instr):
        if isinstance(instr, pikepdf.ContentStreamInlineImage):
            return True
        return str(instr.operator) in _PAINT_OPS

    def flush():
        nonlocal wrapped
        if not run:
            return
        if any(_is_paint(i) for i in run):
            out.append(pikepdf.ContentStreamInstruction(
                [pikepdf.Name("/Artifact")], pikepdf.Operator("BMC")))
            out.extend(run)
            out.append(pikepdf.ContentStreamInstruction([], pikepdf.Operator("EMC")))
            wrapped += 1
        else:
            out.extend(run)
        run.clear()

    for instr in instrs:
        op = "" if isinstance(instr, pikepdf.ContentStreamInlineImage) else str(instr.operator)
        if depth == 0 and op not in _MC_OPEN and op != "EMC":
            run.append(instr)
            continue
        flush()
        out.append(instr)
        if op in _MC_OPEN:
            depth += 1
        elif op == "EMC":
            depth = max(0, depth - 1)
    flush()
    if wrapped:
        page.Contents = pdf.make_stream(pikepdf.unparse_content_stream(out))
    return wrapped


# ── Unreachable-content resolution (veraPDF 7.1, automatic pass) ────────────

_TEXT_SHOW_OPS = {"Tj", "TJ", "'", '"'}


def _page_reachable_mcids(root, page_obj):
    """{mcid} for every MCID on page_obj already reachable from the struct
    tree (a bare int /K entry or an MCR dict), inheriting /Pg the same way
    _remove_struct_kids() does."""
    reachable, visited = set(), set()

    def walk(node, pg):
        key = _node_key(node)
        if key in visited:
            return
        visited.add(key)
        node_pg = node.get("/Pg", pg) if hasattr(node, "get") else pg
        for item in _raw_kids(node):
            if isinstance(item, int):
                if _same_page(node_pg, page_obj):
                    reachable.add(int(item))
            elif hasattr(item, "get") and "/MCID" in item and "/S" not in item:
                item_pg = item.get("/Pg", node_pg)
                if _same_page(item_pg, page_obj):
                    reachable.add(int(item["/MCID"]))
            elif hasattr(item, "get") and "/S" in item:
                walk(item, node_pg)

    walk(root, None)
    return reachable


def _is_text_show(instr):
    return (not isinstance(instr, pikepdf.ContentStreamInlineImage)
            and str(instr.operator) in _TEXT_SHOW_OPS)


def _is_paint_instr(instr):
    if isinstance(instr, pikepdf.ContentStreamInlineImage):
        return True
    return str(instr.operator) in _PAINT_OPS


def resolve_unreachable_content(pdf):
    """Automatic (non-judgment) veraPDF 7.1 pass ("Content shall be marked
    as Artifact or tagged as real content"). Two distinct defects, both
    resolved here:

      - UNMARKED: a depth-0 painting run never wrapped in any BDC/BMC at
        all (the "bare contentItem" veraPDF context).
      - ORPHANED: content already wrapped in BDC/MCID, but that MCID is
        not reachable from the struct tree (the "content[n]{mcid:x}"
        veraPDF context) -- a distinct case from unmarked, and one
        artifact_page_unmarked()/wrap_page_vectors_as_figure() never look
        for, since both only examine depth-0 spans.

    Each resolved span is classified by what it actually contains, never
    by guesswork:

      - genuine text (contains a text-showing operator) is ATTACHED to
        the struct tree -- an unmarked run gets a fresh /P leaf + new
        MCID; an orphaned MCID just gets a /P leaf pointed at the MCID it
        already has. Either way the new leaf is appended to the struct
        element that already owns the surrounding content on that page
        (falling back to the page's top-level struct ancestor), so the
        text rejoins reading order instead of disappearing from it.
      - pure paint with no text (vector fills/strokes/images) is
        ARTIFACTed instead: an unmarked run is wrapped in /Artifact
        BMC..EMC (matching artifact_page_unmarked()'s existing
        decorative-content behavior); an orphaned MCID's BDC is rewritten
        to /Artifact BMC in place and its ParentTree slot is nulled.

    Content already reachable from the struct tree is never touched.
    No text is ever invented -- the text/non-text call is made purely
    from what operators are actually present in that span. Returns
    {"attached", "artifacted"} (marked-content-sequence counts, doc-wide)."""
    root = pdf.Root.get("/StructTreeRoot")
    if root is None:
        return {"attached": 0, "artifacted": 0}
    parent_tree = root.get("/ParentTree")
    if parent_tree is None or parent_tree.get("/Nums") is None:
        return {"attached": 0, "artifacted": 0}  # ponytail: flat ParentTree only

    attached = artifacted = 0
    for page in pdf.pages:
        arr = _page_parenttree_array(pdf, page)
        if arr is None:
            continue
        reachable = _page_reachable_mcids(root, page.obj)

        try:
            instrs = list(pikepdf.parse_content_stream(page))
        except Exception:  # noqa: BLE001
            continue
        props_res = page.get("/Resources", {}).get("/Properties", {})

        def _mcid_of(operands):
            if len(operands) < 2:
                return None
            props = operands[1]
            if isinstance(props, pikepdf.Name):
                props = props_res.get(props)
            if hasattr(props, "get") and "/MCID" in props:
                return int(props.get("/MCID"))
            return None

        # Pass 1 (read-only): find unmarked runs + orphaned-mcid spans.
        segments = []  # (start, end_exclusive, "unmarked"|"orphan", mcid, has_text)
        max_mcid, depth = -1, 0
        run_start, run_has_text = None, False
        orphan_start, orphan_mcid, orphan_has_text = None, None, False

        for i, instr in enumerate(instrs):
            op = "" if isinstance(instr, pikepdf.ContentStreamInlineImage) else str(instr.operator)
            if orphan_start is not None and _is_text_show(instr):
                orphan_has_text = True
            if depth == 0 and op not in _MC_OPEN and op != "EMC":
                if run_start is None:
                    run_start, run_has_text = i, False
                if _is_text_show(instr):
                    run_has_text = True
                continue
            if run_start is not None:
                segments.append((run_start, i, "unmarked", None, run_has_text))
                run_start = None
            if op == "BDC":
                mcid = _mcid_of(instr.operands)
                if mcid is not None:
                    max_mcid = max(max_mcid, mcid)
                    if depth == 0 and mcid not in reachable:
                        orphan_start, orphan_mcid, orphan_has_text = i, mcid, False
            if op in _MC_OPEN:
                depth += 1
            elif op == "EMC":
                depth = max(0, depth - 1)
                if depth == 0 and orphan_start is not None:
                    segments.append((orphan_start, i + 1, "orphan", orphan_mcid, orphan_has_text))
                    orphan_start, orphan_mcid = None, None
        if run_start is not None:
            segments.append((run_start, len(instrs), "unmarked", None, run_has_text))

        def _has_paint(seg):
            s, e, _kind, _mcid, has_text = seg
            return has_text or any(_is_paint_instr(instrs[j]) for j in range(s, e))
        segments = [s for s in segments if _has_paint(s)]
        if not segments:
            continue

        # Pass 2: mutate. New MCIDs (for attached "unmarked" runs) start
        # above both the page's max existing MCID and its ParentTree
        # length -- same sparse-safe rule as wrap_page_vectors_as_figure.
        next_mcid = max(max_mcid + 1, len(arr))
        slots = None  # lazy: only walked if an attach actually happens
        null = pikepdf.Object.parse(b"null")

        def _target_parent():
            nonlocal slots
            if slots is None:
                slots = _page_leaf_slots(root, page.obj)
            if slots:
                return slots[-1][1]
            top = _children(root)
            return top[0] if top else root

        def _attach(mcid, k_value):
            parent = _target_parent()
            elem = pdf.make_indirect(pikepdf.Dictionary({
                "/Type": pikepdf.Name("/StructElem"), "/S": pikepdf.Name("/P"),
                "/Pg": page.obj, "/P": parent, "/K": k_value,
            }))
            _append_struct_kid(parent, elem)
            slots.append((elem, parent))
            while len(arr) <= mcid:
                arr.append(null)
            arr[mcid] = elem

        out, i = [], 0
        seg_by_start = {s: (e, kind, mcid, has_text) for s, e, kind, mcid, has_text in segments}
        while i < len(instrs):
            if i in seg_by_start:
                end, kind, mcid, has_text = seg_by_start[i]
                if kind == "unmarked":
                    if has_text:
                        new_mcid = next_mcid
                        next_mcid += 1
                        _attach(new_mcid, new_mcid)
                        out.append(pikepdf.ContentStreamInstruction(
                            [pikepdf.Name("/P"), pikepdf.Dictionary({"/MCID": new_mcid})],
                            pikepdf.Operator("BDC")))
                        out.extend(instrs[i:end])
                        out.append(pikepdf.ContentStreamInstruction([], pikepdf.Operator("EMC")))
                        attached += 1
                    else:
                        out.append(pikepdf.ContentStreamInstruction(
                            [pikepdf.Name("/Artifact")], pikepdf.Operator("BMC")))
                        out.extend(instrs[i:end])
                        out.append(pikepdf.ContentStreamInstruction([], pikepdf.Operator("EMC")))
                        artifacted += 1
                else:  # orphan -- already BDC/MCID-tagged; only reachability changes
                    if has_text:
                        _attach(mcid, mcid)
                        out.extend(instrs[i:end])
                        attached += 1
                    else:
                        rewritten = list(instrs[i:end])
                        rewritten[0] = pikepdf.ContentStreamInstruction(
                            [pikepdf.Name("/Artifact")], pikepdf.Operator("BMC"))
                        out.extend(rewritten)
                        if 0 <= mcid < len(arr):
                            arr[mcid] = null
                        artifacted += 1
                i = end
            else:
                out.append(instrs[i])
                i += 1
        page.Contents = pdf.make_stream(pikepdf.unparse_content_stream(out))

    return {"attached": attached, "artifacted": artifacted}


# ── Figure alt-text placeholder detection ───────────────────────────────────
#
# veraPDF clause 7.3 is satisfied by an /Alt entry EXISTING; it never checks
# whether the text is useful. A file shipped live (2026-07-28) where
# opendataloader-pdf's own default alt -- "image 1", "image 2", ..., one per
# /Figure -- passed veraPDF 106/106 while a screen reader user would hear
# "image 7" for a figure that carried the argument of an essay about visual
# rhetoric. Worse, those defaults OVERWROTE real printed captions already in
# the document. list_figures() surfaces both facts (is_placeholder + the
# printed caption, when one exists) so remediate_pdf.py can gate on it and an
# agent/human can supply real text via judgment.alt_text -- this module never
# applies a caption automatically (manifest-for-agent, same doctrine as
# pdf_table_weave.py's opt-in table conversions and audit_image_alt.py).

_PLACEHOLDER_ALT_RE = re.compile(
    r"^(?:image|figure|picture|graphic|img|untitled)[\s\-_:.]*\d*[\s\-_:.,;]*$",
    re.IGNORECASE,
)

_CAPTION_START_RE = re.compile(r"^(?:figure|fig\.|table)\s+\d+\.", re.IGNORECASE)
# Line-break hyphen -- either an ordinary "-" or the Unicode soft hyphen
# U+00AD (real-world PDF text extraction, e.g. PyMuPDF on this module's
# reference fixture, emits U+00AD at an author's line-wrap point, not
# ASCII "-"). Either way the defining feature is the newline immediately
# after it: a genuine mid-line compound like "well-known" (ASCII hyphen,
# no newline) is never touched.
_HYPHEN_BREAK_RE = re.compile(r"(\w)[\-­]\s*\n\s*(\w)")
_CAPTION_WS_RE = re.compile(r"\s+")


def is_placeholder_alt(alt):
    """True when `alt` is empty/whitespace, or is ENTIRELY a generic
    placeholder (image/figure/picture/graphic/img/untitled, optionally
    followed by digits/punctuation and nothing else) -- the exact shape an
    auto-tagger leaves behind ("image 1", "image 2", ...) that satisfies
    veraPDF clause 7.3 (an /Alt entry must EXIST) while conveying nothing
    to a screen reader. Deliberately conservative: the WHOLE string must
    match, so "image of a menu with three columns" -- a real description
    that merely starts with a placeholder word -- is never flagged."""
    text = (alt or "").strip()
    if not text:
        return True
    return _PLACEHOLDER_ALT_RE.match(text) is not None


def _normalize_caption_text(text):
    """De-hyphenate a line-break hyphen ("rang-\\ning" -> "ranging" -- only
    when an actual newline sits inside the break, so a genuine mid-line
    compound like "well-known" is left untouched) then collapse all
    remaining whitespace (including any other line breaks) to single
    spaces."""
    text = _HYPHEN_BREAK_RE.sub(r"\1\2", text)
    return _CAPTION_WS_RE.sub(" ", text).strip()


def _page_captions(fitz_page):
    """[(y0, normalized_text), ...], sorted top-to-bottom, for every text
    block on `fitz_page` that begins "Figure <n>." / "Fig. <n>." /
    "Table <n>." (case-insensitive) after normalization."""
    out = []
    for block in fitz_page.get_text("blocks"):
        text = _normalize_caption_text(block[4])
        if _CAPTION_START_RE.match(text):
            out.append((block[1], text))
    out.sort(key=lambda c: c[0])
    return out


def _fill_caption_candidates(fitz_doc, records):
    """Mutates `records` in place. For every page, associate its figures
    (already in document/reading order -- walk_figures is pre-order DFS)
    with its printed captions (sorted top-to-bottom) by position, ONLY
    when the two counts match exactly. Any other page (no caption blocks
    found, or a count mismatch -- e.g. a cover-thumbnail figure with no
    caption at all sharing a page with nothing else) is left at the
    caller's "" default rather than guessing: a wrong caption becomes
    wrong alt text if an agent trusts it, which is worse than surfacing
    nothing (see module docstring above)."""
    by_page = {}
    for rec in records:
        if rec["page"]:
            by_page.setdefault(rec["page"], []).append(rec)
    for page_no, page_records in by_page.items():
        if not (1 <= page_no <= fitz_doc.page_count):
            continue
        captions = _page_captions(fitz_doc[page_no - 1])
        if not captions or len(captions) != len(page_records):
            continue
        for rec, (_y, text) in zip(page_records, captions):
            rec["caption_candidate"] = text


# ── Figure context (SCRIPT half of figure-alt: evidence, never a verdict) ───
#
# figures_image_info / _document_events / _ancestor_has back list_figures'
# extended context fields (docs/plan-figure-alt.md §2) AND figure_alt.py's
# PNG cropping. Every field here is a datum for the AI to weigh -- including
# width/height, which look like an obvious decorative-size signal but are
# explicitly NOT a decider (a 14x14 image can be a status icon; see the
# module docstring's placeholder-alt motivation for the parallel mistake
# a size/position heuristic would make). This module never concludes
# decorative/content on its own.

def _page_mcid_images(page):
    """{mcid: [image_xobject, ...]} for every Image XObject placed (via a
    `Do` operator) directly inside a marked-content sequence on `page`,
    keyed by the innermost enclosing MCID -- mirrors mcid_text._extract's
    BDC/BMC/EMC depth tracking but records `Do` image placements instead of
    text-show operators. Does not descend into Form XObjects (same
    documented limitation as mcid_text.py). Never raises: a parse failure
    yields {}."""
    resources = page.get("/Resources") or {}
    xobj_res = resources.get("/XObject") if "/XObject" in resources else None
    if xobj_res is None:
        return {}
    props_res = resources.get("/Properties") if "/Properties" in resources else None

    def _mcid_of(operands):
        if len(operands) < 2:
            return None
        props = operands[1]
        if isinstance(props, pikepdf.Name):
            props = props_res.get(str(props)) if props_res is not None else None
        if hasattr(props, "get") and "/MCID" in props:
            try:
                return int(props["/MCID"])
            except (TypeError, ValueError):
                return None
        return None

    try:
        instrs = list(pikepdf.parse_content_stream(page))
    except Exception:  # noqa: BLE001
        return {}

    out: dict = {}
    stack, cur = [], None
    for instr in instrs:
        if isinstance(instr, pikepdf.ContentStreamInlineImage):
            continue
        op = str(instr.operator)
        if op == "BDC":
            stack.append(cur)
            mcid = _mcid_of(instr.operands)
            if mcid is not None:
                cur = mcid
            continue
        if op == "BMC":
            stack.append(cur)
            continue
        if op == "EMC":
            cur = stack.pop() if stack else None
            continue
        if op == "Do" and cur is not None and instr.operands:
            name = str(instr.operands[0])
            xobj = xobj_res.get(name) if name in xobj_res else None
            if xobj is not None and str(xobj.get("/Subtype", "")) == "/Image":
                out.setdefault(cur, []).append(xobj)
    return out


def figures_image_info(pdf):
    """[{"objid": int|None, "width": int|None, "height": int|None} | None, ...]
    aligned with walk_figures(pdf) order. For each /Figure, resolves the
    first Image XObject placed within any of its MCIDs on its page (first
    MCID with an image wins when a figure spans more than one). None when
    the figure has no resolvable page, no MCIDs, or no image (e.g. a
    vector-wrapped figure -- see wrap_page_vectors_as_figure). "objid" is the
    PDF object number (pikepdf .objgen[0]) -- the same numbering PyMuPDF's
    `page.get_image_rects(xref)` expects when both libraries open the same
    file, which is how figure_alt.py locates the crop rect for its PNG."""
    figures = walk_figures(pdf)
    page_map = _figure_page_map(pdf)
    page_images_cache: dict = {}

    out = []
    for fig in figures:
        pg = fig.get("/Pg")
        mcids = _leaf_mcids(fig.get("/K"))
        info = None
        if pg is not None and mcids:
            pkey = _page_key(pg)
            page_no = page_map.get(pkey)
            if page_no is not None:
                if pkey not in page_images_cache:
                    page_images_cache[pkey] = _page_mcid_images(pdf.pages[page_no - 1])
                page_images = page_images_cache[pkey]
                imgs = []
                for m in mcids:
                    imgs.extend(page_images.get(m, []))
                if imgs:
                    xobj = imgs[0]
                    try:
                        width = int(xobj["/Width"]) if "/Width" in xobj else None
                    except (TypeError, ValueError):
                        width = None
                    try:
                        height = int(xobj["/Height"]) if "/Height" in xobj else None
                    except (TypeError, ValueError):
                        height = None
                    objid = None
                    try:
                        if getattr(xobj, "is_indirect", False):
                            objid = xobj.objgen[0]
                    except Exception:  # noqa: BLE001
                        objid = None
                    info = {"objid": objid, "width": width, "height": height}
        out.append(info)
    return out


def _ancestor_has(node, matcher, max_depth=64):
    """True if any ancestor reached by following /P from `node` has an /S
    value matcher(s) accepts. Stops at the first node with no /P (or a /P
    that no longer resolves to a dict), and is cycle-guarded -- a document
    that never wires /P at all (some hand-built trees don't) simply yields
    False rather than raising."""
    seen, depth = set(), 0
    cur = node.get("/P") if hasattr(node, "get") else None
    while cur is not None and hasattr(cur, "get") and depth < max_depth:
        key = cur.objgen if getattr(cur, "is_indirect", False) else id(cur)
        if key in seen:
            break
        seen.add(key)
        if matcher(str(cur.get("/S", ""))):
            return True
        cur = cur.get("/P")
        depth += 1
    return False


def _document_events(pdf):
    """[("text", str) | ("figure", node_key), ...] from a single depth-first
    walk of the struct tree in document order -- the same traversal
    reading_order.py's struct_order_runs() performs, except this one also
    emits a ("figure", key) marker when it reaches a /Figure element instead
    of only ever emitting non-empty text runs. That interleaving is what
    lets a figure's neighboring text (list_figures' "surrounding_text") and
    document position ("reading_order_index") be read straight off this one
    list, without a second content-stream pass. A /Figure's own /K entries
    are MCIDs for its image, not text, so this never descends into them."""
    root = pdf.Root.get("/StructTreeRoot")
    if root is None:
        return []

    page_text_cache: dict = {}

    def page_text(pg):
        if pg is None:
            return {}
        try:
            key = pg.objgen
        except Exception:  # noqa: BLE001
            return {}
        if key not in page_text_cache:
            page_text_cache[key] = mcid_text.page_mcid_text(pg)
        return page_text_cache[key]

    events: list = []
    seen: set = set()

    def emit_text(pg, mcid):
        text = re.sub(r"\s+", " ", page_text(pg).get(mcid, "")).strip()
        if text:
            events.append(("text", text))

    def walk(node, pg):
        if node is None:
            return
        if isinstance(node, pikepdf.Array):
            for item in node:
                walk(item, pg)
            return
        if not hasattr(node, "get"):
            return
        key = node.objgen if getattr(node, "is_indirect", False) else id(node)
        if key in seen:
            return
        seen.add(key)
        pg = node.get("/Pg", pg)
        if str(node.get("/S", "")) == "/Figure":
            events.append(("figure", key))
            return
        k = node.get("/K")
        if k is None:
            return
        items = k if isinstance(k, pikepdf.Array) else [k]
        for item in items:
            if isinstance(item, int):
                emit_text(pg, int(item))
            elif isinstance(item, pikepdf.Dictionary):
                if "/MCID" in item and "/S" not in item:
                    emit_text(item.get("/Pg", pg), int(item["/MCID"]))
                else:
                    walk(item, pg)
            elif isinstance(item, pikepdf.Array):
                walk(item, pg)

    for child in _children(root):
        walk(child, None)
    return events


def _figure_context(events, event_idx, chars=200):
    """{"before", "after"} text (each up to `chars` chars, taken from the
    near side) around events[event_idx] -- {"before": "", "after": ""} when
    event_idx is None (figure not found in the document-order walk, e.g. an
    untagged/root-less tree)."""
    if event_idx is None:
        return {"before": "", "after": ""}
    before = " ".join(t for kind, t in events[:event_idx] if kind == "text")
    after = " ".join(t for kind, t in events[event_idx + 1:] if kind == "text")
    return {"before": before[-chars:], "after": after[:chars]}


def list_figures(pdf, path=None):
    """Agent-facing manifest of every /Figure struct element, document
    order -- the placeholder-alt gate's evidence source (see
    remediate_pdf.py) AND figure_alt.py's build-manifest context. Each
    record: {"index", "page", "alt", "is_placeholder", "caption_candidate",
    "width", "height", "surrounding_text", "reading_order_index",
    "repeat_count", "in_link", "in_heading", "mcids"}.

    "alt" is the current /Alt, "" when absent. "is_placeholder" flags the
    exact defect an auto-tagger leaves behind -- see is_placeholder_alt's
    docstring for the (conservative) rule.

    "caption_candidate" is a printed caption found near the figure via
    PyMuPDF text blocks -- requires `path` (pikepdf has no filesystem path
    of its own to hand PyMuPDF); "" when `path` is None, when PyMuPDF/the
    path is unavailable, or when this figure's page has no unambiguous
    figure-count-equals-caption-count match (see _fill_caption_candidates).
    NEVER auto-applied to /Alt -- the agent or human decides which caption
    (if any) becomes the real alt text, via remediate_pdf.py's
    judgment.alt_text.

    New context fields (docs/plan-figure-alt.md §2 -- evidence only, no
    field here is ever auto-classified into a decorative/content verdict):
      "width"/"height": the placed Image XObject's pixel dimensions, or None
        when unresolvable (see figures_image_info) -- a datum, NOT a decider.
      "surrounding_text": {"before", "after"} struct-order text runs (up to
        ~200 chars each) immediately around this figure (see _figure_context).
      "reading_order_index": this figure's position among ALL struct leaf
        elements (text runs and figures alike) in document order, or None
        if the figure isn't reachable from the walk that also yields
        surrounding_text (e.g. no /StructTreeRoot).
      "repeat_count": how many figures in the WHOLE document reference the
        same Image XObject as this one (repetition/branding signal for the
        AI to weigh) -- 1 when this figure's image can't be resolved (no
        evidence of repetition either way, not "definitely unique").
      "in_link"/"in_heading": whether an ancestor struct element (via /P) is
        a /Link or an /Hn heading.
      "mcids": this figure's own MCIDs (page-content-stream identifiers),
        for demote_figures_to_artifact / set_alts targeting."""
    figures = walk_figures(pdf)
    page_map = _figure_page_map(pdf)
    image_info = figures_image_info(pdf)
    events = _document_events(pdf)
    event_idx_by_key = {key: i for i, (kind, key) in enumerate(events) if kind == "figure"}
    image_key_counts = Counter(
        info["objid"] for info in image_info if info and info.get("objid") is not None
    )

    records = []
    for i, fig in enumerate(figures):
        pg = fig.get("/Pg")
        page_no = None
        if pg is not None:
            page_no = page_map.get(_page_key(pg))
        alt = str(fig.get("/Alt")) if "/Alt" in fig else ""
        fig_key = fig.objgen if getattr(fig, "is_indirect", False) else id(fig)
        info = image_info[i]
        objid = info.get("objid") if info else None
        records.append({
            "index": i, "page": page_no, "alt": alt,
            "is_placeholder": is_placeholder_alt(alt), "caption_candidate": "",
            "width": info.get("width") if info else None,
            "height": info.get("height") if info else None,
            "surrounding_text": _figure_context(events, event_idx_by_key.get(fig_key)),
            "reading_order_index": event_idx_by_key.get(fig_key),
            "repeat_count": image_key_counts[objid] if objid is not None else 1,
            "in_link": _ancestor_has(fig, lambda s: s == "/Link"),
            "in_heading": _ancestor_has(fig, lambda s: _H_RE.match(s) is not None),
            "mcids": _leaf_mcids(fig.get("/K")),
        })

    if path is not None:
        fitz_doc = None
        try:
            import fitz  # PyMuPDF
            fitz_doc = fitz.open(path)
        except Exception:  # noqa: BLE001 - best-effort; missing/bad path -> no captions
            fitz_doc = None
        if fitz_doc is not None:
            try:
                _fill_caption_candidates(fitz_doc, records)
            finally:
                fitz_doc.close()
    return records


def main():
    parser = argparse.ArgumentParser(description="PDF tag-tree editor (E2 library)")
    parser.add_argument("--list-figures", metavar="FILE",
                         help="print list_figures() as JSON for FILE")
    args = parser.parse_args()

    if args.list_figures:
        pdf = pikepdf.open(args.list_figures)
        try:
            print(json.dumps(list_figures(pdf, path=args.list_figures), indent=2))
        finally:
            pdf.close()
        return

    print("ERROR: nothing to do -- pass --list-figures FILE", file=sys.stderr)
    sys.exit(2)


if __name__ == "__main__":
    main()
