#!/usr/bin/env python3
"""Table semantics (file-remediation Stage 2, Engine E2 library).

# ponytail: MCID re-parenting into real Table/TR/TH/TD topology is still
# deferred (plan §3, AC6) — opendataloader-pdf already builds real Table/TR/TD
# topology (either /Table -> /TR -> /TD directly, or the PDF-spec-standard
# row-grouped /Table -> /THead|/TBody|/TFoot -> /TR -> /TD — both shapes are
# common in the wild and handled uniformly via _table_rows()), it just never
# promotes a /TD to /TH. weave_tables() below stays the flagged-incomplete
# fallback for callers that don't supply table_headers judgment.
# list_tables()/promote_headers() close the header half of the gap:
# judgment-driven /TD -> /TH promotion with /Scope, which is what Anthology
# Ally's "tables missing headers" finding actually checks for.

Header promotion is opt-in ONLY (per table, via judgment `table_headers`) —
never auto-promoted. A table with no real header row/column (e.g. a plain
3-column word list) would have a screen reader announce ordinary data as
column/row headers if headers were invented, which actively misleads. The
agent must look at the table (list_tables()'s first_row_text, or a page
render) and decide.

The header row defaults to the first cell-bearing /TR, not raw /TR index 0
— real opendataloader-tagged PDFs frequently sweep a cell-less junk row
(e.g. a /Link struct elem) into TR0 ahead of the genuine header row.
`header_row_index` on a spec overrides this resolution explicitly.

Layout-table -> list conversion (convert_tables_to_lists) is the other half
of the table-semantics gap: opendataloader-pdf emits /Table for ANY visual
grid, including ones with no real row/column relationship (e.g. a
multi-column word list authored to read down each column). Tagging that
/Table gives a screen reader meaningless row/column semantics AND forces
row-major reading order, scrambling an author's column-major intent.
convert_tables_to_lists rewrites such a table to /L -> /LI -> /LBody in the
requested order (row-major default, column-major opt-in). Opt-in ONLY, same
rule as header promotion: a table absent from `specs` is never converted,
because converting a genuine DATA table would destroy real row/column
semantics. The agent must inspect the grid (list_tables()'s
first_row_text, or a page render) before listing it. Complements
promote_headers: real data tables get /TH headers, layout tables become
lists.

Mis-detected-table -> figure conversion (convert_tables_to_figures) closes a
third gap: opendataloader-pdf tags ANY visual grid as /Table, including a
chart/plot that is a grid of marks (e.g. a 2x2 panel of regression residual
plots) with no table semantics at all. Tagging that /Table gives a screen
reader meaningless row/column semantics for what is actually a figure. This
rewrites the /Table struct elem to /Figure with a required /Alt, reusing the
table's own marked content (no re-render). Opt-in ONLY, same rule as the
other two conversions, and /Alt is REQUIRED (a /Figure with no /Alt fails
veraPDF 7.3) — the agent must look at the render and describe it. Never
apply to a genuine data table. Completes the trio with promote_headers (real
data tables) and convert_tables_to_lists (layout lists).

Table-fragment merging (merge_tables) closes a fourth gap: a tagger
routinely splits ONE logical table into several sibling /Table structure
elements (a page break mid-table, or the tagger simply losing the thread
partway through). The header row lands in the first fragment; every later
fragment is headerless — and promote_headers correctly refuses to invent
headers on a table that structurally has none (see above), so a genuinely
split table could never be remediated at all until this existed.
merge_tables re-parents each donor fragment's /TR rows onto a chosen
survivor fragment — moving the existing struct elems, never rebuilding
them, same discipline as convert_tables_to_lists — then removes the
emptied donor /Table. Opt-in, same rule as the other three: a fragment
absent from `specs` is never touched. And because merging REMOVES /Table
elements it carries the same index-shift hazard as convert_tables_to_lists
/ convert_tables_to_figures — merge first, re-run list_tables(), then
promote headers or run another conversion on the surviving indices in a
second pass.
"""

from __future__ import annotations

import argparse
import json
import re
import sys
from pathlib import Path

import pikepdf

import mcid_text
import pdf_tree_edit

_WS_RE = re.compile(r"\s+")
_PAGE_CONTEXT_PAGE_CHARS = 400
_PAGE_CONTEXT_SAME_PAGE_CHARS = 300


def _normalize_ws(text: str) -> str:
    """Collapse any whitespace run to a single space, strip ends."""
    return _WS_RE.sub(" ", text).strip()


def _truncate_head(text: str, limit: int) -> str:
    """First `limit` chars of `text`, cut back to the nearest word boundary
    (never mid-word)."""
    if len(text) <= limit:
        return text
    cut = text[:limit]
    if " " in cut:
        cut = cut.rsplit(" ", 1)[0]
    return cut


def _truncate_tail(text: str, limit: int) -> str:
    """Last `limit` chars of `text`, cut forward to the nearest word
    boundary (never mid-word)."""
    if len(text) <= limit:
        return text
    cut = text[-limit:]
    if " " in cut:
        cut = cut.split(" ", 1)[1]
    return cut


def _empty_page_context() -> dict:
    return {"prev_page": "", "next_page": "", "same_page_before": "", "same_page_after": ""}


def _locate_cell_rect(fitz_page, cell_text: str):
    """Bounding rect of `cell_text` on `fitz_page` via search_for(), or None
    when the text is empty or the search doesn't resolve to exactly one
    match. A cell text over 200 chars carries a trailing '…' (see
    _truncate) that never appears verbatim in the page content — stripped
    before searching. Ambiguous (0 or >1 hits) intentionally returns None
    rather than guessing — a wrong same-page split is worse than none (see
    list_tables docstring)."""
    text = (cell_text or "").rstrip("…").strip()
    if not text:
        return None
    try:
        rects = fitz_page.search_for(text)
    except Exception:  # noqa: BLE001 - best-effort, never raises
        return None
    if len(rects) != 1:
        return None
    return rects[0]


def _page_context(fitz_doc, page_no, cell_text: str) -> dict:
    """Surrounding-page context for a table resolved to 1-based `page_no`
    in `fitz_doc` (opened once by the caller). See list_tables docstring
    for field semantics. `fitz_doc` None or `page_no` None -> all-empty."""
    ctx = _empty_page_context()
    if fitz_doc is None or page_no is None:
        return ctx
    n = len(fitz_doc)
    if 1 < page_no <= n:
        prev_text = _normalize_ws(fitz_doc[page_no - 2].get_text())
        ctx["prev_page"] = _truncate_head(prev_text, _PAGE_CONTEXT_PAGE_CHARS)
    if 0 < page_no < n:
        next_text = _normalize_ws(fitz_doc[page_no].get_text())
        ctx["next_page"] = _truncate_head(next_text, _PAGE_CONTEXT_PAGE_CHARS)
    if 1 <= page_no <= n:
        fitz_page = fitz_doc[page_no - 1]
        rect = _locate_cell_rect(fitz_page, cell_text)
        if rect is not None:
            import fitz  # PyMuPDF — local import mirrors judgment_eval.py's usage
            w, h = fitz_page.rect.width, fitz_page.rect.height
            before = fitz_page.get_text("text", clip=fitz.Rect(0, 0, w, rect.y0))
            after = fitz_page.get_text("text", clip=fitz.Rect(0, rect.y1, w, h))
            ctx["same_page_before"] = _truncate_tail(
                _normalize_ws(before), _PAGE_CONTEXT_SAME_PAGE_CHARS)
            ctx["same_page_after"] = _truncate_head(
                _normalize_ws(after), _PAGE_CONTEXT_SAME_PAGE_CHARS)
    return ctx


def weave_tables(pdf, table_specs) -> dict:
    """Record each judgment table_spec as flagged-incomplete. No tree edits.
    table_specs: [{page, bbox, ...}] (judgment.json "tables" entries).
    Returns {"woven": 0, "flagged": [{page, bbox, reason}]}."""
    flagged = [
        {
            "page": spec.get("page"),
            "bbox": spec.get("bbox"),
            "reason": "table semantics incomplete — cells remain /P (fallback default, plan AC6)",
        }
        for spec in (table_specs or [])
    ]
    return {"woven": 0, "flagged": flagged}


# ── Header promotion ────────────────────────────────────────────────────────

def _kids(node):
    """Direct StructElem children of a node's /K, always as a list."""
    k = node.get("/K") if hasattr(node, "get") else None
    if k is None:
        return []
    items = k if isinstance(k, pikepdf.Array) else [k]
    return [c for c in items if hasattr(c, "get") and "/S" in c]


def _table_rows(table):
    """A table's /TR elements in document order. The PDF spec's standard
    tagged-table structure groups rows under /THead, /TBody, /TFoot (i.e.
    /Table -> /TBody -> /TR -> /TD) rather than putting /TR directly under
    /Table — real taggers (opendataloader-pdf included) emit this shape
    routinely. Descends one level into any /THead//TBody//TFoot child to
    collect its /TR kids, while a /TR that IS a direct child of the table
    is also picked up in place — so a mixed table (a /THead group plus a
    loose /TR sibling) still yields rows in document order. Non-row,
    non-group children (/Caption, a swept-in /Link or other annotation)
    are ignored, never counted as rows."""
    rows = []
    for kid in _kids(table):
        tag = str(kid.get("/S", ""))
        if tag == "/TR":
            rows.append(kid)
        elif tag in ("/THead", "/TBody", "/TFoot"):
            rows.extend(c for c in _kids(kid) if str(c.get("/S", "")) == "/TR")
    return rows


def _find_tables(pdf):
    """DFS pre-order list of /Table struct elements."""
    root = pdf.Root.get("/StructTreeRoot")
    if root is None:
        return []
    out, visited, stack = [], set(), list(reversed(_kids(root)))
    while stack:
        node = stack.pop()
        key = node.objgen if node.is_indirect else id(node)
        if key in visited:
            continue
        visited.add(key)
        if str(node.get("/S", "")) == "/Table":
            out.append(node)
        else:
            stack.extend(reversed(_kids(node)))
    return out


def _row_cells(row):
    """Direct /TD or /TH children of a /TR — real cells, excluding junk
    children (e.g. a /Link struct elem ODL sometimes sweeps into TR0)."""
    return [c for c in _kids(row) if str(c.get("/S", "")) in ("/TD", "/TH")]


def _resolve_header_row_index(rows):
    """First /TR index that actually contains a /TD or /TH cell. Real
    opendataloader-tagged PDFs frequently put a cell-less junk row (e.g. a
    swept-in /Link) at index 0 — that row must never be the resolved header
    row. Returns None if no row has any cells."""
    for i, row in enumerate(rows):
        if _row_cells(row):
            return i
    return None


def _cell_row_counts(rows):
    """Cell count per cell-bearing /TR (junk rows with 0 cells excluded —
    they carry no column-count signal, e.g. a swept-in /Link row)."""
    return [len(_row_cells(r)) for r in rows if _row_cells(r)]


def _cell_span(cell):
    """(colspan, rowspan) for `cell`, read from its /Table-owned /A entry
    (via _read_table_owned_attrs — the shared locator, not re-implemented
    here). A missing attribute dict, a missing /ColSpan//RowSpan key, a
    non-integer value, or a value < 1 all default to 1 — this never raises
    and never returns a value below 1."""
    attrs = _read_table_owned_attrs(cell)
    colspan = rowspan = 1
    if attrs is not None:
        for key, current in (("/ColSpan", colspan), ("/RowSpan", rowspan)):
            if key not in attrs:
                continue
            try:
                value = int(attrs[key])
            except (TypeError, ValueError):
                value = 1
            if value < 1:
                value = 1
            if key == "/ColSpan":
                colspan = value
            else:
                rowspan = value
    return colspan, rowspan


def _effective_col_counts_carry(rows):
    """Per cell-bearing /TR (same order/filtering as _effective_col_counts):
    (effective_width, carry_in) — carry_in is the width already
    contributed by an earlier row's /RowSpan reaching down into this row,
    BEFORE this row's own cells are added. _effective_col_counts is a thin
    wrapper that returns just the widths; span synthesis
    (_synthesize_colspans) needs carry_in too, so a lone short cell's
    synthesized /ColSpan nets out columns a /RowSpan already fills rather
    than re-claiming them."""
    cell_rows = [_row_cells(r) for r in rows if _row_cells(r)]
    out = []
    carries = []  # [remaining_rows_after_this_one, colspan] per active span
    for cells in cell_rows:
        carry_in = sum(c for _remaining, c in carries)
        width = carry_in
        next_carries = [[remaining - 1, c] for remaining, c in carries if remaining - 1 > 0]
        for cell in cells:
            colspan, rowspan = _cell_span(cell)
            width += colspan
            if rowspan > 1:
                next_carries.append([rowspan - 1, colspan])
        out.append((width, carry_in))
        carries = next_carries
    return out


def _effective_col_counts(rows):
    """Effective column width per cell-bearing /TR, taking /ColSpan and
    /RowSpan into account (PDF32000-2 14.7.5.4, Table 349) — a row's width
    is the sum of its own cells' /ColSpan (default 1) PLUS a carry-in
    column for every earlier row's cell whose /RowSpan still reaches this
    row. Junk rows with 0 cells are excluded first, same as
    _cell_row_counts — they carry no column-count signal and are never
    counted as a row a /RowSpan spans over."""
    return [width for width, _carry_in in _effective_col_counts_carry(rows)]


def _is_ragged(rows):
    """True when cell-bearing rows' EFFECTIVE column counts differ — each
    row's width is its cells' /ColSpan summed, plus any column carried down
    from an earlier row's /RowSpan (see _effective_col_counts). This is
    what veraPDF clause 7.2 actually checks ("table rows shall have the
    same number of columns, taking spans into account"): a row that reads
    narrower only because one of its cells has /ColSpan 2, or because a
    cell above it carries a /RowSpan into it, is NOT ragged. Attributing
    (/A with /Scope) makes veraPDF actually validate 7.2, which it does not
    check on an un-attributed table — so promoting a genuinely ragged table
    (no spans recorded, real column-count mismatch) turns a compliant
    document into a non-compliant one; promotion refuses those by default
    instead.

    This is a READ of span data the tagger already wrote (opendataloader-
    pdf's JSON exposes "row span"/"column span" per cell, and commonly
    emits /ColSpan//RowSpan on merged cells in the tagged PDF itself) — not
    span synthesis. A table whose tagger omitted /ColSpan on a genuinely
    spanning cell still reads as ragged here, correctly, since a missing
    span defaults to 1; promote_headers calls _synthesize_colspans right
    before this check to close that gap for the one unambiguous shape
    (a lone cell in an otherwise-short row) — see that function's
    docstring. Anything less clear-cut is still left ragged on purpose."""
    return len(set(_effective_col_counts(rows))) > 1


_GRID_MAX_ROWS = 8
_GRID_MAX_COLS = 4
_GRID_CELL_CHARS = 40

_MAX_CELL_TEXT = 200


def _truncate(text: str) -> str:
    """Cap a manifest string at 200 chars, appending '…' when truncated —
    keeps first_row_text/first_col_text/caption readable regardless of how
    much prose a real cell/caption carries."""
    if len(text) <= _MAX_CELL_TEXT:
        return text
    return text[:_MAX_CELL_TEXT] + "…"


def _cell_text(cell, inherited_pg=None):
    """Best-effort header-judgment text for a /TD or /TH cell: /ActualText or
    /Alt on the cell itself, else the same on its first /P descendant.
    (/Caption text uses the separate _caption_text below, NOT this function
    — a caption's text must never come from a descendant's /Alt, see its
    docstring.)

    Real-world tagged PDFs essentially never set /ActualText//Alt on
    ordinary cells — ONLY when both are absent everywhere on the node does
    this fall back to the cell's marked-content text, read from the page's
    content stream via mcid_text.page_mcid_text() (MCID collection reuses
    _collect_figure_mcids' walk rather than duplicating it). `inherited_pg`
    is the nearest ancestor /Pg (the caller passes the owning table's) to
    fall back to when the cell/its descendants carry no /Pg of their own —
    real taggers frequently set /Pg once at the table level, not per cell.

    Known artifact: when a cell's content is split across multiple MCIDs,
    their text is joined with " ". Verified against a real file
    (Cooper_2018, course 256162) that this is genuinely ambiguous, not a
    bug in the join/whitespace-collapse: e.g. MCID 13/14/15 hold the plain
    Tj strings "Decline"/"to"/"state" (three words — needs the space) while
    MCID 17/18 hold "Race/ethnicit"/"y" (ONE word split mid-token by the
    tagger, likely PLOS ONE's justification-driven per-character kerning
    runs — needs NO space) — both shapes are indistinguishable from
    plain per-MCID text alone; neither Tj string carries a trailing/leading
    space or any other signal. Resolving this correctly would require
    tracking text-positioning (Tm/Td/TJ kerning) and font glyph widths to
    measure the actual gap between runs — a real text-layout engine, well
    beyond this best-effort report-field extractor's scope. Left as-is
    (word-boundary preservation is the more common/important case to get
    right); a stray mid-word space is the accepted cost.

    Every returned string is capped at 200 chars (see _truncate)."""
    for key in ("/ActualText", "/Alt"):
        if key in cell:
            return _truncate(str(cell[key]))
    for p in _kids(cell):
        for key in ("/ActualText", "/Alt"):
            if key in p:
                return _truncate(str(p[key]))
    pairs, _captions, _refs = _collect_figure_mcids(cell, inherited_pg=inherited_pg)
    if not pairs:
        return ""
    parts = []
    for mcid, pg in pairs:
        if pg is None:
            continue
        text = mcid_text.page_mcid_text(pg).get(mcid)
        if text:
            parts.append(text)
    return _truncate(" ".join(parts))


def _caption_text(caption, inherited_pg=None):
    """A /Caption struct elem's OWN text: /ActualText or /Alt on the
    /Caption node itself, else its subtree's MCID text. Returns "".

    Deliberately NOT _cell_text: a cell's text is reasonably represented by
    its first /P descendant's /ActualText//Alt when the cell itself lacks
    one, but a caption's content is never a descendant annotation's alt
    string. Real ODL-tagged captions commonly carry a /Link child (e.g. an
    internal cross-reference) whose /Alt is the tagger's generic
    placeholder "Annotation" — descending into it (as _cell_text does for
    cells) would silently substitute that placeholder for the real caption
    text, which is worse than an empty field: it looks like a real,
    decisive table-vs-figure signal ("Table 1 ..." vs "Figure 6.18 ...")
    but is actually noise. So caption resolution stops at the node's own
    attributes and falls straight to MCID text, never a child's /Alt."""
    for key in ("/ActualText", "/Alt"):
        if key in caption:
            return _truncate(str(caption[key]))
    pairs, _captions, _refs = _collect_figure_mcids(caption, inherited_pg=inherited_pg)
    if not pairs:
        return ""
    parts = []
    for mcid, pg in pairs:
        if pg is None:
            continue
        text = mcid_text.page_mcid_text(pg).get(mcid)
        if text:
            parts.append(text)
    return _truncate(" ".join(parts))


def _read_table_owned_attrs(cell):
    """Non-mutating read of `cell`'s /Table-owned attribute dict (an /A
    entry with /O /Table), or None if it has none. Unlike _table_attrs
    (promote_headers' writer, which creates the dict when absent),
    this never creates or mutates anything — list_tables is a read-only
    manifest and must not have write side effects."""
    a = cell.get("/A")
    if a is None:
        return None
    items = a if isinstance(a, pikepdf.Array) else [a]
    for item in items:
        if hasattr(item, "get") and str(item.get("/O", "")) == "/Table":
            return item
    return None


def _grid_entry(cell, inherited_pg=None):
    """One grid preview entry for a /TD or /TH `cell`: {"text", "tag"}, plus
    "colspan"/"rowspan" when the cell's /Table-owned /A carries a /ColSpan
    or /RowSpan attribute other than 1 (PDF32000-2 14.7.5.4, Table 349).
    Text is real cell text via _cell_text (ActualText/Alt/MCID fallback,
    already capped at 200 chars), further truncated to _GRID_CELL_CHARS on
    a word boundary via _truncate_head — the grid is a compact preview, not
    the full-fidelity first_row_text/first_col_text fields."""
    entry = {
        "text": _truncate_head(_cell_text(cell, inherited_pg=inherited_pg), _GRID_CELL_CHARS),
        "tag": str(cell.get("/S", "")).lstrip("/"),
    }
    attrs = _read_table_owned_attrs(cell)
    if attrs is not None:
        for key, out_key in (("/ColSpan", "colspan"), ("/RowSpan", "rowspan")):
            if key in attrs:
                try:
                    value = int(attrs[key])
                except (TypeError, ValueError):
                    continue
                if value != 1:
                    entry[out_key] = value
    return entry


def _table_grid(table, inherited_pg=None):
    """2D preview of a table's actual cell layout: the first
    _GRID_MAX_ROWS cell-bearing rows, each cut to its first _GRID_MAX_COLS
    cells. Deliberately NOT padded to a rectangle — a row with fewer cells
    than others keeps fewer grid entries, because that raggedness is
    exactly the signal _is_ragged / the veraPDF 7.2 guard keys on, and
    padding it away would hide the thing this preview exists to surface
    (see module docstring: a spanning cell with no /ColSpan collapses a
    row's cell count below its neighbors').

    This is what makes a spanning column visible: column 0 holding a
    row-spanning category cell only ever contributes ONE grid entry across
    however many data rows it visually covers, so those rows' grid[1] is
    genuinely column 1 content — unlike first_col_text (a flat list of
    "each row's first cell"), which has no way to represent that a cell
    spans downward and so silently interleaves column 0 and column 1 text
    into one list on exactly this shape.

    Returns {"rows": [[{"text","tag",...}, ...], ...], "truncated_rows":
    <cell-bearing rows beyond the first _GRID_MAX_ROWS, 0 if none>,
    "truncated_cols": <cells beyond _GRID_MAX_COLS in the widest previewed
    row, 0 if none>}. A table with zero cell-bearing rows yields
    {"rows": [], "truncated_rows": 0, "truncated_cols": 0} — the empty case
    falls out of the same counting, no special-casing needed."""
    cell_rows = [_row_cells(r) for r in _table_rows(table) if _row_cells(r)]
    preview_rows = cell_rows[:_GRID_MAX_ROWS]
    widest_previewed = max((len(cells) for cells in preview_rows), default=0)
    return {
        "rows": [
            [_grid_entry(c, inherited_pg=inherited_pg) for c in cells[:_GRID_MAX_COLS]]
            for cells in preview_rows
        ],
        "truncated_rows": len(cell_rows) - len(preview_rows),
        "truncated_cols": max(0, widest_previewed - _GRID_MAX_COLS),
    }


def _page_number_map(pdf):
    """Map from _obj_key(page /Pg object) -> 1-based page number, built once
    per list_tables() call so per-cell page resolution doesn't rescan
    pdf.pages for every cell — a table can have many."""
    return {_obj_key(pg_obj.obj): pno + 1 for pno, pg_obj in enumerate(pdf.pages)}


def _resolve_cell_page(cell, page_map):
    """1-based page number a /TD or /TH `cell` resolves to via its own /Pg,
    falling back to the first /Pg found on a direct child (real taggers
    often set /Pg once at the table/row level, not per cell) — or None when
    unresolved. `page_map` is a _page_number_map(pdf) lookup. This is the
    exact walk list_tables' single-cell `page` field always used, extracted
    so `pages` (every cell a table's rows touch) stays in lockstep with it
    rather than drifting via a second resolver."""
    if cell is None:
        return None
    pg = cell.get("/Pg")
    if pg is None:
        for p in _kids(cell):
            pg = p.get("/Pg")
            if pg is not None:
                break
    if pg is None:
        return None
    return page_map.get(_obj_key(pg))


def list_tables(pdf, path=None) -> list:
    """Agent-facing manifest of every /Table in the tag tree, document order.
    "rows" is the raw /TR count; "cell_rows" is how many of those /TR
    actually contain a cell (real PDFs often have a cell-less junk /TR — a
    swept-in /Link — at index 0). "header_row_index"/"cols"/"first_row_text"
    describe the row promote_headers(header_row=True) will actually act on
    (the first cell-bearing /TR), NOT necessarily raw /TR index 0 — this
    keeps the manifest honest about what promotion does.
    "ragged" is True when cell-bearing rows have differing cell counts
    (merged cells, no /ColSpan recorded) — promote_headers refuses to
    attribute a ragged table by default (see its docstring / module
    docstring for the veraPDF 7.2 hazard).
    "first_col_text" is the first cell of each cell-bearing row (document
    order, capped at 8 rows) — the column-1 content needed to catch a table
    whose real header axis is the first COLUMN, not the first row (a live
    review of real files found this on 3 of 5 sampled data tables — the
    manifest previously had no field describing column 1 at all).
    "caption" is the table's /Caption text if it has one, else "" —
    decisive data-table-vs-figure evidence (e.g. "Figure 6.18 ..." vs
    "Table 1 ...") that was previously invisible to the manifest. Resolved
    via _caption_text, NOT _cell_text — a caption's text is its own
    /ActualText//Alt or its subtree's MCID text, never a descendant
    annotation's /Alt (see _caption_text's docstring for the real-file
    case — a /Link child's generic "Annotation" placeholder — that
    motivated the separate resolver).
    first_row_text/first_col_text resolve real cell text via _cell_text
    (ActualText/Alt, falling back to MCID text from the page content
    stream — see mcid_text.py), the only extractor that actually populates
    these fields on real-world PDFs, which essentially never set
    /ActualText//Alt on ordinary cells.
    "page_context" gives the agent the surrounding text a lone-table view
    can't show — e.g. a rubric table on page 4 that is actually a
    continuation of rows started on page 3 (promoting headers on that
    fragment would tag a data row as headers, which the module docstring
    calls actively misleading). Keys: "prev_page"/"next_page" (first ~400
    chars of the neighboring page's text), "same_page_before"/
    "same_page_after" (up to ~300 chars of this page's own text
    immediately before/after the table). Every value is "" when
    unavailable (no such page, `path` not supplied, or the table's own
    `page` is unresolved) — never omitted, never None. Requires `path` (the
    PDF's file path) because this manifest builds from `pdf` (a pikepdf.Pdf
    with no filesystem path of its own) while the whole-page text comes
    from fitz (PyMuPDF), opened once per call via `path`. `path=None` (the
    default, and the only option for a caller with no path handy) yields
    an all-empty page_context on every record — list_tables(pdf) alone
    keeps working exactly as before. same_page_before/after are located by
    searching the page for the table's own first-cell text
    (fitz.Page.search_for) and splitting the page above/below that rect;
    an empty or ambiguous (non-unique) match returns "" rather than
    guessing — a wrong split misleads the exact judgment this field exists
    to inform.
    "grid" is a 2D preview of the table's real cell layout — see
    _table_grid's docstring. Unlike first_row_text/first_col_text (flat
    lists), grid entries carry each cell's own row position, so a column-0
    cell that visually spans several data rows shows up once, not smeared
    across every row it covers as if it were that row's own first cell —
    the exact ambiguity first_col_text cannot resolve on a table with a
    row-spanning header column (see module docstring). Purely additive:
    first_row_text/first_col_text are unchanged.
    "pages" is the sorted, deduplicated list of every 1-based page number
    any of the table's cells resolve to (via _resolve_cell_page — the same
    walk `page` uses, not a parallel resolver), because a tagged table
    routinely spans a page break and a single `page` field hides that: a
    grading table whose header row repeats mid-table (the tagger's own
    continuation signal) previously reported one page even though its rows
    live on two. `pages` is `[]` when nothing resolves. `page` itself is
    UNCHANGED — same value, same first-cell-of-first-row semantics — kept
    for every existing call site; when both are populated `page ==
    pages[0]` because `page` resolves from the table's own first row, which
    is never later in the document than any other row's page.

    Returns [{"index", "page", "pages", "rows", "cell_rows", "cols",
    "has_th", "header_row_index", "first_row_text", "first_col_text",
    "caption", "ragged", "page_context", "grid"}]."""
    tables = _find_tables(pdf)
    page_map = _page_number_map(pdf)
    out = []
    fitz_doc = None
    if path is not None:
        try:
            import fitz  # PyMuPDF
            fitz_doc = fitz.open(path)
        except Exception:  # noqa: BLE001 - best-effort; missing/bad path -> no context
            fitz_doc = None
    for i, table in enumerate(tables):
        rows = _table_rows(table)
        cell_bearing_rows = [r for r in rows if _row_cells(r)]
        cell_rows = len(cell_bearing_rows)
        header_idx = _resolve_header_row_index(rows)
        header_cells = _row_cells(rows[header_idx]) if header_idx is not None else []
        table_pg = table.get("/Pg")
        has_th = any(
            str(c.get("/S", "")) == "/TH"
            for row in rows
            for c in _kids(row)
        )
        pg_cell = header_cells[0] if header_cells else None
        if pg_cell is None:
            for row in rows:
                cells = _row_cells(row)
                if cells:
                    pg_cell = cells[0]
                    break
        page = _resolve_cell_page(pg_cell, page_map)
        pages = sorted({
            p
            for row in rows
            for cell in _row_cells(row)
            if (p := _resolve_cell_page(cell, page_map)) is not None
        })
        caption_elem = next(
            (c for c in _kids(table) if str(c.get("/S", "")) == "/Caption"), None)
        pg_cell_text = _cell_text(pg_cell, inherited_pg=table_pg) if pg_cell is not None else ""
        out.append({
            "index": i,
            "page": page,
            "pages": pages,
            "rows": len(rows),
            "cell_rows": cell_rows,
            "cols": len(header_cells),
            "has_th": has_th,
            "header_row_index": header_idx,
            "first_row_text": [_cell_text(c, inherited_pg=table_pg) for c in header_cells],
            "first_col_text": [
                _cell_text(_row_cells(r)[0], inherited_pg=table_pg)
                for r in cell_bearing_rows[:8]
            ],
            "caption": _caption_text(caption_elem, inherited_pg=table_pg) if caption_elem is not None else "",
            "ragged": _is_ragged(rows),
            "page_context": _page_context(fitz_doc, page, pg_cell_text),
            "grid": _table_grid(table, inherited_pg=table_pg),
        })
    if fitz_doc is not None:
        fitz_doc.close()
    return out


def _ragged_reason(rows):
    """Shared skip-reason string for both the legacy and hierarchical
    ragged-table guard — identical wording either way, see _is_ragged."""
    counts = sorted(set(_cell_row_counts(rows)))
    return (f"ragged rows (cells per row differ: {counts}) — "
            "attributing would fail veraPDF 7.2")


def _synthesize_colspans(pdf, rows):
    """Write /ColSpan onto UNAMBIGUOUS short rows, called by promote_headers
    immediately before its ragged check (both the legacy and hierarchical
    paths), only for the table spec currently being promoted — never from
    list_tables, which must keep reporting the file's actual recorded
    state untouched.

    A cell-bearing row only qualifies when it holds EXACTLY ONE real cell
    (/TD or /TH) and its effective width (_effective_col_counts_carry) is
    less than the table's target width (the max effective width across
    all its cell-bearing rows). With exactly one cell in the row, that
    cell's content has nowhere else to be — its correct /ColSpan is
    target minus whatever width is already carried into this row from an
    earlier row's /RowSpan (the columns a rowspan already fills need not
    be re-claimed). A short row with two or more cells is genuinely
    ambiguous — which cell spans, and by how much, cannot be inferred
    from row shape alone — and is never touched; it is left to read as
    ragged, same as before this function existed.

    A cell that already carries a /Table-owned /ColSpan is never
    overwritten, in either direction: if that existing value already
    equals the target, the row's effective width already reads >= target
    (via _cell_span, which _effective_col_counts_carry reads through), so
    it never reaches the "short row" branch here in the first place — no
    special-case needed. If the existing value differs from the target,
    it is left exactly as recorded; the row stays exactly as
    ambiguous/ragged as it already was, rather than this function
    silently overwriting a value the tagger (or an earlier
    promote_headers call) already committed to.

    Returns the count of cells that received a brand-new /ColSpan write."""
    detail = _effective_col_counts_carry(rows)
    if not detail:
        return 0
    target = max(width for width, _carry_in in detail)
    cell_rows = [r for r in rows if _row_cells(r)]
    synthesized = 0
    for row, (width, carry_in) in zip(cell_rows, detail):
        if width >= target:
            continue
        cells = _row_cells(row)
        if len(cells) != 1:
            continue
        cell = cells[0]
        existing_attrs = _read_table_owned_attrs(cell)
        if existing_attrs is not None and "/ColSpan" in existing_attrs:
            continue  # already recorded (correct or conflicting) — never overwrite
        needed = target - carry_in
        if needed < 1:
            continue
        _table_attrs(pdf, cell)["/ColSpan"] = pikepdf.Integer(needed)
        synthesized += 1
    return synthesized


def _is_int_list(x):
    """True for a (possibly empty) list of ints — the shape header_rows/
    header_cols/band_rows must have."""
    return isinstance(x, list) and all(isinstance(i, int) for i in x)


def _collect_existing_ids(pdf):
    """Every /ID byte string already present anywhere in the struct tree.
    /ID is required by the PDF spec to be unique among ALL structure
    elements in the document (PDF32000-2 14.7.2), not just the tables this
    module touches — so newly generated header-cell IDs are checked against
    the whole tree, not just against each other."""
    root = pdf.Root.get("/StructTreeRoot")
    if root is None:
        return set()
    ids = set()
    visited, stack = set(), list(_kids(root))
    while stack:
        node = stack.pop()
        key = node.objgen if node.is_indirect else id(node)
        if key in visited:
            continue
        visited.add(key)
        if "/ID" in node:
            ids.add(str(node["/ID"]))
        stack.extend(_kids(node))
    return ids


def _gen_unique_id(existing_ids, base):
    """`base` if not already used (registering it in `existing_ids`), else
    `base` suffixed with an incrementing counter until unique. Collisions
    are expected to be rare in practice (base already encodes table/row/col)
    but the spec's document-wide uniqueness requirement is checked, not
    assumed."""
    candidate = base
    n = 2
    while candidate in existing_ids:
        candidate = f"{base}_{n}"
        n += 1
    existing_ids.add(candidate)
    return candidate


def _table_attrs(pdf, cell):
    """Get-or-create the /Table-owned attribute dictionary on `cell`'s /A
    entry. PDF32000-2 14.7.5.4 (Table 349, "Standard table attributes")
    defines RowSpan/ColSpan/Headers/Scope/Summary as members of ONE
    attribute dictionary whose /O is /Table — never top-level keys on the
    structure element itself. /A may be a single attribute dict, or an
    array of attribute dicts (a cell can carry attributes from more than
    one owner, e.g. /Layout alongside /Table); this never drops an
    existing entry — a single non-/Table dict is promoted to a 2-element
    array, an existing array gets a new /Table dict appended only if none
    already has /O /Table."""
    a = cell.get("/A")
    if a is None:
        attrs = pdf.make_indirect(pikepdf.Dictionary(O=pikepdf.Name("/Table")))
        cell["/A"] = attrs
        return attrs
    if isinstance(a, pikepdf.Array):
        for item in a:
            if hasattr(item, "get") and str(item.get("/O", "")) == "/Table":
                return item
        attrs = pdf.make_indirect(pikepdf.Dictionary(O=pikepdf.Name("/Table")))
        a.append(attrs)
        return attrs
    if hasattr(a, "get") and str(a.get("/O", "")) == "/Table":
        return a
    attrs = pdf.make_indirect(pikepdf.Dictionary(O=pikepdf.Name("/Table")))
    cell["/A"] = pikepdf.Array([a, attrs])
    return attrs


def _promote_headers_hierarchical(pdf, table, table_idx, header_rows, header_cols,
                                   band_rows, existing_ids):
    """Multi-axis /TD -> /TH promotion for one table: every cell in a
    header_rows row, a header_cols column, or a band_rows row becomes /TH
    with a unique /ID; every other (data) cell gets /Headers listing the IDs
    of the header cells that govern it. See promote_headers's docstring for
    the full field semantics. Returns (cells_promoted, touched):
    cells_promoted counts /TD -> /TH transitions only (same meaning as the
    legacy path's metric); touched is True if anything was written at all,
    including backfilling /ID onto a cell that was ALREADY /TH before this
    call — real base-tagger output (opendataloader-pdf included) commonly
    marks the intended header row/column /TH already but never writes /A
    or /ID, so "already /TH" alone cannot mean "fully done, skip": a cell
    already /TH is left with its /S untouched (never re-promoted, never
    double-counted) and any EXISTING /Scope//Headers value in its /A is
    never overwritten, but it still gets a fresh /ID if it doesn't already
    have one (via _table_attrs, which reuses/extends an existing /A rather
    than replacing it) — /ID is only ever added, never replaced, so a cell
    /TH+/ID from an EARLIER run of this function is genuinely a no-op (true
    idempotency), while a cell /TH with no /ID (from this function or an
    outside tagger) still gets hierarchically wired up, which is the only
    way this feature does anything useful on such files.

    Indexing: header_rows/band_rows index into `_table_rows(table)` (the
    same raw, document-order /TR list header_row_index already indexes
    into). header_cols indexes into a row's own _row_cells() position.

    Header-row spans: a header_rows row can itself be narrower than the
    data rows it governs — a single spanning banner cell with no /ColSpan
    recorded (verified against a real file: the PMP exam outline's "DOMAIN
    I PEOPLE—33%" row has exactly one cell while its Task rows have two).
    A data cell's /Headers lookup against such a row clamps its own column
    index to that header row's last real cell rather than requiring an
    exact column match — otherwise a data cell past the header row's cell
    count would silently get NO reference to it at all, defeating the
    entire point on precisely the shape this feature targets.

    Band /Scope: a band row whose lone real cell is the row's only cell (and
    the table has more than one column elsewhere) is treated as genuinely
    spanning the full width and gets /Scope /Column; a band row with more
    than one cell is ambiguous (could be one true span mis-tagged as
    several cells, or several genuine narrower headers) and every cell in it
    gets /A with /O /Table but NO /Scope, rather than asserting a guess.
    (A pre-existing /A on an already-/TH band cell is never overwritten
    either way, per the no-clobber rule above.)"""
    rows = _table_rows(table)
    n = len(rows)
    header_rows = sorted({r for r in header_rows if 0 <= r < n})
    band_rows = sorted({r for r in band_rows if 0 <= r < n})
    header_cols = sorted({c for c in header_cols if c >= 0})
    full_header_rows = set(header_rows) | set(band_rows)

    body_counts = [len(_row_cells(rows[i])) for i in range(n)
                   if i not in full_header_rows and _row_cells(rows[i])]
    body_col_count = max(body_counts) if body_counts else 0
    # a header_rows row narrower than the data rows below it (e.g. one lone
    # cell holding a full-width banner, no /ColSpan recorded — the real
    # PMP-exam-outline shape) has no cell at every data column; its LAST
    # real cell is the fallback header for every column beyond its own
    # count, same reasoning as the band-row "spans full width" heuristic.
    header_row_lens = {r: len(_row_cells(rows[r])) for r in header_rows}

    cells_promoted = 0
    touched = False
    cell_ids = {}  # (row_idx, col_idx) -> /ID string

    def promote(cell, row_idx, col_idx, scope):
        nonlocal cells_promoted, touched
        was_th = str(cell.get("/S", "")) == "/TH"
        if was_th and "/ID" in cell:
            # true no-op: this cell was already fully wired by an earlier
            # call to this function — reuse its ID, touch nothing.
            cell_ids[(row_idx, col_idx)] = str(cell["/ID"])
            return
        cid = _gen_unique_id(existing_ids, f"th_{table_idx}_{row_idx}_{col_idx}")
        cell["/S"] = pikepdf.Name("/TH")
        if scope is not None:
            attrs = _table_attrs(pdf, cell)
            if "/Scope" not in attrs:  # never overwrite an existing Scope
                attrs["/Scope"] = pikepdf.Name(scope)
        cell["/ID"] = pikepdf.String(cid)
        cell_ids[(row_idx, col_idx)] = cid
        touched = True
        if not was_th:
            cells_promoted += 1

    for r in header_rows:
        for c, cell in enumerate(_row_cells(rows[r])):
            promote(cell, r, c, "/Column")

    for r in band_rows:
        cells = _row_cells(rows[r])
        spans_full_width = len(cells) == 1 and body_col_count > 1
        for c, cell in enumerate(cells):
            promote(cell, r, c, "/Column" if spans_full_width else None)

    for r in range(n):
        if r in full_header_rows:
            continue
        cells = _row_cells(rows[r])
        for c in header_cols:
            if 0 <= c < len(cells):
                promote(cells[c], r, c, "/Row")

    for r in range(n):
        if r in full_header_rows:
            continue
        cells = _row_cells(rows[r])
        if not cells:
            continue
        preceding_bands = [b for b in band_rows if b < r]
        nearest_band = max(preceding_bands) if preceding_bands else None
        for c, cell in enumerate(cells):
            if c in header_cols or str(cell.get("/S", "")) == "/TH":
                continue
            ids = []
            for hr in header_rows:
                hr_len = header_row_lens.get(hr, 0)
                if hr_len == 0:
                    continue
                hr_col = c if c < hr_len else hr_len - 1
                hid = cell_ids.get((hr, hr_col))
                if hid:
                    ids.append(hid)
            for hc in header_cols:
                hid = cell_ids.get((r, hc))
                if hid:
                    ids.append(hid)
            if nearest_band is not None:
                for bc in range(len(_row_cells(rows[nearest_band]))):
                    bid = cell_ids.get((nearest_band, bc))
                    if bid:
                        ids.append(bid)
            if not ids:
                continue
            attrs = _table_attrs(pdf, cell)
            existing = None
            if "/Headers" in attrs:
                try:
                    existing = [str(x) for x in attrs["/Headers"]]
                except (TypeError, ValueError):
                    existing = None
            if existing != ids:
                attrs["/Headers"] = pikepdf.Array([pikepdf.String(i) for i in ids])
                touched = True

    return cells_promoted, touched


def promote_headers(pdf, specs) -> dict:
    """Judgment-driven /TD -> /TH promotion. specs: [{"index": int,
    "header_row": bool, "header_col": bool, "header_row_index": int,
    "force": bool}] — opt-in per table, NEVER auto-applied to a table absent
    from specs (inventing headers on a headerless table misleads
    screen-reader users, see module docstring).

    HIERARCHICAL HEADERS (additive, opt-in per spec entry): a spec may
    instead carry "header_rows": [int], "header_cols": [int], "band_rows":
    [int] — 0+ header ROWS, 0+ header COLUMNS, and 0+ full-width spanning
    "band" rows interspersed in the body (e.g. a "Dependent variable = ..."
    banner row splitting a results table into sections). Presence of ANY of
    these three keys on a spec switches that spec entry to the hierarchical
    path INSTEAD of the legacy header_row/header_col boolean path (the two
    are not combined on one spec entry — run legacy fields on one spec,
    hierarchical fields on another, same as any other opt-in judgment).
    "force" still overrides the ragged-table guard for the hierarchical
    path (see below).

    Single /Scope (legacy) cannot express "this cell is governed by main
    header X AND band header Y AND row header Z" — real course tables need
    that (a rubric: row 0 spanning title + column 0 criteria names; an exam
    outline: row 0 domain banner + column 0 task numbers; a results table:
    row 0 main headers + interspersed band sub-headers + column 0 row
    labels). The PDF/UA mechanism for multi-axis header association is /ID
    on each header cell plus /Headers on each governed data cell listing
    those IDs (PDF32000-2 14.8.4.3.5), which is what this path writes.
    Every header cell also still gets /Scope where it remains a true
    single-axis description (Column for header_rows, Row for header_cols);
    a band cell gets /Scope /Column only if it demonstrably spans the full
    row width, otherwise no /Scope at all — see
    _promote_headers_hierarchical's docstring for the exact rule and the
    honest reasoning for each choice.

    /IDTree (NOT populated, by design — see below): PDF32000-2 14.7.2
    defines /StructTreeRoot's /IDTree as an OPTIONAL name-tree index for
    looking up a structure element by its /ID from OUTSIDE the tag tree
    (e.g. a structure destination). It is not what resolves a /Headers
    array — 14.8.4.3.5 defines Headers entries as byte strings matched
    directly against a /TH's own /ID attribute; a conforming AT walks the
    tag tree (or scans by ID) rather than requiring a name-tree lookup.
    This codebase has no prior /IDTree management code (no other script
    touches it), and building a spec-correct sorted PDF name tree (proper
    /Names / /Kids / /Limits balancing) is a materially bigger addition
    than this feature warrants when the thing /Headers actually depends on
    — document-wide /ID uniqueness — is already enforced by
    _collect_existing_ids/_gen_unique_id. If a future consumer specifically
    needs /IDTree (e.g. AT that indexes via IDTree instead of walking the
    tag tree, or another StructTreeRoot editor after this one), it must be
    added as its own change; this is a known, deliberate gap, not a silent
    omission.

    RAGGED TABLES ARE REFUSED BY DEFAULT: a table whose cell-bearing rows
    don't all have the same effective cell count (merged cells with no
    recorded/inferable /ColSpan) is skipped entirely — attributing it (/A
    with /Scope) makes veraPDF actually validate clause 7.2 ("table rows
    shall have the same number of columns, taking spans into account"),
    which an un-attributed table is never checked against. Promoting a
    genuinely ragged table can turn a COMPLIANT document NON-compliant.
    Pass "force": true on the spec to override for a table that is still
    ragged after span synthesis (below) — i.e. a short row with two or
    more cells, where which cell spans cannot be inferred.

    SPAN SYNTHESIS RESOLVES THE FLAGSHIP SHAPE: a header_rows row that
    spans the full table width as one lone cell (no /ColSpan recorded)
    used to always have FEWER cells than the data rows below it, reading
    as ragged unconditionally (this is exactly the PMP-exam-outline
    "DOMAIN I PEOPLE—33%" banner case — see
    _promote_headers_hierarchical's docstring). _synthesize_colspans now
    writes the missing /ColSpan for exactly this shape — a cell-bearing
    row with ONE real cell, narrower than the table's widest row — before
    the ragged check runs, so this table promotes without "force" (a
    corpus review of real remediation-target files found the shape on 6
    of the tables sampled, all single-cell banner/separator rows; see
    _synthesize_colspans's docstring for the exact rule and why a
    multi-cell short row is refused instead of guessed at). "force": true
    remains the fallback for a table that is still ragged after
    synthesis — it does NOT make the table spec-correct, it only lets the
    agent proceed while accepting the pre-existing veraPDF 7.2 risk the
    ragged shape already carries.

    header_row: promotes every real cell (/TD or /TH) of the header row ->
    /TH, /A Scope=/Column. The header row defaults to the first /TR that
    actually contains a cell (_resolve_header_row_index) — real
    opendataloader-tagged PDFs frequently have a cell-less junk /TR (e.g. a
    swept-in /Link) at raw index 0, which must NEVER be treated as the
    header row. Pass "header_row_index" (0-based into the table's raw /TR
    list) to override this resolution explicitly, e.g. when the genuine
    header is the second real row.
    header_col: the first real cell of every cell-bearing /TR -> /TH,
    /A Scope=/Row. Rows with no cells at all (junk rows) are skipped.
    Both: the resolved header row's first cell (the corner) gets
    Scope=/Column (row wins, /A written once).
    Idempotent: a cell already /TH is left alone (its /A is not touched) and
    counted as skipped, not promoted. Out-of-range/malformed index, or a
    header_row_index that can't be resolved (no cell rows / bad override)
    -> skipped, never raises.

    SPAN SYNTHESIS (unambiguous only): immediately before each ragged
    check below (both paths), _synthesize_colspans writes /ColSpan onto
    any cell-bearing row of the table being promoted that holds EXACTLY
    ONE real cell and reads narrower than the table's widest row — that
    cell has nowhere else to be, so its /ColSpan can only be the table's
    target width (net of any /RowSpan already carried into the row). A
    short row with two or more cells stays untouched and, if still
    ragged, is still refused exactly as before this existed. This is not
    a new opt-in: it only touches a table already named in `specs`, never
    a table absent from `specs`, and never list_tables' own reporting.
    Cells synthesized this call are counted in "spans_synthesized" below.

    Returns {"tables_touched": n, "cells_promoted": n,
    "spans_synthesized": n, "skipped": [...]}."""
    tables = _find_tables(pdf)
    tables_touched = 0
    cells_promoted = 0
    spans_synthesized = 0
    skipped = []
    existing_ids = None  # lazily populated, only if a hierarchical spec is seen

    def _promote(cell, scope):
        nonlocal cells_promoted
        if str(cell.get("/S", "")) == "/TH":
            return False
        cell["/S"] = pikepdf.Name("/TH")
        # _table_attrs (get-or-create) rather than a wholesale /A replace —
        # a cell can already carry a /Table-owned /A here (e.g. a /ColSpan
        # just written by _synthesize_colspans immediately before this
        # spec's ragged check ran), and that must survive promotion, not
        # be clobbered by a fresh dict holding only /Scope.
        _table_attrs(pdf, cell)["/Scope"] = pikepdf.Name(scope)
        cells_promoted += 1
        return True

    for spec in (specs or []):
        if not isinstance(spec, dict) or "index" not in spec:
            skipped.append({"spec": spec, "reason": "missing index"})
            continue
        try:
            idx = int(spec["index"])
        except (TypeError, ValueError):
            skipped.append({"spec": spec, "reason": "non-integer index"})
            continue
        if not (0 <= idx < len(tables)):
            skipped.append({"spec": spec, "reason": "index out of range"})
            continue

        # ── Hierarchical path: header_rows/header_cols/band_rows present ──
        # Additive and mutually exclusive with the legacy header_row/
        # header_col booleans on the SAME spec entry — presence of any of
        # these three keys routes this spec entirely through
        # _promote_headers_hierarchical instead of the code below, which
        # stays byte-for-byte identical to before this feature existed.
        if any(k in spec for k in ("header_rows", "header_cols", "band_rows")):
            hr = spec.get("header_rows") or []
            hc = spec.get("header_cols") or []
            br = spec.get("band_rows") or []
            if not (_is_int_list(hr) and _is_int_list(hc) and _is_int_list(br)):
                skipped.append({
                    "spec": spec,
                    "reason": "header_rows/header_cols/band_rows must be lists of int",
                })
                continue
            if not hr and not hc and not br:
                skipped.append({
                    "spec": spec,
                    "reason": "header_rows/header_cols/band_rows all empty",
                })
                continue

            rows = _table_rows(tables[idx])
            spans_synthesized += _synthesize_colspans(pdf, rows)
            if _is_ragged(rows) and not spec.get("force"):
                skipped.append({"spec": spec, "reason": _ragged_reason(rows)})
                continue

            if existing_ids is None:
                existing_ids = _collect_existing_ids(pdf)
            promoted, hier_touched = _promote_headers_hierarchical(
                pdf, tables[idx], idx, hr, hc, br, existing_ids)
            cells_promoted += promoted
            if hier_touched:
                tables_touched += 1
            continue

        header_row = bool(spec.get("header_row"))
        header_col = bool(spec.get("header_col"))
        if not header_row and not header_col:
            skipped.append({"spec": spec, "reason": "neither header_row nor header_col set"})
            continue

        rows = _table_rows(tables[idx])
        spans_synthesized += _synthesize_colspans(pdf, rows)
        if _is_ragged(rows) and not spec.get("force"):
            counts = sorted(set(_cell_row_counts(rows)))
            skipped.append({
                "spec": spec,
                "reason": f"ragged rows (cells per row differ: {counts}) — "
                          "attributing would fail veraPDF 7.2",
            })
            continue
        touched = False

        header_row_idx = None
        if header_row:
            if "header_row_index" in spec:
                try:
                    header_row_idx = int(spec["header_row_index"])
                except (TypeError, ValueError):
                    header_row_idx = None
                if header_row_idx is None or not (0 <= header_row_idx < len(rows)):
                    skipped.append({"spec": spec, "reason": "header_row_index out of range"})
                    header_row = False
            else:
                header_row_idx = _resolve_header_row_index(rows)
                if header_row_idx is None:
                    skipped.append({"spec": spec, "reason": "no cell-bearing row for header_row"})
                    header_row = False

        if header_row:
            for cell in _row_cells(rows[header_row_idx]):
                if _promote(cell, "/Column"):
                    touched = True

        if header_col:
            for i, row in enumerate(rows):
                cells = _row_cells(row)
                if not cells:
                    continue
                corner = header_row and i == header_row_idx
                if _promote(cells[0], "/Column" if corner else "/Row"):
                    touched = True

        if touched:
            tables_touched += 1

    return {
        "tables_touched": tables_touched,
        "cells_promoted": cells_promoted,
        "spans_synthesized": spans_synthesized,
        "skipped": skipped,
    }


# ── Layout-table -> list conversion ─────────────────────────────────────────

def _node_key(n):
    return n.objgen if getattr(n, "is_indirect", False) else id(n)


def _find_parent(pdf, target):
    """Struct-tree parent of `target` (the node whose /K contains it), or
    None if target is a top-level child of StructTreeRoot or not found."""
    root = pdf.Root.get("/StructTreeRoot")
    if root is None:
        return None
    tkey = _node_key(target)
    top = _kids(root)
    if any(_node_key(k) == tkey for k in top):
        return root
    visited, stack = set(), list(top)
    while stack:
        node = stack.pop()
        nkey = _node_key(node)
        if nkey in visited:
            continue
        visited.add(nkey)
        kids = _kids(node)
        if any(_node_key(k) == tkey for k in kids):
            return node
        stack.extend(kids)
    return None


def _find_nested_table(cell):
    """DFS for a /Table struct elem anywhere under a cell — must not be
    silently absorbed into the new /LBody without a record."""
    stack = list(_kids(cell))
    while stack:
        node = stack.pop()
        if str(node.get("/S", "")) == "/Table":
            return node
        stack.extend(_kids(node))
    return None


def _replace_kid(parent, old, new):
    """Swap `old` for `new` in parent's /K (Array or single Dictionary)."""
    okey = _node_key(old)
    k = parent.get("/K")
    if isinstance(k, pikepdf.Array):
        for i, item in enumerate(k):
            if _node_key(item) == okey:
                k[i] = new
                return True
    elif k is not None and _node_key(k) == okey:
        parent["/K"] = new
        return True
    return False


def convert_tables_to_lists(pdf, specs) -> dict:
    """Rewrite a layout /Table (a visual grid with no real row/column
    relationship — e.g. a multi-column word list) to /L -> /LI -> /LBody.
    specs: [{"index": int, "order": "row"|"column"}].

    index = 0-based position in list_tables(pdf) document order (same
    indexing as promote_headers). order defaults to "row" (preserves the
    existing tag order — safe default, no surprise reordering); "column"
    emits items column-major, the authored order for a list that flows down
    columns (see module docstring — the Hinton verb-grid case).

    Opt-in ONLY: a table not named in `specs` is never converted. Converting
    a genuine DATA table to a list would destroy real row/column semantics
    — the agent must inspect the grid (list_tables()'s first_row_text, or a
    page render) before listing it.

    Mechanism per converted table: skip /TR rows with no /TD//TH cells
    (junk rows, same shape promote_headers already skips). Emit one /LI per
    remaining cell in the requested order; the cell's ORIGINAL content
    struct elems are re-parented onto a new /LBody, never rebuilt. Cells
    with no content (/K absent/empty) produce no /LI. A /Caption child of
    the table is preserved as the new /L's first child (not turned into an
    /LI). A nested /Table found inside a cell is left untouched (moved as
    part of that cell's content) and recorded in `skipped` for visibility.
    The new /L replaces the /Table at its position in the parent struct
    element's /K.

    Out-of-range/malformed index, or a table with zero cell-bearing rows /
    zero content cells -> skipped, never raises.

    Returns {"converted": n, "items_created": n, "skipped": [{"spec": ...,
    "reason": ...}]}."""
    tables = _find_tables(pdf)
    converted = 0
    items_created = 0
    skipped = []

    for spec in (specs or []):
        if not isinstance(spec, dict) or "index" not in spec:
            skipped.append({"spec": spec, "reason": "missing index"})
            continue
        try:
            idx = int(spec["index"])
        except (TypeError, ValueError):
            skipped.append({"spec": spec, "reason": "non-integer index"})
            continue
        if not (0 <= idx < len(tables)):
            skipped.append({"spec": spec, "reason": "index out of range"})
            continue
        order = spec.get("order", "row")
        if order not in ("row", "column"):
            skipped.append({"spec": spec, "reason": f"invalid order {order!r}"})
            continue

        table = tables[idx]
        all_kids = _kids(table)
        caption = next((c for c in all_kids if str(c.get("/S", "")) == "/Caption"), None)
        row_nodes = _table_rows(table)
        cell_rows = [_row_cells(r) for r in row_nodes]
        cell_rows = [r for r in cell_rows if r]
        if not cell_rows:
            skipped.append({"spec": spec, "reason": "no cell-bearing rows"})
            continue

        if order == "row":
            ordered_cells = [cell for row in cell_rows for cell in row]
        else:
            max_cols = max(len(r) for r in cell_rows)
            ordered_cells = [row[c] for c in range(max_cols) for row in cell_rows if c < len(row)]

        li_items = []
        for cell in ordered_cells:
            k = cell.get("/K")
            if k is None or (isinstance(k, pikepdf.Array) and len(k) == 0):
                continue  # empty cell -> no /LI
            nested = _find_nested_table(cell)
            if nested is not None:
                skipped.append({"spec": spec, "reason": "cell contains nested table; not converted"})
            body = pdf.make_indirect(pikepdf.Dictionary(
                Type=pikepdf.Name("/StructElem"), S=pikepdf.Name("/LBody"), K=k))
            li = pdf.make_indirect(pikepdf.Dictionary(
                Type=pikepdf.Name("/StructElem"), S=pikepdf.Name("/LI"), K=body))
            body["/P"] = li
            for kid in (k if isinstance(k, pikepdf.Array) else [k]):
                if hasattr(kid, "get") and "/S" in kid:
                    kid["/P"] = body
            if "/Pg" in cell:
                li["/Pg"] = cell["/Pg"]
            li_items.append(li)

        if not li_items:
            skipped.append({"spec": spec, "reason": "no content cells"})
            continue

        parent = _find_parent(pdf, table)
        if parent is None:
            skipped.append({"spec": spec, "reason": "table has no struct-tree parent"})
            continue

        kids = ([caption] if caption is not None else []) + li_items
        lst = pdf.make_indirect(pikepdf.Dictionary(
            Type=pikepdf.Name("/StructElem"), S=pikepdf.Name("/L"),
            K=pikepdf.Array(kids)))
        lst["/P"] = parent
        for li in li_items:
            li["/P"] = lst
        if caption is not None:
            caption["/P"] = lst
        if "/Pg" in table:
            lst["/Pg"] = table["/Pg"]

        if not _replace_kid(parent, table, lst):
            skipped.append({"spec": spec, "reason": "could not locate table in parent /K"})
            continue

        converted += 1
        items_created += len(li_items)

    return {"converted": converted, "items_created": items_created, "skipped": skipped}


# ── Mis-detected-table -> figure conversion ─────────────────────────────────

def _obj_key(obj):
    """Hashable key for a (possibly indirect) pikepdf object — indirect
    pikepdf.Object instances aren't hashable themselves."""
    return obj.objgen if getattr(obj, "is_indirect", False) else id(obj)


def _collect_figure_mcids(node, inherited_pg=None):
    """Recurse `node`'s /K collecting (mcid, page_obj) pairs, EXCLUDING every
    /Caption subtree (returned as a list — a table can carry more than one —
    so callers can re-parent ALL of them onto the new /Figure; dropping any
    but the last would orphan its MCID's ParentTree entry). Any /K entry
    that is a dict with neither /S nor /MCID (an OBJR reference to an
    annotation, e.g. a /Link's /K, or an unrecognized ref) is NEVER int()'d
    — it is collected unchanged in `refs` so the caller can carry it into
    the new /Figure's /K verbatim; content must never be silently dropped
    or crash the walk. `inherited_pg` seeds the page for `node` itself when
    `node` carries no /Pg of its own (default None reproduces the original
    behavior exactly — used by _cell_text to thread a table's /Pg down to
    cells that don't carry their own). Returns (pairs, captions, refs)."""
    pairs = []
    captions = []
    refs = []
    pg = node.get("/Pg", inherited_pg)

    def walk(n, pg):
        pg = n.get("/Pg", pg) if hasattr(n, "get") else pg
        k = n.get("/K") if hasattr(n, "get") else None
        if k is None:
            return
        items = k if isinstance(k, pikepdf.Array) else [k]
        for kid in items:
            if hasattr(kid, "get"):
                if "/S" in kid:
                    if str(kid["/S"]) == "/Caption":
                        captions.append(kid)
                        continue
                    walk(kid, pg)
                elif "/MCID" in kid:  # MCR: dict with /MCID, no /S
                    pairs.append((int(kid["/MCID"]), kid.get("/Pg", pg)))
                else:  # OBJR or unrecognized ref dict — preserve, don't int()
                    refs.append(kid)
            else:
                pairs.append((int(kid), pg))  # bare int MCID

    walk(node, pg)
    return pairs, captions, refs


def convert_tables_to_figures(pdf, specs) -> dict:
    """Rewrite a mis-detected /Table (a visual grid that is actually a
    chart/plot/diagram, not real tabular data — e.g. opendataloader-pdf
    tagging a 2x2 panel of regression residual plots as /Table) to /Figure.
    specs: [{"index": int, "alt": str}].

    index = 0-based position in list_tables(pdf) document order (same
    indexing as promote_headers / convert_tables_to_lists).
    alt is REQUIRED and must be non-empty — a /Figure with no /Alt fails
    veraPDF 7.3. A spec with missing/blank alt is skipped, nothing created.

    Opt-in ONLY: a table not named in `specs` is never converted — the agent
    decides which /Table entries are really figures (and supplies alt text
    from the render) before calling this. Never apply to a genuine data
    table.

    Mechanism: recursively collect the table's descendant marked-content IDs
    (walking /K; a bare int or an MCR dict with /MCID and no /S is an MCID,
    a dict with /S recurses), collecting every /Caption child separately (a
    table can carry more than one — all survive, not just the last) and
    preserving any other reference dict (e.g. an OBJR under a /Link) as-is,
    never int()'d, never dropped. Build one /Figure struct elem whose /K is
    the collected MCIDs + the preserved ref dicts + the captions, with /Alt
    and a representative /Pg (the page most of the MCIDs resolve to). An
    MCID whose own page differs from the figure's representative /Pg is
    emitted as an explicit MCR dict (/Type /MCR /Pg <page> /MCID n) instead
    of a bare int — a bare int always resolves against the struct elem's
    own /Pg, so a multi-page table's off-page MCIDs would otherwise
    silently mis-associate to the wrong page. Replace the /Table at its
    position in the parent struct element's /K. Then remap each moved
    MCID's ParentTree entry (pdf_tree_edit._page_parenttree_array) to point
    at the new /Figure, grouped by page — a page with no flat ParentTree
    entry (no /StructParents, or a nested /Kids number tree) is skipped for
    the remap only, never raised; the /K association alone already
    satisfies veraPDF.

    Index resolved against ALL target tables up front (before any mutation)
    so converting one spec never shifts another spec's index within this
    call. Out-of-range/malformed index, missing/blank alt, or a table with
    zero collectable MCIDs -> skipped, never raises.

    Returns {"converted": n, "figures_created": n, "skipped": [{"spec": ...,
    "reason": ...}]}."""
    tables = _find_tables(pdf)
    converted = 0
    figures_created = 0
    skipped = []

    for spec in (specs or []):
        if not isinstance(spec, dict) or "index" not in spec:
            skipped.append({"spec": spec, "reason": "missing index"})
            continue
        try:
            idx = int(spec["index"])
        except (TypeError, ValueError):
            skipped.append({"spec": spec, "reason": "non-integer index"})
            continue
        if not (0 <= idx < len(tables)):
            skipped.append({"spec": spec, "reason": "index out of range"})
            continue
        alt = spec.get("alt")
        if not isinstance(alt, str) or not alt.strip():
            skipped.append({"spec": spec, "reason": "figure requires non-empty alt text"})
            continue

        table = tables[idx]
        mcid_pairs, captions, refs = _collect_figure_mcids(table)
        if not mcid_pairs:
            skipped.append({"spec": spec, "reason": "no collectable MCIDs"})
            continue

        parent = _find_parent(pdf, table)
        if parent is None:
            skipped.append({"spec": spec, "reason": "table has no struct-tree parent"})
            continue

        # modal page across collected MCIDs -> representative /Pg
        pages = [pg for _, pg in mcid_pairs if pg is not None]
        if pages:
            counts = {}
            for pg in pages:
                counts.setdefault(_obj_key(pg), [0, pg])
                counts[_obj_key(pg)][0] += 1
            rep_pg = max(counts.values(), key=lambda kv: kv[0])[1]
        else:
            rep_pg = table.get("/Pg")
        rep_key = _obj_key(rep_pg) if rep_pg is not None else None

        k_items = []
        for mcid, mpg in mcid_pairs:
            if mpg is not None and rep_key is not None and _obj_key(mpg) != rep_key:
                k_items.append(pikepdf.Dictionary(
                    Type=pikepdf.Name("/MCR"), Pg=mpg, MCID=pikepdf.Integer(mcid)))
            else:
                k_items.append(pikepdf.Integer(mcid))
        k_items.extend(refs)
        k_items.extend(captions)

        fig = pdf.make_indirect(pikepdf.Dictionary(
            Type=pikepdf.Name("/StructElem"), S=pikepdf.Name("/Figure"),
            Alt=pikepdf.String(alt), K=pikepdf.Array(k_items)))
        if rep_pg is not None:
            fig["/Pg"] = rep_pg
        for cap in captions:
            cap["/P"] = fig
        fig["/P"] = parent

        if not _replace_kid(parent, table, fig):
            skipped.append({"spec": spec, "reason": "could not locate table in parent /K"})
            continue

        # ParentTree remap, grouped by page; best-effort, never raises.
        by_page = {}
        for mcid, pg in mcid_pairs:
            if pg is None:
                continue
            by_page.setdefault(_obj_key(pg), [pg, []])[1].append(mcid)
        for pg, mcids in by_page.values():
            page_obj = None
            for p in pdf.pages:
                if p.obj == pg:
                    page_obj = p
                    break
            if page_obj is None:
                continue
            pt = pdf_tree_edit._page_parenttree_array(pdf, page_obj)
            if pt is None:
                continue  # ponytail: no flat ParentTree entry; /K association still validates
            for mcid in mcids:
                if 0 <= mcid < len(pt):
                    pt[mcid] = fig

        converted += 1
        figures_created += 1

    return {"converted": converted, "figures_created": figures_created, "skipped": skipped}


# ── Table-fragment merging ───────────────────────────────────────────────────

def _table_row_group(table, tag):
    """The first direct child of `table` whose /S equals `tag` (e.g.
    "/TBody"), or None. merge_tables uses this to find the survivor's
    EXISTING row-group container — it never creates one the survivor
    didn't already have (see merge_tables' docstring)."""
    return next((k for k in _kids(table) if str(k.get("/S", "")) == tag), None)


def _table_row_parents(table):
    """[(row, direct_parent)] in document order — direct_parent is `table`
    itself for a loose /TR child, or the /THead|/TBody|/TFoot group that
    directly holds it. Same row-group-transparent traversal/filtering as
    _table_rows (reused, not reinvented), but also returns each row's own
    immediate container so merge_tables can detach a row precisely without
    disturbing sibling rows left behind in the same group."""
    out = []
    for kid in _kids(table):
        tag = str(kid.get("/S", ""))
        if tag == "/TR":
            out.append((kid, table))
        elif tag in ("/THead", "/TBody", "/TFoot"):
            out.extend((c, kid) for c in _kids(kid) if str(c.get("/S", "")) == "/TR")
    return out


def _remove_kid(parent, target):
    """Remove `target` from parent's /K (Array or single Dictionary).
    Returns True if found and removed, False otherwise — never raises."""
    tkey = _node_key(target)
    k = parent.get("/K")
    if isinstance(k, pikepdf.Array):
        for i, item in enumerate(k):
            if _node_key(item) == tkey:
                del k[i]
                return True
    elif k is not None and _node_key(k) == tkey:
        del parent["/K"]
        return True
    return False


def _append_kid(parent, child):
    """Append `child` onto parent's /K, creating the array or upgrading a
    single-Dictionary /K to a 2-element array as needed — same upgrade
    pattern _table_attrs already uses for /A."""
    k = parent.get("/K")
    if k is None:
        parent["/K"] = pikepdf.Array([child])
    elif isinstance(k, pikepdf.Array):
        k.append(child)
    else:
        parent["/K"] = pikepdf.Array([k, child])


def _append_caption(table, caption):
    """Append a donor's /Caption onto `table`'s /K, positioned after any
    of the survivor's own existing /Caption children and before its row
    content — keeps captions grouped at the front, the conventional
    tagged-table shape, rather than trailing after every row."""
    k = table.get("/K")
    if k is None:
        table["/K"] = pikepdf.Array([caption])
        return
    if not isinstance(k, pikepdf.Array):
        k = pikepdf.Array([k])
        table["/K"] = k
    insert_at = 0
    for item in k:
        if hasattr(item, "get") and str(item.get("/S", "")) == "/Caption":
            insert_at += 1
        else:
            break
    k.insert(insert_at, caption)


def _table_effective_width(rows):
    """Single effective-column-count number for a table's cell-bearing
    rows: the max of _effective_col_counts (the span-aware helper
    promote_headers' ragged check already uses — reused here, not
    re-derived). 0 for a table with no cell-bearing rows. merge_tables
    compares this across the fragments named in a spec and refuses to
    merge fragments of differing width (see its docstring)."""
    widths = _effective_col_counts(rows)
    return max(widths) if widths else 0


def _collect_mcid_holders(node, inherited_pg=None):
    """[(mcid, page, holder)] for every bare-int or MCR-dict MCID reachable
    under `node`'s /K (recursing into any /S child) — holder is the struct
    elem that directly contains that MCID in its own /K, i.e. the object a
    ParentTree entry must point at. Unlike _collect_figure_mcids, /Caption
    subtrees are NOT excluded — merge_tables calls this once per moved row
    and once per moved caption individually, so there is no ambiguity
    between the two to resolve here."""
    triples = []
    pg0 = node.get("/Pg", inherited_pg) if hasattr(node, "get") else inherited_pg

    def walk(n, pg):
        pg = n.get("/Pg", pg) if hasattr(n, "get") else pg
        k = n.get("/K") if hasattr(n, "get") else None
        if k is None:
            return
        items = k if isinstance(k, pikepdf.Array) else [k]
        for kid in items:
            if hasattr(kid, "get"):
                if "/S" in kid:
                    walk(kid, pg)
                elif "/MCID" in kid:
                    triples.append((int(kid["/MCID"]), kid.get("/Pg", pg), n))
                # else: OBJR/unrecognized ref dict — no MCID, nothing to remap
            else:
                triples.append((int(kid), pg, n))

    walk(node, pg0)
    return triples


def _remap_parenttree(pdf, triples):
    """Point each (mcid, page, holder) triple's ParentTree entry at
    `holder`, grouped by page. A page with no flat ParentTree entry (no
    /StructParents, or a nested /Kids number tree) is skipped for that
    page's triples only — never raised, mirrors
    convert_tables_to_figures' remap exactly. Best-effort by construction:
    merge_tables re-parents EXISTING struct elems rather than rebuilding
    them, so the struct elem that directly holds a moved MCID never
    changes identity — `holder` is almost always already the value on
    disk. This call is the explicit, defensive version of that invariant,
    not a fix for a bug the move mechanism could introduce."""
    by_page = {}
    for mcid, pg, holder in triples:
        if pg is None:
            continue
        by_page.setdefault(_obj_key(pg), [pg, []])[1].append((mcid, holder))
    for pg, items in by_page.values():
        page_obj = next((p for p in pdf.pages if p.obj == pg), None)
        if page_obj is None:
            continue
        arr = pdf_tree_edit._page_parenttree_array(pdf, page_obj)
        if arr is None:
            continue  # ponytail: no flat ParentTree entry; /K association still validates
        for mcid, holder in items:
            if 0 <= mcid < len(arr):
                arr[mcid] = holder


def merge_tables(pdf, specs) -> dict:
    """Join two or more sibling /Table fragments that are really ONE
    logical table into a single /Table, by re-parenting rows — never
    rebuilding them. specs: [{"indices": [int, int, ...], "force": bool}].

    indices are 0-based positions in list_tables(pdf) document order (same
    indexing as promote_headers / convert_tables_to_lists /
    convert_tables_to_figures), given in the order the fragments should be
    joined — TWO OR MORE required. The FIRST index survives (the merged
    table keeps its identity; its own rows stay first, donor rows follow in
    the given order); every other index is a donor: its /TR rows are moved
    onto the survivor in document order, its /Caption (if any) is preserved
    by moving it onto the survivor too, and the emptied donor /Table is
    then removed from its struct-tree parent's /K entirely — no orphaned
    /THead/TBody/TFoot shell is left reachable, because the whole donor
    subtree (including any such shell still nested inside it) goes
    unreachable the instant the donor itself is detached.

    ROW-GROUP DESTINATION: a donor's rows land in the survivor's EXISTING
    /TBody when it has one, else directly under the survivor's own /K —
    this never CREATES a /TBody or /THead the survivor didn't already have.
    A donor's own /THead//TBody//TFoot grouping (if it had one) is not
    itself preserved; only its /TR children are moved, matching
    _table_rows' row-group-transparent traversal (reused here via
    _table_row_parents, the parent-tracking sibling _table_rows doesn't
    provide, needed to detach a row from its specific origin — a group or
    the table itself — precisely, without disturbing rows left behind).

    REFUSED BY DEFAULT — RAGGED-MERGE GUARD: if the named fragments'
    effective column counts differ (via _effective_col_counts, the same
    span-aware helper promote_headers' ragged check already uses — not
    re-derived here), the merge is refused: joining tables of different
    widths produces a ragged table, which trips veraPDF clause 7.2 the
    moment anything attributes it. "force": true overrides for a fragment
    set that is genuinely different widths on purpose.

    ALSO REFUSED, never raises: fewer than two indices; any index out of
    range; a repeated index within the same spec entry. A fragment absent
    from `specs` is never touched — opt-in only, same rule as
    promote_headers / convert_tables_to_lists / convert_tables_to_figures.

    MERGE ORDER IS READING ORDER — READ THIS BEFORE CALLING: the order
    `indices` are given in is the exact order rows land in the merged
    table, and for a tagged PDF that IS the screen-reader reading order.
    Get the order wrong and the merge silently produces a table that reads
    out of sequence — and nothing else in this codebase can catch it:
    veraPDF only checks structural validity (a reordered table is still a
    perfectly well-formed table), and the fidelity gate only checks
    rendered pixels (a merge moves content in the tag tree, never on the
    page, so the pixels never change). A wrong order is therefore
    UNDETECTABLE by every gate this pipeline runs — the only defense is
    supplying the right order in the first place. Document order is
    almost always reading order, so ASCENDING-ORDER GUARD: when `indices`
    are not strictly ascending, the merge is refused by default (a
    `skipped` entry, never an exception) — same shape as the ragged-merge
    guard above. Pass "force": true on a spec that genuinely needs a
    non-ascending join (e.g. a survivor chosen for its header row that
    happens to sort after its donors); `force` still also covers the
    ragged-width guard, so a single "force": true can bypass both checks
    at once when both apply. The agent must inspect the fragments (page
    order, first_row_text, page_context) and choose an order that actually
    reads correctly before merging — this guard catches the common
    mistake, not every possible one.

    PARENTTREE: since rows (and their /TD/P/MCID-bearing descendants) are
    moved as the SAME struct-elem objects rather than rebuilt, the struct
    elem that directly holds each moved MCID never changes identity — a
    correctly-wired ParentTree entry already points at the right object
    both before and after the move. merge_tables still walks every moved
    row's (and moved caption's) MCIDs and rewrites their ParentTree entry
    explicitly, grouped by page, mirroring convert_tables_to_figures' own
    remap — defensive, not a fix for a bug this move mechanism could
    introduce. A page with no flat ParentTree entry (no /StructParents, or
    a nested /Kids number tree) is skipped for that page's entries only,
    never raised — matches convert_tables_to_figures exactly.

    INDEX SAFETY ACROSS CALLS: merging REMOVES /Table elements, so every
    later list_tables() index shifts — the same hazard table_to_list and
    table_to_figure already carry. This is a two-pass operation: merge,
    re-run list_tables() (or remediate_assess) for fresh indices, THEN
    promote headers or run another conversion on the surviving tables in a
    second call. scripts/remediate_pdf.py's _validate_judgment enforces the
    single-judgment half of this: table_merge cannot co-occur with
    table_headers, table_to_list, or table_to_figure in one judgment.json.

    Returns {"tables_merged": n, "rows_moved": n, "donors_removed": n,
    "skipped": [{"spec": ..., "reason": ...}]} — tables_merged counts
    successful spec entries (one entry may join more than two fragments)."""
    tables = _find_tables(pdf)
    tables_merged = 0
    rows_moved = 0
    donors_removed = 0
    skipped = []

    for spec in (specs or []):
        if not isinstance(spec, dict) or "indices" not in spec:
            skipped.append({"spec": spec, "reason": "missing indices"})
            continue
        indices = spec["indices"]
        if not isinstance(indices, list) or not all(isinstance(i, int) for i in indices):
            skipped.append({"spec": spec, "reason": "indices must be a list of int"})
            continue
        if len(indices) < 2:
            skipped.append({"spec": spec, "reason": "at least two indices required"})
            continue
        if len(set(indices)) != len(indices):
            skipped.append({"spec": spec, "reason": "duplicate indices"})
            continue
        if not all(0 <= i < len(tables) for i in indices):
            skipped.append({"spec": spec, "reason": "index out of range"})
            continue
        if indices != sorted(indices) and not spec.get("force"):
            skipped.append({
                "spec": spec,
                "reason": f"indices {indices} are not in ascending document order — "
                          "a non-ascending merge reorders the table's reading order, "
                          "and NO gate this pipeline runs can detect the result "
                          "(veraPDF only checks structural validity; the fidelity "
                          "gate only checks rendered pixels, which a tag-tree-only "
                          "merge never changes). Pass indices in intended reading "
                          "order, or \"force\": true to override deliberately.",
            })
            continue

        frag_tables = [tables[i] for i in indices]
        frag_rows = [_table_rows(t) for t in frag_tables]
        widths = [_table_effective_width(r) for r in frag_rows]
        if len(set(widths)) > 1 and not spec.get("force"):
            skipped.append({
                "spec": spec,
                "reason": f"fragment effective column counts differ: {widths} — "
                          "merging would produce a ragged table",
            })
            continue

        survivor = frag_tables[0]
        donors = frag_tables[1:]
        survivor_tbody = _table_row_group(survivor, "/TBody")
        destination = survivor_tbody if survivor_tbody is not None else survivor

        remap_triples = []
        for donor in donors:
            for row, parent in _table_row_parents(donor):
                remap_triples.extend(_collect_mcid_holders(row, inherited_pg=donor.get("/Pg")))
                _remove_kid(parent, row)
                _append_kid(destination, row)
                row["/P"] = destination
                rows_moved += 1

            donor_caption = next(
                (c for c in _kids(donor) if str(c.get("/S", "")) == "/Caption"), None)
            if donor_caption is not None:
                remap_triples.extend(
                    _collect_mcid_holders(donor_caption, inherited_pg=donor.get("/Pg")))
                _remove_kid(donor, donor_caption)
                _append_caption(survivor, donor_caption)
                donor_caption["/P"] = survivor

            donor_parent = _find_parent(pdf, donor)
            if donor_parent is not None:
                _remove_kid(donor_parent, donor)
            donors_removed += 1

        _remap_parenttree(pdf, remap_triples)
        tables_merged += 1

    return {
        "tables_merged": tables_merged,
        "rows_moved": rows_moved,
        "donors_removed": donors_removed,
        "skipped": skipped,
    }


def main():
    parser = argparse.ArgumentParser(description="PDF table header promotion / list conversion")
    parser.add_argument("--input", required=True)
    parser.add_argument("--output")
    parser.add_argument("--list", action="store_true", help="print list_tables() as JSON")
    parser.add_argument("--specs", help="JSON file of promote_headers specs")
    parser.add_argument("--to-list", help="JSON file of convert_tables_to_lists specs")
    parser.add_argument("--to-figure", help="JSON file of convert_tables_to_figures specs")
    args = parser.parse_args()

    pdf = pikepdf.open(args.input)
    if args.list:
        print(json.dumps(list_tables(pdf, path=args.input), indent=2))
        pdf.close()
        return

    if args.to_list:
        if not args.output:
            print("ERROR: --output required with --to-list", file=sys.stderr)
            pdf.close()
            sys.exit(2)
        specs = json.loads(Path(args.to_list).read_text(encoding="utf-8"))
        result = convert_tables_to_lists(pdf, specs)
        pdf.save(args.output)
        pdf.close()
        print(json.dumps(result, indent=2))
        return

    if args.to_figure:
        if not args.output:
            print("ERROR: --output required with --to-figure", file=sys.stderr)
            pdf.close()
            sys.exit(2)
        specs = json.loads(Path(args.to_figure).read_text(encoding="utf-8"))
        result = convert_tables_to_figures(pdf, specs)
        pdf.save(args.output)
        pdf.close()
        print(json.dumps(result, indent=2))
        return

    if not args.output or not args.specs:
        print("ERROR: --output and --specs required unless --list", file=sys.stderr)
        pdf.close()
        sys.exit(2)
    specs = json.loads(Path(args.specs).read_text(encoding="utf-8"))
    result = promote_headers(pdf, specs)
    pdf.save(args.output)
    pdf.close()
    print(json.dumps(result, indent=2))


if __name__ == "__main__":
    main()
