"""Tests for scripts/judgment_eval.py — AI table-classification accuracy harness.

CI-light: pure pikepdf fixtures built in-memory + saved to tmp_path, no
vendored ODL/java/verapdf, no LLM calls (there is no LLM in this module).
"""

import json
import math
import os
import sys
from pathlib import Path

import pikepdf
import pytest

sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts"))

import judgment_eval as je  # noqa: E402


# ── fixtures (mirrors tests/test_table_headers.py / test_table_to_figure.py) ──

def _cell(pdf, text, pg):
    kids = []
    if text is not None:
        p = pdf.make_indirect(pikepdf.Dictionary(
            {"/Type": pikepdf.Name("/StructElem"), "/S": pikepdf.Name("/P"),
             "/ActualText": pikepdf.String(text)}))
        kids.append(p)
    return pdf.make_indirect(pikepdf.Dictionary({
        "/Type": pikepdf.Name("/StructElem"), "/S": pikepdf.Name("/TD"),
        "/K": pikepdf.Array(kids), "/Pg": pg,
    }))


def _row(pdf, cells):
    return pdf.make_indirect(pikepdf.Dictionary({
        "/Type": pikepdf.Name("/StructElem"), "/S": pikepdf.Name("/TR"),
        "/K": pikepdf.Array(cells),
    }))


def _table(pdf, rows):
    return pdf.make_indirect(pikepdf.Dictionary({
        "/Type": pikepdf.Name("/StructElem"), "/S": pikepdf.Name("/Table"),
        "/K": pikepdf.Array(rows),
    }))


def _build_pdf(path):
    """1-page doc, 2 /Table struct elems, cells carry /Pg so list_tables()
    resolves a page number."""
    pdf = pikepdf.new()
    page = pdf.add_blank_page(page_size=(200, 200))
    t0 = _table(pdf, [
        _row(pdf, [_cell(pdf, "Name", page.obj), _cell(pdf, "Score", page.obj)]),
        _row(pdf, [_cell(pdf, "Alice", page.obj), _cell(pdf, "9", page.obj)]),
    ])
    t1 = _table(pdf, [
        _row(pdf, [_cell(pdf, "apple", page.obj), _cell(pdf, "banana", page.obj)]),
    ])
    doc_elem = pdf.make_indirect(pikepdf.Dictionary({
        "/Type": pikepdf.Name("/StructElem"), "/S": pikepdf.Name("/Document"),
        "/K": pikepdf.Array([t0, t1]),
    }))
    root = pikepdf.Dictionary({
        "/Type": pikepdf.Name("/StructTreeRoot"), "/K": doc_elem,
    })
    pdf.Root.StructTreeRoot = root
    pdf.save(str(path))


# ── candidates mode ──────────────────────────────────────────────────────

def test_candidates_one_record_per_table(tmp_path):
    pdf_path = tmp_path / "sample.pdf"
    _build_pdf(pdf_path)

    candidates, pdf_files = je.build_candidates([str(pdf_path)])

    assert pdf_files == [pdf_path.resolve()]
    assert len(candidates) == 2
    for rec in candidates:
        assert "error" not in rec
        assert rec["file"]
        assert rec["index"] in (0, 1)
        assert rec["page"] == 1


def test_candidates_corrupt_pdf_yields_error_and_does_not_abort(tmp_path):
    good = tmp_path / "good.pdf"
    _build_pdf(good)
    bad = tmp_path / "bad.pdf"
    bad.write_bytes(b"not a pdf at all")

    # sorted() order is deterministic (alphabetical): bad.pdf before good.pdf
    candidates, pdf_files = je.build_candidates([str(bad), str(good)])

    assert len(pdf_files) == 2
    bad_records = [c for c in candidates if c.get("file", "").endswith("bad.pdf")]
    assert len(bad_records) == 1
    assert "error" in bad_records[0]
    assert "index" not in bad_records[0]

    good_records = [c for c in candidates if c.get("file", "").endswith("good.pdf")]
    assert len(good_records) == 2
    assert all("error" not in r for r in good_records)


# ── --render-dir mode ────────────────────────────────────────────────────

def _build_two_page_pdf(path):
    """2-page doc, 1 /Table whose header row repeats on page 2 (the real
    NYU-Torts shape this feature exists for) — cells on rows 0-1 carry
    page 1's /Pg, rows 2-3 carry page 2's, so list_tables() reports
    pages == [1, 2]."""
    pdf = pikepdf.new()
    page1 = pdf.add_blank_page(page_size=(200, 200))
    page2 = pdf.add_blank_page(page_size=(200, 200))
    t0 = _table(pdf, [
        _row(pdf, [_cell(pdf, "Name", page1.obj), _cell(pdf, "Score", page1.obj)]),
        _row(pdf, [_cell(pdf, "Alice", page1.obj), _cell(pdf, "9", page1.obj)]),
        _row(pdf, [_cell(pdf, "Name", page2.obj), _cell(pdf, "Score", page2.obj)]),
        _row(pdf, [_cell(pdf, "Bob", page2.obj), _cell(pdf, "7", page2.obj)]),
    ])
    doc_elem = pdf.make_indirect(pikepdf.Dictionary({
        "/Type": pikepdf.Name("/StructElem"), "/S": pikepdf.Name("/Document"),
        "/K": pikepdf.Array([t0]),
    }))
    root = pikepdf.Dictionary({
        "/Type": pikepdf.Name("/StructTreeRoot"), "/K": doc_elem,
    })
    pdf.Root.StructTreeRoot = root
    pdf.save(str(path))


def _build_pdf_no_page_cells(path):
    """1-page doc, 1 /Table whose cells carry no /Pg at all -> list_tables()
    resolves page=None and pages=[]."""
    pdf = pikepdf.new()
    pdf.add_blank_page(page_size=(200, 200))

    def cell_no_pg(text):
        p = pdf.make_indirect(pikepdf.Dictionary(
            {"/Type": pikepdf.Name("/StructElem"), "/S": pikepdf.Name("/P"),
             "/ActualText": pikepdf.String(text)}))
        return pdf.make_indirect(pikepdf.Dictionary({
            "/Type": pikepdf.Name("/StructElem"), "/S": pikepdf.Name("/TD"),
            "/K": pikepdf.Array([p]),
        }))

    t0 = _table(pdf, [_row(pdf, [cell_no_pg("apple"), cell_no_pg("banana")])])
    doc_elem = pdf.make_indirect(pikepdf.Dictionary({
        "/Type": pikepdf.Name("/StructElem"), "/S": pikepdf.Name("/Document"),
        "/K": pikepdf.Array([t0]),
    }))
    root = pikepdf.Dictionary({
        "/Type": pikepdf.Name("/StructTreeRoot"), "/K": doc_elem,
    })
    pdf.Root.StructTreeRoot = root
    pdf.save(str(path))


def test_render_dir_writes_one_png_per_page_and_lists_them_in_order(tmp_path):
    pdf_path = tmp_path / "sample.pdf"
    _build_two_page_pdf(pdf_path)
    render_dir = tmp_path / "renders"

    candidates, _ = je.build_candidates([str(pdf_path)], render_dir=str(render_dir))

    assert len(candidates) == 1
    rec = candidates[0]
    assert rec["pages"] == [1, 2]
    assert len(rec["renders"]) == 2
    for page, png_path in zip(rec["pages"], rec["renders"]):
        expected = render_dir / f"{pdf_path.stem}_t{rec['index']}_p{page}.png"
        assert png_path == str(expected)
        assert expected.exists()


def test_render_dir_render_key_still_points_at_first_page_png(tmp_path):
    """Regression: `render` must keep meaning what it always meant (the
    first rendered page) even though `renders` is now the full list."""
    pdf_path = tmp_path / "sample.pdf"
    _build_pdf(pdf_path)
    render_dir = tmp_path / "renders"

    candidates, _ = je.build_candidates([str(pdf_path)], render_dir=str(render_dir))

    for rec in candidates:
        assert rec["pages"] == [1]
        assert rec["renders"] == [rec["render"]]
        assert Path(rec["render"]).exists()


def test_render_dir_no_pages_produces_no_render_files_and_does_not_raise(tmp_path):
    pdf_path = tmp_path / "sample.pdf"
    _build_pdf_no_page_cells(pdf_path)
    render_dir = tmp_path / "renders"

    candidates, _ = je.build_candidates([str(pdf_path)], render_dir=str(render_dir))

    assert len(candidates) == 1
    rec = candidates[0]
    assert rec["pages"] == []
    assert "render" not in rec
    assert "renders" not in rec
    assert list(render_dir.iterdir()) == []


# ── scoring math ─────────────────────────────────────────────────────────

def _fixture_labels():
    return [
        {"file": "fileA", "index": 0, "truth": "data_table"},
        {"file": "fileA", "index": 1, "truth": "data_table"},
        {"file": "fileA", "index": 2, "truth": "layout_table"},
        {"file": "fileB", "index": 0, "truth": "figure"},
        {"file": "fileB", "index": 1, "truth": "other"},
    ]


def _fixture_judgments():
    return {
        "fileA": {"0": "data_table", "1": "layout_table", "2": "unsure"},
        # fileB index 1 intentionally absent -> abstain-by-absence
        # fileB index 2 has no label -> unlabeled_judgments
        "fileB": {"0": "figure", "2": "data_table"},
    }


def test_scoring_matches_hand_computed_metrics():
    m = je.score(_fixture_labels(), _fixture_judgments())

    # Hand computation:
    # item1 fileA/0 data_table judged data_table -> correct, acted
    # item2 fileA/1 data_table judged layout_table -> wrong, acted, HARMFUL
    # item3 fileA/2 layout_table judged unsure -> abstain
    # item4 fileB/0 figure judged figure -> correct, acted
    # item5 fileB/1 other judged <absent> -> abstain
    assert m["total"] == 5
    assert m["files"] == 2  # fileA, fileB
    assert m["mean_tables_per_file"] == pytest.approx(5 / 2)
    assert m["abstained"] == 2
    assert m["acted"] == 3
    assert m["acted_files"] == 2  # both fileA and fileB have >=1 acted item
    assert m["correct"] == 2
    assert m["acted_accuracy"] == pytest.approx(2 / 3)
    assert m["coverage"] == pytest.approx(3 / 5)
    assert m["harmful"] == 1

    assert m["confusion"] == {
        "data_table": {"data_table": 1, "layout_table": 1},
        "layout_table": {"abstain": 1},
        "figure": {"figure": 1},
        "other": {"abstain": 1},
    }

    assert m["per_class"] == {
        "data_table": {"n": 2, "correct": 1, "accuracy": pytest.approx(0.5)},
        "figure": {"n": 1, "correct": 1, "accuracy": pytest.approx(1.0)},
    }
    # layout_table and other have zero ACTED items, so they're absent from
    # per_class (which is scoped to acted items only, per spec).
    assert "layout_table" not in m["per_class"]
    assert "other" not in m["per_class"]

    # Independently hand-derived Wilson 95% interval for phat=2/3, n=3
    # (computed with the standard closed-form formula, not by re-calling
    # je.wilson95 for the expected value).
    z = 1.959963984540054
    phat, n = 2 / 3, 3
    z2 = z * z
    denom = 1 + z2 / n
    center = phat + z2 / (2 * n)
    margin = z * math.sqrt((phat * (1 - phat) / n) + (z2 / (4 * n * n)))
    expected_lo = (center - margin) / denom
    expected_hi = (center + margin) / denom
    assert expected_lo == pytest.approx(0.20765960080204768)
    assert expected_hi == pytest.approx(0.9385080552796037)
    assert m["wilson95"][0] == pytest.approx(expected_lo)
    assert m["wilson95"][1] == pytest.approx(expected_hi)

    assert m["unlabeled_judgments"] == [{"file": "fileB", "index": 2}]


def test_abstention_via_unsure_and_via_absence_are_identical():
    labels = [
        {"file": "f", "index": 0, "truth": "data_table"},
        {"file": "f", "index": 1, "truth": "figure"},
    ]
    judgments_unsure = {"f": {"0": "data_table", "1": "unsure"}}
    judgments_absent = {"f": {"0": "data_table"}}

    m_unsure = je.score(labels, judgments_unsure)
    m_absent = je.score(labels, judgments_absent)

    assert m_unsure == m_absent
    assert m_unsure["abstained"] == 1
    assert m_unsure["acted"] == 1


def test_abstention_lowers_coverage_never_lowers_acted_accuracy():
    labels = [
        {"file": "f", "index": 0, "truth": "data_table"},
        {"file": "f", "index": 1, "truth": "data_table"},
    ]
    # Version A: both acted, one wrong.
    judged_both = {"f": {"0": "data_table", "1": "layout_table"}}
    # Version B: the wrong one abstains instead.
    judged_abstain_wrong = {"f": {"0": "data_table", "1": "unsure"}}

    m_both = je.score(labels, judged_both)
    m_abstain = je.score(labels, judged_abstain_wrong)

    assert m_both["coverage"] == pytest.approx(1.0)
    assert m_abstain["coverage"] == pytest.approx(0.5)
    assert m_abstain["coverage"] < m_both["coverage"]

    assert m_both["acted_accuracy"] == pytest.approx(0.5)
    assert m_abstain["acted_accuracy"] == pytest.approx(1.0)
    assert m_abstain["acted_accuracy"] >= m_both["acted_accuracy"]


def test_harmful_is_wrong_and_actionable_not_scoped_to_data_table_truth():
    """harmful = acted AND judged != truth AND judged != 'other'. Harm is a
    property of the ACTION (every non-'other' class drives a structural PDF
    rewrite — promote_headers, convert_tables_to_lists, or
    convert_tables_to_figures), not of the truth class. Covers the cells
    the old (truth==data_table only) definition missed: layout_table-
    >figure, other->figure, layout_table->data_table, other->layout_table.
    A wrong 'other' (no action taken) is a miss, not harmful."""
    labels = [
        {"file": "f", "index": 0, "truth": "data_table"},
        {"file": "f", "index": 1, "truth": "data_table"},
        {"file": "f", "index": 2, "truth": "layout_table"},
        {"file": "f", "index": 3, "truth": "other"},
        {"file": "f", "index": 4, "truth": "layout_table"},
        {"file": "f", "index": 5, "truth": "other"},
        {"file": "f", "index": 6, "truth": "data_table"},
        {"file": "f", "index": 7, "truth": "data_table"},
    ]
    judgments = {
        "f": {
            "0": "layout_table",  # harmful: data_table -> layout_table (old-def case, retained)
            "1": "figure",        # harmful: data_table -> figure (old-def case, retained)
            "2": "figure",        # harmful (NEW): layout_table -> figure
            "3": "figure",        # harmful (NEW): other -> figure
            "4": "data_table",    # harmful (NEW): layout_table -> data_table
            "5": "layout_table",  # harmful (NEW): other -> layout_table
            "6": "other",         # NOT harmful: wrong, but judged=='other' -> a miss, not a rewrite
            "7": "data_table",    # NOT harmful: correct
        }
    }
    m = je.score(labels, judgments)
    assert m["harmful"] == 6


def test_layout_wrapper_is_a_valid_truth_and_judgment_class():
    """layout_wrapper: a single-cell bordered box holding one paragraph,
    used purely for emphasis — not a data table (no rows/cols) and not a
    layout_table either (nothing to list). No unwrap-to-paragraph engine
    exists yet, so it must be accepted as a class without raising."""
    assert "layout_wrapper" in je.TRUTH_CLASSES
    assert "layout_wrapper" in je.JUDGMENT_CLASSES
    labels = [{"file": "f", "index": 0, "truth": "layout_wrapper"}]
    judgments = {"f": {"0": "layout_wrapper"}}
    m = je.score(labels, judgments)
    assert m["acted"] == 1
    assert m["correct"] == 1
    assert m["harmful"] == 0


def test_judged_layout_wrapper_when_wrong_is_not_harmful():
    """judged == 'layout_wrapper' but truth is something else: a MISS, not
    harmful — same treatment as judged == 'other', because no unwrap engine
    exists yet to have done any structural rewrite."""
    labels = [{"file": "f", "index": 0, "truth": "data_table"}]
    judgments = {"f": {"0": "layout_wrapper"}}
    m = je.score(labels, judgments)
    assert m["harmful"] == 0


def test_truth_layout_wrapper_misjudged_into_actionable_class_is_harmful():
    """truth == 'layout_wrapper' but judged into a class that DOES drive a
    real rewrite (e.g. data_table -> promote_headers runs): still harmful,
    same as any other truth class misjudged into an actionable one."""
    labels = [{"file": "f", "index": 0, "truth": "layout_wrapper"}]
    judgments = {"f": {"0": "data_table"}}
    m = je.score(labels, judgments)
    assert m["harmful"] == 1


def test_wilson95_contains_point_estimate_and_stays_in_unit_interval():
    for phat, n in [(0.0, 10), (1.0, 10), (0.5, 1), (0.8, 50), (0.3, 3)]:
        lo, hi = je.wilson95(phat, n)
        assert 0.0 <= lo <= hi <= 1.0
        # Wilson interval always brackets the point estimate (allow float slop).
        assert lo - 1e-9 <= phat <= hi + 1e-9


def _clustered_labels_and_judgments():
    """5 files x 4 tables = 20 acted items, true accuracy 0.6 (12/20).
    3 files are entirely correct, 2 files are entirely wrong — errors
    cluster hard by file (rho=1), the shape the reviewer's simulation says
    makes a per-item Wilson interval falsely narrow."""
    labels = []
    judgments = {}
    correct_files = ["c0", "c1", "c2"]
    wrong_files = ["w0", "w1"]
    for f in correct_files:
        judgments[f] = {}
        for i in range(4):
            labels.append({"file": f, "index": i, "truth": "data_table"})
            judgments[f][str(i)] = "data_table"  # correct
    for f in wrong_files:
        judgments[f] = {}
        for i in range(4):
            labels.append({"file": f, "index": i, "truth": "data_table"})
            judgments[f][str(i)] = "layout_table"  # wrong
    return labels, judgments


def test_cluster_bootstrap_is_deterministic_under_fixed_seed():
    labels, judgments = _clustered_labels_and_judgments()
    m1 = je.score(labels, judgments, seed=777)
    m2 = je.score(labels, judgments, seed=777)
    assert m1["cluster95"] == m2["cluster95"]
    assert m1["cluster95"] is not None


def test_clustered_data_yields_wider_cluster95_than_uncorrected_wilson95():
    labels, judgments = _clustered_labels_and_judgments()
    m = je.score(labels, judgments, seed=12345)

    assert m["acted"] == 20
    assert m["acted_files"] == 5
    assert m["acted_accuracy"] == pytest.approx(0.6)

    wilson_lo, wilson_hi = m["wilson95"]
    cluster_lo, cluster_hi = m["cluster95"]
    wilson_width = wilson_hi - wilson_lo
    cluster_width = cluster_hi - cluster_lo

    # Wilson treats this as n=20 independent trials and is falsely
    # confident; the cluster bootstrap sees only 5 real clusters and must
    # report a substantially wider interval on the same underlying data.
    assert cluster_width > wilson_width
    # Sanity: both intervals still bracket the point estimate and stay in [0,1].
    assert 0.0 <= wilson_lo <= 0.6 <= wilson_hi <= 1.0
    assert 0.0 <= cluster_lo <= 0.6 <= cluster_hi <= 1.0


def test_cluster95_null_when_fewer_than_two_acted_files():
    # Single acted file -> bootstrap over 1 cluster is meaningless.
    labels = [
        {"file": "only_file", "index": 0, "truth": "data_table"},
        {"file": "only_file", "index": 1, "truth": "data_table"},
    ]
    judgments = {"only_file": {"0": "data_table", "1": "layout_table"}}
    m = je.score(labels, judgments)
    assert m["acted_files"] == 1
    assert m["cluster95"] is None
    # acted_accuracy/wilson95 are still real numbers here — only the
    # cluster bootstrap is undefined with <2 clusters.
    assert m["acted_accuracy"] == pytest.approx(0.5)
    assert m["wilson95"] is not None

    # Zero acted files (nothing acted at all) is also null.
    labels_zero = [{"file": "f", "index": 0, "truth": "data_table"}]
    judgments_zero = {"f": {"0": "unsure"}}
    m_zero = je.score(labels_zero, judgments_zero)
    assert m_zero["acted_files"] == 0
    assert m_zero["cluster95"] is None


def test_acted_zero_returns_none_accuracy_without_raising():
    """Zero acted items is zero evidence, not zero accuracy — a 0.0%
    point estimate with a [0.0%, 0.0%] CI reads as a maximally confident
    claim the instrument has no basis for. acted_accuracy/wilson95/
    cluster95 must all be None, and scoring must not raise."""
    labels = [{"file": "f", "index": 0, "truth": "data_table"}]
    judgments = {"f": {"0": "unsure"}}
    m = je.score(labels, judgments)
    assert m["acted"] == 0
    assert m["acted_accuracy"] is None
    assert m["coverage"] == 0.0
    assert m["wilson95"] is None
    assert m["cluster95"] is None


def test_empty_labels_file_exits_2(tmp_path, monkeypatch):
    """An empty (or blank-lines-only) labels.jsonl has nothing to score.
    That must fail loud (exit 2) at load time, distinct from the
    total>0-but-all-abstained case above (which scores fine and reports
    None, not an error)."""
    labels_path = tmp_path / "empty.jsonl"
    labels_path.write_text("\n\n", encoding="utf-8")
    judgments_path = tmp_path / "judgments.json"
    judgments_path.write_text(json.dumps({}), encoding="utf-8")

    with pytest.raises(ValueError):
        je.load_labels(str(labels_path))

    monkeypatch.setattr(
        sys, "argv",
        ["judgment_eval.py", "--score", "--labels", str(labels_path),
         "--judgments", str(judgments_path)],
    )
    rc = je.main()
    assert rc == 2


def test_unlabeled_judgment_reported_and_does_not_affect_accuracy():
    labels = [{"file": "f", "index": 0, "truth": "data_table"}]
    judgments_clean = {"f": {"0": "data_table"}}
    judgments_extra = {"f": {"0": "data_table", "5": "figure"}}

    m_clean = je.score(labels, judgments_clean)
    m_extra = je.score(labels, judgments_extra)

    assert m_clean["unlabeled_judgments"] == []
    assert m_extra["unlabeled_judgments"] == [{"file": "f", "index": 5}]

    assert m_clean["acted_accuracy"] == m_extra["acted_accuracy"]
    assert m_clean["total"] == m_extra["total"]
    assert m_clean["acted"] == m_extra["acted"]
    assert m_clean["correct"] == m_extra["correct"]


# ── invalid input handling ───────────────────────────────────────────────

def test_invalid_truth_value_raises(tmp_path):
    labels_path = tmp_path / "labels.jsonl"
    labels_path.write_text(
        json.dumps({"file": "f", "index": 0, "truth": "not_a_real_class"}) + "\n",
        encoding="utf-8",
    )
    with pytest.raises(ValueError):
        je.load_labels(str(labels_path))


def test_invalid_judgment_class_raises(tmp_path):
    judgments_path = tmp_path / "judgments.json"
    judgments_path.write_text(
        json.dumps({"f": {"0": "not_a_real_class"}}), encoding="utf-8"
    )
    with pytest.raises(ValueError):
        je.load_judgments(str(judgments_path))


def test_non_integer_index_key_in_judgments_raises_value_error(tmp_path):
    judgments_path = tmp_path / "judgments.json"
    judgments_path.write_text(
        json.dumps({"a.pdf": {"abc": "figure"}}), encoding="utf-8"
    )
    with pytest.raises(ValueError):
        je.load_judgments(str(judgments_path))


@pytest.mark.parametrize("bad_key", [" 0", "+0", "00", " 1 ", "01"])
def test_non_canonical_index_key_in_judgments_raises_value_error(tmp_path, bad_key):
    """Reviewer repro: score() looks up str(index) but the
    unlabeled-judgments detector normalizes via int(idx_str) — a key that
    int() accepts but doesn't round-trip back to the same string (" 0",
    "+0", "00") matches neither lookup and silently becomes an invisible
    abstention, shrinking the denominator and inflating accuracy with no
    error surfaced. load_judgments must reject any key where
    str(int(idx)) != idx."""
    judgments_path = tmp_path / "judgments.json"
    judgments_path.write_text(
        json.dumps({"a.pdf": {bad_key: "figure"}}), encoding="utf-8"
    )
    with pytest.raises(ValueError):
        je.load_judgments(str(judgments_path))


def test_cli_exits_2_on_non_integer_judgment_index(tmp_path, monkeypatch):
    """Reviewer repro: non-integer index key must not traceback past main()
    with exit 1 — it must be caught and exit 2, per spec."""
    labels_path = tmp_path / "labels.jsonl"
    labels_path.write_text(
        json.dumps({"file": "a.pdf", "index": 0, "truth": "data_table"}) + "\n",
        encoding="utf-8",
    )
    judgments_path = tmp_path / "bad.json"
    judgments_path.write_text(
        json.dumps({"a.pdf": {"abc": "figure"}}), encoding="utf-8"
    )
    monkeypatch.setattr(
        sys, "argv",
        ["judgment_eval.py", "--score", "--labels", str(labels_path),
         "--judgments", str(judgments_path)],
    )
    rc = je.main()
    assert rc == 2


def test_duplicate_label_for_same_file_index_raises():
    labels_path_lines = [
        json.dumps({"file": "x.pdf", "index": 0, "truth": "data_table"}),
        json.dumps({"file": "x.pdf", "index": 0, "truth": "layout_table"}),
    ]
    import tempfile
    with tempfile.NamedTemporaryFile(
        mode="w", suffix=".jsonl", delete=False, encoding="utf-8"
    ) as f:
        f.write("\n".join(labels_path_lines) + "\n")
        path = f.name
    try:
        with pytest.raises(ValueError):
            je.load_labels(path)
    finally:
        os.unlink(path)


def test_duplicate_label_rejected_even_when_truth_values_agree(tmp_path):
    labels_path = tmp_path / "labels.jsonl"
    labels_path.write_text(
        json.dumps({"file": "x.pdf", "index": 0, "truth": "data_table"}) + "\n"
        + json.dumps({"file": "x.pdf", "index": 0, "truth": "data_table"}) + "\n",
        encoding="utf-8",
    )
    with pytest.raises(ValueError):
        je.load_labels(str(labels_path))


def test_printed_summary_line_distinguishes_candidate_count_from_file_count(
    tmp_path, monkeypatch, capsys
):
    """Reviewer repro: the old line ('N candidates across M files') used M =
    every PDF walked, not the files that actually produced a candidate. The
    fixed line must report distinct numbers for 'files with candidates' vs
    'PDFs swept'."""
    pdf_with_tables = tmp_path / "sample.pdf"
    _build_pdf(pdf_with_tables)
    pdf_without_tables = tmp_path / "empty.pdf"
    pikepdf.new().save(str(pdf_without_tables))

    out_path = tmp_path / "manifest.json"
    monkeypatch.setattr(
        sys, "argv",
        ["judgment_eval.py", "--candidates", str(tmp_path), "--out", str(out_path)],
    )
    rc = je.main()
    captured = capsys.readouterr()

    assert rc == 0
    manifest = json.loads(out_path.read_text(encoding="utf-8"))
    assert len(manifest["candidates"]) == 2  # only sample.pdf's 2 tables

    # 2 candidates, but the sweep walked 2 PDFs, and only 1 file produced
    # candidates. The printed line must surface both the candidate count (2)
    # and the swept-PDF count (2) as distinct labeled numbers, and must not
    # claim 2 files produced candidates.
    assert "2 candidates" in captured.out
    assert "1 files" in captured.out
    assert "2 PDFs swept" in captured.out


def test_cli_exits_2_on_invalid_truth(tmp_path, monkeypatch, capsys):
    labels_path = tmp_path / "labels.jsonl"
    labels_path.write_text(
        json.dumps({"file": "f", "index": 0, "truth": "bogus"}) + "\n",
        encoding="utf-8",
    )
    judgments_path = tmp_path / "judgments.json"
    judgments_path.write_text(json.dumps({"f": {"0": "data_table"}}), encoding="utf-8")

    monkeypatch.setattr(
        sys, "argv",
        ["judgment_eval.py", "--score", "--labels", str(labels_path),
         "--judgments", str(judgments_path)],
    )
    rc = je.main()
    assert rc == 2


def test_cli_exits_2_on_invalid_judgment_class(tmp_path, monkeypatch):
    labels_path = tmp_path / "labels.jsonl"
    labels_path.write_text(
        json.dumps({"file": "f", "index": 0, "truth": "data_table"}) + "\n",
        encoding="utf-8",
    )
    judgments_path = tmp_path / "judgments.json"
    judgments_path.write_text(json.dumps({"f": {"0": "bogus"}}), encoding="utf-8")

    monkeypatch.setattr(
        sys, "argv",
        ["judgment_eval.py", "--score", "--labels", str(labels_path),
         "--judgments", str(judgments_path)],
    )
    rc = je.main()
    assert rc == 2


# ── action spec scoring ──────────────────────────────────────────────────

def _spec_labels():
    return [
        {"file": "f", "index": 0, "truth": "data_table",
         "header_rows": [0], "header_cols": [], "band_rows": []},
        {"file": "f", "index": 1, "truth": "data_table",
         "header_rows": [0], "header_cols": [0], "band_rows": [3, 9]},
        {"file": "f", "index": 2, "truth": "layout_table"},
    ]


def test_backward_compat_bare_string_judgments_score_identically():
    """A judgments file of bare class strings — the pre-existing format —
    must score classification metrics exactly as before. This is the most
    important guarantee: the spec extension must not perturb existing
    scoring for callers who never opted into the new format."""
    labels = _fixture_labels()
    judgments = _fixture_judgments()  # all bare strings, no spec keys anywhere

    m = je.score(labels, judgments)

    # Identical to test_scoring_matches_hand_computed_metrics's assertions.
    assert m["total"] == 5
    assert m["acted"] == 3
    assert m["correct"] == 2
    assert m["acted_accuracy"] == pytest.approx(2 / 3)
    assert m["harmful"] == 1
    assert m["confusion"] == {
        "data_table": {"data_table": 1, "layout_table": 1},
        "layout_table": {"abstain": 1},
        "figure": {"figure": 1},
        "other": {"abstain": 1},
    }
    # No label in this fixture carries spec keys -> nothing is spec-scorable.
    assert m["spec"]["scorable"] == 0
    assert m["spec"]["exact"] == 0
    assert m["spec"]["exact_accuracy"] is None
    assert m["spec"]["skipped_wrong_class"] == 0
    assert m["spec"]["judgment_missing_spec"] == 0


def test_spec_exact_match_all_three_keys():
    labels = [{"file": "f", "index": 0, "truth": "data_table",
               "header_rows": [0], "header_cols": [0, 1], "band_rows": []}]
    judgments = {"f": {"0": {"class": "data_table",
                              "header_rows": [0], "header_cols": [0, 1], "band_rows": []}}}
    m = je.score(labels, judgments)
    assert m["spec"]["scorable"] == 1
    assert m["spec"]["exact"] == 1
    assert m["spec"]["exact_accuracy"] == pytest.approx(1.0)
    assert m["spec"]["judgment_missing_spec"] == 0
    for k in ("header_rows", "header_cols", "band_rows"):
        assert m["spec"]["by_key"][k] == {"n": 1, "correct": 1, "accuracy": pytest.approx(1.0)}


def test_spec_match_is_order_insensitive_and_missing_key_equals_empty_list():
    # Label omits band_rows entirely (== []); judgment gives header_cols in
    # reversed order and omits band_rows too (== [] again) -> still exact.
    labels = [{"file": "f", "index": 0, "truth": "data_table",
               "header_rows": [0], "header_cols": [1, 0]}]
    judgments = {"f": {"0": {"class": "data_table",
                              "header_rows": [0], "header_cols": [0, 1]}}}
    m = je.score(labels, judgments)
    assert m["spec"]["exact"] == 1
    assert m["spec"]["exact_accuracy"] == pytest.approx(1.0)
    assert m["spec"]["by_key"]["band_rows"] == {"n": 1, "correct": 1, "accuracy": pytest.approx(1.0)}


def test_wrong_class_lands_in_skipped_wrong_class_not_scorable():
    labels = [{"file": "f", "index": 0, "truth": "data_table",
               "header_rows": [0], "header_cols": [], "band_rows": []}]
    judgments = {"f": {"0": "layout_table"}}  # misclassified
    m = je.score(labels, judgments)
    assert m["spec"]["skipped_wrong_class"] == 1
    assert m["spec"]["scorable"] == 0
    assert m["spec"]["exact"] == 0
    assert m["spec"]["exact_accuracy"] is None
    assert m["spec"]["judgment_missing_spec"] == 0


def test_bare_string_judgment_against_spec_label_is_judgment_missing_spec():
    """A bare-string judgment against a spec-carrying label is closer to an
    abstention than a wrong answer — parallel to how 'unsure'/absent is
    excluded from acted/acted_accuracy on the classification side. It must
    NOT occupy a spot in spec.scorable (and therefore not in the
    spec.exact_accuracy denominator either), and must not be counted as an
    exact match or as a spec.by_key miss — it is neither exact nor an
    error, just reported separately."""
    labels = [{"file": "f", "index": 0, "truth": "data_table",
               "header_rows": [0], "header_cols": [], "band_rows": []}]
    judgments = {"f": {"0": "data_table"}}  # correct class, no spec given
    m = je.score(labels, judgments)
    assert m["spec"]["judgment_missing_spec"] == 1
    assert m["spec"]["scorable"] == 0  # excluded, not folded in as a miss
    assert m["spec"]["exact"] == 0  # not counted as exact
    # scorable == 0 -> None, never a flattering (or damning) 0.0 over a
    # denominator this item was never actually part of.
    assert m["spec"]["exact_accuracy"] is None
    # by_key: a missing-spec item contributes no evidence at all.
    for k in ("header_rows", "header_cols", "band_rows"):
        assert m["spec"]["by_key"][k] == {"n": 0, "correct": 0, "accuracy": pytest.approx(0.0)}


def test_spec_by_key_isolates_which_key_is_wrong():
    """header_rows right (axis correct), header_cols wrong (depth wrong) —
    the by_key breakdown must surface this, not just an aggregate miss."""
    labels = [{"file": "f", "index": 0, "truth": "data_table",
               "header_rows": [0], "header_cols": [0, 1], "band_rows": []}]
    judgments = {"f": {"0": {"class": "data_table",
                              "header_rows": [0], "header_cols": [0], "band_rows": []}}}
    m = je.score(labels, judgments)
    assert m["spec"]["exact"] == 0  # header_cols mismatch breaks exact match
    assert m["spec"]["by_key"]["header_rows"] == {
        "n": 1, "correct": 1, "accuracy": pytest.approx(1.0)
    }
    assert m["spec"]["by_key"]["header_cols"] == {
        "n": 1, "correct": 0, "accuracy": pytest.approx(0.0)
    }
    assert m["spec"]["by_key"]["band_rows"] == {
        "n": 1, "correct": 1, "accuracy": pytest.approx(1.0)
    }


def test_spec_scorable_zero_yields_none_not_zero_and_does_not_raise():
    # truth=data_table but no spec keys on the label at all -> nothing scorable.
    labels = [{"file": "f", "index": 0, "truth": "data_table"}]
    judgments = {"f": {"0": "data_table"}}
    m = je.score(labels, judgments)
    assert m["spec"]["scorable"] == 0
    assert m["spec"]["exact_accuracy"] is None
    assert m["spec"]["cluster95"] is None


def test_malformed_spec_label_exits_2(tmp_path, monkeypatch):
    labels_path = tmp_path / "labels.jsonl"
    labels_path.write_text(
        json.dumps({"file": "f", "index": 0, "truth": "data_table",
                    "header_rows": "not_a_list"}) + "\n",
        encoding="utf-8",
    )
    judgments_path = tmp_path / "judgments.json"
    judgments_path.write_text(json.dumps({"f": {"0": "data_table"}}), encoding="utf-8")

    with pytest.raises(ValueError):
        je.load_labels(str(labels_path))

    monkeypatch.setattr(
        sys, "argv",
        ["judgment_eval.py", "--score", "--labels", str(labels_path),
         "--judgments", str(judgments_path)],
    )
    assert je.main() == 2


def test_malformed_spec_label_non_integer_index_raises():
    labels_path_content = json.dumps(
        {"file": "f", "index": 0, "truth": "data_table", "header_rows": [0, "a"]}
    )
    import tempfile
    with tempfile.NamedTemporaryFile(
        mode="w", suffix=".jsonl", delete=False, encoding="utf-8"
    ) as fh:
        fh.write(labels_path_content + "\n")
        path = fh.name
    try:
        with pytest.raises(ValueError):
            je.load_labels(path)
    finally:
        os.unlink(path)


def test_malformed_spec_judgment_non_list_raises(tmp_path):
    judgments_path = tmp_path / "judgments.json"
    judgments_path.write_text(
        json.dumps({"f": {"0": {"class": "data_table", "header_rows": "bad"}}}),
        encoding="utf-8",
    )
    with pytest.raises(ValueError):
        je.load_judgments(str(judgments_path))


def test_unknown_key_in_judgment_object_raises(tmp_path):
    judgments_path = tmp_path / "judgments.json"
    judgments_path.write_text(
        json.dumps({"f": {"0": {"class": "data_table", "bogus_key": [0]}}}),
        encoding="utf-8",
    )
    with pytest.raises(ValueError):
        je.load_judgments(str(judgments_path))


def test_judgment_object_missing_class_key_raises(tmp_path):
    judgments_path = tmp_path / "judgments.json"
    judgments_path.write_text(
        json.dumps({"f": {"0": {"header_rows": [0]}}}),
        encoding="utf-8",
    )
    with pytest.raises(ValueError):
        je.load_judgments(str(judgments_path))


def test_load_judgments_accepts_bare_strings_and_objects_together(tmp_path):
    """load_judgments must accept a mix of the two forms in the same file
    without error, and preserve both shapes for score() to interpret."""
    judgments_path = tmp_path / "judgments.json"
    judgments_path.write_text(json.dumps({
        "f": {
            "0": "data_table",
            "1": {"class": "data_table", "header_rows": [0], "header_cols": [], "band_rows": []},
        }
    }), encoding="utf-8")
    data = je.load_judgments(str(judgments_path))
    assert data["f"]["0"] == "data_table"
    assert data["f"]["1"]["class"] == "data_table"


def test_load_labels_preserves_spec_keys_and_omits_when_absent(tmp_path):
    labels_path = tmp_path / "labels.jsonl"
    labels_path.write_text(
        json.dumps({"file": "f", "index": 0, "truth": "data_table",
                    "header_rows": [0], "band_rows": [3, 9]}) + "\n"
        + json.dumps({"file": "f", "index": 1, "truth": "layout_table"}) + "\n",
        encoding="utf-8",
    )
    labels = je.load_labels(str(labels_path))
    assert labels[0]["header_rows"] == [0]
    assert labels[0]["band_rows"] == [3, 9]
    assert "header_cols" not in labels[0]
    assert "header_rows" not in labels[1]


def test_spec_cluster95_reuses_cluster_bootstrap_across_files():
    """Spec-scorable items cluster by file exactly like classification
    items — reuse the same bootstrap machinery, not a bespoke one."""
    labels = []
    judgments = {}
    for f, exact in (("g0", True), ("g1", True), ("g2", False), ("g3", False)):
        judgments[f] = {}
        for i in range(3):
            labels.append({"file": f, "index": i, "truth": "data_table",
                            "header_rows": [0], "header_cols": [], "band_rows": []})
            if exact:
                judgments[f][str(i)] = {"class": "data_table", "header_rows": [0],
                                         "header_cols": [], "band_rows": []}
            else:
                judgments[f][str(i)] = {"class": "data_table", "header_rows": [1],
                                         "header_cols": [], "band_rows": []}
    m = je.score(labels, judgments, seed=999)
    assert m["spec"]["scorable"] == 12
    assert m["spec"]["exact"] == 6
    assert m["spec"]["exact_accuracy"] == pytest.approx(0.5)
    assert m["spec"]["cluster95"] is not None
    lo, hi = m["spec"]["cluster95"]
    assert 0.0 <= lo <= 0.5 <= hi <= 1.0


def test_classification_metrics_unchanged_by_presence_of_spec_data():
    """The same classification outcomes, with and without attached spec
    data, must produce byte-identical classification-side metrics."""
    labels_plain = _fixture_labels()
    judgments_plain = _fixture_judgments()
    m_plain = je.score(labels_plain, judgments_plain)

    labels_with_spec = [dict(item) for item in _fixture_labels()]
    labels_with_spec[0]["header_rows"] = [0]
    labels_with_spec[0]["header_cols"] = []
    labels_with_spec[0]["band_rows"] = []
    judgments_with_spec = {
        "fileA": {"0": {"class": "data_table", "header_rows": [0],
                         "header_cols": [], "band_rows": []},
                  "1": "layout_table", "2": "unsure"},
        "fileB": {"0": "figure", "2": "data_table"},
    }
    m_spec = je.score(labels_with_spec, judgments_with_spec)

    for key in ("total", "files", "abstained", "acted", "acted_files",
                "mean_tables_per_file", "correct", "acted_accuracy", "coverage",
                "harmful", "confusion", "per_class", "wilson95", "cluster95",
                "unlabeled_judgments"):
        assert m_plain[key] == m_spec[key], key
    assert m_spec["spec"]["scorable"] == 1
    assert m_spec["spec"]["exact"] == 1


def test_format_report_shows_classification_and_spec_as_separate_sections():
    labels = _spec_labels()
    judgments = {
        "f": {
            "0": {"class": "data_table", "header_rows": [0],
                  "header_cols": [], "band_rows": []},
            "1": {"class": "data_table", "header_rows": [1],
                  "header_cols": [0], "band_rows": [3, 9]},
            "2": "layout_table",
        }
    }
    m = je.score(labels, judgments)
    report = je.format_report(m)
    assert "CLASSIFICATION" in report
    assert "ACTION SPEC" in report
    # spec section must come after the classification section
    assert report.index("CLASSIFICATION") < report.index("ACTION SPEC")
    assert "spec_exact_accuracy" in report
    assert "acted_accuracy" in report
