docpluck 2.4.75__tar.gz → 2.4.76__tar.gz

docpluck-2.4.76/.claude/skills/_project/canary.json

@@ -0,0 +1,111 @@
+{
+  "schema_version": 2,
+  "project": "docpluck",
+  "skill": "docpluck-iterate",
+  "comment_for_humans": "Originally 2026-05-23; updated same day to DOI keys after the canary papers were onboarded into article-finder (memory feedback_paper_locating_via_article_finder). The `key` is now the canonical DOI key (lowercased, `/`→`__`). `stem` retains the project-local label for human readability and matches `paper_stem` in run-meta phase_5d_runs entries — but the gate's locator queries article-finder via `key`. Fixed-3 + rotating-pool together exercise B1/B2/B3/B4/B6/B7 defect classes.",
+  "canary": {
+    "fixed": [
+      {
+        "stem": "ip_feldman_2025_pspb",
+        "key": "10.1177/01461672251327169",
+        "doi": "10.1177/01461672251327169",
+        "rationale": "Triggered the 2026-05-23 iterate-loop spine. Single paper exercising 4 distinct defect classes: G5d hallucinated subsection headings (Supplemental Materials mid-Method), B3 affiliation/running-header leak into body, B4 mid-text Table 3 caption, running-header 'Ip and Feldman' surfacing as a paragraph. The litmus test for whether the library's structural-defect cluster is closed.",
+        "max_gold_age_days": 30
+      },
+      {
+        "stem": "plos_med_1",
+        "key": "10.1371/journal.pmed.1004323",
+        "doi": "10.1371/journal.pmed.1004323",
+        "rationale": "B1 text_loss canary — Tables 2-5; Table 5 has 13 SAE rows lost. The 1 still-failing Tier-D cell as of run 9 close. Forces the table-completeness story to stay covered every cycle. The architectural decision around modified-Approach-B bbox computation has been outstanding since 2026-05-22 — this canary keeps it visible.",
+        "max_gold_age_days": 30
+      },
+      {
+        "stem": "chandrashekar_2023_mp",
+        "key": "10.15626/MP.2022.3108",
+        "doi": "10.15626/MP.2022.3108",
+        "rationale": "B6 column-interleave canary — pdftotext two-column reading order serialises a multi-column page interleaved. Distinct defect class from ip_feldman / plos_med_1 (extraction-layer, not normalize-layer). Keeps the column-interleave architectural decision (R4 in 2026-05-22-residual handoff) from being silently forgotten.",
+        "max_gold_age_days": 30
+      }
+    ],
+    "rotating_pool": [
+      {
+        "stem": "chan_feldman_2025_cogemo",
+        "key": "10.1080/02699931.2024.2434156",
+        "doi": "10.1080/02699931.2024.2434156",
+        "exercises": "B2 hallucinated/demoted headings; B6 column interleave (Measures section)"
+      },
+      {
+        "stem": "ar_apa_j_jesp_2009_12_011",
+        "key": "10.1016/j.jesp.2009.12.011",
+        "doi": "10.1016/j.jesp.2009.12.011",
+        "exercises": "B2 hallucinated headings (Participants/Overview); B7 deleted-minus glyph (b = .022 → should be -0.022)"
+      }
+    ],
+    "rotation_size": 2,
+    "_pool_size_note": "Pool currently has 2 entries == rotation_size, so every cycle covers both (no rotation effect). Expand to 5+ once efendic_2022_affect, maier_2023_collabra, xiao_2021_crsp are onboarded into article-finder per the locator-via-AF directive. See `onboarding_status.still_to_onboard_before_use`."
+  },
+  "limits": {
+    "max_cycles_without_corpus_sweep": 3,
+    "default_max_gold_age_days": 30
+  },
+  "verification_protocol": {
+    "kind": "library-render-vs-ai-gold",
+    "rendered_ext": ".md",
+    "gold_cache_filename": "reading.md",
+    "gold_view": "reading",
+    "locator_via_article_finder": "python ~/.claude/skills/article-finder/cache-check.py <doi>",
+    "gold_via_article_finder": "python ~/.claude/skills/article-finder/ai-gold.py get <key> --view reading",
+    "render_command": "python tools/render_for_audit.py --key {key} --out {out}",
+    "audit_subagent_prompt": "~/.claude/skills/_shared/iterate-loop/audit-subagent-prompt.md",
+    "audit_subagent_model": "sonnet",
+    "audit_invocation": {
+      "headless": "claude -p --model sonnet --append-system-prompt @<prompt-file> <payload-json>",
+      "in_session": "Agent(subagent_type='general-purpose', model='sonnet', prompt=<assembled-prompt>)",
+      "MUST_NOT_USE": "anthropic SDK, ANTHROPIC_API_KEY, direct api.anthropic.com calls, GH Actions calling Anthropic API. Per memory feedback_no_apis_only_claude_max + Vibe/CLAUDE.md hard rule."
+    },
+    "defect_taxonomy": [
+      {
+        "class": "TEXT-LOSS",
+        "description": "Body text present in the source PDF is missing from rendered .md (entire paragraph, sentence, or table row dropped).",
+        "examples": ["Table 5's 13 SAE rows missing from plos_med_1", "Last paragraph of Method section absent on chan_feldman"]
+      },
+      {
+        "class": "HALLUCINATION",
+        "description": "Text or structure in rendered .md that has no source in the PDF. Subclass G5d covers hallucinated subsection headings (e.g. body text promoted to '### Heading' inappropriately).",
+        "examples": ["'### Supplemental Materials' appearing as a heading mid-Method on ip_feldman when in gold it is body text", "'## KEYWORDS' inserted as heading on xiao_2021"]
+      },
+      {
+        "class": "SECTION-BOUNDARY",
+        "description": "Section headings missing, misplaced, or wrong level. Content attributed to the wrong section.",
+        "examples": ["Discussion content rendered under Method section", "Abstract heading missing entirely"]
+      },
+      {
+        "class": "TABLE",
+        "description": "Table content wrong: missing rows/columns, mid-text caption duplication, column interleave from extraction, header rows in body cells. Subclass B1 covers row-loss, B4 covers mid-text caption duplication, B6 covers column interleave.",
+        "examples": ["'Table 3. Comparison of...' caption appearing in body text on ip_feldman (B4)", "Multi-column page serialized interleaved on chandrashekar_2023 (B6)"]
+      },
+      {
+        "class": "FIGURE",
+        "description": "Figure references missing, captions misplaced, figure caption appearing twice or duplicated as inline text.",
+        "examples": ["Figure 3c caption duplicated inline after the figure block"]
+      },
+      {
+        "class": "METADATA-LEAK",
+        "description": "Front-matter (affiliations, authors, running header, journal banner, page number) leaking into body text. Subclass B3 covers affiliation/banner leak mid-Intro.",
+        "examples": ["'University of Hong Kong, Pok Fu Lam' appearing between Introduction paragraphs on ip_feldman (B3)", "'Ip and Feldman' running header surfacing as a body line"]
+      },
+      {
+        "class": "GLYPH",
+        "description": "Unicode glyph corruption: dropped minus signs (U+2212 → space), Greek letter loss on tight-kerned PDFs, sign-flipped statistics. Subclass B7 covers dropped-minus glyph.",
+        "examples": ["b = .022 rendered when source shows b = -0.022 (B7)"]
+      }
+    ],
+    "_note": "The audit subagent (Sonnet) is dispatched per-paper. For each canary paper: render via render_command, fetch gold via gold_via_article_finder, dispatch Sonnet with this taxonomy + paper-specific signals from canary.fixed[].rationale. Sonnet emits STRICT JSON verdict. The orchestrator (canary-audit.sh) aggregates per-paper verdicts and exits 0/1. Per feedback_no_apis_only_claude_max, Sonnet is ONLY invoked via Claude Max (headless `claude -p --model sonnet` or in-session Agent tool) — NEVER via the Anthropic API."
+  },
+  "onboarding_status": {
+    "onboarded_to_article_finder": ["ip_feldman_2025_pspb", "plos_med_1", "chandrashekar_2023_mp", "chan_feldman_2025_cogemo", "ar_apa_j_jesp_2009_12_011"],
+    "still_to_onboard_before_use": ["efendic_2022_affect", "maier_2023_collabra", "xiao_2021_crsp"],
+    "onboarding_date": "2026-05-23",
+    "onboarding_record": "tmp/onboard_canary.py — run again to onboard the remaining 3 (DOIs need to be extracted first)"
+  }
+}

{docpluck-2.4.75 → docpluck-2.4.76}/.claude/skills/docpluck-deploy/SKILL.md

@@ -4,6 +4,24 @@ description: Deploy Docpluck to production. Pre-flight checks (Next.js build, Py
 tags: [docpluck, nextjs, python, vercel, railway, neon, auth, deploy]
 ---
 
+<!--
+2026-05-25 — Pre-flight Phase 0 gate: canary-audit MUST pass before `git tag v*`.
+
+Before any tag operation, run the canary audit. If FAIL → block the tag with
+the punch-list of findings (see Sonnet's verdict at
+`tmp/iterate/canary-verdict-<HEAD-SHA>.json`).  Do NOT bypass with --no-verify
+on the pre-tag git hook; the audit exists to prevent the v2.4.72-style failure
+mode (cycle declared "shippable" on pytest green while the canary paper was
+demonstrably broken).
+
+Invocation paths:
+- In-session: `Agent(subagent_type="general-purpose", model="sonnet", prompt=<assembled-from-audit-subagent-prompt.md>)`.
+- Headless (after one-time `claude setup-token`): `bash ~/.claude/skills/_shared/iterate-loop/canary-audit.sh --full docpluck-iterate`.
+
+Both paths use Claude Max ONLY — never the Anthropic API (per
+`feedback_no_apis_only_claude_max`).
+-->
+
 ## [MANDATORY FIRST ACTION] preflight (do NOT skip, even if orchestrated by /ship)
 
 **Your very first action in this skill, BEFORE reading anything else, is:**

{docpluck-2.4.75 → docpluck-2.4.76}/.claude/skills/docpluck-iterate/SKILL.md

@@ -6,6 +6,27 @@ user-invocable: true
 argument-hint: "[--goal time:60m | iters:5 | baseline:26/26+full:95/101 | until:\"description\"] [--no-broad-read] [--dry-run]"
 ---
 
+<!--
+2026-05-25 — canary-audit integration (Sonnet-watches-Opus pattern, no API, Claude Max).
+
+At cycle-end (after `iterate-gate.sh --cycle N` returns PASS on deterministic
+rules I1-I11) AND at run-close, invoke the canary audit. The audit is Phase 5d's
+deterministic-gate companion: I1-I11 surface fix targets, Sonnet-via-Claude-Max
+independently judges whether the rendered artifact still matches the AI gold.
+
+Invocation paths:
+- In-session: `Agent(subagent_type="general-purpose", model="sonnet", prompt=<assembled-from-audit-subagent-prompt.md>)`.
+- Headless (after `claude setup-token`): `bash ~/.claude/skills/_shared/iterate-loop/canary-audit.sh --full docpluck-iterate`.
+
+Required reading:
+- `~/.claude/skills/_shared/iterate-loop/audit-subagent-prompt.md` (Sonnet's audit-role prompt template)
+- `.claude/skills/_project/canary.json::verification_protocol` (project-specific defect taxonomy + render command)
+
+Memories: `project_sonnet_watches_opus_audit`, `feedback_audit_nondeterminism_mitigation`, `feedback_no_apis_only_claude_max`.
+
+Reference handoff: `docs/superpowers/handoffs/2026-05-25-canary-audit-architecture-and-cluster-A-B-C-landed.md`.
+-->
+
 ## [MANDATORY FIRST ACTION] preflight + iterate-loop spine (do NOT skip, even if orchestrated by /ship)
 
 **Your very first action in this skill, BEFORE reading anything else, is:**

{docpluck-2.4.75 → docpluck-2.4.76}/.gitignore

@@ -37,9 +37,13 @@ REQUESTS_FROM_ESCIMATE.md
 # captures. Useful locally during iteration but not committed.
 verify_*.txt
 verify_*.err
+verify_*.log
 pytest_run.txt
 pytest_err.txt
 tmp/
+# Ad-hoc one-shot diagnostic scripts dropped at the repo root during
+# iteration (worktree-aware verify runners, scratch reproducers, etc.).
+tmp_*.py
 
 # Verification harness outputs (scripts/harness) — extraction artifacts, not committed.
 verify_out/

{docpluck-2.4.75 → docpluck-2.4.76}/CHANGELOG.md

@@ -1,5 +1,74 @@
 # Changelog
 
+## [2.4.76] — 2026-05-25
+
+**§A R4 column-aware re-extraction LANDED — closes jama-open-1 D4 (Key Points sidebar missing).** `NORMALIZATION_VERSION` 1.9.24 → 1.9.25 (concurrent with EC-T1's bump). Closes the final defect of the 2026-05-25 Haiku-orchestration pretest jama-open-1 cluster (HANDOFF_2026-05-25_pretest-followups.md Issue 1 — 5 of 5 defects now closed).
+
+Two-pronged detector + per-page pdftotext-crop re-extraction:
+
+- **Detector (`docpluck/normalize.py::_detect_column_interleave_pages`):** added Signature B (bimodal-line-length): substantial-content page (≥30 body lines) where ≥30% of lines are short (<40 chars) AND ≥30% are long (>70 chars) is column-fragmented. The canonical fingerprint of JAMA Open's abstract+sidebar interleave that escaped the original Signature A (no-terminator+Title-Case flip count) because period-terminated structured-abstract labels masked the flips.
+- **Column extractor (`docpluck/extract_columns.py`):** new module. `extract_page_text_columns(layout, page_index, pdf_bytes)` detects column midline via word-center histogram (relaxed fallback to single deep gutter when no contiguous run exists — narrow-sidebar pages produce 1-bucket gutters), then `_crop_and_extract` runs pdftotext twice per flagged page with `-x -y -W -H` crop flags (preserves pdftotext's gap-aware word-spacing that pdfplumber's `extract_text()` loses on tight-kerned PDFs). Fall-through to pdfplumber word-join path if pdftotext-crop fails.
+- **Wiring (`docpluck/extract.py::extract_pdf`):** R4 runs at the text-channel layer (after pdftotext, before return) so sections / normalize / render / structured ALL see the corrected text via the single `extract_pdf` call. Method tag gains `+column_corrected:N,M,...` suffix when R4 fires.
+
+**jama-open-1 D4 outcome:** Key Points sidebar (`Question / Findings / Meaning`) now appears as a coherent block rather than line-interleaved through the abstract. Abstract content flows in proper paragraph order. Combined with the v2.4.74 fixes (D1 RUNNING_HEADER_LEAK, D2 HALLUC_HEAD, D3 ABSTRACT_LEVEL_MISMATCH, D5 TABLE_STRUCTURE_CORRUPT), the full jama-open-1 5-defect cluster is closed.
+
+**jama-open-1 D1/D2/D3/D5 follow-up (also this version) ported from v2.4.74:**
+
+- D1 RUNNING_HEADER_LEAK (`normalize.py`): JAMA-style `Downloaded from <bare-domain> ... user on MM/DD/YYYY` watermark + bare standalone date footer.
+- D2 HALLUC_HEAD (`render.py _demote_isolated_table_cell_headings`): demote `### {label}` stranded inside table-cell clusters via bidirectional cell-fragment / column-header-stranded / data-shape signatures.
+- D3 ABSTRACT_LEVEL_MISMATCH (`render.py _demote_abstract_zone_inline_labels`): zone-bounded demoter for JAMA structured-abstract inline labels + Key Points sidebar trio.
+- D5 TABLE_STRUCTURE_CORRUPT (`render.py _strip_phantom_camelot_tables`): strip Camelot tables with masthead `<th>` + section-name `<td>` leak.
+
+**R1-perf threading** (`extract_pdf_structured` + `render.py`): `_layout_doc` kwarg eliminates duplicate `extract_pdf_layout(pdf_bytes)` call.
+
+**R3b widening** (`render.py _suppress_inline_duplicate_figure_captions`): wider 250-char overhang form gated against body-prose starters + stat shapes.
+
+**Other R4-cascade regressions also fixed in this release** (4 separate downstream tests failed when R4 wiring landed; all addressed):
+
+- **`_demote_italic_label_with_comma_headings` allowlist** (`render.py`): the Stream A §B-new-4 demoter fired on generic `## Discussion` when the body started with `In this study, ...` (matches the comma-list shape) and wrecked the rendered output — preventing the orphan-multilevel-number fold from producing `### 5.4. Discussion` on jdm_m.2022.2.pdf. New `_METADATA_LABEL_HEADING_PREFIXES` allowlist restricts the demoter to the open-science / data-availability metadata family (`Data Availability`, `Open Science Disclosures`, `Preregistration`, `Author Contributions`, `Funding`, `CRediT`, etc.). Generic subsection words can no longer be flattened by the heuristic.
+- **`_demote_metadata_label_headings` heading-skipping lookahead** (`render.py`): the §B-new-2 demoter capped lookahead at 3 lines, so when R4 column-aware extraction reordered xiao_2021_crsp front-matter into `KEYWORDS / Introduction / metadata-list`, the bare `## KEYWORDS` heading survived because its keyword payload landed below the intervening `## Introduction`. Extended scan to 15 non-blank lines and explicitly skips intervening heading lines while searching for metadata-shape content.
+- **`_prior_paragraph_is_sentence_terminated` URL handling** (`sections/annotators/text.py`): Stream A's Cluster A canonical-heading guard rejected lines preceded by URL-terminated paragraphs (`...code: https://osf.io/bwmtr/`) because URLs don't end in `.!?`. Added two acceptors: prior line contains `://` (any URL) OR ends with `/` (URL trailing slash). Resolves the v161 text-annotator regression on the lowercase-body Keywords test.
+- **`_strip_recurring_running_headers` truncated-prefix case** (`normalize.py`): when R4 column-aware extraction crops a page footer mid-token (`PLOS Medicine | https://...1004323 Dec` instead of `... December 28, 2023`), the truncated form appeared once while the full form appeared ≥3 times, so P0r's repetition detector stripped the full but left the truncated single. Added a prefix-match arm: when a body line ≥30 chars is a strict prefix of an already-known repeating header, strip it.
+- **Method-tag allowlist + snapshot regen** (`tests/test_v2_backwards_compat.py`): the `+column_corrected:N,M,...` suffix is documented and allowlisted via base-prefix split before the known-strings check. `tests/snapshots/{jama_lattice,ieee_figure_heavy}.txt` snapshots deleted to recapture against the v2.4.76 R4-corrected output.
+
+**R4 false-positive gates** (`extract_columns.py`): three structural-signature gates discovered while reconciling R4 with the full corpus. Each blocks a distinct false-positive class without affecting JAMA Open detection:
+
+1. **Contiguous-run gate** (`_detect_2col_midline`): best_run must span ≥2 buckets before yielding a midline. A length-1 run inside an otherwise populated central region is an alternating-zeros artifact of periodic word x-positioning (justified text, monospaced layouts, synthetic fixtures), not a real gutter. Real 2-column pages produce sustained low-density valleys ≥2 buckets wide because both column peaks are wide enough to push down a stretch of central density.
+2. **Deep-fallback density gates** (`_detect_2col_midline`): the single-bucket trough fallback only fires when (a) ≥50% of surrounding buckets exceed the loose threshold (blocks sparse fixtures + figure-only pages) AND (b) the trough's immediate neighbors both exceed the loose threshold (flanked by genuine peaks).
+3. **Y-row bilateral gate** (`extract_page_text_columns`, NEW in `extract_columns.py:113`): even when midline detection succeeds, R4 is skipped for the page if ≥30% of y-rows have words on BOTH sides of the candidate midline. A real 2-column body-text page has text rows in ONE column at a time (independent baselines per column); a TABLE embedded in a single-column page has rows with cells on both sides at the SAME y. Empirically (2026-05-25): JAMA Open p1 abstract+sidebar = 12.5% bilateral (passes); amle_1 table-heavy pages 10/13/29 = 65.5%/53.0%/38.5% bilateral (rejected).
+
+Verified to preserve R4 firing on jama_open_1 (D4 Key Points sidebar closure unchanged — 3/3 R4 tests pass) while blocking R4 from misreading amle_1's 13 in-paper tables as page-level columns (resolving `test_amle_1_table_captions_not_cell_garbage`). Also updates `tests/test_extract_columns.py::FakePage` to mirror the new LayoutDoc schema (`height`, `words` fields the v2.4.76 R4 rewrite reads) and adds a new bilateral-gate unit test with a synthetic 30-row 2-cell-per-row table.
+
+**Known residuals (v2.4.77+ follow-ups):** column-boundary line truncation on wide titles; `CONCLUSIONS AND RELEVANCE` split across columns becomes two fragments; orphan `(contin` from `(continued)` page markers. Cosmetic — the structural defect (sidebar missing / abstract interleaved) is closed. Full fix needs column-element detection (figures/title/page-spanning banners get rendered FROM the original pdftotext output, not the crops).
+
+**EC-T1: table-row flattening for downstream stat-verification consumers.** `TABLE_EXTRACTION_VERSION` 2.1.5 → 2.2.0. Closes the largest cluster from the ESCIcheck handoffs ([2026-05-24](../ESCIcheckapp/docs/DOCPLUCK_HANDOFF_2026-05-24.md), [2026-05-25](../ESCIcheckapp/docs/DOCPLUCK_HANDOFF_2026-05-25.md)) — ~78 effectcheck rows across 6 canary papers blocked on bare table cells.
+
+**New module: [`docpluck.tables.flatten`](docpluck/tables/flatten.py)**
+
+- `flatten_table(table) -> list[FlattenedRow]` — turns a structured `Table` into per-row records. Each record carries `raw_cells`, `header`, `row_label`, a flattened English `sentence` (e.g. `Importance: t(741) = 3.93, p < .001, d = 0.29`), and a structured `fields` dict (`t`, `df`, `df1`, `df2`, `F`, `r`, `chi2`, `p`, `p_op`, `d`, `eta2`, `M`, `SD`, `n`, `N`, `CI_lower`, `CI_upper`). Three nested fidelity levels so consumers pick what they trust.
+- `flatten_tables_for_paper(tables)` — convenience for paper-level JSONL emission.
+- `render_flattened_inline(records, ...)` — renders the same records as a markdown block bounded by HTML-comment sentinels (`<!-- docpluck:flattened-table id="…" start --> … end -->`).
+- Header→cell binding consolidations: `t + df → t(df)`; `F + (df1, df2)` from a `F(1, 998)` header → `F(df1, df2)`; `r + n → r(n-2)`; `M + SD → M = m, SD = sd`; CI from `[lo, hi]` cell OR separate `lower/upper` columns → `95% CI [lo, hi]`; `p_op + p → p < .001`.
+
+**Render integration**
+
+- `render_pdf_to_markdown` gains `flatten_tables_inline: bool = False`. When True, an `### {label} — rendered as text` block is emitted immediately after each `<table>`, with one bullet per body row. Bounded by HTML-comment sentinels — greppable, diff-tool-friendly, invisible in rendered markdown viewers.
+- Inline block is *generated from* the same `FlattenedRow` records that go into the JSONL sidecar — single source of truth, no drift risk between the two outputs.
+- Default `False` keeps the .md byte-identical to v2.4.75 for callers that don't opt in.
+
+**CLI**
+
+- `docpluck render --tables-jsonl PATH` writes one `FlattenedRow` JSON record per line to `PATH`. Canonical extraction contract for downstream stat-verification tools (effectcheck, escimate, scimeto).
+- `docpluck render --flatten-tables-inline` embeds the human-readable block in the .md output (debug/eyeball mode).
+
+**New top-level exports:** `FlattenedRow`, `flatten_table`, `flatten_tables_for_paper`, `render_flattened_inline`.
+
+**Tests:** 19 new tests in `tests/test_tables_flatten.py` covering the 6 canary table shapes from the handoffs (collabra_57785 T8 t-rows, collabra_90203 T8 F-with-df-header, T10 r+n correlations, collabra_90203 T9 bare-numeric, lee_feldman bare t+p+nodf, majumder effect-size-with-CI) plus inline-render sentinel boundaries and edge cases.
+
+Outstanding from the same handoffs:
+- **EC-T2** — RR/RD/MD per-arm trial-table flattening (`plosmed_1004323`). Specialization of EC-T1; queued.
+- **D-25-C** — gold-label mismatch on inter-rater r (not a docpluck defect); flagged for `article-finder` gold-quality pass.
+
 ## [2.4.75] — 2026-05-25
 
 **EC-T3: CI bracket middle-period → comma (ESCIcheck 2026-05-24 D2).** `NORMALIZATION_VERSION` 1.9.24 → 1.9.25. Closes one of the three defect clusters filed by escicheck-iterate against docpluck ([handoff](../ESCIcheckapp/docs/DOCPLUCK_HANDOFF_2026-05-25.md), [triage EC-T3](docs/TRIAGE_2026-05-14_phase_5d_gold_audit.md)).

{docpluck-2.4.75 → docpluck-2.4.76}/CLAUDE.md

@@ -77,6 +77,7 @@ Skipping step 5 is the most common failure mode. The deploy skill catches it.
 > **READ [`LESSONS.md`](./LESSONS.md) BEFORE TOUCHING `extract*.py`, `normalize.py`, or `sections/`.**
 > It is the durable incident log for the recurring mistakes below.  When in doubt about a change, the answer is almost always already there.
 
+- **NEVER call the Anthropic API. ALL Claude model calls go through Claude Max via Claude Code.** Allowed: `Agent` tool in-session (with `model="sonnet"` for the audit subagent); headless `claude -p --model sonnet` from `.git/hooks/*` and `tools/canary_audit.sh`; `mcp__scheduled-tasks__create_scheduled_task` invoking Claude Code. Forbidden: `import anthropic`, `ANTHROPIC_API_KEY` anywhere in this repo or any related repo (`docpluckapp`, `escicheck`, `2Rmarkdown`, `CitationGuard`), `.github/workflows/*` containing Anthropic-API calls. The canary-audit architecture (Sonnet-watches-Opus) is designed around this constraint: external enforcement is local git hooks + scheduled tasks invoking headless Claude Code, NOT GitHub Actions calling the API. Source: user directive 2026-05-25 (memory `feedback_no_apis_only_claude_max`), re-affirming previous statements across multiple sessions. Failure to follow this rule is the same severity as failing "LEAVE NOTHING BEHIND."
 - **LEAVE NOTHING BEHIND.** If you see an issue — any issue, however small, whether pre-existing, already-known, "out of scope", or unrelated to the task at hand — you fix it in the same run. "Pre-existing", "known", "not introduced by this change", and "out of scope" are NEVER grounds to leave a defect in place; noticing a defect and walking past it is itself a defect. Two — and only two — exceptions: **(a)** the fix needs a product or architecture decision only the user can make — surface it explicitly and immediately, never bury it; **(b)** the fix is genuinely too entangled to land in the current change — then it is queued as an *immediate next cycle in the same run*, never as "later", never as a handoff-doc footnote. Never end a task, cycle, or run with a known issue unaddressed. Established by user directive 2026-05-14, re-affirmed 2026-05-15, 2026-05-17, and **2026-05-19** ("doesn't matter pre-existing or not; this directive holds for all future runs, every skill"). This generalizes and strengthens the rule-0e family (memory `feedback_fix_every_bug_found`). See the prominent top-of-file statement under "Working directive — LEAVE NOTHING BEHIND".
 - **EVERY FIX MUST BE GENERAL — serve all future PDFs, never a one-PDF quick-hack.** docpluck is a meta-science tool that processes arbitrary academic PDFs across many publishers. Every change must be keyed on a STRUCTURAL SIGNATURE — a typographic pattern, layout invariant, glyph-corruption shape, section-structure rule — never on paper identity, filename, or a string hard-coded from one PDF. A change that resolves one paper's quirk but risks regressions on others is the WRONG fix; find the general root cause. Regression tests use specific PDF fixtures, but the fix *logic* must generalize to any PDF with the same structural signature. Always run the full 26-paper baseline to confirm no regression; widen verification (broad-read, more AI-golds) when a fix touches a shared code path. Established by user directive 2026-05-15. See memory `feedback_general_fixes_not_pdf_specific`.
 - **NEVER swap the PDF text-extraction tool as a fix for downstream problems.** The TEXT channel is `extract_pdf` (pdftotext default mode); the LAYOUT channel is `extract_pdf_layout` (pdfplumber).  They are not interchangeable text sources.  Sections / normalize / batch consume the text channel; tables / figures / F0-layout-strip consume the layout channel.  Real-world-paper bugs (watermarks in body, abstract not detected, column interleaving) must be fixed in the layer that owns the artifact (`normalize.py` W0, `sections/annotators/text.py`, `sections/taxonomy.py`, `sections/core.py`) — not by switching extraction tools.  See [LESSONS.md L-001](./LESSONS.md#l-001--never-swap-the-pdf-text-extraction-tool-as-a-fix-for-downstream-problems) for the full incident record.

{docpluck-2.4.75 → docpluck-2.4.76}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docpluck
-Version: 2.4.75
+Version: 2.4.76
 Summary: PDF, DOCX, and HTML text extraction and normalization for academic papers
 Project-URL: Homepage, https://docpluck.app
 Project-URL: Documentation, https://docpluck.app/api-docs

{docpluck-2.4.75 → docpluck-2.4.76}/TODO.md

@@ -73,3 +73,66 @@ Deferred richer modes to consider once the two-mode default is in production and
 - **Custom placeholder template** — caller supplies `f"[{label}: {caption}]"` or similar.
 
 Add only when a real downstream consumer asks for one. YAGNI until then.
+
+---
+
+## 2026-05-25 wrapup punch-list — canary-audit + Cluster A/B/C + leftover v2.4.76
+
+> **Source:** Two-session wrapup combining R4 cycle work (uncommitted v2.4.76 from earlier today) and the canary-audit architecture session. See `docs/superpowers/handoffs/2026-05-25-canary-audit-architecture-and-cluster-A-B-C-landed.md` and `docs/superpowers/handoffs/2026-05-25-wrapup-r4-cycle.md` for full details. The 11th defect (`test_plos_med_1_no_fence_footer`) was fixed in the wrapup itself before commits.
+
+### Must do before v2.4.77 tag
+
+- [ ] **Run full pytest** to confirm combined v2.4.76 (R4 + EC-T1 + A4) + canary-audit (Cluster A/B/C/D-partial + PSPB-style heading + plos_med_1 P0r fix) state is clean. Estimated 24 min.
+- [ ] **Run `scripts/verify_corpus.py`** — R4 fires aggressively on chandrashekar_2023 (24 pages) and ip_feldman (14 pages). Confirm no regression vs the 26-paper baseline before tagging.
+- [ ] **Commit shape** (per 2026-05-25 wrapup decision): single combined v2.4.76 commit for R4 + EC-T1 + A4 + jama-open-1 D4 + plos_med_1 P0r fix, then separate `feat(canary-audit)` commit on top for Cluster A/B/C + audit infrastructure.
+
+### Canary-audit infrastructure — operational
+
+- [ ] **`claude setup-token`** — user runs this once in a regular terminal (interactive browser auth) so headless `claude -p --model sonnet` works from git hooks / scheduled tasks. Required before Phase 1 of canary-audit deploy.
+- [ ] **Wire git hooks** in `.git/hooks/`: pre-commit (quick canary on `docpluck/*.py` changes), pre-push (full canary on push to main), pre-tag (full canary, no exceptions). Each shells out to `~/.claude/skills/_shared/iterate-loop/canary-audit.sh`.
+- [ ] **Wire scheduled-tasks watchdog** via `mcp__scheduled-tasks__create_scheduled_task` — daily audit of HEAD against canary, PushNotification on FAIL.
+- [ ] **Double-audit + finding-union** mode in `canary-audit.sh` — runs Sonnet twice and unions findings (per `feedback_audit_nondeterminism_mitigation.md`). Currently the script runs Sonnet once.
+- [ ] **Persistent open-finding ledger** at `.claude/skills/_project/canary-findings-ledger.json` — once a defect is reported at HEAD SHA X, stays open until a later audit confirms clear.
+
+### Remaining ip_feldman_2025_pspb defects (14 in final audit)
+
+- [ ] **Front-matter leak lines 0-16** — article ID, journal banner, society copyright, DOI fragment emitted as body before Abstract. Needs P0 pre-pass for journal masthead block detection.
+- [ ] **Affiliation fragment "Fu Lam, Hong Kong SAR." (line 37)** — corresponding-author paragraph wrap-tail. Either pre-join wrapped affiliation paragraphs at P0 level OR add an orphan-wrap-tail pattern.
+- [ ] **Missing Method subsections** — `Design and Procedure`, `Power Analysis and Sensitivity Test`, `Measures`, `Data Analysis Strategy` still plain text. Investigate why these specific ones don't promote (others on same paper do).
+- [ ] **Missing Discussion subsection** `Challenging and Reframing Misestimation`.
+- [ ] **Table 10 phantom-guard** — th_section_leak heuristic didn't fire (cell content has hyphens splitting words like "cau-tion"). Debug + retune word-shape detection.
+- [ ] **Data Availability section absent from end-matter** — Cluster A demote-fix may have over-stripped the legitimate `## Data Availability` section that should remain after Author Contributions.
+- [ ] **False positive `### Reasons for change` (line 554)** — over-promotion in some context post-Cluster-B; investigate scope.
+- [ ] **Table 3 malformed** (cluster D-full Camelot — multi-session)
+- [ ] **Table 4 truncated** (Cluster D-full Camelot)
+- [ ] **Table 6 split rows** for multi-word items
+- [ ] **Table 8 split rows** for variable names
+- [ ] **Table 9 caption truncated mid-word** ("Versus" cut off, missing "Replication.")
+- [ ] **Table 9 Interpretation column** split across two `<tr>` rows.
+- [ ] **Table 10 no body** (caption only).
+
+### Cluster D-full Camelot tuning (deferred to its own session)
+
+- [ ] Stream-flavor column-tolerance tuning (`column_tol`, `row_tol`, `edge_tol` per-page or per-paper).
+- [ ] Post-Camelot header-cell splitter for concatenated columns (e.g. "Study 3Replication").
+- [ ] Multi-line cell wrap detection for body rows that span two PDF lines.
+- [ ] Multi-row header collapse (Table 5 fragmented headers).
+- [ ] Full 26-paper corpus regression-test infrastructure (table-specific AI gold view, not just `reading.md`).
+
+### R4 cycle residuals (from `2026-05-25-wrapup-r4-cycle.md`)
+
+- [ ] **R4 title truncation** — `Effect of Time-Restricted Eating on Weight Loss in Adu` cut at column midline.
+- [ ] **R4 multi-word-label splits** — `CONCLUSIONS AND RELEVANCE` becomes `**CONCLUSIONS**` heading + `AND RELEVANCE` orphan.
+- [ ] **R4 (continued) fragment** — `(continued)` page marker splits as `(contin` / `ued)`.
+- [ ] **R5 Path 1** — layout-channel per-char glyph identity recovery (bare table-cell betas, ar_apa_j_jesp_2009_12_011). Architectural multi-day work.
+
+### Article-finder skill issues (separate skill, separate session)
+
+- [ ] **Task #8**: `ai-gold.py resolve` should accept stem names + source-PDF paths (or docs redirect to `check`).
+- [ ] **Task #9**: `ai-gold.py onboard` needs `--skip-legacy` / `--ignore-unresolvable` flag (citationguard onboard halted on 3,018 legacy bare-stem keys).
+
+### Replicate canary-audit pattern to other iterate skills (after docpluck proven)
+
+- [ ] **escicheck-iterate** — easiest pilot (46 successful phase_5d_runs already, well-defined stats-family defect taxonomy). Update `verification_protocol` in its `canary.json`.
+- [ ] **2rmarkdown-iterate** — needs 3-tier verdict-vocabulary (GREEN/YELLOW/RED → PASS/FAIL/FAIL) integration in the orchestrator. Fixture-keyed corpus.
+- [ ] **citationguard-iterate** — needs corpus onboarding into article-finder FIRST (`corpus-query --source citationguard` returns 0 currently). Two-view-per-paper (`citations.v2` + `intext_citations.v1`).

{docpluck-2.4.75 → docpluck-2.4.76}/docpluck/__init__.py

@@ -67,11 +67,17 @@ from .sections import (
     SectionLabel, Confidence, DetectedVia, SECTIONING_VERSION,
 )
 from .tables import Cell, Table
+from .tables.flatten import (
+    FlattenedRow,
+    flatten_table,
+    flatten_tables_for_paper,
+    render_flattened_inline,
+)
 from .figures import Figure
 from .extract_structured import TABLE_EXTRACTION_VERSION, StructuredResult, extract_pdf_structured
 from .render import render_pdf_to_markdown
 
-__version__ = "2.4.75"
+__version__ = "2.4.76"
 __author__ = "Gilad Feldman"
 __license__ = "MIT"
 
@@ -109,6 +115,12 @@ __all__ = [
     "TABLE_EXTRACTION_VERSION",
     "StructuredResult",
     "extract_pdf_structured",
+    # Table row flattening (v2.4.76, EC-T1) — canonical JSONL contract for
+    # downstream stat-verification tools (effectcheck, escimate, scimeto).
+    "FlattenedRow",
+    "flatten_table",
+    "flatten_tables_for_paper",
+    "render_flattened_inline",
     # Markdown rendering (v2.2)
     "render_pdf_to_markdown",
 ]

{docpluck-2.4.75 → docpluck-2.4.76}/docpluck/cli.py

@@ -121,10 +121,40 @@ def _cmd_sections(args: argparse.Namespace) -> int:
 
 
 def _cmd_render(args: argparse.Namespace) -> int:
-    from . import render_pdf_to_markdown, NormalizationLevel
+    from . import (
+        NormalizationLevel,
+        extract_pdf_structured,
+        flatten_tables_for_paper,
+        render_pdf_to_markdown,
+    )
+
     blob = _read_bytes(args.file)
     level = NormalizationLevel(args.level)
-    md = render_pdf_to_markdown(blob, normalization_level=level)
+    flatten_inline = bool(getattr(args, "flatten_tables_inline", False))
+    tables_jsonl = getattr(args, "tables_jsonl", None)
+
+    # If the user asked for a JSONL sidecar, run structured extraction once
+    # and pass it to render so we don't pay Camelot twice.
+    if tables_jsonl:
+        structured = extract_pdf_structured(blob)
+        md = render_pdf_to_markdown(
+            blob,
+            normalization_level=level,
+            flatten_tables_inline=flatten_inline,
+            _structured=structured,
+        )
+        records = flatten_tables_for_paper(structured["tables"])
+        out_path = Path(tables_jsonl)
+        out_path.parent.mkdir(parents=True, exist_ok=True)
+        with out_path.open("w", encoding="utf-8") as fh:
+            for r in records:
+                fh.write(json.dumps(r, ensure_ascii=False) + "\n")
+    else:
+        md = render_pdf_to_markdown(
+            blob,
+            normalization_level=level,
+            flatten_tables_inline=flatten_inline,
+        )
     sys.stdout.write(md)
     return 0
 
@@ -141,7 +171,7 @@ def main(argv: list[str] | None = None) -> int:
         return 0
 
     if args_in[0] in ("-h", "--help", "help"):
-        print("usage: docpluck [--version | extract <file> [--sections L1,L2] [--structured [--thorough] [--text-mode raw|placeholder] [--tables-only|--figures-only] [--html-tables-to DIR]] | sections <file> [--format json|summary] | render <file> [--level none|standard|academic]]")
+        print("usage: docpluck [--version | extract <file> [--sections L1,L2] [--structured [--thorough] [--text-mode raw|placeholder] [--tables-only|--figures-only] [--html-tables-to DIR]] | sections <file> [--format json|summary] | render <file> [--level none|standard|academic] [--flatten-tables-inline] [--tables-jsonl PATH]]")
         return 0
 
     parser = argparse.ArgumentParser(prog="docpluck", add_help=True)
@@ -182,6 +212,25 @@ def main(argv: list[str] | None = None) -> int:
         choices=["none", "standard", "academic"],
         help="Normalization level applied during section detection. Default: standard.",
     )
+    render.add_argument(
+        "--flatten-tables-inline",
+        action="store_true",
+        dest="flatten_tables_inline",
+        help="Emit a human-readable 'rendered as text' block below each <table> "
+             "with one labelled sentence per body row (e.g. "
+             "'Importance: t(741) = 3.93, p < .001, d = 0.29'). Bounded by "
+             "HTML-comment sentinels. Default off — pass --tables-jsonl for "
+             "the structured form consumed by downstream stat-verification tools.",
+    )
+    render.add_argument(
+        "--tables-jsonl",
+        metavar="PATH",
+        default=None,
+        dest="tables_jsonl",
+        help="Write one FlattenedRow JSON record per line to PATH. Canonical "
+             "extraction contract for downstream stat-verification tools "
+             "(effectcheck, escimate, scimeto).",
+    )
     render.set_defaults(func=_cmd_render)
 
     try:

{docpluck-2.4.75 → docpluck-2.4.76}/docpluck/extract.py

@@ -122,6 +122,37 @@ def extract_pdf(pdf_bytes: bytes, *, sections: list[str] | None = None) -> tuple
                     text = patched
                     method = "pdftotext_default+pdfplumber_word_patch"
 
+        # §A R4 / B6 column-aware re-extraction (v2.4.76, 2026-05-25).
+        # Detector runs on form-feed-split pdftotext output (cheap, no
+        # pdfplumber call). If any page is flagged for column-interleave,
+        # extract the layout doc and rewrite those pages via pdftotext
+        # crop-mode (per-column pdftotext gives correct word spacing where
+        # pdfplumber `extract_text()` drops spaces on tight-kerned PDFs).
+        # The corrected text then flows through ALL downstream channels —
+        # sections, normalize, render, structured — because they all go
+        # through extract_pdf. Per CLAUDE.md hard rule 3, this is
+        # conditional fallback, NOT a default tool swap.
+        try:
+            from .normalize import _detect_column_interleave_pages
+            from .extract_columns import splice_column_corrected_pages
+            ff_offsets: list[int] = [0]
+            for idx, ch in enumerate(text):
+                if ch == "\f":
+                    ff_offsets.append(idx + 1)
+            flagged_pages = _detect_column_interleave_pages(text, tuple(ff_offsets))
+            if flagged_pages:
+                from .extract_layout import extract_pdf_layout
+                layout_doc = extract_pdf_layout(pdf_bytes)
+                corrected = splice_column_corrected_pages(
+                    text, layout_doc, ff_offsets, flagged_pages,
+                    pdf_bytes=pdf_bytes,
+                )
+                if corrected and corrected != text:
+                    text = corrected
+                    method = f"{method}+column_corrected:{','.join(map(str, flagged_pages))}"
+        except Exception:
+            pass
+
         if sections is not None:
             from .sections import extract_sections
             doc = extract_sections(pdf_bytes)