citegraph 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- citegraph/__init__.py +44 -0
- citegraph/_progress.py +74 -0
- citegraph/annotations.py +183 -0
- citegraph/author_export.py +294 -0
- citegraph/author_overrides.py +143 -0
- citegraph/authors.py +1997 -0
- citegraph/cli.py +926 -0
- citegraph/cocitation_stats.py +370 -0
- citegraph/config.py +73 -0
- citegraph/cost_estimation.py +259 -0
- citegraph/dedup.py +498 -0
- citegraph/enrich.py +972 -0
- citegraph/extract_metadata.py +81 -0
- citegraph/extract_references.py +376 -0
- citegraph/graph.py +660 -0
- citegraph/html_report.py +1787 -0
- citegraph/identity.py +199 -0
- citegraph/ids.py +106 -0
- citegraph/io.py +566 -0
- citegraph/jel.py +167 -0
- citegraph/journals.py +182 -0
- citegraph/llm.py +186 -0
- citegraph/pdf_to_markdown.py +394 -0
- citegraph/pipeline.py +997 -0
- citegraph/plotting.py +1214 -0
- citegraph/reports.py +214 -0
- citegraph/schemas.py +89 -0
- citegraph/webui.py +624 -0
- citegraph-0.2.0.dist-info/METADATA +870 -0
- citegraph-0.2.0.dist-info/RECORD +33 -0
- citegraph-0.2.0.dist-info/WHEEL +4 -0
- citegraph-0.2.0.dist-info/entry_points.txt +2 -0
- citegraph-0.2.0.dist-info/licenses/LICENSE +21 -0
citegraph/html_report.py
ADDED
|
@@ -0,0 +1,1787 @@
|
|
|
1
|
+
"""Self-contained HTML QC dashboard for a citegraph out directory.
|
|
2
|
+
|
|
3
|
+
``citegraph report --out ./out`` (or :func:`write_report`) reads whatever
|
|
4
|
+
artifacts exist under ``out_dir`` and writes a single ``report.html`` next
|
|
5
|
+
to them. The page is fully self-contained — no CDN scripts, no webfonts,
|
|
6
|
+
no external requests — so it can be opened from disk and archived with the
|
|
7
|
+
CSVs.
|
|
8
|
+
|
|
9
|
+
The module is deliberately split into two halves so the data layer is
|
|
10
|
+
testable without parsing HTML:
|
|
11
|
+
|
|
12
|
+
- :func:`collect_report_data` gathers everything into one JSON-serializable
|
|
13
|
+
dict. Every individual file read is fault-tolerant: a corrupt JSON/CSV is
|
|
14
|
+
recorded as ``{"path": ..., "error": ...}`` in ``data["errors"]`` instead
|
|
15
|
+
of raising. A missing or empty ``out_dir`` yields a valid "nothing yet"
|
|
16
|
+
structure.
|
|
17
|
+
- :func:`build_report_html` renders that dict to HTML. All user-derived
|
|
18
|
+
strings are escaped server-side; the embedded JSON blob additionally has
|
|
19
|
+
``<``/``>``/``&`` escaped as ``\\uXXXX`` so it is inert markup.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import html
|
|
25
|
+
import json
|
|
26
|
+
import math
|
|
27
|
+
from datetime import UTC, datetime
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
from typing import Any
|
|
30
|
+
|
|
31
|
+
import pandas as pd
|
|
32
|
+
|
|
33
|
+
from citegraph.io import (
|
|
34
|
+
AUTHOR_AUDIT_INPUTS,
|
|
35
|
+
AUTHOR_AUDIT_OUTPUTS,
|
|
36
|
+
WORK_AUDIT_INPUTS,
|
|
37
|
+
WORK_AUDIT_OUTPUTS,
|
|
38
|
+
OutLayout,
|
|
39
|
+
artifact_fingerprint,
|
|
40
|
+
read_json,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
__all__ = ["collect_report_data", "build_report_html", "write_report"]
|
|
44
|
+
|
|
45
|
+
# Mirrors ``citegraph.pdf_to_markdown._is_image_only`` but operates on text
|
|
46
|
+
# already read (each markdown file is read exactly once during collection).
|
|
47
|
+
_IMAGE_ONLY_MIN_CHARS = 200
|
|
48
|
+
|
|
49
|
+
# Per-paper preview caps (bytes of markdown, number of references).
|
|
50
|
+
_MARKDOWN_PREVIEW_CHARS = 2_500
|
|
51
|
+
_REFS_PREVIEW_N = 15
|
|
52
|
+
# When the rendered page would exceed this, previews are shrunk and a note
|
|
53
|
+
# is added to the page.
|
|
54
|
+
_MAX_HTML_BYTES = 10 * 1024 * 1024
|
|
55
|
+
_SHRUNK_MARKDOWN_PREVIEW_CHARS = 500
|
|
56
|
+
_SHRUNK_REFS_PREVIEW_N = 5
|
|
57
|
+
|
|
58
|
+
_SUSPICIOUSLY_FEW_REFS = 5
|
|
59
|
+
_SHORT_TITLE_CHARS = 10
|
|
60
|
+
_MERGE_AUDIT_MAX_CLUSTERS = 200
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
# ---------------------------------------------------------------------------
|
|
64
|
+
# Fault-tolerant readers
|
|
65
|
+
# ---------------------------------------------------------------------------
|
|
66
|
+
def _record_error(errors: list[dict], path: Path, exc: Exception) -> None:
|
|
67
|
+
errors.append({"path": str(path), "error": f"{type(exc).__name__}: {exc}"})
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _read_text_safe(path: Path, errors: list[dict]) -> str | None:
|
|
71
|
+
try:
|
|
72
|
+
return path.read_text(encoding="utf-8")
|
|
73
|
+
except Exception as exc: # noqa: BLE001 - corrupt files are a finding, not a crash
|
|
74
|
+
_record_error(errors, path, exc)
|
|
75
|
+
return None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _read_json_safe(path: Path, errors: list[dict]) -> Any:
|
|
79
|
+
try:
|
|
80
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
81
|
+
except Exception as exc: # noqa: BLE001
|
|
82
|
+
_record_error(errors, path, exc)
|
|
83
|
+
return None
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _read_jsonl_safe(path: Path, errors: list[dict]) -> list[dict] | None:
|
|
87
|
+
try:
|
|
88
|
+
text = path.read_text(encoding="utf-8")
|
|
89
|
+
except Exception as exc: # noqa: BLE001
|
|
90
|
+
_record_error(errors, path, exc)
|
|
91
|
+
return None
|
|
92
|
+
rows: list[dict] = []
|
|
93
|
+
for lineno, line in enumerate(text.splitlines(), start=1):
|
|
94
|
+
if not line.strip():
|
|
95
|
+
continue
|
|
96
|
+
try:
|
|
97
|
+
rows.append(json.loads(line))
|
|
98
|
+
except Exception as exc: # noqa: BLE001
|
|
99
|
+
_record_error(errors, path, ValueError(f"line {lineno}: {exc}"))
|
|
100
|
+
return rows
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _read_csv_safe(path: Path, errors: list[dict], **kwargs: Any) -> pd.DataFrame | None:
|
|
104
|
+
try:
|
|
105
|
+
return pd.read_csv(path, **kwargs)
|
|
106
|
+
except Exception as exc: # noqa: BLE001
|
|
107
|
+
_record_error(errors, path, exc)
|
|
108
|
+
return None
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
# ---------------------------------------------------------------------------
|
|
112
|
+
# Small value helpers
|
|
113
|
+
# ---------------------------------------------------------------------------
|
|
114
|
+
def _is_image_only_text(text: str, min_text_chars: int = _IMAGE_ONLY_MIN_CHARS) -> bool:
|
|
115
|
+
stripped = text.replace("<!-- image -->", "")
|
|
116
|
+
lines = [ln for ln in stripped.splitlines() if not ln.lstrip().startswith("#")]
|
|
117
|
+
return len("".join(lines).strip()) < min_text_chars
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _as_int(value: object) -> int | None:
|
|
121
|
+
try:
|
|
122
|
+
if value is None or (isinstance(value, float) and math.isnan(value)):
|
|
123
|
+
return None
|
|
124
|
+
return int(value) # type: ignore[arg-type]
|
|
125
|
+
except (TypeError, ValueError):
|
|
126
|
+
return None
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _as_str(value: object) -> str:
|
|
130
|
+
if value is None:
|
|
131
|
+
return ""
|
|
132
|
+
if isinstance(value, float) and math.isnan(value):
|
|
133
|
+
return ""
|
|
134
|
+
return str(value)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _jsonable(obj: Any) -> Any:
|
|
138
|
+
"""Coerce numpy scalars / NaN into plain JSON-safe Python values."""
|
|
139
|
+
if isinstance(obj, dict):
|
|
140
|
+
return {str(k): _jsonable(v) for k, v in obj.items()}
|
|
141
|
+
if isinstance(obj, (list, tuple)):
|
|
142
|
+
return [_jsonable(v) for v in obj]
|
|
143
|
+
if isinstance(obj, float):
|
|
144
|
+
return None if math.isnan(obj) or math.isinf(obj) else obj
|
|
145
|
+
if isinstance(obj, (str, int, bool)) or obj is None:
|
|
146
|
+
return obj
|
|
147
|
+
item = getattr(obj, "item", None)
|
|
148
|
+
if callable(item): # numpy scalar
|
|
149
|
+
try:
|
|
150
|
+
return _jsonable(item())
|
|
151
|
+
except Exception: # noqa: BLE001
|
|
152
|
+
return str(obj)
|
|
153
|
+
return str(obj)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
# ---------------------------------------------------------------------------
|
|
157
|
+
# Data collection
|
|
158
|
+
# ---------------------------------------------------------------------------
|
|
159
|
+
def collect_report_data(out_dir: Path, *, include_markdown_previews: bool = True) -> dict:
|
|
160
|
+
"""Gather everything the report shows into one JSON-serializable dict.
|
|
161
|
+
|
|
162
|
+
Never raises for missing or corrupt artifacts: a nonexistent ``out_dir``
|
|
163
|
+
returns a valid "nothing yet" structure and every unreadable file is
|
|
164
|
+
recorded in ``data["errors"]``.
|
|
165
|
+
|
|
166
|
+
``include_markdown_previews=False`` omits the per-paper markdown preview
|
|
167
|
+
text (``markdown_preview`` is ``None``) while still computing char counts
|
|
168
|
+
and the image-only check — used by ``citegraph ui``, which fetches
|
|
169
|
+
previews lazily per paper instead of shipping them all in one payload.
|
|
170
|
+
"""
|
|
171
|
+
out_dir = Path(out_dir)
|
|
172
|
+
layout = OutLayout(out_dir)
|
|
173
|
+
errors: list[dict] = []
|
|
174
|
+
|
|
175
|
+
data: dict = {
|
|
176
|
+
"out_dir": str(out_dir),
|
|
177
|
+
"out_dir_exists": out_dir.is_dir(),
|
|
178
|
+
"generated_at": datetime.now(UTC).strftime("%Y-%m-%d %H:%M UTC"),
|
|
179
|
+
"errors": errors,
|
|
180
|
+
"truncated_for_size": False,
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
# --- sidecars -----------------------------------------------------
|
|
184
|
+
conversion_warnings = (
|
|
185
|
+
_read_json_safe(layout.conversion_warnings_json, errors)
|
|
186
|
+
if layout.conversion_warnings_json.exists()
|
|
187
|
+
else None
|
|
188
|
+
)
|
|
189
|
+
source_duplicates = (
|
|
190
|
+
_read_json_safe(layout.source_duplicates_json, errors)
|
|
191
|
+
if layout.source_duplicates_json.exists()
|
|
192
|
+
else None
|
|
193
|
+
)
|
|
194
|
+
papers_no_refs = (
|
|
195
|
+
_read_json_safe(layout.papers_no_references_json, errors)
|
|
196
|
+
if layout.papers_no_references_json.exists()
|
|
197
|
+
else None
|
|
198
|
+
)
|
|
199
|
+
metadata_failures = (
|
|
200
|
+
_read_jsonl_safe(layout.metadata_failures_jsonl, errors)
|
|
201
|
+
if layout.metadata_failures_jsonl.exists()
|
|
202
|
+
else None
|
|
203
|
+
)
|
|
204
|
+
references_failures = (
|
|
205
|
+
_read_jsonl_safe(layout.references_failures_jsonl, errors)
|
|
206
|
+
if layout.references_failures_jsonl.exists()
|
|
207
|
+
else None
|
|
208
|
+
)
|
|
209
|
+
author_review = (
|
|
210
|
+
_read_json_safe(layout.author_review_json, errors)
|
|
211
|
+
if layout.author_review_json.exists()
|
|
212
|
+
else None
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
duplicate_of: dict[str, str] = {} # duplicate stem -> canonical source file
|
|
216
|
+
if isinstance(source_duplicates, list):
|
|
217
|
+
for group in source_duplicates:
|
|
218
|
+
if not isinstance(group, dict):
|
|
219
|
+
continue
|
|
220
|
+
canonical = _as_str(group.get("canonical_source_file"))
|
|
221
|
+
for dup in group.get("duplicate_source_files", []):
|
|
222
|
+
duplicate_of[Path(_as_str(dup)).stem] = canonical
|
|
223
|
+
|
|
224
|
+
metadata_failed_stems = {
|
|
225
|
+
Path(_as_str(f.get("source_file"))).stem for f in (metadata_failures or [])
|
|
226
|
+
}
|
|
227
|
+
references_failed_stems = {
|
|
228
|
+
Path(_as_str(f.get("source_file"))).stem for f in (references_failures or [])
|
|
229
|
+
}
|
|
230
|
+
no_ref_stems = {
|
|
231
|
+
Path(_as_str(e.get("source_file"))).stem
|
|
232
|
+
for e in (papers_no_refs or [])
|
|
233
|
+
if isinstance(e, dict)
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
# --- per-paper rows -----------------------------------------------
|
|
237
|
+
md_stems = (
|
|
238
|
+
{p.stem for p in layout.markdown_dir.glob("*.md")}
|
|
239
|
+
if layout.markdown_dir.is_dir()
|
|
240
|
+
else set()
|
|
241
|
+
)
|
|
242
|
+
meta_stems = (
|
|
243
|
+
{p.stem for p in layout.metadata_dir.glob("*.json")}
|
|
244
|
+
if layout.metadata_dir.is_dir()
|
|
245
|
+
else set()
|
|
246
|
+
)
|
|
247
|
+
ref_stems = (
|
|
248
|
+
{p.stem for p in layout.references_dir.glob("*.json")}
|
|
249
|
+
if layout.references_dir.is_dir()
|
|
250
|
+
else set()
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
papers: list[dict] = []
|
|
254
|
+
n_image_only = 0
|
|
255
|
+
n_zero_refs = 0
|
|
256
|
+
n_pending = 0
|
|
257
|
+
for stem in sorted(md_stems | meta_stems | ref_stems):
|
|
258
|
+
row = _collect_paper_row(
|
|
259
|
+
stem,
|
|
260
|
+
layout,
|
|
261
|
+
errors,
|
|
262
|
+
in_markdown=stem in md_stems,
|
|
263
|
+
in_metadata=stem in meta_stems,
|
|
264
|
+
in_references=stem in ref_stems,
|
|
265
|
+
duplicate_of=duplicate_of.get(stem),
|
|
266
|
+
metadata_failed=stem in metadata_failed_stems,
|
|
267
|
+
references_failed=stem in references_failed_stems,
|
|
268
|
+
no_refs_flagged=stem in no_ref_stems,
|
|
269
|
+
include_markdown_preview=include_markdown_previews,
|
|
270
|
+
)
|
|
271
|
+
if row["markdown"]["status"] == "image-only":
|
|
272
|
+
n_image_only += 1
|
|
273
|
+
if row["references"]["status"] == "zero":
|
|
274
|
+
n_zero_refs += 1
|
|
275
|
+
if row["metadata"]["status"] == "pending" or row["references"]["status"] == "pending":
|
|
276
|
+
n_pending += 1
|
|
277
|
+
papers.append(row)
|
|
278
|
+
data["papers"] = papers
|
|
279
|
+
|
|
280
|
+
# --- flat CSV artifacts -------------------------------------------
|
|
281
|
+
sources_df = (
|
|
282
|
+
_read_csv_safe(layout.sources_csv, errors) if layout.sources_csv.exists() else None
|
|
283
|
+
)
|
|
284
|
+
raw_refs_df = (
|
|
285
|
+
_read_csv_safe(layout.citations_raw_csv, errors)
|
|
286
|
+
if layout.citations_raw_csv.exists()
|
|
287
|
+
else None
|
|
288
|
+
)
|
|
289
|
+
works_df = (
|
|
290
|
+
_read_csv_safe(layout.works_csv, errors, index_col="id")
|
|
291
|
+
if layout.works_csv.exists()
|
|
292
|
+
else None
|
|
293
|
+
)
|
|
294
|
+
graph_df = (
|
|
295
|
+
_read_csv_safe(layout.graph_csv, errors) if layout.graph_csv.exists() else None
|
|
296
|
+
)
|
|
297
|
+
enriched_df = (
|
|
298
|
+
_read_csv_safe(layout.enriched_works_csv, errors)
|
|
299
|
+
if layout.enriched_works_csv.exists()
|
|
300
|
+
else None
|
|
301
|
+
)
|
|
302
|
+
|
|
303
|
+
# Pre-works-model out_dirs: surface as a finding, never a crash.
|
|
304
|
+
legacy_papers_csv = layout.out_dir / "papers.csv"
|
|
305
|
+
if works_df is None and legacy_papers_csv.exists():
|
|
306
|
+
errors.append(
|
|
307
|
+
{
|
|
308
|
+
"path": str(legacy_papers_csv),
|
|
309
|
+
"error": (
|
|
310
|
+
"legacy pre-works-model artifacts detected — re-run "
|
|
311
|
+
"`citegraph metadata`, `citegraph references`, and "
|
|
312
|
+
"`citegraph dedup` (per-paper caches are reused)"
|
|
313
|
+
),
|
|
314
|
+
}
|
|
315
|
+
)
|
|
316
|
+
misses_df = (
|
|
317
|
+
_read_csv_safe(layout.enrichment_misses_csv, errors)
|
|
318
|
+
if layout.enrichment_misses_csv.exists()
|
|
319
|
+
else None
|
|
320
|
+
)
|
|
321
|
+
enrichment_summary = (
|
|
322
|
+
_read_json_safe(layout.enrichment_summary_json, errors)
|
|
323
|
+
if layout.enrichment_summary_json.exists()
|
|
324
|
+
else None
|
|
325
|
+
)
|
|
326
|
+
authors_df = (
|
|
327
|
+
_read_csv_safe(layout.authors_csv, errors, index_col="id")
|
|
328
|
+
if layout.authors_csv.exists()
|
|
329
|
+
else None
|
|
330
|
+
)
|
|
331
|
+
author_citations_df = (
|
|
332
|
+
_read_csv_safe(layout.author_citations_csv, errors)
|
|
333
|
+
if layout.author_citations_csv.exists()
|
|
334
|
+
else None
|
|
335
|
+
)
|
|
336
|
+
run_summary = (
|
|
337
|
+
_read_json_safe(layout.out_dir / "run_summary.json", errors)
|
|
338
|
+
if (layout.out_dir / "run_summary.json").exists()
|
|
339
|
+
else None
|
|
340
|
+
)
|
|
341
|
+
artifact_manifest = (
|
|
342
|
+
_read_json_safe(layout.artifact_manifest_json, errors)
|
|
343
|
+
if layout.artifact_manifest_json.exists()
|
|
344
|
+
else None
|
|
345
|
+
)
|
|
346
|
+
|
|
347
|
+
# --- stage progress strip -----------------------------------------
|
|
348
|
+
n_md = len(md_stems)
|
|
349
|
+
n_meta = len(meta_stems)
|
|
350
|
+
n_ref = len(ref_stems)
|
|
351
|
+
n_ref_expected = n_md
|
|
352
|
+
data["stages"] = _collect_stages(
|
|
353
|
+
layout,
|
|
354
|
+
n_md=n_md,
|
|
355
|
+
n_meta=n_meta,
|
|
356
|
+
n_ref=n_ref,
|
|
357
|
+
n_ref_expected=n_ref_expected,
|
|
358
|
+
works_df=works_df,
|
|
359
|
+
graph_df=graph_df,
|
|
360
|
+
enriched_df=enriched_df,
|
|
361
|
+
authors_df=authors_df,
|
|
362
|
+
)
|
|
363
|
+
|
|
364
|
+
# --- works-model summary ------------------------------------------
|
|
365
|
+
ring_counts: dict[int, int] = {}
|
|
366
|
+
core_to_core: list[dict] = []
|
|
367
|
+
if works_df is not None and "ring" in works_df.columns:
|
|
368
|
+
rings = pd.to_numeric(works_df["ring"], errors="coerce")
|
|
369
|
+
ring_counts = {
|
|
370
|
+
int(k): int(v) for k, v in rings.value_counts().items() if pd.notna(k)
|
|
371
|
+
}
|
|
372
|
+
if graph_df is not None and {"citing_id", "cited_id"} <= set(graph_df.columns):
|
|
373
|
+
ring0 = set(works_df.index[rings == 0])
|
|
374
|
+
cc = graph_df[
|
|
375
|
+
graph_df["citing_id"].isin(ring0) & graph_df["cited_id"].isin(ring0)
|
|
376
|
+
]
|
|
377
|
+
core_to_core = [
|
|
378
|
+
{"citing_id": _as_str(r["citing_id"]), "cited_id": _as_str(r["cited_id"])}
|
|
379
|
+
for _, r in cc.iterrows()
|
|
380
|
+
]
|
|
381
|
+
data["ring_counts"] = ring_counts
|
|
382
|
+
data["core_to_core_edges"] = core_to_core
|
|
383
|
+
|
|
384
|
+
# --- panels -------------------------------------------------------
|
|
385
|
+
data["dedup"] = _collect_dedup_panel(
|
|
386
|
+
raw_refs_df, works_df, graph_df, errors, layout, sources_df=sources_df
|
|
387
|
+
)
|
|
388
|
+
if data["dedup"] is not None:
|
|
389
|
+
data["dedup"]["ring_counts"] = ring_counts
|
|
390
|
+
data["dedup"]["core_to_core"] = core_to_core
|
|
391
|
+
data["enrichment"] = _collect_enrichment_panel(enriched_df, misses_df, enrichment_summary)
|
|
392
|
+
data["authors"] = _collect_authors_panel(authors_df, author_citations_df, author_review)
|
|
393
|
+
if data["authors"] is not None:
|
|
394
|
+
data["authors"].update(_collect_resolution_audit(
|
|
395
|
+
layout.author_resolution_audit_json, errors,
|
|
396
|
+
inputs=AUTHOR_AUDIT_INPUTS, outputs=AUTHOR_AUDIT_OUTPUTS,
|
|
397
|
+
))
|
|
398
|
+
data["authors"].pop("saved_audit", None)
|
|
399
|
+
|
|
400
|
+
# --- problems summary ---------------------------------------------
|
|
401
|
+
n_missing_year = 0
|
|
402
|
+
if data["dedup"] is not None:
|
|
403
|
+
n_missing_year = data["dedup"].get("n_missing_year") or 0
|
|
404
|
+
cards = [
|
|
405
|
+
("conversion_warnings", "image-only markdown", n_image_only, "papers"),
|
|
406
|
+
("metadata_failures", "metadata failures", len(metadata_failures or []), "papers"),
|
|
407
|
+
("references_failures", "references failures", len(references_failures or []), "papers"),
|
|
408
|
+
("zero_references", "papers with zero references", n_zero_refs, "papers"),
|
|
409
|
+
("source_duplicates", "duplicate source PDFs", len(duplicate_of), "papers"),
|
|
410
|
+
("corrupt_files", "corrupt / unreadable files", len(errors), "integrity"),
|
|
411
|
+
("author_review", "author clusters to review", len(author_review or []), "authors"),
|
|
412
|
+
("missing_year", "references without a year", n_missing_year, "dedup"),
|
|
413
|
+
("pending_papers", "papers not fully extracted", n_pending, "papers"),
|
|
414
|
+
]
|
|
415
|
+
data["problems"] = {
|
|
416
|
+
"cards": [
|
|
417
|
+
{"key": key, "label": label, "count": int(count), "anchor": anchor}
|
|
418
|
+
for key, label, count, anchor in cards
|
|
419
|
+
if count
|
|
420
|
+
],
|
|
421
|
+
"total": int(sum(count for _, _, count, _ in cards)),
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
# --- integrity table ----------------------------------------------
|
|
425
|
+
data["integrity"] = _collect_integrity(
|
|
426
|
+
layout,
|
|
427
|
+
errors,
|
|
428
|
+
rows=[
|
|
429
|
+
("sources.csv", layout.sources_csv, sources_df is not None, "stage 2"),
|
|
430
|
+
("citations_raw.csv", layout.citations_raw_csv, raw_refs_df is not None, "stage 3"),
|
|
431
|
+
("works.csv", layout.works_csv, works_df is not None, "stage 4"),
|
|
432
|
+
("citation_graph.csv", layout.graph_csv, graph_df is not None, "stage 4"),
|
|
433
|
+
(
|
|
434
|
+
"enriched_works.csv",
|
|
435
|
+
layout.enriched_works_csv,
|
|
436
|
+
enriched_df is not None,
|
|
437
|
+
"stage 5 (optional)",
|
|
438
|
+
),
|
|
439
|
+
(
|
|
440
|
+
"enrichment_summary.json",
|
|
441
|
+
layout.enrichment_summary_json,
|
|
442
|
+
enrichment_summary is not None,
|
|
443
|
+
"stage 5 (optional)",
|
|
444
|
+
),
|
|
445
|
+
(
|
|
446
|
+
"enrichment_misses.csv",
|
|
447
|
+
layout.enrichment_misses_csv,
|
|
448
|
+
misses_df is not None,
|
|
449
|
+
"stage 5 (optional)",
|
|
450
|
+
),
|
|
451
|
+
("authors.csv", layout.authors_csv, authors_df is not None, "stage 6 (optional)"),
|
|
452
|
+
(
|
|
453
|
+
"author_citations.csv",
|
|
454
|
+
layout.author_citations_csv,
|
|
455
|
+
author_citations_df is not None,
|
|
456
|
+
"stage 6 (optional)",
|
|
457
|
+
),
|
|
458
|
+
(
|
|
459
|
+
"author_review.json",
|
|
460
|
+
layout.author_review_json,
|
|
461
|
+
author_review is not None,
|
|
462
|
+
"sidecar — absent = clean",
|
|
463
|
+
),
|
|
464
|
+
(
|
|
465
|
+
"conversion_warnings.json",
|
|
466
|
+
layout.conversion_warnings_json,
|
|
467
|
+
conversion_warnings is not None,
|
|
468
|
+
"sidecar — absent = clean",
|
|
469
|
+
),
|
|
470
|
+
(
|
|
471
|
+
"source_duplicates.json",
|
|
472
|
+
layout.source_duplicates_json,
|
|
473
|
+
source_duplicates is not None,
|
|
474
|
+
"sidecar — absent = clean",
|
|
475
|
+
),
|
|
476
|
+
(
|
|
477
|
+
"papers_no_references.json",
|
|
478
|
+
layout.papers_no_references_json,
|
|
479
|
+
papers_no_refs is not None,
|
|
480
|
+
"sidecar — absent = clean",
|
|
481
|
+
),
|
|
482
|
+
(
|
|
483
|
+
"metadata_failures.jsonl",
|
|
484
|
+
layout.metadata_failures_jsonl,
|
|
485
|
+
metadata_failures is not None,
|
|
486
|
+
"sidecar — absent = clean",
|
|
487
|
+
),
|
|
488
|
+
(
|
|
489
|
+
"references_failures.jsonl",
|
|
490
|
+
layout.references_failures_jsonl,
|
|
491
|
+
references_failures is not None,
|
|
492
|
+
"sidecar — absent = clean",
|
|
493
|
+
),
|
|
494
|
+
(
|
|
495
|
+
"run_summary.json",
|
|
496
|
+
layout.out_dir / "run_summary.json",
|
|
497
|
+
run_summary is not None,
|
|
498
|
+
"written by `citegraph run`",
|
|
499
|
+
),
|
|
500
|
+
(
|
|
501
|
+
"artifact_manifest.json",
|
|
502
|
+
layout.artifact_manifest_json,
|
|
503
|
+
artifact_manifest is not None,
|
|
504
|
+
"written by `citegraph run`",
|
|
505
|
+
),
|
|
506
|
+
],
|
|
507
|
+
)
|
|
508
|
+
|
|
509
|
+
return _jsonable(data)
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def _collect_paper_row(
|
|
513
|
+
stem: str,
|
|
514
|
+
layout: OutLayout,
|
|
515
|
+
errors: list[dict],
|
|
516
|
+
*,
|
|
517
|
+
in_markdown: bool,
|
|
518
|
+
in_metadata: bool,
|
|
519
|
+
in_references: bool,
|
|
520
|
+
duplicate_of: str | None,
|
|
521
|
+
metadata_failed: bool,
|
|
522
|
+
references_failed: bool,
|
|
523
|
+
no_refs_flagged: bool,
|
|
524
|
+
include_markdown_preview: bool = True,
|
|
525
|
+
) -> dict:
|
|
526
|
+
notes: list[str] = []
|
|
527
|
+
|
|
528
|
+
# -- markdown / conversion
|
|
529
|
+
markdown: dict = {"status": "missing", "chars": None}
|
|
530
|
+
markdown_preview: str | None = None
|
|
531
|
+
if in_markdown:
|
|
532
|
+
text = _read_text_safe(layout.markdown_dir / f"{stem}.md", errors)
|
|
533
|
+
if text is None:
|
|
534
|
+
markdown = {"status": "error", "chars": None}
|
|
535
|
+
else:
|
|
536
|
+
markdown = {
|
|
537
|
+
"status": "image-only" if _is_image_only_text(text) else "ok",
|
|
538
|
+
"chars": len(text),
|
|
539
|
+
}
|
|
540
|
+
if include_markdown_preview:
|
|
541
|
+
markdown_preview = text[:_MARKDOWN_PREVIEW_CHARS]
|
|
542
|
+
|
|
543
|
+
# -- metadata
|
|
544
|
+
metadata: dict = {"status": "pending", "title": "", "authors": "", "journal": "", "year": None}
|
|
545
|
+
if in_metadata:
|
|
546
|
+
obj = _read_json_safe(layout.metadata_dir / f"{stem}.json", errors)
|
|
547
|
+
if not isinstance(obj, dict):
|
|
548
|
+
metadata["status"] = "corrupt"
|
|
549
|
+
else:
|
|
550
|
+
metadata = {
|
|
551
|
+
"status": "ok",
|
|
552
|
+
"title": _as_str(obj.get("Title")),
|
|
553
|
+
"authors": ", ".join(str(a) for a in obj.get("Authors_List") or [])
|
|
554
|
+
or _as_str(obj.get("Authors")),
|
|
555
|
+
"journal": _as_str(obj.get("Journal")),
|
|
556
|
+
"year": _as_int(obj.get("Year")),
|
|
557
|
+
}
|
|
558
|
+
elif metadata_failed:
|
|
559
|
+
metadata["status"] = "failed"
|
|
560
|
+
|
|
561
|
+
# -- references
|
|
562
|
+
references: dict = {"status": "pending", "count": None}
|
|
563
|
+
refs_preview: list[dict] = []
|
|
564
|
+
refs_more = 0
|
|
565
|
+
if in_references:
|
|
566
|
+
obj = _read_json_safe(layout.references_dir / f"{stem}.json", errors)
|
|
567
|
+
if not isinstance(obj, list):
|
|
568
|
+
references["status"] = "corrupt"
|
|
569
|
+
else:
|
|
570
|
+
count = len(obj)
|
|
571
|
+
references = {"status": "zero" if count == 0 else "ok", "count": count}
|
|
572
|
+
if 0 < count < _SUSPICIOUSLY_FEW_REFS:
|
|
573
|
+
notes.append(f"suspiciously few references ({count})")
|
|
574
|
+
for ref in obj[:_REFS_PREVIEW_N]:
|
|
575
|
+
if not isinstance(ref, dict):
|
|
576
|
+
continue
|
|
577
|
+
refs_preview.append(
|
|
578
|
+
{
|
|
579
|
+
"title": _as_str(ref.get("Title")),
|
|
580
|
+
"authors": ", ".join(str(a) for a in ref.get("Authors_List") or [])
|
|
581
|
+
or _as_str(ref.get("Authors")),
|
|
582
|
+
"year": _as_int(ref.get("Year")),
|
|
583
|
+
}
|
|
584
|
+
)
|
|
585
|
+
refs_more = max(count - _REFS_PREVIEW_N, 0)
|
|
586
|
+
elif references_failed:
|
|
587
|
+
references["status"] = "failed"
|
|
588
|
+
elif duplicate_of:
|
|
589
|
+
references["status"] = "skipped"
|
|
590
|
+
|
|
591
|
+
if duplicate_of:
|
|
592
|
+
notes.append(f"duplicate of {Path(duplicate_of).stem}")
|
|
593
|
+
if no_refs_flagged and references["status"] != "zero":
|
|
594
|
+
notes.append("flagged in papers_no_references.json")
|
|
595
|
+
|
|
596
|
+
problem = bool(
|
|
597
|
+
markdown["status"] != "ok"
|
|
598
|
+
or metadata["status"] != "ok"
|
|
599
|
+
or references["status"] not in ("ok", "skipped")
|
|
600
|
+
or notes
|
|
601
|
+
)
|
|
602
|
+
return {
|
|
603
|
+
"stem": stem,
|
|
604
|
+
"markdown": markdown,
|
|
605
|
+
"markdown_preview": markdown_preview,
|
|
606
|
+
"metadata": metadata,
|
|
607
|
+
"references": references,
|
|
608
|
+
"refs_preview": refs_preview,
|
|
609
|
+
"refs_more": refs_more,
|
|
610
|
+
"notes": notes,
|
|
611
|
+
"problem": problem,
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
|
|
615
|
+
def _collect_stages(
|
|
616
|
+
layout: OutLayout,
|
|
617
|
+
*,
|
|
618
|
+
n_md: int,
|
|
619
|
+
n_meta: int,
|
|
620
|
+
n_ref: int,
|
|
621
|
+
n_ref_expected: int,
|
|
622
|
+
works_df: pd.DataFrame | None,
|
|
623
|
+
graph_df: pd.DataFrame | None,
|
|
624
|
+
enriched_df: pd.DataFrame | None,
|
|
625
|
+
authors_df: pd.DataFrame | None,
|
|
626
|
+
) -> list[dict]:
|
|
627
|
+
out = layout.out_dir
|
|
628
|
+
|
|
629
|
+
def cache_stage(label: str, cached: int, expected: int, csv_exists: bool, cmd: str) -> dict:
|
|
630
|
+
if cached == 0 and not csv_exists:
|
|
631
|
+
return {"label": label, "status": "not_run", "detail": "not run", "hint": cmd}
|
|
632
|
+
if expected and cached < expected:
|
|
633
|
+
return {
|
|
634
|
+
"label": label,
|
|
635
|
+
"status": "partial",
|
|
636
|
+
"detail": f"{cached}/{expected} papers cached",
|
|
637
|
+
"hint": cmd,
|
|
638
|
+
}
|
|
639
|
+
return {"label": label, "status": "done", "detail": f"{cached} papers cached", "hint": ""}
|
|
640
|
+
|
|
641
|
+
stages = [
|
|
642
|
+
{
|
|
643
|
+
"label": "convert",
|
|
644
|
+
"status": "done" if n_md else "not_run",
|
|
645
|
+
"detail": f"{n_md} markdown file(s)" if n_md else "not run",
|
|
646
|
+
"hint": f"citegraph convert <pdf_dir> --out {out}" if not n_md else "",
|
|
647
|
+
},
|
|
648
|
+
cache_stage(
|
|
649
|
+
"metadata", n_meta, n_md, layout.sources_csv.exists(), f"citegraph metadata --out {out}"
|
|
650
|
+
),
|
|
651
|
+
cache_stage(
|
|
652
|
+
"references",
|
|
653
|
+
n_ref,
|
|
654
|
+
n_ref_expected,
|
|
655
|
+
layout.citations_raw_csv.exists(),
|
|
656
|
+
f"citegraph references --out {out}",
|
|
657
|
+
),
|
|
658
|
+
]
|
|
659
|
+
|
|
660
|
+
have_works = works_df is not None
|
|
661
|
+
have_graph = graph_df is not None
|
|
662
|
+
if have_works and have_graph:
|
|
663
|
+
stages.append(
|
|
664
|
+
{
|
|
665
|
+
"label": "dedup",
|
|
666
|
+
"status": "done",
|
|
667
|
+
"detail": f"{len(works_df)} works / {len(graph_df)} edges",
|
|
668
|
+
"hint": "",
|
|
669
|
+
}
|
|
670
|
+
)
|
|
671
|
+
elif layout.works_csv.exists() or layout.graph_csv.exists():
|
|
672
|
+
stages.append(
|
|
673
|
+
{
|
|
674
|
+
"label": "dedup",
|
|
675
|
+
"status": "partial",
|
|
676
|
+
"detail": "artifacts incomplete",
|
|
677
|
+
"hint": f"citegraph dedup --out {out}",
|
|
678
|
+
}
|
|
679
|
+
)
|
|
680
|
+
else:
|
|
681
|
+
stages.append(
|
|
682
|
+
{
|
|
683
|
+
"label": "dedup",
|
|
684
|
+
"status": "not_run",
|
|
685
|
+
"detail": "not run",
|
|
686
|
+
"hint": f"citegraph dedup --out {out}",
|
|
687
|
+
}
|
|
688
|
+
)
|
|
689
|
+
|
|
690
|
+
stages.append(
|
|
691
|
+
{
|
|
692
|
+
"label": "enrich",
|
|
693
|
+
"status": "done" if enriched_df is not None else "optional_not_run",
|
|
694
|
+
"detail": f"{len(enriched_df)} works"
|
|
695
|
+
if enriched_df is not None
|
|
696
|
+
else "optional, not run",
|
|
697
|
+
"hint": "" if enriched_df is not None else f"citegraph enrich --out {out}",
|
|
698
|
+
}
|
|
699
|
+
)
|
|
700
|
+
stages.append(
|
|
701
|
+
{
|
|
702
|
+
"label": "authors",
|
|
703
|
+
"status": "done" if authors_df is not None else "optional_not_run",
|
|
704
|
+
"detail": f"{len(authors_df)} canonical authors"
|
|
705
|
+
if authors_df is not None
|
|
706
|
+
else "optional, not run",
|
|
707
|
+
"hint": "" if authors_df is not None else f"citegraph authors --out {out}",
|
|
708
|
+
}
|
|
709
|
+
)
|
|
710
|
+
return stages
|
|
711
|
+
|
|
712
|
+
|
|
713
|
+
def _collect_resolution_audit(
|
|
714
|
+
path: Path, errors: list[dict], *, inputs: set[str], outputs: set[str],
|
|
715
|
+
) -> dict:
|
|
716
|
+
"""Read saved evidence only when every bound checkpoint still matches."""
|
|
717
|
+
result = {"audit_status": "unavailable", "audit_config": None, "audit_decisions": []}
|
|
718
|
+
if not path.exists():
|
|
719
|
+
return result
|
|
720
|
+
try:
|
|
721
|
+
saved = read_json(path)
|
|
722
|
+
if not isinstance(saved, dict):
|
|
723
|
+
raise ValueError("Audit must be a JSON object")
|
|
724
|
+
if saved.get("schema_version") != 2:
|
|
725
|
+
return result
|
|
726
|
+
for key, expected in (("input_fingerprints", inputs), ("output_fingerprints", outputs)):
|
|
727
|
+
values = saved.get(key)
|
|
728
|
+
if not isinstance(values, dict) or set(values) != expected:
|
|
729
|
+
raise ValueError(f"Audit has invalid {key}")
|
|
730
|
+
if any(value is not None and (not isinstance(value, str) or len(value) != 64)
|
|
731
|
+
for value in values.values()):
|
|
732
|
+
raise ValueError(f"Audit has invalid digest in {key}")
|
|
733
|
+
if not isinstance(saved.get("config"), dict) or not isinstance(saved.get("algorithm_version"), str):
|
|
734
|
+
raise ValueError("Audit is missing its algorithm/configuration")
|
|
735
|
+
decisions = saved.get("decisions")
|
|
736
|
+
if not isinstance(decisions, list) or any(not isinstance(row, dict) for row in decisions):
|
|
737
|
+
raise ValueError("Audit decisions must be a list of objects")
|
|
738
|
+
for key in ("input_fingerprints", "output_fingerprints"):
|
|
739
|
+
for name, expected in saved[key].items():
|
|
740
|
+
if artifact_fingerprint(path.parent / name) != expected:
|
|
741
|
+
result["audit_status"] = "stale"
|
|
742
|
+
return result
|
|
743
|
+
result.update(
|
|
744
|
+
audit_status="current", audit_config=saved["config"],
|
|
745
|
+
audit_algorithm=saved["algorithm_version"], audit_decisions=decisions[:200],
|
|
746
|
+
audit_decisions_truncated=max(len(decisions) - 200, 0),
|
|
747
|
+
enrichment_status=saved.get("enrichment_status"), saved_audit=saved,
|
|
748
|
+
)
|
|
749
|
+
except Exception as exc: # noqa: BLE001 - corrupt artifacts are report findings
|
|
750
|
+
result["audit_status"] = "invalid"
|
|
751
|
+
_record_error(errors, path, exc)
|
|
752
|
+
return result
|
|
753
|
+
|
|
754
|
+
|
|
755
|
+
def _collect_dedup_panel(
|
|
756
|
+
raw_refs_df: pd.DataFrame | None,
|
|
757
|
+
works_df: pd.DataFrame | None,
|
|
758
|
+
graph_df: pd.DataFrame | None,
|
|
759
|
+
errors: list[dict],
|
|
760
|
+
layout: OutLayout,
|
|
761
|
+
*,
|
|
762
|
+
sources_df: pd.DataFrame | None = None,
|
|
763
|
+
) -> dict | None:
|
|
764
|
+
if raw_refs_df is None or works_df is None:
|
|
765
|
+
return None
|
|
766
|
+
|
|
767
|
+
panel: dict = {
|
|
768
|
+
"n_raw": int(len(raw_refs_df)),
|
|
769
|
+
"n_canonical": int(len(works_df)),
|
|
770
|
+
"n_edges": int(len(graph_df)) if graph_df is not None else None,
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
# Cheap corpus-level stats over the raw citation events.
|
|
774
|
+
if "Year" in raw_refs_df.columns:
|
|
775
|
+
years = pd.to_numeric(raw_refs_df["Year"], errors="coerce")
|
|
776
|
+
panel["n_missing_year"] = int((years.isna() | (years <= 0)).sum())
|
|
777
|
+
else:
|
|
778
|
+
panel["n_missing_year"] = 0
|
|
779
|
+
if "Authors" in raw_refs_df.columns:
|
|
780
|
+
authors_col = raw_refs_df["Authors"]
|
|
781
|
+
elif "Authors_List" in raw_refs_df.columns:
|
|
782
|
+
authors_col = raw_refs_df["Authors_List"]
|
|
783
|
+
else:
|
|
784
|
+
authors_col = None
|
|
785
|
+
panel["n_empty_authors"] = (
|
|
786
|
+
int(authors_col.fillna("").astype(str).str.strip().isin(["", "[]"]).sum())
|
|
787
|
+
if authors_col is not None
|
|
788
|
+
else 0
|
|
789
|
+
)
|
|
790
|
+
if "Title" in raw_refs_df.columns:
|
|
791
|
+
titles = raw_refs_df["Title"]
|
|
792
|
+
panel["n_short_titles"] = int(
|
|
793
|
+
(titles.fillna("").astype(str).str.strip().str.len() < _SHORT_TITLE_CHARS).sum()
|
|
794
|
+
)
|
|
795
|
+
else:
|
|
796
|
+
panel["n_short_titles"] = 0
|
|
797
|
+
|
|
798
|
+
# Top-10 most-cited canonical works (core works rank here too).
|
|
799
|
+
top_cited: list[dict] = []
|
|
800
|
+
if graph_df is not None and "cited_id" in graph_df.columns:
|
|
801
|
+
counts = graph_df.groupby("cited_id").size().sort_values(ascending=False).head(10)
|
|
802
|
+
for cited_id, n in counts.items():
|
|
803
|
+
title = year = None
|
|
804
|
+
if cited_id in works_df.index:
|
|
805
|
+
work_row = works_df.loc[cited_id]
|
|
806
|
+
if isinstance(work_row, pd.DataFrame): # duplicate ids — take the first
|
|
807
|
+
work_row = work_row.iloc[0]
|
|
808
|
+
title = _as_str(work_row.get("Title"))
|
|
809
|
+
year = _as_int(work_row.get("Year"))
|
|
810
|
+
top_cited.append(
|
|
811
|
+
{"id": _as_str(cited_id), "title": title, "year": year, "count": int(n)}
|
|
812
|
+
)
|
|
813
|
+
panel["top_cited"] = top_cited
|
|
814
|
+
|
|
815
|
+
# Historical membership comes solely from the stage's bound observations.
|
|
816
|
+
panel["merge_audit"] = []
|
|
817
|
+
panel["merge_audit_truncated"] = 0
|
|
818
|
+
panel.update(_collect_resolution_audit(
|
|
819
|
+
layout.canonicalization_audit_json, errors,
|
|
820
|
+
inputs=WORK_AUDIT_INPUTS, outputs=WORK_AUDIT_OUTPUTS,
|
|
821
|
+
))
|
|
822
|
+
if panel["audit_status"] != "current":
|
|
823
|
+
return panel
|
|
824
|
+
try:
|
|
825
|
+
saved_audit = panel.pop("saved_audit")
|
|
826
|
+
source_mapping = saved_audit.get("source_cluster_ids")
|
|
827
|
+
citation_mapping = saved_audit.get("citation_cluster_ids")
|
|
828
|
+
if (not isinstance(source_mapping, list) or sources_df is None
|
|
829
|
+
or len(source_mapping) != len(sources_df)
|
|
830
|
+
or not isinstance(citation_mapping, list) or len(citation_mapping) != len(raw_refs_df)):
|
|
831
|
+
raise ValueError("Audit observation mappings do not match the checkpoint rows")
|
|
832
|
+
mapping = source_mapping + citation_mapping
|
|
833
|
+
if any(not isinstance(cid, str) or cid not in works_df.index for cid in mapping):
|
|
834
|
+
raise ValueError("Audit refers to an unknown canonical work")
|
|
835
|
+
raw = pd.concat([sources_df, raw_refs_df], ignore_index=True)
|
|
836
|
+
for decision in saved_audit["decisions"]:
|
|
837
|
+
for key in ("left_index", "right_index"):
|
|
838
|
+
idx = decision.get(key)
|
|
839
|
+
if not isinstance(idx, int) or not 0 <= idx < len(raw):
|
|
840
|
+
raise ValueError("Audit decision refers to an unknown observation")
|
|
841
|
+
panel["audit_decisions"] = [
|
|
842
|
+
dict(decision, left_title=_as_str(raw.iloc[decision["left_index"]].get("Title")),
|
|
843
|
+
right_title=_as_str(raw.iloc[decision["right_index"]].get("Title")))
|
|
844
|
+
for decision in saved_audit["decisions"] if decision.get("decision") == "review"
|
|
845
|
+
][:200]
|
|
846
|
+
panel["id_collisions"] = saved_audit.get("id_collisions", [])
|
|
847
|
+
members_by_cluster: dict[str, list[int]] = {}
|
|
848
|
+
for idx, cluster_id in enumerate(mapping):
|
|
849
|
+
members_by_cluster.setdefault(str(cluster_id), []).append(int(idx))
|
|
850
|
+
multi = sorted(
|
|
851
|
+
((cid, idxs) for cid, idxs in members_by_cluster.items() if len(idxs) > 1),
|
|
852
|
+
key=lambda item: (-len(item[1]), item[0]),
|
|
853
|
+
)
|
|
854
|
+
panel["merge_audit_truncated"] = max(len(multi) - _MERGE_AUDIT_MAX_CLUSTERS, 0)
|
|
855
|
+
for cluster_id, idxs in multi[:_MERGE_AUDIT_MAX_CLUSTERS]:
|
|
856
|
+
members = []
|
|
857
|
+
for idx in idxs:
|
|
858
|
+
row = raw.iloc[idx]
|
|
859
|
+
members.append(
|
|
860
|
+
{
|
|
861
|
+
"title": _as_str(row.get("Title")),
|
|
862
|
+
"year": _as_int(row.get("Year")),
|
|
863
|
+
"citing_id": _as_str(row.get("citing_id")),
|
|
864
|
+
"source_file": _as_str(row.get("source_file")),
|
|
865
|
+
}
|
|
866
|
+
)
|
|
867
|
+
panel["merge_audit"].append({"id": cluster_id, "members": members})
|
|
868
|
+
except Exception as exc: # noqa: BLE001 - audit is best-effort
|
|
869
|
+
panel.update(audit_status="invalid", merge_audit=[], audit_decisions=[])
|
|
870
|
+
_record_error(errors, layout.canonicalization_audit_json, exc)
|
|
871
|
+
|
|
872
|
+
return panel
|
|
873
|
+
|
|
874
|
+
|
|
875
|
+
def _collect_enrichment_panel(
|
|
876
|
+
enriched_df: pd.DataFrame | None,
|
|
877
|
+
misses_df: pd.DataFrame | None,
|
|
878
|
+
enrichment_summary: object,
|
|
879
|
+
) -> dict | None:
|
|
880
|
+
if enriched_df is None:
|
|
881
|
+
return None
|
|
882
|
+
|
|
883
|
+
status = (
|
|
884
|
+
enriched_df["enrichment_status"].fillna("")
|
|
885
|
+
if "enrichment_status" in enriched_df.columns
|
|
886
|
+
else pd.Series([""] * len(enriched_df))
|
|
887
|
+
)
|
|
888
|
+
|
|
889
|
+
if isinstance(enrichment_summary, dict):
|
|
890
|
+
n_total = _as_int(enrichment_summary.get("n_references")) or len(enriched_df)
|
|
891
|
+
n_matched = _as_int(enrichment_summary.get("n_matched")) or 0
|
|
892
|
+
match_rate = enrichment_summary.get("match_rate")
|
|
893
|
+
sources = enrichment_summary.get("sources") or {}
|
|
894
|
+
else:
|
|
895
|
+
n_total = len(enriched_df)
|
|
896
|
+
n_matched = int((status == "matched").sum())
|
|
897
|
+
match_rate = (n_matched / n_total) if n_total else None
|
|
898
|
+
sources = (
|
|
899
|
+
enriched_df["enrichment_source"].fillna("miss").value_counts().to_dict()
|
|
900
|
+
if "enrichment_source" in enriched_df.columns
|
|
901
|
+
else {}
|
|
902
|
+
)
|
|
903
|
+
|
|
904
|
+
panel: dict = {
|
|
905
|
+
"n_references": int(n_total),
|
|
906
|
+
"n_matched": int(n_matched),
|
|
907
|
+
"match_rate": float(match_rate) if match_rate is not None else None,
|
|
908
|
+
"sources": {str(k): int(v) for k, v in dict(sources).items()},
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
if "enrichment_miss_reason" in enriched_df.columns:
|
|
912
|
+
reasons = enriched_df.loc[status != "matched", "enrichment_miss_reason"]
|
|
913
|
+
panel["miss_reasons"] = {
|
|
914
|
+
str(k): int(v) for k, v in reasons.fillna("(none)").value_counts().items()
|
|
915
|
+
}
|
|
916
|
+
else:
|
|
917
|
+
panel["miss_reasons"] = {}
|
|
918
|
+
|
|
919
|
+
weakest: list[dict] = []
|
|
920
|
+
if "enrichment_title_score" in enriched_df.columns:
|
|
921
|
+
matched = enriched_df[status == "matched"].copy()
|
|
922
|
+
matched["_score"] = pd.to_numeric(matched["enrichment_title_score"], errors="coerce")
|
|
923
|
+
for _, row in matched.sort_values("_score", ascending=True).head(10).iterrows():
|
|
924
|
+
weakest.append(
|
|
925
|
+
{
|
|
926
|
+
"id": _as_str(row.name if matched.index.name else row.get("id")),
|
|
927
|
+
"title": _as_str(row.get("Title")),
|
|
928
|
+
"candidate_title": _as_str(row.get("enrichment_candidate_title")),
|
|
929
|
+
"title_score": row.get("_score"),
|
|
930
|
+
"adjusted_score": row.get("enrichment_adjusted_score"),
|
|
931
|
+
"doi": _as_str(row.get("doi")),
|
|
932
|
+
"year_match": _as_str(row.get("enrichment_year_match")),
|
|
933
|
+
}
|
|
934
|
+
)
|
|
935
|
+
panel["weakest_matches"] = weakest
|
|
936
|
+
|
|
937
|
+
misses_preview: list[dict] = []
|
|
938
|
+
if misses_df is not None:
|
|
939
|
+
for _, row in misses_df.head(15).iterrows():
|
|
940
|
+
misses_preview.append(
|
|
941
|
+
{
|
|
942
|
+
"id": _as_str(row.get("id")),
|
|
943
|
+
"title": _as_str(row.get("Title")),
|
|
944
|
+
"year": _as_int(row.get("Year")),
|
|
945
|
+
"reason": _as_str(row.get("enrichment_miss_reason")),
|
|
946
|
+
"score": row.get("enrichment_title_score"),
|
|
947
|
+
}
|
|
948
|
+
)
|
|
949
|
+
panel["misses_preview"] = misses_preview
|
|
950
|
+
panel["n_misses"] = int(len(misses_df)) if misses_df is not None else None
|
|
951
|
+
return panel
|
|
952
|
+
|
|
953
|
+
|
|
954
|
+
def _collect_authors_panel(
|
|
955
|
+
authors_df: pd.DataFrame | None,
|
|
956
|
+
author_citations_df: pd.DataFrame | None,
|
|
957
|
+
author_review: object,
|
|
958
|
+
) -> dict | None:
|
|
959
|
+
if authors_df is None:
|
|
960
|
+
return None
|
|
961
|
+
|
|
962
|
+
panel: dict = {
|
|
963
|
+
"n_authors": int(len(authors_df)),
|
|
964
|
+
"n_occurrences": int(len(author_citations_df))
|
|
965
|
+
if author_citations_df is not None
|
|
966
|
+
else None,
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
top: list[dict] = []
|
|
970
|
+
if "n_citations_received" in authors_df.columns:
|
|
971
|
+
ranked = authors_df.sort_values("n_citations_received", ascending=False).head(10)
|
|
972
|
+
for author_id, row in ranked.iterrows():
|
|
973
|
+
top.append(
|
|
974
|
+
{
|
|
975
|
+
"id": _as_str(author_id),
|
|
976
|
+
"display_name": _as_str(row.get("display_name")),
|
|
977
|
+
"n_citations_received": _as_int(row.get("n_citations_received")),
|
|
978
|
+
"n_works": _as_int(row.get("n_works")),
|
|
979
|
+
"n_core_works": _as_int(row.get("n_core_works")),
|
|
980
|
+
}
|
|
981
|
+
)
|
|
982
|
+
panel["top_cited"] = top
|
|
983
|
+
|
|
984
|
+
review_entries: list[dict] = []
|
|
985
|
+
if isinstance(author_review, list):
|
|
986
|
+
for entry in author_review:
|
|
987
|
+
if isinstance(entry, dict):
|
|
988
|
+
review_entries.append({str(k): _as_str(v) for k, v in entry.items()})
|
|
989
|
+
panel["review"] = review_entries
|
|
990
|
+
|
|
991
|
+
variant_audit: list[dict] = []
|
|
992
|
+
if (
|
|
993
|
+
author_citations_df is not None
|
|
994
|
+
and "author_id" in author_citations_df.columns
|
|
995
|
+
and "raw_author" in author_citations_df.columns
|
|
996
|
+
):
|
|
997
|
+
grouped = author_citations_df.groupby("author_id")["raw_author"].agg(
|
|
998
|
+
lambda s: sorted({str(v) for v in s.dropna()})
|
|
999
|
+
)
|
|
1000
|
+
ranked_ids = grouped.map(len).sort_values(ascending=False).head(10).index
|
|
1001
|
+
for author_id in ranked_ids:
|
|
1002
|
+
forms = grouped.loc[author_id]
|
|
1003
|
+
display = ""
|
|
1004
|
+
if author_id in authors_df.index:
|
|
1005
|
+
display = _as_str(authors_df.loc[author_id].get("display_name"))
|
|
1006
|
+
variant_audit.append(
|
|
1007
|
+
{
|
|
1008
|
+
"id": _as_str(author_id),
|
|
1009
|
+
"display_name": display,
|
|
1010
|
+
"n_forms": len(forms),
|
|
1011
|
+
"forms": forms,
|
|
1012
|
+
}
|
|
1013
|
+
)
|
|
1014
|
+
panel["variant_audit"] = variant_audit
|
|
1015
|
+
return panel
|
|
1016
|
+
|
|
1017
|
+
|
|
1018
|
+
def _collect_integrity(
|
|
1019
|
+
layout: OutLayout,
|
|
1020
|
+
errors: list[dict],
|
|
1021
|
+
*,
|
|
1022
|
+
rows: list[tuple[str, Path, bool, str]],
|
|
1023
|
+
) -> list[dict]:
|
|
1024
|
+
error_by_path = {e["path"]: e["error"] for e in errors}
|
|
1025
|
+
table: list[dict] = []
|
|
1026
|
+
for name, path, parsed_ok, note in rows:
|
|
1027
|
+
if not path.exists():
|
|
1028
|
+
status = "missing"
|
|
1029
|
+
elif parsed_ok:
|
|
1030
|
+
status = "ok"
|
|
1031
|
+
else:
|
|
1032
|
+
status = "corrupt"
|
|
1033
|
+
table.append(
|
|
1034
|
+
{
|
|
1035
|
+
"name": name,
|
|
1036
|
+
"path": str(path),
|
|
1037
|
+
"status": status,
|
|
1038
|
+
"note": note,
|
|
1039
|
+
"error": error_by_path.get(str(path), ""),
|
|
1040
|
+
}
|
|
1041
|
+
)
|
|
1042
|
+
listed = {row["path"] for row in table}
|
|
1043
|
+
for err in errors: # per-paper caches and anything else that failed to read
|
|
1044
|
+
if err["path"] in listed:
|
|
1045
|
+
continue
|
|
1046
|
+
try:
|
|
1047
|
+
name = str(Path(err["path"]).relative_to(layout.out_dir))
|
|
1048
|
+
except ValueError:
|
|
1049
|
+
name = err["path"]
|
|
1050
|
+
table.append(
|
|
1051
|
+
{
|
|
1052
|
+
"name": name,
|
|
1053
|
+
"path": err["path"],
|
|
1054
|
+
"status": "corrupt",
|
|
1055
|
+
"note": "per-file cache",
|
|
1056
|
+
"error": err["error"],
|
|
1057
|
+
}
|
|
1058
|
+
)
|
|
1059
|
+
listed.add(err["path"])
|
|
1060
|
+
return table
|
|
1061
|
+
|
|
1062
|
+
|
|
1063
|
+
# ---------------------------------------------------------------------------
|
|
1064
|
+
# HTML rendering
|
|
1065
|
+
# ---------------------------------------------------------------------------
|
|
1066
|
+
def _esc(value: object) -> str:
|
|
1067
|
+
if value is None:
|
|
1068
|
+
return ""
|
|
1069
|
+
return html.escape(str(value), quote=True)
|
|
1070
|
+
|
|
1071
|
+
|
|
1072
|
+
def _year_str(year: object) -> str:
|
|
1073
|
+
y = _as_int(year)
|
|
1074
|
+
return "?" if not y else str(y)
|
|
1075
|
+
|
|
1076
|
+
|
|
1077
|
+
def _num(value: object, digits: int = 1) -> str:
|
|
1078
|
+
if value is None or (isinstance(value, float) and math.isnan(value)):
|
|
1079
|
+
return "–"
|
|
1080
|
+
if isinstance(value, float):
|
|
1081
|
+
return f"{value:.{digits}f}"
|
|
1082
|
+
return str(value)
|
|
1083
|
+
|
|
1084
|
+
|
|
1085
|
+
_PILL_KIND = {
|
|
1086
|
+
"ok": "ok",
|
|
1087
|
+
"done": "ok",
|
|
1088
|
+
"zero": "warn",
|
|
1089
|
+
"failed": "warn",
|
|
1090
|
+
"corrupt": "warn",
|
|
1091
|
+
"error": "warn",
|
|
1092
|
+
"image-only": "warn",
|
|
1093
|
+
"missing": "warn",
|
|
1094
|
+
"partial": "warn",
|
|
1095
|
+
"pending": "muted",
|
|
1096
|
+
"skipped": "muted",
|
|
1097
|
+
"not_run": "muted",
|
|
1098
|
+
"optional_not_run": "muted",
|
|
1099
|
+
}
|
|
1100
|
+
|
|
1101
|
+
|
|
1102
|
+
def _pill(label: str, status: str | None = None) -> str:
|
|
1103
|
+
kind = _PILL_KIND.get(status if status is not None else label, "muted")
|
|
1104
|
+
return f'<span class="pill pill-{kind}">{_esc(label)}</span>'
|
|
1105
|
+
|
|
1106
|
+
|
|
1107
|
+
def _placeholder(stage_label: str, cmd: str) -> str:
|
|
1108
|
+
return (
|
|
1109
|
+
'<p class="placeholder">'
|
|
1110
|
+
f"{_esc(stage_label)} not run yet — run <code>{_esc(cmd)}</code>.</p>"
|
|
1111
|
+
)
|
|
1112
|
+
|
|
1113
|
+
|
|
1114
|
+
def _section(anchor: str, eyebrow: str, title: str, body: str) -> str:
|
|
1115
|
+
return (
|
|
1116
|
+
f'<section id="{_esc(anchor)}">'
|
|
1117
|
+
f'<p class="eyebrow">{_esc(eyebrow)}</p>'
|
|
1118
|
+
f"<h2>{_esc(title)}</h2>{body}</section>"
|
|
1119
|
+
)
|
|
1120
|
+
|
|
1121
|
+
|
|
1122
|
+
def _render_stage_strip(stages: list[dict]) -> str:
|
|
1123
|
+
cells = []
|
|
1124
|
+
for stage in stages:
|
|
1125
|
+
status = stage["status"]
|
|
1126
|
+
mark = {"done": "✓ done", "partial": "partial", "not_run": "not run"}.get(
|
|
1127
|
+
status, "optional"
|
|
1128
|
+
)
|
|
1129
|
+
hint = (
|
|
1130
|
+
f'<div class="stage-hint"><code>{_esc(stage["hint"])}</code></div>'
|
|
1131
|
+
if stage.get("hint")
|
|
1132
|
+
else ""
|
|
1133
|
+
)
|
|
1134
|
+
cells.append(
|
|
1135
|
+
f'<div class="stage stage-{_esc(status)}">'
|
|
1136
|
+
f'<div class="stage-name">{_esc(stage["label"])}</div>'
|
|
1137
|
+
f"<div>{_pill(mark, status)}</div>"
|
|
1138
|
+
f'<div class="stage-detail">{_esc(stage["detail"])}</div>{hint}</div>'
|
|
1139
|
+
)
|
|
1140
|
+
return f'<div class="stage-strip">{"".join(cells)}</div>'
|
|
1141
|
+
|
|
1142
|
+
|
|
1143
|
+
def _render_problem_cards(problems: dict) -> str:
|
|
1144
|
+
cards = problems.get("cards", [])
|
|
1145
|
+
if not cards:
|
|
1146
|
+
return (
|
|
1147
|
+
'<p class="all-clear">No problems detected at the stages run so far.</p>'
|
|
1148
|
+
)
|
|
1149
|
+
parts = []
|
|
1150
|
+
for card in cards:
|
|
1151
|
+
parts.append(
|
|
1152
|
+
f'<a class="problem-card" href="#{_esc(card["anchor"])}">'
|
|
1153
|
+
f'<span class="problem-count">{card["count"]}</span>'
|
|
1154
|
+
f'<span class="problem-label">{_esc(card["label"])}</span></a>'
|
|
1155
|
+
)
|
|
1156
|
+
return f'<div class="problem-cards">{"".join(parts)}</div>'
|
|
1157
|
+
|
|
1158
|
+
|
|
1159
|
+
def _render_paper_details(paper: dict) -> str:
|
|
1160
|
+
meta = paper["metadata"]
|
|
1161
|
+
blocks = []
|
|
1162
|
+
if meta["status"] == "ok":
|
|
1163
|
+
blocks.append(
|
|
1164
|
+
'<dl class="kv">'
|
|
1165
|
+
f"<dt>Title</dt><dd>{_esc(meta['title'])}</dd>"
|
|
1166
|
+
f"<dt>Authors</dt><dd>{_esc(meta['authors'])}</dd>"
|
|
1167
|
+
f"<dt>Journal</dt><dd>{_esc(meta['journal'])}</dd>"
|
|
1168
|
+
f"<dt>Year</dt><dd>{_esc(_year_str(meta['year']))}</dd></dl>"
|
|
1169
|
+
)
|
|
1170
|
+
refs = paper["refs_preview"]
|
|
1171
|
+
if refs:
|
|
1172
|
+
items = "".join(
|
|
1173
|
+
f"<li>{_esc(r['title'])} <span class='soft'>— {_esc(r['authors'])} "
|
|
1174
|
+
f"({_esc(_year_str(r['year']))})</span></li>"
|
|
1175
|
+
for r in refs
|
|
1176
|
+
)
|
|
1177
|
+
more = (
|
|
1178
|
+
f'<li class="soft">…and {paper["refs_more"]} more</li>'
|
|
1179
|
+
if paper["refs_more"]
|
|
1180
|
+
else ""
|
|
1181
|
+
)
|
|
1182
|
+
blocks.append(
|
|
1183
|
+
f'<h4>Extracted references (first {len(refs)})</h4><ol class="ref-list">{items}{more}</ol>'
|
|
1184
|
+
)
|
|
1185
|
+
if paper["markdown_preview"]:
|
|
1186
|
+
blocks.append(
|
|
1187
|
+
"<h4>Markdown preview</h4>"
|
|
1188
|
+
f'<pre class="md-preview">{_esc(paper["markdown_preview"])}</pre>'
|
|
1189
|
+
)
|
|
1190
|
+
if not blocks:
|
|
1191
|
+
blocks.append('<p class="soft">Nothing cached for this paper yet.</p>')
|
|
1192
|
+
return "".join(blocks)
|
|
1193
|
+
|
|
1194
|
+
|
|
1195
|
+
def _render_papers_section(papers: list[dict]) -> str:
|
|
1196
|
+
if not papers:
|
|
1197
|
+
return _placeholder("Conversion", "citegraph convert <pdf_dir> --out <out>")
|
|
1198
|
+
|
|
1199
|
+
rows = []
|
|
1200
|
+
for paper in papers:
|
|
1201
|
+
md = paper["markdown"]
|
|
1202
|
+
meta = paper["metadata"]
|
|
1203
|
+
refs = paper["references"]
|
|
1204
|
+
md_cell = _pill(md["status"], md["status"])
|
|
1205
|
+
if md["chars"] is not None:
|
|
1206
|
+
md_cell = f'<span class="mono">{md["chars"]:,}</span> {md_cell}'
|
|
1207
|
+
meta_cell = _pill(meta["status"], meta["status"])
|
|
1208
|
+
if meta["status"] == "ok":
|
|
1209
|
+
meta_cell += (
|
|
1210
|
+
f' <span class="cell-title">{_esc(meta["title"])}'
|
|
1211
|
+
f' <span class="soft">({_esc(_year_str(meta["year"]))})</span></span>'
|
|
1212
|
+
)
|
|
1213
|
+
refs_cell = _pill(refs["status"], refs["status"])
|
|
1214
|
+
if refs["count"] is not None:
|
|
1215
|
+
refs_cell = f'<span class="mono">{refs["count"]}</span> {refs_cell}'
|
|
1216
|
+
notes = "; ".join(paper["notes"])
|
|
1217
|
+
rows.append(
|
|
1218
|
+
f'<details class="paper-row" data-stem="{_esc(paper["stem"])}" '
|
|
1219
|
+
f'data-title="{_esc(meta["title"])}" '
|
|
1220
|
+
f'data-problem="{"1" if paper["problem"] else "0"}">'
|
|
1221
|
+
f'<summary><span class="col col-stem mono">{_esc(paper["stem"])}</span>'
|
|
1222
|
+
f'<span class="col col-md">{md_cell}</span>'
|
|
1223
|
+
f'<span class="col col-meta">{meta_cell}</span>'
|
|
1224
|
+
f'<span class="col col-refs">{refs_cell}</span>'
|
|
1225
|
+
f'<span class="col col-notes">{_esc(notes)}</span></summary>'
|
|
1226
|
+
f'<div class="paper-body">{_render_paper_details(paper)}</div></details>'
|
|
1227
|
+
)
|
|
1228
|
+
|
|
1229
|
+
header = (
|
|
1230
|
+
'<div class="paper-row paper-head">'
|
|
1231
|
+
'<span class="col col-stem">stem</span>'
|
|
1232
|
+
'<span class="col col-md">conversion</span>'
|
|
1233
|
+
'<span class="col col-meta">metadata</span>'
|
|
1234
|
+
'<span class="col col-refs">references</span>'
|
|
1235
|
+
'<span class="col col-notes">notes</span></div>'
|
|
1236
|
+
)
|
|
1237
|
+
controls = (
|
|
1238
|
+
'<div class="controls">'
|
|
1239
|
+
'<input id="filter-text" type="search" placeholder="filter by stem or title…">'
|
|
1240
|
+
'<label><input id="filter-problems" type="checkbox"> problems only</label>'
|
|
1241
|
+
'<span id="paper-count" class="soft"></span></div>'
|
|
1242
|
+
)
|
|
1243
|
+
return (
|
|
1244
|
+
controls
|
|
1245
|
+
+ f'<div class="table-scroll"><div class="paper-table">{header}{"".join(rows)}</div></div>'
|
|
1246
|
+
)
|
|
1247
|
+
|
|
1248
|
+
|
|
1249
|
+
def _render_dedup_section(panel: dict | None) -> str:
|
|
1250
|
+
if panel is None:
|
|
1251
|
+
return _placeholder("Dedup", "citegraph dedup --out <out>")
|
|
1252
|
+
|
|
1253
|
+
ring_counts = panel.get("ring_counts") or {}
|
|
1254
|
+
# The payload is JSON-normalized, so ring keys arrive as strings.
|
|
1255
|
+
n_core = int(ring_counts.get("0", ring_counts.get(0, 0)))
|
|
1256
|
+
stats = (
|
|
1257
|
+
'<div class="stat-row">'
|
|
1258
|
+
f'<div class="stat"><span class="stat-n">{panel["n_raw"]}</span>raw citation events</div>'
|
|
1259
|
+
f'<div class="stat"><span class="stat-n">{panel["n_canonical"]}</span>canonical works</div>'
|
|
1260
|
+
f'<div class="stat"><span class="stat-n">{n_core}</span>core works (ring 0)</div>'
|
|
1261
|
+
f'<div class="stat"><span class="stat-n">{_num(panel["n_edges"], 0)}</span>graph edges</div>'
|
|
1262
|
+
f'<div class="stat"><span class="stat-n">{panel["n_missing_year"]}</span>missing year (Year=0)</div>'
|
|
1263
|
+
f'<div class="stat"><span class="stat-n">{panel["n_empty_authors"]}</span>empty authors</div>'
|
|
1264
|
+
f'<div class="stat"><span class="stat-n">{panel["n_short_titles"]}</span>suspicious short titles</div>'
|
|
1265
|
+
"</div>"
|
|
1266
|
+
)
|
|
1267
|
+
|
|
1268
|
+
core_to_core = panel.get("core_to_core") or []
|
|
1269
|
+
core_rows = "".join(
|
|
1270
|
+
f"<tr><td class='mono'>{_esc(e['citing_id'])}</td>"
|
|
1271
|
+
f"<td class='mono'>{_esc(e['cited_id'])}</td></tr>"
|
|
1272
|
+
for e in core_to_core
|
|
1273
|
+
)
|
|
1274
|
+
core_html = (
|
|
1275
|
+
"<h3>Core cites core</h3>"
|
|
1276
|
+
'<p class="soft">Citations where both ends are ring-0 works — your own '
|
|
1277
|
+
"papers citing each other.</p>"
|
|
1278
|
+
'<div class="table-scroll"><table><thead><tr><th>citing</th><th>cited</th>'
|
|
1279
|
+
f"</tr></thead><tbody>{core_rows}</tbody></table></div>"
|
|
1280
|
+
if core_to_core
|
|
1281
|
+
else ""
|
|
1282
|
+
)
|
|
1283
|
+
|
|
1284
|
+
top_rows = "".join(
|
|
1285
|
+
f"<tr><td class='mono'>{_esc(t['id'])}</td><td>{_esc(t['title'])}</td>"
|
|
1286
|
+
f"<td class='num'>{_esc(_year_str(t['year']))}</td><td class='num'>{t['count']}</td></tr>"
|
|
1287
|
+
for t in panel["top_cited"]
|
|
1288
|
+
)
|
|
1289
|
+
top_html = (
|
|
1290
|
+
"<h3>Top cited works</h3>"
|
|
1291
|
+
'<div class="table-scroll"><table><thead><tr><th>id</th><th>title</th>'
|
|
1292
|
+
"<th>year</th><th>citations</th></tr></thead>"
|
|
1293
|
+
f"<tbody>{top_rows}</tbody></table></div>"
|
|
1294
|
+
if panel["top_cited"]
|
|
1295
|
+
else ""
|
|
1296
|
+
)
|
|
1297
|
+
|
|
1298
|
+
audit_parts = []
|
|
1299
|
+
for cluster in panel["merge_audit"]:
|
|
1300
|
+
members = "".join(
|
|
1301
|
+
f"<li>{_esc(m['title'])} <span class='soft'>({_esc(_year_str(m['year']))}, "
|
|
1302
|
+
f"<span class='mono'>{_esc(m['source_file'] or m['citing_id'])}</span>)</span></li>"
|
|
1303
|
+
for m in cluster["members"]
|
|
1304
|
+
)
|
|
1305
|
+
audit_parts.append(
|
|
1306
|
+
f'<details class="cluster"><summary><span class="mono">{_esc(cluster["id"])}</span>'
|
|
1307
|
+
f' <span class="soft">contains {len(cluster["members"])} observations</span>'
|
|
1308
|
+
f"</summary><ul>{members}</ul></details>"
|
|
1309
|
+
)
|
|
1310
|
+
if audit_parts:
|
|
1311
|
+
note = ""
|
|
1312
|
+
if panel["merge_audit_truncated"]:
|
|
1313
|
+
note = (
|
|
1314
|
+
f'<p class="soft">…and {panel["merge_audit_truncated"]} more clusters '
|
|
1315
|
+
"not shown.</p>"
|
|
1316
|
+
)
|
|
1317
|
+
audit_html = (
|
|
1318
|
+
"<h3>Merge audit</h3>"
|
|
1319
|
+
'<p class="soft">Saved clusters containing multiple source or citation observations. '
|
|
1320
|
+
"Inspect these for false merges.</p>"
|
|
1321
|
+
+ "".join(audit_parts)
|
|
1322
|
+
+ note
|
|
1323
|
+
)
|
|
1324
|
+
else:
|
|
1325
|
+
audit_html = (
|
|
1326
|
+
"<h3>Merge audit</h3><p class='soft'>No saved merged groups to display.</p>"
|
|
1327
|
+
)
|
|
1328
|
+
|
|
1329
|
+
return stats + core_html + top_html + audit_html + _render_resolution_evidence(panel)
|
|
1330
|
+
|
|
1331
|
+
|
|
1332
|
+
def _render_resolution_evidence(panel: dict) -> str:
|
|
1333
|
+
status = panel.get("audit_status", "unavailable")
|
|
1334
|
+
if status != "current":
|
|
1335
|
+
return (f"<p class='soft'>Resolution evidence: {_esc(status)}. "
|
|
1336
|
+
"Re-run the corresponding stage to save evidence for the current artifacts.</p>")
|
|
1337
|
+
result = (
|
|
1338
|
+
"<h3>Resolution evidence</h3><p class='soft'>Saved evidence matches the current artifacts. "
|
|
1339
|
+
f"Algorithm: {_esc(panel.get('audit_algorithm'))}. "
|
|
1340
|
+
f"Settings: {_esc(json.dumps(panel.get('audit_config'), sort_keys=True))}.</p>"
|
|
1341
|
+
)
|
|
1342
|
+
if panel.get("enrichment_status"):
|
|
1343
|
+
result += f"<p>Enrichment evidence: {_esc(panel['enrichment_status'])}.</p>"
|
|
1344
|
+
decisions = panel.get("audit_decisions", [])
|
|
1345
|
+
if decisions:
|
|
1346
|
+
items = "".join(
|
|
1347
|
+
"<li>" + "; ".join(f"<b>{_esc(k)}</b>: {_esc(v)}" for k, v in row.items()) + "</li>"
|
|
1348
|
+
for row in decisions
|
|
1349
|
+
)
|
|
1350
|
+
result += f"<details><summary>Decisions and review evidence</summary><ul>{items}</ul></details>"
|
|
1351
|
+
if panel.get("audit_decisions_truncated"):
|
|
1352
|
+
result += "<p class='soft'>Additional decisions are available in the audit JSON file.</p>"
|
|
1353
|
+
return result
|
|
1354
|
+
|
|
1355
|
+
|
|
1356
|
+
def _render_enrichment_section(panel: dict | None) -> str:
|
|
1357
|
+
if panel is None:
|
|
1358
|
+
return _placeholder("Enrichment", "citegraph enrich --out <out>")
|
|
1359
|
+
|
|
1360
|
+
rate = (
|
|
1361
|
+
f"{panel['match_rate'] * 100:.1f}%" if panel["match_rate"] is not None else "–"
|
|
1362
|
+
)
|
|
1363
|
+
sources = ", ".join(f"{_esc(k)}: {v}" for k, v in panel["sources"].items()) or "–"
|
|
1364
|
+
stats = (
|
|
1365
|
+
'<div class="stat-row">'
|
|
1366
|
+
f'<div class="stat"><span class="stat-n">{panel["n_references"]}</span>references</div>'
|
|
1367
|
+
f'<div class="stat"><span class="stat-n">{panel["n_matched"]}</span>matched</div>'
|
|
1368
|
+
f'<div class="stat"><span class="stat-n">{_esc(rate)}</span>match rate</div>'
|
|
1369
|
+
"</div>"
|
|
1370
|
+
f'<p class="soft">Sources — {sources}</p>'
|
|
1371
|
+
)
|
|
1372
|
+
|
|
1373
|
+
reasons = "".join(
|
|
1374
|
+
f"<tr><td>{_esc(reason)}</td><td class='num'>{count}</td></tr>"
|
|
1375
|
+
for reason, count in panel["miss_reasons"].items()
|
|
1376
|
+
)
|
|
1377
|
+
reasons_html = (
|
|
1378
|
+
"<h3>Miss reasons</h3>"
|
|
1379
|
+
'<div class="table-scroll"><table><thead><tr><th>reason</th><th>count</th></tr>'
|
|
1380
|
+
f"</thead><tbody>{reasons}</tbody></table></div>"
|
|
1381
|
+
if reasons
|
|
1382
|
+
else ""
|
|
1383
|
+
)
|
|
1384
|
+
|
|
1385
|
+
weak_rows = "".join(
|
|
1386
|
+
f"<tr><td>{_esc(w['title'])}</td><td>{_esc(w['candidate_title'])}</td>"
|
|
1387
|
+
f"<td class='num'>{_num(w['title_score'])}</td>"
|
|
1388
|
+
f"<td class='num'>{_num(w['adjusted_score'])}</td>"
|
|
1389
|
+
f"<td class='mono'>{_esc(w['doi'])}</td><td>{_esc(w['year_match'])}</td></tr>"
|
|
1390
|
+
for w in panel["weakest_matches"]
|
|
1391
|
+
)
|
|
1392
|
+
weak_html = (
|
|
1393
|
+
"<h3>Weakest accepted matches</h3>"
|
|
1394
|
+
'<p class="soft">Lowest title scores among status="matched" — check these '
|
|
1395
|
+
"for wrong DOIs.</p>"
|
|
1396
|
+
'<div class="table-scroll"><table><thead><tr><th>title</th><th>candidate title</th>'
|
|
1397
|
+
"<th>score</th><th>adjusted</th><th>doi</th><th>year match</th></tr></thead>"
|
|
1398
|
+
f"<tbody>{weak_rows}</tbody></table></div>"
|
|
1399
|
+
if weak_rows
|
|
1400
|
+
else ""
|
|
1401
|
+
)
|
|
1402
|
+
|
|
1403
|
+
miss_rows = "".join(
|
|
1404
|
+
f"<tr><td class='mono'>{_esc(m['id'])}</td><td>{_esc(m['title'])}</td>"
|
|
1405
|
+
f"<td class='num'>{_esc(_year_str(m['year']))}</td><td>{_esc(m['reason'])}</td>"
|
|
1406
|
+
f"<td class='num'>{_num(m['score'])}</td></tr>"
|
|
1407
|
+
for m in panel["misses_preview"]
|
|
1408
|
+
)
|
|
1409
|
+
n_misses = panel["n_misses"]
|
|
1410
|
+
misses_html = ""
|
|
1411
|
+
if miss_rows:
|
|
1412
|
+
suffix = f" (first {len(panel['misses_preview'])} of {n_misses})" if n_misses else ""
|
|
1413
|
+
misses_html = (
|
|
1414
|
+
f"<h3>Unmatched references{_esc(suffix)}</h3>"
|
|
1415
|
+
'<div class="table-scroll"><table><thead><tr><th>id</th><th>title</th>'
|
|
1416
|
+
"<th>year</th><th>reason</th><th>score</th></tr></thead>"
|
|
1417
|
+
f"<tbody>{miss_rows}</tbody></table></div>"
|
|
1418
|
+
)
|
|
1419
|
+
|
|
1420
|
+
return stats + reasons_html + weak_html + misses_html
|
|
1421
|
+
|
|
1422
|
+
|
|
1423
|
+
def _render_authors_section(panel: dict | None) -> str:
|
|
1424
|
+
if panel is None:
|
|
1425
|
+
return _placeholder("Authors", "citegraph authors --out <out>")
|
|
1426
|
+
|
|
1427
|
+
stats = (
|
|
1428
|
+
'<div class="stat-row">'
|
|
1429
|
+
f'<div class="stat"><span class="stat-n">{panel["n_authors"]}</span>canonical authors</div>'
|
|
1430
|
+
f'<div class="stat"><span class="stat-n">{_num(panel["n_occurrences"], 0)}</span>'
|
|
1431
|
+
"name occurrences</div></div>"
|
|
1432
|
+
)
|
|
1433
|
+
|
|
1434
|
+
top_rows = "".join(
|
|
1435
|
+
f"<tr><td class='mono'>{_esc(t['id'])}</td><td>{_esc(t['display_name'])}</td>"
|
|
1436
|
+
f"<td class='num'>{_num(t['n_citations_received'], 0)}</td>"
|
|
1437
|
+
f"<td class='num'>{_num(t['n_works'], 0)}</td>"
|
|
1438
|
+
f"<td class='num'>{_num(t['n_core_works'], 0)}</td></tr>"
|
|
1439
|
+
for t in panel["top_cited"]
|
|
1440
|
+
)
|
|
1441
|
+
top_html = (
|
|
1442
|
+
"<h3>Most-cited authors</h3>"
|
|
1443
|
+
'<div class="table-scroll"><table><thead><tr><th>id</th><th>name</th>'
|
|
1444
|
+
"<th>citations received</th><th>works</th><th>core works</th></tr></thead>"
|
|
1445
|
+
f"<tbody>{top_rows}</tbody></table></div>"
|
|
1446
|
+
if top_rows
|
|
1447
|
+
else ""
|
|
1448
|
+
)
|
|
1449
|
+
|
|
1450
|
+
review_html = ""
|
|
1451
|
+
if panel["review"]:
|
|
1452
|
+
entries = "".join(
|
|
1453
|
+
"<li>"
|
|
1454
|
+
+ "; ".join(f"<b>{_esc(k)}</b>: {_esc(v)}" for k, v in entry.items())
|
|
1455
|
+
+ "</li>"
|
|
1456
|
+
for entry in panel["review"]
|
|
1457
|
+
)
|
|
1458
|
+
review_html = (
|
|
1459
|
+
f"<h3>Flagged for review ({len(panel['review'])})</h3>"
|
|
1460
|
+
f'<ul class="review-list">{entries}</ul>'
|
|
1461
|
+
)
|
|
1462
|
+
|
|
1463
|
+
variant_parts = []
|
|
1464
|
+
for cluster in panel["variant_audit"]:
|
|
1465
|
+
forms = "".join(f"<li>{_esc(f)}</li>" for f in cluster["forms"])
|
|
1466
|
+
variant_parts.append(
|
|
1467
|
+
f'<details class="cluster"><summary><span class="mono">{_esc(cluster["id"])}</span> '
|
|
1468
|
+
f"{_esc(cluster['display_name'])} "
|
|
1469
|
+
f'<span class="soft">— {cluster["n_forms"]} distinct raw forms</span></summary>'
|
|
1470
|
+
f"<ul>{forms}</ul></details>"
|
|
1471
|
+
)
|
|
1472
|
+
variant_html = ""
|
|
1473
|
+
if variant_parts:
|
|
1474
|
+
variant_html = (
|
|
1475
|
+
"<h3>Name-variant audit</h3>"
|
|
1476
|
+
'<p class="soft">Clusters that merged the most distinct raw author '
|
|
1477
|
+
"strings — the merge audit for people.</p>" + "".join(variant_parts)
|
|
1478
|
+
)
|
|
1479
|
+
|
|
1480
|
+
return stats + top_html + review_html + variant_html + _render_resolution_evidence(panel)
|
|
1481
|
+
|
|
1482
|
+
|
|
1483
|
+
def _render_integrity_section(table: list[dict]) -> str:
|
|
1484
|
+
rows = []
|
|
1485
|
+
for row in table:
|
|
1486
|
+
status = row["status"]
|
|
1487
|
+
pill_status = {"ok": "ok", "corrupt": "corrupt", "missing": "missing"}[status]
|
|
1488
|
+
soft_missing = (
|
|
1489
|
+
"optional" in row["note"] or "absent" in row["note"] or "citegraph run" in row["note"]
|
|
1490
|
+
)
|
|
1491
|
+
if status == "missing" and soft_missing:
|
|
1492
|
+
pill = _pill("absent", "pending")
|
|
1493
|
+
else:
|
|
1494
|
+
pill = _pill(status.upper() if status == "corrupt" else status, pill_status)
|
|
1495
|
+
err = f'<div class="soft">{_esc(row["error"])}</div>' if row["error"] else ""
|
|
1496
|
+
rows.append(
|
|
1497
|
+
f"<tr><td class='mono'>{_esc(row['name'])}</td><td>{pill}</td>"
|
|
1498
|
+
f"<td>{_esc(row['note'])}{err}</td></tr>"
|
|
1499
|
+
)
|
|
1500
|
+
return (
|
|
1501
|
+
'<div class="table-scroll"><table><thead><tr><th>artifact</th><th>status</th>'
|
|
1502
|
+
f"<th>notes</th></tr></thead><tbody>{''.join(rows)}</tbody></table></div>"
|
|
1503
|
+
)
|
|
1504
|
+
|
|
1505
|
+
|
|
1506
|
+
_CSS = """
|
|
1507
|
+
:root {
|
|
1508
|
+
--paper: #f6f7f4; --surface: #fdfdfb; --line: #dde2db;
|
|
1509
|
+
--ink: #1b2420; --ink-soft: #55605a;
|
|
1510
|
+
--accent: #1f6a4e; --accent-soft: #e3eee8;
|
|
1511
|
+
--brass: #a07a2f; --warn: #b4552d; --warn-soft: #f6e9e2;
|
|
1512
|
+
--code-bg: #eef1ec;
|
|
1513
|
+
}
|
|
1514
|
+
@media (prefers-color-scheme: dark) {
|
|
1515
|
+
:root {
|
|
1516
|
+
--paper: #131a17; --surface: #1a231f; --line: #2c3833;
|
|
1517
|
+
--ink: #e6eae4; --ink-soft: #9aa8a0;
|
|
1518
|
+
--accent: #58b78e; --accent-soft: #1e3a2f;
|
|
1519
|
+
--brass: #c9a45c; --warn: #d98a63; --warn-soft: #3a2820;
|
|
1520
|
+
--code-bg: #101613;
|
|
1521
|
+
}
|
|
1522
|
+
}
|
|
1523
|
+
* { box-sizing: border-box; }
|
|
1524
|
+
body {
|
|
1525
|
+
margin: 0; background: var(--paper); color: var(--ink);
|
|
1526
|
+
font-family: system-ui, sans-serif; font-size: 15px; line-height: 1.5;
|
|
1527
|
+
}
|
|
1528
|
+
h1, h2, h3, h4 { font-family: "Iowan Old Style", Georgia, serif; line-height: 1.2; }
|
|
1529
|
+
h1 { font-size: 1.7rem; margin: 0 0 0.25rem; }
|
|
1530
|
+
h2 { font-size: 1.3rem; margin: 0 0 0.75rem; }
|
|
1531
|
+
h3 { font-size: 1.05rem; margin: 1.5rem 0 0.5rem; }
|
|
1532
|
+
h4 { font-size: 0.95rem; margin: 1rem 0 0.35rem; }
|
|
1533
|
+
code, .mono, pre {
|
|
1534
|
+
font-family: ui-monospace, "SF Mono", Menlo, monospace; font-size: 0.86em;
|
|
1535
|
+
}
|
|
1536
|
+
code { background: var(--code-bg); padding: 0.1em 0.35em; border-radius: 4px; }
|
|
1537
|
+
main { max-width: 1180px; margin: 0 auto; padding: 1rem 1.25rem 4rem; }
|
|
1538
|
+
section {
|
|
1539
|
+
background: var(--surface); border: 1px solid var(--line); border-radius: 10px;
|
|
1540
|
+
padding: 1.1rem 1.25rem 1.35rem; margin-top: 1.25rem;
|
|
1541
|
+
}
|
|
1542
|
+
.eyebrow {
|
|
1543
|
+
font-family: ui-monospace, "SF Mono", Menlo, monospace;
|
|
1544
|
+
text-transform: uppercase; letter-spacing: 0.14em; font-size: 0.68rem;
|
|
1545
|
+
color: var(--brass); margin: 0 0 0.35rem;
|
|
1546
|
+
}
|
|
1547
|
+
.soft { color: var(--ink-soft); }
|
|
1548
|
+
nav.mini {
|
|
1549
|
+
position: sticky; top: 0; z-index: 10; background: var(--paper);
|
|
1550
|
+
border-bottom: 1px solid var(--line); padding: 0.5rem 1.25rem;
|
|
1551
|
+
display: flex; gap: 1rem; flex-wrap: wrap; font-size: 0.85rem;
|
|
1552
|
+
}
|
|
1553
|
+
nav.mini a { color: var(--ink-soft); text-decoration: none; }
|
|
1554
|
+
nav.mini a:hover { color: var(--accent); }
|
|
1555
|
+
header.page { padding: 1.5rem 0 0; }
|
|
1556
|
+
header.page .meta { color: var(--ink-soft); font-size: 0.85rem; }
|
|
1557
|
+
.pill {
|
|
1558
|
+
display: inline-block; padding: 0.05em 0.55em; border-radius: 999px;
|
|
1559
|
+
font-family: ui-monospace, "SF Mono", Menlo, monospace; font-size: 0.72rem;
|
|
1560
|
+
white-space: nowrap; vertical-align: middle;
|
|
1561
|
+
}
|
|
1562
|
+
.pill-ok { background: var(--accent-soft); color: var(--accent); }
|
|
1563
|
+
.pill-warn { background: var(--warn-soft); color: var(--warn); }
|
|
1564
|
+
.pill-muted { background: var(--line); color: var(--ink-soft); }
|
|
1565
|
+
.stage-strip {
|
|
1566
|
+
display: grid; grid-template-columns: repeat(auto-fit, minmax(150px, 1fr));
|
|
1567
|
+
gap: 0.6rem; margin-top: 0.9rem;
|
|
1568
|
+
}
|
|
1569
|
+
.stage {
|
|
1570
|
+
border: 1px solid var(--line); border-radius: 8px; padding: 0.6rem 0.7rem;
|
|
1571
|
+
background: var(--surface);
|
|
1572
|
+
}
|
|
1573
|
+
.stage-done { border-color: var(--accent); }
|
|
1574
|
+
.stage-name {
|
|
1575
|
+
font-family: ui-monospace, "SF Mono", Menlo, monospace;
|
|
1576
|
+
text-transform: uppercase; letter-spacing: 0.1em; font-size: 0.68rem;
|
|
1577
|
+
color: var(--ink-soft); margin-bottom: 0.25rem;
|
|
1578
|
+
}
|
|
1579
|
+
.stage-detail { font-size: 0.8rem; color: var(--ink-soft); margin-top: 0.3rem; }
|
|
1580
|
+
.stage-hint { margin-top: 0.3rem; font-size: 0.72rem; overflow-x: auto; }
|
|
1581
|
+
.problem-cards {
|
|
1582
|
+
display: grid; grid-template-columns: repeat(auto-fill, minmax(190px, 1fr));
|
|
1583
|
+
gap: 0.6rem;
|
|
1584
|
+
}
|
|
1585
|
+
.problem-card {
|
|
1586
|
+
display: flex; align-items: baseline; gap: 0.5rem; text-decoration: none;
|
|
1587
|
+
border: 1px solid var(--warn); border-radius: 8px; padding: 0.55rem 0.7rem;
|
|
1588
|
+
background: var(--warn-soft); color: var(--ink);
|
|
1589
|
+
}
|
|
1590
|
+
.problem-count {
|
|
1591
|
+
font-family: ui-monospace, "SF Mono", Menlo, monospace; font-size: 1.15rem;
|
|
1592
|
+
font-variant-numeric: tabular-nums; color: var(--warn); font-weight: 600;
|
|
1593
|
+
}
|
|
1594
|
+
.problem-label { font-size: 0.82rem; }
|
|
1595
|
+
.all-clear { color: var(--accent); margin: 0; }
|
|
1596
|
+
.placeholder { color: var(--ink-soft); margin: 0; }
|
|
1597
|
+
.controls { display: flex; gap: 1rem; align-items: center; margin-bottom: 0.7rem; flex-wrap: wrap; }
|
|
1598
|
+
.controls input[type="search"] {
|
|
1599
|
+
background: var(--paper); color: var(--ink); border: 1px solid var(--line);
|
|
1600
|
+
border-radius: 6px; padding: 0.35rem 0.6rem; min-width: 240px; font: inherit;
|
|
1601
|
+
}
|
|
1602
|
+
.controls label { font-size: 0.85rem; color: var(--ink-soft); }
|
|
1603
|
+
.table-scroll { overflow-x: auto; }
|
|
1604
|
+
.paper-table { min-width: 900px; border-top: 1px solid var(--line); }
|
|
1605
|
+
.paper-row { border-bottom: 1px solid var(--line); }
|
|
1606
|
+
.paper-row summary, .paper-head {
|
|
1607
|
+
display: grid; grid-template-columns: 2.2fr 1.2fr 2.6fr 1.1fr 1.6fr;
|
|
1608
|
+
gap: 0.7rem; padding: 0.45rem 0.3rem; align-items: baseline; cursor: pointer;
|
|
1609
|
+
list-style: none;
|
|
1610
|
+
}
|
|
1611
|
+
.paper-head { font-size: 0.72rem; text-transform: uppercase; letter-spacing: 0.1em;
|
|
1612
|
+
color: var(--ink-soft); font-family: ui-monospace, "SF Mono", Menlo, monospace;
|
|
1613
|
+
cursor: default; }
|
|
1614
|
+
.paper-row summary::-webkit-details-marker { display: none; }
|
|
1615
|
+
.paper-row summary:hover { background: var(--code-bg); }
|
|
1616
|
+
.col { overflow: hidden; text-overflow: ellipsis; }
|
|
1617
|
+
.col-md .mono, .col-refs .mono { font-variant-numeric: tabular-nums; }
|
|
1618
|
+
.col-notes { font-size: 0.8rem; color: var(--warn); }
|
|
1619
|
+
.cell-title { font-size: 0.88rem; }
|
|
1620
|
+
.paper-body { padding: 0.5rem 0.75rem 1rem; background: var(--code-bg); border-radius: 6px;
|
|
1621
|
+
margin: 0 0.3rem 0.7rem; }
|
|
1622
|
+
.kv { display: grid; grid-template-columns: max-content 1fr; gap: 0.15rem 1rem; margin: 0.5rem 0; }
|
|
1623
|
+
.kv dt { color: var(--ink-soft); font-size: 0.8rem; }
|
|
1624
|
+
.kv dd { margin: 0; }
|
|
1625
|
+
.ref-list { margin: 0.25rem 0; padding-left: 1.5rem; font-size: 0.88rem; }
|
|
1626
|
+
.md-preview {
|
|
1627
|
+
max-height: 260px; overflow: auto; background: var(--paper);
|
|
1628
|
+
border: 1px solid var(--line); border-radius: 6px; padding: 0.6rem;
|
|
1629
|
+
white-space: pre-wrap; word-break: break-word; margin: 0.25rem 0;
|
|
1630
|
+
}
|
|
1631
|
+
.stat-row { display: flex; gap: 1.5rem; flex-wrap: wrap; margin: 0.4rem 0 0.8rem; }
|
|
1632
|
+
.stat { font-size: 0.8rem; color: var(--ink-soft); }
|
|
1633
|
+
.stat-n {
|
|
1634
|
+
display: block; font-family: ui-monospace, "SF Mono", Menlo, monospace;
|
|
1635
|
+
font-size: 1.25rem; color: var(--ink); font-variant-numeric: tabular-nums;
|
|
1636
|
+
}
|
|
1637
|
+
table { border-collapse: collapse; width: 100%; font-size: 0.88rem; }
|
|
1638
|
+
th, td { text-align: left; padding: 0.35rem 0.6rem; border-bottom: 1px solid var(--line);
|
|
1639
|
+
vertical-align: top; }
|
|
1640
|
+
th { font-size: 0.72rem; text-transform: uppercase; letter-spacing: 0.1em;
|
|
1641
|
+
color: var(--ink-soft); font-family: ui-monospace, "SF Mono", Menlo, monospace; }
|
|
1642
|
+
td.num { font-variant-numeric: tabular-nums; text-align: right;
|
|
1643
|
+
font-family: ui-monospace, "SF Mono", Menlo, monospace; font-size: 0.82rem; }
|
|
1644
|
+
details.cluster { border: 1px solid var(--line); border-radius: 6px;
|
|
1645
|
+
padding: 0.35rem 0.7rem; margin: 0.35rem 0; background: var(--paper); }
|
|
1646
|
+
details.cluster summary { cursor: pointer; }
|
|
1647
|
+
details.cluster ul { margin: 0.4rem 0; }
|
|
1648
|
+
.review-list { font-size: 0.85rem; }
|
|
1649
|
+
.size-note { color: var(--warn); font-size: 0.85rem; }
|
|
1650
|
+
"""
|
|
1651
|
+
|
|
1652
|
+
_JS = """
|
|
1653
|
+
(function () {
|
|
1654
|
+
var blob = document.getElementById('data');
|
|
1655
|
+
var DATA = { papers: [] };
|
|
1656
|
+
try { DATA = JSON.parse(blob.textContent); } catch (e) { /* blob is advisory */ }
|
|
1657
|
+
var q = document.getElementById('filter-text');
|
|
1658
|
+
var cb = document.getElementById('filter-problems');
|
|
1659
|
+
var count = document.getElementById('paper-count');
|
|
1660
|
+
if (!q || !cb) { return; }
|
|
1661
|
+
var rows = Array.prototype.slice.call(document.querySelectorAll('details.paper-row'));
|
|
1662
|
+
var total = (DATA.papers && DATA.papers.length) || rows.length;
|
|
1663
|
+
function apply() {
|
|
1664
|
+
var needle = q.value.trim().toLowerCase();
|
|
1665
|
+
var shown = 0;
|
|
1666
|
+
rows.forEach(function (el) {
|
|
1667
|
+
var hay = (el.dataset.stem + ' ' + el.dataset.title).toLowerCase();
|
|
1668
|
+
var ok = (!needle || hay.indexOf(needle) !== -1) &&
|
|
1669
|
+
(!cb.checked || el.dataset.problem === '1');
|
|
1670
|
+
el.hidden = !ok;
|
|
1671
|
+
if (ok) { shown += 1; }
|
|
1672
|
+
});
|
|
1673
|
+
if (count) { count.textContent = 'showing ' + shown + ' of ' + total; }
|
|
1674
|
+
}
|
|
1675
|
+
q.addEventListener('input', apply);
|
|
1676
|
+
cb.addEventListener('change', apply);
|
|
1677
|
+
apply();
|
|
1678
|
+
})();
|
|
1679
|
+
"""
|
|
1680
|
+
|
|
1681
|
+
|
|
1682
|
+
def build_report_html(data: dict) -> str:
|
|
1683
|
+
"""Render the collected data dict into one self-contained HTML page."""
|
|
1684
|
+
blob = (
|
|
1685
|
+
json.dumps(_jsonable(data), ensure_ascii=False)
|
|
1686
|
+
.replace("&", "\\u0026")
|
|
1687
|
+
.replace("<", "\\u003c")
|
|
1688
|
+
.replace(">", "\\u003e")
|
|
1689
|
+
)
|
|
1690
|
+
|
|
1691
|
+
size_note = ""
|
|
1692
|
+
if data.get("truncated_for_size"):
|
|
1693
|
+
size_note = (
|
|
1694
|
+
'<p class="size-note">Previews were truncated to keep this report under '
|
|
1695
|
+
"10 MB.</p>"
|
|
1696
|
+
)
|
|
1697
|
+
empty_note = ""
|
|
1698
|
+
if not data.get("papers"):
|
|
1699
|
+
empty_note = (
|
|
1700
|
+
'<p class="placeholder">Nothing processed yet — run '
|
|
1701
|
+
"<code>citegraph convert <pdf_dir> --out "
|
|
1702
|
+
f"{_esc(data['out_dir'])}</code> to get started.</p>"
|
|
1703
|
+
)
|
|
1704
|
+
|
|
1705
|
+
header = (
|
|
1706
|
+
'<header class="page" id="overview">'
|
|
1707
|
+
'<p class="eyebrow">citegraph · quality-control report</p>'
|
|
1708
|
+
f"<h1>{_esc(Path(data['out_dir']).name) or 'out'}</h1>"
|
|
1709
|
+
f'<p class="meta"><span class="mono">{_esc(data["out_dir"])}</span> · '
|
|
1710
|
+
f"generated {_esc(data['generated_at'])}</p>"
|
|
1711
|
+
f"{size_note}{empty_note}"
|
|
1712
|
+
f"{_render_stage_strip(data['stages'])}</header>"
|
|
1713
|
+
)
|
|
1714
|
+
|
|
1715
|
+
nav = (
|
|
1716
|
+
'<nav class="mini">'
|
|
1717
|
+
'<a href="#overview">Overview</a><a href="#problems">Problems</a>'
|
|
1718
|
+
'<a href="#papers">Papers</a><a href="#dedup">Dedup</a>'
|
|
1719
|
+
'<a href="#enrichment">Enrichment</a><a href="#authors">Authors</a>'
|
|
1720
|
+
'<a href="#integrity">Integrity</a></nav>'
|
|
1721
|
+
)
|
|
1722
|
+
|
|
1723
|
+
sections = [
|
|
1724
|
+
_section("problems", "summary", "Problems", _render_problem_cards(data["problems"])),
|
|
1725
|
+
_section(
|
|
1726
|
+
"papers", "per-paper", "Papers", _render_papers_section(data["papers"])
|
|
1727
|
+
),
|
|
1728
|
+
_section("dedup", "stage 4", "Deduplication", _render_dedup_section(data["dedup"])),
|
|
1729
|
+
_section(
|
|
1730
|
+
"enrichment",
|
|
1731
|
+
"stage 5 · optional",
|
|
1732
|
+
"Enrichment",
|
|
1733
|
+
_render_enrichment_section(data["enrichment"]),
|
|
1734
|
+
),
|
|
1735
|
+
_section(
|
|
1736
|
+
"authors", "stage 6 · optional", "Authors", _render_authors_section(data["authors"])
|
|
1737
|
+
),
|
|
1738
|
+
_section("integrity", "artifacts", "Integrity", _render_integrity_section(data["integrity"])),
|
|
1739
|
+
]
|
|
1740
|
+
|
|
1741
|
+
return (
|
|
1742
|
+
"<!doctype html>\n"
|
|
1743
|
+
'<html lang="en"><head><meta charset="utf-8">'
|
|
1744
|
+
'<meta name="viewport" content="width=device-width, initial-scale=1">'
|
|
1745
|
+
f"<title>citegraph report — {_esc(Path(data['out_dir']).name)}</title>"
|
|
1746
|
+
f"<style>{_CSS}</style></head><body>"
|
|
1747
|
+
f"{nav}<main>{header}{''.join(sections)}</main>"
|
|
1748
|
+
f'<script type="application/json" id="data">{blob}</script>'
|
|
1749
|
+
f"<script>{_JS}</script></body></html>"
|
|
1750
|
+
)
|
|
1751
|
+
|
|
1752
|
+
|
|
1753
|
+
def _shrink_for_size(data: dict) -> dict:
|
|
1754
|
+
"""Truncate previews when the rendered page would exceed the size cap."""
|
|
1755
|
+
shrunk = dict(data)
|
|
1756
|
+
shrunk["truncated_for_size"] = True
|
|
1757
|
+
papers = []
|
|
1758
|
+
for paper in data.get("papers", []):
|
|
1759
|
+
p = dict(paper)
|
|
1760
|
+
if p.get("markdown_preview"):
|
|
1761
|
+
p["markdown_preview"] = p["markdown_preview"][:_SHRUNK_MARKDOWN_PREVIEW_CHARS]
|
|
1762
|
+
preview = p.get("refs_preview") or []
|
|
1763
|
+
if len(preview) > _SHRUNK_REFS_PREVIEW_N:
|
|
1764
|
+
count = p["references"].get("count") or 0
|
|
1765
|
+
p["refs_preview"] = preview[:_SHRUNK_REFS_PREVIEW_N]
|
|
1766
|
+
p["refs_more"] = max(count - _SHRUNK_REFS_PREVIEW_N, 0)
|
|
1767
|
+
papers.append(p)
|
|
1768
|
+
shrunk["papers"] = papers
|
|
1769
|
+
return shrunk
|
|
1770
|
+
|
|
1771
|
+
|
|
1772
|
+
def write_report(out_dir: Path, *, data: dict | None = None) -> Path:
|
|
1773
|
+
"""Collect, render, and write ``report.html`` into ``out_dir``.
|
|
1774
|
+
|
|
1775
|
+
Returns the path of the written file. ``out_dir`` is created if it
|
|
1776
|
+
doesn't exist so the command is safe to run at any stage.
|
|
1777
|
+
"""
|
|
1778
|
+
out_dir = Path(out_dir)
|
|
1779
|
+
if data is None:
|
|
1780
|
+
data = collect_report_data(out_dir)
|
|
1781
|
+
html_text = build_report_html(data)
|
|
1782
|
+
if len(html_text.encode("utf-8")) > _MAX_HTML_BYTES:
|
|
1783
|
+
html_text = build_report_html(_shrink_for_size(data))
|
|
1784
|
+
layout = OutLayout(out_dir)
|
|
1785
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
1786
|
+
layout.report_html.write_text(html_text, encoding="utf-8")
|
|
1787
|
+
return layout.report_html
|