citegraph 0.2.0__py3-none-any.whl

citegraph/__init__.py

@@ -0,0 +1,44 @@
+"""citegraph: build a deduplicated citation graph from a folder of PDFs.
+
+See :class:`citegraph.Pipeline` for the high-level entry point::
+
+    from citegraph import Pipeline
+
+    pipe = Pipeline(pdf_dir="./pdfs", out_dir="./out")
+    result = pipe.run()
+    result.works, result.graph
+"""
+
+from __future__ import annotations
+
+from citegraph.authors import (
+    AuthorClusterConfig,
+    ParsedAuthor,
+    normalize_authors,
+    parse_author,
+)
+from citegraph.cost_estimation import ExtractionEstimate
+from citegraph.graph import CitationGraph
+from citegraph.pipeline import Pipeline
+from citegraph.schemas import (
+    CitationLink,
+    PaperMetadata,
+    PipelineResult,
+    Reference,
+)
+
+__all__ = [
+    "Pipeline",
+    "CitationGraph",
+    "ExtractionEstimate",
+    "PipelineResult",
+    "PaperMetadata",
+    "Reference",
+    "CitationLink",
+    "AuthorClusterConfig",
+    "ParsedAuthor",
+    "normalize_authors",
+    "parse_author",
+]
+
+__version__ = "0.2.0"

citegraph/_progress.py

@@ -0,0 +1,74 @@
+"""Internal helper: iterate a list with an optional rich progress bar.
+
+Stage loops in :mod:`citegraph.pipeline` and the PDF -> markdown loop in
+:mod:`citegraph.pdf_to_markdown` all want the same UX (spinner, current
+item label, count, elapsed, ETA) over a list whose body is per-item work.
+This module exists so that affordance lives in one place.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterator
+from typing import TypeVar
+
+T = TypeVar("T")
+
+
+def iter_with_progress(
+    items: list[T],
+    *,
+    show_progress: bool,
+    description: str,
+    item_label: Callable[[T], str] | None = None,
+) -> Iterator[T]:
+    """Yield ``items`` one at a time, optionally with a rich progress bar.
+
+    Parameters
+    ----------
+    items:
+        Sequence to iterate. ``len(items)`` sets the bar total.
+    show_progress:
+        When ``False`` (or ``items`` is empty), yields plainly with no
+        bar — callers don't need to branch on the flag themselves.
+    description:
+        Initial bar description, shown until the first item starts.
+    item_label:
+        Optional callable producing a per-item description (e.g. the
+        current filename). Long labels are truncated to keep the bar
+        layout from jittering.
+    """
+    if not show_progress or not items:
+        yield from items
+        return
+
+    # Imported lazily so callers that pass ``show_progress=False`` don't
+    # pay the import cost — keeps test runs and headless CLI use cheap.
+    from rich.progress import (
+        BarColumn,
+        MofNCompleteColumn,
+        Progress,
+        SpinnerColumn,
+        TextColumn,
+        TimeElapsedColumn,
+        TimeRemainingColumn,
+    )
+
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(),
+        MofNCompleteColumn(),
+        TimeElapsedColumn(),
+        TimeRemainingColumn(),
+        transient=False,
+    ) as progress:
+        task = progress.add_task(description, total=len(items))
+        for item in items:
+            if item_label is not None:
+                label = item_label(item)
+                if len(label) > 50:
+                    label = label[:47] + "..."
+                progress.update(task, description=label)
+            yield item
+            progress.advance(task)
+        progress.update(task, description="Done")

citegraph/annotations.py

@@ -0,0 +1,183 @@
+"""Hand-curated facts about works, joined onto the pipeline's output.
+
+Some things about a paper are never in the paper: the theme a reviewer
+assigned it, whether it counts as the design the manuscript is about. They
+are read off the PDF by a human — or by an agent — and recorded by hand.
+
+They cannot live in ``works.csv``, which stage 4 regenerates from scratch;
+a column typed in there survives exactly until the next ``citegraph dedup``.
+So annotations live beside it in ``work_annotations.csv``, keyed on the work
+``id``, and are joined at read time. This is the arrangement
+``journal_aliases.csv`` and ``author_aliases.csv`` already use: lenient read,
+strict validation, and a loud failure when a curated row no longer matches
+anything.
+
+``annotation_schema.csv`` (``column,type,allowed,description``) declares what
+may be recorded. Declared columns are validated on load — an agent writing
+``Maybe`` into an enum fails immediately, naming the work. Undeclared columns
+are carried untouched, so a reviewer can add a question mid-review without a
+code change. The schema is as much the annotator's instructions as it is a
+constraint, which is why the description column is not decoration.
+"""
+
+from __future__ import annotations
+
+import csv
+from collections.abc import Mapping
+from dataclasses import dataclass
+from pathlib import Path
+
+import pandas as pd
+
+from citegraph.io import OutLayout, read_json
+
+SCHEMA_COLUMNS = ["column", "type", "allowed", "description"]
+COLUMN_TYPES = ("enum", "text")
+
+
+@dataclass(frozen=True)
+class AnnotationColumn:
+    """One declared annotation column."""
+
+    name: str
+    type: str
+    allowed: tuple[str, ...]
+    description: str
+
+
+def load_annotation_schema(path: Path | str) -> dict[str, AnnotationColumn]:
+    """Read ``annotation_schema.csv``; an absent file declares nothing."""
+    path = Path(path)
+    if not path.exists():
+        return {}
+    with path.open(encoding="utf-8", newline="") as stream:
+        rows = list(csv.DictReader(stream))
+    schema: dict[str, AnnotationColumn] = {}
+    for number, row in enumerate(rows, 2):
+        name = (row.get("column") or "").strip()
+        kind = (row.get("type") or "").strip()
+        allowed = tuple(
+            value.strip() for value in (row.get("allowed") or "").split("|") if value.strip()
+        )
+        if not name:
+            raise ValueError(f"Annotation schema row {number}: column name is empty")
+        if kind not in COLUMN_TYPES:
+            raise ValueError(
+                f"Annotation schema row {number}: unknown type {kind!r} for {name!r}; "
+                f"use one of {', '.join(COLUMN_TYPES)}"
+            )
+        if kind == "enum" and not allowed:
+            raise ValueError(
+                f"Annotation schema row {number}: enum column {name!r} declares no "
+                "allowed values; list them separated by '|'"
+            )
+        if name in schema:
+            raise ValueError(f"Annotation schema declares {name!r} twice")
+        schema[name] = AnnotationColumn(
+            name=name,
+            type=kind,
+            allowed=allowed,
+            description=(row.get("description") or "").strip(),
+        )
+    return schema
+
+
+def _redirects(path: Path) -> dict[str, str]:
+    """Work-id redirects recorded by the identity registry beside the file."""
+    registry = OutLayout(path.parent).work_identity_json
+    return read_json(registry).get("redirects", {}) if registry.exists() else {}
+
+
+def _empty() -> pd.DataFrame:
+    return pd.DataFrame(index=pd.Index([], name="id"))
+
+
+def load_annotations(
+    path: Path | str,
+    *,
+    works: pd.DataFrame,
+    schema_path: Path | str | None = None,
+) -> pd.DataFrame:
+    """Read ``work_annotations.csv`` as a frame ready to join onto ``works``.
+
+    Rows are keyed on the persistent work ``id`` and resolved through the
+    identity registry's redirects, so an annotation written before a merge
+    still reaches the work it became. An id that names no work raises rather
+    than being dropped: a curated row that silently applies to nothing is
+    worse than no row at all.
+
+    Columns that ``works`` already has (``Title``, ``Year``, …) are *context*
+    — they are written into the file so a human can read it, and are dropped
+    here rather than joined back over the pipeline's own values.
+    """
+    from citegraph.identity import resolve_redirect
+
+    path = Path(path)
+    schema = load_annotation_schema(schema_path) if schema_path is not None else {}
+    if not path.exists():
+        return _empty()
+
+    with path.open(encoding="utf-8", newline="") as stream:
+        reader = csv.DictReader(stream)
+        fieldnames = reader.fieldnames or []
+        rows = list(reader)
+    if "id" not in fieldnames:
+        raise ValueError(f"{path} must have an 'id' column; got {fieldnames}")
+
+    redirects = _redirects(path)
+    known = set(works.index)
+    records: dict[str, dict[str, object]] = {}
+    for number, row in enumerate(rows, 2):
+        raw_id = (row.get("id") or "").strip()
+        if not raw_id:
+            raise ValueError(f"{path} row {number}: empty work id")
+        work_id = resolve_redirect(raw_id, redirects)
+        if work_id not in known:
+            raise ValueError(
+                f"{path} row {number}: unknown work id {work_id!r}. Re-run the "
+                "import, or delete the row if the work is gone."
+            )
+        if work_id in records:
+            raise ValueError(f"{path} row {number}: work {work_id!r} is annotated twice")
+        records[work_id] = {
+            key: (value.strip() or None)
+            for key, value in row.items()
+            if key is not None and key != "id" and isinstance(value, str)
+        }
+
+    # Context columns are readable padding for the editor, not annotations.
+    columns = [
+        name
+        for name in fieldnames
+        if name != "id" and (name in schema or name not in works.columns)
+    ]
+    for name in schema:
+        if name not in columns:
+            columns.append(name)
+
+    frame = pd.DataFrame(
+        [[records[work_id].get(name) for name in columns] for work_id in records],
+        index=pd.Index(list(records), name="id"),
+        columns=columns,
+        dtype=object,
+    )
+    _validate_values(frame, schema, path)
+    return frame
+
+
+def _validate_values(
+    frame: pd.DataFrame, schema: Mapping[str, AnnotationColumn], path: Path
+) -> None:
+    """Reject any value a declared enum column does not allow."""
+    for name, column in schema.items():
+        if column.type != "enum" or name not in frame.columns:
+            continue
+        for work_id, value in frame[name].items():
+            if value is None or pd.isna(value):
+                continue
+            if value not in column.allowed:
+                raise ValueError(
+                    f"{path}: work {work_id!r} has {name}={value!r}, which is not "
+                    f"one of {', '.join(column.allowed)}. Fix the value, or add it "
+                    "to annotation_schema.csv."
+                )

citegraph/author_export.py

@@ -0,0 +1,294 @@
+"""One spreadsheet holding everything the corpus knows about its own authors.
+
+``authors.csv`` describes every author in the corpus — thousands of them,
+most encountered once in somebody's bibliography. The question a manuscript
+actually asks is narrower: *who wrote my papers, and what do we know about
+them?* That answer is spread over half a dozen artifacts, and often
+reaches outside ``out_dir`` entirely, into hand-collected career metadata.
+
+This module assembles it into a single workbook keyed on ``author_id``.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+import pandas as pd
+
+from citegraph.io import OutLayout, read_json
+
+
+@dataclass(frozen=True)
+class ExtraSheet:
+    """A table from outside ``out_dir``, filtered to the core authors."""
+
+    name: str
+    frame: pd.DataFrame
+    description: str = ""
+    master_columns: tuple[str, ...] = ()
+    """Columns lifted onto the ``Authors`` sheet; requires one row per author."""
+
+
+_GENERATED_SHEETS = ("README", "Authors", "Works", "Review_Flags")
+
+
+def _validate_extra_sheet_names(extra_sheets: tuple[ExtraSheet, ...]) -> None:
+    used = {name.casefold(): name for name in _GENERATED_SHEETS}
+    for sheet in extra_sheets:
+        folded = sheet.name.casefold()
+        if folded in used:
+            raise ValueError(
+                f"Extra sheet {sheet.name!r} conflicts with workbook sheet {used[folded]!r}"
+            )
+        used[folded] = sheet.name
+
+
+def load_extra_sheet(
+    name: str,
+    path: Path | str,
+    *,
+    description: str = "",
+    master_columns: tuple[str, ...] = (),
+) -> ExtraSheet:
+    """Read a curated CSV from outside ``out_dir`` as an :class:`ExtraSheet`."""
+    path = Path(path)
+    if not path.is_file():
+        raise ValueError(f"Extra sheet {name!r}: no such file: {path}")
+    return ExtraSheet(
+        name=name,
+        frame=pd.read_csv(path),
+        description=description or f"Curated table from {path.name}.",
+        master_columns=master_columns,
+    )
+
+
+def _require_openpyxl() -> None:
+    """Excel writing is an extra, so the base install stays light."""
+    try:
+        import openpyxl  # noqa: F401
+    except ImportError as exc:  # pragma: no cover - exercised via monkeypatch
+        raise ImportError(
+            "Writing .xlsx needs openpyxl, which is not a default dependency. "
+            'Install it with: pip install "citegraph[excel]"'
+        ) from exc
+
+
+def _author_id_columns(frame: pd.DataFrame) -> list[str]:
+    return [
+        str(column)
+        for column in frame.columns
+        if str(column) == "author_id" or str(column).endswith("_author_id")
+    ]
+
+
+def build_core_author_workbook(
+    out_dir: Path | str,
+    dest: Path | str | None = None,
+    *,
+    extra_sheets: tuple[ExtraSheet, ...] = (),
+    ring: int = 0,
+) -> Path:
+    """Write ``core_authors.xlsx`` for the authors of ``ring``'s works."""
+    _validate_extra_sheet_names(extra_sheets)
+    _require_openpyxl()
+    layout = OutLayout(Path(out_dir))
+    works = pd.read_csv(layout.works_csv)
+    occurrences = pd.read_csv(layout.author_citations_csv)
+    authors = pd.read_csv(layout.authors_csv)
+
+    core_works = set(works.loc[works["ring"] == ring, "id"])
+    core_ids = set(occurrences.loc[occurrences["record_id"].isin(core_works), "author_id"])
+    master = authors[authors["id"].isin(core_ids)].copy()
+    master["name_variants"] = master["id"].map(_name_variants(occurrences, core_ids))
+    works_sheet = _works_sheet(occurrences, works, core_ids)
+
+    centrality = _optional_frame(layout.author_centrality_csv)
+    if centrality is not None and "author_id" in centrality.columns:
+        master = master.merge(
+            centrality.drop(columns=[c for c in ("Author",) if c in centrality.columns]),
+            left_on="id",
+            right_on="author_id",
+            how="left",
+        ).drop(columns=["author_id"])
+    review = _review_sheet(layout.author_review_json, core_ids)
+
+    filtered: list[tuple[ExtraSheet, pd.DataFrame, str]] = []
+    for sheet in extra_sheets:
+        rows = _filter_to_core(sheet, core_ids)
+        covered: set[str] = set()
+        for column in _author_id_columns(rows):
+            covered |= set(rows[column].dropna())
+        master[f"in_{sheet.name}"] = master["id"].isin(covered)
+        master = _lift_columns(master, sheet, rows)
+        n_covered = len(covered & core_ids)
+        filtered.append((sheet, rows, f"{n_covered} of {len(core_ids)}"))
+
+    tables: list[tuple[str, pd.DataFrame, str, str, str]] = [
+        ("Authors", master, "authors.csv", f"One row per author of a ring-{ring} work.",
+         f"{len(master)} of {len(core_ids)}"),
+        ("Works", works_sheet, "author_citations.csv + works.csv",
+         "One row per (author, work), core and cited alike.", ""),
+    ]
+    if review is not None:
+        tables.append(
+            ("Review_Flags", review, "author_review.json",
+             "Clusters the author stage flagged as low-confidence.", "")
+        )
+    tables.extend(
+        (sheet.name, rows, "supplied by --extra-csv", sheet.description, coverage)
+        for sheet, rows, coverage in filtered
+    )
+
+    path = Path(dest) if dest is not None else layout.core_authors_xlsx
+    with pd.ExcelWriter(path, engine="openpyxl") as writer:
+        _readme(tables).to_excel(writer, sheet_name="README", index=False)
+        for name, frame, _source, _description, _coverage in tables:
+            frame.to_excel(writer, sheet_name=name, index=False)
+    return path
+
+
+def _readme(
+    tables: list[tuple[str, pd.DataFrame, str, str, str]],
+) -> pd.DataFrame:
+    """A contents page, so the workbook explains itself to whoever opens it.
+
+    ``Core_authors_covered`` is the column to read first on a curated sheet:
+    hand-collected metadata never covers everybody, and the gap is a finding
+    about the corpus rather than a fault in the export.
+    """
+    return pd.DataFrame(
+        [
+            {
+                "Sheet": name,
+                "Rows": len(frame),
+                "Core_authors_covered": coverage,
+                "Source": source,
+                "Description": description,
+            }
+            for name, frame, source, description, coverage in tables
+        ]
+    )
+
+
+def _optional_frame(path: Path) -> pd.DataFrame | None:
+    """Read a CSV that may not exist, and never fail the export over it."""
+    if not path.exists():
+        return None
+    try:
+        return pd.read_csv(path)
+    except (OSError, ValueError, pd.errors.ParserError):
+        return None
+
+
+def _review_sheet(path: Path, core_ids: set[str]) -> pd.DataFrame | None:
+    """The author-QC flags, for core authors only."""
+    if not path.exists():
+        return None
+    try:
+        flags = read_json(path)
+    except (OSError, ValueError):
+        return None
+    if not isinstance(flags, list):
+        return None
+    rows = [
+        {
+            key: ("; ".join(str(item) for item in value) if isinstance(value, list) else value)
+            for key, value in flag.items()
+        }
+        for flag in flags
+        if isinstance(flag, dict) and flag.get("author_id") in core_ids
+    ]
+    return pd.DataFrame(rows)
+
+
+def _filter_to_core(sheet: ExtraSheet, core_ids: set[str]) -> pd.DataFrame:
+    """Keep a row when *any* of its author-id columns names a core author.
+
+    One rule covers both shapes the curated tables take: a per-person table
+    keyed on ``author_id``, and an edge table with ``advisor_author_id`` and
+    ``advisee_author_id``, where a tie is worth keeping if either end is
+    somebody we published.
+    """
+    columns = _author_id_columns(sheet.frame)
+    if not columns:
+        raise ValueError(
+            f"Extra sheet {sheet.name!r} has no author-id column; expected "
+            f"'author_id' or a column ending in '_author_id', got "
+            f"{list(sheet.frame.columns)}"
+        )
+    keep = pd.Series(False, index=sheet.frame.index)
+    for column in columns:
+        keep |= sheet.frame[column].isin(core_ids)
+    return sheet.frame[keep].copy()
+
+
+_WORK_COLUMNS = ["ring", "source_file", "Title", "Journal", "Journal_Canonical", "Year"]
+
+
+def _works_sheet(
+    occurrences: pd.DataFrame, works: pd.DataFrame, core_ids: set[str]
+) -> pd.DataFrame:
+    """Every work a core author appears on, core and cited alike.
+
+    Restricting this to ring 0 would answer a question nobody asked: an
+    author is interesting partly *because* the corpus cites their other work.
+    """
+    mine = occurrences[occurrences["author_id"].isin(core_ids)]
+    columns = [column for column in _WORK_COLUMNS if column in works.columns]
+    return mine.merge(
+        works[["id", *columns]].rename(columns={"id": "record_id"}),
+        on="record_id",
+        how="left",
+    )
+
+
+def _name_variants(occurrences: pd.DataFrame, core_ids: set[str]) -> dict[str, str]:
+    """Every distinct spelling the bibliographies used, per author.
+
+    This is the evidence the cluster was built from, and the fastest way for
+    a reader to see that a merge was right — or that it was not.
+    """
+    mine = occurrences[occurrences["author_id"].isin(core_ids)]
+    grouped = mine.groupby("author_id")["raw_author"]
+    return {
+        author_id: "; ".join(sorted({str(name) for name in names if pd.notna(name)}))
+        for author_id, names in grouped
+    }
+
+
+def _lift_columns(
+    master: pd.DataFrame, sheet: ExtraSheet, rows: pd.DataFrame
+) -> pd.DataFrame:
+    """Copy a curated sheet's headline columns onto the master."""
+    if not sheet.master_columns:
+        return master
+    if "author_id" not in rows.columns:
+        raise ValueError(
+            f"Extra sheet {sheet.name!r} lifts columns into the master but has no "
+            "'author_id' column to key them on"
+        )
+    duplicated = rows["author_id"].duplicated().any()
+    if duplicated:
+        raise ValueError(
+            f"Extra sheet {sheet.name!r} lifts columns into the master but does not "
+            "have one row per author; lift from a per-person table instead"
+        )
+    missing = [name for name in sheet.master_columns if name not in rows.columns]
+    if missing:
+        raise ValueError(
+            f"Extra sheet {sheet.name!r} cannot lift {missing}; it has "
+            f"{list(rows.columns)}"
+        )
+    clash = [name for name in sheet.master_columns if name in master.columns]
+    if clash:
+        raise ValueError(
+            f"Extra sheet {sheet.name!r} would overwrite existing column(s) {clash} "
+            "on the Authors sheet; rename them in the source table"
+        )
+    return master.merge(
+        rows[["author_id", *sheet.master_columns]],
+        left_on="id",
+        right_on="author_id",
+        how="left",
+    ).drop(columns=["author_id"])