erga 0.1.0__py3-none-any.whl

erga/__init__.py

@@ -0,0 +1,7 @@
+"""erga: keep a website's academic publications list current.
+
+Fetches works from OpenAlex, deduplicates them across registrars, applies
+maintainer curation, and writes canonical JSON for any static site to render.
+"""
+
+__version__ = "0.1.0"

erga/cli.py

@@ -0,0 +1,96 @@
+"""Console entry point: `erga build` and `erga verify`."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+from erga import __version__
+from erga.config import Config, load_config
+from erga.crossref import CrossrefClient
+from erga.errors import ErgaError
+from erga.http import UrlTransport
+from erga.openalex import OpenAlexClient
+from erga.pipeline import build
+from erga.verify import verify_report
+
+
+def _user_agent(mailto: str) -> str:
+    return f"erga/{__version__} (https://github.com/belalik/erga; mailto:{mailto})"
+
+
+def _clients(config: Config) -> tuple[OpenAlexClient, CrossrefClient]:
+    # One transport for both APIs. The key never touches disk: read from the
+    # configured env var, passed as a query parameter, nothing else. The
+    # transport redacts it from network-error text (requests embeds the full
+    # request URL in its exception messages).
+    api_key = os.environ.get(config.api_key_env) or None
+    if not api_key:
+        print(
+            f"erga: note: {config.api_key_env} not set; using OpenAlex's keyless per-IP quota",
+            file=sys.stderr,
+        )
+    transport = UrlTransport(_user_agent(config.mailto), secrets=[api_key] if api_key else [])
+    return (
+        OpenAlexClient(transport, mailto=config.mailto, api_key=api_key),
+        CrossrefClient(transport, mailto=config.mailto),
+    )
+
+
+def _print_warnings(warnings: list[str]) -> None:
+    for warning in warnings:
+        print(f"erga: warning: {warning}", file=sys.stderr)
+
+
+def _run_build(config: Config, dry_run: bool) -> int:
+    openalex, crossref = _clients(config)
+    stats = build(config, openalex, crossref, dry_run=dry_run)
+    _print_warnings(stats.warnings)
+    if dry_run:
+        print(f"dry run: {stats.summary()}")
+        print(f"would write {stats.total} works to {config.output_path}")
+    else:
+        print(f"wrote {config.output_path} ({stats.total} works)")
+    return 0
+
+
+def _run_verify(config: Config) -> int:
+    openalex, _ = _clients(config)
+    report, warnings = verify_report(config, openalex)
+    print(report, end="")
+    _print_warnings(warnings)
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="erga",
+        description="Keep a website's academic publications list current.",
+    )
+    parser.add_argument("--version", action="version", version=f"erga {__version__}")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    build_parser = subparsers.add_parser("build", help="run the pipeline and write the JSON")
+    build_parser.add_argument("--config", type=Path, default=Path("erga.yml"))
+    build_parser.add_argument(
+        "--dry-run", action="store_true", help="print a summary without writing"
+    )
+
+    verify_parser = subparsers.add_parser("verify", help="author-disambiguation report")
+    verify_parser.add_argument("--config", type=Path, default=Path("erga.yml"))
+
+    args = parser.parse_args(argv)
+    try:
+        config = load_config(args.config)
+        if args.command == "build":
+            return _run_build(config, args.dry_run)
+        return _run_verify(config)
+    except ErgaError as exc:
+        print(f"erga: {exc}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

erga/config.py

@@ -0,0 +1,139 @@
+"""Configuration loading and validation (requirements section 5)."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from erga.errors import ConfigError
+from erga.model import normalize_orcid
+
+_ORCID_RE = re.compile(r"^\d{4}-\d{4}-\d{4}-\d{3}[\dX]$")
+_OPENALEX_AUTHOR_RE = re.compile(r"^A\d+$")
+
+
+@dataclass
+class AuthorConfig:
+    name: str
+    orcid: str | None = None
+    openalex_id: str | None = None
+    aliases: list[str] = field(default_factory=list)
+
+    def match_names(self) -> set[str]:
+        """Casefolded name and aliases, for matching manual entries."""
+        return {n.casefold().strip() for n in [self.name, *self.aliases]}
+
+    @property
+    def tracking_only(self) -> bool:
+        """No registrar ids: contributes names to tracking, fetches nothing."""
+        return self.orcid is None and self.openalex_id is None
+
+
+@dataclass
+class Config:
+    mailto: str
+    authors: list[AuthorConfig]
+    api_key_env: str = "OPENALEX_API_KEY"
+    include_xpac: bool = False
+    output_path: Path = Path("publications.json")
+    manual_path: Path = Path("manual.yml")
+    overrides_path: Path = Path("overrides.yml")
+    tags_path: Path = Path("tags.yml")
+
+
+def load_yaml(path: Path, expect: type) -> Any:
+    """Parse a YAML file and check its top-level type."""
+    try:
+        with open(path, encoding="utf-8") as fh:
+            data = yaml.safe_load(fh)
+    except OSError as exc:
+        raise ConfigError(f"{path}: {exc.strerror or exc}") from exc
+    except yaml.YAMLError as exc:
+        raise ConfigError(f"{path}: invalid YAML: {exc}") from exc
+    if data is None:
+        data = expect()
+    if not isinstance(data, expect):
+        raise ConfigError(f"{path}: expected a {expect.__name__} at top level")
+    return data
+
+
+def reject_unknown_keys(
+    mapping: dict[str, Any], allowed: set[str], where: str, noun: str = "keys"
+) -> None:
+    unknown = set(mapping) - allowed
+    if unknown:
+        raise ConfigError(f"{where}: unknown {noun}: {', '.join(sorted(unknown))}")
+
+
+def expect_str_list(value: Any, where: str) -> list[str]:
+    if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
+        raise ConfigError(f"{where}: must be a list of strings")
+    return list(value)
+
+
+def _parse_author(entry: Any, path: Path, index: int) -> AuthorConfig:
+    where = f"{path}: authors[{index}]"
+    if not isinstance(entry, dict):
+        raise ConfigError(f"{where}: expected a mapping")
+    reject_unknown_keys(entry, {"name", "orcid", "openalex_id", "aliases"}, where)
+    name = entry.get("name")
+    if not isinstance(name, str) or not name.strip():
+        raise ConfigError(f"{where}: 'name' is required")
+    orcid = entry.get("orcid")
+    if orcid is not None:
+        orcid = normalize_orcid(str(orcid))
+        if not _ORCID_RE.match(orcid):
+            raise ConfigError(f"{where}: invalid ORCID iD {orcid!r}")
+    openalex_id = entry.get("openalex_id")
+    if openalex_id is not None:
+        openalex_id = str(openalex_id).strip()
+        if not _OPENALEX_AUTHOR_RE.match(openalex_id):
+            raise ConfigError(f"{where}: invalid OpenAlex author id {openalex_id!r}")
+    # Neither id is fine: the entry contributes its names to the tracked
+    # flag but resolves and fetches nothing (authors without any registrar
+    # identity, or whose works OpenAlex misassigns to a conflated profile).
+    aliases = expect_str_list(entry.get("aliases", []), f"{where}: 'aliases'")
+    return AuthorConfig(name=name.strip(), orcid=orcid, openalex_id=openalex_id, aliases=aliases)
+
+
+def _section(data: dict[str, Any], key: str, path: Path, allowed: set[str]) -> dict[str, Any]:
+    section = data.get(key) or {}
+    if not isinstance(section, dict):
+        raise ConfigError(f"{path}: '{key}' must be a mapping")
+    reject_unknown_keys(section, allowed, f"{path}: {key}")
+    return section
+
+
+def load_config(path: Path) -> Config:
+    """Load and validate erga.yml; relative paths resolve against its directory."""
+    data = load_yaml(path, dict)
+    base = path.resolve().parent
+    reject_unknown_keys(data, {"mailto", "authors", "openalex", "output", "curation"}, str(path))
+
+    mailto = data.get("mailto")
+    if not isinstance(mailto, str) or "@" not in mailto:
+        raise ConfigError(f"{path}: 'mailto' is required (identifies requests to the APIs)")
+
+    raw_authors = data.get("authors")
+    if not isinstance(raw_authors, list) or not raw_authors:
+        raise ConfigError(f"{path}: 'authors' must be a non-empty list")
+    authors = [_parse_author(entry, path, i) for i, entry in enumerate(raw_authors)]
+
+    openalex = _section(data, "openalex", path, {"api_key_env", "include_xpac"})
+    output = _section(data, "output", path, {"path"})
+    curation = _section(data, "curation", path, {"manual", "overrides", "tags"})
+
+    return Config(
+        mailto=mailto.strip(),
+        authors=authors,
+        api_key_env=str(openalex.get("api_key_env", "OPENALEX_API_KEY")),
+        include_xpac=bool(openalex.get("include_xpac", False)),
+        output_path=base / str(output.get("path", "publications.json")),
+        manual_path=base / str(curation.get("manual", "manual.yml")),
+        overrides_path=base / str(curation.get("overrides", "overrides.yml")),
+        tags_path=base / str(curation.get("tags", "tags.yml")),
+    )

erga/crossref.py

@@ -0,0 +1,50 @@
+"""Crossref venue lookup for the backfill stage.
+
+Crossref still operates a mailto polite pool, so the mailto rides along both
+as a query parameter and in the transport's User-Agent.
+"""
+
+from __future__ import annotations
+
+import time
+import urllib.parse
+from collections.abc import Callable
+
+from erga.http import Pacer, Transport, request_with_retry
+
+CROSSREF_BASE = "https://api.crossref.org/works/"
+
+
+class CrossrefClient:
+    def __init__(
+        self,
+        transport: Transport,
+        *,
+        mailto: str,
+        delay: float = 0.5,
+        sleep: Callable[[float], None] = time.sleep,
+    ) -> None:
+        self._transport = transport
+        self._mailto = mailto
+        self._sleep = sleep
+        self._pacer = Pacer(delay, sleep)
+
+    def venue_for_doi(self, doi: str) -> str | None:
+        """Container title for a bare DOI, or None.
+
+        DataCite DOIs 404 here and are skipped silently, as is any record
+        without a container title. Raises FetchError only when retryable
+        failures persist (the caller stops backfilling, keeping nulls).
+        """
+        self._pacer.wait()
+        url = CROSSREF_BASE + urllib.parse.quote(doi, safe="")
+        response = request_with_retry(
+            self._transport, url, {"mailto": self._mailto}, sleep=self._sleep
+        )
+        if response.status_code != 200 or not isinstance(response.data, dict):
+            return None
+        message = response.data.get("message") or {}
+        titles = message.get("container-title") or []
+        if titles and isinstance(titles[0], str) and titles[0].strip():
+            return titles[0].strip()
+        return None

erga/curation.py

@@ -0,0 +1,300 @@
+"""Curation files: manual entries, overrides, tags.
+
+All three survive every refresh; a missing file means "none". Typos fail
+loudly: curation is the maintainer's reviewable artifact, and a silently
+skipped patch is worse than an aborted run.
+"""
+
+from __future__ import annotations
+
+import copy
+import datetime
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from erga.config import AuthorConfig, expect_str_list, load_yaml, reject_unknown_keys
+from erga.errors import ConfigError
+from erga.model import WORK_TYPES, Work, WorkAuthor, doi_key, doi_url, slugify
+
+MANUAL_KEYS = {"title", "authors", "venue", "year", "date", "doi", "type", "tags", "abstract"}
+PATCH_KEYS = {
+    "title",
+    "authors",
+    "venue",
+    "year",
+    "date",
+    "doi",
+    "type",
+    "cited_by_count",
+    "abstract",
+    "open_access",
+    "tags",
+    "is_retracted",
+}
+
+
+def _parse_authors(value: Any, authors_cfg: list[AuthorConfig], where: str) -> list[WorkAuthor]:
+    """Author strings matched to configured authors by name/alias."""
+    if isinstance(value, str):
+        value = [value]
+    if not isinstance(value, list):
+        raise ConfigError(f"{where}: 'authors' must be a string or a list of strings")
+    parsed = []
+    for name in value:
+        if not isinstance(name, str) or not name.strip():
+            raise ConfigError(f"{where}: 'authors' entries must be non-empty strings")
+        name = name.strip()
+        matched = next((a for a in authors_cfg if name.casefold() in a.match_names()), None)
+        parsed.append(
+            WorkAuthor(
+                name=name,
+                orcid=f"https://orcid.org/{matched.orcid}" if matched and matched.orcid else None,
+                tracked=matched is not None,
+            )
+        )
+    return parsed
+
+
+def _parse_type(value: Any, where: str) -> str:
+    if not isinstance(value, str) or value not in WORK_TYPES:
+        allowed = ", ".join(sorted(WORK_TYPES))
+        raise ConfigError(f"{where}: type {value!r} is not one of: {allowed}")
+    return value
+
+
+def _opt_str(entry: dict[str, Any], key: str) -> str | None:
+    value = entry.get(key)
+    return str(value) if value is not None else None
+
+
+def load_manual(path: Path, authors_cfg: list[AuthorConfig]) -> list[Work]:
+    """Manual records the APIs miss; absent file means none."""
+    if not path.exists():
+        return []
+    entries = load_yaml(path, list)
+    works = []
+    used_ids: set[str] = set()
+    for index, entry in enumerate(entries):
+        where = f"{path}: entry {index + 1}"
+        if not isinstance(entry, dict):
+            raise ConfigError(f"{where}: expected a mapping")
+        reject_unknown_keys(entry, MANUAL_KEYS, where)
+        title = entry.get("title")
+        if not isinstance(title, str) or not title.strip():
+            raise ConfigError(f"{where}: 'title' is required")
+        title = title.strip()
+
+        base_id = "manual-" + slugify(title)
+        work_id, suffix = base_id, 2
+        while work_id in used_ids:
+            work_id, suffix = f"{base_id}-{suffix}", suffix + 1
+        used_ids.add(work_id)
+
+        year = entry.get("year")
+        if year is not None and not isinstance(year, int):
+            raise ConfigError(f"{where}: 'year' must be an integer")
+
+        works.append(
+            Work(
+                id=work_id,
+                title=title,
+                authors=_parse_authors(entry.get("authors", []), authors_cfg, where),
+                year=year,
+                date=_opt_str(entry, "date"),
+                venue=_opt_str(entry, "venue"),
+                type=_parse_type(entry.get("type", "other"), where),
+                doi=doi_url(str(entry["doi"])) if entry.get("doi") else None,
+                abstract=_opt_str(entry, "abstract"),
+                tags=expect_str_list(entry.get("tags", []), f"{where}: 'tags'"),
+                source="manual",
+            )
+        )
+    return works
+
+
+@dataclass
+class Override:
+    where: str
+    match_doi: str | None = None  # doi_key form
+    match_id: str | None = None
+    exclude: bool = False
+    keep_distinct: bool = False
+    patch: dict[str, Any] = field(default_factory=dict)
+    matched: bool = False
+    changed: bool = False
+
+
+def load_overrides(path: Path) -> list[Override]:
+    if not path.exists():
+        return []
+    entries = load_yaml(path, list)
+    overrides = []
+    for index, entry in enumerate(entries):
+        where = f"{path}: entry {index + 1}"
+        if not isinstance(entry, dict):
+            raise ConfigError(f"{where}: expected a mapping")
+        if ("doi" in entry) == ("id" in entry):
+            raise ConfigError(f"{where}: needs exactly one of 'doi' or 'id' to match on")
+        patch = {
+            k: v for k, v in entry.items() if k not in {"doi", "id", "exclude", "keep_distinct"}
+        }
+        reject_unknown_keys(patch, PATCH_KEYS, where, noun="fields")
+        overrides.append(
+            Override(
+                where=where,
+                match_doi=doi_key(str(entry["doi"])) if "doi" in entry else None,
+                match_id=str(entry["id"]) if "id" in entry else None,
+                exclude=bool(entry.get("exclude", False)),
+                keep_distinct=bool(entry.get("keep_distinct", False)),
+                patch=patch,
+            )
+        )
+    return overrides
+
+
+def load_tags(path: Path) -> dict[str, list[str]]:
+    """Mapping of tag name to list of DOI/id references."""
+    if not path.exists():
+        return {}
+    data = load_yaml(path, dict)
+    tags: dict[str, list[str]] = {}
+    for name, refs in data.items():
+        where = f"{path}: tag {name!r}"
+        if not isinstance(name, str):
+            raise ConfigError(f"{where}: tag names must be strings")
+        tags[name] = expect_str_list(refs, where)
+    return tags
+
+
+def _iter_matches(overrides: list[Override], works: list[Work]) -> Iterator[tuple[Override, Work]]:
+    """Pair each override with the works it hits, in override file order.
+
+    Indexes the works once, and flips `matched` in this one place so the
+    stale-override detection cannot drift between callers. Pre-dedup, several
+    works can share a DOI, so the indexes map to lists.
+    """
+    by_doi: dict[str, list[Work]] = {}
+    by_id: dict[str, list[Work]] = {}
+    for work in works:
+        if work.doi_key:
+            by_doi.setdefault(work.doi_key, []).append(work)
+        by_id.setdefault(work.id, []).append(work)
+    for override in overrides:
+        if override.match_doi is not None:
+            hits = by_doi.get(override.match_doi, [])
+        else:
+            hits = by_id.get(override.match_id or "", [])
+        for work in hits:
+            override.matched = True
+            yield override, work
+
+
+def mark_keep_distinct(works: list[Work], overrides: list[Override]) -> None:
+    """Applied before title clustering, ahead of the override patch stage."""
+    keep = [o for o in overrides if o.keep_distinct]
+    for _, work in _iter_matches(keep, works):
+        work.keep_distinct = True
+
+
+# Expected value shapes for the scalar patch fields; a mistyped value must
+# fail as a ConfigError at apply time, not as a TypeError deep in the
+# pipeline (sorting, clustering) where the file/entry context is lost.
+_SCALAR_PATCH_TYPES: dict[str, tuple[str, tuple[type, ...]]] = {
+    "title": ("a string", (str,)),
+    "venue": ("a string or null", (str, type(None))),
+    "year": ("an integer or null", (int, type(None))),
+    "cited_by_count": ("an integer", (int,)),
+    "abstract": ("a string or null", (str, type(None))),
+    "is_retracted": ("a boolean", (bool,)),
+}
+
+
+def _patch_work(
+    work: Work, patch: dict[str, Any], authors_cfg: list[AuthorConfig], where: str
+) -> None:
+    for key, value in patch.items():
+        if key == "authors":
+            work.authors = _parse_authors(value, authors_cfg, where)
+        elif key == "open_access":
+            if isinstance(value, dict):
+                value = value.get("url")
+            work.open_access_url = str(value) if value else None
+        elif key == "doi":
+            work.doi = doi_url(str(value)) if value else None
+        elif key == "type":
+            work.type = _parse_type(value, where)
+        elif key == "tags":
+            work.tags = expect_str_list(value, f"{where}: 'tags'")
+        elif key == "date":
+            # YAML parses unquoted ISO dates as date objects; accept both.
+            if value is not None and not isinstance(value, (str, datetime.date)):
+                raise ConfigError(f"{where}: 'date' must be an ISO date string or null")
+            work.date = str(value) if value is not None else None
+        else:
+            description, types = _SCALAR_PATCH_TYPES[key]
+            if not isinstance(value, types) or (isinstance(value, bool) and bool not in types):
+                raise ConfigError(f"{where}: '{key}' must be {description}")
+            setattr(work, key, value)
+
+
+def apply_overrides(
+    works: list[Work], overrides: list[Override], authors_cfg: list[AuthorConfig]
+) -> tuple[list[Work], int]:
+    """Patch or exclude merged records; returns (kept, excluded_count)."""
+    excluded: set[int] = set()
+    for override, work in _iter_matches(overrides, works):
+        if override.exclude:
+            excluded.add(id(work))
+        elif override.patch:
+            # Compare against the pre-patch record: comparing the override
+            # against the output would be circular, the output already has
+            # the override applied and every entry would look load-bearing.
+            # Deep copy so the check stays honest even if a patch branch
+            # ever mutates a list in place instead of reassigning it.
+            before = copy.deepcopy(work)
+            _patch_work(work, override.patch, authors_cfg, override.where)
+            if work != before:
+                override.changed = True
+    return [w for w in works if id(w) not in excluded], len(excluded)
+
+
+def unmatched_overrides(overrides: list[Override]) -> list[str]:
+    """Locations of overrides that touched nothing (stale DOI or id)."""
+    return [o.where for o in overrides if not o.matched]
+
+
+def redundant_overrides(overrides: list[Override]) -> list[str]:
+    """Locations of field patches that no longer change anything.
+
+    Upstream caught up with the correction. Redundant is information, not
+    an instruction to delete: an override may stay as insurance against the
+    upstream regressing again. keep_distinct-only entries drop out via the
+    empty-patch check; `exclude` needs its explicit guard because an
+    exclude entry carrying patch fields never runs them.
+    """
+    return [o.where for o in overrides if o.matched and o.patch and not o.exclude and not o.changed]
+
+
+def apply_tags(works: list[Work], tags: dict[str, list[str]]) -> list[str]:
+    """Attach curated tags; returns unmatched references for warnings.
+
+    References are matched as DOIs when they look like one (URL or 10.x
+    form), else as record ids. Each record's final tag list is sorted for
+    deterministic output.
+    """
+    by_doi = {w.doi_key: w for w in works if w.doi_key}
+    by_id = {w.id: w for w in works}
+    unmatched = []
+    for name, refs in tags.items():
+        for ref in refs:
+            key = doi_key(ref)
+            work = by_doi.get(key) if key.startswith("10.") else by_id.get(ref)
+            if work is None:
+                unmatched.append(f"tag {name!r}: {ref}")
+            elif name not in work.tags:
+                work.tags.append(name)
+    for work in works:
+        work.tags.sort()
+    return unmatched