bibauthors 0.1.0__py3-none-any.whl

bibauthors/__init__.py

@@ -0,0 +1,108 @@
+"""bibauthors: verify bibliography AUTHOR LISTS against authoritative metadata.
+
+Every citation checker matches on title and DOI. This one checks the author
+list, because the most dangerous hallucinated citation is the chimeric one: a
+real title, a real venue, a real year, and an invented author list. It passes
+a title-matching checker with a clean report and it looks real at a glance.
+
+The short version of the API:
+
+    from bibauthors import verify_bibliography, Config, Verdict
+
+    report = verify_bibliography("refs.bib", Config(offline=True))
+    for result in report.results:
+        if result.verdict == Verdict.DISJOINT:
+            print(result.entry.key, "needs a manual look")
+    raise SystemExit(report.exit_code())
+
+A DISJOINT verdict is a prompt to check an entry by hand. It is not an
+accusation of fabrication.
+
+Author: Manu Nicholas Jacob, Independent Researcher.
+ORCID 0009-0007-6589-6572. MIT licensed.
+"""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"
+__author__ = "Manu Nicholas Jacob"
+__license__ = "MIT"
+
+from .cache import Cache, CacheMiss
+from .config import Config
+from .entry import Entry, extract_arxiv_id, normalize_doi
+from .http import HttpClient
+from .names import (
+    ListComparison,
+    NameComparison,
+    PersonName,
+    compare_author_lists,
+    compare_names,
+    fold,
+    parse_name,
+    parse_names,
+)
+from .parsers import ParseError, load_bibliography, parse_bibtex, parse_csl_json
+from .providers import Provider, ProviderUnavailable, Record, build_providers
+from .report import format_github_annotations, format_json, format_text, to_dict
+from .verdict import DESCRIPTIONS, SEVERITY, Thresholds, Verdict
+from .verify import (
+    SCHEMA_VERSION,
+    EntryResult,
+    Report,
+    Verifier,
+    YearFinding,
+    verify_bibliography,
+    verify_entries,
+    verify_entry,
+)
+
+__all__ = [
+    "__version__",
+    # front doors
+    "verify_bibliography",
+    "verify_entries",
+    "verify_entry",
+    "Verifier",
+    # input
+    "load_bibliography",
+    "parse_bibtex",
+    "parse_csl_json",
+    "ParseError",
+    "Entry",
+    "normalize_doi",
+    "extract_arxiv_id",
+    # name matching
+    "PersonName",
+    "parse_name",
+    "parse_names",
+    "fold",
+    "compare_names",
+    "compare_author_lists",
+    "NameComparison",
+    "ListComparison",
+    # verdicts and tuning
+    "Verdict",
+    "Thresholds",
+    "SEVERITY",
+    "DESCRIPTIONS",
+    "Config",
+    # results
+    "EntryResult",
+    "Report",
+    "YearFinding",
+    "SCHEMA_VERSION",
+    # output
+    "format_text",
+    "format_json",
+    "format_github_annotations",
+    "to_dict",
+    # sources
+    "Provider",
+    "Record",
+    "ProviderUnavailable",
+    "build_providers",
+    "HttpClient",
+    "Cache",
+    "CacheMiss",
+]

bibauthors/cache.py

@@ -0,0 +1,199 @@
+"""On-disk HTTP response cache, and the offline mode that makes CI honest.
+
+Every provider response this tool ever sees arrives through :class:`Cache`.
+That has two consequences worth stating up front.
+
+First, reproducibility. A bibliography checker that talks to four live APIs
+cannot be tested, because the APIs change under you and because a test suite
+that hammers a free public service is rude. So the library can run with
+``offline=True``, in which case a cache miss is a hard error and nothing is
+fetched. The test suite runs that way against recorded fixtures, which means a
+failing test is a real regression rather than someone else's outage.
+
+Second, politeness. OpenAlex serves unauthenticated clients out of a small
+shared daily budget per network address, and Semantic Scholar rate-limits
+hard. Serving a repeated lookup from disk is the difference between a tool
+that can be run in a loop and one that gets blocked.
+
+Cache entry files are named by a hash of the request and written with
+``ensure_ascii=True, sort_keys=True, indent=1`` so that they stay pure ASCII
+and produce readable diffs when they are committed as fixtures.
+"""
+
+from __future__ import annotations
+
+import datetime
+import errno
+import hashlib
+import json
+import os
+from typing import Any, Dict, Optional
+
+
+class CacheMiss(Exception):
+    """Raised when a request is not in the cache.
+
+    In offline mode this propagates out of the HTTP client and the providers
+    turn it into :class:`bibauthors.providers.ProviderUnavailable`, which the
+    verifier reports as "could not ask" rather than "not found". The
+    distinction matters: a missing record is weak evidence about a citation,
+    while a missing cache entry is no evidence at all.
+    """
+
+
+#: Timestamp format used for ``recorded_at``. Second resolution, UTC, with the
+#: literal "Z" suffix, so the value sorts lexicographically and stays ASCII.
+TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
+
+
+def utc_now_stamp() -> str:
+    """Current UTC time as an ISO 8601 second-resolution string."""
+    now = datetime.datetime.now(datetime.timezone.utc)
+    return now.strftime(TIMESTAMP_FORMAT)
+
+
+def canonical_request(url: str, params: Optional[Dict[str, Any]]) -> str:
+    """The string that gets hashed into a cache key.
+
+    ``"GET|<url>|<k>=<v>&<k>=<v>"`` with the parameters sorted by name, so the
+    key does not depend on dict ordering. Only GET is ever cached, and only
+    GET is ever issued: this tool has no code path that writes to any API.
+    """
+    pairs = []
+    for key in sorted((params or {}).keys()):
+        pairs.append("%s=%s" % (key, (params or {})[key]))
+    return "GET|%s|%s" % (url, "&".join(pairs))
+
+
+class Cache(object):
+    """A flat directory of JSON files, one per request.
+
+    Parameters
+    ----------
+    directory:
+        Where the entry files live. Created on first write unless offline.
+    offline:
+        When true, :meth:`get` still reads the directory but :meth:`put`
+        becomes a no-op and nothing is ever created. A miss raises
+        :class:`CacheMiss`.
+    ttl_days:
+        When None (the default) entries never expire and ``recorded_at`` is
+        purely informational. Committed fixtures are meant to be read years
+        after they were recorded, so ignoring the timestamp has to be the
+        default rather than an option someone remembers to set.
+    """
+
+    def __init__(
+        self,
+        directory: str,
+        offline: bool = False,
+        ttl_days: Optional[int] = None,
+    ) -> None:
+        self.directory = str(directory)
+        self.offline = bool(offline)
+        self.ttl_days = ttl_days
+
+    # -- keys and paths ---------------------------------------------------
+
+    def key(self, url: str, params: Optional[Dict[str, Any]] = None) -> str:
+        """sha256 hex digest of the canonical request string."""
+        canonical = canonical_request(url, params)
+        return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+    def path_for(self, url: str, params: Optional[Dict[str, Any]] = None) -> str:
+        """Absolute-or-relative path of the entry file for this request."""
+        return os.path.join(self.directory, self.key(url, params) + ".json")
+
+    # -- reading ----------------------------------------------------------
+
+    def get(
+        self, url: str, params: Optional[Dict[str, Any]] = None
+    ) -> Dict[str, Any]:
+        """Return the stored entry dict, or raise :class:`CacheMiss`.
+
+        The returned dict has the keys ``url``, ``params``, ``status``,
+        ``recorded_at``, ``content_type`` and ``body``. A stored non-200
+        status is a legitimate cache entry: the 404 that CrossRef returns for
+        an arXiv DOI is a fact worth recording, not an error to discard.
+        """
+        path = self.path_for(url, params)
+        try:
+            with open(path, "r", encoding="ascii") as handle:
+                entry = json.load(handle)
+        except IOError as exc:
+            if exc.errno in (errno.ENOENT, errno.EISDIR, errno.ENOTDIR):
+                raise CacheMiss("not cached: %s" % url)
+            raise
+        except ValueError as exc:
+            raise CacheMiss("corrupt cache entry for %s: %s" % (url, exc))
+        if not isinstance(entry, dict) or "body" not in entry:
+            raise CacheMiss("malformed cache entry for %s" % url)
+        if self.is_expired(entry):
+            raise CacheMiss("cache entry expired for %s" % url)
+        return entry
+
+    def has(self, url: str, params: Optional[Dict[str, Any]] = None) -> bool:
+        """True when :meth:`get` would succeed."""
+        try:
+            self.get(url, params)
+        except CacheMiss:
+            return False
+        return True
+
+    def is_expired(self, entry: Dict[str, Any]) -> bool:
+        """TTL check. Always false when ``ttl_days`` is None."""
+        if self.ttl_days is None:
+            return False
+        stamp = entry.get("recorded_at")
+        if not stamp:
+            # No timestamp and a TTL in force: treat as too old to trust.
+            return True
+        try:
+            recorded = datetime.datetime.strptime(stamp, TIMESTAMP_FORMAT)
+        except (TypeError, ValueError):
+            return True
+        recorded = recorded.replace(tzinfo=datetime.timezone.utc)
+        age = datetime.datetime.now(datetime.timezone.utc) - recorded
+        return age > datetime.timedelta(days=self.ttl_days)
+
+    # -- writing ----------------------------------------------------------
+
+    def put(
+        self,
+        url: str,
+        params: Optional[Dict[str, Any]],
+        status: int,
+        body: Any,
+        content_type: str,
+    ) -> None:
+        """Store one response. A no-op in offline mode.
+
+        ``body`` is a parsed JSON value for JSON responses and a plain string
+        for XML (arXiv). Strings are stored as JSON strings, so
+        ``ensure_ascii=True`` escapes any non-ASCII character in the XML and
+        the file on disk stays pure ASCII.
+        """
+        if self.offline:
+            return
+        entry = {
+            "url": url,
+            "params": dict(params or {}),
+            "status": int(status),
+            "recorded_at": utc_now_stamp(),
+            "content_type": content_type or "",
+            "body": body,
+        }
+        self.write_entry(self.path_for(url, params), entry)
+
+    def write_entry(self, path: str, entry: Dict[str, Any]) -> None:
+        """Serialize one entry to ``path`` in the committed fixture format."""
+        if self.offline:
+            return
+        directory = os.path.dirname(path)
+        if directory and not os.path.isdir(directory):
+            os.makedirs(directory)
+        temporary = path + ".tmp"
+        with open(temporary, "w", encoding="ascii", newline="\n") as handle:
+            json.dump(entry, handle, ensure_ascii=True, sort_keys=True, indent=1)
+            handle.write("\n")
+        os.replace(temporary, path)

bibauthors/cli.py

@@ -0,0 +1,297 @@
+"""Command line front end.
+
+Exit codes:
+
+    0  clean: every entry MATCH or MATCH_WITH_VARIANTS
+    1  author mismatch found (DISJOINT or PARTIAL_OVERLAP, plus whatever the
+       config promotes)
+    2  no mismatch, but at least one entry could not be verified
+    3  usage or input error
+
+Code 1 wins over code 2 when both apply.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from typing import List, Optional, Sequence
+
+from . import __version__
+from .config import Config, parse_threshold_overrides
+from .parsers import ParseError, load_bibliography
+from .report import format_github_annotations, format_json, format_text
+from .verdict import Thresholds
+from .verify import Verifier
+
+EXIT_CLEAN = 0
+EXIT_MISMATCH = 1
+EXIT_UNVERIFIABLE = 2
+EXIT_USAGE = 3
+
+_EPILOG = """\
+exit codes:
+  0  clean
+  1  author mismatch found
+  2  no mismatch, but some entries could not be verified
+  3  usage or input error
+
+A DISJOINT verdict is a prompt to check the entry by hand. It is not an
+accusation of fabrication.
+"""
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="bibauthors",
+        description=(
+            "Verify the author list of every bibliography entry against "
+            "authoritative metadata."
+        ),
+        epilog=_EPILOG,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "paths",
+        nargs="*",
+        help="bibliography files (.bib or CSL-JSON). Reads stdin when absent.",
+    )
+    parser.add_argument("--version", action="version", version="bibauthors " + __version__)
+    parser.add_argument(
+        "--format",
+        dest="fmt",
+        choices=("bibtex", "csljson"),
+        help="input format (sniffed by default)",
+    )
+
+    out = parser.add_argument_group("output")
+    out.add_argument(
+        "--json",
+        dest="json_path",
+        metavar="PATH",
+        help="write the machine-readable report here ('-' for stdout)",
+    )
+    out.add_argument(
+        "--quiet", action="store_true", help="suppress the human report"
+    )
+    out.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        help="show clean entries and every provider note",
+    )
+    out.add_argument(
+        "--github",
+        action="store_true",
+        help="also emit GitHub Actions annotations on stdout",
+    )
+
+    net = parser.add_argument_group("sources and network")
+    net.add_argument(
+        "--offline",
+        action="store_true",
+        help="use only the on-disk cache; never touch the network",
+    )
+    net.add_argument(
+        "--cache-dir", metavar="DIR", help="where the HTTP cache lives"
+    )
+    net.add_argument(
+        "--providers",
+        metavar="LIST",
+        help="comma-separated subset of: crossref,arxiv,openalex,semanticscholar",
+    )
+    net.add_argument(
+        "--min-interval",
+        type=float,
+        metavar="SECONDS",
+        help="minimum delay between requests to one host (default 1.0)",
+    )
+    net.add_argument(
+        "--timeout", type=float, metavar="SECONDS", help="per-request timeout"
+    )
+    net.add_argument(
+        "--ask-every-provider",
+        action="store_true",
+        help=(
+            "do not stop at the first confident identifier hit. Slower and "
+            "spends more of someone else's rate limit; useful for auditing "
+            "disagreement between sources."
+        ),
+    )
+
+    tune = parser.add_argument_group("thresholds and policy")
+    tune.add_argument(
+        "--config", metavar="PATH", help="TOML config file (default: discovered)"
+    )
+    tune.add_argument(
+        "--set",
+        dest="overrides",
+        action="append",
+        default=[],
+        metavar="NAME=VALUE",
+        help="override one threshold; repeatable. See --list-thresholds.",
+    )
+    tune.add_argument(
+        "--list-thresholds",
+        action="store_true",
+        help="print every threshold with its default and exit",
+    )
+    tune.add_argument(
+        "--strict",
+        action="store_true",
+        help="also fail on ORDER_DIFFERS and TRUNCATED",
+    )
+    tune.add_argument(
+        "--fail-on-unverifiable",
+        action="store_true",
+        help="treat exit code 2 as a failure (returns 1)",
+    )
+    tune.add_argument(
+        "--no-year-check",
+        action="store_true",
+        help="skip the date comparison entirely",
+    )
+    tune.add_argument(
+        "--ignore-key",
+        dest="ignore_keys",
+        action="append",
+        default=[],
+        metavar="KEY",
+        help="skip this citation key; repeatable",
+    )
+    tune.add_argument(
+        "--max-entries", type=int, metavar="N", help="stop after N entries"
+    )
+    return parser
+
+
+def _print_thresholds(stream) -> None:
+    import dataclasses
+
+    defaults = Thresholds()
+    stream.write("thresholds (use --set NAME=VALUE):\n\n")
+    doc = (Thresholds.__doc__ or "").splitlines()
+    descriptions = {}
+    current = None
+    for line in doc:
+        stripped = line.strip()
+        if not stripped:
+            continue
+        if not line.startswith("        ") and stripped.replace("_", "").replace(
+            " / ", ""
+        ).isalnum():
+            current = stripped
+            descriptions[current] = []
+        elif current:
+            descriptions[current].append(stripped)
+    for field in dataclasses.fields(defaults):
+        stream.write("  %-28s %s\n" % (field.name, getattr(defaults, field.name)))
+        for key, lines in descriptions.items():
+            if field.name in [part.strip() for part in key.split("/")]:
+                stream.write("      %s\n" % " ".join(lines))
+                break
+
+
+def _read_inputs(paths: Sequence[str], fmt: Optional[str]):
+    entries = []
+    sources: List[str] = []
+    if not paths:
+        text = sys.stdin.read()
+        if not text.strip():
+            raise ParseError("no input: give a file path or pipe a bibliography in")
+        entries.extend(load_bibliography(text, fmt=fmt, is_text=True))
+        sources.append("<stdin>")
+        return entries, sources
+    for path in paths:
+        if not os.path.exists(path):
+            raise ParseError("no such file: %s" % path)
+        entries.extend(load_bibliography(path, fmt=fmt))
+        sources.append(path)
+    return entries, sources
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    if args.list_thresholds:
+        _print_thresholds(sys.stdout)
+        return EXIT_CLEAN
+
+    try:
+        base = (
+            Config.from_toml(args.config)
+            if args.config
+            else Config.discover(args.paths[0] if args.paths else ".")
+        )
+    except (OSError, ValueError) as exc:
+        sys.stderr.write("bibauthors: config error: %s\n" % exc)
+        return EXIT_USAGE
+
+    try:
+        overrides = parse_threshold_overrides(args.overrides)
+        if args.strict:
+            overrides.setdefault("promote_order_differs", True)
+            overrides.setdefault("promote_truncated", True)
+        providers = (
+            [p.strip() for p in args.providers.split(",") if p.strip()]
+            if args.providers
+            else None
+        )
+        config = base.with_overrides(
+            offline=True if args.offline else None,
+            cache_dir=args.cache_dir,
+            providers=providers,
+            min_interval=args.min_interval,
+            timeout=args.timeout,
+            max_entries=args.max_entries,
+            check_years=False if args.no_year_check else None,
+            stop_after_confident=False if args.ask_every_provider else None,
+            ignore_keys=args.ignore_keys or None,
+            fail_on_unverifiable=True if args.fail_on_unverifiable else None,
+            **overrides
+        )
+    except ValueError as exc:
+        sys.stderr.write("bibauthors: %s\n" % exc)
+        return EXIT_USAGE
+
+    try:
+        entries, sources = _read_inputs(args.paths, args.fmt)
+    except ParseError as exc:
+        sys.stderr.write("bibauthors: %s\n" % exc)
+        return EXIT_USAGE
+    except OSError as exc:
+        sys.stderr.write("bibauthors: %s\n" % exc)
+        return EXIT_USAGE
+
+    if not entries:
+        sys.stderr.write("bibauthors: no entries found in the input\n")
+        return EXIT_USAGE
+
+    verifier = Verifier(config=config)
+    report = verifier.verify_entries(entries, sources=sources)
+
+    if not args.quiet:
+        sys.stdout.write(format_text(report, verbose=args.verbose))
+    if args.github:
+        sys.stdout.write(format_github_annotations(report))
+    if args.json_path:
+        payload = format_json(report)
+        if args.json_path == "-":
+            sys.stdout.write(payload)
+        else:
+            directory = os.path.dirname(os.path.abspath(args.json_path))
+            if directory and not os.path.isdir(directory):
+                os.makedirs(directory, exist_ok=True)
+            with open(args.json_path, "w", encoding="ascii") as handle:
+                handle.write(payload)
+
+    code = report.exit_code()
+    if code == EXIT_UNVERIFIABLE and config.fail_on_unverifiable:
+        return EXIT_MISMATCH
+    return code
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())