risforge 0.1.0__py3-none-any.whl

risforge/__init__.py

@@ -0,0 +1,38 @@
+"""risforge: clean, deduplicate, and enrich RIS bibliographic files.
+
+Typical usage::
+
+    from risforge import clean_ris_file, RisEnricher, run_pipeline
+
+    # Clean + deduplicate only
+    records, errors = clean_ris_file("raw.ris", "clean.ris")
+
+    # Enrich only
+    enricher = RisEnricher(email="you@example.com")
+    stats = enricher.enrich_file("clean.ris", "enriched.ris")
+
+    # Both, in one call
+    result = run_pipeline(
+        "raw.ris", "clean.ris", "enriched.ris", email="you@example.com"
+    )
+
+See the ``risforge`` console script (``risforge --help``) for the
+equivalent command-line interface.
+"""
+
+from risforge.cleaning import clean_ris_file, process_ris_file
+from risforge.enrichment import RisEnricher
+from risforge.exceptions import RisForgeError, RisParsingError
+from risforge.pipeline import PipelineResult, run_pipeline
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "clean_ris_file",
+    "process_ris_file",
+    "RisEnricher",
+    "run_pipeline",
+    "PipelineResult",
+    "RisForgeError",
+    "RisParsingError",
+]

risforge/cleaning.py

@@ -0,0 +1,318 @@
+"""Clean, normalize, and deduplicate RIS bibliographic records.
+
+This module contains no behavioral changes from the original
+``clean_ris.py`` script: the union-find-based clustering, the DOI and
+title+author matching heuristics, and the "most complete record wins,
+then merge in whatever the others have that it's missing" merge
+strategy are all preserved exactly. What changed is purely structural:
+type hints, docstrings, PEP 8 formatting, and turning the script's
+module-level ``logging.basicConfig()`` call into a plain
+``logging.getLogger(__name__)`` (a library should never configure the
+root logger on import -- see :mod:`risforge.cli` for where that
+configuration now happens, only for CLI usage).
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import unicodedata
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+import rispy
+import rispy.writer
+
+logger = logging.getLogger(__name__)
+
+RisRecord = dict[str, Any]
+
+
+class _CleanRisWriter(rispy.writer.RisWriter):
+    """Writer that strips rispy's default "1.", "2." record numbering.
+
+    Enforces ``\\r\\n`` line endings for compatibility with reference
+    managers (EndNote, Zotero, etc.) that expect the RIS spec's
+    canonical newline convention.
+    """
+
+    NEWLINE = "\r\n"
+
+    def set_header(self, count: int) -> str:
+        return ""
+
+
+def normalize_title(title: str | None) -> str:
+    """Normalize a title for deduplication comparison.
+
+    Lowercases, strips diacritics, and removes punctuation so that
+    casing and formatting differences across citation exports don't
+    produce false-negative duplicate matches.
+    """
+    if not isinstance(title, str) or not title:
+        return ""
+
+    normalized = unicodedata.normalize("NFKD", title).lower()
+    normalized = re.sub(r"[^a-z0-9\s]", "", normalized)
+    return re.sub(r"\s+", " ", normalized).strip()
+
+
+def extract_first_author(author_list: list[str] | None) -> str:
+    """Extract a normalized "last name + initials" key for the first author.
+
+    Used as half of the title+author composite deduplication key, since
+    full author-list formatting is rarely consistent across sources but
+    the first author's surname usually is.
+    """
+    if not author_list or not isinstance(author_list[0], str):
+        return ""
+
+    author_parts = author_list[0].split(",")
+    last_name = author_parts[0].strip()
+    initials = ""
+
+    if len(author_parts) > 1:
+        tokens = re.split(r"[\s.\-]+", author_parts[1].strip())
+        initials = "".join(token[0] for token in tokens if token)
+
+    name_str = f"{last_name} {initials}"
+    name_str = unicodedata.normalize("NFKD", name_str).lower()
+    name_str = re.sub(r"[^a-z0-9\s]", "", name_str)
+    return re.sub(r"\s+", " ", name_str).strip()
+
+
+def normalize_doi(doi: str | None) -> str:
+    """Normalize a DOI by stripping URL prefixes, whitespace, and trailing punctuation.
+
+    DOIs are the strongest deduplication key available, but sources
+    frequently prepend ``https://doi.org/`` or ``doi:`` in ways that
+    would otherwise defeat exact matching.
+    """
+    if not isinstance(doi, str) or not doi:
+        return ""
+
+    cleaned = doi.strip().lower()
+    cleaned = re.sub(r"^(https?://)?(dx\.)?doi\.org/|^doi:", "", cleaned)
+    return re.sub(r"[\s.,;:]+$", "", cleaned)
+
+
+def count_fields(record: RisRecord) -> int:
+    """Count populated fields in a record.
+
+    Used to pick the "most complete" record in a duplicate cluster as
+    the merge base.
+    """
+    count = 0
+    for key, value in record.items():
+        if key == "unknown_tag":
+            for unknown_values in value.values():
+                count += sum(bool(item) for item in unknown_values)
+        elif isinstance(value, list):
+            count += sum(bool(item) for item in value)
+        elif value:
+            count += 1
+    return count
+
+
+def merge_cluster(cluster: list[RisRecord]) -> RisRecord:
+    """Merge a cluster of duplicate records into one, losing no data.
+
+    The most complete record (by :func:`count_fields`) is used as the
+    base; every other record in the cluster then supplements it with
+    any fields or list items it's missing.
+    """
+    if len(cluster) == 1:
+        return cluster[0]
+
+    best_record = max(cluster, key=count_fields)
+    merged = dict(best_record)
+    merged["unknown_tag"] = defaultdict(list, merged.get("unknown_tag", {}))
+
+    for record in cluster:
+        if record is best_record:
+            continue
+
+        for key, value in record.items():
+            if key == "unknown_tag":
+                for unknown_key, unknown_values in value.items():
+                    for item in unknown_values:
+                        if item not in merged["unknown_tag"][unknown_key]:
+                            merged["unknown_tag"][unknown_key].append(item)
+            elif key not in merged:
+                merged[key] = value
+            elif isinstance(merged[key], list) and isinstance(value, list):
+                for item in value:
+                    if item not in merged[key]:
+                        merged[key].append(item)
+            elif (
+                isinstance(merged[key], str)
+                and not merged[key]
+                and isinstance(value, str)
+            ):
+                merged[key] = value
+
+    return merged
+
+
+class _RecordUnionFind:
+    """Disjoint-set structure grouping records into duplicate clusters.
+
+    Two records may only be unioned if neither has a normalized DOI
+    that conflicts with the other's -- this prevents the fuzzy
+    title+author heuristic from ever merging two records that carry
+    different, explicit DOIs.
+    """
+
+    def __init__(self, records: list[RisRecord]) -> None:
+        self._parents = list(range(len(records)))
+        self._root_dois = [normalize_doi(record.get("doi", "")) for record in records]
+
+    def find(self, node_index: int) -> int:
+        root = node_index
+        while self._parents[root] != root:
+            root = self._parents[root]
+
+        current = node_index
+        while current != root:
+            nxt = self._parents[current]
+            self._parents[current] = root
+            current = nxt
+
+        return root
+
+    def union(self, node_i: int, node_j: int) -> bool:
+        root_i = self.find(node_i)
+        root_j = self.find(node_j)
+
+        if root_i == root_j:
+            return False
+
+        doi_i = self._root_dois[root_i]
+        doi_j = self._root_dois[root_j]
+
+        if doi_i and doi_j and doi_i != doi_j:
+            return False
+
+        self._parents[root_i] = root_j
+        self._root_dois[root_j] = doi_i or doi_j
+        return True
+
+
+def clean_ris_file(
+    input_path: str | Path, output_path: str | Path
+) -> tuple[list[RisRecord], list[tuple[int, str]]]:
+    """Clean, deduplicate, and write out a RIS file.
+
+    Parses ``input_path`` block-by-block (so a single malformed record
+    doesn't take down the whole parse), deduplicates first on exact
+    normalized DOI, then on a normalized title+first-author composite
+    key, merges each resulting cluster into a single complete record,
+    and writes the result to ``output_path``.
+
+    Args:
+        input_path: Path to the source ``.ris`` file.
+        output_path: Path the cleaned, deduplicated ``.ris`` file is
+            written to.
+
+    Returns:
+        A ``(records, errors)`` tuple: the final deduplicated records
+        (as rispy record dicts), and a list of ``(block_number,
+        message)`` pairs for any blocks that failed to parse.
+
+    Raises:
+        FileNotFoundError: If ``input_path`` does not exist.
+    """
+    input_path = Path(input_path)
+    output_path = Path(output_path)
+
+    if not input_path.exists():
+        raise FileNotFoundError(f"Input file '{input_path}' not found.")
+
+    text = input_path.read_text(encoding="utf-8")
+
+    blocks = re.split(r"(?m)^TY\s+-", text)
+    records: list[RisRecord] = []
+    errors: list[tuple[int, str]] = []
+
+    for index, block in enumerate(blocks):
+        if not block.strip():
+            continue
+
+        block_text = f"TY  -{block}"
+        try:
+            parsed_records = rispy.loads(block_text)
+            if not parsed_records:
+                errors.append((index + 1, "Empty parse result (malformed record)"))
+            else:
+                records.extend(parsed_records)
+        except (ValueError, TypeError, KeyError, AttributeError) as error:
+            errors.append((index + 1, str(error)))
+
+    if errors:
+        logger.warning("Encountered %d malformed record block(s), skipped.", len(errors))
+
+    total_records = len(records)
+    if total_records == 0:
+        logger.warning("No valid records found to process.")
+        return [], errors
+
+    union_find = _RecordUnionFind(records)
+
+    doi_map: dict[str, int] = {}
+    doi_duplicates_removed = 0
+
+    # Pass 1: deduplicate by exact DOI match.
+    for index, record in enumerate(records):
+        doi = normalize_doi(record.get("doi", ""))
+        if doi:
+            if doi in doi_map:
+                if union_find.union(index, doi_map[doi]):
+                    doi_duplicates_removed += 1
+            else:
+                doi_map[doi] = index
+
+    composite_key_map: dict[str, int] = {}
+    title_author_duplicates_removed = 0
+
+    # Pass 2: deduplicate by composite title + first-author key (fallback heuristic).
+    for index, record in enumerate(records):
+        norm_title = normalize_title(record.get("title", ""))
+        norm_author = extract_first_author(record.get("authors", []))
+
+        if norm_title and norm_author:
+            composite_key = f"{norm_title}|{norm_author}"
+
+            if composite_key in composite_key_map:
+                if union_find.union(index, composite_key_map[composite_key]):
+                    title_author_duplicates_removed += 1
+                    composite_key_map[composite_key] = union_find.find(index)
+            else:
+                composite_key_map[composite_key] = union_find.find(index)
+
+    clusters: dict[int, list[int]] = defaultdict(list)
+    for index in range(total_records):
+        clusters[union_find.find(index)].append(index)
+
+    final_records = [
+        merge_cluster([records[i] for i in indices]) for indices in clusters.values()
+    ]
+
+    total_duplicates_removed = doi_duplicates_removed + title_author_duplicates_removed
+
+    logger.info("--- SUMMARY ---")
+    logger.info("Input records successfully parsed: %d", len(records))
+    logger.info("Malformed records skipped: %d", len(errors))
+    logger.info("Total duplicates removed: %d", total_duplicates_removed)
+    logger.info("Final unique records: %d", len(final_records))
+    logger.info("Output saved to: %s", output_path)
+    logger.info("---------------")
+
+    out_text = rispy.dumps(final_records, implementation=_CleanRisWriter)
+    output_path.write_text(out_text, encoding="utf-8")
+
+    return final_records, errors
+
+
+# Backward-compatible alias for the original script's public function name.
+process_ris_file = clean_ris_file

risforge/cli.py

@@ -0,0 +1,197 @@
+"""Command-line interface for risforge.
+
+This is where the original scripts' "just run me directly" behavior
+now lives, generalized into three subcommands so a single installed
+``risforge`` command replaces having three separate scripts each
+hardcoding its own paths:
+
+    risforge clean input.ris output_clean.ris
+    risforge enrich input.ris output_enriched.ris --email you@example.com
+    risforge pipeline input.ris --email you@example.com
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+from pathlib import Path
+
+from risforge.cleaning import clean_ris_file
+from risforge.enrichment import RisEnricher
+from risforge.pipeline import run_pipeline
+
+logger = logging.getLogger("risforge")
+
+
+def _configure_logging(verbose: bool) -> None:
+    """Configure logging for CLI usage only.
+
+    Library modules never do this themselves (see the module
+    docstrings in :mod:`risforge.cleaning` and
+    :mod:`risforge.enrichment`) -- only the CLI entry point, which is
+    the one context where it's actually risforge's call to make.
+    """
+    logging.basicConfig(
+        level=logging.DEBUG if verbose else logging.INFO,
+        format="%(levelname)s: %(message)s",
+    )
+
+
+def _add_common_args(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "-v", "--verbose", action="store_true", help="Enable debug-level logging."
+    )
+
+
+def _cmd_clean(args: argparse.Namespace) -> int:
+    try:
+        records, errors = clean_ris_file(args.input, args.output)
+    except FileNotFoundError as error:
+        logger.error("%s", error)
+        return 1
+    except (OSError, ValueError, RuntimeError) as error:
+        logger.error("Cleaning failed: %s", error)
+        return 1
+
+    logger.info("Successfully processed %d unique records.", len(records))
+    return 1 if errors and args.strict else 0
+
+
+def _cmd_enrich(args: argparse.Namespace) -> int:
+    try:
+        enricher = RisEnricher(email=args.email, cache_name=args.cache_name)
+        stats = enricher.enrich_file(
+            input_path=args.input,
+            output_path=args.output,
+            fail_report_path=args.fail_report,
+        )
+    except (OSError, ValueError, RuntimeError) as error:
+        logger.error("Enrichment failed: %s", error)
+        return 1
+
+    logger.info("Enriched %d/%d records.", stats["enriched"], stats["processed"])
+    return 0
+
+
+def _cmd_pipeline(args: argparse.Namespace) -> int:
+    dedup_path = args.dedup_output or _default_sibling(args.input, "_clean")
+    enriched_path = args.output or _default_sibling(args.input, "_enriched")
+
+    try:
+        result = run_pipeline(
+            input_path=args.input,
+            dedup_path=dedup_path,
+            enriched_path=enriched_path,
+            email=args.email,
+            fail_report_path=args.fail_report,
+        )
+    except FileNotFoundError as error:
+        logger.error("Input file missing: %s", error)
+        return 1
+    except (OSError, ValueError, RuntimeError) as error:
+        logger.error("Pipeline failed: %s", error)
+        return 1
+
+    logger.info(
+        "Pipeline finished: %d cleaned records, %d/%d enriched.",
+        result.cleaned_record_count,
+        result.enrichment_stats.get("enriched", 0),
+        result.enrichment_stats.get("processed", 0),
+    )
+    return 0
+
+
+def _default_sibling(input_path: str | Path, suffix: str) -> Path:
+    path = Path(input_path)
+    return path.with_name(f"{path.stem}{suffix}{path.suffix}")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Build the top-level argument parser (exposed for testing/docs)."""
+    parser = argparse.ArgumentParser(
+        prog="risforge",
+        description="Clean, deduplicate, and enrich RIS bibliographic files.",
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    clean_parser = subparsers.add_parser(
+        "clean", help="Deduplicate and normalize a RIS file."
+    )
+    clean_parser.add_argument("input", help="Input RIS file path.")
+    clean_parser.add_argument("output", help="Output cleaned RIS file path.")
+    clean_parser.add_argument(
+        "--strict",
+        action="store_true",
+        help="Exit with a non-zero status if any records failed to parse.",
+    )
+    _add_common_args(clean_parser)
+    clean_parser.set_defaults(func=_cmd_clean)
+
+    enrich_parser = subparsers.add_parser(
+        "enrich", help="Enrich a RIS file with metadata from scholarly APIs."
+    )
+    enrich_parser.add_argument("input", help="Input RIS file path.")
+    enrich_parser.add_argument("output", help="Output enriched RIS file path.")
+    enrich_parser.add_argument(
+        "--email",
+        required=True,
+        help="Contact email for Crossref/OpenAlex/Unpaywall polite-pool access.",
+    )
+    enrich_parser.add_argument(
+        "--fail-report",
+        dest="fail_report",
+        default="failed_records.json",
+        help="Where to write unresolved-DOI records (default: %(default)s).",
+    )
+    enrich_parser.add_argument(
+        "--cache-name",
+        dest="cache_name",
+        default=".api_cache",
+        help="Base filename for the on-disk HTTP response cache.",
+    )
+    _add_common_args(enrich_parser)
+    enrich_parser.set_defaults(func=_cmd_enrich)
+
+    pipeline_parser = subparsers.add_parser(
+        "pipeline", help="Run clean then enrich in one step."
+    )
+    pipeline_parser.add_argument("input", help="Input RIS file path.")
+    pipeline_parser.add_argument(
+        "--email",
+        required=True,
+        help="Contact email for Crossref/OpenAlex/Unpaywall polite-pool access.",
+    )
+    pipeline_parser.add_argument(
+        "--dedup-output",
+        dest="dedup_output",
+        default=None,
+        help="Path for the intermediate cleaned file (default: <input>_clean.ris).",
+    )
+    pipeline_parser.add_argument(
+        "--output",
+        default=None,
+        help="Path for the final enriched file (default: <input>_enriched.ris).",
+    )
+    pipeline_parser.add_argument(
+        "--fail-report",
+        dest="fail_report",
+        default="failed_records.json",
+        help="Where to write unresolved-DOI records (default: %(default)s).",
+    )
+    _add_common_args(pipeline_parser)
+    pipeline_parser.set_defaults(func=_cmd_pipeline)
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> None:
+    """CLI entry point (registered as the ``risforge`` console script)."""
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    _configure_logging(args.verbose)
+    sys.exit(args.func(args))
+
+
+if __name__ == "__main__":
+    main()

risforge/enrichment.py

@@ -0,0 +1,420 @@
+"""Enrich RIS citation records by aggregating metadata from scholarly APIs.
+
+Behavior is unchanged from the original ``enrich_ris.py``: the same
+four providers (Crossref, OpenAlex, Semantic Scholar, Unpaywall) are
+queried in the same order, existing fields are still never
+overwritten (only gaps are filled), and the same retry/backoff and
+7-day response cache are used.
+
+Two structural changes were made, both in service of the same goal --
+letting this be imported as a library, not just run as a script:
+
+1. The module no longer attaches a ``StreamHandler`` to its logger at
+   import time. A library should never configure logging as a side
+   effect of being imported; the calling application (or
+   :mod:`risforge.cli`) decides where log records go.
+2. :class:`RisEnricher` now accepts an optional pre-built ``session``,
+   so tests (and callers with their own HTTP session/retry policy) can
+   inject one instead of always getting a fresh
+   ``requests_cache.CachedSession`` pointed at a file on disk.
+
+One genuine bug from the original script *was* fixed here, not just
+restructured: the original ``RIS_MAPPING`` (and ``extract_doi``)
+addressed record fields by RIS tag mnemonic (``"DO"``, ``"TI"``,
+``"T2"``, ...). But ``rispy`` -- both the parser and the writer --
+represents records by *logical* field name (``"doi"``, ``"title"``,
+``"secondary_title"``, ...), not by tag. Writing to ``record["DO"]``
+therefore silently created a key rispy's writer doesn't recognize, and
+``rispy.dump()`` dropped it on write (emitting a ``UserWarning: label
+`DO` not exported`` in the process). In practice, every field the
+original enricher "added" never actually made it into the output
+file, and ``extract_doi()`` could never find a DOI that was already
+present on the record either, since it looked for ``"DO"`` instead of
+``"doi"``. :data:`RISPY_FIELD_MAP` below uses rispy's real field
+names, and :meth:`RisEnricher.extract_doi` reads ``"doi"``/``"urls"``.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+import time
+from difflib import SequenceMatcher
+from pathlib import Path
+from typing import Any
+from urllib.parse import quote
+
+import requests
+import requests_cache
+import rispy
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+TITLE_MATCH_THRESHOLD = 0.90
+CACHE_EXPIRE_DAYS = 7
+
+logger = logging.getLogger(__name__)
+
+# Maps our enrichment payload's logical keys to rispy's actual record
+# field names (NOT RIS tag mnemonics -- see the module docstring).
+RISPY_FIELD_MAP: dict[str, str] = {
+    "doi": "doi",
+    "title": "title",
+    "authors": "authors",
+    "journal": "secondary_title",  # rispy has no single canonical "journal" field;
+    "publisher": "publisher",  # secondary_title (T2) is where journal names live
+    "year": "year",  # in the vast majority of real-world RIS exports.
+    "date": "date",
+    "volume": "volume",
+    "issue": "number",
+    "start_page": "start_page",
+    "end_page": "end_page",
+    "abstract": "abstract",
+    "issn": "issn",
+    "url": "urls",  # rispy stores URLs as a list (RIS "UR" is a list-type tag).
+    "pdf_url": "file_attachments1",  # RIS "L1", used for PDF links in Zotero/EndNote.
+    "keywords": "keywords",
+}
+
+# rispy fields that are always lists, regardless of what the source API gives us.
+_LIST_TYPE_FIELDS = {"authors", "keywords", "urls"}
+
+
+def _build_default_session(cache_name: str) -> requests.Session:
+    """Build the package's default cached, retrying HTTP session."""
+    session = requests_cache.CachedSession(
+        cache_name,
+        expire_after=CACHE_EXPIRE_DAYS * 86400,
+        allowable_codes=[200, 404],
+    )
+    retries = Retry(
+        total=5,
+        backoff_factor=1,
+        status_forcelist=[429, 500, 502, 503, 504],
+        allowed_methods=["GET"],
+    )
+    session.mount("https://", HTTPAdapter(max_retries=retries))
+    return session
+
+
+class RisEnricher:
+    """Enriches RIS citation records with metadata from scholarly APIs.
+
+    Args:
+        email: Contact email sent to Crossref/OpenAlex/Unpaywall as
+            required by their "polite pool" usage terms.
+        cache_name: Base filename for the on-disk HTTP response cache
+            (only used when ``session`` is not provided).
+        session: Optional pre-built ``requests.Session`` (or
+            ``requests_cache.CachedSession``). Mainly useful for
+            testing -- pass a mocked session to avoid real network
+            calls. When omitted, a cached, retrying session is built
+            automatically.
+    """
+
+    def __init__(
+        self,
+        email: str,
+        cache_name: str | Path = ".api_cache",
+        session: requests.Session | None = None,
+    ) -> None:
+        self.email = email
+        self.stats: dict[str, Any] = {
+            "processed": 0,
+            "enriched": 0,
+            "failed": 0,
+            "api_calls": {
+                "crossref": 0,
+                "openalex": 0,
+                "semanticscholar": 0,
+                "unpaywall": 0,
+            },
+        }
+        self.failed_records: list[dict[str, Any]] = []
+        self.session = session or _build_default_session(str(cache_name))
+        self.session.headers.update(
+            {"User-Agent": f"risforge/1.0 (mailto:{self.email})"}
+        )
+
+    # --- Core identification -------------------------------------------------
+
+    def extract_doi(self, record: dict[str, Any]) -> str | None:
+        """Extract and validate a DOI from a rispy record's doi/urls fields."""
+        doi = record.get("doi", "")
+
+        if not doi:
+            urls = record.get("urls", "")
+            doi = " ".join(urls) if isinstance(urls, list) else urls
+
+        if not isinstance(doi, str):
+            doi = str(doi)
+
+        match = re.search(r"(10\.\d{4,9}/[-._;()/:A-Z0-9]+)", doi, re.IGNORECASE)
+        return match.group(1).lower() if match else None
+
+    def string_similarity(self, source_text: str, target_text: str) -> float:
+        """Fuzzy-match ratio between two strings (used for title matching)."""
+        if not source_text or not target_text:
+            return 0.0
+        return SequenceMatcher(None, source_text.lower(), target_text.lower()).ratio()
+
+    def resolve_doi_by_title(self, title: str) -> str | None:
+        """Fall back to a Crossref title search when a record has no DOI."""
+        if not title:
+            return None
+
+        url = (
+            f"https://api.crossref.org/works?query.title={quote(title)}"
+            f"&select=DOI,title&rows=3&mailto={self.email}"
+        )
+        try:
+            response = self.session.get(url, timeout=10)
+            self.stats["api_calls"]["crossref"] += 1
+
+            if response.status_code == 200:
+                items = response.json().get("message", {}).get("items", [])
+                for item in items:
+                    api_titles = item.get("title", [""])
+                    api_title = api_titles[0] if api_titles else ""
+
+                    if self.string_similarity(title, api_title) >= TITLE_MATCH_THRESHOLD:
+                        doi_value = item.get("DOI")
+                        if doi_value:
+                            return str(doi_value).lower()
+
+        except (requests.RequestException, ValueError, KeyError) as error:
+            logger.warning("Title resolution failed for '%s': %s", title, error)
+
+        return None
+
+    # --- API integrations ------------------------------------------------------
+
+    def fetch_crossref_metadata(self, doi: str) -> dict[str, Any]:
+        """Fetch authoritative bibliographic metadata from Crossref."""
+        url = f"https://api.crossref.org/works/{quote(doi)}?mailto={self.email}"
+        try:
+            response = self.session.get(url, timeout=10)
+            self.stats["api_calls"]["crossref"] += 1
+
+            if response.status_code == 200:
+                payload = response.json().get("message", {})
+
+                titles = payload.get("title", [""])
+                article_title = titles[0] if titles else ""
+
+                authors_list = []
+                for author_data in payload.get("author", []):
+                    family = author_data.get("family", "")
+                    given = author_data.get("given", "")
+                    author_str = f"{family}, {given}".strip(", ")
+                    if author_str:
+                        authors_list.append(author_str)
+
+                containers = payload.get("container-title", [""])
+                journal_title = containers[0] if containers else ""
+
+                issued_parts = payload.get("issued", {}).get("date-parts", [[None]])
+                published_year = None
+                if issued_parts and issued_parts[0] and issued_parts[0][0] is not None:
+                    published_year = str(issued_parts[0][0])
+
+                page_string = payload.get("page", "")
+                start_page, end_page = None, None
+                if page_string:
+                    pages = page_string.split("-")
+                    start_page = pages[0] if pages else None
+                    end_page = pages[1] if len(pages) > 1 else None
+
+                issns = payload.get("ISSN", [""])
+                issn_value = issns[0] if issns else None
+
+                return {
+                    "title": article_title,
+                    "authors": authors_list,
+                    "journal": journal_title,
+                    "publisher": payload.get("publisher"),
+                    "year": published_year,
+                    "volume": payload.get("volume"),
+                    "issue": payload.get("issue"),
+                    "start_page": start_page,
+                    "end_page": end_page,
+                    "issn": issn_value,
+                }
+
+        except (requests.RequestException, ValueError, KeyError, IndexError) as error:
+            logger.warning("Crossref failed for %s: %s", doi, error)
+
+        return {}
+
+    def fetch_openalex_metadata(self, doi: str) -> dict[str, Any]:
+        """Fetch open-access status and subject concepts from OpenAlex."""
+        url = f"https://api.openalex.org/works/doi:{quote(doi)}?mailto={self.email}"
+        try:
+            response = self.session.get(url, timeout=10)
+            self.stats["api_calls"]["openalex"] += 1
+
+            if response.status_code == 200:
+                payload = response.json()
+                concepts = [
+                    c.get("display_name")
+                    for c in payload.get("concepts", [])
+                    if c.get("level", 99) <= 1 and c.get("display_name")
+                ]
+                oa_url = payload.get("open_access", {}).get("oa_url")
+                return {
+                    "keywords": concepts,
+                    "pdf_url": oa_url,
+                    "url": payload.get("id"),
+                }
+
+        except (requests.RequestException, ValueError, KeyError) as error:
+            logger.warning("OpenAlex failed for %s: %s", doi, error)
+
+        return {}
+
+    def fetch_semanticscholar_metadata(self, doi: str) -> dict[str, Any]:
+        """Fetch parsed abstract text from Semantic Scholar."""
+        url = (
+            f"https://api.semanticscholar.org/graph/v1/paper/DOI:{quote(doi)}"
+            f"?fields=abstract,referenceCount,citationCount"
+        )
+        try:
+            response = self.session.get(url, timeout=10)
+            self.stats["api_calls"]["semanticscholar"] += 1
+
+            if response.status_code == 200:
+                payload = response.json()
+                return {"abstract": payload.get("abstract")}
+
+        except (requests.RequestException, ValueError, KeyError) as error:
+            logger.warning("Semantic Scholar failed for %s: %s", doi, error)
+
+        return {}
+
+    def fetch_unpaywall_pdf(self, doi: str) -> dict[str, Any]:
+        """Fetch the best open-access PDF location from Unpaywall."""
+        url = f"https://api.unpaywall.org/v2/{quote(doi)}?email={self.email}"
+        try:
+            response = self.session.get(url, timeout=10)
+            self.stats["api_calls"]["unpaywall"] += 1
+
+            if response.status_code == 200:
+                payload = response.json()
+                best_oa = payload.get("best_oa_location")
+                if best_oa and best_oa.get("url_for_pdf"):
+                    return {"pdf_url": best_oa.get("url_for_pdf")}
+
+        except (requests.RequestException, ValueError, KeyError) as error:
+            logger.warning("Unpaywall failed for %s: %s", doi, error)
+
+        return {}
+
+    # --- Processing logic --------------------------------------------------------
+
+    def enrich_record(self, record: dict[str, Any]) -> dict[str, Any]:
+        """Aggregate metadata from all providers and fill gaps in one record.
+
+        Existing fields are never overwritten -- only empty/missing
+        fields are populated, so user-curated data always wins.
+        """
+        self.stats["processed"] += 1
+
+        doi = self.extract_doi(record)
+        title = record.get("title")
+
+        if not doi and title:
+            doi = self.resolve_doi_by_title(str(title))
+
+        if not doi:
+            self.stats["failed"] += 1
+            self.failed_records.append(
+                {"original_title": title, "reason": "Unresolved DOI via Title Matching"}
+            )
+            return record
+
+        enriched_data: dict[str, Any] = {"doi": doi}
+
+        crossref_metadata = self.fetch_crossref_metadata(doi)
+        scholar_metadata = self.fetch_semanticscholar_metadata(doi)
+        openalex_metadata = self.fetch_openalex_metadata(doi)
+        unpaywall_metadata = self.fetch_unpaywall_pdf(doi)
+
+        enriched_data.update(crossref_metadata)
+        enriched_data.update(scholar_metadata)
+        enriched_data.update(openalex_metadata)
+
+        if unpaywall_metadata.get("pdf_url"):
+            enriched_data["pdf_url"] = unpaywall_metadata["pdf_url"]
+
+        modified = False
+        for logical_key, rispy_field in RISPY_FIELD_MAP.items():
+            new_value = enriched_data.get(logical_key)
+            if not new_value:
+                continue
+
+            if not record.get(rispy_field):
+                if rispy_field in _LIST_TYPE_FIELDS:
+                    record[rispy_field] = (
+                        new_value if isinstance(new_value, list) else [str(new_value)]
+                    )
+                else:
+                    record[rispy_field] = (
+                        new_value if isinstance(new_value, str) else str(new_value)
+                    )
+                modified = True
+
+        if modified:
+            self.stats["enriched"] += 1
+
+        return record
+
+    def enrich_file(
+        self,
+        input_path: str | Path,
+        output_path: str | Path,
+        fail_report_path: str | Path = "failed_records.json",
+        request_delay_seconds: float = 0.1,
+    ) -> dict[str, Any]:
+        """Enrich every record in a RIS file and write the result.
+
+        Args:
+            input_path: Source ``.ris`` file to enrich.
+            output_path: Destination for the enriched ``.ris`` file.
+            fail_report_path: Where to write a JSON report of records
+                whose DOI could not be resolved. Only written if at
+                least one record failed.
+            request_delay_seconds: Delay between records, to stay
+                within provider rate limits beyond the built-in retry
+                policy.
+
+        Returns:
+            The accumulated ``self.stats`` dict.
+        """
+        input_path = Path(input_path)
+        output_path = Path(output_path)
+        fail_report_path = Path(fail_report_path)
+
+        logger.info("Loading %s...", input_path)
+        try:
+            with input_path.open("r", encoding="utf-8") as file:
+                records = list(rispy.load(file))
+        except (OSError, TypeError, ValueError) as error:
+            logger.error("Failed to parse RIS file: %s", error)
+            return self.stats
+
+        enriched_records = []
+        for record in records:
+            enriched_records.append(self.enrich_record(record))
+            if request_delay_seconds:
+                time.sleep(request_delay_seconds)
+
+        with output_path.open("w", encoding="utf-8") as file:
+            rispy.dump(enriched_records, file)
+
+        if self.failed_records:
+            with fail_report_path.open("w", encoding="utf-8") as file:
+                json.dump(self.failed_records, file, indent=2)
+
+        logger.info("Complete. Stats: %s", json.dumps(self.stats, indent=2))
+        return self.stats

risforge/exceptions.py

@@ -0,0 +1,23 @@
+"""Exception types for risforge.
+
+We deliberately keep this hierarchy small. Most failure modes in this
+package are already well described by the standard library's own
+exceptions (``FileNotFoundError``, ``ValueError``, etc.), and the
+original scripts caught those precisely rather than reaching for
+generic ``Exception``. We keep that pattern. ``RisForgeError`` exists
+only as a common base for the handful of errors that are specific to
+this package's domain logic, so library users can catch
+``RisForgeError`` if they want a single net for "something in risforge
+itself went wrong" without having to also catch unrelated stdlib
+errors they may want to handle differently.
+"""
+
+from __future__ import annotations
+
+
+class RisForgeError(Exception):
+    """Base class for errors raised directly by risforge's own logic."""
+
+
+class RisParsingError(RisForgeError):
+    """Raised when a RIS file cannot be parsed into any usable records."""

risforge/pipeline.py

@@ -0,0 +1,88 @@
+"""Orchestrates the clean -> enrich pipeline for a RIS file.
+
+This is a generalization of the original ``help.py``. The workflow is
+identical (clean/dedupe, then enrich), but two things changed:
+
+1. File paths are now parameters instead of hardcoded to a specific
+   machine's ``/home/<user>/Desktop`` -- a package published to PyPI
+   has to run on other people's filesystems.
+2. It's a plain importable function returning a result object, with
+   :mod:`risforge.cli` providing the "run this as a script with these
+   fixed paths" behavior that ``help.py`` used to hardcode.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from risforge.cleaning import clean_ris_file
+from risforge.enrichment import RisEnricher
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class PipelineResult:
+    """Outcome of a full clean+enrich pipeline run."""
+
+    cleaned_record_count: int
+    cleaning_errors: list[tuple[int, str]]
+    enrichment_stats: dict[str, Any] = field(default_factory=dict)
+
+
+def run_pipeline(
+    input_path: str | Path,
+    dedup_path: str | Path,
+    enriched_path: str | Path,
+    email: str,
+    fail_report_path: str | Path = "failed_records.json",
+) -> PipelineResult:
+    """Run the two-phase clean -> enrich pipeline end to end.
+
+    Args:
+        input_path: Raw ``.ris`` export to process.
+        dedup_path: Where the cleaned/deduplicated intermediate file
+            is written.
+        enriched_path: Where the final enriched ``.ris`` file is
+            written.
+        email: Contact email passed to Crossref/OpenAlex/Unpaywall
+            (required by their usage policies).
+        fail_report_path: Where a JSON report of unresolved records is
+            written, if any.
+
+    Returns:
+        A :class:`PipelineResult` summarizing both phases.
+
+    Raises:
+        FileNotFoundError: If ``input_path`` does not exist.
+    """
+    logger.info("Starting RIS bibliographic pipeline...")
+
+    logger.info("Phase 1: Deduplicating %s", input_path)
+    records, errors = clean_ris_file(input_path, dedup_path)
+    logger.info("Phase 1 complete: generated %d clean records.", len(records))
+    if errors:
+        logger.warning("Encountered %d parsing errors during Phase 1.", len(errors))
+
+    logger.info("Phase 2: Initializing metadata enrichment via APIs")
+    enricher = RisEnricher(email=email)
+    stats = enricher.enrich_file(
+        input_path=dedup_path,
+        output_path=enriched_path,
+        fail_report_path=fail_report_path,
+    )
+    logger.info(
+        "Phase 2 complete: enriched %d/%d records.",
+        stats.get("enriched", 0),
+        stats.get("processed", 0),
+    )
+
+    logger.info("Pipeline execution finished successfully.")
+    return PipelineResult(
+        cleaned_record_count=len(records),
+        cleaning_errors=errors,
+        enrichment_stats=stats,
+    )

risforge-0.1.0.dist-info/METADATA

@@ -0,0 +1,182 @@
+Metadata-Version: 2.4
+Name: risforge
+Version: 0.1.0
+Summary: Clean, deduplicate, and enrich RIS bibliographic files for systematic reviews.
+Author-email: Amyr <amyrhexa@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/amyr/risforge
+Project-URL: Repository, https://github.com/amyr/risforge
+Project-URL: Issues, https://github.com/amyr/risforge/issues
+Project-URL: Changelog, https://github.com/amyr/risforge/blob/main/CHANGELOG.md
+Keywords: ris,bibliography,systematic-review,deduplication,crossref,openalex,citation-management,prisma
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Text Processing :: Filters
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: rispy>=0.9.0
+Requires-Dist: requests>=2.31
+Requires-Dist: requests-cache>=1.1
+Requires-Dist: urllib3>=2.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-cov>=5.0; extra == "dev"
+Requires-Dist: build>=1.2; extra == "dev"
+Requires-Dist: twine>=5.0; extra == "dev"
+Dynamic: license-file
+
+# risforge
+
+**Clean, deduplicate, and enrich RIS bibliographic files for systematic reviews.**
+
+`risforge` takes a raw `.ris` export from Scopus, Web of Science, PubMed,
+EndNote, or any other reference manager and:
+
+1. **Cleans & deduplicates** it — normalizing titles, DOIs, and author
+   names, then clustering duplicate records (by exact DOI, and by a
+   title + first-author fallback) and merging each cluster into one
+   complete, data-loss-free record.
+2. **Enriches** it — filling in missing abstracts, journal names,
+   volumes/issues/pages, ISSNs, keywords, and open-access PDF links by
+   querying [Crossref](https://www.crossref.org/), [OpenAlex](https://openalex.org/),
+   [Semantic Scholar](https://www.semanticscholar.org/), and
+   [Unpaywall](https://unpaywall.org/). Existing fields are never
+   overwritten — only gaps are filled.
+
+It's built for the kind of unglamorous but essential prep work that
+comes before title/abstract screening in a systematic review: getting
+one clean, complete, deduplicated `.ris` file out of a pile of messy,
+overlapping database exports.
+
+## Features
+
+- **DOI-first deduplication** with a normalized title + first-author
+  fallback for records that lack a DOI, using union-find clustering so
+  transitively-linked duplicates across three, four, or more sources
+  all collapse into one record.
+- **Zero data loss on merge** — the most complete record in a
+  duplicate cluster is used as the base, and every other record in the
+  cluster supplements it with whatever fields it's missing.
+- **Non-destructive enrichment** — only empty fields are filled;
+  anything you (or an upstream export) already populated is left
+  alone.
+- **Resilient HTTP** — automatic retries with exponential backoff on
+  429/5xx responses, and a 7-day on-disk response cache so re-running
+  a pipeline doesn't re-hit the same APIs for records you've already
+  enriched.
+- **Library or CLI** — use it as `import risforge` in a script/notebook,
+  or as a single `risforge` command.
+
+## Installation
+
+```bash
+pip install risforge
+```
+
+Requires Python 3.10+.
+
+## Quick start
+
+### Command line
+
+```bash
+# Clean and deduplicate only
+risforge clean raw_export.ris clean.ris
+
+# Enrich an already-clean file
+risforge enrich clean.ris enriched.ris --email you@example.com
+
+# Both steps in one call
+risforge pipeline raw_export.ris --email you@example.com
+```
+
+The `pipeline` subcommand writes `<input>_clean.ris` and
+`<input>_enriched.ris` next to your input file by default; pass
+`--dedup-output` / `--output` to control that explicitly.
+
+An email address is required by Crossref, OpenAlex, and Unpaywall's
+"polite pool" usage policies — it's sent as a contact address in your
+requests, never stored or transmitted anywhere else.
+
+### Python API
+
+```python
+from risforge import clean_ris_file, RisEnricher, run_pipeline
+
+# Clean + deduplicate only
+records, errors = clean_ris_file("raw_export.ris", "clean.ris")
+print(f"{len(records)} unique records, {len(errors)} parse errors")
+
+# Enrich only
+enricher = RisEnricher(email="you@example.com")
+stats = enricher.enrich_file("clean.ris", "enriched.ris")
+print(f"Enriched {stats['enriched']}/{stats['processed']} records")
+
+# Both, in one call
+result = run_pipeline(
+    input_path="raw_export.ris",
+    dedup_path="clean.ris",
+    enriched_path="enriched.ris",
+    email="you@example.com",
+)
+print(result.cleaned_record_count, result.enrichment_stats)
+```
+
+## Configuration
+
+`RisEnricher` accepts a few constructor arguments beyond `email`:
+
+```python
+RisEnricher(
+    email="you@example.com",
+    cache_name=".api_cache",   # base filename for the on-disk HTTP cache
+    session=None,              # inject your own requests.Session (mainly for testing)
+)
+```
+
+`enrich_file()` also accepts `fail_report_path` (where unresolved-DOI
+records are written as JSON) and `request_delay_seconds` (delay
+between records; defaults to 0.1s to stay within provider rate limits).
+
+## Troubleshooting
+
+- **"Unresolved DOI via Title Matching" in the failure report** — the
+  record had no DOI and its title didn't match any Crossref result
+  above the 90% similarity threshold closely enough to resolve one.
+  These records are returned unmodified rather than guessed at.
+- **Enrichment seems slow** — each record makes up to 5 API calls
+  (1 for DOI resolution if needed, 4 for metadata) with a small delay
+  between records. Re-running against the same input is fast, since
+  responses are cached for 7 days.
+- **A field I already had got left alone even though enrichment "ran"** —
+  that's intentional. Enrichment only fills empty fields; it never
+  overwrites existing data.
+
+## Contributing
+
+1. Clone the repo and install in editable mode with dev dependencies:
+   ```bash
+   pip install -e ".[dev]"
+   ```
+2. Run the test suite:
+   ```bash
+   pytest
+   ```
+3. Open a PR. Please include tests for any behavioral change, and note
+   any deduplication/merge/enrichment logic changes explicitly in
+   `CHANGELOG.md` — this package's core value is *predictability* for
+   systematic review workflows, so silent behavior changes are treated
+   as bugs.
+
+## License
+
+MIT — see [LICENSE](LICENSE).

risforge-0.1.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+risforge/__init__.py,sha256=laXP4k6nnIaAYIp6imw8HKOnII5QnKH9n0dQnUSlLnw,1035
+risforge/cleaning.py,sha256=au7Kohr1fXvSdltdT4i7clu7Z-W_Wl0ZPVCxP_VJs68,10985
+risforge/cli.py,sha256=UuJqAKOvjJrNv57CZ5CkC72KxQpu_BVM6hD_ZxwZ3M4,6554
+risforge/enrichment.py,sha256=5ty-fa_G_MHWZfq_rTub4jsFNo7UeeXzydpkiIuYres,16520
+risforge/exceptions.py,sha256=azQRv98OEqgAHXgdog68LhbS-607GTSKXLNL8w9WJwI,937
+risforge/pipeline.py,sha256=XF3iaOIggZ_2OP1X9hIpo4WfrZu_KVfLL-K3bZ-Mc08,2919
+risforge-0.1.0.dist-info/licenses/LICENSE,sha256=vQS5uJ3KfJmWzhi4CsbEevI_26nx1RxO3hnUWb5D9yA,1061
+risforge-0.1.0.dist-info/METADATA,sha256=Ml38gRN-0o38uWgw76oHy8WzvbYZWc3ZV_QE0YNkAYI,6682
+risforge-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+risforge-0.1.0.dist-info/entry_points.txt,sha256=POs_X6Bu61aO5qoaA__Fll1dKcUl2rvB50QZpAcb6ss,47
+risforge-0.1.0.dist-info/top_level.txt,sha256=Xo-Ia33NjkuMGYmEMLAuHpOxYOMYfMa8SOFbyUXwE7k,9
+risforge-0.1.0.dist-info/RECORD,,

risforge-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

risforge-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+risforge = risforge.cli:main

risforge-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Amyr
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

risforge-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+risforge