preprint-fulltext 0.1.0__py3-none-any.whl

preprint_fulltext/__init__.py

@@ -0,0 +1,8 @@
+"""preprint-fulltext — retrieve full text of bioRxiv/medRxiv preprints.
+
+Ports-and-adapters toolkit: sources map into a canonical pydantic model
+(:mod:`preprint_fulltext.core.models`); the CLI and MCP server are thin
+frontends over the same core library.
+"""
+
+__version__ = "0.1.0"

preprint_fulltext/cli.py

@@ -0,0 +1,324 @@
+"""Typer CLI — thin frontend over the core library.
+
+Subcommands: `get` (single full text), `discover`/`search` (SearchHit JSONL),
+`ingest` (bulk chunked corpus). All heavy lifting lives in the core library and
+`pipeline/`; this module only parses args, calls the router/pipeline, and formats.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import sys
+from collections.abc import Iterable, Iterator
+from pathlib import Path
+from typing import Optional
+
+import orjson
+import typer
+
+from .config import get_settings
+from .core.models import FullText, SearchHit
+
+app = typer.Typer(
+    name="preprint-fulltext",
+    help="Retrieve full text of bioRxiv/medRxiv preprints into embedding-ready corpora.",
+    no_args_is_help=True,
+    add_completion=False,
+)
+
+
+@app.callback()
+def _main() -> None:
+    """preprint-fulltext — CLI frontend over the shared core library."""
+
+
+def _dumps(obj) -> bytes:
+    return orjson.dumps(obj, option=orjson.OPT_INDENT_2)
+
+
+def _write_hits_jsonl(hits: Iterable[SearchHit], out: Optional[Path], limit: int) -> int:
+    """Stream SearchHit records as one JSON object per line; returns the count."""
+    fh = out.open("wb") if out else sys.stdout.buffer
+    n = 0
+    try:
+        for hit in hits:
+            fh.write(orjson.dumps(hit.model_dump(mode="json")))
+            fh.write(b"\n")
+            n += 1
+            if n >= limit:
+                break
+    finally:
+        if out:
+            fh.close()
+    return n
+
+
+def _since_to_date(since: Optional[str]) -> Optional[dt.date]:
+    """Accept YYYY-MM or YYYY-MM-DD; map YYYY-MM to the first of the month."""
+    if not since:
+        return None
+    parts = since.split("-")
+    if len(parts) == 2:
+        return dt.date(int(parts[0]), int(parts[1]), 1)
+    return dt.date.fromisoformat(since)
+
+
+def _render_markdown(ft: FullText) -> str:
+    p = ft.preprint
+    lines = [f"# {p.title or p.doi}", ""]
+    meta = [f"**DOI:** {p.doi}", f"**Server:** {p.server.value}"]
+    if p.version is not None:
+        meta.append(f"**Version:** {p.version}")
+    if p.authors:
+        meta.append(f"**Authors:** {', '.join(p.authors)}")
+    if p.license:
+        lic = p.license.spdx_id or p.license.raw or "unknown"
+        meta.append(f"**License:** {lic} (redistributable={p.license.redistributable})")
+    meta.append(f"**Retrieved from:** {ft.retrieved_from.value}")
+    lines.append("  \n".join(meta))
+    lines.append("")
+    for s in ft.sections:
+        heading = s.title or s.kind.value.capitalize()
+        lines.append(f"## {heading}")
+        lines.append("")
+        lines.append(s.text)
+        lines.append("")
+    return "\n".join(lines).rstrip() + "\n"
+
+
+@app.command()
+def version() -> None:
+    """Print the installed version."""
+    from . import __version__
+
+    typer.echo(__version__)
+
+
+@app.command()
+def get(
+    doi: str = typer.Argument(..., help="DOI, arXiv id, or a doi.org/bioRxiv/arxiv.org URL."),
+    version: Optional[int] = typer.Option(None, "--version", "-v", help="Specific version."),
+    source: Optional[str] = typer.Option(
+        None, "--source", help="Force a source: europepmc | s3 | html | arxiv (default: auto)."
+    ),
+    server: Optional[str] = typer.Option(None, "--server", help="biorxiv | medrxiv (hint for S3/HTML)."),
+    html: bool = typer.Option(
+        False, "--html", help="Allow the public HTML page as a last-resort fallback (interactive only)."
+    ),
+    markdown: bool = typer.Option(False, "--markdown", "-m", help="Render sections as markdown."),
+    out: Optional[Path] = typer.Option(None, "--out", "-o", help="Write to file instead of stdout."),
+) -> None:
+    """Retrieve one preprint's full text as structured JSON (default) or markdown.
+
+    Accepts a bioRxiv/medRxiv DOI or content URL, a https://doi.org/… URL, or an arXiv id
+    / arxiv.org URL / 10.48550/arXiv.* DOI (routed to arXiv's LaTeXML full text).
+    """
+    from .pipeline.router import Router
+
+    result = Router(get_settings()).get_fulltext(
+        doi, version=version, source=source, server=server, allow_html=html
+    )
+    if result.fulltext is None:
+        typer.secho(
+            f"No full text for {doi} (tried: {', '.join(result.tried) or 'none'}). "
+            f"{result.reason or ''}".strip(),
+            err=True,
+            fg=typer.colors.YELLOW,
+        )
+        raise typer.Exit(code=2)
+
+    if markdown:
+        payload = _render_markdown(result.fulltext).encode("utf-8")
+    else:
+        payload = _dumps(result.fulltext.model_dump(mode="json"))
+
+    if out:
+        out.write_bytes(payload)
+        typer.secho(f"wrote {out}", err=True, fg=typer.colors.GREEN)
+    else:
+        sys.stdout.buffer.write(payload)
+        if not payload.endswith(b"\n"):
+            sys.stdout.buffer.write(b"\n")
+
+
+@app.command()
+def search(
+    query: str = typer.Argument(..., help="Keyword query."),
+    field: str = typer.Option("fulltext", "--field", help="fulltext | title | abstract | author."),
+    source: str = typer.Option("europepmc", "--source", help="europepmc | openalex | arxiv."),
+    limit: int = typer.Option(25, "--limit", "-n", help="Max hits."),
+    out: Optional[Path] = typer.Option(None, "--out", "-o", help="Write JSONL to file."),
+) -> None:
+    """Keyword search → SearchHit JSONL. --field scopes to title/abstract/author/fulltext."""
+    settings = get_settings()
+    if source == "europepmc":
+        from .sources.europepmc import EuropePMC
+
+        hits: Iterator[SearchHit] = EuropePMC(settings).search(query, limit, field=field)
+    elif source == "openalex":
+        from .sources.openalex import OpenAlex
+
+        hits = OpenAlex(settings).search(query, limit, field=field)
+    elif source == "arxiv":
+        from .sources.arxiv import ArxivAPI
+
+        hits = ArxivAPI(settings).search(query, limit, field=field)
+    else:
+        typer.secho(f"unknown source {source!r} (europepmc|openalex|arxiv)", err=True, fg="red")
+        raise typer.Exit(code=2)
+    _run_hits(hits, out, limit)
+
+
+@app.command()
+def discover(
+    query: Optional[str] = typer.Option(None, "--query", "-q", help="Free-text topic."),
+    category: Optional[str] = typer.Option(None, "--category", help="Subject/concept."),
+    since: Optional[str] = typer.Option(None, "--since", help="From date (YYYY-MM or YYYY-MM-DD)."),
+    to: Optional[str] = typer.Option(None, "--to", help="To date (YYYY-MM-DD)."),
+    source: str = typer.Option("openalex", "--source", help="openalex | europepmc | arxiv."),
+    limit: int = typer.Option(100, "--limit", "-n", help="Max hits."),
+    out: Optional[Path] = typer.Option(None, "--out", "-o", help="Write JSONL to file."),
+) -> None:
+    """Discover preprints by topic/category/date window → SearchHit JSONL."""
+    from .sources.base import DiscoverQuery
+
+    settings = get_settings()
+    dq = DiscoverQuery(
+        query=query, category=category,
+        from_date=_since_to_date(since), to_date=_since_to_date(to), limit=limit,
+    )
+    if source == "openalex":
+        from .sources.openalex import OpenAlex
+
+        hits: Iterator[SearchHit] = OpenAlex(settings).discover(dq)
+    elif source == "europepmc":
+        from .sources.europepmc import EuropePMC
+
+        hits = EuropePMC(settings).discover(dq)
+    elif source == "arxiv":
+        from .sources.arxiv import ArxivAPI
+
+        hits = ArxivAPI(settings).discover(dq)
+    else:
+        typer.secho(f"unknown source {source!r} (openalex|europepmc|arxiv)", err=True, fg="red")
+        raise typer.Exit(code=2)
+    _run_hits(hits, out, limit)
+
+
+@app.command()
+def ingest(
+    out: Path = typer.Argument(..., help="Output corpus path (JSONL, or .parquet with --format parquet)."),
+    source: str = typer.Option("s3", "--source", help="s3 | europepmc."),
+    server: str = typer.Option("biorxiv", "--server", help="biorxiv | medrxiv | both (s3)."),
+    since: Optional[str] = typer.Option(None, "--since", help="From month YYYY-MM (Current_Content only)."),
+    query: Optional[str] = typer.Option(None, "--query", "-q", help="EPMC query (source=europepmc)."),
+    include_back_content: bool = typer.Option(
+        False, "--include-back-content", help="Also ingest undated S3 Back_Content batches."
+    ),
+    redistribution: bool = typer.Option(
+        False, "--redistribution", help="Gate for a shareable corpus (degrade non-CC to stubs)."
+    ),
+    limit: Optional[int] = typer.Option(None, "--limit", "-n", help="Max preprints (testing/sampling)."),
+    fmt: str = typer.Option("jsonl", "--format", help="jsonl | parquet."),
+) -> None:
+    """Bulk full text → chunked corpus JSONL/Parquet. Resumable and incremental."""
+    from .pipeline.export import ExportMode
+    from .pipeline.ingest import Ingest, Target
+
+    settings = get_settings()
+    mode = ExportMode.REDISTRIBUTION if redistribution else ExportMode.ANALYSIS
+    since_date = _since_to_date(since)
+
+    targets: Iterator[Target]
+    bytes_source = None
+    if source == "s3":
+        from .core.models import Server
+        from .sources.biorxiv_s3 import BiorxivS3
+
+        s3 = BiorxivS3(settings)
+        bytes_source = s3
+        servers = [Server.BIORXIV, Server.MEDRXIV] if server == "both" else [Server(server)]
+
+        def _s3_targets() -> Iterator[Target]:
+            count = 0
+            for srv in servers:
+                for key in s3.iter_keys(srv, since=since_date, include_back_content=include_back_content):
+                    yield Target(
+                        id=f"{srv.value}:{key}",
+                        load=lambda srv=srv, key=key: s3.parse_meca(s3.fetch_meca(srv, key), srv, key),
+                    )
+                    count += 1
+                    if limit and count >= limit:
+                        return
+
+        targets = _s3_targets()
+    elif source == "europepmc":
+        from .sources.europepmc import EuropePMC
+
+        epmc = EuropePMC(settings)
+        q = query or "*"
+
+        def _epmc_targets() -> Iterator[Target]:
+            for i, hit in enumerate(epmc.search(q, limit or 1000)):
+                if not hit.doi:
+                    continue
+                yield Target(id=f"doi:{hit.doi}", load=lambda doi=hit.doi: epmc.get_fulltext(doi))
+                if limit and i + 1 >= limit:
+                    return
+
+        targets = _epmc_targets()
+    elif source == "arxiv":
+        typer.secho(
+            "arXiv bulk ingest is not supported (arXiv full text is HTML, single-document). "
+            "Use `get`/`search` for arXiv, or arXiv's own S3 LaTeX-source bucket for bulk.",
+            err=True, fg="red",
+        )
+        raise typer.Exit(code=2)
+    else:
+        typer.secho(f"unknown source {source!r} (s3|europepmc)", err=True, fg="red")
+        raise typer.Exit(code=2)
+
+    ing = Ingest(out, mode=mode, settings=settings,
+                 on_reminder=lambda m: typer.secho(m, err=True, fg="yellow"),
+                 bytes_source=bytes_source)
+    report = ing.run(targets)
+
+    if fmt == "parquet":
+        _to_parquet(out)
+
+    typer.secho(
+        f"ingest done: {report.processed} processed, {report.skipped} skipped, "
+        f"{report.degraded} degraded, {report.not_found} not-found, "
+        f"{report.n_chunks} chunks, {report.bytes_downloaded} bytes (requester-pays).",
+        err=True, fg="green",
+    )
+
+
+def _to_parquet(jsonl_out: Path) -> None:
+    """Convert the durable JSONL corpus to Parquet (best-effort, needs pyarrow)."""
+    try:
+        import pyarrow as pa
+        import pyarrow.parquet as pq
+    except ImportError:
+        typer.secho("pyarrow not installed; keeping JSONL only", err=True, fg="yellow")
+        return
+    rows = [orjson.loads(line) for line in jsonl_out.read_bytes().splitlines() if line.strip()]
+    if not rows:
+        return
+    table = pa.Table.from_pylist(rows)
+    pq.write_table(table, str(jsonl_out.with_suffix(".parquet")))
+
+
+def _run_hits(hits: Iterator[SearchHit], out: Optional[Path], limit: int) -> None:
+    from .sources.base import SourceError
+
+    try:
+        n = _write_hits_jsonl(hits, out, limit)
+    except SourceError as e:
+        typer.secho(str(e), err=True, fg="red")
+        raise typer.Exit(code=2)
+    typer.secho(f"{n} hit(s)" + (f" → {out}" if out else ""), err=True, fg="green")
+
+
+if __name__ == "__main__":  # pragma: no cover
+    app()

preprint_fulltext/config.py

@@ -0,0 +1,127 @@
+"""Runtime configuration.
+
+Backed by ``pydantic-settings``: values come from (highest precedence first)
+constructor args, environment variables, a ``.env`` file, then an optional
+``preprint-fulltext.toml``. AWS credentials themselves are left to the standard
+boto3 chain; only the region and bucket names live here.
+
+Externally-standard names (``CONTACT_EMAIL``, ``OPENALEX_API_KEY``,
+``AWS_REGION``) are accepted both bare and with the ``PREPRINT_FULLTEXT_`` prefix.
+"""
+
+from __future__ import annotations
+
+from functools import lru_cache
+from pathlib import Path
+
+from pydantic import AliasChoices, Field
+from pydantic_settings import (
+    BaseSettings,
+    PydanticBaseSettingsSource,
+    SettingsConfigDict,
+    TomlConfigSettingsSource,
+)
+
+from . import __version__
+
+# Both buckets confirmed against the openRxiv TDM pages (biorxiv.org/tdm,
+# medrxiv.org/tdm, 2026-07-17): requester-pays, us-east-1, $0.09/GB (~6 TB total).
+DEFAULT_BIORXIV_BUCKET = "biorxiv-src-monthly"
+DEFAULT_MEDRXIV_BUCKET = "medrxiv-src-monthly"
+
+
+class Settings(BaseSettings):
+    """Process-wide settings. Instantiate via :func:`get_settings`."""
+
+    model_config = SettingsConfigDict(
+        env_prefix="PREPRINT_FULLTEXT_",
+        env_file=".env",
+        env_file_encoding="utf-8",
+        toml_file="preprint-fulltext.toml",
+        extra="ignore",
+        populate_by_name=True,  # accept the field name too, not only validation_alias
+    )
+
+    # --- polite-pool identity -------------------------------------------------
+    contact_email: str | None = Field(
+        default=None,
+        validation_alias=AliasChoices("PREPRINT_FULLTEXT_CONTACT_EMAIL", "CONTACT_EMAIL"),
+        description="Email for OpenAlex/EPMC polite pools; sent in User-Agent and mailto.",
+    )
+    openalex_api_key: str | None = Field(
+        default=None,
+        validation_alias=AliasChoices("PREPRINT_FULLTEXT_OPENALEX_API_KEY", "OPENALEX_API_KEY"),
+        description="OpenAlex API key (required since 2026-02-13, still free).",
+    )
+
+    # --- AWS / S3 -------------------------------------------------------------
+    aws_region: str = Field(
+        default="us-east-1",
+        validation_alias=AliasChoices("PREPRINT_FULLTEXT_AWS_REGION", "AWS_REGION"),
+        description="Region for the requester-pays openRxiv buckets (us-east-1).",
+    )
+    biorxiv_bucket: str = DEFAULT_BIORXIV_BUCKET
+    medrxiv_bucket: str = DEFAULT_MEDRXIV_BUCKET
+
+    # --- arXiv ----------------------------------------------------------------
+    arxiv_api_base: str = "https://export.arxiv.org/api/query"
+    arxiv_html_base: str = "https://arxiv.org/html"
+    ar5iv_base: str = "https://ar5iv.labs.arxiv.org/html"
+
+    # --- cache ----------------------------------------------------------------
+    cache_dir: Path = Field(
+        default=Path.home() / ".cache" / "preprint-fulltext",
+        description="Content-addressed cache of fetched .meca/XML.",
+    )
+
+    # --- chunking -------------------------------------------------------------
+    chunk_tokens: int = Field(default=512, ge=1, description="Max tokens per chunk.")
+    chunk_overlap: int = Field(default=64, ge=0, description="Token overlap within a section.")
+    tokenizer: str = Field(
+        default="tiktoken:cl100k_base",
+        description="Tokenizer spec, 'tiktoken:<encoding>'.",
+    )
+    cross_section: bool = Field(
+        default=False, description="Allow a chunk to span section boundaries."
+    )
+
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls: type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> tuple[PydanticBaseSettingsSource, ...]:
+        # env/.env win over the TOML file; TOML wins over hard-coded defaults.
+        return (
+            init_settings,
+            env_settings,
+            dotenv_settings,
+            TomlConfigSettingsSource(settings_cls),
+            file_secret_settings,
+        )
+
+    @property
+    def user_agent(self) -> str:
+        """User-Agent for polite-pool HTTP requests."""
+        base = f"preprint-fulltext/{__version__}"
+        return f"{base} (mailto:{self.contact_email})" if self.contact_email else base
+
+    def bucket_for(self, server: str) -> str:
+        """Return the S3 bucket name for a server ('biorxiv' | 'medrxiv')."""
+        if server == "biorxiv":
+            return self.biorxiv_bucket
+        if server == "medrxiv":
+            return self.medrxiv_bucket
+        raise ValueError(f"unknown server: {server!r}")
+
+
+@lru_cache(maxsize=1)
+def get_settings() -> Settings:
+    """Return the process-wide settings singleton."""
+    return Settings()
+
+
+__all__ = ["Settings", "get_settings", "DEFAULT_BIORXIV_BUCKET", "DEFAULT_MEDRXIV_BUCKET"]

preprint_fulltext/core/__init__.py

@@ -0,0 +1 @@
+"""core subpackage."""

preprint_fulltext/core/assemble.py

@@ -0,0 +1,72 @@
+"""assemble.py — map a pydantic-free :class:`ParsedArticle` into the canonical model.
+
+This is the single seam between the shared JATS parser and the canonical model,
+used by BOTH the Europe PMC and S3 paths so their `FullText` is byte-for-byte
+identical for the same document. The abstract is emitted as an
+``abstract``-kind :class:`Section` so the chunker treats it uniformly; sections
+are renumbered from 1 with ``id = str(order)``.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+
+from .jats import ParsedArticle
+from .licenses import parse_license
+from .models import FullText, Preprint, Section, SectionKind, Server, SourceName
+
+
+def build_fulltext(
+    parsed: ParsedArticle,
+    *,
+    doi: str,
+    server: Server | str,
+    source: SourceName | str,
+    version: int | None = None,
+    raw_ref: str | None = None,
+    published_doi: str | None = None,
+    date: dt.date | None = None,
+    category: str | None = None,
+    provenance: dict | None = None,
+) -> FullText:
+    """Assemble a :class:`FullText` from parser output plus caller-supplied metadata."""
+    license_ = parse_license(parsed.license_raw)
+
+    preprint = Preprint(
+        doi=doi,
+        version=version,
+        server=Server(server),
+        title=parsed.title,
+        authors=list(parsed.authors),
+        date=date,
+        category=category,
+        abstract=parsed.abstract,
+        license=license_,
+        published_doi=published_doi,
+        provenance=provenance or {},
+    )
+
+    sections: list[Section] = []
+    order = 0
+    if parsed.abstract:
+        order += 1
+        sections.append(
+            Section(id=str(order), kind=SectionKind.ABSTRACT, title="Abstract",
+                    text=parsed.abstract, order=order)
+        )
+    for ps in parsed.sections:
+        order += 1
+        sections.append(
+            Section(id=str(order), kind=SectionKind(ps.kind), title=ps.title,
+                    text=ps.text, order=order)
+        )
+
+    return FullText(
+        preprint=preprint,
+        sections=sections,
+        retrieved_from=SourceName(source),
+        raw_ref=raw_ref,
+    )
+
+
+__all__ = ["build_fulltext"]

preprint_fulltext/core/cache.py

@@ -0,0 +1,49 @@
+"""cache.py — content-addressed local cache of fetched bytes (.meca/XML).
+
+Keyed by ``sha256(source | id | version)`` so re-runs skip S3/EPMC. Sharded into
+256 subdirectories to keep any one directory small.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Callable
+from pathlib import Path
+
+from ..config import Settings, get_settings
+
+
+class Cache:
+    def __init__(self, cache_dir: Path | str | None = None, settings: Settings | None = None):
+        settings = settings or get_settings()
+        self.dir = Path(cache_dir or settings.cache_dir)
+
+    @staticmethod
+    def key(*parts: object) -> str:
+        raw = "|".join("" if p is None else str(p) for p in parts)
+        return hashlib.sha256(raw.encode("utf-8")).hexdigest()
+
+    def _path(self, key: str) -> Path:
+        return self.dir / key[:2] / key
+
+    def get(self, key: str) -> bytes | None:
+        p = self._path(key)
+        return p.read_bytes() if p.exists() else None
+
+    def put(self, key: str, data: bytes) -> None:
+        p = self._path(key)
+        p.parent.mkdir(parents=True, exist_ok=True)
+        tmp = p.with_suffix(".tmp")
+        tmp.write_bytes(data)
+        tmp.replace(p)  # atomic within the same filesystem
+
+    def get_or_fetch(self, key: str, fetch_fn: Callable[[], bytes]) -> bytes:
+        cached = self.get(key)
+        if cached is not None:
+            return cached
+        data = fetch_fn()
+        self.put(key, data)
+        return data
+
+
+__all__ = ["Cache"]