scipaperlib 0.1.0__py3-none-any.whl

scipaperlib/__init__.py

@@ -0,0 +1,35 @@
+"""Local publication acquisition and evidence-preserving retrieval."""
+
+from .storage import Library
+from .service import Service
+from .models import (
+    Filters,
+    Evidence,
+    EvidenceContext,
+    DiscoveryReport,
+    SciPaperlibError,
+    Assertion,
+)
+from .discovery import discover, parse_list
+from .importing import import_papers
+from .acquisition import sync
+from .search import build_index, search_papers, read_context
+
+__version__ = "0.1.0"
+__all__ = [
+    "Library",
+    "Service",
+    "Filters",
+    "Evidence",
+    "EvidenceContext",
+    "DiscoveryReport",
+    "SciPaperlibError",
+    "Assertion",
+    "discover",
+    "parse_list",
+    "import_papers",
+    "sync",
+    "build_index",
+    "search_papers",
+    "read_context",
+]

scipaperlib/acquisition.py

@@ -0,0 +1,268 @@
+from __future__ import annotations
+import shutil
+import tempfile
+from pathlib import Path
+from .models import Manifest, Artifact, SciPaperlibError, now
+from .storage import digest
+from .extraction import extract
+
+
+def sync(
+    library,
+    selection=None,
+    *,
+    providers,
+    artifacts=("pdf", "source"),
+    refresh=False,
+    progress=None,
+    cancelled=None,
+):
+    outcomes = []
+    with library.lock():
+        papers = (
+            [library.get(i) for i in selection]
+            if selection is not None
+            else library.papers()
+        )
+        total = len(papers) * len(artifacts)
+        completed = 0
+
+        def emit(stage, **values):
+            nonlocal completed
+            if stage == "completed":
+                completed += 1
+            if progress:
+                progress(
+                    {
+                        "stage": stage,
+                        "completed": completed,
+                        "total_files": total,
+                        **values,
+                    }
+                )
+
+        emit("planned", papers=len(papers), artifacts=list(artifacts))
+        for paper in papers:
+            if cancelled and cancelled():
+                break
+            arxiv = next((i for i in paper.identifiers if i.provider == "arxiv"), None)
+            direct = next((i for i in paper.identifiers if i.provider == "pdf"), None)
+            identifier = arxiv.alias if arxiv else paper.paper_id
+            emit("resolving", paper_id=paper.paper_id, identifier=identifier)
+            revision = paper.active_revision
+            urls = {}
+            try:
+                if arxiv:
+                    if paper.requested_arxiv_version and not refresh:
+                        version = paper.requested_arxiv_version
+                    elif revision and not refresh:
+                        version = int(revision.removeprefix("arxiv-v"))
+                    elif arxiv.version and not refresh:
+                        version = arxiv.version
+                    else:
+                        resolved = providers.arxiv(
+                            arxiv.model_copy(update={"version": None})
+                            if refresh
+                            else arxiv,
+                            refresh=refresh,
+                        )
+                        version = resolved["identifiers"][0].version
+                        paper.title = resolved["title"] or paper.title
+                    revision = f"arxiv-v{version}"
+                    value = arxiv.value + f"v{version}"
+                    urls = {
+                        "pdf": f"https://arxiv.org/pdf/{value}",
+                        "source": f"https://arxiv.org/src/{value}",
+                    }
+                elif direct:
+                    urls = {"pdf": direct.value}
+                else:
+                    documents = paper.bibliographic.get("documents", [])
+                    if documents:
+                        urls = {"pdf": documents[0]["url"]}
+                if not urls:
+                    raise SciPaperlibError(
+                        "source_unavailable", "No supported full-text location"
+                    )
+                manifest = Manifest(revision=revision or "pending")
+                if (
+                    revision
+                    and library.path(
+                        f"papers/{paper.directory}/revisions/{revision}/manifest.json"
+                    ).exists()
+                ):
+                    manifest = Manifest.model_validate(
+                        library.read(
+                            f"papers/{paper.directory}/revisions/{revision}/manifest.json"
+                        )
+                    )
+                for kind in artifacts:
+                    if cancelled and cancelled():
+                        break
+                    emit(
+                        "file_started",
+                        paper_id=paper.paper_id,
+                        identifier=identifier,
+                        artifact=kind,
+                    )
+                    old = manifest.artifacts.get(kind)
+                    if (
+                        old
+                        and old.status == "succeeded"
+                        and old.path
+                        and library.path(old.path).exists()
+                        and digest(library.path(old.path).read_bytes()) == old.sha256
+                    ):
+                        outcomes.append(
+                            {
+                                "paper_id": paper.paper_id,
+                                "artifact": kind,
+                                "status": "skipped",
+                            }
+                        )
+                        emit("completed", identifier=identifier, **outcomes[-1])
+                        continue
+                    artifact = Artifact(
+                        kind=kind,
+                        status="running",
+                        requested_url=urls.get(kind),
+                        provider_revision=revision,
+                    )
+                    stage = Path(
+                        tempfile.mkdtemp(prefix="acquire-", dir=library.path("jobs"))
+                    )
+                    try:
+                        if kind not in urls:
+                            raise SciPaperlibError(
+                                "source_unavailable", f"No {kind} offered by provider"
+                            )
+                        partial = stage / "artifact.part"
+                        _, final, headers = providers.network.get(
+                            urls[kind],
+                            destination=partial,
+                            progress=lambda event: emit(
+                                event["stage"],
+                                paper_id=paper.paper_id,
+                                identifier=identifier,
+                                artifact=kind,
+                                **{k: v for k, v in event.items() if k != "stage"},
+                            ),
+                        )
+                        emit(
+                            "processing",
+                            paper_id=paper.paper_id,
+                            identifier=identifier,
+                            artifact=kind,
+                        )
+                        data = partial.read_bytes()
+                        checksum = digest(data)
+                        if kind == "pdf":
+                            if not data.lstrip().startswith(b"%PDF-"):
+                                raise SciPaperlibError(
+                                    "invalid_payload", "Response is not a PDF"
+                                )
+                            fmt = "pdf"
+                        else:
+                            fmt = extract(data, stage / "source")
+                        if not revision:
+                            revision = "sha256-" + checksum
+                            manifest.revision = revision
+                        root = f"papers/{paper.directory}/revisions/{revision}"
+                        relative = f"{root}/original/" + (
+                            "paper.pdf" if kind == "pdf" else "source.bin"
+                        )
+                        target = library.path(relative)
+                        target.parent.mkdir(parents=True, exist_ok=True)
+                        if target.exists() and digest(target.read_bytes()) != checksum:
+                            raise SciPaperlibError(
+                                "revision_conflict",
+                                "Provider changed bytes for an existing immutable revision",
+                            )
+                        if not target.exists():
+                            partial.replace(target)
+                        source_target = library.path(f"{root}/source")
+                        if (
+                            kind == "source"
+                            and fmt != "pdf-only"
+                            and not source_target.exists()
+                        ):
+                            manifest.extracted_checksums = {
+                                str(path.relative_to(stage / "source")): digest(
+                                    path.read_bytes()
+                                )
+                                for path in (stage / "source").rglob("*")
+                                if path.is_file()
+                            }
+                            (stage / "source").replace(source_target)
+                        artifact = Artifact(
+                            kind=kind,
+                            status="succeeded",
+                            requested_url=urls[kind],
+                            final_url=final,
+                            fetched=now(),
+                            byte_count=len(data),
+                            sha256=checksum,
+                            path=relative,
+                            format=fmt,
+                            provider_revision=revision,
+                        )
+                        if fmt == "pdf-only":
+                            manifest.warnings.append(
+                                "Source endpoint returned PDF; TeX source unavailable"
+                            )
+                    except SciPaperlibError as exc:
+                        artifact.status = (
+                            "unavailable"
+                            if exc.error.code == "source_unavailable"
+                            else "failed"
+                        )
+                        artifact.error = exc.error
+                    finally:
+                        shutil.rmtree(stage)
+                    manifest.artifacts[kind] = artifact
+                    outcomes.append(
+                        {
+                            "paper_id": paper.paper_id,
+                            "artifact": kind,
+                            "status": artifact.status,
+                            "error": artifact.error.model_dump()
+                            if artifact.error
+                            else None,
+                        }
+                    )
+                    if revision:
+                        library.write(
+                            f"papers/{paper.directory}/revisions/{revision}/manifest.json",
+                            manifest,
+                        )
+                    emit("completed", identifier=identifier, **outcomes[-1])
+                if revision:
+                    paper.active_revision = revision
+                paper.status = (
+                    "available"
+                    if any(a.status == "succeeded" for a in manifest.artifacts.values())
+                    else "unavailable"
+                )
+                library.save(paper)
+            except SciPaperlibError as exc:
+                # Failure to resolve a location accounts for every planned file.
+                for kind in artifacts:
+                    outcomes.append(
+                        {
+                            "paper_id": paper.paper_id,
+                            "artifact": kind,
+                            "status": "unavailable"
+                            if exc.error.code == "source_unavailable"
+                            else "failed",
+                            "error": exc.error.model_dump(),
+                        }
+                    )
+                    emit("completed", identifier=identifier, **outcomes[-1])
+        library.rebuild_catalogue()
+        report = {
+            "schema_version": 1,
+            "outcomes": outcomes,
+            "interrupted": bool(cancelled and cancelled()),
+        }
+        library.write("logs/last-download.json", report)
+    return report