flowapy 0.1.3__tar.gz → 0.2.0__tar.gz

{flowapy-0.1.3 → flowapy-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowapy
-Version: 0.1.3
+Version: 0.2.0
 Summary: Variant literature assessment pipeline with AI extraction
 Project-URL: Homepage, https://github.com/populationgenomics/flowa
 Project-URL: Source, https://github.com/populationgenomics/flowa
@@ -47,6 +47,7 @@ Requires-Dist: pypdf
 Requires-Dist: s3fs
 Requires-Dist: tenacity
 Requires-Dist: typer
+Requires-Dist: zstandard
 Provides-Extra: anthropic
 Requires-Dist: pydantic-ai-slim[anthropic]==1.101.0; extra == 'anthropic'
 Provides-Extra: bedrock

{flowapy-0.1.3 → flowapy-0.2.0}/examples/demo/README.md

@@ -166,10 +166,10 @@ VARIANT=NM_001035_3-c_14174A_G
 rm -f assessments/$VARIANT/aggregation.json \
       assessments/$VARIANT/aggregation_raw.json
 rm -rf assessments/$VARIANT/extractions/ assessments/$VARIANT/runs/
-# Re-runs flowa.convert (which uses anchorite for PDF chunking).
-# Drop this line to reuse the cached markdown and only redo extract +
-# aggregate.
-rm -f papers/*/markdown.md papers/*/convert_raw.json
+# Re-runs flowa.convert (which uses anchorite for PDF chunking and
+# builds pdf_index.pkl.zst). Drop this line to reuse the cached markdown
+# + index and only redo extract + aggregate.
+rm -f papers/*/markdown.md papers/*/convert_raw.json papers/*/pdf_index.pkl.zst
 ```
 
 Then drive the pipeline. The demo's `scripts/start.ts` translates the
@@ -208,11 +208,13 @@ and not needed by anything downstream.
 For papers whose source license blocks redistribution (CC-BY-NC-ND,
 paywalled; see `fixtures/LICENSES.md` for the rule), do **not** delete
 the whole `papers/{encodedDoi}/` directory — only delete `source.pdf`,
-`markdown.md`, and `convert_raw.json`. Keep `metadata.json` (the
-bibliographic fields are factual data, not copyrightable) but replace
-its `abstract` field with a sentinel string, so the omission reads as
-deliberate (not a missing-data bug) when the literature view renders
-the row:
+`markdown.md`, `convert_raw.json`, and `pdf_index.pkl.zst`. The
+`pdf_index.pkl.zst` embeds the PDF's extracted text (anchorite's char
+index), so it carries the same copyright as `source.pdf` and must not
+ship in the open-source repo. Keep `metadata.json` (the bibliographic
+fields are factual data, not copyrightable) but replace its `abstract`
+field with a sentinel string, so the omission reads as deliberate (not
+a missing-data bug) when the literature view renders the row:
 
 ```bash
 python3 -c "

{flowapy-0.1.3 → flowapy-0.2.0}/examples/demo-gateway/src/demo_gateway/main.py

@@ -18,9 +18,13 @@ from typing import Annotated
 import uvicorn
 from fastapi import APIRouter, Depends, FastAPI, HTTPException, Query, Request, status
 from fastapi.middleware.cors import CORSMiddleware
-from flowa.resolve import ResolvedCitations, ResolveRequest, resolve_citations
+from flowa.resolve import (
+    ResolvedCitations,
+    ResolveRequest,
+    load_pdf_index_from_storage,
+    resolve_citations,
+)
 from flowa.schema import VariantSpec
-from flowa.storage import paper_url, read_bytes, read_text
 from pydantic import BaseModel, Field
 
 from .config import Settings
@@ -111,24 +115,15 @@ def resolve_citations_route(
 ) -> ResolvedCitations:
     """Align verbatim quotes to PDF bboxes.
 
-    Sync `def` so FastAPI auto-runs it in the threadpool — anchorite's PDF
-    parsing is CPU-bound and would block the asyncio loop otherwise.
+    Sync `def` so FastAPI auto-runs it in the threadpool — deserialising the
+    PdfIndex pickle and aligning quotes is CPU-bound and would block the
+    asyncio loop otherwise.
     """
     base = str(settings.demo_data_dir)
-
-    def pdf_loader(doi: str) -> bytes | None:
-        try:
-            return read_bytes(paper_url(base, doi, 'source.pdf'))
-        except FileNotFoundError:
-            return None
-
-    def md_loader(doi: str) -> str | None:
-        try:
-            return read_text(paper_url(base, doi, 'markdown.md'))
-        except FileNotFoundError:
-            return None
-
-    return resolve_citations(body.citations, pdf_loader=pdf_loader, markdown_loader=md_loader)
+    return resolve_citations(
+        body.citations,
+        index_provider=lambda doi: load_pdf_index_from_storage(base, doi),
+    )
 
 
 @asynccontextmanager

{flowapy-0.1.3 → flowapy-0.2.0}/examples/demo-gateway/tests/test_resolve.py

@@ -1,24 +1,13 @@
 """HTTP shape tests for /resolve-citations.
 
 Library-level resolver behaviour is covered in flowa's `tests/test_resolve.py`;
-here we just check that the route plumbs Settings → loader → flowa.resolve
-correctly and returns the expected wire shape.
+here we just check that the route plumbs Settings → index_provider →
+flowa.resolve correctly and returns the expected wire shape.
 """
 
-from pathlib import Path
-
 from fastapi.testclient import TestClient
-from flowa import resolve as flowa_resolve_module
-from flowa.storage import encode_doi
-
-from demo_gateway.config import Settings
-
 
-def _write_fake_paper(data_dir: Path, doi: str, pdf: bytes = b'fake-pdf', markdown: str = '# fake md') -> None:
-    paper_dir = data_dir / 'papers' / encode_doi(doi)
-    paper_dir.mkdir(parents=True, exist_ok=True)
-    (paper_dir / 'source.pdf').write_bytes(pdf)
-    (paper_dir / 'markdown.md').write_text(markdown)
+import demo_gateway.main as demo_main
 
 
 def test_post_resolve_citations_rejects_malformed_body(client: TestClient) -> None:
@@ -26,8 +15,8 @@ def test_post_resolve_citations_rejects_malformed_body(client: TestClient) -> No
     assert response.status_code == 422
 
 
-def test_post_resolve_citations_returns_errors_for_missing_pdfs(client: TestClient) -> None:
-    """When source.pdf is absent, the DOI surfaces in `errors` rather than `resolved`."""
+def test_post_resolve_citations_returns_errors_for_missing_index(client: TestClient) -> None:
+    """When pdf_index.pkl.zst is absent, the DOI surfaces in `errors` rather than `resolved`."""
     response = client.post(
         '/resolve-citations',
         json={'citations': [{'doi': '10.1/missing', 'quotes': ['anything']}]},
@@ -35,15 +24,11 @@ def test_post_resolve_citations_returns_errors_for_missing_pdfs(client: TestClie
     assert response.status_code == 200
     body = response.json()
     assert body['resolved'] == {}
-    assert body['errors'] == {'10.1/missing': 'source.pdf not found'}
+    assert body['errors'] == {'10.1/missing': 'pdf_index not available'}
 
 
-def test_post_resolve_citations_returns_resolved_bboxes(
-    client: TestClient,
-    settings: Settings,
-    monkeypatch,
-) -> None:
-    """When source.pdf exists, the route resolves quotes to bboxes via the library."""
+def test_post_resolve_citations_returns_resolved_bboxes(client: TestClient, monkeypatch) -> None:
+    """When the index loads, the route resolves quotes to bboxes via the library."""
 
     class _FakeBbox:
         def __init__(self, top: int, left: int, bottom: int, right: int) -> None:
@@ -53,16 +38,16 @@ def test_post_resolve_citations_returns_resolved_bboxes(
             self.right = right
 
     class _FakePdfIndex:
-        def __init__(self, _pdf_bytes: bytes, *, markdown: str | None = None) -> None:
-            pass
-
         def resolve(self, quotes: list[str]) -> dict[str, list[tuple[int, _FakeBbox]]]:
             # 0-indexed page from anchorite — the +1 boundary wrap in resolve.py
             # turns this into page=1 on the wire.
             return {q: [(0, _FakeBbox(top=10, left=20, bottom=30, right=40))] for q in quotes}
 
-    monkeypatch.setattr(flowa_resolve_module, 'PdfIndex', _FakePdfIndex)
-    _write_fake_paper(settings.demo_data_dir, '10.1/present')
+    monkeypatch.setattr(
+        demo_main,
+        'load_pdf_index_from_storage',
+        lambda _base, doi: _FakePdfIndex() if doi == '10.1/present' else None,
+    )
 
     response = client.post(
         '/resolve-citations',

{flowapy-0.1.3 → flowapy-0.2.0}/examples/demo-gateway/uv.lock

@@ -415,7 +415,7 @@ wheels = [
 
 [[package]]
 name = "flowapy"
-version = "0.1.3"
+version = "0.2.0"
 source = { editable = "../../" }
 dependencies = [
     { name = "anchorite" },
@@ -431,6 +431,7 @@ dependencies = [
     { name = "s3fs" },
     { name = "tenacity" },
     { name = "typer" },
+    { name = "zstandard" },
 ]
 
 [package.optional-dependencies]
@@ -466,6 +467,7 @@ requires-dist = [
     { name = "s3fs" },
     { name = "tenacity" },
     { name = "typer" },
+    { name = "zstandard" },
 ]
 provides-extras = ["anthropic", "bedrock", "google", "openai"]
 
@@ -2109,3 +2111,28 @@ sdist = { url = "https://files.pythonhosted.org/packages/30/21/093488dfc7cc8964d
 wheels = [
     { url = "https://files.pythonhosted.org/packages/08/8a/0861bec20485572fbddf3dfba2910e38fe249796cb73ecdeb74e07eeb8d3/zipp-3.23.1-py3-none-any.whl", hash = "sha256:0b3596c50a5c700c9cb40ba8d86d9f2cc4807e9bedb06bcdf7fac85633e444dc", size = 10378, upload-time = "2026-04-13T23:21:45.386Z" },
 ]
+
+[[package]]
+name = "zstandard"
+version = "0.25.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/fd/aa/3e0508d5a5dd96529cdc5a97011299056e14c6505b678fd58938792794b1/zstandard-0.25.0.tar.gz", hash = "sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b", size = 711513, upload-time = "2025-09-14T22:15:54.002Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/35/0b/8df9c4ad06af91d39e94fa96cc010a24ac4ef1378d3efab9223cc8593d40/zstandard-0.25.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec996f12524f88e151c339688c3897194821d7f03081ab35d31d1e12ec975e94", size = 795735, upload-time = "2025-09-14T22:17:26.042Z" },
+    { url = "https://files.pythonhosted.org/packages/3f/06/9ae96a3e5dcfd119377ba33d4c42a7d89da1efabd5cb3e366b156c45ff4d/zstandard-0.25.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a1a4ae2dec3993a32247995bdfe367fc3266da832d82f8438c8570f989753de1", size = 640440, upload-time = "2025-09-14T22:17:27.366Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/14/933d27204c2bd404229c69f445862454dcc101cd69ef8c6068f15aaec12c/zstandard-0.25.0-cp313-cp313-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:e96594a5537722fdfb79951672a2a63aec5ebfb823e7560586f7484819f2a08f", size = 5343070, upload-time = "2025-09-14T22:17:28.896Z" },
+    { url = "https://files.pythonhosted.org/packages/6d/db/ddb11011826ed7db9d0e485d13df79b58586bfdec56e5c84a928a9a78c1c/zstandard-0.25.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bfc4e20784722098822e3eee42b8e576b379ed72cca4a7cb856ae733e62192ea", size = 5063001, upload-time = "2025-09-14T22:17:31.044Z" },
+    { url = "https://files.pythonhosted.org/packages/db/00/87466ea3f99599d02a5238498b87bf84a6348290c19571051839ca943777/zstandard-0.25.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:457ed498fc58cdc12fc48f7950e02740d4f7ae9493dd4ab2168a47c93c31298e", size = 5394120, upload-time = "2025-09-14T22:17:32.711Z" },
+    { url = "https://files.pythonhosted.org/packages/2b/95/fc5531d9c618a679a20ff6c29e2b3ef1d1f4ad66c5e161ae6ff847d102a9/zstandard-0.25.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:fd7a5004eb1980d3cefe26b2685bcb0b17989901a70a1040d1ac86f1d898c551", size = 5451230, upload-time = "2025-09-14T22:17:34.41Z" },
+    { url = "https://files.pythonhosted.org/packages/63/4b/e3678b4e776db00f9f7b2fe58e547e8928ef32727d7a1ff01dea010f3f13/zstandard-0.25.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a", size = 5547173, upload-time = "2025-09-14T22:17:36.084Z" },
+    { url = "https://files.pythonhosted.org/packages/4e/d5/ba05ed95c6b8ec30bd468dfeab20589f2cf709b5c940483e31d991f2ca58/zstandard-0.25.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3a39c94ad7866160a4a46d772e43311a743c316942037671beb264e395bdd611", size = 5046736, upload-time = "2025-09-14T22:17:37.891Z" },
+    { url = "https://files.pythonhosted.org/packages/50/d5/870aa06b3a76c73eced65c044b92286a3c4e00554005ff51962deef28e28/zstandard-0.25.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:172de1f06947577d3a3005416977cce6168f2261284c02080e7ad0185faeced3", size = 5576368, upload-time = "2025-09-14T22:17:40.206Z" },
+    { url = "https://files.pythonhosted.org/packages/5d/35/398dc2ffc89d304d59bc12f0fdd931b4ce455bddf7038a0a67733a25f550/zstandard-0.25.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3c83b0188c852a47cd13ef3bf9209fb0a77fa5374958b8c53aaa699398c6bd7b", size = 4954022, upload-time = "2025-09-14T22:17:41.879Z" },
+    { url = "https://files.pythonhosted.org/packages/9a/5c/36ba1e5507d56d2213202ec2b05e8541734af5f2ce378c5d1ceaf4d88dc4/zstandard-0.25.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1673b7199bbe763365b81a4f3252b8e80f44c9e323fc42940dc8843bfeaf9851", size = 5267889, upload-time = "2025-09-14T22:17:43.577Z" },
+    { url = "https://files.pythonhosted.org/packages/70/e8/2ec6b6fb7358b2ec0113ae202647ca7c0e9d15b61c005ae5225ad0995df5/zstandard-0.25.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0be7622c37c183406f3dbf0cba104118eb16a4ea7359eeb5752f0794882fc250", size = 5433952, upload-time = "2025-09-14T22:17:45.271Z" },
+    { url = "https://files.pythonhosted.org/packages/7b/01/b5f4d4dbc59ef193e870495c6f1275f5b2928e01ff5a81fecb22a06e22fb/zstandard-0.25.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:5f5e4c2a23ca271c218ac025bd7d635597048b366d6f31f420aaeb715239fc98", size = 5814054, upload-time = "2025-09-14T22:17:47.08Z" },
+    { url = "https://files.pythonhosted.org/packages/b2/e5/fbd822d5c6f427cf158316d012c5a12f233473c2f9c5fe5ab1ae5d21f3d8/zstandard-0.25.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf", size = 5360113, upload-time = "2025-09-14T22:17:48.893Z" },
+    { url = "https://files.pythonhosted.org/packages/8e/e0/69a553d2047f9a2c7347caa225bb3a63b6d7704ad74610cb7823baa08ed7/zstandard-0.25.0-cp313-cp313-win32.whl", hash = "sha256:7030defa83eef3e51ff26f0b7bfb229f0204b66fe18e04359ce3474ac33cbc09", size = 436936, upload-time = "2025-09-14T22:17:52.658Z" },
+    { url = "https://files.pythonhosted.org/packages/d9/82/b9c06c870f3bd8767c201f1edbdf9e8dc34be5b0fbc5682c4f80fe948475/zstandard-0.25.0-cp313-cp313-win_amd64.whl", hash = "sha256:1f830a0dac88719af0ae43b8b2d6aef487d437036468ef3c2ea59c51f9d55fd5", size = 506232, upload-time = "2025-09-14T22:17:50.402Z" },
+    { url = "https://files.pythonhosted.org/packages/d4/57/60c3c01243bb81d381c9916e2a6d9e149ab8627c0c7d7abb2d73384b3c0c/zstandard-0.25.0-cp313-cp313-win_arm64.whl", hash = "sha256:85304a43f4d513f5464ceb938aa02c1e78c2943b29f44a750b48b25ac999a049", size = 462671, upload-time = "2025-09-14T22:17:51.533Z" },
+]

{flowapy-0.1.3 → flowapy-0.2.0}/packages/chat-service/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@flowajs/chat-service",
-  "version": "0.0.3",
+  "version": "0.0.4",
   "description": "Stateless service that orchestrates LLM conversations over flowa artifacts.",
   "license": "MIT",
   "type": "module",
@@ -71,7 +71,7 @@
     "zod": "4.4.3"
   },
   "peerDependencies": {
-    "@ai-sdk/amazon-bedrock": "^4.0.0",
+    "@ai-sdk/amazon-bedrock": "^4.0.101",
     "@ai-sdk/anthropic": "^3.0.0",
     "@ai-sdk/google": "^3.0.0",
     "@ai-sdk/google-vertex": "^4.0.0",

{flowapy-0.1.3 → flowapy-0.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "flowapy"
-version = "0.1.3"
+version = "0.2.0"
 description = "Variant literature assessment pipeline with AI extraction"
 readme = "README.md"
 requires-python = "==3.13.*"
@@ -27,6 +27,7 @@ dependencies = [
     "s3fs",   # S3/MinIO support for fsspec
     "tenacity",
     "typer",
+    "zstandard",
 ]
 
 [project.optional-dependencies]

{flowapy-0.1.3 → flowapy-0.2.0}/src/flowa/aggregate.py

@@ -15,7 +15,7 @@ from pydantic_ai import Agent, ModelRetry, NativeOutput, RunContext
 from flowa.clinvar import format_clinvar_for_prompt, query_clinvar
 from flowa.models import create_model, get_model_settings
 from flowa.prompts import load_prompt_and_schema
-from flowa.resolve import CitationQuery, resolve_citations
+from flowa.resolve import CitationQuery, load_pdf_index_from_storage, resolve_citations
 from flowa.schema import AGGREGATION_SCHEMA_VERSION, with_schema_version
 from flowa.settings import ModelConfig, Settings
 from flowa.storage import (
@@ -23,9 +23,7 @@ from flowa.storage import (
     encode_doi,
     exists,
     paper_url,
-    read_bytes,
     read_json,
-    read_text,
     write_bytes,
     write_json,
 )
@@ -158,16 +156,14 @@ def create_aggregate_agent(
 def resolve_aggregate_citations(
     aggregate_dict: dict[str, Any],
     paper_id_to_doi: dict[str, str],
-    pdf_bytes_cache: dict[str, bytes],
-    markdown_cache: dict[str, str],
+    base: str,
     metadata_cache: dict[str, dict[str, Any]],
 ) -> None:
     """Post-process aggregate output: resolve quotes to bboxes on claim citations.
 
-    Delegates the actual alignment to `flowa.resolve.resolve_citations` with
-    cache-backed loaders, then attaches the resulting bboxes onto each claim's
-    citations in place. Claims are grouped by paper_id so every (paper_id,
-    quote) pair resolves to exactly one paper.
+    Loads each paper's pre-built `pdf_index.pkl.zst` via the same path the
+    gateway uses; the convert step that ran earlier in this pipeline wrote
+    the artifact, so it's guaranteed to be present.
     """
     # Collect all (doi, quote) pairs, grouped by DOI.
     doi_quotes: dict[str, list[str]] = {}
@@ -180,8 +176,7 @@ def resolve_aggregate_citations(
     citations_input = [CitationQuery(doi=doi, quotes=quotes) for doi, quotes in doi_quotes.items()]
     result = resolve_citations(
         citations_input,
-        pdf_loader=pdf_bytes_cache.get,
-        markdown_loader=markdown_cache.get,
+        index_provider=lambda doi: load_pdf_index_from_storage(base, doi),
     )
 
     # Attach resolved bboxes onto each claim's citations.
@@ -229,14 +224,11 @@ async def aggregate_evidence_async(
     clinvar_data = query_clinvar(hgvs_c_full, ncbi_api_key)
     clinvar_text = format_clinvar_for_prompt(clinvar_data)
 
-    # Load extractions and metadata for each paper. PDF bytes and Markdown are
-    # cached for post-LLM citation resolution: PdfIndex takes both (markdown
-    # denoises the indexed PDF chars). Both files are produced together by the
-    # pipeline; a missing markdown.md at this point is a storage corruption and
-    # surfaces as FileNotFoundError below.
+    # Load extractions and metadata for each paper. PDF bytes and markdown
+    # are NOT loaded here — the post-LLM citation resolver loads the paper's
+    # pre-built `pdf_index.pkl.zst` directly from storage, so this step only
+    # needs the LLM inputs.
     evidence_extractions: list[dict[str, Any]] = []
-    pdf_bytes_cache: dict[str, bytes] = {}
-    markdown_cache: dict[str, str] = {}
     metadata_cache: dict[str, dict[str, Any]] = {}
 
     for doi in dois:
@@ -252,8 +244,6 @@ async def aggregate_evidence_async(
             log.info('Skipping %s: variant not discussed', doi)
             continue
 
-        pdf_bytes_cache[doi] = read_bytes(paper_url(base, doi, 'source.pdf'))
-        markdown_cache[doi] = read_text(paper_url(base, doi, 'markdown.md'))
         metadata = read_json(paper_url(base, doi, 'metadata.json'))
         metadata_cache[doi] = metadata
 
@@ -332,7 +322,7 @@ async def aggregate_evidence_async(
     # Post-LLM: resolve quotes to bboxes, replace paper_id with DOI
     aggregate_dict = output.model_dump()
     with logfire.span('flowa.resolve_citations', paper_count=len(paper_id_to_doi)):
-        resolve_aggregate_citations(aggregate_dict, paper_id_to_doi, pdf_bytes_cache, markdown_cache, metadata_cache)
+        resolve_aggregate_citations(aggregate_dict, paper_id_to_doi, base, metadata_cache)
 
     # Store structured aggregation result
     write_json(aggregation_url, with_schema_version(aggregate_dict, AGGREGATION_SCHEMA_VERSION))

{flowapy-0.1.3 → flowapy-0.2.0}/src/flowa/convert.py

@@ -17,9 +17,11 @@ from pydantic_ai import Agent
 from pydantic_ai.messages import BinaryContent
 
 from flowa.models import create_model, get_model_settings
+from flowa.pdf_index_cache import build as build_pdf_index_payload
+from flowa.pdf_index_cache import serialize as serialize_pdf_index_payload
 from flowa.prompts import load_text_prompt
 from flowa.settings import ModelConfig, Settings
-from flowa.storage import exists, paper_url, read_bytes, write_bytes, write_text
+from flowa.storage import exists, paper_url, read_bytes, read_text, write_bytes, write_text
 
 log = logging.getLogger(__name__)
 
@@ -120,14 +122,22 @@ async def transcribe(
 
 
 async def convert_paper_async(base: str, doi: str, model: ModelConfig, prompt_set: str = 'generic') -> None:
-    """Convert a single paper's PDF to Markdown.
+    """Convert a single paper's PDF to Markdown and persist its `PdfIndex`.
 
-    Reads PDF from papers/{encoded_doi}/source.pdf in object storage.
-    Stores result to papers/{encoded_doi}/markdown.md.
+    Reads PDF from papers/{encoded_doi}/source.pdf and writes
+    papers/{encoded_doi}/markdown.md plus papers/{encoded_doi}/pdf_index.pkl.zst
+    (consumed by the gateway's resolve endpoint).
+
+    Either artifact can be missing independently — if a previous run failed
+    or pre-dates the pdf_index step, the next call fills in only what's
+    missing without re-transcribing or re-building work that's already done.
     """
     md_url = paper_url(base, doi, 'markdown.md')
+    index_url = paper_url(base, doi, 'pdf_index.pkl.zst')
+    md_needed = not exists(md_url)
+    index_needed = not exists(index_url)
 
-    if exists(md_url):
+    if not md_needed and not index_needed:
         log.info('Already converted: %s', md_url)
         return
 
@@ -138,21 +148,36 @@ async def convert_paper_async(base: str, doi: str, model: ModelConfig, prompt_se
         log.info('Skipping DOI %s: PDF not available', doi)
         return
 
-    log.info(
-        'Converting DOI %s (%d bytes, model: %s, chunk: %d pages)', doi, len(pdf_bytes), model.name, PAGES_PER_CHUNK
-    )
-
-    prompt = load_text_prompt('transcription', prompt_set)
-    t0 = time.monotonic()
-    result = await transcribe(pdf_bytes, model=model, prompt=prompt, page_count=PAGES_PER_CHUNK)
-    elapsed = time.monotonic() - t0
-
-    write_text(md_url, result.markdown)
-
-    raw_url = paper_url(base, doi, 'convert_raw.json')
-    write_bytes(raw_url, json.dumps(result.all_messages).encode())
-
-    log.info('Converted DOI %s: %d chars in %.1fs', doi, len(result.markdown), elapsed)
+    markdown: str | None = None
+    if md_needed:
+        log.info(
+            'Converting DOI %s (%d bytes, model: %s, chunk: %d pages)', doi, len(pdf_bytes), model.name, PAGES_PER_CHUNK
+        )
+        prompt = load_text_prompt('transcription', prompt_set)
+        t0 = time.monotonic()
+        result = await transcribe(pdf_bytes, model=model, prompt=prompt, page_count=PAGES_PER_CHUNK)
+        elapsed = time.monotonic() - t0
+
+        markdown = result.markdown
+        write_text(md_url, markdown)
+        write_bytes(paper_url(base, doi, 'convert_raw.json'), json.dumps(result.all_messages).encode())
+        log.info('Converted DOI %s: %d chars in %.1fs', doi, len(markdown), elapsed)
+
+    if index_needed:
+        # PdfIndex construction is CPU-bound (~8s on the deployed gateway
+        # hardware) and dominates per-call latency at `/api/v1/resolve` if
+        # rebuilt on every call. Pay the cost here once per paper and ship
+        # the result; see `flowa.pdf_index_cache` for the storage format.
+        # `asyncio.to_thread` keeps the rest of the convert pipeline (other
+        # papers being transcribed concurrently) unblocked.
+        if markdown is None:  # index missing but markdown already on disk
+            markdown = read_text(md_url)
+        t0 = time.monotonic()
+        blob = await asyncio.to_thread(
+            lambda: serialize_pdf_index_payload(build_pdf_index_payload(pdf_bytes, markdown))
+        )
+        write_bytes(index_url, blob)
+        log.info('Wrote pdf_index for DOI %s: %.1f MB in %.1fs', doi, len(blob) / 1e6, time.monotonic() - t0)
 
 
 def convert_paper(

flowapy-0.2.0/src/flowa/pdf_index_cache.py

@@ -0,0 +1,133 @@
+"""Build, serialise, and load cached `PdfIndex` artifacts.
+
+The gateway's per-call cost is dominated by `PdfIndex(pdf_bytes)` construction
+— ~8s on the deployed gateway hardware for a typical paper, against ~300ms
+for the actual quote alignment afterwards. To avoid paying that on every
+`/api/v1/resolve` call, the pipeline persists the constructed index at
+`papers/{encoded_doi}/pdf_index.pkl.zst` so the gateway can load instead of
+rebuild.
+
+On-the-wire format: zstd-compressed pickle of a single dict:
+
+    {
+        "format_version":   int,    # bumped when this module changes what it serialises
+        "source_pdf_sha256": str,   # sha256 hex digest of source.pdf bytes
+        "pdf_index":        PdfIndex,
+    }
+
+The header fields let `deserialize` reject artifacts that are out of sync
+with the current source.pdf or with the current cache format — a stale
+pickle would silently produce wrong bboxes. The format version is *our*
+contract over the persisted shape: bump it when this module changes which
+fields it stores or how it stores them. It is not tied to anchorite's
+release cadence — anchorite patch/minor releases that preserve the pickle
+shape of `PdfIndex` deserialise fine.
+
+zstd level 3 was chosen empirically (see specs/supplements.md): ~5x faster
+compression than gzip -6 at comparable ratio (~1/5 of the pickled size),
+and a touch faster decompression. The pipeline pays the compress cost once
+per paper; the gateway pays the decompress cost on every cold load, so
+optimising for both directions matters.
+
+Anchorite documents that `PdfIndex` pickles cleanly (state is str/bytes/
+list[int]/frozen dataclasses); no custom reducers needed.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import pickle
+from dataclasses import dataclass
+
+import zstandard
+from anchorite import PdfIndex
+
+# Bump when the persisted shape changes: new fields, removed fields, semantic
+# meaning changes. Anchorite version bumps that preserve `PdfIndex` pickle
+# compatibility do NOT require a bump here — those deserialise correctly
+# under the existing format. If a new anchorite release changes `PdfIndex`'s
+# internals such that old pickles still load but produce different bboxes,
+# bump this to force a re-backfill.
+FORMAT_VERSION = 1
+
+ZSTD_LEVEL = 3
+PICKLE_PROTOCOL = pickle.HIGHEST_PROTOCOL
+
+
+class StaleIndexError(Exception):
+    """Persisted artifact's header doesn't match the runtime expectation.
+
+    Raised by `deserialize` when the pickle was written under a different
+    `FORMAT_VERSION` or against a source.pdf with a different sha256. The
+    caller decides whether to rebuild or surface an error — this module
+    deliberately doesn't fall back, because silent rebuild masks pipeline
+    drift.
+    """
+
+
+@dataclass(frozen=True)
+class PdfIndexPayload:
+    """In-memory view of the persisted artifact."""
+
+    format_version: int
+    source_pdf_sha256: str
+    pdf_index: PdfIndex
+
+
+def build(pdf_bytes: bytes, markdown: str) -> PdfIndexPayload:
+    """Construct a PdfIndex pinned to its source PDF by sha256.
+
+    `markdown` is the paper's transcription, used by anchorite to denoise the
+    indexed PDF char string (drop running heads, page numbers, footnote
+    markers the LLM didn't transcribe), which improves quote alignment.
+    """
+    # markdown is threaded through but not yet forwarded to PdfIndex: the
+    # anchorite-#19 markdown denoise drops entire pages of atoms when the
+    # markdown reorders content relative to PDF page order. Switch to
+    # `markdown=markdown` once the upstream fix lands — and bump
+    # FORMAT_VERSION + re-backfill, since the denoised index resolves quotes
+    # against a different cached char string (existing pickles would silently
+    # produce different bboxes).
+    return PdfIndexPayload(
+        format_version=FORMAT_VERSION,
+        source_pdf_sha256=hashlib.sha256(pdf_bytes).hexdigest(),
+        pdf_index=PdfIndex(pdf_bytes, markdown=None),
+    )
+
+
+def serialize(payload: PdfIndexPayload) -> bytes:
+    """Pickle + zstd-compress for upload to S3."""
+    pkl = pickle.dumps(
+        {
+            'format_version': payload.format_version,
+            'source_pdf_sha256': payload.source_pdf_sha256,
+            'pdf_index': payload.pdf_index,
+        },
+        protocol=PICKLE_PROTOCOL,
+    )
+    return zstandard.ZstdCompressor(level=ZSTD_LEVEL).compress(pkl)
+
+
+def deserialize(blob: bytes, *, expected_pdf_sha256: str | None = None) -> PdfIndexPayload:
+    """Decompress + unpickle. Verifies the header before returning.
+
+    The runtime `FORMAT_VERSION` is always checked. The `source_pdf_sha256`
+    check is only performed when the caller supplies a value to compare
+    against — the gateway typically skips it (would require fetching source.pdf
+    just to hash it). Callers that already have the source bytes pass the
+    digest in to catch pipeline drift.
+    """
+    pkl = zstandard.ZstdDecompressor().decompress(blob)
+    raw = pickle.loads(pkl)
+    payload = PdfIndexPayload(
+        format_version=raw['format_version'],
+        source_pdf_sha256=raw['source_pdf_sha256'],
+        pdf_index=raw['pdf_index'],
+    )
+    if payload.format_version != FORMAT_VERSION:
+        raise StaleIndexError(f'format version mismatch: pickle={payload.format_version!r} runtime={FORMAT_VERSION!r}')
+    if expected_pdf_sha256 is not None and payload.source_pdf_sha256 != expected_pdf_sha256:
+        raise StaleIndexError(
+            f'source.pdf hash mismatch: pickle={payload.source_pdf_sha256!r} actual={expected_pdf_sha256!r}'
+        )
+    return payload