flowapy 0.4.1__tar.gz → 0.4.3__tar.gz

{flowapy-0.4.1 → flowapy-0.4.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowapy
-Version: 0.4.1
+Version: 0.4.3
 Summary: Variant literature assessment pipeline with AI extraction
 Project-URL: Homepage, https://github.com/populationgenomics/flowa
 Project-URL: Source, https://github.com/populationgenomics/flowa
@@ -115,12 +115,20 @@ The PyPI distribution is named `flowapy` (the `flowa` name was already taken), b
 
 ## Usage
 
+A variant is passed as `--variant-spec`, either inline JSON or `@path/to/spec.json`. The
+envelope is `{"schema_version": 1, "variants": [{ "kind": "hgvs_c", "transcript": ...,
+"hgvs_c": ... }]}` — `hgvs_c` is the bare coding change with no transcript prefix. The gene
+symbol and genomic coordinates are derived downstream, so they aren't part of the spec.
+
 ```bash
+# Example variant spec (GAA / Pompe, NM_000152.5:c.2238G>C)
+SPEC='{"schema_version":1,"variants":[{"kind":"hgvs_c","transcript":"NM_000152.5","hgvs_c":"c.2238G>C"}]}'
+
 # Full pipeline
-flowa run --variant-id VAR123 --gene GAA --hgvs-c "NM_000152.5:c.2238G>C" --source litvar
+flowa run --variant-id VAR123 --variant-spec "$SPEC" --source litvar
 
 # Individual steps (for debugging)
-flowa query --variant-id VAR123 --gene GAA --hgvs-c "NM_000152.5:c.2238G>C" --source litvar
+flowa query --variant-id VAR123 --variant-spec "$SPEC" --source litvar
 flowa download --doi '10.1038/s41586-020-2308-7'
 flowa convert --doi '10.1038/s41586-020-2308-7'
 flowa extract --variant-id VAR123 --doi '10.1038/s41586-020-2308-7'
@@ -294,7 +302,9 @@ Run the pipeline directly from a checkout (the [Quickstart](#quickstart) demo wr
 export FLOWA_STORAGE_BASE=file:///tmp/flowa
 export FLOWA_CONVERT_MODEL__NAME=bedrock:au.anthropic.claude-sonnet-4-6
 export FLOWA_EXTRACTION_MODEL__NAME=bedrock:au.anthropic.claude-opus-4-6
-uv run flowa run --variant-id test --gene GAA --hgvs-c "NM_000152.5:c.2238G>C" --source litvar
+uv run flowa run --variant-id test \
+  --variant-spec '{"schema_version":1,"variants":[{"kind":"hgvs_c","transcript":"NM_000152.5","hgvs_c":"c.2238G>C"}]}' \
+  --source litvar
 ```
 
 ### Docker
@@ -306,7 +316,9 @@ docker run \
   -e FLOWA_CONVERT_MODEL__NAME=bedrock:au.anthropic.claude-sonnet-4-6 \
   -e FLOWA_EXTRACTION_MODEL__NAME=bedrock:au.anthropic.claude-opus-4-6 \
   -e AWS_REGION=ap-southeast-2 \
-  flowa run --variant-id VAR123 --gene GAA --hgvs-c "NM_000152.5:c.2238G>C" --source litvar
+  flowa run --variant-id VAR123 \
+  --variant-spec '{"schema_version":1,"variants":[{"kind":"hgvs_c","transcript":"NM_000152.5","hgvs_c":"c.2238G>C"}]}' \
+  --source litvar
 ```
 
 ### AWS Batch
@@ -341,6 +353,6 @@ aws batch submit-job \
   --job-definition flowa-worker \
   --job-queue flowa-queue \
   --container-overrides '{
-    "command": ["run", "--variant-id", "VAR123", "--gene", "GAA", "--hgvs-c", "NM_000152.5:c.2238G>C", "--source", "litvar"]
+    "command": ["run", "--variant-id", "VAR123", "--variant-spec", "{\"schema_version\":1,\"variants\":[{\"kind\":\"hgvs_c\",\"transcript\":\"NM_000152.5\",\"hgvs_c\":\"c.2238G>C\"}]}", "--source", "litvar"]
   }'
 ```

{flowapy-0.4.1 → flowapy-0.4.3}/README.md

@@ -53,12 +53,20 @@ The PyPI distribution is named `flowapy` (the `flowa` name was already taken), b
 
 ## Usage
 
+A variant is passed as `--variant-spec`, either inline JSON or `@path/to/spec.json`. The
+envelope is `{"schema_version": 1, "variants": [{ "kind": "hgvs_c", "transcript": ...,
+"hgvs_c": ... }]}` — `hgvs_c` is the bare coding change with no transcript prefix. The gene
+symbol and genomic coordinates are derived downstream, so they aren't part of the spec.
+
 ```bash
+# Example variant spec (GAA / Pompe, NM_000152.5:c.2238G>C)
+SPEC='{"schema_version":1,"variants":[{"kind":"hgvs_c","transcript":"NM_000152.5","hgvs_c":"c.2238G>C"}]}'
+
 # Full pipeline
-flowa run --variant-id VAR123 --gene GAA --hgvs-c "NM_000152.5:c.2238G>C" --source litvar
+flowa run --variant-id VAR123 --variant-spec "$SPEC" --source litvar
 
 # Individual steps (for debugging)
-flowa query --variant-id VAR123 --gene GAA --hgvs-c "NM_000152.5:c.2238G>C" --source litvar
+flowa query --variant-id VAR123 --variant-spec "$SPEC" --source litvar
 flowa download --doi '10.1038/s41586-020-2308-7'
 flowa convert --doi '10.1038/s41586-020-2308-7'
 flowa extract --variant-id VAR123 --doi '10.1038/s41586-020-2308-7'
@@ -232,7 +240,9 @@ Run the pipeline directly from a checkout (the [Quickstart](#quickstart) demo wr
 export FLOWA_STORAGE_BASE=file:///tmp/flowa
 export FLOWA_CONVERT_MODEL__NAME=bedrock:au.anthropic.claude-sonnet-4-6
 export FLOWA_EXTRACTION_MODEL__NAME=bedrock:au.anthropic.claude-opus-4-6
-uv run flowa run --variant-id test --gene GAA --hgvs-c "NM_000152.5:c.2238G>C" --source litvar
+uv run flowa run --variant-id test \
+  --variant-spec '{"schema_version":1,"variants":[{"kind":"hgvs_c","transcript":"NM_000152.5","hgvs_c":"c.2238G>C"}]}' \
+  --source litvar
 ```
 
 ### Docker
@@ -244,7 +254,9 @@ docker run \
   -e FLOWA_CONVERT_MODEL__NAME=bedrock:au.anthropic.claude-sonnet-4-6 \
   -e FLOWA_EXTRACTION_MODEL__NAME=bedrock:au.anthropic.claude-opus-4-6 \
   -e AWS_REGION=ap-southeast-2 \
-  flowa run --variant-id VAR123 --gene GAA --hgvs-c "NM_000152.5:c.2238G>C" --source litvar
+  flowa run --variant-id VAR123 \
+  --variant-spec '{"schema_version":1,"variants":[{"kind":"hgvs_c","transcript":"NM_000152.5","hgvs_c":"c.2238G>C"}]}' \
+  --source litvar
 ```
 
 ### AWS Batch
@@ -279,6 +291,6 @@ aws batch submit-job \
   --job-definition flowa-worker \
   --job-queue flowa-queue \
   --container-overrides '{
-    "command": ["run", "--variant-id", "VAR123", "--gene", "GAA", "--hgvs-c", "NM_000152.5:c.2238G>C", "--source", "litvar"]
+    "command": ["run", "--variant-id", "VAR123", "--variant-spec", "{\"schema_version\":1,\"variants\":[{\"kind\":\"hgvs_c\",\"transcript\":\"NM_000152.5\",\"hgvs_c\":\"c.2238G>C\"}]}", "--source", "litvar"]
   }'
 ```

{flowapy-0.4.1 → flowapy-0.4.3}/examples/demo/README.md

@@ -187,7 +187,8 @@ if [ -n "${BEDROCK_INFERENCE_PROFILE:-}" ]; then
   export FLOWA_EXTRACTION_MODEL__BEDROCK_INFERENCE_PROFILE="$BEDROCK_INFERENCE_PROFILE"
   export FLOWA_CONVERT_MODEL__BEDROCK_INFERENCE_PROFILE="$BEDROCK_INFERENCE_PROFILE"
 fi
-uv run flowa run --variant-id $VARIANT --gene RYR2 --hgvs-c "NM_001035.3:c.14174A>G"
+uv run flowa run --variant-id $VARIANT \
+  --variant-spec '{"schema_version":1,"variants":[{"kind":"hgvs_c","transcript":"NM_001035.3","hgvs_c":"c.14174A>G"}]}'
 ```
 
 Open `http://localhost:7700/viewer/$VARIANT/acmg_classification` (with

{flowapy-0.4.1 → flowapy-0.4.3}/examples/demo-gateway/README.md

@@ -35,7 +35,7 @@ imports as load-bearing and the rest as something to replace.
 
 | Method | Path | Behaviour |
 |---|---|---|
-| `POST` | `/runs` | Body `{ variant_id, gene, hgvs_c }`. Spawns the pipeline as an `asyncio` task and returns `{ run_id, started_at, status }` immediately. 409 if a run is already in flight for the variant; 429 at the concurrency cap. |
+| `POST` | `/runs` | Body `{ variant_id, variant_spec }`, where `variant_spec` is the `flowa.schema.VariantSpec` envelope. Spawns the pipeline as an `asyncio` task and returns `{ run_id, started_at, status }` immediately. 409 if a run is already in flight for the variant; 429 at the concurrency cap. |
 | `GET`  | `/runs/active?variant_id=X` | Most recent run for `variant_id` as `{ run_id, started_at, status }`. 404 if none. `status` is `running` while in flight; `success` / `error` once terminal. |
 | `POST` | `/resolve-citations` | Body `{ citations: [{ doi, quotes[] }] }`. Builds fsspec-based PDF and Markdown loaders rooted at `DEMO_DATA_DIR/papers/{encoded_doi}/` and calls `flowa.resolve.resolve_citations(...)` synchronously. Returns `{ resolved: { doi: { quote: HighlightBbox[] } }, errors: { doi: string } }`. DOIs whose source PDF or markdown can't be loaded land in `errors`; quotes the aligner searches for but can't locate land as empty arrays in `resolved`. |
 | `GET`  | `/health` | `{ "status": "ok" }`. |

{flowapy-0.4.1 → flowapy-0.4.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "flowapy"
-version = "0.4.1"
+version = "0.4.3"
 description = "Variant literature assessment pipeline with AI extraction"
 readme = "README.md"
 requires-python = "==3.13.*"

{flowapy-0.4.1 → flowapy-0.4.3}/src/flowa/content_validation.py

@@ -13,8 +13,11 @@ anchorite-backed resolver, not here.
 
 Keep in sync with `packages/chat-service/src/chat.ts` (`validateArtifactContent`).
 One rule here has no edit-time counterpart — `claim_quote_not_in_extraction`:
-every claim citation quote must be one the extraction surfaced. It's the same
-kind of exact, model-vs-model check; chat-service can't run it because an edit
+every claim citation quote must be byte-present in a quote the extraction
+surfaced — exact, or a contiguous substring of one (the aggregator legitimately
+trims a long surfaced passage to its load-bearing clause; that span is still
+verbatim source text and still resolves to a highlight downstream). It is still
+byte-exact, not a fuzzy threshold. chat-service can't run it because an edit
 session has no extraction input to compare against.
 """
 
@@ -28,6 +31,20 @@ from flowa.artifact import CategoryResult
 _CITE_LINK_RE = re.compile(r'\[[^\]]*\]\(#cite:([^ )"]+)(?:\s+"([^"]*)")?\)')
 
 
+def _quote_grounded(quote: str, allowed: set[str]) -> bool:
+    """True if `quote` is byte-present in any extraction quote the model was fed.
+
+    Exact set-membership (fast path), or a contiguous substring of a surfaced
+    quote — the aggregator trimming a long extraction passage to its load-bearing
+    clause is grounded, not fabricated. An empty quote grounds nothing.
+    """
+    if not quote:
+        return False
+    if quote in allowed:
+        return True
+    return any(quote in surfaced for surfaced in allowed)
+
+
 def validate_aggregate_category(
     cat_result: CategoryResult,
     valid_paper_ids: set[str],
@@ -90,11 +107,18 @@ def validate_aggregate_category(
             )
             break
 
-    # Every claim citation quote must be one the extraction surfaced.
+    # Every claim citation quote must be grounded in what the extraction surfaced.
+    # The aggregate model is fed only the extraction quotes (never the paper
+    # Markdown), so a claim quote is grounded iff it is byte-present in a surfaced
+    # quote for that paper. A contiguous SUBSTRING of a surfaced quote counts:
+    # the extraction deliberately captures long passages (a full sentence / table
+    # row) and the aggregator legitimately trims to the load-bearing clause, which
+    # is still verbatim source text and still resolves to a highlight downstream.
+    # Only a quote contained in NO surfaced quote is a fabrication/alteration.
     for claim in cat_result.claims:
         allowed = extraction_quotes_by_paper.get(claim.paper_id, set())
         for citation in claim.citations:
-            if citation.quote not in allowed:
+            if not _quote_grounded(citation.quote, allowed):
                 errors.append(
                     (
                         'claim_quote_not_in_extraction',

{flowapy-0.4.1 → flowapy-0.4.3}/src/flowa/convert.py

@@ -22,6 +22,7 @@ from flowa.assemble import assemble_paper
 from flowa.models import create_model, get_model_settings
 from flowa.pdf_index_cache import build as build_pdf_index_payload
 from flowa.pdf_index_cache import serialize as serialize_pdf_index_payload
+from flowa.pdfium import run_pdfium
 from flowa.prompts import load_text_prompt
 from flowa.settings import ModelConfig, Settings
 from flowa.storage import (
@@ -174,7 +175,10 @@ async def transcribe(
     Splits the PDF into chunks (``page_count`` pages each), converts each chunk
     concurrently, then joins the results with ``<!--page-->`` separators.
     """
-    doc_chunks = list(chunks(pdf_bytes, page_count=page_count))
+    # chunks() drives PDFium (page splitting); route it through the single
+    # PDFium lane and fully materialise the generator there so no PDFium call
+    # leaks onto the event-loop thread during iteration.
+    doc_chunks = await run_pdfium(lambda: list(chunks(pdf_bytes, page_count=page_count)))
 
     coros = [_generate_markdown(chunk.data, model, prompt) for chunk in doc_chunks]
     chunk_results = list(await asyncio.gather(*coros))
@@ -295,9 +299,11 @@ async def convert_paper_async(base: str, doi: str, model: ModelConfig, prompt_se
         full_pdf_bytes = read_bytes(full_pdf_url(base, doi))
         markdown = read_text(full_md_url(base, doi))
         t0 = time.monotonic()
-        blob = await asyncio.to_thread(
-            lambda: serialize_pdf_index_payload(build_pdf_index_payload(full_pdf_bytes, markdown))
-        )
+        # PdfIndex construction drives PDFium, so it goes on the single PDFium
+        # lane; the zstd serialisation is plain CPU work, so it goes back to the
+        # general pool and doesn't hold the scarce lane while other papers wait.
+        payload = await run_pdfium(lambda: build_pdf_index_payload(full_pdf_bytes, markdown))
+        blob = await asyncio.to_thread(lambda: serialize_pdf_index_payload(payload))
         write_bytes(index_url, blob)
         log.info('Wrote pdf_index for DOI %s: %.1f MB in %.1fs', doi, len(blob) / 1e6, time.monotonic() - t0)
 

flowapy-0.4.3/src/flowa/pdfium.py

@@ -0,0 +1,43 @@
+"""Process-wide single-thread lane for all PDFium (pypdfium2) work.
+
+PDFium holds global process state and is not thread-safe. anchorite documents
+that `PdfIndex(...)` construction — and, by the same token, `chunks()`'s page
+splitting — must be serialised by the caller (see `anchorite.PdfIndex`:
+"construction is *not* thread-safe (PDFium isn't); serialise concurrent
+`PdfIndex(...)` calls in the caller").
+
+flowa drives up to `--convert-concurrency` papers at once, and its two PDFium
+entry points run on different threads: chunk-splitting on the event-loop thread
+and index construction on a worker thread. Left unserialised, those calls hit
+PDFium's shared state from several threads simultaneously and corrupt it — an
+intermittent native crash (SIGTRAP) that spikes on index-dense runs (e.g. a
+backfill where most papers already have transcriptions, so convert is mostly
+the index step). The GIL does not protect them: ctypes releases it across the
+PDFium FFI calls.
+
+Routing every PDFium-touching callable through this one-worker executor makes
+them mutually exclusive *by construction* — there is no lock to forget, and
+PDFium is physically isolated from the general thread pool. Throughput is
+unaffected: the serialised PDFium time (chunk-split is tens of ms, index build
+0.2-8s) hides under the much longer, fully-concurrent Bedrock transcription.
+"""
+
+import asyncio
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
+
+# One worker => every callable submitted here runs to completion before the
+# next starts, so all PDFium work is serialised. Module-level (process-global)
+# because PDFium's state is process-global; the lone idle thread is negligible.
+_PDFIUM_EXECUTOR = ThreadPoolExecutor(max_workers=1, thread_name_prefix='pdfium')
+
+
+async def run_pdfium[T](fn: Callable[[], T]) -> T:
+    """Run a synchronous PDFium-touching callable on the single PDFium thread.
+
+    `fn` must perform *all* of its PDFium work before returning — fully
+    materialise any lazy generator (e.g. `list(chunks(...))`), don't hand back
+    an iterator that would call into PDFium later on another thread.
+    """
+    loop = asyncio.get_running_loop()
+    return await loop.run_in_executor(_PDFIUM_EXECUTOR, fn)

{flowapy-0.4.1 → flowapy-0.4.3}/tests/test_content_validation.py

@@ -162,3 +162,24 @@ def test_claim_quote_must_come_from_extraction() -> None:
         extraction_quotes_by_paper={'Smith2024': {'a quote the extraction actually surfaced'}},
     )
     assert 'claim_quote_not_in_extraction' in _rules(errors)
+
+
+def test_claim_quote_substring_of_extraction_passes() -> None:
+    # The extraction surfaces a long passage; the aggregator legitimately trims
+    # it to its load-bearing clause. That contiguous substring is still verbatim
+    # source text (it resolves to a highlight downstream), so it is grounded.
+    surfaced = (
+        'Each parent was heterozygous for one allele, and an unaffected sister did not have a mutation at either site.'
+    )
+    trimmed = 'an unaffected sister did not have a mutation at either site'
+    cat = _cat(
+        notes=f'Non-carrier sib ([sib](#cite:Smith2024 "{trimmed}")).',
+        papers=('Smith2024',),
+        claims=(_claim('Smith2024', trimmed),),
+    )
+    errors = validate_aggregate_category(
+        cat,
+        valid_paper_ids={'Smith2024'},
+        extraction_quotes_by_paper={'Smith2024': {surfaced}},
+    )
+    assert errors == []

flowapy-0.4.3/tests/test_pdfium.py

@@ -0,0 +1,47 @@
+"""Tests for `flowa.pdfium` — the single-thread PDFium lane.
+
+PDFium is not thread-safe, so `run_pdfium` must serialise every callable it
+runs onto one dedicated worker thread. These tests assert that property
+directly — concurrently-submitted callables never overlap in time and all run
+on the same thread — which makes the mutual-exclusion guarantee verifiable
+without provoking a native PDFium race.
+"""
+
+import asyncio
+import threading
+import time
+from itertools import pairwise
+
+from flowa.pdfium import run_pdfium
+
+
+def _record(hold: float) -> tuple[int, str, float, float]:
+    """Return (thread_id, thread_name, enter, exit) around a short sleep.
+
+    Runs on the PDFium worker thread; the sleep widens each call's interval so
+    a second worker would produce a detectable overlap.
+    """
+    enter = time.perf_counter()
+    time.sleep(hold)
+    return (threading.get_ident(), threading.current_thread().name, enter, time.perf_counter())
+
+
+def test_run_pdfium_serialises_concurrent_calls() -> None:
+    hold = 0.02
+    n = 8
+
+    async def drive() -> list[tuple[int, str, float, float]]:
+        return list(await asyncio.gather(*(run_pdfium(lambda: _record(hold)) for _ in range(n))))
+
+    records = asyncio.run(drive())
+    assert len(records) == n
+
+    # Every callable ran on the one PDFium worker thread.
+    assert len({tid for tid, _, _, _ in records}) == 1
+    assert all(name.startswith('pdfium') for _, name, _, _ in records)
+
+    # Intervals are pairwise non-overlapping: sorted by entry, each call starts
+    # only after the previous one finished — a single worker runs them serially.
+    intervals = sorted((enter, exit_) for _, _, enter, exit_ in records)
+    for (_, prev_exit), (next_enter, _) in pairwise(intervals):
+        assert next_enter >= prev_exit

{flowapy-0.4.1 → flowapy-0.4.3}/uv.lock

@@ -492,7 +492,7 @@ wheels = [
 
 [[package]]
 name = "flowapy"
-version = "0.4.1"
+version = "0.4.3"
 source = { editable = "." }
 dependencies = [
     { name = "anchorite" },