flowapy 0.4.1__tar.gz → 0.4.2__tar.gz

{flowapy-0.4.1 → flowapy-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowapy
-Version: 0.4.1
+Version: 0.4.2
 Summary: Variant literature assessment pipeline with AI extraction
 Project-URL: Homepage, https://github.com/populationgenomics/flowa
 Project-URL: Source, https://github.com/populationgenomics/flowa

{flowapy-0.4.1 → flowapy-0.4.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "flowapy"
-version = "0.4.1"
+version = "0.4.2"
 description = "Variant literature assessment pipeline with AI extraction"
 readme = "README.md"
 requires-python = "==3.13.*"

{flowapy-0.4.1 → flowapy-0.4.2}/src/flowa/convert.py

@@ -22,6 +22,7 @@ from flowa.assemble import assemble_paper
 from flowa.models import create_model, get_model_settings
 from flowa.pdf_index_cache import build as build_pdf_index_payload
 from flowa.pdf_index_cache import serialize as serialize_pdf_index_payload
+from flowa.pdfium import run_pdfium
 from flowa.prompts import load_text_prompt
 from flowa.settings import ModelConfig, Settings
 from flowa.storage import (
@@ -174,7 +175,10 @@ async def transcribe(
     Splits the PDF into chunks (``page_count`` pages each), converts each chunk
     concurrently, then joins the results with ``<!--page-->`` separators.
     """
-    doc_chunks = list(chunks(pdf_bytes, page_count=page_count))
+    # chunks() drives PDFium (page splitting); route it through the single
+    # PDFium lane and fully materialise the generator there so no PDFium call
+    # leaks onto the event-loop thread during iteration.
+    doc_chunks = await run_pdfium(lambda: list(chunks(pdf_bytes, page_count=page_count)))
 
     coros = [_generate_markdown(chunk.data, model, prompt) for chunk in doc_chunks]
     chunk_results = list(await asyncio.gather(*coros))
@@ -295,9 +299,11 @@ async def convert_paper_async(base: str, doi: str, model: ModelConfig, prompt_se
         full_pdf_bytes = read_bytes(full_pdf_url(base, doi))
         markdown = read_text(full_md_url(base, doi))
         t0 = time.monotonic()
-        blob = await asyncio.to_thread(
-            lambda: serialize_pdf_index_payload(build_pdf_index_payload(full_pdf_bytes, markdown))
-        )
+        # PdfIndex construction drives PDFium, so it goes on the single PDFium
+        # lane; the zstd serialisation is plain CPU work, so it goes back to the
+        # general pool and doesn't hold the scarce lane while other papers wait.
+        payload = await run_pdfium(lambda: build_pdf_index_payload(full_pdf_bytes, markdown))
+        blob = await asyncio.to_thread(lambda: serialize_pdf_index_payload(payload))
         write_bytes(index_url, blob)
         log.info('Wrote pdf_index for DOI %s: %.1f MB in %.1fs', doi, len(blob) / 1e6, time.monotonic() - t0)
 

flowapy-0.4.2/src/flowa/pdfium.py

@@ -0,0 +1,43 @@
+"""Process-wide single-thread lane for all PDFium (pypdfium2) work.
+
+PDFium holds global process state and is not thread-safe. anchorite documents
+that `PdfIndex(...)` construction — and, by the same token, `chunks()`'s page
+splitting — must be serialised by the caller (see `anchorite.PdfIndex`:
+"construction is *not* thread-safe (PDFium isn't); serialise concurrent
+`PdfIndex(...)` calls in the caller").
+
+flowa drives up to `--convert-concurrency` papers at once, and its two PDFium
+entry points run on different threads: chunk-splitting on the event-loop thread
+and index construction on a worker thread. Left unserialised, those calls hit
+PDFium's shared state from several threads simultaneously and corrupt it — an
+intermittent native crash (SIGTRAP) that spikes on index-dense runs (e.g. a
+backfill where most papers already have transcriptions, so convert is mostly
+the index step). The GIL does not protect them: ctypes releases it across the
+PDFium FFI calls.
+
+Routing every PDFium-touching callable through this one-worker executor makes
+them mutually exclusive *by construction* — there is no lock to forget, and
+PDFium is physically isolated from the general thread pool. Throughput is
+unaffected: the serialised PDFium time (chunk-split is tens of ms, index build
+0.2-8s) hides under the much longer, fully-concurrent Bedrock transcription.
+"""
+
+import asyncio
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor
+
+# One worker => every callable submitted here runs to completion before the
+# next starts, so all PDFium work is serialised. Module-level (process-global)
+# because PDFium's state is process-global; the lone idle thread is negligible.
+_PDFIUM_EXECUTOR = ThreadPoolExecutor(max_workers=1, thread_name_prefix='pdfium')
+
+
+async def run_pdfium[T](fn: Callable[[], T]) -> T:
+    """Run a synchronous PDFium-touching callable on the single PDFium thread.
+
+    `fn` must perform *all* of its PDFium work before returning — fully
+    materialise any lazy generator (e.g. `list(chunks(...))`), don't hand back
+    an iterator that would call into PDFium later on another thread.
+    """
+    loop = asyncio.get_running_loop()
+    return await loop.run_in_executor(_PDFIUM_EXECUTOR, fn)

flowapy-0.4.2/tests/test_pdfium.py

@@ -0,0 +1,47 @@
+"""Tests for `flowa.pdfium` — the single-thread PDFium lane.
+
+PDFium is not thread-safe, so `run_pdfium` must serialise every callable it
+runs onto one dedicated worker thread. These tests assert that property
+directly — concurrently-submitted callables never overlap in time and all run
+on the same thread — which makes the mutual-exclusion guarantee verifiable
+without provoking a native PDFium race.
+"""
+
+import asyncio
+import threading
+import time
+from itertools import pairwise
+
+from flowa.pdfium import run_pdfium
+
+
+def _record(hold: float) -> tuple[int, str, float, float]:
+    """Return (thread_id, thread_name, enter, exit) around a short sleep.
+
+    Runs on the PDFium worker thread; the sleep widens each call's interval so
+    a second worker would produce a detectable overlap.
+    """
+    enter = time.perf_counter()
+    time.sleep(hold)
+    return (threading.get_ident(), threading.current_thread().name, enter, time.perf_counter())
+
+
+def test_run_pdfium_serialises_concurrent_calls() -> None:
+    hold = 0.02
+    n = 8
+
+    async def drive() -> list[tuple[int, str, float, float]]:
+        return list(await asyncio.gather(*(run_pdfium(lambda: _record(hold)) for _ in range(n))))
+
+    records = asyncio.run(drive())
+    assert len(records) == n
+
+    # Every callable ran on the one PDFium worker thread.
+    assert len({tid for tid, _, _, _ in records}) == 1
+    assert all(name.startswith('pdfium') for _, name, _, _ in records)
+
+    # Intervals are pairwise non-overlapping: sorted by entry, each call starts
+    # only after the previous one finished — a single worker runs them serially.
+    intervals = sorted((enter, exit_) for _, _, enter, exit_ in records)
+    for (_, prev_exit), (next_enter, _) in pairwise(intervals):
+        assert next_enter >= prev_exit

{flowapy-0.4.1 → flowapy-0.4.2}/uv.lock

@@ -492,7 +492,7 @@ wheels = [
 
 [[package]]
 name = "flowapy"
-version = "0.4.1"
+version = "0.4.2"
 source = { editable = "." }
 dependencies = [
     { name = "anchorite" },