scigantic-bindingdb 0.2.2__tar.gz → 0.2.4__tar.gz

{scigantic_bindingdb-0.2.2/src/scigantic_bindingdb.egg-info → scigantic_bindingdb-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scigantic-bindingdb
-Version: 0.2.2
+Version: 0.2.4
 Summary: Query BindingDB directly from a public S3 mirror with DuckDB, including a ChEMBL cross-reference bridge table and a ready drug-target-interaction training table.
 Author: Scigantic
 License: MIT-0
@@ -136,7 +136,7 @@ bindingdb.enable_cache()
 df = bindingdb.chembl_bridge()  # downloads the bridge table once, then reads from disk
 ```
 
-`chembl_bridge()` and `dti_pairs()` each need exactly one derived file, so caching downloads that one file to `~/.cache/scigantic-bindingdb` (override with `enable_cache(cache_dir=...)` or the `SCIGANTIC_BINDINGDB_CACHE` environment variable) and reuses it after that.
+`chembl_bridge()` and `dti_pairs()` each need exactly one derived file, so caching downloads that one file to `~/.cache/scigantic-bindingdb` (override with `enable_cache(cache_dir=...)` or the `SCIGANTIC_BINDINGDB_CACHE` environment variable) and reuses it after that. Concurrent callers racing the first download of the same file wait for it rather than each downloading their own copy.
 
 `connect()`, `query()` and `measurements()` don't participate in this: `connect()` registers five core tables as views the first time a release is used, so caching them there would mean the first call for any release eagerly downloads everything regardless of what the query actually touches. Cache one table yourself if you want it locally: `bindingdb.cache_resolve("202608/parquet/measurements.parquet")` downloads it and returns the local path, usable directly in `read_parquet(...)`.
 

{scigantic_bindingdb-0.2.2 → scigantic_bindingdb-0.2.4}/README.md

@@ -105,7 +105,7 @@ bindingdb.enable_cache()
 df = bindingdb.chembl_bridge()  # downloads the bridge table once, then reads from disk
 ```
 
-`chembl_bridge()` and `dti_pairs()` each need exactly one derived file, so caching downloads that one file to `~/.cache/scigantic-bindingdb` (override with `enable_cache(cache_dir=...)` or the `SCIGANTIC_BINDINGDB_CACHE` environment variable) and reuses it after that.
+`chembl_bridge()` and `dti_pairs()` each need exactly one derived file, so caching downloads that one file to `~/.cache/scigantic-bindingdb` (override with `enable_cache(cache_dir=...)` or the `SCIGANTIC_BINDINGDB_CACHE` environment variable) and reuses it after that. Concurrent callers racing the first download of the same file wait for it rather than each downloading their own copy.
 
 `connect()`, `query()` and `measurements()` don't participate in this: `connect()` registers five core tables as views the first time a release is used, so caching them there would mean the first call for any release eagerly downloads everything regardless of what the query actually touches. Cache one table yourself if you want it locally: `bindingdb.cache_resolve("202608/parquet/measurements.parquet")` downloads it and returns the local path, usable directly in `read_parquet(...)`.
 

{scigantic_bindingdb-0.2.2 → scigantic_bindingdb-0.2.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "scigantic-bindingdb"
-version = "0.2.2"
+version = "0.2.4"
 description = "Query BindingDB directly from a public S3 mirror with DuckDB, including a ChEMBL cross-reference bridge table and a ready drug-target-interaction training table."
 readme = "README.md"
 requires-python = ">=3.10"

{scigantic_bindingdb-0.2.2 → scigantic_bindingdb-0.2.4}/src/scigantic_bindingdb/cache.py

@@ -23,6 +23,7 @@ from __future__ import annotations
 
 import os
 import sys
+import threading
 import urllib.request
 import uuid
 from pathlib import Path
@@ -34,6 +35,24 @@ _cache_dir: Path | None = None
 
 _CHUNK_BYTES = 1024 * 1024
 
+# One lock per key, created on first use. Guards resolve()'s
+# check-then-download against concurrent callers asking for the same key
+# at once: without this, N threads racing the first resolve() of a key
+# each see it missing and each download it in full, rather than one
+# downloading while the rest wait and reuse the result. Verified directly:
+# 16 threads calling chembl_bridge() concurrently on an empty cache
+# triggered 16 separate downloads of the same file before this fix.
+_resolve_locks: dict[str, threading.Lock] = {}
+_resolve_locks_guard = threading.Lock()
+
+
+def _lock_for(key: str) -> threading.Lock:
+    lock = _resolve_locks.get(key)
+    if lock is not None:
+        return lock
+    with _resolve_locks_guard:
+        return _resolve_locks.setdefault(key, threading.Lock())
+
 
 def _default_cache_dir() -> Path:
     if sys.platform == "win32":
@@ -114,7 +133,9 @@ def resolve(key: str) -> str:
     `key` is a path relative to the bucket root, e.g.
     "202608/derived/bindingdb_chembl_bridge.parquet". Downloads to the
     cache on first access; later calls for the same key reuse the local
-    file without touching the network.
+    file without touching the network. Concurrent callers asking for the
+    same key while it's still downloading wait for that download rather
+    than each starting their own.
     """
     if not _enabled:
         return f"s3://{BUCKET}/{key}"
@@ -124,8 +145,11 @@ def resolve(key: str) -> str:
     if local_path.exists():
         return str(local_path)
 
-    local_path.parent.mkdir(parents=True, exist_ok=True)
-    url = f"https://{BUCKET}.s3.{REGION}.amazonaws.com/{key}"
-    print(f"scigantic-bindingdb: caching {key} ...", file=sys.stderr, flush=True)
-    _atomic_download(url, local_path)
+    with _lock_for(key):
+        if local_path.exists():  # another thread finished it while we waited
+            return str(local_path)
+        local_path.parent.mkdir(parents=True, exist_ok=True)
+        url = f"https://{BUCKET}.s3.{REGION}.amazonaws.com/{key}"
+        print(f"scigantic-bindingdb: caching {key} ...", file=sys.stderr, flush=True)
+        _atomic_download(url, local_path)
     return str(local_path)

{scigantic_bindingdb-0.2.2 → scigantic_bindingdb-0.2.4}/src/scigantic_bindingdb/chembl_bridge.py

@@ -49,10 +49,6 @@ def chembl_bridge(
     con = connect(release)
     try:
         bridge_path = _resolve(f"{release}/derived/bindingdb_chembl_bridge.parquet")
-        con.execute(
-            "CREATE OR REPLACE VIEW chembl_bridge AS "
-            f"SELECT * FROM read_parquet('{bridge_path}')"
-        )
 
         where: list[str] = []
         params: list[str | int] = []
@@ -61,12 +57,22 @@ def chembl_bridge(
             params.append(reactant_set_id)
         clause = f"WHERE {' AND '.join(where)}" if where else ""
 
+        # read_parquet(bridge_path) inline rather than a named
+        # CREATE VIEW: connect() now hands out a cursor on a base
+        # connection shared across calls (see connection.py), and a
+        # named view is catalog state on that shared connection.
+        # Concurrent calls each doing CREATE OR REPLACE VIEW on the same
+        # name raced each other's DDL transaction: reproduced directly,
+        # concurrent chembl_bridge()/dti_pairs() calls from a thread pool
+        # mostly failed with DuckDB's "Catalog write-write conflict".
+        # Referencing the parquet file directly in the query has no
+        # catalog side effect at all, so there's nothing to conflict on.
         if with_names:
             mol_dict = f"s3://{CHEMBL_BUCKET}/{chembl_release}/parquet/molecule_dictionary.parquet"
             sql = f"""
                 SELECT b.reactant_set_id, b.chembl_molregno, b.chembl_id, b.match_method,
                        m.ligand_smiles, d.pref_name AS chembl_pref_name
-                FROM chembl_bridge b
+                FROM read_parquet('{bridge_path}') b
                 JOIN measurements m ON m.reactant_set_id = b.reactant_set_id
                 LEFT JOIN read_parquet('{mol_dict}') d ON d.molregno = b.chembl_molregno
                 {clause}
@@ -75,7 +81,7 @@ def chembl_bridge(
             sql = f"""
                 SELECT b.reactant_set_id, b.chembl_molregno, b.chembl_id, b.match_method,
                        m.ligand_smiles
-                FROM chembl_bridge b
+                FROM read_parquet('{bridge_path}') b
                 JOIN measurements m ON m.reactant_set_id = b.reactant_set_id
                 {clause}
             """

{scigantic_bindingdb-0.2.2 → scigantic_bindingdb-0.2.4}/src/scigantic_bindingdb/dti_pairs.py

@@ -54,7 +54,6 @@ def dti_pairs(
     con = connect(release)
     try:
         path = _resolve(f"{release}/derived/dti_pairs.parquet")
-        con.execute(f"CREATE OR REPLACE VIEW dti_pairs AS SELECT * FROM read_parquet('{path}')")
 
         where: list[str] = []
         params: list[str | int] = []
@@ -68,7 +67,13 @@ def dti_pairs(
             where.append("n_chains_declared = 1")
         clause = f"WHERE {' AND '.join(where)}" if where else ""
 
-        sql = f"SELECT * FROM dti_pairs {clause} ORDER BY p_affinity DESC"
+        # read_parquet(path) inline rather than a named CREATE VIEW: see
+        # chembl_bridge.py's comment on the same change. connect() now
+        # returns a cursor on a base connection shared across calls, and a
+        # named view is catalog state on that shared connection --
+        # concurrent calls racing CREATE OR REPLACE VIEW on the same name
+        # hit DuckDB's "Catalog write-write conflict", reproduced directly.
+        sql = f"SELECT * FROM read_parquet('{path}') {clause} ORDER BY p_affinity DESC"
         if limit is not None:
             sql += " LIMIT ?"
             params.append(int(limit))

{scigantic_bindingdb-0.2.2 → scigantic_bindingdb-0.2.4/src/scigantic_bindingdb.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scigantic-bindingdb
-Version: 0.2.2
+Version: 0.2.4
 Summary: Query BindingDB directly from a public S3 mirror with DuckDB, including a ChEMBL cross-reference bridge table and a ready drug-target-interaction training table.
 Author: Scigantic
 License: MIT-0
@@ -136,7 +136,7 @@ bindingdb.enable_cache()
 df = bindingdb.chembl_bridge()  # downloads the bridge table once, then reads from disk
 ```
 
-`chembl_bridge()` and `dti_pairs()` each need exactly one derived file, so caching downloads that one file to `~/.cache/scigantic-bindingdb` (override with `enable_cache(cache_dir=...)` or the `SCIGANTIC_BINDINGDB_CACHE` environment variable) and reuses it after that.
+`chembl_bridge()` and `dti_pairs()` each need exactly one derived file, so caching downloads that one file to `~/.cache/scigantic-bindingdb` (override with `enable_cache(cache_dir=...)` or the `SCIGANTIC_BINDINGDB_CACHE` environment variable) and reuses it after that. Concurrent callers racing the first download of the same file wait for it rather than each downloading their own copy.
 
 `connect()`, `query()` and `measurements()` don't participate in this: `connect()` registers five core tables as views the first time a release is used, so caching them there would mean the first call for any release eagerly downloads everything regardless of what the query actually touches. Cache one table yourself if you want it locally: `bindingdb.cache_resolve("202608/parquet/measurements.parquet")` downloads it and returns the local path, usable directly in `read_parquet(...)`.
 

{scigantic_bindingdb-0.2.2 → scigantic_bindingdb-0.2.4}/tests/test_cache.py

@@ -127,6 +127,50 @@ def test_concurrent_downloads_of_the_same_key_never_raise(tmp_path):
     assert list(local_path.parent.glob("key.parquet.*.part")) == []
 
 
+def test_concurrent_first_resolve_of_the_same_key_downloads_once(tmp_path):
+    # Regression test for a real inefficiency: resolve()'s check-then-
+    # download wasn't coordinated across threads, so N threads racing the
+    # first resolve() of a key each saw it missing and each downloaded it
+    # in full. Verified directly before this was fixed with a per-key
+    # lock: 16 threads calling resolve() concurrently on an empty cache
+    # triggered 16 separate downloads of the same file instead of one.
+    #
+    # mock.patch is entered/exited only once here, by the main thread,
+    # before the pool starts; the worker threads only read the already-
+    # patched attribute, so this doesn't hit the same-target mock.patch
+    # race that test_concurrent_downloads_of_the_same_key_never_raise's
+    # docstring describes for patching from multiple threads at once.
+    body = b"cached-body"
+
+    class FakeConnection:
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *exc_info):
+            return False
+
+        def read(self, *_args):
+            if getattr(self, "_served", False):
+                return b""
+            self._served = True
+            return body
+
+    bindingdb.enable_cache(cache_dir=str(tmp_path))
+    try:
+        with mock.patch(
+            "urllib.request.urlopen", side_effect=lambda *a, **k: FakeConnection()
+        ) as urlopen:
+            with ThreadPoolExecutor(max_workers=16) as pool:
+                paths = list(pool.map(lambda _i: resolve("shared/once.parquet"), range(16)))
+
+        assert len(set(paths)) == 1
+        assert urlopen.call_count == 1  # only one thread actually downloaded
+        final = tmp_path / "shared" / "once.parquet"
+        assert final.read_bytes() == body
+    finally:
+        bindingdb.disable_cache()
+
+
 def test_cached_bridge_matches_uncached(tmp_path):
     # The one integration-level check: caching actually gets used by a real
     # function, not just by resolve() directly, and returns the same data.

{scigantic_bindingdb-0.2.2 → scigantic_bindingdb-0.2.4}/tests/test_chembl_bridge.py

@@ -1,3 +1,5 @@
+from concurrent.futures import ThreadPoolExecutor
+
 import scigantic_bindingdb as bindingdb
 
 # A gefitinib measurement, verified against the live mirror to resolve
@@ -29,3 +31,25 @@ def test_bridge_total_row_count():
         "'s3://scigantic-bindingdb/202608/derived/bindingdb_chembl_bridge.parquet')"
     )
     assert total["n"].iloc[0] == 2272063
+
+
+def test_concurrent_calls_do_not_conflict_on_catalog():
+    # Regression test for a real bug introduced by connect()'s move to a
+    # shared base connection (see connection.py): chembl_bridge() used to
+    # register its result as a named CREATE OR REPLACE VIEW, which is
+    # catalog-mutating DDL. Two calls racing that DDL on the same shared
+    # connection from different threads mostly failed with DuckDB's
+    # "Catalog write-write conflict", reproduced directly (46/48 calls
+    # failed in a 24-thread stress run) before this was fixed by
+    # referencing the parquet file inline instead of via a named view.
+    def call(i):
+        return bindingdb.chembl_bridge(
+            reactant_set_id=_GEFITINIB_REACTANT_SET_ID, with_names=(i % 2 == 0)
+        )
+
+    with ThreadPoolExecutor(max_workers=16) as pool:
+        results = list(pool.map(call, range(16)))
+
+    for df in results:
+        assert len(df) == 1
+        assert df["chembl_id"].iloc[0] == "CHEMBL939"

{scigantic_bindingdb-0.2.2 → scigantic_bindingdb-0.2.4}/tests/test_dti_pairs.py

@@ -1,4 +1,5 @@
 import math
+from concurrent.futures import ThreadPoolExecutor
 
 import scigantic_bindingdb as bindingdb
 
@@ -69,3 +70,26 @@ def test_uniprot_filter():
 def test_sorted_most_potent_first():
     df = bindingdb.dti_pairs(endpoint="ki", uniprot_id="P00533")
     assert df["p_affinity"].is_monotonic_decreasing
+
+
+def test_concurrent_calls_do_not_conflict_on_catalog():
+    # Regression test for a real bug introduced by connect()'s move to a
+    # shared base connection (see connection.py): dti_pairs() used to
+    # register its result as a named CREATE OR REPLACE VIEW, which is
+    # catalog-mutating DDL. Two calls racing that DDL on the same shared
+    # connection from different threads mostly failed with DuckDB's
+    # "Catalog write-write conflict" (reproduced directly before this was
+    # fixed by referencing the parquet file inline instead of via a named
+    # view; see chembl_bridge.py's equivalent test).
+    endpoints = ("ki", "ic50", "kd", "ec50")
+
+    def call(i):
+        endpoint = endpoints[i % len(endpoints)]
+        return endpoint, bindingdb.dti_pairs(endpoint=endpoint, limit=50)
+
+    with ThreadPoolExecutor(max_workers=16) as pool:
+        results = list(pool.map(call, range(16)))
+
+    for endpoint, df in results:
+        assert len(df) == 50
+        assert (df["endpoint"] == endpoint).all()