ftmq 4.2.3__py3-none-any.whl → 4.2.5__py3-none-any.whl

ftmq/__init__.py

@@ -2,7 +2,7 @@ from ftmq.io import smart_read_proxies, smart_write_proxies
 from ftmq.query import Query
 from ftmq.util import make_entity
 
-__version__ = "4.2.3"
+__version__ = "4.2.5"
 __all__ = [
     "smart_read_proxies",
     "smart_write_proxies",

ftmq/store/fragments/dataset.py

@@ -1,4 +1,5 @@
 import logging
+from contextlib import contextmanager
 from datetime import datetime
 from typing import Generator, Iterable, TypeAlias
 
@@ -9,6 +10,7 @@ from normality import slugify
 from sqlalchemy import (
     JSON,
     Column,
+    Connection,
     DateTime,
     String,
     Table,
@@ -43,6 +45,27 @@ except ImportError:
 EntityFragments: TypeAlias = Generator[EntityProxy, None, None]
 
 
+@contextmanager
+def disable_timeout(conn: Connection, store):
+    # for long running iterations (e.g. re-index in OpenAleph), for postgres we
+    # don't want to get cancelled if a idle_in_transaction_timeout is configured
+    # on the server
+    if store.is_postgres:
+        raw_conn = conn.connection.driver_connection
+        with raw_conn.cursor() as cursor:
+            cursor.execute("SET idle_in_transaction_session_timeout = 0")
+    try:
+        yield conn
+    finally:
+        if store.is_postgres:
+            try:
+                raw_conn = conn.connection.driver_connection
+                with raw_conn.cursor() as cursor:
+                    cursor.execute("SET idle_in_transaction_session_timeout = DEFAULT")
+            except Exception:
+                pass  # Connection might be closed
+
+
 class Fragments(object):
     def __init__(self, store, name, origin=NULL_ORIGIN):
         self.store = store
@@ -123,12 +146,13 @@ class Fragments(object):
         # stmt = stmt.order_by(self.table.c.fragment)
         conn = self.store.engine.connect()
         try:
-            conn = conn.execution_options(stream_results=True)
-            for ent in conn.execute(stmt):
-                data = {"id": ent.id, "datasets": [self.name], **ent.entity}
-                if ent.origin != NULL_ORIGIN:
-                    data["origin"] = ent.origin
-                yield data
+            with disable_timeout(conn, self.store) as conn:
+                conn = conn.execution_options(stream_results=True)
+                for ent in conn.execute(stmt):
+                    data = {"id": ent.id, "datasets": [self.name], **ent.entity}
+                    if ent.origin != NULL_ORIGIN:
+                        data["origin"] = ent.origin
+                    yield data
         except Exception:
             self.reset()
             raise

ftmq/store/fragments/store.py

@@ -39,14 +39,12 @@ class Store(object):
         self.database_uri = self._adjust_psycopg3_uri(database_uri)
 
         # Configure connection pooling for psycopg3
+        config.setdefault("pool_size", 1)
         if self.database_uri.startswith("postgresql+psycopg://"):
-            config.setdefault("pool_size", 10)
-            config.setdefault("max_overflow", 20)
-            config.setdefault("pool_timeout", 30)
+            config.setdefault("max_overflow", 5)
+            config.setdefault("pool_timeout", 60)
             config.setdefault("pool_recycle", 3600)
             config.setdefault("pool_pre_ping", True)
-        else:
-            config.setdefault("pool_size", 5)
 
         self.engine = create_engine(self.database_uri, future=True, **config)
         self.is_postgres = self.engine.dialect.name == "postgresql"

{ftmq-4.2.3.dist-info → ftmq-4.2.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ftmq
-Version: 4.2.3
+Version: 4.2.5
 Summary: followthemoney query dsl and io helpers
 License: AGPLv3+
 Author: Simon Wörpel
@@ -19,15 +19,15 @@ Provides-Extra: postgres
 Provides-Extra: redis
 Provides-Extra: sql
 Requires-Dist: alephclient (>=2.6.0,<3.0.0) ; extra == "aleph"
-Requires-Dist: anystore (>=0.3.10,<0.4.0)
+Requires-Dist: anystore (>=0.4.0,<0.5.0)
 Requires-Dist: click (>=8.2.1,<9.0.0)
 Requires-Dist: click-default-group (>=1.2.4,<2.0.0)
 Requires-Dist: deltalake (>=1.1.4,<2.0.0) ; extra == "lake"
-Requires-Dist: duckdb (>=1.3.2,<2.0.0) ; extra == "lake"
+Requires-Dist: duckdb (>=1.4.0,<2.0.0) ; extra == "lake"
 Requires-Dist: fakeredis (>=2.26.2,<3.0.0) ; extra == "redis"
-Requires-Dist: followthemoney (>=4.2.0,<5.0.0)
+Requires-Dist: followthemoney (>=4.2.2,<5.0.0)
 Requires-Dist: furl (>=2.1.4,<3.0.0) ; extra == "aleph"
-Requires-Dist: nomenklatura (>=4.1.2,<5.0.0)
+Requires-Dist: nomenklatura (>=4.1.9,<5.0.0)
 Requires-Dist: orjson (>=3.10.18,<4.0.0)
 Requires-Dist: pandas (>=2.3.2,<3.0.0) ; extra == "lake"
 Requires-Dist: plyvel (>=1.5.1,<2.0.0) ; extra == "level"
@@ -37,7 +37,7 @@ Requires-Dist: pycountry (>=24.6.1,<25.0.0)
 Requires-Dist: pydantic (>=2.11.3,<3.0.0)
 Requires-Dist: pyicu (>=2.15.2,<3.0.0)
 Requires-Dist: redis (>=5.2.1,<6.0.0) ; extra == "redis"
-Requires-Dist: rigour (>=1.3.0,<2.0.0)
+Requires-Dist: rigour (>=1.3.10,<2.0.0)
 Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0) ; extra == "postgres"
 Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0) ; extra == "sql"
 Project-URL: Documentation, https://docs.investigraph.dev/lib/ftmq

{ftmq-4.2.3.dist-info → ftmq-4.2.5.dist-info}/RECORD

@@ -1,4 +1,4 @@
-ftmq/__init__.py,sha256=7COsy0UJm_rlN5cEgGx_gSKRNS9CDHUg1m1ra8lUqZs,245
+ftmq/__init__.py,sha256=uJMQgEub6cttDVImesGqkC2HkgxIkYMHTloeCMrl2nQ,245
 ftmq/aggregate.py,sha256=nyAI5w6jKG1T4Jf2yy1ruhPh0vY6p7JWOEYh0SLBdZY,2163
 ftmq/aggregations.py,sha256=YLu1WF3WgcI3tadWKqsoZk3f_3bYUJetIREy2N1u_EM,4794
 ftmq/cli.py,sha256=GScVXHnRlnZ-A22iMR5hCX4H3sHLrqyXnXMtKK-2b04,9837
@@ -18,10 +18,10 @@ ftmq/store/__init__.py,sha256=HH30KAHqo1kPr4qbjo6oKxdvgpF_XrqPgk8hSCkPbAY,3152
 ftmq/store/aleph.py,sha256=vENWpMgV1rViMwcef9wAwOGyrciCnSPRN-dSI103478,3977
 ftmq/store/base.py,sha256=1IgX4haeNUA9NTBCi-hBFVe0u44ra1nGdHAjSTOUEAs,4762
 ftmq/store/fragments/__init__.py,sha256=jHXHejqXe6sBNllAt-BuU24Ou8m5evmsD1UPac5J2GE,750
-ftmq/store/fragments/dataset.py,sha256=k6tl4atXOU8XU2c_Q70-tjUHIfyrhZy3yMT11Jg27Cw,8189
+ftmq/store/fragments/dataset.py,sha256=qVTkAl3erpNunbXb35cjwCB3nYRW8qjfLGuyRICLmLc,9135
 ftmq/store/fragments/loader.py,sha256=iVh8F22IApe9MRY_Z2fOLvT80fCYstFyxu410l4pPQY,4066
 ftmq/store/fragments/settings.py,sha256=4c-BW-blVM9gC_IGPch03eExbZYFZ3V5h9yTfhcHvOI,303
-ftmq/store/fragments/store.py,sha256=SurMQJjKhoPOhbrXH9upGMIDy5JPlDNiuxb40sOH0dc,2600
+ftmq/store/fragments/store.py,sha256=LiSfg95LjEmyq2IUpX4CMtp2tE37SEBLtAw0EWufImM,2534
 ftmq/store/fragments/utils.py,sha256=SDoLPFF5O_oJLIPrCEab5iGn-pl6y0AhYZDYIPYxYkk,1098
 ftmq/store/lake.py,sha256=snbEZXTuR9Oy3o7p6XA61IVpsd4-70a6PgoL63PBrRg,11683
 ftmq/store/level.py,sha256=ZGx-mMtJZJNWkpvbe7ajTREnW5MPcnw0ct3nSFLVF0I,781
@@ -30,9 +30,9 @@ ftmq/store/redis.py,sha256=d0hkGF_BezdIfCMUshXWoQwvGmqT8JFblUMcCxzwkDA,433
 ftmq/store/sql.py,sha256=6h3-gDaTAlD-IkiOONcX-JbaAO9-QfSsMjjMPupclcQ,5216
 ftmq/types.py,sha256=HgF8eT3ynKnDUxBYFtoDytS-uN_CS7Yr3DHIX2r4tnk,774
 ftmq/util.py,sha256=CmbZXYAbsKbAjoWn8WxR1Sz4VPXc2gj9CkHwaTqpBG0,15691
-ftmq-4.2.3.dist-info/LICENSE,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
-ftmq-4.2.3.dist-info/METADATA,sha256=M1eiaNsDCACJPteGCWLUhaJ-M1zPBxOv19p8KnJq67k,5281
-ftmq-4.2.3.dist-info/NOTICE,sha256=LNgfzuMbk3kIP_KnyDiXO8rQJmDrLy_PQ7cAY8lCmMM,463
-ftmq-4.2.3.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-ftmq-4.2.3.dist-info/entry_points.txt,sha256=YGDCjEiPgAMaQ5MqFKH8m-XIybehSXgarDucSlmeK3E,37
-ftmq-4.2.3.dist-info/RECORD,,
+ftmq-4.2.5.dist-info/LICENSE,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
+ftmq-4.2.5.dist-info/METADATA,sha256=oyc17dvkbMkwDZH4qnNeUknc5toSzPvSYgJagx2Pgi4,5281
+ftmq-4.2.5.dist-info/NOTICE,sha256=LNgfzuMbk3kIP_KnyDiXO8rQJmDrLy_PQ7cAY8lCmMM,463
+ftmq-4.2.5.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+ftmq-4.2.5.dist-info/entry_points.txt,sha256=YGDCjEiPgAMaQ5MqFKH8m-XIybehSXgarDucSlmeK3E,37
+ftmq-4.2.5.dist-info/RECORD,,