ftmq 4.2.5__py3-none-any.whl → 4.3.0__py3-none-any.whl

ftmq/__init__.py

@@ -2,7 +2,7 @@ from ftmq.io import smart_read_proxies, smart_write_proxies
 from ftmq.query import Query
 from ftmq.util import make_entity
 
-__version__ = "4.2.5"
+__version__ = "4.3.0"
 __all__ = [
     "smart_read_proxies",
     "smart_write_proxies",

ftmq/cli.py

@@ -11,6 +11,9 @@ from ftmq.model.dataset import Catalog, Dataset
 from ftmq.model.stats import Collector
 from ftmq.query import Query
 from ftmq.store import get_store
+from ftmq.store.fragments import get_fragments
+from ftmq.store.fragments import get_store as get_fragments_store
+from ftmq.store.fragments.settings import Settings as FragmentsSettings
 from ftmq.util import apply_dataset, parse_unknown_filters
 
 log = get_logger(__name__)
@@ -311,6 +314,61 @@ def store_iterate(
     smart_write_proxies(output_uri, store.iterate())
 
 
+@cli.group()
+def fragments():
+    pass
+
+
+fragments_settings = FragmentsSettings()
+
+
+@fragments.command("list-datasets")
+@click.option(
+    "-i",
+    "--input-uri",
+    default=fragments_settings.database_uri,
+    show_default=True,
+    help="input file or uri",
+)
+@click.option(
+    "-o", "--output-uri", default="-", show_default=True, help="output file or uri"
+)
+def fragments_list_datasets(
+    input_uri: str = fragments_settings.database_uri,
+    output_uri: str = "-",
+):
+    """
+    List datasets within a fragments store
+    """
+    store = get_fragments_store(input_uri)
+    datasets = [ds.name for ds in store.all()]
+    smart_write(output_uri, "\n".join(datasets).encode() + b"\n")
+
+
+@fragments.command("iterate")
+@click.option(
+    "-i",
+    "--input-uri",
+    default=fragments_settings.database_uri,
+    show_default=True,
+    help="fragments store input uri",
+)
+@click.option(
+    "-o", "--output-uri", default="-", show_default=True, help="output file or uri"
+)
+@click.option("-d", "--dataset", required=True, help="Dataset name to iterate")
+def fragments_iterate(
+    input_uri: str = fragments_settings.database_uri,
+    output_uri: str = "-",
+    dataset: str = None,
+):
+    """
+    Iterate all entities from a fragments dataset
+    """
+    fragments = get_fragments(dataset, database_uri=input_uri)
+    smart_write_proxies(output_uri, fragments.iterate())
+
+
 @cli.command("aggregate")
 @click.option(
     "-i", "--input-uri", default="-", show_default=True, help="input file or uri"

ftmq/store/fragments/dataset.py

@@ -170,6 +170,10 @@ class Fragments(object):
                 raise
 
     def iterate(self, entity_id=None, skip_errors=False) -> EntityFragments:
+        if entity_id is None:
+            log.info("Using batched iteration for complete dataset.")
+            yield from self.iterate_batched()
+            return
         entity = None
         invalid = None
         fragments = 1
@@ -205,6 +209,39 @@ class Fragments(object):
         if entity is not None:
             yield entity
 
+    def iterate_batched(self, skip_errors=False, batch_size=10_000) -> EntityFragments:
+        """
+        For large datasets an overall sort is not feasible, so we iterate in
+        sorted batched IDs.
+        """
+        for entity_ids in self.get_sorted_id_batches(batch_size):
+            yield from self.iterate(entity_id=entity_ids, skip_errors=skip_errors)
+
+    def get_sorted_id_batches(
+        self, batch_size=10_000
+    ) -> Generator[list[str], None, None]:
+        """
+        Get sorted ID batches to speed up iteration and useful to parallelize
+        processing of iterator Entities
+        """
+        last_id = None
+        with self.store.engine.connect() as conn:
+            while True:
+                stmt = select(self.table.c.id).distinct()
+                if last_id is not None:
+                    stmt = stmt.where(self.table.c.id > last_id)
+                stmt = stmt.order_by(self.table.c.id).limit(batch_size)
+                try:
+                    res = conn.execute(stmt)
+                    entity_ids = [r.id for r in res.fetchall()]
+                    if not entity_ids:
+                        return
+                    yield entity_ids
+                    last_id = entity_ids[-1]
+                except Exception:
+                    self.reset()
+                    raise
+
     def statements(
         self,
         entity_ids: Iterable[str] | None = None,

{ftmq-4.2.5.dist-info → ftmq-4.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: ftmq
-Version: 4.2.5
+Version: 4.3.0
 Summary: followthemoney query dsl and io helpers
 License: AGPLv3+
 Author: Simon Wörpel
@@ -22,14 +22,14 @@ Requires-Dist: alephclient (>=2.6.0,<3.0.0) ; extra == "aleph"
 Requires-Dist: anystore (>=0.4.0,<0.5.0)
 Requires-Dist: click (>=8.2.1,<9.0.0)
 Requires-Dist: click-default-group (>=1.2.4,<2.0.0)
-Requires-Dist: deltalake (>=1.1.4,<2.0.0) ; extra == "lake"
-Requires-Dist: duckdb (>=1.4.0,<2.0.0) ; extra == "lake"
+Requires-Dist: deltalake (>=1.2.0,<2.0.0) ; extra == "lake"
+Requires-Dist: duckdb (>=1.4.1,<2.0.0) ; extra == "lake"
 Requires-Dist: fakeredis (>=2.26.2,<3.0.0) ; extra == "redis"
-Requires-Dist: followthemoney (>=4.2.2,<5.0.0)
+Requires-Dist: followthemoney (>=4.3.0,<5.0.0)
 Requires-Dist: furl (>=2.1.4,<3.0.0) ; extra == "aleph"
 Requires-Dist: nomenklatura (>=4.1.9,<5.0.0)
 Requires-Dist: orjson (>=3.10.18,<4.0.0)
-Requires-Dist: pandas (>=2.3.2,<3.0.0) ; extra == "lake"
+Requires-Dist: pandas (>=2.3.3,<3.0.0) ; extra == "lake"
 Requires-Dist: plyvel (>=1.5.1,<2.0.0) ; extra == "level"
 Requires-Dist: psycopg[pool] (>=3.2.9,<4.0.0) ; extra == "postgres"
 Requires-Dist: pyarrow (>=21.0.0,<22.0.0) ; extra == "lake"
@@ -37,7 +37,7 @@ Requires-Dist: pycountry (>=24.6.1,<25.0.0)
 Requires-Dist: pydantic (>=2.11.3,<3.0.0)
 Requires-Dist: pyicu (>=2.15.2,<3.0.0)
 Requires-Dist: redis (>=5.2.1,<6.0.0) ; extra == "redis"
-Requires-Dist: rigour (>=1.3.10,<2.0.0)
+Requires-Dist: rigour (>=1.3.13,<2.0.0)
 Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0) ; extra == "postgres"
 Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0) ; extra == "sql"
 Project-URL: Documentation, https://docs.investigraph.dev/lib/ftmq

{ftmq-4.2.5.dist-info → ftmq-4.3.0.dist-info}/RECORD

@@ -1,7 +1,7 @@
-ftmq/__init__.py,sha256=uJMQgEub6cttDVImesGqkC2HkgxIkYMHTloeCMrl2nQ,245
+ftmq/__init__.py,sha256=yG0oVaBCNY1c6a2EIhTGutZEjNyiMgcn10v_8khS6sQ,245
 ftmq/aggregate.py,sha256=nyAI5w6jKG1T4Jf2yy1ruhPh0vY6p7JWOEYh0SLBdZY,2163
 ftmq/aggregations.py,sha256=YLu1WF3WgcI3tadWKqsoZk3f_3bYUJetIREy2N1u_EM,4794
-ftmq/cli.py,sha256=GScVXHnRlnZ-A22iMR5hCX4H3sHLrqyXnXMtKK-2b04,9837
+ftmq/cli.py,sha256=roKQ1k-PUm15-mxD0fvjOwOhmpCBpuelYoECycfVjHw,11400
 ftmq/enums.py,sha256=4AJ6Ii8Bnbfz0BSznyK0IUopy-XMquuAuvud8ZcrD54,2521
 ftmq/filters.py,sha256=TKs454wbSvA5tPj8WbpIrMojctE2jGTLnrrVQKM9PE4,7908
 ftmq/io.py,sha256=gUbqoZuYXWwxoiJ505H7FhmXLYr5MEUiCEvqIiOkbgo,3789
@@ -18,7 +18,7 @@ ftmq/store/__init__.py,sha256=HH30KAHqo1kPr4qbjo6oKxdvgpF_XrqPgk8hSCkPbAY,3152
 ftmq/store/aleph.py,sha256=vENWpMgV1rViMwcef9wAwOGyrciCnSPRN-dSI103478,3977
 ftmq/store/base.py,sha256=1IgX4haeNUA9NTBCi-hBFVe0u44ra1nGdHAjSTOUEAs,4762
 ftmq/store/fragments/__init__.py,sha256=jHXHejqXe6sBNllAt-BuU24Ou8m5evmsD1UPac5J2GE,750
-ftmq/store/fragments/dataset.py,sha256=qVTkAl3erpNunbXb35cjwCB3nYRW8qjfLGuyRICLmLc,9135
+ftmq/store/fragments/dataset.py,sha256=8mHJo7CW_9kyYgnXapvvZamaG33DXBy8lx8qmNbuD3s,10623
 ftmq/store/fragments/loader.py,sha256=iVh8F22IApe9MRY_Z2fOLvT80fCYstFyxu410l4pPQY,4066
 ftmq/store/fragments/settings.py,sha256=4c-BW-blVM9gC_IGPch03eExbZYFZ3V5h9yTfhcHvOI,303
 ftmq/store/fragments/store.py,sha256=LiSfg95LjEmyq2IUpX4CMtp2tE37SEBLtAw0EWufImM,2534
@@ -30,9 +30,9 @@ ftmq/store/redis.py,sha256=d0hkGF_BezdIfCMUshXWoQwvGmqT8JFblUMcCxzwkDA,433
 ftmq/store/sql.py,sha256=6h3-gDaTAlD-IkiOONcX-JbaAO9-QfSsMjjMPupclcQ,5216
 ftmq/types.py,sha256=HgF8eT3ynKnDUxBYFtoDytS-uN_CS7Yr3DHIX2r4tnk,774
 ftmq/util.py,sha256=CmbZXYAbsKbAjoWn8WxR1Sz4VPXc2gj9CkHwaTqpBG0,15691
-ftmq-4.2.5.dist-info/LICENSE,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
-ftmq-4.2.5.dist-info/METADATA,sha256=oyc17dvkbMkwDZH4qnNeUknc5toSzPvSYgJagx2Pgi4,5281
-ftmq-4.2.5.dist-info/NOTICE,sha256=LNgfzuMbk3kIP_KnyDiXO8rQJmDrLy_PQ7cAY8lCmMM,463
-ftmq-4.2.5.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-ftmq-4.2.5.dist-info/entry_points.txt,sha256=YGDCjEiPgAMaQ5MqFKH8m-XIybehSXgarDucSlmeK3E,37
-ftmq-4.2.5.dist-info/RECORD,,
+ftmq-4.3.0.dist-info/LICENSE,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
+ftmq-4.3.0.dist-info/METADATA,sha256=uOlBTQdtpSar7Bc4GRiLh3RQd9tlxWhuX0UOUd9pbn8,5281
+ftmq-4.3.0.dist-info/NOTICE,sha256=LNgfzuMbk3kIP_KnyDiXO8rQJmDrLy_PQ7cAY8lCmMM,463
+ftmq-4.3.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+ftmq-4.3.0.dist-info/entry_points.txt,sha256=YGDCjEiPgAMaQ5MqFKH8m-XIybehSXgarDucSlmeK3E,37
+ftmq-4.3.0.dist-info/RECORD,,