ftmq 4.3.0__py3-none-any.whl → 4.3.2__py3-none-any.whl

ftmq/__init__.py

@@ -2,7 +2,7 @@ from ftmq.io import smart_read_proxies, smart_write_proxies
 from ftmq.query import Query
 from ftmq.util import make_entity
 
-__version__ = "4.3.0"
+__version__ = "4.3.2"
 __all__ = [
     "smart_read_proxies",
     "smart_write_proxies",

ftmq/cli.py

@@ -1,3 +1,5 @@
+from datetime import datetime
+
 import click
 from anystore.io import smart_write, smart_write_json, smart_write_model
 from click_default_group import DefaultGroup
@@ -357,16 +359,37 @@ def fragments_list_datasets(
     "-o", "--output-uri", default="-", show_default=True, help="output file or uri"
 )
 @click.option("-d", "--dataset", required=True, help="Dataset name to iterate")
+@click.option("-s", "--schema", default=None, help="Filter by schema")
+@click.option(
+    "--since",
+    default=None,
+    help="Filter by timestamp (since), ISO format: YYYY-MM-DDTHH:MM:SS",
+)
+@click.option(
+    "--until",
+    default=None,
+    help="Filter by timestamp (until), ISO format: YYYY-MM-DDTHH:MM:SS",
+)
 def fragments_iterate(
     input_uri: str = fragments_settings.database_uri,
     output_uri: str = "-",
     dataset: str = None,
+    schema: str | None = None,
+    since: str | None = None,
+    until: str | None = None,
 ):
     """
     Iterate all entities from a fragments dataset
     """
     fragments = get_fragments(dataset, database_uri=input_uri)
-    smart_write_proxies(output_uri, fragments.iterate())
+
+    # Parse timestamp strings to datetime objects
+    since_dt = datetime.fromisoformat(since) if since else None
+    until_dt = datetime.fromisoformat(until) if until else None
+
+    smart_write_proxies(
+        output_uri, fragments.iterate(schema=schema, since=since_dt, until=until_dt)
+    )
 
 
 @cli.command("aggregate")

ftmq/model/dataset.py

@@ -16,6 +16,7 @@ ContentType = Literal["documents", "structured", "mixed"]
 
 
 class Dataset(BaseModel, _DatasetModel):
+    prefix: str | None = None
     maintainer: DataPublisher | None = None
     stats: DatasetStats = DatasetStats()
     git_repo: AnyUrl | None = None

ftmq/store/fragments/dataset.py

@@ -132,7 +132,9 @@ class Fragments(object):
     def bulk(self, size=1000):
         return BulkLoader(self, size)
 
-    def fragments(self, entity_ids=None, fragment=None):
+    def fragments(
+        self, entity_ids=None, fragment=None, schema=None, since=None, until=None
+    ):
         stmt = self.table.select()
         entity_ids = ensure_list(entity_ids)
         if len(entity_ids) == 1:
@@ -141,6 +143,18 @@ class Fragments(object):
             stmt = stmt.where(self.table.c.id.in_(entity_ids))
         if fragment is not None:
             stmt = stmt.where(self.table.c.fragment == fragment)
+        if schema is not None:
+            if self.store.is_postgres:
+                stmt = stmt.where(self.table.c.entity["schema"].astext == schema)
+            else:
+                # SQLite JSON support - use json_extract function
+                stmt = stmt.where(
+                    func.json_extract(self.table.c.entity, "$.schema") == schema
+                )
+        if since is not None:
+            stmt = stmt.where(self.table.c.timestamp >= since)
+        if until is not None:
+            stmt = stmt.where(self.table.c.timestamp <= until)
         stmt = stmt.order_by(self.table.c.id)
         # stmt = stmt.order_by(self.table.c.origin)
         # stmt = stmt.order_by(self.table.c.fragment)
@@ -159,8 +173,12 @@ class Fragments(object):
         finally:
             conn.close()
 
-    def partials(self, entity_id=None, skip_errors=False) -> EntityFragments:
-        for fragment in self.fragments(entity_ids=entity_id):
+    def partials(
+        self, entity_id=None, skip_errors=False, schema=None, since=None, until=None
+    ) -> EntityFragments:
+        for fragment in self.fragments(
+            entity_ids=entity_id, schema=schema, since=since, until=until
+        ):
             try:
                 yield EntityProxy.from_dict(fragment, cleaned=True)
             except Exception:
@@ -169,22 +187,32 @@ class Fragments(object):
                     continue
                 raise
 
-    def iterate(self, entity_id=None, skip_errors=False) -> EntityFragments:
+    def iterate(
+        self, entity_id=None, skip_errors=False, schema=None, since=None, until=None
+    ) -> EntityFragments:
         if entity_id is None:
             log.info("Using batched iteration for complete dataset.")
-            yield from self.iterate_batched()
+            yield from self.iterate_batched(
+                skip_errors=skip_errors, schema=schema, since=since, until=until
+            )
             return
         entity = None
         invalid = None
         fragments = 1
-        for partial in self.partials(entity_id=entity_id, skip_errors=skip_errors):
+        for partial in self.partials(
+            entity_id=entity_id,
+            skip_errors=skip_errors,
+            schema=schema,
+            since=since,
+            until=until,
+        ):
             if partial.id == invalid:
                 continue
             if entity is not None:
                 if entity.id == partial.id:
                     fragments += 1
                     if fragments % 10000 == 0:
-                        log.debug(
+                        log.warning(
                             "[%s:%s] aggregated %d fragments...",
                             entity.schema.name,
                             entity.id,
@@ -209,16 +237,26 @@ class Fragments(object):
         if entity is not None:
             yield entity
 
-    def iterate_batched(self, skip_errors=False, batch_size=10_000) -> EntityFragments:
+    def iterate_batched(
+        self, skip_errors=False, batch_size=10_000, schema=None, since=None, until=None
+    ) -> EntityFragments:
         """
         For large datasets an overall sort is not feasible, so we iterate in
         sorted batched IDs.
         """
-        for entity_ids in self.get_sorted_id_batches(batch_size):
-            yield from self.iterate(entity_id=entity_ids, skip_errors=skip_errors)
+        for entity_ids in self.get_sorted_id_batches(
+            batch_size, schema=schema, since=since, until=until
+        ):
+            yield from self.iterate(
+                entity_id=entity_ids,
+                skip_errors=skip_errors,
+                schema=schema,
+                since=since,
+                until=until,
+            )
 
     def get_sorted_id_batches(
-        self, batch_size=10_000
+        self, batch_size=10_000, schema=None, since=None, until=None
     ) -> Generator[list[str], None, None]:
         """
         Get sorted ID batches to speed up iteration and useful to parallelize
@@ -230,6 +268,20 @@ class Fragments(object):
                 stmt = select(self.table.c.id).distinct()
                 if last_id is not None:
                     stmt = stmt.where(self.table.c.id > last_id)
+                if schema is not None:
+                    if self.store.is_postgres:
+                        stmt = stmt.where(
+                            self.table.c.entity["schema"].astext == schema
+                        )
+                    else:
+                        # SQLite JSON support - use json_extract function
+                        stmt = stmt.where(
+                            func.json_extract(self.table.c.entity, "$.schema") == schema
+                        )
+                if since is not None:
+                    stmt = stmt.where(self.table.c.timestamp >= since)
+                if until is not None:
+                    stmt = stmt.where(self.table.c.timestamp <= until)
                 stmt = stmt.order_by(self.table.c.id).limit(batch_size)
                 try:
                     res = conn.execute(stmt)
@@ -242,11 +294,19 @@ class Fragments(object):
                     self.reset()
                     raise
 
+    def get_sorted_ids(
+        self, batch_size=10_000, schema=None, since=None, until=None
+    ) -> Generator[str, None, None]:
+        """Get sorted IDs, optionally filtered by schema"""
+        for batch in self.get_sorted_id_batches(batch_size, schema, since, until):
+            yield from batch
+
     def statements(
         self,
         entity_ids: Iterable[str] | None = None,
         origin: str | None = None,
         since: datetime | None = None,
+        until: datetime | None = None,
     ) -> Statements:
         """Iterate unsorted statements with its fragment origins"""
         stmt = self.table.select()
@@ -258,7 +318,9 @@ class Fragments(object):
         if origin is not None:
             stmt = stmt.where(self.table.c.origin == origin)
         if since is not None:
-            stmt = stmt.where(self.table.c.timestamp > since)
+            stmt = stmt.where(self.table.c.timestamp >= since)
+        if until is not None:
+            stmt = stmt.where(self.table.c.timestamp <= until)
         conn = self.store.engine.connect()
         default_dataset = make_dataset(self.name)
         try:

ftmq/util.py

@@ -13,7 +13,7 @@ from followthemoney.proxy import EntityProxy
 from followthemoney.schema import Schema
 from followthemoney.types import registry
 from followthemoney.util import make_entity_id, sanitize_text
-from normality import collapse_spaces, latinize_text, slugify
+from normality import latinize_text, slugify, squash_spaces
 from rigour.names import Name, Symbol, tag_org_name, tag_person_name
 from rigour.names.tokenize import normalize_name
 from rigour.text.scripts import can_latinize
@@ -321,7 +321,7 @@ def clean_string(value: Any) -> str | None:
     value = sanitize_text(value)
     if value is None:
         return
-    return collapse_spaces(value)
+    return squash_spaces(value)
 
 
 def clean_name(value: Any) -> str | None:

{ftmq-4.3.0.dist-info → ftmq-4.3.2.dist-info}/METADATA

@@ -1,8 +1,10 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: ftmq
-Version: 4.3.0
+Version: 4.3.2
 Summary: followthemoney query dsl and io helpers
 License: AGPLv3+
+License-File: LICENSE
+License-File: NOTICE
 Author: Simon Wörpel
 Author-email: simon.woerpel@pm.me
 Requires-Python: >=3.11,<4
@@ -22,12 +24,12 @@ Requires-Dist: alephclient (>=2.6.0,<3.0.0) ; extra == "aleph"
 Requires-Dist: anystore (>=0.4.0,<0.5.0)
 Requires-Dist: click (>=8.2.1,<9.0.0)
 Requires-Dist: click-default-group (>=1.2.4,<2.0.0)
-Requires-Dist: deltalake (>=1.2.0,<2.0.0) ; extra == "lake"
+Requires-Dist: deltalake (>=1.2.1,<2.0.0) ; extra == "lake"
 Requires-Dist: duckdb (>=1.4.1,<2.0.0) ; extra == "lake"
 Requires-Dist: fakeredis (>=2.26.2,<3.0.0) ; extra == "redis"
-Requires-Dist: followthemoney (>=4.3.0,<5.0.0)
+Requires-Dist: followthemoney (>=4.3.2,<5.0.0)
 Requires-Dist: furl (>=2.1.4,<3.0.0) ; extra == "aleph"
-Requires-Dist: nomenklatura (>=4.1.9,<5.0.0)
+Requires-Dist: nomenklatura (>=4.1.10,<5.0.0)
 Requires-Dist: orjson (>=3.10.18,<4.0.0)
 Requires-Dist: pandas (>=2.3.3,<3.0.0) ; extra == "lake"
 Requires-Dist: plyvel (>=1.5.1,<2.0.0) ; extra == "level"
@@ -37,7 +39,7 @@ Requires-Dist: pycountry (>=24.6.1,<25.0.0)
 Requires-Dist: pydantic (>=2.11.3,<3.0.0)
 Requires-Dist: pyicu (>=2.15.2,<3.0.0)
 Requires-Dist: redis (>=5.2.1,<6.0.0) ; extra == "redis"
-Requires-Dist: rigour (>=1.3.13,<2.0.0)
+Requires-Dist: rigour (>=1.4.1,<2.0.0)
 Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0) ; extra == "postgres"
 Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0) ; extra == "sql"
 Project-URL: Documentation, https://docs.investigraph.dev/lib/ftmq

{ftmq-4.3.0.dist-info → ftmq-4.3.2.dist-info}/RECORD

@@ -1,13 +1,13 @@
-ftmq/__init__.py,sha256=yG0oVaBCNY1c6a2EIhTGutZEjNyiMgcn10v_8khS6sQ,245
+ftmq/__init__.py,sha256=ueHfqk0txp2sdGHbXsKPFwErBru33CPWbewYF8swWfA,245
 ftmq/aggregate.py,sha256=nyAI5w6jKG1T4Jf2yy1ruhPh0vY6p7JWOEYh0SLBdZY,2163
 ftmq/aggregations.py,sha256=YLu1WF3WgcI3tadWKqsoZk3f_3bYUJetIREy2N1u_EM,4794
-ftmq/cli.py,sha256=roKQ1k-PUm15-mxD0fvjOwOhmpCBpuelYoECycfVjHw,11400
+ftmq/cli.py,sha256=e4wLGfGFmn-8Z6_EhEcOJvUwQM3R-J1i40qh2UVHcGo,12078
 ftmq/enums.py,sha256=4AJ6Ii8Bnbfz0BSznyK0IUopy-XMquuAuvud8ZcrD54,2521
 ftmq/filters.py,sha256=TKs454wbSvA5tPj8WbpIrMojctE2jGTLnrrVQKM9PE4,7908
 ftmq/io.py,sha256=gUbqoZuYXWwxoiJ505H7FhmXLYr5MEUiCEvqIiOkbgo,3789
 ftmq/logging.py,sha256=p6UvN8qkki8bOjI_NfIjKoLRhdtAdGm4swMWPssETKY,3211
 ftmq/model/__init__.py,sha256=hgdLNeoYm4o6XoY24_7gtIYf8SUq3d8s_2mda1EQDzg,197
-ftmq/model/dataset.py,sha256=Z8RQNz4wvYsbeX77EabI9wH-7difC32_Y_b1QrJkkXA,2198
+ftmq/model/dataset.py,sha256=d6KQDhvxhfQv7J4oF76_ePANDJQ8onxpzhc4-wnui_Q,2228
 ftmq/model/entity.py,sha256=MIJWORs6X4Z22DDQkHRQlcR3Hug8M_YjZwVEo5KU538,2073
 ftmq/model/mixins.py,sha256=O_z3Pxv0HzXxu5K-lNFglrIAhh4jejmbcXUjQfVe0_o,435
 ftmq/model/stats.py,sha256=BiOK7x-JymI5f3dRpwgr3J7OknDP1pe9955Y7AvOJGo,4048
@@ -18,7 +18,7 @@ ftmq/store/__init__.py,sha256=HH30KAHqo1kPr4qbjo6oKxdvgpF_XrqPgk8hSCkPbAY,3152
 ftmq/store/aleph.py,sha256=vENWpMgV1rViMwcef9wAwOGyrciCnSPRN-dSI103478,3977
 ftmq/store/base.py,sha256=1IgX4haeNUA9NTBCi-hBFVe0u44ra1nGdHAjSTOUEAs,4762
 ftmq/store/fragments/__init__.py,sha256=jHXHejqXe6sBNllAt-BuU24Ou8m5evmsD1UPac5J2GE,750
-ftmq/store/fragments/dataset.py,sha256=8mHJo7CW_9kyYgnXapvvZamaG33DXBy8lx8qmNbuD3s,10623
+ftmq/store/fragments/dataset.py,sha256=P1ljcpoRpl_IIApA2A3vizpB2goRxxue_NNoeI1JQN8,13032
 ftmq/store/fragments/loader.py,sha256=iVh8F22IApe9MRY_Z2fOLvT80fCYstFyxu410l4pPQY,4066
 ftmq/store/fragments/settings.py,sha256=4c-BW-blVM9gC_IGPch03eExbZYFZ3V5h9yTfhcHvOI,303
 ftmq/store/fragments/store.py,sha256=LiSfg95LjEmyq2IUpX4CMtp2tE37SEBLtAw0EWufImM,2534
@@ -29,10 +29,10 @@ ftmq/store/memory.py,sha256=lZ_pDzrBWNljbNb1MXJeCoO7TnAdqEfG4kfLDOU5rME,551
 ftmq/store/redis.py,sha256=d0hkGF_BezdIfCMUshXWoQwvGmqT8JFblUMcCxzwkDA,433
 ftmq/store/sql.py,sha256=6h3-gDaTAlD-IkiOONcX-JbaAO9-QfSsMjjMPupclcQ,5216
 ftmq/types.py,sha256=HgF8eT3ynKnDUxBYFtoDytS-uN_CS7Yr3DHIX2r4tnk,774
-ftmq/util.py,sha256=CmbZXYAbsKbAjoWn8WxR1Sz4VPXc2gj9CkHwaTqpBG0,15691
-ftmq-4.3.0.dist-info/LICENSE,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
-ftmq-4.3.0.dist-info/METADATA,sha256=uOlBTQdtpSar7Bc4GRiLh3RQd9tlxWhuX0UOUd9pbn8,5281
-ftmq-4.3.0.dist-info/NOTICE,sha256=LNgfzuMbk3kIP_KnyDiXO8rQJmDrLy_PQ7cAY8lCmMM,463
-ftmq-4.3.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-ftmq-4.3.0.dist-info/entry_points.txt,sha256=YGDCjEiPgAMaQ5MqFKH8m-XIybehSXgarDucSlmeK3E,37
-ftmq-4.3.0.dist-info/RECORD,,
+ftmq/util.py,sha256=s6TypjCvOy5FVgxn-FeCsIDtwkmUsyj6oIZ60Q56PBo,15687
+ftmq-4.3.2.dist-info/METADATA,sha256=Un8mFQdd6tOfyD2wAKvQlcmU9dZ8tUFGynX0MN2Xv-E,5324
+ftmq-4.3.2.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+ftmq-4.3.2.dist-info/entry_points.txt,sha256=YGDCjEiPgAMaQ5MqFKH8m-XIybehSXgarDucSlmeK3E,37
+ftmq-4.3.2.dist-info/licenses/LICENSE,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
+ftmq-4.3.2.dist-info/licenses/NOTICE,sha256=LNgfzuMbk3kIP_KnyDiXO8rQJmDrLy_PQ7cAY8lCmMM,463
+ftmq-4.3.2.dist-info/RECORD,,

{ftmq-4.3.0.dist-info → ftmq-4.3.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.3
+Generator: poetry-core 2.2.1
 Root-Is-Purelib: true
 Tag: py3-none-any