ftmq 4.3.2__tar.gz → 4.5.3__tar.gz

{ftmq-4.3.2 → ftmq-4.5.3}/PKG-INFO

@@ -1,13 +1,13 @@
 Metadata-Version: 2.4
 Name: ftmq
-Version: 4.3.2
+Version: 4.5.3
 Summary: followthemoney query dsl and io helpers
 License: AGPLv3+
 License-File: LICENSE
 License-File: NOTICE
 Author: Simon Wörpel
 Author-email: simon.woerpel@pm.me
-Requires-Python: >=3.11,<4
+Requires-Python: >=3.11,<3.14
 Classifier: Intended Audience :: Developers
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
@@ -21,25 +21,24 @@ Provides-Extra: postgres
 Provides-Extra: redis
 Provides-Extra: sql
 Requires-Dist: alephclient (>=2.6.0,<3.0.0) ; extra == "aleph"
-Requires-Dist: anystore (>=0.4.0,<0.5.0)
+Requires-Dist: anystore (>=1.0.1,<2.0.0)
 Requires-Dist: click (>=8.2.1,<9.0.0)
 Requires-Dist: click-default-group (>=1.2.4,<2.0.0)
-Requires-Dist: deltalake (>=1.2.1,<2.0.0) ; extra == "lake"
-Requires-Dist: duckdb (>=1.4.1,<2.0.0) ; extra == "lake"
+Requires-Dist: deltalake (>=1.4.1,<2.0.0) ; extra == "lake"
+Requires-Dist: duckdb (>=1.4.4,<2.0.0) ; extra == "lake"
 Requires-Dist: fakeredis (>=2.26.2,<3.0.0) ; extra == "redis"
-Requires-Dist: followthemoney (>=4.3.2,<5.0.0)
+Requires-Dist: followthemoney (>=4.5.2,<5.0.0)
 Requires-Dist: furl (>=2.1.4,<3.0.0) ; extra == "aleph"
-Requires-Dist: nomenklatura (>=4.1.10,<5.0.0)
+Requires-Dist: nomenklatura (>=4.6.1,<5.0.0)
 Requires-Dist: orjson (>=3.10.18,<4.0.0)
-Requires-Dist: pandas (>=2.3.3,<3.0.0) ; extra == "lake"
+Requires-Dist: pandas (>=3.0.0,<4.0.0) ; extra == "lake"
 Requires-Dist: plyvel (>=1.5.1,<2.0.0) ; extra == "level"
 Requires-Dist: psycopg[pool] (>=3.2.9,<4.0.0) ; extra == "postgres"
-Requires-Dist: pyarrow (>=21.0.0,<22.0.0) ; extra == "lake"
-Requires-Dist: pycountry (>=24.6.1,<25.0.0)
+Requires-Dist: pyarrow (>=23.0.0,<24.0.0) ; extra == "lake"
 Requires-Dist: pydantic (>=2.11.3,<3.0.0)
 Requires-Dist: pyicu (>=2.15.2,<3.0.0)
 Requires-Dist: redis (>=5.2.1,<6.0.0) ; extra == "redis"
-Requires-Dist: rigour (>=1.4.1,<2.0.0)
+Requires-Dist: rigour (>=1.6.2,<2.0.0)
 Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0) ; extra == "postgres"
 Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0) ; extra == "sql"
 Project-URL: Documentation, https://docs.investigraph.dev/lib/ftmq

{ftmq-4.3.2 → ftmq-4.5.3}/ftmq/__init__.py

@@ -2,7 +2,7 @@ from ftmq.io import smart_read_proxies, smart_write_proxies
 from ftmq.query import Query
 from ftmq.util import make_entity
 
-__version__ = "4.3.2"
+__version__ = "4.5.3"
 __all__ = [
     "smart_read_proxies",
     "smart_write_proxies",

{ftmq-4.3.2 → ftmq-4.5.3}/ftmq/cli.py

@@ -2,13 +2,13 @@ from datetime import datetime
 
 import click
 from anystore.io import smart_write, smart_write_json, smart_write_model
+from anystore.logging import configure_logging, get_logger
 from click_default_group import DefaultGroup
 from followthemoney import ValueEntity
 from nomenklatura import settings
 
 from ftmq.aggregate import aggregate
 from ftmq.io import smart_read_proxies, smart_write_proxies
-from ftmq.logging import configure_logging, get_logger
 from ftmq.model.dataset import Catalog, Dataset
 from ftmq.model.stats import Collector
 from ftmq.query import Query

{ftmq-4.3.2 → ftmq-4.5.3}/ftmq/io.py

@@ -1,11 +1,12 @@
 from typing import Any, Iterable, Type
 
 import orjson
-from anystore.io import Uri, smart_open, smart_stream
+from anystore.io import smart_open, smart_stream
+from anystore.logging import get_logger
+from anystore.types import Uri
 from banal import is_listish
 from followthemoney import E, StatementEntity, ValueEntity
 
-from ftmq.logging import get_logger
 from ftmq.query import Query
 from ftmq.store import Store, get_store
 from ftmq.types import Entities, Entity
@@ -13,8 +14,6 @@ from ftmq.util import ensure_entity, make_entity
 
 log = get_logger(__name__)
 
-DEFAULT_MODE = "rb"
-
 
 def smart_get_store(uri: Uri, **kwargs) -> Store | None:
     try:

{ftmq-4.3.2 → ftmq-4.5.3}/ftmq/model/dataset.py

@@ -1,19 +1,22 @@
 from datetime import datetime
-from typing import Literal
+from typing import Literal, TypeVar
 
 from anystore.io import logged_items
-from anystore.types import SDict
+from anystore.types import HttpUrlStr, SDict
 from followthemoney.dataset import DataPublisher
 from followthemoney.dataset.dataset import DatasetModel as _DatasetModel
-from pydantic import AnyUrl, HttpUrl
+from pydantic import AnyUrl
 from rigour.mime.types import FTM
 
 from ftmq.model.mixins import BaseModel
 from ftmq.model.stats import DatasetStats
 from ftmq.types import Entities
+from ftmq.util import DEFAULT_DATASET
 
 ContentType = Literal["documents", "structured", "mixed"]
 
+D = TypeVar("D", bound="Dataset")
+
 
 class Dataset(BaseModel, _DatasetModel):
     prefix: str | None = None
@@ -57,9 +60,9 @@ class Catalog(BaseModel):
     description: str | None = None
     maintainer: DataPublisher | None = None
     publisher: DataPublisher | None = None
-    url: HttpUrl | None = None
+    url: HttpUrlStr | None = None
     uri: str | None = None
-    logo_url: HttpUrl | None = None
+    logo_url: HttpUrlStr | None = None
     git_repo: AnyUrl | None = None
 
     def iterate(self) -> Entities:
@@ -70,3 +73,8 @@ class Catalog(BaseModel):
     def names(self) -> set[str]:
         """Get the names of all datasets in the catalog."""
         return {d.name for d in self.datasets}
+
+
+def make_dataset(name: str = DEFAULT_DATASET, cls: type[D] = Dataset, **kwargs) -> D:
+    kwargs["title"] = kwargs.pop("title", name)
+    return cls(name=name, **kwargs)

{ftmq-4.3.2 → ftmq-4.5.3}/ftmq/model/mixins.py

@@ -1,4 +1,4 @@
-from anystore.mixins import BaseModel as _BaseModel
+from anystore.model.base import BaseModel as _BaseModel
 from pydantic import Field
 
 

{ftmq-4.3.2 → ftmq-4.5.3}/ftmq/model/stats.py

@@ -2,8 +2,9 @@ from collections import Counter
 from datetime import datetime
 from typing import Any
 
+from anystore.model import BaseModel
 from followthemoney import model
-from pydantic import BaseModel, model_validator
+from pydantic import model_validator
 
 from ftmq.types import Entities, Entity
 from ftmq.util import get_country_name, get_year_from_iso

{ftmq-4.3.2 → ftmq-4.5.3}/ftmq/store/base.py

@@ -1,25 +1,26 @@
-from typing import Generator, Iterable
+from typing import Generator, Generic, Iterable, TypeVar
 from urllib.parse import urlparse
 
 from anystore.functools import weakref_cache as cache
-from followthemoney import DefaultDataset
+from anystore.logging import get_logger
 from followthemoney.dataset.dataset import Dataset
 from nomenklatura import store as nk
 from nomenklatura.db import get_engine
 from nomenklatura.resolver import Resolver
 
 from ftmq.aggregations import AggregatorResult
-from ftmq.logging import get_logger
 from ftmq.model.stats import Collector, DatasetStats
 from ftmq.query import Query
 from ftmq.similar import get_similar
 from ftmq.types import StatementEntities, StatementEntity
-from ftmq.util import ensure_dataset
+from ftmq.util import DEFAULT_DATASET, ensure_dataset
 
 log = get_logger(__name__)
 
 DEFAULT_ORIGIN = "default"
 
+V = TypeVar("V", bound="View")
+
 
 @cache
 def get_resolver(uri: str | None = None) -> Resolver[StatementEntity]:
@@ -28,7 +29,7 @@ def get_resolver(uri: str | None = None) -> Resolver[StatementEntity]:
     return Resolver.make_default(get_engine("sqlite:///:memory:"))
 
 
-class Store(nk.Store):
+class Store(nk.Store[Dataset, StatementEntity], Generic[V]):
     """
     Feature add-ons to `nomenklatura.store.Store`
     """
@@ -51,7 +52,7 @@ class Store(nk.Store):
         linker = linker or get_resolver(kwargs.get("uri"))
         super().__init__(dataset=dataset, linker=linker, **kwargs)
         # implicit set all datasets as default store scope:
-        if dataset == DefaultDataset and not dataset.leaf_names:
+        if dataset.name == DEFAULT_DATASET and not dataset.leaf_names:
             self.dataset = self.get_scope()
 
     def get_scope(self) -> Dataset:
@@ -60,6 +61,12 @@ class Store(nk.Store):
         """
         raise NotImplementedError
 
+    def view(self, scope: Dataset | None = None, external: bool = False) -> V:
+        raise NotImplementedError
+
+    def default_view(self, external: bool = False) -> V:
+        return self.view(self.dataset, external)
+
     def iterate(self, dataset: str | Dataset | None = None) -> StatementEntities:
         """
         Iterate all the entities, optional filter for a dataset.

{ftmq-4.3.2 → ftmq-4.5.3}/ftmq/store/lake.py

@@ -19,16 +19,15 @@ Layout:
 """
 
 from pathlib import Path
-from typing import Any, Generator, Iterable
+from typing import Any, Generator
 from urllib.parse import urlparse
 
 import duckdb
-import numpy as np
-import pandas as pd
+import pyarrow as pa
 from anystore.functools import weakref_cache as cache
-from anystore.lock import Lock
+from anystore.interface.lock import Lock
 from anystore.logging import get_logger
-from anystore.store.fs import Store as FSStore
+from anystore.store import Store as FSStore
 from anystore.types import SDict
 from anystore.util import clean_dict
 from deltalake import (
@@ -39,6 +38,7 @@ from deltalake import (
     write_deltalake,
 )
 from deltalake._internal import TableNotFoundError
+from deltalake.table import FilterConjunctionType
 from followthemoney import EntityProxy, StatementEntity, model
 from followthemoney.dataset.dataset import Dataset
 from followthemoney.statement import Statement
@@ -51,17 +51,19 @@ from sqlalchemy import Boolean, DateTime, column, select, table
 from sqlalchemy.sql import Select
 
 from ftmq.query import Query
-from ftmq.store.base import Store
+from ftmq.store.base import DEFAULT_ORIGIN, Store
 from ftmq.store.sql import SQLQueryView, SQLStore
 from ftmq.types import StatementEntities
 from ftmq.util import apply_dataset, ensure_entity, get_scope_dataset
 
 log = get_logger(__name__)
 
-Z_ORDER = ["canonical_id", "entity_id", "schema", "prop"]
+Z_ORDER = ["canonical_id", "prop"]  # don't add more columns here
 TARGET_SIZE = 50 * 10_485_760  # 500 MB
 PARTITION_BY = ["dataset", "bucket", "origin"]
-DEFAULT_ORIGIN = "default"
+BUCKET_MENTION = "mention"
+BUCKET_PAGE = "page"
+BUCKET_PAGES = "pages"
 BUCKET_DOCUMENT = "document"
 BUCKET_INTERVAL = "interval"
 BUCKET_THING = "thing"
@@ -82,9 +84,15 @@ WRITER = WriterProperties(
         "schema": STATISTICS,
         "prop": STATISTICS_BLOOM,
         "value": STATISTICS_BLOOM,
+        "last_seen": ColumnProperties(statistics_enabled="CHUNK"),
     },
 )
 
+SA_TO_ARROW: dict[type, pa.DataType] = {
+    Boolean: pa.bool_(),
+    DateTime: pa.timestamp("us"),
+}
+
 TABLE = table(
     nks.STATEMENT_TABLE,
     column("id"),
@@ -93,6 +101,7 @@ TABLE = table(
     column("dataset"),
     column("bucket"),
     column("origin"),
+    column("source"),
     column("schema"),
     column("prop"),
     column("prop_type"),
@@ -104,6 +113,10 @@ TABLE = table(
     column("last_seen", DateTime),
 )
 
+ARROW_SCHEMA = pa.schema(
+    [(col.name, SA_TO_ARROW.get(type(col.type), pa.string())) for col in TABLE.columns]
+)
+
 
 class StorageSettings(BaseSettings):
     model_config = SettingsConfigDict(env_file=".env", extra="ignore")
@@ -148,22 +161,26 @@ def storage_options() -> SDict:
 @cache
 def setup_duckdb_storage() -> None:
     if storage_settings.secret:
-        duckdb.query(
-            f"""CREATE OR REPLACE SECRET secret (
+        duckdb.query(f"""CREATE OR REPLACE SECRET secret (
             TYPE s3,
             PROVIDER config,
             KEY_ID '{storage_settings.key}',
             SECRET '{storage_settings.secret}',
-            ENDPOINT '{storage_settings.endpoint}',
+            ENDPOINT '{storage_settings.duckdb_endpoint}',
             URL_STYLE 'path',
             USE_SSL '{not storage_settings.allow_http}'
-            );"""
-        )
+            );""")
 
 
 @cache
 def get_schema_bucket(schema_name: str) -> str:
     s = model[schema_name]
+    if s.is_a("Mention"):
+        return BUCKET_MENTION
+    if s.is_a("Page"):
+        return BUCKET_PAGE
+    if s.is_a("Pages"):
+        return BUCKET_PAGES
     if s.is_a("Document"):
         return BUCKET_DOCUMENT
     if s.is_a("Interval"):
@@ -171,19 +188,13 @@ def get_schema_bucket(schema_name: str) -> str:
     return BUCKET_THING
 
 
-def pack_statement(stmt: Statement) -> SDict:
+def pack_statement(stmt: Statement, source: str | None = None) -> SDict:
     data = stmt.to_db_row()
     data["bucket"] = get_schema_bucket(data["schema"])
+    data["source"] = source
     return data
 
 
-def pack_statements(statements: Iterable[Statement]) -> pd.DataFrame:
-    df = pd.DataFrame(map(pack_statement, statements))
-    df = df.drop_duplicates()  # .sort_values(Z_ORDER)
-    df = df.fillna(np.nan)
-    return df
-
-
 def compile_query(q: Select) -> str:
     table = nks.STATEMENT_TABLE
     sql = str(q.compile(compile_kwargs={"literal_binds": True}))
@@ -237,15 +248,12 @@ class LakeQueryView(SQLQueryView):
             yield from super().query(query)
 
 
-class LakeStore(SQLStore):
+class LakeStore(SQLStore[LakeQueryView]):
     def __init__(self, *args, **kwargs) -> None:
-        self._backend: FSStore = FSStore(uri=kwargs.pop("uri"))
+        self._backend = FSStore(uri=kwargs.pop("uri"))
         self._partition_by = kwargs.pop("partition_by", PARTITION_BY)
         self._lock: Lock = kwargs.pop("lock", Lock(self._backend))
         self._enforce_dataset = kwargs.pop("enforce_dataset", False)
-        assert isinstance(
-            self._backend, FSStore
-        ), f"Invalid store backend: `{self._backend.__class__}"
         kwargs["uri"] = "sqlite:///:memory:"  # fake it till you make it
         get_metadata.cache_clear()
         super().__init__(*args, **kwargs)
@@ -275,12 +283,14 @@ class LakeStore(SQLStore):
 
     def view(
         self, scope: Dataset | None = None, external: bool = False
-    ) -> SQLQueryView:
+    ) -> LakeQueryView:
         scope = scope or self.dataset
         return LakeQueryView(self, scope, external)
 
-    def writer(self, origin: str | None = DEFAULT_ORIGIN) -> "LakeWriter":
-        return LakeWriter(self, origin=origin or DEFAULT_ORIGIN)
+    def writer(
+        self, origin: str | None = DEFAULT_ORIGIN, source: str | None = None
+    ) -> "LakeWriter":
+        return LakeWriter(self, origin=origin or DEFAULT_ORIGIN, source=source)
 
     def get_origins(self) -> set[str]:
         q = select(self.table.c.origin).distinct()
@@ -291,32 +301,55 @@ class LakeWriter(nk.Writer):
     store: LakeStore
     BATCH_STATEMENTS = 1_000_000
 
-    def __init__(self, store: Store, origin: str | None = DEFAULT_ORIGIN):
+    def __init__(
+        self,
+        store: Store,
+        origin: str | None = DEFAULT_ORIGIN,
+        source: str | None = None,
+    ):
         super().__init__(store)
-        self.batch: set[Statement] = set()
+        self.batch: dict[str, tuple[Statement, str | None]] = {}
         self.origin = origin or DEFAULT_ORIGIN
+        self.source = source
 
-    def add_statement(self, stmt: Statement) -> None:
+    def add_statement(self, stmt: Statement, source: str | None = None) -> None:
         if stmt.entity_id is None:
             return
         stmt.origin = stmt.origin or self.origin
         canonical_id = self.store.linker.get_canonical(stmt.entity_id)
         stmt.canonical_id = canonical_id
-        self.batch.add(stmt)
-
-    def add_entity(self, entity: EntityProxy, origin: str | None = None) -> None:
+        key = f"{canonical_id}\t{stmt.id}"
+        self.batch[key] = (stmt, source or self.source)
+
+    def add_entity(
+        self,
+        entity: EntityProxy,
+        origin: str | None = None,
+        source: str | None = None,
+    ) -> None:
         e = ensure_entity(entity, StatementEntity, self.store.dataset)
         if self.store._enforce_dataset:
             e = apply_dataset(e, self.store.dataset, replace=True)
         for stmt in e.statements:
             if origin:
                 stmt.origin = origin
-            self.add_statement(stmt)
+            self.add_statement(stmt, source=source)
         # we check here instead of in `add_statement` as this will keep entities
-        # together in the same parquet files`
+        # together in the same parquet files
         if len(self.batch) >= self.BATCH_STATEMENTS:
             self.flush()
 
+    def _pack_batches(self) -> Generator[pa.RecordBatch, None, None]:
+        batch: list[SDict] = []
+        for key in sorted(self.batch):
+            stmt, source = self.batch[key]
+            batch.append(pack_statement(stmt, source))
+            if len(batch) >= 100_000:
+                yield pa.RecordBatch.from_pylist(batch, schema=ARROW_SCHEMA)
+                batch = []
+        if batch:
+            yield pa.RecordBatch.from_pylist(batch, schema=ARROW_SCHEMA)
+
     def flush(self) -> None:
         if self.batch:
             log.info(
@@ -324,18 +357,22 @@ class LakeWriter(nk.Writer):
                 uri=self.store.uri,
             )
             with self.store._lock:
+                reader = pa.RecordBatchReader.from_batches(
+                    ARROW_SCHEMA, self._pack_batches()
+                )
                 write_deltalake(
                     str(self.store.uri),
-                    pack_statements(self.batch),
+                    reader,
                     partition_by=self.store._partition_by,
                     mode="append",
                     schema_mode="merge",
                     writer_properties=WRITER,
                     target_file_size=TARGET_SIZE,
                     storage_options=storage_options(),
+                    configuration={"delta.enableChangeDataFeed": "true"},
                 )
 
-        self.batch = set()
+        self.batch = {}
 
     def pop(self, entity_id: str) -> list[Statement]:
         q = select(TABLE)
@@ -348,18 +385,42 @@ class LakeWriter(nk.Writer):
         return statements
 
     def optimize(
-        self, vacuum: bool | None = False, vacuum_keep_hours: int | None = 0
+        self,
+        vacuum: bool | None = False,
+        vacuum_keep_hours: int | None = 0,
+        dataset: str | None = None,
+        bucket: str | None = None,
+        origin: str | None = None,
     ) -> None:
         """
         Optimize the storage: Z-Ordering and compacting
+
+        Args:
+            vacuum: Run vacuum after optimization
+            vacuum_keep_hours: Retention hours for vacuum
+            dataset: Filter optimization to specific dataset partition
+            bucket: Filter optimization to specific bucket partition
+            origin: Filter optimization to specific origin partition
         """
-        self.store.deltatable.optimize.z_order(
-            Z_ORDER, writer_properties=WRITER, target_size=TARGET_SIZE
-        )
-        if vacuum:
-            self.store.deltatable.vacuum(
-                retention_hours=vacuum_keep_hours,
-                enforce_retention_duration=False,
-                dry_run=False,
-                full=True,
+        filters: FilterConjunctionType = []
+        if dataset is not None:
+            filters.append(("dataset", "=", dataset))
+        if bucket is not None:
+            filters.append(("bucket", "=", bucket))
+        if origin is not None:
+            filters.append(("origin", "=", origin))
+
+        with self.store._lock:
+            self.store.deltatable.optimize.z_order(
+                Z_ORDER,
+                writer_properties=WRITER,
+                target_size=TARGET_SIZE,
+                partition_filters=filters or None,
             )
+            if vacuum:
+                self.store.deltatable.vacuum(
+                    retention_hours=vacuum_keep_hours,
+                    enforce_retention_duration=False,
+                    dry_run=False,
+                    full=True,
+                )

{ftmq-4.3.2 → ftmq-4.5.3}/ftmq/store/sql.py

@@ -1,12 +1,14 @@
 import os
 from collections import defaultdict
 from decimal import Decimal
+from typing import Generic
 
 from anystore.util import clean_dict
 from followthemoney.dataset.dataset import Dataset
 from nomenklatura.db import get_metadata
 from nomenklatura.store import sql as nk
 from sqlalchemy import select
+from typing_extensions import TypeVar
 
 from ftmq.aggregations import AggregatorResult
 from ftmq.enums import Fields
@@ -16,6 +18,8 @@ from ftmq.store.base import Store, View
 from ftmq.types import StatementEntities
 from ftmq.util import get_scope_dataset
 
+V = TypeVar("V", bound=View, default="SQLQueryView")
+
 MAX_SQL_AGG_GROUPS = int(os.environ.get("MAX_SQL_AGG_GROUPS", 10))
 
 
@@ -125,7 +129,7 @@ class SQLQueryView(View, nk.SQLView):
         return res
 
 
-class SQLStore(Store, nk.SQLStore):
+class SQLStore(Store[V], nk.SQLStore, Generic[V]):
     def __init__(self, *args, **kwargs) -> None:
         get_metadata.cache_clear()  # FIXME
         super().__init__(*args, **kwargs)
@@ -137,8 +141,6 @@ class SQLStore(Store, nk.SQLStore):
             names.add(row[0])
         return get_scope_dataset(*names)
 
-    def view(
-        self, scope: Dataset | None = None, external: bool = False
-    ) -> SQLQueryView:
+    def view(self, scope: Dataset | None = None, external: bool = False) -> V:
         scope = scope or self.dataset
-        return SQLQueryView(self, scope, external=external)
+        return SQLQueryView(self, scope, external=external)  # type: ignore[return-value]

{ftmq-4.3.2 → ftmq-4.5.3}/ftmq/util.py

@@ -1,7 +1,5 @@
-from functools import lru_cache
 from typing import Any, Generator, Type
 
-import pycountry
 from anystore.functools import weakref_cache as cache
 from anystore.types import SDict, StrGenerator
 from banal import ensure_list, is_listish
@@ -16,6 +14,7 @@ from followthemoney.util import make_entity_id, sanitize_text
 from normality import latinize_text, slugify, squash_spaces
 from rigour.names import Name, Symbol, tag_org_name, tag_person_name
 from rigour.names.tokenize import normalize_name
+from rigour.territories import lookup_territory
 from rigour.text.scripts import can_latinize
 
 from ftmq.enums import Comparators
@@ -149,7 +148,7 @@ def apply_dataset(entity: E, dataset: str | Dataset, replace: bool | None = Fals
 def get_country_name(code: str) -> str:
     """
     Get the (english) country name for the given 2-letter iso code via
-    [pycountry](https://pypi.org/project/pycountry/)
+    [rigour.territories](https://rigour.followthemoney.tech/territories/)
 
     Examples:
         >>> get_country_name("de")
@@ -165,22 +164,17 @@ def get_country_name(code: str) -> str:
     Returns:
         Either the country name for a valid code or the code as fallback.
     """
-    code_clean = get_country_code(code)
-    if code_clean is None:
-        code_clean = code.lower()
-    try:
-        country = pycountry.countries.get(alpha_2=code_clean)
-        if country is not None:
-            return country.name
-    except (LookupError, AttributeError):
-        return code
-    return code_clean
+    territory = lookup_territory(code)
+    if territory is not None:
+        return territory.name
+    return code
 
 
-@lru_cache(1024)
+@cache
 def get_country_code(value: Any, splitter: str | None = ",") -> str | None:
     """
-    Get the 2-letter iso country code for an arbitrary country name
+    Get the 2-letter iso country code for an arbitrary country name via
+    [rigour.territories](https://rigour.followthemoney.tech/territories/)
 
     Examples:
         >>> get_country_code("Germany")
@@ -201,15 +195,16 @@ def get_country_code(value: Any, splitter: str | None = ",") -> str | None:
     """
     value = clean_string(value)
     if not value:
-        return
-    code = registry.country.clean_text(value)
-    if code:
-        return code
-    for token in value.split(splitter):
-        code = registry.country.clean_text(token)
-        if code:
-            return code
-    return
+        return None
+    territory = lookup_territory(value)
+    if territory is not None:
+        return territory.ftm_country
+    if splitter:
+        for token in value.split(splitter):
+            territory = lookup_territory(token.strip())
+            if territory is not None:
+                return territory.ftm_country
+    return None
 
 
 def join_slug(

{ftmq-4.3.2 → ftmq-4.5.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "ftmq"
-version = "4.3.2"
+version = "4.5.3"
 description = "followthemoney query dsl and io helpers"
 authors = [{ name = "Simon Wörpel", email = "simon.woerpel@pm.me" }]
 license = "AGPLv3+"
@@ -12,19 +12,19 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    # "Programming Language :: Python :: 3.14",
 ]
-requires-python = ">=3.11,<4"
+requires-python = ">=3.11,<3.14"
 dependencies = [
-    "anystore (>=0.4.0,<0.5.0)",
-    "followthemoney (>=4.3.2,<5.0.0)",
-    "nomenklatura (>=4.1.10,<5.0.0)",
-    "rigour (>=1.4.1,<2.0.0)",
+    "anystore (>=1.0.1,<2.0.0)",
+    "followthemoney (>=4.5.2,<5.0.0)",
+    "nomenklatura (>=4.6.1,<5.0.0)",
+    "rigour (>=1.6.2,<2.0.0)",
     "click (>=8.2.1,<9.0.0)",
     "click-default-group (>=1.2.4,<2.0.0)",
     "orjson (>=3.10.18,<4.0.0)",
     "pyicu (>=2.15.2,<3.0.0)",
     "pydantic (>=2.11.3,<3.0.0)",
-    "pycountry (>=24.6.1,<25.0.0)",
 ]
 
 [project.optional-dependencies]
@@ -33,10 +33,10 @@ sql = ["sqlalchemy (>=2.0.36,<3.0.0)"]
 postgres = ["sqlalchemy (>=2.0.36,<3.0.0)", "psycopg[pool] (>=3.2.9,<4.0.0)"]
 redis = ["redis (>=5.2.1,<6.0.0)", "fakeredis (>=2.26.2,<3.0.0)"]
 lake = [
-    "duckdb (>=1.4.1,<2.0.0)",
-    "pandas (>=2.3.3,<3.0.0)",
-    "deltalake (>=1.2.1,<2.0.0)",
-    "pyarrow (>=21.0.0,<22.0.0)",
+    "duckdb (>=1.4.4,<2.0.0)",
+    "deltalake (>=1.4.1,<2.0.0)",
+    "pyarrow (>=23.0.0,<24.0.0)",
+    "pandas (>=3.0.0,<4.0.0)",
 ]
 aleph = ["furl (>=2.1.4,<3.0.0)", "alephclient (>=2.6.0,<3.0.0)"]
 
@@ -50,10 +50,10 @@ Repository = "https://github.com/dataresearchcenter/ftmq"
 Issues = "https://github.com/dataresearchcenter/ftmq/issues"
 
 [tool.poetry.group.dev.dependencies]
-pytest = ">=7.4.3,<9.0.0"
+pytest = ">=7.4.3,<10.0.0"
 pytest-cov = ">=4.1,<8.0"
 pytest-env = "^1.1.1"
-black = ">=23.11,<26.0"
+black = ">=23.11,<27.0"
 isort = "^7.0.0"
 mypy = "^1.17.1"
 pre-commit = "^4.0.1"
@@ -62,7 +62,7 @@ ipdb = "^0.13.13"
 bump2version = "^1.0.1"
 mkdocs = "^1.6.1"
 mkdocs-autorefs = "^1.4.3"
-mkdocstrings-python = "^1.18.2"
+mkdocstrings-python = "^2.0.0"
 mkdocs-material = "^9.6.18"
 mkdocs-click = "^0.9.0"
 

ftmq-4.3.2/ftmq/logging.py

@@ -1,105 +0,0 @@
-import logging
-import os
-import sys
-from logging import Filter, LogRecord
-from typing import Any, Dict, List
-
-import structlog
-from banal import as_bool
-from structlog.contextvars import merge_contextvars
-from structlog.dev import ConsoleRenderer, set_exc_info
-from structlog.processors import (
-    JSONRenderer,
-    TimeStamper,
-    UnicodeDecoder,
-    add_log_level,
-    format_exc_info,
-)
-from structlog.stdlib import (
-    BoundLogger,
-    LoggerFactory,
-    ProcessorFormatter,
-    add_logger_name,
-)
-from structlog.stdlib import get_logger as get_raw_logger
-
-LOG_JSON = as_bool(os.environ.get("LOG_JSON"))
-LOG_LEVEL = os.environ.get("LOG_LEVEL", "info").upper()
-
-
-def get_logger(name: str, *args, **kwargs) -> BoundLogger:
-    return get_raw_logger(name, *args, **kwargs)
-
-
-def configure_logging(level: int = logging.INFO) -> None:
-    """Configure log levels and structured logging"""
-    shared_processors: List[Any] = [
-        add_log_level,
-        add_logger_name,
-        # structlog.stdlib.PositionalArgumentsFormatter(),
-        # structlog.processors.StackInfoRenderer(),
-        merge_contextvars,
-        set_exc_info,
-        TimeStamper(fmt="iso"),
-        # format_exc_info,
-        UnicodeDecoder(),
-    ]
-
-    if LOG_JSON:
-        shared_processors.append(format_exc_info)
-        shared_processors.append(format_json)
-        formatter = ProcessorFormatter(
-            foreign_pre_chain=shared_processors,
-            processor=JSONRenderer(),
-        )
-    else:
-        formatter = ProcessorFormatter(
-            foreign_pre_chain=shared_processors,
-            processor=ConsoleRenderer(
-                exception_formatter=structlog.dev.plain_traceback
-            ),
-        )
-
-    processors = shared_processors + [
-        ProcessorFormatter.wrap_for_formatter,
-    ]
-
-    # configuration for structlog based loggers
-    structlog.configure(
-        cache_logger_on_first_use=True,
-        # wrapper_class=AsyncBoundLogger,
-        wrapper_class=BoundLogger,
-        processors=processors,
-        context_class=dict,
-        logger_factory=LoggerFactory(),
-    )
-
-    # handler for low level logs that should be sent to STDERR
-    out_handler = logging.StreamHandler(sys.stderr)
-    out_handler.setLevel(level)
-    out_handler.addFilter(_MaxLevelFilter(logging.WARNING))
-    out_handler.setFormatter(formatter)
-    # handler for high level logs that should be sent to STDERR
-    error_handler = logging.StreamHandler(sys.stderr)
-    error_handler.setLevel(logging.ERROR)
-    error_handler.setFormatter(formatter)
-
-    root_logger = logging.getLogger()
-    root_logger.setLevel(LOG_LEVEL)
-    root_logger.addHandler(out_handler)
-    root_logger.addHandler(error_handler)
-
-
-def format_json(_: Any, __: Any, ed: Dict[str, str]) -> Dict[str, str]:
-    """Stackdriver uses `message` and `severity` keys to display logs"""
-    ed["message"] = ed.pop("event")
-    ed["severity"] = ed.pop("level", "info").upper()
-    return ed
-
-
-class _MaxLevelFilter(Filter):
-    def __init__(self, highest_log_level: int) -> None:
-        self._highest_log_level = highest_log_level
-
-    def filter(self, log_record: LogRecord) -> bool:
-        return log_record.levelno <= self._highest_log_level