followthemoney 4.5.0__py3-none-any.whl → 4.5.1__py3-none-any.whl

followthemoney/__init__.py

@@ -2,14 +2,14 @@ from followthemoney.entity import ValueEntity, VE
 from followthemoney.model import Model
 from followthemoney.schema import Schema
 from followthemoney.property import Property
-from followthemoney.types import registry
+from followthemoney.types import registry, PropertyType
 from followthemoney.value import Value, Values
 from followthemoney.proxy import EntityProxy, E
 from followthemoney.statement import Statement, StatementEntity, SE
-from followthemoney.dataset import Dataset, DefaultDataset, DS
+from followthemoney.dataset import Dataset, UndefinedDataset, DS
 from followthemoney.util import set_model_locale
 
-__version__ = "4.5.0"
+__version__ = "4.5.1"
 
 # Data model singleton
 model = Model.instance()
@@ -20,13 +20,14 @@ __all__ = [
     "Model",
     "Schema",
     "Property",
+    "PropertyType",
     "Value",
     "Values",
     "EntityProxy",
     "E",
     "registry",
     "Dataset",
-    "DefaultDataset",
+    "UndefinedDataset",
     "DS",
     "Statement",
     "StatementEntity",

followthemoney/cli/statement.py

@@ -1,12 +1,12 @@
 import click
 from pathlib import Path
-from typing import Generator, List
+from typing import Generator, List, Optional
 
 
 from followthemoney.cli.cli import cli
 from followthemoney.cli.util import InPath, OutPath
 from followthemoney.cli.util import path_entities, write_entity, path_writer
-from followthemoney.dataset import Dataset, DefaultDataset
+from followthemoney.dataset import Dataset, UndefinedDataset
 from followthemoney.statement import Statement, StatementEntity
 from followthemoney.statement import FORMATS, CSV
 from followthemoney.statement import write_statements
@@ -16,12 +16,18 @@ from followthemoney.statement import read_path_statements
 @cli.command("statements", help="Export entities to statements")
 @click.argument("path", type=InPath)
 @click.option("-o", "--outpath", type=OutPath, default="-")
-@click.option("-d", "--dataset", type=str, required=True)
+@click.option("-d", "--dataset", type=str)
 @click.option("-f", "--format", type=click.Choice(FORMATS), default=CSV)
-def entity_statements(path: Path, outpath: Path, dataset: str, format: str) -> None:
+def entity_statements(
+    path: Path, outpath: Path, dataset: Optional[str], format: str
+) -> None:
     def make_statements() -> Generator[Statement, None, None]:
+        dataset_ = dataset or Dataset.UNDEFINED
         for entity in path_entities(path, StatementEntity):
-            yield from Statement.from_entity(entity, dataset=dataset)
+            for stmt in Statement.from_entity(entity, dataset=dataset_):
+                if dataset is not None:
+                    stmt = stmt.clone(dataset=dataset)
+                yield stmt
 
     with path_writer(outpath) as outfh:
         write_statements(outfh, format, make_statements())
@@ -43,12 +49,12 @@ def format_statements(
 @cli.command("aggregate-statements", help="Roll up statements into entities")
 @click.option("-i", "--infile", type=InPath, default="-")
 @click.option("-o", "--outpath", type=OutPath, default="-")
-@click.option("-d", "--dataset", type=str, default=DefaultDataset.name)
+@click.option("-d", "--dataset", type=str, default=UndefinedDataset.name)
 @click.option("-f", "--format", type=click.Choice(FORMATS), default=CSV)
 def statements_aggregate(
     infile: Path, outpath: Path, dataset: str, format: str
 ) -> None:
-    dataset_ = Dataset.make({"name": dataset, "title": dataset})
+    dataset_ = Dataset.make({"name": dataset})
     with path_writer(outpath) as outfh:
         statements: List[Statement] = []
         for stmt in read_path_statements(infile, format=format):

followthemoney/cli/util.py

@@ -6,7 +6,7 @@ import click
 import orjson
 from pathlib import Path
 from warnings import warn
-from typing import Any, BinaryIO, Generator, Optional, TextIO, Type
+from typing import Any, BinaryIO, Generator, List, Optional, TextIO, Type
 from banal import is_mapping, is_listish, ensure_list
 
 from followthemoney.export.common import Exporter
@@ -26,7 +26,7 @@ def write_object(stream: TextIO, obj: Any) -> None:
     stream.write(data + "\n")
 
 
-def write_entity(fh: BinaryIO, entity: E) -> None:
+def write_entity(fh: BinaryIO, entity: EntityProxy) -> None:
     data = entity.to_dict()
     entity_id = data.pop("id")
     assert entity_id is not None, data
@@ -131,7 +131,7 @@ def resolve_includes(file_path: PathLike, data: Any) -> Any:
     if is_listish(data):
         return [resolve_includes(file_path, i) for i in data]
     if is_mapping(data):
-        include_paths = ensure_list(data.pop("include", []))
+        include_paths: List[str] = ensure_list(data.pop("include", []))
         for include_path in include_paths:
             dir_prefix = os.path.dirname(file_path)
             include_path = os.path.join(dir_prefix, include_path)

followthemoney/dataset/__init__.py

@@ -4,11 +4,11 @@ from followthemoney.dataset.resource import DataResource
 from followthemoney.dataset.publisher import DataPublisher
 from followthemoney.dataset.coverage import DataCoverage
 
-DefaultDataset = Dataset.make({"name": "default"})
+UndefinedDataset = Dataset.make({"name": Dataset.UNDEFINED})
 
 __all__ = [
     "Dataset",
-    "DefaultDataset",
+    "UndefinedDataset",
     "DataCatalog",
     "DataResource",
     "DataPublisher",

followthemoney/dataset/dataset.py

@@ -83,6 +83,8 @@ class Dataset:
     """A container for entities, often from one source or related to one topic.
     A dataset is a set of data, sez W3C."""
 
+    UNDEFINED = "undefined"
+
     def __init__(self: Self, data: Dict[str, Any]) -> None:
         self.model = DatasetModel.model_validate(data)
         self.name = self.model.name

followthemoney/statement/entity.py

@@ -15,7 +15,7 @@ from followthemoney.proxy import P
 from followthemoney.types import registry
 from followthemoney.value import string_list, Values
 from followthemoney.proxy import EntityProxy
-from followthemoney.dataset import Dataset, DefaultDataset
+from followthemoney.dataset import Dataset, UndefinedDataset
 from followthemoney.statement.statement import Statement
 from followthemoney.statement.util import BASE_ID
 
@@ -190,6 +190,11 @@ class StatementEntity(EntityProxy):
             return []
         return list(self._statements[prop_name])
 
+    @property
+    def has_statements(self) -> bool:
+        """Return whether the entity has any statements."""
+        return len(self._statements) > 0
+
     def set(
         self,
         prop: P,
@@ -433,7 +438,7 @@ class StatementEntity(EntityProxy):
                     origins.add(stmt.origin)
 
         data["referents"] = list(referents)
-        data["datasets"] = list(datasets)
+        data["datasets"] = [d for d in datasets if d != Dataset.UNDEFINED]
         if origins:
             data["origin"] = list(origins)
 
@@ -484,7 +489,7 @@ class StatementEntity(EntityProxy):
         default_dataset: Optional[Dataset] = None,
     ) -> SE:
         # Exists only for backwards compatibility.
-        dataset = default_dataset or DefaultDataset
+        dataset = default_dataset or UndefinedDataset
         return cls(dataset, data, cleaned=cleaned)
 
     @classmethod

followthemoney/statement/serialize.py

@@ -1,4 +1,5 @@
 import csv
+import sys
 import click
 import orjson
 import logging
@@ -49,6 +50,7 @@ LEGACY_PACK_COLUMNS = [
     "first_seen",
     "last_seen",
 ]
+csv.field_size_limit(sys.maxsize)
 
 
 def read_json_statements(
@@ -69,6 +71,8 @@ def read_csv_statements(fh: BinaryIO) -> Generator[Statement, None, None]:
             data["lang"] = None
         if row.get("original_value") == "":
             data["original_value"] = None
+        if row.get("origin") == "":
+            data["origin"] = None
         yield Statement.from_dict(data)
 
 
@@ -101,7 +105,7 @@ def read_pack_statements_decoded(fh: TextIO) -> Generator[Statement, None, None]
             dataset=data["dataset"],
             lang=data["lang"] or None,
             original_value=data["original_value"] or None,
-            origin=data.get("origin"),
+            origin=data.get("origin") or None,
             first_seen=data["first_seen"],
             external=data["external"] == "t",
             canonical_id=data["entity_id"],

{followthemoney-4.5.0.dist-info → followthemoney-4.5.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: followthemoney
-Version: 4.5.0
+Version: 4.5.1
 Summary: A data model for anti corruption data modeling and analysis.
 Project-URL: Documentation, https://followthemoney.tech/
 Project-URL: Repository, https://github.com/opensanctions/followthemoney.git

{followthemoney-4.5.0.dist-info → followthemoney-4.5.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-followthemoney/__init__.py,sha256=3Mvq7FWl0cNbTovkJhys0iHU24UdFT44P2AqszoGs-8,856
+followthemoney/__init__.py,sha256=xxSC9m3THm5tFr_l8TY-RAEKb-ymcueZsmV4JVEjPyM,894
 followthemoney/compare.py,sha256=6y6fqtbbfW7ee4_EVXPcKCIr75GjPqXvtHfvExJ-KK0,6119
 followthemoney/entity.py,sha256=biAjuuHlwsVT02imAsaWP0YtgdfU8skCntzBU3mgJpg,4052
 followthemoney/exc.py,sha256=GyMgwY4QVm87hLevDfV7gM1MJsDqfNCi_UQw7F_A8X8,858
@@ -22,12 +22,12 @@ followthemoney/cli/cli.py,sha256=cWSQIrMS0b40uzIveoIfR9CEBbQEwcfonYhDTpioqBM,358
 followthemoney/cli/exports.py,sha256=arWgIfDhMdEEoAnANOPbgirL68qAR2pCt0otz9MEXmg,4074
 followthemoney/cli/mapping.py,sha256=PGQ-9T5ss6w6qnZg7IjUZZ3PplY15CcPSxZxkyMFLDM,3370
 followthemoney/cli/sieve.py,sha256=wLB35fCVp1ArZ7FDTbARevBk8jH4vnp65fyBZU7Lk_k,1937
-followthemoney/cli/statement.py,sha256=kjcOzcwWT_cHCwriiYA8B05BJpwVoI3WMJjghzPOCsY,2871
-followthemoney/cli/util.py,sha256=C3nGMVY3-9JHSFLn3AGvTNcAdvGcgfFS-7jXIzKg6Ik,4735
-followthemoney/dataset/__init__.py,sha256=rOKsI39dccDaYcSa7ASoNKkhmbFYUArxMCRqtrxy2iE,477
+followthemoney/cli/statement.py,sha256=g1Ksi3iS-YsuVA40AnKpIsVDPRc4SovAe1JCoXNIW4g,3041
+followthemoney/cli/util.py,sha256=EyhR6qLIfueG1OK5_DwDA_7mcDPnhNs5dyijbP3WWZY,4762
+followthemoney/dataset/__init__.py,sha256=1-I1MkuTvu72tpZZcxD2GK-jq0_TwNJ29zaFQIo2ORQ,489
 followthemoney/dataset/catalog.py,sha256=bIpxr0jvJeutNSmCaXREQac7TyvZak2Y_QoCFdCM0d4,3001
 followthemoney/dataset/coverage.py,sha256=rBnKs7VngCtIuaDqrF5D0ygCHg8NAMkYbmtl7336PSI,724
-followthemoney/dataset/dataset.py,sha256=7lP3gz94AAaxX6J7OWlNmvPoWkoKvH7ISwoNqYsC4Go,5495
+followthemoney/dataset/dataset.py,sha256=TV3enQCWJCo2cXZCEz_yPWsF-09W0p_A3RSroRbPWV8,5524
 followthemoney/dataset/publisher.py,sha256=nexZe9XexV8WI5Id999vf5OH_DPUmiKQ_GT3c59eF44,893
 followthemoney/dataset/resource.py,sha256=S_-tNjMwHQ8LcSOsZO_xhXD-vLK90wyxtIRBbyCJ0Xo,1164
 followthemoney/dataset/util.py,sha256=mfVTXdbNnWly6cXo4SjNzHuJK1c1uNBwULYOVg1gK5I,1617
@@ -115,8 +115,8 @@ followthemoney/schema/Vessel.yaml,sha256=zWHUfSK8g6Pz58ZyCaK0AFJ4u_UHjEIUGC4c_7o
 followthemoney/schema/Video.yaml,sha256=LY3DYMWTHXiAhL0hxBCNCz50cp2sPbUlEhhig5Fbjos,327
 followthemoney/schema/Workbook.yaml,sha256=iikWPElz4klA7SkWH7eae6xqhbkMCIP_3zdeXzFEMU0,354
 followthemoney/statement/__init__.py,sha256=7m2VUCAuqNZXIY0WFJRFkw5UG14QuxATL4f_xbqKwhw,633
-followthemoney/statement/entity.py,sha256=vznbPTMSWcezYt--bu2RretMresgwp-bUqCsv4w-U90,19568
-followthemoney/statement/serialize.py,sha256=PcG2Qf1jYcF_rF1YybYWBhX7NiX6WZ94u1WUv9Mh-Lw,7386
+followthemoney/statement/entity.py,sha256=jwT1OiM6NiGbCWTZIG4wi0a-ROGy-GN6Vpq9pyp5mbs,19759
+followthemoney/statement/serialize.py,sha256=TLpeCrpgFrQ6_AA3tcNxZhcOjlo_-c1MNfh3zE3idj4,7509
 followthemoney/statement/statement.py,sha256=bAwrrKyYRdJVxRGuXDQIOA7cdEngk1NKUaij_gAwSd4,11876
 followthemoney/statement/util.py,sha256=jHBwK3FIBynUJZRlBBOHayalAFrqpXf2f2JwkHi0zAU,1450
 followthemoney/translations/messages.pot,sha256=JhtY9NJ9wP_EAX4APxOqMyvKcX53oIC9kAxBsliJkf4,107703
@@ -162,8 +162,8 @@ followthemoney/types/phone.py,sha256=_HanfxxTV7jp75gZO2evBc9HWwQTxEMQRaoVDcoXDIQ
 followthemoney/types/string.py,sha256=SEh3xqQCnm377PGvwfR6ao85pHJCNeCUWBKnvccrJ7I,1216
 followthemoney/types/topic.py,sha256=9FIH_WmwVOFg1CJRBF4KeE6vNTn-QQkzsKU5XaMqNJ0,4604
 followthemoney/types/url.py,sha256=sSHKtzvm4kc-VTvNCPIDykOG1hUoawhORj6Bklo0a2A,1434
-followthemoney-4.5.0.dist-info/METADATA,sha256=dteJlaqiJHM7Du9BTLMiOgl-DEjNp2ewkkFU8nMcCYA,6747
-followthemoney-4.5.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-followthemoney-4.5.0.dist-info/entry_points.txt,sha256=caoFTlf213jhg5sz3TNSofutjUTzaKtWATuSIdd9Cps,653
-followthemoney-4.5.0.dist-info/licenses/LICENSE,sha256=H6_EVXisnJC0-18CjXIaqrBSFq_VH3OnS7u3dccOv6g,1148
-followthemoney-4.5.0.dist-info/RECORD,,
+followthemoney-4.5.1.dist-info/METADATA,sha256=kRhvZB4Bvc1aVmGx2n2dyvYoOGhU16uUBqKf69C7bck,6747
+followthemoney-4.5.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+followthemoney-4.5.1.dist-info/entry_points.txt,sha256=caoFTlf213jhg5sz3TNSofutjUTzaKtWATuSIdd9Cps,653
+followthemoney-4.5.1.dist-info/licenses/LICENSE,sha256=H6_EVXisnJC0-18CjXIaqrBSFq_VH3OnS7u3dccOv6g,1148
+followthemoney-4.5.1.dist-info/RECORD,,