followthemoney 3.8.5__py3-none-any.whl → 4.0.1__py3-none-any.whl

followthemoney/__init__.py

@@ -1,16 +1,36 @@
-import os
-
+from followthemoney.entity import ValueEntity, VE
 from followthemoney.model import Model
+from followthemoney.schema import Schema
+from followthemoney.property import Property
+from followthemoney.types import registry
+from followthemoney.value import Value, Values
+from followthemoney.proxy import EntityProxy, E
+from followthemoney.statement import Statement, StatementEntity, SE
+from followthemoney.dataset import Dataset, DefaultDataset, DS
 from followthemoney.util import set_model_locale
 
-__version__ = "3.8.5"
-
-
-model_path = os.path.dirname(__file__)
-model_path = os.path.join(model_path, "schema")
-model_path = os.environ.get("FTM_MODEL_PATH", model_path)
+__version__ = "4.0.1"
 
 # Data model singleton
-model = Model(model_path)
+model = Model.instance()
 
-__all__ = ["model", "set_model_locale"]
+__all__ = [
+    "model",
+    "set_model_locale",
+    "Model",
+    "Schema",
+    "Property",
+    "Value",
+    "Values",
+    "EntityProxy",
+    "E",
+    "registry",
+    "Dataset",
+    "DefaultDataset",
+    "DS",
+    "Statement",
+    "StatementEntity",
+    "SE",
+    "ValueEntity",
+    "VE",
+]

followthemoney/cli/cli.py

@@ -56,7 +56,7 @@ def import_vis(infile: Path, outfile: Path) -> None:
             else:
                 raise click.ClickException("No entities found in VIS file")
             for entity_data in ensure_list(entities):
-                entity = EntityProxy.from_dict(model, entity_data)
+                entity = EntityProxy.from_dict(entity_data)
                 write_entity(outfh, entity)
 
 

followthemoney/cli/exports.py

@@ -6,8 +6,6 @@ from contextlib import contextmanager
 from followthemoney.cli.cli import cli
 from followthemoney.cli.util import InPath, OutPath, export_stream
 from followthemoney.export.csv import CSVExporter
-from followthemoney.export.rdf import RDFExporter
-from followthemoney.export.excel import ExcelExporter
 from followthemoney.export.graph import edge_types, DEFAULT_EDGE_TYPES
 from followthemoney.export.graph import NXGraphExporter
 from followthemoney.export.neo4j import Neo4JCSVExporter
@@ -46,6 +44,9 @@ def export_csv(infile: Path, outdir: Path) -> None:
     required=True,
 )
 def export_excel(infile: Path, outfile: Path) -> None:
+    # lazt load openpyxl
+    from followthemoney.export.excel import ExcelExporter
+
     exporter = ExcelExporter(outfile)
     export_stream(exporter, infile)
 
@@ -60,6 +61,9 @@ def export_excel(infile: Path, outfile: Path) -> None:
     help="Generate full predicates",
 )
 def export_rdf(infile: Path, outfile: Path, qualified: bool = True) -> None:
+    # Lazy load rdflib
+    from followthemoney.export.rdf import RDFExporter
+
     with text_out(outfile) as fh:
         exporter = RDFExporter(fh, qualified=qualified)
         export_stream(exporter, infile)

followthemoney/cli/statement.py

@@ -0,0 +1,62 @@
+import click
+from pathlib import Path
+from typing import Generator, List
+
+
+from followthemoney.cli.cli import cli
+from followthemoney.cli.util import InPath, OutPath
+from followthemoney.cli.util import path_entities, write_entity, path_writer
+from followthemoney.dataset import Dataset, DefaultDataset
+from followthemoney.statement import Statement, StatementEntity
+from followthemoney.statement import FORMATS, CSV
+from followthemoney.statement import write_statements
+from followthemoney.statement import read_path_statements
+
+
+@cli.command("statements", help="Export entities to statements")
+@click.argument("path", type=InPath)
+@click.option("-o", "--outpath", type=OutPath, default="-")
+@click.option("-d", "--dataset", type=str, required=True)
+@click.option("-f", "--format", type=click.Choice(FORMATS), default=CSV)
+def entity_statements(path: Path, outpath: Path, dataset: str, format: str) -> None:
+    def make_statements() -> Generator[Statement, None, None]:
+        for entity in path_entities(path, StatementEntity):
+            yield from Statement.from_entity(entity, dataset=dataset)
+
+    with path_writer(outpath) as outfh:
+        write_statements(outfh, format, make_statements())
+
+
+@cli.command("format-statements", help="Convert entity data formats")
+@click.option("-i", "--infile", type=InPath, default="-")
+@click.option("-o", "--outpath", type=OutPath, default="-")
+@click.option("-f", "--in-format", type=click.Choice(FORMATS), default=CSV)
+@click.option("-x", "--out-format", type=click.Choice(FORMATS), default=CSV)
+def format_statements(
+    infile: Path, outpath: Path, in_format: str, out_format: str
+) -> None:
+    statements = read_path_statements(infile, format=in_format)
+    with path_writer(outpath) as outfh:
+        write_statements(outfh, out_format, statements)
+
+
+@cli.command("aggregate-statements", help="Roll up statements into entities")
+@click.option("-i", "--infile", type=InPath, default="-")
+@click.option("-o", "--outpath", type=OutPath, default="-")
+@click.option("-d", "--dataset", type=str, default=DefaultDataset.name)
+@click.option("-f", "--format", type=click.Choice(FORMATS), default=CSV)
+def statements_aggregate(
+    infile: Path, outpath: Path, dataset: str, format: str
+) -> None:
+    dataset_ = Dataset.make({"name": dataset, "title": dataset})
+    with path_writer(outpath) as outfh:
+        statements: List[Statement] = []
+        for stmt in read_path_statements(infile, format=format):
+            if len(statements) and statements[0].canonical_id != stmt.canonical_id:
+                entity = StatementEntity.from_statements(dataset_, statements)
+                write_entity(outfh, entity)
+                statements = []
+            statements.append(stmt)
+        if len(statements):
+            entity = StatementEntity.from_statements(dataset_, statements)
+            write_entity(outfh, entity)

followthemoney/cli/util.py

@@ -9,7 +9,6 @@ from warnings import warn
 from typing import Any, BinaryIO, Generator, Optional, TextIO, Type
 from banal import is_mapping, is_listish, ensure_list
 
-from followthemoney import model
 from followthemoney.export.common import Exporter
 from followthemoney.proxy import E, EntityProxy
 from followthemoney.util import MEGABYTE, PathLike
@@ -39,7 +38,7 @@ def write_entity(fh: BinaryIO, entity: E) -> None:
 
 def _read_one(data: Any, cleaned: bool = True) -> Generator[EntityProxy, None, None]:
     if is_mapping(data) and "schema" in data:
-        yield model.get_proxy(data, cleaned=cleaned)
+        yield EntityProxy.from_dict(data, cleaned=cleaned)
 
 
 def read_entities(
@@ -79,7 +78,7 @@ def binary_entities(
 ) -> Generator[E, None, None]:
     while line := fh.readline(max_line):
         data = orjson.loads(line)
-        yield entity_type.from_dict(model, data, cleaned=cleaned)
+        yield entity_type.from_dict(data, cleaned=cleaned)
 
 
 def path_entities(

followthemoney/compare.py

@@ -1,10 +1,11 @@
 import math
-import itertools
+from itertools import islice, product
 from typing import Dict, Generator, Iterable, List, Optional
-import fingerprints
 from normality import normalize
+from rigour.names import tokenize_name, remove_person_prefixes
+from rigour.names import replace_org_types_compare
 from followthemoney.exc import InvalidData
-from followthemoney.model import Model
+from followthemoney.schema import Schema
 from followthemoney.types import registry
 from followthemoney.proxy import EntityProxy
 from followthemoney.types.common import PropertyType
@@ -21,16 +22,15 @@ COMPARE_WEIGHTS: Weights = {
     registry.address: 6.456137299747168,
     registry.phone: 3.538892687331418,
     registry.email: 14.115925628770384,
-    registry.iban: 0.019140301711998726,
     registry.url: 3.211995327345834,
     None: -11.91521189545115,
 }
 
 
-def compare_scores(model: Model, left: EntityProxy, right: EntityProxy) -> Scores:
+def compare_scores(left: EntityProxy, right: EntityProxy) -> Scores:
     """Compare two entities and return a match score for each property."""
     try:
-        model.common_schema(left.schema, right.schema)
+        common = left.schema.model.common_schema(left.schema, right.schema)
     except InvalidData:
         return {}
     scores: Scores = {}
@@ -42,7 +42,7 @@ def compare_scores(model: Model, left: EntityProxy, right: EntityProxy) -> Score
         group = registry.groups[group_name]
         try:
             if group == registry.name:
-                score = compare_names(left, right)
+                score = compare_names(common, left, right)
             elif group == registry.country:
                 score = compare_countries(left, right)
             else:
@@ -71,28 +71,38 @@ def _compare(scores: Scores, weights: Weights, n_std: int = 1) -> float:
 
 
 def compare(
-    model: Model,
     left: EntityProxy,
     right: EntityProxy,
     weights: Weights = COMPARE_WEIGHTS,
 ) -> float:
     """Compare two entities and return a match score."""
-    scores = compare_scores(model, left, right)
+    scores = compare_scores(left, right)
     return _compare(scores, weights)
 
 
-def _normalize_names(names: Iterable[str]) -> Generator[str, None, None]:
+def _normalize_names(
+    schema: Schema, names: Iterable[str]
+) -> Generator[str, None, None]:
     """Generate a sequence of comparable names for an entity. This also
-    generates a `fingerprint`, i.e. a version of the name where all tokens
+    generates a fingerprint, i.e. a version of the name where all tokens
     are sorted alphabetically, and some parts, such as company suffixes,
     have been removed."""
     seen = set()
+    can_person = schema.is_a("LegalEntity") and not schema.is_a("Organization")
+    can_org = schema.is_a("LegalEntity") and not schema.is_a("Person")
     for name in names:
         plain = normalize(name, ascii=True)
         if plain is not None and plain not in seen:
             seen.add(plain)
             yield plain
-        fp = fingerprints.generate(name)
+        if not can_org and not can_person:
+            continue
+        if can_person:
+            name = remove_person_prefixes(name)
+        if can_org:
+            name = replace_org_types_compare(name)
+        tokens = tokenize_name(name.lower())
+        fp = " ".join(sorted(tokens))
         if fp is not None and len(fp) > 6 and fp not in seen:
             seen.add(fp)
             yield fp
@@ -109,16 +119,16 @@ def compare_group(
 
 
 def compare_names(
-    left: EntityProxy, right: EntityProxy, max_names: int = 200
+    common: Schema, left: EntityProxy, right: EntityProxy, max_names: int = 200
 ) -> Optional[float]:
     result = 0.0
-    left_list = list(itertools.islice(_normalize_names(left.names), max_names))
-    right_list = list(itertools.islice(_normalize_names(right.names), max_names))
+    left_list = list(islice(_normalize_names(common, left.names), max_names))
+    right_list = list(islice(_normalize_names(common, right.names), max_names))
     if not left_list and not right_list:
         raise ValueError("At least one proxy must have name properties")
     elif not left_list or not right_list:
         return None
-    for (left_val, right_val) in itertools.product(left_list, right_list):
+    for left_val, right_val in product(left_list, right_list):
         similarity = registry.name.compare(left_val, right_val)
         result = max(result, similarity)
         if result == 1.0:

followthemoney/dataset/__init__.py

@@ -0,0 +1,17 @@
+from followthemoney.dataset.dataset import Dataset, DS
+from followthemoney.dataset.catalog import DataCatalog
+from followthemoney.dataset.resource import DataResource
+from followthemoney.dataset.publisher import DataPublisher
+from followthemoney.dataset.coverage import DataCoverage
+
+DefaultDataset = Dataset.make({"name": "default"})
+
+__all__ = [
+    "Dataset",
+    "DefaultDataset",
+    "DataCatalog",
+    "DataResource",
+    "DataPublisher",
+    "DataCoverage",
+    "DS",
+]

followthemoney/dataset/catalog.py

@@ -0,0 +1,77 @@
+import yaml
+from typing import Optional, Dict, Any, Generic, Set, Type, List
+
+from followthemoney.types import registry
+from followthemoney.dataset.dataset import DS
+from followthemoney.exc import MetadataException
+from followthemoney.util import PathLike
+
+
+class DataCatalog(Generic[DS]):
+    """A data catalog is a collection of datasets. It provides methods for retrieving or
+    creating datasets, and for checking if a dataset exists in the catalog."""
+
+    def __init__(self, dataset_type: Type[DS], data: Dict[str, Any]) -> None:
+        self.dataset_type = dataset_type
+        self.datasets: List[DS] = []
+        for ddata in data.get("datasets", []):
+            self.make_dataset(ddata)
+        self.updated_at: Optional[str] = None
+        if "updated_at" in data:
+            raw = data.get("updated_at")
+            self.updated_at = registry.date.clean(raw)
+            if self.updated_at is None:
+                raise MetadataException("Invalid update date: %r" % raw)
+
+    def add(self, dataset: "DS") -> None:
+        """Add a dataset to the catalog. If the dataset already exists, it will be updated."""
+        for existing in self.datasets:
+            if existing.name in dataset.model.children:
+                dataset.children.add(existing)
+            if dataset.name in existing.model.children:
+                existing.children.add(dataset)
+        self.datasets.append(dataset)
+
+    def make_dataset(self, data: Dict[str, Any]) -> "DS":
+        """Create a new dataset from the given data. If a dataset with the same name already
+        exists, it will be updated."""
+        dataset = self.dataset_type(data)
+        self.add(dataset)
+        return dataset
+
+    def get(self, name: str) -> Optional["DS"]:
+        """Get a dataset by name. Returns None if the dataset does not exist."""
+        for ds in self.datasets:
+            if ds.name == name:
+                return ds
+        return None
+
+    def require(self, name: str) -> "DS":
+        """Get a dataset by name. Raises MetadataException if the dataset does not exist."""
+        dataset = self.get(name)
+        if dataset is None:
+            raise MetadataException("No such dataset: %s" % name)
+        return dataset
+
+    def has(self, name: str) -> bool:
+        """Check if a dataset exists in the catalog."""
+        return name in self.names
+
+    @property
+    def names(self) -> Set[str]:
+        """Get the names of all datasets in the catalog."""
+        return {d.name for d in self.datasets}
+
+    def __repr__(self) -> str:  # pragma: no cover
+        return f"<DataCatalog[{self.dataset_type.__name__}]({self.names!r})>"
+
+    def to_dict(self) -> Dict[str, Any]:
+        return {
+            "datasets": [d.to_dict() for d in self.datasets],
+            "updated_at": self.updated_at,
+        }
+
+    @classmethod
+    def from_path(cls, dataset_type: Type[DS], path: PathLike) -> "DataCatalog[DS]":
+        with open(path, "r") as fh:
+            return cls(dataset_type, yaml.safe_load(fh))

followthemoney/dataset/coverage.py

@@ -0,0 +1,29 @@
+from typing import List, Literal, Optional, TypeAlias
+from pydantic import BaseModel
+
+from followthemoney.dataset.util import CountryCode, PartialDate
+
+
+# Derived from Aleph
+FREQUENCY_TYPE: TypeAlias = Literal[
+    "unknown",
+    "never",
+    "hourly",
+    "daily",
+    "weekly",
+    "monthly",
+    "annually",
+]
+
+
+class DataCoverage(BaseModel):
+    """Details on the temporal and geographic scope of a dataset."""
+
+    start: Optional[PartialDate] = None
+    end: Optional[PartialDate] = None
+    countries: List[CountryCode] = []
+    frequency: FREQUENCY_TYPE = "unknown"
+    schedule: Optional[str] = None
+
+    def __repr__(self) -> str:
+        return f"<DataCoverage({self.start!r}, {self.end!r}, {self.countries!r})>"

followthemoney/dataset/dataset.py

@@ -0,0 +1,146 @@
+import yaml
+import logging
+from functools import cached_property
+from typing import TYPE_CHECKING
+from typing_extensions import Self
+from typing import Any, Dict, List, Optional, Set, Type, TypeVar
+from pydantic import BaseModel, field_validator, model_validator
+
+from followthemoney.dataset.coverage import DataCoverage
+from followthemoney.dataset.publisher import DataPublisher
+from followthemoney.dataset.resource import DataResource
+from followthemoney.dataset.util import Url, DateTimeISO, dataset_name_check
+from followthemoney.util import PathLike
+
+if TYPE_CHECKING:
+    from followthemoney.dataset.catalog import DataCatalog
+
+DS = TypeVar("DS", bound="Dataset")
+
+log = logging.getLogger(__name__)
+
+
+class DatasetModel(BaseModel):
+    name: str
+    title: str
+    license: Optional[Url] = None
+    summary: Optional[str] = None
+    description: Optional[str] = None
+    url: Optional[Url] = None
+    updated_at: Optional[DateTimeISO] = None
+    last_export: Optional[DateTimeISO] = None
+    entity_count: Optional[int] = None
+    thing_count: Optional[int] = None
+    version: Optional[str] = None
+    category: Optional[str] = None
+    tags: List[str] = []
+    publisher: DataPublisher | None = None
+    coverage: DataCoverage | None = None
+    resources: List[DataResource] = []
+    children: Set[str] = set()
+
+    @field_validator("name", mode="after")
+    @classmethod
+    def check_name(cls, value: str) -> str:
+        return dataset_name_check(value)
+
+    @model_validator(mode="before")
+    @classmethod
+    def ensure_data(cls, data: Any) -> Any:
+        if isinstance(data, dict):
+            if "name" not in data:
+                raise ValueError("Missing dataset name")
+            data["title"] = data.get("title", data["name"])
+            children = set(data.get("children", []))
+            children.update(data.get("datasets", []))
+            children.update(data.get("scopes", []))
+            data["children"] = children
+        return data
+
+    def get_resource(self, name: str) -> DataResource:
+        for res in self.resources:
+            if res.name == name:
+                return res
+        raise ValueError("No resource named %r!" % name)
+
+
+class Dataset:
+    """A container for entities, often from one source or related to one topic.
+    A dataset is a set of data, sez W3C."""
+
+    Model = DatasetModel
+
+    def __init__(self: Self, data: Dict[str, Any]) -> None:
+        self.model = self.Model.model_validate(data)
+        self.name = self.model.name
+        self.children: Set[Self] = set()
+
+    @cached_property
+    def is_collection(self: Self) -> bool:
+        return len(self.model.children) > 0
+
+    @property
+    def datasets(self: Self) -> Set[Self]:
+        current: Set[Self] = set([self])
+        for child in self.children:
+            current.update(child.datasets)
+        return current
+
+    @property
+    def dataset_names(self: Self) -> List[str]:
+        return [d.name for d in self.datasets]
+
+    @property
+    def leaves(self: Self) -> Set[Self]:
+        """All contained datasets which are not collections (can be 'self')."""
+        return set([d for d in self.datasets if not d.is_collection])
+
+    @property
+    def leaf_names(self: Self) -> Set[str]:
+        return {d.name for d in self.leaves}
+
+    def __hash__(self) -> int:
+        return hash(repr(self))
+
+    def __repr__(self) -> str:
+        if not hasattr(self, "name"):
+            return "<Dataset>"
+        return f"<Dataset({self.name})>"  # pragma: no cover
+
+    def get_resource(self, name: str) -> DataResource:
+        for res in self.model.resources:
+            if res.name == name:
+                return res
+        raise ValueError("No resource named %r!" % name)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert the dataset to a dictionary representation."""
+        return self.model.model_dump(mode="json", exclude_none=True)
+
+    @classmethod
+    def from_path(
+        cls: Type[DS], path: PathLike, catalog: Optional["DataCatalog[DS]"] = None
+    ) -> DS:
+        from followthemoney.dataset.catalog import DataCatalog
+
+        with open(path, "r") as fh:
+            data = yaml.safe_load(fh)
+            if catalog is None:
+                catalog = DataCatalog(cls, {})
+            return catalog.make_dataset(data)
+
+    @classmethod
+    def make(cls: Type[DS], data: Dict[str, Any]) -> DS:
+        from followthemoney.dataset.catalog import DataCatalog
+
+        catalog = DataCatalog(cls, {})
+        return catalog.make_dataset(data)
+
+    def __eq__(self, other: Any) -> bool:
+        try:
+            return not not self.name == other.name
+        except AttributeError:
+            return False
+
+    def __lt__(self, other: Any) -> bool:
+        return self.name.__lt__(other.name)

followthemoney/dataset/publisher.py

@@ -0,0 +1,25 @@
+from typing import Optional
+
+from pydantic import BaseModel
+
+from followthemoney.dataset.util import CountryCode, Url
+from followthemoney.types import registry
+
+
+class DataPublisher(BaseModel):
+    """Publisher information, eg. the government authority."""
+
+    name: str
+    url: Optional[Url] = None
+    name_en: Optional[str] = None
+    acronym: Optional[str] = None
+    description: Optional[str] = None
+    country: Optional[CountryCode] = None
+    official: Optional[bool] = False
+    logo_url: Optional[Url] = None
+
+    @property
+    def country_label(self) -> Optional[str]:
+        if self.country is None:
+            return None
+        return registry.country.caption(self.country)

followthemoney/dataset/resource.py

@@ -0,0 +1,30 @@
+from typing import Optional
+from pydantic import BaseModel, field_validator
+
+from followthemoney.dataset.util import Url, DateTimeISO
+from followthemoney.types import registry
+
+
+class DataResource(BaseModel):
+    """A downloadable resource that is part of a dataset."""
+
+    name: str
+    url: Optional[Url] = None
+    checksum: Optional[str] = None
+    timestamp: Optional[DateTimeISO] = None
+    mime_type: Optional[str] = None
+    title: Optional[str] = None
+    size: Optional[int] = None
+
+    @field_validator("mime_type", mode="after")
+    @classmethod
+    def ensure_mime_type(cls, value: str) -> Optional[str]:
+        if not registry.mimetype.validate(value):
+            raise ValueError(f"Invalid MIME type: {value!r}")
+        return value
+
+    @property
+    def mime_type_label(self) -> Optional[str]:
+        if self.mime_type is None:
+            return None
+        return registry.mimetype.caption(self.mime_type)

followthemoney/dataset/util.py

@@ -0,0 +1,55 @@
+from datetime import datetime
+from normality import slugify
+from typing import Annotated, Any
+from rigour.time import datetime_iso
+from pydantic import AfterValidator, BeforeValidator, HttpUrl, PlainSerializer
+
+from followthemoney.types import registry
+
+
+def dataset_name_check(value: str) -> str:
+    """Check that the given value is a valid dataset name. This doesn't convert
+    or clean invalid names, but raises an error if they are not compliant to
+    force the user to fix an invalid name"""
+    if slugify(value, sep="_") != value:
+        raise ValueError("Invalid %s: %r" % ("dataset name", value))
+    return value
+
+
+def type_check_date(value: Any) -> str:
+    """Check that the given value is a valid date string."""
+    cleaned = registry.date.clean(value)
+    if cleaned is None:
+        raise ValueError("Invalid date: %r" % value)
+    return cleaned
+
+
+PartialDate = Annotated[str, BeforeValidator(type_check_date)]
+
+
+def type_check_country(value: Any) -> str:
+    """Check that the given value is a valid country code."""
+    cleaned = registry.country.clean(value)
+    if cleaned is None:
+        raise ValueError("Invalid country code: %r" % value)
+    return cleaned
+
+
+CountryCode = Annotated[str, BeforeValidator(type_check_country)]
+
+
+def type_check_http_url(v: str) -> str:
+    url = HttpUrl(v)
+    return str(url)
+
+
+Url = Annotated[str, AfterValidator(type_check_http_url)]
+
+
+def serialize_dt(dt: datetime) -> str:
+    text = datetime_iso(dt)
+    assert text is not None, "Invalid datetime: %r" % dt
+    return text
+
+
+DateTimeISO = Annotated[datetime, PlainSerializer(serialize_dt)]