statcatlib 0.8.0__py3-none-any.whl

statcatlib/expr.py

@@ -0,0 +1,167 @@
+"""Sub-traversal expression language for scope-based graph queries.
+
+Expressions describe computations over related graph data without executing
+any SQL.  They are context-free objects — like ``pl.col()`` in Polars — and
+can be reused across ``.enrich()``, ``.filter()``, and ``.sort_by()`` on any
+collection scope.
+
+Usage::
+
+    import statcatlib as sc
+
+    n_datasets = sc.each("datasets").count()
+    has_data = sc.each("datasets").exists()
+
+    store.themes["subjects"].enrich(n=n_datasets).filter(has_data).collect()
+"""
+
+from __future__ import annotations
+
+from attrs import define
+
+# ---------------------------------------------------------------------------
+# Base expression types
+# ---------------------------------------------------------------------------
+
+
+@define(frozen=True)
+class Expr:
+    """Base class for all expression nodes."""
+
+    def alias(self, name: str) -> AliasedExpr:
+        """Name this expression result (like Polars ``.alias()``)."""
+        return AliasedExpr(expr=self, name=name)
+
+    def count(self) -> CountExpr:
+        """Count items at the far end of the traversal."""
+        return CountExpr(source=self)
+
+    def exists(self) -> ExistsExpr:
+        """Check if any items exist at the far end."""
+        return ExistsExpr(source=self)
+
+    def sum(self, field: str) -> SumExpr:
+        """Sum a numeric field at the far end."""
+        return SumExpr(source=self, field=field)
+
+    def ids(self) -> IdsExpr:
+        """Collect identifiers at the far end."""
+        return IdsExpr(source=self)
+
+    def distinct(self, field: str) -> DistinctExpr:
+        """Select distinct values of a field at the far end."""
+        return DistinctExpr(source=self, field=field)
+
+    def filter(self, **kwargs: object) -> FilteredExpr:
+        """Filter far-side nodes by attribute values."""
+        return FilteredExpr(source=self, conditions=kwargs)
+
+    def count_distinct(self) -> CountDistinctExpr:
+        """Count distinct items at the far end."""
+        return CountDistinctExpr(source=self)
+
+
+@define(frozen=True)
+class TraversalExpr(Expr):
+    """A traversal along one or more edges from the current scope position.
+
+    Created by ``sc.each("edge1", "edge2", ...)``.  Strings are always
+    edge labels (table names or relationship names in the graph).
+    """
+
+    edges: tuple[str, ...]
+
+
+@define(frozen=True)
+class AliasedExpr:
+    """An expression with a named output column.
+
+    Created by ``expr.alias("name")`` or via ``**kwargs`` in ``.enrich()``.
+    """
+
+    expr: Expr
+    name: str
+
+
+# ---------------------------------------------------------------------------
+# Terminal reduction expressions
+# ---------------------------------------------------------------------------
+
+
+@define(frozen=True)
+class CountExpr(Expr):
+    """COUNT(*) at the far end of a traversal."""
+
+    source: Expr
+
+
+@define(frozen=True)
+class ExistsExpr(Expr):
+    """EXISTS(...) — boolean, true if any far-side items exist."""
+
+    source: Expr
+
+
+@define(frozen=True)
+class SumExpr(Expr):
+    """SUM(field) at the far end of a traversal."""
+
+    source: Expr
+    field: str
+
+
+@define(frozen=True)
+class IdsExpr(Expr):
+    """LIST(id) — collect identifiers at the far end."""
+
+    source: Expr
+
+
+@define(frozen=True)
+class DistinctExpr(Expr):
+    """SELECT DISTINCT field — narrows to unique values."""
+
+    source: Expr
+    field: str
+
+
+@define(frozen=True)
+class CountDistinctExpr(Expr):
+    """COUNT(DISTINCT ...) at the far end."""
+
+    source: Expr
+
+
+@define(frozen=True)
+class FilteredExpr(Expr):
+    """A traversal with far-side filtering applied.
+
+    Created by ``sc.each("datasets").filter(status="current")``.
+    """
+
+    source: Expr
+    conditions: dict[str, object]
+
+
+# ---------------------------------------------------------------------------
+# Public entry point
+# ---------------------------------------------------------------------------
+
+
+def each(*edges: str) -> TraversalExpr:
+    """Create a sub-traversal expression starting from the current scope.
+
+    Each argument is an edge label (relationship name).  Multiple arguments
+    describe a multi-hop path::
+
+        sc.each("datasets")  # one hop
+        sc.each("datasets", "dimensions")  # two hops
+        sc.each("datasets", "dimensions", "codelist")  # three hops
+
+    The returned expression is context-free and can be used in ``.enrich()``,
+    ``.filter()``, and ``.sort_by()`` on any collection scope.
+    """
+    if not edges:
+        msg = "each() requires at least one edge label"
+        raise ValueError(msg)
+    return TraversalExpr(edges=edges)

statcatlib/model/__init__.py

@@ -0,0 +1,56 @@
+"""Domain model for statcatlib."""
+
+from statcatlib.model.agents import Agent, ContactPoint
+from statcatlib.model.catalog import CatalogDescription, CatalogRecord
+from statcatlib.model.classification import Standard
+from statcatlib.model.dataset import (
+    AggregateDataset,
+    BaseDataset,
+    Dataset,
+    DatasetT,
+    GeospatialDataset,
+    MicrodataDataset,
+    OpenDataset,
+    PublicationDataset,
+    StatisticalDataset,
+    Variable,
+)
+from statcatlib.model.distribution import DataService, Distribution
+from statcatlib.model.licence import LicenceDocument
+from statcatlib.model.quality import ProvenanceStatement, QualityAnnotation
+from statcatlib.model.spatial import SpatialCoverage
+from statcatlib.model.temporal import Frequency, TemporalCoverage
+from statcatlib.model.skos import Concept, ConceptRef, ConceptScheme, ConceptSchemeRef, is_concept_in_scheme
+from statcatlib.model.types import InternationalString
+
+__all__ = [
+    "Agent",
+    "AggregateDataset",
+    "BaseDataset",
+    "CatalogDescription",
+    "CatalogRecord",
+    "Concept",
+    "ConceptRef",
+    "ConceptScheme",
+    "ConceptSchemeRef",
+    "ContactPoint",
+    "DataService",
+    "Dataset",
+    "DatasetT",
+    "Distribution",
+    "Frequency",
+    "GeospatialDataset",
+    "InternationalString",
+    "LicenceDocument",
+    "MicrodataDataset",
+    "OpenDataset",
+    "ProvenanceStatement",
+    "PublicationDataset",
+    "QualityAnnotation",
+    "SpatialCoverage",
+    "Standard",
+    "StatisticalDataset",
+    "TemporalCoverage",
+    "Variable",
+    "is_concept_in_scheme",
+]

statcatlib/model/agents.py

@@ -0,0 +1,32 @@
+"""Agent and contact types for the statcatlib domain model."""
+
+from attrs import define
+from sdmxlib import InternationalString
+
+__all__ = ["Agent", "ContactPoint"]
+
+
+@define
+class Agent:
+    """An organization or person responsible for a dataset.
+
+    Maps to foaf:Agent / dct:publisher / dct:creator in DCAT-AP.
+    """
+
+    name: InternationalString
+    id: str | None = None
+    type: str | None = None
+    homepage: str | None = None
+
+
+@define
+class ContactPoint:
+    """Contact information for a dataset.
+
+    Maps to dcat:contactPoint → vcard:Kind in DCAT-AP.
+    """
+
+    name: InternationalString | None = None
+    email: str | None = None
+    phone: str | None = None
+    url: str | None = None

statcatlib/model/catalog.py

@@ -0,0 +1,71 @@
+"""CatalogDescription and CatalogRecord types for the statcatlib domain model.
+
+DCAT-AP describes a catalog as a top-level container of datasets. In the
+statcatlib runtime, a *catalog* is the DuckDB-backed `statcatlib.Catalog`
+store. The DCAT-style metadata record that describes such a catalog (its
+title, publisher, list of datasets…) is `CatalogDescription` here.
+This split avoids the name collision between "DCAT Catalog the model
+record" and "Catalog the store".
+"""
+
+from datetime import datetime
+
+from attrs import Factory, define
+from sdmxlib import InternationalString
+
+from statcatlib.model.agents import Agent
+from statcatlib.model.classification import Standard
+from statcatlib.model.dataset import BaseDataset
+from statcatlib.model.distribution import DataService
+from statcatlib.model.licence import LicenceDocument
+from statcatlib.model.skos import ConceptSchemeRef
+from statcatlib.model.spatial import SpatialCoverage
+from statcatlib.model.temporal import TemporalCoverage
+
+__all__ = ["CatalogDescription", "CatalogRecord"]
+
+
+@define
+class CatalogRecord:
+    """Metadata about a dataset's entry in the catalog.
+
+    Maps to dcat:CatalogRecord in DCAT-AP. This is metadata about
+    the catalog entry itself — when it was listed, where it was
+    harvested from, etc. Optional — not all catalogs use this.
+    """
+
+    identifier: str
+    primary_topic: BaseDataset
+    listing_date: datetime | None = None
+    modified: datetime | None = None
+    source_catalog: str | None = None
+    conforms_to: Standard | None = None
+
+
+@define
+class CatalogDescription:
+    """A DCAT-style description record for a catalog of datasets.
+
+    Maps to dcat:Catalog in DCAT-AP. This is metadata describing the
+    catalog as a published artefact — title, publisher, list of datasets
+    and services, licence, etc. The runtime store with the same
+    DCAT-shaped role is `statcatlib.Catalog`; the two are separate
+    concerns. Use this when serialising to / from DCAT-AP RDF or when
+    a catalog needs a self-describing manifest record.
+    """
+
+    identifier: str
+    title: InternationalString
+    description: InternationalString
+    publisher: Agent
+    datasets: list[BaseDataset] = Factory(list)
+    services: list[DataService] = Factory(list)
+    records: list[CatalogRecord] = Factory(list)
+    homepage: str | None = None
+    language: list[str] = Factory(list)
+    licence: LicenceDocument | None = None
+    spatial_coverage: list[SpatialCoverage] = Factory(list)
+    temporal_coverage: TemporalCoverage | None = None
+    theme_taxonomy: list[ConceptSchemeRef] = Factory(list)
+    issued: datetime | None = None
+    modified: datetime | None = None

statcatlib/model/classification.py

@@ -0,0 +1,19 @@
+"""Classification types for the statcatlib domain model."""
+
+from attrs import define
+from sdmxlib import InternationalString
+
+__all__ = ["Standard"]
+
+
+@define
+class Standard:
+    """A standard or specification that the dataset conforms to.
+
+    Maps to dct:conformsTo in DCAT-AP. Used to link to SDMX DSDs,
+    CSV schemas, API specifications, etc.
+    """
+
+    id: str
+    label: InternationalString | None = None
+    uri: str | None = None

statcatlib/model/dataset.py

@@ -0,0 +1,198 @@
+"""Core dataset types for the statcatlib domain model."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import ClassVar, Literal, TypeVar
+
+from attrs import Factory, define
+from sdmxlib import InternationalString
+
+from statcatlib.model.agents import Agent, ContactPoint
+from statcatlib.model.classification import Standard
+from statcatlib.model.distribution import Distribution
+from statcatlib.model.licence import LicenceDocument
+from statcatlib.model.quality import ProvenanceStatement, QualityAnnotation
+from statcatlib.model.skos import ConceptRef
+from statcatlib.model.spatial import SpatialCoverage
+from statcatlib.model.temporal import Frequency, TemporalCoverage
+
+__all__ = [
+    "DATASET_KINDS",
+    "AggregateDataset",
+    "BaseDataset",
+    "Dataset",
+    "DatasetKind",
+    "DatasetT",
+    "GeospatialDataset",
+    "MicrodataDataset",
+    "OpenDataset",
+    "PublicationDataset",
+    "StatisticalDataset",
+    "Variable",
+]
+
+
+#: The domain of the ``dataset.kind`` column. `BaseDataset` deliberately has
+#: no ``kind``: it is never instantiated, so it carries no tag — only the five
+#: concrete variants do, each annotated with its own single-member `Literal`.
+#: Annotating a variant with `DatasetKind` itself would widen it back to "any
+#: of the five" and lose the narrowing that makes the tag useful.
+type DatasetKind = Literal["open", "aggregate", "microdata", "geospatial", "publication"]
+
+
+@define
+class BaseDataset:
+    """Common fields for all dataset variants. Maps to DCAT-AP 3.0.
+
+    Not intended for direct construction — use one of the five concrete
+    variants: OpenDataset, AggregateDataset, MicrodataDataset,
+    GeospatialDataset, PublicationDataset.
+    """
+
+    # --- Identity (always required) ---
+    identifier: str
+    title: InternationalString
+    description: InternationalString
+
+    # --- Core catalog metadata ---
+    publisher: Agent | None = None
+    contact_point: ContactPoint | None = None
+    keywords: list[InternationalString] = Factory(list)
+    themes: list[ConceptRef] = Factory(list)  # dcat:theme — URI refs, not owned
+    subject: list[ConceptRef] = Factory(list)  # dcterms:subject — URI refs
+    dataset_type: ConceptRef | None = None  # dcterms:type — URI ref
+    issued: datetime | None = None
+    modified: datetime | None = None
+    frequency: Frequency | None = None
+    spatial_coverage: list[SpatialCoverage] = Factory(list)
+    temporal_coverage: TemporalCoverage | None = None
+    distributions: list[Distribution] = Factory(list)
+    licence: LicenceDocument | None = None
+    language: list[str] = Factory(list)
+
+    # --- Extended metadata ---
+    landing_page: str | None = None
+    version: str | None = None
+    version_notes: InternationalString | None = None
+    conforms_to: list[Standard] = Factory(list)
+    creator: Agent | None = None
+    access_rights: str | None = None
+    status: str | None = None
+    provenance: list[ProvenanceStatement] = Factory(list)
+    quality_annotations: list[QualityAnnotation] = Factory(list)
+
+    # --- Extensibility ---
+    #: Free-form key/value pairs, both sides strings.
+    #:
+    #: Stored in a ``MAP(VARCHAR, VARCHAR)`` column, and declared to match
+    #: rather than to flatter: as ``dict[str, object]`` it accepted values
+    #: it could not return, so `None` read back as the *string* ``"None"``
+    #: — present and truthy where an absence was stored (#114). This is
+    #: also the shape the vocabulary uses; CKAN's package extras are
+    #: string-valued key/value pairs.
+    #:
+    #: Structured values are the caller's to encode — ``json.dumps`` on
+    #: the way in, ``json.loads`` on the way out — so that the lossy step
+    #: is one they chose and can reverse.
+    extras: dict[str, str] = Factory(dict)
+
+
+@define
+class OpenDataset(BaseDataset):
+    """Generic open dataset — the default bucket for non-specialized data."""
+
+    kind: ClassVar[Literal["open"]] = "open"
+
+
+@define
+class AggregateDataset(BaseDataset):
+    """A statistical aggregate dataset (tabular, with dimensions and measures).
+
+    Extends BaseDataset with StatDCAT-AP discovery fields and an optional link
+    to the full SDMX structural record in sdmxlib's shared DuckDB tables.
+    dimension_names carries enough info for catalog search without a JOIN.
+    """
+
+    kind: ClassVar[Literal["aggregate"]] = "aggregate"
+    dimension_names: list[InternationalString] = Factory(list)
+    num_series: int | None = None
+    # SDMX link: FK into sdmxlib's dataflows.urn in the shared DuckDB
+    sdmx_dataflow_urn: str | None = None
+    # Pivot axis dimension for ingest_data(). None = auto-detect from DSD.
+    measure_dim: str | None = None
+
+
+@define
+class Variable:
+    """A variable in a microdata dataset."""
+
+    id: str
+    label: InternationalString
+    description: InternationalString | None = None
+    value_type: str | None = None
+    classification_id: str | None = None
+    universe: str | None = None
+    question_text: str | None = None
+    valid_range: str | None = None
+    derivation: str | None = None
+
+
+@define
+class MicrodataDataset(BaseDataset):
+    """A microdata (survey/record-level) dataset."""
+
+    kind: ClassVar[Literal["microdata"]] = "microdata"
+    variables: list[Variable] = Factory(list)
+    sample_size: int | None = None
+    universe: str | None = None
+    collection_method: str | None = None
+    anonymization: str | None = None
+    survey_id: str | None = None
+
+
+@define
+class GeospatialDataset(BaseDataset):
+    """A geospatial dataset."""
+
+    kind: ClassVar[Literal["geospatial"]] = "geospatial"
+    crs: str | None = None
+    spatial_resolution: str | None = None
+    feature_types: list[str] = Factory(list)
+    scale: str | None = None
+
+
+@define
+class PublicationDataset(BaseDataset):
+    """A publication (article, report, monograph)."""
+
+    kind: ClassVar[Literal["publication"]] = "publication"
+    authors: list[str] = Factory(list)
+    doi: str | None = None
+    isbn: str | None = None
+    publication_type: str | None = None
+    pages: int | None = None
+    series_title: InternationalString | None = None
+
+
+#: Every tag, in declaration order. Built from the classes rather than from
+#: `DatasetKind`, so each variant's `Literal` is checked to be a member of the
+#: alias — a typo'd tag is a type error here, not a silent mismatch with the
+#: ``dataset.kind`` column.
+DATASET_KINDS: tuple[DatasetKind, ...] = (
+    OpenDataset.kind,
+    AggregateDataset.kind,
+    MicrodataDataset.kind,
+    GeospatialDataset.kind,
+    PublicationDataset.kind,
+)
+
+# Public union type alias — the type you work with day-to-day.
+# Use BaseDataset as the generic bound in query() and other generic contexts.
+type Dataset = OpenDataset | AggregateDataset | MicrodataDataset | GeospatialDataset | PublicationDataset
+
+# TypeVar for pre-PEP-695 generic code
+DatasetT = TypeVar("DatasetT", bound=BaseDataset)
+
+# Backward compatibility alias — AggregateDataset is the successor
+StatisticalDataset = AggregateDataset

statcatlib/model/distribution.py

@@ -0,0 +1,50 @@
+"""Distribution and DataService types for the statcatlib domain model."""
+
+from datetime import datetime
+
+from attrs import Factory, define
+from sdmxlib import InternationalString
+
+from statcatlib.model.classification import Standard
+from statcatlib.model.licence import LicenceDocument
+
+__all__ = ["DataService", "Distribution"]
+
+
+@define
+class DataService:
+    """An API endpoint that serves one or more datasets.
+
+    Maps to dcat:DataService in DCAT-AP. For SDMX sources, the
+    endpoint_url is the SDMX REST base URL.
+    """
+
+    id: str
+    title: InternationalString
+    endpoint_url: str
+    description: InternationalString | None = None
+    serves_datasets: list[str] = Factory(list)
+    conforms_to: list[Standard] = Factory(list)
+
+
+@define
+class Distribution:
+    """A specific available form of a dataset.
+
+    Maps to dcat:Distribution in DCAT-AP. A dataset may have
+    multiple distributions: CSV download, API endpoint, visualization, etc.
+    """
+
+    id: str
+    title: InternationalString | None = None
+    description: InternationalString | None = None
+    access_url: str | None = None
+    download_url: str | None = None
+    media_type: str | None = None
+    format: str | None = None
+    byte_size: int | None = None
+    issued: datetime | None = None
+    modified: datetime | None = None
+    licence: LicenceDocument | None = None
+    access_service: DataService | None = None
+    distribution_type: str | None = None

statcatlib/model/licence.py

@@ -0,0 +1,18 @@
+"""Licence types for the statcatlib domain model."""
+
+from attrs import define
+from sdmxlib import InternationalString
+
+__all__ = ["LicenceDocument"]
+
+
+@define
+class LicenceDocument:
+    """A licence under which the dataset is made available.
+
+    Maps to dct:license → dct:LicenseDocument in DCAT-AP.
+    """
+
+    id: str
+    label: InternationalString | None = None
+    uri: str | None = None

statcatlib/model/quality.py

@@ -0,0 +1,31 @@
+"""Quality and provenance types for the statcatlib domain model."""
+
+from attrs import define
+from sdmxlib import InternationalString
+
+__all__ = ["ProvenanceStatement", "QualityAnnotation"]
+
+
+@define
+class QualityAnnotation:
+    """A quality statement about the dataset.
+
+    Maps to dqv:hasQualityAnnotation in StatDCAT-AP. Intentionally
+    simple — a label + optional body. Richer quality modelling
+    can be added later.
+    """
+
+    dimension: str | None = None
+    label: InternationalString | None = None
+    body: InternationalString | None = None
+
+
+@define
+class ProvenanceStatement:
+    """A statement about the origin/methodology of the dataset.
+
+    Maps to dct:provenance in DCAT-AP.
+    """
+
+    label: InternationalString
+    uri: str | None = None