pinaxlib 5.7.0__py3-none-any.whl

pinax/__init__.py

@@ -0,0 +1,135 @@
+"""pinax — a queryable open data catalog engine."""
+
+from pinax.model import (
+    Agent,
+    AggregateDataset,
+    BaseDataset,
+    Catalog,
+    CatalogRecord,
+    Concept,
+    ConceptRef,
+    ConceptScheme,
+    ConceptSchemeRef,
+    ContactPoint,
+    Dataset,
+    DataService,
+    DatasetT,
+    Distribution,
+    Frequency,
+    GeospatialDataset,
+    InternationalString,
+    LicenceDocument,
+    MicrodataDataset,
+    OpenDataset,
+    ProvenanceStatement,
+    PublicationDataset,
+    QualityAnnotation,
+    SpatialCoverage,
+    Standard,
+    StatisticalDataset,
+    TemporalCoverage,
+    Variable,
+    is_concept_in_scheme,
+)
+from pinax.sources import (
+    dataset_from_dataflow,
+    ingest_ckan,
+    ingest_data,
+    ingest_sdmx,
+    ingest_sdmx_registry,
+    prepare_sdmx,
+)
+from pinax import query, urn
+from pinax.expr import (
+    AliasedExpr,
+    CountExpr,
+    ExistsExpr,
+    Expr,
+    TraversalExpr,
+    each,
+)
+from pinax.store import (
+    CatalogStore,
+    CodeScope,
+    CodeSearchResult,
+    CodelistScope,
+    CodelistsScope,
+    ConceptScope,
+    ConceptSchemeScope,
+    ConceptSchemesScope,
+    DatasetScope,
+    DimensionInfo,
+    DimensionScope,
+    DimensionsScope,
+    QueryBuilder,
+    Row,
+    SchemeScope,
+    SearchResult,
+    ThemesScope,
+)
+from pinax.store.sentinel import NotLoadedError, is_unloaded
+
+__all__ = [
+    "Agent",
+    "AggregateDataset",
+    "AliasedExpr",
+    "BaseDataset",
+    "Catalog",
+    "CatalogRecord",
+    "CatalogStore",
+    "CodeScope",
+    "CodeSearchResult",
+    "CodelistScope",
+    "CodelistsScope",
+    "Concept",
+    "ConceptRef",
+    "ConceptScheme",
+    "ConceptSchemeRef",
+    "ConceptSchemeScope",
+    "ConceptSchemesScope",
+    "ConceptScope",
+    "ContactPoint",
+    "CountExpr",
+    "DataService",
+    "Dataset",
+    "DatasetScope",
+    "DatasetT",
+    "DimensionInfo",
+    "DimensionScope",
+    "DimensionsScope",
+    "Distribution",
+    "ExistsExpr",
+    "Expr",
+    "Frequency",
+    "GeospatialDataset",
+    "InternationalString",
+    "LicenceDocument",
+    "MicrodataDataset",
+    "NotLoadedError",
+    "OpenDataset",
+    "ProvenanceStatement",
+    "PublicationDataset",
+    "QualityAnnotation",
+    "QueryBuilder",
+    "Row",
+    "SchemeScope",
+    "SearchResult",
+    "SpatialCoverage",
+    "Standard",
+    "StatisticalDataset",
+    "TemporalCoverage",
+    "ThemesScope",
+    "TraversalExpr",
+    "Variable",
+    "dataset_from_dataflow",
+    "each",
+    "ingest_ckan",
+    "ingest_data",
+    "ingest_sdmx",
+    "ingest_sdmx_registry",
+    "is_concept_in_scheme",
+    "is_unloaded",
+    "prepare_sdmx",
+    "query",
+    "urn",
+]

pinax/expr.py

@@ -0,0 +1,167 @@
+"""Sub-traversal expression language for scope-based graph queries.
+
+Expressions describe computations over related graph data without executing
+any SQL.  They are context-free objects — like ``pl.col()`` in Polars — and
+can be reused across ``.enrich()``, ``.filter()``, and ``.sort_by()`` on any
+collection scope.
+
+Usage::
+
+    import pinax as pk
+
+    n_datasets = pk.each("datasets").count()
+    has_data = pk.each("datasets").exists()
+
+    store.themes["statcan"].enrich(n=n_datasets).filter(has_data).collect()
+"""
+
+from __future__ import annotations
+
+from attrs import define
+
+# ---------------------------------------------------------------------------
+# Base expression types
+# ---------------------------------------------------------------------------
+
+
+@define(frozen=True)
+class Expr:
+    """Base class for all expression nodes."""
+
+    def alias(self, name: str) -> AliasedExpr:
+        """Name this expression result (like Polars ``.alias()``)."""
+        return AliasedExpr(expr=self, name=name)
+
+    def count(self) -> CountExpr:
+        """Count items at the far end of the traversal."""
+        return CountExpr(source=self)
+
+    def exists(self) -> ExistsExpr:
+        """Check if any items exist at the far end."""
+        return ExistsExpr(source=self)
+
+    def sum(self, field: str) -> SumExpr:
+        """Sum a numeric field at the far end."""
+        return SumExpr(source=self, field=field)
+
+    def ids(self) -> IdsExpr:
+        """Collect identifiers at the far end."""
+        return IdsExpr(source=self)
+
+    def distinct(self, field: str) -> DistinctExpr:
+        """Select distinct values of a field at the far end."""
+        return DistinctExpr(source=self, field=field)
+
+    def filter(self, **kwargs: object) -> FilteredExpr:
+        """Filter far-side nodes by attribute values."""
+        return FilteredExpr(source=self, conditions=kwargs)
+
+    def count_distinct(self) -> CountDistinctExpr:
+        """Count distinct items at the far end."""
+        return CountDistinctExpr(source=self)
+
+
+@define(frozen=True)
+class TraversalExpr(Expr):
+    """A traversal along one or more edges from the current scope position.
+
+    Created by ``pk.each("edge1", "edge2", ...)``.  Strings are always
+    edge labels (table names or relationship names in the graph).
+    """
+
+    edges: tuple[str, ...]
+
+
+@define(frozen=True)
+class AliasedExpr:
+    """An expression with a named output column.
+
+    Created by ``expr.alias("name")`` or via ``**kwargs`` in ``.enrich()``.
+    """
+
+    expr: Expr
+    name: str
+
+
+# ---------------------------------------------------------------------------
+# Terminal reduction expressions
+# ---------------------------------------------------------------------------
+
+
+@define(frozen=True)
+class CountExpr(Expr):
+    """COUNT(*) at the far end of a traversal."""
+
+    source: Expr
+
+
+@define(frozen=True)
+class ExistsExpr(Expr):
+    """EXISTS(...) — boolean, true if any far-side items exist."""
+
+    source: Expr
+
+
+@define(frozen=True)
+class SumExpr(Expr):
+    """SUM(field) at the far end of a traversal."""
+
+    source: Expr
+    field: str
+
+
+@define(frozen=True)
+class IdsExpr(Expr):
+    """LIST(id) — collect identifiers at the far end."""
+
+    source: Expr
+
+
+@define(frozen=True)
+class DistinctExpr(Expr):
+    """SELECT DISTINCT field — narrows to unique values."""
+
+    source: Expr
+    field: str
+
+
+@define(frozen=True)
+class CountDistinctExpr(Expr):
+    """COUNT(DISTINCT ...) at the far end."""
+
+    source: Expr
+
+
+@define(frozen=True)
+class FilteredExpr(Expr):
+    """A traversal with far-side filtering applied.
+
+    Created by ``pk.each("datasets").filter(status="current")``.
+    """
+
+    source: Expr
+    conditions: dict[str, object]
+
+
+# ---------------------------------------------------------------------------
+# Public entry point
+# ---------------------------------------------------------------------------
+
+
+def each(*edges: str) -> TraversalExpr:
+    """Create a sub-traversal expression starting from the current scope.
+
+    Each argument is an edge label (relationship name).  Multiple arguments
+    describe a multi-hop path::
+
+        pk.each("datasets")  # one hop
+        pk.each("datasets", "dimensions")  # two hops
+        pk.each("datasets", "dimensions", "codelist")  # three hops
+
+    The returned expression is context-free and can be used in ``.enrich()``,
+    ``.filter()``, and ``.sort_by()`` on any collection scope.
+    """
+    if not edges:
+        msg = "each() requires at least one edge label"
+        raise ValueError(msg)
+    return TraversalExpr(edges=edges)

pinax/model/__init__.py

@@ -0,0 +1,56 @@
+"""Domain model for pinax."""
+
+from pinax.model.agents import Agent, ContactPoint
+from pinax.model.catalog import Catalog, CatalogRecord
+from pinax.model.classification import Standard
+from pinax.model.dataset import (
+    AggregateDataset,
+    BaseDataset,
+    Dataset,
+    DatasetT,
+    GeospatialDataset,
+    MicrodataDataset,
+    OpenDataset,
+    PublicationDataset,
+    StatisticalDataset,
+    Variable,
+)
+from pinax.model.distribution import DataService, Distribution
+from pinax.model.licence import LicenceDocument
+from pinax.model.quality import ProvenanceStatement, QualityAnnotation
+from pinax.model.spatial import SpatialCoverage
+from pinax.model.temporal import Frequency, TemporalCoverage
+from pinax.model.skos import Concept, ConceptRef, ConceptScheme, ConceptSchemeRef, is_concept_in_scheme
+from pinax.model.types import InternationalString
+
+__all__ = [
+    "Agent",
+    "AggregateDataset",
+    "BaseDataset",
+    "Catalog",
+    "CatalogRecord",
+    "Concept",
+    "ConceptRef",
+    "ConceptScheme",
+    "ConceptSchemeRef",
+    "ContactPoint",
+    "DataService",
+    "Dataset",
+    "DatasetT",
+    "Distribution",
+    "Frequency",
+    "GeospatialDataset",
+    "InternationalString",
+    "LicenceDocument",
+    "MicrodataDataset",
+    "OpenDataset",
+    "ProvenanceStatement",
+    "PublicationDataset",
+    "QualityAnnotation",
+    "SpatialCoverage",
+    "Standard",
+    "StatisticalDataset",
+    "TemporalCoverage",
+    "Variable",
+    "is_concept_in_scheme",
+]

pinax/model/agents.py

@@ -0,0 +1,32 @@
+"""Agent and contact types for the pinax domain model."""
+
+from attrs import define
+from sdmxlib import InternationalString
+
+__all__ = ["Agent", "ContactPoint"]
+
+
+@define
+class Agent:
+    """An organization or person responsible for a dataset.
+
+    Maps to foaf:Agent / dct:publisher / dct:creator in DCAT-AP.
+    """
+
+    name: InternationalString
+    id: str | None = None
+    type: str | None = None
+    homepage: str | None = None
+
+
+@define
+class ContactPoint:
+    """Contact information for a dataset.
+
+    Maps to dcat:contactPoint → vcard:Kind in DCAT-AP.
+    """
+
+    name: InternationalString | None = None
+    email: str | None = None
+    phone: str | None = None
+    url: str | None = None

pinax/model/catalog.py

@@ -0,0 +1,58 @@
+"""Catalog and CatalogRecord types for the pinax domain model."""
+
+from datetime import datetime
+
+from attrs import Factory, define
+from sdmxlib import InternationalString
+
+from pinax.model.agents import Agent
+from pinax.model.classification import Standard
+from pinax.model.dataset import BaseDataset
+from pinax.model.distribution import DataService
+from pinax.model.licence import LicenceDocument
+from pinax.model.skos import ConceptSchemeRef
+from pinax.model.spatial import SpatialCoverage
+from pinax.model.temporal import TemporalCoverage
+
+__all__ = ["Catalog", "CatalogRecord"]
+
+
+@define
+class CatalogRecord:
+    """Metadata about a dataset's entry in the catalog.
+
+    Maps to dcat:CatalogRecord in DCAT-AP. This is metadata about
+    the catalog entry itself — when it was listed, where it was
+    harvested from, etc. Optional — not all catalogs use this.
+    """
+
+    identifier: str
+    primary_topic: BaseDataset
+    listing_date: datetime | None = None
+    modified: datetime | None = None
+    source_catalog: str | None = None
+    conforms_to: Standard | None = None
+
+
+@define
+class Catalog:
+    """A catalog of datasets.
+
+    Maps to dcat:Catalog in DCAT-AP. The top-level container.
+    """
+
+    identifier: str
+    title: InternationalString
+    description: InternationalString
+    publisher: Agent
+    datasets: list[BaseDataset] = Factory(list)
+    services: list[DataService] = Factory(list)
+    records: list[CatalogRecord] = Factory(list)
+    homepage: str | None = None
+    language: list[str] = Factory(list)
+    licence: LicenceDocument | None = None
+    spatial_coverage: list[SpatialCoverage] = Factory(list)
+    temporal_coverage: TemporalCoverage | None = None
+    theme_taxonomy: list[ConceptSchemeRef] = Factory(list)
+    issued: datetime | None = None
+    modified: datetime | None = None

pinax/model/classification.py

@@ -0,0 +1,19 @@
+"""Classification types for the pinax domain model."""
+
+from attrs import define
+from sdmxlib import InternationalString
+
+__all__ = ["Standard"]
+
+
+@define
+class Standard:
+    """A standard or specification that the dataset conforms to.
+
+    Maps to dct:conformsTo in DCAT-AP. Used to link to SDMX DSDs,
+    CSV schemas, API specifications, etc.
+    """
+
+    id: str
+    label: InternationalString | None = None
+    uri: str | None = None

pinax/model/dataset.py

@@ -0,0 +1,165 @@
+"""Core dataset types for the pinax domain model."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Literal, TypeVar
+
+from attrs import Factory, define
+from sdmxlib import InternationalString
+
+from pinax.model.agents import Agent, ContactPoint
+from pinax.model.classification import Standard
+from pinax.model.distribution import Distribution
+from pinax.model.licence import LicenceDocument
+from pinax.model.quality import ProvenanceStatement, QualityAnnotation
+from pinax.model.skos import ConceptRef
+from pinax.model.spatial import SpatialCoverage
+from pinax.model.temporal import Frequency, TemporalCoverage
+
+__all__ = [
+    "AggregateDataset",
+    "BaseDataset",
+    "Dataset",
+    "DatasetT",
+    "GeospatialDataset",
+    "MicrodataDataset",
+    "OpenDataset",
+    "PublicationDataset",
+    "StatisticalDataset",
+    "Variable",
+]
+
+
+@define
+class BaseDataset:
+    """Common fields for all dataset variants. Maps to DCAT-AP 3.0.
+
+    Not intended for direct construction — use one of the five concrete
+    variants: OpenDataset, AggregateDataset, MicrodataDataset,
+    GeospatialDataset, PublicationDataset.
+    """
+
+    # --- Identity (always required) ---
+    identifier: str
+    title: InternationalString
+    description: InternationalString
+    kind: str  # discriminator — each variant narrows to Literal and provides a default
+
+    # --- Core catalog metadata ---
+    publisher: Agent | None = None
+    contact_point: ContactPoint | None = None
+    keywords: list[InternationalString] = Factory(list)
+    themes: list[ConceptRef] = Factory(list)  # dcat:theme — URI refs, not owned
+    subject: list[ConceptRef] = Factory(list)  # dcterms:subject — URI refs
+    dataset_type: ConceptRef | None = None  # dcterms:type — URI ref
+    issued: datetime | None = None
+    modified: datetime | None = None
+    frequency: Frequency | None = None
+    spatial_coverage: list[SpatialCoverage] = Factory(list)
+    temporal_coverage: TemporalCoverage | None = None
+    distributions: list[Distribution] = Factory(list)
+    licence: LicenceDocument | None = None
+    language: list[str] = Factory(list)
+
+    # --- Extended metadata ---
+    landing_page: str | None = None
+    version: str | None = None
+    version_notes: InternationalString | None = None
+    conforms_to: list[Standard] = Factory(list)
+    creator: Agent | None = None
+    access_rights: str | None = None
+    status: str | None = None
+    provenance: list[ProvenanceStatement] = Factory(list)
+    quality_annotations: list[QualityAnnotation] = Factory(list)
+
+    # --- Extensibility ---
+    extras: dict[str, Any] = Factory(dict)
+
+
+@define
+class OpenDataset(BaseDataset):
+    """Generic open dataset — the default bucket for non-specialized data."""
+
+    kind: Literal["open"] = "open"  # pyright: ignore[reportIncompatibleVariableOverride]
+
+
+@define
+class AggregateDataset(BaseDataset):
+    """A statistical aggregate dataset (tabular, with dimensions and measures).
+
+    Extends BaseDataset with StatDCAT-AP discovery fields and an optional link
+    to the full SDMX structural record in sdmxlib's shared DuckDB tables.
+    dimension_names carries enough info for catalog search without a JOIN.
+    """
+
+    kind: Literal["aggregate"] = "aggregate"  # pyright: ignore[reportIncompatibleVariableOverride]
+    dimension_names: list[InternationalString] = Factory(list)
+    num_series: int | None = None
+    # SDMX link: FK into sdmxlib's dataflows.urn in the shared DuckDB
+    sdmx_dataflow_urn: str | None = None
+    # Pivot axis dimension for ingest_data(). None = auto-detect from DSD.
+    measure_dim: str | None = None
+
+
+@define
+class Variable:
+    """A variable in a microdata dataset."""
+
+    id: str
+    label: InternationalString
+    description: InternationalString | None = None
+    value_type: str | None = None
+    classification_id: str | None = None
+    universe: str | None = None
+    question_text: str | None = None
+    valid_range: str | None = None
+    derivation: str | None = None
+
+
+@define
+class MicrodataDataset(BaseDataset):
+    """A microdata (survey/record-level) dataset."""
+
+    kind: Literal["microdata"] = "microdata"  # pyright: ignore[reportIncompatibleVariableOverride]
+    variables: list[Variable] = Factory(list)
+    sample_size: int | None = None
+    universe: str | None = None
+    collection_method: str | None = None
+    anonymization: str | None = None
+    survey_id: str | None = None
+
+
+@define
+class GeospatialDataset(BaseDataset):
+    """A geospatial dataset."""
+
+    kind: Literal["geospatial"] = "geospatial"  # pyright: ignore[reportIncompatibleVariableOverride]
+    crs: str | None = None
+    spatial_resolution: str | None = None
+    feature_types: list[str] = Factory(list)
+    scale: str | None = None
+
+
+@define
+class PublicationDataset(BaseDataset):
+    """A publication (article, report, monograph)."""
+
+    kind: Literal["publication"] = "publication"  # pyright: ignore[reportIncompatibleVariableOverride]
+    authors: list[str] = Factory(list)
+    doi: str | None = None
+    isbn: str | None = None
+    publication_type: str | None = None
+    pages: int | None = None
+    series_title: InternationalString | None = None
+
+
+# Public union type alias — the type you work with day-to-day.
+# Use BaseDataset as the generic bound in query() and other generic contexts.
+type Dataset = OpenDataset | AggregateDataset | MicrodataDataset | GeospatialDataset | PublicationDataset
+
+# TypeVar for pre-PEP-695 generic code
+DatasetT = TypeVar("DatasetT", bound=BaseDataset)
+
+# Backward compatibility alias — AggregateDataset is the successor
+StatisticalDataset = AggregateDataset

pinax/model/distribution.py

@@ -0,0 +1,50 @@
+"""Distribution and DataService types for the pinax domain model."""
+
+from datetime import datetime
+
+from attrs import Factory, define
+from sdmxlib import InternationalString
+
+from pinax.model.classification import Standard
+from pinax.model.licence import LicenceDocument
+
+__all__ = ["DataService", "Distribution"]
+
+
+@define
+class DataService:
+    """An API endpoint that serves one or more datasets.
+
+    Maps to dcat:DataService in DCAT-AP. For SDMX sources, the
+    endpoint_url is the SDMX REST base URL.
+    """
+
+    id: str
+    title: InternationalString
+    endpoint_url: str
+    description: InternationalString | None = None
+    serves_datasets: list[str] = Factory(list)
+    conforms_to: list[Standard] = Factory(list)
+
+
+@define
+class Distribution:
+    """A specific available form of a dataset.
+
+    Maps to dcat:Distribution in DCAT-AP. A dataset may have
+    multiple distributions: CSV download, API endpoint, visualization, etc.
+    """
+
+    id: str
+    title: InternationalString | None = None
+    description: InternationalString | None = None
+    access_url: str | None = None
+    download_url: str | None = None
+    media_type: str | None = None
+    format: str | None = None
+    byte_size: int | None = None
+    issued: datetime | None = None
+    modified: datetime | None = None
+    licence: LicenceDocument | None = None
+    access_service: DataService | None = None
+    distribution_type: str | None = None

pinax/model/licence.py

@@ -0,0 +1,18 @@
+"""Licence types for the pinax domain model."""
+
+from attrs import define
+from sdmxlib import InternationalString
+
+__all__ = ["LicenceDocument"]
+
+
+@define
+class LicenceDocument:
+    """A licence under which the dataset is made available.
+
+    Maps to dct:license → dct:LicenseDocument in DCAT-AP.
+    """
+
+    id: str
+    label: InternationalString | None = None
+    uri: str | None = None