sdmxlib 0.8.0__py3-none-any.whl

sdmxlib/local/__init__.py

@@ -0,0 +1,6 @@
+"""Local persistent SDMX artefact store backed by DuckDB."""
+
+from sdmxlib.local._registry import LocalRegistry
+from sdmxlib.storage.lazy import LazyCodelist, LazyItemList
+
+__all__ = ["LazyCodelist", "LazyItemList", "LocalRegistry"]

sdmxlib/local/_registry.py

@@ -0,0 +1,368 @@
+"""LocalRegistry — DuckDB-backed persistent SDMX artefact store.
+
+Provides the same conceptual interface as the in-memory ``Registry``
+with normalized tables for codelists, codes, hierarchies, DSDs,
+dataflows, and concept schemes.
+
+Example::
+
+    import sdmxlib as sl
+
+    # Populate from a remote registry
+    with sl.RestRegistry("https://sdmx.oecd.org/public/rest/") as remote:
+        dsd = remote.DataStructure(agency="OECD", id="DSD_BLI").resolve()
+        local = sl.LocalRegistry("oecd.duckdb")
+        local.ingest(remote.registry)
+        local.close()
+
+    # Use later without network
+    with sl.LocalRegistry("oecd.duckdb") as local:
+        dsd = local.get(dsd_urn)
+        cm = local.code_map(codelist_urn, "en")
+
+    # Shared connection (e.g. with cataloglib)
+    import duckdb
+
+    conn = duckdb.connect("shared.duckdb")
+    local = sl.LocalRegistry(conn, languages=["en", "fr"])
+"""
+
+import json
+from collections.abc import Iterable, Sequence
+from pathlib import Path
+from typing import Any, Self
+
+import attrs
+import duckdb
+
+from sdmxlib.model.collections import ItemList
+from sdmxlib.model.convert import from_dict
+from sdmxlib.model.ref import Ref
+from sdmxlib.model.registry import Registry
+from sdmxlib.model.urn import SdmxUrn
+from sdmxlib.storage import readers, writers
+from sdmxlib.storage.lazy import LazyCodelist
+from sdmxlib.storage.readers import _TYPE_REGISTRY
+from sdmxlib.storage.schema import create_tables, tables_exist
+
+# Map item-level artefact class → (parent scheme type name, items attribute name)
+_ITEM_TO_SCHEME: dict[str, tuple[str, str]] = {
+    "Concept": ("ConceptScheme", "concepts"),
+    "Code": ("Codelist", "codes"),
+}
+
+
+class LocalRegistry:
+    """DuckDB-backed persistent SDMX artefact store.
+
+    Provides the same interface as the in-memory ``Registry``, plus
+    mutations (``drop``, ``replace``) and direct SQL helpers
+    (``code_map``, ``dsd_code_map``) that avoid creating Python objects.
+
+    Use as a context manager::
+
+        with LocalRegistry("store.duckdb") as local:
+            local.add(codelist)
+            cl = local.get(codelist_urn)  # LazyCodelist
+
+    Accepts a DuckDB connection for shared-database workflows::
+
+        conn = duckdb.connect("shared.duckdb")
+        local = LocalRegistry(conn, languages=["en", "fr"])
+    """
+
+    def __init__(
+        self,
+        path_or_connection: str | Path | duckdb.DuckDBPyConnection,
+        *,
+        languages: Sequence[str] | None = None,
+    ) -> None:
+        if isinstance(path_or_connection, duckdb.DuckDBPyConnection):
+            self._conn = path_or_connection
+            self._owns_connection = False
+        else:
+            self._conn = duckdb.connect(str(path_or_connection))
+            self._owns_connection = True
+
+        if not tables_exist(self._conn):
+            create_tables(self._conn, languages=languages or ("en",))
+        elif languages is not None:
+            # Re-create with new languages (idempotent if same)
+            create_tables(self._conn, languages=languages)
+
+    @property
+    def connection(self) -> duckdb.DuckDBPyConnection:
+        """The underlying DuckDB connection (for sharing)."""
+        return self._conn
+
+    def __enter__(self) -> Self:
+        return self
+
+    def __exit__(self, *args: object) -> None:
+        self.close()
+
+    def close(self) -> None:
+        """Close the database connection (only if we own it)."""
+        if self._owns_connection:
+            self._conn.close()
+
+    # ── Add ───────────────────────────────────────────────────────────────
+
+    def add(self, artefact: Any) -> None:
+        """Add or replace an artefact in the store.
+
+        Dispatches to the appropriate writer based on artefact type.
+        Codelists, DataStructures, Dataflows, ConceptSchemes, and
+        Hierarchies are written to normalized tables. Others are stored
+        as JSON blobs.
+        """
+        writers.put_artefact(self._conn, artefact)
+
+    def add_all(self, artefacts: Iterable[Any]) -> None:
+        """Add multiple artefacts."""
+        for artefact in artefacts:
+            self.add(artefact)
+
+    # ── Get ───────────────────────────────────────────────────────────────
+
+    def get[T](self, urn: "SdmxUrn[T]", *, eager: bool = False) -> "T | None":
+        """Look up an artefact by URN.
+
+        For Codelists, returns a ``LazyCodelist`` by default (codes not
+        materialized). Pass ``eager=True`` to get a real ``Codelist``.
+
+        All ``Ref`` fields on the returned artefact are resolved from
+        DuckDB so that standard traversal syntax works::
+
+            dsd = local.get(dsd_urn)
+            dsd.dimensions["FREQ"].codelist()  # → LazyCodelist
+            dsd.code_map("en")  # works
+        """
+        result = readers.get_artefact(self._conn, str(urn), eager=eager)
+        if result is None:
+            return None
+        if not isinstance(result, LazyCodelist):
+            self._resolve_refs(result, set())
+        return result  # type: ignore[return-value]
+
+    def get_all[T](self, artefact_type: "type[T]") -> "list[T]":
+        """All artefacts of the given Python type."""
+        results = readers.get_all(self._conn, artefact_type)
+        for r in results:
+            if not isinstance(r, LazyCodelist):
+                self._resolve_refs(r, set())
+        return results  # type: ignore[return-value]
+
+    # ── Ref resolution ─────────────────────────────────────────────────
+
+    def _resolve_refs(self, obj: Any, visited: set[int]) -> None:
+        """Walk attrs fields, resolving ``Ref`` targets from DuckDB.
+
+        Codelist Refs resolve to ``LazyCodelist``. Other artefacts are
+        reconstructed via ``from_dict``. Missing targets stay unresolved.
+        """
+        if not attrs.has(type(obj)) or isinstance(obj, type):
+            return
+        oid = id(obj)
+        if oid in visited:
+            return
+        visited.add(oid)
+
+        for f in attrs.fields(type(obj)):
+            value = getattr(obj, f.name, None)
+            if value is None:
+                continue
+            if isinstance(value, Ref):
+                self._resolve_ref_field(obj, f.name, value)
+            elif isinstance(value, ItemList):
+                for item in value:
+                    self._resolve_refs(item, visited)
+            elif attrs.has(type(value)) and not isinstance(value, type):
+                self._resolve_refs(value, visited)
+
+    def _resolve_ref_field(self, obj: Any, name: str, ref: "Ref[Any]") -> None:
+        """Resolve a single Ref field from DuckDB and set it on *obj*."""
+        if ref._resolved is not None:  # noqa: SLF001
+            return
+
+        urn_str = str(ref.urn)
+        target = self._resolve_target(urn_str)
+        if target is not None:
+            resolved = Ref.of(ref.urn, target)
+            setattr(obj, name, resolved)
+
+    def _resolve_target(self, urn_str: str) -> Any | None:
+        """Load an artefact by URN string for Ref resolution.
+
+        Returns LazyCodelist for codelists, reconstructed object for others,
+        or None if the URN is not in the store.
+
+        Handles item-level URNs (e.g. Concept within a ConceptScheme) by
+        loading the parent scheme and extracting the item.
+        """
+        parsed = SdmxUrn.parse(urn_str)
+        if parsed.item_id is not None:
+            return self._resolve_item_urn(parsed)
+
+        row = self._conn.execute(
+            """
+            SELECT artefact_type, _document
+            FROM artefacts
+            WHERE urn = $urn
+            """,
+            {"urn": urn_str},
+        ).fetchone()
+        if row is None:
+            return None
+
+        artefact_type, doc_str = row
+        data = json.loads(doc_str)
+
+        if artefact_type == "Codelist":
+            return readers.make_lazy_codelist(self._conn, urn_str, data)
+
+        cls = _TYPE_REGISTRY.get(artefact_type)
+        if cls is None:
+            return None
+
+        if artefact_type == "Hierarchy":
+            data["tree"] = readers.load_hierarchy_tree(self._conn, urn_str)
+
+        return from_dict(cls, data)
+
+    def _resolve_item_urn(self, parsed: "SdmxUrn[Any]") -> Any | None:
+        """Resolve an item-level URN by loading its parent scheme."""
+        scheme_info = _ITEM_TO_SCHEME.get(parsed.artefact_class)
+        if scheme_info is None:
+            return None
+
+        scheme_class_name, items_attr = scheme_info
+        scheme_cls = _TYPE_REGISTRY.get(scheme_class_name)
+        if scheme_cls is None:
+            return None
+
+        parent_urn = SdmxUrn.make(
+            scheme_cls,
+            package=parsed.package,
+            artefact_class=scheme_class_name,
+            agency=parsed.agency,
+            id=parsed.id,
+            version=parsed.version,
+        )
+        parent_urn_str = str(parent_urn)
+
+        row = self._conn.execute(
+            """
+            SELECT urn, _document
+            FROM artefacts
+            WHERE urn = $urn AND artefact_type = $type
+            """,
+            {"urn": parent_urn_str, "type": scheme_class_name},
+        ).fetchone()
+
+        if row is None and parsed.version == "latest":
+            row = self._conn.execute(
+                """
+                SELECT urn, _document
+                FROM artefacts
+                WHERE artefact_type = $type AND agency = $agency AND id = $id
+                """,
+                {"type": scheme_class_name, "agency": parsed.agency, "id": parsed.id},
+            ).fetchone()
+
+        if row is None:
+            return None
+
+        parent_urn_str, doc_str = row
+        data = json.loads(doc_str)
+
+        if scheme_class_name == "Codelist":
+            data["codes"] = readers.load_code_dicts(self._conn, parent_urn_str)
+
+        parent = from_dict(scheme_cls, data)
+        items = getattr(parent, items_attr, None)
+        if items is None:
+            return None
+
+        return items.get(parsed.item_id) if hasattr(items, "get") else None
+
+    # ── Ref ────────────────────────────────────────────────────────────────
+
+    def ref[T](self, urn: "SdmxUrn[T]") -> "Ref[T]":
+        """Return an unresolved Ref for this URN."""
+        return Ref[T].unresolved(urn)
+
+    # ── Contains / len ────────────────────────────────────────────────────
+
+    def __contains__(self, urn: object) -> bool:
+        if isinstance(urn, SdmxUrn):
+            row = self._conn.execute(
+                """
+                SELECT 1 FROM artefacts WHERE urn = $urn
+                """,
+                {"urn": str(urn)},
+            ).fetchone()
+            return row is not None
+        return False
+
+    def __len__(self) -> int:
+        row = self._conn.execute("""
+            SELECT COUNT(*) FROM artefacts
+        """).fetchone()
+        return row[0]  # type: ignore[index]
+
+    def __repr__(self) -> str:
+        rows = self._conn.execute("""
+            SELECT artefact_type, COUNT(*)
+            FROM artefacts
+            GROUP BY artefact_type
+        """).fetchall()
+        counts = ", ".join(f"{name}={cnt}" for name, cnt in rows)
+        return f"LocalRegistry({counts or 'empty'})"
+
+    # ── Mutations ─────────────────────────────────────────────────────────
+
+    def drop(self, urn: "SdmxUrn[Any]") -> bool:
+        """Remove an artefact by URN. Returns True if it existed."""
+        return writers.drop_artefact(self._conn, str(urn))
+
+    def replace(self, old_urn: "SdmxUrn[Any]", new: Any) -> None:
+        """Remove the old artefact and add the new one."""
+        writers.drop_artefact(self._conn, str(old_urn))
+        self.add(new)
+
+    # ── Direct SQL helpers ────────────────────────────────────────────────
+
+    def code_map(self, codelist_urn: "str | SdmxUrn[Any]", lang: str = "en") -> dict[str, str]:
+        """Return {code_id: label} for a codelist without creating Code objects."""
+        return readers.code_map(self._conn, str(codelist_urn), lang)
+
+    def dsd_code_map(self, dsd_urn: "str | SdmxUrn[Any]", lang: str = "en") -> dict[str, dict[str, str]]:
+        """Return {component_id: {code_id: label}} for all enumerated dimensions."""
+        return readers.code_map_for_dsd(self._conn, str(dsd_urn), lang)
+
+    # ── Bulk ──────────────────────────────────────────────────────────────
+
+    def ingest(self, registry: Registry) -> None:
+        """Copy all artefacts from an in-memory Registry into this store."""
+        self.add_all(registry._store.values())  # noqa: SLF001
+
+    # ── Escape hatch ──────────────────────────────────────────────────────
+
+    def to_registry(self) -> Registry:
+        """Load everything into an in-memory Registry with full Ref interning."""
+        reg = Registry()
+        rows = self._conn.execute("""
+            SELECT urn, artefact_type, _document FROM artefacts
+        """).fetchall()
+        for urn_str, artefact_type, doc_str in rows:
+            cls = _TYPE_REGISTRY.get(artefact_type)
+            if cls is None:
+                continue
+            data = json.loads(doc_str)
+            if artefact_type == "Codelist":
+                data["codes"] = readers.load_code_dicts(self._conn, urn_str)
+            elif artefact_type == "Hierarchy":
+                data["tree"] = readers.load_hierarchy_tree(self._conn, urn_str)
+            reg.add(from_dict(cls, data))
+        return reg

sdmxlib/model/__init__.py

@@ -0,0 +1,122 @@
+"""sdmxlib.model — domain model, stdlib only."""
+
+from sdmxlib.model.annotations import Annotation, Annotations
+from sdmxlib.model.convert import from_dict, to_dict
+from sdmxlib.model.validation import ReferenceIssue, ValidationIssue, validate, validate_references
+from sdmxlib.model.base import MaintainableArtefact
+from sdmxlib.model.category import Categorisation, Category, CategoryScheme
+from sdmxlib.model.constraint import (
+    ConstraintAttachment,
+    ContentConstraint,
+    CubeRegion,
+    KeyValue,
+    ReferencePeriod,
+    ReleaseCalendar,
+)
+from sdmxlib.model.mapping import (
+    CategoryMap,
+    CategorySchemeMap,
+    CodelistMap,
+    CodeMap,
+    ComponentMap,
+    ConceptMap,
+    ConceptSchemeMap,
+    DataStructureMap,
+    DataflowMap,
+    StructureSet,
+)
+from sdmxlib.model.codelist import Code, Codelist
+from sdmxlib.model.collections import ArtefactList, ItemList, Maintainable
+from sdmxlib.model.expr import ItemExpr, ItemPredicate, item
+from sdmxlib.model.concept import Concept, ConceptScheme
+from sdmxlib.model.dataflow import Dataflow
+from sdmxlib.model.dataset import Dataset
+from sdmxlib.model.hierarchy import Hierarchy, HierarchicalCode
+from sdmxlib.model.datastructure import (
+    AttachmentLevel,
+    Attribute,
+    DataStructure,
+    Dimension,
+    Group,
+    Measure,
+    MeasureDimension,
+    TimeDimension,
+)
+from sdmxlib.model.istring import InternationalString
+from sdmxlib.model.message import StructureMessage
+from sdmxlib.model.organisation import Agency, AgencyScheme, Contact, DataProvider, DataProviderScheme
+from sdmxlib.model.provision import ProvisionAgreement
+from sdmxlib.model.ref import AnyRef, Ref, SdmxRef, UnresolvedRef
+from sdmxlib.model.registry import Registry
+from sdmxlib.model.representation import DataType, Facet, Representation
+from sdmxlib.model.urn import SdmxUrn
+
+__all__ = [
+    "Agency",
+    "AgencyScheme",
+    "Annotation",
+    "Annotations",
+    "AnyRef",
+    "ArtefactList",
+    "AttachmentLevel",
+    "Attribute",
+    "Categorisation",
+    "Category",
+    "CategoryMap",
+    "CategoryScheme",
+    "CategorySchemeMap",
+    "Code",
+    "CodeMap",
+    "Codelist",
+    "CodelistMap",
+    "ComponentMap",
+    "Concept",
+    "ConceptMap",
+    "ConceptScheme",
+    "ConceptSchemeMap",
+    "ConstraintAttachment",
+    "Contact",
+    "ContentConstraint",
+    "CubeRegion",
+    "DataProvider",
+    "DataProviderScheme",
+    "DataStructure",
+    "DataStructureMap",
+    "DataType",
+    "Dataflow",
+    "DataflowMap",
+    "Dataset",
+    "Dimension",
+    "Facet",
+    "Group",
+    "HierarchicalCode",
+    "Hierarchy",
+    "InternationalString",
+    "ItemExpr",
+    "ItemList",
+    "ItemPredicate",
+    "KeyValue",
+    "Maintainable",
+    "MaintainableArtefact",
+    "Measure",
+    "MeasureDimension",
+    "ProvisionAgreement",
+    "Ref",
+    "ReferenceIssue",
+    "ReferencePeriod",
+    "Registry",
+    "ReleaseCalendar",
+    "Representation",
+    "SdmxRef",
+    "SdmxUrn",
+    "StructureMessage",
+    "StructureSet",
+    "TimeDimension",
+    "UnresolvedRef",
+    "ValidationIssue",
+    "from_dict",
+    "item",
+    "to_dict",
+    "validate",
+    "validate_references",
+]

sdmxlib/model/annotations.py

@@ -0,0 +1,103 @@
+"""Annotation and Annotations — SDMX annotation support."""
+
+from collections.abc import Iterator, Sequence
+from typing import overload
+
+from attrs import define, field
+
+from sdmxlib.model.istring import InternationalString
+
+
+def _to_istring(v: InternationalString | dict[str, str]) -> InternationalString:
+    return InternationalString.of(v) if isinstance(v, dict) else v
+
+
+@define
+class Annotation:
+    """A single SDMX annotation attached to an artefact or item."""
+
+    id: str | None = None
+    title: str | None = None
+    type: str | None = None
+    url: str | None = None
+    text: InternationalString = field(factory=InternationalString, converter=_to_istring)
+
+
+class Annotations(Sequence[Annotation]):
+    """Immutable collection of Annotation with type-based querying.
+
+    Example:
+        "DEPRECATED" in annotations  # checks annotation type
+        annotations.by_type("CONTACT")  # list[Annotation]
+        annotations.first("CONTACT")  # Annotation | None
+        annotations.types  # list[str]
+    """
+
+    __slots__ = ("_items",)
+
+    def __init__(self, items: list[Annotation] | None = None) -> None:
+        self._items: tuple[Annotation, ...] = tuple(items or [])
+
+    # ── Sequence ──────────────────────────────────────────────────────────
+
+    @overload
+    def __getitem__(self, index: int) -> Annotation: ...
+
+    @overload
+    def __getitem__(self, index: slice) -> "Annotations": ...
+
+    def __getitem__(self, index: int | slice) -> "Annotation | Annotations":
+        if isinstance(index, slice):
+            return Annotations(list(self._items[index]))
+        return self._items[index]
+
+    def __len__(self) -> int:
+        return len(self._items)
+
+    def __iter__(self) -> Iterator[Annotation]:
+        return iter(self._items)
+
+    def __bool__(self) -> bool:
+        return bool(self._items)
+
+    # ── Type-based querying ───────────────────────────────────────────────
+
+    def __contains__(self, key: object) -> bool:
+        """True if any annotation has type == key."""
+        if isinstance(key, str):
+            return any(a.type == key for a in self._items)
+        return super().__contains__(key)
+
+    def by_type(self, annotation_type: str) -> list[Annotation]:
+        """All annotations with the given type."""
+        return [a for a in self._items if a.type == annotation_type]
+
+    def first(self, annotation_type: str) -> Annotation | None:
+        """First annotation with the given type, or None."""
+        for a in self._items:
+            if a.type == annotation_type:
+                return a
+        return None
+
+    @property
+    def types(self) -> list[str]:
+        """All type values present (deduplicated, order preserved)."""
+        seen: set[str] = set()
+        result: list[str] = []
+        for a in self._items:
+            if a.type and a.type not in seen:
+                seen.add(a.type)
+                result.append(a.type)
+        return result
+
+    # ── Equality & repr ───────────────────────────────────────────────────
+
+    def __eq__(self, other: object) -> bool:
+        if isinstance(other, Annotations):
+            return self._items == other._items
+        return NotImplemented
+
+    __hash__ = None  # type: ignore[assignment]  # mutable container, explicitly unhashable
+
+    def __repr__(self) -> str:
+        return f"Annotations([{', '.join(repr(a) for a in self._items)}])"

sdmxlib/model/base.py

@@ -0,0 +1,61 @@
+"""MaintainableArtefact mixin — provides the .urn() classmethod."""
+
+from typing import TYPE_CHECKING, ClassVar, Self
+
+from sdmxlib.model.urn import SdmxUrn
+
+if TYPE_CHECKING:
+    from sdmxlib.model.organisation import Agency
+
+
+class MaintainableArtefact:
+    """Mixin for top-level SDMX artefacts that have a globally unique URN.
+
+    Subclasses must declare ``sdmx_package`` and ``sdmx_class`` as
+    class variables. The ``.urn()`` classmethod then constructs a typed
+    ``SdmxUrn[Self]`` without repeating the logic in every class.
+
+    NOTE: Subclasses must redeclare the standard fields (id, maintainer,
+    version, name, description, is_final, annotations) because slotted
+    dataclasses cannot inherit slots from a non-dataclass parent.
+    Python requires that all ``__slots__`` are declared on the class that
+    uses them.
+
+    Example:
+        Codelist.urn(agency="SDMX", id="CL_FREQ")  # SdmxUrn[Codelist]
+        DataStructure.urn(agency="ESTAT", id="CPI", version="2.0")
+    """
+
+    sdmx_package: ClassVar[str]
+    sdmx_class: ClassVar[str]
+    msg_field: ClassVar[str]
+
+    # Declared for type checking — concrete subclasses provide via dataclass fields
+    maintainer: "Agency | str"
+
+    @property
+    def agency_id(self) -> str:
+        """Maintainer agency identifier. Derived from the ``maintainer`` field."""
+        # After __post_init__, maintainer is always Agency (str is coerced)
+        m = self.maintainer
+        if isinstance(m, str):
+            return m
+        return m.id
+
+    @classmethod
+    def urn(
+        cls,
+        *,
+        agency: str,
+        id: str,  # noqa: A002
+        version: str = "latest",
+    ) -> SdmxUrn[Self]:
+        """Construct a typed SdmxUrn for this artefact class."""
+        return SdmxUrn.make(
+            cls,
+            package=cls.sdmx_package,
+            artefact_class=cls.sdmx_class,
+            agency=agency,
+            id=id,
+            version=version,
+        )