sdmxlib 0.8.0__tar.gz → 0.16.2__tar.gz

{sdmxlib-0.8.0 → sdmxlib-0.16.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sdmxlib
-Version: 0.8.0
+Version: 0.16.2
 Summary: SDMX structural metadata library for Python
 Keywords: sdmx,statistics,metadata,datastructure
 Author: gabrielgellner
@@ -10,18 +10,18 @@ Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Science/Research
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Requires-Dist: attrs>=26.1.0
 Requires-Dist: duckdb>=1.0.0
 Requires-Dist: httpx>=0.28.1
 Requires-Dist: lxml>=6.0.2
 Requires-Dist: polars>=1.39.3
-Requires-Python: >=3.12
-Project-URL: Homepage, https://gitlab.com/gabrielgellner/sdmxlib
-Project-URL: Repository, https://gitlab.com/gabrielgellner/sdmxlib
-Project-URL: Documentation, https://gabrielgellner.gitlab.io/sdmxlib
-Project-URL: Changelog, https://gitlab.com/gabrielgellner/sdmxlib/-/blob/main/CHANGELOG.md
+Requires-Python: >=3.13
+Project-URL: Homepage, https://gitlab.com/pinax-suite/sdmxlib
+Project-URL: Repository, https://gitlab.com/pinax-suite/sdmxlib
+Project-URL: Documentation, https://pinax-suite.gitlab.io/sdmxlib
+Project-URL: Changelog, https://gitlab.com/pinax-suite/sdmxlib/-/blob/main/CHANGELOG.md
 Description-Content-Type: text/markdown
 
 # sdmxlib
@@ -74,7 +74,14 @@ uv add "sdmxlib @ git+https://gitlab.com/gabrielgellner/sdmxlib.git"
 - Build SDMX artefacts locally and serialise to SDMX-ML or SDMX-JSON
 - `LocalRegistry` -- a DuckDB-backed persistent store for offline access
   with lazy codelist loading, automatic Ref resolution, and direct SQL
-  code maps; supports shared connections and configurable language columns
+  code maps; supports shared connections and any-language queries
+- Reference metadata via `MetadataStructure` (MSD), `Metadataflow`, and
+  `MetadataSet` — attach typed footnotes, methodology notes, or quality
+  reports to any identifiable artefact. Definitions ride in
+  `StructureMessage`; populated payloads ride in `MetadataMessage`
+  (`<mes:GenericMetadata>` in SDMX-ML 3.0; metadata-schema envelope in
+  SDMX-JSON 2.0). Both message kinds round-trip through
+  `LocalRegistry`, JSON, and XML.
 - Integrates with [Polars](https://pola.rs) for schema generation and data
   validation
 - Fully typed — IDE completion and static analysis work throughout

{sdmxlib-0.8.0 → sdmxlib-0.16.2}/README.md

@@ -48,7 +48,14 @@ uv add "sdmxlib @ git+https://gitlab.com/gabrielgellner/sdmxlib.git"
 - Build SDMX artefacts locally and serialise to SDMX-ML or SDMX-JSON
 - `LocalRegistry` -- a DuckDB-backed persistent store for offline access
   with lazy codelist loading, automatic Ref resolution, and direct SQL
-  code maps; supports shared connections and configurable language columns
+  code maps; supports shared connections and any-language queries
+- Reference metadata via `MetadataStructure` (MSD), `Metadataflow`, and
+  `MetadataSet` — attach typed footnotes, methodology notes, or quality
+  reports to any identifiable artefact. Definitions ride in
+  `StructureMessage`; populated payloads ride in `MetadataMessage`
+  (`<mes:GenericMetadata>` in SDMX-ML 3.0; metadata-schema envelope in
+  SDMX-JSON 2.0). Both message kinds round-trip through
+  `LocalRegistry`, JSON, and XML.
 - Integrates with [Polars](https://pola.rs) for schema generation and data
   validation
 - Fully typed — IDE completion and static analysis work throughout

{sdmxlib-0.8.0 → sdmxlib-0.16.2}/pyproject.toml

@@ -1,18 +1,18 @@
 [project]
 name = "sdmxlib"
-version = "0.8.0"
+version = "0.16.2"
 description = "SDMX structural metadata library for Python"
 readme = "README.md"
 license = { text = "Apache-2.0" }
 authors = [{ name = "gabrielgellner", email = "gabrielgellner@gmail.com" }]
-requires-python = ">=3.12"
+requires-python = ">=3.13"
 keywords = ["sdmx", "statistics", "metadata", "datastructure"]
 classifiers = [
     "Development Status :: 4 - Beta",
     "Intended Audience :: Developers",
     "Intended Audience :: Science/Research",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
     "Topic :: Scientific/Engineering :: Information Analysis",
 ]
 dependencies = [
@@ -24,10 +24,10 @@ dependencies = [
 ]
 
 [project.urls]
-Homepage      = "https://gitlab.com/gabrielgellner/sdmxlib"
-Repository    = "https://gitlab.com/gabrielgellner/sdmxlib"
-Documentation = "https://gabrielgellner.gitlab.io/sdmxlib"
-Changelog     = "https://gitlab.com/gabrielgellner/sdmxlib/-/blob/main/CHANGELOG.md"
+Homepage      = "https://gitlab.com/pinax-suite/sdmxlib"
+Repository    = "https://gitlab.com/pinax-suite/sdmxlib"
+Documentation = "https://pinax-suite.gitlab.io/sdmxlib"
+Changelog     = "https://gitlab.com/pinax-suite/sdmxlib/-/blob/main/CHANGELOG.md"
 
 [build-system]
 requires = ["uv_build>=0.11.2,<0.12.0"]
@@ -38,7 +38,8 @@ testpaths = ["tests"]
 markers = [
   "unit: fast unit tests (no I/O)",
   "docs: tests that verify documentation examples",
-  "integration: live endpoint tests (require network)",
+  "integration: tests that use real DuckDB and sdmxlib but no network",
+  "live: tests that make real HTTP calls to external SDMX endpoints",
 ]
 
 [tool.coverage.run]
@@ -57,7 +58,7 @@ exclude_lines = [
 ]
 
 [tool.basedpyright]
-pythonVersion = "3.12"
+pythonVersion = "3.13"
 pythonPlatform = "Linux"
 venvPath = "."
 venv = ".venv"

{sdmxlib-0.8.0 → sdmxlib-0.16.2}/src/sdmxlib/__init__.py

@@ -1,11 +1,16 @@
 """sdmxlib — SDMX structural metadata library for Python."""
 
+from sdmxlib.api.federated import FederatedRegistry
 from sdmxlib.api.filters import DimFilter, dim
+from sdmxlib.api.policy import FailurePolicy, RegistryUnavailable
 from sdmxlib.api.providers import Provider
 from sdmxlib.api.query import DataQuery
 from sdmxlib.api.registry import BearerToken, FmrRegistry, RestRegistry
+from sdmxlib.catalog import DataflowEntry, FederatedCatalog
+from sdmxlib.data_store import DataStore
 from sdmxlib.formats import parse, read, serialize, to_json, to_xml
 from sdmxlib.local import LazyCodelist, LocalRegistry
+from sdmxlib.local._data_store import DuckDBDataStore
 from sdmxlib.model.convert import from_dict, to_dict
 from sdmxlib.model.validation import ReferenceIssue, ValidationIssue, validate, validate_references
 from sdmxlib.model import (
@@ -20,6 +25,7 @@ from sdmxlib.model import (
     Category,
     CategoryScheme,
     Code,
+    CodeQuery,
     Codelist,
     Concept,
     ConceptScheme,
@@ -32,7 +38,7 @@ from sdmxlib.model import (
     ConceptSchemeMap,
     ConstraintAttachment,
     Contact,
-    ContentConstraint,
+    DataConstraint,
     CubeRegion,
     ReferencePeriod,
     ReleaseCalendar,
@@ -42,22 +48,33 @@ from sdmxlib.model import (
     Dataset,
     DataStructureMap,
     Dataflow,
+    DataflowBinding,
     DataflowMap,
     Dimension,
+    DriftIssue,
     Hierarchy,
     HierarchicalCode,
     Group,
+    InMemoryRegistry,
     InternationalString,
     ItemExpr,
     ItemList,
     ItemPredicate,
     KeyValue,
+    Level,
     item,
     Maintainable,
     MaintainableArtefact,
     Measure,
     MeasureDimension,
+    MetadataAttribute,
+    MetadataMessage,
+    MetadataSet,
+    MetadataStructure,
+    Metadataflow,
     ProvisionAgreement,
+    ReportedAttribute,
+    ReportedAttributeValue,
     DataType,
     Facet,
     Ref,
@@ -69,6 +86,9 @@ from sdmxlib.model import (
     StructureSet,
     TimeDimension,
     UnresolvedRef,
+    artefact_urn,
+    code,
+    validate_binding,
 )
 
 __all__ = [
@@ -87,6 +107,7 @@ __all__ = [
     "CategorySchemeMap",
     "Code",
     "CodeMap",
+    "CodeQuery",
     "Codelist",
     "CodelistMap",
     "ComponentMap",
@@ -96,42 +117,60 @@ __all__ = [
     "ConceptSchemeMap",
     "ConstraintAttachment",
     "Contact",
-    "ContentConstraint",
     "CubeRegion",
+    "DataConstraint",
     "DataProvider",
     "DataProviderScheme",
     "DataQuery",
+    "DataStore",
     "DataStructure",
     "DataStructureMap",
     "DataType",
     "Dataflow",
+    "DataflowBinding",
+    "DataflowEntry",
     "DataflowMap",
     "Dataset",
     "DimFilter",
     "Dimension",
+    "DriftIssue",
+    "DuckDBDataStore",
     "Facet",
+    "FailurePolicy",
+    "FederatedCatalog",
+    "FederatedRegistry",
     "FmrRegistry",
     "Group",
     "HierarchicalCode",
     "Hierarchy",
+    "InMemoryRegistry",
     "InternationalString",
     "ItemExpr",
     "ItemList",
     "ItemPredicate",
     "KeyValue",
     "LazyCodelist",
+    "Level",
     "LocalRegistry",
     "Maintainable",
     "MaintainableArtefact",
     "Measure",
     "MeasureDimension",
+    "MetadataAttribute",
+    "MetadataMessage",
+    "MetadataSet",
+    "MetadataStructure",
+    "Metadataflow",
     "Provider",
     "ProvisionAgreement",
     "Ref",
     "ReferenceIssue",
     "ReferencePeriod",
     "Registry",
+    "RegistryUnavailable",
     "ReleaseCalendar",
+    "ReportedAttribute",
+    "ReportedAttributeValue",
     "Representation",
     "RestRegistry",
     "SdmxRef",
@@ -141,6 +180,8 @@ __all__ = [
     "TimeDimension",
     "UnresolvedRef",
     "ValidationIssue",
+    "artefact_urn",
+    "code",
     "dim",
     "from_dict",
     "item",
@@ -151,5 +192,6 @@ __all__ = [
     "to_json",
     "to_xml",
     "validate",
+    "validate_binding",
     "validate_references",
 ]

{sdmxlib-0.8.0 → sdmxlib-0.16.2}/src/sdmxlib/api/client.py

@@ -17,7 +17,7 @@ _RESOURCE_PATH: dict[str, str] = {
     "ProvisionAgreement": "provisionagreement",
     "CategoryScheme": "categoryscheme",
     "Categorisation": "categorisation",
-    "ContentConstraint": "contentconstraint",
+    "DataConstraint": "dataconstraint",
     "AgencyScheme": "agencyscheme",
     "DataProviderScheme": "dataproviderorganisationscheme",
     "Hierarchy": "hierarchy",
@@ -133,7 +133,9 @@ class SdmxClient:
             params["references"] = references
         response = self._client.get(url, params=params)
         response.raise_for_status()
-        return parse(response.content, version=self._parse_version)
+        msg = parse(response.content, version=self._parse_version)
+        assert isinstance(msg, StructureMessage)  # noqa: S101  # structure endpoint only
+        return msg
 
     def get_all(
         self,
@@ -150,7 +152,9 @@ class SdmxClient:
             params["references"] = references
         response = self._client.get(url, params=params)
         response.raise_for_status()
-        return parse(response.content, version=self._parse_version)
+        msg = parse(response.content, version=self._parse_version)
+        assert isinstance(msg, StructureMessage)  # noqa: S101  # structure endpoint only
+        return msg
 
     def _data_url(
         self,

sdmxlib-0.16.2/src/sdmxlib/api/federated.py

@@ -0,0 +1,166 @@
+"""FederatedRegistry — agency-routed dispatch across multiple sources.
+
+The headline of #46. Composes any number of :class:`~sdmxlib.model.registry.Registry`
+instances (in-memory, local DuckDB, REST) and dispatches each ``get(urn)``
+call to the source responsible for the URN's maintenance agency. The user
+sees a single registry that resolves URNs from many backends.
+
+Cache-mode v1 — each agency has exactly one source. Peer-mode (multiple
+sources cooperating to answer a single URN) is intentionally out of
+scope; see #46 for that conversation.
+
+Example:
+    local = sl.LocalRegistry("cache.duckdb", source_registry="local")
+    estat = sl.RestRegistry(sl.Provider.ESTAT, materialise_to=local)
+    sdmx_global = sl.RestRegistry(sl.Provider.SDMX_GLOBAL, materialise_to=local)
+
+    fed = sl.FederatedRegistry(
+        routing={"ESTAT": estat, "SDMX": sdmx_global},
+        default=local,  # locally-authored agencies dispatch here
+    )
+
+    # Routes by maintenance agency:
+    dsd = fed.get(sl.DataStructure.urn(agency="ESTAT", id="X", version="1.0"))
+    # → estat → HTTP fetch → materialise to local
+
+    cl = fed.get(sl.Codelist.urn(agency="SDMX", id="CL_FREQ", version="2.1"))
+    # → sdmx_global → HTTP fetch → materialise to local
+
+    own = fed.get(sl.Codelist.urn(agency="MY_ORG", id="CL_X", version="1.0"))
+    # → falls through to local (no routing entry)
+"""
+
+from typing import TYPE_CHECKING
+
+import httpx
+
+from sdmxlib.api.policy import FailurePolicy, RegistryUnavailable
+
+if TYPE_CHECKING:
+    from sdmxlib.model.registry import Registry
+    from sdmxlib.model.urn import SdmxUrn
+
+
+class FederatedRegistry:
+    """Read-side dispatcher across multiple Registry sources.
+
+    Routes each URN-keyed lookup by maintenance agency. Configure with a
+    ``routing`` map from agency id to a :class:`Registry` source, plus
+    an optional ``default`` source for agencies absent from the map.
+
+    Satisfies the :class:`~sdmxlib.model.registry.Registry` Protocol —
+    a ``FederatedRegistry`` can itself be a source in another
+    ``FederatedRegistry`` (composition).
+
+    Args:
+        routing: Mapping of agency id (e.g. ``"ESTAT"``, ``"SDMX"``) to a
+            :class:`Registry` source. Lookup keys are matched against
+            ``SdmxUrn.agency``.
+        default: Optional fallback source for URNs whose agency is not
+            in ``routing``. ``None`` (the default) means "no fallback":
+            unrouted URNs raise :class:`KeyError`.
+        failure: How to treat exceptions raised by a routed source.
+            Defaults to :attr:`FailurePolicy.HARD_FAIL` — propagate the
+            underlying error wrapped in :class:`RegistryUnavailable`.
+
+    Note:
+        FederatedRegistry is a *router*, not a *store*. It has no
+        ``add()`` method — write to specific concrete registries
+        (typically the local one in your routing chain).
+    """
+
+    def __init__(
+        self,
+        *,
+        routing: "dict[str, Registry]",
+        default: "Registry | None" = None,
+        failure: FailurePolicy = FailurePolicy.HARD_FAIL,
+    ) -> None:
+        self._routing = dict(routing)
+        self._default = default
+        self._failure = failure
+
+    @property
+    def routing(self) -> "dict[str, Registry]":
+        """Read-only view of the agency → source map."""
+        return dict(self._routing)
+
+    @property
+    def default(self) -> "Registry | None":
+        """The fallback source used when no routing entry matches."""
+        return self._default
+
+    @property
+    def failure(self) -> FailurePolicy:
+        """The configured failure policy."""
+        return self._failure
+
+    # ── Routing ──────────────────────────────────────────────────────────
+
+    def _route(self, urn: "SdmxUrn[object]") -> "Registry | None":
+        """Pick the source responsible for ``urn``'s agency."""
+        return self._routing.get(urn.agency, self._default)
+
+    def source_for[T](self, urn: "SdmxUrn[T]") -> "Registry | None":
+        """Public version of the routing decision — useful for diagnostics."""
+        return self._route(urn)
+
+    # ── Registry Protocol ────────────────────────────────────────────────
+
+    def get[T](self, urn: "SdmxUrn[T]") -> "T | None":
+        """Resolve ``urn`` via the routed source.
+
+        Routing rules:
+
+        - URN agency in ``routing`` → dispatch to that source.
+        - URN agency absent and ``default`` is set → dispatch to default.
+        - URN agency absent and no default → raise ``KeyError``.
+
+        Errors from the routed source are wrapped in
+        :class:`RegistryUnavailable` when ``failure`` is
+        :attr:`FailurePolicy.HARD_FAIL` (default), or swallowed (returning
+        ``None``) when ``failure`` is :attr:`FailurePolicy.RETURN_NONE`.
+        """
+        source = self._route(urn)
+        if source is None:
+            err = (
+                f"No source registered for agency {urn.agency!r} and no default — "
+                f"add an entry to routing or supply default=..."
+            )
+            raise KeyError(err)
+
+        try:
+            return source.get(urn)  # type: ignore[return-value]
+        except (httpx.HTTPError, OSError, RegistryUnavailable) as exc:
+            return self._on_failure(source, exc)
+
+    def __contains__(self, urn: object) -> bool:
+        """True if the routed source already has ``urn`` cached.
+
+        Like the underlying Registry membership: never triggers a network
+        round-trip. ``False`` if no source is routed for the URN's agency.
+        """
+        from sdmxlib.model.urn import SdmxUrn  # noqa: PLC0415
+
+        if not isinstance(urn, SdmxUrn):
+            return False
+        source = self._route(urn)
+        return source is not None and urn in source
+
+    # ── Failure handling ─────────────────────────────────────────────────
+
+    def _on_failure[T](self, source: "Registry", exc: Exception) -> "T | None":
+        """Apply the configured ``FailurePolicy`` to an upstream error."""
+        source_id = getattr(source, "id", None)
+        if self._failure is FailurePolicy.RETURN_NONE:
+            return None
+        # HARD_FAIL — wrap and re-raise
+        raise RegistryUnavailable(source_id, exc) from exc
+
+    def __repr__(self) -> str:
+        agencies = ", ".join(sorted(self._routing))
+        default = " +default" if self._default is not None else ""
+        return f"FederatedRegistry(routes=[{agencies}]{default}, failure={self._failure.name})"
+
+
+__all__ = ["FederatedRegistry"]

sdmxlib-0.16.2/src/sdmxlib/api/policy.py

@@ -0,0 +1,39 @@
+"""FailurePolicy — how a federated read should behave when a source errors.
+
+Per #46's pitfall list ("Partial failure handling"), there are several
+valid behaviours when a routed source is unreachable: hard-fail, return
+``None`` (treat as not-found), or fall through to a stale local cache.
+The policy is configurable per source or globally on
+:class:`~sdmxlib.api.federated.FederatedRegistry`.
+"""
+
+from enum import Enum
+
+
+class FailurePolicy(Enum):
+    """How a federated registry handles errors from a routed source.
+
+    Attributes:
+        HARD_FAIL: Re-raise the underlying error (default). The caller
+            sees the network/HTTP exception unchanged.
+        RETURN_NONE: Catch errors that look like "source unreachable"
+            and return ``None``. The caller cannot distinguish "agency
+            doesn't have this artefact" from "source was down."
+    """
+
+    HARD_FAIL = "hard_fail"
+    RETURN_NONE = "return_none"
+
+
+class RegistryUnavailable(Exception):  # noqa: N818  # name matches #46 vocabulary
+    """Raised when a routed source cannot be reached and the policy is HARD_FAIL.
+
+    Wraps the underlying httpx / connection error so callers can catch a
+    single exception type without depending on the HTTP layer.
+    """
+
+    def __init__(self, source_id: str | None, cause: Exception) -> None:
+        self.source_id = source_id
+        self.cause = cause
+        sid = f" (source={source_id!r})" if source_id else ""
+        super().__init__(f"Registry unavailable{sid}: {cause}")

{sdmxlib-0.8.0 → sdmxlib-0.16.2}/src/sdmxlib/api/providers.py

@@ -13,10 +13,20 @@ from attrs import frozen
 
 @frozen
 class ProviderInfo:
-    """URL and REST API version for a known SDMX endpoint."""
+    """URL and REST API version for a known SDMX endpoint.
+
+    Attributes:
+        url: Base REST endpoint URL.
+        api: SDMX REST API version (``"2.1"`` or ``"3.0"``).
+        id: Stable identifier used as the ``source_registry`` provenance
+            tag when a :class:`RestRegistry` materialises through to a
+            local store. Required for federation; ``None`` for ad-hoc
+            URLs without a known label.
+    """
 
     url: str
     api: str = "2.1"
+    id: str | None = None
 
 
 class Provider:
@@ -39,17 +49,17 @@ class Provider:
     """
 
     #: European Central Bank
-    ECB = ProviderInfo("https://data-api.ecb.europa.eu/service", api="2.1")
+    ECB = ProviderInfo("https://data-api.ecb.europa.eu/service", api="2.1", id="ECB")
     #: Bank for International Settlements
-    BIS = ProviderInfo("https://stats.bis.org/api/v1", api="2.1")
+    BIS = ProviderInfo("https://stats.bis.org/api/v1", api="2.1", id="BIS")
     #: Eurostat
-    ESTAT = ProviderInfo("https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1", api="2.1")
+    ESTAT = ProviderInfo("https://ec.europa.eu/eurostat/api/dissemination/sdmx/2.1", api="2.1", id="ESTAT")
     #: International Monetary Fund
-    IMF = ProviderInfo("https://sdmxcentral.imf.org/ws/public/sdmxapi/rest", api="2.1")
+    IMF = ProviderInfo("https://sdmxcentral.imf.org/ws/public/sdmxapi/rest", api="2.1", id="IMF")
     #: OECD
-    OECD = ProviderInfo("https://sdmx.oecd.org/public/rest", api="2.1")
+    OECD = ProviderInfo("https://sdmx.oecd.org/public/rest", api="2.1", id="OECD")
     #: SDMX Global Registry (FMR — SDMX 3.0 endpoint)
-    SDMX_GLOBAL = ProviderInfo("https://registry.sdmx.org/sdmx/v2", api="3.0")
+    SDMX_GLOBAL = ProviderInfo("https://registry.sdmx.org/sdmx/v2", api="3.0", id="SDMX_GLOBAL")
 
     def __class_getitem__(cls, key: str) -> ProviderInfo:
         val = getattr(cls, key, None)

{sdmxlib-0.8.0 → sdmxlib-0.16.2}/src/sdmxlib/api/query.py

@@ -66,7 +66,7 @@ class GetQuery[T]:
             reference_type: If given, resolve only Refs of this type.
                             If ``None`` (default), resolve all references.
             partial: If ``True`` (default), referenced ItemSchemes are trimmed
-                     to items allowed by ContentConstraints
+                     to items allowed by DataConstraints
                      (``detail=referencepartial``). Produces smaller, more
                      useful responses for the common case. Set to ``False``
                      for full metadata audits (``detail=full``).
@@ -204,7 +204,7 @@ class Query[T]:
         Args:
             reference_type: If given, resolve only Refs of this type.
             partial: Controls whether referenced ItemSchemes are trimmed by
-                     ContentConstraints. ``True`` (default) is faster and
+                     DataConstraints. ``True`` (default) is faster and
                      more useful for data queries; ``False`` for full audits.
         """
         ref_str = "descendants" if reference_type is None else _to_references(reference_type)