sdmxlib 0.8.0__py3-none-any.whl

sdmxlib/polars.py

@@ -0,0 +1,392 @@
+"""Polars integration for sdmxlib.
+
+Low-level helpers for schema generation, validation, and label views.
+Used internally by :class:`~sdmxlib.model.dataset.Dataset`; also importable
+directly for standalone use::
+
+    import sdmxlib.polars as slpl
+
+    schema = slpl.schema(dsd)
+    df = pl.read_csv("data.csv", schema_overrides=schema)
+    report = slpl.validation_report(dsd, df)
+    labelled = slpl.label(df.lazy(), dsd).collect()
+    meta = slpl.structure_frame(dsd)
+
+    # Higher-level Dataset API
+    ds = slpl.scan_csv("data.csv", dsd)
+    df = ds.label().collect()
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import polars as pl
+
+from sdmxlib.model.ref import Ref
+from sdmxlib.model.representation import Facet
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from sdmxlib.model.dataset import Dataset
+    from sdmxlib.model.datastructure import Attribute, DataStructure, Dimension, Measure, MeasureDimension
+
+# ── Facet → Polars dtype mapping ─────────────────────────────────────────────
+
+_TEXT_TYPE_MAP: dict[str, pl.DataType] = {
+    "String": pl.String(),
+    "Alpha": pl.String(),
+    "AlphaNumeric": pl.String(),
+    "Numeric": pl.Float64(),
+    "Integer": pl.Int64(),
+    "Long": pl.Int64(),
+    "Short": pl.Int32(),
+    "Float": pl.Float64(),
+    "Double": pl.Float64(),
+    "Decimal": pl.Float64(),
+    "Boolean": pl.Boolean(),
+    "Date": pl.Date(),
+    "DateTime": pl.Datetime(),
+    "Time": pl.Time(),
+    # All SDMX period types are represented as strings
+    "ObservationalTimePeriod": pl.String(),
+    "StandardTimePeriod": pl.String(),
+    "BasicTimePeriod": pl.String(),
+    "GregorianTimePeriod": pl.String(),
+    "GregorianYear": pl.String(),
+    "GregorianYearMonth": pl.String(),
+    "GregorianDay": pl.Date(),
+    "ReportingTimePeriod": pl.String(),
+    "ReportingYear": pl.String(),
+    "ReportingSemester": pl.String(),
+    "ReportingTrimester": pl.String(),
+    "ReportingQuarter": pl.String(),
+    "ReportingMonth": pl.String(),
+    "ReportingWeek": pl.String(),
+    "ReportingDay": pl.String(),
+    "Year": pl.String(),
+    "Month": pl.String(),
+    "YearMonth": pl.String(),
+    "Duration": pl.String(),
+    "URI": pl.String(),
+    "XHTML": pl.String(),
+}
+
+
+def _dtype_for(component: Dimension | MeasureDimension | Attribute | Measure) -> pl.DataType:
+    """Infer the Polars dtype for a single DSD component."""
+    if isinstance(component.representation, Ref) and component.representation.resolved:
+        code_ids = sorted(c.id for c in component.representation().codes)  # type: ignore[union-attr]
+        return pl.Enum(code_ids)
+    match component.representation:
+        case Facet(type=dtype) if dtype is not None:
+            return _TEXT_TYPE_MAP.get(dtype, pl.String())
+        case _:
+            return pl.String()
+
+
+# ── Public API ────────────────────────────────────────────────────────────────
+
+
+def schema(dsd: DataStructure) -> dict[str, pl.DataType]:
+    """Return a Polars schema dict for all DSD components.
+
+    Enumerated components become ``pl.Enum``; text components are mapped
+    from their ``Facet.type``; everything else defaults to
+    ``pl.String``.
+
+    Example:
+        df = pl.read_csv("data.csv", schema_overrides=slpl.schema(dsd))
+    """
+    result: dict[str, pl.DataType] = {}
+    md = [dsd.measure_dimension] if dsd.measure_dimension is not None else []
+    for comp in [*dsd.dimensions, *md, *dsd.attributes, *dsd.measures]:
+        result[comp.id] = _dtype_for(comp)
+    if dsd.time_dimension is not None:
+        td = dsd.time_dimension
+        match td.representation:
+            case Facet(type=tt) if tt is not None:
+                result[td.id] = _TEXT_TYPE_MAP.get(tt, pl.String())
+            case _:
+                result[td.id] = pl.String()
+    return result
+
+
+def validation_report(
+    dsd: DataStructure,
+    df: pl.LazyFrame | pl.DataFrame,
+) -> pl.DataFrame:
+    """Validate a DataFrame against enumerated DSD components.
+
+    Returns a DataFrame of violations::
+
+        ┌──────────┬───────────────┬───────┐
+        │ column   │ invalid_value │ count │
+        └──────────┴───────────────┴───────┘
+
+    Only columns present in ``df`` are checked. Non-enumerated components
+    and unresolved codelists are skipped.
+
+    Example:
+        report = slpl.validation_report(dsd, df)
+        if report.is_empty():
+            print("All values valid")
+    """
+    lf = df.lazy() if isinstance(df, pl.DataFrame) else df
+    schema_cols = set(lf.collect_schema().names())
+
+    violations: list[pl.DataFrame] = []
+    md = [dsd.measure_dimension] if dsd.measure_dimension is not None else []
+    for comp in [*dsd.dimensions, *md, *dsd.attributes, *dsd.measures]:
+        if comp.id not in schema_cols:
+            continue
+        if not (isinstance(comp.representation, Ref) and comp.representation.resolved):
+            continue
+        valid_codes = {c.id for c in comp.representation().codes}  # type: ignore[union-attr]
+        bad = (
+            lf.select(pl.col(comp.id).cast(pl.String))
+            .filter(~pl.col(comp.id).is_in(valid_codes) & pl.col(comp.id).is_not_null())
+            .group_by(comp.id)
+            .agg(pl.len().alias("count"))
+            .rename({comp.id: "invalid_value"})
+            .with_columns(pl.lit(comp.id).alias("column"))
+            .select("column", "invalid_value", "count")
+            .collect()
+        )
+        if not bad.is_empty():
+            violations.append(bad)
+
+    if violations:
+        return pl.concat(violations)
+    return pl.DataFrame(
+        {
+            "column": pl.Series([], dtype=pl.String),
+            "invalid_value": pl.Series([], dtype=pl.String),
+            "count": pl.Series([], dtype=pl.UInt32),
+        },
+    )
+
+
+def _label_view(
+    lf: pl.LazyFrame,
+    dsd: DataStructure,
+    lang: str = "en",
+    suffix: str = "_label",
+) -> pl.LazyFrame:
+    """Replace enumerated code columns with label columns, preserving column order.
+
+    For each enumerated component present in ``lf``, the code column is replaced
+    by a ``{col}{suffix}`` column containing the human-readable label as a
+    ``pl.Enum`` (DSD-defined order preserved). Non-enumerated columns are
+    unchanged.
+    """
+    enum_map: dict[str, tuple[list[str], list[str]]] = {}
+    md = [dsd.measure_dimension] if dsd.measure_dimension is not None else []
+    for comp in [*dsd.dimensions, *md, *dsd.attributes, *dsd.measures]:
+        if not (isinstance(comp.representation, Ref) and comp.representation.resolved):
+            continue
+        codes = [c.id for c in comp.representation().codes]  # type: ignore[union-attr]
+        labels = [c.name.get(lang) or c.id for c in comp.representation().codes]  # type: ignore[union-attr]
+        enum_map[comp.id] = (codes, labels)
+
+    select_exprs = []
+    for col_name in lf.collect_schema().names():
+        if col_name in enum_map:
+            codes, labels = enum_map[col_name]
+            unique_labels = list(dict.fromkeys(labels))
+            select_exprs.append(
+                pl.col(col_name)
+                .cast(pl.String)
+                .replace_strict(old=codes, new=labels, default=None)
+                .cast(pl.Enum(unique_labels))
+                .alias(f"{col_name}{suffix}")
+            )
+        else:
+            select_exprs.append(pl.col(col_name))
+
+    return lf.select(select_exprs)
+
+
+def label(
+    df: pl.LazyFrame,
+    dsd: DataStructure,
+    lang: str = "en",
+    suffix: str = "_label",
+) -> pl.LazyFrame:
+    """Add human-readable label columns for every enumerated component.
+
+    For each enumerated dimension/attribute/measure that exists in ``df``,
+    appends a ``{col}{suffix}`` column with the code label in ``lang``.
+    Codes with no label fall back to the code id.
+
+    Example:
+        labelled = slpl.label(pl.scan_csv("data.csv"), dsd).collect()
+    """
+    schema_cols = set(df.collect_schema().names())
+    mds = [dsd.measure_dimension] if dsd.measure_dimension is not None else []
+    for comp in [*dsd.dimensions, *mds, *dsd.attributes, *dsd.measures]:
+        if comp.id not in schema_cols:
+            continue
+        if not (isinstance(comp.representation, Ref) and comp.representation.resolved):
+            continue
+        mapping = {c.id: c.name.get(lang) or c.id for c in comp.representation().codes}  # type: ignore[union-attr]
+        old = list(mapping.keys())
+        new = list(mapping.values())
+        label_col = f"{comp.id}{suffix}"
+        df = df.with_columns(
+            pl.col(comp.id).cast(pl.String).replace_strict(old=old, new=new, default=None).alias(label_col)
+        )
+    return df
+
+
+def structure_frame(dsd: DataStructure, lang: str = "en") -> pl.DataFrame:
+    """Export the DSD structure as a tidy DataFrame.
+
+    Returns one row per component::
+
+        ┌────────────┬───────────┬──────────┬───────────┬──────────┬─────────┐
+        │ component  │ role      │ position │ concept   │ codelist │ n_codes │
+        └────────────┴───────────┴──────────┴───────────┴──────────┴─────────┘
+
+    ``role`` is one of ``"dimension"``, ``"time_dimension"``,
+    ``"attribute"``, ``"measure"``.
+
+    Example:
+        meta = slpl.structure_frame(dsd)
+    """
+    rows: list[dict[str, object]] = []
+
+    for i, dim in enumerate(dsd.dimensions, start=1):
+        concept_label = (
+            dim.concept().name.get(lang) or dim.id if isinstance(dim.concept, Ref) and dim.concept.resolved else dim.id
+        )
+        _dim_rep = dim.representation if isinstance(dim.representation, Ref) and dim.representation.resolved else None
+        cl_id = _dim_rep().id if _dim_rep else None  # type: ignore[union-attr]
+        n_codes = len(_dim_rep().codes) if _dim_rep else 0  # type: ignore[union-attr]
+        rows.append(
+            {
+                "component": dim.id,
+                "role": "dimension",
+                "position": i,
+                "concept": concept_label,
+                "codelist": cl_id,
+                "n_codes": n_codes,
+            }
+        )
+
+    if dsd.time_dimension is not None:
+        td = dsd.time_dimension
+        concept_label = (
+            td.concept().name.get(lang) or td.id if isinstance(td.concept, Ref) and td.concept.resolved else td.id
+        )
+        rows.append(
+            {
+                "component": td.id,
+                "role": "time_dimension",
+                "position": None,
+                "concept": concept_label,
+                "codelist": None,
+                "n_codes": 0,
+            }
+        )
+
+    if dsd.measure_dimension is not None:
+        md = dsd.measure_dimension
+        concept_label = (
+            md.concept().name.get(lang) or md.id if isinstance(md.concept, Ref) and md.concept.resolved else md.id
+        )
+        _md_rep = md.representation if isinstance(md.representation, Ref) and md.representation.resolved else None
+        cl_id = _md_rep().id if _md_rep else None  # type: ignore[union-attr]
+        n_codes = len(_md_rep().codes) if _md_rep else 0  # type: ignore[union-attr]
+        rows.append(
+            {
+                "component": md.id,
+                "role": "measure_dimension",
+                "position": None,
+                "concept": concept_label,
+                "codelist": cl_id,
+                "n_codes": n_codes,
+            }
+        )
+
+    for attr in dsd.attributes:
+        if isinstance(attr.concept, Ref) and attr.concept.resolved:
+            concept_label = attr.concept().name.get(lang) or attr.id
+        else:
+            concept_label = attr.id
+        _attr_rep = (
+            attr.representation if isinstance(attr.representation, Ref) and attr.representation.resolved else None
+        )
+        cl_id = _attr_rep().id if _attr_rep else None  # type: ignore[union-attr]
+        n_codes = len(_attr_rep().codes) if _attr_rep else 0  # type: ignore[union-attr]
+        rows.append(
+            {
+                "component": attr.id,
+                "role": "attribute",
+                "position": None,
+                "concept": concept_label,
+                "codelist": cl_id,
+                "n_codes": n_codes,
+            }
+        )
+
+    for meas in dsd.measures:
+        if isinstance(meas.concept, Ref) and meas.concept.resolved:
+            concept_label = meas.concept().name.get(lang) or meas.id
+        else:
+            concept_label = meas.id
+        rows.append(
+            {
+                "component": meas.id,
+                "role": "measure",
+                "position": None,
+                "concept": concept_label,
+                "codelist": None,
+                "n_codes": 0,
+            }
+        )
+
+    return pl.DataFrame(
+        {
+            "component": [r["component"] for r in rows],
+            "role": [r["role"] for r in rows],
+            "position": [r["position"] for r in rows],
+            "concept": [r["concept"] for r in rows],
+            "codelist": [r["codelist"] for r in rows],
+            "n_codes": [r["n_codes"] for r in rows],
+        },
+        schema={
+            "component": pl.String,
+            "role": pl.String,
+            "position": pl.Int32,
+            "concept": pl.String,
+            "codelist": pl.String,
+            "n_codes": pl.Int32,
+        },
+    )
+
+
+def scan_csv(source: str | Path | bytes, structure: DataStructure) -> Dataset:
+    """Parse an SDMX-CSV file or bytes into a :class:`~sdmxlib.model.dataset.Dataset`.
+
+    File paths are scanned lazily (``pl.scan_csv``); bytes are read eagerly
+    and wrapped in a ``LazyFrame``. Either way ``Dataset.data`` is always a
+    ``pl.LazyFrame`` — nothing is executed until a terminal call.
+
+    Convenience entry point. For full options see
+    :func:`sdmxlib.formats.sdmx_csv.reader.scan_csv`.
+
+    Args:
+        source: File path or raw bytes from an SDMX-CSV response.
+        structure: Resolved :class:`~sdmxlib.model.datastructure.DataStructure`.
+
+    Example:
+        import sdmxlib.polars as slpl
+
+        ds = slpl.scan_csv("data.csv", dsd)
+        df = ds.label().collect()
+    """
+    from sdmxlib.formats.sdmx_csv.reader import scan_csv as _scan_csv  # noqa: PLC0415
+
+    return _scan_csv(source, structure)

sdmxlib/storage/__init__.py

@@ -0,0 +1,28 @@
+"""Shared DuckDB storage layer for SDMX structural metadata.
+
+Provides normalized tables for codelists, DSDs, dataflows, concept
+schemes, and hierarchies. Used internally by ``LocalRegistry`` and
+importable by cataloglib for shared-database workflows.
+
+Example::
+
+    import duckdb
+    from sdmxlib.storage import schema, writers, readers
+
+    conn = duckdb.connect("store.duckdb")
+    schema.create_tables(conn, languages=["en", "fr"])
+    writers.put_codelist(conn, codelist)
+    cm = readers.code_map(conn, codelist_urn, "en")
+"""
+
+from sdmxlib.storage import lazy, readers, schema, writers
+from sdmxlib.storage.schema import create_tables, get_languages
+
+__all__ = [
+    "create_tables",
+    "get_languages",
+    "lazy",
+    "readers",
+    "schema",
+    "writers",
+]