csvw-safe 0.0.1__py3-none-any.whl

csvw_safe/__init__.py

@@ -0,0 +1,50 @@
+"""
+Top-level public interface for csvw_safe.
+
+This module provides a simplified API by re-exporting the most commonly used
+functions, classes, and constants for working with CSVW-style metadata.
+
+It includes utilities to:
+
+- Generate metadata from datasets
+- Generate dummy datasets from metadata
+- Validate metadata (standard and SHACL-based validation)
+- Convert metadata to OpenDP and SmartNoise SQL contexts
+- Assert structural equivalence between datasets
+- Work with metadata models and datatypes
+"""
+
+from .assert_same_structure import assert_same_structure
+from .constants import COL_LIST, COL_NAME, MAXIMUM, MINIMUM, TABLE_SCHEMA
+from .csvw_to_opendp_context import csvw_to_opendp_context
+from .csvw_to_smartnoise_sql import csvw_to_smartnoise_sql
+from .datatypes import XSD_GROUP_MAP, DataTypesGroups, to_pandas_dtype
+from .make_dummy_from_metadata import make_dummy_from_metadata
+from .make_metadata_from_data import make_metadata_from_data
+from .metadata_structure import ColumnMetadata, TableMetadata
+from .validate_metadata import validate_metadata
+from .validate_metadata_shacl import validate_metadata_shacl
+
+__all__ = [  # noqa: RUF022
+    # Core functionality
+    "assert_same_structure",
+    "csvw_to_opendp_context",
+    "csvw_to_smartnoise_sql",
+    "make_dummy_from_metadata",
+    "make_metadata_from_data",
+    "validate_metadata",
+    "validate_metadata_shacl",
+    # Metadata models
+    "TableMetadata",
+    "ColumnMetadata",
+    # Constants
+    "COL_LIST",
+    "COL_NAME",
+    "MAXIMUM",
+    "MINIMUM",
+    "TABLE_SCHEMA",
+    # Datatypes
+    "XSD_GROUP_MAP",
+    "DataTypesGroups",
+    "to_pandas_dtype",
+]

csvw_safe/assert_same_structure.py

@@ -0,0 +1,133 @@
+"""
+Utility script to verify that a generated dummy CSV preserves the structural.
+
+properties of an original CSV dataset.
+
+The script checks:
+- column names and order
+- inferred CSVW-SAFE datatypes
+- nullability (required vs optional columns)
+- optional categorical value compatibility
+
+It does NOT check statistical similarity, only structural compatibility.
+"""
+
+import argparse
+import sys
+from pathlib import Path
+
+import pandas as pd
+
+from csvw_safe.datatypes import (
+    XSD_GROUP_MAP,
+    DataTypesGroups,
+    infer_xmlschema_datatype,
+    is_categorical,
+)
+
+
+def assert_same_structure(
+    df1: pd.DataFrame,
+    df2: pd.DataFrame,
+    check_categories: bool = True,
+) -> None:
+    """
+    Verify that two CSV files share the same structural schema.
+
+    The function checks column names/order, inferred datatypes,
+    nullability constraints, and optionally categorical value sets.
+
+    Parameters
+    ----------
+    df1 : pd.DataFrame
+        Original dataframe.
+    df2 : pd.DataFrame
+        Dummy dataframe.
+    check_categories : bool, default=True
+        Whether to verify that categorical values in the dummy data
+        are subsets of those in the original data.
+
+    Raises
+    ------
+    AssertionError
+        If any structural mismatch is detected.
+
+    """
+    # Columns: order and names
+    if list(df1.columns) != list(df2.columns):
+        raise AssertionError(
+            f"Column names/order differ:\nOriginal: {list(df1.columns)}\nDummy:{list(df2.columns)}"
+        )
+
+    # Data types
+    for col in df1.columns:
+        dtype1 = infer_xmlschema_datatype(df1[col])
+        dtype2 = infer_xmlschema_datatype(df2[col])
+
+        group1 = XSD_GROUP_MAP.get(dtype1)
+        group2 = XSD_GROUP_MAP.get(dtype2)
+
+        # If both are integer types, accept subtype differences
+        if group1 == DataTypesGroups.INTEGER and group2 == DataTypesGroups.INTEGER:
+            continue
+
+        if dtype1 != dtype2:
+            raise AssertionError(f"Column '{col}' dtype mismatch: original={dtype1}, dummy={dtype2}")
+
+    # Nullability
+    for col in df1.columns:
+        required1: bool = df1[col].notna().all()
+        required2: bool = df2[col].notna().all()
+
+        if required1 != required2:
+            raise AssertionError(
+                f"Column '{col}' nullability mismatch: original required={required1}, "
+                f"dummy required={required2}"
+            )
+
+    # Categorical subset check
+    if check_categories:
+        cat_cols = [col for col in df1.columns if is_categorical(df1[col])]
+        for col in cat_cols:
+            vals1 = set(df1[col].dropna().unique())
+            vals2 = set(df2[col].dropna().unique())
+
+            if not vals2.issubset(vals1):
+                raise AssertionError(
+                    f"Column '{col}' dummy values {vals2} are not subset of original {vals1}"
+                )
+
+
+def main() -> None:
+    """Command-line entry point for the CSV structure validator."""
+    parser = argparse.ArgumentParser(
+        description="Assert that two CSV files match CSVW-SAFE structural properties"
+    )
+    parser.add_argument("original_csv", type=str, help="Original CSV file")
+    parser.add_argument("dummy_csv", type=str, help="Dummy CSV file")
+    parser.add_argument(
+        "--no-categories",
+        action="store_true",
+        help="Skip categorical subset validation",
+    )
+
+    args = parser.parse_args()
+
+    df1 = pd.read_csv(Path(args.original_csv), parse_dates=True)
+    df2 = pd.read_csv(Path(args.dummy_csv), parse_dates=True)
+    try:
+        assert_same_structure(
+            df1,
+            df2,
+            check_categories=not args.no_categories,
+        )
+    except AssertionError as e:
+        print(f"Structure mismatch: {e}")  # noqa: T201
+        sys.exit(1)
+    except Exception as e:
+        print(f"ERROR: {e}")  # noqa: T201
+        sys.exit(2)
+
+
+if __name__ == "__main__":
+    main()

csvw_safe/constants.py

@@ -0,0 +1,79 @@
+"""Defaults, constants and metadata objects for csvw-safe."""
+
+import string
+from enum import StrEnum
+from pathlib import Path
+
+# ============================================================
+# CSVW
+# ============================================================
+CSVW_CONTEXT = "http://www.w3.org/ns/csvw"
+COL_NAME = "name"
+DATATYPE = "datatype"
+REQUIRED = "required"
+MINIMUM = "minimum"
+MAXIMUM = "maximum"
+TABLE_SCHEMA = "tableSchema"
+COL_LIST = "columns"
+COL_TYPE = "Column"
+TABLE_TYPE = "Table"
+
+# ============================================================
+# CSVW_SAFE Namespaces
+# ============================================================
+CSVW_SAFE_CONTEXT = str((Path(__file__).resolve().parents[2] / "csvw-safe-context.jsonld").resolve())  # tmp
+
+# Column groups / partitions
+COLUMN_GROUP = "ColumnGroup"
+PARTITION = "Partition"
+COLUMNS_IN_GROUP = "columnsInGroup"
+PUBLIC_PARTITIONS = "partitions"
+KEY_VALUES = "keyValues"
+EXHAUSTIVE_KEYS = "exhaustiveKeys"
+INVARIANT_PUBLIC_KEYS = "invariantPublicKeys"
+MAX_NUM_PARTITIONS = "maxNumPartitions"
+PUBLIC_LENGTH = "publicLength"
+PRIVACY_UNIT = "privacyUnit"
+PRIVACY_ID = "privacyId"
+ADD_INFO = "additionalInformation"
+
+# Differential privacy bounds
+MAX_LENGTH = "maxLength"
+MAX_GROUPS = "maxGroupsPerUnit"
+MAX_CONTRIB = "maxContributions"
+
+# Partition predicates
+PREDICATE = "predicate"
+PARTITION_VALUE = "partitionValue"
+LOWER_BOUND = "lowerBound"
+UPPER_BOUND = "upperBound"
+
+# Synthetic modeling
+NULL_PROP = "nullableProportion"
+ROW_DEP = "rowDependencies"
+DEPENDS_ON = "dependsOn"
+DEPENDENCY_TYPE = "dependencyType"
+VALUE_MAP = "valueMap"
+
+
+# ============================================================
+# Make and generate metadata
+# ============================================================
+class DependencyType(StrEnum):
+    """Types of column dependency relationships."""
+
+    MAPPING = "mapping"
+    BIGGER = "bigger"
+    # SMALLER = "smaller"  # redundant with bigger
+    FIXED = "fixedPerEntity"
+
+
+# ============================================================
+# Default Values
+# ============================================================
+DATE_LENGTH = 10  # YYYY-MM-DD only
+DEFAULT_LOWER_INCLUSIVE = True
+DEFAULT_UPPER_INCLUSIVE = True
+
+DEFAULT_NUMBER_PARTITIONS = 10
+RANDOM_STRINGS = list(string.ascii_lowercase + string.ascii_uppercase + string.digits)

csvw_safe/csvw_to_opendp_context.py

@@ -0,0 +1,173 @@
+"""
+Create an OpenDP Context from CSVW-SAFE metadata and a dataset.
+
+This module:
+- Converts CSVW-SAFE metadata into OpenDP margins
+- Builds an OpenDP Context using a provided dataset
+- Supports epsilon-based (Laplace) and rho-based (Gaussian) DP
+- Exposes both a Python API and CLI
+
+The resulting context can be used for differentially private queries.
+"""
+
+from collections.abc import Sequence
+from typing import Any, Union
+
+import opendp.prelude as dp
+import polars as pl
+from opendp.extras.polars import Bound
+from opendp.mod import Measure, Metric, enable_features
+
+from csvw_safe.constants import MAX_CONTRIB  # , PRIVACY_UNIT
+from csvw_safe.csvw_to_opendp_margins import csvw_to_opendp_margins
+
+enable_features("contrib")
+
+
+def get_privacy_loss(
+    epsilon: float | None = None,
+    rho: float | None = None,
+    delta: float | None = None,
+) -> tuple[Measure, Any]:
+    """
+    Create an opendp privacy loss object.
+
+    Parameters
+    ----------
+    epsilon : float, optional
+        Privacy budget epsilon (for Laplace DP).
+    rho : float, optional
+        Privacy budget rho (for Gaussian / zCDP).
+    delta : float, optional
+        Privacy budget delta (if using approximate DP).
+
+    Returns
+    -------
+    privacy_loss
+        opendp privacy loss object
+
+    Raises
+    ------
+    ValueError
+        If neither epsilon nor rho is provided.
+
+    """
+    if epsilon is None and rho is None:
+        raise ValueError("Either epsilon or rho must be provided")
+
+    if epsilon is not None and rho is not None:
+        raise ValueError("Specify only one of epsilon or rho")
+
+    if epsilon is not None:
+        return dp.loss_of(epsilon=epsilon, delta=delta)
+
+    return dp.loss_of(rho=rho, delta=delta)
+
+
+def get_privacy_unit(
+    csvw_meta: dict[str, Any], distance: str
+) -> tuple[Metric, Union[float, Sequence[Bound]]]:
+    """
+    Construct an OpenDP privacy unit from CSVW-SAFE metadata.
+
+    Parameters
+    ----------
+    csvw_meta : Dict[str, Any]
+        CSVW-SAFE metadata dictionary.
+    distance : str
+        Type of privacy distance metric to use (e.g. "contributions", "changes").
+
+    Returns
+    -------
+    privacy_unit
+        OpenDP privacy unit descriptor.
+
+    """
+    if MAX_CONTRIB not in csvw_meta:
+        raise ValueError("Missing max_contributions in metadata")
+
+    max_contrib = csvw_meta[MAX_CONTRIB]
+
+    kwargs: dict[str, Any] = {}
+
+    # Map distance type → correct argument
+    if distance == "contributions":
+        kwargs["contributions"] = max_contrib
+    elif distance == "changes":
+        kwargs["changes"] = max_contrib
+    # elif distance == "absolute":
+    # kwargs["absolute"] = max_contrib
+    # elif distance == "l1":
+    # kwargs["l1"] = float(max_contrib)
+    # elif distance == "l2":
+    # kwargs["l2"] = float(max_contrib)
+    else:
+        raise ValueError(f"Unsupported distance type: {distance}")
+
+    # identifier = csvw_meta.get(PRIVACY_UNIT)
+    # if identifier is not None:
+    #     kwargs["identifier"] = pl.col(identifier)  # TODO: investigate more
+
+    return dp.unit_of(**kwargs)
+
+
+def csvw_to_opendp_context(  # noqa: PLR0913
+    csvw_meta: dict[str, Any],
+    data: pl.LazyFrame,
+    epsilon: float | None = None,
+    rho: float | None = None,
+    delta: float | None = None,
+    split_evenly_over: int | None = None,
+    split_by_weights: list[float] | None = None,
+    distance: str = "contributions",
+) -> dp.Context:
+    """
+    Create an OpenDP Context from CSVW-SAFE metadata and a dataset.
+
+    Parameters
+    ----------
+    csvw_meta : Dict[str, Any]
+        CSVW-SAFE metadata dictionary.
+        Must include `csvw-safe.dp.maxContributions`.
+    data : pl.LazyFrame
+        Input dataset (recommended as LazyFrame).
+    epsilon : float, optional
+        Privacy budget epsilon (for Laplace DP).
+    rho : float, optional
+        Privacy budget rho (for Gaussian / zCDP).
+    delta : float, optional
+        Privacy budget delta (if using approximate DP).
+    split_evenly_over : int
+        Number of queries to split privacy budget across.
+    split_by_weights: list[float]
+        List of privacy budget weight by query.
+    distance: str, default='contributions'
+        Distance metric for privacy unit.
+
+    Returns
+    -------
+    Context
+        OpenDP Context object ready for queries.
+
+    Raises
+    ------
+    ValueError
+        If required metadata (max_contributions) is missing.
+        If neither epsilon nor rho is provided.
+
+    """
+    if split_evenly_over is not None and split_by_weights is not None:
+        raise ValueError("Specify only one of split_evenly_over or split_by_weights")
+
+    kwargs: dict[str, Any] = {
+        "data": data,
+        "privacy_unit": get_privacy_unit(csvw_meta, distance),
+        "privacy_loss": get_privacy_loss(epsilon, rho, delta),
+        "margins": csvw_to_opendp_margins(csvw_meta),
+    }
+    if split_by_weights is not None:
+        kwargs["split_by_weights"] = split_by_weights
+    else:
+        kwargs["split_evenly_over"] = split_evenly_over
+
+    return dp.Context.compositor(**kwargs)

csvw_safe/csvw_to_opendp_margins.py

@@ -0,0 +1,124 @@
+"""
+Convert CSVW-SAFE JSON metadata into OpenDP margin descriptors.
+
+This module provides:
+- A function to translate CSVW-SAFE differential privacy metadata into
+  OpenDP `dp.polars.Margin` objects.
+- A CLI for generating margin specifications from a JSON metadata file.
+
+The resulting margins can be used in an OpenDP context, for example:
+
+    dp.Context.compositor(
+        data=...,
+        privacy_unit=dp.unit_of(contributions=...),
+        privacy_loss=dp.loss_of(epsilon=...),
+        margins=[...],
+    )
+"""
+
+from typing import Any
+
+from opendp.extras.polars import Margin
+
+from csvw_safe.constants import (
+    ADD_INFO,
+    COL_LIST,
+    COL_NAME,
+    COLUMNS_IN_GROUP,
+    INVARIANT_PUBLIC_KEYS,
+    MAX_GROUPS,
+    MAX_LENGTH,
+    MAX_NUM_PARTITIONS,
+    PUBLIC_LENGTH,
+    TABLE_SCHEMA,
+)
+
+
+def get_margins(col_meta: dict[str, Any], by: list[str]) -> dict[str, Any]:
+    """
+    Build margin keyword arguments for a given column or column group.
+
+    Parameters
+    ----------
+    col_meta : Dict[str, Any]
+        Metadata describing a column or group of columns, including
+        differential privacy constraints (e.g., max_length, max_groups).
+    by : List[str]
+        Column name(s) to group by when defining the margin.
+
+    Returns
+    -------
+    Dict[str, Any]
+        Dictionary of keyword arguments suitable for constructing an
+        OpenDP Margin object.
+
+    """
+    margin_kwargs: dict[str, Any] = {"by": by}
+
+    # max_length per column
+    if MAX_LENGTH in col_meta:
+        margin_kwargs["max_length"] = col_meta[MAX_LENGTH]
+
+    # max_groups per column
+    if MAX_GROUPS in col_meta:
+        margin_kwargs["max_groups"] = col_meta[MAX_GROUPS]
+    elif MAX_NUM_PARTITIONS in col_meta:
+        margin_kwargs["max_groups"] = col_meta[MAX_NUM_PARTITIONS]
+
+    # Exhaustive partitions --> invariant keys
+    if col_meta.get(INVARIANT_PUBLIC_KEYS):
+        margin_kwargs["invariant"] = "keys"
+
+    if col_meta.get(PUBLIC_LENGTH):
+        margin_kwargs["invariant"] = "lengths"
+
+    return margin_kwargs
+
+
+def csvw_to_opendp_margins(csvw_meta: dict[str, Any]) -> list["Margin"]:
+    """
+    Convert CSVW-SAFE metadata to a list of OpenDP Margin objects.
+
+    Parameters
+    ----------
+    csvw_meta : Dict[str, Any]
+        CSVW-SAFE metadata dictionary.
+
+    Returns
+    -------
+    List["Margin"]
+        List of OpenDP margin descriptors.
+
+    Raises
+    ------
+    ValueError
+        If required metadata (e.g., max_contributions) is missing.
+
+    """
+    margins: list[Margin] = []
+
+    # Table-level margins: non groupby queries (by=[], max_length=10, ...)
+    margin_kwargs: dict[str, Any] = {}
+
+    # Max length (for non count queries)
+    if csvw_meta.get(MAX_LENGTH, False):
+        margin_kwargs["max_length"] = csvw_meta[MAX_LENGTH]
+
+    # If length is public --> invariant lengths
+    if csvw_meta.get(PUBLIC_LENGTH, False):
+        margin_kwargs["invariant"] = "lengths"
+
+    if margin_kwargs:
+        margins.append(Margin(**margin_kwargs))
+
+    # Column-level margins: groupby queries (by=['col_name'], max_length=100, ...)
+    for col_meta in csvw_meta[TABLE_SCHEMA][COL_LIST]:
+        margin_kwargs = get_margins(col_meta, by=[col_meta[COL_NAME]])
+        margins.append(Margin(**margin_kwargs))
+
+    # Multi-columns-level margins: groupby queries (by=['col_1', 'col_2'], max_length=100, ...)
+    for cols_meta in csvw_meta.get(ADD_INFO, []):
+        margin_kwargs = get_margins(cols_meta, by=cols_meta[COLUMNS_IN_GROUP])
+        margins.append(Margin(**margin_kwargs))
+
+    return margins