sdsa 1.1.0__tar.gz

sdsa-1.1.0/PKG-INFO

@@ -0,0 +1,20 @@
+Metadata-Version: 2.4
+Name: sdsa
+Version: 1.1.0
+Summary: Secure Data Sanitization App backend
+Requires-Python: >=3.11
+Requires-Dist: fastapi>=0.115
+Requires-Dist: uvicorn[standard]>=0.30
+Requires-Dist: python-multipart>=0.0.9
+Requires-Dist: polars>=1.0
+Requires-Dist: pyarrow>=16
+Requires-Dist: opendp>=0.11
+Requires-Dist: phonenumbers>=8.13
+Requires-Dist: chardet>=5.2
+Requires-Dist: pydantic>=2.8
+Requires-Dist: python-dateutil>=2.9
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-asyncio>=0.24; extra == "dev"
+Requires-Dist: httpx>=0.27; extra == "dev"
+Requires-Dist: ruff>=0.6; extra == "dev"

sdsa-1.1.0/pyproject.toml

@@ -0,0 +1,50 @@
+[project]
+name = "sdsa"
+version = "1.1.0"
+description = "Secure Data Sanitization App backend"
+requires-python = ">=3.11"
+dependencies = [
+    "fastapi>=0.115",
+    "uvicorn[standard]>=0.30",
+    "python-multipart>=0.0.9",
+    "polars>=1.0",
+    "pyarrow>=16",
+    "opendp>=0.11",
+    "phonenumbers>=8.13",
+    "chardet>=5.2",
+    "pydantic>=2.8",
+    "python-dateutil>=2.9",
+]
+
+[project.scripts]
+sdsa-server = "sdsa.cli:main"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.24",
+    "httpx>=0.27",
+    "ruff>=0.6",
+]
+
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+sdsa = ["frontend/*.html", "frontend/*.css", "frontend/*.js"]
+
+[tool.pytest.ini_options]
+pythonpath = ["src"]
+testpaths = ["tests"]
+asyncio_mode = "auto"
+filterwarnings = [
+    "ignore:Field name \"schema\".*:UserWarning",
+]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"

sdsa-1.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

sdsa-1.1.0/src/sdsa/__init__.py

@@ -0,0 +1 @@
+__version__ = "1.1.0"

sdsa-1.1.0/src/sdsa/anonymize/policy.py

@@ -0,0 +1,92 @@
+"""Column policy model + apply function.
+
+A ColumnPolicy describes how one column should be transformed. The pipeline
+iterates over policies and invokes the matching primitive or DP mechanism.
+"""
+from __future__ import annotations
+
+from typing import Any, Literal
+
+import polars as pl
+from pydantic import BaseModel, Field, field_validator
+
+from . import primitives as prim
+
+Action = Literal[
+    "retain", "mask", "hash", "tokenize", "redact",
+    "numeric_bin", "date_truncate", "string_truncate",
+    "dp_laplace",
+    "drop",
+]
+
+
+class PolicyApplicationError(ValueError):
+    pass
+
+
+class ColumnPolicy(BaseModel):
+    column: str = Field(min_length=1)
+    action: Action
+    params: dict[str, Any] = Field(default_factory=dict)
+    is_quasi_identifier: bool = False
+
+    # For dp_laplace: caller must supply `epsilon` and column bounds `lower`/`upper`.
+
+    @field_validator("column")
+    @classmethod
+    def validate_column(cls, value: str) -> str:
+        if "\n" in value or "\r" in value or "\x00" in value:
+            raise ValueError("column names must not contain newlines or null bytes")
+        if len(value) > 200:
+            raise ValueError("column names must not exceed 200 characters")
+        return value
+
+
+def apply_policy(df: pl.DataFrame, policy: ColumnPolicy, hmac_key: bytes) -> pl.DataFrame:
+    """Apply a single non-DP policy to a DataFrame. DP is applied separately."""
+    col = policy.column
+    if col not in df.columns:
+        return df
+    s = df[col]
+    action = policy.action
+    p = policy.params
+
+    try:
+        if action == "retain":
+            return df
+        if action == "drop":
+            return df.drop(col)
+        if action == "mask":
+            out = prim.mask(s, keep_prefix=p.get("keep_prefix", 0),
+                           keep_suffix=p.get("keep_suffix", 0),
+                           mask_char=p.get("mask_char", "*"))
+        elif action == "hash":
+            out = prim.hmac_hash(s, hmac_key)
+        elif action == "tokenize":
+            out = prim.tokenize(s, hmac_key, prefix=p.get("prefix", "tok_"))
+        elif action == "redact":
+            out = prim.redact(s, replacement=p.get("replacement", "[REDACTED]"))
+        elif action == "numeric_bin":
+            if "bin_width" not in p:
+                raise PolicyApplicationError(
+                    f"column '{col}' with action 'numeric_bin' requires param 'bin_width'"
+                )
+            out = prim.numeric_bin(s, bin_width=float(p["bin_width"]))
+        elif action == "date_truncate":
+            out = prim.date_truncate(s, granularity=p.get("granularity", "month"))
+        elif action == "string_truncate":
+            out = prim.string_truncate(s, keep=int(p.get("keep", 3)),
+                                       pad_char=p.get("pad_char", "*"))
+        elif action == "dp_laplace":
+            # Applied by the DP pass, not here.
+            return df
+        else:
+            raise PolicyApplicationError(f"unknown action {action}")
+    except PolicyApplicationError:
+        raise
+    except (KeyError, TypeError, ValueError) as e:
+        raise PolicyApplicationError(
+            f"invalid params for column '{col}' action '{action}': {e}"
+        ) from e
+
+    return df.with_columns(out.alias(col))

sdsa-1.1.0/src/sdsa/anonymize/primitives.py

@@ -0,0 +1,195 @@
+"""Per-column anonymization primitives.
+
+Each function takes a Polars Series and returns a transformed Series.
+Row count is preserved except for suppression (done by the k-anonymity step).
+"""
+from __future__ import annotations
+
+from decimal import Decimal, ROUND_FLOOR
+import hashlib
+import hmac
+import math
+import secrets
+from datetime import date, datetime
+
+import polars as pl
+
+# --- direct-identifier primitives --------------------------------------------
+
+def mask(series: pl.Series, keep_prefix: int = 0, keep_suffix: int = 0,
+         mask_char: str = "*") -> pl.Series:
+    """Replace characters with mask_char, optionally keeping a prefix/suffix.
+
+    Guarantees at least one masked character when the input is non-empty.
+    If keep_prefix + keep_suffix >= len(s), both are scaled down
+    proportionally so that at least one character is masked — otherwise
+    a short value like "hi" with keep_prefix=5 would leak unchanged.
+    """
+    if keep_prefix < 0 or keep_suffix < 0:
+        raise ValueError("keep_prefix and keep_suffix must be >= 0")
+    if not mask_char:
+        raise ValueError("mask_char must be a non-empty string")
+
+    def _mask(v):
+        if v is None:
+            return None
+        s = str(v)
+        n = len(s)
+        if n == 0:
+            return s
+        p = keep_prefix
+        q = keep_suffix
+        # Enforce the privacy invariant: at least one character is masked.
+        # If the caller's prefix+suffix would leave zero masked chars, we
+        # shrink them proportionally (rounding down) so 1 char gets masked.
+        if p + q >= n:
+            # Scale so p + q = n - 1 (at least one char masked).
+            target = max(n - 1, 0)
+            if p + q > 0:
+                scale = target / (p + q)
+                p = int(p * scale)
+                q = int(q * scale)
+            else:
+                p = q = 0
+        p = min(p, n)
+        q = min(q, max(n - p, 0))
+        middle = mask_char * (n - p - q)
+        return s[:p] + middle + (s[n - q:] if q else "")
+    return series.map_elements(_mask, return_dtype=pl.Utf8)
+
+
+def hmac_hash(series: pl.Series, key: bytes) -> pl.Series:
+    """HMAC-SHA256, hex-truncated to 16 chars. Keyed → resists rainbow tables."""
+    def _h(v):
+        if v is None:
+            return None
+        digest = hmac.new(key, str(v).encode("utf-8"), hashlib.sha256).hexdigest()
+        return digest[:16]
+    return series.map_elements(_h, return_dtype=pl.Utf8)
+
+
+def tokenize(series: pl.Series, key: bytes, prefix: str = "tok_") -> pl.Series:
+    """Deterministic-within-session token. Uses HMAC to prevent rainbow tables."""
+    def _t(v):
+        if v is None:
+            return None
+        digest = hmac.new(key, str(v).encode("utf-8"), hashlib.sha256).hexdigest()
+        return f"{prefix}{digest[:12]}"
+    return series.map_elements(_t, return_dtype=pl.Utf8)
+
+
+def redact(series: pl.Series, replacement: str = "[REDACTED]") -> pl.Series:
+    return pl.Series(series.name, [replacement if v is not None else None for v in series],
+                     dtype=pl.Utf8)
+
+
+# --- generalization primitives -----------------------------------------------
+
+def numeric_bin(series: pl.Series, bin_width: float) -> pl.Series:
+    """Equal-width binning: value → [lo, lo+width)."""
+    if bin_width <= 0:
+        raise ValueError("bin_width must be > 0")
+    step = Decimal(str(bin_width))
+
+    def _fmt_decimal(value: Decimal) -> str:
+        normalized = format(value.normalize(), "f")
+        if "." in normalized:
+            normalized = normalized.rstrip("0").rstrip(".")
+        return normalized or "0"
+
+    def _bin(v):
+        if v is None:
+            return None
+        try:
+            fv = float(v)
+        except (TypeError, ValueError):
+            raise ValueError(
+                f"numeric_bin cannot convert a value in column '{series.name}' to float; "
+                "column must contain numeric data"
+            ) from None
+        if not math.isfinite(fv):
+            return None
+        dec_value = Decimal(str(v))
+        bucket = (dec_value / step).to_integral_value(rounding=ROUND_FLOOR)
+        lo = bucket * step
+        hi = lo + step
+        return f"[{_fmt_decimal(lo)}, {_fmt_decimal(hi)})"
+    return series.map_elements(_bin, return_dtype=pl.Utf8)
+
+
+def date_truncate(series: pl.Series, granularity: str = "month") -> pl.Series:
+    """Truncate dates/datetimes to year / month / day.
+
+    Requires a Date/Datetime/Time column. For strings (e.g. a column that
+    Polars couldn't auto-parse because of a non-ISO format), we attempt a
+    best-effort parse with dateutil; if that fails for any non-null value
+    we raise rather than silently passing the original value through
+    (which would leak full-resolution dates).
+    """
+    if granularity not in ("year", "month", "day"):
+        raise ValueError("granularity must be year/month/day")
+
+    if series.dtype == pl.Time:
+        raise ValueError(
+            f"date_truncate does not support pl.Time columns ('{series.name}'); "
+            "use a Date or Datetime column"
+        )
+    if series.dtype in (pl.Date, pl.Datetime):
+        fmt = {"year": "%Y", "month": "%Y-%m", "day": "%F"}[granularity]
+        return series.dt.strftime(fmt).alias(series.name)
+
+    from dateutil import parser as _date_parser
+
+    def _t(v):
+        if v is None:
+            return None
+        if isinstance(v, datetime):
+            d = v.date()
+        elif isinstance(v, date):
+            d = v
+        else:
+            # Best-effort string parse. We intentionally raise on failure —
+            # silently stringifying the value would leak it.
+            try:
+                d = _date_parser.parse(str(v)).date()
+            except (ValueError, TypeError, OverflowError) as e:
+                raise ValueError(
+                    f"date_truncate cannot parse a value in column '{series.name}' as a date "
+                    f"(row value withheld for privacy): {e}"
+                ) from e
+        if granularity == "year":
+            return f"{d.year:04d}"
+        if granularity == "month":
+            return f"{d.year:04d}-{d.month:02d}"
+        return d.isoformat()
+    return series.map_elements(_t, return_dtype=pl.Utf8)
+
+
+def string_truncate(series: pl.Series, keep: int = 3, pad_char: str = "*") -> pl.Series:
+    """Keep first `keep` chars, pad the rest (e.g., ZIP 12345 → 123**).
+
+    Guarantees at least one character is masked for any non-empty value, even
+    when keep >= len(s) — without this, short values like two-letter state
+    codes pass through completely unmasked.
+    """
+    if keep < 0:
+        raise ValueError("keep must be >= 0")
+    if len(pad_char) != 1:
+        raise ValueError("pad_char must be a single character")
+
+    def _t(v):
+        if v is None:
+            return None
+        s = str(v)
+        n = len(s)
+        if n == 0:
+            return s
+        effective_keep = min(keep, max(n - 1, 0))
+        return s[:effective_keep] + pad_char * (n - effective_keep)
+    return series.map_elements(_t, return_dtype=pl.Utf8)
+
+
+# --- utility -----------------------------------------------------------------
+
+def new_session_key() -> bytes:
+    return secrets.token_bytes(32)