tabdat-synth 0.1.0__py3-none-any.whl

tabdat_synth/__init__.py

@@ -0,0 +1,67 @@
+"""Synthetic data generator package."""
+
+from tabdat_synth.version import __version__
+from tabdat_synth.domain.contracts import (
+  BernoulliResponseSpec,
+  CategoricalModelStepArtifact,
+  ColumnSpec,
+  CoefficientSumSpec,
+  CoefficientTerm,
+  ConfiguredStepArtifact,
+  EmpiricalStepArtifact,
+  DatasetSchema,
+  EvaluationReport,
+  FittedSynthesizer,
+  GenerationSpec,
+  GenerationStep,
+  MetricResult,
+  ModelBackendSpec,
+  TruncatedNormalStepArtifact,
+)
+from tabdat_synth.interfaces.bundles import (
+  load_fitted_synthesizer,
+  save_fitted_synthesizer,
+)
+from tabdat_synth.interfaces.io import load_generation_spec
+from tabdat_synth.interfaces.reports import (
+  evaluation_report_to_dict,
+  save_evaluation_report,
+)
+from tabdat_synth.workflows.evaluation import (
+  evaluate_from_spec,
+  evaluate_synthetic_data,
+)
+from tabdat_synth.workflows.pipelines import (
+  fit_synthesizer,
+  generate_from_fitted,
+  generate_from_spec,
+)
+
+__all__ = [
+  "__version__",
+  "BernoulliResponseSpec",
+  "CategoricalModelStepArtifact",
+  "ColumnSpec",
+  "CoefficientSumSpec",
+  "CoefficientTerm",
+  "ConfiguredStepArtifact",
+  "DatasetSchema",
+  "EmpiricalStepArtifact",
+  "EvaluationReport",
+  "MetricResult",
+  "FittedSynthesizer",
+  "GenerationSpec",
+  "GenerationStep",
+  "ModelBackendSpec",
+  "TruncatedNormalStepArtifact",
+  "evaluate_from_spec",
+  "evaluate_synthetic_data",
+  "evaluation_report_to_dict",
+  "fit_synthesizer",
+  "generate_from_fitted",
+  "generate_from_spec",
+  "load_fitted_synthesizer",
+  "load_generation_spec",
+  "save_evaluation_report",
+  "save_fitted_synthesizer",
+]

tabdat_synth/core/__init__.py

@@ -0,0 +1 @@
+"""Pure scientific kernels for synthetic data generation."""

tabdat_synth/core/evaluation.py

@@ -0,0 +1,269 @@
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+import numpy as np
+import polars as pl
+from scipy.stats import ks_2samp
+
+
+def null_rate(series: pl.Series) -> float:
+  """Return the fraction of null rows in a series."""
+  if len(series) == 0:
+    return 0.0
+  return series.null_count() / len(series)
+
+
+def null_rate_delta(source: pl.Series, synthetic: pl.Series) -> float:
+  """Return synthetic null-rate minus source null-rate."""
+  return null_rate(synthetic) - null_rate(source)
+
+
+def categorical_total_variation_distance(
+  source: pl.Series,
+  synthetic: pl.Series,
+) -> float:
+  """Compare categorical distributions with total-variation distance."""
+  source_values = _string_values(source)
+  synthetic_values = _string_values(synthetic)
+  if source_values.size == 0 and synthetic_values.size == 0:
+    return 0.0
+  if source_values.size == 0 or synthetic_values.size == 0:
+    return 1.0
+
+  source_freq = _category_proportions(source_values)
+  synthetic_freq = _category_proportions(synthetic_values)
+  categories = set(source_freq) | set(synthetic_freq)
+  distance = sum(
+    abs(synthetic_freq.get(category, 0.0) - source_freq.get(category, 0.0))
+    for category in categories
+  )
+  return 0.5 * distance
+
+
+def unseen_synthetic_category_rate(
+  source: pl.Series,
+  synthetic: pl.Series,
+) -> float:
+  """Return the share of synthetic non-null values unseen in source data."""
+  source_values = _string_values(source)
+  synthetic_values = _string_values(synthetic)
+  if synthetic_values.size == 0:
+    return 0.0
+  if source_values.size == 0:
+    return 1.0
+
+  observed = set(source_values.tolist())
+  unseen_count = sum(value not in observed for value in synthetic_values.tolist())
+  return unseen_count / synthetic_values.size
+
+
+def numeric_summary_deltas(
+  source: pl.Series,
+  synthetic: pl.Series,
+) -> dict[str, float | None]:
+  """Return numeric summary statistics and synthetic-minus-source deltas."""
+  source_summary = _numeric_summary(_finite_numeric_values(source))
+  synthetic_summary = _numeric_summary(_finite_numeric_values(synthetic))
+  return {
+    "source_mean": source_summary["mean"],
+    "synthetic_mean": synthetic_summary["mean"],
+    "mean_delta": _optional_delta(
+      source_summary["mean"], synthetic_summary["mean"]
+    ),
+    "source_std": source_summary["std"],
+    "synthetic_std": synthetic_summary["std"],
+    "std_delta": _optional_delta(source_summary["std"], synthetic_summary["std"]),
+    "source_min": source_summary["min"],
+    "synthetic_min": synthetic_summary["min"],
+    "min_delta": _optional_delta(source_summary["min"], synthetic_summary["min"]),
+    "source_max": source_summary["max"],
+    "synthetic_max": synthetic_summary["max"],
+    "max_delta": _optional_delta(source_summary["max"], synthetic_summary["max"]),
+  }
+
+
+def numeric_ks_statistic(source: pl.Series, synthetic: pl.Series) -> float | None:
+  """Return a two-sample Kolmogorov-Smirnov statistic for numeric values."""
+  source_values = _finite_numeric_values(source)
+  synthetic_values = _finite_numeric_values(synthetic)
+  if source_values.size == 0 or synthetic_values.size == 0:
+    return None
+  return float(ks_2samp(source_values, synthetic_values).statistic)
+
+
+def exact_row_match_rate(source_df: pl.DataFrame, synthetic_df: pl.DataFrame) -> float:
+  """Return the fraction of synthetic rows exactly present in the source data."""
+  if synthetic_df.height == 0:
+    return 0.0
+  _validate_matching_columns(source_df, synthetic_df)
+  if source_df.height == 0 or source_df.width == 0:
+    return 0.0
+
+  source_rows = set(source_df.iter_rows())
+  matched = sum(row in source_rows for row in synthetic_df.iter_rows())
+  return matched / synthetic_df.height
+
+
+def duplicate_row_rate(df: pl.DataFrame) -> float:
+  """Return the fraction of rows beyond the unique-row count."""
+  if df.height == 0:
+    return 0.0
+  return (df.height - df.unique().height) / df.height
+
+
+def nearest_neighbor_distance_summary(
+  source_df: pl.DataFrame,
+  synthetic_df: pl.DataFrame,
+  columns: Iterable[str],
+  *,
+  max_pairwise_rows: int = 1000,
+  random_seed: int = 343,
+) -> dict[str, float | int | None]:
+  """Return synthetic-to-source nearest-neighbor distances over numeric columns."""
+  if max_pairwise_rows <= 0:
+    raise ValueError("max_pairwise_rows must be positive")
+
+  resolved_columns = tuple(columns)
+  source_matrix = _numeric_matrix(source_df, resolved_columns)
+  synthetic_matrix = _numeric_matrix(synthetic_df, resolved_columns)
+  if (
+    not resolved_columns
+    or source_matrix.shape[0] == 0
+    or synthetic_matrix.shape[0] == 0
+  ):
+    return _empty_nearest_neighbor_summary(
+      source_matrix.shape[0],
+      synthetic_matrix.shape[0],
+      len(resolved_columns),
+    )
+
+  rng = np.random.default_rng(random_seed)
+  source_sample = _sample_rows(source_matrix, max_pairwise_rows, rng)
+  synthetic_sample = _sample_rows(synthetic_matrix, max_pairwise_rows, rng)
+  scaled_source, scaled_synthetic = _scale_against_source(
+    source_sample,
+    synthetic_sample,
+  )
+  nearest = _nearest_distances(scaled_source, scaled_synthetic)
+
+  return {
+    "source_rows_compared": int(source_sample.shape[0]),
+    "synthetic_rows_compared": int(synthetic_sample.shape[0]),
+    "column_count": len(resolved_columns),
+    "min": float(np.min(nearest)),
+    "p50": float(np.percentile(nearest, 50)),
+    "p95": float(np.percentile(nearest, 95)),
+    "mean": float(np.mean(nearest)),
+    "max": float(np.max(nearest)),
+  }
+
+
+def _string_values(series: pl.Series) -> np.ndarray:
+  return series.cast(pl.String, strict=False).drop_nulls().to_numpy()
+
+
+def _category_proportions(values: np.ndarray) -> dict[str, float]:
+  unique_values, counts = np.unique(values.astype(str), return_counts=True)
+  total = float(values.size)
+  return {
+    str(value): float(count) / total
+    for value, count in zip(unique_values, counts, strict=True)
+  }
+
+
+def _finite_numeric_values(series: pl.Series) -> np.ndarray:
+  values = series.cast(pl.Float64, strict=False).drop_nulls().to_numpy()
+  numeric = np.asarray(values, dtype=float)
+  return numeric[np.isfinite(numeric)]
+
+
+def _numeric_summary(values: np.ndarray) -> dict[str, float | None]:
+  if values.size == 0:
+    return {"mean": None, "std": None, "min": None, "max": None}
+  return {
+    "mean": float(np.mean(values)),
+    "std": float(np.std(values)),
+    "min": float(np.min(values)),
+    "max": float(np.max(values)),
+  }
+
+
+def _optional_delta(
+  source_value: float | None,
+  synthetic_value: float | None,
+) -> float | None:
+  if source_value is None or synthetic_value is None:
+    return None
+  return synthetic_value - source_value
+
+
+def _validate_matching_columns(
+  source_df: pl.DataFrame,
+  synthetic_df: pl.DataFrame,
+) -> None:
+  if source_df.columns != synthetic_df.columns:
+    raise ValueError("source_df and synthetic_df must have matching columns")
+
+
+def _numeric_matrix(df: pl.DataFrame, columns: tuple[str, ...]) -> np.ndarray:
+  if not columns or df.height == 0:
+    return np.empty((0, len(columns)), dtype=float)
+  numeric_df = df.select(
+    pl.col(column).cast(pl.Float64, strict=False).alias(column)
+    for column in columns
+  )
+  matrix = np.asarray(numeric_df.to_numpy(), dtype=float)
+  if matrix.size == 0:
+    return np.empty((0, len(columns)), dtype=float)
+  finite_rows = np.isfinite(matrix).all(axis=1)
+  return matrix[finite_rows]
+
+
+def _empty_nearest_neighbor_summary(
+  source_rows: int,
+  synthetic_rows: int,
+  column_count: int,
+) -> dict[str, float | int | None]:
+  return {
+    "source_rows_compared": source_rows,
+    "synthetic_rows_compared": synthetic_rows,
+    "column_count": column_count,
+    "min": None,
+    "p50": None,
+    "p95": None,
+    "mean": None,
+    "max": None,
+  }
+
+
+def _sample_rows(
+  matrix: np.ndarray,
+  max_rows: int,
+  rng: np.random.Generator,
+) -> np.ndarray:
+  if matrix.shape[0] <= max_rows:
+    return matrix
+  indices = rng.choice(matrix.shape[0], size=max_rows, replace=False)
+  return matrix[np.sort(indices)]
+
+
+def _scale_against_source(
+  source_matrix: np.ndarray,
+  synthetic_matrix: np.ndarray,
+) -> tuple[np.ndarray, np.ndarray]:
+  center = np.mean(source_matrix, axis=0)
+  scale = np.std(source_matrix, axis=0)
+  scale = np.where(scale > 0, scale, 1.0)
+  return (source_matrix - center) / scale, (synthetic_matrix - center) / scale
+
+
+def _nearest_distances(
+  source_matrix: np.ndarray,
+  synthetic_matrix: np.ndarray,
+) -> np.ndarray:
+  distances = np.empty(synthetic_matrix.shape[0], dtype=float)
+  for index, synthetic_row in enumerate(synthetic_matrix):
+    row_distances = np.linalg.norm(source_matrix - synthetic_row, axis=1)
+    distances[index] = float(np.min(row_distances))
+  return distances

tabdat_synth/core/probability.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+import numpy as np
+
+
+def sigmoid(x: np.ndarray) -> np.ndarray:
+  """Apply sigmoid after clipping extreme log-odds for numerical stability."""
+  return 1 / (1 + np.exp(-np.clip(x, -500, 500)))
+
+
+def logit(p: float | np.ndarray, epsilon: float = 1e-6):
+  """Safely compute logit values for probabilities."""
+  p_clipped = np.clip(p, epsilon, 1 - epsilon)
+  logit_p = np.log(p_clipped / (1 - p_clipped))
+  return logit_p
+
+
+def derive_child_seed(base_seed: int, stream_id: int) -> int:
+  """Derive an independent child seed for a specific RNG stream."""
+  ss = np.random.SeedSequence([base_seed, stream_id])
+  return int(ss.generate_state(1, dtype=np.uint32)[0])

tabdat_synth/core/sampling.py

@@ -0,0 +1,268 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+
+import numpy as np
+import polars as pl
+from scipy.stats import truncnorm
+
+from tabdat_synth.core.probability import logit, sigmoid
+from tabdat_synth.core.statistics import compute_category_frequencies
+from tabdat_synth.domain.contracts import FreqShift
+
+MissingCategoryHandler = Callable[[str], None]
+MISSING_CATEGORY_LABEL = "__MISSING__"
+_AGE_GROUP_BREAKS = [54, 64, 74, 84]
+_AGE_GROUP_LABELS = ["1", "2", "3", "4", "5"]
+_AGE_GROUP_TO_CONTINUOUS = {"1": 57, "2": 67, "3": 77, "4": 87, "5": 92}
+_UNKNOWN_AGE_GROUP_FALLBACK = 57
+
+
+def estimate_empirical_distribution(
+  series: pl.Series,
+) -> tuple[list[str | int | float], list[float]]:
+  """Estimate a discrete distribution from a prepared source series."""
+  values = series.to_list()
+  if not values:
+    raise ValueError(f"cannot sample from empty series '{series.name}'")
+
+  counts: dict[str | int | float, int] = {}
+  for value in values:
+    if value is None:
+      raise ValueError(f"series '{series.name}' contains null values")
+    counts[value] = counts.get(value, 0) + 1
+
+  total = float(len(values))
+  return list(counts.keys()), [count / total for count in counts.values()]
+
+
+def sample_empirical_distribution(
+  n_samples: int,
+  values: list[str | int | float] | tuple[str | int | float, ...],
+  probabilities: list[float] | tuple[float, ...],
+  seed: int,
+) -> np.ndarray:
+  """Sample from a persisted discrete empirical distribution."""
+  if not values:
+    raise ValueError("cannot sample from an empty empirical distribution")
+  if len(values) != len(probabilities):
+    raise ValueError("empirical distribution values/probabilities length mismatch")
+
+  rng = np.random.default_rng(seed=seed)
+  return rng.choice(np.asarray(values, dtype=object), size=n_samples, p=probabilities)
+
+
+def sample_empirical_values(
+  n_samples: int, series: pl.Series, seed: int
+) -> np.ndarray:
+  """Sample observed values with replacement from a prepared source series."""
+  values, probabilities = estimate_empirical_distribution(series)
+  return sample_empirical_distribution(n_samples, values, probabilities, seed)
+
+
+def summarize_truncated_normal_source(
+  series: pl.Series,
+  minimum: float | None = None,
+  maximum: float | None = None,
+) -> tuple[float, float, float, float]:
+  """Return observed moments and effective bounds for a source series."""
+  values = series.cast(pl.Float64).to_numpy()
+  if values.size == 0:
+    raise ValueError(f"cannot sample from empty series '{series.name}'")
+
+  observed_mean = float(np.mean(values))
+  observed_std = float(np.std(values, ddof=1)) if values.size > 1 else 0.0
+  lower = minimum if minimum is not None else float(np.min(values))
+  upper = maximum if maximum is not None else float(np.max(values))
+
+  if lower > upper:
+    raise ValueError(f"invalid bounds for '{series.name}': {lower} > {upper}")
+
+  return observed_mean, observed_std, lower, upper
+
+
+def sample_truncated_normal_from_summary(
+  n_samples: int,
+  observed_mean: float,
+  observed_std: float,
+  minimum: float,
+  maximum: float,
+  seed: int,
+  mean_shift: float = 0.0,
+  round_to_int: bool = False,
+) -> np.ndarray:
+  """Sample bounded numeric values from persisted source moments."""
+  shifted_mean = observed_mean + mean_shift
+  if observed_std == 0.0 or np.isnan(observed_std):
+    sampled = np.full(n_samples, np.clip(shifted_mean, minimum, maximum))
+  else:
+    a = (minimum - shifted_mean) / observed_std
+    b = (maximum - shifted_mean) / observed_std
+    sampled = truncnorm.rvs(
+      a,
+      b,
+      loc=shifted_mean,
+      scale=observed_std,
+      size=n_samples,
+      random_state=seed,
+    )
+
+  if round_to_int:
+    sampled = np.rint(sampled).astype(int)
+
+  return sampled
+
+
+def sample_truncated_normal_values(
+  n_samples: int,
+  series: pl.Series,
+  seed: int,
+  minimum: float | None = None,
+  maximum: float | None = None,
+  mean_shift: float = 0.0,
+  round_to_int: bool = False,
+) -> np.ndarray:
+  """Sample bounded numeric values using observed moments from a source series."""
+  observed_mean, observed_std, lower, upper = summarize_truncated_normal_source(
+    series,
+    minimum=minimum,
+    maximum=maximum,
+  )
+  return sample_truncated_normal_from_summary(
+    n_samples,
+    observed_mean,
+    observed_std,
+    lower,
+    upper,
+    seed,
+    mean_shift=mean_shift,
+    round_to_int=round_to_int,
+  )
+
+
+def apply_freq_shifts(
+  original_freqs: Mapping[str, float],
+  shifts: tuple[FreqShift, ...],
+  missing_category_handler: MissingCategoryHandler | None = None,
+) -> dict[str, float]:
+  """Apply category frequency shifts in percentage points."""
+  if not shifts:
+    return dict(original_freqs)
+
+  shifted_freqs = dict(original_freqs)
+
+  for shift in shifts:
+    if shift.source not in shifted_freqs:
+      if missing_category_handler:
+        missing_category_handler(
+          f"source category '{shift.source}' not found in distribution"
+        )
+      continue
+    if shift.dest not in shifted_freqs:
+      if missing_category_handler:
+        missing_category_handler(
+          f"destination category '{shift.dest}' not found in distribution"
+        )
+      continue
+
+    shift_amount = shift.shift_pct / 100.0
+    shifted_freqs[shift.source] -= shift_amount
+    shifted_freqs[shift.dest] += shift_amount
+
+  if any(freq < 0 for freq in shifted_freqs.values()):
+    negative_cats = [cat for cat, freq in shifted_freqs.items() if freq < 0]
+    raise ValueError(f"negative frequencies after shifts: {negative_cats}")
+
+  total = sum(shifted_freqs.values())
+  return {cat: freq / total for cat, freq in shifted_freqs.items()}
+
+
+def generate_continuous_age(
+  n_samples: int, original_df: pl.DataFrame, age_drift: float, seed: int
+) -> pl.DataFrame:
+  """Generate continuous age using a truncated normal distribution with drift."""
+  age_stats = original_df.select(
+    [
+      pl.col("age_cont").mean().alias("mean"),
+      pl.col("age_cont").std().alias("std"),
+      pl.col("age_cont").min().alias("min"),
+      pl.col("age_cont").max().alias("max"),
+    ]
+  ).row(0)
+
+  age_mean, age_std, age_min, age_max = age_stats
+  age_mean += age_drift
+
+  a = (age_min - age_mean) / age_std
+  b = (age_max - age_mean) / age_std
+
+  ages = truncnorm.rvs(
+    a, b, loc=age_mean, scale=age_std, size=n_samples, random_state=seed
+  )
+
+  return pl.DataFrame({"age_cont": ages.round().astype(int)}).with_columns(
+    [
+      pl.col("age_cont").cast(pl.Int8),
+      pl.col("age_cont")
+      .cut(breaks=_AGE_GROUP_BREAKS, labels=_AGE_GROUP_LABELS)
+      .alias("agegrp"),
+    ]
+  )
+
+
+def generate_categorical_start(
+  n_samples: int,
+  original_df: pl.DataFrame,
+  feature_name: str,
+  freq_shifts: tuple[FreqShift, ...],
+  seed: int,
+  missing_category_handler: MissingCategoryHandler | None = None,
+) -> pl.DataFrame:
+  """Generate a categorical starting feature with optional frequency shifts."""
+  rng = np.random.default_rng(seed=seed)
+
+  original_freqs = compute_category_frequencies(original_df[feature_name])
+  shifted_freqs = apply_freq_shifts(
+    original_freqs, freq_shifts, missing_category_handler
+  )
+
+  categories = list(shifted_freqs.keys())
+  probabilities = list(shifted_freqs.values())
+  sampled_values = rng.choice(categories, size=n_samples, p=probabilities)
+
+  df = pl.DataFrame({feature_name: sampled_values}).with_columns(
+    pl.col(feature_name).cast(pl.String).cast(pl.Categorical)
+  )
+
+  if feature_name == "agegrp":
+    df = df.with_columns(
+      pl.col("agegrp")
+      .map_elements(
+        lambda x: _AGE_GROUP_TO_CONTINUOUS.get(x, _UNKNOWN_AGE_GROUP_FALLBACK),
+        return_dtype=pl.Int8,
+      )
+      .alias("age_cont")
+    )
+
+  return df
+
+
+def generate_responses(
+  log_odds: np.ndarray, intercept: float, slope: float, seed: int
+) -> tuple[np.ndarray, np.ndarray]:
+  """Generate risk probabilities and binary outcomes from log-odds."""
+  rng = np.random.default_rng(seed=seed)
+
+  risk_probs = sigmoid(log_odds)
+  mean_risk_proba = risk_probs.mean()
+  mean_risk_log_odds = logit(mean_risk_proba)
+
+  response_log_odds = np.where(
+    log_odds >= mean_risk_log_odds,
+    intercept + slope * (log_odds - mean_risk_log_odds) + mean_risk_log_odds,
+    intercept + log_odds,
+  )
+  response_probs = sigmoid(response_log_odds)
+  outcomes = rng.binomial(n=1, p=response_probs)
+
+  return risk_probs, outcomes