oissyntheticdata 0.2.0__py3-none-any.whl

oissyntheticdata/__init__.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+"""
+oissyntheticdata — pure-Python sequential CART synthesis, in the synthpop tradition.
+
+Zero third-party dependencies (standard library only). Designed for secure
+research environments: develop and debug your analysis on the synthetic data
+off-site, then run the final code on the real data on-premises.
+
+Single table
+------------
+    import oissyntheticdata
+    oissyntheticdata.synthesize_file("real.csv", "synthetic.csv",
+                            drop=["national_id"], min_leaf=5)
+
+Related tables (referential integrity preserved)
+-------------------------------------------------
+    oissyntheticdata.synthesize_relational_files(
+        {"inmates": "inmates.csv", "judgements": "judgements.csv"},
+        schema={
+            "inmates":    {"key": "prisoner_id"},
+            "judgements": {"key": "judgement_id",
+                           "parent": "inmates", "foreign_key": "prisoner_id"},
+        },
+        out_dir="out", min_leaf=5)
+"""
+
+from ._io import read_table, write_table
+from ._synth import synthesize
+from ._relational import synthesize_relational, synthesize_relational_files
+
+__version__ = "0.2.0"
+__all__ = [
+    "read_table", "write_table", "synthesize", "synthesize_file",
+    "synthesize_relational", "synthesize_relational_files",
+]
+
+
+def synthesize_file(in_path, out_path, n=None, visit=None, drop=None,
+                    min_leaf=5, max_depth=12, smoothing=0.0, seed=12345):
+    """Read a CSV/XLSX, synthesize one flat table, and write a CSV."""
+    header, cols = read_table(in_path)
+    out_header, out_cols = synthesize(
+        header, cols, n=n, visit=visit, drop=drop,
+        min_leaf=min_leaf, max_depth=max_depth, smoothing=smoothing, seed=seed)
+    write_table(out_path, out_header, out_cols)
+    nrows = len(out_cols[out_header[0]]) if out_header else 0
+    return nrows, len(out_header)

oissyntheticdata/__main__.py

@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+"""Command-line interface:  python -m oissyntheticdata real.csv -o synthetic.csv"""
+
+import sys
+import argparse
+from . import synthesize_file, __version__
+
+
+def main(argv=None):
+    p = argparse.ArgumentParser(
+        prog="oissyntheticdata",
+        description="Pure-Python sequential CART synthesis (synthpop tradition, zero deps).")
+    p.add_argument("input", help="real CSV or XLSX file")
+    p.add_argument("-o", "--output", default="synthetic.csv", help="output CSV path")
+    p.add_argument("-n", "--rows", type=int, default=None, help="number of synthetic rows")
+    p.add_argument("--drop", default="", help="comma-separated columns to exclude (e.g. identifiers)")
+    p.add_argument("--visit", default="", help="comma-separated synthesis order (default: file order)")
+    p.add_argument("--min-leaf", type=int, default=5, help="minimum real records per leaf/cell (k)")
+    p.add_argument("--max-depth", type=int, default=12, help="maximum tree depth")
+    p.add_argument("--smoothing", type=float, default=0.0, help="continuous jitter (0 = off)")
+    p.add_argument("--seed", type=int, default=12345)
+    p.add_argument("--version", action="version", version="oissyntheticdata " + __version__)
+    a = p.parse_args(argv)
+
+    drop = [c.strip() for c in a.drop.split(",") if c.strip()]
+    visit = [c.strip() for c in a.visit.split(",") if c.strip()] or None
+    rows, cols = synthesize_file(a.input, a.output, n=a.rows, visit=visit, drop=drop,
+                                 min_leaf=a.min_leaf, max_depth=a.max_depth,
+                                 smoothing=a.smoothing, seed=a.seed)
+    sys.stderr.write("[oissyntheticdata] wrote %d rows x %d cols -> %s\n" % (rows, cols, a.output))
+
+
+if __name__ == "__main__":
+    main()

oissyntheticdata/_io.py

@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+"""oissyntheticdata._io — read CSV/XLSX and write CSV using ONLY the standard library."""
+
+import os
+import csv
+import zipfile
+import xml.etree.ElementTree as ET
+
+MISSING_TOKENS = {"", "na", "n/a", ".", "nan", "null", "none"}
+
+
+def _col_index(ref):
+    letters = "".join(ch for ch in ref if ch.isalpha())
+    n = 0
+    for ch in letters:
+        n = n * 26 + (ord(ch.upper()) - 64)
+    return n - 1
+
+
+def _read_xlsx(path):
+    ns = {"a": "http://schemas.openxmlformats.org/spreadsheetml/2006/main"}
+    T = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}t"
+    with zipfile.ZipFile(path) as z:
+        names = z.namelist()
+        shared = []
+        if "xl/sharedStrings.xml" in names:
+            root = ET.fromstring(z.read("xl/sharedStrings.xml"))
+            for si in root.findall("a:si", ns):
+                shared.append("".join(t.text or "" for t in si.iter(T)))
+        sheet = "xl/worksheets/sheet1.xml"
+        if sheet not in names:
+            sheet = sorted(n for n in names
+                           if n.startswith("xl/worksheets/") and n.endswith(".xml"))[0]
+        root = ET.fromstring(z.read(sheet))
+        rows = []
+        for row in root.iter("{%s}row" % ns["a"]):
+            cells, maxi = {}, -1
+            for c in row.findall("a:c", ns):
+                ref = c.get("r", "")
+                idx = _col_index(ref) if ref else len(cells)
+                t = c.get("t")
+                v = c.find("a:v", ns)
+                if t == "s" and v is not None:
+                    val = shared[int(v.text)]
+                elif t == "inlineStr":
+                    is_ = c.find("a:is", ns)
+                    val = "".join(x.text or "" for x in is_.iter(T)) if is_ is not None else ""
+                else:
+                    val = v.text if v is not None else ""
+                cells[idx] = val if val is not None else ""
+                maxi = max(maxi, idx)
+            rows.append([cells.get(i, "") for i in range(maxi + 1)])
+        return rows
+
+
+def _read_csv(path):
+    with open(path, "r", encoding="utf-8-sig", newline="") as f:
+        return [row for row in csv.reader(f)]
+
+
+def read_table(path):
+    """Return (header: list[str], columns: dict[str, list[str]])."""
+    raw = _read_xlsx(path) if path.lower().endswith((".xlsx", ".xls")) else _read_csv(path)
+    raw = [r for r in raw if any(str(c).strip() for c in r)]
+    if not raw:
+        return [], {}
+    header = [str(h).strip() for h in raw[0]]
+    cols = {h: [] for h in header}
+    for r in raw[1:]:
+        for i, h in enumerate(header):
+            cols[h].append(str(r[i]).strip() if i < len(r) else "")
+    return header, cols
+
+
+def write_table(path, header, columns):
+    n = len(columns[header[0]]) if header else 0
+    with open(path, "w", encoding="utf-8", newline="") as f:
+        w = csv.writer(f)
+        w.writerow(header)
+        for i in range(n):
+            w.writerow([columns[h][i] for h in header])
+
+
+def is_missing(v):
+    return str(v).strip().lower() in MISSING_TOKENS

oissyntheticdata/_relational.py

@@ -0,0 +1,188 @@
+# -*- coding: utf-8 -*-
+"""oissyntheticdata._relational — multi-table (relational) synthesis.
+
+Extends sequential CART synthesis to a parent -> child schema while keeping
+**referential integrity** (every synthetic foreign key points at a synthetic
+parent) and the **parent->child structure** (fan-out and attribute correlation).
+
+For each table, in parent-before-child order:
+  1. Synthesize the table's attributes (sequential CART). For a child, the
+     parent's synthetic attributes are supplied as fixed predictors, so child
+     attributes are drawn conditioned on the parent they belong to.
+  2. Mint fresh surrogate primary keys (1..n) — real identifiers are never
+     reproduced.
+  3. For each child, a regression CART models the number of children per parent
+     from the parent's attributes (the fan-out), so realistic counts — and which
+     parents have many vs. few children — are preserved. Foreign keys are drawn
+     from the synthetic parent keys, guaranteeing valid joins.
+
+Scope: a single-parent DAG (star / snowflake / chains). A table has at most one
+parent; a parent may have many children; children may themselves be parents.
+"""
+
+import random
+from . import _io
+from . import _tree
+from ._synth import type_columns, stringify, synth_core, _to_float
+
+
+def _topo_order(schema):
+    order, seen = [], set()
+    tables = list(schema.keys())
+    guard = 0
+    while len(order) < len(tables):
+        guard += 1
+        if guard > len(tables) + 2:
+            raise ValueError("Cyclic or unresolved parent reference in schema.")
+        for t in tables:
+            if t in seen:
+                continue
+            parent = schema[t].get("parent")
+            if parent is None or parent in seen:
+                order.append(t); seen.add(t)
+    return order
+
+
+def _child_counts(parent_keys_real, child_fk_real):
+    """counts[parent_key] = number of real child rows with that foreign key."""
+    counts = {}
+    for k in child_fk_real:
+        counts[k] = counts.get(k, 0) + 1
+    return [float(counts.get(pk, 0)) for pk in parent_keys_real]
+
+
+def synthesize_relational(tables, schema, n=None, drop=None,
+                          min_leaf=5, max_depth=12, smoothing=0.0, seed=12345):
+    """Synthesize a set of related tables.
+
+    tables : dict  table_name -> (header, columns)   [from oissyntheticdata.read_table]
+    schema : dict  table_name -> {"key": pk,
+                                   "parent": parent_table (optional),
+                                   "foreign_key": fk (required if parent set)}
+    n      : dict table_name -> rows for ROOT tables (children sized by fan-out);
+             a single int applies to all roots; None = each root's real row count.
+    drop   : dict table_name -> [columns to exclude] (besides keys), or a flat
+             list applied to every table.
+
+    Returns dict table_name -> (out_header, out_columns).
+    """
+    rng = random.Random(seed)
+    drop = drop or {}
+    if isinstance(drop, (list, tuple, set)):
+        drop = {t: list(drop) for t in tables}
+    n_map = {} if n is None else (n if isinstance(n, dict) else {t: n for t in tables})
+
+    synth_attr = {}   # table -> typed dict of synthesized attribute columns
+    synth_key  = {}   # table -> list of surrogate pk strings
+    is_num_of  = {}   # table -> is_num map for its attributes
+    results    = {}
+
+    for t in _topo_order(schema):
+        header, cols = tables[t]
+        spec = schema[t]
+        pk = spec["key"]
+        parent = spec.get("parent")
+        fk = spec.get("foreign_key")
+        drop_t = set(drop.get(t, [])) | {pk}
+        if fk:
+            drop_t.add(fk)
+        attrs = [c for c in header if c not in drop_t]
+        is_num, real = type_columns(cols, attrs)
+        is_num_of[t] = is_num
+        n_real = len(cols[header[0]])
+
+        if parent is None:
+            # -------- root table --------
+            n_t = int(n_map.get(t, n_real))
+            out = synth_core(real, is_num, attrs, n_t, rng,
+                             min_leaf=min_leaf, max_depth=max_depth, smoothing=smoothing)
+            synth_attr[t] = out
+            synth_key[t] = [str(i + 1) for i in range(n_t)]
+        else:
+            # -------- child table --------
+            p_attrs = list(synth_attr[parent].keys())
+            p_is_num = is_num_of[parent]
+            # real parent attributes, typed + a key->row lookup
+            p_header, p_cols = tables[parent]
+            _, p_real = type_columns(p_cols, p_attrs)
+            p_pk_real = p_cols[schema[parent]["key"]]
+            lookup = {k: i for i, k in enumerate(p_pk_real)}
+            # fan-out: counts of real children per real parent
+            counts = _child_counts(p_pk_real, cols[fk])
+
+            # count model: counts ~ parent attributes (regression CART)
+            n_parent = len(synth_key[parent])
+            drawn = []
+            if p_attrs:
+                _tree.set_predictors({a: p_real[a] for a in p_attrs})
+                croot = _tree.build_tree(list(range(len(counts))), counts, p_attrs,
+                                         p_is_num, "num", min_leaf=min_leaf, max_depth=max_depth)
+                for i in range(n_parent):
+                    row = {a: synth_attr[parent][a][i] for a in p_attrs}
+                    c = _tree.sample_leaf(croot, row, p_is_num, rng, 0.0)
+                    drawn.append(max(0, int(round(c if c is not None else 0))))
+            else:
+                pos = [c for c in counts]
+                for _ in range(n_parent):
+                    drawn.append(max(0, int(round(rng.choice(pos)))))
+
+            # expand: child foreign keys + the parent attrs carried to each child row
+            child_fk, parent_carry = [], {("p__" + a): [] for a in p_attrs}
+            for i in range(n_parent):
+                key_i = synth_key[parent][i]
+                for _ in range(drawn[i]):
+                    child_fk.append(key_i)
+                    for a in p_attrs:
+                        parent_carry["p__" + a].append(synth_attr[parent][a][i])
+            total = len(child_fk)
+
+            # real fitting data, restricted to child rows whose parent exists
+            valid = [j for j in range(n_real) if cols[fk][j] in lookup]
+            child_real = {a: [real[a][j] for j in valid] for a in attrs}
+            fixed_real = {("p__" + a): [p_real[a][lookup[cols[fk][j]]] for j in valid] for a in p_attrs}
+            combined_is_num = dict(is_num)
+            for a in p_attrs:
+                combined_is_num["p__" + a] = p_is_num[a]
+
+            out = synth_core(child_real, combined_is_num, attrs, total, rng,
+                             fixed_real=fixed_real, fixed_synth=parent_carry,
+                             min_leaf=min_leaf, max_depth=max_depth, smoothing=smoothing)
+            synth_attr[t] = out
+            synth_key[t] = [str(i + 1) for i in range(total)]
+            results[t] = ("__child__", child_fk)   # stash fk for assembly
+
+        # ---- assemble this table's output in original column order ----
+        out_header = [c for c in header if c not in (set(drop.get(t, [])))]
+        # keep pk and fk columns in output even though they aren't "attrs"
+        out_cols = {}
+        strattr = stringify(synth_attr[t], list(synth_attr[t].keys()))
+        nrows = len(synth_key[t])
+        for c in out_header:
+            if c == pk:
+                out_cols[c] = list(synth_key[t])
+            elif parent and c == fk:
+                out_cols[c] = list(results[t][1])
+            elif c in strattr:
+                out_cols[c] = strattr[c]
+            else:
+                out_cols[c] = [""] * nrows
+        results[t] = (out_header, out_cols)
+
+    return results
+
+
+def synthesize_relational_files(paths, schema, out_dir=".", **kw):
+    """Read CSV/XLSX tables, synthesize relationally, write synthetic_<name>.csv.
+
+    paths : dict table_name -> input file path
+    Returns dict table_name -> (rows, cols).
+    """
+    import os
+    tables = {t: _io.read_table(p) for t, p in paths.items()}
+    res = synthesize_relational(tables, schema, **kw)
+    summary = {}
+    for t, (hdr, cols) in res.items():
+        out_path = os.path.join(out_dir, "synthetic_%s.csv" % t)
+        _io.write_table(out_path, hdr, cols)
+        summary[t] = (len(cols[hdr[0]]) if hdr else 0, len(hdr))
+    return summary

oissyntheticdata/_synth.py

@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+"""oissyntheticdata._synth — sequential CART synthesis (the synthpop paradigm).
+
+Columns are synthesized one at a time in `visit` order. The first column is
+drawn from its own (disclosure-controlled) marginal. Each later column is
+synthesized by growing a CART that predicts it from the columns ALREADY
+synthesized, fitted on the real data, then drawing a donor from the matching
+leaf for every synthetic row. Because predictors at draw time are the
+synthetic values, the joint distribution is built up sequentially.
+
+Confidentiality:
+  * `min_leaf` (k): no leaf / no marginal cell is built from fewer than k real
+    records, so a drawn value never isolates one person.
+  * `smoothing`: optional jitter on continuous donors so exact real values are
+    not echoed verbatim.
+  * direct identifiers should be dropped before synthesis (see `drop` arg).
+"""
+
+import random
+from . import _io
+from . import _tree
+
+
+def _is_numeric(values):
+    present = [v for v in values if not _io.is_missing(v)]
+    if not present:
+        return False
+    for v in present:
+        try:
+            float(str(v).replace(",", ""))
+        except ValueError:
+            return False
+    return True
+
+
+def _to_float(v):
+    try:
+        return float(str(v).replace(",", ""))
+    except (ValueError, TypeError):
+        return None
+
+
+def _marginal_draw(values, n, min_leaf, rng):
+    """Sample n values from the empirical marginal, suppressing rare cells."""
+    counts = {}
+    for v in values:
+        counts[v] = counts.get(v, 0) + 1
+    pool = [v for v in values if counts[v] >= min_leaf]
+    if not pool:                       # everything rare -> fall back to all
+        pool = list(values)
+    return [rng.choice(pool) for _ in range(n)]
+
+
+def type_columns(columns, names):
+    """Return (is_num, typed) for the given column names.
+    Numeric columns become floats (missing -> None); others become strings."""
+    is_num, typed = {}, {}
+    for c in names:
+        numeric = _is_numeric(columns[c])
+        is_num[c] = numeric
+        if numeric:
+            typed[c] = [(_to_float(v) if not _io.is_missing(v) else None) for v in columns[c]]
+        else:
+            typed[c] = [("" if _io.is_missing(v) else str(v)) for v in columns[c]]
+    return is_num, typed
+
+
+def stringify(typed, names):
+    """Turn typed synthetic columns back into CSV-ready strings."""
+    out = {}
+    for c in names:
+        vals = []
+        for v in typed[c]:
+            if v is None:
+                vals.append("")
+            elif isinstance(v, float):
+                vals.append(str(int(v)) if v.is_integer() else ("%.6g" % v))
+            else:
+                vals.append(str(v))
+        out[c] = vals
+    return out
+
+
+def synth_core(real, is_num, visit, n, rng, fixed_real=None, fixed_synth=None,
+               min_leaf=5, max_depth=12, smoothing=0.0):
+    """Sequentially synthesize the `visit` columns and return typed output.
+
+    real        : dict name->typed list (real data, for fitting)
+    is_num      : dict name->bool covering every visited AND fixed column
+    fixed_real  : dict name->typed list aligned to real rows — predictors that are
+                  GIVEN, not synthesized (e.g. a child row's parent attributes).
+    fixed_synth : dict name->typed list aligned to the n synthetic rows — the
+                  given predictor values for each synthetic row.
+    """
+    fixed_real = fixed_real or {}
+    fixed_synth = fixed_synth or {}
+    fixed_names = list(fixed_real.keys())
+    if visit:
+        n_real = len(real[visit[0]])
+    elif fixed_real:
+        n_real = len(next(iter(fixed_real.values())))
+    else:
+        n_real = 0
+
+    out = {c: [] for c in visit}
+    done = []
+    for c in visit:
+        target_kind = "num" if is_num[c] else "cat"
+        preds = fixed_names + done
+        if not preds:
+            donors = [v for v in real[c] if v is not None] if is_num[c] else real[c]
+            out[c] = _marginal_draw(donors if donors else real[c], n, min_leaf, rng)
+        else:
+            _tree.set_predictors({p: (fixed_real[p] if p in fixed_real else real[p]) for p in preds})
+            idx = list(range(n_real))
+            root = _tree.build_tree(idx, real[c], preds, is_num, target_kind,
+                                    min_leaf=min_leaf, max_depth=max_depth)
+            col_out = []
+            for i in range(n):
+                row = {}
+                for p in preds:
+                    row[p] = fixed_synth[p][i] if p in fixed_synth else out[p][i]
+                col_out.append(_tree.sample_leaf(root, row, is_num, rng, smoothing))
+            out[c] = col_out
+        done.append(c)
+    return out
+
+
+def synthesize(header, columns, n=None, visit=None, drop=None,
+               min_leaf=5, max_depth=12, smoothing=0.0, seed=12345):
+    """Return (out_header, out_columns) of synthetic data for one flat table."""
+    rng = random.Random(seed)
+    drop = set(drop or [])
+    visit = [c for c in (visit or header) if c in columns and c not in drop]
+    n_real = len(columns[header[0]]) if header else 0
+    n = n_real if n is None else n
+    is_num, real = type_columns(columns, visit)
+    out = synth_core(real, is_num, visit, n, rng,
+                     min_leaf=min_leaf, max_depth=max_depth, smoothing=smoothing)
+    return visit, stringify(out, visit)

oissyntheticdata/_tree.py

@@ -0,0 +1,163 @@
+# -*- coding: utf-8 -*-
+"""oissyntheticdata._tree — a small CART grown from scratch (no numpy/sklearn).
+
+Each leaf keeps the list of REAL target values that reached it ("donors").
+Synthesis samples from a leaf's donors rather than predicting a point value,
+which reproduces the conditional distribution (Reiter, 2005). A minimum leaf
+size (`min_leaf`) guarantees every donor pool blends >= k real records, so a
+synthetic value is never traceable to one individual.
+"""
+
+import math
+import random
+
+MAX_THRESHOLDS = 40   # cap numeric split candidates for speed
+
+
+def _gini(counts, n):
+    if n == 0:
+        return 0.0
+    s = 0.0
+    for c in counts.values():
+        p = c / n
+        s += p * p
+    return 1.0 - s
+
+
+def _sse(values):
+    n = len(values)
+    if n == 0:
+        return 0.0
+    m = sum(values) / n
+    return sum((v - m) ** 2 for v in values)
+
+
+def _candidate_thresholds(vals):
+    uniq = sorted(set(vals))
+    if len(uniq) <= 1:
+        return []
+    if len(uniq) <= MAX_THRESHOLDS:
+        cuts = uniq
+    else:
+        step = len(uniq) / float(MAX_THRESHOLDS)
+        cuts = [uniq[int(i * step)] for i in range(1, MAX_THRESHOLDS)]
+    mids = []
+    for i in range(1, len(cuts)):
+        mids.append((cuts[i - 1] + cuts[i]) / 2.0)
+    return mids
+
+
+class _Node(object):
+    __slots__ = ("leaf", "donors", "feature", "kind", "threshold", "category",
+                 "left", "right")
+
+    def __init__(self):
+        self.leaf = False
+        self.donors = None
+        self.feature = None
+        self.kind = None          # 'num' or 'cat'
+        self.threshold = None
+        self.category = None
+        self.left = None
+        self.right = None
+
+
+def _impurity(idx, y, target_kind):
+    if target_kind == "cat":
+        counts = {}
+        for i in idx:
+            counts[y[i]] = counts.get(y[i], 0) + 1
+        return _gini(counts, len(idx)) * len(idx)
+    else:
+        return _sse([y[i] for i in idx])
+
+
+def build_tree(idx, y, predictors, num_cols, target_kind,
+               min_leaf=5, max_depth=12, depth=0):
+    """idx: row indices; y: target list; predictors: list of feature names;
+    num_cols: dict name->bool (True if numeric predictor)."""
+    node = _Node()
+    distinct_y = set(y[i] for i in idx)
+    if (len(idx) < 2 * min_leaf or depth >= max_depth or len(distinct_y) <= 1
+            or not predictors):
+        node.leaf = True
+        node.donors = [y[i] for i in idx]
+        return node
+
+    base = _impurity(idx, y, target_kind)
+    best = None  # (reduction, feature, kind, thr/cat, left_idx, right_idx)
+
+    for f in predictors:
+        is_num = num_cols[f]
+        if is_num:
+            colvals = [PRED[f][i] for i in idx]
+            present = [v for v in colvals if v is not None]
+            for thr in _candidate_thresholds(present):
+                left = [i for i in idx if PRED[f][i] is not None and PRED[f][i] <= thr]
+                right = [i for i in idx if not (PRED[f][i] is not None and PRED[f][i] <= thr)]
+                if len(left) < min_leaf or len(right) < min_leaf:
+                    continue
+                red = base - _impurity(left, y, target_kind) - _impurity(right, y, target_kind)
+                if best is None or red > best[0]:
+                    best = (red, f, "num", thr, left, right)
+        else:
+            cats = set(PRED[f][i] for i in idx)
+            if len(cats) <= 1:
+                continue
+            for c in cats:
+                left = [i for i in idx if PRED[f][i] == c]
+                right = [i for i in idx if PRED[f][i] != c]
+                if len(left) < min_leaf or len(right) < min_leaf:
+                    continue
+                red = base - _impurity(left, y, target_kind) - _impurity(right, y, target_kind)
+                if best is None or red > best[0]:
+                    best = (red, f, "cat", c, left, right)
+
+    if best is None or best[0] <= 1e-12:
+        node.leaf = True
+        node.donors = [y[i] for i in idx]
+        return node
+
+    _, f, kind, val, left_idx, right_idx = best
+    node.feature, node.kind = f, kind
+    if kind == "num":
+        node.threshold = val
+    else:
+        node.category = val
+    node.left = build_tree(left_idx, y, predictors, num_cols, target_kind,
+                           min_leaf, max_depth, depth + 1)
+    node.right = build_tree(right_idx, y, predictors, num_cols, target_kind,
+                            min_leaf, max_depth, depth + 1)
+    return node
+
+
+# PRED is module-level so the recursive builder can read predictor columns by
+# row index without copying. set_predictors() installs it for one fit.
+PRED = {}
+
+
+def set_predictors(pred_cols):
+    global PRED
+    PRED = pred_cols
+
+
+def sample_leaf(node, row, num_cols, rng, smoothing=0.0):
+    """Route a synthetic row (dict feature->value) to a leaf and draw a donor."""
+    while not node.leaf:
+        if node.kind == "num":
+            v = row.get(node.feature)
+            go_left = (v is not None and v <= node.threshold)
+        else:
+            go_left = (row.get(node.feature) == node.category)
+        node = node.left if go_left else node.right
+    donors = node.donors
+    val = rng.choice(donors)
+    if smoothing and val is not None and isinstance(val, float):
+        nums = [d for d in donors if isinstance(d, float)]
+        if len(nums) > 2:
+            m = sum(nums) / len(nums)
+            sd = math.sqrt(sum((x - m) ** 2 for x in nums) / len(nums))
+            if sd > 0:
+                lo, hi = min(nums), max(nums)
+                val = min(hi, max(lo, val + rng.gauss(0.0, smoothing * sd)))
+    return val

oissyntheticdata-0.2.0.dist-info/METADATA

@@ -0,0 +1,297 @@
+Metadata-Version: 2.4
+Name: oissyntheticdata
+Version: 0.2.0
+Summary: Zero-dependency sequential CART synthesis for secure research (synthpop tradition), with relational support. An OIS tool.
+Author-email: Yohanan Ouaknine <yohanan.ouaknine@ois.co.il>
+Maintainer-email: OIS <yohanan.ouaknine@ois.co.il>
+License: MIT License
+        
+        Copyright (c) 2026 Yohanan Ouaknine and OIS (https://ois.co.il)
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://ois.co.il
+Project-URL: Repository, https://github.com/yohananouaknine/oissyntheticdata
+Project-URL: Issues, https://github.com/yohananouaknine/oissyntheticdata/issues
+Keywords: synthetic data,synthpop,statistical disclosure control,CART,privacy,secure research,microdata,anonymization
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Classifier: Topic :: Security
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# oissyntheticdata
+
+**Pure-Python sequential CART synthesis — in the `synthpop` tradition, with zero third-party dependencies.**
+
+> An **OIS** tool · [ois.co.il](https://ois.co.il) · maintained by Dr Yohanan Ouaknine
+> ([ORCID 0000-0002-4186-7351](https://orcid.org/0000-0002-4186-7351))
+
+`oissyntheticdata` generates a synthetic copy of a sensitive dataset that preserves the
+*relationships between variables*, not just each column's marginal shape. It is
+built for the secure-research workflow used by statistical agencies: **develop
+and debug your analysis on the synthetic data off-site, then run the final code
+on the real data on-premises and release only vetted aggregate results.**
+
+It imports only the Python standard library (`csv`, `json`, `math`, `random`,
+`statistics`, `zipfile`, `xml.etree`), so it can run inside a locked secure
+environment with no `pip install` and is small enough to read and audit in full.
+
+The approach was first deployed in a secure justice-research setting (a study of
+terrorist recidivism after the 2011 Shalit prisoner exchange, run on-premises at
+the Israel Prison Service under Research Committee authorization); this package
+generalises and opens it. OIS offers deployment, validation, and training services
+to government research units and academic researchers around the open core.
+
+---
+
+## Why this exists
+
+This follows a well-established paradigm in statistical disclosure control. The
+synthetic data is *test data* that should resemble the real data closely but is
+never used for final inference; the code developed on it is what gets run on the
+confidential data (Nowok, Raab & Dibben 2016; US Census Bureau SIPP Synthetic
+Beta). `oissyntheticdata` is a dependency-free re-implementation of the core engine those
+tools use — **sequential CART synthesis** (Reiter 2005) — packaged for locked
+environments.
+
+It complements a metadata-only synthesizer (which preserves each column's shape
+but not the joint structure): `oissyntheticdata` fits on the real microdata on-premises
+and therefore reproduces conditional relationships, at the cost of touching raw
+records (so it must run inside the secure environment).
+
+---
+
+## How it works (the engine)
+
+Synthesis proceeds **one column at a time** in a chosen visit order:
+
+1. **First column** — drawn from its own empirical marginal, with cells smaller
+   than `min_leaf` suppressed.
+2. **Each later column `Y`** — a CART (classification tree if `Y` is categorical,
+   regression tree if continuous) is grown on the **real data** to predict `Y`
+   from the columns already synthesized. Every leaf keeps the list of *real*
+   `Y` values that reached it (its "donors").
+3. **Drawing** — for each synthetic row, route it down the tree using the values
+   already generated for that row, reach a leaf, and **sample a donor** from that
+   leaf (optionally jittered for continuous columns). Sampling from donors — not
+   predicting a point — is what reproduces the conditional distribution.
+
+Because each column is predicted from the previously synthesized columns, the
+joint distribution is assembled sequentially (the standard `synthpop` approach).
+
+```
+visit:  c1 -> c2 -> c3 -> ...
+c1 ~ marginal(c1)
+c2 ~ leaf_donor( CART(c2 ~ c1) , synthetic c1 )
+c3 ~ leaf_donor( CART(c3 ~ c1,c2) , synthetic c1,c2 )
+...
+```
+
+---
+
+## Confidentiality model
+
+- **`min_leaf` (k, default 5):** no leaf and no marginal cell is built from fewer
+  than `k` real records, so every drawn value blends ≥ k individuals and is never
+  traceable to one person. This also caps tree depth and prevents the tree from
+  memorizing individuals.
+- **`smoothing` (default 0):** optional Gaussian jitter on continuous donors,
+  bounded to the leaf's range, so exact real values are not echoed verbatim.
+- **`drop`:** direct identifiers (national ID, names, record keys) should be
+  dropped before synthesis — `oissyntheticdata` does not attempt to anonymize them.
+- **Only synthetic data leaves; the real data never does.** The intended use is
+  to take the synthetic file off-site for development and re-run final code on the
+  real data in place.
+
+`oissyntheticdata` is a disclosure-control aid, not a formal privacy guarantee. For a
+mathematical guarantee, combine it with differential privacy or apply output
+checking (statistical disclosure control) to anything released.
+
+---
+
+## Design decisions and trade-offs
+
+The value of `oissyntheticdata` is in its design choices, which are deliberately narrow:
+
+- **Where the synthesizer may run is a first-class concern.** `oissyntheticdata` fits on
+  real microdata to preserve joint structure, so it runs *on-premises*; only the
+  synthetic output leaves. A metadata-only synthesizer can run off-site but
+  preserves only per-column structure. Choosing fidelity-with-on-prem-execution
+  over portability-with-lower-fidelity is intentional, and the two roles are kept
+  as separate tools so the confidentiality reasoning stays explicit.
+- **Donor-leaf sampling, not point prediction.** Drawing a real value from the
+  matching leaf reproduces the conditional distribution; predicting a mean would
+  not.
+- **One confidentiality invariant.** `min_leaf` (`k`) applies the same `k`-record
+  floor to every marginal cell, tree leaf, fan-out estimate, and surrogate key,
+  instead of scattering ad hoc thresholds.
+- **Relational by conditioning, not joining.** Children are synthesized
+  conditioned on the parent's synthetic attributes and linked by surrogate keys,
+  preserving referential integrity without materialising a real join.
+- **Build on, don't reinvent.** The estimator is the established CART-synthesis
+  method; the new work is the dependency-free, auditable, relational realisation
+  for locked environments.
+
+Scope boundaries are equally deliberate: single-parent schemas only, no enforced
+high-order interactions or arithmetic identities, and no formal privacy guarantee
+(see Limitations).
+
+## Governance, support & contributing
+
+`oissyntheticdata` is maintained in the open under the MIT license. Questions, bug reports,
+and change proposals go through public GitHub Issues and Pull Requests; see
+[`CONTRIBUTING.md`](CONTRIBUTING.md). Decisions are made by the maintainer(s)
+listed in [`CITATION.cff`](CITATION.cff) via the public issue/PR process. There is
+no private support channel — keeping development and discussion public is part of
+the project's auditability goal. Releases are versioned and recorded in
+[`CHANGELOG.md`](CHANGELOG.md).
+
+## Generative AI disclosure
+
+A generative AI assistant (Claude, Anthropic) was used to help draft and refactor
+parts of the code and documentation. All output was reviewed, tested, and edited
+by the author(s), who take full responsibility for the design, correctness, and
+integrity of the software. The design decisions and abstractions above, and the
+testing and documentation practices, are the author(s)' own. Contributors are
+asked to disclose non-trivial AI assistance (see `CONTRIBUTING.md`).
+
+## Install
+
+```bash
+pip install oissyntheticdata          # once published
+# or, in a locked environment, just copy the oissyntheticdata/ folder next to your code
+```
+
+No dependencies. Python 3.7+.
+
+## Usage
+
+Command line:
+
+```bash
+python -m oissyntheticdata real.csv -o synthetic.csv --drop national_id --min-leaf 5
+python -m oissyntheticdata data.xlsx -o synthetic.csv --visit "age,offense,violent" --smoothing 0.5
+```
+
+Library:
+
+```python
+import oissyntheticdata
+
+# one call
+oissyntheticdata.synthesize_file("real.csv", "synthetic.csv",
+                        drop=["national_id"], min_leaf=5)
+
+# or step by step
+header, cols = oissyntheticdata.read_table("real.xlsx")
+out_header, out_cols = oissyntheticdata.synthesize(header, cols,
+                                          drop=["national_id"], min_leaf=5)
+oissyntheticdata.write_table("synthetic.csv", out_header, out_cols)
+```
+
+Key parameters: `n` (rows, default = real), `visit` (column order),
+`drop` (identifiers to exclude), `min_leaf` (k), `max_depth`, `smoothing`, `seed`.
+
+### Related tables (multi-table synthesis)
+
+For data split across linked tables (e.g. one row per inmate, many judgements per
+inmate), `synthesize_relational` keeps **referential integrity** and the
+**parent → child structure**:
+
+```python
+import oissyntheticdata
+
+oissyntheticdata.synthesize_relational_files(
+    {"inmates": "inmates.csv", "judgements": "judgements.csv"},
+    schema={
+        "inmates":    {"key": "prisoner_id"},
+        "judgements": {"key": "judgement_id",
+                       "parent": "inmates", "foreign_key": "prisoner_id"},
+    },
+    out_dir="out", min_leaf=5,
+)
+# -> out/synthetic_inmates.csv, out/synthetic_judgements.csv
+```
+
+How it works: the parent is synthesized first and given fresh surrogate keys; a
+regression CART models how many children each parent has (the fan-out) from the
+parent's attributes; and each child's attributes are synthesized **conditioned on
+its parent's synthetic attributes**. The result: every synthetic foreign key
+points at a synthetic parent (0 orphan joins), the number of children per parent
+is realistic, and parent → child relationships survive (e.g. high-risk parents
+keep their child-row patterns). Supports a single-parent DAG — star, snowflake,
+and parent → child → grandchild chains.
+
+---
+
+## Limitations
+
+- Fits on real microdata, so **run it on-premises**; the synthetic *output* is
+  what you take off-site.
+- Relational synthesis covers a single-parent DAG (star / snowflake / chains).
+  Many-to-many relationships and compound keys are not modelled — pre-join or
+  pre-resolve them to a surrogate key first.
+- CART captures pairwise/low-order structure well; very high-order interactions
+  and exact arithmetic identities (e.g. `rate = a/b`) are not enforced.
+- Pure Python: comfortable to a few thousand rows × a few dozen columns; larger
+  data is slower than a compiled implementation.
+
+---
+
+## Lineage & sources
+
+- Rubin, D.B. (1993). *Statistical disclosure limitation.* J. Official Statistics 9(2).
+- Little, R.J.A. (1993). *Statistical analysis of masked data.* J. Official Statistics 9(2).
+- Reiter, J.P. (2005). *Using CART to generate partially synthetic public use microdata.*
+  J. Official Statistics 21(3).
+- Reiter, Oganian & Karr (2009). *Verification servers.* Comput. Stat. Data Anal. 53(4):1475–1482.
+  https://doi.org/10.1016/j.csda.2008.10.006
+- Nowok, Raab & Dibben (2016). *synthpop: Bespoke Creation of Synthetic Data in R.*
+  J. Statistical Software 74(11). https://doi.org/10.18637/jss.v074.i11
+- Drechsler, J. (2011). *Synthetic Datasets for Statistical Disclosure Control.* Springer.
+- US Census Bureau, *SIPP Synthetic Beta* + Cornell Synthetic Data Server (synthetic
+  development data + validation on confidential files).
+
+## Maintainer
+
+Dr **Yohanan Ouaknine** — OIS ([ois.co.il](https://ois.co.il)),
+[yohanan.ouaknine@ois.co.il](mailto:yohanan.ouaknine@ois.co.il),
+[ORCID 0000-0002-4186-7351](https://orcid.org/0000-0002-4186-7351).
+Department of Criminology, Ashkelon Academic College; formerly Head of the
+Research Branch, Israel Prison Service.
+
+## License
+
+MIT — see `LICENSE`.
+
+## Citation
+
+If you use `oissyntheticdata`, please cite this software (see `CITATION.cff`) and
+the methodological lineage above (Reiter 2005; Nowok, Raab & Dibben 2016). The
+method was first applied in Ouaknine, Elisha & Hasisi (2026), *The Effect of Mass
+Prisoner Release on Terrorist Recidivism: A Propensity Score Analysis of the Shalit
+Deal* (in publication).

oissyntheticdata-0.2.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+oissyntheticdata/__init__.py,sha256=jBKFw02tnTZE8_CQfO8vxxsEcLujtwyrGbdmpqrF_EQ,1861
+oissyntheticdata/__main__.py,sha256=7IP9flqUD4hlc97VAXW28XaI0KM8IfKI8xUhB5p1fOw,1767
+oissyntheticdata/_io.py,sha256=Qd-gLzyn9-6KvyvHDPs_fjij9kR5I3tdU6y0kEmePJI,3029
+oissyntheticdata/_relational.py,sha256=YZFTj2ixyinpcagF5yEygddKnqlqvSZLWR3Xs-UGxlo,8296
+oissyntheticdata/_synth.py,sha256=WXpJkHv7aGohnLCWHV9Ct9MhvrZVjiUOOj_NpgSlK2U,5321
+oissyntheticdata/_tree.py,sha256=vOePQ7xosqOpgG-_k6Op68O5nEHsfymyugMHlhv3cFg,5566
+oissyntheticdata-0.2.0.dist-info/licenses/LICENSE,sha256=VRnrt9oHAA6oSVzz6w_OzRBg2WnvjsjgCENdpx9SiOk,1101
+oissyntheticdata-0.2.0.dist-info/METADATA,sha256=lmhbOCO8004uHrGgf71rURCaJtvu7ZuqyzUEReSGWFk,14200
+oissyntheticdata-0.2.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+oissyntheticdata-0.2.0.dist-info/entry_points.txt,sha256=hrXxwSv1b0C2gUhfVO-68zmh5Ot8oFmazt749kzG8Pw,107
+oissyntheticdata-0.2.0.dist-info/top_level.txt,sha256=TjXJ9Cnt9DmspFooICOHMq1yNHg4ywh7v0UyX5N-JY4,17
+oissyntheticdata-0.2.0.dist-info/RECORD,,

oissyntheticdata-0.2.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (82.0.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

oissyntheticdata-0.2.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+oissd = oissyntheticdata.__main__:main
+oissyntheticdata = oissyntheticdata.__main__:main

oissyntheticdata-0.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Yohanan Ouaknine and OIS (https://ois.co.il)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

oissyntheticdata-0.2.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+oissyntheticdata