synthschema 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- synthschema/__init__.py +56 -0
- synthschema/_version.py +1 -0
- synthschema/cli.py +134 -0
- synthschema/constraints.py +438 -0
- synthschema/copula.py +79 -0
- synthschema/datasets.py +106 -0
- synthschema/evaluation.py +446 -0
- synthschema/generators.py +456 -0
- synthschema/marginals.py +143 -0
- synthschema/metadata.py +547 -0
- synthschema/multi_table.py +272 -0
- synthschema/py.typed +0 -0
- synthschema/schema.py +490 -0
- synthschema/single_table.py +311 -0
- synthschema/transformers.py +247 -0
- synthschema-0.1.0.dist-info/METADATA +253 -0
- synthschema-0.1.0.dist-info/RECORD +20 -0
- synthschema-0.1.0.dist-info/WHEEL +4 -0
- synthschema-0.1.0.dist-info/entry_points.txt +2 -0
- synthschema-0.1.0.dist-info/licenses/LICENSE +201 -0
synthschema/__init__.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""synthschema: open-source, schema-driven synthetic data generation.
|
|
2
|
+
|
|
3
|
+
Quick start::
|
|
4
|
+
|
|
5
|
+
from synthschema import GaussianCopulaSynthesizer, evaluate_quality
|
|
6
|
+
|
|
7
|
+
synth = GaussianCopulaSynthesizer()
|
|
8
|
+
synth.fit(real_df)
|
|
9
|
+
fake_df = synth.sample(1_000, seed=42)
|
|
10
|
+
print(evaluate_quality(real_df, fake_df))
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from ._version import __version__
|
|
14
|
+
from .constraints import Custom, FixedCombinations, Inequality, OneOf, Range, parse_rule
|
|
15
|
+
from .datasets import load_demo
|
|
16
|
+
from .evaluation import (
|
|
17
|
+
Certificate,
|
|
18
|
+
DiagnosticReport,
|
|
19
|
+
PrivacyReport,
|
|
20
|
+
QualityReport,
|
|
21
|
+
certify,
|
|
22
|
+
evaluate_privacy,
|
|
23
|
+
evaluate_quality,
|
|
24
|
+
run_diagnostic,
|
|
25
|
+
)
|
|
26
|
+
from .metadata import Column, Metadata, Relationship, Table
|
|
27
|
+
from .multi_table import MultiTableSynthesizer
|
|
28
|
+
from .schema import SchemaSynthesizer, parse_ddl
|
|
29
|
+
from .single_table import GaussianCopulaSynthesizer
|
|
30
|
+
|
|
31
|
+
__all__ = [
|
|
32
|
+
"__version__",
|
|
33
|
+
"Metadata",
|
|
34
|
+
"Table",
|
|
35
|
+
"Column",
|
|
36
|
+
"Relationship",
|
|
37
|
+
"GaussianCopulaSynthesizer",
|
|
38
|
+
"MultiTableSynthesizer",
|
|
39
|
+
"SchemaSynthesizer",
|
|
40
|
+
"parse_ddl",
|
|
41
|
+
"Inequality",
|
|
42
|
+
"Range",
|
|
43
|
+
"OneOf",
|
|
44
|
+
"FixedCombinations",
|
|
45
|
+
"Custom",
|
|
46
|
+
"parse_rule",
|
|
47
|
+
"evaluate_quality",
|
|
48
|
+
"run_diagnostic",
|
|
49
|
+
"evaluate_privacy",
|
|
50
|
+
"certify",
|
|
51
|
+
"QualityReport",
|
|
52
|
+
"DiagnosticReport",
|
|
53
|
+
"PrivacyReport",
|
|
54
|
+
"Certificate",
|
|
55
|
+
"load_demo",
|
|
56
|
+
]
|
synthschema/_version.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
synthschema/cli.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Command-line interface: ``synthschema --help``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
import pandas as pd
|
|
11
|
+
|
|
12
|
+
from ._version import __version__
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _read(path: str) -> pd.DataFrame:
|
|
16
|
+
p = Path(path)
|
|
17
|
+
if p.suffix.lower() == ".parquet":
|
|
18
|
+
return pd.read_parquet(p)
|
|
19
|
+
if p.suffix.lower() == ".json":
|
|
20
|
+
return pd.read_json(p)
|
|
21
|
+
return pd.read_csv(p)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _write(df: pd.DataFrame, path: Path) -> None:
|
|
25
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
26
|
+
if path.suffix.lower() == ".parquet":
|
|
27
|
+
df.to_parquet(path, index=False)
|
|
28
|
+
elif path.suffix.lower() == ".json":
|
|
29
|
+
df.to_json(path, orient="records", indent=2, date_format="iso")
|
|
30
|
+
else:
|
|
31
|
+
df.to_csv(path, index=False)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _cmd_synthesize(args) -> int:
|
|
35
|
+
from .evaluation import certify
|
|
36
|
+
from .metadata import Metadata
|
|
37
|
+
from .single_table import GaussianCopulaSynthesizer
|
|
38
|
+
|
|
39
|
+
real = _read(args.input)
|
|
40
|
+
md = Metadata.load(args.metadata) if args.metadata else None
|
|
41
|
+
synth = GaussianCopulaSynthesizer(md, default_distribution=args.distribution)
|
|
42
|
+
synth.fit(real)
|
|
43
|
+
rows = args.rows if args.rows is not None else len(real)
|
|
44
|
+
fake = synth.sample(rows, seed=args.seed)
|
|
45
|
+
_write(fake, Path(args.output))
|
|
46
|
+
print(f"Wrote {len(fake)} rows to {args.output}")
|
|
47
|
+
if args.report:
|
|
48
|
+
cert = certify(real, fake, synth.get_metadata())
|
|
49
|
+
Path(args.report).write_text(cert.to_json())
|
|
50
|
+
print(cert)
|
|
51
|
+
return 0
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _cmd_from_ddl(args) -> int:
|
|
55
|
+
from .schema import SchemaSynthesizer
|
|
56
|
+
|
|
57
|
+
synth = SchemaSynthesizer.from_ddl(Path(args.ddl).read_text(), children_per_parent=args.children)
|
|
58
|
+
tables = synth.sample(num_rows=args.rows, seed=args.seed)
|
|
59
|
+
out = Path(args.output)
|
|
60
|
+
for name, df in tables.items():
|
|
61
|
+
_write(df, out / f"{name}.{args.format}")
|
|
62
|
+
print(f"Wrote {len(df)} rows to {out / f'{name}.{args.format}'}")
|
|
63
|
+
return 0
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _cmd_evaluate(args) -> int:
|
|
67
|
+
from .evaluation import certify
|
|
68
|
+
from .metadata import Metadata
|
|
69
|
+
|
|
70
|
+
real, fake = _read(args.real), _read(args.synthetic)
|
|
71
|
+
md = Metadata.load(args.metadata) if args.metadata else None
|
|
72
|
+
cert = certify(real, fake, md, min_quality=args.min_quality)
|
|
73
|
+
print(json.dumps(cert.to_dict(), indent=2, default=str))
|
|
74
|
+
return 0 if cert.passed else 1
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _cmd_detect(args) -> int:
|
|
78
|
+
from .metadata import Metadata
|
|
79
|
+
|
|
80
|
+
md = Metadata.detect_from_dataframe(_read(args.input), args.table)
|
|
81
|
+
if args.output:
|
|
82
|
+
md.save(args.output)
|
|
83
|
+
print(f"Wrote metadata to {args.output}")
|
|
84
|
+
else:
|
|
85
|
+
print(json.dumps(md.to_dict(), indent=2, default=str))
|
|
86
|
+
return 0
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
90
|
+
parser = argparse.ArgumentParser(prog="synthschema", description="Schema-driven synthetic data generation.")
|
|
91
|
+
parser.add_argument("--version", action="version", version=f"synthschema {__version__}")
|
|
92
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
93
|
+
|
|
94
|
+
p = sub.add_parser("synthesize", help="learn from a CSV/Parquet file and write synthetic rows")
|
|
95
|
+
p.add_argument("input")
|
|
96
|
+
p.add_argument("-o", "--output", required=True)
|
|
97
|
+
p.add_argument("-n", "--rows", type=int)
|
|
98
|
+
p.add_argument("-m", "--metadata", help="metadata JSON/YAML (detected if omitted)")
|
|
99
|
+
p.add_argument("--seed", type=int)
|
|
100
|
+
p.add_argument("--distribution", default="empirical")
|
|
101
|
+
p.add_argument("--report", help="write a certificate JSON here")
|
|
102
|
+
p.set_defaults(func=_cmd_synthesize)
|
|
103
|
+
|
|
104
|
+
p = sub.add_parser("from-ddl", help="generate tables from SQL CREATE TABLE statements (no data)")
|
|
105
|
+
p.add_argument("ddl")
|
|
106
|
+
p.add_argument("-o", "--output", required=True, help="output directory")
|
|
107
|
+
p.add_argument("-n", "--rows", type=int, default=100, help="rows per root table")
|
|
108
|
+
p.add_argument("--children", type=float, default=3.0, help="child rows per parent row")
|
|
109
|
+
p.add_argument("--format", choices=["csv", "parquet", "json"], default="csv")
|
|
110
|
+
p.add_argument("--seed", type=int)
|
|
111
|
+
p.set_defaults(func=_cmd_from_ddl)
|
|
112
|
+
|
|
113
|
+
p = sub.add_parser("evaluate", help="certify synthetic data against real data")
|
|
114
|
+
p.add_argument("real")
|
|
115
|
+
p.add_argument("synthetic")
|
|
116
|
+
p.add_argument("-m", "--metadata")
|
|
117
|
+
p.add_argument("--min-quality", type=float, default=0.8)
|
|
118
|
+
p.set_defaults(func=_cmd_evaluate)
|
|
119
|
+
|
|
120
|
+
p = sub.add_parser("detect", help="infer metadata from a data file")
|
|
121
|
+
p.add_argument("input")
|
|
122
|
+
p.add_argument("-t", "--table", default="table")
|
|
123
|
+
p.add_argument("-o", "--output", help="write JSON/YAML here")
|
|
124
|
+
p.set_defaults(func=_cmd_detect)
|
|
125
|
+
return parser
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def main(argv: list[str] | None = None) -> int:
|
|
129
|
+
args = build_parser().parse_args(argv)
|
|
130
|
+
return args.func(args)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
if __name__ == "__main__": # pragma: no cover
|
|
134
|
+
sys.exit(main())
|
|
@@ -0,0 +1,438 @@
|
|
|
1
|
+
"""Business rules that must hold on 100% of generated rows.
|
|
2
|
+
|
|
3
|
+
Rules are enforced outside the model. Where possible a rule becomes a reversible transform
|
|
4
|
+
during training (``ship_date >= order_date`` is learned as a non-negative difference), so
|
|
5
|
+
the model never has to discover it. Every rule also has a row-level validity check;
|
|
6
|
+
synthesizers resample any row that fails it.
|
|
7
|
+
|
|
8
|
+
Rules can be written as objects, as dicts (for JSON/YAML blueprints) or as SQL-like strings::
|
|
9
|
+
|
|
10
|
+
Inequality(low="order_date", high="ship_date")
|
|
11
|
+
{"type": "range", "column": "discount", "min": 0, "max": 0.4}
|
|
12
|
+
"ship_date >= order_date"
|
|
13
|
+
"age BETWEEN 18 AND 90"
|
|
14
|
+
"status IN ('open', 'closed')"
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
import re
|
|
21
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
22
|
+
from typing import Any
|
|
23
|
+
|
|
24
|
+
import numpy as np
|
|
25
|
+
import pandas as pd
|
|
26
|
+
|
|
27
|
+
from .metadata import Column, Table
|
|
28
|
+
from .transformers import (
|
|
29
|
+
datetime_to_seconds,
|
|
30
|
+
format_datetime,
|
|
31
|
+
learn_decimals,
|
|
32
|
+
learn_granularity,
|
|
33
|
+
seconds_to_datetime,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _is_datetime(table: Table | None, column: str, series: pd.Series | None = None) -> bool:
|
|
38
|
+
if table is not None and column in table.columns:
|
|
39
|
+
return table.columns[column].sdtype == "datetime"
|
|
40
|
+
return series is not None and pd.api.types.is_datetime64_any_dtype(series)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _fmt(table: Table | None, column: str) -> str | None:
|
|
44
|
+
if table is not None and column in table.columns:
|
|
45
|
+
return table.columns[column].datetime_format
|
|
46
|
+
return None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _as_number(value: Any, is_dt: bool) -> float:
|
|
50
|
+
if is_dt:
|
|
51
|
+
return pd.Timestamp(value).value / 1e9
|
|
52
|
+
return float(value)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class Constraint:
|
|
56
|
+
"""Base class. Subclasses override the hooks they need."""
|
|
57
|
+
|
|
58
|
+
type = "constraint"
|
|
59
|
+
|
|
60
|
+
#: Columns that the transform removes from the training frame.
|
|
61
|
+
consumed_columns: tuple[str, ...] = ()
|
|
62
|
+
|
|
63
|
+
def fit(self, data: pd.DataFrame, table: Table | None = None) -> Constraint:
|
|
64
|
+
self.table = table
|
|
65
|
+
return self
|
|
66
|
+
|
|
67
|
+
def transform(self, data: pd.DataFrame) -> pd.DataFrame:
|
|
68
|
+
return data
|
|
69
|
+
|
|
70
|
+
def reverse(self, data: pd.DataFrame) -> pd.DataFrame:
|
|
71
|
+
return data
|
|
72
|
+
|
|
73
|
+
def is_valid(self, data: pd.DataFrame) -> np.ndarray:
|
|
74
|
+
return np.ones(len(data), dtype=bool)
|
|
75
|
+
|
|
76
|
+
def repair(self, data: pd.DataFrame) -> pd.DataFrame:
|
|
77
|
+
"""Make invalid rows valid with the smallest change (used by schema-only generation)."""
|
|
78
|
+
return data
|
|
79
|
+
|
|
80
|
+
def derived_columns(self) -> dict[str, Column]:
|
|
81
|
+
"""Columns the transform adds, with their sdtypes."""
|
|
82
|
+
return {}
|
|
83
|
+
|
|
84
|
+
@property
|
|
85
|
+
def columns(self) -> tuple[str, ...]: # pragma: no cover - overridden
|
|
86
|
+
return ()
|
|
87
|
+
|
|
88
|
+
def to_dict(self) -> dict: # pragma: no cover - overridden
|
|
89
|
+
raise NotImplementedError
|
|
90
|
+
|
|
91
|
+
def __repr__(self) -> str:
|
|
92
|
+
try:
|
|
93
|
+
return f"{type(self).__name__}({self.to_dict()})"
|
|
94
|
+
except NotImplementedError:
|
|
95
|
+
return f"{type(self).__name__}()"
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class Inequality(Constraint):
|
|
99
|
+
"""``low <= high`` (or ``low < high`` when ``strict``) for numbers or datetimes."""
|
|
100
|
+
|
|
101
|
+
type = "inequality"
|
|
102
|
+
|
|
103
|
+
def __init__(self, low: str, high: str, strict: bool = False) -> None:
|
|
104
|
+
self.low, self.high, self.strict = low, high, strict
|
|
105
|
+
self.consumed_columns = (high,)
|
|
106
|
+
self._diff = f"__{low}__{high}__diff"
|
|
107
|
+
|
|
108
|
+
@property
|
|
109
|
+
def columns(self):
|
|
110
|
+
return (self.low, self.high)
|
|
111
|
+
|
|
112
|
+
def _num(self, series: pd.Series, column: str) -> np.ndarray:
|
|
113
|
+
if self._dt:
|
|
114
|
+
return datetime_to_seconds(series, _fmt(self.table, column))
|
|
115
|
+
return pd.to_numeric(series, errors="coerce").astype(float).to_numpy()
|
|
116
|
+
|
|
117
|
+
def fit(self, data, table=None):
|
|
118
|
+
self.table = table
|
|
119
|
+
self._dt = _is_datetime(table, self.low, data[self.low]) or _is_datetime(
|
|
120
|
+
table, self.high, data[self.high]
|
|
121
|
+
)
|
|
122
|
+
high = self._num(data[self.high], self.high)
|
|
123
|
+
diff = high - self._num(data[self.low], self.low)
|
|
124
|
+
positive = diff[np.isfinite(diff) & (diff > 0)]
|
|
125
|
+
self._min_gap = float(positive.min()) if len(positive) else 1.0
|
|
126
|
+
if self._dt:
|
|
127
|
+
self._granularity = learn_granularity(high)
|
|
128
|
+
self._as_string = not pd.api.types.is_datetime64_any_dtype(data[self.high])
|
|
129
|
+
else:
|
|
130
|
+
self._decimals = learn_decimals(high[np.isfinite(high)])
|
|
131
|
+
return self
|
|
132
|
+
|
|
133
|
+
def transform(self, data):
|
|
134
|
+
out = data.copy()
|
|
135
|
+
out[self._diff] = self._num(data[self.high], self.high) - self._num(data[self.low], self.low)
|
|
136
|
+
return out.drop(columns=[self.high])
|
|
137
|
+
|
|
138
|
+
def reverse(self, data):
|
|
139
|
+
out = data.copy()
|
|
140
|
+
diff = pd.to_numeric(out.pop(self._diff), errors="coerce").to_numpy(dtype=float)
|
|
141
|
+
diff = np.where(diff < 0, 0.0, diff)
|
|
142
|
+
if self.strict:
|
|
143
|
+
diff = np.where(diff <= 0, self._min_gap, diff)
|
|
144
|
+
high = self._num(out[self.low], self.low) + diff
|
|
145
|
+
if self._dt:
|
|
146
|
+
dt = seconds_to_datetime(high, self._granularity)
|
|
147
|
+
out[self.high] = format_datetime(dt, _fmt(self.table, self.high), self._as_string).to_numpy()
|
|
148
|
+
else:
|
|
149
|
+
if self._decimals is not None:
|
|
150
|
+
high = np.round(high, self._decimals)
|
|
151
|
+
out[self.high] = high
|
|
152
|
+
return out
|
|
153
|
+
|
|
154
|
+
def is_valid(self, data):
|
|
155
|
+
low = self._num(data[self.low], self.low)
|
|
156
|
+
high = self._num(data[self.high], self.high)
|
|
157
|
+
missing = ~np.isfinite(low) | ~np.isfinite(high)
|
|
158
|
+
ok = high > low if self.strict else high >= low
|
|
159
|
+
return ok | missing
|
|
160
|
+
|
|
161
|
+
def repair(self, data):
|
|
162
|
+
out = data.copy()
|
|
163
|
+
bad = ~self.is_valid(out)
|
|
164
|
+
if bad.any():
|
|
165
|
+
lo, hi = out.loc[bad, self.low].copy(), out.loc[bad, self.high].copy()
|
|
166
|
+
out.loc[bad, self.low], out.loc[bad, self.high] = hi.to_numpy(), lo.to_numpy()
|
|
167
|
+
return out
|
|
168
|
+
|
|
169
|
+
def derived_columns(self):
|
|
170
|
+
return {self._diff: Column("numerical")}
|
|
171
|
+
|
|
172
|
+
def to_dict(self):
|
|
173
|
+
out = {"type": self.type, "low": self.low, "high": self.high}
|
|
174
|
+
if self.strict:
|
|
175
|
+
out["strict"] = True
|
|
176
|
+
return out
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
class Range(Constraint):
|
|
180
|
+
"""``min <= column <= max``; either bound may be omitted."""
|
|
181
|
+
|
|
182
|
+
type = "range"
|
|
183
|
+
|
|
184
|
+
def __init__(self, column: str, min: Any = None, max: Any = None, strict: bool = False) -> None:
|
|
185
|
+
self.column, self.min, self.max, self.strict = column, min, max, strict
|
|
186
|
+
|
|
187
|
+
@property
|
|
188
|
+
def columns(self):
|
|
189
|
+
return (self.column,)
|
|
190
|
+
|
|
191
|
+
def fit(self, data, table=None):
|
|
192
|
+
self.table = table
|
|
193
|
+
self._dt = _is_datetime(table, self.column, data.get(self.column))
|
|
194
|
+
return self
|
|
195
|
+
|
|
196
|
+
def _values(self, data):
|
|
197
|
+
s = data[self.column]
|
|
198
|
+
if getattr(self, "_dt", False):
|
|
199
|
+
return datetime_to_seconds(s, _fmt(getattr(self, "table", None), self.column))
|
|
200
|
+
return pd.to_numeric(s, errors="coerce").astype(float).to_numpy()
|
|
201
|
+
|
|
202
|
+
def is_valid(self, data):
|
|
203
|
+
x = self._values(data)
|
|
204
|
+
ok = np.ones(len(x), dtype=bool)
|
|
205
|
+
dt = getattr(self, "_dt", False)
|
|
206
|
+
if self.min is not None:
|
|
207
|
+
lo = _as_number(self.min, dt)
|
|
208
|
+
ok &= (x > lo) if self.strict else (x >= lo)
|
|
209
|
+
if self.max is not None:
|
|
210
|
+
hi = _as_number(self.max, dt)
|
|
211
|
+
ok &= (x < hi) if self.strict else (x <= hi)
|
|
212
|
+
return ok | ~np.isfinite(x)
|
|
213
|
+
|
|
214
|
+
def repair(self, data):
|
|
215
|
+
if getattr(self, "_dt", False):
|
|
216
|
+
return data
|
|
217
|
+
out = data.copy()
|
|
218
|
+
x = pd.to_numeric(out[self.column], errors="coerce")
|
|
219
|
+
out[self.column] = x.clip(
|
|
220
|
+
lower=None if self.min is None else float(self.min),
|
|
221
|
+
upper=None if self.max is None else float(self.max),
|
|
222
|
+
)
|
|
223
|
+
return out
|
|
224
|
+
|
|
225
|
+
def to_dict(self):
|
|
226
|
+
out = {"type": self.type, "column": self.column}
|
|
227
|
+
if self.min is not None:
|
|
228
|
+
out["min"] = self.min
|
|
229
|
+
if self.max is not None:
|
|
230
|
+
out["max"] = self.max
|
|
231
|
+
if self.strict:
|
|
232
|
+
out["strict"] = True
|
|
233
|
+
return out
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
class OneOf(Constraint):
|
|
237
|
+
"""``column IN (values)``."""
|
|
238
|
+
|
|
239
|
+
type = "one_of"
|
|
240
|
+
|
|
241
|
+
def __init__(self, column: str, values: Sequence) -> None:
|
|
242
|
+
self.column, self.values = column, list(values)
|
|
243
|
+
|
|
244
|
+
@property
|
|
245
|
+
def columns(self):
|
|
246
|
+
return (self.column,)
|
|
247
|
+
|
|
248
|
+
def is_valid(self, data):
|
|
249
|
+
s = data[self.column]
|
|
250
|
+
return (s.isin(self.values) | s.isna()).to_numpy()
|
|
251
|
+
|
|
252
|
+
def to_dict(self):
|
|
253
|
+
return {"type": self.type, "column": self.column, "values": self.values}
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
class FixedCombinations(Constraint):
|
|
257
|
+
"""Only combinations of these columns seen in the real data may appear
|
|
258
|
+
(e.g. country/currency, category/brand/SKU)."""
|
|
259
|
+
|
|
260
|
+
type = "fixed_combinations"
|
|
261
|
+
|
|
262
|
+
def __init__(self, columns: Sequence[str]) -> None:
|
|
263
|
+
if len(columns) < 2:
|
|
264
|
+
raise ValueError("FixedCombinations needs at least two columns")
|
|
265
|
+
self._columns = tuple(columns)
|
|
266
|
+
self.consumed_columns = self._columns
|
|
267
|
+
self._combo = "__" + "__".join(self._columns) + "__combo"
|
|
268
|
+
|
|
269
|
+
@property
|
|
270
|
+
def columns(self):
|
|
271
|
+
return self._columns
|
|
272
|
+
|
|
273
|
+
@staticmethod
|
|
274
|
+
def _key(values) -> str:
|
|
275
|
+
return json.dumps([None if pd.isna(v) else v for v in values], default=str)
|
|
276
|
+
|
|
277
|
+
def fit(self, data, table=None):
|
|
278
|
+
self.table = table
|
|
279
|
+
keys = [self._key(r) for r in data[list(self._columns)].itertuples(index=False)]
|
|
280
|
+
self._lookup: dict[str, tuple] = {}
|
|
281
|
+
for key, row in zip(keys, data[list(self._columns)].itertuples(index=False), strict=False):
|
|
282
|
+
self._lookup.setdefault(key, tuple(row))
|
|
283
|
+
return self
|
|
284
|
+
|
|
285
|
+
def transform(self, data):
|
|
286
|
+
out = data.copy()
|
|
287
|
+
out[self._combo] = [self._key(r) for r in data[list(self._columns)].itertuples(index=False)]
|
|
288
|
+
return out.drop(columns=list(self._columns))
|
|
289
|
+
|
|
290
|
+
def reverse(self, data):
|
|
291
|
+
out = data.copy()
|
|
292
|
+
keys = out.pop(self._combo)
|
|
293
|
+
rows = [self._lookup.get(k, (None,) * len(self._columns)) for k in keys]
|
|
294
|
+
for i, col in enumerate(self._columns):
|
|
295
|
+
out[col] = pd.Series([r[i] for r in rows], index=out.index, dtype=object)
|
|
296
|
+
return out
|
|
297
|
+
|
|
298
|
+
def is_valid(self, data):
|
|
299
|
+
keys = [self._key(r) for r in data[list(self._columns)].itertuples(index=False)]
|
|
300
|
+
return np.array([k in self._lookup for k in keys], dtype=bool)
|
|
301
|
+
|
|
302
|
+
def derived_columns(self):
|
|
303
|
+
return {self._combo: Column("categorical")}
|
|
304
|
+
|
|
305
|
+
def to_dict(self):
|
|
306
|
+
return {"type": self.type, "columns": list(self._columns)}
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
class Custom(Constraint):
|
|
310
|
+
"""Any row-level rule given as a function ``DataFrame -> boolean array``."""
|
|
311
|
+
|
|
312
|
+
type = "custom"
|
|
313
|
+
|
|
314
|
+
def __init__(self, is_valid: Callable[[pd.DataFrame], Any], columns: Sequence[str] = ()) -> None:
|
|
315
|
+
self._fn = is_valid
|
|
316
|
+
self._columns = tuple(columns)
|
|
317
|
+
|
|
318
|
+
@property
|
|
319
|
+
def columns(self):
|
|
320
|
+
return self._columns
|
|
321
|
+
|
|
322
|
+
def is_valid(self, data):
|
|
323
|
+
return np.asarray(self._fn(data), dtype=bool)
|
|
324
|
+
|
|
325
|
+
def to_dict(self):
|
|
326
|
+
raise NotImplementedError("Custom constraints cannot be serialised")
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
# ---------------------------------------------------------------------- parsing
|
|
330
|
+
|
|
331
|
+
_IDENT = r"[A-Za-z_][A-Za-z0-9_]*"
|
|
332
|
+
_BETWEEN_RE = re.compile(rf"({_IDENT})\s+BETWEEN\s+(\S+)\s+AND\s+(\S+)", re.IGNORECASE)
|
|
333
|
+
_IN_RE = re.compile(rf"^({_IDENT})\s+IN\s*\((.*)\)$", re.IGNORECASE | re.DOTALL)
|
|
334
|
+
_CMP_RE = re.compile(rf"^({_IDENT})\s*(>=|<=|>|<)\s*(.+)$")
|
|
335
|
+
_AND_RE = re.compile(r"\s+AND\s+", re.IGNORECASE)
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _literal(text: str) -> Any:
|
|
339
|
+
text = text.strip()
|
|
340
|
+
if len(text) >= 2 and text[0] == text[-1] and text[0] in "'\"":
|
|
341
|
+
return text[1:-1]
|
|
342
|
+
try:
|
|
343
|
+
return int(text)
|
|
344
|
+
except ValueError:
|
|
345
|
+
pass
|
|
346
|
+
try:
|
|
347
|
+
return float(text)
|
|
348
|
+
except ValueError:
|
|
349
|
+
return None
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def _strip_outer_parens(text: str) -> str:
|
|
353
|
+
text = text.strip()
|
|
354
|
+
while text.startswith("(") and text.endswith(")"):
|
|
355
|
+
depth = 0
|
|
356
|
+
for i, ch in enumerate(text):
|
|
357
|
+
depth += ch == "("
|
|
358
|
+
depth -= ch == ")"
|
|
359
|
+
if depth == 0 and i < len(text) - 1:
|
|
360
|
+
return text # the first "(" closes before the end: not a wrapper
|
|
361
|
+
text = text[1:-1].strip()
|
|
362
|
+
return text
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def parse_rule(rule: str) -> list[Constraint]:
|
|
366
|
+
"""Parse a SQL-like rule string into constraints. Raises ``ValueError`` if unsupported."""
|
|
367
|
+
text = _strip_outer_parens(rule)
|
|
368
|
+
out: list[Constraint] = []
|
|
369
|
+
for m in _BETWEEN_RE.finditer(text):
|
|
370
|
+
lo, hi = _literal(m.group(2)), _literal(m.group(3))
|
|
371
|
+
if lo is None or hi is None:
|
|
372
|
+
raise ValueError(f"Unsupported BETWEEN bounds in rule {rule!r}")
|
|
373
|
+
out.append(Range(m.group(1), min=lo, max=hi))
|
|
374
|
+
text = _BETWEEN_RE.sub("", text).strip()
|
|
375
|
+
parts = [_strip_outer_parens(p) for p in _AND_RE.split(text) if p.strip()] if text else []
|
|
376
|
+
for part in parts:
|
|
377
|
+
m = _IN_RE.match(part)
|
|
378
|
+
if m:
|
|
379
|
+
values = [_literal(v) for v in re.split(r",(?=(?:[^']*'[^']*')*[^']*$)", m.group(2))]
|
|
380
|
+
out.append(OneOf(m.group(1), [v for v in values if v is not None]))
|
|
381
|
+
continue
|
|
382
|
+
m = _CMP_RE.match(part)
|
|
383
|
+
if not m:
|
|
384
|
+
raise ValueError(f"Unsupported rule {rule!r}")
|
|
385
|
+
left, op, right = m.group(1), m.group(2), m.group(3).strip()
|
|
386
|
+
strict = op in {">", "<"}
|
|
387
|
+
if re.fullmatch(_IDENT, right):
|
|
388
|
+
low, high = (right, left) if op in {">=", ">"} else (left, right)
|
|
389
|
+
out.append(Inequality(low, high, strict=strict))
|
|
390
|
+
continue
|
|
391
|
+
value = _literal(right)
|
|
392
|
+
if value is None:
|
|
393
|
+
raise ValueError(f"Unsupported rule {rule!r}")
|
|
394
|
+
if op in {">=", ">"}:
|
|
395
|
+
out.append(Range(left, min=value, strict=strict))
|
|
396
|
+
else:
|
|
397
|
+
out.append(Range(left, max=value, strict=strict))
|
|
398
|
+
if not out:
|
|
399
|
+
raise ValueError(f"Unsupported rule {rule!r}")
|
|
400
|
+
return out
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def from_spec(spec: Any) -> list[Constraint]:
|
|
404
|
+
"""Turn a constraint object, dict or rule string into constraint objects."""
|
|
405
|
+
if isinstance(spec, Constraint):
|
|
406
|
+
return [spec]
|
|
407
|
+
if isinstance(spec, str):
|
|
408
|
+
return parse_rule(spec)
|
|
409
|
+
if isinstance(spec, Mapping):
|
|
410
|
+
if "rule" in spec:
|
|
411
|
+
return parse_rule(spec["rule"])
|
|
412
|
+
kind = str(spec.get("type", "")).lower()
|
|
413
|
+
if kind == "inequality":
|
|
414
|
+
return [Inequality(spec["low"], spec["high"], bool(spec.get("strict", False)))]
|
|
415
|
+
if kind in {"range", "scalar_range"}:
|
|
416
|
+
return [
|
|
417
|
+
Range(
|
|
418
|
+
spec["column"],
|
|
419
|
+
spec.get("min", spec.get("min_value")),
|
|
420
|
+
spec.get("max", spec.get("max_value")),
|
|
421
|
+
bool(spec.get("strict", False)),
|
|
422
|
+
)
|
|
423
|
+
]
|
|
424
|
+
if kind == "positive":
|
|
425
|
+
return [Range(spec["column"], min=0, strict=bool(spec.get("strict", True)))]
|
|
426
|
+
if kind == "one_of":
|
|
427
|
+
return [OneOf(spec["column"], spec["values"])]
|
|
428
|
+
if kind == "fixed_combinations":
|
|
429
|
+
return [FixedCombinations(spec["columns"])]
|
|
430
|
+
raise ValueError(f"Unknown constraint type {spec.get('type')!r}")
|
|
431
|
+
raise TypeError(f"Cannot build a constraint from {type(spec).__name__}")
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
def build_constraints(specs: Sequence[Any] | None) -> list[Constraint]:
|
|
435
|
+
out: list[Constraint] = []
|
|
436
|
+
for spec in specs or []:
|
|
437
|
+
out.extend(from_spec(spec))
|
|
438
|
+
return out
|