fauxdata-cli 0.1.3__tar.gz → 0.1.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/.claude/settings.local.json +2 -1
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/.gitignore +1 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/LOG.md +18 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/PKG-INFO +26 -3
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/README.md +24 -1
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/pyproject.toml +2 -2
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/src/fauxdata/__init__.py +1 -1
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/src/fauxdata/commands/generate.py +27 -3
- fauxdata_cli-0.1.5/src/fauxdata/commands/infer.py +178 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/src/fauxdata/commands/init.py +26 -13
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/src/fauxdata/generator.py +17 -2
- fauxdata_cli-0.1.5/src/fauxdata/main.py +153 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/src/fauxdata/schema.py +11 -0
- fauxdata_cli-0.1.5/tasks/todo.md +52 -0
- fauxdata_cli-0.1.5/tests/test_infer.py +171 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/tests/test_new_fields.py +37 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/uv.lock +188 -9
- fauxdata_cli-0.1.3/.coverage +0 -0
- fauxdata_cli-0.1.3/src/fauxdata/main.py +0 -87
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/.python-version +0 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/LICENSE +0 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/docs/deployment.md +0 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/docs/index.html +0 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/docs/share.png +0 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/schemas/events.yml +0 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/schemas/orders.yml +0 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/schemas/people.yml +0 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/share.png +0 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/src/fauxdata/commands/__init__.py +0 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/src/fauxdata/commands/preview.py +0 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/src/fauxdata/commands/validate.py +0 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/src/fauxdata/output.py +0 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/src/fauxdata/validator.py +0 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/tests/__init__.py +0 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/tests/conftest.py +0 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/tests/test_cli.py +0 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/tests/test_generator.py +0 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/tests/test_output.py +0 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/tests/test_schema.py +0 -0
- {fauxdata_cli-0.1.3 → fauxdata_cli-0.1.5}/tests/test_validator.py +0 -0
|
@@ -1,5 +1,23 @@
|
|
|
1
1
|
# Log
|
|
2
2
|
|
|
3
|
+
## 2026-06-12 — pointblank 0.25 + comando `infer`
|
|
4
|
+
|
|
5
|
+
- Bump `pointblank>=0.25` (uscito su PyPI). Schemi inclusi (people/orders/events) verificati: tutti PASS.
|
|
6
|
+
- Fase B regressioni: `rows_distinct` ora include righe con null (correttezza migliorata), lazy frame ok. fauxdata passa sempre DataFrame eager → nessun impatto.
|
|
7
|
+
- Nuovo comando **`fauxdata infer DATASET`**: da tabella reale (csv/parquet/json/jsonl) genera schema YAML (`columns:` + `validation:`) via `pb.schema_from_tbl`. Flag: `--out/-o`, `--name/-n`, `--rows/-r`, `--format/-f`, `--categorical-threshold`, `--detect-presets/--no-detect-presets`, `--sample-size`.
|
|
8
|
+
- `ColumnSchema`: nuovi campi `min_length`/`max_length` per stringhe (round-trip fedele); passati a `pb.string_field`. Validazione parsing (negativi, min>max).
|
|
9
|
+
- infer CSV: `try_parse_dates=True` → date/datetime tipizzati invece che congelati come categorici.
|
|
10
|
+
- generate: errore chiaro su conflitto unique+range stretto ("too narrow a range") invece del traceback criptico di pointblank.
|
|
11
|
+
- Limite noto: colonne a bassa cardinalità congelate ai valori reali (possibile leak su colonne sensibili) — documentato nel README.
|
|
12
|
+
- Test: +14 (`tests/test_infer.py` + length in `test_new_fields.py`). 102/102 pass, coverage 83.2%.
|
|
13
|
+
|
|
14
|
+
- `init`: aggiunto `--description`, `--rows`, `--format`, `--yes`; questionary diventa fallback
|
|
15
|
+
- Tutti i comandi: `epilog` con esempi in `--help`
|
|
16
|
+
- `generate`: aggiunto `--dry-run` (mostra piano senza scrivere)
|
|
17
|
+
- `generate`: output su successo mostra `output_path`, `format`, `rows` come chiave: valore
|
|
18
|
+
- `init`: output su successo aggiunge `schema_path: <file>`
|
|
19
|
+
- Fix venv: ricreato con `uv venv --clear` (cartella rinominata da `real_fake_datasets`)
|
|
20
|
+
|
|
3
21
|
## 2026-03-06 — v0.1.3
|
|
4
22
|
|
|
5
23
|
- Add Python classifiers to pyproject.toml (3.11, 3.12, 3.13, MIT) — fixes pyversions badge
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: fauxdata-cli
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.5
|
|
4
4
|
Summary: CLI for generating and validating fake datasets
|
|
5
5
|
Project-URL: Homepage, https://aborruso.github.io/fauxdata/
|
|
6
6
|
Project-URL: Repository, https://github.com/aborruso/fauxdata
|
|
@@ -13,7 +13,7 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.12
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.13
|
|
15
15
|
Requires-Python: >=3.11
|
|
16
|
-
Requires-Dist: pointblank>=0.
|
|
16
|
+
Requires-Dist: pointblank>=0.25
|
|
17
17
|
Requires-Dist: polars>=1.0
|
|
18
18
|
Requires-Dist: pyfiglet>=1.0
|
|
19
19
|
Requires-Dist: pyyaml>=6.0
|
|
@@ -171,7 +171,7 @@ validation:
|
|
|
171
171
|
|------|-------------|---------|
|
|
172
172
|
| `int` | Integer | `min`, `max`, `unique` |
|
|
173
173
|
| `float` | Floating point | `min`, `max` |
|
|
174
|
-
| `string` | Text | `preset`, `values`, `unique` |
|
|
174
|
+
| `string` | Text | `preset`, `values`, `pattern`, `unique`, `min_length`, `max_length` |
|
|
175
175
|
| `bool` | Boolean | — |
|
|
176
176
|
| `date` | Date | `min`, `max` (ISO format) |
|
|
177
177
|
| `datetime` | Datetime | `min`, `max` (ISO format) |
|
|
@@ -259,6 +259,29 @@ fauxdata generate schemas/people.yml --out - --format jsonl | wc -l
|
|
|
259
259
|
|
|
260
260
|
When `--out -` is used, all output messages are suppressed and only data is written to stdout.
|
|
261
261
|
|
|
262
|
+
### `fauxdata infer DATASET`
|
|
263
|
+
|
|
264
|
+
Build a YAML schema from a **real** dataset. fauxdata inspects the table (via pointblank's schema inference) and writes a schema with inferred ranges, categorical value sets, string presets (email, url…), uniqueness, null rates, and string lengths — plus matching `validation:` rules. Feed the result back to `fauxdata generate` to produce synthetic data that mirrors the real shape, without ever shipping the real values.
|
|
265
|
+
|
|
266
|
+
```
|
|
267
|
+
fauxdata infer real_data.csv # -> real_data.yml
|
|
268
|
+
fauxdata infer real_data.parquet --out schema.yml --name people
|
|
269
|
+
fauxdata infer real_data.csv --categorical-threshold 50 --no-detect-presets
|
|
270
|
+
fauxdata infer big.parquet --sample-size 10000 --out - | head # stream to stdout
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
| Option | Short | Default | Description |
|
|
274
|
+
|--------|-------|---------|-------------|
|
|
275
|
+
| `--out` | `-o` | `<name>.yml` | Output YAML path — use `-` for stdout |
|
|
276
|
+
| `--name` | `-n` | dataset stem | Schema name |
|
|
277
|
+
| `--rows` | `-r` | source rows | Rows to generate (written into the schema) |
|
|
278
|
+
| `--format` | `-f` | `csv` | Default output format written into the schema |
|
|
279
|
+
| `--categorical-threshold` | | `20` | Max unique values (int) or fraction (`0`–`1`) to treat a column as categorical |
|
|
280
|
+
| `--detect-presets` / `--no-detect-presets` | | on | Match string columns to known presets (email, url…) |
|
|
281
|
+
| `--sample-size` | | all rows | Sample N rows before analysis (for very large tables) |
|
|
282
|
+
|
|
283
|
+
> **Notes.** Low-cardinality columns (≤ `--categorical-threshold` distinct values) are frozen to their exact source values via `values:` — desirable for categories like `status`, but it means sensitive low-cardinality columns are reproduced verbatim. Raise the threshold or edit the schema if that's not what you want. Also, a column inferred as `unique` over a narrow integer range cannot generate many more rows than the source; widen `min`/`max` or drop `unique` to amplify.
|
|
284
|
+
|
|
262
285
|
### `fauxdata validate DATASET SCHEMA`
|
|
263
286
|
|
|
264
287
|
```
|
|
@@ -147,7 +147,7 @@ validation:
|
|
|
147
147
|
|------|-------------|---------|
|
|
148
148
|
| `int` | Integer | `min`, `max`, `unique` |
|
|
149
149
|
| `float` | Floating point | `min`, `max` |
|
|
150
|
-
| `string` | Text | `preset`, `values`, `unique` |
|
|
150
|
+
| `string` | Text | `preset`, `values`, `pattern`, `unique`, `min_length`, `max_length` |
|
|
151
151
|
| `bool` | Boolean | — |
|
|
152
152
|
| `date` | Date | `min`, `max` (ISO format) |
|
|
153
153
|
| `datetime` | Datetime | `min`, `max` (ISO format) |
|
|
@@ -235,6 +235,29 @@ fauxdata generate schemas/people.yml --out - --format jsonl | wc -l
|
|
|
235
235
|
|
|
236
236
|
When `--out -` is used, all output messages are suppressed and only data is written to stdout.
|
|
237
237
|
|
|
238
|
+
### `fauxdata infer DATASET`
|
|
239
|
+
|
|
240
|
+
Build a YAML schema from a **real** dataset. fauxdata inspects the table (via pointblank's schema inference) and writes a schema with inferred ranges, categorical value sets, string presets (email, url…), uniqueness, null rates, and string lengths — plus matching `validation:` rules. Feed the result back to `fauxdata generate` to produce synthetic data that mirrors the real shape, without ever shipping the real values.
|
|
241
|
+
|
|
242
|
+
```
|
|
243
|
+
fauxdata infer real_data.csv # -> real_data.yml
|
|
244
|
+
fauxdata infer real_data.parquet --out schema.yml --name people
|
|
245
|
+
fauxdata infer real_data.csv --categorical-threshold 50 --no-detect-presets
|
|
246
|
+
fauxdata infer big.parquet --sample-size 10000 --out - | head # stream to stdout
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
| Option | Short | Default | Description |
|
|
250
|
+
|--------|-------|---------|-------------|
|
|
251
|
+
| `--out` | `-o` | `<name>.yml` | Output YAML path — use `-` for stdout |
|
|
252
|
+
| `--name` | `-n` | dataset stem | Schema name |
|
|
253
|
+
| `--rows` | `-r` | source rows | Rows to generate (written into the schema) |
|
|
254
|
+
| `--format` | `-f` | `csv` | Default output format written into the schema |
|
|
255
|
+
| `--categorical-threshold` | | `20` | Max unique values (int) or fraction (`0`–`1`) to treat a column as categorical |
|
|
256
|
+
| `--detect-presets` / `--no-detect-presets` | | on | Match string columns to known presets (email, url…) |
|
|
257
|
+
| `--sample-size` | | all rows | Sample N rows before analysis (for very large tables) |
|
|
258
|
+
|
|
259
|
+
> **Notes.** Low-cardinality columns (≤ `--categorical-threshold` distinct values) are frozen to their exact source values via `values:` — desirable for categories like `status`, but it means sensitive low-cardinality columns are reproduced verbatim. Raise the threshold or edit the schema if that's not what you want. Also, a column inferred as `unique` over a narrow integer range cannot generate many more rows than the source; widen `min`/`max` or drop `unique` to amplify.
|
|
260
|
+
|
|
238
261
|
### `fauxdata validate DATASET SCHEMA`
|
|
239
262
|
|
|
240
263
|
```
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "fauxdata-cli"
|
|
3
|
-
version = "0.1.
|
|
3
|
+
version = "0.1.5"
|
|
4
4
|
description = "CLI for generating and validating fake datasets"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.11"
|
|
@@ -13,7 +13,7 @@ classifiers = [
|
|
|
13
13
|
"Operating System :: OS Independent",
|
|
14
14
|
]
|
|
15
15
|
dependencies = [
|
|
16
|
-
"pointblank>=0.
|
|
16
|
+
"pointblank>=0.25",
|
|
17
17
|
"typer>=0.12",
|
|
18
18
|
"rich>=13",
|
|
19
19
|
"pyfiglet>=1.0",
|
|
@@ -19,6 +19,15 @@ from fauxdata.validator import validate_dataset
|
|
|
19
19
|
console = Console()
|
|
20
20
|
|
|
21
21
|
|
|
22
|
+
def _generate(schema, n, rng_seed):
|
|
23
|
+
"""Generate, translating generation errors into a clean CLI exit."""
|
|
24
|
+
try:
|
|
25
|
+
return generate_dataset(schema, rows=n, seed=rng_seed)
|
|
26
|
+
except ValueError as e:
|
|
27
|
+
rprint(f"[red]Error:[/red] {e}")
|
|
28
|
+
raise typer.Exit(code=1)
|
|
29
|
+
|
|
30
|
+
|
|
22
31
|
def run(
|
|
23
32
|
schema_path: str,
|
|
24
33
|
rows: Optional[int] = None,
|
|
@@ -26,6 +35,7 @@ def run(
|
|
|
26
35
|
fmt: Optional[str] = None,
|
|
27
36
|
seed: Optional[int] = None,
|
|
28
37
|
validate: bool = False,
|
|
38
|
+
dry_run: bool = False,
|
|
29
39
|
):
|
|
30
40
|
"""Generate a fake dataset from a YAML schema."""
|
|
31
41
|
schema = load_schema(schema_path)
|
|
@@ -37,21 +47,35 @@ def run(
|
|
|
37
47
|
|
|
38
48
|
stdout_mode = output_path == "-"
|
|
39
49
|
|
|
50
|
+
if dry_run:
|
|
51
|
+
rprint(Panel(f"[bold cyan]fauxdata generate --dry-run[/bold cyan] [dim]{schema_path}[/dim]", expand=False))
|
|
52
|
+
rprint("[yellow]Dry run — no files will be written.[/yellow]\n")
|
|
53
|
+
rprint(f" schema: {schema_path}")
|
|
54
|
+
rprint(f" rows: {n}")
|
|
55
|
+
rprint(f" seed: {rng_seed}")
|
|
56
|
+
rprint(f" format: {output_fmt}")
|
|
57
|
+
rprint(f" output_path: {output_path}")
|
|
58
|
+
rprint(f" validate: {validate}")
|
|
59
|
+
rprint(f" columns: {', '.join(c.name for c in schema.columns)}")
|
|
60
|
+
return
|
|
61
|
+
|
|
40
62
|
if not stdout_mode:
|
|
41
63
|
rprint(Panel(f"[bold cyan]fauxdata generate[/bold cyan] [dim]{schema_path}[/dim]", expand=False))
|
|
42
64
|
|
|
43
65
|
if stdout_mode:
|
|
44
|
-
df =
|
|
66
|
+
df = _generate(schema, n, rng_seed)
|
|
45
67
|
write_stdout(df, output_fmt)
|
|
46
68
|
return
|
|
47
69
|
|
|
48
70
|
with console.status(f"[bold green]Generating {n} rows...[/bold green]"):
|
|
49
|
-
df =
|
|
71
|
+
df = _generate(schema, n, rng_seed)
|
|
50
72
|
|
|
51
73
|
_print_schema_table(schema, n, rng_seed)
|
|
52
74
|
|
|
53
75
|
saved = export_dataset(df, output_path, output_fmt)
|
|
54
|
-
rprint(f"\
|
|
76
|
+
rprint(f"\noutput_path: {saved}")
|
|
77
|
+
rprint(f"format: {output_fmt}")
|
|
78
|
+
rprint(f"rows: {n}")
|
|
55
79
|
|
|
56
80
|
if validate:
|
|
57
81
|
_run_validation(df, schema)
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"""fauxdata infer command - build a YAML schema from a real dataset.
|
|
2
|
+
|
|
3
|
+
Uses pointblank's ``schema_from_tbl`` (>=0.25) to infer rich constraints
|
|
4
|
+
(min/max, unique, null rate, allowed values, presets, string lengths) from an
|
|
5
|
+
existing table, then serializes them into a fauxdata YAML schema that can be
|
|
6
|
+
fed back to ``fauxdata generate``.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
import pointblank as pb
|
|
14
|
+
import polars as pl
|
|
15
|
+
import typer
|
|
16
|
+
import yaml
|
|
17
|
+
from rich import print as rprint
|
|
18
|
+
from rich.panel import Panel
|
|
19
|
+
|
|
20
|
+
# Polars dtype name (str) -> fauxdata col_type
|
|
21
|
+
_DTYPE_MAP = {
|
|
22
|
+
"Int": "int",
|
|
23
|
+
"UInt": "int",
|
|
24
|
+
"Float": "float",
|
|
25
|
+
"String": "string",
|
|
26
|
+
"Utf8": "string",
|
|
27
|
+
"Boolean": "bool",
|
|
28
|
+
"Datetime": "datetime",
|
|
29
|
+
"Date": "date",
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _col_type_from_dtype(dtype: str) -> str:
|
|
34
|
+
"""Map a pointblank/polars dtype name to a fauxdata col_type."""
|
|
35
|
+
for prefix, ctype in _DTYPE_MAP.items():
|
|
36
|
+
if dtype.startswith(prefix):
|
|
37
|
+
return ctype
|
|
38
|
+
return "string"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _field_to_spec(field) -> dict:
|
|
42
|
+
"""Convert an inferred pointblank Field into a fauxdata column spec dict."""
|
|
43
|
+
attrs = vars(field)
|
|
44
|
+
col_type = _col_type_from_dtype(attrs.get("dtype", "String"))
|
|
45
|
+
spec: dict = {"type": col_type}
|
|
46
|
+
|
|
47
|
+
if attrs.get("unique"):
|
|
48
|
+
spec["unique"] = True
|
|
49
|
+
if attrs.get("nullable"):
|
|
50
|
+
spec["nullable"] = True
|
|
51
|
+
np = attrs.get("null_probability")
|
|
52
|
+
if np:
|
|
53
|
+
spec["null_probability"] = round(float(np), 4)
|
|
54
|
+
|
|
55
|
+
if col_type in ("int", "float"):
|
|
56
|
+
if attrs.get("min_val") is not None:
|
|
57
|
+
spec["min"] = attrs["min_val"]
|
|
58
|
+
if attrs.get("max_val") is not None:
|
|
59
|
+
spec["max"] = attrs["max_val"]
|
|
60
|
+
if attrs.get("allowed"):
|
|
61
|
+
spec["values"] = list(attrs["allowed"])
|
|
62
|
+
|
|
63
|
+
elif col_type in ("date", "datetime"):
|
|
64
|
+
if attrs.get("min_date") is not None:
|
|
65
|
+
spec["min"] = str(attrs["min_date"])
|
|
66
|
+
if attrs.get("max_date") is not None:
|
|
67
|
+
spec["max"] = str(attrs["max_date"])
|
|
68
|
+
|
|
69
|
+
elif col_type == "string":
|
|
70
|
+
if attrs.get("preset"):
|
|
71
|
+
spec["preset"] = attrs["preset"]
|
|
72
|
+
elif attrs.get("allowed"):
|
|
73
|
+
spec["values"] = list(attrs["allowed"])
|
|
74
|
+
elif attrs.get("pattern"):
|
|
75
|
+
spec["pattern"] = attrs["pattern"]
|
|
76
|
+
if attrs.get("min_length") is not None:
|
|
77
|
+
spec["min_length"] = attrs["min_length"]
|
|
78
|
+
if attrs.get("max_length") is not None:
|
|
79
|
+
spec["max_length"] = attrs["max_length"]
|
|
80
|
+
|
|
81
|
+
return spec
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _build_validation(columns: list[tuple[str, dict]]) -> list[dict]:
|
|
85
|
+
"""Derive validation rules from the inferred column specs."""
|
|
86
|
+
rules: list[dict] = []
|
|
87
|
+
|
|
88
|
+
not_null = [name for name, spec in columns if not spec.get("nullable")]
|
|
89
|
+
if not_null:
|
|
90
|
+
rules.append({"rule": "col_vals_not_null", "columns": not_null})
|
|
91
|
+
|
|
92
|
+
for name, spec in columns:
|
|
93
|
+
if spec["type"] in ("int", "float") and "min" in spec and "max" in spec:
|
|
94
|
+
rules.append({"rule": "col_vals_between", "column": name,
|
|
95
|
+
"min": spec["min"], "max": spec["max"]})
|
|
96
|
+
|
|
97
|
+
for name, spec in columns:
|
|
98
|
+
if "values" in spec:
|
|
99
|
+
rules.append({"rule": "col_vals_in_set", "column": name, "values": list(spec["values"])})
|
|
100
|
+
|
|
101
|
+
unique_cols = [name for name, spec in columns if spec.get("unique")]
|
|
102
|
+
if unique_cols:
|
|
103
|
+
rules.append({"rule": "rows_distinct", "columns": unique_cols})
|
|
104
|
+
|
|
105
|
+
return rules
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _load_table(path: str) -> pl.DataFrame:
|
|
109
|
+
ext = path.rsplit(".", 1)[-1].lower()
|
|
110
|
+
if ext == "csv":
|
|
111
|
+
# try_parse_dates so date/datetime columns are typed (not frozen as categorical strings)
|
|
112
|
+
return pl.read_csv(path, try_parse_dates=True)
|
|
113
|
+
if ext == "parquet":
|
|
114
|
+
return pl.read_parquet(path)
|
|
115
|
+
if ext == "json":
|
|
116
|
+
return pl.read_json(path)
|
|
117
|
+
if ext in ("jsonl", "ndjson"):
|
|
118
|
+
return pl.read_ndjson(path)
|
|
119
|
+
rprint(f"[red]Unsupported file format: .{ext}[/red]")
|
|
120
|
+
raise typer.Exit(code=1)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def run(
|
|
124
|
+
dataset_path: str,
|
|
125
|
+
out: str | None = None,
|
|
126
|
+
name: str | None = None,
|
|
127
|
+
rows: int | None = None,
|
|
128
|
+
fmt: str = "csv",
|
|
129
|
+
categorical_threshold: float = 20,
|
|
130
|
+
detect_presets: bool = True,
|
|
131
|
+
sample_size: int | None = None,
|
|
132
|
+
):
|
|
133
|
+
"""Infer a fauxdata YAML schema from a real dataset."""
|
|
134
|
+
rprint(Panel(
|
|
135
|
+
f"[bold cyan]fauxdata infer[/bold cyan] [dim]{dataset_path}[/dim]",
|
|
136
|
+
expand=False,
|
|
137
|
+
))
|
|
138
|
+
|
|
139
|
+
df = _load_table(dataset_path)
|
|
140
|
+
rprint(f" Loaded [bold]{len(df)}[/bold] rows, [bold]{len(df.columns)}[/bold] columns")
|
|
141
|
+
|
|
142
|
+
pb_schema = pb.schema_from_tbl(
|
|
143
|
+
df,
|
|
144
|
+
categorical_threshold=categorical_threshold,
|
|
145
|
+
detect_presets=detect_presets,
|
|
146
|
+
sample_size=sample_size,
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
columns = [(col_name, _field_to_spec(field)) for col_name, field in pb_schema.columns]
|
|
150
|
+
|
|
151
|
+
schema_name = name or Path(dataset_path).stem
|
|
152
|
+
out_rows = rows if rows is not None else len(df)
|
|
153
|
+
|
|
154
|
+
schema_dict = {
|
|
155
|
+
"name": schema_name,
|
|
156
|
+
"description": f"Inferred from {Path(dataset_path).name}",
|
|
157
|
+
"rows": out_rows,
|
|
158
|
+
"seed": 42,
|
|
159
|
+
"output": {"format": fmt, "path": f"{schema_name}.{fmt}"},
|
|
160
|
+
"columns": {col_name: spec for col_name, spec in columns},
|
|
161
|
+
"validation": _build_validation(columns),
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
text = yaml.safe_dump(schema_dict, sort_keys=False, allow_unicode=True, default_flow_style=False)
|
|
165
|
+
|
|
166
|
+
out_path = out or f"{schema_name}.yml"
|
|
167
|
+
if out_path == "-":
|
|
168
|
+
import sys
|
|
169
|
+
sys.stdout.write(text)
|
|
170
|
+
return
|
|
171
|
+
|
|
172
|
+
Path(out_path).write_text(text)
|
|
173
|
+
rprint(f"[green]Created[/green] [bold]{out_path}[/bold] "
|
|
174
|
+
f"([bold]{len(columns)}[/bold] columns, "
|
|
175
|
+
f"[bold]{len(schema_dict['validation'])}[/bold] validation rules)")
|
|
176
|
+
rprint("[dim]Generate synthetic data with:[/dim]")
|
|
177
|
+
rprint(f" [cyan]fauxdata generate {out_path} --validate[/cyan]")
|
|
178
|
+
rprint(f"schema_path: {out_path}")
|
|
@@ -75,8 +75,14 @@ validation:
|
|
|
75
75
|
"""
|
|
76
76
|
|
|
77
77
|
|
|
78
|
-
def run(
|
|
79
|
-
|
|
78
|
+
def run(
|
|
79
|
+
name: str | None = None,
|
|
80
|
+
description: str | None = None,
|
|
81
|
+
rows: str | None = None,
|
|
82
|
+
fmt: str | None = None,
|
|
83
|
+
yes: bool = False,
|
|
84
|
+
):
|
|
85
|
+
"""Create a schema template."""
|
|
80
86
|
rprint(Panel("[bold cyan]fauxdata init[/bold cyan] — schema template creator", expand=False))
|
|
81
87
|
|
|
82
88
|
schema_name = name or questionary.text(
|
|
@@ -87,30 +93,37 @@ def run(name: str | None = None):
|
|
|
87
93
|
if schema_name is None:
|
|
88
94
|
raise typer.Abort()
|
|
89
95
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
96
|
+
desc = description if description is not None else (
|
|
97
|
+
questionary.text(
|
|
98
|
+
"Short description:",
|
|
99
|
+
default=f"{schema_name} dataset",
|
|
100
|
+
).ask() or ""
|
|
101
|
+
)
|
|
94
102
|
|
|
95
|
-
|
|
103
|
+
n_rows = rows if rows is not None else (
|
|
104
|
+
questionary.text("Default number of rows:", default="1000").ask() or "1000"
|
|
105
|
+
)
|
|
96
106
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
107
|
+
output_fmt = fmt if fmt is not None else (
|
|
108
|
+
questionary.select(
|
|
109
|
+
"Default output format:",
|
|
110
|
+
choices=["csv", "parquet", "json", "jsonl"],
|
|
111
|
+
).ask() or "csv"
|
|
112
|
+
)
|
|
101
113
|
|
|
102
114
|
output_file = f"{schema_name}.yml"
|
|
103
115
|
out_path = Path(output_file)
|
|
104
116
|
|
|
105
|
-
if out_path.exists():
|
|
117
|
+
if out_path.exists() and not yes:
|
|
106
118
|
overwrite = questionary.confirm(f"{output_file} already exists. Overwrite?", default=False).ask()
|
|
107
119
|
if not overwrite:
|
|
108
120
|
rprint("[yellow]Aborted.[/yellow]")
|
|
109
121
|
raise typer.Exit()
|
|
110
122
|
|
|
111
|
-
content = TEMPLATE.format(name=schema_name, description=
|
|
123
|
+
content = TEMPLATE.format(name=schema_name, description=desc, rows=n_rows, fmt=output_fmt)
|
|
112
124
|
out_path.write_text(content)
|
|
113
125
|
|
|
114
126
|
rprint(f"[green]Created[/green] [bold]{output_file}[/bold]")
|
|
115
127
|
rprint("[dim]Edit the schema then run:[/dim]")
|
|
116
128
|
rprint(f" [cyan]fauxdata generate {output_file} --validate[/cyan]")
|
|
129
|
+
rprint(f"schema_path: {output_file}")
|
|
@@ -16,7 +16,17 @@ def generate_dataset(schema: SchemaConfig, rows: int | None = None, seed: int |
|
|
|
16
16
|
pb_schema = _build_pb_schema(schema)
|
|
17
17
|
country = schema.locale or "US"
|
|
18
18
|
|
|
19
|
-
|
|
19
|
+
try:
|
|
20
|
+
df = pb.generate_dataset(pb_schema, n=n, seed=rng_seed, country=country)
|
|
21
|
+
except ValueError as e:
|
|
22
|
+
if "unique values" in str(e):
|
|
23
|
+
unique_cols = [c.name for c in schema.columns if c.unique]
|
|
24
|
+
raise ValueError(
|
|
25
|
+
f"Cannot generate {n} rows: a unique column has too narrow a range "
|
|
26
|
+
f"(unique columns: {', '.join(unique_cols) or 'n/a'}). "
|
|
27
|
+
f"Reduce --rows, widen min/max, or drop 'unique'."
|
|
28
|
+
) from e
|
|
29
|
+
raise
|
|
20
30
|
return df
|
|
21
31
|
|
|
22
32
|
|
|
@@ -82,7 +92,12 @@ def _col_to_field(col: ColumnSchema):
|
|
|
82
92
|
elif col.preset:
|
|
83
93
|
return pb.string_field(preset=col.preset, nullable=nullable, unique=unique, **np_kwargs)
|
|
84
94
|
else:
|
|
85
|
-
|
|
95
|
+
len_kwargs = {}
|
|
96
|
+
if col.min_length is not None:
|
|
97
|
+
len_kwargs["min_length"] = col.min_length
|
|
98
|
+
if col.max_length is not None:
|
|
99
|
+
len_kwargs["max_length"] = col.max_length
|
|
100
|
+
return pb.string_field(nullable=nullable, unique=unique, **len_kwargs, **np_kwargs)
|
|
86
101
|
|
|
87
102
|
else:
|
|
88
103
|
return pb.string_field(nullable=nullable, **np_kwargs)
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
"""fauxdata CLI entry point."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
import pyfiglet
|
|
8
|
+
import typer
|
|
9
|
+
from rich import print as rprint
|
|
10
|
+
from rich.console import Console
|
|
11
|
+
|
|
12
|
+
from fauxdata import __version__
|
|
13
|
+
|
|
14
|
+
app = typer.Typer(
|
|
15
|
+
name="fauxdata",
|
|
16
|
+
help="Generate and validate fake datasets from YAML schemas.",
|
|
17
|
+
add_completion=False,
|
|
18
|
+
)
|
|
19
|
+
console = Console()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _banner():
|
|
23
|
+
banner = pyfiglet.figlet_format("fauxdata", font="slant")
|
|
24
|
+
rprint(f"[bold cyan]{banner}[/bold cyan]")
|
|
25
|
+
rprint("[dim]Generate and validate realistic fake datasets[/dim]\n")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _version_callback(value: bool):
|
|
29
|
+
if value:
|
|
30
|
+
rprint(f"fauxdata {__version__}")
|
|
31
|
+
raise typer.Exit()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@app.callback(invoke_without_command=True)
|
|
35
|
+
def main(
|
|
36
|
+
ctx: typer.Context,
|
|
37
|
+
version: Optional[bool] = typer.Option(
|
|
38
|
+
None, "--version", "-V", callback=_version_callback, is_eager=True,
|
|
39
|
+
help="Show version and exit.",
|
|
40
|
+
),
|
|
41
|
+
):
|
|
42
|
+
if ctx.invoked_subcommand is None:
|
|
43
|
+
_banner()
|
|
44
|
+
rprint(ctx.get_help())
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@app.command(
|
|
48
|
+
"init",
|
|
49
|
+
epilog=(
|
|
50
|
+
"Examples:\n\n"
|
|
51
|
+
" fauxdata init --name people\n\n"
|
|
52
|
+
" fauxdata init --name orders --rows 500 --format parquet\n\n"
|
|
53
|
+
" fauxdata init --name events --description 'clickstream events' --rows 10000 --format jsonl --yes\n"
|
|
54
|
+
),
|
|
55
|
+
)
|
|
56
|
+
def init_cmd(
|
|
57
|
+
name: Optional[str] = typer.Option(None, "--name", "-n", help="Schema name"),
|
|
58
|
+
description: Optional[str] = typer.Option(None, "--description", "-d", help="Short description"),
|
|
59
|
+
rows: Optional[str] = typer.Option(None, "--rows", "-r", help="Default number of rows (default: 1000)"),
|
|
60
|
+
fmt: Optional[str] = typer.Option(None, "--format", "-f", help="Output format: csv, parquet, json, jsonl (default: csv)"),
|
|
61
|
+
yes: bool = typer.Option(False, "--yes", "-y", help="Overwrite existing file without confirmation"),
|
|
62
|
+
):
|
|
63
|
+
"""Create a schema template (interactive if flags are omitted)."""
|
|
64
|
+
from fauxdata.commands.init import run
|
|
65
|
+
run(name=name, description=description, rows=rows, fmt=fmt, yes=yes)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@app.command(
|
|
69
|
+
"infer",
|
|
70
|
+
epilog=(
|
|
71
|
+
"Examples:\n\n"
|
|
72
|
+
" fauxdata infer real_data.csv\n\n"
|
|
73
|
+
" fauxdata infer real_data.parquet --out schema.yml --name people\n\n"
|
|
74
|
+
" fauxdata infer real_data.csv --categorical-threshold 50 --no-detect-presets\n\n"
|
|
75
|
+
" fauxdata infer big.parquet --sample-size 10000 --out - | head\n"
|
|
76
|
+
),
|
|
77
|
+
)
|
|
78
|
+
def infer_cmd(
|
|
79
|
+
dataset: str = typer.Argument(..., help="Path to a real dataset (csv, parquet, json, jsonl)"),
|
|
80
|
+
out: Optional[str] = typer.Option(None, "--out", "-o", help="Output YAML path (use - for stdout; default: <name>.yml)"),
|
|
81
|
+
name: Optional[str] = typer.Option(None, "--name", "-n", help="Schema name (default: dataset file stem)"),
|
|
82
|
+
rows: Optional[int] = typer.Option(None, "--rows", "-r", help="Rows to generate (default: rows in source table)"),
|
|
83
|
+
fmt: str = typer.Option("csv", "--format", "-f", help="Default output format in the schema: csv, parquet, json, jsonl"),
|
|
84
|
+
categorical_threshold: float = typer.Option(20, "--categorical-threshold", help="Max unique values (int) or fraction (0-1) to treat a column as categorical"),
|
|
85
|
+
detect_presets: bool = typer.Option(True, "--detect-presets/--no-detect-presets", help="Match string columns to known presets (email, url, ...)"),
|
|
86
|
+
sample_size: Optional[int] = typer.Option(None, "--sample-size", help="Sample N rows before analysis (for very large tables)"),
|
|
87
|
+
):
|
|
88
|
+
"""Infer a YAML schema from a real dataset (powered by pointblank schema inference)."""
|
|
89
|
+
from fauxdata.commands.infer import run
|
|
90
|
+
run(
|
|
91
|
+
dataset_path=dataset, out=out, name=name, rows=rows, fmt=fmt,
|
|
92
|
+
categorical_threshold=categorical_threshold, detect_presets=detect_presets,
|
|
93
|
+
sample_size=sample_size,
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@app.command(
|
|
98
|
+
"generate",
|
|
99
|
+
epilog=(
|
|
100
|
+
"Examples:\n\n"
|
|
101
|
+
" fauxdata generate people.yml\n\n"
|
|
102
|
+
" fauxdata generate people.yml --rows 500 --format csv --out out.csv\n\n"
|
|
103
|
+
" fauxdata generate people.yml --seed 42 --validate\n\n"
|
|
104
|
+
" fauxdata generate people.yml --out - --format jsonl | jq .\n\n"
|
|
105
|
+
" fauxdata generate people.yml --dry-run\n"
|
|
106
|
+
),
|
|
107
|
+
)
|
|
108
|
+
def generate_cmd(
|
|
109
|
+
schema: str = typer.Argument(..., help="Path to YAML schema file"),
|
|
110
|
+
rows: Optional[int] = typer.Option(None, "--rows", "-r", help="Number of rows to generate"),
|
|
111
|
+
out: Optional[str] = typer.Option(None, "--out", "-o", help="Output file path (use - for stdout)"),
|
|
112
|
+
fmt: Optional[str] = typer.Option(None, "--format", "-f", help="Output format: csv, parquet, json, jsonl"),
|
|
113
|
+
seed: Optional[int] = typer.Option(None, "--seed", "-s", help="Random seed for reproducibility"),
|
|
114
|
+
validate: bool = typer.Option(False, "--validate", "-v", help="Run validation after generating"),
|
|
115
|
+
dry_run: bool = typer.Option(False, "--dry-run", help="Show what would be generated without writing files"),
|
|
116
|
+
):
|
|
117
|
+
"""Generate a fake dataset from a YAML schema."""
|
|
118
|
+
from fauxdata.commands.generate import run
|
|
119
|
+
run(schema_path=schema, rows=rows, out=out, fmt=fmt, seed=seed, validate=validate, dry_run=dry_run)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@app.command(
|
|
123
|
+
"validate",
|
|
124
|
+
epilog=(
|
|
125
|
+
"Examples:\n\n"
|
|
126
|
+
" fauxdata validate people.csv people.yml\n\n"
|
|
127
|
+
" fauxdata validate out/orders.parquet schemas/orders.yml\n"
|
|
128
|
+
),
|
|
129
|
+
)
|
|
130
|
+
def validate_cmd(
|
|
131
|
+
dataset: str = typer.Argument(..., help="Path to dataset file (csv, parquet, json, jsonl)"),
|
|
132
|
+
schema: str = typer.Argument(..., help="Path to YAML schema file"),
|
|
133
|
+
):
|
|
134
|
+
"""Validate an existing dataset against a YAML schema."""
|
|
135
|
+
from fauxdata.commands.validate import run
|
|
136
|
+
run(dataset_path=dataset, schema_path=schema)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@app.command(
|
|
140
|
+
"preview",
|
|
141
|
+
epilog=(
|
|
142
|
+
"Examples:\n\n"
|
|
143
|
+
" fauxdata preview people.csv\n\n"
|
|
144
|
+
" fauxdata preview people.parquet --rows 20\n"
|
|
145
|
+
),
|
|
146
|
+
)
|
|
147
|
+
def preview_cmd(
|
|
148
|
+
dataset: str = typer.Argument(..., help="Path to dataset file"),
|
|
149
|
+
rows: int = typer.Option(10, "--rows", "-r", help="Number of rows to preview"),
|
|
150
|
+
):
|
|
151
|
+
"""Show a preview and column statistics for a dataset."""
|
|
152
|
+
from fauxdata.commands.preview import run
|
|
153
|
+
run(dataset_path=dataset, rows=rows)
|