fauxdata-cli 0.1.6__tar.gz → 0.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/PKG-INFO +55 -9
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/README.md +52 -6
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/pyproject.toml +5 -2
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/src/fauxdata/__init__.py +1 -1
- fauxdata_cli-0.2.1/src/fauxdata/commands/agent.py +160 -0
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/src/fauxdata/commands/generate.py +34 -2
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/src/fauxdata/commands/infer.py +37 -9
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/src/fauxdata/commands/preview.py +38 -24
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/src/fauxdata/commands/validate.py +22 -6
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/src/fauxdata/generator.py +24 -11
- fauxdata_cli-0.2.1/src/fauxdata/jsonmode.py +19 -0
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/src/fauxdata/main.py +34 -6
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/src/fauxdata/schema.py +1 -0
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/src/fauxdata/validator.py +27 -18
- fauxdata_cli-0.2.1/tests/test_agent.py +163 -0
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/tests/test_cli.py +13 -0
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/tests/test_generator.py +40 -0
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/tests/test_infer.py +51 -0
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/tests/test_schema.py +11 -0
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/tests/test_validator.py +67 -1
- fauxdata_cli-0.1.6/.claude/settings.local.json +0 -12
- fauxdata_cli-0.1.6/.python-version +0 -1
- fauxdata_cli-0.1.6/LOG.md +0 -63
- fauxdata_cli-0.1.6/docs/deployment.md +0 -90
- fauxdata_cli-0.1.6/docs/index.html +0 -1029
- fauxdata_cli-0.1.6/docs/share.png +0 -0
- fauxdata_cli-0.1.6/share.png +0 -0
- fauxdata_cli-0.1.6/tasks/todo.md +0 -52
- fauxdata_cli-0.1.6/uv.lock +0 -900
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/.gitignore +0 -0
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/LICENSE +0 -0
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/schemas/events.yml +0 -0
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/schemas/orders.yml +0 -0
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/schemas/people.yml +0 -0
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/src/fauxdata/commands/__init__.py +0 -0
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/src/fauxdata/commands/init.py +0 -0
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/src/fauxdata/output.py +0 -0
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/tests/__init__.py +0 -0
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/tests/conftest.py +0 -0
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/tests/test_new_fields.py +0 -0
- {fauxdata_cli-0.1.6 → fauxdata_cli-0.2.1}/tests/test_output.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
2
|
Name: fauxdata-cli
|
|
3
|
-
Version: 0.1
|
|
3
|
+
Version: 0.2.1
|
|
4
4
|
Summary: CLI for generating and validating fake datasets
|
|
5
5
|
Project-URL: Homepage, https://aborruso.github.io/fauxdata/
|
|
6
6
|
Project-URL: Repository, https://github.com/aborruso/fauxdata
|
|
@@ -13,7 +13,7 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.12
|
|
14
14
|
Classifier: Programming Language :: Python :: 3.13
|
|
15
15
|
Requires-Python: >=3.11
|
|
16
|
-
Requires-Dist: pointblank>=0
|
|
16
|
+
Requires-Dist: pointblank>=1.1.0
|
|
17
17
|
Requires-Dist: polars>=1.0
|
|
18
18
|
Requires-Dist: pyfiglet>=1.0
|
|
19
19
|
Requires-Dist: pyyaml>=6.0
|
|
@@ -93,7 +93,7 @@ fauxdata init --name orders
|
|
|
93
93
|
|
|
94
94
|
## Schema format
|
|
95
95
|
|
|
96
|
-
A schema is a YAML file that describes the structure of your dataset. Here is a realistic example for a people dataset:
|
|
96
|
+
A schema is a YAML file that describes the structure of your dataset. Here is a realistic example for a people dataset, adapted from [`schemas/people.yml`](https://github.com/aborruso/fauxdata/blob/main/schemas/people.yml):
|
|
97
97
|
|
|
98
98
|
```yaml
|
|
99
99
|
name: people
|
|
@@ -170,12 +170,26 @@ validation:
|
|
|
170
170
|
| Type | Description | Options |
|
|
171
171
|
|------|-------------|---------|
|
|
172
172
|
| `int` | Integer | `min`, `max`, `unique` |
|
|
173
|
-
| `float` | Floating point | `min`, `max` |
|
|
173
|
+
| `float` | Floating point | `min`, `max`, `precision` |
|
|
174
174
|
| `string` | Text | `preset`, `values`, `pattern`, `unique`, `min_length`, `max_length` |
|
|
175
175
|
| `bool` | Boolean | — |
|
|
176
176
|
| `date` | Date | `min`, `max` (ISO format) |
|
|
177
177
|
| `datetime` | Datetime | `min`, `max` (ISO format) |
|
|
178
178
|
|
|
179
|
+
### Float precision
|
|
180
|
+
|
|
181
|
+
`precision` sets the number of decimal places of a `float` column, e.g. `precision: 2` for prices:
|
|
182
|
+
|
|
183
|
+
```yaml
|
|
184
|
+
price:
|
|
185
|
+
type: float
|
|
186
|
+
min: 1.0
|
|
187
|
+
max: 199.999
|
|
188
|
+
precision: 2
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
Rounding never pushes a value outside `min`/`max`. When a bound has more decimals than `precision`, fauxdata moves it inward to the nearest value that fits: in the example above `max: 199.999` becomes `199.99`, so no generated price can round up to `200.00`. If no value with the requested decimals fits between `min` and `max` (e.g. `min: 1.001`, `max: 1.009`, `precision: 2`), `generate` stops with an error instead of producing out-of-range data. With `unique: true`, values are unique after rounding.
|
|
192
|
+
|
|
179
193
|
### String presets
|
|
180
194
|
|
|
181
195
|
Presets generate realistic, locale-aware values. Set `locale` at the schema level to control the country.
|
|
@@ -230,6 +244,7 @@ Supported locales include: `US`, `IT`, `DE`, `FR`, `ES`, `JP`, `BR`, `PL`, `NL`,
|
|
|
230
244
|
| `col_vals_between` | Value in range | `column`, `min`, `max` |
|
|
231
245
|
| `col_vals_regex` | Matches pattern | `column`, `pattern` |
|
|
232
246
|
| `col_vals_in_set` | Value in allowed set | `column`, `values` |
|
|
247
|
+
| `col_vals_str_len` | String length in range (nulls are ignored) | `column`, `min` and/or `max` |
|
|
233
248
|
| `col_vals_gt` / `col_vals_lt` | Greater / less than | `column`, `min` / `max` |
|
|
234
249
|
| `col_vals_ge` / `col_vals_le` | Greater / less or equal | `column`, `min` / `max` |
|
|
235
250
|
| `rows_distinct` | Unique rows | `columns` |
|
|
@@ -256,6 +271,7 @@ fauxdata generate schemas/people.yml --out - --format jsonl | wc -l
|
|
|
256
271
|
| `--format` | `-f` | from schema | Output format: `csv`, `parquet`, `json`, `jsonl`, `jsonlines` |
|
|
257
272
|
| `--seed` | `-s` | from schema | Random seed for reproducibility |
|
|
258
273
|
| `--validate` | `-v` | off | Run validation rules after generating |
|
|
274
|
+
| `--json` | | off | Print only a JSON summary to stdout (see [Using fauxdata with AI agents](#using-fauxdata-with-ai-agents)) |
|
|
259
275
|
|
|
260
276
|
When `--out -` is used, all output messages are suppressed and only data is written to stdout.
|
|
261
277
|
|
|
@@ -317,15 +333,45 @@ Interactive wizard to create a new schema template. Asks for name, description,
|
|
|
317
333
|
|
|
318
334
|
---
|
|
319
335
|
|
|
336
|
+
## Using fauxdata with AI agents
|
|
337
|
+
|
|
338
|
+
fauxdata is designed to be driven by AI agents as well as by people.
|
|
339
|
+
|
|
340
|
+
**Start with `fauxdata agent`.** It prints a Markdown guide with everything an agent needs to use the tool without reading this README: the workflow, the schema format, column types and options, presets, validation rules, the JSON output of each command, exit codes and a complete example schema. The lists of types, presets and rules are built from the code, so the guide always matches the installed version.
|
|
341
|
+
|
|
342
|
+
```
|
|
343
|
+
fauxdata agent > fauxdata-guide.md
|
|
344
|
+
```
|
|
345
|
+
|
|
346
|
+
**Add `--json` to get machine-readable results.** `generate`, `validate`, `infer` and `preview` accept `--json`: stdout then contains a single JSON object and nothing else (no panels, tables or colors).
|
|
347
|
+
|
|
348
|
+
```
|
|
349
|
+
fauxdata infer real.csv --out schema.yml --json
|
|
350
|
+
{"schema_path": "schema.yml", "columns": ["id", "email", "age"], "validation_rules": 4}
|
|
351
|
+
|
|
352
|
+
fauxdata generate schema.yml --out data.csv --validate --json
|
|
353
|
+
{"output_path": "data.csv", "format": "csv", "rows": 1000, "seed": 42, "dry_run": false, "validation": {"passed": true, "results": [...]}}
|
|
354
|
+
|
|
355
|
+
fauxdata validate data.csv schema.yml --json
|
|
356
|
+
{"passed": true, "rows": 1000, "results": [{"step": 1, "rule": "col_vals_not_null", "column": "id", "passed": 1000, "failed": 0, "total": 1000, "ok": true, "evaluated": true}, ...]}
|
|
357
|
+
```
|
|
358
|
+
|
|
359
|
+
- Errors come back as `{"error": "..."}` on stdout with exit code `1`.
|
|
360
|
+
- A step that could not run (for example, its column is missing from the data) has `"evaluated": false` and `"ok": false`; the human-readable table shows it as `NOT RUN`.
|
|
361
|
+
- Exit codes: `0` success, `1` error or at least one validation rule failed.
|
|
362
|
+
- `--json` cannot be combined with `--out -`, which already writes data (or YAML, for `infer`) to stdout.
|
|
363
|
+
|
|
364
|
+
---
|
|
365
|
+
|
|
320
366
|
## Example schemas
|
|
321
367
|
|
|
322
|
-
Three ready-to-use schemas are included in `schemas
|
|
368
|
+
Three ready-to-use schemas are included in [`schemas/`](https://github.com/aborruso/fauxdata/tree/main/schemas):
|
|
323
369
|
|
|
324
370
|
| Schema | Domain | Columns |
|
|
325
371
|
|--------|--------|---------|
|
|
326
|
-
| `people.yml` | Personal data | id, name, email, age, city, country_code, active, signup_date, score |
|
|
327
|
-
| `orders.yml` | E-commerce | order_id, customer_id, product, amount, status, created_at |
|
|
328
|
-
| `events.yml` | Analytics | event_id, user_id, event_type, timestamp, ip, user_agent, session_duration |
|
|
372
|
+
| [`people.yml`](https://github.com/aborruso/fauxdata/blob/main/schemas/people.yml) | Personal data | id, name, email, age, city, country_code, active, signup_date, score |
|
|
373
|
+
| [`orders.yml`](https://github.com/aborruso/fauxdata/blob/main/schemas/orders.yml) | E-commerce | order_id, customer_id, product, amount, status, created_at |
|
|
374
|
+
| [`events.yml`](https://github.com/aborruso/fauxdata/blob/main/schemas/events.yml) | Analytics | event_id, user_id, event_type, timestamp, ip, user_agent, session_duration |
|
|
329
375
|
|
|
330
376
|
---
|
|
331
377
|
|
|
@@ -69,7 +69,7 @@ fauxdata init --name orders
|
|
|
69
69
|
|
|
70
70
|
## Schema format
|
|
71
71
|
|
|
72
|
-
A schema is a YAML file that describes the structure of your dataset. Here is a realistic example for a people dataset:
|
|
72
|
+
A schema is a YAML file that describes the structure of your dataset. Here is a realistic example for a people dataset, adapted from [`schemas/people.yml`](https://github.com/aborruso/fauxdata/blob/main/schemas/people.yml):
|
|
73
73
|
|
|
74
74
|
```yaml
|
|
75
75
|
name: people
|
|
@@ -146,12 +146,26 @@ validation:
|
|
|
146
146
|
| Type | Description | Options |
|
|
147
147
|
|------|-------------|---------|
|
|
148
148
|
| `int` | Integer | `min`, `max`, `unique` |
|
|
149
|
-
| `float` | Floating point | `min`, `max` |
|
|
149
|
+
| `float` | Floating point | `min`, `max`, `precision` |
|
|
150
150
|
| `string` | Text | `preset`, `values`, `pattern`, `unique`, `min_length`, `max_length` |
|
|
151
151
|
| `bool` | Boolean | — |
|
|
152
152
|
| `date` | Date | `min`, `max` (ISO format) |
|
|
153
153
|
| `datetime` | Datetime | `min`, `max` (ISO format) |
|
|
154
154
|
|
|
155
|
+
### Float precision
|
|
156
|
+
|
|
157
|
+
`precision` sets the number of decimal places of a `float` column, e.g. `precision: 2` for prices:
|
|
158
|
+
|
|
159
|
+
```yaml
|
|
160
|
+
price:
|
|
161
|
+
type: float
|
|
162
|
+
min: 1.0
|
|
163
|
+
max: 199.999
|
|
164
|
+
precision: 2
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
Rounding never pushes a value outside `min`/`max`. When a bound has more decimals than `precision`, fauxdata moves it inward to the nearest value that fits: in the example above `max: 199.999` becomes `199.99`, so no generated price can round up to `200.00`. If no value with the requested decimals fits between `min` and `max` (e.g. `min: 1.001`, `max: 1.009`, `precision: 2`), `generate` stops with an error instead of producing out-of-range data. With `unique: true`, values are unique after rounding.
|
|
168
|
+
|
|
155
169
|
### String presets
|
|
156
170
|
|
|
157
171
|
Presets generate realistic, locale-aware values. Set `locale` at the schema level to control the country.
|
|
@@ -206,6 +220,7 @@ Supported locales include: `US`, `IT`, `DE`, `FR`, `ES`, `JP`, `BR`, `PL`, `NL`,
|
|
|
206
220
|
| `col_vals_between` | Value in range | `column`, `min`, `max` |
|
|
207
221
|
| `col_vals_regex` | Matches pattern | `column`, `pattern` |
|
|
208
222
|
| `col_vals_in_set` | Value in allowed set | `column`, `values` |
|
|
223
|
+
| `col_vals_str_len` | String length in range (nulls are ignored) | `column`, `min` and/or `max` |
|
|
209
224
|
| `col_vals_gt` / `col_vals_lt` | Greater / less than | `column`, `min` / `max` |
|
|
210
225
|
| `col_vals_ge` / `col_vals_le` | Greater / less or equal | `column`, `min` / `max` |
|
|
211
226
|
| `rows_distinct` | Unique rows | `columns` |
|
|
@@ -232,6 +247,7 @@ fauxdata generate schemas/people.yml --out - --format jsonl | wc -l
|
|
|
232
247
|
| `--format` | `-f` | from schema | Output format: `csv`, `parquet`, `json`, `jsonl`, `jsonlines` |
|
|
233
248
|
| `--seed` | `-s` | from schema | Random seed for reproducibility |
|
|
234
249
|
| `--validate` | `-v` | off | Run validation rules after generating |
|
|
250
|
+
| `--json` | | off | Print only a JSON summary to stdout (see [Using fauxdata with AI agents](#using-fauxdata-with-ai-agents)) |
|
|
235
251
|
|
|
236
252
|
When `--out -` is used, all output messages are suppressed and only data is written to stdout.
|
|
237
253
|
|
|
@@ -293,15 +309,45 @@ Interactive wizard to create a new schema template. Asks for name, description,
|
|
|
293
309
|
|
|
294
310
|
---
|
|
295
311
|
|
|
312
|
+
## Using fauxdata with AI agents
|
|
313
|
+
|
|
314
|
+
fauxdata is designed to be driven by AI agents as well as by people.
|
|
315
|
+
|
|
316
|
+
**Start with `fauxdata agent`.** It prints a Markdown guide with everything an agent needs to use the tool without reading this README: the workflow, the schema format, column types and options, presets, validation rules, the JSON output of each command, exit codes and a complete example schema. The lists of types, presets and rules are built from the code, so the guide always matches the installed version.
|
|
317
|
+
|
|
318
|
+
```
|
|
319
|
+
fauxdata agent > fauxdata-guide.md
|
|
320
|
+
```
|
|
321
|
+
|
|
322
|
+
**Add `--json` to get machine-readable results.** `generate`, `validate`, `infer` and `preview` accept `--json`: stdout then contains a single JSON object and nothing else (no panels, tables or colors).
|
|
323
|
+
|
|
324
|
+
```
|
|
325
|
+
fauxdata infer real.csv --out schema.yml --json
|
|
326
|
+
{"schema_path": "schema.yml", "columns": ["id", "email", "age"], "validation_rules": 4}
|
|
327
|
+
|
|
328
|
+
fauxdata generate schema.yml --out data.csv --validate --json
|
|
329
|
+
{"output_path": "data.csv", "format": "csv", "rows": 1000, "seed": 42, "dry_run": false, "validation": {"passed": true, "results": [...]}}
|
|
330
|
+
|
|
331
|
+
fauxdata validate data.csv schema.yml --json
|
|
332
|
+
{"passed": true, "rows": 1000, "results": [{"step": 1, "rule": "col_vals_not_null", "column": "id", "passed": 1000, "failed": 0, "total": 1000, "ok": true, "evaluated": true}, ...]}
|
|
333
|
+
```
|
|
334
|
+
|
|
335
|
+
- Errors come back as `{"error": "..."}` on stdout with exit code `1`.
|
|
336
|
+
- A step that could not run (for example, its column is missing from the data) has `"evaluated": false` and `"ok": false`; the human-readable table shows it as `NOT RUN`.
|
|
337
|
+
- Exit codes: `0` success, `1` error or at least one validation rule failed.
|
|
338
|
+
- `--json` cannot be combined with `--out -`, which already writes data (or YAML, for `infer`) to stdout.
|
|
339
|
+
|
|
340
|
+
---
|
|
341
|
+
|
|
296
342
|
## Example schemas
|
|
297
343
|
|
|
298
|
-
Three ready-to-use schemas are included in `schemas
|
|
344
|
+
Three ready-to-use schemas are included in [`schemas/`](https://github.com/aborruso/fauxdata/tree/main/schemas):
|
|
299
345
|
|
|
300
346
|
| Schema | Domain | Columns |
|
|
301
347
|
|--------|--------|---------|
|
|
302
|
-
| `people.yml` | Personal data | id, name, email, age, city, country_code, active, signup_date, score |
|
|
303
|
-
| `orders.yml` | E-commerce | order_id, customer_id, product, amount, status, created_at |
|
|
304
|
-
| `events.yml` | Analytics | event_id, user_id, event_type, timestamp, ip, user_agent, session_duration |
|
|
348
|
+
| [`people.yml`](https://github.com/aborruso/fauxdata/blob/main/schemas/people.yml) | Personal data | id, name, email, age, city, country_code, active, signup_date, score |
|
|
349
|
+
| [`orders.yml`](https://github.com/aborruso/fauxdata/blob/main/schemas/orders.yml) | E-commerce | order_id, customer_id, product, amount, status, created_at |
|
|
350
|
+
| [`events.yml`](https://github.com/aborruso/fauxdata/blob/main/schemas/events.yml) | Analytics | event_id, user_id, event_type, timestamp, ip, user_agent, session_duration |
|
|
305
351
|
|
|
306
352
|
---
|
|
307
353
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "fauxdata-cli"
|
|
3
|
-
version = "0.1
|
|
3
|
+
version = "0.2.1"
|
|
4
4
|
description = "CLI for generating and validating fake datasets"
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
requires-python = ">=3.11"
|
|
@@ -13,7 +13,7 @@ classifiers = [
|
|
|
13
13
|
"Operating System :: OS Independent",
|
|
14
14
|
]
|
|
15
15
|
dependencies = [
|
|
16
|
-
"pointblank>=0
|
|
16
|
+
"pointblank>=1.1.0",
|
|
17
17
|
"typer>=0.12",
|
|
18
18
|
"rich>=13",
|
|
19
19
|
"pyfiglet>=1.0",
|
|
@@ -46,3 +46,6 @@ build-backend = "hatchling.build"
|
|
|
46
46
|
|
|
47
47
|
[tool.hatch.build.targets.wheel]
|
|
48
48
|
packages = ["src/fauxdata"]
|
|
49
|
+
|
|
50
|
+
[tool.hatch.build.targets.sdist]
|
|
51
|
+
only-include = ["src", "tests", "schemas", "README.md", "LICENSE", "pyproject.toml"]
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
"""fauxdata agent command - a Markdown guide for AI agents.
|
|
2
|
+
|
|
3
|
+
The lists of column types, presets and validation rules are built from the
|
|
4
|
+
constants in ``fauxdata.schema``, so the guide cannot drift from the code.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import sys
|
|
10
|
+
|
|
11
|
+
from fauxdata import __version__
|
|
12
|
+
from fauxdata.schema import STRING_PRESETS, VALID_RULES, VALID_TYPES
|
|
13
|
+
|
|
14
|
+
TYPE_OPTIONS = {
|
|
15
|
+
"int": "`min`, `max`, `unique`, `values`",
|
|
16
|
+
"float": "`min`, `max`, `precision`, `unique`, `values`",
|
|
17
|
+
"string": "`preset`, `values`, `pattern`, `unique`, `min_length`, `max_length`",
|
|
18
|
+
"bool": "-",
|
|
19
|
+
"date": "`min`, `max` (ISO date), `unique`",
|
|
20
|
+
"datetime": "`min`, `max` (ISO datetime), `unique`",
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
RULE_PARAMS = {
|
|
24
|
+
"col_vals_not_null": ("`columns`", "no nulls"),
|
|
25
|
+
"col_vals_between": ("`column`, `min`, `max`", "value in range"),
|
|
26
|
+
"col_vals_gt": ("`column`, `min`", "value > min"),
|
|
27
|
+
"col_vals_lt": ("`column`, `max`", "value < max"),
|
|
28
|
+
"col_vals_ge": ("`column`, `min`", "value >= min"),
|
|
29
|
+
"col_vals_le": ("`column`, `max`", "value <= max"),
|
|
30
|
+
"col_vals_regex": ("`column`, `pattern`", "value matches the regex"),
|
|
31
|
+
"col_vals_in_set": ("`column`, `values`", "value in the allowed set"),
|
|
32
|
+
"col_vals_str_len": ("`column`, `min` and/or `max`", "string length in range"),
|
|
33
|
+
"rows_distinct": ("`columns`", "no duplicate keys (rows with a null key are skipped)"),
|
|
34
|
+
"col_exists": ("`columns`", "column present"),
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
EXAMPLE = """\
|
|
38
|
+
name: people
|
|
39
|
+
rows: 1000
|
|
40
|
+
seed: 42
|
|
41
|
+
locale: IT
|
|
42
|
+
output:
|
|
43
|
+
format: csv
|
|
44
|
+
path: people.csv
|
|
45
|
+
columns:
|
|
46
|
+
id:
|
|
47
|
+
type: int
|
|
48
|
+
unique: true
|
|
49
|
+
min: 1
|
|
50
|
+
max: 99999
|
|
51
|
+
name:
|
|
52
|
+
type: string
|
|
53
|
+
preset: name
|
|
54
|
+
email:
|
|
55
|
+
type: string
|
|
56
|
+
preset: email
|
|
57
|
+
age:
|
|
58
|
+
type: int
|
|
59
|
+
min: 18
|
|
60
|
+
max: 90
|
|
61
|
+
null_probability: 0.05
|
|
62
|
+
score:
|
|
63
|
+
type: float
|
|
64
|
+
min: 0
|
|
65
|
+
max: 100
|
|
66
|
+
precision: 1
|
|
67
|
+
status:
|
|
68
|
+
type: string
|
|
69
|
+
values: [active, inactive, pending]
|
|
70
|
+
validation:
|
|
71
|
+
- rule: col_vals_not_null
|
|
72
|
+
columns: [id, name, email]
|
|
73
|
+
- rule: col_vals_between
|
|
74
|
+
column: age
|
|
75
|
+
min: 18
|
|
76
|
+
max: 90
|
|
77
|
+
- rule: col_vals_in_set
|
|
78
|
+
column: status
|
|
79
|
+
values: [active, inactive, pending]
|
|
80
|
+
- rule: rows_distinct
|
|
81
|
+
columns: [id]
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def guide() -> str:
|
|
86
|
+
types = "\n".join(f"| `{t}` | {TYPE_OPTIONS.get(t, '-')} |" for t in sorted(VALID_TYPES))
|
|
87
|
+
rules = "\n".join(
|
|
88
|
+
f"| `{r}` | {RULE_PARAMS.get(r, ('-', '-'))[0]} | {RULE_PARAMS.get(r, ('-', '-'))[1]} |"
|
|
89
|
+
for r in sorted(VALID_RULES)
|
|
90
|
+
)
|
|
91
|
+
presets = ", ".join(f"`{p}`" for p in sorted(STRING_PRESETS))
|
|
92
|
+
return f"""# fauxdata {__version__} - guide for AI agents
|
|
93
|
+
|
|
94
|
+
fauxdata generates realistic fake datasets from a YAML schema and validates datasets against the same schema.
|
|
95
|
+
|
|
96
|
+
## Workflow
|
|
97
|
+
|
|
98
|
+
1. Write a schema (see the example below), or infer one from a real table:
|
|
99
|
+
`fauxdata infer real.csv --out schema.yml --json`
|
|
100
|
+
2. Generate data and validate it in one step:
|
|
101
|
+
`fauxdata generate schema.yml --out data.csv --validate --json`
|
|
102
|
+
3. Validate any existing file against a schema:
|
|
103
|
+
`fauxdata validate data.csv schema.yml --json`
|
|
104
|
+
4. Inspect a file (column stats and first rows):
|
|
105
|
+
`fauxdata preview data.csv --json`
|
|
106
|
+
|
|
107
|
+
## Machine-readable output
|
|
108
|
+
|
|
109
|
+
- With `--json`, stdout contains exactly one JSON object and nothing else.
|
|
110
|
+
- On error: `{{"error": "..."}}` on stdout, exit code 1.
|
|
111
|
+
- `validate --json`: `{{"passed": bool, "rows": int, "results": [{{"step", "rule", "column", "passed", "failed", "total", "ok", "evaluated"}}]}}`
|
|
112
|
+
- `generate --json`: `{{"output_path", "format", "rows", "seed", "dry_run", "validation": null | {{"passed", "results"}}}}`
|
|
113
|
+
- `infer --json`: `{{"schema_path", "columns": [names], "validation_rules": int}}`
|
|
114
|
+
- `preview --json`: `{{"rows", "columns": [{{"name", "type", "nulls", "unique", "min", "max"}}], "head": [rows]}}`
|
|
115
|
+
- `evaluated: false` means the step could not run (e.g. its column is missing from the data); such a step has `ok: false`.
|
|
116
|
+
- `--json` cannot be combined with `--out -` (that mode writes data or YAML to stdout).
|
|
117
|
+
|
|
118
|
+
## Exit codes
|
|
119
|
+
|
|
120
|
+
- `0`: success, all validation rules passed
|
|
121
|
+
- `1`: error, or at least one validation rule failed
|
|
122
|
+
|
|
123
|
+
## Schema
|
|
124
|
+
|
|
125
|
+
Top-level keys: `name` (required), `columns` (required), `rows`, `seed`, `locale` (ISO country code, e.g. `US`, `IT`, `DE`, `JP`), `description`, `output.format` (`csv`, `parquet`, `json`, `jsonl`), `output.path`, `validation`.
|
|
126
|
+
|
|
127
|
+
Every column needs `type`. Any column also accepts `nullable` and `null_probability` (0-1).
|
|
128
|
+
|
|
129
|
+
| Type | Options |
|
|
130
|
+
|------|---------|
|
|
131
|
+
{types}
|
|
132
|
+
|
|
133
|
+
Notes:
|
|
134
|
+
- `precision` rounds floats; bounds with more decimals are moved inward so values never leave `min`/`max`.
|
|
135
|
+
- `values` restricts a column to a fixed set (strings, ints or floats).
|
|
136
|
+
- `pattern` generates strings matching a regex.
|
|
137
|
+
|
|
138
|
+
### String presets
|
|
139
|
+
|
|
140
|
+
{presets}
|
|
141
|
+
|
|
142
|
+
Presets are locale-aware and coherent within a row (the email matches the name, the city matches the country).
|
|
143
|
+
|
|
144
|
+
## Validation rules
|
|
145
|
+
|
|
146
|
+
| Rule | Parameters | Checks |
|
|
147
|
+
|------|------------|--------|
|
|
148
|
+
{rules}
|
|
149
|
+
|
|
150
|
+
Value rules ignore nulls: use `col_vals_not_null` to forbid them.
|
|
151
|
+
|
|
152
|
+
## Example schema
|
|
153
|
+
|
|
154
|
+
```yaml
|
|
155
|
+
{EXAMPLE}```
|
|
156
|
+
"""
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def run():
|
|
160
|
+
sys.stdout.write(guide())
|
|
@@ -11,6 +11,7 @@ from rich.console import Console
|
|
|
11
11
|
from rich.panel import Panel
|
|
12
12
|
from rich.table import Table
|
|
13
13
|
|
|
14
|
+
from fauxdata import jsonmode
|
|
14
15
|
from fauxdata.generator import generate_dataset
|
|
15
16
|
from fauxdata.output import default_output_path, export_dataset, write_stdout
|
|
16
17
|
from fauxdata.schema import load_schema
|
|
@@ -19,11 +20,13 @@ from fauxdata.validator import validate_dataset
|
|
|
19
20
|
console = Console()
|
|
20
21
|
|
|
21
22
|
|
|
22
|
-
def _generate(schema, n, rng_seed):
|
|
23
|
+
def _generate(schema, n, rng_seed, json_out=False):
|
|
23
24
|
"""Generate, translating generation errors into a clean CLI exit."""
|
|
24
25
|
try:
|
|
25
26
|
return generate_dataset(schema, rows=n, seed=rng_seed)
|
|
26
27
|
except ValueError as e:
|
|
28
|
+
if json_out:
|
|
29
|
+
jsonmode.fail(str(e))
|
|
27
30
|
rprint(f"[red]Error:[/red] {e}")
|
|
28
31
|
raise typer.Exit(code=1)
|
|
29
32
|
|
|
@@ -36,6 +39,7 @@ def run(
|
|
|
36
39
|
seed: Optional[int] = None,
|
|
37
40
|
validate: bool = False,
|
|
38
41
|
dry_run: bool = False,
|
|
42
|
+
json_out: bool = False,
|
|
39
43
|
):
|
|
40
44
|
"""Generate a fake dataset from a YAML schema."""
|
|
41
45
|
schema = load_schema(schema_path)
|
|
@@ -47,6 +51,10 @@ def run(
|
|
|
47
51
|
|
|
48
52
|
stdout_mode = output_path == "-"
|
|
49
53
|
|
|
54
|
+
if json_out:
|
|
55
|
+
_run_json(schema, n, rng_seed, output_fmt, output_path, validate, dry_run)
|
|
56
|
+
return
|
|
57
|
+
|
|
50
58
|
if dry_run:
|
|
51
59
|
rprint(Panel(f"[bold cyan]fauxdata generate --dry-run[/bold cyan] [dim]{schema_path}[/dim]", expand=False))
|
|
52
60
|
rprint("[yellow]Dry run — no files will be written.[/yellow]\n")
|
|
@@ -81,6 +89,25 @@ def run(
|
|
|
81
89
|
_run_validation(df, schema)
|
|
82
90
|
|
|
83
91
|
|
|
92
|
+
def _run_json(schema, n, rng_seed, output_fmt, output_path, validate, dry_run):
|
|
93
|
+
if output_path == "-":
|
|
94
|
+
jsonmode.fail("--json cannot be combined with --out - (the data would go to stdout)")
|
|
95
|
+
summary = {"output_path": output_path, "format": output_fmt, "rows": n, "seed": rng_seed,
|
|
96
|
+
"dry_run": dry_run, "validation": None}
|
|
97
|
+
if dry_run:
|
|
98
|
+
jsonmode.emit(summary)
|
|
99
|
+
return
|
|
100
|
+
df = _generate(schema, n, rng_seed, json_out=True)
|
|
101
|
+
summary["output_path"] = str(export_dataset(df, output_path, output_fmt))
|
|
102
|
+
passed = True
|
|
103
|
+
if validate:
|
|
104
|
+
passed, results = validate_dataset(df, schema)
|
|
105
|
+
summary["validation"] = {"passed": passed, "results": results}
|
|
106
|
+
jsonmode.emit(summary)
|
|
107
|
+
if not passed:
|
|
108
|
+
raise typer.Exit(code=1)
|
|
109
|
+
|
|
110
|
+
|
|
84
111
|
def _print_schema_table(schema, n: int, seed):
|
|
85
112
|
t = Table(title=f"Schema: {schema.name}", show_header=True, header_style="bold magenta")
|
|
86
113
|
t.add_column("Column", style="cyan")
|
|
@@ -132,7 +159,12 @@ def _run_validation(df, schema):
|
|
|
132
159
|
t.add_column("Status")
|
|
133
160
|
|
|
134
161
|
for r in results:
|
|
135
|
-
|
|
162
|
+
if r["ok"]:
|
|
163
|
+
status = "[green]PASS[/green]"
|
|
164
|
+
elif not r["evaluated"]:
|
|
165
|
+
status = "[red]NOT RUN[/red]"
|
|
166
|
+
else:
|
|
167
|
+
status = "[red]FAIL[/red]"
|
|
136
168
|
t.add_row(
|
|
137
169
|
str(r["step"]),
|
|
138
170
|
r["rule"],
|
|
@@ -14,6 +14,8 @@ import pointblank as pb
|
|
|
14
14
|
import polars as pl
|
|
15
15
|
import typer
|
|
16
16
|
import yaml
|
|
17
|
+
|
|
18
|
+
from fauxdata import jsonmode
|
|
17
19
|
from rich import print as rprint
|
|
18
20
|
from rich.panel import Panel
|
|
19
21
|
|
|
@@ -98,6 +100,13 @@ def _build_validation(columns: list[tuple[str, dict]]) -> list[dict]:
|
|
|
98
100
|
if "values" in spec:
|
|
99
101
|
rules.append({"rule": "col_vals_in_set", "column": name, "values": list(spec["values"])})
|
|
100
102
|
|
|
103
|
+
for name, spec in columns:
|
|
104
|
+
# only free-text columns: the generator ignores lengths for preset/pattern/values
|
|
105
|
+
free_text = not any(k in spec for k in ("preset", "pattern", "values"))
|
|
106
|
+
if spec["type"] == "string" and free_text and ("min_length" in spec or "max_length" in spec):
|
|
107
|
+
rules.append({"rule": "col_vals_str_len", "column": name,
|
|
108
|
+
"min": spec.get("min_length"), "max": spec.get("max_length")})
|
|
109
|
+
|
|
101
110
|
unique_cols = [name for name, spec in columns if spec.get("unique")]
|
|
102
111
|
if unique_cols:
|
|
103
112
|
rules.append({"rule": "rows_distinct", "columns": unique_cols})
|
|
@@ -105,7 +114,7 @@ def _build_validation(columns: list[tuple[str, dict]]) -> list[dict]:
|
|
|
105
114
|
return rules
|
|
106
115
|
|
|
107
116
|
|
|
108
|
-
def _load_table(path: str) -> pl.DataFrame:
|
|
117
|
+
def _load_table(path: str, json_out: bool = False) -> pl.DataFrame:
|
|
109
118
|
ext = path.rsplit(".", 1)[-1].lower()
|
|
110
119
|
if ext == "csv":
|
|
111
120
|
# try_parse_dates so date/datetime columns are typed (not frozen as categorical strings)
|
|
@@ -116,10 +125,17 @@ def _load_table(path: str) -> pl.DataFrame:
|
|
|
116
125
|
return pl.read_json(path)
|
|
117
126
|
if ext in ("jsonl", "ndjson"):
|
|
118
127
|
return pl.read_ndjson(path)
|
|
128
|
+
if json_out:
|
|
129
|
+
jsonmode.fail(f"Unsupported file format: .{ext}")
|
|
119
130
|
rprint(f"[red]Unsupported file format: .{ext}[/red]")
|
|
120
131
|
raise typer.Exit(code=1)
|
|
121
132
|
|
|
122
133
|
|
|
134
|
+
def _threshold(value: float) -> int | float:
|
|
135
|
+
"""pointblank reads a float as a fraction of rows: whole numbers >= 1 are counts."""
|
|
136
|
+
return int(value) if value >= 1 and float(value).is_integer() else value
|
|
137
|
+
|
|
138
|
+
|
|
123
139
|
def run(
|
|
124
140
|
dataset_path: str,
|
|
125
141
|
out: str | None = None,
|
|
@@ -129,19 +145,24 @@ def run(
|
|
|
129
145
|
categorical_threshold: float = 20,
|
|
130
146
|
detect_presets: bool = True,
|
|
131
147
|
sample_size: int | None = None,
|
|
148
|
+
json_out: bool = False,
|
|
132
149
|
):
|
|
133
150
|
"""Infer a fauxdata YAML schema from a real dataset."""
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
151
|
+
if json_out and out == "-":
|
|
152
|
+
jsonmode.fail("--json cannot be combined with --out - (the YAML would go to stdout)")
|
|
153
|
+
if not json_out:
|
|
154
|
+
rprint(Panel(
|
|
155
|
+
f"[bold cyan]fauxdata infer[/bold cyan] [dim]{dataset_path}[/dim]",
|
|
156
|
+
expand=False,
|
|
157
|
+
))
|
|
158
|
+
|
|
159
|
+
df = _load_table(dataset_path, json_out)
|
|
160
|
+
if not json_out:
|
|
161
|
+
rprint(f" Loaded [bold]{len(df)}[/bold] rows, [bold]{len(df.columns)}[/bold] columns")
|
|
141
162
|
|
|
142
163
|
pb_schema = pb.schema_from_tbl(
|
|
143
164
|
df,
|
|
144
|
-
categorical_threshold=categorical_threshold,
|
|
165
|
+
categorical_threshold=_threshold(categorical_threshold),
|
|
145
166
|
detect_presets=detect_presets,
|
|
146
167
|
sample_size=sample_size,
|
|
147
168
|
)
|
|
@@ -170,6 +191,13 @@ def run(
|
|
|
170
191
|
return
|
|
171
192
|
|
|
172
193
|
Path(out_path).write_text(text)
|
|
194
|
+
if json_out:
|
|
195
|
+
jsonmode.emit({
|
|
196
|
+
"schema_path": out_path,
|
|
197
|
+
"columns": [col_name for col_name, _ in columns],
|
|
198
|
+
"validation_rules": len(schema_dict["validation"]),
|
|
199
|
+
})
|
|
200
|
+
return
|
|
173
201
|
rprint(f"[green]Created[/green] [bold]{out_path}[/bold] "
|
|
174
202
|
f"([bold]{len(columns)}[/bold] columns, "
|
|
175
203
|
f"[bold]{len(schema_dict['validation'])}[/bold] validation rules)")
|