data-sampler 3.3.1__tar.gz → 3.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {data_sampler-3.3.1 → data_sampler-3.4.1}/CHANGELOG.md +63 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/PKG-INFO +74 -14
- {data_sampler-3.3.1 → data_sampler-3.4.1}/README.md +73 -13
- {data_sampler-3.3.1 → data_sampler-3.4.1}/pyproject.toml +1 -1
- {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/__init__.py +9 -2
- {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/cli.py +135 -3
- data_sampler-3.4.1/src/data_sampler/reduce.py +343 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/report.py +76 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/stats.py +7 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/tui/app.py +444 -45
- {data_sampler-3.3.1 → data_sampler-3.4.1}/tests/test_cli.py +136 -0
- data_sampler-3.4.1/tests/test_reduce.py +291 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/tests/test_stats.py +19 -0
- data_sampler-3.4.1/tests/test_tui.py +620 -0
- data_sampler-3.3.1/tests/test_tui.py +0 -283
- {data_sampler-3.3.1 → data_sampler-3.4.1}/.gitattributes +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/.github/workflows/release.yml +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/.gitignore +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/LICENSE +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/RELEASING.md +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/ROADMAP.md +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/TROUBLESHOOTING.md +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/examples/employees.csv +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/examples/using_data_sampler.ipynb +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/requirements.txt +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/scripts/run-tui.bat +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/scripts/run-tui.sh +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/__main__.py +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/_logging.py +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/_names.py +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/anonymize.py +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/engine.py +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/io.py +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/sampling.py +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/tui/__init__.py +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/workflow.py +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/tests/conftest.py +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/tests/test_anonymize.py +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/tests/test_engine.py +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/tests/test_io.py +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/tests/test_report.py +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/tests/test_sampling.py +0 -0
- {data_sampler-3.3.1 → data_sampler-3.4.1}/tests/test_workflow.py +0 -0
|
@@ -1,5 +1,68 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## v3.4.1 — 2026-07-24
|
|
4
|
+
|
|
5
|
+
- **TUI columns table now breaks the single summary cell into separate
|
|
6
|
+
`mean` / `median` / `mode` / `sd` columns.** Numeric columns fill all four;
|
|
7
|
+
non-numeric columns show `mode` (their most frequent value) with dashes for
|
|
8
|
+
the numeric-only stats. `ColumnStats` gained a `mode` field, populated for
|
|
9
|
+
every column kind by `compute_column_stats`.
|
|
10
|
+
- **Bulk configuration via multi-select on the columns screen.** Ctrl-click
|
|
11
|
+
toggles individual rows, shift-click selects a contiguous range (keyboard:
|
|
12
|
+
`space` toggles the cursor row, `x` clears the selection). Any anonymizer
|
|
13
|
+
choice, stratification-skip, or reduction-skip then applies to every
|
|
14
|
+
selected column at once; the panel title shows the selection count.
|
|
15
|
+
- **Columns can now be excluded from the PCA reduction from the TUI.** A new
|
|
16
|
+
“skip from reduction (PCA)” switch (and the `d` key) sets a per-column
|
|
17
|
+
`skip_reduce` flag that is passed to `reduce_columns(exclude=…)`, so chosen
|
|
18
|
+
numeric columns pass through the reduction unchanged. The columns table
|
|
19
|
+
gained a `reduce` column showing each column's candidacy / skip state, and
|
|
20
|
+
excluded columns are listed in the run report.
|
|
21
|
+
- **Undo / redo for column configuration** (`ctrl+z` / `ctrl+y`), keeping well
|
|
22
|
+
over the last ten steps. Bursts of edits to one option field coalesce into a
|
|
23
|
+
single undo step.
|
|
24
|
+
- **File screen: refresh the directory browser** with `ctrl+r` or the new
|
|
25
|
+
⟳ refresh button, so files created after launch appear without restarting.
|
|
26
|
+
- **Report screen: save the report + column histograms to a text file** with
|
|
27
|
+
`ctrl+s` or the 💾 save .txt button; it writes `<sample-name>_report.txt`
|
|
28
|
+
next to the sample output.
|
|
29
|
+
|
|
30
|
+
## v3.4.0 — 2026-07-24
|
|
31
|
+
|
|
32
|
+
- **Added optional PCA column reduction** (`data_sampler.reduce`): after
|
|
33
|
+
sampling (and anonymization, if any), the numeric block of the outgoing
|
|
34
|
+
sample can be collapsed into its first *k* principal components — so the
|
|
35
|
+
delivered file is narrow as well as short. *k* is controlled one of two
|
|
36
|
+
ways: a component count (`--reduce-components N` /
|
|
37
|
+
`reduce_columns(df, n_components=N)`, capped at the number of usable
|
|
38
|
+
numeric columns) or a variance target (`--reduce-variance R` /
|
|
39
|
+
`variance_ratio=R`: the fewest components whose cumulative
|
|
40
|
+
explained-variance ratio reaches R). Non-numeric columns are always
|
|
41
|
+
preserved; `--reduce-exclude` passes chosen numeric columns (e.g.
|
|
42
|
+
identifiers) through unchanged, `--reduce-prefix` renames the `PC*`
|
|
43
|
+
output columns when the source already has one, and the tool warns when
|
|
44
|
+
an included column looks like an identifier. Reduce flags are validated
|
|
45
|
+
before sampling starts, so a typo never costs a full out-of-core run. Missing values are mean-imputed (the row count
|
|
46
|
+
never changes), constant/all-missing columns pass through with a note, and
|
|
47
|
+
columns are z-scored by default (`--reduce-no-standardize` opts out) so
|
|
48
|
+
large-unit columns cannot dominate. The exact-SVD implementation is pure
|
|
49
|
+
numpy — no new dependency — and deterministic across runs and platforms
|
|
50
|
+
(sign-fixed singular vectors). New public API: `reduce_columns`,
|
|
51
|
+
`ReductionResult`, `format_reduction_report`.
|
|
52
|
+
- **Every reduction reports its rationale**, not just the variance kept: the
|
|
53
|
+
report lists per-component and cumulative explained variance, **the groups
|
|
54
|
+
of correlated columns that move together** (connected components of the
|
|
55
|
+
|r| ≥ 0.7 correlation graph — on standardized data PCA diagonalizes
|
|
56
|
+
exactly that correlation matrix, so the groups are a faithful account of
|
|
57
|
+
what compressed), each component's top driving columns, and any
|
|
58
|
+
imputation/passthrough/identifier notes. The full labelled correlation
|
|
59
|
+
matrix is available on `ReductionResult.correlation_matrix`.
|
|
60
|
+
- Wired into all three surfaces: CLI flags on both the pandas and DuckDB
|
|
61
|
+
engine paths (the reduction always runs on the small sampled frame, never
|
|
62
|
+
the full source), a `reduce` control in the TUI run bar with the rationale
|
|
63
|
+
shown on the report screen, and the Python API. Output files gain a
|
|
64
|
+
`_pca{k}` tag suffix (e.g. `data_sample_500_anon_pca3.csv`).
|
|
65
|
+
|
|
3
66
|
## v3.3.1 — 2026-07-23
|
|
4
67
|
|
|
5
68
|
- **Fixed the second CI-only TUI race** (caught, again, by the release CI —
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: data-sampler
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.4.1
|
|
4
4
|
Summary: Hand someone data that looks and behaves like your production data, isn't your production data, and provably kept its statistical variety — stratified sampling + consistent anonymization, with a terminal UI.
|
|
5
5
|
Project-URL: Homepage, https://github.com/aaronified/data-sampler
|
|
6
6
|
Project-URL: Changelog, https://github.com/aaronified/data-sampler/blob/main/CHANGELOG.md
|
|
@@ -120,23 +120,35 @@ data_sampler.run_tui("data.csv") # pre-load a file
|
|
|
120
120
|
|
|
121
121
|
The TUI is a panel-based dashboard (think btop / lazydocker):
|
|
122
122
|
|
|
123
|
-
1. **File screen** — type a path or pick a file from the directory browser
|
|
124
|
-
|
|
123
|
+
1. **File screen** — type a path or pick a file from the directory browser
|
|
124
|
+
(**`ctrl+r`** or the **⟳ refresh** button re-scans the folder so files
|
|
125
|
+
created since launch show up); Excel files take an optional sheet name.
|
|
125
126
|
2. **Columns screen** — every column with its type, missing %, unique count,
|
|
126
|
-
distribution sparkline, and
|
|
127
|
-
VS Code extension). Select a column to
|
|
128
|
-
bars, choose an anonymizer for it, and
|
|
129
|
-
skipped when preserving statistical variety
|
|
130
|
-
|
|
127
|
+
distribution sparkline, and per-stat **mean / median / mode / sd** columns
|
|
128
|
+
(modelled after the Data Wrangler VS Code extension). Select a column to
|
|
129
|
+
see full stats and distribution bars, choose an anonymizer for it, and
|
|
130
|
+
toggle whether it should be skipped when preserving statistical variety
|
|
131
|
+
(stratification) or excluded from the PCA reduction. **Multi-select** rows
|
|
132
|
+
to configure them in bulk: **ctrl-click** toggles individual rows,
|
|
133
|
+
**shift-click** selects a range (**space** toggles the cursor row from the
|
|
134
|
+
keyboard, **`x`** clears) — then any anonymizer / skip / reduce choice
|
|
135
|
+
applies to every selected column at once. **`ctrl+z` / `ctrl+y`** undo and
|
|
136
|
+
redo column-config changes (at least the last ten steps). Set the sample
|
|
137
|
+
size, output folder, optional seed, an optional PCA column reduction
|
|
138
|
+
(`reduce`: N components or a variance target), and run.
|
|
131
139
|
3. **Report screen** — the stratification comparison and anonymization summary
|
|
132
140
|
on the left, and a **column histograms** panel on the right showing every
|
|
133
141
|
column's source-vs-sample distribution (numeric columns share bin edges;
|
|
134
142
|
others use the source's top categories) so you can see at a glance how well
|
|
135
|
-
the sample preserved each column. The output path is shown too
|
|
143
|
+
the sample preserved each column. The output path is shown too, and
|
|
144
|
+
**`ctrl+s`** (or **💾 save .txt**) writes the report and histograms to a
|
|
145
|
+
`*_report.txt` beside the sample.
|
|
136
146
|
|
|
137
|
-
Key bindings: `ctrl+r` run sample, `a` auto-suggest
|
|
138
|
-
`s` toggle stratification skip,
|
|
139
|
-
`
|
|
147
|
+
Key bindings — columns screen: `ctrl+r` run sample, `a` auto-suggest
|
|
148
|
+
anonymizer types, `s` toggle stratification skip, `d` toggle reduction skip,
|
|
149
|
+
`space` select row, `x` clear selection, `ctrl+z`/`ctrl+y` undo/redo,
|
|
150
|
+
`escape` back. Report screen: `ctrl+s` save report to text, `n` new file.
|
|
151
|
+
Everywhere: `ctrl+q` quit.
|
|
140
152
|
|
|
141
153
|
## CLI (headless)
|
|
142
154
|
|
|
@@ -154,6 +166,11 @@ data-sampler <source> <count> [options]
|
|
|
154
166
|
| `--anon COL=KIND[:k=v,...]` | Anonymize a column (repeatable) |
|
|
155
167
|
| `-i`, `--interactive` | Guided workflow: choose an anonymizer type per column from a menu |
|
|
156
168
|
| `--suggest` | Auto-assign a suggested anonymizer type to each column from its stats |
|
|
169
|
+
| `--reduce-components N` | PCA column reduction: replace the numeric columns with the first `N` principal components |
|
|
170
|
+
| `--reduce-variance R` | PCA column reduction: keep the fewest components whose cumulative explained variance reaches `R` (0 < R < 1) |
|
|
171
|
+
| `--reduce-exclude COL[,COL]` | Numeric column(s) to keep out of the reduction, e.g. identifiers (repeatable) |
|
|
172
|
+
| `--reduce-prefix PREFIX` | Name prefix for the component columns (default `PC` → `PC1`, `PC2`, …) |
|
|
173
|
+
| `--reduce-no-standardize` | Skip the per-column z-scoring before PCA |
|
|
157
174
|
| `--engine {auto,pandas,duckdb}` | Sampling engine (default `auto`: DuckDB for Parquet/large inputs, pandas otherwise) |
|
|
158
175
|
| `--threads N` | DuckDB engine: number of threads (default: all cores) |
|
|
159
176
|
| `--memory-limit SIZE` | DuckDB engine: memory limit before spilling to disk (e.g. `8GB`) |
|
|
@@ -172,6 +189,11 @@ data-sampler data.csv 100 --skip region,notes --seed 7 \
|
|
|
172
189
|
|
|
173
190
|
# large / out-of-core: sample a Parquet file in parallel with DuckDB
|
|
174
191
|
data-sampler huge.parquet 10000 --engine duckdb --threads 8 --memory-limit 8GB --suggest
|
|
192
|
+
|
|
193
|
+
# narrow the sample too: collapse the numeric columns into 3 principal
|
|
194
|
+
# components (or keep however many retain 90% of the variance)
|
|
195
|
+
data-sampler wide.csv 500 --reduce-components 3 --reduce-exclude cust_id
|
|
196
|
+
data-sampler wide.csv 500 --reduce-variance 0.9
|
|
175
197
|
```
|
|
176
198
|
|
|
177
199
|
## Python API
|
|
@@ -204,7 +226,11 @@ anon = ds.anonymize(
|
|
|
204
226
|
seed=7,
|
|
205
227
|
)
|
|
206
228
|
|
|
207
|
-
|
|
229
|
+
# optionally collapse the numeric columns into principal components
|
|
230
|
+
red = ds.reduce_columns(anon, variance_ratio=0.9, exclude=["cust_id"])
|
|
231
|
+
print(ds.format_reduction_report(red)) # variance kept + correlated groups
|
|
232
|
+
|
|
233
|
+
ds.save_output(red.data, "data.xlsx", tag="sample_500_anon_pca")
|
|
208
234
|
```
|
|
209
235
|
|
|
210
236
|
## Try it: bundled example
|
|
@@ -372,6 +398,34 @@ category. A side-by-side distribution report is produced for every run.
|
|
|
372
398
|
If no suitable stratification columns exist, the tool falls back to pure
|
|
373
399
|
random sampling automatically.
|
|
374
400
|
|
|
401
|
+
### Reducing columns (PCA)
|
|
402
|
+
|
|
403
|
+
Sampling narrows the *rows*; the optional PCA step narrows the *columns* of
|
|
404
|
+
the outgoing sample. It replaces the numeric block with its first *k*
|
|
405
|
+
principal components (`PC1..PCk`), controlled one of two ways:
|
|
406
|
+
|
|
407
|
+
- **`--reduce-components N`** — `N` components in the output (capped at the
|
|
408
|
+
number of usable numeric columns, with a note when fewer are possible);
|
|
409
|
+
- **`--reduce-variance R`** — the fewest components whose cumulative
|
|
410
|
+
explained-variance ratio reaches `R` (e.g. `0.9` keeps ≥ 90 % of the
|
|
411
|
+
numeric variance).
|
|
412
|
+
|
|
413
|
+
Non-numeric columns (ids, categories, text, booleans, datetimes) are always
|
|
414
|
+
preserved, and `--reduce-exclude` keeps chosen numeric columns out too —
|
|
415
|
+
identifiers should be excluded, since an all-unique id forms its own
|
|
416
|
+
artificial component (the tool warns when it spots one). Missing values are
|
|
417
|
+
mean-imputed so the row count never changes; constant columns carry no signal
|
|
418
|
+
and pass through unchanged. Columns are z-scored first by default (PCA on the
|
|
419
|
+
correlation matrix), so a large-unit column such as a salary cannot dominate
|
|
420
|
+
the components; `--reduce-no-standardize` turns that off.
|
|
421
|
+
|
|
422
|
+
Every reduction prints its rationale: the variance each component retains,
|
|
423
|
+
**the groups of correlated columns** that move together (which is exactly the
|
|
424
|
+
redundancy PCA collapses — on standardized data PCA diagonalizes the
|
|
425
|
+
correlation matrix), and each component's top driving columns. The reduction
|
|
426
|
+
runs after anonymization, on the already-sampled rows, so the stratification
|
|
427
|
+
and histogram reports still describe the original columns.
|
|
428
|
+
|
|
375
429
|
## Large data: the out-of-core DuckDB engine
|
|
376
430
|
|
|
377
431
|
The default pandas path loads the whole file into memory. For inputs that are
|
|
@@ -493,6 +547,11 @@ columns, in one place.
|
|
|
493
547
|
(~1.9× faster stats), report histograms skip near-unique columns instead
|
|
494
548
|
of hashing millions of ids for a meaningless top-8, and the TUI computes
|
|
495
549
|
stats in a worker thread so the UI never freezes on load.
|
|
550
|
+
- **PCA column reduction on the way out.** `--reduce-components` /
|
|
551
|
+
`--reduce-variance` collapse a wide numeric block into a handful of
|
|
552
|
+
principal components *after* sampling, so the SVD runs on the small sampled
|
|
553
|
+
frame (never the full source) and the delivered file is narrow as well as
|
|
554
|
+
short.
|
|
496
555
|
|
|
497
556
|
### Correctness under scale
|
|
498
557
|
|
|
@@ -520,7 +579,8 @@ multi-threading for reproducibility (unseeded runs use all cores).
|
|
|
520
579
|
|
|
521
580
|
Output keeps the source format and is named
|
|
522
581
|
`{stem}_sample_{count}{ext}` — with an `_anon` suffix when anonymization ran
|
|
523
|
-
|
|
582
|
+
and a `_pca{k}` suffix when PCA column reduction ran
|
|
583
|
+
(e.g. `data_sample_500_anon_pca3.csv`).
|
|
524
584
|
|
|
525
585
|
## Development
|
|
526
586
|
|
|
@@ -65,23 +65,35 @@ data_sampler.run_tui("data.csv") # pre-load a file
|
|
|
65
65
|
|
|
66
66
|
The TUI is a panel-based dashboard (think btop / lazydocker):
|
|
67
67
|
|
|
68
|
-
1. **File screen** — type a path or pick a file from the directory browser
|
|
69
|
-
|
|
68
|
+
1. **File screen** — type a path or pick a file from the directory browser
|
|
69
|
+
(**`ctrl+r`** or the **⟳ refresh** button re-scans the folder so files
|
|
70
|
+
created since launch show up); Excel files take an optional sheet name.
|
|
70
71
|
2. **Columns screen** — every column with its type, missing %, unique count,
|
|
71
|
-
distribution sparkline, and
|
|
72
|
-
VS Code extension). Select a column to
|
|
73
|
-
bars, choose an anonymizer for it, and
|
|
74
|
-
skipped when preserving statistical variety
|
|
75
|
-
|
|
72
|
+
distribution sparkline, and per-stat **mean / median / mode / sd** columns
|
|
73
|
+
(modelled after the Data Wrangler VS Code extension). Select a column to
|
|
74
|
+
see full stats and distribution bars, choose an anonymizer for it, and
|
|
75
|
+
toggle whether it should be skipped when preserving statistical variety
|
|
76
|
+
(stratification) or excluded from the PCA reduction. **Multi-select** rows
|
|
77
|
+
to configure them in bulk: **ctrl-click** toggles individual rows,
|
|
78
|
+
**shift-click** selects a range (**space** toggles the cursor row from the
|
|
79
|
+
keyboard, **`x`** clears) — then any anonymizer / skip / reduce choice
|
|
80
|
+
applies to every selected column at once. **`ctrl+z` / `ctrl+y`** undo and
|
|
81
|
+
redo column-config changes (at least the last ten steps). Set the sample
|
|
82
|
+
size, output folder, optional seed, an optional PCA column reduction
|
|
83
|
+
(`reduce`: N components or a variance target), and run.
|
|
76
84
|
3. **Report screen** — the stratification comparison and anonymization summary
|
|
77
85
|
on the left, and a **column histograms** panel on the right showing every
|
|
78
86
|
column's source-vs-sample distribution (numeric columns share bin edges;
|
|
79
87
|
others use the source's top categories) so you can see at a glance how well
|
|
80
|
-
the sample preserved each column. The output path is shown too
|
|
88
|
+
the sample preserved each column. The output path is shown too, and
|
|
89
|
+
**`ctrl+s`** (or **💾 save .txt**) writes the report and histograms to a
|
|
90
|
+
`*_report.txt` beside the sample.
|
|
81
91
|
|
|
82
|
-
Key bindings: `ctrl+r` run sample, `a` auto-suggest
|
|
83
|
-
`s` toggle stratification skip,
|
|
84
|
-
`
|
|
92
|
+
Key bindings — columns screen: `ctrl+r` run sample, `a` auto-suggest
|
|
93
|
+
anonymizer types, `s` toggle stratification skip, `d` toggle reduction skip,
|
|
94
|
+
`space` select row, `x` clear selection, `ctrl+z`/`ctrl+y` undo/redo,
|
|
95
|
+
`escape` back. Report screen: `ctrl+s` save report to text, `n` new file.
|
|
96
|
+
Everywhere: `ctrl+q` quit.
|
|
85
97
|
|
|
86
98
|
## CLI (headless)
|
|
87
99
|
|
|
@@ -99,6 +111,11 @@ data-sampler <source> <count> [options]
|
|
|
99
111
|
| `--anon COL=KIND[:k=v,...]` | Anonymize a column (repeatable) |
|
|
100
112
|
| `-i`, `--interactive` | Guided workflow: choose an anonymizer type per column from a menu |
|
|
101
113
|
| `--suggest` | Auto-assign a suggested anonymizer type to each column from its stats |
|
|
114
|
+
| `--reduce-components N` | PCA column reduction: replace the numeric columns with the first `N` principal components |
|
|
115
|
+
| `--reduce-variance R` | PCA column reduction: keep the fewest components whose cumulative explained variance reaches `R` (0 < R < 1) |
|
|
116
|
+
| `--reduce-exclude COL[,COL]` | Numeric column(s) to keep out of the reduction, e.g. identifiers (repeatable) |
|
|
117
|
+
| `--reduce-prefix PREFIX` | Name prefix for the component columns (default `PC` → `PC1`, `PC2`, …) |
|
|
118
|
+
| `--reduce-no-standardize` | Skip the per-column z-scoring before PCA |
|
|
102
119
|
| `--engine {auto,pandas,duckdb}` | Sampling engine (default `auto`: DuckDB for Parquet/large inputs, pandas otherwise) |
|
|
103
120
|
| `--threads N` | DuckDB engine: number of threads (default: all cores) |
|
|
104
121
|
| `--memory-limit SIZE` | DuckDB engine: memory limit before spilling to disk (e.g. `8GB`) |
|
|
@@ -117,6 +134,11 @@ data-sampler data.csv 100 --skip region,notes --seed 7 \
|
|
|
117
134
|
|
|
118
135
|
# large / out-of-core: sample a Parquet file in parallel with DuckDB
|
|
119
136
|
data-sampler huge.parquet 10000 --engine duckdb --threads 8 --memory-limit 8GB --suggest
|
|
137
|
+
|
|
138
|
+
# narrow the sample too: collapse the numeric columns into 3 principal
|
|
139
|
+
# components (or keep however many retain 90% of the variance)
|
|
140
|
+
data-sampler wide.csv 500 --reduce-components 3 --reduce-exclude cust_id
|
|
141
|
+
data-sampler wide.csv 500 --reduce-variance 0.9
|
|
120
142
|
```
|
|
121
143
|
|
|
122
144
|
## Python API
|
|
@@ -149,7 +171,11 @@ anon = ds.anonymize(
|
|
|
149
171
|
seed=7,
|
|
150
172
|
)
|
|
151
173
|
|
|
152
|
-
|
|
174
|
+
# optionally collapse the numeric columns into principal components
|
|
175
|
+
red = ds.reduce_columns(anon, variance_ratio=0.9, exclude=["cust_id"])
|
|
176
|
+
print(ds.format_reduction_report(red)) # variance kept + correlated groups
|
|
177
|
+
|
|
178
|
+
ds.save_output(red.data, "data.xlsx", tag="sample_500_anon_pca")
|
|
153
179
|
```
|
|
154
180
|
|
|
155
181
|
## Try it: bundled example
|
|
@@ -317,6 +343,34 @@ category. A side-by-side distribution report is produced for every run.
|
|
|
317
343
|
If no suitable stratification columns exist, the tool falls back to pure
|
|
318
344
|
random sampling automatically.
|
|
319
345
|
|
|
346
|
+
### Reducing columns (PCA)
|
|
347
|
+
|
|
348
|
+
Sampling narrows the *rows*; the optional PCA step narrows the *columns* of
|
|
349
|
+
the outgoing sample. It replaces the numeric block with its first *k*
|
|
350
|
+
principal components (`PC1..PCk`), controlled one of two ways:
|
|
351
|
+
|
|
352
|
+
- **`--reduce-components N`** — `N` components in the output (capped at the
|
|
353
|
+
number of usable numeric columns, with a note when fewer are possible);
|
|
354
|
+
- **`--reduce-variance R`** — the fewest components whose cumulative
|
|
355
|
+
explained-variance ratio reaches `R` (e.g. `0.9` keeps ≥ 90 % of the
|
|
356
|
+
numeric variance).
|
|
357
|
+
|
|
358
|
+
Non-numeric columns (ids, categories, text, booleans, datetimes) are always
|
|
359
|
+
preserved, and `--reduce-exclude` keeps chosen numeric columns out too —
|
|
360
|
+
identifiers should be excluded, since an all-unique id forms its own
|
|
361
|
+
artificial component (the tool warns when it spots one). Missing values are
|
|
362
|
+
mean-imputed so the row count never changes; constant columns carry no signal
|
|
363
|
+
and pass through unchanged. Columns are z-scored first by default (PCA on the
|
|
364
|
+
correlation matrix), so a large-unit column such as a salary cannot dominate
|
|
365
|
+
the components; `--reduce-no-standardize` turns that off.
|
|
366
|
+
|
|
367
|
+
Every reduction prints its rationale: the variance each component retains,
|
|
368
|
+
**the groups of correlated columns** that move together (which is exactly the
|
|
369
|
+
redundancy PCA collapses — on standardized data PCA diagonalizes the
|
|
370
|
+
correlation matrix), and each component's top driving columns. The reduction
|
|
371
|
+
runs after anonymization, on the already-sampled rows, so the stratification
|
|
372
|
+
and histogram reports still describe the original columns.
|
|
373
|
+
|
|
320
374
|
## Large data: the out-of-core DuckDB engine
|
|
321
375
|
|
|
322
376
|
The default pandas path loads the whole file into memory. For inputs that are
|
|
@@ -438,6 +492,11 @@ columns, in one place.
|
|
|
438
492
|
(~1.9× faster stats), report histograms skip near-unique columns instead
|
|
439
493
|
of hashing millions of ids for a meaningless top-8, and the TUI computes
|
|
440
494
|
stats in a worker thread so the UI never freezes on load.
|
|
495
|
+
- **PCA column reduction on the way out.** `--reduce-components` /
|
|
496
|
+
`--reduce-variance` collapse a wide numeric block into a handful of
|
|
497
|
+
principal components *after* sampling, so the SVD runs on the small sampled
|
|
498
|
+
frame (never the full source) and the delivered file is narrow as well as
|
|
499
|
+
short.
|
|
441
500
|
|
|
442
501
|
### Correctness under scale
|
|
443
502
|
|
|
@@ -465,7 +524,8 @@ multi-threading for reproducibility (unseeded runs use all cores).
|
|
|
465
524
|
|
|
466
525
|
Output keeps the source format and is named
|
|
467
526
|
`{stem}_sample_{count}{ext}` — with an `_anon` suffix when anonymization ran
|
|
468
|
-
|
|
527
|
+
and a `_pca{k}` suffix when PCA column reduction ran
|
|
528
|
+
(e.g. `data_sample_500_anon_pca3.csv`).
|
|
469
529
|
|
|
470
530
|
## Development
|
|
471
531
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "data-sampler"
|
|
7
|
-
version = "3.
|
|
7
|
+
version = "3.4.1"
|
|
8
8
|
description = "Hand someone data that looks and behaves like your production data, isn't your production data, and provably kept its statistical variety — stratified sampling + consistent anonymization, with a terminal UI."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = { file = "LICENSE" }
|
|
@@ -9,20 +9,24 @@ Public API::
|
|
|
9
9
|
result = ds.sample(df, 500, exclude_columns=["notes"])
|
|
10
10
|
print(ds.format_stratification_report(df, result))
|
|
11
11
|
anon = ds.anonymize(result.data, {"name": "names", "id": ("sequential_id", {"start": 1000})})
|
|
12
|
-
ds.
|
|
12
|
+
red = ds.reduce_columns(anon, variance_ratio=0.9, exclude=["id"]) # optional PCA
|
|
13
|
+
print(ds.format_reduction_report(red))
|
|
14
|
+
ds.save_output(red.data, "data.csv", tag="sample_500_anon_pca")
|
|
13
15
|
|
|
14
16
|
ds.run_tui() # launch the terminal UI
|
|
15
17
|
"""
|
|
16
18
|
|
|
17
19
|
from __future__ import annotations
|
|
18
20
|
|
|
19
|
-
__version__ = "3.
|
|
21
|
+
__version__ = "3.4.1"
|
|
20
22
|
|
|
21
23
|
from .io import SUPPORTED_EXTENSIONS, list_sheets, load_file, save_output
|
|
24
|
+
from .reduce import ReductionResult, reduce_columns
|
|
22
25
|
from .report import (
|
|
23
26
|
column_histogram_data,
|
|
24
27
|
format_column_histograms,
|
|
25
28
|
format_distribution,
|
|
29
|
+
format_reduction_report,
|
|
26
30
|
format_stratification_report,
|
|
27
31
|
)
|
|
28
32
|
from .sampling import (
|
|
@@ -47,7 +51,10 @@ __all__ = [
|
|
|
47
51
|
"sample",
|
|
48
52
|
"stratified_sample",
|
|
49
53
|
"find_stratification_columns",
|
|
54
|
+
"ReductionResult",
|
|
55
|
+
"reduce_columns",
|
|
50
56
|
"format_stratification_report",
|
|
57
|
+
"format_reduction_report",
|
|
51
58
|
"format_distribution",
|
|
52
59
|
"column_histogram_data",
|
|
53
60
|
"format_column_histograms",
|
|
@@ -13,6 +13,12 @@ which picks DuckDB for Parquet/large files) samples out-of-core in parallel:
|
|
|
13
13
|
|
|
14
14
|
data-sampler huge.parquet 10000 --engine duckdb --threads 8 \\
|
|
15
15
|
--memory-limit 8GB --suggest
|
|
16
|
+
|
|
17
|
+
Optionally reduce the numeric columns of the outgoing sample to K principal
|
|
18
|
+
components — or to however many retain a target share of the variance:
|
|
19
|
+
|
|
20
|
+
data-sampler data.csv 500 --reduce-components 3 --reduce-exclude id
|
|
21
|
+
data-sampler data.csv 500 --reduce-variance 0.9
|
|
16
22
|
"""
|
|
17
23
|
|
|
18
24
|
from __future__ import annotations
|
|
@@ -24,7 +30,12 @@ from . import __version__
|
|
|
24
30
|
from ._logging import get_logger
|
|
25
31
|
from .anonymize import anonymize, make_anonymizer
|
|
26
32
|
from .io import load_file, save_output
|
|
27
|
-
from .
|
|
33
|
+
from .reduce import reduce_columns
|
|
34
|
+
from .report import (
|
|
35
|
+
format_column_histograms,
|
|
36
|
+
format_reduction_report,
|
|
37
|
+
format_stratification_report,
|
|
38
|
+
)
|
|
28
39
|
from .sampling import sample
|
|
29
40
|
|
|
30
41
|
log = get_logger(__name__)
|
|
@@ -41,6 +52,24 @@ def _coerce_value(value: str):
|
|
|
41
52
|
return value
|
|
42
53
|
|
|
43
54
|
|
|
55
|
+
def _positive_int(text: str) -> int:
|
|
56
|
+
"""argparse type: an integer >= 1 (rejects bad values at parse time)."""
|
|
57
|
+
value = int(text)
|
|
58
|
+
if value < 1:
|
|
59
|
+
raise argparse.ArgumentTypeError(f"must be a positive integer, got {text}")
|
|
60
|
+
return value
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _unit_interval(text: str) -> float:
|
|
64
|
+
"""argparse type: a float strictly between 0 and 1."""
|
|
65
|
+
value = float(text)
|
|
66
|
+
if not 0.0 < value < 1.0:
|
|
67
|
+
raise argparse.ArgumentTypeError(
|
|
68
|
+
f"must be strictly between 0 and 1, got {text}"
|
|
69
|
+
)
|
|
70
|
+
return value
|
|
71
|
+
|
|
72
|
+
|
|
44
73
|
def parse_anon_option(text: str) -> tuple[str, str, dict]:
|
|
45
74
|
"""Parse ``COL=KIND[:key=value,key=value]`` into (column, kind, options)."""
|
|
46
75
|
col, sep, rhs = text.partition("=")
|
|
@@ -110,6 +139,46 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
110
139
|
"stats (columns set via --anon are left as given)"
|
|
111
140
|
),
|
|
112
141
|
)
|
|
142
|
+
reduce_group = parser.add_mutually_exclusive_group()
|
|
143
|
+
reduce_group.add_argument(
|
|
144
|
+
"--reduce-components", type=_positive_int, metavar="N",
|
|
145
|
+
dest="reduce_components",
|
|
146
|
+
help=(
|
|
147
|
+
"PCA column reduction: replace the numeric columns with the first "
|
|
148
|
+
"N principal components (non-numeric columns are preserved; N is "
|
|
149
|
+
"capped at the number of usable numeric columns)"
|
|
150
|
+
),
|
|
151
|
+
)
|
|
152
|
+
reduce_group.add_argument(
|
|
153
|
+
"--reduce-variance", type=_unit_interval, metavar="R",
|
|
154
|
+
dest="reduce_variance",
|
|
155
|
+
help=(
|
|
156
|
+
"PCA column reduction: keep the fewest principal components whose "
|
|
157
|
+
"cumulative explained-variance ratio reaches R (0 < R < 1)"
|
|
158
|
+
),
|
|
159
|
+
)
|
|
160
|
+
parser.add_argument(
|
|
161
|
+
"--reduce-exclude", action="append", default=[], metavar="COL[,COL...]",
|
|
162
|
+
dest="reduce_exclude",
|
|
163
|
+
help=(
|
|
164
|
+
"numeric column(s) to exclude from PCA reduction and pass through "
|
|
165
|
+
"unchanged, e.g. identifiers (repeatable)"
|
|
166
|
+
),
|
|
167
|
+
)
|
|
168
|
+
parser.add_argument(
|
|
169
|
+
"--reduce-prefix", default="PC", metavar="PREFIX", dest="reduce_prefix",
|
|
170
|
+
help=(
|
|
171
|
+
"name prefix for the principal-component columns (default PC → "
|
|
172
|
+
"PC1, PC2, …); change it if the source already has such a column"
|
|
173
|
+
),
|
|
174
|
+
)
|
|
175
|
+
parser.add_argument(
|
|
176
|
+
"--reduce-no-standardize", action="store_true", dest="reduce_no_standardize",
|
|
177
|
+
help=(
|
|
178
|
+
"PCA column reduction: skip the per-column z-scoring (by default "
|
|
179
|
+
"columns are standardized so large-unit columns cannot dominate)"
|
|
180
|
+
),
|
|
181
|
+
)
|
|
113
182
|
parser.add_argument(
|
|
114
183
|
"--engine", choices=("auto", "pandas", "duckdb"), default="auto",
|
|
115
184
|
help=(
|
|
@@ -136,6 +205,65 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
136
205
|
return parser
|
|
137
206
|
|
|
138
207
|
|
|
208
|
+
def _reduce_requested(args) -> bool:
|
|
209
|
+
return args.reduce_components is not None or args.reduce_variance is not None
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _validate_reduce_args(args, parser, columns) -> list[str]:
|
|
213
|
+
"""Validate the reduce flags as soon as the source's columns are known —
|
|
214
|
+
BEFORE sampling, so a typo never costs a full (possibly out-of-core) run.
|
|
215
|
+
|
|
216
|
+
Returns the parsed ``--reduce-exclude`` column list.
|
|
217
|
+
"""
|
|
218
|
+
exclude = [
|
|
219
|
+
c.strip() for chunk in args.reduce_exclude for c in chunk.split(",") if c.strip()
|
|
220
|
+
]
|
|
221
|
+
if not _reduce_requested(args):
|
|
222
|
+
if exclude:
|
|
223
|
+
parser.error(
|
|
224
|
+
"--reduce-exclude requires --reduce-components or --reduce-variance"
|
|
225
|
+
)
|
|
226
|
+
if args.reduce_no_standardize:
|
|
227
|
+
parser.error(
|
|
228
|
+
"--reduce-no-standardize requires --reduce-components or "
|
|
229
|
+
"--reduce-variance"
|
|
230
|
+
)
|
|
231
|
+
return exclude
|
|
232
|
+
# compare stringified: Excel sources can carry non-string column labels
|
|
233
|
+
names = {str(c) for c in columns}
|
|
234
|
+
unknown = [c for c in exclude if c not in names]
|
|
235
|
+
if unknown:
|
|
236
|
+
parser.error(
|
|
237
|
+
f"--reduce-exclude: column(s) not found in file: {', '.join(unknown)}"
|
|
238
|
+
)
|
|
239
|
+
return exclude
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _apply_reduction(args, parser, data, exclude):
|
|
243
|
+
"""Run the optional PCA column reduction on the outgoing sample.
|
|
244
|
+
|
|
245
|
+
Returns ``(data, tag_suffix)`` — unchanged when no reduction was requested.
|
|
246
|
+
"""
|
|
247
|
+
if not _reduce_requested(args):
|
|
248
|
+
return data, ""
|
|
249
|
+
try:
|
|
250
|
+
reduction = reduce_columns(
|
|
251
|
+
data,
|
|
252
|
+
n_components=args.reduce_components,
|
|
253
|
+
variance_ratio=args.reduce_variance,
|
|
254
|
+
exclude=exclude,
|
|
255
|
+
standardize=not args.reduce_no_standardize,
|
|
256
|
+
prefix=args.reduce_prefix,
|
|
257
|
+
seed=args.seed,
|
|
258
|
+
)
|
|
259
|
+
except ValueError as exc:
|
|
260
|
+
parser.error(f"column reduction: {exc}")
|
|
261
|
+
print()
|
|
262
|
+
print(format_reduction_report(reduction))
|
|
263
|
+
suffix = f"_pca{reduction.n_components}" if reduction.n_components else ""
|
|
264
|
+
return reduction.data, suffix
|
|
265
|
+
|
|
266
|
+
|
|
139
267
|
def _build_engine_spec(args, parser, engine, cols) -> dict:
|
|
140
268
|
"""Build the anonymizer spec for the engine path (validates against ``cols``,
|
|
141
269
|
supports explicit --anon and --suggest via the engine's approximate stats)."""
|
|
@@ -189,6 +317,7 @@ def _run_with_engine(args, parser, skip) -> int:
|
|
|
189
317
|
parser.error(
|
|
190
318
|
f"--skip: column(s) not found in file: {', '.join(unknown_skip)}"
|
|
191
319
|
)
|
|
320
|
+
reduce_exclude = _validate_reduce_args(args, parser, cols)
|
|
192
321
|
spec = _build_engine_spec(args, parser, engine, cols)
|
|
193
322
|
result = engine.sample(
|
|
194
323
|
args.source, args.count,
|
|
@@ -213,7 +342,8 @@ def _run_with_engine(args, parser, skip) -> int:
|
|
|
213
342
|
parser.error(f"anonymization failed: {exc}")
|
|
214
343
|
print(f"Anonymized columns: {', '.join(spec)}")
|
|
215
344
|
|
|
216
|
-
|
|
345
|
+
data, reduce_tag = _apply_reduction(args, parser, data, reduce_exclude)
|
|
346
|
+
tag = f"sample_{args.count}" + ("_anon" if spec else "") + reduce_tag
|
|
217
347
|
out_path = save_output(data, args.source, tag, output_folder=args.outdir)
|
|
218
348
|
print(f"\nSampled {len(data):,} rows.")
|
|
219
349
|
print(f"Output saved to: {out_path}")
|
|
@@ -288,6 +418,7 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
288
418
|
unknown_skip = [c for c in skip if c not in df.columns]
|
|
289
419
|
if unknown_skip:
|
|
290
420
|
parser.error(f"--skip: column(s) not found in file: {', '.join(unknown_skip)}")
|
|
421
|
+
reduce_exclude = _validate_reduce_args(args, parser, df.columns)
|
|
291
422
|
|
|
292
423
|
# parse explicit --anon flags into (column, kind, options) triples
|
|
293
424
|
anon_specs: list[tuple[str, str, dict]] = []
|
|
@@ -352,7 +483,8 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
352
483
|
parser.error(f"anonymization failed: {exc}")
|
|
353
484
|
print(f"Anonymized columns: {', '.join(spec)}")
|
|
354
485
|
|
|
355
|
-
|
|
486
|
+
data, reduce_tag = _apply_reduction(args, parser, data, reduce_exclude)
|
|
487
|
+
tag = f"sample_{args.count}" + ("_anon" if spec else "") + reduce_tag
|
|
356
488
|
out_path = save_output(data, args.source, tag, output_folder=args.outdir)
|
|
357
489
|
print(f"\nSampled {len(data)} rows.")
|
|
358
490
|
print(f"Output saved to: {out_path}")
|