data-sampler 3.3.1__tar.gz → 3.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. {data_sampler-3.3.1 → data_sampler-3.4.1}/CHANGELOG.md +63 -0
  2. {data_sampler-3.3.1 → data_sampler-3.4.1}/PKG-INFO +74 -14
  3. {data_sampler-3.3.1 → data_sampler-3.4.1}/README.md +73 -13
  4. {data_sampler-3.3.1 → data_sampler-3.4.1}/pyproject.toml +1 -1
  5. {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/__init__.py +9 -2
  6. {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/cli.py +135 -3
  7. data_sampler-3.4.1/src/data_sampler/reduce.py +343 -0
  8. {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/report.py +76 -0
  9. {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/stats.py +7 -0
  10. {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/tui/app.py +444 -45
  11. {data_sampler-3.3.1 → data_sampler-3.4.1}/tests/test_cli.py +136 -0
  12. data_sampler-3.4.1/tests/test_reduce.py +291 -0
  13. {data_sampler-3.3.1 → data_sampler-3.4.1}/tests/test_stats.py +19 -0
  14. data_sampler-3.4.1/tests/test_tui.py +620 -0
  15. data_sampler-3.3.1/tests/test_tui.py +0 -283
  16. {data_sampler-3.3.1 → data_sampler-3.4.1}/.gitattributes +0 -0
  17. {data_sampler-3.3.1 → data_sampler-3.4.1}/.github/workflows/release.yml +0 -0
  18. {data_sampler-3.3.1 → data_sampler-3.4.1}/.gitignore +0 -0
  19. {data_sampler-3.3.1 → data_sampler-3.4.1}/LICENSE +0 -0
  20. {data_sampler-3.3.1 → data_sampler-3.4.1}/RELEASING.md +0 -0
  21. {data_sampler-3.3.1 → data_sampler-3.4.1}/ROADMAP.md +0 -0
  22. {data_sampler-3.3.1 → data_sampler-3.4.1}/TROUBLESHOOTING.md +0 -0
  23. {data_sampler-3.3.1 → data_sampler-3.4.1}/examples/employees.csv +0 -0
  24. {data_sampler-3.3.1 → data_sampler-3.4.1}/examples/using_data_sampler.ipynb +0 -0
  25. {data_sampler-3.3.1 → data_sampler-3.4.1}/requirements.txt +0 -0
  26. {data_sampler-3.3.1 → data_sampler-3.4.1}/scripts/run-tui.bat +0 -0
  27. {data_sampler-3.3.1 → data_sampler-3.4.1}/scripts/run-tui.sh +0 -0
  28. {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/__main__.py +0 -0
  29. {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/_logging.py +0 -0
  30. {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/_names.py +0 -0
  31. {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/anonymize.py +0 -0
  32. {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/engine.py +0 -0
  33. {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/io.py +0 -0
  34. {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/sampling.py +0 -0
  35. {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/tui/__init__.py +0 -0
  36. {data_sampler-3.3.1 → data_sampler-3.4.1}/src/data_sampler/workflow.py +0 -0
  37. {data_sampler-3.3.1 → data_sampler-3.4.1}/tests/conftest.py +0 -0
  38. {data_sampler-3.3.1 → data_sampler-3.4.1}/tests/test_anonymize.py +0 -0
  39. {data_sampler-3.3.1 → data_sampler-3.4.1}/tests/test_engine.py +0 -0
  40. {data_sampler-3.3.1 → data_sampler-3.4.1}/tests/test_io.py +0 -0
  41. {data_sampler-3.3.1 → data_sampler-3.4.1}/tests/test_report.py +0 -0
  42. {data_sampler-3.3.1 → data_sampler-3.4.1}/tests/test_sampling.py +0 -0
  43. {data_sampler-3.3.1 → data_sampler-3.4.1}/tests/test_workflow.py +0 -0
@@ -1,5 +1,68 @@
1
1
  # Changelog
2
2
 
3
+ ## v3.4.1 — 2026-07-24
4
+
5
+ - **TUI columns table now breaks the single summary cell into separate
6
+ `mean` / `median` / `mode` / `sd` columns.** Numeric columns fill all four;
7
+ non-numeric columns show `mode` (their most frequent value) with dashes for
8
+ the numeric-only stats. `ColumnStats` gained a `mode` field, populated for
9
+ every column kind by `compute_column_stats`.
10
+ - **Bulk configuration via multi-select on the columns screen.** Ctrl-click
11
+ toggles individual rows, shift-click selects a contiguous range (keyboard:
12
+ `space` toggles the cursor row, `x` clears the selection). Any anonymizer
13
+ choice, stratification-skip, or reduction-skip then applies to every
14
+ selected column at once; the panel title shows the selection count.
15
+ - **Columns can now be excluded from the PCA reduction from the TUI.** A new
16
+ “skip from reduction (PCA)” switch (and the `d` key) sets a per-column
17
+ `skip_reduce` flag that is passed to `reduce_columns(exclude=…)`, so chosen
18
+ numeric columns pass through the reduction unchanged. The columns table
19
+ gained a `reduce` column showing each column's candidacy / skip state, and
20
+ excluded columns are listed in the run report.
21
+ - **Undo / redo for column configuration** (`ctrl+z` / `ctrl+y`), keeping well
22
+ over the last ten steps. Bursts of edits to one option field coalesce into a
23
+ single undo step.
24
+ - **File screen: refresh the directory browser** with `ctrl+r` or the new
25
+ ⟳ refresh button, so files created after launch appear without restarting.
26
+ - **Report screen: save the report + column histograms to a text file** with
27
+ `ctrl+s` or the 💾 save .txt button; it writes `<sample-name>_report.txt`
28
+ next to the sample output.
29
+
30
+ ## v3.4.0 — 2026-07-24
31
+
32
+ - **Added optional PCA column reduction** (`data_sampler.reduce`): after
33
+ sampling (and anonymization, if any), the numeric block of the outgoing
34
+ sample can be collapsed into its first *k* principal components — so the
35
+ delivered file is narrow as well as short. *k* is controlled one of two
36
+ ways: a component count (`--reduce-components N` /
37
+ `reduce_columns(df, n_components=N)`, capped at the number of usable
38
+ numeric columns) or a variance target (`--reduce-variance R` /
39
+ `variance_ratio=R`: the fewest components whose cumulative
40
+ explained-variance ratio reaches R). Non-numeric columns are always
41
+ preserved; `--reduce-exclude` passes chosen numeric columns (e.g.
42
+ identifiers) through unchanged, `--reduce-prefix` renames the `PC*`
43
+ output columns when the source already has one, and the tool warns when
44
+ an included column looks like an identifier. Reduce flags are validated
45
+ before sampling starts, so a typo never costs a full out-of-core run. Missing values are mean-imputed (the row count
46
+ never changes), constant/all-missing columns pass through with a note, and
47
+ columns are z-scored by default (`--reduce-no-standardize` opts out) so
48
+ large-unit columns cannot dominate. The exact-SVD implementation is pure
49
+ numpy — no new dependency — and deterministic across runs and platforms
50
+ (sign-fixed singular vectors). New public API: `reduce_columns`,
51
+ `ReductionResult`, `format_reduction_report`.
52
+ - **Every reduction reports its rationale**, not just the variance kept: the
53
+ report lists per-component and cumulative explained variance, **the groups
54
+ of correlated columns that move together** (connected components of the
55
+ |r| ≥ 0.7 correlation graph — on standardized data PCA diagonalizes
56
+ exactly that correlation matrix, so the groups are a faithful account of
57
+ what compressed), each component's top driving columns, and any
58
+ imputation/passthrough/identifier notes. The full labelled correlation
59
+ matrix is available on `ReductionResult.correlation_matrix`.
60
+ - Wired into all three surfaces: CLI flags on both the pandas and DuckDB
61
+ engine paths (the reduction always runs on the small sampled frame, never
62
+ the full source), a `reduce` control in the TUI run bar with the rationale
63
+ shown on the report screen, and the Python API. Output files gain a
64
+ `_pca{k}` tag suffix (e.g. `data_sample_500_anon_pca3.csv`).
65
+
3
66
  ## v3.3.1 — 2026-07-23
4
67
 
5
68
  - **Fixed the second CI-only TUI race** (caught, again, by the release CI —
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: data-sampler
3
- Version: 3.3.1
3
+ Version: 3.4.1
4
4
  Summary: Hand someone data that looks and behaves like your production data, isn't your production data, and provably kept its statistical variety — stratified sampling + consistent anonymization, with a terminal UI.
5
5
  Project-URL: Homepage, https://github.com/aaronified/data-sampler
6
6
  Project-URL: Changelog, https://github.com/aaronified/data-sampler/blob/main/CHANGELOG.md
@@ -120,23 +120,35 @@ data_sampler.run_tui("data.csv") # pre-load a file
120
120
 
121
121
  The TUI is a panel-based dashboard (think btop / lazydocker):
122
122
 
123
- 1. **File screen** — type a path or pick a file from the directory browser;
124
- Excel files take an optional sheet name.
123
+ 1. **File screen** — type a path or pick a file from the directory browser
124
+ (**`ctrl+r`** or the **⟳ refresh** button re-scans the folder so files
125
+ created since launch show up); Excel files take an optional sheet name.
125
126
  2. **Columns screen** — every column with its type, missing %, unique count,
126
- distribution sparkline, and summary (modelled after the Data Wrangler
127
- VS Code extension). Select a column to see full stats and distribution
128
- bars, choose an anonymizer for it, and toggle whether it should be
129
- skipped when preserving statistical variety (stratification). Set the
130
- sample size, output folder, optional seed, and run.
127
+ distribution sparkline, and per-stat **mean / median / mode / sd** columns
128
+ (modelled after the Data Wrangler VS Code extension). Select a column to
129
+ see full stats and distribution bars, choose an anonymizer for it, and
130
+ toggle whether it should be skipped when preserving statistical variety
131
+ (stratification) or excluded from the PCA reduction. **Multi-select** rows
132
+ to configure them in bulk: **ctrl-click** toggles individual rows,
133
+ **shift-click** selects a range (**space** toggles the cursor row from the
134
+ keyboard, **`x`** clears) — then any anonymizer / skip / reduce choice
135
+ applies to every selected column at once. **`ctrl+z` / `ctrl+y`** undo and
136
+ redo column-config changes (at least the last ten steps). Set the sample
137
+ size, output folder, optional seed, an optional PCA column reduction
138
+ (`reduce`: N components or a variance target), and run.
131
139
  3. **Report screen** — the stratification comparison and anonymization summary
132
140
  on the left, and a **column histograms** panel on the right showing every
133
141
  column's source-vs-sample distribution (numeric columns share bin edges;
134
142
  others use the source's top categories) so you can see at a glance how well
135
- the sample preserved each column. The output path is shown too.
143
+ the sample preserved each column. The output path is shown too, and
144
+ **`ctrl+s`** (or **💾 save .txt**) writes the report and histograms to a
145
+ `*_report.txt` beside the sample.
136
146
 
137
- Key bindings: `ctrl+r` run sample, `a` auto-suggest anonymizer types,
138
- `s` toggle stratification skip,
139
- `escape` back, `ctrl+q` quit.
147
+ Key bindings — columns screen: `ctrl+r` run sample, `a` auto-suggest
148
+ anonymizer types, `s` toggle stratification skip, `d` toggle reduction skip,
149
+ `space` select row, `x` clear selection, `ctrl+z`/`ctrl+y` undo/redo,
150
+ `escape` back. Report screen: `ctrl+s` save report to text, `n` new file.
151
+ Everywhere: `ctrl+q` quit.
140
152
 
141
153
  ## CLI (headless)
142
154
 
@@ -154,6 +166,11 @@ data-sampler <source> <count> [options]
154
166
  | `--anon COL=KIND[:k=v,...]` | Anonymize a column (repeatable) |
155
167
  | `-i`, `--interactive` | Guided workflow: choose an anonymizer type per column from a menu |
156
168
  | `--suggest` | Auto-assign a suggested anonymizer type to each column from its stats |
169
+ | `--reduce-components N` | PCA column reduction: replace the numeric columns with the first `N` principal components |
170
+ | `--reduce-variance R` | PCA column reduction: keep the fewest components whose cumulative explained variance reaches `R` (0 < R < 1) |
171
+ | `--reduce-exclude COL[,COL]` | Numeric column(s) to keep out of the reduction, e.g. identifiers (repeatable) |
172
+ | `--reduce-prefix PREFIX` | Name prefix for the component columns (default `PC` → `PC1`, `PC2`, …) |
173
+ | `--reduce-no-standardize` | Skip the per-column z-scoring before PCA |
157
174
  | `--engine {auto,pandas,duckdb}` | Sampling engine (default `auto`: DuckDB for Parquet/large inputs, pandas otherwise) |
158
175
  | `--threads N` | DuckDB engine: number of threads (default: all cores) |
159
176
  | `--memory-limit SIZE` | DuckDB engine: memory limit before spilling to disk (e.g. `8GB`) |
@@ -172,6 +189,11 @@ data-sampler data.csv 100 --skip region,notes --seed 7 \
172
189
 
173
190
  # large / out-of-core: sample a Parquet file in parallel with DuckDB
174
191
  data-sampler huge.parquet 10000 --engine duckdb --threads 8 --memory-limit 8GB --suggest
192
+
193
+ # narrow the sample too: collapse the numeric columns into 3 principal
194
+ # components (or keep however many retain 90% of the variance)
195
+ data-sampler wide.csv 500 --reduce-components 3 --reduce-exclude cust_id
196
+ data-sampler wide.csv 500 --reduce-variance 0.9
175
197
  ```
176
198
 
177
199
  ## Python API
@@ -204,7 +226,11 @@ anon = ds.anonymize(
204
226
  seed=7,
205
227
  )
206
228
 
207
- ds.save_output(anon, "data.xlsx", tag="sample_500_anon")
229
+ # optionally collapse the numeric columns into principal components
230
+ red = ds.reduce_columns(anon, variance_ratio=0.9, exclude=["cust_id"])
231
+ print(ds.format_reduction_report(red)) # variance kept + correlated groups
232
+
233
+ ds.save_output(red.data, "data.xlsx", tag="sample_500_anon_pca")
208
234
  ```
209
235
 
210
236
  ## Try it: bundled example
@@ -372,6 +398,34 @@ category. A side-by-side distribution report is produced for every run.
372
398
  If no suitable stratification columns exist, the tool falls back to pure
373
399
  random sampling automatically.
374
400
 
401
+ ### Reducing columns (PCA)
402
+
403
+ Sampling narrows the *rows*; the optional PCA step narrows the *columns* of
404
+ the outgoing sample. It replaces the numeric block with its first *k*
405
+ principal components (`PC1..PCk`), controlled one of two ways:
406
+
407
+ - **`--reduce-components N`** — `N` components in the output (capped at the
408
+ number of usable numeric columns, with a note when fewer are possible);
409
+ - **`--reduce-variance R`** — the fewest components whose cumulative
410
+ explained-variance ratio reaches `R` (e.g. `0.9` keeps ≥ 90 % of the
411
+ numeric variance).
412
+
413
+ Non-numeric columns (ids, categories, text, booleans, datetimes) are always
414
+ preserved, and `--reduce-exclude` keeps chosen numeric columns out too —
415
+ identifiers should be excluded, since an all-unique id forms its own
416
+ artificial component (the tool warns when it spots one). Missing values are
417
+ mean-imputed so the row count never changes; constant columns carry no signal
418
+ and pass through unchanged. Columns are z-scored first by default (PCA on the
419
+ correlation matrix), so a large-unit column such as a salary cannot dominate
420
+ the components; `--reduce-no-standardize` turns that off.
421
+
422
+ Every reduction prints its rationale: the variance each component retains,
423
+ **the groups of correlated columns** that move together (which is exactly the
424
+ redundancy PCA collapses — on standardized data PCA diagonalizes the
425
+ correlation matrix), and each component's top driving columns. The reduction
426
+ runs after anonymization, on the already-sampled rows, so the stratification
427
+ and histogram reports still describe the original columns.
428
+
375
429
  ## Large data: the out-of-core DuckDB engine
376
430
 
377
431
  The default pandas path loads the whole file into memory. For inputs that are
@@ -493,6 +547,11 @@ columns, in one place.
493
547
  (~1.9× faster stats), report histograms skip near-unique columns instead
494
548
  of hashing millions of ids for a meaningless top-8, and the TUI computes
495
549
  stats in a worker thread so the UI never freezes on load.
550
+ - **PCA column reduction on the way out.** `--reduce-components` /
551
+ `--reduce-variance` collapse a wide numeric block into a handful of
552
+ principal components *after* sampling, so the SVD runs on the small sampled
553
+ frame (never the full source) and the delivered file is narrow as well as
554
+ short.
496
555
 
497
556
  ### Correctness under scale
498
557
 
@@ -520,7 +579,8 @@ multi-threading for reproducibility (unseeded runs use all cores).
520
579
 
521
580
  Output keeps the source format and is named
522
581
  `{stem}_sample_{count}{ext}` — with an `_anon` suffix when anonymization ran
523
- (e.g. `data_sample_500_anon.csv`).
582
+ and a `_pca{k}` suffix when PCA column reduction ran
583
+ (e.g. `data_sample_500_anon_pca3.csv`).
524
584
 
525
585
  ## Development
526
586
 
@@ -65,23 +65,35 @@ data_sampler.run_tui("data.csv") # pre-load a file
65
65
 
66
66
  The TUI is a panel-based dashboard (think btop / lazydocker):
67
67
 
68
- 1. **File screen** — type a path or pick a file from the directory browser;
69
- Excel files take an optional sheet name.
68
+ 1. **File screen** — type a path or pick a file from the directory browser
69
+ (**`ctrl+r`** or the **⟳ refresh** button re-scans the folder so files
70
+ created since launch show up); Excel files take an optional sheet name.
70
71
  2. **Columns screen** — every column with its type, missing %, unique count,
71
- distribution sparkline, and summary (modelled after the Data Wrangler
72
- VS Code extension). Select a column to see full stats and distribution
73
- bars, choose an anonymizer for it, and toggle whether it should be
74
- skipped when preserving statistical variety (stratification). Set the
75
- sample size, output folder, optional seed, and run.
72
+ distribution sparkline, and per-stat **mean / median / mode / sd** columns
73
+ (modelled after the Data Wrangler VS Code extension). Select a column to
74
+ see full stats and distribution bars, choose an anonymizer for it, and
75
+ toggle whether it should be skipped when preserving statistical variety
76
+ (stratification) or excluded from the PCA reduction. **Multi-select** rows
77
+ to configure them in bulk: **ctrl-click** toggles individual rows,
78
+ **shift-click** selects a range (**space** toggles the cursor row from the
79
+ keyboard, **`x`** clears) — then any anonymizer / skip / reduce choice
80
+ applies to every selected column at once. **`ctrl+z` / `ctrl+y`** undo and
81
+ redo column-config changes (at least the last ten steps). Set the sample
82
+ size, output folder, optional seed, an optional PCA column reduction
83
+ (`reduce`: N components or a variance target), and run.
76
84
  3. **Report screen** — the stratification comparison and anonymization summary
77
85
  on the left, and a **column histograms** panel on the right showing every
78
86
  column's source-vs-sample distribution (numeric columns share bin edges;
79
87
  others use the source's top categories) so you can see at a glance how well
80
- the sample preserved each column. The output path is shown too.
88
+ the sample preserved each column. The output path is shown too, and
89
+ **`ctrl+s`** (or **💾 save .txt**) writes the report and histograms to a
90
+ `*_report.txt` beside the sample.
81
91
 
82
- Key bindings: `ctrl+r` run sample, `a` auto-suggest anonymizer types,
83
- `s` toggle stratification skip,
84
- `escape` back, `ctrl+q` quit.
92
+ Key bindings — columns screen: `ctrl+r` run sample, `a` auto-suggest
93
+ anonymizer types, `s` toggle stratification skip, `d` toggle reduction skip,
94
+ `space` select row, `x` clear selection, `ctrl+z`/`ctrl+y` undo/redo,
95
+ `escape` back. Report screen: `ctrl+s` save report to text, `n` new file.
96
+ Everywhere: `ctrl+q` quit.
85
97
 
86
98
  ## CLI (headless)
87
99
 
@@ -99,6 +111,11 @@ data-sampler <source> <count> [options]
99
111
  | `--anon COL=KIND[:k=v,...]` | Anonymize a column (repeatable) |
100
112
  | `-i`, `--interactive` | Guided workflow: choose an anonymizer type per column from a menu |
101
113
  | `--suggest` | Auto-assign a suggested anonymizer type to each column from its stats |
114
+ | `--reduce-components N` | PCA column reduction: replace the numeric columns with the first `N` principal components |
115
+ | `--reduce-variance R` | PCA column reduction: keep the fewest components whose cumulative explained variance reaches `R` (0 < R < 1) |
116
+ | `--reduce-exclude COL[,COL]` | Numeric column(s) to keep out of the reduction, e.g. identifiers (repeatable) |
117
+ | `--reduce-prefix PREFIX` | Name prefix for the component columns (default `PC` → `PC1`, `PC2`, …) |
118
+ | `--reduce-no-standardize` | Skip the per-column z-scoring before PCA |
102
119
  | `--engine {auto,pandas,duckdb}` | Sampling engine (default `auto`: DuckDB for Parquet/large inputs, pandas otherwise) |
103
120
  | `--threads N` | DuckDB engine: number of threads (default: all cores) |
104
121
  | `--memory-limit SIZE` | DuckDB engine: memory limit before spilling to disk (e.g. `8GB`) |
@@ -117,6 +134,11 @@ data-sampler data.csv 100 --skip region,notes --seed 7 \
117
134
 
118
135
  # large / out-of-core: sample a Parquet file in parallel with DuckDB
119
136
  data-sampler huge.parquet 10000 --engine duckdb --threads 8 --memory-limit 8GB --suggest
137
+
138
+ # narrow the sample too: collapse the numeric columns into 3 principal
139
+ # components (or keep however many retain 90% of the variance)
140
+ data-sampler wide.csv 500 --reduce-components 3 --reduce-exclude cust_id
141
+ data-sampler wide.csv 500 --reduce-variance 0.9
120
142
  ```
121
143
 
122
144
  ## Python API
@@ -149,7 +171,11 @@ anon = ds.anonymize(
149
171
  seed=7,
150
172
  )
151
173
 
152
- ds.save_output(anon, "data.xlsx", tag="sample_500_anon")
174
+ # optionally collapse the numeric columns into principal components
175
+ red = ds.reduce_columns(anon, variance_ratio=0.9, exclude=["cust_id"])
176
+ print(ds.format_reduction_report(red)) # variance kept + correlated groups
177
+
178
+ ds.save_output(red.data, "data.xlsx", tag="sample_500_anon_pca")
153
179
  ```
154
180
 
155
181
  ## Try it: bundled example
@@ -317,6 +343,34 @@ category. A side-by-side distribution report is produced for every run.
317
343
  If no suitable stratification columns exist, the tool falls back to pure
318
344
  random sampling automatically.
319
345
 
346
+ ### Reducing columns (PCA)
347
+
348
+ Sampling narrows the *rows*; the optional PCA step narrows the *columns* of
349
+ the outgoing sample. It replaces the numeric block with its first *k*
350
+ principal components (`PC1..PCk`), controlled one of two ways:
351
+
352
+ - **`--reduce-components N`** — `N` components in the output (capped at the
353
+ number of usable numeric columns, with a note when fewer are possible);
354
+ - **`--reduce-variance R`** — the fewest components whose cumulative
355
+ explained-variance ratio reaches `R` (e.g. `0.9` keeps ≥ 90 % of the
356
+ numeric variance).
357
+
358
+ Non-numeric columns (ids, categories, text, booleans, datetimes) are always
359
+ preserved, and `--reduce-exclude` keeps chosen numeric columns out too —
360
+ identifiers should be excluded, since an all-unique id forms its own
361
+ artificial component (the tool warns when it spots one). Missing values are
362
+ mean-imputed so the row count never changes; constant columns carry no signal
363
+ and pass through unchanged. Columns are z-scored first by default (PCA on the
364
+ correlation matrix), so a large-unit column such as a salary cannot dominate
365
+ the components; `--reduce-no-standardize` turns that off.
366
+
367
+ Every reduction prints its rationale: the variance each component retains,
368
+ **the groups of correlated columns** that move together (which is exactly the
369
+ redundancy PCA collapses — on standardized data PCA diagonalizes the
370
+ correlation matrix), and each component's top driving columns. The reduction
371
+ runs after anonymization, on the already-sampled rows, so the stratification
372
+ and histogram reports still describe the original columns.
373
+
320
374
  ## Large data: the out-of-core DuckDB engine
321
375
 
322
376
  The default pandas path loads the whole file into memory. For inputs that are
@@ -438,6 +492,11 @@ columns, in one place.
438
492
  (~1.9× faster stats), report histograms skip near-unique columns instead
439
493
  of hashing millions of ids for a meaningless top-8, and the TUI computes
440
494
  stats in a worker thread so the UI never freezes on load.
495
+ - **PCA column reduction on the way out.** `--reduce-components` /
496
+ `--reduce-variance` collapse a wide numeric block into a handful of
497
+ principal components *after* sampling, so the SVD runs on the small sampled
498
+ frame (never the full source) and the delivered file is narrow as well as
499
+ short.
441
500
 
442
501
  ### Correctness under scale
443
502
 
@@ -465,7 +524,8 @@ multi-threading for reproducibility (unseeded runs use all cores).
465
524
 
466
525
  Output keeps the source format and is named
467
526
  `{stem}_sample_{count}{ext}` — with an `_anon` suffix when anonymization ran
468
- (e.g. `data_sample_500_anon.csv`).
527
+ and a `_pca{k}` suffix when PCA column reduction ran
528
+ (e.g. `data_sample_500_anon_pca3.csv`).
469
529
 
470
530
  ## Development
471
531
 
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "data-sampler"
7
- version = "3.3.1"
7
+ version = "3.4.1"
8
8
  description = "Hand someone data that looks and behaves like your production data, isn't your production data, and provably kept its statistical variety — stratified sampling + consistent anonymization, with a terminal UI."
9
9
  readme = "README.md"
10
10
  license = { file = "LICENSE" }
@@ -9,20 +9,24 @@ Public API::
9
9
  result = ds.sample(df, 500, exclude_columns=["notes"])
10
10
  print(ds.format_stratification_report(df, result))
11
11
  anon = ds.anonymize(result.data, {"name": "names", "id": ("sequential_id", {"start": 1000})})
12
- ds.save_output(anon, "data.csv", tag="sample_500_anon")
12
+ red = ds.reduce_columns(anon, variance_ratio=0.9, exclude=["id"]) # optional PCA
13
+ print(ds.format_reduction_report(red))
14
+ ds.save_output(red.data, "data.csv", tag="sample_500_anon_pca")
13
15
 
14
16
  ds.run_tui() # launch the terminal UI
15
17
  """
16
18
 
17
19
  from __future__ import annotations
18
20
 
19
- __version__ = "3.3.1"
21
+ __version__ = "3.4.1"
20
22
 
21
23
  from .io import SUPPORTED_EXTENSIONS, list_sheets, load_file, save_output
24
+ from .reduce import ReductionResult, reduce_columns
22
25
  from .report import (
23
26
  column_histogram_data,
24
27
  format_column_histograms,
25
28
  format_distribution,
29
+ format_reduction_report,
26
30
  format_stratification_report,
27
31
  )
28
32
  from .sampling import (
@@ -47,7 +51,10 @@ __all__ = [
47
51
  "sample",
48
52
  "stratified_sample",
49
53
  "find_stratification_columns",
54
+ "ReductionResult",
55
+ "reduce_columns",
50
56
  "format_stratification_report",
57
+ "format_reduction_report",
51
58
  "format_distribution",
52
59
  "column_histogram_data",
53
60
  "format_column_histograms",
@@ -13,6 +13,12 @@ which picks DuckDB for Parquet/large files) samples out-of-core in parallel:
13
13
 
14
14
  data-sampler huge.parquet 10000 --engine duckdb --threads 8 \\
15
15
  --memory-limit 8GB --suggest
16
+
17
+ Optionally reduce the numeric columns of the outgoing sample to K principal
18
+ components — or to however many retain a target share of the variance:
19
+
20
+ data-sampler data.csv 500 --reduce-components 3 --reduce-exclude id
21
+ data-sampler data.csv 500 --reduce-variance 0.9
16
22
  """
17
23
 
18
24
  from __future__ import annotations
@@ -24,7 +30,12 @@ from . import __version__
24
30
  from ._logging import get_logger
25
31
  from .anonymize import anonymize, make_anonymizer
26
32
  from .io import load_file, save_output
27
- from .report import format_column_histograms, format_stratification_report
33
+ from .reduce import reduce_columns
34
+ from .report import (
35
+ format_column_histograms,
36
+ format_reduction_report,
37
+ format_stratification_report,
38
+ )
28
39
  from .sampling import sample
29
40
 
30
41
  log = get_logger(__name__)
@@ -41,6 +52,24 @@ def _coerce_value(value: str):
41
52
  return value
42
53
 
43
54
 
55
+ def _positive_int(text: str) -> int:
56
+ """argparse type: an integer >= 1 (rejects bad values at parse time)."""
57
+ value = int(text)
58
+ if value < 1:
59
+ raise argparse.ArgumentTypeError(f"must be a positive integer, got {text}")
60
+ return value
61
+
62
+
63
+ def _unit_interval(text: str) -> float:
64
+ """argparse type: a float strictly between 0 and 1."""
65
+ value = float(text)
66
+ if not 0.0 < value < 1.0:
67
+ raise argparse.ArgumentTypeError(
68
+ f"must be strictly between 0 and 1, got {text}"
69
+ )
70
+ return value
71
+
72
+
44
73
  def parse_anon_option(text: str) -> tuple[str, str, dict]:
45
74
  """Parse ``COL=KIND[:key=value,key=value]`` into (column, kind, options)."""
46
75
  col, sep, rhs = text.partition("=")
@@ -110,6 +139,46 @@ def build_parser() -> argparse.ArgumentParser:
110
139
  "stats (columns set via --anon are left as given)"
111
140
  ),
112
141
  )
142
+ reduce_group = parser.add_mutually_exclusive_group()
143
+ reduce_group.add_argument(
144
+ "--reduce-components", type=_positive_int, metavar="N",
145
+ dest="reduce_components",
146
+ help=(
147
+ "PCA column reduction: replace the numeric columns with the first "
148
+ "N principal components (non-numeric columns are preserved; N is "
149
+ "capped at the number of usable numeric columns)"
150
+ ),
151
+ )
152
+ reduce_group.add_argument(
153
+ "--reduce-variance", type=_unit_interval, metavar="R",
154
+ dest="reduce_variance",
155
+ help=(
156
+ "PCA column reduction: keep the fewest principal components whose "
157
+ "cumulative explained-variance ratio reaches R (0 < R < 1)"
158
+ ),
159
+ )
160
+ parser.add_argument(
161
+ "--reduce-exclude", action="append", default=[], metavar="COL[,COL...]",
162
+ dest="reduce_exclude",
163
+ help=(
164
+ "numeric column(s) to exclude from PCA reduction and pass through "
165
+ "unchanged, e.g. identifiers (repeatable)"
166
+ ),
167
+ )
168
+ parser.add_argument(
169
+ "--reduce-prefix", default="PC", metavar="PREFIX", dest="reduce_prefix",
170
+ help=(
171
+ "name prefix for the principal-component columns (default PC → "
172
+ "PC1, PC2, …); change it if the source already has such a column"
173
+ ),
174
+ )
175
+ parser.add_argument(
176
+ "--reduce-no-standardize", action="store_true", dest="reduce_no_standardize",
177
+ help=(
178
+ "PCA column reduction: skip the per-column z-scoring (by default "
179
+ "columns are standardized so large-unit columns cannot dominate)"
180
+ ),
181
+ )
113
182
  parser.add_argument(
114
183
  "--engine", choices=("auto", "pandas", "duckdb"), default="auto",
115
184
  help=(
@@ -136,6 +205,65 @@ def build_parser() -> argparse.ArgumentParser:
136
205
  return parser
137
206
 
138
207
 
208
+ def _reduce_requested(args) -> bool:
209
+ return args.reduce_components is not None or args.reduce_variance is not None
210
+
211
+
212
+ def _validate_reduce_args(args, parser, columns) -> list[str]:
213
+ """Validate the reduce flags as soon as the source's columns are known —
214
+ BEFORE sampling, so a typo never costs a full (possibly out-of-core) run.
215
+
216
+ Returns the parsed ``--reduce-exclude`` column list.
217
+ """
218
+ exclude = [
219
+ c.strip() for chunk in args.reduce_exclude for c in chunk.split(",") if c.strip()
220
+ ]
221
+ if not _reduce_requested(args):
222
+ if exclude:
223
+ parser.error(
224
+ "--reduce-exclude requires --reduce-components or --reduce-variance"
225
+ )
226
+ if args.reduce_no_standardize:
227
+ parser.error(
228
+ "--reduce-no-standardize requires --reduce-components or "
229
+ "--reduce-variance"
230
+ )
231
+ return exclude
232
+ # compare stringified: Excel sources can carry non-string column labels
233
+ names = {str(c) for c in columns}
234
+ unknown = [c for c in exclude if c not in names]
235
+ if unknown:
236
+ parser.error(
237
+ f"--reduce-exclude: column(s) not found in file: {', '.join(unknown)}"
238
+ )
239
+ return exclude
240
+
241
+
242
+ def _apply_reduction(args, parser, data, exclude):
243
+ """Run the optional PCA column reduction on the outgoing sample.
244
+
245
+ Returns ``(data, tag_suffix)`` — unchanged when no reduction was requested.
246
+ """
247
+ if not _reduce_requested(args):
248
+ return data, ""
249
+ try:
250
+ reduction = reduce_columns(
251
+ data,
252
+ n_components=args.reduce_components,
253
+ variance_ratio=args.reduce_variance,
254
+ exclude=exclude,
255
+ standardize=not args.reduce_no_standardize,
256
+ prefix=args.reduce_prefix,
257
+ seed=args.seed,
258
+ )
259
+ except ValueError as exc:
260
+ parser.error(f"column reduction: {exc}")
261
+ print()
262
+ print(format_reduction_report(reduction))
263
+ suffix = f"_pca{reduction.n_components}" if reduction.n_components else ""
264
+ return reduction.data, suffix
265
+
266
+
139
267
  def _build_engine_spec(args, parser, engine, cols) -> dict:
140
268
  """Build the anonymizer spec for the engine path (validates against ``cols``,
141
269
  supports explicit --anon and --suggest via the engine's approximate stats)."""
@@ -189,6 +317,7 @@ def _run_with_engine(args, parser, skip) -> int:
189
317
  parser.error(
190
318
  f"--skip: column(s) not found in file: {', '.join(unknown_skip)}"
191
319
  )
320
+ reduce_exclude = _validate_reduce_args(args, parser, cols)
192
321
  spec = _build_engine_spec(args, parser, engine, cols)
193
322
  result = engine.sample(
194
323
  args.source, args.count,
@@ -213,7 +342,8 @@ def _run_with_engine(args, parser, skip) -> int:
213
342
  parser.error(f"anonymization failed: {exc}")
214
343
  print(f"Anonymized columns: {', '.join(spec)}")
215
344
 
216
- tag = f"sample_{args.count}" + ("_anon" if spec else "")
345
+ data, reduce_tag = _apply_reduction(args, parser, data, reduce_exclude)
346
+ tag = f"sample_{args.count}" + ("_anon" if spec else "") + reduce_tag
217
347
  out_path = save_output(data, args.source, tag, output_folder=args.outdir)
218
348
  print(f"\nSampled {len(data):,} rows.")
219
349
  print(f"Output saved to: {out_path}")
@@ -288,6 +418,7 @@ def main(argv: list[str] | None = None) -> int:
288
418
  unknown_skip = [c for c in skip if c not in df.columns]
289
419
  if unknown_skip:
290
420
  parser.error(f"--skip: column(s) not found in file: {', '.join(unknown_skip)}")
421
+ reduce_exclude = _validate_reduce_args(args, parser, df.columns)
291
422
 
292
423
  # parse explicit --anon flags into (column, kind, options) triples
293
424
  anon_specs: list[tuple[str, str, dict]] = []
@@ -352,7 +483,8 @@ def main(argv: list[str] | None = None) -> int:
352
483
  parser.error(f"anonymization failed: {exc}")
353
484
  print(f"Anonymized columns: {', '.join(spec)}")
354
485
 
355
- tag = f"sample_{args.count}" + ("_anon" if spec else "")
486
+ data, reduce_tag = _apply_reduction(args, parser, data, reduce_exclude)
487
+ tag = f"sample_{args.count}" + ("_anon" if spec else "") + reduce_tag
356
488
  out_path = save_output(data, args.source, tag, output_folder=args.outdir)
357
489
  print(f"\nSampled {len(data)} rows.")
358
490
  print(f"Output saved to: {out_path}")