data-sampler 3.2.1__tar.gz

data_sampler-3.2.1/.gitattributes

@@ -0,0 +1,2 @@
+# Auto detect text files and perform LF normalization
+* text=auto

data_sampler-3.2.1/.github/workflows/release.yml

@@ -0,0 +1,75 @@
+name: Release to PyPI
+
+# Publishes to PyPI via Trusted Publishing (OIDC) — no API token stored anywhere.
+# One-time setup on the PyPI side is documented in RELEASING.md.
+#
+# Triggered by publishing a GitHub Release (recommended) or manually from the
+# Actions tab. It will NOT fire on ordinary pushes or tags, so nothing is
+# uploaded by accident.
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+  test:
+    name: Test suite
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        python-version: ["3.10", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install package with dev extras
+        run: python -m pip install -e ".[dev]"
+      - name: Run tests
+        run: python -m pytest -q
+
+  build:
+    name: Build distributions
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Build sdist and wheel
+        run: |
+          python -m pip install build twine
+          python -m build
+          python -m twine check dist/*
+      - name: Smoke-test the wheel
+        run: |
+          python -m pip install dist/*.whl
+          python -c "import data_sampler; print('wheel ok', data_sampler.__version__)"
+          data-sampler --version
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    # The 'pypi' environment lets you require manual approval in repo settings
+    # before anything is uploaded (Settings > Environments > pypi).
+    environment:
+      name: pypi
+      url: https://pypi.org/p/data-sampler
+    permissions:
+      id-token: write  # required for Trusted Publishing (OIDC)
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - name: Publish
+        uses: pypa/gh-action-pypi-publish@release/v1

data_sampler-3.2.1/.gitignore

@@ -0,0 +1,192 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+build/
+dist/
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Cursor  
+#  Cursor is an AI-powered code editor.`.cursorignore` specifies files/directories to 
+#  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
+#  refer to https://docs.cursor.com/context/ignore-files
+.cursorignore
+.cursorindexingignore
+
+# VS Code
+.vscode/*
+
+# Built Visual Studio Code Extensions
+*.vsix
+
+# Claude folder
+.claude/

data_sampler-3.2.1/CHANGELOG.md

@@ -0,0 +1,224 @@
+# Changelog
+
+## v3.2.1 — 2026-07-23
+
+- **Fixed a TUI race that only surfaced on slow machines** (caught by the
+  v3.2.0 release CI on Windows runners; it blocked the PyPI publish): a late
+  mount-time row-highlight for the already-selected column could re-sync the
+  config panel and queue a stale `Changed` message that reset a just-made
+  anonymizer choice back to `none`. Duplicate highlights are now no-ops and
+  the panel sync no longer queues gratuitous widget updates. This is the
+  first release published to PyPI (v3.2.0's publish run failed on this bug).
+
+## v3.2.0 — 2026-07-23
+
+- **Measured parallel scaling** (20M-row Parquet, 12-core machine,
+  10k-row sample): stratified sampling 14.5 s → 3.9 s from 1 → 8 threads
+  (3.7×), `stats()` 9.5 s → 1.7 s (5.6×), reservoir sampling 0.10 s —
+  ~50× faster than the pandas path on the same file, with no ~0.9 GB
+  in-memory materialization.
+
+- **Vectorized every anonymizer** (Block P1 of the v3.2 performance & scale
+  effort): the transform pipeline now does a single `pd.factorize`
+  (dictionary-encode) plus a vectorized gather instead of `pd.unique` + a
+  per-unique Python dict + `Series.map` (the in-process equivalent of a native
+  join against a mapping table). Sequential IDs use `np.arange`;
+  numeric/datetime jitter use vectorized numpy draws. Benchmarks: ~4–6× faster
+  on `sequential_id` and up to ~3.3× on `numeric_jitter`, with `sequential_id`
+  output verified bit-identical to the old path. The public `build_mapping`
+  API, consistent-mapping guarantee, seed reproducibility, and NaN/dtype
+  handling are preserved; nullable string dtypes now round-trip. Docstrings
+  clarified that relabelling anonymizers (`names`, `sequential_id`,
+  `random_string`, `hex`) are bijective while jitter anonymizers
+  (`numeric_jitter`, `datetime_jitter`) are bounded noise (distinct nearby
+  values may collide).
+- **Added an optional DuckDB out-of-core engine** (`data_sampler.engine`,
+  Blocks P2–P4 of the v3.2 performance & scale effort): install
+  `pip install "data-sampler[large]"` to push loading, stratification, and
+  sampling into DuckDB instead of pandas. Multi-threaded (`PRAGMA threads`)
+  and memory-limited with spill-to-disk; reads CSV/TSV/JSON and Parquet
+  natively (Parquet with projection pushdown, the biggest I/O win) plus
+  pandas DataFrames — only the resulting sample is ever materialized.
+  Sampling: reservoir sampling for the random case (exact count, single
+  pass, reproducible via `REPEATABLE`) and two-pass proportional stratified
+  sampling, with auto-stratification picking low-cardinality columns via
+  HyperLogLog (`approx_count_distinct`). Seed-reproducible end to end
+  (seeded stratified runs go single-threaded, since DuckDB's `random()`
+  ordering is only reproducible that way); NaN strata are joined with
+  `IS NOT DISTINCT FROM` so they aren't silently dropped; column identifiers
+  are quoted against injection. New `should_use_engine` auto-selects the
+  engine for Parquet/large inputs, and `large_materialization_warning` warns
+  before loading a large dataset fully into pandas — the pandas path stays
+  the default for small/medium data. Adversarially verified across 4 review
+  lenses with zero findings; a 2M-row Parquet file sampled to 1000 rows in
+  ~1s, out-of-core.
+- **Added approximate stats at scale** (`DuckDBEngine.stats()` and
+  module-level `engine.stats()`, Block P5 of the v3.2 performance & scale
+  effort): per-column `ColumnStats` computed in DuckDB — distinct counts via
+  HyperLogLog (`approx_count_distinct`), median via `approx_quantile`, plus
+  min/max/mean/std, missing counts, equal-width numeric histograms, and
+  categorical/datetime top-values. Scalar aggregates run in one streaming
+  pass so stats stay cheap over billions of rows; `approximate=False` gives
+  exact counts/quantiles for small inputs, and `distributions=False` skips
+  the per-column passes for a single cheap scalar pass across very wide
+  inputs. A new `ColumnStats.approximate` flag marks approximate results.
+- **Wired the DuckDB engine into the CLI** (Block P6 of the v3.2 performance
+  & scale effort): new `--engine {auto,pandas,duckdb}` flag (default `auto`
+  — DuckDB for Parquet/large inputs, pandas otherwise), plus `--threads` and
+  `--memory-limit` to tune the engine. The engine path supports
+  `--random`/`--skip`/`--seed`/`--anon` and `--suggest` (suggestions computed
+  from the engine's approximate stats, without materializing the full
+  input); `--interactive` remains pandas-only. When the pandas path is used
+  on a large input, a note suggests the engine. Auto-selection falls back to
+  pandas if the optional `duckdb` dependency is absent.
+- **Fixed 22 issues found in a pre-release six-lens adversarial audit** (one
+  accepted divergence documented in `TROUBLESHOOTING.md`): the DuckDB
+  engine's `stats()` no longer crashes on LIST/STRUCT columns or on float
+  files containing real NaN values (every NaN-sensitive aggregate is now
+  NaN-filtered); `TIME`/`INTERVAL` and nested types (`LIST`, `STRUCT`,
+  `MAP`, …) classify as `"other"` and are never treated as numeric or
+  datetime; seeded stratified sampling is now deterministic even under
+  allocation-remainder ties (stratum order is pinned before allocation);
+  columns-oriented JSON (the pandas `to_json()` default) is detected and
+  refused with guidance instead of silently sampling a single row; `--skip`
+  column names are validated against the source; `--engine auto` now
+  genuinely falls back to pandas on a DuckDB read failure instead of
+  raising.
+- Alias anonymizer kinds (e.g. `"seq"` for `sequential_id`) now canonicalize
+  on `assign`, so the interactive wizard no longer silently resets them to
+  `none`; accepting a seeded assignment in the wizard preserves its options
+  instead of discarding them. HyperLogLog-based unique counts are clamped to
+  the row count so they can no longer push `unique_pct` past 100%.
+- Performance/robustness: row counts are cached per source (3–4 fewer full
+  scans per CLI run); `compute_stats` skips the useless top-values pass for
+  numeric columns (~1.9× faster); the TUI now computes stats off the event
+  loop so loading a large file no longer freezes the UI; report histograms
+  exclude ±inf from their percentage denominators and skip near-unique
+  columns instead of drawing a meaningless top-8.
+
+## v3.1.0 — 2026-07-23 (released with v3.2.0)
+
+- **Added a `datetime_jitter` anonymizer** (`DatetimeJitterAnonymizer`):
+  shifts each date/time by a random offset within a ±window (±7 days by
+  default), with consistent value mapping preserved so the column's
+  distribution shape survives. `NaT` is left untouched, string-date columns
+  are coerced via `pandas.to_datetime`, and timezone-aware inputs keep their
+  zone. Raises `ValueError` if the window is finer than its `unit`. Reachable
+  as kinds `datetime_jitter` / `datetime` / `dates`, and wired into the CLI
+  `--anon` option and the TUI anonymizer config panel.
+- **Added a guided anonymization workflow** (`data_sampler.workflow`): an
+  `AnonymizationPlan` maps each column to a "type" (anonymizer), buildable
+  three ways — programmatically (`assign`/`suggest`), interactively via a
+  menu wizard (`choose_interactively`, CLI `-i`/`--interactive`), or by
+  clicking in the TUI (new `a` auto-suggest action). `suggest_type` infers a
+  type per column from its stats, including date-named string columns →
+  `datetime_jitter`, using token-based matching so "candidate"/"mandate"
+  aren't misread as dates. New public API: `AnonymizationPlan`,
+  `suggest_type`, `TYPE_OPTIONS`. CLI gains `--suggest`. The TUI columns
+  table was reordered so the anonymizer/strat columns stay visible.
+- **Added column-level histograms** (`column_histogram_data`,
+  `format_column_histograms` in `report`): per-column source-vs-sample
+  distributions — numeric columns share bin edges, other columns use the
+  source's top categories — computed from the pre-anonymization sample so
+  they show how faithfully the sample preserved each column. Surfaced as a
+  new right-hand "column histograms" panel on the TUI report screen and a
+  "COLUMN DISTRIBUTIONS" section in the CLI output. New public API:
+  `column_histogram_data`, `format_column_histograms`.
+
+## v3.0.1 — 2026-07-19
+
+- **Fixed the bundled name library.** The v3.0.0 library contained invented
+  non-name strings (e.g. "Argon", "Ardwin"), so the `names` anonymizer could
+  produce gibberish. Replaced with a curated list of 200 first, 60 middle,
+  and 220 last real names, globally diverse.
+- **Added a bundled example dataset** (`examples/employees.csv`, 1,000 rows,
+  stratifiable) and a worked example section in the README covering the
+  TUI, the Python API, and the CLI.
+
+## v3.0 — 2026-07-19
+
+### Package restructure
+
+- **Proper Python package.** The flat scripts are gone; everything now lives
+  in a src-layout package (`src/data_sampler/`) built with hatchling and
+  installable via `pip install -e .` (wheel-ready for a future PyPI
+  release — the release itself stays manual, after extensive testing).
+- **Public API.** `load_file`, `list_sheets`, `save_output`,
+  `compute_stats`/`ColumnStats`, `sample`/`SampleResult`,
+  `stratified_sample`, `find_stratification_columns`,
+  `format_stratification_report`, `anonymize`, `make_anonymizer`, and
+  `run_tui` are importable from `data_sampler`.
+- **Old Tkinter GUI removed** (`data-sampler-gui.py`), replaced by the TUI.
+- **Central logging** behind `DATA_SAMPLER_LOG` / `DATA_SAMPLER_LOG_FILE`.
+
+### Terminal UI (new)
+
+- Colorful, panel-based Textual dashboard (btop / lazydocker style):
+  file picker with directory browser, Data Wrangler-style column stats
+  table (type, missing %, unique count, distribution sparkline, summary),
+  per-column detail panel with distribution bars, anonymizer configuration,
+  stratification skip toggles, and a post-run report screen.
+- Launch via `data-sampler` (no args), `data-sampler-tui`,
+  `python -m data_sampler`, or `data_sampler.run_tui()`.
+
+### Anonymizers (new)
+
+- Optional per-column anonymization with **consistent value mapping**
+  (repeated values stay repeated, so distributions survive) and NaN
+  passthrough: `names` (bundled first/middle/last name library, five
+  styles), `sequential_id` (start + interval, optional prefix/zero-pad),
+  `numeric_jitter` (±20 % by default), `random_string` / `hex`
+  (configurable length/charset). Seedable for reproducible runs.
+
+### Sampling engine
+
+- `sample()` accepts `exclude_columns` (columns the user marks as skipped
+  never become stratifiers) and `random_state` for reproducible sampling.
+- Sampling functions no longer print; they return a `SampleResult` that the
+  CLI/TUI render via `format_stratification_report`.
+- Fixed string-column classification under pandas 3.0 (new default
+  `StringDtype` no longer matched the `object` dtype checks).
+
+### CLI
+
+- `data-sampler <source> <count>` now supports `--seed`,
+  `--skip COL[,COL]`, and repeatable `--anon "COL=KIND[:k=v,...]"`
+  anonymization options; with no arguments it opens the TUI.
+
+## v2.0 — 2026-04-13
+
+### Sampling engine
+
+- **Joint intersection stratification.** Stratification now groups rows by the *simultaneous* combination of all selected columns (e.g. `A=x AND B=y AND C=z`), rather than applying stratifiers sequentially. Proportional allocation is computed once per intersection group, ensuring the sample reflects the true joint distribution.
+
+- **Missing values included as a category.** Rows with `NaN` in any stratifier column are no longer silently dropped. They are treated as their own `(missing)` category and sampled proportionally alongside non-null values.
+
+- **NaN-safe indexing throughout.** All allocation and sampling operations now use positional (integer) indexing to avoid `KeyError` failures when a MultiIndex contains `NaN` keys — a limitation of pandas label-based `.at[]` / `.get_loc()` accessors.
+
+### Stratification report
+
+- **Method statement.** The report now opens with a plain-language explanation of how groups were formed, so it is clear whether allocation was joint or single-column.
+
+- **Fixed 13-character value labels.** All category labels in the report are right-justified to exactly 13 characters. Values longer than 13 characters are truncated to `10 chars + "..."`. Missing values display as `(missing)`.
+
+- **Zero-allocation flags.** Any category that received 0 samples is marked inline with `← not represented`.
+
+- **Per-category diagnosis.** When categories are unrepresented, a warning block appears showing — for each such value — the number of intersection groups it appeared in, the size of the largest group, and the minimum group size required to earn at least 1 sample. This confirms whether the cause is data sparsity or cross-column splitting.
+
+### GUI
+
+- **"Open output folder" button.** Opens the output directory in Windows Explorer after a successful run. Disabled until a run completes.
+
+- **"Copy log" button.** Copies the full contents of the log panel to the clipboard in one click.
+
+- **GUI no longer duplicates sampling logic.** The GUI now loads `data-sampler.py` directly at runtime via `importlib`, so both the CLI and the GUI always execute identical code. This applies to the built EXE as well — `data-sampler.py` is bundled alongside the executable and loaded from there.
+
+---
+
+## v1.0 — 2024
+
+- Initial release with GUI front-end and CLI.
+- Stratified sampling with automatic column selection.
+- Supports CSV, TSV, JSON, Excel, and Parquet.
+- Side-by-side distribution bar charts in the stratification report.
+- Standalone Windows EXE built with PyInstaller.

data_sampler-3.2.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 programanalytics-isb
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.