tabalyst 0.7.0__tar.gz → 0.7.2__tar.gz

{tabalyst-0.7.0/src/tabalyst.egg-info → tabalyst-0.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tabalyst
-Version: 0.7.0
+Version: 0.7.2
 Summary: Open-source toolkit for understanding and working with unfamiliar data.
 Author: Gregory Borelli
 License-Expression: MPL-2.0
@@ -82,6 +82,7 @@ Update before each new test.
 | [**Tabalyst Scan**](https://docs.tabalyst.com/scan/) | Profile datasets and generate JSON and HTML scan summaries |
 | [**Tabalyst Inspect**](https://docs.tabalyst.com/inspect/) | Discover and configure JSON collections and Excel tables |
 | [**Tabalyst Sample**](https://docs.tabalyst.com/sample/) | Create random, stratified and smart CSV samples |
+| [**Tabalyst Dedup**](https://docs.tabalyst.com/dedup/) | Find the exact duplicate rows of CSV files and write the unique and repeated rows |
 | [**Tabalyst Generate**](https://docs.tabalyst.com/generate/) | Generate synthetic CSV datasets for testing and demos |
 
 ## Tabalyst Report
@@ -145,6 +146,12 @@ Details: [Tabalyst Report](https://docs.tabalyst.com/report/),
 - Dataset dimensions, missing cells, duplicates, and quality observations.
 - A family and a type for each column (identifier, text, date, boolean...),
   with the evidence behind them, and the technical type with its error rates.
+- For an identifier column, whether the ID is unique across all rows and after
+  row deduplication: the repetitions that exact copies of whole rows explain
+  are told apart from those between distinct records, with the evidence (the
+  Scan, the second pass or a [Dedup](https://docs.tabalyst.com/dedup/) report),
+  the case of missing IDs, and a non-binding near-unique reading of the few
+  repetitions that remain.
 - A summary table with one row per column: its type, a sentence about its
   values, the missing share, its Errors and Issues with a preview of each, and
   its details; above it, the Column quality and Signals cards.
@@ -235,6 +242,34 @@ estimate the population from it. Sampling reads records as a stream.
 
 Details: [Tabalyst Sample](https://docs.tabalyst.com/sample/).
 
+## Tabalyst Dedup
+
+Find the rows that occur more than once in a CSV file, without modifying it:
+
+```console
+tabalyst dedup customers.csv
+tabalyst dedup "*.csv" -d deduped
+```
+
+Two rows are the same only when every cell is identical (case, spaces and
+leading zeros count). Dedup writes, beside the source or in `-d`,
+`customers.unique.csv` (every distinct row once, at its first place),
+`customers.duplicates.csv` (each repeated row once) and, when a row occurs three
+times or more, `customers.duplicates_all.csv` (every occurrence of the repeated
+rows), plus `customers.dedup.json`, a report whose counts always add up. A file
+without duplicate gets its report only. A batch also writes
+`dedup.summary.json`; a file that is already processed and unchanged is skipped.
+
+A result is exact or the file fails with a reason: the rows are kept in memory up
+to a budget (`--memory-budget-mb`, 512 by default) and then on disk, a file may
+hold 10,000,000 records by default (`--max-rows`), and nothing is ever
+approximated. Dedup reads CSV files only and asks no question.
+
+Details: [Tabalyst Dedup](https://docs.tabalyst.com/dedup/),
+[batches and already processed files](https://docs.tabalyst.com/dedup/batch/),
+[exactness, memory and limits](https://docs.tabalyst.com/dedup/limits/) and the
+[`dedup` command reference](https://docs.tabalyst.com/reference/cli/dedup/).
+
 ## Tabalyst Generate
 
 Generate synthetic CSV data from a packaged example (`crm`, `insurance` or
@@ -264,6 +299,7 @@ import tabalyst
 result = tabalyst.analyze("customers.csv", "customers.html")
 batch = tabalyst.generate_reports(["*.csv"], output_dir="reports")
 sample = tabalyst.sample_csv("customers.csv", method="random", rows=1000, seed=42)
+dedup = tabalyst.dedup_csv("customers.csv")
 scan = tabalyst.scan("orders.json")
 inspection = tabalyst.inspect("orders.json")
 ```
@@ -297,7 +333,8 @@ Generated JSON and HTML may contain source values and should be shared
 accordingly.
 
 Reports, samples, and scans read CSV, JSON and JSONL files as a stream, with
-memory bounded by configurable limits. An Excel workbook is read by sheet into
+memory bounded by configurable limits; Dedup keeps the rows it has seen in memory
+up to a budget, then on disk. An Excel workbook is read by sheet into
 memory, and a sheet above 1 GiB of XML is refused. See
 [limitations](https://docs.tabalyst.com/about/limitations/).
 

{tabalyst-0.7.0 → tabalyst-0.7.2}/README.md

@@ -42,6 +42,7 @@ Update before each new test.
 | [**Tabalyst Scan**](https://docs.tabalyst.com/scan/) | Profile datasets and generate JSON and HTML scan summaries |
 | [**Tabalyst Inspect**](https://docs.tabalyst.com/inspect/) | Discover and configure JSON collections and Excel tables |
 | [**Tabalyst Sample**](https://docs.tabalyst.com/sample/) | Create random, stratified and smart CSV samples |
+| [**Tabalyst Dedup**](https://docs.tabalyst.com/dedup/) | Find the exact duplicate rows of CSV files and write the unique and repeated rows |
 | [**Tabalyst Generate**](https://docs.tabalyst.com/generate/) | Generate synthetic CSV datasets for testing and demos |
 
 ## Tabalyst Report
@@ -105,6 +106,12 @@ Details: [Tabalyst Report](https://docs.tabalyst.com/report/),
 - Dataset dimensions, missing cells, duplicates, and quality observations.
 - A family and a type for each column (identifier, text, date, boolean...),
   with the evidence behind them, and the technical type with its error rates.
+- For an identifier column, whether the ID is unique across all rows and after
+  row deduplication: the repetitions that exact copies of whole rows explain
+  are told apart from those between distinct records, with the evidence (the
+  Scan, the second pass or a [Dedup](https://docs.tabalyst.com/dedup/) report),
+  the case of missing IDs, and a non-binding near-unique reading of the few
+  repetitions that remain.
 - A summary table with one row per column: its type, a sentence about its
   values, the missing share, its Errors and Issues with a preview of each, and
   its details; above it, the Column quality and Signals cards.
@@ -195,6 +202,34 @@ estimate the population from it. Sampling reads records as a stream.
 
 Details: [Tabalyst Sample](https://docs.tabalyst.com/sample/).
 
+## Tabalyst Dedup
+
+Find the rows that occur more than once in a CSV file, without modifying it:
+
+```console
+tabalyst dedup customers.csv
+tabalyst dedup "*.csv" -d deduped
+```
+
+Two rows are the same only when every cell is identical (case, spaces and
+leading zeros count). Dedup writes, beside the source or in `-d`,
+`customers.unique.csv` (every distinct row once, at its first place),
+`customers.duplicates.csv` (each repeated row once) and, when a row occurs three
+times or more, `customers.duplicates_all.csv` (every occurrence of the repeated
+rows), plus `customers.dedup.json`, a report whose counts always add up. A file
+without duplicate gets its report only. A batch also writes
+`dedup.summary.json`; a file that is already processed and unchanged is skipped.
+
+A result is exact or the file fails with a reason: the rows are kept in memory up
+to a budget (`--memory-budget-mb`, 512 by default) and then on disk, a file may
+hold 10,000,000 records by default (`--max-rows`), and nothing is ever
+approximated. Dedup reads CSV files only and asks no question.
+
+Details: [Tabalyst Dedup](https://docs.tabalyst.com/dedup/),
+[batches and already processed files](https://docs.tabalyst.com/dedup/batch/),
+[exactness, memory and limits](https://docs.tabalyst.com/dedup/limits/) and the
+[`dedup` command reference](https://docs.tabalyst.com/reference/cli/dedup/).
+
 ## Tabalyst Generate
 
 Generate synthetic CSV data from a packaged example (`crm`, `insurance` or
@@ -224,6 +259,7 @@ import tabalyst
 result = tabalyst.analyze("customers.csv", "customers.html")
 batch = tabalyst.generate_reports(["*.csv"], output_dir="reports")
 sample = tabalyst.sample_csv("customers.csv", method="random", rows=1000, seed=42)
+dedup = tabalyst.dedup_csv("customers.csv")
 scan = tabalyst.scan("orders.json")
 inspection = tabalyst.inspect("orders.json")
 ```
@@ -257,7 +293,8 @@ Generated JSON and HTML may contain source values and should be shared
 accordingly.
 
 Reports, samples, and scans read CSV, JSON and JSONL files as a stream, with
-memory bounded by configurable limits. An Excel workbook is read by sheet into
+memory bounded by configurable limits; Dedup keeps the rows it has seen in memory
+up to a budget, then on disk. An Excel workbook is read by sheet into
 memory, and a sheet above 1 GiB of XML is refused. See
 [limitations](https://docs.tabalyst.com/about/limitations/).
 

{tabalyst-0.7.0 → tabalyst-0.7.2}/examples/README.md

@@ -16,7 +16,7 @@ examples/
 |-- showcase/ (what the website and the documentation publish, nothing else)
 |   |-- basic/
 |   |-- insurance-customers/ (report.html, report.json, scan.json; 100,000 rows)
-|   |-- insurance-customers-scan/ (scan, second pass, smart plan and sample; 100,000 rows)
+|   |-- insurance-customers.scan/ (scan, second pass, smart plan and sample; 100,000 rows)
 |   |-- orders/
 |   |-- sales/
 |   `-- web-events/
@@ -67,7 +67,7 @@ fails when it is stale; `regenerate.py` runs it), so it never drifts.
 | --- | --- | --- | --- |
 | `showcase/basic/` | `report` | no | `get-started/examples.md`, `reference/history.md` |
 | `showcase/insurance-customers/` | `report` | yes | `get-started/examples.md` |
-| `showcase/insurance-customers-scan/` | `scan` | no | `get-started/simple-clear-smart.md` |
+| `showcase/insurance-customers.scan/` | `scan` | no | `get-started/simple-clear-smart.md` |
 | `showcase/orders/` | `report` | yes | `get-started/examples.md` |
 | `showcase/sales/` | `report` | yes | `get-started/examples.md` |
 | `showcase/web-events/` | `report` | yes | `get-started/examples.md` |
@@ -96,17 +96,17 @@ tabalyst report examples/datasets/web-events.jsonl -o examples/showcase/web-even
 tabalyst report examples/datasets/sales.xlsx -o examples/showcase/sales/report.html
 ```
 
-The folder `insurance-customers-scan/` shows the three steps of Tabalyst Scan on
+The folder `insurance-customers.scan/` shows the three steps of Tabalyst Scan on
 `insurance-customers-100k.csv`, the generated base of 100,000 rows (see "Simple, Clear and Smart" in the documentation):
 the scan (Simple, with its web page, Clear), the second pass and the plan of a
 smart sample (Smart), and the smart sample with its report and page. Its four steps
 are in its `demo.json`:
 
 ```console
-tabalyst scan {datasets}/insurance-customers-100k.csv -o examples/showcase/insurance-customers-scan/insurance-customers.scan.json --config examples/config.json --deep --yes --force
+tabalyst scan {datasets}/insurance-customers-100k.csv -o examples/showcase/insurance-customers.scan/insurance-customers.scan.json --config examples/config.json --deep --yes --force
 tabalyst scan {datasets}/insurance-customers-100k.csv --config examples/config.json --deep --yes --force --no-html
 tabalyst scan {datasets}/insurance-customers-100k.csv --plan --smart --sample-size 3000 --config examples/config.json
-tabalyst sample {datasets}/insurance-customers-100k.csv --smart --rows 3000 --seed 42 -o examples/showcase/insurance-customers-scan/insurance-customers.sample.csv --config examples/config.json --force
+tabalyst sample {datasets}/insurance-customers-100k.csv --smart --rows 3000 --seed 42 -o examples/showcase/insurance-customers.scan/insurance-customers.sample.csv --config examples/config.json --force
 ```
 
 `make_datasets.py` generates the base (45 seconds, 27 MB, SHA-256 checked) and leaves it outside the

{tabalyst-0.7.0 → tabalyst-0.7.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "tabalyst"
-version = "0.7.0"
+version = "0.7.2"
 description = "Open-source toolkit for understanding and working with unfamiliar data."
 readme = "README.md"
 requires-python = ">=3.11"
@@ -77,7 +77,11 @@ markers = [
     "scan_json_lot(name): Scan and JSON work lot (plan 2026-10-06) that provides the tested behavior; lot 0 tests describe current behavior",
     "report_lot(name): Report overview lot (plan 2026-10-16) that provides the tested behavior; skipped until the lot is enabled in tests/scan/report_gate.py",
     "inspect_lot(name): Tabalyst Inspect implementation lot that provides the tested behavior",
-    "shipped_sections: the test reads the sections the shipped report writes (some are hidden for now), not every section the suite writes",
+    "dedup_lot(name): Tabalyst Dedup implementation lot (plan 2026-10-22) that provides the tested behavior; skipped until the lot is enabled in tests/dedup/conftest.py",
+    "dedup_t(*ids): requirement tests (T01 to T30 of the Dedup specification) that the test covers",
+    "dedup_heavy: Dedup large-scale test, never run by default; select it with -m dedup_heavy",
+    "uid_lot(name): Unique ID plan (2026-10-23) lot that provides the tested behavior; skipped until the lot is enabled in tests/report/conftest.py",
+    "uid_t(*ids): requirement tests (T01 to T34 of the Unique ID specification) that the test covers",
     "shipped_exposure_default: the test reads the shipped default of scan.exposure.sensitive_values (show), not the masking default the suite runs with",
 ]
 

{tabalyst-0.7.0 → tabalyst-0.7.2}/src/tabalyst/__init__.py

@@ -16,6 +16,10 @@ from tabalyst.errors import ConfigurationError, InputError, ReportError, Tabalys
 if TYPE_CHECKING:
     from tabalyst.config import DeepSettings as DeepSettings
     from tabalyst.config import PearsonSettings as PearsonSettings
+    from tabalyst.dedup_service import BatchDedupResult as BatchDedupResult
+    from tabalyst.dedup_service import DedupResult as DedupResult
+    from tabalyst.dedup_service import dedup_csv as dedup_csv
+    from tabalyst.dedup_service import generate_dedups as generate_dedups
     from tabalyst.deep_service import execute_deep as execute_deep
     from tabalyst.deep_service import generate_deep as generate_deep
     from tabalyst.deep_service import prepare_deep as prepare_deep
@@ -76,6 +80,10 @@ _LAZY: dict[str, str] = {
     "generate_deep": "tabalyst.deep_service",
     "prepare_deep": "tabalyst.deep_service",
     "ConfirmationRequired": "tabalyst.deep_store",
+    "BatchDedupResult": "tabalyst.dedup_service",
+    "DedupResult": "tabalyst.dedup_service",
+    "dedup_csv": "tabalyst.dedup_service",
+    "generate_dedups": "tabalyst.dedup_service",
     "ComputerPlan": "tabalyst.generate_service",
     "GeneratePlan": "tabalyst.generate_service",
     "GenerateResult": "tabalyst.generate_service",
@@ -140,9 +148,11 @@ def __dir__() -> list[str]:
 __all__ = [
     "AnalysisPlan",
     "AnalysisRequest",
+    "BatchDedupResult",
     "ComputerPlan",
     "ConfigurationError",
     "ConfirmationRequired",
+    "DedupResult",
     "DeepResult",
     "DeepSettings",
     "GeneratePlan",
@@ -166,9 +176,11 @@ __all__ = [
     "__version__",
     "analyze",
     "build_generate_plan",
+    "dedup_csv",
     "execute_deep",
     "execute_smart_samples",
     "generate_datasets",
+    "generate_dedups",
     "generate_deep",
     "generate_inspections",
     "generate_plans",

{tabalyst-0.7.0 → tabalyst-0.7.2}/src/tabalyst/cli/app.py

@@ -10,6 +10,7 @@ import typer
 
 from tabalyst import __version__
 from tabalyst.cli.cache import cache_app
+from tabalyst.cli.dedup import dedup_command
 from tabalyst.cli.generate import generate_command
 from tabalyst.cli.inspect import inspect_command
 from tabalyst.cli.report import report_command
@@ -45,6 +46,7 @@ def root(
     """Understand, explore, clean, and validate structured data."""
 
 
+app.command("dedup")(dedup_command)
 app.command("inspect")(inspect_command)
 app.command("generate")(generate_command)
 app.command("report")(report_command)