tabalyst 0.7.1__tar.gz → 0.7.3__tar.gz

{tabalyst-0.7.1/src/tabalyst.egg-info → tabalyst-0.7.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tabalyst
-Version: 0.7.1
+Version: 0.7.3
 Summary: Open-source toolkit for understanding and working with unfamiliar data.
 Author: Gregory Borelli
 License-Expression: MPL-2.0
@@ -146,6 +146,12 @@ Details: [Tabalyst Report](https://docs.tabalyst.com/report/),
 - Dataset dimensions, missing cells, duplicates, and quality observations.
 - A family and a type for each column (identifier, text, date, boolean...),
   with the evidence behind them, and the technical type with its error rates.
+- For an identifier column, whether the ID is unique across all rows and after
+  row deduplication: the repetitions that exact copies of whole rows explain
+  are told apart from those between distinct records, with the evidence (the
+  Scan, the second pass or a [Dedup](https://docs.tabalyst.com/dedup/) report),
+  the case of missing IDs, and a non-binding near-unique reading of the few
+  repetitions that remain.
 - A summary table with one row per column: its type, a sentence about its
   values, the missing share, its Errors and Issues with a preview of each, and
   its details; above it, the Column quality and Signals cards.
@@ -274,10 +280,15 @@ tabalyst generate --example crm -o crm-contacts.csv
 tabalyst generate --example insurance --rows 3000 -d insurance-output/
 tabalyst generate --example computer --rows 10000 -d computer-output/
 tabalyst generate --example insurance --rows 3000 --anomaly-profile dirty_realistic -d insurance-dirty/
+tabalyst generate --example computer --rows 30000 -o pcs.csv --anomaly-profile dirty_realistic
 ```
 
 An anomaly profile writes an altered CSV, a cell change log and a summary
-beside the untouched clean CSV.
+beside the untouched clean CSV. For Computer, `-o` writes one CSV with the
+errors and its `.changes.csv` log, and `--anomaly-profile` accepts several
+profiles with `-d` to get one variant of each. Computer rows are generated by
+several processes from 5,000 rows (`--workers` sets the number), and the command
+shows its progress and ends with a timing report.
 
 Details: [Tabalyst Generate](https://docs.tabalyst.com/generate/),
 [`examples/generate/README.md`](examples/generate/README.md) and the

{tabalyst-0.7.1 → tabalyst-0.7.3}/README.md

@@ -106,6 +106,12 @@ Details: [Tabalyst Report](https://docs.tabalyst.com/report/),
 - Dataset dimensions, missing cells, duplicates, and quality observations.
 - A family and a type for each column (identifier, text, date, boolean...),
   with the evidence behind them, and the technical type with its error rates.
+- For an identifier column, whether the ID is unique across all rows and after
+  row deduplication: the repetitions that exact copies of whole rows explain
+  are told apart from those between distinct records, with the evidence (the
+  Scan, the second pass or a [Dedup](https://docs.tabalyst.com/dedup/) report),
+  the case of missing IDs, and a non-binding near-unique reading of the few
+  repetitions that remain.
 - A summary table with one row per column: its type, a sentence about its
   values, the missing share, its Errors and Issues with a preview of each, and
   its details; above it, the Column quality and Signals cards.
@@ -234,10 +240,15 @@ tabalyst generate --example crm -o crm-contacts.csv
 tabalyst generate --example insurance --rows 3000 -d insurance-output/
 tabalyst generate --example computer --rows 10000 -d computer-output/
 tabalyst generate --example insurance --rows 3000 --anomaly-profile dirty_realistic -d insurance-dirty/
+tabalyst generate --example computer --rows 30000 -o pcs.csv --anomaly-profile dirty_realistic
 ```
 
 An anomaly profile writes an altered CSV, a cell change log and a summary
-beside the untouched clean CSV.
+beside the untouched clean CSV. For Computer, `-o` writes one CSV with the
+errors and its `.changes.csv` log, and `--anomaly-profile` accepts several
+profiles with `-d` to get one variant of each. Computer rows are generated by
+several processes from 5,000 rows (`--workers` sets the number), and the command
+shows its progress and ends with a timing report.
 
 Details: [Tabalyst Generate](https://docs.tabalyst.com/generate/),
 [`examples/generate/README.md`](examples/generate/README.md) and the

{tabalyst-0.7.1 → tabalyst-0.7.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "tabalyst"
-version = "0.7.1"
+version = "0.7.3"
 description = "Open-source toolkit for understanding and working with unfamiliar data."
 readme = "README.md"
 requires-python = ">=3.11"
@@ -80,6 +80,8 @@ markers = [
     "dedup_lot(name): Tabalyst Dedup implementation lot (plan 2026-10-22) that provides the tested behavior; skipped until the lot is enabled in tests/dedup/conftest.py",
     "dedup_t(*ids): requirement tests (T01 to T30 of the Dedup specification) that the test covers",
     "dedup_heavy: Dedup large-scale test, never run by default; select it with -m dedup_heavy",
+    "uid_lot(name): Unique ID plan (2026-10-23) lot that provides the tested behavior; skipped until the lot is enabled in tests/report/conftest.py",
+    "uid_t(*ids): requirement tests (T01 to T34 of the Unique ID specification) that the test covers",
     "shipped_exposure_default: the test reads the shipped default of scan.exposure.sensitive_values (show), not the masking default the suite runs with",
 ]
 

tabalyst-0.7.3/src/tabalyst/cli/generate.py

@@ -0,0 +1,190 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+"""CLI adapter for Tabalyst Generate."""
+
+import sys
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from tabalyst.cli.terminal import error_exit_code
+from tabalyst.errors import TabalystError
+from tabalyst.generate_service import (
+    GenerateProgress,
+    GenerateResult,
+    build_generate_plan,
+    generate_datasets,
+)
+
+PHASE_NAMES = {"rows": "generating rows", "errors": "injecting errors", "publish": "writing files"}
+# Below this, a phase is noise in the report.
+MIN_REPORTED_SECONDS = 0.05
+
+
+def format_duration(seconds: float) -> str:
+    """A duration for people: tenths of a second under a minute, then minutes and seconds."""
+    if seconds < 60:
+        return f"{seconds:.1f}s"
+    minutes, rest = divmod(round(seconds), 60)
+    if minutes < 60:
+        return f"{minutes}m {rest:02d}s"
+    hours, minutes = divmod(minutes, 60)
+    return f"{hours}h {minutes:02d}m"
+
+
+class GenerateProgressPrinter:
+    """One rewritten standard error line per stage, with the time left once it is known."""
+
+    def __init__(self, enabled: bool) -> None:
+        self.enabled = enabled
+        self._stage: str | None = None
+        self._stage_start = 0.0
+        self._width = 0
+
+    def __call__(self, event: GenerateProgress) -> None:
+        if not self.enabled or not event.total:
+            return
+        if event.stage != self._stage:
+            self._stage = event.stage
+            self._stage_start = event.elapsed_seconds
+        spent = event.elapsed_seconds - self._stage_start
+        message = f"{event.stage} {event.done:,}/{event.total:,} ({100 * event.done // event.total}%)"
+        if event.done >= event.total:
+            message += f" - {format_duration(spent)}"
+        elif event.done and spent >= 1:
+            message += (f" - {format_duration(spent)}, "
+                        f"about {format_duration(spent * (event.total - event.done) / event.done)} left")
+        finished = event.done >= event.total
+        typer.echo(f"\r{message.ljust(self._width)}", nl=finished, err=True)
+        self._width = 0 if finished else max(self._width, len(message))
+
+
+def timing_report(result: GenerateResult) -> str:
+    """The closing line: rows, total time, rate and where the time went."""
+    rows = sum(result.row_counts.values()) if len(result.row_counts) == 1 else None
+    text = f"Done in {format_duration(result.seconds)}"
+    if rows is not None:
+        text = f"{rows:,} rows in {format_duration(result.seconds)}"
+        if result.seconds > 0:
+            processes = f", {result.workers} processes" if result.workers > 1 else ""
+            text += f" ({rows / result.seconds:,.0f} rows/s{processes})"
+    changed = sum(result.anomaly_counts.values())
+    if changed:
+        profiles = {key.split("/", 1)[1] for key in result.anomaly_counts if "/" in key}
+        text += f", {changed:,} cells changed"
+        if len(profiles) > 1:
+            text += f" in {len(profiles)} profiles"
+    parts = [f"{PHASE_NAMES[name]} {format_duration(value)}"
+             for name, value in result.timings.items()
+             if name in PHASE_NAMES and value >= MIN_REPORTED_SECONDS]
+    return f"{text}: {', '.join(parts)}" if len(parts) > 1 else text
+
+
+def generate_command(
+    definition: Annotated[
+        Path | None,
+        typer.Argument(help="Generate definition or package directory."),
+    ] = None,
+    example: Annotated[
+        str | None,
+        typer.Option("--example", help="Packaged example: crm, insurance, or computer."),
+    ] = None,
+    output: Annotated[
+        Path | None,
+        typer.Option("--output", "-o", help="One CSV file (with --anomaly-profile: the CSV with errors, plus FILE.changes.csv)."),
+    ] = None,
+    output_dir: Annotated[
+        Path | None,
+        typer.Option("--output-dir", "-d", help="Directory for all artifacts."),
+    ] = None,
+    rows: Annotated[
+        int | None,
+        typer.Option("--rows", help="Number of generated rows."),
+    ] = None,
+    seed: Annotated[
+        int | None,
+        typer.Option("--seed", help="Root random seed."),
+    ] = None,
+    as_of_date: Annotated[
+        str | None,
+        typer.Option("--as-of-date", help="Reference date (YYYY-MM-DD)."),
+    ] = None,
+    anomaly_profile: Annotated[
+        str | None,
+        typer.Option("--anomaly-profile", help="Inject errors. Computer: dirty_realistic, dirty_stress, unit_variants, unit_variants_extended; several, comma separated, with -d."),
+    ] = None,
+    request: Annotated[
+        Path | None,
+        typer.Option("--request", help="JSON generation request."),
+    ] = None,
+    generation_profile: Annotated[
+        str | None,
+        typer.Option("--generation-profile", help="Catalog selection profile."),
+    ] = None,
+    output_profile: Annotated[
+        str | None,
+        typer.Option("--output-profile", help="Output column profile."),
+    ] = None,
+    market: Annotated[
+        str | None,
+        typer.Option("--market", help="Offer market: ca or us."),
+    ] = None,
+    locales: Annotated[
+        str | None,
+        typer.Option("--locales", help="Computer locales: en, fr or en,fr (default en with -o, en,fr with -d)."),
+    ] = None,
+    unit_style: Annotated[
+        str | None,
+        typer.Option("--unit-style", help="Computer values: numeric or localized units."),
+    ] = None,
+    unit_columns: Annotated[
+        str | None,
+        typer.Option("--unit-columns", help="Comma-separated Computer measurement columns to format with units."),
+    ] = None,
+    validate_only: Annotated[
+        bool,
+        typer.Option("--validate-only", help="Validate a package without writing."),
+    ] = False,
+    force: Annotated[
+        bool,
+        typer.Option("--force", "-f", help="Replace existing artifacts."),
+    ] = False,
+    no_progress: Annotated[
+        bool,
+        typer.Option("--no-progress", help="Hide the progress line. It is also hidden outside a terminal."),
+    ] = False,
+    workers: Annotated[
+        int | None,
+        typer.Option(
+            "--workers", min=1,
+            help=("Computer: processes that generate the rows, 1 for one process. By default, "
+                  "5,000 rows or more use one per spare processor, at most 6."),
+        ),
+    ] = None,
+) -> None:
+    """Validate and generate datasets from a definition or package."""
+    try:
+        plan = build_generate_plan(
+            definition, example=example, output=output, output_dir=output_dir,
+            rows=rows, seed=seed, as_of_date=as_of_date,
+            anomaly_profile=anomaly_profile, force=force, request=request,
+            generation_profile=generation_profile, output_profile=output_profile,
+            market=market, locales=locales, unit_style=unit_style,
+            unit_columns=unit_columns,
+            validate_only=validate_only, workers=workers,
+        )
+        if validate_only:
+            typer.echo("Valid Generate package.", err=True)
+            return
+        result = generate_datasets(
+            plan, on_progress=GenerateProgressPrinter(not no_progress and sys.stderr.isatty()),
+        )
+    except TabalystError as exc:
+        typer.echo(f"Error: {exc}", err=True)
+        raise typer.Exit(error_exit_code(exc)) from exc
+    for path in result.paths:
+        typer.echo(f"Created: {path}", err=True)
+    typer.echo(timing_report(result), err=True)

{tabalyst-0.7.1 → tabalyst-0.7.3}/src/tabalyst/cli/report.py

@@ -250,6 +250,18 @@ def report_command(
             ),
         ),
     ] = False,
+    dedup_report: Annotated[
+        Path | None,
+        typer.Option(
+            "--dedup-report",
+            help=(
+                "A Dedup report (.dedup.json), or the folder holding the one of each "
+                "source, that proves the copies of rows behind the uniqueness of "
+                "identifier columns. CSV sources only. Without it, the report beside "
+                "each source is tried."
+            ),
+        ),
+    ] = None,
     yes: Annotated[
         bool,
         typer.Option(
@@ -292,6 +304,7 @@ def report_command(
             all_collections=all_collections,
             deep=deep,
             confirm=confirm,
+            dedup_report=dedup_report,
         )
     except ConfirmationRequired as exc:
         if decisions and decisions[-1] is Decision.DECLINED:

{tabalyst-0.7.1 → tabalyst-0.7.3}/src/tabalyst/config.py

@@ -118,6 +118,10 @@ class PresentationSettings(BaseModel):
         default_factory=ValueExamplesSettings
     )
     row_sample: RowSampleSettings = Field(default_factory=RowSampleSettings)
+    # A Dedup report (file) or the folder holding ``<source stem>.dedup.json``: the proof of
+    # the copies of rows behind ``identifier_uniqueness``. A CSV source only; the path is not
+    # stored in the Report, which names the file only.
+    dedup_report: str | None = None
 
 
 class OutputSettings(BaseModel):
@@ -349,6 +353,23 @@ class DuplicateSettings(BaseModel):
     records: bool = False
 
 
+class CopySettings(BaseModel):
+    """The ceilings of the count of the extra copies of rows (design 20.40).
+
+    ``memory_mb`` is the accounted size of the table of distinct rows, ``disk_mb``
+    the estimated size of the temporary database that takes over when the table
+    passes it (``0`` forbids the disk), ``max_rows`` the records read. A count that
+    meets a ceiling stops and is ``limited``: its copies are a lower bound. All are
+    part of the key of a result, and none depends on a clock or a machine.
+    """
+
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    memory_mb: int = Field(default=256, ge=1, le=1_048_576)
+    disk_mb: int = Field(default=2_048, ge=0, le=16_777_216)
+    max_rows: int = Field(default=10_000_000, ge=1)
+
+
 class PatternCountSettings(BaseModel):
     """How the validation of the patterns is bounded in memory (design 20.9).
 
@@ -520,6 +541,7 @@ class DeepSettings(BaseModel):
     patterns: PatternCountSettings = Field(default_factory=PatternCountSettings)
     residuals: ResidualSettings = Field(default_factory=ResidualSettings)
     duplicates: DuplicateSettings = Field(default_factory=DuplicateSettings)
+    copies: CopySettings = Field(default_factory=CopySettings)
     drift: DriftSettings = Field(default_factory=DriftSettings)
     date_order: DateOrderSettings = Field(default_factory=DateOrderSettings)
     missing: MissingSettings = Field(default_factory=MissingSettings)

{tabalyst-0.7.1 → tabalyst-0.7.3}/src/tabalyst/dedup/analysis.py

@@ -123,7 +123,7 @@ def analyze_dedup(
     path = Path(source)
     started = time.perf_counter()
     before = stat_source(path)
-    forward = _staged(on_progress)
+    forward = staged(on_progress)
     spilled_at: int | None = None
 
     def spilled(event: SpillEvent) -> None:
@@ -204,7 +204,7 @@ def source_changed_error(records_read: int) -> DedupInputError:
     )
 
 
-def _staged(callback: ProgressCallback | None) -> ProgressCallback | None:
+def staged(callback: ProgressCallback | None) -> ProgressCallback | None:
     """Name the stage on every event that the pass emits."""
     if callback is None:
         return None

tabalyst-0.7.3/src/tabalyst/dedup/calculation.py

@@ -0,0 +1,14 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+"""The versions of the counts of the copies of rows, apart from the engine so the planner can read them."""
+
+# Changes with any change of what the count means (identity, ceilings, order of the checks). The
+# analyzer ``duplicate_rows_count`` takes it as its version, so stored results of an older count
+# are stale (Unique ID lot 8, cahier 12.5).
+CALCULATION_VERSION = 1
+
+# The same for the light count of the copies of the rows that have no ID (``missing_id_copies``,
+# Unique ID lot 10): what it counts, the identity of the rows, the order of the checks.
+MISSING_CALCULATION_VERSION = 1