tabalyst 0.7.2__tar.gz → 0.7.3__tar.gz

{tabalyst-0.7.2/src/tabalyst.egg-info → tabalyst-0.7.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tabalyst
-Version: 0.7.2
+Version: 0.7.3
 Summary: Open-source toolkit for understanding and working with unfamiliar data.
 Author: Gregory Borelli
 License-Expression: MPL-2.0
@@ -280,10 +280,15 @@ tabalyst generate --example crm -o crm-contacts.csv
 tabalyst generate --example insurance --rows 3000 -d insurance-output/
 tabalyst generate --example computer --rows 10000 -d computer-output/
 tabalyst generate --example insurance --rows 3000 --anomaly-profile dirty_realistic -d insurance-dirty/
+tabalyst generate --example computer --rows 30000 -o pcs.csv --anomaly-profile dirty_realistic
 ```
 
 An anomaly profile writes an altered CSV, a cell change log and a summary
-beside the untouched clean CSV.
+beside the untouched clean CSV. For Computer, `-o` writes one CSV with the
+errors and its `.changes.csv` log, and `--anomaly-profile` accepts several
+profiles with `-d` to get one variant of each. Computer rows are generated by
+several processes from 5,000 rows (`--workers` sets the number), and the command
+shows its progress and ends with a timing report.
 
 Details: [Tabalyst Generate](https://docs.tabalyst.com/generate/),
 [`examples/generate/README.md`](examples/generate/README.md) and the

{tabalyst-0.7.2 → tabalyst-0.7.3}/README.md

@@ -240,10 +240,15 @@ tabalyst generate --example crm -o crm-contacts.csv
 tabalyst generate --example insurance --rows 3000 -d insurance-output/
 tabalyst generate --example computer --rows 10000 -d computer-output/
 tabalyst generate --example insurance --rows 3000 --anomaly-profile dirty_realistic -d insurance-dirty/
+tabalyst generate --example computer --rows 30000 -o pcs.csv --anomaly-profile dirty_realistic
 ```
 
 An anomaly profile writes an altered CSV, a cell change log and a summary
-beside the untouched clean CSV.
+beside the untouched clean CSV. For Computer, `-o` writes one CSV with the
+errors and its `.changes.csv` log, and `--anomaly-profile` accepts several
+profiles with `-d` to get one variant of each. Computer rows are generated by
+several processes from 5,000 rows (`--workers` sets the number), and the command
+shows its progress and ends with a timing report.
 
 Details: [Tabalyst Generate](https://docs.tabalyst.com/generate/),
 [`examples/generate/README.md`](examples/generate/README.md) and the

{tabalyst-0.7.2 → tabalyst-0.7.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "tabalyst"
-version = "0.7.2"
+version = "0.7.3"
 description = "Open-source toolkit for understanding and working with unfamiliar data."
 readme = "README.md"
 requires-python = ">=3.11"

tabalyst-0.7.3/src/tabalyst/cli/generate.py

@@ -0,0 +1,190 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+"""CLI adapter for Tabalyst Generate."""
+
+import sys
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from tabalyst.cli.terminal import error_exit_code
+from tabalyst.errors import TabalystError
+from tabalyst.generate_service import (
+    GenerateProgress,
+    GenerateResult,
+    build_generate_plan,
+    generate_datasets,
+)
+
+PHASE_NAMES = {"rows": "generating rows", "errors": "injecting errors", "publish": "writing files"}
+# Below this, a phase is noise in the report.
+MIN_REPORTED_SECONDS = 0.05
+
+
+def format_duration(seconds: float) -> str:
+    """A duration for people: tenths of a second under a minute, then minutes and seconds."""
+    if seconds < 60:
+        return f"{seconds:.1f}s"
+    minutes, rest = divmod(round(seconds), 60)
+    if minutes < 60:
+        return f"{minutes}m {rest:02d}s"
+    hours, minutes = divmod(minutes, 60)
+    return f"{hours}h {minutes:02d}m"
+
+
+class GenerateProgressPrinter:
+    """One rewritten standard error line per stage, with the time left once it is known."""
+
+    def __init__(self, enabled: bool) -> None:
+        self.enabled = enabled
+        self._stage: str | None = None
+        self._stage_start = 0.0
+        self._width = 0
+
+    def __call__(self, event: GenerateProgress) -> None:
+        if not self.enabled or not event.total:
+            return
+        if event.stage != self._stage:
+            self._stage = event.stage
+            self._stage_start = event.elapsed_seconds
+        spent = event.elapsed_seconds - self._stage_start
+        message = f"{event.stage} {event.done:,}/{event.total:,} ({100 * event.done // event.total}%)"
+        if event.done >= event.total:
+            message += f" - {format_duration(spent)}"
+        elif event.done and spent >= 1:
+            message += (f" - {format_duration(spent)}, "
+                        f"about {format_duration(spent * (event.total - event.done) / event.done)} left")
+        finished = event.done >= event.total
+        typer.echo(f"\r{message.ljust(self._width)}", nl=finished, err=True)
+        self._width = 0 if finished else max(self._width, len(message))
+
+
+def timing_report(result: GenerateResult) -> str:
+    """The closing line: rows, total time, rate and where the time went."""
+    rows = sum(result.row_counts.values()) if len(result.row_counts) == 1 else None
+    text = f"Done in {format_duration(result.seconds)}"
+    if rows is not None:
+        text = f"{rows:,} rows in {format_duration(result.seconds)}"
+        if result.seconds > 0:
+            processes = f", {result.workers} processes" if result.workers > 1 else ""
+            text += f" ({rows / result.seconds:,.0f} rows/s{processes})"
+    changed = sum(result.anomaly_counts.values())
+    if changed:
+        profiles = {key.split("/", 1)[1] for key in result.anomaly_counts if "/" in key}
+        text += f", {changed:,} cells changed"
+        if len(profiles) > 1:
+            text += f" in {len(profiles)} profiles"
+    parts = [f"{PHASE_NAMES[name]} {format_duration(value)}"
+             for name, value in result.timings.items()
+             if name in PHASE_NAMES and value >= MIN_REPORTED_SECONDS]
+    return f"{text}: {', '.join(parts)}" if len(parts) > 1 else text
+
+
+def generate_command(
+    definition: Annotated[
+        Path | None,
+        typer.Argument(help="Generate definition or package directory."),
+    ] = None,
+    example: Annotated[
+        str | None,
+        typer.Option("--example", help="Packaged example: crm, insurance, or computer."),
+    ] = None,
+    output: Annotated[
+        Path | None,
+        typer.Option("--output", "-o", help="One CSV file (with --anomaly-profile: the CSV with errors, plus FILE.changes.csv)."),
+    ] = None,
+    output_dir: Annotated[
+        Path | None,
+        typer.Option("--output-dir", "-d", help="Directory for all artifacts."),
+    ] = None,
+    rows: Annotated[
+        int | None,
+        typer.Option("--rows", help="Number of generated rows."),
+    ] = None,
+    seed: Annotated[
+        int | None,
+        typer.Option("--seed", help="Root random seed."),
+    ] = None,
+    as_of_date: Annotated[
+        str | None,
+        typer.Option("--as-of-date", help="Reference date (YYYY-MM-DD)."),
+    ] = None,
+    anomaly_profile: Annotated[
+        str | None,
+        typer.Option("--anomaly-profile", help="Inject errors. Computer: dirty_realistic, dirty_stress, unit_variants, unit_variants_extended; several, comma separated, with -d."),
+    ] = None,
+    request: Annotated[
+        Path | None,
+        typer.Option("--request", help="JSON generation request."),
+    ] = None,
+    generation_profile: Annotated[
+        str | None,
+        typer.Option("--generation-profile", help="Catalog selection profile."),
+    ] = None,
+    output_profile: Annotated[
+        str | None,
+        typer.Option("--output-profile", help="Output column profile."),
+    ] = None,
+    market: Annotated[
+        str | None,
+        typer.Option("--market", help="Offer market: ca or us."),
+    ] = None,
+    locales: Annotated[
+        str | None,
+        typer.Option("--locales", help="Computer locales: en, fr or en,fr (default en with -o, en,fr with -d)."),
+    ] = None,
+    unit_style: Annotated[
+        str | None,
+        typer.Option("--unit-style", help="Computer values: numeric or localized units."),
+    ] = None,
+    unit_columns: Annotated[
+        str | None,
+        typer.Option("--unit-columns", help="Comma-separated Computer measurement columns to format with units."),
+    ] = None,
+    validate_only: Annotated[
+        bool,
+        typer.Option("--validate-only", help="Validate a package without writing."),
+    ] = False,
+    force: Annotated[
+        bool,
+        typer.Option("--force", "-f", help="Replace existing artifacts."),
+    ] = False,
+    no_progress: Annotated[
+        bool,
+        typer.Option("--no-progress", help="Hide the progress line. It is also hidden outside a terminal."),
+    ] = False,
+    workers: Annotated[
+        int | None,
+        typer.Option(
+            "--workers", min=1,
+            help=("Computer: processes that generate the rows, 1 for one process. By default, "
+                  "5,000 rows or more use one per spare processor, at most 6."),
+        ),
+    ] = None,
+) -> None:
+    """Validate and generate datasets from a definition or package."""
+    try:
+        plan = build_generate_plan(
+            definition, example=example, output=output, output_dir=output_dir,
+            rows=rows, seed=seed, as_of_date=as_of_date,
+            anomaly_profile=anomaly_profile, force=force, request=request,
+            generation_profile=generation_profile, output_profile=output_profile,
+            market=market, locales=locales, unit_style=unit_style,
+            unit_columns=unit_columns,
+            validate_only=validate_only, workers=workers,
+        )
+        if validate_only:
+            typer.echo("Valid Generate package.", err=True)
+            return
+        result = generate_datasets(
+            plan, on_progress=GenerateProgressPrinter(not no_progress and sys.stderr.isatty()),
+        )
+    except TabalystError as exc:
+        typer.echo(f"Error: {exc}", err=True)
+        raise typer.Exit(error_exit_code(exc)) from exc
+    for path in result.paths:
+        typer.echo(f"Created: {path}", err=True)
+    typer.echo(timing_report(result), err=True)

{tabalyst-0.7.2 → tabalyst-0.7.3}/src/tabalyst/generate_anomalies.py

@@ -354,7 +354,8 @@ def _choose_rows(clean: Path, delimiter: str, operations: Sequence[Mapping[str,
 def inject_anomalies(clean: Path, altered: Path, changes: Path, *,
                      dataset: str, profile: Mapping[str, object], seed: int, count: int,
                      delimiter: str, locale: str = "en",
-                     unit_suffixes: Mapping[str, tuple[str, str]] | None = None) -> dict[str, object]:
+                     unit_suffixes: Mapping[str, tuple[str, str]] | None = None,
+                     on_row: Callable[[int], None] | None = None) -> dict[str, object]:
     """Write altered CSV and cell log; return requested and actual counts.
 
     Computer unit suffixes are supplied by its output register. An operation
@@ -369,6 +370,7 @@ def inject_anomalies(clean: Path, altered: Path, changes: Path, *,
     row had already received are then gone from the file, so their journal
     entries are dropped and their operations count the row as skipped: the
     journal always describes the file.
+    ``on_row`` receives the number of rows written, about 200 times per file and at the end.
     """
     # Imported here because generate_values reaches generate_definition, which
     # imports this module for the operation rules.
@@ -416,8 +418,11 @@ def inject_anomalies(clean: Path, altered: Path, changes: Path, *,
         writer.writeheader()
         log.writeheader()
         observed = 0
+        report_every = max(1, count // 200)
         for row_index, clean_row in enumerate(reader):
             observed += 1
+            if on_row is not None and row_index % report_every == 0:
+                on_row(row_index)
             row = clean_row.copy()
             # (operation index, cell updates) of this row, written once it is final.
             pending: list[tuple[int, list[tuple[str, str, str, str]]]] = []
@@ -488,6 +493,8 @@ def inject_anomalies(clean: Path, altered: Path, changes: Path, *,
                     history[field].append(value)
         if observed != count:
             raise ValueError(f"Clean row count changed for {dataset}: {observed} != {count}")
+        if on_row is not None:
+            on_row(observed)
     return {
         "dataset": dataset, "profile": profile_id, "seed": seed,
         "total_rows": count, "affected_rows": affected_rows,

{tabalyst-0.7.2 → tabalyst-0.7.3}/src/tabalyst/generate_computer.py

@@ -26,12 +26,11 @@ MARKET_CURRENCY = {"ca": "CAD", "us": "USD"}
 
 
 def _digest(parts: tuple[str, ...]) -> str:
-    digest = hashlib.sha256()
-    for part in parts:
-        encoded = part.encode("utf-8")
-        digest.update(len(encoded).to_bytes(8, "big"))
-        digest.update(encoded)
-    return digest.hexdigest()[:40]
+    # One hash call over the same length-prefixed bytes: parts hashed one by one would cost more.
+    encoded = [part.encode("utf-8") for part in parts]
+    return hashlib.sha256(b"".join(
+        [len(item).to_bytes(8, "big") + item for item in encoded]
+    )).hexdigest()[:40]
 
 
 def _stream(seed: int, dataset_id: str, row_index: int, decision: str,
@@ -39,13 +38,45 @@ def _stream(seed: int, dataset_id: str, row_index: int, decision: str,
     return random.Random(int(_digest((str(seed), dataset_id, str(row_index), decision, *context)), 16))
 
 
+_ID_INDEXES: dict[tuple[int, str], tuple[DatasetPackage, dict[Cell, list[dict[str, Cell]]]]] = {}
+_MARKET_CHOICES: dict[tuple[int, str, str], tuple[DatasetPackage, tuple[str, ...], tuple[str, ...]]] = {}
+
+
 def _entry(package: DatasetPackage, table_id: str, selected_id: Cell) -> dict[str, Cell]:
-    matches = [row for row in package.tables[table_id].rows if row["id"] == selected_id]
+    key = (id(package), table_id)
+    cached = _ID_INDEXES.get(key)
+    # The cache keeps the package alive, so a matching id is always the same object.
+    if cached is None or cached[0] is not package:
+        index: dict[Cell, list[dict[str, Cell]]] = {}
+        for row in package.tables[table_id].rows:
+            index.setdefault(row["id"], []).append(row)
+        cached = _ID_INDEXES[key] = (package, index)
+    matches = cached[1].get(selected_id, [])
     if len(matches) != 1:
         raise ConfigurationError(f"PACKAGE_STEP_REFERENCE: unknown {table_id} ID {selected_id!r}")
     return matches[0]
 
 
+def _market_choices(package: DatasetPackage, market: str, brand_id: str
+                    ) -> tuple[tuple[str, ...], tuple[str, ...]]:
+    """Sellers a brand may use and warehouses of a market, in the order the draws index them."""
+    key = (id(package), market, brand_id)
+    cached = _MARKET_CHOICES.get(key)
+    if cached is None or cached[0] is not package:
+        sellers = tuple(sorted(
+            str(row["id"]) for row in package.tables["seller"].rows
+            if row["market"] == market
+            and (row["seller_type_id"] != "manufacturer"
+                 or row["id"] == f"{brand_id}_direct_{market}")
+        ))
+        warehouses = tuple(sorted(
+            str(row["id"]) for row in package.tables["warehouse"].rows
+            if row["market"] == market
+        ))
+        cached = _MARKET_CHOICES[key] = (package, sellers, warehouses)
+    return cached[1], cached[2]
+
+
 def _integer(value: Cell, field: str) -> int:
     if type(value) is not int:
         raise ConfigurationError(f"PACKAGE_STEP_REFERENCE: expected integer Computer field {field!r}")
@@ -274,18 +305,9 @@ def derive_computer_row(package: DatasetPackage, selected: dict[str, Cell],
     return_rate = (Decimal("0.10") - (rating_value - 3) * Decimal("0.02")
                    + Decimal(_stream(seed, dataset_id, row_index, "returns", context).randrange(0, 6))
                    / 1000).quantize(Decimal("0.001"))
-    seller_ids = tuple(sorted(
-        str(row["id"]) for row in package.tables["seller"].rows
-        if row["market"] == market
-        and (row["seller_type_id"] != "manufacturer"
-             or row["id"] == f"{values['brand_id']}_direct_{market}")
-    ))
+    seller_ids, warehouse_ids = _market_choices(package, market, str(values["brand_id"]))
     seller_id = _stream(seed, dataset_id, row_index, "seller", context).choice(seller_ids)
     seller = _entry(package, "seller", seller_id)
-    warehouse_ids = tuple(sorted(
-        str(row["id"]) for row in package.tables["warehouse"].rows
-        if row["market"] == market
-    ))
     warehouse_id = _stream(seed, dataset_id, row_index, "warehouse", context).choice(
         warehouse_ids
     )

{tabalyst-0.7.2 → tabalyst-0.7.3}/src/tabalyst/generate_computer_output.py

@@ -8,8 +8,11 @@ from __future__ import annotations
 
 import csv
 import hashlib
+import io
 import json
 import sys
+import tempfile
+from collections.abc import Callable
 from contextlib import ExitStack
 from datetime import date
 from pathlib import Path
@@ -209,6 +212,33 @@ def render_computer_row(package: DatasetPackage, row: dict[str, Cell],
     return rendered
 
 
+def _write_rows_in_workers(
+    package: DatasetPackage, outputs: dict[str, Path], locales: tuple[str, ...],
+    columns: tuple[str, ...], dataset_id: str, rows: int, seed: int, as_of_date: date,
+    generation_profile: str, output_profile: str, market: str, unit_style: str,
+    selected_units: tuple[str, ...], workers: int, on_rows: Callable[[int], None] | None,
+) -> None:
+    """Write the CSVs from rows rendered by worker processes."""
+    # Imported here because the workers module imports this one.
+    from tabalyst.generate_computer_parallel import SpanSettings, write_rows
+
+    header = io.StringIO()
+    csv.writer(header, lineterminator="\n").writerow(columns)
+    with ExitStack() as stack:
+        # The parts sit beside the outputs: millions of rows would fill a small system temp.
+        directory = Path(stack.enter_context(tempfile.TemporaryDirectory(
+            prefix=".generate-parts-", dir=outputs[locales[0]].parent)))
+        streams = {locale: stack.enter_context(outputs[locale].open("wb")) for locale in locales}
+        for stream in streams.values():
+            stream.write(header.getvalue().encode("utf-8"))
+        write_rows(
+            SpanSettings(package.root, dataset_id, seed, as_of_date, generation_profile,
+                         output_profile, market, unit_style, selected_units, locales,
+                         columns, directory),
+            rows, workers, streams, on_rows,
+        )
+
+
 def write_computer_csvs(package: DatasetPackage, outputs: dict[str, Path],
                         metadata_path: Path, *, dataset_id: str = "computers",
                         rows: int = 10, seed: int = 42,
@@ -216,15 +246,23 @@ def write_computer_csvs(package: DatasetPackage, outputs: dict[str, Path],
                         generation_profile: str = "balanced",
                         output_profile: str = "compact", market: str = "ca",
                         unit_style: str = "numeric",
-                        unit_columns: tuple[str, ...] | None = None) -> dict[str, object]:
+                        unit_columns: tuple[str, ...] | None = None,
+                        on_rows: Callable[[int], None] | None = None,
+                        workers: int = 1) -> dict[str, object]:
     """Write paired CSVs and reproducible metadata to caller supplied paths.
 
     The caller owns path planning, staging, collision checks, and publication.
-    One selected and derived record feeds both locale writers per iteration.
+    One selected and derived record feeds every requested locale writer per iteration.
+    `outputs` holds `en`, `fr` or both; a single locale writes only that file.
+    `on_rows` receives the number of rows written, about 200 times and at the end.
+    With `workers` above 1 the rows are rendered by that many processes and the
+    files are byte for byte those of one process; `on_rows` then moves by span.
     Localized units are recorded with the effective column selection.
     """
-    if set(outputs) != {"en", "fr"} or len(set(outputs.values()) | {metadata_path}) != 3:
-        raise ValueError("Computer outputs need distinct en, fr, and metadata paths")
+    locales = tuple(locale for locale in ("en", "fr") if locale in outputs)
+    if (not locales or len(locales) != len(outputs)
+            or len(set(outputs.values()) | {metadata_path}) != len(outputs) + 1):
+        raise ValueError("Computer outputs need distinct en or fr paths and a metadata path")
     if type(rows) is not int or rows < 1:
         raise ValueError("rows must be positive")
     if output_profile not in PROFILE_COLUMNS:
@@ -241,23 +279,33 @@ def write_computer_csvs(package: DatasetPackage, outputs: dict[str, Path],
         or any(column not in columns or column not in UNIT_COLUMNS for column in selected_units)
         or unit_style == "numeric" and unit_columns is not None):
         raise ValueError("invalid Computer unit columns")
-    with ExitStack() as stack:
-        writers: dict[str, csv.DictWriter] = {}
-        for locale in ("en", "fr"):
-            stream = stack.enter_context(outputs[locale].open("w", encoding="utf-8", newline=""))
-            writer = csv.DictWriter(stream, fieldnames=columns,
-                                    extrasaction="raise", lineterminator="\n")
-            writer.writeheader()
-            writers[locale] = writer
-        for index in range(rows):
-            selected = select_catalog_row(package, dataset_id, seed, index,
-                                          generation_profile)
-            internal = derive_computer_row(package, selected, dataset_id, seed,
-                                           index, as_of_date, market)
-            for locale in ("en", "fr"):
-                writers[locale].writerow(render_computer_row(
-                    package, internal, locale, output_profile, unit_style, selected_units,
-                ))
+    if workers > 1:
+        _write_rows_in_workers(package, outputs, locales, columns, dataset_id, rows, seed,
+                               as_of_date, generation_profile, output_profile, market,
+                               unit_style, selected_units, workers, on_rows)
+    else:
+        with ExitStack() as stack:
+            writers: dict[str, csv.DictWriter] = {}
+            for locale in locales:
+                stream = stack.enter_context(outputs[locale].open("w", encoding="utf-8", newline=""))
+                writer = csv.DictWriter(stream, fieldnames=columns,
+                                        extrasaction="raise", lineterminator="\n")
+                writer.writeheader()
+                writers[locale] = writer
+            report_every = max(1, rows // 200)
+            for index in range(rows):
+                if on_rows is not None and index % report_every == 0:
+                    on_rows(index)
+                selected = select_catalog_row(package, dataset_id, seed, index,
+                                              generation_profile)
+                internal = derive_computer_row(package, selected, dataset_id, seed,
+                                               index, as_of_date, market)
+                for locale in locales:
+                    writers[locale].writerow(render_computer_row(
+                        package, internal, locale, output_profile, unit_style, selected_units,
+                    ))
+        if on_rows is not None:
+            on_rows(rows)
     resources: dict[str, str] = {}
     for path in sorted(package.input_paths):
         with path.open("rb") as source:
@@ -273,10 +321,10 @@ def write_computer_csvs(package: DatasetPackage, outputs: dict[str, Path],
         "request": {"dataset": dataset_id, "rows": rows, "seed": seed,
                     "as_of_date": as_of_date.isoformat(),
                     "generation_profile": generation_profile,
-                    "output_profile": output_profile, "locales": ["en", "fr"],
+                    "output_profile": output_profile, "locales": list(locales),
                     "market": market},
         "outputs": {locale: {"rows": rows, "profile": output_profile,
-                             "columns": len(columns)} for locale in ("en", "fr")},
+                             "columns": len(columns)} for locale in locales},
         "engine_version": __version__,
         "python_version": f"{sys.version_info.major}.{sys.version_info.minor}",
     }