tabalyst 0.6.1__tar.gz → 0.6.2__tar.gz

{tabalyst-0.6.1 → tabalyst-0.6.2}/MANIFEST.in

@@ -3,3 +3,4 @@ include README.md
 include examples/README.md
 include examples/config.json
 recursive-include examples/input *.csv
+recursive-include src/tabalyst/generate_resources *.csv *.md

{tabalyst-0.6.1/src/tabalyst.egg-info → tabalyst-0.6.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tabalyst
-Version: 0.6.1
+Version: 0.6.2
 Summary: Open-source toolkit for understanding and working with unfamiliar data.
 Author: Gregory Borelli
 License-Expression: MPL-2.0
@@ -27,6 +27,7 @@ Requires-Dist: jinja2>=3.1
 Requires-Dist: platformdirs>=4
 Requires-Dist: pydantic>=2.11
 Requires-Dist: python-calamine>=0.8
+Requires-Dist: PyYAML>=6.0
 Requires-Dist: typer>=0.12
 Provides-Extra: dev
 Requires-Dist: build>=1.2; extra == "dev"
@@ -48,7 +49,8 @@ Its first tool is **Tabalyst Report**. The current CSV implementation,
 profile and a self-contained interactive HTML report.
 **Tabalyst Scan** describes CSV, JSON, JSONL and Excel files in a scan document,
 **Tabalyst Inspect** finds how to read a JSON, JSONL or Excel file, and
-**Tabalyst Sample** creates smaller CSV files.
+**Tabalyst Sample** creates smaller CSV files by sampling, and **Tabalyst
+Generate** creates synthetic CSV datasets.
 
 Tabalyst is in beta. Its interfaces may still change while
 the shared toolkit architecture is being established.
@@ -273,6 +275,33 @@ Sampling reads CSV records as a stream. Random and stratified sampling keep
 only the requested sample, plus stratum counts, in memory. Existing outputs
 require `--force`, and an input file is never overwritten.
 
+## Tabalyst Generate
+
+Generate synthetic CSV data from a versioned YAML definition. The packaged
+`crm` example writes one contacts file; `insurance`
+writes three branch files and a combined file:
+
+```console
+tabalyst generate --example crm -o crm-contacts.csv
+tabalyst generate --example insurance --rows 3000 -d insurance-output/
+tabalyst generate --example insurance --rows 3000 --anomaly-profile dirty_realistic -d insurance-dirty/
+```
+
+You can pass `examples/generate/input/crm.yaml` or
+`examples/generate/input/insurance.yaml` instead of `--example`. A selected anomaly
+profile writes an altered CSV, a cell change log and a summary beside the
+untouched clean CSV. Insurance's profile changes only the combined file.
+`--rows 3000` apportions 2,182, 273 and 545 rows to the three branches.
+Existing outputs require `--force`.
+
+Run `python examples/generate/generate_examples.py` to generate every YAML
+from `examples/generate/input/` into ignored local output folders.
+
+Insurance preserves the 33 source columns and the combined `filiale` column of
+the existing public demo, but its generated rows are different. The branch date
+and identifier formats are preserved; `filiale` is the last column in the new
+combined CSV. The current public demo and report remain unchanged.
+
 ## Tabalyst Inspect
 
 A JSON file can hold several arrays, and an Excel workbook several sheets.

{tabalyst-0.6.1 → tabalyst-0.6.2}/README.md

@@ -10,7 +10,8 @@ Its first tool is **Tabalyst Report**. The current CSV implementation,
 profile and a self-contained interactive HTML report.
 **Tabalyst Scan** describes CSV, JSON, JSONL and Excel files in a scan document,
 **Tabalyst Inspect** finds how to read a JSON, JSONL or Excel file, and
-**Tabalyst Sample** creates smaller CSV files.
+**Tabalyst Sample** creates smaller CSV files by sampling, and **Tabalyst
+Generate** creates synthetic CSV datasets.
 
 Tabalyst is in beta. Its interfaces may still change while
 the shared toolkit architecture is being established.
@@ -235,6 +236,33 @@ Sampling reads CSV records as a stream. Random and stratified sampling keep
 only the requested sample, plus stratum counts, in memory. Existing outputs
 require `--force`, and an input file is never overwritten.
 
+## Tabalyst Generate
+
+Generate synthetic CSV data from a versioned YAML definition. The packaged
+`crm` example writes one contacts file; `insurance`
+writes three branch files and a combined file:
+
+```console
+tabalyst generate --example crm -o crm-contacts.csv
+tabalyst generate --example insurance --rows 3000 -d insurance-output/
+tabalyst generate --example insurance --rows 3000 --anomaly-profile dirty_realistic -d insurance-dirty/
+```
+
+You can pass `examples/generate/input/crm.yaml` or
+`examples/generate/input/insurance.yaml` instead of `--example`. A selected anomaly
+profile writes an altered CSV, a cell change log and a summary beside the
+untouched clean CSV. Insurance's profile changes only the combined file.
+`--rows 3000` apportions 2,182, 273 and 545 rows to the three branches.
+Existing outputs require `--force`.
+
+Run `python examples/generate/generate_examples.py` to generate every YAML
+from `examples/generate/input/` into ignored local output folders.
+
+Insurance preserves the 33 source columns and the combined `filiale` column of
+the existing public demo, but its generated rows are different. The branch date
+and identifier formats are preserved; `filiale` is the last column in the new
+combined CSV. The current public demo and report remain unchanged.
+
 ## Tabalyst Inspect
 
 A JSON file can hold several arrays, and an Excel workbook several sheets.

tabalyst-0.6.2/examples/README.md

@@ -0,0 +1,144 @@
+# Tabalyst examples
+
+The examples are organized by input name so additional public datasets can be
+added without mixing source data and generated artifacts.
+
+```text
+examples/
+|-- config.json
+|-- generate/
+|   |-- generate_examples.py
+|   |-- input/ (four YAML definitions)
+|   `-- output/ (generated locally; ignored by Git)
+|-- input/
+|   |-- basic.csv
+|   |-- insurance-customers.csv
+|   |-- orders.json
+|   |-- sales.xlsx
+|   `-- web-events.jsonl
+`-- output/
+    |-- basic/
+    |   |-- report.html
+    |   |-- report.json
+    |   `-- executions.json
+    |-- insurance-customers/
+    |   |-- report.html
+    |   |-- report.json
+    |   |-- report/ (34 standalone column pages with --details)
+    |   `-- executions.json
+    |-- orders/
+    |   |-- report.html
+    |   |-- report.json
+    |   `-- executions.json
+    |-- sales/
+    |   |-- report.html
+    |   |-- report.json
+    |   `-- executions.json
+    `-- web-events/
+        |-- report.html
+        |-- report.json
+        `-- executions.json
+```
+
+`basic.csv` is a five-row smoke example. `insurance-customers.csv` contains 3,000
+synthetic customer and contract records across 34 columns. Its people, contact
+details, and contracts are fictional; email addresses use reserved `.example`
+domains. `orders.json` is a synthetic shop export: 60 customers with nested
+addresses, tags and orders. Its report has one dataset, `$.customers[]`, the only
+collection of arrays of objects that Tabalyst Inspect finds in it.
+`web-events.jsonl` is a synthetic web analytics log in JSON Lines: 300 events with
+nested `user` and `device` objects, tags arrays, null values and an optional
+field that appears late, plus three bad lines (a truncated record, a line that
+is not JSON and a JSON array instead of an object). Its report is partial on
+purpose: the three lines are excluded and counted, the 300 events are analyzed.
+Its people and identifiers are fictional. `sales.xlsx` is a synthetic Excel
+workbook: a sheet `Orders` of 120 fictional orders under two title lines, a
+five-row `Regions` lookup sheet and a `Notes` sheet. Inspect selects `$.Orders`,
+which has more than ten times the rows of `$.Regions`, so the report needs no
+choice. The workbook is written by `make_sales_workbook.py`, with the same
+bytes on every run; it needs `pip install xlsxwriter` and is only to run when the
+input itself must be regenerated.
+
+From the repository root, regenerate the outputs with:
+
+```console
+tabalyst report examples/input/basic.csv -o examples/output/basic/report.html --config examples/config.json --force
+tabalyst report examples/input/insurance-customers.csv -o examples/output/insurance-customers/report.html --config examples/config.json --details --force
+tabalyst report examples/input/orders.json -o examples/output/orders/report.html --config examples/config.json --force
+tabalyst report examples/input/web-events.jsonl -o examples/output/web-events/report.html --config examples/config.json --force
+tabalyst report examples/input/sales.xlsx -o examples/output/sales/report.html --config examples/config.json --force
+```
+
+Every output folder is independent and contains its own HTML report, canonical
+JSON profile and cumulative `executions.json` history. The insurance example
+also has standalone HTML pages under `report/`. Delete an output folder
+before regenerating when a fresh one-entry execution history is required.
+
+The input datasets and generated output are public repository examples. The wheel
+contains the runtime package, report resources, and packaged copies of the
+`examples/generate/input/` YAML definitions. CRM and Insurance are local development
+demos. Generate is under development
+and is not advertised as a released feature. The CRM data is synthetic, with
+reserved `.example` domains and fictional phone numbers. The configured 20,000
+rows, generic account names and CRM audit story were approved for the example
+on 2026-10-05. Generate remains local development work until a release.
+
+To review the CRM export outside the checkout:
+
+```console
+tabalyst generate --example crm --rows 500 --anomaly-profile dirty_realistic -d OUTPUT_DIR
+tabalyst scan OUTPUT_DIR/crm-contacts.dirty_realistic.csv -o OUTPUT_DIR/scan.json
+tabalyst report OUTPUT_DIR/scan.json --scan -o OUTPUT_DIR/report.html
+```
+
+Scan and Report expose missing values, fragmented categories and normalized
+variants. The mutation summary counts requested, applied and skipped changes.
+Neither tool currently diagnoses cross-column CRM rules such as stage versus
+probability.
+
+`examples/generate/input/common-references.yaml` is a lot 3R development fixture.
+It generates coherent synthetic Canadian person, address and contact fields
+from packaged references. Its prefixes and area codes are synthetic, email
+domains end in `.example`, and phone line numbers use reserved `555-01xx`.
+The Insurance example is now complete in `examples/generate/input/insurance.yaml`
+and the packaged `--example insurance` copy. Its seven domain tables are in
+`src/tabalyst/generate_resources/insurance/`; their provenance is in
+`insurance/SOURCES.md`. To review 3,000 rows outside the checkout:
+
+```console
+tabalyst generate --example insurance --rows 3000 --anomaly-profile dirty_realistic -d OUTPUT_DIR
+```
+
+This creates clean files for Principal, Ontario and Quebec, their clean union,
+and three anomaly artifacts for that union. The 2,182/273/545 branch allocation
+comes from the declared 200,000/25,000/50,000 weights. The new clean CSV has
+the same 34 column names as the existing public demo, but places `filiale` last
+and has different rows and identifiers. Do not replace
+`examples/input/insurance-customers.csv` or its report without a separate
+decision to change the published demo.
+
+`examples/generate/input/cohorts-anomalies.yaml` is a lot 4 development fixture. It
+shows two normal status conventions in the clean CSV and a separate `dirty`
+profile that writes an altered CSV, cell-level change log and summary. Run it
+with `tabalyst generate examples/generate/input/cohorts-anomalies.yaml --anomaly-profile dirty -d OUTPUT_DIR`.
+Keep `OUTPUT_DIR` outside the repository for one-off local trials. The packaged
+Insurance YAML is the same complete example as the visible file.
+
+## Regenerate all Generate examples
+
+Run the script from the repository root with an editable installation of
+Tabalyst. It reads every YAML in `examples/generate/input/` and writes into
+`examples/generate/output/<definition>/`. The output folder is ignored by Git.
+By default it uses each YAML's declared row counts and generates every declared
+anomaly profile along with its clean CSV. Insurance therefore writes 275,000
+combined rows; allow enough disk space for the clean and altered copies.
+
+```console
+python examples/generate/generate_examples.py
+```
+
+Existing outputs are protected; pass `--force` to replace generated files.
+For a quick review run, `--rows 30` overrides each definition's row count,
+and `--output-dir PATH` writes outside the checkout. `--clean-only` skips the
+profile passes. A definition with several profiles gets one subfolder per
+profile so its clean files cannot collide.

{tabalyst-0.6.1 → tabalyst-0.6.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "tabalyst"
-version = "0.6.1"
+version = "0.6.2"
 description = "Open-source toolkit for understanding and working with unfamiliar data."
 readme = "README.md"
 requires-python = ">=3.11"
@@ -40,6 +40,7 @@ dependencies = [
     "platformdirs>=4",
     "pydantic>=2.11",
     "python-calamine>=0.8",
+    "PyYAML>=6.0",
     "typer>=0.12",
 ]
 
@@ -59,7 +60,7 @@ tabalyst = "tabalyst.cli:main"
 where = ["src"]
 
 [tool.setuptools.package-data]
-tabalyst = ["templates/*.html", "static/*.css", "static/*.js"]
+tabalyst = ["templates/*.html", "static/*.css", "static/*.js", "generate_examples/*.yaml", "generate_resources/**/*.csv", "generate_resources/*.md"]
 
 [project.urls]
 Homepage = "https://github.com/loribel-labs/tabalyst"

{tabalyst-0.6.1 → tabalyst-0.6.2}/src/tabalyst/__init__.py

@@ -6,6 +6,10 @@
 
 from tabalyst._version import __version__
 from tabalyst.errors import ConfigurationError, InputError, ReportError, TabalystError
+from tabalyst.generate_service import GeneratePlan as GeneratePlan
+from tabalyst.generate_service import GenerateResult as GenerateResult
+from tabalyst.generate_service import build_generate_plan as build_generate_plan
+from tabalyst.generate_service import generate_datasets as generate_datasets
 from tabalyst.inspect_service import InspectResult as InspectResult
 from tabalyst.inspect_service import generate_inspections as generate_inspections
 from tabalyst.inspect_service import inspect as inspect
@@ -26,6 +30,8 @@ from tabalyst.service import analyze_scan as analyze_scan
 
 __all__ = [
     "ConfigurationError",
+    "GeneratePlan",
+    "GenerateResult",
     "InputError",
     "InspectResult",
     "ReportError",
@@ -34,6 +40,8 @@ __all__ = [
     "TabalystError",
     "__version__",
     "analyze",
+    "build_generate_plan",
+    "generate_datasets",
     "generate_inspections",
     "generate_reports",
     "generate_samples",

{tabalyst-0.6.1 → tabalyst-0.6.2}/src/tabalyst/cli/app.py

@@ -10,6 +10,7 @@ import typer
 
 from tabalyst import __version__
 from tabalyst.cli.cache import cache_app
+from tabalyst.cli.generate import generate_command
 from tabalyst.cli.inspect import inspect_command
 from tabalyst.cli.report import report_command
 from tabalyst.cli.sample import sample_command
@@ -45,6 +46,7 @@ def root(
 
 
 app.command("inspect")(inspect_command)
+app.command("generate")(generate_command)
 app.command("report")(report_command)
 app.command("sample")(sample_command)
 app.command("scan")(scan_command)

tabalyst-0.6.2/src/tabalyst/cli/generate.py

@@ -0,0 +1,67 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+"""CLI adapter for Tabalyst Generate."""
+
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from tabalyst.cli.terminal import error_exit_code
+from tabalyst.errors import TabalystError
+from tabalyst.generate_service import build_generate_plan, generate_datasets
+
+
+def generate_command(
+    definition: Annotated[
+        Path | None,
+        typer.Argument(help="YAML Generate definition."),
+    ] = None,
+    example: Annotated[
+        str | None,
+        typer.Option("--example", help="Packaged example: crm or insurance."),
+    ] = None,
+    output: Annotated[
+        Path | None,
+        typer.Option("--output", "-o", help="One clean CSV file."),
+    ] = None,
+    output_dir: Annotated[
+        Path | None,
+        typer.Option("--output-dir", "-d", help="Directory for all artifacts."),
+    ] = None,
+    rows: Annotated[
+        int | None,
+        typer.Option("--rows", help="Number of generated rows."),
+    ] = None,
+    seed: Annotated[
+        int | None,
+        typer.Option("--seed", help="Root random seed."),
+    ] = None,
+    as_of_date: Annotated[
+        str | None,
+        typer.Option("--as-of-date", help="Reference date (YYYY-MM-DD)."),
+    ] = None,
+    anomaly_profile: Annotated[
+        str | None,
+        typer.Option("--anomaly-profile", help="Selected anomaly profile."),
+    ] = None,
+    force: Annotated[
+        bool,
+        typer.Option("--force", "-f", help="Replace existing artifacts."),
+    ] = False,
+) -> None:
+    """Validate and generate datasets from a YAML definition."""
+    try:
+        plan = build_generate_plan(
+            definition, example=example, output=output, output_dir=output_dir,
+            rows=rows, seed=seed, as_of_date=as_of_date,
+            anomaly_profile=anomaly_profile, force=force,
+        )
+        result = generate_datasets(plan)
+    except TabalystError as exc:
+        typer.echo(f"Error: {exc}", err=True)
+        raise typer.Exit(error_exit_code(exc)) from exc
+    for path in result.paths:
+        typer.echo(f"Created: {path}", err=True)

tabalyst-0.6.2/src/tabalyst/generate_anomalies.py

@@ -0,0 +1,126 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+"""Measured, streaming mutation pass over a staged clean Generate CSV."""
+
+from __future__ import annotations
+
+import csv
+import random
+from collections import Counter
+from pathlib import Path
+
+from tabalyst.generate_values import _stream
+
+CHANGE_FIELDS = ["event_id", "dataset", "profile", "row_number", "operation_index",
+                 "kind", "column", "before", "after", "cause"]
+
+
+def _mutate(row: dict[str, str], operation: dict, history: dict[str, list[str]],
+            rng: random.Random) -> dict[str, str]:
+    column = operation["column"]
+    before = row[column]
+    kind = operation["kind"]
+    if kind == "set_missing":
+        return {column: ""} if before else {}
+    if kind == "replace":
+        return {column: operation["value"]} if operation["value"] != before else {}
+    if kind == "duplicate":
+        choices = [value for value in history[column] if value and value != before]
+        return {column: rng.choice(choices)} if choices else {}
+    if kind == "swap":
+        other = operation["value"]
+        return {column: row[other], other: before} if before != row[other] else {}
+    transforms = {
+        "upper": str.upper,
+        "lower": str.lower,
+        "strip": str.strip,
+        "date_slash": lambda value: value.replace("-", "/"),
+        "phone_spaces": lambda value: value.replace("-", " "),
+    }
+    after = transforms[operation["value"]](before)
+    return {column: after} if before and after != before else {}
+
+
+def inject_anomalies(clean: Path, altered: Path, changes: Path, *,
+                     dataset: str, profile: dict, seed: int, count: int,
+                     delimiter: str) -> dict:
+    """Write altered CSV and cell log; return requested and actual counts."""
+    operations = profile["operations"]
+    requested = [round(count * operation["rate"]) for operation in operations]
+    remaining = requested.copy()
+    selection_rngs = [_stream(seed, dataset, f"{profile['id']}:{index}",
+                              "anomaly_selection") for index in range(len(operations))]
+    mutation_rngs = [_stream(seed, dataset, f"{profile['id']}:{index}", "anomaly_value")
+                     for index in range(len(operations))]
+    applied: Counter[int] = Counter()
+    skipped: Counter[int] = Counter()
+    history: dict[str, list[str]] = {}
+    affected_rows = 0
+    changed_cells = 0
+    events = 0
+    with (clean.open("r", encoding="utf-8", newline="") as source,
+          altered.open("w", encoding="utf-8", newline="") as dirty,
+          changes.open("w", encoding="utf-8", newline="") as journal):
+        reader = csv.DictReader(source, delimiter=delimiter)
+        fields = reader.fieldnames or []
+        history = {field: [] for field in fields}
+        writer = csv.DictWriter(dirty, fieldnames=fields, delimiter=delimiter,
+                                lineterminator="\n", extrasaction="raise")
+        log = csv.DictWriter(journal, fieldnames=CHANGE_FIELDS, lineterminator="\n")
+        writer.writeheader()
+        log.writeheader()
+        observed = 0
+        for row_index, clean_row in enumerate(reader):
+            observed += 1
+            row = clean_row.copy()
+            changed = False
+            for index, operation in enumerate(operations):
+                rows_left = count - row_index
+                selected = (remaining[index] == rows_left or
+                            (remaining[index] > 0 and
+                             selection_rngs[index].random() < remaining[index] / rows_left))
+                if not selected:
+                    continue
+                remaining[index] -= 1
+                when = operation.get("when", {})
+                if any(row.get(field) != value for field, value in when.items()):
+                    skipped[index] += 1
+                    continue
+                updates = _mutate(row, operation, history, mutation_rngs[index])
+                if not updates:
+                    skipped[index] += 1
+                    continue
+                events += 1
+                for column, after in updates.items():
+                    before = row[column]
+                    row[column] = after
+                    changed_cells += 1
+                    log.writerow({"event_id": events, "dataset": dataset,
+                                  "profile": profile["id"], "row_number": row_index + 2,
+                                  "operation_index": index + 1, "kind": operation["kind"],
+                                  "column": column, "before": before, "after": after,
+                                  "cause": f"{profile['id']}.operations[{index}]"})
+                applied[index] += 1
+                changed = True
+            writer.writerow(row)
+            affected_rows += int(changed)
+            for field, value in clean_row.items():
+                if value and len(history[field]) < 1000:
+                    history[field].append(value)
+        if observed != count:
+            raise ValueError(f"Clean row count changed for {dataset}: {observed} != {count}")
+    return {
+        "dataset": dataset, "profile": profile["id"], "seed": seed,
+        "total_rows": count, "affected_rows": affected_rows,
+        "affected_row_rate": affected_rows / count if count else 0.0,
+        "events": events, "changed_cells": changed_cells,
+        "operations": [
+            {"index": index + 1, "kind": operation["kind"], "column": operation["column"],
+             "rate": operation["rate"], "requested_rows": requested[index],
+             "applied_rows": applied[index], "skipped_rows": skipped[index],
+             "applied_rate": applied[index] / count if count else 0.0}
+            for index, operation in enumerate(operations)
+        ],
+    }

tabalyst-0.6.2/src/tabalyst/generate_csv.py

@@ -0,0 +1,34 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+"""Streaming CSV output adapted from dataset_factory.exporters.csv."""
+
+import csv
+from collections.abc import Iterable
+from pathlib import Path
+
+
+def write_csv(
+    path: Path,
+    fieldnames: list[str],
+    rows: Iterable[dict[str, str]],
+    *,
+    delimiter: str = ",",
+) -> int:
+    """Write a staged CSV using stable UTF-8/LF and strict field names."""
+    count = 0
+    with path.open("w", encoding="utf-8", newline="") as stream:
+        writer = csv.DictWriter(
+            stream,
+            fieldnames=fieldnames,
+            extrasaction="raise",
+            lineterminator="\n",
+            delimiter=delimiter,
+        )
+        writer.writeheader()
+        for row in rows:
+            writer.writerow(row)
+            count += 1
+        stream.flush()
+    return count