tabalyst 0.3.0__tar.gz → 0.4.0__tar.gz

{tabalyst-0.3.0/src/tabalyst.egg-info → tabalyst-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tabalyst
-Version: 0.3.0
+Version: 0.4.0
 Summary: Open-source toolkit for understanding and working with unfamiliar data.
 Author: Gregory Borelli
 License-Expression: MIT
@@ -21,9 +21,9 @@ Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: ijson>=3.3
 Requires-Dist: jinja2>=3.1
-Requires-Dist: pandas>=2.2
-Requires-Dist: pydantic>=2.10
+Requires-Dist: pydantic>=2.11
 Requires-Dist: typer>=0.12
 Provides-Extra: dev
 Requires-Dist: build>=1.2; extra == "dev"
@@ -65,6 +65,8 @@ changes often: update it before each new test.
 | One file, custom name | `tabalyst report data.csv -o report.html` | The file `report.html` |
 | Several files, automatic names | `tabalyst report *.csv` | Beside each source |
 | Several files, one directory | `tabalyst report *.csv -d reports/` | The directory `reports/` |
+| One JSON file | `tabalyst report data.json` | `data.report.html` beside the source |
+| From a scan document | `tabalyst report --scan data.scan.json` | `data.html` beside the scan |
 
 The simplest command keeps the source filename:
 
@@ -87,6 +89,18 @@ Use `-o` to choose a different HTML filename for one source:
 tabalyst report customers.csv -o customer-analysis.html
 ```
 
+### JSON files
+
+A JSON file gets a report too, named `<stem>.report.html` so its profile
+`<stem>.report.json` never replaces the source. Each collection of records,
+such as the `customers` array of `{"customers": [...]}`, is a dataset of the
+report, and nested fields are columns named by their path, such as
+`address.city`:
+
+```console
+tabalyst report orders.json
+```
+
 ### Multiple files
 
 Report several CSV files at once:
@@ -201,16 +215,53 @@ Sampling reads CSV records as a stream. Random and stratified sampling keep
 only the requested sample, plus stratum counts, in memory. Existing outputs
 require `--force`, and an input file is never overwritten.
 
+## Tabalyst Scan
+
+Describe every field of a CSV or JSON file in one JSON document:
+
+```console
+tabalyst scan customers.csv
+tabalyst scan orders.json --collection "$.customers[]"
+```
+
+The default output is `customers.scan.json` beside the source; `-o`, `-d` and
+`--force` work as for the other commands. Tabalyst Scan reads the file once as
+a stream, with memory bounded by configurable limits, and records for each
+field its presence, native types, missing values, frequencies, exact
+statistics, normalization variants, technical type and the result of every
+detector: numbers with decimal commas, dates, booleans, enumerations, email
+addresses, URLs, phone numbers, postal codes, currency amounts, percentages,
+quantities, UUIDs and IP addresses. Values of sensitive fields, such as email
+addresses, are masked by default. Results are written atomically: an
+interrupted scan never leaves a partial file.
+
+Build the report from a scan document without reading the source again:
+
+```console
+tabalyst report --scan customers.scan.json
+```
+
+Tabalyst refuses a scan whose source changed since it was written, or whose
+settings differ from the `scan` settings of `--config`.
+
+See [Scan CSV and JSON files](https://github.com/loribel-labs/tabalyst/blob/main/docs/en/how-to/scan-files.md)
+and the [scan format](https://github.com/loribel-labs/tabalyst/blob/main/docs/en/reference/scan-format.md).
+
 ## What the report analyzes
 
 - Dataset dimensions, missing cells, duplicates, and quality observations.
 - Physical and semantic types with confidence and error rates.
 - Numeric, date, string-length, normalization, and value distributions.
-- Distinct values, representative examples, enum candidates, and date formats.
+- Distinct values, representative examples, date formats, and semantic types
+  such as enumerations, email addresses, phone numbers, and postal codes.
 - CSV record widths, quoting, encoding, and delimiter configuration.
 - A bounded raw-data preview while every record is analyzed.
 
-The HTML report is self-contained and works without a CDN or network connection.
+The report is built on Tabalyst Scan: it reads the CSV once as a stream and
+masks values of sensitive columns, such as email addresses, by default.
+Ambiguous dates stay ambiguous; the report shows the evidence of the column
+without applying it. The HTML report is self-contained and works without a CDN
+or network connection.
 The JSON profile contains the same canonical analysis result for scripts and
 future Tabalyst tools.
 
@@ -238,10 +289,17 @@ sample = tabalyst.sample_csv(
     rows=1000,
     seed=42,
 )
+
+scan = tabalyst.scan("orders.json")
+scans = tabalyst.generate_scans(["data/*.json"], output_dir="scans")
+reports = tabalyst.generate_reports(["scans/*.scan.json"], from_scan=True)
 ```
 
 `analyze()` returns the JSON-serializable profile for one report.
 `generate_reports()` returns the complete batch plan, successes, and failures.
+`scan()` returns a scan result without writing anything; `generate_scans()`
+writes one `.scan.json` document per source; `from_scan=True` builds reports
+from such documents.
 Expected failures derive from `tabalyst.TabalystError`.
 
 Existing artifacts are never replaced silently. Pass `force=True` when
@@ -253,15 +311,21 @@ Configuration files are strict JSON. A minimal file is:
 
 ```json
 {
-  "csv": {
-    "delimiter": ";",
-    "encoding": "cp1252"
+  "scan": {
+    "csv": {
+      "delimiter": ";",
+      "encoding": "cp1252"
+    },
+    "values": {"null_markers": ["N/A"]}
   }
 }
 ```
 
-Explicit CLI or Python arguments override the configuration file, which overrides
-Tabalyst defaults. See the
+Analysis settings, CSV reading included, go in the `scan` object, shared by
+`tabalyst report` and `tabalyst scan`. Top-level settings shape the report
+presentation, and `csv` configures `tabalyst sample`. Explicit CLI or Python
+arguments override the configuration file, which overrides Tabalyst defaults.
+No configuration file is loaded unless it is passed with `--config`. See the
 [configuration reference](https://github.com/loribel-labs/tabalyst/blob/main/docs/en/reference/configuration.md)
 for all analysis settings.
 
@@ -271,10 +335,11 @@ Tabalyst performs analysis locally and adds no telemetry or remote processing.
 Generated JSON and HTML may contain source values and should be shared
 accordingly.
 
-Report analysis currently loads the complete CSV into memory. Sampling uses a
-streaming reader with bounded row storage. File and phase progress is available
-for reports; row percentages, throughput estimates, recursive directory input,
-and parallel batch execution will require later ingestion work.
+Reports, samples, and scans read files as a stream, with memory bounded by
+configurable limits, including duplicate-row detection
+(`scan.limits.max_tracked_records`). Reports and scans show the share of the file read;
+throughput estimates, recursive directory input, and parallel batch execution
+will require later work.
 
 ## Examples and development
 

{tabalyst-0.3.0 → tabalyst-0.4.0}/README.md

@@ -31,6 +31,8 @@ changes often: update it before each new test.
 | One file, custom name | `tabalyst report data.csv -o report.html` | The file `report.html` |
 | Several files, automatic names | `tabalyst report *.csv` | Beside each source |
 | Several files, one directory | `tabalyst report *.csv -d reports/` | The directory `reports/` |
+| One JSON file | `tabalyst report data.json` | `data.report.html` beside the source |
+| From a scan document | `tabalyst report --scan data.scan.json` | `data.html` beside the scan |
 
 The simplest command keeps the source filename:
 
@@ -53,6 +55,18 @@ Use `-o` to choose a different HTML filename for one source:
 tabalyst report customers.csv -o customer-analysis.html
 ```
 
+### JSON files
+
+A JSON file gets a report too, named `<stem>.report.html` so its profile
+`<stem>.report.json` never replaces the source. Each collection of records,
+such as the `customers` array of `{"customers": [...]}`, is a dataset of the
+report, and nested fields are columns named by their path, such as
+`address.city`:
+
+```console
+tabalyst report orders.json
+```
+
 ### Multiple files
 
 Report several CSV files at once:
@@ -167,16 +181,53 @@ Sampling reads CSV records as a stream. Random and stratified sampling keep
 only the requested sample, plus stratum counts, in memory. Existing outputs
 require `--force`, and an input file is never overwritten.
 
+## Tabalyst Scan
+
+Describe every field of a CSV or JSON file in one JSON document:
+
+```console
+tabalyst scan customers.csv
+tabalyst scan orders.json --collection "$.customers[]"
+```
+
+The default output is `customers.scan.json` beside the source; `-o`, `-d` and
+`--force` work as for the other commands. Tabalyst Scan reads the file once as
+a stream, with memory bounded by configurable limits, and records for each
+field its presence, native types, missing values, frequencies, exact
+statistics, normalization variants, technical type and the result of every
+detector: numbers with decimal commas, dates, booleans, enumerations, email
+addresses, URLs, phone numbers, postal codes, currency amounts, percentages,
+quantities, UUIDs and IP addresses. Values of sensitive fields, such as email
+addresses, are masked by default. Results are written atomically: an
+interrupted scan never leaves a partial file.
+
+Build the report from a scan document without reading the source again:
+
+```console
+tabalyst report --scan customers.scan.json
+```
+
+Tabalyst refuses a scan whose source changed since it was written, or whose
+settings differ from the `scan` settings of `--config`.
+
+See [Scan CSV and JSON files](https://github.com/loribel-labs/tabalyst/blob/main/docs/en/how-to/scan-files.md)
+and the [scan format](https://github.com/loribel-labs/tabalyst/blob/main/docs/en/reference/scan-format.md).
+
 ## What the report analyzes
 
 - Dataset dimensions, missing cells, duplicates, and quality observations.
 - Physical and semantic types with confidence and error rates.
 - Numeric, date, string-length, normalization, and value distributions.
-- Distinct values, representative examples, enum candidates, and date formats.
+- Distinct values, representative examples, date formats, and semantic types
+  such as enumerations, email addresses, phone numbers, and postal codes.
 - CSV record widths, quoting, encoding, and delimiter configuration.
 - A bounded raw-data preview while every record is analyzed.
 
-The HTML report is self-contained and works without a CDN or network connection.
+The report is built on Tabalyst Scan: it reads the CSV once as a stream and
+masks values of sensitive columns, such as email addresses, by default.
+Ambiguous dates stay ambiguous; the report shows the evidence of the column
+without applying it. The HTML report is self-contained and works without a CDN
+or network connection.
 The JSON profile contains the same canonical analysis result for scripts and
 future Tabalyst tools.
 
@@ -204,10 +255,17 @@ sample = tabalyst.sample_csv(
     rows=1000,
     seed=42,
 )
+
+scan = tabalyst.scan("orders.json")
+scans = tabalyst.generate_scans(["data/*.json"], output_dir="scans")
+reports = tabalyst.generate_reports(["scans/*.scan.json"], from_scan=True)
 ```
 
 `analyze()` returns the JSON-serializable profile for one report.
 `generate_reports()` returns the complete batch plan, successes, and failures.
+`scan()` returns a scan result without writing anything; `generate_scans()`
+writes one `.scan.json` document per source; `from_scan=True` builds reports
+from such documents.
 Expected failures derive from `tabalyst.TabalystError`.
 
 Existing artifacts are never replaced silently. Pass `force=True` when
@@ -219,15 +277,21 @@ Configuration files are strict JSON. A minimal file is:
 
 ```json
 {
-  "csv": {
-    "delimiter": ";",
-    "encoding": "cp1252"
+  "scan": {
+    "csv": {
+      "delimiter": ";",
+      "encoding": "cp1252"
+    },
+    "values": {"null_markers": ["N/A"]}
   }
 }
 ```
 
-Explicit CLI or Python arguments override the configuration file, which overrides
-Tabalyst defaults. See the
+Analysis settings, CSV reading included, go in the `scan` object, shared by
+`tabalyst report` and `tabalyst scan`. Top-level settings shape the report
+presentation, and `csv` configures `tabalyst sample`. Explicit CLI or Python
+arguments override the configuration file, which overrides Tabalyst defaults.
+No configuration file is loaded unless it is passed with `--config`. See the
 [configuration reference](https://github.com/loribel-labs/tabalyst/blob/main/docs/en/reference/configuration.md)
 for all analysis settings.
 
@@ -237,10 +301,11 @@ Tabalyst performs analysis locally and adds no telemetry or remote processing.
 Generated JSON and HTML may contain source values and should be shared
 accordingly.
 
-Report analysis currently loads the complete CSV into memory. Sampling uses a
-streaming reader with bounded row storage. File and phase progress is available
-for reports; row percentages, throughput estimates, recursive directory input,
-and parallel batch execution will require later ingestion work.
+Reports, samples, and scans read files as a stream, with memory bounded by
+configurable limits, including duplicate-row detection
+(`scan.limits.max_tracked_records`). Reports and scans show the share of the file read;
+throughput estimates, recursive directory input, and parallel batch execution
+will require later work.
 
 ## Examples and development
 

{tabalyst-0.3.0 → tabalyst-0.4.0}/examples/README.md

@@ -8,13 +8,18 @@ examples/
 |-- config.json
 |-- input/
 |   |-- basic.csv
-|   `-- insurance-customers.csv
+|   |-- insurance-customers.csv
+|   `-- orders.json
 `-- output/
     |-- basic/
     |   |-- report.html
     |   |-- report.json
     |   `-- executions.json
-    `-- insurance-customers/
+    |-- insurance-customers/
+    |   |-- report.html
+    |   |-- report.json
+    |   `-- executions.json
+    `-- orders/
         |-- report.html
         |-- report.json
         `-- executions.json
@@ -23,13 +28,16 @@ examples/
 `basic.csv` is a five-row smoke example. `insurance-customers.csv` contains 3,000
 synthetic customer and contract records across 34 columns. Its people, contact
 details, and contracts are fictional; email addresses use reserved `.example`
-domains.
+domains. `orders.json` is a synthetic shop export: 60 customers with nested
+addresses, tags and orders, whose report has two datasets, `$` and
+`$.customers[]`.
 
-From the repository root, regenerate both outputs with:
+From the repository root, regenerate the outputs with:
 
 ```console
 tabalyst report examples/input/basic.csv -o examples/output/basic/report.html --config examples/config.json --force
 tabalyst report examples/input/insurance-customers.csv -o examples/output/insurance-customers/report.html --config examples/config.json --force
+tabalyst report examples/input/orders.json -o examples/output/orders/report.html --config examples/config.json --force
 ```
 
 Every output folder is independent and contains its own HTML report, canonical

tabalyst-0.4.0/examples/config.json

@@ -0,0 +1,60 @@
+{
+  "string_analysis": {
+    "very_short_max_length": 5,
+    "short_max_length": 20,
+    "medium_max_length": 50,
+    "long_max_length": 255,
+    "length_distribution_max_length": 50,
+    "examples_per_length": 10
+  },
+  "value_examples": {
+    "full_distribution_max_distinct": 50,
+    "short_text_max_length": 20,
+    "short_text_percentile": 0.95,
+    "short_text_result_size": 20,
+    "long_text_result_size": 20,
+    "long_text_truncate_at": 30,
+    "truncation_suffix": "...",
+    "inline_display_size": 3
+  },
+  "scan": {
+    "csv": {
+      "encoding": "utf-8-sig",
+      "delimiter": ","
+    },
+    "values": {
+      "null_markers": ["N/A", "NULL"],
+      "missing": ["absent", "null", "empty", "blank", "marker"]
+    },
+    "normalization": {
+      "trim": true,
+      "collapse_whitespace": true
+    },
+    "types": {
+      "minimum_confidence": 0.95
+    },
+    "detectors": {
+      "date": {
+        "orders": ["YMD", "MDY", "DMY"],
+        "separators": ["-", "/", "."],
+        "ambiguous_order": null
+      },
+      "enumeration": {
+        "minimum_values": 500,
+        "maximum_distinct": 49,
+        "case_sensitive": true
+      }
+    },
+    "limits": {
+      "max_samples": 100
+    },
+    "records": {
+      "preview": 20,
+      "duplicates": true
+    },
+    "exposure": {
+      "sensitive_values": "mask"
+    },
+    "random_seed": 42
+  }
+}

{tabalyst-0.3.0 → tabalyst-0.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "tabalyst"
-version = "0.3.0"
+version = "0.4.0"
 description = "Open-source toolkit for understanding and working with unfamiliar data."
 readme = "README.md"
 requires-python = ">=3.11"
@@ -34,9 +34,9 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Information Analysis",
 ]
 dependencies = [
+    "ijson>=3.3",
     "jinja2>=3.1",
-    "pandas>=2.2",
-    "pydantic>=2.10",
+    "pydantic>=2.11",
     "typer>=0.12",
 ]
 
@@ -65,6 +65,9 @@ Issues = "https://github.com/loribel-labs/tabalyst/issues"
 [tool.pytest.ini_options]
 testpaths = ["tests"]
 pythonpath = ["src"]
+markers = [
+    "lot(name): Tabalyst Scan implementation lot that provides the tested behavior",
+]
 
 [tool.ruff]
 line-length = 88

{tabalyst-0.3.0 → tabalyst-0.4.0}/src/tabalyst/__init__.py

@@ -1,28 +1,34 @@
 """Public Python interface for Tabalyst."""
 
 from tabalyst._version import __version__
-from tabalyst.analysis import analyze_column as analyze_column
-from tabalyst.config import AnalysisConfig as AnalysisConfig
 from tabalyst.errors import ConfigurationError, InputError, ReportError, TabalystError
+from tabalyst.report_config import ReportConfig as ReportConfig
 from tabalyst.report_service import generate_reports as generate_reports
 from tabalyst.reporting import render_report as render_report
 from tabalyst.sampling import SampleMethod as SampleMethod
 from tabalyst.sampling import SampleResult as SampleResult
 from tabalyst.sampling import sample_csv as sample_csv
 from tabalyst.sampling_service import generate_samples as generate_samples
+from tabalyst.scan_service import generate_scans as generate_scans
+from tabalyst.scanner import ScanConfig as ScanConfig
+from tabalyst.scanner import ScanResult as ScanResult
+from tabalyst.scanner import scan as scan
 from tabalyst.service import analyze
 from tabalyst.service import analyze_csv as analyze_csv
+from tabalyst.service import analyze_scan as analyze_scan
 
 __all__ = [
     "ConfigurationError",
     "InputError",
     "ReportError",
+    "ScanConfig",
+    "ScanResult",
     "TabalystError",
     "__version__",
     "analyze",
     "generate_reports",
     "generate_samples",
+    "generate_scans",
     "sample_csv",
+    "scan",
 ]
-
-# Alpha compatibility imports remain available, but are not part of the 0.1.0 API.

tabalyst-0.4.0/src/tabalyst/batch.py

@@ -0,0 +1,157 @@
+"""Batch planning shared by the commands: inputs, outputs and safe writing.
+
+Every command resolves its inputs, maps each one to its outputs and rejects the
+whole batch before any work when the plan is unsafe: output name collisions,
+outputs that would replace an input, or existing outputs without ``force``.
+"""
+
+import glob
+import os
+import tempfile
+from collections.abc import Callable, Iterable, Sequence
+from pathlib import Path
+
+from tabalyst.errors import ConfigurationError, InputError, ReportError
+
+
+def path_key(path: Path) -> str:
+    """Identity of a path for comparisons, whatever its spelling and case."""
+    return os.path.normcase(str(path.resolve()))
+
+
+def resolve_input_specs(input_specs: Sequence[str | Path]) -> list[Path]:
+    """Resolve explicit files and shell-independent, non-recursive glob patterns."""
+    if not input_specs:
+        raise ConfigurationError("At least one input file is required.")
+
+    sources: list[Path] = []
+    known: set[str] = set()
+    for value in input_specs:
+        text = os.fspath(value)
+        if "**" in text:
+            raise ConfigurationError(
+                f"Recursive input patterns are not supported yet: {text}"
+            )
+        if glob.has_magic(text):
+            matches = sorted(
+                (Path(match) for match in glob.glob(text) if Path(match).is_file()),
+                key=path_key,
+            )
+            if not matches:
+                raise InputError(f'Input pattern matched no files: "{text}"')
+        else:
+            path = Path(text)
+            if not path.exists():
+                raise InputError(f"Input file does not exist: {path}")
+            if not path.is_file():
+                raise InputError(f"Input path is not a file: {path}")
+            matches = [path]
+
+        for path in matches:
+            key = path_key(path)
+            if key not in known:
+                sources.append(path)
+                known.add(key)
+    return sources
+
+
+def plan_outputs(
+    input_specs: Sequence[str | Path],
+    *,
+    output: str | Path | None,
+    output_dir: str | Path | None,
+    output_name: Callable[[Path], str],
+) -> list[tuple[Path, Path]]:
+    """Resolve the inputs and pair each one with its main output.
+
+    ``output`` names the output of a single input; otherwise each output is
+    named ``output_name(source)``, in ``output_dir`` or beside its source.
+    """
+    if output is not None and output_dir is not None:
+        raise ConfigurationError("--output and --output-dir cannot be used together.")
+    sources = resolve_input_specs(input_specs)
+    if output is not None and len(sources) != 1:
+        raise ConfigurationError("--output can only be used with one input file.")
+
+    destination = Path(output_dir) if output_dir is not None else None
+    if destination is not None and destination.exists() and not destination.is_dir():
+        raise ReportError(f"Output directory path is a file: {destination}")
+    return [
+        (
+            source,
+            Path(output)
+            if output is not None
+            else (destination if destination is not None else source.parent)
+            / output_name(source),
+        )
+        for source in sources
+    ]
+
+
+def validate_outputs(
+    artifacts: Iterable[tuple[Path, Path]],
+    *,
+    sources: Iterable[Path],
+    force: bool,
+    label: str,
+) -> None:
+    """Reject unsafe ``(source, output)`` pairs of a whole batch.
+
+    ``label`` names the outputs in messages, such as ``"Report"``.
+    """
+    input_keys = {path_key(source) for source in sources}
+    owners: dict[str, Path] = {}
+    existing: list[Path] = []
+    for source, artifact in artifacts:
+        if artifact.exists() and artifact.is_dir():
+            raise InputError(f"{label} path is a directory: {artifact}")
+        key = path_key(artifact)
+        if key in input_keys:
+            raise InputError(f"{label} would overwrite an input file: {artifact}")
+        previous = owners.get(key)
+        if previous is not None:
+            raise ReportError(
+                "Output name collision: "
+                f"{previous} and {source} both map to {artifact}."
+            )
+        owners[key] = source
+        if not force and artifact.exists():
+            existing.append(artifact)
+    if existing:
+        lines = "\n".join(f"  {path}" for path in existing)
+        raise ReportError(
+            f"{label} output already exists. Use --force to replace it:\n" + lines
+        )
+
+
+def apply_default_file_mode(path: Path) -> None:
+    """Give ``path`` the permissions of a plain write under the current umask.
+
+    Temporary files are created for their owner only, and ``os.replace`` keeps
+    that mode on the final file.
+    """
+    umask = os.umask(0)
+    os.umask(umask)
+    os.chmod(path, 0o666 & ~umask)
+
+
+def write_text_atomic(path: Path, content: str) -> None:
+    """Write through a temporary file in the target directory, then replace.
+
+    An interrupted write never leaves a partial file at ``path``.
+    """
+    path.parent.mkdir(parents=True, exist_ok=True)
+    descriptor, name = tempfile.mkstemp(
+        dir=path.parent, prefix=f".{path.name}.", suffix=".tmp"
+    )
+    temporary = Path(name)
+    try:
+        apply_default_file_mode(temporary)
+        with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as handle:
+            handle.write(content)
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.replace(temporary, path)
+    except BaseException:
+        temporary.unlink(missing_ok=True)
+        raise

{tabalyst-0.3.0 → tabalyst-0.4.0}/src/tabalyst/cli/app.py

@@ -7,6 +7,7 @@ import typer
 from tabalyst import __version__
 from tabalyst.cli.report import report_command
 from tabalyst.cli.sample import sample_command
+from tabalyst.cli.scan import scan_command
 
 app = typer.Typer(
     add_completion=False,
@@ -39,6 +40,7 @@ def root(
 
 app.command("report")(report_command)
 app.command("sample")(sample_command)
+app.command("scan")(scan_command)
 
 
 def main() -> None: