tabalyst 0.1.0__py3-none-any.whl

tabalyst/cli.py

@@ -0,0 +1,208 @@
+"""Thin command-line adapters over the public Tabalyst API."""
+
+import json
+import sys
+import tempfile
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from tabalyst import __version__
+from tabalyst.config import load_config
+from tabalyst.errors import TabalystError
+from tabalyst.execution_log import EXECUTION_LOG_NAME
+from tabalyst.reporting import load_profile, render_report
+from tabalyst.service import analyze as analyze_report
+
+PROJECT_CONFIG_NAME = "tabalyst.json"
+
+# The official 0.1.0 interface is a single command with two positional paths.
+official_app = typer.Typer(
+    add_completion=False,
+    help="Analyze CSV data and generate sibling JSON and HTML reports.",
+)
+
+# Keep the alpha subcommands available during the transition to the stable CLI.
+app = typer.Typer(
+    add_completion=False,
+    no_args_is_help=True,
+    help="Legacy alpha commands retained for compatibility.",
+)
+
+
+def analysis_config_paths(explicit: list[Path]) -> list[Path]:
+    """Put the current project's automatic config before explicit overrides."""
+    project_config = Path.cwd() / PROJECT_CONFIG_NAME
+    paths = [project_config] if project_config.is_file() else []
+    known = {path.resolve() for path in paths}
+    for path in explicit:
+        if path.resolve() not in known:
+            paths.append(path)
+            known.add(path.resolve())
+    return paths
+
+
+def check_outputs(inputs: list[Path], outputs: list[Path]) -> None:
+    """Protect legacy commands from overwriting inputs or aliased outputs."""
+    resolved = [path.resolve() for path in outputs]
+    if len(set(resolved)) != len(resolved):
+        raise ValueError("Output paths must be different.")
+    for output in outputs:
+        if output.resolve() in {path.resolve() for path in inputs}:
+            raise ValueError(f"Output would overwrite an input file: {output}")
+        if output.exists() and any(output.samefile(path) for path in inputs):
+            raise ValueError(f"Output would overwrite an input file: {output}")
+        if output.is_dir():
+            raise ValueError(f"Output is a directory: {output}")
+    if (
+        len(outputs) > 1
+        and all(path.exists() for path in outputs)
+        and outputs[0].samefile(outputs[1])
+    ):
+        raise ValueError("Outputs refer to the same file.")
+
+
+def write_output(path: Path, content: str) -> None:
+    """Write a UTF-8 text artifact for the legacy render command."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(content, encoding="utf-8", newline="\n")
+
+
+def _print_success(result: dict, report_path: Path) -> None:
+    summary = result["summary"]
+    typer.echo(
+        f"Analyzed {summary['row_count']:,} rows and "
+        f"{summary['column_count']} columns."
+    )
+    typer.echo(f"JSON: {report_path.with_suffix('.json').resolve()}")
+    typer.echo(f"HTML: {report_path.resolve()}")
+    execution_path = (report_path.parent / EXECUTION_LOG_NAME).resolve()
+    typer.echo(f"Execution history: {execution_path}")
+
+
+def _version_callback(value: bool) -> None:
+    if value:
+        typer.echo(__version__)
+        raise typer.Exit()
+
+
+@official_app.command()
+def run(
+    csv_path: Annotated[Path, typer.Argument(help="CSV source file.")],
+    report_path: Annotated[
+        Path, typer.Argument(help="Complete output filename ending in .html.")
+    ],
+    separator: Annotated[
+        str | None, typer.Option("--separator", help="One-character CSV separator.")
+    ] = None,
+    encoding: Annotated[
+        str | None, typer.Option("--encoding", help="CSV text encoding.")
+    ] = None,
+    config: Annotated[
+        Path | None, typer.Option("--config", help="JSON configuration file.")
+    ] = None,
+    show_version: Annotated[
+        bool,
+        typer.Option(
+            "--version",
+            help="Show the installed Tabalyst version and exit.",
+            is_eager=True,
+            callback=_version_callback,
+        ),
+    ] = False,
+) -> None:
+    """Analyze CSV_PATH into REPORT_PATH and its sibling JSON profile."""
+    try:
+        result = analyze_report(
+            csv_path,
+            report_path,
+            separator=separator,
+            encoding=encoding,
+            config_path=config,
+        )
+    except TabalystError as exc:
+        typer.echo(f"Error: {exc}", err=True)
+        raise typer.Exit(1) from exc
+    _print_success(result, report_path)
+
+
+@app.command("analyze")
+def analyze_legacy(
+    source: Annotated[Path, typer.Argument(exists=True, dir_okay=False, readable=True)],
+    output: Annotated[Path, typer.Option("--output", "-o")] = Path("report.html"),
+    delimiter: Annotated[
+        str | None, typer.Option("--delimiter", "--separator")
+    ] = None,
+    encoding: Annotated[str | None, typer.Option("--encoding")] = None,
+    preview_rows: Annotated[
+        int | None, typer.Option("--preview-rows", min=0, max=100)
+    ] = None,
+    config: Annotated[
+        list[Path] | None, typer.Option("--config", exists=True, dir_okay=False)
+    ] = None,
+) -> None:
+    """Run the alpha command and retain its cumulative execution history."""
+    json_output = output.with_suffix(".json")
+    execution_output = output.parent / EXECUTION_LOG_NAME
+    temporary_config: Path | None = None
+    try:
+        config_paths = analysis_config_paths(config or [])
+        check_outputs(
+            [source, *config_paths], [output, json_output, execution_output]
+        )
+        effective_config = config_paths
+        if preview_rows is not None:
+            settings = load_config(config_paths).model_dump()
+            settings["preview_rows"] = preview_rows
+            with tempfile.NamedTemporaryFile(
+                "w", encoding="utf-8", suffix=".json", delete=False
+            ) as stream:
+                json.dump(settings, stream, indent=2, ensure_ascii=False)
+                stream.write("\n")
+                temporary_config = Path(stream.name)
+            effective_config = [temporary_config]
+
+        result = analyze_report(
+            source,
+            output,
+            separator=delimiter,
+            encoding=encoding,
+            config_path=effective_config,
+        )
+    except (TabalystError, OSError, ValueError, LookupError) as exc:
+        typer.echo(f"Error: {exc}", err=True)
+        raise typer.Exit(1) from exc
+    finally:
+        if temporary_config is not None:
+            temporary_config.unlink(missing_ok=True)
+
+    _print_success(result, output)
+    if config_paths:
+        typer.echo(
+            "Configuration: "
+            + ", ".join(str(path.resolve()) for path in config_paths)
+        )
+
+
+@app.command()
+def render(
+    source: Annotated[Path, typer.Argument(exists=True, dir_okay=False, readable=True)],
+    output: Annotated[Path, typer.Option("--output", "-o")] = Path("report.html"),
+) -> None:
+    """Regenerate HTML from a report JSON profile without rereading the CSV."""
+    try:
+        check_outputs([source], [output])
+        write_output(output, render_report(load_profile(source)))
+    except (OSError, ValueError) as exc:
+        typer.echo(f"Error: {exc}", err=True)
+        raise typer.Exit(1) from exc
+    typer.echo(f"HTML: {output.resolve()}")
+
+
+def main() -> None:
+    """Dispatch stable syntax while preserving the two alpha subcommands."""
+    if len(sys.argv) > 1 and sys.argv[1] in {"analyze", "render"}:
+        app()
+    else:
+        official_app()

tabalyst/config.py

@@ -0,0 +1,214 @@
+"""Experimental configuration. Later files override earlier settings."""
+
+import codecs
+from collections.abc import Iterable
+from itertools import pairwise
+from pathlib import Path
+from typing import Literal
+
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    ValidationError,
+    field_validator,
+    model_validator,
+)
+
+from tabalyst.errors import ConfigurationError
+
+
+class CsvConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    encoding: str = "utf-8-sig"
+    delimiter: str = ","
+
+    @field_validator("encoding")
+    @classmethod
+    def known_encoding(cls, value: str) -> str:
+        try:
+            codecs.lookup(value)
+        except LookupError as exc:
+            raise ValueError(f"Unknown encoding: {value}") from exc
+        return value
+
+    @field_validator("delimiter")
+    @classmethod
+    def single_separator(cls, value: str) -> str:
+        if len(value) != 1 or value in '\r\n"\0':
+            raise ValueError(
+                "Delimiter must be one character, excluding quotes/NUL/newlines"
+            )
+        return value
+
+
+class ValueExamplesConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    full_distribution_max_distinct: int = Field(default=50, ge=1)
+    candidate_sample_size: int = Field(default=100, ge=1)
+    short_text_max_length: int = Field(default=20, ge=1)
+    short_text_percentile: float = Field(default=0.95, gt=0, le=1)
+    short_text_result_size: int = Field(default=20, ge=1)
+    long_text_result_size: int = Field(default=20, ge=1)
+    long_text_truncate_at: int = Field(default=30, ge=1)
+    truncation_suffix: str = "..."
+    inline_display_size: int = Field(default=3, ge=1)
+    random_seed: int = 42
+
+
+class NormalizationConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    trim: bool = True
+    collapse_internal_whitespace: bool = True
+
+
+class DateDetectionConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    enabled: bool = True
+    orders: list[Literal["YMD", "MDY", "DMY"]] = Field(
+        default_factory=lambda: ["YMD", "MDY", "DMY"]
+    )
+    separators: list[str] = Field(default_factory=lambda: ["-", "/", "."])
+    ambiguous_order: Literal["MDY", "DMY"] | None = None
+
+    @field_validator("separators")
+    @classmethod
+    def valid_separators(cls, values: list[str]) -> list[str]:
+        if not values or any(
+            len(value) != 1 or value.isdigit() or value in "\r\n" for value in values
+        ):
+            raise ValueError("Date separators must be non-digit single characters")
+        if len(set(values)) != len(values):
+            raise ValueError("Date separators must be unique")
+        return values
+
+    @model_validator(mode="after")
+    def ambiguous_order_must_be_enabled(self):
+        if self.ambiguous_order and self.ambiguous_order not in self.orders:
+            raise ValueError("ambiguous_order must also appear in date orders")
+        return self
+
+
+class TypeInferenceConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    minimum_confidence: float = Field(default=0.95, gt=0.5, le=1)
+
+
+class StringAnalysisConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    very_short_max_length: int = Field(default=5, ge=1)
+    short_max_length: int = Field(default=20, ge=1)
+    medium_max_length: int = Field(default=50, ge=1)
+    long_max_length: int = Field(default=255, ge=1)
+    length_distribution_max_length: int = Field(default=50, ge=1)
+    examples_per_length: int = Field(default=10, ge=1)
+
+    @model_validator(mode="after")
+    def length_thresholds_must_increase(self):
+        thresholds = [
+            self.very_short_max_length,
+            self.short_max_length,
+            self.medium_max_length,
+            self.long_max_length,
+        ]
+        if any(left >= right for left, right in pairwise(thresholds)):
+            raise ValueError("String length thresholds must increase")
+        if self.length_distribution_max_length > self.medium_max_length:
+            raise ValueError(
+                "length_distribution_max_length cannot exceed medium_max_length"
+            )
+        return self
+
+
+class EnumDetectionConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    enabled: bool = True
+    minimum_row_count: int = Field(default=500, ge=1)
+    maximum_distinct_values: int = Field(default=49, ge=1)
+    eligible_types: list[str] = Field(default_factory=lambda: ["text"])
+    case_sensitive: bool = True
+
+
+class AnalysisConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    csv: CsvConfig = Field(default_factory=CsvConfig)
+    missing_values: list[str] = Field(default_factory=lambda: [""])
+    preview_rows: int = Field(default=10, ge=0, le=100)
+    normalization: NormalizationConfig = Field(default_factory=NormalizationConfig)
+    date_detection: DateDetectionConfig = Field(default_factory=DateDetectionConfig)
+    type_inference: TypeInferenceConfig = Field(default_factory=TypeInferenceConfig)
+    string_analysis: StringAnalysisConfig = Field(default_factory=StringAnalysisConfig)
+    value_examples: ValueExamplesConfig = Field(default_factory=ValueExamplesConfig)
+    enum_detection: EnumDetectionConfig = Field(default_factory=EnumDetectionConfig)
+
+
+def _validation_message(exc: ValidationError) -> str:
+    first = exc.errors(include_url=False)[0]
+    location = ".".join(str(part) for part in first["loc"])
+    message = first["msg"]
+    return f"{location}: {message}" if location else message
+
+
+def load_config(paths: Iterable[Path]) -> AnalysisConfig:
+    """Load and recursively merge strict JSON configuration files."""
+
+    def merge(base: dict, override: dict) -> dict:
+        result = dict(base)
+        for key, value in override.items():
+            if isinstance(value, dict) and isinstance(result.get(key), dict):
+                result[key] = merge(result[key], value)
+            else:
+                result[key] = value
+        return result
+
+    merged = {}
+    for path in paths:
+        try:
+            content = path.read_text(encoding="utf-8-sig")
+        except OSError as exc:
+            raise ConfigurationError(
+                f"Cannot read configuration file {path}: {exc}"
+            ) from exc
+        try:
+            current = AnalysisConfig.model_validate_json(content).model_dump(
+                exclude_unset=True
+            )
+        except ValidationError as exc:
+            raise ConfigurationError(
+                f"Invalid configuration in {path}: {_validation_message(exc)}"
+            ) from exc
+        merged = merge(merged, current)
+    try:
+        return AnalysisConfig.model_validate(merged)
+    except ValidationError as exc:
+        raise ConfigurationError(
+            f"Invalid merged configuration: {_validation_message(exc)}"
+        ) from exc
+
+
+def resolve_config(
+    paths: Iterable[Path] = (),
+    *,
+    separator: str | None = None,
+    encoding: str | None = None,
+) -> AnalysisConfig:
+    """Resolve defaults, files and explicit values in one shared layer."""
+    settings = load_config(paths).model_dump()
+    if separator is not None:
+        settings["csv"]["delimiter"] = separator
+    if encoding is not None:
+        settings["csv"]["encoding"] = encoding
+    try:
+        return AnalysisConfig.model_validate(settings)
+    except ValidationError as exc:
+        raise ConfigurationError(
+            f"Invalid configuration: {_validation_message(exc)}"
+        ) from exc

tabalyst/errors.py

@@ -0,0 +1,17 @@
+"""Public exceptions raised by Tabalyst."""
+
+
+class TabalystError(Exception):
+    """Base class for expected Tabalyst failures."""
+
+
+class InputError(TabalystError, ValueError):
+    """The CSV input or requested report path is invalid."""
+
+
+class ConfigurationError(TabalystError, ValueError):
+    """A configuration file or explicit configuration value is invalid."""
+
+
+class ReportError(TabalystError):
+    """Tabalyst could not write or render the requested report."""

tabalyst/execution_log.py

@@ -0,0 +1,102 @@
+"""Persistent performance history for successful CLI analyses."""
+
+import json
+import subprocess
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from tabalyst._version import get_version
+from tabalyst.models import DatasetProfile
+
+EXECUTION_LOG_NAME = "executions.json"
+EXECUTION_SCHEMA_VERSION = "1.0"
+
+
+def tabalyst_version() -> str:
+    """Return the package version for execution-history compatibility."""
+    return get_version()
+
+
+def git_state(directory: Path) -> dict[str, Any]:
+    """Describe the current checkout without requiring Git in production."""
+    try:
+        commit = subprocess.run(
+            ["git", "rev-parse", "HEAD"],
+            cwd=directory,
+            check=True,
+            capture_output=True,
+            text=True,
+            timeout=2,
+        ).stdout.strip()
+        dirty = bool(
+            subprocess.run(
+                ["git", "status", "--porcelain"],
+                cwd=directory,
+                check=True,
+                capture_output=True,
+                text=True,
+                timeout=2,
+            ).stdout.strip()
+        )
+    except (OSError, subprocess.SubprocessError):
+        return {"available": False, "commit": None, "dirty": None, "state": None}
+    suffix = "+working" if dirty else ""
+    return {
+        "available": True,
+        "commit": commit,
+        "dirty": dirty,
+        "state": f"{commit[:8]}{suffix}",
+    }
+
+
+def build_execution_entry(
+    *,
+    source: Path,
+    html_output: Path,
+    json_output: Path,
+    profile: DatasetProfile,
+    total_seconds: float,
+    git_directory: Path,
+) -> dict[str, Any]:
+    """Build one portable history item using names relative to its output folder."""
+    return {
+        "timestamp": datetime.now(UTC).isoformat().replace("+00:00", "Z"),
+        "tabalyst_version": tabalyst_version(),
+        "source_file": source.name,
+        "html_file": html_output.name,
+        "json_file": json_output.name,
+        "rows": profile.summary.row_count,
+        "columns": profile.summary.column_count,
+        "analysis_seconds": profile.processing_seconds,
+        "total_seconds": round(total_seconds, 4),
+        "git": git_state(git_directory),
+    }
+
+
+def append_execution(path: Path, entry: dict[str, Any]) -> None:
+    """Append an item and atomically replace the JSON history file."""
+    path.parent.mkdir(parents=True, exist_ok=True)
+    if path.exists():
+        history = json.loads(path.read_text(encoding="utf-8"))
+        if not isinstance(history, dict) or not isinstance(
+            history.get("executions"), list
+        ):
+            raise ValueError(f"Invalid execution history: {path}")
+        if history.get("schema_version") != EXECUTION_SCHEMA_VERSION:
+            raise ValueError(
+                f"Unsupported execution history schema: {path}"
+            )
+    else:
+        history = {
+            "schema_version": EXECUTION_SCHEMA_VERSION,
+            "executions": [],
+        }
+    history["executions"].append(entry)
+    temporary = path.with_name(f".{path.name}.tmp")
+    temporary.write_text(
+        json.dumps(history, indent=2, ensure_ascii=False) + "\n",
+        encoding="utf-8",
+        newline="\n",
+    )
+    temporary.replace(path)

tabalyst/ingestion.py

@@ -0,0 +1,86 @@
+"""Validate CSV structure before asking pandas to load the raw strings."""
+
+import csv
+import hashlib
+import io
+from dataclasses import dataclass
+from pathlib import Path
+
+import pandas as pd
+
+from tabalyst.config import CsvConfig
+from tabalyst.errors import InputError
+from tabalyst.models import SourceInfo
+
+
+class CsvInputError(InputError):
+    pass
+
+
+@dataclass
+class CsvDataset:
+    frame: pd.DataFrame
+    headers: list[str]
+    source: SourceInfo
+
+
+def read_csv(path: Path, config: CsvConfig) -> CsvDataset:
+    raw = path.read_bytes()
+    try:
+        content = raw.decode(config.encoding)
+    except UnicodeError as exc:
+        raise CsvInputError(
+            f"Cannot decode {path.name} as {config.encoding}. Specify --encoding."
+        ) from exc
+    if "\0" in content:
+        raise CsvInputError("CSV contains NUL characters; verify the file encoding.")
+
+    reader = csv.reader(
+        io.StringIO(content, newline=""), delimiter=config.delimiter, strict=True
+    )
+    row_count = 0
+    try:
+        headers = next(reader, None)
+        if not headers:
+            raise CsvInputError("CSV is empty or has no header on its first record.")
+        for row_count, row in enumerate(reader, start=1):
+            if len(row) != len(headers):
+                raise CsvInputError(
+                    f"Data record {row_count} (ending at physical line {reader.line_num}): "
+                    f"expected {len(headers)} fields, found {len(row)}. "
+                    "Verify the delimiter and quoting. No rows were skipped."
+                )
+    except csv.Error as exc:
+        raise CsvInputError(
+            f"Invalid CSV near physical line {reader.line_num}: {exc}"
+        ) from exc
+
+    # Internal IDs preserve duplicate/blank headers without pandas renaming them.
+    ids = [f"column_{position}" for position in range(1, len(headers) + 1)]
+    frame = pd.read_csv(
+        io.StringIO(content, newline=""),
+        sep=config.delimiter,
+        engine="python",
+        header=0,
+        names=ids,
+        dtype=str,
+        na_filter=False,
+        keep_default_na=False,
+        skip_blank_lines=False,
+        on_bad_lines="error",
+    )
+    if len(frame) != row_count:
+        raise CsvInputError(
+            "CSV record count changed during parsing; analysis was stopped."
+        )
+    return CsvDataset(
+        frame=frame,
+        headers=headers,
+        source=SourceInfo(
+            filename=path.name,
+            size_bytes=len(raw),
+            sha256=hashlib.sha256(raw).hexdigest(),
+            encoding=config.encoding,
+            delimiter=config.delimiter,
+        ),
+    )