tabcaddy 0.1.0__py3-none-any.whl

tabcaddy/__init__.py

@@ -0,0 +1,3 @@
+from tabcaddy.cli.app import app
+
+__all__ = ["app"]

tabcaddy/__main__.py

@@ -0,0 +1,5 @@
+from tabcaddy.cli.app import main
+
+
+if __name__ == "__main__":
+    main()

tabcaddy/application/compile_dataset.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from tabcaddy.domain.models import DatasetSource, ProfileMode, SourceType
+from tabcaddy.domain.serialization import analysis_to_dict
+from tabcaddy.infrastructure.analysis_builder import AnalysisBuilder
+from tabcaddy.infrastructure.csv_reader import read_csv
+from tabcaddy.infrastructure.feather_reader import read_feather
+from tabcaddy.infrastructure.parquet_dataset_writer import write_parquet_dataset
+
+
+def _read_dataframe(path: Path):
+    if path.suffix.lower() == ".csv":
+        return read_csv(path)
+    return read_feather(path)
+
+
+class CompileDataset:
+    def __init__(self, analysis_builder: AnalysisBuilder | None = None) -> None:
+        self._analysis_builder = analysis_builder or AnalysisBuilder()
+
+    def run(
+        self, source: DatasetSource, output_path: Path, schema_index: int | None = None
+    ) -> tuple[Path, list[str]]:
+        if source.source_type != SourceType.FOLDER:
+            raise ValueError("Compile expects a folder source.")
+        build_result = self._analysis_builder.build(source, ProfileMode.STANDARD)
+        schemas = build_result.analysis.schemas
+        if not schemas:
+            raise ValueError("No schemas found to compile.")
+        if len(schemas) > 1 and schema_index is None:
+            labels = [
+                f"Schema {index} ({schema.occurrence_count} files)"
+                for index, schema in enumerate(schemas, start=1)
+            ]
+            raise ValueError(
+                "Multiple schemas detected. Re-run with --schema. Available: "
+                + ", ".join(labels)
+            )
+        chosen_index = schema_index or 1
+        if chosen_index < 1 or chosen_index > len(schemas):
+            raise ValueError(f"Schema index must be between 1 and {len(schemas)}")
+        selected_schema = schemas[chosen_index - 1]
+        selected_files = [
+            record.path
+            for record in build_result.files
+            if record.schema_hash == selected_schema.hash
+        ]
+        output_path.mkdir(parents=True, exist_ok=True)
+        written = write_parquet_dataset(
+            (_read_dataframe(path) for path in selected_files), output_path
+        )
+        selected_analysis = self._analysis_builder.build_file_set(
+            files=selected_files,
+            base_path=source.path,
+            source_type=SourceType.FOLDER,
+            profile_mode=ProfileMode.DEEP,
+        ).analysis
+        payload = analysis_to_dict(selected_analysis)
+        payload["compiled"] = {
+            "source": str(source.path),
+            "selected_schema_hash": selected_schema.hash,
+            "written_parts": [str(path.relative_to(output_path)) for path in written],
+        }
+        (output_path / "metadata.json").write_text(
+            json.dumps(payload, indent=2), encoding="utf-8"
+        )
+        skipped = [
+            record.relative_path.as_posix()
+            for record in build_result.files
+            if record.schema_hash != selected_schema.hash
+        ]
+        return output_path, skipped

tabcaddy/application/diff_datasets.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from tabcaddy.domain.models import DatasetSource, DiffLevel, DiffReport, SourceType
+from tabcaddy.infrastructure.compiled_dataset_differ import CompiledDatasetDiffer
+from tabcaddy.infrastructure.file_differ import FileDiffer
+from tabcaddy.infrastructure.folder_differ import FolderDiffer
+
+
+class DiffDatasets:
+    def __init__(self, generate_analysis) -> None:
+        self._file_differ = FileDiffer(generate_analysis)
+        self._folder_differ = FolderDiffer(generate_analysis)
+        self._compiled_differ = CompiledDatasetDiffer(generate_analysis)
+
+    def run(
+        self, left: DatasetSource, right: DatasetSource, level: DiffLevel
+    ) -> DiffReport:
+        if left.source_type == right.source_type == SourceType.FILE:
+            return self._file_differ.diff(left, right, level)
+        if left.source_type == right.source_type == SourceType.FOLDER:
+            return self._folder_differ.diff(left, right, level)
+        if left.source_type == right.source_type == SourceType.COMPILED_DATASET:
+            return self._compiled_differ.diff(left, right, level)
+        return self._file_differ.diff(left, right, level)

tabcaddy/application/generate_analysis.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from tabcaddy.domain.models import (
+    DatasetAnalysis,
+    DatasetSource,
+    ProfileMode,
+    SourceType,
+)
+from tabcaddy.infrastructure.analysis_builder import AnalysisBuilder
+from tabcaddy.infrastructure.cache_manager import CacheManager
+
+
+class GenerateAnalysis:
+    def __init__(
+        self,
+        analysis_builder: AnalysisBuilder | None = None,
+        cache_manager: CacheManager | None = None,
+    ) -> None:
+        self._analysis_builder = analysis_builder or AnalysisBuilder()
+        self._cache_manager = cache_manager or CacheManager()
+
+    def run(self, source: DatasetSource, profile_mode: ProfileMode) -> DatasetAnalysis:
+        if source.source_type == SourceType.COMPILED_DATASET:
+            compiled = self._analysis_builder.load_compiled_analysis(source)
+            if compiled is not None:
+                return compiled
+        cached = self._cache_manager.get(source, profile_mode)
+        if cached is not None:
+            return cached
+        analysis = self._analysis_builder.build(source, profile_mode).analysis
+        self._cache_manager.set(source, profile_mode, analysis)
+        return analysis

tabcaddy/application/scaffold_transform.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from tabcaddy.domain.models import DatasetSource, ProfileMode
+from tabcaddy.infrastructure.analysis_builder import AnalysisBuilder
+
+
+class ScaffoldTransform:
+    def __init__(self, analysis_builder: AnalysisBuilder | None = None) -> None:
+        self._analysis_builder = analysis_builder or AnalysisBuilder()
+
+    def run(self, source: DatasetSource, output_path: Path) -> Path:
+        analysis = self._analysis_builder.build(source, ProfileMode.STANDARD).analysis
+        lines = [
+            '"""TabCaddy transform scaffold."""',
+            "",
+            "import polars as pl",
+            "",
+            "",
+            "# Observed schemas",
+        ]
+        for index, schema in enumerate(analysis.schemas, start=1):
+            lines.append(
+                f"# Schema {index}: {schema.occurrence_count} files, hash={schema.hash}"
+            )
+            for column in schema.columns:
+                lines.append(f"#   - {column.name}: {column.dtype}")
+        lines.extend(
+            [
+                "",
+                "def transform(df: pl.DataFrame, context=None) -> pl.DataFrame:",
+                "    # Example: rename a column",
+                "    # if 'old_name' in df.columns:",
+                "    #     df = df.rename({'old_name': 'new_name'})",
+                "",
+                "    # Example: filter rows",
+                "    # df = df.filter(pl.col('quantity') > 0)",
+                "",
+                "    return df",
+                "",
+            ]
+        )
+        output_path.write_text("\n".join(lines), encoding="utf-8")
+        return output_path

tabcaddy/application/transform_dataset.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+import polars as pl
+
+from tabcaddy.domain.models import DatasetSource, SourceType
+from tabcaddy.infrastructure.csv_reader import read_csv
+from tabcaddy.infrastructure.csv_writer import write_csv
+from tabcaddy.infrastructure.feather_reader import read_feather
+from tabcaddy.infrastructure.feather_writer import write_feather
+from tabcaddy.infrastructure.schema_analyzer import SchemaAnalyzer
+from tabcaddy.infrastructure.source_resolver import iter_dataset_files
+from tabcaddy.infrastructure.transform_loader import (
+    TransformContext,
+    TransformLoader,
+    TransformMetadata,
+)
+
+
+def _read_dataframe(path: Path) -> pl.DataFrame:
+    if path.suffix.lower() == ".csv":
+        return read_csv(path)
+    return read_feather(path)
+
+
+def _write_dataframe(df: pl.DataFrame, path: Path) -> None:
+    if path.suffix.lower() == ".csv":
+        write_csv(df, path)
+        return
+    write_feather(df, path)
+
+
+class TransformDataset:
+    def __init__(
+        self,
+        transform_loader: TransformLoader | None = None,
+        schema_analyzer: SchemaAnalyzer | None = None,
+    ) -> None:
+        self._transform_loader = transform_loader or TransformLoader()
+        self._schema_analyzer = schema_analyzer or SchemaAnalyzer()
+
+    def run(
+        self,
+        source: DatasetSource,
+        transform_path: Path,
+        output_path: Path | None,
+        workers: int,
+    ) -> Path:
+        if source.source_type == SourceType.COMPILED_DATASET:
+            raise ValueError(
+                "Transform currently supports files and folders, not compiled datasets."
+            )
+        output_root = output_path or self._default_output_path(source.path)
+        output_root.mkdir(parents=True, exist_ok=True)
+        transform, expects_context = self._transform_loader.load(transform_path)
+        files = iter_dataset_files(source)
+        schema_result = self._schema_analyzer.analyze_files(
+            files, base_path=source.path, source_type=source.source_type
+        )
+        record_map = {record.path: record for record in schema_result.files}
+
+        def process(path: Path) -> None:
+            record = record_map[path]
+            df = _read_dataframe(path)
+            context = TransformContext(
+                file_name=path.name,
+                file_path=str(path),
+                schema=[
+                    {"name": column.name, "dtype": column.dtype}
+                    for column in record.columns
+                ],
+                metadata=TransformMetadata(
+                    row_count=record.row_count, schema_hash=record.schema_hash
+                ),
+            )
+            result = transform(df, context) if expects_context else transform(df)
+            if not isinstance(result, pl.DataFrame):
+                raise TypeError(
+                    f"Transform must return a Polars DataFrame for {path.name}"
+                )
+            relative_path = (
+                record.relative_path
+                if source.source_type != SourceType.FILE
+                else Path(path.name)
+            )
+            _write_dataframe(result, output_root / relative_path)
+
+        if workers <= 1:
+            for path in files:
+                process(path)
+        else:
+            with ThreadPoolExecutor(max_workers=workers) as executor:
+                list(executor.map(process, files))
+        return output_root
+
+    def _default_output_path(self, input_path: Path) -> Path:
+        if input_path.is_dir():
+            return input_path.parent / f"{input_path.name}_transformed"
+        return input_path.parent / f"{input_path.stem}_transformed"

tabcaddy/cli/app.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+
+from tabcaddy.application.compile_dataset import CompileDataset
+from tabcaddy.application.diff_datasets import DiffDatasets
+from tabcaddy.application.generate_analysis import GenerateAnalysis
+from tabcaddy.application.scaffold_transform import ScaffoldTransform
+from tabcaddy.application.transform_dataset import TransformDataset
+from tabcaddy.domain.models import DiffLevel
+from tabcaddy.domain.models import ProfileMode
+from tabcaddy.infrastructure.analysis_builder import AnalysisBuilder
+from tabcaddy.infrastructure.source_resolver import resolve_source
+from tabcaddy.rendering.console import create_console
+from tabcaddy.rendering.views.diff import build_diff_view
+from tabcaddy.rendering.views.schema import build_schema_view
+from tabcaddy.rendering.views.summary import build_summary_view
+
+
+app = typer.Typer(
+    add_completion=False,
+    help="Explore, compile, transform, and compare datasets.",
+    no_args_is_help=True,
+)
+
+
+@app.callback()
+def root() -> None:
+    """TabCaddy command line interface."""
+
+
+@app.command()
+def summary(
+    source: Path,
+    profile: ProfileMode = typer.Option(ProfileMode.STANDARD, "--profile"),
+) -> None:
+    console = create_console()
+    analysis = GenerateAnalysis().run(resolve_source(source), profile)
+    console.print(build_summary_view(analysis))
+
+
+@app.command()
+def schema(
+    source: Path,
+    profile: ProfileMode = typer.Option(ProfileMode.STANDARD, "--profile"),
+) -> None:
+    console = create_console()
+    scan = AnalysisBuilder().build(resolve_source(source), profile)
+    console.print(build_schema_view(scan.analysis, scan.files))
+
+
+@app.command()
+def compile(
+    folder: Path,
+    output: Path = typer.Option(Path("compiled_dataset"), "--output"),
+    schema: int | None = typer.Option(None, "--schema"),
+    interactive: bool = typer.Option(False, "--interactive"),
+) -> None:
+    console = create_console()
+    source = resolve_source(folder)
+    schema_index = schema
+    if interactive and schema_index is None:
+        preview = AnalysisBuilder().build(source, ProfileMode.QUICK)
+        if len(preview.analysis.schemas) > 1:
+            schema_index = typer.prompt(
+                "Multiple schemas detected. Choose schema number", type=int
+            )
+    output_path, skipped = CompileDataset().run(source, output, schema_index)
+    console.print(f"Compiled dataset written to [green]{output_path}[/green]")
+    if skipped:
+        console.print(
+            f"Skipped {len(skipped)} files from non-selected schemas.", style="yellow"
+        )
+
+
+@app.command()
+def transform(
+    input_path: Path,
+    transform_path: Path,
+    output_path: Path | None = typer.Argument(None),
+    workers: int = typer.Option(1, "--workers", min=1),
+) -> None:
+    console = create_console()
+    source = resolve_source(input_path)
+    destination = TransformDataset().run(source, transform_path, output_path, workers)
+    console.print(f"Transformed files written to [green]{destination}[/green]")
+
+
+@app.command("scaffold-transform")
+def scaffold_transform(
+    source: Path,
+    output: Path = typer.Option(Path("transform_template.py"), "--output"),
+) -> None:
+    console = create_console()
+    destination = ScaffoldTransform().run(resolve_source(source), output)
+    console.print(f"Transform scaffold written to [green]{destination}[/green]")
+
+
+@app.command()
+def diff(
+    left: Path,
+    right: Path,
+    level: DiffLevel = typer.Option(DiffLevel.FULL, "--level"),
+) -> None:
+    console = create_console()
+    generator = GenerateAnalysis()
+    report = DiffDatasets(generator).run(
+        resolve_source(left), resolve_source(right), level
+    )
+    console.print(build_diff_view(report))
+
+
+def main() -> None:
+    app()

tabcaddy/domain/__init__.py

@@ -0,0 +1,25 @@
+from tabcaddy.domain.models import ColumnDefinition
+from tabcaddy.domain.models import ColumnStatistics
+from tabcaddy.domain.models import DatasetAnalysis
+from tabcaddy.domain.models import DatasetMetadata
+from tabcaddy.domain.models import DatasetSource
+from tabcaddy.domain.models import DatasetStatistics
+from tabcaddy.domain.models import DiffLevel
+from tabcaddy.domain.models import DiffReport
+from tabcaddy.domain.models import ProfileMode
+from tabcaddy.domain.models import SchemaSignature
+from tabcaddy.domain.models import SourceType
+
+__all__ = [
+    "ColumnDefinition",
+    "ColumnStatistics",
+    "DatasetAnalysis",
+    "DatasetMetadata",
+    "DatasetSource",
+    "DatasetStatistics",
+    "DiffLevel",
+    "DiffReport",
+    "ProfileMode",
+    "SchemaSignature",
+    "SourceType",
+]

tabcaddy/domain/models.py

@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from pathlib import Path
+from typing import Any
+
+
+class SourceType(str, Enum):
+    FILE = "file"
+    FOLDER = "folder"
+    COMPILED_DATASET = "compiled_dataset"
+
+
+class ProfileMode(str, Enum):
+    QUICK = "quick"
+    STANDARD = "standard"
+    DEEP = "deep"
+
+
+class DiffLevel(str, Enum):
+    METADATA = "metadata"
+    STATISTICS = "statistics"
+    FULL = "full"
+
+
+@dataclass(frozen=True)
+class DatasetSource:
+    path: Path
+    source_type: SourceType
+
+
+@dataclass(frozen=True)
+class ColumnDefinition:
+    name: str
+    dtype: str
+
+
+@dataclass
+class SchemaSignature:
+    columns: list[ColumnDefinition]
+    hash: str
+    occurrence_count: int
+
+
+@dataclass
+class DatasetMetadata:
+    version: int
+    created_at: datetime
+    row_count: int
+    column_count: int
+    source_file_count: int
+    schema_hash: str | None
+    column_hashes: dict[str, str] | None
+
+
+@dataclass
+class ColumnStatistics:
+    dtype: str
+    null_rate: float
+    unique_estimate: int | None
+    min_value: Any | None
+    max_value: Any | None
+    mean: float | None
+    median: float | None
+    stddev: float | None
+    histogram: list[tuple[str, int]] | None = None
+
+
+@dataclass
+class DatasetStatistics:
+    columns: dict[str, ColumnStatistics]
+
+
+@dataclass
+class DatasetAnalysis:
+    metadata: DatasetMetadata
+    schemas: list[SchemaSignature]
+    statistics: DatasetStatistics | None
+    warnings: list[str] = field(default_factory=list)
+
+
+@dataclass
+class DiffReport:
+    metadata_changes: list[str]
+    schema_changes: list[str]
+    statistics_changes: list[str]
+    warnings: list[str] = field(default_factory=list)

tabcaddy/domain/serialization.py

@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+from datetime import date, datetime, time
+from typing import Any
+
+from tabcaddy.domain.models import (
+    ColumnDefinition,
+    ColumnStatistics,
+    DatasetAnalysis,
+    DatasetMetadata,
+    DatasetStatistics,
+    DiffReport,
+    SchemaSignature,
+)
+
+
+def _serialize_value(value: Any) -> Any:
+    if isinstance(value, (datetime, date, time)):
+        return value.isoformat()
+    if isinstance(value, list):
+        return [_serialize_value(item) for item in value]
+    if isinstance(value, dict):
+        return {key: _serialize_value(item) for key, item in value.items()}
+    return value
+
+
+def analysis_to_dict(analysis: DatasetAnalysis) -> dict[str, Any]:
+    return {
+        "metadata": {
+            "version": analysis.metadata.version,
+            "created_at": analysis.metadata.created_at.isoformat(),
+            "row_count": analysis.metadata.row_count,
+            "column_count": analysis.metadata.column_count,
+            "source_file_count": analysis.metadata.source_file_count,
+            "schema_hash": analysis.metadata.schema_hash,
+            "column_hashes": analysis.metadata.column_hashes,
+        },
+        "schemas": [
+            {
+                "columns": [
+                    {"name": column.name, "dtype": column.dtype}
+                    for column in schema.columns
+                ],
+                "hash": schema.hash,
+                "occurrence_count": schema.occurrence_count,
+            }
+            for schema in analysis.schemas
+        ],
+        "statistics": None
+        if analysis.statistics is None
+        else {
+            "columns": {
+                name: {
+                    "dtype": stats.dtype,
+                    "null_rate": stats.null_rate,
+                    "unique_estimate": stats.unique_estimate,
+                    "min_value": _serialize_value(stats.min_value),
+                    "max_value": _serialize_value(stats.max_value),
+                    "mean": stats.mean,
+                    "median": stats.median,
+                    "stddev": stats.stddev,
+                    "histogram": None
+                    if stats.histogram is None
+                    else [
+                        {"label": label, "count": count}
+                        for label, count in stats.histogram
+                    ],
+                }
+                for name, stats in analysis.statistics.columns.items()
+            }
+        },
+        "warnings": list(analysis.warnings),
+    }
+
+
+def analysis_from_dict(payload: dict[str, Any]) -> DatasetAnalysis:
+    metadata_payload = payload["metadata"]
+    statistics_payload = payload.get("statistics")
+    return DatasetAnalysis(
+        metadata=DatasetMetadata(
+            version=metadata_payload["version"],
+            created_at=datetime.fromisoformat(metadata_payload["created_at"]),
+            row_count=metadata_payload["row_count"],
+            column_count=metadata_payload["column_count"],
+            source_file_count=metadata_payload["source_file_count"],
+            schema_hash=metadata_payload.get("schema_hash"),
+            column_hashes=metadata_payload.get("column_hashes"),
+        ),
+        schemas=[
+            SchemaSignature(
+                columns=[
+                    ColumnDefinition(name=column["name"], dtype=column["dtype"])
+                    for column in schema["columns"]
+                ],
+                hash=schema["hash"],
+                occurrence_count=schema["occurrence_count"],
+            )
+            for schema in payload.get("schemas", [])
+        ],
+        statistics=None
+        if statistics_payload is None
+        else DatasetStatistics(
+            columns={
+                name: ColumnStatistics(
+                    dtype=stats["dtype"],
+                    null_rate=stats["null_rate"],
+                    unique_estimate=stats.get("unique_estimate"),
+                    min_value=stats.get("min_value"),
+                    max_value=stats.get("max_value"),
+                    mean=stats.get("mean"),
+                    median=stats.get("median"),
+                    stddev=stats.get("stddev"),
+                    histogram=None
+                    if stats.get("histogram") is None
+                    else [
+                        (entry["label"], entry["count"]) for entry in stats["histogram"]
+                    ],
+                )
+                for name, stats in statistics_payload.get("columns", {}).items()
+            }
+        ),
+        warnings=list(payload.get("warnings", [])),
+    )
+
+
+def diff_report_to_dict(report: DiffReport) -> dict[str, Any]:
+    return {
+        "metadata_changes": list(report.metadata_changes),
+        "schema_changes": list(report.schema_changes),
+        "statistics_changes": list(report.statistics_changes),
+        "warnings": list(report.warnings),
+    }