truthound 1.0.8__py3-none-any.whl

truthound/cli_modules/profiler/auto_profile.py

@@ -0,0 +1,175 @@
+"""Auto-profile command.
+
+This module implements the `truthound auto-profile` command for
+comprehensive data profiling with auto-detection.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Annotated, Optional
+
+import typer
+
+from truthound.cli_modules.common.errors import error_boundary, require_file
+
+
+def _read_file_as_lazy(path: Path):
+    """Read a file as a Polars LazyFrame."""
+    import polars as pl
+
+    suffix = path.suffix.lower()
+    readers = {
+        ".parquet": pl.scan_parquet,
+        ".csv": pl.scan_csv,
+        ".json": pl.scan_ndjson,
+        ".ndjson": pl.scan_ndjson,
+    }
+
+    if suffix not in readers:
+        raise ValueError(
+            f"Unsupported file type: {suffix}. "
+            f"Supported: {list(readers.keys())}"
+        )
+
+    return readers[suffix](path)
+
+
+def _print_profile_summary(profile) -> None:
+    """Print a summary of the profile to console."""
+    typer.echo(f"\n{'='*60}")
+    typer.echo(f"Profile: {profile.name or 'unnamed'}")
+    typer.echo(f"{'='*60}")
+    typer.echo(f"Rows: {profile.row_count:,}")
+    typer.echo(f"Columns: {profile.column_count}")
+    typer.echo(f"Estimated Memory: {profile.estimated_memory_bytes / 1024 / 1024:.2f} MB")
+
+    if profile.duplicate_row_ratio > 0:
+        typer.echo(
+            f"Duplicate Rows: {profile.duplicate_row_count:,} "
+            f"({profile.duplicate_row_ratio*100:.1f}%)"
+        )
+
+    typer.echo(f"\n{'Column Details':^60}")
+    typer.echo("-" * 60)
+
+    for col in profile.columns:
+        typer.echo(f"\n{col.name}")
+        typer.echo(f"  Type: {col.physical_type} -> {col.inferred_type.value}")
+        typer.echo(f"  Nulls: {col.null_count:,} ({col.null_ratio*100:.1f}%)")
+        typer.echo(f"  Unique: {col.distinct_count:,} ({col.unique_ratio*100:.1f}%)")
+
+        if col.distribution:
+            dist = col.distribution
+            typer.echo(f"  Range: [{dist.min}, {dist.max}]")
+            if dist.mean is not None:
+                typer.echo(f"  Mean: {dist.mean:.2f}, Std: {dist.std:.2f}")
+
+        if col.min_length is not None:
+            typer.echo(f"  Length: [{col.min_length}, {col.max_length}], avg={col.avg_length:.1f}")
+
+        if col.detected_patterns:
+            patterns = [p.pattern for p in col.detected_patterns[:3]]
+            typer.echo(f"  Patterns: {', '.join(patterns)}")
+
+        if col.suggested_validators:
+            typer.echo(f"  Suggested: {len(col.suggested_validators)} validators")
+
+
+@error_boundary
+def auto_profile_cmd(
+    file: Annotated[Path, typer.Argument(help="Path to the data file")],
+    output: Annotated[
+        Optional[Path],
+        typer.Option("--output", "-o", help="Output file path for profile JSON"),
+    ] = None,
+    format: Annotated[
+        str,
+        typer.Option("--format", "-f", help="Output format (console, json, yaml)"),
+    ] = "console",
+    include_patterns: Annotated[
+        bool,
+        typer.Option("--patterns/--no-patterns", help="Include pattern detection"),
+    ] = True,
+    include_correlations: Annotated[
+        bool,
+        typer.Option("--correlations/--no-correlations", help="Include correlation analysis"),
+    ] = False,
+    sample_size: Annotated[
+        Optional[int],
+        typer.Option("--sample", "-s", help="Sample size for profiling (default: all rows)"),
+    ] = None,
+    top_n: Annotated[
+        int,
+        typer.Option("--top-n", help="Number of top/bottom values to include"),
+    ] = 10,
+) -> None:
+    """Profile data with auto-detection of types and patterns.
+
+    This performs comprehensive profiling including:
+    - Column statistics (null ratio, unique ratio, distribution)
+    - Type inference (email, phone, UUID, etc.)
+    - Pattern detection
+    - Suggested validation rules
+
+    Examples:
+        truthound auto-profile data.csv
+        truthound auto-profile data.parquet -o profile.json
+        truthound auto-profile data.csv --no-patterns --sample 10000
+        truthound auto-profile data.csv --format yaml
+    """
+    require_file(file)
+
+    try:
+        from truthound.profiler import (
+            DataProfiler,
+            ProfilerConfig,
+            save_profile,
+        )
+
+        config = ProfilerConfig(
+            include_patterns=include_patterns,
+            include_correlations=include_correlations,
+            sample_size=sample_size,
+            top_n_values=top_n,
+        )
+
+        profiler = DataProfiler(config=config)
+
+        typer.echo(f"Profiling {file}...")
+        profile_result = profiler.profile(
+            _read_file_as_lazy(file),
+            name=file.stem,
+            source=str(file),
+        )
+
+        if format == "json":
+            result = json.dumps(profile_result.to_dict(), indent=2, default=str)
+            if output:
+                with open(output, "w", encoding="utf-8") as f:
+                    f.write(result)
+                typer.echo(f"Profile saved to {output}")
+            else:
+                typer.echo(result)
+
+        elif format == "yaml":
+            import yaml
+
+            result = yaml.dump(profile_result.to_dict(), default_flow_style=False)
+            if output:
+                with open(output, "w", encoding="utf-8") as f:
+                    f.write(result)
+                typer.echo(f"Profile saved to {output}")
+            else:
+                typer.echo(result)
+
+        else:  # console
+            _print_profile_summary(profile_result)
+            if output:
+                save_profile(profile_result, output)
+                typer.echo(f"\nFull profile saved to {output}")
+
+    except Exception as e:
+        typer.echo(f"Error: {e}", err=True)
+        raise typer.Exit(1)

truthound/cli_modules/profiler/metadata.py

@@ -0,0 +1,107 @@
+"""Metadata listing commands for profiler.
+
+This module implements commands for listing available formats, presets,
+and categories for suite generation.
+"""
+
+from __future__ import annotations
+
+import typer
+
+from truthound.cli_modules.common.errors import error_boundary
+
+
+@error_boundary
+def list_formats_cmd() -> None:
+    """List available output formats for suite generation.
+
+    Displays all supported output formats with descriptions.
+    """
+    try:
+        from truthound.profiler import get_available_formats
+
+        typer.echo("Available output formats:")
+        typer.echo("")
+
+        formats_info = {
+            "yaml": "Human-readable YAML format (default)",
+            "json": "Machine-readable JSON format",
+            "python": "Executable Python code with validators",
+            "toml": "TOML configuration format",
+            "checkpoint": "Truthound checkpoint format for CI/CD",
+        }
+
+        for fmt in get_available_formats():
+            desc = formats_info.get(fmt, "")
+            typer.echo(f"  {fmt:12} - {desc}")
+
+    except Exception as e:
+        typer.echo(f"Error: {e}", err=True)
+        raise typer.Exit(1)
+
+
+@error_boundary
+def list_presets_cmd() -> None:
+    """List available configuration presets for suite generation.
+
+    Displays all supported presets with descriptions.
+    """
+    try:
+        from truthound.profiler import get_available_presets
+
+        typer.echo("Available configuration presets:")
+        typer.echo("")
+
+        presets_info = {
+            "default": "Balanced settings (medium strictness, all categories)",
+            "strict": "Strict validation rules with high confidence",
+            "loose": "Relaxed validation for flexible data",
+            "minimal": "Only high-confidence schema rules",
+            "comprehensive": "All generators with detailed output",
+            "schema_only": "Schema and completeness rules only",
+            "format_only": "Format and pattern rules only",
+            "ci_cd": "Optimized for CI/CD pipelines (checkpoint format)",
+            "development": "Development-friendly (Python code output)",
+            "production": "Production-ready (strict, high confidence)",
+        }
+
+        for preset in get_available_presets():
+            desc = presets_info.get(preset, "")
+            typer.echo(f"  {preset:16} - {desc}")
+
+    except Exception as e:
+        typer.echo(f"Error: {e}", err=True)
+        raise typer.Exit(1)
+
+
+@error_boundary
+def list_categories_cmd() -> None:
+    """List available rule categories for suite generation.
+
+    Displays all supported rule categories with descriptions.
+    """
+    try:
+        from truthound.profiler import get_available_categories
+
+        typer.echo("Available rule categories:")
+        typer.echo("")
+
+        categories_info = {
+            "schema": "Column existence, types, and structure",
+            "completeness": "Null values and data completeness",
+            "uniqueness": "Unique constraints and cardinality",
+            "format": "Data format validation (email, phone, etc.)",
+            "distribution": "Statistical distribution checks",
+            "pattern": "Regex pattern matching",
+            "temporal": "Date/time validation",
+            "relationship": "Cross-column relationships",
+            "anomaly": "Anomaly detection rules",
+        }
+
+        for cat in get_available_categories():
+            desc = categories_info.get(cat, "")
+            typer.echo(f"  {cat:14} - {desc}")
+
+    except Exception as e:
+        typer.echo(f"Error: {e}", err=True)
+        raise typer.Exit(1)

truthound/cli_modules/profiler/suite.py

@@ -0,0 +1,283 @@
+"""Suite generation commands.
+
+This module implements the `truthound generate-suite` and `truthound quick-suite`
+commands for generating validation rules from profiles.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Annotated, Optional
+
+import typer
+
+from truthound.cli_modules.common.errors import error_boundary, require_file
+from truthound.cli_modules.common.options import parse_list_callback
+
+
+@error_boundary
+def generate_suite_cmd(
+    profile_file: Annotated[
+        Path,
+        typer.Argument(help="Path to profile JSON file (from auto-profile)"),
+    ],
+    output: Annotated[
+        Optional[Path],
+        typer.Option("--output", "-o", help="Output file path"),
+    ] = None,
+    format: Annotated[
+        str,
+        typer.Option(
+            "--format",
+            "-f",
+            help="Output format (yaml, json, python, toml, checkpoint)",
+        ),
+    ] = "yaml",
+    strictness: Annotated[
+        str,
+        typer.Option("--strictness", "-s", help="Rule strictness (loose, medium, strict)"),
+    ] = "medium",
+    include: Annotated[
+        Optional[list[str]],
+        typer.Option("--include", "-i", help="Include only these categories"),
+    ] = None,
+    exclude: Annotated[
+        Optional[list[str]],
+        typer.Option("--exclude", "-e", help="Exclude these categories"),
+    ] = None,
+    min_confidence: Annotated[
+        Optional[str],
+        typer.Option("--min-confidence", help="Minimum rule confidence (low, medium, high)"),
+    ] = None,
+    name: Annotated[
+        Optional[str],
+        typer.Option("--name", "-n", help="Name for the validation suite"),
+    ] = None,
+    preset: Annotated[
+        Optional[str],
+        typer.Option(
+            "--preset",
+            "-p",
+            help="Configuration preset (default, strict, loose, minimal, comprehensive, ci_cd)",
+        ),
+    ] = None,
+    config: Annotated[
+        Optional[Path],
+        typer.Option("--config", "-c", help="Path to configuration file"),
+    ] = None,
+    group_by_category: Annotated[
+        bool,
+        typer.Option("--group-by-category", help="Group rules by category in output"),
+    ] = False,
+    code_style: Annotated[
+        str,
+        typer.Option(
+            "--code-style",
+            help="Python code style (functional, class_based, declarative)",
+        ),
+    ] = "functional",
+) -> None:
+    """Generate validation rules from a profile.
+
+    This creates a validation suite based on the data profile.
+    Categories available: schema, completeness, uniqueness, format,
+    distribution, pattern, temporal, relationship, anomaly
+
+    Output formats:
+        - yaml: Human-readable YAML (default)
+        - json: Machine-readable JSON
+        - python: Executable Python code
+        - toml: TOML configuration
+        - checkpoint: Truthound checkpoint format for CI/CD
+
+    Examples:
+        truthound generate-suite profile.json -o rules.yaml
+        truthound generate-suite profile.json -i schema -i format
+        truthound generate-suite profile.json --preset strict
+        truthound generate-suite profile.json -f python --code-style class_based
+        truthound generate-suite profile.json -f checkpoint -o ci_rules.yaml
+    """
+    require_file(profile_file, "Profile file")
+
+    try:
+        from truthound.profiler import (
+            run_generate_suite,
+            get_available_formats,
+            get_available_presets,
+        )
+
+        # Validate format
+        available_formats = get_available_formats()
+        if format not in available_formats:
+            typer.echo(
+                f"Error: Invalid format '{format}'. "
+                f"Available: {', '.join(available_formats)}",
+                err=True,
+            )
+            raise typer.Exit(1)
+
+        # Validate preset
+        if preset:
+            available_presets = get_available_presets()
+            if preset not in available_presets:
+                typer.echo(
+                    f"Error: Invalid preset '{preset}'. "
+                    f"Available: {', '.join(available_presets)}",
+                    err=True,
+                )
+                raise typer.Exit(1)
+
+        # Parse categories
+        include_cats = parse_list_callback(include) if include else None
+        exclude_cats = parse_list_callback(exclude) if exclude else None
+
+        # Run generation using the handler
+        exit_code = run_generate_suite(
+            profile_file=profile_file,
+            output=output,
+            format=format,
+            strictness=strictness,
+            include=include_cats,
+            exclude=exclude_cats,
+            min_confidence=min_confidence,
+            name=name,
+            preset=preset,
+            config=config,
+            group_by_category=group_by_category,
+            echo=typer.echo,
+            verbose=True,
+        )
+
+        if exit_code != 0:
+            raise typer.Exit(exit_code)
+
+    except typer.Exit:
+        raise
+    except Exception as e:
+        typer.echo(f"Error: {e}", err=True)
+        raise typer.Exit(1)
+
+
+@error_boundary
+def quick_suite_cmd(
+    file: Annotated[Path, typer.Argument(help="Path to the data file")],
+    output: Annotated[
+        Optional[Path],
+        typer.Option("--output", "-o", help="Output file path"),
+    ] = None,
+    format: Annotated[
+        str,
+        typer.Option(
+            "--format",
+            "-f",
+            help="Output format (yaml, json, python, toml, checkpoint)",
+        ),
+    ] = "yaml",
+    strictness: Annotated[
+        str,
+        typer.Option("--strictness", "-s", help="Rule strictness (loose, medium, strict)"),
+    ] = "medium",
+    include: Annotated[
+        Optional[list[str]],
+        typer.Option("--include", "-i", help="Include only these categories"),
+    ] = None,
+    exclude: Annotated[
+        Optional[list[str]],
+        typer.Option("--exclude", "-e", help="Exclude these categories"),
+    ] = None,
+    min_confidence: Annotated[
+        Optional[str],
+        typer.Option("--min-confidence", help="Minimum rule confidence (low, medium, high)"),
+    ] = None,
+    name: Annotated[
+        Optional[str],
+        typer.Option("--name", "-n", help="Name for the validation suite"),
+    ] = None,
+    preset: Annotated[
+        Optional[str],
+        typer.Option(
+            "--preset",
+            "-p",
+            help="Configuration preset (default, strict, loose, minimal, comprehensive, ci_cd)",
+        ),
+    ] = None,
+    sample_size: Annotated[
+        Optional[int],
+        typer.Option("--sample-size", help="Sample size for profiling (default: auto)"),
+    ] = None,
+) -> None:
+    """Profile data and generate validation rules in one step.
+
+    This is a convenience command that combines auto-profile and generate-suite.
+
+    Output formats:
+        - yaml: Human-readable YAML (default)
+        - json: Machine-readable JSON
+        - python: Executable Python code
+        - toml: TOML configuration
+        - checkpoint: Truthound checkpoint format for CI/CD
+
+    Examples:
+        truthound quick-suite data.parquet -o rules.yaml
+        truthound quick-suite data.csv -s strict -f python -o validators.py
+        truthound quick-suite data.parquet --preset ci_cd -o ci_rules.yaml
+        truthound quick-suite large_data.parquet --sample-size 10000
+    """
+    require_file(file)
+
+    try:
+        from truthound.profiler import (
+            run_quick_suite,
+            get_available_formats,
+            get_available_presets,
+        )
+
+        # Validate format
+        available_formats = get_available_formats()
+        if format not in available_formats:
+            typer.echo(
+                f"Error: Invalid format '{format}'. "
+                f"Available: {', '.join(available_formats)}",
+                err=True,
+            )
+            raise typer.Exit(1)
+
+        # Validate preset
+        if preset:
+            available_presets = get_available_presets()
+            if preset not in available_presets:
+                typer.echo(
+                    f"Error: Invalid preset '{preset}'. "
+                    f"Available: {', '.join(available_presets)}",
+                    err=True,
+                )
+                raise typer.Exit(1)
+
+        # Parse categories
+        include_cats = parse_list_callback(include) if include else None
+        exclude_cats = parse_list_callback(exclude) if exclude else None
+
+        # Run quick suite using the handler
+        exit_code = run_quick_suite(
+            file=file,
+            output=output,
+            format=format,
+            strictness=strictness,
+            include=include_cats,
+            exclude=exclude_cats,
+            min_confidence=min_confidence,
+            name=name,
+            preset=preset,
+            sample_size=sample_size,
+            echo=typer.echo,
+            verbose=True,
+        )
+
+        if exit_code != 0:
+            raise typer.Exit(exit_code)
+
+    except typer.Exit:
+        raise
+    except Exception as e:
+        typer.echo(f"Error: {e}", err=True)
+        raise typer.Exit(1)