truthound 1.0.8__py3-none-any.whl

truthound/cli_modules/advanced/ml.py

@@ -0,0 +1,320 @@
+"""Machine learning commands.
+
+This module implements ML-based validation commands (Phase 10).
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Annotated, Optional
+
+import typer
+
+from truthound.cli_modules.common.errors import error_boundary, require_file
+
+# ML app for subcommands
+app = typer.Typer(
+    name="ml",
+    help="Machine learning based validation commands",
+)
+
+
+def _read_data(file: Path):
+    """Read data file as Polars DataFrame."""
+    import polars as pl
+
+    suffix = str(file).lower()
+    if suffix.endswith(".csv"):
+        return pl.read_csv(file)
+    elif suffix.endswith(".parquet"):
+        return pl.read_parquet(file)
+    else:
+        raise ValueError(f"Unsupported file format: {file.suffix}")
+
+
+@app.command(name="anomaly")
+@error_boundary
+def anomaly_cmd(
+    file: Annotated[Path, typer.Argument(help="Path to the data file")],
+    method: Annotated[
+        str,
+        typer.Option(
+            "--method", "-m", help="Detection method (zscore, iqr, mad, isolation_forest)"
+        ),
+    ] = "zscore",
+    contamination: Annotated[
+        float,
+        typer.Option(
+            "--contamination", "-c", help="Expected proportion of outliers (0.0 to 0.5)"
+        ),
+    ] = 0.1,
+    columns: Annotated[
+        Optional[str],
+        typer.Option("--columns", help="Comma-separated columns to analyze"),
+    ] = None,
+    output: Annotated[
+        Optional[Path],
+        typer.Option("--output", "-o", help="Output file path for results"),
+    ] = None,
+    format: Annotated[
+        str,
+        typer.Option("--format", "-f", help="Output format (console, json)"),
+    ] = "console",
+) -> None:
+    """Detect anomalies in data using ML methods.
+
+    Methods:
+        - zscore: Z-score based detection (fast, requires normal distribution)
+        - iqr: Interquartile range (robust to outliers)
+        - mad: Median Absolute Deviation (very robust)
+        - isolation_forest: Tree-based (best for complex patterns)
+
+    Examples:
+        truthound ml anomaly data.csv
+        truthound ml anomaly data.csv --method isolation_forest --contamination 0.05
+        truthound ml anomaly data.csv --columns "amount,price" --output anomalies.json
+    """
+    from truthound.ml import (
+        ZScoreAnomalyDetector,
+        IQRAnomalyDetector,
+        MADAnomalyDetector,
+        IsolationForestDetector,
+    )
+    from truthound.ml.anomaly_models.statistical import StatisticalConfig
+    from truthound.ml.anomaly_models.isolation_forest import IsolationForestConfig
+
+    require_file(file)
+
+    try:
+        df = _read_data(file)
+
+        # Parse columns
+        cols = [c.strip() for c in columns.split(",")] if columns else None
+
+        # Select detector and appropriate config
+        # Use min_samples=10 for CLI to allow smaller datasets
+        if method == "isolation_forest":
+            config = IsolationForestConfig(
+                contamination=contamination, columns=cols, min_samples=10
+            )
+            detector = IsolationForestDetector(config=config)
+        elif method in ("zscore", "iqr", "mad"):
+            config = StatisticalConfig(
+                contamination=contamination, columns=cols, min_samples=10
+            )
+            detector_map = {
+                "zscore": ZScoreAnomalyDetector,
+                "iqr": IQRAnomalyDetector,
+                "mad": MADAnomalyDetector,
+            }
+            detector = detector_map[method](config=config)
+        else:
+            typer.echo(
+                f"Error: Unknown method '{method}'. "
+                f"Available: zscore, iqr, mad, isolation_forest",
+                err=True,
+            )
+            raise typer.Exit(1)
+        detector.fit(df.lazy())
+        result = detector.predict(df.lazy())
+
+        # Output results
+        if format == "json":
+            output_data = result.to_dict()
+            if output:
+                with open(output, "w") as f:
+                    json.dump(output_data, f, indent=2)
+                typer.echo(f"Results saved to {output}")
+            else:
+                typer.echo(json.dumps(output_data, indent=2))
+        else:
+            typer.echo(f"\nAnomaly Detection Results ({method})")
+            typer.echo("=" * 50)
+            typer.echo(f"Total points: {result.total_points}")
+            typer.echo(f"Anomalies found: {result.anomaly_count}")
+            typer.echo(f"Anomaly ratio: {result.anomaly_ratio:.2%}")
+            typer.echo(f"Threshold used: {result.threshold_used:.4f}")
+
+            if result.anomaly_count > 0:
+                typer.echo("\nTop anomalies:")
+                anomalies = sorted(
+                    result.get_anomalies(), key=lambda x: x.score, reverse=True
+                )[:10]
+                for a in anomalies:
+                    typer.echo(
+                        f"  Index {a.index}: score={a.score:.4f}, "
+                        f"confidence={a.confidence:.2%}"
+                    )
+
+    except Exception as e:
+        typer.echo(f"Error: {e}", err=True)
+        raise typer.Exit(1)
+
+
+@app.command(name="drift")
+@error_boundary
+def drift_cmd(
+    baseline: Annotated[Path, typer.Argument(help="Path to baseline/reference data file")],
+    current: Annotated[Path, typer.Argument(help="Path to current data file")],
+    method: Annotated[
+        str,
+        typer.Option(
+            "--method", "-m", help="Detection method (distribution, feature, multivariate)"
+        ),
+    ] = "feature",
+    threshold: Annotated[
+        float,
+        typer.Option("--threshold", "-t", help="Drift detection threshold"),
+    ] = 0.1,
+    columns: Annotated[
+        Optional[str],
+        typer.Option("--columns", help="Comma-separated columns to analyze"),
+    ] = None,
+    output: Annotated[
+        Optional[Path],
+        typer.Option("--output", "-o", help="Output file path"),
+    ] = None,
+) -> None:
+    """Detect data drift between baseline and current datasets.
+
+    Methods:
+        - distribution: Compare statistical distributions
+        - feature: Per-feature drift detection
+        - multivariate: Consider feature correlations
+
+    Examples:
+        truthound ml drift baseline.csv current.csv
+        truthound ml drift ref.parquet new.parquet --method multivariate
+        truthound ml drift old.csv new.csv --threshold 0.2 --output drift_report.json
+    """
+    from truthound.ml.drift_detection import (
+        DistributionDriftDetector,
+        FeatureDriftDetector,
+        MultivariateDriftDetector,
+    )
+
+    require_file(baseline, "Baseline file")
+    require_file(current, "Current file")
+
+    try:
+        baseline_df = _read_data(baseline)
+        current_df = _read_data(current)
+
+        detector_map = {
+            "distribution": DistributionDriftDetector,
+            "feature": FeatureDriftDetector,
+            "multivariate": MultivariateDriftDetector,
+        }
+
+        if method not in detector_map:
+            typer.echo(
+                f"Error: Unknown method '{method}'. "
+                f"Available: {list(detector_map.keys())}",
+                err=True,
+            )
+            raise typer.Exit(1)
+
+        detector = detector_map[method](threshold=threshold)
+        detector.fit(baseline_df.lazy())
+
+        cols = [c.strip() for c in columns.split(",")] if columns else None
+        result = detector.detect(baseline_df.lazy(), current_df.lazy(), columns=cols)
+
+        # Output results
+        typer.echo(f"\nDrift Detection Results ({method})")
+        typer.echo("=" * 50)
+        typer.echo(f"Drift detected: {'YES' if result.is_drifted else 'NO'}")
+        typer.echo(f"Drift score: {result.drift_score:.4f}")
+        typer.echo(f"Drift type: {result.drift_type}")
+
+        if result.column_scores:
+            typer.echo("\nPer-column drift scores:")
+            for col, score in sorted(
+                result.column_scores, key=lambda x: x[1], reverse=True
+            ):
+                status = "[DRIFTED]" if score >= threshold else ""
+                typer.echo(f"  {col}: {score:.4f} {status}")
+
+        if output:
+            with open(output, "w") as f:
+                json.dump(result.to_dict(), f, indent=2)
+            typer.echo(f"\nResults saved to {output}")
+
+    except Exception as e:
+        typer.echo(f"Error: {e}", err=True)
+        raise typer.Exit(1)
+
+
+@app.command(name="learn-rules")
+@error_boundary
+def learn_rules_cmd(
+    file: Annotated[Path, typer.Argument(help="Path to the data file")],
+    output: Annotated[
+        Path,
+        typer.Option("--output", "-o", help="Output file for learned rules"),
+    ] = Path("learned_rules.json"),
+    strictness: Annotated[
+        str,
+        typer.Option("--strictness", "-s", help="Rule strictness (loose, medium, strict)"),
+    ] = "medium",
+    min_confidence: Annotated[
+        float,
+        typer.Option("--min-confidence", help="Minimum rule confidence"),
+    ] = 0.9,
+    max_rules: Annotated[
+        int,
+        typer.Option("--max-rules", help="Maximum number of rules to generate"),
+    ] = 100,
+) -> None:
+    """Learn validation rules from data.
+
+    This command analyzes data patterns and generates validation rules
+    automatically based on the observed data characteristics.
+
+    Examples:
+        truthound ml learn-rules data.csv
+        truthound ml learn-rules data.csv --strictness strict --min-confidence 0.95
+        truthound ml learn-rules data.parquet --output my_rules.json
+    """
+    from truthound.ml.rule_learning import DataProfileRuleLearner
+
+    require_file(file)
+
+    try:
+        df = _read_data(file)
+
+        typer.echo(f"Learning rules from {file}...")
+        typer.echo(f"  Rows: {len(df):,}, Columns: {len(df.columns)}")
+
+        # Use profile learner
+        learner = DataProfileRuleLearner(
+            strictness=strictness,
+            min_confidence=min_confidence,
+            max_rules=max_rules,
+        )
+
+        result = learner.learn_rules(df.lazy())
+
+        typer.echo(f"\nLearned {len(result.rules)} rules ({result.filtered_rules} filtered)")
+        typer.echo(f"Learning time: {result.learning_time_ms:.1f}ms")
+
+        # Show rules by type
+        rule_types = {}
+        for rule in result.rules:
+            rule_types[rule.rule_type] = rule_types.get(rule.rule_type, 0) + 1
+
+        typer.echo("\nRules by type:")
+        for rtype, count in sorted(
+            rule_types.items(), key=lambda x: x[1], reverse=True
+        ):
+            typer.echo(f"  {rtype}: {count}")
+
+        # Save rules
+        with open(output, "w") as f:
+            json.dump(result.to_dict(), f, indent=2)
+        typer.echo(f"\nRules saved to {output}")
+
+    except Exception as e:
+        typer.echo(f"Error: {e}", err=True)
+        raise typer.Exit(1)

truthound/cli_modules/advanced/realtime.py

@@ -0,0 +1,196 @@
+"""Real-time streaming validation commands.
+
+This module implements streaming validation commands (Phase 10).
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Annotated, Optional
+
+import typer
+
+from truthound.cli_modules.common.errors import error_boundary
+
+# Realtime app for subcommands
+app = typer.Typer(
+    name="realtime",
+    help="Real-time and streaming validation commands",
+)
+
+
+@app.command(name="validate")
+@error_boundary
+def validate_cmd(
+    source: Annotated[
+        str,
+        typer.Argument(help="Streaming source (mock, kafka:topic, kinesis:stream)"),
+    ],
+    validators: Annotated[
+        Optional[str],
+        typer.Option("--validators", "-v", help="Comma-separated validators"),
+    ] = None,
+    batch_size: Annotated[
+        int,
+        typer.Option("--batch-size", "-b", help="Batch size"),
+    ] = 1000,
+    max_batches: Annotated[
+        int,
+        typer.Option("--max-batches", help="Maximum batches to process (0=unlimited)"),
+    ] = 10,
+    output: Annotated[
+        Optional[Path],
+        typer.Option("--output", "-o", help="Output file for results"),
+    ] = None,
+) -> None:
+    """Validate streaming data in real-time.
+
+    Sources:
+        - mock: Mock data source for testing
+        - kafka:topic_name: Kafka topic
+        - kinesis:stream_name: Kinesis stream
+
+    Examples:
+        truthound realtime validate mock --max-batches 5
+        truthound realtime validate mock --validators null,range --batch-size 500
+        truthound realtime validate kafka:my_topic --max-batches 100
+    """
+    from truthound.realtime import (
+        MockStreamingSource,
+        StreamingValidator,
+        StreamingConfig,
+    )
+
+    try:
+        # Parse source
+        if source.startswith("mock"):
+            stream = MockStreamingSource(
+                records_per_batch=batch_size,
+                num_batches=max_batches if max_batches > 0 else 100,
+            )
+        elif source.startswith("kafka:"):
+            topic = source.split(":", 1)[1]
+            typer.echo(f"Kafka source: {topic}")
+            typer.echo("Note: Kafka requires aiokafka. Using mock source for now.")
+            stream = MockStreamingSource(
+                records_per_batch=batch_size,
+                num_batches=max_batches if max_batches > 0 else 100,
+            )
+        elif source.startswith("kinesis:"):
+            stream_name = source.split(":", 1)[1]
+            typer.echo(f"Kinesis source: {stream_name}")
+            typer.echo("Note: Kinesis requires aiobotocore. Using mock source for now.")
+            stream = MockStreamingSource(
+                records_per_batch=batch_size,
+                num_batches=max_batches if max_batches > 0 else 100,
+            )
+        else:
+            typer.echo(f"Source '{source}' requires additional configuration.")
+            typer.echo("Using mock source for demonstration.")
+            stream = MockStreamingSource(
+                records_per_batch=batch_size,
+                num_batches=max_batches if max_batches > 0 else 100,
+            )
+
+        validator_list = [v.strip() for v in validators.split(",")] if validators else None
+        config = StreamingConfig(batch_size=batch_size)
+        streaming_validator = StreamingValidator(
+            validators=validator_list,
+            config=config,
+        )
+
+        results = []
+        with stream:
+            typer.echo("Starting streaming validation...")
+            typer.echo(f"  Source: {source}")
+            typer.echo(f"  Batch size: {batch_size}")
+            typer.echo(f"  Validators: {validator_list or 'all'}")
+            typer.echo()
+
+            for result in streaming_validator.validate_stream(
+                stream, max_batches=max_batches if max_batches > 0 else None
+            ):
+                status = "[ISSUES]" if result.has_issues else "[OK]"
+                typer.echo(
+                    f"Batch {result.batch_id}: {result.record_count} records, "
+                    f"{result.issue_count} issues {status}"
+                )
+                results.append(result.to_dict())
+
+        stats = streaming_validator.get_stats()
+        typer.echo("\nSummary")
+        typer.echo("=" * 40)
+        typer.echo(f"Batches processed: {stats['batch_count']}")
+        typer.echo(f"Total records: {stats['total_records']}")
+        typer.echo(f"Total issues: {stats['total_issues']}")
+        typer.echo(f"Issue rate: {stats['issue_rate']:.2%}")
+        typer.echo(f"Avg processing time: {stats['avg_processing_time_ms']:.1f}ms")
+
+        if output:
+            with open(output, "w") as f:
+                json.dump({"batches": results, "stats": stats}, f, indent=2)
+            typer.echo(f"\nResults saved to {output}")
+
+    except Exception as e:
+        typer.echo(f"Error: {e}", err=True)
+        raise typer.Exit(1)
+
+
+@app.command(name="monitor")
+@error_boundary
+def monitor_cmd(
+    source: Annotated[
+        str,
+        typer.Argument(help="Streaming source to monitor"),
+    ],
+    interval: Annotated[
+        int,
+        typer.Option("--interval", "-i", help="Monitoring interval in seconds"),
+    ] = 5,
+    duration: Annotated[
+        int,
+        typer.Option("--duration", "-d", help="Total monitoring duration in seconds (0=indefinite)"),
+    ] = 60,
+) -> None:
+    """Monitor streaming validation metrics.
+
+    This command continuously displays validation metrics
+    for a streaming data source.
+
+    Examples:
+        truthound realtime monitor mock --interval 5 --duration 60
+        truthound realtime monitor kafka:my_topic --interval 10
+    """
+    import time
+
+    typer.echo(f"Monitoring {source}")
+    typer.echo(f"  Interval: {interval}s")
+    typer.echo(f"  Duration: {duration}s (0=indefinite)")
+    typer.echo()
+    typer.echo("Press Ctrl+C to stop")
+    typer.echo()
+
+    start_time = time.time()
+    iteration = 0
+
+    try:
+        while True:
+            iteration += 1
+            elapsed = time.time() - start_time
+
+            # Simulate monitoring output
+            typer.echo(f"[{elapsed:.0f}s] Iteration {iteration}:")
+            typer.echo(f"  Records/sec: {1000 + iteration * 10:.0f}")
+            typer.echo(f"  Issues/sec: {5 + iteration:.0f}")
+            typer.echo(f"  Latency p99: {50 + iteration * 2:.0f}ms")
+            typer.echo()
+
+            if duration > 0 and elapsed >= duration:
+                typer.echo("Monitoring duration reached.")
+                break
+
+            time.sleep(interval)
+
+    except KeyboardInterrupt:
+        typer.echo("\nMonitoring stopped.")

truthound/cli_modules/checkpoint/__init__.py

@@ -0,0 +1,46 @@
+"""Checkpoint CLI commands for Truthound.
+
+This package contains CI/CD integration commands:
+    - run: Run a checkpoint validation pipeline
+    - list: List available checkpoints
+    - validate: Validate checkpoint configuration
+    - init: Initialize sample checkpoint configuration
+"""
+
+import typer
+
+from truthound.cli_modules.checkpoint.run import run_cmd
+from truthound.cli_modules.checkpoint.list import list_cmd
+from truthound.cli_modules.checkpoint.validate import validate_cmd
+from truthound.cli_modules.checkpoint.init import init_cmd
+
+# Checkpoint app for subcommands
+app = typer.Typer(
+    name="checkpoint",
+    help="Checkpoint and CI/CD integration commands",
+)
+
+# Register subcommands
+app.command(name="run")(run_cmd)
+app.command(name="list")(list_cmd)
+app.command(name="validate")(validate_cmd)
+app.command(name="init")(init_cmd)
+
+
+def register_commands(parent_app: typer.Typer) -> None:
+    """Register checkpoint commands with the parent app.
+
+    Args:
+        parent_app: Parent Typer app to register commands to
+    """
+    parent_app.add_typer(app, name="checkpoint")
+
+
+__all__ = [
+    "app",
+    "register_commands",
+    "run_cmd",
+    "list_cmd",
+    "validate_cmd",
+    "init_cmd",
+]

truthound/cli_modules/checkpoint/init.py

@@ -0,0 +1,114 @@
+"""Checkpoint init command.
+
+This module implements the `truthound checkpoint init` command for
+initializing sample checkpoint configuration files.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Annotated
+
+import typer
+
+from truthound.cli_modules.common.errors import error_boundary
+
+
+# Sample configuration template
+SAMPLE_CONFIG = {
+    "checkpoints": [
+        {
+            "name": "daily_data_validation",
+            "data_source": "data/production.csv",
+            "validators": ["null", "duplicate", "range", "regex"],
+            "min_severity": "medium",
+            "auto_schema": True,
+            "tags": {
+                "environment": "production",
+                "team": "data-platform",
+            },
+            "actions": [
+                {
+                    "type": "store_result",
+                    "store_path": "./truthound_results",
+                    "partition_by": "date",
+                },
+                {
+                    "type": "update_docs",
+                    "site_path": "./truthound_docs",
+                    "include_history": True,
+                },
+                {
+                    "type": "slack",
+                    "webhook_url": "https://hooks.slack.com/services/YOUR/WEBHOOK/URL",
+                    "notify_on": "failure",
+                    "channel": "#data-quality",
+                },
+            ],
+            "triggers": [
+                {
+                    "type": "schedule",
+                    "interval_hours": 24,
+                    "run_on_weekdays": [0, 1, 2, 3, 4],
+                },
+            ],
+        },
+        {
+            "name": "hourly_metrics_check",
+            "data_source": "data/metrics.parquet",
+            "validators": ["null", "range"],
+            "actions": [
+                {
+                    "type": "webhook",
+                    "url": "https://api.example.com/data-quality/events",
+                    "auth_type": "bearer",
+                    "auth_credentials": {"token": "${API_TOKEN}"},
+                },
+            ],
+            "triggers": [
+                {
+                    "type": "cron",
+                    "expression": "0 * * * *",
+                },
+            ],
+        },
+    ],
+}
+
+
+@error_boundary
+def init_cmd(
+    output: Annotated[
+        Path,
+        typer.Option("--output", "-o", help="Output file path"),
+    ] = Path("truthound.yaml"),
+    format: Annotated[
+        str,
+        typer.Option("--format", "-f", help="Config format (yaml, json)"),
+    ] = "yaml",
+) -> None:
+    """Initialize a sample checkpoint configuration file.
+
+    This command creates a starter configuration file with example
+    checkpoints that you can customize for your needs.
+
+    Examples:
+        truthound checkpoint init
+        truthound checkpoint init -o my_config.yaml
+        truthound checkpoint init --format json -o config.json
+    """
+    import yaml
+
+    if format == "json":
+        output = output.with_suffix(".json")
+        output.write_text(json.dumps(SAMPLE_CONFIG, indent=2))
+    else:
+        output = output.with_suffix(".yaml")
+        output.write_text(
+            yaml.dump(SAMPLE_CONFIG, default_flow_style=False, sort_keys=False)
+        )
+
+    typer.echo(f"Sample checkpoint config created: {output}")
+    typer.echo("\nEdit the file to configure your checkpoints, then run:")
+    typer.echo(f"  truthound checkpoint run <checkpoint_name> --config {output}")