truthound 1.0.8__py3-none-any.whl

truthound/__init__.py

@@ -0,0 +1,162 @@
+"""Truthound - Zero-Configuration Data Quality Framework Powered by Polars."""
+
+from truthound.api import check, mask, profile, scan
+from truthound.decorators import validator
+from truthound.drift import compare
+from truthound.report import Report
+from truthound.schema import Schema, learn
+
+# Data sources and execution engines (Phase 5)
+from truthound import datasources
+from truthound import execution
+from truthound.datasources import get_datasource, get_sql_datasource
+
+# Checkpoint and CI/CD integration (Phase 6)
+from truthound import checkpoint
+
+# Auto-profiling and rule generation (Phase 7)
+from truthound import profiler
+from truthound.profiler import (
+    DataProfiler,
+    profile_file,
+    profile_dataframe,
+    generate_suite,
+    ValidationSuite,
+    TableProfile,
+    ColumnProfile,
+)
+
+# Data Docs - HTML Reports and Dashboard (Phase 8)
+from truthound import datadocs
+from truthound.datadocs import (
+    HTMLReportBuilder,
+    generate_html_report,
+    generate_report_from_file,
+    ReportConfig,
+    ReportTheme,
+    ChartLibrary,
+)
+
+# Advanced Features - ML, Lineage, Realtime (Phase 10)
+from truthound import ml
+from truthound import lineage
+from truthound import realtime
+
+# ML exports
+from truthound.ml import (
+    ModelRegistry,
+    AnomalyDetector,
+    MLDriftDetector,
+    RuleLearner,
+    ModelType,
+    ModelState,
+)
+from truthound.ml.anomaly_models import (
+    ZScoreAnomalyDetector,
+    IQRAnomalyDetector,
+    IsolationForestDetector,
+    EnsembleAnomalyDetector,
+)
+from truthound.ml.drift_detection import (
+    DistributionDriftDetector,
+    FeatureDriftDetector,
+)
+
+# Lineage exports
+from truthound.lineage import (
+    LineageGraph,
+    LineageNode,
+    LineageEdge,
+    LineageTracker,
+    ImpactAnalyzer,
+    NodeType,
+    EdgeType,
+)
+
+# Realtime exports
+from truthound.realtime import (
+    StreamingValidator,
+    IncrementalValidator,
+    StreamingConfig,
+    CheckpointManager,
+    MemoryStateStore,
+    BatchResult,
+    StreamingMode,
+)
+
+# Version: Single source of truth from pyproject.toml
+try:
+    from importlib.metadata import version, PackageNotFoundError
+
+    __version__ = version("truthound")
+except PackageNotFoundError:
+    # Package not installed (development mode)
+    __version__ = "0.0.0.dev"
+__all__ = [
+    # Core API
+    "check",
+    "scan",
+    "mask",
+    "profile",
+    "learn",
+    "compare",
+    "validator",
+    "Report",
+    "Schema",
+    # Phase 5: Data sources
+    "datasources",
+    "execution",
+    "get_datasource",
+    "get_sql_datasource",
+    # Phase 6: Checkpoint & CI/CD
+    "checkpoint",
+    # Phase 7: Auto-profiling & Rule Generation
+    "profiler",
+    "DataProfiler",
+    "profile_file",
+    "profile_dataframe",
+    "generate_suite",
+    "ValidationSuite",
+    "TableProfile",
+    "ColumnProfile",
+    # Phase 8: Data Docs (HTML Reports & Dashboard)
+    "datadocs",
+    "HTMLReportBuilder",
+    "generate_html_report",
+    "generate_report_from_file",
+    "ReportConfig",
+    "ReportTheme",
+    "ChartLibrary",
+    # Phase 10: Advanced Features - ML
+    "ml",
+    "ModelRegistry",
+    "AnomalyDetector",
+    "MLDriftDetector",
+    "RuleLearner",
+    "ModelType",
+    "ModelState",
+    "ZScoreAnomalyDetector",
+    "IQRAnomalyDetector",
+    "IsolationForestDetector",
+    "EnsembleAnomalyDetector",
+    "DistributionDriftDetector",
+    "FeatureDriftDetector",
+    # Phase 10: Advanced Features - Lineage
+    "lineage",
+    "LineageGraph",
+    "LineageNode",
+    "LineageEdge",
+    "LineageTracker",
+    "ImpactAnalyzer",
+    "NodeType",
+    "EdgeType",
+    # Phase 10: Advanced Features - Realtime
+    "realtime",
+    "StreamingValidator",
+    "IncrementalValidator",
+    "StreamingConfig",
+    "CheckpointManager",
+    "MemoryStateStore",
+    "BatchResult",
+    "StreamingMode",
+]

truthound/adapters.py

@@ -0,0 +1,100 @@
+"""Input adapters for converting various data formats to Polars LazyFrame."""
+
+from pathlib import Path
+from typing import Any
+
+import polars as pl
+
+
+def to_lazyframe(data: Any) -> pl.LazyFrame:
+    """Convert various input formats to a Polars LazyFrame.
+
+    Supports:
+        - str: File path (CSV, JSON, Parquet)
+        - pl.DataFrame: Polars DataFrame
+        - pl.LazyFrame: Polars LazyFrame (passthrough)
+        - dict: Python dictionary
+        - pd.DataFrame: pandas DataFrame
+
+    Args:
+        data: Input data in any supported format.
+
+    Returns:
+        Polars LazyFrame for lazy evaluation.
+
+    Raises:
+        ValueError: If the input format is not supported.
+        FileNotFoundError: If a file path is provided but doesn't exist.
+    """
+    # Already a LazyFrame
+    if isinstance(data, pl.LazyFrame):
+        return data
+
+    # Polars DataFrame
+    if isinstance(data, pl.DataFrame):
+        return data.lazy()
+
+    # Dictionary
+    if isinstance(data, dict):
+        return pl.DataFrame(data).lazy()
+
+    # File path
+    if isinstance(data, str):
+        return _load_file(data)
+
+    # Try pandas DataFrame
+    if _is_pandas_dataframe(data):
+        return pl.from_pandas(data).lazy()
+
+    raise ValueError(
+        f"Unsupported input type: {type(data).__name__}. "
+        "Supported types: str (file path), pl.DataFrame, pl.LazyFrame, dict, pd.DataFrame"
+    )
+
+
+def _load_file(path: str) -> pl.LazyFrame:
+    """Load a file into a Polars LazyFrame based on extension.
+
+    Args:
+        path: Path to the file.
+
+    Returns:
+        Polars LazyFrame.
+
+    Raises:
+        FileNotFoundError: If the file doesn't exist.
+        ValueError: If the file extension is not supported.
+    """
+    file_path = Path(path)
+
+    if not file_path.exists():
+        raise FileNotFoundError(f"File not found: {path}")
+
+    suffix = file_path.suffix.lower()
+
+    if suffix == ".csv":
+        return pl.scan_csv(path)
+    elif suffix == ".json":
+        # JSON doesn't have a scan_ method, read eagerly then convert to lazy
+        return pl.read_json(path).lazy()
+    elif suffix == ".parquet":
+        return pl.scan_parquet(path)
+    elif suffix == ".ndjson" or suffix == ".jsonl":
+        return pl.scan_ndjson(path)
+    else:
+        raise ValueError(
+            f"Unsupported file extension: {suffix}. "
+            "Supported extensions: .csv, .json, .parquet, .ndjson, .jsonl"
+        )
+
+
+def _is_pandas_dataframe(obj: Any) -> bool:
+    """Check if an object is a pandas DataFrame without importing pandas.
+
+    Args:
+        obj: Object to check.
+
+    Returns:
+        True if the object is a pandas DataFrame.
+    """
+    return type(obj).__name__ == "DataFrame" and type(obj).__module__.startswith("pandas")

truthound/api.py

@@ -0,0 +1,365 @@
+"""Main API functions for Truthound."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import polars as pl
+
+from truthound.adapters import to_lazyframe
+from truthound.maskers import mask_data
+from truthound.profiler import profile_data
+from truthound.report import PIIReport, ProfileReport, Report
+from truthound.scanners import scan_pii
+from truthound.types import Severity
+from truthound.validators import BUILTIN_VALIDATORS, Validator, get_validator
+
+if TYPE_CHECKING:
+    from truthound.schema import Schema
+    from truthound.datasources.base import BaseDataSource
+    from truthound.execution.base import BaseExecutionEngine
+
+
+def check(
+    data: Any = None,
+    source: "BaseDataSource | None" = None,
+    validators: list[str | Validator] | None = None,
+    min_severity: str | Severity | None = None,
+    schema: str | Path | Schema | None = None,
+    auto_schema: bool = False,
+    use_engine: bool = False,
+    parallel: bool = False,
+    max_workers: int | None = None,
+    pushdown: bool | None = None,
+) -> Report:
+    """Perform data quality validation on the input data.
+
+    This is the main entry point for data quality validation. It accepts various
+    input types and automatically converts them to Polars LazyFrame internally.
+
+    Supported Input Types:
+        - str: File path (CSV, JSON, Parquet)
+        - pl.DataFrame: Polars DataFrame (converted to LazyFrame)
+        - pl.LazyFrame: Polars LazyFrame (used directly)
+        - pd.DataFrame: pandas DataFrame (converted via Polars)
+        - dict: Python dictionary (converted to DataFrame then LazyFrame)
+        - BaseDataSource: DataSource instance for SQL databases, Spark, etc.
+
+    Note:
+        Individual Validator classes only accept pl.LazyFrame directly.
+        This API handles the conversion for convenience. If using validators
+        directly, use ``truthound.adapters.to_lazyframe()`` to convert your data.
+
+    Args:
+        data: Input data (file path, DataFrame, dict, etc.)
+        source: Optional DataSource instance. If provided, data is ignored.
+                This enables validation on SQL databases, Spark, etc.
+        validators: Optional list of validator names or Validator instances.
+                   If None, all built-in validators are used.
+        min_severity: Minimum severity level to include in results.
+                     Can be "low", "medium", "high", or "critical".
+        schema: Optional schema for validation. Can be:
+               - Path to a schema YAML file
+               - Schema object from th.learn()
+               When provided, schema validation runs in addition to other validators.
+        auto_schema: If True, automatically learns and caches a schema from the data.
+                    On subsequent runs with the same data source, validates against
+                    the cached schema. This enables true "zero-config" validation.
+        use_engine: If True, uses execution engine for validation (experimental).
+                   Currently validators still use Polars LazyFrame fallback.
+        parallel: If True, uses DAG-based parallel execution for validators.
+                 Validators are grouped by dependency and executed in parallel
+                 when possible. This can significantly improve performance for
+                 large datasets with many validators.
+        max_workers: Maximum number of worker threads for parallel execution.
+                    Only used when parallel=True. Defaults to min(32, cpu_count + 4).
+        pushdown: If True, enables query pushdown for SQL data sources.
+                 Validation logic is executed server-side when possible,
+                 reducing data transfer and improving performance.
+                 If None (default), auto-detects based on data source type.
+
+    Returns:
+        Report containing all validation issues found.
+
+    Example:
+        >>> import truthound as th
+        >>> report = th.check("data.csv")
+        >>> print(report)
+
+        >>> # With specific validators
+        >>> report = th.check(df, validators=["null", "duplicate"])
+
+        >>> # Filter by severity
+        >>> report = th.check(df, min_severity="medium")
+
+        >>> # With schema validation
+        >>> schema = th.learn("baseline.csv")
+        >>> report = th.check("new_data.csv", schema=schema)
+
+        >>> # Zero-config with auto schema caching
+        >>> report = th.check("data.csv", auto_schema=True)
+
+        >>> # Using DataSource for SQL database
+        >>> from truthound.datasources.sql import PostgreSQLDataSource
+        >>> source = PostgreSQLDataSource(
+        ...     table="users",
+        ...     host="localhost",
+        ...     database="mydb",
+        ...     user="postgres",
+        ... )
+        >>> report = th.check(source=source, validators=["null", "duplicate"])
+
+        >>> # Using auto-detection with DataSource
+        >>> from truthound.datasources import get_datasource
+        >>> source = get_datasource(spark_df)  # PySpark DataFrame
+        >>> if source.needs_sampling():
+        ...     source = source.sample(n=100_000)
+        >>> report = th.check(source=source)
+
+        >>> # With query pushdown for SQL data sources
+        >>> from truthound.datasources.sql import PostgreSQLDataSource
+        >>> source = PostgreSQLDataSource(table="users", host="localhost", database="mydb")
+        >>> report = th.check(source=source, pushdown=True)  # Execute validations server-side
+    """
+    # Handle DataSource if provided
+    use_pushdown = False
+    sql_source = None
+
+    if source is not None:
+        from truthound.datasources.base import BaseDataSource
+        from truthound.datasources._protocols import DataSourceCapability
+
+        if not isinstance(source, BaseDataSource):
+            raise ValueError(
+                f"source must be a DataSource instance, got {type(source).__name__}"
+            )
+
+        # Determine if pushdown should be used
+        if pushdown is True:
+            use_pushdown = True
+        elif pushdown is None:
+            # Auto-detect: use pushdown for SQL sources with SQL_PUSHDOWN capability
+            use_pushdown = DataSourceCapability.SQL_PUSHDOWN in source.capabilities
+
+        if use_pushdown:
+            # Verify it's actually a SQL data source
+            try:
+                from truthound.datasources.sql.base import BaseSQLDataSource
+                if isinstance(source, BaseSQLDataSource):
+                    sql_source = source
+                else:
+                    use_pushdown = False
+            except ImportError:
+                use_pushdown = False
+
+        # Check size limits and warn if needed (only if not using pushdown)
+        if not use_pushdown and source.needs_sampling():
+            import warnings
+            warnings.warn(
+                f"Data source '{source.name}' has {source.row_count:,} rows, "
+                f"which exceeds the limit of {source.config.max_rows:,}. "
+                "Consider using source.sample() for better performance.",
+                UserWarning,
+            )
+
+        source_name = source.name
+    else:
+        if data is None:
+            raise ValueError("Either 'data' or 'source' must be provided")
+
+        # Convert input to LazyFrame (legacy path)
+        source_name = str(data) if isinstance(data, str) else type(data).__name__
+
+    # For pushdown path, get metadata without loading all data
+    if use_pushdown and sql_source is not None:
+        row_count = sql_source.row_count or 0
+        column_count = len(sql_source.columns)
+        lf = None  # Will be loaded lazily if needed for non-pushdown validators
+    else:
+        # Standard path: load data into Polars
+        if source is not None:
+            lf = source.to_polars_lazyframe()
+        else:
+            lf = to_lazyframe(data)
+
+        # Collect metadata
+        polars_schema = lf.collect_schema()
+        df_collected = lf.collect()
+        row_count = len(df_collected)
+        column_count = len(polars_schema)
+
+        # Re-create lazy frame after collecting metadata
+        lf = df_collected.lazy()
+
+    # Determine which validators to use
+    validator_instances: list[Validator] = []
+
+    # Add schema validator if schema is provided or auto_schema is enabled
+    if schema is not None or auto_schema:
+        from truthound.schema import Schema as SchemaClass
+        from truthound.validators.schema_validator import SchemaValidator
+
+        if schema is not None:
+            if isinstance(schema, (str, Path)):
+                schema_obj = SchemaClass.load(schema)
+            else:
+                schema_obj = schema
+        else:
+            # Auto schema mode: get from cache or learn new
+            from truthound.cache import get_or_learn_schema
+            schema_obj, was_cached = get_or_learn_schema(data)
+
+        validator_instances.append(SchemaValidator(schema_obj))
+
+    if validators is None:
+        # Use all built-in validators
+        validator_instances.extend([cls() for cls in BUILTIN_VALIDATORS.values()])
+    else:
+        for v in validators:
+            if isinstance(v, str):
+                validator_cls = get_validator(v)
+                validator_instances.append(validator_cls())
+            elif isinstance(v, Validator):
+                validator_instances.append(v)
+            else:
+                raise ValueError(f"Invalid validator: {v}. Expected str or Validator instance.")
+
+    # Run all validators and collect issues
+    all_issues = []
+
+    if use_pushdown and sql_source is not None:
+        # Use pushdown validation engine for SQL data sources
+        from truthound.validators.pushdown_support import PushdownValidationEngine
+
+        engine = PushdownValidationEngine(sql_source)
+        all_issues = engine.validate(validator_instances)
+
+    elif parallel and len(validator_instances) > 1:
+        # Use DAG-based parallel execution
+        from truthound.validators.optimization.orchestrator import (
+            ValidatorDAG,
+            ParallelExecutionStrategy,
+            AdaptiveExecutionStrategy,
+        )
+
+        dag = ValidatorDAG()
+        dag.add_validators(validator_instances)
+        plan = dag.build_execution_plan()
+
+        # Choose strategy based on max_workers
+        if max_workers is not None:
+            strategy = ParallelExecutionStrategy(max_workers=max_workers)
+        else:
+            strategy = AdaptiveExecutionStrategy()
+
+        result = plan.execute(lf, strategy)
+        all_issues = result.all_issues
+    else:
+        # Sequential execution (original behavior)
+        for validator in validator_instances:
+            issues = validator.validate(lf)
+            all_issues.extend(issues)
+
+    # Create report
+    report = Report(
+        issues=all_issues,
+        source=source_name,
+        row_count=row_count,
+        column_count=column_count,
+    )
+
+    # Filter by severity if specified
+    if min_severity is not None:
+        if isinstance(min_severity, str):
+            min_severity = Severity(min_severity.lower())
+        report = report.filter_by_severity(min_severity)
+
+    return report
+
+
+def scan(data: Any) -> PIIReport:
+    """Scan data for personally identifiable information (PII).
+
+    Args:
+        data: Input data (file path, DataFrame, dict, etc.)
+
+    Returns:
+        PIIReport containing all PII findings.
+
+    Example:
+        >>> import truthound as th
+        >>> pii_report = th.scan("data.csv")
+        >>> print(pii_report)
+    """
+    lf = to_lazyframe(data)
+    source = str(data) if isinstance(data, str) else type(data).__name__
+
+    df = lf.collect()
+    row_count = len(df)
+
+    findings = scan_pii(df.lazy())
+
+    return PIIReport(
+        findings=findings,
+        source=source,
+        row_count=row_count,
+    )
+
+
+def mask(
+    data: Any,
+    columns: list[str] | None = None,
+    strategy: str = "redact",
+) -> pl.DataFrame:
+    """Mask sensitive data in the input.
+
+    Args:
+        data: Input data (file path, DataFrame, dict, etc.)
+        columns: Optional list of columns to mask.
+                If None, auto-detects PII columns.
+        strategy: Masking strategy - "redact", "hash", or "fake".
+
+    Returns:
+        Polars DataFrame with masked values.
+
+    Example:
+        >>> import truthound as th
+        >>> masked_df = th.mask("data.csv")
+
+        >>> # Mask specific columns
+        >>> masked_df = th.mask(df, columns=["email", "phone"])
+
+        >>> # Use hash strategy
+        >>> masked_df = th.mask(df, strategy="hash")
+    """
+    lf = to_lazyframe(data)
+    return mask_data(lf, columns=columns, strategy=strategy)
+
+
+def profile(data: Any) -> ProfileReport:
+    """Generate a statistical profile of the dataset.
+
+    Args:
+        data: Input data (file path, DataFrame, dict, etc.)
+
+    Returns:
+        ProfileReport containing statistical summary.
+
+    Example:
+        >>> import truthound as th
+        >>> profile = th.profile("data.csv")
+        >>> print(profile)
+    """
+    lf = to_lazyframe(data)
+    source = str(data) if isinstance(data, str) else type(data).__name__
+
+    profile_dict = profile_data(lf, source=source)
+
+    return ProfileReport(
+        source=profile_dict["source"],
+        row_count=profile_dict["row_count"],
+        column_count=profile_dict["column_count"],
+        size_bytes=profile_dict["size_bytes"],
+        columns=profile_dict["columns"],
+    )