truthound-dashboard 1.3.0__py3-none-any.whl → 1.4.0__py3-none-any.whl

truthound_dashboard/api/sources.py

@@ -221,9 +221,53 @@ async def test_source_connection(
 async def get_supported_types() -> dict:
     """Get list of supported source types.
 
+    Returns comprehensive information about each source type including
+    field definitions for dynamic form rendering.
+
+    Returns:
+        List of supported source types with field definitions.
+    """
+    from truthound_dashboard.core.connections import (
+        get_source_type_categories,
+        get_supported_source_types,
+    )
+
+    return {
+        "success": True,
+        "data": {
+            "types": get_supported_source_types(),
+            "categories": get_source_type_categories(),
+        },
+    }
+
+
+@router.post(
+    "/test-connection",
+    response_model=dict,
+    summary="Test connection configuration",
+    description="Test a connection configuration before creating a source",
+)
+async def test_connection_config(
+    request: dict,
+) -> dict:
+    """Test connection configuration before creating a source.
+
+    This endpoint allows testing connection settings without
+    persisting them to the database.
+
+    Args:
+        request: Dictionary with 'type' and 'config' keys.
+
     Returns:
-        List of supported source types with required/optional fields.
+        Connection test result with success status and message.
     """
-    from truthound_dashboard.core.connections import get_supported_source_types
+    from truthound_dashboard.core.connections import test_connection
+
+    source_type = request.get("type")
+    config = request.get("config", {})
 
-    return {"success": True, "data": get_supported_source_types()}
+    if not source_type:
+        raise HTTPException(status_code=400, detail="Source type is required")
+
+    result = await test_connection(source_type, config)
+    return {"success": True, "data": result}

truthound_dashboard/api/triggers.py

@@ -0,0 +1,190 @@
+"""Trigger monitoring and webhook API endpoints.
+
+This module provides endpoints for:
+- Trigger monitoring status
+- Webhook trigger reception
+- Trigger status queries
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from fastapi import APIRouter, Header, HTTPException
+
+from truthound_dashboard.core.scheduler import get_scheduler
+from truthound_dashboard.schemas.triggers import (
+    TriggerCheckStatus,
+    TriggerMonitoringResponse,
+    TriggerMonitoringStats,
+    WebhookTriggerRequest,
+    WebhookTriggerResponse,
+)
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/triggers", tags=["triggers"])
+
+
+@router.get(
+    "/monitoring",
+    response_model=TriggerMonitoringResponse,
+    summary="Get trigger monitoring status",
+)
+async def get_trigger_monitoring() -> TriggerMonitoringResponse:
+    """Get current trigger monitoring status.
+
+    Returns:
+        Trigger monitoring stats and schedule statuses.
+    """
+    scheduler = get_scheduler()
+    status = scheduler.get_trigger_monitoring_status()
+    schedules = await scheduler.get_trigger_check_statuses()
+
+    # Calculate aggregate stats
+    active_data_change = sum(
+        1 for s in schedules if s["trigger_type"] == "data_change"
+    )
+    active_webhook = sum(
+        1 for s in schedules if s["trigger_type"] == "webhook"
+    )
+    active_composite = sum(
+        1 for s in schedules if s["trigger_type"] == "composite"
+    )
+
+    # Calculate average check interval
+    check_intervals = [
+        s.get("check_interval_minutes", 5) * 60 for s in schedules
+    ]
+    avg_interval = (
+        sum(check_intervals) / len(check_intervals)
+        if check_intervals
+        else 300.0
+    )
+
+    # Find next scheduled check
+    next_checks = [
+        s["next_check_at"] for s in schedules if s.get("next_check_at")
+    ]
+    next_check = min(next_checks) if next_checks else None
+
+    stats = TriggerMonitoringStats(
+        total_schedules=len(schedules),
+        active_data_change_triggers=active_data_change,
+        active_webhook_triggers=active_webhook,
+        active_composite_triggers=active_composite,
+        total_checks_last_hour=status.get("checks_last_hour", 0),
+        total_triggers_last_hour=status.get("triggers_last_hour", 0),
+        average_check_interval_seconds=avg_interval,
+        next_scheduled_check_at=next_check,
+    )
+
+    schedule_statuses = [
+        TriggerCheckStatus(**s) for s in schedules
+    ]
+
+    return TriggerMonitoringResponse(
+        stats=stats,
+        schedules=schedule_statuses,
+        checker_running=status.get("checker_running", False),
+        checker_interval_seconds=status.get("checker_interval_seconds", 300),
+        last_checker_run_at=status.get("last_checker_run_at"),
+    )
+
+
+@router.get(
+    "/schedules/{schedule_id}/status",
+    summary="Get trigger status for a specific schedule",
+)
+async def get_schedule_trigger_status(schedule_id: str) -> dict[str, Any]:
+    """Get trigger status for a specific schedule.
+
+    Args:
+        schedule_id: Schedule ID to query.
+
+    Returns:
+        Trigger status for the schedule.
+    """
+    scheduler = get_scheduler()
+    schedules = await scheduler.get_trigger_check_statuses()
+
+    for schedule in schedules:
+        if schedule["schedule_id"] == schedule_id:
+            return schedule
+
+    raise HTTPException(
+        status_code=404,
+        detail=f"Schedule {schedule_id} not found or not using monitored trigger type",
+    )
+
+
+@router.post(
+    "/webhook",
+    response_model=WebhookTriggerResponse,
+    summary="Receive webhook trigger",
+)
+async def receive_webhook(
+    request: WebhookTriggerRequest,
+    x_webhook_signature: str | None = Header(
+        default=None, description="HMAC-SHA256 signature for verification"
+    ),
+) -> WebhookTriggerResponse:
+    """Receive and process an incoming webhook trigger.
+
+    This endpoint is called by external systems (Airflow, Dagster, Prefect, etc.)
+    to trigger validations when data pipelines complete.
+
+    Args:
+        request: Webhook trigger request.
+        x_webhook_signature: Optional signature for verification.
+
+    Returns:
+        Webhook trigger response with triggered schedules.
+    """
+    scheduler = get_scheduler()
+
+    result = await scheduler.trigger_webhook(
+        source=request.source,
+        event_type=request.event_type,
+        payload=request.payload,
+        schedule_id=request.schedule_id,
+        source_id=request.source_id,
+        signature=x_webhook_signature,
+    )
+
+    return WebhookTriggerResponse(
+        accepted=result["accepted"],
+        triggered_schedules=result["triggered_schedules"],
+        message=result["message"],
+        request_id=result["request_id"],
+    )
+
+
+@router.post(
+    "/webhook/test",
+    summary="Test webhook configuration",
+)
+async def test_webhook(
+    source: str = "test",
+    event_type: str = "test_event",
+) -> dict[str, Any]:
+    """Test webhook endpoint without triggering any schedules.
+
+    Useful for verifying connectivity and configuration.
+
+    Args:
+        source: Test source name.
+        event_type: Test event type.
+
+    Returns:
+        Test result.
+    """
+    return {
+        "success": True,
+        "message": "Webhook endpoint is accessible",
+        "received": {
+            "source": source,
+            "event_type": event_type,
+        },
+    }

truthound_dashboard/api/validations.py

@@ -71,10 +71,23 @@ async def run_validation(
             # Simple mode: use validator names list (backward compatible)
             validators = request.validators
 
+        # Convert custom validators to internal format
+        custom_validators = None
+        if request.custom_validators:
+            custom_validators = [
+                {
+                    "validator_id": cv.validator_id,
+                    "column": cv.column,
+                    "params": cv.params or {},
+                }
+                for cv in request.custom_validators
+            ]
+
         validation = await service.run_validation(
             source_id,
             validators=validators,
             validator_params=validator_params,
+            custom_validators=custom_validators,
             schema_path=request.schema_path,
             auto_schema=request.auto_schema,
             columns=request.columns,

truthound_dashboard/api/validators.py

@@ -1,29 +1,50 @@
 """Validators API endpoints.
 
 This module provides API endpoints for validator discovery and configuration.
+Includes both built-in truthound validators and user-defined custom validators.
 """
 
 from __future__ import annotations
 
-from fastapi import APIRouter, Query
+import logging
+from collections import defaultdict
+from typing import Annotated, Any
+
+from fastapi import APIRouter, Depends, HTTPException, Path, Query, status
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from truthound_dashboard.core.plugins import CustomValidatorExecutor
+from truthound_dashboard.core.plugins.registry import plugin_registry
+from truthound_dashboard.core.plugins.validator_executor import ValidatorContext
 
 from ..schemas.validators import (
     VALIDATOR_REGISTRY,
+    CustomValidatorExecuteRequest,
+    CustomValidatorExecuteResponse,
+    UnifiedValidatorDefinition,
+    UnifiedValidatorListResponse,
     ValidatorCategory,
     ValidatorDefinition,
+    ValidatorSource,
     get_validator_by_name,
     get_validators_by_category,
     search_validators,
 )
+from .deps import SourceServiceDep, get_session
+
+logger = logging.getLogger(__name__)
 
 router = APIRouter()
 
+# Dependencies
+SessionDep = Annotated[AsyncSession, Depends(get_session)]
+
 
 @router.get(
     "/validators",
     response_model=list[ValidatorDefinition],
-    summary="List all validators",
-    description="Returns all available validators with their parameter definitions.",
+    summary="List built-in validators",
+    description="Returns all built-in validators with their parameter definitions.",
 )
 async def list_validators(
     category: ValidatorCategory | None = Query(
@@ -33,7 +54,7 @@ async def list_validators(
         default=None, description="Search by name, description, or tags"
     ),
 ) -> list[ValidatorDefinition]:
-    """List all validators, optionally filtered.
+    """List built-in validators, optionally filtered.
 
     Args:
         category: Optional category filter.
@@ -49,6 +70,122 @@ async def list_validators(
     return VALIDATOR_REGISTRY
 
 
+@router.get(
+    "/validators/unified",
+    response_model=UnifiedValidatorListResponse,
+    summary="List all validators (built-in + custom)",
+    description="Returns unified list of both built-in and custom validators.",
+)
+async def list_unified_validators(
+    session: SessionDep,
+    category: str | None = Query(
+        default=None, description="Filter by category"
+    ),
+    source: ValidatorSource | None = Query(
+        default=None, description="Filter by source (builtin or custom)"
+    ),
+    search: str | None = Query(
+        default=None, description="Search by name, description, or tags"
+    ),
+    enabled_only: bool = Query(
+        default=False, description="Only return enabled validators"
+    ),
+    offset: int = Query(default=0, ge=0),
+    limit: int = Query(default=100, ge=1, le=500),
+) -> UnifiedValidatorListResponse:
+    """List all validators (built-in + custom).
+
+    This endpoint provides a unified view of all available validators,
+    combining truthound's built-in validators with user-defined custom validators.
+
+    Args:
+        session: Database session.
+        category: Optional category filter.
+        source: Optional source filter (builtin or custom).
+        search: Optional search query.
+        enabled_only: Only return enabled validators.
+        offset: Pagination offset.
+        limit: Pagination limit.
+
+    Returns:
+        Unified list of validators with metadata.
+    """
+    unified_validators: list[UnifiedValidatorDefinition] = []
+
+    # 1. Get built-in validators
+    builtin_count = 0
+    if source is None or source == ValidatorSource.BUILTIN:
+        builtin_validators = VALIDATOR_REGISTRY
+
+        # Apply filters
+        if search:
+            search_lower = search.lower()
+            builtin_validators = [
+                v for v in builtin_validators
+                if (
+                    search_lower in v.name.lower()
+                    or search_lower in v.display_name.lower()
+                    or search_lower in v.description.lower()
+                    or any(search_lower in tag.lower() for tag in v.tags)
+                )
+            ]
+
+        if category:
+            builtin_validators = [
+                v for v in builtin_validators
+                if v.category.value == category
+            ]
+
+        for v in builtin_validators:
+            unified_validators.append(UnifiedValidatorDefinition.from_builtin(v))
+        builtin_count = len(builtin_validators)
+
+    # 2. Get custom validators
+    custom_count = 0
+    if source is None or source == ValidatorSource.CUSTOM:
+        custom_validators, total_custom = await plugin_registry.list_validators(
+            session=session,
+            category=category,
+            enabled_only=enabled_only,
+            search=search,
+            offset=0,
+            limit=500,  # Get all for now, apply pagination later
+        )
+        for cv in custom_validators:
+            unified_validators.append(UnifiedValidatorDefinition.from_custom(cv))
+        custom_count = len(custom_validators)
+
+    # Calculate category summary
+    category_counts: dict[str, dict[str, int]] = defaultdict(
+        lambda: {"builtin": 0, "custom": 0}
+    )
+    for v in unified_validators:
+        category_counts[v.category][v.source.value] += 1
+
+    categories = [
+        {
+            "name": cat,
+            "label": cat.replace("_", " ").title(),
+            "builtin_count": counts["builtin"],
+            "custom_count": counts["custom"],
+            "total": counts["builtin"] + counts["custom"],
+        }
+        for cat, counts in sorted(category_counts.items())
+    ]
+
+    # Apply pagination
+    total = len(unified_validators)
+    paginated = unified_validators[offset : offset + limit]
+
+    return UnifiedValidatorListResponse(
+        data=paginated,
+        total=total,
+        builtin_count=builtin_count,
+        custom_count=custom_count,
+        categories=categories,
+    )
+
+
 @router.get(
     "/validators/categories",
     response_model=list[dict[str, str]],
@@ -83,3 +220,195 @@ async def get_validator(name: str) -> ValidatorDefinition | None:
         Validator definition if found.
     """
     return get_validator_by_name(name)
+
+
+# =============================================================================
+# Custom Validator Execution Endpoints
+# =============================================================================
+
+
+@router.post(
+    "/validators/custom/{validator_id}/execute",
+    response_model=CustomValidatorExecuteResponse,
+    summary="Execute custom validator",
+    description="Execute a custom validator against a data source column.",
+)
+async def execute_custom_validator(
+    session: SessionDep,
+    source_service: SourceServiceDep,
+    validator_id: Annotated[str, Path(description="Custom validator ID")],
+    request: CustomValidatorExecuteRequest,
+) -> CustomValidatorExecuteResponse:
+    """Execute a custom validator on a specific data source and column.
+
+    This endpoint allows direct execution of a custom validator without
+    going through the full validation pipeline. Useful for testing and
+    one-off validations.
+
+    Args:
+        session: Database session.
+        source_service: Source service for data access.
+        validator_id: ID of the custom validator.
+        request: Execution request with source_id, column_name, and params.
+
+    Returns:
+        Execution result with validation status and issues.
+
+    Raises:
+        HTTPException: 404 if validator or source not found.
+    """
+    # Get the custom validator
+    validator = await plugin_registry.get_validator(session, validator_id=validator_id)
+    if not validator:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Custom validator {validator_id} not found",
+        )
+
+    if not validator.is_enabled:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"Custom validator {validator.name} is disabled",
+        )
+
+    # Get the data source
+    source = await source_service.get_by_id(request.source_id)
+    if not source:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Data source {request.source_id} not found",
+        )
+
+    # Load data from source
+    try:
+        import polars as pl
+
+        # Read data based on source type
+        if source.type == "csv":
+            df = pl.read_csv(source.path)
+        elif source.type == "parquet":
+            df = pl.read_parquet(source.path)
+        elif source.type == "json":
+            df = pl.read_json(source.path)
+        else:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=f"Unsupported source type: {source.type}",
+            )
+
+        # Check if column exists
+        if request.column_name not in df.columns:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=f"Column '{request.column_name}' not found in source",
+            )
+
+        # Apply sample size if specified
+        if request.sample_size and request.sample_size < len(df):
+            df = df.sample(request.sample_size)
+
+        # Get column values
+        column_values = df[request.column_name].to_list()
+
+        # Get column schema info
+        column_schema = {
+            "dtype": str(df[request.column_name].dtype),
+            "null_count": df[request.column_name].null_count(),
+        }
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Failed to load data from source: {e}")
+        return CustomValidatorExecuteResponse(
+            success=False,
+            error=f"Failed to load data: {str(e)}",
+        )
+
+    # Create execution context
+    context = ValidatorContext(
+        column_name=request.column_name,
+        column_values=column_values,
+        parameters=request.param_values,
+        schema=column_schema,
+        row_count=len(column_values),
+    )
+
+    # Execute the validator
+    executor = CustomValidatorExecutor()
+    result = await executor.execute(
+        validator=validator,
+        context=context,
+        session=session,
+        source_id=request.source_id,
+    )
+
+    await session.commit()
+
+    return CustomValidatorExecuteResponse(
+        success=True,
+        passed=result.passed,
+        execution_time_ms=result.execution_time_ms,
+        issues=result.issues,
+        message=result.message,
+        details=result.details,
+    )
+
+
+@router.post(
+    "/validators/custom/{validator_id}/execute-preview",
+    response_model=CustomValidatorExecuteResponse,
+    summary="Preview custom validator execution",
+    description="Execute a custom validator on sample data for preview.",
+)
+async def preview_custom_validator_execution(
+    session: SessionDep,
+    validator_id: Annotated[str, Path(description="Custom validator ID")],
+    test_data: dict[str, Any],
+) -> CustomValidatorExecuteResponse:
+    """Preview custom validator execution with provided test data.
+
+    This endpoint allows testing a saved custom validator with arbitrary
+    test data without needing a data source.
+
+    Args:
+        session: Database session.
+        validator_id: ID of the custom validator.
+        test_data: Test data containing column_name, values, and params.
+
+    Returns:
+        Execution result with validation status and issues.
+    """
+    # Get the custom validator
+    validator = await plugin_registry.get_validator(session, validator_id=validator_id)
+    if not validator:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Custom validator {validator_id} not found",
+        )
+
+    # Create context from test data
+    context = ValidatorContext(
+        column_name=test_data.get("column_name", "test_column"),
+        column_values=test_data.get("values", []),
+        parameters=test_data.get("params", {}),
+        schema=test_data.get("schema", {}),
+        row_count=len(test_data.get("values", [])),
+    )
+
+    # Execute the validator (without logging)
+    executor = CustomValidatorExecutor(log_executions=False)
+    result = await executor.execute(
+        validator=validator,
+        context=context,
+        session=None,
+    )
+
+    return CustomValidatorExecuteResponse(
+        success=True,
+        passed=result.passed,
+        execution_time_ms=result.execution_time_ms,
+        issues=result.issues,
+        message=result.message,
+        details=result.details,
+    )