truthound-dashboard 1.2.1__py3-none-any.whl → 1.3.0__py3-none-any.whl

truthound_dashboard/api/deps.py

@@ -22,6 +22,8 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from truthound_dashboard.core import (
     DriftService,
     HistoryService,
+    MaskService,
+    PIIScanService,
     ProfileService,
     RuleService,
     ScheduleService,
@@ -142,6 +144,30 @@ async def get_schedule_service(session: SessionDep) -> ScheduleService:
     return ScheduleService(session)
 
 
+async def get_pii_scan_service(session: SessionDep) -> PIIScanService:
+    """Get PII scan service dependency.
+
+    Args:
+        session: Database session.
+
+    Returns:
+        PIIScanService instance.
+    """
+    return PIIScanService(session)
+
+
+async def get_mask_service(session: SessionDep) -> MaskService:
+    """Get mask service dependency.
+
+    Args:
+        session: Database session.
+
+    Returns:
+        MaskService instance.
+    """
+    return MaskService(session)
+
+
 # Type aliases for service dependencies
 SourceServiceDep = Annotated[SourceService, Depends(get_source_service)]
 ValidationServiceDep = Annotated[ValidationService, Depends(get_validation_service)]
@@ -151,3 +177,5 @@ RuleServiceDep = Annotated[RuleService, Depends(get_rule_service)]
 HistoryServiceDep = Annotated[HistoryService, Depends(get_history_service)]
 DriftServiceDep = Annotated[DriftService, Depends(get_drift_service)]
 ScheduleServiceDep = Annotated[ScheduleService, Depends(get_schedule_service)]
+PIIScanServiceDep = Annotated[PIIScanService, Depends(get_pii_scan_service)]
+MaskServiceDep = Annotated[MaskService, Depends(get_mask_service)]

truthound_dashboard/api/drift.py

@@ -54,6 +54,7 @@ async def compare_datasets(
             columns=request.columns,
             method=request.method,
             threshold=request.threshold,
+            correction=request.correction,
             sample_size=request.sample_size,
         )
 

truthound_dashboard/api/mask.py

@@ -0,0 +1,164 @@
+"""Data masking API endpoints.
+
+Provides endpoints for running th.mask() operations with three strategies:
+- redact: Replace values with asterisks
+- hash: Replace values with SHA256 hash
+- fake: Replace values with realistic fake data
+"""
+
+from __future__ import annotations
+
+from fastapi import APIRouter, HTTPException, status
+
+from truthound_dashboard.schemas import (
+    MaskListItem,
+    MaskListResponse,
+    MaskRequest,
+    MaskResponse,
+)
+
+from .deps import MaskServiceDep
+
+router = APIRouter(prefix="/masks", tags=["masks"])
+
+
+@router.post(
+    "/sources/{source_id}/mask",
+    response_model=MaskResponse,
+    status_code=status.HTTP_201_CREATED,
+    summary="Run data masking",
+    description="""
+Run data masking on a source using th.mask().
+
+Supports three masking strategies:
+- **redact**: Replace values with asterisks (default)
+- **hash**: Replace values with SHA256 hash (deterministic, can be used for joins)
+- **fake**: Replace values with realistic fake data
+
+If `columns` is not specified, PII columns are auto-detected.
+""",
+)
+async def run_mask(
+    source_id: str,
+    request: MaskRequest,
+    service: MaskServiceDep,
+) -> MaskResponse:
+    """Run data masking on a source.
+
+    Args:
+        source_id: Source ID to mask.
+        request: Masking options.
+        service: Mask service dependency.
+
+    Returns:
+        MaskResponse with operation details.
+
+    Raises:
+        HTTPException: If source not found or masking fails.
+    """
+    try:
+        mask = await service.run_mask(
+            source_id,
+            columns=request.columns,
+            strategy=request.strategy,
+            output_format=request.output_format,
+        )
+        return MaskResponse.from_db(mask)
+    except ValueError as e:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=str(e),
+        ) from e
+    except Exception as e:
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Masking failed: {e}",
+        ) from e
+
+
+@router.get(
+    "/{mask_id}",
+    response_model=MaskResponse,
+    summary="Get mask operation by ID",
+)
+async def get_mask(
+    mask_id: str,
+    service: MaskServiceDep,
+) -> MaskResponse:
+    """Get a masking operation by ID.
+
+    Args:
+        mask_id: Mask operation ID.
+        service: Mask service dependency.
+
+    Returns:
+        MaskResponse with operation details.
+
+    Raises:
+        HTTPException: If mask operation not found.
+    """
+    mask = await service.get_mask(mask_id)
+    if mask is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"Mask operation '{mask_id}' not found",
+        )
+    return MaskResponse.from_db(mask)
+
+
+@router.get(
+    "/sources/{source_id}/masks",
+    response_model=MaskListResponse,
+    summary="List mask operations for a source",
+)
+async def list_masks(
+    source_id: str,
+    service: MaskServiceDep,
+    limit: int = 20,
+) -> MaskListResponse:
+    """List masking operations for a source.
+
+    Args:
+        source_id: Source ID.
+        service: Mask service dependency.
+        limit: Maximum number of results (default: 20).
+
+    Returns:
+        MaskListResponse with list of operations.
+    """
+    masks = await service.list_for_source(source_id, limit=limit)
+    return MaskListResponse(
+        data=[MaskListItem.from_db(m) for m in masks],
+        total=len(masks),
+        limit=limit,
+    )
+
+
+@router.get(
+    "/sources/{source_id}/masks/latest",
+    response_model=MaskResponse,
+    summary="Get latest mask operation for a source",
+)
+async def get_latest_mask(
+    source_id: str,
+    service: MaskServiceDep,
+) -> MaskResponse:
+    """Get the most recent masking operation for a source.
+
+    Args:
+        source_id: Source ID.
+        service: Mask service dependency.
+
+    Returns:
+        MaskResponse with operation details.
+
+    Raises:
+        HTTPException: If no mask operations found for source.
+    """
+    mask = await service.get_latest_for_source(source_id)
+    if mask is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail=f"No mask operations found for source '{source_id}'",
+        )
+    return MaskResponse.from_db(mask)

truthound_dashboard/api/profile.py

@@ -9,7 +9,7 @@ from typing import Annotated
 
 from fastapi import APIRouter, HTTPException, Path
 
-from truthound_dashboard.schemas import ProfileResponse
+from truthound_dashboard.schemas import ProfileRequest, ProfileResponse
 
 from .deps import ProfileServiceDep, SourceServiceDep
 
@@ -20,12 +20,13 @@ router = APIRouter()
     "/sources/{source_id}/profile",
     response_model=ProfileResponse,
     summary="Profile source",
-    description="Run data profiling on a source",
+    description="Run data profiling on a source with optional sampling",
 )
 async def profile_source(
     service: ProfileServiceDep,
     source_service: SourceServiceDep,
     source_id: Annotated[str, Path(description="Source ID to profile")],
+    request: ProfileRequest | None = None,
 ) -> ProfileResponse:
     """Run data profiling on a source.
 
@@ -33,6 +34,7 @@ async def profile_source(
         service: Injected profile service.
         source_service: Injected source service.
         source_id: Source to profile.
+        request: Optional profiling configuration with sample_size.
 
     Returns:
         Profiling result with column statistics.
@@ -45,8 +47,14 @@ async def profile_source(
     if source is None:
         raise HTTPException(status_code=404, detail="Source not found")
 
+    # Extract sample_size from request if provided
+    sample_size = request.sample_size if request else None
+
     try:
-        result = await service.profile_source(source_id)
+        result = await service.profile_source(
+            source_id,
+            sample_size=sample_size,
+        )
         return ProfileResponse.from_result(result)
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))

truthound_dashboard/api/router.py

@@ -10,13 +10,16 @@ from . import (
     drift,
     health,
     history,
+    mask,
     notifications,
     profile,
     rules,
+    scan,
     schedules,
     schemas,
     sources,
     validations,
+    validators,
     # Phase 5
     catalog,
     collaboration,
@@ -57,6 +60,12 @@ api_router.include_router(
     tags=["validations"],
 )
 
+# Validator registry endpoints
+api_router.include_router(
+    validators.router,
+    tags=["validators"],
+)
+
 # Profiling endpoints
 api_router.include_router(
     profile.router,
@@ -75,6 +84,19 @@ api_router.include_router(
     tags=["drift"],
 )
 
+# PII scan endpoints
+api_router.include_router(
+    scan.router,
+    prefix="/scans",
+    tags=["pii-scan"],
+)
+
+# Data masking endpoints
+api_router.include_router(
+    mask.router,
+    tags=["masks"],
+)
+
 # Schedule management endpoints (Phase 2)
 api_router.include_router(
     schedules.router,

truthound_dashboard/api/scan.py

@@ -0,0 +1,168 @@
+"""PII scan API endpoints.
+
+This module provides endpoints for running PII scans using th.scan().
+Detects personally identifiable information and checks compliance with
+privacy regulations (GDPR, CCPA, LGPD).
+"""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+from fastapi import APIRouter, HTTPException, Path, Query
+
+from truthound_dashboard.schemas import (
+    PIIScanListItem,
+    PIIScanListResponse,
+    PIIScanRequest,
+    PIIScanResponse,
+)
+
+from .deps import PIIScanServiceDep, SourceServiceDep
+
+router = APIRouter()
+
+
+@router.post(
+    "/sources/{source_id}/scan",
+    response_model=PIIScanResponse,
+    summary="Run PII scan",
+    description="Scan data source for personally identifiable information (PII)",
+)
+async def run_pii_scan(
+    service: PIIScanServiceDep,
+    source_id: Annotated[str, Path(description="Source ID to scan")],
+    request: PIIScanRequest,
+) -> PIIScanResponse:
+    """Run PII scan on a data source.
+
+    Supports all th.scan() parameters for maximum flexibility:
+    - columns: Specific columns to scan
+    - regulations: Privacy regulations to check (gdpr, ccpa, lgpd)
+    - min_confidence: Confidence threshold for PII detection
+
+    Args:
+        service: Injected PII scan service.
+        source_id: Source to scan.
+        request: Scan options.
+
+    Returns:
+        PII scan result with findings and violations.
+
+    Raises:
+        HTTPException: 404 if source not found.
+    """
+    try:
+        scan = await service.run_scan(
+            source_id,
+            columns=request.columns,
+            regulations=request.regulations,
+            min_confidence=request.min_confidence,
+        )
+        return PIIScanResponse.from_model(scan)
+    except ValueError as e:
+        raise HTTPException(status_code=404, detail=str(e))
+
+
+@router.get(
+    "/{scan_id}",
+    response_model=PIIScanResponse,
+    summary="Get PII scan",
+    description="Get a specific PII scan result by ID",
+)
+async def get_pii_scan(
+    service: PIIScanServiceDep,
+    scan_id: Annotated[str, Path(description="Scan ID")],
+) -> PIIScanResponse:
+    """Get a specific PII scan result.
+
+    Args:
+        service: Injected PII scan service.
+        scan_id: Scan unique identifier.
+
+    Returns:
+        PII scan details with findings and violations.
+
+    Raises:
+        HTTPException: 404 if scan not found.
+    """
+    scan = await service.get_scan(scan_id)
+    if scan is None:
+        raise HTTPException(status_code=404, detail="PII scan not found")
+    return PIIScanResponse.from_model(scan)
+
+
+@router.get(
+    "/sources/{source_id}/scans",
+    response_model=PIIScanListResponse,
+    summary="List source PII scans",
+    description="Get PII scan history for a source",
+)
+async def list_source_pii_scans(
+    service: PIIScanServiceDep,
+    source_service: SourceServiceDep,
+    source_id: Annotated[str, Path(description="Source ID")],
+    limit: Annotated[int, Query(ge=1, le=100, description="Maximum items")] = 20,
+) -> PIIScanListResponse:
+    """List PII scan history for a source.
+
+    Args:
+        service: Injected PII scan service.
+        source_service: Injected source service.
+        source_id: Source to get scans for.
+        limit: Maximum scans to return.
+
+    Returns:
+        List of PII scan summaries.
+
+    Raises:
+        HTTPException: 404 if source not found.
+    """
+    # Verify source exists
+    source = await source_service.get_by_id(source_id)
+    if source is None:
+        raise HTTPException(status_code=404, detail="Source not found")
+
+    scans = await service.list_for_source(source_id, limit=limit)
+
+    return PIIScanListResponse(
+        data=[PIIScanListItem.from_model(s) for s in scans],
+        total=len(scans),
+        limit=limit,
+    )
+
+
+@router.get(
+    "/sources/{source_id}/scans/latest",
+    response_model=PIIScanResponse,
+    summary="Get latest PII scan",
+    description="Get the most recent PII scan for a source",
+)
+async def get_latest_pii_scan(
+    service: PIIScanServiceDep,
+    source_service: SourceServiceDep,
+    source_id: Annotated[str, Path(description="Source ID")],
+) -> PIIScanResponse:
+    """Get the most recent PII scan for a source.
+
+    Args:
+        service: Injected PII scan service.
+        source_service: Injected source service.
+        source_id: Source to get latest scan for.
+
+    Returns:
+        Latest PII scan result.
+
+    Raises:
+        HTTPException: 404 if source or scan not found.
+    """
+    # Verify source exists
+    source = await source_service.get_by_id(source_id)
+    if source is None:
+        raise HTTPException(status_code=404, detail="Source not found")
+
+    scan = await service.get_latest_for_source(source_id)
+    if scan is None:
+        raise HTTPException(status_code=404, detail="No PII scans found for source")
+
+    return PIIScanResponse.from_model(scan)

truthound_dashboard/api/schemas.py

@@ -59,7 +59,7 @@ async def get_schema(
     "/sources/{source_id}/learn",
     response_model=SchemaResponse,
     summary="Learn schema",
-    description="Auto-learn schema from data source",
+    description="Auto-learn schema from data source with customizable inference options",
 )
 async def learn_schema(
     service: SchemaServiceDep,
@@ -69,17 +69,24 @@ async def learn_schema(
 ) -> SchemaResponse:
     """Learn schema from a data source.
 
+    Analyzes the data source and generates a schema with inferred types
+    and optional constraints. Supports customization of categorical detection
+    and sampling for large datasets.
+
     Args:
         service: Injected schema service.
         source_service: Injected source service.
         source_id: Source to learn schema from.
-        request: Learning options.
+        request: Learning options including:
+            - infer_constraints: Whether to infer min/max and allowed values
+            - categorical_threshold: Max unique values for categorical detection
+            - sample_size: Number of rows to sample for large datasets
 
     Returns:
-        Learned schema.
+        Learned schema with column types and constraints.
 
     Raises:
-        HTTPException: 404 if source not found.
+        HTTPException: 404 if source not found, 500 on learning error.
     """
     # Verify source exists
     source = await source_service.get_by_id(source_id)
@@ -90,6 +97,8 @@ async def learn_schema(
         schema = await service.learn_schema(
             source_id,
             infer_constraints=request.infer_constraints,
+            categorical_threshold=request.categorical_threshold,
+            sample_size=request.sample_size,
         )
         return SchemaResponse.from_model(schema)
     except Exception as e:

truthound_dashboard/api/validations.py

@@ -15,6 +15,7 @@ from truthound_dashboard.schemas import (
     ValidationResponse,
     ValidationRunRequest,
 )
+from truthound_dashboard.schemas.validators import configs_to_truthound_format
 
 from .deps import SourceServiceDep, ValidationServiceDep
 
@@ -34,6 +35,17 @@ async def run_validation(
 ) -> ValidationResponse:
     """Run validation on a data source.
 
+    Supports all th.check() parameters for maximum flexibility:
+    - validators: Specific validators to run
+    - schema_path: Path to schema YAML file
+    - auto_schema: Auto-learn and cache schema
+    - columns: Specific columns to validate
+    - min_severity: Minimum severity to report
+    - strict: Raise exception on failures
+    - parallel: Use parallel execution
+    - max_workers: Max threads for parallel
+    - pushdown: Enable query pushdown for SQL
+
     Args:
         service: Injected validation service.
         source_id: Source to validate.
@@ -46,11 +58,31 @@ async def run_validation(
         HTTPException: 404 if source not found.
     """
     try:
+        # Determine validators and params based on request mode
+        validators: list[str] | None = None
+        validator_params: dict | None = None
+
+        if request.validator_configs:
+            # Advanced mode: use validator_configs (takes precedence)
+            validators, validator_params = configs_to_truthound_format(
+                request.validator_configs
+            )
+        elif request.validators:
+            # Simple mode: use validator names list (backward compatible)
+            validators = request.validators
+
         validation = await service.run_validation(
             source_id,
-            validators=request.validators,
+            validators=validators,
+            validator_params=validator_params,
             schema_path=request.schema_path,
             auto_schema=request.auto_schema,
+            columns=request.columns,
+            min_severity=request.min_severity,
+            strict=request.strict,
+            parallel=request.parallel,
+            max_workers=request.max_workers,
+            pushdown=request.pushdown,
         )
         return ValidationResponse.from_model(validation)
     except ValueError as e:

truthound_dashboard/api/validators.py

@@ -0,0 +1,85 @@
+"""Validators API endpoints.
+
+This module provides API endpoints for validator discovery and configuration.
+"""
+
+from __future__ import annotations
+
+from fastapi import APIRouter, Query
+
+from ..schemas.validators import (
+    VALIDATOR_REGISTRY,
+    ValidatorCategory,
+    ValidatorDefinition,
+    get_validator_by_name,
+    get_validators_by_category,
+    search_validators,
+)
+
+router = APIRouter()
+
+
+@router.get(
+    "/validators",
+    response_model=list[ValidatorDefinition],
+    summary="List all validators",
+    description="Returns all available validators with their parameter definitions.",
+)
+async def list_validators(
+    category: ValidatorCategory | None = Query(
+        default=None, description="Filter by category"
+    ),
+    search: str | None = Query(
+        default=None, description="Search by name, description, or tags"
+    ),
+) -> list[ValidatorDefinition]:
+    """List all validators, optionally filtered.
+
+    Args:
+        category: Optional category filter.
+        search: Optional search query.
+
+    Returns:
+        List of validator definitions.
+    """
+    if search:
+        return search_validators(search)
+    if category:
+        return get_validators_by_category(category)
+    return VALIDATOR_REGISTRY
+
+
+@router.get(
+    "/validators/categories",
+    response_model=list[dict[str, str]],
+    summary="List validator categories",
+    description="Returns all validator categories with their labels.",
+)
+async def list_categories() -> list[dict[str, str]]:
+    """List all validator categories.
+
+    Returns:
+        List of category objects with value and label.
+    """
+    return [
+        {"value": c.value, "label": c.value.replace("_", " ").title()}
+        for c in ValidatorCategory
+    ]
+
+
+@router.get(
+    "/validators/{name}",
+    response_model=ValidatorDefinition | None,
+    summary="Get validator by name",
+    description="Returns a single validator definition by its name.",
+)
+async def get_validator(name: str) -> ValidatorDefinition | None:
+    """Get a validator by name.
+
+    Args:
+        name: Validator name.
+
+    Returns:
+        Validator definition if found.
+    """
+    return get_validator_by_name(name)