pycharter 0.0.25__py3-none-any.whl → 0.0.26__py3-none-any.whl

pycharter/__init__.py

@@ -166,6 +166,9 @@ from pycharter.runtime_validator import (
     validate_input,
     validate_output,
     validate_with_contract_decorator,
+    # Schema utilities
+    merge_rules_into_schema,
+    get_merged_schema_from_contract,
 )
 
 # ============================================================================
@@ -345,6 +348,9 @@ __all__ = [
     "validate_input",
     "validate_output",
     "validate_with_contract_decorator",
+    # Schema utilities
+    "merge_rules_into_schema",
+    "get_merged_schema_from_contract",
     # Quality
     "QualityCheck",
     "QualityCheckOptions",

pycharter/api/README.md

@@ -183,7 +183,7 @@ List all schemas stored in the metadata store.
 Get a schema by ID (optional version parameter).
 
 #### `GET /api/v1/metadata/schemas/{schema_id}/complete`
-Get complete schema with coercion and validation rules merged.
+Get complete schema with coercion and validation rules merged (for display/docs). For validation, use the Validator class, which merges rules internally.
 
 #### `POST /api/v1/metadata/schemas`
 Store a schema in the metadata store.

pycharter/api/dependencies/auth.py

@@ -0,0 +1,158 @@
+"""
+Authentication dependency for API routes.
+
+Supports:
+- Local/dev: initial credentials from env or pycharter.cfg; JWT issued and verified in-process.
+- Optional auth service: token introspect via HTTP (when auth_service_url + introspect_path set).
+"""
+
+import hmac
+import time
+from typing import Any, Optional
+
+import jwt
+from fastapi import Depends, HTTPException, status
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+
+from pycharter.config import (
+    get_auth_initial_credentials,
+    get_auth_jwt_secret,
+    get_auth_service_introspect_path,
+    get_auth_service_url,
+    is_auth_disabled,
+)
+
+# HTTPBearer returns the token from Authorization: Bearer <token>
+security = HTTPBearer(auto_error=False)
+
+# Default JWT expiry (seconds)
+ACCESS_TOKEN_EXPIRE_SECONDS = 3600  # 1 hour
+
+
+def _constant_time_compare(a: str, b: str) -> bool:
+    """Constant-time string comparison to avoid timing attacks."""
+    return hmac.compare_digest(a.encode("utf-8"), b.encode("utf-8"))
+
+
+def verify_initial_credentials(username: str, password: str) -> bool:
+    """Verify username/password against initial credentials (env or cfg). Returns True if valid."""
+    creds = get_auth_initial_credentials()
+    if not creds:
+        return False
+    u, p = creds
+    return _constant_time_compare(username, u) and _constant_time_compare(password, p)
+
+
+def create_access_token(username: str, expires_delta_seconds: int = ACCESS_TOKEN_EXPIRE_SECONDS) -> str:
+    """Create a JWT access token for the given username."""
+    secret = get_auth_jwt_secret()
+    if not secret:
+        raise ValueError("JWT secret not configured (set PYCHARTER_AUTH_JWT_SECRET or auth.jwt_secret)")
+    payload = {
+        "sub": username,
+        "exp": int(time.time()) + expires_delta_seconds,
+        "iat": int(time.time()),
+    }
+    return jwt.encode(payload, secret, algorithm="HS256")
+
+
+def verify_jwt(token: str) -> Optional[dict[str, Any]]:
+    """Verify JWT and return payload (with 'sub' = username) or None if invalid."""
+    secret = get_auth_jwt_secret()
+    if not secret:
+        return None
+    try:
+        payload = jwt.decode(token, secret, algorithms=["HS256"])
+        return payload
+    except jwt.PyJWTError:
+        return None
+
+
+async def introspect_token(token: str) -> Optional[dict[str, Any]]:
+    """Call auth service introspect endpoint. Return payload if active, else None."""
+    base = get_auth_service_url()
+    path = get_auth_service_introspect_path()
+    if not base or not path:
+        return None
+    url = f"{base.rstrip('/')}{path}"
+    try:
+        import httpx
+        async with httpx.AsyncClient() as client:
+            # Common pattern: POST with token in body or header
+            r = await client.post(
+                url,
+                data={"token": token},
+                headers={"Content-Type": "application/x-www-form-urlencoded"},
+                timeout=5.0,
+            )
+            if r.status_code != 200:
+                return None
+            data = r.json()
+            # Typical introspect: {"active": true, "sub": "username", ...}
+            if not data.get("active", False):
+                return None
+            return data
+    except Exception:
+        return None
+
+
+async def get_current_user(
+    credentials: Optional[HTTPAuthorizationCredentials] = Depends(security),
+) -> dict[str, Any]:
+    """
+    Resolve current user from Bearer token. Uses JWT verify or auth service introspect.
+    When auth is disabled, returns a dummy user so routes do not need to branch.
+    """
+    if is_auth_disabled():
+        return {"username": "anonymous", "auth_disabled": True}
+
+    token = None
+    if credentials and credentials.credentials:
+        token = credentials.credentials
+
+    if not token:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Not authenticated",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    # Prefer JWT verify (no HTTP); fall back to introspect if configured
+    payload = verify_jwt(token)
+    if payload is None and (get_auth_service_url() and get_auth_service_introspect_path()):
+        payload = await introspect_token(token)
+
+    if payload is None:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid or expired token",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    username = payload.get("sub") or payload.get("username")
+    if not username:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid token payload",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    return {"username": username}
+
+
+async def get_optional_user(
+    credentials: Optional[HTTPAuthorizationCredentials] = Depends(security),
+) -> Optional[dict[str, Any]]:
+    """Like get_current_user but returns None instead of 401 when no/invalid token. Useful for public endpoints."""
+    if is_auth_disabled():
+        return {"username": "anonymous", "auth_disabled": True}
+    token = credentials.credentials if credentials and credentials.credentials else None
+    if not token:
+        return None
+    payload = verify_jwt(token)
+    if payload is None and (get_auth_service_url() and get_auth_service_introspect_path()):
+        payload = await introspect_token(token)
+    if payload is None:
+        return None
+    username = payload.get("sub") or payload.get("username")
+    return {"username": username} if username else None

pycharter/api/main.py

@@ -17,6 +17,7 @@ from pycharter import __version__ as pycharter_version
 
 # Import routers from v1
 from pycharter.api.routes.v1 import (
+    auth,
     contracts,
     metadata,
     quality,
@@ -27,7 +28,10 @@ from pycharter.api.routes.v1 import (
     docs,
     tracking,
     evolution,
+    etl,
 )
+from pycharter.api.dependencies.auth import get_current_user
+from fastapi import Depends
 
 # Try to import validation_jobs router (requires worker component)
 try:
@@ -102,57 +106,80 @@ def create_application() -> FastAPI:
         allow_headers=["*"],
     )
     
-    # Include routers from v1 with automatic /api/v1 prefix
-    # All routers in routes/v1/ are automatically included with the /api/v1 prefix
+    # Auth router: no global auth dependency (login/logout public; /auth/me uses Depends in route)
+    app.include_router(
+        auth.router,
+        prefix=f"/api/{API_VERSION}",
+        tags=["Auth"],
+    )
+
+    # Protected routers: require valid Bearer token (when auth enabled; when disabled, get_current_user returns dummy user)
+    _auth_dep = [Depends(get_current_user)]
     app.include_router(
         contracts.router,
         prefix=f"/api/{API_VERSION}",
         tags=["Contracts"],
+        dependencies=_auth_dep,
     )
     app.include_router(
         metadata.router,
         prefix=f"/api/{API_VERSION}",
         tags=["Metadata"],
+        dependencies=_auth_dep,
     )
     app.include_router(
         schemas.router,
         prefix=f"/api/{API_VERSION}",
         tags=["Schemas"],
+        dependencies=_auth_dep,
     )
     app.include_router(
         validation.router,
         prefix=f"/api/{API_VERSION}",
         tags=["Validation"],
+        dependencies=_auth_dep,
     )
     app.include_router(
         quality.router,
         prefix=f"/api/{API_VERSION}",
         tags=["Quality"],
+        dependencies=_auth_dep,
     )
     app.include_router(
         templates.router,
         prefix=f"/api/{API_VERSION}",
         tags=["Templates"],
+        dependencies=_auth_dep,
+    )
+    app.include_router(
+        etl.router,
+        prefix=f"/api/{API_VERSION}",
+        tags=["ETL"],
+        dependencies=_auth_dep,
     )
     app.include_router(
         settings.router,
         prefix=f"/api/{API_VERSION}",
         tags=["Settings"],
+        dependencies=_auth_dep,
     )
     app.include_router(
         docs.router,
         prefix=f"/api/{API_VERSION}",
         tags=["Documentation"],
+        dependencies=_auth_dep,
     )
     app.include_router(
         tracking.router,
         prefix=f"/api/{API_VERSION}",
         tags=["Quality Tracking"],
+        dependencies=_auth_dep,
     )
     app.include_router(
         evolution.router,
         prefix=f"/api/{API_VERSION}",
         tags=["Schema Evolution"],
+        dependencies=_auth_dep,
     )
 
     # Include validation_jobs router if worker component is available
@@ -161,6 +188,7 @@ def create_application() -> FastAPI:
             validation_jobs.router,
             prefix=f"/api/{API_VERSION}",
             tags=["Validation Jobs"],
+            dependencies=_auth_dep,
         )
     
     # Root endpoint

pycharter/api/models/etl.py

@@ -0,0 +1,66 @@
+"""
+Request/Response models for ETL run endpoint.
+"""
+
+from typing import Dict, List, Optional
+
+from pydantic import BaseModel, Field
+
+
+class EtlRunRequest(BaseModel):
+    """Request model for running an ETL pipeline from YAML configs."""
+
+    extract_yaml: str = Field(..., description="Extract config as YAML string")
+    load_yaml: str = Field(..., description="Load config as YAML string")
+    transform_yaml: Optional[str] = Field(
+        default=None,
+        description="Optional transform config as YAML string",
+    )
+    variables: Optional[Dict[str, str]] = Field(
+        default=None,
+        description="Optional variables for ${VAR} substitution in configs",
+    )
+    dry_run: bool = Field(
+        default=False,
+        description="If True, extract and transform but do not load",
+    )
+
+
+class EtlRunResponse(BaseModel):
+    """Response model for ETL pipeline run result (mirrors PipelineResult.to_dict())."""
+
+    success: bool = Field(..., description="Whether the pipeline run succeeded")
+    rows_extracted: int = Field(0, description="Number of rows extracted")
+    rows_transformed: int = Field(0, description="Number of rows transformed")
+    rows_loaded: int = Field(0, description="Number of rows loaded")
+    rows_failed: int = Field(0, description="Number of rows that failed to load")
+    # Validation tracking
+    rows_quarantined_extract: int = Field(
+        0,
+        description="Number of rows quarantined at extract stage due to validation",
+    )
+    rows_quarantined_load: int = Field(
+        0,
+        description="Number of rows quarantined at load stage due to validation",
+    )
+    total_quarantined: int = Field(
+        0,
+        description="Total rows quarantined at both stages",
+    )
+    validation_errors_extract: List[str] = Field(
+        default_factory=list,
+        description="Validation error messages from extract stage",
+    )
+    validation_errors_load: List[str] = Field(
+        default_factory=list,
+        description="Validation error messages from load stage",
+    )
+    # Timing and metadata
+    duration_seconds: Optional[float] = Field(
+        None,
+        description="Total run duration in seconds",
+    )
+    batches_processed: int = Field(0, description="Number of batches processed")
+    errors: List[str] = Field(default_factory=list, description="Error messages if any")
+    pipeline_name: Optional[str] = Field(None, description="Pipeline name if set")
+    run_id: Optional[str] = Field(None, description="Run identifier")

pycharter/api/routes/v1/__init__.py

@@ -7,6 +7,7 @@ with the `/api/v1` prefix.
 """
 
 from pycharter.api.routes.v1 import (
+    auth,
     contracts,
     metadata,
     quality,
@@ -16,10 +17,12 @@ from pycharter.api.routes.v1 import (
     docs,
     tracking,
     evolution,
+    etl,
 )
 
 # Export all routers for automatic inclusion in main.py
 __all__ = [
+    "auth",
     "contracts",
     "metadata",
     "quality",
@@ -29,5 +32,6 @@ __all__ = [
     "docs",
     "tracking",
     "evolution",
+    "etl",
 ]
 

pycharter/api/routes/v1/auth.py

@@ -0,0 +1,97 @@
+"""
+Authentication routes: login, logout, me.
+"""
+
+from fastapi import APIRouter, Depends, HTTPException, status
+from pydantic import BaseModel
+
+from pycharter.api.dependencies.auth import (
+    create_access_token,
+    get_current_user,
+    verify_initial_credentials,
+    ACCESS_TOKEN_EXPIRE_SECONDS,
+)
+from pycharter.config import (
+    get_auth_initial_credentials,
+    get_auth_jwt_secret,
+    is_auth_disabled,
+)
+
+router = APIRouter()
+
+
+class LoginRequest(BaseModel):
+    username: str
+    password: str
+
+
+class LoginResponse(BaseModel):
+    access_token: str
+    token_type: str = "bearer"
+    expires_in: int = ACCESS_TOKEN_EXPIRE_SECONDS
+
+
+class UserResponse(BaseModel):
+    username: str
+    auth_disabled: bool = False
+
+
+@router.post(
+    "/auth/login",
+    response_model=LoginResponse,
+    status_code=status.HTTP_200_OK,
+    summary="Login",
+    description="Exchange username and password for an access token. Uses initial credentials (env or pycharter.cfg) when configured.",
+)
+def login(request: LoginRequest) -> LoginResponse:
+    if is_auth_disabled():
+        raise HTTPException(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail="Authentication is disabled",
+        )
+    creds = get_auth_initial_credentials()
+    if not creds:
+        raise HTTPException(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail="Authentication not configured (set initial credentials or auth service)",
+        )
+    if not get_auth_jwt_secret():
+        raise HTTPException(
+            status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            detail="JWT secret not configured (set PYCHARTER_AUTH_JWT_SECRET or auth.jwt_secret)",
+        )
+    if not verify_initial_credentials(request.username, request.password):
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid username or password",
+        )
+    token = create_access_token(request.username)
+    return LoginResponse(
+        access_token=token,
+        token_type="bearer",
+        expires_in=ACCESS_TOKEN_EXPIRE_SECONDS,
+    )
+
+
+@router.post(
+    "/auth/logout",
+    status_code=status.HTTP_200_OK,
+    summary="Logout",
+    description="Client should discard the token. Server-side invalidation can be added later.",
+)
+def logout() -> dict:
+    return {"message": "OK"}
+
+
+@router.get(
+    "/auth/me",
+    response_model=UserResponse,
+    status_code=status.HTTP_200_OK,
+    summary="Current user",
+    description="Return the current user from the Bearer token. Protected.",
+)
+def me(current_user: dict = Depends(get_current_user)) -> UserResponse:
+    return UserResponse(
+        username=current_user["username"],
+        auth_disabled=current_user.get("auth_disabled", False),
+    )

pycharter/api/routes/v1/contracts.py

@@ -269,10 +269,11 @@ async def build_contract_from_id_endpoint(
             model_name="CoercionRule"
         )
         if coercion_rules and coercion_rules.rules:
-            # Format coercion rules - the rules dict itself, with version if needed
-            # The builder will extract the "rules" key if present, otherwise uses the whole dict
-            coercion_rules_data = dict(coercion_rules.rules) if coercion_rules.rules else {}
-            coercion_rules_data["version"] = coercion_rules.version
+            # Builder expects {"version": "...", "rules": {...}} so _extract_rules returns the inner dict
+            coercion_rules_data = {
+                "version": coercion_rules.version,
+                "rules": dict(coercion_rules.rules),
+            }
     
     # Get validation rules (optional)
     validation_rules_data = None
@@ -282,10 +283,11 @@ async def build_contract_from_id_endpoint(
             model_name="ValidationRule"
         )
         if validation_rules and validation_rules.rules:
-            # Format validation rules - the rules dict itself, with version if needed
-            # The builder will extract the "rules" key if present, otherwise uses the whole dict
-            validation_rules_data = dict(validation_rules.rules) if validation_rules.rules else {}
-            validation_rules_data["version"] = validation_rules.version
+            # Builder expects {"version": "...", "rules": {...}} so _extract_rules returns the inner dict
+            validation_rules_data = {
+                "version": validation_rules.version,
+                "rules": dict(validation_rules.rules),
+            }
     
     # Get metadata (optional)
     metadata_data = None

pycharter/api/routes/v1/etl.py

@@ -0,0 +1,131 @@
+"""
+Route handlers for ETL pipeline execution.
+
+Allows running an ETL pipeline from YAML config strings (extract, transform, load).
+"""
+
+import logging
+from typing import Any, Dict, List, Union
+
+import yaml
+from fastapi import APIRouter, HTTPException, status
+
+from pycharter import Pipeline
+from pycharter.api.models.etl import EtlRunRequest, EtlRunResponse
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+def _parse_yaml_config(yaml_str: str, field_name: str) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
+    """
+    Parse a YAML string into a dict or list of dicts.
+    Empty or whitespace-only string returns {}.
+    Raises HTTPException 400 on parse error or invalid type.
+    """
+    if not yaml_str or not yaml_str.strip():
+        return {}
+    try:
+        parsed = yaml.safe_load(yaml_str)
+    except yaml.YAMLError as e:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"Invalid YAML in {field_name}: {str(e)}",
+        )
+    if parsed is None:
+        return {}
+    if isinstance(parsed, dict):
+        return parsed
+    if isinstance(parsed, list):
+        if all(isinstance(item, dict) for item in parsed):
+            return parsed
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"{field_name} must be a YAML object or list of objects",
+        )
+    raise HTTPException(
+        status_code=status.HTTP_400_BAD_REQUEST,
+        detail=f"{field_name} must be a YAML object or list of objects, got {type(parsed).__name__}",
+    )
+
+
+@router.post(
+    "/etl/run",
+    response_model=EtlRunResponse,
+    status_code=status.HTTP_200_OK,
+    summary="Run ETL pipeline",
+    description="Run an ETL pipeline from extract, transform, and load YAML config strings. Optional variables are applied for ${VAR} substitution. Use dry_run=True to extract and transform without loading.",
+    response_description="Pipeline run result with row counts and any errors",
+    tags=["ETL"],
+)
+async def run_etl_pipeline(request: EtlRunRequest) -> EtlRunResponse:
+    """
+    Run an ETL pipeline from the provided YAML configs.
+
+    Parses extract_yaml, load_yaml, and optional transform_yaml; builds a pipeline
+    via Pipeline.from_dict() (so variable substitution is applied), runs it, and
+    returns the result (row counts, success, errors).
+    """
+    # Parse YAML configs
+    extract_dict = _parse_yaml_config(request.extract_yaml, "extract")
+    load_dict = _parse_yaml_config(request.load_yaml, "load")
+    transform_raw = _parse_yaml_config(request.transform_yaml or "", "transform")
+
+    if not isinstance(extract_dict, dict) or not extract_dict:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Extract config is required and must be a non-empty YAML object",
+        )
+    if not isinstance(load_dict, dict) or not load_dict:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Load config is required and must be a non-empty YAML object",
+        )
+
+    # Build config for Pipeline.from_dict
+    # The transform YAML can contain: transform (simple ops), jsonata, custom_function
+    # We pass the full parsed dict to from_dict which will extract these
+    config: Dict[str, Any] = {
+        "extract": extract_dict,
+        "load": load_dict,
+    }
+    
+    # If transform_raw is a dict, include its keys in config (transform, jsonata, custom_function)
+    if isinstance(transform_raw, dict):
+        if "transform" in transform_raw:
+            config["transform"] = transform_raw["transform"]
+        if "jsonata" in transform_raw:
+            config["jsonata"] = transform_raw["jsonata"]
+        if "custom_function" in transform_raw:
+            config["custom_function"] = transform_raw["custom_function"]
+    elif isinstance(transform_raw, list):
+        # If it's a list, treat it as transform steps
+        config["transform"] = transform_raw
+
+    variables = request.variables or {}
+
+    try:
+        pipeline = Pipeline.from_dict(config, variables=variables)
+    except ValueError as e:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=str(e),
+        )
+    except Exception as e:
+        logger.exception("Pipeline build failed")
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"Pipeline config invalid: {str(e)}",
+        )
+
+    try:
+        result = await pipeline.run(dry_run=request.dry_run)
+    except Exception as e:
+        logger.exception("Pipeline run failed")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=str(e),
+        )
+
+    return EtlRunResponse(**result.to_dict())

pycharter/cli.py

@@ -127,7 +127,7 @@ def main():
     seed_parser.add_argument(
         "seed_dir",
         nargs="?",
-        help="Directory containing seed YAML files (default: data/seed)",
+        help="Directory containing seed YAML files (default: package data/seed)",
     )
     seed_parser.add_argument(
         "database_url",