pycharter 0.0.24__py3-none-any.whl → 0.0.26__py3-none-any.whl

pycharter/__init__.py

@@ -166,6 +166,9 @@ from pycharter.runtime_validator import (
     validate_input,
     validate_output,
     validate_with_contract_decorator,
+    # Schema utilities
+    merge_rules_into_schema,
+    get_merged_schema_from_contract,
 )
 
 # ============================================================================
@@ -345,6 +348,9 @@ __all__ = [
     "validate_input",
     "validate_output",
     "validate_with_contract_decorator",
+    # Schema utilities
+    "merge_rules_into_schema",
+    "get_merged_schema_from_contract",
     # Quality
     "QualityCheck",
     "QualityCheckOptions",

pycharter/api/README.md

@@ -0,0 +1,340 @@
+# PyCharter API
+
+REST API wrapper for PyCharter services using FastAPI.
+
+## Overview
+
+The PyCharter API is located at the root level of the repository (`api/`) and provides HTTP endpoints for all core PyCharter services:
+- **Contract Parsing**: Parse data contract files into components
+- **Contract Building**: Reconstruct contracts from metadata store
+- **Metadata Storage**: Store and retrieve schemas, metadata, and rules
+- **Schema Generation**: Generate Pydantic models from JSON Schemas
+- **Schema Conversion**: Convert Pydantic models to JSON Schemas
+- **Runtime Validation**: Validate data against schemas
+- **Quality Assurance**: Run quality checks, query metrics and violations
+
+## Installation
+
+Install PyCharter with API dependencies:
+
+```bash
+pip install pycharter[api]
+```
+
+## Running the API Server
+
+### Using the CLI command:
+
+```bash
+pycharter api
+```
+
+### Using uvicorn directly:
+
+```bash
+uvicorn api.main:app --reload
+```
+
+### Using Python:
+
+```python
+from api.main import main
+main()
+```
+
+## Configuration
+
+The API requires a database connection for most endpoints. Configure it using environment variables:
+
+### Database URL Configuration
+
+The API uses `PYCHARTER_DATABASE_URL` environment variable to connect to the database. You can set it in several ways:
+
+#### Method 1: Export in Shell (Recommended)
+
+```bash
+# Set the environment variable
+export PYCHARTER_DATABASE_URL="postgresql://user:password@localhost:5432/pycharter"
+
+# Then run the API
+pycharter api
+```
+
+#### Method 2: Inline with Command
+
+```bash
+# Set and run in one line
+PYCHARTER_DATABASE_URL="postgresql://user:password@localhost:5432/pycharter" pycharter api
+```
+
+#### Method 3: Add to Shell Profile (Persistent)
+
+Add to your `~/.bashrc`, `~/.zshrc`, or `~/.profile`:
+
+```bash
+export PYCHARTER_DATABASE_URL="postgresql://user:password@localhost:5432/pycharter"
+```
+
+Then reload your shell:
+```bash
+source ~/.bashrc  # or source ~/.zshrc
+```
+
+#### Method 4: Using a .env File
+
+Create a `.env` file in your project root:
+
+```bash
+PYCHARTER_DATABASE_URL=postgresql://user:password@localhost:5432/pycharter
+```
+
+Then load it before running (requires `python-dotenv`):
+```bash
+export $(cat .env | xargs)
+pycharter api
+```
+
+Or use `dotenv-cli`:
+```bash
+pip install dotenv-cli
+dotenv run pycharter api
+```
+
+### Alternative Environment Variable
+
+PyCharter also supports Airflow-style configuration:
+
+```bash
+export PYCHARTER__DATABASE__SQL_ALCHEMY_CONN="postgresql://user:password@localhost:5432/pycharter"
+```
+
+### Metadata Store Configuration
+
+The API can be configured to use different metadata store backends:
+
+```bash
+# Metadata store type (in_memory, postgres, mongodb, redis)
+export PYCHARTER_API_STORE_TYPE=postgres
+
+# Connection string for database-backed stores
+export PYCHARTER_API_CONNECTION_STRING=postgresql://user:pass@localhost:5432/pycharter
+```
+
+**Note**: If `PYCHARTER_DATABASE_URL` is set, it will be used for both the database connection and metadata store (for PostgreSQL).
+
+## API Endpoints
+
+### Contract Endpoints
+
+#### `POST /api/v1/contracts/parse`
+Parse a data contract dictionary into components.
+
+**Request Body:**
+```json
+{
+  "contract": {
+    "schema": {...},
+    "metadata": {...},
+    "ownership": {...},
+    "governance_rules": {...}
+  }
+}
+```
+
+**Response:**
+```json
+{
+  "schema": {...},
+  "metadata": {...},
+  "ownership": {...},
+  "governance_rules": {...},
+  "coercion_rules": {...},
+  "validation_rules": {...},
+  "versions": {...}
+}
+```
+
+#### `POST /api/v1/contracts/build`
+Build a complete contract from metadata store.
+
+**Request Body:**
+```json
+{
+  "schema_id": "user_schema",
+  "version": "1.0.0",
+  "include_metadata": true,
+  "include_ownership": true,
+  "include_governance": true
+}
+```
+
+#### `GET /api/v1/contracts`
+List all data contracts stored in the database.
+
+#### `GET /api/v1/contracts/{contract_id}`
+Get a specific data contract by ID.
+
+### Metadata Endpoints
+
+#### `GET /api/v1/metadata/schemas`
+List all schemas stored in the metadata store.
+
+#### `GET /api/v1/metadata/schemas/{schema_id}`
+Get a schema by ID (optional version parameter).
+
+#### `GET /api/v1/metadata/schemas/{schema_id}/complete`
+Get complete schema with coercion and validation rules merged (for display/docs). For validation, use the Validator class, which merges rules internally.
+
+#### `POST /api/v1/metadata/schemas`
+Store a schema in the metadata store.
+
+#### `GET /api/v1/metadata/metadata/{schema_id}`
+Get metadata for a schema (optional version parameter).
+
+#### `POST /api/v1/metadata/metadata`
+Store metadata for a schema.
+
+#### `GET /api/v1/metadata/coercion-rules/{schema_id}`
+Get coercion rules for a schema (optional version parameter).
+
+#### `POST /api/v1/metadata/coercion-rules`
+Store coercion rules for a schema.
+
+#### `GET /api/v1/metadata/validation-rules/{schema_id}`
+Get validation rules for a schema (optional version parameter).
+
+#### `POST /api/v1/metadata/validation-rules`
+Store validation rules for a schema.
+
+### Quality Assurance Endpoints
+
+#### `GET /api/v1/quality/metrics`
+List quality metrics (optional filtering by schema_id, pagination).
+
+#### `GET /api/v1/quality/metrics/{metric_id}`
+Get a specific quality metric by ID.
+
+#### `GET /api/v1/quality/reports/{schema_id}`
+Get quality reports for a schema/data feed (optional data_source filter).
+
+#### `POST /api/v1/quality/check`
+Run a quality check against a data contract.
+
+#### `POST /api/v1/quality/violations`
+Query data quality violations.
+
+### Schema Endpoints
+
+#### `POST /api/v1/schemas/generate`
+Generate Pydantic model from JSON Schema.
+
+#### `POST /api/v1/schemas/convert`
+Convert Pydantic model to JSON Schema.
+
+### Validation Endpoints
+
+#### `POST /api/v1/validation/validate`
+Validate a single record against a schema.
+
+#### `POST /api/v1/validation/validate-batch`
+Validate a batch of records against a schema.
+
+## API Documentation
+
+Once the API server is running, you can access:
+
+- **Swagger UI**: http://localhost:8000/docs
+- **ReDoc**: http://localhost:8000/redoc
+- **OpenAPI JSON**: http://localhost:8000/openapi.json
+
+## Example Usage
+
+### Using curl:
+
+```bash
+# List all contracts
+curl http://localhost:8000/api/v1/contracts
+
+# Get a specific contract
+curl http://localhost:8000/api/v1/contracts/{contract_id}
+
+# List all schemas
+curl http://localhost:8000/api/v1/metadata/schemas
+
+# Get a schema
+curl http://localhost:8000/api/v1/metadata/schemas/{schema_id}
+
+# Get quality metrics
+curl http://localhost:8000/api/v1/quality/metrics
+
+# Get quality report for a schema
+curl http://localhost:8000/api/v1/quality/reports/{schema_id}
+```
+
+### Using Python requests:
+
+```python
+import requests
+
+# List contracts
+response = requests.get("http://localhost:8000/api/v1/contracts")
+contracts = response.json()
+
+# Get quality metrics
+response = requests.get("http://localhost:8000/api/v1/quality/metrics")
+metrics = response.json()
+```
+
+## Troubleshooting
+
+### Database Connection Issues
+
+If you see errors about database URL not being configured:
+
+1. **Check environment variable is set:**
+   ```bash
+   echo $PYCHARTER_DATABASE_URL
+   ```
+
+2. **Verify database is accessible:**
+   ```bash
+   psql $PYCHARTER_DATABASE_URL -c "SELECT 1;"
+   ```
+
+3. **Ensure database is initialized:**
+   ```bash
+   pycharter db init $PYCHARTER_DATABASE_URL
+   ```
+
+### API Not Starting
+
+If the API fails to start:
+
+1. **Check if uvicorn is installed:**
+   ```bash
+   pip install pycharter[api]
+   ```
+
+2. **Check for port conflicts:**
+   ```bash
+   # Use a different port
+   pycharter api --port 8001
+   ```
+
+3. **Check logs for errors:**
+   The API will print error messages to stderr if there are configuration issues.
+
+## Production Deployment
+
+For production, consider:
+
+1. **Use a process manager** (systemd, supervisor, etc.)
+2. **Set environment variables** in your deployment configuration
+3. **Use a reverse proxy** (nginx, traefik) in front of the API
+4. **Enable HTTPS** for secure connections
+5. **Configure CORS** appropriately in `api/main.py`
+6. **Use a production ASGI server** like Gunicorn with Uvicorn workers:
+
+```bash
+gunicorn api.main:app -w 4 -k uvicorn.workers.UvicornWorker
+```

{api → pycharter/api}/__init__.py

@@ -17,7 +17,7 @@ __version__ = "0.0.3"
 
 # Check if FastAPI is available
 try:
-    from api.main import app, create_application
+    from pycharter.api.main import app, create_application
     __all__ = ["app", "create_application"]
 except (ImportError, ModuleNotFoundError):
     # FastAPI not installed or api module not available

{api → pycharter/api}/dependencies/__init__.py

@@ -4,7 +4,7 @@ Dependencies for API routes.
 This module provides FastAPI dependency injection for shared resources.
 """
 
-from api.dependencies.database import get_db_session
-from api.dependencies.store import get_metadata_store
+from pycharter.api.dependencies.database import get_db_session
+from pycharter.api.dependencies.store import get_metadata_store
 
 __all__ = ["get_db_session", "get_metadata_store"]

pycharter/api/dependencies/auth.py

@@ -0,0 +1,158 @@
+"""
+Authentication dependency for API routes.
+
+Supports:
+- Local/dev: initial credentials from env or pycharter.cfg; JWT issued and verified in-process.
+- Optional auth service: token introspect via HTTP (when auth_service_url + introspect_path set).
+"""
+
+import hmac
+import time
+from typing import Any, Optional
+
+import jwt
+from fastapi import Depends, HTTPException, status
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+
+from pycharter.config import (
+    get_auth_initial_credentials,
+    get_auth_jwt_secret,
+    get_auth_service_introspect_path,
+    get_auth_service_url,
+    is_auth_disabled,
+)
+
+# HTTPBearer returns the token from Authorization: Bearer <token>
+security = HTTPBearer(auto_error=False)
+
+# Default JWT expiry (seconds)
+ACCESS_TOKEN_EXPIRE_SECONDS = 3600  # 1 hour
+
+
+def _constant_time_compare(a: str, b: str) -> bool:
+    """Constant-time string comparison to avoid timing attacks."""
+    return hmac.compare_digest(a.encode("utf-8"), b.encode("utf-8"))
+
+
+def verify_initial_credentials(username: str, password: str) -> bool:
+    """Verify username/password against initial credentials (env or cfg). Returns True if valid."""
+    creds = get_auth_initial_credentials()
+    if not creds:
+        return False
+    u, p = creds
+    return _constant_time_compare(username, u) and _constant_time_compare(password, p)
+
+
+def create_access_token(username: str, expires_delta_seconds: int = ACCESS_TOKEN_EXPIRE_SECONDS) -> str:
+    """Create a JWT access token for the given username."""
+    secret = get_auth_jwt_secret()
+    if not secret:
+        raise ValueError("JWT secret not configured (set PYCHARTER_AUTH_JWT_SECRET or auth.jwt_secret)")
+    payload = {
+        "sub": username,
+        "exp": int(time.time()) + expires_delta_seconds,
+        "iat": int(time.time()),
+    }
+    return jwt.encode(payload, secret, algorithm="HS256")
+
+
+def verify_jwt(token: str) -> Optional[dict[str, Any]]:
+    """Verify JWT and return payload (with 'sub' = username) or None if invalid."""
+    secret = get_auth_jwt_secret()
+    if not secret:
+        return None
+    try:
+        payload = jwt.decode(token, secret, algorithms=["HS256"])
+        return payload
+    except jwt.PyJWTError:
+        return None
+
+
+async def introspect_token(token: str) -> Optional[dict[str, Any]]:
+    """Call auth service introspect endpoint. Return payload if active, else None."""
+    base = get_auth_service_url()
+    path = get_auth_service_introspect_path()
+    if not base or not path:
+        return None
+    url = f"{base.rstrip('/')}{path}"
+    try:
+        import httpx
+        async with httpx.AsyncClient() as client:
+            # Common pattern: POST with token in body or header
+            r = await client.post(
+                url,
+                data={"token": token},
+                headers={"Content-Type": "application/x-www-form-urlencoded"},
+                timeout=5.0,
+            )
+            if r.status_code != 200:
+                return None
+            data = r.json()
+            # Typical introspect: {"active": true, "sub": "username", ...}
+            if not data.get("active", False):
+                return None
+            return data
+    except Exception:
+        return None
+
+
+async def get_current_user(
+    credentials: Optional[HTTPAuthorizationCredentials] = Depends(security),
+) -> dict[str, Any]:
+    """
+    Resolve current user from Bearer token. Uses JWT verify or auth service introspect.
+    When auth is disabled, returns a dummy user so routes do not need to branch.
+    """
+    if is_auth_disabled():
+        return {"username": "anonymous", "auth_disabled": True}
+
+    token = None
+    if credentials and credentials.credentials:
+        token = credentials.credentials
+
+    if not token:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Not authenticated",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    # Prefer JWT verify (no HTTP); fall back to introspect if configured
+    payload = verify_jwt(token)
+    if payload is None and (get_auth_service_url() and get_auth_service_introspect_path()):
+        payload = await introspect_token(token)
+
+    if payload is None:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid or expired token",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    username = payload.get("sub") or payload.get("username")
+    if not username:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Invalid token payload",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+
+    return {"username": username}
+
+
+async def get_optional_user(
+    credentials: Optional[HTTPAuthorizationCredentials] = Depends(security),
+) -> Optional[dict[str, Any]]:
+    """Like get_current_user but returns None instead of 401 when no/invalid token. Useful for public endpoints."""
+    if is_auth_disabled():
+        return {"username": "anonymous", "auth_disabled": True}
+    token = credentials.credentials if credentials and credentials.credentials else None
+    if not token:
+        return None
+    payload = verify_jwt(token)
+    if payload is None and (get_auth_service_url() and get_auth_service_introspect_path()):
+        payload = await introspect_token(token)
+    if payload is None:
+        return None
+    username = payload.get("sub") or payload.get("username")
+    return {"username": username} if username else None

{api → pycharter/api}/main.py

@@ -16,7 +16,8 @@ from fastapi.routing import APIRoute
 from pycharter import __version__ as pycharter_version
 
 # Import routers from v1
-from api.routes.v1 import (
+from pycharter.api.routes.v1 import (
+    auth,
     contracts,
     metadata,
     quality,
@@ -27,7 +28,10 @@ from api.routes.v1 import (
     docs,
     tracking,
     evolution,
+    etl,
 )
+from pycharter.api.dependencies.auth import get_current_user
+from fastapi import Depends
 
 # Try to import validation_jobs router (requires worker component)
 try:
@@ -102,57 +106,80 @@ def create_application() -> FastAPI:
         allow_headers=["*"],
     )
     
-    # Include routers from v1 with automatic /api/v1 prefix
-    # All routers in routes/v1/ are automatically included with the /api/v1 prefix
+    # Auth router: no global auth dependency (login/logout public; /auth/me uses Depends in route)
+    app.include_router(
+        auth.router,
+        prefix=f"/api/{API_VERSION}",
+        tags=["Auth"],
+    )
+
+    # Protected routers: require valid Bearer token (when auth enabled; when disabled, get_current_user returns dummy user)
+    _auth_dep = [Depends(get_current_user)]
     app.include_router(
         contracts.router,
         prefix=f"/api/{API_VERSION}",
         tags=["Contracts"],
+        dependencies=_auth_dep,
     )
     app.include_router(
         metadata.router,
         prefix=f"/api/{API_VERSION}",
         tags=["Metadata"],
+        dependencies=_auth_dep,
     )
     app.include_router(
         schemas.router,
         prefix=f"/api/{API_VERSION}",
         tags=["Schemas"],
+        dependencies=_auth_dep,
     )
     app.include_router(
         validation.router,
         prefix=f"/api/{API_VERSION}",
         tags=["Validation"],
+        dependencies=_auth_dep,
     )
     app.include_router(
         quality.router,
         prefix=f"/api/{API_VERSION}",
         tags=["Quality"],
+        dependencies=_auth_dep,
     )
     app.include_router(
         templates.router,
         prefix=f"/api/{API_VERSION}",
         tags=["Templates"],
+        dependencies=_auth_dep,
+    )
+    app.include_router(
+        etl.router,
+        prefix=f"/api/{API_VERSION}",
+        tags=["ETL"],
+        dependencies=_auth_dep,
     )
     app.include_router(
         settings.router,
         prefix=f"/api/{API_VERSION}",
         tags=["Settings"],
+        dependencies=_auth_dep,
     )
     app.include_router(
         docs.router,
         prefix=f"/api/{API_VERSION}",
         tags=["Documentation"],
+        dependencies=_auth_dep,
     )
     app.include_router(
         tracking.router,
         prefix=f"/api/{API_VERSION}",
         tags=["Quality Tracking"],
+        dependencies=_auth_dep,
     )
     app.include_router(
         evolution.router,
         prefix=f"/api/{API_VERSION}",
         tags=["Schema Evolution"],
+        dependencies=_auth_dep,
     )
 
     # Include validation_jobs router if worker component is available
@@ -161,6 +188,7 @@ def create_application() -> FastAPI:
             validation_jobs.router,
             prefix=f"/api/{API_VERSION}",
             tags=["Validation Jobs"],
+            dependencies=_auth_dep,
         )
     
     # Root endpoint
@@ -296,7 +324,7 @@ def main():
     import uvicorn
     
     uvicorn.run(
-        "api.main:app",
+        "pycharter.api.main:app",
         host="0.0.0.0",
         port=8000,
         reload=True,  # Set to False in production

{api → pycharter/api}/models/__init__.py

@@ -4,13 +4,13 @@ API request and response models.
 This module contains Pydantic models for API request/response validation.
 """
 
-from api.models.contracts import (
+from pycharter.api.models.contracts import (
     ContractParseRequest,
     ContractParseResponse,
     ContractBuildRequest,
     ContractBuildResponse,
 )
-from api.models.metadata import (
+from pycharter.api.models.metadata import (
     MetadataStoreRequest,
     MetadataStoreResponse,
     MetadataGetRequest,
@@ -20,13 +20,13 @@ from api.models.metadata import (
     SchemaGetRequest,
     SchemaGetResponse,
 )
-from api.models.schemas import (
+from pycharter.api.models.schemas import (
     SchemaGenerateRequest,
     SchemaGenerateResponse,
     SchemaConvertRequest,
     SchemaConvertResponse,
 )
-from api.models.validation import (
+from pycharter.api.models.validation import (
     ValidationRequest,
     ValidationResponse,
     ValidationBatchRequest,