pycharter 0.0.22__py3-none-any.whl → 0.0.24__py3-none-any.whl

api/main.py

@@ -16,7 +16,18 @@ from fastapi.routing import APIRoute
 from pycharter import __version__ as pycharter_version
 
 # Import routers from v1
-from api.routes.v1 import contracts, metadata, quality, schemas, templates, validation, settings
+from api.routes.v1 import (
+    contracts,
+    metadata,
+    quality,
+    schemas,
+    templates,
+    validation,
+    settings,
+    docs,
+    tracking,
+    evolution,
+)
 
 # Try to import validation_jobs router (requires worker component)
 try:
@@ -128,6 +139,21 @@ def create_application() -> FastAPI:
         prefix=f"/api/{API_VERSION}",
         tags=["Settings"],
     )
+    app.include_router(
+        docs.router,
+        prefix=f"/api/{API_VERSION}",
+        tags=["Documentation"],
+    )
+    app.include_router(
+        tracking.router,
+        prefix=f"/api/{API_VERSION}",
+        tags=["Quality Tracking"],
+    )
+    app.include_router(
+        evolution.router,
+        prefix=f"/api/{API_VERSION}",
+        tags=["Schema Evolution"],
+    )
 
     # Include validation_jobs router if worker component is available
     if VALIDATION_JOBS_AVAILABLE:

api/models/docs.py

@@ -0,0 +1,68 @@
+"""
+API models for documentation generation endpoints.
+"""
+
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class DocsFormat(str, Enum):
+    """Supported documentation output formats."""
+
+    MARKDOWN = "markdown"
+    HTML = "html"
+
+
+class DocsRequest(BaseModel):
+    """Request model for generating documentation from contract data."""
+
+    contract: dict = Field(..., description="Contract data to generate documentation for")
+    format: DocsFormat = Field(
+        default=DocsFormat.MARKDOWN, description="Output format for documentation"
+    )
+    include_schema: bool = Field(default=True, description="Include schema fields section")
+    include_coercions: bool = Field(
+        default=True, description="Include coercion rules section"
+    )
+    include_validations: bool = Field(
+        default=True, description="Include validation rules section"
+    )
+    include_metadata: bool = Field(
+        default=True, description="Include metadata/ownership section"
+    )
+
+
+class DocsResponse(BaseModel):
+    """Response model for generated documentation."""
+
+    documentation: str = Field(..., description="Generated documentation content")
+    format: DocsFormat = Field(..., description="Format of the generated documentation")
+    schema_name: Optional[str] = Field(
+        default=None, description="Name of the schema documented"
+    )
+    version: Optional[str] = Field(
+        default=None, description="Version of the schema documented"
+    )
+
+
+class DocsSectionRequest(BaseModel):
+    """Request model for generating a specific documentation section."""
+
+    contract: dict = Field(..., description="Contract data to generate documentation for")
+    section: str = Field(
+        ...,
+        description="Section to generate: 'schema', 'coercions', 'validations', 'metadata'",
+    )
+    format: DocsFormat = Field(
+        default=DocsFormat.MARKDOWN, description="Output format for documentation"
+    )
+
+
+class DocsSectionResponse(BaseModel):
+    """Response model for a documentation section."""
+
+    section: str = Field(..., description="Name of the section generated")
+    content: str = Field(..., description="Generated section content")
+    format: DocsFormat = Field(..., description="Format of the generated content")

api/models/evolution.py

@@ -0,0 +1,117 @@
+"""
+API models for schema evolution endpoints.
+"""
+
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, Field
+
+
+class CompatibilityModeEnum(str, Enum):
+    """Compatibility checking modes."""
+
+    BACKWARD = "backward"
+    FORWARD = "forward"
+    FULL = "full"
+
+
+class ChangeTypeEnum(str, Enum):
+    """Types of schema changes."""
+
+    FIELD_ADDED = "field_added"
+    FIELD_REMOVED = "field_removed"
+    FIELD_RENAMED = "field_renamed"
+    TYPE_CHANGED = "type_changed"
+    TYPE_WIDENED = "type_widened"
+    TYPE_NARROWED = "type_narrowed"
+    CONSTRAINT_ADDED = "constraint_added"
+    CONSTRAINT_REMOVED = "constraint_removed"
+    CONSTRAINT_MODIFIED = "constraint_modified"
+    REQUIRED_ADDED = "required_added"
+    REQUIRED_REMOVED = "required_removed"
+    ENUM_VALUE_ADDED = "enum_value_added"
+    ENUM_VALUE_REMOVED = "enum_value_removed"
+    DEFAULT_ADDED = "default_added"
+    DEFAULT_REMOVED = "default_removed"
+    DEFAULT_CHANGED = "default_changed"
+    DESCRIPTION_CHANGED = "description_changed"
+    TITLE_CHANGED = "title_changed"
+    FORMAT_CHANGED = "format_changed"
+    PATTERN_CHANGED = "pattern_changed"
+
+
+class SchemaChangeResponse(BaseModel):
+    """Response model for a single schema change."""
+
+    path: str = Field(..., description="JSON path to the changed element")
+    change_type: ChangeTypeEnum = Field(..., description="Type of change")
+    old_value: Optional[Any] = Field(default=None, description="Value in old schema")
+    new_value: Optional[Any] = Field(default=None, description="Value in new schema")
+    breaking: bool = Field(..., description="Whether this is a breaking change")
+    message: str = Field(..., description="Human-readable description")
+
+
+class SchemaDiffResponse(BaseModel):
+    """Response model for schema diff."""
+
+    changes: List[SchemaChangeResponse] = Field(..., description="All detected changes")
+    breaking_changes: List[SchemaChangeResponse] = Field(
+        ..., description="Only breaking changes"
+    )
+    additions: List[SchemaChangeResponse] = Field(..., description="Added elements")
+    removals: List[SchemaChangeResponse] = Field(..., description="Removed elements")
+    modifications: List[SchemaChangeResponse] = Field(
+        ..., description="Modified elements"
+    )
+    has_breaking_changes: bool = Field(
+        ..., description="Whether any breaking changes exist"
+    )
+    total_changes: int = Field(..., description="Total number of changes")
+
+
+class CompatibilityCheckRequest(BaseModel):
+    """Request model for checking compatibility between two schemas."""
+
+    old_schema: Dict[str, Any] = Field(..., description="Original/existing schema")
+    new_schema: Dict[str, Any] = Field(..., description="New schema to check")
+    mode: CompatibilityModeEnum = Field(
+        default=CompatibilityModeEnum.BACKWARD,
+        description="Compatibility mode to check against",
+    )
+
+
+class CompatibilityCheckResponse(BaseModel):
+    """Response model for compatibility check."""
+
+    compatible: bool = Field(
+        ..., description="Whether schemas are compatible in the specified mode"
+    )
+    mode: CompatibilityModeEnum = Field(..., description="Compatibility mode used")
+    diff: Optional[SchemaDiffResponse] = Field(
+        default=None, description="Detailed schema diff"
+    )
+    issues: List[str] = Field(
+        default_factory=list, description="Breaking change descriptions"
+    )
+    warnings: List[str] = Field(
+        default_factory=list, description="Non-breaking issues that may need attention"
+    )
+    breaking_change_count: int = Field(
+        ..., description="Number of breaking changes detected"
+    )
+
+
+class DiffRequest(BaseModel):
+    """Request model for computing schema diff."""
+
+    old_schema: Dict[str, Any] = Field(..., description="Original schema")
+    new_schema: Dict[str, Any] = Field(..., description="New schema")
+
+
+class StoredSchemaDiffRequest(BaseModel):
+    """Request model for diffing stored schema versions."""
+
+    schema_name: str = Field(..., description="Name of the schema")
+    old_version: str = Field(..., description="Old version to compare")
+    new_version: str = Field(..., description="New version to compare")

api/models/tracking.py

@@ -0,0 +1,111 @@
+"""
+API models for quality tracking endpoints.
+"""
+
+from datetime import datetime
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, Field
+
+
+class ExportFormat(str, Enum):
+    """Supported export formats."""
+
+    JSON = "json"
+    PROMETHEUS = "prometheus"
+    CSV = "csv"
+
+
+class ValidationMetricResponse(BaseModel):
+    """Response model for a single validation metric."""
+
+    id: str = Field(..., description="Unique metric identifier")
+    schema_name: str = Field(..., description="Name of the schema validated")
+    version: str = Field(..., description="Version of the schema")
+    timestamp: datetime = Field(..., description="When the validation occurred")
+    record_count: int = Field(..., description="Number of records validated")
+    valid_count: int = Field(..., description="Number of valid records")
+    error_count: int = Field(..., description="Number of records with errors")
+    validity_rate: float = Field(..., description="Ratio of valid records (0.0 to 1.0)")
+    completeness: float = Field(..., description="Data completeness ratio (0.0 to 1.0)")
+    field_completeness: Dict[str, float] = Field(
+        default_factory=dict, description="Per-field completeness ratios"
+    )
+    duration_ms: float = Field(..., description="Validation duration in milliseconds")
+    errors_by_type: Dict[str, int] = Field(
+        default_factory=dict, description="Error counts by type"
+    )
+    metadata: Dict[str, Any] = Field(
+        default_factory=dict, description="Additional metadata"
+    )
+
+
+class MetricsSummaryResponse(BaseModel):
+    """Response model for aggregated metrics summary."""
+
+    schema_name: str = Field(..., description="Name of the schema")
+    period_start: datetime = Field(..., description="Start of the summary period")
+    period_end: datetime = Field(..., description="End of the summary period")
+    total_validations: int = Field(..., description="Total validation runs in period")
+    total_records: int = Field(..., description="Total records validated")
+    total_valid: int = Field(..., description="Total valid records")
+    total_errors: int = Field(..., description="Total records with errors")
+    avg_validity_rate: float = Field(..., description="Average validity rate")
+    min_validity_rate: float = Field(..., description="Minimum validity rate")
+    max_validity_rate: float = Field(..., description="Maximum validity rate")
+    avg_completeness: float = Field(..., description="Average completeness")
+    avg_duration_ms: float = Field(..., description="Average validation duration")
+    overall_validity_rate: float = Field(..., description="Overall validity rate from totals")
+    top_error_types: Dict[str, int] = Field(
+        default_factory=dict, description="Most common error types"
+    )
+
+
+class MetricsQueryResponse(BaseModel):
+    """Response model for metrics query."""
+
+    metrics: List[ValidationMetricResponse] = Field(
+        ..., description="List of matching metrics"
+    )
+    total: int = Field(..., description="Total number of metrics matching filters")
+    limit: int = Field(..., description="Maximum results returned")
+    offset: int = Field(..., description="Pagination offset")
+
+
+class RecordMetricRequest(BaseModel):
+    """Request model for recording a validation metric."""
+
+    schema_name: str = Field(..., description="Name of the schema validated")
+    version: str = Field(..., description="Version of the schema")
+    record_count: int = Field(default=1, description="Number of records validated")
+    valid_count: int = Field(default=1, description="Number of valid records")
+    error_count: int = Field(default=0, description="Number of records with errors")
+    validity_rate: Optional[float] = Field(
+        default=None, description="Validity rate (calculated if not provided)"
+    )
+    completeness: float = Field(default=1.0, description="Data completeness")
+    field_completeness: Dict[str, float] = Field(
+        default_factory=dict, description="Per-field completeness"
+    )
+    duration_ms: float = Field(default=0.0, description="Validation duration")
+    errors_by_type: Dict[str, int] = Field(
+        default_factory=dict, description="Error counts by type"
+    )
+    metadata: Dict[str, Any] = Field(
+        default_factory=dict, description="Additional metadata"
+    )
+
+
+class ExportResponse(BaseModel):
+    """Response model for metrics export."""
+
+    format: ExportFormat = Field(..., description="Export format used")
+    data: str = Field(..., description="Exported data as string")
+    count: int = Field(..., description="Number of metrics exported")
+
+
+class SchemaListResponse(BaseModel):
+    """Response model for list of schemas with metrics."""
+
+    schemas: List[str] = Field(..., description="List of schema names with recorded metrics")

api/models/validation.py

@@ -15,6 +15,7 @@ class ValidationRequest(BaseModel):
     data: Dict[str, Any] = Field(..., description="Data to validate")
     version: Optional[str] = Field(None, description="Schema version (default: latest)")
     strict: bool = Field(False, description="If True, raise exceptions on validation errors")
+    include_quality: bool = Field(False, description="If True, include quality metrics in response")
     
     class Config:
         json_schema_extra = {
@@ -25,7 +26,8 @@ class ValidationRequest(BaseModel):
                     "age": 30
                 },
                 "version": "1.0.0",
-                "strict": False
+                "strict": False,
+                "include_quality": False
             }
         }
 
@@ -47,6 +49,29 @@ class ValidationErrorDetail(BaseModel):
         }
 
 
+class ValidationQualityMetrics(BaseModel):
+    """Quality metrics for validated data."""
+    
+    completeness: float = Field(..., description="Ratio of non-null fields (0.0 to 1.0)")
+    field_completeness: Dict[str, float] = Field(default_factory=dict, description="Per-field completeness ratios")
+    record_count: int = Field(0, description="Number of records validated")
+    valid_count: int = Field(0, description="Number of valid records")
+    error_count: int = Field(0, description="Number of records with errors")
+    validity_rate: float = Field(..., description="Ratio of valid records")
+    
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "completeness": 0.95,
+                "field_completeness": {"name": 1.0, "age": 0.9, "email": 0.85},
+                "record_count": 100,
+                "valid_count": 95,
+                "error_count": 5,
+                "validity_rate": 0.95
+            }
+        }
+
+
 class ValidationResponse(BaseModel):
     """Response model for validation result."""
     
@@ -54,6 +79,7 @@ class ValidationResponse(BaseModel):
     data: Optional[Dict[str, Any]] = Field(None, description="Validated data (if valid)")
     errors: List[ValidationErrorDetail] = Field(default_factory=list, description="Validation errors (if invalid)")
     error_count: int = Field(0, description="Number of validation errors")
+    quality: Optional[ValidationQualityMetrics] = Field(None, description="Quality metrics (if requested)")
     
     class Config:
         json_schema_extra = {
@@ -64,7 +90,8 @@ class ValidationResponse(BaseModel):
                     "age": 30
                 },
                 "errors": [],
-                "error_count": 0
+                "error_count": 0,
+                "quality": None
             }
         }
 
@@ -77,6 +104,7 @@ class ValidationBatchRequest(BaseModel):
     data_list: List[Dict[str, Any]] = Field(..., description="List of data dictionaries to validate")
     version: Optional[str] = Field(None, description="Schema version (default: latest)")
     strict: bool = Field(False, description="If True, raise exceptions on validation errors")
+    include_quality: bool = Field(False, description="If True, include quality metrics in response")
     
     class Config:
         json_schema_extra = {
@@ -87,7 +115,8 @@ class ValidationBatchRequest(BaseModel):
                     {"name": "Bob", "age": 25}
                 ],
                 "version": "1.0.0",
-                "strict": False
+                "strict": False,
+                "include_quality": False
             }
         }
 
@@ -99,6 +128,7 @@ class ValidationBatchResponse(BaseModel):
     total_count: int = Field(..., description="Total number of items validated")
     valid_count: int = Field(..., description="Number of valid items")
     invalid_count: int = Field(..., description="Number of invalid items")
+    quality: Optional[ValidationQualityMetrics] = Field(None, description="Aggregate quality metrics (if requested)")
     
     class Config:
         json_schema_extra = {
@@ -108,7 +138,8 @@ class ValidationBatchResponse(BaseModel):
                         "is_valid": True,
                         "data": {"name": "Alice", "age": 30},
                         "errors": [],
-                        "error_count": 0
+                        "error_count": 0,
+                        "quality": None
                     },
                     {
                         "is_valid": False,
@@ -120,12 +151,21 @@ class ValidationBatchResponse(BaseModel):
                                 "input_value": -5
                             }
                         ],
-                        "error_count": 1
+                        "error_count": 1,
+                        "quality": None
                     }
                 ],
                 "total_count": 2,
                 "valid_count": 1,
-                "invalid_count": 1
+                "invalid_count": 1,
+                "quality": {
+                    "completeness": 0.95,
+                    "field_completeness": {"name": 1.0, "age": 0.9},
+                    "record_count": 2,
+                    "valid_count": 1,
+                    "error_count": 1,
+                    "validity_rate": 0.5
+                }
             }
         }
 

api/routes/v1/__init__.py

@@ -6,7 +6,17 @@ All routers defined here are automatically included in the main application
 with the `/api/v1` prefix.
 """
 
-from api.routes.v1 import contracts, metadata, quality, schemas, validation, settings
+from api.routes.v1 import (
+    contracts,
+    metadata,
+    quality,
+    schemas,
+    validation,
+    settings,
+    docs,
+    tracking,
+    evolution,
+)
 
 # Export all routers for automatic inclusion in main.py
 __all__ = [
@@ -16,5 +26,8 @@ __all__ = [
     "schemas",
     "validation",
     "settings",
+    "docs",
+    "tracking",
+    "evolution",
 ]
 

api/routes/v1/docs.py

@@ -0,0 +1,187 @@
+"""
+API routes for documentation generation.
+
+Provides endpoints to generate human-readable documentation
+from data contracts.
+"""
+
+from typing import Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+
+from api.dependencies.store import get_metadata_store
+from api.models.docs import (
+    DocsFormat,
+    DocsRequest,
+    DocsResponse,
+    DocsSectionRequest,
+    DocsSectionResponse,
+)
+from pycharter.contract_parser import parse_contract
+from pycharter.docs_generator import DocsGenerator, generate_docs
+from pycharter.docs_generator.renderers import HTMLRenderer, MarkdownRenderer
+from pycharter.metadata_store import MetadataStoreClient
+
+router = APIRouter(prefix="/docs", tags=["Documentation"])
+
+
+@router.post(
+    "/generate",
+    response_model=DocsResponse,
+    summary="Generate documentation from contract",
+    description="Generate human-readable documentation from contract data",
+)
+async def generate_documentation(request: DocsRequest) -> DocsResponse:
+    """
+    Generate documentation from a contract.
+
+    Accepts contract data and generates formatted documentation
+    in the specified format (Markdown or HTML).
+    """
+    try:
+        # Parse the contract
+        contract = parse_contract(request.contract, validate=False)
+
+        # Generate documentation
+        docs = generate_docs(
+            contract,
+            format=request.format.value,
+            include_schema=request.include_schema,
+            include_coercions=request.include_coercions,
+            include_validations=request.include_validations,
+            include_metadata=request.include_metadata,
+        )
+
+        # Extract schema name and version
+        schema_name = contract.schema.get("title") or contract.metadata.get("title")
+        version = contract.versions.get("schema") or contract.schema.get("version")
+
+        return DocsResponse(
+            documentation=docs,
+            format=request.format,
+            schema_name=schema_name,
+            version=version,
+        )
+
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=f"Failed to generate documentation: {e}")
+
+
+@router.post(
+    "/section",
+    response_model=DocsSectionResponse,
+    summary="Generate a specific documentation section",
+    description="Generate a specific section of documentation from contract data",
+)
+async def generate_section(request: DocsSectionRequest) -> DocsSectionResponse:
+    """
+    Generate a specific section of documentation.
+
+    Sections: 'schema', 'coercions', 'validations', 'metadata'
+    """
+    try:
+        # Parse the contract
+        contract = parse_contract(request.contract, validate=False)
+
+        # Select renderer
+        if request.format == DocsFormat.HTML:
+            renderer = HTMLRenderer()
+        else:
+            renderer = MarkdownRenderer()
+
+        generator = DocsGenerator(renderer=renderer)
+
+        # Generate the requested section
+        section = request.section.lower()
+        if section == "schema":
+            content = generator.generate_schema_section(contract.schema)
+        elif section in ("coercions", "coercion_rules"):
+            content = generator.generate_coercion_section(contract.coercion_rules)
+        elif section in ("validations", "validation_rules"):
+            content = generator.generate_validation_section(contract.validation_rules)
+        elif section == "metadata":
+            content = generator.generate_metadata_section(contract)
+        else:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Unknown section: {section}. Valid sections: schema, coercions, validations, metadata",
+            )
+
+        return DocsSectionResponse(
+            section=section,
+            content=content,
+            format=request.format,
+        )
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=f"Failed to generate section: {e}")
+
+
+@router.get(
+    "/{schema_name}",
+    response_model=DocsResponse,
+    summary="Generate documentation for a stored schema",
+    description="Generate documentation for a schema stored in the metadata store",
+)
+async def get_schema_docs(
+    schema_name: str,
+    version: Optional[str] = Query(default=None, description="Schema version (latest if not specified)"),
+    format: DocsFormat = Query(default=DocsFormat.MARKDOWN, description="Output format"),
+    store: MetadataStoreClient = Depends(get_metadata_store),
+) -> DocsResponse:
+    """
+    Generate documentation for a schema from the metadata store.
+
+    Retrieves the schema from the store and generates documentation.
+    """
+    try:
+        # Get the complete schema from store
+        schema_data = store.get_complete_schema(schema_name, version=version)
+
+        if not schema_data:
+            raise HTTPException(
+                status_code=404, detail=f"Schema not found: {schema_name}"
+            )
+
+        # Build contract data from stored components
+        contract_data = {"schema": schema_data}
+
+        # Try to get additional components
+        try:
+            coercion_rules = store.get_coercion_rules(schema_name, version=version)
+            if coercion_rules:
+                contract_data["coercion_rules"] = coercion_rules
+        except Exception:
+            pass
+
+        try:
+            validation_rules = store.get_validation_rules(schema_name, version=version)
+            if validation_rules:
+                contract_data["validation_rules"] = validation_rules
+        except Exception:
+            pass
+
+        try:
+            metadata = store.get_metadata(schema_name, version=version)
+            if metadata:
+                contract_data["metadata"] = metadata
+        except Exception:
+            pass
+
+        # Parse and generate docs
+        contract = parse_contract(contract_data, validate=False)
+        docs = generate_docs(contract, format=format.value)
+
+        return DocsResponse(
+            documentation=docs,
+            format=format,
+            schema_name=schema_name,
+            version=version or contract.versions.get("schema"),
+        )
+
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to generate documentation: {e}")