pycharter 0.0.22__py3-none-any.whl → 0.0.24__py3-none-any.whl

pycharter/data/templates/etl/transform_jsonata.yaml

@@ -0,0 +1,51 @@
+# Template: JSONata Transformation
+# Copy to your pipeline directory as transform.yaml
+# Requires: pip install pyjsonata
+#
+# JSONata is a query and transformation language for JSON
+# Docs: https://jsonata.org/
+
+title: jsonata_transformation
+description: Transform data using JSONata expressions
+version: "1.0.0"
+
+# Mode: record (per-record) or batch (all records at once)
+jsonata:
+  mode: record
+
+  # JSONata expression
+  # Available variables:
+  #   $ - current record (in record mode) or array (in batch mode)
+  #   $now() - current timestamp
+  #   $uuid() - generate UUID
+
+  expression: |
+    {
+      "id": id,
+      "full_name": firstName & " " & lastName,
+      "email": $lowercase(email),
+      "age": $number(age),
+      "is_adult": age >= 18,
+      "tags": $split(tags_string, ","),
+      "created_at": $now()
+    }
+
+# Example: Batch mode for aggregations
+# jsonata:
+#   mode: batch
+#   expression: |
+#     {
+#       "total_records": $count($),
+#       "total_amount": $sum($.amount),
+#       "records": $
+#     }
+
+# Example: Filter and transform
+# jsonata:
+#   mode: batch
+#   expression: |
+#     $[status = "active"].{
+#       "id": id,
+#       "name": name,
+#       "score_normalized": score / 100
+#     }

pycharter/data/templates/etl/transform_simple.yaml

@@ -0,0 +1,59 @@
+# Template: Simple Transformations
+# Copy to your pipeline directory as transform.yaml
+#
+# Transformation order: rename → convert → defaults → add → select → drop
+#
+# Usage with Pipeline:
+#   pipeline = (
+#       Pipeline(HTTPExtractor(...))
+#       | Rename({"old": "new"})
+#       | Convert({"price": float})
+#       | PostgresLoader(...)
+#   )
+
+title: simple_transformation
+description: Rename, convert, defaults, add, select, drop
+version: "1.0.0"
+
+transform:
+  # Rename fields
+  rename:
+    oldFieldName: new_field_name
+    userId: user_id
+    createdAt: created_at
+
+  # Convert field types
+  # Supported: int, integer, float, str, string, bool, boolean, datetime
+  convert:
+    price: float
+    quantity: int
+    is_active: bool
+    created_at: datetime
+
+  # Set default values for missing/null fields
+  defaults:
+    status: "pending"
+    region: "unknown"
+    count: 0
+
+  # Add new computed fields
+  # Supported expressions: ${field}, now(), uuid(), literal values
+  add:
+    full_name: "${first_name} ${last_name}"
+    processed_at: "now()"
+    record_id: "uuid()"
+    source: "api"
+
+  # Keep only these fields (omit to keep all)
+  select:
+    - id
+    - user_id
+    - full_name
+    - email
+    - status
+    - processed_at
+
+  # Remove these fields (applied after select)
+  drop:
+    - _internal_id
+    - _metadata

pycharter/db/schemas/.ipynb_checkpoints/data_contract-checkpoint.py

@@ -0,0 +1,160 @@
+"""
+Pydantic models for data contract validation.
+
+These models ensure that data contracts strictly adhere to the database table design.
+Based on the database schema:
+- data_contracts table
+- schemas table
+- metadata_records table
+- owners table
+- coercion_rules table
+- validation_rules table
+"""
+
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, Field, field_validator
+
+
+class SchemaComponent(BaseModel):
+    """
+    Schema component - required field.
+    
+    Must be a valid JSON Schema object. Stored in schemas table.
+    """
+    type: str = Field(..., description="JSON Schema type (e.g., 'object')")
+    properties: Optional[Dict[str, Any]] = Field(None, description="JSON Schema properties")
+    required: Optional[List[str]] = Field(None, description="Required fields")
+    version: Optional[str] = Field(None, max_length=50, description="Schema version")
+    title: Optional[str] = Field(None, description="Schema title")
+    
+    model_config = {
+        "extra": "allow",  # Allow additional JSON Schema fields
+    }
+
+
+class MetadataComponent(BaseModel):
+    """
+    Metadata component - optional field.
+    
+    Stored in metadata_records table. Matches database column constraints.
+    """
+    version: Optional[str] = Field(None, max_length=50, description="Metadata version")
+    title: Optional[str] = Field(None, max_length=255, description="Title (matches metadata_records.title)")
+    status: Optional[str] = Field(None, max_length=50, description="Status: active, deprecated, or draft")
+    type: Optional[str] = Field(None, max_length=50, description="Type (matches metadata_records.type)")
+    description: Optional[str] = Field(None, description="Description")
+    created_by: Optional[str] = Field(None, max_length=255, description="Created by (matches metadata_records.created_by)")
+    updated_by: Optional[str] = Field(None, max_length=255, description="Updated by (matches metadata_records.updated_by)")
+    
+    # Ownership fields (stored in metadata_records as JSON arrays)
+    business_owners: Optional[List[str]] = Field(None, description="Business owners (stored in metadata_records.business_owners as JSON)")
+    bu_sme: Optional[List[str]] = Field(None, description="BU SMEs (stored in metadata_records.bu_sme as JSON)")
+    it_application_owners: Optional[List[str]] = Field(None, description="IT Application Owners (stored in metadata_records.it_application_owners as JSON)")
+    it_sme: Optional[List[str]] = Field(None, description="IT SMEs (stored in metadata_records.it_sme as JSON)")
+    support_lead: Optional[List[str]] = Field(None, description="Support Lead (stored in metadata_records.support_lead as JSON)")
+    
+    @field_validator("status")
+    @classmethod
+    def validate_status(cls, v: Optional[str]) -> Optional[str]:
+        """Validate status enum values."""
+        if v is not None and v not in ["active", "deprecated", "draft"]:
+            raise ValueError("status must be one of: active, deprecated, draft")
+        return v
+    
+    model_config = {
+        "extra": "allow",  # Allow additional metadata fields (stored as JSON in database)
+    }
+
+
+class OwnershipComponent(BaseModel):
+    """
+    Ownership component - optional field.
+    
+    Stored in owners table. Matches database column constraints.
+    """
+    owner: Optional[str] = Field(None, max_length=255, description="Primary owner (matches owners.owner)")
+    team: Optional[str] = Field(None, max_length=255, description="Team name (matches owners.team)")
+    additional_info: Optional[Dict[str, Any]] = Field(None, description="Additional info (stored in owners.additional_info as JSON)")
+    
+    model_config = {
+        "extra": "allow",  # Allow additional ownership fields
+    }
+
+
+class GovernanceRulesComponent(BaseModel):
+    """
+    Governance rules component - optional field.
+    
+    Stored in metadata_records.governance_rules as JSON.
+    """
+    model_config = {
+        "extra": "allow",  # Governance rules can contain any structure (stored as JSON)
+    }
+
+
+class CoercionRulesComponent(BaseModel):
+    """
+    Coercion rules component - optional field.
+    
+    Stored in coercion_rules table. Structure matches database design.
+    """
+    version: Optional[str] = Field(None, max_length=50, description="Coercion rules version")
+    rules: Optional[Dict[str, str]] = Field(None, description="Coercion rules mapping field names to coercion function names")
+    
+    model_config = {
+        "extra": "allow",  # Allow additional fields in coercion rules
+    }
+
+
+class ValidationRulesComponent(BaseModel):
+    """
+    Validation rules component - optional field.
+    
+    Stored in validation_rules table. Structure matches database design.
+    """
+    version: Optional[str] = Field(None, max_length=50, description="Validation rules version")
+    rules: Optional[Dict[str, Dict[str, Any]]] = Field(None, description="Validation rules mapping field names to validation configurations")
+    
+    model_config = {
+        "extra": "allow",  # Allow additional fields in validation rules
+    }
+
+
+class VersionsComponent(BaseModel):
+    """
+    Versions component - optional field.
+    
+    Tracks versions of all components. Stored in data_contracts table version columns.
+    """
+    schema: Optional[str] = Field(None, max_length=50, description="Schema version")
+    metadata: Optional[str] = Field(None, max_length=50, description="Metadata version")
+    coercion_rules: Optional[str] = Field(None, max_length=50, description="Coercion rules version")
+    validation_rules: Optional[str] = Field(None, max_length=50, description="Validation rules version")
+    
+    model_config = {
+        "extra": "forbid",  # Versions should only contain these fields
+        "populate_by_name": True,
+    }
+
+
+class DataContractSchema(BaseModel):
+    """
+    Pydantic model for validating data contract structure.
+    
+    Ensures contracts strictly adhere to the database table design.
+    All fields match the database schema constraints.
+    """
+    schema: SchemaComponent = Field(..., description="JSON Schema definition (required, stored in schemas table)")
+    metadata: Optional[MetadataComponent] = Field(None, description="Metadata (optional, stored in metadata_records table)")
+    ownership: Optional[OwnershipComponent] = Field(None, description="Ownership (optional, stored in owners table)")
+    governance_rules: Optional[GovernanceRulesComponent] = Field(None, description="Governance rules (optional, stored in metadata_records.governance_rules as JSON)")
+    coercion_rules: Optional[CoercionRulesComponent] = Field(None, description="Coercion rules (optional, stored in coercion_rules table)")
+    validation_rules: Optional[ValidationRulesComponent] = Field(None, description="Validation rules (optional, stored in validation_rules table)")
+    versions: Optional[VersionsComponent] = Field(None, description="Version tracking (optional, stored in data_contracts table)")
+    
+    model_config = {
+        "extra": "forbid",  # Only allow defined fields to ensure strict adherence to database schema
+        "populate_by_name": True,
+    }
+

pycharter/docs_generator/__init__.py

@@ -0,0 +1,43 @@
+"""
+Documentation Generator - Generate documentation from data contracts.
+
+This module provides tools to generate human-readable documentation
+from ContractMetadata objects, supporting multiple output formats.
+
+Primary Interface:
+    - DocsGenerator: Main class for generating documentation
+    - generate_docs: Convenience function for quick documentation generation
+
+Renderers:
+    - MarkdownRenderer: Render documentation as Markdown
+    - HTMLRenderer: Render documentation as HTML
+
+Example:
+    >>> from pycharter import parse_contract_file
+    >>> from pycharter.docs_generator import generate_docs, DocsGenerator
+    >>>
+    >>> # Quick generation
+    >>> contract = parse_contract_file("contract.yaml")
+    >>> markdown = generate_docs(contract)
+    >>>
+    >>> # Custom configuration
+    >>> generator = DocsGenerator(renderer=HTMLRenderer())
+    >>> html = generator.generate(contract)
+"""
+
+from pycharter.docs_generator.generator import DocsGenerator, generate_docs
+from pycharter.docs_generator.renderers import (
+    DocsRenderer,
+    HTMLRenderer,
+    MarkdownRenderer,
+)
+
+__all__ = [
+    # Primary interface
+    "DocsGenerator",
+    "generate_docs",
+    # Renderers
+    "DocsRenderer",
+    "MarkdownRenderer",
+    "HTMLRenderer",
+]