pycharter 0.0.22__py3-none-any.whl → 0.0.24__py3-none-any.whl

pycharter/__init__.py

@@ -1,22 +1,97 @@
 """
-PyCharter - Data Contract Management and Validation
+PyCharter - Data Contract Management, ETL Pipelines, and Validation
 
 Core Services:
-1. Contract Parser - Reads and decomposes data contract files
-2. Contract Builder - Constructs consolidated contracts from separate artifacts
-3. Metadata Store - Database operations for metadata storage
-4. Pydantic Generator - Generates Pydantic models from JSON Schema
-5. JSON Schema Converter - Converts Pydantic models to JSON Schema
-6. Runtime Validator - Validation utilities
-7. Quality Assurance - Data quality checking and monitoring
-
-API Organization:
-- Tier 1: Primary Interfaces (Classes - Use these for best performance)
-- Tier 2: Convenience Functions (Quick start - one-off use cases)
-- Tier 3: Low-Level Utilities (When you already have models/schemas)
+1. ETL Pipelines - Build and run ETL pipelines with | operator
+2. Contract Parser - Reads and decomposes data contract files
+3. Contract Builder - Constructs consolidated contracts from separate artifacts
+4. Metadata Store - Database operations for metadata storage
+5. Pydantic Generator - Generates Pydantic models from JSON Schema
+6. JSON Schema Converter - Converts Pydantic models to JSON Schema
+7. Runtime Validator - Validation utilities
+8. Quality Assurance - Data quality checking and monitoring
+
+ETL API:
+    >>> from pycharter import Pipeline, HTTPExtractor, PostgresLoader, Rename, AddField
+    >>> 
+    >>> # Programmatic pipeline with | operator
+    >>> pipeline = (
+    ...     Pipeline(HTTPExtractor(url="https://api.example.com/data"))
+    ...     | Rename({"old": "new"})
+    ...     | AddField("processed_at", "now()")
+    ...     | PostgresLoader(connection_string="...", table="users")
+    ... )
+    >>> result = await pipeline.run()  # run() is async; use asyncio.run() from scripts
+    >>> 
+    >>> # Config-driven (explicit files)
+    >>> pipeline = Pipeline.from_config_files(
+    ...     extract="configs/extract.yaml",
+    ...     load="configs/load.yaml",
+    ...     variables={"API_KEY": "secret"}
+    ... )
+    >>> 
+    >>> # Config-driven (directory with extract.yaml, transform.yaml, load.yaml)
+    >>> pipeline = Pipeline.from_config_dir("pipelines/users/")
+    >>> 
+    >>> # Config-driven (single file)
+    >>> pipeline = Pipeline.from_config_file("pipelines/users/pipeline.yaml")
+
+Validator API:
+    >>> from pycharter import Validator
+    >>> 
+    >>> # From explicit files
+    >>> validator = Validator.from_files(schema="schema.yaml")
+    >>> 
+    >>> # From directory
+    >>> validator = Validator.from_dir("contracts/users/")
+    >>> 
+    >>> result = validator.validate({"name": "Alice", "age": 30})
 """
 
-__version__ = "0.0.22"
+__version__ = "0.1.0"
+
+# ============================================================================
+# ETL PIPELINES
+# ============================================================================
+
+from pycharter.etl_generator import (
+    # Core
+    Pipeline,
+    PipelineBuilder,
+    PipelineContext,
+    PipelineResult,
+    BatchResult,
+    LoadResult,
+    # Protocols
+    Extractor,
+    Transformer,
+    Loader,
+    # Extractors
+    BaseExtractor,
+    HTTPExtractor,
+    FileExtractor,
+    DatabaseExtractor,
+    CloudStorageExtractor,
+    # Transformers
+    BaseTransformer,
+    TransformerChain,
+    Rename,
+    AddField,
+    Drop,
+    Select,
+    Filter,
+    Convert,
+    Default,
+    Map,
+    FlatMap,
+    CustomFunction,
+    # Loaders
+    BaseLoader,
+    PostgresLoader,
+    DatabaseLoader,
+    FileLoader,
+    CloudStorageLoader,
+)
 
 # ============================================================================
 # TIER 1: PRIMARY INTERFACES (Classes - Use these for best performance)
@@ -25,7 +100,9 @@ __version__ = "0.0.22"
 # Runtime Validator (PRIMARY INTERFACE)
 from pycharter.runtime_validator import (
     Validator,  # ⭐ PRIMARY: Use this for validation
+    ValidatorBuilder,  # Fluent API for building validators
     create_validator,
+    QualityMetrics as ValidationQualityMetrics,  # Quality metrics from validation
 )
 
 # Quality Assurance (PRIMARY INTERFACE)
@@ -140,22 +217,145 @@ from pycharter.quality import (
     ViolationTracker,
     ViolationRecord,
     DataProfiler,
+    # Tracking submodule
+    MetricsCollector,
+    ValidationMetric,
+    MetricsSummary,
+    InMemoryMetricsStore,
+    SQLiteMetricsStore,
+)
+
+# ============================================================================
+# DOCUMENTATION GENERATION
+# ============================================================================
+
+from pycharter.docs_generator import (
+    DocsGenerator,
+    generate_docs,
+    MarkdownRenderer,
+    HTMLRenderer,
+)
+
+# ============================================================================
+# SCHEMA EVOLUTION
+# ============================================================================
+
+from pycharter.schema_evolution import (
+    check_compatibility,
+    compute_diff,
+    CompatibilityResult,
+    SchemaDiff,
+    SchemaChange,
+    ChangeType,
+    CompatibilityMode,
+)
+
+# ============================================================================
+# PROTOCOLS AND ERROR HANDLING
+# ============================================================================
+
+from pycharter.shared import (
+    # Protocols for extensibility
+    MetadataStore,
+    CoercionRegistry,
+    ValidationRegistry,
+    DataValidator,
+    # Exception hierarchy (catch PyCharterError for any pycharter failure)
+    PyCharterError,
+    ConfigError,
+    ConfigValidationError,
+    ConfigLoadError,
+    ExpressionError,
+    # Error handling
+    ErrorMode,
+    ErrorContext,
+    StrictMode,
+    LenientMode,
+    set_error_mode,
 )
 
 __all__ = [
     # ========================================================================
-    # TIER 1: PRIMARY INTERFACES
+    # ETL PIPELINES
+    # ========================================================================
+    # Core
+    "Pipeline",
+    "PipelineBuilder",
+    "PipelineContext",
+    "PipelineResult",
+    "BatchResult",
+    "LoadResult",
+    # Protocols
+    "Extractor",
+    "Transformer",
+    "Loader",
+    # Extractors
+    "BaseExtractor",
+    "HTTPExtractor",
+    "FileExtractor",
+    "DatabaseExtractor",
+    "CloudStorageExtractor",
+    # Transformers
+    "BaseTransformer",
+    "TransformerChain",
+    "Rename",
+    "AddField",
+    "Drop",
+    "Select",
+    "Filter",
+    "Convert",
+    "Default",
+    "Map",
+    "FlatMap",
+    "CustomFunction",
+    # Loaders
+    "BaseLoader",
+    "PostgresLoader",
+    "DatabaseLoader",
+    "FileLoader",
+    "CloudStorageLoader",
+    # Exceptions and error handling
+    "PyCharterError",
+    "ConfigError",
+    "ConfigValidationError",
+    "ConfigLoadError",
+    "ExpressionError",
+    "ErrorMode",
+    "ErrorContext",
+    "StrictMode",
+    "LenientMode",
+    "set_error_mode",
     # ========================================================================
-    "Validator",  # ⭐ PRIMARY: Use for validation
+    # DATA CONTRACT SERVICES
+    # ========================================================================
+    # Validation
+    "Validator",
+    "ValidatorBuilder",
     "create_validator",
-    "QualityCheck",  # ⭐ PRIMARY: Use for quality checks
+    "ValidationResult",
+    "ValidationQualityMetrics",
+    "validate_with_store",
+    "validate_batch_with_store",
+    "validate_with_contract",
+    "validate_batch_with_contract",
+    "get_model_from_store",
+    "get_model_from_contract",
+    "validate",
+    "validate_batch",
+    "validate_input",
+    "validate_output",
+    "validate_with_contract_decorator",
+    # Quality
+    "QualityCheck",
     "QualityCheckOptions",
     "QualityReport",
     "QualityThresholds",
-    "MetadataStoreClient",
-    # ========================================================================
-    # TIER 2: CONVENIENCE FUNCTIONS
-    # ========================================================================
+    "QualityMetrics",
+    "QualityScore",
+    "FieldQualityMetrics",
+    "ViolationTracker",
+    "ViolationRecord",
+    "DataProfiler",
     # Contract Management
     "parse_contract",
     "parse_contract_file",
@@ -163,49 +363,59 @@ __all__ = [
     "build_contract",
     "build_contract_from_store",
     "ContractArtifacts",
-    # Pydantic Generator (input type helpers)
-    "from_dict",  # Quick: dict → model
-    "from_file",  # Quick: file → model
-    "from_json",  # Quick: JSON string → model
-    "from_url",  # Quick: URL → model
-    "generate_model",  # Advanced: more control
+    # Pydantic Generator
+    "from_dict",
+    "from_file",
+    "from_json",
+    "from_url",
+    "generate_model",
     "generate_model_file",
-    # JSON Schema Converter (output type helpers)
-    "to_dict",  # Quick: model → dict
-    "to_file",  # Quick: model → file
-    "to_json",  # Quick: model → JSON string
-    "model_to_schema",  # Advanced: core conversion
-    # Runtime Validator (data source helpers)
-    "ValidationResult",
-    "validate_with_store",  # Quick: store → validate
-    "validate_batch_with_store",  # Quick: batch validate with store
-    "validate_with_contract",  # Quick: contract → validate
-    "validate_batch_with_contract",  # Quick: batch validate with contract
-    "get_model_from_store",  # Quick: store → model
-    "get_model_from_contract",  # Quick: contract → model
-    "validate_input",  # Decorator
-    "validate_output",  # Decorator
-    "validate_with_contract_decorator",
-    # ========================================================================
-    # TIER 3: LOW-LEVEL UTILITIES
-    # ========================================================================
-    "validate",  # Low-level: model → validate
-    "validate_batch",  # Low-level: model → batch validate
-    # ========================================================================
-    # METADATA STORE IMPLEMENTATIONS
-    # ========================================================================
+    # JSON Schema Converter
+    "to_dict",
+    "to_file",
+    "to_json",
+    "model_to_schema",
+    # Metadata Store
+    "MetadataStoreClient",
     "InMemoryMetadataStore",
     "MongoDBMetadataStore",
     "PostgresMetadataStore",
     "RedisMetadataStore",
     "SQLiteMetadataStore",
+    # Protocols
+    "MetadataStore",
+    "CoercionRegistry",
+    "ValidationRegistry",
+    "DataValidator",
+    # Error handling
+    "ErrorMode",
+    "ErrorContext",
+    "StrictMode",
+    "LenientMode",
+    "set_error_mode",
     # ========================================================================
-    # QUALITY ASSURANCE - Additional utilities
+    # QUALITY TRACKING
     # ========================================================================
-    "QualityMetrics",
-    "QualityScore",
-    "FieldQualityMetrics",
-    "ViolationTracker",
-    "ViolationRecord",
-    "DataProfiler",
+    "MetricsCollector",
+    "ValidationMetric",
+    "MetricsSummary",
+    "InMemoryMetricsStore",
+    "SQLiteMetricsStore",
+    # ========================================================================
+    # DOCUMENTATION GENERATION
+    # ========================================================================
+    "DocsGenerator",
+    "generate_docs",
+    "MarkdownRenderer",
+    "HTMLRenderer",
+    # ========================================================================
+    # SCHEMA EVOLUTION
+    # ========================================================================
+    "check_compatibility",
+    "compute_diff",
+    "CompatibilityResult",
+    "SchemaDiff",
+    "SchemaChange",
+    "ChangeType",
+    "CompatibilityMode",
 ]

pycharter/data/templates/contract/template_coercion_rules.yaml

@@ -0,0 +1,57 @@
+# Coercion Rules Template
+# Pre-validation data type transformations
+#
+# Usage:
+#   Coercion rules transform input data BEFORE Pydantic validation.
+#   Useful for handling messy data from external sources.
+#
+# Available coercions:
+#   - coerce_to_string          : Convert to string
+#   - coerce_to_integer         : Convert to int (handles "123", 123.0)
+#   - coerce_to_float           : Convert to float
+#   - coerce_to_boolean         : Convert to bool ("true"/"false", 0/1)
+#   - coerce_to_datetime        : Parse datetime strings (ISO format)
+#   - coerce_to_date            : Parse date strings
+#   - coerce_to_uuid            : Parse UUID strings
+#   - coerce_to_lowercase       : Convert string to lowercase
+#   - coerce_to_uppercase       : Convert string to uppercase
+#   - coerce_to_stripped_string : Trim whitespace
+#   - coerce_to_list            : Wrap single value in list
+#   - coerce_to_json            : Parse JSON string
+#   - coerce_empty_to_null      : Convert "" to None
+#   - coerce_to_none            : Always return None
+#
+# Nullable variants (preserve None values):
+#   - coerce_to_nullable_string
+#   - coerce_to_nullable_integer
+#   - coerce_to_nullable_float
+#   - coerce_to_nullable_datetime
+#   - coerce_to_nullable_uuid
+#   - coerce_to_nullable_json
+
+title: my_coercion_rules
+description: Coercion rules for data transformation
+version: "1.0.0"
+
+rules:
+  # String fields
+  id: coerce_to_string
+  name: coerce_to_stripped_string
+  status: coerce_to_lowercase
+
+  # Numeric fields
+  count: coerce_to_integer
+  amount: coerce_to_float
+
+  # Boolean
+  is_active: coerce_to_boolean
+
+  # Datetime
+  created_at: coerce_to_datetime
+
+  # Nullable fields (use nullable variants)
+  notes: coerce_to_nullable_string
+  email: coerce_to_nullable_string
+
+  # UUID
+  uuid: coerce_to_uuid

pycharter/data/templates/contract/template_contract.yaml

@@ -0,0 +1,122 @@
+# Complete Data Contract Template
+# Combines schema, coercion rules, validation rules, and metadata
+#
+# Usage:
+#   from pycharter import parse_contract, validate_with_contract
+#
+#   contract = parse_contract(contract_dict)  # or parse_contract_file("contract.yaml")
+#   result = validate_with_contract(contract_dict, data)
+
+schema:
+  type: object
+  title: User
+  version: "1.0.0"
+
+  properties:
+    user_id:
+      type: string
+      description: Unique user identifier
+      minLength: 1
+      # PyCharter extension: coercion
+      coercion: coerce_to_string
+      # PyCharter extension: validations
+      validations:
+        non_empty_string: null
+
+    name:
+      type: string
+      description: User's full name
+      minLength: 1
+      maxLength: 100
+      coercion: coerce_to_stripped_string
+      validations:
+        non_empty_string: null
+        max_length:
+          threshold: 100
+
+    email:
+      type: string
+      format: email
+      description: User email address
+      coercion: coerce_to_lowercase
+      validations:
+        is_email: null
+
+    age:
+      type: integer
+      description: User age
+      minimum: 0
+      maximum: 150
+      coercion: coerce_to_integer
+      validations:
+        greater_than_or_equal_to:
+          threshold: 0
+        less_than_or_equal_to:
+          threshold: 150
+
+    status:
+      type: string
+      description: Account status
+      enum: [active, inactive, pending]
+      coercion: coerce_to_lowercase
+      validations:
+        only_allow:
+          allowed_values: [active, inactive, pending]
+
+    created_at:
+      type: string
+      format: date-time
+      description: Account creation timestamp
+      coercion: coerce_to_datetime
+
+    # Nullable field example
+    notes:
+      anyOf:
+        - type: string
+          maxLength: 500
+        - type: "null"
+      default: null
+      description: Optional notes
+      coercion: coerce_to_nullable_string
+      validations:
+        max_length:
+          threshold: 500
+
+  required:
+    - user_id
+    - name
+    - email
+    - status
+    - created_at
+
+# Contract metadata (ownership and governance nested inside)
+metadata:
+  version: "1.0.0"
+  description: User data contract
+  status: active
+  created_at: "2025-01-01T00:00:00Z"
+  created_by: data-team
+
+  # Ownership information (nested in metadata)
+  ownership:
+    owner: data-team
+    team: engineering
+
+  # Governance rules (nested in metadata)
+  governance_rules:
+    data_retention:
+      days: 365
+      reason: Standard user data retention
+    pii_fields:
+      fields: [email, name]
+    sensitivity: confidential
+    access_control:
+      read: [analytics-team, data-team]
+      write: [data-team]
+
+# Version tracking (optional - for build_contract_from_store)
+versions:
+  schema: "1.0.0"
+  coercion_rules: "1.0.0"
+  validation_rules: "1.0.0"
+  metadata: "1.0.0"

pycharter/data/templates/contract/template_metadata.yaml

@@ -0,0 +1,68 @@
+# Metadata Template
+# Business and governance metadata for your data contract
+#
+# Usage:
+#   Metadata provides context, ownership, and governance information
+#   about your data asset. This is separate from the schema itself.
+
+# Basic Information
+title: my_entity
+version: "1.0.0"
+status: active
+description: Description of what this data represents
+
+# Audit Trail
+created_at: "2025-01-01T00:00:00Z"
+updated_at: "2025-01-01T00:00:00Z"
+created_by: your_username
+updated_by: your_username
+
+# Ownership
+business_owners:
+  - team-name
+  - owner-email@company.com
+
+# Domain Classification
+domain: domain_name
+# domains: [domain1, domain2]  # Alternative: multiple domains
+
+# Data Lineage
+pulls_from:
+  - source_system_1
+  - source_system_2
+pushes_to:
+  - destination_system_1
+system_sources:
+  - origin_system
+
+# Data Quality
+data_quality:
+  accuracy: 0.0
+  monitoring_enabled: false
+  sla: null
+  total_records: 0
+  last_updated: "2025-01-01T00:00:00Z"
+
+# Governance Rules
+governance_rules:
+  data_retention:
+    days: 365
+    reason: Standard retention policy
+  pii_fields:
+    fields: []
+  sensitivity: internal  # internal, confidential, restricted, public
+  access_control:
+    read:
+      - data-team
+      - analytics-team
+    write:
+      - data-team
+
+# Changelog
+changelog:
+  - version: "1.0.0"
+    date: "2025-01-01T00:00:00Z"
+    author: your_username
+    type: initial
+    changes:
+      - Initial contract creation