pycharter 0.0.24__py3-none-any.whl → 0.0.26__py3-none-any.whl

pycharter/contract_builder/builder.py

@@ -3,7 +3,15 @@ Contract Builder - Constructs consolidated data contracts from separate artifact
 
 This module provides functionality to combine separate artifacts (schema, coercion rules,
 validation rules, metadata) into a single consolidated data contract that tracks all
-component versions and can be used for runtime validation.
+component versions.
+
+Note: The contract dict contains RAW schema + separate rules. The Validator class
+handles merging rules into the schema internally during validation. This separation
+ensures clarity:
+- contract["schema"] = raw schema (editable, source of truth)
+- contract["coercion_rules"] = separate coercion rules
+- contract["validation_rules"] = separate validation rules
+- For merged schema, use Validator or merge_rules_into_schema() utility
 """
 
 import copy
@@ -11,10 +19,6 @@ from dataclasses import dataclass
 from typing import Any, Dict, Optional
 
 from pycharter.metadata_store import MetadataStoreClient
-from pycharter.metadata_store.client import (
-    _merge_coercion_rules,
-    _merge_validation_rules,
-)
 
 
 @dataclass
@@ -82,8 +86,12 @@ def build_contract(
     Build a consolidated data contract from separate artifacts.
 
     This function combines schema, coercion rules, validation rules, and metadata
-    into a single consolidated contract. It tracks versions of all components
-    and produces a contract suitable for runtime validation.
+    into a single consolidated contract. It tracks versions of all components.
+
+    IMPORTANT: The schema in the returned contract is RAW (not merged with rules).
+    The Validator class handles merging rules into the schema internally during
+    validation. To get a merged schema explicitly, use merge_rules_into_schema()
+    from pycharter.runtime_validator.utils.
 
     Args:
         artifacts: ContractArtifacts object containing all separate artifacts
@@ -93,9 +101,9 @@ def build_contract(
 
     Returns:
         Consolidated contract dictionary with:
-        - schema: Complete schema with coercion/validation rules merged (must have version)
-        - coercion_rules: Coercion rules (if provided)
-        - validation_rules: Validation rules (if provided)
+        - schema: RAW schema (no coercion/validation merged; must have version)
+        - coercion_rules: Coercion rules dictionary (always present, may be empty)
+        - validation_rules: Validation rules dictionary (always present, may be empty)
         - metadata: Metadata containing ownership and governance_rules (if include_metadata=True)
         - versions: Dictionary tracking versions of all components
 
@@ -116,10 +124,10 @@ def build_contract(
         >>> contract = build_contract(artifacts)
         >>> contract["versions"]
         {'schema': '1.0.0', 'coercion_rules': '1.0.0', 'validation_rules': '1.0.0', 'metadata': '1.0.0'}
-        >>> contract["metadata"]["ownership"]
-        {'owner': 'data-team', 'team': 'engineering'}
-        >>> contract["metadata"]["governance_rules"]
-        {'data_retention': {'days': 365}}
+        >>> contract["coercion_rules"]  # Separate from schema
+        {'age': 'coerce_to_integer'}
+        >>> contract["schema"]["properties"]["age"].get("coercion")  # Not in schema
+        None
     """
     if not artifacts.schema:
         raise ValueError("Schema is required to build a contract")
@@ -131,34 +139,21 @@ def build_contract(
             "Please ensure the schema dictionary contains 'version': '<version_string>'."
         )
 
-    # Deep copy schema to avoid modifying original
-    complete_schema = copy.deepcopy(artifacts.schema)
-
-    # Extract and merge coercion rules
-    coercion_rules = _extract_rules(artifacts.coercion_rules)
-    if coercion_rules:
-        _merge_coercion_rules(complete_schema, coercion_rules)
+    # Deep copy schema to avoid modifying original - NO MERGING
+    # The Validator class handles merging internally during validation
+    raw_schema = copy.deepcopy(artifacts.schema)
 
-    # Extract and merge validation rules
-    validation_rules = _extract_rules(artifacts.validation_rules)
-    if validation_rules:
-        _merge_validation_rules(complete_schema, validation_rules)
+    # Extract rules (but do NOT merge into schema)
+    coercion_rules = _extract_rules(artifacts.coercion_rules) or {}
+    validation_rules = _extract_rules(artifacts.validation_rules) or {}
 
-    # Build contract
+    # Build contract with raw schema and separate rules
     contract: Dict[str, Any] = {
-        "schema": complete_schema,
+        "schema": raw_schema,
+        "coercion_rules": copy.deepcopy(coercion_rules) if coercion_rules else {},
+        "validation_rules": copy.deepcopy(validation_rules) if validation_rules else {},
     }
 
-    # Add coercion rules as top-level field (for UI display)
-    # Note: coercion_rules is already extracted above
-    if coercion_rules:
-        contract["coercion_rules"] = copy.deepcopy(coercion_rules)
-
-    # Add validation rules as top-level field (for UI display)
-    # Note: validation_rules is already extracted above
-    if validation_rules:
-        contract["validation_rules"] = copy.deepcopy(validation_rules)
-
     # Track versions
     versions: Dict[str, str] = {}
 

pycharter/data/seed/compliance_frameworks.yaml

@@ -0,0 +1,22 @@
+# Generic seed: compliance_frameworks (name required; code, description, framework_type, is_active, additional_metadata optional)
+
+- name: "internal-policy"
+  code: "INTERNAL"
+  description: "Internal data policy"
+  framework_type: "policy"
+  is_active: true
+  additional_metadata: {}
+
+- name: "soc2"
+  code: "SOC2"
+  description: "System and Organization Controls 2"
+  framework_type: "certification"
+  is_active: true
+  additional_metadata: {}
+
+- name: "gdpr"
+  code: "GDPR"
+  description: "General Data Protection Regulation"
+  framework_type: "regulation"
+  is_active: true
+  additional_metadata: {}

pycharter/data/seed/contracts.yaml

@@ -0,0 +1,130 @@
+# Example contract artifacts for pycharter db seed
+# Each entry creates a data contract with schema, coercion rules, validation rules, and metadata.
+# Skipped if a contract with the same name and version already exists.
+
+- name: example_user_contract
+  version: "1.0.0"
+  status: active
+  description: Example user data contract with schema, coercion, validation, and metadata
+
+  schema:
+    type: object
+    title: example_user
+    version: "1.0.0"
+    properties:
+      user_id:
+        type: string
+        description: Unique user identifier
+        minLength: 1
+      name:
+        type: string
+        description: Full name
+        minLength: 1
+        maxLength: 100
+      email:
+        type: string
+        format: email
+        description: Email address
+      status:
+        type: string
+        enum: [active, inactive, pending]
+        description: Account status
+      created_at:
+        type: string
+        format: date-time
+        description: Creation timestamp
+    required:
+      - user_id
+      - name
+      - email
+      - status
+      - created_at
+
+  coercion_rules:
+    title: example_user_coercion
+    version: "1.0.0"
+    rules:
+      user_id: coerce_to_string
+      name: coerce_to_stripped_string
+      email: coerce_to_lowercase
+      status: coerce_to_lowercase
+      created_at: coerce_to_datetime
+
+  validation_rules:
+    title: example_user_validation
+    version: "1.0.0"
+    rules:
+      user_id:
+        non_empty_string: null
+      name:
+        non_empty_string: null
+        max_length:
+          threshold: 100
+      email:
+        is_email: null
+      status:
+        only_allow:
+          allowed_values: [active, inactive, pending]
+
+  metadata:
+    title: example_user_metadata
+    version: "1.0.0"
+    status: active
+    description: Example user contract metadata
+    governance_rules:
+      data_retention:
+        days: 365
+        reason: Standard retention
+      sensitivity: internal
+      access_control:
+        read: [data-team]
+        write: [data-team]
+
+# Second example: minimal event contract
+- name: example_event_contract
+  version: "1.0.0"
+  status: active
+  description: Minimal event contract for demos
+
+  schema:
+    type: object
+    title: example_event
+    version: "1.0.0"
+    properties:
+      event_id:
+        type: string
+        description: Event identifier
+      event_type:
+        type: string
+        description: Type of event
+      occurred_at:
+        type: string
+        format: date-time
+    required:
+      - event_id
+      - event_type
+      - occurred_at
+
+  coercion_rules:
+    title: example_event_coercion
+    version: "1.0.0"
+    rules:
+      event_id: coerce_to_string
+      event_type: coerce_to_lowercase
+      occurred_at: coerce_to_datetime
+
+  validation_rules:
+    title: example_event_validation
+    version: "1.0.0"
+    rules:
+      event_id:
+        non_empty_string: null
+      event_type:
+        non_empty_string: null
+
+  metadata:
+    title: example_event_metadata
+    version: "1.0.0"
+    status: active
+    description: Example event contract metadata
+    governance_rules: {}

pycharter/data/seed/data_feeds.yaml

@@ -0,0 +1,22 @@
+# Generic seed: data_feeds (name required; title, feed_type, description, is_active, additional_metadata optional)
+
+- name: "default_feed"
+  title: "Default Feed"
+  feed_type: "batch"
+  description: "Default data feed"
+  is_active: true
+  additional_metadata: {}
+
+- name: "streaming_feed"
+  title: "Streaming Feed"
+  feed_type: "streaming"
+  description: "Streaming data feed"
+  is_active: true
+  additional_metadata: {}
+
+- name: "operational_feed"
+  title: "Operational Feed"
+  feed_type: "operational"
+  description: "Operational data feed"
+  is_active: true
+  additional_metadata: {}

pycharter/data/seed/domains.yaml

@@ -0,0 +1,13 @@
+# Generic seed: domains (name required; description optional)
+
+- name: "default"
+  description: "Default domain"
+
+- name: "operations"
+  description: "Operations and runbooks"
+
+- name: "analytics"
+  description: "Analytics and reporting"
+
+- name: "platform"
+  description: "Platform and infrastructure"

pycharter/data/seed/environments.yaml

@@ -0,0 +1,19 @@
+# Generic seed: environments (name required; description, environment_type, is_production, additional_metadata optional)
+
+- name: "development"
+  description: "Development environment"
+  environment_type: "non_production"
+  is_production: false
+  additional_metadata: {}
+
+- name: "staging"
+  description: "Staging environment"
+  environment_type: "non_production"
+  is_production: false
+  additional_metadata: {}
+
+- name: "production"
+  description: "Production environment"
+  environment_type: "production"
+  is_production: true
+  additional_metadata: {}

pycharter/data/seed/owners.yaml

@@ -0,0 +1,21 @@
+# Generic seed: owners (id is required; name, email, team, additional_info optional)
+
+- id: "default-owner"
+  name: "Default Owner"
+  email: "owner@example.com"
+  team: "data-engineering"
+  additional_info: {}
+
+- id: "platform-team"
+  name: "Platform Team"
+  email: "platform@example.com"
+  team: "platform-engineering"
+  additional_info:
+    description: "Platform and infrastructure"
+
+- id: "product-team"
+  name: "Product Team"
+  email: "product@example.com"
+  team: "product"
+  additional_info:
+    description: "Product and analytics"

pycharter/data/seed/systems.yaml

@@ -0,0 +1,13 @@
+# Generic seed: systems (name required; app_id, description optional)
+
+- name: "core-api"
+  app_id: "APP-CORE"
+  description: "Core API service"
+
+- name: "pipeline-engine"
+  app_id: "APP-PIPELINE"
+  description: "Data pipeline engine"
+
+- name: "catalog"
+  app_id: "APP-CATALOG"
+  description: "Data catalog and metadata"

pycharter/data/seed/tags.yaml

@@ -0,0 +1,25 @@
+# Generic seed: tags (name required; description, category, color, additional_metadata optional)
+
+- name: "default"
+  description: "Default tag"
+  category: "general"
+  color: "#808080"
+  additional_metadata: {}
+
+- name: "critical"
+  description: "Critical data"
+  category: "classification"
+  color: "#FF0000"
+  additional_metadata: {}
+
+- name: "internal"
+  description: "Internal use only"
+  category: "classification"
+  color: "#FFA500"
+  additional_metadata: {}
+
+- name: "pii"
+  description: "Personally identifiable information"
+  category: "classification"
+  color: "#9933CC"
+  additional_metadata: {}

pycharter/data/templates/contract/README.md

@@ -0,0 +1,161 @@
+# Data Contract Templates
+
+Templates for creating data contracts with schema, coercion rules, validation rules, and metadata.
+
+## Quick Start
+
+```python
+from pycharter import parse_contract, validate_with_contract
+
+# Option 1: Complete contract in one file
+contract = {
+    "schema": {
+        "type": "object",
+        "properties": {
+            "id": {"type": "string"},
+            "name": {"type": "string", "minLength": 1},
+        },
+        "required": ["id", "name"]
+    },
+    "metadata": {"version": "1.0.0"},
+    "ownership": {"owner": "data-team"}
+}
+
+# Validate data
+result = validate_with_contract(contract, {"id": "123", "name": "Alice"})
+print(f"Valid: {result.is_valid}")
+```
+
+## Template Files
+
+| Template | Description |
+|----------|-------------|
+| `template_contract.yaml` | Complete contract (schema + rules + metadata) |
+| `template_schema.yaml` | JSON Schema with PyCharter extensions |
+| `template_coercion_rules.yaml` | Data type transformation rules |
+| `template_validation_rules.yaml` | Business validation rules |
+| `template_metadata.yaml` | Ownership and governance metadata |
+
+## Contract Structure
+
+```yaml
+# Complete contract structure (parse_contract / create-mixed compatible)
+schema:               # JSON Schema (required). May include inline coercion/validations
+  type: object
+  title: my_entity
+  version: "1.0.0"
+  properties:
+    field_name:
+      type: string
+      coercion: coerce_to_string      # Optional: inline coercion
+      validations:                     # Optional: inline validation rules
+        max_length: {threshold: 100}
+
+# Optional: top-level rules (same format as template_coercion_rules / template_validation_rules)
+coercion_rules:
+  version: "1.0.0"
+  rules:
+    field_name: coerce_to_string
+validation_rules:
+  version: "1.0.0"
+  rules:
+    field_name: { max_length: { threshold: 100 } }
+
+# Metadata: version, description; ownership and governance_rules must be nested here
+metadata:
+  version: "1.0.0"
+  description: "Contract description"
+  ownership:          # Nested in metadata (not top-level)
+    owner: team-name
+    team: department
+  governance_rules:   # Nested in metadata
+    data_retention: {days: 365}
+    pii_fields: {fields: [email]}
+
+versions:             # Optional: explicit version tracking
+  schema: "1.0.0"
+  coercion_rules: "1.0.0"
+  validation_rules: "1.0.0"
+  metadata: "1.0.0"
+```
+
+## Coercion Rules
+
+Applied BEFORE Pydantic validation to transform input data:
+
+| Rule | Description |
+|------|-------------|
+| `coerce_to_string` | Convert to string |
+| `coerce_to_integer` | Convert to int |
+| `coerce_to_float` | Convert to float |
+| `coerce_to_boolean` | Convert to bool |
+| `coerce_to_datetime` | Parse datetime |
+| `coerce_to_date` | Parse date |
+| `coerce_to_uuid` | Parse UUID |
+| `coerce_to_lowercase` | Lowercase string |
+| `coerce_to_uppercase` | Uppercase string |
+| `coerce_to_stripped_string` | Trim whitespace |
+| `coerce_to_nullable_*` | Handle null values |
+
+## Validation Rules
+
+Applied AFTER Pydantic validation for business rules:
+
+| Rule | Parameters | Description |
+|------|------------|-------------|
+| `min_length` | `{threshold: N}` | Minimum string length |
+| `max_length` | `{threshold: N}` | Maximum string length |
+| `greater_than_or_equal_to` | `{threshold: N}` | Minimum value |
+| `less_than_or_equal_to` | `{threshold: N}` | Maximum value |
+| `only_allow` | `{allowed_values: [...]}` | Enum validation |
+| `non_empty_string` | `null` | Non-empty string |
+| `is_email` | `null` | Email format |
+| `is_url` | `null` | URL format |
+| `matches_regex` | `{pattern: "..."}` | Regex match |
+
+## Usage Patterns
+
+### 1. Validate with Contract
+
+```python
+from pycharter import validate_with_contract
+
+result = validate_with_contract(
+    "path/to/contract.yaml",  # or dict
+    {"id": "123", "name": "Alice"}
+)
+
+if result.is_valid:
+    print(f"Data: {result.data}")
+else:
+    print(f"Errors: {result.errors}")
+```
+
+### 2. Build Contract from Artifacts
+
+```python
+from pycharter import build_contract
+
+contract = build_contract(
+    schema=schema_dict,
+    metadata=metadata_dict,
+    coercion_rules=coercion_dict,
+    validation_rules=validation_dict,
+)
+```
+
+### 3. Quality Check
+
+```python
+from pycharter import QualityCheck
+
+report = QualityCheck().run(
+    contract=contract,
+    data=records
+)
+print(f"Score: {report.quality_score.overall_score}/100")
+```
+
+## Examples
+
+See `docs/notebooks/03_contracts.ipynb` and `docs/notebooks/04_validation_quality.ipynb` for working examples.

pycharter/data/templates/contract/template_contract.yaml

@@ -89,6 +89,43 @@ schema:
     - status
     - created_at
 
+# Optional: top-level coercion rules (same as template_coercion_rules.yaml structure)
+# When present, parse_contract returns them; use with create-mixed or store for separate artifacts.
+coercion_rules:
+  version: "1.0.0"
+  rules:
+    user_id: coerce_to_string
+    name: coerce_to_stripped_string
+    email: coerce_to_lowercase
+    age: coerce_to_integer
+    status: coerce_to_lowercase
+    created_at: coerce_to_datetime
+    notes: coerce_to_nullable_string
+
+# Optional: top-level validation rules (same as template_validation_rules.yaml structure)
+validation_rules:
+  version: "1.0.0"
+  rules:
+    user_id:
+      non_empty_string: null
+    name:
+      non_empty_string: null
+      max_length:
+        threshold: 100
+    email:
+      is_email: null
+    age:
+      greater_than_or_equal_to:
+        threshold: 0
+      less_than_or_equal_to:
+        threshold: 150
+    status:
+      only_allow:
+        allowed_values: [active, inactive, pending]
+    notes:
+      max_length:
+        threshold: 500
+
 # Contract metadata (ownership and governance nested inside)
 metadata:
   version: "1.0.0"

pycharter/data/templates/etl/README.md

@@ -221,4 +221,4 @@ result = await pipeline.run()
 
 ## Examples
 
-See `examples/etl_config_example/` for working examples with config files.
+See `docs/notebooks/02_etl_pipelines.ipynb` (config-driven section) for pipeline-from-YAML examples.

pycharter/data/templates/etl/extract_with_validation.yaml

@@ -0,0 +1,86 @@
+# Template: HTTP Extraction with Source Validation
+# Copy to your pipeline directory as extract.yaml
+#
+# This template demonstrates source schema validation after extraction.
+# Invalid records can be skipped, quarantined (sent to DLQ), or cause failures.
+#
+# Usage:
+#   pipeline = Pipeline.from_config_dir("path/to/pipeline")
+#   result = await pipeline.run()
+#   print(f"Valid records: {result.rows_transformed}")
+#   print(f"Quarantined at extract: {result.rows_quarantined_extract}")
+
+title: http_extraction_with_validation
+description: Extract from HTTP API with source schema validation
+version: "1.0.0"
+
+# Source type (required)
+type: http
+
+# API URL
+url: https://api.example.com/v1/orders
+
+# HTTP method
+method: GET
+
+# Request headers
+headers:
+  Accept: application/json
+  Authorization: Bearer ${API_TOKEN}
+
+# Response handling
+response_format: json
+response_path: data.orders  # Path to data array in response
+
+# Batch size
+batch_size: 1000
+
+# ============================================================================
+# Source Schema Validation (Optional)
+# ============================================================================
+# Validate extracted data against a JSON/YAML schema before transformation.
+# This catches malformed data early in the pipeline.
+
+validation:
+  # Path to source schema file (JSON Schema or Pydantic schema YAML)
+  # Can be absolute or relative to config directory
+  schema: ./source_schema.yaml
+  
+  # Action when validation fails:
+  # - fail: Stop pipeline immediately (default)
+  # - warn: Log warning but continue with all records
+  # - skip: Skip invalid records silently
+  # - quarantine: Send invalid records to DLQ
+  on_error: quarantine
+  
+  # DLQ configuration (optional, overrides settings.yaml)
+  # Can be true (use settings.yaml defaults), false (disable), or object
+  dlq: true  # Use default DLQ from settings.yaml
+  
+  # Or customize per-stage:
+  # dlq:
+  #   enabled: true
+  #   backend: file
+  #   path: ./dlq/extract
+
+# ============================================================================
+# Example Source Schema (create as source_schema.yaml)
+# ============================================================================
+# version: "1.0.0"
+# type: object
+# properties:
+#   order_id:
+#     type: string
+#     pattern: "^ORD-[0-9]{6}$"
+#   customer_id:
+#     type: integer
+#   status:
+#     type: string
+#     enum: ["pending", "processing", "shipped", "delivered"]
+#   total_amount:
+#     type: number
+#     minimum: 0
+# required:
+#   - order_id
+#   - customer_id
+#   - status