pycharter 0.0.25__py3-none-any.whl → 0.0.26__py3-none-any.whl

pycharter/data/templates/etl/load_with_validation.yaml

@@ -0,0 +1,111 @@
+# Template: PostgreSQL Load with Target Validation
+# Copy to your pipeline directory as load.yaml
+#
+# This template demonstrates target contract validation before loading.
+# Data is validated against a contract (schema + coercion + validation rules)
+# to ensure it meets the target system's requirements.
+#
+# Usage:
+#   pipeline = Pipeline.from_config_dir("path/to/pipeline")
+#   result = await pipeline.run()
+#   print(f"Records loaded: {result.rows_loaded}")
+#   print(f"Quarantined at load: {result.rows_quarantined_load}")
+
+title: postgres_load_with_validation
+description: Load to PostgreSQL with target contract validation
+version: "1.0.0"
+
+# Destination type
+type: postgres
+
+# Database connection
+connection_string: ${DATABASE_URL}
+# Or use database object:
+# database:
+#   url: ${DATABASE_URL}
+
+# Target table
+schema: public
+table: orders
+
+# Write method
+write_method: upsert  # insert, upsert, replace, truncate_and_load
+primary_key: order_id
+
+# Batch size
+batch_size: 500
+
+# ============================================================================
+# Target Contract Validation (Optional)
+# ============================================================================
+# Validate transformed data against a contract before loading.
+# This ensures data conforms to the target table schema and business rules.
+
+validation:
+  # Option 1: Path to contract directory or file
+  # Contract directory should contain: schema.yaml, coercion_rules.yaml (optional),
+  # validation_rules.yaml (optional)
+  contract: ./contracts/orders
+  
+  # Option 2: Reference contract from metadata store (requires settings.yaml metadata_store)
+  # contract:
+  #   type: database
+  #   name: orders_contract
+  #   version: "2.0"  # Optional, defaults to latest
+  
+  # Option 3: Use default contract from settings.yaml
+  # use_contract: true
+  
+  # Action when validation fails:
+  # - fail: Stop pipeline immediately (default)
+  # - warn: Log warning but continue with all records
+  # - skip: Skip invalid records (don't load them)
+  # - quarantine: Send invalid records to DLQ
+  on_error: quarantine
+  
+  # DLQ configuration (optional, overrides settings.yaml)
+  dlq: true  # Use default DLQ from settings.yaml
+
+# ============================================================================
+# Example Contract Structure (create in ./contracts/orders/)
+# ============================================================================
+#
+# contracts/orders/schema.yaml:
+# ---
+# version: "1.0.0"
+# type: object
+# properties:
+#   order_id:
+#     type: string
+#     maxLength: 20
+#   customer_id:
+#     type: integer
+#   status:
+#     type: string
+#     enum: ["pending", "processing", "shipped", "delivered", "cancelled"]
+#   total_amount:
+#     type: number
+#     minimum: 0
+#   created_at:
+#     type: string
+#     format: date-time
+# required:
+#   - order_id
+#   - customer_id
+#   - status
+#
+# contracts/orders/coercion_rules.yaml:
+# ---
+# order_id: string
+# customer_id: int
+# total_amount: float
+# created_at: datetime
+#
+# contracts/orders/validation_rules.yaml:
+# ---
+# order_id:
+#   - not_empty
+#   - max_length: 20
+# total_amount:
+#   - min_value: 0
+#   - max_value: 1000000

pycharter/data/templates/etl/settings.yaml

@@ -0,0 +1,55 @@
+# Template: Pipeline Settings
+# Copy to your pipeline directory as settings.yaml
+#
+# This file provides shared configuration for the pipeline:
+# - Dead Letter Queue (DLQ) settings for quarantining invalid records
+# - Metadata store connection for database-backed contracts
+# - Default contract name for load validation
+#
+# Usage:
+#   pipeline = Pipeline.from_config_dir("path/to/pipeline")
+#   result = await pipeline.run()
+
+title: pipeline_settings
+description: Shared pipeline configuration
+version: "1.0.0"
+
+# ============================================================================
+# Dead Letter Queue (DLQ) Configuration
+# ============================================================================
+# Records that fail validation can be sent to a DLQ for later review.
+# Configure this at the settings level to share across extract and load stages.
+
+dlq:
+  # Enable/disable DLQ
+  enabled: true
+  
+  # Storage backend: "file", "database", or "memory"
+  backend: file
+  
+  # File backend options
+  path: ./dlq  # Directory to store quarantined records
+  
+  # Database backend options (uncomment to use)
+  # backend: database
+  # connection_string: ${DATABASE_URL}
+  # schema: pycharter  # Database schema for DLQ table
+  # table: dead_letter_queue  # Optional custom table name
+
+# ============================================================================
+# Metadata Store Configuration (Optional)
+# ============================================================================
+# If you store contracts in a database (using MetadataStoreClient), configure
+# the connection here. This allows load validation to reference contracts by name.
+
+# metadata_store:
+#   type: postgres  # or sqlite, mongodb, redis, memory
+#   connection_string: ${DATABASE_URL}
+
+# ============================================================================
+# Default Contract (Optional)
+# ============================================================================
+# Specify a default contract name for load validation when using `use_contract: true`
+# in your load config. The contract must exist in the metadata_store.
+
+# contract: orders_v2

pycharter/db/cli.py

@@ -27,15 +27,20 @@ except ImportError:
 from pycharter.config import get_database_url, set_database_url
 from pycharter.db.models import (
     APIEndpointModel,
+    CoercionRuleModel,
     ComplianceFrameworkModel,
+    DataContractModel,
     DataFeedModel,
     DomainModel,
     EnvironmentModel,
+    MetadataRecordModel,
     OwnerModel,
+    SchemaModel,
     SystemModel,
     TagModel,
+    ValidationRuleModel,
 )
-from pycharter.db.models.base import Base, get_session
+from pycharter.db.models.base import Base, get_engine, get_session
 
 
 # Helper functions
@@ -268,7 +273,7 @@ def _init_sqlite(database_url: str, force: bool = False) -> int:
         if db_path != ":memory:":
             Path(db_path).parent.mkdir(parents=True, exist_ok=True)
         
-        engine = create_engine(database_url)
+        engine = get_engine(database_url)
         existing_tables = inspect(engine).get_table_names()
         
         if existing_tables and not force:
@@ -276,6 +281,15 @@ def _init_sqlite(database_url: str, force: bool = False) -> int:
             print("   Use --force to reinitialize, or run 'pycharter db upgrade' to apply migrations.")
             return 1
         
+        if existing_tables and force:
+            with engine.connect() as conn:
+                conn.execute(text("PRAGMA foreign_keys=OFF"))
+                for name in existing_tables:
+                    conn.execute(text(f'DROP TABLE IF EXISTS "{name}"'))
+                conn.execute(text("PRAGMA foreign_keys=ON"))
+                conn.commit()
+            print("✓ Dropped existing tables")
+        
         Base.metadata.create_all(engine)
         print("✓ Created tables using SQLAlchemy models")
         
@@ -502,8 +516,9 @@ def cmd_seed(seed_dir: Optional[str] = None, database_url: Optional[str] = None)
         if seed_dir:
             seed_path = Path(seed_dir)
         else:
-            project_root = Path(__file__).resolve().parent.parent.parent
-            seed_path = project_root / "data" / "seed"
+            # Default: package-bundled seed under src/pycharter/data/seed
+            pkg_root = Path(__file__).resolve().parent.parent
+            seed_path = pkg_root / "data" / "seed"
         
         if not seed_path.exists():
             print(f"❌ Error: Seed directory not found: {seed_path}")
@@ -522,6 +537,112 @@ def cmd_seed(seed_dir: Optional[str] = None, database_url: Optional[str] = None)
         return 1
 
 
+def _seed_contract_artifacts(seed_path: Path, session: Any) -> int:
+    """Seed contract artifacts from contracts.yaml (data contract + schema, coercion, validation, metadata)."""
+    contracts_file = seed_path / "contracts.yaml"
+    if not contracts_file.exists():
+        print("⚠ No contracts.yaml found, skipping contract artifacts")
+        return 0
+    with open(contracts_file, "r") as f:
+        contracts = yaml.safe_load(f) or []
+    if not contracts:
+        return 0
+    print("Loading contract artifacts...")
+    count = 0
+    for entry in contracts:
+        name = entry.get("name")
+        version = entry.get("version")
+        if not name or not version:
+            print("⚠ Skipping contract entry missing name or version")
+            continue
+        existing = session.query(DataContractModel).filter(
+            DataContractModel.name == name,
+            DataContractModel.version == version,
+        ).first()
+        if existing:
+            print(f"  Skipped (exists): {name}@{version}")
+            continue
+        # Create data contract with null FKs
+        contract = DataContractModel(
+            name=name,
+            version=version,
+            status=entry.get("status") or "active",
+            description=entry.get("description"),
+        )
+        session.add(contract)
+        session.flush()
+        data_contract_id = contract.id
+        # Schema
+        schema_def = entry.get("schema") or {}
+        schema_title = schema_def.get("title") or f"{name}_schema"
+        schema_version = schema_def.get("version") or version
+        schema_data = {k: v for k, v in schema_def.items() if k not in ("title", "version")}
+        schema_data["title"] = schema_def.get("title", schema_title)
+        schema_data["version"] = schema_version
+        schema_row = SchemaModel(
+            title=schema_title,
+            data_contract_id=data_contract_id,
+            version=schema_version,
+            schema_data=schema_data,
+        )
+        session.add(schema_row)
+        session.flush()
+        schema_id = schema_row.id
+        # Coercion rules
+        cr_def = entry.get("coercion_rules") or {}
+        cr_title = cr_def.get("title") or f"{name}_coercion"
+        cr_version = cr_def.get("version") or version
+        cr_row = CoercionRuleModel(
+            title=cr_title,
+            data_contract_id=data_contract_id,
+            version=cr_version,
+            rules=cr_def.get("rules") or {},
+            description=cr_def.get("description"),
+            schema_id=schema_id,
+        )
+        session.add(cr_row)
+        session.flush()
+        # Validation rules
+        vr_def = entry.get("validation_rules") or {}
+        vr_title = vr_def.get("title") or f"{name}_validation"
+        vr_version = vr_def.get("version") or version
+        vr_row = ValidationRuleModel(
+            title=vr_title,
+            data_contract_id=data_contract_id,
+            version=vr_version,
+            rules=vr_def.get("rules") or {},
+            description=vr_def.get("description"),
+            schema_id=schema_id,
+        )
+        session.add(vr_row)
+        session.flush()
+        # Metadata record
+        meta_def = entry.get("metadata") or {}
+        meta_title = meta_def.get("title") or f"{name}_metadata"
+        meta_version = meta_def.get("version") or version
+        meta_row = MetadataRecordModel(
+            title=meta_title,
+            data_contract_id=data_contract_id,
+            version=meta_version,
+            status=meta_def.get("status"),
+            description=meta_def.get("description"),
+            governance_rules=meta_def.get("governance_rules"),
+        )
+        session.add(meta_row)
+        session.flush()
+        # Link contract to artifacts
+        contract.schema_id = schema_id
+        contract.coercion_rules_id = cr_row.id
+        contract.validation_rules_id = vr_row.id
+        contract.metadata_record_id = meta_row.id
+        session.commit()
+        print(f"  Created contract: {name}@{version}")
+        count += 1
+    if count:
+        print(f"✓ Loaded {count} contract(s)")
+    return count
+
+
 def _seed_postgresql(seed_path: Path, db_url: str) -> int:
     """Seed PostgreSQL database with data from YAML files."""
     _require_alembic()
@@ -535,6 +656,7 @@ def _seed_postgresql(seed_path: Path, db_url: str) -> int:
         _seed_model(session, DataFeedModel, seed_path / "data_feeds.yaml", "data_feeds", "name")
         _seed_model(session, ComplianceFrameworkModel, seed_path / "compliance_frameworks.yaml", "compliance_frameworks", "name")
         _seed_model(session, TagModel, seed_path / "tags.yaml", "tags", "name")
+        _seed_contract_artifacts(seed_path, session)
         print("\n✓ Seed data loaded successfully!")
         return 0
     except Exception as e:

pycharter/db/migrations/versions/20260122000000_change_artifact_unique_constraints_to_title_version.py

@@ -1,6 +1,6 @@
 """Change artifact unique constraints from (data_contract_id, version) to (title, version)
 
-Revision ID: change_artifact_unique_constraints
+Revision ID: change_artifact_uniq_constraints
 Revises: remove_artifact_versions
 Create Date: 2026-01-22 00:00:00.000000
 
@@ -20,7 +20,7 @@ from alembic import op
 import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
-revision: str = 'change_artifact_unique_constraints'
+revision: str = 'change_artifact_uniq_constraints'
 down_revision: Union[str, None] = 'remove_artifact_versions'
 branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None

pycharter/etl_generator/INTERFACES.md

@@ -49,16 +49,15 @@ Discovers pipelines from a root directory: each subdir that contains `extract.ya
 
 **Import:** `from pycharter.etl_generator.extractors import extract_with_pagination_streaming`
 
-Async generator that yields batches of records. Dispatches to the right extractor via `ExtractorFactory` using `source_type` (or auto-detection from `extract_config`).
+Async generator that yields batches of records. Dispatches to the right extractor via `ExtractorFactory` using `type` (or auto-detection from `extract_config`).
 
 - **Args:** `extract_config`, `params`, `headers`, `contract_dir=None`, `batch_size=1000`, `max_records=None`, `config_context=None`
 
-### `ExtractorFactory` / `get_extractor(extract_config) -> BaseExtractor`
+### `ExtractorFactory.create(extract_config) -> BaseExtractor`
 
-**Import:** `from pycharter.etl_generator.extractors import ExtractorFactory, get_extractor`
+**Import:** `from pycharter.etl_generator.extractors import ExtractorFactory`
 
-- `ExtractorFactory.get_extractor(extract_config)` — Returns an extractor instance for the given config.
-- `get_extractor(extract_config)` — Same, module-level helper.
+- `ExtractorFactory.create(extract_config)` — Returns an extractor instance for the given config.
 - **Auto-detection:** `base_url`/`api_endpoint` → http; `file_path` → file; `database` → database; `storage` → cloud_storage.
 
 ### Extractors (implementations of `BaseExtractor`)
@@ -71,7 +70,7 @@ Async generator that yields batches of records. Dispatches to the right extracto
 - **DatabaseExtractor** — SQL over PostgreSQL, MySQL, SQLite, MSSQL, Oracle.
 - **CloudStorageExtractor** — S3, GCS, Azure Blob.
 
-Custom extractors: `ExtractorFactory.register_extractor(source_type, extractor_class)`.
+Custom extractors: `ExtractorFactory.register(type_name, extractor_class)`.
 
 ---
 
@@ -136,7 +135,7 @@ Helpers to produce ETL config dicts (extract/transform/load) from contracts or f
 |--------------------------|--------|
 | Run ETL                  | `from pycharter.etl_generator import ETLOrchestrator, create_orchestrator` |
 | Discover pipelines       | `from pycharter.etl_generator import PipelineFactory` |
-| Extract (streaming)      | `from pycharter.etl_generator.extractors import extract_with_pagination_streaming, ExtractorFactory, get_extractor` |
+| Extract (streaming)      | `from pycharter.etl_generator.extractors import extract_with_pagination_streaming, ExtractorFactory` |
 | Transform                | `from pycharter.etl_generator.transformers import apply_transforms` |
 | Load to file/cloud       | `from pycharter.etl_generator.loaders import load_to_file, load_to_cloud_storage` |
 | Config generation        | `from pycharter.etl_generator import generate_etl_config, generate_etl_config_from_contract, generate_etl_config_from_store` |

pycharter/etl_generator/__init__.py

@@ -70,19 +70,39 @@ from pycharter.etl_generator.context import PipelineContext
 from pycharter.etl_generator.result import PipelineResult, BatchResult, LoadResult
 from pycharter.etl_generator.protocols import Extractor, Transformer, Loader
 
-# Config loading and validation
-from pycharter.etl_generator.config_loader import (
-    ConfigLoader,
-    PipelineConfig,
-    load_pipeline_config,
-    ConfigLoadError,
-)
+# Config validation and errors
+from pycharter.shared.errors import ConfigLoadError
 from pycharter.etl_generator.config_validator import (
     ConfigValidator,
     ConfigValidationError,
     validate_config,
 )
 
+# Config models (Pydantic)
+from pycharter.etl_generator.config_models import (
+    ExtractConfig,
+    LoadConfig,
+    TransformConfig,
+    SettingsConfig,
+    ExtractValidationConfig,
+    LoadValidationConfig,
+    DLQConfig,
+    parse_extract_config,
+    parse_load_config,
+    parse_transform_config,
+    parse_settings_config,
+)
+
+# ETL Validation
+from pycharter.etl_generator.validation import (
+    ETLValidator,
+    ETLValidationError,
+    resolve_schema,
+    resolve_contract,
+    create_dlq,
+    create_etl_validator,
+)
+
 # Expression evaluation
 from pycharter.etl_generator.expression import (
     ExpressionEvaluator,
@@ -139,14 +159,30 @@ __all__ = [
     "Extractor",
     "Transformer",
     "Loader",
-    # Config loading
-    "ConfigLoader",
-    "PipelineConfig",
-    "load_pipeline_config",
+    # Config validation
     "ConfigLoadError",
     "ConfigValidator",
     "ConfigValidationError",
     "validate_config",
+    # Config models
+    "ExtractConfig",
+    "LoadConfig",
+    "TransformConfig",
+    "SettingsConfig",
+    "ExtractValidationConfig",
+    "LoadValidationConfig",
+    "DLQConfig",
+    "parse_extract_config",
+    "parse_load_config",
+    "parse_transform_config",
+    "parse_settings_config",
+    # ETL Validation
+    "ETLValidator",
+    "ETLValidationError",
+    "resolve_schema",
+    "resolve_contract",
+    "create_dlq",
+    "create_etl_validator",
     # Expressions
     "ExpressionEvaluator",
     "ExpressionError",