pycharter 0.0.24__py3-none-any.whl → 0.0.26__py3-none-any.whl

pycharter/data/templates/etl/load_with_validation.yaml

@@ -0,0 +1,111 @@
+# Template: PostgreSQL Load with Target Validation
+# Copy to your pipeline directory as load.yaml
+#
+# This template demonstrates target contract validation before loading.
+# Data is validated against a contract (schema + coercion + validation rules)
+# to ensure it meets the target system's requirements.
+#
+# Usage:
+#   pipeline = Pipeline.from_config_dir("path/to/pipeline")
+#   result = await pipeline.run()
+#   print(f"Records loaded: {result.rows_loaded}")
+#   print(f"Quarantined at load: {result.rows_quarantined_load}")
+
+title: postgres_load_with_validation
+description: Load to PostgreSQL with target contract validation
+version: "1.0.0"
+
+# Destination type
+type: postgres
+
+# Database connection
+connection_string: ${DATABASE_URL}
+# Or use database object:
+# database:
+#   url: ${DATABASE_URL}
+
+# Target table
+schema: public
+table: orders
+
+# Write method
+write_method: upsert  # insert, upsert, replace, truncate_and_load
+primary_key: order_id
+
+# Batch size
+batch_size: 500
+
+# ============================================================================
+# Target Contract Validation (Optional)
+# ============================================================================
+# Validate transformed data against a contract before loading.
+# This ensures data conforms to the target table schema and business rules.
+
+validation:
+  # Option 1: Path to contract directory or file
+  # Contract directory should contain: schema.yaml, coercion_rules.yaml (optional),
+  # validation_rules.yaml (optional)
+  contract: ./contracts/orders
+  
+  # Option 2: Reference contract from metadata store (requires settings.yaml metadata_store)
+  # contract:
+  #   type: database
+  #   name: orders_contract
+  #   version: "2.0"  # Optional, defaults to latest
+  
+  # Option 3: Use default contract from settings.yaml
+  # use_contract: true
+  
+  # Action when validation fails:
+  # - fail: Stop pipeline immediately (default)
+  # - warn: Log warning but continue with all records
+  # - skip: Skip invalid records (don't load them)
+  # - quarantine: Send invalid records to DLQ
+  on_error: quarantine
+  
+  # DLQ configuration (optional, overrides settings.yaml)
+  dlq: true  # Use default DLQ from settings.yaml
+
+# ============================================================================
+# Example Contract Structure (create in ./contracts/orders/)
+# ============================================================================
+#
+# contracts/orders/schema.yaml:
+# ---
+# version: "1.0.0"
+# type: object
+# properties:
+#   order_id:
+#     type: string
+#     maxLength: 20
+#   customer_id:
+#     type: integer
+#   status:
+#     type: string
+#     enum: ["pending", "processing", "shipped", "delivered", "cancelled"]
+#   total_amount:
+#     type: number
+#     minimum: 0
+#   created_at:
+#     type: string
+#     format: date-time
+# required:
+#   - order_id
+#   - customer_id
+#   - status
+#
+# contracts/orders/coercion_rules.yaml:
+# ---
+# order_id: string
+# customer_id: int
+# total_amount: float
+# created_at: datetime
+#
+# contracts/orders/validation_rules.yaml:
+# ---
+# order_id:
+#   - not_empty
+#   - max_length: 20
+# total_amount:
+#   - min_value: 0
+#   - max_value: 1000000

pycharter/data/templates/etl/settings.yaml

@@ -0,0 +1,55 @@
+# Template: Pipeline Settings
+# Copy to your pipeline directory as settings.yaml
+#
+# This file provides shared configuration for the pipeline:
+# - Dead Letter Queue (DLQ) settings for quarantining invalid records
+# - Metadata store connection for database-backed contracts
+# - Default contract name for load validation
+#
+# Usage:
+#   pipeline = Pipeline.from_config_dir("path/to/pipeline")
+#   result = await pipeline.run()
+
+title: pipeline_settings
+description: Shared pipeline configuration
+version: "1.0.0"
+
+# ============================================================================
+# Dead Letter Queue (DLQ) Configuration
+# ============================================================================
+# Records that fail validation can be sent to a DLQ for later review.
+# Configure this at the settings level to share across extract and load stages.
+
+dlq:
+  # Enable/disable DLQ
+  enabled: true
+  
+  # Storage backend: "file", "database", or "memory"
+  backend: file
+  
+  # File backend options
+  path: ./dlq  # Directory to store quarantined records
+  
+  # Database backend options (uncomment to use)
+  # backend: database
+  # connection_string: ${DATABASE_URL}
+  # schema: pycharter  # Database schema for DLQ table
+  # table: dead_letter_queue  # Optional custom table name
+
+# ============================================================================
+# Metadata Store Configuration (Optional)
+# ============================================================================
+# If you store contracts in a database (using MetadataStoreClient), configure
+# the connection here. This allows load validation to reference contracts by name.
+
+# metadata_store:
+#   type: postgres  # or sqlite, mongodb, redis, memory
+#   connection_string: ${DATABASE_URL}
+
+# ============================================================================
+# Default Contract (Optional)
+# ============================================================================
+# Specify a default contract name for load validation when using `use_contract: true`
+# in your load config. The contract must exist in the metadata_store.
+
+# contract: orders_v2

pycharter/db/README.md

@@ -0,0 +1,179 @@
+# Database Management with Alembic
+
+PyCharter uses **Alembic** for database schema migrations, following the same pattern as Apache Airflow.
+
+## Overview
+
+- **SQLAlchemy Models**: Database schema defined as Python models in `pycharter/db/models/`
+- **Alembic Migrations**: Version-controlled schema changes in `pycharter/db/migrations/`
+- **CLI Commands**: `pycharter db init`, `pycharter db upgrade`, etc.
+
+## Quick Start
+
+### Initialize Database
+
+```bash
+# Initialize a fresh database
+pycharter db init postgresql://user:pass@localhost/pycharter
+```
+
+This will:
+1. Create all tables defined in SQLAlchemy models
+2. Create the initial Alembic migration
+3. Stamp the database with the current revision
+
+### Upgrade Database
+
+```bash
+# Upgrade to latest version
+pycharter db upgrade postgresql://user:pass@localhost/pycharter
+
+# Or use environment variable
+export PYCHARTER_DATABASE_URL=postgresql://user:pass@localhost/pycharter
+pycharter db upgrade
+```
+
+### Check Current Version
+
+```bash
+pycharter db current postgresql://user:pass@localhost/pycharter
+```
+
+### View Migration History
+
+```bash
+pycharter db history
+```
+
+## Commands
+
+| Command | Description | Example |
+|---------|-------------|---------|
+| `pycharter db init` | Initialize database from scratch | `pycharter db init postgresql://...` |
+| `pycharter db upgrade` | Upgrade to latest revision | `pycharter db upgrade postgresql://...` |
+| `pycharter db downgrade` | Downgrade to previous revision | `pycharter db downgrade postgresql://... --revision -1` |
+| `pycharter db current` | Show current revision | `pycharter db current postgresql://...` |
+| `pycharter db history` | Show migration history | `pycharter db history` |
+
+## Creating New Migrations
+
+When you modify SQLAlchemy models, create a new migration:
+
+```bash
+export PYCHARTER_DATABASE_URL=postgresql://user:pass@localhost/pycharter
+alembic revision --autogenerate -m "Description of changes"
+```
+
+Then review and edit the generated migration file in `pycharter/db/migrations/versions/`.
+
+## Database Models
+
+All models are defined in `pycharter/db/models/`:
+
+- `SchemaModel` - JSON Schema definitions
+- `CoercionRuleModel` - Coercion rules
+- `ValidationRuleModel` - Validation rules
+- `MetadataRecordModel` - Comprehensive metadata storage (includes governance rules and ownership fields)
+- `OwnerModel` - Direct ownership information (owner/team)
+- `DataContractModel` - Central table linking all components
+- `SystemModel` - System information
+- `DomainModel` - Domain information
+
+## Comparison with Airflow
+
+| Feature | Airflow | PyCharter |
+|---------|---------|-----------|
+| **CLI Command** | `airflow db init` | `pycharter db init` |
+| **Upgrade** | `airflow db upgrade` | `pycharter db upgrade` |
+| **Migration Tool** | Alembic | Alembic |
+| **Models** | SQLAlchemy | SQLAlchemy |
+| **Auto-generate** | Yes | Yes |
+
+## Best Practices
+
+1. **Always create migrations** when changing models
+2. **Test migrations** on a development database first
+3. **Review autogenerated migrations** before applying
+4. **Use version control** for all migration files
+5. **Backup database** before running migrations in production
+
+## Configuration
+
+PyCharter follows Airflow's configuration pattern. Database connection can be configured via:
+
+### 1. Environment Variables (Recommended)
+
+**Airflow-style:**
+```bash
+export PYCHARTER__DATABASE__SQL_ALCHEMY_CONN=postgresql://user:password@localhost:5432/pycharter
+```
+
+**Simpler alternative:**
+```bash
+export PYCHARTER_DATABASE_URL=postgresql://user:password@localhost:5432/pycharter
+```
+
+### 2. Config File
+
+Create `pycharter.cfg` in your project root or `~/.pycharter/`:
+
+```ini
+[database]
+sql_alchemy_conn = postgresql://user:password@localhost:5432/pycharter
+```
+
+### 3. Alembic Config
+
+Set in `alembic.ini`:
+
+```ini
+[alembic]
+sqlalchemy.url = postgresql://user:password@localhost:5432/pycharter
+```
+
+### Priority Order
+
+1. Command-line argument (if provided)
+2. `PYCHARTER__DATABASE__SQL_ALCHEMY_CONN` environment variable
+3. `PYCHARTER_DATABASE_URL` environment variable
+4. `pycharter.cfg` config file
+5. `alembic.ini` config file
+
+### Usage
+
+Once configured, you can run commands without passing the database URL:
+
+```bash
+# Database URL from configuration
+pycharter db init
+pycharter db upgrade
+pycharter db current
+
+# Or override with command-line argument
+pycharter db init postgresql://other:db@localhost:5432/other_db
+```
+
+## Troubleshooting
+
+### Tables Already Exist
+
+If tables already exist (from old migration system), stamp the database:
+
+```bash
+alembic stamp head
+```
+
+### Migration Conflicts
+
+If you have conflicts, you can:
+1. Review the migration file
+2. Manually edit if needed
+3. Test on a development database first
+
+### Connection Issues
+
+Make sure:
+- Database is running
+- Connection string is correct
+- User has proper permissions
+

pycharter/db/cli.py

@@ -27,15 +27,20 @@ except ImportError:
 from pycharter.config import get_database_url, set_database_url
 from pycharter.db.models import (
     APIEndpointModel,
+    CoercionRuleModel,
     ComplianceFrameworkModel,
+    DataContractModel,
     DataFeedModel,
     DomainModel,
     EnvironmentModel,
+    MetadataRecordModel,
     OwnerModel,
+    SchemaModel,
     SystemModel,
     TagModel,
+    ValidationRuleModel,
 )
-from pycharter.db.models.base import Base, get_session
+from pycharter.db.models.base import Base, get_engine, get_session
 
 
 # Helper functions
@@ -268,7 +273,7 @@ def _init_sqlite(database_url: str, force: bool = False) -> int:
         if db_path != ":memory:":
             Path(db_path).parent.mkdir(parents=True, exist_ok=True)
         
-        engine = create_engine(database_url)
+        engine = get_engine(database_url)
         existing_tables = inspect(engine).get_table_names()
         
         if existing_tables and not force:
@@ -276,6 +281,15 @@ def _init_sqlite(database_url: str, force: bool = False) -> int:
             print("   Use --force to reinitialize, or run 'pycharter db upgrade' to apply migrations.")
             return 1
         
+        if existing_tables and force:
+            with engine.connect() as conn:
+                conn.execute(text("PRAGMA foreign_keys=OFF"))
+                for name in existing_tables:
+                    conn.execute(text(f'DROP TABLE IF EXISTS "{name}"'))
+                conn.execute(text("PRAGMA foreign_keys=ON"))
+                conn.commit()
+            print("✓ Dropped existing tables")
+        
         Base.metadata.create_all(engine)
         print("✓ Created tables using SQLAlchemy models")
         
@@ -502,8 +516,9 @@ def cmd_seed(seed_dir: Optional[str] = None, database_url: Optional[str] = None)
         if seed_dir:
             seed_path = Path(seed_dir)
         else:
-            project_root = Path(__file__).resolve().parent.parent.parent
-            seed_path = project_root / "data" / "seed"
+            # Default: package-bundled seed under src/pycharter/data/seed
+            pkg_root = Path(__file__).resolve().parent.parent
+            seed_path = pkg_root / "data" / "seed"
         
         if not seed_path.exists():
             print(f"❌ Error: Seed directory not found: {seed_path}")
@@ -522,6 +537,112 @@ def cmd_seed(seed_dir: Optional[str] = None, database_url: Optional[str] = None)
         return 1
 
 
+def _seed_contract_artifacts(seed_path: Path, session: Any) -> int:
+    """Seed contract artifacts from contracts.yaml (data contract + schema, coercion, validation, metadata)."""
+    contracts_file = seed_path / "contracts.yaml"
+    if not contracts_file.exists():
+        print("⚠ No contracts.yaml found, skipping contract artifacts")
+        return 0
+    with open(contracts_file, "r") as f:
+        contracts = yaml.safe_load(f) or []
+    if not contracts:
+        return 0
+    print("Loading contract artifacts...")
+    count = 0
+    for entry in contracts:
+        name = entry.get("name")
+        version = entry.get("version")
+        if not name or not version:
+            print("⚠ Skipping contract entry missing name or version")
+            continue
+        existing = session.query(DataContractModel).filter(
+            DataContractModel.name == name,
+            DataContractModel.version == version,
+        ).first()
+        if existing:
+            print(f"  Skipped (exists): {name}@{version}")
+            continue
+        # Create data contract with null FKs
+        contract = DataContractModel(
+            name=name,
+            version=version,
+            status=entry.get("status") or "active",
+            description=entry.get("description"),
+        )
+        session.add(contract)
+        session.flush()
+        data_contract_id = contract.id
+        # Schema
+        schema_def = entry.get("schema") or {}
+        schema_title = schema_def.get("title") or f"{name}_schema"
+        schema_version = schema_def.get("version") or version
+        schema_data = {k: v for k, v in schema_def.items() if k not in ("title", "version")}
+        schema_data["title"] = schema_def.get("title", schema_title)
+        schema_data["version"] = schema_version
+        schema_row = SchemaModel(
+            title=schema_title,
+            data_contract_id=data_contract_id,
+            version=schema_version,
+            schema_data=schema_data,
+        )
+        session.add(schema_row)
+        session.flush()
+        schema_id = schema_row.id
+        # Coercion rules
+        cr_def = entry.get("coercion_rules") or {}
+        cr_title = cr_def.get("title") or f"{name}_coercion"
+        cr_version = cr_def.get("version") or version
+        cr_row = CoercionRuleModel(
+            title=cr_title,
+            data_contract_id=data_contract_id,
+            version=cr_version,
+            rules=cr_def.get("rules") or {},
+            description=cr_def.get("description"),
+            schema_id=schema_id,
+        )
+        session.add(cr_row)
+        session.flush()
+        # Validation rules
+        vr_def = entry.get("validation_rules") or {}
+        vr_title = vr_def.get("title") or f"{name}_validation"
+        vr_version = vr_def.get("version") or version
+        vr_row = ValidationRuleModel(
+            title=vr_title,
+            data_contract_id=data_contract_id,
+            version=vr_version,
+            rules=vr_def.get("rules") or {},
+            description=vr_def.get("description"),
+            schema_id=schema_id,
+        )
+        session.add(vr_row)
+        session.flush()
+        # Metadata record
+        meta_def = entry.get("metadata") or {}
+        meta_title = meta_def.get("title") or f"{name}_metadata"
+        meta_version = meta_def.get("version") or version
+        meta_row = MetadataRecordModel(
+            title=meta_title,
+            data_contract_id=data_contract_id,
+            version=meta_version,
+            status=meta_def.get("status"),
+            description=meta_def.get("description"),
+            governance_rules=meta_def.get("governance_rules"),
+        )
+        session.add(meta_row)
+        session.flush()
+        # Link contract to artifacts
+        contract.schema_id = schema_id
+        contract.coercion_rules_id = cr_row.id
+        contract.validation_rules_id = vr_row.id
+        contract.metadata_record_id = meta_row.id
+        session.commit()
+        print(f"  Created contract: {name}@{version}")
+        count += 1
+    if count:
+        print(f"✓ Loaded {count} contract(s)")
+    return count
+
+
 def _seed_postgresql(seed_path: Path, db_url: str) -> int:
     """Seed PostgreSQL database with data from YAML files."""
     _require_alembic()
@@ -535,6 +656,7 @@ def _seed_postgresql(seed_path: Path, db_url: str) -> int:
         _seed_model(session, DataFeedModel, seed_path / "data_feeds.yaml", "data_feeds", "name")
         _seed_model(session, ComplianceFrameworkModel, seed_path / "compliance_frameworks.yaml", "compliance_frameworks", "name")
         _seed_model(session, TagModel, seed_path / "tags.yaml", "tags", "name")
+        _seed_contract_artifacts(seed_path, session)
         print("\n✓ Seed data loaded successfully!")
         return 0
     except Exception as e:

pycharter/db/migrations/versions/20260122000000_change_artifact_unique_constraints_to_title_version.py

@@ -1,6 +1,6 @@
 """Change artifact unique constraints from (data_contract_id, version) to (title, version)
 
-Revision ID: change_artifact_unique_constraints
+Revision ID: change_artifact_uniq_constraints
 Revises: remove_artifact_versions
 Create Date: 2026-01-22 00:00:00.000000
 
@@ -20,7 +20,7 @@ from alembic import op
 import sqlalchemy as sa
 
 # revision identifiers, used by Alembic.
-revision: str = 'change_artifact_unique_constraints'
+revision: str = 'change_artifact_uniq_constraints'
 down_revision: Union[str, None] = 'remove_artifact_versions'
 branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None

pycharter/db/schemas/README.md

@@ -0,0 +1,96 @@
+# Data Contract Validation Schemas
+
+This directory contains Pydantic models for validating data contract structure.
+
+## Purpose
+
+The Pydantic models in this directory ensure that all data contracts parsed by `parse_contract()` and `parse_contract_file()` strictly adhere to the database table design.
+
+## Files
+
+- `data_contract.py` - Main Pydantic models for data contract validation
+  - `DataContractSchema` - Main model for validating complete contracts
+  - `SchemaComponent` - Validates the schema component (required)
+  - `MetadataComponent` - Validates metadata component (optional)
+  - `OwnershipComponent` - Validates ownership component (optional)
+  - `GovernanceRulesComponent` - Validates governance rules (optional)
+  - `CoercionRulesComponent` - Validates coercion rules (optional)
+  - `ValidationRulesComponent` - Validates validation rules (optional)
+  - `VersionsComponent` - Validates version tracking (optional)
+
+## Database Alignment
+
+All models are designed to match the database table structure exactly:
+
+- **Field constraints** (max_length) match database column sizes
+- **Required fields** match database NOT NULL constraints
+- **Optional fields** match database nullable columns
+- **Enum values** match database CHECK constraints
+- **Field types** match database column types
+
+## Usage
+
+The models are automatically used by the contract parser:
+
+```python
+from pycharter import parse_contract, parse_contract_file
+
+# Validation is enabled by default
+contract_metadata = parse_contract_file("contract.yaml")  # Validates automatically
+
+# You can disable validation if needed (not recommended)
+contract_metadata = parse_contract_file("contract.yaml", validate=False)
+```
+
+## Validation Behavior
+
+- **Valid contracts**: Pass validation and parse successfully
+- **Invalid contracts**: Raise `ValueError` with detailed Pydantic error messages
+- **Missing required fields**: Validation fails with clear error messages
+- **Extra fields**: Rejected (except in flexible components like metadata)
+
+## Model Design Principles
+
+1. **Strict Top-Level**: `DataContractSchema` uses `extra = "forbid"` to prevent unexpected fields
+2. **Flexible Components**: Metadata, governance_rules allow `extra = "allow"` for flexible JSON storage
+3. **Database Constraints**: All field constraints match database column definitions
+4. **Type Safety**: Full Python type hints for IDE support and static analysis
+
+## Updating Models
+
+When the database schema changes:
+
+1. Update the corresponding Pydantic model in `data_contract.py`
+2. Update field constraints to match new database columns
+3. Add/remove fields as needed
+4. Test with existing contracts to ensure backward compatibility
+5. Update this README if model structure changes significantly
+
+## Testing
+
+To test the validation models:
+
+```python
+from pycharter.db.schemas.data_contract import DataContractSchema
+
+# Valid contract
+valid_contract = {
+    "schema": {"type": "object", "properties": {"name": {"type": "string"}}}
+}
+result = DataContractSchema.model_validate(valid_contract)  # ✓ Passes
+
+# Invalid contract (missing required schema)
+invalid_contract = {
+    "metadata": {"version": "1.0.0"}
+}
+result = DataContractSchema.model_validate(invalid_contract)  # ✗ Raises ValidationError
+```
+
+
+
+
+
+
+
+
+