pycharter 0.0.22__py3-none-any.whl → 0.0.24__py3-none-any.whl

pycharter/data/templates/contract/template_schema.yaml

@@ -0,0 +1,100 @@
+# JSON Schema Template
+# Defines the structure and constraints of your data
+#
+# Usage:
+#   from pycharter import from_dict
+#   Model = from_dict(schema, "MyModel")
+
+type: object
+title: my_entity
+version: "1.0.0"
+
+properties:
+  # Required string field
+  id:
+    type: string
+    description: Unique identifier
+    title: ID
+    minLength: 1
+
+  # Required string with max length
+  name:
+    type: string
+    description: Entity name
+    title: Name
+    minLength: 1
+    maxLength: 100
+
+  # Enum field
+  status:
+    type: string
+    description: Current status
+    enum: [active, inactive, pending]
+
+  # Integer with constraints
+  count:
+    type: integer
+    description: Item count
+    minimum: 0
+
+  # Number (float) with constraints
+  amount:
+    type: number
+    description: Monetary amount
+    minimum: 0
+
+  # Boolean field
+  is_active:
+    type: boolean
+    description: Active flag
+    default: true
+
+  # Date-time field
+  created_at:
+    type: string
+    format: date-time
+    description: Creation timestamp
+
+  # Email field
+  email:
+    type: string
+    format: email
+    description: Contact email
+
+  # UUID field
+  uuid:
+    type: string
+    format: uuid
+    description: Unique UUID
+
+  # Optional/nullable field (use anyOf for nullable)
+  notes:
+    anyOf:
+      - type: string
+        maxLength: 500
+      - type: "null"
+    default: null
+    description: Optional notes
+
+  # Nested object
+  metadata:
+    type: object
+    properties:
+      source:
+        type: string
+      version:
+        type: string
+    required: [source]
+
+  # Array of strings
+  tags:
+    type: array
+    items:
+      type: string
+    description: Tags list
+
+required:
+  - id
+  - name
+  - status
+  - created_at

pycharter/data/templates/contract/template_validation_rules.yaml

@@ -0,0 +1,75 @@
+# Validation Rules Template
+# Post-validation business rule checks
+#
+# Usage:
+#   Validation rules check data AFTER Pydantic validation.
+#   Useful for business rules that go beyond type checking.
+#
+# Available validations:
+#   - min_length:              {threshold: N}
+#   - max_length:              {threshold: N}
+#   - greater_than_or_equal_to:{threshold: N}
+#   - less_than_or_equal_to:   {threshold: N}
+#   - only_allow:              {allowed_values: [a, b, c]}
+#   - only_allow_if:           {condition_field: x, condition_value: y, allowed_values: [...]}
+#   - non_empty_string:        null  (no params needed)
+#   - is_positive:             null
+#   - is_email:                null
+#   - is_url:                  null
+#   - is_alphanumeric:         null
+#   - is_numeric_string:       null
+#   - no_capital_characters:   null
+#   - no_special_characters:   null
+#   - matches_regex:           {pattern: "^[A-Z]{2}[0-9]{4}$"}
+
+title: my_validation_rules
+description: Business validation rules
+version: "1.0.0"
+
+rules:
+  # String length validation
+  id:
+    min_length:
+      threshold: 1
+    max_length:
+      threshold: 50
+
+  name:
+    non_empty_string: null
+    max_length:
+      threshold: 100
+
+  # Enum-like validation
+  status:
+    only_allow:
+      allowed_values:
+        - active
+        - inactive
+        - pending
+
+  # Numeric range validation
+  count:
+    greater_than_or_equal_to:
+      threshold: 0
+
+  amount:
+    greater_than_or_equal_to:
+      threshold: 0
+    less_than_or_equal_to:
+      threshold: 1000000
+
+  # Email validation
+  email:
+    is_email: null
+
+  # Regex pattern validation
+  # code:
+  #   matches_regex:
+  #     pattern: "^[A-Z]{2}[0-9]{4}$"
+
+  # Conditional validation
+  # discount_rate:
+  #   only_allow_if:
+  #     condition_field: customer_type
+  #     condition_value: premium
+  #     allowed_values: [0.1, 0.15, 0.2]

pycharter/data/templates/etl/README.md

@@ -0,0 +1,224 @@
+# ETL Config Templates
+
+Templates for ETL pipeline configuration. Supports two formats:
+
+1. **Single-file format**: `pipeline.yaml` with extract, transform, load sections
+2. **Multi-file format**: Separate `extract.yaml`, `transform.yaml`, `load.yaml` files
+
+## Quick Start
+
+### Single-File Format (Recommended)
+
+```yaml
+# pipelines/users/pipeline.yaml
+name: users_pipeline
+version: "1.0.0"
+
+extract:
+  type: http  # Required: http | file | database | cloud_storage
+  url: https://api.example.com/users
+
+transform:
+  - rename:
+      userId: user_id
+  - convert:
+      user_id: int
+  - add:
+      full_name: "${first_name} ${last_name}"
+      loaded_at: now()
+
+load:
+  type: postgres  # Required: postgres | sqlite | file | cloud_storage
+  table: users
+  database:
+    url: ${DATABASE_URL}
+```
+
+```python
+from pycharter import Pipeline
+import asyncio
+
+async def main():
+    pipeline = Pipeline.from_config("pipelines/users/pipeline.yaml")
+    result = await pipeline.run()
+    print(f"Loaded {result.rows_loaded} rows")
+
+asyncio.run(main())
+```
+
+### Multi-File Format
+
+```bash
+mkdir -p pipelines/users
+cp pycharter/data/templates/etl/extract_http_simple.yaml pipelines/users/extract.yaml
+cp pycharter/data/templates/etl/transform_simple.yaml pipelines/users/transform.yaml
+cp pycharter/data/templates/etl/load_postgresql.yaml pipelines/users/load.yaml
+```
+
+```python
+pipeline = Pipeline.from_config("pipelines/users/")
+```
+
+## Type Field (Required)
+
+All extract and load configs require an explicit `type` field:
+
+**Extract types:**
+- `http` - HTTP/API extraction
+- `file` - Local file extraction (CSV, JSON, Parquet, etc.)
+- `database` - SQL database extraction
+- `cloud_storage` - Cloud storage (S3, GCS, Azure)
+
+**Load types:**
+- `postgres` / `postgresql` - PostgreSQL
+- `sqlite` - SQLite
+- `file` - Local file (JSON, CSV, Parquet)
+- `cloud_storage` - Cloud storage (S3, GCS, Azure)
+
+## Extract Templates
+
+| Template | Type | Description |
+|----------|------|-------------|
+| `extract_http_simple.yaml` | http | Single HTTP request (no pagination) |
+| `extract_http_paginated.yaml` | http | HTTP with pagination (page/offset/cursor) |
+| `extract_http_path_params.yaml` | http | HTTP with `{param}` path substitution |
+| `extract_file_csv.yaml` | file | CSV file extraction |
+| `extract_file_json.yaml` | file | JSON file extraction |
+| `extract_file_parquet.yaml` | file | Parquet file extraction |
+| `extract_file_glob.yaml` | file | Multiple files via glob pattern |
+| `extract_database.yaml` | database | SQL database extraction |
+| `extract_database_ssh.yaml` | database | Database via SSH tunnel |
+| `extract_cloud_s3.yaml` | cloud_storage | AWS S3 extraction |
+| `extract_cloud_gcs.yaml` | cloud_storage | Google Cloud Storage |
+| `extract_cloud_azure.yaml` | cloud_storage | Azure Blob Storage |
+
+## Transform Templates
+
+| Template | Description |
+|----------|-------------|
+| `transform_simple.yaml` | rename, convert, defaults, add, select, drop |
+| `transform_custom_function.yaml` | Call Python function |
+| `transform_jsonata.yaml` | JSONata expressions |
+| `transform_combined.yaml` | Simple + JSONata + custom |
+
+### Transform Formats
+
+**List format (ordered)** - Transforms applied in specified order:
+
+```yaml
+transform:
+  - rename: {old_field: new_field}
+  - convert: {field: int}
+  - add:
+      full_name: "${first_name} ${last_name}"
+      timestamp: now()
+  - select: [id, name, email]
+```
+
+**Dict format (legacy)** - Fixed order: rename → convert → defaults → add → select → drop
+
+```yaml
+transform:
+  rename:
+    old_field: new_field
+  convert:
+    field: int
+```
+
+### Expression Syntax
+
+In `add` fields, you can use expressions:
+
+- `${field_name}` - Reference field value
+- `${field_name:-default}` - Field with default if missing
+- `now()` - Current timestamp (ISO format)
+- `uuid()` - Generate UUID
+- `concat(${a}, " ", ${b})` - Concatenate values
+- `lower(${field})` / `upper(${field})` - Case conversion
+
+## Load Templates
+
+| Template | Type | Description |
+|----------|------|-------------|
+| `load_postgresql.yaml` | postgres | PostgreSQL (upsert, insert, etc.) |
+| `load_sqlite.yaml` | sqlite | SQLite database |
+| `load_file.yaml` | file | JSON, CSV, Parquet file |
+| `load_upsert.yaml` | postgres | Upsert by primary key |
+| `load_insert.yaml` | postgres | Insert only |
+| `load_truncate_and_load.yaml` | postgres | Truncate then insert |
+| `load_with_dlq.yaml` | postgres | With dead letter queue |
+| `load_with_ssh_tunnel.yaml` | postgres | Via SSH tunnel |
+| `load_cloud_s3.yaml` | cloud_storage | AWS S3 |
+| `load_cloud_gcs.yaml` | cloud_storage | Google Cloud Storage |
+| `load_cloud_azure.yaml` | cloud_storage | Azure Blob Storage |
+
+## Complete Pipeline Template
+
+See `pipeline_http_to_db.yaml` for a complete single-file pipeline example.
+
+## Variable Substitution
+
+Configs support `${VAR}` syntax for flexible configuration:
+
+```yaml
+path: ${DATA_DIR}/input.json                  # From variables
+params:
+  api_key: ${API_KEY:?API_KEY is required}    # Required - error if missing
+  limit: ${BATCH_LIMIT:-100}                  # With default value
+database:
+  url: ${DATABASE_URL}                        # From environment
+```
+
+Provide values via the `variables` parameter (recommended) or environment variables:
+
+```python
+# Recommended: explicit variables - no assumptions about structure
+pipeline = Pipeline.from_config_files(
+    extract="my_extract.yaml",
+    load="my_load.yaml",
+    variables={
+        "DATA_DIR": "./data",
+        "OUTPUT_DIR": "./output",
+        "API_KEY": "xxx",
+        "DATABASE_URL": "postgresql://..."
+    }
+)
+
+# Or with from_config() for directory-based loading
+pipeline = Pipeline.from_config(
+    "pipelines/users",
+    variables={"API_KEY": "xxx"}
+)
+```
+
+## Programmatic API
+
+Instead of config files, use the Pipeline API directly:
+
+```python
+from pycharter import (
+    Pipeline, HTTPExtractor, PostgresLoader,
+    Rename, Select, Filter, Convert, AddField
+)
+
+pipeline = (
+    Pipeline(HTTPExtractor(url="https://api.example.com/users"))
+    | Rename({"userName": "user_name"})
+    | AddField("full_name", "${first_name} ${last_name}")  # Expression support!
+    | Select(["id", "user_name", "email", "full_name"])
+    | Convert({"id": int})
+    | Filter(lambda r: r.get("email"))
+    | PostgresLoader(
+        connection_string="postgresql://localhost/db",
+        table="users",
+        write_method="upsert",
+        primary_key="id"
+    )
+)
+
+result = await pipeline.run()
+```
+
+## Examples
+
+See `examples/etl_config_example/` for working examples with config files.

pycharter/data/templates/etl/extract_cloud_azure.yaml

@@ -0,0 +1,24 @@
+# Template: Extract from Azure Blob Storage
+# Copy to your pipeline directory as extract.yaml
+# Requires: pip install azure-storage-blob
+
+title: azure_extraction
+description: Extract data from Azure Blob Storage
+version: "1.0.0"
+
+# Source type (required)
+type: cloud_storage
+
+storage:
+  provider: azure
+  container: my-container
+  path: data/input.json
+
+  # Or prefix for multiple blobs
+  # prefix: data/inputs/
+
+  credentials:
+    connection_string: ${AZURE_STORAGE_CONNECTION_STRING}
+
+format: json
+batch_size: 1000

pycharter/data/templates/etl/extract_cloud_gcs.yaml

@@ -0,0 +1,25 @@
+# Template: Extract from Google Cloud Storage
+# Copy to your pipeline directory as extract.yaml
+# Requires: pip install google-cloud-storage
+
+title: gcs_extraction
+description: Extract data from Google Cloud Storage
+version: "1.0.0"
+
+# Source type (required)
+type: cloud_storage
+
+storage:
+  provider: gcs
+  bucket: my-bucket-name
+  path: data/input.json
+
+  # Or prefix for multiple objects
+  # prefix: data/inputs/
+
+  # Credentials (optional - uses default credentials if omitted)
+  # credentials:
+  #   service_account_file: /path/to/service-account.json
+
+format: json
+batch_size: 1000

pycharter/data/templates/etl/extract_cloud_s3.yaml

@@ -0,0 +1,30 @@
+# Template: Extract from AWS S3
+# Copy to your pipeline directory as extract.yaml
+# Requires: pip install boto3
+
+title: s3_extraction
+description: Extract data from AWS S3
+version: "1.0.0"
+
+# Source type (required)
+type: cloud_storage
+
+storage:
+  provider: s3
+  bucket: my-bucket-name
+
+  # Single object
+  path: data/input.json
+
+  # Or prefix for multiple objects (uncomment)
+  # prefix: data/inputs/
+  # pattern: "*.json"  # Filter by pattern
+
+  # Credentials (optional - uses AWS credential chain if omitted)
+  # credentials:
+  #   aws_access_key_id: ${AWS_ACCESS_KEY_ID}
+  #   aws_secret_access_key: ${AWS_SECRET_ACCESS_KEY}
+  #   region: us-east-1
+
+format: json
+batch_size: 1000

pycharter/data/templates/etl/extract_database.yaml

@@ -0,0 +1,34 @@
+# Template: Extract from Database
+# Copy to your pipeline directory as extract.yaml
+#
+# Supported databases: PostgreSQL, MySQL, SQLite, MSSQL, Oracle
+
+title: database_extraction
+description: Extract data from SQL database
+version: "1.0.0"
+
+# Source type (required)
+type: database
+
+# SQL query to execute
+query: |
+  SELECT id, name, email, created_at
+  FROM users
+  WHERE status = :status
+  ORDER BY created_at DESC
+  LIMIT :limit
+
+# Query parameters (substituted into :param placeholders)
+params:
+  status: active
+  limit: 10000
+
+# Database connection
+connection_string: ${DATABASE_URL:?DATABASE_URL is required}
+# Formats:
+#   PostgreSQL: postgresql://user:pass@host:5432/db
+#   MySQL:      mysql://user:pass@host:3306/db
+#   SQLite:     sqlite:///./data/local.db
+#   MSSQL:      mssql+pyodbc://user:pass@host/db?driver=ODBC+Driver+17+for+SQL+Server
+
+batch_size: 1000

pycharter/data/templates/etl/extract_database_ssh.yaml

@@ -0,0 +1,40 @@
+# Template: Extract from Database via SSH Tunnel
+# Copy to your pipeline directory as extract.yaml
+# Use when database is only accessible through a bastion/jump host
+
+title: database_ssh_extraction
+description: Extract from database through SSH tunnel
+version: "1.0.0"
+
+# Source type (required)
+type: database
+
+query: |
+  SELECT id, name, email, created_at
+  FROM users
+  WHERE status = :status
+
+params:
+  status: active
+
+# Database connection (use localhost since tunnel forwards)
+connection_string: postgresql://user:pass@localhost:5433/database
+
+# SSH tunnel configuration
+ssh_tunnel:
+  enabled: true
+  host: bastion.example.com
+  port: 22
+  username: ${SSH_USER}
+  key_file: ~/.ssh/id_rsa
+  # Or use password:
+  # password: ${SSH_PASSWORD}
+
+  # Remote database endpoint (the actual database host)
+  remote_host: db.internal.example.com
+  remote_port: 5432
+
+  # Local port for tunnel (connect to this)
+  local_port: 5433
+
+batch_size: 1000

pycharter/data/templates/etl/extract_file_csv.yaml

@@ -0,0 +1,21 @@
+# Template: Extract from CSV File
+# Copy to your pipeline directory as extract.yaml
+
+title: csv_file_extraction
+description: Extract data from CSV file
+version: "1.0.0"
+
+# Source type (required)
+type: file
+
+path: ./data/input.csv
+format: csv
+batch_size: 1000
+
+# CSV-specific options
+csv_options:
+  delimiter: ","
+  quotechar: '"'
+  has_header: true
+  # encoding: utf-8
+  # skip_rows: 0

pycharter/data/templates/etl/extract_file_glob.yaml

@@ -0,0 +1,25 @@
+# Template: Extract from Multiple Files (Glob Pattern)
+# Copy to your pipeline directory as extract.yaml
+
+title: glob_file_extraction
+description: Extract from multiple files matching pattern
+version: "1.0.0"
+
+# Source type (required)
+type: file
+
+# Glob pattern to match files
+path: ./data/inputs/*.json
+# Examples:
+#   ./data/*.csv           - All CSV files in data/
+#   ./data/**/*.json       - All JSON files recursively
+#   ./data/2024-*.parquet  - Parquet files starting with 2024-
+
+# Format (auto-detected from first file if omitted)
+format: json
+
+batch_size: 1000
+
+# Processing options
+# sort_files: true        # Process files in sorted order
+# fail_on_empty: false    # Don't fail if no files match

pycharter/data/templates/etl/extract_file_json.yaml

@@ -0,0 +1,24 @@
+# Template: Extract from JSON File
+# Copy to your pipeline directory as extract.yaml
+
+title: json_file_extraction
+description: Extract data from JSON file
+version: "1.0.0"
+
+# Source type (required)
+type: file
+
+# File path
+path: ./data/input.json
+# Or use environment variable:
+# path: ${INPUT_DIR:-./data}/input.json
+
+# Format (auto-detected from extension if omitted)
+format: json
+
+# Batch size for yielding records
+batch_size: 1000
+
+# For JSON files containing an object with data array:
+# response_path: data.items
+# response_path: results

pycharter/data/templates/etl/extract_file_parquet.yaml

@@ -0,0 +1,20 @@
+# Template: Extract from Parquet File
+# Copy to your pipeline directory as extract.yaml
+# Requires: pip install pyarrow or pip install fastparquet
+
+title: parquet_file_extraction
+description: Extract data from Parquet file
+version: "1.0.0"
+
+# Source type (required)
+type: file
+
+path: ./data/input.parquet
+format: parquet
+batch_size: 1000
+
+# Optional: select specific columns
+# columns:
+#   - id
+#   - name
+#   - created_at