pycharter 0.0.22__py3-none-any.whl → 0.0.23__py3-none-any.whl

api/routes/v1/templates.py

@@ -18,40 +18,59 @@ router = APIRouter()
 
 def _find_template_dir() -> Path:
     """
-    Find template directory in multiple locations (package, source, etc.).
-    
+    Find contract template directory in multiple locations (package, source, etc.).
+
+    Contract templates (schema, metadata, coercion/validation rules, contract)
+    live under templates/contract/.
+
     Priority:
-    1. Installed package location (pycharter/data/templates/)
+    1. Installed package location (pycharter/data/templates/contract/)
     2. Source location (relative to api/routes/v1/)
     3. Legacy location (data/aviation_examples/template/)
-    
+
     Returns:
-        Path to template directory
+        Path to contract template directory (templates/contract/)
     """
     # Try installed package location (when package is installed)
     try:
         import pycharter
+
         pycharter_path = Path(pycharter.__file__).parent
-        package_templates = pycharter_path / "data" / "templates"
-        if package_templates.exists() and (package_templates / "template_schema.yaml").exists():
-            return package_templates
+        package_contract = pycharter_path / "data" / "templates" / "contract"
+        if (
+            package_contract.exists()
+            and (package_contract / "template_schema.yaml").exists()
+        ):
+            return package_contract
     except (ImportError, AttributeError):
         pass
-    
+
     # Try source locations (development)
     possible_paths = [
-        Path(__file__).parent.parent.parent.parent / "pycharter" / "data" / "templates",
-        Path(__file__).parent.parent.parent.parent / "data" / "aviation_examples" / "template",
-        Path.cwd() / "pycharter" / "data" / "templates",
+        Path(__file__).parent.parent.parent.parent
+        / "pycharter"
+        / "data"
+        / "templates"
+        / "contract",
+        Path(__file__).parent.parent.parent.parent
+        / "data"
+        / "aviation_examples"
+        / "template",
+        Path.cwd() / "pycharter" / "data" / "templates" / "contract",
         Path.cwd() / "data" / "aviation_examples" / "template",
     ]
-    
+
     for template_path in possible_paths:
         if template_path.exists() and (template_path / "template_schema.yaml").exists():
             return template_path
-    
+
     # Fallback to original location
-    fallback = Path(__file__).parent.parent.parent.parent / "data" / "aviation_examples" / "template"
+    fallback = (
+        Path(__file__).parent.parent.parent.parent
+        / "data"
+        / "aviation_examples"
+        / "template"
+    )
     return fallback
 
 
@@ -77,7 +96,7 @@ def _get_template_path(filename: str) -> Path:
 async def download_schema_template() -> FileResponse:
     """
     Download the schema template file.
-    
+
     Returns:
         YAML file containing a template schema structure
     """
@@ -111,7 +130,7 @@ async def download_schema_template() -> FileResponse:
 async def download_metadata_template() -> FileResponse:
     """
     Download the metadata template file.
-    
+
     Returns:
         YAML file containing a template metadata structure
     """
@@ -145,7 +164,7 @@ async def download_metadata_template() -> FileResponse:
 async def download_coercion_rules_template() -> FileResponse:
     """
     Download the coercion rules template file.
-    
+
     Returns:
         YAML file containing a template coercion rules structure
     """
@@ -179,7 +198,7 @@ async def download_coercion_rules_template() -> FileResponse:
 async def download_validation_rules_template() -> FileResponse:
     """
     Download the validation rules template file.
-    
+
     Returns:
         YAML file containing a template validation rules structure
     """
@@ -213,14 +232,14 @@ async def download_validation_rules_template() -> FileResponse:
 async def download_contract_artifacts() -> Response:
     """
     Download all contract artifact templates as a ZIP archive.
-    
+
     The ZIP file contains:
     - template_schema.yaml
     - template_metadata.yaml
     - template_coercion_rules.yaml
     - template_validation_rules.yaml
     - template_contract.yaml
-    
+
     Returns:
         ZIP file containing all template files
     """
@@ -233,7 +252,7 @@ async def download_contract_artifacts() -> Response:
             "template_validation_rules.yaml",
             "template_contract.yaml",
         ]
-        
+
         # Create ZIP archive in memory
         zip_buffer = io.BytesIO()
         with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zip_file:
@@ -241,12 +260,12 @@ async def download_contract_artifacts() -> Response:
                 template_path = _get_template_path(template_file)
                 # Read file content and add to ZIP
                 zip_file.write(template_path, arcname=template_file)
-        
+
         # Get ZIP content
         zip_buffer.seek(0)
         zip_content = zip_buffer.read()
         zip_buffer.close()
-        
+
         return Response(
             content=zip_content,
             media_type="application/zip",

pycharter/data/templates/etl/README.md

@@ -0,0 +1,91 @@
+# ETL Config Templates
+
+Templates for ETL pipeline configuration files. Copy into your contract directory as `extract.yaml`, `transform.yaml`, or `load.yaml`, and adjust for your environment.
+
+## Directory layout
+
+Use a contract directory that contains at least:
+
+- `schema.json` or `schema.yaml` (required)
+- `extract.yaml` (required) – pick one of the extract templates below
+- `transform.yaml` (optional) – pick one of the transform templates
+- `load.yaml` (required) – pick one of the load templates
+
+## Extract templates
+
+| Template | Description |
+|----------|-------------|
+| `extract_http_simple.yaml` | HTTP API, single request, no pagination |
+| `extract_http_paginated.yaml` | HTTP API with pagination (page / offset / cursor / next_url / link_header) |
+| `extract_http_path_params.yaml` | HTTP API with `{param}` path substitution |
+| `extract_file_csv.yaml` | Local CSV file |
+| `extract_file_json.yaml` | Local JSON file (array or object with data/results/items) |
+| `extract_file_parquet.yaml` | Local Parquet file |
+| `extract_file_glob.yaml` | Multiple files via glob (e.g. `*.csv`, `data/*.json`) |
+| `extract_database.yaml` | SQL query against PostgreSQL/MySQL/SQLite/MSSQL/Oracle |
+| `extract_database_ssh.yaml` | Database extraction through an SSH tunnel |
+| `extract_cloud_s3.yaml` | AWS S3 (single object or prefix) |
+| `extract_cloud_gcs.yaml` | Google Cloud Storage |
+| `extract_cloud_azure.yaml` | Azure Blob Storage |
+
+Source type is chosen by `source_type` or by auto-detection:
+
+- `base_url` / `api_endpoint` → `http`
+- `file_path` → `file`
+- `database` → `database`
+- `storage` → `cloud_storage`
+
+## Transform templates
+
+| Template | Description |
+|----------|-------------|
+| `transform_simple.yaml` | Rename, convert, defaults, add, select, drop |
+| `transform_jsonata.yaml` | JSONata expressions (record or batch) |
+| `transform_custom_function.yaml` | Call a Python function (module.function) |
+| `transform_combined.yaml` | Simple ops + JSONata + optional custom function |
+
+Order in the pipeline: **Simple operations → JSONata → Custom function**.
+
+## Load templates
+
+| Template | Description |
+|----------|-------------|
+| `load_upsert.yaml` | Upsert by primary key (default) |
+| `load_insert.yaml` | Insert only |
+| `load_truncate_and_load.yaml` | Truncate table then load (full refresh) |
+| `load_with_dlq.yaml` | Load with Dead Letter Queue for failed rows |
+| `load_postgresql.yaml` | Load into PostgreSQL |
+| `load_sqlite.yaml` | Load into SQLite |
+| `load_with_ssh_tunnel.yaml` | Load into a DB reachable only via SSH tunnel |
+
+Write methods: `insert`, `upsert`, `replace`, `update`, `delete`, `append`, `truncate_and_load`.
+
+## Variable injection
+
+Configs support `${VAR}` and `${VAR:-default}`. Provide values via:
+
+- Environment variables
+- `config_context` when creating `ETLOrchestrator(...)`
+- Optional `source_file` for resolution (e.g. path to `extract.yaml`)
+
+Examples:
+
+- `url: ${TARGET_DATABASE_URL:?TARGET_DATABASE_URL is required}`
+- `api_key: ${API_KEY}`
+- `rate_limit_delay: ${RATE_LIMIT_DELAY:-0.2}`
+
+## Full pipeline reference
+
+See `pipeline_http_to_db.yaml` for a combined extract/transform/load example. Real pipelines use separate `extract.yaml`, `transform.yaml`, and `load.yaml` in the contract directory.
+
+## Usage
+
+```python
+from pycharter.etl_generator import ETLOrchestrator
+
+# From contract dir (contains schema + extract/transform/load from these templates)
+orchestrator = ETLOrchestrator(contract_dir="path/to/contract_dir")
+result = await orchestrator.run(**input_params)
+```
+
+For more examples, see `examples/10_basic_etl.py` through `examples/14_etl_with_transforms.py`.

pycharter/data/templates/etl/extract_cloud_azure.yaml

@@ -0,0 +1,23 @@
+# Template: Azure Blob Storage extraction
+# source_type: cloud_storage. Requires azure-storage-blob.
+# Copy to your contract dir as extract.yaml and customize.
+
+title: azure_blob_extraction
+description: Extract data from Azure Blob Storage
+version: 1.0.0
+
+source_type: cloud_storage
+storage:
+  provider: azure
+  bucket: my-container   # Azure container name
+  path: data/records.json
+  credentials:
+    connection_string: ${AZURE_STORAGE_CONNECTION_STRING}
+    # Or: account_name + account_key
+
+format: json
+batch_size: 1000
+max_records: null
+
+# Prefix for multiple blobs:
+# path: data/daily/

pycharter/data/templates/etl/extract_cloud_gcs.yaml

@@ -0,0 +1,22 @@
+# Template: Google Cloud Storage extraction
+# source_type: cloud_storage. Requires google-cloud-storage.
+# Copy to your contract dir as extract.yaml and customize.
+
+title: gcs_extraction
+description: Extract data from Google Cloud Storage
+version: 1.0.0
+
+source_type: cloud_storage
+storage:
+  provider: gcs
+  bucket: my-gcs-bucket
+  path: exports/data.csv
+  credentials: /path/to/service-account-key.json
+  # Or inline: credentials: { "type": "service_account", ... }
+
+format: csv
+batch_size: 1000
+max_records: null
+
+# Prefix for multiple files:
+# path: exports/2024/

pycharter/data/templates/etl/extract_cloud_s3.yaml

@@ -0,0 +1,24 @@
+# Template: AWS S3 extraction
+# source_type: cloud_storage. Requires boto3 (pip install boto3 or pycharter[etl]).
+# Copy to your contract dir as extract.yaml and customize.
+
+title: s3_extraction
+description: Extract data from AWS S3
+version: 1.0.0
+
+source_type: cloud_storage
+storage:
+  provider: s3
+  bucket: my-bucket-name
+  path: data/records.parquet   # Single file
+  credentials:
+    aws_access_key_id: ${AWS_ACCESS_KEY_ID}
+    aws_secret_access_key: ${AWS_SECRET_ACCESS_KEY}
+    region: us-east-1
+
+format: parquet  # csv | json | parquet | excel | xml
+batch_size: 1000
+max_records: null
+
+# Multiple files: use prefix (trailing slash or path without filename)
+# path: data/daily/   # All objects with this prefix

pycharter/data/templates/etl/extract_database.yaml

@@ -0,0 +1,28 @@
+# Template: Database extraction
+# source_type: database. Supports PostgreSQL, MySQL, SQLite, MSSQL, Oracle.
+# Copy to your contract dir as extract.yaml and customize.
+
+title: database_extraction
+description: Extract data from database via SQL query
+version: 1.0.0
+
+source_type: database
+database:
+  url: postgresql://user:pass@host:5432/dbname
+  # type: postgresql  # Optional; auto-detected from URL
+
+query: |
+  SELECT id, name, email, created_at
+  FROM users
+  WHERE created_at > :start_date
+  ORDER BY id
+
+query_params:
+  start_date: "2024-01-01"
+
+batch_size: 1000
+max_records: null
+
+# Use variable injection for credentials:
+# database:
+#   url: ${SOURCE_DATABASE_URL}

pycharter/data/templates/etl/extract_database_ssh.yaml

@@ -0,0 +1,27 @@
+# Template: Database extraction with SSH tunnel
+# For databases not directly reachable. Requires sshtunnel (pip install pycharter[etl]).
+# Copy to your contract dir as extract.yaml and customize.
+
+title: database_ssh_extraction
+description: Extract from database via SSH tunnel
+version: 1.0.0
+
+source_type: database
+database:
+  url: postgresql://user:pass@localhost:5433/remote_db
+  ssh_tunnel:
+    enabled: true
+    host: jump.example.com
+    port: 22
+    username: ${SSH_USER}
+    password: ${SSH_PASSWORD}
+    # key_file: ~/.ssh/id_rsa  # Alternative to password
+    remote_host: db.internal.example.com
+    remote_port: 5432
+    local_port: 5433
+
+query: "SELECT id, name, value FROM data_table"
+query_params: {}
+
+batch_size: 1000
+max_records: null

pycharter/data/templates/etl/extract_file_csv.yaml

@@ -0,0 +1,17 @@
+# Template: File-based extraction - CSV
+# source_type: file. Supports CSV, TSV, JSON, JSONL, Parquet, Excel, XML.
+# Copy to your contract dir as extract.yaml and customize.
+
+title: file_csv_extraction
+description: Extract data from CSV file
+version: 1.0.0
+
+source_type: file
+file_path: /path/to/data.csv
+format: csv  # Optional; auto-detected from extension if omitted
+
+batch_size: 1000
+max_records: null
+
+# Variable injection supported:
+# file_path: ${DATA_DIR}/exports/records.csv

pycharter/data/templates/etl/extract_file_glob.yaml

@@ -0,0 +1,17 @@
+# Template: File-based extraction - multiple files via glob pattern
+# Processes all files matching the pattern. Copy as extract.yaml and customize.
+
+title: file_glob_extraction
+description: Extract from multiple files using glob pattern
+version: 1.0.0
+
+source_type: file
+file_path: /path/to/daily/exports/*.csv
+format: csv  # Same format for all matched files
+
+batch_size: 1000
+max_records: null
+
+# Examples:
+# file_path: /data/2024/*/sales_*.json
+# file_path: ${EXPORTS_DIR}/**/*.parquet

pycharter/data/templates/etl/extract_file_json.yaml

@@ -0,0 +1,14 @@
+# Template: File-based extraction - JSON
+# For JSON: expects array or object with key in [data, results, items, records, values].
+# Copy to your contract dir as extract.yaml and customize.
+
+title: file_json_extraction
+description: Extract data from JSON file
+version: 1.0.0
+
+source_type: file
+file_path: /path/to/data.json
+format: json
+
+batch_size: 1000
+max_records: null

pycharter/data/templates/etl/extract_file_parquet.yaml

@@ -0,0 +1,13 @@
+# Template: File-based extraction - Parquet
+# Requires pandas. Copy to your contract dir as extract.yaml and customize.
+
+title: file_parquet_extraction
+description: Extract data from Parquet file
+version: 1.0.0
+
+source_type: file
+file_path: /path/to/data.parquet
+format: parquet
+
+batch_size: 1000
+max_records: null

pycharter/data/templates/etl/extract_http_paginated.yaml

@@ -0,0 +1,75 @@
+# Template: HTTP/API extraction with pagination
+# Strategies: page | offset | cursor | next_url | link_header
+# Copy to your contract dir as extract.yaml and customize.
+
+title: http_paginated_extraction
+description: Extract data from HTTP API with pagination
+version: 1.0.0
+
+source_type: http
+base_url: https://api.example.com
+api_endpoint: /v1/records
+method: GET
+
+params:
+  api_key: ${API_KEY}
+  limit: 100
+  page: 0  # Updated by pagination; use param_name from page config
+
+input_params: {}
+headers:
+  Accept: application/json
+
+batch_size: 1000
+max_records: null
+
+response_format: json
+response_path: data  # Path to array in response, e.g. "data.results"
+
+retry:
+  max_attempts: 3
+  backoff_factor: 2
+  retry_on_status: [429, 500, 502, 503, 504]
+
+timeout:
+  connect: 10
+  read: 60
+  write: 10
+  pool: 10
+
+rate_limit_delay: 0.2
+
+# Pagination config
+pagination:
+  enabled: true
+  strategy: page  # page | offset | cursor | next_url | link_header
+  page_delay: 0.1
+
+  # Strategy: page
+  page:
+    param_name: page
+    start: 0
+    increment: 1
+
+  # Strategy: offset (alternative)
+  # offset:
+  #   param_name: offset
+  #   start: 0
+  #   increment_by: limit
+
+  # Strategy: cursor (alternative)
+  # cursor:
+  #   param_name: cursor
+  #   response_path: next_cursor
+
+  # Strategy: next_url (alternative) - next URL in response
+  # next_url:
+  #   response_path: links.next
+
+  stop_conditions:
+    - type: empty_response
+    - type: fewer_records
+    - type: max_pages
+      value: 1000
+    # - type: max_records
+    #   value: 50000

pycharter/data/templates/etl/extract_http_path_params.yaml

@@ -0,0 +1,45 @@
+# Template: HTTP extraction with path parameters in URL
+# Use {param_name} in api_endpoint; values come from params or input_params.
+# Copy to your contract dir as extract.yaml and customize.
+
+title: http_path_params_extraction
+description: HTTP API with path parameter substitution
+version: 1.0.0
+
+source_type: http
+base_url: https://api.example.com
+api_endpoint: /v1/symbols/{symbol}/prices/{start_date}/{end_date}
+method: GET
+
+params:
+  api_key: ${API_KEY}
+  symbol: AAPL      # Or from input_params at runtime
+  start_date: 2024-01-01
+  end_date: 2024-12-31
+
+input_params:
+  symbol:
+    type: string
+    required: true
+    description: Stock or symbol code
+  start_date:
+    type: string
+    required: false
+  end_date:
+    type: string
+    required: false
+
+headers:
+  Accept: application/json
+
+batch_size: 1000
+response_format: json
+response_path: null
+
+retry:
+  max_attempts: 3
+  backoff_factor: 2
+  retry_on_status: [429, 500, 502, 503, 504]
+
+pagination:
+  enabled: false

pycharter/data/templates/etl/extract_http_simple.yaml

@@ -0,0 +1,52 @@
+# Template: HTTP/API extraction (single request, no pagination)
+# Use with source_type: http (or omit; http is default when base_url/api_endpoint present)
+# Copy to your contract dir as extract.yaml and customize.
+
+title: http_api_extraction
+description: Extract data from HTTP API - single request
+version: 1.0.0
+
+# Optional: set explicitly. Auto-detected from base_url/api_endpoint if omitted.
+# source_type: http
+
+base_url: https://api.example.com
+api_endpoint: /v1/records
+method: GET
+
+params:
+  api_key: ${API_KEY:?API_KEY is required}
+  # Add query params as needed
+
+# Input parameters (passed at runtime via orchestrator.run(**kwargs))
+input_params:
+  id:
+    type: string
+    required: false
+    description: Optional filter by ID
+
+headers:
+  Accept: application/json
+  User-Agent: PyCharter-ETL/1.0
+
+batch_size: 1000
+max_records: null
+
+# Response handling
+response_format: json
+response_path: null  # e.g. "data.items" to extract nested array
+
+retry:
+  max_attempts: 3
+  backoff_factor: 2
+  retry_on_status: [429, 500, 502, 503, 504]
+
+timeout:
+  connect: 10
+  read: 30
+  write: 10
+  pool: 10
+
+rate_limit_delay: 0.2
+
+pagination:
+  enabled: false

pycharter/data/templates/etl/load_insert.yaml

@@ -0,0 +1,17 @@
+# Template: Load with insert only (no conflict handling)
+# Use for append-only or when duplicates are impossible.
+# Copy to your contract dir as load.yaml and customize.
+
+title: load_insert
+description: Insert new rows only
+version: 1.0.0
+
+target_table: events
+schema_name: public
+write_method: insert
+# primary_key not required for insert
+
+batch_size: 1000
+
+database:
+  url: ${TARGET_DATABASE_URL}

pycharter/data/templates/etl/load_postgresql.yaml

@@ -0,0 +1,17 @@
+# Template: Load to PostgreSQL
+# Copy to your contract dir as load.yaml and customize.
+
+title: load_postgresql
+description: Load into PostgreSQL
+version: 1.0.0
+
+target_table: my_table
+schema_name: public
+write_method: upsert
+primary_key: id
+batch_size: 1000
+
+database:
+  type: postgresql
+  url: postgresql://user:pass@host:5432/dbname
+  # With async (if supported): postgresql+asyncpg://...

pycharter/data/templates/etl/load_sqlite.yaml

@@ -0,0 +1,16 @@
+# Template: Load to SQLite
+# Copy to your contract dir as load.yaml and customize.
+
+title: load_sqlite
+description: Load into SQLite
+version: 1.0.0
+
+target_table: my_table
+schema_name: main   # SQLite default schema
+write_method: upsert
+primary_key: id
+batch_size: 1000
+
+database:
+  type: sqlite
+  url: sqlite:///./data.db

pycharter/data/templates/etl/load_truncate_and_load.yaml

@@ -0,0 +1,18 @@
+# Template: Truncate table then load (full refresh)
+# Destructive: removes all existing rows. Use for dim tables or full snapshots.
+# Copy to your contract dir as load.yaml and customize.
+
+title: load_truncate_and_load
+description: Full table refresh - truncate then insert all
+version: 1.0.0
+
+target_table: dimension_customers
+schema_name: public
+write_method: truncate_and_load
+primary_key: id   # Optional for truncate; useful for schema/docs
+
+batch_size: 1000
+
+database:
+  type: postgresql
+  url: ${TARGET_DATABASE_URL}