pycharter 0.0.22__py3-none-any.whl → 0.0.24__py3-none-any.whl

pycharter/data/templates/etl/extract_http_paginated.yaml

@@ -0,0 +1,79 @@
+# Template: HTTP/API Extraction with Pagination
+# Copy to your pipeline directory as extract.yaml
+#
+# Supported pagination strategies:
+#   - page: Page number pagination (?page=1, ?page=2)
+#   - offset: Offset pagination (?offset=0, ?offset=100)
+#   - cursor: Cursor-based pagination (response contains next_cursor)
+#   - next_url: URL-based pagination (response contains next URL)
+#   - link_header: RFC 5988 Link header pagination
+
+title: http_paginated_extraction
+description: Extract from HTTP API with pagination
+version: "1.0.0"
+
+# Source type (required)
+type: http
+
+base_url: https://api.example.com
+api_endpoint: /v1/users
+method: GET
+
+params:
+  api_key: ${API_KEY}
+  limit: 100  # Records per page
+
+headers:
+  Accept: application/json
+
+batch_size: 1000
+response_format: json
+response_path: data.items  # Path to data array in response
+
+# Pagination configuration
+pagination:
+  enabled: true
+
+  # Strategy: page | offset | cursor | next_url | link_header
+  strategy: page
+
+  # Page-based pagination
+  page:
+    param_name: page     # Query param name
+    start: 1             # Starting page number
+    increment: 1         # Page increment
+
+  # Offset-based pagination (uncomment to use)
+  # offset:
+  #   param_name: offset
+  #   start: 0
+  #   increment_by: limit  # or a number like 100
+
+  # Cursor-based pagination (uncomment to use)
+  # cursor:
+  #   param_name: cursor
+  #   response_path: meta.next_cursor  # Path to cursor in response
+
+  # URL-based pagination (uncomment to use)
+  # next_url:
+  #   response_path: links.next  # Path to next URL in response
+
+  # Stop conditions
+  stop_conditions:
+    - type: fewer_records  # Stop when page returns fewer records than limit
+    - type: max_pages
+      value: 100           # Maximum pages to fetch
+    # - type: max_records
+    #   value: 10000       # Maximum total records
+    # - type: empty_response
+
+  # Delay between page requests (seconds)
+  page_delay: 0.1
+
+retry:
+  max_attempts: 3
+  backoff_factor: 2
+
+timeout:
+  connect: 10
+  read: 30

pycharter/data/templates/etl/extract_http_path_params.yaml

@@ -0,0 +1,38 @@
+# Template: HTTP Extraction with Path Parameters
+# Copy to your pipeline directory as extract.yaml
+# Use when API endpoint contains path parameters like /users/{user_id}
+
+title: http_path_params_extraction
+description: HTTP extraction with path parameter substitution
+version: "1.0.0"
+
+# Source type (required)
+type: http
+
+base_url: https://api.example.com
+api_endpoint: /v1/users/{user_id}/orders
+
+method: GET
+
+# Path parameters (substituted into {param} placeholders in endpoint)
+# Pass values at runtime: pipeline.run(user_id="123")
+input_params:
+  user_id:
+    type: string
+    required: true
+    description: User ID to fetch orders for
+
+# Query parameters
+params:
+  api_key: ${API_KEY}
+  limit: 100
+
+headers:
+  Accept: application/json
+
+batch_size: 1000
+response_format: json
+response_path: data.orders  # Path to data array in response
+
+pagination:
+  enabled: false

pycharter/data/templates/etl/extract_http_simple.yaml

@@ -0,0 +1,62 @@
+# Template: HTTP/API Extraction (Single Request)
+# Copy to your pipeline directory as extract.yaml
+#
+# Usage:
+#   pipeline = Pipeline.from_config("path/to/pipeline")
+#   result = await pipeline.run()
+
+title: http_extraction
+description: Extract from HTTP API
+version: "1.0.0"
+
+# Source type (required)
+type: http
+
+# Option 1: Full URL
+url: https://api.example.com/v1/users
+
+# Option 2: Base URL + endpoint (uncomment to use)
+# base_url: https://api.example.com
+# api_endpoint: /v1/users
+
+# HTTP method
+method: GET
+
+# Query parameters
+params:
+  api_key: ${API_KEY:?API_KEY is required}
+  # limit: 100
+  # offset: 0
+
+# Request headers
+headers:
+  Accept: application/json
+  User-Agent: PyCharter-ETL/1.0
+  # Authorization: Bearer ${AUTH_TOKEN}
+
+# Batch size for yielding records
+batch_size: 1000
+
+# Response handling
+response_format: json
+response_path: null  # e.g. "data.items" for nested response
+
+# Retry configuration
+retry:
+  max_attempts: 3
+  backoff_factor: 2
+  retry_on_status: [429, 500, 502, 503, 504]
+
+# Timeout configuration (seconds)
+timeout:
+  connect: 10
+  read: 30
+  write: 10
+  pool: 10
+
+# Rate limiting delay between requests (seconds)
+rate_limit_delay: 0.2
+
+# Pagination (disabled for simple extraction)
+pagination:
+  enabled: false

pycharter/data/templates/etl/load_cloud_azure.yaml

@@ -0,0 +1,24 @@
+# Template: Load to Azure Blob Storage
+# Copy to your pipeline directory as load.yaml
+# Requires: pip install azure-storage-blob
+
+title: load_azure
+description: Load data to Azure Blob Storage
+version: "1.0.0"
+
+# Target type (required)
+type: cloud_storage
+
+storage:
+  provider: azure
+  container: my-container  # Azure uses "container" instead of "bucket"
+  path: output/data.json   # Blob path
+
+  # Credentials
+  credentials:
+    connection_string: ${AZURE_STORAGE_CONNECTION_STRING}
+    # Or use account name + key:
+    # account_name: mystorageaccount
+    # account_key: ${AZURE_STORAGE_KEY}
+
+format: json

pycharter/data/templates/etl/load_cloud_gcs.yaml

@@ -0,0 +1,22 @@
+# Template: Load to Google Cloud Storage
+# Copy to your pipeline directory as load.yaml
+# Requires: pip install google-cloud-storage
+
+title: load_gcs
+description: Load data to Google Cloud Storage
+version: "1.0.0"
+
+# Target type (required)
+type: cloud_storage
+
+storage:
+  provider: gcs
+  bucket: my-bucket-name
+  path: output/data.json  # Object path
+
+  # Credentials (optional - uses default credentials if omitted)
+  # credentials:
+  #   service_account_file: /path/to/service-account.json
+  #   project: my-project-id
+
+format: json

pycharter/data/templates/etl/load_cloud_s3.yaml

@@ -0,0 +1,27 @@
+# Template: Load to AWS S3
+# Copy to your pipeline directory as load.yaml
+# Requires: pip install boto3
+
+title: load_s3
+description: Load data to AWS S3
+version: "1.0.0"
+
+# Target type (required)
+type: cloud_storage
+
+storage:
+  provider: s3
+  bucket: my-bucket-name
+  path: output/data.json  # Object key/prefix
+
+  # Credentials (optional - uses AWS credential chain if omitted)
+  # credentials:
+  #   aws_access_key_id: ${AWS_ACCESS_KEY_ID}
+  #   aws_secret_access_key: ${AWS_SECRET_ACCESS_KEY}
+  #   region: us-east-1
+
+# Output format: json, jsonl, csv, parquet
+format: json
+
+# For partitioned output:
+# partition_by: [year, month]  # Creates year=2024/month=01/data.parquet

pycharter/data/templates/etl/load_file.yaml

@@ -0,0 +1,34 @@
+# Template: Load to File
+# Copy to your pipeline directory as load.yaml
+#
+# Supported formats: json, jsonl, csv, parquet
+# Write modes: overwrite, append
+
+title: load_file
+description: Load data to local file
+version: "1.0.0"
+
+# Target type (required)
+type: file
+
+# File path
+path: ./output/data.json
+# Or use environment variable:
+# path: ${OUTPUT_DIR:-./output}/data.json
+
+# Output format: json, jsonl, csv, parquet
+format: json
+
+# Write mode: overwrite, append
+write_mode: overwrite
+
+# CSV-specific options (when format: csv)
+# csv_options:
+#   delimiter: ","
+#   quotechar: '"'
+#   include_header: true
+
+# JSON-specific options (when format: json)
+# json_options:
+#   indent: 2
+#   ensure_ascii: false

pycharter/data/templates/etl/load_insert.yaml

@@ -0,0 +1,18 @@
+# Template: Load with Insert Only
+# Inserts new records, fails on duplicates
+# Copy to your pipeline directory as load.yaml
+
+title: load_insert
+description: Insert only (no updates)
+version: "1.0.0"
+
+# Target type (required)
+type: postgres
+
+table: my_table
+schema: public
+write_method: insert
+batch_size: 1000
+
+database:
+  url: ${DATABASE_URL:?DATABASE_URL is required}

pycharter/data/templates/etl/load_postgresql.yaml

@@ -0,0 +1,39 @@
+# Template: Load to PostgreSQL
+# Copy to your pipeline directory as load.yaml
+#
+# Write methods:
+#   - insert: Insert only (fails on duplicates)
+#   - upsert: Insert or update by primary key (default)
+#   - replace: Delete and insert
+#   - truncate_and_load: Truncate table then insert
+#   - update: Update existing records only
+#   - delete: Delete matching records
+
+title: load_postgresql
+description: Load data into PostgreSQL
+version: "1.0.0"
+
+# Target type (required)
+type: postgres
+
+# Target table configuration
+table: my_table
+schema: public
+write_method: upsert
+primary_key: id  # or ["col1", "col2"] for composite key
+batch_size: 1000
+
+# Database connection
+database:
+  url: ${DATABASE_URL:?DATABASE_URL is required}
+  # Format: postgresql://user:password@host:port/database
+  # Async: postgresql+asyncpg://user:password@host:port/database
+
+# SSH Tunnel (optional)
+# ssh_tunnel:
+#   enabled: true
+#   host: bastion.example.com
+#   port: 22
+#   username: ssh_user
+#   key_file: ~/.ssh/id_rsa
+#   local_port: 5433

pycharter/data/templates/etl/load_sqlite.yaml

@@ -0,0 +1,21 @@
+# Template: Load to SQLite
+# Copy to your pipeline directory as load.yaml
+#
+# Good for: Local development, testing, small datasets
+
+title: load_sqlite
+description: Load data into SQLite
+version: "1.0.0"
+
+# Target type (required)
+type: sqlite
+
+table: my_table
+write_method: upsert
+primary_key: id
+batch_size: 1000
+
+database:
+  url: sqlite:///./data/my_database.db
+  # Or use environment variable:
+  # url: ${SQLITE_DB_PATH:-sqlite:///./data/local.db}

pycharter/data/templates/etl/load_truncate_and_load.yaml

@@ -0,0 +1,20 @@
+# Template: Truncate and Load
+# Clears the table before loading (full refresh)
+# Copy to your pipeline directory as load.yaml
+
+title: load_truncate_and_load
+description: Truncate table then insert all records
+version: "1.0.0"
+
+# Target type (required)
+type: postgres
+
+table: my_table
+schema: public
+write_method: truncate_and_load
+batch_size: 1000
+
+database:
+  url: ${DATABASE_URL:?DATABASE_URL is required}
+
+# Warning: This deletes all existing data before loading!

pycharter/data/templates/etl/load_upsert.yaml

@@ -0,0 +1,25 @@
+# Template: Load with Upsert
+# Insert new records, update existing ones based on primary key
+# Copy to your pipeline directory as load.yaml
+
+title: load_upsert
+description: Upsert (insert or update) into database
+version: "1.0.0"
+
+# Target type (required)
+type: postgres
+
+table: my_table
+schema: public
+write_method: upsert
+primary_key: id  # or ["col1", "col2"] for composite key
+batch_size: 1000
+
+database:
+  url: ${DATABASE_URL:?DATABASE_URL is required}
+
+# Optional: specify which columns to update on conflict
+# update_columns:
+#   - name
+#   - updated_at
+#   - status

pycharter/data/templates/etl/load_with_dlq.yaml

@@ -0,0 +1,34 @@
+# Template: Load with Dead Letter Queue
+# Copy to your pipeline directory as load.yaml
+# Failed records are written to a DLQ table/file for later processing
+
+title: load_with_dlq
+description: Load with dead letter queue for failed records
+version: "1.0.0"
+
+# Target type (required)
+type: postgres
+
+table: my_table
+schema: public
+write_method: upsert
+primary_key: id
+batch_size: 1000
+
+database:
+  url: ${DATABASE_URL}
+
+# Dead Letter Queue configuration
+# Note: Use 'dead_letter_queue' as the key (not 'dlq')
+dead_letter_queue:
+  enabled: true
+
+  # DLQ backend type: "database", "file", or "memory"
+  backend: database
+
+  # Schema for DLQ table (optional, defaults to 'public')
+  schema_name: public
+
+  # For file-based DLQ, specify storage_path:
+  # backend: file
+  # storage_path: ./dlq/my_table_failures.jsonl

pycharter/data/templates/etl/load_with_ssh_tunnel.yaml

@@ -0,0 +1,35 @@
+# Template: Load to Database via SSH Tunnel
+# Copy to your pipeline directory as load.yaml
+# Use when database is only accessible through a bastion/jump host
+
+title: load_ssh_tunnel
+description: Load to database through SSH tunnel
+version: "1.0.0"
+
+# Target type (required)
+type: postgres
+
+table: my_table
+schema: public
+write_method: upsert
+primary_key: id
+batch_size: 1000
+
+# Database connection (use localhost since tunnel forwards)
+database:
+  url: postgresql://user:pass@localhost:5433/database
+
+# SSH tunnel configuration
+ssh_tunnel:
+  enabled: true
+  host: bastion.example.com
+  port: 22
+  username: ${SSH_USER}
+  key_file: ~/.ssh/id_rsa
+
+  # Remote database endpoint
+  remote_host: db.internal.example.com
+  remote_port: 5432
+
+  # Local port for tunnel
+  local_port: 5433

pycharter/data/templates/etl/pipeline_http_to_db.yaml

@@ -0,0 +1,75 @@
+# Template: Complete HTTP-to-Database Pipeline
+# Single-file format: extract, transform, load in one file
+# Alternative: Use separate extract.yaml, transform.yaml, load.yaml files
+
+name: http_to_db_pipeline
+version: "1.0.0"
+description: Complete ETL pipeline example
+
+# =============================================================================
+# EXTRACT: HTTP API
+# =============================================================================
+extract:
+  type: http  # Required: http | file | database | cloud_storage
+  url: https://api.example.com/v1/users
+  method: GET
+  params:
+    api_key: ${API_KEY}
+    limit: 100
+  headers:
+    Accept: application/json
+  batch_size: 1000
+  response_path: data.users
+  pagination:
+    enabled: true
+    strategy: page
+    page:
+      param_name: page
+      start: 1
+    stop_conditions:
+      - type: fewer_records
+
+# =============================================================================
+# TRANSFORM: Ordered list of operations (new format)
+# =============================================================================
+transform:
+  # List format for explicit ordering
+  - rename:
+      userId: user_id
+      firstName: first_name
+      lastName: last_name
+      createdAt: created_at
+
+  - convert:
+      user_id: int
+      created_at: datetime
+
+  - defaults:
+      status: "active"
+
+  - add:
+      full_name: "${first_name} ${last_name}"
+      loaded_at: now()
+
+  - select:
+      - user_id
+      - first_name
+      - last_name
+      - full_name
+      - email
+      - status
+      - created_at
+      - loaded_at
+
+# =============================================================================
+# LOAD: PostgreSQL
+# =============================================================================
+load:
+  type: postgres  # Required: postgres | sqlite | file | cloud_storage
+  table: users
+  schema: public
+  write_method: upsert
+  primary_key: user_id
+  batch_size: 1000
+  database:
+    url: ${DATABASE_URL}

pycharter/data/templates/etl/transform_combined.yaml

@@ -0,0 +1,48 @@
+# Template: Combined Transformations
+# Apply simple ops, then JSONata, then custom function
+# Copy to your pipeline directory as transform.yaml
+
+title: combined_transformation
+description: Simple operations + JSONata + custom function
+version: "1.0.0"
+
+# Step 1: Simple operations (applied first)
+transform:
+  rename:
+    userId: user_id
+    fullName: full_name
+
+  convert:
+    score: float
+    count: int
+
+  defaults:
+    source: "api"
+
+  add:
+    loaded_at: "now()"
+
+  select:
+    - user_id
+    - full_name
+    - score
+    - source
+    - loaded_at
+
+# Step 2: JSONata expressions (applied after simple ops)
+# Requires: pip install pyjsonata
+jsonata:
+  # Mode: record (per-record) or batch (all records)
+  mode: record
+  expression: |
+    $ ~> |$|{
+      "score_pct": score * 100,
+      "name_upper": $uppercase(full_name)
+    }|
+
+# Step 3: Custom function (applied last)
+# custom_function:
+#   module: mypackage.transforms
+#   function: enrich_records
+#   kwargs:
+#     add_metadata: true

pycharter/data/templates/etl/transform_custom_function.yaml

@@ -0,0 +1,58 @@
+# Template: Custom Function Transformation
+# Copy to your pipeline directory as transform.yaml
+#
+# Calls a Python function for complex transformations
+#
+# Your function signature:
+#   def my_transform(records: List[Dict], **kwargs) -> List[Dict]:
+#       return [transform_record(r) for r in records]
+
+title: custom_function_transformation
+description: Call custom Python function for transformation
+version: "1.0.0"
+
+transform:
+  # Simple operations applied first (optional)
+  rename:
+    userId: user_id
+
+  # Custom function called after simple operations
+  custom_function:
+    # Python module path (relative to working directory or installed package)
+    module: mypackage.transforms
+    # Function name to call
+    function: transform_records
+    # Optional keyword arguments passed to function
+    kwargs:
+      validate: true
+      enrich: true
+      default_region: "US"
+
+# Example custom function implementation:
+#
+# # mypackage/transforms.py
+# from typing import Any, Dict, List
+#
+# def transform_records(
+#     records: List[Dict[str, Any]],
+#     validate: bool = False,
+#     enrich: bool = False,
+#     default_region: str = "US",
+# ) -> List[Dict[str, Any]]:
+#     """Transform a batch of records."""
+#     result = []
+#     for record in records:
+#         # Add region if missing
+#         if "region" not in record:
+#             record["region"] = default_region
+#
+#         # Enrich with computed fields
+#         if enrich:
+#             record["full_name"] = f"{record.get('first', '')} {record.get('last', '')}".strip()
+#
+#         # Validate and skip invalid records
+#         if validate and not record.get("id"):
+#             continue
+#
+#         result.append(record)
+#     return result