pycharter 0.0.25__py3-none-any.whl → 0.0.26__py3-none-any.whl

pycharter/etl_generator/schemas/extract.json

@@ -1,234 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "$id": "https://pycharter.io/schemas/etl/extract.json",
-  "title": "ETL Extract Configuration",
-  "description": "Configuration for ETL data extraction",
-  "type": "object",
-  "required": ["type"],
-  "properties": {
-    "type": {
-      "type": "string",
-      "enum": ["http", "file", "database", "cloud_storage"],
-      "description": "Type of data source"
-    },
-    "batch_size": {
-      "type": "integer",
-      "minimum": 1,
-      "default": 1000,
-      "description": "Number of records per batch"
-    }
-  },
-  "allOf": [
-    {
-      "if": {
-        "properties": { "type": { "const": "http" } }
-      },
-      "then": {
-        "properties": {
-          "url": {
-            "type": "string",
-            "format": "uri",
-            "description": "Full URL for the HTTP request"
-          },
-          "base_url": {
-            "type": "string",
-            "format": "uri",
-            "description": "Base URL (used with endpoint)"
-          },
-          "endpoint": {
-            "type": "string",
-            "description": "API endpoint path (used with base_url)"
-          },
-          "method": {
-            "type": "string",
-            "enum": ["GET", "POST", "PUT", "PATCH", "DELETE"],
-            "default": "GET",
-            "description": "HTTP method"
-          },
-          "headers": {
-            "type": "object",
-            "additionalProperties": { "type": "string" },
-            "description": "HTTP headers"
-          },
-          "params": {
-            "type": "object",
-            "description": "Query parameters"
-          },
-          "body": {
-            "type": "object",
-            "description": "Request body (for POST/PUT/PATCH)"
-          },
-          "response_format": {
-            "type": "string",
-            "enum": ["json", "text", "xml"],
-            "default": "json",
-            "description": "Expected response format"
-          },
-          "response_path": {
-            "type": "string",
-            "description": "JSONPath to data array in response"
-          },
-          "pagination": {
-            "type": "object",
-            "properties": {
-              "enabled": { "type": "boolean", "default": false },
-              "strategy": {
-                "type": "string",
-                "enum": ["page", "offset", "cursor", "link"],
-                "description": "Pagination strategy"
-              },
-              "page_param": { "type": "string" },
-              "offset_param": { "type": "string" },
-              "limit_param": { "type": "string" },
-              "cursor_param": { "type": "string" },
-              "cursor_path": { "type": "string" },
-              "next_link_path": { "type": "string" },
-              "page_size": { "type": "integer", "minimum": 1 },
-              "max_pages": { "type": "integer", "minimum": 1 }
-            }
-          },
-          "retry": {
-            "type": "object",
-            "properties": {
-              "max_attempts": { "type": "integer", "minimum": 1, "default": 3 },
-              "backoff_factor": { "type": "number", "minimum": 0, "default": 2 },
-              "retry_on_status": {
-                "type": "array",
-                "items": { "type": "integer" },
-                "default": [429, 500, 502, 503, 504]
-              }
-            }
-          },
-          "timeout": {
-            "type": "object",
-            "properties": {
-              "connect": { "type": "number", "minimum": 0 },
-              "read": { "type": "number", "minimum": 0 },
-              "write": { "type": "number", "minimum": 0 }
-            }
-          },
-          "rate_limit_delay": {
-            "type": "number",
-            "minimum": 0,
-            "description": "Delay between requests in seconds"
-          }
-        },
-        "anyOf": [
-          { "required": ["url"] },
-          { "required": ["base_url", "endpoint"] }
-        ]
-      }
-    },
-    {
-      "if": {
-        "properties": { "type": { "const": "file" } }
-      },
-      "then": {
-        "required": ["path"],
-        "properties": {
-          "path": {
-            "type": "string",
-            "description": "File path (supports glob patterns)"
-          },
-          "format": {
-            "type": "string",
-            "enum": ["csv", "json", "jsonl", "parquet", "excel", "tsv", "xml"],
-            "description": "File format (auto-detected if not specified)"
-          },
-          "csv_options": {
-            "type": "object",
-            "properties": {
-              "delimiter": { "type": "string", "default": "," },
-              "quote_char": { "type": "string", "default": "\"" },
-              "escape_char": { "type": "string" },
-              "has_header": { "type": "boolean", "default": true },
-              "encoding": { "type": "string", "default": "utf-8" }
-            }
-          },
-          "json_options": {
-            "type": "object",
-            "properties": {
-              "encoding": { "type": "string", "default": "utf-8" },
-              "lines": { "type": "boolean", "default": false }
-            }
-          }
-        }
-      }
-    },
-    {
-      "if": {
-        "properties": { "type": { "const": "database" } }
-      },
-      "then": {
-        "required": ["query"],
-        "properties": {
-          "query": {
-            "type": "string",
-            "description": "SQL query to execute"
-          },
-          "connection_string": {
-            "type": "string",
-            "description": "Database connection URL"
-          },
-          "database": {
-            "type": "object",
-            "properties": {
-              "url": { "type": "string" }
-            }
-          },
-          "params": {
-            "type": "object",
-            "description": "Query parameter bindings"
-          },
-          "ssh_tunnel": {
-            "type": "object",
-            "properties": {
-              "host": { "type": "string" },
-              "port": { "type": "integer", "default": 22 },
-              "username": { "type": "string" },
-              "private_key_path": { "type": "string" },
-              "remote_host": { "type": "string" },
-              "remote_port": { "type": "integer" }
-            },
-            "required": ["host", "username"]
-          }
-        },
-        "anyOf": [
-          { "required": ["connection_string"] },
-          { "required": ["database"] }
-        ]
-      }
-    },
-    {
-      "if": {
-        "properties": { "type": { "const": "cloud_storage" } }
-      },
-      "then": {
-        "required": ["storage"],
-        "properties": {
-          "storage": {
-            "type": "object",
-            "required": ["provider"],
-            "properties": {
-              "provider": {
-                "type": "string",
-                "enum": ["s3", "gcs", "azure"],
-                "description": "Cloud storage provider"
-              },
-              "bucket": { "type": "string" },
-              "container": { "type": "string" },
-              "path": { "type": "string" },
-              "prefix": { "type": "string" },
-              "credentials": { "type": "object" }
-            }
-          },
-          "format": {
-            "type": "string",
-            "enum": ["csv", "json", "jsonl", "parquet"],
-            "description": "File format"
-          }
-        }
-      }
-    }
-  ]
-}

pycharter/etl_generator/schemas/load.json

@@ -1,202 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "$id": "https://pycharter.io/schemas/etl/load.json",
-  "title": "ETL Load Configuration",
-  "description": "Configuration for ETL data loading",
-  "type": "object",
-  "required": ["type"],
-  "properties": {
-    "type": {
-      "type": "string",
-      "enum": ["postgres", "postgresql", "sqlite", "database", "file", "cloud_storage"],
-      "description": "Type of data destination"
-    },
-    "batch_size": {
-      "type": "integer",
-      "minimum": 1,
-      "default": 1000,
-      "description": "Number of records per batch"
-    }
-  },
-  "allOf": [
-    {
-      "if": {
-        "properties": { 
-          "type": { "enum": ["postgres", "postgresql", "database"] }
-        }
-      },
-      "then": {
-        "required": ["table"],
-        "properties": {
-          "table": {
-            "type": "string",
-            "description": "Target table name"
-          },
-          "schema": {
-            "type": "string",
-            "default": "public",
-            "description": "Database schema"
-          },
-          "write_method": {
-            "type": "string",
-            "enum": ["insert", "upsert", "replace", "update", "delete", "truncate_and_load"],
-            "default": "insert",
-            "description": "Write method"
-          },
-          "primary_key": {
-            "oneOf": [
-              { "type": "string" },
-              { "type": "array", "items": { "type": "string" } }
-            ],
-            "description": "Primary key column(s) for upsert/update"
-          },
-          "update_columns": {
-            "type": "array",
-            "items": { "type": "string" },
-            "description": "Columns to update on conflict (upsert)"
-          },
-          "connection_string": {
-            "type": "string",
-            "description": "Database connection URL"
-          },
-          "database": {
-            "type": "object",
-            "properties": {
-              "url": { "type": "string" }
-            }
-          },
-          "ssh_tunnel": {
-            "type": "object",
-            "properties": {
-              "host": { "type": "string" },
-              "port": { "type": "integer", "default": 22 },
-              "username": { "type": "string" },
-              "private_key_path": { "type": "string" },
-              "remote_host": { "type": "string" },
-              "remote_port": { "type": "integer" }
-            },
-            "required": ["host", "username"]
-          },
-          "dead_letter_queue": {
-            "type": "object",
-            "properties": {
-              "enabled": { "type": "boolean", "default": false },
-              "table": { "type": "string" },
-              "path": { "type": "string" }
-            }
-          }
-        }
-      }
-    },
-    {
-      "if": {
-        "properties": { "type": { "const": "sqlite" } }
-      },
-      "then": {
-        "required": ["table", "path"],
-        "properties": {
-          "table": {
-            "type": "string",
-            "description": "Target table name"
-          },
-          "path": {
-            "type": "string",
-            "description": "SQLite database file path"
-          },
-          "write_method": {
-            "type": "string",
-            "enum": ["insert", "upsert", "replace", "truncate_and_load"],
-            "default": "insert"
-          },
-          "primary_key": {
-            "oneOf": [
-              { "type": "string" },
-              { "type": "array", "items": { "type": "string" } }
-            ]
-          }
-        }
-      }
-    },
-    {
-      "if": {
-        "properties": { "type": { "const": "file" } }
-      },
-      "then": {
-        "required": ["path"],
-        "properties": {
-          "path": {
-            "type": "string",
-            "description": "Output file path"
-          },
-          "format": {
-            "type": "string",
-            "enum": ["json", "jsonl", "csv", "parquet"],
-            "description": "Output file format (auto-detected if not specified)"
-          },
-          "write_mode": {
-            "type": "string",
-            "enum": ["overwrite", "append"],
-            "default": "overwrite",
-            "description": "File write mode"
-          },
-          "csv_options": {
-            "type": "object",
-            "properties": {
-              "delimiter": { "type": "string", "default": "," },
-              "quote_char": { "type": "string", "default": "\"" },
-              "include_header": { "type": "boolean", "default": true },
-              "encoding": { "type": "string", "default": "utf-8" }
-            }
-          },
-          "json_options": {
-            "type": "object",
-            "properties": {
-              "indent": { "type": "integer" },
-              "ensure_ascii": { "type": "boolean", "default": false }
-            }
-          }
-        }
-      }
-    },
-    {
-      "if": {
-        "properties": { "type": { "const": "cloud_storage" } }
-      },
-      "then": {
-        "required": ["storage"],
-        "properties": {
-          "storage": {
-            "type": "object",
-            "required": ["provider"],
-            "properties": {
-              "provider": {
-                "type": "string",
-                "enum": ["s3", "gcs", "azure"],
-                "description": "Cloud storage provider"
-              },
-              "bucket": { "type": "string" },
-              "container": { "type": "string" },
-              "path": { "type": "string" },
-              "credentials": { "type": "object" }
-            }
-          },
-          "format": {
-            "type": "string",
-            "enum": ["json", "jsonl", "csv", "parquet"],
-            "description": "Output file format"
-          },
-          "partition_by": {
-            "type": "array",
-            "items": { "type": "string" },
-            "description": "Columns to partition by"
-          },
-          "write_mode": {
-            "type": "string",
-            "enum": ["overwrite", "append"],
-            "default": "overwrite"
-          }
-        }
-      }
-    }
-  ]
-}

pycharter/etl_generator/schemas/pipeline.json

@@ -1,94 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-07/schema#",
-  "$id": "https://pycharter.io/schemas/etl/pipeline.json",
-  "title": "ETL Pipeline Configuration",
-  "description": "Complete ETL pipeline configuration (single-file format)",
-  "type": "object",
-  "required": ["extract", "load"],
-  "properties": {
-    "name": {
-      "type": "string",
-      "description": "Pipeline name"
-    },
-    "version": {
-      "type": "string",
-      "description": "Pipeline version"
-    },
-    "description": {
-      "type": "string",
-      "description": "Pipeline description"
-    },
-    "extract": {
-      "$ref": "extract.json",
-      "description": "Extract configuration"
-    },
-    "transform": {
-      "description": "Transform configuration - can be a list (ordered) or object (legacy)",
-      "oneOf": [
-        {
-          "type": "array",
-          "description": "List of transform operations (executed in order)",
-          "items": {
-            "type": "object",
-            "minProperties": 1,
-            "maxProperties": 1,
-            "properties": {
-              "rename": {
-                "type": "object",
-                "additionalProperties": { "type": "string" }
-              },
-              "convert": {
-                "type": "object",
-                "additionalProperties": { "type": "string" }
-              },
-              "defaults": {
-                "type": "object"
-              },
-              "add": {
-                "type": "object"
-              },
-              "select": {
-                "type": "array",
-                "items": { "type": "string" }
-              },
-              "drop": {
-                "type": "array",
-                "items": { "type": "string" }
-              },
-              "filter": {
-                "type": "object"
-              },
-              "custom_function": {
-                "type": "object",
-                "required": ["module", "function"],
-                "properties": {
-                  "module": { "type": "string" },
-                  "function": { "type": "string" },
-                  "kwargs": { "type": "object" }
-                }
-              }
-            }
-          }
-        },
-        {
-          "type": "object",
-          "description": "Legacy object format",
-          "properties": {
-            "rename": { "type": "object" },
-            "convert": { "type": "object" },
-            "defaults": { "type": "object" },
-            "add": { "type": "object" },
-            "select": { "type": "array" },
-            "drop": { "type": "array" },
-            "filter": { "type": "object" },
-            "custom_function": { "type": "object" }
-          }
-        }
-      ]
-    },
-    "load": {
-      "$ref": "load.json",
-      "description": "Load configuration"
-    }
-  }
-}