pycharter 0.0.22__py3-none-any.whl → 0.0.24__py3-none-any.whl

pycharter/docs_generator/renderers.py

@@ -0,0 +1,247 @@
+"""
+Documentation Renderers - Output format implementations.
+
+Provides Protocol definition and concrete implementations for
+rendering documentation in various formats.
+"""
+
+from typing import Any, Dict, List, Optional, Protocol
+
+
+class DocsRenderer(Protocol):
+    """Protocol for documentation renderers."""
+
+    def render_header(self, title: str, version: Optional[str] = None) -> str:
+        """Render document header with title and optional version."""
+        ...
+
+    def render_section(self, title: str, content: str) -> str:
+        """Render a section with title and content."""
+        ...
+
+    def render_table(
+        self, headers: List[str], rows: List[List[str]], caption: Optional[str] = None
+    ) -> str:
+        """Render a table with headers and rows."""
+        ...
+
+    def render_code_block(self, code: str, language: str = "") -> str:
+        """Render a code block."""
+        ...
+
+    def render_list(self, items: List[str], ordered: bool = False) -> str:
+        """Render a list of items."""
+        ...
+
+    def render_description(self, text: str) -> str:
+        """Render description/paragraph text."""
+        ...
+
+    def render_badge(self, label: str, value: str) -> str:
+        """Render a badge/tag (e.g., required, optional)."""
+        ...
+
+
+class MarkdownRenderer:
+    """Render documentation as Markdown."""
+
+    def render_header(self, title: str, version: Optional[str] = None) -> str:
+        """Render document header with title and optional version."""
+        header = f"# {title}\n\n"
+        if version:
+            header += f"**Version:** `{version}`\n\n"
+        return header
+
+    def render_section(self, title: str, content: str) -> str:
+        """Render a section with title and content."""
+        return f"## {title}\n\n{content}\n\n"
+
+    def render_subsection(self, title: str, content: str) -> str:
+        """Render a subsection with title and content."""
+        return f"### {title}\n\n{content}\n\n"
+
+    def render_table(
+        self, headers: List[str], rows: List[List[str]], caption: Optional[str] = None
+    ) -> str:
+        """Render a Markdown table."""
+        if not headers or not rows:
+            return ""
+
+        lines = []
+        if caption:
+            lines.append(f"*{caption}*\n")
+
+        # Header row
+        lines.append("| " + " | ".join(headers) + " |")
+        # Separator
+        lines.append("| " + " | ".join(["---"] * len(headers)) + " |")
+        # Data rows
+        for row in rows:
+            # Ensure row has same length as headers
+            padded_row = row + [""] * (len(headers) - len(row))
+            lines.append("| " + " | ".join(padded_row[: len(headers)]) + " |")
+
+        return "\n".join(lines) + "\n"
+
+    def render_code_block(self, code: str, language: str = "") -> str:
+        """Render a fenced code block."""
+        return f"```{language}\n{code}\n```\n"
+
+    def render_list(self, items: List[str], ordered: bool = False) -> str:
+        """Render a list of items."""
+        if not items:
+            return ""
+        if ordered:
+            return "\n".join(f"{i+1}. {item}" for i, item in enumerate(items)) + "\n"
+        return "\n".join(f"- {item}" for item in items) + "\n"
+
+    def render_description(self, text: str) -> str:
+        """Render description/paragraph text."""
+        return f"{text}\n\n"
+
+    def render_badge(self, label: str, value: str) -> str:
+        """Render a badge as inline code."""
+        return f"`{label}: {value}`"
+
+    def render_key_value(self, key: str, value: Any) -> str:
+        """Render a key-value pair."""
+        if isinstance(value, (dict, list)):
+            return f"**{key}:**\n{self.render_code_block(str(value), 'json')}"
+        return f"**{key}:** {value}\n"
+
+
+class HTMLRenderer:
+    """Render documentation as HTML."""
+
+    def __init__(self, include_styles: bool = True):
+        """
+        Initialize HTML renderer.
+
+        Args:
+            include_styles: If True, include inline CSS styles
+        """
+        self.include_styles = include_styles
+
+    def _get_styles(self) -> str:
+        """Get CSS styles for HTML output."""
+        return """
+<style>
+    body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; 
+           line-height: 1.6; max-width: 900px; margin: 0 auto; padding: 20px; }
+    h1 { border-bottom: 2px solid #333; padding-bottom: 10px; }
+    h2 { border-bottom: 1px solid #ddd; padding-bottom: 5px; margin-top: 30px; }
+    table { border-collapse: collapse; width: 100%; margin: 15px 0; }
+    th, td { border: 1px solid #ddd; padding: 10px; text-align: left; }
+    th { background: #f5f5f5; font-weight: 600; }
+    tr:nth-child(even) { background: #fafafa; }
+    code { background: #f4f4f4; padding: 2px 6px; border-radius: 3px; font-size: 0.9em; }
+    pre { background: #f4f4f4; padding: 15px; border-radius: 5px; overflow-x: auto; }
+    pre code { background: none; padding: 0; }
+    .badge { display: inline-block; padding: 2px 8px; border-radius: 3px; 
+             font-size: 0.8em; font-weight: 500; }
+    .badge-required { background: #fee2e2; color: #991b1b; }
+    .badge-optional { background: #e0f2fe; color: #0369a1; }
+    .version { color: #666; font-size: 0.9em; }
+</style>
+"""
+
+    def render_header(self, title: str, version: Optional[str] = None) -> str:
+        """Render document header with title and optional version."""
+        styles = self._get_styles() if self.include_styles else ""
+        version_html = (
+            f'<p class="version">Version: <code>{version}</code></p>'
+            if version
+            else ""
+        )
+        return f"""<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="UTF-8">
+    <title>{title}</title>
+    {styles}
+</head>
+<body>
+<h1>{title}</h1>
+{version_html}
+"""
+
+    def render_footer(self) -> str:
+        """Render document footer."""
+        return "</body>\n</html>"
+
+    def render_section(self, title: str, content: str) -> str:
+        """Render a section with title and content."""
+        return f"<h2>{title}</h2>\n{content}\n"
+
+    def render_subsection(self, title: str, content: str) -> str:
+        """Render a subsection with title and content."""
+        return f"<h3>{title}</h3>\n{content}\n"
+
+    def render_table(
+        self, headers: List[str], rows: List[List[str]], caption: Optional[str] = None
+    ) -> str:
+        """Render an HTML table."""
+        if not headers or not rows:
+            return ""
+
+        lines = ["<table>"]
+        if caption:
+            lines.append(f"<caption>{caption}</caption>")
+
+        # Header
+        lines.append("<thead><tr>")
+        for h in headers:
+            lines.append(f"<th>{h}</th>")
+        lines.append("</tr></thead>")
+
+        # Body
+        lines.append("<tbody>")
+        for row in rows:
+            lines.append("<tr>")
+            for i, cell in enumerate(row):
+                if i < len(headers):
+                    lines.append(f"<td>{cell}</td>")
+            # Pad if needed
+            for _ in range(len(headers) - len(row)):
+                lines.append("<td></td>")
+            lines.append("</tr>")
+        lines.append("</tbody>")
+        lines.append("</table>")
+
+        return "\n".join(lines) + "\n"
+
+    def render_code_block(self, code: str, language: str = "") -> str:
+        """Render a code block."""
+        return f"<pre><code>{self._escape_html(code)}</code></pre>\n"
+
+    def render_list(self, items: List[str], ordered: bool = False) -> str:
+        """Render a list of items."""
+        if not items:
+            return ""
+        tag = "ol" if ordered else "ul"
+        items_html = "\n".join(f"<li>{item}</li>" for item in items)
+        return f"<{tag}>\n{items_html}\n</{tag}>\n"
+
+    def render_description(self, text: str) -> str:
+        """Render description/paragraph text."""
+        return f"<p>{text}</p>\n"
+
+    def render_badge(self, label: str, value: str) -> str:
+        """Render a badge span."""
+        css_class = "badge-required" if value.lower() == "required" else "badge-optional"
+        return f'<span class="badge {css_class}">{label}: {value}</span>'
+
+    def render_key_value(self, key: str, value: Any) -> str:
+        """Render a key-value pair."""
+        if isinstance(value, (dict, list)):
+            return f"<p><strong>{key}:</strong></p>\n{self.render_code_block(str(value), 'json')}"
+        return f"<p><strong>{key}:</strong> {value}</p>\n"
+
+    def _escape_html(self, text: str) -> str:
+        """Escape HTML special characters."""
+        return (
+            text.replace("&", "&amp;")
+            .replace("<", "&lt;")
+            .replace(">", "&gt;")
+            .replace('"', "&quot;")
+        )

pycharter/etl_generator/__init__.py

@@ -1,94 +1,182 @@
 """
-ETL Orchestrator - Runtime ETL pipeline execution from contract artifacts.
+ETL Generator - Modern ETL pipeline framework.
 
-PRIMARY INTERFACE: ETLOrchestrator Class
-========================================
+Two APIs:
+=========
+1. Programmatic API (Pipeline with | operator):
+   >>> pipeline = Pipeline(HTTPExtractor(url="...")) | Rename(...) | PostgresLoader(...)
+   >>> result = await pipeline.run()
 
-The ETLOrchestrator class is the recommended and primary way to execute ETL pipelines
-in pycharter. It can be instantiated with contract artifacts from various sources:
+2. Config-driven API (YAML files):
+   >>> # From explicit files (most flexible)
+   >>> pipeline = Pipeline.from_config_files(
+   ...     extract="configs/extract.yaml",
+   ...     load="configs/load.yaml",
+   ...     variables={"API_KEY": "secret"}
+   ... )
+   >>> 
+   >>> # From directory (expects extract.yaml, transform.yaml, load.yaml)
+   >>> pipeline = Pipeline.from_config_dir("pipelines/users/")
+   >>> 
+   >>> # From single file (pipeline.yaml)
+   >>> pipeline = Pipeline.from_config_file("pipelines/users/pipeline.yaml")
 
-    >>> from pycharter.etl_generator import ETLOrchestrator
-    >>> 
-    >>> # From contract directory (contains schema, coercion_rules, validation_rules, extract, transform, load)
-    >>> orchestrator = ETLOrchestrator(contract_dir="data/contracts/user")
-    >>> await orchestrator.run()
-    >>> 
-    >>> # From contract file
-    >>> orchestrator = ETLOrchestrator(contract_file="contracts/user.yaml")
-    >>> await orchestrator.run(dry_run=False)
-    >>> 
-    >>> # With direct ETL config dictionaries (overrides directory configs)
-    >>> orchestrator = ETLOrchestrator(
-    ...     contract_dir="data/contracts/user",
-    ...     extract_config={"base_url": "https://api.example.com", "api_endpoint": "/v1/data"},
-    ...     transform_config={"rename": {"oldName": "new_name"}},
-    ...     load_config={"target_table": "users"}
-    ... )
-    >>> await orchestrator.run()
-    >>> 
-    >>> # With separate ETL config file paths (overrides directory configs)
-    >>> orchestrator = ETLOrchestrator(
-    ...     contract_dir="data/contracts/user",
-    ...     extract_file="custom/extract.yaml",
-    ...     transform_file="custom/transform.yaml",
-    ...     load_file="custom/load.yaml"
-    ... )
-    >>> await orchestrator.run()
-    >>> 
-    >>> # Mixed approach - override specific configs
-    >>> orchestrator = ETLOrchestrator(
-    ...     contract_dir="data/contracts/user",
-    ...     extract_config={"base_url": "https://custom.api.com"}  # Override extract only
-    ...     # transform.yaml and load.yaml loaded from contract_dir
-    ... )
-    >>> await orchestrator.run()
+Quick Start:
+============
+    from pycharter.etl_generator import (
+        Pipeline, HTTPExtractor, PostgresLoader,
+        Rename, AddField, Filter
+    )
+    
+    # Programmatic
+    pipeline = (
+        Pipeline(HTTPExtractor(url="https://api.example.com/users"))
+        | Rename({"user_name": "name"})
+        | AddField("full_name", "${first_name} ${last_name}")
+        | PostgresLoader(connection_string="...", table="users")
+    )
+    result = await pipeline.run()
+    
+    # Config-driven (explicit files)
+    pipeline = Pipeline.from_config_files(
+        extract="pipelines/users/extract.yaml",
+        load="pipelines/users/load.yaml",
+        variables={"DATA_DIR": "./data"}
+    )
+    result = await pipeline.run()
 
-Core Components:
-- orchestrator: ETLOrchestrator class (PRIMARY INTERFACE - use this for all ETL execution)
-- config_generator: Utility functions for generating ETL configs from contracts (helper utilities)
+Config Format:
+==============
+    Single-file (pipeline.yaml):
+        name: users_pipeline
+        extract:
+          type: http  # Required: http | file | database | cloud_storage
+          url: https://api.example.com/users
+        transform:
+          - rename: {userId: user_id}
+          - add:
+              full_name: "${first_name} ${last_name}"
+        load:
+          type: postgres  # Required: postgres | sqlite | file | cloud_storage
+          table: users
+          database:
+            url: ${DATABASE_URL}
 """
 
-from pycharter.etl_generator.checkpoint import (
-    CheckpointManager,
-    CheckpointState,
+# Core pipeline classes
+from pycharter.etl_generator.pipeline import Pipeline
+from pycharter.etl_generator.builder import PipelineBuilder
+from pycharter.etl_generator.context import PipelineContext
+from pycharter.etl_generator.result import PipelineResult, BatchResult, LoadResult
+from pycharter.etl_generator.protocols import Extractor, Transformer, Loader
+
+# Config loading and validation
+from pycharter.etl_generator.config_loader import (
+    ConfigLoader,
+    PipelineConfig,
+    load_pipeline_config,
+    ConfigLoadError,
 )
-from pycharter.etl_generator.config_generator import (
-    generate_etl_config,
-    generate_etl_config_from_contract,
-    generate_etl_config_from_store,
+from pycharter.etl_generator.config_validator import (
+    ConfigValidator,
+    ConfigValidationError,
+    validate_config,
 )
-from pycharter.etl_generator.dlq import (
-    DeadLetterQueue,
-    DeadLetterRecord,
-    DLQReason,
+
+# Expression evaluation
+from pycharter.etl_generator.expression import (
+    ExpressionEvaluator,
+    ExpressionError,
+    evaluate_expression,
+    is_expression,
 )
-from pycharter.etl_generator.orchestrator import (
-    ETLOrchestrator,
-    create_orchestrator,
+
+# Extractors
+from pycharter.etl_generator.extractors import (
+    BaseExtractor,
+    HTTPExtractor,
+    FileExtractor,
+    DatabaseExtractor,
+    CloudStorageExtractor,
+    ExtractorFactory,
 )
-from pycharter.etl_generator.factory import PipelineFactory
-from pycharter.etl_generator.progress import (
-    ETLProgress,
-    ProgressTracker,
+
+# Transformers
+from pycharter.etl_generator.transformers import (
+    BaseTransformer,
+    TransformerChain,
+    Rename,
+    AddField,
+    Drop,
+    Select,
+    Filter,
+    Convert,
+    Default,
+    Map,
+    FlatMap,
+    CustomFunction,
 )
+
+# Loaders
+from pycharter.etl_generator.loaders import (
+    BaseLoader,
+    PostgresLoader,
+    DatabaseLoader,
+    FileLoader,
+    CloudStorageLoader,
+    LoaderFactory,
+)
+
 __all__ = [
-    # PRIMARY INTERFACE: ETLOrchestrator class (use this for all ETL execution)
-    "ETLOrchestrator",
-    "create_orchestrator",
-    # Pipeline factory (auto-discovery of config directories)
-    "PipelineFactory",
-    # Config generation utilities (helper functions for generating ETL configs)
-    "generate_etl_config",
-    "generate_etl_config_from_contract",
-    "generate_etl_config_from_store",
-    # Progress tracking
-    "ETLProgress",
-    "ProgressTracker",
-    # Checkpoint/resume
-    "CheckpointManager",
-    "CheckpointState",
-    # Dead Letter Queue
-    "DeadLetterQueue",
-    "DeadLetterRecord",
-    "DLQReason",
+    # Core
+    "Pipeline",
+    "PipelineBuilder",
+    "PipelineContext",
+    "PipelineResult",
+    "BatchResult",
+    "LoadResult",
+    # Protocols
+    "Extractor",
+    "Transformer",
+    "Loader",
+    # Config loading
+    "ConfigLoader",
+    "PipelineConfig",
+    "load_pipeline_config",
+    "ConfigLoadError",
+    "ConfigValidator",
+    "ConfigValidationError",
+    "validate_config",
+    # Expressions
+    "ExpressionEvaluator",
+    "ExpressionError",
+    "evaluate_expression",
+    "is_expression",
+    # Extractors
+    "BaseExtractor",
+    "HTTPExtractor",
+    "FileExtractor",
+    "DatabaseExtractor",
+    "CloudStorageExtractor",
+    "ExtractorFactory",
+    # Transformers
+    "BaseTransformer",
+    "TransformerChain",
+    "Rename",
+    "AddField",
+    "Drop",
+    "Select",
+    "Filter",
+    "Convert",
+    "Default",
+    "Map",
+    "FlatMap",
+    "CustomFunction",
+    # Loaders
+    "BaseLoader",
+    "PostgresLoader",
+    "DatabaseLoader",
+    "FileLoader",
+    "CloudStorageLoader",
+    "LoaderFactory",
 ]

pycharter/etl_generator/builder.py

@@ -0,0 +1,121 @@
+"""
+Fluent PipelineBuilder API.
+"""
+
+from typing import Any, Dict, List, Optional
+
+from pycharter.etl_generator.context import PipelineContext
+from pycharter.etl_generator.pipeline import Pipeline
+from pycharter.etl_generator.protocols import Extractor, Transformer, Loader
+
+
+class PipelineBuilder:
+    """
+    Fluent API for building ETL pipelines.
+    
+    Example:
+        >>> pipeline = (
+        ...     PipelineBuilder()
+        ...     .name("user_pipeline")
+        ...     .extract_from("http", url="https://api.example.com/users")
+        ...     .transform(Rename({"old": "new"}))
+        ...     .load_to("postgres", table="users", connection_string="...")
+        ...     .build()
+        ... )
+    """
+    
+    def __init__(self):
+        self._name: Optional[str] = None
+        self._extractor: Optional[Extractor] = None
+        self._transformers: List[Transformer] = []
+        self._loader: Optional[Loader] = None
+        self._context: Optional[PipelineContext] = None
+    
+    def name(self, name: str) -> "PipelineBuilder":
+        """Set pipeline name."""
+        self._name = name
+        return self
+    
+    def with_context(self, context: PipelineContext) -> "PipelineBuilder":
+        """Set pipeline context."""
+        self._context = context
+        return self
+    
+    def extract_from(self, source_type: str, **config) -> "PipelineBuilder":
+        """Configure extractor by type."""
+        self._extractor = self._create_extractor(source_type, config)
+        return self
+    
+    def extractor(self, extractor: Extractor) -> "PipelineBuilder":
+        """Set extractor directly."""
+        self._extractor = extractor
+        return self
+    
+    def transform(self, transformer: Transformer) -> "PipelineBuilder":
+        """Add transformer to pipeline."""
+        self._transformers.append(transformer)
+        return self
+    
+    def load_to(self, target_type: str, **config) -> "PipelineBuilder":
+        """Configure loader by type."""
+        self._loader = self._create_loader(target_type, config)
+        return self
+    
+    def loader(self, loader: Loader) -> "PipelineBuilder":
+        """Set loader directly."""
+        self._loader = loader
+        return self
+    
+    def build(self) -> Pipeline:
+        """Build the pipeline."""
+        if not self._extractor:
+            raise ValueError("Extractor required. Call extract_from() or extractor()")
+        
+        return Pipeline(
+            extractor=self._extractor,
+            transformers=self._transformers,
+            loader=self._loader,
+            context=self._context or PipelineContext(),
+            name=self._name,
+        )
+    
+    def _create_extractor(self, source_type: str, config: Dict[str, Any]) -> Extractor:
+        """Create extractor from type and config."""
+        from pycharter.etl_generator.extractors import (
+            HTTPExtractor,
+            FileExtractor,
+            DatabaseExtractor,
+            CloudStorageExtractor,
+        )
+        
+        source_type = source_type.lower()
+        
+        if source_type == "http":
+            return HTTPExtractor(**config)
+        elif source_type == "file":
+            return FileExtractor(**config)
+        elif source_type in ("database", "db", "sql"):
+            return DatabaseExtractor(**config)
+        elif source_type in ("s3", "cloud", "cloud_storage"):
+            return CloudStorageExtractor(**config)
+        else:
+            raise ValueError(f"Unknown extractor type: {source_type}")
+    
+    def _create_loader(self, target_type: str, config: Dict[str, Any]) -> Loader:
+        """Create loader from type and config."""
+        from pycharter.etl_generator.loaders import (
+            PostgresLoader,
+            FileLoader,
+            CloudStorageLoader,
+        )
+        
+        target_type = target_type.lower()
+        
+        if target_type in ("postgres", "postgresql", "database", "db"):
+            return PostgresLoader(**config)
+        elif target_type == "file":
+            return FileLoader(**config)
+        elif target_type in ("s3", "cloud", "cloud_storage"):
+            return CloudStorageLoader(**config)
+        else:
+            raise ValueError(f"Unknown loader type: {target_type}")