pycharter 0.0.22__py3-none-any.whl → 0.0.23__py3-none-any.whl

pycharter/etl_generator/extraction.py

@@ -1,13 +1,14 @@
 """
-HTTP extraction utilities for ETL orchestrator.
-
-This module handles:
-- HTTP request configuration
-- Retry logic
-- Response parsing
-- Data extraction from various response formats
-- Pagination support (page, offset, cursor, next_url, link_header)
-- Streaming extraction for high-volume data
+Extraction utilities for ETL orchestrator.
+
+This module provides the main entry point for data extraction from various sources:
+- HTTP/API extraction
+- File-based extraction (CSV, JSON, Parquet, Excel, TSV, XML)
+- Database extraction (PostgreSQL, MySQL, SQLite, MSSQL, Oracle)
+- Cloud storage extraction (S3, GCS, Azure Blob)
+
+The module maintains backward compatibility with the original HTTP-only interface
+while supporting the new modular extractor architecture.
 """
 
 import asyncio
@@ -648,269 +649,53 @@ async def extract_with_pagination_streaming(
     """
     Extract data with pagination support, yielding batches for memory-efficient processing.
     
+    This is the main entry point for data extraction. It supports multiple source types:
+    - HTTP/API (default for backward compatibility)
+    - File-based (CSV, JSON, Parquet, Excel, TSV, XML)
+    - Database (PostgreSQL, MySQL, SQLite, MSSQL, Oracle)
+    - Cloud storage (S3, GCS, Azure Blob)
+    
+    The source type is auto-detected from extract_config or can be explicitly set
+    via 'source_type' field.
+    
     Yields batches as they are extracted, preventing memory exhaustion for large datasets.
     
     Args:
         extract_config: Extract configuration dictionary
-        params: Request parameters
-        headers: Request headers
+        params: Request/query parameters (source-specific)
+        headers: Request headers (source-specific, mainly for HTTP)
         contract_dir: Contract directory (for variable resolution)
         batch_size: Number of records to yield per batch
         max_records: Maximum total records to extract (None = all)
+        config_context: Optional context dictionary for value injection
     
     Yields:
         Batches of extracted records (lists of dictionaries)
-    """
-    pagination_config = extract_config.get('pagination', {})
-    
-    # If pagination is not enabled, extract all and yield in batches
-    if not pagination_config.get('enabled', False):
-        logger.info("Pagination disabled, extracting all data in single request")
-        all_data = await extract_with_retry(extract_config, params, headers, contract_dir, config_context=config_context)
-        if max_records:
-            logger.info(f"Limiting to {max_records} records (extracted {len(all_data)})")
-            all_data = all_data[:max_records]
-        
-        logger.info(f"Yielding {len(all_data)} records in batches of {batch_size}")
-        for i in range(0, len(all_data), batch_size):
-            batch = all_data[i:i + batch_size]
-            logger.debug(f"Yielding batch {i // batch_size + 1} with {len(batch)} records")
-            yield batch
-        return
-    
-    # Pagination enabled - stream pages and yield in batches
-    strategy = pagination_config.get('strategy', 'page')
-    stop_conditions = pagination_config.get('stop_conditions', [])
-    page_delay = float(pagination_config.get('page_delay', 0.1))
-    max_pages = 1000
-    max_records_from_config = None
-    
-    # Get max_pages and max_records from stop conditions
-    for condition in stop_conditions:
-        if condition.get('type') == 'max_pages':
-            max_pages = condition.get('value', 1000)
-        elif condition.get('type') == 'max_records':
-            max_records_from_config = condition.get('value')
-    
-    # Use config max_records if not provided as parameter
-    if max_records is None:
-        max_records = max_records_from_config
-    
-    current_batch = []
-    total_extracted = 0
-    page_count = 0
-    current_url = None
-    current_cursor = None
-    
-    # Initialize pagination state
-    if strategy == 'page':
-        page_config = pagination_config.get('page', {})
-        current_page = page_config.get('start', 0)
-        page_increment = page_config.get('increment', 1)
-        page_param_name = page_config.get('param_name', 'page')
-    elif strategy == 'offset':
-        offset_config = pagination_config.get('offset', {})
-        current_offset = offset_config.get('start', 0)
-        offset_param_name = offset_config.get('param_name', 'offset')
-        increment_by = offset_config.get('increment_by', 'limit')
-    elif strategy == 'cursor':
-        cursor_config = pagination_config.get('cursor', {})
-        cursor_param_name = cursor_config.get('param_name', 'cursor')
-        cursor_response_path = cursor_config.get('response_path', 'next_cursor')
-    elif strategy == 'next_url':
-        next_url_config = pagination_config.get('next_url', {})
-        next_url_response_path = next_url_config.get('response_path', 'next_url')
-    elif strategy == 'link_header':
-        pass
-    else:
-        raise ValueError(f"Unsupported pagination strategy: {strategy}")
-    
-    extract_config_copy = extract_config.copy()
-    original_endpoint = extract_config_copy.get('api_endpoint')
-    original_base_url = extract_config_copy.get('base_url', '')
     
-    logger.info(
-        f"Starting paginated extraction (strategy: {strategy}, "
-        f"max_pages: {max_pages}, batch_size: {batch_size}, "
-        f"page_delay: {page_delay}s)"
-    )
-    
-    while page_count < max_pages:
-        # Check max_records limit
-        if max_records and total_extracted >= max_records:
-            logger.info(
-                f"Reached max_records limit ({max_records}), stopping pagination "
-                f"(extracted {total_extracted} records from {page_count} pages)"
-            )
-            if current_batch:
-                yield current_batch
-            return
-        
-        # Update params/URL based on strategy
-        if strategy == 'page':
-            params[page_param_name] = current_page
-            logger.debug(f"Fetching page {current_page} (page_count: {page_count + 1}/{max_pages})")
-        elif strategy == 'offset':
-            params[offset_param_name] = current_offset
-        elif strategy == 'cursor' and current_cursor:
-            params[cursor_param_name] = current_cursor
-        elif strategy == 'next_url' and current_url:
-            extract_config_copy['api_endpoint'] = current_url
-            extract_config_copy['base_url'] = ''
-        
-        # Make request
-        need_full_response = strategy in ['cursor', 'next_url', 'link_header']
-        try:
-            logger.debug(f"Extracting page {page_count + 1} (total extracted so far: {total_extracted})")
-            page_data, full_response_data, response_obj = await _extract_single_page(
-                extract_config_copy, params, headers, contract_dir, return_full_response=need_full_response, config_context=config_context
-            )
-            logger.info(f"Page {page_count + 1} extracted: {len(page_data)} records")
-        except Exception as e:
-            logger.error(
-                f"Error extracting page {page_count + 1}",
-                extra={
-                    'page': page_count + 1,
-                    'extracted': total_extracted,
-                },
-                exc_info=True
-            )
-            # Yield what we have so far before raising
-            if current_batch:
-                yield current_batch
-            raise
-        
-        # Restore original endpoint if modified
-        if strategy == 'next_url' and current_url:
-            extract_config_copy['api_endpoint'] = original_endpoint
-            extract_config_copy['base_url'] = original_base_url
-        
-        # Check for empty page first (before adding to batch)
-        if not page_data:
-            logger.info(f"Empty page {page_count + 1} received, stopping pagination")
-            if current_batch:
-                yield current_batch
-            break
-        
-        # Check stop conditions BEFORE adding records to batch
-        # This prevents unnecessary API calls when we know we should stop
-        page_count += 1
-        limit_value = params.get('limit', 100)
-        record_count = len(page_data)
-        logger.info(
-            f"Evaluating stop conditions for page {page_count}: "
-            f"{record_count} records returned, limit={limit_value}"
-        )
-        should_stop = _check_stop_conditions(page_data, stop_conditions, params, full_response_data)
-        if should_stop:
-            logger.info(
-                f"✅ Stop condition met at page {page_count} "
-                f"(page returned {record_count} records, limit: {limit_value})"
-            )
-            # Add the final page's records to current batch before yielding
-            for record in page_data:
-                current_batch.append(record)
-                total_extracted += 1
-                # Yield batch if it reaches batch_size during this final page
-                if len(current_batch) >= batch_size:
-                    yield current_batch
-                    current_batch = []
-            # Yield any remaining records
-            if current_batch:
-                yield current_batch
-            break
-        
-        # Add page data to current batch (only if we're not stopping)
-        for record in page_data:
-            current_batch.append(record)
-            total_extracted += 1
-            
-            # Yield batch when full
-            if len(current_batch) >= batch_size:
-                yield current_batch
-                current_batch = []
-            
-            # Check max_records limit
-            if max_records and total_extracted >= max_records:
-                if current_batch:
-                    yield current_batch
-                return
+    Example:
+        >>> # HTTP extraction (backward compatible)
+        >>> extract_config = {
+        ...     'base_url': 'https://api.example.com',
+        ...     'api_endpoint': '/v1/data'
+        ... }
+        >>> async for batch in extract_with_pagination_streaming(extract_config, {}, {}):
+        ...     print(f"Extracted {len(batch)} records")
         
-        # Extract pagination token/URL for next iteration
-        if strategy == 'cursor' and full_response_data:
-            try:
-                current = full_response_data
-                for part in cursor_response_path.split('.'):
-                    if isinstance(current, dict):
-                        current = current.get(part)
-                    elif isinstance(current, list) and part.isdigit():
-                        current = current[int(part)]
-                    else:
-                        current = None
-                        break
-                
-                if current and isinstance(current, str):
-                    current_cursor = current
-                elif current:
-                    current_cursor = str(current)
-                else:
-                    if current_batch:
-                        yield current_batch
-                    break
-            except (KeyError, IndexError, TypeError, ValueError):
-                if current_batch:
-                    yield current_batch
-                break
-        
-        elif strategy == 'next_url' and full_response_data:
-            try:
-                current = full_response_data
-                for part in next_url_response_path.split('.'):
-                    if isinstance(current, dict):
-                        current = current.get(part)
-                    elif isinstance(current, list) and part.isdigit():
-                        current = current[int(part)]
-                    else:
-                        current = None
-                        break
-                
-                if current and isinstance(current, str):
-                    current_url = current
-                else:
-                    current_url = None
-                
-                if not current_url:
-                    if current_batch:
-                        yield current_batch
-                    break
-            except (KeyError, IndexError, TypeError, ValueError):
-                if current_batch:
-                    yield current_batch
-                break
-        
-        elif strategy == 'link_header' and response_obj:
-            current_url = _extract_link_header_url(response_obj)
-            if not current_url:
-                if current_batch:
-                    yield current_batch
-                break
-            extract_config_copy['api_endpoint'] = current_url
-            extract_config_copy['base_url'] = ''
-        
-        # Update pagination state
-        if strategy == 'page':
-            current_page += page_increment
-        elif strategy == 'offset':
-            limit = params.get('limit', 100)
-            if increment_by == 'limit':
-                current_offset += limit
-            else:
-                current_offset += int(increment_by)
-        
-        # Delay between pages
-        if page_delay > 0:
-            await asyncio.sleep(page_delay)
+        >>> # File extraction
+        >>> extract_config = {
+        ...     'source_type': 'file',
+        ...     'file_path': '/path/to/data.csv'
+        ... }
+        >>> async for batch in extract_with_pagination_streaming(extract_config, {}, {}):
+        ...     print(f"Extracted {len(batch)} records")
+    """
+    # Use factory to get appropriate extractor
+    from pycharter.etl_generator.extractors.factory import ExtractorFactory
+    
+    extractor = ExtractorFactory.get_extractor(extract_config)
     
-    # Yield remaining records
-    if current_batch:
-        yield current_batch
+    # Delegate to extractor
+    async for batch in extractor.extract_streaming(
+        extract_config, params, headers, contract_dir, batch_size, max_records, config_context
+    ):
+        yield batch

pycharter/etl_generator/extractors/__init__.py

@@ -0,0 +1,26 @@
+"""
+Extractors module for ETL orchestrator.
+
+This module provides a modular architecture for data extraction from various sources:
+- HTTP/API extraction
+- File-based extraction (CSV, JSON, Parquet, Excel, TSV, XML)
+- Database extraction (PostgreSQL, MySQL, SQLite, MSSQL, Oracle)
+- Cloud storage extraction (S3, GCS, Azure Blob)
+"""
+
+from pycharter.etl_generator.extractors.base import BaseExtractor
+from pycharter.etl_generator.extractors.cloud_storage import CloudStorageExtractor
+from pycharter.etl_generator.extractors.database import DatabaseExtractor
+from pycharter.etl_generator.extractors.file import FileExtractor
+from pycharter.etl_generator.extractors.factory import ExtractorFactory, get_extractor
+from pycharter.etl_generator.extractors.http import HTTPExtractor
+
+__all__ = [
+    "BaseExtractor",
+    "ExtractorFactory",
+    "get_extractor",
+    "HTTPExtractor",
+    "FileExtractor",
+    "DatabaseExtractor",
+    "CloudStorageExtractor",
+]

pycharter/etl_generator/extractors/base.py

@@ -0,0 +1,70 @@
+"""
+Base extractor interface for ETL orchestrator.
+
+All extractors must implement this interface to ensure consistent behavior
+across different data sources.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, AsyncIterator, Dict, List, Optional
+
+
+class BaseExtractor(ABC):
+    """
+    Base class for all data extractors.
+    
+    All extractors must implement the extract_streaming method which yields
+    batches of records as dictionaries. Extractors are schema-agnostic and
+    focus purely on data retrieval from their respective sources.
+    """
+    
+    @abstractmethod
+    async def extract_streaming(
+        self,
+        extract_config: Dict[str, Any],
+        params: Dict[str, Any],
+        headers: Dict[str, Any],
+        contract_dir: Optional[Any] = None,
+        batch_size: int = 1000,
+        max_records: Optional[int] = None,
+        config_context: Optional[Dict[str, Any]] = None,
+    ) -> AsyncIterator[List[Dict[str, Any]]]:
+        """
+        Extract data in batches using async generator.
+        
+        This is the main interface that all extractors must implement.
+        It yields batches of records as lists of dictionaries, allowing
+        for memory-efficient processing of large datasets.
+        
+        Args:
+            extract_config: Extract configuration dictionary (source-specific)
+            params: Request/query parameters (may be source-specific)
+            headers: Request headers (may be source-specific)
+            contract_dir: Contract directory path (for variable resolution)
+            batch_size: Number of records to yield per batch
+            max_records: Maximum total records to extract (None = all)
+            config_context: Optional context dictionary for value injection
+        
+        Yields:
+            Batches of extracted records (lists of dictionaries)
+        
+        Raises:
+            RuntimeError: If extraction fails
+            ValueError: If configuration is invalid
+        """
+        pass
+    
+    def validate_config(self, extract_config: Dict[str, Any]) -> None:
+        """
+        Validate extractor-specific configuration.
+        
+        Override this method in subclasses to validate source-specific
+        configuration requirements.
+        
+        Args:
+            extract_config: Extract configuration dictionary
+        
+        Raises:
+            ValueError: If configuration is invalid
+        """
+        pass