ergon-framework-python 0.1.0__py3-none-any.whl

ergon/connector/connector.py

@@ -0,0 +1,97 @@
+from abc import ABC, abstractmethod
+from typing import Any, List, Union
+
+from pydantic import BaseModel, model_validator
+
+from . import transaction
+
+
+# ---------------------------------------------------------
+# SYNC CONNECTOR
+# ---------------------------------------------------------
+class Connector(ABC):
+    @abstractmethod
+    def fetch_transactions(self, *args, **kwargs) -> List[transaction.Transaction]:
+        """Fetch one or many transactions (sync)."""
+
+    @abstractmethod
+    def dispatch_transactions(self, transactions: List[transaction.Transaction], *args, **kwargs) -> Any:
+        """Publish a transaction (sync)."""
+        pass
+
+    def fetch_transaction_by_id(self, transaction_id: str, *args, **kwargs) -> transaction.Transaction:
+        """
+        Fetches a transaction by its id for individual processing.
+        """
+        raise NotImplementedError
+
+    def get_transactions_count(self, *args, **kwargs) -> int:
+        """Get the total number of transactions in the connector target system."""
+        raise NotImplementedError
+
+
+# ---------------------------------------------------------
+# ASYNC CONNECTOR
+# ---------------------------------------------------------
+class AsyncConnector(ABC):
+    @abstractmethod
+    async def fetch_transactions_async(self, *args, **kwargs) -> List[transaction.Transaction]:
+        """Fetch transactions asynchronously."""
+        pass
+
+    @abstractmethod
+    async def dispatch_transactions_async(self, transactions: List[transaction.Transaction], *args, **kwargs) -> Any:
+        """Publish a transaction asynchronously."""
+        pass
+
+    async def fetch_transaction_by_id_async(self, transaction_id: str, *args, **kwargs) -> transaction.Transaction:
+        """Fetches a transaction by its id for individual processing asynchronously."""
+        raise NotImplementedError
+
+    async def get_transactions_count_async(self, *args, **kwargs) -> int:
+        """Get the total number of transactions in the connector target system asynchronously."""
+        raise NotImplementedError
+
+    async def ack_transaction(self, transaction: transaction.Transaction) -> None:
+        """Acknowledge a successfully processed transaction.
+
+        Default raises NotImplementedError so non-broker connectors (database,
+        HTTP, file-based, etc.) can ignore it. Broker connectors that have
+        per-message ack semantics (RabbitMQ, SQS, Kafka, ...) must override.
+        """
+        raise NotImplementedError
+
+    async def nack_transaction(self, transaction: transaction.Transaction, requeue: bool = True) -> None:
+        """Negatively acknowledge a transaction.
+
+        ``requeue=True`` asks the broker to redeliver; ``requeue=False`` routes
+        to the broker's dead-letter destination if one is configured. Default
+        raises NotImplementedError; broker connectors must override.
+        """
+        raise NotImplementedError
+
+
+# ---------------------------------------------------------
+# CONNECTOR CONFIG (supports both sync + async)
+# ---------------------------------------------------------
+class ConnectorConfig(BaseModel):
+    """
+    Generic connector configuration.
+
+    connector: class implementing either Connector or AsyncConnector
+    args: positional arguments passed to connector's __init__
+    kwargs: keyword arguments passed to connector's __init__
+    """
+
+    connector: Union[type[Connector], type[AsyncConnector]]
+    args: tuple[Any, ...] = ()
+    kwargs: dict[str, Any] = {}
+
+    @model_validator(mode="after")
+    def validate_connector(self) -> "ConnectorConfig":
+        """Validate that the connector inherits from Connector or AsyncConnector."""
+        if not issubclass(self.connector, Connector) and not issubclass(self.connector, AsyncConnector):
+            raise ValueError(
+                f"Connector '{self.connector}' must inherit from Connector or AsyncConnector. Got: {self.connector}"
+            )
+        return self

ergon/connector/excel/__init__.py

@@ -0,0 +1,18 @@
+"""
+Excel connector for ergon-framework.
+Provides transaction-based access to Excel files with support for:
+- Batch mode (multiple rows per transaction for high-throughput processing)
+- Normal mode (one row per transaction)
+- Sharding for parallel processing
+"""
+
+from .connector import ExcelConnector
+from .models import ExcelFetchConfig, ExcelRow
+from .service import ExcelService
+
+__all__ = [
+    "ExcelConnector",
+    "ExcelService",
+    "ExcelFetchConfig",
+    "ExcelRow",
+]

ergon/connector/excel/connector.py

@@ -0,0 +1,175 @@
+"""
+Excel connector for ergon-framework.
+Provides transaction-based access to Excel files with support for:
+- Batch mode (multiple rows per transaction)
+- Normal mode (one row per transaction)
+- Sharding for parallel processing
+"""
+
+import logging
+import uuid
+from typing import Any, List, Optional
+
+from ..connector import Connector
+from ..transaction import Transaction
+from .service import ExcelService
+
+logger = logging.getLogger(__name__)
+
+
+class ExcelConnector(Connector):
+    """
+    Connector for reading transactions from Excel files.
+    Features:
+    - Iterator-based reading (memory efficient)
+    - Batch mode for high-throughput processing
+    - Automatic sharding support for parallel workers
+    Example:
+        service = ExcelService()
+        connector = ExcelConnector(service)
+        transactions = connector.fetch_transactions(
+            limit=100,
+            file_path="data.xlsx",
+            filter_col="Type",
+            filter_val="Active",
+            batch_mode=True,
+        )
+    """
+
+    def __init__(self, service: ExcelService):
+        """
+        Initialize the Excel connector.
+        Args:
+            service: ExcelService instance for reading Excel files.
+        """
+        self.service = service
+        self._iterators = {}
+
+    def fetch_transactions(
+        self,
+        limit: int = 10,
+        file_path: Optional[str] = None,
+        filter_col: Optional[str] = None,
+        filter_val: Optional[Any] = None,
+        metadata: Optional[dict] = None,
+        worker_id: Optional[int] = None,
+        max_workers: Optional[int] = None,
+        batch_mode: bool = False,
+        sheet_name: Optional[str] = None,
+        *args,
+        **kwargs,
+    ) -> List[Transaction]:
+        """
+        Fetch transactions from Excel file.
+        Args:
+            limit: Number of rows to fetch per call (batch size).
+            file_path: Path to Excel file.
+            filter_col: Column to filter by.
+            filter_val: Value to filter for.
+            metadata: Metadata to attach to transactions.
+            worker_id: Worker ID for sharding (0-indexed).
+            max_workers: Total number of workers for sharding.
+            batch_mode: When True, returns a single transaction with payload
+                        containing a list of rows instead of individual transactions.
+                        This enables batch processing (e.g., batch embedding, batch upsert).
+            sheet_name: Optional sheet name to read.
+        Returns:
+            List of Transaction objects.
+        """
+        if not file_path:
+            return []
+
+        # Create a key to identify this specific data stream
+        key = (file_path, filter_col, filter_val, worker_id, max_workers, sheet_name)
+
+        # Create iterator if needed
+        if key not in self._iterators:
+            data = self.service.read_excel(
+                file_path,
+                filter_col,
+                filter_val,
+                shard_id=worker_id,
+                total_shards=max_workers,
+                sheet_name=sheet_name,
+            )
+            self._iterators[key] = iter(data)
+
+        iterator = self._iterators[key]
+        base_metadata = metadata or {}
+
+        if batch_mode:
+            return self._fetch_batch_mode(iterator, key, limit, base_metadata)
+        else:
+            return self._fetch_normal_mode(iterator, key, limit, base_metadata)
+
+    def _fetch_batch_mode(
+        self,
+        iterator,
+        key,
+        limit: int,
+        base_metadata: dict,
+    ) -> List[Transaction]:
+        """Bundle multiple rows into a single transaction."""
+        rows = []
+        try:
+            for _ in range(limit):
+                row = next(iterator)
+                rows.append(row)
+        except StopIteration:
+            pass
+
+        if not rows:
+            if key in self._iterators:
+                del self._iterators[key]
+            return []
+
+        batch_id = str(uuid.uuid4())
+        return [
+            Transaction(
+                id=batch_id,
+                payload=rows,
+                metadata={**base_metadata, "batch_mode": True, "batch_size": len(rows)},
+            )
+        ]
+
+    def _fetch_normal_mode(
+        self,
+        iterator,
+        key,
+        limit: int,
+        base_metadata: dict,
+    ) -> List[Transaction]:
+        """Return one transaction per row."""
+        transactions = []
+        try:
+            for _ in range(limit):
+                row = next(iterator)
+                row_id = str(uuid.uuid4())
+                transactions.append(
+                    Transaction(
+                        id=row_id,
+                        payload=row,
+                        metadata=base_metadata,
+                    )
+                )
+        except StopIteration:
+            pass
+
+        if not transactions:
+            if key in self._iterators:
+                del self._iterators[key]
+
+        return transactions
+
+    def dispatch_transactions(
+        self,
+        transactions: List[Transaction],
+        *args,
+        **kwargs,
+    ) -> None:
+        """
+        Excel connector does not support writing transactions.
+        Raises:
+            NotImplementedError: Always raised.
+        """
+        raise NotImplementedError("ExcelConnector does not support producing transactions.")

ergon/connector/excel/models.py

@@ -0,0 +1,24 @@
+"""
+Excel connector models for ergon-framework.
+"""
+
+from typing import Any, Dict, Optional
+
+from pydantic import BaseModel, Field
+
+
+class ExcelFetchConfig(BaseModel):
+    """Configuration for fetching transactions from an Excel file."""
+
+    file_path: str = Field(..., description="Path to the Excel file")
+    filter_col: Optional[str] = Field(None, description="Column name to filter by")
+    filter_val: Optional[Any] = Field(None, description="Value to filter for in filter_col")
+    sheet_name: Optional[str] = Field(None, description="Sheet name to read (defaults to active sheet)")
+    batch_mode: bool = Field(False, description="When True, bundle rows into single transaction")
+
+
+class ExcelRow(BaseModel):
+    """Represents a row from an Excel file."""
+
+    data: Dict[str, Any] = Field(default_factory=dict)
+    row_index: int = Field(..., description="Original row index in the Excel file")

ergon/connector/excel/service.py

@@ -0,0 +1,98 @@
+"""
+Excel service (client) for reading Excel files.
+"""
+
+import logging
+from typing import Any, Generator, Optional
+
+from openpyxl import load_workbook
+
+logger = logging.getLogger(__name__)
+
+
+class ExcelService:
+    """
+    Service for reading Excel files with streaming support.
+    Supports:
+    - Streaming row-by-row reading (memory efficient)
+    - Column-based filtering
+    - Sharding for parallel processing
+    """
+
+    def __init__(self):
+        # Cache for loaded data (optional future enhancement)
+        self._cache = {}
+
+    def read_excel(
+        self,
+        file_path: str,
+        filter_col: Optional[str] = None,
+        filter_val: Optional[Any] = None,
+        shard_id: Optional[int] = None,
+        total_shards: Optional[int] = None,
+        sheet_name: Optional[str] = None,
+    ) -> Generator[dict, None, None]:
+        """
+        Read Excel file and yield rows as dictionaries.
+        Args:
+            file_path: Path to the Excel file.
+            filter_col: Optional column name to filter by.
+            filter_val: Optional value to filter for in filter_col.
+            shard_id: Worker shard ID (0-indexed). Used for parallel processing.
+            total_shards: Total number of shards/workers. Used for parallel processing.
+            sheet_name: Optional sheet name to read (defaults to active sheet).
+        When shard_id and total_shards are provided, only rows where
+        (matched_index % total_shards == shard_id) are yielded. This allows
+        multiple workers to process different partitions of the same file.
+        Yields:
+            dict: Row data as a dictionary with column headers as keys.
+        """
+        wb = load_workbook(filename=file_path, read_only=True, data_only=True)
+        try:
+            # Select sheet
+            if sheet_name:
+                ws = wb[sheet_name]
+            else:
+                ws = wb.active
+
+            if ws is None:
+                return
+
+            rows = ws.rows
+
+            try:
+                headers = [cell.value for cell in next(rows)]
+            except StopIteration:
+                return
+
+            sharding_enabled = shard_id is not None and total_shards is not None
+            matched_index = 0
+            yielded_count = 0
+
+            logger.debug(f"Sharding enabled: {sharding_enabled}")
+            if sharding_enabled:
+                logger.info(f"[EXCEL] Worker {shard_id}/{total_shards} starting to read {file_path}")
+
+            for row in rows:
+                row_values = [cell.value for cell in row]
+                row_dict = dict(zip(headers, row_values))
+
+                # Apply filter first
+                if filter_col and filter_val:
+                    val = row_dict.get(filter_col)
+                    if val != filter_val:
+                        continue
+
+                # Apply sharding after filtering
+                if sharding_enabled and total_shards is not None and (matched_index % total_shards != shard_id):
+                    matched_index += 1
+                    continue
+
+                matched_index += 1
+                yielded_count += 1
+                yield row_dict
+
+            if sharding_enabled:
+                logger.info(f"[EXCEL] Worker {shard_id} finished. Matched={matched_index}, Yielded={yielded_count}")
+        finally:
+            wb.close()

ergon/connector/pipefy/__init__.py

@@ -0,0 +1,21 @@
+from .async_connector import AsyncPipefyConnector
+from .async_service import AsyncPipefyService
+from .connector import PipefyConnector
+from .models import (
+    CreateCardInput,
+    FieldFilter,
+    FieldFilterOperator,
+    PipefyClient,
+)
+from .service import PipefyService
+
+__all__ = [
+    "AsyncPipefyConnector",
+    "AsyncPipefyService",
+    "PipefyConnector",
+    "PipefyService",
+    "PipefyClient",
+    "CreateCardInput",
+    "FieldFilter",
+    "FieldFilterOperator",
+]

ergon/connector/pipefy/async_connector.py

@@ -0,0 +1,48 @@
+from typing import List, Optional
+
+from ..connector import AsyncConnector
+from ..transaction import Transaction
+from .async_service import AsyncPipefyService
+from .models import PipefyClient
+
+
+class AsyncPipefyConnector(AsyncConnector):
+    service: AsyncPipefyService
+
+    def __init__(self, client: PipefyClient, *args, **kwargs):
+        self.service = AsyncPipefyService(client)
+        self._authenticated = False
+
+    async def _ensure_authenticated(self):
+        if not self._authenticated:
+            await self.service.authenticate()
+            self._authenticated = True
+
+    async def fetch_transactions_async(
+        self, batch_size: int = 1, phase_id: Optional[str] = None, field_filters=None, **kwargs
+    ) -> List[Transaction]:
+        if not phase_id:
+            raise ValueError("AsyncPipefyConnector.fetch requires a phase_id")
+
+        await self._ensure_authenticated()
+
+        cards = await self.service.get_next_card(
+            phase_id=phase_id, field_filters=field_filters, batch_size=batch_size, **kwargs
+        )
+        if not cards:
+            return []
+
+        return [Transaction(id=card.get("id"), payload=card) for card in cards]
+
+    async def dispatch_transactions_async(self, transactions: List[Transaction], *args, **kwargs):
+        await self._ensure_authenticated()
+
+        for transaction in transactions:
+            await self.service.create_card(transaction.payload)
+        return True
+
+    async def fetch_transaction_by_id_async(self, transaction_id: str, *args, **kwargs) -> Transaction:
+        await self._ensure_authenticated()
+
+        transaction = await self.service.get_card_by_id(transaction_id)
+        return Transaction(id=transaction.get("id"), payload=transaction)