aiecs 1.5.1__py3-none-any.whl

aiecs/tools/apisource/utils/__init__.py

@@ -0,0 +1,9 @@
+"""
+Utils Module
+
+Contains shared validation and utility functions.
+"""
+
+from aiecs.tools.apisource.utils.validators import DataValidator
+
+__all__ = ["DataValidator"]

aiecs/tools/apisource/utils/validators.py

@@ -0,0 +1,338 @@
+"""
+Shared Validation Utilities for API Providers
+
+Common validation functions for data quality assessment:
+- Detect outliers in numeric data
+- Find gaps in time series
+- Check data completeness
+- Validate data types and ranges
+"""
+
+import logging
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class DataValidator:
+    """
+    Provides common data validation methods for API providers.
+    """
+
+    @staticmethod
+    def detect_outliers(
+        values: List[float], method: str = "iqr", threshold: float = 1.5
+    ) -> List[int]:
+        """
+        Detect outliers in numeric data.
+
+        Args:
+            values: List of numeric values
+            method: Detection method ('iqr' or 'zscore')
+            threshold: Threshold for outlier detection
+                      - For IQR: typically 1.5 or 3.0
+                      - For Z-score: typically 2.0 or 3.0
+
+        Returns:
+            List of indices where outliers were detected
+        """
+        if not values or len(values) < 4:
+            return []
+
+        outlier_indices = []
+
+        if method == "iqr":
+            # Interquartile Range method
+            sorted_values = sorted(values)
+            n = len(sorted_values)
+
+            q1_idx = n // 4
+            q3_idx = 3 * n // 4
+
+            q1 = sorted_values[q1_idx]
+            q3 = sorted_values[q3_idx]
+            iqr = q3 - q1
+
+            lower_bound = q1 - threshold * iqr
+            upper_bound = q3 + threshold * iqr
+
+            for i, value in enumerate(values):
+                if value < lower_bound or value > upper_bound:
+                    outlier_indices.append(i)
+
+        elif method == "zscore":
+            # Z-score method
+            mean = sum(values) / len(values)
+            variance = sum((x - mean) ** 2 for x in values) / len(values)
+            std_dev = variance**0.5
+
+            if std_dev == 0:
+                return []
+
+            for i, value in enumerate(values):
+                z_score = abs((value - mean) / std_dev)
+                if z_score > threshold:
+                    outlier_indices.append(i)
+
+        return outlier_indices
+
+    @staticmethod
+    def detect_time_gaps(
+        data: List[Dict[str, Any]],
+        date_field: str = "date",
+        expected_frequency: Optional[str] = None,
+    ) -> List[Dict[str, Any]]:
+        """
+        Detect gaps in time series data.
+
+        Args:
+            data: List of data items with date fields
+            date_field: Name of the date field
+            expected_frequency: Expected frequency ('daily', 'weekly', 'monthly', 'quarterly', 'annual')
+
+        Returns:
+            List of gap information dictionaries
+        """
+        if len(data) < 2:
+            return []
+
+        gaps = []
+
+        # Parse dates
+        dates = []
+        for i, item in enumerate(data):
+            if date_field in item:
+                try:
+                    date_str = str(item[date_field])
+                    if "T" in date_str:
+                        date_obj = datetime.fromisoformat(date_str.replace("Z", "+00:00"))
+                    else:
+                        date_obj = datetime.strptime(date_str[:10], "%Y-%m-%d")
+                    dates.append((i, date_obj))
+                except (ValueError, TypeError):
+                    continue
+
+        if len(dates) < 2:
+            return []
+
+        # Sort by date
+        dates.sort(key=lambda x: x[1])
+
+        # Determine expected gap if not specified
+        if expected_frequency is None:
+            # Estimate from first few intervals
+            if len(dates) >= 3:
+                intervals = [
+                    (dates[i + 1][1] - dates[i][1]).days for i in range(min(3, len(dates) - 1))
+                ]
+                avg_interval = sum(intervals) / len(intervals)
+
+                if avg_interval <= 2:
+                    expected_frequency = "daily"
+                elif avg_interval <= 10:
+                    expected_frequency = "weekly"
+                elif avg_interval <= 40:
+                    expected_frequency = "monthly"
+                elif avg_interval <= 120:
+                    expected_frequency = "quarterly"
+                else:
+                    expected_frequency = "annual"
+
+        # Define expected gaps in days
+        frequency_gaps = {
+            "daily": 1,
+            "weekly": 7,
+            "monthly": 31,
+            "quarterly": 92,
+            "annual": 365,
+        }
+
+        expected_gap_days = frequency_gaps.get(expected_frequency, 31)
+        tolerance = expected_gap_days * 0.5  # 50% tolerance
+
+        # Check for gaps
+        for i in range(len(dates) - 1):
+            idx1, date1 = dates[i]
+            idx2, date2 = dates[i + 1]
+
+            gap_days = (date2 - date1).days
+
+            if gap_days > expected_gap_days + tolerance:
+                gaps.append(
+                    {
+                        "start_index": idx1,
+                        "end_index": idx2,
+                        "start_date": date1.isoformat(),
+                        "end_date": date2.isoformat(),
+                        "gap_days": gap_days,
+                        "expected_days": expected_gap_days,
+                    }
+                )
+
+        return gaps
+
+    @staticmethod
+    def check_data_completeness(
+        data: List[Dict[str, Any]],
+        value_field: str = "value",
+        missing_indicators: Optional[List[Any]] = None,
+    ) -> Dict[str, Any]:
+        """
+        Check completeness of data.
+
+        Args:
+            data: List of data items
+            value_field: Name of the value field to check
+            missing_indicators: Values that indicate missing data (e.g., ['.', None, 'NA'])
+
+        Returns:
+            Completeness statistics dictionary
+        """
+        if missing_indicators is None:
+            missing_indicators = [".", None, "NA", "N/A", "", "null"]
+
+        total_records = len(data)
+        if total_records == 0:
+            return {
+                "total_records": 0,
+                "missing_count": 0,
+                "completeness": 1.0,
+                "missing_indices": [],
+            }
+
+        missing_count = 0
+        missing_indices = []
+
+        for i, item in enumerate(data):
+            if value_field in item:
+                value = item[value_field]
+                # Check if value is missing
+                if value in missing_indicators:
+                    missing_count += 1
+                    missing_indices.append(i)
+                elif isinstance(value, str) and value.strip() in missing_indicators:
+                    missing_count += 1
+                    missing_indices.append(i)
+            else:
+                # Field doesn't exist
+                missing_count += 1
+                missing_indices.append(i)
+
+        completeness = (total_records - missing_count) / total_records
+
+        return {
+            "total_records": total_records,
+            "missing_count": missing_count,
+            "present_count": total_records - missing_count,
+            "completeness": round(completeness, 4),
+            "missing_indices": missing_indices[:10],  # Limit to first 10
+        }
+
+    @staticmethod
+    def calculate_value_range(
+        data: List[Dict[str, Any]],
+        value_field: str = "value",
+        missing_indicators: Optional[List[Any]] = None,
+    ) -> Optional[Dict[str, float]]:
+        """
+        Calculate min, max, mean of numeric values.
+
+        Args:
+            data: List of data items
+            value_field: Name of the value field
+            missing_indicators: Values to skip
+
+        Returns:
+            Dictionary with min, max, mean, or None if no valid data
+        """
+        if missing_indicators is None:
+            missing_indicators = [".", None, "NA", "N/A", "", "null"]
+
+        numeric_values = []
+
+        for item in data:
+            if value_field in item:
+                value = item[value_field]
+
+                # Skip missing indicators
+                if value in missing_indicators:
+                    continue
+
+                # Try to convert to float
+                try:
+                    if isinstance(value, (int, float)):
+                        numeric_values.append(float(value))
+                    elif isinstance(value, str):
+                        # Clean string (remove commas, etc.)
+                        cleaned = value.strip().replace(",", "")
+                        if cleaned and cleaned not in missing_indicators:
+                            numeric_values.append(float(cleaned))
+                except (ValueError, TypeError):
+                    continue
+
+        if not numeric_values:
+            return None
+
+        return {
+            "min": min(numeric_values),
+            "max": max(numeric_values),
+            "mean": sum(numeric_values) / len(numeric_values),
+            "count": len(numeric_values),
+        }
+
+    @staticmethod
+    def infer_data_frequency(data: List[Dict[str, Any]], date_field: str = "date") -> Optional[str]:
+        """
+        Infer the frequency of time series data.
+
+        Args:
+            data: List of data items with dates
+            date_field: Name of the date field
+
+        Returns:
+            Frequency string or None
+        """
+        if len(data) < 3:
+            return None
+
+        # Parse dates
+        dates = []
+        for item in data:
+            if date_field in item:
+                try:
+                    date_str = str(item[date_field])
+                    if "T" in date_str:
+                        date_obj = datetime.fromisoformat(date_str.replace("Z", "+00:00"))
+                    else:
+                        date_obj = datetime.strptime(date_str[:10], "%Y-%m-%d")
+                    dates.append(date_obj)
+                except (ValueError, TypeError):
+                    continue
+
+        if len(dates) < 3:
+            return None
+
+        # Sort dates
+        dates.sort()
+
+        # Calculate intervals
+        intervals = [(dates[i + 1] - dates[i]).days for i in range(len(dates) - 1)]
+
+        # Calculate median interval
+        intervals.sort()
+        median_interval = intervals[len(intervals) // 2]
+
+        # Classify frequency
+        if median_interval <= 2:
+            return "daily"
+        elif median_interval <= 10:
+            return "weekly"
+        elif median_interval <= 40:
+            return "monthly"
+        elif median_interval <= 120:
+            return "quarterly"
+        elif median_interval <= 400:
+            return "annual"
+        else:
+            return "irregular"

aiecs/tools/base_tool.py

@@ -0,0 +1,201 @@
+import inspect
+import logging
+from typing import Any, Dict, List, Optional, Type
+
+from pydantic import BaseModel, ValidationError
+import re
+
+from aiecs.tools.tool_executor import (
+    InputValidationError,
+    SecurityError,
+    get_executor,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class BaseTool:
+    """
+    Base class for all tools, providing common functionality:
+    - Input validation with Pydantic schemas
+    - Caching with TTL and content-based keys
+    - Concurrency with async/sync execution
+    - Error handling with retries and context
+    - Performance optimization with metrics
+    - Logging with structured output
+
+    Tools inheriting from this class focus on business logic, leveraging
+    the executor's cross-cutting concerns.
+
+    Example:
+        class MyTool(BaseTool):
+            class ReadSchema(BaseModel):
+                path: str
+
+            @validate_input(ReadSchema)
+            @cache_result(ttl=300)
+            @run_in_executor
+            @measure_execution_time
+            @sanitize_input
+            def read(self, path: str):
+                # Implementation
+                pass
+    """
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        """
+        Initialize the tool with optional configuration.
+
+        Args:
+            config (Dict[str, Any], optional): Tool-specific configuration.
+
+        Raises:
+            ValueError: If config is invalid.
+        """
+        self._executor = get_executor(config)
+        self._config = config or {}
+        self._schemas: Dict[str, Type[BaseModel]] = {}
+        self._async_methods: List[str] = []
+        self._register_schemas()
+        self._register_async_methods()
+
+    def _register_schemas(self) -> None:
+        """
+        Register Pydantic schemas for operations by inspecting inner Schema classes.
+
+        Example:
+            class MyTool(BaseTool):
+                class ReadSchema(BaseModel):
+                    path: str
+                def read(self, path: str):
+                    pass
+            # Registers 'read' -> ReadSchema
+        """
+        for attr_name in dir(self.__class__):
+            attr = getattr(self.__class__, attr_name)
+            if (
+                isinstance(attr, type)
+                and issubclass(attr, BaseModel)
+                and attr.__name__.endswith("Schema")
+            ):
+                op_name = attr.__name__.replace("Schema", "").lower()
+                self._schemas[op_name] = attr
+
+    def _register_async_methods(self) -> None:
+        """
+        Register async methods for proper execution handling.
+        """
+        for attr_name in dir(self.__class__):
+            attr = getattr(self.__class__, attr_name)
+            if inspect.iscoroutinefunction(attr) and not attr_name.startswith("_"):
+                self._async_methods.append(attr_name)
+
+    def _sanitize_kwargs(self, kwargs: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Sanitize keyword arguments to prevent injection attacks.
+
+        Args:
+            kwargs (Dict[str, Any]): Input keyword arguments.
+
+        Returns:
+            Dict[str, Any]: Sanitized keyword arguments.
+
+        Raises:
+            SecurityError: If kwargs contain malicious content.
+        """
+        sanitized = {}
+        for k, v in kwargs.items():
+            if isinstance(v, str) and re.search(
+                r"(\bSELECT\b|\bINSERT\b|--|;|/\*)", v, re.IGNORECASE
+            ):
+                raise SecurityError(f"Input parameter '{k}' contains potentially malicious content")
+            sanitized[k] = v
+        return sanitized
+
+    def run(self, op: str, **kwargs) -> Any:
+        """
+        Execute a synchronous operation with parameters.
+
+        Args:
+            op (str): The name of the operation to execute.
+            **kwargs: The parameters to pass to the operation.
+
+        Returns:
+            Any: The result of the operation.
+
+        Raises:
+            ToolExecutionError: If the operation fails.
+            InputValidationError: If input parameters are invalid.
+            SecurityError: If inputs contain malicious content.
+        """
+        schema_class = self._schemas.get(op)
+        if schema_class:
+            try:
+                schema = schema_class(**kwargs)
+                kwargs = schema.model_dump(exclude_unset=True)
+            except ValidationError as e:
+                raise InputValidationError(f"Invalid input parameters: {e}")
+        kwargs = self._sanitize_kwargs(kwargs)
+        return self._executor.execute(self, op, **kwargs)
+
+    async def run_async(self, op: str, **kwargs) -> Any:
+        """
+        Execute an asynchronous operation with parameters.
+
+        Args:
+            op (str): The name of the operation to execute.
+            **kwargs: The parameters to pass to the operation.
+
+        Returns:
+            Any: The result of the operation.
+
+        Raises:
+            ToolExecutionError: If the operation fails.
+            InputValidationError: If input parameters are invalid.
+            SecurityError: If inputs contain malicious content.
+        """
+        schema_class = self._schemas.get(op)
+        if schema_class:
+            try:
+                schema = schema_class(**kwargs)
+                kwargs = schema.model_dump(exclude_unset=True)
+            except ValidationError as e:
+                raise InputValidationError(f"Invalid input parameters: {e}")
+        kwargs = self._sanitize_kwargs(kwargs)
+        return await self._executor.execute_async(self, op, **kwargs)
+
+    async def run_batch(self, operations: List[Dict[str, Any]]) -> List[Any]:
+        """
+        Execute multiple operations in parallel.
+
+        Args:
+            operations (List[Dict[str, Any]]): List of operation dictionaries with 'op' and 'kwargs'.
+
+        Returns:
+            List[Any]: List of operation results.
+
+        Raises:
+            ToolExecutionError: If any operation fails.
+            InputValidationError: If input parameters are invalid.
+        """
+        return await self._executor.execute_batch(self, operations)
+
+    def _get_method_schema(self, method_name: str) -> Optional[Type[BaseModel]]:
+        """
+        Get the schema for a method if it exists.
+
+        Args:
+            method_name (str): The name of the method.
+
+        Returns:
+            Optional[Type[BaseModel]]: The schema class or None.
+        """
+        if method_name in self._schemas:
+            return self._schemas[method_name]
+        schema_name = method_name[0].upper() + method_name[1:] + "Schema"
+        for attr_name in dir(self.__class__):
+            if attr_name == schema_name:
+                attr = getattr(self.__class__, attr_name)
+                if isinstance(attr, type) and issubclass(attr, BaseModel):
+                    return attr
+        return None

aiecs/tools/docs/__init__.py

@@ -0,0 +1,121 @@
+# python-middleware/aiecs/tools/docs/__init__.py
+
+"""
+Document Tools Module
+
+This module contains specialized tools for document processing and analysis:
+- document_parser_tool: Modern high-performance document parsing with AI
+- ai_document_orchestrator: AI-powered document processing orchestrator
+"""
+
+# Lazy import all document tools to avoid heavy dependencies at import time
+import os
+
+# Define available document tools for lazy loading
+_AVAILABLE_DOC_TOOLS = [
+    "document_parser_tool",
+    "ai_document_orchestrator",
+    "document_writer_tool",
+    "ai_document_writer_orchestrator",
+    "document_creator_tool",
+    "document_layout_tool",
+    "content_insertion_tool",
+]
+
+# Track which tools have been loaded
+_LOADED_DOC_TOOLS = set()
+
+
+def _lazy_load_doc_tool(tool_name: str):
+    """Lazy load a specific document tool module"""
+    if tool_name in _LOADED_DOC_TOOLS:
+        return
+
+    # Mark as loading to prevent infinite recursion
+    _LOADED_DOC_TOOLS.add(tool_name)
+
+    try:
+        if tool_name == "document_parser_tool":
+            from . import document_parser_tool
+
+            globals()["document_parser_tool"] = document_parser_tool
+        elif tool_name == "ai_document_orchestrator":
+            from . import ai_document_orchestrator
+
+            globals()["ai_document_orchestrator"] = ai_document_orchestrator
+        elif tool_name == "document_writer_tool":
+            from . import document_writer_tool
+
+            globals()["document_writer_tool"] = document_writer_tool
+        elif tool_name == "ai_document_writer_orchestrator":
+            from . import ai_document_writer_orchestrator
+
+            globals()["ai_document_writer_orchestrator"] = ai_document_writer_orchestrator
+        elif tool_name == "document_creator_tool":
+            from . import document_creator_tool
+
+            globals()["document_creator_tool"] = document_creator_tool
+        elif tool_name == "document_layout_tool":
+            from . import document_layout_tool
+
+            globals()["document_layout_tool"] = document_layout_tool
+        elif tool_name == "content_insertion_tool":
+            from . import content_insertion_tool
+
+            globals()["content_insertion_tool"] = content_insertion_tool
+
+    except ImportError as e:
+        # Remove from loaded set if import failed
+        _LOADED_DOC_TOOLS.discard(tool_name)
+        print(f"Warning: Could not import {tool_name}: {e}")
+
+
+def __getattr__(name: str):
+    """
+    Lazy loading mechanism for document tools.
+
+    This allows importing tools like:
+    from aiecs.tools.docs import document_parser_tool
+    """
+    if name in _AVAILABLE_DOC_TOOLS:
+        _lazy_load_doc_tool(name)
+        if name in globals():
+            return globals()[name]
+
+    raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
+
+
+def list_doc_tools():
+    """List all available document tools"""
+    return _AVAILABLE_DOC_TOOLS.copy()
+
+
+def load_all_doc_tools():
+    """Load all available document tools"""
+    for tool_name in _AVAILABLE_DOC_TOOLS:
+        _lazy_load_doc_tool(tool_name)
+
+
+# Auto-discovery of tool modules in this directory
+
+
+def _discover_doc_tools():
+    """Discover document tool modules in the current directory"""
+    current_dir = os.path.dirname(__file__)
+    if not current_dir:
+        return
+
+    for filename in os.listdir(current_dir):
+        if filename.endswith("_tool.py") and not filename.startswith("__"):
+            tool_name = filename[:-3]  # Remove .py extension
+            if tool_name not in _AVAILABLE_DOC_TOOLS:
+                _AVAILABLE_DOC_TOOLS.append(tool_name)
+
+
+# Discover tools on import
+_discover_doc_tools()
+
+__all__ = [
+    "list_doc_tools",
+    "load_all_doc_tools",
+] + _AVAILABLE_DOC_TOOLS