aiecs 1.0.8__py3-none-any.whl → 1.2.0__py3-none-any.whl

aiecs/tools/langchain_adapter.py

@@ -13,6 +13,9 @@ import logging
 from typing import Any, Dict, List, Optional, Type, Union, get_type_hints
 from pydantic import BaseModel, Field
 
+# Import schema generator
+from aiecs.tools.schema_generator import generate_schema_from_method
+
 try:
     from langchain.tools import BaseTool as LangchainBaseTool
     from langchain.callbacks.manager import CallbackManagerForToolRun, AsyncCallbackManagerForToolRun
@@ -33,24 +36,27 @@ logger = logging.getLogger(__name__)
 class LangchainToolAdapter(LangchainBaseTool):
     """
     Langchain tool adapter for single operation
-    
+
     Wraps one operation method of BaseTool as an independent Langchain tool
     """
-    
+
     # Define class attributes
     name: str = ""
     description: str = ""
-    
+    base_tool_name: str = ""
+    operation_name: str = ""
+    operation_schema: Optional[Type[BaseModel]] = None
+
     def __init__(
-        self, 
+        self,
         base_tool_name: str,
-        operation_name: str, 
+        operation_name: str,
         operation_schema: Optional[Type[BaseModel]] = None,
         description: Optional[str] = None
     ):
         """
         Initialize adapter
-        
+
         Args:
             base_tool_name: Original tool name
             operation_name: Operation name
@@ -58,56 +64,55 @@ class LangchainToolAdapter(LangchainBaseTool):
             description: Tool description
         """
         # Construct tool name and description
-        self.name = f"{base_tool_name}_{operation_name}"
-        self.description = description or f"Execute {operation_name} operation from {base_tool_name} tool"
-        
-        # Store tool information (use self.__dict__ to set directly to avoid pydantic validation)
-        self.__dict__['base_tool_name'] = base_tool_name
-        self.__dict__['operation_name'] = operation_name
-        self.__dict__['operation_schema'] = operation_schema
-        
-        # Set parameter Schema
-        if operation_schema:
-            self.args_schema = operation_schema
-        
-        super().__init__()
+        tool_name = f"{base_tool_name}_{operation_name}"
+        tool_description = description or f"Execute {operation_name} operation from {base_tool_name} tool"
+
+        # Initialize parent class with all required fields
+        super().__init__(
+            name=tool_name,
+            description=tool_description,
+            base_tool_name=base_tool_name,
+            operation_name=operation_name,
+            operation_schema=operation_schema,
+            args_schema=operation_schema
+        )
     
     def _run(
-        self, 
-        run_manager: Optional[CallbackManagerForToolRun] = None, 
+        self,
+        run_manager: Optional[CallbackManagerForToolRun] = None,
         **kwargs: Any
     ) -> Any:
         """Execute operation synchronously"""
         try:
             # Get original tool instance
-            base_tool = get_tool(self.__dict__['base_tool_name'])
-            
+            base_tool = get_tool(self.base_tool_name)
+
             # Execute operation
-            result = base_tool.run(self.__dict__['operation_name'], **kwargs)
-            
+            result = base_tool.run(self.operation_name, **kwargs)
+
             logger.info(f"Successfully executed {self.name} with result type: {type(result)}")
             return result
-            
+
         except Exception as e:
             logger.error(f"Error executing {self.name}: {str(e)}")
             raise
-    
+
     async def _arun(
-        self, 
+        self,
         run_manager: Optional[AsyncCallbackManagerForToolRun] = None,
         **kwargs: Any
     ) -> Any:
         """Execute operation asynchronously"""
         try:
             # Get original tool instance
-            base_tool = get_tool(self.__dict__['base_tool_name'])
-            
+            base_tool = get_tool(self.base_tool_name)
+
             # Execute asynchronous operation
-            result = await base_tool.run_async(self.__dict__['operation_name'], **kwargs)
-            
+            result = await base_tool.run_async(self.operation_name, **kwargs)
+
             logger.info(f"Successfully executed {self.name} async with result type: {type(result)}")
             return result
-            
+
         except Exception as e:
             logger.error(f"Error executing {self.name} async: {str(e)}")
             raise
@@ -121,47 +126,90 @@ class ToolRegistry:
     def discover_operations(self, base_tool_class: Type[BaseTool]) -> List[Dict[str, Any]]:
         """
         Discover all operation methods and Schemas of BaseTool class
-        
+
         Args:
             base_tool_class: BaseTool subclass
-            
+
         Returns:
             List of operation information, including method names, Schemas, descriptions, etc.
         """
         operations = []
-        
+
         # Get all Schema classes
+        # Build a mapping from normalized names to Schema classes
+        # Check both class-level and module-level schemas
         schemas = {}
+
+        # 1. Check class-level schemas (e.g., ChartTool)
         for attr_name in dir(base_tool_class):
             attr = getattr(base_tool_class, attr_name)
             if isinstance(attr, type) and issubclass(attr, BaseModel) and attr.__name__.endswith('Schema'):
-                op_name = attr.__name__.replace('Schema', '').lower()
-                schemas[op_name] = attr
-        
+                # Normalize: remove 'Schema' suffix, convert to lowercase, remove underscores
+                schema_base_name = attr.__name__.replace('Schema', '')
+                normalized_name = schema_base_name.replace('_', '').lower()
+                schemas[normalized_name] = attr
+                logger.debug(f"Found class-level schema {attr.__name__} -> normalized: {normalized_name}")
+
+        # 2. Check module-level schemas (e.g., ImageTool)
+        tool_module = inspect.getmodule(base_tool_class)
+        if tool_module:
+            for attr_name in dir(tool_module):
+                if attr_name.startswith('_'):
+                    continue
+                attr = getattr(tool_module, attr_name)
+                if isinstance(attr, type) and issubclass(attr, BaseModel) and attr.__name__.endswith('Schema'):
+                    # Skip if already found at class level
+                    schema_base_name = attr.__name__.replace('Schema', '')
+                    normalized_name = schema_base_name.replace('_', '').lower()
+                    if normalized_name not in schemas:
+                        schemas[normalized_name] = attr
+                        logger.debug(f"Found module-level schema {attr.__name__} -> normalized: {normalized_name}")
+
         # Get all public methods
         for method_name in dir(base_tool_class):
             if method_name.startswith('_'):
                 continue
-                
+
             method = getattr(base_tool_class, method_name)
             if not callable(method):
                 continue
-                
-            # Skip base class methods
+
+            # Skip base class methods and Schema classes themselves
             if method_name in ['run', 'run_async', 'run_batch']:
                 continue
-            
+
+            # Skip if it's a class (like Config or Schema classes)
+            if isinstance(method, type):
+                continue
+
+            # Normalize method name: remove underscores and convert to lowercase
+            normalized_method_name = method_name.replace('_', '').lower()
+
+            # Try to find matching schema
+            matching_schema = schemas.get(normalized_method_name)
+
+            if matching_schema:
+                logger.debug(f"Matched method {method_name} with manual schema {matching_schema.__name__}")
+            else:
+                # Auto-generate schema if not found
+                auto_schema = generate_schema_from_method(method, method_name)
+                if auto_schema:
+                    matching_schema = auto_schema
+                    logger.debug(f"Auto-generated schema for method {method_name}: {auto_schema.__name__}")
+                else:
+                    logger.debug(f"No schema found or generated for method {method_name}")
+
             # Get method information
             operation_info = {
                 'name': method_name,
                 'method': method,
-                'schema': schemas.get(method_name),
+                'schema': matching_schema,
                 'description': inspect.getdoc(method) or f"Execute {method_name} operation",
                 'is_async': inspect.iscoroutinefunction(method)
             }
-            
+
             operations.append(operation_info)
-        
+
         return operations
     
     def _extract_description(self, method, base_tool_name: str, operation_name: str, schema: Optional[Type[BaseModel]] = None) -> str:
@@ -255,20 +303,23 @@ class ToolRegistry:
     def create_all_langchain_tools(self) -> List[LangchainToolAdapter]:
         """
         Create Langchain adapters for all registered BaseTools
-        
+
         Returns:
             List of all Langchain tool adapters
         """
         all_tools = []
-        
-        for tool_name in list_tools():
+
+        # list_tools() returns a list of dicts, extract tool names
+        tool_infos = list_tools()
+        for tool_info in tool_infos:
+            tool_name = tool_info['name']
             try:
                 tools = self.create_langchain_tools(tool_name)
                 all_tools.extend(tools)
             except Exception as e:
                 logger.error(f"Failed to create Langchain tools for {tool_name}: {e}")
-        
-        logger.info(f"Created total {len(all_tools)} Langchain tools from {len(list_tools())} base tools")
+
+        logger.info(f"Created total {len(all_tools)} Langchain tools from {len(tool_infos)} base tools")
         return all_tools
     
     def get_tool(self, name: str) -> Optional[LangchainToolAdapter]:

aiecs/tools/schema_generator.py

@@ -0,0 +1,265 @@
+"""
+Automatic Schema Generation Tool
+
+Automatically generate Pydantic Schema from method signatures and type annotations
+"""
+
+import inspect
+import logging
+from typing import Any, Dict, List, Optional, Type, get_type_hints, Union
+from pydantic import BaseModel, Field, create_model, ConfigDict
+
+logger = logging.getLogger(__name__)
+
+
+def _normalize_type(param_type: Type) -> Type:
+    """
+    Normalize types, handle unsupported types
+
+    Map complex types like pandas.DataFrame to Any
+    """
+    # Get type name
+    type_name = getattr(param_type, '__name__', str(param_type))
+
+    # Check if it's a pandas type
+    if 'DataFrame' in type_name or 'Series' in type_name:
+        return Any
+
+    return param_type
+
+
+def _extract_param_description_from_docstring(docstring: str, param_name: str) -> Optional[str]:
+    """
+    Extract parameter description from docstring
+
+    Supported formats:
+    - Google style: Args: param_name: description
+    - NumPy style: Parameters: param_name : type description
+    """
+    if not docstring:
+        return None
+
+    lines = docstring.split('\n')
+    in_args_section = False
+    current_param = None
+    description_lines = []
+
+    for line in lines:
+        stripped = line.strip()
+
+        # Detect Args/Parameters section
+        if stripped in ['Args:', 'Arguments:', 'Parameters:']:
+            in_args_section = True
+            continue
+
+        # Detect end
+        if in_args_section and stripped in ['Returns:', 'Raises:', 'Yields:', 'Examples:', 'Note:', 'Notes:']:
+            break
+
+        if in_args_section:
+            # Google style: param_name: description or param_name (type): description
+            if ':' in stripped and not stripped.startswith(' '):
+                # Save previous parameter
+                if current_param == param_name and description_lines:
+                    return ' '.join(description_lines).strip()
+
+                # Parse new parameter
+                parts = stripped.split(':', 1)
+                if len(parts) == 2:
+                    # Remove possible type annotation (type)
+                    param_part = parts[0].strip()
+                    if '(' in param_part:
+                        param_part = param_part.split('(')[0].strip()
+
+                    current_param = param_part
+                    description_lines = [parts[1].strip()]
+            elif current_param and stripped:
+                # Continue description
+                description_lines.append(stripped)
+
+    # Check last parameter
+    if current_param == param_name and description_lines:
+        return ' '.join(description_lines).strip()
+
+    return None
+
+
+def generate_schema_from_method(
+    method: callable,
+    method_name: str,
+    base_class: Type[BaseModel] = BaseModel
+) -> Optional[Type[BaseModel]]:
+    """
+    Automatically generate Pydantic Schema from method signature
+
+    Args:
+        method: Method to generate Schema for
+        method_name: Method name
+        base_class: Schema base class
+
+    Returns:
+        Generated Pydantic Schema class, returns None if unable to generate
+    """
+    try:
+        # Get method signature
+        sig = inspect.signature(method)
+
+        # Get type annotations
+        try:
+            type_hints = get_type_hints(method)
+        except Exception as e:
+            logger.debug(f"Failed to get type hints for {method_name}: {e}")
+            type_hints = {}
+
+        # Get docstring
+        docstring = inspect.getdoc(method) or f"Execute {method_name} operation"
+
+        # Extract short description (first line)
+        first_line = docstring.split('\n')[0].strip()
+        schema_description = first_line if first_line else f"Execute {method_name} operation"
+
+        # Build field definitions
+        field_definitions = {}
+
+        for param_name, param in sig.parameters.items():
+            # Skip self parameter
+            if param_name == 'self':
+                continue
+
+            # Get parameter type and normalize
+            param_type = type_hints.get(param_name, Any)
+            param_type = _normalize_type(param_type)
+
+            # Get default value
+            has_default = param.default != inspect.Parameter.empty
+            default_value = param.default if has_default else ...
+
+            # Extract parameter description from docstring
+            field_description = _extract_param_description_from_docstring(docstring, param_name)
+            if not field_description:
+                field_description = f"Parameter {param_name}"
+
+            # Create Field
+            if has_default:
+                if default_value is None:
+                    # Optional parameter
+                    field_definitions[param_name] = (
+                        param_type,
+                        Field(default=None, description=field_description)
+                    )
+                else:
+                    field_definitions[param_name] = (
+                        param_type,
+                        Field(default=default_value, description=field_description)
+                    )
+            else:
+                # Required parameter
+                field_definitions[param_name] = (
+                    param_type,
+                    Field(description=field_description)
+                )
+
+        # If no parameters (except self), return None
+        if not field_definitions:
+            logger.debug(f"No parameters found for {method_name}, skipping schema generation")
+            return None
+
+        # Generate Schema class name
+        schema_name = f"{method_name.title().replace('_', '')}Schema"
+
+        # Create Schema class, allow arbitrary types
+        schema_class = create_model(
+            schema_name,
+            __base__=base_class,
+            __doc__=schema_description,
+            __config__=ConfigDict(arbitrary_types_allowed=True),
+            **field_definitions
+        )
+
+        logger.debug(f"Generated schema {schema_name} for method {method_name}")
+        return schema_class
+
+    except Exception as e:
+        logger.warning(f"Failed to generate schema for {method_name}: {e}")
+        return None
+
+
+def generate_schemas_for_tool(tool_class: Type) -> Dict[str, Type[BaseModel]]:
+    """
+    Generate Schema for all methods of a tool class
+    
+    Args:
+        tool_class: Tool class
+        
+    Returns:
+        Mapping from method names to Schema classes
+    """
+    schemas = {}
+    
+    for method_name in dir(tool_class):
+        # Skip private methods and special methods
+        if method_name.startswith('_'):
+            continue
+        
+        # Skip base class methods
+        if method_name in ['run', 'run_async', 'run_batch']:
+            continue
+        
+        method = getattr(tool_class, method_name)
+        
+        # Skip non-method attributes
+        if not callable(method):
+            continue
+        
+        # Skip classes (like Config, Schema, etc.)
+        if isinstance(method, type):
+            continue
+        
+        # Generate Schema
+        schema = generate_schema_from_method(method, method_name)
+        
+        if schema:
+            # Normalize method name (remove underscores, convert to lowercase)
+            normalized_name = method_name.replace('_', '').lower()
+            schemas[normalized_name] = schema
+            logger.info(f"Generated schema for {method_name}")
+    
+    return schemas
+
+
+
+
+
+# Usage example
+if __name__ == '__main__':
+    import sys
+    sys.path.insert(0, '/home/coder1/python-middleware-dev')
+
+    from aiecs.tools import discover_tools, TOOL_CLASSES
+
+    # Configure logging
+    logging.basicConfig(level=logging.INFO)
+
+    # Discover tools
+    discover_tools()
+
+    # Generate Schema for PandasTool
+    print("Generating Schema for PandasTool:")
+    print("=" * 80)
+
+    pandas_tool = TOOL_CLASSES['pandas']
+    schemas = generate_schemas_for_tool(pandas_tool)
+
+    print(f"\nGenerated {len(schemas)} Schemas:\n")
+
+    # Show first 3 examples
+    for method_name, schema in list(schemas.items())[:3]:
+        print(f"{schema.__name__}:")
+        print(f"  Description: {schema.__doc__}")
+        print(f"  Fields:")
+        for field_name, field_info in schema.model_fields.items():
+            required = "Required" if field_info.is_required() else "Optional"
+            default = f" (default: {field_info.default})" if not field_info.is_required() and field_info.default is not None else ""
+            print(f"    - {field_name}: {field_info.description} [{required}]{default}")
+        print()
+

aiecs/tools/statistics/__init__.py

@@ -0,0 +1,82 @@
+"""
+Statistics and Data Analysis Tools Module
+
+This module contains specialized tools for data analysis and statistical operations:
+- data_loader: Universal data loading from multiple file formats
+- data_profiler: Comprehensive data profiling and quality assessment
+- data_transformer: Data cleaning, transformation, and feature engineering
+- data_visualizer: Smart data visualization and chart generation
+- statistical_analyzer: Advanced statistical analysis and hypothesis testing
+- model_trainer: AutoML and machine learning model training
+- ai_data_analysis_orchestrator: AI-powered end-to-end analysis orchestration
+- ai_insight_generator: AI-driven insight discovery and pattern detection
+- ai_report_orchestrator: AI-powered comprehensive report generation
+"""
+
+# Lazy import strategy to avoid heavy dependencies at import time
+import os
+import logging
+
+logger = logging.getLogger(__name__)
+
+# Define available tools for lazy loading
+_AVAILABLE_STATISTICS_TOOLS = [
+    'data_loader_tool',
+    'data_profiler_tool',
+    'data_transformer_tool',
+    'data_visualizer_tool',
+    'statistical_analyzer_tool',
+    'model_trainer_tool',
+    'ai_data_analysis_orchestrator',
+    'ai_insight_generator_tool',
+    'ai_report_orchestrator_tool'
+]
+
+# Track which tools have been loaded
+_LOADED_STATISTICS_TOOLS = set()
+
+def _lazy_load_statistics_tool(tool_name: str):
+    """Lazy load a specific statistics tool module"""
+    if tool_name in _LOADED_STATISTICS_TOOLS:
+        return
+        
+    try:
+        if tool_name == 'data_loader_tool':
+            from . import data_loader_tool
+        elif tool_name == 'data_profiler_tool':
+            from . import data_profiler_tool
+        elif tool_name == 'data_transformer_tool':
+            from . import data_transformer_tool
+        elif tool_name == 'data_visualizer_tool':
+            from . import data_visualizer_tool
+        elif tool_name == 'statistical_analyzer_tool':
+            from . import statistical_analyzer_tool
+        elif tool_name == 'model_trainer_tool':
+            from . import model_trainer_tool
+        elif tool_name == 'ai_data_analysis_orchestrator':
+            from . import ai_data_analysis_orchestrator
+        elif tool_name == 'ai_insight_generator_tool':
+            from . import ai_insight_generator_tool
+        elif tool_name == 'ai_report_orchestrator_tool':
+            from . import ai_report_orchestrator_tool
+        
+        _LOADED_STATISTICS_TOOLS.add(tool_name)
+        logger.info(f"Successfully loaded statistics tool: {tool_name}")
+        
+    except Exception as e:
+        logger.warning(f"Failed to load statistics tool {tool_name}: {e}")
+
+def load_all_statistics_tools():
+    """Load all available statistics tools"""
+    for tool_name in _AVAILABLE_STATISTICS_TOOLS:
+        _lazy_load_statistics_tool(tool_name)
+
+# Auto-load all tools when module is imported
+# This ensures all tools are registered
+load_all_statistics_tools()
+
+__all__ = [
+    'load_all_statistics_tools',
+    '_lazy_load_statistics_tool'
+]
+