aiecs 1.5.4__tar.gz → 1.8.2__tar.gz

{aiecs-1.5.4 → aiecs-1.8.2}/MANIFEST.in

@@ -11,4 +11,5 @@ global-exclude test*
 global-exclude tests*
 global-exclude htmlcov*
 global-exclude setup.py
-global-exclude *test*.py
+global-exclude test_*.py
+global-exclude *_test.py

{aiecs-1.5.4/aiecs.egg-info → aiecs-1.8.2}/PKG-INFO

@@ -1,11 +1,11 @@
 Metadata-Version: 2.4
 Name: aiecs
-Version: 1.5.4
+Version: 1.8.2
 Summary: AI Execute Services - A middleware framework for AI-powered task execution and tool orchestration
 Author-email: AIECS Team <iretbl@gmail.com>
 License-Expression: MIT
-Project-URL: Homepage, https://github.com/aiecs-team/aiecs
-Project-URL: Bug Tracker, https://github.com/aiecs-team/aiecs/issues
+Project-URL: Homepage, https://github.com/Howmany-Zeta/AI-Execute-Services
+Project-URL: Bug Tracker, https://github.com/Howmany-Zeta/AI-Execute-Services/issues
 Project-URL: Documentation, https://aiecs.readthedocs.io
 Keywords: ai,middleware,llm,orchestration,async,tools
 Classifier: Development Status :: 4 - Beta
@@ -18,7 +18,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Framework :: FastAPI
 Classifier: Framework :: Celery
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
-Requires-Python: <3.13,>=3.10
+Requires-Python: <3.13,>=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: fastapi<0.116.0,>=0.115.12
@@ -38,7 +38,7 @@ Requires-Dist: tenacity<10.0.0,>=9.1.2
 Requires-Dist: flower<3.0.0,>=2.0.1
 Requires-Dist: openai<1.76.0,>=1.68.2
 Requires-Dist: google-cloud-aiplatform<2.0.0,>=1.80.0
-Requires-Dist: google-generativeai<1.0.0,>=0.8.0
+Requires-Dist: google-genai<2.0.0,>=1.0.0
 Requires-Dist: google-api-python-client<3.0.0,>=2.108.0
 Requires-Dist: google-auth<3.0.0,>=2.25.0
 Requires-Dist: google-auth-httplib2<1.0.0,>=0.2.0
@@ -48,19 +48,20 @@ Requires-Dist: langgraph<0.6.0,>=0.5.3
 Requires-Dist: weasel==0.4.1
 Requires-Dist: spacy<4.0.0,>=3.8.7
 Requires-Dist: rake-nltk<2.0.0,>=1.0.6
-Requires-Dist: numpy<3.0.0,>=2.2.6
+Requires-Dist: numpy<3.0.0,>=2.4.1
 Requires-Dist: pandas<3.0.0,>=2.2.3
 Requires-Dist: scipy<2.0.0,>=1.15.3
 Requires-Dist: scikit-learn<2.0.0,>=1.5.0
 Requires-Dist: statsmodels<0.15.0,>=0.14.4
 Requires-Dist: pyreadstat<2.0.0,>=1.2.9
+Requires-Dist: pyarrow<22.0.0,>=21.0.0
 Requires-Dist: tabulate<0.10.0,>=0.9.0
 Requires-Dist: python-docx<2.0.0,>=1.1.2
 Requires-Dist: python-pptx<2.0.0,>=1.0.2
 Requires-Dist: openpyxl<4.0.0,>=3.1.5
 Requires-Dist: pdfplumber<0.12.0,>=0.11.7
 Requires-Dist: pdfminer-six==20250506
-Requires-Dist: tika<3.0.0,>=2.6.0
+Requires-Dist: tika<4.0.0,>=3.1.0
 Requires-Dist: matplotlib<4.0.0,>=3.10.3
 Requires-Dist: seaborn<0.14.0,>=0.13.2
 Requires-Dist: jinja2<4.0.0,>=3.1.6
@@ -82,12 +83,30 @@ Requires-Dist: prometheus-client<1.0.0,>=0.21.1
 Requires-Dist: jaeger-client<5.0.0,>=4.8.0
 Requires-Dist: opentracing<3.0.0,>=2.4.0
 Requires-Dist: psutil<8.0.0,>=7.0.0
+Provides-Extra: dev
+Requires-Dist: pytest<9.0.0,>=8.4.2; extra == "dev"
+Requires-Dist: pytest-asyncio<2.0.0,>=1.2.0; extra == "dev"
+Requires-Dist: pytest-cov<8.0.0,>=7.0.0; extra == "dev"
+Requires-Dist: pytest-timeout<3.0.0,>=2.4.0; extra == "dev"
+Requires-Dist: mypy<2.0.0,>=1.18.2; extra == "dev"
+Requires-Dist: flake8<8.0.0,>=7.3.0; extra == "dev"
+Requires-Dist: black<26.0.0,>=25.1.0; extra == "dev"
+Requires-Dist: types-aiofiles>=25.1.0.20251011; extra == "dev"
+Requires-Dist: types-cachetools>=6.2.0.20251022; extra == "dev"
+Requires-Dist: types-colorama>=0.4.15.20250801; extra == "dev"
+Requires-Dist: types-tqdm>=4.67.0.20250809; extra == "dev"
+Requires-Dist: types-pyyaml>=6.0.12.20250915; extra == "dev"
+Requires-Dist: types-opentracing>=2.4.10.20250622; extra == "dev"
+Requires-Dist: types-lxml>=2025.8.25; extra == "dev"
+Requires-Dist: types-bleach>=6.3.0.20251115; extra == "dev"
 Provides-Extra: docs
 Requires-Dist: sphinx<8.0.0,>=7.0.0; extra == "docs"
 Requires-Dist: sphinx-rtd-theme<3.0.0,>=2.0.0; extra == "docs"
 Requires-Dist: sphinx-autodoc-typehints<3.0.0,>=2.0.0; extra == "docs"
 Requires-Dist: myst-parser<3.0.0,>=2.0.0; extra == "docs"
 Requires-Dist: sphinx-copybutton<1.0.0,>=0.5.0; extra == "docs"
+Provides-Extra: scraper
+Requires-Dist: playwright-stealth<2.0.0,>=1.0.6; extra == "scraper"
 Dynamic: license-file
 
 # AIECS - AI Execute Services
@@ -150,6 +169,22 @@ aiecs-fix-deps
 aiecs-patch-weasel
 ```
 
+### Container Deployment
+
+When installing aiecs in a container (e.g., Docker), you may encounter a warning about scripts not being on PATH:
+
+```
+WARNING: The scripts aiecs, aiecs-check-deps, ... are installed in '/tmp/.local/bin' which is not on PATH.
+```
+
+**Quick Fix:** Add the user bin directory to PATH in your Dockerfile:
+
+```dockerfile
+ENV PATH="${PATH}:/root/.local/bin"
+```
+
+For detailed troubleshooting and best practices, see [Deployment Troubleshooting Guide](docs/user/DEPLOYMENT_TROUBLESHOOTING.md).
+
 ## Quick Start
 
 ### Basic Usage

{aiecs-1.5.4 → aiecs-1.8.2}/README.md

@@ -58,6 +58,22 @@ aiecs-fix-deps
 aiecs-patch-weasel
 ```
 
+### Container Deployment
+
+When installing aiecs in a container (e.g., Docker), you may encounter a warning about scripts not being on PATH:
+
+```
+WARNING: The scripts aiecs, aiecs-check-deps, ... are installed in '/tmp/.local/bin' which is not on PATH.
+```
+
+**Quick Fix:** Add the user bin directory to PATH in your Dockerfile:
+
+```dockerfile
+ENV PATH="${PATH}:/root/.local/bin"
+```
+
+For detailed troubleshooting and best practices, see [Deployment Troubleshooting Guide](docs/user/DEPLOYMENT_TROUBLESHOOTING.md).
+
 ## Quick Start
 
 ### Basic Usage

{aiecs-1.5.4 → aiecs-1.8.2}/aiecs/__init__.py

@@ -5,7 +5,7 @@ A powerful Python middleware framework for building AI-powered applications
 with tool orchestration, task execution, and multi-provider LLM support.
 """
 
-__version__ = "1.5.4"
+__version__ = "1.8.2"
 __author__ = "AIECS Team"
 __email__ = "iretbl@gmail.com"
 

{aiecs-1.5.4 → aiecs-1.8.2}/aiecs/aiecs_client.py

@@ -160,14 +160,21 @@ class AIECS:
                     "mode": "simple",
                 }
 
-            # TODO: Implement direct task execution without Celery
-            # For now, return a placeholder
-            return {
-                "status": "completed",
-                "result": "Simple mode execution (placeholder)",
-                "mode": "simple",
-                "context_id": context.user_id,
-            }
+            # Direct task execution using process_task_async (no Celery required)
+            try:
+                result = await self.process_task_async(context)
+                return {
+                    **result,
+                    "mode": "simple",
+                }
+            except Exception as e:
+                logger.error(f"Simple mode execution failed: {e}", exc_info=True)
+                return {
+                    "status": "failed",
+                    "error": str(e),
+                    "mode": "simple",
+                    "context_id": context.user_id if hasattr(context, "user_id") else "unknown",
+                }
 
         elif self.mode == "full":
             if not self.task_manager:
@@ -203,11 +210,28 @@ class AIECS:
         if not tool:
             raise ValueError(f"Tool '{tool_name}' not found")
 
-        # Prepare parameters with operation
-        tool_params = {**params, "op": operation}
-
-        # Execute tool
-        return await tool.execute(tool_params)
+        # Check if tool has execute() method (some tools like KnowledgeGraphBuilderTool)
+        if hasattr(tool, "execute") and callable(getattr(tool, "execute")):
+            # Tools with execute() method expect kwargs unpacked
+            # Some tools use "op", others use "action" - include both for compatibility
+            tool_params = {**params, "op": operation, "action": operation}
+            return await tool.execute(**tool_params)
+        
+        # Check if tool has run_async() method (BaseTool-based tools)
+        elif hasattr(tool, "run_async") and callable(getattr(tool, "run_async")):
+            # BaseTool.run_async expects op as first parameter, then kwargs
+            return await tool.run_async(operation, **params)
+        
+        # Check if tool has run() method (synchronous BaseTool)
+        elif hasattr(tool, "run") and callable(getattr(tool, "run")):
+            # BaseTool.run expects op as first parameter, then kwargs
+            return tool.run(operation, **params)
+        
+        else:
+            raise ValueError(
+                f"Tool '{tool_name}' does not have an 'execute()', 'run_async()', or 'run()' method. "
+                f"Available methods: {[m for m in dir(tool) if not m.startswith('_')]}"
+            )
 
     async def get_available_tools(self) -> List[Dict[str, Any]]:
         """Get list of all available tools"""

aiecs-1.8.2/aiecs/application/knowledge_graph/builder/data_quality.py

@@ -0,0 +1,302 @@
+"""
+Data Quality Validation for Knowledge Graph Import
+
+Provides validation capabilities to ensure data quality during import,
+including range validation, outlier detection, completeness checks, and
+type consistency validation.
+"""
+
+from typing import Dict, List, Optional, Any, Set, Union
+from dataclasses import dataclass, field
+from enum import Enum
+import logging
+
+logger = logging.getLogger(__name__)
+
+# Check for pandas and numpy availability
+try:
+    import pandas as pd
+    import numpy as np
+    PANDAS_AVAILABLE = True
+except ImportError:
+    PANDAS_AVAILABLE = False
+
+
+class ViolationType(Enum):
+    """Types of data quality violations"""
+    RANGE_VIOLATION = "range_violation"
+    OUTLIER = "outlier"
+    MISSING_VALUE = "missing_value"
+    TYPE_MISMATCH = "type_mismatch"
+
+
+@dataclass
+class ValidationViolation:
+    """
+    Represents a single data quality violation
+    
+    Attributes:
+        violation_type: Type of violation
+        property_name: Property that violated the rule
+        row_id: Identifier of the row with violation
+        value: The violating value
+        expected: Expected value or constraint
+        message: Human-readable description
+    """
+    violation_type: ViolationType
+    property_name: str
+    row_id: Any
+    value: Any
+    expected: Any
+    message: str
+
+
+@dataclass
+class QualityReport:
+    """
+    Data quality validation report
+    
+    Attributes:
+        total_rows: Total number of rows validated
+        violations: List of all violations found
+        completeness: Completeness percentage per property
+        outlier_count: Number of outliers detected per property
+        range_violations: Number of range violations per property
+        type_violations: Number of type violations per property
+        passed: Whether validation passed (no critical violations)
+    """
+    total_rows: int
+    violations: List[ValidationViolation] = field(default_factory=list)
+    completeness: Dict[str, float] = field(default_factory=dict)
+    outlier_count: Dict[str, int] = field(default_factory=dict)
+    range_violations: Dict[str, int] = field(default_factory=dict)
+    type_violations: Dict[str, int] = field(default_factory=dict)
+    passed: bool = True
+    
+    def add_violation(self, violation: ValidationViolation):
+        """Add a violation to the report"""
+        self.violations.append(violation)
+        
+        # Update counts
+        if violation.violation_type == ViolationType.RANGE_VIOLATION:
+            self.range_violations[violation.property_name] = \
+                self.range_violations.get(violation.property_name, 0) + 1
+        elif violation.violation_type == ViolationType.OUTLIER:
+            self.outlier_count[violation.property_name] = \
+                self.outlier_count.get(violation.property_name, 0) + 1
+        elif violation.violation_type == ViolationType.TYPE_MISMATCH:
+            self.type_violations[violation.property_name] = \
+                self.type_violations.get(violation.property_name, 0) + 1
+    
+    def get_summary(self) -> Dict[str, Any]:
+        """Get a summary of the quality report"""
+        return {
+            "total_rows": self.total_rows,
+            "total_violations": len(self.violations),
+            "range_violations": sum(self.range_violations.values()),
+            "outliers": sum(self.outlier_count.values()),
+            "type_violations": sum(self.type_violations.values()),
+            "completeness": self.completeness,
+            "passed": self.passed
+        }
+
+
+@dataclass
+class RangeRule:
+    """Range validation rule for numeric properties"""
+    min_value: Optional[float] = None
+    max_value: Optional[float] = None
+
+
+@dataclass
+class ValidationConfig:
+    """
+    Configuration for data quality validation
+    
+    Attributes:
+        range_rules: Range validation rules per property
+        required_properties: Set of required properties
+        detect_outliers: Whether to detect outliers (3 std devs)
+        fail_on_violations: Whether to fail import on violations
+        max_violation_rate: Maximum allowed violation rate (0.0-1.0)
+    """
+    range_rules: Dict[str, RangeRule] = field(default_factory=dict)
+    required_properties: Set[str] = field(default_factory=set)
+    detect_outliers: bool = False
+    fail_on_violations: bool = False
+    max_violation_rate: float = 0.1  # 10% by default
+
+
+class DataQualityValidator:
+    """
+    Validates data quality during knowledge graph import
+
+    Provides range validation, outlier detection, completeness checks,
+    and type consistency validation.
+    """
+
+    def __init__(self, config: Optional[ValidationConfig] = None):
+        """
+        Initialize validator with configuration
+
+        Args:
+            config: Validation configuration
+        """
+        self.config = config or ValidationConfig()
+        self._property_stats: Dict[str, Dict[str, float]] = {}
+
+    def validate_dataframe(self, df: 'pd.DataFrame', id_column: Optional[str] = None) -> QualityReport:
+        """
+        Validate a pandas DataFrame
+
+        Args:
+            df: DataFrame to validate
+            id_column: Column to use as row identifier
+
+        Returns:
+            QualityReport with validation results
+        """
+        if not PANDAS_AVAILABLE:
+            raise ImportError("pandas and numpy are required for data quality validation")
+
+        report = QualityReport(total_rows=len(df))
+
+        # Use index as row ID if no id_column specified
+        row_ids = df[id_column] if id_column and id_column in df.columns else df.index
+
+        # Check completeness
+        self._check_completeness(df, report)
+
+        # Check required properties
+        self._check_required_properties(df, row_ids, report)
+
+        # Validate ranges
+        self._validate_ranges(df, row_ids, report)
+
+        # Detect outliers
+        if self.config.detect_outliers:
+            self._detect_outliers(df, row_ids, report)
+
+        # Check if validation passed
+        violation_rate = len(report.violations) / max(report.total_rows, 1)
+        if self.config.fail_on_violations and violation_rate > self.config.max_violation_rate:
+            report.passed = False
+
+        return report
+
+    def _check_completeness(self, df: 'pd.DataFrame', report: QualityReport):
+        """Check completeness of properties"""
+        for col in df.columns:
+            non_null_count = df[col].notna().sum()
+            completeness = non_null_count / len(df) if len(df) > 0 else 0.0
+            report.completeness[col] = completeness
+
+    def _check_required_properties(self, df: 'pd.DataFrame', row_ids: Any, report: QualityReport):
+        """Check that required properties are present and non-null"""
+        for prop in self.config.required_properties:
+            if prop not in df.columns:
+                # Property missing entirely
+                violation = ValidationViolation(
+                    violation_type=ViolationType.MISSING_VALUE,
+                    property_name=prop,
+                    row_id="ALL",
+                    value=None,
+                    expected="required property",
+                    message=f"Required property '{prop}' is missing from dataset"
+                )
+                report.add_violation(violation)
+            else:
+                # Check for null values in required property
+                null_mask = df[prop].isna()
+                for idx in df[null_mask].index:
+                    row_id = row_ids.iloc[idx] if hasattr(row_ids, 'iloc') else row_ids[idx]
+                    violation = ValidationViolation(
+                        violation_type=ViolationType.MISSING_VALUE,
+                        property_name=prop,
+                        row_id=row_id,
+                        value=None,
+                        expected="non-null value",
+                        message=f"Required property '{prop}' is null in row {row_id}"
+                    )
+                    report.add_violation(violation)
+
+    def _validate_ranges(self, df: 'pd.DataFrame', row_ids: Any, report: QualityReport):
+        """Validate numeric properties are within specified ranges"""
+        for prop, rule in self.config.range_rules.items():
+            if prop not in df.columns:
+                continue
+
+            # Only validate numeric columns
+            if not pd.api.types.is_numeric_dtype(df[prop]):
+                continue
+
+            # Check min value
+            if rule.min_value is not None:
+                violations_mask = df[prop] < rule.min_value
+                for idx in df[violations_mask].index:
+                    row_id = row_ids.iloc[idx] if hasattr(row_ids, 'iloc') else row_ids[idx]
+                    value = df[prop].iloc[idx]
+                    violation = ValidationViolation(
+                        violation_type=ViolationType.RANGE_VIOLATION,
+                        property_name=prop,
+                        row_id=row_id,
+                        value=value,
+                        expected=f">= {rule.min_value}",
+                        message=f"Value {value} is below minimum {rule.min_value} for property '{prop}' in row {row_id}"
+                    )
+                    report.add_violation(violation)
+
+            # Check max value
+            if rule.max_value is not None:
+                violations_mask = df[prop] > rule.max_value
+                for idx in df[violations_mask].index:
+                    row_id = row_ids.iloc[idx] if hasattr(row_ids, 'iloc') else row_ids[idx]
+                    value = df[prop].iloc[idx]
+                    violation = ValidationViolation(
+                        violation_type=ViolationType.RANGE_VIOLATION,
+                        property_name=prop,
+                        row_id=row_id,
+                        value=value,
+                        expected=f"<= {rule.max_value}",
+                        message=f"Value {value} is above maximum {rule.max_value} for property '{prop}' in row {row_id}"
+                    )
+                    report.add_violation(violation)
+
+    def _detect_outliers(self, df: 'pd.DataFrame', row_ids: Any, report: QualityReport):
+        """Detect outliers using 3 standard deviations rule"""
+        numeric_cols = df.select_dtypes(include=[np.number]).columns
+
+        for col in numeric_cols:
+            # Skip if all values are null
+            if df[col].isna().all():
+                continue
+
+            # Calculate mean and std
+            mean = df[col].mean()
+            std = df[col].std()
+
+            # Skip if std is 0 or NaN
+            if pd.isna(std) or std == 0:
+                continue
+
+            # Store stats for later use
+            self._property_stats[col] = {"mean": mean, "std": std}
+
+            # Detect outliers (beyond 3 standard deviations)
+            lower_bound = mean - 3 * std
+            upper_bound = mean + 3 * std
+            outliers_mask = (df[col] < lower_bound) | (df[col] > upper_bound)
+
+            for idx in df[outliers_mask].index:
+                row_id = row_ids.iloc[idx] if hasattr(row_ids, 'iloc') else row_ids[idx]
+                value = df[col].iloc[idx]
+                violation = ValidationViolation(
+                    violation_type=ViolationType.OUTLIER,
+                    property_name=col,
+                    row_id=row_id,
+                    value=value,
+                    expected=f"within [{lower_bound:.2f}, {upper_bound:.2f}]",
+                    message=f"Value {value} is an outlier (>3 std devs) for property '{col}' in row {row_id}"
+                )
+                report.add_violation(violation)
+