aiecs 1.0.1__py3-none-any.whl → 1.7.6__py3-none-any.whl

aiecs/application/executors/operation_executor.py

@@ -1,6 +1,6 @@
 import asyncio
 import logging
-from typing import Dict, List, Any, Optional
+from typing import Dict, List, Any
 from aiecs.tools import get_tool
 from aiecs.tools.tool_executor import ToolExecutor
 from aiecs.utils.execution_utils import ExecutionUtils
@@ -14,27 +14,33 @@ class OperationExecutor:
     Core logic for handling operation execution
     """
 
-    def __init__(self, tool_executor: ToolExecutor, execution_utils: ExecutionUtils, config: Dict[str, Any]):
+    def __init__(
+        self,
+        tool_executor: ToolExecutor,
+        execution_utils: ExecutionUtils,
+        config: Dict[str, Any],
+    ):
         self.tool_executor = tool_executor
         self.execution_utils = execution_utils
         self.config = config
-        self._tool_instances = {}
-        self.semaphore = asyncio.Semaphore(config.get('rate_limit_requests_per_second', 5))
+        self._tool_instances: Dict[str, Any] = {}
+        self.semaphore = asyncio.Semaphore(config.get("rate_limit_requests_per_second", 5))
 
     def _filter_tool_params(self, params: Dict[str, Any]) -> Dict[str, Any]:
         """
         Filter out system-related parameters, keeping only parameters needed by tool methods
         """
         # System-related parameters that should not be passed to tool methods
-        system_params = {'user_id', 'task_id', 'op'}
+        system_params = {"user_id", "task_id", "op"}
         return {k: v for k, v in params.items() if k not in system_params}
 
     def _filter_tool_call_params(self, params: Dict[str, Any]) -> Dict[str, Any]:
         """
         Filter out system-related parameters in tool calls, but keep 'op' parameter (needed by BaseTool.run())
         """
-        # Only filter user and task IDs, keep 'op' parameter for BaseTool.run() to use
-        system_params = {'user_id', 'task_id'}
+        # Only filter user and task IDs, keep 'op' parameter for BaseTool.run()
+        # to use
+        system_params = {"user_id", "task_id"}
         return {k: v for k, v in params.items() if k not in system_params}
 
     async def execute_operation(self, operation_spec: str, params: Dict[str, Any]) -> Any:
@@ -44,7 +50,9 @@ class OperationExecutor:
         if "." not in operation_spec:
             raise ValueError(f"Invalid operation spec: {operation_spec}, expected 'tool_name.operation_name'")
 
-        tool_name, operation_name = operation_spec.split(".", 1)
+        parts = operation_spec.split(".", 1)
+        tool_name: str = parts[0]
+        operation_name: str = parts[1]
 
         # Get or create tool instance
         if tool_name not in self._tool_instances:
@@ -69,29 +77,37 @@ class OperationExecutor:
         Batch execute operations with rate limiting
         """
         results = []
-        batch_size = self.config.get('batch_size', 10)
-        rate_limit = self.config.get('rate_limit_requests_per_second', 5)
+        batch_size = self.config.get("batch_size", 10)
+        rate_limit = self.config.get("rate_limit_requests_per_second", 5)
 
         for i in range(0, len(operations), batch_size):
-            batch = operations[i:i + batch_size]
+            batch = operations[i : i + batch_size]
             batch_results = await asyncio.gather(
                 *[self.execute_operation(op["operation"], op.get("params", {})) for op in batch],
-                return_exceptions=True
+                return_exceptions=True,
             )
             results.extend(batch_results)
             await asyncio.sleep(1.0 / rate_limit)
 
         return results
 
-    async def execute_operations_sequence(self, operations: List[Dict[str, Any]], user_id: str, task_id: str,
-                                        stop_on_failure: bool = False, save_callback=None) -> List[TaskStepResult]:
+    async def execute_operations_sequence(
+        self,
+        operations: List[Dict[str, Any]],
+        user_id: str,
+        task_id: str,
+        stop_on_failure: bool = False,
+        save_callback=None,
+    ) -> List[TaskStepResult]:
         """
         Execute operations sequence sequentially, with option to stop on failure
         """
-        results = []
+        results: List[TaskStepResult] = []
 
         for step, op_info in enumerate(operations):
             operation_spec = op_info.get("operation")
+            if not isinstance(operation_spec, str):
+                raise ValueError(f"Invalid operation spec: {operation_spec}, expected string")
             params = op_info.get("params", {})
 
             # Process parameter references
@@ -104,7 +120,7 @@ class OperationExecutor:
                     result=result,
                     completed=True,
                     message=f"Completed operation {operation_spec}",
-                    status=TaskStatus.COMPLETED.value
+                    status=TaskStatus.COMPLETED.value,
                 )
             except Exception as e:
                 step_result = TaskStepResult(
@@ -114,7 +130,7 @@ class OperationExecutor:
                     message=f"Failed to execute {operation_spec}",
                     status=TaskStatus.FAILED.value,
                     error_code=ErrorCode.EXECUTION_ERROR.value,
-                    error_message=str(e)
+                    error_message=str(e),
                 )
 
                 if stop_on_failure:
@@ -138,9 +154,9 @@ class OperationExecutor:
         processed = {}
 
         for name, value in params.items():
-            if isinstance(value, str) and value.startswith('$result['):
+            if isinstance(value, str) and value.startswith("$result["):
                 try:
-                    ref_parts = value[8:].split(']', 1)
+                    ref_parts = value[8:].split("]", 1)
                     idx = int(ref_parts[0])
 
                     if idx >= len(results):
@@ -148,9 +164,10 @@ class OperationExecutor:
 
                     ref_value = results[idx].result
 
-                    # Handle nested attribute access, such as $result[0].data.field
-                    if len(ref_parts) > 1 and ref_parts[1].startswith('.'):
-                        for attr in ref_parts[1][1:].split('.'):
+                    # Handle nested attribute access, such as
+                    # $result[0].data.field
+                    if len(ref_parts) > 1 and ref_parts[1].startswith("."):
+                        for attr in ref_parts[1][1:].split("."):
                             if attr:
                                 if isinstance(ref_value, dict):
                                     ref_value = ref_value.get(attr)
@@ -171,14 +188,14 @@ class OperationExecutor:
         Execute batch tool calls with rate limiting
         """
         results = []
-        batch_size = self.config.get('batch_size', 10)
-        rate_limit = self.config.get('rate_limit_requests_per_second', 5)
+        batch_size = self.config.get("batch_size", 10)
+        rate_limit = self.config.get("rate_limit_requests_per_second", 5)
 
         for i in range(0, len(tool_calls), batch_size):
-            batch = tool_calls[i:i + batch_size]
+            batch = tool_calls[i : i + batch_size]
             batch_results = await asyncio.gather(
                 *[self._execute_tool_call(call, tool_executor_func) for call in batch],
-                return_exceptions=True
+                return_exceptions=True,
             )
             results.extend(batch_results)
             await asyncio.sleep(1.0 / rate_limit)
@@ -190,11 +207,14 @@ class OperationExecutor:
         Execute a single tool call with rate limiting
         """
         async with self.semaphore:
-            tool_name = call.get("tool")
+            tool_name_raw = call.get("tool")
+            if not isinstance(tool_name_raw, str):
+                raise ValueError(f"Invalid tool name: {tool_name_raw}, expected string")
+            tool_name: str = tool_name_raw
             params = call.get("params", {})
 
             # Use context-aware caching
-            if self.config.get('enable_cache', True):
+            if self.config.get("enable_cache", True):
                 user_id = params.get("user_id", "anonymous")
                 task_id = params.get("task_id", "none")
                 cache_key = self.execution_utils.generate_cache_key("tool_call", user_id, task_id, (), params)
@@ -211,14 +231,16 @@ class OperationExecutor:
                 if tool_name not in self._tool_instances:
                     self._tool_instances[tool_name] = get_tool(tool_name)
                 tool = self._tool_instances[tool_name]
-                
-                # Filter parameters, remove system-related parameters (but keep 'op' parameter)
+
+                # Filter parameters, remove system-related parameters (but keep
+                # 'op' parameter)
                 tool_params = self._filter_tool_call_params(params)
-                # Execute through BaseTool.run method, passing filtered parameters
+                # Execute through BaseTool.run method, passing filtered
+                # parameters
                 result = await self.tool_executor.execute_async(tool, "run", **tool_params)
 
             # Cache result
-            if self.config.get('enable_cache', True):
+            if self.config.get("enable_cache", True):
                 self.execution_utils.add_to_cache(cache_key, result)
 
             return result
@@ -230,7 +252,7 @@ class OperationExecutor:
         import re
 
         tool_calls = []
-        tool_pattern = r'\{\{(\w+)\((.*?)\)\}\}'
+        tool_pattern = r"\{\{(\w+)\((.*?)\)\}\}"
         matches = re.finditer(tool_pattern, description)
 
         for match in matches:
@@ -256,10 +278,7 @@ class OperationExecutor:
 
                 params[param_name] = param_value
 
-            tool_calls.append({
-                "tool": tool_name,
-                "params": params
-            })
+            tool_calls.append({"tool": tool_name, "params": params})
 
         return tool_calls
 
@@ -271,9 +290,11 @@ class OperationExecutor:
 
         for i, op_info in enumerate(operations):
             operation_spec = op_info.get("operation")
+            if not isinstance(operation_spec, str):
+                raise ValueError(f"Invalid operation spec: {operation_spec}, expected string")
             params = op_info.get("params", {})
 
-            async def execute_single_op(spec, p, index):
+            async def execute_single_op(spec: str, p: Dict[str, Any], index: int) -> TaskStepResult:
                 try:
                     result = await self.execute_operation(spec, p)
                     return TaskStepResult(
@@ -281,7 +302,7 @@ class OperationExecutor:
                         result=result,
                         completed=True,
                         message=f"Completed parallel operation {spec}",
-                        status=TaskStatus.COMPLETED.value
+                        status=TaskStatus.COMPLETED.value,
                     )
                 except Exception as e:
                     return TaskStepResult(
@@ -291,7 +312,7 @@ class OperationExecutor:
                         message=f"Failed parallel operation {spec}",
                         status=TaskStatus.FAILED.value,
                         error_code=ErrorCode.EXECUTION_ERROR.value,
-                        error_message=str(e)
+                        error_message=str(e),
                     )
 
             tasks.append(execute_single_op(operation_spec, params, i))
@@ -299,19 +320,23 @@ class OperationExecutor:
         results = await asyncio.gather(*tasks, return_exceptions=True)
 
         # Handle exception results
-        processed_results = []
+        processed_results: List[TaskStepResult] = []
         for i, result in enumerate(results):
             if isinstance(result, Exception):
-                processed_results.append(TaskStepResult(
-                    step=f"parallel_{i}_error",
-                    result=None,
-                    completed=False,
-                    message=f"Parallel operation failed with exception",
-                    status=TaskStatus.FAILED.value,
-                    error_code=ErrorCode.EXECUTION_ERROR.value,
-                    error_message=str(result)
-                ))
+                processed_results.append(
+                    TaskStepResult(
+                        step=f"parallel_{i}_error",
+                        result=None,
+                        completed=False,
+                        message="Parallel operation failed with exception",
+                        status=TaskStatus.FAILED.value,
+                        error_code=ErrorCode.EXECUTION_ERROR.value,
+                        error_message=str(result),
+                    )
+                )
             else:
+                # result is TaskStepResult here because execute_single_op always returns TaskStepResult
+                assert isinstance(result, TaskStepResult), f"Expected TaskStepResult, got {type(result)}"
                 processed_results.append(result)
 
         return processed_results
@@ -334,8 +359,8 @@ class OperationExecutor:
             "tool_names": list(self._tool_instances.keys()),
             "semaphore_value": self.semaphore._value,
             "config": {
-                "batch_size": self.config.get('batch_size', 10),
-                "rate_limit": self.config.get('rate_limit_requests_per_second', 5),
-                "enable_cache": self.config.get('enable_cache', True)
-            }
+                "batch_size": self.config.get("batch_size", 10),
+                "rate_limit": self.config.get("rate_limit_requests_per_second", 5),
+                "enable_cache": self.config.get("enable_cache", True),
+            },
         }

aiecs/application/knowledge_graph/__init__.py

@@ -0,0 +1,7 @@
+"""
+Knowledge Graph Application Layer
+
+This module contains application services and use cases for knowledge graph operations.
+"""
+
+__all__ = []

aiecs/application/knowledge_graph/builder/__init__.py

@@ -0,0 +1,37 @@
+"""
+Knowledge Graph Builder Pipeline
+
+Orchestrates document-to-graph conversion workflow.
+"""
+
+from aiecs.application.knowledge_graph.builder.graph_builder import (
+    GraphBuilder,
+)
+from aiecs.application.knowledge_graph.builder.document_builder import (
+    DocumentGraphBuilder,
+)
+from aiecs.application.knowledge_graph.builder.text_chunker import TextChunker
+from aiecs.application.knowledge_graph.builder.schema_mapping import (
+    SchemaMapping,
+    EntityMapping,
+    RelationMapping,
+    PropertyTransformation,
+    TransformationType,
+)
+from aiecs.application.knowledge_graph.builder.structured_pipeline import (
+    StructuredDataPipeline,
+    ImportResult,
+)
+
+__all__ = [
+    "GraphBuilder",
+    "DocumentGraphBuilder",
+    "TextChunker",
+    "SchemaMapping",
+    "EntityMapping",
+    "RelationMapping",
+    "PropertyTransformation",
+    "TransformationType",
+    "StructuredDataPipeline",
+    "ImportResult",
+]

aiecs/application/knowledge_graph/builder/data_quality.py

@@ -0,0 +1,302 @@
+"""
+Data Quality Validation for Knowledge Graph Import
+
+Provides validation capabilities to ensure data quality during import,
+including range validation, outlier detection, completeness checks, and
+type consistency validation.
+"""
+
+from typing import Dict, List, Optional, Any, Set, Union
+from dataclasses import dataclass, field
+from enum import Enum
+import logging
+
+logger = logging.getLogger(__name__)
+
+# Check for pandas and numpy availability
+try:
+    import pandas as pd
+    import numpy as np
+    PANDAS_AVAILABLE = True
+except ImportError:
+    PANDAS_AVAILABLE = False
+
+
+class ViolationType(Enum):
+    """Types of data quality violations"""
+    RANGE_VIOLATION = "range_violation"
+    OUTLIER = "outlier"
+    MISSING_VALUE = "missing_value"
+    TYPE_MISMATCH = "type_mismatch"
+
+
+@dataclass
+class ValidationViolation:
+    """
+    Represents a single data quality violation
+    
+    Attributes:
+        violation_type: Type of violation
+        property_name: Property that violated the rule
+        row_id: Identifier of the row with violation
+        value: The violating value
+        expected: Expected value or constraint
+        message: Human-readable description
+    """
+    violation_type: ViolationType
+    property_name: str
+    row_id: Any
+    value: Any
+    expected: Any
+    message: str
+
+
+@dataclass
+class QualityReport:
+    """
+    Data quality validation report
+    
+    Attributes:
+        total_rows: Total number of rows validated
+        violations: List of all violations found
+        completeness: Completeness percentage per property
+        outlier_count: Number of outliers detected per property
+        range_violations: Number of range violations per property
+        type_violations: Number of type violations per property
+        passed: Whether validation passed (no critical violations)
+    """
+    total_rows: int
+    violations: List[ValidationViolation] = field(default_factory=list)
+    completeness: Dict[str, float] = field(default_factory=dict)
+    outlier_count: Dict[str, int] = field(default_factory=dict)
+    range_violations: Dict[str, int] = field(default_factory=dict)
+    type_violations: Dict[str, int] = field(default_factory=dict)
+    passed: bool = True
+    
+    def add_violation(self, violation: ValidationViolation):
+        """Add a violation to the report"""
+        self.violations.append(violation)
+        
+        # Update counts
+        if violation.violation_type == ViolationType.RANGE_VIOLATION:
+            self.range_violations[violation.property_name] = \
+                self.range_violations.get(violation.property_name, 0) + 1
+        elif violation.violation_type == ViolationType.OUTLIER:
+            self.outlier_count[violation.property_name] = \
+                self.outlier_count.get(violation.property_name, 0) + 1
+        elif violation.violation_type == ViolationType.TYPE_MISMATCH:
+            self.type_violations[violation.property_name] = \
+                self.type_violations.get(violation.property_name, 0) + 1
+    
+    def get_summary(self) -> Dict[str, Any]:
+        """Get a summary of the quality report"""
+        return {
+            "total_rows": self.total_rows,
+            "total_violations": len(self.violations),
+            "range_violations": sum(self.range_violations.values()),
+            "outliers": sum(self.outlier_count.values()),
+            "type_violations": sum(self.type_violations.values()),
+            "completeness": self.completeness,
+            "passed": self.passed
+        }
+
+
+@dataclass
+class RangeRule:
+    """Range validation rule for numeric properties"""
+    min_value: Optional[float] = None
+    max_value: Optional[float] = None
+
+
+@dataclass
+class ValidationConfig:
+    """
+    Configuration for data quality validation
+    
+    Attributes:
+        range_rules: Range validation rules per property
+        required_properties: Set of required properties
+        detect_outliers: Whether to detect outliers (3 std devs)
+        fail_on_violations: Whether to fail import on violations
+        max_violation_rate: Maximum allowed violation rate (0.0-1.0)
+    """
+    range_rules: Dict[str, RangeRule] = field(default_factory=dict)
+    required_properties: Set[str] = field(default_factory=set)
+    detect_outliers: bool = False
+    fail_on_violations: bool = False
+    max_violation_rate: float = 0.1  # 10% by default
+
+
+class DataQualityValidator:
+    """
+    Validates data quality during knowledge graph import
+
+    Provides range validation, outlier detection, completeness checks,
+    and type consistency validation.
+    """
+
+    def __init__(self, config: Optional[ValidationConfig] = None):
+        """
+        Initialize validator with configuration
+
+        Args:
+            config: Validation configuration
+        """
+        self.config = config or ValidationConfig()
+        self._property_stats: Dict[str, Dict[str, float]] = {}
+
+    def validate_dataframe(self, df: 'pd.DataFrame', id_column: Optional[str] = None) -> QualityReport:
+        """
+        Validate a pandas DataFrame
+
+        Args:
+            df: DataFrame to validate
+            id_column: Column to use as row identifier
+
+        Returns:
+            QualityReport with validation results
+        """
+        if not PANDAS_AVAILABLE:
+            raise ImportError("pandas and numpy are required for data quality validation")
+
+        report = QualityReport(total_rows=len(df))
+
+        # Use index as row ID if no id_column specified
+        row_ids = df[id_column] if id_column and id_column in df.columns else df.index
+
+        # Check completeness
+        self._check_completeness(df, report)
+
+        # Check required properties
+        self._check_required_properties(df, row_ids, report)
+
+        # Validate ranges
+        self._validate_ranges(df, row_ids, report)
+
+        # Detect outliers
+        if self.config.detect_outliers:
+            self._detect_outliers(df, row_ids, report)
+
+        # Check if validation passed
+        violation_rate = len(report.violations) / max(report.total_rows, 1)
+        if self.config.fail_on_violations and violation_rate > self.config.max_violation_rate:
+            report.passed = False
+
+        return report
+
+    def _check_completeness(self, df: 'pd.DataFrame', report: QualityReport):
+        """Check completeness of properties"""
+        for col in df.columns:
+            non_null_count = df[col].notna().sum()
+            completeness = non_null_count / len(df) if len(df) > 0 else 0.0
+            report.completeness[col] = completeness
+
+    def _check_required_properties(self, df: 'pd.DataFrame', row_ids: Any, report: QualityReport):
+        """Check that required properties are present and non-null"""
+        for prop in self.config.required_properties:
+            if prop not in df.columns:
+                # Property missing entirely
+                violation = ValidationViolation(
+                    violation_type=ViolationType.MISSING_VALUE,
+                    property_name=prop,
+                    row_id="ALL",
+                    value=None,
+                    expected="required property",
+                    message=f"Required property '{prop}' is missing from dataset"
+                )
+                report.add_violation(violation)
+            else:
+                # Check for null values in required property
+                null_mask = df[prop].isna()
+                for idx in df[null_mask].index:
+                    row_id = row_ids.iloc[idx] if hasattr(row_ids, 'iloc') else row_ids[idx]
+                    violation = ValidationViolation(
+                        violation_type=ViolationType.MISSING_VALUE,
+                        property_name=prop,
+                        row_id=row_id,
+                        value=None,
+                        expected="non-null value",
+                        message=f"Required property '{prop}' is null in row {row_id}"
+                    )
+                    report.add_violation(violation)
+
+    def _validate_ranges(self, df: 'pd.DataFrame', row_ids: Any, report: QualityReport):
+        """Validate numeric properties are within specified ranges"""
+        for prop, rule in self.config.range_rules.items():
+            if prop not in df.columns:
+                continue
+
+            # Only validate numeric columns
+            if not pd.api.types.is_numeric_dtype(df[prop]):
+                continue
+
+            # Check min value
+            if rule.min_value is not None:
+                violations_mask = df[prop] < rule.min_value
+                for idx in df[violations_mask].index:
+                    row_id = row_ids.iloc[idx] if hasattr(row_ids, 'iloc') else row_ids[idx]
+                    value = df[prop].iloc[idx]
+                    violation = ValidationViolation(
+                        violation_type=ViolationType.RANGE_VIOLATION,
+                        property_name=prop,
+                        row_id=row_id,
+                        value=value,
+                        expected=f">= {rule.min_value}",
+                        message=f"Value {value} is below minimum {rule.min_value} for property '{prop}' in row {row_id}"
+                    )
+                    report.add_violation(violation)
+
+            # Check max value
+            if rule.max_value is not None:
+                violations_mask = df[prop] > rule.max_value
+                for idx in df[violations_mask].index:
+                    row_id = row_ids.iloc[idx] if hasattr(row_ids, 'iloc') else row_ids[idx]
+                    value = df[prop].iloc[idx]
+                    violation = ValidationViolation(
+                        violation_type=ViolationType.RANGE_VIOLATION,
+                        property_name=prop,
+                        row_id=row_id,
+                        value=value,
+                        expected=f"<= {rule.max_value}",
+                        message=f"Value {value} is above maximum {rule.max_value} for property '{prop}' in row {row_id}"
+                    )
+                    report.add_violation(violation)
+
+    def _detect_outliers(self, df: 'pd.DataFrame', row_ids: Any, report: QualityReport):
+        """Detect outliers using 3 standard deviations rule"""
+        numeric_cols = df.select_dtypes(include=[np.number]).columns
+
+        for col in numeric_cols:
+            # Skip if all values are null
+            if df[col].isna().all():
+                continue
+
+            # Calculate mean and std
+            mean = df[col].mean()
+            std = df[col].std()
+
+            # Skip if std is 0 or NaN
+            if pd.isna(std) or std == 0:
+                continue
+
+            # Store stats for later use
+            self._property_stats[col] = {"mean": mean, "std": std}
+
+            # Detect outliers (beyond 3 standard deviations)
+            lower_bound = mean - 3 * std
+            upper_bound = mean + 3 * std
+            outliers_mask = (df[col] < lower_bound) | (df[col] > upper_bound)
+
+            for idx in df[outliers_mask].index:
+                row_id = row_ids.iloc[idx] if hasattr(row_ids, 'iloc') else row_ids[idx]
+                value = df[col].iloc[idx]
+                violation = ValidationViolation(
+                    violation_type=ViolationType.OUTLIER,
+                    property_name=col,
+                    row_id=row_id,
+                    value=value,
+                    expected=f"within [{lower_bound:.2f}, {upper_bound:.2f}]",
+                    message=f"Value {value} is an outlier (>3 std devs) for property '{col}' in row {row_id}"
+                )
+                report.add_violation(violation)
+