pygeai-orchestration 0.1.0b2__py3-none-any.whl

pygeai_orchestration/core/utils/metrics.py

@@ -0,0 +1,184 @@
+import time
+from collections import defaultdict
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Any, Optional
+
+
+class MetricType(Enum):
+    COUNTER = "counter"
+    GAUGE = "gauge"
+    HISTOGRAM = "histogram"
+    TIMER = "timer"
+
+
+@dataclass
+class Metric:
+    name: str
+    type: MetricType
+    value: float
+    timestamp: datetime = field(default_factory=datetime.now)
+    labels: dict[str, str] = field(default_factory=dict)
+
+
+@dataclass
+class TimerContext:
+    metric_name: str
+    labels: dict[str, str]
+    collector: "MetricsCollector"
+    start_time: float = field(default_factory=time.perf_counter)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        elapsed = time.perf_counter() - self.start_time
+        self.collector.record_timer(self.metric_name, elapsed, labels=self.labels)
+        return False
+
+
+class MetricsCollector:
+    def __init__(self):
+        self._metrics: list[Metric] = []
+        self._counters: dict[str, float] = defaultdict(float)
+        self._gauges: dict[str, float] = {}
+        self._histograms: dict[str, list[float]] = defaultdict(list)
+        self._timers: dict[str, list[float]] = defaultdict(list)
+
+    def increment(self, name: str, value: float = 1.0, labels: Optional[dict[str, str]] = None) -> None:
+        key = self._make_key(name, labels)
+        self._counters[key] += value
+        self._record_metric(name, MetricType.COUNTER, self._counters[key], labels)
+
+    def set_gauge(self, name: str, value: float, labels: Optional[dict[str, str]] = None) -> None:
+        key = self._make_key(name, labels)
+        self._gauges[key] = value
+        self._record_metric(name, MetricType.GAUGE, value, labels)
+
+    def record_histogram(self, name: str, value: float, labels: Optional[dict[str, str]] = None) -> None:
+        key = self._make_key(name, labels)
+        self._histograms[key].append(value)
+        self._record_metric(name, MetricType.HISTOGRAM, value, labels)
+
+    def record_timer(self, name: str, duration: float, labels: Optional[dict[str, str]] = None) -> None:
+        key = self._make_key(name, labels)
+        self._timers[key].append(duration)
+        self._record_metric(name, MetricType.TIMER, duration, labels)
+
+    def timer(self, name: str, labels: Optional[dict[str, str]] = None) -> TimerContext:
+        return TimerContext(name, labels or {}, self)
+
+    def _make_key(self, name: str, labels: Optional[dict[str, str]]) -> str:
+        if not labels:
+            return name
+        label_str = ",".join(f"{k}={v}" for k, v in sorted(labels.items()))
+        return f"{name}{{{label_str}}}"
+
+    def _record_metric(
+        self, name: str, metric_type: MetricType, value: float, labels: Optional[dict[str, str]]
+    ) -> None:
+        metric = Metric(
+            name=name, type=metric_type, value=value, labels=labels or {}
+        )
+        self._metrics.append(metric)
+
+    def get_counter(self, name: str, labels: Optional[dict[str, str]] = None) -> float:
+        key = self._make_key(name, labels)
+        return self._counters.get(key, 0.0)
+
+    def get_gauge(self, name: str, labels: Optional[dict[str, str]] = None) -> Optional[float]:
+        key = self._make_key(name, labels)
+        return self._gauges.get(key)
+
+    def get_histogram_stats(self, name: str, labels: Optional[dict[str, str]] = None) -> dict[str, float]:
+        key = self._make_key(name, labels)
+        values = self._histograms.get(key, [])
+        if not values:
+            return {}
+
+        sorted_values = sorted(values)
+        return {
+            "count": len(values),
+            "sum": sum(values),
+            "min": min(values),
+            "max": max(values),
+            "mean": sum(values) / len(values),
+            "p50": self._percentile(sorted_values, 50),
+            "p95": self._percentile(sorted_values, 95),
+            "p99": self._percentile(sorted_values, 99),
+        }
+
+    def get_timer_stats(self, name: str, labels: Optional[dict[str, str]] = None) -> dict[str, float]:
+        key = self._make_key(name, labels)
+        values = self._timers.get(key, [])
+        if not values:
+            return {}
+
+        sorted_values = sorted(values)
+        return {
+            "count": len(values),
+            "sum": sum(values),
+            "min": min(values),
+            "max": max(values),
+            "mean": sum(values) / len(values),
+            "p50": self._percentile(sorted_values, 50),
+            "p95": self._percentile(sorted_values, 95),
+            "p99": self._percentile(sorted_values, 99),
+        }
+
+    def _percentile(self, sorted_values: list[float], percentile: int) -> float:
+        if not sorted_values:
+            return 0.0
+        index = int(len(sorted_values) * percentile / 100)
+        return sorted_values[min(index, len(sorted_values) - 1)]
+
+    def get_all_metrics(self) -> list[Metric]:
+        return self._metrics.copy()
+
+    def get_summary(self) -> dict[str, Any]:
+        return {
+            "counters": dict(self._counters),
+            "gauges": dict(self._gauges),
+            "histograms": {
+                name: self.get_histogram_stats(name.split("{")[0], self._parse_labels(name))
+                for name in self._histograms.keys()
+            },
+            "timers": {
+                name: self.get_timer_stats(name.split("{")[0], self._parse_labels(name))
+                for name in self._timers.keys()
+            },
+        }
+
+    def _parse_labels(self, key: str) -> Optional[dict[str, str]]:
+        if "{" not in key:
+            return None
+        label_str = key.split("{")[1].rstrip("}")
+        if not label_str:
+            return None
+        labels = {}
+        for pair in label_str.split(","):
+            k, v = pair.split("=")
+            labels[k] = v
+        return labels
+
+    def clear(self) -> None:
+        self._metrics.clear()
+        self._counters.clear()
+        self._gauges.clear()
+        self._histograms.clear()
+        self._timers.clear()
+
+
+class GlobalMetrics:
+    _instance: Optional[MetricsCollector] = None
+
+    @classmethod
+    def get_collector(cls) -> MetricsCollector:
+        if cls._instance is None:
+            cls._instance = MetricsCollector()
+        return cls._instance
+
+    @classmethod
+    def reset(cls) -> None:
+        cls._instance = None

pygeai_orchestration/core/utils/validators.py

@@ -0,0 +1,140 @@
+"""
+Validation utilities for configuration validation.
+
+This module provides validation functions for agent, pattern, and tool
+configurations, ensuring correctness before instantiation.
+"""
+
+from typing import Any, Dict, List
+from pydantic import BaseModel, ValidationError
+
+
+class ValidationResult(BaseModel):
+    """
+    Result of configuration validation.
+
+    :param valid: bool - Whether validation passed.
+    :param errors: List[str] - Validation errors (blocking).
+    :param warnings: List[str] - Validation warnings (non-blocking).
+    """
+
+    valid: bool
+    errors: List[str] = []
+    warnings: List[str] = []
+
+
+def validate_agent_config(config: Dict[str, Any]) -> ValidationResult:
+    """
+    Validate agent configuration dictionary.
+
+    Checks for required fields and validates parameter ranges for
+    agent configuration before creating an AgentConfig instance.
+
+    :param config: Dict[str, Any] - Agent configuration to validate.
+    :return: ValidationResult - Validation outcome with errors/warnings.
+    """
+    errors = []
+    warnings = []
+
+    required_fields = ["name", "model"]
+    for field in required_fields:
+        if field not in config:
+            errors.append(f"Missing required field: {field}")
+
+    if "temperature" in config:
+        temp = config["temperature"]
+        if not isinstance(temp, (int, float)) or temp < 0 or temp > 2:
+            errors.append("Temperature must be between 0 and 2")
+
+    if "max_tokens" in config:
+        tokens = config["max_tokens"]
+        if not isinstance(tokens, int) or tokens < 1:
+            errors.append("Max tokens must be a positive integer")
+
+    return ValidationResult(valid=len(errors) == 0, errors=errors, warnings=warnings)
+
+
+def validate_pattern_config(config: Dict[str, Any]) -> ValidationResult:
+    """
+    Validate pattern configuration dictionary.
+
+    Checks for required fields and validates parameter ranges for
+    pattern configuration before creating a PatternConfig instance.
+
+    :param config: Dict[str, Any] - Pattern configuration to validate.
+    :return: ValidationResult - Validation outcome with errors/warnings.
+    """
+    errors = []
+    warnings = []
+
+    required_fields = ["name", "pattern_type"]
+    for field in required_fields:
+        if field not in config:
+            errors.append(f"Missing required field: {field}")
+
+    if "max_iterations" in config:
+        iterations = config["max_iterations"]
+        if not isinstance(iterations, int) or iterations < 1:
+            errors.append("Max iterations must be a positive integer")
+        elif iterations > 100:
+            warnings.append("Max iterations > 100 may cause performance issues")
+
+    if "timeout" in config:
+        timeout = config["timeout"]
+        if timeout is not None and (not isinstance(timeout, (int, float)) or timeout <= 0):
+            errors.append("Timeout must be a positive number")
+
+    valid_pattern_types = ["reflection", "tool_use", "react", "planning", "multi_agent"]
+    if "pattern_type" in config and config["pattern_type"] not in valid_pattern_types:
+        errors.append(f"Invalid pattern_type. Must be one of: {', '.join(valid_pattern_types)}")
+
+    return ValidationResult(valid=len(errors) == 0, errors=errors, warnings=warnings)
+
+
+def validate_tool_config(config: Dict[str, Any]) -> ValidationResult:
+    """
+    Validate tool configuration dictionary.
+
+    Checks for required fields and validates parameter ranges for
+    tool configuration before creating a ToolConfig instance.
+
+    :param config: Dict[str, Any] - Tool configuration to validate.
+    :return: ValidationResult - Validation outcome with errors/warnings.
+    """
+    errors = []
+    warnings = []
+
+    required_fields = ["name", "description"]
+    for field in required_fields:
+        if field not in config:
+            errors.append(f"Missing required field: {field}")
+
+    if "timeout" in config:
+        timeout = config["timeout"]
+        if timeout is not None and (not isinstance(timeout, (int, float)) or timeout <= 0):
+            errors.append("Timeout must be a positive number")
+
+    valid_categories = ["search", "computation", "data_access", "communication", "custom"]
+    if "category" in config and config["category"] not in valid_categories:
+        errors.append(f"Invalid category. Must be one of: {', '.join(valid_categories)}")
+
+    return ValidationResult(valid=len(errors) == 0, errors=errors, warnings=warnings)
+
+
+def validate_pydantic_model(model_class: type[BaseModel], data: Dict[str, Any]) -> ValidationResult:
+    """
+    Validate data against a Pydantic model.
+
+    Attempts to instantiate the model with the provided data and
+    captures any validation errors.
+
+    :param model_class: type[BaseModel] - Pydantic model class to validate against.
+    :param data: Dict[str, Any] - Data to validate.
+    :return: ValidationResult - Validation outcome with errors.
+    """
+    try:
+        model_class(**data)
+        return ValidationResult(valid=True)
+    except ValidationError as e:
+        errors = [f"{err['loc'][0]}: {err['msg']}" for err in e.errors()]
+        return ValidationResult(valid=False, errors=errors)

pygeai_orchestration/dev/__init__.py

@@ -0,0 +1,15 @@
+"""Development utilities for PyGEAI Orchestration."""
+
+from pygeai_orchestration.dev.debug import DebugTracer, PatternInspector
+from pygeai_orchestration.dev.templates import PatternTemplate, TemplateGenerator
+from pygeai_orchestration.dev.testing import MockAgent, PatternTestCase, create_test_pattern
+
+__all__ = [
+    "DebugTracer",
+    "PatternInspector",
+    "PatternTemplate",
+    "TemplateGenerator",
+    "MockAgent",
+    "PatternTestCase",
+    "create_test_pattern",
+]

pygeai_orchestration/dev/debug.py

@@ -0,0 +1,288 @@
+"""Debugging utilities for pattern development."""
+
+import inspect
+import time
+from contextlib import contextmanager
+from typing import Any, Callable, Dict, List, Optional
+
+from pygeai_orchestration.core.base import BasePattern, PatternResult
+
+
+class DebugTracer:
+    """Trace pattern execution for debugging."""
+
+    def __init__(self, enabled: bool = True):
+        """Initialize debug tracer.
+
+        Args:
+            enabled: Enable tracing
+        """
+        self.enabled = enabled
+        self.traces: List[Dict[str, Any]] = []
+
+    def trace(
+        self,
+        event: str,
+        pattern: Optional[str] = None,
+        data: Optional[Dict[str, Any]] = None
+    ) -> None:
+        """Record a trace event.
+
+        Args:
+            event: Event description
+            pattern: Pattern name
+            data: Additional data
+        """
+        if not self.enabled:
+            return
+
+        trace_entry = {
+            "timestamp": time.time(),
+            "event": event,
+            "pattern": pattern,
+            "data": data or {}
+        }
+
+        self.traces.append(trace_entry)
+
+    def get_traces(self, pattern: Optional[str] = None) -> List[Dict[str, Any]]:
+        """Get recorded traces.
+
+        Args:
+            pattern: Filter by pattern name
+
+        Returns:
+            List of trace entries
+        """
+        if pattern:
+            return [t for t in self.traces if t.get("pattern") == pattern]
+        return self.traces
+
+    def clear(self) -> None:
+        """Clear all traces."""
+        self.traces.clear()
+
+    def format_traces(self, pattern: Optional[str] = None) -> str:
+        """Format traces as readable string.
+
+        Args:
+            pattern: Filter by pattern name
+
+        Returns:
+            Formatted trace output
+        """
+        traces = self.get_traces(pattern)
+
+        if not traces:
+            return "No traces recorded"
+
+        lines = ["Debug Traces:", "=" * 60]
+
+        for i, trace in enumerate(traces, 1):
+            timestamp = time.strftime(
+                "%H:%M:%S",
+                time.localtime(trace["timestamp"])
+            )
+            event = trace["event"]
+            pattern_name = trace.get("pattern", "N/A")
+
+            lines.append(f"\n[{i}] {timestamp} - {pattern_name}")
+            lines.append(f"    Event: {event}")
+
+            if trace.get("data"):
+                lines.append("    Data:")
+                for key, value in trace["data"].items():
+                    value_str = str(value)[:100]
+                    lines.append(f"      {key}: {value_str}")
+
+        return "\n".join(lines)
+
+    @contextmanager
+    def trace_execution(self, pattern_name: str, task: str):
+        """Context manager to trace pattern execution.
+
+        Args:
+            pattern_name: Pattern name
+            task: Task being executed
+
+        Yields:
+            Tracer instance
+        """
+        self.trace("execution_start", pattern_name, {"task": task})
+        start_time = time.time()
+
+        try:
+            yield self
+        except Exception as e:
+            self.trace(
+                "execution_error",
+                pattern_name,
+                {"error": str(e), "type": type(e).__name__}
+            )
+            raise
+        finally:
+            duration = time.time() - start_time
+            self.trace(
+                "execution_end",
+                pattern_name,
+                {"duration": duration}
+            )
+
+
+class PatternInspector:
+    """Inspect and analyze patterns."""
+
+    @staticmethod
+    def inspect_pattern(pattern: BasePattern) -> Dict[str, Any]:
+        """Inspect pattern details.
+
+        Args:
+            pattern: Pattern to inspect
+
+        Returns:
+            Pattern inspection data
+        """
+        pattern_class = type(pattern)
+
+        methods = []
+        for name, method in inspect.getmembers(pattern_class, predicate=inspect.isfunction):
+            if not name.startswith("_"):
+                sig = inspect.signature(method)
+                methods.append({
+                    "name": name,
+                    "signature": str(sig),
+                    "is_async": inspect.iscoroutinefunction(method)
+                })
+
+        return {
+            "class_name": pattern_class.__name__,
+            "module": pattern_class.__module__,
+            "config": pattern.config.model_dump() if hasattr(pattern, "config") else {},
+            "methods": methods,
+            "docstring": inspect.getdoc(pattern_class),
+        }
+
+    @staticmethod
+    def format_inspection(inspection: Dict[str, Any]) -> str:
+        """Format inspection data as readable string.
+
+        Args:
+            inspection: Inspection data
+
+        Returns:
+            Formatted output
+        """
+        lines = [
+            f"Pattern: {inspection['class_name']}",
+            f"Module: {inspection['module']}",
+            "=" * 60,
+        ]
+
+        if inspection.get("docstring"):
+            lines.append(f"\n{inspection['docstring']}\n")
+
+        lines.append("\nConfiguration:")
+        for key, value in inspection.get("config", {}).items():
+            lines.append(f"  {key}: {value}")
+
+        lines.append("\nMethods:")
+        for method in inspection.get("methods", []):
+            async_marker = " (async)" if method["is_async"] else ""
+            lines.append(f"  {method['name']}{method['signature']}{async_marker}")
+
+        return "\n".join(lines)
+
+    @staticmethod
+    def compare_results(result1: PatternResult, result2: PatternResult) -> Dict[str, Any]:
+        """Compare two pattern results.
+
+        Args:
+            result1: First result
+            result2: Second result
+
+        Returns:
+            Comparison data
+        """
+        return {
+            "success_match": result1.success == result2.success,
+            "result_match": result1.result == result2.result,
+            "iterations_diff": result1.iterations - result2.iterations,
+            "metadata_diff": {
+                "added": set(result2.metadata.keys()) - set(result1.metadata.keys()),
+                "removed": set(result1.metadata.keys()) - set(result2.metadata.keys()),
+                "changed": {
+                    k for k in result1.metadata.keys() & result2.metadata.keys()
+                    if result1.metadata[k] != result2.metadata[k]
+                }
+            }
+        }
+
+
+def trace_method(tracer: DebugTracer):
+    """Decorator to trace method calls.
+
+    Args:
+        tracer: Debug tracer instance
+
+    Returns:
+        Decorator function
+    """
+    def decorator(func: Callable) -> Callable:
+        async def async_wrapper(*args, **kwargs):
+            pattern_name = args[0].__class__.__name__ if args else "Unknown"
+            func_name = func.__name__
+
+            tracer.trace(
+                f"method_call: {func_name}",
+                pattern_name,
+                {"args": str(args[1:])[:100], "kwargs": str(kwargs)[:100]}
+            )
+
+            try:
+                result = await func(*args, **kwargs)
+                tracer.trace(
+                    f"method_return: {func_name}",
+                    pattern_name,
+                    {"result": str(result)[:100]}
+                )
+                return result
+            except Exception as e:
+                tracer.trace(
+                    f"method_error: {func_name}",
+                    pattern_name,
+                    {"error": str(e)}
+                )
+                raise
+
+        def sync_wrapper(*args, **kwargs):
+            pattern_name = args[0].__class__.__name__ if args else "Unknown"
+            func_name = func.__name__
+
+            tracer.trace(
+                f"method_call: {func_name}",
+                pattern_name,
+                {"args": str(args[1:])[:100], "kwargs": str(kwargs)[:100]}
+            )
+
+            try:
+                result = func(*args, **kwargs)
+                tracer.trace(
+                    f"method_return: {func_name}",
+                    pattern_name,
+                    {"result": str(result)[:100]}
+                )
+                return result
+            except Exception as e:
+                tracer.trace(
+                    f"method_error: {func_name}",
+                    pattern_name,
+                    {"error": str(e)}
+                )
+                raise
+
+        if inspect.iscoroutinefunction(func):
+            return async_wrapper
+        else:
+            return sync_wrapper
+
+    return decorator