levelapp 0.1.15__py3-none-any.whl

levelapp/core/session.py

@@ -0,0 +1,336 @@
+"""levelapp/core/session.py"""
+
+import asyncio
+import threading
+
+from abc import ABC
+
+from dataclasses import dataclass, field
+from typing import Dict, List, Any
+
+from datetime import datetime
+from humanize import precisedelta
+
+from levelapp.workflow import MainFactory, WorkflowConfig
+from levelapp.workflow.base import BaseWorkflow
+from levelapp.aspects import MetricType, ExecutionMetrics, MonitoringAspect, logger
+from levelapp.workflow.context import WorkflowContextBuilder
+
+
+class TemporalStatusMixin(ABC):
+    started_at: datetime | None
+    ended_at: datetime | None
+
+    @property
+    def is_active(self) -> bool:
+        """Check if the session is currently active."""
+        return self.ended_at is None
+
+    @property
+    def duration(self) -> float | None:
+        """Calculate the duration of the session in seconds."""
+        if not self.is_active:
+            return (self.ended_at - self.started_at).total_seconds()
+        return None
+
+
+@dataclass
+class SessionMetadata(TemporalStatusMixin):
+    """Metadata for an evaluation session."""
+
+    session_name: str
+    started_at: datetime | None = None
+    ended_at: datetime | None = None
+    total_executions: int = 0
+    total_duration: float = 0.0
+    steps: Dict[str, "StepMetadata"] = field(default_factory=dict)
+
+
+@dataclass
+class StepMetadata(TemporalStatusMixin):
+    """Metadata for a specific step within an evaluation session."""
+
+    step_name: str
+    session_name: str
+    started_at: datetime | None = None
+    ended_at: datetime | None = None
+    memory_peak_mb: float | None = None
+    error_count: int = 0
+    procedures_stats: List[ExecutionMetrics] | None = None
+
+
+class StepContext:
+    """Context manager for an evaluation step within an EvaluationSession."""
+
+    def __init__(
+        self,
+        session: "EvaluationSession",
+        step_name: str,
+        category: MetricType,
+    ):
+        """
+        Initialize StepContext.
+
+        Args:
+            session (EvaluationSession): Evaluation session.
+            step_name (str): Step name.
+            category (MetricType): Metric type.
+        """
+        self.session = session
+        self.step_name = step_name
+        self.category = category
+
+        self.step_meta: StepMetadata | None = None
+        self.full_step_name = f"<{session.session_name}:{step_name}>"
+        self._monitored_func = None
+        self._func_gen = None
+
+    def __enter__(self):
+        with self.session.lock:
+            self.step_meta = StepMetadata(
+                step_name=self.step_name,
+                session_name=self.session.session_name,
+                started_at=datetime.now(),
+            )
+            self.session.session_metadata.steps[self.step_name] = self.step_meta
+
+        if self.session.enable_monitoring:
+            # Wrap with FunctionMonitor
+            self._monitored_func = self.session.monitor.monitor(
+                name=self.full_step_name,
+                category=self.category,
+                enable_timing=True,
+                track_memory=True,
+                verbose=self.session.verbose,
+            )(self._step_wrapper)
+
+            # Start monitoring
+            try:
+                self._func_gen = self._monitored_func()
+                next(self._func_gen)  # Enter monitoring
+            except Exception as e:
+                logger.error(
+                    f"[StepContext] Failed to initialize monitoring for {self.full_step_name}:\n{e}"
+                )
+                raise
+
+        return self  # returning self allows nested instrumentation
+
+    # noinspection PyMethodMayBeStatic
+    def _step_wrapper(self):
+        yield  # Actual user step execution happens here
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.session.enable_monitoring:
+            try:
+                next(self._func_gen)  # Exit monitoring
+            except StopIteration:
+                pass
+
+        with self.session.lock:
+            self.step_meta.ended_at = datetime.now()
+
+            if exc_type:
+                self.step_meta.error_count += 1
+
+            self.session.session_metadata.total_executions += 1
+
+            if self.session.enable_monitoring and self.step_meta.duration:
+                self.session.monitor.update_procedure_duration(
+                    name=self.full_step_name, value=self.step_meta.duration
+                )
+                self.session.session_metadata.total_duration += self.step_meta.duration
+
+        return False
+
+
+class EvaluationSession:
+    """Context manager for LLM evaluation sessions with integrated monitoring."""
+
+    def __init__(
+        self,
+        session_name: str = "test-session",
+        workflow_config: WorkflowConfig | None = None,
+        enable_monitoring: bool = True,
+        verbose: bool = False,
+    ):
+        """
+        Initialize Evaluation Session.
+
+        Args:
+            session_name (str): Name of the session
+            workflow_config (WorkflowConfig): Workflow configuration.
+            enable_monitoring (bool): Switch monitoring on. Defaults to True.
+            verbose (bool): Verbose mode. Defaults to False.
+        """
+        self._NAME = self.__class__.__name__
+
+        self.session_name = session_name
+        self.workflow_config = workflow_config
+        self.enable_monitoring = enable_monitoring
+        self.verbose = verbose
+
+        self.workflow: BaseWorkflow | None = None
+
+        self.session_metadata = SessionMetadata(session_name=session_name)
+        self.monitor = MonitoringAspect if enable_monitoring else None
+        self._lock = threading.RLock()
+
+        logger.info("[EvaluationSession] Evaluation session initialized.")
+
+    @property
+    def lock(self):
+        return self._lock
+
+    def __enter__(self):
+        self.session_metadata.started_at = datetime.now()
+
+        # Instantiate workflow if not already
+        if not self.workflow:
+            if not self.workflow_config:
+                raise ValueError(f"{self._NAME}: Workflow configuration must be provided")
+
+            context_builder = WorkflowContextBuilder(self.workflow_config)
+            context = context_builder.build()
+
+            self.workflow = MainFactory.create_workflow(context=context)
+
+        logger.info(
+            f"[{self._NAME}] Starting evaluation session: {self.session_name} - "
+            f"Workflow: '{self.workflow.name}'"
+        )
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.session_metadata.ended_at = datetime.now()
+        logger.info(
+            f"[{self._NAME}] Completed session '{self.session_name}' "
+            f"in {self.session_metadata.duration:.2f}s"
+        )
+
+        if exc_type:
+            logger.error(
+                f"[{self._NAME}] Session ended with error: {exc_val}", exc_info=True
+            )
+
+        return False
+
+    def step(self, step_name: str, category: MetricType = MetricType.CUSTOM) -> StepContext:
+        """Create a monitored evaluation step."""
+        return StepContext(self, step_name, category)
+
+    def run(self):
+        if not self.workflow:
+            raise RuntimeError(f"{self._NAME} Workflow not initialized")
+
+        with self.step(step_name="setup", category=MetricType.SETUP):
+            self.workflow.setup()
+
+        with self.step(step_name="load_data", category=MetricType.DATA_LOADING):
+            self.workflow.load_data()
+
+        with self.step(step_name="execute", category=MetricType.EXECUTION):
+            self.workflow.execute()
+
+        with self.step(
+            step_name=f"{self.session_name}.collect_results",
+            category=MetricType.RESULTS_COLLECTION,
+        ):
+            self.workflow.collect_results()
+
+    def run_connectivity_test(self, context: Dict[str, Any]) -> Dict[str, Any]:
+        if not self.workflow:
+            raise RuntimeError(f"{self._NAME} Workflow not initialized")
+
+        results = asyncio.run(self.workflow.test_connection(context=context))
+        return results
+
+    def get_stats(self) -> Dict[str, Any]:
+        if self.enable_monitoring:
+            return {
+                "session": {
+                    "name": self.session_name,
+                    "duration": precisedelta(
+                        self.session_metadata.duration, suppress=["minutes"]
+                    ),
+                    "start_time": self.session_metadata.started_at.isoformat(),
+                    "end_time": self.session_metadata.ended_at.isoformat(),
+                    "steps": len(self.session_metadata.steps),
+                    "errors": sum(
+                        s.error_count for s in self.session_metadata.steps.values()
+                    ),
+                },
+                "stats": self.monitor.get_all_stats(),
+            }
+
+        return {
+            "session": {
+                "name": self.session_name,
+                "duration": precisedelta(
+                    self.session_metadata.duration, suppress=["minutes"]
+                ),
+                "start_time": self.session_metadata.started_at.isoformat(),
+                "end_time": self.session_metadata.ended_at.isoformat(),
+                "steps": len(self.session_metadata.steps),
+                "errors": sum(
+                    s.error_count for s in self.session_metadata.steps.values()
+                ),
+            },
+        }
+
+    def visualize_results(
+        self, output_dir: str = "./visualizations", formats: List[str] = None
+    ) -> Dict[str, str]:
+        """
+        Generate visualizations for evaluation results.
+
+        Args:
+            output_dir: Directory to save visualizations (default: ./visualizations)
+            formats: List of export formats (html, png, pdf). Default: ["html"]
+
+        Returns:
+            Dictionary mapping format to file path
+
+        Example:
+            with EvaluationSession("my-eval", config) as session:
+                session.run()
+                files = session.visualize_results(
+                    output_dir="./reports",
+                    formats=["html", "png"]
+                )
+                print(f"Dashboard: {files['html']}")
+        """
+        if formats is None:
+            formats = ["html"]
+
+        logger.info(f"[{self.session_name}] Generating visualizations to: {output_dir}")
+
+        # Import here to avoid circular dependency
+        from levelapp.visualization import ResultsExporter
+
+        # Collect results from workflow
+        results = self.workflow.collect_results()
+
+        if not results:
+            logger.warning(
+                f"[{self.session_name}] No results available for visualization"
+            )
+            return {}
+
+        # Parse results if they're JSON string
+        if isinstance(results, str):
+            import json
+            from levelapp.simulator.schemas import SimulationResults
+
+            results_dict = json.loads(results)
+            results = SimulationResults.model_validate(results_dict)
+
+        # Export visualizations
+        exporter = ResultsExporter(output_dir=output_dir)
+        exported_files = exporter.export_dashboard(results=results, formats=formats)
+
+        logger.info(
+            f"[{self.session_name}] Visualizations generated: {list(exported_files.keys())}"
+        )
+
+        return exported_files

levelapp/endpoint/client.py

@@ -0,0 +1,188 @@
+"""levelapp/endpoint/client.py"""
+import os
+import httpx
+import asyncio
+import backoff
+import logging
+
+from dataclasses import dataclass, field
+from typing import List, Dict, Any
+from pydantic import BaseModel, Field
+
+from levelapp.endpoint.schemas import HttpMethod, HeaderConfig, RequestSchemaConfig, ResponseMappingConfig
+
+
+class EndpointConfig(BaseModel):
+    """Complete endpoint configuration."""
+    name: str
+    base_url: str
+    path: str
+    method: HttpMethod
+
+    headers: List[HeaderConfig] = Field(default_factory=list)
+    request_schema: List[RequestSchemaConfig] = Field(default_factory=list)
+    response_mapping: List[ResponseMappingConfig] = Field(default_factory=list)
+
+    # Timeouts (seconds)
+    connect_timeout: int = 10
+    read_timeout: int = 60
+    write_timeout: int = 10
+    pool_timeout: int = 10
+
+    # Concurrency
+    max_parallel_requests: int = 50
+    max_connections: int = 50
+    max_keepalive_connections: int = 50
+
+    # Retries
+    retry_count: int = 5
+    retry_backoff_base: float = 2.0
+    retry_backoff_max: float = 60.0
+
+    @classmethod
+    def validate_path(cls, v: str) -> str:
+        if not v.startswith('/'):
+            return f"/{v}"
+        return v
+
+
+@dataclass
+class ClientResult:
+    success: bool
+    response: httpx.Response | None = None
+    error: Exception | None = None
+
+
+@dataclass
+class APIClient:
+    """HTTP client for REST API interactions"""
+    config: EndpointConfig
+    client: httpx.AsyncClient = field(init=False)
+    semaphore: asyncio.Semaphore = field(init=False)
+    logger: logging.Logger = field(init=False)
+
+    RETRYABLE_ERRORS = (
+        httpx.ConnectTimeout,
+        httpx.WriteTimeout,
+        httpx.ReadTimeout,
+        httpx.NetworkError
+    )
+
+    def __post_init__(self):
+        self.logger = logging.getLogger(f"AsyncAPIClient.{self.config.name}")
+
+        self.client = httpx.AsyncClient(
+            base_url=self.config.base_url,
+            timeout=httpx.Timeout(
+                connect=self.config.connect_timeout,
+                read=self.config.read_timeout,
+                write=self.config.write_timeout,
+                pool=self.config.pool_timeout,
+            ),
+            limits=httpx.Limits(
+                max_connections=self.config.max_connections,
+                max_keepalive_connections=self.config.max_keepalive_connections,
+            ),
+            follow_redirects=True,
+        )
+
+        self.semaphore = asyncio.Semaphore(self.config.max_parallel_requests)
+
+    async def __aenter__(self) -> "APIClient":
+        return self
+
+    async def __aexit__(self, *args) -> None:
+        try:
+            if hasattr(self, 'client') and not self.client.is_closed:
+                self.logger.warning("[APIClient] Client not properly closed, forcing cleanup.")
+                asyncio.create_task(self.client.aclose())
+        except Exception as e:
+            self.logger.error(f"[APIClient] Error closing client: {e}")
+
+    def _build_headers(self) -> Dict[str, str]:
+        """Build headers with secure value resolution."""
+        headers = {}
+
+        for header in self.config.headers:
+            if header.secure:
+                value = os.getenv(header.value)
+                if value is None:
+                    self.logger.warning(f"Secure header '{header.name}' env var '{header.value}' not found")
+                    continue
+                headers[header.name] = value
+            else:
+                headers[header.name] = header.value
+
+        return headers
+
+    def _on_backoff(self, details):
+        """Callback for backoff logging"""
+        self.logger.warning(
+            f"[APIClient] Retry {details['tries']}/{self.config.retry_count} "
+            f"after {details['wait']:.2f}s (error: {details['exception'].__class__.__name__})"
+        )
+
+    def _on_giveup(self, details):
+        """Callback when all retries exhausted"""
+        self.logger.error(
+            f"[APIClient] Gave up after {details['tries']} tries, "
+            f"elapsed: {details['elapsed']:.2f}s"
+        )
+
+    async def send_request(
+            self,
+            payload: Dict[str, Any] | None = None,
+            query_params: Dict[str, Any] | None = None,
+    ) -> httpx.Response:
+        headers = self._build_headers()
+
+        async with self.semaphore:
+            response = await self.client.request(
+                    method=self.config.method.value,
+                    url=self.config.path,
+                    json=payload,
+                    params=query_params,
+                    headers=headers,
+                )
+
+            if response.is_error:
+                response.raise_for_status()
+
+            return response
+
+    async def execute(
+            self,
+            payload: Dict[str, Any] | None = None,
+            query_params: Dict[str, Any] | None = None,
+    ) -> ClientResult:
+        """
+        Execute asynchronous REST API request with retry logic using backoff.
+
+        Retries on transient errors with exponential backoff and jitter.
+        Non-retryable errors (pool exhaustion, HTTP errors) are raised immediately.
+        """
+        @backoff.on_exception(
+            backoff.expo,
+            self.RETRYABLE_ERRORS,
+            max_tries=self.config.retry_count,
+            max_time=self.config.retry_backoff_max,
+            jitter=backoff.full_jitter,
+            on_backoff=self._on_backoff,
+            on_giveup=self._on_giveup,
+            raise_on_giveup=True,
+        )
+        async def _execute_with_retry() -> httpx.Response:
+            return await self.send_request(payload=payload, query_params=query_params)
+
+        try:
+            response = await _execute_with_retry()
+            response.raise_for_status()
+            return ClientResult(success=True, response=response)
+
+        except httpx.HTTPStatusError as exc:
+            exc_response = exc.response if hasattr(exc, "response") else None
+            return ClientResult(success=False, response=exc_response, error=exc)
+
+        except Exception as exc:
+            exc_response = exc.response if hasattr(exc, "response") else None
+            return ClientResult(success=False, response=exc_response, error=exc)

levelapp/endpoint/client_test.py

@@ -0,0 +1,41 @@
+import asyncio
+import logging
+from levelapp.endpoint.client import APIClient, EndpointConfig
+from levelapp.endpoint.schemas import HttpMethod
+
+logging.basicConfig(level=logging.INFO)
+
+
+async def stress_test():
+    config = EndpointConfig(
+        name="stress-test",
+        base_url="http://127.0.0.1:8000",
+        path="chat",
+        method=HttpMethod.POST,
+        max_parallel_requests=20,
+        max_connections=20,
+        retry_count=3,
+        retry_backoff_max=30,
+    )
+
+    async with APIClient(config) as client:
+
+        async def single_call(i: int):
+            try:
+                response = await client.execute(
+                    payload={"message": f"test-{i}"}
+                )
+                return response.status_code
+            except Exception as e:
+                return f"ERROR: {type(e).__name__}"
+
+        tasks = [single_call(i) for i in range(100)]
+        results = await asyncio.gather(*tasks, return_exceptions=False)
+
+        print("Results summary:")
+        from collections import Counter
+        print(Counter(results))
+
+
+if __name__ == "__main__":
+    asyncio.run(stress_test())

levelapp/endpoint/manager.py

@@ -0,0 +1,114 @@
+"""levelapp/endpoint/manager.py"""
+import httpx
+import yaml
+import logging
+
+from pathlib import Path
+from typing import Dict, List, Any
+from pydantic import ValidationError
+
+from levelapp.endpoint.schemas import ResponseMappingConfig
+from levelapp.endpoint.tester import ConnectivityTester
+from levelapp.endpoint.client import EndpointConfig, APIClient, ClientResult
+from levelapp.endpoint.parsers import RequestPayloadBuilder, ResponseDataExtractor
+
+
+class EndpointConfigManager:
+    """Manages endpoint configurations and creates testers."""
+    def __init__(self, config_path: Path | None = None):
+        self.config_path = config_path
+        self.endpoints: Dict[str, EndpointConfig] = {}
+        self.logger = logging.getLogger("ConfigurationManager")
+
+        if config_path:
+            self._load_config()
+
+    def _load_config(self) -> None:
+        """Load and validate YAML configuration file."""
+        try:
+            with open(self.config_path, "r") as f:
+                data = yaml.safe_load(f)
+
+            for endpoint_data in data.get("endpoints", []):
+                config = EndpointConfig.model_validate(endpoint_data)
+                self.endpoints[config.name] = config
+                self.logger.info(f"Loaded endpoint config: {config.name}")
+
+        except ValidationError as e:
+            self.logger.error(f"Failed to load endpoint config: {e}")
+
+        except Exception as e:
+            self.logger.error(f"Failed to load endpoint config: {e}", exc_info=e)
+            raise RuntimeError("Failed to extract endpoints data from YAML file:\n{e}")
+
+    def set_endpoints(self, endpoints_config: List[EndpointConfig]):
+        for endpoint in endpoints_config:
+            try:
+                config = EndpointConfig.model_validate(endpoint)
+                self.endpoints[config.name] = config
+
+            except ValidationError as e:
+                self.logger.error(f"Failed to load endpoint config: {e}", exc_info=e)
+                continue
+
+    def build_response_mapping(self, content: List[Dict[str, Any]]) -> List[ResponseMappingConfig]:
+        mappings = []
+        for el in content:
+            try:
+                mappings.append(ResponseMappingConfig.model_validate(el))
+            except ValidationError as e:
+                self.logger.error(f"Failed to validate response mapping: {e}", exc_info=e)
+
+        return mappings
+
+    async def send_request(
+            self,
+            endpoint_config: EndpointConfig,
+            context: Dict[str, Any],
+            contextual_mode: bool = False
+    ) -> ClientResult:
+        payload_builder = RequestPayloadBuilder()
+        client = APIClient(config=endpoint_config)
+
+        if not contextual_mode:
+            context = payload_builder.build(
+                schema=endpoint_config.request_schema,
+                context=context,
+            )
+
+        async with client:
+            response = await client.execute(payload=context)
+
+        self.logger.info(f"Response status: {response.error}")
+
+        return response
+
+    @staticmethod
+    def extract_response_data(
+            response: httpx.Response,
+            mappings: List[ResponseMappingConfig],
+    ) -> Dict[str, Any]:
+        extractor = ResponseDataExtractor()
+        response_data = response.json() if response.text else {}
+        extracted = extractor.extract(
+            response_data=response_data,
+            mappings=mappings
+        )
+
+        return extracted
+
+    def get_tester(self, endpoint_name: str) -> ConnectivityTester:
+        """Factory method: create connectivity tester for endpoint."""
+        if endpoint_name not in self.endpoints:
+            raise KeyError(f"Endpoint '{endpoint_name}' not found in configuration")
+
+        return ConnectivityTester(self.endpoints[endpoint_name])
+
+    def test_all(self, context: Dict[str, Any] | None = None) -> Dict[str, Dict[str, Any]]:
+        """Test all configured endpoints."""
+        results = {}
+        for name in self.endpoints:
+            tester = self.get_tester(name)
+            results[name] = tester.test(context)
+
+        return results