crackerjack 0.32.0__py3-none-any.whl → 0.33.1__py3-none-any.whl

crackerjack/services/memory_optimizer.py

@@ -0,0 +1,409 @@
+import gc
+import sys
+import time
+import typing as t
+from collections.abc import Callable
+from dataclasses import dataclass
+from functools import wraps
+from threading import Lock
+from typing import Any
+from weakref import WeakSet
+
+from crackerjack.services.logging import get_logger
+
+
+@dataclass
+class MemoryStats:
+    total_allocated_mb: float
+    peak_usage_mb: float
+    current_usage_mb: float
+    gc_collections: int
+    lazy_objects_created: int
+    lazy_objects_loaded: int
+    resource_pools_active: int
+
+
+class LazyLoader:
+    def __init__(
+        self,
+        factory: Callable[[], Any],
+        name: str = "unnamed",
+        auto_dispose: bool = True,
+    ):
+        self._factory = factory
+        self._name = name
+        self._auto_dispose = auto_dispose
+        self._value: Any | None = None
+        self._loaded = False
+        self._lock = Lock()
+        self._access_count = 0
+        self._last_access = time.time()
+        self._logger = get_logger(f"crackerjack.lazy_loader.{name}")
+
+        MemoryOptimizer.get_instance().register_lazy_object(self)
+
+    @property
+    def is_loaded(self) -> bool:
+        with self._lock:
+            return self._loaded
+
+    @property
+    def access_count(self) -> int:
+        return self._access_count
+
+    def get(self) -> Any:
+        with self._lock:
+            if not self._loaded:
+                self._logger.debug(f"Lazy loading resource: {self._name}")
+                start_time = time.time()
+
+                try:
+                    self._value = self._factory()
+                    self._loaded = True
+                    load_time = time.time() - start_time
+                    self._logger.debug(f"Loaded {self._name} in {load_time: .3f}s")
+
+                    MemoryOptimizer.get_instance().notify_lazy_load(self._name)
+
+                except Exception as e:
+                    self._logger.error(f"Failed to load {self._name}: {e}")
+                    raise
+
+            self._access_count += 1
+            self._last_access = time.time()
+
+            if self._value is None:
+                raise RuntimeError(f"Lazy loader {self._name} has no value")
+
+            return self._value
+
+    def dispose(self) -> None:
+        with self._lock:
+            if self._loaded and self._value is not None:
+                self._logger.debug(f"Disposing lazy resource: {self._name}")
+
+                if hasattr(self._value, "close"):
+                    try:
+                        self._value.close()
+                    except Exception as e:
+                        self._logger.warning(f"Error closing {self._name}: {e}")
+
+                self._value = None
+                self._loaded = False
+
+                gc.collect()
+
+    def __del__(self) -> None:
+        if self._auto_dispose:
+            self.dispose()
+
+
+class ResourcePool:
+    def __init__(
+        self,
+        factory: Callable[[], Any],
+        max_size: int = 5,
+        name: str = "unnamed",
+    ):
+        self._factory = factory
+        self._max_size = max_size
+        self._name = name
+        self._pool: list[Any] = []
+        self._in_use: WeakSet[t.Any] = WeakSet()
+        self._lock = Lock()
+        self._created_count = 0
+        self._reused_count = 0
+        self._logger = get_logger(f"crackerjack.resource_pool.{name}")
+
+    def acquire(self) -> Any:
+        with self._lock:
+            if self._pool:
+                resource = self._pool.pop()
+                self._in_use.add(resource)
+                self._reused_count += 1
+                self._logger.debug(f"Reused resource from {self._name} pool")
+                return resource
+            else:
+                resource = self._factory()
+                self._in_use.add(resource)
+                self._created_count += 1
+                self._logger.debug(f"Created new resource for {self._name} pool")
+                return resource
+
+    def release(self, resource: Any) -> None:
+        with self._lock:
+            if resource in self._in_use:
+                self._in_use.discard(resource)
+
+                if len(self._pool) < self._max_size:
+                    self._pool.append(resource)
+                    self._logger.debug(f"Returned resource to {self._name} pool")
+                else:
+                    if hasattr(resource, "close"):
+                        try:
+                            resource.close()
+                        except Exception as e:
+                            self._logger.warning(f"Error closing resource: {e}")
+
+                    self._logger.debug(
+                        f"Pool full, disposed resource from {self._name}"
+                    )
+
+    def clear(self) -> None:
+        with self._lock:
+            for resource in self._pool:
+                if hasattr(resource, "close"):
+                    try:
+                        resource.close()
+                    except Exception as e:
+                        self._logger.warning(f"Error closing pooled resource: {e}")
+
+            self._pool.clear()
+            self._logger.info(f"Cleared {self._name} resource pool")
+
+    def get_stats(self) -> dict[str, Any]:
+        with self._lock:
+            return {
+                "pool_size": len(self._pool),
+                "in_use": len(self._in_use),
+                "created_total": self._created_count,
+                "reused_total": self._reused_count,
+                "efficiency": (
+                    self._reused_count / (self._created_count + self._reused_count)
+                    if self._created_count + self._reused_count > 0
+                    else 0.0
+                ),
+            }
+
+
+class MemoryProfiler:
+    def __init__(self) -> None:
+        self._start_memory = 0.0
+        self._peak_memory = 0.0
+        self._measurements: list[tuple[float, float]] = []
+        self._logger = get_logger("crackerjack.memory_profiler")
+
+    def start_profiling(self) -> None:
+        self._start_memory = self._get_memory_usage()
+        self._peak_memory = self._start_memory
+        self._measurements.clear()
+        self._logger.debug(f"Started memory profiling at {self._start_memory: .2f} MB")
+
+    def record_checkpoint(self, name: str = "") -> float:
+        current_memory = self._get_memory_usage()
+        self._peak_memory = max(self._peak_memory, current_memory)
+
+        timestamp = time.time()
+        self._measurements.append((timestamp, current_memory))
+
+        if name:
+            self._logger.debug(f"Memory checkpoint '{name}': {current_memory: .2f} MB")
+
+        return current_memory
+
+    def get_summary(self) -> dict[str, Any]:
+        if not self._measurements:
+            return {}
+
+        current_memory = self._get_memory_usage()
+        memory_delta = current_memory - self._start_memory
+
+        return {
+            "start_memory_mb": self._start_memory,
+            "current_memory_mb": current_memory,
+            "peak_memory_mb": self._peak_memory,
+            "memory_delta_mb": memory_delta,
+            "checkpoints": len(self._measurements),
+        }
+
+    def _get_memory_usage(self) -> float:
+        try:
+            import os
+
+            import psutil
+
+            process = psutil.Process(os.getpid())
+            memory_mb: float = process.memory_info().rss / 1024 / 1024
+            return memory_mb
+        except ImportError:
+            import tracemalloc
+
+            if tracemalloc.is_tracing():
+                current, _peak = tracemalloc.get_traced_memory()
+                return current / 1024 / 1024
+            else:
+                return sys.getsizeof(gc.get_objects()) / 1024 / 1024
+
+
+class MemoryOptimizer:
+    _instance: t.Optional["MemoryOptimizer"] = None
+    _lock = Lock()
+
+    def __init__(self) -> None:
+        self._lazy_objects: WeakSet[t.Any] = WeakSet()
+        self._resource_pools: dict[str, ResourcePool] = {}
+        self._profiler = MemoryProfiler()
+        self._stats_lock = Lock()
+        self._lazy_created_count = 0
+        self._lazy_loaded_count = 0
+        self._gc_threshold = 100
+        self._auto_gc = True
+        self._logger = get_logger("crackerjack.memory_optimizer")
+
+    @classmethod
+    def get_instance(cls) -> "MemoryOptimizer":
+        with cls._lock:
+            if cls._instance is None:
+                cls._instance = cls()
+            return cls._instance
+
+    def register_lazy_object(self, lazy_obj: LazyLoader) -> None:
+        self._lazy_objects.add(lazy_obj)
+        with self._stats_lock:
+            self._lazy_created_count += 1
+
+    def notify_lazy_load(self, name: str) -> None:
+        with self._stats_lock:
+            self._lazy_loaded_count += 1
+
+        if self._auto_gc and self._should_run_gc():
+            self._run_memory_cleanup()
+
+    def register_resource_pool(self, name: str, pool: ResourcePool) -> None:
+        self._resource_pools[name] = pool
+        self._logger.debug(f"Registered resource pool: {name}")
+
+    def get_resource_pool(self, name: str) -> ResourcePool | None:
+        return self._resource_pools.get(name)
+
+    def start_profiling(self) -> None:
+        self._profiler.start_profiling()
+
+    def record_checkpoint(self, name: str = "") -> float:
+        return self._profiler.record_checkpoint(name)
+
+    def get_memory_stats(self) -> MemoryStats:
+        profiler_stats = self._profiler.get_summary()
+
+        with self._stats_lock:
+            return MemoryStats(
+                total_allocated_mb=profiler_stats.get("peak_memory_mb", 0.0),
+                peak_usage_mb=profiler_stats.get("peak_memory_mb", 0.0),
+                current_usage_mb=profiler_stats.get("current_memory_mb", 0.0),
+                gc_collections=len(gc.get_stats()) if hasattr(gc, "get_stats") else 0,
+                lazy_objects_created=self._lazy_created_count,
+                lazy_objects_loaded=self._lazy_loaded_count,
+                resource_pools_active=len(self._resource_pools),
+            )
+
+    def optimize_memory(self) -> None:
+        self._logger.info("Running memory optimization")
+
+        self._cleanup_lazy_objects()
+
+        self._cleanup_resource_pools()
+
+        collected = gc.collect()
+        self._logger.debug(f"Garbage collection freed {collected} objects")
+
+    def _should_run_gc(self) -> bool:
+        current_memory = self._profiler.get_summary().get("current_memory_mb", 0)
+        should_gc: bool = current_memory > self._gc_threshold
+        return should_gc
+
+    def _run_memory_cleanup(self) -> None:
+        self._logger.debug("Running automatic memory cleanup")
+
+        before_gc = self._profiler._get_memory_usage()
+        collected = gc.collect()
+        after_gc = self._profiler._get_memory_usage()
+
+        memory_freed = before_gc - after_gc
+
+        if memory_freed > 1.0:
+            self._logger.info(
+                f"Memory cleanup freed {memory_freed: .2f} MB ({collected} objects)"
+            )
+
+    def _cleanup_lazy_objects(self) -> None:
+        disposed_count = 0
+
+        lazy_objects = list[t.Any](self._lazy_objects)
+
+        for lazy_obj in lazy_objects:
+            if (
+                hasattr(lazy_obj, "_last_access")
+                and lazy_obj._last_access < time.time() - 300
+                and lazy_obj.is_loaded
+            ):
+                lazy_obj.dispose()
+                disposed_count += 1
+
+        if disposed_count > 0:
+            self._logger.debug(f"Disposed {disposed_count} unused lazy objects")
+
+    def _cleanup_resource_pools(self) -> None:
+        for name, pool in self._resource_pools.items():
+            stats = pool.get_stats()
+
+            if stats["efficiency"] < 0.1 and stats["created_total"] > 10:
+                pool.clear()
+                self._logger.debug(f"Cleared inefficient resource pool: {name}")
+
+
+def lazy_property(factory: t.Callable[[], t.Any]) -> property:
+    def decorator(self: t.Any) -> Any:
+        attr_name = f"_lazy_{factory.__name__}"
+
+        if not hasattr(self, attr_name):
+            loader = LazyLoader(factory, factory.__name__)
+            setattr(self, attr_name, loader)
+
+        return getattr(self, attr_name).get()
+
+    return property(decorator)
+
+
+def memory_optimized(func: t.Callable[..., t.Any]) -> t.Callable[..., t.Any]:
+    @wraps(func)
+    def wrapper(*args: t.Any, **kwargs: t.Any) -> t.Any:
+        optimizer = MemoryOptimizer.get_instance()
+
+        before_memory = optimizer.record_checkpoint(f"{func.__name__}_start")
+
+        try:
+            result = func(*args, **kwargs)
+
+            after_memory = optimizer.record_checkpoint(f"{func.__name__}_end")
+
+            memory_delta = after_memory - before_memory
+            if memory_delta > 10.0:
+                optimizer._logger.warning(
+                    f"Function {func.__name__} increased memory by {memory_delta: .2f} MB"
+                )
+
+            return result
+
+        finally:
+            if optimizer._should_run_gc():
+                optimizer._run_memory_cleanup()
+
+    return wrapper
+
+
+def get_memory_optimizer() -> MemoryOptimizer:
+    return MemoryOptimizer.get_instance()
+
+
+def create_lazy_service(factory: Callable[[], Any], name: str) -> LazyLoader:
+    return LazyLoader(factory, name)
+
+
+def create_resource_pool(
+    factory: Callable[[], Any],
+    max_size: int = 5,
+    name: str = "unnamed",
+) -> ResourcePool:
+    pool = ResourcePool(factory, max_size, name)
+    MemoryOptimizer.get_instance().register_resource_pool(name, pool)
+    return pool

crackerjack/services/metrics.py

@@ -1,6 +1,7 @@
 import json
 import sqlite3
 import threading
+import typing as t
 from contextlib import contextmanager
 from datetime import date, datetime
 from pathlib import Path
@@ -21,45 +22,45 @@ class MetricsCollector:
     def _init_database(self) -> None:
         with self._get_connection() as conn:
             conn.executescript("""
-                - - Jobs table
+                -- Jobs table
                 CREATE TABLE IF NOT EXISTS jobs (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
                     job_id TEXT UNIQUE NOT NULL,
                     start_time TIMESTAMP NOT NULL,
                     end_time TIMESTAMP,
-                    status TEXT NOT NULL, - - 'running', 'success', 'failed', 'cancelled'
+                    status TEXT NOT NULL, -- 'running', 'success', 'failed', 'cancelled'
                     iterations INTEGER DEFAULT 0,
                     ai_agent BOOLEAN DEFAULT 0,
                     error_message TEXT,
-                    metadata TEXT - - JSON field for additional data
+                    metadata TEXT -- JSON field for additional data
                 );
 
-                - - Errors table
+                -- Errors table
                 CREATE TABLE IF NOT EXISTS errors (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
                     job_id TEXT,
                     timestamp TIMESTAMP NOT NULL,
-                    error_type TEXT NOT NULL, - - 'hook', 'test', 'lint', 'type_check', etc.
-                    error_category TEXT, - - 'ruff', 'pyright', 'pytest', etc.
+                    error_type TEXT NOT NULL, -- 'hook', 'test', 'lint', 'type_check', etc.
+                    error_category TEXT, -- 'ruff', 'pyright', 'pytest', etc.
                     error_message TEXT,
                     file_path TEXT,
                     line_number INTEGER,
                     FOREIGN KEY (job_id) REFERENCES jobs(job_id)
                 );
 
-                - - Hook executions table
+                -- Hook executions table
                 CREATE TABLE IF NOT EXISTS hook_executions (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
                     job_id TEXT,
                     timestamp TIMESTAMP NOT NULL,
                     hook_name TEXT NOT NULL,
-                    hook_type TEXT, - - 'fast', 'comprehensive'
+                    hook_type TEXT, -- 'fast', 'comprehensive'
                     execution_time_ms INTEGER,
-                    status TEXT, - - 'success', 'failed', 'skipped'
+                    status TEXT, -- 'success', 'failed', 'skipped'
                     FOREIGN KEY (job_id) REFERENCES jobs(job_id)
                 );
 
-                - - Test executions table
+                -- Test executions table
                 CREATE TABLE IF NOT EXISTS test_executions (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
                     job_id TEXT,
@@ -73,17 +74,17 @@ class MetricsCollector:
                     FOREIGN KEY (job_id) REFERENCES jobs(job_id)
                 );
 
-                - - Orchestration executions table (NEW)
+                -- Orchestration executions table (NEW)
                 CREATE TABLE IF NOT EXISTS orchestration_executions (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
                     job_id TEXT,
                     timestamp TIMESTAMP NOT NULL,
-                    execution_strategy TEXT NOT NULL, - - 'batch', 'individual', 'adaptive', 'selective'
-                    progress_level TEXT NOT NULL, - - 'basic', 'detailed', 'granular', 'streaming'
-                    ai_mode TEXT NOT NULL, - - 'single - agent', 'multi - agent', 'coordinator'
+                    execution_strategy TEXT NOT NULL, -- 'batch', 'individual', 'adaptive', 'selective'
+                    progress_level TEXT NOT NULL, -- 'basic', 'detailed', 'granular', 'streaming'
+                    ai_mode TEXT NOT NULL, -- 'single-agent', 'multi-agent', 'coordinator'
                     iteration_count INTEGER DEFAULT 1,
-                    strategy_switches INTEGER DEFAULT 0, - - How many times strategy changed
-                    correlation_insights TEXT, - - JSON of correlation analysis results
+                    strategy_switches INTEGER DEFAULT 0, -- How many times strategy changed
+                    correlation_insights TEXT, -- JSON of correlation analysis results
                     total_execution_time_ms INTEGER,
                     hooks_execution_time_ms INTEGER,
                     tests_execution_time_ms INTEGER,
@@ -91,7 +92,7 @@ class MetricsCollector:
                     FOREIGN KEY (job_id) REFERENCES jobs(job_id)
                 );
 
-                - - Strategy decisions table (NEW)
+                -- Strategy decisions table (NEW)
                 CREATE TABLE IF NOT EXISTS strategy_decisions (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
                     job_id TEXT,
@@ -99,29 +100,29 @@ class MetricsCollector:
                     timestamp TIMESTAMP NOT NULL,
                     previous_strategy TEXT,
                     selected_strategy TEXT NOT NULL,
-                    decision_reason TEXT, - - Why this strategy was chosen
-                    context_data TEXT, - - JSON of execution context
-                    effectiveness_score REAL, - - How well the strategy worked (0 - 1)
+                    decision_reason TEXT, -- Why this strategy was chosen
+                    context_data TEXT, -- JSON of execution context
+                    effectiveness_score REAL, -- How well the strategy worked (0 - 1)
                     FOREIGN KEY (job_id) REFERENCES jobs(job_id)
                 );
 
-                - - Individual test executions table (NEW - more granular than test_executions)
+                -- Individual test executions table (NEW - more granular than test_executions)
                 CREATE TABLE IF NOT EXISTS individual_test_executions (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
                     job_id TEXT,
                     timestamp TIMESTAMP NOT NULL,
-                    test_id TEXT NOT NULL, - - Full test identifier
+                    test_id TEXT NOT NULL, -- Full test identifier
                     test_file TEXT NOT NULL,
                     test_class TEXT,
                     test_method TEXT,
-                    status TEXT NOT NULL, - - 'passed', 'failed', 'skipped', 'error'
+                    status TEXT NOT NULL, -- 'passed', 'failed', 'skipped', 'error'
                     execution_time_ms INTEGER,
                     error_message TEXT,
                     error_traceback TEXT,
                     FOREIGN KEY (job_id) REFERENCES jobs(job_id)
                 );
 
-                - - Daily summary table (for quick stats)
+                -- Daily summary table (for quick stats)
                 CREATE TABLE IF NOT EXISTS daily_summary (
                     date DATE PRIMARY KEY,
                     total_jobs INTEGER DEFAULT 0,
@@ -134,9 +135,9 @@ class MetricsCollector:
                     type_errors INTEGER DEFAULT 0,
                     avg_job_duration_ms INTEGER,
                     total_ai_fixes INTEGER DEFAULT 0,
-                    orchestrated_jobs INTEGER DEFAULT 0, - - NEW
-                    avg_orchestration_iterations REAL DEFAULT 0, - - NEW
-                    most_effective_strategy TEXT - - NEW
+                    orchestrated_jobs INTEGER DEFAULT 0, -- NEW
+                    avg_orchestration_iterations REAL DEFAULT 0, -- NEW
+                    most_effective_strategy TEXT -- NEW
                 );
 
                 --Create indexes for performance
@@ -152,7 +153,7 @@ class MetricsCollector:
             """)
 
     @contextmanager
-    def _get_connection(self):
+    def _get_connection(self) -> t.Iterator[sqlite3.Connection]:
         conn = sqlite3.connect(str(self.db_path))
         conn.row_factory = sqlite3.Row
         try:
@@ -442,10 +443,18 @@ class MetricsCollector:
             """).fetchall()
 
             return {
-                "strategy_effectiveness": [dict(row) for row in strategy_stats],
-                "correlation_patterns": [dict(row) for row in correlation_patterns],
-                "performance_by_strategy": [dict(row) for row in performance_stats],
-                "test_failure_patterns": [dict(row) for row in test_failure_patterns],
+                "strategy_effectiveness": [
+                    dict[str, t.Any](row) for row in strategy_stats
+                ],
+                "correlation_patterns": [
+                    dict[str, t.Any](row) for row in correlation_patterns
+                ],
+                "performance_by_strategy": [
+                    dict[str, t.Any](row) for row in performance_stats
+                ],
+                "test_failure_patterns": [
+                    dict[str, t.Any](row) for row in test_failure_patterns
+                ],
             }
 
     def _update_daily_summary(
@@ -564,7 +573,7 @@ class MetricsCollector:
                 "error_breakdown": {
                     row["error_type"]: row["count"] for row in error_stats
                 },
-                "common_errors": [dict(row) for row in common_errors],
+                "common_errors": [dict[str, t.Any](row) for row in common_errors],
             }