crackerjack 0.33.0__py3-none-any.whl → 0.33.2__py3-none-any.whl

crackerjack/services/memory_optimizer.py

@@ -1,9 +1,3 @@
-"""Memory optimization service with lazy loading and resource management.
-
-This module provides memory-efficient patterns for managing heavy resources,
-lazy loading, and memory profiling capabilities.
-"""
-
 import gc
 import sys
 import time
@@ -20,8 +14,6 @@ from crackerjack.services.logging import get_logger
 
 @dataclass
 class MemoryStats:
-    """Memory usage statistics."""
-
     total_allocated_mb: float
     peak_usage_mb: float
     current_usage_mb: float
@@ -32,8 +24,6 @@ class MemoryStats:
 
 
 class LazyLoader:
-    """Lazy loader for expensive resources."""
-
     def __init__(
         self,
         factory: Callable[[], Any],
@@ -50,22 +40,18 @@ class LazyLoader:
         self._last_access = time.time()
         self._logger = get_logger(f"crackerjack.lazy_loader.{name}")
 
-        # Register for memory tracking
         MemoryOptimizer.get_instance().register_lazy_object(self)
 
     @property
     def is_loaded(self) -> bool:
-        """Check if resource is loaded."""
         with self._lock:
             return self._loaded
 
     @property
     def access_count(self) -> int:
-        """Get access count."""
         return self._access_count
 
     def get(self) -> Any:
-        """Get the loaded resource, loading if necessary."""
         with self._lock:
             if not self._loaded:
                 self._logger.debug(f"Lazy loading resource: {self._name}")
@@ -75,9 +61,8 @@ class LazyLoader:
                     self._value = self._factory()
                     self._loaded = True
                     load_time = time.time() - start_time
-                    self._logger.debug(f"Loaded {self._name} in {load_time:.3f}s")
+                    self._logger.debug(f"Loaded {self._name} in {load_time: .3f}s")
 
-                    # Register for memory tracking
                     MemoryOptimizer.get_instance().notify_lazy_load(self._name)
 
                 except Exception as e:
@@ -93,12 +78,10 @@ class LazyLoader:
             return self._value
 
     def dispose(self) -> None:
-        """Dispose of the loaded resource."""
         with self._lock:
             if self._loaded and self._value is not None:
                 self._logger.debug(f"Disposing lazy resource: {self._name}")
 
-                # If the object has a cleanup method, call it
                 if hasattr(self._value, "close"):
                     try:
                         self._value.close()
@@ -108,18 +91,14 @@ class LazyLoader:
                 self._value = None
                 self._loaded = False
 
-                # Force garbage collection for this object
                 gc.collect()
 
-    def __del__(self):
-        """Clean up on deletion."""
+    def __del__(self) -> None:
         if self._auto_dispose:
             self.dispose()
 
 
 class ResourcePool:
-    """Pool for reusable expensive objects."""
-
     def __init__(
         self,
         factory: Callable[[], Any],
@@ -137,7 +116,6 @@ class ResourcePool:
         self._logger = get_logger(f"crackerjack.resource_pool.{name}")
 
     def acquire(self) -> Any:
-        """Acquire a resource from the pool."""
         with self._lock:
             if self._pool:
                 resource = self._pool.pop()
@@ -153,7 +131,6 @@ class ResourcePool:
                 return resource
 
     def release(self, resource: Any) -> None:
-        """Release a resource back to the pool."""
         with self._lock:
             if resource in self._in_use:
                 self._in_use.discard(resource)
@@ -162,7 +139,6 @@ class ResourcePool:
                     self._pool.append(resource)
                     self._logger.debug(f"Returned resource to {self._name} pool")
                 else:
-                    # Pool is full, dispose of resource
                     if hasattr(resource, "close"):
                         try:
                             resource.close()
@@ -174,7 +150,6 @@ class ResourcePool:
                     )
 
     def clear(self) -> None:
-        """Clear all resources from the pool."""
         with self._lock:
             for resource in self._pool:
                 if hasattr(resource, "close"):
@@ -187,7 +162,6 @@ class ResourcePool:
             self._logger.info(f"Cleared {self._name} resource pool")
 
     def get_stats(self) -> dict[str, Any]:
-        """Get pool statistics."""
         with self._lock:
             return {
                 "pool_size": len(self._pool),
@@ -203,23 +177,19 @@ class ResourcePool:
 
 
 class MemoryProfiler:
-    """Simple memory profiler for performance monitoring."""
-
-    def __init__(self):
+    def __init__(self) -> None:
         self._start_memory = 0.0
         self._peak_memory = 0.0
         self._measurements: list[tuple[float, float]] = []
         self._logger = get_logger("crackerjack.memory_profiler")
 
     def start_profiling(self) -> None:
-        """Start memory profiling."""
         self._start_memory = self._get_memory_usage()
         self._peak_memory = self._start_memory
         self._measurements.clear()
-        self._logger.debug(f"Started memory profiling at {self._start_memory:.2f} MB")
+        self._logger.debug(f"Started memory profiling at {self._start_memory: .2f} MB")
 
     def record_checkpoint(self, name: str = "") -> float:
-        """Record memory checkpoint."""
         current_memory = self._get_memory_usage()
         self._peak_memory = max(self._peak_memory, current_memory)
 
@@ -227,12 +197,11 @@ class MemoryProfiler:
         self._measurements.append((timestamp, current_memory))
 
         if name:
-            self._logger.debug(f"Memory checkpoint '{name}': {current_memory:.2f} MB")
+            self._logger.debug(f"Memory checkpoint '{name}': {current_memory: .2f} MB")
 
         return current_memory
 
     def get_summary(self) -> dict[str, Any]:
-        """Get profiling summary."""
         if not self._measurements:
             return {}
 
@@ -248,85 +217,72 @@ class MemoryProfiler:
         }
 
     def _get_memory_usage(self) -> float:
-        """Get current memory usage in MB."""
         try:
             import os
 
             import psutil
 
             process = psutil.Process(os.getpid())
-            return process.memory_info().rss / 1024 / 1024
+            memory_mb: float = process.memory_info().rss / 1024 / 1024
+            return memory_mb
         except ImportError:
-            # Fallback to tracemalloc if psutil not available
             import tracemalloc
 
             if tracemalloc.is_tracing():
                 current, _peak = tracemalloc.get_traced_memory()
                 return current / 1024 / 1024
             else:
-                # Basic fallback using sys.getsizeof (less accurate)
                 return sys.getsizeof(gc.get_objects()) / 1024 / 1024
 
 
 class MemoryOptimizer:
-    """Central memory optimization coordinator."""
-
     _instance: t.Optional["MemoryOptimizer"] = None
     _lock = Lock()
 
-    def __init__(self):
+    def __init__(self) -> None:
         self._lazy_objects: WeakSet[t.Any] = WeakSet()
         self._resource_pools: dict[str, ResourcePool] = {}
         self._profiler = MemoryProfiler()
         self._stats_lock = Lock()
         self._lazy_created_count = 0
         self._lazy_loaded_count = 0
-        self._gc_threshold = 100  # MB
+        self._gc_threshold = 100
         self._auto_gc = True
         self._logger = get_logger("crackerjack.memory_optimizer")
 
     @classmethod
     def get_instance(cls) -> "MemoryOptimizer":
-        """Get singleton instance."""
         with cls._lock:
             if cls._instance is None:
                 cls._instance = cls()
             return cls._instance
 
     def register_lazy_object(self, lazy_obj: LazyLoader) -> None:
-        """Register a lazy object for tracking."""
         self._lazy_objects.add(lazy_obj)
         with self._stats_lock:
             self._lazy_created_count += 1
 
     def notify_lazy_load(self, name: str) -> None:
-        """Notify that a lazy object was loaded."""
         with self._stats_lock:
             self._lazy_loaded_count += 1
 
-        # Check if we should trigger garbage collection
         if self._auto_gc and self._should_run_gc():
             self._run_memory_cleanup()
 
     def register_resource_pool(self, name: str, pool: ResourcePool) -> None:
-        """Register a resource pool."""
         self._resource_pools[name] = pool
         self._logger.debug(f"Registered resource pool: {name}")
 
     def get_resource_pool(self, name: str) -> ResourcePool | None:
-        """Get a registered resource pool."""
         return self._resource_pools.get(name)
 
     def start_profiling(self) -> None:
-        """Start memory profiling."""
         self._profiler.start_profiling()
 
     def record_checkpoint(self, name: str = "") -> float:
-        """Record memory checkpoint."""
         return self._profiler.record_checkpoint(name)
 
     def get_memory_stats(self) -> MemoryStats:
-        """Get comprehensive memory statistics."""
         profiler_stats = self._profiler.get_summary()
 
         with self._stats_lock:
@@ -341,52 +297,43 @@ class MemoryOptimizer:
             )
 
     def optimize_memory(self) -> None:
-        """Run memory optimization."""
         self._logger.info("Running memory optimization")
 
-        # Dispose unused lazy objects
         self._cleanup_lazy_objects()
 
-        # Clear resource pools if needed
         self._cleanup_resource_pools()
 
-        # Force garbage collection
         collected = gc.collect()
         self._logger.debug(f"Garbage collection freed {collected} objects")
 
     def _should_run_gc(self) -> bool:
-        """Check if garbage collection should be triggered."""
         current_memory = self._profiler.get_summary().get("current_memory_mb", 0)
-        return current_memory > self._gc_threshold
+        should_gc: bool = current_memory > self._gc_threshold
+        return should_gc
 
     def _run_memory_cleanup(self) -> None:
-        """Run memory cleanup operations."""
         self._logger.debug("Running automatic memory cleanup")
 
-        # Collect garbage
         before_gc = self._profiler._get_memory_usage()
         collected = gc.collect()
         after_gc = self._profiler._get_memory_usage()
 
         memory_freed = before_gc - after_gc
 
-        if memory_freed > 1.0:  # More than 1MB freed
+        if memory_freed > 1.0:
             self._logger.info(
-                f"Memory cleanup freed {memory_freed:.2f} MB ({collected} objects)"
+                f"Memory cleanup freed {memory_freed: .2f} MB ({collected} objects)"
             )
 
     def _cleanup_lazy_objects(self) -> None:
-        """Clean up unused lazy objects."""
         disposed_count = 0
 
-        # Convert to list to avoid modification during iteration
-        lazy_objects = list(self._lazy_objects)
+        lazy_objects = list[t.Any](self._lazy_objects)
 
         for lazy_obj in lazy_objects:
-            # Dispose objects that haven't been accessed recently
             if (
                 hasattr(lazy_obj, "_last_access")
-                and lazy_obj._last_access < time.time() - 300  # 5 minutes
+                and lazy_obj._last_access < time.time() - 300
                 and lazy_obj.is_loaded
             ):
                 lazy_obj.dispose()
@@ -396,19 +343,15 @@ class MemoryOptimizer:
             self._logger.debug(f"Disposed {disposed_count} unused lazy objects")
 
     def _cleanup_resource_pools(self) -> None:
-        """Clean up resource pools."""
         for name, pool in self._resource_pools.items():
             stats = pool.get_stats()
 
-            # Clear pool if efficiency is very low (lots of created, few reused)
             if stats["efficiency"] < 0.1 and stats["created_total"] > 10:
                 pool.clear()
                 self._logger.debug(f"Cleared inefficient resource pool: {name}")
 
 
-def lazy_property(factory: t.Callable[[], Any]) -> t.Callable[[t.Any], Any]:
-    """Decorator for lazy property loading."""
-
+def lazy_property(factory: t.Callable[[], t.Any]) -> property:
     def decorator(self: t.Any) -> Any:
         attr_name = f"_lazy_{factory.__name__}"
 
@@ -418,51 +361,41 @@ def lazy_property(factory: t.Callable[[], Any]) -> t.Callable[[t.Any], Any]:
 
         return getattr(self, attr_name).get()
 
-    return property(decorator)  # type: ignore[return-value]
+    return property(decorator)
 
 
 def memory_optimized(func: t.Callable[..., t.Any]) -> t.Callable[..., t.Any]:
-    """Decorator to add memory optimization to functions."""
-
     @wraps(func)
     def wrapper(*args: t.Any, **kwargs: t.Any) -> t.Any:
         optimizer = MemoryOptimizer.get_instance()
 
-        # Record memory before function execution
         before_memory = optimizer.record_checkpoint(f"{func.__name__}_start")
 
         try:
             result = func(*args, **kwargs)
 
-            # Record memory after function execution
             after_memory = optimizer.record_checkpoint(f"{func.__name__}_end")
 
-            # Log significant memory increases
             memory_delta = after_memory - before_memory
-            if memory_delta > 10.0:  # More than 10MB increase
+            if memory_delta > 10.0:
                 optimizer._logger.warning(
-                    f"Function {func.__name__} increased memory by {memory_delta:.2f} MB"
+                    f"Function {func.__name__} increased memory by {memory_delta: .2f} MB"
                 )
 
             return result
 
         finally:
-            # Run cleanup if memory usage is high
             if optimizer._should_run_gc():
                 optimizer._run_memory_cleanup()
 
     return wrapper
 
 
-# Global optimizer instance
 def get_memory_optimizer() -> MemoryOptimizer:
-    """Get global memory optimizer instance."""
     return MemoryOptimizer.get_instance()
 
 
-# Factory functions for common patterns
 def create_lazy_service(factory: Callable[[], Any], name: str) -> LazyLoader:
-    """Create a lazy-loaded service."""
     return LazyLoader(factory, name)
 
 
@@ -471,7 +404,6 @@ def create_resource_pool(
     max_size: int = 5,
     name: str = "unnamed",
 ) -> ResourcePool:
-    """Create a resource pool and register it."""
     pool = ResourcePool(factory, max_size, name)
     MemoryOptimizer.get_instance().register_resource_pool(name, pool)
     return pool

crackerjack/services/metrics.py

@@ -1,6 +1,7 @@
 import json
 import sqlite3
 import threading
+import typing as t
 from contextlib import contextmanager
 from datetime import date, datetime
 from pathlib import Path
@@ -21,45 +22,45 @@ class MetricsCollector:
     def _init_database(self) -> None:
         with self._get_connection() as conn:
             conn.executescript("""
-                - - Jobs table
+                -- Jobs table
                 CREATE TABLE IF NOT EXISTS jobs (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
                     job_id TEXT UNIQUE NOT NULL,
                     start_time TIMESTAMP NOT NULL,
                     end_time TIMESTAMP,
-                    status TEXT NOT NULL, - - 'running', 'success', 'failed', 'cancelled'
+                    status TEXT NOT NULL, -- 'running', 'success', 'failed', 'cancelled'
                     iterations INTEGER DEFAULT 0,
                     ai_agent BOOLEAN DEFAULT 0,
                     error_message TEXT,
-                    metadata TEXT - - JSON field for additional data
+                    metadata TEXT -- JSON field for additional data
                 );
 
-                - - Errors table
+                -- Errors table
                 CREATE TABLE IF NOT EXISTS errors (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
                     job_id TEXT,
                     timestamp TIMESTAMP NOT NULL,
-                    error_type TEXT NOT NULL, - - 'hook', 'test', 'lint', 'type_check', etc.
-                    error_category TEXT, - - 'ruff', 'pyright', 'pytest', etc.
+                    error_type TEXT NOT NULL, -- 'hook', 'test', 'lint', 'type_check', etc.
+                    error_category TEXT, -- 'ruff', 'pyright', 'pytest', etc.
                     error_message TEXT,
                     file_path TEXT,
                     line_number INTEGER,
                     FOREIGN KEY (job_id) REFERENCES jobs(job_id)
                 );
 
-                - - Hook executions table
+                -- Hook executions table
                 CREATE TABLE IF NOT EXISTS hook_executions (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
                     job_id TEXT,
                     timestamp TIMESTAMP NOT NULL,
                     hook_name TEXT NOT NULL,
-                    hook_type TEXT, - - 'fast', 'comprehensive'
+                    hook_type TEXT, -- 'fast', 'comprehensive'
                     execution_time_ms INTEGER,
-                    status TEXT, - - 'success', 'failed', 'skipped'
+                    status TEXT, -- 'success', 'failed', 'skipped'
                     FOREIGN KEY (job_id) REFERENCES jobs(job_id)
                 );
 
-                - - Test executions table
+                -- Test executions table
                 CREATE TABLE IF NOT EXISTS test_executions (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
                     job_id TEXT,
@@ -73,17 +74,17 @@ class MetricsCollector:
                     FOREIGN KEY (job_id) REFERENCES jobs(job_id)
                 );
 
-                - - Orchestration executions table (NEW)
+                -- Orchestration executions table (NEW)
                 CREATE TABLE IF NOT EXISTS orchestration_executions (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
                     job_id TEXT,
                     timestamp TIMESTAMP NOT NULL,
-                    execution_strategy TEXT NOT NULL, - - 'batch', 'individual', 'adaptive', 'selective'
-                    progress_level TEXT NOT NULL, - - 'basic', 'detailed', 'granular', 'streaming'
-                    ai_mode TEXT NOT NULL, - - 'single - agent', 'multi - agent', 'coordinator'
+                    execution_strategy TEXT NOT NULL, -- 'batch', 'individual', 'adaptive', 'selective'
+                    progress_level TEXT NOT NULL, -- 'basic', 'detailed', 'granular', 'streaming'
+                    ai_mode TEXT NOT NULL, -- 'single-agent', 'multi-agent', 'coordinator'
                     iteration_count INTEGER DEFAULT 1,
-                    strategy_switches INTEGER DEFAULT 0, - - How many times strategy changed
-                    correlation_insights TEXT, - - JSON of correlation analysis results
+                    strategy_switches INTEGER DEFAULT 0, -- How many times strategy changed
+                    correlation_insights TEXT, -- JSON of correlation analysis results
                     total_execution_time_ms INTEGER,
                     hooks_execution_time_ms INTEGER,
                     tests_execution_time_ms INTEGER,
@@ -91,7 +92,7 @@ class MetricsCollector:
                     FOREIGN KEY (job_id) REFERENCES jobs(job_id)
                 );
 
-                - - Strategy decisions table (NEW)
+                -- Strategy decisions table (NEW)
                 CREATE TABLE IF NOT EXISTS strategy_decisions (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
                     job_id TEXT,
@@ -99,29 +100,29 @@ class MetricsCollector:
                     timestamp TIMESTAMP NOT NULL,
                     previous_strategy TEXT,
                     selected_strategy TEXT NOT NULL,
-                    decision_reason TEXT, - - Why this strategy was chosen
-                    context_data TEXT, - - JSON of execution context
-                    effectiveness_score REAL, - - How well the strategy worked (0 - 1)
+                    decision_reason TEXT, -- Why this strategy was chosen
+                    context_data TEXT, -- JSON of execution context
+                    effectiveness_score REAL, -- How well the strategy worked (0 - 1)
                     FOREIGN KEY (job_id) REFERENCES jobs(job_id)
                 );
 
-                - - Individual test executions table (NEW - more granular than test_executions)
+                -- Individual test executions table (NEW - more granular than test_executions)
                 CREATE TABLE IF NOT EXISTS individual_test_executions (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
                     job_id TEXT,
                     timestamp TIMESTAMP NOT NULL,
-                    test_id TEXT NOT NULL, - - Full test identifier
+                    test_id TEXT NOT NULL, -- Full test identifier
                     test_file TEXT NOT NULL,
                     test_class TEXT,
                     test_method TEXT,
-                    status TEXT NOT NULL, - - 'passed', 'failed', 'skipped', 'error'
+                    status TEXT NOT NULL, -- 'passed', 'failed', 'skipped', 'error'
                     execution_time_ms INTEGER,
                     error_message TEXT,
                     error_traceback TEXT,
                     FOREIGN KEY (job_id) REFERENCES jobs(job_id)
                 );
 
-                - - Daily summary table (for quick stats)
+                -- Daily summary table (for quick stats)
                 CREATE TABLE IF NOT EXISTS daily_summary (
                     date DATE PRIMARY KEY,
                     total_jobs INTEGER DEFAULT 0,
@@ -134,9 +135,9 @@ class MetricsCollector:
                     type_errors INTEGER DEFAULT 0,
                     avg_job_duration_ms INTEGER,
                     total_ai_fixes INTEGER DEFAULT 0,
-                    orchestrated_jobs INTEGER DEFAULT 0, - - NEW
-                    avg_orchestration_iterations REAL DEFAULT 0, - - NEW
-                    most_effective_strategy TEXT - - NEW
+                    orchestrated_jobs INTEGER DEFAULT 0, -- NEW
+                    avg_orchestration_iterations REAL DEFAULT 0, -- NEW
+                    most_effective_strategy TEXT -- NEW
                 );
 
                 --Create indexes for performance
@@ -152,7 +153,7 @@ class MetricsCollector:
             """)
 
     @contextmanager
-    def _get_connection(self):
+    def _get_connection(self) -> t.Iterator[sqlite3.Connection]:
         conn = sqlite3.connect(str(self.db_path))
         conn.row_factory = sqlite3.Row
         try:
@@ -442,10 +443,18 @@ class MetricsCollector:
             """).fetchall()
 
             return {
-                "strategy_effectiveness": [dict(row) for row in strategy_stats],
-                "correlation_patterns": [dict(row) for row in correlation_patterns],
-                "performance_by_strategy": [dict(row) for row in performance_stats],
-                "test_failure_patterns": [dict(row) for row in test_failure_patterns],
+                "strategy_effectiveness": [
+                    dict[str, t.Any](row) for row in strategy_stats
+                ],
+                "correlation_patterns": [
+                    dict[str, t.Any](row) for row in correlation_patterns
+                ],
+                "performance_by_strategy": [
+                    dict[str, t.Any](row) for row in performance_stats
+                ],
+                "test_failure_patterns": [
+                    dict[str, t.Any](row) for row in test_failure_patterns
+                ],
             }
 
     def _update_daily_summary(
@@ -564,7 +573,7 @@ class MetricsCollector:
                 "error_breakdown": {
                     row["error_type"]: row["count"] for row in error_stats
                 },
-                "common_errors": [dict(row) for row in common_errors],
+                "common_errors": [dict[str, t.Any](row) for row in common_errors],
             }