crackerjack 0.32.0__py3-none-any.whl → 0.33.1__py3-none-any.whl

crackerjack/services/bounded_status_operations.py

@@ -1,10 +1,3 @@
-"""
-Bounded Status Operations to prevent DoS attacks.
-
-Provides comprehensive operation limits, circuit breakers, and
-resource protection for status collection operations.
-"""
-
 import asyncio
 import time
 import typing as t
@@ -17,17 +10,13 @@ from .security_logger import SecurityEventLevel, SecurityEventType, get_security
 
 
 class OperationState(str, Enum):
-    """States for circuit breaker pattern."""
-
-    CLOSED = "closed"  # Normal operation
-    OPEN = "open"  # Circuit breaker tripped
-    HALF_OPEN = "half_open"  # Testing if service recovered
+    CLOSED = "closed"
+    OPEN = "open"
+    HALF_OPEN = "half_open"
 
 
 @dataclass
 class OperationLimits:
-    """Limits for status operations."""
-
     max_concurrent_operations: int = 10
     max_operations_per_minute: int = 60
     max_operation_duration: float = 30.0
@@ -41,8 +30,6 @@ class OperationLimits:
 
 @dataclass
 class OperationMetrics:
-    """Metrics for operation tracking."""
-
     operation_id: str
     operation_type: str
     client_id: str
@@ -56,72 +43,46 @@ class OperationMetrics:
 
     @property
     def duration(self) -> float:
-        """Get operation duration in seconds."""
         end = self.end_time or time.time()
         return end - self.start_time
 
     @property
     def is_completed(self) -> bool:
-        """Check if operation is completed."""
         return self.end_time is not None
 
 
 class OperationLimitExceededError(Exception):
-    """Raised when operation limits are exceeded."""
-
     pass
 
 
 class CircuitBreakerOpenError(Exception):
-    """Raised when circuit breaker is open."""
-
     pass
 
 
 class BoundedStatusOperations:
-    """
-    Bounded operations manager for status collection.
-
-    Features:
-    - Concurrent operation limits
-    - Rate limiting per client
-    - Memory and CPU usage monitoring
-    - Circuit breaker pattern for fault tolerance
-    - Operation timeout enforcement
-    - Comprehensive metrics collection
-    """
-
     def __init__(self, limits: OperationLimits | None = None):
-        """
-        Initialize bounded operations manager.
-
-        Args:
-            limits: Operation limits configuration
-        """
         self.limits = limits or OperationLimits()
         self.security_logger = get_security_logger()
 
-        # Thread-safe operation tracking
         self._lock = RLock()
         self._active_operations: dict[str, OperationMetrics] = {}
-        self._client_operations: dict[str, deque[float]] = defaultdict(
-            lambda: deque(maxlen=100)
-        )
+
+        def _create_deque() -> deque[float]:
+            return deque(maxlen=100)
+
+        self._client_operations: dict[str, deque[float]] = defaultdict(_create_deque)
         self._operation_history: list[OperationMetrics] = []
 
-        # Circuit breaker tracking
         self._circuit_states: dict[str, OperationState] = defaultdict(
-            lambda: OperationState.CLOSED
+            lambda: OperationState.CLOSED  # type: ignore[misc]
         )
         self._failure_counts: dict[str, int] = defaultdict(int)
         self._last_failure_times: dict[str, float] = {}
 
-        # Resource usage tracking
         self._total_memory_usage = 0
         self._total_cpu_time = 0.0
         self._total_file_operations = 0
 
-        # Operation type definitions
         self._operation_types = {
             "collect_status": "Status collection operation",
             "get_jobs": "Job information retrieval",
@@ -138,32 +99,10 @@ class BoundedStatusOperations:
         *args: t.Any,
         **kwargs: t.Any,
     ) -> t.Any:
-        """
-        Execute a bounded status operation with comprehensive limits.
-
-        Args:
-            operation_type: Type of operation being performed
-            client_id: Client identifier
-            operation_func: Async function to execute
-            *args: Positional arguments for operation_func
-            **kwargs: Keyword arguments for operation_func
-
-        Returns:
-            Result of the operation
-
-        Raises:
-            OperationLimitExceededError: If operation limits exceeded
-            CircuitBreakerOpenError: If circuit breaker is open
-            asyncio.TimeoutError: If operation times out
-        """
-
-        # Check circuit breaker
         self._check_circuit_breaker(operation_type)
 
-        # Validate operation can be started
         operation_id = self._validate_and_reserve_operation(operation_type, client_id)
 
-        # Create operation metrics
         metrics = OperationMetrics(
             operation_id=operation_id,
             operation_type=operation_type,
@@ -171,23 +110,18 @@ class BoundedStatusOperations:
         )
 
         try:
-            # Register operation
             with self._lock:
                 self._active_operations[operation_id] = metrics
 
-            # Execute with timeout and resource monitoring
             result = await self._execute_with_monitoring(
                 operation_func, metrics, *args, **kwargs
             )
 
-            # Mark as successful
             metrics.success = True
             metrics.end_time = time.time()
 
-            # Update circuit breaker on success
             self._record_operation_success(operation_type)
 
-            # Log successful operation
             self.security_logger.log_security_event(
                 event_type=SecurityEventType.OPERATION_SUCCESS,
                 level=SecurityEventLevel.LOW,
@@ -206,15 +140,12 @@ class BoundedStatusOperations:
             return result
 
         except Exception as e:
-            # Mark as failed
             metrics.success = False
             metrics.end_time = time.time()
             metrics.error_message = str(e)
 
-            # Update circuit breaker on failure
             self._record_operation_failure(operation_type)
 
-            # Log failed operation
             self.security_logger.log_security_event(
                 event_type=SecurityEventType.OPERATION_FAILURE,
                 level=SecurityEventLevel.HIGH,
@@ -231,12 +162,9 @@ class BoundedStatusOperations:
             raise
 
         finally:
-            # Clean up operation
             self._cleanup_operation(operation_id, metrics)
 
     def _check_circuit_breaker(self, operation_type: str) -> None:
-        """Check if circuit breaker allows operation."""
-
         current_time = time.time()
 
         with self._lock:
@@ -245,9 +173,7 @@ class BoundedStatusOperations:
             last_failure = self._last_failure_times.get(operation_type, 0)
 
             if state == OperationState.OPEN:
-                # Check if circuit breaker timeout has passed
                 if current_time - last_failure >= self.limits.circuit_breaker_timeout:
-                    # Move to half-open state for testing
                     self._circuit_states[operation_type] = OperationState.HALF_OPEN
                     self.security_logger.log_security_event(
                         event_type=SecurityEventType.CIRCUIT_BREAKER_HALF_OPEN,
@@ -256,7 +182,6 @@ class BoundedStatusOperations:
                         operation=operation_type,
                     )
                 else:
-                    # Circuit breaker still open
                     self.security_logger.log_security_event(
                         event_type=SecurityEventType.CIRCUIT_BREAKER_OPEN,
                         level=SecurityEventLevel.MEDIUM,
@@ -275,13 +200,10 @@ class BoundedStatusOperations:
     def _validate_and_reserve_operation(
         self, operation_type: str, client_id: str
     ) -> str:
-        """Validate operation can be started and reserve resources."""
-
         current_time = time.time()
         operation_id = f"{operation_type}_{client_id}_{int(current_time * 1000)}"
 
         with self._lock:
-            # Check concurrent operation limit
             if len(self._active_operations) >= self.limits.max_concurrent_operations:
                 self.security_logger.log_security_event(
                     event_type=SecurityEventType.RATE_LIMIT_EXCEEDED,
@@ -294,10 +216,8 @@ class BoundedStatusOperations:
                     f"Maximum concurrent operations exceeded: {len(self._active_operations)}"
                 )
 
-            # Check client rate limiting
             client_ops = self._client_operations[client_id]
 
-            # Remove operations older than 1 minute
             while client_ops and current_time - client_ops[0] > 60.0:
                 client_ops.popleft()
 
@@ -313,7 +233,6 @@ class BoundedStatusOperations:
                     f"Operation rate limit exceeded: {len(client_ops)} operations/min"
                 )
 
-            # Check resource usage limits
             if (
                 self._total_memory_usage
                 >= self.limits.max_memory_usage_mb * 1024 * 1024
@@ -321,15 +240,14 @@ class BoundedStatusOperations:
                 self.security_logger.log_security_event(
                     event_type=SecurityEventType.RESOURCE_EXHAUSTED,
                     level=SecurityEventLevel.HIGH,
-                    message=f"Memory limit exceeded: {self._total_memory_usage / 1024 / 1024:.1f}MB",
+                    message=f"Memory limit exceeded: {self._total_memory_usage / 1024 / 1024: .1f}MB",
                     client_id=client_id,
                     operation=operation_type,
                 )
                 raise OperationLimitExceededError(
-                    f"Memory limit exceeded: {self._total_memory_usage / 1024 / 1024:.1f}MB"
+                    f"Memory limit exceeded: {self._total_memory_usage / 1024 / 1024: .1f}MB"
                 )
 
-            # Record operation start
             client_ops.append(current_time)
 
         return operation_id
@@ -341,13 +259,9 @@ class BoundedStatusOperations:
         *args: t.Any,
         **kwargs: t.Any,
     ) -> t.Any:
-        """Execute operation with resource monitoring and timeout."""
-
-        # Create monitoring task
         monitor_task = asyncio.create_task(self._monitor_operation(metrics))
 
         try:
-            # Execute operation with timeout
             result = await asyncio.wait_for(
                 operation_func(*args, **kwargs),
                 timeout=self.limits.timeout_seconds,
@@ -372,7 +286,6 @@ class BoundedStatusOperations:
             )
 
         finally:
-            # Cancel monitoring
             monitor_task.cancel()
             try:
                 await monitor_task
@@ -380,8 +293,6 @@ class BoundedStatusOperations:
                 pass
 
     async def _monitor_operation(self, metrics: OperationMetrics) -> None:
-        """Monitor operation resource usage."""
-
         import os
 
         import psutil
@@ -392,21 +303,18 @@ class BoundedStatusOperations:
 
             while not metrics.is_completed:
                 try:
-                    # Update CPU time
                     current_cpu_time = (
                         process.cpu_times().user + process.cpu_times().system
                     )
                     metrics.cpu_time = current_cpu_time - initial_cpu_time
 
-                    # Update memory usage
                     metrics.memory_usage = process.memory_info().rss
 
-                    # Check limits
                     if metrics.cpu_time > self.limits.max_cpu_time_seconds:
                         self.security_logger.log_security_event(
                             event_type=SecurityEventType.RESOURCE_LIMIT_EXCEEDED,
                             level=SecurityEventLevel.MEDIUM,
-                            message=f"CPU time limit exceeded: {metrics.cpu_time:.2f}s",
+                            message=f"CPU time limit exceeded: {metrics.cpu_time: .2f}s",
                             client_id=metrics.client_id,
                             operation=metrics.operation_type,
                             additional_data={"operation_id": metrics.operation_id},
@@ -417,20 +325,19 @@ class BoundedStatusOperations:
                         self.security_logger.log_security_event(
                             event_type=SecurityEventType.OPERATION_DURATION_EXCEEDED,
                             level=SecurityEventLevel.MEDIUM,
-                            message=f"Operation duration limit exceeded: {metrics.duration:.2f}s",
+                            message=f"Operation duration limit exceeded: {metrics.duration: .2f}s",
                             client_id=metrics.client_id,
                             operation=metrics.operation_type,
                             additional_data={"operation_id": metrics.operation_id},
                         )
                         break
 
-                    await asyncio.sleep(0.1)  # Monitor every 100ms
+                    await asyncio.sleep(0.1)
 
                 except psutil.NoSuchProcess:
                     break
 
         except Exception as e:
-            # Monitoring failure shouldn't stop the operation
             self.security_logger.log_security_event(
                 event_type=SecurityEventType.MONITORING_ERROR,
                 level=SecurityEventLevel.MEDIUM,
@@ -440,13 +347,10 @@ class BoundedStatusOperations:
             )
 
     def _record_operation_success(self, operation_type: str) -> None:
-        """Record successful operation for circuit breaker."""
-
         with self._lock:
             state = self._circuit_states[operation_type]
 
             if state == OperationState.HALF_OPEN:
-                # Success in half-open state - close circuit breaker
                 self._circuit_states[operation_type] = OperationState.CLOSED
                 self._failure_counts[operation_type] = 0
 
@@ -457,12 +361,9 @@ class BoundedStatusOperations:
                     operation=operation_type,
                 )
             elif state == OperationState.CLOSED:
-                # Reset failure count on success
                 self._failure_counts[operation_type] = 0
 
     def _record_operation_failure(self, operation_type: str) -> None:
-        """Record failed operation for circuit breaker."""
-
         current_time = time.time()
 
         with self._lock:
@@ -474,7 +375,6 @@ class BoundedStatusOperations:
 
             if failure_count >= self.limits.circuit_breaker_threshold:
                 if state != OperationState.OPEN:
-                    # Trip circuit breaker
                     self._circuit_states[operation_type] = OperationState.OPEN
 
                     self.security_logger.log_security_event(
@@ -489,19 +389,14 @@ class BoundedStatusOperations:
                     )
 
     def _cleanup_operation(self, operation_id: str, metrics: OperationMetrics) -> None:
-        """Clean up operation and update metrics."""
-
         with self._lock:
-            # Remove from active operations
             if operation_id in self._active_operations:
                 del self._active_operations[operation_id]
 
-            # Add to history (keep last 1000 operations)
             self._operation_history.append(metrics)
             if len(self._operation_history) > 1000:
                 self._operation_history.pop(0)
 
-            # Update resource usage tracking
             self._total_memory_usage = max(
                 0, self._total_memory_usage - metrics.memory_usage
             )
@@ -509,16 +404,11 @@ class BoundedStatusOperations:
             self._total_file_operations += metrics.file_operations
 
     def get_operation_status(self) -> dict[str, t.Any]:
-        """Get current operation status and limits."""
-
         with self._lock:
             current_time = time.time()
 
-            # Calculate recent operation stats
             recent_ops = [
-                m
-                for m in self._operation_history
-                if current_time - m.start_time < 300  # Last 5 minutes
+                m for m in self._operation_history if current_time - m.start_time < 300
             ]
 
             successful_ops = [m for m in recent_ops if m.success is True]
@@ -558,16 +448,6 @@ class BoundedStatusOperations:
             }
 
     def reset_circuit_breaker(self, operation_type: str) -> bool:
-        """
-        Manually reset a circuit breaker.
-
-        Args:
-            operation_type: Type of operation
-
-        Returns:
-            True if circuit breaker was reset
-        """
-
         with self._lock:
             if operation_type in self._circuit_states:
                 self._circuit_states[operation_type] = OperationState.CLOSED
@@ -585,15 +465,12 @@ class BoundedStatusOperations:
         return False
 
 
-# Global singleton instance
 _bounded_operations: BoundedStatusOperations | None = None
 
 
 def get_bounded_status_operations(
     limits: OperationLimits | None = None,
 ) -> BoundedStatusOperations:
-    """Get the global bounded status operations instance."""
-
     global _bounded_operations
     if _bounded_operations is None:
         _bounded_operations = BoundedStatusOperations(limits)
@@ -607,20 +484,6 @@ async def execute_bounded_status_operation(
     *args: t.Any,
     **kwargs: t.Any,
 ) -> t.Any:
-    """
-    Convenience function for bounded operation execution.
-
-    Args:
-        operation_type: Type of operation
-        client_id: Client identifier
-        operation_func: Async function to execute
-        *args: Positional arguments
-        **kwargs: Keyword arguments
-
-    Returns:
-        Operation result
-    """
-
     operations_manager = get_bounded_status_operations()
     return await operations_manager.execute_bounded_operation(
         operation_type, client_id, operation_func, *args, **kwargs

crackerjack/services/cache.py

@@ -220,12 +220,41 @@ class FileCache:
 
 
 class CrackerjackCache:
+    # Expensive hooks that benefit from disk caching across sessions
+    EXPENSIVE_HOOKS = {
+        "pyright",
+        "bandit",
+        "vulture",
+        "complexipy",
+        "refurb",
+        "gitleaks",
+        "detect-secrets",
+    }
+
+    # TTL configuration for different cache types (in seconds)
+    HOOK_DISK_TTLS = {
+        "pyright": 86400,  # 24 hours - type checking is stable
+        "bandit": 86400 * 3,  # 3 days - security patterns change slowly
+        "vulture": 86400 * 2,  # 2 days - dead code detection is stable
+        "complexipy": 86400,  # 24 hours - complexity analysis
+        "refurb": 86400,  # 24 hours - code improvements
+        "gitleaks": 86400 * 7,  # 7 days - secret detection is very stable
+        "detect-secrets": 86400 * 7,  # 7 days - secret detection
+    }
+
+    # Agent version for cache invalidation when agent logic changes
+    AGENT_VERSION = "1.0.0"
+
     def __init__(
         self,
         cache_dir: Path | None = None,
         enable_disk_cache: bool = True,
     ) -> None:
-        self.cache_dir = cache_dir or Path.cwd() / ".crackerjack_cache"
+        if cache_dir:
+            self.cache_dir = cache_dir
+        else:
+            self.cache_dir = Path.cwd() / ".crackerjack" / "cache"
+
         self.enable_disk_cache = enable_disk_cache
 
         self.hook_results_cache = InMemoryCache(max_entries=500, default_ttl=1800)
@@ -252,6 +281,46 @@ class CrackerjackCache:
         cache_key = self._get_hook_cache_key(hook_name, file_hashes)
         self.hook_results_cache.set(cache_key, result, ttl_seconds=1800)
 
+    def get_expensive_hook_result(
+        self,
+        hook_name: str,
+        file_hashes: list[str],
+        tool_version: str | None = None,
+    ) -> HookResult | None:
+        """Get hook result with disk cache fallback for expensive hooks."""
+        # Always check memory first for speed
+        result = self.get_hook_result(hook_name, file_hashes)
+        if result:
+            return result
+
+        # Fall back to disk cache for expensive hooks
+        if self.enable_disk_cache and hook_name in self.EXPENSIVE_HOOKS:
+            cache_key = self._get_versioned_hook_cache_key(
+                hook_name, file_hashes, tool_version
+            )
+            return self.disk_cache.get(cache_key)
+
+        return None
+
+    def set_expensive_hook_result(
+        self,
+        hook_name: str,
+        file_hashes: list[str],
+        result: HookResult,
+        tool_version: str | None = None,
+    ) -> None:
+        """Set hook result in both memory and disk cache for expensive hooks."""
+        # Always set[t.Any] in memory for current session
+        self.set_hook_result(hook_name, file_hashes, result)
+
+        # Also persist to disk for expensive hooks
+        if self.enable_disk_cache and hook_name in self.EXPENSIVE_HOOKS:
+            cache_key = self._get_versioned_hook_cache_key(
+                hook_name, file_hashes, tool_version
+            )
+            ttl = self.HOOK_DISK_TTLS.get(hook_name, 86400)  # Default 24 hours
+            self.disk_cache.set(cache_key, result, ttl_seconds=ttl)
+
     def get_file_hash(self, file_path: Path) -> str | None:
         stat = file_path.stat()
         cache_key = f"file_hash: {file_path}: {stat.st_mtime}: {stat.st_size}"
@@ -268,6 +337,49 @@ class CrackerjackCache:
     def set_config_data(self, config_key: str, data: t.Any) -> None:
         self.config_cache.set(f"config: {config_key}", data, ttl_seconds=7200)
 
+    def get(self, key: str, default: t.Any = None) -> t.Any:
+        """General purpose get method for metrics and other data."""
+        return self.config_cache.get(key) or default
+
+    def set(self, key: str, value: t.Any, ttl_seconds: int | None = None) -> None:
+        """General purpose set method for metrics and other data."""
+        ttl = ttl_seconds or 3600  # Default 1 hour
+        self.config_cache.set(key, value, ttl_seconds=ttl)
+
+    def get_agent_decision(self, agent_name: str, issue_hash: str) -> t.Any | None:
+        """Get cached AI agent decision based on issue content."""
+        if not self.enable_disk_cache:
+            return None
+
+        cache_key = f"agent:{agent_name}:{issue_hash}:{self.AGENT_VERSION}"
+        return self.disk_cache.get(cache_key)
+
+    def set_agent_decision(
+        self, agent_name: str, issue_hash: str, decision: t.Any
+    ) -> None:
+        """Cache AI agent decision for future use."""
+        if not self.enable_disk_cache:
+            return
+
+        cache_key = f"agent:{agent_name}:{issue_hash}:{self.AGENT_VERSION}"
+        self.disk_cache.set(cache_key, decision, ttl_seconds=604800)  # 7 days
+
+    def get_quality_baseline(self, git_hash: str) -> dict[str, t.Any] | None:
+        """Get quality baseline metrics for a specific git commit."""
+        if not self.enable_disk_cache:
+            return None
+
+        return self.disk_cache.get(f"baseline:{git_hash}")
+
+    def set_quality_baseline(self, git_hash: str, metrics: dict[str, t.Any]) -> None:
+        """Store quality baseline metrics for a git commit."""
+        if not self.enable_disk_cache:
+            return
+
+        self.disk_cache.set(
+            f"baseline:{git_hash}", metrics, ttl_seconds=2592000
+        )  # 30 days
+
     def invalidate_hook_cache(self, hook_name: str | None = None) -> None:
         if hook_name:
             keys_to_remove = [
@@ -310,3 +422,17 @@ class CrackerjackCache:
             usedforsecurity=False,
         ).hexdigest()
         return f"hook_result: {hook_name}: {hash_signature}"
+
+    def _get_versioned_hook_cache_key(
+        self,
+        hook_name: str,
+        file_hashes: list[str],
+        tool_version: str | None = None,
+    ) -> str:
+        """Get cache key with tool version for disk cache invalidation."""
+        hash_signature = hashlib.md5(
+            ", ".join(sorted(file_hashes)).encode(),
+            usedforsecurity=False,
+        ).hexdigest()
+        version_part = f":{tool_version}" if tool_version else ""
+        return f"hook_result: {hook_name}: {hash_signature}{version_part}"