crackerjack 0.30.3__py3-none-any.whl → 0.31.4__py3-none-any.whl

crackerjack/executors/async_hook_executor.py

@@ -0,0 +1,431 @@
+import asyncio
+import time
+import typing as t
+from dataclasses import dataclass
+from pathlib import Path
+
+from rich.console import Console
+
+from crackerjack.config.hooks import HookDefinition, HookStrategy, RetryPolicy
+from crackerjack.models.task import HookResult
+from crackerjack.services.logging import LoggingContext, get_logger
+
+
+@dataclass
+class AsyncHookExecutionResult:
+    strategy_name: str
+    results: list[HookResult]
+    total_duration: float
+    success: bool
+    concurrent_execution: bool = True
+    cache_hits: int = 0
+    cache_misses: int = 0
+    performance_gain: float = 0.0
+
+    @property
+    def failed_count(self) -> int:
+        return sum(1 for r in self.results if r.status == "failed")
+
+    @property
+    def passed_count(self) -> int:
+        return sum(1 for r in self.results if r.status == "passed")
+
+    @property
+    def cache_hit_rate(self) -> float:
+        total_requests = self.cache_hits + self.cache_misses
+        return (self.cache_hits / total_requests * 100) if total_requests > 0 else 0.0
+
+    @property
+    def performance_summary(self) -> dict[str, t.Any]:
+        return {
+            "total_hooks": len(self.results),
+            "passed": self.passed_count,
+            "failed": self.failed_count,
+            "duration_seconds": round(self.total_duration, 2),
+            "concurrent": self.concurrent_execution,
+            "cache_hits": self.cache_hits,
+            "cache_misses": self.cache_misses,
+            "cache_hit_rate_percent": round(self.cache_hit_rate, 1),
+            "performance_gain_percent": round(self.performance_gain, 1),
+        }
+
+
+class AsyncHookExecutor:
+    def __init__(
+        self,
+        console: Console,
+        pkg_path: Path,
+        max_concurrent: int = 4,
+        timeout: int = 300,
+        quiet: bool = False,
+    ) -> None:
+        self.console = console
+        self.pkg_path = pkg_path
+        self.max_concurrent = max_concurrent
+        self.timeout = timeout
+        self.quiet = quiet
+        self.logger = get_logger("crackerjack.async_hook_executor")
+
+        self._semaphore = asyncio.Semaphore(max_concurrent)
+
+    async def execute_strategy(
+        self,
+        strategy: HookStrategy,
+    ) -> AsyncHookExecutionResult:
+        with LoggingContext(
+            "async_hook_strategy",
+            strategy_name=strategy.name,
+            hook_count=len(strategy.hooks),
+        ):
+            start_time = time.time()
+            self.logger.info(
+                "Starting async hook strategy execution",
+                strategy=strategy.name,
+                hooks=len(strategy.hooks),
+                parallel=strategy.parallel,
+                max_workers=getattr(strategy, "max_workers", self.max_concurrent),
+            )
+
+            self._print_strategy_header(strategy)
+
+            estimated_sequential = sum(
+                getattr(hook, "timeout", 30) for hook in strategy.hooks
+            )
+
+            if strategy.parallel and len(strategy.hooks) > 1:
+                results = await self._execute_parallel(strategy)
+            else:
+                results = await self._execute_sequential(strategy)
+
+            if strategy.retry_policy != RetryPolicy.NONE:
+                results = await self._handle_retries(strategy, results)
+
+            total_duration = time.time() - start_time
+            success = all(r.status == "passed" for r in results)
+            performance_gain = max(
+                0,
+                ((estimated_sequential - total_duration) / estimated_sequential) * 100,
+            )
+
+            self.logger.info(
+                "Async hook strategy completed",
+                strategy=strategy.name,
+                success=success,
+                duration_seconds=round(total_duration, 2),
+                performance_gain_percent=round(performance_gain, 1),
+                passed=sum(1 for r in results if r.status == "passed"),
+                failed=sum(1 for r in results if r.status == "failed"),
+                errors=sum(1 for r in results if r.status in ("timeout", "error")),
+            )
+
+            if not self.quiet:
+                self._print_summary(strategy, results, success, performance_gain)
+
+            return AsyncHookExecutionResult(
+                strategy_name=strategy.name,
+                results=results,
+                total_duration=total_duration,
+                success=success,
+                performance_gain=performance_gain,
+            )
+
+    def _print_strategy_header(self, strategy: HookStrategy) -> None:
+        self.console.print("\n" + "-" * 80)
+        if strategy.name == "fast":
+            self.console.print(
+                "[bold bright_cyan]🔍 HOOKS[/bold bright_cyan] [bold bright_white]Running code quality checks (async)[/bold bright_white]",
+            )
+        elif strategy.name == "comprehensive":
+            self.console.print(
+                "[bold bright_cyan]🔍 HOOKS[/bold bright_cyan] [bold bright_white]Running comprehensive quality checks (async)[/bold bright_white]",
+            )
+        else:
+            self.console.print(
+                f"[bold bright_cyan]🔍 HOOKS[/bold bright_cyan] [bold bright_white]Running {strategy.name} hooks (async)[/bold bright_white]",
+            )
+        self.console.print("-" * 80 + "\n")
+
+    async def _execute_sequential(self, strategy: HookStrategy) -> list[HookResult]:
+        results: list[HookResult] = []
+        for hook in strategy.hooks:
+            result = await self._execute_single_hook(hook)
+            results.append(result)
+            self._display_hook_result(result)
+        return results
+
+    async def _execute_parallel(self, strategy: HookStrategy) -> list[HookResult]:
+        results: list[HookResult] = []
+
+        formatting_hooks = [
+            h for h in strategy.hooks if getattr(h, "is_formatting", False)
+        ]
+        other_hooks = [
+            h for h in strategy.hooks if not getattr(h, "is_formatting", False)
+        ]
+
+        for hook in formatting_hooks:
+            result = await self._execute_single_hook(hook)
+            results.append(result)
+            self._display_hook_result(result)
+
+        if other_hooks:
+            tasks = [self._execute_single_hook(hook) for hook in other_hooks]
+            parallel_results = await asyncio.gather(*tasks, return_exceptions=True)
+
+            for i, result in enumerate(parallel_results):
+                if isinstance(result, Exception):
+                    hook = other_hooks[i]
+                    error_result = HookResult(
+                        id=getattr(hook, "name", f"hook_{i}"),
+                        name=getattr(hook, "name", f"hook_{i}"),
+                        status="error",
+                        duration=0.0,
+                        issues_found=[str(result)],
+                        stage=hook.stage.value,
+                    )
+                    results.append(error_result)
+                    self._display_hook_result(error_result)
+                else:
+                    hook_result = t.cast("HookResult", result)
+                    results.append(hook_result)
+                    self._display_hook_result(hook_result)
+
+        return results
+
+    async def _execute_single_hook(self, hook: HookDefinition) -> HookResult:
+        async with self._semaphore:
+            return await self._run_hook_subprocess(hook)
+
+    async def _run_hook_subprocess(self, hook: HookDefinition) -> HookResult:
+        start_time = time.time()
+
+        try:
+            cmd = hook.get_command() if hasattr(hook, "get_command") else [str(hook)]
+            timeout_val = getattr(hook, "timeout", self.timeout)
+
+            self.logger.debug(
+                "Starting hook execution",
+                hook=hook.name,
+                command=" ".join(cmd),
+                timeout=timeout_val,
+            )
+
+            process = await asyncio.create_subprocess_exec(
+                *cmd,
+                cwd=self.pkg_path,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+            )
+
+            try:
+                stdout, stderr = await asyncio.wait_for(
+                    process.communicate(),
+                    timeout=timeout_val,
+                )
+            except TimeoutError:
+                process.kill()
+                await process.wait()
+                duration = time.time() - start_time
+
+                self.logger.warning(
+                    "Hook execution timed out",
+                    hook=hook.name,
+                    timeout=timeout_val,
+                    duration_seconds=round(duration, 2),
+                )
+
+                return HookResult(
+                    id=hook.name,
+                    name=hook.name,
+                    status="timeout",
+                    duration=duration,
+                    issues_found=[f"Hook timed out after {duration: .1f}s"],
+                    stage=hook.stage.value,
+                )
+
+            duration = time.time() - start_time
+            output_text = (
+                (stdout.decode() + stderr.decode()) if stdout and stderr else ""
+            )
+            return_code = process.returncode if process.returncode is not None else -1
+            parsed_output = self._parse_hook_output(return_code, output_text)
+
+            status = "passed" if return_code == 0 else "failed"
+
+            self.logger.info(
+                "Hook execution completed",
+                hook=hook.name,
+                status=status,
+                duration_seconds=round(duration, 2),
+                return_code=process.returncode,
+                files_processed=parsed_output.get("files_processed", 0),
+                issues_count=len(parsed_output.get("issues", [])),
+            )
+
+            return HookResult(
+                id=parsed_output.get("hook_id", hook.name),
+                name=hook.name,
+                status=status,
+                duration=duration,
+                files_processed=parsed_output.get("files_processed", 0),
+                issues_found=parsed_output.get("issues", []),
+                stage=hook.stage.value,
+            )
+
+        except Exception as e:
+            duration = time.time() - start_time
+            self.logger.exception(
+                "Hook execution failed with exception",
+                hook=hook.name,
+                error=str(e),
+                error_type=type(e).__name__,
+                duration_seconds=round(duration, 2),
+            )
+            return HookResult(
+                id=hook.name,
+                name=hook.name,
+                status="error",
+                duration=duration,
+                issues_found=[str(e)],
+                stage=hook.stage.value,
+            )
+
+    def _parse_hook_output(self, returncode: int, output: str) -> dict[str, t.Any]:
+        return {
+            "hook_id": None,
+            "exit_code": returncode,
+            "files_processed": 0,
+            "issues": [],
+            "raw_output": output,
+        }
+
+    def _display_hook_result(self, result: HookResult) -> None:
+        dots = "." * (60 - len(result.name))
+        status_text = "Passed" if result.status == "passed" else "Failed"
+        status_color = "green" if result.status == "passed" else "red"
+
+        self.console.print(
+            f"{result.name}{dots}[{status_color}]{status_text}[/{status_color}]",
+        )
+
+        if result.status != "passed" and result.issues_found:
+            for issue in result.issues_found:
+                if issue and "raw_output" not in issue:
+                    self.console.print(issue)
+
+    async def _handle_retries(
+        self,
+        strategy: HookStrategy,
+        results: list[HookResult],
+    ) -> list[HookResult]:
+        if strategy.retry_policy == RetryPolicy.FORMATTING_ONLY:
+            return await self._retry_formatting_hooks(strategy, results)
+        if strategy.retry_policy == RetryPolicy.ALL_HOOKS:
+            return await self._retry_all_hooks(strategy, results)
+        return results
+
+    async def _retry_formatting_hooks(
+        self,
+        strategy: HookStrategy,
+        results: list[HookResult],
+    ) -> list[HookResult]:
+        formatting_hooks_failed: set[str] = set()
+
+        for i, result in enumerate(results):
+            hook = strategy.hooks[i]
+            if getattr(hook, "is_formatting", False) and result.status == "failed":
+                formatting_hooks_failed.add(hook.name)
+
+        if not formatting_hooks_failed:
+            return results
+
+        retry_tasks = [self._execute_single_hook(hook) for hook in strategy.hooks]
+        retry_results = await asyncio.gather(*retry_tasks, return_exceptions=True)
+
+        updated_results: list[HookResult] = []
+        for i, (prev_result, new_result) in enumerate(
+            zip(results, retry_results, strict=False)
+        ):
+            if isinstance(new_result, Exception):
+                hook = strategy.hooks[i]
+                error_result = HookResult(
+                    id=hook.name,
+                    name=hook.name,
+                    status="error",
+                    duration=prev_result.duration,
+                    issues_found=[str(new_result)],
+                    stage=hook.stage.value,
+                )
+                updated_results.append(error_result)
+            else:
+                hook_result = t.cast("HookResult", new_result)
+                hook_result.duration += prev_result.duration
+                updated_results.append(hook_result)
+
+            self._display_hook_result(updated_results[-1])
+
+        return updated_results
+
+    async def _retry_all_hooks(
+        self,
+        strategy: HookStrategy,
+        results: list[HookResult],
+    ) -> list[HookResult]:
+        failed_indices = [i for i, r in enumerate(results) if r.status == "failed"]
+
+        if not failed_indices:
+            return results
+
+        updated_results = results.copy()
+        retry_tasks: list[t.Awaitable[HookResult]] = []
+        retry_indices: list[int] = []
+
+        for i in failed_indices:
+            hook = strategy.hooks[i]
+            retry_tasks.append(self._execute_single_hook(hook))
+            retry_indices.append(i)
+
+        retry_results = await asyncio.gather(*retry_tasks, return_exceptions=True)
+
+        for result_idx, new_result in zip(retry_indices, retry_results, strict=False):
+            prev_result = results[result_idx]
+
+            if isinstance(new_result, Exception):
+                hook = strategy.hooks[result_idx]
+                error_result = HookResult(
+                    id=hook.name,
+                    name=hook.name,
+                    status="error",
+                    duration=prev_result.duration,
+                    issues_found=[str(new_result)],
+                    stage=hook.stage.value,
+                )
+                updated_results[result_idx] = error_result
+            else:
+                hook_result = t.cast("HookResult", new_result)
+                hook_result.duration += prev_result.duration
+                updated_results[result_idx] = hook_result
+
+            self._display_hook_result(updated_results[result_idx])
+
+        return updated_results
+
+    def _print_summary(
+        self,
+        strategy: HookStrategy,
+        results: list[HookResult],
+        success: bool,
+        performance_gain: float,
+    ) -> None:
+        if success:
+            self.console.print(
+                f"[green]✅[/green] {strategy.name.title()} hooks passed: {len(results)} / {len(results)} "
+                f"(async, {performance_gain: .1f} % faster)",
+            )
+        else:
+            failed_count = sum(1 for r in results if r.status == "failed")
+            error_count = sum(1 for r in results if r.status in ("timeout", "error"))
+            self.console.print(
+                f"[red]❌[/red] {strategy.name.title()} hooks failed: {failed_count} failed, {error_count} errors "
+                f"(async, {performance_gain: .1f} % faster)",
+            )

crackerjack/executors/cached_hook_executor.py

@@ -0,0 +1,242 @@
+import logging
+import time
+import typing as t
+from pathlib import Path
+
+from rich.console import Console
+
+from crackerjack.config.hooks import HookDefinition, HookStrategy
+from crackerjack.models.task import HookResult
+from crackerjack.services.cache import CrackerjackCache
+from crackerjack.services.file_hasher import FileHasher
+
+from .hook_executor import HookExecutionResult, HookExecutor
+
+
+class CachedHookExecutor:
+    def __init__(
+        self,
+        console: Console,
+        pkg_path: Path,
+        cache: CrackerjackCache | None = None,
+        cache_ttl_seconds: int = 1800,
+    ) -> None:
+        self.console = console
+        self.pkg_path = pkg_path
+        self.cache = cache or CrackerjackCache()
+        self.cache_ttl_seconds = cache_ttl_seconds
+        self.file_hasher = FileHasher(self.cache)
+        self.base_executor = HookExecutor(console, pkg_path, quiet=True)
+        self.logger = logging.getLogger("crackerjack.cached_executor")
+
+        self.file_patterns = {
+            "python": [" * .py"],
+            "config": [" * .toml", " * .cfg", " * .ini", " * .yaml", " * .yml"],
+            "all": [
+                " * .py",
+                " * .toml",
+                " * .cfg",
+                " * .ini",
+                " * .yaml",
+                " * .yml",
+                " * .md",
+                " * .txt",
+            ],
+        }
+
+    def execute_strategy(self, strategy: HookStrategy) -> HookExecutionResult:
+        self.logger.info(
+            f"Executing cached strategy '{strategy.name}' with {len(strategy.hooks)} hooks",
+        )
+
+        start_time = time.time()
+        results: list[HookResult] = []
+        cache_hits = 0
+        cache_misses = 0
+
+        relevant_files = self._get_relevant_files_for_strategy(strategy)
+        current_file_hashes = self.file_hasher.get_files_hash_list(relevant_files)
+
+        for hook_def in strategy.hooks:
+            cached_result = None
+            try:
+                cached_result = self.cache.get_hook_result(
+                    hook_def.name,
+                    current_file_hashes,
+                )
+            except Exception as e:
+                self.logger.warning(f"Cache error for hook {hook_def.name}: {e}")
+                cached_result = None
+
+            if cached_result and self._is_cache_valid(cached_result, hook_def):
+                self.logger.debug(f"Using cached result for hook: {hook_def.name}")
+                results.append(cached_result)
+                cache_hits += 1
+            else:
+                self.logger.debug(f"Executing hook (cache miss): {hook_def.name}")
+
+                hook_result = self.base_executor.execute_single_hook(hook_def)
+                results.append(hook_result)
+                cache_misses += 1
+
+                if hook_result.status == "passed":
+                    try:
+                        self.cache.set_hook_result(
+                            hook_def.name,
+                            current_file_hashes,
+                            hook_result,
+                        )
+                    except Exception as e:
+                        self.logger.warning(
+                            f"Failed to cache result for {hook_def.name}: {e}",
+                        )
+
+        total_time = time.time() - start_time
+        success = all(result.status == "passed" for result in results)
+
+        self.logger.info(
+            f"Cached strategy '{strategy.name}' completed in {total_time: .2f}s - "
+            f"Success: {success}, Cache hits: {cache_hits}, Cache misses: {cache_misses}",
+        )
+
+        return HookExecutionResult(
+            strategy_name=strategy.name,
+            results=results,
+            success=success,
+            total_duration=total_time,
+            cache_hits=cache_hits,
+            cache_misses=cache_misses,
+        )
+
+    def _get_relevant_files_for_strategy(self, strategy: HookStrategy) -> list[Path]:
+        if self._strategy_affects_python_only(strategy):
+            patterns = self.file_patterns["python"]
+        elif self._strategy_affects_config_only(strategy):
+            patterns = self.file_patterns["config"]
+        else:
+            patterns = self.file_patterns["all"]
+
+        files: list[Path] = []
+        for pattern in patterns:
+            files.extend(list(self.pkg_path.rglob(pattern)))
+
+        return [f for f in files if f.is_file() and not self._should_ignore_file(f)]
+
+    def _strategy_affects_python_only(self, strategy: HookStrategy) -> bool:
+        python_only_hooks = {
+            "ruff - format",
+            "ruff - check",
+            "pyright",
+            "bandit",
+            "vulture",
+            "refurb",
+            "complexipy",
+        }
+        return all(hook.name in python_only_hooks for hook in strategy.hooks)
+
+    def _strategy_affects_config_only(self, strategy: HookStrategy) -> bool:
+        config_only_hooks = {"creosote"}
+        return all(hook.name in config_only_hooks for hook in strategy.hooks)
+
+    def _should_ignore_file(self, file_path: Path) -> bool:
+        ignore_patterns = [
+            ".git/",
+            ".venv/",
+            "__pycache__/",
+            ".pytest_cache/",
+            ".coverage",
+            ".crackerjack_cache/",
+            "node_modules/",
+            ".tox/",
+            "dist/",
+            "build/",
+            ".egg - info/",
+        ]
+
+        path_str = str(file_path)
+        return any(pattern in path_str for pattern in ignore_patterns)
+
+    def _is_cache_valid(
+        self,
+        cached_result: HookResult,
+        hook_def: HookDefinition,
+    ) -> bool:
+        if cached_result.status != "passed":
+            return False
+
+        cache_age = time.time() - getattr(cached_result, "timestamp", time.time())
+        return not cache_age > self.cache_ttl_seconds
+
+    def invalidate_hook_cache(self, hook_name: str | None = None) -> None:
+        self.cache.invalidate_hook_cache(hook_name)
+        self.logger.info(f"Invalidated cache for hook: {hook_name or 'all hooks'}")
+
+    def get_cache_stats(self) -> dict[str, t.Any]:
+        return self.cache.get_cache_stats()
+
+    def cleanup_cache(self) -> dict[str, int]:
+        return self.cache.cleanup_all()
+
+
+class SmartCacheManager:
+    def __init__(self, cached_executor: CachedHookExecutor) -> None:
+        self.cached_executor = cached_executor
+        self.logger = logging.getLogger("crackerjack.cache_manager")
+
+    def should_use_cache_for_hook(
+        self,
+        hook_name: str,
+        project_state: dict[str, t.Any],
+    ) -> bool:
+        external_hooks = {"detect - secrets"}
+        if hook_name in external_hooks:
+            return False
+
+        expensive_hooks = {"pyright", "bandit", "vulture", "complexipy"}
+        if hook_name in expensive_hooks:
+            return True
+
+        formatting_hooks = {
+            "ruff-format",
+            "trailing-whitespace",
+            "end-of-file-fixer",
+        }
+        if hook_name in formatting_hooks:
+            recent_changes = project_state.get("recent_changes", 0)
+            return recent_changes < 5
+
+        return True
+
+    def get_optimal_cache_strategy(
+        self,
+        hook_strategy: HookStrategy,
+    ) -> dict[str, bool]:
+        project_state = self._analyze_project_state()
+
+        cache_decisions = {}
+        for hook_def in hook_strategy.hooks:
+            cache_decisions[hook_def.name] = self.should_use_cache_for_hook(
+                hook_def.name,
+                project_state,
+            )
+
+        return cache_decisions
+
+    def _analyze_project_state(self) -> dict[str, t.Any]:
+        pkg_path = self.cached_executor.pkg_path
+
+        recent_changes = 0
+        for py_file in pkg_path.rglob("*.py"):
+            if py_file.is_file():
+                from contextlib import suppress
+
+                with suppress(OSError):
+                    mtime = py_file.stat().st_mtime
+                    if time.time() - mtime < 3600:
+                        recent_changes += 1
+
+        return {
+            "recent_changes": recent_changes,
+            "total_python_files": len(list(pkg_path.rglob("*.py"))),
+            "project_size": "large" if recent_changes > 50 else "small",
+        }