crackerjack 0.30.3__py3-none-any.whl → 0.31.4__py3-none-any.whl

crackerjack/services/enhanced_filesystem.py

@@ -0,0 +1,439 @@
+import asyncio
+import hashlib
+import time
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Any
+
+import aiofiles
+
+from crackerjack.errors import FileError
+from crackerjack.models.protocols import FileSystemInterface
+from crackerjack.services.logging import LoggingContext, get_logger
+
+
+class FileCache:
+    def __init__(self, max_size: int = 1000, default_ttl: float = 300.0) -> None:
+        self.max_size = max_size
+        self.default_ttl = default_ttl
+        self._cache: dict[str, dict[str, Any]] = {}
+        self._access_times: dict[str, float] = {}
+        self.logger = get_logger("crackerjack.filesystem.cache")
+
+    def get(self, key: str) -> str | None:
+        if key not in self._cache:
+            return None
+
+        cache_entry = self._cache[key]
+        now = time.time()
+
+        if now - cache_entry["timestamp"] > cache_entry["ttl"]:
+            self._evict(key)
+            return None
+
+        self._access_times[key] = now
+        self.logger.debug("Cache hit", key=key)
+        return cache_entry["content"]
+
+    def put(self, key: str, content: str, ttl: float | None = None) -> None:
+        if len(self._cache) >= self.max_size:
+            self._evict_lru()
+
+        now = time.time()
+        self._cache[key] = {
+            "content": content,
+            "timestamp": now,
+            "ttl": ttl or self.default_ttl,
+            "size": len(content),
+        }
+        self._access_times[key] = now
+        self.logger.debug("Cache put", key=key, size=len(content))
+
+    def _evict(self, key: str) -> None:
+        self._cache.pop(key, None)
+        self._access_times.pop(key, None)
+
+    def _evict_lru(self) -> None:
+        if not self._access_times:
+            return
+
+        lru_key = min(self._access_times, key=lambda k: self._access_times.get(k, 0.0))
+        self._evict(lru_key)
+        self.logger.debug("Cache LRU eviction", key=lru_key)
+
+    def clear(self) -> None:
+        self._cache.clear()
+        self._access_times.clear()
+        self.logger.debug("Cache cleared")
+
+    def get_stats(self) -> dict[str, Any]:
+        total_size = sum(entry["size"] for entry in self._cache.values())
+        return {
+            "entries": len(self._cache),
+            "max_size": self.max_size,
+            "total_content_size": total_size,
+            "memory_usage_mb": total_size / (1024 * 1024),
+        }
+
+
+class BatchFileOperations:
+    def __init__(self, batch_size: int = 10) -> None:
+        self.batch_size = batch_size
+        self.read_queue: list[tuple[Path, asyncio.Future[str]]] = []
+        self.write_queue: list[tuple[Path, str, asyncio.Future[None]]] = []
+        self.logger = get_logger("crackerjack.filesystem.batch")
+
+    async def queue_read(self, path: Path) -> str:
+        future: asyncio.Future[str] = asyncio.Future()
+        self.read_queue.append((path, future))
+
+        if len(self.read_queue) >= self.batch_size:
+            await self._flush_reads()
+
+        return await future
+
+    async def queue_write(self, path: Path, content: str) -> None:
+        future: asyncio.Future[None] = asyncio.Future()
+        self.write_queue.append((path, content, future))
+
+        if len(self.write_queue) >= self.batch_size:
+            await self._flush_writes()
+
+        await future
+
+    async def flush_all(self) -> None:
+        await asyncio.gather(
+            self._flush_reads(),
+            self._flush_writes(),
+            return_exceptions=True,
+        )
+
+    async def _flush_reads(self) -> None:
+        if not self.read_queue:
+            return
+
+        with LoggingContext("batch_file_reads", count=len(self.read_queue)):
+            batch = self.read_queue.copy()
+            self.read_queue.clear()
+
+            from itertools import starmap
+
+            tasks = list(starmap(self._read_single_async, batch))
+
+            await asyncio.gather(*tasks, return_exceptions=True)
+
+    async def _flush_writes(self) -> None:
+        if not self.write_queue:
+            return
+
+        with LoggingContext("batch_file_writes", count=len(self.write_queue)):
+            batch = self.write_queue.copy()
+            self.write_queue.clear()
+
+            from itertools import starmap
+
+            tasks = list(starmap(self._write_single_async, batch))
+
+            await asyncio.gather(*tasks, return_exceptions=True)
+
+    async def _read_single_async(self, path: Path, future: asyncio.Future[str]) -> None:
+        try:
+            async with aiofiles.open(path, encoding="utf-8") as f:
+                content = await f.read()
+            future.set_result(content)
+        except Exception as e:
+            future.set_exception(e)
+
+    async def _write_single_async(
+        self,
+        path: Path,
+        content: str,
+        future: asyncio.Future[None],
+    ) -> None:
+        try:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            async with aiofiles.open(path, "w", encoding="utf-8") as f:
+                await f.write(content)
+            future.set_result(None)
+        except Exception as e:
+            future.set_exception(e)
+
+
+class EnhancedFileSystemService(FileSystemInterface):
+    def __init__(
+        self,
+        cache_size: int = 1000,
+        cache_ttl: float = 300.0,
+        batch_size: int = 10,
+        enable_async: bool = True,
+    ) -> None:
+        self.cache = FileCache(cache_size, cache_ttl)
+        self.batch_ops = BatchFileOperations(batch_size) if enable_async else None
+        self.enable_async = enable_async
+        self.logger = get_logger("crackerjack.filesystem.enhanced")
+
+        self._file_timestamps: dict[str, float] = {}
+
+    def read_file(self, path: str | Path) -> str:
+        path_obj = Path(path) if isinstance(path, str) else path
+
+        with LoggingContext("read_file", path=str(path_obj)):
+            cache_key = self._get_cache_key(path_obj)
+            cached_content = self._get_from_cache(cache_key, path_obj)
+
+            if cached_content is not None:
+                return cached_content
+
+            content = self._read_file_direct(path_obj)
+
+            self.cache.put(cache_key, content)
+            self._file_timestamps[str(path_obj)] = path_obj.stat().st_mtime
+
+            return content
+
+    def write_file(self, path: str | Path, content: str) -> None:
+        if not isinstance(content, str):  # type: ignore[arg-type]
+            raise TypeError("Content must be a string")
+
+        path_obj = Path(path) if isinstance(path, str) else path
+
+        with LoggingContext("write_file", path=str(path_obj), size=len(content)):
+            self._write_file_direct(path_obj, content)
+
+            cache_key = self._get_cache_key(path_obj)
+            self.cache._evict(cache_key)
+            self._file_timestamps[str(path_obj)] = time.time()
+
+    async def read_file_async(self, path: Path) -> str:
+        if not self.enable_async or not self.batch_ops:
+            return self.read_file(path)
+
+        cache_key = self._get_cache_key(path)
+        cached_content = self._get_from_cache(cache_key, path)
+
+        if cached_content is not None:
+            return cached_content
+
+        content = await self.batch_ops.queue_read(path)
+
+        self.cache.put(cache_key, content)
+        self._file_timestamps[str(path)] = path.stat().st_mtime
+
+        return content
+
+    async def write_file_async(self, path: Path, content: str) -> None:
+        if not self.enable_async or not self.batch_ops:
+            self.write_file(path, content)
+            return
+
+        await self.batch_ops.queue_write(path, content)
+
+        cache_key = self._get_cache_key(path)
+        self.cache._evict(cache_key)
+        self._file_timestamps[str(path)] = time.time()
+
+    async def read_multiple_files(self, paths: list[Path]) -> dict[Path, str]:
+        results = {}
+
+        if not self.enable_async or not self.batch_ops:
+            for path in paths:
+                try:
+                    results[path] = self.read_file(path)
+                except Exception as e:
+                    self.logger.exception(
+                        "Failed to read file",
+                        path=str(path),
+                        error=str(e),
+                    )
+                    results[path] = ""
+            return results
+
+        with LoggingContext("read_multiple_files", count=len(paths)):
+            tasks = [self.read_file_async(path) for path in paths]
+
+            results_list = await asyncio.gather(*tasks, return_exceptions=True)
+
+            for path, result in zip(paths, results_list, strict=False):
+                if isinstance(result, Exception):
+                    self.logger.error(
+                        "Failed to read file",
+                        path=str(path),
+                        error=str(result),
+                    )
+                    results[path] = ""
+                else:
+                    results[path] = result
+
+            return results
+
+    async def write_multiple_files(self, file_data: dict[Path, str]) -> None:
+        if not self.enable_async or not self.batch_ops:
+            for path, content in file_data.items():
+                try:
+                    self.write_file(path, content)
+                except Exception as e:
+                    self.logger.exception(
+                        "Failed to write file",
+                        path=str(path),
+                        error=str(e),
+                    )
+            return
+
+        with LoggingContext("write_multiple_files", count=len(file_data)):
+            from itertools import starmap
+
+            tasks = list(starmap(self.write_file_async, file_data.items()))
+
+            await asyncio.gather(*tasks, return_exceptions=True)
+
+    def _get_cache_key(self, path: Path) -> str:
+        path_str = str(path.resolve())
+        return hashlib.md5(path_str.encode(), usedforsecurity=False).hexdigest()
+
+    def _get_from_cache(self, cache_key: str, path: Path) -> str | None:
+        if not path.exists():
+            return None
+
+        path_str = str(path)
+        if path_str in self._file_timestamps:
+            current_mtime = path.stat().st_mtime
+            cached_mtime = self._file_timestamps[path_str]
+
+            if current_mtime > cached_mtime:
+                self.cache._evict(cache_key)
+                del self._file_timestamps[path_str]
+                return None
+
+        return self.cache.get(cache_key)
+
+    def _read_file_direct(self, path: Path) -> str:
+        try:
+            if not path.exists():
+                raise FileError(
+                    message=f"File does not exist: {path}",
+                    details=f"Attempted to read file at {path.absolute()}",
+                    recovery="Check file path and ensure file exists",
+                )
+            return path.read_text(encoding="utf-8")
+        except PermissionError as e:
+            raise FileError(
+                message=f"Permission denied reading file: {path}",
+                details=str(e),
+                recovery="Check file permissions and user access rights",
+            ) from e
+        except UnicodeDecodeError as e:
+            raise FileError(
+                message=f"Unable to decode file as UTF-8: {path}",
+                details=str(e),
+                recovery="Ensure file is text-based and UTF-8 encoded",
+            ) from e
+        except OSError as e:
+            raise FileError(
+                message=f"System error reading file: {path}",
+                details=str(e),
+                recovery="Check disk space and file system integrity",
+            ) from e
+
+    def _write_file_direct(self, path: Path, content: str) -> None:
+        try:
+            try:
+                path.parent.mkdir(parents=True, exist_ok=True)
+            except OSError as e:
+                raise FileError(
+                    message=f"Cannot create parent directory: {path.parent}",
+                    details=str(e),
+                    recovery="Check directory permissions and disk space",
+                ) from e
+
+            path.write_text(content, encoding="utf-8")
+
+        except PermissionError as e:
+            raise FileError(
+                message=f"Permission denied writing file: {path}",
+                details=str(e),
+                recovery="Check file and directory permissions",
+            ) from e
+        except OSError as e:
+            raise FileError(
+                message=f"System error writing file: {path}",
+                details=str(e),
+                recovery="Check disk space and file system integrity",
+            ) from e
+
+    def file_exists(self, path: str | Path) -> bool:
+        return (Path(path) if isinstance(path, str) else path).exists()
+
+    def create_directory(self, path: str | Path) -> None:
+        path_obj = Path(path) if isinstance(path, str) else path
+        try:
+            path_obj.mkdir(parents=True, exist_ok=True)
+            self.logger.debug("Directory created", path=str(path_obj))
+        except OSError as e:
+            raise FileError(
+                message=f"Cannot create directory: {path_obj}",
+                details=str(e),
+                recovery="Check parent directory permissions and disk space",
+            ) from e
+
+    def delete_file(self, path: str | Path) -> None:
+        path_obj = Path(path) if isinstance(path, str) else path
+
+        try:
+            if path_obj.exists():
+                path_obj.unlink()
+
+                cache_key = self._get_cache_key(path_obj)
+                self.cache._evict(cache_key)
+                self._file_timestamps.pop(str(path_obj), None)
+
+                self.logger.debug("File deleted", path=str(path_obj))
+        except OSError as e:
+            raise FileError(
+                message=f"Cannot delete file: {path_obj}",
+                details=str(e),
+                recovery="Check file permissions",
+            ) from e
+
+    def list_files(self, path: str | Path, pattern: str = "*") -> Iterator[Path]:
+        path_obj = Path(path) if isinstance(path, str) else path
+
+        if not path_obj.is_dir():
+            raise FileError(
+                message=f"Path is not a directory: {path_obj}",
+                details=f"Cannot list files in {path_obj}",
+                recovery="Ensure path points to a valid directory",
+            )
+
+        try:
+            yield from path_obj.glob(pattern)
+        except OSError as e:
+            raise FileError(
+                message=f"Cannot list files in directory: {path_obj}",
+                details=str(e),
+                recovery="Check directory permissions",
+            ) from e
+
+    async def flush_operations(self) -> None:
+        if self.batch_ops:
+            await self.batch_ops.flush_all()
+
+    def get_cache_stats(self) -> dict[str, Any]:
+        return self.cache.get_stats()
+
+    def clear_cache(self) -> None:
+        self.cache.clear()
+        self._file_timestamps.clear()
+
+    def exists(self, path: str | Path) -> bool:
+        return (Path(path) if isinstance(path, str) else path).exists()
+
+    def mkdir(self, path: str | Path, parents: bool = False) -> None:
+        path_obj = Path(path) if isinstance(path, str) else path
+        try:
+            path_obj.mkdir(parents=parents, exist_ok=True)
+        except OSError as e:
+            raise FileError(
+                message=f"Cannot create directory: {path_obj}",
+                details=str(e),
+                recovery="Check parent directory permissions",
+            ) from e

crackerjack/services/file_hasher.py

@@ -0,0 +1,151 @@
+import asyncio
+import hashlib
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+from .cache import CrackerjackCache
+
+
+class FileHasher:
+    def __init__(self, cache: CrackerjackCache | None = None) -> None:
+        self.cache = cache or CrackerjackCache()
+        self._executor = ThreadPoolExecutor(max_workers=4)
+
+    def get_file_hash(self, file_path: Path, algorithm: str = "md5") -> str:
+        if not file_path.exists():
+            return ""
+
+        cached_hash = self.cache.get_file_hash(file_path)
+        if cached_hash:
+            return cached_hash
+
+        file_hash = self._compute_file_hash(file_path, algorithm)
+        self.cache.set_file_hash(file_path, file_hash)
+        return file_hash
+
+    def get_directory_hash(
+        self,
+        directory: Path,
+        patterns: list[str] | None = None,
+    ) -> str:
+        if patterns is None:
+            patterns = ["*.py"]
+
+        file_hashes: list[str] = []
+        for pattern in patterns:
+            for file_path in directory.rglob(pattern):
+                if file_path.is_file():
+                    file_hash = self.get_file_hash(file_path)
+                    file_hashes.append(
+                        f"{file_path.relative_to(directory)}: {file_hash}",
+                    )
+
+        file_hashes.sort()
+        combined_content = "\n".join(file_hashes)
+        return hashlib.md5(combined_content.encode(), usedforsecurity=False).hexdigest()
+
+    def get_files_hash_list(self, files: list[Path]) -> list[str]:
+        return [
+            self.get_file_hash(file_path) for file_path in files if file_path.exists()
+        ]
+
+    async def get_files_hash_list_async(self, files: list[Path]) -> list[str]:
+        loop = asyncio.get_running_loop()
+        tasks = [
+            loop.run_in_executor(self._executor, self.get_file_hash, file_path)
+            for file_path in files
+            if file_path.exists()
+        ]
+        return await asyncio.gather(*tasks)
+
+    def has_files_changed(self, files: list[Path], cached_hashes: list[str]) -> bool:
+        if len(files) != len(cached_hashes):
+            return True
+
+        current_hashes = self.get_files_hash_list(files)
+        return current_hashes != cached_hashes
+
+    def _compute_file_hash(self, file_path: Path, algorithm: str = "md5") -> str:
+        hash_func = hashlib.new(algorithm)
+
+        try:
+            with file_path.open("rb") as f:
+                while chunk := f.read(8192):
+                    hash_func.update(chunk)
+            return hash_func.hexdigest()
+        except OSError:
+            return ""
+
+    def get_project_files_hash(self, project_path: Path) -> dict[str, str]:
+        patterns = ["*.py", "*.toml", "*.cfg", "*.ini", "*.yaml", "*.yml"]
+        file_hashes = {}
+
+        for pattern in patterns:
+            for file_path in project_path.rglob(pattern):
+                if file_path.is_file() and not self._should_ignore_file(file_path):
+                    relative_path = str(file_path.relative_to(project_path))
+                    file_hashes[relative_path] = self.get_file_hash(file_path)
+
+        return file_hashes
+
+    def _should_ignore_file(self, file_path: Path) -> bool:
+        ignore_patterns = [
+            ".git",
+            ".venv",
+            "__pycache__",
+            ".pytest_cache",
+            ".coverage",
+            ".crackerjack_cache",
+            "node_modules",
+            ".tox",
+            "dist",
+            "build",
+            "*.egg-info",
+        ]
+
+        path_str = str(file_path)
+        return any(pattern in path_str for pattern in ignore_patterns)
+
+    def invalidate_cache(self, file_path: Path | None = None) -> None:
+        if file_path:
+            stat = file_path.stat() if file_path.exists() else None
+            if stat:
+                cache_key = f"file_hash:{file_path}:{stat.st_mtime}:{stat.st_size}"
+                self.cache.file_hash_cache.invalidate(cache_key)
+        else:
+            self.cache.file_hash_cache.clear()
+
+
+class SmartFileWatcher:
+    def __init__(self, file_hasher: FileHasher) -> None:
+        self.file_hasher = file_hasher
+        self._watched_files: dict[Path, tuple[float, int]] = {}
+
+    def register_files(self, files: list[Path]) -> None:
+        for file_path in files:
+            if file_path.exists():
+                stat = file_path.stat()
+                self._watched_files[file_path] = (stat.st_mtime, stat.st_size)
+
+    def check_changes(self) -> list[Path]:
+        changed_files: list[Path] = []
+
+        for file_path, (old_mtime, old_size) in self._watched_files.items():
+            if not file_path.exists():
+                changed_files.append(file_path)
+                continue
+
+            stat = file_path.stat()
+            if stat.st_mtime != old_mtime or stat.st_size != old_size:
+                changed_files.append(file_path)
+                self._watched_files[file_path] = (stat.st_mtime, stat.st_size)
+
+        return changed_files
+
+    def invalidate_changed_files(self) -> int:
+        changed_files = self.check_changes()
+
+        for file_path in changed_files:
+            self.file_hasher.invalidate_cache(file_path)
+
+        return len(changed_files)