codebatch 0.1.0__py3-none-any.whl

codebatch/paths.py

@@ -0,0 +1,196 @@
+"""Path canonicalization and safety utilities.
+
+All file paths in a snapshot are canonicalized:
+- UTF-8 encoded
+- / as separator
+- No . or .. segments
+- No trailing slash
+- Stable casing preserved
+
+A path_key is included for normalized comparison (lowercase).
+"""
+
+import os
+import re
+from pathlib import Path, PurePosixPath
+from typing import Tuple
+
+
+class PathEscapeError(Exception):
+    """Raised when a path attempts to escape the root directory."""
+
+    def __init__(self, path: str, reason: str):
+        self.path = path
+        self.reason = reason
+        super().__init__(f"Path escape attempt: {path} - {reason}")
+
+
+class InvalidPathError(Exception):
+    """Raised for invalid path characters or structure."""
+
+    def __init__(self, path: str, reason: str):
+        self.path = path
+        self.reason = reason
+        super().__init__(f"Invalid path: {path} - {reason}")
+
+
+# Characters not allowed in paths (Windows restrictions + control chars)
+INVALID_CHARS = re.compile(r'[\x00-\x1f<>:"|?*]')
+
+# Reserved Windows filenames
+RESERVED_NAMES = frozenset([
+    "CON", "PRN", "AUX", "NUL",
+    "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9",
+    "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9",
+])
+
+
+def canonicalize_path(path: str, root: Path = None) -> str:
+    """Canonicalize a file path for storage.
+
+    Args:
+        path: Input path (may use any separator).
+        root: Optional root directory for escape detection.
+
+    Returns:
+        Canonicalized path with / separators, no . or .. segments.
+
+    Raises:
+        PathEscapeError: If path escapes the root.
+        InvalidPathError: If path contains invalid characters.
+    """
+    if not path:
+        raise InvalidPathError(path, "empty path")
+
+    # Check for invalid characters
+    if INVALID_CHARS.search(path):
+        raise InvalidPathError(path, "contains invalid characters")
+
+    # Normalize separators to forward slash
+    normalized = path.replace("\\", "/")
+
+    # Remove leading/trailing whitespace
+    normalized = normalized.strip()
+
+    # Remove trailing slash
+    normalized = normalized.rstrip("/")
+
+    if not normalized:
+        raise InvalidPathError(path, "path is empty after normalization")
+
+    # Split into components and resolve . and ..
+    parts = normalized.split("/")
+    resolved = []
+
+    for part in parts:
+        if not part or part == ".":
+            # Skip empty parts and current directory
+            continue
+        elif part == "..":
+            # Parent directory - check for escape
+            if not resolved:
+                raise PathEscapeError(path, "attempts to go above root")
+            resolved.pop()
+        else:
+            # Check for reserved Windows names
+            base_name = part.split(".")[0].upper()
+            if base_name in RESERVED_NAMES:
+                raise InvalidPathError(path, f"contains reserved name: {part}")
+            resolved.append(part)
+
+    if not resolved:
+        raise InvalidPathError(path, "path resolves to root")
+
+    canonical = "/".join(resolved)
+
+    # If root provided, verify the resolved path doesn't escape
+    if root is not None:
+        try:
+            full_path = (root / canonical).resolve()
+            root_resolved = root.resolve()
+            # Check that full_path is under root
+            try:
+                full_path.relative_to(root_resolved)
+            except ValueError:
+                raise PathEscapeError(path, "resolved path escapes root")
+        except OSError:
+            # Path resolution failed - might be too long or invalid
+            pass
+
+    return canonical
+
+
+def compute_path_key(path: str) -> str:
+    """Compute a normalized path key for comparison.
+
+    The path_key is lowercase with normalized separators,
+    used for case-insensitive comparisons and collision detection.
+
+    Args:
+        path: Canonicalized path.
+
+    Returns:
+        Lowercase path key.
+    """
+    # Path should already be canonicalized (/ separators, no . or ..)
+    return path.lower()
+
+
+def canonicalize_with_key(path: str, root: Path = None) -> Tuple[str, str]:
+    """Canonicalize a path and compute its key in one call.
+
+    Args:
+        path: Input path.
+        root: Optional root directory for escape detection.
+
+    Returns:
+        Tuple of (canonical_path, path_key).
+    """
+    canonical = canonicalize_path(path, root)
+    key = compute_path_key(canonical)
+    return canonical, key
+
+
+def is_safe_path(path: str, root: Path = None) -> bool:
+    """Check if a path is safe (doesn't escape root, valid characters).
+
+    Args:
+        path: Path to check.
+        root: Optional root directory.
+
+    Returns:
+        True if path is safe, False otherwise.
+    """
+    try:
+        canonicalize_path(path, root)
+        return True
+    except (PathEscapeError, InvalidPathError):
+        return False
+
+
+def detect_case_collision(paths: list[str]) -> list[Tuple[str, str]]:
+    """Detect case collisions in a list of paths.
+
+    Args:
+        paths: List of canonicalized paths.
+
+    Returns:
+        List of colliding path pairs.
+    """
+    key_to_paths: dict[str, list[str]] = {}
+
+    for path in paths:
+        key = compute_path_key(path)
+        if key not in key_to_paths:
+            key_to_paths[key] = []
+        key_to_paths[key].append(path)
+
+    collisions = []
+    for key, path_list in key_to_paths.items():
+        if len(path_list) > 1:
+            # Return all pairs
+            for i, p1 in enumerate(path_list):
+                for p2 in path_list[i + 1:]:
+                    collisions.append((p1, p2))
+
+    return collisions

codebatch/query.py

@@ -0,0 +1,242 @@
+"""Query engine for output indexes.
+
+Answers questions from JSONL scans without requiring a database:
+- Which files produced diagnostics?
+- Which outputs exist for a given task?
+- Which files failed a given task?
+- Aggregate counts by kind, severity, or language
+"""
+
+import json
+from collections import Counter
+from pathlib import Path
+from typing import Iterator, Optional
+
+
+class QueryEngine:
+    """Query engine for batch output indexes."""
+
+    def __init__(self, store_root: Path):
+        """Initialize the query engine.
+
+        Args:
+            store_root: Root directory of the CodeBatch store.
+        """
+        self.store_root = Path(store_root)
+        self.batches_dir = self.store_root / "batches"
+
+    def _iter_shard_outputs(
+        self, batch_id: str, task_id: str
+    ) -> Iterator[dict]:
+        """Iterate over all output records for a task.
+
+        Args:
+            batch_id: Batch ID.
+            task_id: Task ID.
+
+        Yields:
+            Output record dicts.
+        """
+        shards_dir = self.batches_dir / batch_id / "tasks" / task_id / "shards"
+
+        if not shards_dir.exists():
+            return
+
+        for shard_dir in sorted(shards_dir.iterdir()):
+            if not shard_dir.is_dir():
+                continue
+
+            outputs_path = shard_dir / "outputs.index.jsonl"
+            if not outputs_path.exists():
+                continue
+
+            with open(outputs_path, "r", encoding="utf-8") as f:
+                for line in f:
+                    line = line.strip()
+                    if line:
+                        yield json.loads(line)
+
+    def query_diagnostics(
+        self,
+        batch_id: str,
+        task_id: str,
+        severity: Optional[str] = None,
+        code: Optional[str] = None,
+        path_pattern: Optional[str] = None,
+    ) -> list[dict]:
+        """Query diagnostic outputs.
+
+        Args:
+            batch_id: Batch ID.
+            task_id: Task ID.
+            severity: Filter by severity (error, warning, info, hint).
+            code: Filter by diagnostic code.
+            path_pattern: Filter by path substring.
+
+        Returns:
+            List of diagnostic records.
+        """
+        results = []
+
+        for record in self._iter_shard_outputs(batch_id, task_id):
+            if record.get("kind") != "diagnostic":
+                continue
+
+            if severity and record.get("severity") != severity:
+                continue
+
+            if code and record.get("code") != code:
+                continue
+
+            if path_pattern and path_pattern.lower() not in record.get("path", "").lower():
+                continue
+
+            results.append(record)
+
+        return results
+
+    def query_outputs(
+        self,
+        batch_id: str,
+        task_id: str,
+        kind: Optional[str] = None,
+        path_pattern: Optional[str] = None,
+    ) -> list[dict]:
+        """Query output records.
+
+        Args:
+            batch_id: Batch ID.
+            task_id: Task ID.
+            kind: Filter by output kind (ast, diagnostic, metric, etc.).
+            path_pattern: Filter by path substring.
+
+        Returns:
+            List of output records.
+        """
+        results = []
+
+        for record in self._iter_shard_outputs(batch_id, task_id):
+            if kind and record.get("kind") != kind:
+                continue
+
+            if path_pattern and path_pattern.lower() not in record.get("path", "").lower():
+                continue
+
+            results.append(record)
+
+        return results
+
+    def query_stats(
+        self,
+        batch_id: str,
+        task_id: str,
+        group_by: str = "kind",
+    ) -> dict[str, int]:
+        """Get aggregate statistics.
+
+        Args:
+            batch_id: Batch ID.
+            task_id: Task ID.
+            group_by: Field to group by (kind, severity, code, lang).
+
+        Returns:
+            Dict mapping group values to counts.
+        """
+        counter: Counter[str] = Counter()
+
+        for record in self._iter_shard_outputs(batch_id, task_id):
+            if group_by == "kind":
+                value = record.get("kind", "unknown")
+            elif group_by == "severity":
+                value = record.get("severity", "none")
+            elif group_by == "code":
+                value = record.get("code", "none")
+            elif group_by == "lang":
+                # Extract language from path extension
+                path = record.get("path", "")
+                ext = path.rsplit(".", 1)[-1] if "." in path else "none"
+                value = ext
+            else:
+                value = record.get(group_by, "unknown")
+
+            counter[value] += 1
+
+        return dict(counter)
+
+    def query_failed_files(
+        self, batch_id: str, task_id: str
+    ) -> list[str]:
+        """Get paths of files that produced error diagnostics.
+
+        Args:
+            batch_id: Batch ID.
+            task_id: Task ID.
+
+        Returns:
+            List of file paths with errors.
+        """
+        failed_paths = set()
+
+        for record in self._iter_shard_outputs(batch_id, task_id):
+            if record.get("kind") == "diagnostic" and record.get("severity") == "error":
+                failed_paths.add(record.get("path", ""))
+
+        return sorted(failed_paths)
+
+    def query_files_with_outputs(
+        self, batch_id: str, task_id: str, kind: str
+    ) -> list[str]:
+        """Get paths of files that produced outputs of a given kind.
+
+        Args:
+            batch_id: Batch ID.
+            task_id: Task ID.
+            kind: Output kind to filter by.
+
+        Returns:
+            List of file paths.
+        """
+        paths = set()
+
+        for record in self._iter_shard_outputs(batch_id, task_id):
+            if record.get("kind") == kind:
+                paths.add(record.get("path", ""))
+
+        return sorted(paths)
+
+    def get_task_summary(self, batch_id: str, task_id: str) -> dict:
+        """Get a summary of task outputs.
+
+        Args:
+            batch_id: Batch ID.
+            task_id: Task ID.
+
+        Returns:
+            Summary dict with counts.
+        """
+        total = 0
+        by_kind: Counter[str] = Counter()
+        by_severity: Counter[str] = Counter()
+        files_with_outputs: set[str] = set()
+        files_with_errors: set[str] = set()
+
+        for record in self._iter_shard_outputs(batch_id, task_id):
+            total += 1
+            kind = record.get("kind", "unknown")
+            by_kind[kind] += 1
+
+            if kind == "diagnostic":
+                severity = record.get("severity", "unknown")
+                by_severity[severity] += 1
+                if severity == "error":
+                    files_with_errors.add(record.get("path", ""))
+
+            files_with_outputs.add(record.get("path", ""))
+
+        return {
+            "total_outputs": total,
+            "by_kind": dict(by_kind),
+            "by_severity": dict(by_severity),
+            "files_with_outputs": len(files_with_outputs),
+            "files_with_errors": len(files_with_errors),
+        }