multi-forge 0.2.0__py3-none-any.whl

forge/core/state/io.py

@@ -0,0 +1,140 @@
+"""Atomic file operations for Forge state files.
+
+All write operations use the tempfile + os.replace() pattern for atomicity.
+This ensures that readers never see partial writes.
+
+Durability policy: No fsync. We rely on the filesystem's default behavior.
+This is a deliberate simplicity choice - if crash safety is needed later,
+add fsync before os.replace() and optionally fsync the directory.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+from pathlib import Path
+from typing import Any
+
+from .exceptions import StateCorruptedError, StateNotFoundError
+
+
+def open_secure_append(path: Path) -> Any:
+    """Open a file for append with 0600 permissions (owner read/write only).
+
+    Used for log files that may contain sensitive payloads (request bodies,
+    tool inputs, error messages). Creates the file with 0600 if missing;
+    chmods to 0600 if it already exists.
+
+    The post-open chmod has a tiny TOCTOU window for pre-existing files but
+    closes it on every subsequent write. New files are created with 0600
+    atomically (subject to umask, which only clears bits we already want clear).
+    """
+    fd = os.open(str(path), os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600)
+    try:
+        os.fchmod(fd, 0o600)
+    except OSError:
+        pass  # best-effort: some filesystems (e.g., CIFS) may not support fchmod
+    return os.fdopen(fd, "a", encoding="utf-8")
+
+
+def atomic_write_text(
+    path: Path,
+    content: str,
+    *,
+    create_parents: bool = True,
+) -> None:
+    """Write text to a file atomically.
+
+    Uses tempfile + os.replace() pattern to ensure readers never see
+    partial writes. The temp file is created in the same directory as
+    the target to ensure atomic rename works (same filesystem).
+
+    Args:
+        path: Target file path.
+        content: Text content to write.
+        create_parents: Create parent directories if they don't exist.
+
+    Raises:
+        OSError: If the write or rename fails.
+    """
+    if create_parents:
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+    # Create temp file in same directory for atomic rename
+    fd, temp_path = tempfile.mkstemp(
+        dir=str(path.parent),
+        prefix=f".{path.stem}.",
+        suffix=".tmp",
+    )
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as f:
+            f.write(content)
+        os.replace(temp_path, str(path))
+    except Exception:
+        # Clean up temp file on failure
+        try:
+            os.unlink(temp_path)
+        except OSError:
+            pass
+        raise
+
+
+def atomic_write_json(
+    path: Path,
+    data: dict[str, Any],
+    *,
+    indent: int = 2,
+    create_parents: bool = True,
+) -> None:
+    """Write JSON to a file atomically.
+
+    Serializes the dict to JSON and writes it atomically using
+    tempfile + os.replace(). Adds a trailing newline for git-friendliness.
+
+    Args:
+        path: Target file path.
+        data: Dict to serialize as JSON.
+        indent: JSON indentation level (default 2).
+        create_parents: Create parent directories if they don't exist.
+
+    Raises:
+        OSError: If the write or rename fails.
+        TypeError: If data contains non-serializable values.
+    """
+    content = json.dumps(data, indent=indent)
+    content += "\n"  # Trailing newline
+    atomic_write_text(path, content, create_parents=create_parents)
+
+
+def read_json(path: Path) -> dict[str, Any]:
+    """Read and parse a JSON file.
+
+    Args:
+        path: Path to JSON file.
+
+    Returns:
+        Parsed JSON as a dict.
+
+    Raises:
+        StateNotFoundError: If the file does not exist.
+        StateCorruptedError: If the file contains invalid JSON or is not a JSON object.
+    """
+    if not path.exists():
+        raise StateNotFoundError(str(path))
+
+    try:
+        with open(path, encoding="utf-8") as f:
+            data = json.load(f)
+    except json.JSONDecodeError as e:
+        raise StateCorruptedError(str(path), f"invalid JSON: {e}") from e
+    except OSError as e:
+        raise StateCorruptedError(str(path), f"read error: {e}") from e
+
+    if not isinstance(data, dict):
+        raise StateCorruptedError(
+            str(path),
+            f"expected JSON object, got {type(data).__name__}",
+        )
+
+    return data

forge/core/state/lock.py

@@ -0,0 +1,99 @@
+"""Cross-process file locking for Forge state.
+
+Forge state files are written atomically via write-temp + os.replace.
+That prevents torn reads, but it does NOT prevent concurrent *read-modify-write*
+flows from overwriting each other.
+
+This module provides a small, advisory lock primitive to serialize those RMW
+operations across processes.
+
+Design notes:
+- Locks are implemented using `fcntl.flock` (macOS/Linux).
+- We always lock a **separate lock file** (e.g., "index.json.lock"), not the
+  target file itself, because the target file inode changes on atomic replace.
+- This is intended to be best-effort. Callers should choose appropriate
+  timeouts (hooks: short/fail-open; CLI: longer).
+"""
+
+from __future__ import annotations
+
+import time
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Iterator
+
+from .exceptions import StateError
+
+
+class FileLockTimeoutError(StateError):
+    """Raised when a lock cannot be acquired within the timeout."""
+
+    def __init__(self, lock_path: Path, timeout_s: float) -> None:
+        self.lock_path = lock_path
+        self.timeout_s = timeout_s
+        super().__init__(f"timed out acquiring lock '{lock_path}' after {timeout_s:.3f}s")
+
+
+def get_lock_path_for_target(target_path: Path) -> Path:
+    """Return the lock file path for a target state file.
+
+    Example:
+        target: /home/user/.forge/sessions/index.json
+        lock:   /home/user/.forge/sessions/index.json.lock
+    """
+
+    return target_path.parent / f"{target_path.name}.lock"
+
+
+@contextmanager
+def file_lock(*, lock_path: Path, timeout_s: float, poll_s: float = 0.05) -> Iterator[None]:
+    """Acquire an exclusive advisory lock for the duration of the context.
+
+    Args:
+        lock_path: Path to the lock file.
+        timeout_s: Maximum time to wait for acquisition.
+        poll_s: Sleep interval between non-blocking retries.
+
+    Raises:
+        FileLockTimeoutError: If the lock cannot be acquired in time.
+        OSError: If the lock file cannot be created/opened.
+    """
+
+    # Local import: avoids importing fcntl on platforms where it may not exist.
+    import fcntl
+
+    lock_path.parent.mkdir(parents=True, exist_ok=True)
+
+    deadline = time.monotonic() + timeout_s
+
+    # Keep the fd open for the duration of the lock.
+    with lock_path.open("a+", encoding="utf-8") as f:
+        while True:
+            try:
+                fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+                break
+            except BlockingIOError:
+                if time.monotonic() >= deadline:
+                    raise FileLockTimeoutError(lock_path=lock_path, timeout_s=timeout_s)
+                time.sleep(poll_s)
+
+        try:
+            yield
+        finally:
+            try:
+                fcntl.flock(f.fileno(), fcntl.LOCK_UN)
+            except OSError:
+                # Best-effort cleanup; fd close will also release.
+                pass
+
+
+@contextmanager
+def file_lock_for_target(*, target_path: Path, timeout_s: float, poll_s: float = 0.05) -> Iterator[None]:
+    """Convenience wrapper to lock a target file by deriving its lock path."""
+
+    with file_lock(
+        lock_path=get_lock_path_for_target(target_path),
+        timeout_s=timeout_s,
+        poll_s=poll_s,
+    ):
+        yield

forge/core/state/timestamps.py

@@ -0,0 +1,60 @@
+"""Timestamp utilities for Forge state files.
+
+All timestamps are stored as ISO8601 strings for JSON compatibility.
+Uses UTC exclusively for consistent timestamps across time zones.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+
+def now_iso() -> str:
+    """Return current UTC time as ISO8601 string.
+
+    Format: '2024-01-15T10:30:00+00:00'
+
+    Returns:
+        ISO8601 formatted string with UTC timezone (+00:00 suffix).
+    """
+    return datetime.now(UTC).replace(microsecond=0).isoformat()
+
+
+def parse_iso(s: str) -> datetime:
+    """Parse ISO8601 string to timezone-aware datetime in UTC.
+
+    Handles common ISO8601 formats:
+    - With 'Z' suffix: '2024-01-15T10:30:00Z'
+    - With offset: '2024-01-15T10:30:00+00:00'
+
+    Args:
+        s: ISO8601 formatted string.
+
+    Returns:
+        Timezone-aware datetime normalized to UTC.
+
+    Raises:
+        ValueError: If the string is not valid ISO8601 or lacks timezone info.
+    """
+    normalized = s.replace("Z", "+00:00")
+    dt = datetime.fromisoformat(normalized)
+
+    if dt.tzinfo is None:
+        raise ValueError(f"ISO8601 string must include timezone info, got naive datetime: '{s}'")
+
+    return dt.astimezone(UTC)
+
+
+def iso_to_timestamp(iso_str: str) -> float:
+    """Convert ISO8601 string to Unix timestamp.
+
+    Args:
+        iso_str: ISO8601 formatted string with timezone.
+
+    Returns:
+        Unix timestamp as float (seconds since epoch).
+
+    Raises:
+        ValueError: If the string is not valid ISO8601 or lacks timezone info.
+    """
+    return parse_iso(iso_str).timestamp()

forge/core/transcript.py

@@ -0,0 +1,78 @@
+"""Shared JSONL transcript parsing primitives.
+
+Low-level parsing of Claude Code transcript files. Used by:
+- forge.search.extractor (content extraction for search indexing)
+- forge.session.handoff (context assembly for session resume)
+
+Only parsing primitives live here — extraction/summarization logic stays
+in each consumer module since they produce different output formats.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+def parse_jsonl_transcript(path: Path) -> list[dict[str, Any]]:
+    """Parse a Claude transcript JSONL file, sorted by timestamp.
+
+    Handles both formats:
+    - requestId/message.role (newer Claude Code)
+    - entry.type (older Claude Code)
+
+    Entries without "message" or "type" keys are silently skipped.
+    Malformed lines are skipped with a debug log.
+
+    Returns:
+        List of parsed entries sorted by timestamp. Empty list on read errors.
+    """
+    entries: list[dict[str, Any]] = []
+
+    if not path.is_file():
+        return entries
+
+    try:
+        with path.open(encoding="utf-8") as f:
+            for line_num, line in enumerate(f, 1):
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    entry = json.loads(line)
+                except json.JSONDecodeError:
+                    logger.debug("Skipping malformed JSON at line %d in %s", line_num, path)
+                    continue
+
+                if "message" not in entry and "type" not in entry:
+                    continue
+
+                entries.append(entry)
+    except Exception as e:
+        logger.warning("Error reading transcript %s: %s", path, e)
+        return []
+
+    entries.sort(key=_get_timestamp)
+    return entries
+
+
+def _get_timestamp(entry: dict[str, Any]) -> str:
+    """Extract timestamp from a transcript entry for sorting.
+
+    Checks top-level "timestamp" first, then "message.timestamp".
+    """
+    ts = entry.get("timestamp", "")
+    if not ts and "message" in entry:
+        ts = entry.get("message", {}).get("timestamp", "")
+    return ts if isinstance(ts, str) else ""
+
+
+def truncate(text: str, max_chars: int) -> str:
+    """Truncate text to max_chars, appending '...' if truncated."""
+    if len(text) <= max_chars:
+        return text
+    return text[:max_chars] + "..."

forge/core/typing_helpers.py

@@ -0,0 +1,24 @@
+"""Typing utility helpers shared across Forge modules."""
+
+from __future__ import annotations
+
+from typing import Any, get_args, get_origin
+
+
+def unwrap_optional(tp: Any) -> Any:
+    """Unwrap Optional[T] (i.e., Union[T, None]) to get T.
+
+    Returns the original type unchanged if it is not Optional.
+    """
+    origin = get_origin(tp)
+    if origin is None:
+        return tp
+
+    # Handle Union types (Optional is Union[T, None])
+    args = get_args(tp)
+    if args:
+        non_none = [a for a in args if a is not type(None)]
+        if len(non_none) == 1:
+            return non_none[0]
+
+    return tp

forge/core/workqueue/__init__.py

@@ -0,0 +1,67 @@
+"""Forge async work queue — general-purpose deferred processing primitive.
+
+Provides a file-based queue where producers enqueue markers and CLI startup
+processes them opportunistically. Markers are dispatched to handlers by kind.
+
+Quick Start:
+    from forge.core.workqueue import enqueue, process_pending_work, enqueue_stop_marker
+
+    # Enqueue a generic marker
+    enqueue(kind="index", marker_id="session-123", payload={"path": "..."})
+
+    # Enqueue a stop marker (convenience)
+    enqueue_stop_marker(session_id="uuid", worktree_path=Path(...), ...)
+
+    # Process with explicit handlers
+    def handle_index(marker):
+        index_session(marker.payload["path"])
+
+    process_pending_work(handlers={"index": handle_index})
+"""
+
+from .queue import (
+    MARKER_LOCK_TIMEOUT_S,
+    PROCESSOR_LOCK_TIMEOUT_S,
+    SAFE_MARKER_ID,
+    enqueue,
+    enqueue_handoff_marker,
+    enqueue_index_marker,
+    enqueue_stop_marker,
+    marker_path,
+    pending_work_dir,
+    process_pending_work,
+)
+from .types import (
+    FAILED_WORK_DIR,
+    MARKER_SCHEMA_VERSION,
+    MAX_ATTEMPTS,
+    MAX_ERROR_LENGTH,
+    PENDING_WORK_DIR,
+    Marker,
+    ProcessResult,
+    WorkHandler,
+)
+
+__all__ = [
+    # Queue operations
+    "enqueue",
+    "enqueue_handoff_marker",
+    "enqueue_index_marker",
+    "enqueue_stop_marker",
+    "marker_path",
+    "pending_work_dir",
+    "process_pending_work",
+    # Types
+    "Marker",
+    "ProcessResult",
+    "WorkHandler",
+    # Constants
+    "MARKER_SCHEMA_VERSION",
+    "MAX_ATTEMPTS",
+    "MAX_ERROR_LENGTH",
+    "PENDING_WORK_DIR",
+    "FAILED_WORK_DIR",
+    "MARKER_LOCK_TIMEOUT_S",
+    "PROCESSOR_LOCK_TIMEOUT_S",
+    "SAFE_MARKER_ID",
+]