chipgraph 0.1.0__py3-none-any.whl

chipgraph/core/state/journal.py

@@ -0,0 +1,238 @@
+"""The run journal: an append-only JSONL log of `Event`s, and the state replayed from it.
+
+DESIGN.md 6.2: "Journal chi ghi them. State duoc dung lai tu journal cong voi hash cua
+artifact." (The journal is append-only. State is rebuilt from the journal plus artifact
+hashes.) This module implements the journal itself and replay into a `RunState`; it does
+not know about artifact hashes.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from chipgraph.core.contracts.event import Event, RunManifest
+from chipgraph.core.contracts.types import FailureLabel
+from chipgraph.core.state.layout import StateLayout
+
+RuleStatus = Literal["pending", "running", "done", "failed", "waiting_gate"]
+"""The status of a single rule instance, as derived from the journal."""
+
+_EVENTS_WITHOUT_RULE_INSTANCE = frozenset({"run_start", "run_stop", "tool_call"})
+
+
+class JournalError(Exception):
+    """The journal is inconsistent: a bad sequence number, run id, or a corrupt line."""
+
+
+class RunState(BaseModel):
+    """The state of a run, rebuilt by replaying its journal."""
+
+    model_config = ConfigDict(frozen=True)
+
+    run_id: str
+    started: bool = False
+    stopped: bool = False
+    rules: dict[str, RuleStatus] = Field(default_factory=dict)
+    failures: dict[str, FailureLabel | None] = Field(default_factory=dict)
+    last_seq: int = -1
+
+
+class JournalRead(BaseModel):
+    """The result of reading a journal file: the events found, and whether it was truncated."""
+
+    model_config = ConfigDict(frozen=True)
+
+    events: tuple[Event, ...] = ()
+    truncated: bool = False
+
+
+class Journal:
+    """An append-only JSONL journal of `Event`s for a single run.
+
+    Each line is one `Event`, written with `model_dump_json()`. `append` fsyncs after every
+    write, so a killed process loses at most the in-flight line. On construction, `repair()`
+    removes any trailing partial line left by a previous crash, so the next `append` sees a
+    clean, complete file.
+    """
+
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._run_id: str | None = None
+        self._last_seq: int = -1
+        self.repair()
+        self._load_tail_state()
+
+    def _load_tail_state(self) -> None:
+        result = read(self.path)
+        if result.events:
+            self._run_id = result.events[-1].run_id
+            self._last_seq = result.events[-1].seq
+
+    def repair(self) -> None:
+        """Truncate the file to the end of its last complete, valid JSON line, if any.
+
+        This undoes a crash mid-write: a missing trailing newline, or a last line that is
+        not valid JSON. Complete lines before that point are untouched.
+        """
+        if not self.path.exists():
+            return
+        raw = self.path.read_bytes()
+        if not raw:
+            return
+        lines = raw.split(b"\n")
+        # split(b"\n") on a file ending in "\n" yields a trailing empty element.
+        if lines and lines[-1] == b"":
+            lines.pop()
+        if not lines:
+            return
+        last = lines[-1]
+        ends_with_newline = raw.endswith(b"\n")
+        valid_last = False
+        if ends_with_newline:
+            try:
+                json.loads(last)
+                valid_last = True
+            except json.JSONDecodeError:
+                valid_last = False
+        if valid_last:
+            return
+        # Truncate off the partial trailing line, keeping all prior complete lines.
+        good_len = sum(len(line) + 1 for line in lines[:-1])
+        with open(self.path, "r+b") as f:
+            f.truncate(good_len)
+            f.flush()
+            os.fsync(f.fileno())
+
+    def next_seq(self) -> int:
+        """The `seq` value the next appended event must use."""
+        return self._last_seq + 1
+
+    def append(self, event: Event) -> None:
+        """Append one event, enforcing strictly increasing `seq` and a stable `run_id`."""
+        if self._run_id is not None and event.run_id != self._run_id:
+            raise JournalError(
+                f"journal {self.path}: run_id mismatch: expected {self._run_id!r}, "
+                f"got {event.run_id!r}"
+            )
+        expected_seq = self.next_seq()
+        if event.seq != expected_seq:
+            raise JournalError(f"journal {self.path}: expected seq {expected_seq}, got {event.seq}")
+        line = event.model_dump_json()
+        with open(self.path, "a", encoding="utf-8") as f:
+            f.write(line + "\n")
+            f.flush()
+            os.fsync(f.fileno())
+        self._run_id = event.run_id
+        self._last_seq = event.seq
+
+
+def read(path: Path) -> JournalRead:
+    """Read every complete event line from a journal file.
+
+    A truncated last line (no trailing newline, or invalid JSON, on the final line only)
+    is ignored and reported via `JournalRead.truncated`. An invalid line that is not the
+    last one raises `JournalError` naming the 1-based line number.
+    """
+    if not path.exists():
+        return JournalRead()
+    raw = path.read_text(encoding="utf-8")
+    if raw == "":
+        return JournalRead()
+    ends_with_newline = raw.endswith("\n")
+    lines = raw.split("\n")
+    if lines and lines[-1] == "":
+        lines.pop()
+    if not lines:
+        return JournalRead()
+
+    events: list[Event] = []
+    truncated = False
+    n = len(lines)
+    for i, line in enumerate(lines):
+        line_no = i + 1
+        is_last = i == n - 1
+        try:
+            event = Event.model_validate(json.loads(line))
+        except Exception as exc:
+            if is_last:
+                truncated = True
+                break
+            raise JournalError(f"journal {path}: invalid line at line {line_no}") from exc
+        if is_last and not ends_with_newline:
+            # Valid JSON, but the file was cut off before its trailing newline: treat as
+            # a partial write and drop it too, so repair() and append() agree on the cut.
+            truncated = True
+            break
+        events.append(event)
+    return JournalRead(events=tuple(events), truncated=truncated)
+
+
+def replay(events: list[Event] | tuple[Event, ...]) -> RunState:
+    """Rebuild a `RunState` by folding a sequence of events in order."""
+    run_id = ""
+    started = False
+    stopped = False
+    rules: dict[str, RuleStatus] = {}
+    failures: dict[str, FailureLabel | None] = {}
+    last_seq = -1
+
+    for event in events:
+        run_id = event.run_id
+        last_seq = event.seq
+
+        if event.type == "run_start":
+            started = True
+            continue
+        if event.type == "run_stop":
+            stopped = True
+            continue
+        if event.type == "tool_call":
+            continue
+
+        rule_instance = event.rule_instance
+        if rule_instance is None:
+            # Lenient: ignore events that should carry a rule_instance but don't.
+            continue
+
+        if event.type == "rule_start":
+            rules[rule_instance] = "running"
+        elif event.type == "rule_done":
+            rules[rule_instance] = "done"
+        elif event.type == "rule_fail":
+            rules[rule_instance] = "failed"
+            failures[rule_instance] = event.failure_label
+        elif event.type == "gate_wait":
+            rules[rule_instance] = "waiting_gate"
+        elif event.type == "gate_decision":
+            rules[rule_instance] = "pending"
+        # check_result, agent_turn: recorded in the journal but do not change rule status.
+
+    return RunState(
+        run_id=run_id,
+        started=started,
+        stopped=stopped,
+        rules=rules,
+        failures=failures,
+        last_seq=last_seq,
+    )
+
+
+def write_manifest(layout: StateLayout, manifest: RunManifest) -> None:
+    """Atomically write a run's manifest: write to a tmp file, then `os.replace`."""
+    path = layout.manifest(manifest.run_id)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp_path = path.with_suffix(path.suffix + ".tmp")
+    tmp_path.write_text(manifest.model_dump_json(indent=2), encoding="utf-8")
+    os.replace(tmp_path, path)
+
+
+def read_manifest(layout: StateLayout, run_id: str) -> RunManifest:
+    """Read a run's manifest."""
+    path = layout.manifest(run_id)
+    return RunManifest.model_validate_json(path.read_text(encoding="utf-8"))

chipgraph/core/state/layout.py

@@ -0,0 +1,82 @@
+"""Where run state lives on disk (DESIGN.md 6.1): ``<repo>/.chipgraph/state/``.
+
+This is machine-local, gitignored state: journals, run manifests, locks, cache and
+idempotency results. It is distinct from ``.chipgraph/decisions/``, which holds
+human-auditable, per-decision files that a team may choose to commit (DESIGN.md 6.1).
+"""
+
+from __future__ import annotations
+
+import secrets
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+
+_RUN_ID_RANDOM_HEX_CHARS = 6
+
+
+@dataclass(frozen=True, slots=True)
+class StateLayout:
+    """Paths for a project's local run state, rooted at the project repo root."""
+
+    root: Path
+
+    @property
+    def state_dir(self) -> Path:
+        """The root of all machine-local run state: ``<root>/.chipgraph/state``."""
+        return self.root / ".chipgraph" / "state"
+
+    @property
+    def runs_dir(self) -> Path:
+        """Where each run gets its own subdirectory."""
+        return self.state_dir / "runs"
+
+    def run_dir(self, run_id: str) -> Path:
+        """The directory for a single run's journal, manifest and results."""
+        return self.runs_dir / run_id
+
+    def journal(self, run_id: str) -> Path:
+        """The append-only JSONL journal file for a run."""
+        return self.run_dir(run_id) / "journal.jsonl"
+
+    def manifest(self, run_id: str) -> Path:
+        """The manifest file for a run."""
+        return self.run_dir(run_id) / "manifest.json"
+
+    @property
+    def locks_dir(self) -> Path:
+        """Where per-block lock files live."""
+        return self.state_dir / "locks"
+
+    @property
+    def cache_dir(self) -> Path:
+        """Where cached, regenerable data lives (not the idempotency store)."""
+        return self.state_dir / "cache"
+
+    @property
+    def tmp_dir(self) -> Path:
+        """Scratch space for disposable work (e.g. fan-out workspaces): ``<state>/tmp``.
+
+        Under the local backend this is inside the repo's working tree, so callers that
+        must never write there (fan-out) use the system temp directory instead.
+        """
+        return self.state_dir / "tmp"
+
+    @property
+    def decisions_dir(self) -> Path:
+        """Human-auditable gate/waiver decisions: ``<root>/.chipgraph/decisions``.
+
+        This is *not* part of run state: it is meant to be readable, hash-pinned, and
+        optionally committed to the project repo (DESIGN.md 6.1).
+        """
+        return self.root / ".chipgraph" / "decisions"
+
+
+def new_run_id() -> str:
+    """A sortable, unique run id: UTC timestamp plus a short random suffix.
+
+    Format: ``YYYYMMDDTHHMMSSZ-<6 hex chars>``, e.g. ``20260928T120000Z-a1b2c3``.
+    """
+    ts = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
+    suffix = secrets.token_hex(_RUN_ID_RANDOM_HEX_CHARS // 2)
+    return f"{ts}-{suffix}"

chipgraph/core/state/lock.py

@@ -0,0 +1,195 @@
+"""Per-block locks: DESIGN.md 6.2, "một block chỉ có một run ghi tại một thời điểm".
+
+A lock is a small JSON file created atomically (`O_CREAT | O_EXCL`) under
+`StateLayout.locks_dir`. It records the owner, the process id and host that took it, and
+when. A lock whose owning process is no longer alive on the same host — or that has been
+held past `stale_after_s` — is considered abandoned and may be taken over.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import os
+import re
+import socket
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from types import TracebackType
+from typing import Self
+
+from chipgraph.core.state.layout import StateLayout
+
+_BLOCK_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$")
+_DEFAULT_STALE_AFTER_S = 12 * 60 * 60
+
+
+class LockHeldError(Exception):
+    """Another live owner already holds this block's lock."""
+
+    def __init__(self, block: str, owner: str) -> None:
+        super().__init__(f"block {block!r} is locked by {owner!r}")
+        self.block = block
+        self.owner = owner
+
+
+@dataclass(frozen=True, slots=True)
+class _LockRecord:
+    owner: str
+    pid: int
+    host: str
+    since: str
+
+
+def _pid_alive(pid: int) -> bool:
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        # Process exists but we can't signal it: treat as alive.
+        return True
+    except OSError:
+        return False
+    return True
+
+
+def _read_record(path: Path) -> _LockRecord:
+    data = json.loads(path.read_text(encoding="utf-8"))
+    return _LockRecord(owner=data["owner"], pid=data["pid"], host=data["host"], since=data["since"])
+
+
+def _write_record_atomic(path: Path, record: _LockRecord) -> None:
+    fd = os.open(str(path), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644)
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as f:
+            json.dump(
+                {
+                    "owner": record.owner,
+                    "pid": record.pid,
+                    "host": record.host,
+                    "since": record.since,
+                },
+                f,
+            )
+    except BaseException:
+        with contextlib.suppress(OSError):
+            path.unlink()
+        raise
+
+
+class BlockLock:
+    """A context manager that holds an exclusive lock on `block` for the duration of a `with`.
+
+    Usage::
+
+        with BlockLock(layout, block="timer", owner="run-123") as lock:
+            ...  # only one live owner may be inside this block at a time
+            lock.took_over_stale  # True if a dead/stale lock was reclaimed
+    """
+
+    def __init__(
+        self,
+        layout: StateLayout,
+        block: str,
+        owner: str,
+        *,
+        stale_after_s: float = _DEFAULT_STALE_AFTER_S,
+    ) -> None:
+        if not _BLOCK_NAME_RE.match(block):
+            raise ValueError(f"invalid block name: {block!r}")
+        self.layout = layout
+        self.block = block
+        self.owner = owner
+        self.stale_after_s = stale_after_s
+        self.took_over_stale = False
+        self._path = layout.locks_dir / f"{block}.lock"
+        self._acquired = False
+
+    @property
+    def path(self) -> Path:
+        return self._path
+
+    def acquire(self) -> None:
+        self.layout.locks_dir.mkdir(parents=True, exist_ok=True)
+        record = _LockRecord(
+            owner=self.owner,
+            pid=os.getpid(),
+            host=socket.gethostname(),
+            since=datetime.now(UTC).isoformat(),
+        )
+        try:
+            _write_record_atomic(self._path, record)
+        except FileExistsError:
+            self._resolve_conflict(record)
+            return
+        self._acquired = True
+        self.took_over_stale = False
+
+    def _resolve_conflict(self, new_record: _LockRecord) -> None:
+        try:
+            existing = _read_record(self._path)
+        except (OSError, json.JSONDecodeError, KeyError):
+            # Unreadable/corrupt lock file: treat as stale and take it over.
+            self._take_over(new_record)
+            return
+
+        same_host = existing.host == new_record.host
+        alive = same_host and _pid_alive(existing.pid)
+        if alive:
+            stale = self._is_stale(existing)
+            if not stale:
+                raise LockHeldError(self.block, existing.owner)
+        self._take_over(new_record)
+
+    def _is_stale(self, existing: _LockRecord) -> bool:
+        try:
+            since = datetime.fromisoformat(existing.since)
+        except ValueError:
+            return True
+        age_s = (datetime.now(UTC) - since).total_seconds()
+        return age_s > self.stale_after_s
+
+    def _take_over(self, new_record: _LockRecord) -> None:
+        tmp_path = self._path.with_suffix(".lock.tmp")
+        with open(tmp_path, "w", encoding="utf-8") as f:
+            json.dump(
+                {
+                    "owner": new_record.owner,
+                    "pid": new_record.pid,
+                    "host": new_record.host,
+                    "since": new_record.since,
+                },
+                f,
+            )
+        os.replace(tmp_path, self._path)
+        self._acquired = True
+        self.took_over_stale = True
+
+    def release(self) -> None:
+        if not self._acquired:
+            return
+        try:
+            current = _read_record(self._path)
+        except (OSError, json.JSONDecodeError, KeyError):
+            return
+        if current.owner == self.owner and current.pid == os.getpid():
+            with contextlib.suppress(FileNotFoundError):
+                self._path.unlink()
+        self._acquired = False
+
+    def __enter__(self) -> Self:
+        self.acquire()
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        self.release()
+
+
+__all__ = ["BlockLock", "LockHeldError"]