whiletrue 0.1.0__py3-none-any.whl

whiletrue/core/events.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+import time
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+#: Every started/output/cost/error/approval/takeover is an event; the log is append-only.
+EventType = Literal[
+    "run.started",
+    "run.finished",
+    "run.hashes",
+    "item.state",
+    "step.started",
+    "step.finished",
+    "step.error",
+    "step.cached",
+    "step.journaled",
+    "workspace.created",
+    "workspace.released",
+    "park.created",
+    "park.decided",
+    "comment.added",
+    "comment.answered",
+    "step.replayed",
+    "agent.event",
+    "log",
+]
+
+
+@dataclass
+class WtEvent:
+    type: EventType
+    run_id: str
+    #: Epoch milliseconds.
+    ts: int
+    data: dict[str, Any] = field(default_factory=lambda: {})
+    item_id: str | None = None
+    step_id: int | None = None
+
+
+EventListener = Callable[[WtEvent], None]
+
+
+def _wall_clock_ms() -> int:
+    return int(time.time() * 1000)
+
+
+class EventBus:
+    def __init__(
+        self,
+        *,
+        now: Callable[[], int] = _wall_clock_ms,
+        on_error: Callable[[Exception, WtEvent], None] | None = None,
+    ) -> None:
+        self._listeners: list[EventListener] = []
+        self._now = now
+        self._on_error = on_error
+
+    def subscribe(self, listener: EventListener) -> Callable[[], None]:
+        self._listeners.append(listener)
+
+        def unsubscribe() -> None:
+            if listener in self._listeners:
+                self._listeners.remove(listener)
+
+        return unsubscribe
+
+    def emit(
+        self,
+        type: EventType,
+        run_id: str,
+        data: dict[str, Any] | None = None,
+        *,
+        item_id: str | None = None,
+        step_id: int | None = None,
+        ts: int | None = None,
+    ) -> WtEvent:
+        """Delivers synchronously, in subscription order. A failing listener never breaks us."""
+        event = WtEvent(
+            type=type,
+            run_id=run_id,
+            ts=self._now() if ts is None else ts,
+            data=data if data is not None else {},
+            item_id=item_id,
+            step_id=step_id,
+        )
+        for listener in list(self._listeners):
+            try:
+                listener(event)
+            except Exception as error:
+                if self._on_error is not None:
+                    self._on_error(error, event)
+        return event

whiletrue/core/expr.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any, cast
+
+import cel
+from cel import stdlib as cel_stdlib
+
+#: Guards against pathological inputs; the native evaluator cannot be interrupted once started.
+MAX_EXPR_LENGTH = 4096
+
+#: CEL's standard, side-effect-free functions and operators. Anything else is rejected by `check`.
+ALLOWED_FUNCTIONS = frozenset(cel_stdlib.STDLIB_FUNCTIONS) | frozenset(
+    {
+        "size", "has", "int", "uint", "double", "string", "bool", "bytes", "type", "dyn",
+        "contains", "startsWith", "endsWith", "matches", "exists", "all", "exists_one", "map",
+        "filter", "min", "max", "duration", "timestamp", "getFullYear", "getMonth", "getDate",
+        "getDayOfMonth", "getDayOfWeek", "getHours", "getMinutes", "getSeconds",
+        "getMilliseconds", "optional", "in",
+    }
+)  # fmt: skip
+
+
+class ExprError(ValueError):
+    """Raised for syntax errors and evaluation failures."""
+
+
+def _is_operator(name: str) -> bool:
+    return name.startswith("_") or name.startswith("@")
+
+
+@dataclass(frozen=True)
+class Expr:
+    """A parsed CEL expression. `check` is static, `eval` runs against a context mapping."""
+
+    source: str
+    _program: Any
+
+    def _refs(self) -> tuple[list[str], list[str]]:
+        refs = cast(dict[str, list[str]], self._program.references())
+        return refs.get("variables", []), refs.get("functions", [])
+
+    def check(self, known_variables: set[str] | frozenset[str]) -> list[str]:
+        """Returns problems found without evaluating: unknown variables, disallowed functions."""
+        variables, functions = self._refs()
+        problems: list[str] = []
+        known = ", ".join(sorted(known_variables)) or "none"
+        for name in dict.fromkeys(variables):
+            if name not in known_variables:
+                problems.append(f'Unknown variable "{name}" (known: {known})')
+        for name in dict.fromkeys(functions):
+            if not _is_operator(name) and name not in ALLOWED_FUNCTIONS:
+                problems.append(f'Function "{name}" is not allowed')
+        return problems
+
+    def eval(self, context: Mapping[str, Any]) -> Any:
+        try:
+            ctx = cel.Context(dict(context))
+            cel_stdlib.add_stdlib_to_context(ctx)  # max, split, lowerAscii, ... (all pure)
+            return self._program.execute(ctx)
+        except Exception as error:
+            raise ExprError(f"Cannot evaluate `{self.source}`: {error}") from error
+
+    def eval_bool(self, context: Mapping[str, Any]) -> bool:
+        value = self.eval(context)
+        if not isinstance(value, bool):
+            raise ExprError(f"`{self.source}` must be a boolean, got {type(value).__name__}")
+        return value
+
+
+def parse_expr(source: str) -> Expr:
+    if len(source) > MAX_EXPR_LENGTH:
+        raise ExprError(f"Expression is too long ({len(source)} > {MAX_EXPR_LENGTH} characters)")
+    try:
+        program = cel.compile(source)
+    except Exception as error:
+        text = str(error)
+        if "reserve" in text.lower():
+            raise ExprError(
+                f"`{source}` uses a CEL reserved word (such as `loop`, `if`, `for`) as a name; "
+                "use `iter` for the loop scope"
+            ) from error
+        raise ExprError(f"Invalid expression (syntax) `{source}`: {text}") from error
+    return Expr(source, program)

whiletrue/core/history.py

@@ -0,0 +1,83 @@
+"""Records kept across runs: human input (parks, comments, steers), side effects and learnings.
+
+Unlike runs and steps, these are keyed by *item*, not run: a gate parked in one process is decided
+from another, and a comment made today is answered by tomorrow's run.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+Decision = Literal["approved", "rejected"]
+LearningStatus = Literal["proposed", "approved", "rejected"]
+
+
+@dataclass
+class ParkRecord:
+    id: int
+    #: The run lineage (the first run of a chain of resumes) that created it.
+    run_id: str
+    item_id: str
+    #: The gate step that parked the item.
+    node_path: str
+    #: What is being decided (`pr`, `merge`, ...), shown in status.
+    what: str
+    #: The wake condition this park waits on.
+    wake: dict[str, Any]
+    reason: str
+    created_at: int
+    decision: Decision | None = None
+    notes: str | None = None
+    decided_by: str | None = None
+    decided_at: int | None = None
+
+
+@dataclass
+class CommentRecord:
+    id: int
+    item_id: str
+    #: Where it came from: `github_pr`, `local`, `ai_review`, ...
+    source: str
+    #: The id in that system, unique per (item, source).
+    external_id: str
+    author: str
+    body: str
+    created_at: int
+    thread: str | None = None
+    url: str | None = None
+    #: `human` comments get answered but never resolved by the agent; `reviewer` is automated.
+    kind: Literal["human", "reviewer"] = "human"
+    answered: bool = False
+    reply: str | None = None
+
+
+@dataclass
+class SteerRecord:
+    id: int
+    run_id: str
+    item_id: str
+    step: str
+    prompt: str
+    created_at: int
+
+
+@dataclass
+class EffectRecord:
+    #: A stable key naming the effect (`github.open_pr:owner/repo:wt/12`), not the run.
+    key: str
+    status: Literal["started", "done"]
+    run_id: str
+    created_at: int
+    ref: Any = None
+
+
+@dataclass
+class LearningRecord:
+    id: int
+    scope: str
+    note: str
+    status: LearningStatus
+    created_at: int
+    run_id: str | None = None
+    provenance: dict[str, Any] = field(default_factory=lambda: {})

whiletrue/core/lock.py

@@ -0,0 +1,86 @@
+"""`whiletrue.lock` (plugin version pins) and content hashes of the files a run depends on."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from pathlib import Path
+from typing import Any
+
+from whiletrue.core.config import PipelineConfig, PipelineNode
+
+LOCK_FILE = "whiletrue.lock"
+LOCK_VERSION = 1
+
+
+class LockError(Exception):
+    pass
+
+
+def write_lock(path: Path, plugins: dict[str, str]) -> None:
+    body = {"version": LOCK_VERSION, "plugins": dict(sorted(plugins.items()))}
+    path.write_text(json.dumps(body, indent=2) + "\n", encoding="utf-8")
+
+
+def read_lock(path: Path) -> dict[str, str] | None:
+    """The pinned plugin versions, or None when there is no lock file."""
+    if not path.is_file():
+        return None
+    try:
+        body: Any = json.loads(path.read_text(encoding="utf-8"))
+        plugins = body["plugins"]
+        if body["version"] != LOCK_VERSION or not all(
+            isinstance(k, str) and isinstance(v, str) for k, v in plugins.items()
+        ):
+            raise ValueError("unsupported format")
+    except (ValueError, KeyError, TypeError, AttributeError) as error:
+        raise LockError(f"Cannot read {path.name}: {error}") from error
+    return dict(plugins)
+
+
+def check_lock(pinned: dict[str, str], actual: dict[str, str]) -> list[str]:
+    """Problems between the lock and the plugins in use; empty means they agree."""
+    problems: list[str] = []
+    for name, version in sorted(actual.items()):
+        if name not in pinned:
+            problems.append(f'Plugin "{name}" ({version}) is not in {LOCK_FILE}')
+        elif pinned[name] != version:
+            problems.append(f'Plugin "{name}" is {version} but {LOCK_FILE} pins {pinned[name]}')
+    problems += [
+        f'{LOCK_FILE} pins "{name}" which is no longer used'
+        for name in sorted(set(pinned) - set(actual))
+    ]
+    return problems
+
+
+def sha256_file(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def _uses_paths(node: PipelineNode) -> list[str]:
+    found: list[str] = []
+    uses = node.config.get("uses")
+    if isinstance(uses, str) and not uses.startswith("recipe:"):
+        found.append(uses)
+    for child in node.children:
+        found += _uses_paths(child)
+    return found
+
+
+def collect_file_hashes(config: PipelineConfig, cwd: str) -> dict[str, str]:
+    """Hashes of the pipeline file, expanded composites and any file a step `uses:` (prompts)."""
+    base = Path(cwd)
+    candidates = [Path(config.file), *map(Path, config.sources)]
+    candidates += [base / p for p in _uses_paths(config.root)]
+    hashes: dict[str, str] = {}
+    for candidate in candidates:
+        path = candidate if candidate.is_absolute() else base / candidate
+        if not path.is_file():
+            continue
+        resolved = path.resolve()
+        try:
+            key = str(resolved.relative_to(base.resolve()))
+        except ValueError:
+            key = str(resolved)
+        hashes[key] = sha256_file(resolved)
+    return dict(sorted(hashes.items()))

whiletrue/core/memory_store.py

@@ -0,0 +1,352 @@
+from __future__ import annotations
+
+import copy
+from dataclasses import replace
+from typing import Any, Literal
+
+from whiletrue.core.events import WtEvent
+from whiletrue.core.history import (
+    CommentRecord,
+    Decision,
+    EffectRecord,
+    LearningRecord,
+    LearningStatus,
+    ParkRecord,
+    SteerRecord,
+)
+from whiletrue.core.outcomes import ItemState
+from whiletrue.core.store import ItemRecord, RunRecord, StepRecord
+
+
+class MemoryStore:
+    """In-memory StateStore for tests and throwaway runs."""
+
+    def __init__(self) -> None:
+        self._runs: list[RunRecord] = []
+        self._items: list[ItemRecord] = []
+        self._steps: list[StepRecord] = []
+        self._events: list[WtEvent] = []
+        self._cache: dict[str, Any] = {}
+        self._parks: list[ParkRecord] = []
+        self._comments: list[CommentRecord] = []
+        self._steers: list[SteerRecord] = []
+        self._effects: dict[str, EffectRecord] = {}
+        self._learnings: list[LearningRecord] = []
+
+    def create_run(
+        self, *, id: str, pipeline_hash: str, started_at: int, lockfile: str | None = None
+    ) -> None:
+        self._runs.append(
+            RunRecord(
+                id=id,
+                pipeline_hash=pipeline_hash,
+                lockfile=lockfile,
+                status="running",
+                started_at=started_at,
+                ended_at=None,
+            )
+        )
+
+    def finish_run(
+        self, id: str, status: Literal["done", "failed", "parked", "cancelled"], ended_at: int
+    ) -> None:
+        run = self.get_run(id)
+        if run is None:
+            raise KeyError(f'No run "{id}"')
+        run.status = status
+        run.ended_at = ended_at
+
+    def interrupt_run(self, id: str, ended_at: int) -> None:
+        for step in self._steps:
+            if step.run_id == id and step.status == "running":
+                step.status, step.error, step.ended_at = "errored", "interrupted", ended_at
+        for item in self._items:
+            if item.run_id == id and item.state == "running":
+                item.state = "parked"
+        self.finish_run(id, "parked", ended_at)
+
+    def get_run(self, id: str) -> RunRecord | None:
+        return next((r for r in self._runs if r.id == id), None)
+
+    def list_runs(self) -> list[RunRecord]:
+        return sorted((replace(r) for r in self._runs), key=lambda r: r.started_at)
+
+    def latest_run(self) -> RunRecord | None:
+        return self._runs[-1] if self._runs else None
+
+    def upsert_item(self, item: ItemRecord) -> None:
+        stored = replace(item, depends_on=list(item.depends_on))
+        for i, existing in enumerate(self._items):
+            if existing.run_id == item.run_id and existing.id == item.id:
+                self._items[i] = stored
+                return
+        self._items.append(stored)
+
+    def set_item_state(self, run_id: str, item_id: str, state: ItemState) -> None:
+        for item in self._items:
+            if item.run_id == run_id and item.id == item_id:
+                item.state = state
+                return
+        raise KeyError(f'No item "{item_id}" in run "{run_id}"')
+
+    def get_item(self, run_id: str, item_id: str) -> ItemRecord | None:
+        found = next((i for i in self._items if i.run_id == run_id and i.id == item_id), None)
+        return replace(found) if found else None
+
+    def list_items(self, run_id: str) -> list[ItemRecord]:
+        return [replace(i) for i in self._items if i.run_id == run_id]
+
+    def start_step(
+        self,
+        *,
+        run_id: str,
+        item_id: str,
+        node_path: str,
+        attempt: int,
+        input: Any,
+        started_at: int,
+    ) -> int:
+        step_id = len(self._steps) + 1
+        self._steps.append(
+            StepRecord(
+                id=step_id,
+                run_id=run_id,
+                item_id=item_id,
+                node_path=node_path,
+                attempt=attempt,
+                status="running",
+                port=None,
+                input=input,
+                output=None,
+                cost_usd=0,
+                error=None,
+                started_at=started_at,
+                ended_at=None,
+            )
+        )
+        return step_id
+
+    def finish_step(
+        self,
+        step_id: int,
+        *,
+        status: Literal["completed", "errored"],
+        ended_at: int,
+        port: str | None = None,
+        output: Any = None,
+        cost_usd: float = 0,
+        error: str | None = None,
+    ) -> None:
+        step = next((s for s in self._steps if s.id == step_id), None)
+        if step is None:
+            raise KeyError(f"No step {step_id}")
+        step.status = status
+        step.port = port
+        step.output = output
+        step.cost_usd = cost_usd
+        step.error = error
+        step.ended_at = ended_at
+
+    def list_steps(self, run_id: str, item_id: str | None = None) -> list[StepRecord]:
+        return [
+            replace(s)
+            for s in self._steps
+            if s.run_id == run_id and (item_id is None or s.item_id == item_id)
+        ]
+
+    def append_event(self, event: WtEvent) -> None:
+        self._events.append(copy.copy(event))
+
+    def list_events(
+        self, run_id: str, *, item_id: str | None = None, type: str | None = None
+    ) -> list[WtEvent]:
+        return [
+            e
+            for e in self._events
+            if e.run_id == run_id
+            and (item_id is None or e.item_id == item_id)
+            and (type is None or e.type == type)
+        ]
+
+    def cache_get(self, key: str) -> Any | None:
+        return self._cache.get(key)
+
+    def cache_put(self, key: str, value: Any, created_at: int) -> None:
+        self._cache[key] = value
+
+    def close(self) -> None:
+        return None
+
+    # ---- human input and side effects ----
+
+    def create_park(
+        self,
+        *,
+        run_id: str,
+        item_id: str,
+        node_path: str,
+        what: str,
+        wake: dict[str, Any],
+        reason: str,
+        created_at: int,
+    ) -> ParkRecord:
+        park = ParkRecord(
+            id=len(self._parks) + 1,
+            run_id=run_id,
+            item_id=item_id,
+            node_path=node_path,
+            what=what,
+            wake=copy.deepcopy(wake),
+            reason=reason,
+            created_at=created_at,
+        )
+        self._parks.append(park)
+        return replace(park)
+
+    def _park(self, park_id: int) -> ParkRecord:
+        park = next((p for p in self._parks if p.id == park_id), None)
+        if park is None:
+            raise KeyError(f"No park {park_id}")
+        return park
+
+    def decide_park(
+        self,
+        park_id: int,
+        decision: Decision,
+        *,
+        decided_by: str,
+        decided_at: int,
+        notes: str | None,
+    ) -> None:
+        park = self._park(park_id)
+        park.decision, park.decided_by, park.decided_at, park.notes = (
+            decision,
+            decided_by,
+            decided_at,
+            notes,
+        )
+
+    def list_parks(
+        self,
+        item_id: str | None = None,
+        *,
+        node_path: str | None = None,
+        run_id: str | None = None,
+        undecided_only: bool = False,
+    ) -> list[ParkRecord]:
+        return [
+            replace(p)
+            for p in self._parks
+            if (item_id is None or p.item_id == item_id)
+            and (node_path is None or p.node_path == node_path)
+            and (run_id is None or p.run_id == run_id)
+            and (not undecided_only or p.decision is None)
+        ]
+
+    def add_comment(
+        self,
+        *,
+        item_id: str,
+        source: str,
+        external_id: str,
+        author: str,
+        body: str,
+        created_at: int,
+        thread: str | None = None,
+        url: str | None = None,
+        kind: Literal["human", "reviewer"] = "human",
+    ) -> CommentRecord | None:
+        if any(
+            c.item_id == item_id and c.source == source and c.external_id == external_id
+            for c in self._comments
+        ):
+            return None
+        comment = CommentRecord(
+            id=len(self._comments) + 1,
+            item_id=item_id,
+            source=source,
+            external_id=external_id,
+            author=author,
+            body=body,
+            created_at=created_at,
+            thread=thread,
+            url=url,
+            kind=kind,
+        )
+        self._comments.append(comment)
+        return replace(comment)
+
+    def list_comments(self, item_id: str, *, unanswered_only: bool = False) -> list[CommentRecord]:
+        return [
+            replace(c)
+            for c in self._comments
+            if c.item_id == item_id and not (unanswered_only and c.answered)
+        ]
+
+    def answer_comment(self, comment_id: int, reply: str) -> None:
+        comment = next((c for c in self._comments if c.id == comment_id), None)
+        if comment is None:
+            raise KeyError(f"No comment {comment_id}")
+        comment.answered, comment.reply = True, reply
+
+    def add_steer(
+        self, *, run_id: str, item_id: str, step: str, prompt: str, created_at: int
+    ) -> SteerRecord:
+        steer = SteerRecord(len(self._steers) + 1, run_id, item_id, step, prompt, created_at)
+        self._steers.append(steer)
+        return replace(steer)
+
+    def list_steers(self, item_id: str) -> list[SteerRecord]:
+        return [replace(s) for s in self._steers if s.item_id == item_id]
+
+    def effect_get(self, key: str) -> EffectRecord | None:
+        found = self._effects.get(key)
+        return None if found is None else replace(found, ref=copy.deepcopy(found.ref))
+
+    def effect_put(
+        self,
+        key: str,
+        *,
+        run_id: str,
+        status: Literal["started", "done"],
+        ref: Any,
+        created_at: int,
+    ) -> None:
+        self._effects[key] = EffectRecord(key, status, run_id, created_at, copy.deepcopy(ref))
+
+    def add_learning(
+        self,
+        *,
+        scope: str,
+        note: str,
+        status: LearningStatus,
+        created_at: int,
+        run_id: str | None = None,
+        provenance: dict[str, Any] | None = None,
+    ) -> LearningRecord:
+        learning = LearningRecord(
+            len(self._learnings) + 1,
+            scope,
+            note,
+            status,
+            created_at,
+            run_id,
+            copy.deepcopy(provenance or {}),
+        )
+        self._learnings.append(learning)
+        return replace(learning)
+
+    def list_learnings(
+        self, *, scope: str | None = None, status: LearningStatus | None = None
+    ) -> list[LearningRecord]:
+        return [
+            replace(item)
+            for item in self._learnings
+            if (scope is None or item.scope == scope) and (status is None or item.status == status)
+        ]
+
+    def set_learning_status(self, learning_id: int, status: LearningStatus) -> None:
+        item = next((x for x in self._learnings if x.id == learning_id), None)
+        if item is None:
+            raise KeyError(f"No learning {learning_id}")
+        item.status = status