whiletrue 0.1.0__py3-none-any.whl

whiletrue/basics/exec_.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel, Field, ValidationError
+
+from whiletrue.basics.process import run_command, tail
+from whiletrue.core import (
+    ActionModule,
+    Emit,
+    Ports,
+    PortSpec,
+    StepContext,
+    define_action,
+    parse_duration,
+)
+from whiletrue.core.module import PortValues
+
+
+class ExecConfig(BaseModel):
+    command: str = Field(min_length=1)
+    args: list[str] = Field(default_factory=lambda: [])
+    cwd: str | None = None
+    env: dict[str, str] | None = None
+    timeout: str | int | float | None = None
+
+
+class _Reply(BaseModel):
+    port: str | None = None
+    value: Any = None
+
+
+async def _run(input: PortValues, ctx: StepContext[ExecConfig]) -> Emit:
+    cfg = ctx.config
+    result = await run_command(
+        cfg.command,
+        cfg.args,
+        cwd=str((Path(ctx.cwd) / (cfg.cwd or ".")).resolve()),
+        env=cfg.env,
+        signal=ctx.signal,
+        timeout_ms=None if cfg.timeout is None else parse_duration(cfg.timeout),
+        stdin=json.dumps(
+            {
+                "input": input.get("in"),
+                "config": ctx.config.model_dump(),
+                "item": ctx.scope.get("item"),
+                "step": {"id": ctx.node_id, "key": ctx.step_key, "attempt": ctx.attempt},
+            }
+        ),
+    )
+    if result.code != 0:
+        code = "unknown" if result.code is None else result.code
+        return Emit(
+            "fail",
+            {
+                "error": f"`{cfg.command}` exited with code {code}",
+                "output": tail(result.stdout + result.stderr),
+            },
+        )
+    try:
+        parsed = json.loads(result.stdout)
+    except ValueError:
+        return Emit("fail", {"error": f"`{cfg.command}` did not print valid JSON on stdout"})
+    try:
+        reply = _Reply.model_validate(parsed)
+    except ValidationError:
+        return Emit("fail", {"error": f"`{cfg.command}` reply must be {{ port?, value? }}"})
+    return Emit(reply.port or "ok", reply.value)
+
+
+#: Escape hatch: runs any executable that speaks JSON over stdio, so a module can be written in
+#: any language. Receives `{ input, config, item, step }` on stdin; replies `{ port?, value? }`.
+exec_module = define_action(
+    ActionModule(
+        id="exec",
+        config=ExecConfig,
+        ports=Ports(inputs=[PortSpec("in", "Any")], outputs=[PortSpec("out", "Any")]),
+        effects="side-effect",
+        idempotency_key=lambda _input, ctx: f"exec:{ctx.step_key}",
+        run=_run,
+    )
+)

whiletrue/basics/history.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel
+
+from whiletrue.core import (
+    ActionModule,
+    Emit,
+    Ports,
+    PortSpec,
+    StateStore,
+    StepContext,
+    define_action,
+)
+from whiletrue.core.module import PortValues
+
+
+class HistoryConfig(BaseModel):
+    #: `item`: this item's parks, decisions, comments and steers. `project`: learnings a person
+    #: approved. `all`: both.
+    scope: Literal["item", "project", "all"] = "item"
+
+
+def render_history(store: StateStore, item_id: str, scope: str) -> str:
+    """The human input an agent should know about, as plain text."""
+    sections: list[str] = []
+    if scope in ("item", "all"):
+        lines: list[str] = []
+        for park in store.list_parks(item_id):
+            if park.decision is not None:
+                note = f": {park.notes}" if park.notes else ""
+                lines.append(f"- At {park.node_path} ({park.what}) a person {park.decision}{note}")
+        for comment in store.list_comments(item_id):
+            lines.append(f"- {comment.author} commented: {comment.body}")
+            if comment.reply:
+                lines.append(f"  answered: {comment.reply}")
+        for steer in store.list_steers(item_id):
+            lines.append(f"- A person steered {steer.step}: {steer.prompt}")
+        if lines:
+            sections.append("Human input on this item so far:\n" + "\n".join(lines))
+    if scope in ("project", "all"):
+        learned = store.list_learnings(status="approved")
+        if learned:
+            sections.append(
+                "Lessons this project has approved:\n" + "\n".join(f"- {n.note}" for n in learned)
+            )
+    return "\n\n".join(sections)
+
+
+async def _context(_input: PortValues, ctx: StepContext[HistoryConfig]) -> Emit:
+    if ctx.store is None:
+        return Emit("fail", {"error": "context.history needs a state store"})
+    return Emit("ok", render_history(ctx.store, ctx.item_id, ctx.config.scope))
+
+
+#: Turns the item's human input (and approved project lessons) into text for the next agent.
+context_history_module: ActionModule[HistoryConfig] = define_action(
+    ActionModule(
+        id="context.history",
+        config=HistoryConfig,
+        ports=Ports(inputs=[PortSpec("in", "Any")], outputs=[PortSpec("out", "Text")]),
+        effects="deterministic",
+        cacheable=False,
+        run=_context,
+    )
+)
+
+
+class DistillConfig(BaseModel):
+    #: Where proposals land until a person approves them.
+    scope: Literal["project", "run"] = "project"
+
+
+async def _distill(input: PortValues, ctx: StepContext[DistillConfig]) -> Emit:
+    """Proposes lessons from rejections and steers. Nothing is used until a person approves it."""
+    store = ctx.store
+    if store is None:
+        return Emit("fail", {"error": "history.distill needs a state store"})
+    known = {n.note for n in store.list_learnings(scope=ctx.config.scope)}
+    proposed: list[int] = []
+
+    def propose(note: str, **provenance: object) -> None:
+        if note in known:
+            return
+        known.add(note)
+        record = store.add_learning(
+            scope=ctx.config.scope,
+            note=note,
+            status="proposed",
+            created_at=ctx.now(),
+            run_id=ctx.run_id,
+            provenance={"item": ctx.item_id, **provenance},
+        )
+        proposed.append(record.id)
+
+    for park in store.list_parks(ctx.item_id):
+        if park.decision == "rejected" and park.notes:
+            propose(
+                f"A reviewer rejected work with: {park.notes}", source="rejection", park=park.id
+            )
+    for steer in store.list_steers(ctx.item_id):
+        propose(f"A person steered the agent: {steer.prompt}", source="steer", steer=steer.id)
+    return Emit("ok", input.get("in") if input.get("in") is not None else {"proposed": proposed})
+
+
+#: Proposes learnings (with provenance) from rejections and steers; a person approves them.
+history_distill_module: ActionModule[DistillConfig] = define_action(
+    ActionModule(
+        id="history.distill",
+        config=DistillConfig,
+        ports=Ports(inputs=[PortSpec("in", "Any")], outputs=[PortSpec("out", "Any")]),
+        effects="deterministic",
+        cacheable=False,
+        run=_distill,
+    )
+)

whiletrue/basics/human.py

@@ -0,0 +1,201 @@
+from __future__ import annotations
+
+import asyncio
+from typing import Any, cast
+
+from pydantic import BaseModel, Field
+
+from whiletrue.core import (
+    CommentsContext,
+    CompositeContext,
+    ConfigIssue,
+    Emit,
+    ParkRecord,
+    PipelineNode,
+    Strategy,
+    ValidateContext,
+    WakeContext,
+    WakeResult,
+    WakeSpec,
+    define_strategy,
+    evaluate_wake,
+    parse_duration,
+    parse_wake,
+    sync_comments,
+    wake_to_dict,
+)
+from whiletrue.core.module import PortValues
+
+
+class ApproveConfig(BaseModel):
+    #: What is being decided; shown in `whiletrue status` and the control panel.
+    what: str = "approval"
+    #: Shown to the person deciding (templates see `input`).
+    show: str | None = None
+    #: What ends the wait besides `whiletrue approve|reject`: see `parse_wake`.
+    wake: Any = Field(default="cli")
+    #: How often a watching gate re-checks (seconds, or a duration like `5s`).
+    poll: str | float = "2s"
+
+
+def _spec(config: ApproveConfig) -> WakeSpec:
+    return parse_wake(config.wake)
+
+
+def _validate(config: ApproveConfig, vctx: ValidateContext) -> list[ConfigIssue]:
+    try:
+        spec = _spec(config)
+    except ValueError as error:
+        return [ConfigIssue("error", str(error))]
+    issues = [
+        ConfigIssue("error", f'Unknown wake condition "{c.name}"')
+        for c in spec.conditions
+        if vctx.registry.get_provider("wake", c.name) is None
+    ]
+    try:
+        parse_duration(config.poll)
+    except ValueError as error:
+        issues.append(ConfigIssue("error", f"`poll`: {error}"))
+    return issues
+
+
+async def _sleep(seconds: float, ctx: CompositeContext[ApproveConfig]) -> None:
+    waiter = asyncio.ensure_future(ctx.signal.wait())
+    try:
+        await asyncio.wait({waiter}, timeout=seconds)
+    finally:
+        waiter.cancel()
+
+
+def _park_for(ctx: CompositeContext[ApproveConfig], spec: WakeSpec) -> ParkRecord:
+    """The Nth run of this gate within a lineage is the Nth park: replays find their old park."""
+    assert ctx.store is not None
+    parks = ctx.store.list_parks(ctx.item_id, node_path=ctx.node_id, run_id=ctx.lineage)
+    if ctx.attempt <= len(parks):
+        return parks[ctx.attempt - 1]
+    park = ctx.store.create_park(
+        run_id=ctx.lineage,
+        item_id=ctx.item_id,
+        node_path=ctx.node_id,
+        what=ctx.config.what,
+        wake=wake_to_dict(spec),
+        reason=f"waiting for {ctx.config.what}",
+        created_at=ctx.now(),
+    )
+    ctx.bus.emit(
+        "park.created",
+        ctx.run_id,
+        {"park": park.id, "node": ctx.node_id, "what": park.what},
+        item_id=ctx.item_id,
+    )
+    return park
+
+
+def _item(ctx: CompositeContext[ApproveConfig]) -> dict[str, Any]:
+    item: dict[str, Any] | None = ctx.scope.get("item")
+    return item or {"id": ctx.item_id}
+
+
+async def _run(
+    children: list[PipelineNode], input: PortValues, ctx: CompositeContext[ApproveConfig]
+) -> Emit:
+    if ctx.store is None:
+        return Emit("fail", {"error": "human.approve needs a state store"})
+    store = ctx.store
+    spec = _spec(ctx.config)
+    park = _park_for(ctx, spec)
+    value = input.get("in")
+    item = _item(ctx)
+    shown = ctx.config.show  # templates in config were rendered before the gate ran
+    poll = parse_duration(ctx.config.poll) / 1000
+    answered_attempts: set[int] = set()
+
+    while True:
+        park = next(p for p in store.list_parks(ctx.item_id) if p.id == park.id)
+        if park.decision is None:
+            woke = await evaluate_wake(
+                spec,
+                WakeContext(ctx.cwd, ctx.now, ctx.item_id, park, ctx.registry, item, value),
+            )
+            if woke is not None:
+                _record(ctx, park, woke)
+                continue
+        else:
+            return _outcome(park, value)
+
+        pending = await _unanswered(ctx, item, value)
+        fresh = [c for c in pending if c.id not in answered_attempts]
+        if fresh and children:
+            answered_attempts.update(c.id for c in fresh)
+            # New comments always get a live response, never a recorded one from an earlier cycle.
+            ctx.scope.set("no_replay", True)
+            cur = Emit("ok", {"comments": [_comment_dict(c) for c in fresh], "input": value})
+            for child in children:
+                cur = await ctx.run(child, {"in": cur.value})
+                if cur.port != "ok":
+                    break
+            ctx.log("info", f"answered {len(fresh)} comment(s) on {ctx.item_id}", None)
+            continue
+
+        if not ctx.watch:
+            return Emit(
+                "needs_human",
+                {"park": park.id, "what": park.what, "show": shown, "waiting_for": "approval"},
+            )
+        if ctx.signal.aborted:
+            return Emit("fail", {"error": "cancelled"})
+        await _sleep(poll, ctx)
+
+
+def _comment_dict(comment: Any) -> dict[str, Any]:
+    return {
+        "id": comment.id,
+        "author": comment.author,
+        "body": comment.body,
+        "thread": comment.thread,
+        "url": comment.url,
+        "source": comment.source,
+        "kind": comment.kind,
+    }
+
+
+async def _unanswered(ctx: CompositeContext[ApproveConfig], item: dict[str, Any], value: Any):
+    assert ctx.store is not None
+    await sync_comments(
+        ctx.registry, ctx.store, CommentsContext(ctx.cwd, ctx.now, ctx.item_id, item, value)
+    )
+    return ctx.store.list_comments(ctx.item_id, unanswered_only=True)
+
+
+def _record(ctx: CompositeContext[ApproveConfig], park: ParkRecord, woke: WakeResult) -> None:
+    assert ctx.store is not None
+    ctx.store.decide_park(
+        park.id, woke.decision, decided_by=woke.by, decided_at=ctx.now(), notes=woke.note
+    )
+    ctx.bus.emit(
+        "park.decided",
+        ctx.run_id,
+        {"park": park.id, "decision": woke.decision, "by": woke.by, "note": woke.note},
+        item_id=ctx.item_id,
+    )
+
+
+def _outcome(park: ParkRecord, value: Any) -> Emit:
+    """Approval passes the input on. A rejection passes it on too, with the reviewer's note under
+    `rejection`, so whatever flows through the gate (the PR, say) survives a fix-and-review loop."""
+    carried: dict[str, Any] = (
+        {k: v for k, v in cast(dict[str, Any], value).items() if k != "rejection"}
+        if isinstance(value, dict)
+        else {"value": value}
+    )
+    if park.decision == "rejected":
+        carried["rejection"] = {"note": park.notes, "decided_by": park.decided_by}
+        return Emit("rejected", carried)
+    return Emit("ok", carried if isinstance(value, dict) else value)
+
+
+#: Parks the item until a person approves or rejects (or a wake condition fires).
+#: `do:` children run whenever new comments arrive, then the gate waits again.
+approve_strategy: Strategy[ApproveConfig] = define_strategy(
+    Strategy(id="human.approve", config=ApproveConfig, run=_run, validate=_validate)
+)

whiletrue/basics/json_source.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, cast
+
+from whiletrue.core import RunItem
+from whiletrue.core.source import SourceError
+
+
+class JsonSource:
+    """Items from a JSON file: a list of objects with an `id`, other fields, and `depends_on`."""
+
+    async def fetch(self, config: dict[str, Any], cwd: str) -> list[RunItem]:
+        path = config.get("path")
+        if not isinstance(path, str) or not path:
+            raise SourceError("`source.json` needs a `path`")
+        try:
+            data = json.loads((Path(cwd) / path).read_text(encoding="utf-8"))
+        except (OSError, ValueError) as error:
+            raise SourceError(f"Cannot read {path}: {error}") from error
+        if not isinstance(data, list):
+            raise SourceError(f"{path} must contain a JSON list of items")
+        items: list[RunItem] = []
+        for index, raw in enumerate(cast(list[Any], data)):
+            if not isinstance(raw, dict) or "id" not in raw:
+                raise SourceError(f"{path}: item {index} needs an `id`")
+            fields = {str(k): v for k, v in cast(dict[Any, Any], raw).items()}
+            depends = fields.pop("depends_on", [])
+            items.append(
+                RunItem(
+                    str(fields.pop("id")), fields, [str(d) for d in cast(list[Any], depends or [])]
+                )
+            )
+        return items

whiletrue/basics/markdown_source.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Any
+
+from whiletrue.core import RunItem
+from whiletrue.core.source import SourceError
+
+_TASK = re.compile(r"^\s*[-*]\s+\[(?P<mark>[ xX])\]\s+(?P<text>.+?)\s*$")
+_META = re.compile(r"\s*\{(?P<meta>[^{}]*)\}\s*$")
+
+
+def _slug(text: str) -> str:
+    return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")[:40] or "task"
+
+
+class MarkdownTasksSource:
+    """Unchecked checklist lines of a markdown file are the items.
+
+    `- [ ] Add the login page {#login after=schema,api}`: the optional `{...}` gives the item an
+    id (default: a slug of the title) and the ids it waits for. Checked lines are done and skipped,
+    so ticking a box in the file removes the item on the next resume (the source wins).
+    """
+
+    async def fetch(self, config: dict[str, Any], cwd: str) -> list[RunItem]:
+        path = config.get("path")
+        if not isinstance(path, str) or not path:
+            raise SourceError("`source.markdown_tasks` needs a `path`")
+        try:
+            text = (Path(cwd) / path).read_text(encoding="utf-8")
+        except OSError as error:
+            raise SourceError(f"Cannot read {path}: {error}") from error
+        items: list[RunItem] = []
+        seen: set[str] = set()
+        for number, line in enumerate(text.splitlines(), start=1):
+            match = _TASK.match(line)
+            if match is None or match["mark"] != " ":
+                continue
+            title = match["text"]
+            item_id, depends = "", []
+            meta = _META.search(title)
+            if meta is not None:
+                title = title[: meta.start()].strip()
+                for token in meta["meta"].split():
+                    if token.startswith("#"):
+                        item_id = token[1:]
+                    elif token.startswith("after="):
+                        depends = [d for d in token[6:].split(",") if d]
+            item_id = item_id or _slug(title)
+            if item_id in seen:
+                raise SourceError(f'{path}:{number}: duplicate task id "{item_id}"')
+            seen.add(item_id)
+            items.append(RunItem(item_id, {"title": title, "line": number}, depends))
+        return items

whiletrue/basics/process.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import os
+import signal as signal_module
+from dataclasses import dataclass
+from typing import Any
+
+from whiletrue.abort import AbortSignal
+
+
+@dataclass
+class CommandResult:
+    #: Exit code; None if the process never started or was killed by a signal.
+    code: int | None
+    stdout: str
+    stderr: str
+    timed_out: bool = False
+    aborted: bool = False
+
+
+def kill_process_group(pid: int) -> None:
+    """SIGKILLs the whole group so grandchildren cannot keep our pipes open."""
+    with contextlib.suppress(ProcessLookupError, PermissionError):
+        os.killpg(pid, signal_module.SIGKILL)
+
+
+async def run_command(
+    command: str,
+    args: list[str],
+    *,
+    cwd: str,
+    env: dict[str, str] | None = None,
+    stdin: str | None = None,
+    signal: AbortSignal | None = None,
+    timeout_ms: float | None = None,
+) -> CommandResult:
+    """Runs a command to completion.
+
+    Never raises for non-zero exits; spawn errors are reported as `code=None` with the OS message
+    on stderr.
+    """
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            command,
+            *args,
+            cwd=cwd,
+            env={**os.environ, **(env or {})},
+            stdin=asyncio.subprocess.PIPE,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+            start_new_session=True,
+        )
+    except OSError as error:
+        return CommandResult(
+            code=None,
+            stdout="",
+            stderr=str(error),
+            aborted=signal.aborted if signal else False,
+        )
+
+    communicate = asyncio.ensure_future(proc.communicate((stdin or "").encode()))
+    abort_wait = asyncio.ensure_future(signal.wait()) if signal else None
+    waiting: set[asyncio.Future[Any]] = {communicate}
+    if abort_wait is not None:
+        waiting.add(abort_wait)
+
+    timed_out = False
+    aborted = False
+    try:
+        done, _ = await asyncio.wait(
+            waiting,
+            timeout=None if timeout_ms is None else timeout_ms / 1000,
+            return_when=asyncio.FIRST_COMPLETED,
+        )
+        if communicate not in done:
+            timed_out = not done
+            aborted = abort_wait is not None and abort_wait in done
+            kill_process_group(proc.pid)
+        stdout_b, stderr_b = await communicate
+        await proc.wait()
+    finally:
+        if abort_wait is not None:
+            abort_wait.cancel()
+
+    code = proc.returncode
+    return CommandResult(
+        code=None if code is None or code < 0 else code,
+        stdout=stdout_b.decode(errors="replace"),
+        stderr=stderr_b.decode(errors="replace"),
+        timed_out=timed_out,
+        aborted=aborted or (signal.aborted if signal else False),
+    )
+
+
+def tail(text: str, n: int = 40) -> str:
+    """Last `n` lines of text, for compact failure output."""
+    return "\n".join(text.rstrip().split("\n")[-n:])

whiletrue/basics/sessions.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from pydantic import BaseModel
+
+from whiletrue.core import (
+    ActionModule,
+    Emit,
+    EmptyConfig,
+    Ports,
+    PortSpec,
+    Scope,
+    StepContext,
+    define_action,
+)
+from whiletrue.core.module import PortValues
+
+
+class SessionEndConfig(BaseModel):
+    #: The shared session to close (`item` is the one `session: item` uses).
+    name: str = "item"
+
+
+def _sessions(scope: Scope) -> dict[str, str]:
+    sessions: dict[str, str] | None = scope.get("sessions")
+    return sessions if sessions is not None else {}
+
+
+async def _end(input: PortValues, ctx: StepContext[SessionEndConfig]) -> Emit:
+    _sessions(ctx.scope).pop(ctx.config.name, None)
+    return Emit("ok", input.get("in"))
+
+
+async def _reset(input: PortValues, ctx: StepContext[EmptyConfig]) -> Emit:
+    _sessions(ctx.scope).clear()
+    return Emit("ok", input.get("in"))
+
+
+_PORTS = Ports(inputs=[PortSpec("in", "Any")], outputs=[PortSpec("out", "Any")])
+
+#: Closes a shared session: the next agent that asks for it starts a new conversation.
+session_end_module: ActionModule[SessionEndConfig] = define_action(
+    ActionModule(
+        id="session.end",
+        config=SessionEndConfig,
+        ports=_PORTS,
+        effects="deterministic",
+        cacheable=False,
+        run=_end,
+    )
+)
+
+#: Drops every shared session of the item, so later agents start with a clean context.
+context_reset_module: ActionModule[EmptyConfig] = define_action(
+    ActionModule(
+        id="context.reset",
+        config=EmptyConfig,
+        ports=_PORTS,
+        effects="deterministic",
+        cacheable=False,
+        run=_reset,
+    )
+)

whiletrue/basics/set_.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel
+
+from whiletrue.core import ActionModule, Emit, Ports, PortSpec, StepContext, define_action
+from whiletrue.core.module import PortValues
+
+
+class SetConfig(BaseModel):
+    #: Any JSON value; `{{...}}` templates in strings are rendered before the step runs.
+    value: Any = None
+
+
+async def _run(_input: PortValues, ctx: StepContext[SetConfig]) -> Emit:
+    return Emit("ok", ctx.config.value)
+
+
+#: Emits a constant (after templating). Useful for markers in branches and for tests.
+set_module: ActionModule[SetConfig] = define_action(
+    ActionModule(
+        id="set",
+        config=SetConfig,
+        ports=Ports(inputs=[PortSpec("in", "Any")], outputs=[PortSpec("out", "Any")]),
+        effects="pure",
+        run=_run,
+    )
+)