mulebox 1.1.1__py3-none-any.whl

mulebox/__init__.py

@@ -0,0 +1,5 @@
+"""mulebox: send AI coding agents' heavy work to Linux machines on your network."""
+
+from mulebox.version import get_version
+
+__all__ = ["get_version"]

mulebox/__main__.py

@@ -0,0 +1,7 @@
+"""`python -m mulebox`."""
+
+import sys
+
+from mulebox.cli import main
+
+sys.exit(main())

mulebox/cli.py

@@ -0,0 +1,62 @@
+"""Single entry point: `mulebox <command>`.
+
+Commands are registered here as they are implemented (see docs/phases.md). This module only
+parses arguments and dispatches; the work happens in the client, coordinator and worker
+packages.
+"""
+
+import argparse
+import sys
+from collections.abc import Sequence
+
+from mulebox.client import commands as client_commands
+from mulebox.client import (
+    doctor_commands,
+    feedback_commands,
+    init_commands,
+    job_commands,
+    job_controls,
+    machine_commands,
+    project_commands,
+    setup_commands,
+    skill_commands,
+    stats_commands,
+)
+from mulebox.coordinator import commands as coordinator_commands
+from mulebox.dashboard import commands as dashboard_commands
+from mulebox.runners import commands as runner_commands
+from mulebox.version import get_version
+from mulebox.worker import commands as worker_commands
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="mulebox",
+        description="Run your agents' builds and tests on build machines through one queue.",
+    )
+    parser.add_argument("--version", action="version", version=f"mulebox {get_version()}")
+    sub = parser.add_subparsers(dest="command", metavar="<command>")
+    sub.add_parser("version", help="print the mulebox version")
+    for package in (client_commands, init_commands, job_commands, job_controls, machine_commands,
+                    project_commands, stats_commands,
+                    setup_commands, doctor_commands, skill_commands, feedback_commands, coordinator_commands, worker_commands, runner_commands,
+                    dashboard_commands):
+        package.register(sub)
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    if args.command == "version":
+        print(get_version())
+        return 0
+    handler = getattr(args, "func", None)
+    if handler is not None:
+        return handler(args)
+    parser.print_help()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

mulebox/client/__init__.py

@@ -0,0 +1 @@
+"""mulebox client (docs/interfaces-m1.md)."""

mulebox/client/attention.py

@@ -0,0 +1,156 @@
+"""What a person or agent has to hear about a job: questions, and results from while away.
+
+Pure text from `status_ok`, `queue_ok` and `submit_ok` bodies, except `warn_here`, which asks
+the coordinator on behalf of a command that does not otherwise see the queue. All of it goes to
+stderr, so `--json` on stdout stays clean. A coordinator too old for any of this sends none of
+these keys and nothing is printed.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, TextIO
+
+from mulebox.client import report
+from mulebox.client.remote import Remote, request
+from mulebox.client.report import format_duration
+from mulebox.common.config import (
+    ConfigError,
+    find_project_config,
+    load_client_config,
+    load_project_config,
+)
+from mulebox.common.errors import MuleboxError
+from mulebox.common.network_errors import LOST
+from mulebox.common.protocol import MessageType
+from mulebox.common import disk, status
+from mulebox.common.status import FETCHING, number
+
+WARNING = "warning: "
+QUEUE_NOTHING_FINISHED = 0
+
+
+def _name(job: dict[str, Any]) -> str:
+    known = [str(job[key]) for key in ("project", "job") if job.get(key)]
+    return f" ({' '.join(known)})" if known else ""
+
+
+def stops_in(view: dict[str, Any]) -> str:
+    """When a job stops if it stays stuck. A working job does not: this time keeps moving."""
+    return f"stops in {format_duration(number(view.get('left_s')))} if it stays stuck"
+
+
+def question_line(job_id: str, view: dict[str, Any], job: dict[str, Any] | None = None) -> str:
+    """The reminder of a question not answered yet; it names both answers."""
+    return (f"{WARNING}job {job_id}{_name(job or {})} has run a long time: is everything OK? "
+            f"While it works it goes on; it {stops_in(view)}. Yes: mulebox ok {job_id}. "
+            f"No: mulebox cancel {job_id}")
+
+
+def result_words(job: dict[str, Any]) -> str:
+    code = job.get("exit_code")
+    words = str(status.outcome_of(job)) + (f", exit {code}" if code is not None else "")
+    commit = job.get("base_commit")
+    return f"{words}, on commit {str(commit)[:10]}" if commit else words
+
+
+def away_line(job: dict[str, Any]) -> str:
+    """A result nobody received: the agent need not run the job again."""
+    job_id = job.get("job_id")
+    return (f"job {job_id}{_name(job)} finished while you were away: {result_words(job)}. "
+            f"No need to retest; its log: mulebox logs {job_id}")
+
+
+def already_line(job: dict[str, Any]) -> str:
+    """`check` found the same job on the same files."""
+    job_id = job.get("job_id")
+    if job.get("state") != "finished":
+        return f"this exact job is already {job.get('state')} as {job_id}: following that one"
+    return (f"already tested: job {job_id}{_name(job)} ran on exactly these files: "
+            f"{result_words(job)}. Not run again; to force a run: --again. "
+            f"Its log: mulebox logs {job_id}")
+
+
+def activity_text(activity: dict[str, Any]) -> str:
+    """The readings there are; the worker leaves out what it could not measure."""
+    parts = []
+    if activity.get("stage"):
+        parts.append(f"stage {activity['stage']}")
+    quiet = number(activity.get("output_idle_s"))
+    if quiet is not None:
+        parts.append(f"last output {format_duration(quiet)} ago")
+    job_cpu = number(activity.get("job_cpu_pct"))
+    if job_cpu is not None:
+        parts.append(f"job CPU {job_cpu:.0f}% of a core")
+    cpu, net = number(activity.get("cpu_pct")), number(activity.get("net_kib_s"))
+    if cpu is not None:
+        parts.append(f"machine CPU {cpu:.0f}% of a core")
+    if net is not None:
+        parts.append(f"machine network {net:.1f} KiB/s")
+    if activity.get("last_line"):
+        parts.append(f"last line: {activity['last_line']}")
+    return ", ".join(parts) or "no readings from the machine"
+
+
+def check_in_line(job_id: str, check_in: dict[str, Any]) -> str:
+    took = format_duration(number(check_in.get("elapsed_s")))
+    return (f"{WARNING}check-in: job {job_id} has run {took}; {activity_text(check_in)}. "
+            f"Is everything OK? Yes: mulebox ok {job_id}. No: mulebox cancel {job_id}")
+
+
+def notice(job_id: str, body: dict[str, Any]) -> tuple[Any, str] | None:
+    """(a key that is the same on a reconnect, the line) for a body that carries a notice."""
+    check_in = body.get("check_in")
+    if isinstance(check_in, dict):
+        return ("check_in", check_in.get("number")), check_in_line(job_id, check_in)
+    return None
+
+
+def network_trouble(activity: dict[str, Any]) -> bool:
+    """Whether a job that is fetching things printed a lost connection last. Not in `command`:
+    a project's own tests can print that on purpose."""
+    line = activity.get("last_line")
+    return activity.get("stage") in FETCHING and bool(line and LOST.search(str(line)))
+
+
+def status_lines(body: dict[str, Any]) -> list[str]:
+    """`mulebox status`: what the job is doing and what it is waiting to hear."""
+    job_id, lines = str(body.get("job_id")), []
+    activity, view = body.get("activity"), body.get("question")
+    if isinstance(activity, dict):
+        took = format_duration(number(activity.get("elapsed_s")))
+        lines.append(f"  activity: running {took}, {activity_text(activity)}")
+        if network_trouble(activity):
+            lines.append("  warning: its last line reads as a lost connection; downloads may be "
+                         "retrying, and this is the machine's network, not the project")
+    if isinstance(view, dict):
+        lines.append(f"  question: is everything OK? It {stops_in(view)}. "
+                     f"Yes: mulebox ok {job_id}. No: mulebox cancel {job_id}")
+    return lines
+
+
+def tell(body: dict[str, Any], err: TextIO) -> None:
+    """Questions waiting for an answer and results from while away, from a reply's body."""
+    for job in _dicts(body.get("questions")):
+        report.status(err, question_line(str(job.get("job_id")), job.get("question") or {}, job))
+    for job in _dicts(body.get("finished_away")):
+        report.status(err, away_line(job))
+    for low in _dicts(body.get("low_disks")):
+        report.status(err, WARNING + disk.warning(str(low.get("worker")), int(low.get("free") or 0)))
+
+
+def _dicts(items: Any) -> list[dict[str, Any]]:
+    return [item for item in items if isinstance(item, dict)] if isinstance(items, list) else []
+
+
+def tell_here(err: TextIO, folder: Path | None = None) -> None:
+    """Tell this project's news. Only a remark on another command's way out: nothing that goes
+    wrong here may change what that command returned."""
+    try:
+        project = load_project_config(find_project_config(folder or Path.cwd())).project
+        remote = Remote(load_client_config())
+        reply = remote.call(request(MessageType.QUEUE, limit=QUEUE_NOTHING_FINISHED,
+                                    news_for=project))
+    except (ConfigError, MuleboxError, OSError):
+        return
+    tell(reply.body, err)

mulebox/client/commands.py

@@ -0,0 +1,304 @@
+"""CLI subcommands of the client package (docs/interfaces-m1.md, CLI).
+
+Exit codes for client failures: 2 usage/config error, 3 build machine unreachable,
+4 protocol/remote error, 5 job could not start, 76 stopped as stuck with no answer from the agent,
+130 Ctrl-C. Job failures return the job's own exit code.
+A build machine that cannot be reached is reported as such; mulebox never runs the job
+anywhere else instead.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+from mulebox.client import (attention, contract, queue_view, report, skill_install, summary,
+                            take_back, tui)
+from mulebox.client.remote import CoordinatorError, Remote, request
+from mulebox.client.snapshot import SnapshotError
+from mulebox.client.submit import Streams, Watcher, WatchLost, run_job
+from mulebox.common.config import (
+    ConfigError,
+    ProjectConfig,
+    ProjectNotFound,
+    find_project_config,
+    load_client_config,
+    load_project_config,
+)
+from mulebox.common.config_jobs import BARE
+from mulebox.common.errors import MuleboxError
+from mulebox.common.protocol import MessageType, ProtocolError
+from mulebox.common.redact import Redactor
+from mulebox.common.ssh import RemoteTimeout, SSHError, SSHUnavailable
+from mulebox.version import get_version
+
+QUEUE_LIMIT = 20
+Handler = Callable[[argparse.Namespace], int]
+
+
+class UsageError(MuleboxError):
+    """The command line asks for something that does not exist."""
+
+
+def register(sub: argparse._SubParsersAction) -> None:
+    """Add this package's subcommands. Each sets func=handler(args) -> int."""
+    check = sub.add_parser("check", help="run a job from .mulebox.yml on a build machine")
+    check.add_argument("job", nargs="?", default="check", help="job name (default: check)")
+    check.add_argument("--detach", action="store_true",
+                       help="print the job id once it is queued and leave it running")
+    check.add_argument("--again", action="store_true",
+                       help="run even if these exact files were already tested")
+    check.set_defaults(func=guarded(_check))
+
+    warm = sub.add_parser("warm", help="prepare a job ahead of time: toolchains, images, caches, "
+                          "install; no command")
+    warm.add_argument("job", nargs="?", default="check", help="job name (default: check)")
+    warm.add_argument("--detach", action="store_true",
+                      help="print the job id once it is queued and leave it running")
+    warm.set_defaults(func=guarded(_warm, notices=True))
+
+    run = sub.add_parser("run", help="run an ad-hoc command on a build machine")
+    run.add_argument("--detach", action="store_true",
+                     help="print the job id once it is queued and leave it running")
+    run.add_argument("--in", dest="service", metavar="SERVICE",
+                     help="run it inside this compose service")
+    run.add_argument("--bare", action="store_true",
+                     help="the command alone: no containers, install or setup")
+    run.add_argument("--take", action="append", default=[], metavar="PATH",
+                     help="once the command passed, copy this file back into the project "
+                          "(repeat for more), e.g. --take api/composer.lock")
+    run.add_argument("argv", nargs=argparse.REMAINDER, help="-- command and arguments")
+    run.set_defaults(func=guarded(_run))
+
+    queue = sub.add_parser("queue", help="list running, waiting and recent jobs")
+    queue.add_argument("--watch", action="store_true", help="full-screen live view")
+    queue.set_defaults(func=guarded(_queue))
+
+    known = sub.add_parser("projects", help="list every project that has run here")
+    known.add_argument("--json", action="store_true", help="stable keys for tools")
+    known.set_defaults(func=guarded(_projects, notices=True))
+
+    logs = sub.add_parser("logs", help="show a job's log")
+    logs.add_argument("job_id")
+    logs.add_argument("-f", "--follow", action="store_true", help="follow until it finishes")
+    logs.add_argument("--summary", action="store_true",
+                      help="only why it failed: stage, exit code, last relevant lines")
+    logs.set_defaults(func=guarded(_logs, notices=True))
+
+    cancel = sub.add_parser("cancel", help="cancel a job")
+    cancel.add_argument("job_id")
+    cancel.set_defaults(func=guarded(_cancel, notices=True))
+
+    terms = sub.add_parser("contract", help="print how to work in this project through "
+                           "mulebox, for agents")
+    terms.add_argument("--json", action="store_true", help="stable keys for tools")
+    terms.set_defaults(func=guarded(_contract))
+
+
+def guarded(handler: Handler, notices: bool = False) -> Handler:
+    """Turn expected errors into one clear stderr line and the contract's exit code. `notices`:
+    a command that reached the machine also tells this project's questions and results."""
+
+    def run(args: argparse.Namespace) -> int:
+        try:
+            code = handler(args)
+            if notices:
+                attention.tell_here(sys.stderr)
+            return code
+        except KeyboardInterrupt:
+            return report.EXIT_CANCELLED
+        except (ConfigError, SnapshotError, UsageError) as exc:
+            return _fail(str(exc), report.EXIT_USAGE)
+        except WatchLost as exc:
+            return _fail(str(exc), report.EXIT_UNREACHABLE)
+        except (SSHUnavailable, RemoteTimeout) as exc:
+            return _fail(
+                f"build machine unreachable: {exc} The job was not run; "
+                "fix the connection to the build machine and retry.",
+                report.EXIT_UNREACHABLE,
+            )
+        except (CoordinatorError, ProtocolError, SSHError) as exc:
+            return _fail(str(exc), report.EXIT_REMOTE)
+
+    return run
+
+
+def _fail(message: str, code: int) -> int:
+    report.status(sys.stderr, f"error: {Redactor().redact(message)}")
+    return code
+
+
+def project_here() -> tuple[ProjectConfig, Path]:
+    path = find_project_config(Path.cwd())
+    return load_project_config(path), path.parent
+
+
+def connect() -> Remote:
+    return Remote(load_client_config())
+
+
+def _contract(args: argparse.Namespace) -> int:
+    try:
+        config = project_here()[0]
+    except ProjectNotFound:
+        config = None
+    terms = contract.build(config, get_version())
+    if stale := skill_install.stale_here():
+        report.status(sys.stderr, stale)
+    sys.stdout.write(json.dumps(terms, indent=2) + "\n" if args.json
+                     else contract.render(terms))
+    return 0
+
+
+def _named_job(name: str) -> tuple[ProjectConfig, Path]:
+    project, root = project_here()
+    if name not in project.jobs:
+        names = ", ".join(sorted(project.jobs))
+        raise UsageError(f"no job {name!r} in .mulebox.yml (jobs: {names})")
+    return project, root
+
+
+def _check(args: argparse.Namespace) -> int:
+    project, root = _named_job(args.job)
+    return run_job(project, args.job, root, connect(), detach=args.detach,
+                   again=args.again)
+
+
+def _warm(args: argparse.Namespace) -> int:
+    project, root = _named_job(args.job)
+    return run_job(project, args.job, root, connect(), detach=args.detach, warm=True)
+
+
+def _run(args: argparse.Namespace) -> int:
+    argv = list(args.argv)
+    if argv and argv[0] == "--":
+        argv = argv[1:]
+    if not argv:
+        raise UsageError("nothing to run; use `mulebox run -- <command> [args...]`")
+    project, root = project_here()
+    if args.service:
+        if args.bare:
+            raise UsageError("--in runs in a service, which --bare does not start")
+        if not project.compose:
+            raise UsageError("--in needs `compose:` in .mulebox.yml")
+        argv = ["docker", "compose", "exec", "-T", args.service, *argv]
+    if args.take and args.detach:
+        raise UsageError("--take brings files back at the end, so it waits; drop --detach")
+    try:
+        take = take_back.check(args.take, project.exclude)
+    except ValueError as exc:
+        raise UsageError(str(exc)) from None
+    return run_job(project, argv, root, connect(), detach=args.detach,
+                   options=BARE if args.bare else None, take=take)
+
+
+def _queue(args: argparse.Namespace) -> int:
+    remote = connect()
+    latest: dict[str, Any] = {}
+
+    def fetch() -> list[dict]:
+        reply = remote.call(request(MessageType.QUEUE, limit=QUEUE_LIMIT))
+        latest.update(reply.body)
+        jobs = reply.body.get("jobs")
+        if not isinstance(jobs, list):
+            raise CoordinatorError("the queue reply has no job list")
+        return [j for j in jobs if isinstance(j, dict)]
+
+    if args.watch:
+        return tui.watch(fetch, remote.target)
+    tui.show_once(fetch, sys.stdout)
+    report.dashboard_hint(sys.stderr, latest)
+    return 0
+
+
+def _with_folders(remote: Remote, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """`rows` with each project's folder and size; the job history alone from an older machine."""
+    try:
+        folders = json.loads(remote.project_home("list", "--json").stdout)
+    except (MuleboxError, ValueError):
+        return rows
+    known = {row["project"]: row for row in folders}
+    return [{**row, **{k: v for k, v in known.get(row["project"], {}).items() if k != "project"}}
+            for row in rows]
+
+
+def _projects(args: argparse.Namespace) -> int:
+    remote = connect()
+    rows = remote.call(request(MessageType.PROJECTS)).body.get("projects")
+    if not isinstance(rows, list):
+        raise CoordinatorError("the projects reply has no project list")
+    rows = _with_folders(remote, rows)
+    if args.json:
+        sys.stdout.write(json.dumps(rows, indent=2) + "\n")
+    else:
+        sys.stdout.write("\n".join(queue_view.render_projects(rows)) + "\n")
+    return 0
+
+
+def _logs(args: argparse.Namespace) -> int:
+    remote = connect()
+    watcher = Watcher(remote, Streams())
+    watcher.job_id = args.job_id
+    if args.summary:
+        return _log_summary(remote, args.job_id)
+    if args.follow:
+        result = watcher.attach()
+        note_expired_log(watcher)
+        watcher.finish(result)
+        return report.exit_code_for(result)
+    _read_log(remote, args.job_id, 0, watcher.write_log)
+    note_expired_log(watcher)
+    return 0
+
+
+def note_expired_log(watcher: Watcher) -> None:
+    """A log that printed nothing may be gone rather than empty. Only a remark: losing the
+    connection here must not cost `wait` the job's exit code."""
+    if watcher.offset:
+        return
+    try:
+        body = watcher.remote.call(request(MessageType.STATUS, job_id=watcher.job_id)).body
+    except (MuleboxError, OSError):
+        return
+    if report.log_expired(body):
+        report.status(watcher.io.err, report.expired_line(watcher.job_id))
+
+
+def _read_log(remote: Remote, job_id: str, offset: int, sink: Callable[[dict], None]) -> None:
+    while True:
+        reply = remote.call(request(MessageType.LOGS, job_id=job_id, offset=offset))
+        sink(reply.body)
+        next_offset = reply.body.get("next_offset", offset)
+        if reply.body.get("eof", True) or next_offset == offset:
+            return
+        offset = next_offset
+
+
+def _log_summary(remote: Remote, job_id: str) -> int:
+    body = remote.call(request(MessageType.STATUS, job_id=job_id)).body
+    state = body.get("state")
+    if state != "finished":
+        print(f"job {job_id} is {state}; a summary needs it to end")
+    elif not summary.wanted(body):
+        print(report.outcome_line(job_id, body))
+    else:
+        tail = summary.LogTail()
+        size = body.get("log_size")
+        start = max(0, size - summary.TAIL_BYTES) if isinstance(size, int) else 0
+        tail.cut = start > 0
+        _read_log(remote, job_id, start, lambda chunk: tail.feed(chunk.get("data") or ""))
+        if not tail.text and report.log_expired(body):
+            report.status(sys.stderr, report.expired_line(job_id))
+        print("\n".join(summary.render(job_id, body, summary.relevant_lines(tail))))
+    return 0
+
+
+def _cancel(args: argparse.Namespace) -> int:
+    reply = connect().call(request(MessageType.CANCEL, job_id=args.job_id))
+    state = reply.body.get("state", "unknown")
+    report.status(sys.stderr, f"cancel requested for job {args.job_id} (state {state})")
+    return 0