fherma-runner 0.2.0__py3-none-any.whl

fherma_runner/__init__.py

@@ -0,0 +1,8 @@
+"""A FHERMA runner: takes measurement jobs and reports what happened.
+
+Nothing here imports anything that does not ship with Python, and docker is
+called as a command rather than through a library. Installing this on a server
+is copying a directory.
+"""
+
+__version__ = "0.2.0"

fherma_runner/__main__.py

@@ -0,0 +1,129 @@
+"""The command line, and the loop that outlives everything under it.
+
+    fherma-runner --api https://fherma.io --name bench-01
+    fherma-runner --once                one assignment, then stop
+    fherma-runner --re-register         forget the key and join again
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+import time
+from importlib import metadata
+
+from . import job, machine
+from .api import Api, Refused, Unreachable
+from .session import KEYFILE, describe, identify
+
+
+def version() -> str:
+    """What is installed, asked of the installation rather than hardcoded."""
+    try:
+        return metadata.version("fherma-runner")
+    except metadata.PackageNotFoundError:
+        return "unknown (running from a checkout)"
+
+
+def parse(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        prog="fherma-runner",
+        description="Takes measurement jobs from a FHERMA platform and reports what happened.",
+    )
+    parser.add_argument("--api", default=os.environ.get("FHERMA_API", "http://localhost:4000"))
+    parser.add_argument("--name", default=os.environ.get("FHERMA_RUNNER_NAME", "runner"))
+    parser.add_argument("--interval", type=float, default=3.0, help="seconds between polls")
+    parser.add_argument("--once", action="store_true", help="one assignment, then stop")
+    parser.add_argument("--token", help="an invitation, exchanged once for this machine's key")
+    parser.add_argument("--re-register", action="store_true", help="forget the key and join again")
+    parser.add_argument("--describe", action="store_true", help="print what this machine is, and stop")
+    # The first question anybody asks after an install that may not have worked.
+    # One line rather than three, because argparse reflows the text it prints
+    # and a newline here comes out as a space.
+    parser.add_argument(
+        "--version",
+        action="version",
+        version=f"fherma-runner {version()} · python {sys.version.split()[0]}",
+    )
+
+    limits = parser.add_argument_group(
+        "limits",
+        "What a solution's container may take. The platform sets the time limits; "
+        "these are the machine's own.",
+    )
+    limits.add_argument("--memory", default=os.environ.get("FHERMA_MEMORY", "8g"))
+    limits.add_argument("--cpus", default=os.environ.get("FHERMA_CPUS"))
+
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    options = parse(argv)
+
+    if options.describe:
+        for name, value in machine.describe().items():
+            print(f"{name:12} {value}")
+        print(f"{'can':12} {', '.join(one['name'] for one in machine.capabilities()) or 'nothing'}")
+        return 0
+
+    if options.re_register and KEYFILE.exists():
+        KEYFILE.unlink()
+
+    api = Api(options.api)
+
+    while True:
+        try:
+            api.key = identify(api, options.name, options.token)
+            break
+        except Refused as failure:
+            print(f"could not register: {failure}", file=sys.stderr)
+            return 1
+        except Unreachable as failure:
+            # A runner boots when its machine boots, which may be before the
+            # platform is up. Waiting is the correct answer; exiting would need
+            # somebody to notice and start it again.
+            print(f"waiting for {options.api}: {failure}", file=sys.stderr)
+            time.sleep(options.interval)
+
+    try:
+        describe(api)
+    except Refused as failure:
+        if failure.status in (401, 403):
+            print("the key was refused — try --re-register", file=sys.stderr)
+            return 1
+        raise
+    except Unreachable as failure:
+        print(f"could not describe this machine: {failure}", file=sys.stderr)
+
+    print(f"polling {options.api} every {options.interval}s — ctrl-c to stop")
+
+    while True:
+        try:
+            did = job.take(api, options)
+        except Refused as failure:
+            # A lapsed lease is the ordinary case: stop working on it and ask
+            # again rather than reporting into a result somebody else now owns.
+            print(f"  refused ({failure.status}) — dropping this one", file=sys.stderr)
+            did = False
+        except Unreachable as failure:
+            print(f"  cannot reach {options.api}: {failure}", file=sys.stderr)
+            did = False
+
+        if did and options.once:
+            return 0
+        if not did:
+            print(".", end="", flush=True)
+        time.sleep(options.interval)
+
+
+def run() -> None:
+    """The console entry point. Ctrl-C is an ending, not a crash."""
+    try:
+        raise SystemExit(main())
+    except KeyboardInterrupt:
+        print()
+
+
+if __name__ == "__main__":
+    run()

fherma_runner/api.py

@@ -0,0 +1,80 @@
+"""Talking to the platform, and the three ways it can go wrong.
+
+The runner is the only one of the three programs on a machine that speaks to
+the platform at all. The bundle is handed a directory; the solution is handed a
+directory; neither has a key and neither has a network.
+
+Nothing here is imported that does not ship with Python. A machine that runs
+this needs nothing installed beyond an interpreter and docker.
+"""
+
+from __future__ import annotations
+
+import json
+import urllib.error
+import urllib.request
+
+
+class Refused(Exception):
+    """The platform answered, and the answer was no."""
+
+    def __init__(self, status: int, detail: str) -> None:
+        super().__init__(f"{status}: {detail}")
+        self.status = status
+        self.detail = detail
+
+
+class Unreachable(Exception):
+    """The platform did not answer. Says nothing about the work."""
+
+
+class Dropped(Exception):
+    """This assignment is no longer ours. Stop, and send nothing."""
+
+
+class Api:
+    """The calls a runner makes, and the one that gets it a key.
+
+    Errors are not swallowed. A runner that carries on after a refusal is a
+    runner reporting into somebody else's result, and the refusals here mean
+    exactly that: the lease lapsed, the assignment went elsewhere.
+    """
+
+    def __init__(self, base: str, key: str | None = None, timeout: float = 30) -> None:
+        self.base = base.rstrip("/")
+        self.key = key
+        self.timeout = timeout
+
+    def call(self, method: str, path: str, body: dict | None = None) -> dict:
+        raw = self._request(method, path, body)
+        return json.loads(raw) if raw else {}
+
+    def download(self, path: str) -> bytes:
+        """A file rather than a document — the bundle archive, and only that."""
+        return self._request("GET", path, None, decode=False)
+
+    def _request(self, method: str, path: str, body: dict | None, decode: bool = True):
+        request = urllib.request.Request(
+            f"{self.base}/api{path}",
+            method=method,
+            data=json.dumps(body).encode() if body is not None else None,
+            headers={
+                "content-type": "application/json",
+                **({"x-fherma-runner": self.key} if self.key else {}),
+            },
+        )
+
+        try:
+            with urllib.request.urlopen(request, timeout=self.timeout) as answer:
+                raw = answer.read()
+                return raw.decode() if decode else raw
+        except urllib.error.HTTPError as failure:
+            detail = failure.read().decode()[:300]
+            raise Refused(failure.code, detail) from None
+        except OSError as failure:
+            # Refused connections arrive as URLError; a read that times out
+            # half-way through arrives as a bare TimeoutError, and a platform
+            # restarting mid-request produces both. They mean one thing to a
+            # runner — nobody answered — and a machine meant to sit in a rack
+            # for weeks must not die of any of them.
+            raise Unreachable(str(getattr(failure, "reason", None) or failure)) from None

fherma_runner/docker.py

@@ -0,0 +1,146 @@
+"""Running containers, by calling the command rather than a library.
+
+A library would be a dependency, and the point of having none is that putting
+this on a server is copying a directory. `docker` is already there — a machine
+that cannot run containers cannot be a runner at all.
+
+Everything here is one container, started and finished. Nothing is left
+running, and nothing is reused between points: a process that outlived its
+point would be measuring the next one with the previous one's memory.
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+class DockerMissing(Exception):
+    """No docker on this machine, which is not something to work around."""
+
+
+class Timeout(Exception):
+    """It ran past its limit and was killed."""
+
+
+@dataclass
+class Mount:
+    """One path from the host, and whether the container may write to it."""
+
+    source: Path
+    target: str
+    write: bool = False
+
+    def arg(self) -> str:
+        return f"{self.source}:{self.target}{'' if self.write else ':ro'}"
+
+
+@dataclass
+class Limits:
+    """What a container may take. None leaves docker's own default."""
+
+    memory: str | None = None
+    cpus: str | None = None
+    pids: int = 512
+
+
+@dataclass
+class Result:
+    code: int
+    out: str
+    err: str
+
+    @property
+    def ok(self) -> bool:
+        return self.code == 0
+
+
+def available() -> bool:
+    return shutil.which("docker") is not None
+
+
+def pull(reference: str, timeout: float = 900) -> Result:
+    """Fetch an image by digest.
+
+    By digest, not by tag: a tag is a pointer, and an image rebuilt under the
+    same tag would silently become what a past measurement claims to have run
+    in.
+    """
+    return _run(["docker", "pull", "--quiet", reference], timeout)
+
+
+def run(
+    image: str,
+    command: list[str],
+    mounts: list[Mount] | None = None,
+    workdir: str | None = None,
+    limits: Limits | None = None,
+    network: bool = False,
+    timeout: float = 600,
+    name: str | None = None,
+) -> Result:
+    """One container, to completion.
+
+    Network is off unless asked for. The only stage that needs it is the build,
+    where dependencies are fetched; a measurement that could reach the network
+    could fetch its answer.
+    """
+    argv = ["docker", "run", "--rm"]
+
+    if name:
+        argv += ["--name", name]
+    if not network:
+        argv += ["--network", "none"]
+
+    limits = limits or Limits()
+    if limits.memory:
+        argv += ["--memory", limits.memory]
+    if limits.cpus:
+        argv += ["--cpus", limits.cpus]
+    if limits.pids:
+        argv += ["--pids-limit", str(limits.pids)]
+
+    # Author code, even when the fleet is ours.
+    argv += ["--cap-drop", "ALL", "--security-opt", "no-new-privileges"]
+
+    # The command is complete: the bundle's comes from its record, the
+    # solution's from its language. An image with an ENTRYPOINT would prepend
+    # to it — `python` in front of `python main.py` — so it is cleared.
+    argv += ["--entrypoint", ""]
+
+    for mount in mounts or []:
+        mount.source.mkdir(parents=True, exist_ok=True) if mount.write else None
+        argv += ["-v", mount.arg()]
+
+    if workdir:
+        argv += ["-w", workdir]
+
+    argv.append(image)
+    argv += command
+
+    try:
+        return _run(argv, timeout)
+    except Timeout:
+        if name:
+            # The container outlives the client that started it, so killing the
+            # subprocess is not killing the work.
+            _run(["docker", "kill", name], timeout=30)
+        raise
+
+
+def _run(argv: list[str], timeout: float) -> Result:
+    if not available():
+        raise DockerMissing("docker is not on the path")
+
+    try:
+        done = subprocess.run(
+            argv, capture_output=True, text=True, timeout=timeout, check=False
+        )
+    except subprocess.TimeoutExpired as expired:
+        raise Timeout(f"ran past {timeout:.0f}s") from expired
+    except OSError as failure:
+        raise DockerMissing(str(failure)) from failure
+
+    return Result(code=done.returncode, out=done.stdout or "", err=done.stderr or "")

fherma_runner/job.py

@@ -0,0 +1,226 @@
+"""One assignment, start to finish.
+
+Take the job, fetch what it names, build once, then a point at a time: make the
+cases, answer them, judge the answers, report, delete. Report per point, so a
+run that dies on the seventh of ten keeps the six it finished.
+
+Send nothing at all if the assignment stopped being ours. Finishing the work
+and reporting it would be reporting into a result somebody else now owns.
+"""
+
+from __future__ import annotations
+
+import argparse
+import shutil
+
+from . import docker, measure, workspace
+from .api import Api, Dropped, Refused
+from .docker import Limits
+from .log import Log
+from .measure import Environment, Plan
+from .session import beat
+from .workspace import Failed, Workspace
+
+
+def take(api: Api, options: argparse.Namespace) -> bool:
+    """One assignment. False when there was nothing to do."""
+    answer = api.call("GET", "/runners/me/work")
+    work = answer.get("work")
+    if not work:
+        return False
+
+    run = work["run"]
+    log = Log()
+    place = Workspace.make(run)
+
+    try:
+        return _do(api, work, place, log, options)
+    except Dropped as why:
+        log.warn(f"dropped: {why}")
+        return True
+    except Failed as why:
+        log.fail(str(why))
+        _give_up(api, run, str(why), log)
+        return True
+    except docker.DockerMissing as why:
+        log.fail(f"docker: {why}")
+        _give_up(api, run, f"this machine cannot run containers: {why}", log)
+        return True
+    finally:
+        place.clear()
+
+
+def _do(api: Api, work: dict, place: Workspace, log: Log, options) -> bool:
+    run = work["run"]
+    points = [Plan(**{k: one[k] for k in ("id", "point", "seeds")}) for one in work.get("points") or []]
+    where = _environment(work)
+    limits = Limits(memory=options.memory, cpus=options.cpus)
+    seconds = work.get("limits") or {}
+
+    log.done(f"run {run[:8]} · {len(points)} point(s) · {sum(len(p.seeds) for p in points)} case(s)")
+    log.info(f"solution runs in {where.solution}")
+    log.info(f"bundle runs in {where.bundle}")
+
+    log.step("pulling images")
+    for image in {where.solution, where.bundle}:
+        pulled = docker.pull(image)
+        if not pulled.ok:
+            raise Failed(f"could not pull {image}: {pulled.err.strip()[:300]}")
+    log.ok("images ready")
+
+    log.step("fetching the bundle")
+    workspace.fetch_bundle(api, place, work.get("bundle") or {})
+    log.ok(f"bundle unpacked · {len(list(place.bundle.iterdir()))} file(s)")
+
+    log.step("cloning the solution")
+    source = work.get("source") or {}
+    workspace.clone(place, source)
+    log.ok(f"{source.get('repo')} at {(source.get('commit') or '')[:12]}")
+
+    laid = workspace.lay_harness(place, work.get("harness") or [])
+    if laid:
+        log.info(f"harness written over the clone: {', '.join(laid)}")
+
+    _build(place, work, where, float(seconds.get("build_s", 600)), log)
+
+    passed = counted = 0
+    for index, plan in enumerate(points, start=1):
+        beat(api, run, f"point {index} of {len(points)} · {plan.id}", "running")
+
+        measured = measure.one_point(
+            place, plan, where, limits, float(seconds.get("point_s", 300)), log
+        )
+        cases = measured.cases
+        _say(log, plan, cases)
+
+        passed += sum(1 for one in cases if one["passed"])
+        counted += len(cases)
+        last = index == len(points)
+
+        if last:
+            log.done(f"{passed} of {counted} seeds passed")
+
+        if not _report(api, run, cases, measured.init_s, log, done=last):
+            return True
+
+        # The point is reported; the data can be made again from the bundle,
+        # the point and the seed at any time. One point on disk, whatever the
+        # plan asked for.
+        shutil.rmtree(place.point(plan.id), ignore_errors=True)
+
+    return True
+
+
+def _environment(work: dict) -> Environment:
+    image = work.get("image") or {}
+    bundle = work.get("bundle") or {}
+    bundle_image = bundle.get("image")
+
+    solution = image.get("reference") if isinstance(image, dict) else image
+    if not solution:
+        raise Failed("the job names no image for the solution")
+    if not bundle_image:
+        raise Failed("the job names no image for the bundle")
+
+    return Environment(
+        solution=solution,
+        bundle=bundle_image,
+        bundle_command=bundle.get("command") or "python main.py",
+        run_command=work.get("run_cmd") or "python main.py",
+    )
+
+
+def _build(place: Workspace, work: dict, where: Environment, limit: float, log: Log) -> None:
+    """Once for the whole job. The same binary answers every point."""
+    command = work.get("build")
+    if not command:
+        log.info("nothing to build")
+        return
+
+    log.step(f"building · {command}")
+    try:
+        done = docker.run(
+            where.solution,
+            ["sh", "-c", command],
+            mounts=[docker.Mount(place.solution, "/solution", write=True)],
+            workdir="/solution",
+            # The one stage that may reach the network: a build fetches
+            # dependencies. Measurement never does.
+            network=True,
+            timeout=limit,
+        )
+    except docker.Timeout as expired:
+        raise Failed(f"the build ran past {limit:.0f}s") from expired
+
+    if not done.ok:
+        tail = (done.err or done.out).strip().splitlines()[-12:]
+        for line in tail:
+            log.fail(line[:200])
+        raise Failed(f"the build failed with {done.code}")
+
+    log.ok("built")
+
+
+def _say(log: Log, plan: Plan, cases: list[dict]) -> None:
+    timed = [one["seconds"] for one in cases if one.get("seconds")]
+    good = sum(1 for one in cases if one["passed"])
+
+    if not timed:
+        log.fail(f"{plan.id}: nothing came back")
+    elif good == len(cases):
+        log.ok(f"{plan.id}: {good}/{len(cases)} · {_span(timed)}")
+    elif good == 0:
+        # Not a shade of partial success. Every seed was wrong, which is as
+        # failed as nothing coming back at all.
+        log.fail(f"{plan.id}: 0/{len(cases)} within tolerance · {_span(timed)}")
+    else:
+        log.warn(f"{plan.id}: {good}/{len(cases)} within tolerance · {_span(timed)}")
+
+
+def _span(timed: list[float]) -> str:
+    """The range, in a unit that has digits in it.
+
+    Two decimals of milliseconds turn a fast implementation into `0.00 ms`,
+    which reads the same as a measurement that failed.
+    """
+    return f"{_time(min(timed))} – {_time(max(timed))}"
+
+
+def _time(seconds: float) -> str:
+    for over, scale, name in ((60, 1 / 60, "min"), (1, 1, "s"), (1e-3, 1e3, "ms"),
+                              (1e-6, 1e6, "µs"), (0, 1e9, "ns")):
+        if abs(seconds) >= over:
+            value = seconds * scale
+            digits = 0 if abs(value) >= 100 else 1 if abs(value) >= 10 else 2
+            return f"{round(value, digits):g} {name}"
+    return "0"
+
+
+def _report(
+    api: Api, run: str, cases: list[dict], init_s: float | None, log: Log, done: bool
+) -> bool:
+    """One point's worth. False when the assignment was taken away."""
+    try:
+        api.call(
+            "POST",
+            f"/runners/me/runs/{run}/report",
+            {"cases": cases, "init_s": init_s, "build_log": log.text(), "done": done},
+        )
+        return True
+    except Refused as failure:
+        if failure.status in (401, 403, 404):
+            log.warn(f"taken away before the report landed: {failure.detail}")
+            return False
+        raise
+
+
+def _give_up(api: Api, run: str, reason: str, log: Log) -> None:
+    """Say why there is nothing, so the page shows a reason and not a silence."""
+    try:
+        api.call(
+            "POST",
+            f"/runners/me/runs/{run}/fail",
+            {"reason": reason[:4000], "build_log": log.text()},
+        )
+    except (Refused, Exception):  # noqa: BLE001 — already failing
+        pass

fherma_runner/log.py

@@ -0,0 +1,104 @@
+"""What the runner says while it works.
+
+Two audiences, one source. A person watching a terminal wants to see where the
+work has got to; the platform wants a transcript it can show on a page months
+later. Both are written here at the same moment, so the page cannot disagree
+with what the operator saw.
+
+Colour goes to the terminal and never into the transcript: escape codes stored
+in a database are a page rendering `[32m` at somebody.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import time
+
+#: Off when the output is not a terminal, so a log file or a pipe stays plain.
+COLOUR = sys.stdout.isatty() and os.environ.get("NO_COLOR") is None
+
+_CODES = {
+    "grey": "\033[90m",
+    "red": "\033[31m",
+    "green": "\033[32m",
+    "amber": "\033[33m",
+    "blue": "\033[34m",
+    "bold": "\033[1m",
+    "off": "\033[0m",
+}
+
+#: How each level is painted, and how wide the column is.
+_LEVELS = {
+    "INFO": "grey",
+    "STEP": "blue",
+    "OK": "green",
+    "WARN": "amber",
+    "FAIL": "red",
+    "DONE": "bold",
+}
+
+
+def paint(text: str, colour: str) -> str:
+    if not COLOUR or colour not in _CODES:
+        return text
+    return f"{_CODES[colour]}{text}{_CODES['off']}"
+
+
+class Log:
+    """A transcript that prints as it is written.
+
+    Kept because a run carries points and timings and not why the build took
+    four minutes or which seed the machine was on when it stopped. Written
+    while the work happens rather than assembled afterwards, so a run that dies
+    half way still leaves everything up to the moment it died.
+
+    One line per event: the clock, a level, the sentence. The page reads the
+    three parts into three columns.
+    """
+
+    def __init__(self, quiet: bool = False) -> None:
+        self.lines: list[str] = []
+        self.quiet = quiet
+
+    def say(self, level: str, text: str) -> None:
+        stamp = time.strftime("%H:%M:%S")
+        self.lines.append(f"{stamp} {level} {text}")
+
+        if self.quiet:
+            return
+
+        colour = _LEVELS.get(level, "grey")
+        print(
+            f"{paint(stamp, 'grey')} {paint(level.ljust(4), colour)} {text}",
+            flush=True,
+        )
+
+    def info(self, text: str) -> None:
+        self.say("INFO", text)
+
+    def step(self, text: str) -> None:
+        """A stage beginning. Printed apart so the shape of the work is visible."""
+        self.say("STEP", paint(text, "bold") if COLOUR else text)
+
+    def ok(self, text: str) -> None:
+        self.say("OK", text)
+
+    def warn(self, text: str) -> None:
+        self.say("WARN", text)
+
+    def fail(self, text: str) -> None:
+        self.say("FAIL", text)
+
+    def done(self, text: str) -> None:
+        self.say("DONE", text)
+
+    def text(self) -> str:
+        """The transcript, without colour. This is what the platform stores."""
+        return "\n".join(_plain(line) for line in self.lines)
+
+
+def _plain(line: str) -> str:
+    for code in _CODES.values():
+        line = line.replace(code, "")
+    return line