kalfa 3.0.0.dev0__py3-none-any.whl

kalfa/commands/sweep/checks.py

@@ -0,0 +1,98 @@
+from kalfa.commands.sweep.space import SPACE_HINT, space_values
+from kalfa.commands.sweep.strategies import STRATEGY_HINT, Grid, strategy_of
+from kalfa.core.check import SERIES_PREFIXES
+from kalfa.core.context import train_of
+from kalfa.core.errors import BuildError, error
+
+
+SWEEP_KEYS = ("strategy", "space", "objective", "record")
+OBJECTIVE_KEYS = ("watch", "mode", "at")
+MODES = ("min", "max")
+MOMENTS = ("best", "last")
+
+
+def series_problem(tree, watch):
+    names = set(tree.section("losses")) | set(tree.section("metrics"))
+    prefixes = set(SERIES_PREFIXES)
+    evaluate = train_of(tree).get("evaluate")
+    if isinstance(evaluate, list):
+        prefixes.update(str(name) for name in evaluate)
+    prefix, _, rest = watch.partition("/")
+    if not rest or prefix not in prefixes:
+        return f"{watch!r} does not look like a series such as val/<metric>"
+    if prefix in ("train", "val", "test") and rest.split("/")[0] not in names:
+        known = ", ".join(sorted(names)) or "none"
+        return f"{watch!r} names no loss or metric; the losses and metrics are {known}"
+    return None
+
+
+def check_space(tree, space, problems):
+    try:
+        dimensions = space_values(space)
+    except BuildError as exc:
+        problems.append(error("sweep_space", str(exc), "sweep.space", hint=SPACE_HINT))
+        return None
+    params = getattr(getattr(tree, "config", None), "params", None) or {}
+    known = ", ".join(sorted(str(name) for name in params))
+    for name in dimensions:
+        if name not in params:
+            problems.append(error("sweep_param", f"{name!r} is not in the params section; a swept name is a param the "
+                                  f"config reads as ${name}$", "sweep.space",
+                                  hint=f"the params are {known}" if known else "define it under params"))
+    return dimensions
+
+
+def check_strategy(raw, space, dimensions, problems):
+    try:
+        strategy = strategy_of(raw)
+    except BuildError as exc:
+        problems.append(error("sweep_strategy", str(exc), "sweep.strategy", hint=STRATEGY_HINT))
+        return
+    if dimensions is not None and isinstance(strategy, Grid):
+        try:
+            strategy.axes(space)
+        except BuildError as exc:
+            problems.append(error("sweep_space", str(exc), "sweep.space"))
+
+
+def check_objective(tree, objective, problems):
+    where = "sweep.objective"
+    if not isinstance(objective, dict):
+        problems.append(error("sweep_objective", "sweep.objective is a mapping {watch, mode, at}", where,
+                              hint="write objective: {watch: val/<metric>, mode: min}"))
+        return
+    for key in objective:
+        if key not in OBJECTIVE_KEYS:
+            problems.append(error("sweep_objective", f"{key!r} is not an objective key; the keys are "
+                                  + ", ".join(OBJECTIVE_KEYS), where))
+    watch = objective.get("watch")
+    if not isinstance(watch, str) or not watch:
+        problems.append(error("sweep_objective", "the objective needs watch, the series that ranks the points", where,
+                              hint="such as watch: val/rmse"))
+    else:
+        message = series_problem(tree, watch)
+        if message:
+            problems.append(error("sweep_objective", message, where))
+    mode = objective.get("mode")
+    if mode is not None and mode not in MODES:
+        problems.append(error("sweep_objective", f"mode is min or max, got {mode!r}", where))
+    at = objective.get("at")
+    if at is not None and at not in MOMENTS:
+        problems.append(error("sweep_objective", f"at is best or last, got {at!r}", where))
+
+
+def check_sweep(tree, raw, problems):
+    if not isinstance(raw, dict):
+        problems.append(error("sweep_key", "sweep is a mapping with strategy, space, objective and record", "sweep"))
+        return
+    for key in raw:
+        if key not in SWEEP_KEYS:
+            problems.append(error("sweep_key", f"{key!r} is not a sweep key; the keys are " + ", ".join(SWEEP_KEYS),
+                                  f"sweep.{key}"))
+    dimensions = check_space(tree, raw.get("space"), problems)
+    check_strategy(raw.get("strategy"), raw.get("space"), dimensions, problems)
+    check_objective(tree, raw.get("objective"), problems)
+    record = raw.get("record")
+    if not isinstance(record, str) or not record.strip():
+        problems.append(error("sweep_record", "the sweep needs record, the root directory its points go under",
+                              "sweep.record", hint="write record: runs/<name>, or give --record"))

kalfa/commands/sweep/plan.py

@@ -0,0 +1,183 @@
+import copy
+import json
+import os
+import shlex
+import shutil
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from ruamel.yaml.error import YAMLError
+
+from kalfa.commands.sweep.points import LOGS, POINTS_FILE, SWEEP_FILE
+from kalfa.commands.sweep.space import is_int
+from kalfa.commands.sweep.strategies import strategy_note, strategy_of
+from kalfa.core import run as running
+from kalfa.core.check import check
+from kalfa.core.errors import KalfaError, ProblemsError
+from kalfa.core.record import atomic, now
+from kalfa.core.spec import parse_value
+from kalfa.records import reader
+
+
+MAPPING_NOTE = "# mapping config: run through the python api"
+
+
+@dataclass
+class Plan:
+    paths: tuple
+    sets: tuple
+    params: tuple
+    root: str
+    strategy: object
+    space: dict
+    objective: dict
+    points: list | None = None
+    config_name: str = "sweep"
+    extra: dict = field(default_factory=dict)
+
+    @property
+    def count(self):
+        return None if self.points is None else len(self.points)
+
+    @property
+    def adaptive(self):
+        return self.points is None
+
+
+def objective_note(raw):
+    raw = raw if isinstance(raw, dict) else {}
+    return {"watch": raw.get("watch"), "mode": raw.get("mode") or "min", "at": raw.get("at") or "best"}
+
+
+def plan_seed(seed):
+    return int(seed) if is_int(seed) else 0
+
+
+def plan_of(paths=(), sets=(), params=(), mappings=(), record=None):
+    paths = tuple(str(path) for path in paths)
+    sets = tuple(running.pairs(sets))
+    params = tuple(running.pairs(params, "param"))
+    mappings = list(mappings)
+    checked = check(paths=paths, sets=list(sets), params=list(params), mappings=mappings)
+    problems = [problem for problem in checked.problems if record is None or problem.code != "sweep_record"]
+    if any(problem.severity == "error" for problem in problems):
+        raise ProblemsError(problems)
+    raw = checked.tree.settings.get("sweep") if checked.tree is not None else None
+    if not isinstance(raw, dict):
+        raise KalfaError("the config has no sweep section; write sweep: {strategy, space, objective, record}")
+    seed = plan_seed(checked.config.data.get("seed"))
+    strategy = strategy_of(raw.get("strategy"))
+    if getattr(strategy, "seed", 0) is None:
+        strategy.seed = seed
+    space = copy.deepcopy(raw["space"])
+    plan = Plan(paths, sets, params, str(record) if record is not None else str(raw["record"]), strategy, space,
+                objective_note(raw.get("objective")), config_name=Path(paths[0]).stem if paths else "sweep",
+                extra={"mappings": mappings, "seed": seed, "strategy": strategy_note(raw.get("strategy"))})
+    if callable(getattr(strategy, "points", None)):
+        plan.points = list(strategy.points(space, seed))
+    return plan
+
+
+def value_text(value):
+    if isinstance(value, str):
+        try:
+            if parse_value(value) == value:
+                return value
+        except YAMLError:
+            pass
+    return json.dumps(value, default=str)
+
+
+def command_args(plan, index=None, absolute=False, repair=False):
+    paths = [os.path.abspath(path) if absolute else path for path in plan.paths] or ["CONFIG"]
+    args = ["sweep", *paths]
+    for path, value in plan.sets:
+        args += ["--set", f"{path}={value_text(value)}"]
+    for name, value in plan.params:
+        args += ["-p", f"{name}={value_text(value)}"]
+    args += ["--record", os.path.abspath(plan.root) if absolute else plan.root]
+    if index is not None:
+        args += ["--point", str(index)]
+    if repair:
+        args.append("--repair")
+    args.append("--quiet")
+    return args
+
+
+def script_text(plan):
+    indices = [None] if plan.adaptive else list(range(plan.count))
+    lines = [shlex.join(["kalfa", *command_args(plan, index, absolute=True)]) for index in indices]
+    if not plan.paths:
+        return "\n".join(["#!/usr/bin/env bash", MAPPING_NOTE, *(f"# {line}" for line in lines)]) + "\n"
+    repair = shlex.join(["kalfa", *command_args(plan, absolute=True, repair=True)])
+    return "\n".join(["#!/usr/bin/env bash", "set -uo pipefail", "status=0", *(f"{line} || status=1" for line in lines),
+                      "# a point that fails leaves the others running; this reruns the failed and missing points:",
+                      f"# {repair}", 'exit "$status"']) + "\n"
+
+
+def condor_arg(text):
+    text = str(text).replace('"', '""')
+    if any(character.isspace() or character == "'" for character in text):
+        return "'" + text.replace("'", "''") + "'"
+    return text
+
+
+def executable():
+    found = shutil.which("kalfa")
+    return (found, []) if found else (sys.executable, ["-m", "kalfa.cli"])
+
+
+def submit_text(plan):
+    index = None if plan.adaptive else "$(Process)"
+    program, prefix = executable()
+    arguments = " ".join(condor_arg(arg) for arg in [*prefix, *command_args(plan, index, absolute=True)])
+    logs = os.path.join(os.path.abspath(plan.root), LOGS)
+    lines = [f"executable = {program}", f'arguments = "{arguments}"', "getenv = True",
+             f"output = {logs}/$(Process).out", f"error = {logs}/$(Process).err", f"log = {logs}/sweep.log",
+             f"queue {1 if plan.adaptive else plan.count}"]
+    if not plan.paths:
+        lines = [MAPPING_NOTE, *(f"# {line}" for line in lines)]
+    return "\n".join(lines) + "\n"
+
+
+def write_text(path, text):
+    with atomic(path) as temporary:
+        temporary.write_text(text, encoding="utf-8")
+
+
+def points_text(points):
+    return "".join(json.dumps({"id": index, "values": values}, default=str) + "\n"
+                   for index, values in enumerate(points or []))
+
+
+def previous_note(root):
+    try:
+        found = reader.read_json(root, SWEEP_FILE)
+    except (OSError, ValueError):
+        return {}
+    return found if isinstance(found, dict) else {}
+
+
+def write_plan(plan, log=print):
+    root = Path(plan.root)
+    root.mkdir(parents=True, exist_ok=True)
+    (root / LOGS).mkdir(exist_ok=True)
+    previous = previous_note(root)
+    note = {"format": 1, "strategy": plan.extra.get("strategy") or {"use": type(plan.strategy).__name__.lower()},
+            "space": plan.space, "objective": plan.objective, "points": plan.count}
+    if plan.adaptive:
+        note["trials"] = getattr(plan.strategy, "trials", None)
+    note["started"] = previous.get("started") or now()
+    note["config_files"] = [os.path.abspath(path) for path in plan.paths]
+    note["record"] = plan.root
+    written = [root / SWEEP_FILE, root / POINTS_FILE, root / "sweep.sh", root / "sweep.sub"]
+    write_text(written[0], json.dumps(note, indent=1, default=str) + "\n")
+    if not plan.adaptive or not written[1].exists():
+        write_text(written[1], points_text(plan.points))
+    write_text(written[2], script_text(plan))
+    written[2].chmod(0o755)
+    write_text(written[3], submit_text(plan))
+    size = f"{plan.count} points" if not plan.adaptive else f"adaptive, {note['trials']} trials"
+    log(f"{plan.root}: {size}; " + ", ".join(path.name for path in written))
+    return [str(path) for path in written]

kalfa/commands/sweep/points.py

@@ -0,0 +1,123 @@
+import json
+import shutil
+from pathlib import Path
+
+from kalfa.commands.sweep.space import is_int
+from kalfa.core.errors import KalfaError
+from kalfa.core.record import STOP_FILE
+from kalfa.records import reader
+from kalfa.records.sweep import objective_of, point_dirs
+
+
+REPAIRED = ("failed", "stopped")
+OUTSIDE_STOPS = ("keyboard", "stop.json")
+SWEEP_FILE = "sweep.json"
+POINTS_FILE = "points.jsonl"
+LOGS = "logs"
+
+
+def point_dir(root, index):
+    return f"{root}/{index:04d}"
+
+
+def manifest_of(directory):
+    try:
+        found = reader.read_manifest(directory)
+    except (OSError, ValueError):
+        return None
+    return found if isinstance(found, dict) else None
+
+
+def stopped_by_rule(directory):
+    ends = [line for line in reader.read_lines(Path(directory) / "events.jsonl") if line.get("kind") == "run_end"]
+    stopping = ends[-1].get("stopping") if ends else None
+    return isinstance(stopping, str) and not stopping.startswith(OUTSIDE_STOPS)
+
+
+def complete(directory, manifest=None):
+    manifest = manifest if manifest is not None else manifest_of(directory)
+    if manifest is None:
+        return False
+    status = manifest.get("status")
+    return status == "done" or (status == "stopped" and stopped_by_rule(directory))
+
+
+def abandoned(directory, manifest):
+    return reader.state_of(manifest) == "stale" and (Path(directory) / STOP_FILE).is_file()
+
+
+def repairable(directory):
+    manifest = manifest_of(directory)
+    if manifest is None:
+        return True
+    if manifest.get("status") == "running":
+        return abandoned(directory, manifest)
+    return manifest.get("status") in REPAIRED and not complete(directory, manifest)
+
+
+def clear_point(directory, index):
+    if not directory.is_dir() or not any(directory.iterdir()):
+        return
+    manifest = manifest_of(directory)
+    if manifest is not None and complete(directory, manifest):
+        raise KalfaError(f"point {index} is {manifest.get('status')} ({directory}); --repair runs only the failed "
+                         f"and missing points")
+    if manifest is not None and manifest.get("status") == "running" and not abandoned(directory, manifest):
+        if reader.state_of(manifest) == "stale":
+            raise KalfaError(f"point {index} was running and its heartbeat is stale ({directory}); if its process is "
+                             f"dead, kalfa stop {directory} marks it and --repair runs it again, or remove the "
+                             f"directory")
+        raise KalfaError(f"point {index} is running ({directory}); kalfa stop {directory} asks it to stop after its "
+                         f"turn")
+    shutil.rmtree(directory)
+
+
+def recorded_values(plan):
+    found = {}
+    for directory in point_dirs(plan.root):
+        manifest = manifest_of(directory)
+        sweep = manifest.get("sweep") if manifest is not None else None
+        if isinstance(sweep, dict) and isinstance(sweep.get("values"), dict):
+            found[int(directory.name)] = dict(sweep["values"])
+    for line in reader.read_lines(Path(plan.root) / POINTS_FILE):
+        if is_int(line.get("id")) and isinstance(line.get("values"), dict):
+            found[line["id"]] = dict(line["values"])
+    return found
+
+
+def planned_values(plan, index):
+    if plan.points is not None:
+        if not 0 <= index < len(plan.points):
+            raise KalfaError(f"point {index} is not in the plan of {plan.root}, which has {len(plan.points)} points")
+        return plan.points[index]
+    found = recorded_values(plan).get(index)
+    if found is None:
+        raise KalfaError(f"{plan.root}: the strategy is adaptive and point {index} was never asked; run the sweep "
+                         f"without --point")
+    return found
+
+
+def next_index(plan):
+    known = [int(directory.name) for directory in point_dirs(plan.root)]
+    known += [line["id"] for line in reader.read_lines(Path(plan.root) / POINTS_FILE) if is_int(line.get("id"))]
+    return max(known) + 1 if known else 0
+
+
+def append_point(root, index, values):
+    with open(Path(root) / POINTS_FILE, "a", encoding="utf-8") as handle:
+        handle.write(json.dumps({"id": index, "values": values}, default=str) + "\n")
+
+
+def recorded_points(plan):
+    points = []
+    for index, values in sorted(recorded_values(plan).items()):
+        directory = Path(point_dir(plan.root, index))
+        summary = reader.summary_of(directory) if directory.is_dir() else None
+        failed = not isinstance(summary, dict) or summary.get("status") == "failed"
+        points.append((values, None if failed else objective_of(summary, plan.objective)))
+    return points
+
+
+def failed_points(plan):
+    indices = range(len(plan.points)) if plan.points is not None else sorted(recorded_values(plan))
+    return [index for index in indices if repairable(Path(point_dir(plan.root, index)))]

kalfa/commands/sweep/runner.py

@@ -0,0 +1,100 @@
+import logging
+import subprocess
+import sys
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+from kalfa.commands.sweep.plan import command_args, write_plan
+from kalfa.commands.sweep.points import (
+    LOGS,
+    OUTSIDE_STOPS,
+    SWEEP_FILE,
+    clear_point,
+    manifest_of,
+    planned_values,
+    point_dir,
+    recorded_points,
+    stopped_by_rule,
+)
+from kalfa.commands.sweep.space import is_int
+from kalfa.core import run as running
+from kalfa.records import reader
+
+
+LOGGER = logging.getLogger("kalfa.commands")
+
+
+def silent(text):
+    return None
+
+
+def say_of(monitor):
+    say = getattr(monitor, "say", None)
+    return say if callable(say) else silent
+
+
+def run_point(plan, index, values=None, monitor=None):
+    index = int(index)
+    if values is None:
+        values = planned_values(plan, index)
+    if not (Path(plan.root) / SWEEP_FILE).exists():
+        write_plan(plan, log=say_of(monitor))
+    directory = Path(point_dir(plan.root, index))
+    clear_point(directory, index)
+    return running.run(paths=plan.paths, sets=[*plan.sets, ("sweep.record", plan.root)],
+                       params=[*plan.params, *values.items()], mappings=plan.extra.get("mappings", ()),
+                       record=str(directory), monitor=monitor,
+                       sweep={"root": plan.root, "id": index, "values": values})
+
+
+def values_text(values):
+    return "  ".join(f"{key}={value}" for key, value in values.items())
+
+
+def interrupted(result):
+    stopping = getattr(getattr(result, "context", None), "stopping", None)
+    return isinstance(stopping, str) and stopping.startswith(OUTSIDE_STOPS)
+
+
+def child(plan, index, logs, halt):
+    if halt.is_set():
+        return index, None
+    command = [sys.executable, "-m", "kalfa.cli", *command_args(plan, index)]
+    with open(logs / f"{index:04d}.log", "w", encoding="utf-8") as out:
+        code = subprocess.run(command, stdout=out, stderr=subprocess.STDOUT, check=False).returncode
+    directory = Path(point_dir(plan.root, index))
+    manifest = manifest_of(directory)
+    if code == 0 and manifest is not None and manifest.get("status") == "stopped" and not stopped_by_rule(directory):
+        halt.set()
+    return index, code
+
+
+def in_children(plan, indices, jobs, log):
+    logs = Path(plan.root) / LOGS
+    logs.mkdir(parents=True, exist_ok=True)
+    halt = threading.Event()
+    results, skipped = [], 0
+    with ThreadPoolExecutor(max_workers=jobs) as pool:
+        for index, code in pool.map(lambda index: child(plan, index, logs, halt), indices):
+            directory = point_dir(plan.root, index)
+            output = logs / f"{index:04d}.log"
+            if code is None:
+                skipped += 1
+            elif code == 0:
+                results.append(running.Result(record=directory, history=reader.read_history(directory), context=None))
+            else:
+                LOGGER.warning("point %04d failed (exit %s); its output is in %s", index, code, output)
+                results.append(None)
+    if skipped:
+        log(f"a point was stopped from outside; {skipped} points did not start and --repair runs them")
+    return results
+
+
+def resumed(plan):
+    resume = getattr(plan.strategy, "resume", None)
+    if not callable(resume):
+        return None
+    points = recorded_points(plan)
+    told = resume(plan.space, plan.objective, points)
+    return told if is_int(told) else len(points)

kalfa/commands/sweep/space.py

@@ -0,0 +1,67 @@
+import copy
+import math
+
+import numpy
+
+from kalfa.core.errors import BuildError
+from kalfa.records import reader
+
+
+RANGES = ("log", "float", "int")
+SPACE_HINT = "a param maps to a list of values or to {log: [a, b]}, {float: [a, b]} or {int: [a, b]}"
+
+
+def is_int(value):
+    return isinstance(value, int) and not isinstance(value, bool)
+
+
+def spread(kind, bounds, steps):
+    low, high = bounds
+    grid = numpy.geomspace(low, high, steps) if kind == "log" else numpy.linspace(low, high, steps)
+    if kind == "int":
+        return list(dict.fromkeys(round(float(value)) for value in grid))
+    return [float(f"{float(value):.12g}") for value in grid]
+
+
+def draw(generator, kind, values):
+    if kind == "list":
+        return copy.deepcopy(values[int(generator.integers(len(values)))])
+    low, high = values
+    if kind == "int":
+        return int(generator.integers(low, high + 1))
+    if kind == "log":
+        return float(min(max(math.exp(generator.uniform(math.log(low), math.log(high))), low), high))
+    return float(generator.uniform(low, high))
+
+
+def space_values(space):
+    if not isinstance(space, dict) or not space:
+        raise BuildError(f"sweep.space must be a mapping of params to values; {SPACE_HINT}")
+    return {str(name): dimension(name, entry) for name, entry in space.items()}
+
+
+def dimension(name, entry):
+    if isinstance(entry, (list, tuple)):
+        if not entry:
+            raise BuildError(f"sweep.space.{name}: the list of values is empty")
+        return "list", list(entry)
+    if isinstance(entry, dict) and len(entry) == 1:
+        kind, bounds = next(iter(entry.items()))
+        if kind in RANGES:
+            return kind, bounds_of(name, kind, bounds)
+    raise BuildError(f"sweep.space.{name}: {entry!r} is neither a list of values nor a range; {SPACE_HINT}")
+
+
+def bounds_of(name, kind, bounds):
+    if not isinstance(bounds, (list, tuple)) or len(bounds) != 2 or not all(map(reader.is_finite, bounds)):
+        raise BuildError(f"sweep.space.{name}: {kind} takes two numbers [low, high], got {bounds!r}")
+    low, high = bounds
+    if kind == "int" and not (is_int(low) and is_int(high)):
+        raise BuildError(f"sweep.space.{name}: int takes two whole numbers [low, high], got {list(bounds)!r}")
+    if low > high:
+        raise BuildError(f"sweep.space.{name}: {kind} [{low}, {high}] has its low end above its high end")
+    if kind == "log" and low <= 0:
+        raise BuildError(f"sweep.space.{name}: log [{low}, {high}] needs a low end above zero")
+    if kind == "int":
+        return [int(low), int(high)]
+    return [float(low), float(high)]