rulesmith 0.1.0__py3-none-any.whl

rulesmith/__init__.py

@@ -0,0 +1 @@
+"""Typed decision graphs, answered by a judge and searched with GEPA."""

rulesmith/ablate.py

@@ -0,0 +1,99 @@
+"""What each input is worth: switch one off and count the decisions that change.
+
+A reading that changes nothing is either a path the state never had, a rule an earlier one always
+beats to the decision, or a threshold real data never trips. All three look identical from a score
+-- the graph runs, decides something, and scores like a graph that simply decides badly.
+"""
+
+from dataclasses import dataclass
+
+from rulesmith.graph import Call, Plan, Read, State, Sum
+from rulesmith.rules import named_condition
+from rulesmith.runtime import (
+    MISSING,
+    DecisionProgram,
+    ExecutionConfig,
+    Models,
+    Remembered,
+    read_path,
+)
+
+
+@dataclass(frozen=True)
+class Effect:
+    """One input's share of the decisions it changes when it is taken away."""
+
+    node: str
+    kind: str
+    changed: int
+    decided: int
+    reason: str = ""
+
+    @property
+    def share(self) -> float:
+        return self.changed / self.decided if self.decided else 0.0
+
+
+def decisions(program: DecisionProgram, states: list[State]) -> list[dict | None]:
+    """Every output the graph gives each state, or None where it cannot decide at all. A graph
+    may decide more than a label, and a change to any output is a changed decision."""
+    decided = []
+    for state in states:
+        prediction = program(state=state)
+        decided.append(None if prediction.undecided else prediction.outputs)
+    return decided
+
+
+def effects(
+    plan: Plan,
+    states: list[State],
+    client=None,
+    model: str = "none",
+    execution: ExecutionConfig | None = None,
+    models: Models | None = None,
+) -> list[Effect]:
+    """Every read, judgment and name, ranked by how much the graph would change without it.
+
+    The judge is asked once per state for the baseline and never again: taking away a reading
+    cannot change what a judge said about the same state, so the replays reuse those answers and
+    a graph full of questions ablates for free."""
+    inputs = {}
+    for name, node in plan.nodes.items():
+        if isinstance(node, (Read, Call)):
+            inputs[name] = "read" if isinstance(node, Read) else "ask"
+        elif isinstance(node, Sum) or named_condition(node):
+            inputs[name] = "let"
+    judge = Remembered(client) if client is not None else None
+    models = (models or Models()).remembered()
+    execution = execution or ExecutionConfig()
+    baseline = decisions(DecisionProgram(plan, judge, model, execution, models=models), states)
+    found = []
+    for name, kind in inputs.items():
+        without = execution.model_copy(update={"suppress": (name,)})
+        suppressed = DecisionProgram(plan, judge, model, without, models=models)
+        changed = sum(
+            mine != theirs
+            for mine, theirs in zip(baseline, decisions(suppressed, states), strict=True)
+        )
+        found.append(
+            Effect(
+                node=name,
+                kind=kind,
+                changed=changed,
+                decided=len(states),
+                reason=blank(plan, name, states) if not changed else "",
+            )
+        )
+    return sorted(found, key=lambda effect: (-effect.changed, effect.node))
+
+
+def blank(plan: Plan, name: str, states: list[State]) -> str:
+    """Why an input changed nothing, when the graph can tell. A read whose path is not in the
+    state is a different problem from a rule that never wins, and only one of them is a typo."""
+    node = plan.nodes[name]
+    if not isinstance(node, Read):
+        return "the graph decides the same without it"
+    present = sum(read_path(node, state) is not MISSING for state in states)
+    if not present:
+        return "this path is never in the state"
+    return f"present in {present} of {len(states)} states and changes no decision"

rulesmith/arena.py

@@ -0,0 +1,172 @@
+"""Rank decision graphs by Elo from round-robin deathmatch duels."""
+
+import math
+import socket
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, replace
+from itertools import combinations
+from statistics import fmean
+
+from rulesmith.doom import DEATHMATCH, DoomEnvironment, DoomTask, Episode, Episodes
+from rulesmith.optimize import Outcome
+from rulesmith.runtime import DecisionProgram
+
+# Elo's conventions: 400 points is tenfold odds, and the field averages 1500.
+ELO_SCALE = 400
+ELO_MEAN = 1500
+CONVERGED = 1e-10
+
+
+@dataclass(frozen=True)
+class Entrant:
+    name: str
+    program: DecisionProgram
+
+
+def free_port() -> int:
+    """A UDP port nothing is using, for one game's host to listen on."""
+    with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as probe:
+        probe.bind(("127.0.0.1", 0))
+        return probe.getsockname()[1]
+
+
+def play(
+    task: DoomTask,
+    program: DecisionProgram,
+    episode: int | Episode,
+    game_args: str = "",
+    frames: list | None = None,
+) -> Outcome:
+    """One seat's episode. A seat whose graph fails leaves the game, and its opponent plays on.
+    A frames list, when given, collects every tic this seat rendered."""
+    with DoomEnvironment(task, game_args=game_args, record=frames is not None) as env:
+        try:
+            return Episodes(task, env)(program, episode)
+        finally:
+            if frames is not None:
+                frames += env.frames
+
+
+def duel(
+    task: DoomTask,
+    a: DecisionProgram,
+    b: DecisionProgram,
+    episode: int | Episode,
+    host_first: bool,
+    frames: list | None = None,
+) -> list[Outcome]:
+    """Both seats' episodes of one deathmatch, a's first, each graph in its own networked game.
+    A frames list, when given, collects the tics a's seat rendered."""
+    port = free_port()
+    order = [a, b] if host_first else [b, a]
+    args = [f"-host 2 -port {port} {DEATHMATCH}", f"-join 127.0.0.1 -port {port}"]
+    seats = [frames if program is a else None for program in order]
+    # The host waits at init until its opponent joins, so both seats start together.
+    with ThreadPoolExecutor(2) as pool:
+        futures = [
+            pool.submit(play, task, program, episode, arg, seat)
+            for program, arg, seat in zip(order, args, seats, strict=True)
+        ]
+    seats = [future.result() for future in futures]
+    return seats if host_first else seats[::-1]
+
+
+class Duels:
+    """Scores a graph by its reward in a deathmatch against a fixed opponent per episode."""
+
+    def __init__(self, task: DoomTask, opponent: DecisionProgram):
+        self.task = task
+        self.opponent = opponent
+
+    def __call__(self, program: DecisionProgram, episode: int | Episode) -> Outcome:
+        # Alternating hosts by seed shares the joiner's latency handicap across episodes.
+        hosted = self.task.episode(episode).seed % 2 == 0
+        mine, theirs = duel(self.task, program, self.opponent, episode, hosted)
+        own, rival = mine.record["stats"]["frags"], theirs.record["stats"]["frags"]
+        result = "won" if own > rival else "lost" if own < rival else "drew"
+        feedback = f"{mine.trace['Feedback']} The opponent scored {rival} frags, so you {result}."
+        return replace(
+            mine,
+            record=mine.record | {"hosted": hosted, "opponent_frags": rival, "result": result},
+            trace=mine.trace | {"Feedback": feedback},
+        )
+
+
+def match(task: DoomTask, a: Entrant, b: Entrant, episode: int | Episode, host_first: bool) -> dict:
+    """One deathmatch between two entrants, scored for the tournament."""
+    seats = duel(task, a.program, b.program, episode, host_first)
+    frags = [seat.record["stats"]["frags"] for seat in seats]
+    failed = [seat.error is not None for seat in seats]
+    if failed[0] != failed[1]:
+        score = [float(failed[1]), float(failed[0])]
+    elif frags[0] == frags[1]:
+        score = [0.5, 0.5]
+    else:
+        score = [float(frags[0] > frags[1]), float(frags[1] > frags[0])]
+    episode = task.episode(episode)
+    return {
+        "players": [a.name, b.name],
+        "seed": episode.seed,
+        "map": episode.map,
+        "host": (a if host_first else b).name,
+        "frags": frags,
+        "score": score,
+        "seats": [seat.record for seat in seats],
+    }
+
+
+def ratings(matches: list[dict], names: list[str]) -> dict[str, float]:
+    """Bradley-Terry strengths fit to every result at once, so match order never matters,
+    reported on the Elo scale. Each pair also shares one virtual draw, which keeps an unbeaten
+    player's rating finite."""
+    points = {(x, y): 0.5 for x in names for y in names if x != y}
+    for record in matches:
+        (x, y), (earned_x, earned_y) = record["players"], record["score"]
+        points[x, y] += earned_x
+        points[y, x] += earned_y
+    strength = dict.fromkeys(names, 1.0)
+    while True:
+        # Hunter's minorization-maximization update, which converges to the maximum likelihood.
+        updated = {
+            x: sum(points[x, y] for y in names if y != x)
+            / sum(
+                (points[x, y] + points[y, x]) / (strength[x] + strength[y]) for y in names if y != x
+            )
+            for x in names
+        }
+        center = math.exp(fmean(math.log(value) for value in updated.values()))
+        updated = {x: value / center for x, value in updated.items()}
+        done = max(abs(updated[x] - strength[x]) for x in names) < CONVERGED
+        strength = updated
+        if done:
+            break
+    return {x: ELO_MEAN + ELO_SCALE * math.log10(strength[x]) for x in names}
+
+
+def tournament(task: DoomTask, entrants: list[Entrant], episodes: list[int | Episode]) -> dict:
+    """Every pair of entrants plays every episode, swapping hosts from one episode to the next."""
+    matches = []
+    for a, b in combinations(entrants, 2):
+        for index, episode in enumerate(episodes):
+            matches.append(match(task, a, b, episode, host_first=index % 2 == 0))
+    names = [entrant.name for entrant in entrants]
+    rated = ratings(matches, names)
+    records = {
+        name: {"wins": 0, "draws": 0, "losses": 0, "frags": 0, "frags_against": 0} for name in names
+    }
+    for record in matches:
+        for side, name in enumerate(record["players"]):
+            earned, own, rival = (
+                record["score"][side],
+                record["frags"][side],
+                record["frags"][1 - side],
+            )
+            records[name]["wins" if earned == 1 else "draws" if earned == 0.5 else "losses"] += 1
+            records[name]["frags"] += own
+            records[name]["frags_against"] += rival
+    standings = sorted(names, key=rated.get, reverse=True)
+    return {
+        "ratings": {name: round(rated[name], 1) for name in standings},
+        "records": {name: records[name] for name in standings},
+        "matches": matches,
+    }