tickwork 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tickwork/__init__.py +11 -0
- tickwork/__main__.py +8 -0
- tickwork/cli.py +288 -0
- tickwork/effect_log.py +319 -0
- tickwork/pool.py +214 -0
- tickwork/providers/__init__.py +10 -0
- tickwork/providers/openai_compat.py +82 -0
- tickwork/replay.py +134 -0
- tickwork/workflow_states.py +226 -0
- tickwork-0.3.0.data/data/share/tickwork/examples/items.jsonl +3 -0
- tickwork-0.3.0.data/data/share/tickwork/examples/logins.yaml +7 -0
- tickwork-0.3.0.data/data/share/tickwork/examples/pr-loop.states.yaml +81 -0
- tickwork-0.3.0.dist-info/METADATA +9 -0
- tickwork-0.3.0.dist-info/RECORD +18 -0
- tickwork-0.3.0.dist-info/WHEEL +5 -0
- tickwork-0.3.0.dist-info/entry_points.txt +2 -0
- tickwork-0.3.0.dist-info/licenses/LICENSE +21 -0
- tickwork-0.3.0.dist-info/top_level.txt +1 -0
tickwork/pool.py
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
"""Role-keyed flock claim and worker-slot primitives.
|
|
2
|
+
|
|
3
|
+
The concurrency primitives a worker pool needs, built on ``fcntl.flock`` with
|
|
4
|
+
non-blocking acquisition: no waiting, no daemon, and no explicit cleanup (the
|
|
5
|
+
lock is released when the holding process exits or crashes).
|
|
6
|
+
|
|
7
|
+
- ``try_claim(role, state_dir, kind, repo, number)`` — claim exclusive
|
|
8
|
+
ownership of an item (e.g. ``kind="issue"`` or ``"pr"``) so two concurrent
|
|
9
|
+
ticks of *role* never work the same item.
|
|
10
|
+
- ``acquire_worker_slot(role, state_dir, pool_size)`` — claim the
|
|
11
|
+
lowest-numbered free slot in ``1..pool_size`` for per-tick disambiguation
|
|
12
|
+
(branch naming, local state).
|
|
13
|
+
- ``lock_worker_slot(role, state_dir, slot)`` — claim a *specific* slot's
|
|
14
|
+
lease (no free-slot scan), for when the slot is pinned from outside the
|
|
15
|
+
process.
|
|
16
|
+
- ``flock_nonblock(path)`` — open *path* and take a non-blocking exclusive
|
|
17
|
+
``flock`` on it; returns the file descriptor, or ``None`` when another
|
|
18
|
+
holder has the lock. Public so a caller can hold an extra lock file
|
|
19
|
+
next to the generic ones.
|
|
20
|
+
|
|
21
|
+
Lock files live under ``<state_dir>/claims/<role>/<kind>-<owner>_<repo>-<n>.lock``
|
|
22
|
+
and ``<state_dir>/slots/<role>/<slot>.lock``. The state directory is
|
|
23
|
+
passed in by the caller; this module reads no environment variables, so it
|
|
24
|
+
names nothing of the instance it runs for.
|
|
25
|
+
|
|
26
|
+
stdlib-only — no new dependencies. ``fcntl.flock`` is POSIX-only; the module
|
|
27
|
+
will fail at import on non-POSIX platforms (consistent with the repo's
|
|
28
|
+
single-host Linux deploy model).
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import contextlib
|
|
34
|
+
import fcntl
|
|
35
|
+
import os
|
|
36
|
+
from collections.abc import Generator
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _item_lock_path(role: str, state_dir: str, kind: str, repo: str | None, number: int) -> str:
|
|
40
|
+
"""Full path to the lock file for one item of *kind* held by *role*.
|
|
41
|
+
|
|
42
|
+
With *repo*, the path is keyed by repo too, so the same item number in
|
|
43
|
+
two repos doesn't share one lock. ``owner/name`` is flattened into one
|
|
44
|
+
path component (``owner_name``).
|
|
45
|
+
"""
|
|
46
|
+
claims_dir = os.path.join(state_dir, "claims", role)
|
|
47
|
+
if repo:
|
|
48
|
+
return os.path.join(claims_dir, f"{kind}-{repo.replace('/', '_')}-{number}.lock")
|
|
49
|
+
return os.path.join(claims_dir, f"{kind}-{number}.lock")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _slot_lock_path(role: str, state_dir: str, slot: int) -> str:
|
|
53
|
+
"""Full path to the lock file for a worker slot (1-indexed)."""
|
|
54
|
+
return os.path.join(state_dir, "slots", role, f"{slot}.lock")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
# ─── helpers ─────────────────────────────────────────────────────────────────
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _ensure_dir(path: str) -> None:
|
|
61
|
+
"""Create *path* (and parents) if absent. Silent on success; raises on
|
|
62
|
+
permission errors so the caller can surface the failure."""
|
|
63
|
+
os.makedirs(path, exist_ok=True)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def flock_nonblock(path: str) -> int | None:
|
|
67
|
+
"""Open *path* for writing (creating it if needed) and try a non-blocking
|
|
68
|
+
exclusive ``flock``. Returns the file descriptor on success, or ``None``
|
|
69
|
+
when another process already holds the lock (``EWOULDBLOCK``).
|
|
70
|
+
|
|
71
|
+
The lock file is created with ``os.O_CREAT`` so it persists on disk even
|
|
72
|
+
after the lock is released — this is intentional: ``flock`` is advisory
|
|
73
|
+
and tied to the file descriptor, so the file's existence on disk is
|
|
74
|
+
harmless and re-using the same path is critical for cross-process
|
|
75
|
+
coordination.
|
|
76
|
+
|
|
77
|
+
On an unrecoverable OS error (e.g. ``PermissionError``, ``IOError`` from
|
|
78
|
+
a full filesystem) the exception propagates so the caller can escalate
|
|
79
|
+
rather than silently proceeding without a lock.
|
|
80
|
+
"""
|
|
81
|
+
try:
|
|
82
|
+
fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0o644)
|
|
83
|
+
except OSError as e:
|
|
84
|
+
# The directory doesn't exist yet — create it and retry once.
|
|
85
|
+
if e.errno == 2: # ENOENT
|
|
86
|
+
_ensure_dir(os.path.dirname(path))
|
|
87
|
+
fd = os.open(path, os.O_WRONLY | os.O_CREAT, 0o644)
|
|
88
|
+
else:
|
|
89
|
+
raise
|
|
90
|
+
try:
|
|
91
|
+
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
92
|
+
except OSError as e:
|
|
93
|
+
os.close(fd)
|
|
94
|
+
if e.errno in (11, 35): # EAGAIN / EWOULDBLOCK
|
|
95
|
+
return None
|
|
96
|
+
raise
|
|
97
|
+
return fd
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@contextlib.contextmanager
|
|
101
|
+
def _lock_holder(fd: int) -> Generator[None, None, None]:
|
|
102
|
+
"""Context manager that holds a ``flock``-ed file descriptor for the
|
|
103
|
+
duration of the ``with`` block, then releases it.
|
|
104
|
+
|
|
105
|
+
The lock is released either by an explicit ``os.close`` (the ``finally``
|
|
106
|
+
block), or automatically when the process exits — the kernel releases all
|
|
107
|
+
file-descriptor locks on process termination, so a crashed/killed process
|
|
108
|
+
never leaves a stale lock behind.
|
|
109
|
+
"""
|
|
110
|
+
try:
|
|
111
|
+
yield
|
|
112
|
+
finally:
|
|
113
|
+
try:
|
|
114
|
+
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
115
|
+
except OSError:
|
|
116
|
+
pass # best-effort; a crash during yield may leave fd stale, but
|
|
117
|
+
# the kernel will clean up on process exit.
|
|
118
|
+
try:
|
|
119
|
+
os.close(fd)
|
|
120
|
+
except OSError:
|
|
121
|
+
pass
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@contextlib.contextmanager
|
|
125
|
+
def _slot_id_holder(fd: int, slot: int) -> Generator[int, None, None]:
|
|
126
|
+
"""Like ``_lock_holder`` but also yields the *slot* integer to the ``with``
|
|
127
|
+
target so ``acquire_worker_slot`` and ``lock_worker_slot`` can be used as::
|
|
128
|
+
|
|
129
|
+
with acquire_worker_slot(role, state, 3) as slot:
|
|
130
|
+
print(slot) # 1, 2, or 3
|
|
131
|
+
"""
|
|
132
|
+
try:
|
|
133
|
+
yield slot
|
|
134
|
+
finally:
|
|
135
|
+
try:
|
|
136
|
+
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
137
|
+
except OSError:
|
|
138
|
+
pass
|
|
139
|
+
try:
|
|
140
|
+
os.close(fd)
|
|
141
|
+
except OSError:
|
|
142
|
+
pass
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
# ─── public API ──────────────────────────────────────────────────────────────
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def try_claim(
|
|
149
|
+
role: str, state_dir: str, kind: str, repo: str | None, number: int
|
|
150
|
+
) -> contextlib.AbstractContextManager | None:
|
|
151
|
+
"""Attempt a non-blocking claim by *role* on item *number* of *kind*.
|
|
152
|
+
|
|
153
|
+
*kind* selects the lock-file prefix (``"issue"``, ``"pr"``, …); each kind
|
|
154
|
+
gets its own lock, so a PR 7 and an issue 7 don't collide. *repo* keys
|
|
155
|
+
the lock by repository when given.
|
|
156
|
+
|
|
157
|
+
Returns a context manager that holds the exclusive lock for the lifetime
|
|
158
|
+
of the ``with`` block, or ``None`` if another process already holds the
|
|
159
|
+
claim. The lock is released on context-manager exit OR on process
|
|
160
|
+
exit/crash (kernel-managed, no cleanup needed).
|
|
161
|
+
|
|
162
|
+
Usage::
|
|
163
|
+
|
|
164
|
+
claim = try_claim(role, state_dir, "issue", repo, 42)
|
|
165
|
+
if claim is None:
|
|
166
|
+
log("issue 42 already claimed by another tick")
|
|
167
|
+
return
|
|
168
|
+
with claim:
|
|
169
|
+
... # compose the work
|
|
170
|
+
"""
|
|
171
|
+
fd = flock_nonblock(_item_lock_path(role, state_dir, kind, repo, number))
|
|
172
|
+
if fd is None:
|
|
173
|
+
return None
|
|
174
|
+
return _lock_holder(fd)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def lock_worker_slot(role: str, state_dir: str, slot: int) -> contextlib.AbstractContextManager | None:
|
|
178
|
+
"""Claim the flock lease on *specifically* slot *slot* — no free-slot scan.
|
|
179
|
+
|
|
180
|
+
Companion to :func:`acquire_worker_slot` for the pinned-slot case: when
|
|
181
|
+
the slot is set from outside the process, the lease must still be *taken*
|
|
182
|
+
so concurrent ticks on the same slot collide instead of silently running
|
|
183
|
+
double — but which slot it is has already been decided, so there is
|
|
184
|
+
nothing to scan for.
|
|
185
|
+
|
|
186
|
+
Returns a context manager yielding *slot*, or ``None`` if the slot is
|
|
187
|
+
already held by another worker (caller should skip the tick, same contract
|
|
188
|
+
as a full :func:`acquire_worker_slot` scan).
|
|
189
|
+
"""
|
|
190
|
+
if slot is None or slot < 1:
|
|
191
|
+
return None
|
|
192
|
+
fd = flock_nonblock(_slot_lock_path(role, state_dir, slot))
|
|
193
|
+
if fd is None:
|
|
194
|
+
return None
|
|
195
|
+
return _slot_id_holder(fd, slot)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def acquire_worker_slot(
|
|
199
|
+
role: str, state_dir: str, pool_size: int
|
|
200
|
+
) -> contextlib.AbstractContextManager | None:
|
|
201
|
+
"""Claim the lowest-numbered free worker slot in ``1..pool_size`` for *role*.
|
|
202
|
+
|
|
203
|
+
Returns a context manager that yields the slot number (``int``) for the
|
|
204
|
+
duration of the ``with`` block, or ``None`` if every slot is taken (or
|
|
205
|
+
``pool_size < 1``). When all slots are occupied, returns ``None``
|
|
206
|
+
immediately — the caller should skip the tick rather than waiting.
|
|
207
|
+
"""
|
|
208
|
+
if pool_size < 1:
|
|
209
|
+
return None
|
|
210
|
+
for slot in range(1, pool_size + 1):
|
|
211
|
+
fd = flock_nonblock(_slot_lock_path(role, state_dir, slot))
|
|
212
|
+
if fd is not None:
|
|
213
|
+
return _slot_id_holder(fd, slot)
|
|
214
|
+
return None
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""Provider implementations for :mod:`tickwork.replay`.
|
|
2
|
+
|
|
3
|
+
Each provider is a stdlib-only adapter speaking one API shape, imported
|
|
4
|
+
here so a CLI can map a ``--provider <name>`` flag to an adapter without
|
|
5
|
+
the replay core knowing about transports.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from tickwork.providers.openai_compat import OpenAICompatProvider
|
|
9
|
+
|
|
10
|
+
__all__ = ["OpenAICompatProvider"]
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""A stdlib-only :class:`~tickwork.replay.Provider` for OpenAI-compatible APIs.
|
|
2
|
+
|
|
3
|
+
Speaks any ``/chat/completions`` endpoint: the base URL and the name of the
|
|
4
|
+
API-key environment variable are constructor parameters, so nothing about a
|
|
5
|
+
deployment is baked in. The key is read from the environment per call and
|
|
6
|
+
used only in the request's ``Authorization`` header — it is never logged,
|
|
7
|
+
printed, or included in an error string.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import time
|
|
15
|
+
import urllib.request
|
|
16
|
+
from typing import Any
|
|
17
|
+
|
|
18
|
+
from tickwork.replay import Reply
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class OpenAICompatProvider:
|
|
22
|
+
"""Provider for any OpenAI-compatible ``/chat/completions`` endpoint."""
|
|
23
|
+
|
|
24
|
+
def __init__(self, base_url: str, key_env: str = "OPENAI_API_KEY", timeout: float = 120.0):
|
|
25
|
+
self.base_url = base_url.rstrip("/")
|
|
26
|
+
self.key_env = key_env
|
|
27
|
+
self.timeout = timeout
|
|
28
|
+
|
|
29
|
+
def generate(self, model: str, messages: list[dict]) -> Reply:
|
|
30
|
+
"""Send *messages* to *model*; failures come back as a Reply."""
|
|
31
|
+
start = time.monotonic()
|
|
32
|
+
response, usage, finish_reason, error = self._chat(model, messages)
|
|
33
|
+
return {
|
|
34
|
+
"response": response,
|
|
35
|
+
"ms": int((time.monotonic() - start) * 1000),
|
|
36
|
+
"prompt_tokens": _usage_int(usage, "prompt_tokens"),
|
|
37
|
+
"completion_tokens": _usage_int(usage, "completion_tokens"),
|
|
38
|
+
"reasoning_tokens": _usage_int(usage.get("completion_tokens_details") or {}, "reasoning_tokens"),
|
|
39
|
+
"cache_hit_tokens": _usage_int(usage.get("prompt_tokens_details") or {}, "cached_tokens"),
|
|
40
|
+
"finish_reason": finish_reason,
|
|
41
|
+
"error": error,
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
def _chat(self, model: str, messages: list[dict]) -> tuple[str | None, Any, str | None, str | None]:
|
|
45
|
+
"""One POST; returns (response text, usage, finish_reason, error).
|
|
46
|
+
|
|
47
|
+
The key is only ever attached as the Authorization header of the
|
|
48
|
+
request object, which is never str()'d or logged — so no failure
|
|
49
|
+
path can carry it into the returned error string.
|
|
50
|
+
"""
|
|
51
|
+
key = os.environ.get(self.key_env)
|
|
52
|
+
if not key:
|
|
53
|
+
return None, {}, None, f"environment variable {self.key_env} is not set"
|
|
54
|
+
request = urllib.request.Request(
|
|
55
|
+
f"{self.base_url}/chat/completions",
|
|
56
|
+
data=json.dumps({"model": model, "messages": messages}).encode("utf-8"),
|
|
57
|
+
headers={"Content-Type": "application/json", "Authorization": f"Bearer {key}"},
|
|
58
|
+
method="POST",
|
|
59
|
+
)
|
|
60
|
+
try:
|
|
61
|
+
with urllib.request.urlopen(request, timeout=self.timeout) as response:
|
|
62
|
+
data = json.loads(response.read().decode("utf-8"))
|
|
63
|
+
except Exception as exc: # one bad call is one errored reply, not a crash
|
|
64
|
+
return None, {}, None, f"{type(exc).__name__}: {exc}"
|
|
65
|
+
if not isinstance(data, dict):
|
|
66
|
+
return None, {}, None, f"malformed response: top level is {type(data).__name__}"
|
|
67
|
+
choices = data.get("choices")
|
|
68
|
+
if not isinstance(choices, list) or not choices or not isinstance(choices[0], dict):
|
|
69
|
+
return None, {}, None, "malformed response: no choices[0] object"
|
|
70
|
+
choice = choices[0]
|
|
71
|
+
message = choice.get("message") or {}
|
|
72
|
+
content = message.get("content") if isinstance(message, dict) else None
|
|
73
|
+
response_text = content if isinstance(content, str) else None
|
|
74
|
+
usage = data.get("usage") or {}
|
|
75
|
+
usage = usage if isinstance(usage, dict) else {}
|
|
76
|
+
return response_text, usage, choice.get("finish_reason"), None
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _usage_int(usage: dict, key: str) -> int | None:
|
|
80
|
+
"""The int value of one usage counter, or None when absent/malformed."""
|
|
81
|
+
val = usage.get(key)
|
|
82
|
+
return int(val) if isinstance(val, (int, float)) else None
|
tickwork/replay.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Generic replay of recorded LLM calls.
|
|
2
|
+
|
|
3
|
+
The instance-neutral core of the offline replay: read the final transcript
|
|
4
|
+
records (the ``final: true`` JSONL records carrying the
|
|
5
|
+
complete ``messages`` array), send each ``messages`` array to a model
|
|
6
|
+
through a caller-supplied :class:`Provider`, deep-merge a variant overlay,
|
|
7
|
+
and score every reply into one metrics dict per (record, model, repeat).
|
|
8
|
+
The module imports nothing beyond the standard library and names no
|
|
9
|
+
deployment: providers and scorers are parameters, never imports.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
from collections.abc import Iterable, Iterator, Mapping, Sequence
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any, Protocol, TypedDict
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class Reply(TypedDict):
|
|
21
|
+
"""One model reply plus the per-call facts a metrics line is built from."""
|
|
22
|
+
|
|
23
|
+
response: str | None
|
|
24
|
+
ms: int
|
|
25
|
+
prompt_tokens: int | None
|
|
26
|
+
completion_tokens: int | None
|
|
27
|
+
reasoning_tokens: int | None
|
|
28
|
+
cache_hit_tokens: int | None
|
|
29
|
+
finish_reason: str | None
|
|
30
|
+
error: str | None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class Provider(Protocol):
|
|
34
|
+
"""Anything that can send a recorded ``messages`` array to a model.
|
|
35
|
+
|
|
36
|
+
Implementations surface call failures as the ``error`` field of a
|
|
37
|
+
:class:`Reply` (never raised), so one bad call skips into the metrics
|
|
38
|
+
instead of aborting the run.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
def generate(self, model: str, messages: list[dict]) -> Reply: ...
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def read_records(paths: Iterable[str | Path]) -> Iterator[dict]:
|
|
45
|
+
"""Yield the final transcript records of *paths* (JSONL files).
|
|
46
|
+
|
|
47
|
+
A final record (the transcript writer marks it ``final: true``) carries
|
|
48
|
+
the call's complete ``messages`` array; every other line is skipped:
|
|
49
|
+
blank lines, lines JSON cannot parse, non-dict
|
|
50
|
+
lines, intermediate (non-final) records, and final records without a
|
|
51
|
+
usable ``messages`` array. Skipping, not failing: a transcript
|
|
52
|
+
legitimately mixes records, and only the finals replay.
|
|
53
|
+
"""
|
|
54
|
+
for path in paths:
|
|
55
|
+
with open(path, encoding="utf-8") as handle:
|
|
56
|
+
for line in handle:
|
|
57
|
+
line = line.strip()
|
|
58
|
+
if not line:
|
|
59
|
+
continue
|
|
60
|
+
try:
|
|
61
|
+
rec = json.loads(line)
|
|
62
|
+
except json.JSONDecodeError:
|
|
63
|
+
continue
|
|
64
|
+
if (
|
|
65
|
+
isinstance(rec, dict)
|
|
66
|
+
and rec.get("final") is True
|
|
67
|
+
and isinstance(rec.get("messages"), list)
|
|
68
|
+
and rec.get("messages")
|
|
69
|
+
):
|
|
70
|
+
yield rec
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def deep_merge(base: dict, overlay: dict) -> dict:
|
|
74
|
+
"""Recursively merge ``overlay`` into ``base`` (returns a new dict).
|
|
75
|
+
|
|
76
|
+
Nested mappings merge key by key; every other value (scalars, lists)
|
|
77
|
+
is replaced wholesale by the overlay's value.
|
|
78
|
+
"""
|
|
79
|
+
out = dict(base)
|
|
80
|
+
for key, val in overlay.items():
|
|
81
|
+
if isinstance(val, dict) and isinstance(out.get(key), dict):
|
|
82
|
+
out[key] = deep_merge(out[key], val)
|
|
83
|
+
else:
|
|
84
|
+
out[key] = val
|
|
85
|
+
return out
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def replay(
|
|
89
|
+
records: Iterable[dict],
|
|
90
|
+
provider: Provider,
|
|
91
|
+
models: Sequence[str],
|
|
92
|
+
scorers: Mapping[tuple[str, str], Any],
|
|
93
|
+
repeats: int = 1,
|
|
94
|
+
skips: dict[str, int] | None = None,
|
|
95
|
+
) -> Iterator[dict]:
|
|
96
|
+
"""Replay every (record, model, repeat) and yield one metrics dict each.
|
|
97
|
+
|
|
98
|
+
``scorers`` maps a ``(role, stage)`` pair — or the wildcard
|
|
99
|
+
``("*", "*")`` fallback — to a ``(record, reply) -> dict`` function whose
|
|
100
|
+
result is merged into that record's metrics line. A record with no
|
|
101
|
+
matching scorer is skipped and counted in ``skips`` under a readable
|
|
102
|
+
reason when the caller passes a mapping. A provider failure arrives as
|
|
103
|
+
the reply's ``error`` field and flows into the metrics line; replay
|
|
104
|
+
itself never raises for one bad call.
|
|
105
|
+
"""
|
|
106
|
+
for rec in records:
|
|
107
|
+
role, stage = rec.get("role"), rec.get("stage")
|
|
108
|
+
scorer = scorers.get((role, stage), scorers.get(("*", "*")))
|
|
109
|
+
if scorer is None:
|
|
110
|
+
if skips is not None:
|
|
111
|
+
reason = f"no scorer for {role}/{stage}"
|
|
112
|
+
skips[reason] = skips.get(reason, 0) + 1
|
|
113
|
+
continue
|
|
114
|
+
for model in models:
|
|
115
|
+
for repeat in range(max(repeats, 0)):
|
|
116
|
+
reply = provider.generate(model, rec["messages"])
|
|
117
|
+
metrics = {
|
|
118
|
+
"role": role,
|
|
119
|
+
"stage": stage,
|
|
120
|
+
"repo": rec.get("repo"),
|
|
121
|
+
"issue": rec.get("issue"),
|
|
122
|
+
"model": model,
|
|
123
|
+
"repeat": repeat,
|
|
124
|
+
"ms": reply.get("ms"),
|
|
125
|
+
"response": reply.get("response"),
|
|
126
|
+
"prompt_tokens": reply.get("prompt_tokens"),
|
|
127
|
+
"completion_tokens": reply.get("completion_tokens"),
|
|
128
|
+
"reasoning_tokens": reply.get("reasoning_tokens"),
|
|
129
|
+
"cache_hit_tokens": reply.get("cache_hit_tokens"),
|
|
130
|
+
"finish_reason": reply.get("finish_reason"),
|
|
131
|
+
"error": reply.get("error"),
|
|
132
|
+
}
|
|
133
|
+
metrics.update(scorer(rec, reply))
|
|
134
|
+
yield metrics
|
|
@@ -0,0 +1,226 @@
|
|
|
1
|
+
"""Loader and validator for a workflow states file.
|
|
2
|
+
|
|
3
|
+
A states file maps each workflow state to a predicate (``when``), a board
|
|
4
|
+
bucket (``board``) and an owner (``owner``: a role name or ``human``). This
|
|
5
|
+
module knows nothing about any instance: the caller supplies the path and the
|
|
6
|
+
set of allowed owner names. It names no role and has no default path.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
import yaml
|
|
15
|
+
|
|
16
|
+
HUMAN = "human"
|
|
17
|
+
|
|
18
|
+
# The predicate vocabulary a `when` block may use, at any nesting depth.
|
|
19
|
+
PREDICATE_KEYS = frozenset(
|
|
20
|
+
{
|
|
21
|
+
"issue.state",
|
|
22
|
+
"issue.assignee",
|
|
23
|
+
"issue.dependencies",
|
|
24
|
+
"pr",
|
|
25
|
+
"pr.state",
|
|
26
|
+
"pr.assignee",
|
|
27
|
+
"pr.requested_reviewers",
|
|
28
|
+
"pr.reviewed_on_head",
|
|
29
|
+
"pr.approved_on_head",
|
|
30
|
+
"any",
|
|
31
|
+
"not",
|
|
32
|
+
}
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
VALID_BOARDS = frozenset(
|
|
36
|
+
{
|
|
37
|
+
"queued",
|
|
38
|
+
"in_progress",
|
|
39
|
+
"waiting_on_human",
|
|
40
|
+
"blocked",
|
|
41
|
+
"done",
|
|
42
|
+
"failed",
|
|
43
|
+
"intake",
|
|
44
|
+
}
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
_STATE_FIELDS = frozenset({"when", "board", "owner", "override", "terminal"})
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def load(path: str | Path, roles: set[str]) -> dict[str, dict[str, Any]]:
|
|
51
|
+
"""Read and validate the states file at *path*.
|
|
52
|
+
|
|
53
|
+
*roles* is the set of allowed role-name owners; ``human`` is always allowed.
|
|
54
|
+
|
|
55
|
+
Returns ``{state name: {when, board, owner, override, terminal}}`` in file
|
|
56
|
+
order (``override``/``terminal`` default to ``False``; a ``terminal`` state
|
|
57
|
+
may omit ``owner``).
|
|
58
|
+
|
|
59
|
+
Raises ``ValueError`` naming the state and the offending field on any
|
|
60
|
+
violation.
|
|
61
|
+
"""
|
|
62
|
+
data = yaml.safe_load(Path(path).read_text(encoding="utf-8"))
|
|
63
|
+
if not isinstance(data, dict) or not isinstance(data.get("states"), dict):
|
|
64
|
+
raise ValueError(f"{path}: top level must be a mapping with a `states` block")
|
|
65
|
+
allowed_owners = set(roles) | {HUMAN}
|
|
66
|
+
states: dict[str, dict[str, Any]] = {}
|
|
67
|
+
for name, spec in data["states"].items():
|
|
68
|
+
if not isinstance(name, str) or not name:
|
|
69
|
+
raise ValueError(f"states: state name {name!r} must be a non-empty string")
|
|
70
|
+
if not isinstance(spec, dict):
|
|
71
|
+
raise ValueError(f"{name}: state must be a mapping, got {type(spec).__name__}")
|
|
72
|
+
unknown = sorted(set(spec) - _STATE_FIELDS)
|
|
73
|
+
if unknown:
|
|
74
|
+
raise ValueError(f"{name}: unknown field(s) {unknown}")
|
|
75
|
+
when = spec.get("when")
|
|
76
|
+
if not isinstance(when, dict) or not when:
|
|
77
|
+
raise ValueError(f"{name}: `when` must be a non-empty mapping")
|
|
78
|
+
_check_predicates(name, when)
|
|
79
|
+
board = spec.get("board")
|
|
80
|
+
if board not in VALID_BOARDS:
|
|
81
|
+
raise ValueError(f"{name}: `board` {board!r} not in {sorted(VALID_BOARDS)}")
|
|
82
|
+
for field in ("override", "terminal"):
|
|
83
|
+
value = spec.get(field, False)
|
|
84
|
+
if not isinstance(value, bool):
|
|
85
|
+
raise ValueError(f"{name}: `{field}` must be a boolean, got {value!r}")
|
|
86
|
+
owner = spec.get("owner")
|
|
87
|
+
if owner is None:
|
|
88
|
+
if not spec.get("terminal", False):
|
|
89
|
+
raise ValueError(f"{name}: missing `owner`")
|
|
90
|
+
elif owner not in allowed_owners:
|
|
91
|
+
raise ValueError(f"{name}: `owner` {owner!r} not in {sorted(allowed_owners)}")
|
|
92
|
+
states[name] = {
|
|
93
|
+
"when": when,
|
|
94
|
+
"board": board,
|
|
95
|
+
"owner": owner,
|
|
96
|
+
"override": spec.get("override", False),
|
|
97
|
+
"terminal": spec.get("terminal", False),
|
|
98
|
+
}
|
|
99
|
+
return states
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _check_predicates(state: str, pred: dict[str, Any]) -> None:
|
|
103
|
+
"""Validate predicate keys at any depth of a `when` block."""
|
|
104
|
+
for key, value in pred.items():
|
|
105
|
+
if key not in PREDICATE_KEYS:
|
|
106
|
+
raise ValueError(f"{state}: `when` has unknown predicate key {key!r}")
|
|
107
|
+
if key == "any":
|
|
108
|
+
if not isinstance(value, list) or not all(isinstance(v, dict) for v in value):
|
|
109
|
+
raise ValueError(f"{state}: `when.any` must be a list of mappings")
|
|
110
|
+
for sub in value:
|
|
111
|
+
_check_predicates(state, sub)
|
|
112
|
+
elif key == "not":
|
|
113
|
+
if not isinstance(value, dict):
|
|
114
|
+
raise ValueError(f"{state}: `when.not` must be a mapping")
|
|
115
|
+
_check_predicates(state, value)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
# ─── predicate evaluation ─────────────────────────────────────────────────────
|
|
119
|
+
|
|
120
|
+
# Predicate leaf → the field of the item dict it reads. Assignee predicates
|
|
121
|
+
# name a ROLE, or the value `none` (`issue.assignee` only) meaning an empty
|
|
122
|
+
# assignees list; role names resolve through the caller-supplied `logins`.
|
|
123
|
+
_ITEM_KEYS = {
|
|
124
|
+
"assignee": "assignees",
|
|
125
|
+
"requested_reviewers": "requested_reviewers",
|
|
126
|
+
"state": "state",
|
|
127
|
+
"dependencies": "open_dependencies",
|
|
128
|
+
"reviewed_on_head": "reviewed_on_head",
|
|
129
|
+
"approved_on_head": "approved_on_head",
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def evaluate(states: dict[str, dict[str, Any]], item: dict[str, Any], logins: dict[str, str]) -> list[str]:
|
|
134
|
+
"""Return the names of all states whose `when` predicate matches *item*.
|
|
135
|
+
|
|
136
|
+
Override states are tested first, in file order, and the first match wins
|
|
137
|
+
alone. Otherwise the result is every matching non-override state.
|
|
138
|
+
|
|
139
|
+
*item* is a plain dict built from adapter data: ``repo``, ``issue``
|
|
140
|
+
(``number``/``state``/``assignees``/``open_dependencies``) and ``pr``
|
|
141
|
+
(``None``, or ``number``/``state``/``assignees``/``requested_reviewers``/
|
|
142
|
+
``reviewed_on_head``/``approved_on_head``). Role names in predicates
|
|
143
|
+
resolve to logins through *logins*; this module never looks logins up
|
|
144
|
+
itself. Well-formed items resolve to exactly one state; zero or 2+ is an
|
|
145
|
+
anomaly that :func:`check` reports.
|
|
146
|
+
"""
|
|
147
|
+
for name, spec in states.items():
|
|
148
|
+
if spec["override"] and _match(spec["when"], item, logins):
|
|
149
|
+
return [name]
|
|
150
|
+
return [
|
|
151
|
+
name for name, spec in states.items() if not spec["override"] and _match(spec["when"], item, logins)
|
|
152
|
+
]
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def check(
|
|
156
|
+
states: dict[str, dict[str, Any]],
|
|
157
|
+
items: list[dict[str, Any]],
|
|
158
|
+
logins: dict[str, str],
|
|
159
|
+
) -> list[dict]:
|
|
160
|
+
"""Report every *item* that matches zero or 2+ states.
|
|
161
|
+
|
|
162
|
+
Returns one anomaly ``{repo, issue, pr, matched: [...]}`` per mis-mapped
|
|
163
|
+
item, in items order: an item matched by nothing is owned by nobody, one
|
|
164
|
+
matched twice is contested between owners.
|
|
165
|
+
"""
|
|
166
|
+
anomalies: list[dict] = []
|
|
167
|
+
for item in items:
|
|
168
|
+
matched = evaluate(states, item, logins)
|
|
169
|
+
if len(matched) != 1:
|
|
170
|
+
anomalies.append(
|
|
171
|
+
{
|
|
172
|
+
"repo": item.get("repo"),
|
|
173
|
+
"issue": (item.get("issue") or {}).get("number"),
|
|
174
|
+
"pr": (item.get("pr") or {}).get("number"),
|
|
175
|
+
"matched": matched,
|
|
176
|
+
}
|
|
177
|
+
)
|
|
178
|
+
return anomalies
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _match(pred: dict[str, Any], item: dict[str, Any], logins: dict[str, str]) -> bool:
|
|
182
|
+
return all(_leaf(key, value, item, logins) for key, value in pred.items())
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _leaf(key: str, expected: Any, item: dict[str, Any], logins: dict[str, str]) -> bool:
|
|
186
|
+
if key == "any":
|
|
187
|
+
return any(_match(sub, item, logins) for sub in expected)
|
|
188
|
+
if key == "not":
|
|
189
|
+
return not _match(expected, item, logins)
|
|
190
|
+
section, _, attr = key.partition(".")
|
|
191
|
+
if not attr:
|
|
192
|
+
# A bare section name (`pr`) tests presence: the only use is `pr: none`.
|
|
193
|
+
has = item.get("pr") is not None
|
|
194
|
+
return has if expected != "none" else not has
|
|
195
|
+
obj = item.get(section) or {}
|
|
196
|
+
if attr == "assignee" or attr == "requested_reviewers":
|
|
197
|
+
if not isinstance(expected, str):
|
|
198
|
+
return False
|
|
199
|
+
if expected == "none":
|
|
200
|
+
# `issue.assignee: none` tests an empty assignees list, in the
|
|
201
|
+
# way `pr: none` tests PR absence. A role name is never `none`.
|
|
202
|
+
if attr != "assignee":
|
|
203
|
+
return False
|
|
204
|
+
return not (obj.get(_ITEM_KEYS[attr]) or [])
|
|
205
|
+
return _role_login(expected, logins, key) in (obj.get(_ITEM_KEYS[attr]) or [])
|
|
206
|
+
if attr == "state":
|
|
207
|
+
return obj.get("state") == expected
|
|
208
|
+
if attr == "dependencies":
|
|
209
|
+
truth = bool(obj.get("open_dependencies"))
|
|
210
|
+
return truth if expected == "open" else not truth
|
|
211
|
+
if attr in ("reviewed_on_head", "approved_on_head"):
|
|
212
|
+
return bool(obj.get(_ITEM_KEYS[attr])) == _as_bool(expected)
|
|
213
|
+
return False # validated files cannot reach an unknown leaf
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _role_login(role: str, logins: dict[str, str], key: str) -> str:
|
|
217
|
+
try:
|
|
218
|
+
return logins[role]
|
|
219
|
+
except KeyError:
|
|
220
|
+
raise ValueError(f"`{key}`: role {role!r} not in the caller-supplied `logins`") from None
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
def _as_bool(value: Any) -> bool:
|
|
224
|
+
if isinstance(value, bool):
|
|
225
|
+
return value
|
|
226
|
+
return str(value).strip().lower() in {"true", "yes"}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
{"repo": "example/widget", "issue": {"number": 101, "state": "open", "assignees": ["dee"], "open_dependencies": []}, "pr": null}
|
|
2
|
+
{"repo": "example/widget", "issue": {"number": 102, "state": "open", "assignees": ["dee"], "open_dependencies": []}, "pr": {"number": 202, "state": "open", "assignees": [], "requested_reviewers": [], "reviewed_on_head": false, "approved_on_head": false}}
|
|
3
|
+
{"repo": "example/widget", "issue": {"number": 103, "state": "open", "assignees": ["dee"], "open_dependencies": []}, "pr": {"number": 203, "state": "open", "assignees": [], "requested_reviewers": ["ria"], "reviewed_on_head": false, "approved_on_head": true}}
|