agent-usage-tracker 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_usage_tracker/__init__.py +8 -0
- agent_usage_tracker/backup.py +225 -0
- agent_usage_tracker/cli.py +479 -0
- agent_usage_tracker/collectors/__init__.py +115 -0
- agent_usage_tracker/collectors/_util.py +310 -0
- agent_usage_tracker/collectors/activity_common.py +294 -0
- agent_usage_tracker/collectors/activitywatch.py +394 -0
- agent_usage_tracker/collectors/claude_code.py +484 -0
- agent_usage_tracker/collectors/claude_code_activity.py +317 -0
- agent_usage_tracker/collectors/codex.py +612 -0
- agent_usage_tracker/collectors/commits.py +154 -0
- agent_usage_tracker/collectors/cursor.py +193 -0
- agent_usage_tracker/collectors/cursor_api.py +247 -0
- agent_usage_tracker/collectors/cursor_cli.py +592 -0
- agent_usage_tracker/collectors/cursor_sdk.py +95 -0
- agent_usage_tracker/collectors/generic_sdk.py +100 -0
- agent_usage_tracker/collectors/pi.py +397 -0
- agent_usage_tracker/collectors/session_digest.py +379 -0
- agent_usage_tracker/collectors/t3.py +490 -0
- agent_usage_tracker/config.py +392 -0
- agent_usage_tracker/coverage.py +615 -0
- agent_usage_tracker/coverage_catalog.json +588 -0
- agent_usage_tracker/cursor_run.py +62 -0
- agent_usage_tracker/diagnostics.py +74 -0
- agent_usage_tracker/episode_store.py +78 -0
- agent_usage_tracker/event.py +132 -0
- agent_usage_tracker/git_day_cache.py +128 -0
- agent_usage_tracker/git_days.py +499 -0
- agent_usage_tracker/git_safe.py +58 -0
- agent_usage_tracker/harness_inventory.py +208 -0
- agent_usage_tracker/history_steers.py +88 -0
- agent_usage_tracker/improve/__init__.py +1 -0
- agent_usage_tracker/improve/case_view.py +112 -0
- agent_usage_tracker/improve/cli.py +138 -0
- agent_usage_tracker/improve/desktop.py +347 -0
- agent_usage_tracker/improve/evidence.py +322 -0
- agent_usage_tracker/improve/judge.py +671 -0
- agent_usage_tracker/improve/proposals.py +72 -0
- agent_usage_tracker/improve/rubric.py +360 -0
- agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
- agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
- agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
- agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
- agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
- agent_usage_tracker/improve/runtime.py +117 -0
- agent_usage_tracker/improve/schedule.py +291 -0
- agent_usage_tracker/improve/search_check.py +563 -0
- agent_usage_tracker/improve/store.py +244 -0
- agent_usage_tracker/improve/studies.py +285 -0
- agent_usage_tracker/improve/study_cli.py +46 -0
- agent_usage_tracker/improve/study_decision.py +121 -0
- agent_usage_tracker/improve/study_followup.py +174 -0
- agent_usage_tracker/improve/t3.py +269 -0
- agent_usage_tracker/improve/trial.py +318 -0
- agent_usage_tracker/improve/validation.py +148 -0
- agent_usage_tracker/inspector.py +350 -0
- agent_usage_tracker/label/__init__.py +22 -0
- agent_usage_tracker/label/backends.py +742 -0
- agent_usage_tracker/label/bakeoff.py +661 -0
- agent_usage_tracker/label/baseline.py +173 -0
- agent_usage_tracker/label/clef.py +258 -0
- agent_usage_tracker/label/context.py +382 -0
- agent_usage_tracker/label/core.py +355 -0
- agent_usage_tracker/label/covered_run.py +92 -0
- agent_usage_tracker/label/evalset.py +154 -0
- agent_usage_tracker/label/gold.py +209 -0
- agent_usage_tracker/label/local.py +326 -0
- agent_usage_tracker/label/local_parallel.py +232 -0
- agent_usage_tracker/label/merge.py +57 -0
- agent_usage_tracker/label/outcome_coverage.py +57 -0
- agent_usage_tracker/label/replay.py +194 -0
- agent_usage_tracker/label/run.py +339 -0
- agent_usage_tracker/label/tasks.py +579 -0
- agent_usage_tracker/label/teach.py +145 -0
- agent_usage_tracker/local_page.py +128 -0
- agent_usage_tracker/manual_time.py +69 -0
- agent_usage_tracker/net.py +23 -0
- agent_usage_tracker/observation_cli.py +65 -0
- agent_usage_tracker/observation_report.py +116 -0
- agent_usage_tracker/observation_server.py +110 -0
- agent_usage_tracker/observations.py +365 -0
- agent_usage_tracker/outcome_git.py +383 -0
- agent_usage_tracker/outcomes.py +464 -0
- agent_usage_tracker/pricing.py +248 -0
- agent_usage_tracker/push.py +530 -0
- agent_usage_tracker/redact.py +45 -0
- agent_usage_tracker/repo_keys.py +165 -0
- agent_usage_tracker/review.py +252 -0
- agent_usage_tracker/scan.py +472 -0
- agent_usage_tracker/scheduled.py +164 -0
- agent_usage_tracker/sdk.py +79 -0
- agent_usage_tracker/sink.py +82 -0
- agent_usage_tracker/sink_stack/README.md +71 -0
- agent_usage_tracker/sink_stack/apply.sh +63 -0
- agent_usage_tracker/sink_stack/compose.yaml +94 -0
- agent_usage_tracker/sink_stack/env.example +20 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
- agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
- agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
- agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
- agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
- agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
- agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
- agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
- agent_usage_tracker/sources.py +180 -0
- agent_usage_tracker/state.py +191 -0
- agent_usage_tracker/telemetry.py +165 -0
- agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
- agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
- agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
- agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
- agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
"""Import provisional historical labels separately from calibrated nightly labels."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import math
|
|
8
|
+
import re
|
|
9
|
+
from collections import Counter
|
|
10
|
+
from datetime import datetime, timezone
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from ..config import AutError, Config
|
|
15
|
+
from ..event import ActivityEvent, make_event_id
|
|
16
|
+
from ..push import send_activity_rows
|
|
17
|
+
|
|
18
|
+
POLICY = "working-sonnet-v1"
|
|
19
|
+
_MODEL = "claude-sonnet-5-5-medium"
|
|
20
|
+
_LABELS = {
|
|
21
|
+
"steer_intent": {
|
|
22
|
+
"framing",
|
|
23
|
+
"instruction",
|
|
24
|
+
"clarification",
|
|
25
|
+
"correction",
|
|
26
|
+
"redirection",
|
|
27
|
+
"review",
|
|
28
|
+
"acceptance",
|
|
29
|
+
"stop",
|
|
30
|
+
"other",
|
|
31
|
+
},
|
|
32
|
+
"soft_outcome": {"resolved", "still_open", "blocked", "ruled_out"},
|
|
33
|
+
}
|
|
34
|
+
_SIGNALS = {
|
|
35
|
+
"steer_intent": {"explicit_halt", "disputes_agent_work"},
|
|
36
|
+
"soft_outcome": {"required_next_step", "external_blocker"},
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _check(condition: bool) -> None:
|
|
41
|
+
if not condition:
|
|
42
|
+
raise ValueError("invalid provisional baseline")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _timestamp(value: Any) -> str:
|
|
46
|
+
_check(isinstance(value, str))
|
|
47
|
+
_check(datetime.fromisoformat(value).utcoffset() is not None)
|
|
48
|
+
return str(value)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _events(bundle: Any, machine: str, labelled_at: str) -> list[ActivityEvent]:
|
|
52
|
+
_check(bundle["version"] == "working-sonnet-baseline-v1")
|
|
53
|
+
_check(isinstance(bundle["rows"], list) and bool(bundle["rows"]))
|
|
54
|
+
baseline_id = hashlib.sha256(
|
|
55
|
+
json.dumps(bundle, sort_keys=True, allow_nan=False).encode()
|
|
56
|
+
).hexdigest()
|
|
57
|
+
stamped = _timestamp(labelled_at)
|
|
58
|
+
result, seen = [], set()
|
|
59
|
+
for row in bundle["rows"]:
|
|
60
|
+
task, unit_id = row["task"], row["unit_id"]
|
|
61
|
+
_check(
|
|
62
|
+
task in _LABELS
|
|
63
|
+
and isinstance(unit_id, str)
|
|
64
|
+
and re.fullmatch(r"[A-Za-z0-9._:-]{1,200}", unit_id) is not None
|
|
65
|
+
)
|
|
66
|
+
_check(row["unit"] == ("steer" if task == "steer_intent" else "episode"))
|
|
67
|
+
_check((task, unit_id) not in seen)
|
|
68
|
+
seen.add((task, unit_id))
|
|
69
|
+
_check(
|
|
70
|
+
row["quality"] == "provisional"
|
|
71
|
+
and row["estimate"] is True
|
|
72
|
+
and row["human_verified"] is False
|
|
73
|
+
)
|
|
74
|
+
_check(
|
|
75
|
+
row["label"] in _LABELS[task] | {"unknown"}
|
|
76
|
+
and row["baseline_label"] in _LABELS[task] | {"unknown"}
|
|
77
|
+
)
|
|
78
|
+
_check(row["label_source"] in {"sonnet", "unknown_review", "unknown_input"})
|
|
79
|
+
_check(row["label_source"] == "sonnet" or row["label"] == "unknown")
|
|
80
|
+
_check(
|
|
81
|
+
row["model_id"] == _MODEL
|
|
82
|
+
or (
|
|
83
|
+
row["model_id"] is None
|
|
84
|
+
and row["label"] == "unknown"
|
|
85
|
+
and row["baseline_label"] == "unknown"
|
|
86
|
+
)
|
|
87
|
+
)
|
|
88
|
+
weight = row["weight"]
|
|
89
|
+
_check(type(weight) in (int, float) and math.isfinite(weight) and weight > 0)
|
|
90
|
+
_check(
|
|
91
|
+
isinstance(row["input_hash"], str)
|
|
92
|
+
and re.fullmatch(r"[a-f0-9]{64}", row["input_hash"]) is not None
|
|
93
|
+
)
|
|
94
|
+
signals = {}
|
|
95
|
+
_check(isinstance(row["reviewed_signals"], dict))
|
|
96
|
+
for question, answer in row["reviewed_signals"].items():
|
|
97
|
+
_check(question in _SIGNALS[task] and answer["answer"] in {"yes", "no", "unknown"})
|
|
98
|
+
_check(answer["source"] == "agent_review" and answer["human_verified"] is False)
|
|
99
|
+
_check(
|
|
100
|
+
isinstance(answer["question_sha256"], str)
|
|
101
|
+
and re.fullmatch(r"[a-f0-9]{64}", answer["question_sha256"]) is not None
|
|
102
|
+
)
|
|
103
|
+
signals[question] = {
|
|
104
|
+
"answer": answer["answer"],
|
|
105
|
+
"source": "agent_review",
|
|
106
|
+
"question_sha256": answer["question_sha256"],
|
|
107
|
+
}
|
|
108
|
+
_check(
|
|
109
|
+
not any(value["answer"] == "unknown" for value in signals.values())
|
|
110
|
+
or row["label"] == "unknown"
|
|
111
|
+
)
|
|
112
|
+
# Only ids, enums and numbers cross the sink boundary, never review reasons or paths.
|
|
113
|
+
attrs = {
|
|
114
|
+
"task": "working." + task,
|
|
115
|
+
"task_ver": 1,
|
|
116
|
+
"unit": row["unit"],
|
|
117
|
+
"unit_id": unit_id,
|
|
118
|
+
"label": row["label"],
|
|
119
|
+
"baseline_label": row["baseline_label"],
|
|
120
|
+
"p": None,
|
|
121
|
+
"below_tau": row["label"] == "unknown",
|
|
122
|
+
"model_id": row["model_id"],
|
|
123
|
+
"label_policy": POLICY,
|
|
124
|
+
"baseline_id": baseline_id,
|
|
125
|
+
"labelled_at": stamped,
|
|
126
|
+
"label_source": row["label_source"],
|
|
127
|
+
"population_weight": weight,
|
|
128
|
+
"input_hash": row["input_hash"],
|
|
129
|
+
"provisional": True,
|
|
130
|
+
"human_verified": False,
|
|
131
|
+
"signals": signals,
|
|
132
|
+
}
|
|
133
|
+
result.append(
|
|
134
|
+
ActivityEvent(
|
|
135
|
+
event_id=make_event_id("baseline", POLICY, baseline_id, task, unit_id),
|
|
136
|
+
ts=_timestamp(row["ts"]),
|
|
137
|
+
agent="label",
|
|
138
|
+
kind="label",
|
|
139
|
+
session_id=None,
|
|
140
|
+
turn_id=None,
|
|
141
|
+
attrs=attrs,
|
|
142
|
+
machine=machine,
|
|
143
|
+
)
|
|
144
|
+
)
|
|
145
|
+
return result
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def events(bundle: Any, machine: str, labelled_at: str) -> list[ActivityEvent]:
|
|
149
|
+
"""Validate the entire bundle before writing; repeated imports have stable event ids."""
|
|
150
|
+
try:
|
|
151
|
+
return _events(bundle, machine, labelled_at)
|
|
152
|
+
except (KeyError, TypeError, ValueError, OverflowError) as exc:
|
|
153
|
+
raise AutError("label import-baseline: invalid provisional baseline bundle") from exc
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def run(cfg: Config, path: Path, *, apply: bool = False) -> dict[str, Any]:
|
|
157
|
+
try:
|
|
158
|
+
bundle = json.loads(path.read_text())
|
|
159
|
+
except (OSError, ValueError) as exc:
|
|
160
|
+
raise AutError("label import-baseline: could not read the baseline JSON") from exc
|
|
161
|
+
rows = events(bundle, cfg.machine, datetime.now(timezone.utc).isoformat())
|
|
162
|
+
if apply:
|
|
163
|
+
if not cfg.pg_dsn:
|
|
164
|
+
raise AutError("label import-baseline: transport.pg_dsn is required for --apply")
|
|
165
|
+
send_activity_rows(cfg.pg_dsn, rows)
|
|
166
|
+
return {
|
|
167
|
+
"mode": "applied" if apply else "preview",
|
|
168
|
+
"policy": POLICY,
|
|
169
|
+
"rows": len(rows),
|
|
170
|
+
"unknown": sum(e.attrs["label"] == "unknown" for e in rows),
|
|
171
|
+
"reviewed_signals": sum(len(e.attrs["signals"]) for e in rows),
|
|
172
|
+
"tasks": dict(Counter(e.attrs["task"] for e in rows)),
|
|
173
|
+
}
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
"""Cloudflare Clef's joint schema reader on MLX, text only: the prompt encoding and the head.
|
|
2
|
+
|
|
3
|
+
Trimmed from clef_mlx.py in mlx-community/clef-flash-4bit (Apache-2.0), which mirrors the reference
|
|
4
|
+
joint_schema_model.encode_record. Left out: images, video, the server, and the mlx-vlm loader.
|
|
5
|
+
Imported only when clef-flash runs; it needs mlx from the extra aut[label].
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import math
|
|
12
|
+
from dataclasses import dataclass
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any, Callable
|
|
15
|
+
|
|
16
|
+
import mlx.core as mx
|
|
17
|
+
import mlx.nn as nn
|
|
18
|
+
|
|
19
|
+
SYSTEM_PROMPT = ("Read the complete state and schema. Decide every field jointly. Each answer "
|
|
20
|
+
"must be exactly one of that field's allowed options.")
|
|
21
|
+
QUESTION_TYPES = {"noul": 0, "choice": 1, "score": 2}
|
|
22
|
+
MAX_LENGTH = 16384
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def render(value: Any) -> str:
|
|
26
|
+
if isinstance(value, str):
|
|
27
|
+
return value
|
|
28
|
+
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def options_of(question: dict[str, Any]) -> list[tuple[str, str]]:
|
|
32
|
+
if question["type"] == "noul":
|
|
33
|
+
return [(k, question["criteria"][k]) for k in ("true", "false")]
|
|
34
|
+
return sorted((str(k), v) for k, v in question["criteria"].items())
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(frozen=True)
|
|
38
|
+
class Encoded:
|
|
39
|
+
question_type: int
|
|
40
|
+
question_span: tuple[int, int]
|
|
41
|
+
option_spans: tuple[tuple[int, int], ...]
|
|
42
|
+
option_ids: tuple[str, ...]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def encode_record(tok: Any, record: dict[str, Any]) -> tuple[list[int], list[Encoded]]:
|
|
46
|
+
"""Token ids and the spans of each question and option. The state is cut to fit MAX_LENGTH,
|
|
47
|
+
like the reference."""
|
|
48
|
+
def tokens(text: str) -> list[int]:
|
|
49
|
+
return tok.encode(text, add_special_tokens=False)
|
|
50
|
+
|
|
51
|
+
schema = tokens("\n\nSCHEMA FIELDS:\n")
|
|
52
|
+
found: list[Encoded] = []
|
|
53
|
+
for qi, (qid, q) in enumerate(record["questions"].items()):
|
|
54
|
+
schema += tokens(f"\nFIELD {qi + 1}\nID: {qid}\nTYPE: {q['type']}\nINSTRUCTION: ")
|
|
55
|
+
q_start = len(schema)
|
|
56
|
+
schema += tokens(render(q["instructions"]))
|
|
57
|
+
q_end = len(schema)
|
|
58
|
+
schema += tokens("\nALLOWED OPTIONS:\n")
|
|
59
|
+
spans, ids = [], []
|
|
60
|
+
for oi, (option, description) in enumerate(options_of(q)):
|
|
61
|
+
schema += tokens(f"OPTION {oi + 1}: ")
|
|
62
|
+
start = len(schema)
|
|
63
|
+
schema += tokens(render({"option_id": option, "description": description}))
|
|
64
|
+
spans.append((start, len(schema)))
|
|
65
|
+
ids.append(option)
|
|
66
|
+
schema += tokens("\n")
|
|
67
|
+
schema += tokens("END FIELD\n")
|
|
68
|
+
found.append(Encoded(QUESTION_TYPES[q["type"]], (q_start, q_end), tuple(spans), tuple(ids)))
|
|
69
|
+
prefix = tokens(f"<|im_start|>system\n{SYSTEM_PROMPT}<|im_end|>\n<|im_start|>user\nSTATE:\n")
|
|
70
|
+
suffix = tokens("\n<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
|
|
71
|
+
"JOINT SCHEMA DECISIONS:")
|
|
72
|
+
state = tokens(render(record["state"]))[:max(0, MAX_LENGTH - len(prefix) - len(schema)
|
|
73
|
+
- len(suffix))]
|
|
74
|
+
off = len(prefix) + len(state)
|
|
75
|
+
return prefix + state + schema + suffix, [
|
|
76
|
+
Encoded(q.question_type, (q.question_span[0] + off, q.question_span[1] + off),
|
|
77
|
+
tuple((s + off, e + off) for s, e in q.option_spans), q.option_ids)
|
|
78
|
+
for q in found]
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class MultiheadAttention(nn.Module):
|
|
82
|
+
"""torch.nn.MultiheadAttention (batch_first, packed in_proj) in MLX."""
|
|
83
|
+
|
|
84
|
+
def __init__(self, width: int, heads: int):
|
|
85
|
+
super().__init__()
|
|
86
|
+
self.heads = heads
|
|
87
|
+
self.in_proj_weight = mx.zeros((3 * width, width))
|
|
88
|
+
self.in_proj_bias = mx.zeros((3 * width,))
|
|
89
|
+
self.out_proj = nn.Linear(width, width)
|
|
90
|
+
|
|
91
|
+
def __call__(self, q: mx.array, k: mx.array, v: mx.array) -> mx.array:
|
|
92
|
+
w, b = self.in_proj_weight, self.in_proj_bias
|
|
93
|
+
d = w.shape[1]
|
|
94
|
+
q = q @ w[:d].T + b[:d]
|
|
95
|
+
k = k @ w[d:2 * d].T + b[d:2 * d]
|
|
96
|
+
v = v @ w[2 * d:].T + b[2 * d:]
|
|
97
|
+
batch, lq, lk, hd = q.shape[0], q.shape[1], k.shape[1], d // self.heads
|
|
98
|
+
q = q.reshape(batch, lq, self.heads, hd).transpose(0, 2, 1, 3)
|
|
99
|
+
k = k.reshape(batch, lk, self.heads, hd).transpose(0, 2, 1, 3)
|
|
100
|
+
v = v.reshape(batch, lk, self.heads, hd).transpose(0, 2, 1, 3)
|
|
101
|
+
o = mx.fast.scaled_dot_product_attention(q, k, v, scale=hd ** -0.5)
|
|
102
|
+
return self.out_proj(o.transpose(0, 2, 1, 3).reshape(batch, lq, d))
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class FeedForward(nn.Module):
|
|
106
|
+
def __init__(self, width: int, feedforward: int):
|
|
107
|
+
super().__init__()
|
|
108
|
+
self.fc1 = nn.Linear(width, feedforward)
|
|
109
|
+
self.fc2 = nn.Linear(feedforward, width)
|
|
110
|
+
|
|
111
|
+
def __call__(self, x: mx.array) -> mx.array:
|
|
112
|
+
return self.fc2(nn.gelu(self.fc1(x)))
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class EvidenceRoutingLayer(nn.Module):
|
|
116
|
+
def __init__(self, width: int, heads: int, feedforward: int):
|
|
117
|
+
super().__init__()
|
|
118
|
+
self.query_norm = nn.LayerNorm(width)
|
|
119
|
+
self.memory_norm = nn.LayerNorm(width)
|
|
120
|
+
self.attention = MultiheadAttention(width, heads)
|
|
121
|
+
self.feedforward_norm = nn.LayerNorm(width)
|
|
122
|
+
self.feedforward = FeedForward(width, feedforward)
|
|
123
|
+
|
|
124
|
+
def __call__(self, queries: mx.array, memory: mx.array) -> mx.array:
|
|
125
|
+
m = self.memory_norm(memory)
|
|
126
|
+
queries = queries + self.attention(self.query_norm(queries), m, m)
|
|
127
|
+
return queries + self.feedforward(self.feedforward_norm(queries))
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class TransformerDecoderLayer(nn.Module):
|
|
131
|
+
"""torch.nn.TransformerDecoderLayer(norm_first=True, activation='gelu')."""
|
|
132
|
+
|
|
133
|
+
def __init__(self, width: int, heads: int, feedforward: int):
|
|
134
|
+
super().__init__()
|
|
135
|
+
self.self_attn = MultiheadAttention(width, heads)
|
|
136
|
+
self.multihead_attn = MultiheadAttention(width, heads)
|
|
137
|
+
self.linear1 = nn.Linear(width, feedforward)
|
|
138
|
+
self.linear2 = nn.Linear(feedforward, width)
|
|
139
|
+
self.norm1 = nn.LayerNorm(width)
|
|
140
|
+
self.norm2 = nn.LayerNorm(width)
|
|
141
|
+
self.norm3 = nn.LayerNorm(width)
|
|
142
|
+
|
|
143
|
+
def __call__(self, x: mx.array, memory: mx.array) -> mx.array:
|
|
144
|
+
h = self.norm1(x)
|
|
145
|
+
x = x + self.self_attn(h, h, h)
|
|
146
|
+
x = x + self.multihead_attn(self.norm2(x), memory, memory)
|
|
147
|
+
return x + self.linear2(nn.gelu(self.linear1(self.norm3(x))))
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _l2norm(x: mx.array, eps: float = 1e-12) -> mx.array:
|
|
151
|
+
return x / mx.maximum(mx.linalg.norm(x, axis=-1, keepdims=True), eps)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
class JointSchemaHead(nn.Module):
|
|
155
|
+
def __init__(self, hidden_size: int, width: int, routing_layers: int, layers: int, heads: int,
|
|
156
|
+
feedforward: int):
|
|
157
|
+
super().__init__()
|
|
158
|
+
self.hidden_norm = nn.LayerNorm(hidden_size)
|
|
159
|
+
self.memory_projection = nn.Linear(hidden_size, width, bias=False)
|
|
160
|
+
self.question_projection = nn.Linear(hidden_size, width, bias=False)
|
|
161
|
+
self.option_question_projection = nn.Linear(hidden_size, width, bias=False)
|
|
162
|
+
self.global_projection = nn.Linear(hidden_size, width, bias=False)
|
|
163
|
+
self.option_context_projection = nn.Linear(hidden_size, width, bias=False)
|
|
164
|
+
self.option_lexical_projection = nn.Linear(hidden_size, width, bias=False)
|
|
165
|
+
self.type_embedding = nn.Embedding(3, width)
|
|
166
|
+
self.evidence_layers = [EvidenceRoutingLayer(width, heads, feedforward)
|
|
167
|
+
for _ in range(routing_layers)]
|
|
168
|
+
self.option_summary_norm = nn.LayerNorm(width)
|
|
169
|
+
self.layers = [TransformerDecoderLayer(width, heads, feedforward) for _ in range(layers)]
|
|
170
|
+
self.field_norm = nn.LayerNorm(width)
|
|
171
|
+
self.option_norm = nn.LayerNorm(width)
|
|
172
|
+
self.scorer1 = nn.Linear(width * 4, width)
|
|
173
|
+
self.scorer2 = nn.Linear(width, 1)
|
|
174
|
+
self.prior_logit_scale = mx.zeros(())
|
|
175
|
+
self.joint_logit_scale = mx.zeros(())
|
|
176
|
+
self.residual_gate = mx.zeros(())
|
|
177
|
+
|
|
178
|
+
def __call__(self, hidden: mx.array, input_ids: mx.array, questions: list[Encoded],
|
|
179
|
+
lexical: Callable[[mx.array], mx.array]) -> list[mx.array]:
|
|
180
|
+
"""hidden: (L, H) final backbone states of one record. One logit array per question."""
|
|
181
|
+
h = self.hidden_norm(hidden)
|
|
182
|
+
memory = self.memory_projection(h)[None]
|
|
183
|
+
global_vector = h[-1]
|
|
184
|
+
question_vectors = mx.stack([h[q.question_span[0]:q.question_span[1]].mean(0)
|
|
185
|
+
for q in questions])
|
|
186
|
+
type_ids = mx.array([q.question_type for q in questions])
|
|
187
|
+
contexts = [mx.stack([h[s:e].mean(0) for s, e in q.option_spans]) for q in questions]
|
|
188
|
+
lexicals = [mx.stack([lexical(input_ids[s:e]).mean(0) for s, e in q.option_spans])
|
|
189
|
+
for q in questions]
|
|
190
|
+
counts = [len(q.option_spans) for q in questions]
|
|
191
|
+
routed = mx.concatenate([
|
|
192
|
+
self.option_context_projection(c) + self.option_lexical_projection(lx)
|
|
193
|
+
+ self.option_question_projection(question_vectors[i])[None]
|
|
194
|
+
for i, (c, lx) in enumerate(zip(contexts, lexicals))], axis=0)[None]
|
|
195
|
+
for layer in self.evidence_layers:
|
|
196
|
+
routed = layer(routed, memory)
|
|
197
|
+
routed = routed[0]
|
|
198
|
+
splits = [int(x) for x in mx.cumsum(mx.array(counts))[:-1].tolist()]
|
|
199
|
+
split_options = mx.split(routed, splits, axis=0) if splits else [routed]
|
|
200
|
+
|
|
201
|
+
base_fields = self.question_projection(question_vectors)
|
|
202
|
+
summaries = []
|
|
203
|
+
for field, options in zip(base_fields, split_options):
|
|
204
|
+
w = mx.softmax((options @ field) / math.sqrt(options.shape[-1]), axis=0)
|
|
205
|
+
summaries.append((w[:, None] * options).sum(0))
|
|
206
|
+
fields = (base_fields + self.option_summary_norm(mx.stack(summaries))
|
|
207
|
+
+ self.global_projection(global_vector)[None]
|
|
208
|
+
+ self.type_embedding(type_ids))[None]
|
|
209
|
+
for layer in self.layers:
|
|
210
|
+
fields = layer(fields, memory)
|
|
211
|
+
fields = self.field_norm(fields[0])
|
|
212
|
+
|
|
213
|
+
prior_scale = mx.exp(mx.minimum(self.prior_logit_scale, math.log(100.0)))
|
|
214
|
+
joint_scale = mx.exp(mx.minimum(self.joint_logit_scale, math.log(100.0)))
|
|
215
|
+
gate = mx.sigmoid(self.residual_gate)
|
|
216
|
+
logits = []
|
|
217
|
+
for i, (field, lx, opts) in enumerate(zip(fields, lexicals, split_options)):
|
|
218
|
+
prior = prior_scale * (_l2norm(lx) @ _l2norm(question_vectors[i] + global_vector))
|
|
219
|
+
options = self.option_norm(opts)
|
|
220
|
+
rf = mx.broadcast_to(field[None], options.shape)
|
|
221
|
+
cosine = (rf * options).sum(-1) / mx.maximum(
|
|
222
|
+
mx.linalg.norm(rf, axis=-1) * mx.linalg.norm(options, axis=-1), 1e-8)
|
|
223
|
+
feats = mx.concatenate([rf, options, rf * options, mx.abs(rf - options)], axis=-1)
|
|
224
|
+
residual = self.scorer2(nn.gelu(self.scorer1(feats)))[:, 0]
|
|
225
|
+
logits.append(prior + gate * (joint_scale * cosine + residual))
|
|
226
|
+
return logits
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def load_head(path: Path) -> JointSchemaHead:
|
|
230
|
+
head = JointSchemaHead(**json.loads((path / "joint_head_config.json").read_text()))
|
|
231
|
+
weights = {k.replace(".feedforward.0.", ".feedforward.fc1.")
|
|
232
|
+
.replace(".feedforward.3.", ".feedforward.fc2.")
|
|
233
|
+
.replace("residual_scorer.0.", "scorer1.")
|
|
234
|
+
.replace("residual_scorer.3.", "scorer2."): v
|
|
235
|
+
for k, v in mx.load(str(path / "joint_head.safetensors")).items()}
|
|
236
|
+
head.load_weights(list(weights.items()), strict=True)
|
|
237
|
+
head.set_dtype(mx.bfloat16)
|
|
238
|
+
mx.eval(head.parameters())
|
|
239
|
+
return head
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def predict(model: Any, tok: Any, head: JointSchemaHead, record: dict[str, Any]) -> dict[str, float]:
|
|
243
|
+
"""{option id: probability} for the one question of a record."""
|
|
244
|
+
lm = model.language_model
|
|
245
|
+
out = lm.lm_head if hasattr(lm, "lm_head") else lm.model.embed_tokens
|
|
246
|
+
|
|
247
|
+
def lexical(ids: mx.array) -> mx.array: # rows of the output matrix, dequantized if needed
|
|
248
|
+
if isinstance(out, (nn.QuantizedLinear, nn.QuantizedEmbedding)):
|
|
249
|
+
return mx.dequantize(out.weight[ids], out.scales[ids],
|
|
250
|
+
out.get("biases")[ids] if "biases" in out else None,
|
|
251
|
+
group_size=out.group_size, bits=out.bits,
|
|
252
|
+
mode=getattr(out, "mode", "affine"))
|
|
253
|
+
return out.weight[ids]
|
|
254
|
+
|
|
255
|
+
tokens, questions = encode_record(tok, record)
|
|
256
|
+
ids = mx.array(tokens)
|
|
257
|
+
logits = head(lm.model(ids[None])[0], ids, questions, lexical)[0]
|
|
258
|
+
return dict(zip(questions[0].option_ids, mx.softmax(logits.astype(mx.float32)).tolist()))
|