agent-usage-tracker 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. agent_usage_tracker/__init__.py +8 -0
  2. agent_usage_tracker/backup.py +225 -0
  3. agent_usage_tracker/cli.py +479 -0
  4. agent_usage_tracker/collectors/__init__.py +115 -0
  5. agent_usage_tracker/collectors/_util.py +310 -0
  6. agent_usage_tracker/collectors/activity_common.py +294 -0
  7. agent_usage_tracker/collectors/activitywatch.py +394 -0
  8. agent_usage_tracker/collectors/claude_code.py +484 -0
  9. agent_usage_tracker/collectors/claude_code_activity.py +317 -0
  10. agent_usage_tracker/collectors/codex.py +612 -0
  11. agent_usage_tracker/collectors/commits.py +154 -0
  12. agent_usage_tracker/collectors/cursor.py +193 -0
  13. agent_usage_tracker/collectors/cursor_api.py +247 -0
  14. agent_usage_tracker/collectors/cursor_cli.py +592 -0
  15. agent_usage_tracker/collectors/cursor_sdk.py +95 -0
  16. agent_usage_tracker/collectors/generic_sdk.py +100 -0
  17. agent_usage_tracker/collectors/pi.py +397 -0
  18. agent_usage_tracker/collectors/session_digest.py +379 -0
  19. agent_usage_tracker/collectors/t3.py +490 -0
  20. agent_usage_tracker/config.py +392 -0
  21. agent_usage_tracker/coverage.py +615 -0
  22. agent_usage_tracker/coverage_catalog.json +588 -0
  23. agent_usage_tracker/cursor_run.py +62 -0
  24. agent_usage_tracker/diagnostics.py +74 -0
  25. agent_usage_tracker/episode_store.py +78 -0
  26. agent_usage_tracker/event.py +132 -0
  27. agent_usage_tracker/git_day_cache.py +128 -0
  28. agent_usage_tracker/git_days.py +499 -0
  29. agent_usage_tracker/git_safe.py +58 -0
  30. agent_usage_tracker/harness_inventory.py +208 -0
  31. agent_usage_tracker/history_steers.py +88 -0
  32. agent_usage_tracker/improve/__init__.py +1 -0
  33. agent_usage_tracker/improve/case_view.py +112 -0
  34. agent_usage_tracker/improve/cli.py +138 -0
  35. agent_usage_tracker/improve/desktop.py +347 -0
  36. agent_usage_tracker/improve/evidence.py +322 -0
  37. agent_usage_tracker/improve/judge.py +671 -0
  38. agent_usage_tracker/improve/proposals.py +72 -0
  39. agent_usage_tracker/improve/rubric.py +360 -0
  40. agent_usage_tracker/improve/rubrics/avoidable_recovery.md +20 -0
  41. agent_usage_tracker/improve/rubrics/code_quality.md +22 -0
  42. agent_usage_tracker/improve/rubrics/completion_supported.md +20 -0
  43. agent_usage_tracker/improve/rubrics/redundant_tests.md +21 -0
  44. agent_usage_tracker/improve/rubrics/verbosity.md +21 -0
  45. agent_usage_tracker/improve/runtime.py +117 -0
  46. agent_usage_tracker/improve/schedule.py +291 -0
  47. agent_usage_tracker/improve/search_check.py +563 -0
  48. agent_usage_tracker/improve/store.py +244 -0
  49. agent_usage_tracker/improve/studies.py +285 -0
  50. agent_usage_tracker/improve/study_cli.py +46 -0
  51. agent_usage_tracker/improve/study_decision.py +121 -0
  52. agent_usage_tracker/improve/study_followup.py +174 -0
  53. agent_usage_tracker/improve/t3.py +269 -0
  54. agent_usage_tracker/improve/trial.py +318 -0
  55. agent_usage_tracker/improve/validation.py +148 -0
  56. agent_usage_tracker/inspector.py +350 -0
  57. agent_usage_tracker/label/__init__.py +22 -0
  58. agent_usage_tracker/label/backends.py +742 -0
  59. agent_usage_tracker/label/bakeoff.py +661 -0
  60. agent_usage_tracker/label/baseline.py +173 -0
  61. agent_usage_tracker/label/clef.py +258 -0
  62. agent_usage_tracker/label/context.py +382 -0
  63. agent_usage_tracker/label/core.py +355 -0
  64. agent_usage_tracker/label/covered_run.py +92 -0
  65. agent_usage_tracker/label/evalset.py +154 -0
  66. agent_usage_tracker/label/gold.py +209 -0
  67. agent_usage_tracker/label/local.py +326 -0
  68. agent_usage_tracker/label/local_parallel.py +232 -0
  69. agent_usage_tracker/label/merge.py +57 -0
  70. agent_usage_tracker/label/outcome_coverage.py +57 -0
  71. agent_usage_tracker/label/replay.py +194 -0
  72. agent_usage_tracker/label/run.py +339 -0
  73. agent_usage_tracker/label/tasks.py +579 -0
  74. agent_usage_tracker/label/teach.py +145 -0
  75. agent_usage_tracker/local_page.py +128 -0
  76. agent_usage_tracker/manual_time.py +69 -0
  77. agent_usage_tracker/net.py +23 -0
  78. agent_usage_tracker/observation_cli.py +65 -0
  79. agent_usage_tracker/observation_report.py +116 -0
  80. agent_usage_tracker/observation_server.py +110 -0
  81. agent_usage_tracker/observations.py +365 -0
  82. agent_usage_tracker/outcome_git.py +383 -0
  83. agent_usage_tracker/outcomes.py +464 -0
  84. agent_usage_tracker/pricing.py +248 -0
  85. agent_usage_tracker/push.py +530 -0
  86. agent_usage_tracker/redact.py +45 -0
  87. agent_usage_tracker/repo_keys.py +165 -0
  88. agent_usage_tracker/review.py +252 -0
  89. agent_usage_tracker/scan.py +472 -0
  90. agent_usage_tracker/scheduled.py +164 -0
  91. agent_usage_tracker/sdk.py +79 -0
  92. agent_usage_tracker/sink.py +82 -0
  93. agent_usage_tracker/sink_stack/README.md +71 -0
  94. agent_usage_tracker/sink_stack/apply.sh +63 -0
  95. agent_usage_tracker/sink_stack/compose.yaml +94 -0
  96. agent_usage_tracker/sink_stack/env.example +20 -0
  97. agent_usage_tracker/sink_stack/grafana/dashboards/agents-effort.json +634 -0
  98. agent_usage_tracker/sink_stack/grafana/dashboards/agents-usage.json +1279 -0
  99. agent_usage_tracker/sink_stack/grafana/provisioning/dashboards/aut.yaml +9 -0
  100. agent_usage_tracker/sink_stack/grafana/provisioning/datasources/agents.yaml +16 -0
  101. agent_usage_tracker/sink_stack/sql/01_schema.sql +1186 -0
  102. agent_usage_tracker/sink_stack/sql/02_price.sql +60 -0
  103. agent_usage_tracker/sink_stack/sql/03_effort.sql +1180 -0
  104. agent_usage_tracker/sink_stack/sql/04_grants.sql +149 -0
  105. agent_usage_tracker/sink_stack/sql/refresh.sql +66 -0
  106. agent_usage_tracker/sources.py +180 -0
  107. agent_usage_tracker/state.py +191 -0
  108. agent_usage_tracker/telemetry.py +165 -0
  109. agent_usage_tracker-0.1.0.dist-info/METADATA +366 -0
  110. agent_usage_tracker-0.1.0.dist-info/RECORD +113 -0
  111. agent_usage_tracker-0.1.0.dist-info/WHEEL +4 -0
  112. agent_usage_tracker-0.1.0.dist-info/entry_points.txt +3 -0
  113. agent_usage_tracker-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,173 @@
1
+ """Import provisional historical labels separately from calibrated nightly labels."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import math
8
+ import re
9
+ from collections import Counter
10
+ from datetime import datetime, timezone
11
+ from pathlib import Path
12
+ from typing import Any
13
+
14
+ from ..config import AutError, Config
15
+ from ..event import ActivityEvent, make_event_id
16
+ from ..push import send_activity_rows
17
+
18
+ POLICY = "working-sonnet-v1"
19
+ _MODEL = "claude-sonnet-5-5-medium"
20
+ _LABELS = {
21
+ "steer_intent": {
22
+ "framing",
23
+ "instruction",
24
+ "clarification",
25
+ "correction",
26
+ "redirection",
27
+ "review",
28
+ "acceptance",
29
+ "stop",
30
+ "other",
31
+ },
32
+ "soft_outcome": {"resolved", "still_open", "blocked", "ruled_out"},
33
+ }
34
+ _SIGNALS = {
35
+ "steer_intent": {"explicit_halt", "disputes_agent_work"},
36
+ "soft_outcome": {"required_next_step", "external_blocker"},
37
+ }
38
+
39
+
40
+ def _check(condition: bool) -> None:
41
+ if not condition:
42
+ raise ValueError("invalid provisional baseline")
43
+
44
+
45
+ def _timestamp(value: Any) -> str:
46
+ _check(isinstance(value, str))
47
+ _check(datetime.fromisoformat(value).utcoffset() is not None)
48
+ return str(value)
49
+
50
+
51
+ def _events(bundle: Any, machine: str, labelled_at: str) -> list[ActivityEvent]:
52
+ _check(bundle["version"] == "working-sonnet-baseline-v1")
53
+ _check(isinstance(bundle["rows"], list) and bool(bundle["rows"]))
54
+ baseline_id = hashlib.sha256(
55
+ json.dumps(bundle, sort_keys=True, allow_nan=False).encode()
56
+ ).hexdigest()
57
+ stamped = _timestamp(labelled_at)
58
+ result, seen = [], set()
59
+ for row in bundle["rows"]:
60
+ task, unit_id = row["task"], row["unit_id"]
61
+ _check(
62
+ task in _LABELS
63
+ and isinstance(unit_id, str)
64
+ and re.fullmatch(r"[A-Za-z0-9._:-]{1,200}", unit_id) is not None
65
+ )
66
+ _check(row["unit"] == ("steer" if task == "steer_intent" else "episode"))
67
+ _check((task, unit_id) not in seen)
68
+ seen.add((task, unit_id))
69
+ _check(
70
+ row["quality"] == "provisional"
71
+ and row["estimate"] is True
72
+ and row["human_verified"] is False
73
+ )
74
+ _check(
75
+ row["label"] in _LABELS[task] | {"unknown"}
76
+ and row["baseline_label"] in _LABELS[task] | {"unknown"}
77
+ )
78
+ _check(row["label_source"] in {"sonnet", "unknown_review", "unknown_input"})
79
+ _check(row["label_source"] == "sonnet" or row["label"] == "unknown")
80
+ _check(
81
+ row["model_id"] == _MODEL
82
+ or (
83
+ row["model_id"] is None
84
+ and row["label"] == "unknown"
85
+ and row["baseline_label"] == "unknown"
86
+ )
87
+ )
88
+ weight = row["weight"]
89
+ _check(type(weight) in (int, float) and math.isfinite(weight) and weight > 0)
90
+ _check(
91
+ isinstance(row["input_hash"], str)
92
+ and re.fullmatch(r"[a-f0-9]{64}", row["input_hash"]) is not None
93
+ )
94
+ signals = {}
95
+ _check(isinstance(row["reviewed_signals"], dict))
96
+ for question, answer in row["reviewed_signals"].items():
97
+ _check(question in _SIGNALS[task] and answer["answer"] in {"yes", "no", "unknown"})
98
+ _check(answer["source"] == "agent_review" and answer["human_verified"] is False)
99
+ _check(
100
+ isinstance(answer["question_sha256"], str)
101
+ and re.fullmatch(r"[a-f0-9]{64}", answer["question_sha256"]) is not None
102
+ )
103
+ signals[question] = {
104
+ "answer": answer["answer"],
105
+ "source": "agent_review",
106
+ "question_sha256": answer["question_sha256"],
107
+ }
108
+ _check(
109
+ not any(value["answer"] == "unknown" for value in signals.values())
110
+ or row["label"] == "unknown"
111
+ )
112
+ # Only ids, enums and numbers cross the sink boundary, never review reasons or paths.
113
+ attrs = {
114
+ "task": "working." + task,
115
+ "task_ver": 1,
116
+ "unit": row["unit"],
117
+ "unit_id": unit_id,
118
+ "label": row["label"],
119
+ "baseline_label": row["baseline_label"],
120
+ "p": None,
121
+ "below_tau": row["label"] == "unknown",
122
+ "model_id": row["model_id"],
123
+ "label_policy": POLICY,
124
+ "baseline_id": baseline_id,
125
+ "labelled_at": stamped,
126
+ "label_source": row["label_source"],
127
+ "population_weight": weight,
128
+ "input_hash": row["input_hash"],
129
+ "provisional": True,
130
+ "human_verified": False,
131
+ "signals": signals,
132
+ }
133
+ result.append(
134
+ ActivityEvent(
135
+ event_id=make_event_id("baseline", POLICY, baseline_id, task, unit_id),
136
+ ts=_timestamp(row["ts"]),
137
+ agent="label",
138
+ kind="label",
139
+ session_id=None,
140
+ turn_id=None,
141
+ attrs=attrs,
142
+ machine=machine,
143
+ )
144
+ )
145
+ return result
146
+
147
+
148
+ def events(bundle: Any, machine: str, labelled_at: str) -> list[ActivityEvent]:
149
+ """Validate the entire bundle before writing; repeated imports have stable event ids."""
150
+ try:
151
+ return _events(bundle, machine, labelled_at)
152
+ except (KeyError, TypeError, ValueError, OverflowError) as exc:
153
+ raise AutError("label import-baseline: invalid provisional baseline bundle") from exc
154
+
155
+
156
+ def run(cfg: Config, path: Path, *, apply: bool = False) -> dict[str, Any]:
157
+ try:
158
+ bundle = json.loads(path.read_text())
159
+ except (OSError, ValueError) as exc:
160
+ raise AutError("label import-baseline: could not read the baseline JSON") from exc
161
+ rows = events(bundle, cfg.machine, datetime.now(timezone.utc).isoformat())
162
+ if apply:
163
+ if not cfg.pg_dsn:
164
+ raise AutError("label import-baseline: transport.pg_dsn is required for --apply")
165
+ send_activity_rows(cfg.pg_dsn, rows)
166
+ return {
167
+ "mode": "applied" if apply else "preview",
168
+ "policy": POLICY,
169
+ "rows": len(rows),
170
+ "unknown": sum(e.attrs["label"] == "unknown" for e in rows),
171
+ "reviewed_signals": sum(len(e.attrs["signals"]) for e in rows),
172
+ "tasks": dict(Counter(e.attrs["task"] for e in rows)),
173
+ }
@@ -0,0 +1,258 @@
1
+ """Cloudflare Clef's joint schema reader on MLX, text only: the prompt encoding and the head.
2
+
3
+ Trimmed from clef_mlx.py in mlx-community/clef-flash-4bit (Apache-2.0), which mirrors the reference
4
+ joint_schema_model.encode_record. Left out: images, video, the server, and the mlx-vlm loader.
5
+ Imported only when clef-flash runs; it needs mlx from the extra aut[label].
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ import math
12
+ from dataclasses import dataclass
13
+ from pathlib import Path
14
+ from typing import Any, Callable
15
+
16
+ import mlx.core as mx
17
+ import mlx.nn as nn
18
+
19
+ SYSTEM_PROMPT = ("Read the complete state and schema. Decide every field jointly. Each answer "
20
+ "must be exactly one of that field's allowed options.")
21
+ QUESTION_TYPES = {"noul": 0, "choice": 1, "score": 2}
22
+ MAX_LENGTH = 16384
23
+
24
+
25
+ def render(value: Any) -> str:
26
+ if isinstance(value, str):
27
+ return value
28
+ return json.dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
29
+
30
+
31
+ def options_of(question: dict[str, Any]) -> list[tuple[str, str]]:
32
+ if question["type"] == "noul":
33
+ return [(k, question["criteria"][k]) for k in ("true", "false")]
34
+ return sorted((str(k), v) for k, v in question["criteria"].items())
35
+
36
+
37
+ @dataclass(frozen=True)
38
+ class Encoded:
39
+ question_type: int
40
+ question_span: tuple[int, int]
41
+ option_spans: tuple[tuple[int, int], ...]
42
+ option_ids: tuple[str, ...]
43
+
44
+
45
+ def encode_record(tok: Any, record: dict[str, Any]) -> tuple[list[int], list[Encoded]]:
46
+ """Token ids and the spans of each question and option. The state is cut to fit MAX_LENGTH,
47
+ like the reference."""
48
+ def tokens(text: str) -> list[int]:
49
+ return tok.encode(text, add_special_tokens=False)
50
+
51
+ schema = tokens("\n\nSCHEMA FIELDS:\n")
52
+ found: list[Encoded] = []
53
+ for qi, (qid, q) in enumerate(record["questions"].items()):
54
+ schema += tokens(f"\nFIELD {qi + 1}\nID: {qid}\nTYPE: {q['type']}\nINSTRUCTION: ")
55
+ q_start = len(schema)
56
+ schema += tokens(render(q["instructions"]))
57
+ q_end = len(schema)
58
+ schema += tokens("\nALLOWED OPTIONS:\n")
59
+ spans, ids = [], []
60
+ for oi, (option, description) in enumerate(options_of(q)):
61
+ schema += tokens(f"OPTION {oi + 1}: ")
62
+ start = len(schema)
63
+ schema += tokens(render({"option_id": option, "description": description}))
64
+ spans.append((start, len(schema)))
65
+ ids.append(option)
66
+ schema += tokens("\n")
67
+ schema += tokens("END FIELD\n")
68
+ found.append(Encoded(QUESTION_TYPES[q["type"]], (q_start, q_end), tuple(spans), tuple(ids)))
69
+ prefix = tokens(f"<|im_start|>system\n{SYSTEM_PROMPT}<|im_end|>\n<|im_start|>user\nSTATE:\n")
70
+ suffix = tokens("\n<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
71
+ "JOINT SCHEMA DECISIONS:")
72
+ state = tokens(render(record["state"]))[:max(0, MAX_LENGTH - len(prefix) - len(schema)
73
+ - len(suffix))]
74
+ off = len(prefix) + len(state)
75
+ return prefix + state + schema + suffix, [
76
+ Encoded(q.question_type, (q.question_span[0] + off, q.question_span[1] + off),
77
+ tuple((s + off, e + off) for s, e in q.option_spans), q.option_ids)
78
+ for q in found]
79
+
80
+
81
+ class MultiheadAttention(nn.Module):
82
+ """torch.nn.MultiheadAttention (batch_first, packed in_proj) in MLX."""
83
+
84
+ def __init__(self, width: int, heads: int):
85
+ super().__init__()
86
+ self.heads = heads
87
+ self.in_proj_weight = mx.zeros((3 * width, width))
88
+ self.in_proj_bias = mx.zeros((3 * width,))
89
+ self.out_proj = nn.Linear(width, width)
90
+
91
+ def __call__(self, q: mx.array, k: mx.array, v: mx.array) -> mx.array:
92
+ w, b = self.in_proj_weight, self.in_proj_bias
93
+ d = w.shape[1]
94
+ q = q @ w[:d].T + b[:d]
95
+ k = k @ w[d:2 * d].T + b[d:2 * d]
96
+ v = v @ w[2 * d:].T + b[2 * d:]
97
+ batch, lq, lk, hd = q.shape[0], q.shape[1], k.shape[1], d // self.heads
98
+ q = q.reshape(batch, lq, self.heads, hd).transpose(0, 2, 1, 3)
99
+ k = k.reshape(batch, lk, self.heads, hd).transpose(0, 2, 1, 3)
100
+ v = v.reshape(batch, lk, self.heads, hd).transpose(0, 2, 1, 3)
101
+ o = mx.fast.scaled_dot_product_attention(q, k, v, scale=hd ** -0.5)
102
+ return self.out_proj(o.transpose(0, 2, 1, 3).reshape(batch, lq, d))
103
+
104
+
105
+ class FeedForward(nn.Module):
106
+ def __init__(self, width: int, feedforward: int):
107
+ super().__init__()
108
+ self.fc1 = nn.Linear(width, feedforward)
109
+ self.fc2 = nn.Linear(feedforward, width)
110
+
111
+ def __call__(self, x: mx.array) -> mx.array:
112
+ return self.fc2(nn.gelu(self.fc1(x)))
113
+
114
+
115
+ class EvidenceRoutingLayer(nn.Module):
116
+ def __init__(self, width: int, heads: int, feedforward: int):
117
+ super().__init__()
118
+ self.query_norm = nn.LayerNorm(width)
119
+ self.memory_norm = nn.LayerNorm(width)
120
+ self.attention = MultiheadAttention(width, heads)
121
+ self.feedforward_norm = nn.LayerNorm(width)
122
+ self.feedforward = FeedForward(width, feedforward)
123
+
124
+ def __call__(self, queries: mx.array, memory: mx.array) -> mx.array:
125
+ m = self.memory_norm(memory)
126
+ queries = queries + self.attention(self.query_norm(queries), m, m)
127
+ return queries + self.feedforward(self.feedforward_norm(queries))
128
+
129
+
130
+ class TransformerDecoderLayer(nn.Module):
131
+ """torch.nn.TransformerDecoderLayer(norm_first=True, activation='gelu')."""
132
+
133
+ def __init__(self, width: int, heads: int, feedforward: int):
134
+ super().__init__()
135
+ self.self_attn = MultiheadAttention(width, heads)
136
+ self.multihead_attn = MultiheadAttention(width, heads)
137
+ self.linear1 = nn.Linear(width, feedforward)
138
+ self.linear2 = nn.Linear(feedforward, width)
139
+ self.norm1 = nn.LayerNorm(width)
140
+ self.norm2 = nn.LayerNorm(width)
141
+ self.norm3 = nn.LayerNorm(width)
142
+
143
+ def __call__(self, x: mx.array, memory: mx.array) -> mx.array:
144
+ h = self.norm1(x)
145
+ x = x + self.self_attn(h, h, h)
146
+ x = x + self.multihead_attn(self.norm2(x), memory, memory)
147
+ return x + self.linear2(nn.gelu(self.linear1(self.norm3(x))))
148
+
149
+
150
+ def _l2norm(x: mx.array, eps: float = 1e-12) -> mx.array:
151
+ return x / mx.maximum(mx.linalg.norm(x, axis=-1, keepdims=True), eps)
152
+
153
+
154
+ class JointSchemaHead(nn.Module):
155
+ def __init__(self, hidden_size: int, width: int, routing_layers: int, layers: int, heads: int,
156
+ feedforward: int):
157
+ super().__init__()
158
+ self.hidden_norm = nn.LayerNorm(hidden_size)
159
+ self.memory_projection = nn.Linear(hidden_size, width, bias=False)
160
+ self.question_projection = nn.Linear(hidden_size, width, bias=False)
161
+ self.option_question_projection = nn.Linear(hidden_size, width, bias=False)
162
+ self.global_projection = nn.Linear(hidden_size, width, bias=False)
163
+ self.option_context_projection = nn.Linear(hidden_size, width, bias=False)
164
+ self.option_lexical_projection = nn.Linear(hidden_size, width, bias=False)
165
+ self.type_embedding = nn.Embedding(3, width)
166
+ self.evidence_layers = [EvidenceRoutingLayer(width, heads, feedforward)
167
+ for _ in range(routing_layers)]
168
+ self.option_summary_norm = nn.LayerNorm(width)
169
+ self.layers = [TransformerDecoderLayer(width, heads, feedforward) for _ in range(layers)]
170
+ self.field_norm = nn.LayerNorm(width)
171
+ self.option_norm = nn.LayerNorm(width)
172
+ self.scorer1 = nn.Linear(width * 4, width)
173
+ self.scorer2 = nn.Linear(width, 1)
174
+ self.prior_logit_scale = mx.zeros(())
175
+ self.joint_logit_scale = mx.zeros(())
176
+ self.residual_gate = mx.zeros(())
177
+
178
+ def __call__(self, hidden: mx.array, input_ids: mx.array, questions: list[Encoded],
179
+ lexical: Callable[[mx.array], mx.array]) -> list[mx.array]:
180
+ """hidden: (L, H) final backbone states of one record. One logit array per question."""
181
+ h = self.hidden_norm(hidden)
182
+ memory = self.memory_projection(h)[None]
183
+ global_vector = h[-1]
184
+ question_vectors = mx.stack([h[q.question_span[0]:q.question_span[1]].mean(0)
185
+ for q in questions])
186
+ type_ids = mx.array([q.question_type for q in questions])
187
+ contexts = [mx.stack([h[s:e].mean(0) for s, e in q.option_spans]) for q in questions]
188
+ lexicals = [mx.stack([lexical(input_ids[s:e]).mean(0) for s, e in q.option_spans])
189
+ for q in questions]
190
+ counts = [len(q.option_spans) for q in questions]
191
+ routed = mx.concatenate([
192
+ self.option_context_projection(c) + self.option_lexical_projection(lx)
193
+ + self.option_question_projection(question_vectors[i])[None]
194
+ for i, (c, lx) in enumerate(zip(contexts, lexicals))], axis=0)[None]
195
+ for layer in self.evidence_layers:
196
+ routed = layer(routed, memory)
197
+ routed = routed[0]
198
+ splits = [int(x) for x in mx.cumsum(mx.array(counts))[:-1].tolist()]
199
+ split_options = mx.split(routed, splits, axis=0) if splits else [routed]
200
+
201
+ base_fields = self.question_projection(question_vectors)
202
+ summaries = []
203
+ for field, options in zip(base_fields, split_options):
204
+ w = mx.softmax((options @ field) / math.sqrt(options.shape[-1]), axis=0)
205
+ summaries.append((w[:, None] * options).sum(0))
206
+ fields = (base_fields + self.option_summary_norm(mx.stack(summaries))
207
+ + self.global_projection(global_vector)[None]
208
+ + self.type_embedding(type_ids))[None]
209
+ for layer in self.layers:
210
+ fields = layer(fields, memory)
211
+ fields = self.field_norm(fields[0])
212
+
213
+ prior_scale = mx.exp(mx.minimum(self.prior_logit_scale, math.log(100.0)))
214
+ joint_scale = mx.exp(mx.minimum(self.joint_logit_scale, math.log(100.0)))
215
+ gate = mx.sigmoid(self.residual_gate)
216
+ logits = []
217
+ for i, (field, lx, opts) in enumerate(zip(fields, lexicals, split_options)):
218
+ prior = prior_scale * (_l2norm(lx) @ _l2norm(question_vectors[i] + global_vector))
219
+ options = self.option_norm(opts)
220
+ rf = mx.broadcast_to(field[None], options.shape)
221
+ cosine = (rf * options).sum(-1) / mx.maximum(
222
+ mx.linalg.norm(rf, axis=-1) * mx.linalg.norm(options, axis=-1), 1e-8)
223
+ feats = mx.concatenate([rf, options, rf * options, mx.abs(rf - options)], axis=-1)
224
+ residual = self.scorer2(nn.gelu(self.scorer1(feats)))[:, 0]
225
+ logits.append(prior + gate * (joint_scale * cosine + residual))
226
+ return logits
227
+
228
+
229
+ def load_head(path: Path) -> JointSchemaHead:
230
+ head = JointSchemaHead(**json.loads((path / "joint_head_config.json").read_text()))
231
+ weights = {k.replace(".feedforward.0.", ".feedforward.fc1.")
232
+ .replace(".feedforward.3.", ".feedforward.fc2.")
233
+ .replace("residual_scorer.0.", "scorer1.")
234
+ .replace("residual_scorer.3.", "scorer2."): v
235
+ for k, v in mx.load(str(path / "joint_head.safetensors")).items()}
236
+ head.load_weights(list(weights.items()), strict=True)
237
+ head.set_dtype(mx.bfloat16)
238
+ mx.eval(head.parameters())
239
+ return head
240
+
241
+
242
+ def predict(model: Any, tok: Any, head: JointSchemaHead, record: dict[str, Any]) -> dict[str, float]:
243
+ """{option id: probability} for the one question of a record."""
244
+ lm = model.language_model
245
+ out = lm.lm_head if hasattr(lm, "lm_head") else lm.model.embed_tokens
246
+
247
+ def lexical(ids: mx.array) -> mx.array: # rows of the output matrix, dequantized if needed
248
+ if isinstance(out, (nn.QuantizedLinear, nn.QuantizedEmbedding)):
249
+ return mx.dequantize(out.weight[ids], out.scales[ids],
250
+ out.get("biases")[ids] if "biases" in out else None,
251
+ group_size=out.group_size, bits=out.bits,
252
+ mode=getattr(out, "mode", "affine"))
253
+ return out.weight[ids]
254
+
255
+ tokens, questions = encode_record(tok, record)
256
+ ids = mx.array(tokens)
257
+ logits = head(lm.model(ids[None])[0], ids, questions, lexical)[0]
258
+ return dict(zip(questions[0].option_ids, mx.softmax(logits.astype(mx.float32)).tolist()))