mcaligner 0.2.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
mcaligner/__init__.py ADDED
@@ -0,0 +1,5 @@
1
+ """Word alignment of parallel texts with LLM agents, on top of MCParser."""
2
+
3
+ from importlib.metadata import version
4
+
5
+ __version__ = version("mcaligner")
mcaligner/__main__.py ADDED
@@ -0,0 +1,3 @@
1
+ from .cli import app
2
+
3
+ app(prog_name="mcaligner")
@@ -0,0 +1 @@
1
+ """The pydantic-ai agents: the aligner and the scorer."""
@@ -0,0 +1,189 @@
1
+ """Compact answers from the agents, checked and applied in code.
2
+
3
+ The aligner answers with one line per link, the scorer with one line per
4
+ rating::
5
+
6
+ <id> 3-5 S 0.95 aligner: SENT_ID A-B S|P CONFIDENCE
7
+ <id> 3-5 0.9 scorer: SENT_ID A-B CONFIDENCE
8
+
9
+ Nothing is written until the whole answer is valid: unknown pairs, malformed
10
+ lines and links to words that do not exist (checked with
11
+ ``set_links(..., dry_run=True)``) go back to the model as a retry with the
12
+ list of problems.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import re
18
+ from dataclasses import dataclass, field
19
+
20
+ from mcparser.corpus import CorpusError, CorpusStore
21
+ from mcparser.corpus.alignment import Link
22
+ from pydantic import BaseModel, Field
23
+ from pydantic_ai import ModelRetry
24
+
25
+ from ..rules import is_punctuation
26
+
27
+ ALIGNER_AGENT = "aligner"
28
+ SCORER_AGENT = "scorer"
29
+
30
+ LINK_LINE = re.compile(r"^(\S+)\s+(\d+)-(\d+)\s*([SP])?\s+(\S+)$")
31
+ SCORE_LINE = re.compile(r"^(\S+)\s+(\d+)-(\d+)[SP]?\s+(\S+)$")
32
+ REMOVE_LINE = re.compile(r"^(\S+)\s+(\d+)-(\d+)[SP]?$")
33
+
34
+
35
+ class Alignment(BaseModel):
36
+ """Your answer: the links of the sentence pairs."""
37
+
38
+ links: str = Field(
39
+ description=(
40
+ 'One line per link, "SENT_ID A-B S|P CONFIDENCE", e.g. "<id> 3-5 S 0.95", with <id> the pair id exactly as in its "## Pair" heading. '
41
+ "Settled links are kept without listing them."
42
+ )
43
+ )
44
+ remove: str = Field("", description='Settled links that are wrong, one per line: "SENT_ID A-B".')
45
+ summary: str = Field("", description="At most 15 words.")
46
+
47
+
48
+ class Scores(BaseModel):
49
+ """Your answer: a confidence for each link to score."""
50
+
51
+ scores: str = Field(description='One line per link, "SENT_ID A-B CONFIDENCE", e.g. "<id> 3-5 0.9", with <id> the pair id exactly as in its "## Pair" heading.')
52
+ summary: str = Field("", description="At most 15 words.")
53
+
54
+
55
+ @dataclass
56
+ class TaskBatch:
57
+ """The pairs of one task; passed to the agent run as its dependencies."""
58
+
59
+ target: str
60
+ sent_ids: list[str]
61
+ settled: dict[str, list[Link]] = field(default_factory=dict)
62
+ """Aligner: the links shown as settled, per pair (they are kept)."""
63
+ review: dict[str, list[Link]] = field(default_factory=dict)
64
+ """Scorer: the links to score, per pair."""
65
+ dropped: int = 0
66
+ """Aligner: links to punctuation that code removed from the answer."""
67
+
68
+
69
+ def _lines(text: str) -> list[str]:
70
+ return [line.strip() for line in text.splitlines() if line.strip() and not line.lstrip().startswith("#")]
71
+
72
+
73
+ def _confidence(text: str) -> float | None:
74
+ try:
75
+ return min(max(float(text), 0.0), 1.0)
76
+ except ValueError:
77
+ return None
78
+
79
+
80
+ def parse_links(text: str, batch: TaskBatch) -> tuple[dict[str, list[Link]], list[str]]:
81
+ """``SENT_ID A-B S|P CONFIDENCE`` lines as links per pair, plus error messages."""
82
+ links: dict[str, list[Link]] = {}
83
+ errors = []
84
+ for number, line in enumerate(_lines(text), start=1):
85
+ match = LINK_LINE.match(line)
86
+ if match is None:
87
+ errors.append(f"links line {number}: expected SENT_ID A-B S|P CONFIDENCE, got {line!r}")
88
+ continue
89
+ sent_id, source, target, kind, confidence = match.groups()
90
+ score = _confidence(confidence)
91
+ if sent_id not in batch.sent_ids:
92
+ errors.append(f"links line {number}: pair {sent_id!r} is not in this task")
93
+ elif score is None:
94
+ errors.append(f"links line {number}: confidence {confidence!r} is not a number")
95
+ else:
96
+ links.setdefault(sent_id, []).append(Link(source=source, target=target, sure=kind != "P", confidence=score))
97
+ return links, errors
98
+
99
+
100
+ def parse_removals(text: str, batch: TaskBatch) -> tuple[dict[str, set[str]], list[str]]:
101
+ removals: dict[str, set[str]] = {}
102
+ errors = []
103
+ for number, line in enumerate(_lines(text), start=1):
104
+ match = REMOVE_LINE.match(line)
105
+ if match is None or match.group(1) not in batch.sent_ids:
106
+ errors.append(f"remove line {number}: expected SENT_ID A-B for a pair of this task, got {line!r}")
107
+ continue
108
+ sent_id, source, target = match.groups()
109
+ removals.setdefault(sent_id, set()).add(f"{source}-{target}")
110
+ return removals, errors
111
+
112
+
113
+ def parse_scores(text: str, batch: TaskBatch) -> tuple[dict[str, dict[str, float]], list[str]]:
114
+ """``SENT_ID A-B CONFIDENCE`` lines as {link key: confidence} per pair, plus error messages."""
115
+ scores: dict[str, dict[str, float]] = {}
116
+ errors = []
117
+ for number, line in enumerate(_lines(text), start=1):
118
+ match = SCORE_LINE.match(line)
119
+ if match is None:
120
+ errors.append(f"scores line {number}: expected SENT_ID A-B CONFIDENCE, got {line!r}")
121
+ continue
122
+ sent_id, source, target, confidence = match.groups()
123
+ key, score = f"{source}-{target}", _confidence(confidence)
124
+ if sent_id not in batch.sent_ids:
125
+ errors.append(f"scores line {number}: pair {sent_id!r} is not in this task")
126
+ elif score is None:
127
+ errors.append(f"scores line {number}: confidence {confidence!r} is not a number")
128
+ elif key not in {link.key for link in batch.review.get(sent_id, [])}:
129
+ errors.append(f"scores line {number}: {sent_id} {key} is not one of the links to score")
130
+ else:
131
+ scores.setdefault(sent_id, {})[key] = score
132
+ return scores, errors
133
+
134
+
135
+ class Applier:
136
+ """Checks an agent's answer against the store, then applies it."""
137
+
138
+ def __init__(self, store: CorpusStore, link_punctuation: bool = False):
139
+ self.store = store
140
+ self.link_punctuation = link_punctuation
141
+
142
+ def without_punctuation(self, batch: TaskBatch, links: dict[str, list[Link]]) -> dict[str, list[Link]]:
143
+ """The answer's links without those that touch punctuation (counted in ``batch.dropped``)."""
144
+ aset = self.store.alignment(batch.target)
145
+ kept, batch.dropped = {}, 0 # reset: a retried answer is filtered again
146
+ for sent_id, group in links.items():
147
+ skip_a = {w.id for w in aset.source_a.sentence(sent_id).words if is_punctuation(w)}
148
+ skip_b = {w.id for w in aset.source_b.sentence(sent_id).words if is_punctuation(w)}
149
+ kept[sent_id] = [link for link in group if link.source not in skip_a and link.target not in skip_b]
150
+ batch.dropped += len(group) - len(kept[sent_id])
151
+ return kept
152
+
153
+ @staticmethod
154
+ def _retry(errors: list[str]) -> None:
155
+ if errors:
156
+ raise ModelRetry("Fix these problems and answer again:\n" + "\n".join(f"- {e}" for e in errors[:20]))
157
+
158
+ def merged(self, batch: TaskBatch, sent_id: str, links: list[Link], removals: set[str]) -> list[Link]:
159
+ """Settled links (minus removals) plus the answer's links (which win on the same word pair)."""
160
+ result = {link.key: link for link in batch.settled.get(sent_id, []) if link.key not in removals}
161
+ result.update({link.key: link for link in links})
162
+ return list(result.values())
163
+
164
+ def alignment(self, answer: Alignment, batch: TaskBatch) -> Alignment:
165
+ links, errors = parse_links(answer.links, batch)
166
+ if not self.link_punctuation and not errors:
167
+ links = self.without_punctuation(batch, links)
168
+ removals, removal_errors = parse_removals(answer.remove, batch)
169
+ errors += removal_errors
170
+ new = {sent_id: self.merged(batch, sent_id, links.get(sent_id, []), removals.get(sent_id, set())) for sent_id in batch.sent_ids}
171
+ for sent_id, merged in new.items():
172
+ try:
173
+ self.store.set_links(batch.target, sent_id, merged, agent=ALIGNER_AGENT, dry_run=True)
174
+ except CorpusError as error:
175
+ errors.append(f"pair {sent_id}: {error}")
176
+ self._retry(errors)
177
+ for sent_id, merged in new.items(): # every pair is written, so the history shows the aligner saw it
178
+ self.store.set_links(batch.target, sent_id, merged, reason=answer.summary, agent=ALIGNER_AGENT)
179
+ return answer
180
+
181
+ def scores(self, answer: Scores, batch: TaskBatch) -> Scores:
182
+ scores, errors = parse_scores(answer.scores, batch)
183
+ self._retry(errors)
184
+ for sent_id, keyed in scores.items():
185
+ rated = [
186
+ link.model_copy(update={"confidence": keyed[link.key]}) for link in batch.review[sent_id] if link.key in keyed
187
+ ]
188
+ self.store.set_links(batch.target, sent_id, rated, replace=False, reason=answer.summary, agent=SCORER_AGENT)
189
+ return answer
@@ -0,0 +1,237 @@
1
+ """Instructions and task prompts for the aligner and the scorer.
2
+
3
+ A task covers several sentence pairs. Each pair is rendered compactly: one line
4
+ per word (``id form lemma UPOS head deprel`` and glosses from MISC), the links
5
+ already settled, the links to review, and for each unlinked word how its lemma
6
+ is linked elsewhere (translation evidence). The answers are compact lines that
7
+ code checks and applies (see :mod:`mcaligner.agents.outputs`).
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ from dataclasses import dataclass, field
13
+
14
+ from mcparser.corpus import Sentence
15
+ from mcparser.corpus.alignment import Link
16
+
17
+ from ..lexicon import TranslationTable, lemma_of
18
+ from ..settings import COLUMNS, DEFAULT_COLUMNS, TOOL_NAMES
19
+
20
+
21
+ @dataclass
22
+ class PairItem:
23
+ """One sentence pair of a task."""
24
+
25
+ sent_id: str
26
+ a: Sentence
27
+ b: Sentence
28
+ settled: list[Link] = field(default_factory=list)
29
+ """Links kept as they are (at or above the threshold)."""
30
+ review: list[Link] = field(default_factory=list)
31
+ """Links below the threshold (aligner) or links to score (scorer)."""
32
+
33
+ @property
34
+ def words(self) -> int:
35
+ return len(self.a.words) + len(self.b.words)
36
+
37
+
38
+ ALIGNER = """\
39
+ You align the words of two versions of the same text: source A ({a}) and
40
+ source B ({b}), for example a translation and its original. Sentences are
41
+ paired by id. Words are numbered as in CoNLL-U; a multiword token (e.g. a word
42
+ with clitics) is split into syntactic words, and links join words, not tokens.
43
+
44
+ A link joins a word of A and a word of B that translate each other: "A-B S"
45
+ (sure) or "A-B P" (probable: a looser or partial correspondence, e.g. a
46
+ paraphrase). One-to-many and many-to-many alignments are several links (2-3 S
47
+ and 2-4 S when word 2 of A renders words 3 and 4 of B). A word may contain
48
+ morphemes that the parse did not split off (a pronoun suffix, an attached
49
+ preposition or article, the person of a verb): link it to every word it
50
+ renders, e.g. a verb with an object suffix "blessed-us" to both the verb and
51
+ the pronoun. Leave a word unlinked when it has no counterpart (a word the
52
+ translator added, or a function word the other language leaves out).
53
+
54
+ The task gives, for each pair: both sentences ({columns}, glosses where
55
+ available), the settled links (by code rules or earlier work; they are kept),
56
+ links to review, and for unlinked words how their lemma is linked elsewhere in
57
+ the corpus. Follow that evidence unless the context rules it out. {tools}
58
+
59
+ Answer with ONE final result. In `links`, one line per link:
60
+ "SENT_ID A-B S|P CONFIDENCE", e.g. "<id> 3-5 S 0.95", with <id> the pair id exactly as in its "## Pair" heading. List every link of
61
+ each pair except the settled ones, which are kept automatically; a link to
62
+ review that you do not list is removed. To remove a settled link that is
63
+ wrong, put "SENT_ID A-B" in `remove`. Confidence: 0.95 or more = clear; 0.8 =
64
+ probable; 0.6 = doubtful; below 0.5 = a guess. Keep `summary` under 15 words.
65
+ """
66
+
67
+ SCORER = """\
68
+ You check word alignments between two versions of the same text: source A
69
+ ({a}) and source B ({b}). Sentences are paired by id; words are numbered as in
70
+ CoNLL-U. A link "A-B S|P" says that word A and word B translate each other
71
+ (S = sure, P = probable, a looser correspondence). A word may contain
72
+ morphemes the parse did not split off (a pronoun suffix, an attached
73
+ preposition, the person of a verb), so a link from such a word to a pronoun
74
+ or preposition on the other side is right when that morpheme renders it.
75
+
76
+ For each pair the task gives both sentences ({columns}, glosses where
77
+ available), all its links, and the links to score. Rate each
78
+ link to score: how likely is it that these two words correspond in this
79
+ sentence? 0.95 or more = clearly right; 0.8 = probably right; 0.6 = doubtful;
80
+ below 0.5 = probably wrong (e.g. the wrong one of two similar words, or a word
81
+ that has a better counterpart). Links below {threshold} go back to the aligner. {tools}
82
+
83
+ Answer with ONE final result: in `scores`, one line per link to score,
84
+ "SENT_ID A-B CONFIDENCE", e.g. "<id> 3-5 0.9", with <id> the pair id exactly as in its "## Pair" heading. Keep `summary` under 15 words.
85
+ """
86
+
87
+
88
+ GUIDELINES = """\
89
+ ## Alignment guidelines
90
+
91
+ The human annotators of this text followed these guidelines. Apply them: they
92
+ decide which words are linked and which are left unlinked.
93
+
94
+ {guidelines}
95
+ """
96
+
97
+
98
+ COLUMN_NAMES = {"upos": "UPOS", "xpos": "XPOS", "feats": "FEATS"}
99
+
100
+
101
+ def column_text(columns: list[str] | tuple[str, ...]) -> str:
102
+ """``id form lemma UPOS head deprel`` for the columns shown."""
103
+ return " ".join(["id", "form", *(COLUMN_NAMES.get(name, name) for name in columns)])
104
+
105
+
106
+ def tools_text(tools: set[str] | frozenset[str] | list[str] | None, columns: list[str] | tuple[str, ...]) -> str:
107
+ """What the instructions say about the tools the agent has (None = all of them)."""
108
+ names = TOOL_NAMES if tools is None else set(tools)
109
+ if not names:
110
+ return "No tools are available: answer from the task."
111
+ text = (
112
+ f"Tools are available ({', '.join(sorted(names))}) but call them only when the evidence given is not enough, "
113
+ "and make independent calls in parallel."
114
+ )
115
+ hidden = [COLUMN_NAMES.get(name, name) for name in COLUMNS if name not in columns]
116
+ if "pair_words" in names and hidden:
117
+ text += f" The prompt omits {', '.join(hidden)}: pair_words gives every column of a pair's words"
118
+ text += (
119
+ ", and category_links shows how words of a category (UPOS, XPOS, FEATS such as Case=Gen, deprel) are "
120
+ "usually linked or left unlinked. Use them when the grammar decides a link (an article, a case against a "
121
+ "preposition, tense and voice, a participle)."
122
+ if "category_links" in names
123
+ else "."
124
+ )
125
+ return text
126
+
127
+
128
+ def instructions(
129
+ role: str,
130
+ a: str,
131
+ b: str,
132
+ target: str,
133
+ threshold: float,
134
+ extra: str | None = None,
135
+ guidelines: str | None = None,
136
+ columns: list[str] | tuple[str, ...] = DEFAULT_COLUMNS,
137
+ tools: list[str] | None = None,
138
+ ) -> str:
139
+ template = ALIGNER if role == "aligner" else SCORER
140
+ text = template.format(
141
+ a=a, b=b, target=target, threshold=threshold, columns=column_text(columns), tools=tools_text(tools, columns)
142
+ )
143
+ if guidelines:
144
+ text = f"{text}\n{GUIDELINES.format(guidelines=guidelines.strip())}"
145
+ return f"{text}\n{extra.strip()}\n" if extra else text
146
+
147
+
148
+ def word_lines(sentence: Sentence, misc_keys: list[str], columns: list[str] | tuple[str, ...] = DEFAULT_COLUMNS) -> list[str]:
149
+ """``id form`` and the ``columns`` (default lemma UPOS head deprel), then the MISC values, per word; a line per multiword token."""
150
+ lines = []
151
+ for token in sentence.tokens:
152
+ if token.is_multiword:
153
+ lines.append(f"{token.id} {token.form} (token of words {token.id})")
154
+ elif token.is_word:
155
+ extra = [value for key in misc_keys if (value := token.misc_value(key))]
156
+ lines.append(" ".join([token.id, token.form, *(getattr(token, name) for name in columns), *extra]))
157
+ return lines
158
+
159
+
160
+ def link_text(link: Link, confidence: float) -> str:
161
+ agent = f" {link.agent}" if link.agent else ""
162
+ return f"{link.ugarit()} {confidence:.2f}{agent}"
163
+
164
+
165
+ def evidence_lines(item: PairItem, table: TranslationTable, source_a: str, source_b: str, limit: int) -> list[str]:
166
+ """For unlinked words (not punctuation): their lemma's linked lemmas elsewhere (this pair left out)."""
167
+ if limit <= 0:
168
+ return []
169
+ linked_a = {link.source for link in item.settled + item.review}
170
+ linked_b = {link.target for link in item.settled + item.review}
171
+ lines = []
172
+ for side, source, sentence, linked in (("A", source_a, item.a, linked_a), ("B", source_b, item.b, linked_b)):
173
+ seen: set[str] = set()
174
+ for word in sentence.words:
175
+ lemma = lemma_of(word)
176
+ if word.id in linked or lemma in seen or word.upos == "PUNCT":
177
+ continue
178
+ seen.add(lemma)
179
+ entry = table.entry(source, lemma)
180
+ if entry is None:
181
+ continue
182
+ candidates = entry.candidates(item.sent_id).most_common(limit)
183
+ if not candidates:
184
+ continue
185
+ unlinked = entry.unaligned_without(item.sent_id)
186
+ shown = ", ".join(f"{value}×{count}" for value, count in candidates)
187
+ note = f"of {entry.occurrences_without(item.sent_id)}" + (f", {unlinked} unlinked" if unlinked else "")
188
+ lines.append(f"{side}{word.id} {lemma}: {shown} ({note})")
189
+ return lines
190
+
191
+
192
+ def pair_block(
193
+ item: PairItem,
194
+ role: str,
195
+ confidence: dict[str, float],
196
+ table: TranslationTable | None,
197
+ names: tuple[str, str],
198
+ misc_keys: list[str],
199
+ candidates: int,
200
+ columns: list[str] | tuple[str, ...] = DEFAULT_COLUMNS,
201
+ ) -> str:
202
+ """The text of one pair in a task prompt."""
203
+ lines = [f"## Pair {item.sent_id}", f"A: {item.a.text}"]
204
+ lines += word_lines(item.a, misc_keys, columns)
205
+ lines += [f"B: {item.b.text}", *word_lines(item.b, misc_keys, columns)]
206
+ settled = ", ".join(link_text(link, confidence[link.key]) for link in item.settled) or "none"
207
+ lines.append(f"Settled links: {settled}")
208
+ if role == "aligner":
209
+ if item.review:
210
+ lines.append("Links to review: " + ", ".join(link_text(link, confidence[link.key]) for link in item.review))
211
+ hints = evidence_lines(item, table, *names, candidates) if table is not None else []
212
+ if hints:
213
+ lines += ["Evidence for unlinked words (linked lemmas elsewhere):", *hints]
214
+ else:
215
+ forms_a = {word.id: word.form for word in item.a.words}
216
+ forms_b = {word.id: word.form for word in item.b.words}
217
+ lines.append("Links to score:")
218
+ lines += [f"{link.ugarit()} {forms_a[link.source]}→{forms_b[link.target]}" for link in item.review]
219
+ return "\n".join(lines)
220
+
221
+
222
+ def task_prompt(
223
+ role: str,
224
+ items: list[PairItem],
225
+ confidence: dict[str, dict[str, float]],
226
+ table: TranslationTable | None,
227
+ names: tuple[str, str],
228
+ misc_keys: list[str],
229
+ candidates: int,
230
+ columns: list[str] | tuple[str, ...] = DEFAULT_COLUMNS,
231
+ ) -> str:
232
+ header = (
233
+ f"Align these {len(items)} sentence pairs." if role == "aligner" else f"Score the listed links of these {len(items)} sentence pairs."
234
+ )
235
+ header += f' Use the pair ids exactly as in the "## Pair" headings ({", ".join(item.sent_id for item in items)}), without any prefix.'
236
+ blocks = [pair_block(item, role, confidence[item.sent_id], table, names, misc_keys, candidates, columns) for item in items]
237
+ return "\n\n".join([header, *blocks])
@@ -0,0 +1,165 @@
1
+ """The pydantic-ai agents (aligner, scorer), sharing one MCP connection to the store."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Mapping
6
+ from typing import Any
7
+
8
+ from mcparser.corpus import CorpusStore
9
+ from mcparser.pricing import estimate
10
+ from pydantic import BaseModel, Field
11
+ from pydantic_ai import Agent, RunContext
12
+ from pydantic_ai.mcp import MCPToolset
13
+ from pydantic_ai.messages import ModelResponse, RetryPromptPart, ToolCallPart
14
+ from pydantic_ai.models import Model
15
+ from pydantic_ai.settings import ModelSettings
16
+ from pydantic_ai.toolsets import AbstractToolset
17
+ from pydantic_ai.usage import RunUsage, UsageLimits
18
+
19
+ from ..mcp import build_server
20
+ from ..settings import Settings
21
+ from .outputs import Alignment, Applier, Scores, TaskBatch
22
+ from .prompts import instructions
23
+ from .toolset import READ_TOOLS, read_toolset
24
+
25
+ Answer = Alignment | Scores
26
+
27
+
28
+ class Details(BaseModel):
29
+ """What happened inside one agent task."""
30
+
31
+ requests: int = 0
32
+ tools: dict[str, int] = Field(default_factory=dict, description="Tool calls by tool name.")
33
+ retries: list[str] = Field(default_factory=list, description="Why the model was asked again.")
34
+ dropped: int = Field(0, description="Links to punctuation that code removed from the answer.")
35
+ USAGE_FIELDS = ("tasks", "requests", "input_tokens", "cache_read_tokens", "cache_write_tokens", "output_tokens")
36
+
37
+
38
+ class ModelUsage(BaseModel):
39
+ """Token use of one model over a run."""
40
+
41
+ tasks: int = 0
42
+ requests: int = 0
43
+ input_tokens: int = 0
44
+ cache_read_tokens: int = 0
45
+ cache_write_tokens: int = 0
46
+ output_tokens: int = 0
47
+ cost: float | None = Field(None, description="Estimated US dollars (see mcparser.pricing).")
48
+
49
+ def add(self, usage: RunUsage, model: str) -> None:
50
+ self.tasks += 1
51
+ self.requests += usage.requests
52
+ self.input_tokens += usage.input_tokens
53
+ self.cache_read_tokens += usage.cache_read_tokens
54
+ self.cache_write_tokens += usage.cache_write_tokens
55
+ self.output_tokens += usage.output_tokens
56
+ self.cost = estimate(model, self.input_tokens, self.cache_read_tokens, self.cache_write_tokens, self.output_tokens)
57
+
58
+
59
+ class AlignerTeam:
60
+ """Builds and runs the agents for one writable alignment set (``target``).
61
+
62
+ By default the agents reach the store through an in-process MCP server;
63
+ pass ``toolset`` to use another MCP connection. ``models`` maps a role (or
64
+ ``"*"``) to a model name or instance, overriding the settings; tests use it
65
+ for scripted models.
66
+ """
67
+
68
+ def __init__(
69
+ self,
70
+ store: CorpusStore,
71
+ target: str,
72
+ settings: Settings,
73
+ models: Mapping[str, Model | str] | None = None,
74
+ toolset: AbstractToolset[Any] | None = None,
75
+ ):
76
+ self.store = store
77
+ self.target = target
78
+ self.settings = settings
79
+ self.models = dict(models or {})
80
+ self.toolset = toolset or MCPToolset(build_server(store))
81
+ self.applier = Applier(store, settings.rules.link_punctuation)
82
+ self._agents: dict[str, Agent[TaskBatch, Answer]] = {}
83
+ self.usage: dict[str, ModelUsage] = {}
84
+ """Token use and estimated cost per model."""
85
+
86
+ def model(self, role: str) -> Model | str:
87
+ return self.models.get(role) or self.models.get("*") or self.settings.model_for(role)
88
+
89
+ @staticmethod
90
+ def model_name(model: Model | str) -> str:
91
+ return model if isinstance(model, str) else model.model_name
92
+
93
+ def model_settings(self, role: str) -> ModelSettings:
94
+ settings = ModelSettings(cache=self.settings.cache)
95
+ thinking = self.settings.thinking_for(role)
96
+ if thinking is not None:
97
+ settings["thinking"] = thinking
98
+ return settings
99
+
100
+ def agent(self, role: str) -> Agent[TaskBatch, Answer]:
101
+ if role not in self._agents:
102
+ aset = self.store.alignment(self.target)
103
+ agent = Agent(
104
+ self.model(role),
105
+ name=role,
106
+ deps_type=TaskBatch,
107
+ instructions=instructions(
108
+ role, aset.source_a.name, aset.source_b.name, self.target, self.settings.threshold,
109
+ self.settings.role(role).instructions, self.settings.guidelines_text,
110
+ self.settings.columns, self.settings.tools,
111
+ ), # fmt: skip
112
+ output_type=Alignment if role == "aligner" else Scores,
113
+ toolsets=[read_toolset(self.toolset, self.settings.tools)],
114
+ retries=3,
115
+ model_settings=self.model_settings(role),
116
+ defer_model_check=True,
117
+ )
118
+ apply = self.applier.alignment if role == "aligner" else self.applier.scores
119
+
120
+ @agent.output_validator
121
+ def check_and_apply(ctx: RunContext[TaskBatch], answer: Answer) -> Answer:
122
+ return apply(answer, ctx.deps) # type: ignore[arg-type]
123
+
124
+ self._agents[role] = agent
125
+ return self._agents[role]
126
+
127
+ async def __aenter__(self) -> AlignerTeam:
128
+ await self.toolset.__aenter__()
129
+ return self
130
+
131
+ async def __aexit__(self, *args: Any) -> None:
132
+ await self.toolset.__aexit__(*args)
133
+
134
+ async def run(self, role: str, prompt: str, batch: TaskBatch, details: Details | None = None) -> Answer:
135
+ """Run ``role`` on a task; the answer is applied when it is accepted.
136
+
137
+ If ``details`` is given, the task's requests, tool calls (by tool) and the messages that made the model
138
+ retry are added to it.
139
+ """
140
+ result = await self.agent(role).run(
141
+ prompt, deps=batch, usage_limits=UsageLimits(request_limit=self.settings.request_limit)
142
+ )
143
+ name = self.model_name(self.model(role))
144
+ self.usage.setdefault(name, ModelUsage()).add(result.usage, name)
145
+ if details is not None:
146
+ details.dropped += batch.dropped
147
+ details.requests += result.usage.requests
148
+ for message in result.all_messages():
149
+ parts = message.parts
150
+ if isinstance(message, ModelResponse):
151
+ for part in parts:
152
+ if isinstance(part, ToolCallPart) and part.tool_name in READ_TOOLS:
153
+ details.tools[part.tool_name] = details.tools.get(part.tool_name, 0) + 1
154
+ else: # a ModelRequest
155
+ details.retries += [str(part.content)[:300] for part in parts if isinstance(part, RetryPromptPart)]
156
+ return result.output
157
+
158
+ def total(self) -> ModelUsage:
159
+ """Usage summed over all models (cost is None if any model has no price)."""
160
+ total = ModelUsage(cost=0.0)
161
+ for usage in self.usage.values():
162
+ for name in USAGE_FIELDS:
163
+ setattr(total, name, getattr(total, name) + getattr(usage, name))
164
+ total.cost = None if total.cost is None or usage.cost is None else total.cost + usage.cost
165
+ return total