mcaligner 0.2.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mcaligner/__init__.py +5 -0
- mcaligner/__main__.py +3 -0
- mcaligner/agents/__init__.py +1 -0
- mcaligner/agents/outputs.py +189 -0
- mcaligner/agents/prompts.py +237 -0
- mcaligner/agents/team.py +165 -0
- mcaligner/agents/toolset.py +139 -0
- mcaligner/cli.py +298 -0
- mcaligner/evaluate.py +147 -0
- mcaligner/guidelines/bicross.md +140 -0
- mcaligner/guidelines/got-lat.md +132 -0
- mcaligner/guidelines/grc-ar.md +146 -0
- mcaligner/guidelines/grc-cop.md +166 -0
- mcaligner/guidelines/grc-lat.md +162 -0
- mcaligner/guidelines/grc-syr.md +65 -0
- mcaligner/guidelines/lat-ar.md +73 -0
- mcaligner/guidelines/syr-ar.md +57 -0
- mcaligner/lexicon.py +151 -0
- mcaligner/mcp/__init__.py +5 -0
- mcaligner/mcp/cli.py +67 -0
- mcaligner/mcp/server.py +123 -0
- mcaligner/mcp/tools.py +307 -0
- mcaligner/pipeline.py +286 -0
- mcaligner/py.typed +0 -0
- mcaligner/rules.py +323 -0
- mcaligner/settings.py +206 -0
- mcaligner-0.2.0a1.dist-info/METADATA +195 -0
- mcaligner-0.2.0a1.dist-info/RECORD +31 -0
- mcaligner-0.2.0a1.dist-info/WHEEL +4 -0
- mcaligner-0.2.0a1.dist-info/entry_points.txt +2 -0
- mcaligner-0.2.0a1.dist-info/licenses/LICENSE +202 -0
mcaligner/__init__.py
ADDED
mcaligner/__main__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""The pydantic-ai agents: the aligner and the scorer."""
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
"""Compact answers from the agents, checked and applied in code.
|
|
2
|
+
|
|
3
|
+
The aligner answers with one line per link, the scorer with one line per
|
|
4
|
+
rating::
|
|
5
|
+
|
|
6
|
+
<id> 3-5 S 0.95 aligner: SENT_ID A-B S|P CONFIDENCE
|
|
7
|
+
<id> 3-5 0.9 scorer: SENT_ID A-B CONFIDENCE
|
|
8
|
+
|
|
9
|
+
Nothing is written until the whole answer is valid: unknown pairs, malformed
|
|
10
|
+
lines and links to words that do not exist (checked with
|
|
11
|
+
``set_links(..., dry_run=True)``) go back to the model as a retry with the
|
|
12
|
+
list of problems.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import re
|
|
18
|
+
from dataclasses import dataclass, field
|
|
19
|
+
|
|
20
|
+
from mcparser.corpus import CorpusError, CorpusStore
|
|
21
|
+
from mcparser.corpus.alignment import Link
|
|
22
|
+
from pydantic import BaseModel, Field
|
|
23
|
+
from pydantic_ai import ModelRetry
|
|
24
|
+
|
|
25
|
+
from ..rules import is_punctuation
|
|
26
|
+
|
|
27
|
+
ALIGNER_AGENT = "aligner"
|
|
28
|
+
SCORER_AGENT = "scorer"
|
|
29
|
+
|
|
30
|
+
LINK_LINE = re.compile(r"^(\S+)\s+(\d+)-(\d+)\s*([SP])?\s+(\S+)$")
|
|
31
|
+
SCORE_LINE = re.compile(r"^(\S+)\s+(\d+)-(\d+)[SP]?\s+(\S+)$")
|
|
32
|
+
REMOVE_LINE = re.compile(r"^(\S+)\s+(\d+)-(\d+)[SP]?$")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class Alignment(BaseModel):
|
|
36
|
+
"""Your answer: the links of the sentence pairs."""
|
|
37
|
+
|
|
38
|
+
links: str = Field(
|
|
39
|
+
description=(
|
|
40
|
+
'One line per link, "SENT_ID A-B S|P CONFIDENCE", e.g. "<id> 3-5 S 0.95", with <id> the pair id exactly as in its "## Pair" heading. '
|
|
41
|
+
"Settled links are kept without listing them."
|
|
42
|
+
)
|
|
43
|
+
)
|
|
44
|
+
remove: str = Field("", description='Settled links that are wrong, one per line: "SENT_ID A-B".')
|
|
45
|
+
summary: str = Field("", description="At most 15 words.")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class Scores(BaseModel):
|
|
49
|
+
"""Your answer: a confidence for each link to score."""
|
|
50
|
+
|
|
51
|
+
scores: str = Field(description='One line per link, "SENT_ID A-B CONFIDENCE", e.g. "<id> 3-5 0.9", with <id> the pair id exactly as in its "## Pair" heading.')
|
|
52
|
+
summary: str = Field("", description="At most 15 words.")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass
|
|
56
|
+
class TaskBatch:
|
|
57
|
+
"""The pairs of one task; passed to the agent run as its dependencies."""
|
|
58
|
+
|
|
59
|
+
target: str
|
|
60
|
+
sent_ids: list[str]
|
|
61
|
+
settled: dict[str, list[Link]] = field(default_factory=dict)
|
|
62
|
+
"""Aligner: the links shown as settled, per pair (they are kept)."""
|
|
63
|
+
review: dict[str, list[Link]] = field(default_factory=dict)
|
|
64
|
+
"""Scorer: the links to score, per pair."""
|
|
65
|
+
dropped: int = 0
|
|
66
|
+
"""Aligner: links to punctuation that code removed from the answer."""
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _lines(text: str) -> list[str]:
|
|
70
|
+
return [line.strip() for line in text.splitlines() if line.strip() and not line.lstrip().startswith("#")]
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _confidence(text: str) -> float | None:
|
|
74
|
+
try:
|
|
75
|
+
return min(max(float(text), 0.0), 1.0)
|
|
76
|
+
except ValueError:
|
|
77
|
+
return None
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def parse_links(text: str, batch: TaskBatch) -> tuple[dict[str, list[Link]], list[str]]:
|
|
81
|
+
"""``SENT_ID A-B S|P CONFIDENCE`` lines as links per pair, plus error messages."""
|
|
82
|
+
links: dict[str, list[Link]] = {}
|
|
83
|
+
errors = []
|
|
84
|
+
for number, line in enumerate(_lines(text), start=1):
|
|
85
|
+
match = LINK_LINE.match(line)
|
|
86
|
+
if match is None:
|
|
87
|
+
errors.append(f"links line {number}: expected SENT_ID A-B S|P CONFIDENCE, got {line!r}")
|
|
88
|
+
continue
|
|
89
|
+
sent_id, source, target, kind, confidence = match.groups()
|
|
90
|
+
score = _confidence(confidence)
|
|
91
|
+
if sent_id not in batch.sent_ids:
|
|
92
|
+
errors.append(f"links line {number}: pair {sent_id!r} is not in this task")
|
|
93
|
+
elif score is None:
|
|
94
|
+
errors.append(f"links line {number}: confidence {confidence!r} is not a number")
|
|
95
|
+
else:
|
|
96
|
+
links.setdefault(sent_id, []).append(Link(source=source, target=target, sure=kind != "P", confidence=score))
|
|
97
|
+
return links, errors
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def parse_removals(text: str, batch: TaskBatch) -> tuple[dict[str, set[str]], list[str]]:
|
|
101
|
+
removals: dict[str, set[str]] = {}
|
|
102
|
+
errors = []
|
|
103
|
+
for number, line in enumerate(_lines(text), start=1):
|
|
104
|
+
match = REMOVE_LINE.match(line)
|
|
105
|
+
if match is None or match.group(1) not in batch.sent_ids:
|
|
106
|
+
errors.append(f"remove line {number}: expected SENT_ID A-B for a pair of this task, got {line!r}")
|
|
107
|
+
continue
|
|
108
|
+
sent_id, source, target = match.groups()
|
|
109
|
+
removals.setdefault(sent_id, set()).add(f"{source}-{target}")
|
|
110
|
+
return removals, errors
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def parse_scores(text: str, batch: TaskBatch) -> tuple[dict[str, dict[str, float]], list[str]]:
|
|
114
|
+
"""``SENT_ID A-B CONFIDENCE`` lines as {link key: confidence} per pair, plus error messages."""
|
|
115
|
+
scores: dict[str, dict[str, float]] = {}
|
|
116
|
+
errors = []
|
|
117
|
+
for number, line in enumerate(_lines(text), start=1):
|
|
118
|
+
match = SCORE_LINE.match(line)
|
|
119
|
+
if match is None:
|
|
120
|
+
errors.append(f"scores line {number}: expected SENT_ID A-B CONFIDENCE, got {line!r}")
|
|
121
|
+
continue
|
|
122
|
+
sent_id, source, target, confidence = match.groups()
|
|
123
|
+
key, score = f"{source}-{target}", _confidence(confidence)
|
|
124
|
+
if sent_id not in batch.sent_ids:
|
|
125
|
+
errors.append(f"scores line {number}: pair {sent_id!r} is not in this task")
|
|
126
|
+
elif score is None:
|
|
127
|
+
errors.append(f"scores line {number}: confidence {confidence!r} is not a number")
|
|
128
|
+
elif key not in {link.key for link in batch.review.get(sent_id, [])}:
|
|
129
|
+
errors.append(f"scores line {number}: {sent_id} {key} is not one of the links to score")
|
|
130
|
+
else:
|
|
131
|
+
scores.setdefault(sent_id, {})[key] = score
|
|
132
|
+
return scores, errors
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
class Applier:
|
|
136
|
+
"""Checks an agent's answer against the store, then applies it."""
|
|
137
|
+
|
|
138
|
+
def __init__(self, store: CorpusStore, link_punctuation: bool = False):
|
|
139
|
+
self.store = store
|
|
140
|
+
self.link_punctuation = link_punctuation
|
|
141
|
+
|
|
142
|
+
def without_punctuation(self, batch: TaskBatch, links: dict[str, list[Link]]) -> dict[str, list[Link]]:
|
|
143
|
+
"""The answer's links without those that touch punctuation (counted in ``batch.dropped``)."""
|
|
144
|
+
aset = self.store.alignment(batch.target)
|
|
145
|
+
kept, batch.dropped = {}, 0 # reset: a retried answer is filtered again
|
|
146
|
+
for sent_id, group in links.items():
|
|
147
|
+
skip_a = {w.id for w in aset.source_a.sentence(sent_id).words if is_punctuation(w)}
|
|
148
|
+
skip_b = {w.id for w in aset.source_b.sentence(sent_id).words if is_punctuation(w)}
|
|
149
|
+
kept[sent_id] = [link for link in group if link.source not in skip_a and link.target not in skip_b]
|
|
150
|
+
batch.dropped += len(group) - len(kept[sent_id])
|
|
151
|
+
return kept
|
|
152
|
+
|
|
153
|
+
@staticmethod
|
|
154
|
+
def _retry(errors: list[str]) -> None:
|
|
155
|
+
if errors:
|
|
156
|
+
raise ModelRetry("Fix these problems and answer again:\n" + "\n".join(f"- {e}" for e in errors[:20]))
|
|
157
|
+
|
|
158
|
+
def merged(self, batch: TaskBatch, sent_id: str, links: list[Link], removals: set[str]) -> list[Link]:
|
|
159
|
+
"""Settled links (minus removals) plus the answer's links (which win on the same word pair)."""
|
|
160
|
+
result = {link.key: link for link in batch.settled.get(sent_id, []) if link.key not in removals}
|
|
161
|
+
result.update({link.key: link for link in links})
|
|
162
|
+
return list(result.values())
|
|
163
|
+
|
|
164
|
+
def alignment(self, answer: Alignment, batch: TaskBatch) -> Alignment:
|
|
165
|
+
links, errors = parse_links(answer.links, batch)
|
|
166
|
+
if not self.link_punctuation and not errors:
|
|
167
|
+
links = self.without_punctuation(batch, links)
|
|
168
|
+
removals, removal_errors = parse_removals(answer.remove, batch)
|
|
169
|
+
errors += removal_errors
|
|
170
|
+
new = {sent_id: self.merged(batch, sent_id, links.get(sent_id, []), removals.get(sent_id, set())) for sent_id in batch.sent_ids}
|
|
171
|
+
for sent_id, merged in new.items():
|
|
172
|
+
try:
|
|
173
|
+
self.store.set_links(batch.target, sent_id, merged, agent=ALIGNER_AGENT, dry_run=True)
|
|
174
|
+
except CorpusError as error:
|
|
175
|
+
errors.append(f"pair {sent_id}: {error}")
|
|
176
|
+
self._retry(errors)
|
|
177
|
+
for sent_id, merged in new.items(): # every pair is written, so the history shows the aligner saw it
|
|
178
|
+
self.store.set_links(batch.target, sent_id, merged, reason=answer.summary, agent=ALIGNER_AGENT)
|
|
179
|
+
return answer
|
|
180
|
+
|
|
181
|
+
def scores(self, answer: Scores, batch: TaskBatch) -> Scores:
|
|
182
|
+
scores, errors = parse_scores(answer.scores, batch)
|
|
183
|
+
self._retry(errors)
|
|
184
|
+
for sent_id, keyed in scores.items():
|
|
185
|
+
rated = [
|
|
186
|
+
link.model_copy(update={"confidence": keyed[link.key]}) for link in batch.review[sent_id] if link.key in keyed
|
|
187
|
+
]
|
|
188
|
+
self.store.set_links(batch.target, sent_id, rated, replace=False, reason=answer.summary, agent=SCORER_AGENT)
|
|
189
|
+
return answer
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
"""Instructions and task prompts for the aligner and the scorer.
|
|
2
|
+
|
|
3
|
+
A task covers several sentence pairs. Each pair is rendered compactly: one line
|
|
4
|
+
per word (``id form lemma UPOS head deprel`` and glosses from MISC), the links
|
|
5
|
+
already settled, the links to review, and for each unlinked word how its lemma
|
|
6
|
+
is linked elsewhere (translation evidence). The answers are compact lines that
|
|
7
|
+
code checks and applies (see :mod:`mcaligner.agents.outputs`).
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
|
|
14
|
+
from mcparser.corpus import Sentence
|
|
15
|
+
from mcparser.corpus.alignment import Link
|
|
16
|
+
|
|
17
|
+
from ..lexicon import TranslationTable, lemma_of
|
|
18
|
+
from ..settings import COLUMNS, DEFAULT_COLUMNS, TOOL_NAMES
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass
|
|
22
|
+
class PairItem:
|
|
23
|
+
"""One sentence pair of a task."""
|
|
24
|
+
|
|
25
|
+
sent_id: str
|
|
26
|
+
a: Sentence
|
|
27
|
+
b: Sentence
|
|
28
|
+
settled: list[Link] = field(default_factory=list)
|
|
29
|
+
"""Links kept as they are (at or above the threshold)."""
|
|
30
|
+
review: list[Link] = field(default_factory=list)
|
|
31
|
+
"""Links below the threshold (aligner) or links to score (scorer)."""
|
|
32
|
+
|
|
33
|
+
@property
|
|
34
|
+
def words(self) -> int:
|
|
35
|
+
return len(self.a.words) + len(self.b.words)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
ALIGNER = """\
|
|
39
|
+
You align the words of two versions of the same text: source A ({a}) and
|
|
40
|
+
source B ({b}), for example a translation and its original. Sentences are
|
|
41
|
+
paired by id. Words are numbered as in CoNLL-U; a multiword token (e.g. a word
|
|
42
|
+
with clitics) is split into syntactic words, and links join words, not tokens.
|
|
43
|
+
|
|
44
|
+
A link joins a word of A and a word of B that translate each other: "A-B S"
|
|
45
|
+
(sure) or "A-B P" (probable: a looser or partial correspondence, e.g. a
|
|
46
|
+
paraphrase). One-to-many and many-to-many alignments are several links (2-3 S
|
|
47
|
+
and 2-4 S when word 2 of A renders words 3 and 4 of B). A word may contain
|
|
48
|
+
morphemes that the parse did not split off (a pronoun suffix, an attached
|
|
49
|
+
preposition or article, the person of a verb): link it to every word it
|
|
50
|
+
renders, e.g. a verb with an object suffix "blessed-us" to both the verb and
|
|
51
|
+
the pronoun. Leave a word unlinked when it has no counterpart (a word the
|
|
52
|
+
translator added, or a function word the other language leaves out).
|
|
53
|
+
|
|
54
|
+
The task gives, for each pair: both sentences ({columns}, glosses where
|
|
55
|
+
available), the settled links (by code rules or earlier work; they are kept),
|
|
56
|
+
links to review, and for unlinked words how their lemma is linked elsewhere in
|
|
57
|
+
the corpus. Follow that evidence unless the context rules it out. {tools}
|
|
58
|
+
|
|
59
|
+
Answer with ONE final result. In `links`, one line per link:
|
|
60
|
+
"SENT_ID A-B S|P CONFIDENCE", e.g. "<id> 3-5 S 0.95", with <id> the pair id exactly as in its "## Pair" heading. List every link of
|
|
61
|
+
each pair except the settled ones, which are kept automatically; a link to
|
|
62
|
+
review that you do not list is removed. To remove a settled link that is
|
|
63
|
+
wrong, put "SENT_ID A-B" in `remove`. Confidence: 0.95 or more = clear; 0.8 =
|
|
64
|
+
probable; 0.6 = doubtful; below 0.5 = a guess. Keep `summary` under 15 words.
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
SCORER = """\
|
|
68
|
+
You check word alignments between two versions of the same text: source A
|
|
69
|
+
({a}) and source B ({b}). Sentences are paired by id; words are numbered as in
|
|
70
|
+
CoNLL-U. A link "A-B S|P" says that word A and word B translate each other
|
|
71
|
+
(S = sure, P = probable, a looser correspondence). A word may contain
|
|
72
|
+
morphemes the parse did not split off (a pronoun suffix, an attached
|
|
73
|
+
preposition, the person of a verb), so a link from such a word to a pronoun
|
|
74
|
+
or preposition on the other side is right when that morpheme renders it.
|
|
75
|
+
|
|
76
|
+
For each pair the task gives both sentences ({columns}, glosses where
|
|
77
|
+
available), all its links, and the links to score. Rate each
|
|
78
|
+
link to score: how likely is it that these two words correspond in this
|
|
79
|
+
sentence? 0.95 or more = clearly right; 0.8 = probably right; 0.6 = doubtful;
|
|
80
|
+
below 0.5 = probably wrong (e.g. the wrong one of two similar words, or a word
|
|
81
|
+
that has a better counterpart). Links below {threshold} go back to the aligner. {tools}
|
|
82
|
+
|
|
83
|
+
Answer with ONE final result: in `scores`, one line per link to score,
|
|
84
|
+
"SENT_ID A-B CONFIDENCE", e.g. "<id> 3-5 0.9", with <id> the pair id exactly as in its "## Pair" heading. Keep `summary` under 15 words.
|
|
85
|
+
"""
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
GUIDELINES = """\
|
|
89
|
+
## Alignment guidelines
|
|
90
|
+
|
|
91
|
+
The human annotators of this text followed these guidelines. Apply them: they
|
|
92
|
+
decide which words are linked and which are left unlinked.
|
|
93
|
+
|
|
94
|
+
{guidelines}
|
|
95
|
+
"""
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
COLUMN_NAMES = {"upos": "UPOS", "xpos": "XPOS", "feats": "FEATS"}
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def column_text(columns: list[str] | tuple[str, ...]) -> str:
|
|
102
|
+
"""``id form lemma UPOS head deprel`` for the columns shown."""
|
|
103
|
+
return " ".join(["id", "form", *(COLUMN_NAMES.get(name, name) for name in columns)])
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def tools_text(tools: set[str] | frozenset[str] | list[str] | None, columns: list[str] | tuple[str, ...]) -> str:
|
|
107
|
+
"""What the instructions say about the tools the agent has (None = all of them)."""
|
|
108
|
+
names = TOOL_NAMES if tools is None else set(tools)
|
|
109
|
+
if not names:
|
|
110
|
+
return "No tools are available: answer from the task."
|
|
111
|
+
text = (
|
|
112
|
+
f"Tools are available ({', '.join(sorted(names))}) but call them only when the evidence given is not enough, "
|
|
113
|
+
"and make independent calls in parallel."
|
|
114
|
+
)
|
|
115
|
+
hidden = [COLUMN_NAMES.get(name, name) for name in COLUMNS if name not in columns]
|
|
116
|
+
if "pair_words" in names and hidden:
|
|
117
|
+
text += f" The prompt omits {', '.join(hidden)}: pair_words gives every column of a pair's words"
|
|
118
|
+
text += (
|
|
119
|
+
", and category_links shows how words of a category (UPOS, XPOS, FEATS such as Case=Gen, deprel) are "
|
|
120
|
+
"usually linked or left unlinked. Use them when the grammar decides a link (an article, a case against a "
|
|
121
|
+
"preposition, tense and voice, a participle)."
|
|
122
|
+
if "category_links" in names
|
|
123
|
+
else "."
|
|
124
|
+
)
|
|
125
|
+
return text
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def instructions(
|
|
129
|
+
role: str,
|
|
130
|
+
a: str,
|
|
131
|
+
b: str,
|
|
132
|
+
target: str,
|
|
133
|
+
threshold: float,
|
|
134
|
+
extra: str | None = None,
|
|
135
|
+
guidelines: str | None = None,
|
|
136
|
+
columns: list[str] | tuple[str, ...] = DEFAULT_COLUMNS,
|
|
137
|
+
tools: list[str] | None = None,
|
|
138
|
+
) -> str:
|
|
139
|
+
template = ALIGNER if role == "aligner" else SCORER
|
|
140
|
+
text = template.format(
|
|
141
|
+
a=a, b=b, target=target, threshold=threshold, columns=column_text(columns), tools=tools_text(tools, columns)
|
|
142
|
+
)
|
|
143
|
+
if guidelines:
|
|
144
|
+
text = f"{text}\n{GUIDELINES.format(guidelines=guidelines.strip())}"
|
|
145
|
+
return f"{text}\n{extra.strip()}\n" if extra else text
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def word_lines(sentence: Sentence, misc_keys: list[str], columns: list[str] | tuple[str, ...] = DEFAULT_COLUMNS) -> list[str]:
|
|
149
|
+
"""``id form`` and the ``columns`` (default lemma UPOS head deprel), then the MISC values, per word; a line per multiword token."""
|
|
150
|
+
lines = []
|
|
151
|
+
for token in sentence.tokens:
|
|
152
|
+
if token.is_multiword:
|
|
153
|
+
lines.append(f"{token.id} {token.form} (token of words {token.id})")
|
|
154
|
+
elif token.is_word:
|
|
155
|
+
extra = [value for key in misc_keys if (value := token.misc_value(key))]
|
|
156
|
+
lines.append(" ".join([token.id, token.form, *(getattr(token, name) for name in columns), *extra]))
|
|
157
|
+
return lines
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def link_text(link: Link, confidence: float) -> str:
|
|
161
|
+
agent = f" {link.agent}" if link.agent else ""
|
|
162
|
+
return f"{link.ugarit()} {confidence:.2f}{agent}"
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def evidence_lines(item: PairItem, table: TranslationTable, source_a: str, source_b: str, limit: int) -> list[str]:
|
|
166
|
+
"""For unlinked words (not punctuation): their lemma's linked lemmas elsewhere (this pair left out)."""
|
|
167
|
+
if limit <= 0:
|
|
168
|
+
return []
|
|
169
|
+
linked_a = {link.source for link in item.settled + item.review}
|
|
170
|
+
linked_b = {link.target for link in item.settled + item.review}
|
|
171
|
+
lines = []
|
|
172
|
+
for side, source, sentence, linked in (("A", source_a, item.a, linked_a), ("B", source_b, item.b, linked_b)):
|
|
173
|
+
seen: set[str] = set()
|
|
174
|
+
for word in sentence.words:
|
|
175
|
+
lemma = lemma_of(word)
|
|
176
|
+
if word.id in linked or lemma in seen or word.upos == "PUNCT":
|
|
177
|
+
continue
|
|
178
|
+
seen.add(lemma)
|
|
179
|
+
entry = table.entry(source, lemma)
|
|
180
|
+
if entry is None:
|
|
181
|
+
continue
|
|
182
|
+
candidates = entry.candidates(item.sent_id).most_common(limit)
|
|
183
|
+
if not candidates:
|
|
184
|
+
continue
|
|
185
|
+
unlinked = entry.unaligned_without(item.sent_id)
|
|
186
|
+
shown = ", ".join(f"{value}×{count}" for value, count in candidates)
|
|
187
|
+
note = f"of {entry.occurrences_without(item.sent_id)}" + (f", {unlinked} unlinked" if unlinked else "")
|
|
188
|
+
lines.append(f"{side}{word.id} {lemma}: {shown} ({note})")
|
|
189
|
+
return lines
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def pair_block(
|
|
193
|
+
item: PairItem,
|
|
194
|
+
role: str,
|
|
195
|
+
confidence: dict[str, float],
|
|
196
|
+
table: TranslationTable | None,
|
|
197
|
+
names: tuple[str, str],
|
|
198
|
+
misc_keys: list[str],
|
|
199
|
+
candidates: int,
|
|
200
|
+
columns: list[str] | tuple[str, ...] = DEFAULT_COLUMNS,
|
|
201
|
+
) -> str:
|
|
202
|
+
"""The text of one pair in a task prompt."""
|
|
203
|
+
lines = [f"## Pair {item.sent_id}", f"A: {item.a.text}"]
|
|
204
|
+
lines += word_lines(item.a, misc_keys, columns)
|
|
205
|
+
lines += [f"B: {item.b.text}", *word_lines(item.b, misc_keys, columns)]
|
|
206
|
+
settled = ", ".join(link_text(link, confidence[link.key]) for link in item.settled) or "none"
|
|
207
|
+
lines.append(f"Settled links: {settled}")
|
|
208
|
+
if role == "aligner":
|
|
209
|
+
if item.review:
|
|
210
|
+
lines.append("Links to review: " + ", ".join(link_text(link, confidence[link.key]) for link in item.review))
|
|
211
|
+
hints = evidence_lines(item, table, *names, candidates) if table is not None else []
|
|
212
|
+
if hints:
|
|
213
|
+
lines += ["Evidence for unlinked words (linked lemmas elsewhere):", *hints]
|
|
214
|
+
else:
|
|
215
|
+
forms_a = {word.id: word.form for word in item.a.words}
|
|
216
|
+
forms_b = {word.id: word.form for word in item.b.words}
|
|
217
|
+
lines.append("Links to score:")
|
|
218
|
+
lines += [f"{link.ugarit()} {forms_a[link.source]}→{forms_b[link.target]}" for link in item.review]
|
|
219
|
+
return "\n".join(lines)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def task_prompt(
|
|
223
|
+
role: str,
|
|
224
|
+
items: list[PairItem],
|
|
225
|
+
confidence: dict[str, dict[str, float]],
|
|
226
|
+
table: TranslationTable | None,
|
|
227
|
+
names: tuple[str, str],
|
|
228
|
+
misc_keys: list[str],
|
|
229
|
+
candidates: int,
|
|
230
|
+
columns: list[str] | tuple[str, ...] = DEFAULT_COLUMNS,
|
|
231
|
+
) -> str:
|
|
232
|
+
header = (
|
|
233
|
+
f"Align these {len(items)} sentence pairs." if role == "aligner" else f"Score the listed links of these {len(items)} sentence pairs."
|
|
234
|
+
)
|
|
235
|
+
header += f' Use the pair ids exactly as in the "## Pair" headings ({", ".join(item.sent_id for item in items)}), without any prefix.'
|
|
236
|
+
blocks = [pair_block(item, role, confidence[item.sent_id], table, names, misc_keys, candidates, columns) for item in items]
|
|
237
|
+
return "\n\n".join([header, *blocks])
|
mcaligner/agents/team.py
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"""The pydantic-ai agents (aligner, scorer), sharing one MCP connection to the store."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from mcparser.corpus import CorpusStore
|
|
9
|
+
from mcparser.pricing import estimate
|
|
10
|
+
from pydantic import BaseModel, Field
|
|
11
|
+
from pydantic_ai import Agent, RunContext
|
|
12
|
+
from pydantic_ai.mcp import MCPToolset
|
|
13
|
+
from pydantic_ai.messages import ModelResponse, RetryPromptPart, ToolCallPart
|
|
14
|
+
from pydantic_ai.models import Model
|
|
15
|
+
from pydantic_ai.settings import ModelSettings
|
|
16
|
+
from pydantic_ai.toolsets import AbstractToolset
|
|
17
|
+
from pydantic_ai.usage import RunUsage, UsageLimits
|
|
18
|
+
|
|
19
|
+
from ..mcp import build_server
|
|
20
|
+
from ..settings import Settings
|
|
21
|
+
from .outputs import Alignment, Applier, Scores, TaskBatch
|
|
22
|
+
from .prompts import instructions
|
|
23
|
+
from .toolset import READ_TOOLS, read_toolset
|
|
24
|
+
|
|
25
|
+
Answer = Alignment | Scores
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class Details(BaseModel):
|
|
29
|
+
"""What happened inside one agent task."""
|
|
30
|
+
|
|
31
|
+
requests: int = 0
|
|
32
|
+
tools: dict[str, int] = Field(default_factory=dict, description="Tool calls by tool name.")
|
|
33
|
+
retries: list[str] = Field(default_factory=list, description="Why the model was asked again.")
|
|
34
|
+
dropped: int = Field(0, description="Links to punctuation that code removed from the answer.")
|
|
35
|
+
USAGE_FIELDS = ("tasks", "requests", "input_tokens", "cache_read_tokens", "cache_write_tokens", "output_tokens")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class ModelUsage(BaseModel):
|
|
39
|
+
"""Token use of one model over a run."""
|
|
40
|
+
|
|
41
|
+
tasks: int = 0
|
|
42
|
+
requests: int = 0
|
|
43
|
+
input_tokens: int = 0
|
|
44
|
+
cache_read_tokens: int = 0
|
|
45
|
+
cache_write_tokens: int = 0
|
|
46
|
+
output_tokens: int = 0
|
|
47
|
+
cost: float | None = Field(None, description="Estimated US dollars (see mcparser.pricing).")
|
|
48
|
+
|
|
49
|
+
def add(self, usage: RunUsage, model: str) -> None:
|
|
50
|
+
self.tasks += 1
|
|
51
|
+
self.requests += usage.requests
|
|
52
|
+
self.input_tokens += usage.input_tokens
|
|
53
|
+
self.cache_read_tokens += usage.cache_read_tokens
|
|
54
|
+
self.cache_write_tokens += usage.cache_write_tokens
|
|
55
|
+
self.output_tokens += usage.output_tokens
|
|
56
|
+
self.cost = estimate(model, self.input_tokens, self.cache_read_tokens, self.cache_write_tokens, self.output_tokens)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class AlignerTeam:
|
|
60
|
+
"""Builds and runs the agents for one writable alignment set (``target``).
|
|
61
|
+
|
|
62
|
+
By default the agents reach the store through an in-process MCP server;
|
|
63
|
+
pass ``toolset`` to use another MCP connection. ``models`` maps a role (or
|
|
64
|
+
``"*"``) to a model name or instance, overriding the settings; tests use it
|
|
65
|
+
for scripted models.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
def __init__(
|
|
69
|
+
self,
|
|
70
|
+
store: CorpusStore,
|
|
71
|
+
target: str,
|
|
72
|
+
settings: Settings,
|
|
73
|
+
models: Mapping[str, Model | str] | None = None,
|
|
74
|
+
toolset: AbstractToolset[Any] | None = None,
|
|
75
|
+
):
|
|
76
|
+
self.store = store
|
|
77
|
+
self.target = target
|
|
78
|
+
self.settings = settings
|
|
79
|
+
self.models = dict(models or {})
|
|
80
|
+
self.toolset = toolset or MCPToolset(build_server(store))
|
|
81
|
+
self.applier = Applier(store, settings.rules.link_punctuation)
|
|
82
|
+
self._agents: dict[str, Agent[TaskBatch, Answer]] = {}
|
|
83
|
+
self.usage: dict[str, ModelUsage] = {}
|
|
84
|
+
"""Token use and estimated cost per model."""
|
|
85
|
+
|
|
86
|
+
def model(self, role: str) -> Model | str:
|
|
87
|
+
return self.models.get(role) or self.models.get("*") or self.settings.model_for(role)
|
|
88
|
+
|
|
89
|
+
@staticmethod
|
|
90
|
+
def model_name(model: Model | str) -> str:
|
|
91
|
+
return model if isinstance(model, str) else model.model_name
|
|
92
|
+
|
|
93
|
+
def model_settings(self, role: str) -> ModelSettings:
|
|
94
|
+
settings = ModelSettings(cache=self.settings.cache)
|
|
95
|
+
thinking = self.settings.thinking_for(role)
|
|
96
|
+
if thinking is not None:
|
|
97
|
+
settings["thinking"] = thinking
|
|
98
|
+
return settings
|
|
99
|
+
|
|
100
|
+
def agent(self, role: str) -> Agent[TaskBatch, Answer]:
|
|
101
|
+
if role not in self._agents:
|
|
102
|
+
aset = self.store.alignment(self.target)
|
|
103
|
+
agent = Agent(
|
|
104
|
+
self.model(role),
|
|
105
|
+
name=role,
|
|
106
|
+
deps_type=TaskBatch,
|
|
107
|
+
instructions=instructions(
|
|
108
|
+
role, aset.source_a.name, aset.source_b.name, self.target, self.settings.threshold,
|
|
109
|
+
self.settings.role(role).instructions, self.settings.guidelines_text,
|
|
110
|
+
self.settings.columns, self.settings.tools,
|
|
111
|
+
), # fmt: skip
|
|
112
|
+
output_type=Alignment if role == "aligner" else Scores,
|
|
113
|
+
toolsets=[read_toolset(self.toolset, self.settings.tools)],
|
|
114
|
+
retries=3,
|
|
115
|
+
model_settings=self.model_settings(role),
|
|
116
|
+
defer_model_check=True,
|
|
117
|
+
)
|
|
118
|
+
apply = self.applier.alignment if role == "aligner" else self.applier.scores
|
|
119
|
+
|
|
120
|
+
@agent.output_validator
|
|
121
|
+
def check_and_apply(ctx: RunContext[TaskBatch], answer: Answer) -> Answer:
|
|
122
|
+
return apply(answer, ctx.deps) # type: ignore[arg-type]
|
|
123
|
+
|
|
124
|
+
self._agents[role] = agent
|
|
125
|
+
return self._agents[role]
|
|
126
|
+
|
|
127
|
+
async def __aenter__(self) -> AlignerTeam:
|
|
128
|
+
await self.toolset.__aenter__()
|
|
129
|
+
return self
|
|
130
|
+
|
|
131
|
+
async def __aexit__(self, *args: Any) -> None:
|
|
132
|
+
await self.toolset.__aexit__(*args)
|
|
133
|
+
|
|
134
|
+
async def run(self, role: str, prompt: str, batch: TaskBatch, details: Details | None = None) -> Answer:
|
|
135
|
+
"""Run ``role`` on a task; the answer is applied when it is accepted.
|
|
136
|
+
|
|
137
|
+
If ``details`` is given, the task's requests, tool calls (by tool) and the messages that made the model
|
|
138
|
+
retry are added to it.
|
|
139
|
+
"""
|
|
140
|
+
result = await self.agent(role).run(
|
|
141
|
+
prompt, deps=batch, usage_limits=UsageLimits(request_limit=self.settings.request_limit)
|
|
142
|
+
)
|
|
143
|
+
name = self.model_name(self.model(role))
|
|
144
|
+
self.usage.setdefault(name, ModelUsage()).add(result.usage, name)
|
|
145
|
+
if details is not None:
|
|
146
|
+
details.dropped += batch.dropped
|
|
147
|
+
details.requests += result.usage.requests
|
|
148
|
+
for message in result.all_messages():
|
|
149
|
+
parts = message.parts
|
|
150
|
+
if isinstance(message, ModelResponse):
|
|
151
|
+
for part in parts:
|
|
152
|
+
if isinstance(part, ToolCallPart) and part.tool_name in READ_TOOLS:
|
|
153
|
+
details.tools[part.tool_name] = details.tools.get(part.tool_name, 0) + 1
|
|
154
|
+
else: # a ModelRequest
|
|
155
|
+
details.retries += [str(part.content)[:300] for part in parts if isinstance(part, RetryPromptPart)]
|
|
156
|
+
return result.output
|
|
157
|
+
|
|
158
|
+
def total(self) -> ModelUsage:
|
|
159
|
+
"""Usage summed over all models (cost is None if any model has no price)."""
|
|
160
|
+
total = ModelUsage(cost=0.0)
|
|
161
|
+
for usage in self.usage.values():
|
|
162
|
+
for name in USAGE_FIELDS:
|
|
163
|
+
setattr(total, name, getattr(total, name) + getattr(usage, name))
|
|
164
|
+
total.cost = None if total.cost is None or usage.cost is None else total.cost + usage.cost
|
|
165
|
+
return total
|