mcaligner 0.2.0a1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. mcaligner-0.2.0a1/.github/workflows/docs.yml +62 -0
  2. mcaligner-0.2.0a1/.github/workflows/publish.yml +31 -0
  3. mcaligner-0.2.0a1/.github/workflows/testing.yml +52 -0
  4. mcaligner-0.2.0a1/.gitignore +15 -0
  5. mcaligner-0.2.0a1/.python-version +1 -0
  6. mcaligner-0.2.0a1/AGENTS.md +383 -0
  7. mcaligner-0.2.0a1/CHANGELOG.md +20 -0
  8. mcaligner-0.2.0a1/CITATION.cff +19 -0
  9. mcaligner-0.2.0a1/CLAUDE.md +1 -0
  10. mcaligner-0.2.0a1/CONTRIBUTING.md +37 -0
  11. mcaligner-0.2.0a1/INSTRUCTIONS.md +201 -0
  12. mcaligner-0.2.0a1/LICENSE +202 -0
  13. mcaligner-0.2.0a1/PKG-INFO +195 -0
  14. mcaligner-0.2.0a1/README.rst +172 -0
  15. mcaligner-0.2.0a1/docs/_static/.gitkeep +0 -0
  16. mcaligner-0.2.0a1/docs/_static/mc.svg +512 -0
  17. mcaligner-0.2.0a1/docs/_static/mcaligner-text-dark.svg +53 -0
  18. mcaligner-0.2.0a1/docs/_static/mcaligner-text.svg +53 -0
  19. mcaligner-0.2.0a1/docs/agents.md +87 -0
  20. mcaligner-0.2.0a1/docs/api.md +14 -0
  21. mcaligner-0.2.0a1/docs/conf.py +62 -0
  22. mcaligner-0.2.0a1/docs/data.md +49 -0
  23. mcaligner-0.2.0a1/docs/development.md +28 -0
  24. mcaligner-0.2.0a1/docs/evaluation.md +39 -0
  25. mcaligner-0.2.0a1/docs/index.md +80 -0
  26. mcaligner-0.2.0a1/docs/mcp.md +35 -0
  27. mcaligner-0.2.0a1/docs/preparing.md +72 -0
  28. mcaligner-0.2.0a1/docs/quickstart.md +49 -0
  29. mcaligner-0.2.0a1/docs/results.md +55 -0
  30. mcaligner-0.2.0a1/docs/rules.md +38 -0
  31. mcaligner-0.2.0a1/mcaligner/__init__.py +5 -0
  32. mcaligner-0.2.0a1/mcaligner/__main__.py +3 -0
  33. mcaligner-0.2.0a1/mcaligner/agents/__init__.py +1 -0
  34. mcaligner-0.2.0a1/mcaligner/agents/outputs.py +189 -0
  35. mcaligner-0.2.0a1/mcaligner/agents/prompts.py +237 -0
  36. mcaligner-0.2.0a1/mcaligner/agents/team.py +165 -0
  37. mcaligner-0.2.0a1/mcaligner/agents/toolset.py +139 -0
  38. mcaligner-0.2.0a1/mcaligner/cli.py +298 -0
  39. mcaligner-0.2.0a1/mcaligner/evaluate.py +147 -0
  40. mcaligner-0.2.0a1/mcaligner/guidelines/bicross.md +140 -0
  41. mcaligner-0.2.0a1/mcaligner/guidelines/got-lat.md +132 -0
  42. mcaligner-0.2.0a1/mcaligner/guidelines/grc-ar.md +146 -0
  43. mcaligner-0.2.0a1/mcaligner/guidelines/grc-cop.md +166 -0
  44. mcaligner-0.2.0a1/mcaligner/guidelines/grc-lat.md +162 -0
  45. mcaligner-0.2.0a1/mcaligner/guidelines/grc-syr.md +65 -0
  46. mcaligner-0.2.0a1/mcaligner/guidelines/lat-ar.md +73 -0
  47. mcaligner-0.2.0a1/mcaligner/guidelines/syr-ar.md +57 -0
  48. mcaligner-0.2.0a1/mcaligner/lexicon.py +151 -0
  49. mcaligner-0.2.0a1/mcaligner/mcp/__init__.py +5 -0
  50. mcaligner-0.2.0a1/mcaligner/mcp/cli.py +67 -0
  51. mcaligner-0.2.0a1/mcaligner/mcp/server.py +123 -0
  52. mcaligner-0.2.0a1/mcaligner/mcp/tools.py +307 -0
  53. mcaligner-0.2.0a1/mcaligner/pipeline.py +286 -0
  54. mcaligner-0.2.0a1/mcaligner/py.typed +0 -0
  55. mcaligner-0.2.0a1/mcaligner/rules.py +323 -0
  56. mcaligner-0.2.0a1/mcaligner/settings.py +206 -0
  57. mcaligner-0.2.0a1/mkdocs.sh +12 -0
  58. mcaligner-0.2.0a1/pyproject.toml +63 -0
  59. mcaligner-0.2.0a1/tests/conftest.py +74 -0
  60. mcaligner-0.2.0a1/tests/data/ar.conllu +67 -0
  61. mcaligner-0.2.0a1/tests/data/grc.conllu +63 -0
  62. mcaligner-0.2.0a1/tests/data/silver.jsonl +3 -0
  63. mcaligner-0.2.0a1/tests/scripted.py +93 -0
  64. mcaligner-0.2.0a1/tests/test_agents.py +279 -0
  65. mcaligner-0.2.0a1/tests/test_cli.py +212 -0
  66. mcaligner-0.2.0a1/tests/test_evaluate.py +57 -0
  67. mcaligner-0.2.0a1/tests/test_lexicon.py +42 -0
  68. mcaligner-0.2.0a1/tests/test_mcp.py +160 -0
  69. mcaligner-0.2.0a1/tests/test_pipeline.py +111 -0
  70. mcaligner-0.2.0a1/tests/test_rules.py +121 -0
  71. mcaligner-0.2.0a1/tests/test_settings.py +82 -0
  72. mcaligner-0.2.0a1/uv.lock +2399 -0
@@ -0,0 +1,62 @@
1
+ name: docs
2
+
3
+ # Builds the Sphinx docs and the coverage report on every push (so a broken
4
+ # docs build shows up on any branch), and deploys them to GitHub Pages from main.
5
+
6
+ on:
7
+ push:
8
+ workflow_dispatch:
9
+
10
+ permissions:
11
+ contents: read
12
+ pages: write
13
+ id-token: write
14
+
15
+ concurrency:
16
+ group: pages-${{ github.ref }}
17
+ cancel-in-progress: false
18
+
19
+ jobs:
20
+ build:
21
+ runs-on: ubuntu-latest
22
+ steps:
23
+ - uses: actions/checkout@v7
24
+
25
+ - name: Install uv and Python
26
+ uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
27
+ with:
28
+ enable-cache: true
29
+ python-version: "3.13"
30
+
31
+ - name: Install dependencies
32
+ run: uv sync --locked --all-groups
33
+
34
+ - name: Build Sphinx docs
35
+ run: uv run --locked sphinx-build -W -b html -d docs/_build/doctrees docs gh-pages
36
+
37
+ - name: Run tests and generate coverage report
38
+ run: |
39
+ uv run --locked pytest --cov-report=html:gh-pages/coverage
40
+ rm -f gh-pages/coverage/.gitignore
41
+
42
+ - name: Setup Pages
43
+ if: github.ref == 'refs/heads/main'
44
+ uses: actions/configure-pages@v6
45
+
46
+ - name: Upload artifact
47
+ if: github.ref == 'refs/heads/main'
48
+ uses: actions/upload-pages-artifact@v5
49
+ with:
50
+ path: gh-pages
51
+
52
+ deploy:
53
+ if: github.ref == 'refs/heads/main'
54
+ needs: build
55
+ runs-on: ubuntu-latest
56
+ environment:
57
+ name: github-pages
58
+ url: ${{ steps.deployment.outputs.page_url }}
59
+ steps:
60
+ - name: Deploy to GitHub Pages
61
+ id: deployment
62
+ uses: actions/deploy-pages@v5
@@ -0,0 +1,31 @@
1
+ name: publish
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - "v*.*.*"
7
+
8
+ jobs:
9
+ publish:
10
+ runs-on: ubuntu-latest
11
+ steps:
12
+ - uses: actions/checkout@v7
13
+
14
+ - name: Install uv and Python
15
+ uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
16
+ with:
17
+ enable-cache: true
18
+ python-version: "3.13"
19
+
20
+ - name: Run tests
21
+ run: |
22
+ uv sync --locked --group dev
23
+ uv run --locked pytest
24
+
25
+ - name: Build library
26
+ run: uv build
27
+
28
+ - name: Publish library
29
+ env:
30
+ UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }}
31
+ run: uv publish
@@ -0,0 +1,52 @@
1
+ name: testing
2
+
3
+ on:
4
+ push:
5
+ pull_request:
6
+ workflow_dispatch:
7
+
8
+ jobs:
9
+ test:
10
+ runs-on: ubuntu-latest
11
+ strategy:
12
+ fail-fast: false
13
+ matrix:
14
+ python-version: ["3.11", "3.12", "3.13", "3.14"]
15
+ steps:
16
+ - uses: actions/checkout@v7
17
+
18
+ - name: Install uv and Python ${{ matrix.python-version }}
19
+ uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
20
+ with:
21
+ enable-cache: true
22
+ python-version: ${{ matrix.python-version }}
23
+
24
+ - name: Install dependencies
25
+ run: uv sync --locked --group dev
26
+
27
+ # Coverage is reported but does not fail the job; only failing tests do.
28
+ # The tests never call a real model, so no API key is needed.
29
+ - name: Run tests
30
+ run: uv run --locked pytest
31
+
32
+ package:
33
+ runs-on: ubuntu-latest
34
+ steps:
35
+ - uses: actions/checkout@v7
36
+
37
+ - name: Install uv
38
+ uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
39
+ with:
40
+ python-version: "3.13"
41
+
42
+ - name: Build and check the package
43
+ run: |
44
+ uv build
45
+ uvx twine check dist/*
46
+
47
+ - name: Install the wheel and run the CLI
48
+ run: |
49
+ uv venv --python 3.13 /tmp/wheel-test
50
+ uv pip install --python /tmp/wheel-test/bin/python dist/*.whl
51
+ /tmp/wheel-test/bin/mcaligner --version
52
+ /tmp/wheel-test/bin/mcaligner --help
@@ -0,0 +1,15 @@
1
+ results/
2
+ llmalignerold
3
+ .venv/
4
+ .coverage
5
+ htmlcov/
6
+ .pytest_cache/
7
+ __pycache__/
8
+ *.history.jsonl
9
+ *.runs.jsonl
10
+ docs/_build/
11
+ docs/_generated/
12
+ /data/
13
+ /out/
14
+ scripts/
15
+ dist/
@@ -0,0 +1 @@
1
+ 3.13
@@ -0,0 +1,383 @@
1
+ # mcaligner — implementation notes
2
+
3
+ Notes for coding agents (Codex, Claude Code) and humans picking up this repo.
4
+ Keep this file current when you change the design or finish a milestone.
5
+ Most of `../llmparser/AGENTS.md` (conventions, environment gotchas) applies here too.
6
+
7
+ ## Goal
8
+
9
+ Align the **words** of two parsed versions of the same text (e.g. an Arabic
10
+ translation of Ephesians and its Greek Vorlage). Sentences (verses) are paired
11
+ by `sent_id`; links are Ugarit style (`3-5S` sure, `12-18P` probable); the
12
+ links feed a bilingual lexicon. All corpus and alignment data lives in
13
+ MCParser's `CorpusStore` and is reached by the agents through MCParser's MCP
14
+ server. The handoff brief is `INSTRUCTIONS.md`.
15
+
16
+ ## Status (2026-10-14)
17
+
18
+ | Milestone | State |
19
+ |---|---|
20
+ | 1. Package set-up, fixture tests (`convert` has since been removed) | done, 100% covered |
21
+ | 2. `evaluate` (AER, P/R/F1) and `lexicon` | done, 100% covered |
22
+ | 3. Rules, precision measured on the 155 verses | done, 100% covered (see Rules below) |
23
+ | 4. Aligner agent, scorer, pipeline; small real run | done, 100% covered; 2 real 5-verse runs ($0.21, $0.18) |
24
+ | 5. Sphinx docs, AGENTS.md | done; `./mkdocs.sh` (`sphinx-build -E -W`) clean |
25
+
26
+ The old implementation is in `llmalignerold/` (LangChain + retriv/chroma). It is
27
+ not imported.
28
+
29
+ ## Layout
30
+
31
+ ```
32
+ mcaligner/
33
+ evaluate.py # P/R/F1 over sure and over all links, AER; per pair + micro total
34
+ lexicon.py # TranslationTable (lemma -> linked lemmas, one pass, leave-one-pair-out) + lexicon export
35
+ guidelines/ # bundled annotation guidelines (grc-lat.md, bicross.md, grc-ar.md: a draft for Greek-Arabic, unvalidated; grc-cop.md: a draft for Greek-Coptic, unvalidated), added to the agents' instructions (setting `guidelines`, default grc-lat)
36
+ rules.py # code-only links: identical symbols, transliterated names, dominant lexicon match; measure()
37
+ settings.py # Settings (YAML): tiers, per-role model/thinking/instructions/enabled, batching, rules
38
+ agents/
39
+ prompts.py # instructions (aligner, scorer); compact pair blocks with settled/review links + evidence
40
+ outputs.py # Alignment/Scores answers (lines), parsing, Applier (dry-run set_links, then apply)
41
+ toolset.py # ReadToolset: 5 read MCP tools, JSON-string list args repaired, compact results
42
+ team.py # AlignerTeam: one pydantic-ai Agent per role over MCPToolset(build_server(store)); usage/cost
43
+ mcp/ # tools.py (similar_pairs, lemma_links, link_examples, pair_words, category_links, alignment_stats), server.py (MCParser's server + these), cli.py (mcp serve/config)
44
+ pipeline.py # rules -> align -> score -> rules rounds, batching, concurrency, RunReport
45
+ cli.py # typer app: evaluate, lexicon, rules, run, settings, mcp (MCParser's mcp_app)
46
+ docs/ # Sphinx (myst-parser + furo); CLI page rendered from --help in conf.py
47
+ tests/ # pytest; tests/data/{ar,grc}.conllu + silver.jsonl = verses B10K1V1-3 of results/ArabicAligned.yaml;
48
+ # tests/scripted.py = scripted FunctionModels (align(), score()) for agent tests
49
+ ```
50
+
51
+ ## Data
52
+
53
+ - `results/ArabicAligned.yaml` (untracked): 155 verses of Ephesians, Arabic
54
+ (`ArabGr1-Arabic-Ephesians-Parsed`, with multiword tokens, MISC
55
+ `Gloss=`/`Translit=`/`LTranslit=`/`Root=`) and Greek
56
+ (`ArabGr1-GreekVorlage-Ephesians-Parsed`, PROIEL-style, unaccented forms,
57
+ accented lemmas). `alignment` = old LLM output: **silver**, not gold.
58
+ - It was split once (by a `convert` command that has been removed, along with
59
+ its code) into `data/<corpus>.conllu` ×2 and `data/ArabicAligned.jsonl`
60
+ (`/data/` is ignored; keep them, they cannot be regenerated from the repo).
61
+ Silver totals: 2095 links (1967 S, 128 P); in the split 2094 links in 155 pairs.
62
+ - Data quirks found then: 4 Arabic multiword lines had 9 columns (padded with
63
+ `_`); one silver link pointed to a missing word (`B10K2V16 16-16P`, dropped);
64
+ duplicate links were merged (first kept).
65
+ - The Greek side has **no punctuation tokens** (Arabic has 116 PUNCT), so a
66
+ punctuation rule cannot fire on this text.
67
+
68
+ ## Design decisions (and why)
69
+
70
+ - **Package**: uv + hatchling; MCParser (package and CLI `mcparser`, formerly
71
+ llmparser) from PyPI (`mcparser>=0.2.0a1`, a pre-release, hence
72
+ `[tool.uv] prerelease = "allow"`). To develop against the local repository
73
+ `../llmparser` add `[tool.uv.sources] mcparser = { path = "../llmparser", editable = true }`.
74
+ - **`mcp serve` / `mcp config`** are `mcaligner/mcp/cli.py`: MCParser's options
75
+ (reusing its typer option types) over `mcaligner.mcp.build_server`, which
76
+ is MCParser's server plus the exploration tools; the Claude Desktop config
77
+ runs `python -m mcaligner mcp serve`.
78
+ - **Alignment files** are MCParser's JSON Lines sets. Imported links carry no
79
+ confidence, so a read-only set gives them its default (1.0, or e.g. `:ro:0.8`
80
+ in the alignment spec). `mcparser align import/export` (MCParser, after 0.2.0a1)
81
+ converts to and from Pharaoh/Ugarit text and YAML.
82
+
83
+ - **Evaluation** (`evaluate.py`): A = predicted, S = gold sure, P = gold all.
84
+ "sure" P/R/F1 compares predicted *sure* links with S; "all" compares A with P
85
+ ignoring S/P; `AER = 1 − (|A∩S| + |A∩P|) / (|A| + |S|)`. Only gold pairs are
86
+ scored (gold often covers part of a text); a gold pair missing from the
87
+ predictions counts as empty; `--annotated-only` scores only pairs with
88
+ predicted links; `--min-confidence` drops weak predicted links (links without
89
+ a confidence are kept). Totals are micro-averaged. Empty denominators give 0.
90
+ It reads the JSONL files directly: no sources needed.
91
+ - **Translation table** (`lexicon.TranslationTable`): same counting as
92
+ MCParser's `translation_candidates` (lemma falls back to form;
93
+ `min_confidence` on links), keyed by **source name** (not side), built in one
94
+ pass, with per-pair counts so a pair can be left out (`exclude=(set, sent_id)`)
95
+ — needed when measuring rules on the same data the table comes from. Code uses
96
+ the table (calling `translation_candidates` per word is O(pairs) each); agents
97
+ can still call the MCP tool.
98
+ - **Lexicon export** (`mcaligner lexicon -s ... -a ... [-o x.yaml|x.json]`):
99
+ `{source: {lemma: {part_of_speech, occurrences, unaligned, alignments:
100
+ {lemma: {count, part_of_speech}}}}}`; default `--min-confidence 0.8`. On the
101
+ silver set it reproduces the old `results/ArabicLexicon.yaml` exactly (1138
102
+ lemmas) except for the dropped `B10K2V16 16-16P` link (old code counted a link
103
+ to a missing word as "aligned").
104
+
105
+ ## Greek-Latin gold standard
106
+
107
+ `/Users/rturnbull/MDAP/BICROSS/Alignment/Alignment-Gold-Standards/grc-lat`: 100 DFHG
108
+ fragments (raw text only, 1983 grc / 1860 lat tokens, 1593 links, 0-based Ugarit).
109
+ Workflow: `mcparser init grc.txt -o grc.conllu` (and lat), parse with the PROIEL
110
+ treebanks (`UD/UD_Ancient_Greek-PROIEL`, `UD/UD_Latin-PROIEL`), then
111
+ `uv run python scripts/grc_lat_gold.py GOLD.json grc.conllu lat.conllu -o gold.jsonl` (a disposable script, not part of the package) maps each
112
+ gold token to the parsed words covering the same characters (a link goes to every
113
+ overlapping word), so it survives tokenisation differences. `init` only detaches `«` and the
114
+ full stop of `rep.` (3 fragments: grc 64, lat 26, lat 64; 1593 -> 1597 links). Work files are in `data/grc-lat/`.
115
+ Do not let a parser change the segmentation after the gold is mapped.
116
+
117
+ ### Greek-Latin DFHG dev set (20 fragments; `data/grc-lat/`)
118
+
119
+ Dev ids in `dev-ids.txt` (stratified by length, seed 42; `scripts/grc_lat_split.py`); gold-dev 313 links.
120
+ Parses: grc $17.74, lat $16.83 (cheap-first, syntax on Opus, 3 rounds; the PROIEL treebanks as examples).
121
+ Cold start (no evidence sets), 2 rounds, Opus aligner + Sonnet scorer, evaluated against gold-dev:
122
+
123
+ | Run | Cost | Requests | All P/R/F1 | AER |
124
+ |---|---|---|---|---|
125
+ | 1 (old prompt) | $0.84 | 34 | 0.889/0.869/0.879 | 0.088 |
126
+ | 2 (old prompt; run-to-run noise) | $0.58 | 36 | 0.886/0.866/0.876 | 0.089 |
127
+ | 3 (ids fixed, see below) | $0.60 | 23 | 0.895/0.869/0.882 | 0.083 |
128
+ | 3 without its 15 punctuation links (post-hoc) | – | – | 0.941/0.869/0.904 | 0.058 |
129
+ | 4 (+ guidelines in the prompts, punctuation dropped in code) | $0.77 | 22 | 0.955/0.885/0.919 | 0.044 |
130
+
131
+ - The prompt's answer example used an Ephesians id (`B10K1V1`), so models wrote `B98` for pair `98` and were
132
+ retried (15 of 36 requests). The examples are now `<id>` and the task header lists the ids: 1 retry in run 3.
133
+ - `TaskResult.details` (requests, tool calls by name, retry messages) is in the report. Almost no tool calls were
134
+ made (1 `pair_words` in run 3, none in the others): the prompt evidence is enough.
135
+ - Error analysis (pairs 80, 43, 60, 96, 84): (1) the aligner links punctuation (`,`~`,`, `«`~`«`), which the guidelines
136
+ forbid; (2) links the guidelines rule out (`τούτου`~`Sabinorum`, `τῇ`~`libro`, an article to `filius`); (3) misses
137
+ particle groups (`ἔτι δὲ καὶ`~`nec non`) and crossed pairs (`τότε πρῶτον`~`tunc demum`); (4) the names rule's wrong
138
+ `Ἡφαίστου`~`festi` (0.95) where `Vulcani` is right. Done in run 4: `mcaligner/guidelines/grc-lat.md` (a condensed version, from `../guidelines/grc-lat.md`) is in the
139
+ aligner's and scorer's instructions by default (`guidelines`), and links to punctuation are dropped in code
140
+ (`rules.link_punctuation`); the model made 0 such links with the guidelines. Human-human on the DFHG gold:
141
+ Dice 0.927 (from the README's flattened counts 1487/1486, 1378 agreed); BICROSS NT annotators: 0.948.
142
+ The dev set was used to choose these changes, so confirm on the 80 test fragments before quoting a number.
143
+
144
+ **Column ablation** (same 20 dev pairs; guidelines, punctuation filter, rules, models and `--tools none` fixed; only
145
+ `--columns` varies; one run each; `data/grc-lat/ablation/`; F1 over all links, 95% bootstrap over the 20 pairs):
146
+
147
+ | Columns shown | Cost | F1 | AER |
148
+ |---|---|---|---|
149
+ | forms only (no parse) | $0.40 | 0.912 [0.881, 0.944] | 0.053 |
150
+ | lemma upos | $0.48 | 0.916 | 0.047 |
151
+ | lemma upos head deprel (default) | $0.54 | 0.907 | 0.052 |
152
+ | + feats | $0.73 | 0.913 | 0.045 |
153
+ | + feats xpos | $0.71 | 0.916 | 0.047 |
154
+ | default columns with all tools (run 4) | $0.77 | 0.919 | 0.044 |
155
+
156
+ No difference is detectable: every paired difference is within about ±0.02 F1 (e.g. forms vs default +0.005 [-0.016, +0.025]),
157
+ which is the resolution of 20 pairs / 313 links. So on DFHG Greek-Latin the agents do not need the parse in the
158
+ prompt, and FEATS only adds cost (+35%). Caveats: the rules (names rule: 43 links) still use the parse; cold start,
159
+ so the lemma-based prompt evidence was empty; Opus knows classical Greek and Latin well. The same test on the BICROSS
160
+ manuscript text (variant spellings, nomina sacra) and with accumulated evidence is still to do.
161
+
162
+ **Column ablation on BICROSS** (20 verses chosen like the DFHG dev set, `scripts/grc_lat_split.py`; ids in
163
+ `data/bicross/dev-ids.txt`, the other 79 verses are the untouched test set; consensus-dev = 322 links; same settings):
164
+
165
+ | Columns shown | Cost | F1 vs consensus | AER |
166
+ |---|---|---|---|
167
+ | forms only | $0.29 | 0.905 | 0.084 |
168
+ | lemma upos | $0.31 | 0.903 | 0.081 |
169
+ | lemma upos head deprel | $0.37 | 0.904 | 0.079 |
170
+ | + feats | $0.55 | 0.902 | 0.084 |
171
+ | + feats xpos | $0.59 | 0.903 | 0.084 |
172
+
173
+ Again no difference (paired differences within ±0.01). The two annotators agree at F1 0.955 on these 20 verses. The
174
+ whole gap is one word class: the model misses 46 of the 53 consensus links that start at a Greek article
175
+ (`το~aqua`, `το~spiritus`) and almost nothing else (precision 0.98, recall 0.84). The BICROSS annotators link the
176
+ article to its noun's translation (88-95% of articles, mostly to NOUN), whereas DFHG's gold leaves ~90% of articles
177
+ unlinked, which is what the bundled `grc-lat` guidelines say. Guidelines are therefore per corpus: BICROSS needs its
178
+ own article rule (to be tested; the dev set was used to find this).
179
+
180
+ **BICROSS guidelines** (`--guidelines bicross`, `mcaligner/guidelines/bicross.md` from `Adam/.../BICROSS_guidelines_grc-lat.odt`,
181
+ 13 Aug 2024; the v2 .odt of 28 May says the opposite about articles and predates both workbooks). Same 20 dev verses, no tools:
182
+
183
+ | Run | Cost | F1 vs consensus | vs Kreinecker | vs Tilke | AER (consensus) |
184
+ |---|---|---|---|---|---|
185
+ | `grc-lat` guidelines, forms only | $0.29 | 0.905 | 0.893 | 0.895 | 0.084 |
186
+ | `bicross` guidelines, forms only | $0.40 | 0.978 | 0.957 | 0.967 | 0.016 |
187
+ | `bicross` guidelines, lemma upos head deprel | $0.54 | 0.977 | 0.955 | 0.965 | 0.018 |
188
+
189
+ The annotators agree with each other at F1 0.955 on these verses, so the aligner is at the level of the human agreement
190
+ (against each annotator separately). The consensus (links of either annotator) is a lenient reference; compare with the
191
+ per-annotator figures. The guidelines were chosen after seeing the article gap on this dev set: confirm on the 79 test verses.
192
+
193
+ **BICROSS held-out test** (79 verses never used for tuning; `data/bicross/align-test.sh`; defaults except `--guidelines bicross`:
194
+ columns lemma upos head deprel, all tools allowed, rules on, 2 rounds, Opus aligner + Sonnet scorer, cold start). Cost $2.64
195
+ (Opus $1.84, Sonnet $0.80), 86 tasks / 86 requests, 0 retries, 0 tool calls, 0 failed, 0 punctuation links dropped. 1412 links.
196
+ F1 over all links (AER): vs Kreinecker 0.947 (0.053), vs Tilke 0.975 (0.025), vs consensus 0.969 (0.013). The two annotators
197
+ agree with each other at F1 0.946 (AER 0.054) on the same 79 verses, so the pipeline is at the level of human agreement. It
198
+ agrees more with Tilke (who links more, 1404 vs 1371 links) than with Kreinecker. Dev (tuned) was 0.957 / 0.967 / 0.978.
199
+
200
+ **DFHG held-out test** (80 fragments never used for tuning; `data/grc-lat/align-test.sh`; defaults, `grc-lat` guidelines, cold start,
201
+ 2 rounds): $2.34 (Opus $1.69, Sonnet $0.66), 88 tasks / 88 requests, 0 retries, 0 tool calls, 0 failed, 2 punctuation links
202
+ dropped. 1215 links vs 1284 gold: sure P/R/F1 0.967/0.877/0.920; all links 0.948/0.897/0.922, AER 0.042 (dev: 0.919, 0.044). The
203
+ DFHG annotators agree at Dice about 0.927 (README counts, all 100 fragments), so the pipeline is at that level.
204
+ Remaining errors (132 missed, 63 extra): missed = verbs and participles linked to relatives or subordinators (`παροικοῦντες~qui`,
205
+ `ἠνάγκαζε~ut`), prepositions the Latin renders by case (`ἐν~Babyloniae`), adverbs; extra = `ἐν~libro` (the v2 guideline's "libro"
206
+ exception and the `μέν`/`δέ` rule are not in the short `grc-lat.md`). The test errors have now been seen: tune guidelines on dev only.
207
+
208
+ **Column ablation on the held-out test sets** (`scripts/ablation_test.sh`; corpus guidelines, `--tools none`, defaults otherwise; one run each;
209
+ F1 over all links with 95% bootstrap over the test pairs):
210
+
211
+ | Test set | words only | + lemma, UPOS | default columns + all tools (earlier run) | lemma-upos minus words only |
212
+ |---|---|---|---|---|
213
+ | DFHG (80) vs gold | 0.927 [0.911, 0.941], $1.55 | 0.922, $1.88 | 0.922, $2.34 | -0.005 [-0.011, +0.002] |
214
+ | BICROSS (79) vs Kreinecker | 0.948, $1.76 | 0.946, $1.95 | 0.946, $2.64 | -0.002 [-0.006, +0.003] |
215
+ | BICROSS vs Tilke | 0.974 | 0.973 | 0.975 | -0.002 [-0.005, +0.002] |
216
+
217
+ Words only is as good as with the parse in all four comparisons (never worse than -0.005) and 10-17% cheaper than lemma+UPOS,
218
+ about 35% cheaper than the default configuration. The parse in the prompt does not help alignment on these two corpora (cold start,
219
+ rules unchanged). Not tested: the evidence path (lemma-based hints with accumulated links) and lower-resource languages.
220
+ - Rules on the dev set: names 43 links (41 in gold); `identical` linked punctuation only (0 right): now off
221
+ (`link_punctuation`); lexicon with the full gold as leave-one-out evidence adds 3 (all right).
222
+ - BICROSS (`data/bicross/`): Greek and Latin manuscript text parsed the same way (resumable `parse-*.sh`); two
223
+ annotators' alignments, IRR AER 0.052 (`scripts/bicross_prepare.py`, `irr.json`).
224
+
225
+ ## Rules (no model)
226
+
227
+ `rules.py`; `mcaligner rules -s ... -a ... --target pred` applies them,
228
+ `--measure SET [--show-errors]` measures them. Rules only link words that have
229
+ no link yet, only 1-1, only when the match is unique in the pair, and write
230
+ sure links with confidence 0.95 (identical: 0.98), agent `rules`.
231
+
232
+ - **identical**: same form on both sides, unique on each side, and a symbol
233
+ (punctuation only with `link_punctuation`, off by default: the Greek-Latin DFHG
234
+ gold never aligns it, and the first measure linked 7 punctuation marks, 0 right)
235
+ (UPOS PUNCT/NUM/SYM or no letters).
236
+ - **names**: one side PROPN, neither side a function word; consonant
237
+ skeletons (romanise: NFD, strip marks, Greek → Latin; then drop vowels and
238
+ y/w/h/j, merge consonant classes b/p/f/v/ph, t/d/th, k/g/q/c/kh, s/z; collapse
239
+ doubles) of MISC `LTranslit`/`Translit`, lemma and form; skeletons shorter than
240
+ 3 must be equal, otherwise `SequenceMatcher` ≥ `name_similarity` (0.9); the
241
+ best match must have no equal-or-better rival sharing one end.
242
+ - **lexicon**: from a `TranslationTable` over the evidence sets (default all),
243
+ counting links ≥ `evidence_confidence` (0.96, above the rules' 0.95, so rules
244
+ never feed themselves — the MCParser trick). Fires when the lemma's top
245
+ translation has ≥ `min_count` (5) links and ≥ `min_share` (0.95) of **all**
246
+ the lemma's occurrences, unlinked ones included (so mostly-unlinked words like
247
+ the Greek article never fire), the lemma is unique in its sentence and the
248
+ translation occurs exactly once (unlinked) in the other. Both directions.
249
+ **The sentence under work is always left out of the evidence.**
250
+
251
+ **Measured** on the 155 Ephesians verses against the silver links, starting each
252
+ pair from no links, leave-one-out evidence (`--measure silver`):
253
+
254
+ | Rule | Links | In silver | Real precision (after reading the disagreements) |
255
+ |---|---|---|---|
256
+ | identical | 0 | – | – (the Greek has no punctuation) |
257
+ | names (threshold 0.9) | 20 | 18 | 20/20: both disagreements are silver errors (silver links Jesus↔Christ crosswise in B10K3V6, and Ἰησοῦ↔و in B10K3V21) |
258
+ | lexicon (5, 0.95) | 129 | 127 | 128/129: B10K4V29 لاكن→ἀλλ' is right (silver is wrong); B10K3V13 أنا "I" → μου "my" is wrong (pronoun lemmas cover every case) |
259
+
260
+ 149 of the 2094 silver links (7%) are settled by rules at the start; the lexicon
261
+ rule grows as confident agent links accumulate. Settings tried: names at 0.8
262
+ added جسد "body" ~ χριστῷ (ǧasad vs khristo, 0.86) → threshold 0.9. Lexicon at
263
+ (3, 0.95): 198 links, same precision; at (3, 0.9): 229 links with 3 real errors
264
+ (ὑπό→مِن, مثل→ὡς, أنا→μου) ≈ 98.7%; at (5, 0.8): 265 links, 94.7% vs silver.
265
+ Kept MCParser's conservative (5, 0.95). **The silver set is noisy** (crossing
266
+ errors in "Jesus Christ"/"Christ Jesus"), so precision against silver is a
267
+ lower bound.
268
+
269
+ ## Agents and pipeline
270
+
271
+ - **Two roles**: `aligner` (strong tier, Opus 5.5) and `scorer` (efficient
272
+ tier, Sonnet 5.5; `--no-scorer` or `roles: {scorer: {enabled: false}}`).
273
+ Per-role `model`, `thinking`, `instructions`; tiers and `model` as in MCParser.
274
+ - **Data access**: agents get only read MCP tools (`get_parallel`,
275
+ `translation_candidates`, `analyse_form`, `find_tokens`, `get_context`)
276
+ through `MCPToolset(build_server(store))`; answers are applied by code through
277
+ the store (`set_links(dry_run=True)` for every pair, retry on any error, then
278
+ `set_links`), as MCParser's Applier does. Results of the alignment tools
279
+ are compacted here; the monolingual ones use MCParser's `compact_result`.
280
+ - **Aligner answer**: `links` lines `SENT_ID A-B S|P CONFIDENCE` (S/P may be
281
+ attached: `3-5S`; missing = S) and `remove` lines `SENT_ID A-B`. New links of a
282
+ pair = settled links (≥ threshold, shown in the prompt and kept) − removals +
283
+ answer lines. Links to review that the answer omits are dropped. **Every pair
284
+ of the task is written** (even unchanged), so the history shows the aligner
285
+ saw it (`seen_by_aligner`).
286
+ - **Scorer answer**: `scores` lines `SENT_ID A-B CONFIDENCE`, only for the links
287
+ listed to score; applied with `replace=False` (S/P kept).
288
+ - **Provenance** comes from MCParser's `Link.agent` (added for this project):
289
+ rules write `rules`, the aligner `aligner`, the scorer `scorer`; an unchanged
290
+ link keeps its agent. The scorer gets links whose agent is `aligner`. If the
291
+ scorer gives the same confidence, the link stays `aligner`, so the pipeline
292
+ also remembers `(sent_id, key, S/P, confidence)` it has rated (in memory).
293
+ - **Prompt** per pair: `id form lemma UPOS head deprel Gloss` lines (MISC keys
294
+ in `misc_keys`), multiword tokens as `5-6 form (token of words 5-6)`, settled
295
+ links and links to review with confidence and agent, and for each unlinked
296
+ non-punctuation word the top `candidates` translations from a
297
+ `TranslationTable` (links ≥ `evidence_confidence` 0.8, this sentence left
298
+ out): `B10 Χριστός: مَسِيح×25, المسيح×11, يسوع×5 (of 43, 3 unlinked)`.
299
+ Two Ephesians verses ≈ 5k characters.
300
+ - **Rounds**: rules; then per round align (unseen pairs + pairs with links <
301
+ threshold), score, rules again (more evidence). Stop when no task, no pair's
302
+ links changed (Ugarit text, ignoring confidences), nothing below threshold,
303
+ or the round limit (default 2). Batches: ≤ `batch_sentences` (5) pairs and
304
+ ≤ `batch_words` (150) words. A failing task is recorded and the run goes on;
305
+ `UserError` (missing API key) aborts. Each run appends its report to
306
+ `<set>.runs.jsonl`; `--report` writes it as JSON. Tokens and cost per model are
307
+ printed (`mcparser.pricing.estimate`).
308
+
309
+ ## Real-model observations (Opus 5.5 aligner, Sonnet 5.5 scorer)
310
+
311
+ Ephesians 1:1–5 (B10K1V1–5), evidence = silver **without those 5 verses**
312
+ (`data/silver-heldout.jsonl`), so neither the rules, the prompt evidence nor the
313
+ tools can see the answer. 2 rounds, batch 5. Evaluated against silver
314
+ (`evaluate --annotated-only`):
315
+
316
+ | Run | Cost | Requests | All P/R/F1 | AER | ≥0.8 links: P/R | below 0.8 |
317
+ |---|---|---|---|---|---|---|
318
+ | 1 | $0.21 (Opus $0.16) | 6 (1 per task) | 0.825/0.930/0.874 | 0.130 | 0.952/0.831 (62 links) | 18 of 80 |
319
+ | 2 (morpheme guidance) | $0.18 (Opus $0.14) | 6 | 0.835/0.930/0.880 | 0.124 | 0.892/0.817 (65 links) | 14 of 79 |
320
+
321
+ - Rules settled 13 of ~80 links (names 5, lexicon 8). No tool calls were made:
322
+ the prompt evidence was enough, every task was a single request. ~1 minute.
323
+ - **Most "false positives" are right.** (1) Silver is wrong where Arabic has
324
+ "Jesus Messiah" for Greek "Christ Jesus" (it links them crosswise); the
325
+ aligner and the names rule get it right. (2) The Arabic parse does not split
326
+ pronoun suffixes (ربنا "our Lord", باركنا "blessed us", اختارنا "chose us",
327
+ هواه "his desire") or some prepositions (بكل "with all"); the aligner links
328
+ these to ἡμῶν/ἡμᾶς/αὐτοῦ/πάσῃ (one-to-many), which silver mostly lacks.
329
+ So precision against silver underestimates; a small gold set is needed.
330
+ - Run 1: the scorer rated those suffix links 0.3–0.4 ("doubtful or wrong"),
331
+ so round 2 re-sent them, the aligner kept them, nothing improved. Run 2 added
332
+ "a word may contain morphemes the parse did not split off" to both prompts:
333
+ fewer links below threshold, but round 2 still changed 1 of 4 pairs both
334
+ times for ~1/3 of the cost. **For large runs consider `--rounds 1`** (the
335
+ scorer still sets confidences, e.g. for `lexicon --min-confidence`).
336
+ - Cache writes exceed reads in a run of 6 requests (writes cost 1.25× input);
337
+ caching pays off on long runs where instructions and tool definitions repeat.
338
+
339
+ ## MCParser additions (for this project)
340
+
341
+ - `Link.agent` (who last set or changed a link; unchanged links keep it;
342
+ `get_parallel` returns `agents`), and `set_links` now copies the links it is
343
+ given (it used to be able to mutate the caller's/stored `Link` objects).
344
+ - `dry_run` on the MCP `set_links` tool (the store method had it).
345
+
346
+ ## Environment gotchas
347
+
348
+ - `uv sync --python /opt/homebrew/opt/python@3.13/bin/python3.13` (the
349
+ uv-managed 3.12 is x86_64 and breaks `cryptography`).
350
+ - **MCParser was called mcparser** (2026-10-15 rename): the package and CLI are
351
+ now `mcparser`, but the repository is still `../llmparser`. Imports are
352
+ `from mcparser...`; the Claude Desktop config runs `python -m mcparser`. Older
353
+ run reports and notes may still say mcparser.
354
+ - If `VIRTUAL_ENV` points at MCParser's venv (e.g. from the IDE), uv warns and
355
+ ignores it; `unset VIRTUAL_ENV` to silence it.
356
+ - The test fixture YAML is dumped with escaped tabs (`"1\tبولس..."`), so edit
357
+ it through `yaml.safe_load`, not with text replacement.
358
+
359
+ ## Conventions
360
+
361
+ - Docs: `./mkdocs.sh` (= `uv run --group docs sphinx-build -E -W ...`). The
362
+ CLI page is generated from `--help` (typer vendors click, so no sphinx-click);
363
+ add new commands to `CLI_COMMANDS` in `docs/conf.py`.
364
+
365
+ - `uv run pytest` should stay at **100% line+branch coverage** (reported, not enforced: CI
366
+ only fails when a test fails) and fast. No network: `conftest.py` sets
367
+ `pydantic_ai.models.ALLOW_MODEL_REQUESTS = False`.
368
+ - Generic store/MCP functionality goes into MCParser (tested at 100%); alignment
369
+ exploration tools specific to this project live in `mcaligner/mcp/`.
370
+ - Raise `CorpusError` subclasses for expected failures; the CLI turns them
371
+ (and `OSError`, pydantic `ValidationError`, pydantic-ai `UserError`) into
372
+ `Error: ...` with exit status 1.
373
+
374
+ ## Ideas / next steps
375
+
376
+ - A small hand-checked **gold** set (silver is noisy: Jesus/Christ crossings,
377
+ missing suffix links), then tune threshold, rounds and rule settings on it.
378
+ - `--rounds 1` as the default if round 2 keeps changing little on larger runs.
379
+ - Anthropic Message Batches (50% off): most tasks are a single request.
380
+ - Many-to-one rules (e.g. Arabic suffixed pronoun → Greek pronoun when the
381
+ host word is linked) once a gold set shows how often they hold.
382
+ - A per-lemma index in MCParser's `translation_candidates` if agents call it
383
+ a lot on large sets (it scans all pairs per call).
@@ -0,0 +1,20 @@
1
+ # Changelog
2
+
3
+ All notable changes to MCAligner. The format follows [Keep a Changelog](https://keepachangelog.com/).
4
+
5
+ ## [0.2.0a1]
6
+
7
+ First alpha release on PyPI. Install with `pip install --pre mcaligner`.
8
+
9
+ ### Added
10
+ - `mcaligner run`: rules, aligner agent, scorer agent and refinement rounds over two parsed CoNLL-U files, with
11
+ batching, concurrency, a cost report and a resumable alignment set (JSON Lines, MCParser's format).
12
+ - Rules without a model (identical symbols, transliterated names, dominant lexicon match).
13
+ - `mcaligner evaluate` (P/R/F1 over sure and all links, AER), `lexicon` (bilingual lexicon export), `rules`
14
+ (apply or measure the rules), `settings`, and `mcp serve` / `mcp config` (MCParser's MCP server plus the
15
+ exploration tools `similar_pairs`, `lemma_links`, `link_examples`, `pair_words`, `category_links`,
16
+ `alignment_stats`).
17
+ - Bundled annotation guidelines (`--guidelines`): `grc-lat` (default), `bicross`, and drafts for `grc-cop`,
18
+ `grc-ar`, `lat-ar`, `grc-syr`, `syr-ar`, `got-lat`.
19
+ - `--columns`, `--tools` and `--link-punctuation` to control what the agents see and may do.
20
+ - `mcaligner --version`, documentation, and a project logo.
@@ -0,0 +1,19 @@
1
+ cff-version: 1.2.0
2
+ message: "If you use MCAligner, please cite it as below."
3
+ title: "MCAligner: word alignment of parallel texts with LLM agents"
4
+ type: software
5
+ authors:
6
+ - family-names: Turnbull
7
+ given-names: Robert
8
+ affiliation: "Melbourne Data Analytics Platform, University of Melbourne"
9
+ website: "https://robturnbull.com"
10
+ version: 0.2.0a1
11
+ license: Apache-2.0
12
+ repository-code: "https://github.com/rbturnbull/mcaligner"
13
+ url: "https://rbturnbull.github.io/mcaligner/"
14
+ keywords:
15
+ - digital humanities
16
+ - word alignment
17
+ - parallel texts
18
+ - LLM agents
19
+ - MCP
@@ -0,0 +1 @@
1
+ @AGENTS.md
@@ -0,0 +1,37 @@
1
+ # Contributing
2
+
3
+ Thanks for helping with MCAligner. Bug reports, feature ideas and pull requests are welcome.
4
+
5
+ ## Setting up
6
+
7
+ ```bash
8
+ git clone https://github.com/rbturnbull/mcaligner && cd mcaligner
9
+ uv sync --all-groups
10
+ uv run pytest
11
+ ```
12
+
13
+ Python 3.11 to 3.14 are supported. The repository pins 3.13 for development (`.python-version`).
14
+
15
+ ## Before opening a pull request
16
+
17
+ - `uv run pytest` must pass. Aim for **100% line and branch coverage** (it is reported, but a shortfall does not fail CI). The tests never call a real model or
18
+ the network; agents are tested with the scripted models in `tests/scripted.py`.
19
+ - `./mkdocs.sh` (Sphinx with warnings as errors) must build the documentation. Update the docs and
20
+ `CHANGELOG.md` for user-visible changes.
21
+ - Keep `uv.lock` current when you change dependencies (`uv lock`); CI installs with `--locked`.
22
+ - Match the surrounding code: type hints, short docstrings, pydantic models for return values, and
23
+ `CorpusError` subclasses for expected failures.
24
+
25
+ `AGENTS.md` records the design decisions and the measurements behind them. Read it before changing
26
+ how the agents, rules or scorer behave, and add what you learned to it.
27
+
28
+ ## Changes that affect accuracy or cost
29
+
30
+ Say what you measured. The agents' output varies between runs, so a difference of about 0.015 F1
31
+ on 80 sentence pairs is noise. Give the corpus, the number of pairs, the guidelines, the models,
32
+ the cost and ideally two runs. Tune on a dev subset and report on a held-out one.
33
+
34
+ ## Reporting a bug
35
+
36
+ Include `mcaligner --version`, the command, the settings and the error. Do not paste API keys or text
37
+ you may not share.