mcaligner 0.2.0a1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mcaligner-0.2.0a1/.github/workflows/docs.yml +62 -0
- mcaligner-0.2.0a1/.github/workflows/publish.yml +31 -0
- mcaligner-0.2.0a1/.github/workflows/testing.yml +52 -0
- mcaligner-0.2.0a1/.gitignore +15 -0
- mcaligner-0.2.0a1/.python-version +1 -0
- mcaligner-0.2.0a1/AGENTS.md +383 -0
- mcaligner-0.2.0a1/CHANGELOG.md +20 -0
- mcaligner-0.2.0a1/CITATION.cff +19 -0
- mcaligner-0.2.0a1/CLAUDE.md +1 -0
- mcaligner-0.2.0a1/CONTRIBUTING.md +37 -0
- mcaligner-0.2.0a1/INSTRUCTIONS.md +201 -0
- mcaligner-0.2.0a1/LICENSE +202 -0
- mcaligner-0.2.0a1/PKG-INFO +195 -0
- mcaligner-0.2.0a1/README.rst +172 -0
- mcaligner-0.2.0a1/docs/_static/.gitkeep +0 -0
- mcaligner-0.2.0a1/docs/_static/mc.svg +512 -0
- mcaligner-0.2.0a1/docs/_static/mcaligner-text-dark.svg +53 -0
- mcaligner-0.2.0a1/docs/_static/mcaligner-text.svg +53 -0
- mcaligner-0.2.0a1/docs/agents.md +87 -0
- mcaligner-0.2.0a1/docs/api.md +14 -0
- mcaligner-0.2.0a1/docs/conf.py +62 -0
- mcaligner-0.2.0a1/docs/data.md +49 -0
- mcaligner-0.2.0a1/docs/development.md +28 -0
- mcaligner-0.2.0a1/docs/evaluation.md +39 -0
- mcaligner-0.2.0a1/docs/index.md +80 -0
- mcaligner-0.2.0a1/docs/mcp.md +35 -0
- mcaligner-0.2.0a1/docs/preparing.md +72 -0
- mcaligner-0.2.0a1/docs/quickstart.md +49 -0
- mcaligner-0.2.0a1/docs/results.md +55 -0
- mcaligner-0.2.0a1/docs/rules.md +38 -0
- mcaligner-0.2.0a1/mcaligner/__init__.py +5 -0
- mcaligner-0.2.0a1/mcaligner/__main__.py +3 -0
- mcaligner-0.2.0a1/mcaligner/agents/__init__.py +1 -0
- mcaligner-0.2.0a1/mcaligner/agents/outputs.py +189 -0
- mcaligner-0.2.0a1/mcaligner/agents/prompts.py +237 -0
- mcaligner-0.2.0a1/mcaligner/agents/team.py +165 -0
- mcaligner-0.2.0a1/mcaligner/agents/toolset.py +139 -0
- mcaligner-0.2.0a1/mcaligner/cli.py +298 -0
- mcaligner-0.2.0a1/mcaligner/evaluate.py +147 -0
- mcaligner-0.2.0a1/mcaligner/guidelines/bicross.md +140 -0
- mcaligner-0.2.0a1/mcaligner/guidelines/got-lat.md +132 -0
- mcaligner-0.2.0a1/mcaligner/guidelines/grc-ar.md +146 -0
- mcaligner-0.2.0a1/mcaligner/guidelines/grc-cop.md +166 -0
- mcaligner-0.2.0a1/mcaligner/guidelines/grc-lat.md +162 -0
- mcaligner-0.2.0a1/mcaligner/guidelines/grc-syr.md +65 -0
- mcaligner-0.2.0a1/mcaligner/guidelines/lat-ar.md +73 -0
- mcaligner-0.2.0a1/mcaligner/guidelines/syr-ar.md +57 -0
- mcaligner-0.2.0a1/mcaligner/lexicon.py +151 -0
- mcaligner-0.2.0a1/mcaligner/mcp/__init__.py +5 -0
- mcaligner-0.2.0a1/mcaligner/mcp/cli.py +67 -0
- mcaligner-0.2.0a1/mcaligner/mcp/server.py +123 -0
- mcaligner-0.2.0a1/mcaligner/mcp/tools.py +307 -0
- mcaligner-0.2.0a1/mcaligner/pipeline.py +286 -0
- mcaligner-0.2.0a1/mcaligner/py.typed +0 -0
- mcaligner-0.2.0a1/mcaligner/rules.py +323 -0
- mcaligner-0.2.0a1/mcaligner/settings.py +206 -0
- mcaligner-0.2.0a1/mkdocs.sh +12 -0
- mcaligner-0.2.0a1/pyproject.toml +63 -0
- mcaligner-0.2.0a1/tests/conftest.py +74 -0
- mcaligner-0.2.0a1/tests/data/ar.conllu +67 -0
- mcaligner-0.2.0a1/tests/data/grc.conllu +63 -0
- mcaligner-0.2.0a1/tests/data/silver.jsonl +3 -0
- mcaligner-0.2.0a1/tests/scripted.py +93 -0
- mcaligner-0.2.0a1/tests/test_agents.py +279 -0
- mcaligner-0.2.0a1/tests/test_cli.py +212 -0
- mcaligner-0.2.0a1/tests/test_evaluate.py +57 -0
- mcaligner-0.2.0a1/tests/test_lexicon.py +42 -0
- mcaligner-0.2.0a1/tests/test_mcp.py +160 -0
- mcaligner-0.2.0a1/tests/test_pipeline.py +111 -0
- mcaligner-0.2.0a1/tests/test_rules.py +121 -0
- mcaligner-0.2.0a1/tests/test_settings.py +82 -0
- mcaligner-0.2.0a1/uv.lock +2399 -0
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
name: docs
|
|
2
|
+
|
|
3
|
+
# Builds the Sphinx docs and the coverage report on every push (so a broken
|
|
4
|
+
# docs build shows up on any branch), and deploys them to GitHub Pages from main.
|
|
5
|
+
|
|
6
|
+
on:
|
|
7
|
+
push:
|
|
8
|
+
workflow_dispatch:
|
|
9
|
+
|
|
10
|
+
permissions:
|
|
11
|
+
contents: read
|
|
12
|
+
pages: write
|
|
13
|
+
id-token: write
|
|
14
|
+
|
|
15
|
+
concurrency:
|
|
16
|
+
group: pages-${{ github.ref }}
|
|
17
|
+
cancel-in-progress: false
|
|
18
|
+
|
|
19
|
+
jobs:
|
|
20
|
+
build:
|
|
21
|
+
runs-on: ubuntu-latest
|
|
22
|
+
steps:
|
|
23
|
+
- uses: actions/checkout@v7
|
|
24
|
+
|
|
25
|
+
- name: Install uv and Python
|
|
26
|
+
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
|
|
27
|
+
with:
|
|
28
|
+
enable-cache: true
|
|
29
|
+
python-version: "3.13"
|
|
30
|
+
|
|
31
|
+
- name: Install dependencies
|
|
32
|
+
run: uv sync --locked --all-groups
|
|
33
|
+
|
|
34
|
+
- name: Build Sphinx docs
|
|
35
|
+
run: uv run --locked sphinx-build -W -b html -d docs/_build/doctrees docs gh-pages
|
|
36
|
+
|
|
37
|
+
- name: Run tests and generate coverage report
|
|
38
|
+
run: |
|
|
39
|
+
uv run --locked pytest --cov-report=html:gh-pages/coverage
|
|
40
|
+
rm -f gh-pages/coverage/.gitignore
|
|
41
|
+
|
|
42
|
+
- name: Setup Pages
|
|
43
|
+
if: github.ref == 'refs/heads/main'
|
|
44
|
+
uses: actions/configure-pages@v6
|
|
45
|
+
|
|
46
|
+
- name: Upload artifact
|
|
47
|
+
if: github.ref == 'refs/heads/main'
|
|
48
|
+
uses: actions/upload-pages-artifact@v5
|
|
49
|
+
with:
|
|
50
|
+
path: gh-pages
|
|
51
|
+
|
|
52
|
+
deploy:
|
|
53
|
+
if: github.ref == 'refs/heads/main'
|
|
54
|
+
needs: build
|
|
55
|
+
runs-on: ubuntu-latest
|
|
56
|
+
environment:
|
|
57
|
+
name: github-pages
|
|
58
|
+
url: ${{ steps.deployment.outputs.page_url }}
|
|
59
|
+
steps:
|
|
60
|
+
- name: Deploy to GitHub Pages
|
|
61
|
+
id: deployment
|
|
62
|
+
uses: actions/deploy-pages@v5
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
name: publish
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- "v*.*.*"
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
publish:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
steps:
|
|
12
|
+
- uses: actions/checkout@v7
|
|
13
|
+
|
|
14
|
+
- name: Install uv and Python
|
|
15
|
+
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
|
|
16
|
+
with:
|
|
17
|
+
enable-cache: true
|
|
18
|
+
python-version: "3.13"
|
|
19
|
+
|
|
20
|
+
- name: Run tests
|
|
21
|
+
run: |
|
|
22
|
+
uv sync --locked --group dev
|
|
23
|
+
uv run --locked pytest
|
|
24
|
+
|
|
25
|
+
- name: Build library
|
|
26
|
+
run: uv build
|
|
27
|
+
|
|
28
|
+
- name: Publish library
|
|
29
|
+
env:
|
|
30
|
+
UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }}
|
|
31
|
+
run: uv publish
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
name: testing
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
pull_request:
|
|
6
|
+
workflow_dispatch:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
strategy:
|
|
12
|
+
fail-fast: false
|
|
13
|
+
matrix:
|
|
14
|
+
python-version: ["3.11", "3.12", "3.13", "3.14"]
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v7
|
|
17
|
+
|
|
18
|
+
- name: Install uv and Python ${{ matrix.python-version }}
|
|
19
|
+
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
|
|
20
|
+
with:
|
|
21
|
+
enable-cache: true
|
|
22
|
+
python-version: ${{ matrix.python-version }}
|
|
23
|
+
|
|
24
|
+
- name: Install dependencies
|
|
25
|
+
run: uv sync --locked --group dev
|
|
26
|
+
|
|
27
|
+
# Coverage is reported but does not fail the job; only failing tests do.
|
|
28
|
+
# The tests never call a real model, so no API key is needed.
|
|
29
|
+
- name: Run tests
|
|
30
|
+
run: uv run --locked pytest
|
|
31
|
+
|
|
32
|
+
package:
|
|
33
|
+
runs-on: ubuntu-latest
|
|
34
|
+
steps:
|
|
35
|
+
- uses: actions/checkout@v7
|
|
36
|
+
|
|
37
|
+
- name: Install uv
|
|
38
|
+
uses: astral-sh/setup-uv@bec219d24cd3e171d82865faccec33120bb574f4 # v10.1.0
|
|
39
|
+
with:
|
|
40
|
+
python-version: "3.13"
|
|
41
|
+
|
|
42
|
+
- name: Build and check the package
|
|
43
|
+
run: |
|
|
44
|
+
uv build
|
|
45
|
+
uvx twine check dist/*
|
|
46
|
+
|
|
47
|
+
- name: Install the wheel and run the CLI
|
|
48
|
+
run: |
|
|
49
|
+
uv venv --python 3.13 /tmp/wheel-test
|
|
50
|
+
uv pip install --python /tmp/wheel-test/bin/python dist/*.whl
|
|
51
|
+
/tmp/wheel-test/bin/mcaligner --version
|
|
52
|
+
/tmp/wheel-test/bin/mcaligner --help
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.13
|
|
@@ -0,0 +1,383 @@
|
|
|
1
|
+
# mcaligner — implementation notes
|
|
2
|
+
|
|
3
|
+
Notes for coding agents (Codex, Claude Code) and humans picking up this repo.
|
|
4
|
+
Keep this file current when you change the design or finish a milestone.
|
|
5
|
+
Most of `../llmparser/AGENTS.md` (conventions, environment gotchas) applies here too.
|
|
6
|
+
|
|
7
|
+
## Goal
|
|
8
|
+
|
|
9
|
+
Align the **words** of two parsed versions of the same text (e.g. an Arabic
|
|
10
|
+
translation of Ephesians and its Greek Vorlage). Sentences (verses) are paired
|
|
11
|
+
by `sent_id`; links are Ugarit style (`3-5S` sure, `12-18P` probable); the
|
|
12
|
+
links feed a bilingual lexicon. All corpus and alignment data lives in
|
|
13
|
+
MCParser's `CorpusStore` and is reached by the agents through MCParser's MCP
|
|
14
|
+
server. The handoff brief is `INSTRUCTIONS.md`.
|
|
15
|
+
|
|
16
|
+
## Status (2026-10-14)
|
|
17
|
+
|
|
18
|
+
| Milestone | State |
|
|
19
|
+
|---|---|
|
|
20
|
+
| 1. Package set-up, fixture tests (`convert` has since been removed) | done, 100% covered |
|
|
21
|
+
| 2. `evaluate` (AER, P/R/F1) and `lexicon` | done, 100% covered |
|
|
22
|
+
| 3. Rules, precision measured on the 155 verses | done, 100% covered (see Rules below) |
|
|
23
|
+
| 4. Aligner agent, scorer, pipeline; small real run | done, 100% covered; 2 real 5-verse runs ($0.21, $0.18) |
|
|
24
|
+
| 5. Sphinx docs, AGENTS.md | done; `./mkdocs.sh` (`sphinx-build -E -W`) clean |
|
|
25
|
+
|
|
26
|
+
The old implementation is in `llmalignerold/` (LangChain + retriv/chroma). It is
|
|
27
|
+
not imported.
|
|
28
|
+
|
|
29
|
+
## Layout
|
|
30
|
+
|
|
31
|
+
```
|
|
32
|
+
mcaligner/
|
|
33
|
+
evaluate.py # P/R/F1 over sure and over all links, AER; per pair + micro total
|
|
34
|
+
lexicon.py # TranslationTable (lemma -> linked lemmas, one pass, leave-one-pair-out) + lexicon export
|
|
35
|
+
guidelines/ # bundled annotation guidelines (grc-lat.md, bicross.md, grc-ar.md: a draft for Greek-Arabic, unvalidated; grc-cop.md: a draft for Greek-Coptic, unvalidated), added to the agents' instructions (setting `guidelines`, default grc-lat)
|
|
36
|
+
rules.py # code-only links: identical symbols, transliterated names, dominant lexicon match; measure()
|
|
37
|
+
settings.py # Settings (YAML): tiers, per-role model/thinking/instructions/enabled, batching, rules
|
|
38
|
+
agents/
|
|
39
|
+
prompts.py # instructions (aligner, scorer); compact pair blocks with settled/review links + evidence
|
|
40
|
+
outputs.py # Alignment/Scores answers (lines), parsing, Applier (dry-run set_links, then apply)
|
|
41
|
+
toolset.py # ReadToolset: 5 read MCP tools, JSON-string list args repaired, compact results
|
|
42
|
+
team.py # AlignerTeam: one pydantic-ai Agent per role over MCPToolset(build_server(store)); usage/cost
|
|
43
|
+
mcp/ # tools.py (similar_pairs, lemma_links, link_examples, pair_words, category_links, alignment_stats), server.py (MCParser's server + these), cli.py (mcp serve/config)
|
|
44
|
+
pipeline.py # rules -> align -> score -> rules rounds, batching, concurrency, RunReport
|
|
45
|
+
cli.py # typer app: evaluate, lexicon, rules, run, settings, mcp (MCParser's mcp_app)
|
|
46
|
+
docs/ # Sphinx (myst-parser + furo); CLI page rendered from --help in conf.py
|
|
47
|
+
tests/ # pytest; tests/data/{ar,grc}.conllu + silver.jsonl = verses B10K1V1-3 of results/ArabicAligned.yaml;
|
|
48
|
+
# tests/scripted.py = scripted FunctionModels (align(), score()) for agent tests
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
## Data
|
|
52
|
+
|
|
53
|
+
- `results/ArabicAligned.yaml` (untracked): 155 verses of Ephesians, Arabic
|
|
54
|
+
(`ArabGr1-Arabic-Ephesians-Parsed`, with multiword tokens, MISC
|
|
55
|
+
`Gloss=`/`Translit=`/`LTranslit=`/`Root=`) and Greek
|
|
56
|
+
(`ArabGr1-GreekVorlage-Ephesians-Parsed`, PROIEL-style, unaccented forms,
|
|
57
|
+
accented lemmas). `alignment` = old LLM output: **silver**, not gold.
|
|
58
|
+
- It was split once (by a `convert` command that has been removed, along with
|
|
59
|
+
its code) into `data/<corpus>.conllu` ×2 and `data/ArabicAligned.jsonl`
|
|
60
|
+
(`/data/` is ignored; keep them, they cannot be regenerated from the repo).
|
|
61
|
+
Silver totals: 2095 links (1967 S, 128 P); in the split 2094 links in 155 pairs.
|
|
62
|
+
- Data quirks found then: 4 Arabic multiword lines had 9 columns (padded with
|
|
63
|
+
`_`); one silver link pointed to a missing word (`B10K2V16 16-16P`, dropped);
|
|
64
|
+
duplicate links were merged (first kept).
|
|
65
|
+
- The Greek side has **no punctuation tokens** (Arabic has 116 PUNCT), so a
|
|
66
|
+
punctuation rule cannot fire on this text.
|
|
67
|
+
|
|
68
|
+
## Design decisions (and why)
|
|
69
|
+
|
|
70
|
+
- **Package**: uv + hatchling; MCParser (package and CLI `mcparser`, formerly
|
|
71
|
+
llmparser) from PyPI (`mcparser>=0.2.0a1`, a pre-release, hence
|
|
72
|
+
`[tool.uv] prerelease = "allow"`). To develop against the local repository
|
|
73
|
+
`../llmparser` add `[tool.uv.sources] mcparser = { path = "../llmparser", editable = true }`.
|
|
74
|
+
- **`mcp serve` / `mcp config`** are `mcaligner/mcp/cli.py`: MCParser's options
|
|
75
|
+
(reusing its typer option types) over `mcaligner.mcp.build_server`, which
|
|
76
|
+
is MCParser's server plus the exploration tools; the Claude Desktop config
|
|
77
|
+
runs `python -m mcaligner mcp serve`.
|
|
78
|
+
- **Alignment files** are MCParser's JSON Lines sets. Imported links carry no
|
|
79
|
+
confidence, so a read-only set gives them its default (1.0, or e.g. `:ro:0.8`
|
|
80
|
+
in the alignment spec). `mcparser align import/export` (MCParser, after 0.2.0a1)
|
|
81
|
+
converts to and from Pharaoh/Ugarit text and YAML.
|
|
82
|
+
|
|
83
|
+
- **Evaluation** (`evaluate.py`): A = predicted, S = gold sure, P = gold all.
|
|
84
|
+
"sure" P/R/F1 compares predicted *sure* links with S; "all" compares A with P
|
|
85
|
+
ignoring S/P; `AER = 1 − (|A∩S| + |A∩P|) / (|A| + |S|)`. Only gold pairs are
|
|
86
|
+
scored (gold often covers part of a text); a gold pair missing from the
|
|
87
|
+
predictions counts as empty; `--annotated-only` scores only pairs with
|
|
88
|
+
predicted links; `--min-confidence` drops weak predicted links (links without
|
|
89
|
+
a confidence are kept). Totals are micro-averaged. Empty denominators give 0.
|
|
90
|
+
It reads the JSONL files directly: no sources needed.
|
|
91
|
+
- **Translation table** (`lexicon.TranslationTable`): same counting as
|
|
92
|
+
MCParser's `translation_candidates` (lemma falls back to form;
|
|
93
|
+
`min_confidence` on links), keyed by **source name** (not side), built in one
|
|
94
|
+
pass, with per-pair counts so a pair can be left out (`exclude=(set, sent_id)`)
|
|
95
|
+
— needed when measuring rules on the same data the table comes from. Code uses
|
|
96
|
+
the table (calling `translation_candidates` per word is O(pairs) each); agents
|
|
97
|
+
can still call the MCP tool.
|
|
98
|
+
- **Lexicon export** (`mcaligner lexicon -s ... -a ... [-o x.yaml|x.json]`):
|
|
99
|
+
`{source: {lemma: {part_of_speech, occurrences, unaligned, alignments:
|
|
100
|
+
{lemma: {count, part_of_speech}}}}}`; default `--min-confidence 0.8`. On the
|
|
101
|
+
silver set it reproduces the old `results/ArabicLexicon.yaml` exactly (1138
|
|
102
|
+
lemmas) except for the dropped `B10K2V16 16-16P` link (old code counted a link
|
|
103
|
+
to a missing word as "aligned").
|
|
104
|
+
|
|
105
|
+
## Greek-Latin gold standard
|
|
106
|
+
|
|
107
|
+
`/Users/rturnbull/MDAP/BICROSS/Alignment/Alignment-Gold-Standards/grc-lat`: 100 DFHG
|
|
108
|
+
fragments (raw text only, 1983 grc / 1860 lat tokens, 1593 links, 0-based Ugarit).
|
|
109
|
+
Workflow: `mcparser init grc.txt -o grc.conllu` (and lat), parse with the PROIEL
|
|
110
|
+
treebanks (`UD/UD_Ancient_Greek-PROIEL`, `UD/UD_Latin-PROIEL`), then
|
|
111
|
+
`uv run python scripts/grc_lat_gold.py GOLD.json grc.conllu lat.conllu -o gold.jsonl` (a disposable script, not part of the package) maps each
|
|
112
|
+
gold token to the parsed words covering the same characters (a link goes to every
|
|
113
|
+
overlapping word), so it survives tokenisation differences. `init` only detaches `«` and the
|
|
114
|
+
full stop of `rep.` (3 fragments: grc 64, lat 26, lat 64; 1593 -> 1597 links). Work files are in `data/grc-lat/`.
|
|
115
|
+
Do not let a parser change the segmentation after the gold is mapped.
|
|
116
|
+
|
|
117
|
+
### Greek-Latin DFHG dev set (20 fragments; `data/grc-lat/`)
|
|
118
|
+
|
|
119
|
+
Dev ids in `dev-ids.txt` (stratified by length, seed 42; `scripts/grc_lat_split.py`); gold-dev 313 links.
|
|
120
|
+
Parses: grc $17.74, lat $16.83 (cheap-first, syntax on Opus, 3 rounds; the PROIEL treebanks as examples).
|
|
121
|
+
Cold start (no evidence sets), 2 rounds, Opus aligner + Sonnet scorer, evaluated against gold-dev:
|
|
122
|
+
|
|
123
|
+
| Run | Cost | Requests | All P/R/F1 | AER |
|
|
124
|
+
|---|---|---|---|---|
|
|
125
|
+
| 1 (old prompt) | $0.84 | 34 | 0.889/0.869/0.879 | 0.088 |
|
|
126
|
+
| 2 (old prompt; run-to-run noise) | $0.58 | 36 | 0.886/0.866/0.876 | 0.089 |
|
|
127
|
+
| 3 (ids fixed, see below) | $0.60 | 23 | 0.895/0.869/0.882 | 0.083 |
|
|
128
|
+
| 3 without its 15 punctuation links (post-hoc) | – | – | 0.941/0.869/0.904 | 0.058 |
|
|
129
|
+
| 4 (+ guidelines in the prompts, punctuation dropped in code) | $0.77 | 22 | 0.955/0.885/0.919 | 0.044 |
|
|
130
|
+
|
|
131
|
+
- The prompt's answer example used an Ephesians id (`B10K1V1`), so models wrote `B98` for pair `98` and were
|
|
132
|
+
retried (15 of 36 requests). The examples are now `<id>` and the task header lists the ids: 1 retry in run 3.
|
|
133
|
+
- `TaskResult.details` (requests, tool calls by name, retry messages) is in the report. Almost no tool calls were
|
|
134
|
+
made (1 `pair_words` in run 3, none in the others): the prompt evidence is enough.
|
|
135
|
+
- Error analysis (pairs 80, 43, 60, 96, 84): (1) the aligner links punctuation (`,`~`,`, `«`~`«`), which the guidelines
|
|
136
|
+
forbid; (2) links the guidelines rule out (`τούτου`~`Sabinorum`, `τῇ`~`libro`, an article to `filius`); (3) misses
|
|
137
|
+
particle groups (`ἔτι δὲ καὶ`~`nec non`) and crossed pairs (`τότε πρῶτον`~`tunc demum`); (4) the names rule's wrong
|
|
138
|
+
`Ἡφαίστου`~`festi` (0.95) where `Vulcani` is right. Done in run 4: `mcaligner/guidelines/grc-lat.md` (a condensed version, from `../guidelines/grc-lat.md`) is in the
|
|
139
|
+
aligner's and scorer's instructions by default (`guidelines`), and links to punctuation are dropped in code
|
|
140
|
+
(`rules.link_punctuation`); the model made 0 such links with the guidelines. Human-human on the DFHG gold:
|
|
141
|
+
Dice 0.927 (from the README's flattened counts 1487/1486, 1378 agreed); BICROSS NT annotators: 0.948.
|
|
142
|
+
The dev set was used to choose these changes, so confirm on the 80 test fragments before quoting a number.
|
|
143
|
+
|
|
144
|
+
**Column ablation** (same 20 dev pairs; guidelines, punctuation filter, rules, models and `--tools none` fixed; only
|
|
145
|
+
`--columns` varies; one run each; `data/grc-lat/ablation/`; F1 over all links, 95% bootstrap over the 20 pairs):
|
|
146
|
+
|
|
147
|
+
| Columns shown | Cost | F1 | AER |
|
|
148
|
+
|---|---|---|---|
|
|
149
|
+
| forms only (no parse) | $0.40 | 0.912 [0.881, 0.944] | 0.053 |
|
|
150
|
+
| lemma upos | $0.48 | 0.916 | 0.047 |
|
|
151
|
+
| lemma upos head deprel (default) | $0.54 | 0.907 | 0.052 |
|
|
152
|
+
| + feats | $0.73 | 0.913 | 0.045 |
|
|
153
|
+
| + feats xpos | $0.71 | 0.916 | 0.047 |
|
|
154
|
+
| default columns with all tools (run 4) | $0.77 | 0.919 | 0.044 |
|
|
155
|
+
|
|
156
|
+
No difference is detectable: every paired difference is within about ±0.02 F1 (e.g. forms vs default +0.005 [-0.016, +0.025]),
|
|
157
|
+
which is the resolution of 20 pairs / 313 links. So on DFHG Greek-Latin the agents do not need the parse in the
|
|
158
|
+
prompt, and FEATS only adds cost (+35%). Caveats: the rules (names rule: 43 links) still use the parse; cold start,
|
|
159
|
+
so the lemma-based prompt evidence was empty; Opus knows classical Greek and Latin well. The same test on the BICROSS
|
|
160
|
+
manuscript text (variant spellings, nomina sacra) and with accumulated evidence is still to do.
|
|
161
|
+
|
|
162
|
+
**Column ablation on BICROSS** (20 verses chosen like the DFHG dev set, `scripts/grc_lat_split.py`; ids in
|
|
163
|
+
`data/bicross/dev-ids.txt`, the other 79 verses are the untouched test set; consensus-dev = 322 links; same settings):
|
|
164
|
+
|
|
165
|
+
| Columns shown | Cost | F1 vs consensus | AER |
|
|
166
|
+
|---|---|---|---|
|
|
167
|
+
| forms only | $0.29 | 0.905 | 0.084 |
|
|
168
|
+
| lemma upos | $0.31 | 0.903 | 0.081 |
|
|
169
|
+
| lemma upos head deprel | $0.37 | 0.904 | 0.079 |
|
|
170
|
+
| + feats | $0.55 | 0.902 | 0.084 |
|
|
171
|
+
| + feats xpos | $0.59 | 0.903 | 0.084 |
|
|
172
|
+
|
|
173
|
+
Again no difference (paired differences within ±0.01). The two annotators agree at F1 0.955 on these 20 verses. The
|
|
174
|
+
whole gap is one word class: the model misses 46 of the 53 consensus links that start at a Greek article
|
|
175
|
+
(`το~aqua`, `το~spiritus`) and almost nothing else (precision 0.98, recall 0.84). The BICROSS annotators link the
|
|
176
|
+
article to its noun's translation (88-95% of articles, mostly to NOUN), whereas DFHG's gold leaves ~90% of articles
|
|
177
|
+
unlinked, which is what the bundled `grc-lat` guidelines say. Guidelines are therefore per corpus: BICROSS needs its
|
|
178
|
+
own article rule (to be tested; the dev set was used to find this).
|
|
179
|
+
|
|
180
|
+
**BICROSS guidelines** (`--guidelines bicross`, `mcaligner/guidelines/bicross.md` from `Adam/.../BICROSS_guidelines_grc-lat.odt`,
|
|
181
|
+
13 Aug 2024; the v2 .odt of 28 May says the opposite about articles and predates both workbooks). Same 20 dev verses, no tools:
|
|
182
|
+
|
|
183
|
+
| Run | Cost | F1 vs consensus | vs Kreinecker | vs Tilke | AER (consensus) |
|
|
184
|
+
|---|---|---|---|---|---|
|
|
185
|
+
| `grc-lat` guidelines, forms only | $0.29 | 0.905 | 0.893 | 0.895 | 0.084 |
|
|
186
|
+
| `bicross` guidelines, forms only | $0.40 | 0.978 | 0.957 | 0.967 | 0.016 |
|
|
187
|
+
| `bicross` guidelines, lemma upos head deprel | $0.54 | 0.977 | 0.955 | 0.965 | 0.018 |
|
|
188
|
+
|
|
189
|
+
The annotators agree with each other at F1 0.955 on these verses, so the aligner is at the level of the human agreement
|
|
190
|
+
(against each annotator separately). The consensus (links of either annotator) is a lenient reference; compare with the
|
|
191
|
+
per-annotator figures. The guidelines were chosen after seeing the article gap on this dev set: confirm on the 79 test verses.
|
|
192
|
+
|
|
193
|
+
**BICROSS held-out test** (79 verses never used for tuning; `data/bicross/align-test.sh`; defaults except `--guidelines bicross`:
|
|
194
|
+
columns lemma upos head deprel, all tools allowed, rules on, 2 rounds, Opus aligner + Sonnet scorer, cold start). Cost $2.64
|
|
195
|
+
(Opus $1.84, Sonnet $0.80), 86 tasks / 86 requests, 0 retries, 0 tool calls, 0 failed, 0 punctuation links dropped. 1412 links.
|
|
196
|
+
F1 over all links (AER): vs Kreinecker 0.947 (0.053), vs Tilke 0.975 (0.025), vs consensus 0.969 (0.013). The two annotators
|
|
197
|
+
agree with each other at F1 0.946 (AER 0.054) on the same 79 verses, so the pipeline is at the level of human agreement. It
|
|
198
|
+
agrees more with Tilke (who links more, 1404 vs 1371 links) than with Kreinecker. Dev (tuned) was 0.957 / 0.967 / 0.978.
|
|
199
|
+
|
|
200
|
+
**DFHG held-out test** (80 fragments never used for tuning; `data/grc-lat/align-test.sh`; defaults, `grc-lat` guidelines, cold start,
|
|
201
|
+
2 rounds): $2.34 (Opus $1.69, Sonnet $0.66), 88 tasks / 88 requests, 0 retries, 0 tool calls, 0 failed, 2 punctuation links
|
|
202
|
+
dropped. 1215 links vs 1284 gold: sure P/R/F1 0.967/0.877/0.920; all links 0.948/0.897/0.922, AER 0.042 (dev: 0.919, 0.044). The
|
|
203
|
+
DFHG annotators agree at Dice about 0.927 (README counts, all 100 fragments), so the pipeline is at that level.
|
|
204
|
+
Remaining errors (132 missed, 63 extra): missed = verbs and participles linked to relatives or subordinators (`παροικοῦντες~qui`,
|
|
205
|
+
`ἠνάγκαζε~ut`), prepositions the Latin renders by case (`ἐν~Babyloniae`), adverbs; extra = `ἐν~libro` (the v2 guideline's "libro"
|
|
206
|
+
exception and the `μέν`/`δέ` rule are not in the short `grc-lat.md`). The test errors have now been seen: tune guidelines on dev only.
|
|
207
|
+
|
|
208
|
+
**Column ablation on the held-out test sets** (`scripts/ablation_test.sh`; corpus guidelines, `--tools none`, defaults otherwise; one run each;
|
|
209
|
+
F1 over all links with 95% bootstrap over the test pairs):
|
|
210
|
+
|
|
211
|
+
| Test set | words only | + lemma, UPOS | default columns + all tools (earlier run) | lemma-upos minus words only |
|
|
212
|
+
|---|---|---|---|---|
|
|
213
|
+
| DFHG (80) vs gold | 0.927 [0.911, 0.941], $1.55 | 0.922, $1.88 | 0.922, $2.34 | -0.005 [-0.011, +0.002] |
|
|
214
|
+
| BICROSS (79) vs Kreinecker | 0.948, $1.76 | 0.946, $1.95 | 0.946, $2.64 | -0.002 [-0.006, +0.003] |
|
|
215
|
+
| BICROSS vs Tilke | 0.974 | 0.973 | 0.975 | -0.002 [-0.005, +0.002] |
|
|
216
|
+
|
|
217
|
+
Words only is as good as with the parse in all four comparisons (never worse than -0.005) and 10-17% cheaper than lemma+UPOS,
|
|
218
|
+
about 35% cheaper than the default configuration. The parse in the prompt does not help alignment on these two corpora (cold start,
|
|
219
|
+
rules unchanged). Not tested: the evidence path (lemma-based hints with accumulated links) and lower-resource languages.
|
|
220
|
+
- Rules on the dev set: names 43 links (41 in gold); `identical` linked punctuation only (0 right): now off
|
|
221
|
+
(`link_punctuation`); lexicon with the full gold as leave-one-out evidence adds 3 (all right).
|
|
222
|
+
- BICROSS (`data/bicross/`): Greek and Latin manuscript text parsed the same way (resumable `parse-*.sh`); two
|
|
223
|
+
annotators' alignments, IRR AER 0.052 (`scripts/bicross_prepare.py`, `irr.json`).
|
|
224
|
+
|
|
225
|
+
## Rules (no model)
|
|
226
|
+
|
|
227
|
+
`rules.py`; `mcaligner rules -s ... -a ... --target pred` applies them,
|
|
228
|
+
`--measure SET [--show-errors]` measures them. Rules only link words that have
|
|
229
|
+
no link yet, only 1-1, only when the match is unique in the pair, and write
|
|
230
|
+
sure links with confidence 0.95 (identical: 0.98), agent `rules`.
|
|
231
|
+
|
|
232
|
+
- **identical**: same form on both sides, unique on each side, and a symbol
|
|
233
|
+
(punctuation only with `link_punctuation`, off by default: the Greek-Latin DFHG
|
|
234
|
+
gold never aligns it, and the first measure linked 7 punctuation marks, 0 right)
|
|
235
|
+
(UPOS PUNCT/NUM/SYM or no letters).
|
|
236
|
+
- **names**: one side PROPN, neither side a function word; consonant
|
|
237
|
+
skeletons (romanise: NFD, strip marks, Greek → Latin; then drop vowels and
|
|
238
|
+
y/w/h/j, merge consonant classes b/p/f/v/ph, t/d/th, k/g/q/c/kh, s/z; collapse
|
|
239
|
+
doubles) of MISC `LTranslit`/`Translit`, lemma and form; skeletons shorter than
|
|
240
|
+
3 must be equal, otherwise `SequenceMatcher` ≥ `name_similarity` (0.9); the
|
|
241
|
+
best match must have no equal-or-better rival sharing one end.
|
|
242
|
+
- **lexicon**: from a `TranslationTable` over the evidence sets (default all),
|
|
243
|
+
counting links ≥ `evidence_confidence` (0.96, above the rules' 0.95, so rules
|
|
244
|
+
never feed themselves — the MCParser trick). Fires when the lemma's top
|
|
245
|
+
translation has ≥ `min_count` (5) links and ≥ `min_share` (0.95) of **all**
|
|
246
|
+
the lemma's occurrences, unlinked ones included (so mostly-unlinked words like
|
|
247
|
+
the Greek article never fire), the lemma is unique in its sentence and the
|
|
248
|
+
translation occurs exactly once (unlinked) in the other. Both directions.
|
|
249
|
+
**The sentence under work is always left out of the evidence.**
|
|
250
|
+
|
|
251
|
+
**Measured** on the 155 Ephesians verses against the silver links, starting each
|
|
252
|
+
pair from no links, leave-one-out evidence (`--measure silver`):
|
|
253
|
+
|
|
254
|
+
| Rule | Links | In silver | Real precision (after reading the disagreements) |
|
|
255
|
+
|---|---|---|---|
|
|
256
|
+
| identical | 0 | – | – (the Greek has no punctuation) |
|
|
257
|
+
| names (threshold 0.9) | 20 | 18 | 20/20: both disagreements are silver errors (silver links Jesus↔Christ crosswise in B10K3V6, and Ἰησοῦ↔و in B10K3V21) |
|
|
258
|
+
| lexicon (5, 0.95) | 129 | 127 | 128/129: B10K4V29 لاكن→ἀλλ' is right (silver is wrong); B10K3V13 أنا "I" → μου "my" is wrong (pronoun lemmas cover every case) |
|
|
259
|
+
|
|
260
|
+
149 of the 2094 silver links (7%) are settled by rules at the start; the lexicon
|
|
261
|
+
rule grows as confident agent links accumulate. Settings tried: names at 0.8
|
|
262
|
+
added جسد "body" ~ χριστῷ (ǧasad vs khristo, 0.86) → threshold 0.9. Lexicon at
|
|
263
|
+
(3, 0.95): 198 links, same precision; at (3, 0.9): 229 links with 3 real errors
|
|
264
|
+
(ὑπό→مِن, مثل→ὡς, أنا→μου) ≈ 98.7%; at (5, 0.8): 265 links, 94.7% vs silver.
|
|
265
|
+
Kept MCParser's conservative (5, 0.95). **The silver set is noisy** (crossing
|
|
266
|
+
errors in "Jesus Christ"/"Christ Jesus"), so precision against silver is a
|
|
267
|
+
lower bound.
|
|
268
|
+
|
|
269
|
+
## Agents and pipeline
|
|
270
|
+
|
|
271
|
+
- **Two roles**: `aligner` (strong tier, Opus 5.5) and `scorer` (efficient
|
|
272
|
+
tier, Sonnet 5.5; `--no-scorer` or `roles: {scorer: {enabled: false}}`).
|
|
273
|
+
Per-role `model`, `thinking`, `instructions`; tiers and `model` as in MCParser.
|
|
274
|
+
- **Data access**: agents get only read MCP tools (`get_parallel`,
|
|
275
|
+
`translation_candidates`, `analyse_form`, `find_tokens`, `get_context`)
|
|
276
|
+
through `MCPToolset(build_server(store))`; answers are applied by code through
|
|
277
|
+
the store (`set_links(dry_run=True)` for every pair, retry on any error, then
|
|
278
|
+
`set_links`), as MCParser's Applier does. Results of the alignment tools
|
|
279
|
+
are compacted here; the monolingual ones use MCParser's `compact_result`.
|
|
280
|
+
- **Aligner answer**: `links` lines `SENT_ID A-B S|P CONFIDENCE` (S/P may be
|
|
281
|
+
attached: `3-5S`; missing = S) and `remove` lines `SENT_ID A-B`. New links of a
|
|
282
|
+
pair = settled links (≥ threshold, shown in the prompt and kept) − removals +
|
|
283
|
+
answer lines. Links to review that the answer omits are dropped. **Every pair
|
|
284
|
+
of the task is written** (even unchanged), so the history shows the aligner
|
|
285
|
+
saw it (`seen_by_aligner`).
|
|
286
|
+
- **Scorer answer**: `scores` lines `SENT_ID A-B CONFIDENCE`, only for the links
|
|
287
|
+
listed to score; applied with `replace=False` (S/P kept).
|
|
288
|
+
- **Provenance** comes from MCParser's `Link.agent` (added for this project):
|
|
289
|
+
rules write `rules`, the aligner `aligner`, the scorer `scorer`; an unchanged
|
|
290
|
+
link keeps its agent. The scorer gets links whose agent is `aligner`. If the
|
|
291
|
+
scorer gives the same confidence, the link stays `aligner`, so the pipeline
|
|
292
|
+
also remembers `(sent_id, key, S/P, confidence)` it has rated (in memory).
|
|
293
|
+
- **Prompt** per pair: `id form lemma UPOS head deprel Gloss` lines (MISC keys
|
|
294
|
+
in `misc_keys`), multiword tokens as `5-6 form (token of words 5-6)`, settled
|
|
295
|
+
links and links to review with confidence and agent, and for each unlinked
|
|
296
|
+
non-punctuation word the top `candidates` translations from a
|
|
297
|
+
`TranslationTable` (links ≥ `evidence_confidence` 0.8, this sentence left
|
|
298
|
+
out): `B10 Χριστός: مَسِيح×25, المسيح×11, يسوع×5 (of 43, 3 unlinked)`.
|
|
299
|
+
Two Ephesians verses ≈ 5k characters.
|
|
300
|
+
- **Rounds**: rules; then per round align (unseen pairs + pairs with links <
|
|
301
|
+
threshold), score, rules again (more evidence). Stop when no task, no pair's
|
|
302
|
+
links changed (Ugarit text, ignoring confidences), nothing below threshold,
|
|
303
|
+
or the round limit (default 2). Batches: ≤ `batch_sentences` (5) pairs and
|
|
304
|
+
≤ `batch_words` (150) words. A failing task is recorded and the run goes on;
|
|
305
|
+
`UserError` (missing API key) aborts. Each run appends its report to
|
|
306
|
+
`<set>.runs.jsonl`; `--report` writes it as JSON. Tokens and cost per model are
|
|
307
|
+
printed (`mcparser.pricing.estimate`).
|
|
308
|
+
|
|
309
|
+
## Real-model observations (Opus 5.5 aligner, Sonnet 5.5 scorer)
|
|
310
|
+
|
|
311
|
+
Ephesians 1:1–5 (B10K1V1–5), evidence = silver **without those 5 verses**
|
|
312
|
+
(`data/silver-heldout.jsonl`), so neither the rules, the prompt evidence nor the
|
|
313
|
+
tools can see the answer. 2 rounds, batch 5. Evaluated against silver
|
|
314
|
+
(`evaluate --annotated-only`):
|
|
315
|
+
|
|
316
|
+
| Run | Cost | Requests | All P/R/F1 | AER | ≥0.8 links: P/R | below 0.8 |
|
|
317
|
+
|---|---|---|---|---|---|---|
|
|
318
|
+
| 1 | $0.21 (Opus $0.16) | 6 (1 per task) | 0.825/0.930/0.874 | 0.130 | 0.952/0.831 (62 links) | 18 of 80 |
|
|
319
|
+
| 2 (morpheme guidance) | $0.18 (Opus $0.14) | 6 | 0.835/0.930/0.880 | 0.124 | 0.892/0.817 (65 links) | 14 of 79 |
|
|
320
|
+
|
|
321
|
+
- Rules settled 13 of ~80 links (names 5, lexicon 8). No tool calls were made:
|
|
322
|
+
the prompt evidence was enough, every task was a single request. ~1 minute.
|
|
323
|
+
- **Most "false positives" are right.** (1) Silver is wrong where Arabic has
|
|
324
|
+
"Jesus Messiah" for Greek "Christ Jesus" (it links them crosswise); the
|
|
325
|
+
aligner and the names rule get it right. (2) The Arabic parse does not split
|
|
326
|
+
pronoun suffixes (ربنا "our Lord", باركنا "blessed us", اختارنا "chose us",
|
|
327
|
+
هواه "his desire") or some prepositions (بكل "with all"); the aligner links
|
|
328
|
+
these to ἡμῶν/ἡμᾶς/αὐτοῦ/πάσῃ (one-to-many), which silver mostly lacks.
|
|
329
|
+
So precision against silver underestimates; a small gold set is needed.
|
|
330
|
+
- Run 1: the scorer rated those suffix links 0.3–0.4 ("doubtful or wrong"),
|
|
331
|
+
so round 2 re-sent them, the aligner kept them, nothing improved. Run 2 added
|
|
332
|
+
"a word may contain morphemes the parse did not split off" to both prompts:
|
|
333
|
+
fewer links below threshold, but round 2 still changed 1 of 4 pairs both
|
|
334
|
+
times for ~1/3 of the cost. **For large runs consider `--rounds 1`** (the
|
|
335
|
+
scorer still sets confidences, e.g. for `lexicon --min-confidence`).
|
|
336
|
+
- Cache writes exceed reads in a run of 6 requests (writes cost 1.25× input);
|
|
337
|
+
caching pays off on long runs where instructions and tool definitions repeat.
|
|
338
|
+
|
|
339
|
+
## MCParser additions (for this project)
|
|
340
|
+
|
|
341
|
+
- `Link.agent` (who last set or changed a link; unchanged links keep it;
|
|
342
|
+
`get_parallel` returns `agents`), and `set_links` now copies the links it is
|
|
343
|
+
given (it used to be able to mutate the caller's/stored `Link` objects).
|
|
344
|
+
- `dry_run` on the MCP `set_links` tool (the store method had it).
|
|
345
|
+
|
|
346
|
+
## Environment gotchas
|
|
347
|
+
|
|
348
|
+
- `uv sync --python /opt/homebrew/opt/python@3.13/bin/python3.13` (the
|
|
349
|
+
uv-managed 3.12 is x86_64 and breaks `cryptography`).
|
|
350
|
+
- **MCParser was called mcparser** (2026-10-15 rename): the package and CLI are
|
|
351
|
+
now `mcparser`, but the repository is still `../llmparser`. Imports are
|
|
352
|
+
`from mcparser...`; the Claude Desktop config runs `python -m mcparser`. Older
|
|
353
|
+
run reports and notes may still say mcparser.
|
|
354
|
+
- If `VIRTUAL_ENV` points at MCParser's venv (e.g. from the IDE), uv warns and
|
|
355
|
+
ignores it; `unset VIRTUAL_ENV` to silence it.
|
|
356
|
+
- The test fixture YAML is dumped with escaped tabs (`"1\tبولس..."`), so edit
|
|
357
|
+
it through `yaml.safe_load`, not with text replacement.
|
|
358
|
+
|
|
359
|
+
## Conventions
|
|
360
|
+
|
|
361
|
+
- Docs: `./mkdocs.sh` (= `uv run --group docs sphinx-build -E -W ...`). The
|
|
362
|
+
CLI page is generated from `--help` (typer vendors click, so no sphinx-click);
|
|
363
|
+
add new commands to `CLI_COMMANDS` in `docs/conf.py`.
|
|
364
|
+
|
|
365
|
+
- `uv run pytest` should stay at **100% line+branch coverage** (reported, not enforced: CI
|
|
366
|
+
only fails when a test fails) and fast. No network: `conftest.py` sets
|
|
367
|
+
`pydantic_ai.models.ALLOW_MODEL_REQUESTS = False`.
|
|
368
|
+
- Generic store/MCP functionality goes into MCParser (tested at 100%); alignment
|
|
369
|
+
exploration tools specific to this project live in `mcaligner/mcp/`.
|
|
370
|
+
- Raise `CorpusError` subclasses for expected failures; the CLI turns them
|
|
371
|
+
(and `OSError`, pydantic `ValidationError`, pydantic-ai `UserError`) into
|
|
372
|
+
`Error: ...` with exit status 1.
|
|
373
|
+
|
|
374
|
+
## Ideas / next steps
|
|
375
|
+
|
|
376
|
+
- A small hand-checked **gold** set (silver is noisy: Jesus/Christ crossings,
|
|
377
|
+
missing suffix links), then tune threshold, rounds and rule settings on it.
|
|
378
|
+
- `--rounds 1` as the default if round 2 keeps changing little on larger runs.
|
|
379
|
+
- Anthropic Message Batches (50% off): most tasks are a single request.
|
|
380
|
+
- Many-to-one rules (e.g. Arabic suffixed pronoun → Greek pronoun when the
|
|
381
|
+
host word is linked) once a gold set shows how often they hold.
|
|
382
|
+
- A per-lemma index in MCParser's `translation_candidates` if agents call it
|
|
383
|
+
a lot on large sets (it scans all pairs per call).
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to MCAligner. The format follows [Keep a Changelog](https://keepachangelog.com/).
|
|
4
|
+
|
|
5
|
+
## [0.2.0a1]
|
|
6
|
+
|
|
7
|
+
First alpha release on PyPI. Install with `pip install --pre mcaligner`.
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
- `mcaligner run`: rules, aligner agent, scorer agent and refinement rounds over two parsed CoNLL-U files, with
|
|
11
|
+
batching, concurrency, a cost report and a resumable alignment set (JSON Lines, MCParser's format).
|
|
12
|
+
- Rules without a model (identical symbols, transliterated names, dominant lexicon match).
|
|
13
|
+
- `mcaligner evaluate` (P/R/F1 over sure and all links, AER), `lexicon` (bilingual lexicon export), `rules`
|
|
14
|
+
(apply or measure the rules), `settings`, and `mcp serve` / `mcp config` (MCParser's MCP server plus the
|
|
15
|
+
exploration tools `similar_pairs`, `lemma_links`, `link_examples`, `pair_words`, `category_links`,
|
|
16
|
+
`alignment_stats`).
|
|
17
|
+
- Bundled annotation guidelines (`--guidelines`): `grc-lat` (default), `bicross`, and drafts for `grc-cop`,
|
|
18
|
+
`grc-ar`, `lat-ar`, `grc-syr`, `syr-ar`, `got-lat`.
|
|
19
|
+
- `--columns`, `--tools` and `--link-punctuation` to control what the agents see and may do.
|
|
20
|
+
- `mcaligner --version`, documentation, and a project logo.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If you use MCAligner, please cite it as below."
|
|
3
|
+
title: "MCAligner: word alignment of parallel texts with LLM agents"
|
|
4
|
+
type: software
|
|
5
|
+
authors:
|
|
6
|
+
- family-names: Turnbull
|
|
7
|
+
given-names: Robert
|
|
8
|
+
affiliation: "Melbourne Data Analytics Platform, University of Melbourne"
|
|
9
|
+
website: "https://robturnbull.com"
|
|
10
|
+
version: 0.2.0a1
|
|
11
|
+
license: Apache-2.0
|
|
12
|
+
repository-code: "https://github.com/rbturnbull/mcaligner"
|
|
13
|
+
url: "https://rbturnbull.github.io/mcaligner/"
|
|
14
|
+
keywords:
|
|
15
|
+
- digital humanities
|
|
16
|
+
- word alignment
|
|
17
|
+
- parallel texts
|
|
18
|
+
- LLM agents
|
|
19
|
+
- MCP
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
@AGENTS.md
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
Thanks for helping with MCAligner. Bug reports, feature ideas and pull requests are welcome.
|
|
4
|
+
|
|
5
|
+
## Setting up
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
git clone https://github.com/rbturnbull/mcaligner && cd mcaligner
|
|
9
|
+
uv sync --all-groups
|
|
10
|
+
uv run pytest
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
Python 3.11 to 3.14 are supported. The repository pins 3.13 for development (`.python-version`).
|
|
14
|
+
|
|
15
|
+
## Before opening a pull request
|
|
16
|
+
|
|
17
|
+
- `uv run pytest` must pass. Aim for **100% line and branch coverage** (it is reported, but a shortfall does not fail CI). The tests never call a real model or
|
|
18
|
+
the network; agents are tested with the scripted models in `tests/scripted.py`.
|
|
19
|
+
- `./mkdocs.sh` (Sphinx with warnings as errors) must build the documentation. Update the docs and
|
|
20
|
+
`CHANGELOG.md` for user-visible changes.
|
|
21
|
+
- Keep `uv.lock` current when you change dependencies (`uv lock`); CI installs with `--locked`.
|
|
22
|
+
- Match the surrounding code: type hints, short docstrings, pydantic models for return values, and
|
|
23
|
+
`CorpusError` subclasses for expected failures.
|
|
24
|
+
|
|
25
|
+
`AGENTS.md` records the design decisions and the measurements behind them. Read it before changing
|
|
26
|
+
how the agents, rules or scorer behave, and add what you learned to it.
|
|
27
|
+
|
|
28
|
+
## Changes that affect accuracy or cost
|
|
29
|
+
|
|
30
|
+
Say what you measured. The agents' output varies between runs, so a difference of about 0.015 F1
|
|
31
|
+
on 80 sentence pairs is noise. Give the corpus, the number of pairs, the guidelines, the models,
|
|
32
|
+
the cost and ideally two runs. Tune on a dev subset and report on a held-out one.
|
|
33
|
+
|
|
34
|
+
## Reporting a bug
|
|
35
|
+
|
|
36
|
+
Include `mcaligner --version`, the command, the settings and the error. Do not paste API keys or text
|
|
37
|
+
you may not share.
|