hexapla 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hexapla/__init__.py +3 -0
- hexapla/about.py +63 -0
- hexapla/analysis.py +370 -0
- hexapla/bible.py +285 -0
- hexapla/build.py +509 -0
- hexapla/config.py +221 -0
- hexapla/conllu.py +166 -0
- hexapla/corpus.py +302 -0
- hexapla/languages.py +36 -0
- hexapla/licences.py +24 -0
- hexapla/links.py +105 -0
- hexapla/main.py +166 -0
- hexapla/static/img/hexapla-logo.svg +50 -0
- hexapla/static/img/hexapla-small.svg +50 -0
- hexapla/static/js/bootstrap.bundle.min.js +7 -0
- hexapla/static/js/common.js +122 -0
- hexapla/static/js/d3.v7.min.js +2 -0
- hexapla/static/js/navigator.js +419 -0
- hexapla/static/js/pages.js +311 -0
- hexapla/static/scss/_hexapla.scss +339 -0
- hexapla/static/scss/bootstrap/LICENSE +21 -0
- hexapla/static/scss/bootstrap/scss/_accordion.scss +153 -0
- hexapla/static/scss/bootstrap/scss/_alert.scss +68 -0
- hexapla/static/scss/bootstrap/scss/_badge.scss +38 -0
- hexapla/static/scss/bootstrap/scss/_breadcrumb.scss +40 -0
- hexapla/static/scss/bootstrap/scss/_button-group.scss +147 -0
- hexapla/static/scss/bootstrap/scss/_buttons.scss +216 -0
- hexapla/static/scss/bootstrap/scss/_card.scss +238 -0
- hexapla/static/scss/bootstrap/scss/_carousel.scss +226 -0
- hexapla/static/scss/bootstrap/scss/_close.scss +66 -0
- hexapla/static/scss/bootstrap/scss/_containers.scss +41 -0
- hexapla/static/scss/bootstrap/scss/_dropdown.scss +250 -0
- hexapla/static/scss/bootstrap/scss/_forms.scss +9 -0
- hexapla/static/scss/bootstrap/scss/_functions.scss +302 -0
- hexapla/static/scss/bootstrap/scss/_grid.scss +39 -0
- hexapla/static/scss/bootstrap/scss/_helpers.scss +12 -0
- hexapla/static/scss/bootstrap/scss/_images.scss +42 -0
- hexapla/static/scss/bootstrap/scss/_list-group.scss +199 -0
- hexapla/static/scss/bootstrap/scss/_maps.scss +174 -0
- hexapla/static/scss/bootstrap/scss/_mixins.scss +42 -0
- hexapla/static/scss/bootstrap/scss/_modal.scss +240 -0
- hexapla/static/scss/bootstrap/scss/_nav.scss +197 -0
- hexapla/static/scss/bootstrap/scss/_navbar.scss +289 -0
- hexapla/static/scss/bootstrap/scss/_offcanvas.scss +147 -0
- hexapla/static/scss/bootstrap/scss/_pagination.scss +109 -0
- hexapla/static/scss/bootstrap/scss/_placeholders.scss +51 -0
- hexapla/static/scss/bootstrap/scss/_popover.scss +196 -0
- hexapla/static/scss/bootstrap/scss/_progress.scss +68 -0
- hexapla/static/scss/bootstrap/scss/_reboot.scss +617 -0
- hexapla/static/scss/bootstrap/scss/_root.scss +187 -0
- hexapla/static/scss/bootstrap/scss/_spinners.scss +86 -0
- hexapla/static/scss/bootstrap/scss/_tables.scss +171 -0
- hexapla/static/scss/bootstrap/scss/_toasts.scss +73 -0
- hexapla/static/scss/bootstrap/scss/_tooltip.scss +119 -0
- hexapla/static/scss/bootstrap/scss/_transitions.scss +27 -0
- hexapla/static/scss/bootstrap/scss/_type.scss +106 -0
- hexapla/static/scss/bootstrap/scss/_utilities.scss +806 -0
- hexapla/static/scss/bootstrap/scss/_variables-dark.scss +102 -0
- hexapla/static/scss/bootstrap/scss/_variables.scss +1753 -0
- hexapla/static/scss/bootstrap/scss/bootstrap-grid.scss +62 -0
- hexapla/static/scss/bootstrap/scss/bootstrap-reboot.scss +10 -0
- hexapla/static/scss/bootstrap/scss/bootstrap-utilities.scss +19 -0
- hexapla/static/scss/bootstrap/scss/bootstrap.scss +52 -0
- hexapla/static/scss/bootstrap/scss/forms/_floating-labels.scss +97 -0
- hexapla/static/scss/bootstrap/scss/forms/_form-check.scss +189 -0
- hexapla/static/scss/bootstrap/scss/forms/_form-control.scss +214 -0
- hexapla/static/scss/bootstrap/scss/forms/_form-range.scss +91 -0
- hexapla/static/scss/bootstrap/scss/forms/_form-select.scss +80 -0
- hexapla/static/scss/bootstrap/scss/forms/_form-text.scss +11 -0
- hexapla/static/scss/bootstrap/scss/forms/_input-group.scss +132 -0
- hexapla/static/scss/bootstrap/scss/forms/_labels.scss +36 -0
- hexapla/static/scss/bootstrap/scss/forms/_validation.scss +12 -0
- hexapla/static/scss/bootstrap/scss/helpers/_clearfix.scss +3 -0
- hexapla/static/scss/bootstrap/scss/helpers/_color-bg.scss +7 -0
- hexapla/static/scss/bootstrap/scss/helpers/_colored-links.scss +30 -0
- hexapla/static/scss/bootstrap/scss/helpers/_focus-ring.scss +5 -0
- hexapla/static/scss/bootstrap/scss/helpers/_icon-link.scss +25 -0
- hexapla/static/scss/bootstrap/scss/helpers/_position.scss +36 -0
- hexapla/static/scss/bootstrap/scss/helpers/_ratio.scss +26 -0
- hexapla/static/scss/bootstrap/scss/helpers/_stacks.scss +15 -0
- hexapla/static/scss/bootstrap/scss/helpers/_stretched-link.scss +15 -0
- hexapla/static/scss/bootstrap/scss/helpers/_text-truncation.scss +7 -0
- hexapla/static/scss/bootstrap/scss/helpers/_visually-hidden.scss +8 -0
- hexapla/static/scss/bootstrap/scss/helpers/_vr.scss +8 -0
- hexapla/static/scss/bootstrap/scss/mixins/_alert.scss +18 -0
- hexapla/static/scss/bootstrap/scss/mixins/_backdrop.scss +14 -0
- hexapla/static/scss/bootstrap/scss/mixins/_banner.scss +7 -0
- hexapla/static/scss/bootstrap/scss/mixins/_border-radius.scss +78 -0
- hexapla/static/scss/bootstrap/scss/mixins/_box-shadow.scss +24 -0
- hexapla/static/scss/bootstrap/scss/mixins/_breakpoints.scss +127 -0
- hexapla/static/scss/bootstrap/scss/mixins/_buttons.scss +70 -0
- hexapla/static/scss/bootstrap/scss/mixins/_caret.scss +69 -0
- hexapla/static/scss/bootstrap/scss/mixins/_clearfix.scss +9 -0
- hexapla/static/scss/bootstrap/scss/mixins/_color-mode.scss +21 -0
- hexapla/static/scss/bootstrap/scss/mixins/_color-scheme.scss +7 -0
- hexapla/static/scss/bootstrap/scss/mixins/_container.scss +11 -0
- hexapla/static/scss/bootstrap/scss/mixins/_deprecate.scss +10 -0
- hexapla/static/scss/bootstrap/scss/mixins/_forms.scss +163 -0
- hexapla/static/scss/bootstrap/scss/mixins/_gradients.scss +47 -0
- hexapla/static/scss/bootstrap/scss/mixins/_grid.scss +151 -0
- hexapla/static/scss/bootstrap/scss/mixins/_image.scss +16 -0
- hexapla/static/scss/bootstrap/scss/mixins/_list-group.scss +26 -0
- hexapla/static/scss/bootstrap/scss/mixins/_lists.scss +7 -0
- hexapla/static/scss/bootstrap/scss/mixins/_pagination.scss +10 -0
- hexapla/static/scss/bootstrap/scss/mixins/_reset-text.scss +17 -0
- hexapla/static/scss/bootstrap/scss/mixins/_resize.scss +6 -0
- hexapla/static/scss/bootstrap/scss/mixins/_table-variants.scss +24 -0
- hexapla/static/scss/bootstrap/scss/mixins/_text-truncate.scss +8 -0
- hexapla/static/scss/bootstrap/scss/mixins/_transition.scss +26 -0
- hexapla/static/scss/bootstrap/scss/mixins/_utilities.scss +97 -0
- hexapla/static/scss/bootstrap/scss/mixins/_visually-hidden.scss +38 -0
- hexapla/static/scss/bootstrap/scss/utilities/_api.scss +47 -0
- hexapla/static/scss/bootstrap/scss/vendor/_rfs.scss +348 -0
- hexapla/templates/about.html +7 -0
- hexapla/templates/alignment.html +121 -0
- hexapla/templates/base.html +82 -0
- hexapla/templates/home.html +29 -0
- hexapla/templates/insights.html +61 -0
- hexapla/templates/lemma.html +67 -0
- hexapla/templates/lexicon.html +38 -0
- hexapla/templates/manuscript.html +86 -0
- hexapla/templates/review.html +70 -0
- hexapla/templates/verse.html +106 -0
- hexapla/templates/verse.js +4 -0
- hexapla/theme.py +37 -0
- hexapla/transcription.py +396 -0
- hexapla-0.1.0.dist-info/METADATA +290 -0
- hexapla-0.1.0.dist-info/RECORD +131 -0
- hexapla-0.1.0.dist-info/WHEEL +4 -0
- hexapla-0.1.0.dist-info/entry_points.txt +2 -0
- hexapla-0.1.0.dist-info/licenses/LICENSE +202 -0
hexapla/__init__.py
ADDED
hexapla/about.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The about page: the theme's Markdown file, rendered to HTML.
|
|
3
|
+
|
|
4
|
+
Images and links in it are written relative to the Markdown file, as they are when it is previewed on GitHub.
|
|
5
|
+
Images can be given a size or Bootstrap classes: ``{width=180 .float-end}``.
|
|
6
|
+
Those that point into the theme's ``assets`` folder (````) are rewritten to the copy
|
|
7
|
+
of the folder in the site; any other local link would be broken on the site, so validation reports it.
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from urllib.parse import urlsplit
|
|
13
|
+
|
|
14
|
+
from markdown_it import MarkdownIt
|
|
15
|
+
from mdit_py_plugins.attrs import attrs_plugin
|
|
16
|
+
|
|
17
|
+
ATTRIBUTES = {"image": "src", "link_open": "href"}
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _markdown() -> MarkdownIt:
|
|
21
|
+
return MarkdownIt().use(attrs_plugin)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _link_tokens(tokens):
|
|
25
|
+
for token in tokens:
|
|
26
|
+
for child in token.children or []:
|
|
27
|
+
if child.type in ATTRIBUTES:
|
|
28
|
+
yield child, ATTRIBUTES[child.type]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _is_local(url: str) -> bool:
|
|
32
|
+
parts = urlsplit(url)
|
|
33
|
+
return bool(parts.path) and not parts.scheme and not parts.netloc and not url.startswith("/")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def local_links(text: str) -> list[str]:
|
|
37
|
+
""" The images and links in a Markdown text that point to local files, in order, without repeats. """
|
|
38
|
+
links = [str(token.attrs[attr]) for token, attr in _link_tokens(_markdown().parse(text))]
|
|
39
|
+
return list(dict.fromkeys(link for link in links if _is_local(link)))
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def asset_path(link: str, about: Path, assets: Path | None) -> str | None:
|
|
43
|
+
""" Where a link from the about file lands in the assets folder (``img/logo.png``), or None if it is not there. """
|
|
44
|
+
if assets is None:
|
|
45
|
+
return None
|
|
46
|
+
target = (about.parent / urlsplit(link).path).resolve()
|
|
47
|
+
assets = assets.resolve()
|
|
48
|
+
if not target.is_file() or not target.is_relative_to(assets):
|
|
49
|
+
return None
|
|
50
|
+
return target.relative_to(assets).as_posix()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def render_about(about: Path, assets: Path | None, root: str) -> str:
|
|
54
|
+
""" The about page's HTML, with links into the assets folder pointing at its copy in the site. """
|
|
55
|
+
md = _markdown()
|
|
56
|
+
tokens = md.parse(about.read_text(encoding="utf-8"))
|
|
57
|
+
for token, attr in _link_tokens(tokens):
|
|
58
|
+
link = str(token.attrs[attr])
|
|
59
|
+
if _is_local(link) and (path := asset_path(link, about, assets)):
|
|
60
|
+
parts = urlsplit(link)
|
|
61
|
+
suffix = (f"?{parts.query}" if parts.query else "") + (f"#{parts.fragment}" if parts.fragment else "")
|
|
62
|
+
token.attrs[attr] = f"{root}assets/{path}{suffix}"
|
|
63
|
+
return md.renderer.render(tokens, md.options, {})
|
hexapla/analysis.py
ADDED
|
@@ -0,0 +1,370 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Statistics over a dataset for the overview, lexicon, insights and review pages.
|
|
3
|
+
|
|
4
|
+
Everything here is plain data (dicts, lists, numbers) so it can be dumped to JSON for the pages.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from collections import Counter, defaultdict
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
|
|
11
|
+
from .links import Link
|
|
12
|
+
from .conllu import Sentence, Word
|
|
13
|
+
from .corpus import AlignedPair as Dataset, Verse
|
|
14
|
+
|
|
15
|
+
LOW_CONFIDENCE = 0.8 # MCParser's default refinement threshold
|
|
16
|
+
CONFIDENCE_COLUMNS = ("lemma", "upos", "feats", "head", "deprel")
|
|
17
|
+
NOMINAL_UPOS = {"NOUN", "PROPN", "PRON", "ADJ", "DET", "NUM"}
|
|
18
|
+
VERBAL_UPOS = {"VERB", "AUX"}
|
|
19
|
+
MISSING = "—"
|
|
20
|
+
MAX_EXAMPLES = 12
|
|
21
|
+
|
|
22
|
+
# The grammatical correspondences shown on the insights page: (key, title, feature, description)
|
|
23
|
+
FEATURES = [
|
|
24
|
+
("Case", "Case", "How the cases of one language are rendered in the other"),
|
|
25
|
+
("Number", "Number", "Singular, plural and dual across the translation"),
|
|
26
|
+
("Tense", "Tense", "Tense of aligned verbs"),
|
|
27
|
+
("Aspect", "Aspect", "Verbal aspect of aligned verbs"),
|
|
28
|
+
("Mood", "Mood", "Indicative, subjunctive, imperative... of aligned verbs"),
|
|
29
|
+
("VerbForm", "Verb form", "Finite verbs, participles, infinitives..."),
|
|
30
|
+
("Voice", "Voice", "Active, middle and passive"),
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def word_confidence(word: Word) -> float | None:
|
|
35
|
+
""" The lowest MCParser confidence (MISC ``Conf=``) over the columns that matter for display. """
|
|
36
|
+
conf = word.misc_dict.get("Conf")
|
|
37
|
+
if not conf:
|
|
38
|
+
return None
|
|
39
|
+
values = {}
|
|
40
|
+
for item in conf.split(","):
|
|
41
|
+
column, _, value = item.partition(":")
|
|
42
|
+
try:
|
|
43
|
+
values[column] = float(value)
|
|
44
|
+
except ValueError:
|
|
45
|
+
continue
|
|
46
|
+
relevant = [values[c] for c in CONFIDENCE_COLUMNS if c in values]
|
|
47
|
+
return min(relevant) if relevant else None
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def feats_dict(word: Word) -> dict[str, str]:
|
|
51
|
+
if not word.feats or word.feats == "_":
|
|
52
|
+
return {}
|
|
53
|
+
return dict(item.split("=", 1) for item in word.feats.split("|") if "=" in item)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def reordering(links: list[Link]) -> float | None:
|
|
57
|
+
"""
|
|
58
|
+
Share of link pairs that cross (0 = the two texts keep the same word order, 1 = fully reversed).
|
|
59
|
+
Pairs sharing a word on either side are ignored. None when there are fewer than two such pairs.
|
|
60
|
+
"""
|
|
61
|
+
pairs = crossing = 0
|
|
62
|
+
for i, first in enumerate(links):
|
|
63
|
+
for second in links[i + 1:]:
|
|
64
|
+
ds = first.source - second.source
|
|
65
|
+
dt = first.target - second.target
|
|
66
|
+
if ds == 0 or dt == 0:
|
|
67
|
+
continue
|
|
68
|
+
pairs += 1
|
|
69
|
+
if ds * dt < 0:
|
|
70
|
+
crossing += 1
|
|
71
|
+
return crossing / pairs if pairs >= 2 else None
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def alignment_groups(links: list[Link]) -> list[tuple[set[int], set[int]]]:
|
|
75
|
+
""" Connected groups of linked words: ({A ids}, {B ids}). """
|
|
76
|
+
parent: dict[tuple[str, int], tuple[str, int]] = {}
|
|
77
|
+
|
|
78
|
+
def find(x):
|
|
79
|
+
parent.setdefault(x, x)
|
|
80
|
+
while parent[x] != x:
|
|
81
|
+
parent[x] = parent[parent[x]]
|
|
82
|
+
x = parent[x]
|
|
83
|
+
return x
|
|
84
|
+
|
|
85
|
+
for link in links:
|
|
86
|
+
parent[find(("a", link.source))] = find(("b", link.target))
|
|
87
|
+
groups: dict[tuple[str, int], tuple[set[int], set[int]]] = defaultdict(lambda: (set(), set()))
|
|
88
|
+
for node in list(parent):
|
|
89
|
+
side, word_id = node
|
|
90
|
+
groups[find(node)][0 if side == "a" else 1].add(word_id)
|
|
91
|
+
return list(groups.values())
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def group_shape(a_ids: set[int], b_ids: set[int]) -> str:
|
|
95
|
+
a = "1" if len(a_ids) == 1 else "n"
|
|
96
|
+
b = "1" if len(b_ids) == 1 else "n"
|
|
97
|
+
return {"11": "1:1", "1n": "1:n", "n1": "n:1", "nn": "n:m"}[a + b]
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@dataclass
|
|
101
|
+
class VerseStats:
|
|
102
|
+
verse: Verse
|
|
103
|
+
index: int
|
|
104
|
+
|
|
105
|
+
def as_dict(self) -> dict:
|
|
106
|
+
verse = self.verse
|
|
107
|
+
aligned_a = {link.source for link in verse.links}
|
|
108
|
+
aligned_b = {link.target for link in verse.links}
|
|
109
|
+
words_a, words_b = len(verse.a.words), len(verse.b.words)
|
|
110
|
+
confidences = [link.confidence for link in verse.links if link.confidence is not None]
|
|
111
|
+
word_confs = [c for w in verse.a.words + verse.b.words if (c := word_confidence(w)) is not None]
|
|
112
|
+
return {
|
|
113
|
+
"i": self.index,
|
|
114
|
+
"id": verse.id,
|
|
115
|
+
"label": verse.label,
|
|
116
|
+
"words_a": words_a,
|
|
117
|
+
"words_b": words_b,
|
|
118
|
+
"links": len(verse.links),
|
|
119
|
+
"probable": sum(1 for link in verse.links if not link.sure),
|
|
120
|
+
"coverage_a": round(len(aligned_a) / words_a, 3) if words_a else 0,
|
|
121
|
+
"coverage_b": round(len(aligned_b) / words_b, 3) if words_b else 0,
|
|
122
|
+
"coverage": round((len(aligned_a) + len(aligned_b)) / (words_a + words_b), 3) if words_a + words_b else 0,
|
|
123
|
+
"link_confidence": round(sum(confidences) / len(confidences), 3) if confidences else None,
|
|
124
|
+
"reordering": None if (r := reordering(verse.links)) is None else round(r, 3),
|
|
125
|
+
"parse_confidence": round(sum(word_confs) / len(word_confs), 3) if word_confs else None,
|
|
126
|
+
"low_confidence_words": sum(1 for c in word_confs if c < LOW_CONFIDENCE),
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def verse_stats(dataset: Dataset) -> list[dict]:
|
|
131
|
+
return [VerseStats(verse, i).as_dict() for i, verse in enumerate(dataset.verses)]
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def summary(dataset: Dataset, verses: list[dict] | None = None) -> dict:
|
|
135
|
+
verses = verses if verses is not None else verse_stats(dataset)
|
|
136
|
+
links = [link for verse in dataset.verses for link in verse.links]
|
|
137
|
+
words_a = sum(v["words_a"] for v in verses)
|
|
138
|
+
words_b = sum(v["words_b"] for v in verses)
|
|
139
|
+
aligned_a = sum(round(v["coverage_a"] * v["words_a"]) for v in verses)
|
|
140
|
+
aligned_b = sum(round(v["coverage_b"] * v["words_b"]) for v in verses)
|
|
141
|
+
confidences = [link.confidence for link in links if link.confidence is not None]
|
|
142
|
+
reorderings = [v["reordering"] for v in verses if v["reordering"] is not None]
|
|
143
|
+
parse = [v["parse_confidence"] for v in verses if v["parse_confidence"] is not None]
|
|
144
|
+
|
|
145
|
+
shapes = Counter()
|
|
146
|
+
for verse in dataset.verses:
|
|
147
|
+
for a_ids, b_ids in alignment_groups(verse.links):
|
|
148
|
+
shapes[group_shape(a_ids, b_ids)] += 1
|
|
149
|
+
|
|
150
|
+
return {
|
|
151
|
+
"verses": len(verses),
|
|
152
|
+
"aligned_verses": sum(1 for v in verses if v["links"]),
|
|
153
|
+
"words_a": words_a,
|
|
154
|
+
"words_b": words_b,
|
|
155
|
+
"aligned_a": aligned_a,
|
|
156
|
+
"aligned_b": aligned_b,
|
|
157
|
+
"coverage_a": aligned_a / words_a if words_a else 0,
|
|
158
|
+
"coverage_b": aligned_b / words_b if words_b else 0,
|
|
159
|
+
"links": len(links),
|
|
160
|
+
"sure": sum(1 for link in links if link.sure),
|
|
161
|
+
"probable": sum(1 for link in links if not link.sure),
|
|
162
|
+
"link_confidence": sum(confidences) / len(confidences) if confidences else None,
|
|
163
|
+
"reordering": sum(reorderings) / len(reorderings) if reorderings else None,
|
|
164
|
+
"parse_confidence": sum(parse) / len(parse) if parse else None,
|
|
165
|
+
"low_confidence_words": sum(v["low_confidence_words"] for v in verses),
|
|
166
|
+
"shapes": {shape: shapes.get(shape, 0) for shape in ("1:1", "1:n", "n:1", "n:m")},
|
|
167
|
+
"books": books(dataset),
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def books(dataset: Dataset) -> list[dict]:
|
|
172
|
+
""" Verse counts per book, in order of first appearance. """
|
|
173
|
+
counts: dict[str, int] = {}
|
|
174
|
+
for verse in dataset.verses:
|
|
175
|
+
book = verse.label.rsplit(" ", 1)[0] if ":" in verse.label else "Other"
|
|
176
|
+
counts[book] = counts.get(book, 0) + 1
|
|
177
|
+
return [{"book": book, "verses": n} for book, n in counts.items()]
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
class Matrix:
|
|
181
|
+
""" Counts of (A value, B value) pairs with a few examples per cell. """
|
|
182
|
+
|
|
183
|
+
def __init__(self):
|
|
184
|
+
self.counts: Counter = Counter()
|
|
185
|
+
self.examples: dict[tuple[str, str], list] = defaultdict(list)
|
|
186
|
+
|
|
187
|
+
def add(self, a_value: str, b_value: str, verse_index: int, a_id: int, b_id: int):
|
|
188
|
+
key = (a_value, b_value)
|
|
189
|
+
self.counts[key] += 1
|
|
190
|
+
examples = self.examples[key]
|
|
191
|
+
if len(examples) < MAX_EXAMPLES and not any(e[0] == verse_index and e[1] == a_id for e in examples):
|
|
192
|
+
examples.append([verse_index, a_id, b_id])
|
|
193
|
+
|
|
194
|
+
def as_dict(self) -> dict:
|
|
195
|
+
row_totals, col_totals = Counter(), Counter()
|
|
196
|
+
for (a, b), n in self.counts.items():
|
|
197
|
+
row_totals[a] += n
|
|
198
|
+
col_totals[b] += n
|
|
199
|
+
|
|
200
|
+
def order(totals):
|
|
201
|
+
# Most frequent first, the "missing" value last.
|
|
202
|
+
return sorted(totals, key=lambda v: (v == MISSING, -totals[v], v))
|
|
203
|
+
|
|
204
|
+
return {
|
|
205
|
+
"rows": order(row_totals),
|
|
206
|
+
"cols": order(col_totals),
|
|
207
|
+
"cells": [
|
|
208
|
+
{"a": a, "b": b, "n": n, "ex": self.examples[(a, b)]}
|
|
209
|
+
for (a, b), n in sorted(self.counts.items(), key=lambda item: -item[1])
|
|
210
|
+
],
|
|
211
|
+
"total": sum(self.counts.values()),
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def correspondences(dataset: Dataset) -> list[dict]:
|
|
216
|
+
""" Part of speech and morphological feature correspondences across alignment links. """
|
|
217
|
+
upos = Matrix()
|
|
218
|
+
features = {key: Matrix() for key, _, _ in FEATURES}
|
|
219
|
+
|
|
220
|
+
for index, verse in enumerate(dataset.verses):
|
|
221
|
+
words_a = {w.id: w for w in verse.a.words}
|
|
222
|
+
words_b = {w.id: w for w in verse.b.words}
|
|
223
|
+
for link in verse.links:
|
|
224
|
+
a, b = words_a[link.source], words_b[link.target]
|
|
225
|
+
if a.upos == "PUNCT" or b.upos == "PUNCT":
|
|
226
|
+
continue
|
|
227
|
+
upos.add(a.upos, b.upos, index, a.id, b.id)
|
|
228
|
+
fa, fb = feats_dict(a), feats_dict(b)
|
|
229
|
+
for key, _, _ in FEATURES:
|
|
230
|
+
if key in fa or key in fb:
|
|
231
|
+
features[key].add(fa.get(key, MISSING), fb.get(key, MISSING), index, a.id, b.id)
|
|
232
|
+
|
|
233
|
+
result = [{"key": "upos", "title": "Part of speech",
|
|
234
|
+
"description": "Universal part-of-speech tags of aligned words", **upos.as_dict()}]
|
|
235
|
+
for key, title, description in FEATURES:
|
|
236
|
+
matrix = features[key].as_dict()
|
|
237
|
+
if matrix["total"]:
|
|
238
|
+
result.append({"key": key.lower(), "title": title, "description": description, **matrix})
|
|
239
|
+
return result
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def unaligned_by_upos(dataset: Dataset) -> dict[str, list[dict]]:
|
|
243
|
+
""" For each side: per part of speech, how many words have no counterpart. """
|
|
244
|
+
result = {}
|
|
245
|
+
for side in ("a", "b"):
|
|
246
|
+
totals, unaligned = Counter(), Counter()
|
|
247
|
+
examples: dict[str, list] = defaultdict(list)
|
|
248
|
+
for index, verse in enumerate(dataset.verses):
|
|
249
|
+
sentence = verse.a if side == "a" else verse.b
|
|
250
|
+
linked = {link.source if side == "a" else link.target for link in verse.links}
|
|
251
|
+
if not verse.links:
|
|
252
|
+
continue # unaligned verses say nothing about translation technique
|
|
253
|
+
for word in sentence.words:
|
|
254
|
+
if word.upos == "PUNCT":
|
|
255
|
+
continue
|
|
256
|
+
totals[word.upos] += 1
|
|
257
|
+
if word.id not in linked:
|
|
258
|
+
unaligned[word.upos] += 1
|
|
259
|
+
if len(examples[word.upos]) < MAX_EXAMPLES:
|
|
260
|
+
examples[word.upos].append([index, word.id])
|
|
261
|
+
rows = [
|
|
262
|
+
{"upos": upos, "total": totals[upos], "unaligned": unaligned[upos],
|
|
263
|
+
"share": unaligned[upos] / totals[upos], "ex": examples[upos]}
|
|
264
|
+
for upos in totals
|
|
265
|
+
]
|
|
266
|
+
result[side] = sorted(rows, key=lambda r: (-r["share"], -r["total"]))
|
|
267
|
+
return result
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
GAP = "…"
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def equivalent(partner_ids: list[int], other_words: dict) -> tuple[str, ...]:
|
|
274
|
+
"""
|
|
275
|
+
The lemmas a word is linked to, in text order, as one equivalent: "in Christus" for one word
|
|
276
|
+
rather than "in" and "Christus". Words that are not next to each other are separated by "…".
|
|
277
|
+
"""
|
|
278
|
+
lemmas: list[str] = []
|
|
279
|
+
previous = None
|
|
280
|
+
for partner_id in sorted(set(partner_ids)):
|
|
281
|
+
partner = other_words.get(partner_id)
|
|
282
|
+
if partner and partner.lemma and partner.lemma != "_":
|
|
283
|
+
if lemmas and previous is not None and partner_id - previous > 1 and lemmas[-1] != GAP:
|
|
284
|
+
lemmas.append(GAP)
|
|
285
|
+
lemmas.append(partner.lemma)
|
|
286
|
+
previous = partner_id
|
|
287
|
+
return tuple(lemmas)
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def lexicon(dataset: Dataset, side: str, top: int = 6) -> list[dict]:
|
|
291
|
+
""" Each lemma of one side with its frequency, share aligned and most frequent equivalents. """
|
|
292
|
+
entries: dict[str, dict] = {}
|
|
293
|
+
for verse in dataset.verses:
|
|
294
|
+
this, other = (verse.a, verse.b) if side == "a" else (verse.b, verse.a)
|
|
295
|
+
other_words = {w.id: w for w in other.words}
|
|
296
|
+
partners: dict[int, list[int]] = defaultdict(list)
|
|
297
|
+
for link in verse.links:
|
|
298
|
+
if side == "a":
|
|
299
|
+
partners[link.source].append(link.target)
|
|
300
|
+
else:
|
|
301
|
+
partners[link.target].append(link.source)
|
|
302
|
+
for word in this.words:
|
|
303
|
+
if word.upos == "PUNCT" or not word.lemma or word.lemma == "_":
|
|
304
|
+
continue
|
|
305
|
+
entry = entries.setdefault(word.lemma, {"count": 0, "aligned": 0, "in_aligned": 0, "upos": Counter(),
|
|
306
|
+
"forms": Counter(), "eq": Counter(), "verses": set()})
|
|
307
|
+
entry["count"] += 1
|
|
308
|
+
entry["in_aligned"] += bool(verse.links)
|
|
309
|
+
entry["upos"][word.upos] += 1
|
|
310
|
+
entry["forms"][word.form] += 1
|
|
311
|
+
entry["verses"].add(verse.id)
|
|
312
|
+
ids = partners.get(word.id, [])
|
|
313
|
+
if ids:
|
|
314
|
+
entry["aligned"] += 1
|
|
315
|
+
if lemmas := equivalent(ids, other_words):
|
|
316
|
+
entry["eq"][lemmas] += 1
|
|
317
|
+
return sorted(
|
|
318
|
+
(
|
|
319
|
+
{
|
|
320
|
+
"lemma": lemma,
|
|
321
|
+
"upos": entry["upos"].most_common(1)[0][0],
|
|
322
|
+
"count": entry["count"],
|
|
323
|
+
"aligned": entry["aligned"],
|
|
324
|
+
"in_aligned": entry["in_aligned"],
|
|
325
|
+
"verses": len(entry["verses"]),
|
|
326
|
+
"forms": len(entry["forms"]),
|
|
327
|
+
# [label, count] or, for several linked words, [label, count, [lemma, ...]]
|
|
328
|
+
"eq": [[" ".join(lemmas), n] + ([list(lemmas)] if len(lemmas) > 1 else [])
|
|
329
|
+
for lemmas, n in entry["eq"].most_common(top)],
|
|
330
|
+
"eq_total": len(entry["eq"]),
|
|
331
|
+
}
|
|
332
|
+
for lemma, entry in entries.items()
|
|
333
|
+
),
|
|
334
|
+
key=lambda e: (-e["count"], e["lemma"]),
|
|
335
|
+
)
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def review(dataset: Dataset, limit: int = 300) -> dict:
|
|
339
|
+
""" The links and words most worth checking: least confident first. """
|
|
340
|
+
links, words = [], []
|
|
341
|
+
for index, verse in enumerate(dataset.verses):
|
|
342
|
+
words_a = {w.id: w for w in verse.a.words}
|
|
343
|
+
words_b = {w.id: w for w in verse.b.words}
|
|
344
|
+
for link in verse.links:
|
|
345
|
+
if link.confidence is None and link.sure:
|
|
346
|
+
continue
|
|
347
|
+
links.append({
|
|
348
|
+
"v": index, "a": link.source, "b": link.target, "sure": link.sure,
|
|
349
|
+
"confidence": link.confidence, "agent": link.agent,
|
|
350
|
+
"a_form": words_a[link.source].form, "b_form": words_b[link.target].form,
|
|
351
|
+
})
|
|
352
|
+
for side, sentence in (("a", verse.a), ("b", verse.b)):
|
|
353
|
+
for word in sentence.words:
|
|
354
|
+
if word.upos == "PUNCT":
|
|
355
|
+
continue
|
|
356
|
+
confidence = word_confidence(word)
|
|
357
|
+
if confidence is not None and confidence < LOW_CONFIDENCE:
|
|
358
|
+
words.append({"v": index, "side": side, "id": word.id, "form": word.form, "lemma": word.lemma,
|
|
359
|
+
"upos": word.upos, "deprel": word.deprel, "confidence": confidence})
|
|
360
|
+
links.sort(key=lambda l: (l["sure"], l["confidence"] if l["confidence"] is not None else 1))
|
|
361
|
+
words.sort(key=lambda w: w["confidence"])
|
|
362
|
+
return {"links": links[:limit], "links_total": len(links), "words": words[:limit], "words_total": len(words)}
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def corpus(dataset: Dataset) -> list[dict]:
|
|
366
|
+
""" Compact verse texts for pages that show examples: forms, ids and no-space-after ids per side. """
|
|
367
|
+
def side(sentence: Sentence):
|
|
368
|
+
return {"ids": [w.id for w in sentence.words], "forms": [w.form for w in sentence.words],
|
|
369
|
+
"ns": sentence.no_space_after}
|
|
370
|
+
return [{"id": v.id, "label": v.label, "a": side(v.a), "b": side(v.b)} for v in dataset.verses]
|