hexapla 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. hexapla/__init__.py +3 -0
  2. hexapla/about.py +63 -0
  3. hexapla/analysis.py +370 -0
  4. hexapla/bible.py +285 -0
  5. hexapla/build.py +509 -0
  6. hexapla/config.py +221 -0
  7. hexapla/conllu.py +166 -0
  8. hexapla/corpus.py +302 -0
  9. hexapla/languages.py +36 -0
  10. hexapla/licences.py +24 -0
  11. hexapla/links.py +105 -0
  12. hexapla/main.py +166 -0
  13. hexapla/static/img/hexapla-logo.svg +50 -0
  14. hexapla/static/img/hexapla-small.svg +50 -0
  15. hexapla/static/js/bootstrap.bundle.min.js +7 -0
  16. hexapla/static/js/common.js +122 -0
  17. hexapla/static/js/d3.v7.min.js +2 -0
  18. hexapla/static/js/navigator.js +419 -0
  19. hexapla/static/js/pages.js +311 -0
  20. hexapla/static/scss/_hexapla.scss +339 -0
  21. hexapla/static/scss/bootstrap/LICENSE +21 -0
  22. hexapla/static/scss/bootstrap/scss/_accordion.scss +153 -0
  23. hexapla/static/scss/bootstrap/scss/_alert.scss +68 -0
  24. hexapla/static/scss/bootstrap/scss/_badge.scss +38 -0
  25. hexapla/static/scss/bootstrap/scss/_breadcrumb.scss +40 -0
  26. hexapla/static/scss/bootstrap/scss/_button-group.scss +147 -0
  27. hexapla/static/scss/bootstrap/scss/_buttons.scss +216 -0
  28. hexapla/static/scss/bootstrap/scss/_card.scss +238 -0
  29. hexapla/static/scss/bootstrap/scss/_carousel.scss +226 -0
  30. hexapla/static/scss/bootstrap/scss/_close.scss +66 -0
  31. hexapla/static/scss/bootstrap/scss/_containers.scss +41 -0
  32. hexapla/static/scss/bootstrap/scss/_dropdown.scss +250 -0
  33. hexapla/static/scss/bootstrap/scss/_forms.scss +9 -0
  34. hexapla/static/scss/bootstrap/scss/_functions.scss +302 -0
  35. hexapla/static/scss/bootstrap/scss/_grid.scss +39 -0
  36. hexapla/static/scss/bootstrap/scss/_helpers.scss +12 -0
  37. hexapla/static/scss/bootstrap/scss/_images.scss +42 -0
  38. hexapla/static/scss/bootstrap/scss/_list-group.scss +199 -0
  39. hexapla/static/scss/bootstrap/scss/_maps.scss +174 -0
  40. hexapla/static/scss/bootstrap/scss/_mixins.scss +42 -0
  41. hexapla/static/scss/bootstrap/scss/_modal.scss +240 -0
  42. hexapla/static/scss/bootstrap/scss/_nav.scss +197 -0
  43. hexapla/static/scss/bootstrap/scss/_navbar.scss +289 -0
  44. hexapla/static/scss/bootstrap/scss/_offcanvas.scss +147 -0
  45. hexapla/static/scss/bootstrap/scss/_pagination.scss +109 -0
  46. hexapla/static/scss/bootstrap/scss/_placeholders.scss +51 -0
  47. hexapla/static/scss/bootstrap/scss/_popover.scss +196 -0
  48. hexapla/static/scss/bootstrap/scss/_progress.scss +68 -0
  49. hexapla/static/scss/bootstrap/scss/_reboot.scss +617 -0
  50. hexapla/static/scss/bootstrap/scss/_root.scss +187 -0
  51. hexapla/static/scss/bootstrap/scss/_spinners.scss +86 -0
  52. hexapla/static/scss/bootstrap/scss/_tables.scss +171 -0
  53. hexapla/static/scss/bootstrap/scss/_toasts.scss +73 -0
  54. hexapla/static/scss/bootstrap/scss/_tooltip.scss +119 -0
  55. hexapla/static/scss/bootstrap/scss/_transitions.scss +27 -0
  56. hexapla/static/scss/bootstrap/scss/_type.scss +106 -0
  57. hexapla/static/scss/bootstrap/scss/_utilities.scss +806 -0
  58. hexapla/static/scss/bootstrap/scss/_variables-dark.scss +102 -0
  59. hexapla/static/scss/bootstrap/scss/_variables.scss +1753 -0
  60. hexapla/static/scss/bootstrap/scss/bootstrap-grid.scss +62 -0
  61. hexapla/static/scss/bootstrap/scss/bootstrap-reboot.scss +10 -0
  62. hexapla/static/scss/bootstrap/scss/bootstrap-utilities.scss +19 -0
  63. hexapla/static/scss/bootstrap/scss/bootstrap.scss +52 -0
  64. hexapla/static/scss/bootstrap/scss/forms/_floating-labels.scss +97 -0
  65. hexapla/static/scss/bootstrap/scss/forms/_form-check.scss +189 -0
  66. hexapla/static/scss/bootstrap/scss/forms/_form-control.scss +214 -0
  67. hexapla/static/scss/bootstrap/scss/forms/_form-range.scss +91 -0
  68. hexapla/static/scss/bootstrap/scss/forms/_form-select.scss +80 -0
  69. hexapla/static/scss/bootstrap/scss/forms/_form-text.scss +11 -0
  70. hexapla/static/scss/bootstrap/scss/forms/_input-group.scss +132 -0
  71. hexapla/static/scss/bootstrap/scss/forms/_labels.scss +36 -0
  72. hexapla/static/scss/bootstrap/scss/forms/_validation.scss +12 -0
  73. hexapla/static/scss/bootstrap/scss/helpers/_clearfix.scss +3 -0
  74. hexapla/static/scss/bootstrap/scss/helpers/_color-bg.scss +7 -0
  75. hexapla/static/scss/bootstrap/scss/helpers/_colored-links.scss +30 -0
  76. hexapla/static/scss/bootstrap/scss/helpers/_focus-ring.scss +5 -0
  77. hexapla/static/scss/bootstrap/scss/helpers/_icon-link.scss +25 -0
  78. hexapla/static/scss/bootstrap/scss/helpers/_position.scss +36 -0
  79. hexapla/static/scss/bootstrap/scss/helpers/_ratio.scss +26 -0
  80. hexapla/static/scss/bootstrap/scss/helpers/_stacks.scss +15 -0
  81. hexapla/static/scss/bootstrap/scss/helpers/_stretched-link.scss +15 -0
  82. hexapla/static/scss/bootstrap/scss/helpers/_text-truncation.scss +7 -0
  83. hexapla/static/scss/bootstrap/scss/helpers/_visually-hidden.scss +8 -0
  84. hexapla/static/scss/bootstrap/scss/helpers/_vr.scss +8 -0
  85. hexapla/static/scss/bootstrap/scss/mixins/_alert.scss +18 -0
  86. hexapla/static/scss/bootstrap/scss/mixins/_backdrop.scss +14 -0
  87. hexapla/static/scss/bootstrap/scss/mixins/_banner.scss +7 -0
  88. hexapla/static/scss/bootstrap/scss/mixins/_border-radius.scss +78 -0
  89. hexapla/static/scss/bootstrap/scss/mixins/_box-shadow.scss +24 -0
  90. hexapla/static/scss/bootstrap/scss/mixins/_breakpoints.scss +127 -0
  91. hexapla/static/scss/bootstrap/scss/mixins/_buttons.scss +70 -0
  92. hexapla/static/scss/bootstrap/scss/mixins/_caret.scss +69 -0
  93. hexapla/static/scss/bootstrap/scss/mixins/_clearfix.scss +9 -0
  94. hexapla/static/scss/bootstrap/scss/mixins/_color-mode.scss +21 -0
  95. hexapla/static/scss/bootstrap/scss/mixins/_color-scheme.scss +7 -0
  96. hexapla/static/scss/bootstrap/scss/mixins/_container.scss +11 -0
  97. hexapla/static/scss/bootstrap/scss/mixins/_deprecate.scss +10 -0
  98. hexapla/static/scss/bootstrap/scss/mixins/_forms.scss +163 -0
  99. hexapla/static/scss/bootstrap/scss/mixins/_gradients.scss +47 -0
  100. hexapla/static/scss/bootstrap/scss/mixins/_grid.scss +151 -0
  101. hexapla/static/scss/bootstrap/scss/mixins/_image.scss +16 -0
  102. hexapla/static/scss/bootstrap/scss/mixins/_list-group.scss +26 -0
  103. hexapla/static/scss/bootstrap/scss/mixins/_lists.scss +7 -0
  104. hexapla/static/scss/bootstrap/scss/mixins/_pagination.scss +10 -0
  105. hexapla/static/scss/bootstrap/scss/mixins/_reset-text.scss +17 -0
  106. hexapla/static/scss/bootstrap/scss/mixins/_resize.scss +6 -0
  107. hexapla/static/scss/bootstrap/scss/mixins/_table-variants.scss +24 -0
  108. hexapla/static/scss/bootstrap/scss/mixins/_text-truncate.scss +8 -0
  109. hexapla/static/scss/bootstrap/scss/mixins/_transition.scss +26 -0
  110. hexapla/static/scss/bootstrap/scss/mixins/_utilities.scss +97 -0
  111. hexapla/static/scss/bootstrap/scss/mixins/_visually-hidden.scss +38 -0
  112. hexapla/static/scss/bootstrap/scss/utilities/_api.scss +47 -0
  113. hexapla/static/scss/bootstrap/scss/vendor/_rfs.scss +348 -0
  114. hexapla/templates/about.html +7 -0
  115. hexapla/templates/alignment.html +121 -0
  116. hexapla/templates/base.html +82 -0
  117. hexapla/templates/home.html +29 -0
  118. hexapla/templates/insights.html +61 -0
  119. hexapla/templates/lemma.html +67 -0
  120. hexapla/templates/lexicon.html +38 -0
  121. hexapla/templates/manuscript.html +86 -0
  122. hexapla/templates/review.html +70 -0
  123. hexapla/templates/verse.html +106 -0
  124. hexapla/templates/verse.js +4 -0
  125. hexapla/theme.py +37 -0
  126. hexapla/transcription.py +396 -0
  127. hexapla-0.1.0.dist-info/METADATA +290 -0
  128. hexapla-0.1.0.dist-info/RECORD +131 -0
  129. hexapla-0.1.0.dist-info/WHEEL +4 -0
  130. hexapla-0.1.0.dist-info/entry_points.txt +2 -0
  131. hexapla-0.1.0.dist-info/licenses/LICENSE +202 -0
hexapla/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ """ Hexapla: static websites for reading and exploring word-aligned manuscript texts. """
2
+
3
+ __version__ = "0.1.0"
hexapla/about.py ADDED
@@ -0,0 +1,63 @@
1
+ """
2
+ The about page: the theme's Markdown file, rendered to HTML.
3
+
4
+ Images and links in it are written relative to the Markdown file, as they are when it is previewed on GitHub.
5
+ Images can be given a size or Bootstrap classes: ``![ERC](assets/erc-logo.jpg){width=180 .float-end}``.
6
+ Those that point into the theme's ``assets`` folder (``![ERC](assets/erc-logo.jpg)``) are rewritten to the copy
7
+ of the folder in the site; any other local link would be broken on the site, so validation reports it.
8
+ """
9
+ from __future__ import annotations
10
+
11
+ from pathlib import Path
12
+ from urllib.parse import urlsplit
13
+
14
+ from markdown_it import MarkdownIt
15
+ from mdit_py_plugins.attrs import attrs_plugin
16
+
17
+ ATTRIBUTES = {"image": "src", "link_open": "href"}
18
+
19
+
20
+ def _markdown() -> MarkdownIt:
21
+ return MarkdownIt().use(attrs_plugin)
22
+
23
+
24
+ def _link_tokens(tokens):
25
+ for token in tokens:
26
+ for child in token.children or []:
27
+ if child.type in ATTRIBUTES:
28
+ yield child, ATTRIBUTES[child.type]
29
+
30
+
31
+ def _is_local(url: str) -> bool:
32
+ parts = urlsplit(url)
33
+ return bool(parts.path) and not parts.scheme and not parts.netloc and not url.startswith("/")
34
+
35
+
36
+ def local_links(text: str) -> list[str]:
37
+ """ The images and links in a Markdown text that point to local files, in order, without repeats. """
38
+ links = [str(token.attrs[attr]) for token, attr in _link_tokens(_markdown().parse(text))]
39
+ return list(dict.fromkeys(link for link in links if _is_local(link)))
40
+
41
+
42
+ def asset_path(link: str, about: Path, assets: Path | None) -> str | None:
43
+ """ Where a link from the about file lands in the assets folder (``img/logo.png``), or None if it is not there. """
44
+ if assets is None:
45
+ return None
46
+ target = (about.parent / urlsplit(link).path).resolve()
47
+ assets = assets.resolve()
48
+ if not target.is_file() or not target.is_relative_to(assets):
49
+ return None
50
+ return target.relative_to(assets).as_posix()
51
+
52
+
53
+ def render_about(about: Path, assets: Path | None, root: str) -> str:
54
+ """ The about page's HTML, with links into the assets folder pointing at its copy in the site. """
55
+ md = _markdown()
56
+ tokens = md.parse(about.read_text(encoding="utf-8"))
57
+ for token, attr in _link_tokens(tokens):
58
+ link = str(token.attrs[attr])
59
+ if _is_local(link) and (path := asset_path(link, about, assets)):
60
+ parts = urlsplit(link)
61
+ suffix = (f"?{parts.query}" if parts.query else "") + (f"#{parts.fragment}" if parts.fragment else "")
62
+ token.attrs[attr] = f"{root}assets/{path}{suffix}"
63
+ return md.renderer.render(tokens, md.options, {})
hexapla/analysis.py ADDED
@@ -0,0 +1,370 @@
1
+ """
2
+ Statistics over a dataset for the overview, lexicon, insights and review pages.
3
+
4
+ Everything here is plain data (dicts, lists, numbers) so it can be dumped to JSON for the pages.
5
+ """
6
+ from __future__ import annotations
7
+
8
+ from collections import Counter, defaultdict
9
+ from dataclasses import dataclass
10
+
11
+ from .links import Link
12
+ from .conllu import Sentence, Word
13
+ from .corpus import AlignedPair as Dataset, Verse
14
+
15
+ LOW_CONFIDENCE = 0.8 # MCParser's default refinement threshold
16
+ CONFIDENCE_COLUMNS = ("lemma", "upos", "feats", "head", "deprel")
17
+ NOMINAL_UPOS = {"NOUN", "PROPN", "PRON", "ADJ", "DET", "NUM"}
18
+ VERBAL_UPOS = {"VERB", "AUX"}
19
+ MISSING = "—"
20
+ MAX_EXAMPLES = 12
21
+
22
+ # The grammatical correspondences shown on the insights page: (key, title, feature, description)
23
+ FEATURES = [
24
+ ("Case", "Case", "How the cases of one language are rendered in the other"),
25
+ ("Number", "Number", "Singular, plural and dual across the translation"),
26
+ ("Tense", "Tense", "Tense of aligned verbs"),
27
+ ("Aspect", "Aspect", "Verbal aspect of aligned verbs"),
28
+ ("Mood", "Mood", "Indicative, subjunctive, imperative... of aligned verbs"),
29
+ ("VerbForm", "Verb form", "Finite verbs, participles, infinitives..."),
30
+ ("Voice", "Voice", "Active, middle and passive"),
31
+ ]
32
+
33
+
34
+ def word_confidence(word: Word) -> float | None:
35
+ """ The lowest MCParser confidence (MISC ``Conf=``) over the columns that matter for display. """
36
+ conf = word.misc_dict.get("Conf")
37
+ if not conf:
38
+ return None
39
+ values = {}
40
+ for item in conf.split(","):
41
+ column, _, value = item.partition(":")
42
+ try:
43
+ values[column] = float(value)
44
+ except ValueError:
45
+ continue
46
+ relevant = [values[c] for c in CONFIDENCE_COLUMNS if c in values]
47
+ return min(relevant) if relevant else None
48
+
49
+
50
+ def feats_dict(word: Word) -> dict[str, str]:
51
+ if not word.feats or word.feats == "_":
52
+ return {}
53
+ return dict(item.split("=", 1) for item in word.feats.split("|") if "=" in item)
54
+
55
+
56
+ def reordering(links: list[Link]) -> float | None:
57
+ """
58
+ Share of link pairs that cross (0 = the two texts keep the same word order, 1 = fully reversed).
59
+ Pairs sharing a word on either side are ignored. None when there are fewer than two such pairs.
60
+ """
61
+ pairs = crossing = 0
62
+ for i, first in enumerate(links):
63
+ for second in links[i + 1:]:
64
+ ds = first.source - second.source
65
+ dt = first.target - second.target
66
+ if ds == 0 or dt == 0:
67
+ continue
68
+ pairs += 1
69
+ if ds * dt < 0:
70
+ crossing += 1
71
+ return crossing / pairs if pairs >= 2 else None
72
+
73
+
74
+ def alignment_groups(links: list[Link]) -> list[tuple[set[int], set[int]]]:
75
+ """ Connected groups of linked words: ({A ids}, {B ids}). """
76
+ parent: dict[tuple[str, int], tuple[str, int]] = {}
77
+
78
+ def find(x):
79
+ parent.setdefault(x, x)
80
+ while parent[x] != x:
81
+ parent[x] = parent[parent[x]]
82
+ x = parent[x]
83
+ return x
84
+
85
+ for link in links:
86
+ parent[find(("a", link.source))] = find(("b", link.target))
87
+ groups: dict[tuple[str, int], tuple[set[int], set[int]]] = defaultdict(lambda: (set(), set()))
88
+ for node in list(parent):
89
+ side, word_id = node
90
+ groups[find(node)][0 if side == "a" else 1].add(word_id)
91
+ return list(groups.values())
92
+
93
+
94
+ def group_shape(a_ids: set[int], b_ids: set[int]) -> str:
95
+ a = "1" if len(a_ids) == 1 else "n"
96
+ b = "1" if len(b_ids) == 1 else "n"
97
+ return {"11": "1:1", "1n": "1:n", "n1": "n:1", "nn": "n:m"}[a + b]
98
+
99
+
100
+ @dataclass
101
+ class VerseStats:
102
+ verse: Verse
103
+ index: int
104
+
105
+ def as_dict(self) -> dict:
106
+ verse = self.verse
107
+ aligned_a = {link.source for link in verse.links}
108
+ aligned_b = {link.target for link in verse.links}
109
+ words_a, words_b = len(verse.a.words), len(verse.b.words)
110
+ confidences = [link.confidence for link in verse.links if link.confidence is not None]
111
+ word_confs = [c for w in verse.a.words + verse.b.words if (c := word_confidence(w)) is not None]
112
+ return {
113
+ "i": self.index,
114
+ "id": verse.id,
115
+ "label": verse.label,
116
+ "words_a": words_a,
117
+ "words_b": words_b,
118
+ "links": len(verse.links),
119
+ "probable": sum(1 for link in verse.links if not link.sure),
120
+ "coverage_a": round(len(aligned_a) / words_a, 3) if words_a else 0,
121
+ "coverage_b": round(len(aligned_b) / words_b, 3) if words_b else 0,
122
+ "coverage": round((len(aligned_a) + len(aligned_b)) / (words_a + words_b), 3) if words_a + words_b else 0,
123
+ "link_confidence": round(sum(confidences) / len(confidences), 3) if confidences else None,
124
+ "reordering": None if (r := reordering(verse.links)) is None else round(r, 3),
125
+ "parse_confidence": round(sum(word_confs) / len(word_confs), 3) if word_confs else None,
126
+ "low_confidence_words": sum(1 for c in word_confs if c < LOW_CONFIDENCE),
127
+ }
128
+
129
+
130
+ def verse_stats(dataset: Dataset) -> list[dict]:
131
+ return [VerseStats(verse, i).as_dict() for i, verse in enumerate(dataset.verses)]
132
+
133
+
134
+ def summary(dataset: Dataset, verses: list[dict] | None = None) -> dict:
135
+ verses = verses if verses is not None else verse_stats(dataset)
136
+ links = [link for verse in dataset.verses for link in verse.links]
137
+ words_a = sum(v["words_a"] for v in verses)
138
+ words_b = sum(v["words_b"] for v in verses)
139
+ aligned_a = sum(round(v["coverage_a"] * v["words_a"]) for v in verses)
140
+ aligned_b = sum(round(v["coverage_b"] * v["words_b"]) for v in verses)
141
+ confidences = [link.confidence for link in links if link.confidence is not None]
142
+ reorderings = [v["reordering"] for v in verses if v["reordering"] is not None]
143
+ parse = [v["parse_confidence"] for v in verses if v["parse_confidence"] is not None]
144
+
145
+ shapes = Counter()
146
+ for verse in dataset.verses:
147
+ for a_ids, b_ids in alignment_groups(verse.links):
148
+ shapes[group_shape(a_ids, b_ids)] += 1
149
+
150
+ return {
151
+ "verses": len(verses),
152
+ "aligned_verses": sum(1 for v in verses if v["links"]),
153
+ "words_a": words_a,
154
+ "words_b": words_b,
155
+ "aligned_a": aligned_a,
156
+ "aligned_b": aligned_b,
157
+ "coverage_a": aligned_a / words_a if words_a else 0,
158
+ "coverage_b": aligned_b / words_b if words_b else 0,
159
+ "links": len(links),
160
+ "sure": sum(1 for link in links if link.sure),
161
+ "probable": sum(1 for link in links if not link.sure),
162
+ "link_confidence": sum(confidences) / len(confidences) if confidences else None,
163
+ "reordering": sum(reorderings) / len(reorderings) if reorderings else None,
164
+ "parse_confidence": sum(parse) / len(parse) if parse else None,
165
+ "low_confidence_words": sum(v["low_confidence_words"] for v in verses),
166
+ "shapes": {shape: shapes.get(shape, 0) for shape in ("1:1", "1:n", "n:1", "n:m")},
167
+ "books": books(dataset),
168
+ }
169
+
170
+
171
+ def books(dataset: Dataset) -> list[dict]:
172
+ """ Verse counts per book, in order of first appearance. """
173
+ counts: dict[str, int] = {}
174
+ for verse in dataset.verses:
175
+ book = verse.label.rsplit(" ", 1)[0] if ":" in verse.label else "Other"
176
+ counts[book] = counts.get(book, 0) + 1
177
+ return [{"book": book, "verses": n} for book, n in counts.items()]
178
+
179
+
180
+ class Matrix:
181
+ """ Counts of (A value, B value) pairs with a few examples per cell. """
182
+
183
+ def __init__(self):
184
+ self.counts: Counter = Counter()
185
+ self.examples: dict[tuple[str, str], list] = defaultdict(list)
186
+
187
+ def add(self, a_value: str, b_value: str, verse_index: int, a_id: int, b_id: int):
188
+ key = (a_value, b_value)
189
+ self.counts[key] += 1
190
+ examples = self.examples[key]
191
+ if len(examples) < MAX_EXAMPLES and not any(e[0] == verse_index and e[1] == a_id for e in examples):
192
+ examples.append([verse_index, a_id, b_id])
193
+
194
+ def as_dict(self) -> dict:
195
+ row_totals, col_totals = Counter(), Counter()
196
+ for (a, b), n in self.counts.items():
197
+ row_totals[a] += n
198
+ col_totals[b] += n
199
+
200
+ def order(totals):
201
+ # Most frequent first, the "missing" value last.
202
+ return sorted(totals, key=lambda v: (v == MISSING, -totals[v], v))
203
+
204
+ return {
205
+ "rows": order(row_totals),
206
+ "cols": order(col_totals),
207
+ "cells": [
208
+ {"a": a, "b": b, "n": n, "ex": self.examples[(a, b)]}
209
+ for (a, b), n in sorted(self.counts.items(), key=lambda item: -item[1])
210
+ ],
211
+ "total": sum(self.counts.values()),
212
+ }
213
+
214
+
215
+ def correspondences(dataset: Dataset) -> list[dict]:
216
+ """ Part of speech and morphological feature correspondences across alignment links. """
217
+ upos = Matrix()
218
+ features = {key: Matrix() for key, _, _ in FEATURES}
219
+
220
+ for index, verse in enumerate(dataset.verses):
221
+ words_a = {w.id: w for w in verse.a.words}
222
+ words_b = {w.id: w for w in verse.b.words}
223
+ for link in verse.links:
224
+ a, b = words_a[link.source], words_b[link.target]
225
+ if a.upos == "PUNCT" or b.upos == "PUNCT":
226
+ continue
227
+ upos.add(a.upos, b.upos, index, a.id, b.id)
228
+ fa, fb = feats_dict(a), feats_dict(b)
229
+ for key, _, _ in FEATURES:
230
+ if key in fa or key in fb:
231
+ features[key].add(fa.get(key, MISSING), fb.get(key, MISSING), index, a.id, b.id)
232
+
233
+ result = [{"key": "upos", "title": "Part of speech",
234
+ "description": "Universal part-of-speech tags of aligned words", **upos.as_dict()}]
235
+ for key, title, description in FEATURES:
236
+ matrix = features[key].as_dict()
237
+ if matrix["total"]:
238
+ result.append({"key": key.lower(), "title": title, "description": description, **matrix})
239
+ return result
240
+
241
+
242
+ def unaligned_by_upos(dataset: Dataset) -> dict[str, list[dict]]:
243
+ """ For each side: per part of speech, how many words have no counterpart. """
244
+ result = {}
245
+ for side in ("a", "b"):
246
+ totals, unaligned = Counter(), Counter()
247
+ examples: dict[str, list] = defaultdict(list)
248
+ for index, verse in enumerate(dataset.verses):
249
+ sentence = verse.a if side == "a" else verse.b
250
+ linked = {link.source if side == "a" else link.target for link in verse.links}
251
+ if not verse.links:
252
+ continue # unaligned verses say nothing about translation technique
253
+ for word in sentence.words:
254
+ if word.upos == "PUNCT":
255
+ continue
256
+ totals[word.upos] += 1
257
+ if word.id not in linked:
258
+ unaligned[word.upos] += 1
259
+ if len(examples[word.upos]) < MAX_EXAMPLES:
260
+ examples[word.upos].append([index, word.id])
261
+ rows = [
262
+ {"upos": upos, "total": totals[upos], "unaligned": unaligned[upos],
263
+ "share": unaligned[upos] / totals[upos], "ex": examples[upos]}
264
+ for upos in totals
265
+ ]
266
+ result[side] = sorted(rows, key=lambda r: (-r["share"], -r["total"]))
267
+ return result
268
+
269
+
270
+ GAP = "…"
271
+
272
+
273
+ def equivalent(partner_ids: list[int], other_words: dict) -> tuple[str, ...]:
274
+ """
275
+ The lemmas a word is linked to, in text order, as one equivalent: "in Christus" for one word
276
+ rather than "in" and "Christus". Words that are not next to each other are separated by "…".
277
+ """
278
+ lemmas: list[str] = []
279
+ previous = None
280
+ for partner_id in sorted(set(partner_ids)):
281
+ partner = other_words.get(partner_id)
282
+ if partner and partner.lemma and partner.lemma != "_":
283
+ if lemmas and previous is not None and partner_id - previous > 1 and lemmas[-1] != GAP:
284
+ lemmas.append(GAP)
285
+ lemmas.append(partner.lemma)
286
+ previous = partner_id
287
+ return tuple(lemmas)
288
+
289
+
290
+ def lexicon(dataset: Dataset, side: str, top: int = 6) -> list[dict]:
291
+ """ Each lemma of one side with its frequency, share aligned and most frequent equivalents. """
292
+ entries: dict[str, dict] = {}
293
+ for verse in dataset.verses:
294
+ this, other = (verse.a, verse.b) if side == "a" else (verse.b, verse.a)
295
+ other_words = {w.id: w for w in other.words}
296
+ partners: dict[int, list[int]] = defaultdict(list)
297
+ for link in verse.links:
298
+ if side == "a":
299
+ partners[link.source].append(link.target)
300
+ else:
301
+ partners[link.target].append(link.source)
302
+ for word in this.words:
303
+ if word.upos == "PUNCT" or not word.lemma or word.lemma == "_":
304
+ continue
305
+ entry = entries.setdefault(word.lemma, {"count": 0, "aligned": 0, "in_aligned": 0, "upos": Counter(),
306
+ "forms": Counter(), "eq": Counter(), "verses": set()})
307
+ entry["count"] += 1
308
+ entry["in_aligned"] += bool(verse.links)
309
+ entry["upos"][word.upos] += 1
310
+ entry["forms"][word.form] += 1
311
+ entry["verses"].add(verse.id)
312
+ ids = partners.get(word.id, [])
313
+ if ids:
314
+ entry["aligned"] += 1
315
+ if lemmas := equivalent(ids, other_words):
316
+ entry["eq"][lemmas] += 1
317
+ return sorted(
318
+ (
319
+ {
320
+ "lemma": lemma,
321
+ "upos": entry["upos"].most_common(1)[0][0],
322
+ "count": entry["count"],
323
+ "aligned": entry["aligned"],
324
+ "in_aligned": entry["in_aligned"],
325
+ "verses": len(entry["verses"]),
326
+ "forms": len(entry["forms"]),
327
+ # [label, count] or, for several linked words, [label, count, [lemma, ...]]
328
+ "eq": [[" ".join(lemmas), n] + ([list(lemmas)] if len(lemmas) > 1 else [])
329
+ for lemmas, n in entry["eq"].most_common(top)],
330
+ "eq_total": len(entry["eq"]),
331
+ }
332
+ for lemma, entry in entries.items()
333
+ ),
334
+ key=lambda e: (-e["count"], e["lemma"]),
335
+ )
336
+
337
+
338
+ def review(dataset: Dataset, limit: int = 300) -> dict:
339
+ """ The links and words most worth checking: least confident first. """
340
+ links, words = [], []
341
+ for index, verse in enumerate(dataset.verses):
342
+ words_a = {w.id: w for w in verse.a.words}
343
+ words_b = {w.id: w for w in verse.b.words}
344
+ for link in verse.links:
345
+ if link.confidence is None and link.sure:
346
+ continue
347
+ links.append({
348
+ "v": index, "a": link.source, "b": link.target, "sure": link.sure,
349
+ "confidence": link.confidence, "agent": link.agent,
350
+ "a_form": words_a[link.source].form, "b_form": words_b[link.target].form,
351
+ })
352
+ for side, sentence in (("a", verse.a), ("b", verse.b)):
353
+ for word in sentence.words:
354
+ if word.upos == "PUNCT":
355
+ continue
356
+ confidence = word_confidence(word)
357
+ if confidence is not None and confidence < LOW_CONFIDENCE:
358
+ words.append({"v": index, "side": side, "id": word.id, "form": word.form, "lemma": word.lemma,
359
+ "upos": word.upos, "deprel": word.deprel, "confidence": confidence})
360
+ links.sort(key=lambda l: (l["sure"], l["confidence"] if l["confidence"] is not None else 1))
361
+ words.sort(key=lambda w: w["confidence"])
362
+ return {"links": links[:limit], "links_total": len(links), "words": words[:limit], "words_total": len(words)}
363
+
364
+
365
+ def corpus(dataset: Dataset) -> list[dict]:
366
+ """ Compact verse texts for pages that show examples: forms, ids and no-space-after ids per side. """
367
+ def side(sentence: Sentence):
368
+ return {"ids": [w.id for w in sentence.words], "forms": [w.form for w in sentence.words],
369
+ "ns": sentence.no_space_after}
370
+ return [{"id": v.id, "label": v.label, "a": side(v.a), "b": side(v.b)} for v in dataset.verses]