sci2audio 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sci2audio/__init__.py +3 -0
- sci2audio/app/__init__.py +110 -0
- sci2audio/app/audio.py +847 -0
- sci2audio/app/chapters.py +109 -0
- sci2audio/app/chunks.py +102 -0
- sci2audio/app/config.py +136 -0
- sci2audio/app/estimate.py +192 -0
- sci2audio/app/files.py +98 -0
- sci2audio/app/job.py +285 -0
- sci2audio/app/lock.py +92 -0
- sci2audio/app/report.py +400 -0
- sci2audio/app/runner.py +92 -0
- sci2audio/app/script.py +95 -0
- sci2audio/app/settings.py +58 -0
- sci2audio/app/voice.py +70 -0
- sci2audio/cli.py +1013 -0
- sci2audio/core/__init__.py +81 -0
- sci2audio/core/audio.py +39 -0
- sci2audio/core/content.py +266 -0
- sci2audio/core/document.py +225 -0
- sci2audio/core/edits.py +121 -0
- sci2audio/core/export.py +76 -0
- sci2audio/extraction/__init__.py +28 -0
- sci2audio/extraction/cover.py +68 -0
- sci2audio/extraction/epub.py +576 -0
- sci2audio/extraction/errors.py +7 -0
- sci2audio/extraction/markdown.py +250 -0
- sci2audio/extraction/markup.py +356 -0
- sci2audio/extraction/pdf.py +623 -0
- sci2audio/extraction/text.py +136 -0
- sci2audio/layout/__init__.py +13 -0
- sci2audio/layout/build.py +265 -0
- sci2audio/layout/captions.py +77 -0
- sci2audio/layout/footnotes.py +180 -0
- sci2audio/layout/formulas.py +392 -0
- sci2audio/layout/furniture.py +326 -0
- sci2audio/layout/geometry.py +124 -0
- sci2audio/layout/graphics.py +569 -0
- sci2audio/layout/headings.py +358 -0
- sci2audio/layout/lists.py +167 -0
- sci2audio/layout/maths.py +47 -0
- sci2audio/layout/order.py +374 -0
- sci2audio/layout/paragraphs.py +518 -0
- sci2audio/layout/settings.py +142 -0
- sci2audio/layout/split.py +113 -0
- sci2audio/layout/tagged.py +442 -0
- sci2audio/output/__init__.py +44 -0
- sci2audio/output/ffmpeg.py +111 -0
- sci2audio/output/model.py +125 -0
- sci2audio/output/names.py +70 -0
- sci2audio/output/readalong.py +160 -0
- sci2audio/output/timeline.py +113 -0
- sci2audio/output/wav.py +83 -0
- sci2audio/output/write.py +280 -0
- sci2audio/testing/__init__.py +32 -0
- sci2audio/testing/documents.py +124 -0
- sci2audio/testing/regression.py +197 -0
- sci2audio/testing/text_steps.py +262 -0
- sci2audio/testing/voices.py +164 -0
- sci2audio/textprep/__init__.py +105 -0
- sci2audio/textprep/abbreviations.py +138 -0
- sci2audio/textprep/characters.py +267 -0
- sci2audio/textprep/citations.py +683 -0
- sci2audio/textprep/data/en/abbreviations.toml +209 -0
- sci2audio/textprep/data/en/units.toml +548 -0
- sci2audio/textprep/footnotes.py +418 -0
- sci2audio/textprep/furniture.py +46 -0
- sci2audio/textprep/hyphens.py +228 -0
- sci2audio/textprep/links.py +628 -0
- sci2audio/textprep/notation.py +170 -0
- sci2audio/textprep/numbers.py +529 -0
- sci2audio/textprep/placeholders.py +68 -0
- sci2audio/textprep/references.py +107 -0
- sci2audio/textprep/registry.py +83 -0
- sci2audio/textprep/runner.py +206 -0
- sci2audio/textprep/spoken.py +33 -0
- sci2audio/textprep/statistics.py +322 -0
- sci2audio/textprep/steps.py +153 -0
- sci2audio/textprep/units.py +449 -0
- sci2audio/textprep/utterances.py +315 -0
- sci2audio/textprep/wording.py +93 -0
- sci2audio/textprep/wordlists.py +189 -0
- sci2audio/voices/__init__.py +102 -0
- sci2audio/voices/download.py +340 -0
- sci2audio/voices/engine.py +136 -0
- sci2audio/voices/errors.py +88 -0
- sci2audio/voices/kokoro_engine.py +346 -0
- sci2audio/voices/runtime.py +37 -0
- sci2audio/voices/types.py +206 -0
- sci2audio/voices/wav.py +82 -0
- sci2audio/web/__init__.py +13 -0
- sci2audio/web/api.py +658 -0
- sci2audio/web/config.py +97 -0
- sci2audio/web/documents.py +496 -0
- sci2audio/web/i18n.py +188 -0
- sci2audio/web/jobs.py +265 -0
- sci2audio/web/launch.py +121 -0
- sci2audio/web/limits.py +64 -0
- sci2audio/web/models.py +178 -0
- sci2audio/web/pages.py +509 -0
- sci2audio/web/player.py +549 -0
- sci2audio/web/security.py +194 -0
- sci2audio/web/server.py +116 -0
- sci2audio/web/state.py +88 -0
- sci2audio/web/static/app.css +150 -0
- sci2audio/web/static/app.js +499 -0
- sci2audio/web/static/player.js +277 -0
- sci2audio/web/strings/de.toml +348 -0
- sci2audio/web/strings/en.toml +343 -0
- sci2audio/web/tasks.py +280 -0
- sci2audio/web/templates/_remove.html +17 -0
- sci2audio/web/templates/base.html +32 -0
- sci2audio/web/templates/document.html +153 -0
- sci2audio/web/templates/index.html +41 -0
- sci2audio/web/templates/job.html +66 -0
- sci2audio/web/templates/listen.html +75 -0
- sci2audio/web/templates/missing.html +7 -0
- sci2audio/web/templates/privacy.html +47 -0
- sci2audio/web/templates/voice.html +31 -0
- sci2audio-1.0.0.dist-info/METADATA +171 -0
- sci2audio-1.0.0.dist-info/RECORD +125 -0
- sci2audio-1.0.0.dist-info/WHEEL +4 -0
- sci2audio-1.0.0.dist-info/entry_points.txt +2 -0
- sci2audio-1.0.0.dist-info/licenses/LICENSE +674 -0
- sci2audio-1.0.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +86 -0
|
@@ -0,0 +1,529 @@
|
|
|
1
|
+
# ruff: noqa: RUF001, RUF002, RUF003
|
|
2
|
+
# (minus signs, dashes, primes and math letters are what these rules are about)
|
|
3
|
+
"""Numbers the voice reads wrong (TP-09, TP-13, #67): ranges, minus signs, counts, powers.
|
|
4
|
+
|
|
5
|
+
misaki, the voice's English front end, reads ordinary numbers, years and dates well and
|
|
6
|
+
they are left to it (D-54, D-176). This step proposes `replace` edits only where it reads a
|
|
7
|
+
number wrong; digits stay digits for the voice wherever it can read them, and each reason
|
|
8
|
+
quotes the printed text.
|
|
9
|
+
|
|
10
|
+
- N1: a minus sign (U+2212) before a number read "minus"; misaki loses the point of "−.32".
|
|
11
|
+
- N2 (D-183): ranges read "to": "3–5", "1.2–3.4", "2010–15", and "18-25" with a hyphen
|
|
12
|
+
between plain numbers only.
|
|
13
|
+
- N3 (D-182): misaki reads every bare four-digit number as a year; one that the context
|
|
14
|
+
marks as a count gets a thousands comma ("1234 participants" → "1,234").
|
|
15
|
+
- N4 (D-184): trailing zeros kept ("2.50" → "2 point 5 0"); misaki drops them.
|
|
16
|
+
- N5, N6 (D-187): powers: "10⁶", "10^-4", "× 10−4", "x²".
|
|
17
|
+
- N7, N8 (D-186): ratios "3:1" read "3 to 1"; fractions "1/2" read "1 half", "17/20" read
|
|
18
|
+
"17 out of 20".
|
|
19
|
+
- N9 (D-188): ISO dates read "12 March 2019".
|
|
20
|
+
- N10 (D-189): digits grouped by thin or no-break spaces ("50 000") read as one number.
|
|
21
|
+
- R1 (D-185): Roman numerals after "Phase", "Type", "War" and similar read as numbers.
|
|
22
|
+
|
|
23
|
+
Units, %, ≈, ×, ° and µ are the units step's (#68, D-178).
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import calendar
|
|
29
|
+
import re
|
|
30
|
+
|
|
31
|
+
from sci2audio.core import Document, Edit
|
|
32
|
+
from sci2audio.textprep.notation import (
|
|
33
|
+
SKIPPED_KINDS,
|
|
34
|
+
Candidate,
|
|
35
|
+
fold,
|
|
36
|
+
pick,
|
|
37
|
+
through_unread,
|
|
38
|
+
to_edits,
|
|
39
|
+
)
|
|
40
|
+
from sci2audio.textprep.steps import Category, Stage, StepContext, TextStep
|
|
41
|
+
|
|
42
|
+
_UNSIGNED = r"\d+(?:\.\d+)?"
|
|
43
|
+
|
|
44
|
+
# --- N1 minus sign ---------------------------------------------------------------------------
|
|
45
|
+
# U+2212 minus, and U+2012 figure dash where no number stands before it (between two numbers
|
|
46
|
+
# it is a range dash, N2); the ASCII hyphen-minus is read right by the voice
|
|
47
|
+
_MINUS = re.compile(r"(?<![\w.,])(?:\u2212|(?<!\d)(?<!\d\s)\u2012)(?P<space> ?)(?=\.?\d)")
|
|
48
|
+
|
|
49
|
+
# --- N2 ranges ------------------------------------------------------------------------------
|
|
50
|
+
# an en dash or figure dash, spaces allowed around it, or an em dash without spaces
|
|
51
|
+
_DASH_RANGE = re.compile(r"(?<=\d)(?P<dash>\s*[\u2013\u2012]\s*|\u2014)(?=[\u2212-]?\.?\d)")
|
|
52
|
+
# a hyphen between two plain numbers that are not part of a longer chain
|
|
53
|
+
_HYPHEN_RANGE = re.compile(
|
|
54
|
+
r"(?<![\w.,/:-])(?<!-\n)(?P<a>\d{1,4}(?:\.\d+)?)(?P<dash>-\n?)(?P<b>\d{1,4}(?:\.\d+)?)"
|
|
55
|
+
r"(?![\w/:-]|[.,]\d)"
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
# --- N3 four-digit counts -------------------------------------------------------------------
|
|
59
|
+
_FOUR_DIGITS = re.compile(r"(?<![\w.,/])[1-9]\d{3}(?![\w]|[.,]\d|\s+point\s+\d)")
|
|
60
|
+
_AFTER_OPERATOR = re.compile(r"[=<>\u2264\u2265\u2248~]\s*\Z")
|
|
61
|
+
_NEXT_WORD = re.compile(r"\s+(?P<word>[a-z][a-z'-]*)")
|
|
62
|
+
_PREVIOUS_WORD = re.compile(r"(?P<word>[A-Za-z]+)\.?\s+\Z")
|
|
63
|
+
# words ending in "s" that are not plural nouns: "ACL 2023 is", "in 2019 was"
|
|
64
|
+
_NOT_PLURAL = frozenset(
|
|
65
|
+
{"is", "was", "has", "his", "its", "this", "thus", "as", "yes", "does", "goes", "whereas"}
|
|
66
|
+
| {"versus", "plus", "minus", "less", "unless", "always", "perhaps", "towards", "besides"}
|
|
67
|
+
)
|
|
68
|
+
_PLURALS = frozenset({"people", "children", "men", "women", "mice", "data", "persons"})
|
|
69
|
+
_MONTHS = {name.lower() for name in calendar.month_name[1:] + calendar.month_abbr[1:]}
|
|
70
|
+
_YEAR_WORDS = frozenset(
|
|
71
|
+
{"in", "since", "from", "until", "till", "to", "by", "during", "of", "before", "after"}
|
|
72
|
+
| {"around", "circa", "c", "ca", "the", "this", "that", "these", "those"}
|
|
73
|
+
| {"early", "mid", "late", "year", "years", "spring", "summer", "autumn", "fall"}
|
|
74
|
+
| {"winter", "sept"}
|
|
75
|
+
| _MONTHS
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
# --- N4 trailing zeros -----------------------------------------------------------------------
|
|
79
|
+
# at most four digits after the point, not inside an identifier such as "arXiv:2401.17620",
|
|
80
|
+
# and not over a slash ("2.50/3" has no fraction reading, so it is left as printed) or in an
|
|
81
|
+
# exponent ("10^22.50", "10−42.50")
|
|
82
|
+
_TRAILING_ZERO = re.compile(
|
|
83
|
+
r"(?<![\w.,/^])(?<![^\W\d_]:)(?<!\d[\u2212-])(?P<whole>\d{1,3}(?:,\d{3})*|\d+)\.(?P<fraction>\d{0,3}0)"
|
|
84
|
+
r"(?![\w]|[.,/]\d)"
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
# --- N5, N6 powers ---------------------------------------------------------------------------
|
|
88
|
+
_SUPERSCRIPTS = "⁰¹²³⁴⁵⁶⁷⁸⁹"
|
|
89
|
+
_SUPER_DIGIT = {char: str(n) for n, char in enumerate(_SUPERSCRIPTS)}
|
|
90
|
+
# a power ends where its superscripts end: "10⁶⁻⁴" is no power the voice could say
|
|
91
|
+
_SUPER = f"[⁺⁻]?[{_SUPERSCRIPTS}]+(?![⁺⁻^])"
|
|
92
|
+
_POWER_OF_TEN = re.compile(r"(?<![\w.,^])(?P<base>10)(?P<exp>" + _SUPER + ")")
|
|
93
|
+
_SIGNED_POWER = re.compile(
|
|
94
|
+
r"(?<![\w.,^])(?P<base>\d+(?:\.\d+)?)(?P<exp>[⁺⁻][" + _SUPERSCRIPTS + "]+)(?![⁺⁻^])"
|
|
95
|
+
)
|
|
96
|
+
_CARET = re.compile(
|
|
97
|
+
r"(?<![\w.,])(?P<base>\d+(?:\.\d+)?|[^\W\d_" + _SUPERSCRIPTS + r"])"
|
|
98
|
+
r"(?P<exp>\^(?:\{(?P<braced>[+\u2212-]?\d+)\}|\((?P<paren>[+\u2212-]?\d+)\)|(?P<bare>[+\u2212-]?\d+)))"
|
|
99
|
+
r"(?![\w.⁺⁻])"
|
|
100
|
+
)
|
|
101
|
+
_TIMES_TEN = re.compile(r"(?<=\d)\s*[×x]\s*(?P<base>10)(?P<exp>[\u2212-]\d{1,3})(?![\w.,⁺⁻])")
|
|
102
|
+
# not after a number and a space ("5 m²" is a unit, the units step's); a caret exponent before
|
|
103
|
+
# the space is no such number, since the caret rule may turn it into a word ("x^2 x⁶")
|
|
104
|
+
_LETTER_POWER = re.compile(
|
|
105
|
+
r"(?<![\w/])(?<!(?<!\^)\d )(?<![.,] )(?P<base>[^\W\d_"
|
|
106
|
+
+ _SUPERSCRIPTS
|
|
107
|
+
+ r"])(?P<exp>"
|
|
108
|
+
+ _SUPER
|
|
109
|
+
+ r")"
|
|
110
|
+
r"(?![\w" + _SUPERSCRIPTS + "])"
|
|
111
|
+
)
|
|
112
|
+
_SQUARED_STATISTICS = frozenset("Rrηωεχ") # read by the statistics step ("R squared")
|
|
113
|
+
# a unit chain after a number ("2 m s⁻¹", "3 W m⁻²"): the units step reads it (#68, D-178)
|
|
114
|
+
# (the number is no caret exponent, and a unit is letters, not superscript digits: "x^2 x⁶x⁶")
|
|
115
|
+
_UNIT_CHAIN = re.compile(
|
|
116
|
+
r"(?<![\w^.])\d+(?:\.\d+)?\s*(?:[^\W\d_"
|
|
117
|
+
+ _SUPERSCRIPTS
|
|
118
|
+
+ r"]{1,3}["
|
|
119
|
+
+ _SUPERSCRIPTS
|
|
120
|
+
+ r"⁺⁻]*[\s·⋅]+)+\Z"
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
# --- N7 ratios, N8 fractions -----------------------------------------------------------------
|
|
124
|
+
# a ratio not after a hyphen: "10-23:3" is a hyphenated pair, left as printed
|
|
125
|
+
_RATIO = re.compile(
|
|
126
|
+
r"(?<![\w.:/,-])(?<!-\n)(?P<a>" + _UNSIGNED + r"):(?P<b>" + _UNSIGNED + r")(?![\w:/^]|[.,]\d)"
|
|
127
|
+
)
|
|
128
|
+
_TIME_BEFORE = re.compile(
|
|
129
|
+
r"(?i:\b(?:at|from|until|till|by|before|after|around|between|and|to)\s+)\Z"
|
|
130
|
+
)
|
|
131
|
+
_TIME_AFTER = re.compile(r"\s*(?:[ap]\.?m\b\.?|h\b|hrs?\b|hours?\b|o'clock)", re.IGNORECASE)
|
|
132
|
+
# a range after "8:64" makes it a volume and its pages; "to" is how this step reads that range
|
|
133
|
+
_RANGE_AFTER = re.compile(r"\s*(?:[\u2013\u2012\u2014-]|\s+to\s)\s*\d")
|
|
134
|
+
# not before a signed power, colon or hyphen ("3/3⁻⁴", "3/33:3", "3/33-53": no plain denominator)
|
|
135
|
+
_FRACTION = re.compile(
|
|
136
|
+
r"(?<![\w.,/:\u2212-])(?<!-\n)(?P<n>\d{1,3})/(?P<d>\d{1,3})(?![\w/:⁺⁻^-]|[.,]\d)"
|
|
137
|
+
)
|
|
138
|
+
_FRACTION_WORDS = {
|
|
139
|
+
2: ("half", "halves"),
|
|
140
|
+
3: ("third", "thirds"),
|
|
141
|
+
4: ("quarter", "quarters"),
|
|
142
|
+
5: ("fifth", "fifths"),
|
|
143
|
+
6: ("sixth", "sixths"),
|
|
144
|
+
7: ("seventh", "sevenths"),
|
|
145
|
+
8: ("eighth", "eighths"),
|
|
146
|
+
9: ("ninth", "ninths"),
|
|
147
|
+
10: ("tenth", "tenths"),
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
# --- N9 ISO dates ----------------------------------------------------------------------------
|
|
151
|
+
_ISO_DATE = re.compile(
|
|
152
|
+
r"(?<![\w.:/\u2013-])(?P<y>1[5-9]\d\d|20\d\d)-\n?(?P<m>0[1-9]|1[0-2])-\n?(?P<d>0[1-9]|[12]\d|3[01])"
|
|
153
|
+
r"(?![\w⁺⁻^-]|[.,]\d)" # not before a power: "2019-03-12⁻⁴" is no date
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
# --- N10 grouped digits ----------------------------------------------------------------------
|
|
157
|
+
_GROUPED = re.compile(
|
|
158
|
+
r"(?<![\w.,])\d{1,3}(?:[ \u00a0\u2009\u202f]\d{3})+(?![\w]|[.,]\d|[ \u00a0\u2009\u202f]\d)"
|
|
159
|
+
)
|
|
160
|
+
_NARROW = re.compile(r"[\u00a0\u2009\u202f]")
|
|
161
|
+
|
|
162
|
+
# --- R1 Roman numerals -----------------------------------------------------------------------
|
|
163
|
+
_KEYWORDS = frozenset(
|
|
164
|
+
{"war", "phase", "type", "class", "stage", "grade", "level", "tier", "part", "chapter"}
|
|
165
|
+
| {"volume", "book", "section", "appendix", "study", "experiment", "trial", "table"}
|
|
166
|
+
| {"figure"}
|
|
167
|
+
)
|
|
168
|
+
_PLURAL_KEYWORDS = {"studies": "study", "classes": "class", "wars": "war"}
|
|
169
|
+
# keywords printed short with a full stop, as the abbreviations step (#68) reads them in full:
|
|
170
|
+
# "Fig. II" must read "Figure 2" on the first run, not "Figure II" (review of PR #132)
|
|
171
|
+
_ABBREVIATED_KEYWORDS = {
|
|
172
|
+
"fig": "figure",
|
|
173
|
+
"figs": "figure",
|
|
174
|
+
"tab": "table",
|
|
175
|
+
"tabs": "table",
|
|
176
|
+
"sec": "section",
|
|
177
|
+
"secs": "section",
|
|
178
|
+
"sect": "section",
|
|
179
|
+
"sects": "section",
|
|
180
|
+
"ch": "chapter",
|
|
181
|
+
"chap": "chapter",
|
|
182
|
+
"exp": "experiment",
|
|
183
|
+
"app": "appendix",
|
|
184
|
+
}
|
|
185
|
+
_ROMAN = r"[IVX]{1,6}"
|
|
186
|
+
_ROMAN_RUN = re.compile(
|
|
187
|
+
r"\b(?P<keyword>[A-Za-z]+)(?P<dot>\.)?(?(dot)\s*|\s+)(?P<run>"
|
|
188
|
+
+ _ROMAN
|
|
189
|
+
+ r"(?:(?:\s*[/\u2013-]\s*|,\s*|\s+(?:and|or|&)\s+)"
|
|
190
|
+
+ _ROMAN
|
|
191
|
+
# a power after it is the powers rule's: "Phase II⁻⁴", "Phase I²" (later "I squared")
|
|
192
|
+
+ r")*)(?![\w⁺⁻]|\s+(?:squared|cubed|to the power of)\b)"
|
|
193
|
+
)
|
|
194
|
+
_ROMAN_PART = re.compile(r"(?P<numeral>" + _ROMAN + r")|(?P<sep>\s*[/\u2013-]\s*)")
|
|
195
|
+
_VALID_ROMAN = re.compile(r"\A(?P<tens>X{0,3})(?P<units>IX|IV|V?I{0,3})\Z")
|
|
196
|
+
_UNITS = {"": 0, "I": 1, "II": 2, "III": 3, "IV": 4, "V": 5, "VI": 6, "VII": 7, "VIII": 8, "IX": 9}
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
class NumbersStep(TextStep):
|
|
200
|
+
"""Read the numbers the voice gets wrong; one edit per number or sign."""
|
|
201
|
+
|
|
202
|
+
id = "numbers"
|
|
203
|
+
languages = ("en",)
|
|
204
|
+
category = Category.NUMBERS_AND_UNITS
|
|
205
|
+
stage = Stage.INLINE
|
|
206
|
+
|
|
207
|
+
def find(self, document: Document, context: StepContext) -> list[Edit]:
|
|
208
|
+
edits = []
|
|
209
|
+
for index, block in enumerate(document.blocks):
|
|
210
|
+
if block.kind in SKIPPED_KINDS or not block.text:
|
|
211
|
+
continue
|
|
212
|
+
edits += to_edits(index, through_unread(block.text, find_numbers), self.id)
|
|
213
|
+
return edits
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def find_numbers(text: str) -> list[Candidate]:
|
|
217
|
+
"""The number edits for one block's text, without overlap, in text order.
|
|
218
|
+
|
|
219
|
+
Rules come most specific first, so a date is not also read as a range and a power's
|
|
220
|
+
minus is not also read as a minus sign.
|
|
221
|
+
"""
|
|
222
|
+
plain = fold(text)
|
|
223
|
+
return pick(
|
|
224
|
+
[
|
|
225
|
+
*_iso_dates(text),
|
|
226
|
+
*_unjoined(text, _roman_numerals(plain, text)),
|
|
227
|
+
*_grouped_digits(text),
|
|
228
|
+
*_unjoined(text, _powers(text, plain)),
|
|
229
|
+
*_minus_signs(text),
|
|
230
|
+
*_ranges(text),
|
|
231
|
+
*_unjoined(text, _trailing_zeros(text)),
|
|
232
|
+
*_counts(text),
|
|
233
|
+
*_ratios(text),
|
|
234
|
+
*_fractions(text),
|
|
235
|
+
]
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
# a number joined to the next or the last one by a dash, colon or slash, by a space, or by
|
|
240
|
+
# the "to" or "minus" this step reads a range, ratio or figure dash with, or by nothing at
|
|
241
|
+
# all ("10⁶3")
|
|
242
|
+
_JOINED_AFTER = re.compile(r"\s*[\u2013\u2014\u2012:/-]\s*\.?\d|\.?\d|\s\d|\s+(?:to|minus)\s+\.?\d")
|
|
243
|
+
_GLUED_MINUS = re.compile(r"\d\u2212\Z")
|
|
244
|
+
_JOINED_BEFORE = re.compile(r"\d\s*(?:[\u2013\u2014\u2012:/-]|\sto\s)\s*\Z|\d\s\Z")
|
|
245
|
+
# every look back before a match reads at most this many characters, so a long block takes
|
|
246
|
+
# linear time (review of PR #132); what the rules look for before a number is a few words
|
|
247
|
+
WINDOW = 64
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _before(text: str, at: int) -> str:
|
|
251
|
+
"""The text just before `at`: at most `WINDOW` characters."""
|
|
252
|
+
return text[max(0, at - WINDOW) : at]
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _joined_before(text: str, at: int) -> bool:
|
|
256
|
+
return bool(_JOINED_BEFORE.search(text, max(0, at - WINDOW), at))
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _unjoined(text: str, candidates: list[Candidate]) -> list[Candidate]:
|
|
260
|
+
"""The candidates whose number stands on its own.
|
|
261
|
+
|
|
262
|
+
These rules write digits next to words ("2 point 5 0", "Phase 3", "of 4"). Where the
|
|
263
|
+
printed number is joined to another one ("3:2.50", "Phase I 395", "10⁻⁴–5"), that new
|
|
264
|
+
reading would join differently on a second run, so such a number is left as printed:
|
|
265
|
+
the step proposes nothing that a second run would change again.
|
|
266
|
+
"""
|
|
267
|
+
return [
|
|
268
|
+
c
|
|
269
|
+
for c in candidates
|
|
270
|
+
if not _JOINED_AFTER.match(text, c.end) and not _joined_before(text, c.start)
|
|
271
|
+
]
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _iso_dates(text: str) -> list[Candidate]:
|
|
275
|
+
found = []
|
|
276
|
+
for match in _ISO_DATE.finditer(text):
|
|
277
|
+
year, month, day = int(match["y"]), int(match["m"]), int(match["d"])
|
|
278
|
+
if day > calendar.monthrange(year, month)[1]:
|
|
279
|
+
continue
|
|
280
|
+
words = f"{day} {calendar.month_name[month]} {year}"
|
|
281
|
+
found.append(Candidate(*match.span(), words, f'date "{match.group()}" read as "{words}"'))
|
|
282
|
+
return found
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _roman_value(numeral: str) -> int:
|
|
286
|
+
match = _VALID_ROMAN.match(numeral)
|
|
287
|
+
if not match or not numeral:
|
|
288
|
+
return 0
|
|
289
|
+
return 10 * len(match["tens"]) + _UNITS[match["units"]]
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _roman_numerals(plain: str, text: str) -> list[Candidate]:
|
|
293
|
+
found = []
|
|
294
|
+
for match in _ROMAN_RUN.finditer(plain):
|
|
295
|
+
keyword = match["keyword"]
|
|
296
|
+
if match["dot"]:
|
|
297
|
+
if keyword.lower() not in _ABBREVIATED_KEYWORDS:
|
|
298
|
+
continue # "the end. II": a sentence end, not a short keyword
|
|
299
|
+
keyword += "."
|
|
300
|
+
else:
|
|
301
|
+
base = keyword.lower()
|
|
302
|
+
base = _PLURAL_KEYWORDS.get(base, base[:-1] if base.endswith("s") else base)
|
|
303
|
+
if keyword.lower() not in _KEYWORDS and base not in _KEYWORDS:
|
|
304
|
+
continue
|
|
305
|
+
offset = match.start("run")
|
|
306
|
+
parts = list(_ROMAN_PART.finditer(match["run"]))
|
|
307
|
+
values = [_roman_value(p["numeral"]) for p in parts if p["numeral"]]
|
|
308
|
+
if not all(values):
|
|
309
|
+
continue # "Type IIX": not a numeral, so nothing in the run is read as one
|
|
310
|
+
if match["run"] == "I" and not keyword[0].isupper():
|
|
311
|
+
continue # "the part I think": a lone "I" after a lowercase word is the pronoun
|
|
312
|
+
for part in parts:
|
|
313
|
+
start, end = offset + part.start(), offset + part.end()
|
|
314
|
+
if part["numeral"]:
|
|
315
|
+
value = str(_roman_value(part["numeral"]))
|
|
316
|
+
reason = f'Roman numeral "{part["numeral"]}" after "{keyword}" read as "{value}"'
|
|
317
|
+
found.append(Candidate(start, end, value, reason))
|
|
318
|
+
else:
|
|
319
|
+
sep = part["sep"].strip()
|
|
320
|
+
words = " and " if sep == "/" else " to "
|
|
321
|
+
reason = f'"{sep}" between Roman numerals read as "{words.strip()}"'
|
|
322
|
+
found.append(Candidate(start, end, words, reason))
|
|
323
|
+
return found
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def _grouped_digits(text: str) -> list[Candidate]:
|
|
327
|
+
found = []
|
|
328
|
+
for match in _GROUPED.finditer(text):
|
|
329
|
+
number = match.group()
|
|
330
|
+
groups = re.split(r"[ \u00a0\u2009\u202f]", number)
|
|
331
|
+
if not _NARROW.search(number) and len(groups) < 3:
|
|
332
|
+
continue # "Table 2 000" could be two numbers; one plain-space group is not enough
|
|
333
|
+
if _NARROW.search(number) and " " in number:
|
|
334
|
+
continue # mixed separators: not one grouped number
|
|
335
|
+
words = ",".join(groups)
|
|
336
|
+
found.append(
|
|
337
|
+
Candidate(*match.span(), words, f'digits grouped as "{number}" read as "{words}"')
|
|
338
|
+
)
|
|
339
|
+
return found
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _exponent_words(exponent: str) -> str:
|
|
343
|
+
digits = "".join(_SUPER_DIGIT.get(c, c) for c in exponent)
|
|
344
|
+
sign = ""
|
|
345
|
+
if digits[:1] in "⁻\u2212-":
|
|
346
|
+
sign, digits = "minus ", digits[1:]
|
|
347
|
+
elif digits[:1] in "⁺+":
|
|
348
|
+
digits = digits[1:]
|
|
349
|
+
return f"{sign}{digits}"
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def _powers(text: str, plain: str) -> list[Candidate]:
|
|
353
|
+
found: list[tuple[Candidate, int | None]] = []
|
|
354
|
+
for pattern in (_POWER_OF_TEN, _SIGNED_POWER, _TIMES_TEN):
|
|
355
|
+
for match in pattern.finditer(text):
|
|
356
|
+
start, end = match.span("exp")
|
|
357
|
+
words = f" to the power of {_exponent_words(match['exp'])}"
|
|
358
|
+
found.append(
|
|
359
|
+
(
|
|
360
|
+
Candidate(start, end, words, _power_reason(text, match, words)),
|
|
361
|
+
match.start("base"),
|
|
362
|
+
)
|
|
363
|
+
)
|
|
364
|
+
for match in _CARET.finditer(plain):
|
|
365
|
+
start, end = match.span("exp")
|
|
366
|
+
exponent = match["braced"] or match["paren"] or match["bare"]
|
|
367
|
+
words = _letter_power(exponent) if match["base"].isalpha() else None
|
|
368
|
+
words = words or f" to the power of {_exponent_words(exponent)}"
|
|
369
|
+
base = None if match["base"].isalpha() else match.start("base")
|
|
370
|
+
found.append((Candidate(start, end, words, _power_reason(text, match, words)), base))
|
|
371
|
+
for match in _LETTER_POWER.finditer(plain):
|
|
372
|
+
if match["base"] in _SQUARED_STATISTICS and match["exp"] == "²":
|
|
373
|
+
continue
|
|
374
|
+
if _UNIT_CHAIN.search(_before(plain, match.start("base"))):
|
|
375
|
+
continue
|
|
376
|
+
start, end = match.span("exp")
|
|
377
|
+
exponent = "".join(_SUPER_DIGIT.get(c, c) for c in match["exp"])
|
|
378
|
+
words = _letter_power(exponent) or f" to the power of {_exponent_words(match['exp'])}"
|
|
379
|
+
found.append((Candidate(start, end, words, _power_reason(text, match, words)), None))
|
|
380
|
+
# the base is the number whose reading changes: "3:10²" is left as printed (`_unjoined`);
|
|
381
|
+
# a letter base (None) is read the same whatever stands before it. A minus sign glued to
|
|
382
|
+
# a number before the base joins it too ("10−4⁻⁴" would read "10−4" as a power next time),
|
|
383
|
+
# but a spaced one is read as "minus" in the same run ("5 − 10⁶").
|
|
384
|
+
return [
|
|
385
|
+
c
|
|
386
|
+
for c, base in found
|
|
387
|
+
if base is None
|
|
388
|
+
or not (_joined_before(text, base) or _GLUED_MINUS.match(text, max(0, base - 2), base))
|
|
389
|
+
]
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def _letter_power(exponent: str) -> str | None:
|
|
393
|
+
return {"2": " squared", "3": " cubed"}.get(exponent)
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _power_reason(text: str, match: re.Match[str], words: str) -> str:
|
|
397
|
+
printed = text[match.start("base") : match.end("exp")]
|
|
398
|
+
return f'power "{printed}" read with "{words.strip()}"'
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def _minus_signs(text: str) -> list[Candidate]:
|
|
402
|
+
# the reason quotes the sign as printed: the minus sign or a figure dash
|
|
403
|
+
return [
|
|
404
|
+
Candidate(m.start(), m.end(), "minus ", f'"{m[0].strip()}" before a number read as "minus"')
|
|
405
|
+
for m in _MINUS.finditer(text)
|
|
406
|
+
]
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def _ranges(text: str) -> list[Candidate]:
|
|
410
|
+
found = []
|
|
411
|
+
for match in _DASH_RANGE.finditer(text):
|
|
412
|
+
start, end = match.span("dash")
|
|
413
|
+
found.append(Candidate(start, end, " to ", _range_reason(text, start, end)))
|
|
414
|
+
for match in _HYPHEN_RANGE.finditer(text):
|
|
415
|
+
if float(match["a"]) >= float(match["b"]):
|
|
416
|
+
continue # "3-1" is a score or a code, not a range
|
|
417
|
+
start, end = match.span("dash")
|
|
418
|
+
found.append(Candidate(start, end, " to ", _range_reason(text, start, end)))
|
|
419
|
+
return found
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
_LEFT_NUMBER = re.compile(r"[\d.,]*\d\Z")
|
|
423
|
+
_RIGHT_NUMBER = re.compile(r"[\u2212-]?[\d.,]*\d")
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _range_reason(text: str, start: int, end: int) -> str:
|
|
427
|
+
left = _LEFT_NUMBER.search(_before(text, start))
|
|
428
|
+
right = _RIGHT_NUMBER.match(text, end)
|
|
429
|
+
printed = (left.group() if left else "") + text[start:end] + (right.group() if right else "")
|
|
430
|
+
return f'range "{" ".join(printed.split())}" read with "to"'
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def _counts(text: str) -> list[Candidate]:
|
|
434
|
+
found = []
|
|
435
|
+
for match in _FOUR_DIGITS.finditer(text):
|
|
436
|
+
number = match.group()
|
|
437
|
+
value = int(number)
|
|
438
|
+
if value % 100 == 0:
|
|
439
|
+
continue # "1000", "1200": read right as a year and as a number
|
|
440
|
+
before = _before(text, match.start())
|
|
441
|
+
if not _AFTER_OPERATOR.search(before) and not _counted(text, match.end(), before, value):
|
|
442
|
+
continue
|
|
443
|
+
words = f"{number[0]},{number[1:]}"
|
|
444
|
+
found.append(Candidate(*match.span(), words, f'"{number}" read as a number, not a year'))
|
|
445
|
+
return found
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def _counted(text: str, end: int, before: str, value: int) -> bool:
|
|
449
|
+
"""True if a plural word follows and no word before makes the number a year."""
|
|
450
|
+
following = _NEXT_WORD.match(text, end)
|
|
451
|
+
if not following:
|
|
452
|
+
return False
|
|
453
|
+
word = following["word"]
|
|
454
|
+
plural = word in _PLURALS or (
|
|
455
|
+
word.endswith("s") and not word.endswith(("ss", "us", "'s")) and word not in _NOT_PLURAL
|
|
456
|
+
)
|
|
457
|
+
if not plural:
|
|
458
|
+
return False
|
|
459
|
+
if not 1500 <= value <= 2099:
|
|
460
|
+
return True
|
|
461
|
+
previous = _PREVIOUS_WORD.search(before)
|
|
462
|
+
# "in 2019 studies", "the 2019 guidelines", "ACL 2023 papers": the number is a year
|
|
463
|
+
return previous is None or not (
|
|
464
|
+
previous["word"].lower() in _YEAR_WORDS or previous["word"][0].isupper()
|
|
465
|
+
)
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def _trailing_zeros(text: str) -> list[Candidate]:
|
|
469
|
+
found = []
|
|
470
|
+
for match in _TRAILING_ZERO.finditer(text):
|
|
471
|
+
words = f"{match['whole']} point {' '.join(match['fraction'])}"
|
|
472
|
+
found.append(
|
|
473
|
+
Candidate(
|
|
474
|
+
*match.span(),
|
|
475
|
+
words,
|
|
476
|
+
f'"{match.group()}" read with its trailing zeros as "{words}"',
|
|
477
|
+
)
|
|
478
|
+
)
|
|
479
|
+
return found
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def _ratios(text: str) -> list[Candidate]:
|
|
483
|
+
found = []
|
|
484
|
+
for match in _RATIO.finditer(text):
|
|
485
|
+
if _RANGE_AFTER.match(text, match.end()):
|
|
486
|
+
continue # "8:64–77": a volume and its pages
|
|
487
|
+
if _is_time(text, match):
|
|
488
|
+
continue
|
|
489
|
+
start = match.start() + len(match["a"])
|
|
490
|
+
words = f"{match['a']} to {match['b']}"
|
|
491
|
+
found.append(
|
|
492
|
+
Candidate(start, start + 1, " to ", f'ratio "{match.group()}" read as "{words}"')
|
|
493
|
+
)
|
|
494
|
+
return found
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def _is_time(text: str, match: re.Match[str]) -> bool:
|
|
498
|
+
hours, minutes = match["a"], match["b"]
|
|
499
|
+
if not (hours.isdigit() and minutes.isdigit() and len(minutes) == 2):
|
|
500
|
+
return False
|
|
501
|
+
if int(hours) > 24 or int(minutes) > 59:
|
|
502
|
+
return False
|
|
503
|
+
return bool(
|
|
504
|
+
_TIME_BEFORE.search(_before(text, match.start())) or _TIME_AFTER.match(text, match.end())
|
|
505
|
+
)
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def _fractions(text: str) -> list[Candidate]:
|
|
509
|
+
found = []
|
|
510
|
+
for match in _FRACTION.finditer(text):
|
|
511
|
+
n, d = int(match["n"]), int(match["d"])
|
|
512
|
+
if n > d or d == 0:
|
|
513
|
+
continue # "24/7", "3/2": no reading that is right everywhere
|
|
514
|
+
start = match.start("n") + len(match["n"])
|
|
515
|
+
# a word for the denominator would part it from a number joined to it ("3/5 2.50")
|
|
516
|
+
if d in _FRACTION_WORDS and 0 < n < d and not _JOINED_AFTER.match(text, match.end()):
|
|
517
|
+
one, many = _FRACTION_WORDS[d]
|
|
518
|
+
words = f" {one if n == 1 else many}"
|
|
519
|
+
else:
|
|
520
|
+
words = " out of "
|
|
521
|
+
start, end = start, start + 1
|
|
522
|
+
found.append(
|
|
523
|
+
Candidate(start, end, words, f'"{match.group()}" read as "{n} out of {d}"')
|
|
524
|
+
)
|
|
525
|
+
continue
|
|
526
|
+
found.append(
|
|
527
|
+
Candidate(start, match.end(), words, f'fraction "{match.group()}" read as "{n}{words}"')
|
|
528
|
+
)
|
|
529
|
+
return found
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""Formulas, tables and figures are not read; a short notice says so (FX-01, FX-02, D-55).
|
|
2
|
+
|
|
3
|
+
Layout keeps each display formula, table and figure as a placeholder block (D-63), with its
|
|
4
|
+
caption as a block of its own that points at it (`describes`) and a formula's printed number
|
|
5
|
+
as its label. Each placeholder's whole text is replaced by one short notice that names it
|
|
6
|
+
(Q3, D-145): through the labels of the captions that describe it, as printed ("Table 2",
|
|
7
|
+
"Algorithm 1"; a short "Fig. 3" read "Figure 3"), else through a formula's own number
|
|
8
|
+
("Formula 16a"), else without a name ("A figure is skipped."). The caption is not repeated:
|
|
9
|
+
it reads as printed, in its printed place, and already starts with its label.
|
|
10
|
+
|
|
11
|
+
Formulas and tables-and-figures are separate categories (D-108), so they are two steps.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from sci2audio.core import BlockKind, Document, Edit, EditKind, Span
|
|
17
|
+
from sci2audio.textprep.steps import Category, Stage, StepContext, TextStep
|
|
18
|
+
from sci2audio.textprep.wording import wording_for
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class _NoticeStep(TextStep):
|
|
22
|
+
"""Replace each placeholder block of the given kinds by its notice."""
|
|
23
|
+
|
|
24
|
+
kinds: frozenset[BlockKind]
|
|
25
|
+
languages = ("en",)
|
|
26
|
+
stage = Stage.BLOCK
|
|
27
|
+
|
|
28
|
+
def find(self, document: Document, context: StepContext) -> list[Edit]:
|
|
29
|
+
wording = wording_for(context.language)
|
|
30
|
+
captions: dict[int, list[str]] = {}
|
|
31
|
+
for block in document.blocks:
|
|
32
|
+
if block.describes is not None and block.label:
|
|
33
|
+
captions.setdefault(block.describes, []).append(wording.label(block.label))
|
|
34
|
+
edits = []
|
|
35
|
+
for index, block in enumerate(document.blocks):
|
|
36
|
+
if block.kind not in self.kinds:
|
|
37
|
+
continue
|
|
38
|
+
names = captions.get(index, [])
|
|
39
|
+
if not names and block.kind is BlockKind.FORMULA and block.label:
|
|
40
|
+
names = [wording.formula.format(block.label)]
|
|
41
|
+
notice = wording.skipped(names, block.kind)
|
|
42
|
+
what = names[0] if len(names) == 1 else f"this {block.kind}"
|
|
43
|
+
edits.append(
|
|
44
|
+
Edit(
|
|
45
|
+
Span(index, 0, len(block.text)),
|
|
46
|
+
EditKind.REPLACE,
|
|
47
|
+
reason=f"{what} is not read; a short notice is read instead (D-55)",
|
|
48
|
+
step=self.id,
|
|
49
|
+
replacement=notice,
|
|
50
|
+
)
|
|
51
|
+
)
|
|
52
|
+
return edits
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class FormulaNoticeStep(_NoticeStep):
|
|
56
|
+
"""A display formula is read as "Formula 4 is skipped." (FX-01)."""
|
|
57
|
+
|
|
58
|
+
id = "formula-notices"
|
|
59
|
+
category = Category.FORMULAS
|
|
60
|
+
kinds = frozenset({BlockKind.FORMULA})
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class TableFigureNoticeStep(_NoticeStep):
|
|
64
|
+
"""A table or figure is read as "Table 2 is skipped." (FX-02)."""
|
|
65
|
+
|
|
66
|
+
id = "table-figure-notices"
|
|
67
|
+
category = Category.TABLES_AND_FIGURES
|
|
68
|
+
kinds = frozenset({BlockKind.TABLE, BlockKind.FIGURE})
|