calkit-python 0.50.2__py3-none-any.whl → 0.50.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- calkit/cli/latex.py +860 -38
- calkit/docx.py +616 -2
- calkit/latex.py +361 -10
- calkit/tests/test_docx.py +336 -8
- calkit/tests/test_latex.py +124 -0
- {calkit_python-0.50.2.dist-info → calkit_python-0.50.3.dist-info}/METADATA +1 -1
- {calkit_python-0.50.2.dist-info → calkit_python-0.50.3.dist-info}/RECORD +23 -23
- {calkit_python-0.50.2.data → calkit_python-0.50.3.data}/data/etc/jupyter/jupyter_server_config.d/calkit.json +0 -0
- {calkit_python-0.50.2.data → calkit_python-0.50.3.data}/data/share/jupyter/labextensions/calkit/package.json +0 -0
- {calkit_python-0.50.2.data → calkit_python-0.50.3.data}/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig +0 -0
- {calkit_python-0.50.2.data → calkit_python-0.50.3.data}/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json +0 -0
- {calkit_python-0.50.2.data → calkit_python-0.50.3.data}/data/share/jupyter/labextensions/calkit/static/506.c34b070098184e33.js +0 -0
- {calkit_python-0.50.2.data → calkit_python-0.50.3.data}/data/share/jupyter/labextensions/calkit/static/57d13a7399cec3c5.png +0 -0
- {calkit_python-0.50.2.data → calkit_python-0.50.3.data}/data/share/jupyter/labextensions/calkit/static/616.528427eb54a4a0c0.js +0 -0
- {calkit_python-0.50.2.data → calkit_python-0.50.3.data}/data/share/jupyter/labextensions/calkit/static/740.6bf87276e9e5788a.js +0 -0
- {calkit_python-0.50.2.data → calkit_python-0.50.3.data}/data/share/jupyter/labextensions/calkit/static/899.f9b9f9bf705a6493.js +0 -0
- {calkit_python-0.50.2.data → calkit_python-0.50.3.data}/data/share/jupyter/labextensions/calkit/static/935.46ecc6bf99aa593a.js +0 -0
- {calkit_python-0.50.2.data → calkit_python-0.50.3.data}/data/share/jupyter/labextensions/calkit/static/remoteEntry.109a3a8379365e59.js +0 -0
- {calkit_python-0.50.2.data → calkit_python-0.50.3.data}/data/share/jupyter/labextensions/calkit/static/style.js +0 -0
- {calkit_python-0.50.2.data → calkit_python-0.50.3.data}/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json +0 -0
- {calkit_python-0.50.2.dist-info → calkit_python-0.50.3.dist-info}/WHEEL +0 -0
- {calkit_python-0.50.2.dist-info → calkit_python-0.50.3.dist-info}/entry_points.txt +0 -0
- {calkit_python-0.50.2.dist-info → calkit_python-0.50.3.dist-info}/licenses/LICENSE +0 -0
calkit/cli/latex.py
CHANGED
|
@@ -11,7 +11,7 @@ import shutil
|
|
|
11
11
|
import string
|
|
12
12
|
import subprocess
|
|
13
13
|
import sys
|
|
14
|
-
from collections.abc import Callable
|
|
14
|
+
from collections.abc import Callable, Iterator
|
|
15
15
|
from copy import deepcopy
|
|
16
16
|
from pathlib import Path
|
|
17
17
|
|
|
@@ -1982,9 +1982,11 @@ def to_docx(
|
|
|
1982
1982
|
"--engine",
|
|
1983
1983
|
help=(
|
|
1984
1984
|
"What makes the Word copy: 'word', which imports the "
|
|
1985
|
-
"compiled PDF so the copy looks like it,
|
|
1985
|
+
"compiled PDF so the copy looks like it, 'libreoffice', "
|
|
1986
1986
|
"which converts the source with TeX4ht's make4ht and then "
|
|
1987
|
-
"LibreOffice
|
|
1987
|
+
"LibreOffice, or 'pandoc', experimental, which converts the "
|
|
1988
|
+
"source with Pandoc, with numbers and citations from the "
|
|
1989
|
+
"build. Defaults to Word where it's installed."
|
|
1988
1990
|
),
|
|
1989
1991
|
),
|
|
1990
1992
|
] = None,
|
|
@@ -2019,6 +2021,405 @@ def to_docx(
|
|
|
2019
2021
|
from calkit.models.docx import LatexDocxExport
|
|
2020
2022
|
from calkit.models.pipeline import LatexStage
|
|
2021
2023
|
|
|
2024
|
+
def restore_formulas(
|
|
2025
|
+
doc: calkit.docx.Document, blks: list[calkit.latex.Block]
|
|
2026
|
+
) -> None:
|
|
2027
|
+
# A chemical formula, e.g., \ce{C3H8}, that Word's import drew
|
|
2028
|
+
# partly as a picture, e.g., its subscripts, goes back as text
|
|
2029
|
+
# formatted from the source
|
|
2030
|
+
from xml.etree import ElementTree as ET
|
|
2031
|
+
|
|
2032
|
+
w_r = calkit.docx._tag(calkit.docx.W, "r")
|
|
2033
|
+
w_t = calkit.docx._tag(calkit.docx.W, "t")
|
|
2034
|
+
w_rpr = calkit.docx._tag(calkit.docx.W, "rPr")
|
|
2035
|
+
w_vert = calkit.docx._tag(calkit.docx.W, "vertAlign")
|
|
2036
|
+
by_name = {
|
|
2037
|
+
calkit.latex.make_bookmark_name(b.path, b.lineno): b for b in blks
|
|
2038
|
+
}
|
|
2039
|
+
for p in doc.paragraphs:
|
|
2040
|
+
# Without a numbered piece's suffix, e.g., ck_ab12_7_2
|
|
2041
|
+
name = "_".join((p.bookmark or "").split("_")[:3])
|
|
2042
|
+
fblk = by_name.get(name)
|
|
2043
|
+
if fblk is None or p.element is None:
|
|
2044
|
+
continue
|
|
2045
|
+
src = calkit.latex._strip_comments_joined(
|
|
2046
|
+
"\n".join(ln.text for ln in fblk.lines)
|
|
2047
|
+
)
|
|
2048
|
+
for fm in re.finditer(r"\\ce\s*\{((?:[^{}]|\{[^{}]*\})*)\}", src):
|
|
2049
|
+
text = doc.linear_text(p.element)
|
|
2050
|
+
if "\ufffc" not in text:
|
|
2051
|
+
break
|
|
2052
|
+
formula = fm.group(1)
|
|
2053
|
+
shown = calkit.latex.detex(f"\\ce{{{formula}}}")
|
|
2054
|
+
before = calkit.latex.keep_letters_and_digits(
|
|
2055
|
+
calkit.latex.detex(src[: fm.start()])
|
|
2056
|
+
)[-12:]
|
|
2057
|
+
# The formula with some of it as the picture, after the
|
|
2058
|
+
# prose before it
|
|
2059
|
+
spot = None
|
|
2060
|
+
for i in range(len(shown) + 1):
|
|
2061
|
+
for j in range(i, len(shown) + 1):
|
|
2062
|
+
pattern = (
|
|
2063
|
+
re.escape(shown[:i])
|
|
2064
|
+
+ r"\s*\ufffc\s*"
|
|
2065
|
+
+ re.escape(shown[j:])
|
|
2066
|
+
)
|
|
2067
|
+
for sm in re.finditer(pattern, text):
|
|
2068
|
+
if calkit.latex.keep_letters_and_digits(
|
|
2069
|
+
text[: sm.start()]
|
|
2070
|
+
).endswith(before):
|
|
2071
|
+
spot = sm
|
|
2072
|
+
break
|
|
2073
|
+
if spot:
|
|
2074
|
+
break
|
|
2075
|
+
if spot:
|
|
2076
|
+
break
|
|
2077
|
+
if spot is None:
|
|
2078
|
+
continue
|
|
2079
|
+
# Its runs, as the text before it is set, with numbers
|
|
2080
|
+
# after a symbol lowered and charges raised
|
|
2081
|
+
pieces: list[tuple[str, str | None]] = []
|
|
2082
|
+
k = 0
|
|
2083
|
+
while k < len(formula):
|
|
2084
|
+
ch = formula[k]
|
|
2085
|
+
if ch in "^_" and k + 1 < len(formula):
|
|
2086
|
+
if formula[k + 1] == "{":
|
|
2087
|
+
close = formula.find("}", k + 2)
|
|
2088
|
+
close = len(formula) if close < 0 else close
|
|
2089
|
+
arg, k = formula[k + 2 : close], close + 1
|
|
2090
|
+
else:
|
|
2091
|
+
arg, k = formula[k + 1], k + 2
|
|
2092
|
+
pieces.append(
|
|
2093
|
+
(
|
|
2094
|
+
arg,
|
|
2095
|
+
"superscript" if ch == "^" else "subscript",
|
|
2096
|
+
)
|
|
2097
|
+
)
|
|
2098
|
+
continue
|
|
2099
|
+
if (
|
|
2100
|
+
ch.isdigit()
|
|
2101
|
+
and k
|
|
2102
|
+
and (
|
|
2103
|
+
formula[k - 1].isalpha() or formula[k - 1] in ")]"
|
|
2104
|
+
)
|
|
2105
|
+
):
|
|
2106
|
+
end = k
|
|
2107
|
+
while end < len(formula) and formula[end].isdigit():
|
|
2108
|
+
end += 1
|
|
2109
|
+
pieces.append((formula[k:end], "subscript"))
|
|
2110
|
+
k = end
|
|
2111
|
+
continue
|
|
2112
|
+
if (
|
|
2113
|
+
ch in "+-"
|
|
2114
|
+
and k
|
|
2115
|
+
and formula[k - 1] not in " "
|
|
2116
|
+
and (k + 1 == len(formula) or formula[k + 1] == " ")
|
|
2117
|
+
):
|
|
2118
|
+
pieces.append((ch, "superscript"))
|
|
2119
|
+
k += 1
|
|
2120
|
+
continue
|
|
2121
|
+
pieces.append((ch, None))
|
|
2122
|
+
k += 1
|
|
2123
|
+
prior = [
|
|
2124
|
+
r for r in p.element.iter(w_r) if r.find(w_t) is not None
|
|
2125
|
+
]
|
|
2126
|
+
offset = len(text[: spot.start()])
|
|
2127
|
+
style = None
|
|
2128
|
+
pos = 0
|
|
2129
|
+
for r in prior:
|
|
2130
|
+
pos += len(doc._run_text(r))
|
|
2131
|
+
if pos >= offset:
|
|
2132
|
+
style = r.find(w_rpr)
|
|
2133
|
+
break
|
|
2134
|
+
runs: list[tuple[str, str | None]] = []
|
|
2135
|
+
for chunk, vert in pieces:
|
|
2136
|
+
if runs and runs[-1][1] == vert:
|
|
2137
|
+
runs[-1] = (runs[-1][0] + chunk, vert)
|
|
2138
|
+
else:
|
|
2139
|
+
runs.append((chunk, vert))
|
|
2140
|
+
elements = []
|
|
2141
|
+
for chunk, vert in runs:
|
|
2142
|
+
run = ET.Element(w_r)
|
|
2143
|
+
if style is not None:
|
|
2144
|
+
rpr = deepcopy(style)
|
|
2145
|
+
for old in rpr.findall(w_vert):
|
|
2146
|
+
rpr.remove(old)
|
|
2147
|
+
run.append(rpr)
|
|
2148
|
+
if vert is not None:
|
|
2149
|
+
has_rpr = run.find(w_rpr)
|
|
2150
|
+
rpr = (
|
|
2151
|
+
has_rpr
|
|
2152
|
+
if has_rpr is not None
|
|
2153
|
+
else ET.SubElement(run, w_rpr)
|
|
2154
|
+
)
|
|
2155
|
+
v = ET.Element(w_vert)
|
|
2156
|
+
v.set(calkit.docx._tag(calkit.docx.W, "val"), vert)
|
|
2157
|
+
# In schema order, before what follows it
|
|
2158
|
+
later = {
|
|
2159
|
+
calkit.docx._tag(calkit.docx.W, n)
|
|
2160
|
+
for n in (
|
|
2161
|
+
"rtl",
|
|
2162
|
+
"cs",
|
|
2163
|
+
"em",
|
|
2164
|
+
"lang",
|
|
2165
|
+
"eastAsianLayout",
|
|
2166
|
+
"specVanish",
|
|
2167
|
+
"oMath",
|
|
2168
|
+
)
|
|
2169
|
+
}
|
|
2170
|
+
rpr.insert(
|
|
2171
|
+
next(
|
|
2172
|
+
(
|
|
2173
|
+
i
|
|
2174
|
+
for i, c in enumerate(rpr)
|
|
2175
|
+
if c.tag in later
|
|
2176
|
+
),
|
|
2177
|
+
len(rpr),
|
|
2178
|
+
),
|
|
2179
|
+
v,
|
|
2180
|
+
)
|
|
2181
|
+
t = ET.SubElement(run, w_t)
|
|
2182
|
+
t.text = chunk
|
|
2183
|
+
t.set(calkit.docx.XML_SPACE, "preserve")
|
|
2184
|
+
elements.append(run)
|
|
2185
|
+
if elements and doc.replace_range(
|
|
2186
|
+
p.element, spot.start(), spot.end(), elements[0]
|
|
2187
|
+
):
|
|
2188
|
+
at = list(p.element).index(elements[0])
|
|
2189
|
+
for k, run in enumerate(elements[1:], 1):
|
|
2190
|
+
p.element.insert(at + k, run)
|
|
2191
|
+
doc._parents = {c: q for q in doc.doc.iter() for c in q}
|
|
2192
|
+
|
|
2193
|
+
def rebuild_tables(
|
|
2194
|
+
doc: calkit.docx.Document,
|
|
2195
|
+
paras: list[calkit.docx.Paragraph],
|
|
2196
|
+
matched: list[calkit.latex.Block | None],
|
|
2197
|
+
table_blks: list[calkit.latex.Block],
|
|
2198
|
+
preamble: str,
|
|
2199
|
+
) -> tuple[dict[Any, calkit.latex.Block | None], set[int]]:
|
|
2200
|
+
# Word's tables replaced by ones converted from the source, giving
|
|
2201
|
+
# the block each paragraph is anchored to after and the ids of the
|
|
2202
|
+
# blocks whose tables were replaced
|
|
2203
|
+
rebuilt: set[int] = set()
|
|
2204
|
+
from collections import Counter
|
|
2205
|
+
from xml.etree import ElementTree as ET
|
|
2206
|
+
|
|
2207
|
+
w_body = doc.doc.find(calkit.docx._tag(calkit.docx.W, "body"))
|
|
2208
|
+
assert w_body is not None
|
|
2209
|
+
w_p = calkit.docx._tag(calkit.docx.W, "p")
|
|
2210
|
+
w_tbl = calkit.docx._tag(calkit.docx.W, "tbl")
|
|
2211
|
+
w_ppr = calkit.docx._tag(calkit.docx.W, "pPr")
|
|
2212
|
+
w_sect_tag = calkit.docx._tag(calkit.docx.W, "sectPr")
|
|
2213
|
+
w_sect = f"{w_ppr}/{w_sect_tag}"
|
|
2214
|
+
block_of = {p.element: b for p, b in zip(paras, matched)}
|
|
2215
|
+
|
|
2216
|
+
def top_of(el: Any) -> Any:
|
|
2217
|
+
while doc._parents.get(el) is not w_body:
|
|
2218
|
+
el = doc._parents[el]
|
|
2219
|
+
return el
|
|
2220
|
+
|
|
2221
|
+
def number(el: Any, attr: str, default: int) -> int:
|
|
2222
|
+
value = el.get(attr) if el is not None else None
|
|
2223
|
+
try:
|
|
2224
|
+
return int(float(value)) if value else default
|
|
2225
|
+
except ValueError:
|
|
2226
|
+
return default
|
|
2227
|
+
|
|
2228
|
+
def is_prose(q: Any, blk: calkit.latex.Block) -> bool:
|
|
2229
|
+
# Anchored elsewhere, and more than a cell's few words, which
|
|
2230
|
+
# can match, e.g., a caption by chance
|
|
2231
|
+
return (
|
|
2232
|
+
block_of.get(q) not in (None, blk)
|
|
2233
|
+
and len(doc.linear_text(q).split()) >= 4
|
|
2234
|
+
)
|
|
2235
|
+
|
|
2236
|
+
def tokens(text: str) -> list[str]:
|
|
2237
|
+
return re.findall(
|
|
2238
|
+
r"[A-Z]?[a-z]+|[A-Z]+(?![a-z])|\d+(?:\.\d+)?", text
|
|
2239
|
+
)
|
|
2240
|
+
|
|
2241
|
+
# Each table's Word table to replace, and the width of the text
|
|
2242
|
+
# where it is, as its section sets it
|
|
2243
|
+
anchors: list[Any] = []
|
|
2244
|
+
widths: list[int] = []
|
|
2245
|
+
sizes: list[int | None] = []
|
|
2246
|
+
for blk in table_blks:
|
|
2247
|
+
mine = Counter(
|
|
2248
|
+
top_of(p.element)
|
|
2249
|
+
for p, b in zip(paras, matched)
|
|
2250
|
+
if b is blk and p.element is not None
|
|
2251
|
+
)
|
|
2252
|
+
anchor = mine.most_common(1)[0][0] if mine else None
|
|
2253
|
+
others = (
|
|
2254
|
+
[q for q in anchor.iter(w_p) if is_prose(q, blk)]
|
|
2255
|
+
if anchor is not None
|
|
2256
|
+
else []
|
|
2257
|
+
)
|
|
2258
|
+
if (
|
|
2259
|
+
anchor is None
|
|
2260
|
+
or anchor.tag != w_tbl
|
|
2261
|
+
or 3 * len(others) > mine[anchor]
|
|
2262
|
+
):
|
|
2263
|
+
anchors.append(None)
|
|
2264
|
+
widths.append(0)
|
|
2265
|
+
sizes.append(None)
|
|
2266
|
+
continue
|
|
2267
|
+
anchors.append(anchor)
|
|
2268
|
+
# In the size the PDF has it, as most of Word's runs in it are
|
|
2269
|
+
sz = Counter(
|
|
2270
|
+
el.get(calkit.docx._tag(calkit.docx.W, "val"))
|
|
2271
|
+
for el in anchor.iter(calkit.docx._tag(calkit.docx.W, "sz"))
|
|
2272
|
+
).most_common(1)
|
|
2273
|
+
sizes.append(
|
|
2274
|
+
int(sz[0][0]) if sz and (sz[0][0] or "").isdigit() else None
|
|
2275
|
+
)
|
|
2276
|
+
children = list(w_body)
|
|
2277
|
+
sect = next(
|
|
2278
|
+
(
|
|
2279
|
+
c if c.tag == w_sect_tag else c.find(w_sect)
|
|
2280
|
+
for c in children[children.index(anchor) :]
|
|
2281
|
+
if c.tag == w_sect_tag or c.find(w_sect) is not None
|
|
2282
|
+
),
|
|
2283
|
+
None,
|
|
2284
|
+
)
|
|
2285
|
+
width = 9360
|
|
2286
|
+
if sect is not None:
|
|
2287
|
+
size = sect.find(calkit.docx._tag(calkit.docx.W, "pgSz"))
|
|
2288
|
+
margin = sect.find(calkit.docx._tag(calkit.docx.W, "pgMar"))
|
|
2289
|
+
columns = sect.find(calkit.docx._tag(calkit.docx.W, "cols"))
|
|
2290
|
+
w_attr = calkit.docx._tag(calkit.docx.W, "w")
|
|
2291
|
+
n_cols = number(
|
|
2292
|
+
columns, calkit.docx._tag(calkit.docx.W, "num"), 1
|
|
2293
|
+
)
|
|
2294
|
+
width = (
|
|
2295
|
+
number(size, w_attr, 12240)
|
|
2296
|
+
- number(
|
|
2297
|
+
margin, calkit.docx._tag(calkit.docx.W, "left"), 1440
|
|
2298
|
+
)
|
|
2299
|
+
- number(
|
|
2300
|
+
margin, calkit.docx._tag(calkit.docx.W, "right"), 1440
|
|
2301
|
+
)
|
|
2302
|
+
- (n_cols - 1)
|
|
2303
|
+
* number(
|
|
2304
|
+
columns, calkit.docx._tag(calkit.docx.W, "space"), 720
|
|
2305
|
+
)
|
|
2306
|
+
) // max(1, n_cols)
|
|
2307
|
+
widths.append(width)
|
|
2308
|
+
tables = calkit.docx.latex_to_tables(
|
|
2309
|
+
[
|
|
2310
|
+
"\n".join(ln.text for ln in b.lines) if a is not None else ""
|
|
2311
|
+
for b, a in zip(table_blks, anchors)
|
|
2312
|
+
],
|
|
2313
|
+
preamble,
|
|
2314
|
+
widths,
|
|
2315
|
+
sizes,
|
|
2316
|
+
)
|
|
2317
|
+
for blk, table, anchor in zip(table_blks, tables, anchors):
|
|
2318
|
+
if table is None or anchor is None:
|
|
2319
|
+
continue
|
|
2320
|
+
# Beside it, what's anchored to the table or reads only as its
|
|
2321
|
+
# words, e.g., "Error Rate" from a header row
|
|
2322
|
+
words = set(tokens(calkit.latex.detex(blk.text)))
|
|
2323
|
+
children = list(w_body)
|
|
2324
|
+
at = children.index(anchor)
|
|
2325
|
+
going = [anchor]
|
|
2326
|
+
for step in (-1, 1):
|
|
2327
|
+
k = at + step
|
|
2328
|
+
while 0 <= k < len(children):
|
|
2329
|
+
unit = children[k]
|
|
2330
|
+
# Its text runs, since a rule Word drew as a picture
|
|
2331
|
+
# can sit in the same paragraph
|
|
2332
|
+
text = "".join(
|
|
2333
|
+
t.text or ""
|
|
2334
|
+
for t in unit.iter(
|
|
2335
|
+
calkit.docx._tag(calkit.docx.W, "t")
|
|
2336
|
+
)
|
|
2337
|
+
)
|
|
2338
|
+
if unit.tag != w_p or unit.find(w_sect) is not None:
|
|
2339
|
+
break
|
|
2340
|
+
held = block_of.get(unit)
|
|
2341
|
+
if not (
|
|
2342
|
+
held is blk
|
|
2343
|
+
or (
|
|
2344
|
+
held is None
|
|
2345
|
+
and tokens(text)
|
|
2346
|
+
and set(tokens(text)) <= words
|
|
2347
|
+
)
|
|
2348
|
+
):
|
|
2349
|
+
break
|
|
2350
|
+
going.append(unit)
|
|
2351
|
+
k += step
|
|
2352
|
+
# Anchored prose inside Word's table moves out of it, before the
|
|
2353
|
+
# table if it came first
|
|
2354
|
+
inside = list(anchor.iter(w_p))
|
|
2355
|
+
first_mine = next(
|
|
2356
|
+
(i for i, q in enumerate(inside) if block_of.get(q) is blk), 0
|
|
2357
|
+
)
|
|
2358
|
+
moved = [
|
|
2359
|
+
(i > first_mine, q)
|
|
2360
|
+
for i, q in enumerate(inside)
|
|
2361
|
+
if is_prose(q, blk)
|
|
2362
|
+
]
|
|
2363
|
+
for _, q in moved:
|
|
2364
|
+
doc._parents[q].remove(q)
|
|
2365
|
+
at = list(w_body).index(anchor)
|
|
2366
|
+
pieces = (
|
|
2367
|
+
[q for after, q in moved if not after]
|
|
2368
|
+
+ [table]
|
|
2369
|
+
+ [q for after, q in moved if after]
|
|
2370
|
+
)
|
|
2371
|
+
for k, piece in enumerate(pieces):
|
|
2372
|
+
w_body.insert(at + k, piece)
|
|
2373
|
+
for unit in going:
|
|
2374
|
+
w_body.remove(unit)
|
|
2375
|
+
# Set off from the text after, as LaTeX sets a table
|
|
2376
|
+
children = list(w_body)
|
|
2377
|
+
after_tbl = children[children.index(table) + 1 :][:1]
|
|
2378
|
+
if after_tbl and after_tbl[0].tag == w_p:
|
|
2379
|
+
ppr = after_tbl[0].find(w_ppr)
|
|
2380
|
+
if ppr is None:
|
|
2381
|
+
ppr = ET.Element(w_ppr)
|
|
2382
|
+
after_tbl[0].insert(0, ppr)
|
|
2383
|
+
spacing = ppr.find(calkit.docx._tag(calkit.docx.W, "spacing"))
|
|
2384
|
+
if spacing is None:
|
|
2385
|
+
spacing = ET.Element(
|
|
2386
|
+
calkit.docx._tag(calkit.docx.W, "spacing")
|
|
2387
|
+
)
|
|
2388
|
+
# In schema order, before what follows spacing
|
|
2389
|
+
later = {
|
|
2390
|
+
calkit.docx._tag(calkit.docx.W, t)
|
|
2391
|
+
for t in (
|
|
2392
|
+
"ind",
|
|
2393
|
+
"contextualSpacing",
|
|
2394
|
+
"mirrorIndents",
|
|
2395
|
+
"suppressOverlap",
|
|
2396
|
+
"jc",
|
|
2397
|
+
"textDirection",
|
|
2398
|
+
"textAlignment",
|
|
2399
|
+
"textboxTightWrap",
|
|
2400
|
+
"outlineLvl",
|
|
2401
|
+
"divId",
|
|
2402
|
+
"cnfStyle",
|
|
2403
|
+
"rPr",
|
|
2404
|
+
"sectPr",
|
|
2405
|
+
"pPrChange",
|
|
2406
|
+
)
|
|
2407
|
+
}
|
|
2408
|
+
ppr.insert(
|
|
2409
|
+
next(
|
|
2410
|
+
(k for k, c in enumerate(ppr) if c.tag in later),
|
|
2411
|
+
len(ppr),
|
|
2412
|
+
),
|
|
2413
|
+
spacing,
|
|
2414
|
+
)
|
|
2415
|
+
before_attr = calkit.docx._tag(calkit.docx.W, "before")
|
|
2416
|
+
if int(spacing.get(before_attr, "0") or 0) < 240:
|
|
2417
|
+
spacing.set(before_attr, "240")
|
|
2418
|
+
rebuilt.add(id(blk))
|
|
2419
|
+
for q in table.iter(w_p):
|
|
2420
|
+
block_of[q] = blk
|
|
2421
|
+
return block_of, rebuilt
|
|
2422
|
+
|
|
2022
2423
|
ck_info = calkit.load_calkit_info()
|
|
2023
2424
|
stages = ck_info.get("pipeline", {}).get("stages", {})
|
|
2024
2425
|
stage_name = calkit.pipeline.get_stage_for_output(pdf_path, ck_info)
|
|
@@ -2054,10 +2455,13 @@ def to_docx(
|
|
|
2054
2455
|
raise_error(f"{output} already exists; use --force to overwrite it")
|
|
2055
2456
|
if engine is None:
|
|
2056
2457
|
engine = "word" if calkit.docx.word_installed() else "libreoffice"
|
|
2458
|
+
# Converted beside the output, then read in and removed, so a failed
|
|
2459
|
+
# export doesn't leave the bare conversion where the export goes
|
|
2460
|
+
bare = Path(output).with_suffix(".bare.docx").as_posix()
|
|
2057
2461
|
if engine == "word":
|
|
2058
2462
|
typer.echo("Converting the PDF with Word")
|
|
2059
2463
|
try:
|
|
2060
|
-
calkit.docx.pdf_to_docx(pdf_path,
|
|
2464
|
+
calkit.docx.pdf_to_docx(pdf_path, bare)
|
|
2061
2465
|
except RuntimeError as e:
|
|
2062
2466
|
raise_error(
|
|
2063
2467
|
f"{e}\nTo convert from the source instead, use "
|
|
@@ -2210,16 +2614,135 @@ def to_docx(
|
|
|
2210
2614
|
f"{build}. Its make4ht comes with TeX Live's tex4ht package."
|
|
2211
2615
|
)
|
|
2212
2616
|
try:
|
|
2213
|
-
calkit.docx.odt_to_docx(odt,
|
|
2617
|
+
calkit.docx.odt_to_docx(odt, bare)
|
|
2214
2618
|
except RuntimeError as e:
|
|
2215
2619
|
raise_error(f"{e}\nTeX4ht's logs are in {build}")
|
|
2620
|
+
elif engine == "pandoc":
|
|
2621
|
+
import tempfile
|
|
2622
|
+
|
|
2623
|
+
if calkit.docx.find_pandoc() is None:
|
|
2624
|
+
raise_error(
|
|
2625
|
+
"Converting with Pandoc requires Pandoc; run "
|
|
2626
|
+
"'calkit install pandoc'"
|
|
2627
|
+
)
|
|
2628
|
+
typer.echo("Converting the source with Pandoc")
|
|
2629
|
+
# What TeX worked out, e.g., numbers and citations, comes from the
|
|
2630
|
+
# build beside the PDF or the source
|
|
2631
|
+
found_aux: dict[str, str] = {}
|
|
2632
|
+
for ext in (".aux", ".bbl"):
|
|
2633
|
+
for base in (Path(pdf_path), Path(source)):
|
|
2634
|
+
picture = base.with_suffix(ext)
|
|
2635
|
+
if picture.is_file():
|
|
2636
|
+
found_aux[ext] = picture.read_text(
|
|
2637
|
+
encoding="utf-8", errors="replace"
|
|
2638
|
+
)
|
|
2639
|
+
break
|
|
2640
|
+
if ".aux" not in found_aux:
|
|
2641
|
+
warn(
|
|
2642
|
+
"No .aux beside the PDF or source, so references and "
|
|
2643
|
+
"citations show as ??; build the document first"
|
|
2644
|
+
)
|
|
2645
|
+
tex_dir = Path(source).parent
|
|
2646
|
+
drawn = tempfile.mkdtemp()
|
|
2647
|
+
|
|
2648
|
+
def find_image(path: str) -> str | None:
|
|
2649
|
+
# A picture in the source as one Word can show, drawing a PDF
|
|
2650
|
+
for picture in [tex_dir / path] + [
|
|
2651
|
+
tex_dir / (path + ext)
|
|
2652
|
+
for ext in (".pdf", ".png", ".jpg", ".jpeg")
|
|
2653
|
+
]:
|
|
2654
|
+
if not picture.is_file():
|
|
2655
|
+
continue
|
|
2656
|
+
if picture.suffix.lower() != ".pdf":
|
|
2657
|
+
return picture.resolve().as_posix()
|
|
2658
|
+
png = os.path.join(
|
|
2659
|
+
drawn, f"{len(os.listdir(drawn))}-{picture.stem}.png"
|
|
2660
|
+
)
|
|
2661
|
+
if calkit.docx.pdf_to_png(str(picture), png):
|
|
2662
|
+
return png
|
|
2663
|
+
warn(f"Couldn't draw {picture} for Word")
|
|
2664
|
+
return None
|
|
2665
|
+
return None
|
|
2666
|
+
|
|
2667
|
+
try:
|
|
2668
|
+
text = calkit.latex.prepare_for_pandoc(
|
|
2669
|
+
calkit.latex.flatten(source),
|
|
2670
|
+
found_aux.get(".aux", ""),
|
|
2671
|
+
found_aux.get(".bbl", ""),
|
|
2672
|
+
find_image,
|
|
2673
|
+
)
|
|
2674
|
+
reference = os.path.join(drawn, "reference.docx")
|
|
2675
|
+
Path(reference).write_bytes(calkit.docx.make_reference_doc())
|
|
2676
|
+
pandoc_docx = calkit.docx._pandoc(
|
|
2677
|
+
[
|
|
2678
|
+
"-f",
|
|
2679
|
+
"latex",
|
|
2680
|
+
"-t",
|
|
2681
|
+
"docx",
|
|
2682
|
+
"-o",
|
|
2683
|
+
"-",
|
|
2684
|
+
"--resource-path",
|
|
2685
|
+
tex_dir.as_posix() or ".",
|
|
2686
|
+
"--reference-doc",
|
|
2687
|
+
reference,
|
|
2688
|
+
],
|
|
2689
|
+
text.encode(),
|
|
2690
|
+
)
|
|
2691
|
+
except RuntimeError as e:
|
|
2692
|
+
raise_error(str(e))
|
|
2693
|
+
finally:
|
|
2694
|
+
shutil.rmtree(drawn, ignore_errors=True)
|
|
2695
|
+
Path(bare).write_bytes(pandoc_docx)
|
|
2216
2696
|
else:
|
|
2217
|
-
raise_error("--engine must be 'word' or '
|
|
2218
|
-
|
|
2697
|
+
raise_error("--engine must be 'word', 'libreoffice', or 'pandoc'")
|
|
2698
|
+
try:
|
|
2699
|
+
doc = calkit.docx.Document(bare)
|
|
2700
|
+
finally:
|
|
2701
|
+
os.remove(bare)
|
|
2702
|
+
doc.path = output
|
|
2219
2703
|
lines = calkit.latex.flatten(source)
|
|
2220
2704
|
blks = calkit.latex.blocks(lines)
|
|
2705
|
+
# The document's macro definitions, for converting math and tables
|
|
2706
|
+
preamble = "\n".join(
|
|
2707
|
+
ln.text
|
|
2708
|
+
for ln in lines
|
|
2709
|
+
if re.match(
|
|
2710
|
+
r"\s*\\(newcommand|renewcommand|providecommand|"
|
|
2711
|
+
r"DeclareMathOperator|def)\b",
|
|
2712
|
+
ln.text,
|
|
2713
|
+
)
|
|
2714
|
+
)
|
|
2221
2715
|
paras = doc.paragraphs
|
|
2222
2716
|
matched = calkit.latex.align([p.text for p in paras], blks)
|
|
2717
|
+
# Tables come from the source too, converted by Pandoc, since Word's
|
|
2718
|
+
# import can merge their rows, drop a header, or leave math as text.
|
|
2719
|
+
# Each replaces the Word table holding most of its cells, with what
|
|
2720
|
+
# Word put beside it from the table, e.g., a header row, and nothing
|
|
2721
|
+
# anchored elsewhere.
|
|
2722
|
+
rebuilt: set[int] = set()
|
|
2723
|
+
table_blks = [
|
|
2724
|
+
b
|
|
2725
|
+
for b in blks
|
|
2726
|
+
if b.display is None
|
|
2727
|
+
and re.search(
|
|
2728
|
+
r"\\begin\s*\{("
|
|
2729
|
+
+ "|".join(re.escape(e) for e in calkit.docx.TABLE_ENVS)
|
|
2730
|
+
+ r")\}",
|
|
2731
|
+
"\n".join(ln.text for ln in b.lines),
|
|
2732
|
+
)
|
|
2733
|
+
]
|
|
2734
|
+
if (
|
|
2735
|
+
engine in ("word", "pandoc")
|
|
2736
|
+
and table_blks
|
|
2737
|
+
and calkit.docx.find_pandoc() is not None
|
|
2738
|
+
):
|
|
2739
|
+
block_of, rebuilt = rebuild_tables(
|
|
2740
|
+
doc, paras, matched, table_blks, preamble
|
|
2741
|
+
)
|
|
2742
|
+
if rebuilt:
|
|
2743
|
+
doc._parents = {c: p for p in doc.doc.iter() for c in p}
|
|
2744
|
+
paras = doc.paragraphs
|
|
2745
|
+
matched = [block_of.get(p.element) for p in paras]
|
|
2223
2746
|
# Pieces of a paragraph Word's import split, e.g., at a page break,
|
|
2224
2747
|
# join again, with anything floated between them after
|
|
2225
2748
|
if engine == "word":
|
|
@@ -2396,6 +2919,21 @@ def to_docx(
|
|
|
2396
2919
|
for k in reversed(range(1, len(m.groups()) + 1)):
|
|
2397
2920
|
if not m.group(k):
|
|
2398
2921
|
doc.insert_text(para.element, m.start(k), "-")
|
|
2922
|
+
# Likewise the space between a number and a word after it, e.g.,
|
|
2923
|
+
# a unit, "20 K" as "20K", where the source never
|
|
2924
|
+
# joins them
|
|
2925
|
+
pairs = re.findall(
|
|
2926
|
+
r"(?<![\w.])(\d+(?:[.,]\d+)*) ([^\W\d_]\w*)", blk.text
|
|
2927
|
+
)
|
|
2928
|
+
for num, word in dict.fromkeys(pairs):
|
|
2929
|
+
joined = (
|
|
2930
|
+
rf"(?<![\w.]){re.escape(num)}(?={re.escape(word)}(?!\w))"
|
|
2931
|
+
)
|
|
2932
|
+
if re.search(joined, blk.text):
|
|
2933
|
+
continue
|
|
2934
|
+
text = doc.linear_text(para.element)
|
|
2935
|
+
for m in reversed(list(re.finditer(joined, text))):
|
|
2936
|
+
doc.insert_text(para.element, m.end(), " ")
|
|
2399
2937
|
paras = doc.paragraphs
|
|
2400
2938
|
# Bookmark each anchored paragraph; a block rendering as several
|
|
2401
2939
|
# paragraphs gets numbered suffixes
|
|
@@ -2415,6 +2953,9 @@ def to_docx(
|
|
|
2415
2953
|
# Display math goes in as Word equations converted from the source,
|
|
2416
2954
|
# replacing what Word's PDF import or TeX4ht made of it
|
|
2417
2955
|
equations: dict[str, str] = {}
|
|
2956
|
+
# Inline math placed from the source, by element, with which of its
|
|
2957
|
+
# block's inline math it is
|
|
2958
|
+
placed_math: dict[int, tuple[Any, int]] = {}
|
|
2418
2959
|
left_as_imported = 0
|
|
2419
2960
|
has_display = any(b.display for b in blks)
|
|
2420
2961
|
# Word's PDF import leaves inline math as text and pictures, which
|
|
@@ -2646,15 +3187,6 @@ def to_docx(
|
|
|
2646
3187
|
for blk in group
|
|
2647
3188
|
for k, (tex, numbered) in enumerate(blk.rows)
|
|
2648
3189
|
]
|
|
2649
|
-
preamble = "\n".join(
|
|
2650
|
-
ln.text
|
|
2651
|
-
for ln in lines
|
|
2652
|
-
if re.match(
|
|
2653
|
-
r"\s*\\(newcommand|renewcommand|providecommand|"
|
|
2654
|
-
r"DeclareMathOperator|def)\b",
|
|
2655
|
-
ln.text,
|
|
2656
|
-
)
|
|
2657
|
-
)
|
|
2658
3190
|
maths = calkit.docx.latex_to_omml([p[2] for p in pieces], preamble)
|
|
2659
3191
|
converted = dict(zip([(id(p[0]), p[1]) for p in pieces], maths))
|
|
2660
3192
|
# What each link within the document shows, e.g., an equation's
|
|
@@ -2883,10 +3415,26 @@ def to_docx(
|
|
|
2883
3415
|
rendered = calkit.latex.keep_letters_and_digits(
|
|
2884
3416
|
calkit.latex.get_math_text(tex)
|
|
2885
3417
|
)
|
|
3418
|
+
# Or with big operators as the letters TeX's math extension
|
|
3419
|
+
# font has them at, which Word's import can read, e.g., \sum as P
|
|
3420
|
+
drawn = calkit.latex.keep_letters_and_digits(
|
|
3421
|
+
calkit.latex.get_math_text(
|
|
3422
|
+
re.sub(
|
|
3423
|
+
r"\\(sum|prod|int|oint)(?![A-Za-z])",
|
|
3424
|
+
lambda m: {
|
|
3425
|
+
"sum": " P ",
|
|
3426
|
+
"prod": " Q ",
|
|
3427
|
+
"int": " R ",
|
|
3428
|
+
}.get(str(m.group(1)), " H "),
|
|
3429
|
+
tex,
|
|
3430
|
+
)
|
|
3431
|
+
)
|
|
3432
|
+
)
|
|
2886
3433
|
return (
|
|
2887
3434
|
bool(region.strip())
|
|
2888
3435
|
and all(
|
|
2889
3436
|
calkit.latex.keep_letters_and_digits(w) in rendered
|
|
3437
|
+
or calkit.latex.keep_letters_and_digits(w) in drawn
|
|
2890
3438
|
for w in words
|
|
2891
3439
|
)
|
|
2892
3440
|
and set(re.findall(r"\d", region))
|
|
@@ -2956,11 +3504,12 @@ def to_docx(
|
|
|
2956
3504
|
for p, b in zip(paras, matched):
|
|
2957
3505
|
if b is not None and p.element is not None:
|
|
2958
3506
|
by_block.setdefault(id(b), []).append(p.element)
|
|
3507
|
+
math_index: dict[int, int] = {}
|
|
2959
3508
|
for b in blks:
|
|
2960
3509
|
els = by_block.get(id(b), [])
|
|
2961
3510
|
texts = [doc.linear_text(el) for el in els]
|
|
2962
3511
|
pi, cursor = 0, 0
|
|
2963
|
-
found = b.inline_math if els else []
|
|
3512
|
+
found = b.inline_math if els and id(b) not in rebuilt else []
|
|
2964
3513
|
for n, (pre, tex, post) in enumerate(found):
|
|
2965
3514
|
# Only a superscript or subscript, e.g., $^{13}$C, stays
|
|
2966
3515
|
# as Word's formatted text, easier to edit
|
|
@@ -2974,8 +3523,13 @@ def to_docx(
|
|
|
2974
3523
|
)
|
|
2975
3524
|
# In the paragraph where the last one was, or a later
|
|
2976
3525
|
# one when the block continues past a page or float
|
|
3526
|
+
start_len = len(inline)
|
|
2977
3527
|
for k in range(pi, len(els)):
|
|
2978
3528
|
if place(k, pre, tex, shown, post, is_last):
|
|
3529
|
+
# Which of the block's inline math it is
|
|
3530
|
+
for j in range(start_len, len(inline)):
|
|
3531
|
+
if inline[j][3] is not None:
|
|
3532
|
+
math_index[j] = n
|
|
2979
3533
|
break
|
|
2980
3534
|
inline_maths = iter(
|
|
2981
3535
|
calkit.docx.latex_to_omml(
|
|
@@ -3002,6 +3556,8 @@ def to_docx(
|
|
|
3002
3556
|
s0, e0 = s0 + 1, e0 + 1
|
|
3003
3557
|
if doc.replace_range(el, s0, e0, math) and itex is not None:
|
|
3004
3558
|
inline_count += 1
|
|
3559
|
+
if math is not None and i in math_index:
|
|
3560
|
+
placed_math[id(math)] = (math, math_index[i])
|
|
3005
3561
|
# After a picture there was no text to put it in, so
|
|
3006
3562
|
# it starts the text after
|
|
3007
3563
|
if pad_trail and not trailed and math is not None:
|
|
@@ -3060,6 +3616,49 @@ def to_docx(
|
|
|
3060
3616
|
for p in doc.paragraphs
|
|
3061
3617
|
if p.bookmark and p.bookmark in original
|
|
3062
3618
|
}
|
|
3619
|
+
if engine == "word":
|
|
3620
|
+
restore_formulas(doc, blks)
|
|
3621
|
+
doc.prune_media()
|
|
3622
|
+
original = {
|
|
3623
|
+
p.bookmark: p.text
|
|
3624
|
+
for p in doc.paragraphs
|
|
3625
|
+
if p.bookmark and p.bookmark in original
|
|
3626
|
+
}
|
|
3627
|
+
# The inline math placed, as the converter reads it back, by paragraph,
|
|
3628
|
+
# so a merge can tell what a reviewer changed in it
|
|
3629
|
+
inline_eqs: dict[str, list[tuple[int, str]]] = {}
|
|
3630
|
+
held_math: list[tuple[str | None, tuple[Any, int]]] = [
|
|
3631
|
+
(p.bookmark, placed_math[id(child)])
|
|
3632
|
+
for p in doc.paragraphs
|
|
3633
|
+
if p.bookmark and p.element is not None
|
|
3634
|
+
for child in p.element
|
|
3635
|
+
if id(child) in placed_math
|
|
3636
|
+
]
|
|
3637
|
+
# Converted with the document, a paragraph's inline math is its
|
|
3638
|
+
# source's in order, where there are as many
|
|
3639
|
+
if engine != "word":
|
|
3640
|
+
by_mark = {
|
|
3641
|
+
calkit.latex.make_bookmark_name(b.path, b.lineno): b
|
|
3642
|
+
for b in blks
|
|
3643
|
+
if not b.display
|
|
3644
|
+
}
|
|
3645
|
+
m_omath = calkit.docx._tag(calkit.docx.M, "oMath")
|
|
3646
|
+
for p in doc.paragraphs:
|
|
3647
|
+
mblk = by_mark.get(p.bookmark or "")
|
|
3648
|
+
if mblk is None or p.element is None:
|
|
3649
|
+
continue
|
|
3650
|
+
found_math = [c for c in p.element if c.tag == m_omath]
|
|
3651
|
+
if found_math and len(found_math) == len(mblk.inline_math):
|
|
3652
|
+
held_math += [
|
|
3653
|
+
(p.bookmark, (el, k)) for k, el in enumerate(found_math)
|
|
3654
|
+
]
|
|
3655
|
+
if held_math:
|
|
3656
|
+
read_back_inline = calkit.docx.omml_to_latex(
|
|
3657
|
+
[el for _, (el, _) in held_math]
|
|
3658
|
+
)
|
|
3659
|
+
for (mark, (_, index)), mtex in zip(held_math, read_back_inline):
|
|
3660
|
+
if mtex is not None and mark is not None:
|
|
3661
|
+
inline_eqs.setdefault(mark, []).append((index, mtex))
|
|
3063
3662
|
doc.even_margins()
|
|
3064
3663
|
# Existing comment blocks in the source go out as Word comments
|
|
3065
3664
|
threads, anchors, highlights, resolved = [], [], [], []
|
|
@@ -3112,6 +3711,7 @@ def to_docx(
|
|
|
3112
3711
|
original,
|
|
3113
3712
|
doc.media_hashes,
|
|
3114
3713
|
equations=equations,
|
|
3714
|
+
inline=inline_eqs,
|
|
3115
3715
|
)
|
|
3116
3716
|
)
|
|
3117
3717
|
doc.set_identifier(f"calkit-latex-export:{export_id}:{rev or ''}:{source}")
|
|
@@ -3209,7 +3809,10 @@ def merge_docx(
|
|
|
3209
3809
|
comment blocks above the paragraph; threads resolved in Word are
|
|
3210
3810
|
marked resolved.
|
|
3211
3811
|
"""
|
|
3812
|
+
import dataclasses
|
|
3212
3813
|
import datetime
|
|
3814
|
+
import difflib
|
|
3815
|
+
from typing import Any
|
|
3213
3816
|
|
|
3214
3817
|
import calkit.docx
|
|
3215
3818
|
import calkit.git
|
|
@@ -3246,6 +3849,19 @@ def merge_docx(
|
|
|
3246
3849
|
}
|
|
3247
3850
|
edits: dict[str, list[tuple[int, int, list[str]]]] = {}
|
|
3248
3851
|
changes: list[LatexDocxMergeChange] = []
|
|
3852
|
+
# Each block as edited so far, so edits to its pieces, e.g., a table's
|
|
3853
|
+
# cells, or its text and its math, all apply
|
|
3854
|
+
working: dict[int, calkit.latex.Block] = {}
|
|
3855
|
+
|
|
3856
|
+
def update(blk: calkit.latex.Block, new_lines: list[str]) -> None:
|
|
3857
|
+
working[id(blk)] = dataclasses.replace(
|
|
3858
|
+
blk,
|
|
3859
|
+
lines=[
|
|
3860
|
+
calkit.latex.SourceLine(blk.path, blk.lineno + i, text)
|
|
3861
|
+
for i, text in enumerate(new_lines)
|
|
3862
|
+
],
|
|
3863
|
+
)
|
|
3864
|
+
|
|
3249
3865
|
for para in doc.paragraphs:
|
|
3250
3866
|
if para.bookmark is None or para.bookmark not in original.paragraphs:
|
|
3251
3867
|
continue
|
|
@@ -3279,16 +3895,17 @@ def merge_docx(
|
|
|
3279
3895
|
continue
|
|
3280
3896
|
sent_tex = calkit.latex.from_word_text(sent)
|
|
3281
3897
|
new_tex = calkit.latex.from_word_text(para.text)
|
|
3282
|
-
|
|
3898
|
+
current = working.get(id(blk), blk)
|
|
3899
|
+
if calkit.latex.already_applied(current, sent_tex, new_tex):
|
|
3283
3900
|
changes.append(
|
|
3284
3901
|
LatexDocxMergeChange(
|
|
3285
3902
|
path=blk.path, lineno=blk.lineno, status="already-applied"
|
|
3286
3903
|
)
|
|
3287
3904
|
)
|
|
3288
3905
|
continue
|
|
3289
|
-
new_lines = calkit.latex.apply_edit(
|
|
3906
|
+
new_lines = calkit.latex.apply_edit(current, sent_tex, new_tex)
|
|
3290
3907
|
if new_lines == [
|
|
3291
|
-
ln.text.rstrip() for ln in
|
|
3908
|
+
ln.text.rstrip() for ln in current.lines if ln.text.strip()
|
|
3292
3909
|
]:
|
|
3293
3910
|
changes.append(
|
|
3294
3911
|
LatexDocxMergeChange(
|
|
@@ -3306,15 +3923,226 @@ def merge_docx(
|
|
|
3306
3923
|
)
|
|
3307
3924
|
)
|
|
3308
3925
|
continue
|
|
3309
|
-
|
|
3310
|
-
(blk.lineno, len(blk.lines), new_lines)
|
|
3311
|
-
)
|
|
3926
|
+
update(blk, new_lines)
|
|
3312
3927
|
changes.append(
|
|
3313
3928
|
LatexDocxMergeChange(
|
|
3314
3929
|
path=blk.path, lineno=blk.lineno, status="applied"
|
|
3315
3930
|
)
|
|
3316
3931
|
)
|
|
3317
3932
|
typer.echo(f"Applied edit at {loc}")
|
|
3933
|
+
# Inline math edited in Word goes back through the LaTeX the converter
|
|
3934
|
+
# reads it as, to the same piece of the paragraph's math in the source.
|
|
3935
|
+
# An export from before that was kept has each lined up with the
|
|
3936
|
+
# source's math by order instead.
|
|
3937
|
+
m_omath = calkit.docx._tag(calkit.docx.M, "oMath")
|
|
3938
|
+
# Lined up by order only for Word's import, which converts the math as
|
|
3939
|
+
# this does, going by the export's record on this machine
|
|
3940
|
+
older_word_export = False
|
|
3941
|
+
if not original.kept_inline:
|
|
3942
|
+
for folder in (
|
|
3943
|
+
calkit.latex.LOCAL_DOCX_EXPORTS_DIR,
|
|
3944
|
+
calkit.latex.DOCX_EXPORTS_DIR,
|
|
3945
|
+
):
|
|
3946
|
+
for record_path in glob.glob(
|
|
3947
|
+
os.path.join(folder, f"*-{original.id}.json")
|
|
3948
|
+
):
|
|
3949
|
+
try:
|
|
3950
|
+
with open(record_path, encoding="utf-8") as f:
|
|
3951
|
+
exported = json.load(f)
|
|
3952
|
+
except (OSError, ValueError):
|
|
3953
|
+
continue
|
|
3954
|
+
older_word_export |= exported.get("engine") == "word"
|
|
3955
|
+
with_math: list[
|
|
3956
|
+
tuple[calkit.docx.Paragraph, list[tuple[int, str]] | None, list[Any]]
|
|
3957
|
+
] = [
|
|
3958
|
+
(
|
|
3959
|
+
para,
|
|
3960
|
+
original.inline.get(para.bookmark or ""),
|
|
3961
|
+
[c for c in para.element if c.tag == m_omath],
|
|
3962
|
+
)
|
|
3963
|
+
for para in doc.paragraphs
|
|
3964
|
+
if para.element is not None
|
|
3965
|
+
and (
|
|
3966
|
+
para.bookmark in original.inline
|
|
3967
|
+
or (older_word_export and para.bookmark in original.paragraphs)
|
|
3968
|
+
)
|
|
3969
|
+
]
|
|
3970
|
+
with_math = [w for w in with_math if w[2]]
|
|
3971
|
+
all_math = [m for *_, math_els in with_math for m in math_els]
|
|
3972
|
+
if all_math and calkit.docx.find_pandoc() is None:
|
|
3973
|
+
warn(
|
|
3974
|
+
"Pandoc isn't installed, so edits to inline math aren't "
|
|
3975
|
+
"merged; run 'calkit install pandoc'"
|
|
3976
|
+
)
|
|
3977
|
+
with_math = []
|
|
3978
|
+
converted_source: dict[int, list[str | None]] = {}
|
|
3979
|
+
preamble = "\n".join(
|
|
3980
|
+
ln.text
|
|
3981
|
+
for ln in lines
|
|
3982
|
+
if re.match(
|
|
3983
|
+
r"\s*\\(newcommand|renewcommand|providecommand|"
|
|
3984
|
+
r"DeclareMathOperator|def)\b",
|
|
3985
|
+
ln.text,
|
|
3986
|
+
)
|
|
3987
|
+
)
|
|
3988
|
+
math_back: Iterator[str | None] = iter(
|
|
3989
|
+
calkit.docx.omml_to_latex(all_math) if with_math else []
|
|
3990
|
+
)
|
|
3991
|
+
for para, sent_maths, math_els in with_math:
|
|
3992
|
+
math_now = [next(math_back) for _ in math_els]
|
|
3993
|
+
assert para.bookmark is not None
|
|
3994
|
+
parts = para.bookmark.split("_")
|
|
3995
|
+
path, lineno = path_for_hash.get(parts[1], ""), int(parts[2])
|
|
3996
|
+
if sent_maths is None:
|
|
3997
|
+
sent = original.paragraphs.get(para.bookmark, "")
|
|
3998
|
+
blk = calkit.latex.find_block(
|
|
3999
|
+
blks, path, lineno, sent
|
|
4000
|
+
) or calkit.latex.find_block(blks, path, lineno, para.text)
|
|
4001
|
+
if blk is None or blk.display:
|
|
4002
|
+
continue
|
|
4003
|
+
# The source's math as the converter reads it back, as the
|
|
4004
|
+
# export would have kept it
|
|
4005
|
+
if id(blk) not in converted_source:
|
|
4006
|
+
texs = [t for _, t, _ in blk.inline_math]
|
|
4007
|
+
converted_source[id(blk)] = (
|
|
4008
|
+
calkit.docx.omml_to_latex(
|
|
4009
|
+
[
|
|
4010
|
+
m
|
|
4011
|
+
for m in calkit.docx.latex_to_omml(
|
|
4012
|
+
[f"${t}$" for t in texs], preamble
|
|
4013
|
+
)
|
|
4014
|
+
if m is not None
|
|
4015
|
+
]
|
|
4016
|
+
)
|
|
4017
|
+
if texs
|
|
4018
|
+
else []
|
|
4019
|
+
)
|
|
4020
|
+
source_back = converted_source[id(blk)]
|
|
4021
|
+
matcher = difflib.SequenceMatcher(
|
|
4022
|
+
a=[
|
|
4023
|
+
tuple(calkit.latex.math_tokens(t)) if t else None
|
|
4024
|
+
for t in source_back
|
|
4025
|
+
],
|
|
4026
|
+
b=[
|
|
4027
|
+
tuple(calkit.latex.math_tokens(t)) if t else None
|
|
4028
|
+
for t in math_now
|
|
4029
|
+
],
|
|
4030
|
+
autojunk=False,
|
|
4031
|
+
)
|
|
4032
|
+
lined_up: dict[int, int] = {}
|
|
4033
|
+
for tag, i1, i2, j1, j2 in matcher.get_opcodes():
|
|
4034
|
+
if tag == "equal" or (tag == "replace" and i2 - i1 == j2 - j1):
|
|
4035
|
+
lined_up.update(zip(range(j1, j2), range(i1, i2)))
|
|
4036
|
+
keep = [j for j in range(len(math_now)) if j in lined_up]
|
|
4037
|
+
sent_maths = [
|
|
4038
|
+
(lined_up[j], source_back[lined_up[j]] or "") for j in keep
|
|
4039
|
+
]
|
|
4040
|
+
math_now = [math_now[j] for j in keep]
|
|
4041
|
+
math_els = [math_els[j] for j in keep]
|
|
4042
|
+
edited_maths = [
|
|
4043
|
+
(k, sent_m, edited_m)
|
|
4044
|
+
for (k, sent_m), edited_m in zip(sent_maths, math_now)
|
|
4045
|
+
if edited_m is None
|
|
4046
|
+
or calkit.latex.math_tokens(edited_m)
|
|
4047
|
+
!= calkit.latex.math_tokens(sent_m)
|
|
4048
|
+
]
|
|
4049
|
+
if not edited_maths and len(math_els) == len(sent_maths):
|
|
4050
|
+
continue
|
|
4051
|
+
sent = original.paragraphs.get(para.bookmark, "")
|
|
4052
|
+
blk = calkit.latex.find_block(
|
|
4053
|
+
blks, path, lineno, sent
|
|
4054
|
+
) or calkit.latex.find_block(blks, path, lineno, para.text)
|
|
4055
|
+
loc = f"{path}:{lineno}" if blk is None else f"{blk.path}:{blk.lineno}"
|
|
4056
|
+
if blk is None or len(math_els) != len(sent_maths):
|
|
4057
|
+
warn(
|
|
4058
|
+
f"Inline math was added or removed at {loc}; apply it by hand"
|
|
4059
|
+
)
|
|
4060
|
+
changes.append(
|
|
4061
|
+
LatexDocxMergeChange(
|
|
4062
|
+
path=path, lineno=lineno, status="unplaced"
|
|
4063
|
+
)
|
|
4064
|
+
)
|
|
4065
|
+
continue
|
|
4066
|
+
if any(
|
|
4067
|
+
e.tag
|
|
4068
|
+
in (
|
|
4069
|
+
calkit.docx._tag(calkit.docx.W, "ins"),
|
|
4070
|
+
calkit.docx._tag(calkit.docx.W, "del"),
|
|
4071
|
+
)
|
|
4072
|
+
for m in math_els
|
|
4073
|
+
for e in m.iter()
|
|
4074
|
+
):
|
|
4075
|
+
warn(f"Tracked change at {loc} not yet accepted or rejected")
|
|
4076
|
+
changes.append(
|
|
4077
|
+
LatexDocxMergeChange(
|
|
4078
|
+
path=blk.path, lineno=blk.lineno, status="pending"
|
|
4079
|
+
)
|
|
4080
|
+
)
|
|
4081
|
+
continue
|
|
4082
|
+
current = working.get(id(blk), blk)
|
|
4083
|
+
src = "\n".join(ln.text for ln in current.lines)
|
|
4084
|
+
# The source's inline math in order, not counting any in comments
|
|
4085
|
+
math_spans = [
|
|
4086
|
+
m.span(1) if m.group(1) is not None else m.span(2)
|
|
4087
|
+
for m in calkit.latex._INLINE_MATH_RE.finditer(src)
|
|
4088
|
+
if not re.search(
|
|
4089
|
+
r"(?<!\\)%", src[src.rfind("\n", 0, m.start()) + 1 : m.start()]
|
|
4090
|
+
)
|
|
4091
|
+
]
|
|
4092
|
+
for k, sent_m, edited_m in edited_maths:
|
|
4093
|
+
if edited_m is None or k >= len(math_spans):
|
|
4094
|
+
warn(f"Can't place an inline math edit at {loc}")
|
|
4095
|
+
changes.append(
|
|
4096
|
+
LatexDocxMergeChange(
|
|
4097
|
+
path=blk.path, lineno=blk.lineno, status="unplaced"
|
|
4098
|
+
)
|
|
4099
|
+
)
|
|
4100
|
+
continue
|
|
4101
|
+
start, end = math_spans[k]
|
|
4102
|
+
math_piece = calkit.latex.Block(
|
|
4103
|
+
[calkit.latex.SourceLine(blk.path, blk.lineno, src[start:end])]
|
|
4104
|
+
)
|
|
4105
|
+
if calkit.latex.math_similarity(
|
|
4106
|
+
math_piece, edited_m
|
|
4107
|
+
) > calkit.latex.math_similarity(math_piece, sent_m):
|
|
4108
|
+
changes.append(
|
|
4109
|
+
LatexDocxMergeChange(
|
|
4110
|
+
path=blk.path,
|
|
4111
|
+
lineno=blk.lineno,
|
|
4112
|
+
status="already-applied",
|
|
4113
|
+
)
|
|
4114
|
+
)
|
|
4115
|
+
continue
|
|
4116
|
+
edited_piece = calkit.latex.apply_math_edit(
|
|
4117
|
+
math_piece, sent_m, edited_m
|
|
4118
|
+
)
|
|
4119
|
+
if edited_piece is None:
|
|
4120
|
+
warn(
|
|
4121
|
+
f"Inline math edit at {loc} doesn't fit the source; "
|
|
4122
|
+
f"apply it by hand: {' '.join(edited_m.split())}"
|
|
4123
|
+
)
|
|
4124
|
+
changes.append(
|
|
4125
|
+
LatexDocxMergeChange(
|
|
4126
|
+
path=blk.path, lineno=blk.lineno, status="unplaced"
|
|
4127
|
+
)
|
|
4128
|
+
)
|
|
4129
|
+
continue
|
|
4130
|
+
edited_text = "\n".join(edited_piece)
|
|
4131
|
+
src = src[:start] + edited_text + src[end:]
|
|
4132
|
+
# Later pieces moved by the change in length
|
|
4133
|
+
shift = len(edited_text) - (end - start)
|
|
4134
|
+
math_spans = [
|
|
4135
|
+
(a + shift, b + shift) if a >= end else (a, b)
|
|
4136
|
+
for a, b in math_spans
|
|
4137
|
+
]
|
|
4138
|
+
update(blk, src.split("\n"))
|
|
4139
|
+
current = working[id(blk)]
|
|
4140
|
+
changes.append(
|
|
4141
|
+
LatexDocxMergeChange(
|
|
4142
|
+
path=blk.path, lineno=blk.lineno, status="applied"
|
|
4143
|
+
)
|
|
4144
|
+
)
|
|
4145
|
+
typer.echo(f"Applied inline math edit at {loc}")
|
|
3318
4146
|
# Equations edited in Word go back through the LaTeX the converter
|
|
3319
4147
|
# reads them as, applied to the source's own spelling; one that can't
|
|
3320
4148
|
# be placed that way goes in as a comment on the equation
|
|
@@ -3332,7 +4160,6 @@ def merge_docx(
|
|
|
3332
4160
|
n: (next(it) if m is not None else None, authors)
|
|
3333
4161
|
for n, (m, authors) in found.items()
|
|
3334
4162
|
}
|
|
3335
|
-
working: dict[int, calkit.latex.Block] = {}
|
|
3336
4163
|
for name, (eq_tex, pending) in now.items():
|
|
3337
4164
|
sent_eq = original.equations[name]
|
|
3338
4165
|
if eq_tex is not None and calkit.latex.math_tokens(
|
|
@@ -3406,27 +4233,22 @@ def merge_docx(
|
|
|
3406
4233
|
)
|
|
3407
4234
|
)
|
|
3408
4235
|
continue
|
|
3409
|
-
|
|
3410
|
-
[
|
|
3411
|
-
calkit.latex.SourceLine(blk.path, blk.lineno + i, text)
|
|
3412
|
-
for i, text in enumerate(new_lines)
|
|
3413
|
-
]
|
|
3414
|
-
)
|
|
4236
|
+
update(blk, new_lines)
|
|
3415
4237
|
changes.append(
|
|
3416
4238
|
LatexDocxMergeChange(
|
|
3417
4239
|
path=blk.path, lineno=blk.lineno, status="applied"
|
|
3418
4240
|
)
|
|
3419
4241
|
)
|
|
3420
4242
|
typer.echo(f"Applied equation edit at {loc}")
|
|
3421
|
-
|
|
3422
|
-
|
|
3423
|
-
|
|
3424
|
-
|
|
3425
|
-
|
|
3426
|
-
|
|
3427
|
-
|
|
3428
|
-
)
|
|
4243
|
+
for blk in blks:
|
|
4244
|
+
if id(blk) in working:
|
|
4245
|
+
edits.setdefault(blk.path, []).append(
|
|
4246
|
+
(
|
|
4247
|
+
blk.lineno,
|
|
4248
|
+
len(blk.lines),
|
|
4249
|
+
[ln.text for ln in working[id(blk)].lines],
|
|
3429
4250
|
)
|
|
4251
|
+
)
|
|
3430
4252
|
files = {
|
|
3431
4253
|
p: Path(p).read_text(encoding="utf-8").split("\n")
|
|
3432
4254
|
for p in {ln.path for ln in lines}
|