calkit-python 0.47.2__py3-none-any.whl → 0.47.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- calkit/cli/latex.py +407 -40
- calkit/cli/main/core.py +20 -5
- calkit/conda.py +68 -7
- calkit/environments.py +11 -1
- calkit/git.py +0 -21
- calkit/latex.py +42 -23
- calkit/models/pipeline.py +63 -43
- calkit/pipeline.py +86 -27
- calkit/tests/cli/main/test_core.py +40 -0
- calkit/tests/cli/test_latex.py +229 -0
- calkit/tests/models/test_pipeline.py +34 -6
- calkit/tests/test_conda.py +59 -0
- calkit/tests/test_pipeline.py +99 -20
- {calkit_python-0.47.2.data → calkit_python-0.47.4.data}/data/share/jupyter/labextensions/calkit/package.json +5 -5
- {calkit_python-0.47.2.data → calkit_python-0.47.4.data}/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig +4 -4
- calkit_python-0.47.4.data/data/share/jupyter/labextensions/calkit/static/506.c34b070098184e33.js +156 -0
- calkit_python-0.47.4.data/data/share/jupyter/labextensions/calkit/static/616.528427eb54a4a0c0.js +1 -0
- calkit_python-0.47.4.data/data/share/jupyter/labextensions/calkit/static/740.6bf87276e9e5788a.js +1 -0
- calkit_python-0.47.4.data/data/share/jupyter/labextensions/calkit/static/899.f9b9f9bf705a6493.js +2632 -0
- calkit_python-0.47.4.data/data/share/jupyter/labextensions/calkit/static/935.46ecc6bf99aa593a.js +1 -0
- calkit_python-0.47.4.data/data/share/jupyter/labextensions/calkit/static/remoteEntry.109a3a8379365e59.js +1 -0
- {calkit_python-0.47.2.data → calkit_python-0.47.4.data}/data/share/jupyter/labextensions/calkit/static/style.js +1 -1
- {calkit_python-0.47.2.dist-info → calkit_python-0.47.4.dist-info}/METADATA +1 -1
- {calkit_python-0.47.2.dist-info → calkit_python-0.47.4.dist-info}/RECORD +31 -32
- calkit_python-0.47.2.data/data/share/jupyter/labextensions/calkit/static/502.9a2c5772a15466e923ef.js +0 -1
- calkit_python-0.47.2.data/data/share/jupyter/labextensions/calkit/static/695.2c41003a452d43d2b358.js +0 -1
- calkit_python-0.47.2.data/data/share/jupyter/labextensions/calkit/static/867.a42a046aa5108f54f8fb.js +0 -1
- calkit_python-0.47.2.data/data/share/jupyter/labextensions/calkit/static/909.6d8285ce7c45878ac508.js +0 -1
- calkit_python-0.47.2.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js +0 -2
- calkit_python-0.47.2.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js.LICENSE.txt +0 -9
- calkit_python-0.47.2.data/data/share/jupyter/labextensions/calkit/static/remoteEntry.d7a43c7948f690d37d19.js +0 -1
- {calkit_python-0.47.2.data → calkit_python-0.47.4.data}/data/etc/jupyter/jupyter_server_config.d/calkit.json +0 -0
- {calkit_python-0.47.2.data → calkit_python-0.47.4.data}/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json +0 -0
- /calkit_python-0.47.2.data/data/share/jupyter/labextensions/calkit/static/b2f1c3efe70cb539d121.png → /calkit_python-0.47.4.data/data/share/jupyter/labextensions/calkit/static/57d13a7399cec3c5.png +0 -0
- {calkit_python-0.47.2.data → calkit_python-0.47.4.data}/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json +0 -0
- {calkit_python-0.47.2.dist-info → calkit_python-0.47.4.dist-info}/WHEEL +0 -0
- {calkit_python-0.47.2.dist-info → calkit_python-0.47.4.dist-info}/entry_points.txt +0 -0
- {calkit_python-0.47.2.dist-info → calkit_python-0.47.4.dist-info}/licenses/LICENSE +0 -0
calkit/cli/latex.py
CHANGED
|
@@ -5,6 +5,7 @@ from __future__ import annotations
|
|
|
5
5
|
import hashlib
|
|
6
6
|
import json
|
|
7
7
|
import os
|
|
8
|
+
import re
|
|
8
9
|
import shutil
|
|
9
10
|
import string
|
|
10
11
|
import subprocess
|
|
@@ -16,7 +17,7 @@ from typing_extensions import Annotated
|
|
|
16
17
|
|
|
17
18
|
import calkit
|
|
18
19
|
import calkit.latex
|
|
19
|
-
from calkit.cli import raise_error
|
|
20
|
+
from calkit.cli import raise_error, warn
|
|
20
21
|
|
|
21
22
|
latex_app = typer.Typer(no_args_is_help=True)
|
|
22
23
|
|
|
@@ -300,9 +301,14 @@ def build(
|
|
|
300
301
|
|
|
301
302
|
|
|
302
303
|
DIFF_TMP_DIR = calkit.latex.DIFF_TMP_DIR
|
|
303
|
-
|
|
304
|
+
# A verbatim input named by a macro parameter, e.g., \verbatiminput{#1.wcsum}
|
|
305
|
+
# in a \newcommand, which latexdiff dies trying to open. Breaking the line
|
|
306
|
+
# after the command reads the same to TeX, but not to latexdiff's pattern.
|
|
307
|
+
_VERBATIM_PARAM_RE = re.compile(
|
|
308
|
+
r"(\\(?:verbatiminput\*?|lstinputlisting))"
|
|
309
|
+
r"(?=\s*(?:\[[^\]\n]*\])?\s*\{[^}\n]*#[0-9])"
|
|
310
|
+
)
|
|
304
311
|
get_diff_path = calkit.latex.get_diff_path
|
|
305
|
-
_is_immutable_ref = calkit.latex._is_immutable_ref
|
|
306
312
|
_default_base_ref = calkit.latex.default_base_ref
|
|
307
313
|
|
|
308
314
|
|
|
@@ -361,14 +367,68 @@ def diff(
|
|
|
361
367
|
),
|
|
362
368
|
),
|
|
363
369
|
] = None,
|
|
370
|
+
latexmk_rc_path: Annotated[
|
|
371
|
+
str | None,
|
|
372
|
+
typer.Option(
|
|
373
|
+
"--latexmk-rc",
|
|
374
|
+
"-r",
|
|
375
|
+
help="Path to a latexmkrc file to build the marked-up document with.",
|
|
376
|
+
),
|
|
377
|
+
] = None,
|
|
378
|
+
latexmk_args: Annotated[
|
|
379
|
+
list[str],
|
|
380
|
+
typer.Option(
|
|
381
|
+
"--latexmk-arg",
|
|
382
|
+
help=(
|
|
383
|
+
"Extra argument to pass through to latexmk. Repeat the option "
|
|
384
|
+
"to pass more than one."
|
|
385
|
+
),
|
|
386
|
+
),
|
|
387
|
+
] = [],
|
|
388
|
+
latexdiff_args: Annotated[
|
|
389
|
+
list[str],
|
|
390
|
+
typer.Option(
|
|
391
|
+
"--latexdiff-arg",
|
|
392
|
+
help=(
|
|
393
|
+
"Extra argument to pass through to latexdiff, e.g., "
|
|
394
|
+
"'--type=CFONT'. Changed figures are shown old and new by "
|
|
395
|
+
"default; pass '--graphics-markup=new-only' to show only the "
|
|
396
|
+
"new. Repeat the option to pass more "
|
|
397
|
+
"than one."
|
|
398
|
+
),
|
|
399
|
+
),
|
|
400
|
+
] = [],
|
|
401
|
+
inputs: Annotated[
|
|
402
|
+
list[str],
|
|
403
|
+
typer.Option(
|
|
404
|
+
"--input",
|
|
405
|
+
help=(
|
|
406
|
+
"File or directory the document reads. Anything in it "
|
|
407
|
+
"tracked with DVC is fetched as it was at each revision. "
|
|
408
|
+
"Defaults to the inputs detected in the document. Repeat "
|
|
409
|
+
"the option to pass more than one."
|
|
410
|
+
),
|
|
411
|
+
),
|
|
412
|
+
] = [],
|
|
413
|
+
revision_key: Annotated[
|
|
414
|
+
str | None,
|
|
415
|
+
typer.Option(
|
|
416
|
+
"--revision-key",
|
|
417
|
+
hidden=True,
|
|
418
|
+
help=(
|
|
419
|
+
"Set by the pipeline so a diff's stage reruns when either "
|
|
420
|
+
"revision's inputs change. Not used by the command."
|
|
421
|
+
),
|
|
422
|
+
),
|
|
423
|
+
] = None,
|
|
364
424
|
force: Annotated[
|
|
365
425
|
bool,
|
|
366
426
|
typer.Option(
|
|
367
427
|
"--force",
|
|
368
428
|
"-f",
|
|
369
429
|
help=(
|
|
370
|
-
"Rebuild even if
|
|
371
|
-
"
|
|
430
|
+
"Rebuild even if nothing the diff depends on has changed "
|
|
431
|
+
"since it was last built."
|
|
372
432
|
),
|
|
373
433
|
),
|
|
374
434
|
] = False,
|
|
@@ -405,7 +465,191 @@ def diff(
|
|
|
405
465
|
With the default `--to`, the newer side is the working tree, so the
|
|
406
466
|
marked-up document is built with the current figures and bibliography
|
|
407
467
|
and what's marked is what changed in the text.
|
|
468
|
+
|
|
469
|
+
A revision's DVC-tracked inputs, e.g., figures and tables a pipeline
|
|
470
|
+
generates, are fetched as they were at that revision, so each side of
|
|
471
|
+
the comparison shows its own.
|
|
408
472
|
"""
|
|
473
|
+
|
|
474
|
+
def fetch_dvc_inputs(root: str, rev: str, paths: list[str]) -> list[str]:
|
|
475
|
+
"""Fetch the DVC-tracked content of ``paths`` at ``rev`` into ``root``.
|
|
476
|
+
|
|
477
|
+
Returns each fetched path with its hash, since that content changes
|
|
478
|
+
the PDF without changing the marked-up source.
|
|
479
|
+
"""
|
|
480
|
+
from dvc.exceptions import NotDvcRepoError
|
|
481
|
+
from dvc.fs import DVCFileSystem
|
|
482
|
+
|
|
483
|
+
import calkit.dvc
|
|
484
|
+
|
|
485
|
+
def reason(e: BaseException) -> str:
|
|
486
|
+
# DVC wraps a remote's own error, e.g., needing to log in, in
|
|
487
|
+
# several layers that each say less
|
|
488
|
+
while e.__cause__ is not None:
|
|
489
|
+
e = e.__cause__
|
|
490
|
+
return str(e)
|
|
491
|
+
|
|
492
|
+
fetched: list[str] = []
|
|
493
|
+
try:
|
|
494
|
+
# A project using Calkit's own remote needs its scheme known
|
|
495
|
+
calkit.dvc.register_ck_scheme()
|
|
496
|
+
fs = DVCFileSystem(url=".", rev=rev)
|
|
497
|
+
except NotDvcRepoError:
|
|
498
|
+
return fetched
|
|
499
|
+
found: list[str] = []
|
|
500
|
+
for path in paths:
|
|
501
|
+
try:
|
|
502
|
+
found += list(fs.find(path.rstrip("/")))
|
|
503
|
+
except FileNotFoundError:
|
|
504
|
+
continue
|
|
505
|
+
except Exception as e:
|
|
506
|
+
# A directory whose listing is in neither the cache nor a
|
|
507
|
+
# reachable remote can't be expanded, but the rest of the
|
|
508
|
+
# comparison can still be built
|
|
509
|
+
warn(
|
|
510
|
+
f"Can't list {path} at {rev[:7]}, so the diff will be "
|
|
511
|
+
f"missing it: {reason(e)}"
|
|
512
|
+
)
|
|
513
|
+
for rpath in dict.fromkeys(found):
|
|
514
|
+
dvc_info = fs.info(rpath).get("dvc_info")
|
|
515
|
+
if not dvc_info:
|
|
516
|
+
continue
|
|
517
|
+
fetched.append(f"{rpath} {dvc_info.get('md5')}")
|
|
518
|
+
dest = os.path.join(root, rpath)
|
|
519
|
+
if os.path.exists(dest):
|
|
520
|
+
continue
|
|
521
|
+
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
|
522
|
+
try:
|
|
523
|
+
fs.get_file(rpath, dest)
|
|
524
|
+
except Exception as e:
|
|
525
|
+
warn(
|
|
526
|
+
f"Can't fetch {rpath} at {rev[:7]}, so the diff will be "
|
|
527
|
+
f"missing it: {reason(e)}"
|
|
528
|
+
)
|
|
529
|
+
return fetched
|
|
530
|
+
|
|
531
|
+
def copy_uncached_outputs(
|
|
532
|
+
root: str, rev: str, paths: list[str]
|
|
533
|
+
) -> list[str]:
|
|
534
|
+
"""Copy outputs DVC records but doesn't store from the working tree.
|
|
535
|
+
|
|
536
|
+
An output with caching off, e.g., a copy another stage makes, has
|
|
537
|
+
only its hash in ``dvc.lock``, so no revision's checkout can have
|
|
538
|
+
it. The working tree's copy stands in when it's the same content.
|
|
539
|
+
Returns each copied path with its hash.
|
|
540
|
+
"""
|
|
541
|
+
try:
|
|
542
|
+
lock = calkit.ryaml.load(repo.git.show(f"{rev}:dvc.lock")) or {}
|
|
543
|
+
except Exception:
|
|
544
|
+
return []
|
|
545
|
+
prefixes = [p.rstrip("/") for p in paths]
|
|
546
|
+
copied: list[str] = []
|
|
547
|
+
for stage in (lock.get("stages") or {}).values():
|
|
548
|
+
for out in stage.get("outs") or []:
|
|
549
|
+
path, md5 = out.get("path"), out.get("md5")
|
|
550
|
+
if not path or not md5 or str(md5).endswith(".dir"):
|
|
551
|
+
continue
|
|
552
|
+
if not any(
|
|
553
|
+
path == p or path.startswith(p + "/") for p in prefixes
|
|
554
|
+
):
|
|
555
|
+
continue
|
|
556
|
+
dest = os.path.join(root, path)
|
|
557
|
+
if os.path.exists(dest) or not os.path.isfile(path):
|
|
558
|
+
continue
|
|
559
|
+
if hashlib.md5(Path(path).read_bytes()).hexdigest() != md5:
|
|
560
|
+
continue
|
|
561
|
+
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
|
562
|
+
shutil.copy2(path, dest)
|
|
563
|
+
copied.append(f"{path} {md5}")
|
|
564
|
+
return copied
|
|
565
|
+
|
|
566
|
+
def break_verbatim_params(root: str) -> None:
|
|
567
|
+
"""Break verbatim inputs named by macro parameters in a checkout.
|
|
568
|
+
|
|
569
|
+
Done to the checked-out files directly rather than with latexdiff's
|
|
570
|
+
--filter-script, which hands text to the script and reads it back
|
|
571
|
+
with no encoding, mangling anything outside Latin-1, e.g., a curly
|
|
572
|
+
apostrophe.
|
|
573
|
+
"""
|
|
574
|
+
sources = [tex_file] + [
|
|
575
|
+
path
|
|
576
|
+
for path in calkit.latex.detect_inputs(tex_file, wdir=root)
|
|
577
|
+
if Path(path).suffix in calkit.latex._SOURCE_EXTS
|
|
578
|
+
]
|
|
579
|
+
for source in sources:
|
|
580
|
+
source_path = Path(root, source)
|
|
581
|
+
try:
|
|
582
|
+
text = source_path.read_text(encoding="utf-8")
|
|
583
|
+
except (OSError, UnicodeDecodeError):
|
|
584
|
+
continue
|
|
585
|
+
new_text = _VERBATIM_PARAM_RE.sub("\\1%\n", text)
|
|
586
|
+
if new_text != text:
|
|
587
|
+
source_path.write_text(new_text, encoding="utf-8")
|
|
588
|
+
|
|
589
|
+
def default_inputs(root: str) -> list[str]:
|
|
590
|
+
"""What to fetch or note for a side when no inputs were given.
|
|
591
|
+
|
|
592
|
+
Detection finds a file by its own name or pointer, but a file in a
|
|
593
|
+
directory DVC tracks as a whole has only the directory's pointer,
|
|
594
|
+
which can't say what's inside, so those directories beside the
|
|
595
|
+
document are included whole.
|
|
596
|
+
"""
|
|
597
|
+
doc_dir = Path(root, os.path.dirname(tex_file))
|
|
598
|
+
pointed = [
|
|
599
|
+
Path(os.path.relpath(p, root)).as_posix().removesuffix(".dvc")
|
|
600
|
+
for p in sorted(doc_dir.rglob("*.dvc"))
|
|
601
|
+
if p.is_file()
|
|
602
|
+
]
|
|
603
|
+
return calkit.latex.detect_inputs(tex_file, wdir=root) + pointed
|
|
604
|
+
|
|
605
|
+
def point_changed_figures_at_base(base_root: str, head_root: str) -> None:
|
|
606
|
+
"""Make the older side's changed figures refer to its own copies.
|
|
607
|
+
|
|
608
|
+
The marked-up document is built beside the newer side, where the
|
|
609
|
+
older side's figure names find the newer figures. Left alone, a
|
|
610
|
+
figure that changed would show as it is now on both sides, and
|
|
611
|
+
latexdiff wouldn't mark it, since its name didn't change.
|
|
612
|
+
"""
|
|
613
|
+
import filecmp
|
|
614
|
+
|
|
615
|
+
tex_dir = os.path.dirname(tex_file)
|
|
616
|
+
build_dir = os.path.join(head_root, tex_dir)
|
|
617
|
+
graphic_re = re.compile(
|
|
618
|
+
r"(\\includegraphics\*?\s*(?:\[[^\]]*\])*\s*\{)([^}#\\]+)(\})"
|
|
619
|
+
)
|
|
620
|
+
|
|
621
|
+
def repoint(match: re.Match[str]) -> str:
|
|
622
|
+
name = match.group(2).strip()
|
|
623
|
+
exts = calkit.latex._GRAPHICS_EXTS
|
|
624
|
+
for candidate in [name] + [name + ext for ext in exts]:
|
|
625
|
+
rel = os.path.normpath(os.path.join(tex_dir, candidate))
|
|
626
|
+
base_path = os.path.join(base_root, rel)
|
|
627
|
+
if not os.path.isfile(base_path):
|
|
628
|
+
continue
|
|
629
|
+
head_path = os.path.join(head_root, rel)
|
|
630
|
+
if os.path.isfile(head_path) and filecmp.cmp(
|
|
631
|
+
base_path, head_path, shallow=False
|
|
632
|
+
):
|
|
633
|
+
return match.group(0)
|
|
634
|
+
new_name = Path(os.path.relpath(base_path, build_dir))
|
|
635
|
+
return match.group(1) + new_name.as_posix() + match.group(3)
|
|
636
|
+
return match.group(0)
|
|
637
|
+
|
|
638
|
+
sources = [tex_file] + [
|
|
639
|
+
path
|
|
640
|
+
for path in calkit.latex.detect_inputs(tex_file, wdir=base_root)
|
|
641
|
+
if Path(path).suffix in calkit.latex._SOURCE_EXTS
|
|
642
|
+
]
|
|
643
|
+
for source in sources:
|
|
644
|
+
source_path = Path(base_root, source)
|
|
645
|
+
try:
|
|
646
|
+
text = source_path.read_text(encoding="utf-8")
|
|
647
|
+
except (OSError, UnicodeDecodeError):
|
|
648
|
+
continue
|
|
649
|
+
new_text = graphic_re.sub(repoint, text)
|
|
650
|
+
if new_text != text:
|
|
651
|
+
source_path.write_text(new_text, encoding="utf-8")
|
|
652
|
+
|
|
409
653
|
repo = calkit.git.get_repo()
|
|
410
654
|
if repo.bare:
|
|
411
655
|
raise_error("This is not a working Git repo")
|
|
@@ -424,16 +668,8 @@ def diff(
|
|
|
424
668
|
to_ref=to_ref,
|
|
425
669
|
output_dir=output_dir,
|
|
426
670
|
)
|
|
427
|
-
# A comparison between two revisions that can't move is the same
|
|
428
|
-
# comparison forever, and LaTeX writes a timestamp into every PDF, so
|
|
429
|
-
# rebuilding one would change the file without changing what it says
|
|
430
|
-
fixed = _is_immutable_ref(repo, from_ref) and _is_immutable_ref(
|
|
431
|
-
repo, to_ref
|
|
432
|
-
)
|
|
433
|
-
if fixed and os.path.isfile(output) and not force:
|
|
434
|
-
typer.echo(f"{output} is already built; use --force to rebuild it")
|
|
435
|
-
return
|
|
436
671
|
checkouts: dict[str, str] = {}
|
|
672
|
+
shas: dict[str, str] = {}
|
|
437
673
|
try:
|
|
438
674
|
for name, ref in [("base", from_ref), ("head", to_ref)]:
|
|
439
675
|
if ref is None:
|
|
@@ -441,6 +677,7 @@ def diff(
|
|
|
441
677
|
sha = calkit.git.resolve_ref(repo, ref)
|
|
442
678
|
if sha is None:
|
|
443
679
|
raise_error(f"Git ref '{ref}' was not found")
|
|
680
|
+
shas[name] = sha
|
|
444
681
|
# A worktree, not a temp directory: a document is rarely one
|
|
445
682
|
# file, and \input needs the rest of them as they were then
|
|
446
683
|
path = os.path.join(DIFF_TMP_DIR, name)
|
|
@@ -465,12 +702,43 @@ def diff(
|
|
|
465
702
|
if not os.path.isfile(side):
|
|
466
703
|
raise_error(f"{tex_file} does not exist at {ref}")
|
|
467
704
|
sides[name] = side
|
|
705
|
+
# The working tree already has its DVC-tracked files, if pulled
|
|
706
|
+
head_root = checkouts.get("head", ".")
|
|
707
|
+
context: list[str] = []
|
|
708
|
+
for name, root in checkouts.items():
|
|
709
|
+
paths = inputs or default_inputs(root)
|
|
710
|
+
context += fetch_dvc_inputs(root, rev=shas[name], paths=paths)
|
|
711
|
+
context += copy_uncached_outputs(root, rev=shas[name], paths=paths)
|
|
712
|
+
if "head" not in checkouts:
|
|
713
|
+
# The working tree has no revision to name its files by, so
|
|
714
|
+
# note what's there instead
|
|
715
|
+
for path in inputs or default_inputs("."):
|
|
716
|
+
if os.path.isfile(path):
|
|
717
|
+
files = [Path(path)]
|
|
718
|
+
else:
|
|
719
|
+
files = sorted(
|
|
720
|
+
p for p in Path(path).rglob("*") if p.is_file()
|
|
721
|
+
)
|
|
722
|
+
for file_path in files:
|
|
723
|
+
stat = file_path.stat()
|
|
724
|
+
context.append(
|
|
725
|
+
f"{file_path.as_posix()} {stat.st_size} "
|
|
726
|
+
f"{stat.st_mtime_ns}"
|
|
727
|
+
)
|
|
728
|
+
for root in checkouts.values():
|
|
729
|
+
break_verbatim_params(root)
|
|
730
|
+
point_changed_figures_at_base(checkouts["base"], head_root)
|
|
468
731
|
_build_diff(
|
|
469
732
|
base_tex=sides["base"],
|
|
470
733
|
head_tex=sides["head"],
|
|
471
734
|
tex_file=tex_file,
|
|
735
|
+
head_root=head_root,
|
|
472
736
|
output=output,
|
|
473
737
|
environment=environment,
|
|
738
|
+
latexmk_rc_path=latexmk_rc_path,
|
|
739
|
+
latexmk_args=latexmk_args,
|
|
740
|
+
latexdiff_args=latexdiff_args,
|
|
741
|
+
context=context,
|
|
474
742
|
no_check=no_check,
|
|
475
743
|
keep_tex=keep_tex,
|
|
476
744
|
force=force,
|
|
@@ -481,19 +749,23 @@ def diff(
|
|
|
481
749
|
_remove_worktree(path)
|
|
482
750
|
|
|
483
751
|
|
|
484
|
-
def _marked_up_digest(marked_up: bytes) -> str:
|
|
752
|
+
def _marked_up_digest(marked_up: bytes, context: list[str] = []) -> str:
|
|
485
753
|
"""Hash a marked-up document by what actually determines the PDF.
|
|
486
754
|
|
|
487
755
|
latexdiff writes the two inputs' paths and modification times into a
|
|
488
756
|
header comment, and the older side is a fresh checkout every time, so
|
|
489
757
|
hashing the file as-is would say "changed" on every run when nothing
|
|
490
758
|
had. Those lines are comments; the PDF doesn't depend on them.
|
|
759
|
+
|
|
760
|
+
``context`` is anything else the PDF depends on, e.g., the hashes of
|
|
761
|
+
DVC-tracked figures and the options it's built with.
|
|
491
762
|
"""
|
|
492
763
|
kept = [
|
|
493
764
|
line
|
|
494
765
|
for line in marked_up.splitlines(keepends=True)
|
|
495
766
|
if not line.startswith((b"%DIF DEL ", b"%DIF ADD "))
|
|
496
767
|
]
|
|
768
|
+
kept += [f"{item}\n".encode() for item in context]
|
|
497
769
|
return hashlib.sha256(b"".join(kept)).hexdigest()
|
|
498
770
|
|
|
499
771
|
|
|
@@ -519,28 +791,75 @@ def _build_diff(
|
|
|
519
791
|
base_tex: str,
|
|
520
792
|
head_tex: str,
|
|
521
793
|
tex_file: str,
|
|
794
|
+
head_root: str,
|
|
522
795
|
output: str,
|
|
523
796
|
environment: str | None,
|
|
797
|
+
latexmk_rc_path: str | None,
|
|
798
|
+
latexmk_args: list[str],
|
|
799
|
+
latexdiff_args: list[str],
|
|
800
|
+
context: list[str],
|
|
524
801
|
no_check: bool,
|
|
525
802
|
keep_tex: bool,
|
|
526
803
|
force: bool,
|
|
527
804
|
verbose: bool,
|
|
528
805
|
) -> None:
|
|
529
806
|
"""Mark up one document against another and build the result."""
|
|
530
|
-
# Built beside the
|
|
531
|
-
#
|
|
532
|
-
#
|
|
533
|
-
# for it, but a DVC-tracked figure isn't in Git: a checkout has the
|
|
534
|
-
# pointer file and not the image, and the marked-up document would
|
|
535
|
-
# come out with its figures missing.
|
|
807
|
+
# Built beside the newer side, so \graphicspath, \bibliography, and
|
|
808
|
+
# relative \includegraphics resolve the way they do for the real thing,
|
|
809
|
+
# against that revision's own files
|
|
536
810
|
tex_dir = os.path.dirname(tex_file) or "."
|
|
811
|
+
build_dir = os.path.normpath(os.path.join(head_root, tex_dir))
|
|
537
812
|
stem = Path(tex_file).stem
|
|
538
|
-
diff_tex = os.path.join(
|
|
813
|
+
diff_tex = os.path.join(build_dir, f"{stem}-diff.tex")
|
|
814
|
+
# Where --keep-tex leaves it: beside the document, since a checkout is
|
|
815
|
+
# removed afterwards
|
|
816
|
+
kept_tex = os.path.normpath(os.path.join(tex_dir, f"{stem}-diff.tex"))
|
|
817
|
+
aux_dir = os.path.join(build_dir, calkit.latex.DIFF_AUX_DIRNAME)
|
|
539
818
|
try:
|
|
540
819
|
# --flatten pulls \input and \include files into one document on
|
|
541
820
|
# each side, so a multi-file paper compares as a whole
|
|
821
|
+
latexdiff_cmd = ["latexdiff", "--flatten", "--encoding=utf8"]
|
|
822
|
+
# A checkout's verbatim inputs named by macro parameters are already
|
|
823
|
+
# broken, so this only finds the working tree's, which can't be
|
|
824
|
+
# edited. A filter breaks those instead, though latexdiff's
|
|
825
|
+
# --filter-script mangles anything outside Latin-1.
|
|
826
|
+
sources = [base_tex, head_tex] + [
|
|
827
|
+
path
|
|
828
|
+
for side in (base_tex, head_tex)
|
|
829
|
+
for path in calkit.latex.detect_inputs(side)
|
|
830
|
+
if Path(path).suffix in calkit.latex._SOURCE_EXTS
|
|
831
|
+
]
|
|
832
|
+
if any(
|
|
833
|
+
os.path.isfile(path)
|
|
834
|
+
and _VERBATIM_PARAM_RE.search(
|
|
835
|
+
Path(path).read_text(encoding="utf-8", errors="replace")
|
|
836
|
+
)
|
|
837
|
+
for path in sources
|
|
838
|
+
):
|
|
839
|
+
filter_path = Path(DIFF_TMP_DIR, "verbatim-param-filter.pl")
|
|
840
|
+
os.makedirs(filter_path.parent, exist_ok=True)
|
|
841
|
+
# latexdiff appends a newline to what it sends, so drop it
|
|
842
|
+
filter_path.write_text(
|
|
843
|
+
"local $/;\n"
|
|
844
|
+
"my $text = <STDIN>;\n"
|
|
845
|
+
"$text =~ s/\\n\\z//;\n"
|
|
846
|
+
"$text =~ s/(\\\\(?:verbatiminput\\*?|lstinputlisting))"
|
|
847
|
+
"(?=\\s*(?:\\[[^\\]\\n]*\\])?\\s*\\{[^}\\n]*#[0-9])/$1%\\n/g;\n"
|
|
848
|
+
"print $text;\n"
|
|
849
|
+
)
|
|
850
|
+
latexdiff_cmd.append(
|
|
851
|
+
f"--filter-script=perl {filter_path.as_posix()}"
|
|
852
|
+
)
|
|
853
|
+
# Each side has its own revision's figures, so a changed one is
|
|
854
|
+
# shown both ways rather than only as it is now, unless the user
|
|
855
|
+
# chose otherwise
|
|
856
|
+
if not any(a.startswith("--graphics-markup") for a in latexdiff_args):
|
|
857
|
+
latexdiff_cmd.append("--graphics-markup=both")
|
|
858
|
+
# User pass-through args come last so they can override Calkit's
|
|
859
|
+
# defaults
|
|
860
|
+
latexdiff_cmd += latexdiff_args
|
|
542
861
|
cmd = _tex_cmd(
|
|
543
|
-
|
|
862
|
+
latexdiff_cmd + [base_tex, head_tex],
|
|
544
863
|
environment=environment,
|
|
545
864
|
no_check=no_check,
|
|
546
865
|
verbose=verbose,
|
|
@@ -554,14 +873,32 @@ def _build_diff(
|
|
|
554
873
|
"latexdiff was not found; it ships with TeX Live, so a "
|
|
555
874
|
"minimal install may not have it"
|
|
556
875
|
)
|
|
557
|
-
except subprocess.CalledProcessError:
|
|
558
|
-
raise_error("latexdiff failed")
|
|
559
|
-
# The
|
|
560
|
-
#
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
876
|
+
except subprocess.CalledProcessError as e:
|
|
877
|
+
raise_error(f"latexdiff failed with exit status {e.returncode}")
|
|
878
|
+
# The newer side's copy, which is what the document was built with
|
|
879
|
+
# at that revision
|
|
880
|
+
rc_path = None
|
|
881
|
+
rc_hash = None
|
|
882
|
+
if latexmk_rc_path is not None:
|
|
883
|
+
rc_path = os.path.join(head_root, latexmk_rc_path)
|
|
884
|
+
if not os.path.isfile(rc_path):
|
|
885
|
+
rc_path = latexmk_rc_path
|
|
886
|
+
rc_path = Path(os.path.normpath(rc_path)).as_posix()
|
|
887
|
+
if os.path.isfile(rc_path):
|
|
888
|
+
rc_hash = hashlib.sha256(
|
|
889
|
+
Path(rc_path).read_bytes()
|
|
890
|
+
).hexdigest()
|
|
891
|
+
# The PDF is a function of this marked-up source and how it's built,
|
|
892
|
+
# so if neither has changed there's nothing to build. Worth checking
|
|
893
|
+
# because the common case produces nothing at all: on the default
|
|
894
|
+
# branch the merge base is usually HEAD, so the comparison is empty,
|
|
895
|
+
# and latexmk is the expensive half of this.
|
|
896
|
+
digest = _marked_up_digest(
|
|
897
|
+
marked_up,
|
|
898
|
+
context=context
|
|
899
|
+
+ [f"latexmkrc {rc_path} {rc_hash}"]
|
|
900
|
+
+ latexmk_args,
|
|
901
|
+
)
|
|
565
902
|
state_path = calkit.latex.diff_state_path(output)
|
|
566
903
|
if (
|
|
567
904
|
not force
|
|
@@ -572,11 +909,14 @@ def _build_diff(
|
|
|
572
909
|
return
|
|
573
910
|
with open(diff_tex, "wb") as f:
|
|
574
911
|
f.write(marked_up)
|
|
575
|
-
aux_dir = DIFF_AUX_DIR
|
|
576
912
|
os.makedirs(aux_dir, exist_ok=True)
|
|
577
|
-
rel_aux =
|
|
578
|
-
latexmk_cmd = [
|
|
579
|
-
|
|
913
|
+
rel_aux = calkit.latex.DIFF_AUX_DIRNAME
|
|
914
|
+
latexmk_cmd = ["latexmk"]
|
|
915
|
+
# First, since latexmk reads an rc file where it appears, so the
|
|
916
|
+
# directories below override any the rc file sets
|
|
917
|
+
if rc_path is not None:
|
|
918
|
+
latexmk_cmd += ["-r", rc_path]
|
|
919
|
+
latexmk_cmd += [
|
|
580
920
|
"-pdf",
|
|
581
921
|
"-cd",
|
|
582
922
|
"-interaction=nonstopmode",
|
|
@@ -585,6 +925,9 @@ def _build_diff(
|
|
|
585
925
|
]
|
|
586
926
|
if not verbose:
|
|
587
927
|
latexmk_cmd.append("-silent")
|
|
928
|
+
# User pass-through args come last so they can override Calkit's
|
|
929
|
+
# defaults
|
|
930
|
+
latexmk_cmd += latexmk_args
|
|
588
931
|
latexmk_cmd.append(diff_tex)
|
|
589
932
|
cmd = _tex_cmd(
|
|
590
933
|
latexmk_cmd,
|
|
@@ -596,23 +939,47 @@ def _build_diff(
|
|
|
596
939
|
typer.echo("Building the marked-up document")
|
|
597
940
|
try:
|
|
598
941
|
subprocess.check_call(cmd)
|
|
599
|
-
except subprocess.CalledProcessError:
|
|
942
|
+
except subprocess.CalledProcessError as e:
|
|
943
|
+
# -silent hides why, so show the errors LaTeX logged
|
|
944
|
+
log_path = Path(aux_dir, f"{stem}-diff.log")
|
|
945
|
+
try:
|
|
946
|
+
log_lines = log_path.read_text(
|
|
947
|
+
encoding="utf-8", errors="replace"
|
|
948
|
+
).splitlines()
|
|
949
|
+
except OSError:
|
|
950
|
+
log_lines = []
|
|
951
|
+
shown: list[str] = []
|
|
952
|
+
for i, line in enumerate(log_lines):
|
|
953
|
+
if line.startswith("!"):
|
|
954
|
+
shown += log_lines[i : i + 3]
|
|
955
|
+
excerpt = shown[:60] or log_lines[-20:]
|
|
956
|
+
if excerpt:
|
|
957
|
+
typer.echo(f"From {log_path.as_posix()}:", err=True)
|
|
958
|
+
typer.echo("\n".join(excerpt), err=True)
|
|
600
959
|
raise_error(
|
|
601
|
-
"latexmk failed on the marked-up document
|
|
602
|
-
"--keep-tex to inspect it"
|
|
960
|
+
"latexmk failed on the marked-up document with exit status "
|
|
961
|
+
f"{e.returncode}; rerun with --keep-tex to inspect it"
|
|
603
962
|
)
|
|
604
963
|
built = os.path.join(aux_dir, f"{stem}-diff.pdf")
|
|
605
964
|
if not os.path.isfile(built):
|
|
606
965
|
raise_error("latexmk did not produce a PDF")
|
|
607
966
|
os.makedirs(os.path.dirname(output) or ".", exist_ok=True)
|
|
608
967
|
shutil.move(built, output)
|
|
968
|
+
# A checkout is removed afterwards, but the working tree's document
|
|
969
|
+
# directory is the user's, so nothing is left behind in it
|
|
970
|
+
if os.path.abspath(head_root) == os.path.abspath("."):
|
|
971
|
+
shutil.rmtree(aux_dir, ignore_errors=True)
|
|
609
972
|
os.makedirs(os.path.dirname(state_path), exist_ok=True)
|
|
610
973
|
with open(state_path, "w") as f:
|
|
611
974
|
f.write(digest)
|
|
612
975
|
typer.echo(f"Wrote {output}")
|
|
613
976
|
finally:
|
|
614
|
-
if
|
|
615
|
-
os.
|
|
977
|
+
if os.path.isfile(diff_tex):
|
|
978
|
+
in_place = os.path.abspath(diff_tex) == os.path.abspath(kept_tex)
|
|
979
|
+
if keep_tex and not in_place:
|
|
980
|
+
shutil.copy(diff_tex, kept_tex)
|
|
981
|
+
if not keep_tex or not in_place:
|
|
982
|
+
os.remove(diff_tex)
|
|
616
983
|
|
|
617
984
|
|
|
618
985
|
@latex_app.command(name="to-docx")
|
calkit/cli/main/core.py
CHANGED
|
@@ -1672,6 +1672,14 @@ def push(
|
|
|
1672
1672
|
git_cmd = ["git", "push"]
|
|
1673
1673
|
if not no_recursive and "--recurse-submodules" not in git_args:
|
|
1674
1674
|
git_cmd.append("--recurse-submodules=on-demand")
|
|
1675
|
+
# A plain `git push` leaves tags behind, so annotated tags end
|
|
1676
|
+
# up only on the machine that made them
|
|
1677
|
+
if not {
|
|
1678
|
+
"--follow-tags",
|
|
1679
|
+
"--no-follow-tags",
|
|
1680
|
+
"--tags",
|
|
1681
|
+
}.intersection(git_args):
|
|
1682
|
+
git_cmd.append("--follow-tags")
|
|
1675
1683
|
subprocess.check_call(git_cmd + git_args)
|
|
1676
1684
|
except subprocess.CalledProcessError:
|
|
1677
1685
|
raise_error("Git push failed")
|
|
@@ -2769,9 +2777,13 @@ def run(
|
|
|
2769
2777
|
dvc_data_status_before.pop("git", None) # Remove git status
|
|
2770
2778
|
if targets is None:
|
|
2771
2779
|
targets = []
|
|
2772
|
-
|
|
2773
|
-
|
|
2774
|
-
|
|
2780
|
+
try:
|
|
2781
|
+
args, isolated_sp_targets = calkit.pipeline.translate_run_targets(
|
|
2782
|
+
deepcopy(targets), ck_info=ck_info
|
|
2783
|
+
)
|
|
2784
|
+
except ValueError as e:
|
|
2785
|
+
os.environ.pop("CALKIT_PIPELINE_RUNNING", None)
|
|
2786
|
+
raise_error(str(e))
|
|
2775
2787
|
# Extract any boolean args
|
|
2776
2788
|
for name in [
|
|
2777
2789
|
"quiet",
|
|
@@ -3470,8 +3482,11 @@ def run_in_env(
|
|
|
3470
3482
|
typer.echo(f"Running command: {docker_cmd}")
|
|
3471
3483
|
try:
|
|
3472
3484
|
subprocess.check_call(docker_cmd, cwd=wdir)
|
|
3473
|
-
except subprocess.CalledProcessError:
|
|
3474
|
-
raise_error(
|
|
3485
|
+
except subprocess.CalledProcessError as e:
|
|
3486
|
+
raise_error(
|
|
3487
|
+
"Failed to run in Docker environment: command exited with "
|
|
3488
|
+
f"status {e.returncode}"
|
|
3489
|
+
)
|
|
3475
3490
|
elif env["kind"] == "conda":
|
|
3476
3491
|
with open(env["path"]) as f:
|
|
3477
3492
|
conda_env = calkit.ryaml.load(f)
|