calkit-python 0.47.3__py3-none-any.whl → 0.47.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- calkit/cli/latex.py +378 -48
- calkit/cli/main/core.py +12 -5
- calkit/environments.py +11 -1
- calkit/git.py +0 -21
- calkit/latex.py +42 -23
- calkit/models/pipeline.py +63 -43
- calkit/pipeline.py +86 -27
- calkit/tests/cli/test_latex.py +214 -0
- calkit/tests/models/test_pipeline.py +34 -6
- calkit/tests/test_pipeline.py +99 -20
- {calkit_python-0.47.3.data → calkit_python-0.47.4.data}/data/share/jupyter/labextensions/calkit/package.json +5 -5
- {calkit_python-0.47.3.data → calkit_python-0.47.4.data}/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig +4 -4
- calkit_python-0.47.4.data/data/share/jupyter/labextensions/calkit/static/506.c34b070098184e33.js +156 -0
- calkit_python-0.47.4.data/data/share/jupyter/labextensions/calkit/static/616.528427eb54a4a0c0.js +1 -0
- calkit_python-0.47.4.data/data/share/jupyter/labextensions/calkit/static/740.6bf87276e9e5788a.js +1 -0
- calkit_python-0.47.4.data/data/share/jupyter/labextensions/calkit/static/899.f9b9f9bf705a6493.js +2632 -0
- calkit_python-0.47.4.data/data/share/jupyter/labextensions/calkit/static/935.46ecc6bf99aa593a.js +1 -0
- calkit_python-0.47.4.data/data/share/jupyter/labextensions/calkit/static/remoteEntry.109a3a8379365e59.js +1 -0
- {calkit_python-0.47.3.data → calkit_python-0.47.4.data}/data/share/jupyter/labextensions/calkit/static/style.js +1 -1
- {calkit_python-0.47.3.dist-info → calkit_python-0.47.4.dist-info}/METADATA +1 -1
- {calkit_python-0.47.3.dist-info → calkit_python-0.47.4.dist-info}/RECORD +28 -29
- calkit_python-0.47.3.data/data/share/jupyter/labextensions/calkit/static/502.9a2c5772a15466e923ef.js +0 -1
- calkit_python-0.47.3.data/data/share/jupyter/labextensions/calkit/static/695.2c41003a452d43d2b358.js +0 -1
- calkit_python-0.47.3.data/data/share/jupyter/labextensions/calkit/static/867.a42a046aa5108f54f8fb.js +0 -1
- calkit_python-0.47.3.data/data/share/jupyter/labextensions/calkit/static/909.6d8285ce7c45878ac508.js +0 -1
- calkit_python-0.47.3.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js +0 -2
- calkit_python-0.47.3.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js.LICENSE.txt +0 -9
- calkit_python-0.47.3.data/data/share/jupyter/labextensions/calkit/static/remoteEntry.d7a43c7948f690d37d19.js +0 -1
- {calkit_python-0.47.3.data → calkit_python-0.47.4.data}/data/etc/jupyter/jupyter_server_config.d/calkit.json +0 -0
- {calkit_python-0.47.3.data → calkit_python-0.47.4.data}/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json +0 -0
- /calkit_python-0.47.3.data/data/share/jupyter/labextensions/calkit/static/b2f1c3efe70cb539d121.png → /calkit_python-0.47.4.data/data/share/jupyter/labextensions/calkit/static/57d13a7399cec3c5.png +0 -0
- {calkit_python-0.47.3.data → calkit_python-0.47.4.data}/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json +0 -0
- {calkit_python-0.47.3.dist-info → calkit_python-0.47.4.dist-info}/WHEEL +0 -0
- {calkit_python-0.47.3.dist-info → calkit_python-0.47.4.dist-info}/entry_points.txt +0 -0
- {calkit_python-0.47.3.dist-info → calkit_python-0.47.4.dist-info}/licenses/LICENSE +0 -0
calkit/cli/latex.py
CHANGED
|
@@ -17,7 +17,7 @@ from typing_extensions import Annotated
|
|
|
17
17
|
|
|
18
18
|
import calkit
|
|
19
19
|
import calkit.latex
|
|
20
|
-
from calkit.cli import raise_error
|
|
20
|
+
from calkit.cli import raise_error, warn
|
|
21
21
|
|
|
22
22
|
latex_app = typer.Typer(no_args_is_help=True)
|
|
23
23
|
|
|
@@ -301,9 +301,14 @@ def build(
|
|
|
301
301
|
|
|
302
302
|
|
|
303
303
|
DIFF_TMP_DIR = calkit.latex.DIFF_TMP_DIR
|
|
304
|
-
|
|
304
|
+
# A verbatim input named by a macro parameter, e.g., \verbatiminput{#1.wcsum}
|
|
305
|
+
# in a \newcommand, which latexdiff dies trying to open. Breaking the line
|
|
306
|
+
# after the command reads the same to TeX, but not to latexdiff's pattern.
|
|
307
|
+
_VERBATIM_PARAM_RE = re.compile(
|
|
308
|
+
r"(\\(?:verbatiminput\*?|lstinputlisting))"
|
|
309
|
+
r"(?=\s*(?:\[[^\]\n]*\])?\s*\{[^}\n]*#[0-9])"
|
|
310
|
+
)
|
|
305
311
|
get_diff_path = calkit.latex.get_diff_path
|
|
306
|
-
_is_immutable_ref = calkit.latex._is_immutable_ref
|
|
307
312
|
_default_base_ref = calkit.latex.default_base_ref
|
|
308
313
|
|
|
309
314
|
|
|
@@ -362,14 +367,68 @@ def diff(
|
|
|
362
367
|
),
|
|
363
368
|
),
|
|
364
369
|
] = None,
|
|
370
|
+
latexmk_rc_path: Annotated[
|
|
371
|
+
str | None,
|
|
372
|
+
typer.Option(
|
|
373
|
+
"--latexmk-rc",
|
|
374
|
+
"-r",
|
|
375
|
+
help="Path to a latexmkrc file to build the marked-up document with.",
|
|
376
|
+
),
|
|
377
|
+
] = None,
|
|
378
|
+
latexmk_args: Annotated[
|
|
379
|
+
list[str],
|
|
380
|
+
typer.Option(
|
|
381
|
+
"--latexmk-arg",
|
|
382
|
+
help=(
|
|
383
|
+
"Extra argument to pass through to latexmk. Repeat the option "
|
|
384
|
+
"to pass more than one."
|
|
385
|
+
),
|
|
386
|
+
),
|
|
387
|
+
] = [],
|
|
388
|
+
latexdiff_args: Annotated[
|
|
389
|
+
list[str],
|
|
390
|
+
typer.Option(
|
|
391
|
+
"--latexdiff-arg",
|
|
392
|
+
help=(
|
|
393
|
+
"Extra argument to pass through to latexdiff, e.g., "
|
|
394
|
+
"'--type=CFONT'. Changed figures are shown old and new by "
|
|
395
|
+
"default; pass '--graphics-markup=new-only' to show only the "
|
|
396
|
+
"new. Repeat the option to pass more "
|
|
397
|
+
"than one."
|
|
398
|
+
),
|
|
399
|
+
),
|
|
400
|
+
] = [],
|
|
401
|
+
inputs: Annotated[
|
|
402
|
+
list[str],
|
|
403
|
+
typer.Option(
|
|
404
|
+
"--input",
|
|
405
|
+
help=(
|
|
406
|
+
"File or directory the document reads. Anything in it "
|
|
407
|
+
"tracked with DVC is fetched as it was at each revision. "
|
|
408
|
+
"Defaults to the inputs detected in the document. Repeat "
|
|
409
|
+
"the option to pass more than one."
|
|
410
|
+
),
|
|
411
|
+
),
|
|
412
|
+
] = [],
|
|
413
|
+
revision_key: Annotated[
|
|
414
|
+
str | None,
|
|
415
|
+
typer.Option(
|
|
416
|
+
"--revision-key",
|
|
417
|
+
hidden=True,
|
|
418
|
+
help=(
|
|
419
|
+
"Set by the pipeline so a diff's stage reruns when either "
|
|
420
|
+
"revision's inputs change. Not used by the command."
|
|
421
|
+
),
|
|
422
|
+
),
|
|
423
|
+
] = None,
|
|
365
424
|
force: Annotated[
|
|
366
425
|
bool,
|
|
367
426
|
typer.Option(
|
|
368
427
|
"--force",
|
|
369
428
|
"-f",
|
|
370
429
|
help=(
|
|
371
|
-
"Rebuild even if
|
|
372
|
-
"
|
|
430
|
+
"Rebuild even if nothing the diff depends on has changed "
|
|
431
|
+
"since it was last built."
|
|
373
432
|
),
|
|
374
433
|
),
|
|
375
434
|
] = False,
|
|
@@ -406,7 +465,191 @@ def diff(
|
|
|
406
465
|
With the default `--to`, the newer side is the working tree, so the
|
|
407
466
|
marked-up document is built with the current figures and bibliography
|
|
408
467
|
and what's marked is what changed in the text.
|
|
468
|
+
|
|
469
|
+
A revision's DVC-tracked inputs, e.g., figures and tables a pipeline
|
|
470
|
+
generates, are fetched as they were at that revision, so each side of
|
|
471
|
+
the comparison shows its own.
|
|
409
472
|
"""
|
|
473
|
+
|
|
474
|
+
def fetch_dvc_inputs(root: str, rev: str, paths: list[str]) -> list[str]:
|
|
475
|
+
"""Fetch the DVC-tracked content of ``paths`` at ``rev`` into ``root``.
|
|
476
|
+
|
|
477
|
+
Returns each fetched path with its hash, since that content changes
|
|
478
|
+
the PDF without changing the marked-up source.
|
|
479
|
+
"""
|
|
480
|
+
from dvc.exceptions import NotDvcRepoError
|
|
481
|
+
from dvc.fs import DVCFileSystem
|
|
482
|
+
|
|
483
|
+
import calkit.dvc
|
|
484
|
+
|
|
485
|
+
def reason(e: BaseException) -> str:
|
|
486
|
+
# DVC wraps a remote's own error, e.g., needing to log in, in
|
|
487
|
+
# several layers that each say less
|
|
488
|
+
while e.__cause__ is not None:
|
|
489
|
+
e = e.__cause__
|
|
490
|
+
return str(e)
|
|
491
|
+
|
|
492
|
+
fetched: list[str] = []
|
|
493
|
+
try:
|
|
494
|
+
# A project using Calkit's own remote needs its scheme known
|
|
495
|
+
calkit.dvc.register_ck_scheme()
|
|
496
|
+
fs = DVCFileSystem(url=".", rev=rev)
|
|
497
|
+
except NotDvcRepoError:
|
|
498
|
+
return fetched
|
|
499
|
+
found: list[str] = []
|
|
500
|
+
for path in paths:
|
|
501
|
+
try:
|
|
502
|
+
found += list(fs.find(path.rstrip("/")))
|
|
503
|
+
except FileNotFoundError:
|
|
504
|
+
continue
|
|
505
|
+
except Exception as e:
|
|
506
|
+
# A directory whose listing is in neither the cache nor a
|
|
507
|
+
# reachable remote can't be expanded, but the rest of the
|
|
508
|
+
# comparison can still be built
|
|
509
|
+
warn(
|
|
510
|
+
f"Can't list {path} at {rev[:7]}, so the diff will be "
|
|
511
|
+
f"missing it: {reason(e)}"
|
|
512
|
+
)
|
|
513
|
+
for rpath in dict.fromkeys(found):
|
|
514
|
+
dvc_info = fs.info(rpath).get("dvc_info")
|
|
515
|
+
if not dvc_info:
|
|
516
|
+
continue
|
|
517
|
+
fetched.append(f"{rpath} {dvc_info.get('md5')}")
|
|
518
|
+
dest = os.path.join(root, rpath)
|
|
519
|
+
if os.path.exists(dest):
|
|
520
|
+
continue
|
|
521
|
+
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
|
522
|
+
try:
|
|
523
|
+
fs.get_file(rpath, dest)
|
|
524
|
+
except Exception as e:
|
|
525
|
+
warn(
|
|
526
|
+
f"Can't fetch {rpath} at {rev[:7]}, so the diff will be "
|
|
527
|
+
f"missing it: {reason(e)}"
|
|
528
|
+
)
|
|
529
|
+
return fetched
|
|
530
|
+
|
|
531
|
+
def copy_uncached_outputs(
|
|
532
|
+
root: str, rev: str, paths: list[str]
|
|
533
|
+
) -> list[str]:
|
|
534
|
+
"""Copy outputs DVC records but doesn't store from the working tree.
|
|
535
|
+
|
|
536
|
+
An output with caching off, e.g., a copy another stage makes, has
|
|
537
|
+
only its hash in ``dvc.lock``, so no revision's checkout can have
|
|
538
|
+
it. The working tree's copy stands in when it's the same content.
|
|
539
|
+
Returns each copied path with its hash.
|
|
540
|
+
"""
|
|
541
|
+
try:
|
|
542
|
+
lock = calkit.ryaml.load(repo.git.show(f"{rev}:dvc.lock")) or {}
|
|
543
|
+
except Exception:
|
|
544
|
+
return []
|
|
545
|
+
prefixes = [p.rstrip("/") for p in paths]
|
|
546
|
+
copied: list[str] = []
|
|
547
|
+
for stage in (lock.get("stages") or {}).values():
|
|
548
|
+
for out in stage.get("outs") or []:
|
|
549
|
+
path, md5 = out.get("path"), out.get("md5")
|
|
550
|
+
if not path or not md5 or str(md5).endswith(".dir"):
|
|
551
|
+
continue
|
|
552
|
+
if not any(
|
|
553
|
+
path == p or path.startswith(p + "/") for p in prefixes
|
|
554
|
+
):
|
|
555
|
+
continue
|
|
556
|
+
dest = os.path.join(root, path)
|
|
557
|
+
if os.path.exists(dest) or not os.path.isfile(path):
|
|
558
|
+
continue
|
|
559
|
+
if hashlib.md5(Path(path).read_bytes()).hexdigest() != md5:
|
|
560
|
+
continue
|
|
561
|
+
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
|
562
|
+
shutil.copy2(path, dest)
|
|
563
|
+
copied.append(f"{path} {md5}")
|
|
564
|
+
return copied
|
|
565
|
+
|
|
566
|
+
def break_verbatim_params(root: str) -> None:
|
|
567
|
+
"""Break verbatim inputs named by macro parameters in a checkout.
|
|
568
|
+
|
|
569
|
+
Done to the checked-out files directly rather than with latexdiff's
|
|
570
|
+
--filter-script, which hands text to the script and reads it back
|
|
571
|
+
with no encoding, mangling anything outside Latin-1, e.g., a curly
|
|
572
|
+
apostrophe.
|
|
573
|
+
"""
|
|
574
|
+
sources = [tex_file] + [
|
|
575
|
+
path
|
|
576
|
+
for path in calkit.latex.detect_inputs(tex_file, wdir=root)
|
|
577
|
+
if Path(path).suffix in calkit.latex._SOURCE_EXTS
|
|
578
|
+
]
|
|
579
|
+
for source in sources:
|
|
580
|
+
source_path = Path(root, source)
|
|
581
|
+
try:
|
|
582
|
+
text = source_path.read_text(encoding="utf-8")
|
|
583
|
+
except (OSError, UnicodeDecodeError):
|
|
584
|
+
continue
|
|
585
|
+
new_text = _VERBATIM_PARAM_RE.sub("\\1%\n", text)
|
|
586
|
+
if new_text != text:
|
|
587
|
+
source_path.write_text(new_text, encoding="utf-8")
|
|
588
|
+
|
|
589
|
+
def default_inputs(root: str) -> list[str]:
|
|
590
|
+
"""What to fetch or note for a side when no inputs were given.
|
|
591
|
+
|
|
592
|
+
Detection finds a file by its own name or pointer, but a file in a
|
|
593
|
+
directory DVC tracks as a whole has only the directory's pointer,
|
|
594
|
+
which can't say what's inside, so those directories beside the
|
|
595
|
+
document are included whole.
|
|
596
|
+
"""
|
|
597
|
+
doc_dir = Path(root, os.path.dirname(tex_file))
|
|
598
|
+
pointed = [
|
|
599
|
+
Path(os.path.relpath(p, root)).as_posix().removesuffix(".dvc")
|
|
600
|
+
for p in sorted(doc_dir.rglob("*.dvc"))
|
|
601
|
+
if p.is_file()
|
|
602
|
+
]
|
|
603
|
+
return calkit.latex.detect_inputs(tex_file, wdir=root) + pointed
|
|
604
|
+
|
|
605
|
+
def point_changed_figures_at_base(base_root: str, head_root: str) -> None:
|
|
606
|
+
"""Make the older side's changed figures refer to its own copies.
|
|
607
|
+
|
|
608
|
+
The marked-up document is built beside the newer side, where the
|
|
609
|
+
older side's figure names find the newer figures. Left alone, a
|
|
610
|
+
figure that changed would show as it is now on both sides, and
|
|
611
|
+
latexdiff wouldn't mark it, since its name didn't change.
|
|
612
|
+
"""
|
|
613
|
+
import filecmp
|
|
614
|
+
|
|
615
|
+
tex_dir = os.path.dirname(tex_file)
|
|
616
|
+
build_dir = os.path.join(head_root, tex_dir)
|
|
617
|
+
graphic_re = re.compile(
|
|
618
|
+
r"(\\includegraphics\*?\s*(?:\[[^\]]*\])*\s*\{)([^}#\\]+)(\})"
|
|
619
|
+
)
|
|
620
|
+
|
|
621
|
+
def repoint(match: re.Match[str]) -> str:
|
|
622
|
+
name = match.group(2).strip()
|
|
623
|
+
exts = calkit.latex._GRAPHICS_EXTS
|
|
624
|
+
for candidate in [name] + [name + ext for ext in exts]:
|
|
625
|
+
rel = os.path.normpath(os.path.join(tex_dir, candidate))
|
|
626
|
+
base_path = os.path.join(base_root, rel)
|
|
627
|
+
if not os.path.isfile(base_path):
|
|
628
|
+
continue
|
|
629
|
+
head_path = os.path.join(head_root, rel)
|
|
630
|
+
if os.path.isfile(head_path) and filecmp.cmp(
|
|
631
|
+
base_path, head_path, shallow=False
|
|
632
|
+
):
|
|
633
|
+
return match.group(0)
|
|
634
|
+
new_name = Path(os.path.relpath(base_path, build_dir))
|
|
635
|
+
return match.group(1) + new_name.as_posix() + match.group(3)
|
|
636
|
+
return match.group(0)
|
|
637
|
+
|
|
638
|
+
sources = [tex_file] + [
|
|
639
|
+
path
|
|
640
|
+
for path in calkit.latex.detect_inputs(tex_file, wdir=base_root)
|
|
641
|
+
if Path(path).suffix in calkit.latex._SOURCE_EXTS
|
|
642
|
+
]
|
|
643
|
+
for source in sources:
|
|
644
|
+
source_path = Path(base_root, source)
|
|
645
|
+
try:
|
|
646
|
+
text = source_path.read_text(encoding="utf-8")
|
|
647
|
+
except (OSError, UnicodeDecodeError):
|
|
648
|
+
continue
|
|
649
|
+
new_text = graphic_re.sub(repoint, text)
|
|
650
|
+
if new_text != text:
|
|
651
|
+
source_path.write_text(new_text, encoding="utf-8")
|
|
652
|
+
|
|
410
653
|
repo = calkit.git.get_repo()
|
|
411
654
|
if repo.bare:
|
|
412
655
|
raise_error("This is not a working Git repo")
|
|
@@ -425,16 +668,8 @@ def diff(
|
|
|
425
668
|
to_ref=to_ref,
|
|
426
669
|
output_dir=output_dir,
|
|
427
670
|
)
|
|
428
|
-
# A comparison between two revisions that can't move is the same
|
|
429
|
-
# comparison forever, and LaTeX writes a timestamp into every PDF, so
|
|
430
|
-
# rebuilding one would change the file without changing what it says
|
|
431
|
-
fixed = _is_immutable_ref(repo, from_ref) and _is_immutable_ref(
|
|
432
|
-
repo, to_ref
|
|
433
|
-
)
|
|
434
|
-
if fixed and os.path.isfile(output) and not force:
|
|
435
|
-
typer.echo(f"{output} is already built; use --force to rebuild it")
|
|
436
|
-
return
|
|
437
671
|
checkouts: dict[str, str] = {}
|
|
672
|
+
shas: dict[str, str] = {}
|
|
438
673
|
try:
|
|
439
674
|
for name, ref in [("base", from_ref), ("head", to_ref)]:
|
|
440
675
|
if ref is None:
|
|
@@ -442,6 +677,7 @@ def diff(
|
|
|
442
677
|
sha = calkit.git.resolve_ref(repo, ref)
|
|
443
678
|
if sha is None:
|
|
444
679
|
raise_error(f"Git ref '{ref}' was not found")
|
|
680
|
+
shas[name] = sha
|
|
445
681
|
# A worktree, not a temp directory: a document is rarely one
|
|
446
682
|
# file, and \input needs the rest of them as they were then
|
|
447
683
|
path = os.path.join(DIFF_TMP_DIR, name)
|
|
@@ -466,12 +702,43 @@ def diff(
|
|
|
466
702
|
if not os.path.isfile(side):
|
|
467
703
|
raise_error(f"{tex_file} does not exist at {ref}")
|
|
468
704
|
sides[name] = side
|
|
705
|
+
# The working tree already has its DVC-tracked files, if pulled
|
|
706
|
+
head_root = checkouts.get("head", ".")
|
|
707
|
+
context: list[str] = []
|
|
708
|
+
for name, root in checkouts.items():
|
|
709
|
+
paths = inputs or default_inputs(root)
|
|
710
|
+
context += fetch_dvc_inputs(root, rev=shas[name], paths=paths)
|
|
711
|
+
context += copy_uncached_outputs(root, rev=shas[name], paths=paths)
|
|
712
|
+
if "head" not in checkouts:
|
|
713
|
+
# The working tree has no revision to name its files by, so
|
|
714
|
+
# note what's there instead
|
|
715
|
+
for path in inputs or default_inputs("."):
|
|
716
|
+
if os.path.isfile(path):
|
|
717
|
+
files = [Path(path)]
|
|
718
|
+
else:
|
|
719
|
+
files = sorted(
|
|
720
|
+
p for p in Path(path).rglob("*") if p.is_file()
|
|
721
|
+
)
|
|
722
|
+
for file_path in files:
|
|
723
|
+
stat = file_path.stat()
|
|
724
|
+
context.append(
|
|
725
|
+
f"{file_path.as_posix()} {stat.st_size} "
|
|
726
|
+
f"{stat.st_mtime_ns}"
|
|
727
|
+
)
|
|
728
|
+
for root in checkouts.values():
|
|
729
|
+
break_verbatim_params(root)
|
|
730
|
+
point_changed_figures_at_base(checkouts["base"], head_root)
|
|
469
731
|
_build_diff(
|
|
470
732
|
base_tex=sides["base"],
|
|
471
733
|
head_tex=sides["head"],
|
|
472
734
|
tex_file=tex_file,
|
|
735
|
+
head_root=head_root,
|
|
473
736
|
output=output,
|
|
474
737
|
environment=environment,
|
|
738
|
+
latexmk_rc_path=latexmk_rc_path,
|
|
739
|
+
latexmk_args=latexmk_args,
|
|
740
|
+
latexdiff_args=latexdiff_args,
|
|
741
|
+
context=context,
|
|
475
742
|
no_check=no_check,
|
|
476
743
|
keep_tex=keep_tex,
|
|
477
744
|
force=force,
|
|
@@ -482,19 +749,23 @@ def diff(
|
|
|
482
749
|
_remove_worktree(path)
|
|
483
750
|
|
|
484
751
|
|
|
485
|
-
def _marked_up_digest(marked_up: bytes) -> str:
|
|
752
|
+
def _marked_up_digest(marked_up: bytes, context: list[str] = []) -> str:
|
|
486
753
|
"""Hash a marked-up document by what actually determines the PDF.
|
|
487
754
|
|
|
488
755
|
latexdiff writes the two inputs' paths and modification times into a
|
|
489
756
|
header comment, and the older side is a fresh checkout every time, so
|
|
490
757
|
hashing the file as-is would say "changed" on every run when nothing
|
|
491
758
|
had. Those lines are comments; the PDF doesn't depend on them.
|
|
759
|
+
|
|
760
|
+
``context`` is anything else the PDF depends on, e.g., the hashes of
|
|
761
|
+
DVC-tracked figures and the options it's built with.
|
|
492
762
|
"""
|
|
493
763
|
kept = [
|
|
494
764
|
line
|
|
495
765
|
for line in marked_up.splitlines(keepends=True)
|
|
496
766
|
if not line.startswith((b"%DIF DEL ", b"%DIF ADD "))
|
|
497
767
|
]
|
|
768
|
+
kept += [f"{item}\n".encode() for item in context]
|
|
498
769
|
return hashlib.sha256(b"".join(kept)).hexdigest()
|
|
499
770
|
|
|
500
771
|
|
|
@@ -520,35 +791,38 @@ def _build_diff(
|
|
|
520
791
|
base_tex: str,
|
|
521
792
|
head_tex: str,
|
|
522
793
|
tex_file: str,
|
|
794
|
+
head_root: str,
|
|
523
795
|
output: str,
|
|
524
796
|
environment: str | None,
|
|
797
|
+
latexmk_rc_path: str | None,
|
|
798
|
+
latexmk_args: list[str],
|
|
799
|
+
latexdiff_args: list[str],
|
|
800
|
+
context: list[str],
|
|
525
801
|
no_check: bool,
|
|
526
802
|
keep_tex: bool,
|
|
527
803
|
force: bool,
|
|
528
804
|
verbose: bool,
|
|
529
805
|
) -> None:
|
|
530
806
|
"""Mark up one document against another and build the result."""
|
|
531
|
-
# Built beside the
|
|
532
|
-
#
|
|
533
|
-
#
|
|
534
|
-
# for it, but a DVC-tracked figure isn't in Git: a checkout has the
|
|
535
|
-
# pointer file and not the image, and the marked-up document would
|
|
536
|
-
# come out with its figures missing.
|
|
807
|
+
# Built beside the newer side, so \graphicspath, \bibliography, and
|
|
808
|
+
# relative \includegraphics resolve the way they do for the real thing,
|
|
809
|
+
# against that revision's own files
|
|
537
810
|
tex_dir = os.path.dirname(tex_file) or "."
|
|
811
|
+
build_dir = os.path.normpath(os.path.join(head_root, tex_dir))
|
|
538
812
|
stem = Path(tex_file).stem
|
|
539
|
-
diff_tex = os.path.join(
|
|
813
|
+
diff_tex = os.path.join(build_dir, f"{stem}-diff.tex")
|
|
814
|
+
# Where --keep-tex leaves it: beside the document, since a checkout is
|
|
815
|
+
# removed afterwards
|
|
816
|
+
kept_tex = os.path.normpath(os.path.join(tex_dir, f"{stem}-diff.tex"))
|
|
817
|
+
aux_dir = os.path.join(build_dir, calkit.latex.DIFF_AUX_DIRNAME)
|
|
540
818
|
try:
|
|
541
819
|
# --flatten pulls \input and \include files into one document on
|
|
542
820
|
# each side, so a multi-file paper compares as a whole
|
|
543
821
|
latexdiff_cmd = ["latexdiff", "--flatten", "--encoding=utf8"]
|
|
544
|
-
#
|
|
545
|
-
#
|
|
546
|
-
# filter breaks
|
|
547
|
-
#
|
|
548
|
-
verbatim_param = re.compile(
|
|
549
|
-
r"\\(?:verbatiminput\*?|lstinputlisting)"
|
|
550
|
-
r"\s*(?:\[[^\]\n]*\])?\s*\{[^}\n]*#[0-9]"
|
|
551
|
-
)
|
|
822
|
+
# A checkout's verbatim inputs named by macro parameters are already
|
|
823
|
+
# broken, so this only finds the working tree's, which can't be
|
|
824
|
+
# edited. A filter breaks those instead, though latexdiff's
|
|
825
|
+
# --filter-script mangles anything outside Latin-1.
|
|
552
826
|
sources = [base_tex, head_tex] + [
|
|
553
827
|
path
|
|
554
828
|
for side in (base_tex, head_tex)
|
|
@@ -557,7 +831,7 @@ def _build_diff(
|
|
|
557
831
|
]
|
|
558
832
|
if any(
|
|
559
833
|
os.path.isfile(path)
|
|
560
|
-
and
|
|
834
|
+
and _VERBATIM_PARAM_RE.search(
|
|
561
835
|
Path(path).read_text(encoding="utf-8", errors="replace")
|
|
562
836
|
)
|
|
563
837
|
for path in sources
|
|
@@ -576,6 +850,14 @@ def _build_diff(
|
|
|
576
850
|
latexdiff_cmd.append(
|
|
577
851
|
f"--filter-script=perl {filter_path.as_posix()}"
|
|
578
852
|
)
|
|
853
|
+
# Each side has its own revision's figures, so a changed one is
|
|
854
|
+
# shown both ways rather than only as it is now, unless the user
|
|
855
|
+
# chose otherwise
|
|
856
|
+
if not any(a.startswith("--graphics-markup") for a in latexdiff_args):
|
|
857
|
+
latexdiff_cmd.append("--graphics-markup=both")
|
|
858
|
+
# User pass-through args come last so they can override Calkit's
|
|
859
|
+
# defaults
|
|
860
|
+
latexdiff_cmd += latexdiff_args
|
|
579
861
|
cmd = _tex_cmd(
|
|
580
862
|
latexdiff_cmd + [base_tex, head_tex],
|
|
581
863
|
environment=environment,
|
|
@@ -591,14 +873,32 @@ def _build_diff(
|
|
|
591
873
|
"latexdiff was not found; it ships with TeX Live, so a "
|
|
592
874
|
"minimal install may not have it"
|
|
593
875
|
)
|
|
594
|
-
except subprocess.CalledProcessError:
|
|
595
|
-
raise_error("latexdiff failed")
|
|
596
|
-
# The
|
|
597
|
-
#
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
876
|
+
except subprocess.CalledProcessError as e:
|
|
877
|
+
raise_error(f"latexdiff failed with exit status {e.returncode}")
|
|
878
|
+
# The newer side's copy, which is what the document was built with
|
|
879
|
+
# at that revision
|
|
880
|
+
rc_path = None
|
|
881
|
+
rc_hash = None
|
|
882
|
+
if latexmk_rc_path is not None:
|
|
883
|
+
rc_path = os.path.join(head_root, latexmk_rc_path)
|
|
884
|
+
if not os.path.isfile(rc_path):
|
|
885
|
+
rc_path = latexmk_rc_path
|
|
886
|
+
rc_path = Path(os.path.normpath(rc_path)).as_posix()
|
|
887
|
+
if os.path.isfile(rc_path):
|
|
888
|
+
rc_hash = hashlib.sha256(
|
|
889
|
+
Path(rc_path).read_bytes()
|
|
890
|
+
).hexdigest()
|
|
891
|
+
# The PDF is a function of this marked-up source and how it's built,
|
|
892
|
+
# so if neither has changed there's nothing to build. Worth checking
|
|
893
|
+
# because the common case produces nothing at all: on the default
|
|
894
|
+
# branch the merge base is usually HEAD, so the comparison is empty,
|
|
895
|
+
# and latexmk is the expensive half of this.
|
|
896
|
+
digest = _marked_up_digest(
|
|
897
|
+
marked_up,
|
|
898
|
+
context=context
|
|
899
|
+
+ [f"latexmkrc {rc_path} {rc_hash}"]
|
|
900
|
+
+ latexmk_args,
|
|
901
|
+
)
|
|
602
902
|
state_path = calkit.latex.diff_state_path(output)
|
|
603
903
|
if (
|
|
604
904
|
not force
|
|
@@ -609,11 +909,14 @@ def _build_diff(
|
|
|
609
909
|
return
|
|
610
910
|
with open(diff_tex, "wb") as f:
|
|
611
911
|
f.write(marked_up)
|
|
612
|
-
aux_dir = DIFF_AUX_DIR
|
|
613
912
|
os.makedirs(aux_dir, exist_ok=True)
|
|
614
|
-
rel_aux =
|
|
615
|
-
latexmk_cmd = [
|
|
616
|
-
|
|
913
|
+
rel_aux = calkit.latex.DIFF_AUX_DIRNAME
|
|
914
|
+
latexmk_cmd = ["latexmk"]
|
|
915
|
+
# First, since latexmk reads an rc file where it appears, so the
|
|
916
|
+
# directories below override any the rc file sets
|
|
917
|
+
if rc_path is not None:
|
|
918
|
+
latexmk_cmd += ["-r", rc_path]
|
|
919
|
+
latexmk_cmd += [
|
|
617
920
|
"-pdf",
|
|
618
921
|
"-cd",
|
|
619
922
|
"-interaction=nonstopmode",
|
|
@@ -622,6 +925,9 @@ def _build_diff(
|
|
|
622
925
|
]
|
|
623
926
|
if not verbose:
|
|
624
927
|
latexmk_cmd.append("-silent")
|
|
928
|
+
# User pass-through args come last so they can override Calkit's
|
|
929
|
+
# defaults
|
|
930
|
+
latexmk_cmd += latexmk_args
|
|
625
931
|
latexmk_cmd.append(diff_tex)
|
|
626
932
|
cmd = _tex_cmd(
|
|
627
933
|
latexmk_cmd,
|
|
@@ -633,23 +939,47 @@ def _build_diff(
|
|
|
633
939
|
typer.echo("Building the marked-up document")
|
|
634
940
|
try:
|
|
635
941
|
subprocess.check_call(cmd)
|
|
636
|
-
except subprocess.CalledProcessError:
|
|
942
|
+
except subprocess.CalledProcessError as e:
|
|
943
|
+
# -silent hides why, so show the errors LaTeX logged
|
|
944
|
+
log_path = Path(aux_dir, f"{stem}-diff.log")
|
|
945
|
+
try:
|
|
946
|
+
log_lines = log_path.read_text(
|
|
947
|
+
encoding="utf-8", errors="replace"
|
|
948
|
+
).splitlines()
|
|
949
|
+
except OSError:
|
|
950
|
+
log_lines = []
|
|
951
|
+
shown: list[str] = []
|
|
952
|
+
for i, line in enumerate(log_lines):
|
|
953
|
+
if line.startswith("!"):
|
|
954
|
+
shown += log_lines[i : i + 3]
|
|
955
|
+
excerpt = shown[:60] or log_lines[-20:]
|
|
956
|
+
if excerpt:
|
|
957
|
+
typer.echo(f"From {log_path.as_posix()}:", err=True)
|
|
958
|
+
typer.echo("\n".join(excerpt), err=True)
|
|
637
959
|
raise_error(
|
|
638
|
-
"latexmk failed on the marked-up document
|
|
639
|
-
"--keep-tex to inspect it"
|
|
960
|
+
"latexmk failed on the marked-up document with exit status "
|
|
961
|
+
f"{e.returncode}; rerun with --keep-tex to inspect it"
|
|
640
962
|
)
|
|
641
963
|
built = os.path.join(aux_dir, f"{stem}-diff.pdf")
|
|
642
964
|
if not os.path.isfile(built):
|
|
643
965
|
raise_error("latexmk did not produce a PDF")
|
|
644
966
|
os.makedirs(os.path.dirname(output) or ".", exist_ok=True)
|
|
645
967
|
shutil.move(built, output)
|
|
968
|
+
# A checkout is removed afterwards, but the working tree's document
|
|
969
|
+
# directory is the user's, so nothing is left behind in it
|
|
970
|
+
if os.path.abspath(head_root) == os.path.abspath("."):
|
|
971
|
+
shutil.rmtree(aux_dir, ignore_errors=True)
|
|
646
972
|
os.makedirs(os.path.dirname(state_path), exist_ok=True)
|
|
647
973
|
with open(state_path, "w") as f:
|
|
648
974
|
f.write(digest)
|
|
649
975
|
typer.echo(f"Wrote {output}")
|
|
650
976
|
finally:
|
|
651
|
-
if
|
|
652
|
-
os.
|
|
977
|
+
if os.path.isfile(diff_tex):
|
|
978
|
+
in_place = os.path.abspath(diff_tex) == os.path.abspath(kept_tex)
|
|
979
|
+
if keep_tex and not in_place:
|
|
980
|
+
shutil.copy(diff_tex, kept_tex)
|
|
981
|
+
if not keep_tex or not in_place:
|
|
982
|
+
os.remove(diff_tex)
|
|
653
983
|
|
|
654
984
|
|
|
655
985
|
@latex_app.command(name="to-docx")
|
calkit/cli/main/core.py
CHANGED
|
@@ -2777,9 +2777,13 @@ def run(
|
|
|
2777
2777
|
dvc_data_status_before.pop("git", None) # Remove git status
|
|
2778
2778
|
if targets is None:
|
|
2779
2779
|
targets = []
|
|
2780
|
-
|
|
2781
|
-
|
|
2782
|
-
|
|
2780
|
+
try:
|
|
2781
|
+
args, isolated_sp_targets = calkit.pipeline.translate_run_targets(
|
|
2782
|
+
deepcopy(targets), ck_info=ck_info
|
|
2783
|
+
)
|
|
2784
|
+
except ValueError as e:
|
|
2785
|
+
os.environ.pop("CALKIT_PIPELINE_RUNNING", None)
|
|
2786
|
+
raise_error(str(e))
|
|
2783
2787
|
# Extract any boolean args
|
|
2784
2788
|
for name in [
|
|
2785
2789
|
"quiet",
|
|
@@ -3478,8 +3482,11 @@ def run_in_env(
|
|
|
3478
3482
|
typer.echo(f"Running command: {docker_cmd}")
|
|
3479
3483
|
try:
|
|
3480
3484
|
subprocess.check_call(docker_cmd, cwd=wdir)
|
|
3481
|
-
except subprocess.CalledProcessError:
|
|
3482
|
-
raise_error(
|
|
3485
|
+
except subprocess.CalledProcessError as e:
|
|
3486
|
+
raise_error(
|
|
3487
|
+
"Failed to run in Docker environment: command exited with "
|
|
3488
|
+
f"status {e.returncode}"
|
|
3489
|
+
)
|
|
3483
3490
|
elif env["kind"] == "conda":
|
|
3484
3491
|
with open(env["path"]) as f:
|
|
3485
3492
|
conda_env = calkit.ryaml.load(f)
|
calkit/environments.py
CHANGED
|
@@ -1140,10 +1140,20 @@ def check_all_in_pipeline(
|
|
|
1140
1140
|
for t in targets
|
|
1141
1141
|
if t in md_stages
|
|
1142
1142
|
]
|
|
1143
|
+
# A latex stage's diffs run in its environment, whether they're
|
|
1144
|
+
# named one at a time or all together
|
|
1145
|
+
import calkit.latex
|
|
1146
|
+
|
|
1143
1147
|
stages = {
|
|
1144
1148
|
k: v
|
|
1145
1149
|
for k, v in stages.items()
|
|
1146
|
-
if k in targets
|
|
1150
|
+
if k in targets
|
|
1151
|
+
or any(k.startswith(p) for p in prefixes)
|
|
1152
|
+
or k + calkit.latex.DIFFS_TARGET_SUFFIX in targets
|
|
1153
|
+
or any(
|
|
1154
|
+
name in targets
|
|
1155
|
+
for name in calkit.latex.get_diff_stage_names(k, v)
|
|
1156
|
+
)
|
|
1147
1157
|
}
|
|
1148
1158
|
envs_in_pipeline = [stage.get("environment") for stage in stages.values()]
|
|
1149
1159
|
envs_in_pipeline = [
|
calkit/git.py
CHANGED
|
@@ -625,24 +625,3 @@ def resolve_ref(repo: git.Repo, ref: str) -> str | None:
|
|
|
625
625
|
warnings.warn(f"Failed to fetch history: {e}")
|
|
626
626
|
return None
|
|
627
627
|
return parse()
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
def last_change(repo: git.Repo, ref: str, paths: list[str]) -> str | None:
|
|
631
|
-
"""The last commit at or before ``ref`` that touched any of ``paths``.
|
|
632
|
-
|
|
633
|
-
Used to name a revision by its content rather than its position. A
|
|
634
|
-
comparison against ``HEAD`` means the document as it stands, and the
|
|
635
|
-
document doesn't change when something else in the project is
|
|
636
|
-
committed, so resolving to the commit that last changed it keeps the
|
|
637
|
-
answer stable and keeps a pipeline from invalidating itself.
|
|
638
|
-
|
|
639
|
-
Returns None if nothing matches, including when the paths were never
|
|
640
|
-
tracked.
|
|
641
|
-
"""
|
|
642
|
-
if not paths:
|
|
643
|
-
return None
|
|
644
|
-
try:
|
|
645
|
-
out = str(repo.git.rev_list("-1", ref, "--", *paths)).strip()
|
|
646
|
-
except Exception:
|
|
647
|
-
return None
|
|
648
|
-
return out or None
|