calkit-python 0.49.3__py3-none-any.whl → 0.49.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- calkit/cli/__init__.py +10 -0
- calkit/cli/core.py +21 -5
- calkit/cli/main/core.py +36 -9
- calkit/cli/overleaf.py +136 -19
- calkit/cli/update.py +1 -1
- calkit/core.py +49 -0
- calkit/environments.py +50 -20
- calkit/hub.py +3 -1
- calkit/models/pipeline.py +5 -0
- calkit/overleaf.py +2 -1
- calkit/pipeline.py +30 -8
- calkit/tests/cli/test_core.py +22 -0
- calkit/tests/cli/test_overleaf.py +135 -6
- calkit/tests/models/test_pipeline.py +14 -0
- calkit/tests/test_core.py +75 -0
- calkit/tests/test_environments.py +28 -1
- calkit/tests/test_pipeline.py +41 -0
- calkit/upgrade.py +3 -1
- {calkit_python-0.49.3.dist-info → calkit_python-0.49.4.dist-info}/METADATA +1 -1
- {calkit_python-0.49.3.dist-info → calkit_python-0.49.4.dist-info}/RECORD +36 -36
- {calkit_python-0.49.3.data → calkit_python-0.49.4.data}/data/etc/jupyter/jupyter_server_config.d/calkit.json +0 -0
- {calkit_python-0.49.3.data → calkit_python-0.49.4.data}/data/share/jupyter/labextensions/calkit/package.json +0 -0
- {calkit_python-0.49.3.data → calkit_python-0.49.4.data}/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig +0 -0
- {calkit_python-0.49.3.data → calkit_python-0.49.4.data}/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json +0 -0
- {calkit_python-0.49.3.data → calkit_python-0.49.4.data}/data/share/jupyter/labextensions/calkit/static/506.c34b070098184e33.js +0 -0
- {calkit_python-0.49.3.data → calkit_python-0.49.4.data}/data/share/jupyter/labextensions/calkit/static/57d13a7399cec3c5.png +0 -0
- {calkit_python-0.49.3.data → calkit_python-0.49.4.data}/data/share/jupyter/labextensions/calkit/static/616.528427eb54a4a0c0.js +0 -0
- {calkit_python-0.49.3.data → calkit_python-0.49.4.data}/data/share/jupyter/labextensions/calkit/static/740.6bf87276e9e5788a.js +0 -0
- {calkit_python-0.49.3.data → calkit_python-0.49.4.data}/data/share/jupyter/labextensions/calkit/static/899.f9b9f9bf705a6493.js +0 -0
- {calkit_python-0.49.3.data → calkit_python-0.49.4.data}/data/share/jupyter/labextensions/calkit/static/935.46ecc6bf99aa593a.js +0 -0
- {calkit_python-0.49.3.data → calkit_python-0.49.4.data}/data/share/jupyter/labextensions/calkit/static/remoteEntry.109a3a8379365e59.js +0 -0
- {calkit_python-0.49.3.data → calkit_python-0.49.4.data}/data/share/jupyter/labextensions/calkit/static/style.js +0 -0
- {calkit_python-0.49.3.data → calkit_python-0.49.4.data}/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json +0 -0
- {calkit_python-0.49.3.dist-info → calkit_python-0.49.4.dist-info}/WHEEL +0 -0
- {calkit_python-0.49.3.dist-info → calkit_python-0.49.4.dist-info}/entry_points.txt +0 -0
- {calkit_python-0.49.3.dist-info → calkit_python-0.49.4.dist-info}/licenses/LICENSE +0 -0
calkit/cli/__init__.py
CHANGED
|
@@ -19,6 +19,16 @@ def run() -> None:
|
|
|
19
19
|
# otherwise print the *current* CLI's help instead of re-execing
|
|
20
20
|
# under the requested version.
|
|
21
21
|
_maybe_exec_with_version()
|
|
22
|
+
if sys.platform == "win32":
|
|
23
|
+
import io
|
|
24
|
+
|
|
25
|
+
from calkit.core import is_mintty
|
|
26
|
+
|
|
27
|
+
# mintty shows UTF-8, so emoji needn't be replaced, and output is
|
|
28
|
+
# line buffered as it would be to a terminal
|
|
29
|
+
for stream in (sys.stdout, sys.stderr):
|
|
30
|
+
if isinstance(stream, io.TextIOWrapper) and is_mintty(stream):
|
|
31
|
+
stream.reconfigure(encoding="utf-8", line_buffering=True)
|
|
22
32
|
from .main import app
|
|
23
33
|
|
|
24
34
|
# Completion lists commands, so needs them all
|
calkit/cli/core.py
CHANGED
|
@@ -5,6 +5,7 @@ from __future__ import annotations
|
|
|
5
5
|
import os
|
|
6
6
|
import re
|
|
7
7
|
import subprocess
|
|
8
|
+
import sys
|
|
8
9
|
from enum import Enum
|
|
9
10
|
from typing import Any, NoReturn
|
|
10
11
|
|
|
@@ -152,8 +153,24 @@ def echo_json(obj: Any) -> None:
|
|
|
152
153
|
typer.echo(json.dumps(obj, default=str))
|
|
153
154
|
|
|
154
155
|
|
|
156
|
+
def _echo_above_spinner(text: str, err: bool) -> bool:
|
|
157
|
+
"""Print through a spinner's stand-in for the stream, if there is one.
|
|
158
|
+
|
|
159
|
+
Click writes to a Windows console's handle directly, which would put the
|
|
160
|
+
text on the spinner's line rather than above it.
|
|
161
|
+
"""
|
|
162
|
+
file_proxy = sys.modules.get("rich.file_proxy")
|
|
163
|
+
stream = sys.stderr if err else sys.stdout
|
|
164
|
+
if file_proxy is None or not isinstance(stream, file_proxy.FileProxy):
|
|
165
|
+
return False
|
|
166
|
+
print(text, file=stream, flush=True)
|
|
167
|
+
return True
|
|
168
|
+
|
|
169
|
+
|
|
155
170
|
def raise_error(txt: str) -> NoReturn:
|
|
156
|
-
typer.
|
|
171
|
+
text = typer.style("Error: " + str(txt), fg="red")
|
|
172
|
+
if not _echo_above_spinner(text, err=True):
|
|
173
|
+
typer.echo(text, err=True)
|
|
157
174
|
raise typer.Exit(1)
|
|
158
175
|
|
|
159
176
|
|
|
@@ -164,7 +181,6 @@ def warn(txt: str, prefix: str = "Warning: ", err: bool = False):
|
|
|
164
181
|
# defined.
|
|
165
182
|
from calkit.core import encode_safe
|
|
166
183
|
|
|
167
|
-
typer.
|
|
168
|
-
|
|
169
|
-
err=err
|
|
170
|
-
)
|
|
184
|
+
text = typer.style(prefix + encode_safe(str(txt), err=err), fg="yellow")
|
|
185
|
+
if not _echo_above_spinner(text, err=err):
|
|
186
|
+
typer.echo(text, err=err)
|
calkit/cli/main/core.py
CHANGED
|
@@ -542,6 +542,18 @@ def get_status(
|
|
|
542
542
|
if isinstance(spinner, Status):
|
|
543
543
|
spinner.update(text)
|
|
544
544
|
|
|
545
|
+
def start_docker() -> None:
|
|
546
|
+
# Asked with the spinner out of the way, as in 'calkit run'
|
|
547
|
+
import calkit.docker
|
|
548
|
+
|
|
549
|
+
if isinstance(spinner, Status):
|
|
550
|
+
spinner.stop()
|
|
551
|
+
try:
|
|
552
|
+
calkit.docker.ensure_ready(interactive=True)
|
|
553
|
+
finally:
|
|
554
|
+
if isinstance(spinner, Status):
|
|
555
|
+
spinner.start()
|
|
556
|
+
|
|
545
557
|
def get_dvc_data_status() -> dict:
|
|
546
558
|
# Reuse what the pipeline status computed under the same DVC lock
|
|
547
559
|
if (
|
|
@@ -624,15 +636,23 @@ def get_status(
|
|
|
624
636
|
# while, e.g., when environments are checked again, so a spinner shows
|
|
625
637
|
# until then. It writes to its own copy of the terminal, since what the
|
|
626
638
|
# environment checks' tools print is discarded.
|
|
639
|
+
# Whether there's someone to ask is worked out before the spinner starts
|
|
640
|
+
can_ask = not as_json and calkit.core.is_tty()
|
|
627
641
|
spinner: AbstractContextManager = nullcontext()
|
|
628
|
-
|
|
642
|
+
mintty = calkit.core.is_mintty(sys.stderr)
|
|
643
|
+
if not as_json and (sys.stderr.isatty() or mintty):
|
|
629
644
|
import logging
|
|
630
645
|
|
|
631
646
|
from rich.console import Console
|
|
632
647
|
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
648
|
+
# mintty takes ANSI and UTF-8, though it isn't a console
|
|
649
|
+
spinner = Console(
|
|
650
|
+
file=os.fdopen(
|
|
651
|
+
os.dup(2), "w", encoding="utf-8" if mintty else None
|
|
652
|
+
),
|
|
653
|
+
force_terminal=mintty or None,
|
|
654
|
+
legacy_windows=False if mintty else None,
|
|
655
|
+
).status("Checking status")
|
|
636
656
|
# DVC's progress bars would each be printed above the spinner, and
|
|
637
657
|
# they're shown only if this is at or below their level
|
|
638
658
|
logging.getLogger("dvc.progress").setLevel(logging.CRITICAL + 1)
|
|
@@ -665,6 +685,7 @@ def get_status(
|
|
|
665
685
|
with_storage_problems="dvc" in categories,
|
|
666
686
|
quiet_environment_checks=True,
|
|
667
687
|
progress=step,
|
|
688
|
+
start_docker=start_docker if can_ask else None,
|
|
668
689
|
)
|
|
669
690
|
if pipeline_status.failed_environment_checks:
|
|
670
691
|
warn(
|
|
@@ -3128,12 +3149,18 @@ def run(
|
|
|
3128
3149
|
# someone to ask is worked out first
|
|
3129
3150
|
can_ask = is_tty()
|
|
3130
3151
|
spinner: Any = None
|
|
3131
|
-
|
|
3152
|
+
mintty = calkit.core.is_mintty(sys.stderr)
|
|
3153
|
+
if not (quiet or verbose) and (sys.stderr.isatty() or mintty):
|
|
3132
3154
|
from rich.console import Console
|
|
3133
3155
|
|
|
3134
|
-
|
|
3135
|
-
|
|
3136
|
-
|
|
3156
|
+
# mintty takes ANSI and UTF-8, though it isn't a console
|
|
3157
|
+
spinner = Console(
|
|
3158
|
+
file=os.fdopen(
|
|
3159
|
+
os.dup(2), "w", encoding="utf-8" if mintty else None
|
|
3160
|
+
),
|
|
3161
|
+
force_terminal=mintty or None,
|
|
3162
|
+
legacy_windows=False if mintty else None,
|
|
3163
|
+
).status("Getting ready to run")
|
|
3137
3164
|
with spinner or contextlib.nullcontext():
|
|
3138
3165
|
step("Getting system information", "💻")
|
|
3139
3166
|
# Only the stages this run can reach are checked, so nothing else
|
|
@@ -4781,7 +4808,7 @@ def install_app(
|
|
|
4781
4808
|
ok = _install.prompt_and_install(name, interactive=is_tty())
|
|
4782
4809
|
if not ok:
|
|
4783
4810
|
raise_error(f"Failed to install '{name}'")
|
|
4784
|
-
|
|
4811
|
+
calkit.echo(f"✅ Installed '{name}'")
|
|
4785
4812
|
|
|
4786
4813
|
|
|
4787
4814
|
@app.command(name="xproc|runproc", help="Execute a procedure.")
|
calkit/cli/overleaf.py
CHANGED
|
@@ -62,7 +62,8 @@ def _get_overleaf_token(rejected: bool = False) -> str:
|
|
|
62
62
|
"A new one can be generated at:\n\n"
|
|
63
63
|
" https://www.overleaf.com/user/settings\n\n"
|
|
64
64
|
"under the 'Git Integration' section.\n"
|
|
65
|
-
"Without Overleaf Premium, run 'calkit overleaf login' instead
|
|
65
|
+
"Without Overleaf Premium, run 'calkit overleaf login' instead, "
|
|
66
|
+
"or connect from the Calkit browser extension on overleaf.com.\n"
|
|
66
67
|
)
|
|
67
68
|
if not (sys.stdin.isatty() and sys.stdout.isatty()):
|
|
68
69
|
raise_error(
|
|
@@ -82,27 +83,54 @@ def _get_overleaf_token(rejected: bool = False) -> str:
|
|
|
82
83
|
return overleaf_token
|
|
83
84
|
|
|
84
85
|
|
|
86
|
+
def _get_overleaf_session_from_hub(stale: str | None = None) -> str | None:
|
|
87
|
+
"""Get an Overleaf session saved to the hub, e.g., by the browser
|
|
88
|
+
extension, other than a ``stale`` one, and store it in config.
|
|
89
|
+
"""
|
|
90
|
+
try:
|
|
91
|
+
# Not signed in to the hub fails before any request, and
|
|
92
|
+
# allow_login=False keeps that from starting a sign-in
|
|
93
|
+
session = calkit.hub.get(
|
|
94
|
+
"/user/overleaf-session", allow_login=False, max_retries=2
|
|
95
|
+
)["session"]
|
|
96
|
+
except Exception:
|
|
97
|
+
return None
|
|
98
|
+
# Compared as the client sends it, without a cookie name or whitespace
|
|
99
|
+
if not session or calkit.overleaf.WebClient(session).session == stale:
|
|
100
|
+
return None
|
|
101
|
+
calkit_config = calkit.config.read()
|
|
102
|
+
calkit_config.overleaf_session = session
|
|
103
|
+
calkit_config.write()
|
|
104
|
+
return str(session)
|
|
105
|
+
|
|
106
|
+
|
|
85
107
|
def _get_overleaf_auth() -> str | calkit.overleaf.WebClient:
|
|
86
108
|
"""Get a Git token, or, if only a session is available, a web client
|
|
87
109
|
for syncing without Overleaf Premium.
|
|
88
110
|
|
|
89
|
-
Credentials in config come first, then a token saved to the hub,
|
|
90
|
-
prompting for a token.
|
|
111
|
+
Credentials in config come first, then a token saved to the hub, then a
|
|
112
|
+
session saved to the hub, before prompting for a token.
|
|
91
113
|
"""
|
|
92
114
|
calkit_config = calkit.config.read()
|
|
93
115
|
if calkit_config.overleaf_token:
|
|
94
116
|
return calkit_config.overleaf_token
|
|
95
117
|
if calkit_config.overleaf_session:
|
|
96
118
|
return calkit.overleaf.WebClient(calkit_config.overleaf_session)
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
119
|
+
try:
|
|
120
|
+
token = calkit.hub.get(
|
|
121
|
+
"/user/overleaf-token", allow_login=False, max_retries=2
|
|
122
|
+
)["access_token"]
|
|
123
|
+
except Exception:
|
|
124
|
+
token = None
|
|
125
|
+
if token:
|
|
126
|
+
typer.echo("Using the Overleaf token saved to the hub")
|
|
127
|
+
calkit_config.overleaf_token = token
|
|
128
|
+
calkit_config.write()
|
|
129
|
+
return str(token)
|
|
130
|
+
session = _get_overleaf_session_from_hub()
|
|
131
|
+
if session:
|
|
132
|
+
typer.echo("Using the Overleaf session saved to the hub")
|
|
133
|
+
return calkit.overleaf.WebClient(session)
|
|
106
134
|
return _get_overleaf_token()
|
|
107
135
|
|
|
108
136
|
|
|
@@ -116,9 +144,16 @@ def _get_pulled_overleaf_repo(
|
|
|
116
144
|
them too.
|
|
117
145
|
"""
|
|
118
146
|
|
|
119
|
-
def get_new_session() -> str:
|
|
120
|
-
"""
|
|
121
|
-
|
|
147
|
+
def get_new_session(stale: str, error: Exception) -> str:
|
|
148
|
+
"""Get a new Overleaf session from the hub, or prompt for one, and
|
|
149
|
+
store it.
|
|
150
|
+
"""
|
|
151
|
+
# A newer session on the hub, e.g., from the browser extension,
|
|
152
|
+
# means nothing failed as far as the user is concerned
|
|
153
|
+
session = _get_overleaf_session_from_hub(stale=stale)
|
|
154
|
+
if session:
|
|
155
|
+
return session
|
|
156
|
+
warn(str(error), prefix="")
|
|
122
157
|
typer.echo(calkit.overleaf.SESSION_HELP + "\n")
|
|
123
158
|
if not (sys.stdin.isatty() and sys.stdout.isatty()):
|
|
124
159
|
raise_error(
|
|
@@ -145,18 +180,28 @@ def _get_pulled_overleaf_repo(
|
|
|
145
180
|
return repo, auth
|
|
146
181
|
except calkit.overleaf.AuthError as e:
|
|
147
182
|
if isinstance(auth, calkit.overleaf.WebClient):
|
|
148
|
-
|
|
149
|
-
|
|
183
|
+
auth = calkit.overleaf.WebClient(
|
|
184
|
+
get_new_session(stale=auth.session, error=e)
|
|
185
|
+
)
|
|
150
186
|
else:
|
|
151
187
|
# A token stops working when Premium ends, but a browser
|
|
152
188
|
# session still does
|
|
153
|
-
session =
|
|
189
|
+
session = (
|
|
190
|
+
calkit.config.read().overleaf_session
|
|
191
|
+
or _get_overleaf_session_from_hub()
|
|
192
|
+
)
|
|
154
193
|
if session:
|
|
155
194
|
warn(
|
|
156
195
|
"Overleaf rejected your Git token; "
|
|
157
196
|
"using your browser session instead",
|
|
158
197
|
prefix="",
|
|
159
198
|
)
|
|
199
|
+
# Forget the token, so later syncs go straight to the
|
|
200
|
+
# session
|
|
201
|
+
calkit_config = calkit.config.read()
|
|
202
|
+
if calkit_config.overleaf_token == auth:
|
|
203
|
+
calkit_config.overleaf_token = None
|
|
204
|
+
calkit_config.write()
|
|
160
205
|
auth = calkit.overleaf.WebClient(str(session))
|
|
161
206
|
else:
|
|
162
207
|
auth = _get_overleaf_token(rejected=True)
|
|
@@ -173,6 +218,13 @@ def login(
|
|
|
173
218
|
help="Value of the 'overleaf_session2' cookie from your browser.",
|
|
174
219
|
),
|
|
175
220
|
] = None,
|
|
221
|
+
hub: Annotated[
|
|
222
|
+
bool,
|
|
223
|
+
typer.Option(
|
|
224
|
+
"--hub",
|
|
225
|
+
help="Also save session to Calkit hub, so it can sync there.",
|
|
226
|
+
),
|
|
227
|
+
] = False,
|
|
176
228
|
) -> None:
|
|
177
229
|
"""Sign in with a browser session to sync without Overleaf Premium."""
|
|
178
230
|
if session is None:
|
|
@@ -190,6 +242,12 @@ def login(
|
|
|
190
242
|
)
|
|
191
243
|
calkit_config.write()
|
|
192
244
|
typer.echo("Stored Overleaf session in Calkit config")
|
|
245
|
+
if hub:
|
|
246
|
+
try:
|
|
247
|
+
calkit.hub.put("/user/overleaf-session", json={"session": session})
|
|
248
|
+
except Exception as e:
|
|
249
|
+
raise_error(f"Failed to save Overleaf session to hub: {e}")
|
|
250
|
+
typer.echo("Saved Overleaf session to hub")
|
|
193
251
|
if calkit_config.overleaf_token:
|
|
194
252
|
warn(
|
|
195
253
|
"An Overleaf Git token is also set and will be used instead; "
|
|
@@ -708,6 +766,16 @@ def sync(
|
|
|
708
766
|
help="When rebuilding, also run stages the build depends on.",
|
|
709
767
|
),
|
|
710
768
|
] = False,
|
|
769
|
+
no_pull: Annotated[
|
|
770
|
+
bool,
|
|
771
|
+
typer.Option(
|
|
772
|
+
"--no-pull",
|
|
773
|
+
help=(
|
|
774
|
+
"Don't pull new commits from the project remote first, "
|
|
775
|
+
"e.g., from a sync on the hub."
|
|
776
|
+
),
|
|
777
|
+
),
|
|
778
|
+
] = False,
|
|
711
779
|
):
|
|
712
780
|
"""Sync folders with Overleaf.
|
|
713
781
|
|
|
@@ -752,6 +820,52 @@ def sync(
|
|
|
752
820
|
return {path: (content.get(path), u) for path, u in untracked.items()}
|
|
753
821
|
|
|
754
822
|
repo = calkit.git.get_repo()
|
|
823
|
+
in_am_session = "in the middle of an am session" in repo.git.status()
|
|
824
|
+
# A sync on the hub or another machine pushes commits here, and syncing
|
|
825
|
+
# without them would apply the same Overleaf edits again. Pulled before
|
|
826
|
+
# reading the sync settings, since the new commits can change them.
|
|
827
|
+
if not no_pull and not resolve and not in_am_session:
|
|
828
|
+
import git
|
|
829
|
+
|
|
830
|
+
try:
|
|
831
|
+
tracking = repo.active_branch.tracking_branch()
|
|
832
|
+
except TypeError:
|
|
833
|
+
# Detached HEAD
|
|
834
|
+
tracking = None
|
|
835
|
+
fetched = False
|
|
836
|
+
if tracking is not None:
|
|
837
|
+
try:
|
|
838
|
+
repo.git.fetch(tracking.remote_name, tracking.remote_head)
|
|
839
|
+
fetched = True
|
|
840
|
+
except git.GitCommandError:
|
|
841
|
+
warn(
|
|
842
|
+
f"Could not fetch {tracking.name}; "
|
|
843
|
+
"syncing without any new commits there"
|
|
844
|
+
)
|
|
845
|
+
if tracking is not None and fetched:
|
|
846
|
+
behind = int(
|
|
847
|
+
repo.git.rev_list("--count", f"HEAD..{tracking.name}")
|
|
848
|
+
)
|
|
849
|
+
merge_head = os.path.join(repo.git_dir, "MERGE_HEAD")
|
|
850
|
+
if behind and os.path.exists(merge_head):
|
|
851
|
+
raise_error(
|
|
852
|
+
f"There are new commits on {tracking.name}, but a merge "
|
|
853
|
+
"is in progress; finish it, then sync again"
|
|
854
|
+
)
|
|
855
|
+
if behind:
|
|
856
|
+
typer.echo(
|
|
857
|
+
f"Pulling {behind} new commit(s) from {tracking.name}"
|
|
858
|
+
)
|
|
859
|
+
try:
|
|
860
|
+
repo.git.merge("--no-edit", tracking.name)
|
|
861
|
+
except git.GitCommandError as e:
|
|
862
|
+
if os.path.exists(merge_head):
|
|
863
|
+
repo.git.merge("--abort")
|
|
864
|
+
raise_error(
|
|
865
|
+
f"Could not pull new commits from {tracking.name}:\n"
|
|
866
|
+
+ str(e.stderr).strip()
|
|
867
|
+
+ "\nPull them yourself, then sync again"
|
|
868
|
+
)
|
|
755
869
|
# Uncommitted edits to calkit.yaml aren't the sync's to touch or commit,
|
|
756
870
|
# so fixing the legacy schema waits until there are none
|
|
757
871
|
calkit_yaml_dirty = bool(repo.git.diff("HEAD", "--", "calkit.yaml"))
|
|
@@ -799,7 +913,6 @@ def sync(
|
|
|
799
913
|
)
|
|
800
914
|
overleaf_auth = _get_overleaf_auth()
|
|
801
915
|
conflict_fpath = calkit.overleaf.get_conflict_fpath()
|
|
802
|
-
in_am_session = "in the middle of an am session" in repo.git.status()
|
|
803
916
|
# Check if we're in the middle of resolving a merge conflict
|
|
804
917
|
if in_am_session and not resolve:
|
|
805
918
|
raise_error(
|
|
@@ -1337,6 +1450,8 @@ def push_to_overleaf(
|
|
|
1337
1450
|
allow_stale=True,
|
|
1338
1451
|
any_branch=True,
|
|
1339
1452
|
force=force,
|
|
1453
|
+
# Pulling is part of preparing, or was skipped with --no-pull
|
|
1454
|
+
no_pull=True,
|
|
1340
1455
|
)
|
|
1341
1456
|
calkit.echo("Overleaf is up-to-date with this project ✅")
|
|
1342
1457
|
|
|
@@ -1448,6 +1563,8 @@ def pull_from_overleaf(
|
|
|
1448
1563
|
any_branch=True,
|
|
1449
1564
|
force=force,
|
|
1450
1565
|
no_build=no_run,
|
|
1566
|
+
# Pulling is part of preparing, or was skipped with --no-pull
|
|
1567
|
+
no_pull=True,
|
|
1451
1568
|
)
|
|
1452
1569
|
# An Overleaf edit to a map-paths copy lands in the file it's copied
|
|
1453
1570
|
# from, so the document is only rebuilt from it after a run
|
calkit/cli/update.py
CHANGED
|
@@ -2212,7 +2212,7 @@ def update_hub(
|
|
|
2212
2212
|
typer.echo("Committing changes")
|
|
2213
2213
|
repo.git.commit(to_commit + ["-m", f"Connect to {hub_url}"])
|
|
2214
2214
|
_adopt_hub_history(repo)
|
|
2215
|
-
|
|
2215
|
+
calkit.echo(f"✅ Connected; project is at {hub_url}/{owner}/{name}")
|
|
2216
2216
|
|
|
2217
2217
|
|
|
2218
2218
|
@update_app.command(name="comment")
|
calkit/core.py
CHANGED
|
@@ -778,6 +778,55 @@ def is_tty() -> bool:
|
|
|
778
778
|
return False
|
|
779
779
|
|
|
780
780
|
|
|
781
|
+
def is_mintty(stream: Any) -> bool:
|
|
782
|
+
"""True when a stream is a Git Bash or Cygwin (mintty) terminal.
|
|
783
|
+
|
|
784
|
+
mintty gives native programs a named pipe rather than a console, so
|
|
785
|
+
Python sees no terminal and picks the ANSI code page, even though
|
|
786
|
+
mintty shows UTF-8 and ANSI escapes. It's told apart by the pipe's
|
|
787
|
+
name, as Git for Windows does.
|
|
788
|
+
"""
|
|
789
|
+
if sys.platform != "win32":
|
|
790
|
+
return False
|
|
791
|
+
import ctypes
|
|
792
|
+
import msvcrt
|
|
793
|
+
from ctypes import wintypes
|
|
794
|
+
|
|
795
|
+
class FileNameInfo(ctypes.Structure):
|
|
796
|
+
_fields_ = [
|
|
797
|
+
("FileNameLength", wintypes.DWORD),
|
|
798
|
+
("FileName", wintypes.WCHAR * 260),
|
|
799
|
+
]
|
|
800
|
+
|
|
801
|
+
try:
|
|
802
|
+
handle = msvcrt.get_osfhandle(stream.fileno())
|
|
803
|
+
except (AttributeError, OSError, ValueError):
|
|
804
|
+
return False
|
|
805
|
+
kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
|
|
806
|
+
kernel32.GetFileType.argtypes = [wintypes.HANDLE]
|
|
807
|
+
kernel32.GetFileInformationByHandleEx.argtypes = [
|
|
808
|
+
wintypes.HANDLE,
|
|
809
|
+
ctypes.c_int,
|
|
810
|
+
ctypes.c_void_p,
|
|
811
|
+
wintypes.DWORD,
|
|
812
|
+
]
|
|
813
|
+
kernel32.GetFileInformationByHandleEx.restype = wintypes.BOOL
|
|
814
|
+
# FILE_TYPE_PIPE
|
|
815
|
+
if kernel32.GetFileType(handle) != 3:
|
|
816
|
+
return False
|
|
817
|
+
info = FileNameInfo()
|
|
818
|
+
# FileNameInfo
|
|
819
|
+
if not kernel32.GetFileInformationByHandleEx(
|
|
820
|
+
handle, 2, ctypes.byref(info), ctypes.sizeof(info)
|
|
821
|
+
):
|
|
822
|
+
return False
|
|
823
|
+
name = info.FileName[: info.FileNameLength // 2]
|
|
824
|
+
return (
|
|
825
|
+
re.match(r"\\(cygwin|msys)-[0-9a-f]+-pty\d+-(from|to)-master", name)
|
|
826
|
+
is not None
|
|
827
|
+
)
|
|
828
|
+
|
|
829
|
+
|
|
781
830
|
def check_dep_exists(
|
|
782
831
|
name: str,
|
|
783
832
|
kind: Literal["app", "env-var", "calkit-config"] = "app",
|
calkit/environments.py
CHANGED
|
@@ -2236,31 +2236,61 @@ def check_all_in_pipeline(
|
|
|
2236
2236
|
|
|
2237
2237
|
@contextmanager
|
|
2238
2238
|
def silenced() -> Iterator[None]:
|
|
2239
|
-
# Tools write to the terminal directly, so this is done
|
|
2240
|
-
#
|
|
2239
|
+
# Tools write to the terminal directly, so this is done where they
|
|
2240
|
+
# inherit it from as well as Python's. Nobody can see a prompt here,
|
|
2241
|
+
# so input is from devnull, which answers one rather than hangs.
|
|
2241
2242
|
if not quiet:
|
|
2242
2243
|
yield
|
|
2243
2244
|
return
|
|
2244
2245
|
sys.stdout.flush()
|
|
2245
2246
|
sys.stderr.flush()
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
2247
|
+
saved_stdin = sys.stdin
|
|
2248
|
+
with (
|
|
2249
|
+
open(os.devnull, "r+") as null,
|
|
2250
|
+
redirect_stdout(null),
|
|
2251
|
+
redirect_stderr(null),
|
|
2252
|
+
):
|
|
2253
|
+
sys.stdin = null
|
|
2254
|
+
try:
|
|
2255
|
+
if sys.platform == "win32":
|
|
2256
|
+
# Replacing descriptors 0-2 would close the console
|
|
2257
|
+
# handles Click uses, so only the standard handles that
|
|
2258
|
+
# subprocesses inherit are swapped
|
|
2259
|
+
import ctypes
|
|
2260
|
+
import msvcrt
|
|
2261
|
+
from ctypes import wintypes
|
|
2262
|
+
|
|
2263
|
+
kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
|
|
2264
|
+
kernel32.GetStdHandle.argtypes = [wintypes.DWORD]
|
|
2265
|
+
kernel32.GetStdHandle.restype = wintypes.HANDLE
|
|
2266
|
+
kernel32.SetStdHandle.argtypes = [
|
|
2267
|
+
wintypes.DWORD,
|
|
2268
|
+
wintypes.HANDLE,
|
|
2269
|
+
]
|
|
2270
|
+
kernel32.SetStdHandle.restype = wintypes.BOOL
|
|
2271
|
+
# STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, STD_ERROR_HANDLE
|
|
2272
|
+
std_ids = [-10, -11, -12]
|
|
2273
|
+
null_handle = msvcrt.get_osfhandle(null.fileno())
|
|
2274
|
+
saved = [kernel32.GetStdHandle(i) for i in std_ids]
|
|
2275
|
+
try:
|
|
2276
|
+
for i in std_ids:
|
|
2277
|
+
kernel32.SetStdHandle(i, null_handle)
|
|
2278
|
+
yield
|
|
2279
|
+
finally:
|
|
2280
|
+
for i, h in zip(std_ids, saved):
|
|
2281
|
+
kernel32.SetStdHandle(i, h)
|
|
2282
|
+
else:
|
|
2283
|
+
saved = [os.dup(fd) for fd in (0, 1, 2)]
|
|
2284
|
+
try:
|
|
2285
|
+
for fd in (0, 1, 2):
|
|
2286
|
+
os.dup2(null.fileno(), fd)
|
|
2287
|
+
yield
|
|
2288
|
+
finally:
|
|
2289
|
+
for fd, saved_fd in enumerate(saved):
|
|
2290
|
+
os.dup2(saved_fd, fd)
|
|
2291
|
+
os.close(saved_fd)
|
|
2292
|
+
finally:
|
|
2293
|
+
sys.stdin = saved_stdin
|
|
2264
2294
|
|
|
2265
2295
|
# TODO: ``check_environment`` should be able to take a wdir argument
|
|
2266
2296
|
if wdir is not None:
|
calkit/hub.py
CHANGED
|
@@ -143,7 +143,9 @@ def run_device_flow() -> str:
|
|
|
143
143
|
raise DeviceLoginError(
|
|
144
144
|
f"Failed to save token in config: {e}"
|
|
145
145
|
) from e
|
|
146
|
-
|
|
146
|
+
from calkit.core import encode_safe
|
|
147
|
+
|
|
148
|
+
print(encode_safe("Logged in successfully ✅"), flush=True)
|
|
147
149
|
return access_token
|
|
148
150
|
sleep_seconds = min(
|
|
149
151
|
interval, max(0.0, deadline - time.monotonic())
|
calkit/models/pipeline.py
CHANGED
|
@@ -2537,6 +2537,11 @@ class MarkdownStage(Stage):
|
|
|
2537
2537
|
def markdown_path(self) -> str:
|
|
2538
2538
|
return Path(self.target_path).as_posix()
|
|
2539
2539
|
|
|
2540
|
+
@property
|
|
2541
|
+
def dvc_outs(self) -> list[str | dict]:
|
|
2542
|
+
# The outputs belong to the stages the file expands into
|
|
2543
|
+
return []
|
|
2544
|
+
|
|
2540
2545
|
|
|
2541
2546
|
class Pipeline(Model):
|
|
2542
2547
|
"""The project's reproducible pipeline."""
|
calkit/overleaf.py
CHANGED
|
@@ -225,7 +225,8 @@ SESSION_HELP = (
|
|
|
225
225
|
SESSION_REJECTED_ADVICE = (
|
|
226
226
|
"Your Overleaf session may have expired, e.g., by signing out of "
|
|
227
227
|
"overleaf.com, or you may not have edit access to this project. "
|
|
228
|
-
"Sign in again with 'calkit overleaf login'
|
|
228
|
+
"Sign in again with 'calkit overleaf login', or reconnect from the "
|
|
229
|
+
"Calkit browser extension on overleaf.com."
|
|
229
230
|
)
|
|
230
231
|
|
|
231
232
|
|
calkit/pipeline.py
CHANGED
|
@@ -1223,6 +1223,7 @@ def get_status(
|
|
|
1223
1223
|
with_storage_problems: bool = False,
|
|
1224
1224
|
quiet_environment_checks: bool = False,
|
|
1225
1225
|
progress: Callable[[str], None] | None = None,
|
|
1226
|
+
start_docker: Callable[[], None] | None = None,
|
|
1226
1227
|
) -> PipelineStatus:
|
|
1227
1228
|
"""Get pipeline status after optional prep checks.
|
|
1228
1229
|
|
|
@@ -1231,12 +1232,27 @@ def get_status(
|
|
|
1231
1232
|
With ``with_data_status``, DVC's data status is attached as well, left
|
|
1232
1233
|
None if it fails so the caller can ask again and report why.
|
|
1233
1234
|
``progress`` is told what's being done as each step starts.
|
|
1235
|
+
``start_docker`` is called when a stage needs Docker and it isn't
|
|
1236
|
+
ready, e.g., to offer to start it, and raises ``DockerNotReadyError``
|
|
1237
|
+
if it still isn't.
|
|
1234
1238
|
"""
|
|
1235
1239
|
|
|
1236
1240
|
def step(text: str) -> None:
|
|
1237
1241
|
if progress is not None:
|
|
1238
1242
|
progress(text)
|
|
1239
1243
|
|
|
1244
|
+
def check_envs(docker_required: bool) -> dict:
|
|
1245
|
+
return calkit.environments.check_all_in_pipeline(
|
|
1246
|
+
ck_info=ck_info,
|
|
1247
|
+
targets=targets,
|
|
1248
|
+
force=force_env_check,
|
|
1249
|
+
skip_stages=list(result["gated_stages"]),
|
|
1250
|
+
quiet=quiet_environment_checks,
|
|
1251
|
+
progress=progress,
|
|
1252
|
+
docker_required=docker_required,
|
|
1253
|
+
)
|
|
1254
|
+
|
|
1255
|
+
import calkit.docker
|
|
1240
1256
|
import calkit.environments
|
|
1241
1257
|
import calkit.markdown
|
|
1242
1258
|
|
|
@@ -1300,14 +1316,20 @@ def get_status(
|
|
|
1300
1316
|
remote=False,
|
|
1301
1317
|
)
|
|
1302
1318
|
result["gated_stages"] = gated | gate_errors
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1319
|
+
|
|
1320
|
+
try:
|
|
1321
|
+
env_checks = check_envs(docker_required=True)
|
|
1322
|
+
except calkit.docker.DockerNotReadyError:
|
|
1323
|
+
# Docker not running doesn't make anything stale, so
|
|
1324
|
+
# without it only the environments that need it fail
|
|
1325
|
+
ready = False
|
|
1326
|
+
if start_docker is not None:
|
|
1327
|
+
try:
|
|
1328
|
+
start_docker()
|
|
1329
|
+
ready = True
|
|
1330
|
+
except calkit.docker.DockerNotReadyError:
|
|
1331
|
+
pass
|
|
1332
|
+
env_checks = check_envs(docker_required=ready)
|
|
1311
1333
|
result["environment_checks"] = env_checks
|
|
1312
1334
|
except Exception as e:
|
|
1313
1335
|
result["errors"].append(
|
calkit/tests/cli/test_core.py
CHANGED
|
@@ -44,3 +44,25 @@ def test_optional_value_command():
|
|
|
44
44
|
# Tokens after -- are not rewritten (they're arguments, not options)
|
|
45
45
|
result = runner.invoke(app, ["--", "--hub"])
|
|
46
46
|
assert "hub=None" in result.output
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def test_warn(capsys) -> None:
|
|
50
|
+
import io
|
|
51
|
+
|
|
52
|
+
import pytest
|
|
53
|
+
from rich.console import Console
|
|
54
|
+
|
|
55
|
+
from calkit.cli.core import raise_error, warn
|
|
56
|
+
|
|
57
|
+
warn("plain")
|
|
58
|
+
assert capsys.readouterr().out == "Warning: plain\n"
|
|
59
|
+
# Under a spinner, a warning or error goes above it rather than straight
|
|
60
|
+
# to the terminal, where it would share the spinner's line
|
|
61
|
+
out = io.StringIO()
|
|
62
|
+
with Console(file=out, force_terminal=True).status("Working"):
|
|
63
|
+
warn("careful")
|
|
64
|
+
with pytest.raises(typer.Exit):
|
|
65
|
+
raise_error("broken")
|
|
66
|
+
assert capsys.readouterr() == ("", "")
|
|
67
|
+
assert "Warning: careful" in out.getvalue()
|
|
68
|
+
assert "Error: broken" in out.getvalue()
|