calkit-python 0.41.16__py3-none-any.whl → 0.41.18__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- calkit/cli/notebooks.py +74 -4
- calkit/cli/scheduler.py +61 -30
- calkit/models/core.py +23 -2
- calkit/tests/cli/test_notebooks.py +85 -0
- calkit/tests/cli/test_scheduler.py +49 -18
- {calkit_python-0.41.16.dist-info → calkit_python-0.41.18.dist-info}/METADATA +52 -28
- {calkit_python-0.41.16.dist-info → calkit_python-0.41.18.dist-info}/RECORD +25 -25
- {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/etc/jupyter/jupyter_server_config.d/calkit.json +0 -0
- {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/install.json +0 -0
- {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/package.json +0 -0
- {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig +0 -0
- {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json +0 -0
- {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/static/502.9a2c5772a15466e923ef.js +0 -0
- {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/static/695.2c41003a452d43d2b358.js +0 -0
- {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/static/867.a42a046aa5108f54f8fb.js +0 -0
- {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/static/909.e3f9cc3408834a7fdcc3.js +0 -0
- {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js +0 -0
- {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js.LICENSE.txt +0 -0
- {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/static/b2f1c3efe70cb539d121.png +0 -0
- {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/static/remoteEntry.ac9035764d5b2adbb542.js +0 -0
- {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/static/style.js +0 -0
- {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json +0 -0
- {calkit_python-0.41.16.dist-info → calkit_python-0.41.18.dist-info}/WHEEL +0 -0
- {calkit_python-0.41.16.dist-info → calkit_python-0.41.18.dist-info}/entry_points.txt +0 -0
- {calkit_python-0.41.16.dist-info → calkit_python-0.41.18.dist-info}/licenses/LICENSE +0 -0
calkit/cli/notebooks.py
CHANGED
|
@@ -490,7 +490,17 @@ def execute_notebook(
|
|
|
490
490
|
typer.echo(f"Using {language} as notebook language")
|
|
491
491
|
# First, ensure the specified environment has a kernel we can use
|
|
492
492
|
# We need to check the environment type and create the kernel if needed
|
|
493
|
-
|
|
493
|
+
docker_env = env.get("kind") == "docker"
|
|
494
|
+
if docker_env:
|
|
495
|
+
# Docker environments run the kernel inside the container, so there is
|
|
496
|
+
# no host kernel to register. Use the container's own kernel (default
|
|
497
|
+
# 'python3', overridable with the env's 'jupyter_kernel' key) and
|
|
498
|
+
# execute with Papermill inside the container further below.
|
|
499
|
+
kernel_name = env.get("jupyter_kernel") or {"r": "ir"}.get(
|
|
500
|
+
language.lower(), "python3"
|
|
501
|
+
)
|
|
502
|
+
display_name = kernel_name
|
|
503
|
+
elif language.lower() in ["python", "julia", "r"]:
|
|
494
504
|
kernel_name, display_name = check_env_kernel(
|
|
495
505
|
env_name=env_name,
|
|
496
506
|
no_check=no_check,
|
|
@@ -563,10 +573,70 @@ def execute_notebook(
|
|
|
563
573
|
typer.echo(f"Using kernel: {kernel_name}")
|
|
564
574
|
typer.echo(f"Running with cwd: {notebook_dir}")
|
|
565
575
|
typer.echo(f"Output will be saved to: {fpath_out_exec}")
|
|
566
|
-
# If this is a Python, Julia, or R notebook, we can use Papermill
|
|
576
|
+
# If this is a Python, Julia, or R notebook, we can use Papermill.
|
|
567
577
|
# If it's a MATLAB notebook, we need to use the MATLAB kernel inside the
|
|
568
|
-
# specified environment
|
|
569
|
-
|
|
578
|
+
# specified environment.
|
|
579
|
+
# Exception: If the environment is a Docker environment, we run Papermill
|
|
580
|
+
# inside the container so the kernel (which lives in the image) and
|
|
581
|
+
# Papermill share a process namespace---this avoids registering a host
|
|
582
|
+
# kernel and cross-container networking entirely
|
|
583
|
+
if docker_env:
|
|
584
|
+
# run_in_env mounts the project at the working directory and applies
|
|
585
|
+
# the env's user/platform/args, so we only need to invoke Papermill
|
|
586
|
+
# ipykernel is in the image, but Papermill may not be, so install it
|
|
587
|
+
# on demand (warning the user) before executing. We install into a
|
|
588
|
+
# writable temp dir on PYTHONPATH rather than the image's site-packages:
|
|
589
|
+
# run_in_env maps the host user into the container, so writing to the
|
|
590
|
+
# system location is typically denied, but /tmp is world-writable
|
|
591
|
+
pm_target = "/tmp/calkit-papermill"
|
|
592
|
+
pythonpath = f"PYTHONPATH={pm_target}"
|
|
593
|
+
pm_cmd = [
|
|
594
|
+
pythonpath,
|
|
595
|
+
"python",
|
|
596
|
+
"-c",
|
|
597
|
+
"import papermill",
|
|
598
|
+
"2>/dev/null",
|
|
599
|
+
"||",
|
|
600
|
+
"(",
|
|
601
|
+
"echo",
|
|
602
|
+
"papermill not found in image; installing it with pip...",
|
|
603
|
+
"&&",
|
|
604
|
+
"python",
|
|
605
|
+
"-m",
|
|
606
|
+
"pip",
|
|
607
|
+
"install",
|
|
608
|
+
"-q",
|
|
609
|
+
"--target",
|
|
610
|
+
pm_target,
|
|
611
|
+
"papermill",
|
|
612
|
+
")",
|
|
613
|
+
"&&",
|
|
614
|
+
pythonpath,
|
|
615
|
+
"python",
|
|
616
|
+
"-m",
|
|
617
|
+
"papermill",
|
|
618
|
+
path,
|
|
619
|
+
fpath_out_exec,
|
|
620
|
+
"-k",
|
|
621
|
+
kernel_name,
|
|
622
|
+
"--log-output",
|
|
623
|
+
]
|
|
624
|
+
if notebook_dir:
|
|
625
|
+
pm_cmd += ["--cwd", notebook_dir]
|
|
626
|
+
if parsed_params:
|
|
627
|
+
# JSON is valid YAML, so pass parameters as base64-encoded JSON to
|
|
628
|
+
# avoid shell-quoting issues across the container boundary
|
|
629
|
+
params_b64 = base64.b64encode(
|
|
630
|
+
json.dumps(parsed_params).encode()
|
|
631
|
+
).decode()
|
|
632
|
+
pm_cmd += ["-b", params_b64]
|
|
633
|
+
run_in_env(
|
|
634
|
+
pm_cmd,
|
|
635
|
+
env_name=env_name,
|
|
636
|
+
no_check=no_check,
|
|
637
|
+
verbose=verbose,
|
|
638
|
+
)
|
|
639
|
+
elif language.lower() in ["python", "julia", "r"]:
|
|
570
640
|
papermill.execute_notebook(
|
|
571
641
|
input_path=path,
|
|
572
642
|
output_path=fpath_out_exec,
|
calkit/cli/scheduler.py
CHANGED
|
@@ -304,6 +304,25 @@ def _slurm_exit_code(job_id: str) -> int | None:
|
|
|
304
304
|
return None
|
|
305
305
|
|
|
306
306
|
|
|
307
|
+
def _parse_pbs_state(stdout: str) -> tuple[str | None, int | None]:
|
|
308
|
+
# `qstat -f` output is a flat list of `key = value` lines; pull out the
|
|
309
|
+
# job's state and, for a finished job, its exit status. The status field is
|
|
310
|
+
# spelled `Exit_status` on PBS Pro and `exit_status` on some Torque builds,
|
|
311
|
+
# so match it case-insensitively.
|
|
312
|
+
state = None
|
|
313
|
+
exit_code = None
|
|
314
|
+
for line in stdout.splitlines():
|
|
315
|
+
stripped = line.strip()
|
|
316
|
+
if stripped.startswith("job_state"):
|
|
317
|
+
state = stripped.split("=", 1)[-1].strip()
|
|
318
|
+
elif stripped.lower().startswith("exit_status"):
|
|
319
|
+
try:
|
|
320
|
+
exit_code = int(stripped.split("=", 1)[-1].strip())
|
|
321
|
+
except ValueError:
|
|
322
|
+
exit_code = None
|
|
323
|
+
return state, exit_code
|
|
324
|
+
|
|
325
|
+
|
|
307
326
|
def _poll_job(kind: str, job_id: str) -> tuple[bool, int | None]:
|
|
308
327
|
"""Return ``(active, exit_code)`` for a scheduler job.
|
|
309
328
|
|
|
@@ -339,39 +358,51 @@ def _poll_job(kind: str, job_id: str) -> tuple[bool, int | None]:
|
|
|
339
358
|
if "invalid job id" in stderr:
|
|
340
359
|
return False, _slurm_exit_code(job_id)
|
|
341
360
|
return True, None
|
|
342
|
-
#
|
|
343
|
-
#
|
|
344
|
-
#
|
|
345
|
-
#
|
|
346
|
-
#
|
|
361
|
+
# PBS reports a finished job differently across variants, so poll in two
|
|
362
|
+
# steps that mirror the SLURM path above. First consult the active-queue
|
|
363
|
+
# view with `qstat -f`: every live state (R, Q, H, E) shows here, and on
|
|
364
|
+
# Torque a completed job lingers here in state `C` carrying its
|
|
365
|
+
# `Exit_status`. `qstat -f` is used rather than plain `qstat <id>` because
|
|
366
|
+
# the latter exits 0 even for completed jobs, so the return code alone
|
|
367
|
+
# could not tell running from done.
|
|
347
368
|
p = subprocess.run(
|
|
348
369
|
["qstat", "-f", job_id], capture_output=True, text=True, check=False
|
|
349
370
|
)
|
|
350
|
-
if p.returncode
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
371
|
+
if p.returncode == 0:
|
|
372
|
+
state, exit_code = _parse_pbs_state(p.stdout)
|
|
373
|
+
if state in ("C", "F"):
|
|
374
|
+
return False, exit_code
|
|
375
|
+
if state is not None:
|
|
376
|
+
return True, None
|
|
377
|
+
# The job is no longer in the active queue---on PBS Pro a finished job is
|
|
378
|
+
# hidden from plain `qstat` and shows (as state `F` with its `Exit_status`)
|
|
379
|
+
# only under `qstat -x`---or qstat failed. Ask the history view for the
|
|
380
|
+
# finished record. Without this, a PBS Pro job that has finished is never
|
|
381
|
+
# seen as done and the wait hangs forever. `-x` means "include finished
|
|
382
|
+
# jobs" on PBS Pro; on Torque it switches qstat to XML output, which simply
|
|
383
|
+
# won't parse here, leaving the stderr check below to settle it (Torque
|
|
384
|
+
# already reported completion above via state `C`).
|
|
385
|
+
hist = subprocess.run(
|
|
386
|
+
["qstat", "-x", "-f", job_id],
|
|
387
|
+
capture_output=True,
|
|
388
|
+
text=True,
|
|
389
|
+
check=False,
|
|
390
|
+
)
|
|
391
|
+
if hist.returncode == 0:
|
|
392
|
+
state, exit_code = _parse_pbs_state(hist.stdout)
|
|
393
|
+
if state in ("C", "F"):
|
|
394
|
+
return False, exit_code
|
|
395
|
+
if state is not None:
|
|
396
|
+
return True, None
|
|
397
|
+
# Neither view resolved a state. A non-zero exit is ambiguous: either the
|
|
398
|
+
# job is gone (finished and purged from history) or qstat failed
|
|
399
|
+
# transiently---a busy PBS server periodically refuses connections or times
|
|
400
|
+
# out. Treating a transient failure as completion would stop the wait while
|
|
401
|
+
# the job is still running and writing its log, so only conclude the job is
|
|
402
|
+
# done when qstat positively reports it is unknown; otherwise keep waiting.
|
|
403
|
+
# A purged job is done but with an unknowable exit status.
|
|
404
|
+
stderr = (p.stderr or "").lower()
|
|
405
|
+
return ("unknown job" not in stderr), None
|
|
375
406
|
|
|
376
407
|
|
|
377
408
|
def _is_active(kind: str, job_id: str) -> bool:
|
calkit/models/core.py
CHANGED
|
@@ -96,8 +96,6 @@ class Environment(BaseModel):
|
|
|
96
96
|
# without a spec file (e.g. Matlab/Slurm/PBS) don't carry an unused field
|
|
97
97
|
# and a required ``path`` doesn't conflict with the base's optional one.
|
|
98
98
|
description: str | None = None
|
|
99
|
-
stage: str | None = None
|
|
100
|
-
default: bool | None = None
|
|
101
99
|
|
|
102
100
|
|
|
103
101
|
class CondaEnvironment(Environment):
|
|
@@ -112,6 +110,8 @@ class VenvEnvironment(Environment):
|
|
|
112
110
|
# If unset, the prefix is resolved on the fly (defaults to .venv next to
|
|
113
111
|
# the spec file, nesting under .calkit/envs/{name}/.venv on conflict)
|
|
114
112
|
prefix: str | None = None
|
|
113
|
+
# Python version to use when creating the environment
|
|
114
|
+
python: str | None = None
|
|
115
115
|
|
|
116
116
|
|
|
117
117
|
class UvEnvironment(Environment):
|
|
@@ -125,6 +125,8 @@ class UvVenvEnvironment(Environment):
|
|
|
125
125
|
# If unset, the prefix is resolved on the fly (defaults to .venv next to
|
|
126
126
|
# the spec file, nesting under .calkit/envs/{name}/.venv on conflict)
|
|
127
127
|
prefix: str | None = None
|
|
128
|
+
# Python version to use when creating the environment
|
|
129
|
+
python: str | None = None
|
|
128
130
|
|
|
129
131
|
|
|
130
132
|
class PixiEnvironment(Environment):
|
|
@@ -153,6 +155,24 @@ class DockerEnvironment(Environment):
|
|
|
153
155
|
shell: Literal["bash", "sh"] = "sh"
|
|
154
156
|
command_mode: Literal["shell", "entrypoint"] = "shell"
|
|
155
157
|
platform: str | None = None
|
|
158
|
+
# Working directory inside the container; defaults to ``/work`` at run time
|
|
159
|
+
wdir: str | None = None
|
|
160
|
+
# User to run the container as; defaults to the host user at run time
|
|
161
|
+
user: str | None = None
|
|
162
|
+
# Files added to the container as dependencies
|
|
163
|
+
deps: list[str] | None = None
|
|
164
|
+
# Environment variables to set in the container
|
|
165
|
+
env_vars: dict[str, str] | None = None
|
|
166
|
+
# Ports to expose, e.g. ``"8080:80"``
|
|
167
|
+
ports: list[str] | None = None
|
|
168
|
+
# GPUs to make available, passed to ``docker run --gpus``
|
|
169
|
+
gpus: str | None = None
|
|
170
|
+
# Extra arguments passed to ``docker run``
|
|
171
|
+
args: list[str] | None = None
|
|
172
|
+
# Name of the Jupyter kernel inside the image, used when executing
|
|
173
|
+
# notebooks with ``calkit nb execute``; defaults to ``python3`` (or ``ir``
|
|
174
|
+
# for R images)
|
|
175
|
+
jupyter_kernel: str | None = None
|
|
156
176
|
|
|
157
177
|
|
|
158
178
|
class REnvironment(Environment):
|
|
@@ -195,6 +215,7 @@ class SSHEnvironment(BaseModel):
|
|
|
195
215
|
key: str | None = None
|
|
196
216
|
send_paths: list[str] = ["./*"]
|
|
197
217
|
get_paths: list[str] = ["*"]
|
|
218
|
+
description: str | None = None
|
|
198
219
|
|
|
199
220
|
|
|
200
221
|
class Software(BaseModel):
|
|
@@ -10,6 +10,7 @@ import sys
|
|
|
10
10
|
import pytest
|
|
11
11
|
|
|
12
12
|
import calkit
|
|
13
|
+
import calkit.notebooks
|
|
13
14
|
|
|
14
15
|
|
|
15
16
|
def test_clean_notebook_outputs(tmp_dir):
|
|
@@ -264,3 +265,87 @@ def test_execute_notebook_auto_env(tmp_dir):
|
|
|
264
265
|
# TODO: Check if the notebook is saved in the project and if it
|
|
265
266
|
# has an environment specified, either in the notebooks section or a
|
|
266
267
|
# pipeline stage
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
@pytest.mark.skipif(
|
|
271
|
+
sys.platform == "win32",
|
|
272
|
+
reason="Docker Linux images are unavailable on windows-latest GHA runners",
|
|
273
|
+
)
|
|
274
|
+
def test_execute_notebook_docker(tmp_dir):
|
|
275
|
+
# A Docker environment runs the kernel inside the container, so we execute
|
|
276
|
+
# with Papermill inside the container rather than registering a host kernel
|
|
277
|
+
if shutil.which("docker") is None:
|
|
278
|
+
pytest.skip("Docker is not available")
|
|
279
|
+
image = "calkit-nb-docker-test:latest"
|
|
280
|
+
# Bake ipykernel and Papermill into the image so execution is deterministic
|
|
281
|
+
with open("Dockerfile", "w") as f:
|
|
282
|
+
f.write(
|
|
283
|
+
"FROM python:3.10-slim\n"
|
|
284
|
+
"RUN pip install --no-cache-dir ipykernel papermill\n"
|
|
285
|
+
"WORKDIR /work\n"
|
|
286
|
+
)
|
|
287
|
+
subprocess.check_call(["docker", "build", "-t", image, "."])
|
|
288
|
+
subprocess.check_call(["calkit", "init"])
|
|
289
|
+
ck_info = {
|
|
290
|
+
"environments": {
|
|
291
|
+
"docker-env": {"kind": "docker", "image": image, "wdir": "/work"}
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
with open("calkit.yaml", "w") as f:
|
|
295
|
+
calkit.ryaml.dump(ck_info, f)
|
|
296
|
+
# A notebook with a 'parameters' cell so we can verify injection
|
|
297
|
+
nb = {
|
|
298
|
+
"cells": [
|
|
299
|
+
{
|
|
300
|
+
"cell_type": "code",
|
|
301
|
+
"execution_count": None,
|
|
302
|
+
"id": "params",
|
|
303
|
+
"metadata": {"tags": ["parameters"]},
|
|
304
|
+
"outputs": [],
|
|
305
|
+
"source": ["a = 1\n"],
|
|
306
|
+
},
|
|
307
|
+
{
|
|
308
|
+
"cell_type": "code",
|
|
309
|
+
"execution_count": None,
|
|
310
|
+
"id": "compute",
|
|
311
|
+
"metadata": {},
|
|
312
|
+
"outputs": [],
|
|
313
|
+
"source": ["result = a * 10\n", "print('result =', result)\n"],
|
|
314
|
+
},
|
|
315
|
+
],
|
|
316
|
+
"metadata": {},
|
|
317
|
+
"nbformat": 4,
|
|
318
|
+
"nbformat_minor": 5,
|
|
319
|
+
}
|
|
320
|
+
os.makedirs("notebooks")
|
|
321
|
+
with open(os.path.join("notebooks", "main.ipynb"), "w") as f:
|
|
322
|
+
json.dump(nb, f)
|
|
323
|
+
subprocess.check_call(
|
|
324
|
+
[
|
|
325
|
+
"calkit",
|
|
326
|
+
"nb",
|
|
327
|
+
"exec",
|
|
328
|
+
"-e",
|
|
329
|
+
"docker-env",
|
|
330
|
+
"--no-check",
|
|
331
|
+
"-p",
|
|
332
|
+
"a=4",
|
|
333
|
+
"notebooks/main.ipynb",
|
|
334
|
+
]
|
|
335
|
+
)
|
|
336
|
+
# The injected parameter should have produced result = 40 in the executed
|
|
337
|
+
# notebook saved under .calkit/notebooks/executed
|
|
338
|
+
exec_path = calkit.notebooks.get_executed_notebook_path(
|
|
339
|
+
notebook_path="notebooks/main.ipynb",
|
|
340
|
+
to="notebook",
|
|
341
|
+
parameters={"a": 4},
|
|
342
|
+
)
|
|
343
|
+
with open(exec_path) as f:
|
|
344
|
+
executed = json.load(f)
|
|
345
|
+
text = "".join(
|
|
346
|
+
"".join(o.get("text", []))
|
|
347
|
+
for cell in executed["cells"]
|
|
348
|
+
for o in cell.get("outputs", [])
|
|
349
|
+
if o.get("name") == "stdout"
|
|
350
|
+
)
|
|
351
|
+
assert "result = 40" in text
|
|
@@ -215,44 +215,75 @@ def test_poll_job_pbs(monkeypatch):
|
|
|
215
215
|
|
|
216
216
|
import calkit.cli.scheduler as sched
|
|
217
217
|
|
|
218
|
-
|
|
218
|
+
# PBS is polled in two steps: plain `qstat -f` for the active queue, then
|
|
219
|
+
# `qstat -x -f` for the finished-job history view. The fake dispatches on
|
|
220
|
+
# whether `-x` is present so each step can be simulated independently, the
|
|
221
|
+
# way a real PBS Pro server answers them.
|
|
222
|
+
active: dict = {}
|
|
223
|
+
history: dict = {}
|
|
219
224
|
|
|
220
|
-
def _fake_run(*args, **kwargs):
|
|
225
|
+
def _fake_run(cmd, *args, **kwargs):
|
|
226
|
+
outcomes = history if "-x" in cmd else active
|
|
221
227
|
return subprocess.CompletedProcess(
|
|
222
|
-
|
|
228
|
+
cmd,
|
|
223
229
|
returncode=outcomes["returncode"],
|
|
224
230
|
stdout=outcomes.get("stdout", ""),
|
|
225
231
|
stderr=outcomes.get("stderr", ""),
|
|
226
232
|
)
|
|
227
233
|
|
|
228
234
|
monkeypatch.setattr(sched.subprocess, "run", _fake_run)
|
|
229
|
-
# A running job (state R) is active with no exit code yet
|
|
230
|
-
|
|
235
|
+
# A running job (state R) is active with no exit code yet, and resolves
|
|
236
|
+
# from the active queue alone---the history view is never consulted.
|
|
237
|
+
active.update(returncode=0, stdout=" job_state = R\n", stderr="")
|
|
238
|
+
history.clear()
|
|
231
239
|
assert _poll_job("pbs", "1.pbs") == (True, None)
|
|
232
|
-
#
|
|
233
|
-
#
|
|
234
|
-
#
|
|
235
|
-
|
|
240
|
+
# The reported bug: on PBS Pro a finished job leaves the active queue and
|
|
241
|
+
# is shown only under `qstat -x`, where it appears as state F with its
|
|
242
|
+
# Exit_status. Plain `qstat -f` errors with a message that is NOT "unknown
|
|
243
|
+
# job", which used to be read as "still active" and hang the wait forever.
|
|
244
|
+
active.update(
|
|
245
|
+
returncode=1,
|
|
246
|
+
stdout="",
|
|
247
|
+
stderr="qstat: 1.pbs Job has finished, use -x to obtain historical "
|
|
248
|
+
"job information\n",
|
|
249
|
+
)
|
|
250
|
+
history.update(
|
|
236
251
|
returncode=0, stdout=" job_state = F\n Exit_status = 0\n"
|
|
237
252
|
)
|
|
238
253
|
assert _poll_job("pbs", "1.pbs") == (False, 0)
|
|
239
|
-
|
|
254
|
+
# A non-zero exit code from the finished job is reported back so the caller
|
|
255
|
+
# can fail the stage.
|
|
256
|
+
history.update(
|
|
257
|
+
returncode=0, stdout=" job_state = F\n Exit_status = 137\n"
|
|
258
|
+
)
|
|
259
|
+
assert _poll_job("pbs", "1.pbs") == (False, 137)
|
|
260
|
+
# On Torque a completed job lingers in the active queue as state C with its
|
|
261
|
+
# exit status, so it is done without ever needing the history view.
|
|
262
|
+
active.update(
|
|
240
263
|
returncode=0, stdout=" job_state = C\n exit_status = 137\n"
|
|
241
264
|
)
|
|
265
|
+
history.update(returncode=1, stdout="", stderr="should not be reached\n")
|
|
242
266
|
assert _poll_job("pbs", "1.pbs") == (False, 137)
|
|
243
267
|
# A finished job whose record lacks an exit status yields an unknown code.
|
|
244
|
-
|
|
268
|
+
active.update(returncode=1, stdout="", stderr="qstat: Unknown Job Id\n")
|
|
269
|
+
history.update(returncode=0, stdout=" job_state = F\n")
|
|
245
270
|
assert _poll_job("pbs", "1.pbs") == (False, None)
|
|
246
|
-
# Once purged
|
|
247
|
-
# with no way to recover the exit status.
|
|
248
|
-
|
|
271
|
+
# Once purged from history too, both views error with "unknown job": done,
|
|
272
|
+
# but with no way to recover the exit status.
|
|
273
|
+
active.update(
|
|
274
|
+
returncode=1, stdout="", stderr="qstat: Unknown Job Id 1.pbs\n"
|
|
275
|
+
)
|
|
276
|
+
history.update(
|
|
249
277
|
returncode=1, stdout="", stderr="qstat: Unknown Job Id 1.pbs\n"
|
|
250
278
|
)
|
|
251
279
|
assert _poll_job("pbs", "1.pbs") == (False, None)
|
|
252
|
-
# A transient qstat failure (busy/unreachable server) is NOT
|
|
253
|
-
# treating it as done would stop the wait while the job still
|
|
254
|
-
# job is reported active and the caller keeps polling.
|
|
255
|
-
|
|
280
|
+
# A transient qstat failure (busy/unreachable server) on both views is NOT
|
|
281
|
+
# completion: treating it as done would stop the wait while the job still
|
|
282
|
+
# runs, so the job is reported active and the caller keeps polling.
|
|
283
|
+
active.update(
|
|
284
|
+
returncode=1, stdout="", stderr="qstat: cannot connect to server\n"
|
|
285
|
+
)
|
|
286
|
+
history.update(
|
|
256
287
|
returncode=1, stdout="", stderr="qstat: cannot connect to server\n"
|
|
257
288
|
)
|
|
258
289
|
assert _poll_job("pbs", "1.pbs") == (True, None)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: calkit-python
|
|
3
|
-
Version: 0.41.
|
|
3
|
+
Version: 0.41.18
|
|
4
4
|
Summary: Reproducibility simplified.
|
|
5
5
|
Project-URL: Homepage, https://calkit.org
|
|
6
6
|
Project-URL: Issues, https://github.com/calkit/calkit/issues
|
|
@@ -78,35 +78,30 @@ Description-Content-Type: text/markdown
|
|
|
78
78
|
|
|
79
79
|
<!-- INCLUDE: docs/index.md -->
|
|
80
80
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
- Automating a step can and should take roughly
|
|
100
|
-
the same amount of time as doing it once manually,
|
|
101
|
-
therefore it's almost always worth it.
|
|
102
|
-
- Working in a "quick and dirty" way can easily become _not quick_ when
|
|
103
|
-
the dirtiness results in mistakes and/or discourages working in small steps.
|
|
81
|
+
It's six months since you submitted your paper,
|
|
82
|
+
do you know exactly how your figures and results were generated?
|
|
83
|
+
|
|
84
|
+
You will if they're part of a Calkit project.
|
|
85
|
+
|
|
86
|
+
Calkit helps you integrate code, data, figures, results, publications,
|
|
87
|
+
and more into a cohesive, traceable, and portable _knowledge creation system_,
|
|
88
|
+
so every output can be traced back to its source and reproduced with a
|
|
89
|
+
single command.
|
|
90
|
+
|
|
91
|
+
With industry standard tools combined into a unified and simplified experience
|
|
92
|
+
tailored for research,
|
|
93
|
+
you can reap the rewards of reproducibility and automation
|
|
94
|
+
without the cognitive overhead.
|
|
95
|
+
|
|
96
|
+
<!-- https://docs.google.com/drawings/d/1XMGnbgYYNFAVUBDyUaCyLfRB7efvJdrnrKmFlNmT19o/edit -->
|
|
97
|
+
|
|
98
|
+

|
|
104
99
|
|
|
105
100
|
## Features
|
|
106
101
|
|
|
107
102
|
- A simplified [version control](https://docs.calkit.org/version-control)
|
|
108
103
|
interface that unifies Git and DVC (Data Version Control),
|
|
109
|
-
so
|
|
104
|
+
so everything can be kept in the same project repository.
|
|
110
105
|
This way, code doesn't need to be siloed away from other
|
|
111
106
|
important artifacts like datasets, models, figures, or article PDFs,
|
|
112
107
|
allowing you to work on all parts of a project without hopping around to
|
|
@@ -129,9 +124,10 @@ and build upon the results.
|
|
|
129
124
|
- [Overleaf integration](https://docs.calkit.org/overleaf/), so
|
|
130
125
|
analysis, visualization, and writing can all stay in sync
|
|
131
126
|
(no more manual uploads!)
|
|
132
|
-
- Support for running on high performance computing (HPC) systems
|
|
133
|
-
|
|
134
|
-
- Support for running with
|
|
127
|
+
- Support for running on [high performance computing (HPC)](https://docs.calkit.org/hpc) systems
|
|
128
|
+
that use PBS or SLURM schedulers.
|
|
129
|
+
- Support for automated running with
|
|
130
|
+
[GitHub Actions](https://docs.calkit.org/tutorials/github-actions).
|
|
135
131
|
- Extensions for doing all of the above graphically in
|
|
136
132
|
[JupyterLab](https://docs.calkit.org/jupyterlab) and
|
|
137
133
|
[VS Code](https://marketplace.visualstudio.com/items?itemName=Calkit.calkit-vscode).
|
|
@@ -373,9 +369,37 @@ To back up or save the project, call:
|
|
|
373
369
|
calkit save -am "Run pipeline"
|
|
374
370
|
```
|
|
375
371
|
|
|
372
|
+
### With an AI coding agent
|
|
373
|
+
|
|
374
|
+
Simply tell the [AI agent](https://docs.calkit.org/ai-tools):
|
|
375
|
+
|
|
376
|
+
> Turn this folder into a Calkit project
|
|
377
|
+
|
|
378
|
+
or
|
|
379
|
+
|
|
380
|
+
> Create me a new Calkit project for investigating...
|
|
381
|
+
|
|
376
382
|
<!-- END INCLUDE -->
|
|
377
383
|
|
|
378
384
|
## Get involved
|
|
379
385
|
|
|
380
386
|
We welcome all kinds of contributions!
|
|
381
387
|
See [CONTRIBUTING.md](CONTRIBUTING.md) to learn how to get involved.
|
|
388
|
+
|
|
389
|
+
## Acknowledgements
|
|
390
|
+
|
|
391
|
+
<!-- INCLUDE: docs/acknowledgements.md +1 -->
|
|
392
|
+
|
|
393
|
+
Calkit is supported by the
|
|
394
|
+
[Caltech Schmidt Academy of Software Engineering](https://sase.caltech.edu).
|
|
395
|
+
|
|
396
|
+
<p align="center">
|
|
397
|
+
<a href="https://sase.caltech.edu" target="_blank">
|
|
398
|
+
<img width="40%" src="docs/img/caltech.png" alt="Caltech SASE">
|
|
399
|
+
</a>
|
|
400
|
+
<a href="https://schmidtsciences.org/" target="_blank">
|
|
401
|
+
<img width="40%" src="docs/img/schmidt-sciences.png" alt="Schmidt Sciences">
|
|
402
|
+
</a>
|
|
403
|
+
</p>
|
|
404
|
+
|
|
405
|
+
<!-- END INCLUDE -->
|
|
@@ -41,10 +41,10 @@ calkit/cli/import_.py,sha256=5j2ANLq6v7HOku9kJUWylwvQnJfkb_OUKCNOdv1BQXU,18264
|
|
|
41
41
|
calkit/cli/latex.py,sha256=dhcbVNcR3YjVSD5mo-OBtZN1FhH4mv4mlGnuP19EbjU,6220
|
|
42
42
|
calkit/cli/list.py,sha256=rV3gCVvoZ4CkUYXYfNRow11-AV99b98w7sabcDq08uk,10521
|
|
43
43
|
calkit/cli/new.py,sha256=OtXcXmhtywXM54dgyjxQgtYEfCYBAi_UVKvMlPjSI8k,129111
|
|
44
|
-
calkit/cli/notebooks.py,sha256=
|
|
44
|
+
calkit/cli/notebooks.py,sha256=mMNHE8yT0BDN0iJE07GCBFAseyDF0jSRshIm1CQjgrY,24645
|
|
45
45
|
calkit/cli/office.py,sha256=jVSTvbl91TCzlglST5O2b8hdApZA_QHw_UiEQFJqxdc,1754
|
|
46
46
|
calkit/cli/overleaf.py,sha256=bS-SkYsSqqhC9YsP-NCzpYJCGS3820w18w_s8JOFihg,25216
|
|
47
|
-
calkit/cli/scheduler.py,sha256=
|
|
47
|
+
calkit/cli/scheduler.py,sha256=Nt1wQMJzuY3ZNkq1KVEV0dZk4T8Gr0XPLQ6mJpXPrZE,40125
|
|
48
48
|
calkit/cli/update.py,sha256=KaO-VbUxMpR_a9FV42HG8d9p-iaA4-2-L-fAgb2LiYI,43499
|
|
49
49
|
calkit/cli/main/__init__.py,sha256=qu1POZPyqs33ZKfasOxv_Wc-EzVcEgK3Dt_vwFL8Bi8,65
|
|
50
50
|
calkit/cli/main/core.py,sha256=MMKEAI4bAJm1YJcS4E5MTQAR-XluwtGn3wlsNYjK-NE,123878
|
|
@@ -55,7 +55,7 @@ calkit/dvc/zip.py,sha256=nsAkcTLinH5S2UyZMdoYrBESKPb7s-PARmnuY9dyWVQ,21256
|
|
|
55
55
|
calkit/jupyterlab/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
56
56
|
calkit/jupyterlab/routes.py,sha256=jnNd289CpNQFY6am5p3qaiL_Tcnkd6J_vDVdjIXlb1A,51465
|
|
57
57
|
calkit/models/__init__.py,sha256=vGYF4MZBGzDk1BPBzpp1bRb59RCJ2Bj2kzHuVVRacug,120
|
|
58
|
-
calkit/models/core.py,sha256=
|
|
58
|
+
calkit/models/core.py,sha256=m6r0R-6loWet_O0nU5Su3wIm3BTONaaTUE4qVWbU3fQ,12829
|
|
59
59
|
calkit/models/io.py,sha256=FgItZDxrSyph91YRMOEfJo9xWmy3drMin_632vy9WhA,1087
|
|
60
60
|
calkit/models/iteration.py,sha256=lodRMjj818ysec303PkuepOUr3dMPOa0cwKmTUaw9Ok,2947
|
|
61
61
|
calkit/models/pipeline.py,sha256=Ov0vDogxe4PjWath80cmUR9uVx3XfoAT66PAuNYxUMo,46069
|
|
@@ -101,9 +101,9 @@ calkit/tests/cli/test_import.py,sha256=e2BToVPebf4NalFQuIgyTNJHwbc9VMc6AYO-FAhYb
|
|
|
101
101
|
calkit/tests/cli/test_latex.py,sha256=SqDbenoFtnsgkfqFjR2Jhw6TJsYQBtLqIGjM3PJmddc,3604
|
|
102
102
|
calkit/tests/cli/test_list.py,sha256=ejMg6q8vtjCbKxvS9zZ32rv_oeKH94JbvpEiVjnw5QA,3859
|
|
103
103
|
calkit/tests/cli/test_new.py,sha256=_-lQH53OkKC-nyLPrdnwndTDyzXv8-aGnXOaDQ_bu-s,46568
|
|
104
|
-
calkit/tests/cli/test_notebooks.py,sha256=
|
|
104
|
+
calkit/tests/cli/test_notebooks.py,sha256=2t1KiGhEz9H9LcIdWy4jGM05REUxJpWiIg6fSpX12ME,10678
|
|
105
105
|
calkit/tests/cli/test_overleaf.py,sha256=Y-ud4kZHh3nhyoZlXrtMfx61z-GPNytUpfDHMad3tuA,16474
|
|
106
|
-
calkit/tests/cli/test_scheduler.py,sha256=
|
|
106
|
+
calkit/tests/cli/test_scheduler.py,sha256=UBJCcZzx9lvPwymFi3SOt5bdwuNZ8wzDy2obMZ-Qn5E,17205
|
|
107
107
|
calkit/tests/cli/test_update.py,sha256=bwRF0kpqbCVxNvGH777LHFX3oUP2DyN3XXUBDCEKNoc,6356
|
|
108
108
|
calkit/tests/cli/main/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
109
109
|
calkit/tests/cli/main/test_core.py,sha256=lowrbL1WCUPAEQjzMWFLPTbrARyFGEQKtVWGp2Lqrpg,63418
|
|
@@ -123,23 +123,23 @@ calkit/tests/models/test_pipeline.py,sha256=AJsMAX4928U3_Omt0_8K96U5LU_KoNErgpHe
|
|
|
123
123
|
calkit/agent_skills/add-pipeline-stage/SKILL.md,sha256=kpYYb-rJsS4B9p1Ub3gL21irCG5ZZ4DvWZpdJeEwPZk,3772
|
|
124
124
|
calkit/agent_skills/conventions/SKILL.md,sha256=nAM2FjSMk6ED56Dr5zh2bL_dhfzDra-h2ZgzHU7ruS4,9524
|
|
125
125
|
calkit/agent_skills/create-pipeline/SKILL.md,sha256=2FGw1iDQVRk5FUq79GP3lPdgvgof2Wi1O9O-abGGtgs,5422
|
|
126
|
-
calkit_python-0.41.
|
|
127
|
-
calkit_python-0.41.
|
|
128
|
-
calkit_python-0.41.
|
|
129
|
-
calkit_python-0.41.
|
|
130
|
-
calkit_python-0.41.
|
|
131
|
-
calkit_python-0.41.
|
|
132
|
-
calkit_python-0.41.
|
|
133
|
-
calkit_python-0.41.
|
|
134
|
-
calkit_python-0.41.
|
|
135
|
-
calkit_python-0.41.
|
|
136
|
-
calkit_python-0.41.
|
|
137
|
-
calkit_python-0.41.
|
|
138
|
-
calkit_python-0.41.
|
|
139
|
-
calkit_python-0.41.
|
|
140
|
-
calkit_python-0.41.
|
|
141
|
-
calkit_python-0.41.
|
|
142
|
-
calkit_python-0.41.
|
|
143
|
-
calkit_python-0.41.
|
|
144
|
-
calkit_python-0.41.
|
|
145
|
-
calkit_python-0.41.
|
|
126
|
+
calkit_python-0.41.18.data/data/etc/jupyter/jupyter_server_config.d/calkit.json,sha256=CWrZP--JDGz8fsvbAlKr_duTL1vDriGT48SL1YCtkY0,81
|
|
127
|
+
calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/package.json,sha256=39PQEWOzw0qtqD7Kep3OPbUR113vI7CG5TeD84v7clM,6224
|
|
128
|
+
calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig,sha256=NBG3t7gFYb6_2J-yPU29qS7Ck9pYgm8-wC5Q1LrzQeI,6082
|
|
129
|
+
calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json,sha256=YSpIrwpwB-lQBk9Mwv-npedWTOOZreO6nlWZhqlWyGI,840
|
|
130
|
+
calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/502.9a2c5772a15466e923ef.js,sha256=mixXcqFUZukj7_jQVxHTavsYl7cdZv83sEe4phJgkh0,59893
|
|
131
|
+
calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/695.2c41003a452d43d2b358.js,sha256=LEEAOkUtQ9KzWEHn-QFv5yGi70B-sBOnrvztzHr0Shw,223
|
|
132
|
+
calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/867.a42a046aa5108f54f8fb.js,sha256=pCoEaqUQj1T4-zAc9SUd__9ZGm7uL2wHQve2xwL89NI,8156
|
|
133
|
+
calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/909.e3f9cc3408834a7fdcc3.js,sha256=4_nMNAiDSn_cw7UjIHEwone4fizrvqrqSgbwUWvjmoU,114571
|
|
134
|
+
calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js,sha256=n_a0yu_gE7l6fauyoXXv9BZ7ZEYt2387mTfs7CbLcs4,51939
|
|
135
|
+
calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js.LICENSE.txt,sha256=eNJ8gc9n9IF8nW1d9sI9niuHstYzjNz5vqXx9UgWSPc,249
|
|
136
|
+
calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/b2f1c3efe70cb539d121.png,sha256=svHD7-cMtTnRITFwugwsVaB9nZ-h8A61ose8z32HiRE,24850
|
|
137
|
+
calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/remoteEntry.ac9035764d5b2adbb542.js,sha256=rJA1dk1bKtu1QkURrp5HEC2WCxzZppPV1SCYz-35GTc,8737
|
|
138
|
+
calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/style.js,sha256=r89Jlk5v1drcwhCpv9FnC3Jig_JH4k24kZNIvxh6Htk,149
|
|
139
|
+
calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json,sha256=2BGdItLJwO3fAopLl4eJvp26TEwuscYC0-yFaF-LaLU,13683
|
|
140
|
+
calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/install.json,sha256=DK9d8G-q-rMVlcT3rAeGIyo3REWKw6FySBZceLU9yaw,187
|
|
141
|
+
calkit_python-0.41.18.dist-info/METADATA,sha256=zexxBw6ZYpD61wV5bcv8i25G0lGpGxrhJ0HKM3CAh1s,12437
|
|
142
|
+
calkit_python-0.41.18.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
|
|
143
|
+
calkit_python-0.41.18.dist-info/entry_points.txt,sha256=2iQBBzjTAOdk66CwS-f3ecXXW5DjUqkG1KBRj8mHI1I,133
|
|
144
|
+
calkit_python-0.41.18.dist-info/licenses/LICENSE,sha256=9ZamCaSUTZk9rcrnf-sWFKLOHr3ws-S_dgKMegW4nw8,1056
|
|
145
|
+
calkit_python-0.41.18.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|