calkit-python 0.41.16__py3-none-any.whl → 0.41.18__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. calkit/cli/notebooks.py +74 -4
  2. calkit/cli/scheduler.py +61 -30
  3. calkit/models/core.py +23 -2
  4. calkit/tests/cli/test_notebooks.py +85 -0
  5. calkit/tests/cli/test_scheduler.py +49 -18
  6. {calkit_python-0.41.16.dist-info → calkit_python-0.41.18.dist-info}/METADATA +52 -28
  7. {calkit_python-0.41.16.dist-info → calkit_python-0.41.18.dist-info}/RECORD +25 -25
  8. {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/etc/jupyter/jupyter_server_config.d/calkit.json +0 -0
  9. {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/install.json +0 -0
  10. {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/package.json +0 -0
  11. {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig +0 -0
  12. {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json +0 -0
  13. {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/static/502.9a2c5772a15466e923ef.js +0 -0
  14. {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/static/695.2c41003a452d43d2b358.js +0 -0
  15. {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/static/867.a42a046aa5108f54f8fb.js +0 -0
  16. {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/static/909.e3f9cc3408834a7fdcc3.js +0 -0
  17. {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js +0 -0
  18. {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js.LICENSE.txt +0 -0
  19. {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/static/b2f1c3efe70cb539d121.png +0 -0
  20. {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/static/remoteEntry.ac9035764d5b2adbb542.js +0 -0
  21. {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/static/style.js +0 -0
  22. {calkit_python-0.41.16.data → calkit_python-0.41.18.data}/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json +0 -0
  23. {calkit_python-0.41.16.dist-info → calkit_python-0.41.18.dist-info}/WHEEL +0 -0
  24. {calkit_python-0.41.16.dist-info → calkit_python-0.41.18.dist-info}/entry_points.txt +0 -0
  25. {calkit_python-0.41.16.dist-info → calkit_python-0.41.18.dist-info}/licenses/LICENSE +0 -0
calkit/cli/notebooks.py CHANGED
@@ -490,7 +490,17 @@ def execute_notebook(
490
490
  typer.echo(f"Using {language} as notebook language")
491
491
  # First, ensure the specified environment has a kernel we can use
492
492
  # We need to check the environment type and create the kernel if needed
493
- if language.lower() in ["python", "julia", "r"]:
493
+ docker_env = env.get("kind") == "docker"
494
+ if docker_env:
495
+ # Docker environments run the kernel inside the container, so there is
496
+ # no host kernel to register. Use the container's own kernel (default
497
+ # 'python3', overridable with the env's 'jupyter_kernel' key) and
498
+ # execute with Papermill inside the container further below.
499
+ kernel_name = env.get("jupyter_kernel") or {"r": "ir"}.get(
500
+ language.lower(), "python3"
501
+ )
502
+ display_name = kernel_name
503
+ elif language.lower() in ["python", "julia", "r"]:
494
504
  kernel_name, display_name = check_env_kernel(
495
505
  env_name=env_name,
496
506
  no_check=no_check,
@@ -563,10 +573,70 @@ def execute_notebook(
563
573
  typer.echo(f"Using kernel: {kernel_name}")
564
574
  typer.echo(f"Running with cwd: {notebook_dir}")
565
575
  typer.echo(f"Output will be saved to: {fpath_out_exec}")
566
- # If this is a Python, Julia, or R notebook, we can use Papermill
576
+ # If this is a Python, Julia, or R notebook, we can use Papermill.
567
577
  # If it's a MATLAB notebook, we need to use the MATLAB kernel inside the
568
- # specified environment
569
- if language.lower() in ["python", "julia", "r"]:
578
+ # specified environment.
579
+ # Exception: If the environment is a Docker environment, we run Papermill
580
+ # inside the container so the kernel (which lives in the image) and
581
+ # Papermill share a process namespace---this avoids registering a host
582
+ # kernel and cross-container networking entirely
583
+ if docker_env:
584
+ # run_in_env mounts the project at the working directory and applies
585
+ # the env's user/platform/args, so we only need to invoke Papermill
586
+ # ipykernel is in the image, but Papermill may not be, so install it
587
+ # on demand (warning the user) before executing. We install into a
588
+ # writable temp dir on PYTHONPATH rather than the image's site-packages:
589
+ # run_in_env maps the host user into the container, so writing to the
590
+ # system location is typically denied, but /tmp is world-writable
591
+ pm_target = "/tmp/calkit-papermill"
592
+ pythonpath = f"PYTHONPATH={pm_target}"
593
+ pm_cmd = [
594
+ pythonpath,
595
+ "python",
596
+ "-c",
597
+ "import papermill",
598
+ "2>/dev/null",
599
+ "||",
600
+ "(",
601
+ "echo",
602
+ "papermill not found in image; installing it with pip...",
603
+ "&&",
604
+ "python",
605
+ "-m",
606
+ "pip",
607
+ "install",
608
+ "-q",
609
+ "--target",
610
+ pm_target,
611
+ "papermill",
612
+ ")",
613
+ "&&",
614
+ pythonpath,
615
+ "python",
616
+ "-m",
617
+ "papermill",
618
+ path,
619
+ fpath_out_exec,
620
+ "-k",
621
+ kernel_name,
622
+ "--log-output",
623
+ ]
624
+ if notebook_dir:
625
+ pm_cmd += ["--cwd", notebook_dir]
626
+ if parsed_params:
627
+ # JSON is valid YAML, so pass parameters as base64-encoded JSON to
628
+ # avoid shell-quoting issues across the container boundary
629
+ params_b64 = base64.b64encode(
630
+ json.dumps(parsed_params).encode()
631
+ ).decode()
632
+ pm_cmd += ["-b", params_b64]
633
+ run_in_env(
634
+ pm_cmd,
635
+ env_name=env_name,
636
+ no_check=no_check,
637
+ verbose=verbose,
638
+ )
639
+ elif language.lower() in ["python", "julia", "r"]:
570
640
  papermill.execute_notebook(
571
641
  input_path=path,
572
642
  output_path=fpath_out_exec,
calkit/cli/scheduler.py CHANGED
@@ -304,6 +304,25 @@ def _slurm_exit_code(job_id: str) -> int | None:
304
304
  return None
305
305
 
306
306
 
307
+ def _parse_pbs_state(stdout: str) -> tuple[str | None, int | None]:
308
+ # `qstat -f` output is a flat list of `key = value` lines; pull out the
309
+ # job's state and, for a finished job, its exit status. The status field is
310
+ # spelled `Exit_status` on PBS Pro and `exit_status` on some Torque builds,
311
+ # so match it case-insensitively.
312
+ state = None
313
+ exit_code = None
314
+ for line in stdout.splitlines():
315
+ stripped = line.strip()
316
+ if stripped.startswith("job_state"):
317
+ state = stripped.split("=", 1)[-1].strip()
318
+ elif stripped.lower().startswith("exit_status"):
319
+ try:
320
+ exit_code = int(stripped.split("=", 1)[-1].strip())
321
+ except ValueError:
322
+ exit_code = None
323
+ return state, exit_code
324
+
325
+
307
326
  def _poll_job(kind: str, job_id: str) -> tuple[bool, int | None]:
308
327
  """Return ``(active, exit_code)`` for a scheduler job.
309
328
 
@@ -339,39 +358,51 @@ def _poll_job(kind: str, job_id: str) -> tuple[bool, int | None]:
339
358
  if "invalid job id" in stderr:
340
359
  return False, _slurm_exit_code(job_id)
341
360
  return True, None
342
- # Use `qstat -f` and parse job_state: on Torque/OpenPBS, plain `qstat
343
- # <id>` returns exit 0 even for completed (C) jobs, so checking the
344
- # return code alone would cause `calkit sched batch` to hang forever
345
- # after a PBS job finishes. States C and F mean the job is done, and a
346
- # finished job's record carries its `Exit_status`.
361
+ # PBS reports a finished job differently across variants, so poll in two
362
+ # steps that mirror the SLURM path above. First consult the active-queue
363
+ # view with `qstat -f`: every live state (R, Q, H, E) shows here, and on
364
+ # Torque a completed job lingers here in state `C` carrying its
365
+ # `Exit_status`. `qstat -f` is used rather than plain `qstat <id>` because
366
+ # the latter exits 0 even for completed jobs, so the return code alone
367
+ # could not tell running from done.
347
368
  p = subprocess.run(
348
369
  ["qstat", "-f", job_id], capture_output=True, text=True, check=False
349
370
  )
350
- if p.returncode != 0:
351
- # A non-zero exit is ambiguous: either the job is gone (completed and
352
- # purged from history) or `qstat` itself failed transiently---a busy
353
- # PBS server periodically refuses connections or times out. Treating a
354
- # transient failure as completion would stop the wait while the job is
355
- # still running and writing its log, so only conclude the job is done
356
- # when qstat positively reports it is unknown; otherwise keep waiting.
357
- # A purged job (unknown to qstat) is done but with an unknowable exit
358
- # status; any other error is transient, so report the job as active.
359
- stderr = (p.stderr or "").lower()
360
- return ("unknown job" not in stderr), None
361
- state = None
362
- exit_code = None
363
- for line in p.stdout.splitlines():
364
- stripped = line.strip()
365
- if stripped.startswith("job_state"):
366
- state = stripped.split("=", 1)[-1].strip()
367
- elif stripped.lower().startswith("exit_status"):
368
- try:
369
- exit_code = int(stripped.split("=", 1)[-1].strip())
370
- except ValueError:
371
- exit_code = None
372
- if state in ("C", "F"):
373
- return False, exit_code
374
- return True, None
371
+ if p.returncode == 0:
372
+ state, exit_code = _parse_pbs_state(p.stdout)
373
+ if state in ("C", "F"):
374
+ return False, exit_code
375
+ if state is not None:
376
+ return True, None
377
+ # The job is no longer in the active queue---on PBS Pro a finished job is
378
+ # hidden from plain `qstat` and shows (as state `F` with its `Exit_status`)
379
+ # only under `qstat -x`---or qstat failed. Ask the history view for the
380
+ # finished record. Without this, a PBS Pro job that has finished is never
381
+ # seen as done and the wait hangs forever. `-x` means "include finished
382
+ # jobs" on PBS Pro; on Torque it switches qstat to XML output, which simply
383
+ # won't parse here, leaving the stderr check below to settle it (Torque
384
+ # already reported completion above via state `C`).
385
+ hist = subprocess.run(
386
+ ["qstat", "-x", "-f", job_id],
387
+ capture_output=True,
388
+ text=True,
389
+ check=False,
390
+ )
391
+ if hist.returncode == 0:
392
+ state, exit_code = _parse_pbs_state(hist.stdout)
393
+ if state in ("C", "F"):
394
+ return False, exit_code
395
+ if state is not None:
396
+ return True, None
397
+ # Neither view resolved a state. A non-zero exit is ambiguous: either the
398
+ # job is gone (finished and purged from history) or qstat failed
399
+ # transiently---a busy PBS server periodically refuses connections or times
400
+ # out. Treating a transient failure as completion would stop the wait while
401
+ # the job is still running and writing its log, so only conclude the job is
402
+ # done when qstat positively reports it is unknown; otherwise keep waiting.
403
+ # A purged job is done but with an unknowable exit status.
404
+ stderr = (p.stderr or "").lower()
405
+ return ("unknown job" not in stderr), None
375
406
 
376
407
 
377
408
  def _is_active(kind: str, job_id: str) -> bool:
calkit/models/core.py CHANGED
@@ -96,8 +96,6 @@ class Environment(BaseModel):
96
96
  # without a spec file (e.g. Matlab/Slurm/PBS) don't carry an unused field
97
97
  # and a required ``path`` doesn't conflict with the base's optional one.
98
98
  description: str | None = None
99
- stage: str | None = None
100
- default: bool | None = None
101
99
 
102
100
 
103
101
  class CondaEnvironment(Environment):
@@ -112,6 +110,8 @@ class VenvEnvironment(Environment):
112
110
  # If unset, the prefix is resolved on the fly (defaults to .venv next to
113
111
  # the spec file, nesting under .calkit/envs/{name}/.venv on conflict)
114
112
  prefix: str | None = None
113
+ # Python version to use when creating the environment
114
+ python: str | None = None
115
115
 
116
116
 
117
117
  class UvEnvironment(Environment):
@@ -125,6 +125,8 @@ class UvVenvEnvironment(Environment):
125
125
  # If unset, the prefix is resolved on the fly (defaults to .venv next to
126
126
  # the spec file, nesting under .calkit/envs/{name}/.venv on conflict)
127
127
  prefix: str | None = None
128
+ # Python version to use when creating the environment
129
+ python: str | None = None
128
130
 
129
131
 
130
132
  class PixiEnvironment(Environment):
@@ -153,6 +155,24 @@ class DockerEnvironment(Environment):
153
155
  shell: Literal["bash", "sh"] = "sh"
154
156
  command_mode: Literal["shell", "entrypoint"] = "shell"
155
157
  platform: str | None = None
158
+ # Working directory inside the container; defaults to ``/work`` at run time
159
+ wdir: str | None = None
160
+ # User to run the container as; defaults to the host user at run time
161
+ user: str | None = None
162
+ # Files added to the container as dependencies
163
+ deps: list[str] | None = None
164
+ # Environment variables to set in the container
165
+ env_vars: dict[str, str] | None = None
166
+ # Ports to expose, e.g. ``"8080:80"``
167
+ ports: list[str] | None = None
168
+ # GPUs to make available, passed to ``docker run --gpus``
169
+ gpus: str | None = None
170
+ # Extra arguments passed to ``docker run``
171
+ args: list[str] | None = None
172
+ # Name of the Jupyter kernel inside the image, used when executing
173
+ # notebooks with ``calkit nb execute``; defaults to ``python3`` (or ``ir``
174
+ # for R images)
175
+ jupyter_kernel: str | None = None
156
176
 
157
177
 
158
178
  class REnvironment(Environment):
@@ -195,6 +215,7 @@ class SSHEnvironment(BaseModel):
195
215
  key: str | None = None
196
216
  send_paths: list[str] = ["./*"]
197
217
  get_paths: list[str] = ["*"]
218
+ description: str | None = None
198
219
 
199
220
 
200
221
  class Software(BaseModel):
@@ -10,6 +10,7 @@ import sys
10
10
  import pytest
11
11
 
12
12
  import calkit
13
+ import calkit.notebooks
13
14
 
14
15
 
15
16
  def test_clean_notebook_outputs(tmp_dir):
@@ -264,3 +265,87 @@ def test_execute_notebook_auto_env(tmp_dir):
264
265
  # TODO: Check if the notebook is saved in the project and if it
265
266
  # has an environment specified, either in the notebooks section or a
266
267
  # pipeline stage
268
+
269
+
270
+ @pytest.mark.skipif(
271
+ sys.platform == "win32",
272
+ reason="Docker Linux images are unavailable on windows-latest GHA runners",
273
+ )
274
+ def test_execute_notebook_docker(tmp_dir):
275
+ # A Docker environment runs the kernel inside the container, so we execute
276
+ # with Papermill inside the container rather than registering a host kernel
277
+ if shutil.which("docker") is None:
278
+ pytest.skip("Docker is not available")
279
+ image = "calkit-nb-docker-test:latest"
280
+ # Bake ipykernel and Papermill into the image so execution is deterministic
281
+ with open("Dockerfile", "w") as f:
282
+ f.write(
283
+ "FROM python:3.10-slim\n"
284
+ "RUN pip install --no-cache-dir ipykernel papermill\n"
285
+ "WORKDIR /work\n"
286
+ )
287
+ subprocess.check_call(["docker", "build", "-t", image, "."])
288
+ subprocess.check_call(["calkit", "init"])
289
+ ck_info = {
290
+ "environments": {
291
+ "docker-env": {"kind": "docker", "image": image, "wdir": "/work"}
292
+ }
293
+ }
294
+ with open("calkit.yaml", "w") as f:
295
+ calkit.ryaml.dump(ck_info, f)
296
+ # A notebook with a 'parameters' cell so we can verify injection
297
+ nb = {
298
+ "cells": [
299
+ {
300
+ "cell_type": "code",
301
+ "execution_count": None,
302
+ "id": "params",
303
+ "metadata": {"tags": ["parameters"]},
304
+ "outputs": [],
305
+ "source": ["a = 1\n"],
306
+ },
307
+ {
308
+ "cell_type": "code",
309
+ "execution_count": None,
310
+ "id": "compute",
311
+ "metadata": {},
312
+ "outputs": [],
313
+ "source": ["result = a * 10\n", "print('result =', result)\n"],
314
+ },
315
+ ],
316
+ "metadata": {},
317
+ "nbformat": 4,
318
+ "nbformat_minor": 5,
319
+ }
320
+ os.makedirs("notebooks")
321
+ with open(os.path.join("notebooks", "main.ipynb"), "w") as f:
322
+ json.dump(nb, f)
323
+ subprocess.check_call(
324
+ [
325
+ "calkit",
326
+ "nb",
327
+ "exec",
328
+ "-e",
329
+ "docker-env",
330
+ "--no-check",
331
+ "-p",
332
+ "a=4",
333
+ "notebooks/main.ipynb",
334
+ ]
335
+ )
336
+ # The injected parameter should have produced result = 40 in the executed
337
+ # notebook saved under .calkit/notebooks/executed
338
+ exec_path = calkit.notebooks.get_executed_notebook_path(
339
+ notebook_path="notebooks/main.ipynb",
340
+ to="notebook",
341
+ parameters={"a": 4},
342
+ )
343
+ with open(exec_path) as f:
344
+ executed = json.load(f)
345
+ text = "".join(
346
+ "".join(o.get("text", []))
347
+ for cell in executed["cells"]
348
+ for o in cell.get("outputs", [])
349
+ if o.get("name") == "stdout"
350
+ )
351
+ assert "result = 40" in text
@@ -215,44 +215,75 @@ def test_poll_job_pbs(monkeypatch):
215
215
 
216
216
  import calkit.cli.scheduler as sched
217
217
 
218
- outcomes: dict = {}
218
+ # PBS is polled in two steps: plain `qstat -f` for the active queue, then
219
+ # `qstat -x -f` for the finished-job history view. The fake dispatches on
220
+ # whether `-x` is present so each step can be simulated independently, the
221
+ # way a real PBS Pro server answers them.
222
+ active: dict = {}
223
+ history: dict = {}
219
224
 
220
- def _fake_run(*args, **kwargs):
225
+ def _fake_run(cmd, *args, **kwargs):
226
+ outcomes = history if "-x" in cmd else active
221
227
  return subprocess.CompletedProcess(
222
- args=args[0],
228
+ cmd,
223
229
  returncode=outcomes["returncode"],
224
230
  stdout=outcomes.get("stdout", ""),
225
231
  stderr=outcomes.get("stderr", ""),
226
232
  )
227
233
 
228
234
  monkeypatch.setattr(sched.subprocess, "run", _fake_run)
229
- # A running job (state R) is active with no exit code yet.
230
- outcomes.update(returncode=0, stdout=" job_state = R\n", stderr="")
235
+ # A running job (state R) is active with no exit code yet, and resolves
236
+ # from the active queue alone---the history view is never consulted.
237
+ active.update(returncode=0, stdout=" job_state = R\n", stderr="")
238
+ history.clear()
231
239
  assert _poll_job("pbs", "1.pbs") == (True, None)
232
- # Terminal states C (Torque) and F (PBS Pro) mean the job is done; qstat
233
- # still exits 0 while the record lingers in history and carries the exit
234
- # status, which is reported back so the caller can fail a bad job.
235
- outcomes.update(
240
+ # The reported bug: on PBS Pro a finished job leaves the active queue and
241
+ # is shown only under `qstat -x`, where it appears as state F with its
242
+ # Exit_status. Plain `qstat -f` errors with a message that is NOT "unknown
243
+ # job", which used to be read as "still active" and hang the wait forever.
244
+ active.update(
245
+ returncode=1,
246
+ stdout="",
247
+ stderr="qstat: 1.pbs Job has finished, use -x to obtain historical "
248
+ "job information\n",
249
+ )
250
+ history.update(
236
251
  returncode=0, stdout=" job_state = F\n Exit_status = 0\n"
237
252
  )
238
253
  assert _poll_job("pbs", "1.pbs") == (False, 0)
239
- outcomes.update(
254
+ # A non-zero exit code from the finished job is reported back so the caller
255
+ # can fail the stage.
256
+ history.update(
257
+ returncode=0, stdout=" job_state = F\n Exit_status = 137\n"
258
+ )
259
+ assert _poll_job("pbs", "1.pbs") == (False, 137)
260
+ # On Torque a completed job lingers in the active queue as state C with its
261
+ # exit status, so it is done without ever needing the history view.
262
+ active.update(
240
263
  returncode=0, stdout=" job_state = C\n exit_status = 137\n"
241
264
  )
265
+ history.update(returncode=1, stdout="", stderr="should not be reached\n")
242
266
  assert _poll_job("pbs", "1.pbs") == (False, 137)
243
267
  # A finished job whose record lacks an exit status yields an unknown code.
244
- outcomes.update(returncode=0, stdout=" job_state = F\n")
268
+ active.update(returncode=1, stdout="", stderr="qstat: Unknown Job Id\n")
269
+ history.update(returncode=0, stdout=" job_state = F\n")
245
270
  assert _poll_job("pbs", "1.pbs") == (False, None)
246
- # Once purged, qstat exits non-zero and reports the job unknown: done, but
247
- # with no way to recover the exit status.
248
- outcomes.update(
271
+ # Once purged from history too, both views error with "unknown job": done,
272
+ # but with no way to recover the exit status.
273
+ active.update(
274
+ returncode=1, stdout="", stderr="qstat: Unknown Job Id 1.pbs\n"
275
+ )
276
+ history.update(
249
277
  returncode=1, stdout="", stderr="qstat: Unknown Job Id 1.pbs\n"
250
278
  )
251
279
  assert _poll_job("pbs", "1.pbs") == (False, None)
252
- # A transient qstat failure (busy/unreachable server) is NOT completion:
253
- # treating it as done would stop the wait while the job still runs, so the
254
- # job is reported active and the caller keeps polling.
255
- outcomes.update(
280
+ # A transient qstat failure (busy/unreachable server) on both views is NOT
281
+ # completion: treating it as done would stop the wait while the job still
282
+ # runs, so the job is reported active and the caller keeps polling.
283
+ active.update(
284
+ returncode=1, stdout="", stderr="qstat: cannot connect to server\n"
285
+ )
286
+ history.update(
256
287
  returncode=1, stdout="", stderr="qstat: cannot connect to server\n"
257
288
  )
258
289
  assert _poll_job("pbs", "1.pbs") == (True, None)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: calkit-python
3
- Version: 0.41.16
3
+ Version: 0.41.18
4
4
  Summary: Reproducibility simplified.
5
5
  Project-URL: Homepage, https://calkit.org
6
6
  Project-URL: Issues, https://github.com/calkit/calkit/issues
@@ -78,35 +78,30 @@ Description-Content-Type: text/markdown
78
78
 
79
79
  <!-- INCLUDE: docs/index.md -->
80
80
 
81
- Calkit helps you manage and automate research projects like a software
82
- engineer.
83
-
84
- Define computational environments,
85
- steps that process your data, create figures,
86
- presentations, and publications, connect to external tools,
87
- then iterate quickly and painlessly until your research questions are
88
- answered, tracking changes to all files along the way.
89
- At the end, deliver your entire project as a self-contained, self-documenting,
90
- version-controlled, and
91
- [single button reproducible](https://doi.org/10.1190/1.1822162)
92
- "calculation kit" so you and others can easily verify
93
- and build upon the results.
94
-
95
- ## Guiding principles
96
-
97
- - Quality comes from iteration. Automation reduces the time and effort
98
- needed to iterate, thereby increasing iteration and quality.
99
- - Automating a step can and should take roughly
100
- the same amount of time as doing it once manually,
101
- therefore it's almost always worth it.
102
- - Working in a "quick and dirty" way can easily become _not quick_ when
103
- the dirtiness results in mistakes and/or discourages working in small steps.
81
+ It's six months since you submitted your paper,
82
+ do you know exactly how your figures and results were generated?
83
+
84
+ You will if they're part of a Calkit project.
85
+
86
+ Calkit helps you integrate code, data, figures, results, publications,
87
+ and more into a cohesive, traceable, and portable _knowledge creation system_,
88
+ so every output can be traced back to its source and reproduced with a
89
+ single command.
90
+
91
+ With industry standard tools combined into a unified and simplified experience
92
+ tailored for research,
93
+ you can reap the rewards of reproducibility and automation
94
+ without the cognitive overhead.
95
+
96
+ <!-- https://docs.google.com/drawings/d/1XMGnbgYYNFAVUBDyUaCyLfRB7efvJdrnrKmFlNmT19o/edit -->
97
+
98
+ ![pipeline](docs/img/pipeline.png)
104
99
 
105
100
  ## Features
106
101
 
107
102
  - A simplified [version control](https://docs.calkit.org/version-control)
108
103
  interface that unifies Git and DVC (Data Version Control),
109
- so all materials can be kept in the same project repository.
104
+ so everything can be kept in the same project repository.
110
105
  This way, code doesn't need to be siloed away from other
111
106
  important artifacts like datasets, models, figures, or article PDFs,
112
107
  allowing you to work on all parts of a project without hopping around to
@@ -129,9 +124,10 @@ and build upon the results.
129
124
  - [Overleaf integration](https://docs.calkit.org/overleaf/), so
130
125
  analysis, visualization, and writing can all stay in sync
131
126
  (no more manual uploads!)
132
- - Support for running on high performance computing (HPC) systems that use
133
- [SLURM schedulers](https://docs.calkit.org/pipeline/slurm).
134
- - Support for running with [GitHub Actions](https://docs.calkit.org/tutorials/github-actions).
127
+ - Support for running on [high performance computing (HPC)](https://docs.calkit.org/hpc) systems
128
+ that use PBS or SLURM schedulers.
129
+ - Support for automated running with
130
+ [GitHub Actions](https://docs.calkit.org/tutorials/github-actions).
135
131
  - Extensions for doing all of the above graphically in
136
132
  [JupyterLab](https://docs.calkit.org/jupyterlab) and
137
133
  [VS Code](https://marketplace.visualstudio.com/items?itemName=Calkit.calkit-vscode).
@@ -373,9 +369,37 @@ To back up or save the project, call:
373
369
  calkit save -am "Run pipeline"
374
370
  ```
375
371
 
372
+ ### With an AI coding agent
373
+
374
+ Simply tell the [AI agent](https://docs.calkit.org/ai-tools):
375
+
376
+ > Turn this folder into a Calkit project
377
+
378
+ or
379
+
380
+ > Create me a new Calkit project for investigating...
381
+
376
382
  <!-- END INCLUDE -->
377
383
 
378
384
  ## Get involved
379
385
 
380
386
  We welcome all kinds of contributions!
381
387
  See [CONTRIBUTING.md](CONTRIBUTING.md) to learn how to get involved.
388
+
389
+ ## Acknowledgements
390
+
391
+ <!-- INCLUDE: docs/acknowledgements.md +1 -->
392
+
393
+ Calkit is supported by the
394
+ [Caltech Schmidt Academy of Software Engineering](https://sase.caltech.edu).
395
+
396
+ <p align="center">
397
+ <a href="https://sase.caltech.edu" target="_blank">
398
+ <img width="40%" src="docs/img/caltech.png" alt="Caltech SASE">
399
+ </a>
400
+ <a href="https://schmidtsciences.org/" target="_blank">
401
+ <img width="40%" src="docs/img/schmidt-sciences.png" alt="Schmidt Sciences">
402
+ </a>
403
+ </p>
404
+
405
+ <!-- END INCLUDE -->
@@ -41,10 +41,10 @@ calkit/cli/import_.py,sha256=5j2ANLq6v7HOku9kJUWylwvQnJfkb_OUKCNOdv1BQXU,18264
41
41
  calkit/cli/latex.py,sha256=dhcbVNcR3YjVSD5mo-OBtZN1FhH4mv4mlGnuP19EbjU,6220
42
42
  calkit/cli/list.py,sha256=rV3gCVvoZ4CkUYXYfNRow11-AV99b98w7sabcDq08uk,10521
43
43
  calkit/cli/new.py,sha256=OtXcXmhtywXM54dgyjxQgtYEfCYBAi_UVKvMlPjSI8k,129111
44
- calkit/cli/notebooks.py,sha256=QV3_0wwgIm9cdTBM7CwlUth1xbTnPGp2LJUFVSAoBm4,21941
44
+ calkit/cli/notebooks.py,sha256=mMNHE8yT0BDN0iJE07GCBFAseyDF0jSRshIm1CQjgrY,24645
45
45
  calkit/cli/office.py,sha256=jVSTvbl91TCzlglST5O2b8hdApZA_QHw_UiEQFJqxdc,1754
46
46
  calkit/cli/overleaf.py,sha256=bS-SkYsSqqhC9YsP-NCzpYJCGS3820w18w_s8JOFihg,25216
47
- calkit/cli/scheduler.py,sha256=Mn_G2tx78A_wJ3XYE0hRtTkPj376sKTWpSxMZKirHuA,38660
47
+ calkit/cli/scheduler.py,sha256=Nt1wQMJzuY3ZNkq1KVEV0dZk4T8Gr0XPLQ6mJpXPrZE,40125
48
48
  calkit/cli/update.py,sha256=KaO-VbUxMpR_a9FV42HG8d9p-iaA4-2-L-fAgb2LiYI,43499
49
49
  calkit/cli/main/__init__.py,sha256=qu1POZPyqs33ZKfasOxv_Wc-EzVcEgK3Dt_vwFL8Bi8,65
50
50
  calkit/cli/main/core.py,sha256=MMKEAI4bAJm1YJcS4E5MTQAR-XluwtGn3wlsNYjK-NE,123878
@@ -55,7 +55,7 @@ calkit/dvc/zip.py,sha256=nsAkcTLinH5S2UyZMdoYrBESKPb7s-PARmnuY9dyWVQ,21256
55
55
  calkit/jupyterlab/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
56
56
  calkit/jupyterlab/routes.py,sha256=jnNd289CpNQFY6am5p3qaiL_Tcnkd6J_vDVdjIXlb1A,51465
57
57
  calkit/models/__init__.py,sha256=vGYF4MZBGzDk1BPBzpp1bRb59RCJ2Bj2kzHuVVRacug,120
58
- calkit/models/core.py,sha256=QYbnHHOjd1Y-ublDE-hBL7WzqNIAAGW9e5H1BLIJvyg,11833
58
+ calkit/models/core.py,sha256=m6r0R-6loWet_O0nU5Su3wIm3BTONaaTUE4qVWbU3fQ,12829
59
59
  calkit/models/io.py,sha256=FgItZDxrSyph91YRMOEfJo9xWmy3drMin_632vy9WhA,1087
60
60
  calkit/models/iteration.py,sha256=lodRMjj818ysec303PkuepOUr3dMPOa0cwKmTUaw9Ok,2947
61
61
  calkit/models/pipeline.py,sha256=Ov0vDogxe4PjWath80cmUR9uVx3XfoAT66PAuNYxUMo,46069
@@ -101,9 +101,9 @@ calkit/tests/cli/test_import.py,sha256=e2BToVPebf4NalFQuIgyTNJHwbc9VMc6AYO-FAhYb
101
101
  calkit/tests/cli/test_latex.py,sha256=SqDbenoFtnsgkfqFjR2Jhw6TJsYQBtLqIGjM3PJmddc,3604
102
102
  calkit/tests/cli/test_list.py,sha256=ejMg6q8vtjCbKxvS9zZ32rv_oeKH94JbvpEiVjnw5QA,3859
103
103
  calkit/tests/cli/test_new.py,sha256=_-lQH53OkKC-nyLPrdnwndTDyzXv8-aGnXOaDQ_bu-s,46568
104
- calkit/tests/cli/test_notebooks.py,sha256=pJX6KV45tNBd-ZpqyoayDzNop9-maRIN_fC2j3EcPf0,7899
104
+ calkit/tests/cli/test_notebooks.py,sha256=2t1KiGhEz9H9LcIdWy4jGM05REUxJpWiIg6fSpX12ME,10678
105
105
  calkit/tests/cli/test_overleaf.py,sha256=Y-ud4kZHh3nhyoZlXrtMfx61z-GPNytUpfDHMad3tuA,16474
106
- calkit/tests/cli/test_scheduler.py,sha256=MY2tmhcDLPypNS2yFONUeLjvQog37iw2XGHc9c2wtCY,15715
106
+ calkit/tests/cli/test_scheduler.py,sha256=UBJCcZzx9lvPwymFi3SOt5bdwuNZ8wzDy2obMZ-Qn5E,17205
107
107
  calkit/tests/cli/test_update.py,sha256=bwRF0kpqbCVxNvGH777LHFX3oUP2DyN3XXUBDCEKNoc,6356
108
108
  calkit/tests/cli/main/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
109
109
  calkit/tests/cli/main/test_core.py,sha256=lowrbL1WCUPAEQjzMWFLPTbrARyFGEQKtVWGp2Lqrpg,63418
@@ -123,23 +123,23 @@ calkit/tests/models/test_pipeline.py,sha256=AJsMAX4928U3_Omt0_8K96U5LU_KoNErgpHe
123
123
  calkit/agent_skills/add-pipeline-stage/SKILL.md,sha256=kpYYb-rJsS4B9p1Ub3gL21irCG5ZZ4DvWZpdJeEwPZk,3772
124
124
  calkit/agent_skills/conventions/SKILL.md,sha256=nAM2FjSMk6ED56Dr5zh2bL_dhfzDra-h2ZgzHU7ruS4,9524
125
125
  calkit/agent_skills/create-pipeline/SKILL.md,sha256=2FGw1iDQVRk5FUq79GP3lPdgvgof2Wi1O9O-abGGtgs,5422
126
- calkit_python-0.41.16.data/data/etc/jupyter/jupyter_server_config.d/calkit.json,sha256=CWrZP--JDGz8fsvbAlKr_duTL1vDriGT48SL1YCtkY0,81
127
- calkit_python-0.41.16.data/data/share/jupyter/labextensions/calkit/package.json,sha256=39PQEWOzw0qtqD7Kep3OPbUR113vI7CG5TeD84v7clM,6224
128
- calkit_python-0.41.16.data/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig,sha256=NBG3t7gFYb6_2J-yPU29qS7Ck9pYgm8-wC5Q1LrzQeI,6082
129
- calkit_python-0.41.16.data/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json,sha256=YSpIrwpwB-lQBk9Mwv-npedWTOOZreO6nlWZhqlWyGI,840
130
- calkit_python-0.41.16.data/data/share/jupyter/labextensions/calkit/static/502.9a2c5772a15466e923ef.js,sha256=mixXcqFUZukj7_jQVxHTavsYl7cdZv83sEe4phJgkh0,59893
131
- calkit_python-0.41.16.data/data/share/jupyter/labextensions/calkit/static/695.2c41003a452d43d2b358.js,sha256=LEEAOkUtQ9KzWEHn-QFv5yGi70B-sBOnrvztzHr0Shw,223
132
- calkit_python-0.41.16.data/data/share/jupyter/labextensions/calkit/static/867.a42a046aa5108f54f8fb.js,sha256=pCoEaqUQj1T4-zAc9SUd__9ZGm7uL2wHQve2xwL89NI,8156
133
- calkit_python-0.41.16.data/data/share/jupyter/labextensions/calkit/static/909.e3f9cc3408834a7fdcc3.js,sha256=4_nMNAiDSn_cw7UjIHEwone4fizrvqrqSgbwUWvjmoU,114571
134
- calkit_python-0.41.16.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js,sha256=n_a0yu_gE7l6fauyoXXv9BZ7ZEYt2387mTfs7CbLcs4,51939
135
- calkit_python-0.41.16.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js.LICENSE.txt,sha256=eNJ8gc9n9IF8nW1d9sI9niuHstYzjNz5vqXx9UgWSPc,249
136
- calkit_python-0.41.16.data/data/share/jupyter/labextensions/calkit/static/b2f1c3efe70cb539d121.png,sha256=svHD7-cMtTnRITFwugwsVaB9nZ-h8A61ose8z32HiRE,24850
137
- calkit_python-0.41.16.data/data/share/jupyter/labextensions/calkit/static/remoteEntry.ac9035764d5b2adbb542.js,sha256=rJA1dk1bKtu1QkURrp5HEC2WCxzZppPV1SCYz-35GTc,8737
138
- calkit_python-0.41.16.data/data/share/jupyter/labextensions/calkit/static/style.js,sha256=r89Jlk5v1drcwhCpv9FnC3Jig_JH4k24kZNIvxh6Htk,149
139
- calkit_python-0.41.16.data/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json,sha256=2BGdItLJwO3fAopLl4eJvp26TEwuscYC0-yFaF-LaLU,13683
140
- calkit_python-0.41.16.data/data/share/jupyter/labextensions/calkit/install.json,sha256=DK9d8G-q-rMVlcT3rAeGIyo3REWKw6FySBZceLU9yaw,187
141
- calkit_python-0.41.16.dist-info/METADATA,sha256=tUnzT2-o6dU9IUfQO05ysFgxfISBJ4ws9pUnrNGZU9Q,12071
142
- calkit_python-0.41.16.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
143
- calkit_python-0.41.16.dist-info/entry_points.txt,sha256=2iQBBzjTAOdk66CwS-f3ecXXW5DjUqkG1KBRj8mHI1I,133
144
- calkit_python-0.41.16.dist-info/licenses/LICENSE,sha256=9ZamCaSUTZk9rcrnf-sWFKLOHr3ws-S_dgKMegW4nw8,1056
145
- calkit_python-0.41.16.dist-info/RECORD,,
126
+ calkit_python-0.41.18.data/data/etc/jupyter/jupyter_server_config.d/calkit.json,sha256=CWrZP--JDGz8fsvbAlKr_duTL1vDriGT48SL1YCtkY0,81
127
+ calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/package.json,sha256=39PQEWOzw0qtqD7Kep3OPbUR113vI7CG5TeD84v7clM,6224
128
+ calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/schemas/calkit/package.json.orig,sha256=NBG3t7gFYb6_2J-yPU29qS7Ck9pYgm8-wC5Q1LrzQeI,6082
129
+ calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/schemas/calkit/plugin.json,sha256=YSpIrwpwB-lQBk9Mwv-npedWTOOZreO6nlWZhqlWyGI,840
130
+ calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/502.9a2c5772a15466e923ef.js,sha256=mixXcqFUZukj7_jQVxHTavsYl7cdZv83sEe4phJgkh0,59893
131
+ calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/695.2c41003a452d43d2b358.js,sha256=LEEAOkUtQ9KzWEHn-QFv5yGi70B-sBOnrvztzHr0Shw,223
132
+ calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/867.a42a046aa5108f54f8fb.js,sha256=pCoEaqUQj1T4-zAc9SUd__9ZGm7uL2wHQve2xwL89NI,8156
133
+ calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/909.e3f9cc3408834a7fdcc3.js,sha256=4_nMNAiDSn_cw7UjIHEwone4fizrvqrqSgbwUWvjmoU,114571
134
+ calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js,sha256=n_a0yu_gE7l6fauyoXXv9BZ7ZEYt2387mTfs7CbLcs4,51939
135
+ calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/946.050af2abf7845cfbdbd2.js.LICENSE.txt,sha256=eNJ8gc9n9IF8nW1d9sI9niuHstYzjNz5vqXx9UgWSPc,249
136
+ calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/b2f1c3efe70cb539d121.png,sha256=svHD7-cMtTnRITFwugwsVaB9nZ-h8A61ose8z32HiRE,24850
137
+ calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/remoteEntry.ac9035764d5b2adbb542.js,sha256=rJA1dk1bKtu1QkURrp5HEC2WCxzZppPV1SCYz-35GTc,8737
138
+ calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/style.js,sha256=r89Jlk5v1drcwhCpv9FnC3Jig_JH4k24kZNIvxh6Htk,149
139
+ calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/static/third-party-licenses.json,sha256=2BGdItLJwO3fAopLl4eJvp26TEwuscYC0-yFaF-LaLU,13683
140
+ calkit_python-0.41.18.data/data/share/jupyter/labextensions/calkit/install.json,sha256=DK9d8G-q-rMVlcT3rAeGIyo3REWKw6FySBZceLU9yaw,187
141
+ calkit_python-0.41.18.dist-info/METADATA,sha256=zexxBw6ZYpD61wV5bcv8i25G0lGpGxrhJ0HKM3CAh1s,12437
142
+ calkit_python-0.41.18.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
143
+ calkit_python-0.41.18.dist-info/entry_points.txt,sha256=2iQBBzjTAOdk66CwS-f3ecXXW5DjUqkG1KBRj8mHI1I,133
144
+ calkit_python-0.41.18.dist-info/licenses/LICENSE,sha256=9ZamCaSUTZk9rcrnf-sWFKLOHr3ws-S_dgKMegW4nw8,1056
145
+ calkit_python-0.41.18.dist-info/RECORD,,