calkit-python 0.38.1__tar.gz → 0.38.2__tar.gz

{calkit_python-0.38.1 → calkit_python-0.38.2}/AGENTS.md

@@ -16,3 +16,5 @@ To sync the docs and format all the code, run `make format`.
 
 Wrap prose at natural breakpoints in phrases or punctuation to keep max
 line length below 80 characters.
+
+Agents should never make commits to Git.

{calkit_python-0.38.1 → calkit_python-0.38.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.38.1
+Version: 0.38.2
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.38.1 → calkit_python-0.38.2}/calkit/dvc/zip.py

@@ -10,6 +10,7 @@ efficient.
 import json
 import os
 import shutil
+import stat
 import subprocess
 import zipfile
 from pathlib import Path
@@ -376,9 +377,25 @@ def unzip(workspace_path: str, zip_path: str):
                 "System unzip failed; falling back to Python zipfile.",
                 err=True,
             )
+    workspace_abs = os.path.realpath(workspace_path)
     with ZipFile(zip_path, "r") as zip_file:
         members = zip_file.infolist()
         for member in tqdm(members, desc="Unzipping", unit="file"):
+            target = os.path.normpath(
+                os.path.join(workspace_abs, member.filename)
+            )
+            if not os.path.realpath(target).startswith(workspace_abs + os.sep):
+                raise ValueError(
+                    f"Zip entry {member.filename!r} would extract outside "
+                    "the workspace directory"
+                )
+            if os.path.isfile(target) and not os.access(target, os.W_OK):
+                # File may be read-only (e.g. created by Docker as root or by
+                # DVC). Try chmod first; if that fails, remove and re-extract.
+                try:
+                    os.chmod(target, stat.S_IWRITE | stat.S_IREAD)
+                except OSError:
+                    os.remove(target)
             zip_file.extract(member, workspace_path)
 
 

{calkit_python-0.38.1 → calkit_python-0.38.2}/calkit/releases.py

@@ -10,11 +10,13 @@ import subprocess
 import sys
 import tempfile
 import zipfile
+from pathlib import Path
 from typing import Literal
 
 import git
 
 import calkit
+import calkit.dvc.zip
 
 SERVICES = {
     "caltechdata": {"name": "CaltechDATA", "url": "https://data.caltech.edu"},
@@ -165,7 +167,42 @@ def ls_files() -> list[str]:
                 fpath = os.path.join(root, filename)
                 if os.path.isfile(fpath):
                     cache_files.append(fpath)
-    return list(dict.fromkeys(git_files + dvc_files + cache_files))
+    # Include files from unzipped dvc-zip workspace folders, which are
+    # ignored by both Git and DVC and would otherwise be missing from the
+    # release archive
+    dvc_zip_files: list[str] = []
+    repo_root = Path(repo.working_dir).resolve()
+    zip_path_map = calkit.dvc.zip.get_zip_path_map()
+    for workspace_path in zip_path_map:
+        abs_workspace = (repo_root / workspace_path).resolve()
+        if not abs_workspace.is_relative_to(repo_root):
+            raise ValueError(
+                f"dvc-zip workspace path {workspace_path!r} is not within "
+                "the repository root; this may indicate a bug or tampering "
+                "with .calkit/zip/paths.json"
+            )
+        if abs_workspace.is_dir():
+            for root, _, files in os.walk(abs_workspace):
+                for filename in files:
+                    abs_fpath = Path(root) / filename
+                    if abs_fpath.is_file():
+                        dvc_zip_files.append(
+                            str(abs_fpath.relative_to(repo_root))
+                        )
+        elif abs_workspace.exists():
+            raise ValueError(
+                f"dvc-zip workspace path {workspace_path!r} exists but is "
+                "not a directory; this may indicate a bug or tampering with "
+                ".calkit/zip/paths.json"
+            )
+    zip_files = {Path(p).as_posix() for p in zip_path_map.values()}
+    return [
+        f
+        for f in dict.fromkeys(
+            git_files + dvc_files + cache_files + dvc_zip_files
+        )
+        if Path(f).as_posix() not in zip_files
+    ]
 
 
 def make_dvc_md5s(
@@ -205,7 +242,7 @@ def populate_dvc_cache():
     """Populate DVC cache from releases."""
     ck_info = calkit.load_calkit_info()
     releases = ck_info.get("releases", {})
-    for name, release in releases.items():
+    for name, _ in releases.items():
         md5s_fpath = f".calkit/releases/{name}/dvc-md5s.yaml"
         with open(md5s_fpath) as f:
             md5s = calkit.ryaml.load(f)

{calkit_python-0.38.1 → calkit_python-0.38.2}/calkit/tests/test_releases.py

@@ -9,6 +9,7 @@ import bibtexparser
 import git
 import pytest
 
+from calkit.dvc.zip import write_zip_path_map
 from calkit.releases import (
     check_project_release_archive,
     create_bibtex,
@@ -66,6 +67,29 @@ def test_ls_files(tmp_dir):
     assert ".dvc/cache/aa/legacy" in files
     assert ".dvc/cache/files/md5/bb/modern" in files
     assert ".dvc/cache/runs/cc/hash/run" in files
+    # Ensure files from unzipped dvc-zip workspace folders are included;
+    # these folders are ignored by both Git and DVC
+    (tmp_dir / "my-zip-workspace" / "sub").mkdir(parents=True, exist_ok=True)
+    (tmp_dir / "my-zip-workspace" / "data.txt").write_text("data")
+    (tmp_dir / "my-zip-workspace" / "sub" / "nested.txt").write_text("nested")
+    # Create the zip file and DVC-track it so the exclusion filter is exercised
+    zip_dir = tmp_dir / ".calkit" / "zip" / "files"
+    zip_dir.mkdir(parents=True, exist_ok=True)
+    zip_fpath = zip_dir / "my-zip-workspace.zip"
+    with zipfile.ZipFile(zip_fpath, "w") as zf:
+        zf.writestr("data.txt", "data")
+        zf.writestr("sub/nested.txt", "nested")
+    subprocess.run(
+        ["calkit", "dvc", "add", str(zip_fpath.relative_to(tmp_dir))],
+        check=True,
+    )
+    write_zip_path_map(
+        {"my-zip-workspace": ".calkit/zip/files/my-zip-workspace.zip"}
+    )
+    files = ls_files()
+    assert "my-zip-workspace/data.txt" in files
+    assert "my-zip-workspace/sub/nested.txt" in files
+    assert ".calkit/zip/files/my-zip-workspace.zip" not in files
 
 
 def test_check_project_release_archive_passes_when_pipeline_is_current(

{calkit_python-0.38.1 → calkit_python-0.38.2}/docs/governance.md

@@ -16,7 +16,9 @@ and our code of conduct can be found
 
 ## Vision
 
-We envision a future where nearly every research article is delivered as part
+We envision a future where reliable and impactful knowledge is created
+10x faster than it is today.
+In this future nearly every research article is delivered as part
 of a [single-button](https://petebachant.me/single-button/)
 reproducible
 [compendium](https://book.the-turing-way.org/reproducible-research/compendia/)
@@ -46,6 +48,10 @@ by enabling faster:
    making it easier to find something similar to a desired workflow to start
    from and adapt to new questions.
 
+Most importantly,
+full automation speeds up iteration cycle time,
+and more iterations means greater likelihood of success.
+
 ### The status quo
 
 The figure below from the PLOS Open Science Indicators dataset
@@ -63,10 +69,15 @@ Code sharing rates from
 
 ## Strategy
 
-We believe one major hurdle preventing researchers from working reproducibly is
-the expectation that they become software engineering experts,
-choose and integrate multiple tools,
-and assemble a custom workflow.
+Telling researchers they need to work reproducibly is like telling them
+they need to eat their vegetables.
+In other words, reproducibility needs to be a side effect of a process
+that is more productive and more enjoyable than the status quo.
+
+The current "gold standard"
+requires researchers to become de facto software engineers,
+choosing and integrating multiple tools,
+designing bespoke project layouts and workflows.
 We want to provide a vertically-integrated, purpose-built,
 and user-friendly project format and toolset that reduces the required
 expertise and decision fatigue.
@@ -75,7 +86,7 @@ solution—they certainly are—but we are focused on improving
 tooling and infrastructure.
 
 - **Path of least resistance:**
-  Make it faster to work in a clean,
+  Make it faster to work in a clean, automated,
   reproducible way than it is to work in an ad-hoc, disorganized way.
 - **Intuitive tooling:**
   Simplify the "hard parts" of modern scientific computing:
@@ -88,6 +99,9 @@ tooling and infrastructure.
 - **Builder's pride:**
   Enable researchers to take pride in what they create so
   they will be more likely to share their projects openly.
+- **Continuous delivery:** Continuous delivery (CD) practices have greatly
+  enhanced the quality and speed at which software products are created.
+  These same principles can be applied to research.
 
 ## Objective and key results (OKRs)
 

{calkit_python-0.38.1 → calkit_python-0.38.2}/vscode-ext/src/environments.ts

@@ -223,3 +223,14 @@ export function findCalkitEnvKernelSourceCandidate(
     (candidate) => candidate.environmentName === environmentName,
   );
 }
+
+// Environment kinds that can have their Jupyter kernel registered and selected
+// natively in VS Code — no Jupyter server launch required.
+export const kernelRegistrationKinds: ReadonlySet<EnvKind> = new Set<EnvKind>([
+  "uv",
+  "uv-venv",
+  "venv",
+  "conda",
+  "pixi",
+  "julia",
+]);

{calkit_python-0.38.1 → calkit_python-0.38.2}/vscode-ext/src/extension.ts

@@ -10,6 +10,7 @@ import {
   compactSlurmOptions,
   findCalkitEnvKernelSourceCandidate,
   getDefaultSlurmOptions,
+  kernelRegistrationKinds,
   makeCalkitEnvKernelSourceCandidates,
   slurmOptionsToOptionList,
   type CalkitEnvNotebookKernelSource,
@@ -440,6 +441,7 @@ async function selectCalkitEnvironment(
       picked.innerEnvironment,
       "-e",
       targetNotebookUri,
+      picked.innerKind,
     );
     await refreshNotebookToolbarContext(context);
     return kernelId;
@@ -472,7 +474,7 @@ async function selectCalkitEnvironment(
     slurmAutoStartSuppressedThisSession.delete(targetNotebookUri);
   }
 
-  // For uv/julia in a non-nested setup, only register/check the kernel and
+  // For local/non-nested environments, only register/check the kernel and
   // then select it in VS Code. No server launch is needed here.
   if (
     !picked.outerSlurmEnvironment &&
@@ -483,6 +485,7 @@ async function selectCalkitEnvironment(
       picked.innerEnvironment,
       "-e",
       targetNotebookUri,
+      picked.innerKind,
     );
     await refreshNotebookToolbarContext(context);
     return kernelId;
@@ -1870,6 +1873,7 @@ async function autoSelectEnvironmentForActiveNotebook(
       configuredCandidate.innerEnvironment,
       "-e",
       notebookUri,
+      configuredCandidate.innerKind,
     );
     if (kernelId) {
       log(
@@ -2225,7 +2229,7 @@ function buildDockerRunCommand(
 }
 
 function needsKernelRegistration(kind: EnvKind): boolean {
-  return kind === "uv" || kind === "julia";
+  return kernelRegistrationKinds.has(kind);
 }
 
 function createServerToken(): string {
@@ -2389,8 +2393,9 @@ async function registerAndSelectKernel(
   envName: string,
   envFlag: "-n" | "-e",
   expectedNotebookUri?: string,
+  kind?: EnvKind,
 ): Promise<string | undefined> {
-  log(`Registering and selecting kernel for env: ${envName}`);
+  log(`Registering and selecting kernel for env: ${envName} (kind=${kind})`);
 
   return await withKernelProgress(
     "Checking environment and setting kernel...",

calkit_python-0.38.2/vscode-ext/src/test/kernel-selection.test.ts

@@ -0,0 +1,88 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import {
+  kernelRegistrationKinds,
+  makeCalkitEnvKernelSourceCandidates,
+  type EnvKind,
+} from "../environments";
+
+const REGISTRATION_KINDS: EnvKind[] = [
+  "uv",
+  "uv-venv",
+  "venv",
+  "conda",
+  "pixi",
+  "julia",
+];
+
+const SERVER_KINDS: EnvKind[] = ["docker", "slurm", "ssh", "_system"];
+
+test("kernelRegistrationKinds includes all local env types", () => {
+  for (const kind of REGISTRATION_KINDS) {
+    assert.ok(
+      kernelRegistrationKinds.has(kind),
+      `expected '${kind}' to be in kernelRegistrationKinds`,
+    );
+  }
+});
+
+test("kernelRegistrationKinds excludes server-launch env types", () => {
+  for (const kind of SERVER_KINDS) {
+    assert.ok(
+      !kernelRegistrationKinds.has(kind),
+      `expected '${kind}' to NOT be in kernelRegistrationKinds`,
+    );
+  }
+});
+
+test("non-nested local env candidate has no outerSlurmEnvironment", () => {
+  // A standalone uv-venv env should produce a candidate with no outer
+  // SLURM environment, which is the signal that no server launch is needed.
+  const candidates = makeCalkitEnvKernelSourceCandidates({
+    main: { kind: "uv-venv", path: "pyproject.toml" },
+  });
+  assert.equal(candidates.length, 1);
+  assert.equal(candidates[0].innerKind, "uv-venv");
+  assert.equal(candidates[0].outerSlurmEnvironment, undefined);
+  assert.ok(kernelRegistrationKinds.has(candidates[0].innerKind));
+});
+
+test("nested slurm+uv-venv candidate has outerSlurmEnvironment set", () => {
+  // A SLURM-wrapped uv-venv env must still launch a server — the outer
+  // SLURM environment is what triggers that path. The guard in extension.ts
+  // is: !outerSlurmEnvironment && needsKernelRegistration(innerKind)
+  const candidates = makeCalkitEnvKernelSourceCandidates({
+    cluster: { kind: "slurm", host: "cluster.example.edu" },
+    main: { kind: "uv-venv", path: "pyproject.toml" },
+  });
+  const nested = candidates.find((c) => c.outerSlurmEnvironment === "cluster");
+  assert.ok(nested, "expected a nested slurm:uv-venv candidate");
+  assert.equal(nested.innerKind, "uv-venv");
+  assert.equal(nested.outerSlurmEnvironment, "cluster");
+  assert.ok(kernelRegistrationKinds.has(nested.innerKind));
+  assert.ok(nested.outerSlurmEnvironment !== undefined);
+});
+
+test("nested slurm+docker candidate has outerSlurmEnvironment set and docker does not use kernel registration", () => {
+  const candidates = makeCalkitEnvKernelSourceCandidates({
+    cluster: { kind: "slurm", host: "cluster.example.edu" },
+    myContainer: { kind: "docker", image: "jupyter/minimal-notebook:latest" },
+  });
+  const nested = candidates.find((c) => c.outerSlurmEnvironment === "cluster");
+  assert.ok(nested);
+  assert.equal(nested.innerKind, "docker");
+  assert.ok(!kernelRegistrationKinds.has(nested.innerKind));
+});
+
+test("all standalone registration-kind envs produce candidates with no outer env", () => {
+  const envs: Record<string, { kind: EnvKind }> = {};
+  for (const kind of REGISTRATION_KINDS) {
+    envs[kind] = { kind };
+  }
+  const candidates = makeCalkitEnvKernelSourceCandidates(envs);
+  for (const kind of REGISTRATION_KINDS) {
+    const c = candidates.find((x) => x.environmentName === kind);
+    assert.ok(c, `expected standalone candidate for kind '${kind}'`);
+    assert.equal(c.outerSlurmEnvironment, undefined);
+  }
+});