calkit-python 0.41.17__tar.gz → 0.41.18__tar.gz

{calkit_python-0.41.17 → calkit_python-0.41.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.41.17
+Version: 0.41.18
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.41.17 → calkit_python-0.41.18}/calkit/cli/scheduler.py

@@ -304,6 +304,25 @@ def _slurm_exit_code(job_id: str) -> int | None:
     return None
 
 
+def _parse_pbs_state(stdout: str) -> tuple[str | None, int | None]:
+    # `qstat -f` output is a flat list of `key = value` lines; pull out the
+    # job's state and, for a finished job, its exit status. The status field is
+    # spelled `Exit_status` on PBS Pro and `exit_status` on some Torque builds,
+    # so match it case-insensitively.
+    state = None
+    exit_code = None
+    for line in stdout.splitlines():
+        stripped = line.strip()
+        if stripped.startswith("job_state"):
+            state = stripped.split("=", 1)[-1].strip()
+        elif stripped.lower().startswith("exit_status"):
+            try:
+                exit_code = int(stripped.split("=", 1)[-1].strip())
+            except ValueError:
+                exit_code = None
+    return state, exit_code
+
+
 def _poll_job(kind: str, job_id: str) -> tuple[bool, int | None]:
     """Return ``(active, exit_code)`` for a scheduler job.
 
@@ -339,39 +358,51 @@ def _poll_job(kind: str, job_id: str) -> tuple[bool, int | None]:
         if "invalid job id" in stderr:
             return False, _slurm_exit_code(job_id)
         return True, None
-    # Use `qstat -f` and parse job_state: on Torque/OpenPBS, plain `qstat
-    # <id>` returns exit 0 even for completed (C) jobs, so checking the
-    # return code alone would cause `calkit sched batch` to hang forever
-    # after a PBS job finishes.  States C and F mean the job is done, and a
-    # finished job's record carries its `Exit_status`.
+    # PBS reports a finished job differently across variants, so poll in two
+    # steps that mirror the SLURM path above. First consult the active-queue
+    # view with `qstat -f`: every live state (R, Q, H, E) shows here, and on
+    # Torque a completed job lingers here in state `C` carrying its
+    # `Exit_status`. `qstat -f` is used rather than plain `qstat <id>` because
+    # the latter exits 0 even for completed jobs, so the return code alone
+    # could not tell running from done.
     p = subprocess.run(
         ["qstat", "-f", job_id], capture_output=True, text=True, check=False
     )
-    if p.returncode != 0:
-        # A non-zero exit is ambiguous: either the job is gone (completed and
-        # purged from history) or `qstat` itself failed transiently---a busy
-        # PBS server periodically refuses connections or times out. Treating a
-        # transient failure as completion would stop the wait while the job is
-        # still running and writing its log, so only conclude the job is done
-        # when qstat positively reports it is unknown; otherwise keep waiting.
-        # A purged job (unknown to qstat) is done but with an unknowable exit
-        # status; any other error is transient, so report the job as active.
-        stderr = (p.stderr or "").lower()
-        return ("unknown job" not in stderr), None
-    state = None
-    exit_code = None
-    for line in p.stdout.splitlines():
-        stripped = line.strip()
-        if stripped.startswith("job_state"):
-            state = stripped.split("=", 1)[-1].strip()
-        elif stripped.lower().startswith("exit_status"):
-            try:
-                exit_code = int(stripped.split("=", 1)[-1].strip())
-            except ValueError:
-                exit_code = None
-    if state in ("C", "F"):
-        return False, exit_code
-    return True, None
+    if p.returncode == 0:
+        state, exit_code = _parse_pbs_state(p.stdout)
+        if state in ("C", "F"):
+            return False, exit_code
+        if state is not None:
+            return True, None
+    # The job is no longer in the active queue---on PBS Pro a finished job is
+    # hidden from plain `qstat` and shows (as state `F` with its `Exit_status`)
+    # only under `qstat -x`---or qstat failed. Ask the history view for the
+    # finished record. Without this, a PBS Pro job that has finished is never
+    # seen as done and the wait hangs forever. `-x` means "include finished
+    # jobs" on PBS Pro; on Torque it switches qstat to XML output, which simply
+    # won't parse here, leaving the stderr check below to settle it (Torque
+    # already reported completion above via state `C`).
+    hist = subprocess.run(
+        ["qstat", "-x", "-f", job_id],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if hist.returncode == 0:
+        state, exit_code = _parse_pbs_state(hist.stdout)
+        if state in ("C", "F"):
+            return False, exit_code
+        if state is not None:
+            return True, None
+    # Neither view resolved a state. A non-zero exit is ambiguous: either the
+    # job is gone (finished and purged from history) or qstat failed
+    # transiently---a busy PBS server periodically refuses connections or times
+    # out. Treating a transient failure as completion would stop the wait while
+    # the job is still running and writing its log, so only conclude the job is
+    # done when qstat positively reports it is unknown; otherwise keep waiting.
+    # A purged job is done but with an unknowable exit status.
+    stderr = (p.stderr or "").lower()
+    return ("unknown job" not in stderr), None
 
 
 def _is_active(kind: str, job_id: str) -> bool:

{calkit_python-0.41.17 → calkit_python-0.41.18}/calkit/tests/cli/test_scheduler.py

@@ -215,44 +215,75 @@ def test_poll_job_pbs(monkeypatch):
 
     import calkit.cli.scheduler as sched
 
-    outcomes: dict = {}
+    # PBS is polled in two steps: plain `qstat -f` for the active queue, then
+    # `qstat -x -f` for the finished-job history view. The fake dispatches on
+    # whether `-x` is present so each step can be simulated independently, the
+    # way a real PBS Pro server answers them.
+    active: dict = {}
+    history: dict = {}
 
-    def _fake_run(*args, **kwargs):
+    def _fake_run(cmd, *args, **kwargs):
+        outcomes = history if "-x" in cmd else active
         return subprocess.CompletedProcess(
-            args=args[0],
+            cmd,
             returncode=outcomes["returncode"],
             stdout=outcomes.get("stdout", ""),
             stderr=outcomes.get("stderr", ""),
         )
 
     monkeypatch.setattr(sched.subprocess, "run", _fake_run)
-    # A running job (state R) is active with no exit code yet.
-    outcomes.update(returncode=0, stdout="    job_state = R\n", stderr="")
+    # A running job (state R) is active with no exit code yet, and resolves
+    # from the active queue alone---the history view is never consulted.
+    active.update(returncode=0, stdout="    job_state = R\n", stderr="")
+    history.clear()
     assert _poll_job("pbs", "1.pbs") == (True, None)
-    # Terminal states C (Torque) and F (PBS Pro) mean the job is done; qstat
-    # still exits 0 while the record lingers in history and carries the exit
-    # status, which is reported back so the caller can fail a bad job.
-    outcomes.update(
+    # The reported bug: on PBS Pro a finished job leaves the active queue and
+    # is shown only under `qstat -x`, where it appears as state F with its
+    # Exit_status. Plain `qstat -f` errors with a message that is NOT "unknown
+    # job", which used to be read as "still active" and hang the wait forever.
+    active.update(
+        returncode=1,
+        stdout="",
+        stderr="qstat: 1.pbs Job has finished, use -x to obtain historical "
+        "job information\n",
+    )
+    history.update(
         returncode=0, stdout="    job_state = F\n    Exit_status = 0\n"
     )
     assert _poll_job("pbs", "1.pbs") == (False, 0)
-    outcomes.update(
+    # A non-zero exit code from the finished job is reported back so the caller
+    # can fail the stage.
+    history.update(
+        returncode=0, stdout="    job_state = F\n    Exit_status = 137\n"
+    )
+    assert _poll_job("pbs", "1.pbs") == (False, 137)
+    # On Torque a completed job lingers in the active queue as state C with its
+    # exit status, so it is done without ever needing the history view.
+    active.update(
         returncode=0, stdout="    job_state = C\n    exit_status = 137\n"
     )
+    history.update(returncode=1, stdout="", stderr="should not be reached\n")
     assert _poll_job("pbs", "1.pbs") == (False, 137)
     # A finished job whose record lacks an exit status yields an unknown code.
-    outcomes.update(returncode=0, stdout="    job_state = F\n")
+    active.update(returncode=1, stdout="", stderr="qstat: Unknown Job Id\n")
+    history.update(returncode=0, stdout="    job_state = F\n")
     assert _poll_job("pbs", "1.pbs") == (False, None)
-    # Once purged, qstat exits non-zero and reports the job unknown: done, but
-    # with no way to recover the exit status.
-    outcomes.update(
+    # Once purged from history too, both views error with "unknown job": done,
+    # but with no way to recover the exit status.
+    active.update(
+        returncode=1, stdout="", stderr="qstat: Unknown Job Id 1.pbs\n"
+    )
+    history.update(
         returncode=1, stdout="", stderr="qstat: Unknown Job Id 1.pbs\n"
     )
     assert _poll_job("pbs", "1.pbs") == (False, None)
-    # A transient qstat failure (busy/unreachable server) is NOT completion:
-    # treating it as done would stop the wait while the job still runs, so the
-    # job is reported active and the caller keeps polling.
-    outcomes.update(
+    # A transient qstat failure (busy/unreachable server) on both views is NOT
+    # completion: treating it as done would stop the wait while the job still
+    # runs, so the job is reported active and the caller keeps polling.
+    active.update(
+        returncode=1, stdout="", stderr="qstat: cannot connect to server\n"
+    )
+    history.update(
         returncode=1, stdout="", stderr="qstat: cannot connect to server\n"
     )
     assert _poll_job("pbs", "1.pbs") == (True, None)

{calkit_python-0.41.17 → calkit_python-0.41.18}/vscode-ext/src/extension.ts

@@ -24,7 +24,12 @@ import {
   getConfiguredCandidateForNotebookPath as resolveConfiguredCandidateForNotebookPath,
   getExecutedNotebookHtmlPath,
 } from "./notebooks";
-import type { CalkitInfo, DvcYaml, EnvDescription } from "./types";
+import type {
+  CalkitInfo,
+  DvcYaml,
+  EnvDescription,
+  StaleStageDetail,
+} from "./types";
 import { dvcStageOutputPaths } from "./pipeline/core";
 import { CalkitSidebarProvider } from "./sidebar";
 import {
@@ -142,6 +147,10 @@ let hasCheckedCalkitCli = false;
 const pipelineOutputUris = new Set<string>();
 const staleOutputUris = new Set<string>();
 const staleStageNames = new Set<string>();
+// What is stale about each stale stage (modified inputs/outputs, script,
+// environment, command), keyed by base stage name. Surfaced as per-property
+// markers under a stale stage in the sidebar.
+let staleStageDetails: Record<string, StaleStageDetail> = {};
 // Stages currently being executed by an in-progress `calkit run` (reported by
 // `calkit status --json` as pipeline.running_stages); shown with a spinner.
 const runningStageNames = new Set<string>();
@@ -1228,6 +1237,29 @@ export function activate(context: vscode.ExtensionContext): void {
     }),
   );
 
+  // Re-check pipeline status when a workspace file is saved, so a stage goes
+  // stale in the sidebar right after editing its script/input/notebook without
+  // a manual refresh (issue #959). The dvc.yaml/calkit.yaml watchers already
+  // cover config edits. The scheduled refresh is debounced and only runs
+  // `calkit status` when auto-refresh is enabled.
+  const onDocumentSaved = (uri: vscode.Uri): void => {
+    const root = getWorkspaceRoot();
+    if (!root || uri.scheme !== "file") {
+      return;
+    }
+    const rel = path.relative(root, uri.fsPath);
+    if (rel.startsWith("..") || path.isAbsolute(rel)) {
+      return; // saved file is outside the workspace
+    }
+    scheduleRefreshPipelineOutputContext(context);
+  };
+  context.subscriptions.push(
+    vscode.workspace.onDidSaveTextDocument((doc) => onDocumentSaved(doc.uri)),
+    vscode.workspace.onDidSaveNotebookDocument((doc) =>
+      onDocumentSaved(doc.uri),
+    ),
+  );
+
   // Image/figure files open in a custom editor, so the active-text-editor event
   // above doesn't fire for them. Re-evaluate the figure-output context on tab
   // changes (a tab becoming active) and tab-group changes (the active split
@@ -1940,6 +1972,7 @@ function renderSidebarFromState(): void {
     currentDetectedResults,
     currentDetectedPresentations,
     getHiddenSections(),
+    staleStageDetails,
   );
   updateSidebarBadge();
 }
@@ -1949,6 +1982,7 @@ function renderSidebarFromState(): void {
 function applyStaleStages(
   workspaceRoot: string,
   freshStaleStageNames: Set<string>,
+  freshStaleStageDetails: Record<string, StaleStageDetail> = {},
 ): void {
   const outputMap = buildPipelineOutputMapFromYaml(
     workspaceRoot,
@@ -1970,6 +2004,8 @@ function applyStaleStages(
   for (const n of freshStaleStageNames) {
     staleStageNames.add(n);
   }
+  const prevDetails = staleStageDetails;
+  staleStageDetails = freshStaleStageDetails;
   const changedUris = [...new Set([...prevStale, ...staleOutputUris])].map(
     (p) => vscode.Uri.file(p),
   );
@@ -1978,7 +2014,10 @@ function applyStaleStages(
   }
   const staleChanged =
     prevStageNames.size !== staleStageNames.size ||
-    [...staleStageNames].some((n) => !prevStageNames.has(n));
+    [...staleStageNames].some((n) => !prevStageNames.has(n)) ||
+    // The set of stale stages can be unchanged while *what* is stale about them
+    // changes (e.g. a different input edited), so compare the details too.
+    JSON.stringify(prevDetails) !== JSON.stringify(staleStageDetails);
   if (staleChanged) {
     renderSidebarFromState();
   }
@@ -2010,6 +2049,55 @@ let runStatusPolling = false;
 let suppressRwlockUntil = 0;
 const RUN_STATUS_POLL_INTERVAL_MS = 2500;
 
+function unionPaths(
+  a: string[] | undefined,
+  b: string[] | undefined,
+): string[] {
+  return [...new Set([...(a ?? []), ...(b ?? [])])];
+}
+
+// Merge the per-stage staleness detail from `calkit status --json` down to base
+// stage names. iterate_over stages report one entry per expansion
+// ("benchmark@zdt1"); the sidebar keys stages by the base name, so union each
+// expansion's modified paths/flags under "benchmark".
+function collapseStaleStageDetails(
+  raw: Record<string, StaleStageDetail>,
+): Record<string, StaleStageDetail> {
+  const merged: Record<string, StaleStageDetail> = {};
+  for (const [key, detail] of Object.entries(raw)) {
+    const base = key.split("@")[0];
+    const existing = merged[base];
+    if (!existing) {
+      merged[base] = {
+        stale_outputs: [...(detail.stale_outputs ?? [])],
+        modified_inputs: [...(detail.modified_inputs ?? [])],
+        modified_outputs: [...(detail.modified_outputs ?? [])],
+        modified_command: !!detail.modified_command,
+        always_run: !!detail.always_run,
+        is_subproject: !!detail.is_subproject,
+      };
+      continue;
+    }
+    existing.stale_outputs = unionPaths(
+      existing.stale_outputs,
+      detail.stale_outputs,
+    );
+    existing.modified_inputs = unionPaths(
+      existing.modified_inputs,
+      detail.modified_inputs,
+    );
+    existing.modified_outputs = unionPaths(
+      existing.modified_outputs,
+      detail.modified_outputs,
+    );
+    existing.modified_command =
+      existing.modified_command || !!detail.modified_command;
+    existing.always_run = existing.always_run || !!detail.always_run;
+    existing.is_subproject = existing.is_subproject || !!detail.is_subproject;
+  }
+  return merged;
+}
+
 async function pollPipelineRunStatus(
   context: vscode.ExtensionContext,
 ): Promise<void> {
@@ -2032,6 +2120,7 @@ async function pollPipelineRunStatus(
         running?: boolean;
         running_stages?: string[];
         stale_stage_names?: string[];
+        stale_stages?: Record<string, StaleStageDetail>;
       };
     };
     const pipeline = status?.pipeline ?? {};
@@ -2057,6 +2146,7 @@ async function pollPipelineRunStatus(
         new Set(
           (pipeline.stale_stage_names ?? []).map((name) => name.split("@")[0]),
         ),
+        collapseStaleStageDetails(pipeline.stale_stages ?? {}),
       );
     }
   } catch (error) {

calkit_python-0.41.18/vscode-ext/src/pipeline/core.ts

@@ -0,0 +1,173 @@
+import type { DvcStage, StaleStageDetail } from "../types";
+
+// Pure helpers (no vscode imports) for resolving pipeline stage outputs, so they
+// can be unit-tested under plain `node --test` (where the `vscode` module is
+// absent). The crux is iterate_over (DVC "matrix") stages: calkit writes their
+// deps/outs templated with `${item.*}` placeholders plus a `matrix` of values,
+// so the literal templated path never exists on disk and the concrete
+// per-iteration files would otherwise go unrecognized as pipeline outputs.
+
+// Expand a DVC `matrix` into the list of concrete `${item.*}` substitution maps,
+// one per cartesian-product combination. Mirrors calkit's `_expand_matrix` in
+// calkit/pipeline.py, including flattening nested-dict values (produced by
+// list-of-lists iterations) into dotted "parent.child" keys.
+export function expandDvcMatrix(
+  matrix: Record<string, unknown[]>,
+): Record<string, string>[] {
+  let combos: Record<string, unknown>[] = [{}];
+  for (const [key, values] of Object.entries(matrix)) {
+    // calkit always writes matrix values as arrays, but a hand-edited dvc.yaml
+    // could hold a scalar (which would iterate characters) or a non-iterable
+    // (which would throw); skip anything that isn't an array so a malformed
+    // file can't break the sidebar render.
+    if (!Array.isArray(values)) {
+      continue;
+    }
+    const next: Record<string, unknown>[] = [];
+    for (const combo of combos) {
+      for (const value of values) {
+        next.push({ ...combo, [key]: value });
+      }
+    }
+    combos = next;
+  }
+  return combos.map((combo) => {
+    const flat: Record<string, string> = {};
+    for (const [key, value] of Object.entries(combo)) {
+      // A nested-dict value substitutes `${item.key.subkey}`; any other value
+      // is a scalar substituting `${item.key}`.
+      if (value !== null && typeof value === "object") {
+        for (const [subKey, subVal] of Object.entries(
+          value as Record<string, unknown>,
+        )) {
+          flat[`${key}.${subKey}`] = String(subVal);
+        }
+      } else {
+        flat[key] = String(value);
+      }
+    }
+    return flat;
+  });
+}
+
+// Substitute one matrix combination's `${item.*}` placeholders into a template.
+function applyMatrixItem(
+  template: string,
+  item: Record<string, string>,
+): string {
+  let result = template;
+  for (const [variable, value] of Object.entries(item)) {
+    result = result.split(`\${item.${variable}}`).join(value);
+  }
+  return result;
+}
+
+// The concrete output paths declared by a DVC stage. A plain stage returns its
+// `outs` as written; an iterate_over (matrix) stage has its templated `outs`
+// expanded into one concrete path per iteration so the real files---rather than
+// a single non-existent templated path---are recognized as pipeline outputs.
+export function dvcStageOutputPaths(stage: DvcStage): string[] {
+  const rawPaths = (stage.outs ?? []).flatMap((out) =>
+    typeof out === "string" ? [out] : Object.keys(out),
+  );
+  const matrix = stage.matrix as Record<string, unknown[]> | undefined;
+  if (!matrix || Object.keys(matrix).length === 0) {
+    return rawPaths;
+  }
+  const items = expandDvcMatrix(matrix);
+  return rawPaths.flatMap((p) =>
+    p.includes("${item.") ? items.map((item) => applyMatrixItem(p, item)) : [p],
+  );
+}
+
+// The pieces of a stage's configuration needed to attribute its DVC-reported
+// changes to the meaningful categories shown in the sidebar (script, notebook,
+// source, environment, declared inputs). Paths are repo-relative, matching how
+// `calkit status` reports modified deps/outs.
+export interface StageStaleContext {
+  scriptPath?: string;
+  notebookPath?: string;
+  targetPath?: string;
+  configuredInputs?: string[];
+  // The environment's spec/lock files (e.g. pyproject.toml + uv.lock). When any
+  // of these is a modified dep, the stage is stale because its environment
+  // changed.
+  envFilePaths?: string[];
+}
+
+// How a stale stage's changes map onto the categories surfaced in the tree, so
+// the sidebar can flag exactly which inputs/outputs/script/environment are out
+// of date rather than just marking the whole stage stale.
+export interface StaleClassification {
+  modifiedInputs: Set<string>;
+  modifiedOutputs: Set<string>;
+  staleOutputs: Set<string>;
+  scriptStale: boolean;
+  notebookStale: boolean;
+  sourceStale: boolean;
+  envStale: boolean;
+  commandModified: boolean;
+  // Modified deps not attributable to any structured row above (e.g. an
+  // auto-added module dependency). Internal `.calkit/` bookkeeping paths are
+  // excluded since they mirror the script/notebook rows.
+  extraModifiedInputs: string[];
+}
+
+// calkit tracks a notebook stage's *cleaned* copy under `.calkit/` as the DVC
+// dep, so a source-notebook edit surfaces as a change to this path rather than
+// the notebook's own path.
+function cleanedNotebookPath(notebookPath: string): string {
+  return `.calkit/notebooks/cleaned/${notebookPath}`;
+}
+
+// Attribute a stale stage's modified deps/outs to the script, notebook, source,
+// environment, and declared inputs/outputs so the sidebar can show what is
+// stale about it. Pure (no vscode imports) so it can be unit-tested.
+export function classifyStaleStage(
+  detail: StaleStageDetail,
+  context: StageStaleContext,
+): StaleClassification {
+  const modifiedInputs = new Set(detail.modified_inputs ?? []);
+  const modifiedOutputs = new Set(detail.modified_outputs ?? []);
+  const staleOutputs = new Set(detail.stale_outputs ?? []);
+  const scriptStale =
+    !!context.scriptPath && modifiedInputs.has(context.scriptPath);
+  const notebookStale =
+    !!context.notebookPath &&
+    (modifiedInputs.has(context.notebookPath) ||
+      modifiedInputs.has(cleanedNotebookPath(context.notebookPath)));
+  const sourceStale =
+    !!context.targetPath && modifiedInputs.has(context.targetPath);
+  const envFilePaths = context.envFilePaths ?? [];
+  const envStale = envFilePaths.some((p) => modifiedInputs.has(p));
+  // Deps already represented by a structured row, so they aren't repeated as
+  // generic "extra" inputs.
+  const covered = new Set<string>(context.configuredInputs ?? []);
+  if (context.scriptPath) {
+    covered.add(context.scriptPath);
+  }
+  if (context.notebookPath) {
+    covered.add(context.notebookPath);
+    covered.add(cleanedNotebookPath(context.notebookPath));
+  }
+  if (context.targetPath) {
+    covered.add(context.targetPath);
+  }
+  for (const p of envFilePaths) {
+    covered.add(p);
+  }
+  const extraModifiedInputs = [...modifiedInputs].filter(
+    (p) => !covered.has(p) && !p.startsWith(".calkit/"),
+  );
+  return {
+    modifiedInputs,
+    modifiedOutputs,
+    staleOutputs,
+    scriptStale,
+    notebookStale,
+    sourceStale,
+    envStale,
+    commandModified: !!detail.modified_command,
+    extraModifiedInputs,
+  };
+}