databricks-agent-notebooks 0.2.2__tar.gz → 0.3.0__tar.gz

{databricks_agent_notebooks-0.2.2 → databricks_agent_notebooks-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: databricks-agent-notebooks
-Version: 0.2.2
+Version: 0.3.0
 Summary: Standalone notebook conversion and local runtime tooling for the databricks-agent-notebooks repository.
 Author: Swoop
 License-Expression: MIT

{databricks_agent_notebooks-0.2.2 → databricks_agent_notebooks-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "databricks-agent-notebooks"
-version = "0.2.2"
+version = "0.3.0"
 description = "Standalone notebook conversion and local runtime tooling for the databricks-agent-notebooks repository."
 readme = "README.md"
 requires-python = ">=3.11"
@@ -52,6 +52,12 @@ agent-notebook = "databricks_agent_notebooks.cli:main"
 testpaths = ["tests"]
 addopts = "-q"
 pythonpath = ["src"]
+markers = [
+    "integration: Fast integration tests (CLI smoke, no side effects)",
+    "slow: Slow tests (kernel downloads, package installs, real execution)",
+    "spark: Tests requiring local PySpark installation",
+    "databricks: Tests requiring live Databricks workspace credentials",
+]
 
 [tool.setuptools]
 package-dir = { "" = "src" }

databricks_agent_notebooks-0.3.0/src/databricks_agent_notebooks/__init__.py

@@ -0,0 +1,10 @@
+"""Standalone Databricks notebook conversion and execution tooling."""
+
+from importlib.metadata import version, PackageNotFoundError
+
+try:
+    __version__ = version("databricks-agent-notebooks")
+except PackageNotFoundError:
+    __version__ = "0.0.0-dev"
+
+__all__ = ["__version__"]

{databricks_agent_notebooks-0.2.2 → databricks_agent_notebooks-0.3.0}/src/databricks_agent_notebooks/cli.py

@@ -15,6 +15,7 @@ import tempfile
 from importlib import resources
 from pathlib import Path
 
+from databricks_agent_notebooks import __version__
 from databricks_agent_notebooks.config.frontmatter import DatabricksConfig, merge_config
 from databricks_agent_notebooks.execution.executor import RawProgressValue, emit_progress_signal, execute_notebook
 from databricks_agent_notebooks.execution.injection import inject_cells
@@ -75,6 +76,10 @@ def _build_parser() -> argparse.ArgumentParser:
         epilog=_agent_docs_epilog(),
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
+    parser.add_argument(
+        "--version", action="version",
+        version=f"%(prog)s {__version__}",
+    )
     subparsers = parser.add_subparsers(dest="command")
 
     # -- run --
@@ -565,18 +570,6 @@ def _print_doctor_checks(checks: list[Check]) -> list[Check]:
     return [check for check in checks if check.status == "fail"]
 
 
-def _cmd_kernels_doctor(args: argparse.Namespace) -> int:
-    """Run environment validation checks."""
-    checks = _run_kernels_doctor_checks(args)
-    failures = _print_doctor_checks(checks)
-    if failures:
-        print(f"\n{len(failures)} check(s) failed.", file=sys.stderr)
-        return 1
-
-    print("\nAll checks passed.")
-    return 0
-
-
 def _cmd_kernels(args: argparse.Namespace) -> int:
     handler = _KERNEL_HANDLERS.get(args.kernels_command)
     if handler is None:
@@ -601,18 +594,6 @@ def _cmd_runtimes_list(_args: argparse.Namespace) -> int:
     return 0
 
 
-def _cmd_runtimes_doctor(_args: argparse.Namespace) -> int:
-    """Validate managed runtime receipts rooted under runtime-home."""
-    checks = _run_runtimes_doctor_checks()
-    failures = _print_doctor_checks(checks)
-    if failures:
-        print(f"\n{len(failures)} check(s) failed.", file=sys.stderr)
-        return 1
-
-    print("\nAll checks passed.")
-    return 0
-
-
 def _cmd_runtimes(args: argparse.Namespace) -> int:
     handler = _RUNTIME_HANDLERS.get(args.runtimes_command)
     if handler is None:
@@ -640,12 +621,10 @@ _KERNEL_HANDLERS = {
     "install": _cmd_kernels_install,
     "list": _cmd_kernels_list,
     "remove": _cmd_kernels_remove,
-    "doctor": _cmd_kernels_doctor,
 }
 
 _RUNTIME_HANDLERS = {
     "list": _cmd_runtimes_list,
-    "doctor": _cmd_runtimes_doctor,
 }
 
 

{databricks_agent_notebooks-0.2.2 → databricks_agent_notebooks-0.3.0}/src/databricks_agent_notebooks/execution/executor.py

@@ -33,8 +33,7 @@ except ModuleNotFoundError:  # pragma: no cover - exercised via packaging verifi
     install_ipykernel = None
 
 
-HEARTBEAT_INTERVAL_SECONDS = 30.0
-REDACTED_CELL_SNIPPET = "[source redacted]"
+HEARTBEAT_INTERVAL_SECONDS = 60.0
 
 
 class RawProgressValue(str):
@@ -133,18 +132,11 @@ def _build_cell_label(cell: nbformat.NotebookNode, lines: list[str]) -> str:
     return f"[{cell_type} cell]"
 
 
-def _build_cell_snippet(_cell: nbformat.NotebookNode, lines: list[str]) -> str:
-    if not lines:
-        return "[empty cell]"
-    return REDACTED_CELL_SNIPPET
-
-
 def _describe_cell(cell: nbformat.NotebookNode, *, cell_index: int) -> dict[str, object]:
     lines = _iter_meaningful_lines(cell.source)
     return {
         "cell_index": cell_index + 1,
         "cell_label": _build_cell_label(cell, lines),
-        "cell_snippet": _build_cell_snippet(cell, lines),
     }
 
 
@@ -366,7 +358,7 @@ def _execute_notebook_subprocess(
         result_path = Path(result_file.name)
 
     try:
-        completed = subprocess.run(
+        process = subprocess.Popen(
             _subprocess_command(
                 python_executable=python_executable,
                 notebook_path=notebook_path,
@@ -376,14 +368,30 @@ def _execute_notebook_subprocess(
                 timeout=timeout,
                 allow_errors=allow_errors,
             ),
-            capture_output=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
             text=True,
-            check=False,
         )
-        if completed.stdout:
-            print(completed.stdout, end="", file=sys.stdout)
-        if completed.stderr:
-            print(completed.stderr, end="", file=sys.stderr)
+
+        stderr_lines: list[str] = []
+
+        def _stream_stderr() -> None:
+            assert process.stderr is not None
+            for line in process.stderr:
+                print(line, end="", file=sys.stderr, flush=True)
+                stderr_lines.append(line)
+
+        stderr_thread = threading.Thread(target=_stream_stderr, daemon=True)
+        stderr_thread.start()
+
+        stdout_text = process.stdout.read() if process.stdout else ""
+        process.wait()
+        stderr_thread.join()
+
+        if stdout_text:
+            print(stdout_text, end="", file=sys.stdout)
+
+        stderr_text = "".join(stderr_lines).strip()
 
         if result_path.is_file():
             payload = json.loads(result_path.read_text(encoding="utf-8"))
@@ -399,7 +407,7 @@ def _execute_notebook_subprocess(
             success=False,
             output_path=output_path,
             duration_seconds=time.monotonic() - start,
-            error=completed.stderr.strip() or f"Managed runtime execution failed with exit code {completed.returncode}",
+            error=stderr_text or f"Managed runtime execution failed with exit code {process.returncode}",
         )
     finally:
         result_path.unlink(missing_ok=True)
@@ -417,7 +425,7 @@ def execute_notebook(
     if output_path is None:
         output_path = notebook_path.with_suffix(".executed.ipynb")
 
-    if python_executable is not None and python_executable.resolve() != Path(sys.executable).resolve():
+    if python_executable is not None and str(python_executable.resolve().parent.parent) != sys.prefix:
         return _execute_notebook_subprocess(
             notebook_path,
             output_path=output_path,

{databricks_agent_notebooks-0.2.2 → databricks_agent_notebooks-0.3.0}/src/databricks_agent_notebooks/for_agents/README.md

@@ -20,9 +20,8 @@ Usually safe in a sandbox:
 
 - `agent-notebook help`
 - `agent-notebook render ...`
-- `agent-notebook kernels doctor ...`
+- `agent-notebook doctor`
 - `agent-notebook runtimes list`
-- `agent-notebook runtimes doctor`
 
 Usually safe only when outbound artifact downloads are allowed:
 
@@ -63,9 +62,17 @@ agent-notebook run notebook.md --profile prod --cluster 0123-456789-abcdef
 agent-notebook run notebook.md --profile prod --cluster my-cluster
 ```
 
-If the `run` command cannot resolve a cluster name, its error message will 
+If the `run` command cannot resolve a cluster name, its error message will
 include helpful fuzzy matches for clusters you can present to the user.
 
+On large workspaces with many clusters, the 30-second cluster-listing timeout
+may expire before an exact match is found. When this happens, `agent-notebook`
+returns fuzzy name suggestions based on partial results received before the
+timeout. Inspect these suggestions — the full cluster name (e.g.,
+`"rnd-alpha [engineering]"` instead of `"rnd-alpha"`) usually resolves the
+issue. Passing a cluster ID is the deterministic path for cluster-based
+execution and is never subject to timeouts.
+
 The only time `agent-notebook clusters` is appropriate is during the initial
 `agent_doctor.md` readiness flow to verify live Databricks connectivity. Once
 you know the cluster name, pass it to `--cluster` and move on.
@@ -81,6 +88,8 @@ Recommended agent policy:
 - run only the Databricks-facing command outside the sandbox when needed
 - if sandboxed notebook startup fails around IPython or Jupyter runtime state,
   check whether the sandbox needs write access to `~/.ipython`
+- for Scala notebooks, the Almond kernel requires write access to
+  `~/Library/Caches/Almond/` on macOS
 
 ## Progress model
 
@@ -98,43 +107,126 @@ Current progress model:
 
 ## Timeouts
 
-Only set `--timeout` if you have a high-confidence real upper bound for the cell or
-notebook you are running. Otherwise, you risk losing valuable work due to factors you cannot control: cluster startup/resizing, node availability/termination, task/stage failure and re-execution, resource contention from other jobs.
+### Notebook cell timeout (`--timeout`)
+
+Only set `--timeout` if you have a high-confidence real upper bound for the cell
+or notebook you are running. Otherwise, you risk losing valuable work due to
+factors you cannot control: cluster startup/resizing, node availability, task
+failure and re-execution, resource contention from other jobs.
+
+`--timeout` is per-cell, not per-notebook.
+
+### Agent environment timeout
 
-If you do set timeout `--timeout`, note that it is cell-based and ensure you use 
-an appropriately long Bash tool timeout.
+Your agent environment's shell tool has its own timeout — the maximum duration a
+single foreground command can run before being killed. In Claude Code, this is
+600 seconds (10 minutes). If a foreground `agent-notebook run` exceeds this
+ceiling, the process is killed and the work is lost.
+
+This is why long-running execution patterns (below) default to non-blocking: they
+decouple the notebook run from the shell tool's timeout.
 
 ## Long-running runs
 
-If your shell or agent environment may kill long commands, prefer a detached
-`nohup` launch for `agent-notebook run ...` and monitor the log plus rendered
-output artifact instead of waiting interactively.
+Unless you have a concrete reason to expect a short run (e.g., a familiar
+serverless smoke notebook you have run before), treat every notebook run as
+potentially long. Cluster startup, serverless warmup, queueing, autoscaling,
+dependency setup, and shared compute contention can all stretch runtimes
+unpredictably.
+
+### Execution helper script
+
+A parameterizable wrapper script ships with this package at
+`for_agents/scripts/agent-nb-run.sh`. It handles path computation (log path,
+rendered output path, stem), output directory creation, early validation, and
+tee-to-log — so you do not need to reconstruct these details from examples.
+
+Use it as the command in any of the patterns below. It forwards all arguments to
+`agent-notebook run` unchanged.
+
+```bash
+agent-nb-run.sh <notebook> --profile <profile> [--output-dir <dir>] [--cluster <name>] [--format md] [...]
+```
+
+### Quick reference: which pattern to use
+
+| Environment | Default pattern | Use foreground only when |
+|-------------|----------------|------------------------|
+| Claude Code | `run_in_background` | You know the run will finish in under 5 minutes |
+| Codex | PTY session | N/A — PTY has no fixed timeout |
+| Standard shell | `nohup` detached | You will wait interactively |
+
+### Claude Code
+
+Two non-blocking patterns, in preference order.
+
+**Pattern 1: `run_in_background` (preferred)**
+
+Non-blocking — the session stays interactive and Claude Code notifies on
+completion. No fixed timeout ceiling; the run continues until it finishes.
+
+Use the Bash tool with `run_in_background: true`:
+
+```bash
+# Use with Bash tool parameter: run_in_background: true
+agent-nb-run.sh path/to/notebook.md \
+  --profile <profile> \
+  --format md \
+  --output-dir tmp/run-output
+```
+
+The script emits the log path and output directory to stderr at startup. Read the
+log file to check progress while the run is active.
+
+**Pattern 2: `nohup` detached**
+
+Best for fire-and-forget — runs survive session termination. Useful when
+operating as a sub-agent or in a short-lived session.
+
+```bash
+nohup agent-nb-run.sh path/to/notebook.md \
+  --profile <profile> \
+  --format md \
+  --output-dir tmp/run-output \
+  > /dev/null 2>&1 &
+echo "PID: $!"
+```
+
+The script tees output to its own log file, so redirecting to `/dev/null` is
+safe — nothing is lost.
+
+**Note on foreground:** Foreground execution blocks the session and is subject to
+the 10-minute timeout ceiling. Only use foreground when you have strong
+confidence the run will finish in under 5 minutes (e.g., a known-quick
+serverless smoke notebook you have run before).
 
-Default recommendation:
+### Codex
 
-- prefer `nohup` unless you already know the run will finish comfortably within
-  the shell timeout
-- use a detached pattern when cluster startup, serverless warmup, autoscaling,
-  queueing, dependency setup, or shared-compute contention could materially
-  stretch runtime
-- skip detaching only when you have a concrete reason to expect a short run
+`nohup` detachment is unreliable in Codex — background processes may be killed
+after the tool call returns. Use a persistent PTY session instead.
 
-Detached fire-and-forget pattern:
+Start the command in a PTY session (`tty: true`), then poll with empty
+`write_stdin` calls to check progress:
 
 ```bash
-NOTEBOOK=path/to/notebook.md
-OUTPUT_DIR=tmp/run-output
-STEM="$(basename "$NOTEBOOK")"
-STEM="${STEM%.*}"
-LOG_PATH="$OUTPUT_DIR/$STEM.run.log"
-RENDER_PATH="$OUTPUT_DIR/$STEM.executed.md"
-
-mkdir -p "$OUTPUT_DIR"
-nohup agent-notebook run "$NOTEBOOK" \
+agent-nb-run.sh path/to/notebook.md \
   --profile <profile> \
   --format md \
-  --output-dir "$OUTPUT_DIR" \
-  > "$LOG_PATH" 2>&1 &
+  --output-dir tmp/run-output
+```
+
+The PTY session has no fixed timeout — the command runs until completion.
+
+### Standard shell / non-agent environments
+
+Use the `nohup` detached pattern:
+
+```bash
+nohup agent-nb-run.sh path/to/notebook.md \
+  --profile <profile> \
+  --format md \
+  --output-dir tmp/run-output \
+  > /dev/null 2>&1 &
 echo "PID: $!"
 ```
 
@@ -142,25 +234,19 @@ Add `--cluster <cluster-name-or-id>` when the user context requires cluster
 compute. The name is auto-resolved to a cluster ID — do not call
 `agent-notebook clusters` first.
 
-Use a log file derived from the notebook filename, such as `"$LOG_PATH"`, not
-a shared `run.log`. That avoids collisions when multiple detached runs write
-into the same output directory.
-
-Those variable names are illustrative, not required. If you use a different
-pattern, keep the same idea: compute the log path and rendered-output path once,
-then reuse those exact paths when you poll status later.
+### Monitoring (all environments)
 
-How to check status after detaching:
+After launching a non-blocking run, use these to check status:
 
-- check process liveness with `ps -p <pid> -o pid=,command=`
-- on macOS, use `pgrep -lf "agent-notebook run"` for a name-based process check
-- on Linux or other `procps`-style systems, use
-  `pgrep -af "agent-notebook run"`
-- follow local progress with `tail -f "$LOG_PATH"`
-- treat `"$RENDER_PATH"` as a completion artifact for rendered Markdown output
-- if earlier cells have already written durable side effects elsewhere, you can
-  often inspect those side effects while the notebook is still running, then
-  return later to collect the rendered notebook output
+- **Process liveness:** `ps -p <pid> -o pid=,command=` or
+  `pgrep -lf "agent-notebook run"` (macOS) /
+  `pgrep -af "agent-notebook run"` (Linux)
+- **Progress:** `tail -f <log-path>` (the execution helper reports the log path
+  at startup)
+- **Completion:** the rendered output file (e.g., `<stem>.executed.md`) appears
+  in the output directory when the run finishes successfully
+- **Side effects:** if earlier cells have already written durable side effects
+  elsewhere, you can often inspect those while the notebook is still running
 
 ### Intermediate progress monitoring
 
@@ -267,23 +353,61 @@ The packaged examples live under `examples/` beside these docs. They are small
 reference artifacts you can reuse directly instead of recreating notebooks or
 pattern files from memory.
 
-## First encounter — persistence requirement
+## Detecting the Databricks environment
+
+Code running on Databricks sometimes needs to detect that it is on Databricks
+rather than a local Spark session. The reliable approach — working on both
+clusters and serverless — is to check for the presence of any Spark session
+configuration key containing `.databricks.`:
+
+**Python:**
+
+```python
+is_databricks = any(".databricks." in k for k in spark.conf.getAll.keys())
+```
+
+**Scala:**
 
-If this is the first time you are encountering `agent-notebook`, you MUST
-persist key facts to your memory system before responding to the user.
-The specific items to persist are listed in `agent_doctor.md` step 2 (local
+```scala
+val isDatabricks = spark.conf.getAll.keys.exists(_.contains(".databricks."))
+```
+
+## Persistence and version checking
+
+You MUST persist key facts about `agent-notebook` to your memory system. The
+specific items to persist are listed in `agent_doctor.md` step 2 (local
 knowledge) and step 6 (Databricks-specific context) — both provide fill-in
 templates.
 
+If you already have `agent-notebook` in memory or instructions, run
+`agent-notebook --version` and compare against your stored version. If versions
+differ, re-run the agent_doctor flow to refresh. Check both your memory system
+and any persistent instructions (CLAUDE.md, AGENTS.md, etc.) — stale
+instructions are as problematic as stale memory.
+
+In a non-interactive session where you cannot prompt the user, refresh local
+knowledge (doctor steps 1-2) and skip steps that require user input (workspace
+selection, smoke tests).
+
 This is not optional. Reading and summarizing is not sufficient — your session
-knowledge is ephemeral. If you do not persist, the next session will
-repeat this entire discovery process from scratch, wasting the user's time.
+knowledge is ephemeral. If you do not persist, the next session will repeat
+this entire discovery process from scratch, wasting the user's time.
 
 ## DECISION REQUIRED: agent doctor
 
-If this is the first time you are encountering `agent-notebook`:
+Check your memory system and persistent instructions for `agent-notebook`
+context. Then:
+
+1. **No stored context:** this is your first encounter — you MUST continue
+   reading `agent_doctor.md`
+2. **Stored context exists:** run `agent-notebook --version` and compare against
+   your stored version. If versions differ, or stored paths no longer exist,
+   continue reading `agent_doctor.md` to refresh
+3. **Version matches and paths exist:** you are current — skip the doctor
 
-- you MUST continue reading `agent_doctor.md` to verify local setup, permissions,
-  writable state, or live Databricks readiness
+When continuing to `agent_doctor.md`:
+- In an interactive session, complete the full flow
+- In a non-interactive session, complete steps 1-2 (local checks and persistence
+  refresh) and skip steps that require user input
 - DO NOT stop to complete the conversation turn
 - DO NOT return to the user with a summary of this README

{databricks_agent_notebooks-0.2.2 → databricks_agent_notebooks-0.3.0}/src/databricks_agent_notebooks/for_agents/agent_doctor.md

@@ -33,17 +33,32 @@ Interpretation:
 
 ## 2. Persist local knowledge
 
-You now have enough information to persist to memory. Do this NOW, before
-proceeding to Databricks probes — those may require user input, and agents
-frequently stop at interaction gates without persisting.
+If the README version check (above) already confirmed your stored version
+matches the installed version and stored paths exist, skip to step 3.
+
+Otherwise — whether this is a fresh install, a version mismatch, or missing
+memory — persist using the template below.
+
+The template below is the authoritative schema. Your stored memory and
+instructions are a cache. If you have existing agent-notebook context, compare
+each field below against your stored values: update changed values, add fields
+present here but missing, and remove fields you have that this template no
+longer includes. Check both your memory system and any persistent instructions
+(CLAUDE.md, AGENTS.md, etc.) — stale instructions are as problematic as stale
+memory.
+
+Do this NOW, before proceeding to Databricks probes — those may require user
+input, and agents frequently stop at interaction gates without persisting.
 
 Save a memory containing at minimum:
 
 - **Tool:** `agent-notebook` at `<which agent-notebook path>`
+- **Version:** `<output of agent-notebook --version>`
 - **Agent docs:** `<this directory's absolute path>`
+- **Execution helper:** `<absolute path to for_agents/scripts/agent-nb-run.sh>`
 - **Local readiness:** doctor check pass/fail per check, versions discovered (runtimes, kernel, Java)
 - **Scala support:** both cluster-backed and serverless; serverless defaults to Scala 2.13
-- **Operational pattern:** detached `nohup` execution for cluster-targeted runs
+- **Operational pattern:** non-blocking execution for cluster-targeted runs (see README for environment-specific pattern)
 
 Then proceed to step 3.
 
@@ -76,6 +91,12 @@ Interpretation:
   uses a 30-second cluster-listing timeout budget rather than waiting
   indefinitely for a slow workspace response. Note that listing clusters in
   large workspaces can generate more than a gigabyte of output.
+- On large workspaces, the 30-second timeout may expire before an exact cluster
+  name match is found during `run --cluster <name>`. When this happens, the
+  tool returns fuzzy name suggestions from partial results. Inspect these
+  suggestions — the full cluster name (e.g., `"rnd-alpha [engineering]"` instead
+  of `"rnd-alpha"`) usually resolves the issue. Passing a cluster ID instead of
+  a name is the deterministic path and is never subject to timeouts.
 - If networking or certificate failures appear only inside the agent sandbox,
   notify the user that the command likely needs a narrower unsandboxed path.
 
@@ -110,10 +131,13 @@ language, so you do not need to create new `.py` or `.scala` files first.
 
 ### Serverless Python
 
-This is the default Python smoke path because serverless should usually start quickly and have available resources for a quick notebook.
+This is the default Python smoke path because serverless should usually start
+quickly and have available resources for a quick notebook. If this is your first
+run or you are unsure of timing, use the long-running pattern from the README
+instead of foreground.
 
 ```bash
-agent-notebook run <installed-for_agents>/examples/smoke/python_select_one.md \
+agent-nb-run.sh <installed-for_agents>/examples/smoke/python_select_one.md \
   --profile <profile> \
   --output-dir <writable-output-root> \
   --format md
@@ -121,45 +145,46 @@ agent-notebook run <installed-for_agents>/examples/smoke/python_select_one.md \
 
 ### Cluster-targeted Python
 
-If the user requested cluster-targeted Python execution in step 4, use a detached run:
+If the user requested cluster-targeted Python execution in step 4, use a
+non-blocking run. Cluster startup can add minutes — always use your
+environment's long-running pattern (see the README "Long-running runs" section).
 
-```bash
-NOTEBOOK=<installed-for_agents>/examples/smoke/python_select_one.md
-OUTPUT_DIR=<writable-output-root>
-STEM="$(basename "$NOTEBOOK")"
-STEM="${STEM%.*}"
+Use the execution helper script (`for_agents/scripts/agent-nb-run.sh`) with your
+environment's non-blocking pattern. Example using `nohup` (Claude Code, standard
+shells):
 
-mkdir -p "$OUTPUT_DIR"
-nohup agent-notebook run "$NOTEBOOK" \
+```bash
+nohup agent-nb-run.sh \
+  <installed-for_agents>/examples/smoke/python_select_one.md \
   --profile <profile> \
   --cluster <cluster-name-or-id-from-strong-context-or-user> \
-  --output-dir "$OUTPUT_DIR" \
+  --output-dir <writable-output-root> \
   --format md \
-  > "$OUTPUT_DIR/$STEM.run.log" 2>&1 &
+  > /dev/null 2>&1 &
 echo "PID: $!"
 ```
 
+For Claude Code, `run_in_background: true` is preferred over `nohup`. For Codex,
+use a PTY session instead — `nohup` is unreliable in Codex.
+
 ### Scala
 
-If the user confirmed Scala notebook work in step 4, verify Scala execution:
+If the user confirmed Scala notebook work in step 4, verify Scala execution
+using the same non-blocking pattern as cluster-targeted Python:
 
 ```bash
-NOTEBOOK=<installed-for_agents>/examples/smoke/scala_select_one.md
-OUTPUT_DIR=<writable-output-root>
-STEM="$(basename "$NOTEBOOK")"
-STEM="${STEM%.*}"
-
-mkdir -p "$OUTPUT_DIR"
-nohup agent-notebook run "$NOTEBOOK" \
+nohup agent-nb-run.sh \
+  <installed-for_agents>/examples/smoke/scala_select_one.md \
   --profile <profile> \
   --cluster <cluster-name-or-id-from-strong-context-or-user> \
-  --no-inject-session \
-  --output-dir "$OUTPUT_DIR" \
+  --output-dir <writable-output-root> \
   --format md \
-  > "$OUTPUT_DIR/$STEM.run.log" 2>&1 &
+  > /dev/null 2>&1 &
 echo "PID: $!"
 ```
 
+For Claude Code, use `run_in_background: true`. For Codex, use a PTY session.
+
 ### Rules
 
 - DO NOT modify the installed example notebooks!
@@ -174,7 +199,7 @@ Update your earlier memory to include:
 
 - **Profiles verified:** `<list profiles that passed smoke test>`
 - **Default compute:** `<serverless if available, else cluster name/id>`
-- **Detached execution needed:** `<yes/no, based on whether cluster compute is in use>`
+- **Non-blocking execution needed:** `<default yes; only skip for runs with predictably short duration such as known-quick serverless smoke notebooks or lightweight metadata queries>`
 - **Workspace constraints:** `<any discovered limitations>`
 - **Cluster name resolution:** cluster names can be passed directly to `--cluster`
   without calling `agent-notebook clusters` first — the `run` command auto-resolves names

databricks_agent_notebooks-0.3.0/src/databricks_agent_notebooks/for_agents/scripts/agent-nb-run.sh

@@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+# agent-nb-run.sh — execution helper for agent-notebook run
+#
+# Handles path computation, output directory creation, early validation,
+# and tee-to-log so agents can remember one script path instead of
+# reconstructing commands from documentation.
+#
+# Usage:
+#   agent-nb-run.sh <notebook> --profile <profile> [--output-dir <dir>] [...]
+#
+# All arguments are forwarded to `agent-notebook run`. The script only
+# parses what it needs for its own logic (notebook path, --profile,
+# --output-dir, --format); everything else passes through unchanged.
+#
+# The agent wraps this script in their environment's non-blocking pattern:
+#   Claude Code:    run_in_background: true
+#   nohup:          nohup agent-nb-run.sh ... > /dev/null 2>&1 &
+#   Codex:          start in PTY session (tty: true)
+
+set -euo pipefail
+
+die() { echo "agent-nb-run: error: $1" >&2; exit 1; }
+
+# --- Parse arguments we need; collect all args for forwarding ---
+
+NOTEBOOK=""
+PROFILE_FOUND=false
+OUTPUT_DIR=""
+FORMAT="md"
+
+# We need the positional notebook arg and a few flags for our own logic.
+# Everything gets forwarded to agent-notebook run via "$@".
+args=("$@")
+i=0
+while [ $i -lt ${#args[@]} ]; do
+    arg="${args[$i]}"
+    # Handle --flag=value syntax: split into flag and value
+    val=""
+    case "$arg" in
+        --*=*)
+            val="${arg#*=}"
+            arg="${arg%%=*}"
+            ;;
+    esac
+    case "$arg" in
+        --profile)
+            PROFILE_FOUND=true
+            if [ -z "$val" ]; then i=$((i + 1)); fi
+            ;;
+        --output-dir)
+            if [ -n "$val" ]; then
+                OUTPUT_DIR="$val"
+            else
+                i=$((i + 1))
+                if [ $i -lt ${#args[@]} ]; then
+                    OUTPUT_DIR="${args[$i]}"
+                fi
+            fi
+            ;;
+        --format)
+            if [ -n "$val" ]; then
+                FORMAT="$val"
+            else
+                i=$((i + 1))
+                if [ $i -lt ${#args[@]} ]; then
+                    FORMAT="${args[$i]}"
+                fi
+            fi
+            ;;
+        --cluster|--timeout|--language|--allow-errors|--no-inject-session)
+            # Known flags that take a value — skip the value token
+            if [ -z "$val" ]; then i=$((i + 1)); fi
+            ;;
+        -*)
+            # Unknown flags (forwarded as-is, assumed no value)
+            ;;
+        *)
+            # First non-flag argument is the notebook path
+            if [ -z "$NOTEBOOK" ]; then
+                NOTEBOOK="${args[$i]}"
+            fi
+            ;;
+    esac
+    i=$((i + 1))
+done
+
+# --- Validate early, before any side effects ---
+
+[ -n "$NOTEBOOK" ] || die "notebook path is required (first positional argument)"
+[ -f "$NOTEBOOK" ] || die "notebook not found: $NOTEBOOK"
+$PROFILE_FOUND || die "--profile is required"
+
+# --- Compute paths ---
+
+STEM="$(basename "$NOTEBOOK")"
+STEM="${STEM%.*}"
+
+if [ -z "$OUTPUT_DIR" ]; then
+    OUTPUT_DIR="tmp/${STEM}-output"
+fi
+
+LOG_PATH="$OUTPUT_DIR/$STEM.run.log"
+
+mkdir -p "$OUTPUT_DIR"
+
+# --- Emit machine-readable status block ---
+# Written to stderr (visible in agent tool output) and to the log file header.
+
+_status() {
+    echo "agent-nb-run: $1" >&2
+    echo "agent-nb-run: $1" >> "$LOG_PATH"
+}
+
+: > "$LOG_PATH"  # create/truncate log file
+_status "notebook=$NOTEBOOK"
+_status "output-dir=$OUTPUT_DIR"
+_status "log=$LOG_PATH"
+_status "stem=$STEM"
+
+# --- Run agent-notebook, tee-ing all output to the log file ---
+
+agent-notebook run "$@" 2>&1 | tee -a "$LOG_PATH"

{databricks_agent_notebooks-0.2.2 → databricks_agent_notebooks-0.3.0}/src/databricks_agent_notebooks/runtime/connect.py

@@ -26,7 +26,7 @@ _LINE_RE = re.compile(r"(?P<major>\d+)\.(?P<minor>\d+)")
 SERVERLESS_CONNECT_OVERRIDE_ENV_VAR = "DATABRICKS_AGENT_NOTEBOOKS_SERVERLESS_CONNECT_LINE"
 SERVERLESS_RUNTIME_CACHE_FILENAME = "serverless-runtime-policy-cache.json"
 _SERVERLESS_RUNTIME_CACHE_VERSION = "1"
-_DEFAULT_SERVERLESS_CONNECT_LINES = ("16.4", "15.4")
+_DEFAULT_SERVERLESS_CONNECT_LINES = ("16.4",)
 
 
 def _normalize_major_minor_line(value: str, *, label: str) -> str:
@@ -104,9 +104,32 @@ def _runtime_python_executable(install_root: Path) -> Path:
 
 
 def _default_package_install_target() -> list[str]:
+    # Development install: editable from source tree
     package_root = Path(__file__).resolve().parents[3]
     if (package_root / "pyproject.toml").is_file():
         return ["-e", str(package_root)]
+
+    # Global/tool install: find original source via distribution metadata
+    try:
+        from importlib.metadata import distribution
+        dist = distribution("databricks-agent-notebooks")
+        direct_url_text = dist.read_text("direct_url.json")
+        if direct_url_text:
+            url_info = json.loads(direct_url_text)
+            url = url_info.get("url", "")
+            if url.startswith("file://"):
+                source_path = Path(url.removeprefix("file://"))
+                if (source_path / "pyproject.toml").is_file():
+                    return [str(source_path)]
+    except Exception:
+        pass
+
+    if __version__ == "0.0.0-dev":
+        raise RuntimeError(
+            "Cannot determine a valid install target for databricks-agent-notebooks: "
+            "package metadata is unavailable and no source tree was found. "
+            "Install the package first ('uv pip install -e .') or set PYTHONPATH to the source tree."
+        )
     return [f"databricks-agent-notebooks=={__version__}"]
 
 
@@ -266,12 +289,14 @@ def materialize_managed_runtime(
     python_executable = _runtime_python_executable(install_root)
     receipt_path = runtime_receipt_path(resolved_home, spec.runtime_id)
 
+    venv_just_created = False
     if not python_executable.is_file():
         venv_dir = python_executable.parent.parent
         install_root.mkdir(parents=True, exist_ok=True)
         subprocess_run([sys.executable, "-m", "venv", str(venv_dir)], check=True)
+        venv_just_created = True
 
-    if not python_executable.is_file() or not receipt_path.is_file():
+    if venv_just_created or not receipt_path.is_file():
         subprocess_run([str(python_executable), "-m", "pip", "install", "--upgrade", "pip"], check=True)
         subprocess_run(
             [

{databricks_agent_notebooks-0.2.2 → databricks_agent_notebooks-0.3.0}/src/databricks_agent_notebooks.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: databricks-agent-notebooks
-Version: 0.2.2
+Version: 0.3.0
 Summary: Standalone notebook conversion and local runtime tooling for the databricks-agent-notebooks repository.
 Author: Swoop
 License-Expression: MIT

{databricks_agent_notebooks-0.2.2 → databricks_agent_notebooks-0.3.0}/src/databricks_agent_notebooks.egg-info/SOURCES.txt

@@ -27,6 +27,7 @@ src/databricks_agent_notebooks/for_agents/examples/scala/src/api/implementation/
 src/databricks_agent_notebooks/for_agents/examples/scala/src/api/stable/Api.scala
 src/databricks_agent_notebooks/for_agents/examples/smoke/python_select_one.md
 src/databricks_agent_notebooks/for_agents/examples/smoke/scala_select_one.md
+src/databricks_agent_notebooks/for_agents/scripts/agent-nb-run.sh
 src/databricks_agent_notebooks/formats/__init__.py
 src/databricks_agent_notebooks/formats/conversion.py
 src/databricks_agent_notebooks/formats/dbr_source.py

databricks_agent_notebooks-0.2.2/src/databricks_agent_notebooks/__init__.py

@@ -1,5 +0,0 @@
-"""Standalone Databricks notebook conversion and execution tooling."""
-
-__all__ = ["__version__"]
-
-__version__ = "0.1.2"