calkit-python 0.39.1__tar.gz → 0.40.0__tar.gz

{calkit_python-0.39.1 → calkit_python-0.40.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.39.1
+Version: 0.40.0
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.39.1 → calkit_python-0.40.0}/calkit/cli/check.py

@@ -264,6 +264,7 @@ def check_environment(
         get_all_docker_lock_fpaths,
         get_all_venv_lock_fpaths,
         get_env_lock_fpath,
+        write_scheduler_env_lock,
     )
 
     dotenv.load_dotenv(dotenv_path=".env", verbose=verbose)
@@ -405,6 +406,12 @@ def check_environment(
             verbose=verbose,
             cache_key=env_name,
         )
+    elif env["kind"] in ("slurm", "pbs"):
+        # Job-scheduler envs have no external manifest to validate; the
+        # "check" is just writing a deterministic JSON lock file from the
+        # env config so DVC stages that depend on the env get invalidated
+        # when the config changes.
+        write_scheduler_env_lock(env_name=env_name, env=env)
     else:
         raise_error(f"Environment kind '{env['kind']}' not supported")
     return get_env_lock_fpath(env=env, env_name=env_name, as_posix=False)

{calkit_python-0.39.1 → calkit_python-0.40.0}/calkit/cli/main/core.py

@@ -57,7 +57,7 @@ from calkit.cli.new import new_app
 from calkit.cli.notebooks import notebooks_app
 from calkit.cli.office import office_app
 from calkit.cli.overleaf import overleaf_app
-from calkit.cli.slurm import slurm_app
+from calkit.cli.scheduler import scheduler_app
 from calkit.cli.update import update_app
 
 app = typer.Typer(
@@ -81,7 +81,11 @@ app.add_typer(check_app, name="check", help="Check things.")
 app.add_typer(latex_app, name="latex|tex", help="Work with LaTeX.")
 app.add_typer(overleaf_app, name="overleaf|ol", help="Interact with Overleaf.")
 app.add_typer(cloud_app, name="cloud", help="Interact with a Calkit Cloud.")
-app.add_typer(slurm_app, name="slurm", help="Work with SLURM.")
+app.add_typer(
+    scheduler_app,
+    name="scheduler|sch",
+    help="Work with a job scheduler (SLURM or PBS).",
+)
 app.add_typer(dev_app, name="dev", help="Developer tools.", hidden=True)
 
 
@@ -669,9 +673,11 @@ def add(
             paths.remove(".")
             if dvc_repo is not None:
                 dvc_status = dvc_repo.data_status()
-                for dvc_uncommitted in dvc_status["uncommitted"].get(
+                uncommitted = dvc_status["uncommitted"]
+                dvc_uncommitted_all = uncommitted.get(
                     "modified", []
-                ):
+                ) + uncommitted.get("deleted", [])
+                for dvc_uncommitted in dvc_uncommitted_all:
                     if os.path.exists(dvc_uncommitted):
                         if dry_run:
                             typer.echo(
@@ -679,12 +685,25 @@ def add(
                             )
                         else:
                             typer.echo(f"Adding {dvc_uncommitted} to DVC")
-                            dvc_repo.commit(dvc_uncommitted, force=True)
+                            dvc_repo.commit(
+                                dvc_uncommitted,
+                                force=True,
+                                allow_missing=True,
+                            )
                     else:
-                        warn(
-                            f"DVC uncommitted '{dvc_uncommitted}' does not "
-                            "exist; skipping"
-                        )
+                        if dry_run:
+                            typer.echo(
+                                f"Would commit deleted {dvc_uncommitted} to DVC"
+                            )
+                        else:
+                            typer.echo(
+                                f"Committing deleted {dvc_uncommitted} to DVC"
+                            )
+                            dvc_repo.commit(
+                                dvc_uncommitted,
+                                force=True,
+                                allow_missing=True,
+                            )
             if not disable_auto_ignore:
                 for untracked_file in untracked_git_files:
                     if (
@@ -1453,13 +1472,8 @@ def run(
             raise_error("Failed to initialize Git repo")
     # Set env vars
     calkit.set_env_vars(ck_info=ck_info)
-    # Clean all notebooks in the pipeline
-    try:
-        calkit.notebooks.clean_all_in_pipeline(ck_info=ck_info)
-    except Exception as e:
-        raise_error(f"Failed to clean notebooks: {e.__class__.__name__}: {e}")
     if not quiet:
-        typer.echo("Getting system information")
+        calkit.echo("💻 Getting system information")
     # Get system information
     system_info = calkit.get_system_info()
     if save_logs:
@@ -1475,7 +1489,7 @@ def run(
             json.dump(system_info, f, indent=2)
     # First check any system-level dependencies exist
     if not quiet:
-        typer.echo("Checking system-level dependencies")
+        calkit.echo("🔗 Checking system-level dependencies")
     try:
         calkit.check_system_deps(ck_info=ck_info, system_info=system_info)
     except Exception as e:
@@ -1483,7 +1497,7 @@ def run(
         raise_error(str(e))
     # Check all environments in the pipeline (with caching)
     # If any failed, warn the user that we might have problems running
-    typer.echo("Checking environments")
+    calkit.echo("📦 Checking environments")
     env_check_results = calkit.environments.check_all_in_pipeline(
         ck_info=ck_info, targets=targets, force=force
     )
@@ -1493,7 +1507,13 @@ def run(
         failed = not result.get("success", False)
         if failed:
             warn(f"Failed to check environment '{env_name}'")
-    # Check environments for each subproject so DVC can detect changes
+    # Clean all notebooks in the pipeline
+    try:
+        calkit.echo("📓 Cleaning notebooks")
+        calkit.notebooks.clean_all_in_pipeline(ck_info=ck_info)
+    except Exception as e:
+        raise_error(f"Failed to clean notebooks: {e.__class__.__name__}: {e}")
+    # Check environments and clean notebooks for each subproject
     subprojects = ck_info.get("subprojects", [])
     if subprojects:
         prev_cwd = os.getcwd()
@@ -1503,11 +1523,13 @@ def run(
             sp = Path(subproject["path"]).as_posix()
             if not os.path.isdir(sp):
                 continue
-            if not quiet:
-                typer.echo(f"Checking environments for subproject: {sp}")
             os.chdir(sp)
             try:
                 sp_ck_info = calkit.load_calkit_info()
+                if not quiet:
+                    calkit.echo(
+                        f"📦 Checking environments for subproject: {sp}"
+                    )
                 sp_env_results = calkit.environments.check_all_in_pipeline(
                     ck_info=sp_ck_info, force=force
                 )
@@ -1519,9 +1541,12 @@ def run(
                             f"Failed to check environment '{env_name}' "
                             f"in subproject '{sp}'"
                         )
+                if not quiet:
+                    calkit.echo(f"📓 Cleaning notebooks for subproject: {sp}")
+                calkit.notebooks.clean_all_in_pipeline(ck_info=sp_ck_info)
             except Exception as e:
                 warn(
-                    f"Failed to check environments for subproject '{sp}': "
+                    f"Failed to prepare subproject '{sp}': "
                     f"{e.__class__.__name__}: {e}"
                 )
             finally:
@@ -1533,7 +1558,7 @@ def run(
     dvc_stages = None
     if ck_info.get("pipeline", {}) or ck_info.get("subprojects"):
         if not quiet:
-            typer.echo("Compiling DVC pipeline")
+            calkit.echo("🔀 Compiling DVC pipeline")
         try:
             dvc_stages = calkit.pipeline.to_dvc(ck_info=ck_info, write=True)
         except Exception as e:

{calkit_python-0.39.1 → calkit_python-0.40.0}/calkit/cli/new.py

@@ -1471,13 +1471,105 @@ def new_slurm_env(
     ck_info["environments"] = envs
     with open("calkit.yaml", "w") as f:
         ryaml.dump(ck_info, f)
+    from calkit.cli.check import check_environment
+
+    env_lock_fpath = check_environment(env_name=name)
     if not no_commit:
         repo = calkit.git.get_repo()
         repo.git.add("calkit.yaml")
-        if repo.git.diff(["--staged", "calkit.yaml"]):
-            repo.git.commit(
-                ["calkit.yaml", "-m", f"Add SLURM environment {name}"]
-            )
+        if env_lock_fpath is not None:
+            repo.git.add(str(env_lock_fpath))
+        if repo.git.diff("--staged"):
+            repo.git.commit(["-m", f"Add SLURM environment {name}"])
+
+
+@new_app.command("pbs-env")
+def new_pbs_env(
+    name: Annotated[
+        str, typer.Option("--name", "-n", help="Environment name.")
+    ],
+    host: Annotated[
+        str,
+        typer.Option("--host", help="Host where PBS commands should run."),
+    ] = "localhost",
+    default_options: Annotated[
+        list[str],
+        typer.Option(
+            "--default-option",
+            help=(
+                "Default qsub option string (for example "
+                "--default-option=-l --default-option=walltime=01:00:00). "
+                "Repeat for multiple options."
+            ),
+        ),
+    ] = [],
+    default_setup: Annotated[
+        list[str],
+        typer.Option(
+            "--default-setup",
+            help=(
+                "Default shell setup command to run before PBS jobs "
+                "(for example 'module load julia/1.11'). Repeat for "
+                "multiple commands."
+            ),
+        ),
+    ] = [],
+    description: Annotated[
+        str | None, typer.Option("--description", help="Description.")
+    ] = None,
+    overwrite: Annotated[
+        bool,
+        typer.Option(
+            "--overwrite",
+            "-f",
+            help="Overwrite any existing environment with this name.",
+        ),
+    ] = False,
+    no_commit: Annotated[
+        bool, typer.Option("--no-commit", help="Do not commit changes.")
+    ] = False,
+):
+    """Create a new PBS environment."""
+    from calkit.cli.check import check_environment
+
+    host = host.strip()
+    if not host:
+        raise_error("Host is required")
+    ck_info = calkit.load_calkit_info()
+    envs = ck_info.get("environments", {})
+    if isinstance(envs, list):
+        typer.echo("Converting environments from list to dict")
+        envs = {env.pop("name"): env for env in envs}
+    if name in envs and not overwrite:
+        raise_error(
+            f"Environment with name {name} already exists "
+            "(use -f to overwrite)"
+        )
+    normalized_default_options = [
+        opt.strip() for opt in default_options if opt.strip()
+    ]
+    normalized_default_setup = [
+        cmd.strip() for cmd in default_setup if cmd.strip()
+    ]
+    env = {"kind": "pbs", "host": host}
+    if normalized_default_options:
+        env["default_options"] = normalized_default_options  # type: ignore
+    if normalized_default_setup:
+        env["default_setup"] = normalized_default_setup  # type: ignore
+    if description is not None:
+        env["description"] = description
+    envs[name] = env
+    ck_info["environments"] = envs
+    with open("calkit.yaml", "w") as f:
+        ryaml.dump(ck_info, f)
+    env_lock_fpath = check_environment(env_name=name)
+    if not no_commit:
+        repo = calkit.git.get_repo()
+        repo.git.add("calkit.yaml")
+        if env_lock_fpath is not None:
+            repo.git.add(str(env_lock_fpath))
+        if repo.git.diff("--staged"):
+            repo.git.commit(["-m", f"Add PBS environment {name}"])
 
 
 @new_app.command("uv-venv")