calkit-python 0.28.4__tar.gz → 0.29.2__tar.gz

{calkit_python-0.28.4 → calkit_python-0.29.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.28.4
+Version: 0.29.2
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.28.4 → calkit_python-0.29.2}/calkit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.28.4"
+__version__ = "0.29.2"
 
 from .core import *  # noqa: F403, I001
 from . import git  # noqa: F401

calkit_python-0.29.2/calkit/cli/config.py

@@ -0,0 +1,240 @@
+"""Config CLI."""
+
+from __future__ import annotations
+
+import glob
+import os
+import subprocess
+
+import git
+import typer
+from git.exc import InvalidGitRepositoryError
+from typing_extensions import Annotated
+
+import calkit
+from calkit import config
+from calkit.cli.core import raise_error
+from calkit.dvc import configure_remote, get_remotes, set_remote_auth
+
+config_app = typer.Typer(no_args_is_help=True)
+
+
+@config_app.command(name="set")
+def set_config_value(key: str, value: str):
+    """Set a value in the config."""
+    keys = config.Settings.model_fields.keys()
+    if key not in keys:
+        raise_error(
+            f"Invalid config key: '{key}'; Valid keys are: {list(keys)}"
+        )
+    try:
+        cfg = config.read()
+        cfg = config.Settings.model_validate(cfg.model_dump() | {key: value})
+    except Exception as e:
+        raise_error(f"Failed to set {key} in config: {e}")
+    cfg.write()
+
+
+@config_app.command(name="get")
+def get_config_value(key: str) -> None:
+    """Get and print a value from the config."""
+    cfg = config.read().model_dump()
+    if key not in cfg:
+        raise_error(
+            f"Invalid config key: '{key}'; Valid keys are: {list(cfg.keys())}"
+        )
+    val = cfg[key]
+    if val is not None:
+        print(val)
+    else:
+        print()
+
+
+@config_app.command(name="unset")
+def unset_config_value(key: str):
+    """Unset a value in the config, returning it to default."""
+    model_fields = config.Settings.model_fields
+    if key not in model_fields:
+        raise_error(
+            f"Invalid config key: '{key}'; "
+            f"Valid keys: {list(model_fields.keys())}"
+        )
+    try:
+        cfg = config.read()
+        setattr(cfg, key, model_fields[key].default)
+    except Exception as e:
+        raise_error(f"Failed to unset {key} in config: {e}")
+    cfg.write()
+
+
+@config_app.command(name="setup-remote", help="Alias for 'remote'.")
+@config_app.command(name="remote")
+def setup_remote(
+    no_commit: Annotated[
+        bool,
+        typer.Option(
+            "--no-commit", help="Do not commit changes to DVC config."
+        ),
+    ] = False,
+):
+    """Setup the Calkit cloud as the default DVC remote and store a token in
+    the local config.
+    """
+    try:
+        configure_remote()
+        set_remote_auth()
+    except subprocess.CalledProcessError:
+        if not os.path.isfile(".dvc/config"):
+            raise_error(
+                "DVC remote config failed; have you run `calkit init`?"
+            )
+        raise_error(
+            "Failed to configure DVC remote; check DVC config for errors"
+        )
+    except InvalidGitRepositoryError:
+        raise_error("Current directory is not a Git repository")
+    except (ValueError, RuntimeError) as e:
+        raise_error(f"Failed to set up DVC remote: {e}")
+    if not no_commit:
+        repo = git.Repo()
+        repo.git.add(".dvc/config")
+        if ".dvc/config" in calkit.git.get_staged_files():
+            typer.echo("Committing changes to DVC config")
+            repo.git.commit([".dvc/config", "-m", "Set DVC remote"])
+
+
+@config_app.command(name="setup-remote-auth", help="Alias for 'remote-auth'.")
+@config_app.command(name="remote-auth")
+def setup_remote_auth():
+    """Store a Calkit cloud token in the local DVC config for all Calkit
+    remotes.
+    """
+    try:
+        remotes = get_remotes()
+    except Exception:
+        raise_error("Cannot list DVC remotes; check DVC config for errors")
+    for name, url in remotes.items():
+        if name == "calkit" or name.startswith("calkit:"):
+            typer.echo(f"Setting up authentication for DVC remote: {name}")
+            set_remote_auth(remote_name=name)
+
+
+@config_app.command(name="list")
+def list_config_keys():
+    """List keys in the config."""
+    cfg = config.read()
+    for key in cfg.model_dump():
+        typer.echo(key)
+
+
+@config_app.command(name="github-ssh")
+def config_github_ssh():
+    """Walk through the process of adding an SSH key to GitHub."""
+    typer.echo("Checking if you can already connect to GitHub via SSH")
+    # First check if we can already connect to GitHub
+    ssh_test_cmd = ["ssh", "-T", "git@github.com"]
+    p = subprocess.run(ssh_test_cmd, capture_output=True, text=True)
+    if "successfully authenticated" in p.stderr:
+        typer.echo("You can already connect to GitHub via SSH")
+        go_on = typer.confirm("Do you want to add a new SSH key anyway?")
+        if not go_on:
+            raise typer.Exit()
+    # If we can, ask the user if they still want to add a new key
+    # First check if the user has any SSH keys
+    ssh_dir = os.path.expanduser("~/.ssh")
+    existing_pub_keys = glob.glob(os.path.join(ssh_dir, "*.pub"))
+    # If not run ssh-keygen
+    if existing_pub_keys:
+        # Ask the user if they want to use an existing key or create a new one
+        typer.echo("Existing SSH public keys found:")
+        for i, key in enumerate(existing_pub_keys):
+            typer.echo(f"{i + 1}: {key}")
+        use_existing = typer.confirm("Do you want to use one of these keys?")
+        if use_existing:
+            key_choice = typer.prompt(
+                "Enter the number of the key to use", type=int
+            )
+            if 1 <= key_choice <= len(existing_pub_keys):
+                key_path = existing_pub_keys[key_choice - 1][:-4]
+            else:
+                typer.echo("Invalid choice")
+                # Keep asking until they give a valid choice
+                while True:
+                    key_choice = typer.prompt(
+                        "Enter the number of the key to use", type=int
+                    )
+                    if 1 <= key_choice <= len(existing_pub_keys):
+                        key_path = existing_pub_keys[key_choice - 1][:-4]
+                        break
+                    else:
+                        typer.echo("Invalid choice, please try again.")
+        else:
+            key_path = typer.prompt(
+                "Enter the path to save the new SSH key",
+                default=os.path.join(ssh_dir, "id_ed25519"),
+            )
+    else:
+        typer.echo("No existing SSH keys found")
+        key_path = typer.prompt(
+            "Enter the path to save the new SSH key",
+            default=os.path.join(ssh_dir, "id_ed25519"),
+        )
+    # Get the user's email from their Git config, and ask them if they want to
+    # use that or a different one
+    try:
+        user_git_email = git.Git().config("--get", "user.email").strip()
+    except Exception:
+        user_git_email = typer.prompt(
+            "No email found in Git config; enter email for SSH key"
+        )
+        git.Git().config("--global", "user.email", user_git_email)
+    # Do the same for user name even though we don't need it
+    try:
+        user_git_name = git.Git().config("--get", "user.name").strip()
+    except Exception:
+        user_git_name = typer.prompt(
+            "No name found in Git config; enter name for SSH key"
+        )
+        git.Git().config("--global", "user.name", user_git_name)
+    keygen_cmd = [
+        "ssh-keygen",
+        "-t",
+        "ed25519",
+        "-C",
+        user_git_email,
+        "-f",
+        key_path,
+    ]
+    subprocess.run(keygen_cmd)
+    # Start the SSH agent in the background
+    typer.echo("Checking that the SSH agent is running")
+    ssh_agent_cmd = subprocess.run(
+        ["ssh-agent", "-s"], capture_output=True, text=True
+    ).stdout
+    p = subprocess.run(ssh_agent_cmd, shell=True)
+    if p.returncode != 0:
+        raise_error("Failed to start ssh-agent")
+    # Add the SSH key to the ssh-agent
+    typer.echo(f"Adding SSH key to ssh-agent: {key_path}")
+    cmd = ["ssh-add", key_path]
+    p = subprocess.run(cmd)
+    if p.returncode != 0:
+        raise_error("Failed to add SSH key to ssh-agent; please try again")
+    # Now add to GitHub
+    gh_ssh_url = "https://github.com/settings/ssh/new"
+    typer.echo(
+        "Add the new SSH key to your GitHub account by visiting:\n"
+        f"{gh_ssh_url}"
+    )
+    with open(key_path + ".pub", "r") as f:
+        pub_key = f.read()
+    typer.echo(f"Paste this into the public key field:\n\n{pub_key}\n")
+    typer.confirm("Press Enter when done", default=True)
+    typer.echo("Testing SSH connection to GitHub")
+    p = subprocess.run(ssh_test_cmd, capture_output=True, text=True)
+    if "successfully authenticated" in p.stderr:
+        typer.echo("Successfully connected to GitHub via SSH!")
+    else:
+        raise_error(
+            "Failed to connect to GitHub via SSH; please check your setup"
+        )

{calkit_python-0.28.4 → calkit_python-0.29.2}/calkit/cli/main.py

@@ -981,22 +981,23 @@ def run(
         except Exception as e:
             os.environ.pop("CALKIT_PIPELINE_RUNNING", None)
             raise_error(f"Pipeline compilation failed: {e}")
-    # Get status of Git repo before running
-    repo = git.Repo()
-    git_rev = repo.head.commit.hexsha
-    try:
-        git_branch = repo.active_branch.name
-    except TypeError:
-        # If no branch is checked out, we are in a detached HEAD state
-        git_branch = None
-    git_changed_files_before = calkit.git.get_changed_files(repo=repo)
-    git_staged_files_before = calkit.git.get_staged_files(repo=repo)
-    git_untracked_files_before = calkit.git.get_untracked_files(repo=repo)
-    # Get status of DVC repo before running
-    dvc_repo = dvc.repo.Repo()
-    dvc_status_before = dvc_repo.status()
-    dvc_data_status_before = dvc_repo.data_status()
-    dvc_data_status_before.pop("git", None)  # Remove git status
+    if save_log:
+        # Get status of Git repo before running
+        repo = git.Repo()
+        git_rev = repo.head.commit.hexsha
+        try:
+            git_branch = repo.active_branch.name
+        except TypeError:
+            # If no branch is checked out, we are in a detached HEAD state
+            git_branch = None
+        git_changed_files_before = calkit.git.get_changed_files(repo=repo)
+        git_staged_files_before = calkit.git.get_staged_files(repo=repo)
+        git_untracked_files_before = calkit.git.get_untracked_files(repo=repo)
+        # Get status of DVC repo before running
+        dvc_repo = dvc.repo.Repo()
+        dvc_status_before = dvc_repo.status()
+        dvc_data_status_before = dvc_repo.data_status()
+        dvc_data_status_before.pop("git", None)  # Remove git status
     if targets is None:
         targets = []
     args = deepcopy(targets)
@@ -1855,3 +1856,51 @@ def run_jupyter(
     """Run a command with the Jupyter CLI."""
     process = subprocess.run([sys.executable, "-m", "jupyter"] + sys.argv[2:])
     sys.exit(process.returncode)
+
+
+@app.command(name="latexmk")
+def run_latexmk(
+    tex_file: Annotated[str, typer.Argument(help="The .tex file to compile.")],
+    environment: Annotated[
+        str | None,
+        typer.Option(
+            "--env",
+            "-e",
+            help=("Environment in which to run latexmk, if applicable."),
+        ),
+    ] = None,
+):
+    """Compile a LaTeX document with latexmk.
+
+    If a Calkit environment is not specified, latexmk will be run in the
+    system environment if available. If not available, a TeX Live Docker
+    container will be used.
+    """
+    latexmk_cmd = [
+        "latexmk",
+        "-pdf",
+        "-cd",
+        "-silent",
+        "-synctex=1",
+        "-interaction=nonstopmode",
+        tex_file,
+    ]
+    if environment is not None:
+        cmd = ["calkit", "xenv", "--name", environment] + latexmk_cmd
+    elif calkit.check_dep_exists("latexmk"):
+        cmd = latexmk_cmd
+    else:
+        cmd = [
+            "docker",
+            "run",
+            "--rm",
+            "-v",
+            f"{os.getcwd()}:/work",
+            "-w",
+            "/work",
+            "texlive/texlive:latest-full",
+        ] + latexmk_cmd
+    try:
+        subprocess.check_call(cmd)
+    except subprocess.CalledProcessError:
+        raise_error("latexmk failed")

{calkit_python-0.28.4 → calkit_python-0.29.2}/calkit/cli/notebooks.py

@@ -163,6 +163,17 @@ def execute_notebook(
             ),
         ),
     ] = None,
+    language: Annotated[
+        str,
+        typer.Option(
+            "--language",
+            "-l",
+            help=(
+                "Notebook language; if 'matlab', MATLAB kernel must be "
+                "available in environment."
+            ),
+        ),
+    ] = "python",
     verbose: Annotated[
         bool, typer.Option("--verbose", "-v", help="Print verbose output.")
     ] = False,
@@ -176,10 +187,24 @@ def execute_notebook(
 
     if os.path.isabs(path):
         raise ValueError("Path must be relative")
+    if language.lower() not in ["python", "matlab", "julia"]:
+        raise ValueError(
+            "Language must be one of 'python', 'matlab', or 'julia'"
+        )
     # First, ensure the specified environment has a kernel we can use
-    kernel_name = check_env_kernel(
-        env_name=env_name, no_check=no_check, verbose=verbose
-    )
+    # We need to check the environment type and create the kernel if needed
+    if language.lower() == "python":
+        kernel_name = check_env_kernel(
+            env_name=env_name, no_check=no_check, verbose=verbose
+        )
+    elif language.lower() == "matlab":
+        kernel_name = "jupyter_matlab_kernel"
+    else:
+        raise_error(f"Language '{language}' not yet supported")
+    # We can't handle parameters unless language is Python
+    if language.lower() != "python":
+        if params or params_json is not None or params_base64 is not None:
+            raise_error("Parameters can only be passed to Python notebooks")
     # Parse parameters
     if params:
         try:
@@ -214,14 +239,34 @@ def execute_notebook(
         typer.echo(f"Using kernel: {kernel_name}")
         typer.echo(f"Running with cwd: {notebook_dir}")
         typer.echo(f"Output will be saved to: {fpath_out_exec}")
-    papermill.execute_notebook(
-        input_path=path,
-        output_path=fpath_out_exec,
-        kernel_name=kernel_name,
-        log_output=True,
-        parameters=parsed_params,
-        cwd=notebook_dir,
-    )
+    # If this is a Python or Julia notebook, we can use Papermill
+    # If it's a MATLAB notebook, we need to use the MATLAB kernel inside the
+    # specified environment
+    if language.lower() in ["python", "julia"]:
+        papermill.execute_notebook(
+            input_path=path,
+            output_path=fpath_out_exec,
+            kernel_name=kernel_name,
+            log_output=True,
+            parameters=parsed_params,
+            cwd=notebook_dir,
+        )
+    elif language.lower() == "matlab":
+        # Use nbconvert to execute the notebook with the MATLAB kernel
+        cmd = [
+            "python",
+            "-m",
+            "jupyter",
+            "nbconvert",
+            "--to",
+            "notebook",
+            "--execute",
+            f"--ExecutePreprocessor.kernel_name={kernel_name}",
+            "--output",
+            fpath_out_exec,
+            path,
+        ]
+        run_in_env(cmd, env_name=env_name, no_check=no_check, verbose=verbose)
     for to_fmt in to:
         if to_fmt != "notebook":
             try:
@@ -250,6 +295,6 @@ def execute_notebook(
                 fname_out,
             ]
             typer.echo(f"Exporting {to_fmt}")
-            run_in_env(
-                cmd=cmd, env_name=env_name, no_check=True, verbose=verbose
-            )
+            p = subprocess.run(cmd)
+            if p.returncode != 0:
+                raise_error(f"nbconvert failed for format '{to_fmt}'")

{calkit_python-0.28.4 → calkit_python-0.29.2}/calkit/dvc.py

@@ -18,7 +18,7 @@ logger = logging.getLogger(__package__)
 logger.setLevel(logging.INFO)
 
 
-def configure_remote(wdir: str = None):
+def configure_remote(wdir: str | None = None):
     try:
         project_name = calkit.detect_project_name(wdir=wdir)
     except ValueError as e:
@@ -70,7 +70,9 @@ def configure_remote(wdir: str = None):
 
 
 def set_remote_auth(
-    remote_name: str = None, always_auth: bool = False, wdir: str = None
+    remote_name: str | None = None,
+    always_auth: bool = False,
+    wdir: str | None = None,
 ):
     """Get a token and set it in the local DVC config so we can interact with
     the cloud as an HTTP remote.
@@ -85,7 +87,7 @@ def set_remote_auth(
         )["access_token"]
         settings.dvc_token = token
         settings.write()
-    subprocess.check_call(
+    p1 = subprocess.run(
         [
             sys.executable,
             "-m",
@@ -99,7 +101,7 @@ def set_remote_auth(
         ],
         cwd=wdir,
     )
-    subprocess.check_call(
+    p2 = subprocess.run(
         [
             sys.executable,
             "-m",
@@ -113,6 +115,10 @@ def set_remote_auth(
         ],
         cwd=wdir,
     )
+    if p1.returncode != 0 or p2.returncode != 0:
+        raise RuntimeError(
+            f"Failed to set DVC remote authentication for {remote_name}"
+        )
 
 
 def add_external_remote(owner_name: str, project_name: str) -> dict:
@@ -159,13 +165,15 @@ def get_remotes(wdir: str | None = None) -> dict[str, str]:
     """Get a dictionary of DVC remotes, keyed by name, with URL as the
     value.
     """
-    out = (
-        subprocess.check_output(
-            [sys.executable, "-m", "dvc", "remote", "list"], cwd=wdir
-        )
-        .decode()
-        .strip()
+    p = subprocess.run(
+        [sys.executable, "-m", "dvc", "remote", "list"],
+        cwd=wdir,
+        capture_output=True,
+        text=True,
     )
+    if p.returncode != 0:
+        raise RuntimeError(f"Error getting DVC remotes: {p.stderr.strip()}")
+    out = p.stdout.strip()
     if not out:
         return {}
     resp = {}
@@ -182,12 +190,14 @@ def get_remotes(wdir: str | None = None) -> dict[str, str]:
     return resp
 
 
-def list_paths(wdir: str = None, recursive=False) -> list[str]:
+def list_paths(wdir: str | None = None, recursive=False) -> list[str]:
     """List paths tracked with DVC."""
-    return [p.get("path") for p in list_files(wdir=wdir, recursive=recursive)]
+    return [
+        p.get("path", "") for p in list_files(wdir=wdir, recursive=recursive)
+    ]
 
 
-def list_files(wdir: str = None, recursive=True) -> list[dict]:
+def list_files(wdir: str | None = None, recursive=True) -> list[dict]:
     """Return a list with all files in DVC, including their path and md5
     checksum.
     """

{calkit_python-0.28.4 → calkit_python-0.29.2}/calkit/models/pipeline.py

@@ -113,6 +113,7 @@ class Stage(BaseModel):
         "python-script",
         "latex",
         "matlab-script",
+        "matlab-command",
         "docker-command",
         "shell-command",
         "shell-script",
@@ -164,7 +165,9 @@ class Stage(BaseModel):
 
     @property
     def xenv_cmd(self) -> str:
-        return f"calkit xenv -n {self.environment} --no-check"
+        if self.environment == "_system":
+            return ""
+        return f"calkit xenv -n {self.environment} --no-check --"
 
     def to_dvc(self) -> dict:
         """Convert to a DVC stage.
@@ -193,7 +196,7 @@ class PythonScriptStage(Stage):
 
     @property
     def dvc_cmd(self) -> str:
-        cmd = f"{self.xenv_cmd} -- python {self.script_path}"
+        cmd = f"{self.xenv_cmd} python {self.script_path}"
         for arg in self.args:
             cmd += f" {arg}"
         return cmd
@@ -212,7 +215,7 @@ class LatexStage(Stage):
 
     @property
     def dvc_cmd(self) -> str:
-        cmd = f"{self.xenv_cmd} -- latexmk -cd -interaction=nonstopmode"
+        cmd = f"{self.xenv_cmd} latexmk -cd -interaction=nonstopmode"
         if not self.verbose:
             cmd += " -silent"
         if self.force:
@@ -247,7 +250,26 @@ class MatlabScriptStage(Stage):
 
     @property
     def dvc_cmd(self) -> str:
-        return f"{self.xenv_cmd} -- \"run('{self.script_path}');\""
+        cmd = self.xenv_cmd
+        if self.environment == "_system":
+            cmd += "matlab -batch"
+        cmd += f" \"run('{self.script_path}');\""
+        return cmd
+
+
+class MatlabCommandStage(Stage):
+    kind: Literal["matlab-command"] = "matlab-command"
+    command: str
+
+    @property
+    def dvc_cmd(self) -> str:
+        # We need to escape quotes in the command
+        matlab_cmd = self.command.replace('"', '\\"')
+        cmd = self.xenv_cmd
+        if self.environment == "_system":
+            cmd += "matlab -batch"
+        cmd += f' "{matlab_cmd}"'
+        return cmd
 
 
 class ShellCommandStage(Stage):
@@ -257,14 +279,13 @@ class ShellCommandStage(Stage):
 
     @property
     def dvc_cmd(self) -> str:
-        cmd = ""
-        if self.environment != "_system":
-            cmd = f"{self.xenv_cmd} -- "
+        shell_cmd = self.command.replace('"', '\\"')
+        cmd = self.xenv_cmd
         if self.shell == "zsh":
             norc_args = "-f"
         else:
             norc_args = "--noprofile --norc"
-        cmd += f'{self.shell} {norc_args} -c "{self.command}"'
+        cmd += f' {self.shell} {norc_args} -c "{shell_cmd}"'
         return cmd
 
 
@@ -280,14 +301,12 @@ class ShellScriptStage(Stage):
 
     @property
     def dvc_cmd(self) -> str:
-        cmd = ""
-        if self.environment != "_system":
-            cmd = f"{self.xenv_cmd} -- "
+        cmd = self.xenv_cmd
         if self.shell == "zsh":
             norc_args = "-f"
         else:
             norc_args = "--noprofile --norc"
-        cmd += f"{self.shell} {norc_args} {self.script_path}"
+        cmd += f" {self.shell} {norc_args} {self.script_path}"
         for arg in self.args:
             cmd += f" {arg}"
         return cmd
@@ -327,7 +346,7 @@ class JuliaScriptStage(Stage):
 
     @property
     def dvc_cmd(self) -> str:
-        cmd = f'{self.xenv_cmd} -- "include(\\"{self.script_path}\\")"'
+        cmd = f'{self.xenv_cmd} "include(\\"{self.script_path}\\")"'
         return cmd
 
     @property
@@ -343,7 +362,7 @@ class JuliaCommandStage(Stage):
     def dvc_cmd(self) -> str:
         # We need to escape quotes in the command
         julia_cmd = self.command.replace('"', '\\"')
-        cmd = f'{self.xenv_cmd} -- "{julia_cmd}"'
+        cmd = f'{self.xenv_cmd} "{julia_cmd}"'
         return cmd
 
 
@@ -373,6 +392,7 @@ class JupyterNotebookStage(Stage):
     executed_ipynb_storage: Literal["git", "dvc"] | None = "dvc"
     html_storage: Literal["git", "dvc"] | None = "dvc"
     parameters: dict[str, Any] = {}
+    language: Literal["python", "matlab", "julia"] = "python"
 
     def update_parameters(self, params: dict) -> None:
         """If we have any templated parameters, update those, e.g., from
@@ -430,7 +450,10 @@ class JupyterNotebookStage(Stage):
 
     @property
     def dvc_cmd(self) -> str:
-        cmd = f"calkit nb execute --environment {self.environment} --no-check"
+        cmd = (
+            f"calkit nb execute --environment {self.environment} "
+            f"--no-check --language {self.language}"
+        )
         if self.html_storage:
             cmd += " --to html"
         if self.parameters:
@@ -533,6 +556,7 @@ class Pipeline(BaseModel):
                 PythonScriptStage
                 | LatexStage
                 | MatlabScriptStage
+                | MatlabCommandStage
                 | ShellCommandStage
                 | ShellScriptStage
                 | DockerCommandStage