calkit-python 0.35.2__tar.gz → 0.35.4__tar.gz

{calkit_python-0.35.2 → calkit_python-0.35.4}/.gitignore

@@ -17,3 +17,5 @@ coverage
 junit.xml
 calk9
 dist
+*.vsix
+vscode-ext/out

{calkit_python-0.35.2 → calkit_python-0.35.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.35.2
+Version: 0.35.4
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

calkit_python-0.35.4/calkit/cli/main/__init__.py

@@ -0,0 +1,2 @@
+from . import xr  # noqa: F401
+from .core import *  # noqa: F403

calkit_python-0.35.2/calkit/cli/main.py → calkit_python-0.35.4/calkit/cli/main/core.py

@@ -1,4 +1,4 @@
-"""Main CLI app."""
+"""Core functionality in the main namespace of the CLI."""
 
 from __future__ import annotations
 
@@ -1325,10 +1325,11 @@ def run(
     if failed:
         raise_error("Pipeline failed")
     else:
+        _enc = sys.stdout.encoding or "utf-8"
         typer.echo(
             "Pipeline completed successfully ✅".encode(
-                "utf-8", errors="replace"
-            )
+                _enc, errors="replace"
+            ).decode(_enc)
         )
     if save_after_run or save_message is not None:
         if save_message is None:
@@ -1888,499 +1889,6 @@ def run_in_env(
         raise_error("Environment kind not supported")
 
 
-@app.command(name="xr")
-def execute_and_record(
-    cmd: Annotated[
-        list[str],
-        typer.Argument(
-            help="Command to execute and record. "
-            "If the first argument is a script, notebook or LaTeX file, "
-            "it will be treated as a stage with that file as "
-            "the target. Any command, including arguments, is supported."
-        ),
-    ],
-    environment: Annotated[
-        str | None,
-        typer.Option(
-            "--environment",
-            "-e",
-            help="Name of or path the spec file for the environment to use.",
-        ),
-    ] = None,
-    inputs: Annotated[
-        list[str],
-        typer.Option(
-            "--input",
-            "-i",
-            help="Input paths to record.",
-        ),
-    ] = [],
-    outputs: Annotated[
-        list[str],
-        typer.Option(
-            "--output",
-            "-o",
-            help="Output paths to record.",
-        ),
-    ] = [],
-    no_detect_io: Annotated[
-        bool,
-        typer.Option(
-            "--no-detect-io",
-            help=(
-                "Don't attempt to detect inputs and outputs from the command, "
-                "script, or notebook."
-            ),
-        ),
-    ] = False,
-    stage_name: Annotated[
-        str | None,
-        typer.Option(
-            "--stage",
-            help=(
-                "Name of the DVC stage to create for this command. If not "
-                "provided, a name will be generated automatically."
-            ),
-        ),
-    ] = None,
-    dry_run: Annotated[
-        bool,
-        typer.Option(
-            "--dry-run",
-            "-d",
-            help=(
-                "Print the environment and stage that would be created "
-                "without modifying calkit.yaml or executing the command."
-            ),
-        ),
-    ] = False,
-    force: Annotated[
-        bool,
-        typer.Option(
-            "--force",
-            "-f",
-            help="Force running stage even if it's up-to-date.",
-        ),
-    ] = False,
-    verbose: Annotated[
-        bool, typer.Option("--verbose", "-v", help="Print verbose output.")
-    ] = False,
-):
-    """Execute a command and if successful, record in the pipeline."""
-    import io
-    import shlex
-
-    from calkit.detect import (
-        detect_io,
-        generate_stage_name,
-    )
-    from calkit.environments import detect_env_for_stage
-    from calkit.models.io import PathOutput
-    from calkit.models.pipeline import (
-        JuliaCommandStage,
-        JuliaScriptStage,
-        JupyterNotebookStage,
-        LatexStage,
-        MatlabCommandStage,
-        MatlabScriptStage,
-        PythonScriptStage,
-        RScriptStage,
-        ShellCommandStage,
-        ShellScriptStage,
-    )
-    from calkit.pipeline import stages_are_similar
-
-    def _determine_output_storage(
-        path: str,
-        repo: git.Repo | None = None,
-        dvc_paths: list[str] | None = None,
-    ) -> str:
-        """Determine storage type (git or dvc) for an output path.
-
-        Parameters
-        ----------
-        path : str
-            Path to check.
-        repo : git.Repo | None
-            Git repository. If None, only extension and size are checked.
-        dvc_paths : list[str] | None
-            List of paths tracked by DVC.
-
-        Returns
-        -------
-        str
-            "git" or "dvc".
-        """
-        if dvc_paths is None:
-            dvc_paths = []
-        # If already tracked by git, use git
-        if repo is not None and repo.git.ls_files(path):
-            return "git"
-        # If already tracked by dvc, use dvc
-        if path in dvc_paths:
-            return "dvc"
-        # Check extension
-        if os.path.splitext(path)[-1] in DVC_EXTENSIONS:
-            return "dvc"
-        # Check size if file exists
-        if os.path.exists(path):
-            try:
-                if calkit.get_size(path) > DVC_SIZE_THRESH_BYTES:
-                    return "dvc"
-            except (OSError, IOError):
-                # If we cannot determine file size (e.g., permission
-                # or I/O issues), fall back to the default "git"
-                # behavior below.
-                pass
-        # Default to git for small/unknown files
-        return "git"
-
-    # First, read the pipeline before running so we can revert if the stage
-    # fails
-    ck_info = calkit.load_calkit_info()
-    ck_info_orig = deepcopy(ck_info)
-    pipeline = ck_info.get("pipeline", {})
-    stages = pipeline.get("stages", {})
-    # Strip and detect uv/pixi run prefix
-    if len(cmd) >= 2 and cmd[0] == "uv" and cmd[1] == "run":
-        if environment is None and os.path.isfile("pyproject.toml"):
-            environment = "pyproject.toml"
-        cmd = cmd[2:]
-    elif len(cmd) >= 2 and cmd[0] == "pixi" and cmd[1] == "run":
-        if environment is None and os.path.isfile("pixi.toml"):
-            environment = "pixi.toml"
-        cmd = cmd[2:]
-    # Guard against empty command after stripping uv/pixi run
-    if not cmd:
-        raise_error(
-            "No command specified after stripping environment prefix. "
-            "Usage: calkit xr [uv|pixi run] <command> [args]"
-        )
-    # Detect what kind of stage this is based on the command
-    # If the first argument is a notebook, we'll treat this as a notebook stage
-    # If the first argument is `python`, check that the second argument is a
-    # script, otherwise it's a shell-command stage
-    # If the first argument ends with .tex, we'll treat this as a LaTeX stage
-    first_arg = cmd[0]
-    stage = {}
-    language = None
-    if first_arg.endswith(".ipynb"):
-        cls = JupyterNotebookStage
-        stage["kind"] = "jupyter-notebook"
-        stage["notebook_path"] = first_arg
-        storage = calkit.notebooks.determine_storage(first_arg)
-        stage["html_storage"] = storage
-        stage["executed_ipynb_storage"] = storage
-    elif first_arg.endswith(".tex"):
-        cls = LatexStage
-        stage["kind"] = "latex"
-        stage["target_path"] = first_arg
-        language = "latex"
-        # Determine PDF storage based on whether it's already in git
-        pdf_path = first_arg.removesuffix(".tex") + ".pdf"
-        try:
-            repo = git.Repo(".")
-            if repo.git.ls_files(pdf_path):
-                stage["pdf_storage"] = "git"
-            else:
-                stage["pdf_storage"] = "dvc"
-        except InvalidGitRepositoryError:
-            # Not a git repo, default to dvc
-            stage["pdf_storage"] = "dvc"
-    elif first_arg == "python" and len(cmd) > 1 and cmd[1].endswith(".py"):
-        cls = PythonScriptStage
-        stage["kind"] = "python-script"
-        stage["script_path"] = cmd[1]
-        if len(cmd) > 2:
-            stage["args"] = cmd[2:]
-        language = "python"
-    elif first_arg.endswith(".py"):
-        cls = PythonScriptStage
-        stage["kind"] = "python-script"
-        stage["script_path"] = first_arg
-        if len(cmd) > 1:
-            stage["args"] = cmd[1:]
-        language = "python"
-    elif first_arg == "julia" and len(cmd) > 1 and cmd[1].endswith(".jl"):
-        cls = JuliaScriptStage
-        stage["kind"] = "julia-script"
-        stage["script_path"] = cmd[1]
-        if len(cmd) > 2:
-            stage["args"] = cmd[2:]
-        language = "julia"
-    elif first_arg.endswith(".jl"):
-        cls = JuliaScriptStage
-        stage["kind"] = "julia-script"
-        stage["script_path"] = first_arg
-        if len(cmd) > 1:
-            stage["args"] = cmd[1:]
-        language = "julia"
-    elif first_arg == "julia" and len(cmd) > 1:
-        cls = JuliaCommandStage
-        stage["kind"] = "julia-command"
-        stage["command"] = " ".join(cmd[1:])
-        language = "julia"
-    elif first_arg == "matlab" and len(cmd) > 1 and cmd[1].endswith(".m"):
-        cls = MatlabScriptStage
-        stage["kind"] = "matlab-script"
-        stage["script_path"] = cmd[1]
-        language = "matlab"
-    elif first_arg.endswith(".m"):
-        cls = MatlabScriptStage
-        stage["kind"] = "matlab-script"
-        stage["script_path"] = first_arg
-        language = "matlab"
-    elif first_arg == "matlab" and len(cmd) > 1:
-        cls = MatlabCommandStage
-        stage["kind"] = "matlab-command"
-        stage["command"] = " ".join(cmd[1:])
-        language = "matlab"
-    elif first_arg == "Rscript" and len(cmd) > 1 and cmd[1].endswith(".R"):
-        cls = RScriptStage
-        stage["kind"] = "r-script"
-        stage["script_path"] = cmd[1]
-        if len(cmd) > 2:
-            stage["args"] = cmd[2:]
-        language = "r"
-    elif first_arg.endswith(".R"):
-        cls = RScriptStage
-        stage["kind"] = "r-script"
-        stage["script_path"] = first_arg
-        if len(cmd) > 1:
-            stage["args"] = cmd[1:]
-        language = "r"
-    elif first_arg.endswith((".sh", ".bash", ".zsh")):
-        cls = ShellScriptStage
-        stage["kind"] = "shell-script"
-        stage["script_path"] = first_arg
-        if len(cmd) > 1:
-            stage["args"] = cmd[1:]
-        # Detect shell type from extension
-        if first_arg.endswith(".bash"):
-            stage["shell"] = "bash"
-        elif first_arg.endswith(".zsh"):
-            stage["shell"] = "zsh"
-        else:
-            stage["shell"] = "sh"
-        language = "shell"
-    else:
-        cls = ShellCommandStage
-        stage["kind"] = "shell-command"
-        stage["command"] = " ".join(shlex.quote(arg) for arg in cmd)
-        language = "shell"
-    # Create a stage name if one isn't provided and check for existing similar
-    # stages
-    if stage_name is None:
-        base_stage_name = generate_stage_name(cmd)
-        stage_name = base_stage_name
-        # If a stage with this name exists and is different, auto-increment
-        if stage_name in stages:
-            existing_stage = stages[stage_name]
-            if not stages_are_similar(existing_stage, stage):
-                # Find the next available increment
-                counter = 2
-                while f"{base_stage_name}-{counter}" in stages:
-                    candidate_stage = stages[f"{base_stage_name}-{counter}"]
-                    if stages_are_similar(candidate_stage, stage):
-                        # Found a matching stage with this incremented name
-                        stage_name = f"{base_stage_name}-{counter}"
-                        break
-                    counter += 1
-                else:
-                    # No matching stage found, use the next available number
-                    stage_name = f"{base_stage_name}-{counter}"
-                typer.echo(
-                    f"Stage '{base_stage_name}' already exists with different "
-                    f"configuration; using '{stage_name}' instead"
-                )
-    # Check if a similar stage already exists and reuse its environment if not
-    # specified
-    if stage_name in stages:
-        existing_stage = stages[stage_name]
-        if not stages_are_similar(existing_stage, stage):
-            raise_error(
-                f"A stage named '{stage_name}' already exists with "
-                "different configuration; "
-                f"Please specify a unique stage name with --stage"
-            )
-            return
-        # If no environment was specified and a similar stage exists,
-        # reuse its environment
-        if environment is None and "environment" in existing_stage:
-            environment = existing_stage["environment"]
-            typer.echo(
-                f"Reusing environment '{environment}' from existing "
-                f"stage '{stage_name}'"
-            )
-    # Detect or create environment for this stage
-    env_result = detect_env_for_stage(
-        stage=stage,
-        environment=environment,
-        ck_info=ck_info,
-        language=language,
-    )
-    # If we created an environment from dependencies, write the spec file
-    if env_result.created_from_dependencies and not dry_run:
-        typer.echo(
-            "No existing environment detected; "
-            "Attempting to create one based on detected dependencies"
-        )
-        if env_result.dependencies:
-            typer.echo(
-                f"Detected {language or 'code'} dependencies: "
-                f"{', '.join(env_result.dependencies)}"
-            )
-        if env_result.spec_path and env_result.spec_content is not None:
-            # Create the spec file
-            spec_dir = os.path.dirname(env_result.spec_path)
-            if spec_dir:
-                os.makedirs(spec_dir, exist_ok=True)
-            with open(env_result.spec_path, "w") as f:
-                f.write(env_result.spec_content)
-            typer.echo(f"Created environment spec: {env_result.spec_path}")
-    # Add environment to calkit.yaml if it doesn't exist
-    if not env_result.exists and not dry_run:
-        envs = ck_info.get("environments", {})
-        envs[env_result.name] = env_result.env
-        ck_info["environments"] = envs
-        with open("calkit.yaml", "w") as f:
-            calkit.ryaml.dump(ck_info, f)
-        ck_info = calkit.load_calkit_info()
-    env_name = env_result.name
-    stage["environment"] = env_name
-    # Detect inputs and outputs if not disabled
-    detected_inputs = []
-    detected_outputs = []
-    if not no_detect_io:
-        try:
-            io_info = detect_io(stage)
-            detected_inputs = io_info["inputs"]
-            detected_outputs = io_info["outputs"]
-        except Exception as e:
-            typer.echo(
-                f"Warning: Failed to detect inputs/outputs: {e}",
-                err=True,
-            )
-    # Initialize git repo and get DVC paths for storage determination
-    try:
-        repo = git.Repo(".")
-    except InvalidGitRepositoryError:
-        repo = None
-    dvc_paths = []
-    if repo is not None:
-        try:
-            dvc_paths = calkit.dvc.list_paths()
-        except Exception:
-            # DVC might not be initialized
-            pass
-    # Merge user-specified inputs/outputs with detected ones
-    # User-specified take precedence and detected ones are added if not already
-    # present
-    all_inputs = list(inputs)  # Start with user-specified
-    for detected in detected_inputs:
-        if detected not in all_inputs:
-            all_inputs.append(detected)
-    # Convert detected outputs to PathOutput models with storage determination
-    detected_output_models: list[str | PathOutput] = []
-    for detected in detected_outputs:
-        storage = _determine_output_storage(detected, repo, dvc_paths)
-        detected_output_models.append(
-            PathOutput(
-                path=detected,
-                storage=storage,  # type: ignore[arg-type]
-            )
-        )
-    # Merge outputs
-    all_outputs: list[str | PathOutput] = list(
-        outputs
-    )  # Start with user-specified
-    for detected in detected_output_models:
-        # Check if this output path is already present
-        detected_path = (
-            detected.path if isinstance(detected, PathOutput) else detected
-        )
-        already_present = False
-        for existing in all_outputs:
-            existing_path = (
-                existing.path if isinstance(existing, PathOutput) else existing
-            )
-            if existing_path == detected_path:
-                already_present = True
-                break
-        if not already_present:
-            all_outputs.append(detected)
-    # Add inputs and outputs to stage
-    if all_inputs:
-        stage["inputs"] = all_inputs
-    if all_outputs:
-        # Convert PathOutput objects to dicts for serialization
-        serialized_outputs = []
-        for output in all_outputs:
-            if isinstance(output, PathOutput):
-                serialized_outputs.append(
-                    output.model_dump(exclude_unset=True)
-                )
-            else:
-                serialized_outputs.append(output)
-        stage["outputs"] = serialized_outputs
-    # Create the stage, write to calkit.yaml, and run it to see if
-    # it's successful
-    try:
-        cls.model_validate(stage)
-    except Exception as e:
-        raise_error(f"Failed to create stage: {e}")
-    # If dry-run, print environment and stage then return
-    if dry_run:
-        # Print environment info
-        if env_result.created_from_dependencies:
-            typer.echo("Environment (would be created):")
-            if env_result.spec_path and env_result.spec_content is not None:
-                typer.echo(f"  Spec file: {env_result.spec_path}")
-                typer.echo("  Content:")
-                for line in env_result.spec_content.split("\n"):
-                    typer.echo(f"    {line}")
-        elif not env_result.exists:
-            typer.echo("Environment (would be added to calkit.yaml):")
-            yaml_output = io.StringIO()
-            calkit.ryaml.dump({env_result.name: env_result.env}, yaml_output)
-            for line in yaml_output.getvalue().rstrip().split("\n"):
-                typer.echo(f"  {line}")
-        else:
-            typer.echo(f"Environment: {env_name} (already exists)")
-        # Print stage
-        typer.echo("\nStage (would be added to pipeline):")
-        yaml_output = io.StringIO()
-        calkit.ryaml.dump({stage_name: stage}, yaml_output)
-        for line in yaml_output.getvalue().rstrip().split("\n"):
-            typer.echo(f"  {line}")
-        return
-    stages[stage_name] = stage
-    pipeline["stages"] = stages
-    ck_info["pipeline"] = pipeline
-    with open("calkit.yaml", "w") as f:
-        calkit.ryaml.dump(ck_info, f)
-    try:
-        # Format stage as YAML for display
-        yaml_output = io.StringIO()
-        calkit.ryaml.dump({stage_name: stage}, yaml_output)
-        # Indent YAML by 2 spaces
-        indented_yaml = "\n".join(
-            "  " + line if line.strip() else line
-            for line in yaml_output.getvalue().rstrip().split("\n")
-        )
-        typer.echo(
-            f"Adding stage to pipeline and attempting to execute:"
-            f"\n{indented_yaml}"
-        )
-        run(targets=[stage_name], force=force, verbose=verbose)
-    except Exception as e:
-        # If the stage failed, write the old ck_info back to calkit.yaml to
-        # remove the stage that we added
-        with open("calkit.yaml", "w") as f:
-            calkit.ryaml.dump(ck_info_orig, f)
-        raise_error(f"Failed to execute stage: {e}")
-
-
 @app.command(name="runproc", help="Execute a procedure (alias for 'xproc').")
 @app.command(name="xproc", help="Execute a procedure.")
 def run_procedure(
@@ -2400,7 +1908,7 @@ def run_procedure(
             typer.echo(out, nl=False)
             time.sleep(dt)
             seconds -= dt
-        typer.echo()
+        typer.echo("Done")
 
     def convert_value(value, dtype):
         if dtype == "int":