calkit-python 0.27.3__tar.gz → 0.28.0__tar.gz

{calkit_python-0.27.3 → calkit_python-0.28.0}/.github/workflows/test.yml

@@ -23,6 +23,10 @@ jobs:
       - name: Install Pixi
         run: |
           curl -fsSL https://pixi.sh/install.sh | bash
+      - name: Setup Julia
+        uses: julia-actions/setup-julia@v2
+        with:
+          version: "1.11"
       - name: Run tests
         run: |
           source ~/.bashrc

{calkit_python-0.27.3 → calkit_python-0.28.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.27.3
+Version: 0.28.0
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues
@@ -15,7 +15,7 @@ Requires-Dist: arithmeval
 Requires-Dist: bibtexparser
 Requires-Dist: checksumdir
 Requires-Dist: docx2pdf
-Requires-Dist: dvc==3.61.0
+Requires-Dist: dvc==3.62.0
 Requires-Dist: fastapi
 Requires-Dist: gitpython
 Requires-Dist: jupyterlab>=4.4.5
@@ -31,7 +31,7 @@ Requires-Dist: python-dotenv>=1
 Requires-Dist: pywin32; platform_system == 'Windows'
 Requires-Dist: requests
 Requires-Dist: tqdm>=4.67.1
-Requires-Dist: typer
+Requires-Dist: typer==0.16.0
 Requires-Dist: uvicorn
 Provides-Extra: data
 Requires-Dist: pandas>=2.2.3; extra == 'data'
@@ -89,7 +89,10 @@ while guiding users away from common reproducibility pitfalls.
 - A CLI to run the project's pipeline to verify it's reproducible,
   regenerating outputs as needed and
   ensuring all
-  computational environments (e.g., [Conda](https://docs.conda.io/en/latest/), [Docker](https://docker.com)) match their specification.
+  computational environments
+  (e.g., [Conda](https://docs.conda.io/en/latest/),
+  [Docker](https://docker.com), uv, Julia)
+  match their specification.
 - A schema to store structured metadata describing the
   project's important outputs (in its `calkit.yaml` file)
   and how they are created

{calkit_python-0.27.3 → calkit_python-0.28.0}/README.md

@@ -49,7 +49,10 @@ while guiding users away from common reproducibility pitfalls.
 - A CLI to run the project's pipeline to verify it's reproducible,
   regenerating outputs as needed and
   ensuring all
-  computational environments (e.g., [Conda](https://docs.conda.io/en/latest/), [Docker](https://docker.com)) match their specification.
+  computational environments
+  (e.g., [Conda](https://docs.conda.io/en/latest/),
+  [Docker](https://docker.com), uv, Julia)
+  match their specification.
 - A schema to store structured metadata describing the
   project's important outputs (in its `calkit.yaml` file)
   and how they are created

{calkit_python-0.27.3 → calkit_python-0.28.0}/calkit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.27.3"
+__version__ = "0.28.0"
 
 from .core import *  # noqa: F403, I001
 from . import git  # noqa: F401

{calkit_python-0.27.3 → calkit_python-0.28.0}/calkit/cli/check.py

@@ -140,6 +140,32 @@ def check_environment(
                 env=env, env_name=env_name, as_posix=False
             ),  # type: ignore
         )
+    elif env["kind"] == "julia":
+        env_path = env.get("path")
+        if env_path is None:
+            raise_error(
+                "Julia environments require a path pointing to Project.toml"
+            )
+        env_fname = os.path.basename(env_path)
+        if not env_fname == "Project.toml":
+            raise_error(
+                "Julia environments require a path pointing to Project.toml"
+            )
+        env_dir = os.path.dirname(env_path)
+        if not env_dir:
+            env_dir = "."
+        cmd = [
+            "julia",
+            f"--project={env_dir}",
+            "-e",
+            "using Pkg; Pkg.instantiate();",
+        ]
+        try:
+            subprocess.check_call(
+                cmd, env=os.environ.copy() | {"JULIA_LOAD_PATH": "@:@stdlib"}
+            )
+        except subprocess.CalledProcessError:
+            raise_error("Failed to check julia environment")
     else:
         raise_error(f"Environment kind '{env['kind']}' not supported")
     return get_env_lock_fpath(env=env, env_name=env_name, as_posix=False)

{calkit_python-0.27.3 → calkit_python-0.28.0}/calkit/cli/main.py

@@ -38,6 +38,7 @@ from calkit.cli.check import (
     check_app,
     check_conda_env,
     check_docker_env,
+    check_environment,
     check_matlab_env,
     check_venv,
 )
@@ -1389,6 +1390,38 @@ def run_in_env(
             subprocess.check_call(cmd, shell=True, cwd=wdir)
         except subprocess.CalledProcessError:
             raise_error(f"Failed to run in {kind}")
+    elif env["kind"] == "julia":
+        if not no_check:
+            check_environment(env_name=env_name, verbose=verbose)
+        env_path = env.get("path")
+        if env_path is None:
+            raise_error(
+                "Julia environments require a path pointing to Project.toml"
+            )
+        env_fname = os.path.basename(env_path)
+        if not env_fname == "Project.toml":
+            raise_error(
+                "Julia environments require a path pointing to Project.toml"
+            )
+        env_dir = os.path.dirname(env_path)
+        if not env_dir:
+            env_dir = "."
+        julia_cmd = [
+            "julia",
+            "--project=" + env_dir,
+            "-e",
+            " ".join(cmd),
+        ]
+        if verbose:
+            typer.echo(f"Running command: {julia_cmd}")
+        try:
+            subprocess.check_call(
+                julia_cmd,
+                cwd=wdir,
+                env=os.environ.copy() | {"JULIA_LOAD_PATH": "@:@stdlib"},
+            )
+        except subprocess.CalledProcessError:
+            raise_error("Failed to run in julia environment")
     elif env["kind"] == "ssh":
         try:
             host = os.path.expandvars(env["host"])

{calkit_python-0.27.3 → calkit_python-0.28.0}/calkit/cli/new.py

@@ -588,7 +588,8 @@ def new_docker_env(
     layers: Annotated[
         list[str],
         typer.Option(
-            "--add-layer", help="Add a layer (options: miniforge, foampy)."
+            "--add-layer",
+            help="Add a layer (options: miniforge, foampy, uv, julia).",
         ),
     ] = [],
     env_vars: Annotated[
@@ -703,11 +704,11 @@ def new_docker_env(
         wdir=wdir,
     )
     if base is not None or path is not None:
-        env["path"] = path
+        env["path"] = path  # type: ignore
     if description is not None:
         env["description"] = description
     if layers:
-        env["layers"] = layers
+        env["layers"] = layers  # type: ignore
     if platform:
         env["platform"] = platform
     if user:
@@ -715,18 +716,18 @@ def new_docker_env(
     if gpus:
         env["gpus"] = gpus
     if env_vars:
-        env["env_vars"] = {}
+        env["env_vars"] = {}  # type: ignore
         for var in env_vars:
             if "=" not in var:
                 raise_error(f"Invalid environment variable format: {var}")
             key, value = var.split("=", 1)
             env["env_vars"][key] = value
     if args:
-        env["args"] = args
+        env["args"] = args  # type: ignore
     if deps:
-        env["deps"] = deps
+        env["deps"] = deps  # type: ignore
     if ports:
-        env["ports"] = ports
+        env["ports"] = ports  # type: ignore
     envs[name] = env
     ck_info["environments"] = envs
     with open("calkit.yaml", "w") as f:
@@ -799,14 +800,14 @@ def new_dataset(
     title: Annotated[str, typer.Option("--title")],
     description: Annotated[str, typer.Option("--description")],
     stage_name: Annotated[
-        str,
+        str | None,
         typer.Option(
             "--stage",
             help="Name of the pipeline stage that generates this dataset.",
         ),
     ] = None,
     cmd: Annotated[
-        str,
+        str | None,
         typer.Option(
             "--cmd", help="Command to add to the stage, if specified."
         ),
@@ -825,7 +826,7 @@ def new_dataset(
         ),
     ] = [],
     outs_from_stage: Annotated[
-        str,
+        str | None,
         typer.Option(
             "--deps-from-stage-outs",
             help="Stage name from which to add outputs as dependencies.",
@@ -1066,7 +1067,7 @@ def new_publication(
         calkit.ryaml.dump(ck_info, f)
     repo.git.add("calkit.yaml")
     # Copy in template files if applicable
-    if template_type == "latex":
+    if template is not None and template_type == "latex":
         if overwrite and os.path.exists(path):
             shutil.rmtree(path)
         calkit.templates.use_template(
@@ -1309,7 +1310,7 @@ def new_venv(
         str, typer.Option("--prefix", help="Prefix for environment location.")
     ] = ".venv",
     description: Annotated[
-        str, typer.Option("--description", help="Description.")
+        str | None, typer.Option("--description", help="Description.")
     ] = None,
     overwrite: Annotated[
         bool,
@@ -1390,7 +1391,7 @@ def new_pixi_env(
         list[str], typer.Option("--pip", help="Packages to install with pip.")
     ] = [],
     description: Annotated[
-        str, typer.Option("--description", help="Description.")
+        str | None, typer.Option("--description", help="Description.")
     ] = None,
     platforms: Annotated[
         list[str], typer.Option("--platform", "-p", help="Platform.")
@@ -1479,6 +1480,86 @@ def new_pixi_env(
         repo.git.commit(["-m", f"Add pixi env {name}"])
 
 
+@new_app.command("julia-env")
+def new_julia_env(
+    packages: Annotated[
+        list[str],
+        typer.Argument(help="Packages to include in the environment."),
+    ],
+    name: Annotated[
+        str, typer.Option("--name", "-n", help="Environment name.")
+    ],
+    path: Annotated[
+        str, typer.Option("--path", help="Path for Project.toml file.")
+    ] = "Project.toml",
+    description: Annotated[
+        str | None, typer.Option("--description", help="Description.")
+    ] = None,
+    overwrite: Annotated[
+        bool,
+        typer.Option(
+            "--overwrite",
+            "-f",
+            help="Overwrite any existing environment with this name.",
+        ),
+    ] = False,
+    no_commit: Annotated[
+        bool, typer.Option("--no-commit", help="Do not commit changes.")
+    ] = False,
+):
+    """Create a new Julia environment."""
+    if not os.path.basename(path) == "Project.toml":
+        raise_error(
+            "Julia environment paths must point to a Project.toml file"
+        )
+    try:
+        repo = git.Repo()
+    except git.InvalidGitRepositoryError:
+        raise_error(
+            "Current directory is not a Git repository; run calkit init"
+        )
+    # Add environment to Calkit info
+    ck_info = calkit.load_calkit_info()
+    # If environments is a list instead of a dict, reformulate it
+    envs = ck_info.get("environments", {})
+    if isinstance(envs, list):
+        typer.echo("Converting environments from list to dict")
+        envs = {env.pop("name"): env for env in envs}
+    if name in envs and not overwrite:
+        raise_error(
+            f"Environment with name {name} already exists "
+            "(use -f to overwrite)"
+        )
+    # Create the environment now
+    env_dir = os.path.dirname(path)
+    if env_dir:
+        os.makedirs(env_dir, exist_ok=True)
+    else:
+        env_dir = "."
+    cmd = ["julia", f"--project={env_dir}", "-e"]
+    install_cmd = "using Pkg;"
+    for package in packages:
+        install_cmd += f' Pkg.add("{package}");'
+    cmd.append(install_cmd)
+    try:
+        subprocess.run(cmd, check=True)
+    except subprocess.CalledProcessError:
+        raise_error("Failed to create new Julia environment")
+    typer.echo("Adding environment to calkit.yaml")
+    env = dict(kind="julia", path=path, name=name)
+    if description is not None:
+        env["description"] = description
+    envs[name] = env
+    ck_info["environments"] = envs
+    with open("calkit.yaml", "w") as f:
+        ryaml.dump(ck_info, f)
+    repo.git.add(path)
+    repo.git.add(os.path.join(env_dir, "Manifest.toml"))
+    repo.git.add("calkit.yaml")
+    if not no_commit and repo.git.diff("--staged"):
+        repo.git.commit(["-m", f"Add Julia env {name}"])
+
+
 class Status(str, Enum):
     in_progress = "in-progress"
     on_hold = "on-hold"
@@ -1786,6 +1867,66 @@ def new_python_script_stage(
     )
 
 
+@new_app.command(name="julia-script-stage")
+def new_julia_script_stage(
+    name: StageArgs.name,
+    environment: StageArgs.environment,
+    script_path: StageArgs.script_path,
+    inputs: StageArgs.inputs = [],
+    outputs: StageArgs.outputs = [],
+    outs_git: StageArgs.outs_git = [],
+    outs_git_no_delete: StageArgs.outs_git_no_delete = [],
+    outs_no_delete: StageArgs.outs_no_delete = [],
+    outs_no_store: StageArgs.outs_no_store = [],
+    outs_no_store_no_delete: StageArgs.outs_no_store_no_delete = [],
+    iter_arg: Annotated[
+        tuple[str, str] | None,
+        typer.Option(
+            "--iter",
+            help=(
+                "Iterate over an argument with a comma-separated list, e.g., "
+                "--iter-arg var_name val1,val2,val3."
+            ),
+        ),
+    ] = None,
+    overwrite: StageArgs.overwrite = False,
+    no_check: StageArgs.no_check = False,
+    no_commit: StageArgs.no_commit = False,
+) -> None:
+    """Add a stage to the pipeline that runs a Julia script."""
+    ck_outs = _to_ck_outs(
+        outputs=outputs,
+        outs_git=outs_git,
+        outs_git_no_delete=outs_git_no_delete,
+        outs_no_delete=outs_no_delete,
+        outs_no_store=outs_no_store,
+        outs_no_store_no_delete=outs_no_store_no_delete,
+    )
+    try:
+        if iter_arg is not None:
+            arg_name, vals = iter_arg
+            i = [StageIteration(arg_name=arg_name, values=vals.split(","))]  # type: ignore
+        else:
+            i = None
+        stage = calkit.models.pipeline.JuliaScriptStage(
+            kind="julia-script",
+            environment=environment,
+            inputs=inputs,  # type: ignore
+            outputs=ck_outs,
+            script_path=script_path,
+            iterate_over=i,
+        )
+    except Exception as e:
+        raise_error(f"Invalid stage specification: {e}")
+    _save_stage(
+        stage=stage,
+        name=name,
+        overwrite=overwrite,
+        no_check=no_check,
+        no_commit=no_commit,
+    )
+
+
 @new_app.command(name="matlab-script-stage")
 def new_matlab_script_stage(
     name: StageArgs.name,

{calkit_python-0.27.3 → calkit_python-0.28.0}/calkit/core.py

@@ -309,6 +309,8 @@ def check_system_deps(
             if "docker" not in deps:
                 deps.append("docker")
             deps.append({"MATLAB_LICENSE_SERVER": {"kind": "env-var"}})
+        elif kind == "julia" and "julia" not in deps:
+            deps.append("julia")
     for dep in deps:
         if isinstance(dep, dict):
             keys = list(dep.keys())
@@ -488,7 +490,7 @@ def detect_project_name(
     if name is not None and not prepend_owner:
         return name
     owner = ck_info.get("owner")
-    if name is None and owner is None:
+    if name is None or owner is None:
         try:
             url = Repo(path=wdir).remote().url
         except ValueError:
@@ -560,6 +562,7 @@ def get_system_info() -> dict:
         "uv",
         "pixi",
         "Rscript",
+        "julia",
     ]:
         system_info[f"{dep}_version"] = get_dep_version(dep)
     system_info_str = json.dumps(system_info, sort_keys=True).encode()

{calkit_python-0.27.3 → calkit_python-0.28.0}/calkit/docker.py

@@ -47,7 +47,17 @@ RUN pip install --no-cache-dir numpy pandas matplotlib h5py \
 """.strip()
 
 UV_LAYER_TEXT = """
-COPY --from=ghcr.io/astral-sh/uv:0.6.14 /uv /uvx /bin/
+COPY --from=ghcr.io/astral-sh/uv:0.8.5 /uv /uvx /bin/
+"""
+
+JULIA_LAYER_TEXT = """
+# Install Julia
+# Ensure base image is a bullseye distribution
+COPY --from=julia:1.11.6-bullseye /usr/local/julia /usr/local/julia
+ENV JULIA_PATH=/usr/local/julia \
+    PATH=$PATH:/usr/local/julia/bin \
+    JULIA_GPG=3673DF529D9049477F76B37566E3C7DC03D6E495 \
+    JULIA_VERSION=1.11.6
 """
 
 LAYERS = {
@@ -55,4 +65,5 @@ LAYERS = {
     "miniforge": MINIFORGE_LAYER_TXT,
     "foampy": FOAMPY_LAYER_TEXT,
     "uv": UV_LAYER_TEXT,
+    "julia": JULIA_LAYER_TEXT,
 }

{calkit_python-0.27.3 → calkit_python-0.28.0}/calkit/environments.py

@@ -28,6 +28,20 @@ def get_env_lock_fpath(
         lock_fpath += ".yml"
     elif env_kind == "matlab":
         lock_fpath += ".json"
+    elif env_kind == "julia":
+        env_path = env.get("path")
+        if env_path is None:
+            raise ValueError(
+                "Julia environments require a path pointing to Project.toml"
+            )
+        env_fname = os.path.basename(env_path)
+        if not env_fname == "Project.toml":
+            raise ValueError(
+                "Julia environments require a path pointing to Project.toml"
+            )
+        # Simply replace Project.toml with Manifest.toml
+        env_dir = os.path.dirname(env_path)
+        lock_fpath = os.path.join(env_dir, "Manifest.toml")
     else:
         return
     if as_posix:

{calkit_python-0.27.3 → calkit_python-0.28.0}/calkit/models/pipeline.py

@@ -109,6 +109,8 @@ class Stage(BaseModel):
         "shell-script",
         "jupyter-notebook",
         "r-script",
+        "julia-script",
+        "word-to-pdf",
     ]
     environment: str
     wdir: str | None = None
@@ -309,6 +311,20 @@ class RScriptStage(Stage):
         return cmd
 
 
+class JuliaScriptStage(Stage):
+    kind: Literal["julia-script"] = "julia-script"
+    script_path: str
+
+    @property
+    def dvc_cmd(self) -> str:
+        cmd = f'{self.xenv_cmd} -- "include(\\"{self.script_path}\\")"'
+        return cmd
+
+    @property
+    def dvc_deps(self) -> list[str]:
+        return [self.script_path] + super().dvc_deps
+
+
 class JupyterNotebookStage(Stage):
     """A stage that runs a Jupyter notebook.
 
@@ -466,6 +482,7 @@ class Pipeline(BaseModel):
                 | RScriptStage
                 | WordToPdfStage
                 | JupyterNotebookStage
+                | JuliaScriptStage
             ),
             Discriminator("kind"),
         ],

{calkit_python-0.27.3 → calkit_python-0.28.0}/calkit/tests/cli/test_main.py

@@ -234,6 +234,41 @@ def test_run_in_venv(tmp_dir):
     assert out == "2.0.0"
 
 
+def test_run_in_julia_env(tmp_dir):
+    subprocess.check_call("calkit init", shell=True)
+    subprocess.check_call(
+        [
+            "calkit",
+            "new",
+            "julia-env",
+            "-n",
+            "my-julia",
+            "--no-commit",
+            "Revise",
+            "PkgVersion",
+        ]
+    )
+    out = (
+        subprocess.check_output(
+            [
+                "calkit",
+                "xenv",
+                "-n",
+                "my-julia",
+                "--",
+                (
+                    "using Revise; using PkgVersion; "
+                    "println(PkgVersion.Version(Revise))"
+                ),
+            ]
+        )
+        .decode()
+        .strip()
+    )
+    # TODO: Allow specifying version
+    assert out
+
+
 def test_to_shell_cmd():
     cmd = ["python", "-c", "import math; print('hello world')"]
     subprocess.check_call(cmd)
@@ -420,6 +455,35 @@ def test_run(tmp_dir):
     repo = git.Repo()
     repo.git.checkout("HEAD^")
     out = subprocess.check_output(["calkit", "run"], text=True)
+    # Test that we can run a Julia script
+    with open("julia_script.jl", "w") as f:
+        f.write('println("Hello from julia_script.jl")')
+    subprocess.check_call(
+        [
+            "calkit",
+            "new",
+            "julia-env",
+            "--name",
+            "j1",
+            "--path",
+            "something/Project.toml",
+            "PkgVersion",
+        ]
+    )
+    subprocess.check_call(
+        [
+            "calkit",
+            "new",
+            "julia-script-stage",
+            "--name",
+            "stage-2",
+            "--script-path",
+            "julia_script.jl",
+            "-e",
+            "j1",
+        ]
+    )
+    subprocess.check_call(["calkit", "run"])
 
 
 def test_stage_run_info_from_log_content():

{calkit_python-0.27.3 → calkit_python-0.28.0}/calkit/tests/cli/test_new.py

@@ -633,3 +633,26 @@ def test_new_matlab_script_stage(tmp_dir):
         "results/output.txt",
         "results/output2.txt",
     ]
+
+
+def test_new_julia_env(tmp_dir):
+    subprocess.check_call(["calkit", "init"])
+    subprocess.check_call(
+        ["calkit", "new", "julia-env", "--name", "j1", "WaterLily"]
+    )
+    assert os.path.isfile("Project.toml")
+    assert os.path.isfile("Manifest.toml")
+    subprocess.check_call(
+        [
+            "calkit",
+            "new",
+            "julia-env",
+            "--name",
+            "j2",
+            "Revise",
+            "--path",
+            "envs/my-env/Project.toml",
+        ]
+    )
+    assert os.path.isfile("envs/my-env/Project.toml")
+    assert os.path.isfile("envs/my-env/Manifest.toml")

{calkit_python-0.27.3 → calkit_python-0.28.0}/docs/environments.md

@@ -269,6 +269,28 @@ Again this highlights Calkit's declarative design philosophy.
 Declare the environment and what command should be executed inside,
 and Calkit will handle the rest.
 
+### Julia
+
+[Julia](https://julialang.org/) environments have paths that point to a
+`Project.toml` file.
+Creating a new Julia environment is similar to creating a Python environment:
+
+```sh
+calkit new julia-env \
+    --name my-julia-env \
+    --path ./envs/my-julia-env/Project.toml \
+    WaterLily \
+    Makie
+```
+
+Unlike Python environments, for which `xenv` runs a shell command,
+`xenv` runs Julia commands in a Julia environment.
+For example:
+
+```sh
+calkit xenv -n my-julia-env -- "println(\"hello world\");"
+```
+
 ### SSH
 
 It's possible to define a remote environment that uses `ssh` to connect

{calkit_python-0.27.3 → calkit_python-0.28.0}/docs/index.md

@@ -32,7 +32,10 @@ while guiding users away from common reproducibility pitfalls.
 - A CLI to run the project's pipeline to verify it's reproducible,
   regenerating outputs as needed and
   ensuring all
-  computational environments (e.g., [Conda](https://docs.conda.io/en/latest/), [Docker](https://docker.com)) match their specification.
+  computational environments
+  (e.g., [Conda](https://docs.conda.io/en/latest/),
+  [Docker](https://docker.com), uv, Julia)
+  match their specification.
 - A schema to store structured metadata describing the
   project's important outputs (in its `calkit.yaml` file)
   and how they are created

{calkit_python-0.27.3 → calkit_python-0.28.0}/docs/pipeline/index.md

@@ -82,6 +82,10 @@ For more details, see `calkit.models.pipeline`.
 - `script_path`
 - `args` (list, optional)
 
+### `julia-script`
+
+- `script_path`
+
 ## Iteration
 
 ### Over a list of values