calkit-python 0.12.0__tar.gz → 0.14.0__tar.gz

{calkit_python-0.12.0 → calkit_python-0.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.12.0
+Version: 0.14.0
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://github.com/calkit/calkit
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.12.0 → calkit_python-0.14.0}/calkit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.12.0"
+__version__ = "0.14.0"
 
 from .core import *
 from . import git

{calkit_python-0.12.0 → calkit_python-0.14.0}/calkit/cli/main.py

@@ -8,6 +8,7 @@ import hashlib
 import json
 import os
 import platform as _platform
+import shlex
 import subprocess
 import sys
 import time
@@ -671,23 +672,42 @@ def run_in_env(
             subprocess.check_call(cmd, cwd=wdir)
         except subprocess.CalledProcessError:
             raise_error(f"Failed to run in {env['kind']} environment")
-    elif env["kind"] == "uv-venv":
+    elif (kind := env["kind"]) in ["uv-venv", "venv"]:
         if "prefix" not in env:
-            raise_error("uv-venv environments require a prefix")
+            raise_error("venv environments require a prefix")
         if "path" not in env:
-            raise_error("uv-venv environments require a path")
+            raise_error("venv environments require a path")
         prefix = env["prefix"]
         path = env["path"]
-        shell_cmd = " ".join(cmd)
+        # Any parts of the raw command with whitespace in them need to be
+        # quoted
+        quoted_cmd = [shlex.quote(part) for part in cmd]
+        shell_cmd = " ".join(quoted_cmd)
+        if verbose:
+            typer.echo(f"Raw command: {cmd}")
+            typer.echo(f"Quoted command: {quoted_cmd}")
+            typer.echo(f"Shell command: {shell_cmd}")
+        create_cmd = (
+            ["uv", "venv"] if kind == "uv-venv" else ["python", "-m", "venv"]
+        )
+        pip_cmd = "pip" if kind == "venv" else "uv pip"
+        pip_install_args = "-q"
+        if "python" in env and kind == "uv-venv":
+            create_cmd += ["--python", env["python"]]
+            pip_install_args += f" --python {env['python']}"
         # Check environment
         if not no_check:
             if not os.path.isdir(prefix):
                 if verbose:
-                    typer.echo(f"Creating uv-venv at {prefix}")
+                    typer.echo(f"Creating {kind} at {prefix}")
                 try:
-                    subprocess.check_call(["uv", "venv", prefix], cwd=wdir)
+                    subprocess.check_call(create_cmd + [prefix], cwd=wdir)
                 except subprocess.CalledProcessError:
-                    raise_error(f"Failed to create uv-venv at {prefix}")
+                    raise_error(f"Failed to create {kind} at {prefix}")
+                # Put a gitignore file in the env dir if one doesn't exist
+                if not os.path.isfile(os.path.join(prefix, ".gitignore")):
+                    with open(os.path.join(prefix, ".gitignore"), "w") as f:
+                        f.write("*\n")
             fname, ext = os.path.splitext(path)
             lock_fpath = fname + "-lock" + ext
             if _platform.system() == "Windows":
@@ -696,16 +716,21 @@ def run_in_env(
                 activate_cmd = f". {prefix}/bin/activate"
             check_cmd = (
                 f"{activate_cmd} "
-                f"&& uv pip install -q -r {path} "
-                f"&& uv pip freeze > {lock_fpath} "
+                f"&& {pip_cmd} install {pip_install_args} -r {path} "
+                f"&& {pip_cmd} freeze > {lock_fpath} "
                 "&& deactivate"
             )
             try:
                 if verbose:
                     typer.echo(f"Running command: {check_cmd}")
-                subprocess.check_output(check_cmd, shell=True, cwd=wdir)
+                subprocess.check_output(
+                    check_cmd,
+                    shell=True,
+                    cwd=wdir,
+                    stderr=subprocess.STDOUT if not verbose else None,
+                )
             except subprocess.CalledProcessError:
-                raise_error("Failed to check uv-venv")
+                raise_error(f"Failed to check {kind}")
         # Now run the command
         cmd = f"{activate_cmd} && {shell_cmd} && deactivate"
         if verbose:
@@ -713,7 +738,7 @@ def run_in_env(
         try:
             subprocess.check_call(cmd, shell=True, cwd=wdir)
         except subprocess.CalledProcessError:
-            raise_error("Failed to run in uv-venv")
+            raise_error(f"Failed to run in {kind}")
     else:
         raise_error("Environment kind not supported")
 

{calkit_python-0.12.0 → calkit_python-0.14.0}/calkit/cli/new.py

@@ -465,9 +465,7 @@ def new_docker_env(
     ] = "Dockerfile",
     stage: Annotated[
         str,
-        typer.Option(
-            "--stage", help="DVC pipeline stage name, deprecated."
-        ),
+        typer.Option("--stage", help="DVC pipeline stage name, deprecated."),
     ] = None,
     layers: Annotated[
         list[str],
@@ -1053,6 +1051,9 @@ def new_uv_venv(
     prefix: Annotated[
         str, typer.Option("--prefix", help="Prefix for environment location.")
     ] = ".venv",
+    python_version: Annotated[
+        str, typer.Option("--python", "-p", help="Python version.")
+    ] = None,
     description: Annotated[
         str, typer.Option("--description", help="Description.")
     ] = None,
@@ -1084,6 +1085,14 @@ def new_uv_venv(
             f"Environment with name {name} already exists "
             "(use -f to overwrite)"
         )
+    # Check prefixes
+    if not overwrite:
+        for env_name, env in envs.items():
+            if env.get("prefix") == prefix:
+                raise_error(
+                    f"Environment '{env_name}' already exists with "
+                    f"prefix '{prefix}'"
+                )
     packages_txt = "\n".join(packages)
     # Write environment to path
     with open(path, "w") as f:
@@ -1091,6 +1100,8 @@ def new_uv_venv(
     repo.git.add(path)
     typer.echo("Adding environment to calkit.yaml")
     env = dict(path=path, kind="uv-venv", prefix=prefix)
+    if python_version is not None:
+        env["python"] = python_version
     if description is not None:
         env["description"] = description
     envs[name] = env
@@ -1100,3 +1111,162 @@ def new_uv_venv(
     repo.git.add("calkit.yaml")
     if not no_commit and repo.git.diff("--staged"):
         repo.git.commit(["-m", f"Add uv venv {name}"])
+
+
+@new_app.command("venv")
+def new_venv(
+    packages: Annotated[
+        list[str],
+        typer.Argument(help="Packages to include in the environment."),
+    ],
+    name: Annotated[
+        str, typer.Option("--name", "-n", help="Environment name.")
+    ],
+    path: Annotated[
+        str, typer.Option("--path", help="Path for requirements file.")
+    ] = "requirements.txt",
+    prefix: Annotated[
+        str, typer.Option("--prefix", help="Prefix for environment location.")
+    ] = ".venv",
+    description: Annotated[
+        str, typer.Option("--description", help="Description.")
+    ] = None,
+    overwrite: Annotated[
+        bool,
+        typer.Option(
+            "--overwrite",
+            "-f",
+            help="Overwrite any existing environment with this name.",
+        ),
+    ] = False,
+    no_commit: Annotated[
+        bool, typer.Option("--no-commit", help="Do not commit changes.")
+    ] = False,
+):
+    """Create a new Python virtual environment with venv."""
+    if os.path.isfile(path) and not overwrite:
+        raise_error("Output path already exists (use -f to overwrite)")
+    repo = git.Repo()
+    # Add environment to Calkit info
+    ck_info = calkit.load_calkit_info()
+    # If environments is a list instead of a dict, reformulate it
+    envs = ck_info.get("environments", {})
+    if isinstance(envs, list):
+        typer.echo("Converting environments from list to dict")
+        envs = {env.pop("name"): env for env in envs}
+    if name in envs and not overwrite:
+        raise_error(
+            f"Environment with name {name} already exists "
+            "(use -f to overwrite)"
+        )
+    # Check prefixes
+    if not overwrite:
+        for env_name, env in envs.items():
+            if env.get("prefix") == prefix:
+                raise_error(
+                    f"Environment '{env_name}' already exists with "
+                    f"prefix '{prefix}'"
+                )
+    packages_txt = "\n".join(packages)
+    # Write environment to path
+    with open(path, "w") as f:
+        f.write(packages_txt)
+    repo.git.add(path)
+    typer.echo("Adding environment to calkit.yaml")
+    env = dict(path=path, kind="venv", prefix=prefix)
+    if description is not None:
+        env["description"] = description
+    envs[name] = env
+    ck_info["environments"] = envs
+    with open("calkit.yaml", "w") as f:
+        ryaml.dump(ck_info, f)
+    repo.git.add("calkit.yaml")
+    if not no_commit and repo.git.diff("--staged"):
+        repo.git.commit(["-m", f"Add venv {name}"])
+
+
+@new_app.command("pixi-env")
+def new_pixi_env(
+    packages: Annotated[
+        list[str],
+        typer.Argument(help="Packages to include in the environment."),
+    ],
+    name: Annotated[
+        str, typer.Option("--name", "-n", help="Environment name.")
+    ],
+    description: Annotated[
+        str, typer.Option("--description", help="Description.")
+    ] = None,
+    overwrite: Annotated[
+        bool,
+        typer.Option(
+            "--overwrite",
+            "-f",
+            help="Overwrite any existing environment with this name.",
+        ),
+    ] = False,
+    no_commit: Annotated[
+        bool, typer.Option("--no-commit", help="Do not commit changes.")
+    ] = False,
+):
+    """Create a new pixi virtual environment."""
+    repo = git.Repo()
+    # Add environment to Calkit info
+    ck_info = calkit.load_calkit_info()
+    # If environments is a list instead of a dict, reformulate it
+    envs = ck_info.get("environments", {})
+    if isinstance(envs, list):
+        typer.echo("Converting environments from list to dict")
+        envs = {env.pop("name"): env for env in envs}
+    if name in envs and not overwrite:
+        raise_error(
+            f"Environment with name {name} already exists "
+            "(use -f to overwrite)"
+        )
+    # Create the environment now
+    if not os.path.isfile("pixi.toml"):
+        subprocess.run(
+            [
+                "pixi",
+                "init",
+                ".",
+                "--format",
+                "pixi",
+                "--platform",
+                "win-64",
+                "--platform",
+                "linux-64",
+                "--platform",
+                "osx-64",
+                "--platform",
+                "osx-arm64",
+            ]
+        )
+    # Install the packages
+    for pkg in packages:
+        subprocess.run(["pixi", "add", pkg, "--feature", name])
+    # Create a pixi environment
+    subprocess.run(
+        [
+            "pixi",
+            "project",
+            "environment",
+            "add",
+            name,
+            "--feature",
+            name,
+            "--force",
+        ]
+    )
+    typer.echo("Adding environment to calkit.yaml")
+    env = dict(kind="pixi", path="pixi.toml", name=name)
+    if description is not None:
+        env["description"] = description
+    envs[name] = env
+    ck_info["environments"] = envs
+    with open("calkit.yaml", "w") as f:
+        ryaml.dump(ck_info, f)
+    repo.git.add("pixi.toml")
+    repo.git.add("calkit.yaml")
+    if not no_commit and repo.git.diff("--staged"):
+        repo.git.commit(["-m", f"Add pixi env {name}"])

{calkit_python-0.12.0 → calkit_python-0.14.0}/calkit/core.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import base64
 import glob
 import json
 import logging
@@ -10,6 +11,8 @@ import pickle
 import re
 import subprocess
 
+import requests
+
 try:
     from datetime import UTC
 except ImportError:
@@ -276,3 +279,73 @@ def check_system_deps(wdir: str | None = None) -> None:
             dep_name = re.split("[=<>]", dep)[0]
         if not check_dep_exists(dep_name):
             raise ValueError(f"{dep_name} not found")
+
+
+def project_and_path_from_path(path: str) -> tuple:
+    """Split a path into project and path, respecting the ``CALKIT_PROJECT``
+    environmental variable if set.
+
+    For example, a path like
+
+        someone/some-project:some/path/to/file.png
+
+    will return
+
+        (someone/some-project, some/path/to/file.png)
+    """
+    path_split = path.split(":")
+    if len(path_split) == 2:
+        project = path_split[0]
+        path = path_split[1]
+    elif len(path_split) == 1:
+        project = None
+    else:
+        raise ValueError("Path has too many colons in it")
+    if project is None:
+        project = os.getenv("CALKIT_PROJECT")
+    return project, path
+
+
+def read_file(path: str, as_bytes: bool = None) -> str | bytes:
+    """Read file content from path, which can optionally include a project
+    identifier, which if specified will indicate we should read from the API.
+    """
+    project, path = project_and_path_from_path(path)
+    if as_bytes is None:
+        _, ext = os.path.splitext(path)
+        as_bytes = ext in [
+            ".png",
+            ".jpg",
+            ".gif",
+            ".jpeg",
+            ".pdf",
+            ".xlsx",
+            ".docx",
+        ]
+    if project is not None:
+        import calkit.cloud
+
+        if len(project.split("/")) != 2:
+            raise ValueError("Invalid project identifier (too many slashes)")
+        resp = calkit.cloud.get(f"/projects/{project}/contents/{path}")
+        # If the response has a content key, that is a base64 encoded string
+        if (content := resp.get("content")) is not None:
+            # Load the content appropriately
+            content_bytes = base64.b64decode(content)
+            if as_bytes:
+                return content_bytes
+            else:
+                return content_bytes.decode()
+        # If the response has a URL, we can fetch from that directly
+        elif (url := resp.get("url")) is not None:
+            resp2 = requests.get(url)
+            resp2.raise_for_status()
+            if as_bytes:
+                return resp2.content
+            else:
+                return resp2.text
+        else:
+            raise ValueError("No content or URL returned from API")
+    # Project is None, so let's just read a local file
+    with open(path, mode="rb" if as_bytes else "r") as f:
+        return f.read()

calkit_python-0.14.0/calkit/datasets.py

@@ -0,0 +1,71 @@
+"""Functionality for working with datasets."""
+
+from __future__ import annotations
+
+import base64
+import io
+from typing import Literal
+
+import calkit
+import calkit.config
+
+DEFAULT_ENGINE = calkit.config.read().dataframe_engine
+
+
+def _get_df_lib(engine: str):
+    if engine == "pandas":
+        import pandas
+
+        return pandas
+    elif engine == "polars":
+        import polars
+
+        return polars
+    else:
+        raise ValueError("Unknown engine")
+
+
+def list_datasets() -> list[dict]:
+    """Read the Calkit metadata file and list out our datasets."""
+    ck_info = calkit.load_calkit_info(as_pydantic=False, process_includes=True)
+    return ck_info.get("datasets", [])
+
+
+def read_dataset(
+    path: str,
+    engine: Literal["pandas", "polars"] = DEFAULT_ENGINE,
+):
+    """Read a dataset from a path.
+
+    Path can include the project owner/name like
+
+        someone/some-project:my-data-folder/data.csv
+
+    When a project is set via the ``CALKIT_PROJECT`` environmental variable,
+    we will use the API to fetch the data.
+    """
+
+    def load_from_fobj(fobj, path: str):
+        """Read from a filelike object or path."""
+        if path.endswith(".csv"):
+            return _get_df_lib(engine).read_csv(fobj)
+        elif path.endswith(".parquet"):
+            return _get_df_lib(engine).read_parquet(fobj)
+
+    project, path = calkit.project_and_path_from_path(path)
+    if project is not None:
+        if len(project.split("/")) != 2:
+            raise ValueError("Invalid project identifier (too many slashes)")
+        resp = calkit.cloud.get(f"/projects/{project}/contents/{path}")
+        # If the response has a content key, that is a base64 encoded string
+        if (content := resp.get("content")) is not None:
+            # Load the content appropriately
+            content_bytes = base64.b64decode(content)
+            return load_from_fobj(io.BytesIO(content_bytes), path=path)
+        # If the response has a URL, we can fetch from that directly
+        elif (url := resp.get("url")) is not None:
+            return load_from_fobj(url, path=path)
+        else:
+            raise ValueError("No content or URL returned from API")
+    # Project is None, so let's just read a local file
+    return load_from_fobj(path, path)

{calkit_python-0.12.0 → calkit_python-0.14.0}/calkit/models.py

@@ -71,6 +71,7 @@ class Environment(BaseModel):
         "remote-ssh",
         "uv",
         "pixi",
+        "venv",
         "uv-venv",
         "renv",
     ]
@@ -80,11 +81,21 @@ class Environment(BaseModel):
     default: bool | None = None
 
 
+class VenvEnvironment(Environment):
+    kind: Literal["venv"]
+    prefix: str
+
+
 class UvVenvEnvironment(Environment):
     kind: Literal["uv-venv"]
     prefix: str
 
 
+class PixiEnvironment(Environment):
+    kind: Literal["pixi"]
+    name: str | None = None
+
+
 class DockerEnvironment(Environment):
     kind: Literal["docker"]
     image: str
@@ -222,7 +233,8 @@ class ProjectInfo(BaseModel):
     publications: list[Publication] = []
     references: list[ReferenceCollection] = []
     environments: dict[
-        str, Environment | DockerEnvironment | UvVenvEnvironment
+        str,
+        Environment | DockerEnvironment | VenvEnvironment | UvVenvEnvironment,
     ] = {}
     software: list[Software] = []
     notebooks: list[Notebook] = []

{calkit_python-0.12.0 → calkit_python-0.14.0}/calkit/tests/cli/test_main.py

@@ -89,23 +89,105 @@ def test_run_in_env(tmp_dir):
     ck_info = calkit.load_calkit_info()
     env = ck_info["environments"]["py3.10"]
     assert env.get("path") is None
+
+
+def test_run_in_venv(tmp_dir):
+    subprocess.check_call("git init", shell=True)
+    subprocess.check_call("dvc init", shell=True)
     # Test uv venv
     subprocess.check_call(
-        ["calkit", "new", "uv-venv", "-n", "uv1", "polars==1.18.0"]
-    )
-    out = subprocess.check_output(
         [
             "calkit",
-            "xenv",
+            "new",
+            "uv-venv",
             "-n",
             "uv1",
-            "--",
-            "python",
-            "-c",
-            "'import polars; print(polars.__version__)'",
+            "--python",
+            "3.13",
+            "--no-commit",
+            "polars==1.18.0",
         ]
-    ).decode().strip()
+    )
+    out = (
+        subprocess.check_output(
+            [
+                "calkit",
+                "xenv",
+                "-n",
+                "uv1",
+                "--",
+                "python",
+                "-c",
+                "import polars; print(polars.__version__)",
+            ]
+        )
+        .decode()
+        .strip()
+    )
     assert out == "1.18.0"
+    # Test regular venvs
+    subprocess.check_call(
+        [
+            "calkit",
+            "new",
+            "venv",
+            "-n",
+            "venv1",
+            "--prefix",
+            ".venv1",
+            "--path",
+            "reqs2.txt",
+            "polars==1.17.0",
+        ]
+    )
+    out = (
+        subprocess.check_output(
+            [
+                "calkit",
+                "xenv",
+                "-n",
+                "venv1",
+                "--",
+                "python",
+                "-c",
+                "import polars; print(polars.__version__)",
+            ]
+        )
+        .decode()
+        .strip()
+    )
+    assert out == "1.17.0"
+    # Test pixi envs
+    subprocess.check_call(
+        [
+            "calkit",
+            "new",
+            "pixi-env",
+            "-n",
+            "my-pixi",
+            "pandas=2.0.0",
+        ]
+    )
+    ck_info = calkit.load_calkit_info(as_pydantic=True)
+    envs = ck_info.environments
+    env = envs["my-pixi"]
+    out = (
+        subprocess.check_output(
+            [
+                "calkit",
+                "xenv",
+                "-n",
+                "my-pixi",
+                "--",
+                "python",
+                "-c",
+                "import pandas; print(pandas.__version__)",
+            ]
+        )
+        .decode()
+        .strip()
+    )
+    assert out == "2.0.0"
 
 
 def test_check_call():

calkit_python-0.12.0/calkit/data.py

@@ -1,60 +0,0 @@
-"""Functionality for working with datasets.
-
-Since the dependencies here are optional, we need to ensure this isn't imported
-by default, or otherwise ensure ``import calkit`` works when the data
-dependencies are not installed.
-"""
-
-from __future__ import annotations
-
-from typing import Literal, Union
-
-import pandas as pd
-import polars as pl
-
-import calkit.config
-
-DEFAULT_ENGINE = calkit.config.read().dataframe_engine
-
-
-def list_data():
-    """Read the Calkit metadata file and list out our datasets."""
-    pass
-
-
-def read_data(
-    path: str, engine: Literal["pandas", "polars"] = DEFAULT_ENGINE
-) -> Union[pd.DataFrame, pl.DataFrame]:
-    """Read (tabular) data from dataset with path ``path`` and return a
-    DataFrame.
-
-    If the dataset doesn't exist locally, but is a DVC object, download it
-    first.
-
-    If the dataset path includes a user and project name, we add it to the
-    project as an imported dataset, and therefore DVC import it?
-
-    For example: someuser/someproject:data/somefile.parquet
-
-    We can run a DVC import command if it needs to be imported. We will need to
-    find the Git repo and path within it? Maybe we should require an explicit
-    import of the data.
-    """
-    pass
-
-
-def write_data(
-    data: Union[pd.DataFrame, pl.DataFrame],
-    path: str,
-    filename: str | None = None,
-    commit=False,
-):
-    """Write ``data`` to the dataset with path ``path``.
-
-    If the dataset path is a directory, the filename must be specified.
-
-    If the path is not a Calkit dataset, it will be created.
-
-    If ``commit`` is specified, create a commit for the dataset update.
-    """
-    pass