calkit-python 0.21.3__tar.gz → 0.21.4__tar.gz

{calkit_python-0.21.3 → calkit_python-0.21.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.21.3
+Version: 0.21.4
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.21.3 → calkit_python-0.21.4}/calkit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.21.3"
+__version__ = "0.21.4"
 
 from .core import *
 from . import git

{calkit_python-0.21.3 → calkit_python-0.21.4}/calkit/check.py

@@ -180,7 +180,7 @@ def check_reproducibility(
     if log_func is None:
         log_func = print
     try:
-        repo = git.Repo(wdir)
+        git.Repo(wdir)
         res["is_git_repo"] = True
     except InvalidGitRepositoryError:
         res["is_git_repo"] = False

{calkit_python-0.21.3 → calkit_python-0.21.4}/calkit/cli/main.py

@@ -905,7 +905,7 @@ def run_in_env(
                 fpath=env["path"],
                 platform=env.get("platform"),
                 deps=env.get("deps", []),
-                quiet=True,
+                quiet=not verbose,
             )
         shell_cmd = _to_shell_cmd(cmd)
         docker_cmd = [
@@ -949,9 +949,18 @@ def run_in_env(
             conda_env = calkit.ryaml.load(f)
         if not no_check:
             check_conda_env(
-                env_fpath=env["path"], relaxed=relaxed_check, quiet=True
+                env_fpath=env["path"],
+                relaxed=relaxed_check,
+                quiet=not verbose,
             )
-        cmd = ["conda", "run", "-n", conda_env["name"]] + cmd
+        # TODO: Prefix should only be in the env file or calkit.yaml, not both?
+        prefix = env.get("prefix")
+        conda_cmd = ["conda", "run"]
+        if prefix is not None:
+            conda_cmd += ["--prefix", os.path.abspath(prefix)]
+        else:
+            conda_cmd += ["-n", conda_env["name"]]
+        cmd = conda_cmd + cmd
         if verbose:
             typer.echo(f"Running command: {cmd}")
         try:

{calkit_python-0.21.3 → calkit_python-0.21.4}/calkit/cli/new.py

@@ -963,6 +963,9 @@ def new_conda_env(
     pip_packages: Annotated[
         list[str], typer.Option("--pip", help="Packages to install with pip.")
     ] = [],
+    prefix: Annotated[
+        str, typer.Option("--prefix", help="Prefix for environment location.")
+    ] = None,
     description: Annotated[
         str, typer.Option("--description", help="Description.")
     ] = None,
@@ -1003,6 +1006,12 @@ def new_conda_env(
     conda_env = dict(
         name=conda_name, channels=["conda-forge"], dependencies=packages
     )
+    if prefix is not None:
+        from calkit.cli.main import ignore
+
+        conda_env["prefix"] = prefix
+        ignore(prefix, no_commit=True)
+        repo.git.add(".gitignore")
     if pip_packages:
         conda_env["dependencies"].append(dict(pip=pip_packages))
     with open(path, "w") as f:
@@ -1010,6 +1019,8 @@ def new_conda_env(
     repo.git.add(path)
     typer.echo("Adding environment to calkit.yaml")
     env = dict(path=path, kind="conda")
+    if prefix is not None:
+        env["prefix"] = prefix
     if description is not None:
         env["description"] = description
     envs[name] = env

{calkit_python-0.21.3 → calkit_python-0.21.4}/calkit/conda.py

@@ -75,35 +75,58 @@ def check_env(
     log_func(f"Checking conda env defined in {env_fpath}")
     res = EnvCheckResult()
     # Use mamba here because it's faster and produces less output
+    info = json.loads(subprocess.check_output(["conda", "info", "--json"]))
+    root_prefix = info["root_prefix"]
+    envs_dir = os.path.join(root_prefix, "envs")
     envs = json.loads(
         subprocess.check_output(["mamba", "env", "list", "--json"]).decode()
     )["envs"]
-    # Get existing env names, but skip the base environment
-    # Note that this could fail for environments with non-default prefixes
+    # Get existing env names for those in the envs directory
     existing_env_names = [
-        os.path.basename(env) for env in envs if "envs" in env
+        os.path.basename(env) for env in envs if env.startswith(envs_dir)
     ]
+    # Get a list of environments defined by prefix instead of name
+    env_prefixes = [e for e in envs if not e.startswith(root_prefix)]
     with open(env_fpath) as f:
         env_spec = ryaml.load(f)
     env_name = env_spec["name"]
-    env_check_fpath = os.path.join(
-        os.path.expanduser("~"),
-        ".calkit",
-        "conda-env-checks",
-        env_name + ".yml",
-    )
+    prefix = env_spec.get("prefix")
+    prefix_orig = prefix
+    if prefix is not None:
+        prefix = os.path.abspath(prefix)
+        env_check_fpath = os.path.join(prefix, "env-export.yml")
+    else:
+        env_check_fpath = os.path.join(
+            os.path.expanduser("~"),
+            ".calkit",
+            "conda-env-checks",
+            env_name + ".yml",
+        )
     env_check_dir = os.path.dirname(env_check_fpath)
     os.makedirs(env_check_dir, exist_ok=True)
+    # Create env export command, which will be used later
+    export_cmd = [
+        "conda",  # Mamba output is slightly different
+        "env",
+        "export",
+        "--no-builds",
+        "--json",
+    ]
+    # Create with conda since newer mamba versions create a strange
+    # "Library" subdirectory, at least on Windows
+    create_cmd = ["conda", "env", "create", "-y", "-f", env_fpath]
+    if prefix is not None:
+        export_cmd += ["--prefix", prefix]
+        create_cmd += ["--prefix", prefix]
+    else:
+        export_cmd += ["-n", env_name]
     # Check if env even exists
-    if env_name not in existing_env_names:
+    # If env has a prefix defined, it will be identified by that
+    if env_name not in existing_env_names and prefix not in env_prefixes:
         log_func(f"Environment {env_name} doesn't exist; creating")
         res.env_exists = False
         # Environment doesn't exist, so create it
-        # Create with conda since newer mamba versions create a strange
-        # "Library" subdirectory, at least on Windows
-        subprocess.check_call(
-            ["conda", "env", "create", "-y", "-f", env_fpath]
-        )
+        subprocess.check_call(create_cmd)
         env_needs_rebuild = False
         env_needs_export = True
     else:
@@ -111,6 +134,7 @@ def check_env(
         env_needs_export = False
         # Environment does exist, so check it
         if os.path.isfile(env_check_fpath):
+            log_func(f"Found env check file at {env_check_fpath}")
             # Open up the env check result file
             with open(env_check_fpath) as f:
                 env_check = ryaml.load(f)
@@ -122,24 +146,16 @@ def check_env(
             current_mtime = os.path.getmtime(
                 os.path.normpath(env_check["prefix"])
             )
+            log_func(f"Env check mtime: {existing_mtime}")
+            log_func(f"Env dir mtime: {current_mtime}")
             env_needs_export = existing_mtime != current_mtime
         else:
+            log_func(f"Env check file at {env_check_fpath} does not exist")
             env_needs_export = True
         if env_needs_export:
-            res.env_needs_export = True
             log_func(f"Exporting existing env to {env_check_fpath}")
             env_check = json.loads(
-                subprocess.check_output(
-                    [
-                        "conda",  # Mamba output is slightly different
-                        "env",
-                        "export",
-                        "-n",
-                        env_name,
-                        "--no-builds",
-                        "--json",
-                    ]
-                ).decode()
+                subprocess.check_output(export_cmd).decode()
             )
             env_check["mtime"] = os.path.getmtime(
                 os.path.normpath(env_check["prefix"])
@@ -188,29 +204,16 @@ def check_env(
     if env_needs_rebuild:
         res.env_needs_rebuild = True
         log_func(f"Rebuilding {env_name} since it does not match spec")
-        subprocess.check_call(
-            ["conda", "env", "create", "-y", "-f", env_fpath]
-        )
+        subprocess.check_call(create_cmd)
         env_needs_export = True
     else:
         log_func(f"Environment {env_name} matches spec")
         res.env_needs_rebuild = False
     # If the env was rebuilt, export the env check
+    res.env_needs_export = env_needs_export
     if env_needs_export:
         log_func(f"Exporting existing env to {env_check_fpath}")
-        env_check = json.loads(
-            subprocess.check_output(
-                [
-                    "conda",  # Mamba output is slightly different
-                    "env",
-                    "export",
-                    "-n",
-                    env_name,
-                    "--no-builds",
-                    "--json",
-                ]
-            ).decode()
-        )
+        env_check = json.loads(subprocess.check_output(export_cmd).decode())
         env_check["mtime"] = os.path.getmtime(
             os.path.normpath(env_check["prefix"])
         )
@@ -225,8 +228,17 @@ def check_env(
         or not os.path.isfile(output_fpath)
     ):
         log_func(f"Exporting lock file to {output_fpath}")
+        env_export = json.loads(
+            subprocess.check_output(
+                [a for a in export_cmd if a != "--no-builds"]
+            ).decode()
+        )
+        # Remove prefix from env export since it will be an absolute path
+        _ = env_export.pop("prefix")
+        # Remove name if prefix is set, since that will be the prefix
+        if prefix is not None:
+            _ = env_export.pop("name")
+            env_export["prefix"] = prefix_orig
         with open(output_fpath, "w") as f:
-            _ = env_check.pop("mtime")
-            _ = env_check.pop("prefix")
-            ryaml.dump(env_check, f)
+            ryaml.dump(env_export, f)
     return res

{calkit_python-0.21.3 → calkit_python-0.21.4}/calkit/tests/test_conda.py

@@ -80,10 +80,11 @@ def test_check_env(tmp_dir, conda_env_name):
     )
     res = check_env()
     assert res.env_exists
-    assert not res.env_needs_export
+    assert res.env_needs_export
     assert res.env_needs_rebuild
     res = check_env()
     assert not res.env_needs_rebuild
+    assert not res.env_needs_export
     # Check relaxed mode, where we allow dependencies to be in either the pip
     # or conda section
     subprocess.check_call(
@@ -163,3 +164,82 @@ def test_check_env(tmp_dir, conda_env_name):
     res = check_env()
     assert not res.env_needs_export
     assert not res.env_needs_rebuild
+
+
+@pytest.fixture
+def conda_env_prefix():
+    prefix = ".conda-envs/my-conda-env"
+    yield prefix
+    subprocess.check_call(["conda", "env", "remove", "-y", "--prefix", prefix])
+
+
+def test_check_prefix_env(tmp_dir, conda_env_prefix):
+    subprocess.check_call(["calkit", "init"])
+    # Test we can use a local prefix
+    subprocess.check_call(
+        [
+            "calkit",
+            "new",
+            "conda-env",
+            "-n",
+            "my-conda-env",
+            "python=3.12",
+            "--prefix",
+            conda_env_prefix,
+        ]
+    )
+    res = check_env()
+    assert not res.env_exists
+    assert res.env_needs_export
+    assert os.path.isfile(os.path.join(conda_env_prefix, "env-export.yml"))
+    res = check_env()
+    assert res.env_exists
+    # Env will need to be exported a second time since we save it inside the
+    # prefix folder, so the mtime will be slightly after creation of the
+    # initial environment
+    assert res.env_needs_export
+    assert not res.env_needs_rebuild
+    # Now the env should be exported okay
+    res = check_env()
+    assert res.env_exists
+    assert not res.env_needs_export
+    assert not res.env_needs_rebuild
+    subprocess.check_call(
+        ["calkit", "xenv", "-n", "my-conda-env", "python", "--version"]
+    )
+    # Test that we can add a new dependency
+    with open("environment.yml") as f:
+        env = calkit.ryaml.load(f)
+    env["dependencies"].append("requests")
+    with open("environment.yml", "w") as f:
+        calkit.ryaml.dump(env, f)
+    res = check_env()
+    assert res.env_exists
+    assert res.env_needs_export
+    assert res.env_needs_rebuild
+    subprocess.check_call(
+        [
+            "calkit",
+            "xenv",
+            "-n",
+            "my-conda-env",
+            "python",
+            "-c",
+            "import requests",
+        ]
+    )
+    # Test that we can specify --wdir
+    os.makedirs("subdir")
+    subprocess.check_call(
+        [
+            "calkit",
+            "xenv",
+            "--wdir",
+            "subdir",
+            "-n",
+            "my-conda-env",
+            "python",
+            "-c",
+            "import requests",
+        ]
+    )

{calkit_python-0.21.3 → calkit_python-0.21.4}/docs/environments.md

@@ -236,6 +236,13 @@ The new Conda environment spec will be written to `environment.yml`
 by default,
 which can be controlled with the `--path` option.
 
+A prefix for the environment can be specified to keep all packages under the
+project directory, e.g., by adding `--prefix .conda-envs/my-conda-env`.
+If this option is omitted, the environment will become part of Conda's
+system-wide collection of environments with a name like
+`{project_name}-{env_name}`,
+where the project name is added to avoid conflicts.
+
 Similar to other environment types,
 any time a command is executed with `calkit xenv`,
 this environment will be checked and created or updated as necessary.