calkit-python 0.8.2__tar.gz → 0.8.4__tar.gz

{calkit_python-0.8.2 → calkit_python-0.8.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: calkit-python
-Version: 0.8.2
+Version: 0.8.4
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://github.com/calkit/calkit
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.8.2 → calkit_python-0.8.4}/calkit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.8.2"
+__version__ = "0.8.4"
 
 from .core import *
 from . import git

{calkit_python-0.8.2 → calkit_python-0.8.4}/calkit/cli/main.py

@@ -560,14 +560,14 @@ def run_in_env(
         ]
         if verbose:
             typer.echo(f"Running command: {docker_cmd}")
-        subprocess.call(docker_cmd, cwd=wdir)
+        subprocess.check_call(docker_cmd, cwd=wdir)
     elif env["kind"] == "conda":
         with open(env["path"]) as f:
             conda_env = calkit.ryaml.load(f)
         cmd = ["conda", "run", "-n", conda_env["name"]] + cmd
         if verbose:
             typer.echo(f"Running command: {cmd}")
-        subprocess.call(cmd, cwd=wdir)
+        subprocess.check_call(cmd, cwd=wdir)
     else:
         raise_error("Environment kind not supported")
 

{calkit_python-0.8.2 → calkit_python-0.8.4}/calkit/conda.py

@@ -21,14 +21,15 @@ def check_env(
     Note that this only works with exact or no version specification.
     Using greater than and less than operators is not supported.
     """
+    conda = "mamba" if use_mamba else "conda"
     if log_func is None:
         log_func = calkit.logger.info
     log_func(f"Checking conda env defined in {env_fpath}")
     envs = json.loads(
-        subprocess.check_output(["conda", "env", "list", "--json"]).decode()
+        subprocess.check_output([conda, "env", "list", "--json"]).decode()
     )["envs"]
     # Get existing env names, but skip the base environment
-    existing_env_names = [env.split("/")[-1] for env in envs[1:]]
+    existing_env_names = [os.path.basename(env) for env in envs[1:]]
     with open(env_fpath) as f:
         env_spec = ryaml.load(f)
     env_name = env_spec["name"]
@@ -40,7 +41,6 @@ def check_env(
     )
     env_check_dir = os.path.dirname(env_check_fpath)
     os.makedirs(env_check_dir, exist_ok=True)
-    conda = "mamba" if use_mamba else "conda"
     # Check if env even exists
     if env_name not in existing_env_names:
         log_func(f"Environment {env_name} doesn't exist; creating")
@@ -60,7 +60,9 @@ def check_env(
             # If they match, the environment saved in env_check is still
             # valid, so we don't need to re-export
             existing_mtime = env_check["mtime"]
-            current_mtime = os.path.getmtime(env_check["prefix"])
+            current_mtime = os.path.getmtime(
+                os.path.normpath(env_check["prefix"])
+            )
             env_needs_export = existing_mtime != current_mtime
         else:
             env_needs_export = True
@@ -69,7 +71,7 @@ def check_env(
             env_check = json.loads(
                 subprocess.check_output(
                     [
-                        "conda",
+                        "conda",  # Mamba output is slightly different
                         "env",
                         "export",
                         "-n",
@@ -79,7 +81,9 @@ def check_env(
                     ]
                 ).decode()
             )
-            env_check["mtime"] = os.path.getmtime(env_check["prefix"])
+            env_check["mtime"] = os.path.getmtime(
+                os.path.normpath(env_check["prefix"])
+            )
             with open(env_check_fpath, "w") as f:
                 ryaml.dump(env_check, f)
         # Determine if the env matches
@@ -149,7 +153,7 @@ def check_env(
         env_check = json.loads(
             subprocess.check_output(
                 [
-                    "conda",
+                    "conda",  # Mamba output is slightly different
                     "env",
                     "export",
                     "-n",
@@ -159,7 +163,9 @@ def check_env(
                 ]
             ).decode()
         )
-        env_check["mtime"] = os.path.getmtime(env_check["prefix"])
+        env_check["mtime"] = os.path.getmtime(
+            os.path.normpath(env_check["prefix"])
+        )
         with open(env_check_fpath, "w") as f:
             ryaml.dump(env_check, f)
     if output_fpath is None:

{calkit_python-0.8.2 → calkit_python-0.8.4}/calkit/core.py

@@ -114,7 +114,7 @@ NOTEBOOK_STAGE_OUT_FORMATS = ["pickle", "parquet", "json", "yaml", "csv"]
 
 
 def get_notebook_stage_dir(stage_name: str) -> str:
-    return f".calkit/notebook-stages/{stage_name}"
+    return os.path.join(".calkit", "notebook-stages", stage_name)
 
 
 def get_notebook_stage_script_path(stage_name: str) -> str:

{calkit_python-0.8.2 → calkit_python-0.8.4}/calkit/magics.py

@@ -2,6 +2,7 @@
 
 import ast
 import os
+import pathlib
 import subprocess
 
 from IPython.core import magic_arguments
@@ -18,6 +19,10 @@ def _parse_string_arg(val: str):
     return val
 
 
+def _posix_path(path: str):
+    return pathlib.Path(path).as_posix()
+
+
 @magics_class
 class Calkit(Magics):
 
@@ -29,6 +34,15 @@ class Calkit(Magics):
         required=True,
         type=_parse_string_arg,
     )
+    @magic_arguments.argument(
+        "--env",
+        nargs="?",
+        const=True,
+        help=(
+            "Whether or not this cell should be run in an environment. "
+            "If no environment name is provided, the default will be used."
+        ),
+    )
     @magic_arguments.argument(
         "--dep",
         "-d",
@@ -106,7 +120,15 @@ class Calkit(Magics):
         Then variables will be loaded back into the user namespace state by
         loading the DVC output.
         """
-        args = magic_arguments.parse_argstring(self.stage, line)
+        # Allow command for this line to continue with backslashes
+        cell_split = cell.split("\n")
+        line_combined = line.strip().removesuffix("\\")
+        while line.strip().endswith("\\"):
+            newline = cell_split.pop(0)
+            line += newline
+            line_combined += " " + newline.strip().removesuffix("\\")
+        cell = "\n".join(cell_split)
+        args = magic_arguments.parse_argstring(self.stage, line_combined)
         # If an output object type is specified, make sure we only have one
         # output
         if args.out_type:
@@ -209,7 +231,7 @@ class Calkit(Magics):
             "--run",
             "--force",
             "-d",
-            script_fpath,
+            _posix_path(script_fpath),
         ]
         if args.dep:
             for dep in args.dep:
@@ -221,11 +243,11 @@ class Calkit(Magics):
                     kws["fmt"] = dep_split[2]
                 cmd += [
                     "-d",
-                    calkit.get_notebook_stage_out_path(**kws),
+                    _posix_path(calkit.get_notebook_stage_out_path(**kws)),
                 ]
         if args.dep_path:
             for dep in args.dep_path:
-                cmd += ["-d", dep]
+                cmd += ["-d", _posix_path(dep)]
         if args.out:
             for out in args.out:
                 out_split = out.split(":")
@@ -235,12 +257,18 @@ class Calkit(Magics):
                     kws["fmt"] = out_split[1]
                 cmd += [
                     "-o",
-                    calkit.get_notebook_stage_out_path(**kws),
+                    _posix_path(calkit.get_notebook_stage_out_path(**kws)),
                 ]
         if args.out_path:
             for path in args.out_path:
-                cmd += ["-o", path]
-        cmd.append(f"python '{script_fpath}'")
+                cmd += ["-o", _posix_path(path)]
+        stage_cmd = f'python "{_posix_path(script_fpath)}"'
+        if args.env:
+            runenv = "calkit runenv"
+            if isinstance(args.env, str):
+                runenv += f" -n {args.env}"
+            stage_cmd = runenv + " -- " + stage_cmd
+        cmd.append(stage_cmd)
         try:
             subprocess.run(cmd, check=True, capture_output=True, text=True)
         except subprocess.CalledProcessError as e:

{calkit_python-0.8.2 → calkit_python-0.8.4}/docs/tutorials/notebook-pipeline.md

@@ -143,7 +143,14 @@ We can add a line that saves the figure and declare an additional output path
 and metadata like (note this requires `plotly` and `kaleido` to be installed):
 
 ```python
-%%stage --name plot --dep get-data:df:parquet:pandas --out fig --out-path figures/plot.png --out-type figure --out-title "A plot of the data" --out-desc "This is a plot of the data."
+%%stage \
+   --name plot \
+   --dep get-data:df:parquet:pandas \
+   --out fig \
+   --out-path figures/plot.png \
+   --out-type figure \
+   --out-title "A plot of the data" \
+   --out-desc "This is a plot of the data."
 
 import os
 

{calkit_python-0.8.2 → calkit_python-0.8.4}/test/pipeline.ipynb

@@ -65,7 +65,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "%%stage --name plot-fig --dep get-data:df:parquet:pandas --out fig --out-path figures/plot.png --out-type figure --out-title \"A plot of the data\" --out-desc \"This is a plot of the data.\"\n",
+    "%%stage \\\n",
+    "    --name plot-fig \\\n",
+    "    --dep get-data:df:parquet:pandas \\\n",
+    "    --out fig \\\n",
+    "    --out-path figures/plot.png \\\n",
+    "    --out-type figure \\\n",
+    "    --out-title \"A plot of the data\" \\\n",
+    "    --out-desc \"This is a plot of the data.\"\n",
     "\n",
     "import os\n",
     "\n",