calkit-python 0.25.0__tar.gz → 0.25.2__tar.gz

{calkit_python-0.25.0 → calkit_python-0.25.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calkit-python
-Version: 0.25.0
+Version: 0.25.2
 Summary: Reproducibility simplified.
 Project-URL: Homepage, https://calkit.org
 Project-URL: Issues, https://github.com/calkit/calkit/issues

{calkit_python-0.25.0 → calkit_python-0.25.2}/calkit/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.25.0"
+__version__ = "0.25.2"
 
 from .core import *  # noqa: F403, I001
 from . import git  # noqa: F401

{calkit_python-0.25.0 → calkit_python-0.25.2}/calkit/cli/main.py

@@ -66,7 +66,7 @@ app.add_typer(overleaf_app, name="overleaf", help="Interact with Overleaf.")
 app.add_typer(cloud_app, name="cloud", help="Interact with a Calkit Cloud.")
 
 # Constants for version control auto-ignore
-AUTO_IGNORE_SUFFIXES = [".DS_Store", ".env", ".pyc"]
+AUTO_IGNORE_SUFFIXES = [".DS_Store", ".env", ".pyc", ".synctex.gz"]
 AUTO_IGNORE_PATHS = [os.path.join(".dvc", "config.local")]
 AUTO_IGNORE_PREFIXES = [".venv", "__pycache__"]
 # Constants for version control auto-add to DVC
@@ -675,45 +675,127 @@ def run_local_server():
     )
 
 
-@app.command(
-    name="run",
-    add_help_option=False,
-)
+@app.command(name="run")
 def run(
-    targets: Optional[list[str]] = typer.Argument(default=None),
-    help: Annotated[bool, typer.Option("-h", "--help")] = False,
-    quiet: Annotated[bool, typer.Option("-q", "--quiet")] = False,
-    verbose: Annotated[bool, typer.Option("-v", "--verbose")] = False,
-    force: Annotated[bool, typer.Option("-f", "--force")] = False,
-    interactive: Annotated[bool, typer.Option("-i", "--interactive")] = False,
-    single_item: Annotated[bool, typer.Option("-s", "--single-item")] = False,
+    targets: Optional[list[str]] = typer.Argument(
+        default=None, help="Stages to run."
+    ),
+    quiet: Annotated[
+        bool, typer.Option("-q", "--quiet", help="Be quiet.")
+    ] = False,
+    verbose: Annotated[
+        bool, typer.Option("-v", "--verbose", help="Print verbose output.")
+    ] = False,
+    force: Annotated[
+        bool,
+        typer.Option(
+            "-f",
+            "--force",
+            help="Run even if stages or inputs have not changed.",
+        ),
+    ] = False,
+    interactive: Annotated[
+        bool,
+        typer.Option(
+            "-i",
+            "--interactive",
+            help="Ask for confirmation before running each stage.",
+        ),
+    ] = False,
+    single_item: Annotated[
+        bool,
+        typer.Option(
+            "-s",
+            "--single-item",
+            help="Run only a single stage without any dependents.",
+        ),
+    ] = False,
     pipeline: Annotated[
         Optional[str], typer.Option("-p", "--pipeline")
     ] = None,
     all_pipelines: Annotated[
-        bool, typer.Option("-P", "--all-pipelines")
+        bool,
+        typer.Option(
+            "-P", "--all-pipelines", help="Run all pipelines in the repo."
+        ),
+    ] = False,
+    recursive: Annotated[
+        bool,
+        typer.Option(
+            "-R", "--recursive", help="Run pipelines in subdirectories."
+        ),
     ] = False,
-    recursive: Annotated[bool, typer.Option("-R", "--recursive")] = False,
     downstream: Annotated[
-        Optional[list[str]], typer.Option("--downstream")
+        Optional[list[str]],
+        typer.Option(
+            "--downstream",
+            help="Start from the specified stage and run all downstream.",
+        ),
     ] = None,
     force_downstream: Annotated[
-        bool, typer.Option("--force-downstream")
+        bool,
+        typer.Option(
+            "--force-downstream",
+            help=(
+                "Force downstream stages to run even if "
+                "they are still up-to-date."
+            ),
+        ),
+    ] = False,
+    pull: Annotated[
+        bool,
+        typer.Option("--pull", help="Try automatically pulling missing data."),
     ] = False,
-    pull: Annotated[bool, typer.Option("--pull")] = False,
-    allow_missing: Annotated[bool, typer.Option("--allow-missing")] = False,
-    dry: Annotated[bool, typer.Option("--dry")] = False,
-    keep_going: Annotated[bool, typer.Option("--keep-going", "-k")] = False,
-    ignore_errors: Annotated[bool, typer.Option("--ignore-errors")] = False,
-    glob: Annotated[bool, typer.Option("--glob")] = False,
-    no_commit: Annotated[bool, typer.Option("--no-commit")] = False,
-    no_run_cache: Annotated[bool, typer.Option("--no-run-cache")] = False,
+    allow_missing: Annotated[
+        bool,
+        typer.Option("--allow-missing", help="Skip stages with missing data."),
+    ] = False,
+    dry: Annotated[
+        bool,
+        typer.Option("--dry", help="Only print commands that would execute."),
+    ] = False,
+    keep_going: Annotated[
+        bool,
+        typer.Option(
+            "--keep-going",
+            "-k",
+            help=(
+                "Continue executing, skipping stages with failed "
+                "inputs from other stages."
+            ),
+        ),
+    ] = False,
+    ignore_errors: Annotated[
+        bool,
+        typer.Option("--ignore-errors", help="Ignore errors from stages."),
+    ] = False,
+    glob: Annotated[
+        bool,
+        typer.Option("--glob", help="Match stages with glob-style patterns."),
+    ] = False,
+    no_commit: Annotated[
+        bool, typer.Option("--no-commit", help="Do not save to the run cache.")
+    ] = False,
+    no_run_cache: Annotated[
+        bool, typer.Option("--no-run-cache", help="Ignore the run cache.")
+    ] = False,
+    save_after_run: Annotated[
+        bool,
+        typer.Option("--save", "-S", help="Save the project after running."),
+    ] = False,
+    save_message: Annotated[
+        str | None,
+        typer.Option(
+            "--save-message", "-m", help="Commit message for saving."
+        ),
+    ] = None,
 ):
     """Check dependencies and run the pipeline."""
     os.environ["CALKIT_PIPELINE_RUNNING"] = "1"
     dotenv.load_dotenv(dotenv_path=".env", verbose=verbose)
     # First check any system-level dependencies exist
-    typer.echo("Checking system-level dependencies")
+    if not quiet:
+        typer.echo("Checking system-level dependencies")
     try:
         calkit.check_system_deps()
     except Exception as e:
@@ -722,7 +804,8 @@ def run(
     # Compile the pipeline
     ck_info = calkit.load_calkit_info()
     if ck_info.get("pipeline", {}):
-        typer.echo("Compiling DVC pipeline")
+        if not quiet:
+            typer.echo("Compiling DVC pipeline")
         try:
             calkit.pipeline.to_dvc(ck_info=ck_info, write=True)
         except Exception as e:
@@ -733,7 +816,6 @@ def run(
     args = targets
     # Extract any boolean args
     for name in [
-        "help",
         "quiet",
         "verbose",
         "force",
@@ -762,6 +844,12 @@ def run(
         os.environ.pop("CALKIT_PIPELINE_RUNNING", None)
         raise_error("DVC pipeline failed")
     os.environ.pop("CALKIT_PIPELINE_RUNNING", None)
+    if save_after_run or save_message is not None:
+        if save_message is None:
+            save_message = "Run pipeline"
+        if not quiet:
+            typer.echo("Saving the project after successful run")
+        save(save_all=True, message=save_message)
 
 
 @app.command(name="manual-step", help="Execute a manual step.")

{calkit_python-0.25.0 → calkit_python-0.25.2}/calkit/cli/new.py

@@ -30,6 +30,13 @@ from calkit.models.pipeline import LatexStage, StageIteration
 new_app = typer.Typer(no_args_is_help=True)
 
 
+def _check_path_dir(path: str):
+    """If path is in a subdirectory, check that it exists."""
+    dirname = os.path.dirname(path)
+    if dirname:
+        os.makedirs(dirname, exist_ok=True)
+
+
 @new_app.command(name="project")
 def new_project(
     path: Annotated[str, typer.Argument(help="Where to create the project.")],
@@ -1075,6 +1082,7 @@ def new_conda_env(
             project_name = os.path.basename(os.getcwd())
         conda_name = calkit.to_kebab_case(project_name) + "-" + name
     # Write environment to path
+    _check_path_dir(path)
     conda_env = dict(
         name=conda_name, channels=["conda-forge"], dependencies=packages
     )
@@ -1174,6 +1182,7 @@ def new_uv_venv(
                 )
     packages_txt = "\n".join(packages)
     # Write environment to path
+    _check_path_dir(path)
     with open(path, "w") as f:
         f.write(packages_txt)
     repo.git.add(path)
@@ -1259,6 +1268,7 @@ def new_venv(
                 )
     packages_txt = "\n".join(packages)
     # Write environment to path
+    _check_path_dir(path)
     with open(path, "w") as f:
         f.write(packages_txt)
     repo.git.add(path)

{calkit_python-0.25.0 → calkit_python-0.25.2}/calkit/models/pipeline.py

@@ -132,16 +132,19 @@ class PythonScriptStage(Stage):
 class LatexStage(Stage):
     kind: Literal["latex"] = "latex"
     target_path: str
-    silent: bool = False
+    verbose: bool = False
     force: bool = False
+    synctex: bool = True
 
     @property
     def dvc_cmd(self) -> str:
         cmd = f"{self.xenv_cmd} -- latexmk -cd -interaction=nonstopmode"
-        if self.silent:
+        if not self.verbose:
             cmd += " -silent"
         if self.force:
             cmd += " -f"
+        if self.synctex:
+            cmd += " -synctex=1"
         cmd += f" -pdf {self.target_path}"
         return cmd
 

{calkit_python-0.25.0 → calkit_python-0.25.2}/calkit/pipeline.py

@@ -44,9 +44,13 @@ def to_dvc(
     except Exception as e:
         raise ValueError(f"Pipeline is not defined properly: {e}")
     dvc_stages = {}
-    # First, create stages for checking/exporting all environments
+    # First, create stages for checking/exporting all environments used in the
+    # pipeline
+    used_envs = set([stage.environment for stage in pipeline.stages.values()])
     env_lock_fpaths = {}
     for env_name, env in ck_info.get("environments", {}).items():
+        if env_name not in used_envs:
+            continue
         env_fpath = env.get("path")
         lock_fpath = get_env_lock_fpath(
             env=env, env_name=env_name, as_posix=True

{calkit_python-0.25.0 → calkit_python-0.25.2}/calkit/tests/cli/test_new.py

@@ -548,7 +548,7 @@ def test_new_latex_stage(tmp_dir):
     pipeline = calkit.dvc.read_pipeline()
     assert pipeline["stages"]["build-paper"]["cmd"] == (
         "calkit xenv -n tex --no-check -- "
-        "latexmk -cd -interaction=nonstopmode -pdf paper.tex"
+        "latexmk -cd -interaction=nonstopmode -silent -synctex=1 -pdf paper.tex"
     )
     assert set(pipeline["stages"]["build-paper"]["deps"]) == set(
         ["paper.tex", ".calkit/env-locks/tex.json"]

{calkit_python-0.25.0 → calkit_python-0.25.2}/calkit/tests/models/test_pipeline.py

@@ -46,8 +46,8 @@ def test_wordtopdfstage():
 
 def test_latexstage():
     s = LatexStage(environment="tex", target_path="my-paper.tex")
-    assert " -silent " not in s.dvc_cmd
-    s.silent = True
     assert " -silent " in s.dvc_cmd
+    s.verbose = True
+    assert " -silent " not in s.dvc_cmd
     assert "my-paper.tex" in s.dvc_deps
     assert "my-paper.pdf" in s.dvc_outs

{calkit_python-0.25.0 → calkit_python-0.25.2}/calkit/tests/test_pipeline.py

@@ -21,7 +21,7 @@ def test_to_dvc():
             "stages": {
                 "get-data": {
                     "kind": "python-script",
-                    "environment": "my-env",
+                    "environment": "py",
                     "script_path": "something/my-cool-script.py",
                     "outputs": [
                         "my-output.out",

calkit_python-0.25.2/docs/notebooks.md

@@ -0,0 +1,236 @@
+# Working with notebooks
+
+While working on a research project,
+Jupyter notebooks can be useful for prototyping and data exploration.
+If while working interactively in a notebook
+you get an output you like, e.g., a figure,
+it can be tempting to simply stop right there
+and copy/paste it into a research article.
+However, in order to keep the project reproducible,
+we need to be able to go from raw data to research article
+[with a single command](https://doi.org/10.1190/1.1822162),
+which of course is not possible in the above scenario.
+
+This is the primary notebook use case Calkit is concerned with:
+generating evidence to back up conclusions or answers to research questions.
+There are other use cases that are out of scope like using notebooks to build
+documentation or interactive web apps for exploring results.
+For building [apps](apps.md) (a different concept in a Calkit project),
+there are probably better tools out there, e.g.,
+[marimo](https://marimo.io/),
+[Dash](https://dash.plotly.com/),
+[Voila](https://voila.readthedocs.io/en/stable/),
+or [Gradio](https://www.gradio.app/).
+
+Here we'll talk about how to take advantage of the interactive nature
+of Jupyter notebooks while incorporating them into a reproducible workflow,
+avoiding some of the pitfalls that have caused a bit of a
+[notebook reproducibility crisis](https://leomurta.github.io/papers/pimentel2019a.pdf).
+Returning to the "one project, one command" requirement,
+we can focus on three rules:
+
+1. The notebook must be kept in version control.
+   This happens naturally since any file included in a Calkit project is
+   kept in version control.
+   However, it's usually a good idea to exclude notebook output from
+   Git commits.
+   This can be done by installing `nbstripout` and running
+   `nbstripout --install` in the project directory.
+1. A notebook must run in one of the project's [environments](environments.md).
+1. Notebooks should be incorporated into the project's
+   [pipeline](pipeline/index.md).
+   It's fine to do some ad hoc work interactively to get the notebook
+   working properly, but
+   "official" outputs should be generated by calling `calkit run`.
+   This means notebooks need to be able to run from top-to-bottom with no
+   manual intervention. We'll see how below.
+
+## Creating an environment for a notebook
+
+Assuming you want to run Python in the notebook, you can create an environment
+for it with `uv`, `venv`, `conda`, or `pixi`.
+For example, if we wanted to create a new `uv-venv` called `py` in our project,
+we can execute:
+
+```sh
+calkit new uv-venv \
+    --name py \
+    --prefix .venv \
+    --python 3.13 \
+    --path requirements.txt \
+    jupyter \
+    "pandas>=2" \
+    numpy \
+    plotly \
+    matplotlib \
+    polars
+```
+
+You can then start JupyterLab in this environment with
+`calkit xenv -n py jupyter lab`.
+
+Note the environment only needs to be created once per project.
+If the project is cloned onto a new machine,
+the environment does not need to be recreated,
+since that will be done automatically when the project is run.
+Also note that it's totally fine and perhaps even preferable to create
+a new environment for each notebook, so long as they have different
+names, prefixes, and paths---there is no limit to the number
+of environments a project can use, and they can be of any type.
+
+## Adding a notebook to the pipeline
+
+A notebook can be added to the pipeline by editing the project's `calkit.yaml`
+file directly, using a `jupyter-notebook` stage.
+For example:
+
+```yaml
+# In calkit.yaml
+environments:
+  py:
+    kind: uv-venv
+    prefix: .venv
+    python: "3.13"
+    path: requirements.txt
+pipeline:
+  stages:
+    my-notebook:
+      kind: jupyter-notebook
+      environment: py
+      notebook_path: notebooks/get-data.ipynb
+      inputs:
+        - config/my-params.json
+      outputs:
+        - data/raw/data.csv
+      html_storage: dvc
+      executed_ipynb_storage: null
+      cleaned_ipynb_storage: git
+# Optional: Add to project notebooks so they can be viewed on Calkit Cloud
+notebooks:
+  - path: notebooks/get-data.ipynb
+    title: Get data
+    stage: my-notebook
+```
+
+For this example, we're declaring that the notebook
+should use the `py` environment, and that it will read an input
+file `config/my-params.json` and produce an output
+file `data/raw/data.csv`.
+These inputs and outputs will be tracked
+along with the notebook and environment content,
+to automatically determine if and when the notebook needs to be rerun.
+Outputs will also be kept in DVC by default so others can pull them down
+without bloating the Git repo.
+Output storage is configurable, however, e.g., if you'd like to keep
+smaller and/or text-based outputs in Git for simplicity's sake.
+
+Copies of the notebook with and without outputs will be generated as the
+notebook is executed, along with an HTML export of the latter.
+Storage for these outputs can be controlled with the `html_storage`,
+`executed_ipynb_storage`, `cleaned_ipynb_storage` properties,
+and they will live inside the project's `.calkit` subdirectory.
+The executed `.ipynb` can be rendered on GitHub or
+[nbviewer.org](https://nbviewer.org),
+and the HTML can be viewed on [calkit.io](https://calkit.io),
+the latter of which allows some level of interactivity, e.g., Plotly figures.
+The cleaned `.ipynb` can be useful for diffing with Git in cases where
+`nbstripout` is not activated.
+
+It's also possible to add a notebook to the pipeline
+inside a notebook with the `declare_notebook` function,
+which will update `calkit.yaml` automatically.
+
+```python
+import calkit
+
+calkit.declare_notebook(
+    path="notebooks/get-data.ipynb",
+    stage_name="my-notebook",
+    environment_name="py",
+    inputs=["config/my-params.json"],
+    outputs=["data/raw/data.csv"],
+    html_storage="dvc",
+    executed_ipynb_storage=None,
+    cleaned_ipynb_storage="git",
+)
+```
+
+Note that for this to run properly `calkit-python` must be installed in
+the notebook's environment, which in this case is named `py` and whose
+packages are listed in `requirements.txt`.
+If we didn't include them when creating the environment,
+we can simply add `calkit-python` to the `requirements.txt` file and rerun
+`calkit xenv -n py jupyter lab`.
+The environment will be updated before starting JupyterLab.
+
+## Working interactively
+
+The main advantage of Jupyter notebooks is the ability to work interactively,
+allowing us to quickly iterate on a smaller chunk of the process
+while the rest remains constant.
+For example, if you need to refine a figure,
+you can keep updating and running the cell that generates the figure,
+without needing to rerun the expensive cell above that generates
+or processes the data for it.
+In this case our notebook might look like this:
+
+```python
+from some_package import run_data_processing
+
+result = run_data_processing(param1=55)
+```
+
+```python
+import matplotlib.pyplot as plt
+
+fig, ax = plt.subplots()
+ax.plot(result["x"], result["y"])
+```
+
+```python
+fig.savefig("figures/my-plot.png")
+```
+
+So, with a fresh Jupyter kernel we'll need to run cell 1 in order to generate
+`result` so we can iterate on cell 2 to get the plot looking the way
+we want it to.
+But what if `run_data_processing`
+takes minutes, hours, or even days, so therefore we don't want to run it
+every time we restart the notebook?
+Well, we can use the Calkit `%%stage` cell magic to automatically cache
+and retrieve the result.
+
+After adding a cell with:
+
+```python
+%load_ext calkit.magics
+```
+
+the first cell can be turned into a pipeline stage by changing it to:
+
+```python
+%%stage --name run-nb-proc --environment py --out result
+
+from some_package import run_data_processing
+
+result = run_data_processing(param1=55)
+```
+
+In the magic command we're giving the cell a unique name,
+declaring which environment it should run in
+(`py` above, but it can be any environment in the project),
+and declaring an output from the cell that we want to be available to
+cells below.
+
+Now, the kernel can be restarted and we can use "run all cells above"
+when working on the figure,
+and we'll have `result` nearly instantaneously.
+`result` will also be versioned with DVC and pushed to the cloud by default,
+so our collaborators can also take advantage of the caching
+without bloating the Git repo.
+Execution as part of the project's pipeline will also take advantage of
+the caching and will not rerun data processing unless something
+about that cell's code or environment has changed.
+
+For a more in-depth look at using the `%%stage` cell magic,
+see [this tutorial](tutorials/notebook-pipeline.md).

{calkit_python-0.25.0 → calkit_python-0.25.2}/mkdocs.yml

@@ -45,6 +45,7 @@ nav:
   - The pipeline:
       - pipeline/index.md
       - pipeline/manual-steps.md
+  - Notebooks: notebooks.md
   - Datasets: datasets.md
   - References: references.md
   - Calculations: calculations.md