calkit-python 0.14.5__tar.gz → 0.16.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {calkit_python-0.14.5 → calkit_python-0.16.0}/.github/workflows/docs.yml +1 -1
- {calkit_python-0.14.5 → calkit_python-0.16.0}/.gitignore +1 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/PKG-INFO +15 -26
- {calkit_python-0.14.5 → calkit_python-0.16.0}/README.md +11 -24
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/__init__.py +1 -1
- calkit_python-0.16.0/calkit/cli/check.py +48 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/cli/main.py +142 -53
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/cli/new.py +1 -2
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/core.py +18 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/dvc.py +8 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/git.py +10 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/models.py +18 -3
- calkit_python-0.16.0/calkit/ops.py +45 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/server.py +1 -1
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/tests/cli/test_main.py +49 -24
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/tests/test_check.py +22 -0
- calkit_python-0.16.0/docs/apps.md +9 -0
- calkit_python-0.16.0/docs/calculations.md +51 -0
- calkit_python-0.16.0/docs/calkit-yaml.md +21 -0
- calkit_python-0.16.0/docs/cli-reference.md +3 -0
- calkit_python-0.16.0/docs/cloud-integration.md +24 -0
- calkit_python-0.16.0/docs/examples.md +66 -0
- calkit_python-0.16.0/docs/help.md +14 -0
- calkit_python-0.16.0/docs/index.md +51 -0
- calkit_python-0.16.0/docs/installation.md +18 -0
- calkit_python-0.16.0/docs/pipeline/index.md +37 -0
- calkit_python-0.16.0/docs/pipeline/manual-steps.md +38 -0
- calkit_python-0.16.0/docs/references.md +6 -0
- calkit_python-0.16.0/docs/tutorials/first-project.md +3 -0
- calkit_python-0.16.0/docs/tutorials/img/latex-codespaces/building-codespace.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/latex-codespaces/codespaces-secrets-2.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/latex-codespaces/editor-split.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/latex-codespaces/go-to-linked-code.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/latex-codespaces/issue-from-selection.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/latex-codespaces/new-project.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/latex-codespaces/new-pub-2.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/latex-codespaces/new-token.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/latex-codespaces/paper.tex.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/latex-codespaces/project-home-3.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/latex-codespaces/push.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/latex-codespaces/stage.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/office/anakin-excel.jpg +0 -0
- calkit_python-0.16.0/docs/tutorials/img/office/chart-more-rows.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/office/create-project.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/office/elsevier-research-data-guidelines.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/office/excel-chart.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/office/excel-data.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/office/insert-link-to-file.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/office/needs-clone.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/office/new-stage.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/office/phd-comics-version-control.webp +0 -0
- calkit_python-0.16.0/docs/tutorials/img/office/pipeline-out-of-date.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/office/status-more-rows.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/office/uncommitted-changes.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/office/untracked-data.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/office/updated-publication.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/office/word-to-pdf-stage-2.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/office/workflow-page.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/openfoam/clone.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/openfoam/create-project.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/openfoam/datasets-page.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/openfoam/figure-on-website-updated.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/openfoam/figure-on-website.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/openfoam/new-token.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/openfoam/reclone.png +0 -0
- calkit_python-0.16.0/docs/tutorials/img/openfoam/status-after-import-dataset.png +0 -0
- calkit_python-0.16.0/docs/tutorials/latex-codespaces.md +365 -0
- calkit_python-0.16.0/docs/tutorials/matlab.md +3 -0
- calkit_python-0.16.0/docs/tutorials/office.md +351 -0
- calkit_python-0.16.0/docs/tutorials/openfoam.md +314 -0
- calkit_python-0.16.0/docs/version-control.md +173 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/mkdocs.yml +34 -1
- {calkit_python-0.14.5 → calkit_python-0.16.0}/pyproject.toml +3 -1
- calkit_python-0.14.5/calkit/cli/check.py +0 -23
- calkit_python-0.14.5/docs/index.md +0 -8
- {calkit_python-0.14.5 → calkit_python-0.16.0}/.github/FUNDING.yml +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/.github/workflows/publish-test.yml +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/.github/workflows/publish.yml +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/LICENSE +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/calc.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/check.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/cli/__init__.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/cli/config.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/cli/core.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/cli/import_.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/cli/list.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/cli/notebooks.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/cli/office.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/cli/update.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/cloud.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/conda.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/config.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/datasets.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/docker.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/gui.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/jupyter.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/magics.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/office.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/templates/__init__.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/templates/core.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/templates/latex/__init__.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/templates/latex/article/paper.tex +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/templates/latex/core.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/templates/latex/jfm/jfm.bst +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/templates/latex/jfm/jfm.cls +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/templates/latex/jfm/lineno-FLM.sty +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/templates/latex/jfm/paper.tex +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/templates/latex/jfm/upmath.sty +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/tests/__init__.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/tests/cli/__init__.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/tests/cli/test_list.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/tests/cli/test_new.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/tests/test_calc.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/tests/test_conda.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/tests/test_core.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/tests/test_dvc.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/tests/test_jupyter.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/tests/test_magics.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/calkit/tests/test_templates.py +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/docs/CNAME +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/docs/environments.md +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/docs/img/c-to-the-k-white.svg +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/docs/img/calkit-no-bg.png +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/docs/tutorials/adding-latex-pub-docker.md +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/docs/tutorials/conda-envs.md +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/docs/tutorials/img/run-proc.png +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/docs/tutorials/notebook-pipeline.md +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/docs/tutorials/procedures.md +0 -0
- {calkit_python-0.14.5 → calkit_python-0.16.0}/test/pipeline.ipynb +0 -0
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: calkit-python
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.16.0
|
|
4
4
|
Summary: Reproducibility simplified.
|
|
5
5
|
Project-URL: Homepage, https://calkit.org
|
|
6
|
-
Project-URL: Repository, https://github.com/calkit/calkit
|
|
7
6
|
Project-URL: Issues, https://github.com/calkit/calkit/issues
|
|
7
|
+
Project-URL: Repository, https://github.com/calkit/calkit
|
|
8
8
|
Author-email: Pete Bachant <petebachant@gmail.com>
|
|
9
9
|
License-File: LICENSE
|
|
10
10
|
Classifier: License :: OSI Approved :: MIT License
|
|
@@ -23,6 +23,7 @@ Requires-Dist: pillow
|
|
|
23
23
|
Requires-Dist: pydantic-settings
|
|
24
24
|
Requires-Dist: pydantic[email]
|
|
25
25
|
Requires-Dist: pyjwt
|
|
26
|
+
Requires-Dist: python-dotenv>=1
|
|
26
27
|
Requires-Dist: pywin32; platform_system == 'Windows'
|
|
27
28
|
Requires-Dist: requests
|
|
28
29
|
Requires-Dist: typer
|
|
@@ -35,6 +36,7 @@ Requires-Dist: pytest; extra == 'dev'
|
|
|
35
36
|
Provides-Extra: docs
|
|
36
37
|
Requires-Dist: mkdocs; extra == 'docs'
|
|
37
38
|
Requires-Dist: mkdocs-material; extra == 'docs'
|
|
39
|
+
Requires-Dist: mkdocs-mermaid2-plugin; extra == 'docs'
|
|
38
40
|
Description-Content-Type: text/markdown
|
|
39
41
|
|
|
40
42
|
<p align="center">
|
|
@@ -43,7 +45,7 @@ Description-Content-Type: text/markdown
|
|
|
43
45
|
</a>
|
|
44
46
|
</p>
|
|
45
47
|
|
|
46
|
-
Calkit is a
|
|
48
|
+
Calkit is a framework and toolkit for reproducible research projects.
|
|
47
49
|
It acts as a top-level layer to integrate and simplify the use of enabling
|
|
48
50
|
technologies such as
|
|
49
51
|
[Git](https://git-scm.com/),
|
|
@@ -54,22 +56,7 @@ Calkit also adds a domain-specific data model
|
|
|
54
56
|
such that all aspects of the research process can be fully described in a
|
|
55
57
|
single repository and therefore easily consumed by others.
|
|
56
58
|
|
|
57
|
-
|
|
58
|
-
To do this, we try to make it easy for users to follow two simple rules:
|
|
59
|
-
|
|
60
|
-
1. **Keep everything in version control.** This includes large files like
|
|
61
|
-
datasets, enabled by DVC.
|
|
62
|
-
The [Calkit cloud](https://github.com/calkit/calkit-cloud)
|
|
63
|
-
serves as a simple default DVC remote storage location for those who do not
|
|
64
|
-
want to manage their own infrastructure.
|
|
65
|
-
2. **Generate all important artifacts with a single pipeline.** There should be
|
|
66
|
-
no special instructions required to reproduce a project's artifacts.
|
|
67
|
-
It should be as simple as calling `calkit run`.
|
|
68
|
-
The DVC pipeline (in a project's `dvc.yaml` file) is therefore the main
|
|
69
|
-
thing to "build" throughout a research project.
|
|
70
|
-
Calkit provides helper functionality to build pipeline stages that
|
|
71
|
-
keep computational environments up-to-date and label their outputs for
|
|
72
|
-
convenient reuse.
|
|
59
|
+
To learn more, see the [documentation](https://docs.calkit.org).
|
|
73
60
|
|
|
74
61
|
## Installation
|
|
75
62
|
|
|
@@ -145,13 +132,15 @@ This will commit and push to both GitHub and the Calkit Cloud.
|
|
|
145
132
|
|
|
146
133
|
## Tutorials
|
|
147
134
|
|
|
148
|
-
- [LaTeX collaboration with GitHub Codespaces](https://
|
|
149
|
-
- [Jupyter notebook as a DVC pipeline](docs/tutorials/notebook-pipeline
|
|
150
|
-
- [Keeping track of conda environments](docs/tutorials/conda-envs
|
|
151
|
-
- [Defining and executing manual procedures](docs/tutorials/procedures
|
|
152
|
-
- [Adding a new LaTeX-based publication with its own Docker build environment](docs/tutorials/adding-latex-pub-docker
|
|
153
|
-
- [A reproducible workflow using Microsoft Office (Word and Excel)](https://
|
|
154
|
-
- [Reproducible OpenFOAM simulations](https://
|
|
135
|
+
- [LaTeX collaboration with GitHub Codespaces](https://docs.calkit.org/tutorials/latex-codespaces/)
|
|
136
|
+
- [Jupyter notebook as a DVC pipeline](https://docs.calkit.org/tutorials/notebook-pipeline/)
|
|
137
|
+
- [Keeping track of conda environments](https://docs.calkit.org/tutorials/conda-envs/)
|
|
138
|
+
- [Defining and executing manual procedures](https://docs.calkit.org/tutorials/procedures/)
|
|
139
|
+
- [Adding a new LaTeX-based publication with its own Docker build environment](https://docs.calkit.org/tutorials/adding-latex-pub-docker/)
|
|
140
|
+
- [A reproducible workflow using Microsoft Office (Word and Excel)](https://docs.calkit.org/tutorials/office/)
|
|
141
|
+
- [Reproducible OpenFOAM simulations](https://docs.calkit.org/tutorials/openfoam/)
|
|
142
|
+
|
|
143
|
+
See more in the [docs](https://docs.calkit.org).
|
|
155
144
|
|
|
156
145
|
## Why does reproducibility matter?
|
|
157
146
|
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
</a>
|
|
5
5
|
</p>
|
|
6
6
|
|
|
7
|
-
Calkit is a
|
|
7
|
+
Calkit is a framework and toolkit for reproducible research projects.
|
|
8
8
|
It acts as a top-level layer to integrate and simplify the use of enabling
|
|
9
9
|
technologies such as
|
|
10
10
|
[Git](https://git-scm.com/),
|
|
@@ -15,22 +15,7 @@ Calkit also adds a domain-specific data model
|
|
|
15
15
|
such that all aspects of the research process can be fully described in a
|
|
16
16
|
single repository and therefore easily consumed by others.
|
|
17
17
|
|
|
18
|
-
|
|
19
|
-
To do this, we try to make it easy for users to follow two simple rules:
|
|
20
|
-
|
|
21
|
-
1. **Keep everything in version control.** This includes large files like
|
|
22
|
-
datasets, enabled by DVC.
|
|
23
|
-
The [Calkit cloud](https://github.com/calkit/calkit-cloud)
|
|
24
|
-
serves as a simple default DVC remote storage location for those who do not
|
|
25
|
-
want to manage their own infrastructure.
|
|
26
|
-
2. **Generate all important artifacts with a single pipeline.** There should be
|
|
27
|
-
no special instructions required to reproduce a project's artifacts.
|
|
28
|
-
It should be as simple as calling `calkit run`.
|
|
29
|
-
The DVC pipeline (in a project's `dvc.yaml` file) is therefore the main
|
|
30
|
-
thing to "build" throughout a research project.
|
|
31
|
-
Calkit provides helper functionality to build pipeline stages that
|
|
32
|
-
keep computational environments up-to-date and label their outputs for
|
|
33
|
-
convenient reuse.
|
|
18
|
+
To learn more, see the [documentation](https://docs.calkit.org).
|
|
34
19
|
|
|
35
20
|
## Installation
|
|
36
21
|
|
|
@@ -106,13 +91,15 @@ This will commit and push to both GitHub and the Calkit Cloud.
|
|
|
106
91
|
|
|
107
92
|
## Tutorials
|
|
108
93
|
|
|
109
|
-
- [LaTeX collaboration with GitHub Codespaces](https://
|
|
110
|
-
- [Jupyter notebook as a DVC pipeline](docs/tutorials/notebook-pipeline
|
|
111
|
-
- [Keeping track of conda environments](docs/tutorials/conda-envs
|
|
112
|
-
- [Defining and executing manual procedures](docs/tutorials/procedures
|
|
113
|
-
- [Adding a new LaTeX-based publication with its own Docker build environment](docs/tutorials/adding-latex-pub-docker
|
|
114
|
-
- [A reproducible workflow using Microsoft Office (Word and Excel)](https://
|
|
115
|
-
- [Reproducible OpenFOAM simulations](https://
|
|
94
|
+
- [LaTeX collaboration with GitHub Codespaces](https://docs.calkit.org/tutorials/latex-codespaces/)
|
|
95
|
+
- [Jupyter notebook as a DVC pipeline](https://docs.calkit.org/tutorials/notebook-pipeline/)
|
|
96
|
+
- [Keeping track of conda environments](https://docs.calkit.org/tutorials/conda-envs/)
|
|
97
|
+
- [Defining and executing manual procedures](https://docs.calkit.org/tutorials/procedures/)
|
|
98
|
+
- [Adding a new LaTeX-based publication with its own Docker build environment](https://docs.calkit.org/tutorials/adding-latex-pub-docker/)
|
|
99
|
+
- [A reproducible workflow using Microsoft Office (Word and Excel)](https://docs.calkit.org/tutorials/office/)
|
|
100
|
+
- [Reproducible OpenFOAM simulations](https://docs.calkit.org/tutorials/openfoam/)
|
|
101
|
+
|
|
102
|
+
See more in the [docs](https://docs.calkit.org).
|
|
116
103
|
|
|
117
104
|
## Why does reproducibility matter?
|
|
118
105
|
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""CLI for checking things."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import subprocess
|
|
6
|
+
from typing import Annotated
|
|
7
|
+
|
|
8
|
+
import typer
|
|
9
|
+
|
|
10
|
+
from calkit.check import check_reproducibility
|
|
11
|
+
from calkit.cli import raise_error
|
|
12
|
+
|
|
13
|
+
check_app = typer.Typer(no_args_is_help=True)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@check_app.command(name="repro")
|
|
17
|
+
def check_repro(
|
|
18
|
+
wdir: Annotated[
|
|
19
|
+
str, typer.Option("--wdir", help="Project working directory.")
|
|
20
|
+
] = ".",
|
|
21
|
+
):
|
|
22
|
+
"""Check the reproducibility of a project."""
|
|
23
|
+
res = check_reproducibility(wdir=wdir, log_func=typer.echo)
|
|
24
|
+
typer.echo(res.to_pretty().encode("utf-8", errors="replace"))
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@check_app.command(name="call")
|
|
28
|
+
def check_call(
|
|
29
|
+
cmd: Annotated[str, typer.Argument(help="Command to check.")],
|
|
30
|
+
if_error: Annotated[
|
|
31
|
+
str,
|
|
32
|
+
typer.Option(
|
|
33
|
+
"--if-error", help="Command to run if there is an error."
|
|
34
|
+
),
|
|
35
|
+
],
|
|
36
|
+
):
|
|
37
|
+
"""Check that a command succeeds and run an alternate if not."""
|
|
38
|
+
try:
|
|
39
|
+
subprocess.check_call(cmd, shell=True)
|
|
40
|
+
typer.echo("Command succeeded")
|
|
41
|
+
except subprocess.CalledProcessError:
|
|
42
|
+
typer.echo("Command failed")
|
|
43
|
+
try:
|
|
44
|
+
typer.echo("Attempting fallback call")
|
|
45
|
+
subprocess.check_call(if_error, shell=True)
|
|
46
|
+
typer.echo("Fallback call succeeded")
|
|
47
|
+
except subprocess.CalledProcessError:
|
|
48
|
+
raise_error("Fallback call failed")
|
|
@@ -12,12 +12,16 @@ import subprocess
|
|
|
12
12
|
import sys
|
|
13
13
|
import time
|
|
14
14
|
|
|
15
|
+
import dotenv
|
|
16
|
+
import dvc.repo
|
|
15
17
|
import git
|
|
16
18
|
import typer
|
|
19
|
+
from dvc.exceptions import NotDvcRepoError
|
|
20
|
+
from git.exc import InvalidGitRepositoryError
|
|
17
21
|
from typing_extensions import Annotated, Optional
|
|
18
22
|
|
|
19
23
|
import calkit
|
|
20
|
-
from calkit.cli import print_sep, raise_error, run_cmd
|
|
24
|
+
from calkit.cli import print_sep, raise_error, run_cmd, warn
|
|
21
25
|
from calkit.cli.check import check_app
|
|
22
26
|
from calkit.cli.config import config_app
|
|
23
27
|
from calkit.cli.import_ import import_app
|
|
@@ -57,8 +61,8 @@ def _to_shell_cmd(cmd: list[str]) -> str:
|
|
|
57
61
|
quoted_cmd = []
|
|
58
62
|
for part in cmd:
|
|
59
63
|
if " " in part or '"' in part or "'" in part:
|
|
60
|
-
part = part.replace('"', r
|
|
61
|
-
quoted_cmd.append(f"
|
|
64
|
+
part = part.replace('"', r"\"")
|
|
65
|
+
quoted_cmd.append(f'"{part}"')
|
|
62
66
|
else:
|
|
63
67
|
quoted_cmd.append(part)
|
|
64
68
|
return " ".join(quoted_cmd)
|
|
@@ -76,6 +80,29 @@ def main(
|
|
|
76
80
|
raise typer.Exit()
|
|
77
81
|
|
|
78
82
|
|
|
83
|
+
@app.command(name="init")
|
|
84
|
+
def init(
|
|
85
|
+
force: Annotated[
|
|
86
|
+
bool,
|
|
87
|
+
typer.Option(
|
|
88
|
+
"--force",
|
|
89
|
+
"-f",
|
|
90
|
+
help="Force reinitializing DVC if already initialized.",
|
|
91
|
+
),
|
|
92
|
+
] = False,
|
|
93
|
+
):
|
|
94
|
+
"""Initialize the current working directory."""
|
|
95
|
+
subprocess.run(["git", "init"])
|
|
96
|
+
dvc_cmd = ["dvc", "init"]
|
|
97
|
+
if force:
|
|
98
|
+
dvc_cmd.append("-f")
|
|
99
|
+
subprocess.run(dvc_cmd)
|
|
100
|
+
# TODO: Initialize `calkit.yaml`
|
|
101
|
+
# TODO: Initialize `dvc.yaml`
|
|
102
|
+
# TODO: Add a sane .gitignore file
|
|
103
|
+
# TODO: Add a sane LICENSE file?
|
|
104
|
+
|
|
105
|
+
|
|
79
106
|
@app.command(name="clone")
|
|
80
107
|
def clone(
|
|
81
108
|
url: Annotated[str, typer.Argument(help="Repo URL.")],
|
|
@@ -183,6 +210,16 @@ def add(
|
|
|
183
210
|
"""
|
|
184
211
|
if to is not None and to not in ["git", "dvc"]:
|
|
185
212
|
raise_error(f"Invalid option for 'to': {to}")
|
|
213
|
+
try:
|
|
214
|
+
repo = git.Repo()
|
|
215
|
+
except InvalidGitRepositoryError:
|
|
216
|
+
warn("Not currently in a Git repo; initializing")
|
|
217
|
+
repo = git.Repo.init()
|
|
218
|
+
try:
|
|
219
|
+
dvc_repo = dvc.repo.Repo()
|
|
220
|
+
except NotDvcRepoError:
|
|
221
|
+
warn("DVC not initialized yet; initializing")
|
|
222
|
+
dvc_repo = dvc.repo.Repo.init()
|
|
186
223
|
# Ensure autostage is enabled for DVC
|
|
187
224
|
subprocess.call(["dvc", "config", "core.autostage", "true"])
|
|
188
225
|
subprocess.call(["git", "add", ".dvc/config"])
|
|
@@ -207,13 +244,11 @@ def add(
|
|
|
207
244
|
".doc",
|
|
208
245
|
]
|
|
209
246
|
dvc_size_thresh_bytes = 1_000_000
|
|
247
|
+
dvc_paths = [
|
|
248
|
+
obj.get("path") for obj in dvc_repo.ls(".", dvc_only=True)
|
|
249
|
+
]
|
|
210
250
|
if "." in paths and to is None:
|
|
211
251
|
raise_error("Cannot add '.' with calkit; use git or dvc")
|
|
212
|
-
if to is None:
|
|
213
|
-
for path in paths:
|
|
214
|
-
if os.path.isdir(path):
|
|
215
|
-
raise_error("Cannot auto-add directories; use git or dvc")
|
|
216
|
-
repo = git.Repo()
|
|
217
252
|
for path in paths:
|
|
218
253
|
# Detect if this file should be tracked with Git or DVC
|
|
219
254
|
# First see if it's in Git
|
|
@@ -222,19 +257,22 @@ def add(
|
|
|
222
257
|
f"Adding {path} to Git since it's already in the repo"
|
|
223
258
|
)
|
|
224
259
|
subprocess.call(["git", "add", path])
|
|
225
|
-
|
|
226
|
-
|
|
260
|
+
elif path in dvc_paths:
|
|
261
|
+
typer.echo(
|
|
262
|
+
f"Adding {path} to DVC since it's already tracked with DVC"
|
|
263
|
+
)
|
|
264
|
+
subprocess.call(["dvc", "add", path])
|
|
265
|
+
elif os.path.splitext(path)[-1] in dvc_extensions:
|
|
227
266
|
typer.echo(f"Adding {path} to DVC per its extension")
|
|
228
267
|
subprocess.call(["dvc", "add", path])
|
|
229
|
-
|
|
230
|
-
if os.path.getsize(path) > dvc_size_thresh_bytes:
|
|
268
|
+
elif calkit.get_size(path) > dvc_size_thresh_bytes:
|
|
231
269
|
typer.echo(
|
|
232
270
|
f"Adding {path} to DVC since it's greater than 1 MB"
|
|
233
271
|
)
|
|
234
272
|
subprocess.call(["dvc", "add", path])
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
273
|
+
else:
|
|
274
|
+
typer.echo(f"Adding {path} to Git")
|
|
275
|
+
subprocess.call(["git", "add", path])
|
|
238
276
|
if commit_message is not None:
|
|
239
277
|
subprocess.call(["git", "commit", "-m", commit_message])
|
|
240
278
|
if push_commit:
|
|
@@ -260,6 +298,10 @@ def commit(
|
|
|
260
298
|
] = False,
|
|
261
299
|
):
|
|
262
300
|
"""Commit a change to the repo."""
|
|
301
|
+
if message is None:
|
|
302
|
+
typer.echo("Please provide a message describing the changes.")
|
|
303
|
+
typer.echo("Example: Update y-label in scripts/plot-data.py")
|
|
304
|
+
message = typer.prompt("Message")
|
|
263
305
|
cmd = ["git", "commit"]
|
|
264
306
|
if all:
|
|
265
307
|
cmd.append("-a")
|
|
@@ -281,10 +323,12 @@ def save(
|
|
|
281
323
|
),
|
|
282
324
|
),
|
|
283
325
|
] = None,
|
|
284
|
-
|
|
326
|
+
save_all: Annotated[
|
|
285
327
|
Optional[bool],
|
|
286
328
|
typer.Option(
|
|
287
|
-
"--all",
|
|
329
|
+
"--all",
|
|
330
|
+
"-a",
|
|
331
|
+
help=("Save all, automatically handling staging and ignoring."),
|
|
288
332
|
),
|
|
289
333
|
] = False,
|
|
290
334
|
message: Annotated[
|
|
@@ -307,8 +351,56 @@ def save(
|
|
|
307
351
|
|
|
308
352
|
This is essentially git/dvc add, commit, and push in one step.
|
|
309
353
|
"""
|
|
354
|
+
if not paths and not save_all:
|
|
355
|
+
raise_error("Paths must be provided if not using --all")
|
|
310
356
|
if paths is not None:
|
|
311
357
|
add(paths, to=to)
|
|
358
|
+
elif save_all:
|
|
359
|
+
# First check to see if we should commit anything to DVC
|
|
360
|
+
dvc_repo = dvc.repo.Repo()
|
|
361
|
+
dvc_status = dvc_repo.data_status()
|
|
362
|
+
for dvc_uncommitted in dvc_status["uncommitted"].get("modified", []):
|
|
363
|
+
typer.echo(f"Adding {dvc_uncommitted} to DVC")
|
|
364
|
+
dvc_repo.commit(dvc_uncommitted, force=True)
|
|
365
|
+
repo = git.Repo()
|
|
366
|
+
untracked_git_files = repo.untracked_files
|
|
367
|
+
auto_ignore_suffixes = [".DS_Store", ".env"]
|
|
368
|
+
auto_ignore_paths = [os.path.join(".dvc", "config.local")]
|
|
369
|
+
auto_ignore_prefixes = [".venv"]
|
|
370
|
+
for untracked_file in untracked_git_files:
|
|
371
|
+
if (
|
|
372
|
+
any(
|
|
373
|
+
[
|
|
374
|
+
untracked_file.endswith(suffix)
|
|
375
|
+
for suffix in auto_ignore_suffixes
|
|
376
|
+
]
|
|
377
|
+
)
|
|
378
|
+
or any(
|
|
379
|
+
[
|
|
380
|
+
untracked_file.startswith(prefix)
|
|
381
|
+
for prefix in auto_ignore_prefixes
|
|
382
|
+
]
|
|
383
|
+
)
|
|
384
|
+
or untracked_file in auto_ignore_paths
|
|
385
|
+
):
|
|
386
|
+
typer.echo(f"Automatically ignoring {untracked_file}")
|
|
387
|
+
with open(".gitignore", "a") as f:
|
|
388
|
+
f.write("\n" + untracked_file + "\n")
|
|
389
|
+
# TODO: Figure out if we should group large folders for dvc
|
|
390
|
+
# Now add untracked files automatically
|
|
391
|
+
for untracked_file in repo.untracked_files:
|
|
392
|
+
add(paths=[untracked_file])
|
|
393
|
+
# Now add changed files
|
|
394
|
+
for changed_file in [d.a_path for d in repo.index.diff(None)]:
|
|
395
|
+
repo.git.add(changed_file)
|
|
396
|
+
if message is None:
|
|
397
|
+
typer.echo("No message provided; entering interactive mode")
|
|
398
|
+
typer.echo("Creating a commit including the following paths:")
|
|
399
|
+
for path in calkit.git.get_staged_files():
|
|
400
|
+
typer.echo(f"- {path}")
|
|
401
|
+
typer.echo("Please provide a message describing the changes.")
|
|
402
|
+
typer.echo("Example: Add new data to data/raw")
|
|
403
|
+
message = typer.prompt("Message")
|
|
312
404
|
commit(all=True if paths is None else False, message=message)
|
|
313
405
|
if not no_push:
|
|
314
406
|
push()
|
|
@@ -316,7 +408,7 @@ def save(
|
|
|
316
408
|
|
|
317
409
|
@app.command(name="pull")
|
|
318
410
|
def pull(
|
|
319
|
-
no_check_auth: Annotated[bool, typer.Option("--no-check-auth")] = False
|
|
411
|
+
no_check_auth: Annotated[bool, typer.Option("--no-check-auth")] = False,
|
|
320
412
|
):
|
|
321
413
|
"""Pull with both Git and DVC."""
|
|
322
414
|
typer.echo("Git pulling")
|
|
@@ -340,7 +432,7 @@ def pull(
|
|
|
340
432
|
|
|
341
433
|
@app.command(name="push")
|
|
342
434
|
def push(
|
|
343
|
-
no_check_auth: Annotated[bool, typer.Option("--no-check-auth")] = False
|
|
435
|
+
no_check_auth: Annotated[bool, typer.Option("--no-check-auth")] = False,
|
|
344
436
|
):
|
|
345
437
|
"""Push with both Git and DVC."""
|
|
346
438
|
typer.echo("Pushing to Git remote")
|
|
@@ -380,7 +472,7 @@ def run_local_server():
|
|
|
380
472
|
name="run",
|
|
381
473
|
add_help_option=False,
|
|
382
474
|
)
|
|
383
|
-
def
|
|
475
|
+
def run(
|
|
384
476
|
targets: Optional[list[str]] = typer.Argument(default=None),
|
|
385
477
|
help: Annotated[bool, typer.Option("-h", "--help")] = False,
|
|
386
478
|
quiet: Annotated[bool, typer.Option("-q", "--quiet")] = False,
|
|
@@ -410,9 +502,8 @@ def run_dvc_repro(
|
|
|
410
502
|
no_commit: Annotated[bool, typer.Option("--no-commit")] = False,
|
|
411
503
|
no_run_cache: Annotated[bool, typer.Option("--no-run-cache")] = False,
|
|
412
504
|
):
|
|
413
|
-
"""
|
|
414
|
-
|
|
415
|
-
"""
|
|
505
|
+
"""Check dependencies, run DVC pipeline, and update Calkit objects."""
|
|
506
|
+
dotenv.load_dotenv(dotenv_path=".env", verbose=verbose)
|
|
416
507
|
# First check any system-level dependencies exist
|
|
417
508
|
typer.echo("Checking system-level dependencies")
|
|
418
509
|
try:
|
|
@@ -596,6 +687,7 @@ def run_in_env(
|
|
|
596
687
|
bool, typer.Option("--verbose", "-v", help="Print verbose output.")
|
|
597
688
|
] = False,
|
|
598
689
|
):
|
|
690
|
+
dotenv.load_dotenv(dotenv_path=".env", verbose=verbose)
|
|
599
691
|
ck_info = calkit.load_calkit_info(process_includes="environments")
|
|
600
692
|
envs = ck_info.get("environments", {})
|
|
601
693
|
if not envs:
|
|
@@ -755,35 +847,6 @@ def run_in_env(
|
|
|
755
847
|
raise_error("Environment kind not supported")
|
|
756
848
|
|
|
757
849
|
|
|
758
|
-
@app.command(
|
|
759
|
-
name="check-call",
|
|
760
|
-
help=(
|
|
761
|
-
"Check that a call to a command succeeds and run another command "
|
|
762
|
-
"if there is an error."
|
|
763
|
-
),
|
|
764
|
-
)
|
|
765
|
-
def check_call(
|
|
766
|
-
cmd: Annotated[str, typer.Argument(help="Command to check.")],
|
|
767
|
-
if_error: Annotated[
|
|
768
|
-
str,
|
|
769
|
-
typer.Option(
|
|
770
|
-
"--if-error", help="Command to run if there is an error."
|
|
771
|
-
),
|
|
772
|
-
],
|
|
773
|
-
):
|
|
774
|
-
try:
|
|
775
|
-
subprocess.check_call(cmd, shell=True)
|
|
776
|
-
typer.echo("Command succeeded")
|
|
777
|
-
except subprocess.CalledProcessError:
|
|
778
|
-
typer.echo("Command failed")
|
|
779
|
-
try:
|
|
780
|
-
typer.echo("Attempting fallback call")
|
|
781
|
-
subprocess.check_call(if_error, shell=True)
|
|
782
|
-
typer.echo("Fallback call succeeded")
|
|
783
|
-
except subprocess.CalledProcessError:
|
|
784
|
-
raise_error("Fallback call failed")
|
|
785
|
-
|
|
786
|
-
|
|
787
850
|
@app.command(
|
|
788
851
|
name="build-docker",
|
|
789
852
|
help="Build Docker image if missing or different from lock file.",
|
|
@@ -900,7 +963,7 @@ def run_procedure(
|
|
|
900
963
|
# Check to make sure the working tree is clean, so we know we ran the
|
|
901
964
|
# committed version of the procedure
|
|
902
965
|
git_status = git_repo.git.status()
|
|
903
|
-
if
|
|
966
|
+
if "working tree clean" not in git_status:
|
|
904
967
|
raise_error(
|
|
905
968
|
f"Cannot execute procedures unless repo is clean:\n\n{git_status}"
|
|
906
969
|
)
|
|
@@ -1079,3 +1142,29 @@ def run_calculation(
|
|
|
1079
1142
|
typer.echo(calc.evaluate_and_format(**parsed_inputs))
|
|
1080
1143
|
except Exception as e:
|
|
1081
1144
|
raise_error(f"Calculation failed: {e}")
|
|
1145
|
+
|
|
1146
|
+
|
|
1147
|
+
@app.command(name="set-env-var")
|
|
1148
|
+
def set_env_var(
|
|
1149
|
+
name: Annotated[str, typer.Argument(help="Name of the variable.")],
|
|
1150
|
+
value: Annotated[str, typer.Argument(help="Value of the variable.")],
|
|
1151
|
+
):
|
|
1152
|
+
"""Set an environmental variable for the project in its '.env' file."""
|
|
1153
|
+
# Ensure that .env is ignored by git
|
|
1154
|
+
repo = git.Repo()
|
|
1155
|
+
if not repo.ignored(".env"):
|
|
1156
|
+
typer.echo("Adding .env to .gitignore")
|
|
1157
|
+
with open(".gitignore", "a") as f:
|
|
1158
|
+
f.write("\n.env\n")
|
|
1159
|
+
dotenv.set_key(dotenv_path=".env", key_to_set=name, value_to_set=value)
|
|
1160
|
+
|
|
1161
|
+
|
|
1162
|
+
@app.command(name="upgrade")
|
|
1163
|
+
def upgrade():
|
|
1164
|
+
"""Upgrade Calkit."""
|
|
1165
|
+
# See if uv is installed first
|
|
1166
|
+
if calkit.check_dep_exists("uv"):
|
|
1167
|
+
cmd = ["uv", "pip", "install", "--system"]
|
|
1168
|
+
else:
|
|
1169
|
+
cmd = ["pip", "install"]
|
|
1170
|
+
subprocess.run(cmd + ["--upgrade", "calkit-python"])
|
|
@@ -89,8 +89,7 @@ def new_project(
|
|
|
89
89
|
] = False,
|
|
90
90
|
):
|
|
91
91
|
"""Create a new project."""
|
|
92
|
-
|
|
93
|
-
docs_url = "https://github.com/calkit/calkit?tab=readme-ov-file#tutorials"
|
|
92
|
+
docs_url = "https://docs.calkit.org"
|
|
94
93
|
success_message = (
|
|
95
94
|
"\nCongrats on creating your new Calkit project!\n\n"
|
|
96
95
|
"Next, you'll probably want to start building your pipeline.\n\n"
|
|
@@ -362,3 +362,21 @@ def read_file(path: str, as_bytes: bool = None) -> str | bytes:
|
|
|
362
362
|
# Project is None, so let's just read a local file
|
|
363
363
|
with open(path, mode="rb" if as_bytes else "r") as f:
|
|
364
364
|
return f.read()
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def get_size(path: str):
|
|
368
|
+
"""Get the size of a path in bytes.
|
|
369
|
+
|
|
370
|
+
This differs from ``os.path.getsize`` in that it is recursive.
|
|
371
|
+
"""
|
|
372
|
+
if os.path.isfile(path):
|
|
373
|
+
return os.path.getsize(path)
|
|
374
|
+
# From https://stackoverflow.com/a/1392549/2284865
|
|
375
|
+
total_size = 0
|
|
376
|
+
for dirpath, dirnames, filenames in os.walk(path):
|
|
377
|
+
for f in filenames:
|
|
378
|
+
fp = os.path.join(dirpath, f)
|
|
379
|
+
# skip if it is symbolic link
|
|
380
|
+
if not os.path.islink(fp):
|
|
381
|
+
total_size += os.path.getsize(fp)
|
|
382
|
+
return total_size
|
|
@@ -6,6 +6,8 @@ import logging
|
|
|
6
6
|
import os
|
|
7
7
|
import subprocess
|
|
8
8
|
|
|
9
|
+
import dvc.repo
|
|
10
|
+
|
|
9
11
|
import calkit
|
|
10
12
|
from calkit.config import get_app_name
|
|
11
13
|
|
|
@@ -102,3 +104,9 @@ def get_remotes(wdir: str = None) -> dict[str, str]:
|
|
|
102
104
|
name, url = line.split("\t")
|
|
103
105
|
resp[name] = url
|
|
104
106
|
return resp
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def list_paths(wdir: str = None) -> list[str]:
|
|
110
|
+
"""List paths tracked with DVC."""
|
|
111
|
+
dvc_repo = dvc.repo.Repo(wdir)
|
|
112
|
+
return [p.get("path") for p in dvc_repo.ls(".", dvc_only=True)]
|
|
@@ -20,3 +20,13 @@ def detect_project_name(path: str = None) -> str:
|
|
|
20
20
|
if owner is None:
|
|
21
21
|
owner = owner_name
|
|
22
22
|
return f"{owner}/{name}"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def get_staged_files(path: str = None) -> list[str]:
|
|
26
|
+
repo = git.Repo(path)
|
|
27
|
+
cmd = ["--staged", "--name-only"]
|
|
28
|
+
if path is not None:
|
|
29
|
+
cmd.append(path)
|
|
30
|
+
diff = repo.git.diff(cmd)
|
|
31
|
+
paths = diff.split("\n")
|
|
32
|
+
return paths
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
-
from datetime import
|
|
5
|
+
from datetime import timedelta
|
|
6
6
|
from typing import Literal
|
|
7
7
|
|
|
8
8
|
from pydantic import BaseModel
|
|
@@ -21,7 +21,7 @@ class _ImportedFromUrl(BaseModel):
|
|
|
21
21
|
class _CalkitObject(BaseModel):
|
|
22
22
|
path: str
|
|
23
23
|
title: str
|
|
24
|
-
description: str
|
|
24
|
+
description: str | None = None
|
|
25
25
|
stage: str | None = None
|
|
26
26
|
|
|
27
27
|
|
|
@@ -201,6 +201,20 @@ class ModelRelease(Release):
|
|
|
201
201
|
path: str
|
|
202
202
|
|
|
203
203
|
|
|
204
|
+
class ShowcaseFigure(BaseModel):
|
|
205
|
+
figure: str
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
class ShowcaseText(BaseModel):
|
|
209
|
+
text: str
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
class DerivedFromProject(BaseModel):
|
|
213
|
+
project: str
|
|
214
|
+
git_repo_url: str
|
|
215
|
+
git_rev: str
|
|
216
|
+
|
|
217
|
+
|
|
204
218
|
class ProjectInfo(BaseModel):
|
|
205
219
|
"""All of the project's information or metadata, written to the
|
|
206
220
|
``calkit.yaml`` file.
|
|
@@ -226,7 +240,7 @@ class ProjectInfo(BaseModel):
|
|
|
226
240
|
description: str | None = None
|
|
227
241
|
name: str | None = None
|
|
228
242
|
git_repo_url: str | None = None
|
|
229
|
-
|
|
243
|
+
derived_from: DerivedFromProject | None = None
|
|
230
244
|
questions: list[str] = []
|
|
231
245
|
datasets: list[Dataset] = []
|
|
232
246
|
figures: list[Figure] = []
|
|
@@ -243,3 +257,4 @@ class ProjectInfo(BaseModel):
|
|
|
243
257
|
str,
|
|
244
258
|
ProjectRelease | PublicationRelease | DatasetRelease | ModelRelease,
|
|
245
259
|
] = {}
|
|
260
|
+
showcase: list[ShowcaseFigure | ShowcaseText] | None = None
|