calkit-python 0.0.3__tar.gz → 0.0.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. {calkit_python-0.0.3 → calkit_python-0.0.5}/PKG-INFO +36 -4
  2. calkit_python-0.0.5/README.md +81 -0
  3. {calkit_python-0.0.3 → calkit_python-0.0.5}/calkit/__init__.py +1 -1
  4. calkit_python-0.0.5/calkit/cli.py +379 -0
  5. {calkit_python-0.0.3 → calkit_python-0.0.5}/calkit/models.py +7 -0
  6. {calkit_python-0.0.3 → calkit_python-0.0.5}/pyproject.toml +1 -0
  7. calkit_python-0.0.3/README.md +0 -50
  8. calkit_python-0.0.3/calkit/cli.py +0 -177
  9. {calkit_python-0.0.3 → calkit_python-0.0.5}/.github/FUNDING.yml +0 -0
  10. {calkit_python-0.0.3 → calkit_python-0.0.5}/.github/workflows/publish-test.yml +0 -0
  11. {calkit_python-0.0.3 → calkit_python-0.0.5}/.github/workflows/publish.yml +0 -0
  12. {calkit_python-0.0.3 → calkit_python-0.0.5}/.gitignore +0 -0
  13. {calkit_python-0.0.3 → calkit_python-0.0.5}/LICENSE +0 -0
  14. {calkit_python-0.0.3 → calkit_python-0.0.5}/calkit/cloud.py +0 -0
  15. {calkit_python-0.0.3 → calkit_python-0.0.5}/calkit/config.py +0 -0
  16. {calkit_python-0.0.3 → calkit_python-0.0.5}/calkit/core.py +0 -0
  17. {calkit_python-0.0.3 → calkit_python-0.0.5}/calkit/data.py +0 -0
  18. {calkit_python-0.0.3 → calkit_python-0.0.5}/calkit/dvc.py +0 -0
  19. {calkit_python-0.0.3 → calkit_python-0.0.5}/calkit/git.py +0 -0
  20. {calkit_python-0.0.3 → calkit_python-0.0.5}/calkit/gui.py +0 -0
  21. {calkit_python-0.0.3 → calkit_python-0.0.5}/calkit/jupyter.py +0 -0
  22. {calkit_python-0.0.3 → calkit_python-0.0.5}/calkit/server.py +0 -0
  23. {calkit_python-0.0.3 → calkit_python-0.0.5}/calkit/tests/__init__.py +0 -0
  24. {calkit_python-0.0.3 → calkit_python-0.0.5}/calkit/tests/test_core.py +0 -0
  25. {calkit_python-0.0.3 → calkit_python-0.0.5}/calkit/tests/test_jupyter.py +0 -0
  26. {calkit_python-0.0.3 → calkit_python-0.0.5}/examples/cfd-study/README.md +0 -0
  27. {calkit_python-0.0.3 → calkit_python-0.0.5}/examples/cfd-study/calkit.yaml +0 -0
  28. {calkit_python-0.0.3 → calkit_python-0.0.5}/examples/cfd-study/config/simulations/runs.csv +0 -0
  29. {calkit_python-0.0.3 → calkit_python-0.0.5}/examples/cfd-study/notebook.ipynb +0 -0
  30. {calkit_python-0.0.3 → calkit_python-0.0.5}/examples/ms-office/.gitignore +0 -0
  31. {calkit_python-0.0.3 → calkit_python-0.0.5}/examples/ms-office/README.md +0 -0
  32. {calkit_python-0.0.3 → calkit_python-0.0.5}/examples/ms-office/calkit.yaml +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: calkit-python
3
- Version: 0.0.3
3
+ Version: 0.0.5
4
4
  Summary: Reproducibility simplified.
5
5
  Project-URL: Homepage, https://github.com/calkit/calkit
6
6
  Project-URL: Issues, https://github.com/calkit/calkit/issues
@@ -15,6 +15,7 @@ Requires-Dist: eval-type-backport; python_version < '3.10'
15
15
  Requires-Dist: fastapi
16
16
  Requires-Dist: gitpython
17
17
  Requires-Dist: keyring
18
+ Requires-Dist: nbconvert
18
19
  Requires-Dist: pydantic-settings
19
20
  Requires-Dist: pydantic[email]
20
21
  Requires-Dist: pyjwt
@@ -27,12 +28,43 @@ Description-Content-Type: text/markdown
27
28
 
28
29
  # Calkit
29
30
 
30
- [Calkit](https://calkit.io) makes reproducible research easy,
31
+ [Calkit](https://calkit.io) simplifies reproducibility,
31
32
  acting as a layer on top of
32
- [Git](https://git-scm.com/) and [DVC](https://dvc.org/), such that all
33
- all materials involved in the research process can be fully described in a
33
+ [Git](https://git-scm.com/), [DVC](https://dvc.org/),
34
+ [Zenodo](https://zenodo.org), and more,
35
+ such that all all aspects of the research process can be fully described in a
34
36
  single repository.
35
37
 
38
+ ## Why does reproducibility matter?
39
+
40
+ If your work is reproducible, that means that someone else can "run" it and
41
+ get the same results or outputs.
42
+ This is a major step towards addressing
43
+ [the replication crisis](https://en.wikipedia.org/wiki/Replication_crisis)
44
+ and has some major benefits for both you as an individual and the research
45
+ community:
46
+
47
+ 1. You will avoid mistakes caused by, e.g., running an old version of a script
48
+ and including a figure that wasn't created after fixing a bug in the data
49
+ processing pipeline.
50
+ 2. Since your project is "runnable," it's more likely that someone else will be
51
+ able to reuse part of your work to run it in a different context, thereby
52
+ producing a bigger impact and accelerating the pace of discovery.
53
+ If someone can take what you've done and use it to calculate a
54
+ prediction, you have just produced truly useful knowledge.
55
+
56
+ ## Why another tool/platform?
57
+
58
+ Git, GitHub, DVC, Zenodo et al. are amazing tools/platforms, but their
59
+ use involves multiple fairly difficult learning curves.
60
+ Our goal is to provide a single tool and platform to unify all of these so
61
+ that there is a single, gentle learning curve.
62
+ However, it is not our goal to hide or replace these underlying components.
63
+ Advanced users can use them directly, but new users aren't forced to, which
64
+ helps them get up and running with less effort and training.
65
+ Calkit should help users understand what is going on under the hood without
66
+ forcing them to work at that lower level of abstraction.
67
+
36
68
  ## Installation
37
69
 
38
70
  Simply run
@@ -0,0 +1,81 @@
1
+ # Calkit
2
+
3
+ [Calkit](https://calkit.io) simplifies reproducibility,
4
+ acting as a layer on top of
5
+ [Git](https://git-scm.com/), [DVC](https://dvc.org/),
6
+ [Zenodo](https://zenodo.org), and more,
7
+ such that all all aspects of the research process can be fully described in a
8
+ single repository.
9
+
10
+ ## Why does reproducibility matter?
11
+
12
+ If your work is reproducible, that means that someone else can "run" it and
13
+ get the same results or outputs.
14
+ This is a major step towards addressing
15
+ [the replication crisis](https://en.wikipedia.org/wiki/Replication_crisis)
16
+ and has some major benefits for both you as an individual and the research
17
+ community:
18
+
19
+ 1. You will avoid mistakes caused by, e.g., running an old version of a script
20
+ and including a figure that wasn't created after fixing a bug in the data
21
+ processing pipeline.
22
+ 2. Since your project is "runnable," it's more likely that someone else will be
23
+ able to reuse part of your work to run it in a different context, thereby
24
+ producing a bigger impact and accelerating the pace of discovery.
25
+ If someone can take what you've done and use it to calculate a
26
+ prediction, you have just produced truly useful knowledge.
27
+
28
+ ## Why another tool/platform?
29
+
30
+ Git, GitHub, DVC, Zenodo et al. are amazing tools/platforms, but their
31
+ use involves multiple fairly difficult learning curves.
32
+ Our goal is to provide a single tool and platform to unify all of these so
33
+ that there is a single, gentle learning curve.
34
+ However, it is not our goal to hide or replace these underlying components.
35
+ Advanced users can use them directly, but new users aren't forced to, which
36
+ helps them get up and running with less effort and training.
37
+ Calkit should help users understand what is going on under the hood without
38
+ forcing them to work at that lower level of abstraction.
39
+
40
+ ## Installation
41
+
42
+ Simply run
43
+
44
+ ```sh
45
+ pip install calkit-python
46
+ ```
47
+
48
+ ## Cloud integration
49
+
50
+ The Calkit cloud platform (https://calkit.io) serves as a project
51
+ management interface and a DVC remote for easily storing all versions of your
52
+ data/code/figures/publications, interacting with your collaborators,
53
+ reusing others' research artifacts, etc.
54
+
55
+ After signing up, visit the [settings](https://calkit.io/settings) page
56
+ and create a token.
57
+ Then run
58
+
59
+ ```sh
60
+ calkit config set token ${YOUR_TOKEN_HERE}
61
+ ```
62
+
63
+ Then, inside a project repo you'd like to connect to the cloud, run
64
+
65
+ ```sh
66
+ calkit config setup-remote
67
+ ```
68
+
69
+ This will setup the Calkit DVC remote, such that commands like `dvc push` will
70
+ allow you to push versions of your data or pipeline outputs to the cloud
71
+ for safe storage and sharing with your collaborators.
72
+
73
+ ## How it works
74
+
75
+ Calkit creates a simple human-readable "database" inside the `calkit.yaml`
76
+ file, which serves as a way to store important information about the project,
77
+ e.g., what question(s) it seeks to answer,
78
+ what files should be considered datasets, figures, publications, etc.
79
+ The Calkit cloud reads this database and registers the various entities
80
+ as part of the entire ecosystem such that if a project is made public,
81
+ other researchers can find and reuse your work to accelerate their own.
@@ -1,4 +1,4 @@
1
- __version__ = "0.0.3"
1
+ __version__ = "0.0.5"
2
2
 
3
3
  from .core import *
4
4
  from . import git
@@ -0,0 +1,379 @@
1
+ """The command line interface."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import pty
7
+ import subprocess
8
+ import sys
9
+
10
+ import git
11
+ import typer
12
+ from typing_extensions import Annotated, Optional
13
+
14
+ import calkit
15
+ from calkit.core import ryaml
16
+
17
+ from . import config
18
+ from .dvc import configure_remote, set_remote_auth
19
+
20
+ app = typer.Typer(
21
+ invoke_without_command=True,
22
+ no_args_is_help=True,
23
+ context_settings=dict(help_option_names=["-h", "--help"]),
24
+ )
25
+ config_app = typer.Typer(no_args_is_help=True)
26
+ new_app = typer.Typer(no_args_is_help=True)
27
+ notebooks_app = typer.Typer(no_args_is_help=True)
28
+ app.add_typer(config_app, name="config", help="Configure Calkit.")
29
+ app.add_typer(
30
+ new_app, name="new", help="Add new Calkit object (to calkit.yaml)."
31
+ )
32
+ app.add_typer(notebooks_app, name="nb", help="Work with Jupyter notebooks.")
33
+
34
+
35
+ @app.callback()
36
+ def main(
37
+ version: Annotated[
38
+ bool,
39
+ typer.Option("--version", help="Show version and exit."),
40
+ ] = False,
41
+ ):
42
+ if version:
43
+ typer.echo(calkit.__version__)
44
+ raise typer.Exit()
45
+
46
+
47
+ @config_app.command(name="set")
48
+ def set_config_value(key: str, value: str):
49
+ try:
50
+ cfg = config.read()
51
+ cfg = config.Settings.model_validate(cfg.model_dump() | {key: value})
52
+ # Kind of a hack for setting the password computed field
53
+ # Types have been validated above, so this won't hurt to do again
54
+ setattr(cfg, key, value)
55
+ except FileNotFoundError:
56
+ # TODO: This fails if we try to set password before any config has
57
+ # been written
58
+ # Username is fine
59
+ cfg = config.Settings.model_validate({key: value})
60
+ cfg.write()
61
+
62
+
63
+ @config_app.command(name="get")
64
+ def get_config_value(key: str) -> None:
65
+ cfg = config.read()
66
+ val = getattr(cfg, key)
67
+ if val is not None:
68
+ print(val)
69
+ else:
70
+ print()
71
+
72
+
73
+ @config_app.command(name="setup-remote")
74
+ def setup_remote():
75
+ configure_remote()
76
+ set_remote_auth()
77
+
78
+
79
+ @app.command(name="status")
80
+ def get_status():
81
+ """Get a unified Git and DVC status."""
82
+
83
+ def print_sep(name: str):
84
+ width = 66
85
+ txt_width = len(name) + 2
86
+ buffer_width = (width - txt_width) // 2
87
+ buffer = "-" * buffer_width
88
+ typer.echo(f"{buffer} {name} {buffer}")
89
+
90
+ def run_cmd(cmd: list[str]):
91
+ if os.name == "nt":
92
+ subprocess.call(cmd)
93
+ else:
94
+ pty.spawn(cmd, lambda fd: os.read(fd, 1024))
95
+
96
+ print_sep("Code (Git)")
97
+ run_cmd(["git", "status"])
98
+ typer.echo()
99
+ print_sep("Data (DVC)")
100
+ run_cmd(["dvc", "data", "status"])
101
+ typer.echo()
102
+ print_sep("Pipeline (DVC)")
103
+ run_cmd(["dvc", "status"])
104
+
105
+
106
+ @app.command(name="add")
107
+ def add(paths: list[str]):
108
+ """Add paths to the repo.
109
+
110
+ Code will be added to Git and data will be added to DVC.
111
+ """
112
+ dvc_extensions = [".png", ".h5", ".parquet", ".pickle"]
113
+ dvc_size_thresh_bytes = 1_000_000
114
+ dvc_folders = ["data", "figures"]
115
+ if "." in paths:
116
+ print("ERROR: Cannot add '.' with calkit; use git or dvc")
117
+ sys.exit(1)
118
+ for path in paths:
119
+ if os.path.isdir(path):
120
+ print("ERROR: Cannot add directories with calkit; use git or dvc")
121
+ sys.exit(1)
122
+ # Detect if this file should be tracked with Git or DVC
123
+ # TODO: Add to whatever
124
+
125
+
126
+ @app.command(name="commit")
127
+ def commit(
128
+ all: Annotated[Optional[bool], typer.Option("--all", "-a")] = False,
129
+ message: Annotated[Optional[str], typer.Option("--message", "-m")] = None,
130
+ ):
131
+ """Commit a change to the repo."""
132
+ print(all, message)
133
+ raise NotImplementedError
134
+
135
+
136
+ @app.command(name="pull", help="Pull with both Git and DVC.")
137
+ def pull():
138
+ typer.echo("Git pulling")
139
+ subprocess.call(["git", "pull"])
140
+ typer.echo("DVC pulling")
141
+ subprocess.call(["dvc", "pull"])
142
+
143
+
144
+ @app.command(name="push", help="Push with both Git and DVC.")
145
+ def push():
146
+ typer.echo("Pushing to Git remote")
147
+ subprocess.call(["git", "push"])
148
+ typer.echo("Pushing to DVC remote")
149
+ subprocess.call(["dvc", "push"])
150
+
151
+
152
+ @new_app.command(name="figure")
153
+ def new_figure(
154
+ path: str,
155
+ title: Annotated[str, typer.Option("--title")],
156
+ description: Annotated[str, typer.Option("--desc")] = None,
157
+ commit: Annotated[bool, typer.Option("--commit")] = False,
158
+ ):
159
+ """Add a new figure."""
160
+ if os.path.isfile("calkit.yaml"):
161
+ with open("calkit.yaml") as f:
162
+ ck_info = ryaml.load(f)
163
+ else:
164
+ ck_info = {}
165
+ figures = ck_info.get("figures", [])
166
+ paths = [f.get("path") for f in figures]
167
+ if path in paths:
168
+ raise ValueError(f"Figure at path {path} already exists")
169
+ obj = dict(path=path, title=title)
170
+ if description is not None:
171
+ obj["description"] = description
172
+ figures.append(obj)
173
+ ck_info["figures"] = figures
174
+ with open("calkit.yaml", "w") as f:
175
+ ryaml.dump(ck_info, f)
176
+ if commit:
177
+ repo = git.Repo()
178
+ repo.git.add("calkit.yaml")
179
+ repo.git.commit(["-m", f"Add figure {path}"])
180
+
181
+
182
+ @new_app.command("question")
183
+ def new_question(
184
+ question: str,
185
+ commit: Annotated[bool, typer.Option("--commit")] = False,
186
+ ):
187
+ """Add a new question."""
188
+ if os.path.isfile("calkit.yaml"):
189
+ with open("calkit.yaml") as f:
190
+ ck_info = ryaml.load(f)
191
+ else:
192
+ ck_info = {}
193
+ questions = ck_info.get("questions", [])
194
+ if question in questions:
195
+ raise ValueError("Question already exists")
196
+ if not question.endswith("?"):
197
+ raise ValueError("Questions must end with a question mark")
198
+ questions.append(question)
199
+ ck_info["questions"] = questions
200
+ with open("calkit.yaml", "w") as f:
201
+ ryaml.dump(ck_info, f)
202
+ if commit:
203
+ repo = git.Repo()
204
+ repo.git.add("calkit.yaml")
205
+ repo.git.commit(["-m", "Add question"])
206
+
207
+
208
+ @new_app.command("notebook")
209
+ def new_notebook(
210
+ path: Annotated[str, typer.Argument(help="Notebook path (relative)")],
211
+ title: Annotated[str, typer.Option("--title")],
212
+ description: Annotated[str, typer.Option("--desc")] = None,
213
+ commit: Annotated[bool, typer.Option("--commit")] = False,
214
+ ):
215
+ """Add a new notebook."""
216
+ if os.path.isabs(path):
217
+ raise ValueError("Path must be relative")
218
+ if not os.path.isfile(path):
219
+ raise ValueError("Path is not a file")
220
+ if not path.endswith(".ipynb"):
221
+ raise ValueError("Path does not have .ipynb extension")
222
+ # TODO: Add option to create stages that run `calkit nb clean` and
223
+ # `calkit nb execute`
224
+ if os.path.isfile("calkit.yaml"):
225
+ with open("calkit.yaml") as f:
226
+ ck_info = ryaml.load(f)
227
+ else:
228
+ ck_info = {}
229
+ notebooks = ck_info.get("notebooks", [])
230
+ paths = [f.get("path") for f in notebooks]
231
+ if path in paths:
232
+ raise ValueError(f"Notebook at path {path} already exists")
233
+ obj = dict(path=path, title=title)
234
+ if description is not None:
235
+ obj["description"] = description
236
+ notebooks.append(obj)
237
+ ck_info["notebooks"] = notebooks
238
+ with open("calkit.yaml", "w") as f:
239
+ ryaml.dump(ck_info, f)
240
+ if commit:
241
+ repo = git.Repo()
242
+ repo.git.add("calkit.yaml")
243
+ repo.git.commit(["-m", f"Add notebook {path}"])
244
+
245
+
246
+ @notebooks_app.command("clean")
247
+ def clean_notebook_outputs(path: str):
248
+ """Clean notebook and place a copy in the cleaned notebooks directory.
249
+
250
+ This can be useful to use as a preprocessing DVC stage to use a clean
251
+ notebook as a dependency for a stage that caches and executed notebook.
252
+ """
253
+ if os.path.isabs(path):
254
+ raise ValueError("Path must be relative")
255
+ fpath_out = os.path.join(".calkit", "notebooks", "cleaned", path)
256
+ folder = os.path.dirname(fpath_out)
257
+ os.makedirs(folder, exist_ok=True)
258
+ subprocess.call(
259
+ [
260
+ "jupyter",
261
+ "nbconvert",
262
+ path,
263
+ "--clear-output",
264
+ "--to",
265
+ "notebook",
266
+ "--output",
267
+ fpath_out,
268
+ ]
269
+ )
270
+
271
+
272
+ @notebooks_app.command("execute")
273
+ def execute_notebook(path: str):
274
+ """Execute notebook and place a copy in the executed notebooks directory.
275
+
276
+ This can be useful to use as a preprocessing DVC stage to use a clean
277
+ notebook as a dependency for a stage that caches and executed notebook.
278
+ """
279
+ if os.path.isabs(path):
280
+ raise ValueError("Path must be relative")
281
+ fpath_out = os.path.join(".calkit", "notebooks", "executed", path)
282
+ folder = os.path.dirname(fpath_out)
283
+ os.makedirs(folder, exist_ok=True)
284
+ subprocess.call(
285
+ [
286
+ "jupyter",
287
+ "nbconvert",
288
+ path,
289
+ "--execute",
290
+ "--to",
291
+ "notebook",
292
+ "--output",
293
+ fpath_out,
294
+ ]
295
+ )
296
+
297
+
298
+ @app.command(name="server", help="Run the local server.")
299
+ def run_server():
300
+ import uvicorn
301
+
302
+ uvicorn.run(
303
+ "calkit.server:app",
304
+ port=8866,
305
+ host="localhost",
306
+ reload=True,
307
+ reload_dirs=[os.path.dirname(__file__)],
308
+ )
309
+
310
+
311
+ @app.command(
312
+ name="run",
313
+ add_help_option=False,
314
+ help="Run DVC pipeline (a wrapper for `dvc repro`).",
315
+ )
316
+ def run_dvc_repro(
317
+ targets: Optional[list[str]] = typer.Argument(default=None),
318
+ help: Annotated[bool, typer.Option("-h", "--help")] = False,
319
+ quiet: Annotated[bool, typer.Option("-q", "--quiet")] = False,
320
+ verbose: Annotated[bool, typer.Option("-v", "--verbose")] = False,
321
+ force: Annotated[bool, typer.Option("-f", "--force")] = False,
322
+ interactive: Annotated[bool, typer.Option("-i", "--interactive")] = False,
323
+ single_item: Annotated[bool, typer.Option("-s", "--single-item")] = False,
324
+ pipeline: Annotated[
325
+ Optional[str], typer.Option("-p", "--pipeline")
326
+ ] = None,
327
+ all_pipelines: Annotated[
328
+ bool, typer.Option("-P", "--all-pipelines")
329
+ ] = False,
330
+ recursive: Annotated[bool, typer.Option("-R", "--recursive")] = False,
331
+ downstream: Annotated[
332
+ Optional[list[str]], typer.Option("--downstream")
333
+ ] = None,
334
+ force_downstream: Annotated[
335
+ bool, typer.Option("--force-downstream")
336
+ ] = False,
337
+ pull: Annotated[bool, typer.Option("--pull")] = False,
338
+ allow_missing: Annotated[bool, typer.Option("--allow-missing")] = False,
339
+ dry: Annotated[bool, typer.Option("--dry")] = False,
340
+ keep_going: Annotated[bool, typer.Option("--keep-going", "-k")] = False,
341
+ ignore_errors: Annotated[bool, typer.Option("--ignore-errors")] = False,
342
+ glob: Annotated[bool, typer.Option("--glob")] = False,
343
+ no_commit: Annotated[bool, typer.Option("--no-commit")] = False,
344
+ no_run_cache: Annotated[bool, typer.Option("--no-run-cache")] = False,
345
+ ):
346
+ """A simple wrapper for ``dvc repro``."""
347
+ if targets is None:
348
+ targets = []
349
+ args = targets
350
+ # Extract any boolean args
351
+ for name in [
352
+ "help",
353
+ "quiet",
354
+ "verbose",
355
+ "force",
356
+ "interactive",
357
+ "single-item",
358
+ "all-pipelines",
359
+ "recursive",
360
+ "pull",
361
+ "allow-missing",
362
+ "dry",
363
+ "keep-going",
364
+ "force-downstream",
365
+ "glob",
366
+ "no-commit",
367
+ "no-run-cache",
368
+ ]:
369
+ if locals()[name.replace("-", "_")]:
370
+ args.append("--" + name)
371
+ if pipeline is not None:
372
+ args += ["--pipeline", pipeline]
373
+ if downstream is not None:
374
+ args += downstream
375
+ subprocess.call(["dvc", "repro"] + args)
376
+
377
+
378
+ def run() -> None:
379
+ app()
@@ -60,6 +60,12 @@ class Software(BaseModel):
60
60
  description: str
61
61
 
62
62
 
63
+ class Notebook(_CalkitObject):
64
+ """A Jupyter notebook."""
65
+
66
+ pass
67
+
68
+
63
69
  class ProjectInfo(BaseModel):
64
70
  """All of the project's information or metadata, written to the
65
71
  ``calkit.yaml`` file.
@@ -72,3 +78,4 @@ class ProjectInfo(BaseModel):
72
78
  references: list[ReferenceCollection] = []
73
79
  environments: list[Environment] = []
74
80
  software: list[Software] = []
81
+ notebooks: list[Notebook] = []
@@ -17,6 +17,7 @@ dependencies = [
17
17
  "fastapi",
18
18
  "gitpython",
19
19
  "keyring",
20
+ "nbconvert",
20
21
  "pydantic[email]",
21
22
  "pydantic-settings",
22
23
  "pyjwt",
@@ -1,50 +0,0 @@
1
- # Calkit
2
-
3
- [Calkit](https://calkit.io) makes reproducible research easy,
4
- acting as a layer on top of
5
- [Git](https://git-scm.com/) and [DVC](https://dvc.org/), such that all
6
- all materials involved in the research process can be fully described in a
7
- single repository.
8
-
9
- ## Installation
10
-
11
- Simply run
12
-
13
- ```sh
14
- pip install calkit-python
15
- ```
16
-
17
- ## Cloud integration
18
-
19
- The Calkit cloud platform (https://calkit.io) serves as a project
20
- management interface and a DVC remote for easily storing all versions of your
21
- data/code/figures/publications, interacting with your collaborators,
22
- reusing others' research artifacts, etc.
23
-
24
- After signing up, visit the [settings](https://calkit.io/settings) page
25
- and create a token.
26
- Then run
27
-
28
- ```sh
29
- calkit config set token ${YOUR_TOKEN_HERE}
30
- ```
31
-
32
- Then, inside a project repo you'd like to connect to the cloud, run
33
-
34
- ```sh
35
- calkit config setup-remote
36
- ```
37
-
38
- This will setup the Calkit DVC remote, such that commands like `dvc push` will
39
- allow you to push versions of your data or pipeline outputs to the cloud
40
- for safe storage and sharing with your collaborators.
41
-
42
- ## How it works
43
-
44
- Calkit creates a simple human-readable "database" inside the `calkit.yaml`
45
- file, which serves as a way to store important information about the project,
46
- e.g., what question(s) it seeks to answer,
47
- what files should be considered datasets, figures, publications, etc.
48
- The Calkit cloud reads this database and registers the various entities
49
- as part of the entire ecosystem such that if a project is made public,
50
- other researchers can find and reuse your work to accelerate their own.
@@ -1,177 +0,0 @@
1
- """The command line interface."""
2
-
3
- from __future__ import annotations
4
-
5
- import os
6
- import pty
7
- import subprocess
8
- import sys
9
-
10
- import git
11
- import typer
12
- from typing_extensions import Annotated, Optional
13
-
14
- from calkit.core import ryaml
15
-
16
- from . import config
17
- from .dvc import configure_remote, set_remote_auth
18
-
19
- app = typer.Typer()
20
- config_app = typer.Typer()
21
- new_app = typer.Typer()
22
- app.add_typer(config_app, name="config", help="Configure Calkit.")
23
- app.add_typer(new_app, name="new", help="Add new Calkit object.")
24
-
25
-
26
- @config_app.command(name="set")
27
- def set_config_value(key: str, value: str):
28
- try:
29
- cfg = config.read()
30
- cfg = config.Settings.model_validate(cfg.model_dump() | {key: value})
31
- # Kind of a hack for setting the password computed field
32
- # Types have been validated above, so this won't hurt to do again
33
- setattr(cfg, key, value)
34
- except FileNotFoundError:
35
- # TODO: This fails if we try to set password before any config has
36
- # been written
37
- # Username is fine
38
- cfg = config.Settings.model_validate({key: value})
39
- cfg.write()
40
-
41
-
42
- @config_app.command(name="get")
43
- def get_config_value(key: str) -> None:
44
- cfg = config.read()
45
- val = getattr(cfg, key)
46
- if val is not None:
47
- print(val)
48
- else:
49
- print()
50
-
51
-
52
- @config_app.command(name="setup-remote")
53
- def setup_remote():
54
- configure_remote()
55
- set_remote_auth()
56
-
57
-
58
- @app.command(name="status")
59
- def get_status():
60
- """Get a unified Git and DVC status."""
61
-
62
- def print_sep(name: str):
63
- print(f"------------ {name} ------------")
64
-
65
- print_sep("Code")
66
- if os.name == "nt":
67
- subprocess.call(["git", "status"])
68
- print()
69
- print_sep("data")
70
- subprocess.call(["dvc", "status"])
71
- else:
72
- pty.spawn(["git", "status"], lambda fd: os.read(fd, 1024))
73
- print()
74
- print_sep("Data")
75
- pty.spawn(["dvc", "status"], lambda fd: os.read(fd, 1024))
76
-
77
-
78
- @app.command(name="add")
79
- def add(paths: list[str]):
80
- """Add paths to the repo.
81
-
82
- Code will be added to Git and data will be added to DVC.
83
- """
84
- dvc_extensions = [".png", ".h5", ".parquet", ".pickle"]
85
- dvc_size_thresh_bytes = 1_000_000
86
- dvc_folders = ["data", "figures"]
87
- if "." in paths:
88
- print("ERROR: Cannot add '.' with calkit; use git or dvc")
89
- sys.exit(1)
90
- for path in paths:
91
- if os.path.isdir(path):
92
- print("ERROR: Cannot add directories with calkit; use git or dvc")
93
- sys.exit(1)
94
- # Detect if this file should be tracked with Git or DVC
95
- # TODO: Add to whatever
96
-
97
-
98
- @app.command(name="commit")
99
- def commit(
100
- all: Annotated[Optional[bool], typer.Option("--all", "-a")] = False,
101
- message: Annotated[Optional[str], typer.Option("--message", "-m")] = None,
102
- ):
103
- """Commit a change to the repo."""
104
- print(all, message)
105
- raise NotImplementedError
106
-
107
-
108
- @new_app.command(name="figure")
109
- def new_figure(
110
- path: str,
111
- title: Annotated[str, typer.Option("--title")],
112
- description: Annotated[str, typer.Option("--desc")] = None,
113
- commit: Annotated[bool, typer.Option("--commit")] = False,
114
- ):
115
- """Add a new figure to ``calkit.yaml``."""
116
- if os.path.isfile("calkit.yaml"):
117
- with open("calkit.yaml") as f:
118
- ck_info = ryaml.load(f)
119
- else:
120
- ck_info = {}
121
- figures = ck_info.get("figures", [])
122
- paths = [f.get("path") for f in figures]
123
- if path in paths:
124
- raise ValueError(f"Figure at path {path} already exists")
125
- obj = dict(path=path, title=title)
126
- if description is not None:
127
- obj["description"] = description
128
- figures.append(obj)
129
- ck_info["figures"] = figures
130
- with open("calkit.yaml", "w") as f:
131
- ryaml.dump(ck_info, f)
132
- if commit:
133
- repo = git.Repo()
134
- repo.git.add("calkit.yaml")
135
- repo.git.commit(["-m", f"Add figure {path}"])
136
-
137
-
138
- @new_app.command("question")
139
- def new_question(
140
- question: str,
141
- commit: Annotated[bool, typer.Option("--commit")] = False,
142
- ):
143
- if os.path.isfile("calkit.yaml"):
144
- with open("calkit.yaml") as f:
145
- ck_info = ryaml.load(f)
146
- else:
147
- ck_info = {}
148
- questions = ck_info.get("questions", [])
149
- if question in questions:
150
- raise ValueError("Question already exists")
151
- if not question.endswith("?"):
152
- raise ValueError("Questions must end with a question mark")
153
- questions.append(question)
154
- ck_info["questions"] = questions
155
- with open("calkit.yaml", "w") as f:
156
- ryaml.dump(ck_info, f)
157
- if commit:
158
- repo = git.Repo()
159
- repo.git.add("calkit.yaml")
160
- repo.git.commit(["-m", "Add question"])
161
-
162
-
163
- @app.command(name="server")
164
- def run_server():
165
- import uvicorn
166
-
167
- uvicorn.run(
168
- "calkit.server:app",
169
- port=8866,
170
- host="localhost",
171
- reload=True,
172
- reload_dirs=[os.path.dirname(__file__)],
173
- )
174
-
175
-
176
- def run() -> None:
177
- app()
File without changes
File without changes