lamin_cli 1.2.0__py2.py3-none-any.whl → 1.4.0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lamin_cli/__init__.py +1 -1
- lamin_cli/__main__.py +78 -27
- lamin_cli/_load.py +3 -1
- lamin_cli/_save.py +139 -58
- lamin_cli/compute/__init__.py +0 -0
- lamin_cli/compute/modal.py +175 -0
- {lamin_cli-1.2.0.dist-info → lamin_cli-1.4.0.dist-info}/METADATA +1 -1
- lamin_cli-1.4.0.dist-info/RECORD +14 -0
- lamin_cli-1.2.0.dist-info/RECORD +0 -12
- {lamin_cli-1.2.0.dist-info → lamin_cli-1.4.0.dist-info}/LICENSE +0 -0
- {lamin_cli-1.2.0.dist-info → lamin_cli-1.4.0.dist-info}/WHEEL +0 -0
- {lamin_cli-1.2.0.dist-info → lamin_cli-1.4.0.dist-info}/entry_points.txt +0 -0
lamin_cli/__init__.py
CHANGED
lamin_cli/__main__.py
CHANGED
|
@@ -2,13 +2,22 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import inspect
|
|
4
4
|
import os
|
|
5
|
+
import shutil
|
|
5
6
|
import sys
|
|
6
7
|
import warnings
|
|
7
8
|
from collections import OrderedDict
|
|
8
9
|
from functools import wraps
|
|
9
10
|
from importlib.metadata import PackageNotFoundError, version
|
|
11
|
+
from pathlib import Path
|
|
10
12
|
from typing import TYPE_CHECKING
|
|
11
13
|
|
|
14
|
+
from lamindb_setup._init_instance import (
|
|
15
|
+
DOC_DB,
|
|
16
|
+
DOC_INSTANCE_NAME,
|
|
17
|
+
DOC_MODULES,
|
|
18
|
+
DOC_STORAGE_ARG,
|
|
19
|
+
)
|
|
20
|
+
|
|
12
21
|
if TYPE_CHECKING:
|
|
13
22
|
from collections.abc import Mapping
|
|
14
23
|
|
|
@@ -41,12 +50,7 @@ else:
|
|
|
41
50
|
"lamin": [
|
|
42
51
|
{
|
|
43
52
|
"name": "Connect to an instance",
|
|
44
|
-
"commands": [
|
|
45
|
-
"connect",
|
|
46
|
-
"disconnect",
|
|
47
|
-
"info",
|
|
48
|
-
"init",
|
|
49
|
-
],
|
|
53
|
+
"commands": ["connect", "disconnect", "info", "init", "run"],
|
|
50
54
|
},
|
|
51
55
|
{
|
|
52
56
|
"name": "Read & write data",
|
|
@@ -155,24 +159,20 @@ def schema_to_modules_callback(ctx, param, value):
|
|
|
155
159
|
|
|
156
160
|
# fmt: off
|
|
157
161
|
@main.command()
|
|
158
|
-
@click.option("--storage", type=str,
|
|
159
|
-
@click.option("--
|
|
160
|
-
@click.option("--
|
|
161
|
-
@click.option("--
|
|
162
|
-
@click.option("--schema", type=str, default=None, help="[DEPRECATED] Use --modules instead.", callback=schema_to_modules_callback)
|
|
162
|
+
@click.option("--storage", type=str, default = ".", help=DOC_STORAGE_ARG)
|
|
163
|
+
@click.option("--name", type=str, default=None, help=DOC_INSTANCE_NAME)
|
|
164
|
+
@click.option("--db", type=str, default=None, help=DOC_DB)
|
|
165
|
+
@click.option("--modules", type=str, default=None, help=DOC_MODULES)
|
|
163
166
|
# fmt: on
|
|
164
167
|
def init(
|
|
165
168
|
storage: str,
|
|
169
|
+
name: str | None,
|
|
166
170
|
db: str | None,
|
|
167
171
|
modules: str | None,
|
|
168
|
-
name: str | None,
|
|
169
|
-
schema: str | None,
|
|
170
172
|
):
|
|
171
173
|
"""Init an instance."""
|
|
172
174
|
from lamindb_setup._init_instance import init as init_
|
|
173
175
|
|
|
174
|
-
modules = modules if modules is not None else schema
|
|
175
|
-
|
|
176
176
|
return init_(storage=storage, db=db, modules=modules, name=name)
|
|
177
177
|
|
|
178
178
|
|
|
@@ -188,6 +188,8 @@ def connect(instance: str):
|
|
|
188
188
|
`lamin connect` switches
|
|
189
189
|
{attr}`~lamindb.setup.core.SetupSettings.auto_connect` to `True` so that you
|
|
190
190
|
auto-connect in a Python session upon importing `lamindb`.
|
|
191
|
+
|
|
192
|
+
For manually connecting in a Python session, use {func}`~lamindb.connect`.
|
|
191
193
|
"""
|
|
192
194
|
from lamindb_setup import connect as connect_
|
|
193
195
|
from lamindb_setup import settings as settings_
|
|
@@ -307,30 +309,79 @@ def get(entity: str, uid: str | None = None, key: str | None = None):
|
|
|
307
309
|
|
|
308
310
|
|
|
309
311
|
@main.command()
|
|
310
|
-
@click.argument("
|
|
311
|
-
@click.option("--key", type=str, default=None)
|
|
312
|
-
@click.option("--description", type=str, default=None)
|
|
313
|
-
@click.option("--
|
|
314
|
-
|
|
312
|
+
@click.argument("path", type=str)
|
|
313
|
+
@click.option("--key", type=str, default=None, help="The key of the artifact or transform.")
|
|
314
|
+
@click.option("--description", type=str, default=None, help="A description of the artifact or transform.")
|
|
315
|
+
@click.option("--stem-uid", type=str, default=None, help="The stem uid of the artifact or transform.")
|
|
316
|
+
@click.option("--project", type=str, default=None, help="A valid project name or uid.")
|
|
317
|
+
@click.option("--registry", type=str, default=None, help="Either 'artifact' or 'transform'. If not passed, chooses based on path suffix.")
|
|
318
|
+
def save(path: str, key: str, description: str, stem_uid: str, project: str, registry: str):
|
|
315
319
|
"""Save a file or folder.
|
|
316
320
|
|
|
317
|
-
|
|
318
|
-
|
|
321
|
+
Example: Given a valid project name "my_project".
|
|
322
|
+
|
|
323
|
+
```
|
|
324
|
+
lamin save my_table.csv --key my_tables/my_table.csv --project my_project
|
|
325
|
+
```
|
|
319
326
|
|
|
320
|
-
|
|
321
|
-
|
|
327
|
+
Note: Defaults to saving `.py`, `.ipynb`, `.R`, `.Rmd`, and `.qmd` as {class}`~lamindb.Transform` and
|
|
328
|
+
other file types and folders as {class}`~lamindb.Artifact`. You can enforce saving a file as
|
|
329
|
+
an {class}`~lamindb.Artifact` by passing `--registry artifact`.
|
|
322
330
|
"""
|
|
323
|
-
from lamin_cli._save import
|
|
331
|
+
from lamin_cli._save import save_from_path_cli
|
|
324
332
|
|
|
325
|
-
if
|
|
333
|
+
if save_from_path_cli(path, key, description, stem_uid, project, registry) is not None:
|
|
326
334
|
sys.exit(1)
|
|
327
335
|
|
|
328
336
|
|
|
337
|
+
@main.command()
|
|
338
|
+
@click.argument("filepath", type=str)
|
|
339
|
+
@click.option("--project", type=str, default=None, help="A valid project name or uid. When running on Modal, creates an app with the same name.", required=True)
|
|
340
|
+
@click.option("--image-url", type=str, default=None, help="A URL to the base docker image to use.")
|
|
341
|
+
@click.option("--packages", type=str, default="lamindb", help="A comma-separated list of additional packages to install.")
|
|
342
|
+
@click.option("--cpu", type=float, default=None, help="Configuration for the CPU.")
|
|
343
|
+
@click.option("--gpu", type=str, default=None, help="The type of GPU to use (only compatible with cuda images).")
|
|
344
|
+
def run(filepath: str, project: str, image_url: str, packages: str, cpu: int, gpu: str | None):
|
|
345
|
+
"""Run a compute job in the cloud.
|
|
346
|
+
|
|
347
|
+
This is an EXPERIMENTAL feature that enables to run a script on Modal.
|
|
348
|
+
|
|
349
|
+
Example: Given a valid project name "my_project".
|
|
350
|
+
|
|
351
|
+
```
|
|
352
|
+
lamin run my_script.py --project my_project
|
|
353
|
+
```
|
|
354
|
+
"""
|
|
355
|
+
from lamin_cli.compute.modal import Runner
|
|
356
|
+
|
|
357
|
+
default_mount_dir = Path('./modal_mount_dir')
|
|
358
|
+
if not default_mount_dir.is_dir():
|
|
359
|
+
default_mount_dir.mkdir(parents=True, exist_ok=True)
|
|
360
|
+
|
|
361
|
+
shutil.copy(filepath, default_mount_dir)
|
|
362
|
+
|
|
363
|
+
filepath_in_mount_dir = default_mount_dir / Path(filepath).name
|
|
364
|
+
|
|
365
|
+
package_list = []
|
|
366
|
+
if packages:
|
|
367
|
+
package_list = [package.strip() for package in packages.split(',')]
|
|
368
|
+
|
|
369
|
+
runner = Runner(
|
|
370
|
+
local_mount_dir=default_mount_dir,
|
|
371
|
+
app_name=project,
|
|
372
|
+
packages=package_list,
|
|
373
|
+
image_url=image_url,
|
|
374
|
+
cpu=cpu,
|
|
375
|
+
gpu=gpu
|
|
376
|
+
)
|
|
377
|
+
|
|
378
|
+
runner.run(filepath_in_mount_dir)
|
|
379
|
+
|
|
380
|
+
|
|
329
381
|
main.add_command(settings)
|
|
330
382
|
main.add_command(cache)
|
|
331
383
|
main.add_command(migrate)
|
|
332
384
|
|
|
333
|
-
|
|
334
385
|
# https://stackoverflow.com/questions/57810659/automatically-generate-all-help-documentation-for-click-commands
|
|
335
386
|
# https://claude.ai/chat/73c28487-bec3-4073-8110-50d1a2dd6b84
|
|
336
387
|
def _generate_help():
|
lamin_cli/_load.py
CHANGED
|
@@ -78,7 +78,9 @@ def load(
|
|
|
78
78
|
)
|
|
79
79
|
if bump_revision:
|
|
80
80
|
uid = transform.uid
|
|
81
|
-
if
|
|
81
|
+
if (
|
|
82
|
+
uid in new_content
|
|
83
|
+
): # this only hits if it has the full uid, not for the stem uid
|
|
82
84
|
new_uid = f"{uid[:-4]}{increment_base62(uid[-4:])}"
|
|
83
85
|
new_content = new_content.replace(uid, new_uid)
|
|
84
86
|
logger.important(f"updated uid: {uid} → {new_uid}")
|
lamin_cli/_save.py
CHANGED
|
@@ -2,26 +2,30 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import re
|
|
4
4
|
import sys
|
|
5
|
-
from
|
|
5
|
+
from typing import TYPE_CHECKING
|
|
6
6
|
|
|
7
|
+
import click
|
|
7
8
|
from lamin_utils import logger
|
|
8
9
|
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
9
13
|
|
|
10
14
|
def parse_uid_from_code(content: str, suffix: str) -> str | None:
|
|
11
15
|
if suffix == ".py":
|
|
12
16
|
track_pattern = re.compile(
|
|
13
|
-
r'ln\.track\(\s*(?:transform\s*=\s*)?(["\'])([a-zA-Z0-9]{16})\1'
|
|
17
|
+
r'ln\.track\(\s*(?:transform\s*=\s*)?(["\'])([a-zA-Z0-9]{12,16})\1'
|
|
14
18
|
)
|
|
15
19
|
uid_pattern = re.compile(r'\.context\.uid\s*=\s*["\']([^"\']+)["\']')
|
|
16
20
|
elif suffix == ".ipynb":
|
|
17
21
|
track_pattern = re.compile(
|
|
18
|
-
r'ln\.track\(\s*(?:transform\s*=\s*)?(?:\\"|\')([a-zA-Z0-9]{16})(?:\\"|\')'
|
|
22
|
+
r'ln\.track\(\s*(?:transform\s*=\s*)?(?:\\"|\')([a-zA-Z0-9]{12,16})(?:\\"|\')'
|
|
19
23
|
)
|
|
20
24
|
# backward compat
|
|
21
25
|
uid_pattern = re.compile(r'\.context\.uid\s*=\s*\\["\']([^"\']+)\\["\']')
|
|
22
26
|
elif suffix in {".R", ".qmd", ".Rmd"}:
|
|
23
27
|
track_pattern = re.compile(
|
|
24
|
-
r'track\(\s*(?:transform\s*=\s*)?([\'"])([a-zA-Z0-9]{16})\1'
|
|
28
|
+
r'track\(\s*(?:transform\s*=\s*)?([\'"])([a-zA-Z0-9]{12,16})\1'
|
|
25
29
|
)
|
|
26
30
|
uid_pattern = None
|
|
27
31
|
else:
|
|
@@ -42,16 +46,16 @@ def parse_uid_from_code(content: str, suffix: str) -> str | None:
|
|
|
42
46
|
return uid
|
|
43
47
|
|
|
44
48
|
|
|
45
|
-
def
|
|
46
|
-
|
|
49
|
+
def save_from_path_cli(
|
|
50
|
+
path: Path | str,
|
|
47
51
|
key: str | None,
|
|
48
52
|
description: str | None,
|
|
53
|
+
stem_uid: str | None,
|
|
54
|
+
project: str | None,
|
|
49
55
|
registry: str | None,
|
|
50
56
|
) -> str | None:
|
|
51
57
|
import lamindb_setup as ln_setup
|
|
52
|
-
|
|
53
|
-
if not isinstance(filepath, Path):
|
|
54
|
-
filepath = Path(filepath)
|
|
58
|
+
from lamindb_setup.core.upath import LocalPathClasses, UPath, create_path
|
|
55
59
|
|
|
56
60
|
# this will be gone once we get rid of lamin load or enable loading multiple
|
|
57
61
|
# instances sequentially
|
|
@@ -66,77 +70,149 @@ def save_from_filepath_cli(
|
|
|
66
70
|
|
|
67
71
|
ln_setup.settings.auto_connect = auto_connect_state
|
|
68
72
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
if
|
|
75
|
-
|
|
76
|
-
filepath.with_suffix(".html").exists()
|
|
77
|
-
or filepath.with_suffix(".nb.html").exists()
|
|
78
|
-
):
|
|
79
|
-
raise SystemExit(
|
|
80
|
-
f"Please export your {filepath.suffix} file as an html file here"
|
|
81
|
-
f" {filepath.with_suffix('.html')}"
|
|
82
|
-
)
|
|
83
|
-
if (
|
|
84
|
-
filepath.with_suffix(".html").exists()
|
|
85
|
-
and filepath.with_suffix(".nb.html").exists()
|
|
86
|
-
):
|
|
87
|
-
raise SystemExit(
|
|
88
|
-
f"Please delete one of\n - {filepath.with_suffix('.html')}\n -"
|
|
89
|
-
f" {filepath.with_suffix('.nb.html')}"
|
|
90
|
-
)
|
|
73
|
+
# this allows to have the correct treatment of credentials in case of cloud paths
|
|
74
|
+
path = create_path(path)
|
|
75
|
+
# isinstance is needed to cast the type of path to UPath
|
|
76
|
+
# to avoid mypy erors
|
|
77
|
+
assert isinstance(path, UPath)
|
|
78
|
+
if not path.exists():
|
|
79
|
+
raise click.BadParameter(f"Path {path} does not exist", param_hint="path")
|
|
91
80
|
|
|
92
81
|
if registry is None:
|
|
82
|
+
suffixes_transform = {
|
|
83
|
+
"py": {".py", ".ipynb"},
|
|
84
|
+
"R": {".R", ".qmd", ".Rmd"},
|
|
85
|
+
}
|
|
93
86
|
registry = (
|
|
94
87
|
"transform"
|
|
95
|
-
if
|
|
96
|
-
in suffixes_transform["py"].union(suffixes_transform["R"])
|
|
88
|
+
if path.suffix in suffixes_transform["py"].union(suffixes_transform["R"])
|
|
97
89
|
else "artifact"
|
|
98
90
|
)
|
|
99
91
|
|
|
92
|
+
if project is not None:
|
|
93
|
+
project_record = ln.Project.filter(
|
|
94
|
+
ln.Q(name=project) | ln.Q(uid=project)
|
|
95
|
+
).one_or_none()
|
|
96
|
+
if project_record is None:
|
|
97
|
+
raise ln.errors.InvalidArgument(
|
|
98
|
+
f"Project '{project}' not found, either create it with `ln.Project(name='...').save()` or fix typos."
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
is_cloud_path = not isinstance(path, LocalPathClasses)
|
|
102
|
+
|
|
100
103
|
if registry == "artifact":
|
|
101
104
|
ln.settings.creation.artifact_silence_missing_run_warning = True
|
|
102
|
-
|
|
105
|
+
revises = None
|
|
106
|
+
if stem_uid is not None:
|
|
107
|
+
revises = (
|
|
108
|
+
ln.Artifact.filter(uid__startswith=stem_uid)
|
|
109
|
+
.order_by("-created_at")
|
|
110
|
+
.first()
|
|
111
|
+
)
|
|
112
|
+
if revises is None:
|
|
113
|
+
raise ln.errors.InvalidArgument("The stem uid is not found.")
|
|
114
|
+
|
|
115
|
+
if is_cloud_path:
|
|
116
|
+
if key is not None:
|
|
117
|
+
logger.error("Do not pass --key for cloud paths")
|
|
118
|
+
return "key-with-cloud-path"
|
|
119
|
+
elif key is None and description is None:
|
|
103
120
|
logger.error("Please pass a key or description via --key or --description")
|
|
104
121
|
return "missing-key-or-description"
|
|
105
|
-
|
|
122
|
+
|
|
123
|
+
artifact = ln.Artifact(
|
|
124
|
+
path, key=key, description=description, revises=revises
|
|
125
|
+
).save()
|
|
106
126
|
logger.important(f"saved: {artifact}")
|
|
107
127
|
logger.important(f"storage path: {artifact.path}")
|
|
108
128
|
if ln_setup.settings.storage.type == "s3":
|
|
109
129
|
logger.important(f"storage url: {artifact.path.to_url()}")
|
|
130
|
+
if project is not None:
|
|
131
|
+
artifact.projects.add(project_record)
|
|
132
|
+
logger.important(f"labeled with project: {project_record.name}")
|
|
110
133
|
if ln_setup.settings.instance.is_remote:
|
|
111
134
|
slug = ln_setup.settings.instance.slug
|
|
112
135
|
logger.important(f"go to: https://lamin.ai/{slug}/artifact/{artifact.uid}")
|
|
113
136
|
return None
|
|
114
|
-
|
|
115
|
-
|
|
137
|
+
|
|
138
|
+
if registry == "transform":
|
|
139
|
+
if is_cloud_path:
|
|
140
|
+
logger.error("Can not register a transform from a cloud path")
|
|
141
|
+
return "transform-with-cloud-path"
|
|
142
|
+
|
|
143
|
+
if path.suffix in {".qmd", ".Rmd"}:
|
|
144
|
+
html_file_exists = path.with_suffix(".html").exists()
|
|
145
|
+
nb_html_file_exists = path.with_suffix(".nb.html").exists()
|
|
146
|
+
|
|
147
|
+
if not html_file_exists and not nb_html_file_exists:
|
|
148
|
+
logger.error(
|
|
149
|
+
f"Please export your {path.suffix} file as an html file here"
|
|
150
|
+
f" {path.with_suffix('.html')}"
|
|
151
|
+
)
|
|
152
|
+
return "export-qmd-Rmd-as-html"
|
|
153
|
+
elif html_file_exists and nb_html_file_exists:
|
|
154
|
+
logger.error(
|
|
155
|
+
f"Please delete one of\n - {path.with_suffix('.html')}\n -"
|
|
156
|
+
f" {path.with_suffix('.nb.html')}"
|
|
157
|
+
)
|
|
158
|
+
return "delete-html-or-nb-html"
|
|
159
|
+
|
|
160
|
+
with path.open() as file:
|
|
116
161
|
content = file.read()
|
|
117
|
-
uid = parse_uid_from_code(content,
|
|
162
|
+
uid = parse_uid_from_code(content, path.suffix)
|
|
163
|
+
|
|
118
164
|
if uid is not None:
|
|
119
|
-
logger.important(f"mapped '{
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
165
|
+
logger.important(f"mapped '{path.name}' on uid '{uid}'")
|
|
166
|
+
if len(uid) == 16:
|
|
167
|
+
# is full uid
|
|
168
|
+
transform = ln.Transform.filter(uid=uid).one_or_none()
|
|
169
|
+
else:
|
|
170
|
+
# is stem uid
|
|
171
|
+
if stem_uid is not None:
|
|
172
|
+
assert stem_uid == uid, (
|
|
173
|
+
"passed stem uid and parsed stem uid do not match"
|
|
174
|
+
)
|
|
175
|
+
else:
|
|
176
|
+
stem_uid = uid
|
|
177
|
+
transform = (
|
|
178
|
+
ln.Transform.filter(uid__startswith=uid)
|
|
179
|
+
.order_by("-created_at")
|
|
180
|
+
.first()
|
|
125
181
|
)
|
|
126
|
-
|
|
182
|
+
if transform is None:
|
|
183
|
+
uid = f"{stem_uid}0000"
|
|
127
184
|
else:
|
|
128
|
-
# TODO:
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
185
|
+
# TODO: account for folders and hash equivalence as we do in ln.track()
|
|
186
|
+
transform = ln.Transform.filter(key=path.name, is_latest=True).one_or_none()
|
|
187
|
+
revises = None
|
|
188
|
+
if stem_uid is not None:
|
|
189
|
+
revises = (
|
|
190
|
+
ln.Transform.filter(uid__startswith=stem_uid)
|
|
191
|
+
.order_by("-created_at")
|
|
192
|
+
.first()
|
|
193
|
+
)
|
|
194
|
+
if revises is None:
|
|
195
|
+
raise ln.errors.InvalidArgument("The stem uid is not found.")
|
|
196
|
+
if transform is None:
|
|
197
|
+
if path.suffix == ".ipynb":
|
|
198
|
+
from nbproject.dev import read_notebook
|
|
199
|
+
from nbproject.dev._meta_live import get_title
|
|
200
|
+
|
|
201
|
+
nb = read_notebook(path)
|
|
202
|
+
description = get_title(nb)
|
|
203
|
+
else:
|
|
204
|
+
description = None
|
|
205
|
+
transform = ln.Transform(
|
|
206
|
+
uid=uid,
|
|
207
|
+
description=description,
|
|
208
|
+
key=path.name,
|
|
209
|
+
type="script" if path.suffix in {".R", ".py"} else "notebook",
|
|
210
|
+
revises=revises,
|
|
211
|
+
).save()
|
|
212
|
+
logger.important(f"created Transform('{transform.uid}')")
|
|
213
|
+
if project is not None:
|
|
214
|
+
transform.projects.add(project_record)
|
|
215
|
+
logger.important(f"labeled with project: {project_record.name}")
|
|
140
216
|
# latest run of this transform by user
|
|
141
217
|
run = ln.Run.filter(transform=transform).order_by("-started_at").first()
|
|
142
218
|
if run is not None and run.created_by.id != ln_setup.settings.user.id:
|
|
@@ -147,10 +223,15 @@ def save_from_filepath_cli(
|
|
|
147
223
|
)
|
|
148
224
|
if response != "y":
|
|
149
225
|
return "aborted-save-notebook-created-by-different-user"
|
|
226
|
+
if run is None and transform.key.endswith(".ipynb"):
|
|
227
|
+
run = ln.Run(transform=transform).save()
|
|
228
|
+
logger.important(
|
|
229
|
+
f"found no run, creating Run('{run.uid}') to display the html"
|
|
230
|
+
)
|
|
150
231
|
return_code = save_context_core(
|
|
151
232
|
run=run,
|
|
152
233
|
transform=transform,
|
|
153
|
-
filepath=
|
|
234
|
+
filepath=path,
|
|
154
235
|
from_cli=True,
|
|
155
236
|
)
|
|
156
237
|
return return_code
|
|
File without changes
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import subprocess
|
|
3
|
+
import sys
|
|
4
|
+
import threading
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import lamindb_setup as ln_setup
|
|
8
|
+
import modal
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def run_script(path: Path) -> dict:
|
|
12
|
+
"""Takes a path to a script for running it as a function through Modal."""
|
|
13
|
+
result = {"success": False, "output": "", "error": ""}
|
|
14
|
+
|
|
15
|
+
def stream_output(stream, capture_list):
|
|
16
|
+
"""Read from stream line by line and print in real-time while also capturing to a list."""
|
|
17
|
+
for line in iter(stream.readline, ""):
|
|
18
|
+
print(line, end="") # Print in real-time
|
|
19
|
+
capture_list.append(line)
|
|
20
|
+
stream.close()
|
|
21
|
+
|
|
22
|
+
if not path.exists():
|
|
23
|
+
raise FileNotFoundError(f"Script file not found: {path}")
|
|
24
|
+
|
|
25
|
+
try:
|
|
26
|
+
# Run the script using subprocess
|
|
27
|
+
process = subprocess.Popen(
|
|
28
|
+
[sys.executable, path.as_posix()],
|
|
29
|
+
stdout=subprocess.PIPE,
|
|
30
|
+
stderr=subprocess.PIPE,
|
|
31
|
+
text=True,
|
|
32
|
+
bufsize=1, # Line buffered
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
# Capture output and error while streaming stdout in real-time
|
|
36
|
+
stdout_lines: list[str] = []
|
|
37
|
+
stderr_lines: list[str] = []
|
|
38
|
+
|
|
39
|
+
# Create threads to handle stdout and stderr streams
|
|
40
|
+
stdout_thread = threading.Thread(
|
|
41
|
+
target=stream_output, args=(process.stdout, stdout_lines)
|
|
42
|
+
)
|
|
43
|
+
stderr_thread = threading.Thread(
|
|
44
|
+
target=stream_output, args=(process.stderr, stderr_lines)
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
# Set as daemon threads so they exit when the main program exits
|
|
48
|
+
stdout_thread.daemon = True
|
|
49
|
+
stderr_thread.daemon = True
|
|
50
|
+
|
|
51
|
+
# Start the threads
|
|
52
|
+
stdout_thread.start()
|
|
53
|
+
stderr_thread.start()
|
|
54
|
+
|
|
55
|
+
# Wait for the process to complete
|
|
56
|
+
return_code = process.wait()
|
|
57
|
+
|
|
58
|
+
# Wait for the threads to finish
|
|
59
|
+
stdout_thread.join()
|
|
60
|
+
stderr_thread.join()
|
|
61
|
+
|
|
62
|
+
# Join the captured output
|
|
63
|
+
stdout_output = "".join(stdout_lines)
|
|
64
|
+
stderr_output = "".join(stderr_lines)
|
|
65
|
+
|
|
66
|
+
# Check return code
|
|
67
|
+
if return_code == 0:
|
|
68
|
+
result["success"] = True
|
|
69
|
+
result["output"] = stdout_output
|
|
70
|
+
else:
|
|
71
|
+
result["error"] = stderr_output
|
|
72
|
+
|
|
73
|
+
except Exception as e:
|
|
74
|
+
import traceback
|
|
75
|
+
|
|
76
|
+
result["error"] = str(e) + "\n" + traceback.format_exc()
|
|
77
|
+
return result
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class Runner:
|
|
81
|
+
def __init__(
|
|
82
|
+
self,
|
|
83
|
+
app_name: str,
|
|
84
|
+
local_mount_dir: str | Path = "./scripts",
|
|
85
|
+
remote_mount_dir: str | Path = "/scripts",
|
|
86
|
+
image_url: str | None = None,
|
|
87
|
+
packages: list[str] | None = None,
|
|
88
|
+
cpu: float | None = None,
|
|
89
|
+
gpu: str | None = None,
|
|
90
|
+
):
|
|
91
|
+
self.app_name = app_name # we use the LaminDB project name as the app name
|
|
92
|
+
self.app = self.create_modal_app(app_name)
|
|
93
|
+
|
|
94
|
+
self.local_mount_dir = local_mount_dir
|
|
95
|
+
self.remote_mount_dir = remote_mount_dir
|
|
96
|
+
|
|
97
|
+
self.image = self.create_modal_image(
|
|
98
|
+
local_dir=local_mount_dir, packages=packages, image_url=image_url
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
local_secrets = self._configure_local_secrets()
|
|
102
|
+
|
|
103
|
+
self.modal_function = self.app.function(
|
|
104
|
+
image=self.image, cpu=cpu, gpu=gpu, secrets=[local_secrets]
|
|
105
|
+
)(run_script)
|
|
106
|
+
|
|
107
|
+
def run(self, script_local_path: Path) -> None:
|
|
108
|
+
script_remote_path = self.local_to_remote_path(str(script_local_path))
|
|
109
|
+
with modal.enable_output(show_progress=True): # Prints out modal logs
|
|
110
|
+
with self.app.run():
|
|
111
|
+
self.modal_function.remote(Path(script_remote_path))
|
|
112
|
+
|
|
113
|
+
def create_modal_app(self, app_name: str) -> modal.App:
|
|
114
|
+
app = modal.App(app_name)
|
|
115
|
+
return app
|
|
116
|
+
|
|
117
|
+
def local_to_remote_path(self, local_path: str | Path) -> str:
|
|
118
|
+
local_path = Path(local_path).absolute()
|
|
119
|
+
local_mount_dir = Path(self.local_mount_dir).absolute()
|
|
120
|
+
remote_mount_dir = Path(self.remote_mount_dir)
|
|
121
|
+
|
|
122
|
+
# Check if local_path is inside local_mount_dir
|
|
123
|
+
try:
|
|
124
|
+
# This will raise ValueError if local_path is not relative to local_mount_dir
|
|
125
|
+
relative_path = local_path.relative_to(local_mount_dir)
|
|
126
|
+
except ValueError as err:
|
|
127
|
+
raise ValueError(
|
|
128
|
+
f"Local path '{local_path}' is not inside the mount directory '{local_mount_dir}'"
|
|
129
|
+
) from err
|
|
130
|
+
|
|
131
|
+
# Join remote_mount_dir with the relative path
|
|
132
|
+
remote_path = remote_mount_dir / relative_path
|
|
133
|
+
|
|
134
|
+
# Return as string with normalized separators
|
|
135
|
+
return remote_path.as_posix()
|
|
136
|
+
|
|
137
|
+
def _configure_local_secrets(self) -> dict:
|
|
138
|
+
if ln_setup.settings.user.api_key is None:
|
|
139
|
+
raise ValueError("Please authenticate via: lamin login")
|
|
140
|
+
|
|
141
|
+
all_env_variables = {
|
|
142
|
+
"LAMIN_API_KEY": ln_setup.settings.user.api_key,
|
|
143
|
+
"LAMIN_CURRENT_PROJECT": self.app_name,
|
|
144
|
+
"LAMIN_CURRENT_INSTANCE": ln_setup.settings.instance.slug,
|
|
145
|
+
}
|
|
146
|
+
local_secrets = modal.Secret.from_dict(all_env_variables)
|
|
147
|
+
return local_secrets
|
|
148
|
+
|
|
149
|
+
def create_modal_image(
|
|
150
|
+
self,
|
|
151
|
+
python_version: str = "3.12",
|
|
152
|
+
packages: list[str] | None = None,
|
|
153
|
+
local_dir: str | Path = "./scripts",
|
|
154
|
+
remote_dir: str = "/scripts/",
|
|
155
|
+
image_url: str | None = None,
|
|
156
|
+
env_variables: dict | None = None,
|
|
157
|
+
) -> modal.Image:
|
|
158
|
+
if env_variables is None:
|
|
159
|
+
env_variables = {}
|
|
160
|
+
if packages is None:
|
|
161
|
+
packages = ["lamindb"]
|
|
162
|
+
else:
|
|
163
|
+
packages.append("lamindb") # Append lamindb to the list of packages
|
|
164
|
+
|
|
165
|
+
if image_url is None:
|
|
166
|
+
image = modal.Image.debian_slim(python_version=python_version)
|
|
167
|
+
else:
|
|
168
|
+
image = modal.Image.from_registry(image_url, add_python=python_version)
|
|
169
|
+
return (
|
|
170
|
+
image.pip_install(packages)
|
|
171
|
+
.env(env_variables)
|
|
172
|
+
.add_local_python_source("lamindb", "lamindb_setup", copy=True)
|
|
173
|
+
.run_commands("lamin settings set auto-connect true")
|
|
174
|
+
.add_local_dir(local_dir, remote_dir)
|
|
175
|
+
)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
lamin_cli/__init__.py,sha256=YeP3WxwX7_OcORoNOBGWEDB-_0yt4bX9J3ib92jScow,40
|
|
2
|
+
lamin_cli/__main__.py,sha256=1GdJbaTQQmy_xPu-6QWglewBVkXdlGe37PbzSWPKCBM,12813
|
|
3
|
+
lamin_cli/_cache.py,sha256=oplwE8AcS_9PYptQUZxff2qTIdNFS81clGPkJNWk098,800
|
|
4
|
+
lamin_cli/_load.py,sha256=ow9IPQog_mPvjoGj1Bn08l4AG1YyRNnEyBpQvQkfLZs,8182
|
|
5
|
+
lamin_cli/_migration.py,sha256=xQi6mwnpBzY5wcv1-TJhveD7a3XJIlpiYx6Z3AJ1NF0,1063
|
|
6
|
+
lamin_cli/_save.py,sha256=LvxEIXKgUPvpwhjgRyxi1MTDRcL6rlAMbMQLfPtfDhs,9047
|
|
7
|
+
lamin_cli/_settings.py,sha256=O2tecCf5EIZu98ima4DTJujo4KuywckOLgw8c-Ke3dY,1142
|
|
8
|
+
lamin_cli/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
9
|
+
lamin_cli/compute/modal.py,sha256=QnR7GyyvWWWkLnou95HxS9xxSQfw1k-SiefM_qRVnU0,6010
|
|
10
|
+
lamin_cli-1.4.0.dist-info/entry_points.txt,sha256=Qms85i9cZPlu-U7RnVZhFsF7vJ9gaLZUFkCjcGcXTpg,49
|
|
11
|
+
lamin_cli-1.4.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
12
|
+
lamin_cli-1.4.0.dist-info/WHEEL,sha256=ssQ84EZ5gH1pCOujd3iW7HClo_O_aDaClUbX4B8bjKY,100
|
|
13
|
+
lamin_cli-1.4.0.dist-info/METADATA,sha256=GZKhv9Y-CgSgG32doVUMA2a42IZtjKyo0F-8YwgptCQ,337
|
|
14
|
+
lamin_cli-1.4.0.dist-info/RECORD,,
|
lamin_cli-1.2.0.dist-info/RECORD
DELETED
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
lamin_cli/__init__.py,sha256=2kLtbQt2_3KEeiaePHI1brXcTblfz2N7yIGKsKcAtb4,40
|
|
2
|
-
lamin_cli/__main__.py,sha256=a14UZj5xS66nZXdL_dUkU9M5AAWvs5unQ9lHJx1S5fI,10839
|
|
3
|
-
lamin_cli/_cache.py,sha256=oplwE8AcS_9PYptQUZxff2qTIdNFS81clGPkJNWk098,800
|
|
4
|
-
lamin_cli/_load.py,sha256=lMhV9AMkybjvj4VChJE_v7IMy6qGqisFlo40BJUibsA,8087
|
|
5
|
-
lamin_cli/_migration.py,sha256=xQi6mwnpBzY5wcv1-TJhveD7a3XJIlpiYx6Z3AJ1NF0,1063
|
|
6
|
-
lamin_cli/_save.py,sha256=bt873beNgog5naWITjPb61cjy00aeEtIv9lwqQttRGI,5908
|
|
7
|
-
lamin_cli/_settings.py,sha256=O2tecCf5EIZu98ima4DTJujo4KuywckOLgw8c-Ke3dY,1142
|
|
8
|
-
lamin_cli-1.2.0.dist-info/entry_points.txt,sha256=Qms85i9cZPlu-U7RnVZhFsF7vJ9gaLZUFkCjcGcXTpg,49
|
|
9
|
-
lamin_cli-1.2.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
10
|
-
lamin_cli-1.2.0.dist-info/WHEEL,sha256=ssQ84EZ5gH1pCOujd3iW7HClo_O_aDaClUbX4B8bjKY,100
|
|
11
|
-
lamin_cli-1.2.0.dist-info/METADATA,sha256=22dRlSUzNnNcaapFD-faPqMrNG5Fi9KZLjuHVe06hEc,337
|
|
12
|
-
lamin_cli-1.2.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|