lamin_cli 0.21.4__tar.gz → 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/.gitignore +1 -11
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/PKG-INFO +2 -2
- lamin_cli-1.0.0/lamin_cli/__init__.py +3 -0
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/lamin_cli/__main__.py +27 -9
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/lamin_cli/_load.py +61 -35
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/lamin_cli/_migration.py +1 -1
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/lamin_cli/_save.py +32 -54
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/tests/scripts/run-track-and-finish.py +2 -2
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/tests/test_load.py +29 -4
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/tests/test_migrate.py +7 -7
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/tests/test_parse_uid_from_code.py +6 -11
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/tests/test_save_files.py +3 -0
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/tests/test_save_notebooks.py +5 -11
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/tests/test_save_scripts.py +18 -6
- lamin_cli-0.21.4/lamin_cli/__init__.py +0 -3
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/.github/workflows/doc-changes.yml +0 -0
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/.pre-commit-config.yaml +0 -0
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/LICENSE +0 -0
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/README.md +0 -0
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/lamin_cli/_cache.py +0 -0
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/lamin_cli/_settings.py +0 -0
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/pyproject.toml +0 -0
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/tests/conftest.py +0 -0
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/tests/notebooks/not-initialized.ipynb +0 -0
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/tests/notebooks/with-title-and-initialized-consecutive.ipynb +0 -0
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/tests/notebooks/with-title-and-initialized-non-consecutive.ipynb +0 -0
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/tests/scripts/merely-import-lamindb.py +0 -0
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/tests/scripts/run-track-and-finish-sync-git.py +0 -0
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/tests/scripts/run-track-with-params.py +0 -0
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/tests/scripts/run-track.R +0 -0
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/tests/scripts/run-track.qmd +0 -0
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/tests/test_cli.py +0 -0
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/tests/test_multi_process.py +0 -0
- {lamin_cli-0.21.4 → lamin_cli-1.0.0}/tests/test_save_r_code.py +0 -0
|
@@ -1,11 +1,5 @@
|
|
|
1
1
|
# LaminDB
|
|
2
|
-
|
|
3
|
-
docs/guide/knowledge.ipynb
|
|
4
|
-
docs/guide/lnschema-bionty.ipynb
|
|
5
|
-
docs/guide/lnschema-core.ipynb
|
|
6
|
-
docs/paradisi05_laminopathic_nuclei.jpg
|
|
7
|
-
lnschema_bionty_docs/
|
|
8
|
-
lnschema_core_docs/
|
|
2
|
+
lamindb_docs/
|
|
9
3
|
_build
|
|
10
4
|
mydata/
|
|
11
5
|
lamin-intro/
|
|
@@ -202,7 +196,3 @@ _build
|
|
|
202
196
|
|
|
203
197
|
# Pycharm
|
|
204
198
|
.idea
|
|
205
|
-
|
|
206
|
-
# fred files
|
|
207
|
-
docs/guide/insert-some-data-into-frederic-hub.ipynb
|
|
208
|
-
docs/guide/insert-data-in-test-user-1-profile.ipynb
|
|
@@ -6,6 +6,7 @@ import inspect
|
|
|
6
6
|
from importlib.metadata import PackageNotFoundError, version
|
|
7
7
|
from typing import Optional, Mapping
|
|
8
8
|
from functools import wraps
|
|
9
|
+
import warnings
|
|
9
10
|
|
|
10
11
|
# https://github.com/ewels/rich-click/issues/19
|
|
11
12
|
# Otherwise rich-click takes over the formatting.
|
|
@@ -135,18 +136,37 @@ def logout():
|
|
|
135
136
|
return logout_()
|
|
136
137
|
|
|
137
138
|
|
|
139
|
+
def schema_to_modules_callback(ctx, param, value):
|
|
140
|
+
if param.name == "schema" and value is not None:
|
|
141
|
+
warnings.warn(
|
|
142
|
+
"The --schema option is deprecated and will be removed in a future version."
|
|
143
|
+
" Please use --modules instead.",
|
|
144
|
+
DeprecationWarning,
|
|
145
|
+
)
|
|
146
|
+
return value
|
|
147
|
+
|
|
148
|
+
|
|
138
149
|
# fmt: off
|
|
139
150
|
@main.command()
|
|
140
151
|
@click.option("--storage", type=str, help="Local directory, s3://bucket_name, gs://bucket_name.") # noqa: E501
|
|
141
152
|
@click.option("--db", type=str, default=None, help="Postgres database connection URL, do not pass for SQLite.") # noqa: E501
|
|
142
|
-
@click.option("--
|
|
153
|
+
@click.option("--modules", type=str, default=None, help="Comma-separated string of modules.") # noqa: E501
|
|
143
154
|
@click.option("--name", type=str, default=None, help="The instance name.")
|
|
155
|
+
@click.option("--schema", type=str, default=None, help="[DEPRECATED] Use --modules instead.", callback=schema_to_modules_callback) # noqa: E501
|
|
144
156
|
# fmt: on
|
|
145
|
-
def init(
|
|
157
|
+
def init(
|
|
158
|
+
storage: str,
|
|
159
|
+
db: Optional[str],
|
|
160
|
+
modules: Optional[str],
|
|
161
|
+
name: Optional[str],
|
|
162
|
+
schema: Optional[str],
|
|
163
|
+
):
|
|
146
164
|
"""Init an instance."""
|
|
147
165
|
from lamindb_setup._init_instance import init as init_
|
|
148
166
|
|
|
149
|
-
|
|
167
|
+
modules = modules if modules is not None else schema
|
|
168
|
+
|
|
169
|
+
return init_(storage=storage, db=db, modules=modules, name=name)
|
|
150
170
|
|
|
151
171
|
|
|
152
172
|
# fmt: off
|
|
@@ -165,7 +185,7 @@ def connect(instance: str):
|
|
|
165
185
|
from lamindb_setup import settings as settings_, connect as connect_
|
|
166
186
|
|
|
167
187
|
settings_.auto_connect = True
|
|
168
|
-
return connect_(slug=instance)
|
|
188
|
+
return connect_(slug=instance, _reload_lamindb=False)
|
|
169
189
|
|
|
170
190
|
|
|
171
191
|
@main.command()
|
|
@@ -180,7 +200,7 @@ def disconnect():
|
|
|
180
200
|
|
|
181
201
|
|
|
182
202
|
@main.command()
|
|
183
|
-
@click.option("--schema", is_flag=True, help="View schema.")
|
|
203
|
+
@click.option("--schema", is_flag=True, help="View database schema.")
|
|
184
204
|
def info(schema: bool):
|
|
185
205
|
"""Show info about current instance."""
|
|
186
206
|
if schema:
|
|
@@ -239,7 +259,7 @@ def load(entity: str, uid: str = None, key: str = None, with_env: bool = False):
|
|
|
239
259
|
# f"! please use: lamin connect {entity}"
|
|
240
260
|
# )
|
|
241
261
|
settings_.auto_connect = True
|
|
242
|
-
return connect(slug=entity)
|
|
262
|
+
return connect(slug=entity, _reload_lamindb=False)
|
|
243
263
|
else:
|
|
244
264
|
from lamin_cli._load import load as load_
|
|
245
265
|
|
|
@@ -265,9 +285,7 @@ def get(entity: str, uid: str = None, key: str = None, with_env: bool = False):
|
|
|
265
285
|
|
|
266
286
|
|
|
267
287
|
@main.command()
|
|
268
|
-
@click.argument(
|
|
269
|
-
"filepath", type=click.Path(exists=True, dir_okay=False, file_okay=True)
|
|
270
|
-
)
|
|
288
|
+
@click.argument("filepath", type=click.Path(exists=True, dir_okay=True, file_okay=True))
|
|
271
289
|
@click.option("--key", type=str, default=None)
|
|
272
290
|
@click.option("--description", type=str, default=None)
|
|
273
291
|
@click.option("--registry", type=str, default=None)
|
|
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
from typing import Tuple
|
|
3
3
|
from lamin_utils import logger
|
|
4
4
|
import shutil
|
|
5
|
+
import re
|
|
5
6
|
from pathlib import Path
|
|
6
7
|
|
|
7
8
|
|
|
@@ -27,7 +28,7 @@ def load(entity: str, uid: str = None, key: str = None, with_env: bool = False):
|
|
|
27
28
|
instance = ln_setup.settings.instance.slug
|
|
28
29
|
|
|
29
30
|
ln_setup.connect(instance)
|
|
30
|
-
|
|
31
|
+
import lamindb as ln
|
|
31
32
|
|
|
32
33
|
def script_to_notebook(
|
|
33
34
|
transform: ln.Transform, notebook_path: Path, bump_revision: bool = False
|
|
@@ -35,16 +36,48 @@ def load(entity: str, uid: str = None, key: str = None, with_env: bool = False):
|
|
|
35
36
|
import jupytext
|
|
36
37
|
from lamin_utils._base62 import increment_base62
|
|
37
38
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
39
|
+
if notebook_path.suffix == ".ipynb":
|
|
40
|
+
# below is backward compat
|
|
41
|
+
if "# # transform.name" in transform.source_code:
|
|
42
|
+
new_content = transform.source_code.replace(
|
|
43
|
+
"# # transform.name", f"# # {transform.description}"
|
|
44
|
+
)
|
|
45
|
+
elif transform.source_code.startswith("# %% [markdown]\n#\n"):
|
|
46
|
+
new_content = transform.source_code.replace(
|
|
47
|
+
"# %% [markdown]\n#\n",
|
|
48
|
+
f"# %% [markdown]\n# # {transform.description}\n",
|
|
49
|
+
)
|
|
50
|
+
else: # R notebook
|
|
51
|
+
# Pattern to match title only within YAML header section
|
|
52
|
+
title_pattern = r'^---\n.*?title:\s*"([^"]*)".*?---'
|
|
53
|
+
title_match = re.search(
|
|
54
|
+
title_pattern, transform.source_code, flags=re.DOTALL | re.MULTILINE
|
|
55
|
+
)
|
|
56
|
+
new_content = transform.source_code
|
|
57
|
+
if title_match:
|
|
58
|
+
current_title = title_match.group(1)
|
|
59
|
+
if current_title != transform.description:
|
|
60
|
+
pattern = r'^(---\n.*?title:\s*)"([^"]*)"(.*?---)'
|
|
61
|
+
replacement = f'\\1"{transform.description}"\\3'
|
|
62
|
+
new_content = re.sub(
|
|
63
|
+
pattern,
|
|
64
|
+
replacement,
|
|
65
|
+
new_content,
|
|
66
|
+
flags=re.DOTALL | re.MULTILINE,
|
|
67
|
+
)
|
|
68
|
+
logger.important(
|
|
69
|
+
f"fixed title: {current_title} → {transform.description}"
|
|
70
|
+
)
|
|
41
71
|
if bump_revision:
|
|
42
72
|
uid = transform.uid
|
|
43
73
|
new_uid = f"{uid[:-4]}{increment_base62(uid[-4:])}"
|
|
44
|
-
|
|
74
|
+
new_content = new_content.replace(uid, new_uid)
|
|
45
75
|
logger.important(f"updated uid: {uid} → {new_uid}")
|
|
46
|
-
|
|
47
|
-
|
|
76
|
+
if notebook_path.suffix == ".ipynb":
|
|
77
|
+
notebook = jupytext.reads(new_content, fmt="py:percent")
|
|
78
|
+
jupytext.write(notebook, notebook_path)
|
|
79
|
+
else:
|
|
80
|
+
notebook_path.write_text(new_content)
|
|
48
81
|
|
|
49
82
|
query_by_uid = uid is not None
|
|
50
83
|
|
|
@@ -61,42 +94,36 @@ def load(entity: str, uid: str = None, key: str = None, with_env: bool = False):
|
|
|
61
94
|
transforms = ln.Transform.objects.filter(key=key, source_code__isnull=False)
|
|
62
95
|
|
|
63
96
|
if (n_transforms := len(transforms)) == 0:
|
|
64
|
-
err_msg =
|
|
65
|
-
f"uid strating with {uid}"
|
|
66
|
-
if query_by_uid
|
|
67
|
-
else f"key={key} and source_code"
|
|
68
|
-
)
|
|
97
|
+
err_msg = f"uid {uid}" if query_by_uid else f"key={key} and source_code"
|
|
69
98
|
raise SystemExit(f"Transform with {err_msg} does not exist.")
|
|
70
99
|
|
|
71
100
|
if n_transforms > 1:
|
|
72
101
|
transforms = transforms.order_by("-created_at")
|
|
73
102
|
transform = transforms.first()
|
|
74
103
|
|
|
75
|
-
|
|
76
|
-
if
|
|
77
|
-
|
|
104
|
+
target_relpath = Path(transform.key)
|
|
105
|
+
if len(target_relpath.parents) > 1:
|
|
106
|
+
logger.important(
|
|
107
|
+
"preserve the folder structure for versioning:"
|
|
108
|
+
f" {target_relpath.parent}/"
|
|
109
|
+
)
|
|
110
|
+
target_relpath.parent.mkdir(parents=True, exist_ok=True)
|
|
111
|
+
if target_relpath.exists():
|
|
112
|
+
response = input(f"! {target_relpath} exists: replace? (y/n)")
|
|
78
113
|
if response != "y":
|
|
79
114
|
raise SystemExit("Aborted.")
|
|
80
|
-
if transform._source_code_artifact_id is not None: # backward compat
|
|
81
|
-
# need lamindb here to have .cache() available
|
|
82
|
-
import lamindb as ln
|
|
83
115
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
target_filename += transform._source_code_artifact.suffix
|
|
88
|
-
shutil.move(filepath_cache, target_filename)
|
|
89
|
-
elif transform.source_code is not None:
|
|
90
|
-
if transform.key.endswith(".ipynb"):
|
|
91
|
-
script_to_notebook(transform, target_filename, bump_revision=True)
|
|
116
|
+
if transform.source_code is not None:
|
|
117
|
+
if target_relpath.suffix in (".ipynb", ".Rmd", ".qmd"):
|
|
118
|
+
script_to_notebook(transform, target_relpath, bump_revision=True)
|
|
92
119
|
else:
|
|
93
|
-
|
|
120
|
+
target_relpath.write_text(transform.source_code)
|
|
94
121
|
else:
|
|
95
122
|
raise SystemExit("No source code available for this transform.")
|
|
96
|
-
logger.important(f"{transform.type} is here: {target_filename}")
|
|
97
|
-
if with_env:
|
|
98
|
-
import lamindb as ln
|
|
99
123
|
|
|
124
|
+
logger.important(f"{transform.type} is here: {target_relpath}")
|
|
125
|
+
|
|
126
|
+
if with_env:
|
|
100
127
|
ln.settings.track_run_inputs = False
|
|
101
128
|
if (
|
|
102
129
|
transform.latest_run is not None
|
|
@@ -104,16 +131,15 @@ def load(entity: str, uid: str = None, key: str = None, with_env: bool = False):
|
|
|
104
131
|
):
|
|
105
132
|
filepath_env_cache = transform.latest_run.environment.cache()
|
|
106
133
|
target_env_filename = (
|
|
107
|
-
|
|
134
|
+
target_relpath.parent / f"{target_relpath.stem}__requirements.txt"
|
|
108
135
|
)
|
|
109
136
|
shutil.move(filepath_env_cache, target_env_filename)
|
|
110
137
|
logger.important(f"environment is here: {target_env_filename}")
|
|
111
138
|
else:
|
|
112
139
|
logger.warning("latest transform run with environment doesn't exist")
|
|
113
|
-
return target_filename
|
|
114
|
-
elif entity == "artifact":
|
|
115
|
-
import lamindb as ln
|
|
116
140
|
|
|
141
|
+
return target_relpath
|
|
142
|
+
elif entity == "artifact":
|
|
117
143
|
ln.settings.track_run_inputs = False
|
|
118
144
|
|
|
119
145
|
if query_by_uid:
|
|
@@ -124,7 +150,7 @@ def load(entity: str, uid: str = None, key: str = None, with_env: bool = False):
|
|
|
124
150
|
artifacts = ln.Artifact.filter(key=key)
|
|
125
151
|
|
|
126
152
|
if (n_artifacts := len(artifacts)) == 0:
|
|
127
|
-
err_msg = f"uid
|
|
153
|
+
err_msg = f"uid={uid}" if query_by_uid else f"key={key}"
|
|
128
154
|
raise SystemExit(f"Artifact with {err_msg} does not exist.")
|
|
129
155
|
|
|
130
156
|
if n_artifacts > 1:
|
|
@@ -1,43 +1,27 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
from pathlib import Path
|
|
3
3
|
from typing import Union
|
|
4
|
-
import lamindb_setup as ln_setup
|
|
5
4
|
from lamin_utils import logger
|
|
6
5
|
import re
|
|
7
6
|
|
|
8
7
|
|
|
9
|
-
def parse_uid_from_code(
|
|
10
|
-
content: str, suffix: str
|
|
11
|
-
) -> tuple[str | None, str | None, str | None]:
|
|
8
|
+
def parse_uid_from_code(content: str, suffix: str) -> str | None:
|
|
12
9
|
if suffix == ".py":
|
|
13
10
|
track_pattern = re.compile(
|
|
14
11
|
r'ln\.track\(\s*(?:transform\s*=\s*)?(["\'])([a-zA-Z0-9]{16})\1'
|
|
15
12
|
)
|
|
16
|
-
# backward compat
|
|
17
13
|
uid_pattern = re.compile(r'\.context\.uid\s*=\s*["\']([^"\']+)["\']')
|
|
18
|
-
stem_uid_pattern = re.compile(
|
|
19
|
-
r'\.transform\.stem_uid\s*=\s*["\']([^"\']+)["\']'
|
|
20
|
-
)
|
|
21
|
-
version_pattern = re.compile(r'\.transform\.version\s*=\s*["\']([^"\']+)["\']')
|
|
22
14
|
elif suffix == ".ipynb":
|
|
23
15
|
track_pattern = re.compile(
|
|
24
16
|
r'ln\.track\(\s*(?:transform\s*=\s*)?(?:\\"|\')([a-zA-Z0-9]{16})(?:\\"|\')'
|
|
25
17
|
)
|
|
26
18
|
# backward compat
|
|
27
19
|
uid_pattern = re.compile(r'\.context\.uid\s*=\s*\\["\']([^"\']+)\\["\']')
|
|
28
|
-
stem_uid_pattern = re.compile(
|
|
29
|
-
r'\.transform\.stem_uid\s*=\s*\\["\']([^"\']+)\\["\']'
|
|
30
|
-
)
|
|
31
|
-
version_pattern = re.compile(
|
|
32
|
-
r'\.transform\.version\s*=\s*\\["\']([^"\']+)\\["\']'
|
|
33
|
-
)
|
|
34
20
|
elif suffix in {".R", ".qmd", ".Rmd"}:
|
|
35
21
|
track_pattern = re.compile(
|
|
36
22
|
r'track\(\s*(?:transform\s*=\s*)?([\'"])([a-zA-Z0-9]{16})\1'
|
|
37
23
|
)
|
|
38
24
|
uid_pattern = None
|
|
39
|
-
stem_uid_pattern = None
|
|
40
|
-
version_pattern = None
|
|
41
25
|
else:
|
|
42
26
|
raise SystemExit(
|
|
43
27
|
"Only .py, .ipynb, .R, .qmd, .Rmd files are supported for saving"
|
|
@@ -48,26 +32,12 @@ def parse_uid_from_code(
|
|
|
48
32
|
uid_match = track_pattern.search(content)
|
|
49
33
|
group_index = 1 if suffix == ".ipynb" else 2
|
|
50
34
|
uid = uid_match.group(group_index) if uid_match else None
|
|
51
|
-
stem_uid = None
|
|
52
|
-
version = None
|
|
53
35
|
|
|
54
36
|
if uid_pattern is not None and uid is None:
|
|
55
37
|
uid_match = uid_pattern.search(content)
|
|
56
38
|
uid = uid_match.group(1) if uid_match else None
|
|
57
|
-
if stem_uid_pattern is not None:
|
|
58
|
-
stem_uid_match = stem_uid_pattern.search(content)
|
|
59
|
-
stem_uid = stem_uid_match.group(1) if stem_uid_match else None
|
|
60
|
-
if version_pattern is not None:
|
|
61
|
-
version_match = version_pattern.search(content)
|
|
62
|
-
version = version_match.group(1) if version_match else None
|
|
63
|
-
|
|
64
|
-
if uid is None and (stem_uid is None or version is None):
|
|
65
|
-
target = "script" if suffix in {".py", ".R"} else "notebook"
|
|
66
|
-
raise SystemExit(
|
|
67
|
-
f"Cannot infer transform uid. Did you run `ln.track()` in your {target}?"
|
|
68
|
-
)
|
|
69
39
|
|
|
70
|
-
return uid
|
|
40
|
+
return uid
|
|
71
41
|
|
|
72
42
|
|
|
73
43
|
def save_from_filepath_cli(
|
|
@@ -76,6 +46,8 @@ def save_from_filepath_cli(
|
|
|
76
46
|
description: str | None,
|
|
77
47
|
registry: str | None,
|
|
78
48
|
) -> str | None:
|
|
49
|
+
import lamindb_setup as ln_setup
|
|
50
|
+
|
|
79
51
|
if not isinstance(filepath, Path):
|
|
80
52
|
filepath = Path(filepath)
|
|
81
53
|
|
|
@@ -94,14 +66,23 @@ def save_from_filepath_cli(
|
|
|
94
66
|
"R": set([".R", ".qmd", ".Rmd"]),
|
|
95
67
|
}
|
|
96
68
|
|
|
97
|
-
if
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
69
|
+
if filepath.suffix in {".qmd", ".Rmd"}:
|
|
70
|
+
if not (
|
|
71
|
+
filepath.with_suffix(".html").exists()
|
|
72
|
+
or filepath.with_suffix(".nb.html").exists()
|
|
73
|
+
):
|
|
74
|
+
raise SystemExit(
|
|
75
|
+
f"Please export your {filepath.suffix} file as an html file here"
|
|
76
|
+
f" {filepath.with_suffix('.html')}"
|
|
77
|
+
)
|
|
78
|
+
if (
|
|
79
|
+
filepath.with_suffix(".html").exists()
|
|
80
|
+
and filepath.with_suffix(".nb.html").exists()
|
|
81
|
+
):
|
|
82
|
+
raise SystemExit(
|
|
83
|
+
f'Please delete one of\n - {filepath.with_suffix(".html")}\n -'
|
|
84
|
+
f' {filepath.with_suffix(".nb.html")}'
|
|
85
|
+
)
|
|
105
86
|
|
|
106
87
|
if registry is None:
|
|
107
88
|
registry = (
|
|
@@ -128,9 +109,9 @@ def save_from_filepath_cli(
|
|
|
128
109
|
elif registry == "transform":
|
|
129
110
|
with open(filepath) as file:
|
|
130
111
|
content = file.read()
|
|
131
|
-
uid
|
|
132
|
-
logger.important(f"mapped '{filepath}' on uid '{uid}'")
|
|
112
|
+
uid = parse_uid_from_code(content, filepath.suffix)
|
|
133
113
|
if uid is not None:
|
|
114
|
+
logger.important(f"mapped '{filepath}' on uid '{uid}'")
|
|
134
115
|
transform = ln.Transform.filter(uid=uid).one_or_none()
|
|
135
116
|
if transform is None:
|
|
136
117
|
logger.error(
|
|
@@ -139,10 +120,17 @@ def save_from_filepath_cli(
|
|
|
139
120
|
)
|
|
140
121
|
return "not-tracked-in-transform-registry"
|
|
141
122
|
else:
|
|
142
|
-
transform = ln.Transform.
|
|
123
|
+
transform = ln.Transform.filter(key=filepath.name).one_or_none()
|
|
124
|
+
if transform is None:
|
|
125
|
+
transform = ln.Transform(
|
|
126
|
+
name=filepath.name,
|
|
127
|
+
key=filepath.name,
|
|
128
|
+
type="script" if filepath.suffix in {".R", ".py"} else "notebook",
|
|
129
|
+
).save()
|
|
130
|
+
logger.important(f"created Transform('{transform.uid}')")
|
|
143
131
|
# latest run of this transform by user
|
|
144
132
|
run = ln.Run.filter(transform=transform).order_by("-started_at").first()
|
|
145
|
-
if run.created_by.id != ln_setup.settings.user.id:
|
|
133
|
+
if run is not None and run.created_by.id != ln_setup.settings.user.id:
|
|
146
134
|
response = input(
|
|
147
135
|
"You are trying to save a transform created by another user: Source"
|
|
148
136
|
" and report files will be tagged with *your* user id. Proceed?"
|
|
@@ -156,16 +144,6 @@ def save_from_filepath_cli(
|
|
|
156
144
|
filepath=filepath,
|
|
157
145
|
from_cli=True,
|
|
158
146
|
)
|
|
159
|
-
if filepath.suffix in {".qmd", ".Rmd"}:
|
|
160
|
-
report_file = ln.Artifact(
|
|
161
|
-
filepath.with_suffix(".html"), # validated at the top that this exists
|
|
162
|
-
description=f"Report of run {run.uid}",
|
|
163
|
-
visibility=0, # hidden file
|
|
164
|
-
run=False,
|
|
165
|
-
)
|
|
166
|
-
report_file.save(upload=True, print_progress=False)
|
|
167
|
-
run.report = report_file
|
|
168
|
-
run.save()
|
|
169
147
|
return return_code
|
|
170
148
|
else:
|
|
171
149
|
raise SystemExit("Allowed values for '--registry' are: 'artifact', 'transform'")
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
from lamin_cli._load import decompose_url
|
|
2
|
+
from pathlib import Path
|
|
2
3
|
import subprocess
|
|
3
4
|
|
|
4
5
|
|
|
@@ -15,26 +16,50 @@ def test_decompose_url():
|
|
|
15
16
|
assert uid == "1GCKs8zLtkc85zKv"
|
|
16
17
|
|
|
17
18
|
|
|
18
|
-
def
|
|
19
|
+
def test_load_transform():
|
|
20
|
+
import lamindb_setup as ln_setup
|
|
21
|
+
|
|
22
|
+
print(ln_setup.settings.instance.slug)
|
|
19
23
|
result = subprocess.run(
|
|
20
24
|
"lamin load"
|
|
21
|
-
" 'https://lamin.ai/laminlabs/
|
|
25
|
+
" 'https://lamin.ai/laminlabs/lamin-dev/transform/VFYCIuaw2GsX0000'"
|
|
22
26
|
" --with-env", # noqa
|
|
23
27
|
shell=True,
|
|
24
28
|
capture_output=True,
|
|
25
29
|
)
|
|
30
|
+
print(result.stdout.decode())
|
|
31
|
+
print(result.stderr.decode())
|
|
26
32
|
assert result.returncode == 0
|
|
27
33
|
|
|
34
|
+
print(ln_setup.settings.instance.slug)
|
|
35
|
+
|
|
36
|
+
path1 = Path("run-track-and-finish.py")
|
|
37
|
+
path2 = Path("run-track-and-finish__requirements.txt")
|
|
38
|
+
assert path1.exists()
|
|
39
|
+
assert path2.exists()
|
|
40
|
+
|
|
41
|
+
# below will fail because it will say "these files already exist"
|
|
42
|
+
result = subprocess.run(
|
|
43
|
+
"lamin load transform --uid VFYCIuaw2GsX --with-env",
|
|
44
|
+
shell=True,
|
|
45
|
+
capture_output=True,
|
|
46
|
+
)
|
|
47
|
+
assert result.returncode == 1
|
|
48
|
+
path1.unlink()
|
|
49
|
+
path2.unlink()
|
|
50
|
+
|
|
28
51
|
# partial uid
|
|
29
52
|
result = subprocess.run(
|
|
30
|
-
"lamin load transform --uid
|
|
53
|
+
"lamin load transform --uid VFYCIuaw2GsX --with-env",
|
|
31
54
|
shell=True,
|
|
32
55
|
capture_output=True,
|
|
33
56
|
)
|
|
34
57
|
assert result.returncode == 0
|
|
58
|
+
path1.unlink()
|
|
59
|
+
path2.unlink()
|
|
35
60
|
|
|
36
61
|
|
|
37
|
-
def
|
|
62
|
+
def test_load_artifact():
|
|
38
63
|
result = subprocess.run(
|
|
39
64
|
"lamin load"
|
|
40
65
|
" 'https://lamin.ai/laminlabs/lamin-site-assets/artifact/e2G7k9EVul4JbfsEYAy5'", # noqa
|
|
@@ -15,12 +15,12 @@ def test_migrate_deploy():
|
|
|
15
15
|
import lamindb as ln
|
|
16
16
|
|
|
17
17
|
instance_slug = ln.setup.settings.instance.slug
|
|
18
|
-
exit_status = os.system(
|
|
19
|
-
|
|
20
|
-
)
|
|
21
|
-
assert exit_status == 0
|
|
22
|
-
exit_status = os.system("lamin migrate deploy")
|
|
23
|
-
assert exit_status == 0
|
|
18
|
+
# exit_status = os.system(
|
|
19
|
+
# "lamin connect testuser1/static-test-instance-private-sqlite"
|
|
20
|
+
# )
|
|
21
|
+
# assert exit_status == 0
|
|
22
|
+
# exit_status = os.system("lamin migrate deploy")
|
|
23
|
+
# assert exit_status == 0
|
|
24
24
|
# now test that the hub got populated with the correct lamindb version
|
|
25
25
|
# test it once we integrated it in the CLI output
|
|
26
26
|
# instance = call_with_fallback_auth(
|
|
@@ -35,6 +35,6 @@ def test_migrate_deploy():
|
|
|
35
35
|
|
|
36
36
|
# def test_migrate_squash():
|
|
37
37
|
# exit_status = os.system(
|
|
38
|
-
# "yes | lamin migrate squash --package-name
|
|
38
|
+
# "yes | lamin migrate squash --package-name lamindb --end-number 0023"
|
|
39
39
|
# )
|
|
40
40
|
# assert exit_status == 0
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import pytest
|
|
2
1
|
from lamin_cli._save import parse_uid_from_code
|
|
3
2
|
|
|
4
3
|
|
|
@@ -33,13 +32,12 @@ def test_python_track_pattern():
|
|
|
33
32
|
|
|
34
33
|
# Test valid cases
|
|
35
34
|
for content, expected_uid in valid_cases:
|
|
36
|
-
uid
|
|
35
|
+
uid = parse_uid_from_code(content, ".py")
|
|
37
36
|
assert uid == expected_uid, f"Failed for valid content: {content}"
|
|
38
37
|
|
|
39
38
|
# Test invalid cases
|
|
40
39
|
for content in invalid_cases:
|
|
41
|
-
|
|
42
|
-
uid, _, _ = parse_uid_from_code(content, ".py")
|
|
40
|
+
assert parse_uid_from_code(content, ".py") is None
|
|
43
41
|
|
|
44
42
|
|
|
45
43
|
def test_jupyter_track_pattern():
|
|
@@ -70,14 +68,12 @@ def test_jupyter_track_pattern():
|
|
|
70
68
|
|
|
71
69
|
# Test valid cases
|
|
72
70
|
for content, expected_uid in valid_cases:
|
|
73
|
-
uid
|
|
71
|
+
uid = parse_uid_from_code(content, ".ipynb")
|
|
74
72
|
assert uid == expected_uid, f"Failed for valid content: {content}"
|
|
75
73
|
|
|
76
74
|
# Test invalid cases
|
|
77
75
|
for content in invalid_cases:
|
|
78
|
-
|
|
79
|
-
with pytest.raises(SystemExit):
|
|
80
|
-
uid, _, _ = parse_uid_from_code(content, ".py")
|
|
76
|
+
assert parse_uid_from_code(content, ".py") is None
|
|
81
77
|
|
|
82
78
|
|
|
83
79
|
def test_edge_cases():
|
|
@@ -99,8 +95,7 @@ def test_edge_cases():
|
|
|
99
95
|
]
|
|
100
96
|
|
|
101
97
|
for content, expected_uid, suffix in test_cases:
|
|
102
|
-
|
|
103
|
-
uid, _, _ = parse_uid_from_code(content, suffix)
|
|
98
|
+
uid = parse_uid_from_code(content, suffix)
|
|
104
99
|
assert uid == expected_uid, f"Failed for content: {content}"
|
|
105
100
|
|
|
106
101
|
|
|
@@ -116,7 +111,7 @@ def test_r_track_pattern():
|
|
|
116
111
|
# Test valid cases across all R-related suffixes
|
|
117
112
|
for suffix in suffixes:
|
|
118
113
|
for content, expected_uid in valid_cases:
|
|
119
|
-
uid
|
|
114
|
+
uid = parse_uid_from_code(content, suffix)
|
|
120
115
|
assert (
|
|
121
116
|
uid == expected_uid
|
|
122
117
|
), f"Failed for valid content with {suffix}: {content}"
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import subprocess
|
|
2
|
+
import lamindb_setup as ln_setup
|
|
2
3
|
from pathlib import Path
|
|
3
4
|
|
|
4
5
|
test_file = Path(__file__).parent.parent.resolve() / ".gitignore"
|
|
@@ -21,6 +22,8 @@ def test_save_file():
|
|
|
21
22
|
)
|
|
22
23
|
assert result.returncode == 1
|
|
23
24
|
|
|
25
|
+
print(ln_setup.settings.instance.slug)
|
|
26
|
+
|
|
24
27
|
result = subprocess.run(
|
|
25
28
|
f"lamin save {filepath} --key mytest",
|
|
26
29
|
shell=True,
|
|
@@ -20,11 +20,7 @@ def test_save_not_initialized():
|
|
|
20
20
|
capture_output=True,
|
|
21
21
|
env=env,
|
|
22
22
|
)
|
|
23
|
-
assert result.returncode ==
|
|
24
|
-
assert (
|
|
25
|
-
"Cannot infer transform uid. Did you run `ln.track()` in your notebook?"
|
|
26
|
-
in result.stderr.decode()
|
|
27
|
-
)
|
|
23
|
+
assert result.returncode == 0
|
|
28
24
|
|
|
29
25
|
|
|
30
26
|
def test_save_non_consecutive():
|
|
@@ -87,7 +83,7 @@ def test_save_consecutive():
|
|
|
87
83
|
transform = ln.Transform.filter(uid="hlsFXswrJjtt0000").one_or_none()
|
|
88
84
|
assert transform is not None
|
|
89
85
|
assert transform.latest_run.report is None
|
|
90
|
-
assert transform.
|
|
86
|
+
assert transform.source_code is None
|
|
91
87
|
assert transform.latest_run.environment is None
|
|
92
88
|
|
|
93
89
|
# and save again
|
|
@@ -107,7 +103,7 @@ def test_save_consecutive():
|
|
|
107
103
|
assert (
|
|
108
104
|
transform.source_code
|
|
109
105
|
== """# %% [markdown]
|
|
110
|
-
#
|
|
106
|
+
#
|
|
111
107
|
|
|
112
108
|
# %%
|
|
113
109
|
import lamindb as ln
|
|
@@ -119,7 +115,7 @@ ln.track("hlsFXswrJjtt0000")
|
|
|
119
115
|
print("my consecutive cell")
|
|
120
116
|
"""
|
|
121
117
|
)
|
|
122
|
-
assert transform.hash == "
|
|
118
|
+
assert transform.hash == "ik5Dilxs2RmwOGydohFolQ"
|
|
123
119
|
# below is the test that we can use if store the run repot as `.ipynb`
|
|
124
120
|
# and not as html as we do right now
|
|
125
121
|
assert transform.latest_run.report.suffix == ".html"
|
|
@@ -133,7 +129,6 @@ print("my consecutive cell")
|
|
|
133
129
|
# }
|
|
134
130
|
# testing for the hash of the report makes no sense because it contains timestamps
|
|
135
131
|
assert transform.latest_run.environment.path.exists()
|
|
136
|
-
assert transform._source_code_artifact is None
|
|
137
132
|
|
|
138
133
|
# edit the notebook
|
|
139
134
|
nb = read_notebook(notebook_path)
|
|
@@ -167,9 +162,8 @@ print("my consecutive cell")
|
|
|
167
162
|
transform = ln.Transform.get("hlsFXswrJjtt0000")
|
|
168
163
|
assert transform.latest_run.report.path.exists()
|
|
169
164
|
assert transform.latest_run.report.path == transform.latest_run.report.path
|
|
170
|
-
assert transform.hash == "
|
|
165
|
+
assert transform.hash == "Jv0_TrZfzM-0erbp1FGdrQ"
|
|
171
166
|
assert transform.latest_run.environment.path.exists()
|
|
172
|
-
assert transform._source_code_artifact is None
|
|
173
167
|
|
|
174
168
|
# get the the source code via command line
|
|
175
169
|
result = subprocess.run(
|
|
@@ -8,7 +8,23 @@ from lamindb_setup import settings
|
|
|
8
8
|
scripts_dir = Path(__file__).parent.resolve() / "scripts"
|
|
9
9
|
|
|
10
10
|
|
|
11
|
-
def
|
|
11
|
+
def test_save_without_uid():
|
|
12
|
+
env = os.environ
|
|
13
|
+
env["LAMIN_TESTING"] = "true"
|
|
14
|
+
filepath = scripts_dir / "run-track-and-finish.py"
|
|
15
|
+
|
|
16
|
+
# attempt to save the script without it yet being run
|
|
17
|
+
result = subprocess.run(
|
|
18
|
+
f"lamin save {filepath}",
|
|
19
|
+
shell=True,
|
|
20
|
+
capture_output=True,
|
|
21
|
+
)
|
|
22
|
+
# print(result.stdout.decode())
|
|
23
|
+
assert result.returncode == 0
|
|
24
|
+
assert "created Transform" in result.stdout.decode()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_run_save_cache_with_git_and_uid():
|
|
12
28
|
env = os.environ
|
|
13
29
|
env["LAMIN_TESTING"] = "true"
|
|
14
30
|
filepath = scripts_dir / "run-track-and-finish-sync-git.py"
|
|
@@ -39,7 +55,6 @@ def test_run_save_cache():
|
|
|
39
55
|
transform = ln.Transform.get("m5uCHTTpJnjQ")
|
|
40
56
|
assert transform.hash == "MoIciBQ0lpVPCKQGofPX6g"
|
|
41
57
|
assert transform.latest_run.environment.path.exists()
|
|
42
|
-
assert transform._source_code_artifact is None
|
|
43
58
|
|
|
44
59
|
# you can rerun the same script
|
|
45
60
|
result = subprocess.run(
|
|
@@ -54,7 +69,6 @@ def test_run_save_cache():
|
|
|
54
69
|
assert "loaded Transform" in result.stdout.decode()
|
|
55
70
|
assert "m5uCHTTp" in result.stdout.decode()
|
|
56
71
|
assert "started Run" in result.stdout.decode()
|
|
57
|
-
assert "source code is already saved" in result.stdout.decode()
|
|
58
72
|
|
|
59
73
|
# you can re-save the script
|
|
60
74
|
result = subprocess.run(
|
|
@@ -66,7 +80,6 @@ def test_run_save_cache():
|
|
|
66
80
|
# print(result.stdout.decode())
|
|
67
81
|
# print(result.stderr.decode())
|
|
68
82
|
assert result.returncode == 0
|
|
69
|
-
assert "source code is already saved" in result.stdout.decode()
|
|
70
83
|
assert "run.environment is already saved" in result.stdout.decode()
|
|
71
84
|
|
|
72
85
|
# edit the script
|
|
@@ -105,7 +118,7 @@ def test_run_save_cache():
|
|
|
105
118
|
# print(result.stdout.decode())
|
|
106
119
|
# print(result.stderr.decode())
|
|
107
120
|
assert result.returncode == 1
|
|
108
|
-
assert "
|
|
121
|
+
assert "✗ source code changed, run:" in result.stderr.decode()
|
|
109
122
|
|
|
110
123
|
# update the uid
|
|
111
124
|
content = filepath.read_text()
|
|
@@ -204,7 +217,6 @@ def test_run_save_with_params():
|
|
|
204
217
|
print(result.stdout.decode())
|
|
205
218
|
print(result.stderr.decode())
|
|
206
219
|
assert result.returncode == 0
|
|
207
|
-
assert "source code is already saved" in result.stdout.decode()
|
|
208
220
|
assert (
|
|
209
221
|
"run-track-with-params.py' on uid 'JjRF4mACd9m00000'" in result.stdout.decode()
|
|
210
222
|
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{lamin_cli-0.21.4 → lamin_cli-1.0.0}/tests/notebooks/with-title-and-initialized-consecutive.ipynb
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|