lamindb 1.3.2__py3-none-any.whl → 1.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lamindb/__init__.py +52 -36
- lamindb/_finish.py +17 -10
- lamindb/_tracked.py +1 -1
- lamindb/base/__init__.py +3 -1
- lamindb/base/fields.py +40 -22
- lamindb/base/ids.py +1 -94
- lamindb/base/types.py +2 -0
- lamindb/base/uids.py +117 -0
- lamindb/core/_context.py +216 -133
- lamindb/core/_settings.py +38 -25
- lamindb/core/datasets/__init__.py +11 -4
- lamindb/core/datasets/_core.py +5 -5
- lamindb/core/datasets/_small.py +0 -93
- lamindb/core/datasets/mini_immuno.py +172 -0
- lamindb/core/loaders.py +1 -1
- lamindb/core/storage/_backed_access.py +100 -6
- lamindb/core/storage/_polars_lazy_df.py +51 -0
- lamindb/core/storage/_pyarrow_dataset.py +15 -30
- lamindb/core/storage/objects.py +6 -0
- lamindb/core/subsettings/__init__.py +2 -0
- lamindb/core/subsettings/_annotation_settings.py +11 -0
- lamindb/curators/__init__.py +7 -3559
- lamindb/curators/_legacy.py +2056 -0
- lamindb/curators/core.py +1546 -0
- lamindb/errors.py +11 -0
- lamindb/examples/__init__.py +27 -0
- lamindb/examples/schemas/__init__.py +12 -0
- lamindb/examples/schemas/_anndata.py +25 -0
- lamindb/examples/schemas/_simple.py +19 -0
- lamindb/integrations/_vitessce.py +8 -5
- lamindb/migrations/0091_alter_featurevalue_options_alter_space_options_and_more.py +24 -0
- lamindb/migrations/0092_alter_artifactfeaturevalue_artifact_and_more.py +75 -0
- lamindb/models/__init__.py +12 -2
- lamindb/models/_describe.py +21 -4
- lamindb/models/_feature_manager.py +384 -301
- lamindb/models/_from_values.py +1 -1
- lamindb/models/_is_versioned.py +5 -15
- lamindb/models/_label_manager.py +8 -2
- lamindb/models/artifact.py +354 -177
- lamindb/models/artifact_set.py +122 -0
- lamindb/models/can_curate.py +4 -1
- lamindb/models/collection.py +79 -56
- lamindb/models/core.py +1 -1
- lamindb/models/feature.py +78 -47
- lamindb/models/has_parents.py +24 -9
- lamindb/models/project.py +3 -3
- lamindb/models/query_manager.py +221 -22
- lamindb/models/query_set.py +251 -206
- lamindb/models/record.py +211 -344
- lamindb/models/run.py +59 -5
- lamindb/models/save.py +9 -5
- lamindb/models/schema.py +673 -196
- lamindb/models/transform.py +5 -14
- lamindb/models/ulabel.py +8 -5
- {lamindb-1.3.2.dist-info → lamindb-1.5.0.dist-info}/METADATA +8 -7
- lamindb-1.5.0.dist-info/RECORD +108 -0
- lamindb-1.3.2.dist-info/RECORD +0 -95
- {lamindb-1.3.2.dist-info → lamindb-1.5.0.dist-info}/LICENSE +0 -0
- {lamindb-1.3.2.dist-info → lamindb-1.5.0.dist-info}/WHEEL +0 -0
lamindb/core/_context.py
CHANGED
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|
2
2
|
|
3
3
|
import builtins
|
4
4
|
import hashlib
|
5
|
+
import os
|
5
6
|
import signal
|
6
7
|
import sys
|
7
8
|
import threading
|
@@ -22,7 +23,6 @@ from lamindb.models import Run, Transform, format_field_value
|
|
22
23
|
|
23
24
|
from ..core._settings import settings
|
24
25
|
from ..errors import (
|
25
|
-
InconsistentKey,
|
26
26
|
InvalidArgument,
|
27
27
|
TrackNotCalled,
|
28
28
|
UpdateContext,
|
@@ -30,7 +30,6 @@ from ..errors import (
|
|
30
30
|
from ..models._is_versioned import bump_version as bump_version_function
|
31
31
|
from ..models._is_versioned import (
|
32
32
|
increment_base62,
|
33
|
-
message_update_key_in_version_family,
|
34
33
|
)
|
35
34
|
from ._sync_git import get_transform_reference_from_git_repo
|
36
35
|
from ._track_environment import track_environment
|
@@ -39,7 +38,7 @@ if TYPE_CHECKING:
|
|
39
38
|
from lamindb_setup.core.types import UPathStr
|
40
39
|
|
41
40
|
from lamindb.base.types import TransformType
|
42
|
-
from lamindb.models import Project
|
41
|
+
from lamindb.models import Project, Space
|
43
42
|
|
44
43
|
is_run_from_ipython = getattr(builtins, "__IPYTHON__", False)
|
45
44
|
|
@@ -55,19 +54,21 @@ def get_uid_ext(version: str) -> str:
|
|
55
54
|
return encodebytes(hashlib.md5(version.encode()).digest())[:4] # noqa: S324
|
56
55
|
|
57
56
|
|
58
|
-
def get_notebook_path() -> Path:
|
57
|
+
def get_notebook_path() -> tuple[Path, str]:
|
59
58
|
from nbproject.dev._jupyter_communicate import (
|
60
59
|
notebook_path as get_notebook_path,
|
61
60
|
)
|
62
61
|
|
63
62
|
path = None
|
64
63
|
try:
|
65
|
-
path = get_notebook_path()
|
66
|
-
except
|
67
|
-
raise
|
64
|
+
path, env = get_notebook_path(return_env=True)
|
65
|
+
except ValueError as ve:
|
66
|
+
raise ve
|
67
|
+
except Exception as error:
|
68
|
+
raise RuntimeError(msg_path_failed) from error
|
68
69
|
if path is None:
|
69
70
|
raise RuntimeError(msg_path_failed) from None
|
70
|
-
return Path(path)
|
71
|
+
return Path(path), env
|
71
72
|
|
72
73
|
|
73
74
|
# from https://stackoverflow.com/questions/61901628
|
@@ -179,20 +180,7 @@ class LogStreamTracker:
|
|
179
180
|
class Context:
|
180
181
|
"""Run context.
|
181
182
|
|
182
|
-
|
183
|
-
upon :meth:`~lamindb.core.Context.track` & :meth:`~lamindb.core.Context.finish`.
|
184
|
-
|
185
|
-
Guide: :doc:`/track`
|
186
|
-
|
187
|
-
Examples:
|
188
|
-
|
189
|
-
Is typically used via the global :class:`~lamindb.context` object via `ln.track()` and `ln.finish()`:
|
190
|
-
|
191
|
-
>>> import lamindb as ln
|
192
|
-
>>> ln.track()
|
193
|
-
>>> # do things
|
194
|
-
>>> ln.finish()
|
195
|
-
|
183
|
+
Is the book keeper for :meth:`~lamindb.core.Context.track`.
|
196
184
|
"""
|
197
185
|
|
198
186
|
def __init__(self):
|
@@ -202,12 +190,14 @@ class Context:
|
|
202
190
|
self._transform: Transform | None = None
|
203
191
|
self._run: Run | None = None
|
204
192
|
self._path: Path | None = None
|
205
|
-
"""A local path to the script that's running."""
|
193
|
+
"""A local path to the script or notebook that's running."""
|
206
194
|
self._project: Project | None = None
|
195
|
+
self._space: Space | None = None
|
207
196
|
self._logging_message_track: str = ""
|
208
197
|
self._logging_message_imports: str = ""
|
209
198
|
self._stream_tracker: LogStreamTracker = LogStreamTracker()
|
210
199
|
self._is_finish_retry: bool = False
|
200
|
+
self._notebook_runner: str | None = None
|
211
201
|
|
212
202
|
@property
|
213
203
|
def transform(self) -> Transform | None:
|
@@ -255,33 +245,38 @@ class Context:
|
|
255
245
|
"""Project to label entities created during the run."""
|
256
246
|
return self._project
|
257
247
|
|
248
|
+
@property
|
249
|
+
def space(self) -> Space | None:
|
250
|
+
"""The space in which entities are created during the run."""
|
251
|
+
return self._space
|
252
|
+
|
258
253
|
@property
|
259
254
|
def run(self) -> Run | None:
|
260
255
|
"""Managed run of context."""
|
261
256
|
return self._run
|
262
257
|
|
263
|
-
def
|
258
|
+
def _track(
|
264
259
|
self,
|
265
260
|
transform: str | Transform | None = None,
|
266
261
|
*,
|
267
262
|
project: str | None = None,
|
263
|
+
space: str | None = None,
|
268
264
|
params: dict | None = None,
|
269
265
|
new_run: bool | None = None,
|
270
266
|
path: str | None = None,
|
271
267
|
) -> None:
|
272
|
-
"""Track a
|
268
|
+
"""Track a run of your notebook or script.
|
273
269
|
|
274
|
-
|
275
|
-
:attr:`~lamindb.core.Context.run` by creating or loading `Transform` &
|
276
|
-
`Run` records
|
277
|
-
- saves Python environment as a `requirements.txt` file: `run.environment`
|
270
|
+
Populates the global run :class:`~lamindb.context` by managing `Transform` & `Run` records and caching the compute environment.
|
278
271
|
|
279
|
-
If :attr:`~lamindb.core.Settings.sync_git_repo` is set, checks whether a
|
280
|
-
script-like transform exists in a git repository and links it.
|
272
|
+
If :attr:`~lamindb.core.Settings.sync_git_repo` is set, checks whether a script-like transform exists in a git repository and links it.
|
281
273
|
|
282
274
|
Args:
|
283
|
-
transform: A transform `uid` or record. If `None`, creates a `uid`.
|
275
|
+
transform: A transform (stem) `uid` (or record). If `None`, auto-creates a `transform` with its `uid`.
|
284
276
|
project: A project `name` or `uid` for labeling entities created during the run.
|
277
|
+
space: A space `name` or `uid` to identify where potentially sensitive entities are created during the run.
|
278
|
+
This doesn't affect `Storage`, `ULabel`, `Feature`, `Schema`, `Param` and bionty entities as these provide mere structure that should typically be commonly accessible.
|
279
|
+
If you want to manually move entities to a different space, set the `.space` field (:doc:`docs:access`).
|
285
280
|
params: A dictionary of parameters to track for the run.
|
286
281
|
new_run: If `False`, loads the latest run of transform
|
287
282
|
(default notebook), if `True`, creates new run (default non-notebook).
|
@@ -290,16 +285,20 @@ class Context:
|
|
290
285
|
|
291
286
|
Examples:
|
292
287
|
|
293
|
-
To track the run of a notebook or script, call
|
288
|
+
To track the run of a notebook or script, call::
|
294
289
|
|
295
|
-
|
290
|
+
ln.track()
|
291
|
+
#> → created Transform('Onv04I53OgtT0000'), started new Run('dpSfd7Ds...') at 2025-04-25 11:00:03 UTC
|
292
|
+
#> • recommendation: to identify the notebook across renames, pass the uid: ln.track("Onv04I53OgtT")
|
296
293
|
|
297
|
-
|
294
|
+
Ensure one version history across file renames::
|
298
295
|
|
299
|
-
|
296
|
+
ln.track("Onv04I53OgtT")
|
297
|
+
#> → created Transform('Onv04I53OgtT0000'), started new Run('dpSfd7Ds...') at 2025-04-25 11:00:03 UTC
|
300
298
|
|
299
|
+
More examples: :doc:`/track`
|
301
300
|
"""
|
302
|
-
from lamindb.models import Project
|
301
|
+
from lamindb.models import Project, Space
|
303
302
|
|
304
303
|
instance_settings = ln_setup.settings.instance
|
305
304
|
# similar logic here: https://github.com/laminlabs/lamindb/pull/2527
|
@@ -307,6 +306,8 @@ class Context:
|
|
307
306
|
if instance_settings.dialect == "postgresql" and "read" in instance_settings.db:
|
308
307
|
logger.warning("skipping track(), connected in read-only mode")
|
309
308
|
return None
|
309
|
+
if project is None:
|
310
|
+
project = os.environ.get("LAMIN_CURRENT_PROJECT")
|
310
311
|
if project is not None:
|
311
312
|
project_record = Project.filter(
|
312
313
|
Q(name=project) | Q(uid=project)
|
@@ -316,11 +317,21 @@ class Context:
|
|
316
317
|
f"Project '{project}' not found, either create it with `ln.Project(name='...').save()` or fix typos."
|
317
318
|
)
|
318
319
|
self._project = project_record
|
320
|
+
if space is not None:
|
321
|
+
space_record = Space.filter(Q(name=space) | Q(uid=space)).one_or_none()
|
322
|
+
if space_record is None:
|
323
|
+
raise InvalidArgument(
|
324
|
+
f"Space '{space}', please check on the hub UI whether you have the correct `uid` or `name`."
|
325
|
+
)
|
326
|
+
self._space = space_record
|
319
327
|
self._logging_message_track = ""
|
320
328
|
self._logging_message_imports = ""
|
321
329
|
if transform is not None and isinstance(transform, str):
|
322
330
|
self.uid = transform
|
323
331
|
transform = None
|
332
|
+
uid_was_none = False
|
333
|
+
else:
|
334
|
+
uid_was_none = True
|
324
335
|
self._path = None
|
325
336
|
if transform is None:
|
326
337
|
description = None
|
@@ -367,7 +378,14 @@ class Context:
|
|
367
378
|
self._transform = transform_exists
|
368
379
|
|
369
380
|
if new_run is None: # for notebooks, default to loading latest runs
|
370
|
-
new_run =
|
381
|
+
new_run = (
|
382
|
+
False
|
383
|
+
if (
|
384
|
+
self._transform.type == "notebook"
|
385
|
+
and self._notebook_runner != "nbconvert"
|
386
|
+
)
|
387
|
+
else True
|
388
|
+
) # type: ignore
|
371
389
|
|
372
390
|
run = None
|
373
391
|
if not new_run: # try loading latest run by same user
|
@@ -414,6 +432,22 @@ class Context:
|
|
414
432
|
logger.important(self._logging_message_track)
|
415
433
|
if self._logging_message_imports:
|
416
434
|
logger.important(self._logging_message_imports)
|
435
|
+
if uid_was_none:
|
436
|
+
notebook_or_script = (
|
437
|
+
"notebook" if self._transform.type == "notebook" else "script"
|
438
|
+
)
|
439
|
+
r_or_python = "."
|
440
|
+
if self._path is not None:
|
441
|
+
r_or_python = "." if self._path.suffix in {".py", ".ipynb"} else "$"
|
442
|
+
project_str = f', project="{project}"' if project is not None else ""
|
443
|
+
space_str = f', space="{space}"' if space is not None else ""
|
444
|
+
params_str = (
|
445
|
+
", params={...}" if params is not None else ""
|
446
|
+
) # do not put the values because typically parameterized by user
|
447
|
+
kwargs_str = f"{project_str}{space_str}{params_str}"
|
448
|
+
logger.important_hint(
|
449
|
+
f'recommendation: to identify the {notebook_or_script} across renames, pass the uid: ln{r_or_python}track("{self.transform.uid[:-4]}"{kwargs_str})'
|
450
|
+
)
|
417
451
|
|
418
452
|
def _track_source_code(
|
419
453
|
self,
|
@@ -455,7 +489,7 @@ class Context:
|
|
455
489
|
path_str: str | None,
|
456
490
|
) -> tuple[Path, str | None]:
|
457
491
|
if path_str is None:
|
458
|
-
path = get_notebook_path()
|
492
|
+
path, self._notebook_runner = get_notebook_path()
|
459
493
|
else:
|
460
494
|
path = Path(path_str)
|
461
495
|
description = None
|
@@ -487,6 +521,53 @@ class Context:
|
|
487
521
|
pass
|
488
522
|
return path, description
|
489
523
|
|
524
|
+
def _process_aux_transform(
|
525
|
+
self,
|
526
|
+
aux_transform: Transform,
|
527
|
+
transform_hash: str,
|
528
|
+
) -> tuple[str, Transform | None, str]:
|
529
|
+
# first part of the if condition: no version bump, second part: version bump
|
530
|
+
message = ""
|
531
|
+
if (
|
532
|
+
# if a user hasn't yet saved the transform source code AND is the same user
|
533
|
+
(
|
534
|
+
aux_transform.source_code is None
|
535
|
+
and aux_transform.created_by_id == ln_setup.settings.user.id
|
536
|
+
)
|
537
|
+
# if the transform source code is unchanged
|
538
|
+
# if aux_transform.type == "notebook", we anticipate the user makes changes to the notebook source code
|
539
|
+
# in an interactive session, hence we *pro-actively bump* the version number by setting `revises` / 'nbconvert' execution is NOT interactive
|
540
|
+
# in the second part of the if condition even though the source code is unchanged at point of running track()
|
541
|
+
or (
|
542
|
+
aux_transform.hash == transform_hash
|
543
|
+
and (
|
544
|
+
aux_transform.type != "notebook"
|
545
|
+
or self._notebook_runner == "nbconvert"
|
546
|
+
)
|
547
|
+
)
|
548
|
+
):
|
549
|
+
uid = aux_transform.uid
|
550
|
+
return uid, aux_transform, message
|
551
|
+
else:
|
552
|
+
uid = f"{aux_transform.uid[:-4]}{increment_base62(aux_transform.uid[-4:])}"
|
553
|
+
message = (
|
554
|
+
f"found {aux_transform.type} {aux_transform.key}, making new version"
|
555
|
+
)
|
556
|
+
if (
|
557
|
+
aux_transform.hash == transform_hash
|
558
|
+
and aux_transform.type == "notebook"
|
559
|
+
):
|
560
|
+
message += " -- anticipating changes"
|
561
|
+
elif aux_transform.hash != transform_hash:
|
562
|
+
message += (
|
563
|
+
"" # could log "source code changed", but this seems too much
|
564
|
+
)
|
565
|
+
elif aux_transform.created_by_id != ln_setup.settings.user.id:
|
566
|
+
message += (
|
567
|
+
f" -- {aux_transform.created_by.handle} already works on this draft"
|
568
|
+
)
|
569
|
+
return uid, None, message
|
570
|
+
|
490
571
|
def _create_or_load_transform(
|
491
572
|
self,
|
492
573
|
*,
|
@@ -495,21 +576,22 @@ class Context:
|
|
495
576
|
transform_ref_type: str | None = None,
|
496
577
|
transform_type: TransformType = None,
|
497
578
|
):
|
498
|
-
|
499
|
-
update_key_note = message_update_key_in_version_family(
|
500
|
-
suid=transform.stem_uid,
|
501
|
-
existing_key=transform.key,
|
502
|
-
new_key=key,
|
503
|
-
registry="Transform",
|
504
|
-
)
|
505
|
-
return (
|
506
|
-
f'Filepath "{key}" clashes with the existing key "{transform.key}" for uid "{transform.uid[:-4]}...."\n\nEither init a new transform with a new uid:\n\n'
|
507
|
-
f'ln.track("{ids.base62_12()}0000")\n\n{update_key_note}'
|
508
|
-
)
|
579
|
+
from .._finish import notebook_to_script
|
509
580
|
|
510
|
-
|
511
|
-
|
512
|
-
|
581
|
+
if not self._path.suffix == ".ipynb":
|
582
|
+
transform_hash, _ = hash_file(self._path)
|
583
|
+
else:
|
584
|
+
# need to convert to stripped py:percent format for hashing
|
585
|
+
source_code_path = ln_setup.settings.cache_dir / self._path.name.replace(
|
586
|
+
".ipynb", ".py"
|
587
|
+
)
|
588
|
+
notebook_to_script(description, self._path, source_code_path)
|
589
|
+
transform_hash, _ = hash_file(source_code_path)
|
590
|
+
# see whether we find a transform with the exact same hash
|
591
|
+
aux_transform = Transform.filter(hash=transform_hash).one_or_none()
|
592
|
+
# if the user did not pass a uid and there is no matching aux_transform
|
593
|
+
# need to search for the transform based on the filename
|
594
|
+
if self.uid is None and aux_transform is None:
|
513
595
|
|
514
596
|
class SlashCount(Func):
|
515
597
|
template = "LENGTH(%(expressions)s) - LENGTH(REPLACE(%(expressions)s, '/', ''))"
|
@@ -524,47 +606,15 @@ class Context:
|
|
524
606
|
uid = f"{base62_12()}0000"
|
525
607
|
key = self._path.name
|
526
608
|
target_transform = None
|
527
|
-
hash, _ = hash_file(self._path)
|
528
609
|
if len(transforms) != 0:
|
529
610
|
message = ""
|
530
611
|
found_key = False
|
531
612
|
for aux_transform in transforms:
|
532
613
|
if aux_transform.key in self._path.as_posix():
|
533
614
|
key = aux_transform.key
|
534
|
-
|
535
|
-
|
536
|
-
|
537
|
-
and (
|
538
|
-
# if the transform source code wasn't yet saved
|
539
|
-
aux_transform.source_code is None
|
540
|
-
# if the transform source code is unchanged
|
541
|
-
# if aux_transform.type == "notebook", we anticipate the user makes changes to the notebook source code
|
542
|
-
# in an interactive session, hence we *pro-actively bump* the version number by setting `revises`
|
543
|
-
# in the second part of the if condition even though the source code is unchanged at point of running track()
|
544
|
-
or (
|
545
|
-
aux_transform.hash == hash
|
546
|
-
and aux_transform.type != "notebook"
|
547
|
-
)
|
548
|
-
)
|
549
|
-
):
|
550
|
-
uid = aux_transform.uid
|
551
|
-
target_transform = aux_transform
|
552
|
-
else:
|
553
|
-
uid = f"{aux_transform.uid[:-4]}{increment_base62(aux_transform.uid[-4:])}"
|
554
|
-
message = f"there already is a {aux_transform.type} with `key` '{aux_transform.key}'"
|
555
|
-
if (
|
556
|
-
aux_transform.hash == hash
|
557
|
-
and aux_transform.type == "notebook"
|
558
|
-
):
|
559
|
-
message += " -- anticipating changes"
|
560
|
-
elif aux_transform.hash != hash:
|
561
|
-
message += "" # could log "source code changed", but this seems too much
|
562
|
-
elif (
|
563
|
-
aux_transform.created_by_id != ln_setup.settings.user.id
|
564
|
-
):
|
565
|
-
message += f" -- {aux_transform.created_by.handle} already works on this draft"
|
566
|
-
message += f", creating new version '{uid}'"
|
567
|
-
revises = aux_transform
|
615
|
+
uid, target_transform, message = self._process_aux_transform(
|
616
|
+
aux_transform, transform_hash
|
617
|
+
)
|
568
618
|
found_key = True
|
569
619
|
break
|
570
620
|
if not found_key:
|
@@ -580,22 +630,48 @@ class Context:
|
|
580
630
|
logger.important(message)
|
581
631
|
self.uid, transform = uid, target_transform
|
582
632
|
# the user did pass the uid
|
583
|
-
|
633
|
+
elif self.uid is not None and len(self.uid) == 16:
|
584
634
|
transform = Transform.filter(uid=self.uid).one_or_none()
|
585
635
|
if transform is not None:
|
586
636
|
if transform.key not in self._path.as_posix():
|
587
637
|
n_parts = len(Path(transform.key).parts)
|
588
|
-
|
638
|
+
(
|
589
639
|
Path(*self._path.parts[-n_parts:]).as_posix()
|
590
640
|
if n_parts > 0
|
591
641
|
else ""
|
592
642
|
)
|
593
|
-
|
594
|
-
|
643
|
+
key = self._path.name
|
644
|
+
else:
|
645
|
+
key = transform.key # type: ignore
|
646
|
+
else:
|
647
|
+
key = self._path.name
|
648
|
+
else:
|
649
|
+
if self.uid is not None:
|
650
|
+
assert len(self.uid) == 12, ( # noqa: S101
|
651
|
+
"uid must be 12 (stem) or 16 (full) characters long"
|
652
|
+
)
|
653
|
+
aux_transform = (
|
654
|
+
Transform.filter(uid__startswith=self.uid)
|
655
|
+
.order_by("-created_at")
|
656
|
+
.first()
|
657
|
+
)
|
658
|
+
if aux_transform is not None:
|
659
|
+
if aux_transform.key.endswith(self._path.name):
|
660
|
+
key = aux_transform.key
|
661
|
+
else:
|
662
|
+
key = "/".join(
|
663
|
+
aux_transform.key.split("/")[:-1] + [self._path.name]
|
595
664
|
)
|
596
|
-
|
665
|
+
uid, target_transform, message = self._process_aux_transform(
|
666
|
+
aux_transform, transform_hash
|
667
|
+
)
|
668
|
+
if message != "":
|
669
|
+
logger.important(message)
|
597
670
|
else:
|
671
|
+
uid = f"{self.uid}0000" if self.uid is not None else None
|
672
|
+
target_transform = None
|
598
673
|
key = self._path.name
|
674
|
+
self.uid, transform = uid, target_transform
|
599
675
|
if self.version is not None:
|
600
676
|
# test inconsistent version passed
|
601
677
|
if (
|
@@ -607,39 +683,28 @@ class Context:
|
|
607
683
|
f"✗ please pass consistent version: ln.context.version = '{transform.version}'" # type: ignore
|
608
684
|
)
|
609
685
|
# test whether version was already used for another member of the family
|
610
|
-
|
611
|
-
|
612
|
-
|
613
|
-
|
614
|
-
|
615
|
-
|
616
|
-
|
617
|
-
|
618
|
-
|
686
|
+
if self.uid is not None and len(self.uid) == 16:
|
687
|
+
suid, vuid = (self.uid[:-4], self.uid[-4:])
|
688
|
+
transform = Transform.filter(
|
689
|
+
uid__startswith=suid, version=self.version
|
690
|
+
).one_or_none()
|
691
|
+
if transform is not None and vuid != transform.uid[-4:]:
|
692
|
+
better_version = bump_version_function(self.version)
|
693
|
+
raise SystemExit(
|
694
|
+
f"✗ version '{self.version}' is already taken by Transform('{transform.uid}'); please set another version, e.g., ln.context.version = '{better_version}'"
|
695
|
+
)
|
619
696
|
# make a new transform record
|
620
697
|
if transform is None:
|
621
698
|
assert key is not None # noqa: S101
|
622
|
-
|
623
|
-
|
624
|
-
|
625
|
-
|
626
|
-
|
627
|
-
|
628
|
-
|
629
|
-
|
630
|
-
|
631
|
-
type=transform_type,
|
632
|
-
).save()
|
633
|
-
except InconsistentKey:
|
634
|
-
raise_update_context = True
|
635
|
-
if raise_update_context:
|
636
|
-
if revises is None:
|
637
|
-
revises = (
|
638
|
-
Transform.filter(uid__startswith=self.uid[:-4], is_latest=True)
|
639
|
-
.order_by("-created_at")
|
640
|
-
.first()
|
641
|
-
)
|
642
|
-
raise UpdateContext(get_key_clashing_message(revises, key))
|
699
|
+
transform = Transform( # type: ignore
|
700
|
+
uid=self.uid,
|
701
|
+
version=self.version,
|
702
|
+
description=description,
|
703
|
+
key=key,
|
704
|
+
reference=transform_ref,
|
705
|
+
reference_type=transform_ref_type,
|
706
|
+
type=transform_type,
|
707
|
+
).save()
|
643
708
|
self._logging_message_track += f"created Transform('{transform.uid}')"
|
644
709
|
else:
|
645
710
|
uid = transform.uid
|
@@ -647,7 +712,11 @@ class Context:
|
|
647
712
|
transform_was_saved = transform.source_code is not None
|
648
713
|
# check whether the transform.key is consistent
|
649
714
|
if transform.key != key:
|
650
|
-
|
715
|
+
self._logging_message_track += (
|
716
|
+
f"renaming transform {transform.key} to {key}"
|
717
|
+
)
|
718
|
+
transform.key = key
|
719
|
+
transform.save()
|
651
720
|
elif transform.description != description and description is not None:
|
652
721
|
transform.description = description
|
653
722
|
transform.save()
|
@@ -659,18 +728,20 @@ class Context:
|
|
659
728
|
and not transform_was_saved
|
660
729
|
):
|
661
730
|
raise UpdateContext(
|
662
|
-
f'{transform.created_by.name} ({transform.created_by.handle}) already works on this draft {transform.type}.\n\nPlease create a revision via `ln.track("{uid[:-4]}{increment_base62(uid[-4:])}")` or a new transform with a *different*
|
731
|
+
f'{transform.created_by.name} ({transform.created_by.handle}) already works on this draft {transform.type}.\n\nPlease create a revision via `ln.track("{uid[:-4]}{increment_base62(uid[-4:])}")` or a new transform with a *different* key and `ln.track("{ids.base62_12()}0000")`.'
|
663
732
|
)
|
664
733
|
# check whether transform source code was already saved
|
665
734
|
if transform_was_saved:
|
666
735
|
bump_revision = False
|
667
|
-
if
|
736
|
+
if (
|
737
|
+
transform.type == "notebook"
|
738
|
+
and self._notebook_runner != "nbconvert"
|
739
|
+
):
|
668
740
|
# we anticipate the user makes changes to the notebook source code
|
669
741
|
# in an interactive session, hence we pro-actively bump the version number
|
670
742
|
bump_revision = True
|
671
743
|
else:
|
672
|
-
|
673
|
-
if hash != transform.hash:
|
744
|
+
if transform_hash != transform.hash:
|
674
745
|
bump_revision = True
|
675
746
|
else:
|
676
747
|
self._logging_message_track += (
|
@@ -679,7 +750,10 @@ class Context:
|
|
679
750
|
if bump_revision:
|
680
751
|
change_type = (
|
681
752
|
"re-running notebook with already-saved source code"
|
682
|
-
if
|
753
|
+
if (
|
754
|
+
transform.type == "notebook"
|
755
|
+
and self._notebook_runner != "nbconvert"
|
756
|
+
)
|
683
757
|
else "source code changed"
|
684
758
|
)
|
685
759
|
raise UpdateContext(
|
@@ -689,11 +763,11 @@ class Context:
|
|
689
763
|
self._logging_message_track += f"loaded Transform('{transform.uid}')"
|
690
764
|
self._transform = transform
|
691
765
|
|
692
|
-
def
|
693
|
-
"""Finish a
|
766
|
+
def _finish(self, ignore_non_consecutive: None | bool = None) -> None:
|
767
|
+
"""Finish the run and write a run report.
|
694
768
|
|
695
769
|
- writes a timestamp: `run.finished_at`
|
696
|
-
- saves the source code: `transform.source_code`
|
770
|
+
- saves the source code if it is not yet saved: `transform.source_code`
|
697
771
|
- saves a run report: `run.report`
|
698
772
|
|
699
773
|
When called in the last cell of a notebook:
|
@@ -737,6 +811,7 @@ class Context:
|
|
737
811
|
finished_at=True,
|
738
812
|
ignore_non_consecutive=ignore_non_consecutive,
|
739
813
|
is_retry=self._is_finish_retry,
|
814
|
+
notebook_runner=self._notebook_runner,
|
740
815
|
)
|
741
816
|
if return_code == "retry":
|
742
817
|
self._is_finish_retry = True
|
@@ -751,5 +826,13 @@ class Context:
|
|
751
826
|
self._version = None
|
752
827
|
self._description = None
|
753
828
|
|
829
|
+
@deprecated("ln.track()")
|
830
|
+
def track(self, *args, **kwargs):
|
831
|
+
return self._track(*args, **kwargs)
|
832
|
+
|
833
|
+
@deprecated("ln.finish()")
|
834
|
+
def finish(self, *args, **kwargs):
|
835
|
+
return self._finish(*args, **kwargs)
|
836
|
+
|
754
837
|
|
755
|
-
context = Context()
|
838
|
+
context: Context = Context()
|