lamindb 0.76.0__py3-none-any.whl → 0.76.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lamindb/__init__.py +12 -11
- lamindb/_artifact.py +81 -54
- lamindb/_can_validate.py +10 -3
- lamindb/_collection.py +17 -18
- lamindb/_curate.py +37 -19
- lamindb/_feature.py +0 -49
- lamindb/_filter.py +6 -5
- lamindb/_finish.py +11 -54
- lamindb/_from_values.py +14 -10
- lamindb/_is_versioned.py +3 -5
- lamindb/_query_manager.py +4 -4
- lamindb/_query_set.py +36 -10
- lamindb/_record.py +44 -43
- lamindb/_save.py +2 -3
- lamindb/_transform.py +23 -10
- lamindb/core/__init__.py +9 -3
- lamindb/core/_context.py +518 -0
- lamindb/core/_data.py +8 -6
- lamindb/core/_feature_manager.py +25 -8
- lamindb/core/_label_manager.py +1 -1
- lamindb/core/_mapped_collection.py +82 -26
- lamindb/core/_settings.py +4 -8
- lamindb/core/datasets/_core.py +1 -0
- lamindb/core/exceptions.py +22 -5
- lamindb/core/storage/__init__.py +1 -1
- lamindb/core/storage/_backed_access.py +2 -38
- lamindb/core/storage/_tiledbsoma.py +229 -0
- lamindb/core/storage/_valid_suffixes.py +2 -0
- lamindb/core/storage/paths.py +2 -6
- lamindb/core/versioning.py +56 -47
- lamindb/integrations/_vitessce.py +2 -0
- {lamindb-0.76.0.dist-info → lamindb-0.76.2.dist-info}/METADATA +7 -15
- lamindb-0.76.2.dist-info/RECORD +59 -0
- lamindb/core/_run_context.py +0 -514
- lamindb-0.76.0.dist-info/RECORD +0 -58
- {lamindb-0.76.0.dist-info → lamindb-0.76.2.dist-info}/LICENSE +0 -0
- {lamindb-0.76.0.dist-info → lamindb-0.76.2.dist-info}/WHEEL +0 -0
lamindb/_feature.py
CHANGED
@@ -109,18 +109,6 @@ def from_df(cls, df: pd.DataFrame, field: FieldAttr | None = None) -> RecordsLis
|
|
109
109
|
for name, col in df.items():
|
110
110
|
if name in categoricals:
|
111
111
|
dtypes[name] = "cat"
|
112
|
-
# below is a harder feature to write, now, because it requires to
|
113
|
-
# query the link tables between the label Record and file or collection
|
114
|
-
# the original implementation fell short
|
115
|
-
# categorical = categoricals[name]
|
116
|
-
# if hasattr(
|
117
|
-
# categorical, "cat"
|
118
|
-
# ): # because .categories > pd2.0, .cat.categories < pd2.0
|
119
|
-
# categorical = categorical.cat
|
120
|
-
# categories = categorical.categories
|
121
|
-
# categoricals_with_unmapped_categories[name] = ULabel.filter(
|
122
|
-
# feature=name
|
123
|
-
# ).inspect(categories, "name", logging=False)["not_mapped"]
|
124
112
|
else:
|
125
113
|
dtypes[name] = convert_numpy_dtype_to_lamin_feature_type(col.dtype)
|
126
114
|
|
@@ -138,46 +126,9 @@ def from_df(cls, df: pd.DataFrame, field: FieldAttr | None = None) -> RecordsLis
|
|
138
126
|
settings.verbosity = verbosity
|
139
127
|
|
140
128
|
assert len(features) == len(df.columns) # noqa: S101
|
141
|
-
|
142
|
-
# if len(categoricals_with_unmapped_categories) > 0:
|
143
|
-
# n_max = 20
|
144
|
-
# categoricals_with_unmapped_categories_formatted = "\n ".join(
|
145
|
-
# [
|
146
|
-
# (
|
147
|
-
# f"{key} ({len(value)}): {', '.join(value)}"
|
148
|
-
# if len(value) <= 5
|
149
|
-
# else f"{key} ({len(value)}): {', '.join(value[:5])} ..."
|
150
|
-
# )
|
151
|
-
# for key, value in take(
|
152
|
-
# n_max, categoricals_with_unmapped_categories.items()
|
153
|
-
# )
|
154
|
-
# ]
|
155
|
-
# )
|
156
|
-
# if len(categoricals_with_unmapped_categories) > n_max:
|
157
|
-
# categoricals_with_unmapped_categories_formatted += "\n ..."
|
158
|
-
# categoricals_with_unmapped_categories_formatted
|
159
|
-
# logger.info(
|
160
|
-
# f"{len(categoricals_with_unmapped_categories)} features have"
|
161
|
-
# f" {colors.yellow('unmapped categories')}:\n "
|
162
|
-
# f" {categoricals_with_unmapped_categories_formatted}"
|
163
|
-
# )
|
164
129
|
return RecordsList(features)
|
165
130
|
|
166
131
|
|
167
|
-
# def from_df(
|
168
|
-
# self,
|
169
|
-
# df: "pd.DataFrame",
|
170
|
-
# field: Optional[FieldAttr] = Feature.name,
|
171
|
-
# **kwargs,
|
172
|
-
# ) -> Dict:
|
173
|
-
# feature_set = FeatureSet.from_df(df, field=field, **kwargs)
|
174
|
-
# if feature_set is not None:
|
175
|
-
# feature_sets = {"columns": feature_set}
|
176
|
-
# else:
|
177
|
-
# feature_sets = {}
|
178
|
-
# return feature_sets
|
179
|
-
|
180
|
-
|
181
132
|
@doc_args(Feature.save.__doc__)
|
182
133
|
def save(self, *args, **kwargs) -> Feature:
|
183
134
|
"""{}""" # noqa: D415
|
lamindb/_filter.py
CHANGED
@@ -1,18 +1,19 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
|
3
|
-
from lnschema_core import Artifact, Collection,
|
3
|
+
from lnschema_core import Artifact, Collection, Record
|
4
4
|
from lnschema_core.types import VisibilityChoice
|
5
5
|
|
6
6
|
from lamindb import settings
|
7
|
-
from lamindb._query_set import QuerySet
|
8
7
|
|
8
|
+
from ._query_set import QuerySet
|
9
9
|
|
10
|
-
|
10
|
+
|
11
|
+
def filter(registry: type[Record], **expressions) -> QuerySet:
|
11
12
|
"""See :meth:`~lamindb.core.Record.filter`."""
|
12
13
|
_using_key = None
|
13
14
|
if "_using_key" in expressions:
|
14
15
|
_using_key = expressions.pop("_using_key")
|
15
|
-
if
|
16
|
+
if registry in {Artifact, Collection}:
|
16
17
|
# visibility is set to 0 unless expressions contains id or uid equality
|
17
18
|
if not (
|
18
19
|
"id" in expressions
|
@@ -29,7 +30,7 @@ def filter(Record: type[Record], **expressions) -> QuerySet:
|
|
29
30
|
# sense for a non-NULLABLE column
|
30
31
|
elif visibility in expressions and expressions[visibility] is None:
|
31
32
|
expressions.pop(visibility)
|
32
|
-
qs = QuerySet(model=
|
33
|
+
qs = QuerySet(model=registry, using=_using_key)
|
33
34
|
if len(expressions) > 0:
|
34
35
|
return qs.filter(**expressions)
|
35
36
|
else:
|
lamindb/_finish.py
CHANGED
@@ -9,9 +9,6 @@ from typing import TYPE_CHECKING
|
|
9
9
|
import lamindb_setup as ln_setup
|
10
10
|
from lamin_utils import logger
|
11
11
|
from lamindb_setup.core.hashing import hash_file
|
12
|
-
from lnschema_core.types import TransformType
|
13
|
-
|
14
|
-
from .core._run_context import is_run_from_ipython, run_context
|
15
12
|
|
16
13
|
if TYPE_CHECKING:
|
17
14
|
from pathlib import Path
|
@@ -21,51 +18,7 @@ if TYPE_CHECKING:
|
|
21
18
|
from ._query_set import QuerySet
|
22
19
|
|
23
20
|
|
24
|
-
|
25
|
-
pass
|
26
|
-
|
27
|
-
|
28
|
-
class NotebookNotSaved(SystemExit):
|
29
|
-
pass
|
30
|
-
|
31
|
-
|
32
|
-
def get_seconds_since_modified(filepath) -> float:
|
33
|
-
return datetime.now().timestamp() - filepath.stat().st_mtime
|
34
|
-
|
35
|
-
|
36
|
-
def finish() -> None:
|
37
|
-
"""Mark a tracked run as finished.
|
38
|
-
|
39
|
-
Saves source code and, for notebooks, a run report to your default storage location.
|
40
|
-
"""
|
41
|
-
if run_context.run is None:
|
42
|
-
raise TrackNotCalled("Please run `ln.track()` before `ln.finish()`")
|
43
|
-
if run_context.path is None:
|
44
|
-
if run_context.transform.type in {"script", "notebook"}:
|
45
|
-
raise ValueError(
|
46
|
-
f"Transform type is not allowed to be 'script' or 'notebook' but is {run_context.transform.type}."
|
47
|
-
)
|
48
|
-
run_context.run.finished_at = datetime.now(timezone.utc)
|
49
|
-
run_context.run.save()
|
50
|
-
# nothing else to do
|
51
|
-
return None
|
52
|
-
if is_run_from_ipython: # notebooks
|
53
|
-
if (
|
54
|
-
get_seconds_since_modified(run_context.path) > 3
|
55
|
-
and os.getenv("LAMIN_TESTING") is None
|
56
|
-
):
|
57
|
-
raise NotebookNotSaved(
|
58
|
-
"Please save the notebook in your editor right before running `ln.finish()`"
|
59
|
-
)
|
60
|
-
save_run_context_core(
|
61
|
-
run=run_context.run,
|
62
|
-
transform=run_context.transform,
|
63
|
-
filepath=run_context.path,
|
64
|
-
finished_at=True,
|
65
|
-
)
|
66
|
-
|
67
|
-
|
68
|
-
def save_run_context_core(
|
21
|
+
def save_context_core(
|
69
22
|
*,
|
70
23
|
run: Run,
|
71
24
|
transform: Transform,
|
@@ -76,6 +29,8 @@ def save_run_context_core(
|
|
76
29
|
) -> str | None:
|
77
30
|
import lamindb as ln
|
78
31
|
|
32
|
+
from .core._context import context, is_run_from_ipython
|
33
|
+
|
79
34
|
ln.settings.verbosity = "success"
|
80
35
|
|
81
36
|
# for scripts, things are easy
|
@@ -182,7 +137,9 @@ def save_run_context_core(
|
|
182
137
|
f"replaced transform._source_code_artifact: {transform._source_code_artifact}"
|
183
138
|
)
|
184
139
|
else:
|
185
|
-
logger.warning(
|
140
|
+
logger.warning(
|
141
|
+
"Please re-run `ln.context.track()` to make a new version"
|
142
|
+
)
|
186
143
|
return "rerun-the-notebook"
|
187
144
|
else:
|
188
145
|
logger.important("source code is already saved")
|
@@ -191,7 +148,7 @@ def save_run_context_core(
|
|
191
148
|
_source_code_artifact_path,
|
192
149
|
description=f"Source of transform {transform.uid}",
|
193
150
|
version=transform.version,
|
194
|
-
|
151
|
+
revises=prev_source,
|
195
152
|
visibility=0, # hidden file
|
196
153
|
run=False,
|
197
154
|
)
|
@@ -254,7 +211,7 @@ def save_run_context_core(
|
|
254
211
|
report_file = ln.Artifact(
|
255
212
|
report_path,
|
256
213
|
description=f"Report of run {run.uid}",
|
257
|
-
|
214
|
+
revises=prev_report,
|
258
215
|
visibility=0, # hidden file
|
259
216
|
run=False,
|
260
217
|
)
|
@@ -282,7 +239,7 @@ def save_run_context_core(
|
|
282
239
|
logger.important(
|
283
240
|
f"if you want to update your {thing} without re-running it, use `lamin save {name}`"
|
284
241
|
)
|
285
|
-
# because run & transform changed, update the global
|
286
|
-
|
287
|
-
|
242
|
+
# because run & transform changed, update the global context
|
243
|
+
context._run = run
|
244
|
+
context._transform = transform
|
288
245
|
return None
|
lamindb/_from_values.py
CHANGED
@@ -25,9 +25,9 @@ def get_or_create_records(
|
|
25
25
|
mute: bool = False,
|
26
26
|
) -> list[Record]:
|
27
27
|
"""Get or create records from iterables."""
|
28
|
-
|
28
|
+
registry = field.field.model
|
29
29
|
if create:
|
30
|
-
return [
|
30
|
+
return [registry(**{field.field.name: value}) for value in iterable]
|
31
31
|
creation_search_names = settings.creation.search_names
|
32
32
|
feature: Feature = None
|
33
33
|
organism = _get_organism_record(field, organism)
|
@@ -57,21 +57,23 @@ def get_or_create_records(
|
|
57
57
|
and records[0].source_id
|
58
58
|
):
|
59
59
|
source_record = records[0].source
|
60
|
-
if not source_record and hasattr(
|
60
|
+
if not source_record and hasattr(registry, "public"):
|
61
61
|
from bionty._bionty import get_source_record
|
62
62
|
|
63
|
-
source_record = get_source_record(
|
63
|
+
source_record = get_source_record(
|
64
|
+
registry.public(organism=organism), registry
|
65
|
+
)
|
64
66
|
if source_record:
|
65
67
|
from bionty.core._add_ontology import check_source_in_db
|
66
68
|
|
67
69
|
check_source_in_db(
|
68
|
-
registry=
|
70
|
+
registry=registry,
|
69
71
|
source=source_record,
|
70
72
|
update=True,
|
71
73
|
)
|
72
74
|
|
73
75
|
from_source = not source_record.in_db
|
74
|
-
elif hasattr(
|
76
|
+
elif hasattr(registry, "source_id"):
|
75
77
|
from_source = True
|
76
78
|
else:
|
77
79
|
from_source = False
|
@@ -97,14 +99,14 @@ def get_or_create_records(
|
|
97
99
|
logger.success(msg)
|
98
100
|
s = "" if len(unmapped_values) == 1 else "s"
|
99
101
|
print_values = colors.yellow(_print_values(unmapped_values))
|
100
|
-
name =
|
102
|
+
name = registry.__name__
|
101
103
|
n_nonval = colors.yellow(f"{len(unmapped_values)} non-validated")
|
102
104
|
if not mute:
|
103
105
|
logger.warning(
|
104
106
|
f"{colors.red('did not create')} {name} record{s} for "
|
105
107
|
f"{n_nonval} {colors.italic(f'{field.field.name}{s}')}: {print_values}"
|
106
108
|
)
|
107
|
-
if
|
109
|
+
if registry.__get_schema_name__() == "bionty" or registry == ULabel:
|
108
110
|
if isinstance(iterable, pd.Series):
|
109
111
|
feature = iterable.name
|
110
112
|
feature_name = None
|
@@ -230,7 +232,7 @@ def create_records_from_source(
|
|
230
232
|
# for custom records that are not created from public sources
|
231
233
|
return records, iterable_idx
|
232
234
|
# add source record to the kwargs
|
233
|
-
source_record = get_source_record(public_ontology)
|
235
|
+
source_record = get_source_record(public_ontology, model)
|
234
236
|
kwargs.update({"source": source_record})
|
235
237
|
|
236
238
|
# filter the columns in bionty df based on fields
|
@@ -373,6 +375,8 @@ def _get_organism_record(
|
|
373
375
|
if _has_organism_field(registry) and check:
|
374
376
|
from bionty._bionty import create_or_get_organism_record
|
375
377
|
|
376
|
-
organism_record = create_or_get_organism_record(
|
378
|
+
organism_record = create_or_get_organism_record(
|
379
|
+
organism=organism, registry=registry
|
380
|
+
)
|
377
381
|
if organism_record is not None:
|
378
382
|
return organism_record
|
lamindb/_is_versioned.py
CHANGED
@@ -7,15 +7,13 @@ from lnschema_core.models import IsVersioned
|
|
7
7
|
|
8
8
|
from lamindb._utils import attach_func_to_class_method
|
9
9
|
|
10
|
-
from .core.versioning import
|
10
|
+
from .core.versioning import create_uid, get_new_path_from_uid
|
11
11
|
|
12
12
|
|
13
13
|
# docstring handled through attach_func_to_class_method
|
14
|
-
def _add_to_version_family(
|
15
|
-
self, is_new_version_of: IsVersioned, version: str | None = None
|
16
|
-
):
|
14
|
+
def _add_to_version_family(self, revises: IsVersioned, version: str | None = None):
|
17
15
|
old_uid = self.uid
|
18
|
-
new_uid,
|
16
|
+
new_uid, revises = create_uid(revises=revises, version=version)
|
19
17
|
if self.__class__.__name__ == "Artifact" and self._key_is_virtual:
|
20
18
|
old_path = self.path
|
21
19
|
new_path = get_new_path_from_uid(
|
lamindb/_query_manager.py
CHANGED
@@ -28,7 +28,7 @@ class QueryManager(models.Manager):
|
|
28
28
|
>>> ln.save(ln.ULabel.from_values(["ULabel1", "ULabel2", "ULabel3"], field="name")) # noqa
|
29
29
|
>>> labels = ln.ULabel.filter(name__icontains = "label").all()
|
30
30
|
>>> ln.ULabel(name="ULabel1").save()
|
31
|
-
>>> label = ln.ULabel.
|
31
|
+
>>> label = ln.ULabel.get(name="ULabel1")
|
32
32
|
>>> label.parents.set(labels)
|
33
33
|
>>> manager = label.parents
|
34
34
|
>>> manager.df()
|
@@ -40,11 +40,11 @@ class QueryManager(models.Manager):
|
|
40
40
|
self.source_field_name == "collection"
|
41
41
|
and self.target_field_name == "artifact"
|
42
42
|
):
|
43
|
+
from lamindb.core._context import context
|
43
44
|
from lamindb.core._data import WARNING_RUN_TRANSFORM, _track_run_input
|
44
|
-
from lamindb.core._run_context import run_context
|
45
45
|
|
46
46
|
if (
|
47
|
-
|
47
|
+
context.run is None
|
48
48
|
and not settings.creation.artifact_silence_missing_run_warning
|
49
49
|
):
|
50
50
|
logger.warning(WARNING_RUN_TRANSFORM)
|
@@ -57,7 +57,7 @@ class QueryManager(models.Manager):
|
|
57
57
|
>>> ln.save(ln.ULabel.from_values(["ULabel1", "ULabel2", "ULabel3"], field="name"))
|
58
58
|
>>> labels = ln.ULabel.filter(name__icontains="label").all()
|
59
59
|
>>> ln.ULabel(name="ULabel1").save()
|
60
|
-
>>> label = ln.ULabel.
|
60
|
+
>>> label = ln.ULabel.get(name="ULabel1")
|
61
61
|
>>> label.parents.set(labels)
|
62
62
|
>>> label.parents.list()
|
63
63
|
>>> label.parents.list("name")
|
lamindb/_query_set.py
CHANGED
@@ -17,14 +17,12 @@ from lnschema_core.models import (
|
|
17
17
|
Transform,
|
18
18
|
)
|
19
19
|
|
20
|
+
from lamindb.core.exceptions import DoesNotExist
|
21
|
+
|
20
22
|
if TYPE_CHECKING:
|
21
23
|
from lnschema_core.types import ListLike, StrField
|
22
24
|
|
23
25
|
|
24
|
-
class NoResultFound(Exception):
|
25
|
-
pass
|
26
|
-
|
27
|
-
|
28
26
|
class MultipleResultsFound(Exception):
|
29
27
|
pass
|
30
28
|
|
@@ -59,13 +57,41 @@ def get_keys_from_df(data: list, registry: Record) -> list[str]:
|
|
59
57
|
|
60
58
|
def one_helper(self):
|
61
59
|
if len(self) == 0:
|
62
|
-
raise
|
60
|
+
raise DoesNotExist
|
63
61
|
elif len(self) > 1:
|
64
62
|
raise MultipleResultsFound(self)
|
65
63
|
else:
|
66
64
|
return self[0]
|
67
65
|
|
68
66
|
|
67
|
+
def get(
|
68
|
+
registry_or_queryset: type[Record] | QuerySet,
|
69
|
+
idlike: int | str | None = None,
|
70
|
+
**expressions,
|
71
|
+
) -> Record:
|
72
|
+
if isinstance(registry_or_queryset, QuerySet):
|
73
|
+
qs = registry_or_queryset
|
74
|
+
registry = qs.model
|
75
|
+
else:
|
76
|
+
qs = QuerySet(model=registry_or_queryset)
|
77
|
+
registry = registry_or_queryset
|
78
|
+
if isinstance(idlike, int):
|
79
|
+
return super(QuerySet, qs).get(id=idlike)
|
80
|
+
elif isinstance(idlike, str):
|
81
|
+
qs = qs.filter(uid__startswith=idlike)
|
82
|
+
if issubclass(registry, IsVersioned):
|
83
|
+
if len(idlike) <= registry._len_stem_uid:
|
84
|
+
return qs.latest_version().one()
|
85
|
+
else:
|
86
|
+
return qs.one()
|
87
|
+
else:
|
88
|
+
return qs.one()
|
89
|
+
else:
|
90
|
+
assert idlike is None # noqa: S101
|
91
|
+
# below behaves exactly like `.one()`
|
92
|
+
return registry.objects.get(**expressions)
|
93
|
+
|
94
|
+
|
69
95
|
class RecordsList(UserList):
|
70
96
|
"""Is ordered, can't be queried, but has `.df()`."""
|
71
97
|
|
@@ -221,12 +247,12 @@ class QuerySet(models.QuerySet, CanValidate):
|
|
221
247
|
return None
|
222
248
|
return self[0]
|
223
249
|
|
224
|
-
def
|
225
|
-
"""
|
250
|
+
def get(self, idlike: int | str | None = None, **expressions) -> Record:
|
251
|
+
"""Query a single record. Raises error if there are more or none."""
|
252
|
+
return get(self, idlike, **expressions)
|
226
253
|
|
227
|
-
|
228
|
-
|
229
|
-
"""
|
254
|
+
def one(self) -> Record:
|
255
|
+
"""Exactly one result. Raises error if there are more or none."""
|
230
256
|
return one_helper(self)
|
231
257
|
|
232
258
|
def one_or_none(self) -> Record | None:
|
lamindb/_record.py
CHANGED
@@ -12,7 +12,7 @@ from lamin_utils._lookup import Lookup
|
|
12
12
|
from lamindb_setup._connect_instance import get_owner_name_from_identifier
|
13
13
|
from lamindb_setup.core._docs import doc_args
|
14
14
|
from lamindb_setup.core._hub_core import connect_instance
|
15
|
-
from lnschema_core.models import IsVersioned, Record
|
15
|
+
from lnschema_core.models import Collection, IsVersioned, Record
|
16
16
|
|
17
17
|
from lamindb._utils import attach_func_to_class_method
|
18
18
|
from lamindb.core._settings import settings
|
@@ -90,13 +90,22 @@ def __init__(record: Record, *args, **kwargs):
|
|
90
90
|
match = suggest_records_with_similar_names(record, kwargs)
|
91
91
|
if match:
|
92
92
|
if "version" in kwargs:
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
93
|
+
if kwargs["version"] is not None:
|
94
|
+
version_comment = " and version"
|
95
|
+
existing_record = record.__class__.filter(
|
96
|
+
name=kwargs["name"], version=kwargs["version"]
|
97
|
+
).one_or_none()
|
98
|
+
else:
|
99
|
+
# for a versioned record, an exact name match is not a
|
100
|
+
# criterion for retrieving a record in case `version`
|
101
|
+
# isn't passed - we'd always pull out many records with exactly the
|
102
|
+
# same name
|
103
|
+
existing_record = None
|
97
104
|
else:
|
98
105
|
version_comment = ""
|
99
|
-
existing_record = record.__class__.filter(
|
106
|
+
existing_record = record.__class__.filter(
|
107
|
+
name=kwargs["name"]
|
108
|
+
).one_or_none()
|
100
109
|
if existing_record is not None:
|
101
110
|
logger.important(
|
102
111
|
f"returning existing {record.__class__.__name__} record with same"
|
@@ -123,27 +132,26 @@ def filter(cls, **expressions) -> QuerySet:
|
|
123
132
|
|
124
133
|
@classmethod # type:ignore
|
125
134
|
@doc_args(Record.get.__doc__)
|
126
|
-
def get(
|
135
|
+
def get(
|
136
|
+
cls,
|
137
|
+
idlike: int | str | None = None,
|
138
|
+
**expressions,
|
139
|
+
) -> Record:
|
127
140
|
"""{}""" # noqa: D415
|
128
|
-
|
141
|
+
# this is the only place in which we need the lamindb queryset
|
142
|
+
# in this file; everywhere else it should be Django's
|
143
|
+
from lamindb._query_set import QuerySet
|
129
144
|
|
130
|
-
|
131
|
-
return filter(cls, id=idlike).one()
|
132
|
-
else:
|
133
|
-
qs = filter(cls, uid__startswith=idlike)
|
134
|
-
if issubclass(cls, IsVersioned):
|
135
|
-
if len(idlike) <= cls._len_stem_uid:
|
136
|
-
return qs.latest_version().one()
|
137
|
-
else:
|
138
|
-
return qs.one()
|
139
|
-
else:
|
140
|
-
return qs.one()
|
145
|
+
return QuerySet(model=cls).get(idlike, **expressions)
|
141
146
|
|
142
147
|
|
143
148
|
@classmethod # type:ignore
|
144
149
|
@doc_args(Record.df.__doc__)
|
145
150
|
def df(
|
146
|
-
cls,
|
151
|
+
cls,
|
152
|
+
include: str | list[str] | None = None,
|
153
|
+
join: str = "inner",
|
154
|
+
limit: int = 100,
|
147
155
|
) -> pd.DataFrame:
|
148
156
|
"""{}""" # noqa: D415
|
149
157
|
from lamindb._filter import filter
|
@@ -151,7 +159,7 @@ def df(
|
|
151
159
|
query_set = filter(cls)
|
152
160
|
if hasattr(cls, "updated_at"):
|
153
161
|
query_set = query_set.order_by("-updated_at")
|
154
|
-
return query_set.df(include=include, join=join)
|
162
|
+
return query_set[:limit].df(include=include, join=join)
|
155
163
|
|
156
164
|
|
157
165
|
# from_values doesn't apply for QuerySet or Manager
|
@@ -382,9 +390,11 @@ def add_db_connection(db: str, using: str):
|
|
382
390
|
@doc_args(Record.using.__doc__)
|
383
391
|
def using(
|
384
392
|
cls,
|
385
|
-
instance: str,
|
393
|
+
instance: str | None,
|
386
394
|
) -> QuerySet:
|
387
395
|
"""{}""" # noqa: D415
|
396
|
+
if instance is None:
|
397
|
+
return QuerySet(model=cls, using=None)
|
388
398
|
from lamindb_setup._connect_instance import (
|
389
399
|
load_instance_settings,
|
390
400
|
update_db_using_local,
|
@@ -473,8 +483,8 @@ def transfer_to_default_db(
|
|
473
483
|
return record_on_default
|
474
484
|
if not mute:
|
475
485
|
logger.hint(f"saving from instance {db} to default instance: {record}")
|
486
|
+
from lamindb.core._context import context
|
476
487
|
from lamindb.core._data import WARNING_RUN_TRANSFORM
|
477
|
-
from lamindb.core._run_context import run_context
|
478
488
|
|
479
489
|
if hasattr(record, "created_by_id"):
|
480
490
|
# this line is needed to point created_by to default db
|
@@ -482,16 +492,16 @@ def transfer_to_default_db(
|
|
482
492
|
record.created_by_id = ln_setup.settings.user.id
|
483
493
|
if hasattr(record, "run_id"):
|
484
494
|
record.run = None
|
485
|
-
if
|
486
|
-
record.run_id =
|
495
|
+
if context.run is not None:
|
496
|
+
record.run_id = context.run.id
|
487
497
|
else:
|
488
498
|
if not settings.creation.artifact_silence_missing_run_warning:
|
489
499
|
logger.warning(WARNING_RUN_TRANSFORM)
|
490
500
|
record.run_id = None
|
491
501
|
if hasattr(record, "transform_id") and record._meta.model_name != "run":
|
492
502
|
record.transform = None
|
493
|
-
if
|
494
|
-
record.transform_id =
|
503
|
+
if context.run is not None:
|
504
|
+
record.transform_id = context.run.transform_id
|
495
505
|
else:
|
496
506
|
record.transform_id = None
|
497
507
|
# transfer other foreign key fields
|
@@ -530,24 +540,15 @@ def save(self, *args, **kwargs) -> Record:
|
|
530
540
|
init_self_from_db(self, result)
|
531
541
|
else:
|
532
542
|
# save versioned record
|
533
|
-
if isinstance(self, IsVersioned) and self.
|
534
|
-
|
535
|
-
|
536
|
-
|
537
|
-
# need one additional request
|
538
|
-
is_new_version_of = self.__class__.objects.get(
|
539
|
-
is_latest=True, uid__startswith=self.stem_uid
|
540
|
-
)
|
541
|
-
logger.warning(
|
542
|
-
f"didn't pass the latest version in `is_new_version_of`, retrieved it: {is_new_version_of}"
|
543
|
-
)
|
544
|
-
is_new_version_of.is_latest = False
|
543
|
+
if isinstance(self, IsVersioned) and self._revises is not None:
|
544
|
+
assert self._revises.is_latest # noqa: S101
|
545
|
+
revises = self._revises
|
546
|
+
revises.is_latest = False
|
545
547
|
with transaction.atomic():
|
546
|
-
|
547
|
-
|
548
|
-
)
|
549
|
-
is_new_version_of.save()
|
548
|
+
revises._revises = None # ensure we don't start a recursion
|
549
|
+
revises.save()
|
550
550
|
super(Record, self).save(*args, **kwargs)
|
551
|
+
self._revises = None
|
551
552
|
# save unversioned record
|
552
553
|
else:
|
553
554
|
super(Record, self).save(*args, **kwargs)
|
lamindb/_save.py
CHANGED
@@ -48,8 +48,7 @@ def save(records: Iterable[Record], ignore_conflicts: bool | None = False) -> No
|
|
48
48
|
|
49
49
|
Examples:
|
50
50
|
|
51
|
-
Save a
|
52
|
-
than writing a loop over ``projects.save()``:
|
51
|
+
Save a list of records:
|
53
52
|
|
54
53
|
>>> labels = [ln.ULabel(f"Label {i}") for i in range(10)]
|
55
54
|
>>> ln.save(projects)
|
@@ -61,7 +60,7 @@ def save(records: Iterable[Record], ignore_conflicts: bool | None = False) -> No
|
|
61
60
|
|
62
61
|
Update a single existing record:
|
63
62
|
|
64
|
-
>>> transform = ln.
|
63
|
+
>>> transform = ln.Transform.get("0Cb86EZj")
|
65
64
|
>>> transform.name = "New name"
|
66
65
|
>>> transform.save()
|
67
66
|
|
lamindb/_transform.py
CHANGED
@@ -2,12 +2,13 @@ from __future__ import annotations
|
|
2
2
|
|
3
3
|
from typing import TYPE_CHECKING
|
4
4
|
|
5
|
+
from lamin_utils import logger
|
5
6
|
from lamindb_setup.core._docs import doc_args
|
6
7
|
from lnschema_core.models import Run, Transform
|
7
8
|
|
8
9
|
from ._parents import _view_parents
|
9
10
|
from ._run import delete_run_artifacts
|
10
|
-
from .core.versioning import
|
11
|
+
from .core.versioning import message_update_key_in_version_family, process_revises
|
11
12
|
|
12
13
|
if TYPE_CHECKING:
|
13
14
|
from lnschema_core.types import TransformType
|
@@ -19,26 +20,38 @@ def __init__(transform: Transform, *args, **kwargs):
|
|
19
20
|
return None
|
20
21
|
name: str | None = kwargs.pop("name") if "name" in kwargs else None
|
21
22
|
key: str | None = kwargs.pop("key") if "key" in kwargs else None
|
22
|
-
|
23
|
-
kwargs.pop("is_new_version_of") if "is_new_version_of" in kwargs else None
|
24
|
-
)
|
23
|
+
revises: Transform | None = kwargs.pop("revises") if "revises" in kwargs else None
|
25
24
|
version: str | None = kwargs.pop("version") if "version" in kwargs else None
|
26
25
|
type: TransformType | None = kwargs.pop("type") if "type" in kwargs else "pipeline"
|
27
26
|
reference: str | None = kwargs.pop("reference") if "reference" in kwargs else None
|
28
27
|
reference_type: str | None = (
|
29
28
|
kwargs.pop("reference_type") if "reference_type" in kwargs else None
|
30
29
|
)
|
30
|
+
if "is_new_version_of" in kwargs:
|
31
|
+
logger.warning("`is_new_version_of` will be removed soon, please use `revises`")
|
32
|
+
revises = kwargs.pop("is_new_version_of")
|
31
33
|
# below is internal use that we'll hopefully be able to eliminate
|
32
34
|
uid: str | None = kwargs.pop("uid") if "uid" in kwargs else None
|
33
35
|
if not len(kwargs) == 0:
|
34
36
|
raise ValueError(
|
35
|
-
"Only name, key, version, type,
|
37
|
+
"Only name, key, version, type, revises, reference, "
|
36
38
|
f"reference_type can be passed, but you passed: {kwargs}"
|
37
39
|
)
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
40
|
+
# Transform allows passing a uid, all others don't
|
41
|
+
if uid is None and key is not None:
|
42
|
+
revises = Transform.filter(key=key).order_by("-created_at").first()
|
43
|
+
if revises is not None and key is not None and revises.key != key:
|
44
|
+
note = message_update_key_in_version_family(
|
45
|
+
suid=revises.stem_uid,
|
46
|
+
existing_key=revises.key,
|
47
|
+
new_key=key,
|
48
|
+
registry="Artifact",
|
49
|
+
)
|
50
|
+
raise ValueError(
|
51
|
+
f"`key` is {key}, but `revises.key` is '{revises.key}'\n\n Either do *not* pass `key`.\n\n{note}"
|
52
|
+
)
|
53
|
+
new_uid, version, name, revises = process_revises(revises, version, name, Transform)
|
54
|
+
# this is only because the user-facing constructor allows passing a uid
|
42
55
|
# most others don't
|
43
56
|
if uid is None:
|
44
57
|
has_consciously_provided_uid = False
|
@@ -54,7 +67,7 @@ def __init__(transform: Transform, *args, **kwargs):
|
|
54
67
|
reference=reference,
|
55
68
|
reference_type=reference_type,
|
56
69
|
_has_consciously_provided_uid=has_consciously_provided_uid,
|
57
|
-
|
70
|
+
revises=revises,
|
58
71
|
)
|
59
72
|
|
60
73
|
|