lamindb 0.77.3__py3-none-any.whl → 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lamindb/__init__.py +39 -32
- lamindb/_artifact.py +95 -64
- lamindb/_can_curate.py +13 -6
- lamindb/_collection.py +51 -49
- lamindb/_feature.py +9 -9
- lamindb/_finish.py +92 -79
- lamindb/_from_values.py +13 -10
- lamindb/_is_versioned.py +2 -1
- lamindb/_parents.py +23 -16
- lamindb/_query_manager.py +3 -3
- lamindb/_query_set.py +85 -18
- lamindb/_record.py +114 -41
- lamindb/_run.py +3 -3
- lamindb/_save.py +5 -6
- lamindb/{_feature_set.py → _schema.py} +34 -31
- lamindb/_storage.py +2 -1
- lamindb/_transform.py +51 -23
- lamindb/_ulabel.py +17 -8
- lamindb/_view.py +13 -13
- lamindb/base/__init__.py +24 -0
- lamindb/base/fields.py +281 -0
- lamindb/base/ids.py +103 -0
- lamindb/base/types.py +51 -0
- lamindb/base/users.py +30 -0
- lamindb/base/validation.py +67 -0
- lamindb/core/__init__.py +18 -15
- lamindb/core/_context.py +295 -224
- lamindb/core/_data.py +44 -49
- lamindb/core/_describe.py +41 -31
- lamindb/core/_django.py +29 -27
- lamindb/core/_feature_manager.py +130 -129
- lamindb/core/_label_manager.py +7 -8
- lamindb/core/_mapped_collection.py +17 -14
- lamindb/core/_settings.py +1 -12
- lamindb/core/_sync_git.py +56 -9
- lamindb/core/_track_environment.py +1 -1
- lamindb/core/datasets/_core.py +5 -6
- lamindb/core/exceptions.py +0 -7
- lamindb/core/fields.py +1 -1
- lamindb/core/loaders.py +0 -1
- lamindb/core/{schema.py → relations.py} +22 -19
- lamindb/core/storage/_anndata_accessor.py +1 -2
- lamindb/core/storage/_backed_access.py +2 -1
- lamindb/core/storage/_tiledbsoma.py +38 -13
- lamindb/core/storage/objects.py +1 -1
- lamindb/core/storage/paths.py +13 -8
- lamindb/core/subsettings/__init__.py +0 -2
- lamindb/core/types.py +2 -23
- lamindb/core/versioning.py +11 -7
- lamindb/{_curate.py → curators/__init__.py} +122 -23
- lamindb/curators/_spatial.py +528 -0
- lamindb/integrations/_vitessce.py +1 -3
- lamindb/migrations/0052_squashed.py +1261 -0
- lamindb/migrations/0053_alter_featureset_hash_alter_paramvalue_created_by_and_more.py +57 -0
- lamindb/migrations/0054_alter_feature_previous_runs_and_more.py +35 -0
- lamindb/migrations/0055_artifact_type_artifactparamvalue_and_more.py +61 -0
- lamindb/migrations/0056_rename_ulabel_ref_is_name_artifactulabel_label_ref_is_name_and_more.py +22 -0
- lamindb/migrations/0057_link_models_latest_report_and_others.py +356 -0
- lamindb/migrations/0058_artifact__actions_collection__actions.py +22 -0
- lamindb/migrations/0059_alter_artifact__accessor_alter_artifact__hash_type_and_more.py +31 -0
- lamindb/migrations/0060_alter_artifact__actions.py +22 -0
- lamindb/migrations/0061_alter_collection_meta_artifact_alter_run_environment_and_more.py +45 -0
- lamindb/migrations/0062_add_is_latest_field.py +32 -0
- lamindb/migrations/0063_populate_latest_field.py +45 -0
- lamindb/migrations/0064_alter_artifact_version_alter_collection_version_and_more.py +33 -0
- lamindb/migrations/0065_remove_collection_feature_sets_and_more.py +22 -0
- lamindb/migrations/0066_alter_artifact__feature_values_and_more.py +352 -0
- lamindb/migrations/0067_alter_featurevalue_unique_together_and_more.py +20 -0
- lamindb/migrations/0068_alter_artifactulabel_unique_together_and_more.py +20 -0
- lamindb/migrations/0069_alter_artifact__accessor_alter_artifact__hash_type_and_more.py +1294 -0
- lamindb/migrations/0069_squashed.py +1770 -0
- lamindb/migrations/0070_lamindbv1_migrate_data.py +78 -0
- lamindb/migrations/0071_lamindbv1_migrate_schema.py +741 -0
- lamindb/migrations/0072_remove_user__branch_code_remove_user_aux_and_more.py +148 -0
- lamindb/migrations/0073_merge_ourprojects.py +945 -0
- lamindb/migrations/0074_lamindbv1_part4.py +374 -0
- lamindb/migrations/0075_lamindbv1_part5.py +276 -0
- lamindb/migrations/0076_lamindbv1_part6.py +621 -0
- lamindb/migrations/0077_lamindbv1_part6b.py +228 -0
- lamindb/migrations/0078_lamindbv1_part6c.py +468 -0
- lamindb/migrations/0079_alter_rundata_value_json_and_more.py +36 -0
- lamindb/migrations/__init__.py +0 -0
- lamindb/models.py +4064 -0
- {lamindb-0.77.3.dist-info → lamindb-1.0.0.dist-info}/METADATA +13 -19
- lamindb-1.0.0.dist-info/RECORD +100 -0
- {lamindb-0.77.3.dist-info → lamindb-1.0.0.dist-info}/WHEEL +1 -1
- lamindb/core/subsettings/_transform_settings.py +0 -21
- lamindb-0.77.3.dist-info/RECORD +0 -63
- {lamindb-0.77.3.dist-info → lamindb-1.0.0.dist-info}/LICENSE +0 -0
lamindb/_parents.py
CHANGED
@@ -5,15 +5,22 @@ from typing import TYPE_CHECKING, Literal
|
|
5
5
|
|
6
6
|
import lamindb_setup as ln_setup
|
7
7
|
from lamin_utils import logger
|
8
|
-
|
9
|
-
from
|
8
|
+
|
9
|
+
from lamindb.models import (
|
10
|
+
Artifact,
|
11
|
+
Collection,
|
12
|
+
HasParents,
|
13
|
+
Record,
|
14
|
+
Run,
|
15
|
+
Transform,
|
16
|
+
format_field_value,
|
17
|
+
)
|
10
18
|
|
11
19
|
from ._record import get_name_field
|
12
20
|
from ._utils import attach_func_to_class_method
|
13
21
|
|
14
22
|
if TYPE_CHECKING:
|
15
|
-
from
|
16
|
-
|
23
|
+
from lamindb.base.types import StrField
|
17
24
|
from lamindb.core import QuerySet
|
18
25
|
|
19
26
|
LAMIN_GREEN_LIGHTER = "#10b981"
|
@@ -25,7 +32,7 @@ TRANSFORM_EMOJIS = {
|
|
25
32
|
"pipeline": "🧩",
|
26
33
|
"script": "📝",
|
27
34
|
"function": "🔧",
|
28
|
-
"
|
35
|
+
"linker": "🧲",
|
29
36
|
}
|
30
37
|
is_run_from_ipython = getattr(builtins, "__IPYTHON__", False)
|
31
38
|
|
@@ -342,8 +349,8 @@ def _record_label(record: Record, field: str | None = None):
|
|
342
349
|
rf' FACE="Monospace">uid={record.uid}<BR/>version={record.version}</FONT>>'
|
343
350
|
)
|
344
351
|
elif isinstance(record, Run):
|
345
|
-
if record.transform.
|
346
|
-
name = f'{record.transform.
|
352
|
+
if record.transform.description:
|
353
|
+
name = f'{record.transform.description.replace("&", "&")}'
|
347
354
|
elif record.transform.key:
|
348
355
|
name = f'{record.transform.key.replace("&", "&")}'
|
349
356
|
else:
|
@@ -395,22 +402,22 @@ def _get_all_parent_runs(data: Artifact | Collection) -> list:
|
|
395
402
|
inputs_run = (
|
396
403
|
r.__getattribute__(f"input_{name}s")
|
397
404
|
.all()
|
398
|
-
.filter(
|
405
|
+
.filter(_branch_code__in=[0, 1])
|
399
406
|
.list()
|
400
407
|
)
|
401
408
|
if name == "artifact":
|
402
409
|
inputs_run += (
|
403
|
-
r.input_collections.all().filter(
|
410
|
+
r.input_collections.all().filter(_branch_code__in=[0, 1]).list()
|
404
411
|
)
|
405
412
|
outputs_run = (
|
406
413
|
r.__getattribute__(f"output_{name}s")
|
407
414
|
.all()
|
408
|
-
.filter(
|
415
|
+
.filter(_branch_code__in=[0, 1])
|
409
416
|
.list()
|
410
417
|
)
|
411
418
|
if name == "artifact":
|
412
419
|
outputs_run += (
|
413
|
-
r.output_collections.all().filter(
|
420
|
+
r.output_collections.all().filter(_branch_code__in=[0, 1]).list()
|
414
421
|
)
|
415
422
|
# if inputs are outputs artifacts are the same, will result infinite loop
|
416
423
|
# so only show as outputs
|
@@ -444,7 +451,7 @@ def _get_all_child_runs(data: Artifact | Collection) -> list:
|
|
444
451
|
{
|
445
452
|
f.run
|
446
453
|
for f in data.run.output_collections.all()
|
447
|
-
.filter(
|
454
|
+
.filter(_branch_code__in=[0, 1])
|
448
455
|
.all()
|
449
456
|
}
|
450
457
|
)
|
@@ -455,24 +462,24 @@ def _get_all_child_runs(data: Artifact | Collection) -> list:
|
|
455
462
|
inputs_run = (
|
456
463
|
r.__getattribute__(f"input_{name}s")
|
457
464
|
.all()
|
458
|
-
.filter(
|
465
|
+
.filter(_branch_code__in=[0, 1])
|
459
466
|
.list()
|
460
467
|
)
|
461
468
|
if name == "artifact":
|
462
469
|
inputs_run += (
|
463
|
-
r.input_collections.all().filter(
|
470
|
+
r.input_collections.all().filter(_branch_code__in=[0, 1]).list()
|
464
471
|
)
|
465
472
|
run_inputs_outputs += [(inputs_run, r)]
|
466
473
|
|
467
474
|
outputs_run = (
|
468
475
|
r.__getattribute__(f"output_{name}s")
|
469
476
|
.all()
|
470
|
-
.filter(
|
477
|
+
.filter(_branch_code__in=[0, 1])
|
471
478
|
.list()
|
472
479
|
)
|
473
480
|
if name == "artifact":
|
474
481
|
outputs_run += (
|
475
|
-
r.output_collections.all().filter(
|
482
|
+
r.output_collections.all().filter(_branch_code__in=[0, 1]).list()
|
476
483
|
)
|
477
484
|
run_inputs_outputs += [(r, outputs_run)]
|
478
485
|
|
lamindb/_query_manager.py
CHANGED
@@ -5,13 +5,13 @@ from typing import TYPE_CHECKING, NamedTuple
|
|
5
5
|
from django.db import models
|
6
6
|
from lamin_utils import logger
|
7
7
|
from lamindb_setup.core._docs import doc_args
|
8
|
-
from lnschema_core.models import Record
|
9
8
|
|
10
|
-
from .
|
9
|
+
from lamindb.models import Record
|
10
|
+
|
11
11
|
from .core._settings import settings
|
12
12
|
|
13
13
|
if TYPE_CHECKING:
|
14
|
-
from
|
14
|
+
from lamindb.base.types import StrField
|
15
15
|
|
16
16
|
|
17
17
|
class QueryManager(models.Manager):
|
lamindb/_query_set.py
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
|
3
3
|
import re
|
4
|
+
import warnings
|
4
5
|
from collections import UserList
|
5
6
|
from collections.abc import Iterable
|
6
7
|
from collections.abc import Iterable as IterableType
|
@@ -12,17 +13,17 @@ from django.db.models import F, ForeignKey, ManyToManyField
|
|
12
13
|
from django.db.models.fields.related import ForeignObjectRel
|
13
14
|
from lamin_utils import logger
|
14
15
|
from lamindb_setup.core._docs import doc_args
|
15
|
-
|
16
|
+
|
17
|
+
from lamindb.models import (
|
16
18
|
Artifact,
|
17
19
|
CanCurate,
|
18
20
|
Collection,
|
19
21
|
Feature,
|
20
22
|
IsVersioned,
|
21
23
|
Record,
|
22
|
-
Registry,
|
23
24
|
Run,
|
25
|
+
Schema,
|
24
26
|
Transform,
|
25
|
-
VisibilityChoice,
|
26
27
|
)
|
27
28
|
|
28
29
|
from .core.exceptions import DoesNotExist
|
@@ -32,7 +33,7 @@ T = TypeVar("T")
|
|
32
33
|
if TYPE_CHECKING:
|
33
34
|
from collections.abc import Iterable
|
34
35
|
|
35
|
-
from
|
36
|
+
from lamindb.base.types import ListLike, StrField
|
36
37
|
|
37
38
|
|
38
39
|
class MultipleResultsFound(Exception):
|
@@ -79,6 +80,58 @@ def one_helper(self):
|
|
79
80
|
return self[0]
|
80
81
|
|
81
82
|
|
83
|
+
def get_backward_compat_filter_kwargs(queryset, expressions):
|
84
|
+
if queryset.model in {Collection, Transform}:
|
85
|
+
name_mappings = {
|
86
|
+
"name": "key",
|
87
|
+
"visibility": "_branch_code", # for convenience (and backward compat <1.0)
|
88
|
+
}
|
89
|
+
elif queryset.model == Artifact:
|
90
|
+
name_mappings = {
|
91
|
+
"n_objects": "n_files",
|
92
|
+
"visibility": "_branch_code", # for convenience (and backward compat <1.0)
|
93
|
+
"transform": "run__transform", # for convenience (and backward compat <1.0)
|
94
|
+
"feature_sets": "_schemas_m2m",
|
95
|
+
"type": "kind",
|
96
|
+
"_accessor": "otype",
|
97
|
+
}
|
98
|
+
elif queryset.model == Schema:
|
99
|
+
name_mappings = {
|
100
|
+
"registry": "itype",
|
101
|
+
"artifacts": "_artifacts_m2m", # will raise warning when we start to migrate over
|
102
|
+
}
|
103
|
+
else:
|
104
|
+
return expressions
|
105
|
+
was_list = False
|
106
|
+
if isinstance(expressions, list):
|
107
|
+
# make a dummy dictionary
|
108
|
+
was_list = True
|
109
|
+
expressions = {field: True for field in expressions}
|
110
|
+
mapped = {}
|
111
|
+
for field, value in expressions.items():
|
112
|
+
parts = field.split("__")
|
113
|
+
if parts[0] in name_mappings:
|
114
|
+
if parts[0] not in {
|
115
|
+
"transform",
|
116
|
+
"visibility",
|
117
|
+
"feature_sets",
|
118
|
+
"schemas",
|
119
|
+
"artifacts",
|
120
|
+
}:
|
121
|
+
warnings.warn(
|
122
|
+
f"{name_mappings[parts[0]]} is deprecated, please query for {parts[0]} instead",
|
123
|
+
DeprecationWarning,
|
124
|
+
stacklevel=2,
|
125
|
+
)
|
126
|
+
new_field = name_mappings[parts[0]] + (
|
127
|
+
"__" + "__".join(parts[1:]) if len(parts) > 1 else ""
|
128
|
+
)
|
129
|
+
mapped[new_field] = value
|
130
|
+
else:
|
131
|
+
mapped[field] = value
|
132
|
+
return list(mapped.keys()) if was_list else mapped
|
133
|
+
|
134
|
+
|
82
135
|
def process_expressions(queryset: QuerySet, expressions: dict) -> dict:
|
83
136
|
def _map_databases(value: Any, key: str, target_db: str) -> tuple[str, Any]:
|
84
137
|
if isinstance(value, Record):
|
@@ -105,23 +158,26 @@ def process_expressions(queryset: QuerySet, expressions: dict) -> dict:
|
|
105
158
|
|
106
159
|
return key, value
|
107
160
|
|
108
|
-
|
109
|
-
|
161
|
+
expressions = get_backward_compat_filter_kwargs(
|
162
|
+
queryset,
|
163
|
+
expressions,
|
164
|
+
)
|
165
|
+
|
166
|
+
if issubclass(queryset.model, Record):
|
167
|
+
# _branch_code is set to 0 unless expressions contains id or uid
|
110
168
|
if not (
|
111
169
|
"id" in expressions
|
112
170
|
or "uid" in expressions
|
113
171
|
or "uid__startswith" in expressions
|
114
172
|
):
|
115
|
-
|
116
|
-
if not any(e.startswith(
|
117
|
-
expressions[
|
118
|
-
|
119
|
-
) # default visibility
|
120
|
-
# if visibility is None, do not apply a filter
|
173
|
+
_branch_code = "_branch_code"
|
174
|
+
if not any(e.startswith(_branch_code) for e in expressions):
|
175
|
+
expressions[_branch_code] = 1 # default _branch_code
|
176
|
+
# if _branch_code is None, do not apply a filter
|
121
177
|
# otherwise, it would mean filtering for NULL values, which doesn't make
|
122
178
|
# sense for a non-NULLABLE column
|
123
|
-
elif
|
124
|
-
expressions.pop(
|
179
|
+
elif _branch_code in expressions and expressions[_branch_code] is None:
|
180
|
+
expressions.pop(_branch_code)
|
125
181
|
if queryset._db is not None:
|
126
182
|
# only check for database mismatch if there is a defined database on the
|
127
183
|
# queryset
|
@@ -213,6 +269,8 @@ def get_basic_field_names(
|
|
213
269
|
"created_at",
|
214
270
|
"created_by_id",
|
215
271
|
"updated_at",
|
272
|
+
"_aux",
|
273
|
+
"_branch_code",
|
216
274
|
]:
|
217
275
|
if field_name in field_names:
|
218
276
|
field_names.remove(field_name)
|
@@ -242,17 +300,17 @@ def get_feature_annotate_kwargs(show_features: bool | list[str]) -> dict[str, An
|
|
242
300
|
link_models_on_models = {
|
243
301
|
getattr(
|
244
302
|
Artifact, obj.related_name
|
245
|
-
).through.
|
303
|
+
).through.__get_name_with_module__(): obj.related_model.__get_name_with_module__()
|
246
304
|
for obj in Artifact._meta.related_objects
|
247
|
-
if obj.related_model.
|
305
|
+
if obj.related_model.__get_name_with_module__() in cat_feature_types
|
248
306
|
}
|
249
307
|
link_models_on_models["ArtifactULabel"] = "ULabel"
|
250
308
|
link_attributes_on_models = {
|
251
309
|
obj.related_name: link_models_on_models[
|
252
|
-
obj.related_model.
|
310
|
+
obj.related_model.__get_name_with_module__()
|
253
311
|
]
|
254
312
|
for obj in Artifact._meta.related_objects
|
255
|
-
if obj.related_model.
|
313
|
+
if obj.related_model.__get_name_with_module__() in link_models_on_models
|
256
314
|
}
|
257
315
|
# Prepare Django's annotate for features
|
258
316
|
annotate_kwargs = {}
|
@@ -478,7 +536,9 @@ class QuerySet(models.QuerySet):
|
|
478
536
|
include = []
|
479
537
|
elif isinstance(include, str):
|
480
538
|
include = [include]
|
539
|
+
include = get_backward_compat_filter_kwargs(self, include)
|
481
540
|
field_names = get_basic_field_names(self, include, features)
|
541
|
+
|
482
542
|
annotate_kwargs = {}
|
483
543
|
if features:
|
484
544
|
annotate_kwargs.update(get_feature_annotate_kwargs(features))
|
@@ -490,6 +550,7 @@ class QuerySet(models.QuerySet):
|
|
490
550
|
queryset = self.annotate(**annotate_kwargs)
|
491
551
|
else:
|
492
552
|
queryset = self
|
553
|
+
|
493
554
|
df = pd.DataFrame(queryset.values(*field_names, *list(annotate_kwargs.keys())))
|
494
555
|
if len(df) == 0:
|
495
556
|
df = pd.DataFrame({}, columns=field_names)
|
@@ -500,6 +561,12 @@ class QuerySet(models.QuerySet):
|
|
500
561
|
pk_column_name = pk_name if pk_name in df.columns else f"{pk_name}_id"
|
501
562
|
if pk_column_name in df_reshaped.columns:
|
502
563
|
df_reshaped = df_reshaped.set_index(pk_column_name)
|
564
|
+
|
565
|
+
# Compatibility code
|
566
|
+
df_reshaped.columns = df_reshaped.columns.str.replace(
|
567
|
+
r"_schemas_m2m", "feature_sets", regex=True
|
568
|
+
)
|
569
|
+
|
503
570
|
return df_reshaped
|
504
571
|
|
505
572
|
def delete(self, *args, **kwargs):
|
lamindb/_record.py
CHANGED
@@ -3,14 +3,21 @@ from __future__ import annotations
|
|
3
3
|
import builtins
|
4
4
|
import re
|
5
5
|
from functools import reduce
|
6
|
+
from pathlib import PurePosixPath
|
6
7
|
from typing import TYPE_CHECKING, NamedTuple
|
7
8
|
|
8
9
|
import dj_database_url
|
9
10
|
import lamindb_setup as ln_setup
|
10
|
-
from django.core.exceptions import FieldDoesNotExist
|
11
11
|
from django.core.exceptions import ValidationError as DjangoValidationError
|
12
12
|
from django.db import connections, transaction
|
13
|
-
from django.db.models import
|
13
|
+
from django.db.models import (
|
14
|
+
IntegerField,
|
15
|
+
Manager,
|
16
|
+
Q,
|
17
|
+
QuerySet,
|
18
|
+
TextField,
|
19
|
+
Value,
|
20
|
+
)
|
14
21
|
from django.db.models.functions import Cast, Coalesce
|
15
22
|
from django.db.models.lookups import (
|
16
23
|
Contains,
|
@@ -32,28 +39,37 @@ from lamindb_setup._connect_instance import (
|
|
32
39
|
from lamindb_setup.core._docs import doc_args
|
33
40
|
from lamindb_setup.core._hub_core import connect_instance_hub
|
34
41
|
from lamindb_setup.core._settings_store import instance_settings_file
|
35
|
-
from
|
42
|
+
from lamindb_setup.core.upath import extract_suffix_from_path
|
43
|
+
|
44
|
+
from lamindb.base.validation import FieldValidationError
|
45
|
+
from lamindb.models import (
|
36
46
|
Artifact,
|
47
|
+
BasicRecord,
|
48
|
+
CanCurate,
|
37
49
|
Collection,
|
38
50
|
Feature,
|
39
|
-
FeatureSet,
|
40
51
|
IsVersioned,
|
41
52
|
Param,
|
42
53
|
Record,
|
43
54
|
Run,
|
55
|
+
Schema,
|
44
56
|
Transform,
|
45
57
|
ULabel,
|
46
58
|
ValidateFields,
|
47
59
|
)
|
48
|
-
from lnschema_core.validation import FieldValidationError
|
49
60
|
|
50
61
|
from ._utils import attach_func_to_class_method
|
51
62
|
from .core._settings import settings
|
52
|
-
from .core.exceptions import
|
63
|
+
from .core.exceptions import (
|
64
|
+
InvalidArgument,
|
65
|
+
RecordNameChangeIntegrityError,
|
66
|
+
ValidationError,
|
67
|
+
)
|
53
68
|
|
54
69
|
if TYPE_CHECKING:
|
55
70
|
import pandas as pd
|
56
|
-
|
71
|
+
|
72
|
+
from lamindb.base.types import StrField
|
57
73
|
|
58
74
|
|
59
75
|
IPYTHON = getattr(builtins, "__IPYTHON__", False)
|
@@ -76,7 +92,7 @@ def update_attributes(record: Record, attributes: dict[str, str]):
|
|
76
92
|
|
77
93
|
|
78
94
|
def validate_fields(record: Record, kwargs):
|
79
|
-
from
|
95
|
+
from lamindb.base.validation import validate_literal_fields
|
80
96
|
|
81
97
|
# validate required fields
|
82
98
|
# a "required field" is a Django field that has `null=False, default=None`
|
@@ -98,7 +114,7 @@ def validate_fields(record: Record, kwargs):
|
|
98
114
|
Run,
|
99
115
|
ULabel,
|
100
116
|
Feature,
|
101
|
-
|
117
|
+
Schema,
|
102
118
|
Param,
|
103
119
|
}:
|
104
120
|
uid_max_length = record.__class__._meta.get_field(
|
@@ -153,7 +169,13 @@ def __init__(record: Record, *args, **kwargs):
|
|
153
169
|
has_consciously_provided_uid = False
|
154
170
|
if "_has_consciously_provided_uid" in kwargs:
|
155
171
|
has_consciously_provided_uid = kwargs.pop("_has_consciously_provided_uid")
|
156
|
-
if
|
172
|
+
if (
|
173
|
+
isinstance(
|
174
|
+
record, (CanCurate, Collection, Transform)
|
175
|
+
) # Collection is only temporary because it'll get a key field
|
176
|
+
and settings.creation.search_names
|
177
|
+
and not has_consciously_provided_uid
|
178
|
+
):
|
157
179
|
name_field = getattr(record, "_name_field", "name")
|
158
180
|
match = suggest_records_with_similar_names(record, name_field, kwargs)
|
159
181
|
if match:
|
@@ -184,7 +206,7 @@ def __init__(record: Record, *args, **kwargs):
|
|
184
206
|
)
|
185
207
|
init_self_from_db(record, existing_record)
|
186
208
|
return None
|
187
|
-
super(
|
209
|
+
super(BasicRecord, record).__init__(**kwargs)
|
188
210
|
if isinstance(record, ValidateFields):
|
189
211
|
# this will trigger validation against django validators
|
190
212
|
try:
|
@@ -199,8 +221,9 @@ def __init__(record: Record, *args, **kwargs):
|
|
199
221
|
raise ValueError("please provide keyword arguments, not plain arguments")
|
200
222
|
else:
|
201
223
|
# object is loaded from DB (**kwargs could be omitted below, I believe)
|
202
|
-
super(
|
224
|
+
super(BasicRecord, record).__init__(*args, **kwargs)
|
203
225
|
_store_record_old_name(record)
|
226
|
+
_store_record_old_key(record)
|
204
227
|
|
205
228
|
|
206
229
|
def _format_django_validation_error(record: Record, e: DjangoValidationError):
|
@@ -285,6 +308,9 @@ def _search(
|
|
285
308
|
using_key: str | None = None,
|
286
309
|
truncate_string: bool = False,
|
287
310
|
) -> QuerySet:
|
311
|
+
if string is None:
|
312
|
+
raise ValueError("Cannot search for None value! Please pass a valid string.")
|
313
|
+
|
288
314
|
input_queryset = _queryset(cls, using_key=using_key)
|
289
315
|
registry = input_queryset.model
|
290
316
|
name_field = getattr(registry, "_name_field", "name")
|
@@ -526,14 +552,14 @@ def using(
|
|
526
552
|
f"Failed to load instance {instance}, please check your permissions!"
|
527
553
|
)
|
528
554
|
iresult, _ = result
|
529
|
-
|
530
|
-
|
555
|
+
source_module = {
|
556
|
+
modules for modules in iresult["schema_str"].split(",") if modules != ""
|
531
557
|
} # type: ignore
|
532
|
-
|
533
|
-
if not
|
534
|
-
missing_members =
|
558
|
+
target_module = ln_setup.settings.instance.modules
|
559
|
+
if not source_module.issubset(target_module):
|
560
|
+
missing_members = source_module - target_module
|
535
561
|
logger.warning(
|
536
|
-
f"source
|
562
|
+
f"source modules has additional modules: {missing_members}\nconsider mounting these registry modules to transfer all metadata"
|
537
563
|
)
|
538
564
|
cache_filepath.write_text(f"{iresult['lnid']}\n{iresult['schema_str']}") # type: ignore
|
539
565
|
settings_file = instance_settings_file(name, owner)
|
@@ -541,7 +567,7 @@ def using(
|
|
541
567
|
else:
|
542
568
|
isettings = load_instance_settings(settings_file)
|
543
569
|
db = isettings.db
|
544
|
-
cache_filepath.write_text(f"{isettings.uid}\n{','.join(isettings.
|
570
|
+
cache_filepath.write_text(f"{isettings.uid}\n{','.join(isettings.modules)}") # type: ignore
|
545
571
|
add_db_connection(db, instance)
|
546
572
|
return QuerySet(model=cls, using=instance)
|
547
573
|
|
@@ -550,6 +576,7 @@ REGISTRY_UNIQUE_FIELD = {
|
|
550
576
|
"storage": "root",
|
551
577
|
"feature": "name",
|
552
578
|
"ulabel": "name",
|
579
|
+
"space": "name", # TODO: this should be updated with the currently used space instead during transfer
|
553
580
|
}
|
554
581
|
|
555
582
|
|
@@ -585,7 +612,6 @@ def update_fk_to_default_db(
|
|
585
612
|
FKBULK = [
|
586
613
|
"organism",
|
587
614
|
"source",
|
588
|
-
"_source_code_artifact", # Transform
|
589
615
|
"report", # Run
|
590
616
|
]
|
591
617
|
|
@@ -598,8 +624,6 @@ def transfer_fk_to_default_db_bulk(
|
|
598
624
|
|
599
625
|
|
600
626
|
def get_transfer_run(record) -> Run:
|
601
|
-
from lamindb_setup import settings as setup_settings
|
602
|
-
|
603
627
|
from lamindb.core._context import context
|
604
628
|
from lamindb.core._data import WARNING_RUN_TRANSFORM
|
605
629
|
|
@@ -619,18 +643,20 @@ def get_transfer_run(record) -> Run:
|
|
619
643
|
uid=uid, name=f"Transfer from `{slug}`", key=key, type="function"
|
620
644
|
).save()
|
621
645
|
settings.creation.search_names = search_names
|
622
|
-
# use the global run context to get the
|
646
|
+
# use the global run context to get the initiated_by_run run id
|
623
647
|
if context.run is not None:
|
624
|
-
|
648
|
+
initiated_by_run = context.run
|
625
649
|
else:
|
626
650
|
if not settings.creation.artifact_silence_missing_run_warning:
|
627
651
|
logger.warning(WARNING_RUN_TRANSFORM)
|
628
|
-
|
652
|
+
initiated_by_run = None
|
629
653
|
# it doesn't seem to make sense to create new runs for every transfer
|
630
|
-
run = Run.filter(
|
654
|
+
run = Run.filter(
|
655
|
+
transform=transform, initiated_by_run=initiated_by_run
|
656
|
+
).one_or_none()
|
631
657
|
if run is None:
|
632
|
-
run = Run(transform=transform,
|
633
|
-
run.
|
658
|
+
run = Run(transform=transform, initiated_by_run=initiated_by_run).save()
|
659
|
+
run.initiated_by_run = initiated_by_run # so that it's available in memory
|
634
660
|
return run
|
635
661
|
|
636
662
|
|
@@ -721,14 +747,19 @@ def save(self, *args, **kwargs) -> Record:
|
|
721
747
|
revises._revises = None # ensure we don't start a recursion
|
722
748
|
revises.save()
|
723
749
|
check_name_change(self)
|
724
|
-
|
750
|
+
check_key_change(self)
|
751
|
+
super(BasicRecord, self).save(*args, **kwargs)
|
725
752
|
_store_record_old_name(self)
|
753
|
+
_store_record_old_key(self)
|
726
754
|
self._revises = None
|
727
755
|
# save unversioned record
|
728
756
|
else:
|
729
757
|
check_name_change(self)
|
730
|
-
|
758
|
+
check_key_change(self)
|
759
|
+
super(BasicRecord, self).save(*args, **kwargs)
|
760
|
+
# update _old_name and _old_key after saving
|
731
761
|
_store_record_old_name(self)
|
762
|
+
_store_record_old_key(self)
|
732
763
|
# perform transfer of many-to-many fields
|
733
764
|
# only supported for Artifact and Collection records
|
734
765
|
if db is not None and db != "default" and using_key is None:
|
@@ -741,7 +772,7 @@ def save(self, *args, **kwargs) -> Record:
|
|
741
772
|
if hasattr(self, "labels"):
|
742
773
|
from copy import copy
|
743
774
|
|
744
|
-
from
|
775
|
+
from lamindb.models import FeatureManager
|
745
776
|
|
746
777
|
# here we go back to original record on the source database
|
747
778
|
self_on_db = copy(self)
|
@@ -759,19 +790,33 @@ def save(self, *args, **kwargs) -> Record:
|
|
759
790
|
def _store_record_old_name(record: Record):
|
760
791
|
# writes the name to the _name attribute, so we can detect renaming upon save
|
761
792
|
if hasattr(record, "_name_field"):
|
762
|
-
record.
|
793
|
+
record._old_name = getattr(record, record._name_field)
|
794
|
+
|
795
|
+
|
796
|
+
def _store_record_old_key(record: Record):
|
797
|
+
# writes the key to the _old_key attribute, so we can detect key changes upon save
|
798
|
+
if isinstance(record, (Artifact, Transform)):
|
799
|
+
record._old_key = record.key
|
763
800
|
|
764
801
|
|
765
802
|
def check_name_change(record: Record):
|
766
803
|
"""Warns if a record's name has changed."""
|
767
804
|
if (
|
768
805
|
not record.pk
|
769
|
-
or not hasattr(record, "
|
806
|
+
or not hasattr(record, "_old_name")
|
770
807
|
or not hasattr(record, "_name_field")
|
771
808
|
):
|
772
809
|
return
|
773
810
|
|
774
|
-
|
811
|
+
# checked in check_key_change or not checked at all
|
812
|
+
if isinstance(record, (Artifact, Collection, Transform)):
|
813
|
+
return
|
814
|
+
|
815
|
+
# renaming feature sets is not checked
|
816
|
+
if isinstance(record, Schema):
|
817
|
+
return
|
818
|
+
|
819
|
+
old_name = record._old_name
|
775
820
|
new_name = getattr(record, record._name_field)
|
776
821
|
registry = record.__class__.__name__
|
777
822
|
|
@@ -785,13 +830,13 @@ def check_name_change(record: Record):
|
|
785
830
|
.exclude(feature_id=None) # must have a feature
|
786
831
|
.exclude(
|
787
832
|
feature_ref_is_name=None
|
788
|
-
) # must be linked via Curator and therefore part of a
|
833
|
+
) # must be linked via Curator and therefore part of a schema
|
789
834
|
.distinct()
|
790
835
|
)
|
791
836
|
artifact_ids = linked_records.list("artifact__uid")
|
792
837
|
n = len(artifact_ids)
|
793
|
-
s = "s" if n > 1 else ""
|
794
838
|
if n > 0:
|
839
|
+
s = "s" if n > 1 else ""
|
795
840
|
logger.error(
|
796
841
|
f"You are trying to {colors.red('rename label')} from '{old_name}' to '{new_name}'!\n"
|
797
842
|
f" → The following {n} artifact{s} {colors.red('will no longer be validated')}: {artifact_ids}\n\n"
|
@@ -805,13 +850,13 @@ def check_name_change(record: Record):
|
|
805
850
|
|
806
851
|
# when a feature is renamed
|
807
852
|
elif isinstance(record, Feature):
|
808
|
-
# only internal features are associated with
|
809
|
-
linked_artifacts = Artifact.filter(
|
853
|
+
# only internal features are associated with schemas
|
854
|
+
linked_artifacts = Artifact.filter(_schemas_m2m__features=record).list(
|
810
855
|
"uid"
|
811
856
|
)
|
812
857
|
n = len(linked_artifacts)
|
813
|
-
s = "s" if n > 1 else ""
|
814
858
|
if n > 0:
|
859
|
+
s = "s" if n > 1 else ""
|
815
860
|
logger.error(
|
816
861
|
f"You are trying to {colors.red('rename feature')} from '{old_name}' to '{new_name}'!\n"
|
817
862
|
f" → The following {n} artifact{s} {colors.red('will no longer be validated')}: {linked_artifacts}\n\n"
|
@@ -824,6 +869,33 @@ def check_name_change(record: Record):
|
|
824
869
|
raise RecordNameChangeIntegrityError
|
825
870
|
|
826
871
|
|
872
|
+
def check_key_change(record: Artifact | Transform):
|
873
|
+
"""Errors if a record's key has falsely changed."""
|
874
|
+
if not isinstance(record, Artifact) or not hasattr(record, "_old_key"):
|
875
|
+
return
|
876
|
+
|
877
|
+
old_key = record._old_key or ""
|
878
|
+
new_key = record.key or ""
|
879
|
+
|
880
|
+
if old_key != new_key:
|
881
|
+
if not record._key_is_virtual:
|
882
|
+
raise InvalidArgument(
|
883
|
+
f"Changing a non-virtual key of an artifact is not allowed! Tried to change key from '{old_key}' to '{new_key}'."
|
884
|
+
)
|
885
|
+
old_key_suffix = (
|
886
|
+
record.suffix
|
887
|
+
if record.suffix
|
888
|
+
else extract_suffix_from_path(PurePosixPath(old_key), arg_name="key")
|
889
|
+
)
|
890
|
+
new_key_suffix = extract_suffix_from_path(
|
891
|
+
PurePosixPath(new_key), arg_name="key"
|
892
|
+
)
|
893
|
+
if old_key_suffix != new_key_suffix:
|
894
|
+
raise InvalidArgument(
|
895
|
+
f"The suffix '{new_key_suffix}' of the provided key is incorrect, it should be '{old_key_suffix}'."
|
896
|
+
)
|
897
|
+
|
898
|
+
|
827
899
|
def delete(self) -> None:
|
828
900
|
"""Delete the record."""
|
829
901
|
# note that the logic below does not fire if a record is moved to the trash
|
@@ -843,10 +915,10 @@ def delete(self) -> None:
|
|
843
915
|
new_latest.is_latest = True
|
844
916
|
with transaction.atomic():
|
845
917
|
new_latest.save()
|
846
|
-
super(
|
918
|
+
super(BasicRecord, self).delete()
|
847
919
|
logger.warning(f"new latest version is {new_latest}")
|
848
920
|
return None
|
849
|
-
super(
|
921
|
+
super(BasicRecord, self).delete()
|
850
922
|
|
851
923
|
|
852
924
|
METHOD_NAMES = [
|
@@ -871,4 +943,5 @@ if ln_setup._TESTING: # type: ignore
|
|
871
943
|
}
|
872
944
|
|
873
945
|
for name in METHOD_NAMES:
|
946
|
+
attach_func_to_class_method(name, BasicRecord, globals())
|
874
947
|
attach_func_to_class_method(name, Record, globals())
|