lamindb 0.77.3__py3-none-any.whl → 1.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. lamindb/__init__.py +39 -32
  2. lamindb/_artifact.py +95 -64
  3. lamindb/_can_curate.py +13 -6
  4. lamindb/_collection.py +51 -49
  5. lamindb/_feature.py +9 -9
  6. lamindb/_finish.py +92 -79
  7. lamindb/_from_values.py +13 -10
  8. lamindb/_is_versioned.py +2 -1
  9. lamindb/_parents.py +23 -16
  10. lamindb/_query_manager.py +3 -3
  11. lamindb/_query_set.py +85 -18
  12. lamindb/_record.py +114 -41
  13. lamindb/_run.py +3 -3
  14. lamindb/_save.py +5 -6
  15. lamindb/{_feature_set.py → _schema.py} +34 -31
  16. lamindb/_storage.py +2 -1
  17. lamindb/_transform.py +51 -23
  18. lamindb/_ulabel.py +17 -8
  19. lamindb/_view.py +13 -13
  20. lamindb/base/__init__.py +24 -0
  21. lamindb/base/fields.py +281 -0
  22. lamindb/base/ids.py +103 -0
  23. lamindb/base/types.py +51 -0
  24. lamindb/base/users.py +30 -0
  25. lamindb/base/validation.py +67 -0
  26. lamindb/core/__init__.py +18 -15
  27. lamindb/core/_context.py +295 -224
  28. lamindb/core/_data.py +44 -49
  29. lamindb/core/_describe.py +41 -31
  30. lamindb/core/_django.py +29 -27
  31. lamindb/core/_feature_manager.py +130 -129
  32. lamindb/core/_label_manager.py +7 -8
  33. lamindb/core/_mapped_collection.py +17 -14
  34. lamindb/core/_settings.py +1 -12
  35. lamindb/core/_sync_git.py +56 -9
  36. lamindb/core/_track_environment.py +1 -1
  37. lamindb/core/datasets/_core.py +5 -6
  38. lamindb/core/exceptions.py +0 -7
  39. lamindb/core/fields.py +1 -1
  40. lamindb/core/loaders.py +0 -1
  41. lamindb/core/{schema.py → relations.py} +22 -19
  42. lamindb/core/storage/_anndata_accessor.py +1 -2
  43. lamindb/core/storage/_backed_access.py +2 -1
  44. lamindb/core/storage/_tiledbsoma.py +38 -13
  45. lamindb/core/storage/objects.py +1 -1
  46. lamindb/core/storage/paths.py +13 -8
  47. lamindb/core/subsettings/__init__.py +0 -2
  48. lamindb/core/types.py +2 -23
  49. lamindb/core/versioning.py +11 -7
  50. lamindb/{_curate.py → curators/__init__.py} +122 -23
  51. lamindb/curators/_spatial.py +528 -0
  52. lamindb/integrations/_vitessce.py +1 -3
  53. lamindb/migrations/0052_squashed.py +1261 -0
  54. lamindb/migrations/0053_alter_featureset_hash_alter_paramvalue_created_by_and_more.py +57 -0
  55. lamindb/migrations/0054_alter_feature_previous_runs_and_more.py +35 -0
  56. lamindb/migrations/0055_artifact_type_artifactparamvalue_and_more.py +61 -0
  57. lamindb/migrations/0056_rename_ulabel_ref_is_name_artifactulabel_label_ref_is_name_and_more.py +22 -0
  58. lamindb/migrations/0057_link_models_latest_report_and_others.py +356 -0
  59. lamindb/migrations/0058_artifact__actions_collection__actions.py +22 -0
  60. lamindb/migrations/0059_alter_artifact__accessor_alter_artifact__hash_type_and_more.py +31 -0
  61. lamindb/migrations/0060_alter_artifact__actions.py +22 -0
  62. lamindb/migrations/0061_alter_collection_meta_artifact_alter_run_environment_and_more.py +45 -0
  63. lamindb/migrations/0062_add_is_latest_field.py +32 -0
  64. lamindb/migrations/0063_populate_latest_field.py +45 -0
  65. lamindb/migrations/0064_alter_artifact_version_alter_collection_version_and_more.py +33 -0
  66. lamindb/migrations/0065_remove_collection_feature_sets_and_more.py +22 -0
  67. lamindb/migrations/0066_alter_artifact__feature_values_and_more.py +352 -0
  68. lamindb/migrations/0067_alter_featurevalue_unique_together_and_more.py +20 -0
  69. lamindb/migrations/0068_alter_artifactulabel_unique_together_and_more.py +20 -0
  70. lamindb/migrations/0069_alter_artifact__accessor_alter_artifact__hash_type_and_more.py +1294 -0
  71. lamindb/migrations/0069_squashed.py +1770 -0
  72. lamindb/migrations/0070_lamindbv1_migrate_data.py +78 -0
  73. lamindb/migrations/0071_lamindbv1_migrate_schema.py +741 -0
  74. lamindb/migrations/0072_remove_user__branch_code_remove_user_aux_and_more.py +148 -0
  75. lamindb/migrations/0073_merge_ourprojects.py +945 -0
  76. lamindb/migrations/0074_lamindbv1_part4.py +374 -0
  77. lamindb/migrations/0075_lamindbv1_part5.py +276 -0
  78. lamindb/migrations/0076_lamindbv1_part6.py +621 -0
  79. lamindb/migrations/0077_lamindbv1_part6b.py +228 -0
  80. lamindb/migrations/0078_lamindbv1_part6c.py +468 -0
  81. lamindb/migrations/0079_alter_rundata_value_json_and_more.py +36 -0
  82. lamindb/migrations/__init__.py +0 -0
  83. lamindb/models.py +4064 -0
  84. {lamindb-0.77.3.dist-info → lamindb-1.0rc1.dist-info}/METADATA +13 -19
  85. lamindb-1.0rc1.dist-info/RECORD +100 -0
  86. {lamindb-0.77.3.dist-info → lamindb-1.0rc1.dist-info}/WHEEL +1 -1
  87. lamindb/core/subsettings/_transform_settings.py +0 -21
  88. lamindb-0.77.3.dist-info/RECORD +0 -63
  89. {lamindb-0.77.3.dist-info → lamindb-1.0rc1.dist-info}/LICENSE +0 -0
lamindb/_parents.py CHANGED
@@ -5,15 +5,22 @@ from typing import TYPE_CHECKING, Literal
5
5
 
6
6
  import lamindb_setup as ln_setup
7
7
  from lamin_utils import logger
8
- from lnschema_core import Artifact, Collection, Record, Run, Transform
9
- from lnschema_core.models import HasParents, format_field_value
8
+
9
+ from lamindb.models import (
10
+ Artifact,
11
+ Collection,
12
+ HasParents,
13
+ Record,
14
+ Run,
15
+ Transform,
16
+ format_field_value,
17
+ )
10
18
 
11
19
  from ._record import get_name_field
12
20
  from ._utils import attach_func_to_class_method
13
21
 
14
22
  if TYPE_CHECKING:
15
- from lnschema_core.types import StrField
16
-
23
+ from lamindb.base.types import StrField
17
24
  from lamindb.core import QuerySet
18
25
 
19
26
  LAMIN_GREEN_LIGHTER = "#10b981"
@@ -25,7 +32,7 @@ TRANSFORM_EMOJIS = {
25
32
  "pipeline": "🧩",
26
33
  "script": "📝",
27
34
  "function": "🔧",
28
- "glue": "🧲",
35
+ "linker": "🧲",
29
36
  }
30
37
  is_run_from_ipython = getattr(builtins, "__IPYTHON__", False)
31
38
 
@@ -342,8 +349,8 @@ def _record_label(record: Record, field: str | None = None):
342
349
  rf' FACE="Monospace">uid={record.uid}<BR/>version={record.version}</FONT>>'
343
350
  )
344
351
  elif isinstance(record, Run):
345
- if record.transform.name:
346
- name = f'{record.transform.name.replace("&", "&amp;")}'
352
+ if record.transform.description:
353
+ name = f'{record.transform.description.replace("&", "&amp;")}'
347
354
  elif record.transform.key:
348
355
  name = f'{record.transform.key.replace("&", "&amp;")}'
349
356
  else:
@@ -395,22 +402,22 @@ def _get_all_parent_runs(data: Artifact | Collection) -> list:
395
402
  inputs_run = (
396
403
  r.__getattribute__(f"input_{name}s")
397
404
  .all()
398
- .filter(visibility__in=[0, 1])
405
+ .filter(_branch_code__in=[0, 1])
399
406
  .list()
400
407
  )
401
408
  if name == "artifact":
402
409
  inputs_run += (
403
- r.input_collections.all().filter(visibility__in=[0, 1]).list()
410
+ r.input_collections.all().filter(_branch_code__in=[0, 1]).list()
404
411
  )
405
412
  outputs_run = (
406
413
  r.__getattribute__(f"output_{name}s")
407
414
  .all()
408
- .filter(visibility__in=[0, 1])
415
+ .filter(_branch_code__in=[0, 1])
409
416
  .list()
410
417
  )
411
418
  if name == "artifact":
412
419
  outputs_run += (
413
- r.output_collections.all().filter(visibility__in=[0, 1]).list()
420
+ r.output_collections.all().filter(_branch_code__in=[0, 1]).list()
414
421
  )
415
422
  # if inputs are outputs artifacts are the same, will result infinite loop
416
423
  # so only show as outputs
@@ -444,7 +451,7 @@ def _get_all_child_runs(data: Artifact | Collection) -> list:
444
451
  {
445
452
  f.run
446
453
  for f in data.run.output_collections.all()
447
- .filter(visibility__in=[0, 1])
454
+ .filter(_branch_code__in=[0, 1])
448
455
  .all()
449
456
  }
450
457
  )
@@ -455,24 +462,24 @@ def _get_all_child_runs(data: Artifact | Collection) -> list:
455
462
  inputs_run = (
456
463
  r.__getattribute__(f"input_{name}s")
457
464
  .all()
458
- .filter(visibility__in=[0, 1])
465
+ .filter(_branch_code__in=[0, 1])
459
466
  .list()
460
467
  )
461
468
  if name == "artifact":
462
469
  inputs_run += (
463
- r.input_collections.all().filter(visibility__in=[0, 1]).list()
470
+ r.input_collections.all().filter(_branch_code__in=[0, 1]).list()
464
471
  )
465
472
  run_inputs_outputs += [(inputs_run, r)]
466
473
 
467
474
  outputs_run = (
468
475
  r.__getattribute__(f"output_{name}s")
469
476
  .all()
470
- .filter(visibility__in=[0, 1])
477
+ .filter(_branch_code__in=[0, 1])
471
478
  .list()
472
479
  )
473
480
  if name == "artifact":
474
481
  outputs_run += (
475
- r.output_collections.all().filter(visibility__in=[0, 1]).list()
482
+ r.output_collections.all().filter(_branch_code__in=[0, 1]).list()
476
483
  )
477
484
  run_inputs_outputs += [(r, outputs_run)]
478
485
 
lamindb/_query_manager.py CHANGED
@@ -5,13 +5,13 @@ from typing import TYPE_CHECKING, NamedTuple
5
5
  from django.db import models
6
6
  from lamin_utils import logger
7
7
  from lamindb_setup.core._docs import doc_args
8
- from lnschema_core.models import Record
9
8
 
10
- from .core._feature_manager import get_feature_set_by_slot_
9
+ from lamindb.models import Record
10
+
11
11
  from .core._settings import settings
12
12
 
13
13
  if TYPE_CHECKING:
14
- from lnschema_core.types import StrField
14
+ from lamindb.base.types import StrField
15
15
 
16
16
 
17
17
  class QueryManager(models.Manager):
lamindb/_query_set.py CHANGED
@@ -1,6 +1,7 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import re
4
+ import warnings
4
5
  from collections import UserList
5
6
  from collections.abc import Iterable
6
7
  from collections.abc import Iterable as IterableType
@@ -12,17 +13,17 @@ from django.db.models import F, ForeignKey, ManyToManyField
12
13
  from django.db.models.fields.related import ForeignObjectRel
13
14
  from lamin_utils import logger
14
15
  from lamindb_setup.core._docs import doc_args
15
- from lnschema_core.models import (
16
+
17
+ from lamindb.models import (
16
18
  Artifact,
17
19
  CanCurate,
18
20
  Collection,
19
21
  Feature,
20
22
  IsVersioned,
21
23
  Record,
22
- Registry,
23
24
  Run,
25
+ Schema,
24
26
  Transform,
25
- VisibilityChoice,
26
27
  )
27
28
 
28
29
  from .core.exceptions import DoesNotExist
@@ -32,7 +33,7 @@ T = TypeVar("T")
32
33
  if TYPE_CHECKING:
33
34
  from collections.abc import Iterable
34
35
 
35
- from lnschema_core.types import ListLike, StrField
36
+ from lamindb.base.types import ListLike, StrField
36
37
 
37
38
 
38
39
  class MultipleResultsFound(Exception):
@@ -79,6 +80,58 @@ def one_helper(self):
79
80
  return self[0]
80
81
 
81
82
 
83
+ def get_backward_compat_filter_kwargs(queryset, expressions):
84
+ if queryset.model in {Collection, Transform}:
85
+ name_mappings = {
86
+ "name": "key",
87
+ "visibility": "_branch_code", # for convenience (and backward compat <1.0)
88
+ }
89
+ elif queryset.model == Artifact:
90
+ name_mappings = {
91
+ "n_objects": "n_files",
92
+ "visibility": "_branch_code", # for convenience (and backward compat <1.0)
93
+ "transform": "run__transform", # for convenience (and backward compat <1.0)
94
+ "feature_sets": "_schemas_m2m",
95
+ "type": "kind",
96
+ "_accessor": "otype",
97
+ }
98
+ elif queryset.model == Schema:
99
+ name_mappings = {
100
+ "registry": "itype",
101
+ "artifacts": "_artifacts_m2m", # will raise warning when we start to migrate over
102
+ }
103
+ else:
104
+ return expressions
105
+ was_list = False
106
+ if isinstance(expressions, list):
107
+ # make a dummy dictionary
108
+ was_list = True
109
+ expressions = {field: True for field in expressions}
110
+ mapped = {}
111
+ for field, value in expressions.items():
112
+ parts = field.split("__")
113
+ if parts[0] in name_mappings:
114
+ if parts[0] not in {
115
+ "transform",
116
+ "visibility",
117
+ "feature_sets",
118
+ "schemas",
119
+ "artifacts",
120
+ }:
121
+ warnings.warn(
122
+ f"{name_mappings[parts[0]]} is deprecated, please query for {parts[0]} instead",
123
+ DeprecationWarning,
124
+ stacklevel=2,
125
+ )
126
+ new_field = name_mappings[parts[0]] + (
127
+ "__" + "__".join(parts[1:]) if len(parts) > 1 else ""
128
+ )
129
+ mapped[new_field] = value
130
+ else:
131
+ mapped[field] = value
132
+ return list(mapped.keys()) if was_list else mapped
133
+
134
+
82
135
  def process_expressions(queryset: QuerySet, expressions: dict) -> dict:
83
136
  def _map_databases(value: Any, key: str, target_db: str) -> tuple[str, Any]:
84
137
  if isinstance(value, Record):
@@ -105,23 +158,26 @@ def process_expressions(queryset: QuerySet, expressions: dict) -> dict:
105
158
 
106
159
  return key, value
107
160
 
108
- if queryset.model in {Artifact, Collection}:
109
- # visibility is set to 0 unless expressions contains id or uid equality
161
+ expressions = get_backward_compat_filter_kwargs(
162
+ queryset,
163
+ expressions,
164
+ )
165
+
166
+ if issubclass(queryset.model, Record):
167
+ # _branch_code is set to 0 unless expressions contains id or uid
110
168
  if not (
111
169
  "id" in expressions
112
170
  or "uid" in expressions
113
171
  or "uid__startswith" in expressions
114
172
  ):
115
- visibility = "visibility"
116
- if not any(e.startswith(visibility) for e in expressions):
117
- expressions[visibility] = (
118
- VisibilityChoice.default.value
119
- ) # default visibility
120
- # if visibility is None, do not apply a filter
173
+ _branch_code = "_branch_code"
174
+ if not any(e.startswith(_branch_code) for e in expressions):
175
+ expressions[_branch_code] = 1 # default _branch_code
176
+ # if _branch_code is None, do not apply a filter
121
177
  # otherwise, it would mean filtering for NULL values, which doesn't make
122
178
  # sense for a non-NULLABLE column
123
- elif visibility in expressions and expressions[visibility] is None:
124
- expressions.pop(visibility)
179
+ elif _branch_code in expressions and expressions[_branch_code] is None:
180
+ expressions.pop(_branch_code)
125
181
  if queryset._db is not None:
126
182
  # only check for database mismatch if there is a defined database on the
127
183
  # queryset
@@ -213,6 +269,8 @@ def get_basic_field_names(
213
269
  "created_at",
214
270
  "created_by_id",
215
271
  "updated_at",
272
+ "_aux",
273
+ "_branch_code",
216
274
  ]:
217
275
  if field_name in field_names:
218
276
  field_names.remove(field_name)
@@ -242,17 +300,17 @@ def get_feature_annotate_kwargs(show_features: bool | list[str]) -> dict[str, An
242
300
  link_models_on_models = {
243
301
  getattr(
244
302
  Artifact, obj.related_name
245
- ).through.__get_name_with_schema__(): obj.related_model.__get_name_with_schema__()
303
+ ).through.__get_name_with_module__(): obj.related_model.__get_name_with_module__()
246
304
  for obj in Artifact._meta.related_objects
247
- if obj.related_model.__get_name_with_schema__() in cat_feature_types
305
+ if obj.related_model.__get_name_with_module__() in cat_feature_types
248
306
  }
249
307
  link_models_on_models["ArtifactULabel"] = "ULabel"
250
308
  link_attributes_on_models = {
251
309
  obj.related_name: link_models_on_models[
252
- obj.related_model.__get_name_with_schema__()
310
+ obj.related_model.__get_name_with_module__()
253
311
  ]
254
312
  for obj in Artifact._meta.related_objects
255
- if obj.related_model.__get_name_with_schema__() in link_models_on_models
313
+ if obj.related_model.__get_name_with_module__() in link_models_on_models
256
314
  }
257
315
  # Prepare Django's annotate for features
258
316
  annotate_kwargs = {}
@@ -478,7 +536,9 @@ class QuerySet(models.QuerySet):
478
536
  include = []
479
537
  elif isinstance(include, str):
480
538
  include = [include]
539
+ include = get_backward_compat_filter_kwargs(self, include)
481
540
  field_names = get_basic_field_names(self, include, features)
541
+
482
542
  annotate_kwargs = {}
483
543
  if features:
484
544
  annotate_kwargs.update(get_feature_annotate_kwargs(features))
@@ -490,6 +550,7 @@ class QuerySet(models.QuerySet):
490
550
  queryset = self.annotate(**annotate_kwargs)
491
551
  else:
492
552
  queryset = self
553
+
493
554
  df = pd.DataFrame(queryset.values(*field_names, *list(annotate_kwargs.keys())))
494
555
  if len(df) == 0:
495
556
  df = pd.DataFrame({}, columns=field_names)
@@ -500,6 +561,12 @@ class QuerySet(models.QuerySet):
500
561
  pk_column_name = pk_name if pk_name in df.columns else f"{pk_name}_id"
501
562
  if pk_column_name in df_reshaped.columns:
502
563
  df_reshaped = df_reshaped.set_index(pk_column_name)
564
+
565
+ # Compatibility code
566
+ df_reshaped.columns = df_reshaped.columns.str.replace(
567
+ r"_schemas_m2m", "feature_sets", regex=True
568
+ )
569
+
503
570
  return df_reshaped
504
571
 
505
572
  def delete(self, *args, **kwargs):
lamindb/_record.py CHANGED
@@ -3,14 +3,21 @@ from __future__ import annotations
3
3
  import builtins
4
4
  import re
5
5
  from functools import reduce
6
+ from pathlib import PurePosixPath
6
7
  from typing import TYPE_CHECKING, NamedTuple
7
8
 
8
9
  import dj_database_url
9
10
  import lamindb_setup as ln_setup
10
- from django.core.exceptions import FieldDoesNotExist
11
11
  from django.core.exceptions import ValidationError as DjangoValidationError
12
12
  from django.db import connections, transaction
13
- from django.db.models import F, IntegerField, Manager, Q, QuerySet, TextField, Value
13
+ from django.db.models import (
14
+ IntegerField,
15
+ Manager,
16
+ Q,
17
+ QuerySet,
18
+ TextField,
19
+ Value,
20
+ )
14
21
  from django.db.models.functions import Cast, Coalesce
15
22
  from django.db.models.lookups import (
16
23
  Contains,
@@ -32,28 +39,37 @@ from lamindb_setup._connect_instance import (
32
39
  from lamindb_setup.core._docs import doc_args
33
40
  from lamindb_setup.core._hub_core import connect_instance_hub
34
41
  from lamindb_setup.core._settings_store import instance_settings_file
35
- from lnschema_core.models import (
42
+ from lamindb_setup.core.upath import extract_suffix_from_path
43
+
44
+ from lamindb.base.validation import FieldValidationError
45
+ from lamindb.models import (
36
46
  Artifact,
47
+ BasicRecord,
48
+ CanCurate,
37
49
  Collection,
38
50
  Feature,
39
- FeatureSet,
40
51
  IsVersioned,
41
52
  Param,
42
53
  Record,
43
54
  Run,
55
+ Schema,
44
56
  Transform,
45
57
  ULabel,
46
58
  ValidateFields,
47
59
  )
48
- from lnschema_core.validation import FieldValidationError
49
60
 
50
61
  from ._utils import attach_func_to_class_method
51
62
  from .core._settings import settings
52
- from .core.exceptions import RecordNameChangeIntegrityError, ValidationError
63
+ from .core.exceptions import (
64
+ InvalidArgument,
65
+ RecordNameChangeIntegrityError,
66
+ ValidationError,
67
+ )
53
68
 
54
69
  if TYPE_CHECKING:
55
70
  import pandas as pd
56
- from lnschema_core.types import ListLike, StrField
71
+
72
+ from lamindb.base.types import StrField
57
73
 
58
74
 
59
75
  IPYTHON = getattr(builtins, "__IPYTHON__", False)
@@ -76,7 +92,7 @@ def update_attributes(record: Record, attributes: dict[str, str]):
76
92
 
77
93
 
78
94
  def validate_fields(record: Record, kwargs):
79
- from lnschema_core.validation import validate_literal_fields
95
+ from lamindb.base.validation import validate_literal_fields
80
96
 
81
97
  # validate required fields
82
98
  # a "required field" is a Django field that has `null=False, default=None`
@@ -98,7 +114,7 @@ def validate_fields(record: Record, kwargs):
98
114
  Run,
99
115
  ULabel,
100
116
  Feature,
101
- FeatureSet,
117
+ Schema,
102
118
  Param,
103
119
  }:
104
120
  uid_max_length = record.__class__._meta.get_field(
@@ -153,7 +169,13 @@ def __init__(record: Record, *args, **kwargs):
153
169
  has_consciously_provided_uid = False
154
170
  if "_has_consciously_provided_uid" in kwargs:
155
171
  has_consciously_provided_uid = kwargs.pop("_has_consciously_provided_uid")
156
- if settings.creation.search_names and not has_consciously_provided_uid:
172
+ if (
173
+ isinstance(
174
+ record, (CanCurate, Collection, Transform)
175
+ ) # Collection is only temporary because it'll get a key field
176
+ and settings.creation.search_names
177
+ and not has_consciously_provided_uid
178
+ ):
157
179
  name_field = getattr(record, "_name_field", "name")
158
180
  match = suggest_records_with_similar_names(record, name_field, kwargs)
159
181
  if match:
@@ -184,7 +206,7 @@ def __init__(record: Record, *args, **kwargs):
184
206
  )
185
207
  init_self_from_db(record, existing_record)
186
208
  return None
187
- super(Record, record).__init__(**kwargs)
209
+ super(BasicRecord, record).__init__(**kwargs)
188
210
  if isinstance(record, ValidateFields):
189
211
  # this will trigger validation against django validators
190
212
  try:
@@ -199,8 +221,9 @@ def __init__(record: Record, *args, **kwargs):
199
221
  raise ValueError("please provide keyword arguments, not plain arguments")
200
222
  else:
201
223
  # object is loaded from DB (**kwargs could be omitted below, I believe)
202
- super(Record, record).__init__(*args, **kwargs)
224
+ super(BasicRecord, record).__init__(*args, **kwargs)
203
225
  _store_record_old_name(record)
226
+ _store_record_old_key(record)
204
227
 
205
228
 
206
229
  def _format_django_validation_error(record: Record, e: DjangoValidationError):
@@ -285,6 +308,9 @@ def _search(
285
308
  using_key: str | None = None,
286
309
  truncate_string: bool = False,
287
310
  ) -> QuerySet:
311
+ if string is None:
312
+ raise ValueError("Cannot search for None value! Please pass a valid string.")
313
+
288
314
  input_queryset = _queryset(cls, using_key=using_key)
289
315
  registry = input_queryset.model
290
316
  name_field = getattr(registry, "_name_field", "name")
@@ -526,14 +552,14 @@ def using(
526
552
  f"Failed to load instance {instance}, please check your permissions!"
527
553
  )
528
554
  iresult, _ = result
529
- source_schema = {
530
- schema for schema in iresult["schema_str"].split(",") if schema != ""
555
+ source_module = {
556
+ modules for modules in iresult["schema_str"].split(",") if modules != ""
531
557
  } # type: ignore
532
- target_schema = ln_setup.settings.instance.schema
533
- if not source_schema.issubset(target_schema):
534
- missing_members = source_schema - target_schema
558
+ target_module = ln_setup.settings.instance.modules
559
+ if not source_module.issubset(target_module):
560
+ missing_members = source_module - target_module
535
561
  logger.warning(
536
- f"source schema has additional modules: {missing_members}\nconsider mounting these schema modules to transfer all metadata"
562
+ f"source modules has additional modules: {missing_members}\nconsider mounting these registry modules to transfer all metadata"
537
563
  )
538
564
  cache_filepath.write_text(f"{iresult['lnid']}\n{iresult['schema_str']}") # type: ignore
539
565
  settings_file = instance_settings_file(name, owner)
@@ -541,7 +567,7 @@ def using(
541
567
  else:
542
568
  isettings = load_instance_settings(settings_file)
543
569
  db = isettings.db
544
- cache_filepath.write_text(f"{isettings.uid}\n{','.join(isettings.schema)}") # type: ignore
570
+ cache_filepath.write_text(f"{isettings.uid}\n{','.join(isettings.modules)}") # type: ignore
545
571
  add_db_connection(db, instance)
546
572
  return QuerySet(model=cls, using=instance)
547
573
 
@@ -550,6 +576,7 @@ REGISTRY_UNIQUE_FIELD = {
550
576
  "storage": "root",
551
577
  "feature": "name",
552
578
  "ulabel": "name",
579
+ "space": "name", # TODO: this should be updated with the currently used space instead during transfer
553
580
  }
554
581
 
555
582
 
@@ -585,7 +612,6 @@ def update_fk_to_default_db(
585
612
  FKBULK = [
586
613
  "organism",
587
614
  "source",
588
- "_source_code_artifact", # Transform
589
615
  "report", # Run
590
616
  ]
591
617
 
@@ -598,8 +624,6 @@ def transfer_fk_to_default_db_bulk(
598
624
 
599
625
 
600
626
  def get_transfer_run(record) -> Run:
601
- from lamindb_setup import settings as setup_settings
602
-
603
627
  from lamindb.core._context import context
604
628
  from lamindb.core._data import WARNING_RUN_TRANSFORM
605
629
 
@@ -619,18 +643,20 @@ def get_transfer_run(record) -> Run:
619
643
  uid=uid, name=f"Transfer from `{slug}`", key=key, type="function"
620
644
  ).save()
621
645
  settings.creation.search_names = search_names
622
- # use the global run context to get the parent run id
646
+ # use the global run context to get the initiated_by_run run id
623
647
  if context.run is not None:
624
- parent = context.run
648
+ initiated_by_run = context.run
625
649
  else:
626
650
  if not settings.creation.artifact_silence_missing_run_warning:
627
651
  logger.warning(WARNING_RUN_TRANSFORM)
628
- parent = None
652
+ initiated_by_run = None
629
653
  # it doesn't seem to make sense to create new runs for every transfer
630
- run = Run.filter(transform=transform, parent=parent).one_or_none()
654
+ run = Run.filter(
655
+ transform=transform, initiated_by_run=initiated_by_run
656
+ ).one_or_none()
631
657
  if run is None:
632
- run = Run(transform=transform, parent=parent).save()
633
- run.parent = parent # so that it's available in memory
658
+ run = Run(transform=transform, initiated_by_run=initiated_by_run).save()
659
+ run.initiated_by_run = initiated_by_run # so that it's available in memory
634
660
  return run
635
661
 
636
662
 
@@ -721,14 +747,19 @@ def save(self, *args, **kwargs) -> Record:
721
747
  revises._revises = None # ensure we don't start a recursion
722
748
  revises.save()
723
749
  check_name_change(self)
724
- super(Record, self).save(*args, **kwargs)
750
+ check_key_change(self)
751
+ super(BasicRecord, self).save(*args, **kwargs)
725
752
  _store_record_old_name(self)
753
+ _store_record_old_key(self)
726
754
  self._revises = None
727
755
  # save unversioned record
728
756
  else:
729
757
  check_name_change(self)
730
- super(Record, self).save(*args, **kwargs)
758
+ check_key_change(self)
759
+ super(BasicRecord, self).save(*args, **kwargs)
760
+ # update _old_name and _old_key after saving
731
761
  _store_record_old_name(self)
762
+ _store_record_old_key(self)
732
763
  # perform transfer of many-to-many fields
733
764
  # only supported for Artifact and Collection records
734
765
  if db is not None and db != "default" and using_key is None:
@@ -741,7 +772,7 @@ def save(self, *args, **kwargs) -> Record:
741
772
  if hasattr(self, "labels"):
742
773
  from copy import copy
743
774
 
744
- from lnschema_core.models import FeatureManager
775
+ from lamindb.models import FeatureManager
745
776
 
746
777
  # here we go back to original record on the source database
747
778
  self_on_db = copy(self)
@@ -759,19 +790,33 @@ def save(self, *args, **kwargs) -> Record:
759
790
  def _store_record_old_name(record: Record):
760
791
  # writes the name to the _name attribute, so we can detect renaming upon save
761
792
  if hasattr(record, "_name_field"):
762
- record._name = getattr(record, record._name_field)
793
+ record._old_name = getattr(record, record._name_field)
794
+
795
+
796
+ def _store_record_old_key(record: Record):
797
+ # writes the key to the _old_key attribute, so we can detect key changes upon save
798
+ if isinstance(record, (Artifact, Transform)):
799
+ record._old_key = record.key
763
800
 
764
801
 
765
802
  def check_name_change(record: Record):
766
803
  """Warns if a record's name has changed."""
767
804
  if (
768
805
  not record.pk
769
- or not hasattr(record, "_name")
806
+ or not hasattr(record, "_old_name")
770
807
  or not hasattr(record, "_name_field")
771
808
  ):
772
809
  return
773
810
 
774
- old_name = record._name
811
+ # checked in check_key_change or not checked at all
812
+ if isinstance(record, (Artifact, Collection, Transform)):
813
+ return
814
+
815
+ # renaming feature sets is not checked
816
+ if isinstance(record, Schema):
817
+ return
818
+
819
+ old_name = record._old_name
775
820
  new_name = getattr(record, record._name_field)
776
821
  registry = record.__class__.__name__
777
822
 
@@ -785,13 +830,13 @@ def check_name_change(record: Record):
785
830
  .exclude(feature_id=None) # must have a feature
786
831
  .exclude(
787
832
  feature_ref_is_name=None
788
- ) # must be linked via Curator and therefore part of a featureset
833
+ ) # must be linked via Curator and therefore part of a schema
789
834
  .distinct()
790
835
  )
791
836
  artifact_ids = linked_records.list("artifact__uid")
792
837
  n = len(artifact_ids)
793
- s = "s" if n > 1 else ""
794
838
  if n > 0:
839
+ s = "s" if n > 1 else ""
795
840
  logger.error(
796
841
  f"You are trying to {colors.red('rename label')} from '{old_name}' to '{new_name}'!\n"
797
842
  f" → The following {n} artifact{s} {colors.red('will no longer be validated')}: {artifact_ids}\n\n"
@@ -805,13 +850,13 @@ def check_name_change(record: Record):
805
850
 
806
851
  # when a feature is renamed
807
852
  elif isinstance(record, Feature):
808
- # only internal features are associated with featuresets
809
- linked_artifacts = Artifact.filter(feature_sets__features=record).list(
853
+ # only internal features are associated with schemas
854
+ linked_artifacts = Artifact.filter(_schemas_m2m__features=record).list(
810
855
  "uid"
811
856
  )
812
857
  n = len(linked_artifacts)
813
- s = "s" if n > 1 else ""
814
858
  if n > 0:
859
+ s = "s" if n > 1 else ""
815
860
  logger.error(
816
861
  f"You are trying to {colors.red('rename feature')} from '{old_name}' to '{new_name}'!\n"
817
862
  f" → The following {n} artifact{s} {colors.red('will no longer be validated')}: {linked_artifacts}\n\n"
@@ -824,6 +869,33 @@ def check_name_change(record: Record):
824
869
  raise RecordNameChangeIntegrityError
825
870
 
826
871
 
872
+ def check_key_change(record: Artifact | Transform):
873
+ """Errors if a record's key has falsely changed."""
874
+ if not isinstance(record, Artifact) or not hasattr(record, "_old_key"):
875
+ return
876
+
877
+ old_key = record._old_key or ""
878
+ new_key = record.key or ""
879
+
880
+ if old_key != new_key:
881
+ if not record._key_is_virtual:
882
+ raise InvalidArgument(
883
+ f"Changing a non-virtual key of an artifact is not allowed! Tried to change key from '{old_key}' to '{new_key}'."
884
+ )
885
+ old_key_suffix = (
886
+ record.suffix
887
+ if record.suffix
888
+ else extract_suffix_from_path(PurePosixPath(old_key), arg_name="key")
889
+ )
890
+ new_key_suffix = extract_suffix_from_path(
891
+ PurePosixPath(new_key), arg_name="key"
892
+ )
893
+ if old_key_suffix != new_key_suffix:
894
+ raise InvalidArgument(
895
+ f"The suffix '{new_key_suffix}' of the provided key is incorrect, it should be '{old_key_suffix}'."
896
+ )
897
+
898
+
827
899
  def delete(self) -> None:
828
900
  """Delete the record."""
829
901
  # note that the logic below does not fire if a record is moved to the trash
@@ -843,10 +915,10 @@ def delete(self) -> None:
843
915
  new_latest.is_latest = True
844
916
  with transaction.atomic():
845
917
  new_latest.save()
846
- super(Record, self).delete()
918
+ super(BasicRecord, self).delete()
847
919
  logger.warning(f"new latest version is {new_latest}")
848
920
  return None
849
- super(Record, self).delete()
921
+ super(BasicRecord, self).delete()
850
922
 
851
923
 
852
924
  METHOD_NAMES = [
@@ -871,4 +943,5 @@ if ln_setup._TESTING: # type: ignore
871
943
  }
872
944
 
873
945
  for name in METHOD_NAMES:
946
+ attach_func_to_class_method(name, BasicRecord, globals())
874
947
  attach_func_to_class_method(name, Record, globals())