lamindb 0.77.2__py3-none-any.whl → 1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lamindb/__init__.py +39 -32
- lamindb/_artifact.py +95 -64
- lamindb/_can_curate.py +19 -10
- lamindb/_collection.py +51 -49
- lamindb/_feature.py +9 -9
- lamindb/_finish.py +99 -86
- lamindb/_from_values.py +20 -17
- lamindb/_is_versioned.py +2 -1
- lamindb/_parents.py +23 -16
- lamindb/_query_manager.py +3 -3
- lamindb/_query_set.py +85 -18
- lamindb/_record.py +121 -46
- lamindb/_run.py +3 -3
- lamindb/_save.py +14 -8
- lamindb/{_feature_set.py → _schema.py} +34 -31
- lamindb/_storage.py +2 -1
- lamindb/_transform.py +51 -23
- lamindb/_ulabel.py +17 -8
- lamindb/_view.py +15 -14
- lamindb/base/__init__.py +24 -0
- lamindb/base/fields.py +281 -0
- lamindb/base/ids.py +103 -0
- lamindb/base/types.py +51 -0
- lamindb/base/users.py +30 -0
- lamindb/base/validation.py +67 -0
- lamindb/core/__init__.py +19 -14
- lamindb/core/_context.py +297 -228
- lamindb/core/_data.py +44 -49
- lamindb/core/_describe.py +41 -31
- lamindb/core/_django.py +59 -44
- lamindb/core/_feature_manager.py +192 -168
- lamindb/core/_label_manager.py +22 -22
- lamindb/core/_mapped_collection.py +17 -14
- lamindb/core/_settings.py +1 -12
- lamindb/core/_sync_git.py +56 -9
- lamindb/core/_track_environment.py +1 -1
- lamindb/core/datasets/_core.py +5 -6
- lamindb/core/exceptions.py +0 -7
- lamindb/core/fields.py +1 -1
- lamindb/core/loaders.py +18 -2
- lamindb/core/{schema.py → relations.py} +22 -19
- lamindb/core/storage/_anndata_accessor.py +1 -2
- lamindb/core/storage/_backed_access.py +2 -1
- lamindb/core/storage/_tiledbsoma.py +40 -13
- lamindb/core/storage/objects.py +1 -1
- lamindb/core/storage/paths.py +13 -8
- lamindb/core/subsettings/__init__.py +0 -2
- lamindb/core/types.py +2 -23
- lamindb/core/versioning.py +11 -7
- lamindb/{_curate.py → curators/__init__.py} +700 -57
- lamindb/curators/_spatial.py +528 -0
- lamindb/integrations/_vitessce.py +1 -3
- lamindb/migrations/0052_squashed.py +1261 -0
- lamindb/migrations/0053_alter_featureset_hash_alter_paramvalue_created_by_and_more.py +57 -0
- lamindb/migrations/0054_alter_feature_previous_runs_and_more.py +35 -0
- lamindb/migrations/0055_artifact_type_artifactparamvalue_and_more.py +61 -0
- lamindb/migrations/0056_rename_ulabel_ref_is_name_artifactulabel_label_ref_is_name_and_more.py +22 -0
- lamindb/migrations/0057_link_models_latest_report_and_others.py +356 -0
- lamindb/migrations/0058_artifact__actions_collection__actions.py +22 -0
- lamindb/migrations/0059_alter_artifact__accessor_alter_artifact__hash_type_and_more.py +31 -0
- lamindb/migrations/0060_alter_artifact__actions.py +22 -0
- lamindb/migrations/0061_alter_collection_meta_artifact_alter_run_environment_and_more.py +45 -0
- lamindb/migrations/0062_add_is_latest_field.py +32 -0
- lamindb/migrations/0063_populate_latest_field.py +45 -0
- lamindb/migrations/0064_alter_artifact_version_alter_collection_version_and_more.py +33 -0
- lamindb/migrations/0065_remove_collection_feature_sets_and_more.py +22 -0
- lamindb/migrations/0066_alter_artifact__feature_values_and_more.py +352 -0
- lamindb/migrations/0067_alter_featurevalue_unique_together_and_more.py +20 -0
- lamindb/migrations/0068_alter_artifactulabel_unique_together_and_more.py +20 -0
- lamindb/migrations/0069_alter_artifact__accessor_alter_artifact__hash_type_and_more.py +1294 -0
- lamindb/migrations/0069_squashed.py +1770 -0
- lamindb/migrations/0070_lamindbv1_migrate_data.py +78 -0
- lamindb/migrations/0071_lamindbv1_migrate_schema.py +741 -0
- lamindb/migrations/0072_remove_user__branch_code_remove_user_aux_and_more.py +148 -0
- lamindb/migrations/0073_merge_ourprojects.py +945 -0
- lamindb/migrations/0074_lamindbv1_part4.py +374 -0
- lamindb/migrations/0075_lamindbv1_part5.py +276 -0
- lamindb/migrations/0076_lamindbv1_part6.py +621 -0
- lamindb/migrations/0077_lamindbv1_part6b.py +228 -0
- lamindb/migrations/0078_lamindbv1_part6c.py +468 -0
- lamindb/migrations/0079_alter_rundata_value_json_and_more.py +36 -0
- lamindb/migrations/__init__.py +0 -0
- lamindb/models.py +4064 -0
- {lamindb-0.77.2.dist-info → lamindb-1.0rc1.dist-info}/METADATA +15 -20
- lamindb-1.0rc1.dist-info/RECORD +100 -0
- {lamindb-0.77.2.dist-info → lamindb-1.0rc1.dist-info}/WHEEL +1 -1
- lamindb/core/subsettings/_transform_settings.py +0 -21
- lamindb-0.77.2.dist-info/RECORD +0 -63
- {lamindb-0.77.2.dist-info → lamindb-1.0rc1.dist-info}/LICENSE +0 -0
lamindb/_record.py
CHANGED
@@ -1,15 +1,23 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
|
3
3
|
import builtins
|
4
|
+
import re
|
4
5
|
from functools import reduce
|
6
|
+
from pathlib import PurePosixPath
|
5
7
|
from typing import TYPE_CHECKING, NamedTuple
|
6
8
|
|
7
9
|
import dj_database_url
|
8
10
|
import lamindb_setup as ln_setup
|
9
|
-
from django.core.exceptions import FieldDoesNotExist
|
10
11
|
from django.core.exceptions import ValidationError as DjangoValidationError
|
11
12
|
from django.db import connections, transaction
|
12
|
-
from django.db.models import
|
13
|
+
from django.db.models import (
|
14
|
+
IntegerField,
|
15
|
+
Manager,
|
16
|
+
Q,
|
17
|
+
QuerySet,
|
18
|
+
TextField,
|
19
|
+
Value,
|
20
|
+
)
|
13
21
|
from django.db.models.functions import Cast, Coalesce
|
14
22
|
from django.db.models.lookups import (
|
15
23
|
Contains,
|
@@ -31,28 +39,37 @@ from lamindb_setup._connect_instance import (
|
|
31
39
|
from lamindb_setup.core._docs import doc_args
|
32
40
|
from lamindb_setup.core._hub_core import connect_instance_hub
|
33
41
|
from lamindb_setup.core._settings_store import instance_settings_file
|
34
|
-
from
|
42
|
+
from lamindb_setup.core.upath import extract_suffix_from_path
|
43
|
+
|
44
|
+
from lamindb.base.validation import FieldValidationError
|
45
|
+
from lamindb.models import (
|
35
46
|
Artifact,
|
47
|
+
BasicRecord,
|
48
|
+
CanCurate,
|
36
49
|
Collection,
|
37
50
|
Feature,
|
38
|
-
FeatureSet,
|
39
51
|
IsVersioned,
|
40
52
|
Param,
|
41
53
|
Record,
|
42
54
|
Run,
|
55
|
+
Schema,
|
43
56
|
Transform,
|
44
57
|
ULabel,
|
45
58
|
ValidateFields,
|
46
59
|
)
|
47
|
-
from lnschema_core.validation import FieldValidationError
|
48
60
|
|
49
61
|
from ._utils import attach_func_to_class_method
|
50
62
|
from .core._settings import settings
|
51
|
-
from .core.exceptions import
|
63
|
+
from .core.exceptions import (
|
64
|
+
InvalidArgument,
|
65
|
+
RecordNameChangeIntegrityError,
|
66
|
+
ValidationError,
|
67
|
+
)
|
52
68
|
|
53
69
|
if TYPE_CHECKING:
|
54
70
|
import pandas as pd
|
55
|
-
|
71
|
+
|
72
|
+
from lamindb.base.types import StrField
|
56
73
|
|
57
74
|
|
58
75
|
IPYTHON = getattr(builtins, "__IPYTHON__", False)
|
@@ -75,7 +92,7 @@ def update_attributes(record: Record, attributes: dict[str, str]):
|
|
75
92
|
|
76
93
|
|
77
94
|
def validate_fields(record: Record, kwargs):
|
78
|
-
from
|
95
|
+
from lamindb.base.validation import validate_literal_fields
|
79
96
|
|
80
97
|
# validate required fields
|
81
98
|
# a "required field" is a Django field that has `null=False, default=None`
|
@@ -97,7 +114,7 @@ def validate_fields(record: Record, kwargs):
|
|
97
114
|
Run,
|
98
115
|
ULabel,
|
99
116
|
Feature,
|
100
|
-
|
117
|
+
Schema,
|
101
118
|
Param,
|
102
119
|
}:
|
103
120
|
uid_max_length = record.__class__._meta.get_field(
|
@@ -152,7 +169,13 @@ def __init__(record: Record, *args, **kwargs):
|
|
152
169
|
has_consciously_provided_uid = False
|
153
170
|
if "_has_consciously_provided_uid" in kwargs:
|
154
171
|
has_consciously_provided_uid = kwargs.pop("_has_consciously_provided_uid")
|
155
|
-
if
|
172
|
+
if (
|
173
|
+
isinstance(
|
174
|
+
record, (CanCurate, Collection, Transform)
|
175
|
+
) # Collection is only temporary because it'll get a key field
|
176
|
+
and settings.creation.search_names
|
177
|
+
and not has_consciously_provided_uid
|
178
|
+
):
|
156
179
|
name_field = getattr(record, "_name_field", "name")
|
157
180
|
match = suggest_records_with_similar_names(record, name_field, kwargs)
|
158
181
|
if match:
|
@@ -183,7 +206,7 @@ def __init__(record: Record, *args, **kwargs):
|
|
183
206
|
)
|
184
207
|
init_self_from_db(record, existing_record)
|
185
208
|
return None
|
186
|
-
super(
|
209
|
+
super(BasicRecord, record).__init__(**kwargs)
|
187
210
|
if isinstance(record, ValidateFields):
|
188
211
|
# this will trigger validation against django validators
|
189
212
|
try:
|
@@ -198,8 +221,9 @@ def __init__(record: Record, *args, **kwargs):
|
|
198
221
|
raise ValueError("please provide keyword arguments, not plain arguments")
|
199
222
|
else:
|
200
223
|
# object is loaded from DB (**kwargs could be omitted below, I believe)
|
201
|
-
super(
|
224
|
+
super(BasicRecord, record).__init__(*args, **kwargs)
|
202
225
|
_store_record_old_name(record)
|
226
|
+
_store_record_old_key(record)
|
203
227
|
|
204
228
|
|
205
229
|
def _format_django_validation_error(record: Record, e: DjangoValidationError):
|
@@ -284,6 +308,9 @@ def _search(
|
|
284
308
|
using_key: str | None = None,
|
285
309
|
truncate_string: bool = False,
|
286
310
|
) -> QuerySet:
|
311
|
+
if string is None:
|
312
|
+
raise ValueError("Cannot search for None value! Please pass a valid string.")
|
313
|
+
|
287
314
|
input_queryset = _queryset(cls, using_key=using_key)
|
288
315
|
registry = input_queryset.model
|
289
316
|
name_field = getattr(registry, "_name_field", "name")
|
@@ -316,6 +343,7 @@ def _search(
|
|
316
343
|
string = string[:n_80_pct]
|
317
344
|
|
318
345
|
string = string.strip()
|
346
|
+
string_escape = re.escape(string)
|
319
347
|
|
320
348
|
exact_lookup = Exact if case_sensitive else IExact
|
321
349
|
regex_lookup = Regex if case_sensitive else IRegex
|
@@ -334,28 +362,28 @@ def _search(
|
|
334
362
|
exact_rank = Cast(exact_expr, output_field=IntegerField()) * 200
|
335
363
|
ranks.append(exact_rank)
|
336
364
|
# exact synonym
|
337
|
-
synonym_expr = regex_lookup(field_expr, rf"(?:^|.*\|){
|
365
|
+
synonym_expr = regex_lookup(field_expr, rf"(?:^|.*\|){string_escape}(?:\|.*|$)")
|
338
366
|
synonym_rank = Cast(synonym_expr, output_field=IntegerField()) * 200
|
339
367
|
ranks.append(synonym_rank)
|
340
368
|
# match as sub-phrase
|
341
369
|
sub_expr = regex_lookup(
|
342
|
-
field_expr, rf"(?:^|.*[ \|\.,;:]){
|
370
|
+
field_expr, rf"(?:^|.*[ \|\.,;:]){string_escape}(?:[ \|\.,;:].*|$)"
|
343
371
|
)
|
344
372
|
sub_rank = Cast(sub_expr, output_field=IntegerField()) * 10
|
345
373
|
ranks.append(sub_rank)
|
346
374
|
# startswith and avoid matching string with " " on the right
|
347
375
|
# mostly for truncated
|
348
376
|
startswith_expr = regex_lookup(
|
349
|
-
field_expr, rf"(?:^|.*\|){
|
377
|
+
field_expr, rf"(?:^|.*\|){string_escape}[^ ]*(?:\|.*|$)"
|
350
378
|
)
|
351
379
|
startswith_rank = Cast(startswith_expr, output_field=IntegerField()) * 8
|
352
380
|
ranks.append(startswith_rank)
|
353
381
|
# match as sub-phrase from the left, mostly for truncated
|
354
|
-
right_expr = regex_lookup(field_expr, rf"(?:^|.*[ \|]){
|
382
|
+
right_expr = regex_lookup(field_expr, rf"(?:^|.*[ \|]){string_escape}.*")
|
355
383
|
right_rank = Cast(right_expr, output_field=IntegerField()) * 2
|
356
384
|
ranks.append(right_rank)
|
357
385
|
# match as sub-phrase from the right
|
358
|
-
left_expr = regex_lookup(field_expr, rf".*{
|
386
|
+
left_expr = regex_lookup(field_expr, rf".*{string_escape}(?:$|[ \|\.,;:].*)")
|
359
387
|
left_rank = Cast(left_expr, output_field=IntegerField()) * 2
|
360
388
|
ranks.append(left_rank)
|
361
389
|
# simple contains filter
|
@@ -524,14 +552,14 @@ def using(
|
|
524
552
|
f"Failed to load instance {instance}, please check your permissions!"
|
525
553
|
)
|
526
554
|
iresult, _ = result
|
527
|
-
|
528
|
-
|
555
|
+
source_module = {
|
556
|
+
modules for modules in iresult["schema_str"].split(",") if modules != ""
|
529
557
|
} # type: ignore
|
530
|
-
|
531
|
-
if not
|
532
|
-
missing_members =
|
558
|
+
target_module = ln_setup.settings.instance.modules
|
559
|
+
if not source_module.issubset(target_module):
|
560
|
+
missing_members = source_module - target_module
|
533
561
|
logger.warning(
|
534
|
-
f"source
|
562
|
+
f"source modules has additional modules: {missing_members}\nconsider mounting these registry modules to transfer all metadata"
|
535
563
|
)
|
536
564
|
cache_filepath.write_text(f"{iresult['lnid']}\n{iresult['schema_str']}") # type: ignore
|
537
565
|
settings_file = instance_settings_file(name, owner)
|
@@ -539,7 +567,7 @@ def using(
|
|
539
567
|
else:
|
540
568
|
isettings = load_instance_settings(settings_file)
|
541
569
|
db = isettings.db
|
542
|
-
cache_filepath.write_text(f"{isettings.uid}\n{','.join(isettings.
|
570
|
+
cache_filepath.write_text(f"{isettings.uid}\n{','.join(isettings.modules)}") # type: ignore
|
543
571
|
add_db_connection(db, instance)
|
544
572
|
return QuerySet(model=cls, using=instance)
|
545
573
|
|
@@ -548,6 +576,7 @@ REGISTRY_UNIQUE_FIELD = {
|
|
548
576
|
"storage": "root",
|
549
577
|
"feature": "name",
|
550
578
|
"ulabel": "name",
|
579
|
+
"space": "name", # TODO: this should be updated with the currently used space instead during transfer
|
551
580
|
}
|
552
581
|
|
553
582
|
|
@@ -583,7 +612,6 @@ def update_fk_to_default_db(
|
|
583
612
|
FKBULK = [
|
584
613
|
"organism",
|
585
614
|
"source",
|
586
|
-
"_source_code_artifact", # Transform
|
587
615
|
"report", # Run
|
588
616
|
]
|
589
617
|
|
@@ -596,8 +624,6 @@ def transfer_fk_to_default_db_bulk(
|
|
596
624
|
|
597
625
|
|
598
626
|
def get_transfer_run(record) -> Run:
|
599
|
-
from lamindb_setup import settings as setup_settings
|
600
|
-
|
601
627
|
from lamindb.core._context import context
|
602
628
|
from lamindb.core._data import WARNING_RUN_TRANSFORM
|
603
629
|
|
@@ -617,18 +643,20 @@ def get_transfer_run(record) -> Run:
|
|
617
643
|
uid=uid, name=f"Transfer from `{slug}`", key=key, type="function"
|
618
644
|
).save()
|
619
645
|
settings.creation.search_names = search_names
|
620
|
-
# use the global run context to get the
|
646
|
+
# use the global run context to get the initiated_by_run run id
|
621
647
|
if context.run is not None:
|
622
|
-
|
648
|
+
initiated_by_run = context.run
|
623
649
|
else:
|
624
650
|
if not settings.creation.artifact_silence_missing_run_warning:
|
625
651
|
logger.warning(WARNING_RUN_TRANSFORM)
|
626
|
-
|
652
|
+
initiated_by_run = None
|
627
653
|
# it doesn't seem to make sense to create new runs for every transfer
|
628
|
-
run = Run.filter(
|
654
|
+
run = Run.filter(
|
655
|
+
transform=transform, initiated_by_run=initiated_by_run
|
656
|
+
).one_or_none()
|
629
657
|
if run is None:
|
630
|
-
run = Run(transform=transform,
|
631
|
-
run.
|
658
|
+
run = Run(transform=transform, initiated_by_run=initiated_by_run).save()
|
659
|
+
run.initiated_by_run = initiated_by_run # so that it's available in memory
|
632
660
|
return run
|
633
661
|
|
634
662
|
|
@@ -719,14 +747,19 @@ def save(self, *args, **kwargs) -> Record:
|
|
719
747
|
revises._revises = None # ensure we don't start a recursion
|
720
748
|
revises.save()
|
721
749
|
check_name_change(self)
|
722
|
-
|
750
|
+
check_key_change(self)
|
751
|
+
super(BasicRecord, self).save(*args, **kwargs)
|
723
752
|
_store_record_old_name(self)
|
753
|
+
_store_record_old_key(self)
|
724
754
|
self._revises = None
|
725
755
|
# save unversioned record
|
726
756
|
else:
|
727
757
|
check_name_change(self)
|
728
|
-
|
758
|
+
check_key_change(self)
|
759
|
+
super(BasicRecord, self).save(*args, **kwargs)
|
760
|
+
# update _old_name and _old_key after saving
|
729
761
|
_store_record_old_name(self)
|
762
|
+
_store_record_old_key(self)
|
730
763
|
# perform transfer of many-to-many fields
|
731
764
|
# only supported for Artifact and Collection records
|
732
765
|
if db is not None and db != "default" and using_key is None:
|
@@ -739,7 +772,7 @@ def save(self, *args, **kwargs) -> Record:
|
|
739
772
|
if hasattr(self, "labels"):
|
740
773
|
from copy import copy
|
741
774
|
|
742
|
-
from
|
775
|
+
from lamindb.models import FeatureManager
|
743
776
|
|
744
777
|
# here we go back to original record on the source database
|
745
778
|
self_on_db = copy(self)
|
@@ -757,19 +790,33 @@ def save(self, *args, **kwargs) -> Record:
|
|
757
790
|
def _store_record_old_name(record: Record):
|
758
791
|
# writes the name to the _name attribute, so we can detect renaming upon save
|
759
792
|
if hasattr(record, "_name_field"):
|
760
|
-
record.
|
793
|
+
record._old_name = getattr(record, record._name_field)
|
794
|
+
|
795
|
+
|
796
|
+
def _store_record_old_key(record: Record):
|
797
|
+
# writes the key to the _old_key attribute, so we can detect key changes upon save
|
798
|
+
if isinstance(record, (Artifact, Transform)):
|
799
|
+
record._old_key = record.key
|
761
800
|
|
762
801
|
|
763
802
|
def check_name_change(record: Record):
|
764
803
|
"""Warns if a record's name has changed."""
|
765
804
|
if (
|
766
805
|
not record.pk
|
767
|
-
or not hasattr(record, "
|
806
|
+
or not hasattr(record, "_old_name")
|
768
807
|
or not hasattr(record, "_name_field")
|
769
808
|
):
|
770
809
|
return
|
771
810
|
|
772
|
-
|
811
|
+
# checked in check_key_change or not checked at all
|
812
|
+
if isinstance(record, (Artifact, Collection, Transform)):
|
813
|
+
return
|
814
|
+
|
815
|
+
# renaming feature sets is not checked
|
816
|
+
if isinstance(record, Schema):
|
817
|
+
return
|
818
|
+
|
819
|
+
old_name = record._old_name
|
773
820
|
new_name = getattr(record, record._name_field)
|
774
821
|
registry = record.__class__.__name__
|
775
822
|
|
@@ -783,13 +830,13 @@ def check_name_change(record: Record):
|
|
783
830
|
.exclude(feature_id=None) # must have a feature
|
784
831
|
.exclude(
|
785
832
|
feature_ref_is_name=None
|
786
|
-
) # must be linked via Curator and therefore part of a
|
833
|
+
) # must be linked via Curator and therefore part of a schema
|
787
834
|
.distinct()
|
788
835
|
)
|
789
836
|
artifact_ids = linked_records.list("artifact__uid")
|
790
837
|
n = len(artifact_ids)
|
791
|
-
s = "s" if n > 1 else ""
|
792
838
|
if n > 0:
|
839
|
+
s = "s" if n > 1 else ""
|
793
840
|
logger.error(
|
794
841
|
f"You are trying to {colors.red('rename label')} from '{old_name}' to '{new_name}'!\n"
|
795
842
|
f" → The following {n} artifact{s} {colors.red('will no longer be validated')}: {artifact_ids}\n\n"
|
@@ -803,13 +850,13 @@ def check_name_change(record: Record):
|
|
803
850
|
|
804
851
|
# when a feature is renamed
|
805
852
|
elif isinstance(record, Feature):
|
806
|
-
# only internal features are associated with
|
807
|
-
linked_artifacts = Artifact.filter(
|
853
|
+
# only internal features are associated with schemas
|
854
|
+
linked_artifacts = Artifact.filter(_schemas_m2m__features=record).list(
|
808
855
|
"uid"
|
809
856
|
)
|
810
857
|
n = len(linked_artifacts)
|
811
|
-
s = "s" if n > 1 else ""
|
812
858
|
if n > 0:
|
859
|
+
s = "s" if n > 1 else ""
|
813
860
|
logger.error(
|
814
861
|
f"You are trying to {colors.red('rename feature')} from '{old_name}' to '{new_name}'!\n"
|
815
862
|
f" → The following {n} artifact{s} {colors.red('will no longer be validated')}: {linked_artifacts}\n\n"
|
@@ -822,6 +869,33 @@ def check_name_change(record: Record):
|
|
822
869
|
raise RecordNameChangeIntegrityError
|
823
870
|
|
824
871
|
|
872
|
+
def check_key_change(record: Artifact | Transform):
|
873
|
+
"""Errors if a record's key has falsely changed."""
|
874
|
+
if not isinstance(record, Artifact) or not hasattr(record, "_old_key"):
|
875
|
+
return
|
876
|
+
|
877
|
+
old_key = record._old_key or ""
|
878
|
+
new_key = record.key or ""
|
879
|
+
|
880
|
+
if old_key != new_key:
|
881
|
+
if not record._key_is_virtual:
|
882
|
+
raise InvalidArgument(
|
883
|
+
f"Changing a non-virtual key of an artifact is not allowed! Tried to change key from '{old_key}' to '{new_key}'."
|
884
|
+
)
|
885
|
+
old_key_suffix = (
|
886
|
+
record.suffix
|
887
|
+
if record.suffix
|
888
|
+
else extract_suffix_from_path(PurePosixPath(old_key), arg_name="key")
|
889
|
+
)
|
890
|
+
new_key_suffix = extract_suffix_from_path(
|
891
|
+
PurePosixPath(new_key), arg_name="key"
|
892
|
+
)
|
893
|
+
if old_key_suffix != new_key_suffix:
|
894
|
+
raise InvalidArgument(
|
895
|
+
f"The suffix '{new_key_suffix}' of the provided key is incorrect, it should be '{old_key_suffix}'."
|
896
|
+
)
|
897
|
+
|
898
|
+
|
825
899
|
def delete(self) -> None:
|
826
900
|
"""Delete the record."""
|
827
901
|
# note that the logic below does not fire if a record is moved to the trash
|
@@ -841,10 +915,10 @@ def delete(self) -> None:
|
|
841
915
|
new_latest.is_latest = True
|
842
916
|
with transaction.atomic():
|
843
917
|
new_latest.save()
|
844
|
-
super(
|
918
|
+
super(BasicRecord, self).delete()
|
845
919
|
logger.warning(f"new latest version is {new_latest}")
|
846
920
|
return None
|
847
|
-
super(
|
921
|
+
super(BasicRecord, self).delete()
|
848
922
|
|
849
923
|
|
850
924
|
METHOD_NAMES = [
|
@@ -869,4 +943,5 @@ if ln_setup._TESTING: # type: ignore
|
|
869
943
|
}
|
870
944
|
|
871
945
|
for name in METHOD_NAMES:
|
946
|
+
attach_func_to_class_method(name, BasicRecord, globals())
|
872
947
|
attach_func_to_class_method(name, Record, globals())
|
lamindb/_run.py
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
|
3
|
-
from
|
3
|
+
from lamindb.models import ParamManager, Run, Transform
|
4
4
|
|
5
5
|
|
6
6
|
def __init__(run: Run, *args, **kwargs):
|
@@ -18,7 +18,7 @@ def __init__(run: Run, *args, **kwargs):
|
|
18
18
|
reference_type: str | None = (
|
19
19
|
kwargs.pop("reference_type") if "reference_type" in kwargs else None
|
20
20
|
)
|
21
|
-
|
21
|
+
initiated_by_run: Run | None = kwargs.pop("initiated_by_run", None)
|
22
22
|
if transform is None:
|
23
23
|
raise TypeError("Pass transform parameter")
|
24
24
|
if transform._state.adding:
|
@@ -27,7 +27,7 @@ def __init__(run: Run, *args, **kwargs):
|
|
27
27
|
super(Run, run).__init__(
|
28
28
|
transform=transform,
|
29
29
|
reference=reference,
|
30
|
-
|
30
|
+
initiated_by_run=initiated_by_run,
|
31
31
|
reference_type=reference_type,
|
32
32
|
)
|
33
33
|
|
lamindb/_save.py
CHANGED
@@ -5,15 +5,14 @@ import shutil
|
|
5
5
|
import traceback
|
6
6
|
from collections import defaultdict
|
7
7
|
from datetime import datetime
|
8
|
-
from
|
9
|
-
from typing import TYPE_CHECKING, overload
|
8
|
+
from typing import TYPE_CHECKING
|
10
9
|
|
11
|
-
import
|
12
|
-
from django.db import IntegrityError, transaction
|
10
|
+
from django.db import transaction
|
13
11
|
from django.utils.functional import partition
|
14
12
|
from lamin_utils import logger
|
15
13
|
from lamindb_setup.core.upath import LocalPathClasses
|
16
|
-
|
14
|
+
|
15
|
+
from lamindb.models import Artifact, Record
|
17
16
|
|
18
17
|
from .core._settings import settings
|
19
18
|
from .core.storage.paths import (
|
@@ -64,7 +63,7 @@ def save(records: Iterable[Record], ignore_conflicts: bool | None = False) -> No
|
|
64
63
|
Update a single existing record:
|
65
64
|
|
66
65
|
>>> transform = ln.Transform.get("0Cb86EZj")
|
67
|
-
>>> transform.
|
66
|
+
>>> transform.description = "New description"
|
68
67
|
>>> transform.save()
|
69
68
|
|
70
69
|
"""
|
@@ -184,10 +183,17 @@ def copy_or_move_to_cache(
|
|
184
183
|
return None
|
185
184
|
# non-local storage_path further
|
186
185
|
if local_path != cache_path:
|
187
|
-
cache_path.
|
188
|
-
|
186
|
+
if cache_path.exists():
|
187
|
+
logger.warning(
|
188
|
+
f"The cache path {cache_path.as_posix()} already exists, replacing it."
|
189
|
+
)
|
189
190
|
if cache_path.is_dir():
|
190
191
|
shutil.rmtree(cache_path)
|
192
|
+
else:
|
193
|
+
cache_path.unlink()
|
194
|
+
else:
|
195
|
+
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
196
|
+
if cache_dir in local_path.parents:
|
191
197
|
local_path.replace(cache_path)
|
192
198
|
else:
|
193
199
|
if is_dir:
|
@@ -7,14 +7,16 @@ import numpy as np
|
|
7
7
|
from lamin_utils import logger
|
8
8
|
from lamindb_setup.core._docs import doc_args
|
9
9
|
from lamindb_setup.core.hashing import hash_set
|
10
|
-
|
11
|
-
from
|
10
|
+
|
11
|
+
from lamindb.base import ids
|
12
|
+
from lamindb.base.types import FieldAttr, ListLike
|
13
|
+
from lamindb.models import Feature, Record, Schema
|
12
14
|
|
13
15
|
from ._feature import convert_pandas_dtype_to_lamin_dtype
|
14
16
|
from ._record import init_self_from_db
|
15
17
|
from ._utils import attach_func_to_class_method
|
16
18
|
from .core.exceptions import ValidationError
|
17
|
-
from .core.
|
19
|
+
from .core.relations import (
|
18
20
|
dict_related_model_to_related_name,
|
19
21
|
get_related_name,
|
20
22
|
)
|
@@ -47,7 +49,7 @@ def validate_features(features: list[Record]) -> Record:
|
|
47
49
|
)
|
48
50
|
feature_types = {feature.__class__ for feature in features}
|
49
51
|
if len(feature_types) > 1:
|
50
|
-
raise TypeError("
|
52
|
+
raise TypeError("schema can only contain a single type")
|
51
53
|
for feature in features:
|
52
54
|
if feature._state.adding:
|
53
55
|
raise ValueError("Can only construct feature sets from validated features")
|
@@ -56,7 +58,7 @@ def validate_features(features: list[Record]) -> Record:
|
|
56
58
|
|
57
59
|
def __init__(self, *args, **kwargs):
|
58
60
|
if len(args) == len(self._meta.concrete_fields):
|
59
|
-
super(
|
61
|
+
super(Schema, self).__init__(*args, **kwargs)
|
60
62
|
return None
|
61
63
|
# now we proceed with the user-facing constructor
|
62
64
|
if len(args) > 1:
|
@@ -72,29 +74,29 @@ def __init__(self, *args, **kwargs):
|
|
72
74
|
dtype = None if features_registry == Feature else NUMBER_TYPE
|
73
75
|
n_features = len(features)
|
74
76
|
features_hash = hash_set({feature.uid for feature in features})
|
75
|
-
|
76
|
-
if
|
77
|
-
logger.debug(f"loaded: {
|
78
|
-
init_self_from_db(self,
|
77
|
+
schema = Schema.filter(hash=features_hash).one_or_none()
|
78
|
+
if schema is not None:
|
79
|
+
logger.debug(f"loaded: {schema}")
|
80
|
+
init_self_from_db(self, schema)
|
79
81
|
return None
|
80
82
|
else:
|
81
83
|
hash = features_hash
|
82
84
|
self._features = (get_related_name(features_registry), features)
|
83
85
|
|
84
|
-
super(
|
86
|
+
super(Schema, self).__init__(
|
85
87
|
uid=ids.base62_20(),
|
86
88
|
name=name,
|
87
89
|
dtype=get_type_str(dtype),
|
88
90
|
n=n_features,
|
89
|
-
registry=features_registry.
|
91
|
+
registry=features_registry.__get_name_with_module__(),
|
90
92
|
hash=hash,
|
91
93
|
)
|
92
94
|
|
93
95
|
|
94
|
-
@doc_args(
|
95
|
-
def save(self, *args, **kwargs) ->
|
96
|
+
@doc_args(Schema.save.__doc__)
|
97
|
+
def save(self, *args, **kwargs) -> Schema:
|
96
98
|
"""{}""" # noqa: D415
|
97
|
-
super(
|
99
|
+
super(Schema, self).save(*args, **kwargs)
|
98
100
|
if hasattr(self, "_features"):
|
99
101
|
related_name, records = self._features
|
100
102
|
getattr(self, related_name).set(records)
|
@@ -110,7 +112,7 @@ def get_type_str(dtype: str | None) -> str | None:
|
|
110
112
|
|
111
113
|
|
112
114
|
@classmethod # type:ignore
|
113
|
-
@doc_args(
|
115
|
+
@doc_args(Schema.from_values.__doc__)
|
114
116
|
def from_values(
|
115
117
|
cls,
|
116
118
|
values: ListLike,
|
@@ -121,7 +123,7 @@ def from_values(
|
|
121
123
|
organism: Record | str | None = None,
|
122
124
|
source: Record | None = None,
|
123
125
|
raise_validation_error: bool = True,
|
124
|
-
) ->
|
126
|
+
) -> Schema:
|
125
127
|
"""{}""" # noqa: D415
|
126
128
|
if not isinstance(field, FieldAttr):
|
127
129
|
raise TypeError("Argument `field` must be a Record field, e.g., `Feature.name`")
|
@@ -152,16 +154,16 @@ def from_values(
|
|
152
154
|
organism=organism,
|
153
155
|
source=source,
|
154
156
|
)
|
155
|
-
|
157
|
+
schema = Schema(
|
156
158
|
features=validated_features,
|
157
159
|
name=name,
|
158
160
|
dtype=get_type_str(type),
|
159
161
|
)
|
160
|
-
return
|
162
|
+
return schema
|
161
163
|
|
162
164
|
|
163
165
|
@classmethod # type:ignore
|
164
|
-
@doc_args(
|
166
|
+
@doc_args(Schema.from_df.__doc__)
|
165
167
|
def from_df(
|
166
168
|
cls,
|
167
169
|
df: pd.DataFrame,
|
@@ -170,7 +172,7 @@ def from_df(
|
|
170
172
|
mute: bool = False,
|
171
173
|
organism: Record | str | None = None,
|
172
174
|
source: Record | None = None,
|
173
|
-
) ->
|
175
|
+
) -> Schema | None:
|
174
176
|
"""{}""" # noqa: D415
|
175
177
|
registry = field.field.model
|
176
178
|
validated = registry.validate(df.columns, field=field, mute=mute, organism=organism)
|
@@ -182,7 +184,7 @@ def from_df(
|
|
182
184
|
validated_features = Feature.from_values(
|
183
185
|
df.columns, field=field, organism=organism
|
184
186
|
)
|
185
|
-
|
187
|
+
schema = Schema(validated_features, name=name, dtype=None)
|
186
188
|
else:
|
187
189
|
dtypes = [col.dtype for (_, col) in df.loc[:, validated].items()]
|
188
190
|
if len(set(dtypes)) != 1:
|
@@ -194,16 +196,16 @@ def from_df(
|
|
194
196
|
organism=organism,
|
195
197
|
source=source,
|
196
198
|
)
|
197
|
-
|
199
|
+
schema = Schema(
|
198
200
|
features=validated_features,
|
199
201
|
name=name,
|
200
202
|
dtype=get_type_str(dtype),
|
201
203
|
)
|
202
|
-
return
|
204
|
+
return schema
|
203
205
|
|
204
206
|
|
205
207
|
@property # type: ignore
|
206
|
-
@doc_args(
|
208
|
+
@doc_args(Schema.members.__doc__)
|
207
209
|
def members(self) -> QuerySet:
|
208
210
|
"""{}""" # noqa: D415
|
209
211
|
if self._state.adding:
|
@@ -216,11 +218,11 @@ def members(self) -> QuerySet:
|
|
216
218
|
return self.__getattribute__(related_name).all()
|
217
219
|
|
218
220
|
|
219
|
-
def _get_related_name(self:
|
220
|
-
|
221
|
+
def _get_related_name(self: Schema) -> str:
|
222
|
+
_schemas_m2m_related_models = dict_related_model_to_related_name(
|
221
223
|
self, instance=self._state.db
|
222
224
|
)
|
223
|
-
related_name =
|
225
|
+
related_name = _schemas_m2m_related_models.get(self.itype)
|
224
226
|
return related_name
|
225
227
|
|
226
228
|
|
@@ -235,13 +237,14 @@ if ln_setup._TESTING:
|
|
235
237
|
from inspect import signature
|
236
238
|
|
237
239
|
SIGS = {
|
238
|
-
name: signature(getattr(
|
240
|
+
name: signature(getattr(Schema, name))
|
239
241
|
for name in METHOD_NAMES
|
240
242
|
if name != "__init__"
|
241
243
|
}
|
242
244
|
|
243
245
|
for name in METHOD_NAMES:
|
244
|
-
attach_func_to_class_method(name,
|
246
|
+
attach_func_to_class_method(name, Schema, globals())
|
245
247
|
|
246
|
-
|
247
|
-
|
248
|
+
Schema.members = members
|
249
|
+
Schema._get_related_name = _get_related_name
|
250
|
+
Schema.feature_sets = Schema._artifacts_m2m # backward compat
|