lamindb 0.77.2__py3-none-any.whl → 1.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. lamindb/__init__.py +39 -32
  2. lamindb/_artifact.py +95 -64
  3. lamindb/_can_curate.py +19 -10
  4. lamindb/_collection.py +51 -49
  5. lamindb/_feature.py +9 -9
  6. lamindb/_finish.py +99 -86
  7. lamindb/_from_values.py +20 -17
  8. lamindb/_is_versioned.py +2 -1
  9. lamindb/_parents.py +23 -16
  10. lamindb/_query_manager.py +3 -3
  11. lamindb/_query_set.py +85 -18
  12. lamindb/_record.py +121 -46
  13. lamindb/_run.py +3 -3
  14. lamindb/_save.py +14 -8
  15. lamindb/{_feature_set.py → _schema.py} +34 -31
  16. lamindb/_storage.py +2 -1
  17. lamindb/_transform.py +51 -23
  18. lamindb/_ulabel.py +17 -8
  19. lamindb/_view.py +15 -14
  20. lamindb/base/__init__.py +24 -0
  21. lamindb/base/fields.py +281 -0
  22. lamindb/base/ids.py +103 -0
  23. lamindb/base/types.py +51 -0
  24. lamindb/base/users.py +30 -0
  25. lamindb/base/validation.py +67 -0
  26. lamindb/core/__init__.py +19 -14
  27. lamindb/core/_context.py +297 -228
  28. lamindb/core/_data.py +44 -49
  29. lamindb/core/_describe.py +41 -31
  30. lamindb/core/_django.py +59 -44
  31. lamindb/core/_feature_manager.py +192 -168
  32. lamindb/core/_label_manager.py +22 -22
  33. lamindb/core/_mapped_collection.py +17 -14
  34. lamindb/core/_settings.py +1 -12
  35. lamindb/core/_sync_git.py +56 -9
  36. lamindb/core/_track_environment.py +1 -1
  37. lamindb/core/datasets/_core.py +5 -6
  38. lamindb/core/exceptions.py +0 -7
  39. lamindb/core/fields.py +1 -1
  40. lamindb/core/loaders.py +18 -2
  41. lamindb/core/{schema.py → relations.py} +22 -19
  42. lamindb/core/storage/_anndata_accessor.py +1 -2
  43. lamindb/core/storage/_backed_access.py +2 -1
  44. lamindb/core/storage/_tiledbsoma.py +40 -13
  45. lamindb/core/storage/objects.py +1 -1
  46. lamindb/core/storage/paths.py +13 -8
  47. lamindb/core/subsettings/__init__.py +0 -2
  48. lamindb/core/types.py +2 -23
  49. lamindb/core/versioning.py +11 -7
  50. lamindb/{_curate.py → curators/__init__.py} +700 -57
  51. lamindb/curators/_spatial.py +528 -0
  52. lamindb/integrations/_vitessce.py +1 -3
  53. lamindb/migrations/0052_squashed.py +1261 -0
  54. lamindb/migrations/0053_alter_featureset_hash_alter_paramvalue_created_by_and_more.py +57 -0
  55. lamindb/migrations/0054_alter_feature_previous_runs_and_more.py +35 -0
  56. lamindb/migrations/0055_artifact_type_artifactparamvalue_and_more.py +61 -0
  57. lamindb/migrations/0056_rename_ulabel_ref_is_name_artifactulabel_label_ref_is_name_and_more.py +22 -0
  58. lamindb/migrations/0057_link_models_latest_report_and_others.py +356 -0
  59. lamindb/migrations/0058_artifact__actions_collection__actions.py +22 -0
  60. lamindb/migrations/0059_alter_artifact__accessor_alter_artifact__hash_type_and_more.py +31 -0
  61. lamindb/migrations/0060_alter_artifact__actions.py +22 -0
  62. lamindb/migrations/0061_alter_collection_meta_artifact_alter_run_environment_and_more.py +45 -0
  63. lamindb/migrations/0062_add_is_latest_field.py +32 -0
  64. lamindb/migrations/0063_populate_latest_field.py +45 -0
  65. lamindb/migrations/0064_alter_artifact_version_alter_collection_version_and_more.py +33 -0
  66. lamindb/migrations/0065_remove_collection_feature_sets_and_more.py +22 -0
  67. lamindb/migrations/0066_alter_artifact__feature_values_and_more.py +352 -0
  68. lamindb/migrations/0067_alter_featurevalue_unique_together_and_more.py +20 -0
  69. lamindb/migrations/0068_alter_artifactulabel_unique_together_and_more.py +20 -0
  70. lamindb/migrations/0069_alter_artifact__accessor_alter_artifact__hash_type_and_more.py +1294 -0
  71. lamindb/migrations/0069_squashed.py +1770 -0
  72. lamindb/migrations/0070_lamindbv1_migrate_data.py +78 -0
  73. lamindb/migrations/0071_lamindbv1_migrate_schema.py +741 -0
  74. lamindb/migrations/0072_remove_user__branch_code_remove_user_aux_and_more.py +148 -0
  75. lamindb/migrations/0073_merge_ourprojects.py +945 -0
  76. lamindb/migrations/0074_lamindbv1_part4.py +374 -0
  77. lamindb/migrations/0075_lamindbv1_part5.py +276 -0
  78. lamindb/migrations/0076_lamindbv1_part6.py +621 -0
  79. lamindb/migrations/0077_lamindbv1_part6b.py +228 -0
  80. lamindb/migrations/0078_lamindbv1_part6c.py +468 -0
  81. lamindb/migrations/0079_alter_rundata_value_json_and_more.py +36 -0
  82. lamindb/migrations/__init__.py +0 -0
  83. lamindb/models.py +4064 -0
  84. {lamindb-0.77.2.dist-info → lamindb-1.0rc1.dist-info}/METADATA +15 -20
  85. lamindb-1.0rc1.dist-info/RECORD +100 -0
  86. {lamindb-0.77.2.dist-info → lamindb-1.0rc1.dist-info}/WHEEL +1 -1
  87. lamindb/core/subsettings/_transform_settings.py +0 -21
  88. lamindb-0.77.2.dist-info/RECORD +0 -63
  89. {lamindb-0.77.2.dist-info → lamindb-1.0rc1.dist-info}/LICENSE +0 -0
lamindb/_record.py CHANGED
@@ -1,15 +1,23 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import builtins
4
+ import re
4
5
  from functools import reduce
6
+ from pathlib import PurePosixPath
5
7
  from typing import TYPE_CHECKING, NamedTuple
6
8
 
7
9
  import dj_database_url
8
10
  import lamindb_setup as ln_setup
9
- from django.core.exceptions import FieldDoesNotExist
10
11
  from django.core.exceptions import ValidationError as DjangoValidationError
11
12
  from django.db import connections, transaction
12
- from django.db.models import F, IntegerField, Manager, Q, QuerySet, TextField, Value
13
+ from django.db.models import (
14
+ IntegerField,
15
+ Manager,
16
+ Q,
17
+ QuerySet,
18
+ TextField,
19
+ Value,
20
+ )
13
21
  from django.db.models.functions import Cast, Coalesce
14
22
  from django.db.models.lookups import (
15
23
  Contains,
@@ -31,28 +39,37 @@ from lamindb_setup._connect_instance import (
31
39
  from lamindb_setup.core._docs import doc_args
32
40
  from lamindb_setup.core._hub_core import connect_instance_hub
33
41
  from lamindb_setup.core._settings_store import instance_settings_file
34
- from lnschema_core.models import (
42
+ from lamindb_setup.core.upath import extract_suffix_from_path
43
+
44
+ from lamindb.base.validation import FieldValidationError
45
+ from lamindb.models import (
35
46
  Artifact,
47
+ BasicRecord,
48
+ CanCurate,
36
49
  Collection,
37
50
  Feature,
38
- FeatureSet,
39
51
  IsVersioned,
40
52
  Param,
41
53
  Record,
42
54
  Run,
55
+ Schema,
43
56
  Transform,
44
57
  ULabel,
45
58
  ValidateFields,
46
59
  )
47
- from lnschema_core.validation import FieldValidationError
48
60
 
49
61
  from ._utils import attach_func_to_class_method
50
62
  from .core._settings import settings
51
- from .core.exceptions import RecordNameChangeIntegrityError, ValidationError
63
+ from .core.exceptions import (
64
+ InvalidArgument,
65
+ RecordNameChangeIntegrityError,
66
+ ValidationError,
67
+ )
52
68
 
53
69
  if TYPE_CHECKING:
54
70
  import pandas as pd
55
- from lnschema_core.types import ListLike, StrField
71
+
72
+ from lamindb.base.types import StrField
56
73
 
57
74
 
58
75
  IPYTHON = getattr(builtins, "__IPYTHON__", False)
@@ -75,7 +92,7 @@ def update_attributes(record: Record, attributes: dict[str, str]):
75
92
 
76
93
 
77
94
  def validate_fields(record: Record, kwargs):
78
- from lnschema_core.validation import validate_literal_fields
95
+ from lamindb.base.validation import validate_literal_fields
79
96
 
80
97
  # validate required fields
81
98
  # a "required field" is a Django field that has `null=False, default=None`
@@ -97,7 +114,7 @@ def validate_fields(record: Record, kwargs):
97
114
  Run,
98
115
  ULabel,
99
116
  Feature,
100
- FeatureSet,
117
+ Schema,
101
118
  Param,
102
119
  }:
103
120
  uid_max_length = record.__class__._meta.get_field(
@@ -152,7 +169,13 @@ def __init__(record: Record, *args, **kwargs):
152
169
  has_consciously_provided_uid = False
153
170
  if "_has_consciously_provided_uid" in kwargs:
154
171
  has_consciously_provided_uid = kwargs.pop("_has_consciously_provided_uid")
155
- if settings.creation.search_names and not has_consciously_provided_uid:
172
+ if (
173
+ isinstance(
174
+ record, (CanCurate, Collection, Transform)
175
+ ) # Collection is only temporary because it'll get a key field
176
+ and settings.creation.search_names
177
+ and not has_consciously_provided_uid
178
+ ):
156
179
  name_field = getattr(record, "_name_field", "name")
157
180
  match = suggest_records_with_similar_names(record, name_field, kwargs)
158
181
  if match:
@@ -183,7 +206,7 @@ def __init__(record: Record, *args, **kwargs):
183
206
  )
184
207
  init_self_from_db(record, existing_record)
185
208
  return None
186
- super(Record, record).__init__(**kwargs)
209
+ super(BasicRecord, record).__init__(**kwargs)
187
210
  if isinstance(record, ValidateFields):
188
211
  # this will trigger validation against django validators
189
212
  try:
@@ -198,8 +221,9 @@ def __init__(record: Record, *args, **kwargs):
198
221
  raise ValueError("please provide keyword arguments, not plain arguments")
199
222
  else:
200
223
  # object is loaded from DB (**kwargs could be omitted below, I believe)
201
- super(Record, record).__init__(*args, **kwargs)
224
+ super(BasicRecord, record).__init__(*args, **kwargs)
202
225
  _store_record_old_name(record)
226
+ _store_record_old_key(record)
203
227
 
204
228
 
205
229
  def _format_django_validation_error(record: Record, e: DjangoValidationError):
@@ -284,6 +308,9 @@ def _search(
284
308
  using_key: str | None = None,
285
309
  truncate_string: bool = False,
286
310
  ) -> QuerySet:
311
+ if string is None:
312
+ raise ValueError("Cannot search for None value! Please pass a valid string.")
313
+
287
314
  input_queryset = _queryset(cls, using_key=using_key)
288
315
  registry = input_queryset.model
289
316
  name_field = getattr(registry, "_name_field", "name")
@@ -316,6 +343,7 @@ def _search(
316
343
  string = string[:n_80_pct]
317
344
 
318
345
  string = string.strip()
346
+ string_escape = re.escape(string)
319
347
 
320
348
  exact_lookup = Exact if case_sensitive else IExact
321
349
  regex_lookup = Regex if case_sensitive else IRegex
@@ -334,28 +362,28 @@ def _search(
334
362
  exact_rank = Cast(exact_expr, output_field=IntegerField()) * 200
335
363
  ranks.append(exact_rank)
336
364
  # exact synonym
337
- synonym_expr = regex_lookup(field_expr, rf"(?:^|.*\|){string}(?:\|.*|$)")
365
+ synonym_expr = regex_lookup(field_expr, rf"(?:^|.*\|){string_escape}(?:\|.*|$)")
338
366
  synonym_rank = Cast(synonym_expr, output_field=IntegerField()) * 200
339
367
  ranks.append(synonym_rank)
340
368
  # match as sub-phrase
341
369
  sub_expr = regex_lookup(
342
- field_expr, rf"(?:^|.*[ \|\.,;:]){string}(?:[ \|\.,;:].*|$)"
370
+ field_expr, rf"(?:^|.*[ \|\.,;:]){string_escape}(?:[ \|\.,;:].*|$)"
343
371
  )
344
372
  sub_rank = Cast(sub_expr, output_field=IntegerField()) * 10
345
373
  ranks.append(sub_rank)
346
374
  # startswith and avoid matching string with " " on the right
347
375
  # mostly for truncated
348
376
  startswith_expr = regex_lookup(
349
- field_expr, rf"(?:^|.*\|){string}[^ ]*(?:\|.*|$)"
377
+ field_expr, rf"(?:^|.*\|){string_escape}[^ ]*(?:\|.*|$)"
350
378
  )
351
379
  startswith_rank = Cast(startswith_expr, output_field=IntegerField()) * 8
352
380
  ranks.append(startswith_rank)
353
381
  # match as sub-phrase from the left, mostly for truncated
354
- right_expr = regex_lookup(field_expr, rf"(?:^|.*[ \|]){string}.*")
382
+ right_expr = regex_lookup(field_expr, rf"(?:^|.*[ \|]){string_escape}.*")
355
383
  right_rank = Cast(right_expr, output_field=IntegerField()) * 2
356
384
  ranks.append(right_rank)
357
385
  # match as sub-phrase from the right
358
- left_expr = regex_lookup(field_expr, rf".*{string}(?:$|[ \|\.,;:].*)")
386
+ left_expr = regex_lookup(field_expr, rf".*{string_escape}(?:$|[ \|\.,;:].*)")
359
387
  left_rank = Cast(left_expr, output_field=IntegerField()) * 2
360
388
  ranks.append(left_rank)
361
389
  # simple contains filter
@@ -524,14 +552,14 @@ def using(
524
552
  f"Failed to load instance {instance}, please check your permissions!"
525
553
  )
526
554
  iresult, _ = result
527
- source_schema = {
528
- schema for schema in iresult["schema_str"].split(",") if schema != ""
555
+ source_module = {
556
+ modules for modules in iresult["schema_str"].split(",") if modules != ""
529
557
  } # type: ignore
530
- target_schema = ln_setup.settings.instance.schema
531
- if not source_schema.issubset(target_schema):
532
- missing_members = source_schema - target_schema
558
+ target_module = ln_setup.settings.instance.modules
559
+ if not source_module.issubset(target_module):
560
+ missing_members = source_module - target_module
533
561
  logger.warning(
534
- f"source schema has additional modules: {missing_members}\nconsider mounting these schema modules to transfer all metadata"
562
+ f"source modules has additional modules: {missing_members}\nconsider mounting these registry modules to transfer all metadata"
535
563
  )
536
564
  cache_filepath.write_text(f"{iresult['lnid']}\n{iresult['schema_str']}") # type: ignore
537
565
  settings_file = instance_settings_file(name, owner)
@@ -539,7 +567,7 @@ def using(
539
567
  else:
540
568
  isettings = load_instance_settings(settings_file)
541
569
  db = isettings.db
542
- cache_filepath.write_text(f"{isettings.uid}\n{','.join(isettings.schema)}") # type: ignore
570
+ cache_filepath.write_text(f"{isettings.uid}\n{','.join(isettings.modules)}") # type: ignore
543
571
  add_db_connection(db, instance)
544
572
  return QuerySet(model=cls, using=instance)
545
573
 
@@ -548,6 +576,7 @@ REGISTRY_UNIQUE_FIELD = {
548
576
  "storage": "root",
549
577
  "feature": "name",
550
578
  "ulabel": "name",
579
+ "space": "name", # TODO: this should be updated with the currently used space instead during transfer
551
580
  }
552
581
 
553
582
 
@@ -583,7 +612,6 @@ def update_fk_to_default_db(
583
612
  FKBULK = [
584
613
  "organism",
585
614
  "source",
586
- "_source_code_artifact", # Transform
587
615
  "report", # Run
588
616
  ]
589
617
 
@@ -596,8 +624,6 @@ def transfer_fk_to_default_db_bulk(
596
624
 
597
625
 
598
626
  def get_transfer_run(record) -> Run:
599
- from lamindb_setup import settings as setup_settings
600
-
601
627
  from lamindb.core._context import context
602
628
  from lamindb.core._data import WARNING_RUN_TRANSFORM
603
629
 
@@ -617,18 +643,20 @@ def get_transfer_run(record) -> Run:
617
643
  uid=uid, name=f"Transfer from `{slug}`", key=key, type="function"
618
644
  ).save()
619
645
  settings.creation.search_names = search_names
620
- # use the global run context to get the parent run id
646
+ # use the global run context to get the initiated_by_run run id
621
647
  if context.run is not None:
622
- parent = context.run
648
+ initiated_by_run = context.run
623
649
  else:
624
650
  if not settings.creation.artifact_silence_missing_run_warning:
625
651
  logger.warning(WARNING_RUN_TRANSFORM)
626
- parent = None
652
+ initiated_by_run = None
627
653
  # it doesn't seem to make sense to create new runs for every transfer
628
- run = Run.filter(transform=transform, parent=parent).one_or_none()
654
+ run = Run.filter(
655
+ transform=transform, initiated_by_run=initiated_by_run
656
+ ).one_or_none()
629
657
  if run is None:
630
- run = Run(transform=transform, parent=parent).save()
631
- run.parent = parent # so that it's available in memory
658
+ run = Run(transform=transform, initiated_by_run=initiated_by_run).save()
659
+ run.initiated_by_run = initiated_by_run # so that it's available in memory
632
660
  return run
633
661
 
634
662
 
@@ -719,14 +747,19 @@ def save(self, *args, **kwargs) -> Record:
719
747
  revises._revises = None # ensure we don't start a recursion
720
748
  revises.save()
721
749
  check_name_change(self)
722
- super(Record, self).save(*args, **kwargs)
750
+ check_key_change(self)
751
+ super(BasicRecord, self).save(*args, **kwargs)
723
752
  _store_record_old_name(self)
753
+ _store_record_old_key(self)
724
754
  self._revises = None
725
755
  # save unversioned record
726
756
  else:
727
757
  check_name_change(self)
728
- super(Record, self).save(*args, **kwargs)
758
+ check_key_change(self)
759
+ super(BasicRecord, self).save(*args, **kwargs)
760
+ # update _old_name and _old_key after saving
729
761
  _store_record_old_name(self)
762
+ _store_record_old_key(self)
730
763
  # perform transfer of many-to-many fields
731
764
  # only supported for Artifact and Collection records
732
765
  if db is not None and db != "default" and using_key is None:
@@ -739,7 +772,7 @@ def save(self, *args, **kwargs) -> Record:
739
772
  if hasattr(self, "labels"):
740
773
  from copy import copy
741
774
 
742
- from lnschema_core.models import FeatureManager
775
+ from lamindb.models import FeatureManager
743
776
 
744
777
  # here we go back to original record on the source database
745
778
  self_on_db = copy(self)
@@ -757,19 +790,33 @@ def save(self, *args, **kwargs) -> Record:
757
790
  def _store_record_old_name(record: Record):
758
791
  # writes the name to the _name attribute, so we can detect renaming upon save
759
792
  if hasattr(record, "_name_field"):
760
- record._name = getattr(record, record._name_field)
793
+ record._old_name = getattr(record, record._name_field)
794
+
795
+
796
+ def _store_record_old_key(record: Record):
797
+ # writes the key to the _old_key attribute, so we can detect key changes upon save
798
+ if isinstance(record, (Artifact, Transform)):
799
+ record._old_key = record.key
761
800
 
762
801
 
763
802
  def check_name_change(record: Record):
764
803
  """Warns if a record's name has changed."""
765
804
  if (
766
805
  not record.pk
767
- or not hasattr(record, "_name")
806
+ or not hasattr(record, "_old_name")
768
807
  or not hasattr(record, "_name_field")
769
808
  ):
770
809
  return
771
810
 
772
- old_name = record._name
811
+ # checked in check_key_change or not checked at all
812
+ if isinstance(record, (Artifact, Collection, Transform)):
813
+ return
814
+
815
+ # renaming feature sets is not checked
816
+ if isinstance(record, Schema):
817
+ return
818
+
819
+ old_name = record._old_name
773
820
  new_name = getattr(record, record._name_field)
774
821
  registry = record.__class__.__name__
775
822
 
@@ -783,13 +830,13 @@ def check_name_change(record: Record):
783
830
  .exclude(feature_id=None) # must have a feature
784
831
  .exclude(
785
832
  feature_ref_is_name=None
786
- ) # must be linked via Curator and therefore part of a featureset
833
+ ) # must be linked via Curator and therefore part of a schema
787
834
  .distinct()
788
835
  )
789
836
  artifact_ids = linked_records.list("artifact__uid")
790
837
  n = len(artifact_ids)
791
- s = "s" if n > 1 else ""
792
838
  if n > 0:
839
+ s = "s" if n > 1 else ""
793
840
  logger.error(
794
841
  f"You are trying to {colors.red('rename label')} from '{old_name}' to '{new_name}'!\n"
795
842
  f" → The following {n} artifact{s} {colors.red('will no longer be validated')}: {artifact_ids}\n\n"
@@ -803,13 +850,13 @@ def check_name_change(record: Record):
803
850
 
804
851
  # when a feature is renamed
805
852
  elif isinstance(record, Feature):
806
- # only internal features are associated with featuresets
807
- linked_artifacts = Artifact.filter(feature_sets__features=record).list(
853
+ # only internal features are associated with schemas
854
+ linked_artifacts = Artifact.filter(_schemas_m2m__features=record).list(
808
855
  "uid"
809
856
  )
810
857
  n = len(linked_artifacts)
811
- s = "s" if n > 1 else ""
812
858
  if n > 0:
859
+ s = "s" if n > 1 else ""
813
860
  logger.error(
814
861
  f"You are trying to {colors.red('rename feature')} from '{old_name}' to '{new_name}'!\n"
815
862
  f" → The following {n} artifact{s} {colors.red('will no longer be validated')}: {linked_artifacts}\n\n"
@@ -822,6 +869,33 @@ def check_name_change(record: Record):
822
869
  raise RecordNameChangeIntegrityError
823
870
 
824
871
 
872
+ def check_key_change(record: Artifact | Transform):
873
+ """Errors if a record's key has falsely changed."""
874
+ if not isinstance(record, Artifact) or not hasattr(record, "_old_key"):
875
+ return
876
+
877
+ old_key = record._old_key or ""
878
+ new_key = record.key or ""
879
+
880
+ if old_key != new_key:
881
+ if not record._key_is_virtual:
882
+ raise InvalidArgument(
883
+ f"Changing a non-virtual key of an artifact is not allowed! Tried to change key from '{old_key}' to '{new_key}'."
884
+ )
885
+ old_key_suffix = (
886
+ record.suffix
887
+ if record.suffix
888
+ else extract_suffix_from_path(PurePosixPath(old_key), arg_name="key")
889
+ )
890
+ new_key_suffix = extract_suffix_from_path(
891
+ PurePosixPath(new_key), arg_name="key"
892
+ )
893
+ if old_key_suffix != new_key_suffix:
894
+ raise InvalidArgument(
895
+ f"The suffix '{new_key_suffix}' of the provided key is incorrect, it should be '{old_key_suffix}'."
896
+ )
897
+
898
+
825
899
  def delete(self) -> None:
826
900
  """Delete the record."""
827
901
  # note that the logic below does not fire if a record is moved to the trash
@@ -841,10 +915,10 @@ def delete(self) -> None:
841
915
  new_latest.is_latest = True
842
916
  with transaction.atomic():
843
917
  new_latest.save()
844
- super(Record, self).delete()
918
+ super(BasicRecord, self).delete()
845
919
  logger.warning(f"new latest version is {new_latest}")
846
920
  return None
847
- super(Record, self).delete()
921
+ super(BasicRecord, self).delete()
848
922
 
849
923
 
850
924
  METHOD_NAMES = [
@@ -869,4 +943,5 @@ if ln_setup._TESTING: # type: ignore
869
943
  }
870
944
 
871
945
  for name in METHOD_NAMES:
946
+ attach_func_to_class_method(name, BasicRecord, globals())
872
947
  attach_func_to_class_method(name, Record, globals())
lamindb/_run.py CHANGED
@@ -1,6 +1,6 @@
1
1
  from __future__ import annotations
2
2
 
3
- from lnschema_core.models import ParamManager, Run, Transform
3
+ from lamindb.models import ParamManager, Run, Transform
4
4
 
5
5
 
6
6
  def __init__(run: Run, *args, **kwargs):
@@ -18,7 +18,7 @@ def __init__(run: Run, *args, **kwargs):
18
18
  reference_type: str | None = (
19
19
  kwargs.pop("reference_type") if "reference_type" in kwargs else None
20
20
  )
21
- parent: Run | None = kwargs.pop("parent", None)
21
+ initiated_by_run: Run | None = kwargs.pop("initiated_by_run", None)
22
22
  if transform is None:
23
23
  raise TypeError("Pass transform parameter")
24
24
  if transform._state.adding:
@@ -27,7 +27,7 @@ def __init__(run: Run, *args, **kwargs):
27
27
  super(Run, run).__init__(
28
28
  transform=transform,
29
29
  reference=reference,
30
- parent=parent,
30
+ initiated_by_run=initiated_by_run,
31
31
  reference_type=reference_type,
32
32
  )
33
33
 
lamindb/_save.py CHANGED
@@ -5,15 +5,14 @@ import shutil
5
5
  import traceback
6
6
  from collections import defaultdict
7
7
  from datetime import datetime
8
- from functools import partial
9
- from typing import TYPE_CHECKING, overload
8
+ from typing import TYPE_CHECKING
10
9
 
11
- import lamindb_setup
12
- from django.db import IntegrityError, transaction
10
+ from django.db import transaction
13
11
  from django.utils.functional import partition
14
12
  from lamin_utils import logger
15
13
  from lamindb_setup.core.upath import LocalPathClasses
16
- from lnschema_core.models import Artifact, Record
14
+
15
+ from lamindb.models import Artifact, Record
17
16
 
18
17
  from .core._settings import settings
19
18
  from .core.storage.paths import (
@@ -64,7 +63,7 @@ def save(records: Iterable[Record], ignore_conflicts: bool | None = False) -> No
64
63
  Update a single existing record:
65
64
 
66
65
  >>> transform = ln.Transform.get("0Cb86EZj")
67
- >>> transform.name = "New name"
66
+ >>> transform.description = "New description"
68
67
  >>> transform.save()
69
68
 
70
69
  """
@@ -184,10 +183,17 @@ def copy_or_move_to_cache(
184
183
  return None
185
184
  # non-local storage_path further
186
185
  if local_path != cache_path:
187
- cache_path.parent.mkdir(parents=True, exist_ok=True)
188
- if cache_dir in local_path.parents:
186
+ if cache_path.exists():
187
+ logger.warning(
188
+ f"The cache path {cache_path.as_posix()} already exists, replacing it."
189
+ )
189
190
  if cache_path.is_dir():
190
191
  shutil.rmtree(cache_path)
192
+ else:
193
+ cache_path.unlink()
194
+ else:
195
+ cache_path.parent.mkdir(parents=True, exist_ok=True)
196
+ if cache_dir in local_path.parents:
191
197
  local_path.replace(cache_path)
192
198
  else:
193
199
  if is_dir:
@@ -7,14 +7,16 @@ import numpy as np
7
7
  from lamin_utils import logger
8
8
  from lamindb_setup.core._docs import doc_args
9
9
  from lamindb_setup.core.hashing import hash_set
10
- from lnschema_core import Feature, FeatureSet, Record, ids
11
- from lnschema_core.types import FieldAttr, ListLike
10
+
11
+ from lamindb.base import ids
12
+ from lamindb.base.types import FieldAttr, ListLike
13
+ from lamindb.models import Feature, Record, Schema
12
14
 
13
15
  from ._feature import convert_pandas_dtype_to_lamin_dtype
14
16
  from ._record import init_self_from_db
15
17
  from ._utils import attach_func_to_class_method
16
18
  from .core.exceptions import ValidationError
17
- from .core.schema import (
19
+ from .core.relations import (
18
20
  dict_related_model_to_related_name,
19
21
  get_related_name,
20
22
  )
@@ -47,7 +49,7 @@ def validate_features(features: list[Record]) -> Record:
47
49
  )
48
50
  feature_types = {feature.__class__ for feature in features}
49
51
  if len(feature_types) > 1:
50
- raise TypeError("feature_set can only contain a single type")
52
+ raise TypeError("schema can only contain a single type")
51
53
  for feature in features:
52
54
  if feature._state.adding:
53
55
  raise ValueError("Can only construct feature sets from validated features")
@@ -56,7 +58,7 @@ def validate_features(features: list[Record]) -> Record:
56
58
 
57
59
  def __init__(self, *args, **kwargs):
58
60
  if len(args) == len(self._meta.concrete_fields):
59
- super(FeatureSet, self).__init__(*args, **kwargs)
61
+ super(Schema, self).__init__(*args, **kwargs)
60
62
  return None
61
63
  # now we proceed with the user-facing constructor
62
64
  if len(args) > 1:
@@ -72,29 +74,29 @@ def __init__(self, *args, **kwargs):
72
74
  dtype = None if features_registry == Feature else NUMBER_TYPE
73
75
  n_features = len(features)
74
76
  features_hash = hash_set({feature.uid for feature in features})
75
- feature_set = FeatureSet.filter(hash=features_hash).one_or_none()
76
- if feature_set is not None:
77
- logger.debug(f"loaded: {feature_set}")
78
- init_self_from_db(self, feature_set)
77
+ schema = Schema.filter(hash=features_hash).one_or_none()
78
+ if schema is not None:
79
+ logger.debug(f"loaded: {schema}")
80
+ init_self_from_db(self, schema)
79
81
  return None
80
82
  else:
81
83
  hash = features_hash
82
84
  self._features = (get_related_name(features_registry), features)
83
85
 
84
- super(FeatureSet, self).__init__(
86
+ super(Schema, self).__init__(
85
87
  uid=ids.base62_20(),
86
88
  name=name,
87
89
  dtype=get_type_str(dtype),
88
90
  n=n_features,
89
- registry=features_registry.__get_name_with_schema__(),
91
+ registry=features_registry.__get_name_with_module__(),
90
92
  hash=hash,
91
93
  )
92
94
 
93
95
 
94
- @doc_args(FeatureSet.save.__doc__)
95
- def save(self, *args, **kwargs) -> FeatureSet:
96
+ @doc_args(Schema.save.__doc__)
97
+ def save(self, *args, **kwargs) -> Schema:
96
98
  """{}""" # noqa: D415
97
- super(FeatureSet, self).save(*args, **kwargs)
99
+ super(Schema, self).save(*args, **kwargs)
98
100
  if hasattr(self, "_features"):
99
101
  related_name, records = self._features
100
102
  getattr(self, related_name).set(records)
@@ -110,7 +112,7 @@ def get_type_str(dtype: str | None) -> str | None:
110
112
 
111
113
 
112
114
  @classmethod # type:ignore
113
- @doc_args(FeatureSet.from_values.__doc__)
115
+ @doc_args(Schema.from_values.__doc__)
114
116
  def from_values(
115
117
  cls,
116
118
  values: ListLike,
@@ -121,7 +123,7 @@ def from_values(
121
123
  organism: Record | str | None = None,
122
124
  source: Record | None = None,
123
125
  raise_validation_error: bool = True,
124
- ) -> FeatureSet:
126
+ ) -> Schema:
125
127
  """{}""" # noqa: D415
126
128
  if not isinstance(field, FieldAttr):
127
129
  raise TypeError("Argument `field` must be a Record field, e.g., `Feature.name`")
@@ -152,16 +154,16 @@ def from_values(
152
154
  organism=organism,
153
155
  source=source,
154
156
  )
155
- feature_set = FeatureSet(
157
+ schema = Schema(
156
158
  features=validated_features,
157
159
  name=name,
158
160
  dtype=get_type_str(type),
159
161
  )
160
- return feature_set
162
+ return schema
161
163
 
162
164
 
163
165
  @classmethod # type:ignore
164
- @doc_args(FeatureSet.from_df.__doc__)
166
+ @doc_args(Schema.from_df.__doc__)
165
167
  def from_df(
166
168
  cls,
167
169
  df: pd.DataFrame,
@@ -170,7 +172,7 @@ def from_df(
170
172
  mute: bool = False,
171
173
  organism: Record | str | None = None,
172
174
  source: Record | None = None,
173
- ) -> FeatureSet | None:
175
+ ) -> Schema | None:
174
176
  """{}""" # noqa: D415
175
177
  registry = field.field.model
176
178
  validated = registry.validate(df.columns, field=field, mute=mute, organism=organism)
@@ -182,7 +184,7 @@ def from_df(
182
184
  validated_features = Feature.from_values(
183
185
  df.columns, field=field, organism=organism
184
186
  )
185
- feature_set = FeatureSet(validated_features, name=name, dtype=None)
187
+ schema = Schema(validated_features, name=name, dtype=None)
186
188
  else:
187
189
  dtypes = [col.dtype for (_, col) in df.loc[:, validated].items()]
188
190
  if len(set(dtypes)) != 1:
@@ -194,16 +196,16 @@ def from_df(
194
196
  organism=organism,
195
197
  source=source,
196
198
  )
197
- feature_set = FeatureSet(
199
+ schema = Schema(
198
200
  features=validated_features,
199
201
  name=name,
200
202
  dtype=get_type_str(dtype),
201
203
  )
202
- return feature_set
204
+ return schema
203
205
 
204
206
 
205
207
  @property # type: ignore
206
- @doc_args(FeatureSet.members.__doc__)
208
+ @doc_args(Schema.members.__doc__)
207
209
  def members(self) -> QuerySet:
208
210
  """{}""" # noqa: D415
209
211
  if self._state.adding:
@@ -216,11 +218,11 @@ def members(self) -> QuerySet:
216
218
  return self.__getattribute__(related_name).all()
217
219
 
218
220
 
219
- def _get_related_name(self: FeatureSet) -> str:
220
- feature_sets_related_models = dict_related_model_to_related_name(
221
+ def _get_related_name(self: Schema) -> str:
222
+ _schemas_m2m_related_models = dict_related_model_to_related_name(
221
223
  self, instance=self._state.db
222
224
  )
223
- related_name = feature_sets_related_models.get(self.registry)
225
+ related_name = _schemas_m2m_related_models.get(self.itype)
224
226
  return related_name
225
227
 
226
228
 
@@ -235,13 +237,14 @@ if ln_setup._TESTING:
235
237
  from inspect import signature
236
238
 
237
239
  SIGS = {
238
- name: signature(getattr(FeatureSet, name))
240
+ name: signature(getattr(Schema, name))
239
241
  for name in METHOD_NAMES
240
242
  if name != "__init__"
241
243
  }
242
244
 
243
245
  for name in METHOD_NAMES:
244
- attach_func_to_class_method(name, FeatureSet, globals())
246
+ attach_func_to_class_method(name, Schema, globals())
245
247
 
246
- FeatureSet.members = members
247
- FeatureSet._get_related_name = _get_related_name
248
+ Schema.members = members
249
+ Schema._get_related_name = _get_related_name
250
+ Schema.feature_sets = Schema._artifacts_m2m # backward compat
lamindb/_storage.py CHANGED
@@ -1,6 +1,7 @@
1
1
  from lamindb_setup.core._docs import doc_args
2
2
  from lamindb_setup.core.upath import UPath, create_path
3
- from lnschema_core import Storage
3
+
4
+ from lamindb.models import Storage
4
5
 
5
6
 
6
7
  @property # type: ignore