lamindb 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lamindb/__init__.py +33 -26
- lamindb/_finish.py +9 -1
- lamindb/_tracked.py +26 -3
- lamindb/_view.py +2 -3
- lamindb/base/__init__.py +1 -1
- lamindb/base/ids.py +1 -10
- lamindb/base/users.py +1 -4
- lamindb/core/__init__.py +7 -65
- lamindb/core/_compat.py +60 -0
- lamindb/core/_context.py +50 -22
- lamindb/core/_mapped_collection.py +4 -2
- lamindb/core/_settings.py +6 -6
- lamindb/core/_sync_git.py +1 -1
- lamindb/core/_track_environment.py +2 -1
- lamindb/core/datasets/_small.py +3 -3
- lamindb/core/loaders.py +43 -20
- lamindb/core/storage/_anndata_accessor.py +8 -3
- lamindb/core/storage/_backed_access.py +14 -7
- lamindb/core/storage/_pyarrow_dataset.py +24 -9
- lamindb/core/storage/_tiledbsoma.py +8 -6
- lamindb/core/storage/_zarr.py +104 -25
- lamindb/core/storage/objects.py +63 -28
- lamindb/core/storage/paths.py +16 -13
- lamindb/core/types.py +10 -0
- lamindb/curators/__init__.py +176 -149
- lamindb/errors.py +1 -1
- lamindb/integrations/_vitessce.py +4 -4
- lamindb/migrations/0089_subsequent_runs.py +159 -0
- lamindb/migrations/0090_runproject_project_runs.py +73 -0
- lamindb/migrations/{0088_squashed.py → 0090_squashed.py} +245 -177
- lamindb/models/__init__.py +79 -0
- lamindb/{core → models}/_describe.py +3 -3
- lamindb/{core → models}/_django.py +8 -5
- lamindb/{core → models}/_feature_manager.py +103 -87
- lamindb/{_from_values.py → models/_from_values.py} +5 -2
- lamindb/{core/versioning.py → models/_is_versioned.py} +94 -6
- lamindb/{core → models}/_label_manager.py +10 -17
- lamindb/{core/relations.py → models/_relations.py} +8 -1
- lamindb/models/artifact.py +2602 -0
- lamindb/{_can_curate.py → models/can_curate.py} +349 -180
- lamindb/models/collection.py +683 -0
- lamindb/models/core.py +135 -0
- lamindb/models/feature.py +643 -0
- lamindb/models/flextable.py +163 -0
- lamindb/{_parents.py → models/has_parents.py} +55 -49
- lamindb/models/project.py +384 -0
- lamindb/{_query_manager.py → models/query_manager.py} +10 -8
- lamindb/{_query_set.py → models/query_set.py} +64 -32
- lamindb/models/record.py +1762 -0
- lamindb/models/run.py +563 -0
- lamindb/{_save.py → models/save.py} +18 -8
- lamindb/models/schema.py +732 -0
- lamindb/models/transform.py +360 -0
- lamindb/models/ulabel.py +249 -0
- {lamindb-1.1.0.dist-info → lamindb-1.2.0.dist-info}/METADATA +6 -6
- lamindb-1.2.0.dist-info/RECORD +95 -0
- lamindb/_artifact.py +0 -1361
- lamindb/_collection.py +0 -440
- lamindb/_feature.py +0 -316
- lamindb/_is_versioned.py +0 -40
- lamindb/_record.py +0 -1065
- lamindb/_run.py +0 -60
- lamindb/_schema.py +0 -347
- lamindb/_storage.py +0 -15
- lamindb/_transform.py +0 -170
- lamindb/_ulabel.py +0 -56
- lamindb/_utils.py +0 -9
- lamindb/base/validation.py +0 -63
- lamindb/core/_data.py +0 -491
- lamindb/core/fields.py +0 -12
- lamindb/models.py +0 -4435
- lamindb-1.1.0.dist-info/RECORD +0 -95
- {lamindb-1.1.0.dist-info → lamindb-1.2.0.dist-info}/LICENSE +0 -0
- {lamindb-1.1.0.dist-info → lamindb-1.2.0.dist-info}/WHEEL +0 -0
@@ -0,0 +1,79 @@
|
|
1
|
+
"""Models library.
|
2
|
+
|
3
|
+
.. autosummary::
|
4
|
+
:toctree: .
|
5
|
+
|
6
|
+
BasicRecord
|
7
|
+
Record
|
8
|
+
Registry
|
9
|
+
QuerySet
|
10
|
+
QueryManager
|
11
|
+
RecordList
|
12
|
+
FeatureManager
|
13
|
+
ParamManager
|
14
|
+
LabelManager
|
15
|
+
IsVersioned
|
16
|
+
CanCurate
|
17
|
+
HasParents
|
18
|
+
TracksRun
|
19
|
+
TracksUpdates
|
20
|
+
ParamValue
|
21
|
+
FeatureValue
|
22
|
+
InspectResult
|
23
|
+
ValidateFields
|
24
|
+
|
25
|
+
"""
|
26
|
+
|
27
|
+
# ruff: noqa: I001
|
28
|
+
from lamin_utils._inspect import InspectResult
|
29
|
+
from ._is_versioned import IsVersioned
|
30
|
+
from .can_curate import CanCurate
|
31
|
+
from .record import (
|
32
|
+
BasicRecord,
|
33
|
+
Record,
|
34
|
+
Registry,
|
35
|
+
Space,
|
36
|
+
ValidateFields,
|
37
|
+
format_field_value,
|
38
|
+
record_repr,
|
39
|
+
LinkORM,
|
40
|
+
)
|
41
|
+
from .core import Storage
|
42
|
+
from .transform import Transform
|
43
|
+
from .run import Run, TracksRun, TracksUpdates, current_run, Param, ParamValue, User
|
44
|
+
from .feature import Feature, FeatureValue
|
45
|
+
from .schema import Schema
|
46
|
+
from .ulabel import ULabel
|
47
|
+
|
48
|
+
# should come last as it needs everything else
|
49
|
+
from .artifact import Artifact
|
50
|
+
from ._feature_manager import FeatureManager
|
51
|
+
from .run import ParamManager
|
52
|
+
from ._label_manager import LabelManager
|
53
|
+
from .collection import Collection, CollectionArtifact
|
54
|
+
from .project import Person, Project, Reference
|
55
|
+
from .flextable import FlexTable, RunData
|
56
|
+
from .query_manager import QueryManager
|
57
|
+
from .query_set import QuerySet, RecordList
|
58
|
+
from .has_parents import HasParents
|
59
|
+
from datetime import datetime as _datetime
|
60
|
+
|
61
|
+
FeatureSet = Schema # backward compat
|
62
|
+
|
63
|
+
# link models
|
64
|
+
from .artifact import ArtifactFeatureValue, ArtifactParamValue
|
65
|
+
from .flextable import FlexTableData
|
66
|
+
from .project import (
|
67
|
+
ArtifactProject,
|
68
|
+
TransformProject,
|
69
|
+
CollectionProject,
|
70
|
+
ULabelProject,
|
71
|
+
FeatureProject,
|
72
|
+
SchemaProject,
|
73
|
+
ArtifactReference,
|
74
|
+
CollectionReference,
|
75
|
+
)
|
76
|
+
from .record import Migration
|
77
|
+
from .run import RunParamValue
|
78
|
+
from .schema import SchemaFeature, SchemaParam, ArtifactSchema, SchemaComponent
|
79
|
+
from .ulabel import ArtifactULabel, TransformULabel, RunULabel, CollectionULabel
|
@@ -7,7 +7,7 @@ from lamin_utils import logger
|
|
7
7
|
from rich.text import Text
|
8
8
|
from rich.tree import Tree
|
9
9
|
|
10
|
-
from ._context import is_run_from_ipython
|
10
|
+
from ..core._context import is_run_from_ipython
|
11
11
|
|
12
12
|
if TYPE_CHECKING:
|
13
13
|
from lamindb.models import Artifact, Collection, Run
|
@@ -73,9 +73,9 @@ def describe_header(self: Artifact | Collection | Run) -> Tree:
|
|
73
73
|
f"This is not the latest version of the {self.__class__.__name__}."
|
74
74
|
)
|
75
75
|
if hasattr(self, "_branch_code"):
|
76
|
-
if self._branch_code == 0:
|
76
|
+
if self._branch_code == 0: # type: ignore
|
77
77
|
logger.warning("This artifact is hidden.")
|
78
|
-
elif self._branch_code == -1:
|
78
|
+
elif self._branch_code == -1: # type: ignore
|
79
79
|
logger.warning("This artifact is in the trash.")
|
80
80
|
# initialize tree
|
81
81
|
suffix = self.suffix if hasattr(self, "suffix") and self.suffix else ""
|
@@ -1,6 +1,7 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
|
3
3
|
from functools import reduce
|
4
|
+
from typing import TYPE_CHECKING
|
4
5
|
|
5
6
|
from django.contrib.postgres.aggregates import ArrayAgg
|
6
7
|
from django.db import connection
|
@@ -9,9 +10,12 @@ from django.db.models.fields.related import ForeignKey, ManyToManyField
|
|
9
10
|
from django.db.models.fields.reverse_related import ManyToManyRel, ManyToOneRel
|
10
11
|
from django.db.models.functions import JSONObject
|
11
12
|
|
12
|
-
from
|
13
|
+
from ._relations import dict_related_model_to_related_name, get_schema_modules
|
14
|
+
from .schema import Schema
|
13
15
|
|
14
|
-
|
16
|
+
if TYPE_CHECKING:
|
17
|
+
from .artifact import Artifact
|
18
|
+
from .record import Record
|
15
19
|
|
16
20
|
|
17
21
|
def get_related_model(model, field_name):
|
@@ -38,9 +42,8 @@ def get_artifact_with_related(
|
|
38
42
|
include_schema: bool = False,
|
39
43
|
) -> dict:
|
40
44
|
"""Fetch an artifact with its related data."""
|
41
|
-
from lamindb._can_curate import get_name_field
|
42
|
-
|
43
45
|
from ._label_manager import EXCLUDE_LABELS
|
46
|
+
from .can_curate import get_name_field
|
44
47
|
|
45
48
|
model = artifact.__class__
|
46
49
|
schema_modules = get_schema_modules(artifact._state.db)
|
@@ -177,7 +180,7 @@ def get_artifact_with_related(
|
|
177
180
|
|
178
181
|
def get_schema_m2m_relations(artifact: Artifact, slot_schema: dict, limit: int = 20):
|
179
182
|
"""Fetch all many-to-many relationships for given feature sets."""
|
180
|
-
from
|
183
|
+
from .can_curate import get_name_field
|
181
184
|
|
182
185
|
m2m_relations = [
|
183
186
|
v
|
@@ -1,6 +1,6 @@
|
|
1
|
+
# ruff: noqa: TC004
|
1
2
|
from __future__ import annotations
|
2
3
|
|
3
|
-
import warnings
|
4
4
|
from collections import defaultdict
|
5
5
|
from collections.abc import Iterable
|
6
6
|
from datetime import date, datetime
|
@@ -20,38 +20,23 @@ from lamindb_setup.core.upath import create_path
|
|
20
20
|
from rich.table import Column, Table
|
21
21
|
from rich.text import Text
|
22
22
|
|
23
|
-
from lamindb.
|
23
|
+
from lamindb.core.storage import LocalPathClasses
|
24
|
+
from lamindb.errors import DoesNotExist, ValidationError
|
25
|
+
from lamindb.models._from_values import _format_values
|
26
|
+
from lamindb.models.feature import (
|
24
27
|
convert_pandas_dtype_to_lamin_dtype,
|
25
28
|
suggest_categorical_for_str_iterable,
|
26
29
|
)
|
27
|
-
from lamindb.
|
28
|
-
from lamindb._record import (
|
30
|
+
from lamindb.models.record import (
|
29
31
|
REGISTRY_UNIQUE_FIELD,
|
30
32
|
get_name_field,
|
31
33
|
transfer_fk_to_default_db_bulk,
|
32
34
|
transfer_to_default_db,
|
33
35
|
)
|
34
|
-
from lamindb.
|
35
|
-
from lamindb.
|
36
|
-
from lamindb.core.storage import LocalPathClasses
|
37
|
-
from lamindb.errors import DoesNotExist, ValidationError
|
38
|
-
from lamindb.models import (
|
39
|
-
Artifact,
|
40
|
-
Collection,
|
41
|
-
Feature,
|
42
|
-
FeatureManager,
|
43
|
-
FeatureValue,
|
44
|
-
LinkORM,
|
45
|
-
Param,
|
46
|
-
ParamManager,
|
47
|
-
ParamManagerArtifact,
|
48
|
-
ParamManagerRun,
|
49
|
-
ParamValue,
|
50
|
-
Record,
|
51
|
-
Run,
|
52
|
-
ULabel,
|
53
|
-
)
|
36
|
+
from lamindb.models.save import save
|
37
|
+
from lamindb.models.schema import DICT_KEYS_TYPE, Schema
|
54
38
|
|
39
|
+
from ..base import deprecated
|
55
40
|
from ._describe import (
|
56
41
|
NAME_WIDTH,
|
57
42
|
TYPE_WIDTH,
|
@@ -61,24 +46,24 @@ from ._describe import (
|
|
61
46
|
)
|
62
47
|
from ._django import get_artifact_with_related
|
63
48
|
from ._label_manager import _get_labels, describe_labels
|
64
|
-
from .
|
65
|
-
from .relations import (
|
49
|
+
from ._relations import (
|
66
50
|
dict_related_model_to_related_name,
|
67
51
|
)
|
52
|
+
from .feature import Feature, FeatureValue
|
53
|
+
from .record import Record
|
54
|
+
from .run import Param, ParamManager, ParamManagerRun, ParamValue, Run
|
55
|
+
from .ulabel import ULabel
|
68
56
|
|
69
57
|
if TYPE_CHECKING:
|
70
58
|
from rich.tree import Tree
|
71
59
|
|
72
|
-
from lamindb._query_set import QuerySet
|
73
60
|
from lamindb.base.types import FieldAttr
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
host_id_field = "collection_id"
|
81
|
-
return host_id_field
|
61
|
+
from lamindb.models import (
|
62
|
+
Artifact,
|
63
|
+
Collection,
|
64
|
+
LinkORM,
|
65
|
+
)
|
66
|
+
from lamindb.models.query_set import QuerySet
|
82
67
|
|
83
68
|
|
84
69
|
def get_accessor_by_registry_(host: Artifact | Collection) -> dict:
|
@@ -91,9 +76,7 @@ def get_accessor_by_registry_(host: Artifact | Collection) -> dict:
|
|
91
76
|
return dictionary
|
92
77
|
|
93
78
|
|
94
|
-
def get_schema_by_slot_(host: Artifact
|
95
|
-
if isinstance(host, Collection):
|
96
|
-
return {}
|
79
|
+
def get_schema_by_slot_(host: Artifact) -> dict:
|
97
80
|
# if the host is not yet saved
|
98
81
|
if host._state.adding:
|
99
82
|
if hasattr(host, "_staged_feature_sets"):
|
@@ -101,8 +84,7 @@ def get_schema_by_slot_(host: Artifact | Collection) -> dict:
|
|
101
84
|
else:
|
102
85
|
return {}
|
103
86
|
host_db = host._state.db
|
104
|
-
|
105
|
-
kwargs = {host_id_field: host.id}
|
87
|
+
kwargs = {"artifact_id": host.id}
|
106
88
|
# otherwise, we need a query
|
107
89
|
links_schema = (
|
108
90
|
host.feature_sets.through.objects.using(host_db)
|
@@ -115,8 +97,7 @@ def get_schema_by_slot_(host: Artifact | Collection) -> dict:
|
|
115
97
|
def get_label_links(
|
116
98
|
host: Artifact | Collection, registry: str, feature: Feature
|
117
99
|
) -> QuerySet:
|
118
|
-
|
119
|
-
kwargs = {host_id_field: host.id, "feature_id": feature.id}
|
100
|
+
kwargs = {"artifact_id": host.id, "feature_id": feature.id}
|
120
101
|
link_records = (
|
121
102
|
getattr(host, host.features._accessor_by_registry[registry]) # type: ignore
|
122
103
|
.through.objects.using(host._state.db)
|
@@ -126,8 +107,7 @@ def get_label_links(
|
|
126
107
|
|
127
108
|
|
128
109
|
def get_schema_links(host: Artifact | Collection) -> QuerySet:
|
129
|
-
|
130
|
-
kwargs = {host_id_field: host.id}
|
110
|
+
kwargs = {"artifact_id": host.id}
|
131
111
|
links_schema = host.feature_sets.through.objects.filter(**kwargs)
|
132
112
|
return links_schema
|
133
113
|
|
@@ -232,6 +212,9 @@ def _get_non_categoricals(
|
|
232
212
|
print_params: bool = False,
|
233
213
|
) -> dict[tuple[str, str], set[Any]]:
|
234
214
|
"""Get non-categorical features and their values."""
|
215
|
+
from .artifact import Artifact
|
216
|
+
from .run import Run
|
217
|
+
|
235
218
|
non_categoricals = {}
|
236
219
|
|
237
220
|
if self.id is not None and isinstance(self, (Artifact, Run)):
|
@@ -299,21 +282,15 @@ def _create_feature_table(
|
|
299
282
|
|
300
283
|
|
301
284
|
def describe_features(
|
302
|
-
self: Artifact
|
285
|
+
self: Artifact,
|
303
286
|
related_data: dict | None = None,
|
304
|
-
print_types: bool = False,
|
305
287
|
to_dict: bool = False,
|
306
288
|
print_params: bool = False,
|
307
289
|
tree: Tree | None = None,
|
308
290
|
with_labels: bool = False,
|
309
291
|
):
|
310
292
|
"""Describe features of an artifact or collection."""
|
311
|
-
|
312
|
-
warnings.warn(
|
313
|
-
"`print_types` parameter is deprecated and will be removed in a future version. Types are now always printed.",
|
314
|
-
DeprecationWarning,
|
315
|
-
stacklevel=2,
|
316
|
-
)
|
293
|
+
from .artifact import Artifact
|
317
294
|
|
318
295
|
# initialize tree
|
319
296
|
if tree is None:
|
@@ -466,7 +443,7 @@ def describe_features(
|
|
466
443
|
Text.assemble(
|
467
444
|
("Dataset features", "bold bright_magenta"),
|
468
445
|
("/", "dim"),
|
469
|
-
("
|
446
|
+
(".feature_sets", "dim bold"),
|
470
447
|
)
|
471
448
|
)
|
472
449
|
for child in int_features_tree_children:
|
@@ -511,6 +488,7 @@ def parse_staged_feature_sets_from_anndata(
|
|
511
488
|
if not isinstance(adata, AnnData): # is a path
|
512
489
|
filepath = create_path(adata) # returns Path for local
|
513
490
|
if not isinstance(filepath, LocalPathClasses):
|
491
|
+
from lamindb import settings
|
514
492
|
from lamindb.core.storage._backed_access import backed_access
|
515
493
|
|
516
494
|
using_key = settings._using_key
|
@@ -631,9 +609,21 @@ def infer_feature_type_convert_json(
|
|
631
609
|
return "?", value, message
|
632
610
|
|
633
611
|
|
612
|
+
class FeatureManager:
|
613
|
+
"""Feature manager."""
|
614
|
+
|
615
|
+
pass
|
616
|
+
|
617
|
+
|
618
|
+
class ParamManagerArtifact(ParamManager):
|
619
|
+
"""Param manager."""
|
620
|
+
|
621
|
+
pass
|
622
|
+
|
623
|
+
|
634
624
|
def __init__(self, host: Artifact | Collection | Run):
|
635
625
|
self._host = host
|
636
|
-
self.
|
626
|
+
self._slots = None
|
637
627
|
self._accessor_by_registry_ = None
|
638
628
|
|
639
629
|
|
@@ -649,19 +639,22 @@ def get_values(self) -> dict[str, Any]:
|
|
649
639
|
) # type: ignore
|
650
640
|
|
651
641
|
|
642
|
+
@deprecated("slots[slot].members")
|
652
643
|
def __getitem__(self, slot) -> QuerySet:
|
653
|
-
if slot not in self.
|
644
|
+
if slot not in self.slots:
|
654
645
|
raise ValueError(
|
655
646
|
f"No linked feature set for slot: {slot}\nDid you get validation"
|
656
647
|
" warnings? Only features that match registered features get validated"
|
657
648
|
" and linked."
|
658
649
|
)
|
659
|
-
schema = self.
|
650
|
+
schema = self.slots[slot]
|
660
651
|
orm_name = schema.itype
|
661
652
|
return getattr(schema, self._accessor_by_registry[orm_name]).all()
|
662
653
|
|
663
654
|
|
664
655
|
def filter_base(cls, **expression):
|
656
|
+
from .artifact import Artifact
|
657
|
+
|
665
658
|
if cls is FeatureManager:
|
666
659
|
model = Feature
|
667
660
|
value_model = FeatureValue
|
@@ -748,11 +741,17 @@ def get(cls, **expression) -> Record:
|
|
748
741
|
|
749
742
|
|
750
743
|
@property # type: ignore
|
751
|
-
def
|
752
|
-
"""
|
753
|
-
|
754
|
-
|
755
|
-
|
744
|
+
def slots(self) -> dict[str, Schema]:
|
745
|
+
"""Schema by slot.
|
746
|
+
|
747
|
+
Example:
|
748
|
+
|
749
|
+
>>> artifact.features.slots
|
750
|
+
{'var': <Schema: var>, 'obs': <Schema: obs>}
|
751
|
+
"""
|
752
|
+
if self._slots is None:
|
753
|
+
self._slots = get_schema_by_slot_(self._host)
|
754
|
+
return self._slots
|
756
755
|
|
757
756
|
|
758
757
|
@property # type: ignore
|
@@ -819,6 +818,7 @@ def _add_values(
|
|
819
818
|
dictionary.
|
820
819
|
"""
|
821
820
|
from .._tracked import get_current_tracked_run
|
821
|
+
from .artifact import Artifact
|
822
822
|
|
823
823
|
# rename to distinguish from the values inside the dict
|
824
824
|
features_values = values
|
@@ -1019,6 +1019,8 @@ def remove_values(
|
|
1019
1019
|
value: An optional value to restrict removal to a single value.
|
1020
1020
|
|
1021
1021
|
"""
|
1022
|
+
from .artifact import Artifact
|
1023
|
+
|
1022
1024
|
if isinstance(feature, str):
|
1023
1025
|
feature = Feature.get(name=feature)
|
1024
1026
|
filter_kwargs = {"feature": feature}
|
@@ -1059,7 +1061,7 @@ def remove_values(
|
|
1059
1061
|
# we can clean the FeatureValue registry periodically if we want to
|
1060
1062
|
|
1061
1063
|
|
1062
|
-
def
|
1064
|
+
def _add_schema(self, schema: Schema, slot: str) -> None:
|
1063
1065
|
"""Annotate artifact with a schema.
|
1064
1066
|
|
1065
1067
|
Args:
|
@@ -1074,9 +1076,8 @@ def add_schema(self, schema: Schema, slot: str) -> None:
|
|
1074
1076
|
)
|
1075
1077
|
host_db = self._host._state.db
|
1076
1078
|
schema.save(using=host_db)
|
1077
|
-
host_id_field = get_host_id_field(self._host)
|
1078
1079
|
kwargs = {
|
1079
|
-
|
1080
|
+
"artifact_id": self._host.id,
|
1080
1081
|
"schema": schema,
|
1081
1082
|
"slot": slot,
|
1082
1083
|
}
|
@@ -1087,9 +1088,9 @@ def add_schema(self, schema: Schema, slot: str) -> None:
|
|
1087
1088
|
)
|
1088
1089
|
if link_record is None:
|
1089
1090
|
self._host.feature_sets.through(**kwargs).save(using=host_db)
|
1090
|
-
if slot in self.
|
1091
|
+
if slot in self.slots:
|
1091
1092
|
logger.debug(f"replaced existing {slot} feature set")
|
1092
|
-
self.
|
1093
|
+
self._slots[slot] = schema # type: ignore
|
1093
1094
|
|
1094
1095
|
|
1095
1096
|
def _add_set_from_df(
|
@@ -1099,11 +1100,7 @@ def _add_set_from_df(
|
|
1099
1100
|
mute: bool = False,
|
1100
1101
|
):
|
1101
1102
|
"""Add feature set corresponding to column names of DataFrame."""
|
1102
|
-
|
1103
|
-
assert self._host.otype == "DataFrame" # noqa: S101
|
1104
|
-
else:
|
1105
|
-
# Collection
|
1106
|
-
assert self._host.artifact.otype == "DataFrame" # noqa: S101
|
1103
|
+
assert self._host.otype == "DataFrame" # noqa: S101
|
1107
1104
|
df = self._host.load()
|
1108
1105
|
schema = Schema.from_df(
|
1109
1106
|
df=df,
|
@@ -1117,16 +1114,13 @@ def _add_set_from_df(
|
|
1117
1114
|
|
1118
1115
|
def _add_set_from_anndata(
|
1119
1116
|
self,
|
1120
|
-
var_field: FieldAttr,
|
1117
|
+
var_field: FieldAttr | None = None,
|
1121
1118
|
obs_field: FieldAttr | None = Feature.name,
|
1122
1119
|
mute: bool = False,
|
1123
1120
|
organism: str | Record | None = None,
|
1124
1121
|
):
|
1125
1122
|
"""Add features from AnnData."""
|
1126
|
-
|
1127
|
-
assert self._host.otype == "AnnData" # noqa: S101
|
1128
|
-
else:
|
1129
|
-
raise NotImplementedError()
|
1123
|
+
assert self._host.otype == "AnnData" # noqa: S101
|
1130
1124
|
|
1131
1125
|
# parse and register features
|
1132
1126
|
adata = self._host.load()
|
@@ -1153,10 +1147,7 @@ def _add_set_from_mudata(
|
|
1153
1147
|
"""Add features from MuData."""
|
1154
1148
|
if obs_fields is None:
|
1155
1149
|
obs_fields = {}
|
1156
|
-
|
1157
|
-
assert self._host.otype == "MuData" # noqa: S101
|
1158
|
-
else:
|
1159
|
-
raise NotImplementedError()
|
1150
|
+
assert self._host.otype == "MuData" # noqa: S101
|
1160
1151
|
|
1161
1152
|
# parse and register features
|
1162
1153
|
mdata = self._host.load()
|
@@ -1197,8 +1188,10 @@ def _add_from(self, data: Artifact | Collection, transfer_logs: dict = None):
|
|
1197
1188
|
# This only covers feature sets
|
1198
1189
|
if transfer_logs is None:
|
1199
1190
|
transfer_logs = {"mapped": [], "transferred": [], "run": None}
|
1191
|
+
from lamindb import settings
|
1192
|
+
|
1200
1193
|
using_key = settings._using_key
|
1201
|
-
for slot, schema in data.features.
|
1194
|
+
for slot, schema in data.features.slots.items(): # type: ignore
|
1202
1195
|
members = schema.members
|
1203
1196
|
if len(members) == 0:
|
1204
1197
|
continue
|
@@ -1246,7 +1239,7 @@ def _add_from(self, data: Artifact | Collection, transfer_logs: dict = None):
|
|
1246
1239
|
if schema_self.hash == schema.hash:
|
1247
1240
|
schema_self.uid = schema.uid
|
1248
1241
|
logger.info(f"saving {slot} schema: {schema_self}")
|
1249
|
-
self._host.features.
|
1242
|
+
self._host.features._add_schema(schema_self, slot)
|
1250
1243
|
|
1251
1244
|
|
1252
1245
|
def make_external(self, feature: Feature) -> None:
|
@@ -1279,6 +1272,27 @@ def make_external(self, feature: Feature) -> None:
|
|
1279
1272
|
fs.delete()
|
1280
1273
|
|
1281
1274
|
|
1275
|
+
@deprecated("_add_schema")
|
1276
|
+
def add_schema(self, schema: Schema, slot: str) -> None:
|
1277
|
+
return self._add_schema(schema, slot)
|
1278
|
+
|
1279
|
+
|
1280
|
+
@deprecated("_add_schema")
|
1281
|
+
def add_feature_set(self, schema: Schema, slot: str) -> None:
|
1282
|
+
return self._add_schema(schema, slot)
|
1283
|
+
|
1284
|
+
|
1285
|
+
@property
|
1286
|
+
@deprecated("slots")
|
1287
|
+
def _schema_by_slot(self):
|
1288
|
+
return self.slots
|
1289
|
+
|
1290
|
+
|
1291
|
+
@property
|
1292
|
+
def _feature_set_by_slot(self):
|
1293
|
+
return self.slots
|
1294
|
+
|
1295
|
+
|
1282
1296
|
# mypy: ignore-errors
|
1283
1297
|
FeatureManager.__init__ = __init__
|
1284
1298
|
ParamManager.__init__ = __init__
|
@@ -1286,12 +1300,14 @@ FeatureManager.__repr__ = __repr__
|
|
1286
1300
|
ParamManager.__repr__ = __repr__
|
1287
1301
|
FeatureManager.__getitem__ = __getitem__
|
1288
1302
|
FeatureManager.get_values = get_values
|
1289
|
-
FeatureManager.
|
1290
|
-
FeatureManager._feature_set_by_slot = _schema_by_slot
|
1291
|
-
FeatureManager._accessor_by_registry = _accessor_by_registry
|
1303
|
+
FeatureManager.slots = slots
|
1292
1304
|
FeatureManager.add_values = add_values_features
|
1293
|
-
FeatureManager.
|
1294
|
-
FeatureManager.
|
1305
|
+
FeatureManager._add_schema = _add_schema
|
1306
|
+
FeatureManager.add_schema = add_schema # deprecated
|
1307
|
+
FeatureManager.add_feature_set = add_feature_set # deprecated
|
1308
|
+
FeatureManager._schema_by_slot = _schema_by_slot # deprecated
|
1309
|
+
FeatureManager._feature_set_by_slot = _feature_set_by_slot # deprecated
|
1310
|
+
FeatureManager._accessor_by_registry = _accessor_by_registry
|
1295
1311
|
FeatureManager._add_set_from_df = _add_set_from_df
|
1296
1312
|
FeatureManager._add_set_from_anndata = _add_set_from_anndata
|
1297
1313
|
FeatureManager._add_set_from_mudata = _add_set_from_mudata
|
@@ -6,14 +6,15 @@ import pandas as pd
|
|
6
6
|
from django.core.exceptions import FieldDoesNotExist
|
7
7
|
from lamin_utils import colors, logger
|
8
8
|
|
9
|
-
from
|
10
|
-
from lamindb.models import Record
|
9
|
+
from .record import Record
|
11
10
|
|
12
11
|
if TYPE_CHECKING:
|
13
12
|
from collections.abc import Iterable
|
14
13
|
|
15
14
|
from lamindb.base.types import ListLike, StrField
|
16
15
|
|
16
|
+
from .query_set import RecordList
|
17
|
+
|
17
18
|
|
18
19
|
# The base function for `from_values`
|
19
20
|
def get_or_create_records(
|
@@ -27,6 +28,8 @@ def get_or_create_records(
|
|
27
28
|
mute: bool = False,
|
28
29
|
) -> RecordList:
|
29
30
|
"""Get or create records from iterables."""
|
31
|
+
from .query_set import RecordList
|
32
|
+
|
30
33
|
registry = field.field.model # type: ignore
|
31
34
|
if create:
|
32
35
|
return RecordList([registry(**{field.field.name: value}) for value in iterable]) # type: ignore
|