lamindb 0.76.1__py3-none-any.whl → 0.76.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lamindb/__init__.py +7 -4
- lamindb/_artifact.py +80 -53
- lamindb/_can_validate.py +10 -3
- lamindb/_collection.py +17 -18
- lamindb/_curate.py +33 -15
- lamindb/_feature.py +0 -49
- lamindb/_filter.py +6 -5
- lamindb/_finish.py +2 -2
- lamindb/_from_values.py +14 -10
- lamindb/_is_versioned.py +3 -5
- lamindb/_query_manager.py +2 -2
- lamindb/_query_set.py +33 -5
- lamindb/_record.py +29 -39
- lamindb/_save.py +2 -3
- lamindb/_transform.py +23 -10
- lamindb/core/_context.py +16 -11
- lamindb/core/_feature_manager.py +25 -8
- lamindb/core/_label_manager.py +1 -1
- lamindb/core/storage/__init__.py +1 -1
- lamindb/core/storage/_backed_access.py +2 -38
- lamindb/core/storage/_tiledbsoma.py +229 -0
- lamindb/core/storage/paths.py +2 -6
- lamindb/core/versioning.py +43 -47
- lamindb/integrations/_vitessce.py +2 -0
- {lamindb-0.76.1.dist-info → lamindb-0.76.2.dist-info}/METADATA +6 -14
- {lamindb-0.76.1.dist-info → lamindb-0.76.2.dist-info}/RECORD +28 -27
- {lamindb-0.76.1.dist-info → lamindb-0.76.2.dist-info}/LICENSE +0 -0
- {lamindb-0.76.1.dist-info → lamindb-0.76.2.dist-info}/WHEEL +0 -0
lamindb/_filter.py
CHANGED
@@ -1,18 +1,19 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
|
3
|
-
from lnschema_core import Artifact, Collection,
|
3
|
+
from lnschema_core import Artifact, Collection, Record
|
4
4
|
from lnschema_core.types import VisibilityChoice
|
5
5
|
|
6
6
|
from lamindb import settings
|
7
|
-
from lamindb._query_set import QuerySet
|
8
7
|
|
8
|
+
from ._query_set import QuerySet
|
9
9
|
|
10
|
-
|
10
|
+
|
11
|
+
def filter(registry: type[Record], **expressions) -> QuerySet:
|
11
12
|
"""See :meth:`~lamindb.core.Record.filter`."""
|
12
13
|
_using_key = None
|
13
14
|
if "_using_key" in expressions:
|
14
15
|
_using_key = expressions.pop("_using_key")
|
15
|
-
if
|
16
|
+
if registry in {Artifact, Collection}:
|
16
17
|
# visibility is set to 0 unless expressions contains id or uid equality
|
17
18
|
if not (
|
18
19
|
"id" in expressions
|
@@ -29,7 +30,7 @@ def filter(Record: type[Record], **expressions) -> QuerySet:
|
|
29
30
|
# sense for a non-NULLABLE column
|
30
31
|
elif visibility in expressions and expressions[visibility] is None:
|
31
32
|
expressions.pop(visibility)
|
32
|
-
qs = QuerySet(model=
|
33
|
+
qs = QuerySet(model=registry, using=_using_key)
|
33
34
|
if len(expressions) > 0:
|
34
35
|
return qs.filter(**expressions)
|
35
36
|
else:
|
lamindb/_finish.py
CHANGED
@@ -148,7 +148,7 @@ def save_context_core(
|
|
148
148
|
_source_code_artifact_path,
|
149
149
|
description=f"Source of transform {transform.uid}",
|
150
150
|
version=transform.version,
|
151
|
-
|
151
|
+
revises=prev_source,
|
152
152
|
visibility=0, # hidden file
|
153
153
|
run=False,
|
154
154
|
)
|
@@ -211,7 +211,7 @@ def save_context_core(
|
|
211
211
|
report_file = ln.Artifact(
|
212
212
|
report_path,
|
213
213
|
description=f"Report of run {run.uid}",
|
214
|
-
|
214
|
+
revises=prev_report,
|
215
215
|
visibility=0, # hidden file
|
216
216
|
run=False,
|
217
217
|
)
|
lamindb/_from_values.py
CHANGED
@@ -25,9 +25,9 @@ def get_or_create_records(
|
|
25
25
|
mute: bool = False,
|
26
26
|
) -> list[Record]:
|
27
27
|
"""Get or create records from iterables."""
|
28
|
-
|
28
|
+
registry = field.field.model
|
29
29
|
if create:
|
30
|
-
return [
|
30
|
+
return [registry(**{field.field.name: value}) for value in iterable]
|
31
31
|
creation_search_names = settings.creation.search_names
|
32
32
|
feature: Feature = None
|
33
33
|
organism = _get_organism_record(field, organism)
|
@@ -57,21 +57,23 @@ def get_or_create_records(
|
|
57
57
|
and records[0].source_id
|
58
58
|
):
|
59
59
|
source_record = records[0].source
|
60
|
-
if not source_record and hasattr(
|
60
|
+
if not source_record and hasattr(registry, "public"):
|
61
61
|
from bionty._bionty import get_source_record
|
62
62
|
|
63
|
-
source_record = get_source_record(
|
63
|
+
source_record = get_source_record(
|
64
|
+
registry.public(organism=organism), registry
|
65
|
+
)
|
64
66
|
if source_record:
|
65
67
|
from bionty.core._add_ontology import check_source_in_db
|
66
68
|
|
67
69
|
check_source_in_db(
|
68
|
-
registry=
|
70
|
+
registry=registry,
|
69
71
|
source=source_record,
|
70
72
|
update=True,
|
71
73
|
)
|
72
74
|
|
73
75
|
from_source = not source_record.in_db
|
74
|
-
elif hasattr(
|
76
|
+
elif hasattr(registry, "source_id"):
|
75
77
|
from_source = True
|
76
78
|
else:
|
77
79
|
from_source = False
|
@@ -97,14 +99,14 @@ def get_or_create_records(
|
|
97
99
|
logger.success(msg)
|
98
100
|
s = "" if len(unmapped_values) == 1 else "s"
|
99
101
|
print_values = colors.yellow(_print_values(unmapped_values))
|
100
|
-
name =
|
102
|
+
name = registry.__name__
|
101
103
|
n_nonval = colors.yellow(f"{len(unmapped_values)} non-validated")
|
102
104
|
if not mute:
|
103
105
|
logger.warning(
|
104
106
|
f"{colors.red('did not create')} {name} record{s} for "
|
105
107
|
f"{n_nonval} {colors.italic(f'{field.field.name}{s}')}: {print_values}"
|
106
108
|
)
|
107
|
-
if
|
109
|
+
if registry.__get_schema_name__() == "bionty" or registry == ULabel:
|
108
110
|
if isinstance(iterable, pd.Series):
|
109
111
|
feature = iterable.name
|
110
112
|
feature_name = None
|
@@ -230,7 +232,7 @@ def create_records_from_source(
|
|
230
232
|
# for custom records that are not created from public sources
|
231
233
|
return records, iterable_idx
|
232
234
|
# add source record to the kwargs
|
233
|
-
source_record = get_source_record(public_ontology)
|
235
|
+
source_record = get_source_record(public_ontology, model)
|
234
236
|
kwargs.update({"source": source_record})
|
235
237
|
|
236
238
|
# filter the columns in bionty df based on fields
|
@@ -373,6 +375,8 @@ def _get_organism_record(
|
|
373
375
|
if _has_organism_field(registry) and check:
|
374
376
|
from bionty._bionty import create_or_get_organism_record
|
375
377
|
|
376
|
-
organism_record = create_or_get_organism_record(
|
378
|
+
organism_record = create_or_get_organism_record(
|
379
|
+
organism=organism, registry=registry
|
380
|
+
)
|
377
381
|
if organism_record is not None:
|
378
382
|
return organism_record
|
lamindb/_is_versioned.py
CHANGED
@@ -7,15 +7,13 @@ from lnschema_core.models import IsVersioned
|
|
7
7
|
|
8
8
|
from lamindb._utils import attach_func_to_class_method
|
9
9
|
|
10
|
-
from .core.versioning import
|
10
|
+
from .core.versioning import create_uid, get_new_path_from_uid
|
11
11
|
|
12
12
|
|
13
13
|
# docstring handled through attach_func_to_class_method
|
14
|
-
def _add_to_version_family(
|
15
|
-
self, is_new_version_of: IsVersioned, version: str | None = None
|
16
|
-
):
|
14
|
+
def _add_to_version_family(self, revises: IsVersioned, version: str | None = None):
|
17
15
|
old_uid = self.uid
|
18
|
-
new_uid,
|
16
|
+
new_uid, revises = create_uid(revises=revises, version=version)
|
19
17
|
if self.__class__.__name__ == "Artifact" and self._key_is_virtual:
|
20
18
|
old_path = self.path
|
21
19
|
new_path = get_new_path_from_uid(
|
lamindb/_query_manager.py
CHANGED
@@ -28,7 +28,7 @@ class QueryManager(models.Manager):
|
|
28
28
|
>>> ln.save(ln.ULabel.from_values(["ULabel1", "ULabel2", "ULabel3"], field="name")) # noqa
|
29
29
|
>>> labels = ln.ULabel.filter(name__icontains = "label").all()
|
30
30
|
>>> ln.ULabel(name="ULabel1").save()
|
31
|
-
>>> label = ln.ULabel.
|
31
|
+
>>> label = ln.ULabel.get(name="ULabel1")
|
32
32
|
>>> label.parents.set(labels)
|
33
33
|
>>> manager = label.parents
|
34
34
|
>>> manager.df()
|
@@ -57,7 +57,7 @@ class QueryManager(models.Manager):
|
|
57
57
|
>>> ln.save(ln.ULabel.from_values(["ULabel1", "ULabel2", "ULabel3"], field="name"))
|
58
58
|
>>> labels = ln.ULabel.filter(name__icontains="label").all()
|
59
59
|
>>> ln.ULabel(name="ULabel1").save()
|
60
|
-
>>> label = ln.ULabel.
|
60
|
+
>>> label = ln.ULabel.get(name="ULabel1")
|
61
61
|
>>> label.parents.set(labels)
|
62
62
|
>>> label.parents.list()
|
63
63
|
>>> label.parents.list("name")
|
lamindb/_query_set.py
CHANGED
@@ -64,6 +64,34 @@ def one_helper(self):
|
|
64
64
|
return self[0]
|
65
65
|
|
66
66
|
|
67
|
+
def get(
|
68
|
+
registry_or_queryset: type[Record] | QuerySet,
|
69
|
+
idlike: int | str | None = None,
|
70
|
+
**expressions,
|
71
|
+
) -> Record:
|
72
|
+
if isinstance(registry_or_queryset, QuerySet):
|
73
|
+
qs = registry_or_queryset
|
74
|
+
registry = qs.model
|
75
|
+
else:
|
76
|
+
qs = QuerySet(model=registry_or_queryset)
|
77
|
+
registry = registry_or_queryset
|
78
|
+
if isinstance(idlike, int):
|
79
|
+
return super(QuerySet, qs).get(id=idlike)
|
80
|
+
elif isinstance(idlike, str):
|
81
|
+
qs = qs.filter(uid__startswith=idlike)
|
82
|
+
if issubclass(registry, IsVersioned):
|
83
|
+
if len(idlike) <= registry._len_stem_uid:
|
84
|
+
return qs.latest_version().one()
|
85
|
+
else:
|
86
|
+
return qs.one()
|
87
|
+
else:
|
88
|
+
return qs.one()
|
89
|
+
else:
|
90
|
+
assert idlike is None # noqa: S101
|
91
|
+
# below behaves exactly like `.one()`
|
92
|
+
return registry.objects.get(**expressions)
|
93
|
+
|
94
|
+
|
67
95
|
class RecordsList(UserList):
|
68
96
|
"""Is ordered, can't be queried, but has `.df()`."""
|
69
97
|
|
@@ -219,12 +247,12 @@ class QuerySet(models.QuerySet, CanValidate):
|
|
219
247
|
return None
|
220
248
|
return self[0]
|
221
249
|
|
222
|
-
def
|
223
|
-
"""
|
250
|
+
def get(self, idlike: int | str | None = None, **expressions) -> Record:
|
251
|
+
"""Query a single record. Raises error if there are more or none."""
|
252
|
+
return get(self, idlike, **expressions)
|
224
253
|
|
225
|
-
|
226
|
-
|
227
|
-
"""
|
254
|
+
def one(self) -> Record:
|
255
|
+
"""Exactly one result. Raises error if there are more or none."""
|
228
256
|
return one_helper(self)
|
229
257
|
|
230
258
|
def one_or_none(self) -> Record | None:
|
lamindb/_record.py
CHANGED
@@ -12,7 +12,7 @@ from lamin_utils._lookup import Lookup
|
|
12
12
|
from lamindb_setup._connect_instance import get_owner_name_from_identifier
|
13
13
|
from lamindb_setup.core._docs import doc_args
|
14
14
|
from lamindb_setup.core._hub_core import connect_instance
|
15
|
-
from lnschema_core.models import IsVersioned, Record
|
15
|
+
from lnschema_core.models import Collection, IsVersioned, Record
|
16
16
|
|
17
17
|
from lamindb._utils import attach_func_to_class_method
|
18
18
|
from lamindb.core._settings import settings
|
@@ -90,13 +90,22 @@ def __init__(record: Record, *args, **kwargs):
|
|
90
90
|
match = suggest_records_with_similar_names(record, kwargs)
|
91
91
|
if match:
|
92
92
|
if "version" in kwargs:
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
93
|
+
if kwargs["version"] is not None:
|
94
|
+
version_comment = " and version"
|
95
|
+
existing_record = record.__class__.filter(
|
96
|
+
name=kwargs["name"], version=kwargs["version"]
|
97
|
+
).one_or_none()
|
98
|
+
else:
|
99
|
+
# for a versioned record, an exact name match is not a
|
100
|
+
# criterion for retrieving a record in case `version`
|
101
|
+
# isn't passed - we'd always pull out many records with exactly the
|
102
|
+
# same name
|
103
|
+
existing_record = None
|
97
104
|
else:
|
98
105
|
version_comment = ""
|
99
|
-
existing_record = record.__class__.filter(
|
106
|
+
existing_record = record.__class__.filter(
|
107
|
+
name=kwargs["name"]
|
108
|
+
).one_or_none()
|
100
109
|
if existing_record is not None:
|
101
110
|
logger.important(
|
102
111
|
f"returning existing {record.__class__.__name__} record with same"
|
@@ -129,23 +138,11 @@ def get(
|
|
129
138
|
**expressions,
|
130
139
|
) -> Record:
|
131
140
|
"""{}""" # noqa: D415
|
132
|
-
|
141
|
+
# this is the only place in which we need the lamindb queryset
|
142
|
+
# in this file; everywhere else it should be Django's
|
143
|
+
from lamindb._query_set import QuerySet
|
133
144
|
|
134
|
-
|
135
|
-
return filter(cls, id=idlike).one()
|
136
|
-
elif isinstance(idlike, str):
|
137
|
-
qs = filter(cls, uid__startswith=idlike)
|
138
|
-
if issubclass(cls, IsVersioned):
|
139
|
-
if len(idlike) <= cls._len_stem_uid:
|
140
|
-
return qs.latest_version().one()
|
141
|
-
else:
|
142
|
-
return qs.one()
|
143
|
-
else:
|
144
|
-
return qs.one()
|
145
|
-
else:
|
146
|
-
assert idlike is None # noqa: S101
|
147
|
-
# below behaves exactly like `.one()`
|
148
|
-
return cls.objects.get(**expressions)
|
145
|
+
return QuerySet(model=cls).get(idlike, **expressions)
|
149
146
|
|
150
147
|
|
151
148
|
@classmethod # type:ignore
|
@@ -393,9 +390,11 @@ def add_db_connection(db: str, using: str):
|
|
393
390
|
@doc_args(Record.using.__doc__)
|
394
391
|
def using(
|
395
392
|
cls,
|
396
|
-
instance: str,
|
393
|
+
instance: str | None,
|
397
394
|
) -> QuerySet:
|
398
395
|
"""{}""" # noqa: D415
|
396
|
+
if instance is None:
|
397
|
+
return QuerySet(model=cls, using=None)
|
399
398
|
from lamindb_setup._connect_instance import (
|
400
399
|
load_instance_settings,
|
401
400
|
update_db_using_local,
|
@@ -541,24 +540,15 @@ def save(self, *args, **kwargs) -> Record:
|
|
541
540
|
init_self_from_db(self, result)
|
542
541
|
else:
|
543
542
|
# save versioned record
|
544
|
-
if isinstance(self, IsVersioned) and self.
|
545
|
-
|
546
|
-
|
547
|
-
|
548
|
-
# need one additional request
|
549
|
-
is_new_version_of = self.__class__.objects.get(
|
550
|
-
is_latest=True, uid__startswith=self.stem_uid
|
551
|
-
)
|
552
|
-
logger.warning(
|
553
|
-
f"didn't pass the latest version in `is_new_version_of`, retrieved it: {is_new_version_of}"
|
554
|
-
)
|
555
|
-
is_new_version_of.is_latest = False
|
543
|
+
if isinstance(self, IsVersioned) and self._revises is not None:
|
544
|
+
assert self._revises.is_latest # noqa: S101
|
545
|
+
revises = self._revises
|
546
|
+
revises.is_latest = False
|
556
547
|
with transaction.atomic():
|
557
|
-
|
558
|
-
|
559
|
-
)
|
560
|
-
is_new_version_of.save()
|
548
|
+
revises._revises = None # ensure we don't start a recursion
|
549
|
+
revises.save()
|
561
550
|
super(Record, self).save(*args, **kwargs)
|
551
|
+
self._revises = None
|
562
552
|
# save unversioned record
|
563
553
|
else:
|
564
554
|
super(Record, self).save(*args, **kwargs)
|
lamindb/_save.py
CHANGED
@@ -48,8 +48,7 @@ def save(records: Iterable[Record], ignore_conflicts: bool | None = False) -> No
|
|
48
48
|
|
49
49
|
Examples:
|
50
50
|
|
51
|
-
Save a
|
52
|
-
than writing a loop over ``projects.save()``:
|
51
|
+
Save a list of records:
|
53
52
|
|
54
53
|
>>> labels = [ln.ULabel(f"Label {i}") for i in range(10)]
|
55
54
|
>>> ln.save(projects)
|
@@ -61,7 +60,7 @@ def save(records: Iterable[Record], ignore_conflicts: bool | None = False) -> No
|
|
61
60
|
|
62
61
|
Update a single existing record:
|
63
62
|
|
64
|
-
>>> transform = ln.
|
63
|
+
>>> transform = ln.Transform.get("0Cb86EZj")
|
65
64
|
>>> transform.name = "New name"
|
66
65
|
>>> transform.save()
|
67
66
|
|
lamindb/_transform.py
CHANGED
@@ -2,12 +2,13 @@ from __future__ import annotations
|
|
2
2
|
|
3
3
|
from typing import TYPE_CHECKING
|
4
4
|
|
5
|
+
from lamin_utils import logger
|
5
6
|
from lamindb_setup.core._docs import doc_args
|
6
7
|
from lnschema_core.models import Run, Transform
|
7
8
|
|
8
9
|
from ._parents import _view_parents
|
9
10
|
from ._run import delete_run_artifacts
|
10
|
-
from .core.versioning import
|
11
|
+
from .core.versioning import message_update_key_in_version_family, process_revises
|
11
12
|
|
12
13
|
if TYPE_CHECKING:
|
13
14
|
from lnschema_core.types import TransformType
|
@@ -19,26 +20,38 @@ def __init__(transform: Transform, *args, **kwargs):
|
|
19
20
|
return None
|
20
21
|
name: str | None = kwargs.pop("name") if "name" in kwargs else None
|
21
22
|
key: str | None = kwargs.pop("key") if "key" in kwargs else None
|
22
|
-
|
23
|
-
kwargs.pop("is_new_version_of") if "is_new_version_of" in kwargs else None
|
24
|
-
)
|
23
|
+
revises: Transform | None = kwargs.pop("revises") if "revises" in kwargs else None
|
25
24
|
version: str | None = kwargs.pop("version") if "version" in kwargs else None
|
26
25
|
type: TransformType | None = kwargs.pop("type") if "type" in kwargs else "pipeline"
|
27
26
|
reference: str | None = kwargs.pop("reference") if "reference" in kwargs else None
|
28
27
|
reference_type: str | None = (
|
29
28
|
kwargs.pop("reference_type") if "reference_type" in kwargs else None
|
30
29
|
)
|
30
|
+
if "is_new_version_of" in kwargs:
|
31
|
+
logger.warning("`is_new_version_of` will be removed soon, please use `revises`")
|
32
|
+
revises = kwargs.pop("is_new_version_of")
|
31
33
|
# below is internal use that we'll hopefully be able to eliminate
|
32
34
|
uid: str | None = kwargs.pop("uid") if "uid" in kwargs else None
|
33
35
|
if not len(kwargs) == 0:
|
34
36
|
raise ValueError(
|
35
|
-
"Only name, key, version, type,
|
37
|
+
"Only name, key, version, type, revises, reference, "
|
36
38
|
f"reference_type can be passed, but you passed: {kwargs}"
|
37
39
|
)
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
40
|
+
# Transform allows passing a uid, all others don't
|
41
|
+
if uid is None and key is not None:
|
42
|
+
revises = Transform.filter(key=key).order_by("-created_at").first()
|
43
|
+
if revises is not None and key is not None and revises.key != key:
|
44
|
+
note = message_update_key_in_version_family(
|
45
|
+
suid=revises.stem_uid,
|
46
|
+
existing_key=revises.key,
|
47
|
+
new_key=key,
|
48
|
+
registry="Artifact",
|
49
|
+
)
|
50
|
+
raise ValueError(
|
51
|
+
f"`key` is {key}, but `revises.key` is '{revises.key}'\n\n Either do *not* pass `key`.\n\n{note}"
|
52
|
+
)
|
53
|
+
new_uid, version, name, revises = process_revises(revises, version, name, Transform)
|
54
|
+
# this is only because the user-facing constructor allows passing a uid
|
42
55
|
# most others don't
|
43
56
|
if uid is None:
|
44
57
|
has_consciously_provided_uid = False
|
@@ -54,7 +67,7 @@ def __init__(transform: Transform, *args, **kwargs):
|
|
54
67
|
reference=reference,
|
55
68
|
reference_type=reference_type,
|
56
69
|
_has_consciously_provided_uid=has_consciously_provided_uid,
|
57
|
-
|
70
|
+
revises=revises,
|
58
71
|
)
|
59
72
|
|
60
73
|
|
lamindb/core/_context.py
CHANGED
@@ -26,7 +26,7 @@ from .exceptions import (
|
|
26
26
|
)
|
27
27
|
from .subsettings._transform_settings import transform_settings
|
28
28
|
from .versioning import bump_version as bump_version_function
|
29
|
-
from .versioning import increment_base62
|
29
|
+
from .versioning import increment_base62, message_update_key_in_version_family
|
30
30
|
|
31
31
|
if TYPE_CHECKING:
|
32
32
|
from lamindb_setup.core.types import UPathStr
|
@@ -87,9 +87,9 @@ def raise_missing_context(transform_type: str, key: str) -> None:
|
|
87
87
|
message = f"To track this {transform_type}, set\n\n"
|
88
88
|
else:
|
89
89
|
uid = transform.uid
|
90
|
-
suid,
|
91
|
-
|
92
|
-
new_uid = f"{suid}{
|
90
|
+
suid, vuid = uid[: Transform._len_stem_uid], uid[Transform._len_stem_uid :]
|
91
|
+
new_vuid = increment_base62(vuid)
|
92
|
+
new_uid = f"{suid}{new_vuid}"
|
93
93
|
message = f"You already have a {transform_type} version family with key '{key}', suid '{transform.stem_uid}' & name '{transform.name}'.\n\n- to create a new {transform_type} version family, rename your file and rerun: ln.context.track()\n- to bump the version, set: "
|
94
94
|
message += f'ln.context.uid = "{new_uid}"'
|
95
95
|
if transform_type == "notebook":
|
@@ -199,7 +199,7 @@ class Context:
|
|
199
199
|
:class:`~lamindb.Transform` object of ``type`` ``"pipeline"``:
|
200
200
|
|
201
201
|
>>> ln.Transform(name="Cell Ranger", version="2", type="pipeline").save()
|
202
|
-
>>> transform = ln.Transform.
|
202
|
+
>>> transform = ln.Transform.get(name="Cell Ranger", version="2")
|
203
203
|
>>> ln.context.track(transform=transform)
|
204
204
|
"""
|
205
205
|
self._path = None
|
@@ -224,7 +224,7 @@ class Context:
|
|
224
224
|
f"Please pass consistent version: ln.context.version = '{transform.version}'"
|
225
225
|
)
|
226
226
|
# test whether version was already used for another member of the family
|
227
|
-
suid,
|
227
|
+
suid, vuid = (
|
228
228
|
self.uid[: Transform._len_stem_uid],
|
229
229
|
self.uid[Transform._len_stem_uid :],
|
230
230
|
)
|
@@ -233,7 +233,7 @@ class Context:
|
|
233
233
|
).one_or_none()
|
234
234
|
if (
|
235
235
|
transform is not None
|
236
|
-
and
|
236
|
+
and vuid != transform.uid[Transform._len_stem_uid :]
|
237
237
|
):
|
238
238
|
better_version = bump_version_function(self.version)
|
239
239
|
raise SystemExit(
|
@@ -432,7 +432,12 @@ class Context:
|
|
432
432
|
suid = transform.stem_uid
|
433
433
|
new_suid = ids.base62_12()
|
434
434
|
transform_type = "Notebook" if is_run_from_ipython else "Script"
|
435
|
-
note =
|
435
|
+
note = message_update_key_in_version_family(
|
436
|
+
suid=suid,
|
437
|
+
existing_key=transform.key,
|
438
|
+
new_key=key,
|
439
|
+
registry="Transform",
|
440
|
+
)
|
436
441
|
raise UpdateContext(
|
437
442
|
f"{transform_type} filename changed.\n\nEither init a new transform family by setting:\n\n"
|
438
443
|
f'ln.context.uid = "{new_suid}0000"\n\n{note}'
|
@@ -460,14 +465,14 @@ class Context:
|
|
460
465
|
if is_run_from_ipython
|
461
466
|
else "Source code changed"
|
462
467
|
)
|
463
|
-
suid,
|
468
|
+
suid, vuid = (
|
464
469
|
uid[: Transform._len_stem_uid],
|
465
470
|
uid[Transform._len_stem_uid :],
|
466
471
|
)
|
467
|
-
|
472
|
+
new_vuid = increment_base62(vuid)
|
468
473
|
raise UpdateContext(
|
469
474
|
f"{change_type}, bump version by setting:\n\n"
|
470
|
-
f'ln.context.uid = "{suid}{
|
475
|
+
f'ln.context.uid = "{suid}{new_vuid}"'
|
471
476
|
)
|
472
477
|
else:
|
473
478
|
self._logging_message += f"loaded Transform('{transform.uid}')"
|
lamindb/core/_feature_manager.py
CHANGED
@@ -361,9 +361,7 @@ def __getitem__(self, slot) -> QuerySet:
|
|
361
361
|
return getattr(feature_set, self._accessor_by_registry[orm_name]).all()
|
362
362
|
|
363
363
|
|
364
|
-
|
365
|
-
def filter(cls, **expression) -> QuerySet:
|
366
|
-
"""Filter features."""
|
364
|
+
def filter_base(cls, **expression):
|
367
365
|
if cls in {FeatureManagerArtifact, FeatureManagerCollection}:
|
368
366
|
model = Feature
|
369
367
|
value_model = FeatureValue
|
@@ -379,14 +377,20 @@ def filter(cls, **expression) -> QuerySet:
|
|
379
377
|
new_expression = {}
|
380
378
|
features = model.filter(name__in=keys_normalized).all().distinct()
|
381
379
|
for key, value in expression.items():
|
382
|
-
|
380
|
+
split_key = key.split("__")
|
381
|
+
normalized_key = split_key[0]
|
382
|
+
comparator = ""
|
383
|
+
if len(split_key) == 2:
|
384
|
+
comparator = f"__{split_key[1]}"
|
383
385
|
feature = features.get(name=normalized_key)
|
384
386
|
if not feature.dtype.startswith("cat"):
|
385
|
-
|
386
|
-
|
387
|
+
expression = {"feature": feature, f"value{comparator}": value}
|
388
|
+
feature_value = value_model.filter(**expression)
|
389
|
+
new_expression["_feature_values__in"] = feature_value
|
387
390
|
else:
|
388
391
|
if isinstance(value, str):
|
389
|
-
|
392
|
+
expression = {f"name{comparator}": value}
|
393
|
+
label = ULabel.get(**expression)
|
390
394
|
new_expression["ulabels"] = label
|
391
395
|
else:
|
392
396
|
raise NotImplementedError
|
@@ -398,6 +402,18 @@ def filter(cls, **expression) -> QuerySet:
|
|
398
402
|
return Run.filter(**new_expression)
|
399
403
|
|
400
404
|
|
405
|
+
@classmethod # type: ignore
|
406
|
+
def filter(cls, **expression) -> QuerySet:
|
407
|
+
"""Query artifacts by features."""
|
408
|
+
return filter_base(cls, **expression)
|
409
|
+
|
410
|
+
|
411
|
+
@classmethod # type: ignore
|
412
|
+
def get(cls, **expression) -> Record:
|
413
|
+
"""Query a single artifact by feature."""
|
414
|
+
return filter_base(cls, **expression).one()
|
415
|
+
|
416
|
+
|
401
417
|
@property # type: ignore
|
402
418
|
def _feature_set_by_slot(self):
|
403
419
|
"""Feature sets by slot."""
|
@@ -474,7 +490,7 @@ def _add_values(
|
|
474
490
|
_feature_values = []
|
475
491
|
not_validated_values = []
|
476
492
|
for key, value in features_values.items():
|
477
|
-
feature = model.
|
493
|
+
feature = model.get(name=key)
|
478
494
|
inferred_type, converted_value = infer_feature_type_convert_json(
|
479
495
|
value,
|
480
496
|
mute=True,
|
@@ -843,5 +859,6 @@ FeatureManager._add_set_from_anndata = _add_set_from_anndata
|
|
843
859
|
FeatureManager._add_set_from_mudata = _add_set_from_mudata
|
844
860
|
FeatureManager._add_from = _add_from
|
845
861
|
FeatureManager.filter = filter
|
862
|
+
FeatureManager.get = get
|
846
863
|
ParamManager.add_values = add_values_params
|
847
864
|
ParamManager.get_values = get_values
|
lamindb/core/_label_manager.py
CHANGED
@@ -234,7 +234,7 @@ class LabelManager:
|
|
234
234
|
if hasattr(self._host, related_name):
|
235
235
|
for feature_name, labels in labels_by_features.items():
|
236
236
|
if feature_name is not None:
|
237
|
-
feature_id = Feature.
|
237
|
+
feature_id = Feature.get(name=feature_name).id
|
238
238
|
else:
|
239
239
|
feature_id = None
|
240
240
|
getattr(self._host, related_name).add(
|
lamindb/core/storage/__init__.py
CHANGED
@@ -18,8 +18,8 @@ Array accessors.
|
|
18
18
|
|
19
19
|
from lamindb_setup.core.upath import LocalPathClasses, UPath, infer_filesystem
|
20
20
|
|
21
|
-
from ._anndata_sizes import size_adata
|
22
21
|
from ._backed_access import AnnDataAccessor, BackedAccessor
|
22
|
+
from ._tiledbsoma import register_for_tiledbsoma_store, write_tiledbsoma_store
|
23
23
|
from ._valid_suffixes import VALID_SUFFIXES
|
24
24
|
from .objects import infer_suffix, write_to_disk
|
25
25
|
from .paths import delete_storage, load_to_memory
|
@@ -1,12 +1,13 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
|
3
3
|
from dataclasses import dataclass
|
4
|
-
from typing import TYPE_CHECKING, Any, Callable
|
4
|
+
from typing import TYPE_CHECKING, Any, Callable
|
5
5
|
|
6
6
|
from anndata._io.specs.registry import get_spec
|
7
7
|
from lnschema_core import Artifact
|
8
8
|
|
9
9
|
from ._anndata_accessor import AnnDataAccessor, StorageType, registry
|
10
|
+
from ._tiledbsoma import _open_tiledbsoma
|
10
11
|
from .paths import filepath_from_artifact
|
11
12
|
|
12
13
|
if TYPE_CHECKING:
|
@@ -52,43 +53,6 @@ def _track_writes_factory(obj: Any, finalize: Callable):
|
|
52
53
|
return obj
|
53
54
|
|
54
55
|
|
55
|
-
def _open_tiledbsoma(
|
56
|
-
filepath: UPath, mode: Literal["r", "w"] = "r"
|
57
|
-
) -> SOMACollection | SOMAExperiment:
|
58
|
-
try:
|
59
|
-
import tiledbsoma as soma
|
60
|
-
except ImportError as e:
|
61
|
-
raise ImportError("Please install tiledbsoma: pip install tiledbsoma") from e
|
62
|
-
filepath_str = filepath.as_posix()
|
63
|
-
if filepath.protocol == "s3":
|
64
|
-
from lamindb_setup.core._settings_storage import get_storage_region
|
65
|
-
|
66
|
-
region = get_storage_region(filepath_str)
|
67
|
-
tiledb_config = {"vfs.s3.region": region}
|
68
|
-
storage_options = filepath.storage_options
|
69
|
-
if "key" in storage_options:
|
70
|
-
tiledb_config["vfs.s3.aws_access_key_id"] = storage_options["key"]
|
71
|
-
if "secret" in storage_options:
|
72
|
-
tiledb_config["vfs.s3.aws_secret_access_key"] = storage_options["secret"]
|
73
|
-
if "token" in storage_options:
|
74
|
-
tiledb_config["vfs.s3.aws_session_token"] = storage_options["token"]
|
75
|
-
ctx = soma.SOMATileDBContext(tiledb_config=tiledb_config)
|
76
|
-
# this is a strange bug
|
77
|
-
# for some reason iterdir futher gives incorrect results
|
78
|
-
# if cache is not invalidated
|
79
|
-
# instead of obs and ms it gives ms and ms in the list of names
|
80
|
-
filepath.fs.invalidate_cache()
|
81
|
-
else:
|
82
|
-
ctx = None
|
83
|
-
|
84
|
-
soma_objects = [obj.name for obj in filepath.iterdir()]
|
85
|
-
if "obs" in soma_objects and "ms" in soma_objects:
|
86
|
-
SOMAType = soma.Experiment
|
87
|
-
else:
|
88
|
-
SOMAType = soma.Collection
|
89
|
-
return SOMAType.open(filepath_str, mode=mode, context=ctx)
|
90
|
-
|
91
|
-
|
92
56
|
@dataclass
|
93
57
|
class BackedAccessor:
|
94
58
|
"""h5py.File or zarr.Group accessor."""
|