lamindb 1.0.5__py3-none-any.whl → 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lamindb/__init__.py +14 -5
- lamindb/_artifact.py +150 -53
- lamindb/_can_curate.py +27 -8
- lamindb/_collection.py +85 -51
- lamindb/_feature.py +177 -41
- lamindb/_finish.py +12 -6
- lamindb/_from_values.py +83 -98
- lamindb/_parents.py +4 -4
- lamindb/_query_set.py +59 -17
- lamindb/_record.py +171 -53
- lamindb/_run.py +4 -4
- lamindb/_save.py +33 -10
- lamindb/_schema.py +135 -38
- lamindb/_storage.py +1 -1
- lamindb/_tracked.py +106 -0
- lamindb/_transform.py +21 -8
- lamindb/_ulabel.py +5 -14
- lamindb/base/validation.py +2 -6
- lamindb/core/__init__.py +13 -14
- lamindb/core/_context.py +7 -7
- lamindb/core/_data.py +29 -25
- lamindb/core/_describe.py +1 -1
- lamindb/core/_django.py +1 -1
- lamindb/core/_feature_manager.py +53 -43
- lamindb/core/_label_manager.py +4 -4
- lamindb/core/_mapped_collection.py +20 -7
- lamindb/core/datasets/__init__.py +6 -1
- lamindb/core/datasets/_core.py +12 -11
- lamindb/core/datasets/_small.py +66 -20
- lamindb/core/exceptions.py +1 -90
- lamindb/core/loaders.py +6 -12
- lamindb/core/relations.py +6 -4
- lamindb/core/storage/_anndata_accessor.py +41 -0
- lamindb/core/storage/_backed_access.py +2 -2
- lamindb/core/storage/_pyarrow_dataset.py +25 -15
- lamindb/core/storage/_tiledbsoma.py +56 -12
- lamindb/core/storage/paths.py +27 -21
- lamindb/core/subsettings/_creation_settings.py +4 -16
- lamindb/curators/__init__.py +2168 -833
- lamindb/curators/_cellxgene_schemas/__init__.py +26 -0
- lamindb/curators/_cellxgene_schemas/schema_versions.yml +104 -0
- lamindb/errors.py +96 -0
- lamindb/integrations/_vitessce.py +3 -3
- lamindb/migrations/0069_squashed.py +76 -75
- lamindb/migrations/0075_lamindbv1_part5.py +4 -5
- lamindb/migrations/0082_alter_feature_dtype.py +21 -0
- lamindb/migrations/0083_alter_feature_is_type_alter_flextable_is_type_and_more.py +94 -0
- lamindb/migrations/0084_alter_schemafeature_feature_and_more.py +35 -0
- lamindb/migrations/0085_alter_feature_is_type_alter_flextable_is_type_and_more.py +63 -0
- lamindb/migrations/0086_various.py +95 -0
- lamindb/migrations/0087_rename__schemas_m2m_artifact_feature_sets_and_more.py +41 -0
- lamindb/migrations/0088_schema_components.py +273 -0
- lamindb/migrations/0088_squashed.py +4372 -0
- lamindb/models.py +420 -153
- {lamindb-1.0.5.dist-info → lamindb-1.1.0.dist-info}/METADATA +9 -7
- lamindb-1.1.0.dist-info/RECORD +95 -0
- lamindb/curators/_spatial.py +0 -528
- lamindb/migrations/0052_squashed.py +0 -1261
- lamindb/migrations/0053_alter_featureset_hash_alter_paramvalue_created_by_and_more.py +0 -57
- lamindb/migrations/0054_alter_feature_previous_runs_and_more.py +0 -35
- lamindb/migrations/0055_artifact_type_artifactparamvalue_and_more.py +0 -61
- lamindb/migrations/0056_rename_ulabel_ref_is_name_artifactulabel_label_ref_is_name_and_more.py +0 -22
- lamindb/migrations/0057_link_models_latest_report_and_others.py +0 -356
- lamindb/migrations/0058_artifact__actions_collection__actions.py +0 -22
- lamindb/migrations/0059_alter_artifact__accessor_alter_artifact__hash_type_and_more.py +0 -31
- lamindb/migrations/0060_alter_artifact__actions.py +0 -22
- lamindb/migrations/0061_alter_collection_meta_artifact_alter_run_environment_and_more.py +0 -45
- lamindb/migrations/0062_add_is_latest_field.py +0 -32
- lamindb/migrations/0063_populate_latest_field.py +0 -45
- lamindb/migrations/0064_alter_artifact_version_alter_collection_version_and_more.py +0 -33
- lamindb/migrations/0065_remove_collection_feature_sets_and_more.py +0 -22
- lamindb/migrations/0066_alter_artifact__feature_values_and_more.py +0 -352
- lamindb/migrations/0067_alter_featurevalue_unique_together_and_more.py +0 -20
- lamindb/migrations/0068_alter_artifactulabel_unique_together_and_more.py +0 -20
- lamindb/migrations/0069_alter_artifact__accessor_alter_artifact__hash_type_and_more.py +0 -1294
- lamindb-1.0.5.dist-info/RECORD +0 -102
- {lamindb-1.0.5.dist-info → lamindb-1.1.0.dist-info}/LICENSE +0 -0
- {lamindb-1.0.5.dist-info → lamindb-1.1.0.dist-info}/WHEEL +0 -0
lamindb/_schema.py
CHANGED
@@ -10,21 +10,23 @@ from lamindb_setup.core.hashing import hash_set
|
|
10
10
|
|
11
11
|
from lamindb.base import ids
|
12
12
|
from lamindb.base.types import FieldAttr, ListLike
|
13
|
+
from lamindb.errors import InvalidArgument
|
13
14
|
from lamindb.models import Feature, Record, Schema
|
14
15
|
|
15
|
-
from ._feature import convert_pandas_dtype_to_lamin_dtype
|
16
|
-
from ._record import init_self_from_db
|
16
|
+
from ._feature import convert_pandas_dtype_to_lamin_dtype, get_dtype_str_from_dtype
|
17
|
+
from ._record import init_self_from_db, update_attributes
|
17
18
|
from ._utils import attach_func_to_class_method
|
18
|
-
from .core.exceptions import ValidationError
|
19
19
|
from .core.relations import (
|
20
20
|
dict_related_model_to_related_name,
|
21
21
|
get_related_name,
|
22
22
|
)
|
23
|
+
from .errors import ValidationError
|
23
24
|
|
24
25
|
if TYPE_CHECKING:
|
25
26
|
from collections.abc import Iterable
|
26
27
|
|
27
28
|
import pandas as pd
|
29
|
+
from django.db.models.query_utils import DeferredAttribute
|
28
30
|
|
29
31
|
from ._query_set import QuerySet
|
30
32
|
|
@@ -60,46 +62,136 @@ def __init__(self, *args, **kwargs):
|
|
60
62
|
if len(args) == len(self._meta.concrete_fields):
|
61
63
|
super(Schema, self).__init__(*args, **kwargs)
|
62
64
|
return None
|
63
|
-
# now we proceed with the user-facing constructor
|
64
65
|
if len(args) > 1:
|
65
66
|
raise ValueError("Only one non-keyword arg allowed: features")
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
67
|
+
|
68
|
+
features: Iterable[Record] | None = args[0] if args else kwargs.pop("features", [])
|
69
|
+
# typing here anticipates transitioning to a ManyToMany
|
70
|
+
# between composites and components similar to feature_sets
|
71
|
+
# in lamindb v2
|
72
|
+
components: dict[str, Schema] = kwargs.pop("components", {})
|
73
|
+
name: str | None = kwargs.pop("name", None)
|
74
|
+
description: str | None = kwargs.pop("description", None)
|
75
|
+
dtype: str | None = kwargs.pop("dtype", None)
|
76
|
+
itype: str | Record | DeferredAttribute | None = kwargs.pop("itype", None)
|
77
|
+
type: Feature | None = kwargs.pop("type", None)
|
78
|
+
is_type: bool = kwargs.pop("is_type", False)
|
79
|
+
otype: str | None = kwargs.pop("otype", None)
|
80
|
+
minimal_set: bool = kwargs.pop("minimal_set", True)
|
81
|
+
ordered_set: bool = kwargs.pop("ordered_set", False)
|
82
|
+
maximal_set: bool = kwargs.pop("maximal_set", False)
|
83
|
+
slot: str | None = kwargs.pop("slot", None)
|
84
|
+
coerce_dtype: bool | None = kwargs.pop("coerce_dtype", None)
|
85
|
+
|
86
|
+
if kwargs:
|
87
|
+
raise ValueError(
|
88
|
+
f"Unexpected keyword arguments: {', '.join(kwargs.keys())}\n"
|
89
|
+
"Valid arguments are: features, description, dtype, itype, type, "
|
90
|
+
"is_type, otype, minimal_set, ordered_set, maximal_set, "
|
91
|
+
"slot, validated_by, coerce_dtype"
|
92
|
+
)
|
93
|
+
|
94
|
+
if features:
|
95
|
+
features_registry = validate_features(features)
|
96
|
+
itype_compare = features_registry.__get_name_with_module__()
|
97
|
+
if itype is not None:
|
98
|
+
assert itype == itype_compare, str(itype_compare) # noqa: S101
|
99
|
+
else:
|
100
|
+
itype = itype_compare
|
101
|
+
n_features = len(features)
|
102
|
+
else:
|
103
|
+
n_features = -1
|
73
104
|
if dtype is None:
|
74
|
-
dtype = None if
|
75
|
-
|
76
|
-
|
77
|
-
|
105
|
+
dtype = None if itype is not None and itype == "Feature" else NUMBER_TYPE
|
106
|
+
else:
|
107
|
+
dtype = get_type_str(dtype)
|
108
|
+
components: dict[str, Schema]
|
109
|
+
if components:
|
110
|
+
itype = "Composite"
|
111
|
+
if otype is None:
|
112
|
+
raise InvalidArgument("Please pass otype != None for composite schemas")
|
113
|
+
if itype is not None and not isinstance(itype, str):
|
114
|
+
itype_str = get_dtype_str_from_dtype(itype, is_itype=True)
|
115
|
+
else:
|
116
|
+
itype_str = itype
|
117
|
+
validated_kwargs = {
|
118
|
+
"name": name,
|
119
|
+
"description": description,
|
120
|
+
"type": type,
|
121
|
+
"dtype": dtype,
|
122
|
+
"is_type": is_type,
|
123
|
+
"otype": otype,
|
124
|
+
"n": n_features,
|
125
|
+
"itype": itype_str,
|
126
|
+
"minimal_set": minimal_set,
|
127
|
+
"ordered_set": ordered_set,
|
128
|
+
"maximal_set": maximal_set,
|
129
|
+
}
|
130
|
+
if coerce_dtype:
|
131
|
+
validated_kwargs["_aux"] = {"af": {"0": coerce_dtype}}
|
132
|
+
if features:
|
133
|
+
hash = hash_set({feature.uid for feature in features})
|
134
|
+
elif components:
|
135
|
+
hash = hash_set({component.hash for component in components.values()})
|
136
|
+
else:
|
137
|
+
hash = hash_set({str(value) for value in validated_kwargs.values()})
|
138
|
+
validated_kwargs["hash"] = hash
|
139
|
+
validated_kwargs["slot"] = slot
|
140
|
+
schema = Schema.filter(hash=hash).one_or_none()
|
78
141
|
if schema is not None:
|
79
|
-
logger.
|
142
|
+
logger.important(f"returning existing schema with same hash: {schema}")
|
80
143
|
init_self_from_db(self, schema)
|
144
|
+
update_attributes(self, validated_kwargs)
|
81
145
|
return None
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
)
|
146
|
+
if features:
|
147
|
+
self._features = (get_related_name(features_registry), features)
|
148
|
+
elif components:
|
149
|
+
for slot, component in components.items():
|
150
|
+
if component._state.adding:
|
151
|
+
raise InvalidArgument(
|
152
|
+
f"component {slot} {component} must be saved before use"
|
153
|
+
)
|
154
|
+
self._components = components
|
155
|
+
validated_kwargs["uid"] = ids.base62_20()
|
156
|
+
super(Schema, self).__init__(**validated_kwargs)
|
94
157
|
|
95
158
|
|
96
159
|
@doc_args(Schema.save.__doc__)
|
97
160
|
def save(self, *args, **kwargs) -> Schema:
|
98
161
|
"""{}""" # noqa: D415
|
162
|
+
from lamindb._save import bulk_create
|
163
|
+
|
99
164
|
super(Schema, self).save(*args, **kwargs)
|
165
|
+
if hasattr(self, "_components"):
|
166
|
+
# analogous to save_schema_links in core._data.py
|
167
|
+
# which is called to save feature sets in artifact.save()
|
168
|
+
links = []
|
169
|
+
for slot, component in self._components.items():
|
170
|
+
kwargs = {
|
171
|
+
"composite_id": self.id,
|
172
|
+
"component_id": component.id,
|
173
|
+
"slot": slot,
|
174
|
+
}
|
175
|
+
links.append(Schema.components.through(**kwargs))
|
176
|
+
bulk_create(links, ignore_conflicts=True)
|
100
177
|
if hasattr(self, "_features"):
|
178
|
+
assert self.n > 0 # noqa: S101
|
101
179
|
related_name, records = self._features
|
102
|
-
|
180
|
+
# only the following method preserves the order
|
181
|
+
# .set() does not preserve the order but orders by
|
182
|
+
# the feature primary key
|
183
|
+
through_model = getattr(self, related_name).through
|
184
|
+
related_model_split = self.itype.split(".")
|
185
|
+
if len(related_model_split) == 1:
|
186
|
+
related_field = related_model_split[0].lower()
|
187
|
+
else:
|
188
|
+
related_field = related_model_split[1].lower()
|
189
|
+
related_field_id = f"{related_field}_id"
|
190
|
+
links = [
|
191
|
+
through_model(**{"schema_id": self.id, related_field_id: record.id})
|
192
|
+
for record in records
|
193
|
+
]
|
194
|
+
through_model.objects.bulk_create(links, ignore_conflicts=True)
|
103
195
|
return self
|
104
196
|
|
105
197
|
|
@@ -181,10 +273,10 @@ def from_df(
|
|
181
273
|
logger.warning("no validated features, skip creating feature set")
|
182
274
|
return None
|
183
275
|
if registry == Feature:
|
184
|
-
validated_features = Feature.from_values(
|
276
|
+
validated_features = Feature.from_values( # type: ignore
|
185
277
|
df.columns, field=field, organism=organism
|
186
278
|
)
|
187
|
-
schema = Schema(validated_features, name=name, dtype=None)
|
279
|
+
schema = Schema(validated_features, name=name, dtype=None, otype="DataFrame")
|
188
280
|
else:
|
189
281
|
dtypes = [col.dtype for (_, col) in df.loc[:, validated].items()]
|
190
282
|
if len(set(dtypes)) != 1:
|
@@ -200,6 +292,7 @@ def from_df(
|
|
200
292
|
features=validated_features,
|
201
293
|
name=name,
|
202
294
|
dtype=get_type_str(dtype),
|
295
|
+
otype="DataFrame",
|
203
296
|
)
|
204
297
|
return schema
|
205
298
|
|
@@ -215,14 +308,12 @@ def members(self) -> QuerySet:
|
|
215
308
|
related_name = self._get_related_name()
|
216
309
|
if related_name is None:
|
217
310
|
related_name = "features"
|
218
|
-
return self.__getattribute__(related_name).
|
311
|
+
return self.__getattribute__(related_name).order_by("links_schema__id")
|
219
312
|
|
220
313
|
|
221
314
|
def _get_related_name(self: Schema) -> str:
|
222
|
-
|
223
|
-
|
224
|
-
)
|
225
|
-
related_name = _schemas_m2m_related_models.get(self.itype)
|
315
|
+
related_models = dict_related_model_to_related_name(self, instance=self._state.db)
|
316
|
+
related_name = related_models.get(self.itype)
|
226
317
|
return related_name
|
227
318
|
|
228
319
|
|
@@ -245,6 +336,12 @@ if ln_setup._TESTING:
|
|
245
336
|
for name in METHOD_NAMES:
|
246
337
|
attach_func_to_class_method(name, Schema, globals())
|
247
338
|
|
248
|
-
Schema.members = members
|
339
|
+
Schema.members = members # type: ignore
|
249
340
|
Schema._get_related_name = _get_related_name
|
250
|
-
|
341
|
+
# excluded on docs via
|
342
|
+
# https://github.com/laminlabs/lndocs/blob/8c1963de65445107ea69b3fd59354c3828e067d1/lndocs/lamin_sphinx/__init__.py#L584-L588
|
343
|
+
delattr(Schema, "validated_by") # we don't want to expose these
|
344
|
+
delattr(Schema, "validated_by_id") # we don't want to expose these
|
345
|
+
delattr(Schema, "validated_schemas") # we don't want to expose these
|
346
|
+
delattr(Schema, "composite") # we don't want to expose these
|
347
|
+
delattr(Schema, "composite_id") # we don't want to expose these
|
lamindb/_storage.py
CHANGED
lamindb/_tracked.py
ADDED
@@ -0,0 +1,106 @@
|
|
1
|
+
import functools
|
2
|
+
import inspect
|
3
|
+
from contextvars import ContextVar
|
4
|
+
from datetime import datetime, timezone
|
5
|
+
from typing import Callable, ParamSpec, TypeVar
|
6
|
+
|
7
|
+
from .core._context import context
|
8
|
+
from .core._feature_manager import infer_feature_type_convert_json
|
9
|
+
from .models import Run, Transform
|
10
|
+
|
11
|
+
P = ParamSpec("P")
|
12
|
+
R = TypeVar("R")
|
13
|
+
|
14
|
+
# Create a context variable to store the current tracked run
|
15
|
+
current_tracked_run: ContextVar[Run | None] = ContextVar(
|
16
|
+
"current_tracked_run", default=None
|
17
|
+
)
|
18
|
+
|
19
|
+
|
20
|
+
def get_current_tracked_run() -> Run | None:
|
21
|
+
"""Get the run object."""
|
22
|
+
run = current_tracked_run.get()
|
23
|
+
if run is None:
|
24
|
+
run = context.run
|
25
|
+
return run
|
26
|
+
|
27
|
+
|
28
|
+
def tracked(uid: str | None = None) -> Callable[[Callable[P, R]], Callable[P, R]]:
|
29
|
+
"""Decorator that tracks function execution.
|
30
|
+
|
31
|
+
Args:
|
32
|
+
uid: Optional unique identifier for the transform
|
33
|
+
"""
|
34
|
+
|
35
|
+
def decorator_tracked(func: Callable[P, R]) -> Callable[P, R]:
|
36
|
+
# Get the original signature
|
37
|
+
sig = inspect.signature(func)
|
38
|
+
|
39
|
+
@functools.wraps(func)
|
40
|
+
def wrapper_tracked(*args: P.args, **kwargs: P.kwargs) -> R:
|
41
|
+
# Get function metadata
|
42
|
+
source_code = inspect.getsource(func)
|
43
|
+
|
44
|
+
initiated_by_run = get_current_tracked_run()
|
45
|
+
if initiated_by_run is None:
|
46
|
+
if context.run is None:
|
47
|
+
raise RuntimeError(
|
48
|
+
"Please track the global run context before using @ln.tracked(): ln.track()"
|
49
|
+
)
|
50
|
+
initiated_by_run = context.run
|
51
|
+
|
52
|
+
# Get fully qualified function name
|
53
|
+
module_name = func.__module__
|
54
|
+
if module_name in {"__main__", "__mp_main__"}:
|
55
|
+
qualified_name = (
|
56
|
+
f"{initiated_by_run.transform.key}/{func.__qualname__}.py"
|
57
|
+
)
|
58
|
+
else:
|
59
|
+
qualified_name = f"{module_name}.{func.__qualname__}.py"
|
60
|
+
|
61
|
+
# Create transform and run objects
|
62
|
+
transform = Transform( # type: ignore
|
63
|
+
uid=uid,
|
64
|
+
key=qualified_name,
|
65
|
+
type="function",
|
66
|
+
source_code=source_code,
|
67
|
+
).save()
|
68
|
+
|
69
|
+
run = Run(transform=transform, initiated_by_run=initiated_by_run) # type: ignore
|
70
|
+
run.started_at = datetime.now(timezone.utc)
|
71
|
+
run.save()
|
72
|
+
|
73
|
+
# Bind arguments to get a mapping of parameter names to values
|
74
|
+
bound_args = sig.bind(*args, **kwargs)
|
75
|
+
bound_args.apply_defaults()
|
76
|
+
params = dict(bound_args.arguments)
|
77
|
+
|
78
|
+
# Remove the run parameter if it exists (we'll inject our own)
|
79
|
+
params.pop("run", None)
|
80
|
+
|
81
|
+
# Deal with non-trivial parameter values
|
82
|
+
filtered_params = {}
|
83
|
+
for key, value in params.items():
|
84
|
+
dtype, _, _ = infer_feature_type_convert_json(
|
85
|
+
key, value, str_as_ulabel=False
|
86
|
+
)
|
87
|
+
if (dtype == "?" or dtype.startswith("cat")) and dtype != "cat ? str":
|
88
|
+
continue
|
89
|
+
filtered_params[key] = value
|
90
|
+
|
91
|
+
# Add parameters to the run
|
92
|
+
run.params.add_values(filtered_params)
|
93
|
+
|
94
|
+
# Set the run in context and execute function
|
95
|
+
token = current_tracked_run.set(run)
|
96
|
+
try:
|
97
|
+
result = func(*args, **kwargs)
|
98
|
+
run.finished_at = datetime.now(timezone.utc)
|
99
|
+
run.save()
|
100
|
+
return result
|
101
|
+
finally:
|
102
|
+
current_tracked_run.reset(token)
|
103
|
+
|
104
|
+
return wrapper_tracked
|
105
|
+
|
106
|
+
return decorator_tracked
|
lamindb/_transform.py
CHANGED
@@ -5,14 +5,16 @@ from typing import TYPE_CHECKING
|
|
5
5
|
|
6
6
|
from lamin_utils import logger
|
7
7
|
from lamindb_setup.core._docs import doc_args
|
8
|
+
from lamindb_setup.core.hashing import hash_string
|
8
9
|
|
9
10
|
from lamindb.models import Run, Transform
|
10
11
|
|
11
12
|
from ._parents import _view_parents
|
13
|
+
from ._record import init_self_from_db, update_attributes
|
12
14
|
from ._run import delete_run_artifacts
|
13
15
|
from .core._settings import settings
|
14
|
-
from .core.exceptions import InconsistentKey
|
15
16
|
from .core.versioning import message_update_key_in_version_family, process_revises
|
17
|
+
from .errors import InconsistentKey
|
16
18
|
|
17
19
|
if TYPE_CHECKING:
|
18
20
|
from lamindb.base.types import TransformType
|
@@ -54,6 +56,9 @@ def __init__(transform: Transform, *args, **kwargs):
|
|
54
56
|
)
|
55
57
|
# below is internal use that we'll hopefully be able to eliminate
|
56
58
|
uid: str | None = kwargs.pop("uid") if "uid" in kwargs else None
|
59
|
+
source_code: str | None = (
|
60
|
+
kwargs.pop("source_code") if "source_code" in kwargs else None
|
61
|
+
)
|
57
62
|
if not len(kwargs) == 0:
|
58
63
|
raise ValueError(
|
59
64
|
"Only key, description, version, type, revises, reference, "
|
@@ -84,8 +89,6 @@ def __init__(transform: Transform, *args, **kwargs):
|
|
84
89
|
)
|
85
90
|
uid = revises.uid
|
86
91
|
if revises is not None and uid is not None and uid == revises.uid:
|
87
|
-
from ._record import init_self_from_db, update_attributes
|
88
|
-
|
89
92
|
if revises.key != key:
|
90
93
|
logger.warning("ignoring inconsistent key")
|
91
94
|
init_self_from_db(transform, revises)
|
@@ -111,7 +114,15 @@ def __init__(transform: Transform, *args, **kwargs):
|
|
111
114
|
uid = new_uid
|
112
115
|
else:
|
113
116
|
has_consciously_provided_uid = True
|
114
|
-
|
117
|
+
hash = None
|
118
|
+
if source_code is not None:
|
119
|
+
hash = hash_string(source_code)
|
120
|
+
transform_candidate = Transform.filter(hash=hash, is_latest=True).one_or_none()
|
121
|
+
if transform_candidate is not None:
|
122
|
+
init_self_from_db(transform, transform_candidate)
|
123
|
+
update_attributes(transform, {"key": key, "description": description})
|
124
|
+
return None
|
125
|
+
super(Transform, transform).__init__( # type: ignore
|
115
126
|
uid=uid,
|
116
127
|
description=description,
|
117
128
|
key=key,
|
@@ -119,6 +130,8 @@ def __init__(transform: Transform, *args, **kwargs):
|
|
119
130
|
version=version,
|
120
131
|
reference=reference,
|
121
132
|
reference_type=reference_type,
|
133
|
+
source_code=source_code,
|
134
|
+
hash=hash,
|
122
135
|
_has_consciously_provided_uid=has_consciously_provided_uid,
|
123
136
|
revises=revises,
|
124
137
|
)
|
@@ -151,7 +164,7 @@ def view_lineage(self, with_successors: bool = False, distance: int = 5):
|
|
151
164
|
)
|
152
165
|
|
153
166
|
|
154
|
-
Transform.__init__ = __init__
|
155
|
-
Transform.delete = delete
|
156
|
-
Transform.latest_run = latest_run
|
157
|
-
Transform.view_lineage = view_lineage
|
167
|
+
Transform.__init__ = __init__ # type: ignore
|
168
|
+
Transform.delete = delete # type: ignore
|
169
|
+
Transform.latest_run = latest_run # type: ignore
|
170
|
+
Transform.view_lineage = view_lineage # type: ignore
|
lamindb/_ulabel.py
CHANGED
@@ -1,8 +1,9 @@
|
|
1
1
|
from __future__ import annotations
|
2
2
|
|
3
3
|
import lamindb_setup as ln_setup
|
4
|
-
from lamin_utils import logger
|
5
4
|
|
5
|
+
from lamindb._record import _get_record_kwargs
|
6
|
+
from lamindb.errors import FieldValidationError
|
6
7
|
from lamindb.models import ULabel
|
7
8
|
|
8
9
|
from ._utils import attach_func_to_class_method
|
@@ -17,7 +18,7 @@ def __init__(self, *args, **kwargs):
|
|
17
18
|
raise ValueError("Only one non-keyword arg allowed")
|
18
19
|
name: str = kwargs.pop("name") if "name" in kwargs else None
|
19
20
|
type: str | None = kwargs.pop("type") if "type" in kwargs else None
|
20
|
-
is_type:
|
21
|
+
is_type: bool = kwargs.pop("is_type") if "is_type" in kwargs else False
|
21
22
|
description: str | None = (
|
22
23
|
kwargs.pop("description") if "description" in kwargs else None
|
23
24
|
)
|
@@ -26,18 +27,8 @@ def __init__(self, *args, **kwargs):
|
|
26
27
|
kwargs.pop("reference_type") if "reference_type" in kwargs else None
|
27
28
|
)
|
28
29
|
if len(kwargs) > 0:
|
29
|
-
|
30
|
-
|
31
|
-
)
|
32
|
-
if is_type:
|
33
|
-
if name.endswith("s"):
|
34
|
-
logger.warning(
|
35
|
-
"`name` ends with 's', in case you're naming with plural, consider the singular for a type name"
|
36
|
-
)
|
37
|
-
if name[0].islower():
|
38
|
-
logger.warning(
|
39
|
-
"`name` starts with lowercase, in case you're naming a type, consider starting with uppercase"
|
40
|
-
)
|
30
|
+
valid_keywords = ", ".join([val[0] for val in _get_record_kwargs(ULabel)])
|
31
|
+
raise FieldValidationError(f"Only {valid_keywords} are valid keyword arguments")
|
41
32
|
super(ULabel, self).__init__(
|
42
33
|
name=name,
|
43
34
|
type=type,
|
lamindb/base/validation.py
CHANGED
@@ -2,16 +2,12 @@ from typing import TYPE_CHECKING, Literal, Union, get_args, get_origin, get_type
|
|
2
2
|
|
3
3
|
from lamin_utils import colors
|
4
4
|
|
5
|
+
from lamindb.errors import FieldValidationError
|
6
|
+
|
5
7
|
if TYPE_CHECKING:
|
6
8
|
from .models import Record
|
7
9
|
|
8
10
|
|
9
|
-
class FieldValidationError(SystemExit):
|
10
|
-
"""Field validation error."""
|
11
|
-
|
12
|
-
pass
|
13
|
-
|
14
|
-
|
15
11
|
def validate_literal_fields(record: "Record", kwargs) -> None:
|
16
12
|
"""Validate all Literal type fields in a record.
|
17
13
|
|
lamindb/core/__init__.py
CHANGED
@@ -10,7 +10,6 @@ Registries:
|
|
10
10
|
Registry
|
11
11
|
QuerySet
|
12
12
|
QueryManager
|
13
|
-
Schema
|
14
13
|
RecordList
|
15
14
|
FeatureManager
|
16
15
|
ParamManager
|
@@ -31,11 +30,11 @@ Curators:
|
|
31
30
|
.. autosummary::
|
32
31
|
:toctree: .
|
33
32
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
33
|
+
CatManager
|
34
|
+
DataFrameCatManager
|
35
|
+
AnnDataCatManager
|
36
|
+
MuDataCatManager
|
37
|
+
TiledbsomaCatManager
|
39
38
|
CurateLookup
|
40
39
|
|
41
40
|
Settings & context:
|
@@ -61,7 +60,6 @@ Modules:
|
|
61
60
|
loaders
|
62
61
|
datasets
|
63
62
|
storage
|
64
|
-
exceptions
|
65
63
|
subsettings
|
66
64
|
logger
|
67
65
|
|
@@ -75,12 +73,13 @@ from lamindb._query_set import QuerySet, RecordList
|
|
75
73
|
from lamindb.core._feature_manager import FeatureManager, ParamManager
|
76
74
|
from lamindb.core._label_manager import LabelManager
|
77
75
|
from lamindb.curators import (
|
78
|
-
|
79
|
-
|
76
|
+
AnnDataCatManager,
|
77
|
+
CatManager,
|
80
78
|
CurateLookup,
|
81
|
-
|
82
|
-
|
83
|
-
|
79
|
+
Curator,
|
80
|
+
DataFrameCatManager,
|
81
|
+
MuDataCatManager,
|
82
|
+
TiledbsomaCatManager,
|
84
83
|
)
|
85
84
|
from lamindb.models import (
|
86
85
|
BasicRecord,
|
@@ -91,13 +90,13 @@ from lamindb.models import (
|
|
91
90
|
ParamValue,
|
92
91
|
Record,
|
93
92
|
Registry,
|
94
|
-
Schema,
|
95
93
|
TracksRun,
|
96
94
|
TracksUpdates,
|
97
95
|
ValidateFields,
|
98
96
|
)
|
99
97
|
|
100
|
-
from
|
98
|
+
from .. import errors as exceptions
|
99
|
+
from . import _data, datasets, fields, loaders, subsettings, types
|
101
100
|
from ._context import Context
|
102
101
|
from ._mapped_collection import MappedCollection
|
103
102
|
from ._settings import Settings
|
lamindb/core/_context.py
CHANGED
@@ -19,14 +19,14 @@ from lamindb.base import ids
|
|
19
19
|
from lamindb.base.ids import base62_12
|
20
20
|
from lamindb.models import Run, Transform, format_field_value
|
21
21
|
|
22
|
-
from
|
23
|
-
from ._sync_git import get_transform_reference_from_git_repo
|
24
|
-
from ._track_environment import track_environment
|
25
|
-
from .exceptions import (
|
22
|
+
from ..errors import (
|
26
23
|
InconsistentKey,
|
27
24
|
TrackNotCalled,
|
28
25
|
UpdateContext,
|
29
26
|
)
|
27
|
+
from ._settings import settings
|
28
|
+
from ._sync_git import get_transform_reference_from_git_repo
|
29
|
+
from ._track_environment import track_environment
|
30
30
|
from .versioning import bump_version as bump_version_function
|
31
31
|
from .versioning import increment_base62, message_update_key_in_version_family
|
32
32
|
|
@@ -315,7 +315,7 @@ class Context:
|
|
315
315
|
description=description,
|
316
316
|
transform_ref=transform_ref,
|
317
317
|
transform_ref_type=transform_ref_type,
|
318
|
-
transform_type=transform_type,
|
318
|
+
transform_type=transform_type, # type: ignore
|
319
319
|
)
|
320
320
|
else:
|
321
321
|
if transform.type in {"notebook", "script"}:
|
@@ -352,7 +352,7 @@ class Context:
|
|
352
352
|
self._logging_message_track += f", re-started Run('{run.uid[:8]}...') at {format_field_value(run.started_at)}"
|
353
353
|
|
354
354
|
if run is None: # create new run
|
355
|
-
run = Run(
|
355
|
+
run = Run( # type: ignore
|
356
356
|
transform=self._transform,
|
357
357
|
params=params,
|
358
358
|
)
|
@@ -587,7 +587,7 @@ class Context:
|
|
587
587
|
assert key is not None # noqa: S101
|
588
588
|
raise_update_context = False
|
589
589
|
try:
|
590
|
-
transform = Transform(
|
590
|
+
transform = Transform( # type: ignore
|
591
591
|
uid=self.uid,
|
592
592
|
version=self.version,
|
593
593
|
description=description,
|