lamindb 1.1.1__py3-none-any.whl → 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lamindb/__init__.py +30 -25
- lamindb/_tracked.py +1 -1
- lamindb/_view.py +2 -3
- lamindb/base/__init__.py +1 -1
- lamindb/base/ids.py +1 -10
- lamindb/core/__init__.py +7 -65
- lamindb/core/_compat.py +60 -0
- lamindb/core/_context.py +43 -20
- lamindb/core/_settings.py +6 -6
- lamindb/core/_sync_git.py +1 -1
- lamindb/core/loaders.py +30 -19
- lamindb/core/storage/_backed_access.py +4 -2
- lamindb/core/storage/_tiledbsoma.py +8 -6
- lamindb/core/storage/_zarr.py +104 -25
- lamindb/core/storage/objects.py +63 -28
- lamindb/core/storage/paths.py +4 -1
- lamindb/core/types.py +10 -0
- lamindb/curators/__init__.py +100 -85
- lamindb/errors.py +1 -1
- lamindb/integrations/_vitessce.py +4 -4
- lamindb/migrations/0089_subsequent_runs.py +159 -0
- lamindb/migrations/0090_runproject_project_runs.py +73 -0
- lamindb/migrations/{0088_squashed.py → 0090_squashed.py} +245 -177
- lamindb/models/__init__.py +79 -0
- lamindb/{core → models}/_describe.py +3 -3
- lamindb/{core → models}/_django.py +8 -5
- lamindb/{core → models}/_feature_manager.py +103 -87
- lamindb/{_from_values.py → models/_from_values.py} +5 -2
- lamindb/{core/versioning.py → models/_is_versioned.py} +94 -6
- lamindb/{core → models}/_label_manager.py +10 -17
- lamindb/{core/relations.py → models/_relations.py} +8 -1
- lamindb/models/artifact.py +2602 -0
- lamindb/{_can_curate.py → models/can_curate.py} +349 -180
- lamindb/models/collection.py +683 -0
- lamindb/models/core.py +135 -0
- lamindb/models/feature.py +643 -0
- lamindb/models/flextable.py +163 -0
- lamindb/{_parents.py → models/has_parents.py} +55 -49
- lamindb/models/project.py +384 -0
- lamindb/{_query_manager.py → models/query_manager.py} +10 -8
- lamindb/{_query_set.py → models/query_set.py} +40 -26
- lamindb/models/record.py +1762 -0
- lamindb/models/run.py +563 -0
- lamindb/{_save.py → models/save.py} +9 -7
- lamindb/models/schema.py +732 -0
- lamindb/models/transform.py +360 -0
- lamindb/models/ulabel.py +249 -0
- {lamindb-1.1.1.dist-info → lamindb-1.2.0.dist-info}/METADATA +6 -6
- {lamindb-1.1.1.dist-info → lamindb-1.2.0.dist-info}/RECORD +51 -51
- lamindb/_artifact.py +0 -1379
- lamindb/_collection.py +0 -440
- lamindb/_feature.py +0 -316
- lamindb/_is_versioned.py +0 -40
- lamindb/_record.py +0 -1064
- lamindb/_run.py +0 -60
- lamindb/_schema.py +0 -347
- lamindb/_storage.py +0 -15
- lamindb/_transform.py +0 -170
- lamindb/_ulabel.py +0 -56
- lamindb/_utils.py +0 -9
- lamindb/base/validation.py +0 -63
- lamindb/core/_data.py +0 -491
- lamindb/core/fields.py +0 -12
- lamindb/models.py +0 -4475
- {lamindb-1.1.1.dist-info → lamindb-1.2.0.dist-info}/LICENSE +0 -0
- {lamindb-1.1.1.dist-info → lamindb-1.2.0.dist-info}/WHEEL +0 -0
lamindb/models/run.py
ADDED
@@ -0,0 +1,563 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
from typing import TYPE_CHECKING, Any, overload
|
4
|
+
|
5
|
+
from django.db import models
|
6
|
+
from django.db.models import (
|
7
|
+
CASCADE,
|
8
|
+
PROTECT,
|
9
|
+
Q,
|
10
|
+
)
|
11
|
+
from django.db.utils import IntegrityError
|
12
|
+
from lamindb_setup import _check_instance_setup
|
13
|
+
from lamindb_setup.core.hashing import HASH_LENGTH, hash_dict
|
14
|
+
|
15
|
+
from lamindb.base.fields import (
|
16
|
+
BooleanField,
|
17
|
+
CharField,
|
18
|
+
DateTimeField,
|
19
|
+
ForeignKey,
|
20
|
+
)
|
21
|
+
from lamindb.base.users import current_user_id
|
22
|
+
from lamindb.errors import ValidationError
|
23
|
+
|
24
|
+
from ..base.ids import base62_20
|
25
|
+
from .can_curate import CanCurate
|
26
|
+
from .record import BasicRecord, LinkORM, Record
|
27
|
+
|
28
|
+
if TYPE_CHECKING:
|
29
|
+
from datetime import datetime
|
30
|
+
|
31
|
+
from .artifact import Artifact
|
32
|
+
from .collection import Collection
|
33
|
+
from .project import Project
|
34
|
+
from .schema import Schema
|
35
|
+
from .transform import Transform
|
36
|
+
from .ulabel import ULabel
|
37
|
+
|
38
|
+
|
39
|
+
_TRACKING_READY: bool | None = None
|
40
|
+
|
41
|
+
|
42
|
+
class ParamManager:
|
43
|
+
"""Param manager."""
|
44
|
+
|
45
|
+
pass
|
46
|
+
|
47
|
+
|
48
|
+
class ParamManagerRun(ParamManager):
|
49
|
+
"""Param manager."""
|
50
|
+
|
51
|
+
pass
|
52
|
+
|
53
|
+
|
54
|
+
def current_run() -> Run | None:
|
55
|
+
global _TRACKING_READY
|
56
|
+
|
57
|
+
if not _TRACKING_READY:
|
58
|
+
_TRACKING_READY = _check_instance_setup()
|
59
|
+
if _TRACKING_READY:
|
60
|
+
import lamindb
|
61
|
+
|
62
|
+
# also see get_run() in core._data
|
63
|
+
run = lamindb._tracked.get_current_tracked_run()
|
64
|
+
if run is None:
|
65
|
+
run = lamindb.context.run
|
66
|
+
return run
|
67
|
+
else:
|
68
|
+
return None
|
69
|
+
|
70
|
+
|
71
|
+
class TracksRun(models.Model):
|
72
|
+
"""Base class tracking latest run, creating user, and `created_at` timestamp."""
|
73
|
+
|
74
|
+
class Meta:
|
75
|
+
abstract = True
|
76
|
+
|
77
|
+
created_at: datetime = DateTimeField(
|
78
|
+
editable=False, db_default=models.functions.Now(), db_index=True
|
79
|
+
)
|
80
|
+
"""Time of creation of record."""
|
81
|
+
created_by: User = ForeignKey(
|
82
|
+
"lamindb.User",
|
83
|
+
PROTECT,
|
84
|
+
editable=False,
|
85
|
+
default=current_user_id,
|
86
|
+
related_name="+",
|
87
|
+
)
|
88
|
+
"""Creator of record."""
|
89
|
+
run: Run | None = ForeignKey(
|
90
|
+
"lamindb.Run", PROTECT, null=True, default=current_run, related_name="+"
|
91
|
+
)
|
92
|
+
"""Run that created record."""
|
93
|
+
|
94
|
+
@overload
|
95
|
+
def __init__(self): ...
|
96
|
+
|
97
|
+
@overload
|
98
|
+
def __init__(
|
99
|
+
self,
|
100
|
+
*db_args,
|
101
|
+
): ...
|
102
|
+
|
103
|
+
def __init__(
|
104
|
+
self,
|
105
|
+
*args,
|
106
|
+
**kwargs,
|
107
|
+
):
|
108
|
+
super().__init__(*args, **kwargs)
|
109
|
+
|
110
|
+
|
111
|
+
class TracksUpdates(models.Model):
|
112
|
+
"""Base class tracking previous runs and `updated_at` timestamp."""
|
113
|
+
|
114
|
+
class Meta:
|
115
|
+
abstract = True
|
116
|
+
|
117
|
+
updated_at: datetime = DateTimeField(
|
118
|
+
editable=False, db_default=models.functions.Now(), db_index=True
|
119
|
+
)
|
120
|
+
"""Time of last update to record."""
|
121
|
+
|
122
|
+
@overload
|
123
|
+
def __init__(self): ...
|
124
|
+
|
125
|
+
@overload
|
126
|
+
def __init__(
|
127
|
+
self,
|
128
|
+
*db_args,
|
129
|
+
): ...
|
130
|
+
|
131
|
+
def __init__(
|
132
|
+
self,
|
133
|
+
*args,
|
134
|
+
**kwargs,
|
135
|
+
):
|
136
|
+
super().__init__(*args, **kwargs)
|
137
|
+
|
138
|
+
|
139
|
+
class User(BasicRecord, CanCurate):
|
140
|
+
"""Users.
|
141
|
+
|
142
|
+
All data in this registry is synced from `lamin.ai` to ensure a universal
|
143
|
+
user identity. There is no need to manually create records.
|
144
|
+
|
145
|
+
Examples:
|
146
|
+
|
147
|
+
Query a user by handle:
|
148
|
+
|
149
|
+
>>> user = ln.User.get(handle="testuser1")
|
150
|
+
>>> user
|
151
|
+
"""
|
152
|
+
|
153
|
+
_name_field: str = "handle"
|
154
|
+
|
155
|
+
id: int = models.AutoField(primary_key=True)
|
156
|
+
"""Internal id, valid only in one DB instance."""
|
157
|
+
uid: str = CharField(editable=False, unique=True, db_index=True, max_length=8)
|
158
|
+
"""Universal id, valid across DB instances."""
|
159
|
+
handle: str = CharField(max_length=30, unique=True, db_index=True)
|
160
|
+
"""User handle, valid across DB instances (required)."""
|
161
|
+
name: str | None = CharField(max_length=150, db_index=True, null=True)
|
162
|
+
"""Full name (optional).""" # has to match hub specification, where it's also optional
|
163
|
+
created_artifacts: Artifact
|
164
|
+
"""Artifacts created by user."""
|
165
|
+
created_transforms: Transform
|
166
|
+
"""Transforms created by user."""
|
167
|
+
created_runs: Run
|
168
|
+
"""Runs created by user."""
|
169
|
+
created_at: datetime = DateTimeField(
|
170
|
+
editable=False, db_default=models.functions.Now(), db_index=True
|
171
|
+
)
|
172
|
+
"""Time of creation of record."""
|
173
|
+
updated_at: datetime = DateTimeField(
|
174
|
+
editable=False, db_default=models.functions.Now(), db_index=True
|
175
|
+
)
|
176
|
+
"""Time of last update to record."""
|
177
|
+
|
178
|
+
@overload
|
179
|
+
def __init__(
|
180
|
+
self,
|
181
|
+
handle: str,
|
182
|
+
email: str,
|
183
|
+
name: str | None,
|
184
|
+
): ...
|
185
|
+
|
186
|
+
@overload
|
187
|
+
def __init__(
|
188
|
+
self,
|
189
|
+
*db_args,
|
190
|
+
): ...
|
191
|
+
|
192
|
+
def __init__(
|
193
|
+
self,
|
194
|
+
*args,
|
195
|
+
**kwargs,
|
196
|
+
):
|
197
|
+
super().__init__(*args, **kwargs)
|
198
|
+
|
199
|
+
|
200
|
+
class Param(Record, CanCurate, TracksRun, TracksUpdates):
|
201
|
+
"""Parameters of runs & models."""
|
202
|
+
|
203
|
+
class Meta(Record.Meta, TracksRun.Meta, TracksUpdates.Meta):
|
204
|
+
abstract = False
|
205
|
+
|
206
|
+
_name_field: str = "name"
|
207
|
+
|
208
|
+
name: str = CharField(max_length=100, db_index=True)
|
209
|
+
dtype: str | None = CharField(db_index=True, null=True)
|
210
|
+
"""Data type ("num", "cat", "int", "float", "bool", "datetime").
|
211
|
+
|
212
|
+
For categorical types, can define from which registry values are
|
213
|
+
sampled, e.g., `cat[ULabel]` or `cat[bionty.CellType]`.
|
214
|
+
"""
|
215
|
+
type: Param | None = ForeignKey("self", PROTECT, null=True, related_name="records")
|
216
|
+
"""Type of param (e.g., 'Pipeline', 'ModelTraining', 'PostProcessing').
|
217
|
+
|
218
|
+
Allows to group features by type, e.g., all read outs, all metrics, etc.
|
219
|
+
"""
|
220
|
+
records: Param
|
221
|
+
"""Records of this type."""
|
222
|
+
is_type: bool = BooleanField(default=False, db_index=True, null=True)
|
223
|
+
"""Distinguish types from instances of the type."""
|
224
|
+
_expect_many: bool = models.BooleanField(default=False, db_default=False)
|
225
|
+
"""Indicates whether values for this param are expected to occur a single or multiple times for an artifact/run (default `False`).
|
226
|
+
|
227
|
+
- if it's `False` (default), the values mean artifact/run-level values and a dtype of `datetime` means `datetime`
|
228
|
+
- if it's `True`, the values are from an aggregation, which this seems like an edge case but when characterizing a model ensemble trained with different parameters it could be relevant
|
229
|
+
"""
|
230
|
+
schemas: Schema = models.ManyToManyField(
|
231
|
+
"Schema", through="SchemaParam", related_name="params"
|
232
|
+
)
|
233
|
+
"""Feature sets linked to this feature."""
|
234
|
+
# backward fields
|
235
|
+
values: ParamValue
|
236
|
+
"""Values for this parameter."""
|
237
|
+
|
238
|
+
def __init__(self, *args, **kwargs):
|
239
|
+
from .feature import process_init_feature_param
|
240
|
+
|
241
|
+
if len(args) == len(self._meta.concrete_fields):
|
242
|
+
super().__init__(*args, **kwargs)
|
243
|
+
return None
|
244
|
+
|
245
|
+
dtype = kwargs.get("dtype", None)
|
246
|
+
kwargs = process_init_feature_param(args, kwargs, is_param=True)
|
247
|
+
super().__init__(*args, **kwargs)
|
248
|
+
dtype_str = kwargs.pop("dtype", None)
|
249
|
+
if not self._state.adding:
|
250
|
+
if not (
|
251
|
+
self.dtype.startswith("cat")
|
252
|
+
if dtype == "cat"
|
253
|
+
else self.dtype == dtype_str
|
254
|
+
):
|
255
|
+
raise ValidationError(
|
256
|
+
f"Feature {self.name} already exists with dtype {self.dtype}, you passed {dtype_str}"
|
257
|
+
)
|
258
|
+
|
259
|
+
|
260
|
+
# FeatureValue behaves in many ways like a link in a LinkORM
|
261
|
+
# in particular, we don't want a _public field on it
|
262
|
+
# Also, we don't inherit from TracksRun because a ParamValue
|
263
|
+
# is typically created before a run is created and we want to
|
264
|
+
# avoid delete cycles (for Model params though it might be helpful)
|
265
|
+
class ParamValue(Record):
|
266
|
+
"""Parameter values.
|
267
|
+
|
268
|
+
Is largely analogous to `FeatureValue`.
|
269
|
+
"""
|
270
|
+
|
271
|
+
# we do not have a unique constraint on param & value because it leads to hashing errors
|
272
|
+
# for large dictionaries: https://lamin.ai/laminlabs/lamindata/transform/jgTrkoeuxAfs0000
|
273
|
+
# we do not hash values because we have `get_or_create` logic all over the place
|
274
|
+
# and also for checking whether the (param, value) combination exists
|
275
|
+
# there does not seem an issue with querying for a dict-like value
|
276
|
+
# https://lamin.ai/laminlabs/lamindata/transform/jgTrkoeuxAfs0001
|
277
|
+
_name_field: str = "value"
|
278
|
+
|
279
|
+
param: Param = ForeignKey(Param, CASCADE, related_name="values")
|
280
|
+
"""The dimension metadata."""
|
281
|
+
value: Any = (
|
282
|
+
models.JSONField()
|
283
|
+
) # stores float, integer, boolean, datetime or dictionaries
|
284
|
+
"""The JSON-like value."""
|
285
|
+
# it'd be confusing and hard to populate a run here because these
|
286
|
+
# values are typically created upon creating a run
|
287
|
+
# hence, ParamValue does _not_ inherit from TracksRun but manually
|
288
|
+
# adds created_at & created_by
|
289
|
+
# because ParamValue cannot be updated, we don't need updated_at
|
290
|
+
created_at: datetime = DateTimeField(
|
291
|
+
editable=False, db_default=models.functions.Now(), db_index=True
|
292
|
+
)
|
293
|
+
"""Time of creation of record."""
|
294
|
+
created_by: User = ForeignKey(
|
295
|
+
User, PROTECT, default=current_user_id, related_name="+"
|
296
|
+
)
|
297
|
+
"""Creator of record."""
|
298
|
+
hash: str = CharField(max_length=HASH_LENGTH, null=True, db_index=True)
|
299
|
+
|
300
|
+
class Meta:
|
301
|
+
constraints = [
|
302
|
+
# For simple types, use direct value comparison
|
303
|
+
models.UniqueConstraint(
|
304
|
+
fields=["param", "value"],
|
305
|
+
name="unique_simple_param_value",
|
306
|
+
condition=Q(hash__isnull=True),
|
307
|
+
),
|
308
|
+
# For complex types (dictionaries), use hash
|
309
|
+
models.UniqueConstraint(
|
310
|
+
fields=["param", "hash"],
|
311
|
+
name="unique_complex_param_value",
|
312
|
+
condition=Q(hash__isnull=False),
|
313
|
+
),
|
314
|
+
]
|
315
|
+
|
316
|
+
@classmethod
|
317
|
+
def get_or_create(cls, param, value):
|
318
|
+
# Simple types: int, float, str, bool
|
319
|
+
if isinstance(value, (int, float, str, bool)):
|
320
|
+
try:
|
321
|
+
return cls.objects.create(param=param, value=value, hash=None), False
|
322
|
+
except IntegrityError:
|
323
|
+
return cls.objects.get(param=param, value=value), True
|
324
|
+
|
325
|
+
# Complex types: dict, list
|
326
|
+
else:
|
327
|
+
hash = hash_dict(value)
|
328
|
+
try:
|
329
|
+
return cls.objects.create(param=param, value=value, hash=hash), False
|
330
|
+
except IntegrityError:
|
331
|
+
return cls.objects.get(param=param, hash=hash), True
|
332
|
+
|
333
|
+
|
334
|
+
class Run(Record):
|
335
|
+
"""Runs.
|
336
|
+
|
337
|
+
A registry to store runs of transforms, such as an executation of a script.
|
338
|
+
|
339
|
+
Args:
|
340
|
+
transform: `Transform` A :class:`~lamindb.Transform` record.
|
341
|
+
reference: `str | None = None` For instance, an external ID or a download URL.
|
342
|
+
reference_type: `str | None = None` For instance, `redun_id`, `nextflow_id` or `url`.
|
343
|
+
|
344
|
+
See Also:
|
345
|
+
:meth:`~lamindb.core.Context.track`
|
346
|
+
Track global runs & transforms for a notebook or script.
|
347
|
+
|
348
|
+
Examples:
|
349
|
+
|
350
|
+
Create a run record:
|
351
|
+
|
352
|
+
>>> ln.Transform(key="Cell Ranger", version="7.2.0", type="pipeline").save()
|
353
|
+
>>> transform = ln.Transform.get(key="Cell Ranger", version="7.2.0")
|
354
|
+
>>> run = ln.Run(transform)
|
355
|
+
|
356
|
+
Create a global run context for a custom transform:
|
357
|
+
|
358
|
+
>>> ln.track(transform=transform)
|
359
|
+
>>> ln.context.run # globally available run
|
360
|
+
|
361
|
+
Track a global run context for a notebook or script:
|
362
|
+
|
363
|
+
>>> ln.track() # Jupyter notebook metadata is automatically parsed
|
364
|
+
>>> ln.context.run
|
365
|
+
"""
|
366
|
+
|
367
|
+
_name_field: str = "started_at"
|
368
|
+
|
369
|
+
params: ParamManager = ParamManagerRun # type: ignore
|
370
|
+
"""Param manager.
|
371
|
+
|
372
|
+
Guide: :ref:`track-run-parameters`
|
373
|
+
|
374
|
+
Example::
|
375
|
+
|
376
|
+
run.params.add_values({
|
377
|
+
"learning_rate": 0.01,
|
378
|
+
"input_dir": "s3://my-bucket/mydataset",
|
379
|
+
"downsample": True,
|
380
|
+
"preprocess_params": {
|
381
|
+
"normalization_type": "cool",
|
382
|
+
"subset_highlyvariable": True,
|
383
|
+
},
|
384
|
+
})
|
385
|
+
"""
|
386
|
+
|
387
|
+
id: int = models.BigAutoField(primary_key=True)
|
388
|
+
"""Internal id, valid only in one DB instance."""
|
389
|
+
uid: str = CharField(
|
390
|
+
editable=False, unique=True, db_index=True, max_length=20, default=base62_20
|
391
|
+
)
|
392
|
+
"""Universal id, valid across DB instances."""
|
393
|
+
name: str | None = CharField(max_length=150, null=True)
|
394
|
+
"""A name."""
|
395
|
+
transform: Transform = ForeignKey("Transform", CASCADE, related_name="runs")
|
396
|
+
"""The transform :class:`~lamindb.Transform` that is being run."""
|
397
|
+
started_at: datetime = DateTimeField(
|
398
|
+
editable=False, db_default=models.functions.Now(), db_index=True
|
399
|
+
)
|
400
|
+
"""Start time of run."""
|
401
|
+
finished_at: datetime | None = DateTimeField(db_index=True, null=True, default=None)
|
402
|
+
"""Finished time of run."""
|
403
|
+
# we don't want to make below a OneToOne because there could be the same trivial report
|
404
|
+
# generated for many different runs
|
405
|
+
report: Artifact | None = ForeignKey(
|
406
|
+
"Artifact", PROTECT, null=True, related_name="_report_of", default=None
|
407
|
+
)
|
408
|
+
"""Report of run, e.g.. n html file."""
|
409
|
+
_logfile: Artifact | None = ForeignKey(
|
410
|
+
"Artifact", PROTECT, null=True, related_name="_logfile_of", default=None
|
411
|
+
)
|
412
|
+
"""Report of run, e.g.. n html file."""
|
413
|
+
environment: Artifact | None = ForeignKey(
|
414
|
+
"Artifact", PROTECT, null=True, related_name="_environment_of", default=None
|
415
|
+
)
|
416
|
+
"""Computational environment for the run.
|
417
|
+
|
418
|
+
For instance, `Dockerfile`, `docker image`, `requirements.txt`, `environment.yml`, etc.
|
419
|
+
"""
|
420
|
+
input_artifacts: Artifact
|
421
|
+
"""The artifacts serving as input for this run.
|
422
|
+
|
423
|
+
Related accessor: :attr:`~lamindb.Artifact.input_of_runs`.
|
424
|
+
"""
|
425
|
+
output_artifacts: Artifact
|
426
|
+
"""The artifacts generated by this run.
|
427
|
+
|
428
|
+
Related accessor: via :attr:`~lamindb.Artifact.run`
|
429
|
+
"""
|
430
|
+
input_collections: Collection
|
431
|
+
"""The collections serving as input for this run."""
|
432
|
+
output_collections: Collection
|
433
|
+
"""The collections generated by this run."""
|
434
|
+
_param_values: ParamValue = models.ManyToManyField(
|
435
|
+
ParamValue, through="RunParamValue", related_name="runs"
|
436
|
+
)
|
437
|
+
"""Parameter values."""
|
438
|
+
reference: str | None = CharField(max_length=255, db_index=True, null=True)
|
439
|
+
"""A reference like a URL or external ID (such as from a workflow manager)."""
|
440
|
+
reference_type: str | None = CharField(max_length=25, db_index=True, null=True)
|
441
|
+
"""Type of reference such as a workflow manager execution ID."""
|
442
|
+
created_at: datetime = DateTimeField(
|
443
|
+
editable=False, db_default=models.functions.Now(), db_index=True
|
444
|
+
)
|
445
|
+
"""Time of first creation. Mismatches ``started_at`` if the run is re-run."""
|
446
|
+
created_by: User = ForeignKey(
|
447
|
+
"User", CASCADE, default=current_user_id, related_name="created_runs"
|
448
|
+
)
|
449
|
+
"""Creator of run."""
|
450
|
+
ulabels: ULabel = models.ManyToManyField(
|
451
|
+
"ULabel", through="RunULabel", related_name="runs"
|
452
|
+
)
|
453
|
+
"""ULabel annotations of this transform."""
|
454
|
+
initiated_by_run: Run | None = ForeignKey(
|
455
|
+
"Run", CASCADE, null=True, related_name="initiated_runs", default=None
|
456
|
+
)
|
457
|
+
"""The run that triggered the current run.
|
458
|
+
|
459
|
+
This is not a preceding run. The preceding runs ("predecessors") is the set
|
460
|
+
of runs that produced the output artifacts that serve as the inputs for the
|
461
|
+
present run.
|
462
|
+
"""
|
463
|
+
initiated_runs: Run
|
464
|
+
"""Runs that were initiated by this run."""
|
465
|
+
projects: Project
|
466
|
+
"""Linked projects."""
|
467
|
+
_is_consecutive: bool | None = BooleanField(null=True)
|
468
|
+
"""Indicates whether code was consecutively executed. Is relevant for notebooks."""
|
469
|
+
_status_code: int = models.SmallIntegerField(default=0, db_index=True)
|
470
|
+
"""Status code of the run.
|
471
|
+
|
472
|
+
- 0: scheduled
|
473
|
+
- 1: started
|
474
|
+
- 2: errored
|
475
|
+
- 3: aborted
|
476
|
+
- 4: completed
|
477
|
+
"""
|
478
|
+
|
479
|
+
@overload
|
480
|
+
def __init__(
|
481
|
+
self,
|
482
|
+
transform: Transform,
|
483
|
+
reference: str | None = None,
|
484
|
+
reference_type: str | None = None,
|
485
|
+
): ...
|
486
|
+
|
487
|
+
@overload
|
488
|
+
def __init__(
|
489
|
+
self,
|
490
|
+
*db_args,
|
491
|
+
): ...
|
492
|
+
|
493
|
+
def __init__(
|
494
|
+
self,
|
495
|
+
*args,
|
496
|
+
**kwargs,
|
497
|
+
):
|
498
|
+
self.params = ParamManager(self) # type: ignore
|
499
|
+
if len(args) == len(self._meta.concrete_fields):
|
500
|
+
super().__init__(*args, **kwargs)
|
501
|
+
return None
|
502
|
+
# now we proceed with the user-facing constructor
|
503
|
+
if len(args) > 1:
|
504
|
+
raise ValueError("Only one non-keyword arg allowed: transform")
|
505
|
+
transform: Transform = None
|
506
|
+
if "transform" in kwargs or len(args) == 1:
|
507
|
+
transform = kwargs.pop("transform") if len(args) == 0 else args[0]
|
508
|
+
reference: str | None = kwargs.pop("reference", None)
|
509
|
+
reference_type: str | None = kwargs.pop("reference_type", None)
|
510
|
+
initiated_by_run: Run | None = kwargs.pop("initiated_by_run", None)
|
511
|
+
if transform is None:
|
512
|
+
raise TypeError("Pass transform parameter")
|
513
|
+
if transform._state.adding:
|
514
|
+
raise ValueError("Please save transform record before creating a run")
|
515
|
+
|
516
|
+
super().__init__( # type: ignore
|
517
|
+
transform=transform,
|
518
|
+
reference=reference,
|
519
|
+
initiated_by_run=initiated_by_run,
|
520
|
+
reference_type=reference_type,
|
521
|
+
)
|
522
|
+
|
523
|
+
def delete(self) -> None:
|
524
|
+
"""Delete."""
|
525
|
+
delete_run_artifacts(self)
|
526
|
+
super().delete()
|
527
|
+
|
528
|
+
|
529
|
+
def delete_run_artifacts(run: Run) -> None:
|
530
|
+
environment = None
|
531
|
+
if run.environment is not None:
|
532
|
+
environment = run.environment
|
533
|
+
run.environment = None
|
534
|
+
report = None
|
535
|
+
if run.report is not None:
|
536
|
+
report = run.report
|
537
|
+
run.report = None
|
538
|
+
if environment is not None or report is not None:
|
539
|
+
run.save()
|
540
|
+
if environment is not None:
|
541
|
+
# only delete if there are no other runs attached to this environment
|
542
|
+
if environment._environment_of.count() == 0:
|
543
|
+
environment.delete(permanent=True)
|
544
|
+
if report is not None:
|
545
|
+
report.delete(permanent=True)
|
546
|
+
|
547
|
+
|
548
|
+
class RunParamValue(BasicRecord, LinkORM):
|
549
|
+
id: int = models.BigAutoField(primary_key=True)
|
550
|
+
run: Run = ForeignKey(Run, CASCADE, related_name="+")
|
551
|
+
# we follow the lower() case convention rather than snake case for link models
|
552
|
+
paramvalue: ParamValue = ForeignKey(ParamValue, PROTECT, related_name="+")
|
553
|
+
created_at: datetime = DateTimeField(
|
554
|
+
editable=False, db_default=models.functions.Now(), db_index=True
|
555
|
+
)
|
556
|
+
"""Time of creation of record."""
|
557
|
+
created_by: User = ForeignKey(
|
558
|
+
"lamindb.User", PROTECT, default=current_user_id, related_name="+"
|
559
|
+
)
|
560
|
+
"""Creator of record."""
|
561
|
+
|
562
|
+
class Meta:
|
563
|
+
unique_together = ("run", "paramvalue")
|
@@ -1,3 +1,4 @@
|
|
1
|
+
# ruff: noqa: TC004
|
1
2
|
from __future__ import annotations
|
2
3
|
|
3
4
|
import os
|
@@ -10,23 +11,22 @@ from typing import TYPE_CHECKING
|
|
10
11
|
from django.db import transaction
|
11
12
|
from django.utils.functional import partition
|
12
13
|
from lamin_utils import logger
|
13
|
-
from lamindb_setup.core.upath import LocalPathClasses
|
14
|
+
from lamindb_setup.core.upath import LocalPathClasses, UPath
|
14
15
|
|
15
|
-
from
|
16
|
-
|
17
|
-
from .core._settings import settings
|
18
|
-
from .core.storage.paths import (
|
16
|
+
from ..core._settings import settings
|
17
|
+
from ..core.storage.paths import (
|
19
18
|
_cache_key_from_artifact_storage,
|
20
19
|
attempt_accessing_path,
|
21
20
|
auto_storage_key_from_artifact,
|
22
21
|
delete_storage_using_key,
|
23
22
|
store_file_or_folder,
|
24
23
|
)
|
24
|
+
from .record import Record
|
25
25
|
|
26
26
|
if TYPE_CHECKING:
|
27
27
|
from collections.abc import Iterable
|
28
28
|
|
29
|
-
from
|
29
|
+
from .artifact import Artifact
|
30
30
|
|
31
31
|
|
32
32
|
def save(records: Iterable[Record], ignore_conflicts: bool | None = False) -> None:
|
@@ -42,7 +42,7 @@ def save(records: Iterable[Record], ignore_conflicts: bool | None = False) -> No
|
|
42
42
|
existing records! Use ``record.save()`` for these use cases.
|
43
43
|
|
44
44
|
Args:
|
45
|
-
records: Multiple :class:`~lamindb.
|
45
|
+
records: Multiple :class:`~lamindb.models.Record` objects.
|
46
46
|
ignore_conflicts: If ``True``, do not error if some records violate a
|
47
47
|
unique or another constraint. However, it won't inplace update the id
|
48
48
|
fields of records. If you need records with ids, you need to query
|
@@ -67,6 +67,8 @@ def save(records: Iterable[Record], ignore_conflicts: bool | None = False) -> No
|
|
67
67
|
>>> transform.save()
|
68
68
|
|
69
69
|
"""
|
70
|
+
from .artifact import Artifact
|
71
|
+
|
70
72
|
if isinstance(records, Record):
|
71
73
|
raise ValueError("Please use record.save() if saving a single record.")
|
72
74
|
|