lamindb 1.5.3__py3-none-any.whl → 1.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lamindb/__init__.py +25 -6
- lamindb/_finish.py +5 -5
- lamindb/_tracked.py +1 -1
- lamindb/_view.py +4 -4
- lamindb/core/_context.py +32 -6
- lamindb/core/_settings.py +1 -1
- lamindb/core/datasets/mini_immuno.py +8 -0
- lamindb/core/loaders.py +1 -1
- lamindb/core/storage/_anndata_accessor.py +9 -9
- lamindb/core/storage/_valid_suffixes.py +1 -0
- lamindb/core/storage/_zarr.py +32 -107
- lamindb/curators/__init__.py +19 -2
- lamindb/curators/_cellxgene_schemas/__init__.py +3 -3
- lamindb/curators/_legacy.py +15 -19
- lamindb/curators/core.py +247 -80
- lamindb/errors.py +2 -2
- lamindb/migrations/0069_squashed.py +8 -8
- lamindb/migrations/0071_lamindbv1_migrate_schema.py +3 -3
- lamindb/migrations/0073_merge_ourprojects.py +7 -7
- lamindb/migrations/0075_lamindbv1_part5.py +1 -1
- lamindb/migrations/0077_lamindbv1_part6b.py +3 -3
- lamindb/migrations/0080_polish_lamindbv1.py +2 -2
- lamindb/migrations/0088_schema_components.py +1 -1
- lamindb/migrations/0090_runproject_project_runs.py +2 -2
- lamindb/migrations/0091_alter_featurevalue_options_alter_space_options_and_more.py +1 -1
- lamindb/migrations/0094_writeloglock_writelogmigrationstate_and_more.py +84 -0
- lamindb/migrations/0095_remove_rundata_flextable.py +155 -0
- lamindb/migrations/0096_remove_artifact__param_values_and_more.py +266 -0
- lamindb/migrations/0097_remove_schemaparam_param_remove_paramvalue_param_and_more.py +27 -0
- lamindb/migrations/0098_alter_feature_type_alter_project_type_and_more.py +656 -0
- lamindb/migrations/0099_alter_writelog_seqno.py +22 -0
- lamindb/migrations/0100_branch_alter_artifact__branch_code_and_more.py +102 -0
- lamindb/migrations/0101_alter_artifact_hash_alter_feature_name_and_more.py +444 -0
- lamindb/migrations/0102_remove_writelog_branch_code_and_more.py +72 -0
- lamindb/migrations/0103_remove_writelog_migration_state_and_more.py +46 -0
- lamindb/migrations/{0090_squashed.py → 0103_squashed.py} +1013 -1009
- lamindb/models/__init__.py +35 -18
- lamindb/models/_describe.py +4 -4
- lamindb/models/_django.py +38 -4
- lamindb/models/_feature_manager.py +66 -123
- lamindb/models/_from_values.py +13 -13
- lamindb/models/_label_manager.py +8 -6
- lamindb/models/_relations.py +7 -7
- lamindb/models/artifact.py +166 -156
- lamindb/models/can_curate.py +25 -25
- lamindb/models/collection.py +48 -18
- lamindb/models/core.py +3 -3
- lamindb/models/feature.py +88 -60
- lamindb/models/has_parents.py +17 -17
- lamindb/models/project.py +52 -24
- lamindb/models/query_manager.py +5 -5
- lamindb/models/query_set.py +61 -37
- lamindb/models/record.py +158 -1583
- lamindb/models/run.py +39 -176
- lamindb/models/save.py +6 -6
- lamindb/models/schema.py +32 -43
- lamindb/models/sqlrecord.py +1743 -0
- lamindb/models/transform.py +17 -33
- lamindb/models/ulabel.py +21 -15
- {lamindb-1.5.3.dist-info → lamindb-1.6.0.dist-info}/METADATA +7 -11
- lamindb-1.6.0.dist-info/RECORD +118 -0
- lamindb/core/storage/_anndata_sizes.py +0 -41
- lamindb/models/flextable.py +0 -163
- lamindb-1.5.3.dist-info/RECORD +0 -109
- {lamindb-1.5.3.dist-info → lamindb-1.6.0.dist-info}/LICENSE +0 -0
- {lamindb-1.5.3.dist-info → lamindb-1.6.0.dist-info}/WHEEL +0 -0
@@ -326,7 +326,7 @@ class Migration(migrations.Migration):
|
|
326
326
|
WHERE dtype = 'number'
|
327
327
|
"""
|
328
328
|
),
|
329
|
-
# an aux field on
|
329
|
+
# an aux field on SQLRecord
|
330
330
|
migrations.AddField(
|
331
331
|
model_name="artifact",
|
332
332
|
name="aux",
|
@@ -547,7 +547,7 @@ class Migration(migrations.Migration):
|
|
547
547
|
options={
|
548
548
|
"unique_together": {("transform", "ulabel")},
|
549
549
|
},
|
550
|
-
bases=(lamindb.models.
|
550
|
+
bases=(lamindb.models.IsLink, models.Model),
|
551
551
|
),
|
552
552
|
migrations.RunSQL(
|
553
553
|
sql="""
|
@@ -670,7 +670,7 @@ class Migration(migrations.Migration):
|
|
670
670
|
old_name="n_objects",
|
671
671
|
new_name="n_files",
|
672
672
|
),
|
673
|
-
# let feature value and paramvalue inherit from
|
673
|
+
# let feature value and paramvalue inherit from SQLRecord
|
674
674
|
migrations.AddField(
|
675
675
|
model_name="featurevalue",
|
676
676
|
name="_branch_code",
|
@@ -173,7 +173,7 @@ def migrate_data(apps, schema_editor):
|
|
173
173
|
)
|
174
174
|
else:
|
175
175
|
cursor.execute("ROLLBACK")
|
176
|
-
raise Exception("Migration failed:
|
176
|
+
raise Exception("Migration failed: SQLRecord count mismatch")
|
177
177
|
|
178
178
|
except Exception as e:
|
179
179
|
cursor.execute("ROLLBACK")
|
@@ -248,7 +248,7 @@ class Migration(migrations.Migration):
|
|
248
248
|
),
|
249
249
|
),
|
250
250
|
],
|
251
|
-
bases=(lamindb.models.
|
251
|
+
bases=(lamindb.models.IsLink, models.Model),
|
252
252
|
),
|
253
253
|
migrations.CreateModel(
|
254
254
|
name="ArtifactReference",
|
@@ -312,7 +312,7 @@ class Migration(migrations.Migration):
|
|
312
312
|
),
|
313
313
|
),
|
314
314
|
],
|
315
|
-
bases=(lamindb.models.
|
315
|
+
bases=(lamindb.models.IsLink, models.Model),
|
316
316
|
),
|
317
317
|
migrations.CreateModel(
|
318
318
|
name="CollectionProject",
|
@@ -353,7 +353,7 @@ class Migration(migrations.Migration):
|
|
353
353
|
),
|
354
354
|
),
|
355
355
|
],
|
356
|
-
bases=(lamindb.models.
|
356
|
+
bases=(lamindb.models.IsLink, models.Model),
|
357
357
|
),
|
358
358
|
migrations.CreateModel(
|
359
359
|
name="CollectionReference",
|
@@ -394,7 +394,7 @@ class Migration(migrations.Migration):
|
|
394
394
|
),
|
395
395
|
),
|
396
396
|
],
|
397
|
-
bases=(lamindb.models.
|
397
|
+
bases=(lamindb.models.IsLink, models.Model),
|
398
398
|
),
|
399
399
|
migrations.CreateModel(
|
400
400
|
name="Person",
|
@@ -851,7 +851,7 @@ class Migration(migrations.Migration):
|
|
851
851
|
options={
|
852
852
|
"unique_together": {("transform", "project")},
|
853
853
|
},
|
854
|
-
bases=(lamindb.models.
|
854
|
+
bases=(lamindb.models.IsLink, models.Model),
|
855
855
|
),
|
856
856
|
migrations.AddField(
|
857
857
|
model_name="project",
|
@@ -913,7 +913,7 @@ class Migration(migrations.Migration):
|
|
913
913
|
options={
|
914
914
|
"unique_together": {("transform", "reference")},
|
915
915
|
},
|
916
|
-
bases=(lamindb.models.
|
916
|
+
bases=(lamindb.models.IsLink, models.Model),
|
917
917
|
),
|
918
918
|
migrations.AddField(
|
919
919
|
model_name="reference",
|
@@ -243,7 +243,7 @@ class Migration(migrations.Migration):
|
|
243
243
|
options={
|
244
244
|
"unique_together": {("schema", "param")},
|
245
245
|
},
|
246
|
-
bases=(models.Model, lamindb.models.
|
246
|
+
bases=(models.Model, lamindb.models.IsLink),
|
247
247
|
),
|
248
248
|
migrations.AddField(
|
249
249
|
model_name="param",
|
@@ -78,7 +78,7 @@ class Migration(migrations.Migration):
|
|
78
78
|
options={
|
79
79
|
"unique_together": {("feature", "project")},
|
80
80
|
},
|
81
|
-
bases=(lamindb.models.
|
81
|
+
bases=(lamindb.models.IsLink, models.Model),
|
82
82
|
),
|
83
83
|
migrations.AddField(
|
84
84
|
model_name="project",
|
@@ -146,7 +146,7 @@ class Migration(migrations.Migration):
|
|
146
146
|
options={
|
147
147
|
"unique_together": {("schema", "project")},
|
148
148
|
},
|
149
|
-
bases=(lamindb.models.
|
149
|
+
bases=(lamindb.models.IsLink, models.Model),
|
150
150
|
),
|
151
151
|
migrations.AddField(
|
152
152
|
model_name="project",
|
@@ -214,7 +214,7 @@ class Migration(migrations.Migration):
|
|
214
214
|
options={
|
215
215
|
"unique_together": {("ulabel", "project")},
|
216
216
|
},
|
217
|
-
bases=(lamindb.models.
|
217
|
+
bases=(lamindb.models.IsLink, models.Model),
|
218
218
|
),
|
219
219
|
migrations.AddField(
|
220
220
|
model_name="project",
|
@@ -277,7 +277,7 @@ class Migration(migrations.Migration):
|
|
277
277
|
options={
|
278
278
|
"unique_together": {("person", "project")},
|
279
279
|
},
|
280
|
-
bases=(lamindb.models.
|
280
|
+
bases=(lamindb.models.IsLink, models.Model),
|
281
281
|
),
|
282
282
|
migrations.AddField(
|
283
283
|
model_name="project",
|
@@ -481,7 +481,7 @@ class Migration(migrations.Migration):
|
|
481
481
|
options={
|
482
482
|
"unique_together": {("run", "ulabel")},
|
483
483
|
},
|
484
|
-
bases=(models.Model, lamindb.models.
|
484
|
+
bases=(models.Model, lamindb.models.IsLink),
|
485
485
|
),
|
486
486
|
migrations.AddField(
|
487
487
|
model_name="run",
|
@@ -97,7 +97,7 @@ class Migration(migrations.Migration):
|
|
97
97
|
options={
|
98
98
|
"unique_together": {("composite", "component")},
|
99
99
|
},
|
100
|
-
bases=(lamindb.models.
|
100
|
+
bases=(lamindb.models.IsLink, models.Model),
|
101
101
|
),
|
102
102
|
migrations.AddField(
|
103
103
|
model_name="schema",
|
@@ -6,7 +6,7 @@ from django.db import migrations, models
|
|
6
6
|
|
7
7
|
import lamindb.base.fields
|
8
8
|
import lamindb.base.users
|
9
|
-
import lamindb.models.
|
9
|
+
import lamindb.models.sqlrecord
|
10
10
|
|
11
11
|
|
12
12
|
class Migration(migrations.Migration):
|
@@ -61,7 +61,7 @@ class Migration(migrations.Migration):
|
|
61
61
|
options={
|
62
62
|
"unique_together": {("run", "project")},
|
63
63
|
},
|
64
|
-
bases=(models.Model, lamindb.models.
|
64
|
+
bases=(models.Model, lamindb.models.sqlrecord.IsLink),
|
65
65
|
),
|
66
66
|
migrations.AddField(
|
67
67
|
model_name="project",
|
@@ -0,0 +1,84 @@
|
|
1
|
+
# Generated by Django 5.1.7 on 2025-05-10 00:32
|
2
|
+
|
3
|
+
import django.db.models.deletion
|
4
|
+
from django.db import migrations, models
|
5
|
+
|
6
|
+
|
7
|
+
class Migration(migrations.Migration):
|
8
|
+
dependencies = [
|
9
|
+
("lamindb", "0093_alter_schemacomponent_unique_together"),
|
10
|
+
]
|
11
|
+
|
12
|
+
operations = [
|
13
|
+
migrations.CreateModel(
|
14
|
+
name="WriteLogLock",
|
15
|
+
fields=[
|
16
|
+
(
|
17
|
+
"id",
|
18
|
+
models.BigAutoField(
|
19
|
+
auto_created=True,
|
20
|
+
primary_key=True,
|
21
|
+
serialize=False,
|
22
|
+
verbose_name="ID",
|
23
|
+
),
|
24
|
+
),
|
25
|
+
("locked", models.BooleanField()),
|
26
|
+
],
|
27
|
+
),
|
28
|
+
migrations.CreateModel(
|
29
|
+
name="MigrationState",
|
30
|
+
fields=[
|
31
|
+
("id", models.SmallAutoField(primary_key=True, serialize=False)),
|
32
|
+
("migration_state_id", models.JSONField()),
|
33
|
+
],
|
34
|
+
),
|
35
|
+
migrations.CreateModel(
|
36
|
+
name="TableState",
|
37
|
+
fields=[
|
38
|
+
("id", models.SmallAutoField(primary_key=True, serialize=False)),
|
39
|
+
("table_name", models.CharField(max_length=255)),
|
40
|
+
("backfilled", models.BooleanField()),
|
41
|
+
],
|
42
|
+
),
|
43
|
+
migrations.CreateModel(
|
44
|
+
name="WriteLog",
|
45
|
+
fields=[
|
46
|
+
("seqno", models.AutoField(primary_key=True, serialize=False)),
|
47
|
+
(
|
48
|
+
"uid",
|
49
|
+
models.CharField(
|
50
|
+
db_index=True, editable=False, max_length=18, unique=True
|
51
|
+
),
|
52
|
+
),
|
53
|
+
("space_uid", models.CharField(max_length=12, null=True)),
|
54
|
+
("created_by_uid", models.CharField(default="00000000", max_length=8)),
|
55
|
+
("branch_code", models.IntegerField(default=1)),
|
56
|
+
(
|
57
|
+
"run_uid",
|
58
|
+
models.CharField(default="0000000000000000", max_length=16),
|
59
|
+
),
|
60
|
+
("record_uid", models.JSONField(null=True)),
|
61
|
+
("record_data", models.JSONField(null=True)),
|
62
|
+
("event_type", models.PositiveSmallIntegerField()),
|
63
|
+
("created_at", models.DateTimeField()),
|
64
|
+
(
|
65
|
+
"migration_state",
|
66
|
+
models.ForeignKey(
|
67
|
+
on_delete=django.db.models.deletion.PROTECT,
|
68
|
+
to="lamindb.migrationstate",
|
69
|
+
),
|
70
|
+
),
|
71
|
+
(
|
72
|
+
"table",
|
73
|
+
models.ForeignKey(
|
74
|
+
on_delete=django.db.models.deletion.PROTECT,
|
75
|
+
to="lamindb.tablestate",
|
76
|
+
),
|
77
|
+
),
|
78
|
+
],
|
79
|
+
options={
|
80
|
+
"verbose_name": "Write Log",
|
81
|
+
"verbose_name_plural": "Write Logs",
|
82
|
+
},
|
83
|
+
),
|
84
|
+
]
|
@@ -0,0 +1,155 @@
|
|
1
|
+
# Generated by Django 5.2 on 2025-05-11 18:42
|
2
|
+
|
3
|
+
import django.db.models.deletion
|
4
|
+
import django.db.models.functions.datetime
|
5
|
+
from django.db import migrations, models
|
6
|
+
|
7
|
+
import lamindb.base.fields
|
8
|
+
import lamindb.base.users
|
9
|
+
import lamindb.models.sqlrecord
|
10
|
+
|
11
|
+
|
12
|
+
class Migration(migrations.Migration):
|
13
|
+
dependencies = [
|
14
|
+
("lamindb", "0094_writeloglock_writelogmigrationstate_and_more"),
|
15
|
+
]
|
16
|
+
|
17
|
+
operations = [
|
18
|
+
migrations.CreateModel(
|
19
|
+
name="RunFeatureValue",
|
20
|
+
fields=[
|
21
|
+
("id", models.BigAutoField(primary_key=True, serialize=False)),
|
22
|
+
(
|
23
|
+
"created_at",
|
24
|
+
lamindb.base.fields.DateTimeField(
|
25
|
+
blank=True,
|
26
|
+
db_default=django.db.models.functions.datetime.Now(),
|
27
|
+
db_index=True,
|
28
|
+
editable=False,
|
29
|
+
),
|
30
|
+
),
|
31
|
+
(
|
32
|
+
"created_by",
|
33
|
+
lamindb.base.fields.ForeignKey(
|
34
|
+
blank=True,
|
35
|
+
default=lamindb.base.users.current_user_id,
|
36
|
+
on_delete=django.db.models.deletion.PROTECT,
|
37
|
+
related_name="+",
|
38
|
+
to="lamindb.user",
|
39
|
+
),
|
40
|
+
),
|
41
|
+
(
|
42
|
+
"featurevalue",
|
43
|
+
lamindb.base.fields.ForeignKey(
|
44
|
+
blank=True,
|
45
|
+
on_delete=django.db.models.deletion.PROTECT,
|
46
|
+
related_name="links_run",
|
47
|
+
to="lamindb.featurevalue",
|
48
|
+
),
|
49
|
+
),
|
50
|
+
(
|
51
|
+
"run",
|
52
|
+
lamindb.base.fields.ForeignKey(
|
53
|
+
blank=True,
|
54
|
+
on_delete=django.db.models.deletion.CASCADE,
|
55
|
+
related_name="links_featurevalue",
|
56
|
+
to="lamindb.run",
|
57
|
+
),
|
58
|
+
),
|
59
|
+
],
|
60
|
+
options={
|
61
|
+
"unique_together": {("run", "featurevalue")},
|
62
|
+
},
|
63
|
+
bases=(models.Model, lamindb.models.sqlrecord.IsLink),
|
64
|
+
),
|
65
|
+
migrations.AddField(
|
66
|
+
model_name="run",
|
67
|
+
name="_feature_values",
|
68
|
+
field=models.ManyToManyField(
|
69
|
+
related_name="runs",
|
70
|
+
through="lamindb.RunFeatureValue",
|
71
|
+
to="lamindb.featurevalue",
|
72
|
+
),
|
73
|
+
),
|
74
|
+
migrations.RemoveField(
|
75
|
+
model_name="flextable",
|
76
|
+
name="created_by",
|
77
|
+
),
|
78
|
+
migrations.RemoveField(
|
79
|
+
model_name="flextable",
|
80
|
+
name="projects",
|
81
|
+
),
|
82
|
+
migrations.RemoveField(
|
83
|
+
model_name="flextable",
|
84
|
+
name="schema",
|
85
|
+
),
|
86
|
+
migrations.RemoveField(
|
87
|
+
model_name="flextable",
|
88
|
+
name="space",
|
89
|
+
),
|
90
|
+
migrations.RemoveField(
|
91
|
+
model_name="flextable",
|
92
|
+
name="type",
|
93
|
+
),
|
94
|
+
migrations.RemoveField(
|
95
|
+
model_name="flextable",
|
96
|
+
name="ulabels",
|
97
|
+
),
|
98
|
+
migrations.RemoveField(
|
99
|
+
model_name="flextabledata",
|
100
|
+
name="space",
|
101
|
+
),
|
102
|
+
migrations.RemoveField(
|
103
|
+
model_name="flextabledata",
|
104
|
+
name="value_artifact",
|
105
|
+
),
|
106
|
+
migrations.RemoveField(
|
107
|
+
model_name="flextabledata",
|
108
|
+
name="value_collection",
|
109
|
+
),
|
110
|
+
migrations.RemoveField(
|
111
|
+
model_name="flextabledata",
|
112
|
+
name="value_person",
|
113
|
+
),
|
114
|
+
migrations.RemoveField(
|
115
|
+
model_name="flextabledata",
|
116
|
+
name="value_project",
|
117
|
+
),
|
118
|
+
migrations.RemoveField(
|
119
|
+
model_name="flextabledata",
|
120
|
+
name="value_ulabel",
|
121
|
+
),
|
122
|
+
migrations.RemoveField(
|
123
|
+
model_name="rundata",
|
124
|
+
name="space",
|
125
|
+
),
|
126
|
+
migrations.RemoveField(
|
127
|
+
model_name="rundata",
|
128
|
+
name="value_artifact",
|
129
|
+
),
|
130
|
+
migrations.RemoveField(
|
131
|
+
model_name="rundata",
|
132
|
+
name="value_collection",
|
133
|
+
),
|
134
|
+
migrations.RemoveField(
|
135
|
+
model_name="rundata",
|
136
|
+
name="value_person",
|
137
|
+
),
|
138
|
+
migrations.RemoveField(
|
139
|
+
model_name="rundata",
|
140
|
+
name="value_project",
|
141
|
+
),
|
142
|
+
migrations.RemoveField(
|
143
|
+
model_name="rundata",
|
144
|
+
name="value_ulabel",
|
145
|
+
),
|
146
|
+
migrations.DeleteModel(
|
147
|
+
name="FlexTable",
|
148
|
+
),
|
149
|
+
migrations.DeleteModel(
|
150
|
+
name="FlexTableData",
|
151
|
+
),
|
152
|
+
migrations.DeleteModel(
|
153
|
+
name="RunData",
|
154
|
+
),
|
155
|
+
]
|
@@ -0,0 +1,266 @@
|
|
1
|
+
# Generated by Django 5.2 on 2025-05-11 18:50
|
2
|
+
|
3
|
+
import random
|
4
|
+
import string
|
5
|
+
|
6
|
+
from django.db import migrations
|
7
|
+
|
8
|
+
|
9
|
+
def generate_base62_uid():
|
10
|
+
"""Generate a 12-character base62 UID."""
|
11
|
+
charset = string.ascii_uppercase + string.ascii_lowercase + string.digits
|
12
|
+
return "".join(random.choice(charset) for _ in range(12))
|
13
|
+
|
14
|
+
|
15
|
+
def copy_params_to_features(apps, schema_editor):
|
16
|
+
"""Copy data from Param to Feature with generated UIDs."""
|
17
|
+
Param = apps.get_model("lamindb", "Param")
|
18
|
+
Feature = apps.get_model("lamindb", "Feature")
|
19
|
+
|
20
|
+
# Create features from params with generated UIDs
|
21
|
+
name_to_feature = {}
|
22
|
+
|
23
|
+
# First pass: create all Features
|
24
|
+
for param in Param.objects.all():
|
25
|
+
feature = Feature.objects.create(
|
26
|
+
uid=generate_base62_uid(),
|
27
|
+
name=param.name,
|
28
|
+
dtype=param.dtype,
|
29
|
+
is_type=param.is_type,
|
30
|
+
_expect_many=param._expect_many,
|
31
|
+
created_at=param.created_at,
|
32
|
+
updated_at=param.updated_at,
|
33
|
+
created_by_id=param.created_by_id,
|
34
|
+
run_id=param.run_id,
|
35
|
+
)
|
36
|
+
name_to_feature[param.name] = feature
|
37
|
+
|
38
|
+
# Second pass: set up type relationships
|
39
|
+
for param in Param.objects.all():
|
40
|
+
if param.type_id is not None:
|
41
|
+
try:
|
42
|
+
# Find the parameter that is the type
|
43
|
+
type_param = Param.objects.get(id=param.type_id)
|
44
|
+
|
45
|
+
# Set the type relationship
|
46
|
+
feature = name_to_feature[param.name]
|
47
|
+
feature.type = name_to_feature[type_param.name]
|
48
|
+
feature.save()
|
49
|
+
except (Param.DoesNotExist, KeyError):
|
50
|
+
# Handle case where the referenced type doesn't exist
|
51
|
+
print(f"Warning: Could not set type for feature {param.name}")
|
52
|
+
|
53
|
+
|
54
|
+
def copy_param_values(apps, schema_editor):
|
55
|
+
"""Copy data from ParamValue to FeatureValue."""
|
56
|
+
Param = apps.get_model("lamindb", "Param")
|
57
|
+
Feature = apps.get_model("lamindb", "Feature")
|
58
|
+
ParamValue = apps.get_model("lamindb", "ParamValue")
|
59
|
+
FeatureValue = apps.get_model("lamindb", "FeatureValue")
|
60
|
+
|
61
|
+
# Create a mapping from param ID to feature ID
|
62
|
+
param_to_feature = {
|
63
|
+
param.id: Feature.objects.get(name=param.name).id
|
64
|
+
for param in Param.objects.all()
|
65
|
+
}
|
66
|
+
|
67
|
+
# Create FeatureValues from ParamValues
|
68
|
+
pv_to_fv = {}
|
69
|
+
|
70
|
+
for pv in ParamValue.objects.all():
|
71
|
+
try:
|
72
|
+
feature_id = param_to_feature[pv.param_id]
|
73
|
+
|
74
|
+
# Create the FeatureValue
|
75
|
+
fv = FeatureValue.objects.create(
|
76
|
+
feature_id=feature_id,
|
77
|
+
value=pv.value,
|
78
|
+
hash=pv.hash,
|
79
|
+
created_at=pv.created_at,
|
80
|
+
created_by_id=pv.created_by_id,
|
81
|
+
)
|
82
|
+
|
83
|
+
# Remember the mapping
|
84
|
+
pv_to_fv[pv.id] = fv.id
|
85
|
+
|
86
|
+
except KeyError:
|
87
|
+
print(f"Warning: Could not map ParamValue {pv.id} to a FeatureValue")
|
88
|
+
|
89
|
+
return pv_to_fv
|
90
|
+
|
91
|
+
|
92
|
+
def copy_run_param_values(apps, schema_editor):
|
93
|
+
"""Copy data from RunParamValue to RunFeatureValue."""
|
94
|
+
ParamValue = apps.get_model("lamindb", "ParamValue")
|
95
|
+
FeatureValue = apps.get_model("lamindb", "FeatureValue")
|
96
|
+
RunParamValue = apps.get_model("lamindb", "RunParamValue")
|
97
|
+
RunFeatureValue = apps.get_model("lamindb", "RunFeatureValue")
|
98
|
+
|
99
|
+
# Create a mapping from ParamValue ID to FeatureValue ID
|
100
|
+
# This is complex because we need to match based on param->feature and value
|
101
|
+
pv_to_fv = {}
|
102
|
+
|
103
|
+
for pv in ParamValue.objects.all():
|
104
|
+
try:
|
105
|
+
# Find the corresponding Feature for this Param
|
106
|
+
feature = apps.get_model("lamindb", "Feature").objects.get(
|
107
|
+
name=apps.get_model("lamindb", "Param").objects.get(id=pv.param_id).name
|
108
|
+
)
|
109
|
+
|
110
|
+
# Find the corresponding FeatureValue
|
111
|
+
if pv.hash is None:
|
112
|
+
fv = FeatureValue.objects.get(
|
113
|
+
feature=feature, value=pv.value, hash__isnull=True
|
114
|
+
)
|
115
|
+
else:
|
116
|
+
fv = FeatureValue.objects.get(feature=feature, hash=pv.hash)
|
117
|
+
|
118
|
+
pv_to_fv[pv.id] = fv.id
|
119
|
+
except Exception as e:
|
120
|
+
print(f"Warning: Could not map ParamValue {pv.id} to a FeatureValue: {e}")
|
121
|
+
|
122
|
+
# Copy RunParamValues to RunFeatureValues
|
123
|
+
for rpv in RunParamValue.objects.all():
|
124
|
+
try:
|
125
|
+
featurevalue_id = pv_to_fv[rpv.paramvalue_id]
|
126
|
+
|
127
|
+
RunFeatureValue.objects.create(
|
128
|
+
run_id=rpv.run_id,
|
129
|
+
featurevalue_id=featurevalue_id,
|
130
|
+
created_at=rpv.created_at,
|
131
|
+
created_by_id=rpv.created_by_id,
|
132
|
+
)
|
133
|
+
except KeyError:
|
134
|
+
print(f"Warning: Could not map RunParamValue {rpv.id} to a RunFeatureValue")
|
135
|
+
|
136
|
+
|
137
|
+
def copy_artifact_param_values(apps, schema_editor):
|
138
|
+
"""Copy data from ArtifactParamValue to ArtifactFeatureValue."""
|
139
|
+
ParamValue = apps.get_model("lamindb", "ParamValue")
|
140
|
+
FeatureValue = apps.get_model("lamindb", "FeatureValue")
|
141
|
+
ArtifactParamValue = apps.get_model("lamindb", "ArtifactParamValue")
|
142
|
+
ArtifactFeatureValue = apps.get_model("lamindb", "ArtifactFeatureValue")
|
143
|
+
|
144
|
+
# Create a mapping from ParamValue ID to FeatureValue ID
|
145
|
+
# This is complex because we need to match based on param->feature and value
|
146
|
+
pv_to_fv = {}
|
147
|
+
|
148
|
+
for pv in ParamValue.objects.all():
|
149
|
+
try:
|
150
|
+
# Find the corresponding Feature for this Param
|
151
|
+
feature = apps.get_model("lamindb", "Feature").objects.get(
|
152
|
+
name=apps.get_model("lamindb", "Param").objects.get(id=pv.param_id).name
|
153
|
+
)
|
154
|
+
|
155
|
+
# Find the corresponding FeatureValue
|
156
|
+
if pv.hash is None:
|
157
|
+
fv = FeatureValue.objects.get(
|
158
|
+
feature=feature, value=pv.value, hash__isnull=True
|
159
|
+
)
|
160
|
+
else:
|
161
|
+
fv = FeatureValue.objects.get(feature=feature, hash=pv.hash)
|
162
|
+
|
163
|
+
pv_to_fv[pv.id] = fv.id
|
164
|
+
except Exception as e:
|
165
|
+
print(f"Warning: Could not map ParamValue {pv.id} to a FeatureValue: {e}")
|
166
|
+
|
167
|
+
# Copy ArtifactParamValues to ArtifactFeatureValues
|
168
|
+
for apv in ArtifactParamValue.objects.all():
|
169
|
+
try:
|
170
|
+
featurevalue_id = pv_to_fv[apv.paramvalue_id]
|
171
|
+
|
172
|
+
ArtifactFeatureValue.objects.create(
|
173
|
+
artifact_id=apv.artifact_id,
|
174
|
+
featurevalue_id=featurevalue_id,
|
175
|
+
created_at=apv.created_at,
|
176
|
+
created_by_id=apv.created_by_id,
|
177
|
+
run_id=apv.run_id,
|
178
|
+
)
|
179
|
+
except KeyError:
|
180
|
+
print(
|
181
|
+
f"Warning: Could not map ArtifactParamValue {apv.id} to an ArtifactFeatureValue"
|
182
|
+
)
|
183
|
+
|
184
|
+
|
185
|
+
def get_features_count(apps, schema_editor):
|
186
|
+
Feature = apps.get_model("lamindb", "Feature")
|
187
|
+
FeatureValue = apps.get_model("lamindb", "FeatureValue")
|
188
|
+
RunFeatureValue = apps.get_model("lamindb", "RunFeatureValue")
|
189
|
+
ArtifactFeatureValue = apps.get_model("lamindb", "ArtifactFeatureValue")
|
190
|
+
|
191
|
+
print("Features are now")
|
192
|
+
print(f"Feature count: {Feature.objects.count()}")
|
193
|
+
print(f"FeatureValue count: {FeatureValue.objects.count()}")
|
194
|
+
print(f"RunFeatureValue count: {RunFeatureValue.objects.count()}")
|
195
|
+
print(f"ArtifactFeatureValue count: {ArtifactFeatureValue.objects.count()}")
|
196
|
+
|
197
|
+
|
198
|
+
def get_params_count(apps, schema_editor):
|
199
|
+
Param = apps.get_model("lamindb", "Param")
|
200
|
+
ParamValue = apps.get_model("lamindb", "ParamValue")
|
201
|
+
RunParamValue = apps.get_model("lamindb", "RunParamValue")
|
202
|
+
ArtifactParamValue = apps.get_model("lamindb", "ArtifactParamValue")
|
203
|
+
|
204
|
+
print("Moving params into features and deleting the params table")
|
205
|
+
print(f"Param count: {Param.objects.count()}")
|
206
|
+
print(f"ParamValue count: {ParamValue.objects.count()}")
|
207
|
+
print(f"RunParamValue count: {RunParamValue.objects.count()}")
|
208
|
+
print(f"ArtifactParamValue count: {ArtifactParamValue.objects.count()}")
|
209
|
+
|
210
|
+
|
211
|
+
class Migration(migrations.Migration):
|
212
|
+
dependencies = [
|
213
|
+
("lamindb", "0095_remove_rundata_flextable"),
|
214
|
+
]
|
215
|
+
|
216
|
+
operations = [
|
217
|
+
# Step 0: Report the results
|
218
|
+
migrations.RunPython(get_params_count),
|
219
|
+
migrations.RunPython(get_features_count),
|
220
|
+
# Step 1: Copy from Param to Feature
|
221
|
+
migrations.RunPython(copy_params_to_features),
|
222
|
+
# Step 2: Copy from ParamValue to FeatureValue
|
223
|
+
migrations.RunPython(copy_param_values),
|
224
|
+
# Step 3: Copy from RunParamValue to RunFeatureValue
|
225
|
+
migrations.RunPython(copy_run_param_values),
|
226
|
+
# Step 4: Copy from ArtifactParamValue to ArtifactFeatureValue
|
227
|
+
migrations.RunPython(copy_artifact_param_values),
|
228
|
+
# Step 5: Report the results
|
229
|
+
migrations.RunPython(get_features_count),
|
230
|
+
migrations.RemoveField(
|
231
|
+
model_name="artifact",
|
232
|
+
name="_param_values",
|
233
|
+
),
|
234
|
+
migrations.RemoveField(
|
235
|
+
model_name="param",
|
236
|
+
name="created_by",
|
237
|
+
),
|
238
|
+
migrations.RemoveField(
|
239
|
+
model_name="param",
|
240
|
+
name="run",
|
241
|
+
),
|
242
|
+
migrations.RemoveField(
|
243
|
+
model_name="param",
|
244
|
+
name="schemas",
|
245
|
+
),
|
246
|
+
migrations.RemoveField(
|
247
|
+
model_name="param",
|
248
|
+
name="space",
|
249
|
+
),
|
250
|
+
migrations.RemoveField(
|
251
|
+
model_name="param",
|
252
|
+
name="type",
|
253
|
+
),
|
254
|
+
migrations.RemoveField(
|
255
|
+
model_name="run",
|
256
|
+
name="_param_values",
|
257
|
+
),
|
258
|
+
migrations.AlterUniqueTogether(
|
259
|
+
name="schemaparam",
|
260
|
+
unique_together=None,
|
261
|
+
),
|
262
|
+
migrations.RemoveField(
|
263
|
+
model_name="schemaparam",
|
264
|
+
name="schema",
|
265
|
+
),
|
266
|
+
]
|