lamindb_setup 0.73.0__py2.py3-none-any.whl → 0.73.1__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lamindb_setup/__init__.py +1 -1
- lamindb_setup/_init_instance.py +2 -2
- lamindb_setup/_migrate.py +5 -0
- lamindb_setup/_schema_metadata.py +479 -0
- lamindb_setup/core/_aws_credentials.py +4 -4
- lamindb_setup/core/_settings_instance.py +2 -2
- lamindb_setup/core/_settings_storage.py +2 -3
- lamindb_setup/core/upath.py +10 -29
- {lamindb_setup-0.73.0.dist-info → lamindb_setup-0.73.1.dist-info}/METADATA +2 -2
- {lamindb_setup-0.73.0.dist-info → lamindb_setup-0.73.1.dist-info}/RECORD +12 -11
- {lamindb_setup-0.73.0.dist-info → lamindb_setup-0.73.1.dist-info}/LICENSE +0 -0
- {lamindb_setup-0.73.0.dist-info → lamindb_setup-0.73.1.dist-info}/WHEEL +0 -0
lamindb_setup/__init__.py
CHANGED
lamindb_setup/_init_instance.py
CHANGED
|
@@ -16,7 +16,7 @@ from ._silence_loggers import silence_loggers
|
|
|
16
16
|
from .core import InstanceSettings
|
|
17
17
|
from .core._settings import settings
|
|
18
18
|
from .core._settings_storage import StorageSettings, init_storage
|
|
19
|
-
from .core.upath import
|
|
19
|
+
from .core.upath import UPath
|
|
20
20
|
|
|
21
21
|
if TYPE_CHECKING:
|
|
22
22
|
from pydantic import PostgresDsn
|
|
@@ -351,7 +351,7 @@ def infer_instance_name(
|
|
|
351
351
|
return str(db).split("/")[-1]
|
|
352
352
|
if storage == "create-s3":
|
|
353
353
|
raise ValueError("pass name to init if storage = 'create-s3'")
|
|
354
|
-
storage_path =
|
|
354
|
+
storage_path = UPath(storage)
|
|
355
355
|
if storage_path.name != "":
|
|
356
356
|
name = storage_path.name
|
|
357
357
|
else:
|
lamindb_setup/_migrate.py
CHANGED
|
@@ -68,6 +68,8 @@ class migrate:
|
|
|
68
68
|
@classmethod
|
|
69
69
|
def deploy(cls) -> None:
|
|
70
70
|
"""Deploy a migration."""
|
|
71
|
+
from ._schema_metadata import update_schema_in_hub
|
|
72
|
+
|
|
71
73
|
if _check_instance_setup():
|
|
72
74
|
raise RuntimeError("Restart Python session to migrate or use CLI!")
|
|
73
75
|
from lamindb_setup.core._hub_client import call_with_fallback_auth
|
|
@@ -104,6 +106,9 @@ class migrate:
|
|
|
104
106
|
# this populates the hub
|
|
105
107
|
if instance_is_on_hub:
|
|
106
108
|
logger.important(f"updating lamindb version in hub: {lamindb.__version__}")
|
|
109
|
+
# TODO: integrate update of instance table within update_schema_in_hub & below
|
|
110
|
+
if settings.instance.dialect != "sqlite":
|
|
111
|
+
update_schema_in_hub()
|
|
107
112
|
call_with_fallback_auth(
|
|
108
113
|
update_instance,
|
|
109
114
|
instance_id=settings.instance._id.hex,
|
|
@@ -0,0 +1,479 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import importlib
|
|
5
|
+
import json
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from typing import TYPE_CHECKING, Dict
|
|
8
|
+
from uuid import UUID
|
|
9
|
+
|
|
10
|
+
import sqlparse
|
|
11
|
+
from django.contrib.postgres.expressions import ArraySubquery
|
|
12
|
+
from django.db.models import (
|
|
13
|
+
Field,
|
|
14
|
+
ForeignObjectRel,
|
|
15
|
+
ManyToManyField,
|
|
16
|
+
ManyToManyRel,
|
|
17
|
+
OuterRef,
|
|
18
|
+
QuerySet,
|
|
19
|
+
Subquery,
|
|
20
|
+
)
|
|
21
|
+
from django.db.models.functions import JSONObject
|
|
22
|
+
from sqlparse.sql import Identifier, IdentifierList
|
|
23
|
+
from sqlparse.tokens import DML, Keyword
|
|
24
|
+
|
|
25
|
+
from lamindb_setup import settings
|
|
26
|
+
from lamindb_setup._init_instance import get_schema_module_name
|
|
27
|
+
from lamindb_setup.core._hub_client import call_with_fallback_auth
|
|
28
|
+
|
|
29
|
+
if TYPE_CHECKING:
|
|
30
|
+
from lnschema_core.models import Registry
|
|
31
|
+
from supabase import Client
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def update_schema_in_hub() -> tuple[bool, UUID, dict]:
|
|
35
|
+
return call_with_fallback_auth(_synchronize_schema)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _synchronize_schema(client: Client) -> tuple[bool, UUID, dict]:
|
|
39
|
+
schema_metadata = SchemaMetadata()
|
|
40
|
+
schema_metadata_dict = schema_metadata.to_json()
|
|
41
|
+
schema_uuid = _dict_to_uuid(schema_metadata_dict)
|
|
42
|
+
schema = _get_schema_by_id(schema_uuid, client)
|
|
43
|
+
|
|
44
|
+
is_new = schema is None
|
|
45
|
+
if is_new:
|
|
46
|
+
module_set_info = schema_metadata._get_module_set_info()
|
|
47
|
+
module_ids = "-".join(str(module_info["id"]) for module_info in module_set_info)
|
|
48
|
+
schema = (
|
|
49
|
+
client.table("schema")
|
|
50
|
+
.insert(
|
|
51
|
+
{
|
|
52
|
+
"id": schema_uuid.hex,
|
|
53
|
+
"module_ids": module_ids,
|
|
54
|
+
"module_set_info": module_set_info,
|
|
55
|
+
"json": schema_metadata_dict,
|
|
56
|
+
}
|
|
57
|
+
)
|
|
58
|
+
.execute()
|
|
59
|
+
.data[0]
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
instance_response = (
|
|
63
|
+
client.table("instance")
|
|
64
|
+
.update({"schema_id": schema_uuid.hex})
|
|
65
|
+
.eq("id", settings.instance._id.hex)
|
|
66
|
+
.execute()
|
|
67
|
+
)
|
|
68
|
+
assert (
|
|
69
|
+
len(instance_response.data) == 1
|
|
70
|
+
), f"schema of instance {settings.instance._id.hex} could not be updated with schema {schema_uuid.hex}"
|
|
71
|
+
|
|
72
|
+
return is_new, schema_uuid, schema
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def get_schema_by_id(id: UUID):
|
|
76
|
+
return call_with_fallback_auth(_get_schema_by_id, id=id)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _get_schema_by_id(id: UUID, client: Client):
|
|
80
|
+
response = client.table("schema").select("*").eq("id", id.hex).execute()
|
|
81
|
+
if len(response.data) == 0:
|
|
82
|
+
return None
|
|
83
|
+
return response.data[0]
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _dict_to_uuid(dict: dict):
|
|
87
|
+
encoded = json.dumps(dict, sort_keys=True).encode("utf-8")
|
|
88
|
+
hash = hashlib.md5(encoded).digest()
|
|
89
|
+
uuid = UUID(bytes=hash[:16])
|
|
90
|
+
return uuid
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class SchemaMetadata:
|
|
94
|
+
def __init__(self) -> None:
|
|
95
|
+
self.included_modules = ["core"] + list(settings.instance.schema)
|
|
96
|
+
self.modules = self._get_modules_metadata()
|
|
97
|
+
|
|
98
|
+
def to_dict(
|
|
99
|
+
self, include_django_objects: bool = True, include_select_terms: bool = True
|
|
100
|
+
):
|
|
101
|
+
return {
|
|
102
|
+
module_name: {
|
|
103
|
+
model_name: model.to_dict(include_django_objects, include_select_terms)
|
|
104
|
+
for model_name, model in module.items()
|
|
105
|
+
}
|
|
106
|
+
for module_name, module in self.modules.items()
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
def to_json(self, include_select_terms: bool = True):
|
|
110
|
+
return self.to_dict(
|
|
111
|
+
include_django_objects=False, include_select_terms=include_select_terms
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
def _get_modules_metadata(self):
|
|
115
|
+
return {
|
|
116
|
+
module_name: {
|
|
117
|
+
model._meta.model_name: ModelMetadata(
|
|
118
|
+
model, module_name, self.included_modules
|
|
119
|
+
)
|
|
120
|
+
for model in self._get_schema_module(
|
|
121
|
+
module_name
|
|
122
|
+
).models.__dict__.values()
|
|
123
|
+
if model.__class__.__name__ == "ModelBase"
|
|
124
|
+
and model.__name__ not in ["Registry", "ORM"]
|
|
125
|
+
and not model._meta.abstract
|
|
126
|
+
and model.__get_schema_name__() == module_name
|
|
127
|
+
}
|
|
128
|
+
for module_name in self.included_modules
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
def _get_module_set_info(self):
|
|
132
|
+
# TODO: rely on schemamodule table for this
|
|
133
|
+
module_set_info = []
|
|
134
|
+
for module_name in self.included_modules:
|
|
135
|
+
module = self._get_schema_module(module_name)
|
|
136
|
+
module_set_info.append(
|
|
137
|
+
{"id": 0, "name": module_name, "version": module.__version__}
|
|
138
|
+
)
|
|
139
|
+
return module_set_info
|
|
140
|
+
|
|
141
|
+
@staticmethod
|
|
142
|
+
def _get_schema_module(module_name):
|
|
143
|
+
return importlib.import_module(get_schema_module_name(module_name))
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
@dataclass
|
|
147
|
+
class FieldMetadata:
|
|
148
|
+
schema_name: str
|
|
149
|
+
model_name: str
|
|
150
|
+
field_name: str
|
|
151
|
+
type: str
|
|
152
|
+
is_link_table: bool
|
|
153
|
+
column: str | None = None
|
|
154
|
+
relation_type: str | None = None
|
|
155
|
+
related_schema_name: str | None = None
|
|
156
|
+
related_model_name: str | None = None
|
|
157
|
+
related_field_name: str | None = None
|
|
158
|
+
through: dict | None = None
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
class ModelRelations:
|
|
162
|
+
def __init__(self, fields: list[ForeignObjectRel]) -> None:
|
|
163
|
+
self.many_to_one = {}
|
|
164
|
+
self.one_to_many = {}
|
|
165
|
+
self.many_to_many = {}
|
|
166
|
+
self.one_to_one = {}
|
|
167
|
+
|
|
168
|
+
for field in fields:
|
|
169
|
+
if field.many_to_one:
|
|
170
|
+
self.many_to_one.update({field.name: field})
|
|
171
|
+
elif field.one_to_many:
|
|
172
|
+
self.one_to_many.update({field.name: field})
|
|
173
|
+
elif field.many_to_many:
|
|
174
|
+
self.many_to_many.update({field.name: field})
|
|
175
|
+
elif field.one_to_one:
|
|
176
|
+
self.one_to_one.update({field.name: field})
|
|
177
|
+
|
|
178
|
+
self.all = {
|
|
179
|
+
**self.many_to_one,
|
|
180
|
+
**self.one_to_many,
|
|
181
|
+
**self.many_to_many,
|
|
182
|
+
**self.one_to_one,
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
class ModelMetadata:
|
|
187
|
+
def __init__(self, model, module_name: str, included_modules: list[str]) -> None:
|
|
188
|
+
self.model = model
|
|
189
|
+
self.class_name = model.__name__
|
|
190
|
+
self.module_name = module_name
|
|
191
|
+
self.model_name = model._meta.model_name
|
|
192
|
+
self.table_name = model._meta.db_table
|
|
193
|
+
self.included_modules = included_modules
|
|
194
|
+
self.fields, self.relations = self._get_fields_metadata(self.model)
|
|
195
|
+
|
|
196
|
+
def to_dict(
|
|
197
|
+
self, include_django_objects: bool = True, include_select_terms: bool = True
|
|
198
|
+
):
|
|
199
|
+
_dict = {
|
|
200
|
+
"fields": self.fields.copy(),
|
|
201
|
+
"class_name": self.class_name,
|
|
202
|
+
"table_name": self.table_name,
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
select_terms = self.select_terms if include_select_terms else []
|
|
206
|
+
|
|
207
|
+
for field_name in self.fields.keys():
|
|
208
|
+
_dict["fields"][field_name] = _dict["fields"][field_name].__dict__
|
|
209
|
+
if field_name in select_terms:
|
|
210
|
+
_dict["fields"][field_name].update(
|
|
211
|
+
{"select_term": select_terms[field_name]}
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
if include_django_objects:
|
|
215
|
+
_dict.update({"model": self.model})
|
|
216
|
+
|
|
217
|
+
return _dict
|
|
218
|
+
|
|
219
|
+
@property
|
|
220
|
+
def select_terms(self):
|
|
221
|
+
return (
|
|
222
|
+
DjangoQueryBuilder(self.module_name, self.model_name)
|
|
223
|
+
.add_all_sub_queries()
|
|
224
|
+
.extract_select_terms()
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
def _get_fields_metadata(self, model):
|
|
228
|
+
related_fields = []
|
|
229
|
+
fields_metadata: dict[str, FieldMetadata] = {}
|
|
230
|
+
|
|
231
|
+
for field in model._meta.get_fields():
|
|
232
|
+
field_metadata = self._get_field_metadata(model, field)
|
|
233
|
+
if field_metadata.related_schema_name is None:
|
|
234
|
+
fields_metadata.update({field.name: field_metadata})
|
|
235
|
+
|
|
236
|
+
if (
|
|
237
|
+
field_metadata.related_schema_name in self.included_modules
|
|
238
|
+
and field_metadata.schema_name in self.included_modules
|
|
239
|
+
):
|
|
240
|
+
related_fields.append(field)
|
|
241
|
+
|
|
242
|
+
model_relations_metadata = ModelRelations(related_fields)
|
|
243
|
+
|
|
244
|
+
related_fields_metadata = self._get_related_fields_metadata(
|
|
245
|
+
model, model_relations_metadata
|
|
246
|
+
)
|
|
247
|
+
|
|
248
|
+
fields_metadata = {**fields_metadata, **related_fields_metadata}
|
|
249
|
+
|
|
250
|
+
return fields_metadata, model_relations_metadata
|
|
251
|
+
|
|
252
|
+
def _get_related_fields_metadata(
|
|
253
|
+
self, model, model_relations_metadata: ModelRelations
|
|
254
|
+
):
|
|
255
|
+
related_fields: dict[str, FieldMetadata] = {}
|
|
256
|
+
|
|
257
|
+
# Many to one (foreign key defined in the model)
|
|
258
|
+
for link_field_name, link_field in model_relations_metadata.many_to_one.items():
|
|
259
|
+
related_fields.update(
|
|
260
|
+
{f"{link_field_name}": self._get_field_metadata(model, link_field)}
|
|
261
|
+
)
|
|
262
|
+
for field in link_field.related_model._meta.fields:
|
|
263
|
+
related_fields.update(
|
|
264
|
+
{
|
|
265
|
+
f"{link_field_name}__{field.name}": self._get_field_metadata(
|
|
266
|
+
model, field
|
|
267
|
+
)
|
|
268
|
+
}
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
# One to many (foreign key defined in the related model)
|
|
272
|
+
for relation_name, relation in model_relations_metadata.one_to_many.items():
|
|
273
|
+
# exclude self reference as it is already included in the many to one
|
|
274
|
+
if relation.related_model == model:
|
|
275
|
+
continue
|
|
276
|
+
related_fields.update(
|
|
277
|
+
{f"{relation_name}": self._get_field_metadata(model, relation.field)}
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
# One to one
|
|
281
|
+
for link_field_name, link_field in model_relations_metadata.one_to_one.items():
|
|
282
|
+
related_fields.update(
|
|
283
|
+
{f"{link_field_name}": self._get_field_metadata(model, link_field)}
|
|
284
|
+
)
|
|
285
|
+
for field in link_field.related_model._meta.fields:
|
|
286
|
+
related_fields.update(
|
|
287
|
+
{
|
|
288
|
+
f"{link_field_name}__{field.name}": self._get_field_metadata(
|
|
289
|
+
model, field
|
|
290
|
+
)
|
|
291
|
+
}
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
# Many to many
|
|
295
|
+
for (
|
|
296
|
+
link_field_name,
|
|
297
|
+
link_field,
|
|
298
|
+
) in model_relations_metadata.many_to_many.items():
|
|
299
|
+
related_fields.update(
|
|
300
|
+
{f"{link_field_name}": self._get_field_metadata(model, link_field)}
|
|
301
|
+
)
|
|
302
|
+
|
|
303
|
+
return related_fields
|
|
304
|
+
|
|
305
|
+
def _get_field_metadata(self, model, field: Field):
|
|
306
|
+
from lnschema_core.models import LinkORM
|
|
307
|
+
|
|
308
|
+
internal_type = field.get_internal_type()
|
|
309
|
+
model_name = field.model._meta.model_name
|
|
310
|
+
relation_type = self._get_relation_type(model, field)
|
|
311
|
+
if field.related_model is None:
|
|
312
|
+
schema_name = field.model.__get_schema_name__()
|
|
313
|
+
related_model_name = None
|
|
314
|
+
related_schema_name = None
|
|
315
|
+
related_field_name = None
|
|
316
|
+
field_name = field.name
|
|
317
|
+
else:
|
|
318
|
+
related_model_name = field.related_model._meta.model_name
|
|
319
|
+
related_schema_name = field.related_model.__get_schema_name__()
|
|
320
|
+
schema_name = field.model.__get_schema_name__()
|
|
321
|
+
related_field_name = field.remote_field.name
|
|
322
|
+
field_name = field.name
|
|
323
|
+
|
|
324
|
+
if relation_type in ["one-to-many"]:
|
|
325
|
+
# For a one-to-many relation, the field belong
|
|
326
|
+
# to the other model as a foreign key.
|
|
327
|
+
# To make usage similar to other relation types
|
|
328
|
+
# we need to invert model and related model.
|
|
329
|
+
schema_name, related_schema_name = related_schema_name, schema_name
|
|
330
|
+
model_name, related_model_name = related_model_name, model_name
|
|
331
|
+
field_name, related_field_name = related_field_name, field_name
|
|
332
|
+
pass
|
|
333
|
+
|
|
334
|
+
column = None
|
|
335
|
+
if relation_type not in ["many-to-many", "one-to-one", "one-to-many"]:
|
|
336
|
+
column = field.column
|
|
337
|
+
|
|
338
|
+
through = None
|
|
339
|
+
if relation_type == "many-to-many":
|
|
340
|
+
through = self._get_through(model, field)
|
|
341
|
+
|
|
342
|
+
return FieldMetadata(
|
|
343
|
+
schema_name,
|
|
344
|
+
model_name,
|
|
345
|
+
field_name,
|
|
346
|
+
internal_type,
|
|
347
|
+
issubclass(field.model, LinkORM),
|
|
348
|
+
column,
|
|
349
|
+
relation_type,
|
|
350
|
+
related_schema_name,
|
|
351
|
+
related_model_name,
|
|
352
|
+
related_field_name,
|
|
353
|
+
through,
|
|
354
|
+
)
|
|
355
|
+
|
|
356
|
+
@staticmethod
|
|
357
|
+
def _get_through(model, field_or_rel: ManyToManyField | ManyToManyRel):
|
|
358
|
+
table_name = model._meta.db_table
|
|
359
|
+
related_table_name = field_or_rel.related_model._meta.db_table
|
|
360
|
+
|
|
361
|
+
if isinstance(field_or_rel, ManyToManyField):
|
|
362
|
+
return {
|
|
363
|
+
"link_table_name": field_or_rel.remote_field.through._meta.db_table,
|
|
364
|
+
table_name: field_or_rel.m2m_column_name(),
|
|
365
|
+
related_table_name: field_or_rel.m2m_reverse_name(),
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
if isinstance(field_or_rel, ManyToManyRel):
|
|
369
|
+
return {
|
|
370
|
+
"link_table_name": field_or_rel.through._meta.db_table,
|
|
371
|
+
table_name: field_or_rel.field.m2m_column_name(),
|
|
372
|
+
related_table_name: field_or_rel.field.m2m_reverse_name(),
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
@staticmethod
|
|
376
|
+
def _get_relation_type(model, field: Field):
|
|
377
|
+
if field.many_to_one:
|
|
378
|
+
# defined in the model
|
|
379
|
+
if model == field.model:
|
|
380
|
+
return "many-to-one"
|
|
381
|
+
# defined in the related model
|
|
382
|
+
else:
|
|
383
|
+
return "one-to-many"
|
|
384
|
+
elif field.one_to_many:
|
|
385
|
+
return "one-to-many"
|
|
386
|
+
elif field.many_to_many:
|
|
387
|
+
return "many-to-many"
|
|
388
|
+
elif field.one_to_one:
|
|
389
|
+
return "one-to-one"
|
|
390
|
+
else:
|
|
391
|
+
return None
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
class DjangoQueryBuilder:
|
|
395
|
+
def __init__(
|
|
396
|
+
self, module_name: str, model_name: str, query_set: QuerySet | None = None
|
|
397
|
+
) -> None:
|
|
398
|
+
self.schema_metadata = SchemaMetadata()
|
|
399
|
+
self.module_name = module_name
|
|
400
|
+
self.model_name = model_name
|
|
401
|
+
self.model_metadata = self.schema_metadata.modules[module_name][model_name]
|
|
402
|
+
self.query_set = query_set if query_set else self.model_metadata.model.objects
|
|
403
|
+
|
|
404
|
+
def add_all_sub_queries(self):
|
|
405
|
+
all_fields = self.model_metadata.fields
|
|
406
|
+
included_relations = [
|
|
407
|
+
field_name
|
|
408
|
+
for field_name, field in all_fields.items()
|
|
409
|
+
if field.relation_type is not None
|
|
410
|
+
]
|
|
411
|
+
self.add_sub_queries(included_relations)
|
|
412
|
+
return self
|
|
413
|
+
|
|
414
|
+
def add_sub_queries(self, included_relations: list[str]):
|
|
415
|
+
sub_queries = {
|
|
416
|
+
f"annotated_{relation_name}": self._get_sub_query(
|
|
417
|
+
self.model_metadata.fields[relation_name]
|
|
418
|
+
)
|
|
419
|
+
for relation_name in included_relations
|
|
420
|
+
}
|
|
421
|
+
self.query_set = self.query_set.annotate(**sub_queries)
|
|
422
|
+
return self
|
|
423
|
+
|
|
424
|
+
def extract_select_terms(self):
|
|
425
|
+
parsed = sqlparse.parse(self.sql_query)
|
|
426
|
+
select_found = False
|
|
427
|
+
select_terms = {}
|
|
428
|
+
|
|
429
|
+
def get_name(identifier):
|
|
430
|
+
name = identifier.get_name()
|
|
431
|
+
return name if name is not None else str(identifier).split(".")
|
|
432
|
+
|
|
433
|
+
for token in parsed[0].tokens:
|
|
434
|
+
if token.ttype is DML and token.value.upper() == "SELECT":
|
|
435
|
+
select_found = True
|
|
436
|
+
elif select_found and isinstance(token, IdentifierList):
|
|
437
|
+
for identifier in token.get_identifiers():
|
|
438
|
+
select_terms[get_name(identifier)] = str(identifier)
|
|
439
|
+
elif select_found and isinstance(token, Identifier):
|
|
440
|
+
select_terms[get_name(token)] = str(token)
|
|
441
|
+
elif token.ttype is Keyword:
|
|
442
|
+
if token.value.upper() in ["FROM", "WHERE", "GROUP BY", "ORDER BY"]:
|
|
443
|
+
break
|
|
444
|
+
|
|
445
|
+
return select_terms
|
|
446
|
+
|
|
447
|
+
def _get_sub_query(self, field_metadata: FieldMetadata):
|
|
448
|
+
module_name = field_metadata.related_schema_name
|
|
449
|
+
model_name = field_metadata.related_model_name
|
|
450
|
+
field_name = field_metadata.related_field_name
|
|
451
|
+
model_metadata = self.schema_metadata.modules[module_name][model_name]
|
|
452
|
+
query_set = model_metadata.model.objects.get_queryset()
|
|
453
|
+
select = {
|
|
454
|
+
field_name: field_name
|
|
455
|
+
for field_name in model_metadata.fields.keys()
|
|
456
|
+
if model_metadata.fields[field_name].relation_type is None
|
|
457
|
+
and "__" not in field_name
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
if field_metadata.relation_type in ["many-to-many", "one-to-many"]:
|
|
461
|
+
return ArraySubquery(
|
|
462
|
+
Subquery(
|
|
463
|
+
query_set.filter(**{field_name: OuterRef("pk")}).values(
|
|
464
|
+
data=JSONObject(**select)
|
|
465
|
+
)[:5]
|
|
466
|
+
)
|
|
467
|
+
)
|
|
468
|
+
if field_metadata.relation_type in ["many-to-one", "one-to-one"]:
|
|
469
|
+
return Subquery(
|
|
470
|
+
query_set.filter(**{field_name: OuterRef("pk")}).values(
|
|
471
|
+
data=JSONObject(**select)
|
|
472
|
+
)[:5]
|
|
473
|
+
)
|
|
474
|
+
|
|
475
|
+
@property
|
|
476
|
+
def sql_query(self):
|
|
477
|
+
sql_template, params = self.query_set.query.sql_with_params()
|
|
478
|
+
sql_query = sql_template % tuple(f"'{p}'" for p in params)
|
|
479
|
+
return sql_query.replace("annotated_", "")
|
|
@@ -67,8 +67,8 @@ class AWSCredentialsManager:
|
|
|
67
67
|
def _path_inject_options(self, path: S3Path, credentials: dict) -> S3Path:
|
|
68
68
|
if credentials == {}:
|
|
69
69
|
# credentials were specified manually for the path
|
|
70
|
-
if "anon" in path.
|
|
71
|
-
anon = path.
|
|
70
|
+
if "anon" in path.storage_options:
|
|
71
|
+
anon = path.storage_options["anon"]
|
|
72
72
|
elif path.fs.key is not None and path.fs.secret is not None:
|
|
73
73
|
anon = False
|
|
74
74
|
else:
|
|
@@ -77,8 +77,8 @@ class AWSCredentialsManager:
|
|
|
77
77
|
else:
|
|
78
78
|
connection_options = credentials
|
|
79
79
|
|
|
80
|
-
if "cache_regions" in path.
|
|
81
|
-
cache_regions = path.
|
|
80
|
+
if "cache_regions" in path.storage_options:
|
|
81
|
+
cache_regions = path.storage_options["cache_regions"]
|
|
82
82
|
else:
|
|
83
83
|
cache_regions = True
|
|
84
84
|
|
|
@@ -18,7 +18,7 @@ from .cloud_sqlite_locker import (
|
|
|
18
18
|
EXPIRATION_TIME,
|
|
19
19
|
InstanceLockedException,
|
|
20
20
|
)
|
|
21
|
-
from .upath import LocalPathClasses, UPath
|
|
21
|
+
from .upath import LocalPathClasses, UPath
|
|
22
22
|
|
|
23
23
|
if TYPE_CHECKING:
|
|
24
24
|
from uuid import UUID
|
|
@@ -205,7 +205,7 @@ class InstanceSettings:
|
|
|
205
205
|
)
|
|
206
206
|
if response != "y":
|
|
207
207
|
return None
|
|
208
|
-
local_root =
|
|
208
|
+
local_root = UPath(local_root)
|
|
209
209
|
assert isinstance(local_root, LocalPathClasses)
|
|
210
210
|
self._storage_local = init_storage(local_root, self._id, register_hub=True) # type: ignore
|
|
211
211
|
register_storage_in_instance(self._storage_local) # type: ignore
|
|
@@ -17,7 +17,6 @@ from ._settings_store import system_storage_settings_file
|
|
|
17
17
|
from .upath import (
|
|
18
18
|
LocalPathClasses,
|
|
19
19
|
UPath,
|
|
20
|
-
convert_pathlike,
|
|
21
20
|
create_path,
|
|
22
21
|
)
|
|
23
22
|
|
|
@@ -73,7 +72,7 @@ def mark_storage_root(root: UPathStr, uid: str):
|
|
|
73
72
|
# we need to touch a 0-byte object in folder-like storage location on S3 to avoid
|
|
74
73
|
# permission errors from leveraging s3fs on an empty hosted storage location
|
|
75
74
|
# for consistency, we write this file everywhere
|
|
76
|
-
root_upath =
|
|
75
|
+
root_upath = UPath(root)
|
|
77
76
|
mark_upath = root_upath / IS_INITIALIZED_KEY
|
|
78
77
|
mark_upath.write_text(uid)
|
|
79
78
|
|
|
@@ -159,7 +158,7 @@ class StorageSettings:
|
|
|
159
158
|
):
|
|
160
159
|
self._uid = uid
|
|
161
160
|
self._uuid_ = uuid
|
|
162
|
-
self._root_init =
|
|
161
|
+
self._root_init = UPath(root)
|
|
163
162
|
if isinstance(self._root_init, LocalPathClasses): # local paths
|
|
164
163
|
try:
|
|
165
164
|
(self._root_init / ".lamindb").mkdir(parents=True, exist_ok=True)
|
lamindb_setup/core/upath.py
CHANGED
|
@@ -59,6 +59,7 @@ VALID_SUFFIXES = {
|
|
|
59
59
|
VALID_COMPOSITE_SUFFIXES = {
|
|
60
60
|
".anndata.zarr",
|
|
61
61
|
".spatialdata.zarr",
|
|
62
|
+
".ome.zarr",
|
|
62
63
|
}
|
|
63
64
|
|
|
64
65
|
TRAILING_SEP = (os.sep, os.altsep) if os.altsep is not None else os.sep
|
|
@@ -295,12 +296,7 @@ def upload_from(
|
|
|
295
296
|
|
|
296
297
|
if local_path_is_dir and not create_folder:
|
|
297
298
|
source = [f for f in local_path.rglob("*") if f.is_file()]
|
|
298
|
-
|
|
299
|
-
# UPath("s3://some-bucket/some-folder/") / "some-key"
|
|
300
|
-
# results in UPath("s3://some-bucket/some-folder//some-key")
|
|
301
|
-
# for upath 0.1.4
|
|
302
|
-
dest_root = convert_pathlike(self) if self._parts[-1] == "" else self
|
|
303
|
-
destination = [str(dest_root / f.relative_to(local_path)) for f in source]
|
|
299
|
+
destination = [str(self / f.relative_to(local_path)) for f in source]
|
|
304
300
|
source = [str(f) for f in source] # type: ignore
|
|
305
301
|
else:
|
|
306
302
|
source = str(local_path) # type: ignore
|
|
@@ -329,9 +325,7 @@ def upload_from(
|
|
|
329
325
|
del self.fs.dircache[bucket]
|
|
330
326
|
|
|
331
327
|
if local_path_is_dir and create_folder:
|
|
332
|
-
|
|
333
|
-
dest_root = convert_pathlike(self) if self._parts[-1] == "" else self
|
|
334
|
-
return dest_root / local_path.name
|
|
328
|
+
return self / local_path.name
|
|
335
329
|
else:
|
|
336
330
|
return self
|
|
337
331
|
|
|
@@ -415,7 +409,7 @@ def synchronize(
|
|
|
415
409
|
origin = f"{self.protocol}://{file}"
|
|
416
410
|
destination = objectpath / file_key
|
|
417
411
|
child = callback.branched(origin, destination.as_posix())
|
|
418
|
-
UPath(origin, **self.
|
|
412
|
+
UPath(origin, **self.storage_options).synchronize(
|
|
419
413
|
destination, callback=child, timestamp=timestamp
|
|
420
414
|
)
|
|
421
415
|
child.close()
|
|
@@ -695,22 +689,8 @@ Args:
|
|
|
695
689
|
"""
|
|
696
690
|
|
|
697
691
|
|
|
698
|
-
def convert_pathlike(pathlike: UPathStr) -> UPath:
|
|
699
|
-
"""Convert pathlike to Path or UPath inheriting options from root."""
|
|
700
|
-
if isinstance(pathlike, (str, UPath)):
|
|
701
|
-
path = UPath(pathlike)
|
|
702
|
-
elif isinstance(pathlike, Path):
|
|
703
|
-
path = UPath(str(pathlike)) # UPath applied on Path gives Path back
|
|
704
|
-
else:
|
|
705
|
-
raise ValueError("pathlike should be of type UPathStr")
|
|
706
|
-
# remove trailing slash
|
|
707
|
-
if path._parts and path._parts[-1] == "":
|
|
708
|
-
path._parts = path._parts[:-1]
|
|
709
|
-
return path
|
|
710
|
-
|
|
711
|
-
|
|
712
692
|
def create_path(path: UPath, access_token: str | None = None) -> UPath:
|
|
713
|
-
path =
|
|
693
|
+
path = UPath(path)
|
|
714
694
|
# test whether we have an AWS S3 path
|
|
715
695
|
if not isinstance(path, S3Path):
|
|
716
696
|
return path
|
|
@@ -719,15 +699,16 @@ def create_path(path: UPath, access_token: str | None = None) -> UPath:
|
|
|
719
699
|
|
|
720
700
|
def get_stat_file_cloud(stat: dict) -> tuple[int, str, str]:
|
|
721
701
|
size = stat["size"]
|
|
702
|
+
etag = stat["ETag"]
|
|
722
703
|
# small files
|
|
723
|
-
if "-" not in
|
|
704
|
+
if "-" not in etag:
|
|
724
705
|
# only store hash for non-multipart uploads
|
|
725
706
|
# we can't rapidly validate multi-part uploaded files client-side
|
|
726
707
|
# we can add more logic later down-the-road
|
|
727
|
-
hash = b16_to_b64(
|
|
708
|
+
hash = b16_to_b64(etag)
|
|
728
709
|
hash_type = "md5"
|
|
729
710
|
else:
|
|
730
|
-
stripped_etag, suffix =
|
|
711
|
+
stripped_etag, suffix = etag.split("-")
|
|
731
712
|
suffix = suffix.strip('"')
|
|
732
713
|
hash = f"{b16_to_b64(stripped_etag)}-{suffix}"
|
|
733
714
|
hash_type = "md5-n" # this is the S3 chunk-hashing strategy
|
|
@@ -759,7 +740,7 @@ class InstanceNotEmpty(Exception):
|
|
|
759
740
|
def check_storage_is_empty(
|
|
760
741
|
root: UPathStr, *, raise_error: bool = True, account_for_sqlite_file: bool = False
|
|
761
742
|
) -> int:
|
|
762
|
-
root_upath =
|
|
743
|
+
root_upath = UPath(root)
|
|
763
744
|
root_string = root_upath.as_posix() # type: ignore
|
|
764
745
|
# we currently touch a 0-byte file in the root of a hosted storage location
|
|
765
746
|
# ({storage_root}/.lamindb/_is_initialized) during storage initialization
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: lamindb_setup
|
|
3
|
-
Version: 0.73.
|
|
3
|
+
Version: 0.73.1
|
|
4
4
|
Summary: Setup & configure LaminDB.
|
|
5
5
|
Author-email: Lamin Labs <laminlabs@gmail.com>
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
@@ -11,7 +11,7 @@ Requires-Dist: dj_database_url>=1.3.0,<3.0.0
|
|
|
11
11
|
Requires-Dist: pydantic[dotenv]<2.0.0
|
|
12
12
|
Requires-Dist: appdirs<2.0.0
|
|
13
13
|
Requires-Dist: requests
|
|
14
|
-
Requires-Dist: universal_pathlib==0.
|
|
14
|
+
Requires-Dist: universal_pathlib==0.2.2
|
|
15
15
|
Requires-Dist: botocore<2.0.0
|
|
16
16
|
Requires-Dist: supabase==2.2.1
|
|
17
17
|
Requires-Dist: urllib3<2 ; extra == "aws"
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
lamindb_setup/__init__.py,sha256=
|
|
1
|
+
lamindb_setup/__init__.py,sha256=XYsA4Sk8k2514_kS3XNzBJs1ZXhDkUcrd1rQTzwsan4,1542
|
|
2
2
|
lamindb_setup/_cache.py,sha256=wA7mbysANwe8hPNbjDo9bOmXJ0xIyaS5iyxIpxSWji4,846
|
|
3
3
|
lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
|
|
4
4
|
lamindb_setup/_check_setup.py,sha256=cNEL9Q4yPpmEkGKHH8JgullWl1VUZwALJ4RHn9wZypY,2613
|
|
@@ -8,15 +8,16 @@ lamindb_setup/_delete.py,sha256=Y8KSFYgY0UHAvjd7cCL6hZ_XiLeJwx50BguVATcj_Xo,5524
|
|
|
8
8
|
lamindb_setup/_django.py,sha256=EoyWvFzH0i9wxjy4JZhcoXCTckztP_Mrl6FbYQnMmLE,1534
|
|
9
9
|
lamindb_setup/_exportdb.py,sha256=uTIZjKKTB7arzEr1j0O6lONiT2pRBKeOFdLvOV8ZwzE,2120
|
|
10
10
|
lamindb_setup/_importdb.py,sha256=yYYShzUajTsR-cTW4CZ-UNDWZY2uE5PAgNbp-wn8Ogc,1874
|
|
11
|
-
lamindb_setup/_init_instance.py,sha256=
|
|
12
|
-
lamindb_setup/_migrate.py,sha256=
|
|
11
|
+
lamindb_setup/_init_instance.py,sha256=Cji3h2kCj8Meukkm8Btl8Lu4Jio4NiWLlzAAKukEzYY,11896
|
|
12
|
+
lamindb_setup/_migrate.py,sha256=49xPkwU-QQjpq0xSUepiVfvRsMhmjfbRlhd72YIq8o8,9059
|
|
13
13
|
lamindb_setup/_register_instance.py,sha256=Jeu0wyvJVSVQ_n-A_7yn7xOZIP0ncJD92DRABqzPIjA,940
|
|
14
14
|
lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
|
|
15
|
+
lamindb_setup/_schema_metadata.py,sha256=G3yXJ46OkVGqHxccCgjvQnnBNKg1uhwxXB1CddlkwYw,16882
|
|
15
16
|
lamindb_setup/_set_managed_storage.py,sha256=mNZrANn-9rwZ0oGWxxg0wS0T0VOQCWyo4nSSyNAE15Q,1419
|
|
16
17
|
lamindb_setup/_setup_user.py,sha256=6Oc7Rke-yRQSZbuntdUAz8QbJ6UuPzYHI9FnYlf_q-A,3670
|
|
17
18
|
lamindb_setup/_silence_loggers.py,sha256=AKF_YcHvX32eGXdsYK8MJlxEaZ-Uo2f6QDRzjKFCtws,1568
|
|
18
19
|
lamindb_setup/core/__init__.py,sha256=dV9S-rQpNK9JcBn4hiEmiLnmNqfpPFJD9pqagMCaIew,416
|
|
19
|
-
lamindb_setup/core/_aws_credentials.py,sha256=
|
|
20
|
+
lamindb_setup/core/_aws_credentials.py,sha256=uKMQO9q42Hnepz8aj3RxwLKDWUJx8pNOYrFnnNh5X40,5325
|
|
20
21
|
lamindb_setup/core/_aws_storage.py,sha256=nEjeUv4xUVpoV0Lx-zjjmyb9w804bDyaeiM-OqbfwM0,1799
|
|
21
22
|
lamindb_setup/core/_deprecated.py,sha256=3qxUI1dnDlSeR0BYrv7ucjqRBEojbqotPgpShXs4KF8,2520
|
|
22
23
|
lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
|
|
@@ -25,10 +26,10 @@ lamindb_setup/core/_hub_core.py,sha256=RGjTqf1owuWmkXAYy0EPaoHAaJ-0T0hAidkqa3cId
|
|
|
25
26
|
lamindb_setup/core/_hub_crud.py,sha256=b1XF7AJpM9Q-ttm9nPG-r3OTRWHQaGzAGIyvmb83NTo,4859
|
|
26
27
|
lamindb_setup/core/_hub_utils.py,sha256=b_M1LkdCjiMWm1EOlSb9GuPdLijwVgQDtATTpeZuXI0,1875
|
|
27
28
|
lamindb_setup/core/_settings.py,sha256=jjZ_AxRXB3Y3UP6m04BAw_dhFbJbdg2-nZWmEv2LNZ8,3141
|
|
28
|
-
lamindb_setup/core/_settings_instance.py,sha256=
|
|
29
|
+
lamindb_setup/core/_settings_instance.py,sha256=O9TtGijSJCXMREePegrxlQmHzKJFbgpC8yrlOe4BwJo,16920
|
|
29
30
|
lamindb_setup/core/_settings_load.py,sha256=NGgCDpN85j1EqoKlrYFIlZBMlBJm33gx2-wc96CP_ZQ,3922
|
|
30
31
|
lamindb_setup/core/_settings_save.py,sha256=d1A-Ex-7H08mb8l7I0Oe0j0GilrfaDuprh_NMxhQAsQ,2704
|
|
31
|
-
lamindb_setup/core/_settings_storage.py,sha256=
|
|
32
|
+
lamindb_setup/core/_settings_storage.py,sha256=k4XyJR6_KpUpQuBYZp4mEdABiT91gTTfbK7tAVwqZCA,13093
|
|
32
33
|
lamindb_setup/core/_settings_store.py,sha256=dagS5c7wAMRnuZTRfCU4sKaIOyF_HwAP5Fnnn8vphno,2084
|
|
33
34
|
lamindb_setup/core/_settings_user.py,sha256=P2lC4WDRAFfT-Xq3MlXJ-wMKIHCoGNhMTQfRGIAyUNQ,1344
|
|
34
35
|
lamindb_setup/core/_setup_bionty_sources.py,sha256=h_pBANsSGK6ujAFsG21mtADHVJoMLKDR4eGgRP4Fgls,3072
|
|
@@ -37,8 +38,8 @@ lamindb_setup/core/django.py,sha256=QUQm3zt5QIiD8uv6o9vbSm_bshqiSWzKSkgD3z2eJCg,
|
|
|
37
38
|
lamindb_setup/core/exceptions.py,sha256=eoI7AXgATgDVzgArtN7CUvpaMUC067vsBg5LHCsWzDM,305
|
|
38
39
|
lamindb_setup/core/hashing.py,sha256=7r96h5JBzuwfOR_gNNqTyWNPKMuiOUfBYwn6sCbZkf8,2269
|
|
39
40
|
lamindb_setup/core/types.py,sha256=bcYnZ0uM_2NXKJCl94Mmc-uYrQlRUUVKG3sK2N-F-N4,532
|
|
40
|
-
lamindb_setup/core/upath.py,sha256=
|
|
41
|
-
lamindb_setup-0.73.
|
|
42
|
-
lamindb_setup-0.73.
|
|
43
|
-
lamindb_setup-0.73.
|
|
44
|
-
lamindb_setup-0.73.
|
|
41
|
+
lamindb_setup/core/upath.py,sha256=dwudkTVsXuyjS-2xR16WomcWtXJAEfRZ0ZzFq8_EDhE,27157
|
|
42
|
+
lamindb_setup-0.73.1.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
|
|
43
|
+
lamindb_setup-0.73.1.dist-info/WHEEL,sha256=Sgu64hAMa6g5FdzHxXv9Xdse9yxpGGMeagVtPMWpJQY,99
|
|
44
|
+
lamindb_setup-0.73.1.dist-info/METADATA,sha256=7iNq1IHqO4W4cHuOXPr_4wG7SxruZvn8MTBHZ_fRNM0,1620
|
|
45
|
+
lamindb_setup-0.73.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|