lamindb_setup 0.77.3__py2.py3-none-any.whl → 0.77.5__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lamindb_setup/__init__.py +1 -1
- lamindb_setup/_cache.py +34 -34
- lamindb_setup/_check.py +7 -7
- lamindb_setup/_check_setup.py +79 -79
- lamindb_setup/_close.py +35 -35
- lamindb_setup/_connect_instance.py +431 -444
- lamindb_setup/_delete.py +141 -139
- lamindb_setup/_django.py +41 -41
- lamindb_setup/_entry_points.py +22 -22
- lamindb_setup/_exportdb.py +68 -68
- lamindb_setup/_importdb.py +50 -50
- lamindb_setup/_init_instance.py +417 -374
- lamindb_setup/_migrate.py +239 -239
- lamindb_setup/_register_instance.py +36 -36
- lamindb_setup/_schema.py +27 -27
- lamindb_setup/_schema_metadata.py +411 -411
- lamindb_setup/_set_managed_storage.py +55 -55
- lamindb_setup/_setup_user.py +137 -137
- lamindb_setup/_silence_loggers.py +44 -44
- lamindb_setup/core/__init__.py +21 -21
- lamindb_setup/core/_aws_credentials.py +151 -151
- lamindb_setup/core/_aws_storage.py +48 -48
- lamindb_setup/core/_deprecated.py +55 -55
- lamindb_setup/core/_docs.py +14 -14
- lamindb_setup/core/_hub_core.py +611 -590
- lamindb_setup/core/_hub_crud.py +211 -211
- lamindb_setup/core/_hub_utils.py +109 -109
- lamindb_setup/core/_private_django_api.py +88 -88
- lamindb_setup/core/_settings.py +138 -138
- lamindb_setup/core/_settings_instance.py +480 -467
- lamindb_setup/core/_settings_load.py +105 -105
- lamindb_setup/core/_settings_save.py +81 -81
- lamindb_setup/core/_settings_storage.py +412 -405
- lamindb_setup/core/_settings_store.py +75 -75
- lamindb_setup/core/_settings_user.py +53 -53
- lamindb_setup/core/_setup_bionty_sources.py +101 -101
- lamindb_setup/core/cloud_sqlite_locker.py +237 -232
- lamindb_setup/core/django.py +114 -114
- lamindb_setup/core/exceptions.py +12 -12
- lamindb_setup/core/hashing.py +114 -114
- lamindb_setup/core/types.py +19 -19
- lamindb_setup/core/upath.py +779 -779
- {lamindb_setup-0.77.3.dist-info → lamindb_setup-0.77.5.dist-info}/METADATA +1 -1
- lamindb_setup-0.77.5.dist-info/RECORD +47 -0
- {lamindb_setup-0.77.3.dist-info → lamindb_setup-0.77.5.dist-info}/WHEEL +1 -1
- lamindb_setup-0.77.3.dist-info/RECORD +0 -47
- {lamindb_setup-0.77.3.dist-info → lamindb_setup-0.77.5.dist-info}/LICENSE +0 -0
|
@@ -1,444 +1,431 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
import os
|
|
4
|
-
from typing import TYPE_CHECKING
|
|
5
|
-
from uuid import UUID
|
|
6
|
-
|
|
7
|
-
from lamin_utils import logger
|
|
8
|
-
|
|
9
|
-
from ._check_setup import _check_instance_setup
|
|
10
|
-
from ._close import close as close_instance
|
|
11
|
-
from ._init_instance import MESSAGE_NO_MULTIPLE_INSTANCE, load_from_isettings
|
|
12
|
-
from ._migrate import check_whether_migrations_in_sync
|
|
13
|
-
from ._silence_loggers import silence_loggers
|
|
14
|
-
from .core._hub_core import connect_instance as load_instance_from_hub
|
|
15
|
-
from .core._hub_core import connect_instance_new as load_instance_from_hub_edge
|
|
16
|
-
from .core._hub_utils import (
|
|
17
|
-
LaminDsn,
|
|
18
|
-
LaminDsnModel,
|
|
19
|
-
)
|
|
20
|
-
from .core._settings import settings
|
|
21
|
-
from .core._settings_instance import InstanceSettings
|
|
22
|
-
from .core._settings_load import load_instance_settings
|
|
23
|
-
from .core._settings_storage import StorageSettings
|
|
24
|
-
from .core._settings_store import instance_settings_file, settings_dir
|
|
25
|
-
from .core.cloud_sqlite_locker import unlock_cloud_sqlite_upon_exception
|
|
26
|
-
|
|
27
|
-
if TYPE_CHECKING:
|
|
28
|
-
from pathlib import Path
|
|
29
|
-
|
|
30
|
-
from .core.
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
#
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
"
|
|
40
|
-
"
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
and db_dsn_hub.
|
|
53
|
-
and db_dsn_hub.
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
# read
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
and
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
"
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
"
|
|
98
|
-
"
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
#
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
hub_result =
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
if
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
else:
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
def update_root_field_in_default_storage(isettings: InstanceSettings):
|
|
433
|
-
from lnschema_core.models import Storage
|
|
434
|
-
|
|
435
|
-
storages = Storage.objects.all()
|
|
436
|
-
if len(storages) != 1:
|
|
437
|
-
raise RuntimeError(
|
|
438
|
-
"You have several storage locations: Can't identify in which storage"
|
|
439
|
-
" location the root column is to be updated!"
|
|
440
|
-
)
|
|
441
|
-
storage = storages[0]
|
|
442
|
-
storage.root = isettings.storage.root_as_str
|
|
443
|
-
storage.save()
|
|
444
|
-
logger.save(f"updated storage root {storage.id} to {isettings.storage.root_as_str}")
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from typing import TYPE_CHECKING
|
|
5
|
+
from uuid import UUID
|
|
6
|
+
|
|
7
|
+
from lamin_utils import logger
|
|
8
|
+
|
|
9
|
+
from ._check_setup import _check_instance_setup
|
|
10
|
+
from ._close import close as close_instance
|
|
11
|
+
from ._init_instance import MESSAGE_NO_MULTIPLE_INSTANCE, load_from_isettings
|
|
12
|
+
from ._migrate import check_whether_migrations_in_sync
|
|
13
|
+
from ._silence_loggers import silence_loggers
|
|
14
|
+
from .core._hub_core import connect_instance as load_instance_from_hub
|
|
15
|
+
from .core._hub_core import connect_instance_new as load_instance_from_hub_edge
|
|
16
|
+
from .core._hub_utils import (
|
|
17
|
+
LaminDsn,
|
|
18
|
+
LaminDsnModel,
|
|
19
|
+
)
|
|
20
|
+
from .core._settings import settings
|
|
21
|
+
from .core._settings_instance import InstanceSettings
|
|
22
|
+
from .core._settings_load import load_instance_settings
|
|
23
|
+
from .core._settings_storage import StorageSettings
|
|
24
|
+
from .core._settings_store import instance_settings_file, settings_dir
|
|
25
|
+
from .core.cloud_sqlite_locker import unlock_cloud_sqlite_upon_exception
|
|
26
|
+
|
|
27
|
+
if TYPE_CHECKING:
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
|
|
30
|
+
from .core._settings_user import UserSettings
|
|
31
|
+
from .core.types import UPathStr
|
|
32
|
+
|
|
33
|
+
# this is for testing purposes only
|
|
34
|
+
# set to True only to test failed load
|
|
35
|
+
_TEST_FAILED_LOAD = False
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
INSTANCE_NOT_FOUND_MESSAGE = (
|
|
39
|
+
"'{owner}/{name}' not found:"
|
|
40
|
+
" '{hub_result}'\nCheck your permissions:"
|
|
41
|
+
" https://lamin.ai/{owner}/{name}"
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class InstanceNotFoundError(SystemExit):
|
|
46
|
+
pass
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def check_db_dsn_equal_up_to_credentials(db_dsn_hub, db_dsn_local):
|
|
50
|
+
return (
|
|
51
|
+
db_dsn_hub.scheme == db_dsn_local.scheme
|
|
52
|
+
and db_dsn_hub.host == db_dsn_local.host
|
|
53
|
+
and db_dsn_hub.database == db_dsn_local.database
|
|
54
|
+
and db_dsn_hub.port == db_dsn_local.port
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def update_db_using_local(
|
|
59
|
+
hub_instance_result: dict[str, str],
|
|
60
|
+
settings_file: Path,
|
|
61
|
+
db: str | None = None,
|
|
62
|
+
raise_permission_error=True,
|
|
63
|
+
) -> str | None:
|
|
64
|
+
db_updated = None
|
|
65
|
+
# check if postgres
|
|
66
|
+
if hub_instance_result["db_scheme"] == "postgresql":
|
|
67
|
+
db_dsn_hub = LaminDsnModel(db=hub_instance_result["db"])
|
|
68
|
+
if db is not None:
|
|
69
|
+
db_dsn_local = LaminDsnModel(db=db)
|
|
70
|
+
else:
|
|
71
|
+
# read directly from the environment
|
|
72
|
+
if os.getenv("LAMINDB_INSTANCE_DB") is not None:
|
|
73
|
+
logger.important("loading db URL from env variable LAMINDB_INSTANCE_DB")
|
|
74
|
+
db_dsn_local = LaminDsnModel(db=os.getenv("LAMINDB_INSTANCE_DB"))
|
|
75
|
+
# read from a cached settings file in case the hub result is only
|
|
76
|
+
# read level or inexistent
|
|
77
|
+
elif settings_file.exists() and (
|
|
78
|
+
db_dsn_hub.db.user is None
|
|
79
|
+
or (db_dsn_hub.db.user is not None and "read" in db_dsn_hub.db.user)
|
|
80
|
+
):
|
|
81
|
+
isettings = load_instance_settings(settings_file)
|
|
82
|
+
db_dsn_local = LaminDsnModel(db=isettings.db)
|
|
83
|
+
else:
|
|
84
|
+
# just take the default hub result and ensure there is actually a user
|
|
85
|
+
if (
|
|
86
|
+
db_dsn_hub.db.user == "none"
|
|
87
|
+
and db_dsn_hub.db.password == "none"
|
|
88
|
+
and raise_permission_error
|
|
89
|
+
):
|
|
90
|
+
raise PermissionError(
|
|
91
|
+
"No database access, please ask your admin to provide you with"
|
|
92
|
+
" a DB URL and pass it via --db <db_url>"
|
|
93
|
+
)
|
|
94
|
+
db_dsn_local = db_dsn_hub
|
|
95
|
+
if not check_db_dsn_equal_up_to_credentials(db_dsn_hub.db, db_dsn_local.db):
|
|
96
|
+
raise ValueError(
|
|
97
|
+
"The local differs from the hub database information:\n 1. did you"
|
|
98
|
+
" pass a wrong db URL with --db?\n 2. did your database get updated by"
|
|
99
|
+
" an admin?\nConsider deleting your cached database environment:\nrm"
|
|
100
|
+
f" {settings_file.as_posix()}"
|
|
101
|
+
)
|
|
102
|
+
db_updated = LaminDsn.build(
|
|
103
|
+
scheme=db_dsn_hub.db.scheme,
|
|
104
|
+
user=db_dsn_local.db.user,
|
|
105
|
+
password=db_dsn_local.db.password,
|
|
106
|
+
host=db_dsn_hub.db.host, # type: ignore
|
|
107
|
+
port=db_dsn_hub.db.port,
|
|
108
|
+
database=db_dsn_hub.db.database,
|
|
109
|
+
)
|
|
110
|
+
return db_updated
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _connect_instance(
|
|
114
|
+
owner: str,
|
|
115
|
+
name: str,
|
|
116
|
+
*,
|
|
117
|
+
db: str | None = None,
|
|
118
|
+
raise_permission_error: bool = True,
|
|
119
|
+
access_token: str | None = None,
|
|
120
|
+
) -> InstanceSettings:
|
|
121
|
+
settings_file = instance_settings_file(name, owner)
|
|
122
|
+
make_hub_request = True
|
|
123
|
+
if settings_file.exists():
|
|
124
|
+
isettings = load_instance_settings(settings_file)
|
|
125
|
+
# skip hub request for a purely local instance
|
|
126
|
+
make_hub_request = isettings.is_remote
|
|
127
|
+
if make_hub_request:
|
|
128
|
+
# the following will return a string if the instance does not exist
|
|
129
|
+
# on the hub
|
|
130
|
+
# do not call hub if the user is anonymous
|
|
131
|
+
if owner != "anonymous":
|
|
132
|
+
if settings.user.handle in {"Koncopd", "sunnyosun", "falexwolf"}:
|
|
133
|
+
hub_result = load_instance_from_hub_edge(
|
|
134
|
+
owner=owner, name=name, access_token=access_token
|
|
135
|
+
)
|
|
136
|
+
else:
|
|
137
|
+
hub_result = load_instance_from_hub(
|
|
138
|
+
owner=owner, name=name, access_token=access_token
|
|
139
|
+
)
|
|
140
|
+
else:
|
|
141
|
+
hub_result = "anonymous-user"
|
|
142
|
+
# if hub_result is not a string, it means it made a request
|
|
143
|
+
# that successfully returned metadata
|
|
144
|
+
if not isinstance(hub_result, str):
|
|
145
|
+
instance_result, storage_result = hub_result
|
|
146
|
+
db_updated = update_db_using_local(
|
|
147
|
+
instance_result,
|
|
148
|
+
settings_file,
|
|
149
|
+
db=db,
|
|
150
|
+
raise_permission_error=raise_permission_error,
|
|
151
|
+
)
|
|
152
|
+
ssettings = StorageSettings(
|
|
153
|
+
root=storage_result["root"],
|
|
154
|
+
region=storage_result["region"],
|
|
155
|
+
uid=storage_result["lnid"],
|
|
156
|
+
uuid=UUID(storage_result["id"]),
|
|
157
|
+
instance_id=UUID(instance_result["id"]),
|
|
158
|
+
)
|
|
159
|
+
isettings = InstanceSettings(
|
|
160
|
+
id=UUID(instance_result["id"]),
|
|
161
|
+
owner=owner,
|
|
162
|
+
name=name,
|
|
163
|
+
storage=ssettings,
|
|
164
|
+
db=db_updated,
|
|
165
|
+
schema=instance_result["schema_str"],
|
|
166
|
+
git_repo=instance_result["git_repo"],
|
|
167
|
+
keep_artifacts_local=bool(instance_result["keep_artifacts_local"]),
|
|
168
|
+
is_on_hub=True,
|
|
169
|
+
)
|
|
170
|
+
check_whether_migrations_in_sync(instance_result["lamindb_version"])
|
|
171
|
+
else:
|
|
172
|
+
if hub_result != "anonymous-user":
|
|
173
|
+
message = INSTANCE_NOT_FOUND_MESSAGE.format(
|
|
174
|
+
owner=owner, name=name, hub_result=hub_result
|
|
175
|
+
)
|
|
176
|
+
else:
|
|
177
|
+
message = "It is not possible to load an anonymous-owned instance from the hub"
|
|
178
|
+
if settings_file.exists():
|
|
179
|
+
isettings = load_instance_settings(settings_file)
|
|
180
|
+
if isettings.is_remote:
|
|
181
|
+
raise InstanceNotFoundError(message)
|
|
182
|
+
else:
|
|
183
|
+
raise InstanceNotFoundError(message)
|
|
184
|
+
return isettings
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
@unlock_cloud_sqlite_upon_exception(ignore_prev_locker=True)
|
|
188
|
+
def connect(slug: str, **kwargs) -> str | tuple | None:
|
|
189
|
+
"""Connect to instance.
|
|
190
|
+
|
|
191
|
+
Args:
|
|
192
|
+
slug: The instance slug `account_handle/instance_name` or URL.
|
|
193
|
+
If the instance is owned by you, it suffices to pass the instance name.
|
|
194
|
+
"""
|
|
195
|
+
# validate kwargs
|
|
196
|
+
valid_kwargs = {
|
|
197
|
+
"_db",
|
|
198
|
+
"_write_settings",
|
|
199
|
+
"_raise_not_found_error",
|
|
200
|
+
"_test",
|
|
201
|
+
"_user",
|
|
202
|
+
}
|
|
203
|
+
for kwarg in kwargs:
|
|
204
|
+
if kwarg not in valid_kwargs:
|
|
205
|
+
raise TypeError(f"connect() got unexpected keyword argument '{kwarg}'")
|
|
206
|
+
|
|
207
|
+
isettings: InstanceSettings = None # type: ignore
|
|
208
|
+
# _db is still needed because it is called in init
|
|
209
|
+
_db: str | None = kwargs.get("_db", None)
|
|
210
|
+
_write_settings: bool = kwargs.get("_write_settings", True)
|
|
211
|
+
_raise_not_found_error: bool = kwargs.get("_raise_not_found_error", True)
|
|
212
|
+
_test: bool = kwargs.get("_test", False)
|
|
213
|
+
|
|
214
|
+
access_token: str | None = None
|
|
215
|
+
_user: UserSettings | None = kwargs.get("_user", None)
|
|
216
|
+
if _user is not None:
|
|
217
|
+
access_token = _user.access_token
|
|
218
|
+
|
|
219
|
+
try:
|
|
220
|
+
owner, name = get_owner_name_from_identifier(slug)
|
|
221
|
+
|
|
222
|
+
if _check_instance_setup() and not _test:
|
|
223
|
+
if (
|
|
224
|
+
settings._instance_exists
|
|
225
|
+
and f"{owner}/{name}" == settings.instance.slug
|
|
226
|
+
):
|
|
227
|
+
logger.info(f"connected lamindb: {settings.instance.slug}")
|
|
228
|
+
return None
|
|
229
|
+
else:
|
|
230
|
+
raise RuntimeError(MESSAGE_NO_MULTIPLE_INSTANCE)
|
|
231
|
+
elif (
|
|
232
|
+
_write_settings
|
|
233
|
+
and settings._instance_exists
|
|
234
|
+
and f"{owner}/{name}" != settings.instance.slug
|
|
235
|
+
):
|
|
236
|
+
close_instance(mute=True)
|
|
237
|
+
|
|
238
|
+
try:
|
|
239
|
+
isettings = _connect_instance(
|
|
240
|
+
owner, name, db=_db, access_token=access_token
|
|
241
|
+
)
|
|
242
|
+
except InstanceNotFoundError as e:
|
|
243
|
+
if _raise_not_found_error:
|
|
244
|
+
raise e
|
|
245
|
+
else:
|
|
246
|
+
return "instance-not-found"
|
|
247
|
+
if isinstance(isettings, str):
|
|
248
|
+
return isettings
|
|
249
|
+
# at this point we have checked already that isettings is not a string
|
|
250
|
+
# _user is passed to lock cloud sqite for this user in isettings._load_db()
|
|
251
|
+
# has no effect if _user is None or if not cloud sqlite instance
|
|
252
|
+
isettings._locker_user = _user
|
|
253
|
+
isettings._persist(write_to_disk=_write_settings)
|
|
254
|
+
if _test:
|
|
255
|
+
return None
|
|
256
|
+
silence_loggers()
|
|
257
|
+
check, msg = isettings._load_db()
|
|
258
|
+
if not check:
|
|
259
|
+
local_db = (
|
|
260
|
+
isettings._is_cloud_sqlite and isettings._sqlite_file_local.exists()
|
|
261
|
+
)
|
|
262
|
+
if local_db:
|
|
263
|
+
logger.warning(
|
|
264
|
+
"SQLite file does not exist in the cloud, but exists locally:"
|
|
265
|
+
f" {isettings._sqlite_file_local}\nTo push the file to the cloud,"
|
|
266
|
+
" call: lamin load --unload"
|
|
267
|
+
)
|
|
268
|
+
elif _raise_not_found_error:
|
|
269
|
+
raise SystemExit(msg)
|
|
270
|
+
else:
|
|
271
|
+
logger.warning(
|
|
272
|
+
f"instance exists with id {isettings._id.hex}, but database is not"
|
|
273
|
+
" loadable: re-initializing"
|
|
274
|
+
)
|
|
275
|
+
return "instance-corrupted-or-deleted"
|
|
276
|
+
# this is for testing purposes only
|
|
277
|
+
if _TEST_FAILED_LOAD:
|
|
278
|
+
raise RuntimeError("Technical testing error.")
|
|
279
|
+
|
|
280
|
+
# below is for backfilling the instance_uid value
|
|
281
|
+
# we'll enable it once more people migrated to 0.71.0
|
|
282
|
+
# ssettings_record = isettings.storage.record
|
|
283
|
+
# if ssettings_record.instance_uid is None:
|
|
284
|
+
# ssettings_record.instance_uid = isettings.uid
|
|
285
|
+
# # try saving if not read-only access
|
|
286
|
+
# try:
|
|
287
|
+
# ssettings_record.save()
|
|
288
|
+
# # raised by django when the access is denied
|
|
289
|
+
# except ProgrammingError:
|
|
290
|
+
# pass
|
|
291
|
+
load_from_isettings(isettings, user=_user, write_settings=_write_settings)
|
|
292
|
+
except Exception as e:
|
|
293
|
+
if isettings is not None:
|
|
294
|
+
if _write_settings:
|
|
295
|
+
isettings._get_settings_file().unlink(missing_ok=True) # type: ignore
|
|
296
|
+
settings._instance_settings = None
|
|
297
|
+
raise e
|
|
298
|
+
# rename lnschema_bionty to bionty for sql tables
|
|
299
|
+
if "bionty" in isettings.schema:
|
|
300
|
+
no_lnschema_bionty_file = (
|
|
301
|
+
settings_dir / f"no_lnschema_bionty-{isettings.slug.replace('/', '')}"
|
|
302
|
+
)
|
|
303
|
+
if not no_lnschema_bionty_file.exists():
|
|
304
|
+
migrate_lnschema_bionty(
|
|
305
|
+
isettings, no_lnschema_bionty_file, write_file=_write_settings
|
|
306
|
+
)
|
|
307
|
+
return None
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def load(slug: str) -> str | tuple | None:
|
|
311
|
+
"""Connect to instance and set ``auto-connect`` to true.
|
|
312
|
+
|
|
313
|
+
This is exactly the same as ``ln.connect()`` except for that
|
|
314
|
+
``ln.connect()`` doesn't change the state of ``auto-connect``.
|
|
315
|
+
"""
|
|
316
|
+
result = connect(slug)
|
|
317
|
+
settings.auto_connect = True
|
|
318
|
+
return result
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def migrate_lnschema_bionty(
|
|
322
|
+
isettings: InstanceSettings, no_lnschema_bionty_file: Path, write_file: bool = True
|
|
323
|
+
):
|
|
324
|
+
"""Migrate lnschema_bionty tables to bionty tables if bionty_source doesn't exist.
|
|
325
|
+
|
|
326
|
+
:param db_uri: str, database URI (e.g., 'sqlite:///path/to/db.sqlite' or 'postgresql://user:password@host:port/dbname')
|
|
327
|
+
"""
|
|
328
|
+
from urllib.parse import urlparse
|
|
329
|
+
|
|
330
|
+
parsed_uri = urlparse(isettings.db)
|
|
331
|
+
db_type = parsed_uri.scheme
|
|
332
|
+
|
|
333
|
+
if db_type == "sqlite":
|
|
334
|
+
import sqlite3
|
|
335
|
+
|
|
336
|
+
conn = sqlite3.connect(parsed_uri.path)
|
|
337
|
+
elif db_type in ["postgresql", "postgres"]:
|
|
338
|
+
import psycopg2
|
|
339
|
+
|
|
340
|
+
conn = psycopg2.connect(isettings.db)
|
|
341
|
+
else:
|
|
342
|
+
raise ValueError("Unsupported database type. Use 'sqlite' or 'postgresql' URI.")
|
|
343
|
+
|
|
344
|
+
cur = conn.cursor()
|
|
345
|
+
|
|
346
|
+
try:
|
|
347
|
+
# check if bionty_source table exists
|
|
348
|
+
if db_type == "sqlite":
|
|
349
|
+
cur.execute(
|
|
350
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name='bionty_source'"
|
|
351
|
+
)
|
|
352
|
+
migrated = cur.fetchone() is not None
|
|
353
|
+
|
|
354
|
+
# tables that need to be renamed
|
|
355
|
+
cur.execute(
|
|
356
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'lnschema_bionty_%'"
|
|
357
|
+
)
|
|
358
|
+
tables_to_rename = [
|
|
359
|
+
row[0][len("lnschema_bionty_") :] for row in cur.fetchall()
|
|
360
|
+
]
|
|
361
|
+
else: # postgres
|
|
362
|
+
cur.execute(
|
|
363
|
+
"SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'bionty_source')"
|
|
364
|
+
)
|
|
365
|
+
migrated = cur.fetchone()[0]
|
|
366
|
+
|
|
367
|
+
# tables that need to be renamed
|
|
368
|
+
cur.execute(
|
|
369
|
+
"SELECT table_name FROM information_schema.tables WHERE table_name LIKE 'lnschema_bionty_%'"
|
|
370
|
+
)
|
|
371
|
+
tables_to_rename = [
|
|
372
|
+
row[0][len("lnschema_bionty_") :] for row in cur.fetchall()
|
|
373
|
+
]
|
|
374
|
+
|
|
375
|
+
if migrated:
|
|
376
|
+
if write_file:
|
|
377
|
+
no_lnschema_bionty_file.touch(exist_ok=True)
|
|
378
|
+
else:
|
|
379
|
+
try:
|
|
380
|
+
# rename tables only if bionty_source doesn't exist and there are tables to rename
|
|
381
|
+
for table in tables_to_rename:
|
|
382
|
+
if db_type == "sqlite":
|
|
383
|
+
cur.execute(
|
|
384
|
+
f"ALTER TABLE lnschema_bionty_{table} RENAME TO bionty_{table}"
|
|
385
|
+
)
|
|
386
|
+
else: # postgres
|
|
387
|
+
cur.execute(
|
|
388
|
+
f"ALTER TABLE lnschema_bionty_{table} RENAME TO bionty_{table};"
|
|
389
|
+
)
|
|
390
|
+
|
|
391
|
+
# update django_migrations table
|
|
392
|
+
cur.execute(
|
|
393
|
+
"UPDATE django_migrations SET app = 'bionty' WHERE app = 'lnschema_bionty'"
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
logger.warning(
|
|
397
|
+
"Please uninstall lnschema-bionty via `pip uninstall lnschema-bionty`!"
|
|
398
|
+
)
|
|
399
|
+
if write_file:
|
|
400
|
+
no_lnschema_bionty_file.touch(exist_ok=True)
|
|
401
|
+
except Exception:
|
|
402
|
+
# read-only users can't rename tables
|
|
403
|
+
pass
|
|
404
|
+
|
|
405
|
+
conn.commit()
|
|
406
|
+
|
|
407
|
+
except Exception as e:
|
|
408
|
+
print(f"An error occurred: {e}")
|
|
409
|
+
conn.rollback()
|
|
410
|
+
|
|
411
|
+
finally:
|
|
412
|
+
# close the cursor and connection
|
|
413
|
+
cur.close()
|
|
414
|
+
conn.close()
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def get_owner_name_from_identifier(identifier: str):
|
|
418
|
+
if "/" in identifier:
|
|
419
|
+
if identifier.startswith("https://lamin.ai/"):
|
|
420
|
+
identifier = identifier.replace("https://lamin.ai/", "")
|
|
421
|
+
split = identifier.split("/")
|
|
422
|
+
if len(split) > 2:
|
|
423
|
+
raise ValueError(
|
|
424
|
+
"The instance identifier needs to be 'owner/name', the instance name"
|
|
425
|
+
" (owner is current user) or the URL: https://lamin.ai/owner/name."
|
|
426
|
+
)
|
|
427
|
+
owner, name = split
|
|
428
|
+
else:
|
|
429
|
+
owner = settings.user.handle
|
|
430
|
+
name = identifier
|
|
431
|
+
return owner, name
|