lamindb_setup 1.10.2__py3-none-any.whl → 1.12.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lamindb_setup/__init__.py +1 -1
- lamindb_setup/_connect_instance.py +52 -38
- lamindb_setup/_delete.py +3 -1
- lamindb_setup/_init_instance.py +22 -24
- lamindb_setup/_migrate.py +8 -0
- lamindb_setup/_schema_metadata.py +20 -16
- lamindb_setup/core/_aws_options.py +2 -4
- lamindb_setup/core/_aws_storage.py +2 -3
- lamindb_setup/core/_hub_client.py +84 -44
- lamindb_setup/core/_hub_core.py +35 -12
- lamindb_setup/core/_settings.py +24 -5
- lamindb_setup/core/_settings_instance.py +7 -7
- lamindb_setup/core/_settings_save.py +2 -2
- lamindb_setup/core/_settings_store.py +9 -9
- lamindb_setup/core/django.py +4 -1
- lamindb_setup/core/upath.py +17 -10
- {lamindb_setup-1.10.2.dist-info → lamindb_setup-1.12.0.dist-info}/METADATA +3 -2
- {lamindb_setup-1.10.2.dist-info → lamindb_setup-1.12.0.dist-info}/RECORD +20 -20
- {lamindb_setup-1.10.2.dist-info → lamindb_setup-1.12.0.dist-info}/LICENSE +0 -0
- {lamindb_setup-1.10.2.dist-info → lamindb_setup-1.12.0.dist-info}/WHEEL +0 -0
lamindb_setup/__init__.py
CHANGED
|
@@ -104,6 +104,7 @@ def _connect_instance(
|
|
|
104
104
|
db: str | None = None,
|
|
105
105
|
raise_permission_error: bool = True,
|
|
106
106
|
use_root_db_user: bool = False,
|
|
107
|
+
use_proxy_db: bool = False,
|
|
107
108
|
access_token: str | None = None,
|
|
108
109
|
) -> InstanceSettings:
|
|
109
110
|
settings_file = instance_settings_file(name, owner)
|
|
@@ -118,8 +119,7 @@ def _connect_instance(
|
|
|
118
119
|
if db is not None and isettings.dialect == "postgresql":
|
|
119
120
|
isettings._db = db
|
|
120
121
|
if make_hub_request:
|
|
121
|
-
# the following will return a string if the instance does not exist
|
|
122
|
-
# on the hub
|
|
122
|
+
# the following will return a string if the instance does not exist on the hub
|
|
123
123
|
# do not call hub if the user is anonymous
|
|
124
124
|
if owner != "anonymous":
|
|
125
125
|
hub_result = connect_instance_hub(
|
|
@@ -127,6 +127,7 @@ def _connect_instance(
|
|
|
127
127
|
name=name,
|
|
128
128
|
access_token=access_token,
|
|
129
129
|
use_root_db_user=use_root_db_user,
|
|
130
|
+
use_proxy_db=use_proxy_db,
|
|
130
131
|
)
|
|
131
132
|
else:
|
|
132
133
|
hub_result = "anonymous-user"
|
|
@@ -192,11 +193,10 @@ def reset_django_module_variables():
|
|
|
192
193
|
# import lamindb as ln
|
|
193
194
|
# ln.connect(...)
|
|
194
195
|
#
|
|
195
|
-
# Then it will **not** work and the `ln` variable becomes stale and hold a reference
|
|
196
|
-
# to the old classes
|
|
196
|
+
# Then it will **not** work and the `ln` variable becomes stale and hold a reference to the old classes
|
|
197
197
|
# Other functions that dynamically import are no problem because the variables
|
|
198
198
|
# are automatically refreshed when the function runs the next time after ln.connect() was called
|
|
199
|
-
logger.
|
|
199
|
+
logger.debug("resetting django module variables")
|
|
200
200
|
|
|
201
201
|
# django.apps needs to be a local import to refresh variables
|
|
202
202
|
from django.apps import apps
|
|
@@ -242,11 +242,15 @@ def reset_django_module_variables():
|
|
|
242
242
|
continue
|
|
243
243
|
|
|
244
244
|
|
|
245
|
-
def _connect_cli(
|
|
245
|
+
def _connect_cli(
|
|
246
|
+
instance: str, use_root_db_user: bool = False, use_proxy_db: bool = False
|
|
247
|
+
) -> None:
|
|
246
248
|
from lamindb_setup import settings as settings_
|
|
247
249
|
|
|
248
250
|
owner, name = get_owner_name_from_identifier(instance)
|
|
249
|
-
isettings = _connect_instance(
|
|
251
|
+
isettings = _connect_instance(
|
|
252
|
+
owner, name, use_root_db_user=use_root_db_user, use_proxy_db=use_proxy_db
|
|
253
|
+
)
|
|
250
254
|
isettings._persist(write_to_disk=True)
|
|
251
255
|
if not isettings.is_on_hub or isettings._is_cloud_sqlite:
|
|
252
256
|
# there are two reasons to call the full-blown connect
|
|
@@ -259,6 +263,36 @@ def _connect_cli(instance: str, use_root_db_user: bool = False) -> None:
|
|
|
259
263
|
return None
|
|
260
264
|
|
|
261
265
|
|
|
266
|
+
def validate_connection_state(
|
|
267
|
+
owner: str, name: str, use_root_db_user: bool = False
|
|
268
|
+
) -> None:
|
|
269
|
+
from django.db import connection
|
|
270
|
+
|
|
271
|
+
if (
|
|
272
|
+
settings._instance_exists
|
|
273
|
+
and f"{owner}/{name}" == settings.instance.slug
|
|
274
|
+
# below is to ensure that if another process interferes
|
|
275
|
+
# we don't use the in-memory mock database
|
|
276
|
+
# could be made more specific by checking whether the django
|
|
277
|
+
# configured database is the same as the one in settings
|
|
278
|
+
and connection.settings_dict["NAME"] != ":memory:"
|
|
279
|
+
and not use_root_db_user # always re-connect for root db user
|
|
280
|
+
):
|
|
281
|
+
logger.important(
|
|
282
|
+
f"doing nothing, already connected lamindb: {settings.instance.slug}"
|
|
283
|
+
)
|
|
284
|
+
return None
|
|
285
|
+
else:
|
|
286
|
+
if settings._instance_exists and settings.instance.slug != "none/none":
|
|
287
|
+
import lamindb as ln
|
|
288
|
+
|
|
289
|
+
if ln.context.transform is not None:
|
|
290
|
+
raise CannotSwitchDefaultInstance(
|
|
291
|
+
"Cannot switch default instance while `ln.track()` is live: call `ln.finish()`"
|
|
292
|
+
)
|
|
293
|
+
reset_django()
|
|
294
|
+
|
|
295
|
+
|
|
262
296
|
@unlock_cloud_sqlite_upon_exception(ignore_prev_locker=True)
|
|
263
297
|
def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
|
|
264
298
|
"""Connect to an instance.
|
|
@@ -274,6 +308,7 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
|
|
|
274
308
|
# validate kwargs
|
|
275
309
|
valid_kwargs = {
|
|
276
310
|
"use_root_db_user",
|
|
311
|
+
"use_proxy_db",
|
|
277
312
|
"_db",
|
|
278
313
|
"_write_settings",
|
|
279
314
|
"_raise_not_found_error",
|
|
@@ -284,15 +319,18 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
|
|
|
284
319
|
for kwarg in kwargs:
|
|
285
320
|
if kwarg not in valid_kwargs:
|
|
286
321
|
raise TypeError(f"connect() got unexpected keyword argument '{kwarg}'")
|
|
287
|
-
|
|
288
|
-
# _db is still needed because it is called in init
|
|
322
|
+
|
|
289
323
|
use_root_db_user: bool = kwargs.get("use_root_db_user", False)
|
|
324
|
+
use_proxy_db = kwargs.get("use_proxy_db", False)
|
|
325
|
+
# _db is still needed because it is called in init
|
|
290
326
|
_db: str | None = kwargs.get("_db", None)
|
|
291
327
|
_write_settings: bool = kwargs.get("_write_settings", False)
|
|
292
328
|
_raise_not_found_error: bool = kwargs.get("_raise_not_found_error", True)
|
|
293
329
|
_reload_lamindb: bool = kwargs.get("_reload_lamindb", True)
|
|
294
330
|
_test: bool = kwargs.get("_test", False)
|
|
295
331
|
|
|
332
|
+
isettings: InstanceSettings = None # type: ignore
|
|
333
|
+
|
|
296
334
|
access_token: str | None = None
|
|
297
335
|
_user: UserSettings | None = kwargs.get("_user", None)
|
|
298
336
|
if _user is not None:
|
|
@@ -317,36 +355,11 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
|
|
|
317
355
|
if _db is not None and isettings.dialect == "postgresql":
|
|
318
356
|
isettings._db = _db
|
|
319
357
|
else:
|
|
320
|
-
from django.db import connection
|
|
321
|
-
|
|
322
358
|
owner, name = get_owner_name_from_identifier(instance)
|
|
323
359
|
if _check_instance_setup() and not _test:
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
# below is to ensure that if another process interferes
|
|
328
|
-
# we don't use the in-memory mock database
|
|
329
|
-
# could be made more specific by checking whether the django
|
|
330
|
-
# configured database is the same as the one in settings
|
|
331
|
-
and connection.settings_dict["NAME"] != ":memory:"
|
|
332
|
-
and not use_root_db_user # always re-connect for root db user
|
|
333
|
-
):
|
|
334
|
-
logger.important(
|
|
335
|
-
f"doing nothing, already connected lamindb: {settings.instance.slug}"
|
|
336
|
-
)
|
|
337
|
-
return None
|
|
338
|
-
else:
|
|
339
|
-
if (
|
|
340
|
-
settings._instance_exists
|
|
341
|
-
and settings.instance.slug != "none/none"
|
|
342
|
-
):
|
|
343
|
-
import lamindb as ln
|
|
344
|
-
|
|
345
|
-
if ln.context.transform is not None:
|
|
346
|
-
raise CannotSwitchDefaultInstance(
|
|
347
|
-
"Cannot switch default instance while `ln.track()` is live: call `ln.finish()`"
|
|
348
|
-
)
|
|
349
|
-
reset_django()
|
|
360
|
+
validate_connection_state(
|
|
361
|
+
owner, name, use_root_db_user=use_root_db_user
|
|
362
|
+
)
|
|
350
363
|
elif (
|
|
351
364
|
_write_settings
|
|
352
365
|
and settings._instance_exists
|
|
@@ -362,6 +375,7 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
|
|
|
362
375
|
db=_db,
|
|
363
376
|
access_token=access_token,
|
|
364
377
|
use_root_db_user=use_root_db_user,
|
|
378
|
+
use_proxy_db=use_proxy_db,
|
|
365
379
|
)
|
|
366
380
|
except InstanceNotFoundError as e:
|
|
367
381
|
if _raise_not_found_error:
|
|
@@ -416,7 +430,7 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
|
|
|
416
430
|
return None
|
|
417
431
|
|
|
418
432
|
|
|
419
|
-
def get_owner_name_from_identifier(identifier: str):
|
|
433
|
+
def get_owner_name_from_identifier(identifier: str) -> tuple[str, str]:
|
|
420
434
|
if "/" in identifier:
|
|
421
435
|
if identifier.startswith("https://lamin.ai/"):
|
|
422
436
|
identifier = identifier.replace("https://lamin.ai/", "")
|
lamindb_setup/_delete.py
CHANGED
|
@@ -21,6 +21,8 @@ if TYPE_CHECKING:
|
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
def delete_cache(isettings: InstanceSettings):
|
|
24
|
+
if isettings.storage is None:
|
|
25
|
+
return
|
|
24
26
|
# avoid init of root
|
|
25
27
|
root = isettings.storage._root_init
|
|
26
28
|
if not isinstance(root, LocalPathClasses):
|
|
@@ -40,7 +42,7 @@ def delete_by_isettings(isettings: InstanceSettings) -> None:
|
|
|
40
42
|
if settings_file.exists():
|
|
41
43
|
settings_file.unlink()
|
|
42
44
|
delete_cache(isettings)
|
|
43
|
-
if isettings.dialect == "sqlite":
|
|
45
|
+
if isettings.dialect == "sqlite" and isettings.storage is not None:
|
|
44
46
|
try:
|
|
45
47
|
if isettings._sqlite_file.exists():
|
|
46
48
|
isettings._sqlite_file.unlink()
|
lamindb_setup/_init_instance.py
CHANGED
|
@@ -16,12 +16,13 @@ from ._silence_loggers import silence_loggers
|
|
|
16
16
|
from .core import InstanceSettings
|
|
17
17
|
from .core._docs import doc_args
|
|
18
18
|
from .core._settings import settings
|
|
19
|
-
from .core._settings_instance import check_is_instance_remote
|
|
19
|
+
from .core._settings_instance import check_is_instance_remote
|
|
20
20
|
from .core._settings_storage import StorageSettings, init_storage
|
|
21
21
|
from .core.upath import UPath
|
|
22
22
|
from .errors import CannotSwitchDefaultInstance
|
|
23
23
|
|
|
24
24
|
if TYPE_CHECKING:
|
|
25
|
+
from lamindb.models import Storage
|
|
25
26
|
from pydantic import PostgresDsn
|
|
26
27
|
|
|
27
28
|
from .core._settings_user import UserSettings
|
|
@@ -49,7 +50,7 @@ def get_schema_module_name(module_name, raise_import_error: bool = True) -> str
|
|
|
49
50
|
return None
|
|
50
51
|
|
|
51
52
|
|
|
52
|
-
def register_storage_in_instance(ssettings: StorageSettings):
|
|
53
|
+
def register_storage_in_instance(ssettings: StorageSettings) -> Storage:
|
|
53
54
|
from lamindb.models import Storage
|
|
54
55
|
|
|
55
56
|
# how do we ensure that this function is only called passing
|
|
@@ -69,7 +70,7 @@ def register_storage_in_instance(ssettings: StorageSettings):
|
|
|
69
70
|
return storage
|
|
70
71
|
|
|
71
72
|
|
|
72
|
-
def register_user(usettings: UserSettings, update_user: bool = True):
|
|
73
|
+
def register_user(usettings: UserSettings, update_user: bool = True) -> None:
|
|
73
74
|
from lamindb.models import User
|
|
74
75
|
|
|
75
76
|
if not update_user and User.objects.filter(uid=usettings.uid).exists():
|
|
@@ -91,7 +92,9 @@ def register_user(usettings: UserSettings, update_user: bool = True):
|
|
|
91
92
|
pass
|
|
92
93
|
|
|
93
94
|
|
|
94
|
-
def register_initial_records(
|
|
95
|
+
def register_initial_records(
|
|
96
|
+
isettings: InstanceSettings, usettings: UserSettings
|
|
97
|
+
) -> None:
|
|
95
98
|
"""Register space, user & storage in DB."""
|
|
96
99
|
from django.db.utils import OperationalError
|
|
97
100
|
from lamindb.models import Branch, Space
|
|
@@ -245,6 +248,15 @@ def init(
|
|
|
245
248
|
See Also:
|
|
246
249
|
Init an instance for via the CLI, see `here <https://docs.lamin.ai/cli#init>`__.
|
|
247
250
|
"""
|
|
251
|
+
from ._check_setup import _check_instance_setup
|
|
252
|
+
from ._connect_instance import (
|
|
253
|
+
reset_django_module_variables,
|
|
254
|
+
validate_connection_state,
|
|
255
|
+
)
|
|
256
|
+
from .core._hub_core import init_instance_hub
|
|
257
|
+
|
|
258
|
+
silence_loggers()
|
|
259
|
+
|
|
248
260
|
isettings = None
|
|
249
261
|
ssettings = None
|
|
250
262
|
|
|
@@ -261,22 +273,6 @@ def init(
|
|
|
261
273
|
access_token: str | None = None if _user is None else _user.access_token
|
|
262
274
|
|
|
263
275
|
try:
|
|
264
|
-
silence_loggers()
|
|
265
|
-
from ._check_setup import _check_instance_setup
|
|
266
|
-
|
|
267
|
-
if _check_instance_setup() and not _test:
|
|
268
|
-
from lamindb_setup.core.django import reset_django
|
|
269
|
-
|
|
270
|
-
if settings._instance_exists:
|
|
271
|
-
raise CannotSwitchDefaultInstance(
|
|
272
|
-
"Cannot init new instance after connecting to an existing instance."
|
|
273
|
-
)
|
|
274
|
-
reset_django()
|
|
275
|
-
elif _write_settings:
|
|
276
|
-
disconnect(mute=True)
|
|
277
|
-
from ._connect_instance import reset_django_module_variables
|
|
278
|
-
from .core._hub_core import init_instance_hub
|
|
279
|
-
|
|
280
276
|
name_str, instance_id, instance_state, _ = validate_init_args(
|
|
281
277
|
storage=storage,
|
|
282
278
|
name=name,
|
|
@@ -288,6 +284,10 @@ def init(
|
|
|
288
284
|
)
|
|
289
285
|
if instance_state == "connected":
|
|
290
286
|
return None
|
|
287
|
+
if _check_instance_setup() and not _test:
|
|
288
|
+
validate_connection_state(user_handle, name_str)
|
|
289
|
+
elif _write_settings:
|
|
290
|
+
disconnect(mute=True)
|
|
291
291
|
isettings = InstanceSettings(
|
|
292
292
|
id=instance_id, # type: ignore
|
|
293
293
|
owner=user_handle,
|
|
@@ -378,10 +378,8 @@ def load_from_isettings(
|
|
|
378
378
|
else:
|
|
379
379
|
# when loading, django is already set up
|
|
380
380
|
#
|
|
381
|
-
# only register user if the instance is connected
|
|
382
|
-
# for the
|
|
383
|
-
# this is our best proxy for that the user might not
|
|
384
|
-
# yet be registered
|
|
381
|
+
# only register user if the instance is connected for the first time in an environment
|
|
382
|
+
# this is our best proxy for that the user might not yet be registered
|
|
385
383
|
if not isettings._get_settings_file().exists():
|
|
386
384
|
# do not try to update the user on fine grained access instances
|
|
387
385
|
# this is blocked anyways, only select and insert are allowed
|
lamindb_setup/_migrate.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import httpx
|
|
3
4
|
from django.db import connection
|
|
4
5
|
from django.db.migrations.loader import MigrationLoader
|
|
5
6
|
from lamin_utils import logger
|
|
@@ -142,6 +143,13 @@ class migrate:
|
|
|
142
143
|
logger.important(f"updating lamindb version in hub: {lamindb.__version__}")
|
|
143
144
|
if settings.instance.dialect != "sqlite":
|
|
144
145
|
update_schema_in_hub()
|
|
146
|
+
logger.warning(
|
|
147
|
+
"clearing instance cache in hub; if this fails, re-run with latest lamindb version"
|
|
148
|
+
)
|
|
149
|
+
httpx.delete(
|
|
150
|
+
f"{settings.instance.api_url}/cache/instances/{settings.instance._id.hex}",
|
|
151
|
+
headers={"Authorization": f"Bearer {settings.user.access_token}"},
|
|
152
|
+
)
|
|
145
153
|
call_with_fallback_auth(
|
|
146
154
|
update_instance,
|
|
147
155
|
instance_id=settings.instance._id.hex,
|
|
@@ -71,9 +71,9 @@ def _synchronize_schema(client: Client) -> tuple[bool, UUID, dict]:
|
|
|
71
71
|
.eq("id", settings.instance._id.hex)
|
|
72
72
|
.execute()
|
|
73
73
|
)
|
|
74
|
-
assert (
|
|
75
|
-
|
|
76
|
-
)
|
|
74
|
+
assert len(instance_response.data) == 1, (
|
|
75
|
+
f"schema of instance {settings.instance._id.hex} could not be updated with schema {schema_uuid.hex}"
|
|
76
|
+
)
|
|
77
77
|
|
|
78
78
|
return is_new, schema_uuid, schema
|
|
79
79
|
|
|
@@ -404,23 +404,27 @@ class _SchemaHandler:
|
|
|
404
404
|
return self.to_dict(include_django_objects=False)
|
|
405
405
|
|
|
406
406
|
def _get_modules_metadata(self):
|
|
407
|
+
from django.apps import apps
|
|
407
408
|
from lamindb.models import Registry, SQLRecord
|
|
408
409
|
|
|
409
|
-
all_models = {
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
).models.__dict__.values()
|
|
417
|
-
if model.__class__ is Registry
|
|
410
|
+
all_models = {module_name: {} for module_name in self.included_modules}
|
|
411
|
+
|
|
412
|
+
# Iterate through all registered Django models
|
|
413
|
+
for model in apps.get_models():
|
|
414
|
+
# Check if model meets the criteria
|
|
415
|
+
if (
|
|
416
|
+
model.__class__ is Registry
|
|
418
417
|
and model is not SQLRecord
|
|
419
418
|
and not model._meta.abstract
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
419
|
+
):
|
|
420
|
+
module_name = model.__get_module_name__()
|
|
421
|
+
# Only include if module is in our included list
|
|
422
|
+
if module_name in self.included_modules:
|
|
423
|
+
model_name = model._meta.model_name
|
|
424
|
+
all_models[module_name][model_name] = _ModelHandler(
|
|
425
|
+
model, module_name, self.included_modules
|
|
426
|
+
)
|
|
427
|
+
|
|
424
428
|
assert all_models
|
|
425
429
|
return all_models
|
|
426
430
|
|
|
@@ -29,8 +29,7 @@ def _keep_trailing_slash(path_str: str) -> str:
|
|
|
29
29
|
AWS_CREDENTIALS_EXPIRATION: int = 11 * 60 * 60 # refresh credentials after 11 hours
|
|
30
30
|
|
|
31
31
|
|
|
32
|
-
# set anon=True for these buckets if credentials fail for a public bucket
|
|
33
|
-
# to be expanded
|
|
32
|
+
# set anon=True for these buckets if credentials fail for a public bucket to be expanded
|
|
34
33
|
PUBLIC_BUCKETS: tuple[str, ...] = ("cellxgene-data-public", "bionty-assets")
|
|
35
34
|
|
|
36
35
|
|
|
@@ -155,8 +154,7 @@ class AWSOptionsManager:
|
|
|
155
154
|
# this option is needed for correct uploads to R2
|
|
156
155
|
path = UPath(path, fixed_upload_size=True)
|
|
157
156
|
return path
|
|
158
|
-
# trailing slash is needed to avoid returning incorrect results
|
|
159
|
-
# with .startswith
|
|
157
|
+
# trailing slash is needed to avoid returning incorrect results with .startswith
|
|
160
158
|
# for example s3://lamindata-eu should not receive cache for s3://lamindata
|
|
161
159
|
path_str = _keep_trailing_slash(path.as_posix())
|
|
162
160
|
root = self._find_root(path_str)
|
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import httpx
|
|
3
4
|
from lamin_utils import logger
|
|
4
5
|
|
|
5
6
|
|
|
6
7
|
def get_location(ip="ipinfo.io"):
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
response = requests.get(f"http://{ip}/json").json()
|
|
8
|
+
response = httpx.get(f"http://{ip}/json").json()
|
|
10
9
|
loc = response["loc"].split(",")
|
|
11
10
|
return {"latitude": float(loc[0]), "longitude": float(loc[1])}
|
|
12
11
|
|
|
@@ -2,10 +2,13 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import json
|
|
4
4
|
import os
|
|
5
|
+
from contextlib import contextmanager
|
|
6
|
+
from datetime import datetime
|
|
5
7
|
from typing import Literal
|
|
6
8
|
from urllib.request import urlretrieve
|
|
7
9
|
|
|
8
|
-
|
|
10
|
+
import httpx
|
|
11
|
+
from httpx_retries import Retry, RetryTransport
|
|
9
12
|
from lamin_utils import logger
|
|
10
13
|
from pydantic_settings import BaseSettings
|
|
11
14
|
from supabase import Client, create_client # type: ignore
|
|
@@ -61,7 +64,17 @@ class Environment:
|
|
|
61
64
|
self.supabase_anon_key: str = key
|
|
62
65
|
|
|
63
66
|
|
|
64
|
-
DEFAULT_TIMEOUT =
|
|
67
|
+
DEFAULT_TIMEOUT = 12
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
# needed to log retries
|
|
71
|
+
class LogRetry(Retry):
|
|
72
|
+
def increment(self):
|
|
73
|
+
new = super().increment()
|
|
74
|
+
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
75
|
+
# new.attempts_made is the 1-based retry count
|
|
76
|
+
logger.warning(f"{now} HTTP retry attempt {new.attempts_made}/{new.total}")
|
|
77
|
+
return new
|
|
65
78
|
|
|
66
79
|
|
|
67
80
|
# runs ~0.5s
|
|
@@ -78,11 +91,33 @@ def connect_hub(
|
|
|
78
91
|
client = create_client(env.supabase_api_url, env.supabase_anon_key, client_options)
|
|
79
92
|
# needed to enable retries for http requests in supabase
|
|
80
93
|
# these are separate clients and need separate transports
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
94
|
+
transports = []
|
|
95
|
+
for _ in range(2):
|
|
96
|
+
transports.append(
|
|
97
|
+
RetryTransport(
|
|
98
|
+
retry=LogRetry(total=2, backoff_factor=0.2),
|
|
99
|
+
transport=httpx.HTTPTransport(verify=True, http2=True),
|
|
100
|
+
)
|
|
101
|
+
)
|
|
102
|
+
client.auth._http_client._transport = transports[0]
|
|
103
|
+
client.postgrest.session._transport = transports[1]
|
|
104
|
+
# POST is not retryable by default, but for our functions it should be safe to retry
|
|
105
|
+
client.functions._client._transport = RetryTransport(
|
|
106
|
+
retry=LogRetry(
|
|
107
|
+
total=2,
|
|
108
|
+
backoff_factor=0.2,
|
|
109
|
+
allowed_methods=[
|
|
110
|
+
"HEAD",
|
|
111
|
+
"GET",
|
|
112
|
+
"PUT",
|
|
113
|
+
"DELETE",
|
|
114
|
+
"OPTIONS",
|
|
115
|
+
"TRACE",
|
|
116
|
+
"POST",
|
|
117
|
+
],
|
|
118
|
+
),
|
|
119
|
+
transport=httpx.HTTPTransport(verify=True, http2=True),
|
|
120
|
+
)
|
|
86
121
|
return client
|
|
87
122
|
|
|
88
123
|
|
|
@@ -140,17 +175,18 @@ def call_with_fallback_auth(
|
|
|
140
175
|
access_token = kwargs.pop("access_token", None)
|
|
141
176
|
|
|
142
177
|
if access_token is not None:
|
|
178
|
+
client = None
|
|
143
179
|
try:
|
|
144
180
|
client = connect_hub_with_auth(access_token=access_token)
|
|
145
181
|
result = callable(**kwargs, client=client)
|
|
146
182
|
finally:
|
|
147
|
-
|
|
183
|
+
if client is not None:
|
|
148
184
|
client.auth.sign_out(options={"scope": "local"})
|
|
149
|
-
|
|
150
|
-
pass
|
|
185
|
+
|
|
151
186
|
return result
|
|
152
187
|
|
|
153
188
|
for renew_token, fallback_env in [(False, False), (True, False), (False, True)]:
|
|
189
|
+
client = None
|
|
154
190
|
try:
|
|
155
191
|
client = connect_hub_with_auth(
|
|
156
192
|
renew_token=renew_token, fallback_env=fallback_env
|
|
@@ -171,10 +207,9 @@ def call_with_fallback_auth(
|
|
|
171
207
|
if fallback_env:
|
|
172
208
|
raise e
|
|
173
209
|
finally:
|
|
174
|
-
|
|
210
|
+
if client is not None:
|
|
175
211
|
client.auth.sign_out(options={"scope": "local"})
|
|
176
|
-
|
|
177
|
-
pass
|
|
212
|
+
|
|
178
213
|
return result
|
|
179
214
|
|
|
180
215
|
|
|
@@ -183,6 +218,7 @@ def call_with_fallback(
|
|
|
183
218
|
**kwargs,
|
|
184
219
|
):
|
|
185
220
|
for fallback_env in [False, True]:
|
|
221
|
+
client = None
|
|
186
222
|
try:
|
|
187
223
|
client = connect_hub(fallback_env=fallback_env)
|
|
188
224
|
result = callable(**kwargs, client=client)
|
|
@@ -191,25 +227,32 @@ def call_with_fallback(
|
|
|
191
227
|
if fallback_env:
|
|
192
228
|
raise e
|
|
193
229
|
finally:
|
|
194
|
-
|
|
230
|
+
if client is not None:
|
|
195
231
|
# in case there was sign in
|
|
196
232
|
client.auth.sign_out(options={"scope": "local"})
|
|
197
|
-
except NameError:
|
|
198
|
-
pass
|
|
199
233
|
return result
|
|
200
234
|
|
|
201
235
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
import requests # type: ignore
|
|
236
|
+
@contextmanager
|
|
237
|
+
def httpx_client():
|
|
238
|
+
client = None
|
|
239
|
+
try:
|
|
240
|
+
# local is used in tests
|
|
241
|
+
if os.environ.get("LAMIN_ENV", "prod") == "local":
|
|
242
|
+
from fastapi.testclient import TestClient
|
|
243
|
+
from laminhub_rest.main import app
|
|
211
244
|
|
|
212
|
-
|
|
245
|
+
client = TestClient(app)
|
|
246
|
+
else:
|
|
247
|
+
transport = RetryTransport(
|
|
248
|
+
retry=LogRetry(total=2, backoff_factor=0.2),
|
|
249
|
+
transport=httpx.HTTPTransport(verify=True, http2=True),
|
|
250
|
+
)
|
|
251
|
+
client = httpx.Client(transport=transport)
|
|
252
|
+
yield client
|
|
253
|
+
finally:
|
|
254
|
+
if client is not None:
|
|
255
|
+
client.close()
|
|
213
256
|
|
|
214
257
|
|
|
215
258
|
def request_with_auth(
|
|
@@ -219,30 +262,27 @@ def request_with_auth(
|
|
|
219
262
|
renew_token: bool = True,
|
|
220
263
|
**kwargs,
|
|
221
264
|
):
|
|
222
|
-
requests = requests_client()
|
|
223
|
-
|
|
224
265
|
headers = kwargs.pop("headers", {})
|
|
225
266
|
headers["Authorization"] = f"Bearer {access_token}"
|
|
226
|
-
|
|
227
|
-
make_request = getattr(requests, method)
|
|
228
267
|
timeout = kwargs.pop("timeout", DEFAULT_TIMEOUT)
|
|
229
268
|
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
269
|
+
with httpx_client() as client:
|
|
270
|
+
make_request = getattr(client, method)
|
|
271
|
+
response = make_request(url, headers=headers, timeout=timeout, **kwargs)
|
|
272
|
+
status_code = response.status_code
|
|
273
|
+
# update access_token and try again if failed
|
|
274
|
+
if not (200 <= status_code < 300) and renew_token:
|
|
275
|
+
from lamindb_setup import settings
|
|
237
276
|
|
|
238
|
-
|
|
239
|
-
settings.user.email, settings.user.password, settings.user.api_key
|
|
240
|
-
)
|
|
277
|
+
logger.debug(f"{method} {url} failed: {status_code} {response.text}")
|
|
241
278
|
|
|
242
|
-
|
|
243
|
-
|
|
279
|
+
access_token = get_access_token(
|
|
280
|
+
settings.user.email, settings.user.password, settings.user.api_key
|
|
281
|
+
)
|
|
244
282
|
|
|
245
|
-
|
|
283
|
+
settings.user.access_token = access_token
|
|
284
|
+
save_user_settings(settings.user)
|
|
246
285
|
|
|
247
|
-
|
|
286
|
+
headers["Authorization"] = f"Bearer {access_token}"
|
|
287
|
+
response = make_request(url, headers=headers, timeout=timeout, **kwargs)
|
|
248
288
|
return response
|
lamindb_setup/core/_hub_core.py
CHANGED
|
@@ -36,7 +36,6 @@ from ._hub_utils import (
|
|
|
36
36
|
from ._settings import settings
|
|
37
37
|
from ._settings_instance import InstanceSettings
|
|
38
38
|
from ._settings_storage import StorageSettings, base62, instance_uid_from_uuid
|
|
39
|
-
from .hashing import hash_and_encode_as_b62
|
|
40
39
|
|
|
41
40
|
if TYPE_CHECKING:
|
|
42
41
|
from supabase import Client # type: ignore
|
|
@@ -194,6 +193,8 @@ def _select_storage_by_settings(
|
|
|
194
193
|
|
|
195
194
|
|
|
196
195
|
def _select_storage_or_parent(path: str, client: Client) -> dict | None:
|
|
196
|
+
# add get=True when we upgrade supabase
|
|
197
|
+
# because otherwise it uses POST which is not retryable
|
|
197
198
|
result = client.rpc("existing_root_or_child", {"_path": path}).execute().data
|
|
198
199
|
if result["root"] is None:
|
|
199
200
|
return None
|
|
@@ -253,8 +254,7 @@ def _init_storage_hub(
|
|
|
253
254
|
from lamindb_setup import settings
|
|
254
255
|
|
|
255
256
|
created_by = settings.user._uuid if created_by is None else created_by
|
|
256
|
-
# storage roots are always stored without the trailing slash in the SQL
|
|
257
|
-
# database
|
|
257
|
+
# storage roots are always stored without the trailing slash in the SQL database
|
|
258
258
|
root = ssettings.root_as_str
|
|
259
259
|
if _select_storage_by_settings(ssettings, update_uid=True, client=client):
|
|
260
260
|
return "hub-record-retrieved"
|
|
@@ -285,8 +285,7 @@ def _init_storage_hub(
|
|
|
285
285
|
"is_default": is_default,
|
|
286
286
|
"space_id": space_id.hex if space_id is not None else None,
|
|
287
287
|
}
|
|
288
|
-
# TODO: add error message for violated unique constraint
|
|
289
|
-
# on root & description
|
|
288
|
+
# TODO: add error message for violated unique constraint on root & description
|
|
290
289
|
client.table("storage").upsert(fields).execute()
|
|
291
290
|
ssettings._uuid_ = id
|
|
292
291
|
return "hub-record-created"
|
|
@@ -341,7 +340,7 @@ def _delete_instance(
|
|
|
341
340
|
)
|
|
342
341
|
# gate storage and instance deletion on empty storage location for
|
|
343
342
|
# normally auth.get_session() doesn't have access_token
|
|
344
|
-
# so this block is useless
|
|
343
|
+
# so this block is useless I think (Sergei)
|
|
345
344
|
# the token is received from user settings inside create_path
|
|
346
345
|
# might be needed in the hub though
|
|
347
346
|
if client.auth.get_session() is not None:
|
|
@@ -425,6 +424,7 @@ def _connect_instance_hub(
|
|
|
425
424
|
owner: str, # account_handle
|
|
426
425
|
name: str, # instance_name
|
|
427
426
|
use_root_db_user: bool,
|
|
427
|
+
use_proxy_db: bool,
|
|
428
428
|
client: Client,
|
|
429
429
|
) -> tuple[dict, dict] | str:
|
|
430
430
|
response = client.functions.invoke(
|
|
@@ -456,7 +456,7 @@ def _connect_instance_hub(
|
|
|
456
456
|
)
|
|
457
457
|
# no instance found, check why is that
|
|
458
458
|
if response == b"{}":
|
|
459
|
-
# try
|
|
459
|
+
# try via separate requests, will take more time
|
|
460
460
|
account = select_account_by_handle(owner, client)
|
|
461
461
|
if account is None:
|
|
462
462
|
return "account-not-exists"
|
|
@@ -500,12 +500,26 @@ def _connect_instance_hub(
|
|
|
500
500
|
db_user["name" if fine_grained_access else "db_user_name"],
|
|
501
501
|
db_user["password" if fine_grained_access else "db_user_password"],
|
|
502
502
|
)
|
|
503
|
+
|
|
504
|
+
if use_proxy_db:
|
|
505
|
+
host = instance.get("proxy_host", None)
|
|
506
|
+
assert host is not None, (
|
|
507
|
+
"Database proxy host is not available, please do not pass 'use_proxy_db'."
|
|
508
|
+
)
|
|
509
|
+
port = instance.get("proxy_port", None)
|
|
510
|
+
assert port is not None, (
|
|
511
|
+
"Database proxy port is not available, please do not pass 'use_proxy_db'."
|
|
512
|
+
)
|
|
513
|
+
else:
|
|
514
|
+
host = instance["db_host"]
|
|
515
|
+
port = instance["db_port"]
|
|
516
|
+
|
|
503
517
|
db_dsn = LaminDsn.build(
|
|
504
518
|
scheme=instance["db_scheme"],
|
|
505
519
|
user=db_user_name if db_user_name is not None else "none",
|
|
506
520
|
password=db_user_password if db_user_password is not None else "none",
|
|
507
|
-
host=
|
|
508
|
-
port=
|
|
521
|
+
host=host,
|
|
522
|
+
port=port,
|
|
509
523
|
database=instance["db_database"],
|
|
510
524
|
)
|
|
511
525
|
instance["db"] = db_dsn
|
|
@@ -519,6 +533,7 @@ def connect_instance_hub(
|
|
|
519
533
|
name: str, # instance_name
|
|
520
534
|
access_token: str | None = None,
|
|
521
535
|
use_root_db_user: bool = False,
|
|
536
|
+
use_proxy_db: bool = False,
|
|
522
537
|
) -> tuple[dict, dict] | str:
|
|
523
538
|
from ._settings import settings
|
|
524
539
|
|
|
@@ -528,6 +543,7 @@ def connect_instance_hub(
|
|
|
528
543
|
owner=owner,
|
|
529
544
|
name=name,
|
|
530
545
|
use_root_db_user=use_root_db_user,
|
|
546
|
+
use_proxy_db=use_proxy_db,
|
|
531
547
|
access_token=access_token,
|
|
532
548
|
)
|
|
533
549
|
else:
|
|
@@ -536,6 +552,7 @@ def connect_instance_hub(
|
|
|
536
552
|
owner=owner,
|
|
537
553
|
name=name,
|
|
538
554
|
use_root_db_user=use_root_db_user,
|
|
555
|
+
use_proxy_db=use_proxy_db,
|
|
539
556
|
)
|
|
540
557
|
|
|
541
558
|
|
|
@@ -610,7 +627,7 @@ def access_db(
|
|
|
610
627
|
else:
|
|
611
628
|
renew_token = False
|
|
612
629
|
# local is used in tests
|
|
613
|
-
url = f"/
|
|
630
|
+
url = f"/instances/{instance_id}/db_token"
|
|
614
631
|
if os.environ.get("LAMIN_ENV", "prod") != "local":
|
|
615
632
|
if instance_api_url is None:
|
|
616
633
|
raise RuntimeError(
|
|
@@ -661,7 +678,12 @@ def _sign_in_hub(email: str, password: str, handle: str | None, client: Client):
|
|
|
661
678
|
"password": password,
|
|
662
679
|
}
|
|
663
680
|
)
|
|
664
|
-
|
|
681
|
+
# normally public.account.id is equal to auth.user.id
|
|
682
|
+
# but it might be not the case in the future
|
|
683
|
+
# this is why we check public.account.user_id that references auth.user.id
|
|
684
|
+
data = (
|
|
685
|
+
client.table("account").select("*").eq("user_id", auth.user.id).execute().data
|
|
686
|
+
)
|
|
665
687
|
if data: # sync data from hub to local cache in case it was updated on the hub
|
|
666
688
|
user = data[0]
|
|
667
689
|
user_uuid = UUID(user["id"])
|
|
@@ -710,8 +732,9 @@ def _sign_in_hub_api_key(api_key: str, client: Client):
|
|
|
710
732
|
# probably need more info here to avoid additional queries
|
|
711
733
|
# like handle, uid etc
|
|
712
734
|
account_id = jwt.decode(access_token, options={"verify_signature": False})["sub"]
|
|
735
|
+
|
|
713
736
|
client.postgrest.auth(access_token)
|
|
714
|
-
|
|
737
|
+
|
|
715
738
|
data = client.table("account").select("*").eq("id", account_id).execute().data
|
|
716
739
|
if data:
|
|
717
740
|
user = data[0]
|
lamindb_setup/core/_settings.py
CHANGED
|
@@ -2,7 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import os
|
|
4
4
|
import sys
|
|
5
|
-
import
|
|
5
|
+
from pathlib import Path
|
|
6
6
|
from typing import TYPE_CHECKING
|
|
7
7
|
|
|
8
8
|
from lamin_utils import logger
|
|
@@ -23,8 +23,6 @@ from ._settings_store import (
|
|
|
23
23
|
from .upath import LocalPathClasses, UPath
|
|
24
24
|
|
|
25
25
|
if TYPE_CHECKING:
|
|
26
|
-
from pathlib import Path
|
|
27
|
-
|
|
28
26
|
from lamindb.models import Branch, Space
|
|
29
27
|
|
|
30
28
|
from lamindb_setup.core import InstanceSettings, StorageSettings, UserSettings
|
|
@@ -60,6 +58,7 @@ class SetupSettings:
|
|
|
60
58
|
|
|
61
59
|
_auto_connect_path: Path = settings_dir / "auto_connect"
|
|
62
60
|
_private_django_api_path: Path = settings_dir / "private_django_api"
|
|
61
|
+
_work_dir: Path = settings_dir / "work_dir.txt"
|
|
63
62
|
|
|
64
63
|
_cache_dir: Path | None = None
|
|
65
64
|
|
|
@@ -70,6 +69,25 @@ class SetupSettings:
|
|
|
70
69
|
def _instance_settings_path(self) -> Path:
|
|
71
70
|
return current_instance_settings_file()
|
|
72
71
|
|
|
72
|
+
@property
|
|
73
|
+
def work_dir(self) -> Path | None:
|
|
74
|
+
"""Get or set the current working directory.
|
|
75
|
+
|
|
76
|
+
If setting it to `None`, the working directory is unset
|
|
77
|
+
"""
|
|
78
|
+
if not self._work_dir.exists():
|
|
79
|
+
return None
|
|
80
|
+
return Path(self._work_dir.read_text())
|
|
81
|
+
|
|
82
|
+
@work_dir.setter
|
|
83
|
+
def work_dir(self, value: str | Path | None) -> None:
|
|
84
|
+
if value is None:
|
|
85
|
+
if self._work_dir.exists():
|
|
86
|
+
self._work_dir.unlink()
|
|
87
|
+
else:
|
|
88
|
+
value_str = Path(value).expanduser().resolve().as_posix()
|
|
89
|
+
self._work_dir.write_text(value_str)
|
|
90
|
+
|
|
73
91
|
@property
|
|
74
92
|
def settings_dir(self) -> Path:
|
|
75
93
|
"""The directory that holds locally persisted settings."""
|
|
@@ -207,8 +225,8 @@ class SetupSettings:
|
|
|
207
225
|
def private_django_api(self) -> bool:
|
|
208
226
|
"""Turn internal Django API private to clean up the API (default `False`).
|
|
209
227
|
|
|
210
|
-
This patches your local pip-installed django installation.
|
|
211
|
-
the patch by setting this back to `False`.
|
|
228
|
+
This patches your local pip-installed django installation.
|
|
229
|
+
You can undo the patch by setting this back to `False`.
|
|
212
230
|
"""
|
|
213
231
|
return self._private_django_api_path.exists()
|
|
214
232
|
|
|
@@ -317,6 +335,7 @@ class SetupSettings:
|
|
|
317
335
|
repr += "\nConfig:\n"
|
|
318
336
|
repr += f" - private Django API: {self.private_django_api}\n"
|
|
319
337
|
repr += "Local directories:\n"
|
|
338
|
+
repr += f" - working directory: {self.work_dir}\n"
|
|
320
339
|
repr += f" - cache: {self.cache_dir.as_posix()}\n"
|
|
321
340
|
repr += f" - user settings: {settings_dir.as_posix()}\n"
|
|
322
341
|
repr += f" - system settings: {system_settings_dir.as_posix()}\n"
|
|
@@ -242,8 +242,8 @@ class InstanceSettings:
|
|
|
242
242
|
def storage(self) -> StorageSettings:
|
|
243
243
|
"""Default storage of instance.
|
|
244
244
|
|
|
245
|
-
For a cloud instance, this is cloud storage.
|
|
246
|
-
is a local directory.
|
|
245
|
+
For a cloud instance, this is cloud storage.
|
|
246
|
+
For a local instance, this is a local directory.
|
|
247
247
|
"""
|
|
248
248
|
return self._storage # type: ignore
|
|
249
249
|
|
|
@@ -350,6 +350,8 @@ class InstanceSettings:
|
|
|
350
350
|
|
|
351
351
|
Use this URL for API calls related to this instance.
|
|
352
352
|
"""
|
|
353
|
+
if "LAMIN_API_URL" in os.environ:
|
|
354
|
+
return os.environ["LAMIN_API_URL"]
|
|
353
355
|
return self._api_url
|
|
354
356
|
|
|
355
357
|
@property
|
|
@@ -461,7 +463,7 @@ class InstanceSettings:
|
|
|
461
463
|
)
|
|
462
464
|
lock_msg += (
|
|
463
465
|
" The instance will be automatically unlocked after"
|
|
464
|
-
f" {int(EXPIRATION_TIME/3600/24)}d of no activity."
|
|
466
|
+
f" {int(EXPIRATION_TIME / 3600 / 24)}d of no activity."
|
|
465
467
|
)
|
|
466
468
|
raise InstanceLockedException(lock_msg)
|
|
467
469
|
|
|
@@ -501,8 +503,6 @@ class InstanceSettings:
|
|
|
501
503
|
|
|
502
504
|
@property
|
|
503
505
|
def _is_cloud_sqlite(self) -> bool:
|
|
504
|
-
# can we make this a private property, Sergei?
|
|
505
|
-
# as it's not relevant to the user
|
|
506
506
|
"""Is this a cloud instance with sqlite db."""
|
|
507
507
|
return self.dialect == "sqlite" and self.storage.type_is_cloud
|
|
508
508
|
|
|
@@ -530,8 +530,8 @@ class InstanceSettings:
|
|
|
530
530
|
def is_on_hub(self) -> bool:
|
|
531
531
|
"""Is this instance on the hub?
|
|
532
532
|
|
|
533
|
-
Can only reliably establish if user has access to the instance.
|
|
534
|
-
return `False` in case the instance isn't found.
|
|
533
|
+
Can only reliably establish if user has access to the instance.
|
|
534
|
+
Will return `False` in case the instance isn't found.
|
|
535
535
|
"""
|
|
536
536
|
if self._is_on_hub is None:
|
|
537
537
|
from ._hub_client import call_with_fallback_auth
|
|
@@ -41,9 +41,9 @@ def save_settings(
|
|
|
41
41
|
):
|
|
42
42
|
with open(settings_file, "w") as f:
|
|
43
43
|
for store_key, type_ in type_hints.items():
|
|
44
|
-
if type_ == Optional[str]:
|
|
44
|
+
if type_ == Optional[str]: # noqa: UP045
|
|
45
45
|
type_ = str
|
|
46
|
-
if type_ == Optional[bool]:
|
|
46
|
+
if type_ == Optional[bool]: # noqa: UP045
|
|
47
47
|
type_ = bool
|
|
48
48
|
if "__" not in store_key:
|
|
49
49
|
if store_key == "model_config":
|
|
@@ -63,19 +63,19 @@ def system_settings_file():
|
|
|
63
63
|
|
|
64
64
|
|
|
65
65
|
class InstanceSettingsStore(BaseSettings):
|
|
66
|
-
api_url:
|
|
66
|
+
api_url: str | None = None
|
|
67
67
|
owner: str
|
|
68
68
|
name: str
|
|
69
69
|
storage_root: str
|
|
70
|
-
storage_region:
|
|
71
|
-
db:
|
|
72
|
-
schema_str:
|
|
73
|
-
schema_id:
|
|
70
|
+
storage_region: str | None # take old type annotations here because pydantic
|
|
71
|
+
db: str | None # doesn't like new types on 3.9 even with future annotations
|
|
72
|
+
schema_str: str | None
|
|
73
|
+
schema_id: str | None = None
|
|
74
74
|
fine_grained_access: bool = False
|
|
75
|
-
db_permissions:
|
|
75
|
+
db_permissions: str | None = None
|
|
76
76
|
id: str
|
|
77
|
-
git_repo:
|
|
78
|
-
keep_artifacts_local:
|
|
77
|
+
git_repo: str | None
|
|
78
|
+
keep_artifacts_local: bool | None
|
|
79
79
|
model_config = SettingsConfigDict(env_prefix="lamindb_instance_", env_file=".env")
|
|
80
80
|
|
|
81
81
|
|
|
@@ -83,7 +83,7 @@ class UserSettingsStore(BaseSettings):
|
|
|
83
83
|
email: str
|
|
84
84
|
password: str
|
|
85
85
|
access_token: str
|
|
86
|
-
api_key:
|
|
86
|
+
api_key: str | None = None
|
|
87
87
|
uid: str
|
|
88
88
|
uuid: str
|
|
89
89
|
handle: str
|
lamindb_setup/core/django.py
CHANGED
|
@@ -168,7 +168,10 @@ def setup_django(
|
|
|
168
168
|
ssl_require = False
|
|
169
169
|
else:
|
|
170
170
|
ssl_require = not is_local_db_url(instance_db)
|
|
171
|
-
options = {
|
|
171
|
+
options = {
|
|
172
|
+
"connect_timeout": os.getenv("PGCONNECT_TIMEOUT", 20),
|
|
173
|
+
"gssencmode": "disable",
|
|
174
|
+
}
|
|
172
175
|
else:
|
|
173
176
|
ssl_require = False
|
|
174
177
|
options = {}
|
lamindb_setup/core/upath.py
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
from __future__ import annotations
|
|
5
5
|
|
|
6
|
+
import math
|
|
6
7
|
import os
|
|
7
8
|
import warnings
|
|
8
9
|
from collections import defaultdict
|
|
@@ -180,7 +181,7 @@ def print_hook(size: int, value: int, objectname: str, action: str):
|
|
|
180
181
|
progress_in_percent = 100.0
|
|
181
182
|
else:
|
|
182
183
|
progress_in_percent = (value / size) * 100
|
|
183
|
-
out = f"... {action} {objectname}:
|
|
184
|
+
out = f"... {action} {objectname}: {min(progress_in_percent, 100):4.1f}%"
|
|
184
185
|
if "NBPRJ_TEST_NBPATH" not in os.environ:
|
|
185
186
|
end = "\n" if progress_in_percent >= 100 else "\r"
|
|
186
187
|
print(out, end=end)
|
|
@@ -353,6 +354,15 @@ def upload_from(
|
|
|
353
354
|
destination = fsspec.utils.other_paths(
|
|
354
355
|
files, self.as_posix(), exists=False, flatten=False
|
|
355
356
|
)
|
|
357
|
+
elif self.protocol == "s3" and "chunksize" not in kwargs:
|
|
358
|
+
size = local_path.stat().st_size
|
|
359
|
+
MiB = 1024**2
|
|
360
|
+
DEFAULT_CHUNKSIZE = 50 * MiB # so in s3fs
|
|
361
|
+
if size / DEFAULT_CHUNKSIZE > 10000: # should be no more than 10k parts for s3
|
|
362
|
+
raw = math.ceil(size / 10000)
|
|
363
|
+
step = 5 * MiB
|
|
364
|
+
rounded = math.ceil(raw / step) * step
|
|
365
|
+
kwargs["chunksize"] = rounded
|
|
356
366
|
|
|
357
367
|
# the below lines are to avoid s3fs triggering create_bucket in upload if
|
|
358
368
|
# dirs are present, it allows to avoid the permission error
|
|
@@ -650,7 +660,7 @@ def view_tree(
|
|
|
650
660
|
skip_suffixes: Skip directories with these suffixes.
|
|
651
661
|
|
|
652
662
|
Examples:
|
|
653
|
-
>>> dir_path = ln.
|
|
663
|
+
>>> dir_path = ln.examples.datasets.generate_cell_ranger_files(
|
|
654
664
|
>>> "sample_001", ln.settings.storage
|
|
655
665
|
>>> )
|
|
656
666
|
>>> ln.UPath(dir_path).view_tree()
|
|
@@ -713,15 +723,12 @@ def to_url(upath):
|
|
|
713
723
|
# Why aren't we subclassing?
|
|
714
724
|
#
|
|
715
725
|
# The problem is that UPath defines a type system of paths
|
|
716
|
-
# Its __new__ method returns instances of different subclasses rather than a
|
|
717
|
-
# UPath object
|
|
726
|
+
# Its __new__ method returns instances of different subclasses rather than a UPath object
|
|
718
727
|
# If we create a custom subclass naively, subclasses of the parent UPath won't
|
|
719
728
|
# be subclasses of our custom subclass
|
|
720
|
-
# This makes life really hard in type checks involving local to cloud
|
|
721
|
-
# comparisons, etc.
|
|
729
|
+
# This makes life really hard in type checks involving local to cloud comparisons, etc.
|
|
722
730
|
# Hence, we extend the existing UPath and amend the docs
|
|
723
|
-
# Some of this might end up in the original UPath implementation over time,
|
|
724
|
-
# we'll see.
|
|
731
|
+
# Some of this might end up in the original UPath implementation over time, we'll see.
|
|
725
732
|
|
|
726
733
|
|
|
727
734
|
# add custom functions
|
|
@@ -893,7 +900,7 @@ def create_path(path: UPathStr, access_token: str | None = None) -> UPath:
|
|
|
893
900
|
storage_options["client_kwargs"] = client_kwargs
|
|
894
901
|
# see download_to for the reason
|
|
895
902
|
if "use_listings_cache" not in upath.storage_options:
|
|
896
|
-
storage_options["use_listings_cache"] = True
|
|
903
|
+
storage_options["use_listings_cache"] = True # type: ignore
|
|
897
904
|
if len(storage_options) > 0:
|
|
898
905
|
return UPath(upath, **storage_options)
|
|
899
906
|
return upath
|
|
@@ -1004,7 +1011,7 @@ def check_storage_is_empty(
|
|
|
1004
1011
|
if raise_error
|
|
1005
1012
|
else "consider deleting them"
|
|
1006
1013
|
)
|
|
1007
|
-
message = f"'{directory_string}' contains {n_diff} objects
|
|
1014
|
+
message = f"'{directory_string}' contains {n_diff} objects - {ask_for_deletion}"
|
|
1008
1015
|
if n_diff > 0:
|
|
1009
1016
|
if raise_error:
|
|
1010
1017
|
raise StorageNotEmpty(message) from None
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: lamindb_setup
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.12.0
|
|
4
4
|
Summary: Setup & configure LaminDB.
|
|
5
5
|
Author-email: Lamin Labs <open-source@lamin.ai>
|
|
6
6
|
Requires-Python: >=3.10
|
|
@@ -10,10 +10,11 @@ Requires-Dist: django>=5.1,<5.2
|
|
|
10
10
|
Requires-Dist: dj_database_url>=1.3.0,<3.0.0
|
|
11
11
|
Requires-Dist: pydantic-settings
|
|
12
12
|
Requires-Dist: platformdirs<5.0.0
|
|
13
|
+
Requires-Dist: httpx_retries<1.0.0
|
|
13
14
|
Requires-Dist: requests
|
|
14
15
|
Requires-Dist: universal_pathlib==0.2.6
|
|
15
16
|
Requires-Dist: botocore<2.0.0
|
|
16
|
-
Requires-Dist: supabase>=2.8.1,<=2.
|
|
17
|
+
Requires-Dist: supabase>=2.8.1,<=2.16.0
|
|
17
18
|
Requires-Dist: gotrue<=2.12.0
|
|
18
19
|
Requires-Dist: storage3!=0.11.2; python_version < '3.11'
|
|
19
20
|
Requires-Dist: pyjwt<3.0.0
|
|
@@ -1,19 +1,19 @@
|
|
|
1
|
-
lamindb_setup/__init__.py,sha256=
|
|
1
|
+
lamindb_setup/__init__.py,sha256=lU96Y8vk4EcJ-Uhu-vEWooTbhhYTb2LQ_bkBjA7jD4Q,2783
|
|
2
2
|
lamindb_setup/_cache.py,sha256=pGvDNVHGx4HWr_6w5ajqEJOdysmaGc6F221qFnXkT-k,2747
|
|
3
3
|
lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
|
|
4
4
|
lamindb_setup/_check_setup.py,sha256=ToKMxsUq8dQBQh8baOrNVlSb1iC8h4zTg5dV8wMu0W4,6760
|
|
5
|
-
lamindb_setup/_connect_instance.py,sha256=
|
|
6
|
-
lamindb_setup/_delete.py,sha256=
|
|
5
|
+
lamindb_setup/_connect_instance.py,sha256=kp4Nke_ksHb6xZ-z64jRN-NcfLKzzzaZpsI1a83AeIA,17701
|
|
6
|
+
lamindb_setup/_delete.py,sha256=KS3r-xGFuDmAbzPUy-9JR-YnPShYdaHjDRQrAmXQ0qM,5863
|
|
7
7
|
lamindb_setup/_disconnect.py,sha256=FT8EpCm5XXDdhDH7QtAnkO3KPatq2HqT9VXGNjgJDbk,1232
|
|
8
8
|
lamindb_setup/_django.py,sha256=uIQflpkp8l3axyPaKURlk3kacgpElVP5KOKmFxYSMGk,1454
|
|
9
9
|
lamindb_setup/_entry_points.py,sha256=sKwXPX9xjOotoAjvgkU5LBwjjHLWVkh0ZGdiSsrch9k,522
|
|
10
10
|
lamindb_setup/_exportdb.py,sha256=QLjoH4dEwqa01A12naKaDPglCCzl2_VLKWFfJRE_uSg,2113
|
|
11
11
|
lamindb_setup/_importdb.py,sha256=fKv9ev5OOj_-bmzC8XZ1GxOcjIjI486yrHSHDWQrJeI,1874
|
|
12
|
-
lamindb_setup/_init_instance.py,sha256=
|
|
13
|
-
lamindb_setup/_migrate.py,sha256=
|
|
12
|
+
lamindb_setup/_init_instance.py,sha256=8ejD6zjV0eF7KR-DvnmDAVJb9Ty0hjaPtIkFbyLDvA0,14806
|
|
13
|
+
lamindb_setup/_migrate.py,sha256=aOWE13LJOW55mC4QiYeCS5bJGSTRsRZPpUYz6e_xoFs,10773
|
|
14
14
|
lamindb_setup/_register_instance.py,sha256=RdUZxZWHLdbvdNZWpF8e0UWROb_T0cStWbzc5yUw34I,1047
|
|
15
15
|
lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
|
|
16
|
-
lamindb_setup/_schema_metadata.py,sha256=
|
|
16
|
+
lamindb_setup/_schema_metadata.py,sha256=At_EAE9mMzMJIJ1mfiOZYXVgBaXRkWUW6a3fLz5Z_lY,15132
|
|
17
17
|
lamindb_setup/_set_managed_storage.py,sha256=y5YQASsWNYVWUYeLgh3N2YBETYP7mBtbpxe3X_Vgb5I,2699
|
|
18
18
|
lamindb_setup/_setup_user.py,sha256=DapdzT3u0f5LN5W9W9A6PWw-n8ejcJciQtHN9b5lidA,5889
|
|
19
19
|
lamindb_setup/_silence_loggers.py,sha256=AKF_YcHvX32eGXdsYK8MJlxEaZ-Uo2f6QDRzjKFCtws,1568
|
|
@@ -21,30 +21,30 @@ lamindb_setup/errors.py,sha256=qZTfSL0rpbY8AIG-Z4-3-_EbLW5zyo2CFEJrVU02-3A,1863
|
|
|
21
21
|
lamindb_setup/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
22
22
|
lamindb_setup/types.py,sha256=XlXLb4nmbc68uBj5Hp3xpDRezYGJIBZv6jAAqqN0p10,614
|
|
23
23
|
lamindb_setup/core/__init__.py,sha256=5M4A6CVHBO_T5Rr9MeLaPW3WTk4-y00cgRYEgUJVU5U,410
|
|
24
|
-
lamindb_setup/core/_aws_options.py,sha256=
|
|
25
|
-
lamindb_setup/core/_aws_storage.py,sha256=
|
|
24
|
+
lamindb_setup/core/_aws_options.py,sha256=SadUhcLCRtvsy3Qvx6799Iv4_CJkb1rwWB6d5GIixHc,8080
|
|
25
|
+
lamindb_setup/core/_aws_storage.py,sha256=QEtV-riQrwfivcwqHnXBbkJ-9YyNEXL4fLoCmOHZ1BI,2003
|
|
26
26
|
lamindb_setup/core/_deprecated.py,sha256=M3vpM4fZPOncxY2qsXQAPeaEph28xWdv7tYaueaUyAA,2554
|
|
27
27
|
lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
|
|
28
|
-
lamindb_setup/core/_hub_client.py,sha256=
|
|
29
|
-
lamindb_setup/core/_hub_core.py,sha256=
|
|
28
|
+
lamindb_setup/core/_hub_client.py,sha256=J0x43at0zb0yWP-RoT2lyqaHV66ewUP3OiYVYQCjxe8,9974
|
|
29
|
+
lamindb_setup/core/_hub_core.py,sha256=axnNugfAehXIB_GLJoJE2zQfsAfPNEw5UELnUvTLWHY,27279
|
|
30
30
|
lamindb_setup/core/_hub_crud.py,sha256=j6516H82kLjFUNPqFGUINbDw9YbofMgjxadGzYb0OS4,6362
|
|
31
31
|
lamindb_setup/core/_hub_utils.py,sha256=6dyDGyzYFgVfR_lE3VN3CP1jGp98gxPtr-T91PAP05U,2687
|
|
32
32
|
lamindb_setup/core/_private_django_api.py,sha256=By63l3vIEtK1pq246FhHq3tslxsaTJGKm5VakYluWp4,2656
|
|
33
|
-
lamindb_setup/core/_settings.py,sha256=
|
|
34
|
-
lamindb_setup/core/_settings_instance.py,sha256=
|
|
33
|
+
lamindb_setup/core/_settings.py,sha256=0nz3HKnBuXdDY4R2UJQts3ZVC7vROpsBAxWIgZNz800,14017
|
|
34
|
+
lamindb_setup/core/_settings_instance.py,sha256=jg-7M-wlfBRQa8GzJLotyCSwt7GIfQpBwl0_Tbejelg,23320
|
|
35
35
|
lamindb_setup/core/_settings_load.py,sha256=j20cy3J56ZBHLDfB2A8oKjekNetMNsy0_W3eWD36pWI,5161
|
|
36
|
-
lamindb_setup/core/_settings_save.py,sha256=
|
|
36
|
+
lamindb_setup/core/_settings_save.py,sha256=jh412jXIAbIYvnSoW9riBFePRAa4vmPm-ScYD0smlnw,3292
|
|
37
37
|
lamindb_setup/core/_settings_storage.py,sha256=pyU25hP5rQYjVe0tFPR8P6TzAYzu1NpT-PIbXoxfV18,15348
|
|
38
|
-
lamindb_setup/core/_settings_store.py,sha256=
|
|
38
|
+
lamindb_setup/core/_settings_store.py,sha256=ykJeBA9IODK4G_jrfBE9pb0c1xkfePkARPpb306DT08,2687
|
|
39
39
|
lamindb_setup/core/_settings_user.py,sha256=gFfyMf-738onbh1Mf4wsmLlenQJPtjQfpUgKnOlqc2o,1453
|
|
40
40
|
lamindb_setup/core/_setup_bionty_sources.py,sha256=ox3X-SHiHa2lNPSWjwZhINypbLacX6kGwH6hVVrSFZc,1505
|
|
41
41
|
lamindb_setup/core/cloud_sqlite_locker.py,sha256=H_CTUCjURFXwD1cCtV_Jn0_60iztZTkaesLLXIBgIxc,7204
|
|
42
|
-
lamindb_setup/core/django.py,sha256=
|
|
42
|
+
lamindb_setup/core/django.py,sha256=kV8W3WZy5Rkhn4rDJv2GNoq8JYvX_8dLBHhDRZdSgwE,10542
|
|
43
43
|
lamindb_setup/core/exceptions.py,sha256=qjMzqy_uzPA7mCOdnoWnS_fdA6OWbdZGftz-YYplrY0,84
|
|
44
44
|
lamindb_setup/core/hashing.py,sha256=Y8Uc5uSGTfU6L2R_gb5w8DdHhGRog7RnkK-e9FEMjPY,3680
|
|
45
45
|
lamindb_setup/core/types.py,sha256=T7NwspfRHgIIpYsXDcApks8jkOlGeGRW-YbVLB7jNIo,67
|
|
46
|
-
lamindb_setup/core/upath.py,sha256
|
|
47
|
-
lamindb_setup-1.
|
|
48
|
-
lamindb_setup-1.
|
|
49
|
-
lamindb_setup-1.
|
|
50
|
-
lamindb_setup-1.
|
|
46
|
+
lamindb_setup/core/upath.py,sha256=uk3LpDA7Jbk1GzUb8hCsxByg5cMYTjPusIvwyXe8g3Y,36023
|
|
47
|
+
lamindb_setup-1.12.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
|
|
48
|
+
lamindb_setup-1.12.0.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
|
|
49
|
+
lamindb_setup-1.12.0.dist-info/METADATA,sha256=4cc1p3XQml5mHnes3thxdnu9HZeectGVXmPNw9C3Al4,1839
|
|
50
|
+
lamindb_setup-1.12.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|