lamindb_setup 0.74.3__tar.gz → 0.76.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/.pre-commit-config.yaml +1 -1
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/PKG-INFO +1 -1
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-cloud/01-init-local-instance.ipynb +4 -1
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-cloud/04-test-bionty.ipynb +3 -3
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-cloud/05-init-hosted-instance.ipynb +1 -1
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-cloud/06-connect-hosted-instance.ipynb +1 -1
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-prod/test-sqlite-lock.ipynb +1 -1
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/__init__.py +1 -1
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_close.py +2 -1
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_connect_instance.py +101 -2
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_init_instance.py +11 -4
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_register_instance.py +4 -1
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_schema_metadata.py +2 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/__init__.py +1 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_hub_core.py +13 -3
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_settings_instance.py +11 -10
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_settings_storage.py +10 -5
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_setup_bionty_sources.py +25 -9
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/hashing.py +6 -4
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/upath.py +1 -1
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/noxfile.py +2 -2
- lamindb_setup-0.76.0/tests/hub-local/conftest.py +29 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-local/test_update_schema_in_hub.py +15 -1
- lamindb_setup-0.74.3/tests/hub-local/conftest.py +0 -23
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/.github/workflows/build.yml +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/.github/workflows/doc-changes.yml +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/.gitignore +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/LICENSE +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/README.md +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/changelog.md +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-cloud/02-connect-local-instance.ipynb +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-cloud/03-add-managed-storage.ipynb +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-cloud/07-keep-artifacts-local.ipynb +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-cloud/test-multi-session.ipynb +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-cloud/test_notebooks.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-prod/test-cache-management.ipynb +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-prod/test-cloud-sync.ipynb +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-prod/test-connect-anonymously.ipynb +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-prod/test-empty-init.ipynb +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-prod/test-import-schema.ipynb +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-prod/test-insufficient-user-info.ipynb +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-prod/test-invalid-schema.ipynb +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-prod/test_notebooks2.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/index.md +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/notebooks.md +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/reference.md +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_cache.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_check.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_check_setup.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_delete.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_django.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_exportdb.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_importdb.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_migrate.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_schema.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_set_managed_storage.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_setup_user.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_silence_loggers.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_aws_credentials.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_aws_storage.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_deprecated.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_docs.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_hub_client.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_hub_crud.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_hub_utils.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_private_django_api.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_settings.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_settings_load.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_settings_save.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_settings_store.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_settings_user.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/cloud_sqlite_locker.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/django.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/exceptions.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/types.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/pyproject.toml +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-cloud/test_connect_instance.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-cloud/test_delete_instance.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-cloud/test_init_instance.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-cloud/test_login.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-cloud/test_migrate.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-cloud/test_set_storage.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-local/test_all.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-prod/conftest.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-prod/test_django.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-prod/test_global_settings.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-prod/test_switch_and_fallback_env.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-prod/test_upath.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/storage/test_hashing.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/storage/test_storage_access.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/storage/test_storage_basis.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/storage/test_storage_stats.py +0 -0
- {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/storage/test_to_url.py +0 -0
|
@@ -48,6 +48,7 @@
|
|
|
48
48
|
"outputs": [],
|
|
49
49
|
"source": [
|
|
50
50
|
"from pathlib import Path\n",
|
|
51
|
+
"from lnschema_core.models import Storage\n",
|
|
51
52
|
"\n",
|
|
52
53
|
"assert ln_setup.settings.instance.storage.type_is_cloud == False\n",
|
|
53
54
|
"assert ln_setup.settings.instance.owner == ln_setup.settings.user.handle\n",
|
|
@@ -59,7 +60,9 @@
|
|
|
59
60
|
"assert (\n",
|
|
60
61
|
" ln_setup.settings.instance.db\n",
|
|
61
62
|
" == f\"sqlite:///{Path('./mydata').resolve().as_posix()}/{ln_setup.settings.instance._id.hex}.lndb\"\n",
|
|
62
|
-
")"
|
|
63
|
+
")\n",
|
|
64
|
+
"assert ln_setup.settings.storage._instance_id == ln_setup.settings.instance._id\n",
|
|
65
|
+
"assert Storage.objects.get(instance_uid=ln_setup.settings.instance.uid).root == ln_setup.settings.storage.root_as_str\n"
|
|
63
66
|
]
|
|
64
67
|
},
|
|
65
68
|
{
|
|
@@ -49,7 +49,7 @@
|
|
|
49
49
|
"metadata": {},
|
|
50
50
|
"outputs": [],
|
|
51
51
|
"source": [
|
|
52
|
-
"from
|
|
52
|
+
"from bionty import Source"
|
|
53
53
|
]
|
|
54
54
|
},
|
|
55
55
|
{
|
|
@@ -70,7 +70,7 @@
|
|
|
70
70
|
},
|
|
71
71
|
"outputs": [],
|
|
72
72
|
"source": [
|
|
73
|
-
"from
|
|
73
|
+
"from bionty.base import settings as bionty_base_settings\n",
|
|
74
74
|
"\n",
|
|
75
75
|
"assert sources_df.shape[0] > 0\n",
|
|
76
76
|
"assert bionty_base_settings.lamindb_sources.exists()"
|
|
@@ -154,7 +154,7 @@
|
|
|
154
154
|
"name": "python",
|
|
155
155
|
"nbconvert_exporter": "python",
|
|
156
156
|
"pygments_lexer": "ipython3",
|
|
157
|
-
"version": "3.
|
|
157
|
+
"version": "3.10.13"
|
|
158
158
|
},
|
|
159
159
|
"vscode": {
|
|
160
160
|
"interpreter": {
|
|
@@ -24,8 +24,9 @@ def close(mute: bool = False) -> None:
|
|
|
24
24
|
logger.warning("did not upload cache file - not enough permissions")
|
|
25
25
|
else:
|
|
26
26
|
raise e
|
|
27
|
+
if "bionty" in settings.instance.schema:
|
|
28
|
+
delete_bionty_sources_yaml()
|
|
27
29
|
current_instance_settings_file().unlink()
|
|
28
|
-
delete_bionty_sources_yaml()
|
|
29
30
|
clear_locker()
|
|
30
31
|
if not mute:
|
|
31
32
|
logger.success(f"closed instance: {instance}")
|
|
@@ -4,7 +4,6 @@ import os
|
|
|
4
4
|
from typing import TYPE_CHECKING
|
|
5
5
|
from uuid import UUID
|
|
6
6
|
|
|
7
|
-
from django.db import ProgrammingError
|
|
8
7
|
from lamin_utils import logger
|
|
9
8
|
|
|
10
9
|
from ._check_setup import _check_instance_setup
|
|
@@ -21,7 +20,7 @@ from .core._settings import settings
|
|
|
21
20
|
from .core._settings_instance import InstanceSettings
|
|
22
21
|
from .core._settings_load import load_instance_settings
|
|
23
22
|
from .core._settings_storage import StorageSettings
|
|
24
|
-
from .core._settings_store import instance_settings_file
|
|
23
|
+
from .core._settings_store import instance_settings_file, settings_dir
|
|
25
24
|
from .core.cloud_sqlite_locker import unlock_cloud_sqlite_upon_exception
|
|
26
25
|
|
|
27
26
|
if TYPE_CHECKING:
|
|
@@ -257,9 +256,109 @@ def connect(
|
|
|
257
256
|
if isettings is not None:
|
|
258
257
|
isettings._get_settings_file().unlink(missing_ok=True) # type: ignore
|
|
259
258
|
raise e
|
|
259
|
+
# rename lnschema_bionty to bionty for sql tables
|
|
260
|
+
if "bionty" in isettings.schema:
|
|
261
|
+
no_lnschema_bionty_file = (
|
|
262
|
+
settings_dir / f"no_lnschema_bionty-{isettings.slug.replace('/', '')}"
|
|
263
|
+
)
|
|
264
|
+
if not no_lnschema_bionty_file.exists():
|
|
265
|
+
migrate_lnschema_bionty(isettings, no_lnschema_bionty_file)
|
|
260
266
|
return None
|
|
261
267
|
|
|
262
268
|
|
|
269
|
+
def migrate_lnschema_bionty(isettings: InstanceSettings, no_lnschema_bionty_file: Path):
|
|
270
|
+
"""Migrate lnschema_bionty tables to bionty tables if bionty_source doesn't exist.
|
|
271
|
+
|
|
272
|
+
:param db_uri: str, database URI (e.g., 'sqlite:///path/to/db.sqlite' or 'postgresql://user:password@host:port/dbname')
|
|
273
|
+
"""
|
|
274
|
+
from urllib.parse import urlparse
|
|
275
|
+
|
|
276
|
+
parsed_uri = urlparse(isettings.db)
|
|
277
|
+
db_type = parsed_uri.scheme
|
|
278
|
+
|
|
279
|
+
if db_type == "sqlite":
|
|
280
|
+
import sqlite3
|
|
281
|
+
|
|
282
|
+
conn = sqlite3.connect(parsed_uri.path)
|
|
283
|
+
elif db_type in ["postgresql", "postgres"]:
|
|
284
|
+
import psycopg2
|
|
285
|
+
|
|
286
|
+
conn = psycopg2.connect(isettings.db)
|
|
287
|
+
else:
|
|
288
|
+
raise ValueError("Unsupported database type. Use 'sqlite' or 'postgresql' URI.")
|
|
289
|
+
|
|
290
|
+
cur = conn.cursor()
|
|
291
|
+
|
|
292
|
+
try:
|
|
293
|
+
# check if bionty_source table exists
|
|
294
|
+
if db_type == "sqlite":
|
|
295
|
+
cur.execute(
|
|
296
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name='bionty_source'"
|
|
297
|
+
)
|
|
298
|
+
migrated = cur.fetchone() is not None
|
|
299
|
+
|
|
300
|
+
# tables that need to be renamed
|
|
301
|
+
cur.execute(
|
|
302
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'lnschema_bionty_%'"
|
|
303
|
+
)
|
|
304
|
+
tables_to_rename = [
|
|
305
|
+
row[0][len("lnschema_bionty_") :] for row in cur.fetchall()
|
|
306
|
+
]
|
|
307
|
+
else: # postgres
|
|
308
|
+
cur.execute(
|
|
309
|
+
"SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'bionty_source')"
|
|
310
|
+
)
|
|
311
|
+
migrated = cur.fetchone()[0]
|
|
312
|
+
|
|
313
|
+
# tables that need to be renamed
|
|
314
|
+
cur.execute(
|
|
315
|
+
"SELECT table_name FROM information_schema.tables WHERE table_name LIKE 'lnschema_bionty_%'"
|
|
316
|
+
)
|
|
317
|
+
tables_to_rename = [
|
|
318
|
+
row[0][len("lnschema_bionty_") :] for row in cur.fetchall()
|
|
319
|
+
]
|
|
320
|
+
|
|
321
|
+
if migrated:
|
|
322
|
+
no_lnschema_bionty_file.touch(exist_ok=True)
|
|
323
|
+
else:
|
|
324
|
+
try:
|
|
325
|
+
# rename tables only if bionty_source doesn't exist and there are tables to rename
|
|
326
|
+
for table in tables_to_rename:
|
|
327
|
+
if db_type == "sqlite":
|
|
328
|
+
cur.execute(
|
|
329
|
+
f"ALTER TABLE lnschema_bionty_{table} RENAME TO bionty_{table}"
|
|
330
|
+
)
|
|
331
|
+
else: # postgres
|
|
332
|
+
cur.execute(
|
|
333
|
+
f"ALTER TABLE lnschema_bionty_{table} RENAME TO bionty_{table};"
|
|
334
|
+
)
|
|
335
|
+
|
|
336
|
+
# update django_migrations table
|
|
337
|
+
cur.execute(
|
|
338
|
+
"UPDATE django_migrations SET app = 'bionty' WHERE app = 'lnschema_bionty'"
|
|
339
|
+
)
|
|
340
|
+
|
|
341
|
+
logger.warning(
|
|
342
|
+
"Please uninstall lnschema-bionty via `pip uninstall lnschema-bionty`!"
|
|
343
|
+
)
|
|
344
|
+
|
|
345
|
+
no_lnschema_bionty_file.touch(exist_ok=True)
|
|
346
|
+
except Exception:
|
|
347
|
+
# read-only users can't rename tables
|
|
348
|
+
pass
|
|
349
|
+
|
|
350
|
+
conn.commit()
|
|
351
|
+
|
|
352
|
+
except Exception as e:
|
|
353
|
+
print(f"An error occurred: {e}")
|
|
354
|
+
conn.rollback()
|
|
355
|
+
|
|
356
|
+
finally:
|
|
357
|
+
# close the cursor and connection
|
|
358
|
+
cur.close()
|
|
359
|
+
conn.close()
|
|
360
|
+
|
|
361
|
+
|
|
263
362
|
def load(
|
|
264
363
|
slug: str,
|
|
265
364
|
*,
|
|
@@ -15,6 +15,7 @@ from ._close import close as close_instance
|
|
|
15
15
|
from ._silence_loggers import silence_loggers
|
|
16
16
|
from .core import InstanceSettings
|
|
17
17
|
from .core._settings import settings
|
|
18
|
+
from .core._settings_instance import is_local_db_url
|
|
18
19
|
from .core._settings_storage import StorageSettings, init_storage
|
|
19
20
|
from .core.upath import UPath
|
|
20
21
|
|
|
@@ -245,10 +246,16 @@ def init(
|
|
|
245
246
|
if instance_state == "connected":
|
|
246
247
|
settings.auto_connect = True # we can also debate this switch here
|
|
247
248
|
return None
|
|
248
|
-
#
|
|
249
|
-
#
|
|
250
|
-
#
|
|
251
|
-
|
|
249
|
+
# the conditions here match `isettings.is_remote`, but I currently don't
|
|
250
|
+
# see a way of making this more elegant; should become possible if we
|
|
251
|
+
# remove the instance.storage_id FK on the hub
|
|
252
|
+
prevent_register_hub = is_local_db_url(db) if db is not None else False
|
|
253
|
+
ssettings = init_storage(
|
|
254
|
+
storage,
|
|
255
|
+
instance_id=instance_id,
|
|
256
|
+
init_instance=True,
|
|
257
|
+
prevent_register_hub=prevent_register_hub,
|
|
258
|
+
)
|
|
252
259
|
isettings = InstanceSettings(
|
|
253
260
|
id=instance_id, # type: ignore
|
|
254
261
|
owner=settings.user.handle,
|
|
@@ -23,7 +23,10 @@ def register(_test: bool = False):
|
|
|
23
23
|
if ssettings._uid is None and _test:
|
|
24
24
|
# because django isn't up, we can't get it from the database
|
|
25
25
|
ssettings._uid = base62(8)
|
|
26
|
-
|
|
26
|
+
# cannot yet populate the instance id here
|
|
27
|
+
ssettings._instance_id = None
|
|
28
|
+
# flag auto_populate_instance can be removed once FK migration is over
|
|
29
|
+
init_storage_hub(ssettings, auto_populate_instance=False)
|
|
27
30
|
init_instance_hub(isettings)
|
|
28
31
|
isettings._is_on_hub = True
|
|
29
32
|
isettings._persist()
|
|
@@ -118,11 +118,13 @@ def _select_storage(
|
|
|
118
118
|
|
|
119
119
|
def init_storage(
|
|
120
120
|
ssettings: StorageSettings,
|
|
121
|
+
auto_populate_instance: bool = True,
|
|
121
122
|
) -> None:
|
|
122
123
|
if settings.user.handle != "anonymous":
|
|
123
124
|
return call_with_fallback_auth(
|
|
124
125
|
_init_storage,
|
|
125
126
|
ssettings=ssettings,
|
|
127
|
+
auto_populate_instance=auto_populate_instance,
|
|
126
128
|
)
|
|
127
129
|
else:
|
|
128
130
|
storage_exists = call_with_fallback(
|
|
@@ -134,7 +136,9 @@ def init_storage(
|
|
|
134
136
|
raise ValueError("Log in to create a storage location on the hub.")
|
|
135
137
|
|
|
136
138
|
|
|
137
|
-
def _init_storage(
|
|
139
|
+
def _init_storage(
|
|
140
|
+
ssettings: StorageSettings, auto_populate_instance: bool, client: Client
|
|
141
|
+
) -> None:
|
|
138
142
|
from lamindb_setup import settings
|
|
139
143
|
|
|
140
144
|
# storage roots are always stored without the trailing slash in the SQL
|
|
@@ -146,13 +150,19 @@ def _init_storage(ssettings: StorageSettings, client: Client) -> None:
|
|
|
146
150
|
id = uuid.uuid5(uuid.NAMESPACE_URL, root)
|
|
147
151
|
else:
|
|
148
152
|
id = uuid.uuid4()
|
|
149
|
-
if
|
|
153
|
+
if (
|
|
154
|
+
ssettings._instance_id is None
|
|
155
|
+
and settings._instance_exists
|
|
156
|
+
and auto_populate_instance
|
|
157
|
+
):
|
|
150
158
|
logger.warning(
|
|
151
159
|
f"will manage storage location {ssettings.root_as_str} with instance {settings.instance.slug}"
|
|
152
160
|
)
|
|
153
161
|
ssettings._instance_id = settings.instance._id
|
|
154
162
|
instance_id_hex = (
|
|
155
|
-
|
|
163
|
+
ssettings._instance_id.hex
|
|
164
|
+
if (ssettings._instance_id is not None and auto_populate_instance)
|
|
165
|
+
else None
|
|
156
166
|
)
|
|
157
167
|
fields = {
|
|
158
168
|
"id": id.hex,
|
|
@@ -29,6 +29,16 @@ def sanitize_git_repo_url(repo_url: str) -> str:
|
|
|
29
29
|
return repo_url.replace(".git", "")
|
|
30
30
|
|
|
31
31
|
|
|
32
|
+
def is_local_db_url(db_url: str) -> bool:
|
|
33
|
+
if "@localhost:" in db_url:
|
|
34
|
+
return True
|
|
35
|
+
if "@0.0.0.0:" in db_url:
|
|
36
|
+
return True
|
|
37
|
+
if "@127.0.0.1" in db_url:
|
|
38
|
+
return True
|
|
39
|
+
return False
|
|
40
|
+
|
|
41
|
+
|
|
32
42
|
class InstanceSettings:
|
|
33
43
|
"""Instance settings."""
|
|
34
44
|
|
|
@@ -370,17 +380,8 @@ class InstanceSettings:
|
|
|
370
380
|
if not self.storage.type_is_cloud:
|
|
371
381
|
return False
|
|
372
382
|
|
|
373
|
-
def is_local_uri(uri: str):
|
|
374
|
-
if "@localhost:" in uri:
|
|
375
|
-
return True
|
|
376
|
-
if "@0.0.0.0:" in uri:
|
|
377
|
-
return True
|
|
378
|
-
if "@127.0.0.1" in uri:
|
|
379
|
-
return True
|
|
380
|
-
return False
|
|
381
|
-
|
|
382
383
|
if self.dialect == "postgresql":
|
|
383
|
-
if
|
|
384
|
+
if is_local_db_url(self.db):
|
|
384
385
|
return False
|
|
385
386
|
# returns True for cloud SQLite
|
|
386
387
|
# and remote postgres
|
|
@@ -78,7 +78,11 @@ def mark_storage_root(root: UPathStr, uid: str):
|
|
|
78
78
|
|
|
79
79
|
|
|
80
80
|
def init_storage(
|
|
81
|
-
root: UPathStr,
|
|
81
|
+
root: UPathStr,
|
|
82
|
+
instance_id: UUID | None = None,
|
|
83
|
+
register_hub: bool | None = None,
|
|
84
|
+
prevent_register_hub: bool = False,
|
|
85
|
+
init_instance: bool = False,
|
|
82
86
|
) -> StorageSettings:
|
|
83
87
|
if root is None:
|
|
84
88
|
raise ValueError("`storage` argument can't be `None`")
|
|
@@ -114,11 +118,12 @@ def init_storage(
|
|
|
114
118
|
instance_id=instance_id,
|
|
115
119
|
)
|
|
116
120
|
# the below might update the uid with one that's already taken on the hub
|
|
117
|
-
if
|
|
118
|
-
|
|
119
|
-
|
|
121
|
+
if not prevent_register_hub:
|
|
122
|
+
if ssettings.type_is_cloud or register_hub:
|
|
123
|
+
from ._hub_core import delete_storage_record
|
|
124
|
+
from ._hub_core import init_storage as init_storage_hub
|
|
120
125
|
|
|
121
|
-
|
|
126
|
+
init_storage_hub(ssettings, auto_populate_instance=not init_instance)
|
|
122
127
|
# below comes last only if everything else was successful
|
|
123
128
|
try:
|
|
124
129
|
# (federated) credentials for AWS access are provisioned under-the-hood
|
|
@@ -7,6 +7,9 @@ from django.db.utils import OperationalError, ProgrammingError
|
|
|
7
7
|
if TYPE_CHECKING:
|
|
8
8
|
from ._settings_instance import InstanceSettings
|
|
9
9
|
|
|
10
|
+
# bionty.Source -> bionty.base
|
|
11
|
+
RENAME = {"name": "source", "description": "source_name"}
|
|
12
|
+
|
|
10
13
|
|
|
11
14
|
def write_bionty_sources(isettings: InstanceSettings) -> None:
|
|
12
15
|
"""Write bionty sources to Source table."""
|
|
@@ -14,9 +17,13 @@ def write_bionty_sources(isettings: InstanceSettings) -> None:
|
|
|
14
17
|
return None
|
|
15
18
|
import shutil
|
|
16
19
|
|
|
17
|
-
import
|
|
18
|
-
|
|
19
|
-
from
|
|
20
|
+
import bionty
|
|
21
|
+
import bionty.base as bionty_base
|
|
22
|
+
from bionty._bionty import list_biorecord_models
|
|
23
|
+
from bionty.base.dev._handle_sources import parse_sources_yaml
|
|
24
|
+
from bionty.models import Source
|
|
25
|
+
|
|
26
|
+
bionty_models = list_biorecord_models(bionty)
|
|
20
27
|
|
|
21
28
|
shutil.copy(
|
|
22
29
|
bionty_base.settings.current_sources, bionty_base.settings.lamindb_sources
|
|
@@ -46,7 +53,10 @@ def write_bionty_sources(isettings: InstanceSettings) -> None:
|
|
|
46
53
|
# won't need this once lamindb is released with the new pin
|
|
47
54
|
kwargs["run"] = None # can't yet access tracking information
|
|
48
55
|
kwargs["in_db"] = False
|
|
49
|
-
|
|
56
|
+
for db_field, base_col in RENAME.items():
|
|
57
|
+
kwargs[db_field] = kwargs.pop(base_col)
|
|
58
|
+
if kwargs["entity"] in bionty_models:
|
|
59
|
+
kwargs["entity"] = f"bionty.{kwargs['entity']}"
|
|
50
60
|
record = Source(**kwargs)
|
|
51
61
|
all_records.append(record)
|
|
52
62
|
|
|
@@ -58,14 +68,20 @@ def load_bionty_sources(isettings: InstanceSettings):
|
|
|
58
68
|
if "bionty" not in isettings.schema:
|
|
59
69
|
return None
|
|
60
70
|
|
|
61
|
-
import bionty_base
|
|
62
|
-
from
|
|
63
|
-
from
|
|
64
|
-
from
|
|
71
|
+
import bionty.base as bionty_base
|
|
72
|
+
from bionty.base.dev._handle_sources import parse_currently_used_sources
|
|
73
|
+
from bionty.base.dev._io import write_yaml
|
|
74
|
+
from bionty.models import Source
|
|
65
75
|
|
|
66
76
|
try:
|
|
67
77
|
# need try except because of integer primary key migration
|
|
68
78
|
active_records = Source.objects.filter(currently_used=True).all().values()
|
|
79
|
+
for kwargs in active_records:
|
|
80
|
+
for db_field, base_col in RENAME.items():
|
|
81
|
+
kwargs[base_col] = kwargs.pop(db_field)
|
|
82
|
+
# TODO: non-bionty schema?
|
|
83
|
+
if db_field == "entity":
|
|
84
|
+
kwargs["entity"] = kwargs["entity"].split(".")[1]
|
|
69
85
|
write_yaml(
|
|
70
86
|
parse_currently_used_sources(active_records),
|
|
71
87
|
bionty_base.settings.lamindb_sources,
|
|
@@ -77,7 +93,7 @@ def load_bionty_sources(isettings: InstanceSettings):
|
|
|
77
93
|
def delete_bionty_sources_yaml():
|
|
78
94
|
"""Delete LAMINDB_SOURCES_PATH in bionty."""
|
|
79
95
|
try:
|
|
80
|
-
import bionty_base
|
|
96
|
+
import bionty.base as bionty_base
|
|
81
97
|
|
|
82
98
|
bionty_base.settings.lamindb_sources.unlink(missing_ok=True)
|
|
83
99
|
except ModuleNotFoundError:
|
|
@@ -17,6 +17,8 @@ from typing import TYPE_CHECKING, Iterable
|
|
|
17
17
|
|
|
18
18
|
import psutil
|
|
19
19
|
|
|
20
|
+
HASH_LENGTH = 22
|
|
21
|
+
|
|
20
22
|
if TYPE_CHECKING:
|
|
21
23
|
from .types import Path, UPathStr
|
|
22
24
|
|
|
@@ -39,8 +41,8 @@ def b16_to_b64(s: str):
|
|
|
39
41
|
# a lot to read about this: lamin-notes/2022/hashing
|
|
40
42
|
def hash_set(s: set[str]) -> str:
|
|
41
43
|
bstr = ":".join(sorted(s)).encode("utf-8")
|
|
42
|
-
# as we're truncating at
|
|
43
|
-
return to_b64_str(hashlib.md5(bstr).digest())[:
|
|
44
|
+
# as we're truncating at 22 b64, we choose md5 over sha512
|
|
45
|
+
return to_b64_str(hashlib.md5(bstr).digest())[:HASH_LENGTH]
|
|
44
46
|
|
|
45
47
|
|
|
46
48
|
def hash_md5s_from_dir(hashes: Iterable[str]) -> tuple[str, str]:
|
|
@@ -50,7 +52,7 @@ def hash_md5s_from_dir(hashes: Iterable[str]) -> tuple[str, str]:
|
|
|
50
52
|
hashlib.md5(hash.encode("utf-8")).digest() for hash in sorted(hashes)
|
|
51
53
|
)
|
|
52
54
|
digest = hashlib.md5(digests).digest()
|
|
53
|
-
return to_b64_str(digest)[:
|
|
55
|
+
return to_b64_str(digest)[:HASH_LENGTH], "md5-d"
|
|
54
56
|
|
|
55
57
|
|
|
56
58
|
def hash_code(file_path: UPathStr):
|
|
@@ -85,7 +87,7 @@ def hash_file(
|
|
|
85
87
|
hashlib.sha1(first_chunk).digest() + hashlib.sha1(last_chunk).digest()
|
|
86
88
|
).digest()
|
|
87
89
|
hash_type = "sha1-fl"
|
|
88
|
-
return to_b64_str(digest)[:
|
|
90
|
+
return to_b64_str(digest)[:HASH_LENGTH], hash_type
|
|
89
91
|
|
|
90
92
|
|
|
91
93
|
def hash_dir(path: Path):
|
|
@@ -772,7 +772,7 @@ def check_storage_is_empty(
|
|
|
772
772
|
hint += " ignored"
|
|
773
773
|
message = (
|
|
774
774
|
f"Storage {directory_string} contains {n_objects - n_offset_objects} objects "
|
|
775
|
-
f"({hint}) - {ask_for_deletion}
|
|
775
|
+
f"({hint}) - {ask_for_deletion}"
|
|
776
776
|
)
|
|
777
777
|
if n_diff > 0:
|
|
778
778
|
if raise_error:
|
|
@@ -27,8 +27,8 @@ def lint(session: nox.Session) -> None:
|
|
|
27
27
|
["hub-local", "hub-prod", "hub-cloud", "storage", "docs"],
|
|
28
28
|
)
|
|
29
29
|
def install(session: nox.Session, group: str) -> None:
|
|
30
|
-
no_deps_packages = "git+https://github.com/laminlabs/lnschema-
|
|
31
|
-
schema_deps = f"""uv pip install --system bionty git+https://github.com/laminlabs/
|
|
30
|
+
no_deps_packages = "git+https://github.com/laminlabs/lnschema-core git+https://github.com/laminlabs/wetlab lamin-cli"
|
|
31
|
+
schema_deps = f"""uv pip install --system git+https://github.com/laminlabs/bionty git+https://github.com/laminlabs/lamindb@main
|
|
32
32
|
uv pip install --system --no-deps {no_deps_packages}
|
|
33
33
|
"""
|
|
34
34
|
if group == "hub-cloud":
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
|
|
5
|
+
from laminhub_rest.dev._seed import seed_local_test
|
|
6
|
+
from laminhub_rest.dev._supabase import SupabaseResources
|
|
7
|
+
from laminhub_rest.dev._utils import remove_lamin_local_settings
|
|
8
|
+
|
|
9
|
+
supabase_resources = SupabaseResources()
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
pytest_plugins = [
|
|
13
|
+
"laminhub_rest.core.account.user.test.fixtures",
|
|
14
|
+
"laminhub_rest.test.fixtures.run_id",
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def pytest_configure():
|
|
19
|
+
os.environ["LAMIN_ENV"] = "local"
|
|
20
|
+
remove_lamin_local_settings()
|
|
21
|
+
supabase_resources.start_local()
|
|
22
|
+
supabase_resources.reset_local()
|
|
23
|
+
supabase_resources.migrate()
|
|
24
|
+
seed_local_test()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def pytest_unconfigure():
|
|
28
|
+
if supabase_resources.edge_function_process:
|
|
29
|
+
supabase_resources.stop_local_edge_functions()
|
|
@@ -55,6 +55,20 @@ def test_update_schema_in_hub(setup_instance):
|
|
|
55
55
|
"related_schema_name": None,
|
|
56
56
|
}
|
|
57
57
|
|
|
58
|
+
assert schema["schema_json"]["core"]["artifact"]["fields"]["_accessor"] == {
|
|
59
|
+
"type": "CharField",
|
|
60
|
+
"column_name": "accessor",
|
|
61
|
+
"through": None,
|
|
62
|
+
"field_name": "_accessor",
|
|
63
|
+
"model_name": "artifact",
|
|
64
|
+
"schema_name": "core",
|
|
65
|
+
"is_link_table": False,
|
|
66
|
+
"relation_type": None,
|
|
67
|
+
"related_field_name": None,
|
|
68
|
+
"related_model_name": None,
|
|
69
|
+
"related_schema_name": None,
|
|
70
|
+
}
|
|
71
|
+
|
|
58
72
|
assert schema["schema_json"]["core"]["artifact"]["fields"]["created_by"] == {
|
|
59
73
|
"type": "ForeignKey",
|
|
60
74
|
"column_name": "created_by_id",
|
|
@@ -79,7 +93,7 @@ def test_update_schema_in_hub(setup_instance):
|
|
|
79
93
|
"through": {
|
|
80
94
|
"left_key": "gene_id",
|
|
81
95
|
"right_key": "pathway_id",
|
|
82
|
-
"link_table_name": "
|
|
96
|
+
"link_table_name": "bionty_pathway_genes",
|
|
83
97
|
},
|
|
84
98
|
"field_name": "pathways",
|
|
85
99
|
"model_name": "gene",
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
import os
|
|
4
|
-
|
|
5
|
-
from lamin_utils import logger
|
|
6
|
-
from laminhub_rest.dev._setup_laminapp_rest import setup_local
|
|
7
|
-
|
|
8
|
-
pytest_plugins = [
|
|
9
|
-
"laminhub_rest.core.account.user.test.fixtures",
|
|
10
|
-
"laminhub_rest.test.fixtures.run_id",
|
|
11
|
-
]
|
|
12
|
-
|
|
13
|
-
local_setup_state = setup_local()
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def pytest_configure():
|
|
17
|
-
assert os.environ["LAMIN_ENV"] == "local"
|
|
18
|
-
local_setup_state.__enter__()
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
def pytest_unconfigure():
|
|
22
|
-
if os.environ["LAMIN_ENV"] == "local":
|
|
23
|
-
local_setup_state.__exit__(None, None, None)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-cloud/02-connect-local-instance.ipynb
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-prod/test-insufficient-user-info.ipynb
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-prod/test_switch_and_fallback_env.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|