lamindb_setup 0.81.4__py3-none-any.whl → 1.0a5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lamindb_setup/__init__.py +31 -2
- lamindb_setup/_check_setup.py +15 -16
- lamindb_setup/_close.py +1 -1
- lamindb_setup/_connect_instance.py +36 -34
- lamindb_setup/_delete.py +2 -2
- lamindb_setup/_django.py +6 -6
- lamindb_setup/_exportdb.py +4 -4
- lamindb_setup/_importdb.py +2 -2
- lamindb_setup/_init_instance.py +34 -53
- lamindb_setup/_migrate.py +5 -3
- lamindb_setup/_schema_metadata.py +15 -11
- lamindb_setup/core/_aws_credentials.py +10 -2
- lamindb_setup/core/_deprecated.py +12 -5
- lamindb_setup/core/_hub_core.py +2 -2
- lamindb_setup/core/_hub_crud.py +8 -11
- lamindb_setup/core/_hub_utils.py +0 -9
- lamindb_setup/core/_settings.py +2 -2
- lamindb_setup/core/_settings_instance.py +32 -9
- lamindb_setup/core/_settings_load.py +1 -1
- lamindb_setup/core/_settings_storage.py +1 -1
- lamindb_setup/core/_settings_user.py +1 -1
- lamindb_setup/core/_setup_bionty_sources.py +3 -3
- lamindb_setup/core/django.py +7 -7
- lamindb_setup/core/hashing.py +16 -5
- lamindb_setup/core/upath.py +79 -45
- {lamindb_setup-0.81.4.dist-info → lamindb_setup-1.0a5.dist-info}/METADATA +4 -5
- lamindb_setup-1.0a5.dist-info/RECORD +47 -0
- {lamindb_setup-0.81.4.dist-info → lamindb_setup-1.0a5.dist-info}/WHEEL +1 -1
- lamindb_setup-0.81.4.dist-info/RECORD +0 -47
- {lamindb_setup-0.81.4.dist-info → lamindb_setup-1.0a5.dist-info}/LICENSE +0 -0
lamindb_setup/__init__.py
CHANGED
|
@@ -33,7 +33,7 @@ Modules & settings:
|
|
|
33
33
|
|
|
34
34
|
"""
|
|
35
35
|
|
|
36
|
-
__version__ = "
|
|
36
|
+
__version__ = "1.0a5" # denote a release candidate for 0.1.0 with 0.1rc1
|
|
37
37
|
|
|
38
38
|
import os as _os
|
|
39
39
|
import sys as _sys
|
|
@@ -51,7 +51,36 @@ from ._register_instance import register
|
|
|
51
51
|
from ._setup_user import login, logout
|
|
52
52
|
from .core._settings import settings
|
|
53
53
|
|
|
54
|
-
|
|
54
|
+
|
|
55
|
+
def _is_CI_environment() -> bool:
|
|
56
|
+
ci_env_vars = [
|
|
57
|
+
"LAMIN_TESTING", # Set by our nox configurations
|
|
58
|
+
"CI", # Commonly set by many CI systems
|
|
59
|
+
"TRAVIS", # Travis CI
|
|
60
|
+
"GITHUB_ACTIONS", # GitHub Actions
|
|
61
|
+
"GITLAB_CI", # GitLab CI/CD
|
|
62
|
+
"CIRCLECI", # CircleCI
|
|
63
|
+
"JENKINS_URL", # Jenkins
|
|
64
|
+
"TEAMCITY_VERSION", # TeamCity
|
|
65
|
+
"BUILDKITE", # Buildkite
|
|
66
|
+
"BITBUCKET_BUILD_NUMBER", # Bitbucket Pipelines
|
|
67
|
+
"APPVEYOR", # AppVeyor
|
|
68
|
+
"AZURE_HTTP_USER_AGENT", # Azure Pipelines
|
|
69
|
+
"BUDDY", # Buddy
|
|
70
|
+
"DRONE", # Drone CI
|
|
71
|
+
"HUDSON_URL", # Hudson
|
|
72
|
+
"CF_BUILD_ID", # Codefresh
|
|
73
|
+
"WERCKER", # Wercker
|
|
74
|
+
"NOW_BUILDER", # ZEIT Now
|
|
75
|
+
"TASKCLUSTER_ROOT_URL", # TaskCluster
|
|
76
|
+
"SEMAPHORE", # Semaphore CI
|
|
77
|
+
"BUILD_ID", # Generic build environments
|
|
78
|
+
]
|
|
79
|
+
return any(env_var in _os.environ for env_var in ci_env_vars)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
_TESTING = _is_CI_environment()
|
|
83
|
+
|
|
55
84
|
|
|
56
85
|
# hide the supabase error in a thread on windows
|
|
57
86
|
if _os.name == "nt":
|
lamindb_setup/_check_setup.py
CHANGED
|
@@ -27,6 +27,7 @@ If you used the CLI to set up lamindb in a notebook, restart the Python session.
|
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
CURRENT_ISETTINGS: InstanceSettings | None = None
|
|
30
|
+
IS_LOADING: bool = False
|
|
30
31
|
|
|
31
32
|
|
|
32
33
|
def _get_current_instance_settings() -> InstanceSettings | None:
|
|
@@ -54,16 +55,11 @@ def _get_current_instance_settings() -> InstanceSettings | None:
|
|
|
54
55
|
|
|
55
56
|
# we make this a private function because in all the places it's used,
|
|
56
57
|
# users should not see it
|
|
57
|
-
def _check_instance_setup(
|
|
58
|
-
from_lamindb: bool = False, from_module: str | None = None
|
|
59
|
-
) -> bool:
|
|
60
|
-
reload_module = from_lamindb or from_module is not None
|
|
61
|
-
from ._init_instance import get_schema_module_name, reload_schema_modules
|
|
62
|
-
|
|
58
|
+
def _check_instance_setup(from_module: str | None = None) -> bool:
|
|
63
59
|
if django.IS_SETUP:
|
|
64
60
|
# reload logic here because module might not yet have been imported
|
|
65
61
|
# upon first setup
|
|
66
|
-
if from_module is not None:
|
|
62
|
+
if from_module is not None and from_module != "lamindb":
|
|
67
63
|
il.reload(il.import_module(from_module))
|
|
68
64
|
return True
|
|
69
65
|
silence_loggers()
|
|
@@ -75,18 +71,21 @@ def _check_instance_setup(
|
|
|
75
71
|
return True
|
|
76
72
|
isettings = _get_current_instance_settings()
|
|
77
73
|
if isettings is not None:
|
|
78
|
-
if
|
|
79
|
-
|
|
74
|
+
if (
|
|
75
|
+
from_module is not None
|
|
76
|
+
and settings.auto_connect
|
|
77
|
+
and not django.IS_SETUP
|
|
78
|
+
and not IS_LOADING
|
|
79
|
+
):
|
|
80
|
+
if not from_module == "lamindb":
|
|
81
|
+
import lamindb
|
|
82
|
+
|
|
83
|
+
il.reload(il.import_module(from_module))
|
|
84
|
+
else:
|
|
80
85
|
django.setup_django(isettings)
|
|
81
|
-
if from_module is not None:
|
|
82
|
-
# this only reloads `from_module`
|
|
83
|
-
il.reload(il.import_module(from_module))
|
|
84
|
-
else:
|
|
85
|
-
# this bulk reloads all schema modules
|
|
86
|
-
reload_schema_modules(isettings)
|
|
87
86
|
logger.important(f"connected lamindb: {isettings.slug}")
|
|
88
87
|
return django.IS_SETUP
|
|
89
88
|
else:
|
|
90
|
-
if
|
|
89
|
+
if from_module is not None and settings.auto_connect:
|
|
91
90
|
logger.warning(InstanceNotSetupError.default_message)
|
|
92
91
|
return False
|
lamindb_setup/_close.py
CHANGED
|
@@ -24,7 +24,7 @@ def close(mute: bool = False) -> None:
|
|
|
24
24
|
logger.warning("did not upload cache file - not enough permissions")
|
|
25
25
|
else:
|
|
26
26
|
raise e
|
|
27
|
-
if "bionty" in settings.instance.
|
|
27
|
+
if "bionty" in settings.instance.modules:
|
|
28
28
|
delete_bionty_sources_yaml()
|
|
29
29
|
current_instance_settings_file().unlink()
|
|
30
30
|
clear_locker()
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import importlib
|
|
3
4
|
import os
|
|
4
|
-
import sys
|
|
5
5
|
from typing import TYPE_CHECKING
|
|
6
6
|
from uuid import UUID
|
|
7
7
|
|
|
@@ -160,7 +160,7 @@ def _connect_instance(
|
|
|
160
160
|
name=name,
|
|
161
161
|
storage=ssettings,
|
|
162
162
|
db=db_updated,
|
|
163
|
-
|
|
163
|
+
modules=instance_result["schema_str"],
|
|
164
164
|
git_repo=instance_result["git_repo"],
|
|
165
165
|
keep_artifacts_local=bool(instance_result["keep_artifacts_local"]),
|
|
166
166
|
is_on_hub=True,
|
|
@@ -198,6 +198,7 @@ def connect(slug: str, **kwargs) -> str | tuple | None:
|
|
|
198
198
|
"_db",
|
|
199
199
|
"_write_settings",
|
|
200
200
|
"_raise_not_found_error",
|
|
201
|
+
"_reload_lamindb",
|
|
201
202
|
"_test",
|
|
202
203
|
"_user",
|
|
203
204
|
}
|
|
@@ -210,6 +211,7 @@ def connect(slug: str, **kwargs) -> str | tuple | None:
|
|
|
210
211
|
_db: str | None = kwargs.get("_db", None)
|
|
211
212
|
_write_settings: bool = kwargs.get("_write_settings", True)
|
|
212
213
|
_raise_not_found_error: bool = kwargs.get("_raise_not_found_error", True)
|
|
214
|
+
_reload_lamindb: bool = kwargs.get("_reload_lamindb", True)
|
|
213
215
|
_test: bool = kwargs.get("_test", False)
|
|
214
216
|
|
|
215
217
|
access_token: str | None = None
|
|
@@ -257,6 +259,14 @@ def connect(slug: str, **kwargs) -> str | tuple | None:
|
|
|
257
259
|
if _test:
|
|
258
260
|
return None
|
|
259
261
|
silence_loggers()
|
|
262
|
+
# migrate away from lnschema-core
|
|
263
|
+
no_lnschema_core_file = (
|
|
264
|
+
settings_dir / f"no_lnschema_core-{isettings.slug.replace('/', '--')}"
|
|
265
|
+
)
|
|
266
|
+
if not no_lnschema_core_file.exists():
|
|
267
|
+
migrate_lnschema_core(
|
|
268
|
+
isettings, no_lnschema_core_file, write_file=_write_settings
|
|
269
|
+
)
|
|
260
270
|
check, msg = isettings._load_db()
|
|
261
271
|
if not check:
|
|
262
272
|
local_db = (
|
|
@@ -292,21 +302,14 @@ def connect(slug: str, **kwargs) -> str | tuple | None:
|
|
|
292
302
|
# except ProgrammingError:
|
|
293
303
|
# pass
|
|
294
304
|
load_from_isettings(isettings, user=_user, write_settings=_write_settings)
|
|
305
|
+
if _reload_lamindb:
|
|
306
|
+
importlib.reload(importlib.import_module("lamindb"))
|
|
295
307
|
except Exception as e:
|
|
296
308
|
if isettings is not None:
|
|
297
309
|
if _write_settings:
|
|
298
310
|
isettings._get_settings_file().unlink(missing_ok=True) # type: ignore
|
|
299
311
|
settings._instance_settings = None
|
|
300
312
|
raise e
|
|
301
|
-
# rename lnschema_bionty to bionty for sql tables
|
|
302
|
-
if "bionty" in isettings.schema:
|
|
303
|
-
no_lnschema_bionty_file = (
|
|
304
|
-
settings_dir / f"no_lnschema_bionty-{isettings.slug.replace('/', '')}"
|
|
305
|
-
)
|
|
306
|
-
if not no_lnschema_bionty_file.exists():
|
|
307
|
-
migrate_lnschema_bionty(
|
|
308
|
-
isettings, no_lnschema_bionty_file, write_file=_write_settings
|
|
309
|
-
)
|
|
310
313
|
return None
|
|
311
314
|
|
|
312
315
|
|
|
@@ -322,13 +325,10 @@ def load(slug: str) -> str | tuple | None:
|
|
|
322
325
|
return result
|
|
323
326
|
|
|
324
327
|
|
|
325
|
-
def
|
|
326
|
-
isettings: InstanceSettings,
|
|
328
|
+
def migrate_lnschema_core(
|
|
329
|
+
isettings: InstanceSettings, no_lnschema_core_file: Path, write_file: bool = True
|
|
327
330
|
):
|
|
328
|
-
"""Migrate
|
|
329
|
-
|
|
330
|
-
:param db_uri: str, database URI (e.g., 'sqlite:///path/to/db.sqlite' or 'postgresql://user:password@host:port/dbname')
|
|
331
|
-
"""
|
|
331
|
+
"""Migrate lnschema_core tables to lamindb tables."""
|
|
332
332
|
from urllib.parse import urlparse
|
|
333
333
|
|
|
334
334
|
parsed_uri = urlparse(isettings.db)
|
|
@@ -348,60 +348,62 @@ def migrate_lnschema_bionty(
|
|
|
348
348
|
cur = conn.cursor()
|
|
349
349
|
|
|
350
350
|
try:
|
|
351
|
-
# check if bionty_source table exists
|
|
352
351
|
if db_type == "sqlite":
|
|
353
352
|
cur.execute(
|
|
354
|
-
"SELECT name FROM sqlite_master WHERE type='table' AND name='
|
|
353
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name='lamindb_user'"
|
|
355
354
|
)
|
|
356
355
|
migrated = cur.fetchone() is not None
|
|
357
356
|
|
|
358
357
|
# tables that need to be renamed
|
|
359
358
|
cur.execute(
|
|
360
|
-
"SELECT name FROM sqlite_master WHERE type='table' AND name LIKE '
|
|
359
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'lnschema_core_%'"
|
|
361
360
|
)
|
|
362
361
|
tables_to_rename = [
|
|
363
|
-
row[0][len("
|
|
362
|
+
row[0][len("lnschema_core_") :] for row in cur.fetchall()
|
|
364
363
|
]
|
|
365
364
|
else: # postgres
|
|
366
365
|
cur.execute(
|
|
367
|
-
"SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = '
|
|
366
|
+
"SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'lamindb_user')"
|
|
368
367
|
)
|
|
369
368
|
migrated = cur.fetchone()[0]
|
|
370
369
|
|
|
371
370
|
# tables that need to be renamed
|
|
372
371
|
cur.execute(
|
|
373
|
-
"SELECT table_name FROM information_schema.tables WHERE table_name LIKE '
|
|
372
|
+
"SELECT table_name FROM information_schema.tables WHERE table_name LIKE 'lnschema_core_%'"
|
|
374
373
|
)
|
|
375
374
|
tables_to_rename = [
|
|
376
|
-
row[0][len("
|
|
375
|
+
row[0][len("lnschema_core_") :] for row in cur.fetchall()
|
|
377
376
|
]
|
|
378
377
|
|
|
379
378
|
if migrated:
|
|
380
379
|
if write_file:
|
|
381
|
-
|
|
380
|
+
no_lnschema_core_file.touch(exist_ok=True)
|
|
382
381
|
else:
|
|
383
382
|
try:
|
|
384
|
-
|
|
383
|
+
response = input(
|
|
384
|
+
f"Do you want to migrate to lamindb 0.78 (integrate lnschema_core into lamindb)? (y/n) -- Will rename {tables_to_rename}"
|
|
385
|
+
)
|
|
386
|
+
if response != "y":
|
|
387
|
+
print("Aborted.")
|
|
388
|
+
quit()
|
|
385
389
|
for table in tables_to_rename:
|
|
386
390
|
if db_type == "sqlite":
|
|
387
391
|
cur.execute(
|
|
388
|
-
f"ALTER TABLE
|
|
392
|
+
f"ALTER TABLE lnschema_core_{table} RENAME TO lamindb_{table}"
|
|
389
393
|
)
|
|
390
394
|
else: # postgres
|
|
391
395
|
cur.execute(
|
|
392
|
-
f"ALTER TABLE
|
|
396
|
+
f"ALTER TABLE lnschema_core_{table} RENAME TO lamindb_{table};"
|
|
393
397
|
)
|
|
394
398
|
|
|
395
|
-
# update django_migrations table
|
|
396
399
|
cur.execute(
|
|
397
|
-
"UPDATE django_migrations SET app = '
|
|
400
|
+
"UPDATE django_migrations SET app = 'lamindb' WHERE app = 'lnschema_core'"
|
|
398
401
|
)
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
"Please uninstall lnschema-bionty via `pip uninstall lnschema-bionty`!"
|
|
402
|
+
print(
|
|
403
|
+
"Renaming tables finished.\nNow, *please* call: lamin migrate deploy"
|
|
402
404
|
)
|
|
403
405
|
if write_file:
|
|
404
|
-
|
|
406
|
+
no_lnschema_core_file.touch(exist_ok=True)
|
|
405
407
|
except Exception:
|
|
406
408
|
# read-only users can't rename tables
|
|
407
409
|
pass
|
lamindb_setup/_delete.py
CHANGED
|
@@ -99,7 +99,7 @@ def delete(slug: str, force: bool = False, require_empty: bool = True) -> int |
|
|
|
99
99
|
)
|
|
100
100
|
require_empty = True
|
|
101
101
|
# first the default storage
|
|
102
|
-
|
|
102
|
+
n_files = check_storage_is_empty(
|
|
103
103
|
isettings.storage.root,
|
|
104
104
|
raise_error=require_empty,
|
|
105
105
|
account_for_sqlite_file=isettings.dialect == "sqlite",
|
|
@@ -132,7 +132,7 @@ def delete(slug: str, force: bool = False, require_empty: bool = True) -> int |
|
|
|
132
132
|
delete_instance_on_hub(isettings._id, require_empty=False)
|
|
133
133
|
delete_by_isettings(isettings)
|
|
134
134
|
# if .lndb file was delete, then we might count -1
|
|
135
|
-
if
|
|
135
|
+
if n_files <= 0 and isettings.storage.type == "local":
|
|
136
136
|
# dir is only empty after sqlite file was delete via delete_by_isettings
|
|
137
137
|
if (isettings.storage.root / ".lamindb").exists():
|
|
138
138
|
(isettings.storage.root / ".lamindb").rmdir()
|
lamindb_setup/_django.py
CHANGED
|
@@ -5,24 +5,24 @@ from .core.django import setup_django
|
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
def django(command: str, package_name: str | None = None, **kwargs):
|
|
8
|
-
r"""
|
|
8
|
+
r"""Call Django commands.
|
|
9
9
|
|
|
10
10
|
Examples:
|
|
11
11
|
|
|
12
12
|
Reset auto-incrementing primary integer ids after a database import:
|
|
13
13
|
|
|
14
14
|
>>> import lamindb as ln
|
|
15
|
-
>>> ln.setup.django("sqlsequencereset", "
|
|
15
|
+
>>> ln.setup.django("sqlsequencereset", "lamindb")
|
|
16
16
|
BEGIN;
|
|
17
|
-
SELECT setval(pg_get_serial_sequence('"
|
|
18
|
-
SELECT setval(pg_get_serial_sequence('"
|
|
17
|
+
SELECT setval(pg_get_serial_sequence('"lamindb_user"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lamindb_user"; # noqa
|
|
18
|
+
SELECT setval(pg_get_serial_sequence('"lamindb_storage"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lamindb_storage"; # noqa
|
|
19
19
|
COMMIT;
|
|
20
20
|
|
|
21
21
|
You can then run the SQL output that you'll see like so:
|
|
22
22
|
|
|
23
23
|
>>> sql = \"\"\"BEGIN;
|
|
24
|
-
SELECT setval(pg_get_serial_sequence('"
|
|
25
|
-
SELECT setval(pg_get_serial_sequence('"
|
|
24
|
+
SELECT setval(pg_get_serial_sequence('"lamindb_user"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lamindb_user"; # noqa
|
|
25
|
+
SELECT setval(pg_get_serial_sequence('"lamindb_storage"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lamindb_storage"; # noqa
|
|
26
26
|
COMMIT;\"\"\"
|
|
27
27
|
>>> from django.db import connection
|
|
28
28
|
>>> with connection.cursor() as cursor:
|
lamindb_setup/_exportdb.py
CHANGED
|
@@ -4,7 +4,7 @@ from importlib import import_module
|
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
|
|
6
6
|
MODELS = {
|
|
7
|
-
"
|
|
7
|
+
"lamindb": {
|
|
8
8
|
"Collection": False,
|
|
9
9
|
"Artifact": False,
|
|
10
10
|
"Transform": False,
|
|
@@ -12,7 +12,7 @@ MODELS = {
|
|
|
12
12
|
"User": False,
|
|
13
13
|
"Storage": False,
|
|
14
14
|
"Feature": False,
|
|
15
|
-
"
|
|
15
|
+
"Schema": False,
|
|
16
16
|
"ULabel": False,
|
|
17
17
|
},
|
|
18
18
|
# "bionty": {
|
|
@@ -57,9 +57,9 @@ def exportdb() -> None:
|
|
|
57
57
|
|
|
58
58
|
# export data to parquet files
|
|
59
59
|
print(f"\nexporting data to parquet files in: {directory}\n")
|
|
60
|
-
for
|
|
60
|
+
for module_name, models in MODELS.items():
|
|
61
61
|
for model_name in models.keys():
|
|
62
|
-
schema_module = import_module(f"lnschema_{
|
|
62
|
+
schema_module = import_module(f"lnschema_{module_name}")
|
|
63
63
|
registry = getattr(schema_module, model_name)
|
|
64
64
|
export_registry(registry, directory)
|
|
65
65
|
many_to_many_names = [field.name for field in registry._meta.many_to_many]
|
lamindb_setup/_importdb.py
CHANGED
|
@@ -36,10 +36,10 @@ def importdb() -> None:
|
|
|
36
36
|
with engine.begin() as connection:
|
|
37
37
|
if ln_setup.settings.instance.dialect == "postgresql":
|
|
38
38
|
connection.execute(text("SET CONSTRAINTS ALL DEFERRED;"))
|
|
39
|
-
for
|
|
39
|
+
for module_name, models in MODELS.items():
|
|
40
40
|
for model_name in models.keys():
|
|
41
41
|
print(model_name)
|
|
42
|
-
schema_module = import_module(f"lnschema_{
|
|
42
|
+
schema_module = import_module(f"lnschema_{module_name}")
|
|
43
43
|
registry = getattr(schema_module, model_name)
|
|
44
44
|
import_registry(registry, directory, connection)
|
|
45
45
|
many_to_many_names = [
|
lamindb_setup/_init_instance.py
CHANGED
|
@@ -2,7 +2,6 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
import importlib
|
|
4
4
|
import os
|
|
5
|
-
import sys
|
|
6
5
|
import uuid
|
|
7
6
|
from typing import TYPE_CHECKING, Literal
|
|
8
7
|
from uuid import UUID
|
|
@@ -26,15 +25,17 @@ if TYPE_CHECKING:
|
|
|
26
25
|
from .core.types import UPathStr
|
|
27
26
|
|
|
28
27
|
|
|
29
|
-
def get_schema_module_name(
|
|
28
|
+
def get_schema_module_name(module_name, raise_import_error: bool = True) -> str | None:
|
|
30
29
|
import importlib.util
|
|
31
30
|
|
|
32
|
-
|
|
31
|
+
if module_name == "core":
|
|
32
|
+
return "lamindb"
|
|
33
|
+
name_attempts = [f"lnschema_{module_name.replace('-', '_')}", module_name]
|
|
33
34
|
for name in name_attempts:
|
|
34
35
|
module_spec = importlib.util.find_spec(name)
|
|
35
36
|
if module_spec is not None:
|
|
36
37
|
return name
|
|
37
|
-
message = f"
|
|
38
|
+
message = f"schema module '{module_name}' is not installed → no access to its labels & registries (resolve via `pip install {module_name}`)"
|
|
38
39
|
if raise_import_error:
|
|
39
40
|
raise ImportError(message)
|
|
40
41
|
logger.warning(message.lower())
|
|
@@ -42,8 +43,8 @@ def get_schema_module_name(schema_name, raise_import_error: bool = True) -> str
|
|
|
42
43
|
|
|
43
44
|
|
|
44
45
|
def register_storage_in_instance(ssettings: StorageSettings):
|
|
45
|
-
from
|
|
46
|
-
from
|
|
46
|
+
from lamindb.base.users import current_user_id
|
|
47
|
+
from lamindb.models import Storage
|
|
47
48
|
|
|
48
49
|
from .core.hashing import hash_and_encode_as_b62
|
|
49
50
|
|
|
@@ -71,7 +72,7 @@ def register_storage_in_instance(ssettings: StorageSettings):
|
|
|
71
72
|
|
|
72
73
|
|
|
73
74
|
def register_user(usettings):
|
|
74
|
-
from
|
|
75
|
+
from lamindb.models import User
|
|
75
76
|
|
|
76
77
|
try:
|
|
77
78
|
# need to have try except because of integer primary key migration
|
|
@@ -99,35 +100,6 @@ def register_user_and_storage_in_instance(isettings: InstanceSettings, usettings
|
|
|
99
100
|
logger.warning(f"instance seems not set up ({error})")
|
|
100
101
|
|
|
101
102
|
|
|
102
|
-
def reload_schema_modules(isettings: InstanceSettings, include_core: bool = True):
|
|
103
|
-
schema_names = ["core"] if include_core else []
|
|
104
|
-
# schema_names += list(isettings.schema)
|
|
105
|
-
schema_module_names = [get_schema_module_name(n) for n in schema_names]
|
|
106
|
-
|
|
107
|
-
for schema_module_name in schema_module_names:
|
|
108
|
-
if schema_module_name in sys.modules:
|
|
109
|
-
schema_module = importlib.import_module(schema_module_name)
|
|
110
|
-
importlib.reload(schema_module)
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
def reload_lamindb_itself(isettings) -> bool:
|
|
114
|
-
reloaded = False
|
|
115
|
-
if "lamindb" in sys.modules:
|
|
116
|
-
import lamindb
|
|
117
|
-
|
|
118
|
-
importlib.reload(lamindb)
|
|
119
|
-
reloaded = True
|
|
120
|
-
return reloaded
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
def reload_lamindb(isettings: InstanceSettings):
|
|
124
|
-
log_message = settings.auto_connect
|
|
125
|
-
if not reload_lamindb_itself(isettings):
|
|
126
|
-
log_message = True
|
|
127
|
-
if log_message:
|
|
128
|
-
logger.important(f"connected lamindb: {isettings.slug}")
|
|
129
|
-
|
|
130
|
-
|
|
131
103
|
ERROR_SQLITE_CACHE = """
|
|
132
104
|
Your cached local SQLite file exists, while your cloud SQLite file ({}) doesn't.
|
|
133
105
|
Either delete your cache ({}) or add it back to the cloud (if delete was accidental).
|
|
@@ -157,12 +129,21 @@ def process_connect_response(
|
|
|
157
129
|
return instance_id, instance_state
|
|
158
130
|
|
|
159
131
|
|
|
132
|
+
def process_modules_arg(modules: str | None = None) -> str:
|
|
133
|
+
if modules is None or modules == "":
|
|
134
|
+
return ""
|
|
135
|
+
# currently no actual validation, can add back if we see a need
|
|
136
|
+
# the following just strips white spaces
|
|
137
|
+
to_be_validated = [s.strip() for s in modules.split(",")]
|
|
138
|
+
return ",".join(to_be_validated)
|
|
139
|
+
|
|
140
|
+
|
|
160
141
|
def validate_init_args(
|
|
161
142
|
*,
|
|
162
143
|
storage: UPathStr,
|
|
163
144
|
name: str | None = None,
|
|
164
145
|
db: PostgresDsn | None = None,
|
|
165
|
-
|
|
146
|
+
modules: str | None = None,
|
|
166
147
|
_test: bool = False,
|
|
167
148
|
_write_settings: bool = True,
|
|
168
149
|
_user: UserSettings | None = None,
|
|
@@ -178,9 +159,6 @@ def validate_init_args(
|
|
|
178
159
|
str,
|
|
179
160
|
]:
|
|
180
161
|
from ._connect_instance import connect
|
|
181
|
-
from .core._hub_utils import (
|
|
182
|
-
validate_schema_arg,
|
|
183
|
-
)
|
|
184
162
|
|
|
185
163
|
if storage is None:
|
|
186
164
|
raise SystemExit("✗ `storage` argument can't be `None`")
|
|
@@ -206,7 +184,7 @@ def validate_init_args(
|
|
|
206
184
|
instance_id = None
|
|
207
185
|
if response is not None:
|
|
208
186
|
instance_id, instance_state = process_connect_response(response, instance_slug)
|
|
209
|
-
|
|
187
|
+
modules = process_modules_arg(modules)
|
|
210
188
|
return name_str, instance_id, instance_state, instance_slug
|
|
211
189
|
|
|
212
190
|
|
|
@@ -230,22 +208,23 @@ def init(
|
|
|
230
208
|
storage: UPathStr,
|
|
231
209
|
name: str | None = None,
|
|
232
210
|
db: PostgresDsn | None = None,
|
|
233
|
-
|
|
211
|
+
modules: str | None = None,
|
|
234
212
|
**kwargs,
|
|
235
213
|
) -> None:
|
|
236
214
|
"""Create and load a LaminDB instance.
|
|
237
215
|
|
|
238
216
|
Args:
|
|
239
|
-
storage: Either
|
|
240
|
-
remote folder (``"s3://..."`` or ``"gs://..."``).
|
|
217
|
+
storage: Either local or remote folder (`"s3://..."` or `"gs://..."`).
|
|
241
218
|
name: Instance name.
|
|
242
219
|
db: Database connection url, do not pass for SQLite.
|
|
243
|
-
|
|
220
|
+
modules: Comma-separated string of modules. None if the lamindb registries are enough.
|
|
244
221
|
"""
|
|
245
222
|
isettings = None
|
|
246
223
|
ssettings = None
|
|
247
224
|
|
|
248
225
|
_write_settings: bool = kwargs.get("_write_settings", True)
|
|
226
|
+
if modules is None:
|
|
227
|
+
modules = kwargs.get("schema", None)
|
|
249
228
|
_test: bool = kwargs.get("_test", False)
|
|
250
229
|
|
|
251
230
|
# use this user instead of settings.user
|
|
@@ -269,7 +248,7 @@ def init(
|
|
|
269
248
|
storage=storage,
|
|
270
249
|
name=name,
|
|
271
250
|
db=db,
|
|
272
|
-
|
|
251
|
+
modules=modules,
|
|
273
252
|
_test=_test,
|
|
274
253
|
_write_settings=_write_settings,
|
|
275
254
|
_user=_user, # will get from settings.user if _user is None
|
|
@@ -296,7 +275,7 @@ def init(
|
|
|
296
275
|
name=name_str,
|
|
297
276
|
storage=ssettings,
|
|
298
277
|
db=db,
|
|
299
|
-
|
|
278
|
+
modules=modules,
|
|
300
279
|
uid=ssettings.uid,
|
|
301
280
|
# to lock passed user in isettings._cloud_sqlite_locker.lock()
|
|
302
281
|
_locker_user=_user, # only has effect if cloud sqlite
|
|
@@ -329,6 +308,8 @@ def init(
|
|
|
329
308
|
update_schema_in_hub(access_token=access_token)
|
|
330
309
|
if _write_settings:
|
|
331
310
|
settings.auto_connect = True
|
|
311
|
+
importlib.reload(importlib.import_module("lamindb"))
|
|
312
|
+
logger.important(f"initialized lamindb: {isettings.slug}")
|
|
332
313
|
except Exception as e:
|
|
333
314
|
from ._delete import delete_by_isettings
|
|
334
315
|
from .core._hub_core import delete_instance_record, delete_storage_record
|
|
@@ -338,16 +319,17 @@ def init(
|
|
|
338
319
|
delete_by_isettings(isettings)
|
|
339
320
|
else:
|
|
340
321
|
settings._instance_settings = None
|
|
341
|
-
if (
|
|
342
|
-
user_handle != "anonymous" or access_token is not None
|
|
343
|
-
) and isettings.is_on_hub:
|
|
344
|
-
delete_instance_record(isettings._id, access_token=access_token)
|
|
345
322
|
if (
|
|
346
323
|
ssettings is not None
|
|
347
324
|
and (user_handle != "anonymous" or access_token is not None)
|
|
348
325
|
and ssettings.is_on_hub
|
|
349
326
|
):
|
|
350
327
|
delete_storage_record(ssettings._uuid, access_token=access_token) # type: ignore
|
|
328
|
+
if isettings is not None:
|
|
329
|
+
if (
|
|
330
|
+
user_handle != "anonymous" or access_token is not None
|
|
331
|
+
) and isettings.is_on_hub:
|
|
332
|
+
delete_instance_record(isettings._id, access_token=access_token)
|
|
351
333
|
raise e
|
|
352
334
|
return None
|
|
353
335
|
|
|
@@ -378,7 +360,6 @@ def load_from_isettings(
|
|
|
378
360
|
if not isettings._get_settings_file().exists():
|
|
379
361
|
register_user(user)
|
|
380
362
|
isettings._persist(write_to_disk=write_settings)
|
|
381
|
-
reload_lamindb(isettings)
|
|
382
363
|
|
|
383
364
|
|
|
384
365
|
def validate_sqlite_state(isettings: InstanceSettings) -> None:
|
|
@@ -414,7 +395,7 @@ def infer_instance_name(
|
|
|
414
395
|
return str(db).split("/")[-1]
|
|
415
396
|
if storage == "create-s3":
|
|
416
397
|
raise ValueError("pass name to init if storage = 'create-s3'")
|
|
417
|
-
storage_path = UPath(storage)
|
|
398
|
+
storage_path = UPath(storage).resolve()
|
|
418
399
|
# not sure if name is ever ""
|
|
419
400
|
if storage_path.name != "":
|
|
420
401
|
name = storage_path.name
|
lamindb_setup/_migrate.py
CHANGED
|
@@ -5,7 +5,7 @@ from django.db.migrations.loader import MigrationLoader
|
|
|
5
5
|
from lamin_utils import logger
|
|
6
6
|
from packaging import version
|
|
7
7
|
|
|
8
|
-
from .
|
|
8
|
+
from . import _check_setup
|
|
9
9
|
from .core._settings import settings
|
|
10
10
|
from .core.django import setup_django
|
|
11
11
|
|
|
@@ -64,16 +64,18 @@ class migrate:
|
|
|
64
64
|
@classmethod
|
|
65
65
|
def create(cls) -> None:
|
|
66
66
|
"""Create a migration."""
|
|
67
|
-
if _check_instance_setup():
|
|
67
|
+
if _check_setup._check_instance_setup():
|
|
68
68
|
raise RuntimeError("Restart Python session to create migration or use CLI!")
|
|
69
|
+
_check_setup.IS_LOADING = True
|
|
69
70
|
setup_django(settings.instance, create_migrations=True)
|
|
71
|
+
_check_setup.IS_LOADING = False
|
|
70
72
|
|
|
71
73
|
@classmethod
|
|
72
74
|
def deploy(cls) -> None:
|
|
73
75
|
"""Deploy a migration."""
|
|
74
76
|
from ._schema_metadata import update_schema_in_hub
|
|
75
77
|
|
|
76
|
-
if _check_instance_setup():
|
|
78
|
+
if _check_setup._check_instance_setup():
|
|
77
79
|
raise RuntimeError("Restart Python session to migrate or use CLI!")
|
|
78
80
|
from lamindb_setup.core._hub_client import call_with_fallback_auth
|
|
79
81
|
from lamindb_setup.core._hub_crud import (
|