lamindb_setup 0.77.4__py2.py3-none-any.whl → 0.77.6__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lamindb_setup/__init__.py +1 -2
- lamindb_setup/_cache.py +34 -34
- lamindb_setup/_check.py +7 -7
- lamindb_setup/_check_setup.py +92 -79
- lamindb_setup/_close.py +35 -35
- lamindb_setup/_connect_instance.py +425 -444
- lamindb_setup/_django.py +41 -41
- lamindb_setup/_entry_points.py +22 -22
- lamindb_setup/_exportdb.py +68 -68
- lamindb_setup/_importdb.py +50 -50
- lamindb_setup/_init_instance.py +411 -374
- lamindb_setup/_migrate.py +239 -239
- lamindb_setup/_register_instance.py +36 -36
- lamindb_setup/_schema.py +27 -27
- lamindb_setup/_schema_metadata.py +411 -411
- lamindb_setup/_set_managed_storage.py +55 -55
- lamindb_setup/_setup_user.py +137 -137
- lamindb_setup/_silence_loggers.py +44 -44
- lamindb_setup/core/__init__.py +21 -21
- lamindb_setup/core/_aws_credentials.py +151 -151
- lamindb_setup/core/_aws_storage.py +48 -48
- lamindb_setup/core/_deprecated.py +55 -55
- lamindb_setup/core/_docs.py +14 -14
- lamindb_setup/core/_hub_client.py +1 -1
- lamindb_setup/core/_hub_core.py +615 -590
- lamindb_setup/core/_hub_crud.py +211 -211
- lamindb_setup/core/_hub_utils.py +109 -109
- lamindb_setup/core/_private_django_api.py +88 -88
- lamindb_setup/core/_settings.py +145 -138
- lamindb_setup/core/_settings_instance.py +480 -467
- lamindb_setup/core/_settings_load.py +105 -105
- lamindb_setup/core/_settings_save.py +81 -81
- lamindb_setup/core/_settings_storage.py +412 -405
- lamindb_setup/core/_settings_store.py +75 -75
- lamindb_setup/core/_settings_user.py +55 -53
- lamindb_setup/core/_setup_bionty_sources.py +101 -101
- lamindb_setup/core/cloud_sqlite_locker.py +237 -232
- lamindb_setup/core/django.py +115 -114
- lamindb_setup/core/exceptions.py +12 -12
- lamindb_setup/core/hashing.py +114 -114
- lamindb_setup/core/types.py +19 -19
- lamindb_setup/core/upath.py +779 -779
- {lamindb_setup-0.77.4.dist-info → lamindb_setup-0.77.6.dist-info}/METADATA +3 -2
- lamindb_setup-0.77.6.dist-info/RECORD +47 -0
- {lamindb_setup-0.77.4.dist-info → lamindb_setup-0.77.6.dist-info}/WHEEL +1 -1
- lamindb_setup-0.77.4.dist-info/RECORD +0 -47
- {lamindb_setup-0.77.4.dist-info → lamindb_setup-0.77.6.dist-info}/LICENSE +0 -0
lamindb_setup/_django.py
CHANGED
|
@@ -1,41 +1,41 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from typing import Optional
|
|
4
|
-
|
|
5
|
-
from .core._settings import settings
|
|
6
|
-
from .core.django import setup_django
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
def django(command: str, package_name: str | None = None, **kwargs):
|
|
10
|
-
r"""Manage migrations.
|
|
11
|
-
|
|
12
|
-
Examples:
|
|
13
|
-
|
|
14
|
-
Reset auto-incrementing primary integer ids after a database import:
|
|
15
|
-
|
|
16
|
-
>>> import lamindb as ln
|
|
17
|
-
>>> ln.setup.django("sqlsequencereset", "lnschema_core")
|
|
18
|
-
BEGIN;
|
|
19
|
-
SELECT setval(pg_get_serial_sequence('"lnschema_core_user"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lnschema_core_user"; # noqa
|
|
20
|
-
SELECT setval(pg_get_serial_sequence('"lnschema_core_storage"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lnschema_core_storage"; # noqa
|
|
21
|
-
COMMIT;
|
|
22
|
-
|
|
23
|
-
You can then run the SQL output that you'll see like so:
|
|
24
|
-
|
|
25
|
-
>>> sql = \"\"\"BEGIN;
|
|
26
|
-
SELECT setval(pg_get_serial_sequence('"lnschema_core_user"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lnschema_core_user"; # noqa
|
|
27
|
-
SELECT setval(pg_get_serial_sequence('"lnschema_core_storage"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lnschema_core_storage"; # noqa
|
|
28
|
-
COMMIT;\"\"\"
|
|
29
|
-
>>> from django.db import connection
|
|
30
|
-
>>> with connection.cursor() as cursor:
|
|
31
|
-
cursor.execute(sql)
|
|
32
|
-
|
|
33
|
-
"""
|
|
34
|
-
from django.core.management import call_command
|
|
35
|
-
|
|
36
|
-
setup_django(settings.instance)
|
|
37
|
-
if package_name is not None:
|
|
38
|
-
args = [package_name]
|
|
39
|
-
else:
|
|
40
|
-
args = []
|
|
41
|
-
call_command(command, *args, **kwargs)
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Optional
|
|
4
|
+
|
|
5
|
+
from .core._settings import settings
|
|
6
|
+
from .core.django import setup_django
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def django(command: str, package_name: str | None = None, **kwargs):
|
|
10
|
+
r"""Manage migrations.
|
|
11
|
+
|
|
12
|
+
Examples:
|
|
13
|
+
|
|
14
|
+
Reset auto-incrementing primary integer ids after a database import:
|
|
15
|
+
|
|
16
|
+
>>> import lamindb as ln
|
|
17
|
+
>>> ln.setup.django("sqlsequencereset", "lnschema_core")
|
|
18
|
+
BEGIN;
|
|
19
|
+
SELECT setval(pg_get_serial_sequence('"lnschema_core_user"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lnschema_core_user"; # noqa
|
|
20
|
+
SELECT setval(pg_get_serial_sequence('"lnschema_core_storage"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lnschema_core_storage"; # noqa
|
|
21
|
+
COMMIT;
|
|
22
|
+
|
|
23
|
+
You can then run the SQL output that you'll see like so:
|
|
24
|
+
|
|
25
|
+
>>> sql = \"\"\"BEGIN;
|
|
26
|
+
SELECT setval(pg_get_serial_sequence('"lnschema_core_user"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lnschema_core_user"; # noqa
|
|
27
|
+
SELECT setval(pg_get_serial_sequence('"lnschema_core_storage"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lnschema_core_storage"; # noqa
|
|
28
|
+
COMMIT;\"\"\"
|
|
29
|
+
>>> from django.db import connection
|
|
30
|
+
>>> with connection.cursor() as cursor:
|
|
31
|
+
cursor.execute(sql)
|
|
32
|
+
|
|
33
|
+
"""
|
|
34
|
+
from django.core.management import call_command
|
|
35
|
+
|
|
36
|
+
setup_django(settings.instance)
|
|
37
|
+
if package_name is not None:
|
|
38
|
+
args = [package_name]
|
|
39
|
+
else:
|
|
40
|
+
args = []
|
|
41
|
+
call_command(command, *args, **kwargs)
|
lamindb_setup/_entry_points.py
CHANGED
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
import sys
|
|
2
|
-
import warnings
|
|
3
|
-
from importlib.metadata import entry_points
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
def call_registered_entry_points(group, **kwargs):
|
|
7
|
-
"""load and call entry points registered under group."""
|
|
8
|
-
if sys.version_info >= (3, 10):
|
|
9
|
-
eps = entry_points(group=group)
|
|
10
|
-
else:
|
|
11
|
-
eps = entry_points().get(group, [])
|
|
12
|
-
|
|
13
|
-
for ep in eps:
|
|
14
|
-
func = ep.load()
|
|
15
|
-
try:
|
|
16
|
-
func(**kwargs)
|
|
17
|
-
except BaseException as e:
|
|
18
|
-
warnings.warn(
|
|
19
|
-
f"Error loading entry point of group {group!r}: {ep} -> {e}",
|
|
20
|
-
RuntimeWarning,
|
|
21
|
-
stacklevel=2,
|
|
22
|
-
)
|
|
1
|
+
import sys
|
|
2
|
+
import warnings
|
|
3
|
+
from importlib.metadata import entry_points
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def call_registered_entry_points(group, **kwargs):
|
|
7
|
+
"""load and call entry points registered under group."""
|
|
8
|
+
if sys.version_info >= (3, 10):
|
|
9
|
+
eps = entry_points(group=group)
|
|
10
|
+
else:
|
|
11
|
+
eps = entry_points().get(group, [])
|
|
12
|
+
|
|
13
|
+
for ep in eps:
|
|
14
|
+
func = ep.load()
|
|
15
|
+
try:
|
|
16
|
+
func(**kwargs)
|
|
17
|
+
except BaseException as e:
|
|
18
|
+
warnings.warn(
|
|
19
|
+
f"Error loading entry point of group {group!r}: {ep} -> {e}",
|
|
20
|
+
RuntimeWarning,
|
|
21
|
+
stacklevel=2,
|
|
22
|
+
)
|
lamindb_setup/_exportdb.py
CHANGED
|
@@ -1,68 +1,68 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from importlib import import_module
|
|
4
|
-
from pathlib import Path
|
|
5
|
-
|
|
6
|
-
MODELS = {
|
|
7
|
-
"core": {
|
|
8
|
-
"Collection": False,
|
|
9
|
-
"Artifact": False,
|
|
10
|
-
"Transform": False,
|
|
11
|
-
"Run": True,
|
|
12
|
-
"User": False,
|
|
13
|
-
"Storage": False,
|
|
14
|
-
"Feature": False,
|
|
15
|
-
"FeatureSet": False,
|
|
16
|
-
"ULabel": False,
|
|
17
|
-
},
|
|
18
|
-
# "bionty": {
|
|
19
|
-
# "Organism": False,
|
|
20
|
-
# "Gene": False,
|
|
21
|
-
# "Protein": False,
|
|
22
|
-
# "CellMarker": False,
|
|
23
|
-
# "Tissue": False,
|
|
24
|
-
# "CellType": False,
|
|
25
|
-
# "Disease": False,
|
|
26
|
-
# "CellLine": False,
|
|
27
|
-
# "Phenotype": False,
|
|
28
|
-
# "Pathway": False,
|
|
29
|
-
# "ExperimentalFactor": False,
|
|
30
|
-
# "DevelopmentalStage": False,
|
|
31
|
-
# "Ethnicity": False,
|
|
32
|
-
# "Source": False,
|
|
33
|
-
# },
|
|
34
|
-
# "wetlab": {
|
|
35
|
-
# "ExperimentType": False,
|
|
36
|
-
# "Experiment": False,
|
|
37
|
-
# "Well": False,
|
|
38
|
-
# "TreatmentTarget": False,
|
|
39
|
-
# "Treatment": False,
|
|
40
|
-
# "Biosample": False,
|
|
41
|
-
# "Techsample": False,
|
|
42
|
-
# },
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
def exportdb() -> None:
|
|
47
|
-
directory = Path("./lamindb_export/")
|
|
48
|
-
directory.mkdir(parents=True, exist_ok=True)
|
|
49
|
-
import pandas as pd
|
|
50
|
-
|
|
51
|
-
import lamindb_setup as ln_setup
|
|
52
|
-
|
|
53
|
-
def export_registry(registry, directory):
|
|
54
|
-
table_name = registry._meta.db_table
|
|
55
|
-
df = pd.read_sql_table(table_name, ln_setup.settings.instance.db)
|
|
56
|
-
df.to_parquet(directory / f"{table_name}.parquet", compression=None)
|
|
57
|
-
|
|
58
|
-
# export data to parquet files
|
|
59
|
-
print(f"\nexporting data to parquet files in: {directory}\n")
|
|
60
|
-
for schema_name, models in MODELS.items():
|
|
61
|
-
for model_name in models.keys():
|
|
62
|
-
schema_module = import_module(f"lnschema_{schema_name}")
|
|
63
|
-
registry = getattr(schema_module, model_name)
|
|
64
|
-
export_registry(registry, directory)
|
|
65
|
-
many_to_many_names = [field.name for field in registry._meta.many_to_many]
|
|
66
|
-
for many_to_many_name in many_to_many_names:
|
|
67
|
-
link_orm = getattr(registry, many_to_many_name).through
|
|
68
|
-
export_registry(link_orm, directory)
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from importlib import import_module
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
MODELS = {
|
|
7
|
+
"core": {
|
|
8
|
+
"Collection": False,
|
|
9
|
+
"Artifact": False,
|
|
10
|
+
"Transform": False,
|
|
11
|
+
"Run": True,
|
|
12
|
+
"User": False,
|
|
13
|
+
"Storage": False,
|
|
14
|
+
"Feature": False,
|
|
15
|
+
"FeatureSet": False,
|
|
16
|
+
"ULabel": False,
|
|
17
|
+
},
|
|
18
|
+
# "bionty": {
|
|
19
|
+
# "Organism": False,
|
|
20
|
+
# "Gene": False,
|
|
21
|
+
# "Protein": False,
|
|
22
|
+
# "CellMarker": False,
|
|
23
|
+
# "Tissue": False,
|
|
24
|
+
# "CellType": False,
|
|
25
|
+
# "Disease": False,
|
|
26
|
+
# "CellLine": False,
|
|
27
|
+
# "Phenotype": False,
|
|
28
|
+
# "Pathway": False,
|
|
29
|
+
# "ExperimentalFactor": False,
|
|
30
|
+
# "DevelopmentalStage": False,
|
|
31
|
+
# "Ethnicity": False,
|
|
32
|
+
# "Source": False,
|
|
33
|
+
# },
|
|
34
|
+
# "wetlab": {
|
|
35
|
+
# "ExperimentType": False,
|
|
36
|
+
# "Experiment": False,
|
|
37
|
+
# "Well": False,
|
|
38
|
+
# "TreatmentTarget": False,
|
|
39
|
+
# "Treatment": False,
|
|
40
|
+
# "Biosample": False,
|
|
41
|
+
# "Techsample": False,
|
|
42
|
+
# },
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def exportdb() -> None:
|
|
47
|
+
directory = Path("./lamindb_export/")
|
|
48
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
49
|
+
import pandas as pd
|
|
50
|
+
|
|
51
|
+
import lamindb_setup as ln_setup
|
|
52
|
+
|
|
53
|
+
def export_registry(registry, directory):
|
|
54
|
+
table_name = registry._meta.db_table
|
|
55
|
+
df = pd.read_sql_table(table_name, ln_setup.settings.instance.db)
|
|
56
|
+
df.to_parquet(directory / f"{table_name}.parquet", compression=None)
|
|
57
|
+
|
|
58
|
+
# export data to parquet files
|
|
59
|
+
print(f"\nexporting data to parquet files in: {directory}\n")
|
|
60
|
+
for schema_name, models in MODELS.items():
|
|
61
|
+
for model_name in models.keys():
|
|
62
|
+
schema_module = import_module(f"lnschema_{schema_name}")
|
|
63
|
+
registry = getattr(schema_module, model_name)
|
|
64
|
+
export_registry(registry, directory)
|
|
65
|
+
many_to_many_names = [field.name for field in registry._meta.many_to_many]
|
|
66
|
+
for many_to_many_name in many_to_many_names:
|
|
67
|
+
link_orm = getattr(registry, many_to_many_name).through
|
|
68
|
+
export_registry(link_orm, directory)
|
lamindb_setup/_importdb.py
CHANGED
|
@@ -1,50 +1,50 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from importlib import import_module
|
|
4
|
-
from pathlib import Path
|
|
5
|
-
|
|
6
|
-
from ._exportdb import MODELS
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
def import_registry(registry, directory, connection):
|
|
10
|
-
import pandas as pd
|
|
11
|
-
|
|
12
|
-
table_name = registry._meta.db_table
|
|
13
|
-
df = pd.read_parquet(directory / f"{table_name}.parquet")
|
|
14
|
-
old_foreign_key_columns = [
|
|
15
|
-
column for column in df.columns if column.endswith("_old")
|
|
16
|
-
]
|
|
17
|
-
for column in old_foreign_key_columns:
|
|
18
|
-
df.drop(column, axis=1, inplace=True)
|
|
19
|
-
df.to_sql(table_name, connection, if_exists="append", index=False)
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
def importdb() -> None:
|
|
23
|
-
# import data from parquet files
|
|
24
|
-
directory = Path("./lamindb_export/")
|
|
25
|
-
if directory.exists():
|
|
26
|
-
response = input(
|
|
27
|
-
f"\n\nDo you want to import registries from here: {directory}? (y/n)\n"
|
|
28
|
-
)
|
|
29
|
-
if response != "y":
|
|
30
|
-
return None
|
|
31
|
-
from sqlalchemy import create_engine, text
|
|
32
|
-
|
|
33
|
-
import lamindb_setup as ln_setup
|
|
34
|
-
|
|
35
|
-
engine = create_engine(ln_setup.settings.instance.db, echo=False)
|
|
36
|
-
with engine.begin() as connection:
|
|
37
|
-
if ln_setup.settings.instance.dialect == "postgresql":
|
|
38
|
-
connection.execute(text("SET CONSTRAINTS ALL DEFERRED;"))
|
|
39
|
-
for schema_name, models in MODELS.items():
|
|
40
|
-
for model_name in models.keys():
|
|
41
|
-
print(model_name)
|
|
42
|
-
schema_module = import_module(f"lnschema_{schema_name}")
|
|
43
|
-
registry = getattr(schema_module, model_name)
|
|
44
|
-
import_registry(registry, directory, connection)
|
|
45
|
-
many_to_many_names = [
|
|
46
|
-
field.name for field in registry._meta.many_to_many
|
|
47
|
-
]
|
|
48
|
-
for many_to_many_name in many_to_many_names:
|
|
49
|
-
link_orm = getattr(registry, many_to_many_name).through
|
|
50
|
-
import_registry(link_orm, directory, connection)
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from importlib import import_module
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from ._exportdb import MODELS
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def import_registry(registry, directory, connection):
|
|
10
|
+
import pandas as pd
|
|
11
|
+
|
|
12
|
+
table_name = registry._meta.db_table
|
|
13
|
+
df = pd.read_parquet(directory / f"{table_name}.parquet")
|
|
14
|
+
old_foreign_key_columns = [
|
|
15
|
+
column for column in df.columns if column.endswith("_old")
|
|
16
|
+
]
|
|
17
|
+
for column in old_foreign_key_columns:
|
|
18
|
+
df.drop(column, axis=1, inplace=True)
|
|
19
|
+
df.to_sql(table_name, connection, if_exists="append", index=False)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def importdb() -> None:
|
|
23
|
+
# import data from parquet files
|
|
24
|
+
directory = Path("./lamindb_export/")
|
|
25
|
+
if directory.exists():
|
|
26
|
+
response = input(
|
|
27
|
+
f"\n\nDo you want to import registries from here: {directory}? (y/n)\n"
|
|
28
|
+
)
|
|
29
|
+
if response != "y":
|
|
30
|
+
return None
|
|
31
|
+
from sqlalchemy import create_engine, text
|
|
32
|
+
|
|
33
|
+
import lamindb_setup as ln_setup
|
|
34
|
+
|
|
35
|
+
engine = create_engine(ln_setup.settings.instance.db, echo=False)
|
|
36
|
+
with engine.begin() as connection:
|
|
37
|
+
if ln_setup.settings.instance.dialect == "postgresql":
|
|
38
|
+
connection.execute(text("SET CONSTRAINTS ALL DEFERRED;"))
|
|
39
|
+
for schema_name, models in MODELS.items():
|
|
40
|
+
for model_name in models.keys():
|
|
41
|
+
print(model_name)
|
|
42
|
+
schema_module = import_module(f"lnschema_{schema_name}")
|
|
43
|
+
registry = getattr(schema_module, model_name)
|
|
44
|
+
import_registry(registry, directory, connection)
|
|
45
|
+
many_to_many_names = [
|
|
46
|
+
field.name for field in registry._meta.many_to_many
|
|
47
|
+
]
|
|
48
|
+
for many_to_many_name in many_to_many_names:
|
|
49
|
+
link_orm = getattr(registry, many_to_many_name).through
|
|
50
|
+
import_registry(link_orm, directory, connection)
|