lamindb_setup 1.17.0__py3-none-any.whl → 1.18.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lamindb_setup/__init__.py +1 -1
- lamindb_setup/_connect_instance.py +14 -10
- lamindb_setup/_migrate.py +56 -16
- lamindb_setup/_set_managed_storage.py +3 -0
- lamindb_setup/core/__init__.py +0 -3
- lamindb_setup/core/_aws_options.py +1 -1
- lamindb_setup/core/_clone.py +1 -125
- lamindb_setup/core/_private_django_api.py +1 -3
- lamindb_setup/core/_settings_instance.py +12 -18
- lamindb_setup/core/_settings_storage.py +0 -5
- lamindb_setup/core/django.py +9 -9
- lamindb_setup/core/hashing.py +5 -5
- lamindb_setup/core/lamin.db.gz +0 -0
- lamindb_setup/errors.py +5 -0
- {lamindb_setup-1.17.0.dist-info → lamindb_setup-1.18.1.dist-info}/METADATA +5 -5
- {lamindb_setup-1.17.0.dist-info → lamindb_setup-1.18.1.dist-info}/RECORD +18 -18
- {lamindb_setup-1.17.0.dist-info → lamindb_setup-1.18.1.dist-info}/WHEEL +1 -1
- {lamindb_setup-1.17.0.dist-info/licenses → lamindb_setup-1.18.1.dist-info}/LICENSE +0 -0
lamindb_setup/__init__.py
CHANGED
|
@@ -22,7 +22,7 @@ from .core._settings_storage import StorageSettings
|
|
|
22
22
|
from .core._settings_store import instance_settings_file
|
|
23
23
|
from .core.cloud_sqlite_locker import unlock_cloud_sqlite_upon_exception
|
|
24
24
|
from .core.django import reset_django
|
|
25
|
-
from .errors import CannotSwitchDefaultInstance
|
|
25
|
+
from .errors import CannotSwitchDefaultInstance, InstanceNotFoundError
|
|
26
26
|
|
|
27
27
|
if TYPE_CHECKING:
|
|
28
28
|
from pathlib import Path
|
|
@@ -33,8 +33,6 @@ if TYPE_CHECKING:
|
|
|
33
33
|
# this is for testing purposes only
|
|
34
34
|
# set to True only to test failed load
|
|
35
35
|
_TEST_FAILED_LOAD = False
|
|
36
|
-
|
|
37
|
-
|
|
38
36
|
INSTANCE_NOT_FOUND_MESSAGE = (
|
|
39
37
|
"'{owner}/{name}' not found:"
|
|
40
38
|
" '{hub_result}'\nCheck your permissions:"
|
|
@@ -42,10 +40,6 @@ INSTANCE_NOT_FOUND_MESSAGE = (
|
|
|
42
40
|
)
|
|
43
41
|
|
|
44
42
|
|
|
45
|
-
class InstanceNotFoundError(SystemExit):
|
|
46
|
-
pass
|
|
47
|
-
|
|
48
|
-
|
|
49
43
|
def check_db_dsn_equal_up_to_credentials(db_dsn_hub, db_dsn_local):
|
|
50
44
|
return (
|
|
51
45
|
db_dsn_hub.scheme == db_dsn_local.scheme
|
|
@@ -102,6 +96,7 @@ def _connect_instance(
|
|
|
102
96
|
use_root_db_user: bool = False,
|
|
103
97
|
use_proxy_db: bool = False,
|
|
104
98
|
access_token: str | None = None,
|
|
99
|
+
raise_systemexit: bool = False,
|
|
105
100
|
) -> InstanceSettings:
|
|
106
101
|
settings_file = instance_settings_file(name, owner)
|
|
107
102
|
make_hub_request = True
|
|
@@ -170,12 +165,17 @@ def _connect_instance(
|
|
|
170
165
|
)
|
|
171
166
|
else:
|
|
172
167
|
message = "It is not possible to load an anonymous-owned instance from the hub"
|
|
168
|
+
exception = (
|
|
169
|
+
SystemExit(message)
|
|
170
|
+
if raise_systemexit
|
|
171
|
+
else InstanceNotFoundError(message)
|
|
172
|
+
)
|
|
173
173
|
if settings_file.exists():
|
|
174
174
|
isettings = load_instance_settings(settings_file)
|
|
175
175
|
if isettings.is_remote:
|
|
176
|
-
raise
|
|
176
|
+
raise exception
|
|
177
177
|
else:
|
|
178
|
-
raise
|
|
178
|
+
raise exception
|
|
179
179
|
return isettings
|
|
180
180
|
|
|
181
181
|
|
|
@@ -239,7 +239,11 @@ def _connect_cli(
|
|
|
239
239
|
|
|
240
240
|
owner, name = get_owner_name_from_identifier(instance)
|
|
241
241
|
isettings = _connect_instance(
|
|
242
|
-
owner,
|
|
242
|
+
owner,
|
|
243
|
+
name,
|
|
244
|
+
use_root_db_user=use_root_db_user,
|
|
245
|
+
use_proxy_db=use_proxy_db,
|
|
246
|
+
raise_systemexit=True,
|
|
243
247
|
)
|
|
244
248
|
isettings._persist(write_to_disk=True)
|
|
245
249
|
if not isettings.is_on_hub or isettings._is_cloud_sqlite:
|
lamindb_setup/_migrate.py
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import os
|
|
4
|
+
|
|
3
5
|
import httpx
|
|
4
6
|
from django.db import connection
|
|
5
7
|
from django.db.migrations.loader import MigrationLoader
|
|
@@ -98,16 +100,19 @@ class migrate:
|
|
|
98
100
|
|
|
99
101
|
@classmethod
|
|
100
102
|
def deploy(cls, package_name: str | None = None, number: int | None = None) -> None:
|
|
101
|
-
|
|
103
|
+
assert settings._instance_exists, (
|
|
104
|
+
"Not connected to an instance, please connect to migrate."
|
|
105
|
+
)
|
|
102
106
|
|
|
103
107
|
# NOTE: this is a temporary solution to avoid breaking tests
|
|
104
108
|
LAMIN_MIGRATE_ON_LAMBDA = (
|
|
105
|
-
os.
|
|
109
|
+
os.getenv("LAMIN_MIGRATE_ON_LAMBDA", "false") == "true"
|
|
106
110
|
)
|
|
111
|
+
isettings = settings.instance
|
|
107
112
|
|
|
108
|
-
if
|
|
113
|
+
if isettings.is_on_hub and LAMIN_MIGRATE_ON_LAMBDA:
|
|
109
114
|
response = httpx.post(
|
|
110
|
-
f"{
|
|
115
|
+
f"{isettings.api_url}/instances/{isettings._id}/migrate",
|
|
111
116
|
headers={"Authorization": f"Bearer {settings.user.access_token}"},
|
|
112
117
|
timeout=None, # this can take time
|
|
113
118
|
)
|
|
@@ -128,32 +133,38 @@ class migrate:
|
|
|
128
133
|
update_instance,
|
|
129
134
|
)
|
|
130
135
|
|
|
131
|
-
|
|
136
|
+
isettings = settings.instance
|
|
137
|
+
is_managed_by_hub = isettings.is_managed_by_hub
|
|
138
|
+
is_on_hub = is_managed_by_hub or isettings.is_on_hub
|
|
139
|
+
|
|
140
|
+
if is_managed_by_hub and "root" not in isettings.db:
|
|
132
141
|
# ensure we connect with the root user
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
142
|
+
connect(use_root_db_user=True)
|
|
143
|
+
assert "root" in (instance_db := settings.instance.db), instance_db
|
|
144
|
+
if is_on_hub:
|
|
136
145
|
# we need lamindb to be installed, otherwise we can't populate the version
|
|
137
146
|
# information in the hub
|
|
147
|
+
# this also connects
|
|
138
148
|
import lamindb
|
|
139
|
-
|
|
149
|
+
# this is needed to avoid connecting on importing apps inside setup_django process
|
|
150
|
+
setup_django_disable_autoconnect = disable_auto_connect(setup_django)
|
|
140
151
|
# this sets up django and deploys the migrations
|
|
141
152
|
if package_name is not None and number is not None:
|
|
142
|
-
|
|
143
|
-
|
|
153
|
+
setup_django_disable_autoconnect(
|
|
154
|
+
isettings,
|
|
144
155
|
deploy_migrations=True,
|
|
145
156
|
appname_number=(package_name, number),
|
|
146
157
|
)
|
|
147
158
|
else:
|
|
148
|
-
|
|
159
|
+
setup_django_disable_autoconnect(isettings, deploy_migrations=True)
|
|
149
160
|
# this populates the hub
|
|
150
|
-
if
|
|
161
|
+
if is_on_hub:
|
|
151
162
|
logger.important(f"updating lamindb version in hub: {lamindb.__version__}")
|
|
152
|
-
if
|
|
163
|
+
if isettings.dialect != "sqlite":
|
|
153
164
|
update_schema_in_hub()
|
|
154
165
|
call_with_fallback_auth(
|
|
155
166
|
update_instance,
|
|
156
|
-
instance_id=
|
|
167
|
+
instance_id=isettings._id.hex,
|
|
157
168
|
instance_fields={"lamindb_version": lamindb.__version__},
|
|
158
169
|
)
|
|
159
170
|
|
|
@@ -161,16 +172,45 @@ class migrate:
|
|
|
161
172
|
@disable_auto_connect
|
|
162
173
|
def check(cls) -> bool:
|
|
163
174
|
"""Check whether Registry definitions are in sync with migrations."""
|
|
175
|
+
import io
|
|
176
|
+
|
|
164
177
|
from django.core.management import call_command
|
|
165
178
|
|
|
166
179
|
setup_django(settings.instance)
|
|
180
|
+
|
|
181
|
+
# Capture stdout/stderr to show what migrations are needed if check fails
|
|
182
|
+
stdout = io.StringIO()
|
|
183
|
+
stderr = io.StringIO()
|
|
184
|
+
|
|
167
185
|
try:
|
|
168
|
-
call_command(
|
|
186
|
+
call_command(
|
|
187
|
+
"makemigrations", check_changes=True, stdout=stdout, stderr=stderr
|
|
188
|
+
)
|
|
169
189
|
except SystemExit:
|
|
170
190
|
logger.error(
|
|
171
191
|
"migrations are not in sync with ORMs, please create a migration: lamin"
|
|
172
192
|
" migrate create"
|
|
173
193
|
)
|
|
194
|
+
# Print captured output from the check
|
|
195
|
+
if stdout.getvalue():
|
|
196
|
+
logger.error(f"makemigrations --check stdout:\n{stdout.getvalue()}")
|
|
197
|
+
if stderr.getvalue():
|
|
198
|
+
logger.error(f"makemigrations --check stderr:\n{stderr.getvalue()}")
|
|
199
|
+
|
|
200
|
+
# Run makemigrations --dry-run to show what would be created
|
|
201
|
+
stdout2 = io.StringIO()
|
|
202
|
+
stderr2 = io.StringIO()
|
|
203
|
+
try:
|
|
204
|
+
call_command(
|
|
205
|
+
"makemigrations", dry_run=True, stdout=stdout2, stderr=stderr2
|
|
206
|
+
)
|
|
207
|
+
except SystemExit:
|
|
208
|
+
pass
|
|
209
|
+
if stdout2.getvalue():
|
|
210
|
+
logger.error(f"makemigrations --dry-run stdout:\n{stdout2.getvalue()}")
|
|
211
|
+
if stderr2.getvalue():
|
|
212
|
+
logger.error(f"makemigrations --dry-run stderr:\n{stderr2.getvalue()}")
|
|
213
|
+
|
|
174
214
|
return False
|
|
175
215
|
return True
|
|
176
216
|
|
|
@@ -16,6 +16,9 @@ if TYPE_CHECKING:
|
|
|
16
16
|
def set_managed_storage(root: UPathStr, host: str | None = None, **fs_kwargs):
|
|
17
17
|
"""Add or switch to another managed storage location.
|
|
18
18
|
|
|
19
|
+
Note: This function should be called `set_writeable_storage_location` instead. But likely it will disappear
|
|
20
|
+
in refactoring that consolidates with the `ln.Storage()` path.
|
|
21
|
+
|
|
19
22
|
Args:
|
|
20
23
|
root: `UPathStr` - The new storage root, e.g., an S3 bucket.
|
|
21
24
|
host: `str | None = None` For a shared local storage location, pass a globally unique host identifier, e.g. `"my-institute-cluster-1"`, `"my-server-abcd"`, ...
|
lamindb_setup/core/__init__.py
CHANGED
|
@@ -20,7 +20,7 @@ lamin_env = os.getenv("LAMIN_ENV")
|
|
|
20
20
|
if lamin_env is None or lamin_env == "prod":
|
|
21
21
|
HOSTED_BUCKETS = tuple([f"s3://lamin-{region}" for region in HOSTED_REGIONS])
|
|
22
22
|
else:
|
|
23
|
-
logger.warning("loaded LAMIN_ENV:
|
|
23
|
+
logger.warning(f"loaded LAMIN_ENV: {lamin_env}")
|
|
24
24
|
HOSTED_BUCKETS = ("s3://lamin-hosted-test",) # type: ignore
|
|
25
25
|
|
|
26
26
|
|
lamindb_setup/core/_clone.py
CHANGED
|
@@ -1,142 +1,18 @@
|
|
|
1
|
-
"""Utilities to
|
|
1
|
+
"""Utilities to work with Postgres Snapshots.
|
|
2
2
|
|
|
3
3
|
.. autosummary::
|
|
4
4
|
:toctree:
|
|
5
5
|
|
|
6
|
-
init_local_sqlite
|
|
7
|
-
connect_local_sqlite
|
|
8
|
-
connect_remote_sqlite
|
|
9
6
|
upload_sqlite_clone
|
|
10
7
|
"""
|
|
11
8
|
|
|
12
9
|
import gzip
|
|
13
|
-
import os
|
|
14
10
|
import shutil
|
|
15
11
|
from pathlib import Path
|
|
16
12
|
|
|
17
|
-
from lamindb_setup.core._settings_load import load_instance_settings
|
|
18
|
-
from lamindb_setup.core._settings_store import instance_settings_file
|
|
19
|
-
from lamindb_setup.core.django import reset_django
|
|
20
13
|
from lamindb_setup.core.upath import create_path
|
|
21
14
|
|
|
22
15
|
|
|
23
|
-
def init_local_sqlite(
|
|
24
|
-
instance: str | None = None, copy_suffix: str | None = None
|
|
25
|
-
) -> None:
|
|
26
|
-
"""Initialize SQLite copy of an existing Postgres instance.
|
|
27
|
-
|
|
28
|
-
Creates a SQLite database with the same schema as the source Postgres instance.
|
|
29
|
-
The copy shares the same storage location as the original instance.
|
|
30
|
-
|
|
31
|
-
The copy is intended for read-only access to instance data without requiring a Postgres connection.
|
|
32
|
-
Data synchronization to complete the clone happens via a separate Lambda function.
|
|
33
|
-
|
|
34
|
-
Note that essential user, branch and storage tables are missing.
|
|
35
|
-
Therefore, it is not possible to store Artifacts without having replayed these records first.
|
|
36
|
-
|
|
37
|
-
Args:
|
|
38
|
-
instance: Pass a slug (`account/name`) or URL (`https://lamin.ai/account/name`).
|
|
39
|
-
If `None`, looks for an environment variable `LAMIN_CURRENT_INSTANCE` to get the instance identifier.
|
|
40
|
-
If it doesn't find this variable, it connects to the instance that was connected with `lamin connect` through the CLI.
|
|
41
|
-
copy_suffix: Optional suffix to append to the local clone name.
|
|
42
|
-
"""
|
|
43
|
-
import lamindb_setup as ln_setup
|
|
44
|
-
|
|
45
|
-
if instance is None: # pragma: no cover
|
|
46
|
-
instance = os.environ.get("LAMIN_CURRENT_INSTANCE")
|
|
47
|
-
|
|
48
|
-
if instance is None:
|
|
49
|
-
raise ValueError(
|
|
50
|
-
"No instance identifier provided and LAMIN_CURRENT_INSTANCE is not set"
|
|
51
|
-
)
|
|
52
|
-
|
|
53
|
-
if ln_setup.settings.instance is None: # pragma: no cover
|
|
54
|
-
ln_setup.connect(instance)
|
|
55
|
-
|
|
56
|
-
name = (
|
|
57
|
-
f"{ln_setup.settings.instance.name}{copy_suffix}"
|
|
58
|
-
if copy_suffix is not None
|
|
59
|
-
else ln_setup.settings.instance.name
|
|
60
|
-
)
|
|
61
|
-
isettings = ln_setup._connect_instance._connect_instance(
|
|
62
|
-
owner=ln_setup.settings.instance.owner, name=name
|
|
63
|
-
)
|
|
64
|
-
isettings._db = None
|
|
65
|
-
isettings._is_on_hub = False
|
|
66
|
-
isettings._fine_grained_access = False
|
|
67
|
-
name = (
|
|
68
|
-
f"{isettings.name}{copy_suffix}" if copy_suffix is not None else isettings.name
|
|
69
|
-
)
|
|
70
|
-
isettings._name = name
|
|
71
|
-
isettings._is_clone = True
|
|
72
|
-
isettings._persist(write_to_disk=True)
|
|
73
|
-
|
|
74
|
-
if not isettings._sqlite_file_local.exists():
|
|
75
|
-
# Reset Django configuration before _init_db() because Django was already configured for the original Postgres instance.
|
|
76
|
-
# Without this reset, the `if not settings.configured`` check in `setup_django()` would skip reconfiguration,
|
|
77
|
-
# causing migrations to run against the old Postgres database instead of the new SQLite clone database.
|
|
78
|
-
reset_django()
|
|
79
|
-
isettings._init_db()
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
def connect_local_sqlite(
|
|
83
|
-
instance: str,
|
|
84
|
-
) -> None:
|
|
85
|
-
"""Load a locally stored SQLite instance of which a remote hub Postgres instance exists.
|
|
86
|
-
|
|
87
|
-
This function bypasses the hub lookup that `lamin connect` performs, loading the SQLite clone directly from local settings files.
|
|
88
|
-
The clone must first be created via `init_local_sqlite()`.
|
|
89
|
-
|
|
90
|
-
Args:
|
|
91
|
-
instance: Instance slug in the form `account/name` (e.g., `laminlabs/privatedata-local`).
|
|
92
|
-
"""
|
|
93
|
-
owner, name = instance.split("/")
|
|
94
|
-
settings_file = instance_settings_file(name=name, owner=owner)
|
|
95
|
-
|
|
96
|
-
if not settings_file.exists():
|
|
97
|
-
raise ValueError(
|
|
98
|
-
"SQLite clone not found."
|
|
99
|
-
" Run `init_local_sqlite()` to create a local copy or connect to a remote copy using `connect_remote_sqlite`."
|
|
100
|
-
)
|
|
101
|
-
|
|
102
|
-
isettings = load_instance_settings(settings_file)
|
|
103
|
-
isettings._persist(write_to_disk=False)
|
|
104
|
-
|
|
105
|
-
# Using `setup_django` instead of `_load_db` to not ping AWS RDS
|
|
106
|
-
from lamindb_setup._check_setup import disable_auto_connect
|
|
107
|
-
|
|
108
|
-
from .django import setup_django
|
|
109
|
-
|
|
110
|
-
disable_auto_connect(setup_django)(isettings)
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
def connect_remote_sqlite(instance: str, *, copy_suffix: str | None = None) -> None:
|
|
114
|
-
"""Load an existing SQLite copy of a hub instance.
|
|
115
|
-
|
|
116
|
-
Args:
|
|
117
|
-
instance: Instance slug in the form `account/name` (e.g., `laminlabs/privatedata-local`).
|
|
118
|
-
copy_suffix: Optional suffix of the local clone.
|
|
119
|
-
"""
|
|
120
|
-
import lamindb_setup as ln_setup
|
|
121
|
-
|
|
122
|
-
owner, name = instance.split("/")
|
|
123
|
-
|
|
124
|
-
# Step 1: Create the settings file
|
|
125
|
-
isettings = ln_setup._connect_instance._connect_instance(owner=owner, name=name)
|
|
126
|
-
isettings._db = None
|
|
127
|
-
isettings._is_on_hub = False
|
|
128
|
-
isettings._fine_grained_access = False
|
|
129
|
-
isettings._db_permissions = "read"
|
|
130
|
-
name = (
|
|
131
|
-
f"{isettings.name}{copy_suffix}" if copy_suffix is not None else isettings.name
|
|
132
|
-
)
|
|
133
|
-
isettings._name = name
|
|
134
|
-
isettings._is_clone = True
|
|
135
|
-
isettings._persist(write_to_disk=True)
|
|
136
|
-
|
|
137
|
-
connect_local_sqlite(instance=instance + (copy_suffix or ""))
|
|
138
|
-
|
|
139
|
-
|
|
140
16
|
def upload_sqlite_clone(
|
|
141
17
|
local_sqlite_path: Path | str | None = None, compress: bool = True
|
|
142
18
|
) -> None:
|
|
@@ -34,8 +34,8 @@ def private_django_api(reverse=False):
|
|
|
34
34
|
# the order here matters
|
|
35
35
|
# changing it might break the tests
|
|
36
36
|
attributes = [
|
|
37
|
-
"MultipleObjectsReturned",
|
|
38
37
|
"add_to_class",
|
|
38
|
+
"arefresh_from_db",
|
|
39
39
|
"adelete",
|
|
40
40
|
"asave",
|
|
41
41
|
"clean",
|
|
@@ -51,10 +51,8 @@ def private_django_api(reverse=False):
|
|
|
51
51
|
"validate_unique",
|
|
52
52
|
]
|
|
53
53
|
if reverse:
|
|
54
|
-
attributes.append("arefresh_from_db")
|
|
55
54
|
attributes.append("full_clean")
|
|
56
55
|
else:
|
|
57
|
-
attributes.append("a_refresh_from_db")
|
|
58
56
|
attributes.append("full__clean")
|
|
59
57
|
|
|
60
58
|
django_path = Path(db.__file__).parent.parent
|
|
@@ -323,16 +323,6 @@ class InstanceSettings:
|
|
|
323
323
|
else:
|
|
324
324
|
logger.warning(f"could not set this local storage location: {local_root}")
|
|
325
325
|
|
|
326
|
-
@property
|
|
327
|
-
@deprecated("local_storage")
|
|
328
|
-
def storage_local(self) -> StorageSettings:
|
|
329
|
-
return self.local_storage
|
|
330
|
-
|
|
331
|
-
@storage_local.setter
|
|
332
|
-
@deprecated("local_storage")
|
|
333
|
-
def storage_local(self, local_root_host: tuple[Path | str, str]):
|
|
334
|
-
self.local_storage = local_root_host # type: ignore
|
|
335
|
-
|
|
336
326
|
@property
|
|
337
327
|
def slug(self) -> str:
|
|
338
328
|
"""Unique semantic identifier of form `"{account_handle}/{instance_name}"`."""
|
|
@@ -418,11 +408,6 @@ class InstanceSettings:
|
|
|
418
408
|
else:
|
|
419
409
|
return {module for module in self._schema_str.split(",") if module != ""}
|
|
420
410
|
|
|
421
|
-
@property
|
|
422
|
-
@deprecated("modules")
|
|
423
|
-
def schema(self) -> set[str]:
|
|
424
|
-
return self.modules
|
|
425
|
-
|
|
426
411
|
@property
|
|
427
412
|
def _sqlite_file(self) -> UPath:
|
|
428
413
|
"""SQLite file."""
|
|
@@ -569,10 +554,10 @@ class InstanceSettings:
|
|
|
569
554
|
|
|
570
555
|
@property
|
|
571
556
|
def is_on_hub(self) -> bool:
|
|
572
|
-
"""Is this instance on the hub?
|
|
557
|
+
"""Is this instance registered on the hub?
|
|
573
558
|
|
|
574
|
-
Can only
|
|
575
|
-
Will return `False` in case the
|
|
559
|
+
Can only establish if user has access to the instance.
|
|
560
|
+
Will return `False` in case the user token can't find the instance.
|
|
576
561
|
"""
|
|
577
562
|
if self._is_on_hub is None:
|
|
578
563
|
from ._hub_client import call_with_fallback_auth
|
|
@@ -594,6 +579,15 @@ class InstanceSettings:
|
|
|
594
579
|
self._is_on_hub = True
|
|
595
580
|
return self._is_on_hub
|
|
596
581
|
|
|
582
|
+
@property
|
|
583
|
+
def is_managed_by_hub(self) -> bool:
|
|
584
|
+
"""Is this instance managed by the hub?
|
|
585
|
+
|
|
586
|
+
Returns `True` if the instance is _managed_ by LaminHub, i.e.,
|
|
587
|
+
it was connected to LaminHub to manage access, migrations, a REST API, a UI, etc.
|
|
588
|
+
"""
|
|
589
|
+
return self.api_url is not None
|
|
590
|
+
|
|
597
591
|
def _get_settings_file(self) -> Path:
|
|
598
592
|
return instance_settings_file(self.name, self.owner)
|
|
599
593
|
|
lamindb_setup/core/django.py
CHANGED
|
@@ -329,20 +329,20 @@ def setup_django(
|
|
|
329
329
|
compressed_sqlite_path = Path(__file__).parent / "lamin.db.gz"
|
|
330
330
|
if "bionty" in modules_beyond_bionty:
|
|
331
331
|
modules_beyond_bionty.remove("bionty")
|
|
332
|
+
# seed from compressed sqlite file
|
|
332
333
|
if (
|
|
333
|
-
isettings.dialect == "
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
334
|
+
isettings.dialect == "sqlite"
|
|
335
|
+
and os.getenv("LAMINDB_INIT_FROM_SCRATCH", "false") != "true"
|
|
336
|
+
and len(modules_beyond_bionty) == 0
|
|
337
|
+
and compressed_sqlite_path.exists()
|
|
337
338
|
):
|
|
338
|
-
global IS_MIGRATING
|
|
339
|
-
IS_MIGRATING = True
|
|
340
|
-
call_command("migrate", verbosity=0)
|
|
341
|
-
IS_MIGRATING = False
|
|
342
|
-
else:
|
|
343
339
|
with gzip.open(compressed_sqlite_path, "rb") as f_in:
|
|
344
340
|
with open(isettings._sqlite_file_local, "wb") as f_out:
|
|
345
341
|
shutil.copyfileobj(f_in, f_out)
|
|
342
|
+
global IS_MIGRATING
|
|
343
|
+
IS_MIGRATING = True
|
|
344
|
+
call_command("migrate", verbosity=0)
|
|
345
|
+
IS_MIGRATING = False
|
|
346
346
|
|
|
347
347
|
global IS_SETUP
|
|
348
348
|
IS_SETUP = True
|
lamindb_setup/core/hashing.py
CHANGED
|
@@ -88,7 +88,7 @@ def hash_file(
|
|
|
88
88
|
file_path: Path,
|
|
89
89
|
file_size: int | None = None,
|
|
90
90
|
chunk_size: int | None = 50 * 1024 * 1024,
|
|
91
|
-
) -> tuple[str, str]:
|
|
91
|
+
) -> tuple[int, str, str]:
|
|
92
92
|
with open(file_path, "rb") as fp:
|
|
93
93
|
if file_size is None:
|
|
94
94
|
fp.seek(0, 2)
|
|
@@ -107,15 +107,15 @@ def hash_file(
|
|
|
107
107
|
hashlib.sha1(first_chunk).digest() + hashlib.sha1(last_chunk).digest()
|
|
108
108
|
).digest()
|
|
109
109
|
hash_type = "sha1-fl"
|
|
110
|
-
return to_b64_str(digest)[:HASH_LENGTH], hash_type
|
|
110
|
+
return file_size, to_b64_str(digest)[:HASH_LENGTH], hash_type
|
|
111
111
|
|
|
112
112
|
|
|
113
|
-
def hash_dir(path: Path):
|
|
113
|
+
def hash_dir(path: Path) -> tuple[int, str, str, int]:
|
|
114
114
|
files = (subpath for subpath in path.rglob("*") if subpath.is_file())
|
|
115
115
|
|
|
116
116
|
def hash_size(file):
|
|
117
|
-
|
|
118
|
-
return
|
|
117
|
+
size, hash, _ = hash_file(file)
|
|
118
|
+
return hash, size
|
|
119
119
|
|
|
120
120
|
try:
|
|
121
121
|
n_workers = len(psutil.Process().cpu_affinity())
|
lamindb_setup/core/lamin.db.gz
CHANGED
|
Binary file
|
lamindb_setup/errors.py
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
.. autoexception:: InstanceLockedException
|
|
8
8
|
.. autoexception:: SettingsEnvFileOutdated
|
|
9
9
|
.. autoexception:: CannotSwitchDefaultInstance
|
|
10
|
+
.. autoexception:: InstanceNotFoundError
|
|
10
11
|
|
|
11
12
|
"""
|
|
12
13
|
|
|
@@ -51,6 +52,10 @@ class StorageNotEmpty(click.ClickException):
|
|
|
51
52
|
pass
|
|
52
53
|
|
|
53
54
|
|
|
55
|
+
class InstanceNotFoundError(Exception):
|
|
56
|
+
pass
|
|
57
|
+
|
|
58
|
+
|
|
54
59
|
# raise if a cloud SQLite instance is already locked
|
|
55
60
|
# ignored by unlock_cloud_sqlite_upon_exception
|
|
56
61
|
class InstanceLockedException(Exception):
|
|
@@ -1,11 +1,10 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
2
|
Name: lamindb_setup
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.18.1
|
|
4
4
|
Summary: Setup & configure LaminDB.
|
|
5
5
|
Author-email: Lamin Labs <open-source@lamin.ai>
|
|
6
6
|
Requires-Python: >=3.10
|
|
7
7
|
Description-Content-Type: text/markdown
|
|
8
|
-
License-File: LICENSE
|
|
9
8
|
Requires-Dist: lamin_utils>=0.3.3
|
|
10
9
|
Requires-Dist: django>=5.2,<5.3
|
|
11
10
|
Requires-Dist: dj_database_url>=1.3.0,<3.0.0
|
|
@@ -17,11 +16,12 @@ Requires-Dist: requests
|
|
|
17
16
|
Requires-Dist: universal_pathlib==0.2.6
|
|
18
17
|
Requires-Dist: botocore<2.0.0
|
|
19
18
|
Requires-Dist: supabase>=2.20.0,<=2.24.0
|
|
19
|
+
Requires-Dist: websockets>=13.0
|
|
20
20
|
Requires-Dist: pyjwt<3.0.0
|
|
21
21
|
Requires-Dist: psutil
|
|
22
22
|
Requires-Dist: packaging
|
|
23
23
|
Requires-Dist: aiobotocore[boto3]>=2.12.4,<3.0.0 ; extra == "aws"
|
|
24
|
-
Requires-Dist: s3fs>=2023.12.2,<=2025.
|
|
24
|
+
Requires-Dist: s3fs>=2023.12.2,<=2025.12.0,!=2024.10.0 ; extra == "aws"
|
|
25
25
|
Requires-Dist: line_profiler ; extra == "dev"
|
|
26
26
|
Requires-Dist: psycopg2-binary ; extra == "dev"
|
|
27
27
|
Requires-Dist: python-dotenv ; extra == "dev"
|
|
@@ -32,7 +32,7 @@ Requires-Dist: pytest-xdist ; extra == "dev"
|
|
|
32
32
|
Requires-Dist: nbproject-test>=0.4.3 ; extra == "dev"
|
|
33
33
|
Requires-Dist: pandas ; extra == "dev"
|
|
34
34
|
Requires-Dist: django-schema-graph ; extra == "erdiagram"
|
|
35
|
-
Requires-Dist: gcsfs>=2023.12.2,<=2025.
|
|
35
|
+
Requires-Dist: gcsfs>=2023.12.2,<=2025.12.0 ; extra == "gcp"
|
|
36
36
|
Project-URL: Home, https://github.com/laminlabs/lamindb-setup
|
|
37
37
|
Provides-Extra: aws
|
|
38
38
|
Provides-Extra: dev
|
|
@@ -1,51 +1,51 @@
|
|
|
1
|
-
lamindb_setup/__init__.py,sha256=
|
|
1
|
+
lamindb_setup/__init__.py,sha256=qNJ5BlP4kXqvMH3dKRQkWpaxb0mhyK4vS2xSa2Y3XLo,3270
|
|
2
2
|
lamindb_setup/_cache.py,sha256=pGvDNVHGx4HWr_6w5ajqEJOdysmaGc6F221qFnXkT-k,2747
|
|
3
3
|
lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
|
|
4
4
|
lamindb_setup/_check_setup.py,sha256=zPQho12dctJYjnZxpumq2r7XRvXwrNyWi5-UMRPSeuE,4297
|
|
5
|
-
lamindb_setup/_connect_instance.py,sha256=
|
|
5
|
+
lamindb_setup/_connect_instance.py,sha256=2sPW9KF7wJvGlnUdi_eshzahfT319RwE3mhkZs1H2Fs,17339
|
|
6
6
|
lamindb_setup/_delete.py,sha256=ZMAik35NNcj4kDfOISDfzpicCi4j8gwmWAor0hEB-Jk,6123
|
|
7
7
|
lamindb_setup/_disconnect.py,sha256=B0K0yTIIyhwc3MzDFL_-cKutm-_DHsiJSTrkp1MMyXA,1470
|
|
8
8
|
lamindb_setup/_django.py,sha256=uIQflpkp8l3axyPaKURlk3kacgpElVP5KOKmFxYSMGk,1454
|
|
9
9
|
lamindb_setup/_entry_points.py,sha256=sKwXPX9xjOotoAjvgkU5LBwjjHLWVkh0ZGdiSsrch9k,522
|
|
10
10
|
lamindb_setup/_init_instance.py,sha256=EBdv0ga_nnqHU-m9fN7EkVg7YEuKOLxR8N87Ww2qEDg,14877
|
|
11
|
-
lamindb_setup/_migrate.py,sha256=
|
|
11
|
+
lamindb_setup/_migrate.py,sha256=mqJxm1EnB_1wJOhw_3djESyoc53bXJOXgBPndSRd5OU,12077
|
|
12
12
|
lamindb_setup/_register_instance.py,sha256=RdUZxZWHLdbvdNZWpF8e0UWROb_T0cStWbzc5yUw34I,1047
|
|
13
13
|
lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
|
|
14
14
|
lamindb_setup/_schema_metadata.py,sha256=Whs-e4ZMnA1niZ2l5Eu8il-33IxI4Hr5ylGEgPxx8wk,15628
|
|
15
|
-
lamindb_setup/_set_managed_storage.py,sha256=
|
|
15
|
+
lamindb_setup/_set_managed_storage.py,sha256=xik6Amz8yS4R7ZIK8BVdNyvOo_ShBMjsDMAadfIwQZM,3274
|
|
16
16
|
lamindb_setup/_setup_user.py,sha256=9CwKUru2jza1vXu4A5V-q-oXnbIXwR_YwjKbVnETK7Q,6931
|
|
17
17
|
lamindb_setup/_silence_loggers.py,sha256=oQnvnvieqxARQQMwQkR82EJVk1TaM-Bo9o0UCOsCcXY,1697
|
|
18
|
-
lamindb_setup/errors.py,sha256=
|
|
18
|
+
lamindb_setup/errors.py,sha256=3u77okYz2NHXbX-TZkhHf5mJAXaaIIPBFPD21iSyRPg,1620
|
|
19
19
|
lamindb_setup/io.py,sha256=stTE6VHCIpLK1ueRuWEJGh35wo9Qd1NcNN1TnLEbK-U,17156
|
|
20
20
|
lamindb_setup/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
21
21
|
lamindb_setup/types.py,sha256=fuQxZJnrGYe7a_Ju9n1RqO-HhkOAr1l1xjpAg9dmBu8,605
|
|
22
|
-
lamindb_setup/core/__init__.py,sha256=
|
|
23
|
-
lamindb_setup/core/_aws_options.py,sha256=
|
|
22
|
+
lamindb_setup/core/__init__.py,sha256=oSTXxNCx6yfoqaHYTZM9qHuZKdMUIUF2DdXw4gBMBU4,592
|
|
23
|
+
lamindb_setup/core/_aws_options.py,sha256=fHc7ndBl7URMYeVliuw1GAoodOuxvYMadTna8m0PORo,10059
|
|
24
24
|
lamindb_setup/core/_aws_storage.py,sha256=QEtV-riQrwfivcwqHnXBbkJ-9YyNEXL4fLoCmOHZ1BI,2003
|
|
25
|
-
lamindb_setup/core/_clone.py,sha256=
|
|
25
|
+
lamindb_setup/core/_clone.py,sha256=IE7d9n4qc8dvUgYbFavIkdUZ_BhsEHfoQzmcPDFuF4A,1537
|
|
26
26
|
lamindb_setup/core/_deprecated.py,sha256=M3vpM4fZPOncxY2qsXQAPeaEph28xWdv7tYaueaUyAA,2554
|
|
27
27
|
lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
|
|
28
28
|
lamindb_setup/core/_hub_client.py,sha256=113SKT3Ki4KxyKqpYOwIB8mEG0ouy9oebLJoh_xGJZY,10426
|
|
29
29
|
lamindb_setup/core/_hub_core.py,sha256=29iPGnmV1hzYIMrErgwJoHwTQ5RHNJ6-icDM81aC4ac,29255
|
|
30
30
|
lamindb_setup/core/_hub_crud.py,sha256=j6516H82kLjFUNPqFGUINbDw9YbofMgjxadGzYb0OS4,6362
|
|
31
31
|
lamindb_setup/core/_hub_utils.py,sha256=6dyDGyzYFgVfR_lE3VN3CP1jGp98gxPtr-T91PAP05U,2687
|
|
32
|
-
lamindb_setup/core/_private_django_api.py,sha256=
|
|
32
|
+
lamindb_setup/core/_private_django_api.py,sha256=e-9WiGkI7g683nOdTcScS_fPG-_neNvrcWuAI9NEeQs,2529
|
|
33
33
|
lamindb_setup/core/_settings.py,sha256=LTeh2jsM01GHW2ZTrcc6rXNz6qIUT0VPEzpE9I8tg3Q,15497
|
|
34
|
-
lamindb_setup/core/_settings_instance.py,sha256=
|
|
34
|
+
lamindb_setup/core/_settings_instance.py,sha256=KoKwvSqOGEieGC-I5Dd0kIr_Q9OknRlno-nj3ZaX8R8,24903
|
|
35
35
|
lamindb_setup/core/_settings_load.py,sha256=D6r0fiqJznHhTN_apGlrKfYzl0g21shIaV4z4zEr7Z8,5780
|
|
36
36
|
lamindb_setup/core/_settings_save.py,sha256=96mWdYLyfvbnG_ok_vK4x7jm-rtqcWCD1OHEt2QSAms,3328
|
|
37
|
-
lamindb_setup/core/_settings_storage.py,sha256=
|
|
37
|
+
lamindb_setup/core/_settings_storage.py,sha256=e1FQ2uh13Cuw9Z5k6d71C825f6ISUnc1LtkDUO0QUIA,15590
|
|
38
38
|
lamindb_setup/core/_settings_store.py,sha256=auZssUBb6qE5oSqdGiHhqI2B46qSpegX89VwObPQksk,2601
|
|
39
39
|
lamindb_setup/core/_settings_user.py,sha256=gFfyMf-738onbh1Mf4wsmLlenQJPtjQfpUgKnOlqc2o,1453
|
|
40
40
|
lamindb_setup/core/_setup_bionty_sources.py,sha256=ox3X-SHiHa2lNPSWjwZhINypbLacX6kGwH6hVVrSFZc,1505
|
|
41
41
|
lamindb_setup/core/cloud_sqlite_locker.py,sha256=H_CTUCjURFXwD1cCtV_Jn0_60iztZTkaesLLXIBgIxc,7204
|
|
42
|
-
lamindb_setup/core/django.py,sha256=
|
|
42
|
+
lamindb_setup/core/django.py,sha256=X_KDP6c9EgqG0JoHg-igv-rvpdZ9mF1LeeoTtZjVICc,13649
|
|
43
43
|
lamindb_setup/core/exceptions.py,sha256=qjMzqy_uzPA7mCOdnoWnS_fdA6OWbdZGftz-YYplrY0,84
|
|
44
|
-
lamindb_setup/core/hashing.py,sha256=
|
|
45
|
-
lamindb_setup/core/lamin.db.gz,sha256=
|
|
44
|
+
lamindb_setup/core/hashing.py,sha256=4EDiMkdn96ct3oVkB64_4ujj6_EY5Vt8WMOMT5qnewI,3695
|
|
45
|
+
lamindb_setup/core/lamin.db.gz,sha256=sFh_Qgg4Krui0lW4g2CrGfKpL59-UjOMgAJ7n0xP4XE,78228
|
|
46
46
|
lamindb_setup/core/types.py,sha256=T7NwspfRHgIIpYsXDcApks8jkOlGeGRW-YbVLB7jNIo,67
|
|
47
47
|
lamindb_setup/core/upath.py,sha256=zlPZcDRHDYT7OcoOiX_w9koAzfBr9q8hB66OPwhTpBo,36504
|
|
48
|
-
lamindb_setup-1.
|
|
49
|
-
lamindb_setup-1.
|
|
50
|
-
lamindb_setup-1.
|
|
51
|
-
lamindb_setup-1.
|
|
48
|
+
lamindb_setup-1.18.1.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
|
|
49
|
+
lamindb_setup-1.18.1.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
|
|
50
|
+
lamindb_setup-1.18.1.dist-info/METADATA,sha256=vP_uPB1gKCchtRUEGIPU-9m-rBcwr3j82VvGw6kvNOA,1778
|
|
51
|
+
lamindb_setup-1.18.1.dist-info/RECORD,,
|
|
File without changes
|