lamindb_setup 1.9.0__py3-none-any.whl → 1.9.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. lamindb_setup/__init__.py +107 -107
  2. lamindb_setup/_cache.py +87 -87
  3. lamindb_setup/_check_setup.py +166 -166
  4. lamindb_setup/_connect_instance.py +328 -342
  5. lamindb_setup/_delete.py +141 -141
  6. lamindb_setup/_disconnect.py +32 -32
  7. lamindb_setup/_init_instance.py +440 -440
  8. lamindb_setup/_migrate.py +266 -266
  9. lamindb_setup/_register_instance.py +35 -35
  10. lamindb_setup/_schema_metadata.py +441 -441
  11. lamindb_setup/_set_managed_storage.py +70 -70
  12. lamindb_setup/_setup_user.py +133 -133
  13. lamindb_setup/core/__init__.py +21 -21
  14. lamindb_setup/core/_aws_options.py +223 -223
  15. lamindb_setup/core/_hub_client.py +248 -248
  16. lamindb_setup/core/_hub_core.py +665 -665
  17. lamindb_setup/core/_hub_crud.py +227 -227
  18. lamindb_setup/core/_private_django_api.py +83 -83
  19. lamindb_setup/core/_settings.py +377 -377
  20. lamindb_setup/core/_settings_instance.py +569 -569
  21. lamindb_setup/core/_settings_load.py +141 -141
  22. lamindb_setup/core/_settings_save.py +95 -95
  23. lamindb_setup/core/_settings_storage.py +429 -429
  24. lamindb_setup/core/_settings_store.py +91 -91
  25. lamindb_setup/core/_settings_user.py +55 -55
  26. lamindb_setup/core/_setup_bionty_sources.py +44 -44
  27. lamindb_setup/core/cloud_sqlite_locker.py +240 -240
  28. lamindb_setup/core/django.py +305 -296
  29. lamindb_setup/core/exceptions.py +1 -1
  30. lamindb_setup/core/hashing.py +134 -134
  31. lamindb_setup/core/types.py +1 -1
  32. lamindb_setup/core/upath.py +1013 -1013
  33. lamindb_setup/errors.py +70 -70
  34. lamindb_setup/types.py +20 -20
  35. {lamindb_setup-1.9.0.dist-info → lamindb_setup-1.9.1.dist-info}/METADATA +1 -1
  36. lamindb_setup-1.9.1.dist-info/RECORD +50 -0
  37. lamindb_setup-1.9.0.dist-info/RECORD +0 -50
  38. {lamindb_setup-1.9.0.dist-info → lamindb_setup-1.9.1.dist-info}/LICENSE +0 -0
  39. {lamindb_setup-1.9.0.dist-info → lamindb_setup-1.9.1.dist-info}/WHEEL +0 -0
lamindb_setup/_migrate.py CHANGED
@@ -1,266 +1,266 @@
1
- from __future__ import annotations
2
-
3
- from django.db import connection
4
- from django.db.migrations.loader import MigrationLoader
5
- from lamin_utils import logger
6
- from packaging import version
7
-
8
- from ._check_setup import _check_instance_setup, disable_auto_connect
9
- from .core._settings import settings
10
- from .core.django import setup_django
11
-
12
-
13
- # for the django-based synching code, see laminhub_rest
14
- def check_whether_migrations_in_sync(db_version_str: str):
15
- from importlib import metadata
16
-
17
- try:
18
- installed_version_str = metadata.version("lamindb")
19
- except metadata.PackageNotFoundError:
20
- return None
21
- if db_version_str is None:
22
- logger.warning("no lamindb version stored to compare with installed version")
23
- return None
24
- installed_version = version.parse(installed_version_str)
25
- db_version = version.parse(db_version_str)
26
- if installed_version.major < db_version.major:
27
- logger.warning(
28
- f"the database ({db_version_str}) is far ahead of your installed lamindb package ({installed_version_str})"
29
- )
30
- logger.important(
31
- f"please update lamindb: pip install lamindb>={db_version.major}"
32
- )
33
- elif (
34
- installed_version.major == db_version.major
35
- and installed_version.minor < db_version.minor
36
- ):
37
- db_version_lower = f"{db_version.major}.{db_version.minor}"
38
- logger.important(
39
- f"the database ({db_version_str}) is ahead of your installed lamindb"
40
- f" package ({installed_version_str})"
41
- )
42
- logger.important(
43
- f"consider updating lamindb: pip install lamindb>={db_version_lower}"
44
- )
45
- elif installed_version.major > db_version.major:
46
- logger.warning(
47
- f"the database ({db_version_str}) is far behind your installed lamindb package"
48
- f" ({installed_version_str})"
49
- )
50
- logger.important(
51
- "if you are an admin, migrate your database: lamin migrate deploy"
52
- )
53
- elif (
54
- installed_version.major == db_version.major
55
- and installed_version.minor > db_version.minor
56
- ):
57
- pass
58
- # if the database is behind by a minor version, we don't want to spam the user
59
- # logger.important(
60
- # f"the database ({db_version_str}) is behind your installed lamindb package"
61
- # f" ({installed_version_str})"
62
- # )
63
- # logger.important("consider migrating your database: lamin migrate deploy")
64
-
65
-
66
- # for tests, see lamin-cli
67
- class migrate:
68
- """Manage migrations.
69
-
70
- Examples:
71
-
72
- >>> import lamindb as ln
73
- >>> ln.setup.migrate.create()
74
- >>> ln.setup.migrate.deploy()
75
- >>> ln.setup.migrate.check()
76
-
77
- """
78
-
79
- @classmethod
80
- @disable_auto_connect
81
- def create(cls) -> None:
82
- """Create a migration."""
83
- if _check_instance_setup():
84
- raise RuntimeError("Restart Python session to create migration or use CLI!")
85
- setup_django(settings.instance, create_migrations=True)
86
-
87
- @classmethod
88
- @disable_auto_connect
89
- def deploy(cls, package_name: str | None = None, number: int | None = None) -> None:
90
- """Deploy a migration."""
91
- from ._schema_metadata import update_schema_in_hub
92
-
93
- if _check_instance_setup():
94
- raise RuntimeError("Restart Python session to migrate or use CLI!")
95
- from lamindb_setup.core._hub_client import call_with_fallback_auth
96
- from lamindb_setup.core._hub_crud import (
97
- select_collaborator,
98
- update_instance,
99
- )
100
-
101
- if settings.instance.is_on_hub:
102
- # double check that user is an admin, otherwise will fail below
103
- # due to insufficient SQL permissions with cryptic error
104
- collaborator = call_with_fallback_auth(
105
- select_collaborator,
106
- instance_id=settings.instance._id,
107
- account_id=settings.user._uuid,
108
- )
109
- if collaborator is None or collaborator["role"] != "admin":
110
- raise SystemExit(
111
- "❌ Only admins can deploy migrations, please ensure that you're an"
112
- f" admin: https://lamin.ai/{settings.instance.slug}/settings"
113
- )
114
- # we need lamindb to be installed, otherwise we can't populate the version
115
- # information in the hub
116
- import lamindb
117
-
118
- # this sets up django and deploys the migrations
119
- if package_name is not None and number is not None:
120
- setup_django(
121
- settings.instance,
122
- deploy_migrations=True,
123
- appname_number=(package_name, number),
124
- )
125
- else:
126
- setup_django(settings.instance, deploy_migrations=True)
127
- # this populates the hub
128
- if settings.instance.is_on_hub:
129
- logger.important(f"updating lamindb version in hub: {lamindb.__version__}")
130
- # TODO: integrate update of instance table within update_schema_in_hub & below
131
- if settings.instance.dialect != "sqlite":
132
- update_schema_in_hub()
133
- call_with_fallback_auth(
134
- update_instance,
135
- instance_id=settings.instance._id.hex,
136
- instance_fields={"lamindb_version": lamindb.__version__},
137
- )
138
-
139
- @classmethod
140
- @disable_auto_connect
141
- def check(cls) -> bool:
142
- """Check whether Registry definitions are in sync with migrations."""
143
- from django.core.management import call_command
144
-
145
- setup_django(settings.instance)
146
- try:
147
- call_command("makemigrations", check_changes=True)
148
- except SystemExit:
149
- logger.error(
150
- "migrations are not in sync with ORMs, please create a migration: lamin"
151
- " migrate create"
152
- )
153
- return False
154
- return True
155
-
156
- @classmethod
157
- @disable_auto_connect
158
- def squash(
159
- cls, package_name, migration_nr, start_migration_nr: str | None = None
160
- ) -> None:
161
- """Squash migrations."""
162
- from django.core.management import call_command
163
-
164
- setup_django(settings.instance)
165
- if start_migration_nr is not None:
166
- call_command(
167
- "squashmigrations", package_name, start_migration_nr, migration_nr
168
- )
169
- else:
170
- call_command("squashmigrations", package_name, migration_nr)
171
-
172
- @classmethod
173
- @disable_auto_connect
174
- def show(cls) -> None:
175
- """Show migrations."""
176
- from django.core.management import call_command
177
-
178
- setup_django(settings.instance)
179
- call_command("showmigrations")
180
-
181
- @classmethod
182
- def defined_migrations(cls, latest: bool = False):
183
- from io import StringIO
184
-
185
- from django.core.management import call_command
186
-
187
- def parse_migration_output(output):
188
- """Parse the output of the showmigrations command to get migration names."""
189
- lines = output.splitlines()
190
-
191
- # Initialize an empty dict to store migration names of each module
192
- migration_names = {}
193
-
194
- # Process each line
195
- for line in lines:
196
- if " " not in line:
197
- # CLI displays the module name in bold
198
- name = line.strip().replace("\x1b[1m", "")
199
- migration_names[name] = []
200
- continue
201
- # Strip whitespace and split the line into status and migration name
202
- migration_name = line.strip().split("] ")[-1].split(" ")[0]
203
- # The second part is the migration name
204
- migration_names[name].append(migration_name)
205
-
206
- return migration_names
207
-
208
- out = StringIO()
209
- call_command("showmigrations", stdout=out)
210
- out.seek(0)
211
- output = out.getvalue()
212
- if latest:
213
- return {k: v[-1] for k, v in parse_migration_output(output).items()}
214
- else:
215
- return parse_migration_output(output)
216
-
217
- @classmethod
218
- def deployed_migrations(cls, latest: bool = False):
219
- """Get the list of deployed migrations from Migration table in DB."""
220
- if latest:
221
- latest_migrations = {}
222
- with connection.cursor() as cursor:
223
- # query to get the latest migration for each app that is not squashed
224
- cursor.execute(
225
- """
226
- SELECT app, name
227
- FROM django_migrations
228
- WHERE id IN (
229
- SELECT MAX(id)
230
- FROM django_migrations
231
- WHERE name NOT LIKE '%%_squashed_%%'
232
- GROUP BY app
233
- )
234
- """
235
- )
236
- # fetch all the results
237
- for app, name in cursor.fetchall():
238
- latest_migrations[app] = name
239
-
240
- return latest_migrations
241
- else:
242
- # Load all migrations using Django's migration loader
243
- loader = MigrationLoader(connection)
244
- squashed_replacements = set()
245
- for _key, migration in loader.disk_migrations.items():
246
- if hasattr(migration, "replaces"):
247
- squashed_replacements.update(migration.replaces)
248
-
249
- deployed_migrations: dict = {}
250
- with connection.cursor() as cursor:
251
- cursor.execute(
252
- """
253
- SELECT app, name, deployed
254
- FROM django_migrations
255
- ORDER BY app, deployed DESC
256
- """
257
- )
258
- for app, name, _deployed in cursor.fetchall():
259
- # skip migrations that are part of a squashed migration
260
- if (app, name) in squashed_replacements:
261
- continue
262
-
263
- if app not in deployed_migrations:
264
- deployed_migrations[app] = []
265
- deployed_migrations[app].append(name)
266
- return deployed_migrations
1
+ from __future__ import annotations
2
+
3
+ from django.db import connection
4
+ from django.db.migrations.loader import MigrationLoader
5
+ from lamin_utils import logger
6
+ from packaging import version
7
+
8
+ from ._check_setup import _check_instance_setup, disable_auto_connect
9
+ from .core._settings import settings
10
+ from .core.django import setup_django
11
+
12
+
13
+ # for the django-based synching code, see laminhub_rest
14
+ def check_whether_migrations_in_sync(db_version_str: str):
15
+ from importlib import metadata
16
+
17
+ try:
18
+ installed_version_str = metadata.version("lamindb")
19
+ except metadata.PackageNotFoundError:
20
+ return None
21
+ if db_version_str is None:
22
+ logger.warning("no lamindb version stored to compare with installed version")
23
+ return None
24
+ installed_version = version.parse(installed_version_str)
25
+ db_version = version.parse(db_version_str)
26
+ if installed_version.major < db_version.major:
27
+ logger.warning(
28
+ f"the database ({db_version_str}) is far ahead of your installed lamindb package ({installed_version_str})"
29
+ )
30
+ logger.important(
31
+ f"please update lamindb: pip install lamindb>={db_version.major}"
32
+ )
33
+ elif (
34
+ installed_version.major == db_version.major
35
+ and installed_version.minor < db_version.minor
36
+ ):
37
+ db_version_lower = f"{db_version.major}.{db_version.minor}"
38
+ logger.important(
39
+ f"the database ({db_version_str}) is ahead of your installed lamindb"
40
+ f" package ({installed_version_str})"
41
+ )
42
+ logger.important(
43
+ f"consider updating lamindb: pip install lamindb>={db_version_lower}"
44
+ )
45
+ elif installed_version.major > db_version.major:
46
+ logger.warning(
47
+ f"the database ({db_version_str}) is far behind your installed lamindb package"
48
+ f" ({installed_version_str})"
49
+ )
50
+ logger.important(
51
+ "if you are an admin, migrate your database: lamin migrate deploy"
52
+ )
53
+ elif (
54
+ installed_version.major == db_version.major
55
+ and installed_version.minor > db_version.minor
56
+ ):
57
+ pass
58
+ # if the database is behind by a minor version, we don't want to spam the user
59
+ # logger.important(
60
+ # f"the database ({db_version_str}) is behind your installed lamindb package"
61
+ # f" ({installed_version_str})"
62
+ # )
63
+ # logger.important("consider migrating your database: lamin migrate deploy")
64
+
65
+
66
+ # for tests, see lamin-cli
67
+ class migrate:
68
+ """Manage migrations.
69
+
70
+ Examples:
71
+
72
+ >>> import lamindb as ln
73
+ >>> ln.setup.migrate.create()
74
+ >>> ln.setup.migrate.deploy()
75
+ >>> ln.setup.migrate.check()
76
+
77
+ """
78
+
79
+ @classmethod
80
+ @disable_auto_connect
81
+ def create(cls) -> None:
82
+ """Create a migration."""
83
+ if _check_instance_setup():
84
+ raise RuntimeError("Restart Python session to create migration or use CLI!")
85
+ setup_django(settings.instance, create_migrations=True)
86
+
87
+ @classmethod
88
+ @disable_auto_connect
89
+ def deploy(cls, package_name: str | None = None, number: int | None = None) -> None:
90
+ """Deploy a migration."""
91
+ from ._schema_metadata import update_schema_in_hub
92
+
93
+ if _check_instance_setup():
94
+ raise RuntimeError("Restart Python session to migrate or use CLI!")
95
+ from lamindb_setup.core._hub_client import call_with_fallback_auth
96
+ from lamindb_setup.core._hub_crud import (
97
+ select_collaborator,
98
+ update_instance,
99
+ )
100
+
101
+ if settings.instance.is_on_hub:
102
+ # double check that user is an admin, otherwise will fail below
103
+ # due to insufficient SQL permissions with cryptic error
104
+ collaborator = call_with_fallback_auth(
105
+ select_collaborator,
106
+ instance_id=settings.instance._id,
107
+ account_id=settings.user._uuid,
108
+ )
109
+ if collaborator is None or collaborator["role"] != "admin":
110
+ raise SystemExit(
111
+ "❌ Only admins can deploy migrations, please ensure that you're an"
112
+ f" admin: https://lamin.ai/{settings.instance.slug}/settings"
113
+ )
114
+ # we need lamindb to be installed, otherwise we can't populate the version
115
+ # information in the hub
116
+ import lamindb
117
+
118
+ # this sets up django and deploys the migrations
119
+ if package_name is not None and number is not None:
120
+ setup_django(
121
+ settings.instance,
122
+ deploy_migrations=True,
123
+ appname_number=(package_name, number),
124
+ )
125
+ else:
126
+ setup_django(settings.instance, deploy_migrations=True)
127
+ # this populates the hub
128
+ if settings.instance.is_on_hub:
129
+ logger.important(f"updating lamindb version in hub: {lamindb.__version__}")
130
+ # TODO: integrate update of instance table within update_schema_in_hub & below
131
+ if settings.instance.dialect != "sqlite":
132
+ update_schema_in_hub()
133
+ call_with_fallback_auth(
134
+ update_instance,
135
+ instance_id=settings.instance._id.hex,
136
+ instance_fields={"lamindb_version": lamindb.__version__},
137
+ )
138
+
139
+ @classmethod
140
+ @disable_auto_connect
141
+ def check(cls) -> bool:
142
+ """Check whether Registry definitions are in sync with migrations."""
143
+ from django.core.management import call_command
144
+
145
+ setup_django(settings.instance)
146
+ try:
147
+ call_command("makemigrations", check_changes=True)
148
+ except SystemExit:
149
+ logger.error(
150
+ "migrations are not in sync with ORMs, please create a migration: lamin"
151
+ " migrate create"
152
+ )
153
+ return False
154
+ return True
155
+
156
+ @classmethod
157
+ @disable_auto_connect
158
+ def squash(
159
+ cls, package_name, migration_nr, start_migration_nr: str | None = None
160
+ ) -> None:
161
+ """Squash migrations."""
162
+ from django.core.management import call_command
163
+
164
+ setup_django(settings.instance)
165
+ if start_migration_nr is not None:
166
+ call_command(
167
+ "squashmigrations", package_name, start_migration_nr, migration_nr
168
+ )
169
+ else:
170
+ call_command("squashmigrations", package_name, migration_nr)
171
+
172
+ @classmethod
173
+ @disable_auto_connect
174
+ def show(cls) -> None:
175
+ """Show migrations."""
176
+ from django.core.management import call_command
177
+
178
+ setup_django(settings.instance)
179
+ call_command("showmigrations")
180
+
181
+ @classmethod
182
+ def defined_migrations(cls, latest: bool = False):
183
+ from io import StringIO
184
+
185
+ from django.core.management import call_command
186
+
187
+ def parse_migration_output(output):
188
+ """Parse the output of the showmigrations command to get migration names."""
189
+ lines = output.splitlines()
190
+
191
+ # Initialize an empty dict to store migration names of each module
192
+ migration_names = {}
193
+
194
+ # Process each line
195
+ for line in lines:
196
+ if " " not in line:
197
+ # CLI displays the module name in bold
198
+ name = line.strip().replace("\x1b[1m", "")
199
+ migration_names[name] = []
200
+ continue
201
+ # Strip whitespace and split the line into status and migration name
202
+ migration_name = line.strip().split("] ")[-1].split(" ")[0]
203
+ # The second part is the migration name
204
+ migration_names[name].append(migration_name)
205
+
206
+ return migration_names
207
+
208
+ out = StringIO()
209
+ call_command("showmigrations", stdout=out)
210
+ out.seek(0)
211
+ output = out.getvalue()
212
+ if latest:
213
+ return {k: v[-1] for k, v in parse_migration_output(output).items()}
214
+ else:
215
+ return parse_migration_output(output)
216
+
217
+ @classmethod
218
+ def deployed_migrations(cls, latest: bool = False):
219
+ """Get the list of deployed migrations from Migration table in DB."""
220
+ if latest:
221
+ latest_migrations = {}
222
+ with connection.cursor() as cursor:
223
+ # query to get the latest migration for each app that is not squashed
224
+ cursor.execute(
225
+ """
226
+ SELECT app, name
227
+ FROM django_migrations
228
+ WHERE id IN (
229
+ SELECT MAX(id)
230
+ FROM django_migrations
231
+ WHERE name NOT LIKE '%%_squashed_%%'
232
+ GROUP BY app
233
+ )
234
+ """
235
+ )
236
+ # fetch all the results
237
+ for app, name in cursor.fetchall():
238
+ latest_migrations[app] = name
239
+
240
+ return latest_migrations
241
+ else:
242
+ # Load all migrations using Django's migration loader
243
+ loader = MigrationLoader(connection)
244
+ squashed_replacements = set()
245
+ for _key, migration in loader.disk_migrations.items():
246
+ if hasattr(migration, "replaces"):
247
+ squashed_replacements.update(migration.replaces)
248
+
249
+ deployed_migrations: dict = {}
250
+ with connection.cursor() as cursor:
251
+ cursor.execute(
252
+ """
253
+ SELECT app, name, deployed
254
+ FROM django_migrations
255
+ ORDER BY app, deployed DESC
256
+ """
257
+ )
258
+ for app, name, _deployed in cursor.fetchall():
259
+ # skip migrations that are part of a squashed migration
260
+ if (app, name) in squashed_replacements:
261
+ continue
262
+
263
+ if app not in deployed_migrations:
264
+ deployed_migrations[app] = []
265
+ deployed_migrations[app].append(name)
266
+ return deployed_migrations
@@ -1,35 +1,35 @@
1
- from __future__ import annotations
2
-
3
- from lamin_utils import logger
4
-
5
- from .core._settings import settings
6
- from .core._settings_storage import base62
7
- from .core.django import setup_django
8
-
9
-
10
- def register(_test: bool = False):
11
- """Register an instance on the hub."""
12
- from ._check_setup import _check_instance_setup
13
- from .core._hub_core import init_instance_hub, init_storage_hub
14
-
15
- logger.warning("note that register() is only for testing purposes")
16
-
17
- isettings = settings.instance
18
- if not _check_instance_setup() and not _test:
19
- setup_django(isettings)
20
-
21
- ssettings = settings.instance.storage
22
- if ssettings._uid is None and _test:
23
- # because django isn't up, we can't get it from the database
24
- ssettings._uid = base62(8)
25
- # cannot yet populate the instance id here
26
- ssettings._instance_id = None
27
- # flag auto_populate_instance can be removed once FK migration is over
28
- init_storage_hub(ssettings, auto_populate_instance=False)
29
- init_instance_hub(isettings)
30
- isettings._is_on_hub = True
31
- isettings._persist()
32
- if isettings.dialect != "sqlite" and not _test:
33
- from ._schema_metadata import update_schema_in_hub
34
-
35
- update_schema_in_hub()
1
+ from __future__ import annotations
2
+
3
+ from lamin_utils import logger
4
+
5
+ from .core._settings import settings
6
+ from .core._settings_storage import base62
7
+ from .core.django import setup_django
8
+
9
+
10
+ def register(_test: bool = False):
11
+ """Register an instance on the hub."""
12
+ from ._check_setup import _check_instance_setup
13
+ from .core._hub_core import init_instance_hub, init_storage_hub
14
+
15
+ logger.warning("note that register() is only for testing purposes")
16
+
17
+ isettings = settings.instance
18
+ if not _check_instance_setup() and not _test:
19
+ setup_django(isettings)
20
+
21
+ ssettings = settings.instance.storage
22
+ if ssettings._uid is None and _test:
23
+ # because django isn't up, we can't get it from the database
24
+ ssettings._uid = base62(8)
25
+ # cannot yet populate the instance id here
26
+ ssettings._instance_id = None
27
+ # flag auto_populate_instance can be removed once FK migration is over
28
+ init_storage_hub(ssettings, auto_populate_instance=False)
29
+ init_instance_hub(isettings)
30
+ isettings._is_on_hub = True
31
+ isettings._persist()
32
+ if isettings.dialect != "sqlite" and not _test:
33
+ from ._schema_metadata import update_schema_in_hub
34
+
35
+ update_schema_in_hub()