lamindb_setup 0.74.3__tar.gz → 0.76.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/.pre-commit-config.yaml +1 -1
  2. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/PKG-INFO +1 -1
  3. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-cloud/01-init-local-instance.ipynb +4 -1
  4. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-cloud/04-test-bionty.ipynb +3 -3
  5. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-cloud/05-init-hosted-instance.ipynb +1 -1
  6. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-cloud/06-connect-hosted-instance.ipynb +1 -1
  7. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-prod/test-sqlite-lock.ipynb +1 -1
  8. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/__init__.py +1 -1
  9. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_close.py +2 -1
  10. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_connect_instance.py +101 -2
  11. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_init_instance.py +11 -4
  12. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_register_instance.py +4 -1
  13. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_schema_metadata.py +2 -0
  14. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/__init__.py +1 -0
  15. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_hub_core.py +13 -3
  16. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_settings_instance.py +11 -10
  17. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_settings_storage.py +10 -5
  18. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_setup_bionty_sources.py +25 -9
  19. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/hashing.py +6 -4
  20. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/upath.py +1 -1
  21. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/noxfile.py +2 -2
  22. lamindb_setup-0.76.0/tests/hub-local/conftest.py +29 -0
  23. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-local/test_update_schema_in_hub.py +15 -1
  24. lamindb_setup-0.74.3/tests/hub-local/conftest.py +0 -23
  25. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/.github/workflows/build.yml +0 -0
  26. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/.github/workflows/doc-changes.yml +0 -0
  27. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/.gitignore +0 -0
  28. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/LICENSE +0 -0
  29. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/README.md +0 -0
  30. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/changelog.md +0 -0
  31. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-cloud/02-connect-local-instance.ipynb +0 -0
  32. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-cloud/03-add-managed-storage.ipynb +0 -0
  33. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-cloud/07-keep-artifacts-local.ipynb +0 -0
  34. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-cloud/test-multi-session.ipynb +0 -0
  35. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-cloud/test_notebooks.py +0 -0
  36. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-prod/test-cache-management.ipynb +0 -0
  37. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-prod/test-cloud-sync.ipynb +0 -0
  38. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-prod/test-connect-anonymously.ipynb +0 -0
  39. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-prod/test-empty-init.ipynb +0 -0
  40. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-prod/test-import-schema.ipynb +0 -0
  41. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-prod/test-insufficient-user-info.ipynb +0 -0
  42. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-prod/test-invalid-schema.ipynb +0 -0
  43. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/hub-prod/test_notebooks2.py +0 -0
  44. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/index.md +0 -0
  45. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/notebooks.md +0 -0
  46. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/docs/reference.md +0 -0
  47. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_cache.py +0 -0
  48. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_check.py +0 -0
  49. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_check_setup.py +0 -0
  50. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_delete.py +0 -0
  51. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_django.py +0 -0
  52. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_exportdb.py +0 -0
  53. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_importdb.py +0 -0
  54. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_migrate.py +0 -0
  55. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_schema.py +0 -0
  56. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_set_managed_storage.py +0 -0
  57. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_setup_user.py +0 -0
  58. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/_silence_loggers.py +0 -0
  59. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_aws_credentials.py +0 -0
  60. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_aws_storage.py +0 -0
  61. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_deprecated.py +0 -0
  62. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_docs.py +0 -0
  63. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_hub_client.py +0 -0
  64. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_hub_crud.py +0 -0
  65. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_hub_utils.py +0 -0
  66. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_private_django_api.py +0 -0
  67. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_settings.py +0 -0
  68. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_settings_load.py +0 -0
  69. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_settings_save.py +0 -0
  70. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_settings_store.py +0 -0
  71. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/_settings_user.py +0 -0
  72. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/cloud_sqlite_locker.py +0 -0
  73. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/django.py +0 -0
  74. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/exceptions.py +0 -0
  75. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/lamindb_setup/core/types.py +0 -0
  76. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/pyproject.toml +0 -0
  77. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-cloud/test_connect_instance.py +0 -0
  78. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-cloud/test_delete_instance.py +0 -0
  79. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-cloud/test_init_instance.py +0 -0
  80. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-cloud/test_login.py +0 -0
  81. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-cloud/test_migrate.py +0 -0
  82. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-cloud/test_set_storage.py +0 -0
  83. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-local/test_all.py +0 -0
  84. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-prod/conftest.py +0 -0
  85. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-prod/test_django.py +0 -0
  86. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-prod/test_global_settings.py +0 -0
  87. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-prod/test_switch_and_fallback_env.py +0 -0
  88. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/hub-prod/test_upath.py +0 -0
  89. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/storage/test_hashing.py +0 -0
  90. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/storage/test_storage_access.py +0 -0
  91. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/storage/test_storage_basis.py +0 -0
  92. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/storage/test_storage_stats.py +0 -0
  93. {lamindb_setup-0.74.3 → lamindb_setup-0.76.0}/tests/storage/test_to_url.py +0 -0
@@ -17,7 +17,7 @@ repos:
17
17
  docs/notes/
18
18
  )
19
19
  - repo: https://github.com/astral-sh/ruff-pre-commit
20
- rev: v0.1.7
20
+ rev: v0.5.5
21
21
  hooks:
22
22
  - id: ruff
23
23
  args: [--fix, --exit-non-zero-on-fix, --unsafe-fixes]
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: lamindb_setup
3
- Version: 0.74.3
3
+ Version: 0.76.0
4
4
  Summary: Setup & configure LaminDB.
5
5
  Author-email: Lamin Labs <open-source@lamin.ai>
6
6
  Description-Content-Type: text/markdown
@@ -48,6 +48,7 @@
48
48
  "outputs": [],
49
49
  "source": [
50
50
  "from pathlib import Path\n",
51
+ "from lnschema_core.models import Storage\n",
51
52
  "\n",
52
53
  "assert ln_setup.settings.instance.storage.type_is_cloud == False\n",
53
54
  "assert ln_setup.settings.instance.owner == ln_setup.settings.user.handle\n",
@@ -59,7 +60,9 @@
59
60
  "assert (\n",
60
61
  " ln_setup.settings.instance.db\n",
61
62
  " == f\"sqlite:///{Path('./mydata').resolve().as_posix()}/{ln_setup.settings.instance._id.hex}.lndb\"\n",
62
- ")"
63
+ ")\n",
64
+ "assert ln_setup.settings.storage._instance_id == ln_setup.settings.instance._id\n",
65
+ "assert Storage.objects.get(instance_uid=ln_setup.settings.instance.uid).root == ln_setup.settings.storage.root_as_str\n"
63
66
  ]
64
67
  },
65
68
  {
@@ -49,7 +49,7 @@
49
49
  "metadata": {},
50
50
  "outputs": [],
51
51
  "source": [
52
- "from lnschema_bionty import Source"
52
+ "from bionty import Source"
53
53
  ]
54
54
  },
55
55
  {
@@ -70,7 +70,7 @@
70
70
  },
71
71
  "outputs": [],
72
72
  "source": [
73
- "from bionty_base import settings as bionty_base_settings\n",
73
+ "from bionty.base import settings as bionty_base_settings\n",
74
74
  "\n",
75
75
  "assert sources_df.shape[0] > 0\n",
76
76
  "assert bionty_base_settings.lamindb_sources.exists()"
@@ -154,7 +154,7 @@
154
154
  "name": "python",
155
155
  "nbconvert_exporter": "python",
156
156
  "pygments_lexer": "ipython3",
157
- "version": "3.9.16"
157
+ "version": "3.10.13"
158
158
  },
159
159
  "vscode": {
160
160
  "interpreter": {
@@ -125,7 +125,7 @@
125
125
  "name": "python",
126
126
  "nbconvert_exporter": "python",
127
127
  "pygments_lexer": "ipython3",
128
- "version": "3.9.17"
128
+ "version": "3.10.13"
129
129
  }
130
130
  },
131
131
  "nbformat": 4,
@@ -170,7 +170,7 @@
170
170
  "name": "python",
171
171
  "nbconvert_exporter": "python",
172
172
  "pygments_lexer": "ipython3",
173
- "version": "3.9.17"
173
+ "version": "3.10.13"
174
174
  }
175
175
  },
176
176
  "nbformat": 4,
@@ -272,7 +272,7 @@
272
272
  "name": "python",
273
273
  "nbconvert_exporter": "python",
274
274
  "pygments_lexer": "ipython3",
275
- "version": "3.9.17"
275
+ "version": "3.10.13"
276
276
  }
277
277
  },
278
278
  "nbformat": 4,
@@ -34,7 +34,7 @@ Modules & settings:
34
34
 
35
35
  """
36
36
 
37
- __version__ = "0.74.3" # denote a release candidate for 0.1.0 with 0.1rc1
37
+ __version__ = "0.76.0" # denote a release candidate for 0.1.0 with 0.1rc1
38
38
 
39
39
  import sys
40
40
  from os import name as _os_name
@@ -24,8 +24,9 @@ def close(mute: bool = False) -> None:
24
24
  logger.warning("did not upload cache file - not enough permissions")
25
25
  else:
26
26
  raise e
27
+ if "bionty" in settings.instance.schema:
28
+ delete_bionty_sources_yaml()
27
29
  current_instance_settings_file().unlink()
28
- delete_bionty_sources_yaml()
29
30
  clear_locker()
30
31
  if not mute:
31
32
  logger.success(f"closed instance: {instance}")
@@ -4,7 +4,6 @@ import os
4
4
  from typing import TYPE_CHECKING
5
5
  from uuid import UUID
6
6
 
7
- from django.db import ProgrammingError
8
7
  from lamin_utils import logger
9
8
 
10
9
  from ._check_setup import _check_instance_setup
@@ -21,7 +20,7 @@ from .core._settings import settings
21
20
  from .core._settings_instance import InstanceSettings
22
21
  from .core._settings_load import load_instance_settings
23
22
  from .core._settings_storage import StorageSettings
24
- from .core._settings_store import instance_settings_file
23
+ from .core._settings_store import instance_settings_file, settings_dir
25
24
  from .core.cloud_sqlite_locker import unlock_cloud_sqlite_upon_exception
26
25
 
27
26
  if TYPE_CHECKING:
@@ -257,9 +256,109 @@ def connect(
257
256
  if isettings is not None:
258
257
  isettings._get_settings_file().unlink(missing_ok=True) # type: ignore
259
258
  raise e
259
+ # rename lnschema_bionty to bionty for sql tables
260
+ if "bionty" in isettings.schema:
261
+ no_lnschema_bionty_file = (
262
+ settings_dir / f"no_lnschema_bionty-{isettings.slug.replace('/', '')}"
263
+ )
264
+ if not no_lnschema_bionty_file.exists():
265
+ migrate_lnschema_bionty(isettings, no_lnschema_bionty_file)
260
266
  return None
261
267
 
262
268
 
269
+ def migrate_lnschema_bionty(isettings: InstanceSettings, no_lnschema_bionty_file: Path):
270
+ """Migrate lnschema_bionty tables to bionty tables if bionty_source doesn't exist.
271
+
272
+ :param db_uri: str, database URI (e.g., 'sqlite:///path/to/db.sqlite' or 'postgresql://user:password@host:port/dbname')
273
+ """
274
+ from urllib.parse import urlparse
275
+
276
+ parsed_uri = urlparse(isettings.db)
277
+ db_type = parsed_uri.scheme
278
+
279
+ if db_type == "sqlite":
280
+ import sqlite3
281
+
282
+ conn = sqlite3.connect(parsed_uri.path)
283
+ elif db_type in ["postgresql", "postgres"]:
284
+ import psycopg2
285
+
286
+ conn = psycopg2.connect(isettings.db)
287
+ else:
288
+ raise ValueError("Unsupported database type. Use 'sqlite' or 'postgresql' URI.")
289
+
290
+ cur = conn.cursor()
291
+
292
+ try:
293
+ # check if bionty_source table exists
294
+ if db_type == "sqlite":
295
+ cur.execute(
296
+ "SELECT name FROM sqlite_master WHERE type='table' AND name='bionty_source'"
297
+ )
298
+ migrated = cur.fetchone() is not None
299
+
300
+ # tables that need to be renamed
301
+ cur.execute(
302
+ "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'lnschema_bionty_%'"
303
+ )
304
+ tables_to_rename = [
305
+ row[0][len("lnschema_bionty_") :] for row in cur.fetchall()
306
+ ]
307
+ else: # postgres
308
+ cur.execute(
309
+ "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'bionty_source')"
310
+ )
311
+ migrated = cur.fetchone()[0]
312
+
313
+ # tables that need to be renamed
314
+ cur.execute(
315
+ "SELECT table_name FROM information_schema.tables WHERE table_name LIKE 'lnschema_bionty_%'"
316
+ )
317
+ tables_to_rename = [
318
+ row[0][len("lnschema_bionty_") :] for row in cur.fetchall()
319
+ ]
320
+
321
+ if migrated:
322
+ no_lnschema_bionty_file.touch(exist_ok=True)
323
+ else:
324
+ try:
325
+ # rename tables only if bionty_source doesn't exist and there are tables to rename
326
+ for table in tables_to_rename:
327
+ if db_type == "sqlite":
328
+ cur.execute(
329
+ f"ALTER TABLE lnschema_bionty_{table} RENAME TO bionty_{table}"
330
+ )
331
+ else: # postgres
332
+ cur.execute(
333
+ f"ALTER TABLE lnschema_bionty_{table} RENAME TO bionty_{table};"
334
+ )
335
+
336
+ # update django_migrations table
337
+ cur.execute(
338
+ "UPDATE django_migrations SET app = 'bionty' WHERE app = 'lnschema_bionty'"
339
+ )
340
+
341
+ logger.warning(
342
+ "Please uninstall lnschema-bionty via `pip uninstall lnschema-bionty`!"
343
+ )
344
+
345
+ no_lnschema_bionty_file.touch(exist_ok=True)
346
+ except Exception:
347
+ # read-only users can't rename tables
348
+ pass
349
+
350
+ conn.commit()
351
+
352
+ except Exception as e:
353
+ print(f"An error occurred: {e}")
354
+ conn.rollback()
355
+
356
+ finally:
357
+ # close the cursor and connection
358
+ cur.close()
359
+ conn.close()
360
+
361
+
263
362
  def load(
264
363
  slug: str,
265
364
  *,
@@ -15,6 +15,7 @@ from ._close import close as close_instance
15
15
  from ._silence_loggers import silence_loggers
16
16
  from .core import InstanceSettings
17
17
  from .core._settings import settings
18
+ from .core._settings_instance import is_local_db_url
18
19
  from .core._settings_storage import StorageSettings, init_storage
19
20
  from .core.upath import UPath
20
21
 
@@ -245,10 +246,16 @@ def init(
245
246
  if instance_state == "connected":
246
247
  settings.auto_connect = True # we can also debate this switch here
247
248
  return None
248
- # cannot past instance_id here because instance does not yet exist!
249
- # the instance_id field of the storage table is populated at the end of
250
- # init_instance_hub
251
- ssettings = init_storage(storage)
249
+ # the conditions here match `isettings.is_remote`, but I currently don't
250
+ # see a way of making this more elegant; should become possible if we
251
+ # remove the instance.storage_id FK on the hub
252
+ prevent_register_hub = is_local_db_url(db) if db is not None else False
253
+ ssettings = init_storage(
254
+ storage,
255
+ instance_id=instance_id,
256
+ init_instance=True,
257
+ prevent_register_hub=prevent_register_hub,
258
+ )
252
259
  isettings = InstanceSettings(
253
260
  id=instance_id, # type: ignore
254
261
  owner=settings.user.handle,
@@ -23,7 +23,10 @@ def register(_test: bool = False):
23
23
  if ssettings._uid is None and _test:
24
24
  # because django isn't up, we can't get it from the database
25
25
  ssettings._uid = base62(8)
26
- init_storage_hub(ssettings)
26
+ # cannot yet populate the instance id here
27
+ ssettings._instance_id = None
28
+ # flag auto_populate_instance can be removed once FK migration is over
29
+ init_storage_hub(ssettings, auto_populate_instance=False)
27
30
  init_instance_hub(isettings)
28
31
  isettings._is_on_hub = True
29
32
  isettings._persist()
@@ -102,6 +102,8 @@ Type = Literal[
102
102
  "JSONField",
103
103
  "DateField",
104
104
  "FloatField",
105
+ "PositiveIntegerField",
106
+ "PositiveBigIntegerField",
105
107
  ]
106
108
 
107
109
 
@@ -11,6 +11,7 @@ Settings:
11
11
  StorageSettings
12
12
 
13
13
  """
14
+
14
15
  from . import django, types, upath
15
16
  from ._deprecated import deprecated
16
17
  from ._docs import doc_args
@@ -118,11 +118,13 @@ def _select_storage(
118
118
 
119
119
  def init_storage(
120
120
  ssettings: StorageSettings,
121
+ auto_populate_instance: bool = True,
121
122
  ) -> None:
122
123
  if settings.user.handle != "anonymous":
123
124
  return call_with_fallback_auth(
124
125
  _init_storage,
125
126
  ssettings=ssettings,
127
+ auto_populate_instance=auto_populate_instance,
126
128
  )
127
129
  else:
128
130
  storage_exists = call_with_fallback(
@@ -134,7 +136,9 @@ def init_storage(
134
136
  raise ValueError("Log in to create a storage location on the hub.")
135
137
 
136
138
 
137
- def _init_storage(ssettings: StorageSettings, client: Client) -> None:
139
+ def _init_storage(
140
+ ssettings: StorageSettings, auto_populate_instance: bool, client: Client
141
+ ) -> None:
138
142
  from lamindb_setup import settings
139
143
 
140
144
  # storage roots are always stored without the trailing slash in the SQL
@@ -146,13 +150,19 @@ def _init_storage(ssettings: StorageSettings, client: Client) -> None:
146
150
  id = uuid.uuid5(uuid.NAMESPACE_URL, root)
147
151
  else:
148
152
  id = uuid.uuid4()
149
- if ssettings._instance_id is None and settings._instance_exists:
153
+ if (
154
+ ssettings._instance_id is None
155
+ and settings._instance_exists
156
+ and auto_populate_instance
157
+ ):
150
158
  logger.warning(
151
159
  f"will manage storage location {ssettings.root_as_str} with instance {settings.instance.slug}"
152
160
  )
153
161
  ssettings._instance_id = settings.instance._id
154
162
  instance_id_hex = (
155
- None if ssettings._instance_id is None else ssettings._instance_id.hex
163
+ ssettings._instance_id.hex
164
+ if (ssettings._instance_id is not None and auto_populate_instance)
165
+ else None
156
166
  )
157
167
  fields = {
158
168
  "id": id.hex,
@@ -29,6 +29,16 @@ def sanitize_git_repo_url(repo_url: str) -> str:
29
29
  return repo_url.replace(".git", "")
30
30
 
31
31
 
32
+ def is_local_db_url(db_url: str) -> bool:
33
+ if "@localhost:" in db_url:
34
+ return True
35
+ if "@0.0.0.0:" in db_url:
36
+ return True
37
+ if "@127.0.0.1" in db_url:
38
+ return True
39
+ return False
40
+
41
+
32
42
  class InstanceSettings:
33
43
  """Instance settings."""
34
44
 
@@ -370,17 +380,8 @@ class InstanceSettings:
370
380
  if not self.storage.type_is_cloud:
371
381
  return False
372
382
 
373
- def is_local_uri(uri: str):
374
- if "@localhost:" in uri:
375
- return True
376
- if "@0.0.0.0:" in uri:
377
- return True
378
- if "@127.0.0.1" in uri:
379
- return True
380
- return False
381
-
382
383
  if self.dialect == "postgresql":
383
- if is_local_uri(self.db):
384
+ if is_local_db_url(self.db):
384
385
  return False
385
386
  # returns True for cloud SQLite
386
387
  # and remote postgres
@@ -78,7 +78,11 @@ def mark_storage_root(root: UPathStr, uid: str):
78
78
 
79
79
 
80
80
  def init_storage(
81
- root: UPathStr, instance_id: UUID | None = None, register_hub: bool | None = None
81
+ root: UPathStr,
82
+ instance_id: UUID | None = None,
83
+ register_hub: bool | None = None,
84
+ prevent_register_hub: bool = False,
85
+ init_instance: bool = False,
82
86
  ) -> StorageSettings:
83
87
  if root is None:
84
88
  raise ValueError("`storage` argument can't be `None`")
@@ -114,11 +118,12 @@ def init_storage(
114
118
  instance_id=instance_id,
115
119
  )
116
120
  # the below might update the uid with one that's already taken on the hub
117
- if ssettings.type_is_cloud or register_hub:
118
- from ._hub_core import delete_storage_record
119
- from ._hub_core import init_storage as init_storage_hub
121
+ if not prevent_register_hub:
122
+ if ssettings.type_is_cloud or register_hub:
123
+ from ._hub_core import delete_storage_record
124
+ from ._hub_core import init_storage as init_storage_hub
120
125
 
121
- init_storage_hub(ssettings)
126
+ init_storage_hub(ssettings, auto_populate_instance=not init_instance)
122
127
  # below comes last only if everything else was successful
123
128
  try:
124
129
  # (federated) credentials for AWS access are provisioned under-the-hood
@@ -7,6 +7,9 @@ from django.db.utils import OperationalError, ProgrammingError
7
7
  if TYPE_CHECKING:
8
8
  from ._settings_instance import InstanceSettings
9
9
 
10
+ # bionty.Source -> bionty.base
11
+ RENAME = {"name": "source", "description": "source_name"}
12
+
10
13
 
11
14
  def write_bionty_sources(isettings: InstanceSettings) -> None:
12
15
  """Write bionty sources to Source table."""
@@ -14,9 +17,13 @@ def write_bionty_sources(isettings: InstanceSettings) -> None:
14
17
  return None
15
18
  import shutil
16
19
 
17
- import bionty_base
18
- from bionty_base.dev._handle_sources import parse_sources_yaml
19
- from lnschema_bionty.models import Source
20
+ import bionty
21
+ import bionty.base as bionty_base
22
+ from bionty._bionty import list_biorecord_models
23
+ from bionty.base.dev._handle_sources import parse_sources_yaml
24
+ from bionty.models import Source
25
+
26
+ bionty_models = list_biorecord_models(bionty)
20
27
 
21
28
  shutil.copy(
22
29
  bionty_base.settings.current_sources, bionty_base.settings.lamindb_sources
@@ -46,7 +53,10 @@ def write_bionty_sources(isettings: InstanceSettings) -> None:
46
53
  # won't need this once lamindb is released with the new pin
47
54
  kwargs["run"] = None # can't yet access tracking information
48
55
  kwargs["in_db"] = False
49
- # kwargs["name"] = kwargs.pop("source")
56
+ for db_field, base_col in RENAME.items():
57
+ kwargs[db_field] = kwargs.pop(base_col)
58
+ if kwargs["entity"] in bionty_models:
59
+ kwargs["entity"] = f"bionty.{kwargs['entity']}"
50
60
  record = Source(**kwargs)
51
61
  all_records.append(record)
52
62
 
@@ -58,14 +68,20 @@ def load_bionty_sources(isettings: InstanceSettings):
58
68
  if "bionty" not in isettings.schema:
59
69
  return None
60
70
 
61
- import bionty_base
62
- from bionty_base.dev._handle_sources import parse_currently_used_sources
63
- from bionty_base.dev._io import write_yaml
64
- from lnschema_bionty.models import Source
71
+ import bionty.base as bionty_base
72
+ from bionty.base.dev._handle_sources import parse_currently_used_sources
73
+ from bionty.base.dev._io import write_yaml
74
+ from bionty.models import Source
65
75
 
66
76
  try:
67
77
  # need try except because of integer primary key migration
68
78
  active_records = Source.objects.filter(currently_used=True).all().values()
79
+ for kwargs in active_records:
80
+ for db_field, base_col in RENAME.items():
81
+ kwargs[base_col] = kwargs.pop(db_field)
82
+ # TODO: non-bionty schema?
83
+ if db_field == "entity":
84
+ kwargs["entity"] = kwargs["entity"].split(".")[1]
69
85
  write_yaml(
70
86
  parse_currently_used_sources(active_records),
71
87
  bionty_base.settings.lamindb_sources,
@@ -77,7 +93,7 @@ def load_bionty_sources(isettings: InstanceSettings):
77
93
  def delete_bionty_sources_yaml():
78
94
  """Delete LAMINDB_SOURCES_PATH in bionty."""
79
95
  try:
80
- import bionty_base
96
+ import bionty.base as bionty_base
81
97
 
82
98
  bionty_base.settings.lamindb_sources.unlink(missing_ok=True)
83
99
  except ModuleNotFoundError:
@@ -17,6 +17,8 @@ from typing import TYPE_CHECKING, Iterable
17
17
 
18
18
  import psutil
19
19
 
20
+ HASH_LENGTH = 22
21
+
20
22
  if TYPE_CHECKING:
21
23
  from .types import Path, UPathStr
22
24
 
@@ -39,8 +41,8 @@ def b16_to_b64(s: str):
39
41
  # a lot to read about this: lamin-notes/2022/hashing
40
42
  def hash_set(s: set[str]) -> str:
41
43
  bstr = ":".join(sorted(s)).encode("utf-8")
42
- # as we're truncating at 20 b64, we choose md5 over sha512
43
- return to_b64_str(hashlib.md5(bstr).digest())[:20]
44
+ # as we're truncating at 22 b64, we choose md5 over sha512
45
+ return to_b64_str(hashlib.md5(bstr).digest())[:HASH_LENGTH]
44
46
 
45
47
 
46
48
  def hash_md5s_from_dir(hashes: Iterable[str]) -> tuple[str, str]:
@@ -50,7 +52,7 @@ def hash_md5s_from_dir(hashes: Iterable[str]) -> tuple[str, str]:
50
52
  hashlib.md5(hash.encode("utf-8")).digest() for hash in sorted(hashes)
51
53
  )
52
54
  digest = hashlib.md5(digests).digest()
53
- return to_b64_str(digest)[:22], "md5-d"
55
+ return to_b64_str(digest)[:HASH_LENGTH], "md5-d"
54
56
 
55
57
 
56
58
  def hash_code(file_path: UPathStr):
@@ -85,7 +87,7 @@ def hash_file(
85
87
  hashlib.sha1(first_chunk).digest() + hashlib.sha1(last_chunk).digest()
86
88
  ).digest()
87
89
  hash_type = "sha1-fl"
88
- return to_b64_str(digest)[:22], hash_type
90
+ return to_b64_str(digest)[:HASH_LENGTH], hash_type
89
91
 
90
92
 
91
93
  def hash_dir(path: Path):
@@ -772,7 +772,7 @@ def check_storage_is_empty(
772
772
  hint += " ignored"
773
773
  message = (
774
774
  f"Storage {directory_string} contains {n_objects - n_offset_objects} objects "
775
- f"({hint}) - {ask_for_deletion}\n{objects}"
775
+ f"({hint}) - {ask_for_deletion}"
776
776
  )
777
777
  if n_diff > 0:
778
778
  if raise_error:
@@ -27,8 +27,8 @@ def lint(session: nox.Session) -> None:
27
27
  ["hub-local", "hub-prod", "hub-cloud", "storage", "docs"],
28
28
  )
29
29
  def install(session: nox.Session, group: str) -> None:
30
- no_deps_packages = "git+https://github.com/laminlabs/lnschema-bionty git+https://github.com/laminlabs/lnschema-core git+https://github.com/laminlabs/wetlab lamin-cli"
31
- schema_deps = f"""uv pip install --system bionty git+https://github.com/laminlabs/bionty-base
30
+ no_deps_packages = "git+https://github.com/laminlabs/lnschema-core git+https://github.com/laminlabs/wetlab lamin-cli"
31
+ schema_deps = f"""uv pip install --system git+https://github.com/laminlabs/bionty git+https://github.com/laminlabs/lamindb@main
32
32
  uv pip install --system --no-deps {no_deps_packages}
33
33
  """
34
34
  if group == "hub-cloud":
@@ -0,0 +1,29 @@
1
+ from __future__ import annotations
2
+
3
+ import os
4
+
5
+ from laminhub_rest.dev._seed import seed_local_test
6
+ from laminhub_rest.dev._supabase import SupabaseResources
7
+ from laminhub_rest.dev._utils import remove_lamin_local_settings
8
+
9
+ supabase_resources = SupabaseResources()
10
+
11
+
12
+ pytest_plugins = [
13
+ "laminhub_rest.core.account.user.test.fixtures",
14
+ "laminhub_rest.test.fixtures.run_id",
15
+ ]
16
+
17
+
18
+ def pytest_configure():
19
+ os.environ["LAMIN_ENV"] = "local"
20
+ remove_lamin_local_settings()
21
+ supabase_resources.start_local()
22
+ supabase_resources.reset_local()
23
+ supabase_resources.migrate()
24
+ seed_local_test()
25
+
26
+
27
+ def pytest_unconfigure():
28
+ if supabase_resources.edge_function_process:
29
+ supabase_resources.stop_local_edge_functions()
@@ -55,6 +55,20 @@ def test_update_schema_in_hub(setup_instance):
55
55
  "related_schema_name": None,
56
56
  }
57
57
 
58
+ assert schema["schema_json"]["core"]["artifact"]["fields"]["_accessor"] == {
59
+ "type": "CharField",
60
+ "column_name": "accessor",
61
+ "through": None,
62
+ "field_name": "_accessor",
63
+ "model_name": "artifact",
64
+ "schema_name": "core",
65
+ "is_link_table": False,
66
+ "relation_type": None,
67
+ "related_field_name": None,
68
+ "related_model_name": None,
69
+ "related_schema_name": None,
70
+ }
71
+
58
72
  assert schema["schema_json"]["core"]["artifact"]["fields"]["created_by"] == {
59
73
  "type": "ForeignKey",
60
74
  "column_name": "created_by_id",
@@ -79,7 +93,7 @@ def test_update_schema_in_hub(setup_instance):
79
93
  "through": {
80
94
  "left_key": "gene_id",
81
95
  "right_key": "pathway_id",
82
- "link_table_name": "lnschema_bionty_pathway_genes",
96
+ "link_table_name": "bionty_pathway_genes",
83
97
  },
84
98
  "field_name": "pathways",
85
99
  "model_name": "gene",
@@ -1,23 +0,0 @@
1
- from __future__ import annotations
2
-
3
- import os
4
-
5
- from lamin_utils import logger
6
- from laminhub_rest.dev._setup_laminapp_rest import setup_local
7
-
8
- pytest_plugins = [
9
- "laminhub_rest.core.account.user.test.fixtures",
10
- "laminhub_rest.test.fixtures.run_id",
11
- ]
12
-
13
- local_setup_state = setup_local()
14
-
15
-
16
- def pytest_configure():
17
- assert os.environ["LAMIN_ENV"] == "local"
18
- local_setup_state.__enter__()
19
-
20
-
21
- def pytest_unconfigure():
22
- if os.environ["LAMIN_ENV"] == "local":
23
- local_setup_state.__exit__(None, None, None)
File without changes
File without changes