lamindb_setup 1.17.0__tar.gz → 1.18.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/.github/workflows/build.yml +1 -1
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/PKG-INFO +5 -5
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/docs/hub-cloud/01-init-local-instance.ipynb +16 -4
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/docs/hub-cloud/02-connect-local-instance.ipynb +1 -1
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/docs/hub-cloud/03-add-managed-storage.ipynb +43 -26
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/docs/hub-cloud/05-init-hosted-instance.ipynb +1 -1
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/docs/hub-prod/test-import-schema.ipynb +2 -2
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/__init__.py +1 -1
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/_connect_instance.py +14 -10
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/_migrate.py +56 -16
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/_set_managed_storage.py +3 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/__init__.py +0 -3
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/_aws_options.py +1 -1
- lamindb_setup-1.18.1/lamindb_setup/core/_clone.py +50 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/_private_django_api.py +1 -3
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/_settings_instance.py +12 -18
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/_settings_storage.py +0 -5
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/django.py +9 -9
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/hashing.py +5 -5
- lamindb_setup-1.18.1/lamindb_setup/core/lamin.db.gz +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/errors.py +5 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/noxfile.py +12 -3
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/pyproject.toml +3 -2
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/hub-local/README.md +2 -2
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/hub-local/conftest.py +18 -13
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/hub-local/test_all.py +10 -5
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/hub-local/test_update_schema_in_hub.py +5 -5
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/storage/test_db_import_export.py +4 -8
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/storage/test_hashing.py +13 -9
- lamindb_setup-1.17.0/lamindb_setup/core/_clone.py +0 -174
- lamindb_setup-1.17.0/lamindb_setup/core/lamin.db.gz +0 -0
- lamindb_setup-1.17.0/tests/hub-cloud/test_clone_instance.py +0 -137
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/.github/workflows/doc-changes.yml +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/.gitignore +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/.pre-commit-config.yaml +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/LICENSE +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/README.md +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/docs/changelog.md +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/docs/hub-cloud/04-test-bionty.ipynb +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/docs/hub-cloud/06-connect-hosted-instance.ipynb +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/docs/hub-cloud/07-keep-artifacts-local.ipynb +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/docs/hub-cloud/08-test-multi-session.ipynb +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/docs/hub-cloud/09-test-migrate.ipynb +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/docs/hub-cloud/test_notebooks.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/docs/hub-prod/test-cache-management.ipynb +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/docs/hub-prod/test-cloud-sync.ipynb +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/docs/hub-prod/test-connect-anonymously.ipynb +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/docs/hub-prod/test-empty-init.ipynb +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/docs/hub-prod/test-init-load-local-anonymously.ipynb +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/docs/hub-prod/test-invalid-schema.ipynb +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/docs/hub-prod/test-sqlite-lock.ipynb +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/docs/hub-prod/test_notebooks2.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/docs/index.md +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/docs/notebooks.md +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/docs/reference.md +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/_cache.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/_check.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/_check_setup.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/_delete.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/_disconnect.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/_django.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/_entry_points.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/_init_instance.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/_register_instance.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/_schema.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/_schema_metadata.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/_setup_user.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/_silence_loggers.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/_aws_storage.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/_deprecated.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/_docs.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/_hub_client.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/_hub_core.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/_hub_crud.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/_hub_utils.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/_settings.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/_settings_load.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/_settings_save.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/_settings_store.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/_settings_user.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/_setup_bionty_sources.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/cloud_sqlite_locker.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/exceptions.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/types.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/core/upath.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/io.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/py.typed +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/lamindb_setup/types.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/connectivity/conftest.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/connectivity/test_proxies_certificates.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/hub-cloud/scripts/script-init-pass-user-no-writes.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/hub-cloud/scripts/script-to-fail-managed-storage.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/hub-cloud/test_connect_instance.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/hub-cloud/test_delete_instance.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/hub-cloud/test_edge_request.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/hub-cloud/test_fail_managed_storage.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/hub-cloud/test_init_instance.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/hub-cloud/test_init_pass_user_no_writes.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/hub-cloud/test_login.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/hub-cloud/test_set_storage.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/hub-local/scripts/script-connect-fine-grained-access.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/hub-prod/conftest.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/hub-prod/test_aws_options_manager.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/hub-prod/test_django.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/hub-prod/test_global_settings.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/hub-prod/test_migrate.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/hub-prod/test_switch_and_fallback_env.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/hub-prod/test_upath.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/storage/conftest.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/storage/test_entry_point.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/storage/test_storage_access.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/storage/test_storage_basis.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/storage/test_storage_settings.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/storage/test_storage_stats.py +0 -0
- {lamindb_setup-1.17.0 → lamindb_setup-1.18.1}/tests/storage/test_to_url.py +0 -0
|
@@ -137,7 +137,7 @@ jobs:
|
|
|
137
137
|
touch .env.local
|
|
138
138
|
echo "AWS_ACCESS_KEY_ID_HOSTED_S3=${{ secrets.AWS_ACCESS_KEY_ID }}" >> .env.local
|
|
139
139
|
echo "AWS_SECRET_ACCESS_KEY_HOSTED_S3=${{ secrets.AWS_SECRET_ACCESS_KEY }}" >> .env.local
|
|
140
|
-
working-directory: laminhub/
|
|
140
|
+
working-directory: laminhub/backend/central/supabase
|
|
141
141
|
- uses: actions/setup-python@v6
|
|
142
142
|
with:
|
|
143
143
|
python-version: "3.11" # we need to run everything for coverage on 3.11
|
|
@@ -1,11 +1,10 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
2
|
Name: lamindb_setup
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.18.1
|
|
4
4
|
Summary: Setup & configure LaminDB.
|
|
5
5
|
Author-email: Lamin Labs <open-source@lamin.ai>
|
|
6
6
|
Requires-Python: >=3.10
|
|
7
7
|
Description-Content-Type: text/markdown
|
|
8
|
-
License-File: LICENSE
|
|
9
8
|
Requires-Dist: lamin_utils>=0.3.3
|
|
10
9
|
Requires-Dist: django>=5.2,<5.3
|
|
11
10
|
Requires-Dist: dj_database_url>=1.3.0,<3.0.0
|
|
@@ -17,11 +16,12 @@ Requires-Dist: requests
|
|
|
17
16
|
Requires-Dist: universal_pathlib==0.2.6
|
|
18
17
|
Requires-Dist: botocore<2.0.0
|
|
19
18
|
Requires-Dist: supabase>=2.20.0,<=2.24.0
|
|
19
|
+
Requires-Dist: websockets>=13.0
|
|
20
20
|
Requires-Dist: pyjwt<3.0.0
|
|
21
21
|
Requires-Dist: psutil
|
|
22
22
|
Requires-Dist: packaging
|
|
23
23
|
Requires-Dist: aiobotocore[boto3]>=2.12.4,<3.0.0 ; extra == "aws"
|
|
24
|
-
Requires-Dist: s3fs>=2023.12.2,<=2025.
|
|
24
|
+
Requires-Dist: s3fs>=2023.12.2,<=2025.12.0,!=2024.10.0 ; extra == "aws"
|
|
25
25
|
Requires-Dist: line_profiler ; extra == "dev"
|
|
26
26
|
Requires-Dist: psycopg2-binary ; extra == "dev"
|
|
27
27
|
Requires-Dist: python-dotenv ; extra == "dev"
|
|
@@ -32,7 +32,7 @@ Requires-Dist: pytest-xdist ; extra == "dev"
|
|
|
32
32
|
Requires-Dist: nbproject-test>=0.4.3 ; extra == "dev"
|
|
33
33
|
Requires-Dist: pandas ; extra == "dev"
|
|
34
34
|
Requires-Dist: django-schema-graph ; extra == "erdiagram"
|
|
35
|
-
Requires-Dist: gcsfs>=2023.12.2,<=2025.
|
|
35
|
+
Requires-Dist: gcsfs>=2023.12.2,<=2025.12.0 ; extra == "gcp"
|
|
36
36
|
Project-URL: Home, https://github.com/laminlabs/lamindb-setup
|
|
37
37
|
Provides-Extra: aws
|
|
38
38
|
Provides-Extra: dev
|
|
@@ -24,7 +24,8 @@
|
|
|
24
24
|
"metadata": {},
|
|
25
25
|
"outputs": [],
|
|
26
26
|
"source": [
|
|
27
|
-
"import lamindb_setup as ln_setup"
|
|
27
|
+
"import lamindb_setup as ln_setup\n",
|
|
28
|
+
"import os"
|
|
28
29
|
]
|
|
29
30
|
},
|
|
30
31
|
{
|
|
@@ -42,7 +43,7 @@
|
|
|
42
43
|
"metadata": {},
|
|
43
44
|
"outputs": [],
|
|
44
45
|
"source": [
|
|
45
|
-
"ln_setup.init(storage=\"./mydata\")"
|
|
46
|
+
"ln_setup.init(storage=\"./mydata\", modules=\"bionty\")"
|
|
46
47
|
]
|
|
47
48
|
},
|
|
48
49
|
{
|
|
@@ -70,10 +71,11 @@
|
|
|
70
71
|
"assert ln_setup.settings.instance.storage.type_is_cloud is False\n",
|
|
71
72
|
"assert ln_setup.settings.instance.owner == ln_setup.settings.user.handle\n",
|
|
72
73
|
"assert ln_setup.settings.instance.name == \"mydata\"\n",
|
|
74
|
+
"assert ln_setup.settings.instance.modules == {\"bionty\"}\n",
|
|
73
75
|
"assert ln_setup.settings.storage.root.as_posix() == Path(\"mydata\").resolve().as_posix()\n",
|
|
74
76
|
"storage_root = ln_setup.settings.storage.root\n",
|
|
75
77
|
"assert storage_root.exists()\n",
|
|
76
|
-
"assert ln_setup.settings.storage.
|
|
78
|
+
"assert ln_setup.settings.storage._id is not None\n",
|
|
77
79
|
"assert (\n",
|
|
78
80
|
" ln_setup.settings.instance.db\n",
|
|
79
81
|
" == f\"sqlite:///{Path('./mydata').resolve().as_posix()}/.lamindb/lamin.db\"\n",
|
|
@@ -85,6 +87,16 @@
|
|
|
85
87
|
")"
|
|
86
88
|
]
|
|
87
89
|
},
|
|
90
|
+
{
|
|
91
|
+
"cell_type": "code",
|
|
92
|
+
"execution_count": null,
|
|
93
|
+
"metadata": {},
|
|
94
|
+
"outputs": [],
|
|
95
|
+
"source": [
|
|
96
|
+
"exit_status = os.system(\"lamin migrate deploy\")\n",
|
|
97
|
+
"assert exit_status == 0"
|
|
98
|
+
]
|
|
99
|
+
},
|
|
88
100
|
{
|
|
89
101
|
"cell_type": "code",
|
|
90
102
|
"execution_count": null,
|
|
@@ -124,7 +136,7 @@
|
|
|
124
136
|
"name": "python",
|
|
125
137
|
"nbconvert_exporter": "python",
|
|
126
138
|
"pygments_lexer": "ipython3",
|
|
127
|
-
"version": "3.
|
|
139
|
+
"version": "3.11.14"
|
|
128
140
|
}
|
|
129
141
|
},
|
|
130
142
|
"nbformat": 4,
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
"tags": []
|
|
8
8
|
},
|
|
9
9
|
"source": [
|
|
10
|
-
"# Add
|
|
10
|
+
"# Add writeable and read-only storage locations to an instance"
|
|
11
11
|
]
|
|
12
12
|
},
|
|
13
13
|
{
|
|
@@ -84,22 +84,12 @@
|
|
|
84
84
|
"ln_setup.init(storage=\"./storage1\", name=\"test-add-managed-storage\", db=pgurl)"
|
|
85
85
|
]
|
|
86
86
|
},
|
|
87
|
-
{
|
|
88
|
-
"cell_type": "code",
|
|
89
|
-
"execution_count": null,
|
|
90
|
-
"id": "95a61826",
|
|
91
|
-
"metadata": {},
|
|
92
|
-
"outputs": [],
|
|
93
|
-
"source": [
|
|
94
|
-
"ln_setup.settings.instance.is_on_hub"
|
|
95
|
-
]
|
|
96
|
-
},
|
|
97
87
|
{
|
|
98
88
|
"cell_type": "markdown",
|
|
99
89
|
"id": "b62e84cb",
|
|
100
90
|
"metadata": {},
|
|
101
91
|
"source": [
|
|
102
|
-
"Test adding referenced storage location."
|
|
92
|
+
"Test adding a referenced read-only storage location."
|
|
103
93
|
]
|
|
104
94
|
},
|
|
105
95
|
{
|
|
@@ -120,7 +110,7 @@
|
|
|
120
110
|
"id": "93234f70",
|
|
121
111
|
"metadata": {},
|
|
122
112
|
"source": [
|
|
123
|
-
"Now continue with
|
|
113
|
+
"Now continue with writeable storage locations."
|
|
124
114
|
]
|
|
125
115
|
},
|
|
126
116
|
{
|
|
@@ -134,6 +124,14 @@
|
|
|
134
124
|
"storage1_uid = ln_setup.settings.storage.uid"
|
|
135
125
|
]
|
|
136
126
|
},
|
|
127
|
+
{
|
|
128
|
+
"cell_type": "markdown",
|
|
129
|
+
"id": "548190d1",
|
|
130
|
+
"metadata": {},
|
|
131
|
+
"source": [
|
|
132
|
+
"This errors at first."
|
|
133
|
+
]
|
|
134
|
+
},
|
|
137
135
|
{
|
|
138
136
|
"cell_type": "code",
|
|
139
137
|
"execution_count": null,
|
|
@@ -149,6 +147,14 @@
|
|
|
149
147
|
")"
|
|
150
148
|
]
|
|
151
149
|
},
|
|
150
|
+
{
|
|
151
|
+
"cell_type": "markdown",
|
|
152
|
+
"id": "845d9c9c",
|
|
153
|
+
"metadata": {},
|
|
154
|
+
"source": [
|
|
155
|
+
"Register the instance on the hub."
|
|
156
|
+
]
|
|
157
|
+
},
|
|
152
158
|
{
|
|
153
159
|
"cell_type": "code",
|
|
154
160
|
"execution_count": null,
|
|
@@ -162,11 +168,12 @@
|
|
|
162
168
|
{
|
|
163
169
|
"cell_type": "code",
|
|
164
170
|
"execution_count": null,
|
|
165
|
-
"id": "
|
|
171
|
+
"id": "1660bb27",
|
|
166
172
|
"metadata": {},
|
|
167
173
|
"outputs": [],
|
|
168
174
|
"source": [
|
|
169
|
-
"ln_setup.settings.instance.is_on_hub"
|
|
175
|
+
"assert ln_setup.settings.instance.is_on_hub\n",
|
|
176
|
+
"assert not ln_setup.settings.instance.is_managed_by_hub"
|
|
170
177
|
]
|
|
171
178
|
},
|
|
172
179
|
{
|
|
@@ -417,7 +424,7 @@
|
|
|
417
424
|
"outputs": [],
|
|
418
425
|
"source": [
|
|
419
426
|
"from lamindb_setup.core._hub_client import connect_hub_with_auth\n",
|
|
420
|
-
"from
|
|
427
|
+
"from lamincentral.client import SupabaseClientWrapper\n",
|
|
421
428
|
"from laminhub_rest.core.instance_collaborator import InstanceCollaboratorHandler\n",
|
|
422
429
|
"from laminhub_rest.core.organization import OrganizationMemberHandler\n",
|
|
423
430
|
"\n",
|
|
@@ -432,11 +439,16 @@
|
|
|
432
439
|
"account_id = ln_setup.settings.user._uuid\n",
|
|
433
440
|
"\n",
|
|
434
441
|
"try:\n",
|
|
435
|
-
"
|
|
436
|
-
"
|
|
437
|
-
"
|
|
438
|
-
"
|
|
439
|
-
"
|
|
442
|
+
" try:\n",
|
|
443
|
+
" organization_member_handler.add(\n",
|
|
444
|
+
" organization_id=organization_id,\n",
|
|
445
|
+
" account_id=account_id,\n",
|
|
446
|
+
" role=\"member\",\n",
|
|
447
|
+
" )\n",
|
|
448
|
+
" except KeyError:\n",
|
|
449
|
+
" # we don't set LAMIN_API_KEY, so broadcasting cache invalidation fails\n",
|
|
450
|
+
" # it should still be fine with adding to the organization\n",
|
|
451
|
+
" pass\n",
|
|
440
452
|
" try:\n",
|
|
441
453
|
" InstanceCollaboratorHandler(admin_hub).add(\n",
|
|
442
454
|
" account_id=account_id,\n",
|
|
@@ -456,10 +468,15 @@
|
|
|
456
468
|
" ln_setup.delete(\"testuser1/test-add-managed-storage\", force=True)\n",
|
|
457
469
|
"\n",
|
|
458
470
|
"finally:\n",
|
|
459
|
-
"
|
|
460
|
-
"
|
|
461
|
-
"
|
|
462
|
-
"
|
|
471
|
+
" try:\n",
|
|
472
|
+
" organization_member_handler.remove(\n",
|
|
473
|
+
" organization_id=organization_id,\n",
|
|
474
|
+
" account_id=account_id,\n",
|
|
475
|
+
" )\n",
|
|
476
|
+
" except KeyError:\n",
|
|
477
|
+
" # we don't set LAMIN_API_KEY, so broadcasting cache invalidation fails\n",
|
|
478
|
+
" # it should still be fine with deletion from the organization\n",
|
|
479
|
+
" pass\n",
|
|
463
480
|
" admin_hub.auth.sign_out(options={\"scope\": \"local\"})"
|
|
464
481
|
]
|
|
465
482
|
},
|
|
@@ -525,7 +542,7 @@
|
|
|
525
542
|
"name": "python",
|
|
526
543
|
"nbconvert_exporter": "python",
|
|
527
544
|
"pygments_lexer": "ipython3",
|
|
528
|
-
"version": "3.
|
|
545
|
+
"version": "3.11.14"
|
|
529
546
|
}
|
|
530
547
|
},
|
|
531
548
|
"nbformat": 4,
|
|
@@ -95,7 +95,7 @@
|
|
|
95
95
|
"assert ln_setup.settings.instance.owner == ln_setup.settings.user.handle\n",
|
|
96
96
|
"assert ln_setup.settings.instance.name == \"my-hosted\"\n",
|
|
97
97
|
"assert ln_setup.settings.storage.root.as_posix().startswith(HOSTED_BUCKETS)\n",
|
|
98
|
-
"assert ln_setup.settings.storage.
|
|
98
|
+
"assert ln_setup.settings.storage._id is not None\n",
|
|
99
99
|
"\n",
|
|
100
100
|
"assert ln_setup.settings.storage._mark_storage_root.exists()"
|
|
101
101
|
]
|
|
@@ -49,7 +49,7 @@
|
|
|
49
49
|
"metadata": {},
|
|
50
50
|
"outputs": [],
|
|
51
51
|
"source": [
|
|
52
|
-
"import wetlab as
|
|
52
|
+
"import wetlab as wl"
|
|
53
53
|
]
|
|
54
54
|
},
|
|
55
55
|
{
|
|
@@ -69,7 +69,7 @@
|
|
|
69
69
|
"metadata": {},
|
|
70
70
|
"outputs": [],
|
|
71
71
|
"source": [
|
|
72
|
-
"
|
|
72
|
+
"wl.Compound"
|
|
73
73
|
]
|
|
74
74
|
},
|
|
75
75
|
{
|
|
@@ -22,7 +22,7 @@ from .core._settings_storage import StorageSettings
|
|
|
22
22
|
from .core._settings_store import instance_settings_file
|
|
23
23
|
from .core.cloud_sqlite_locker import unlock_cloud_sqlite_upon_exception
|
|
24
24
|
from .core.django import reset_django
|
|
25
|
-
from .errors import CannotSwitchDefaultInstance
|
|
25
|
+
from .errors import CannotSwitchDefaultInstance, InstanceNotFoundError
|
|
26
26
|
|
|
27
27
|
if TYPE_CHECKING:
|
|
28
28
|
from pathlib import Path
|
|
@@ -33,8 +33,6 @@ if TYPE_CHECKING:
|
|
|
33
33
|
# this is for testing purposes only
|
|
34
34
|
# set to True only to test failed load
|
|
35
35
|
_TEST_FAILED_LOAD = False
|
|
36
|
-
|
|
37
|
-
|
|
38
36
|
INSTANCE_NOT_FOUND_MESSAGE = (
|
|
39
37
|
"'{owner}/{name}' not found:"
|
|
40
38
|
" '{hub_result}'\nCheck your permissions:"
|
|
@@ -42,10 +40,6 @@ INSTANCE_NOT_FOUND_MESSAGE = (
|
|
|
42
40
|
)
|
|
43
41
|
|
|
44
42
|
|
|
45
|
-
class InstanceNotFoundError(SystemExit):
|
|
46
|
-
pass
|
|
47
|
-
|
|
48
|
-
|
|
49
43
|
def check_db_dsn_equal_up_to_credentials(db_dsn_hub, db_dsn_local):
|
|
50
44
|
return (
|
|
51
45
|
db_dsn_hub.scheme == db_dsn_local.scheme
|
|
@@ -102,6 +96,7 @@ def _connect_instance(
|
|
|
102
96
|
use_root_db_user: bool = False,
|
|
103
97
|
use_proxy_db: bool = False,
|
|
104
98
|
access_token: str | None = None,
|
|
99
|
+
raise_systemexit: bool = False,
|
|
105
100
|
) -> InstanceSettings:
|
|
106
101
|
settings_file = instance_settings_file(name, owner)
|
|
107
102
|
make_hub_request = True
|
|
@@ -170,12 +165,17 @@ def _connect_instance(
|
|
|
170
165
|
)
|
|
171
166
|
else:
|
|
172
167
|
message = "It is not possible to load an anonymous-owned instance from the hub"
|
|
168
|
+
exception = (
|
|
169
|
+
SystemExit(message)
|
|
170
|
+
if raise_systemexit
|
|
171
|
+
else InstanceNotFoundError(message)
|
|
172
|
+
)
|
|
173
173
|
if settings_file.exists():
|
|
174
174
|
isettings = load_instance_settings(settings_file)
|
|
175
175
|
if isettings.is_remote:
|
|
176
|
-
raise
|
|
176
|
+
raise exception
|
|
177
177
|
else:
|
|
178
|
-
raise
|
|
178
|
+
raise exception
|
|
179
179
|
return isettings
|
|
180
180
|
|
|
181
181
|
|
|
@@ -239,7 +239,11 @@ def _connect_cli(
|
|
|
239
239
|
|
|
240
240
|
owner, name = get_owner_name_from_identifier(instance)
|
|
241
241
|
isettings = _connect_instance(
|
|
242
|
-
owner,
|
|
242
|
+
owner,
|
|
243
|
+
name,
|
|
244
|
+
use_root_db_user=use_root_db_user,
|
|
245
|
+
use_proxy_db=use_proxy_db,
|
|
246
|
+
raise_systemexit=True,
|
|
243
247
|
)
|
|
244
248
|
isettings._persist(write_to_disk=True)
|
|
245
249
|
if not isettings.is_on_hub or isettings._is_cloud_sqlite:
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import os
|
|
4
|
+
|
|
3
5
|
import httpx
|
|
4
6
|
from django.db import connection
|
|
5
7
|
from django.db.migrations.loader import MigrationLoader
|
|
@@ -98,16 +100,19 @@ class migrate:
|
|
|
98
100
|
|
|
99
101
|
@classmethod
|
|
100
102
|
def deploy(cls, package_name: str | None = None, number: int | None = None) -> None:
|
|
101
|
-
|
|
103
|
+
assert settings._instance_exists, (
|
|
104
|
+
"Not connected to an instance, please connect to migrate."
|
|
105
|
+
)
|
|
102
106
|
|
|
103
107
|
# NOTE: this is a temporary solution to avoid breaking tests
|
|
104
108
|
LAMIN_MIGRATE_ON_LAMBDA = (
|
|
105
|
-
os.
|
|
109
|
+
os.getenv("LAMIN_MIGRATE_ON_LAMBDA", "false") == "true"
|
|
106
110
|
)
|
|
111
|
+
isettings = settings.instance
|
|
107
112
|
|
|
108
|
-
if
|
|
113
|
+
if isettings.is_on_hub and LAMIN_MIGRATE_ON_LAMBDA:
|
|
109
114
|
response = httpx.post(
|
|
110
|
-
f"{
|
|
115
|
+
f"{isettings.api_url}/instances/{isettings._id}/migrate",
|
|
111
116
|
headers={"Authorization": f"Bearer {settings.user.access_token}"},
|
|
112
117
|
timeout=None, # this can take time
|
|
113
118
|
)
|
|
@@ -128,32 +133,38 @@ class migrate:
|
|
|
128
133
|
update_instance,
|
|
129
134
|
)
|
|
130
135
|
|
|
131
|
-
|
|
136
|
+
isettings = settings.instance
|
|
137
|
+
is_managed_by_hub = isettings.is_managed_by_hub
|
|
138
|
+
is_on_hub = is_managed_by_hub or isettings.is_on_hub
|
|
139
|
+
|
|
140
|
+
if is_managed_by_hub and "root" not in isettings.db:
|
|
132
141
|
# ensure we connect with the root user
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
142
|
+
connect(use_root_db_user=True)
|
|
143
|
+
assert "root" in (instance_db := settings.instance.db), instance_db
|
|
144
|
+
if is_on_hub:
|
|
136
145
|
# we need lamindb to be installed, otherwise we can't populate the version
|
|
137
146
|
# information in the hub
|
|
147
|
+
# this also connects
|
|
138
148
|
import lamindb
|
|
139
|
-
|
|
149
|
+
# this is needed to avoid connecting on importing apps inside setup_django process
|
|
150
|
+
setup_django_disable_autoconnect = disable_auto_connect(setup_django)
|
|
140
151
|
# this sets up django and deploys the migrations
|
|
141
152
|
if package_name is not None and number is not None:
|
|
142
|
-
|
|
143
|
-
|
|
153
|
+
setup_django_disable_autoconnect(
|
|
154
|
+
isettings,
|
|
144
155
|
deploy_migrations=True,
|
|
145
156
|
appname_number=(package_name, number),
|
|
146
157
|
)
|
|
147
158
|
else:
|
|
148
|
-
|
|
159
|
+
setup_django_disable_autoconnect(isettings, deploy_migrations=True)
|
|
149
160
|
# this populates the hub
|
|
150
|
-
if
|
|
161
|
+
if is_on_hub:
|
|
151
162
|
logger.important(f"updating lamindb version in hub: {lamindb.__version__}")
|
|
152
|
-
if
|
|
163
|
+
if isettings.dialect != "sqlite":
|
|
153
164
|
update_schema_in_hub()
|
|
154
165
|
call_with_fallback_auth(
|
|
155
166
|
update_instance,
|
|
156
|
-
instance_id=
|
|
167
|
+
instance_id=isettings._id.hex,
|
|
157
168
|
instance_fields={"lamindb_version": lamindb.__version__},
|
|
158
169
|
)
|
|
159
170
|
|
|
@@ -161,16 +172,45 @@ class migrate:
|
|
|
161
172
|
@disable_auto_connect
|
|
162
173
|
def check(cls) -> bool:
|
|
163
174
|
"""Check whether Registry definitions are in sync with migrations."""
|
|
175
|
+
import io
|
|
176
|
+
|
|
164
177
|
from django.core.management import call_command
|
|
165
178
|
|
|
166
179
|
setup_django(settings.instance)
|
|
180
|
+
|
|
181
|
+
# Capture stdout/stderr to show what migrations are needed if check fails
|
|
182
|
+
stdout = io.StringIO()
|
|
183
|
+
stderr = io.StringIO()
|
|
184
|
+
|
|
167
185
|
try:
|
|
168
|
-
call_command(
|
|
186
|
+
call_command(
|
|
187
|
+
"makemigrations", check_changes=True, stdout=stdout, stderr=stderr
|
|
188
|
+
)
|
|
169
189
|
except SystemExit:
|
|
170
190
|
logger.error(
|
|
171
191
|
"migrations are not in sync with ORMs, please create a migration: lamin"
|
|
172
192
|
" migrate create"
|
|
173
193
|
)
|
|
194
|
+
# Print captured output from the check
|
|
195
|
+
if stdout.getvalue():
|
|
196
|
+
logger.error(f"makemigrations --check stdout:\n{stdout.getvalue()}")
|
|
197
|
+
if stderr.getvalue():
|
|
198
|
+
logger.error(f"makemigrations --check stderr:\n{stderr.getvalue()}")
|
|
199
|
+
|
|
200
|
+
# Run makemigrations --dry-run to show what would be created
|
|
201
|
+
stdout2 = io.StringIO()
|
|
202
|
+
stderr2 = io.StringIO()
|
|
203
|
+
try:
|
|
204
|
+
call_command(
|
|
205
|
+
"makemigrations", dry_run=True, stdout=stdout2, stderr=stderr2
|
|
206
|
+
)
|
|
207
|
+
except SystemExit:
|
|
208
|
+
pass
|
|
209
|
+
if stdout2.getvalue():
|
|
210
|
+
logger.error(f"makemigrations --dry-run stdout:\n{stdout2.getvalue()}")
|
|
211
|
+
if stderr2.getvalue():
|
|
212
|
+
logger.error(f"makemigrations --dry-run stderr:\n{stderr2.getvalue()}")
|
|
213
|
+
|
|
174
214
|
return False
|
|
175
215
|
return True
|
|
176
216
|
|
|
@@ -16,6 +16,9 @@ if TYPE_CHECKING:
|
|
|
16
16
|
def set_managed_storage(root: UPathStr, host: str | None = None, **fs_kwargs):
|
|
17
17
|
"""Add or switch to another managed storage location.
|
|
18
18
|
|
|
19
|
+
Note: This function should be called `set_writeable_storage_location` instead. But likely it will disappear
|
|
20
|
+
in refactoring that consolidates with the `ln.Storage()` path.
|
|
21
|
+
|
|
19
22
|
Args:
|
|
20
23
|
root: `UPathStr` - The new storage root, e.g., an S3 bucket.
|
|
21
24
|
host: `str | None = None` For a shared local storage location, pass a globally unique host identifier, e.g. `"my-institute-cluster-1"`, `"my-server-abcd"`, ...
|
|
@@ -20,7 +20,7 @@ lamin_env = os.getenv("LAMIN_ENV")
|
|
|
20
20
|
if lamin_env is None or lamin_env == "prod":
|
|
21
21
|
HOSTED_BUCKETS = tuple([f"s3://lamin-{region}" for region in HOSTED_REGIONS])
|
|
22
22
|
else:
|
|
23
|
-
logger.warning("loaded LAMIN_ENV:
|
|
23
|
+
logger.warning(f"loaded LAMIN_ENV: {lamin_env}")
|
|
24
24
|
HOSTED_BUCKETS = ("s3://lamin-hosted-test",) # type: ignore
|
|
25
25
|
|
|
26
26
|
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""Utilities to work with Postgres Snapshots.
|
|
2
|
+
|
|
3
|
+
.. autosummary::
|
|
4
|
+
:toctree:
|
|
5
|
+
|
|
6
|
+
upload_sqlite_clone
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import gzip
|
|
10
|
+
import shutil
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from lamindb_setup.core.upath import create_path
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def upload_sqlite_clone(
|
|
17
|
+
local_sqlite_path: Path | str | None = None, compress: bool = True
|
|
18
|
+
) -> None:
|
|
19
|
+
"""Uploads the SQLite clone to the default storage.
|
|
20
|
+
|
|
21
|
+
Args:
|
|
22
|
+
local_sqlite_path: Path to the SQLite file.
|
|
23
|
+
Defaults to the local storage path if not specified.
|
|
24
|
+
compress: Whether to compress the database with gzip before uploading.
|
|
25
|
+
"""
|
|
26
|
+
import lamindb_setup as ln_setup
|
|
27
|
+
|
|
28
|
+
if local_sqlite_path is None:
|
|
29
|
+
local_sqlite_path = ln_setup.settings.instance._sqlite_file_local
|
|
30
|
+
else:
|
|
31
|
+
local_sqlite_path = Path(local_sqlite_path)
|
|
32
|
+
|
|
33
|
+
if not local_sqlite_path.exists():
|
|
34
|
+
raise FileNotFoundError(f"Database not found at {local_sqlite_path}")
|
|
35
|
+
|
|
36
|
+
cloud_db_path = ln_setup.settings.instance._sqlite_file
|
|
37
|
+
|
|
38
|
+
if compress:
|
|
39
|
+
temp_gz_path = local_sqlite_path.with_suffix(".db.gz")
|
|
40
|
+
with (
|
|
41
|
+
open(local_sqlite_path, "rb") as f_in,
|
|
42
|
+
gzip.open(temp_gz_path, "wb") as f_out,
|
|
43
|
+
):
|
|
44
|
+
shutil.copyfileobj(f_in, f_out)
|
|
45
|
+
cloud_destination = create_path(f"{cloud_db_path}.gz")
|
|
46
|
+
cloud_destination.upload_from(temp_gz_path, print_progress=True)
|
|
47
|
+
temp_gz_path.unlink()
|
|
48
|
+
else:
|
|
49
|
+
cloud_destination = create_path(cloud_db_path)
|
|
50
|
+
cloud_destination.upload_from(local_sqlite_path, print_progress=True)
|
|
@@ -34,8 +34,8 @@ def private_django_api(reverse=False):
|
|
|
34
34
|
# the order here matters
|
|
35
35
|
# changing it might break the tests
|
|
36
36
|
attributes = [
|
|
37
|
-
"MultipleObjectsReturned",
|
|
38
37
|
"add_to_class",
|
|
38
|
+
"arefresh_from_db",
|
|
39
39
|
"adelete",
|
|
40
40
|
"asave",
|
|
41
41
|
"clean",
|
|
@@ -51,10 +51,8 @@ def private_django_api(reverse=False):
|
|
|
51
51
|
"validate_unique",
|
|
52
52
|
]
|
|
53
53
|
if reverse:
|
|
54
|
-
attributes.append("arefresh_from_db")
|
|
55
54
|
attributes.append("full_clean")
|
|
56
55
|
else:
|
|
57
|
-
attributes.append("a_refresh_from_db")
|
|
58
56
|
attributes.append("full__clean")
|
|
59
57
|
|
|
60
58
|
django_path = Path(db.__file__).parent.parent
|
|
@@ -323,16 +323,6 @@ class InstanceSettings:
|
|
|
323
323
|
else:
|
|
324
324
|
logger.warning(f"could not set this local storage location: {local_root}")
|
|
325
325
|
|
|
326
|
-
@property
|
|
327
|
-
@deprecated("local_storage")
|
|
328
|
-
def storage_local(self) -> StorageSettings:
|
|
329
|
-
return self.local_storage
|
|
330
|
-
|
|
331
|
-
@storage_local.setter
|
|
332
|
-
@deprecated("local_storage")
|
|
333
|
-
def storage_local(self, local_root_host: tuple[Path | str, str]):
|
|
334
|
-
self.local_storage = local_root_host # type: ignore
|
|
335
|
-
|
|
336
326
|
@property
|
|
337
327
|
def slug(self) -> str:
|
|
338
328
|
"""Unique semantic identifier of form `"{account_handle}/{instance_name}"`."""
|
|
@@ -418,11 +408,6 @@ class InstanceSettings:
|
|
|
418
408
|
else:
|
|
419
409
|
return {module for module in self._schema_str.split(",") if module != ""}
|
|
420
410
|
|
|
421
|
-
@property
|
|
422
|
-
@deprecated("modules")
|
|
423
|
-
def schema(self) -> set[str]:
|
|
424
|
-
return self.modules
|
|
425
|
-
|
|
426
411
|
@property
|
|
427
412
|
def _sqlite_file(self) -> UPath:
|
|
428
413
|
"""SQLite file."""
|
|
@@ -569,10 +554,10 @@ class InstanceSettings:
|
|
|
569
554
|
|
|
570
555
|
@property
|
|
571
556
|
def is_on_hub(self) -> bool:
|
|
572
|
-
"""Is this instance on the hub?
|
|
557
|
+
"""Is this instance registered on the hub?
|
|
573
558
|
|
|
574
|
-
Can only
|
|
575
|
-
Will return `False` in case the
|
|
559
|
+
Can only establish if user has access to the instance.
|
|
560
|
+
Will return `False` in case the user token can't find the instance.
|
|
576
561
|
"""
|
|
577
562
|
if self._is_on_hub is None:
|
|
578
563
|
from ._hub_client import call_with_fallback_auth
|
|
@@ -594,6 +579,15 @@ class InstanceSettings:
|
|
|
594
579
|
self._is_on_hub = True
|
|
595
580
|
return self._is_on_hub
|
|
596
581
|
|
|
582
|
+
@property
|
|
583
|
+
def is_managed_by_hub(self) -> bool:
|
|
584
|
+
"""Is this instance managed by the hub?
|
|
585
|
+
|
|
586
|
+
Returns `True` if the instance is _managed_ by LaminHub, i.e.,
|
|
587
|
+
it was connected to LaminHub to manage access, migrations, a REST API, a UI, etc.
|
|
588
|
+
"""
|
|
589
|
+
return self.api_url is not None
|
|
590
|
+
|
|
597
591
|
def _get_settings_file(self) -> Path:
|
|
598
592
|
return instance_settings_file(self.name, self.owner)
|
|
599
593
|
|