lamindb_setup 0.81.3__tar.gz → 1.0a1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/.github/workflows/build.yml +1 -1
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/.gitignore +1 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/.pre-commit-config.yaml +1 -1
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/PKG-INFO +4 -5
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/hub-cloud/01-init-local-instance.ipynb +7 -4
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/hub-cloud/02-connect-local-instance.ipynb +1 -1
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/hub-cloud/03-add-managed-storage.ipynb +28 -10
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/hub-cloud/05-init-hosted-instance.ipynb +8 -2
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/hub-cloud/06-connect-hosted-instance.ipynb +1 -1
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/hub-cloud/07-keep-artifacts-local.ipynb +13 -9
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/hub-cloud/08-test-multi-session.ipynb +1 -1
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/hub-prod/test-cloud-sync.ipynb +123 -10
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/hub-prod/test-connect-anonymously.ipynb +1 -1
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/hub-prod/test-sqlite-lock.ipynb +4 -1
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/__init__.py +31 -2
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/_check_setup.py +15 -16
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/_connect_instance.py +35 -33
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/_delete.py +2 -2
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/_django.py +6 -6
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/_exportdb.py +1 -1
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/_init_instance.py +13 -39
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/_migrate.py +5 -3
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/_schema_metadata.py +10 -6
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/_aws_credentials.py +10 -2
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/_hub_core.py +6 -3
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/_hub_crud.py +32 -30
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/_settings.py +10 -5
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/_settings_instance.py +17 -3
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/_settings_storage.py +16 -15
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/_settings_user.py +1 -1
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/django.py +1 -3
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/hashing.py +16 -5
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/upath.py +108 -64
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/noxfile.py +4 -7
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/pyproject.toml +7 -8
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/hub-cloud/scripts/script-init-pass-user-no-writes.py +5 -2
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/hub-cloud/scripts/script-to-fail-managed-storage.py +1 -1
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/hub-cloud/test_init_instance.py +37 -12
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/hub-local/test_all.py +6 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/hub-local/test_update_schema_in_hub.py +5 -5
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/hub-prod/test_django.py +1 -1
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/storage/test_storage_stats.py +17 -8
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/.github/workflows/doc-changes.yml +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/LICENSE +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/README.md +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/changelog.md +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/hub-cloud/04-test-bionty.ipynb +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/hub-cloud/test_notebooks.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/hub-prod/test-cache-management.ipynb +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/hub-prod/test-empty-init.ipynb +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/hub-prod/test-import-schema.ipynb +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/hub-prod/test-init-load-local-anonymously.ipynb +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/hub-prod/test-insufficient-user-info.ipynb +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/hub-prod/test-invalid-schema.ipynb +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/hub-prod/test_notebooks2.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/index.md +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/notebooks.md +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/reference.md +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/_cache.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/_check.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/_close.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/_entry_points.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/_importdb.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/_register_instance.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/_schema.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/_set_managed_storage.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/_setup_user.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/_silence_loggers.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/__init__.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/_aws_storage.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/_deprecated.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/_docs.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/_hub_client.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/_hub_utils.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/_private_django_api.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/_settings_load.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/_settings_save.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/_settings_store.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/_setup_bionty_sources.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/cloud_sqlite_locker.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/exceptions.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/lamindb_setup/core/types.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/hub-cloud/test_connect_instance.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/hub-cloud/test_delete_instance.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/hub-cloud/test_edge_request.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/hub-cloud/test_fail_managed_storage.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/hub-cloud/test_init_pass_user_no_writes.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/hub-cloud/test_login.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/hub-cloud/test_migrate.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/hub-cloud/test_set_storage.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/hub-local/conftest.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/hub-prod/conftest.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/hub-prod/test_aws_credentials_manager.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/hub-prod/test_global_settings.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/hub-prod/test_switch_and_fallback_env.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/hub-prod/test_upath.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/storage/test_entry_point.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/storage/test_hashing.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/storage/test_storage_access.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/storage/test_storage_basis.py +0 -0
- {lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/tests/storage/test_to_url.py +0 -0
|
@@ -99,7 +99,7 @@ jobs:
|
|
|
99
99
|
- uses: actions/checkout@v4
|
|
100
100
|
- uses: actions/setup-python@v4
|
|
101
101
|
with:
|
|
102
|
-
python-version: "3.
|
|
102
|
+
python-version: "3.10"
|
|
103
103
|
cache: "pip"
|
|
104
104
|
cache-dependency-path: ".github/workflows/build.yml"
|
|
105
105
|
- run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
|
|
@@ -1,20 +1,19 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
2
|
Name: lamindb_setup
|
|
3
|
-
Version:
|
|
3
|
+
Version: 1.0a1
|
|
4
4
|
Summary: Setup & configure LaminDB.
|
|
5
5
|
Author-email: Lamin Labs <open-source@lamin.ai>
|
|
6
6
|
Requires-Python: >=3.9
|
|
7
7
|
Description-Content-Type: text/markdown
|
|
8
|
-
Requires-Dist: lnschema_core>=0.51.0
|
|
9
8
|
Requires-Dist: lamin_utils>=0.3.3
|
|
10
|
-
Requires-Dist: django
|
|
9
|
+
Requires-Dist: django>=5,<5.2
|
|
11
10
|
Requires-Dist: dj_database_url>=1.3.0,<3.0.0
|
|
12
11
|
Requires-Dist: pydantic-settings
|
|
13
12
|
Requires-Dist: appdirs<2.0.0
|
|
14
13
|
Requires-Dist: requests
|
|
15
14
|
Requires-Dist: universal_pathlib==0.2.5
|
|
16
15
|
Requires-Dist: botocore<2.0.0
|
|
17
|
-
Requires-Dist: supabase>=2.8.1,<=2.
|
|
16
|
+
Requires-Dist: supabase>=2.8.1,<=2.11.0
|
|
18
17
|
Requires-Dist: psutil
|
|
19
18
|
Requires-Dist: urllib3<2 ; extra == "aws"
|
|
20
19
|
Requires-Dist: aiobotocore[boto3]>=2.5.4,<3.0.0 ; extra == "aws"
|
|
@@ -48,9 +48,9 @@
|
|
|
48
48
|
"outputs": [],
|
|
49
49
|
"source": [
|
|
50
50
|
"from pathlib import Path\n",
|
|
51
|
-
"from
|
|
51
|
+
"from lamindb.models import Storage\n",
|
|
52
52
|
"\n",
|
|
53
|
-
"assert ln_setup.settings.instance.storage.type_is_cloud
|
|
53
|
+
"assert ln_setup.settings.instance.storage.type_is_cloud is False\n",
|
|
54
54
|
"assert ln_setup.settings.instance.owner == ln_setup.settings.user.handle\n",
|
|
55
55
|
"assert ln_setup.settings.instance.name == \"mydata\"\n",
|
|
56
56
|
"assert ln_setup.settings.storage.root.as_posix() == Path(\"mydata\").resolve().as_posix()\n",
|
|
@@ -62,7 +62,10 @@
|
|
|
62
62
|
" == f\"sqlite:///{Path('./mydata').resolve().as_posix()}/{ln_setup.settings.instance._id.hex}.lndb\"\n",
|
|
63
63
|
")\n",
|
|
64
64
|
"assert ln_setup.settings.storage._instance_id == ln_setup.settings.instance._id\n",
|
|
65
|
-
"assert
|
|
65
|
+
"assert (\n",
|
|
66
|
+
" Storage.objects.get(instance_uid=ln_setup.settings.instance.uid).root\n",
|
|
67
|
+
" == ln_setup.settings.storage.root_as_str\n",
|
|
68
|
+
")"
|
|
66
69
|
]
|
|
67
70
|
},
|
|
68
71
|
{
|
|
@@ -84,7 +87,7 @@
|
|
|
84
87
|
"\n",
|
|
85
88
|
"settings_file = instance_settings_file(\"mydata\", \"testuser1\")\n",
|
|
86
89
|
"assert not storage_root.exists()\n",
|
|
87
|
-
"assert settings_file.exists()
|
|
90
|
+
"assert settings_file.exists() is False"
|
|
88
91
|
]
|
|
89
92
|
}
|
|
90
93
|
],
|
|
@@ -74,7 +74,7 @@
|
|
|
74
74
|
"source": [
|
|
75
75
|
"from pathlib import Path\n",
|
|
76
76
|
"\n",
|
|
77
|
-
"assert ln_setup.settings.instance.storage.type_is_cloud
|
|
77
|
+
"assert ln_setup.settings.instance.storage.type_is_cloud is False\n",
|
|
78
78
|
"assert ln_setup.settings.instance.name == \"mydata\"\n",
|
|
79
79
|
"\n",
|
|
80
80
|
"root_path = Path(\"./mydata\").resolve()\n",
|
|
@@ -85,7 +85,10 @@
|
|
|
85
85
|
"source": [
|
|
86
86
|
"with pytest.raises(ValueError) as error:\n",
|
|
87
87
|
" set_managed_storage(\"./storage2\")\n",
|
|
88
|
-
"assert
|
|
88
|
+
"assert (\n",
|
|
89
|
+
" error.exconly()\n",
|
|
90
|
+
" == \"ValueError: Can't add additional managed storage locations for instances that aren't managed through the hub.\"\n",
|
|
91
|
+
")"
|
|
89
92
|
]
|
|
90
93
|
},
|
|
91
94
|
{
|
|
@@ -129,7 +132,9 @@
|
|
|
129
132
|
"source": [
|
|
130
133
|
"storage2_uid = ln_setup.settings.storage.uid\n",
|
|
131
134
|
"assert ln_setup.settings.storage.root_as_str == f\"{Path.cwd()}/storage2\"\n",
|
|
132
|
-
"assert (
|
|
135
|
+
"assert (\n",
|
|
136
|
+
" ln_setup.settings.storage.root / \".lamindb/_is_initialized\"\n",
|
|
137
|
+
").read_text() == ln_setup.settings.storage.uid\n",
|
|
133
138
|
"assert ln_setup.settings.storage.is_on_hub"
|
|
134
139
|
]
|
|
135
140
|
},
|
|
@@ -159,7 +164,9 @@
|
|
|
159
164
|
"outputs": [],
|
|
160
165
|
"source": [
|
|
161
166
|
"assert ln_setup.settings.storage.root_as_str == f\"{Path.cwd()}/storage1\"\n",
|
|
162
|
-
"assert (
|
|
167
|
+
"assert (\n",
|
|
168
|
+
" ln_setup.settings.storage.root / \".lamindb/_is_initialized\"\n",
|
|
169
|
+
").read_text() == ln_setup.settings.storage.uid\n",
|
|
163
170
|
"assert ln_setup.settings.storage.is_on_hub\n",
|
|
164
171
|
"assert ln_setup.settings.storage.uid == storage1_uid"
|
|
165
172
|
]
|
|
@@ -190,7 +197,9 @@
|
|
|
190
197
|
"outputs": [],
|
|
191
198
|
"source": [
|
|
192
199
|
"assert ln_setup.settings.storage.root_as_str == f\"{Path.cwd()}/storage1\"\n",
|
|
193
|
-
"assert (
|
|
200
|
+
"assert (\n",
|
|
201
|
+
" ln_setup.settings.storage.root / \".lamindb/_is_initialized\"\n",
|
|
202
|
+
").read_text() == ln_setup.settings.storage.uid\n",
|
|
194
203
|
"assert ln_setup.settings.storage.is_on_hub\n",
|
|
195
204
|
"assert ln_setup.settings.storage.uid == storage1_uid"
|
|
196
205
|
]
|
|
@@ -214,7 +223,9 @@
|
|
|
214
223
|
"source": [
|
|
215
224
|
"storage2_uid = ln_setup.settings.storage.uid\n",
|
|
216
225
|
"assert ln_setup.settings.storage.root_as_str == f\"{Path.cwd()}/storage2\"\n",
|
|
217
|
-
"assert (
|
|
226
|
+
"assert (\n",
|
|
227
|
+
" ln_setup.settings.storage.root / \".lamindb/_is_initialized\"\n",
|
|
228
|
+
").read_text() == ln_setup.settings.storage.uid\n",
|
|
218
229
|
"assert ln_setup.settings.storage.is_on_hub\n",
|
|
219
230
|
"assert ln_setup.settings.storage.uid == storage2_uid"
|
|
220
231
|
]
|
|
@@ -251,7 +262,9 @@
|
|
|
251
262
|
"assert ln_setup.settings.storage.type_is_cloud\n",
|
|
252
263
|
"assert ln_setup.settings.storage.root_as_str == \"s3://lamindb-ci/storage3\"\n",
|
|
253
264
|
"assert ln_setup.settings.storage.region == \"us-west-1\"\n",
|
|
254
|
-
"assert (
|
|
265
|
+
"assert (\n",
|
|
266
|
+
" ln_setup.settings.storage.root / \".lamindb/_is_initialized\"\n",
|
|
267
|
+
").read_text() == ln_setup.settings.storage.uid\n",
|
|
255
268
|
"# root.fs contains the underlying fsspec filesystem\n",
|
|
256
269
|
"assert (\n",
|
|
257
270
|
" ln_setup.settings.storage.root.fs.cache_regions # set by lamindb to True for s3 by default\n",
|
|
@@ -305,7 +318,9 @@
|
|
|
305
318
|
"source": [
|
|
306
319
|
"with pytest.raises(ValueError) as error:\n",
|
|
307
320
|
" set_managed_storage(\"gs://rxrx1-europe-west4/images/test/HEPG2-08\")\n",
|
|
308
|
-
"assert error.exconly().startswith(\"
|
|
321
|
+
"assert error.exconly().startswith(\n",
|
|
322
|
+
" \"ValueError: Cannot manage storage without write access\"\n",
|
|
323
|
+
")"
|
|
309
324
|
]
|
|
310
325
|
},
|
|
311
326
|
{
|
|
@@ -326,7 +341,7 @@
|
|
|
326
341
|
"from laminhub_rest.core.instance.collaborator import InstanceCollaboratorHandler\n",
|
|
327
342
|
"from laminhub_rest.core.account.user import UserAccountHandler\n",
|
|
328
343
|
"from lamindb_setup.core._hub_client import connect_hub_with_auth\n",
|
|
329
|
-
"from
|
|
344
|
+
"from lamindb.models import User\n",
|
|
330
345
|
"\n",
|
|
331
346
|
"admin_hub = connect_hub_with_auth()\n",
|
|
332
347
|
"testuser2 = UserAccountHandler(admin_hub).get_by_handle(\"testuser2\")\n",
|
|
@@ -335,7 +350,7 @@
|
|
|
335
350
|
" account_id=testuser2.id,\n",
|
|
336
351
|
" role=\"write\",\n",
|
|
337
352
|
" schema_id=None,\n",
|
|
338
|
-
" skip_insert_user_table=True
|
|
353
|
+
" skip_insert_user_table=True,\n",
|
|
339
354
|
")\n",
|
|
340
355
|
"User.objects.create(uid=testuser2.lnid, handle=testuser2.handle, name=testuser2.name)\n",
|
|
341
356
|
"admin_hub.auth.close()"
|
|
@@ -418,7 +433,10 @@
|
|
|
418
433
|
"from lamindb_setup.core._hub_client import call_with_fallback_auth\n",
|
|
419
434
|
"from lamindb_setup.core._hub_crud import select_instance_by_id\n",
|
|
420
435
|
"from lamindb_setup.core._hub_core import get_storage_records_for_instance\n",
|
|
421
|
-
"
|
|
436
|
+
"\n",
|
|
437
|
+
"assert (\n",
|
|
438
|
+
" call_with_fallback_auth(select_instance_by_id, instance_id=instance_id.hex) is None\n",
|
|
439
|
+
")\n",
|
|
422
440
|
"assert not get_storage_records_for_instance(instance_id)"
|
|
423
441
|
]
|
|
424
442
|
}
|
|
@@ -91,7 +91,7 @@
|
|
|
91
91
|
"metadata": {},
|
|
92
92
|
"outputs": [],
|
|
93
93
|
"source": [
|
|
94
|
-
"assert ln_setup.settings.instance.storage.type_is_cloud
|
|
94
|
+
"assert ln_setup.settings.instance.storage.type_is_cloud is True\n",
|
|
95
95
|
"assert ln_setup.settings.instance.owner == ln_setup.settings.user.handle\n",
|
|
96
96
|
"assert ln_setup.settings.instance.name == \"my-hosted\"\n",
|
|
97
97
|
"assert ln_setup.settings.storage.root.as_posix().startswith(HOSTED_BUCKETS)\n",
|
|
@@ -114,7 +114,13 @@
|
|
|
114
114
|
"outputs": [],
|
|
115
115
|
"source": [
|
|
116
116
|
"hub = connect_hub_with_auth()\n",
|
|
117
|
-
"response =
|
|
117
|
+
"response = (\n",
|
|
118
|
+
" hub.table(\"storage\")\n",
|
|
119
|
+
" .select(\"*\")\n",
|
|
120
|
+
" .eq(\"root\", ln_setup.settings.storage.root.as_posix())\n",
|
|
121
|
+
" .execute()\n",
|
|
122
|
+
" .data\n",
|
|
123
|
+
")\n",
|
|
118
124
|
"assert len(response) == 1\n",
|
|
119
125
|
"assert response[0][\"is_default\"]"
|
|
120
126
|
]
|
{lamindb_setup-0.81.3 → lamindb_setup-1.0a1}/docs/hub-cloud/06-connect-hosted-instance.ipynb
RENAMED
|
@@ -87,7 +87,7 @@
|
|
|
87
87
|
"outputs": [],
|
|
88
88
|
"source": [
|
|
89
89
|
"target_dir = root / \"test-dir-upload\"\n",
|
|
90
|
-
"target_dir.upload_from(test_dir, create_folder=True)
|
|
90
|
+
"target_dir.upload_from(test_dir, create_folder=True) # default\n",
|
|
91
91
|
"\n",
|
|
92
92
|
"assert target_dir.is_dir()\n",
|
|
93
93
|
"assert (target_dir / \"test-dir-upload\").exists()\n",
|
|
@@ -29,13 +29,10 @@
|
|
|
29
29
|
"\n",
|
|
30
30
|
"assert ln_setup.settings.instance.name == name\n",
|
|
31
31
|
"assert ln_setup.settings.instance.storage.type_is_cloud\n",
|
|
32
|
-
"assert
|
|
33
|
-
" ln_setup.settings.instance.storage.root_as_str\n",
|
|
34
|
-
" == storage\n",
|
|
35
|
-
")\n",
|
|
32
|
+
"assert ln_setup.settings.instance.storage.root_as_str == storage\n",
|
|
36
33
|
"assert (\n",
|
|
37
34
|
" ln_setup.settings.instance._sqlite_file.as_posix()\n",
|
|
38
|
-
" == f\"{storage}/{ln_setup.settings.instance._id.hex}.lndb\"
|
|
35
|
+
" == f\"{storage}/{ln_setup.settings.instance._id.hex}.lndb\"\n",
|
|
39
36
|
")"
|
|
40
37
|
]
|
|
41
38
|
},
|
|
@@ -47,7 +44,10 @@
|
|
|
47
44
|
"source": [
|
|
48
45
|
"with pytest.raises(ValueError) as error:\n",
|
|
49
46
|
" ln_setup.settings.instance.storage_local\n",
|
|
50
|
-
"assert
|
|
47
|
+
"assert (\n",
|
|
48
|
+
" error.exconly()\n",
|
|
49
|
+
" == \"ValueError: `keep_artifacts_local` is not enabled for this instance.\"\n",
|
|
50
|
+
")"
|
|
51
51
|
]
|
|
52
52
|
},
|
|
53
53
|
{
|
|
@@ -74,7 +74,9 @@
|
|
|
74
74
|
" ln_setup.settings.instance.storage_local.root.as_posix()\n",
|
|
75
75
|
" == UPath(\"./my_storage_local\").resolve().as_posix()\n",
|
|
76
76
|
")\n",
|
|
77
|
-
"assert (
|
|
77
|
+
"assert (\n",
|
|
78
|
+
" ln_setup.settings.instance.storage_local.root / \".lamindb/_is_initialized\"\n",
|
|
79
|
+
").read_text() == ln_setup.settings.instance.storage_local.uid\n",
|
|
78
80
|
"assert ln_setup.settings.instance.storage_local is not None\n",
|
|
79
81
|
"# the remote storage location is still in the regular slot\n",
|
|
80
82
|
"assert ln_setup.settings.instance.storage.root.as_posix() == storage"
|
|
@@ -106,7 +108,9 @@
|
|
|
106
108
|
" ln_setup.settings.instance.storage_local.root.as_posix()\n",
|
|
107
109
|
" == UPath(\"./my_storage_local2\").resolve().as_posix()\n",
|
|
108
110
|
")\n",
|
|
109
|
-
"assert (
|
|
111
|
+
"assert (\n",
|
|
112
|
+
" ln_setup.settings.instance.storage_local.root / \".lamindb/_is_initialized\"\n",
|
|
113
|
+
").read_text() == ln_setup.settings.instance.storage_local.uid"
|
|
110
114
|
]
|
|
111
115
|
},
|
|
112
116
|
{
|
|
@@ -154,7 +158,7 @@
|
|
|
154
158
|
"metadata": {},
|
|
155
159
|
"outputs": [],
|
|
156
160
|
"source": [
|
|
157
|
-
"test_file =
|
|
161
|
+
"test_file = ln_setup.settings.instance.storage_local.root / \".lamindb/test_file.txt\"\n",
|
|
158
162
|
"test_file.write_text(\"test\")"
|
|
159
163
|
]
|
|
160
164
|
},
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
"source": [
|
|
18
18
|
"import os\n",
|
|
19
19
|
"\n",
|
|
20
|
-
"instance_name =
|
|
20
|
+
"instance_name = \"test-sqlite-sync\"\n",
|
|
21
21
|
"!lamin connect {instance_name}\n",
|
|
22
22
|
"!yes | lamin delete {instance_name}"
|
|
23
23
|
]
|
|
@@ -33,7 +33,6 @@
|
|
|
33
33
|
"from lamindb_setup.core.upath import UPath, LocalPathClasses\n",
|
|
34
34
|
"import shutil\n",
|
|
35
35
|
"import time\n",
|
|
36
|
-
"import os\n",
|
|
37
36
|
"import pytest"
|
|
38
37
|
]
|
|
39
38
|
},
|
|
@@ -131,8 +130,14 @@
|
|
|
131
130
|
"source": [
|
|
132
131
|
"test_local_path = UPath(\"./some/local/path\")\n",
|
|
133
132
|
"assert settings.paths.cloud_to_local_no_update(test_local_path) == test_local_path\n",
|
|
134
|
-
"assert
|
|
135
|
-
"
|
|
133
|
+
"assert (\n",
|
|
134
|
+
" settings.paths.cloud_to_local_no_update(test_local_path.as_posix())\n",
|
|
135
|
+
" == test_local_path\n",
|
|
136
|
+
")\n",
|
|
137
|
+
"assert (\n",
|
|
138
|
+
" settings.paths.cloud_to_local_no_update(test_local_path, cache_key=\"some/cache/key\")\n",
|
|
139
|
+
" == test_local_path\n",
|
|
140
|
+
")"
|
|
136
141
|
]
|
|
137
142
|
},
|
|
138
143
|
{
|
|
@@ -142,11 +147,66 @@
|
|
|
142
147
|
"metadata": {},
|
|
143
148
|
"outputs": [],
|
|
144
149
|
"source": [
|
|
145
|
-
"assert
|
|
146
|
-
"
|
|
150
|
+
"assert (\n",
|
|
151
|
+
" settings.paths.cloud_to_local_no_update(dir_sync)\n",
|
|
152
|
+
" == settings.cache_dir / f\"lamindb-ci/{instance_name}/dir_sync\"\n",
|
|
153
|
+
")\n",
|
|
154
|
+
"assert (\n",
|
|
155
|
+
" settings.paths.cloud_to_local_no_update(dir_sync.as_posix())\n",
|
|
156
|
+
" == settings.cache_dir / f\"lamindb-ci/{instance_name}/dir_sync\"\n",
|
|
157
|
+
")\n",
|
|
158
|
+
"\n",
|
|
159
|
+
"assert (\n",
|
|
160
|
+
" settings.paths.cloud_to_local_no_update(dir_sync, cache_key=\"dir_cache/key\")\n",
|
|
161
|
+
" == settings.cache_dir / \"dir_cache/key\"\n",
|
|
162
|
+
")\n",
|
|
163
|
+
"assert (\n",
|
|
164
|
+
" settings.paths.cloud_to_local_no_update(\n",
|
|
165
|
+
" dir_sync.as_posix(), cache_key=\"dir_cache/key\"\n",
|
|
166
|
+
" )\n",
|
|
167
|
+
" == settings.cache_dir / \"dir_cache/key\"\n",
|
|
168
|
+
")"
|
|
169
|
+
]
|
|
170
|
+
},
|
|
171
|
+
{
|
|
172
|
+
"cell_type": "code",
|
|
173
|
+
"execution_count": null,
|
|
174
|
+
"id": "eda84820",
|
|
175
|
+
"metadata": {},
|
|
176
|
+
"outputs": [],
|
|
177
|
+
"source": [
|
|
178
|
+
"# for http urls\n",
|
|
179
|
+
"http_path = UPath(\n",
|
|
180
|
+
" \"https://raw.githubusercontent.com/laminlabs/lamindb-setup/refs/heads/main/README.md\"\n",
|
|
181
|
+
")\n",
|
|
182
|
+
"assert http_path.protocol == \"https\"\n",
|
|
147
183
|
"\n",
|
|
148
|
-
"
|
|
149
|
-
"assert
|
|
184
|
+
"http_stat = http_path.stat()\n",
|
|
185
|
+
"assert http_stat.st_size != 0\n",
|
|
186
|
+
"assert http_stat.st_mtime == 0\n",
|
|
187
|
+
"assert http_stat.as_info()[\"type\"] == \"file\""
|
|
188
|
+
]
|
|
189
|
+
},
|
|
190
|
+
{
|
|
191
|
+
"cell_type": "code",
|
|
192
|
+
"execution_count": null,
|
|
193
|
+
"id": "df6a9be4",
|
|
194
|
+
"metadata": {},
|
|
195
|
+
"outputs": [],
|
|
196
|
+
"source": [
|
|
197
|
+
"http_key = \"raw.githubusercontent.com/laminlabs/lamindb-setup/refs/heads/main/README.md\"\n",
|
|
198
|
+
"\n",
|
|
199
|
+
"assert (\n",
|
|
200
|
+
" settings.paths.cloud_to_local_no_update(http_path) == settings.cache_dir / http_key\n",
|
|
201
|
+
")\n",
|
|
202
|
+
"assert (\n",
|
|
203
|
+
" settings.paths.cloud_to_local_no_update(str(http_path))\n",
|
|
204
|
+
" == settings.cache_dir / http_key\n",
|
|
205
|
+
")\n",
|
|
206
|
+
"assert (\n",
|
|
207
|
+
" settings.paths.cloud_to_local_no_update(http_path, cache_key=\"check/README.md\")\n",
|
|
208
|
+
" == settings.cache_dir / \"check/README.md\"\n",
|
|
209
|
+
")"
|
|
150
210
|
]
|
|
151
211
|
},
|
|
152
212
|
{
|
|
@@ -164,7 +224,9 @@
|
|
|
164
224
|
"metadata": {},
|
|
165
225
|
"outputs": [],
|
|
166
226
|
"source": [
|
|
167
|
-
"dir_sync_local = settings.paths.cloud_to_local(
|
|
227
|
+
"dir_sync_local = settings.paths.cloud_to_local(\n",
|
|
228
|
+
" dir_sync.as_posix(), cache_key=\"dir_cache/key\"\n",
|
|
229
|
+
")"
|
|
168
230
|
]
|
|
169
231
|
},
|
|
170
232
|
{
|
|
@@ -191,6 +253,57 @@
|
|
|
191
253
|
"dir_sync_local.rmdir()"
|
|
192
254
|
]
|
|
193
255
|
},
|
|
256
|
+
{
|
|
257
|
+
"cell_type": "markdown",
|
|
258
|
+
"id": "d2246f90",
|
|
259
|
+
"metadata": {},
|
|
260
|
+
"source": [
|
|
261
|
+
"Test `cloud_to_local` for http"
|
|
262
|
+
]
|
|
263
|
+
},
|
|
264
|
+
{
|
|
265
|
+
"cell_type": "code",
|
|
266
|
+
"execution_count": null,
|
|
267
|
+
"id": "bc1b7736",
|
|
268
|
+
"metadata": {},
|
|
269
|
+
"outputs": [],
|
|
270
|
+
"source": [
|
|
271
|
+
"http_local = settings.paths.cloud_to_local(http_path)"
|
|
272
|
+
]
|
|
273
|
+
},
|
|
274
|
+
{
|
|
275
|
+
"cell_type": "code",
|
|
276
|
+
"execution_count": null,
|
|
277
|
+
"id": "ca9f0ba8",
|
|
278
|
+
"metadata": {},
|
|
279
|
+
"outputs": [],
|
|
280
|
+
"source": [
|
|
281
|
+
"assert isinstance(http_local, LocalPathClasses)\n",
|
|
282
|
+
"assert http_local.stat().st_size == http_path.stat().st_size"
|
|
283
|
+
]
|
|
284
|
+
},
|
|
285
|
+
{
|
|
286
|
+
"cell_type": "code",
|
|
287
|
+
"execution_count": null,
|
|
288
|
+
"id": "6ae7cd2b",
|
|
289
|
+
"metadata": {},
|
|
290
|
+
"outputs": [],
|
|
291
|
+
"source": [
|
|
292
|
+
"http_local_mtime = http_local.stat().st_mtime\n",
|
|
293
|
+
"# no changes here because the file exists already\n",
|
|
294
|
+
"assert settings.paths.cloud_to_local(http_path).stat().st_mtime == http_local_mtime"
|
|
295
|
+
]
|
|
296
|
+
},
|
|
297
|
+
{
|
|
298
|
+
"cell_type": "code",
|
|
299
|
+
"execution_count": null,
|
|
300
|
+
"id": "9da41c21",
|
|
301
|
+
"metadata": {},
|
|
302
|
+
"outputs": [],
|
|
303
|
+
"source": [
|
|
304
|
+
"http_local.unlink()"
|
|
305
|
+
]
|
|
306
|
+
},
|
|
194
307
|
{
|
|
195
308
|
"cell_type": "markdown",
|
|
196
309
|
"id": "574c3f95",
|
|
@@ -328,7 +441,7 @@
|
|
|
328
441
|
"time.sleep(1)\n",
|
|
329
442
|
"cloud_file = dir_sync / \"file1\"\n",
|
|
330
443
|
"# update cloud timestamp\n",
|
|
331
|
-
"cloud_file.fs.touch(cloud_file.as_posix(), truncate=True)
|
|
444
|
+
"cloud_file.fs.touch(cloud_file.as_posix(), truncate=True)\n",
|
|
332
445
|
"\n",
|
|
333
446
|
"assert cloud_file.modified.timestamp() > local_file_new.stat().st_mtime"
|
|
334
447
|
]
|
|
@@ -76,7 +76,10 @@
|
|
|
76
76
|
"ln_setup.init(storage=\"s3://lamindb-ci/test-load-lock\", name=\"test-load-lock\")\n",
|
|
77
77
|
"instance_id = ln_setup.settings.instance._id\n",
|
|
78
78
|
"\n",
|
|
79
|
-
"assert
|
|
79
|
+
"assert (\n",
|
|
80
|
+
" ln_setup.settings.instance._cloud_sqlite_locker\n",
|
|
81
|
+
" is ln_setup.settings.instance._cloud_sqlite_locker\n",
|
|
82
|
+
")\n",
|
|
80
83
|
"\n",
|
|
81
84
|
"ln_setup.close()"
|
|
82
85
|
]
|
|
@@ -33,7 +33,7 @@ Modules & settings:
|
|
|
33
33
|
|
|
34
34
|
"""
|
|
35
35
|
|
|
36
|
-
__version__ = "
|
|
36
|
+
__version__ = "1.0a1" # denote a release candidate for 0.1.0 with 0.1rc1
|
|
37
37
|
|
|
38
38
|
import os as _os
|
|
39
39
|
import sys as _sys
|
|
@@ -51,7 +51,36 @@ from ._register_instance import register
|
|
|
51
51
|
from ._setup_user import login, logout
|
|
52
52
|
from .core._settings import settings
|
|
53
53
|
|
|
54
|
-
|
|
54
|
+
|
|
55
|
+
def _is_CI_environment() -> bool:
|
|
56
|
+
ci_env_vars = [
|
|
57
|
+
"LAMIN_TESTING", # Set by our nox configurations
|
|
58
|
+
"CI", # Commonly set by many CI systems
|
|
59
|
+
"TRAVIS", # Travis CI
|
|
60
|
+
"GITHUB_ACTIONS", # GitHub Actions
|
|
61
|
+
"GITLAB_CI", # GitLab CI/CD
|
|
62
|
+
"CIRCLECI", # CircleCI
|
|
63
|
+
"JENKINS_URL", # Jenkins
|
|
64
|
+
"TEAMCITY_VERSION", # TeamCity
|
|
65
|
+
"BUILDKITE", # Buildkite
|
|
66
|
+
"BITBUCKET_BUILD_NUMBER", # Bitbucket Pipelines
|
|
67
|
+
"APPVEYOR", # AppVeyor
|
|
68
|
+
"AZURE_HTTP_USER_AGENT", # Azure Pipelines
|
|
69
|
+
"BUDDY", # Buddy
|
|
70
|
+
"DRONE", # Drone CI
|
|
71
|
+
"HUDSON_URL", # Hudson
|
|
72
|
+
"CF_BUILD_ID", # Codefresh
|
|
73
|
+
"WERCKER", # Wercker
|
|
74
|
+
"NOW_BUILDER", # ZEIT Now
|
|
75
|
+
"TASKCLUSTER_ROOT_URL", # TaskCluster
|
|
76
|
+
"SEMAPHORE", # Semaphore CI
|
|
77
|
+
"BUILD_ID", # Generic build environments
|
|
78
|
+
]
|
|
79
|
+
return any(env_var in _os.environ for env_var in ci_env_vars)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
_TESTING = _is_CI_environment()
|
|
83
|
+
|
|
55
84
|
|
|
56
85
|
# hide the supabase error in a thread on windows
|
|
57
86
|
if _os.name == "nt":
|
|
@@ -27,6 +27,7 @@ If you used the CLI to set up lamindb in a notebook, restart the Python session.
|
|
|
27
27
|
|
|
28
28
|
|
|
29
29
|
CURRENT_ISETTINGS: InstanceSettings | None = None
|
|
30
|
+
IS_LOADING: bool = False
|
|
30
31
|
|
|
31
32
|
|
|
32
33
|
def _get_current_instance_settings() -> InstanceSettings | None:
|
|
@@ -54,16 +55,11 @@ def _get_current_instance_settings() -> InstanceSettings | None:
|
|
|
54
55
|
|
|
55
56
|
# we make this a private function because in all the places it's used,
|
|
56
57
|
# users should not see it
|
|
57
|
-
def _check_instance_setup(
|
|
58
|
-
from_lamindb: bool = False, from_module: str | None = None
|
|
59
|
-
) -> bool:
|
|
60
|
-
reload_module = from_lamindb or from_module is not None
|
|
61
|
-
from ._init_instance import get_schema_module_name, reload_schema_modules
|
|
62
|
-
|
|
58
|
+
def _check_instance_setup(from_module: str | None = None) -> bool:
|
|
63
59
|
if django.IS_SETUP:
|
|
64
60
|
# reload logic here because module might not yet have been imported
|
|
65
61
|
# upon first setup
|
|
66
|
-
if from_module is not None:
|
|
62
|
+
if from_module is not None and from_module != "lamindb":
|
|
67
63
|
il.reload(il.import_module(from_module))
|
|
68
64
|
return True
|
|
69
65
|
silence_loggers()
|
|
@@ -75,18 +71,21 @@ def _check_instance_setup(
|
|
|
75
71
|
return True
|
|
76
72
|
isettings = _get_current_instance_settings()
|
|
77
73
|
if isettings is not None:
|
|
78
|
-
if
|
|
79
|
-
|
|
74
|
+
if (
|
|
75
|
+
from_module is not None
|
|
76
|
+
and settings.auto_connect
|
|
77
|
+
and not django.IS_SETUP
|
|
78
|
+
and not IS_LOADING
|
|
79
|
+
):
|
|
80
|
+
if not from_module == "lamindb":
|
|
81
|
+
import lamindb
|
|
82
|
+
|
|
83
|
+
il.reload(il.import_module(from_module))
|
|
84
|
+
else:
|
|
80
85
|
django.setup_django(isettings)
|
|
81
|
-
if from_module is not None:
|
|
82
|
-
# this only reloads `from_module`
|
|
83
|
-
il.reload(il.import_module(from_module))
|
|
84
|
-
else:
|
|
85
|
-
# this bulk reloads all schema modules
|
|
86
|
-
reload_schema_modules(isettings)
|
|
87
86
|
logger.important(f"connected lamindb: {isettings.slug}")
|
|
88
87
|
return django.IS_SETUP
|
|
89
88
|
else:
|
|
90
|
-
if
|
|
89
|
+
if from_module is not None and settings.auto_connect:
|
|
91
90
|
logger.warning(InstanceNotSetupError.default_message)
|
|
92
91
|
return False
|