lamindb_setup 0.69.5__py2.py3-none-any.whl → 0.71.0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lamindb_setup/__init__.py +15 -15
- lamindb_setup/_add_remote_storage.py +22 -33
- lamindb_setup/_cache.py +4 -1
- lamindb_setup/_check.py +3 -0
- lamindb_setup/_check_setup.py +13 -7
- lamindb_setup/_close.py +2 -0
- lamindb_setup/_connect_instance.py +50 -34
- lamindb_setup/_delete.py +121 -22
- lamindb_setup/_django.py +4 -1
- lamindb_setup/_exportdb.py +4 -2
- lamindb_setup/_importdb.py +5 -1
- lamindb_setup/_init_instance.py +58 -46
- lamindb_setup/_migrate.py +20 -14
- lamindb_setup/_register_instance.py +10 -3
- lamindb_setup/_schema.py +6 -3
- lamindb_setup/_setup_user.py +8 -8
- lamindb_setup/_silence_loggers.py +4 -2
- lamindb_setup/core/__init__.py +4 -3
- lamindb_setup/core/_aws_storage.py +3 -0
- lamindb_setup/core/_deprecated.py +2 -7
- lamindb_setup/core/_docs.py +2 -0
- lamindb_setup/core/_hub_client.py +12 -10
- lamindb_setup/core/_hub_core.py +206 -85
- lamindb_setup/core/_hub_crud.py +15 -11
- lamindb_setup/core/_hub_utils.py +11 -8
- lamindb_setup/core/_settings.py +23 -26
- lamindb_setup/core/_settings_instance.py +164 -42
- lamindb_setup/core/_settings_load.py +13 -8
- lamindb_setup/core/_settings_save.py +11 -8
- lamindb_setup/core/_settings_storage.py +104 -95
- lamindb_setup/core/_settings_store.py +3 -2
- lamindb_setup/core/_settings_user.py +10 -6
- lamindb_setup/core/_setup_bionty_sources.py +9 -2
- lamindb_setup/core/cloud_sqlite_locker.py +13 -10
- lamindb_setup/core/django.py +5 -0
- lamindb_setup/core/exceptions.py +4 -2
- lamindb_setup/core/hashing.py +15 -5
- lamindb_setup/core/types.py +5 -2
- lamindb_setup/core/upath.py +202 -92
- {lamindb_setup-0.69.5.dist-info → lamindb_setup-0.71.0.dist-info}/METADATA +6 -4
- lamindb_setup-0.71.0.dist-info/RECORD +43 -0
- lamindb_setup-0.69.5.dist-info/RECORD +0 -43
- {lamindb_setup-0.69.5.dist-info → lamindb_setup-0.71.0.dist-info}/LICENSE +0 -0
- {lamindb_setup-0.69.5.dist-info → lamindb_setup-0.71.0.dist-info}/WHEEL +0 -0
lamindb_setup/core/upath.py
CHANGED
|
@@ -1,22 +1,28 @@
|
|
|
1
1
|
# we are not documenting UPath here because it's documented at lamindb.UPath
|
|
2
2
|
"""Paths & file systems."""
|
|
3
3
|
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
4
6
|
import os
|
|
7
|
+
from collections import defaultdict
|
|
5
8
|
from datetime import datetime, timezone
|
|
6
|
-
import
|
|
9
|
+
from functools import partial
|
|
10
|
+
from itertools import islice
|
|
7
11
|
from pathlib import Path, PurePosixPath
|
|
8
|
-
from typing import
|
|
12
|
+
from typing import TYPE_CHECKING, Any, Literal
|
|
13
|
+
|
|
14
|
+
import botocore.session
|
|
9
15
|
import fsspec
|
|
10
|
-
from itertools import islice
|
|
11
|
-
from typing import Optional, Set, Any, Tuple, List
|
|
12
|
-
from collections import defaultdict
|
|
13
16
|
from lamin_utils import logger
|
|
14
17
|
from upath import UPath
|
|
15
|
-
from upath.implementations.cloud import CloudPath, S3Path #
|
|
18
|
+
from upath.implementations.cloud import CloudPath, S3Path # keep CloudPath!
|
|
16
19
|
from upath.implementations.local import LocalPath, PosixUPath, WindowsUPath
|
|
17
|
-
|
|
20
|
+
|
|
18
21
|
from .hashing import b16_to_b64, hash_md5s_from_dir
|
|
19
22
|
|
|
23
|
+
if TYPE_CHECKING:
|
|
24
|
+
from .types import UPathStr
|
|
25
|
+
|
|
20
26
|
LocalPathClasses = (PosixUPath, WindowsUPath, LocalPath)
|
|
21
27
|
|
|
22
28
|
# also see https://gist.github.com/securifera/e7eed730cbe1ce43d0c29d7cd2d582f4
|
|
@@ -55,7 +61,7 @@ VALID_SUFFIXES = {
|
|
|
55
61
|
TRAILING_SEP = (os.sep, os.altsep) if os.altsep is not None else os.sep
|
|
56
62
|
|
|
57
63
|
|
|
58
|
-
def extract_suffix_from_path(path: Path, arg_name:
|
|
64
|
+
def extract_suffix_from_path(path: Path, arg_name: str | None = None) -> str:
|
|
59
65
|
def process_digits(suffix: str):
|
|
60
66
|
if suffix[1:].isdigit(): # :1 to skip the dot
|
|
61
67
|
return "" # digits are no valid suffixes
|
|
@@ -139,44 +145,100 @@ def create_mapper(
|
|
|
139
145
|
)
|
|
140
146
|
|
|
141
147
|
|
|
142
|
-
def print_hook(size: int, value: int,
|
|
148
|
+
def print_hook(size: int, value: int, objectname: str, action: str):
|
|
143
149
|
progress_in_percent = (value / size) * 100
|
|
144
|
-
out = (
|
|
145
|
-
f"... {kwargs['action']} {Path(kwargs['filepath']).name}:"
|
|
146
|
-
f" {min(progress_in_percent, 100):4.1f}%"
|
|
147
|
-
)
|
|
148
|
-
if progress_in_percent >= 100:
|
|
149
|
-
out += "\n"
|
|
150
|
+
out = f"... {action} {objectname}:" f" {min(progress_in_percent, 100):4.1f}%"
|
|
150
151
|
if "NBPRJ_TEST_NBPATH" not in os.environ:
|
|
151
152
|
print(out, end="\r")
|
|
152
153
|
|
|
153
154
|
|
|
154
155
|
class ProgressCallback(fsspec.callbacks.Callback):
|
|
155
|
-
def __init__(
|
|
156
|
+
def __init__(
|
|
157
|
+
self,
|
|
158
|
+
objectname: str,
|
|
159
|
+
action: Literal["uploading", "downloading", "synchronizing"],
|
|
160
|
+
adjust_size: bool = False,
|
|
161
|
+
):
|
|
162
|
+
assert action in {"uploading", "downloading", "synchronizing"}
|
|
163
|
+
|
|
156
164
|
super().__init__()
|
|
165
|
+
|
|
157
166
|
self.action = action
|
|
167
|
+
print_progress = partial(print_hook, objectname=objectname, action=action)
|
|
168
|
+
self.hooks = {"print_progress": print_progress}
|
|
169
|
+
|
|
170
|
+
self.adjust_size = adjust_size
|
|
171
|
+
|
|
172
|
+
def absolute_update(self, value):
|
|
173
|
+
pass
|
|
174
|
+
|
|
175
|
+
def relative_update(self, inc=1):
|
|
176
|
+
pass
|
|
177
|
+
|
|
178
|
+
def update_relative_value(self, inc=1):
|
|
179
|
+
self.value += inc
|
|
180
|
+
self.call()
|
|
158
181
|
|
|
159
182
|
def branch(self, path_1, path_2, kwargs):
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
183
|
+
if self.adjust_size:
|
|
184
|
+
if Path(path_2 if self.action != "uploading" else path_1).is_dir():
|
|
185
|
+
self.size -= 1
|
|
186
|
+
kwargs["callback"] = ChildProgressCallback(self)
|
|
187
|
+
|
|
188
|
+
def branched(self, path_1, path_2, **kwargs):
|
|
189
|
+
self.branch(path_1, path_2, kwargs)
|
|
190
|
+
return kwargs["callback"]
|
|
191
|
+
|
|
192
|
+
def wrap(self, iterable):
|
|
193
|
+
if self.adjust_size:
|
|
194
|
+
paths = []
|
|
195
|
+
for lpath, rpath in iterable:
|
|
196
|
+
paths.append((lpath, rpath))
|
|
197
|
+
if Path(lpath).is_dir():
|
|
198
|
+
self.size -= 1
|
|
199
|
+
self.adjust_size = False
|
|
200
|
+
return paths
|
|
201
|
+
else:
|
|
202
|
+
return iterable
|
|
203
|
+
|
|
204
|
+
@classmethod
|
|
205
|
+
def requires_progress(
|
|
206
|
+
cls,
|
|
207
|
+
maybe_callback: fsspec.callbacks.Callback | None,
|
|
208
|
+
print_progress: bool,
|
|
209
|
+
objectname: str,
|
|
210
|
+
action: Literal["uploading", "downloading", "synchronizing"],
|
|
211
|
+
**kwargs,
|
|
212
|
+
):
|
|
213
|
+
if maybe_callback is None:
|
|
214
|
+
if print_progress:
|
|
215
|
+
return cls(objectname, action, **kwargs)
|
|
216
|
+
else:
|
|
217
|
+
return fsspec.callbacks.NoOpCallback()
|
|
218
|
+
return maybe_callback
|
|
163
219
|
|
|
164
|
-
|
|
165
|
-
|
|
220
|
+
|
|
221
|
+
class ChildProgressCallback(fsspec.callbacks.Callback):
|
|
222
|
+
def __init__(self, parent: ProgressCallback):
|
|
223
|
+
super().__init__()
|
|
224
|
+
|
|
225
|
+
self.parent = parent
|
|
226
|
+
|
|
227
|
+
def parent_update(self, inc=1):
|
|
228
|
+
self.parent.update_relative_value(inc)
|
|
229
|
+
|
|
230
|
+
def relative_update(self, inc=1):
|
|
231
|
+
self.parent_update(inc / self.size)
|
|
166
232
|
|
|
167
233
|
|
|
168
234
|
def download_to(self, path: UPathStr, print_progress: bool = False, **kwargs):
|
|
169
235
|
"""Download to a path."""
|
|
170
|
-
if print_progress:
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
else:
|
|
177
|
-
# todo: make proper progress bar for directories
|
|
178
|
-
cb = fsspec.callbacks.NoOpCallback()
|
|
179
|
-
kwargs["callback"] = cb
|
|
236
|
+
if print_progress and "callback" not in kwargs:
|
|
237
|
+
callback = ProgressCallback(
|
|
238
|
+
PurePosixPath(path).name, "downloading", adjust_size=True
|
|
239
|
+
)
|
|
240
|
+
kwargs["callback"] = callback
|
|
241
|
+
|
|
180
242
|
self.fs.download(str(self), str(path), **kwargs)
|
|
181
243
|
|
|
182
244
|
|
|
@@ -188,20 +250,16 @@ def upload_from(
|
|
|
188
250
|
**kwargs,
|
|
189
251
|
):
|
|
190
252
|
"""Upload from a local path."""
|
|
191
|
-
|
|
253
|
+
path = Path(path)
|
|
254
|
+
path_is_dir = path.is_dir()
|
|
192
255
|
if not path_is_dir:
|
|
193
256
|
dir_inplace = False
|
|
194
257
|
|
|
195
|
-
if print_progress:
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
else:
|
|
199
|
-
# todo: make proper progress bar for directories
|
|
200
|
-
cb = fsspec.callbacks.NoOpCallback()
|
|
201
|
-
kwargs["callback"] = cb
|
|
258
|
+
if print_progress and "callback" not in kwargs:
|
|
259
|
+
callback = ProgressCallback(path.name, "uploading")
|
|
260
|
+
kwargs["callback"] = callback
|
|
202
261
|
|
|
203
262
|
if dir_inplace:
|
|
204
|
-
path = Path(path)
|
|
205
263
|
source = [f for f in path.rglob("*") if f.is_file()]
|
|
206
264
|
destination = [str(self / f.relative_to(path)) for f in source]
|
|
207
265
|
source = [str(f) for f in source] # type: ignore
|
|
@@ -231,7 +289,14 @@ def upload_from(
|
|
|
231
289
|
del self.fs.dircache[bucket]
|
|
232
290
|
|
|
233
291
|
|
|
234
|
-
def synchronize(
|
|
292
|
+
def synchronize(
|
|
293
|
+
self,
|
|
294
|
+
objectpath: Path,
|
|
295
|
+
error_no_origin: bool = True,
|
|
296
|
+
print_progress: bool = False,
|
|
297
|
+
callback: fsspec.callbacks.Callback | None = None,
|
|
298
|
+
**kwargs,
|
|
299
|
+
):
|
|
235
300
|
"""Sync to a local destination path."""
|
|
236
301
|
# optimize the number of network requests
|
|
237
302
|
if "timestamp" in kwargs:
|
|
@@ -290,15 +355,23 @@ def synchronize(self, objectpath: Path, error_no_origin: bool = True, **kwargs):
|
|
|
290
355
|
destination_exists = False
|
|
291
356
|
need_synchronize = True
|
|
292
357
|
if need_synchronize:
|
|
358
|
+
callback = ProgressCallback.requires_progress(
|
|
359
|
+
callback, print_progress, objectpath.name, "synchronizing"
|
|
360
|
+
)
|
|
361
|
+
callback.set_size(len(files))
|
|
293
362
|
origin_file_keys = []
|
|
294
|
-
for file, stat in files.items():
|
|
295
|
-
|
|
296
|
-
origin_file_keys.append(
|
|
363
|
+
for file, stat in callback.wrap(files.items()):
|
|
364
|
+
file_key = PurePosixPath(file).relative_to(self.path)
|
|
365
|
+
origin_file_keys.append(file_key.as_posix())
|
|
297
366
|
timestamp = stat[modified_key].timestamp()
|
|
298
|
-
|
|
299
|
-
origin.
|
|
300
|
-
|
|
367
|
+
|
|
368
|
+
origin = f"{self.protocol}://{file}"
|
|
369
|
+
destination = objectpath / file_key
|
|
370
|
+
child = callback.branched(origin, destination.as_posix())
|
|
371
|
+
UPath(origin, **self._kwargs).synchronize(
|
|
372
|
+
destination, timestamp=timestamp, callback=child, **kwargs
|
|
301
373
|
)
|
|
374
|
+
child.close()
|
|
302
375
|
if destination_exists:
|
|
303
376
|
local_files = [file for file in objectpath.rglob("*") if file.is_file()]
|
|
304
377
|
if len(local_files) > len(files):
|
|
@@ -314,6 +387,10 @@ def synchronize(self, objectpath: Path, error_no_origin: bool = True, **kwargs):
|
|
|
314
387
|
return None
|
|
315
388
|
|
|
316
389
|
# synchronization logic for files
|
|
390
|
+
callback = ProgressCallback.requires_progress(
|
|
391
|
+
callback, print_progress, objectpath.name, "synchronizing"
|
|
392
|
+
)
|
|
393
|
+
kwargs["callback"] = callback
|
|
317
394
|
if objectpath.exists():
|
|
318
395
|
local_mts = objectpath.stat().st_mtime # type: ignore
|
|
319
396
|
need_synchronize = cloud_mts > local_mts
|
|
@@ -323,9 +400,13 @@ def synchronize(self, objectpath: Path, error_no_origin: bool = True, **kwargs):
|
|
|
323
400
|
if need_synchronize:
|
|
324
401
|
self.download_to(objectpath, **kwargs)
|
|
325
402
|
os.utime(objectpath, times=(cloud_mts, cloud_mts))
|
|
403
|
+
else:
|
|
404
|
+
# nothing happens if parent_update is not defined
|
|
405
|
+
# because of Callback.no_op
|
|
406
|
+
callback.parent_update()
|
|
326
407
|
|
|
327
408
|
|
|
328
|
-
def modified(self) ->
|
|
409
|
+
def modified(self) -> datetime | None:
|
|
329
410
|
"""Return modified time stamp."""
|
|
330
411
|
mtime = self.fs.modified(str(self))
|
|
331
412
|
if mtime.tzinfo is None:
|
|
@@ -338,15 +419,15 @@ def compute_file_tree(
|
|
|
338
419
|
*,
|
|
339
420
|
level: int = -1,
|
|
340
421
|
only_dirs: bool = False,
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
422
|
+
n_max_files_per_dir_and_type: int = 100,
|
|
423
|
+
n_max_files: int = 1000,
|
|
424
|
+
include_paths: set[Any] | None = None,
|
|
425
|
+
skip_suffixes: list[str] | None = None,
|
|
426
|
+
) -> tuple[str, int]:
|
|
345
427
|
space = " "
|
|
346
428
|
branch = "│ "
|
|
347
429
|
tee = "├── "
|
|
348
430
|
last = "└── "
|
|
349
|
-
max_files_per_dir_per_type = 7
|
|
350
431
|
if skip_suffixes is None:
|
|
351
432
|
skip_suffixes_tuple = ()
|
|
352
433
|
else:
|
|
@@ -380,14 +461,14 @@ def compute_file_tree(
|
|
|
380
461
|
if only_dirs:
|
|
381
462
|
contents = [d for d in contents if d.is_dir()]
|
|
382
463
|
pointers = [tee] * (len(contents) - 1) + [last]
|
|
383
|
-
|
|
384
|
-
for pointer, child_path in zip(pointers, contents):
|
|
464
|
+
n_files_per_dir_and_type = defaultdict(lambda: 0) # type: ignore
|
|
465
|
+
for pointer, child_path in zip(pointers, contents, strict=False): # type: ignore
|
|
385
466
|
if child_path.is_dir():
|
|
386
467
|
if include_dirs and child_path not in include_dirs:
|
|
387
468
|
continue
|
|
388
469
|
yield prefix + pointer + child_path.name
|
|
389
470
|
n_directories += 1
|
|
390
|
-
|
|
471
|
+
n_files_per_dir_and_type = defaultdict(lambda: 0)
|
|
391
472
|
extension = branch if pointer == tee else space
|
|
392
473
|
yield from inner(child_path, prefix=prefix + extension, level=level - 1)
|
|
393
474
|
elif not only_dirs:
|
|
@@ -395,21 +476,21 @@ def compute_file_tree(
|
|
|
395
476
|
continue
|
|
396
477
|
suffix = extract_suffix_from_path(child_path)
|
|
397
478
|
suffixes.add(suffix)
|
|
398
|
-
|
|
479
|
+
n_files_per_dir_and_type[suffix] += 1
|
|
399
480
|
n_objects += 1
|
|
400
|
-
if
|
|
481
|
+
if n_files_per_dir_and_type[suffix] == n_max_files_per_dir_and_type:
|
|
401
482
|
yield prefix + "..."
|
|
402
|
-
elif
|
|
483
|
+
elif n_files_per_dir_and_type[suffix] > n_max_files_per_dir_and_type:
|
|
403
484
|
continue
|
|
404
485
|
else:
|
|
405
486
|
yield prefix + pointer + child_path.name
|
|
406
487
|
|
|
407
488
|
folder_tree = ""
|
|
408
489
|
iterator = inner(path, level=level)
|
|
409
|
-
for line in islice(iterator,
|
|
490
|
+
for line in islice(iterator, n_max_files):
|
|
410
491
|
folder_tree += f"\n{line}"
|
|
411
492
|
if next(iterator, None):
|
|
412
|
-
folder_tree += f"\n... only showing {
|
|
493
|
+
folder_tree += f"\n... only showing {n_max_files} out of {n_objects} files"
|
|
413
494
|
directory_info = "directory" if n_directories == 1 else "directories"
|
|
414
495
|
display_suffixes = ", ".join([f"{suffix!r}" for suffix in suffixes])
|
|
415
496
|
suffix_message = f" with suffixes {display_suffixes}" if n_objects > 0 else ""
|
|
@@ -424,11 +505,12 @@ def compute_file_tree(
|
|
|
424
505
|
def view_tree(
|
|
425
506
|
path: Path,
|
|
426
507
|
*,
|
|
427
|
-
level: int =
|
|
508
|
+
level: int = 2,
|
|
428
509
|
only_dirs: bool = False,
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
510
|
+
n_max_files_per_dir_and_type: int = 100,
|
|
511
|
+
n_max_files: int = 1000,
|
|
512
|
+
include_paths: set[Any] | None = None,
|
|
513
|
+
skip_suffixes: list[str] | None = None,
|
|
432
514
|
) -> None:
|
|
433
515
|
"""Print a visual tree structure of files & directories.
|
|
434
516
|
|
|
@@ -436,7 +518,7 @@ def view_tree(
|
|
|
436
518
|
level: If `1`, only iterate through one level, if `2` iterate through 2
|
|
437
519
|
levels, if `-1` iterate through entire hierarchy.
|
|
438
520
|
only_dirs: Only iterate through directories.
|
|
439
|
-
|
|
521
|
+
n_max_files: Display limit. Will only show this many files. Doesn't affect count.
|
|
440
522
|
include_paths: Restrict to these paths.
|
|
441
523
|
skip_suffixes: Skip directories with these suffixes.
|
|
442
524
|
|
|
@@ -470,7 +552,8 @@ def view_tree(
|
|
|
470
552
|
path,
|
|
471
553
|
level=level,
|
|
472
554
|
only_dirs=only_dirs,
|
|
473
|
-
|
|
555
|
+
n_max_files=n_max_files,
|
|
556
|
+
n_max_files_per_dir_and_type=n_max_files_per_dir_and_type,
|
|
474
557
|
include_paths=include_paths,
|
|
475
558
|
skip_suffixes=skip_suffixes,
|
|
476
559
|
)
|
|
@@ -495,9 +578,10 @@ def to_url(upath):
|
|
|
495
578
|
bucket = upath._url.netloc
|
|
496
579
|
if bucket == "scverse-spatial-eu-central-1":
|
|
497
580
|
region = "eu-central-1"
|
|
498
|
-
elif f"s3://{bucket}" not in
|
|
499
|
-
|
|
500
|
-
|
|
581
|
+
elif f"s3://{bucket}" not in HOSTED_BUCKETS:
|
|
582
|
+
response = upath.fs.call_s3("head_bucket", Bucket=upath._url.netloc)
|
|
583
|
+
headers = response["ResponseMetadata"]["HTTPHeaders"]
|
|
584
|
+
region = headers.get("x-amz-bucket-region")
|
|
501
585
|
else:
|
|
502
586
|
region = bucket.replace("lamin_", "")
|
|
503
587
|
if region == "us-east-1":
|
|
@@ -576,7 +660,7 @@ def convert_pathlike(pathlike: UPathStr) -> UPath:
|
|
|
576
660
|
return path
|
|
577
661
|
|
|
578
662
|
|
|
579
|
-
|
|
663
|
+
HOSTED_REGIONS = [
|
|
580
664
|
"eu-central-1",
|
|
581
665
|
"eu-west-2",
|
|
582
666
|
"us-east-1",
|
|
@@ -586,16 +670,16 @@ hosted_regions = [
|
|
|
586
670
|
]
|
|
587
671
|
lamin_env = os.getenv("LAMIN_ENV")
|
|
588
672
|
if lamin_env is None or lamin_env == "prod":
|
|
589
|
-
hosted_buckets_list = [f"s3://lamin-{region}" for region in
|
|
673
|
+
hosted_buckets_list = [f"s3://lamin-{region}" for region in HOSTED_REGIONS]
|
|
590
674
|
hosted_buckets_list.append("s3://scverse-spatial-eu-central-1")
|
|
591
|
-
|
|
675
|
+
HOSTED_BUCKETS = tuple(hosted_buckets_list)
|
|
592
676
|
else:
|
|
593
|
-
|
|
594
|
-
credentials_cache:
|
|
677
|
+
HOSTED_BUCKETS = ("s3://lamin-hosted-test",) # type: ignore
|
|
678
|
+
credentials_cache: dict[str, dict[str, str]] = {}
|
|
595
679
|
AWS_CREDENTIALS_PRESENT = None
|
|
596
680
|
|
|
597
681
|
|
|
598
|
-
def create_path(path: UPath, access_token:
|
|
682
|
+
def create_path(path: UPath, access_token: str | None = None) -> UPath:
|
|
599
683
|
path = convert_pathlike(path)
|
|
600
684
|
# test whether we have an AWS S3 path
|
|
601
685
|
if not isinstance(path, S3Path):
|
|
@@ -609,9 +693,8 @@ def create_path(path: UPath, access_token: Optional[str] = None) -> UPath:
|
|
|
609
693
|
if path.fs.key is not None and path.fs.secret is not None:
|
|
610
694
|
anon = False
|
|
611
695
|
else:
|
|
612
|
-
# we
|
|
613
|
-
# path.fs.
|
|
614
|
-
# and check path.fs.session._credentials, but it is slower
|
|
696
|
+
# we could do path.fs.connect()
|
|
697
|
+
# and check path.fs.session._credentials, but it'd be slower
|
|
615
698
|
session = botocore.session.get_session()
|
|
616
699
|
credentials = session.get_credentials()
|
|
617
700
|
if credentials is None or credentials.access_key is None:
|
|
@@ -623,7 +706,7 @@ def create_path(path: UPath, access_token: Optional[str] = None) -> UPath:
|
|
|
623
706
|
|
|
624
707
|
# test whether we are on hosted storage or not
|
|
625
708
|
path_str = path.as_posix()
|
|
626
|
-
is_hosted_storage = path_str.startswith(
|
|
709
|
+
is_hosted_storage = path_str.startswith(HOSTED_BUCKETS)
|
|
627
710
|
|
|
628
711
|
if not is_hosted_storage:
|
|
629
712
|
# make anon request if no credentials present
|
|
@@ -650,7 +733,7 @@ def create_path(path: UPath, access_token: Optional[str] = None) -> UPath:
|
|
|
650
733
|
)
|
|
651
734
|
|
|
652
735
|
|
|
653
|
-
def get_stat_file_cloud(stat:
|
|
736
|
+
def get_stat_file_cloud(stat: dict) -> tuple[int, str, str]:
|
|
654
737
|
size = stat["size"]
|
|
655
738
|
# small files
|
|
656
739
|
if "-" not in stat["ETag"]:
|
|
@@ -667,7 +750,7 @@ def get_stat_file_cloud(stat: Dict) -> Tuple[int, str, str]:
|
|
|
667
750
|
return size, hash, hash_type
|
|
668
751
|
|
|
669
752
|
|
|
670
|
-
def get_stat_dir_cloud(path: UPath) ->
|
|
753
|
+
def get_stat_dir_cloud(path: UPath) -> tuple[int, str, str, int]:
|
|
671
754
|
sizes = []
|
|
672
755
|
md5s = []
|
|
673
756
|
objects = path.fs.find(path.as_posix(), detail=True)
|
|
@@ -688,20 +771,47 @@ class InstanceNotEmpty(Exception):
|
|
|
688
771
|
pass
|
|
689
772
|
|
|
690
773
|
|
|
691
|
-
|
|
774
|
+
# is as fast as boto3: https://lamin.ai/laminlabs/lamindata/transform/krGp3hT1f78N5zKv
|
|
775
|
+
def check_storage_is_empty(
|
|
776
|
+
root: UPathStr, *, raise_error: bool = True, account_for_sqlite_file: bool = False
|
|
777
|
+
) -> int:
|
|
692
778
|
root_upath = convert_pathlike(root)
|
|
693
779
|
root_string = root_upath.as_posix() # type: ignore
|
|
694
780
|
# we currently touch a 0-byte file in the root of a hosted storage location
|
|
695
781
|
# ({storage_root}/.lamindb/_is_initialized) during storage initialization
|
|
696
782
|
# since path.fs.find raises a PermissionError on empty hosted
|
|
697
783
|
# subdirectories (see lamindb_setup/core/_settings_storage/init_storage).
|
|
698
|
-
|
|
699
|
-
if root_string.startswith(
|
|
700
|
-
|
|
701
|
-
|
|
784
|
+
n_offset_objects = 1 # because of touched dummy file, see mark_storage_root()
|
|
785
|
+
if root_string.startswith(HOSTED_BUCKETS):
|
|
786
|
+
# in hosted buckets, count across entire root
|
|
787
|
+
directory_string = root_string
|
|
788
|
+
# the SQLite file is not in the ".lamindb" directory
|
|
789
|
+
if account_for_sqlite_file:
|
|
790
|
+
n_offset_objects += 1 # because of SQLite file
|
|
791
|
+
else:
|
|
792
|
+
# in any other storage location, only count in .lamindb
|
|
793
|
+
if not root_string.endswith("/"):
|
|
794
|
+
root_string += "/"
|
|
795
|
+
directory_string = root_string + ".lamindb"
|
|
796
|
+
objects = root_upath.fs.find(directory_string)
|
|
702
797
|
n_objects = len(objects)
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
798
|
+
n_diff = n_objects - n_offset_objects
|
|
799
|
+
ask_for_deletion = (
|
|
800
|
+
"delete them prior to deleting the instance"
|
|
801
|
+
if raise_error
|
|
802
|
+
else "consider deleting them"
|
|
803
|
+
)
|
|
804
|
+
hint = "'./lamindb/_is_initialized' "
|
|
805
|
+
if n_offset_objects == 2:
|
|
806
|
+
hint += "& SQLite file"
|
|
807
|
+
hint += " ignored"
|
|
808
|
+
message = (
|
|
809
|
+
f"Storage {directory_string} contains {n_objects} objects "
|
|
810
|
+
f"({hint}) - {ask_for_deletion}\n{objects}"
|
|
811
|
+
)
|
|
812
|
+
if n_diff > 0:
|
|
813
|
+
if raise_error:
|
|
814
|
+
raise InstanceNotEmpty(message)
|
|
815
|
+
else:
|
|
816
|
+
logger.warning(message)
|
|
817
|
+
return n_diff
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: lamindb_setup
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.71.0
|
|
4
4
|
Summary: Setup & configure LaminDB.
|
|
5
5
|
Author-email: Lamin Labs <laminlabs@gmail.com>
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
@@ -14,7 +14,9 @@ Requires-Dist: requests
|
|
|
14
14
|
Requires-Dist: universal_pathlib==0.1.4
|
|
15
15
|
Requires-Dist: botocore<2.0.0
|
|
16
16
|
Requires-Dist: supabase==2.2.1
|
|
17
|
-
Requires-Dist:
|
|
17
|
+
Requires-Dist: urllib3<2 ; extra == "aws"
|
|
18
|
+
Requires-Dist: aiobotocore[boto3]>=2.5.4,<3.0.0 ; extra == "aws"
|
|
19
|
+
Requires-Dist: s3fs>=2023.12.2,<=2024.3.1 ; extra == "aws"
|
|
18
20
|
Requires-Dist: pyjwt<3.0.0 ; extra == "dev"
|
|
19
21
|
Requires-Dist: psycopg2-binary ; extra == "dev"
|
|
20
22
|
Requires-Dist: python-dotenv ; extra == "dev"
|
|
@@ -25,12 +27,12 @@ Requires-Dist: pytest-xdist ; extra == "dev"
|
|
|
25
27
|
Requires-Dist: nbproject-test>=0.4.3 ; extra == "dev"
|
|
26
28
|
Requires-Dist: pandas ; extra == "dev"
|
|
27
29
|
Requires-Dist: django-schema-graph ; extra == "erdiagram"
|
|
28
|
-
Requires-Dist:
|
|
30
|
+
Requires-Dist: gcsfs>=2023.12.2,<=2024.3.1 ; extra == "gcp"
|
|
29
31
|
Project-URL: Home, https://github.com/laminlabs/lamindb-setup
|
|
30
32
|
Provides-Extra: aws
|
|
31
33
|
Provides-Extra: dev
|
|
32
34
|
Provides-Extra: erdiagram
|
|
33
|
-
Provides-Extra:
|
|
35
|
+
Provides-Extra: gcp
|
|
34
36
|
|
|
35
37
|
[](https://codecov.io/gh/laminlabs/lamindb-setup)
|
|
36
38
|
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
lamindb_setup/__init__.py,sha256=6a3FROJtySXqU0zXTq1xIniLLPzc7_JufrXPqKJ11eI,1542
|
|
2
|
+
lamindb_setup/_add_remote_storage.py,sha256=j-p29pxrr_07Ns5X2tm6yKxa3VESPkjyt35X4Cvr-nk,1325
|
|
3
|
+
lamindb_setup/_cache.py,sha256=wA7mbysANwe8hPNbjDo9bOmXJ0xIyaS5iyxIpxSWji4,846
|
|
4
|
+
lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
|
|
5
|
+
lamindb_setup/_check_setup.py,sha256=cNEL9Q4yPpmEkGKHH8JgullWl1VUZwALJ4RHn9wZypY,2613
|
|
6
|
+
lamindb_setup/_close.py,sha256=1QS9p2SCacgovYn6xqWU4zFvwHN1RgIccvzwJgFvKgU,1186
|
|
7
|
+
lamindb_setup/_connect_instance.py,sha256=ElpbjSaAI9vq3OfmPM8TxvsqfrZyytTZSRyYUDfiv3E,11944
|
|
8
|
+
lamindb_setup/_delete.py,sha256=tOEb8N7Ga9hrMpup2zBugtl0wN_Xxv0MqqKP2CpHArQ,7267
|
|
9
|
+
lamindb_setup/_django.py,sha256=EoyWvFzH0i9wxjy4JZhcoXCTckztP_Mrl6FbYQnMmLE,1534
|
|
10
|
+
lamindb_setup/_exportdb.py,sha256=uTIZjKKTB7arzEr1j0O6lONiT2pRBKeOFdLvOV8ZwzE,2120
|
|
11
|
+
lamindb_setup/_importdb.py,sha256=yYYShzUajTsR-cTW4CZ-UNDWZY2uE5PAgNbp-wn8Ogc,1874
|
|
12
|
+
lamindb_setup/_init_instance.py,sha256=XLf-xkF_YBwvT_sGyW1wFJFbOWpTFABPRt2oeIICQNI,11825
|
|
13
|
+
lamindb_setup/_migrate.py,sha256=4nBTFg5-BK4A2gH-D3_tcFf8EtvMnIo5Mq0e_C6_9-U,8815
|
|
14
|
+
lamindb_setup/_register_instance.py,sha256=Jeu0wyvJVSVQ_n-A_7yn7xOZIP0ncJD92DRABqzPIjA,940
|
|
15
|
+
lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
|
|
16
|
+
lamindb_setup/_setup_user.py,sha256=6Oc7Rke-yRQSZbuntdUAz8QbJ6UuPzYHI9FnYlf_q-A,3670
|
|
17
|
+
lamindb_setup/_silence_loggers.py,sha256=AKF_YcHvX32eGXdsYK8MJlxEaZ-Uo2f6QDRzjKFCtws,1568
|
|
18
|
+
lamindb_setup/core/__init__.py,sha256=dV9S-rQpNK9JcBn4hiEmiLnmNqfpPFJD9pqagMCaIew,416
|
|
19
|
+
lamindb_setup/core/_aws_storage.py,sha256=nEjeUv4xUVpoV0Lx-zjjmyb9w804bDyaeiM-OqbfwM0,1799
|
|
20
|
+
lamindb_setup/core/_deprecated.py,sha256=3qxUI1dnDlSeR0BYrv7ucjqRBEojbqotPgpShXs4KF8,2520
|
|
21
|
+
lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
|
|
22
|
+
lamindb_setup/core/_hub_client.py,sha256=V0qKDsCdRn-tQy2YIk4VgXcpJFmuum6N3gcavAC7gBQ,5504
|
|
23
|
+
lamindb_setup/core/_hub_core.py,sha256=88EZA5G2JLlEZP6ol_km3Cv0dEpV2U2Z6X5vQOorGrs,15776
|
|
24
|
+
lamindb_setup/core/_hub_crud.py,sha256=m8DgbWIBZBLWB-PV_UNBldUebqnxABfJfrqvzgr3t5s,4662
|
|
25
|
+
lamindb_setup/core/_hub_utils.py,sha256=b_M1LkdCjiMWm1EOlSb9GuPdLijwVgQDtATTpeZuXI0,1875
|
|
26
|
+
lamindb_setup/core/_settings.py,sha256=jjZ_AxRXB3Y3UP6m04BAw_dhFbJbdg2-nZWmEv2LNZ8,3141
|
|
27
|
+
lamindb_setup/core/_settings_instance.py,sha256=RFUcnBBUp303dbVEHcAaIm_q7lzlWg56OrKLwdam8Pg,16588
|
|
28
|
+
lamindb_setup/core/_settings_load.py,sha256=0MkFvZa0fVrMt087GL8Y-mHH44RUDQYybKqqj-c8flU,3849
|
|
29
|
+
lamindb_setup/core/_settings_save.py,sha256=sLVNEMsrEO238Px8P8PsQjl4_RxGSKwRqB9Cffg2TrY,2637
|
|
30
|
+
lamindb_setup/core/_settings_storage.py,sha256=yN2KNd6lq1RDNYbnkaCIfoYc3uKqkSU70otUr4qALPM,12739
|
|
31
|
+
lamindb_setup/core/_settings_store.py,sha256=OMBn396BFkMXGg3came1pToBqnBUwpyqBFv2G8GARiM,2043
|
|
32
|
+
lamindb_setup/core/_settings_user.py,sha256=P2lC4WDRAFfT-Xq3MlXJ-wMKIHCoGNhMTQfRGIAyUNQ,1344
|
|
33
|
+
lamindb_setup/core/_setup_bionty_sources.py,sha256=OgPpZxN2_Wffy-ogEBz_97c_k8d2bD-DDVt89-u9GLY,3002
|
|
34
|
+
lamindb_setup/core/cloud_sqlite_locker.py,sha256=NIBNAGq7TTRrip9OzMdiQKj8QOuwhL9esyM0aehUqBA,6893
|
|
35
|
+
lamindb_setup/core/django.py,sha256=m0AKg2lJ1EYCtEtZ8frFFJbAR9qX0gnFcgqp7aeC2k0,3450
|
|
36
|
+
lamindb_setup/core/exceptions.py,sha256=eoI7AXgATgDVzgArtN7CUvpaMUC067vsBg5LHCsWzDM,305
|
|
37
|
+
lamindb_setup/core/hashing.py,sha256=mv9UCvAsSrdHYQAv3Kz7UOvjd5tIjvDTIYv_ettBuVY,2218
|
|
38
|
+
lamindb_setup/core/types.py,sha256=bcYnZ0uM_2NXKJCl94Mmc-uYrQlRUUVKG3sK2N-F-N4,532
|
|
39
|
+
lamindb_setup/core/upath.py,sha256=TcpBLVtI2aImJWn-FIMMThQt8i_ztJdETQ_Wj4Ow4qY,27965
|
|
40
|
+
lamindb_setup-0.71.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
|
|
41
|
+
lamindb_setup-0.71.0.dist-info/WHEEL,sha256=Sgu64hAMa6g5FdzHxXv9Xdse9yxpGGMeagVtPMWpJQY,99
|
|
42
|
+
lamindb_setup-0.71.0.dist-info/METADATA,sha256=yQ8BtbUkb8PMmdgno_coRXNadR3Ib4ikzV6pTW7TSqo,1620
|
|
43
|
+
lamindb_setup-0.71.0.dist-info/RECORD,,
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
lamindb_setup/__init__.py,sha256=UtfzeMDIIg5wzz8ahXKmjc2p9pgSXvoXop6PeSM3fig,1558
|
|
2
|
-
lamindb_setup/_add_remote_storage.py,sha256=S2rFvn-JcGaBOiBNQoAiaTPM590GQh-g4OJeNsben0Q,1802
|
|
3
|
-
lamindb_setup/_cache.py,sha256=wrwlHi2IfxcWDfOW-cHzzkQNlLvasoQPPNxihICbwew,809
|
|
4
|
-
lamindb_setup/_check.py,sha256=xgc5kF2HA4lGuAA9FJT9BJyQV5z1EwYMr92RTbGcQg4,92
|
|
5
|
-
lamindb_setup/_check_setup.py,sha256=LLdTn4JYiyWKji_sexq2eENbmvA6_gdxqMKjDnF-nJc,2543
|
|
6
|
-
lamindb_setup/_close.py,sha256=XtP7uyKRbwCGkXdSkwuiEnLYn5PEN3t3q50dPHNJRSU,1150
|
|
7
|
-
lamindb_setup/_connect_instance.py,sha256=QICzNO3ID3JwJb2yMd6R5DfExT3q5vlkn0e8WHo2lLw,11789
|
|
8
|
-
lamindb_setup/_delete.py,sha256=XzYaB2k6DhNfr5b-nzHZ4qGcAIxnuP8fc-hIOYXvy8E,3111
|
|
9
|
-
lamindb_setup/_django.py,sha256=Yv-bNj80I5Fj9AJ2BFgcv9JZg4WUrlh3UStty82_jXQ,1500
|
|
10
|
-
lamindb_setup/_exportdb.py,sha256=f5XUhRxTsU-yqfPM92OxjZKRxJo3lKaqYTRh9oytDfY,2084
|
|
11
|
-
lamindb_setup/_importdb.py,sha256=bxFZNjtVOAW4yeMQakVYZE4-WNi4QE-MdMynHShMOgI,1836
|
|
12
|
-
lamindb_setup/_init_instance.py,sha256=i0X9ENufdSGPog_8S1dypAvhwfGpVfsGuAIpnpfkjTc,11479
|
|
13
|
-
lamindb_setup/_migrate.py,sha256=JlyjuhPO7qVEzaRfB5WBvkGgy-0qMh60MddM1zemSlo,8672
|
|
14
|
-
lamindb_setup/_register_instance.py,sha256=tfdpL9NFXmtdxAGnym5Q4I5x4cT6_xVBim6pbECKlvk,795
|
|
15
|
-
lamindb_setup/_schema.py,sha256=edhbiUYatbO5FfBA6PYRmVmYWbtBXZBpATa70_MZiWo,642
|
|
16
|
-
lamindb_setup/_setup_user.py,sha256=xwSJpbLtl2gW-frGs5k_kBL6PZZ00pe1gfmmd-a7myc,3657
|
|
17
|
-
lamindb_setup/_silence_loggers.py,sha256=auXnYVSx2ErZmyXZJ7FYYA69UQgUDQ5SnuznFXzfaWs,1548
|
|
18
|
-
lamindb_setup/core/__init__.py,sha256=xfhu3cO_zPtNztmu6vlWtdbX92gPpnjyff5dOqCHTvE,369
|
|
19
|
-
lamindb_setup/core/_aws_storage.py,sha256=GIT9SagV-1oFlLpRIogFgXuTwzMZsppFjqKA4-Vp66s,1762
|
|
20
|
-
lamindb_setup/core/_deprecated.py,sha256=DNO_90uyYGM9on7rzJCu9atS7Z1qOQypsuL9PI6I-OQ,2491
|
|
21
|
-
lamindb_setup/core/_docs.py,sha256=0ha3cZqzvIQnAXVC_JlAiRrFi3VpEEVookP_oURhBr4,240
|
|
22
|
-
lamindb_setup/core/_hub_client.py,sha256=H09wJRck4d3fQ8VBSK1v8Dbj9RfqLdW9UqoTs6OCyUM,5520
|
|
23
|
-
lamindb_setup/core/_hub_core.py,sha256=2ul3KvVorXLZfEefGbm9toRLga-rli8qzzq71vvEfw0,10990
|
|
24
|
-
lamindb_setup/core/_hub_crud.py,sha256=0qKKsnHlx_FceX-jahdS2vJc2IuGRzouQrtAMn6MTSA,4505
|
|
25
|
-
lamindb_setup/core/_hub_utils.py,sha256=4bjtm4QVivDwtVq3Bx9hQFZtdU2WmqrVdpupdvxMHcM,1862
|
|
26
|
-
lamindb_setup/core/_settings.py,sha256=ogeOddaJ9tiolzpGwUIu6i4vhcViXQvsREEzZ95uyvI,3241
|
|
27
|
-
lamindb_setup/core/_settings_instance.py,sha256=XRpHZJT8MZONpIS5Sop6Rk3BLYVmguh2PkC3TrGU7jU,11556
|
|
28
|
-
lamindb_setup/core/_settings_load.py,sha256=65T_lW1xby__PKULufqzc1eWE30SVRNbxS2TAVSCWXM,3744
|
|
29
|
-
lamindb_setup/core/_settings_save.py,sha256=bAnjmZuAKk4wx4EoTx-LfaCY7yyU_4bUfF7hdI71DgY,2563
|
|
30
|
-
lamindb_setup/core/_settings_storage.py,sha256=a_Pa_nPzFuDeYxBlvFelQIkINlVGSNO_2aLvY_nxP-w,13044
|
|
31
|
-
lamindb_setup/core/_settings_store.py,sha256=iM3Ei-YTSQo6wbcUhOliQx_KmHo8sxDx0-elJPt9AwQ,1929
|
|
32
|
-
lamindb_setup/core/_settings_user.py,sha256=wWLrSaohWlQSLOA-7fLSRCV43rupy-ixNvkZMWT9Pgo,1281
|
|
33
|
-
lamindb_setup/core/_setup_bionty_sources.py,sha256=xhIIJqtCzrjTONzU_68bq5kQxsNRDfAprHi2oJvY_LY,2908
|
|
34
|
-
lamindb_setup/core/cloud_sqlite_locker.py,sha256=nFwAupnbIy_sxzlJBayfD2Dh8ZMUGNxbspIbv6BGrp8,6835
|
|
35
|
-
lamindb_setup/core/django.py,sha256=MNlEFS8a7GX1IxmMgJymoRf3M03Eg881zQ2_MKONOQI,3335
|
|
36
|
-
lamindb_setup/core/exceptions.py,sha256=YzkJA51ZAa2w9lawAggSdg-NA-EIYJGNRnFd0DN3_bE,275
|
|
37
|
-
lamindb_setup/core/hashing.py,sha256=nbQO_CSiX09O4bznHABft94fdFZAHPm2NIyvH5xagoo,2011
|
|
38
|
-
lamindb_setup/core/types.py,sha256=fR71SLjoN1MkCjx8TJcjYDgmgO4bPXBX5J_RKpmmy_o,497
|
|
39
|
-
lamindb_setup/core/upath.py,sha256=rK1GvW_-23gE5Wr3VmAv1hw4PZCxN878P-EUxXr0RlM,24510
|
|
40
|
-
lamindb_setup-0.69.5.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
|
|
41
|
-
lamindb_setup-0.69.5.dist-info/WHEEL,sha256=Sgu64hAMa6g5FdzHxXv9Xdse9yxpGGMeagVtPMWpJQY,99
|
|
42
|
-
lamindb_setup-0.69.5.dist-info/METADATA,sha256=o7IyD1LbQF4cqks-lrF4XgOh25jZcnYkrV0o9JU_9Jo,1469
|
|
43
|
-
lamindb_setup-0.69.5.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|