lamindb_setup 0.70.0__py2.py3-none-any.whl → 0.71.0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lamindb_setup/__init__.py +15 -15
- lamindb_setup/_add_remote_storage.py +22 -33
- lamindb_setup/_cache.py +4 -1
- lamindb_setup/_check.py +3 -0
- lamindb_setup/_check_setup.py +13 -7
- lamindb_setup/_close.py +2 -0
- lamindb_setup/_connect_instance.py +33 -26
- lamindb_setup/_delete.py +52 -19
- lamindb_setup/_django.py +4 -1
- lamindb_setup/_exportdb.py +4 -2
- lamindb_setup/_importdb.py +5 -1
- lamindb_setup/_init_instance.py +57 -45
- lamindb_setup/_migrate.py +16 -13
- lamindb_setup/_register_instance.py +10 -3
- lamindb_setup/_schema.py +6 -3
- lamindb_setup/_setup_user.py +7 -7
- lamindb_setup/_silence_loggers.py +4 -2
- lamindb_setup/core/__init__.py +4 -3
- lamindb_setup/core/_aws_storage.py +3 -0
- lamindb_setup/core/_deprecated.py +2 -7
- lamindb_setup/core/_docs.py +2 -0
- lamindb_setup/core/_hub_client.py +12 -10
- lamindb_setup/core/_hub_core.py +198 -88
- lamindb_setup/core/_hub_crud.py +15 -11
- lamindb_setup/core/_hub_utils.py +11 -8
- lamindb_setup/core/_settings.py +23 -26
- lamindb_setup/core/_settings_instance.py +149 -81
- lamindb_setup/core/_settings_load.py +12 -7
- lamindb_setup/core/_settings_save.py +11 -8
- lamindb_setup/core/_settings_storage.py +83 -42
- lamindb_setup/core/_settings_store.py +3 -2
- lamindb_setup/core/_settings_user.py +10 -6
- lamindb_setup/core/_setup_bionty_sources.py +9 -2
- lamindb_setup/core/cloud_sqlite_locker.py +13 -10
- lamindb_setup/core/django.py +3 -1
- lamindb_setup/core/exceptions.py +4 -2
- lamindb_setup/core/hashing.py +15 -5
- lamindb_setup/core/types.py +5 -2
- lamindb_setup/core/upath.py +181 -87
- {lamindb_setup-0.70.0.dist-info → lamindb_setup-0.71.0.dist-info}/METADATA +6 -4
- lamindb_setup-0.71.0.dist-info/RECORD +43 -0
- lamindb_setup-0.70.0.dist-info/RECORD +0 -43
- {lamindb_setup-0.70.0.dist-info → lamindb_setup-0.71.0.dist-info}/LICENSE +0 -0
- {lamindb_setup-0.70.0.dist-info → lamindb_setup-0.71.0.dist-info}/WHEEL +0 -0
lamindb_setup/core/upath.py
CHANGED
|
@@ -4,21 +4,25 @@
|
|
|
4
4
|
from __future__ import annotations
|
|
5
5
|
|
|
6
6
|
import os
|
|
7
|
+
from collections import defaultdict
|
|
7
8
|
from datetime import datetime, timezone
|
|
8
|
-
import
|
|
9
|
+
from functools import partial
|
|
10
|
+
from itertools import islice
|
|
9
11
|
from pathlib import Path, PurePosixPath
|
|
10
|
-
from typing import
|
|
12
|
+
from typing import TYPE_CHECKING, Any, Literal
|
|
13
|
+
|
|
14
|
+
import botocore.session
|
|
11
15
|
import fsspec
|
|
12
|
-
from itertools import islice
|
|
13
|
-
from typing import Optional, Set, Any, Tuple, List
|
|
14
|
-
from collections import defaultdict
|
|
15
16
|
from lamin_utils import logger
|
|
16
17
|
from upath import UPath
|
|
17
|
-
from upath.implementations.cloud import CloudPath, S3Path #
|
|
18
|
+
from upath.implementations.cloud import CloudPath, S3Path # keep CloudPath!
|
|
18
19
|
from upath.implementations.local import LocalPath, PosixUPath, WindowsUPath
|
|
19
|
-
|
|
20
|
+
|
|
20
21
|
from .hashing import b16_to_b64, hash_md5s_from_dir
|
|
21
22
|
|
|
23
|
+
if TYPE_CHECKING:
|
|
24
|
+
from .types import UPathStr
|
|
25
|
+
|
|
22
26
|
LocalPathClasses = (PosixUPath, WindowsUPath, LocalPath)
|
|
23
27
|
|
|
24
28
|
# also see https://gist.github.com/securifera/e7eed730cbe1ce43d0c29d7cd2d582f4
|
|
@@ -57,7 +61,7 @@ VALID_SUFFIXES = {
|
|
|
57
61
|
TRAILING_SEP = (os.sep, os.altsep) if os.altsep is not None else os.sep
|
|
58
62
|
|
|
59
63
|
|
|
60
|
-
def extract_suffix_from_path(path: Path, arg_name:
|
|
64
|
+
def extract_suffix_from_path(path: Path, arg_name: str | None = None) -> str:
|
|
61
65
|
def process_digits(suffix: str):
|
|
62
66
|
if suffix[1:].isdigit(): # :1 to skip the dot
|
|
63
67
|
return "" # digits are no valid suffixes
|
|
@@ -141,44 +145,100 @@ def create_mapper(
|
|
|
141
145
|
)
|
|
142
146
|
|
|
143
147
|
|
|
144
|
-
def print_hook(size: int, value: int,
|
|
148
|
+
def print_hook(size: int, value: int, objectname: str, action: str):
|
|
145
149
|
progress_in_percent = (value / size) * 100
|
|
146
|
-
out = (
|
|
147
|
-
f"... {kwargs['action']} {Path(kwargs['filepath']).name}:"
|
|
148
|
-
f" {min(progress_in_percent, 100):4.1f}%"
|
|
149
|
-
)
|
|
150
|
-
if progress_in_percent >= 100:
|
|
151
|
-
out += "\n"
|
|
150
|
+
out = f"... {action} {objectname}:" f" {min(progress_in_percent, 100):4.1f}%"
|
|
152
151
|
if "NBPRJ_TEST_NBPATH" not in os.environ:
|
|
153
152
|
print(out, end="\r")
|
|
154
153
|
|
|
155
154
|
|
|
156
155
|
class ProgressCallback(fsspec.callbacks.Callback):
|
|
157
|
-
def __init__(
|
|
156
|
+
def __init__(
|
|
157
|
+
self,
|
|
158
|
+
objectname: str,
|
|
159
|
+
action: Literal["uploading", "downloading", "synchronizing"],
|
|
160
|
+
adjust_size: bool = False,
|
|
161
|
+
):
|
|
162
|
+
assert action in {"uploading", "downloading", "synchronizing"}
|
|
163
|
+
|
|
158
164
|
super().__init__()
|
|
165
|
+
|
|
159
166
|
self.action = action
|
|
167
|
+
print_progress = partial(print_hook, objectname=objectname, action=action)
|
|
168
|
+
self.hooks = {"print_progress": print_progress}
|
|
169
|
+
|
|
170
|
+
self.adjust_size = adjust_size
|
|
171
|
+
|
|
172
|
+
def absolute_update(self, value):
|
|
173
|
+
pass
|
|
174
|
+
|
|
175
|
+
def relative_update(self, inc=1):
|
|
176
|
+
pass
|
|
177
|
+
|
|
178
|
+
def update_relative_value(self, inc=1):
|
|
179
|
+
self.value += inc
|
|
180
|
+
self.call()
|
|
160
181
|
|
|
161
182
|
def branch(self, path_1, path_2, kwargs):
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
183
|
+
if self.adjust_size:
|
|
184
|
+
if Path(path_2 if self.action != "uploading" else path_1).is_dir():
|
|
185
|
+
self.size -= 1
|
|
186
|
+
kwargs["callback"] = ChildProgressCallback(self)
|
|
187
|
+
|
|
188
|
+
def branched(self, path_1, path_2, **kwargs):
|
|
189
|
+
self.branch(path_1, path_2, kwargs)
|
|
190
|
+
return kwargs["callback"]
|
|
191
|
+
|
|
192
|
+
def wrap(self, iterable):
|
|
193
|
+
if self.adjust_size:
|
|
194
|
+
paths = []
|
|
195
|
+
for lpath, rpath in iterable:
|
|
196
|
+
paths.append((lpath, rpath))
|
|
197
|
+
if Path(lpath).is_dir():
|
|
198
|
+
self.size -= 1
|
|
199
|
+
self.adjust_size = False
|
|
200
|
+
return paths
|
|
201
|
+
else:
|
|
202
|
+
return iterable
|
|
203
|
+
|
|
204
|
+
@classmethod
|
|
205
|
+
def requires_progress(
|
|
206
|
+
cls,
|
|
207
|
+
maybe_callback: fsspec.callbacks.Callback | None,
|
|
208
|
+
print_progress: bool,
|
|
209
|
+
objectname: str,
|
|
210
|
+
action: Literal["uploading", "downloading", "synchronizing"],
|
|
211
|
+
**kwargs,
|
|
212
|
+
):
|
|
213
|
+
if maybe_callback is None:
|
|
214
|
+
if print_progress:
|
|
215
|
+
return cls(objectname, action, **kwargs)
|
|
216
|
+
else:
|
|
217
|
+
return fsspec.callbacks.NoOpCallback()
|
|
218
|
+
return maybe_callback
|
|
165
219
|
|
|
166
|
-
|
|
167
|
-
|
|
220
|
+
|
|
221
|
+
class ChildProgressCallback(fsspec.callbacks.Callback):
|
|
222
|
+
def __init__(self, parent: ProgressCallback):
|
|
223
|
+
super().__init__()
|
|
224
|
+
|
|
225
|
+
self.parent = parent
|
|
226
|
+
|
|
227
|
+
def parent_update(self, inc=1):
|
|
228
|
+
self.parent.update_relative_value(inc)
|
|
229
|
+
|
|
230
|
+
def relative_update(self, inc=1):
|
|
231
|
+
self.parent_update(inc / self.size)
|
|
168
232
|
|
|
169
233
|
|
|
170
234
|
def download_to(self, path: UPathStr, print_progress: bool = False, **kwargs):
|
|
171
235
|
"""Download to a path."""
|
|
172
|
-
if print_progress:
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
else:
|
|
179
|
-
# todo: make proper progress bar for directories
|
|
180
|
-
cb = fsspec.callbacks.NoOpCallback()
|
|
181
|
-
kwargs["callback"] = cb
|
|
236
|
+
if print_progress and "callback" not in kwargs:
|
|
237
|
+
callback = ProgressCallback(
|
|
238
|
+
PurePosixPath(path).name, "downloading", adjust_size=True
|
|
239
|
+
)
|
|
240
|
+
kwargs["callback"] = callback
|
|
241
|
+
|
|
182
242
|
self.fs.download(str(self), str(path), **kwargs)
|
|
183
243
|
|
|
184
244
|
|
|
@@ -190,20 +250,16 @@ def upload_from(
|
|
|
190
250
|
**kwargs,
|
|
191
251
|
):
|
|
192
252
|
"""Upload from a local path."""
|
|
193
|
-
|
|
253
|
+
path = Path(path)
|
|
254
|
+
path_is_dir = path.is_dir()
|
|
194
255
|
if not path_is_dir:
|
|
195
256
|
dir_inplace = False
|
|
196
257
|
|
|
197
|
-
if print_progress:
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
else:
|
|
201
|
-
# todo: make proper progress bar for directories
|
|
202
|
-
cb = fsspec.callbacks.NoOpCallback()
|
|
203
|
-
kwargs["callback"] = cb
|
|
258
|
+
if print_progress and "callback" not in kwargs:
|
|
259
|
+
callback = ProgressCallback(path.name, "uploading")
|
|
260
|
+
kwargs["callback"] = callback
|
|
204
261
|
|
|
205
262
|
if dir_inplace:
|
|
206
|
-
path = Path(path)
|
|
207
263
|
source = [f for f in path.rglob("*") if f.is_file()]
|
|
208
264
|
destination = [str(self / f.relative_to(path)) for f in source]
|
|
209
265
|
source = [str(f) for f in source] # type: ignore
|
|
@@ -233,7 +289,14 @@ def upload_from(
|
|
|
233
289
|
del self.fs.dircache[bucket]
|
|
234
290
|
|
|
235
291
|
|
|
236
|
-
def synchronize(
|
|
292
|
+
def synchronize(
|
|
293
|
+
self,
|
|
294
|
+
objectpath: Path,
|
|
295
|
+
error_no_origin: bool = True,
|
|
296
|
+
print_progress: bool = False,
|
|
297
|
+
callback: fsspec.callbacks.Callback | None = None,
|
|
298
|
+
**kwargs,
|
|
299
|
+
):
|
|
237
300
|
"""Sync to a local destination path."""
|
|
238
301
|
# optimize the number of network requests
|
|
239
302
|
if "timestamp" in kwargs:
|
|
@@ -292,15 +355,23 @@ def synchronize(self, objectpath: Path, error_no_origin: bool = True, **kwargs):
|
|
|
292
355
|
destination_exists = False
|
|
293
356
|
need_synchronize = True
|
|
294
357
|
if need_synchronize:
|
|
358
|
+
callback = ProgressCallback.requires_progress(
|
|
359
|
+
callback, print_progress, objectpath.name, "synchronizing"
|
|
360
|
+
)
|
|
361
|
+
callback.set_size(len(files))
|
|
295
362
|
origin_file_keys = []
|
|
296
|
-
for file, stat in files.items():
|
|
297
|
-
|
|
298
|
-
origin_file_keys.append(
|
|
363
|
+
for file, stat in callback.wrap(files.items()):
|
|
364
|
+
file_key = PurePosixPath(file).relative_to(self.path)
|
|
365
|
+
origin_file_keys.append(file_key.as_posix())
|
|
299
366
|
timestamp = stat[modified_key].timestamp()
|
|
300
|
-
|
|
301
|
-
origin.
|
|
302
|
-
|
|
367
|
+
|
|
368
|
+
origin = f"{self.protocol}://{file}"
|
|
369
|
+
destination = objectpath / file_key
|
|
370
|
+
child = callback.branched(origin, destination.as_posix())
|
|
371
|
+
UPath(origin, **self._kwargs).synchronize(
|
|
372
|
+
destination, timestamp=timestamp, callback=child, **kwargs
|
|
303
373
|
)
|
|
374
|
+
child.close()
|
|
304
375
|
if destination_exists:
|
|
305
376
|
local_files = [file for file in objectpath.rglob("*") if file.is_file()]
|
|
306
377
|
if len(local_files) > len(files):
|
|
@@ -316,6 +387,10 @@ def synchronize(self, objectpath: Path, error_no_origin: bool = True, **kwargs):
|
|
|
316
387
|
return None
|
|
317
388
|
|
|
318
389
|
# synchronization logic for files
|
|
390
|
+
callback = ProgressCallback.requires_progress(
|
|
391
|
+
callback, print_progress, objectpath.name, "synchronizing"
|
|
392
|
+
)
|
|
393
|
+
kwargs["callback"] = callback
|
|
319
394
|
if objectpath.exists():
|
|
320
395
|
local_mts = objectpath.stat().st_mtime # type: ignore
|
|
321
396
|
need_synchronize = cloud_mts > local_mts
|
|
@@ -325,9 +400,13 @@ def synchronize(self, objectpath: Path, error_no_origin: bool = True, **kwargs):
|
|
|
325
400
|
if need_synchronize:
|
|
326
401
|
self.download_to(objectpath, **kwargs)
|
|
327
402
|
os.utime(objectpath, times=(cloud_mts, cloud_mts))
|
|
403
|
+
else:
|
|
404
|
+
# nothing happens if parent_update is not defined
|
|
405
|
+
# because of Callback.no_op
|
|
406
|
+
callback.parent_update()
|
|
328
407
|
|
|
329
408
|
|
|
330
|
-
def modified(self) ->
|
|
409
|
+
def modified(self) -> datetime | None:
|
|
331
410
|
"""Return modified time stamp."""
|
|
332
411
|
mtime = self.fs.modified(str(self))
|
|
333
412
|
if mtime.tzinfo is None:
|
|
@@ -340,15 +419,15 @@ def compute_file_tree(
|
|
|
340
419
|
*,
|
|
341
420
|
level: int = -1,
|
|
342
421
|
only_dirs: bool = False,
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
422
|
+
n_max_files_per_dir_and_type: int = 100,
|
|
423
|
+
n_max_files: int = 1000,
|
|
424
|
+
include_paths: set[Any] | None = None,
|
|
425
|
+
skip_suffixes: list[str] | None = None,
|
|
426
|
+
) -> tuple[str, int]:
|
|
347
427
|
space = " "
|
|
348
428
|
branch = "│ "
|
|
349
429
|
tee = "├── "
|
|
350
430
|
last = "└── "
|
|
351
|
-
max_files_per_dir_per_type = 7
|
|
352
431
|
if skip_suffixes is None:
|
|
353
432
|
skip_suffixes_tuple = ()
|
|
354
433
|
else:
|
|
@@ -382,14 +461,14 @@ def compute_file_tree(
|
|
|
382
461
|
if only_dirs:
|
|
383
462
|
contents = [d for d in contents if d.is_dir()]
|
|
384
463
|
pointers = [tee] * (len(contents) - 1) + [last]
|
|
385
|
-
|
|
386
|
-
for pointer, child_path in zip(pointers, contents):
|
|
464
|
+
n_files_per_dir_and_type = defaultdict(lambda: 0) # type: ignore
|
|
465
|
+
for pointer, child_path in zip(pointers, contents, strict=False): # type: ignore
|
|
387
466
|
if child_path.is_dir():
|
|
388
467
|
if include_dirs and child_path not in include_dirs:
|
|
389
468
|
continue
|
|
390
469
|
yield prefix + pointer + child_path.name
|
|
391
470
|
n_directories += 1
|
|
392
|
-
|
|
471
|
+
n_files_per_dir_and_type = defaultdict(lambda: 0)
|
|
393
472
|
extension = branch if pointer == tee else space
|
|
394
473
|
yield from inner(child_path, prefix=prefix + extension, level=level - 1)
|
|
395
474
|
elif not only_dirs:
|
|
@@ -397,21 +476,21 @@ def compute_file_tree(
|
|
|
397
476
|
continue
|
|
398
477
|
suffix = extract_suffix_from_path(child_path)
|
|
399
478
|
suffixes.add(suffix)
|
|
400
|
-
|
|
479
|
+
n_files_per_dir_and_type[suffix] += 1
|
|
401
480
|
n_objects += 1
|
|
402
|
-
if
|
|
481
|
+
if n_files_per_dir_and_type[suffix] == n_max_files_per_dir_and_type:
|
|
403
482
|
yield prefix + "..."
|
|
404
|
-
elif
|
|
483
|
+
elif n_files_per_dir_and_type[suffix] > n_max_files_per_dir_and_type:
|
|
405
484
|
continue
|
|
406
485
|
else:
|
|
407
486
|
yield prefix + pointer + child_path.name
|
|
408
487
|
|
|
409
488
|
folder_tree = ""
|
|
410
489
|
iterator = inner(path, level=level)
|
|
411
|
-
for line in islice(iterator,
|
|
490
|
+
for line in islice(iterator, n_max_files):
|
|
412
491
|
folder_tree += f"\n{line}"
|
|
413
492
|
if next(iterator, None):
|
|
414
|
-
folder_tree += f"\n... only showing {
|
|
493
|
+
folder_tree += f"\n... only showing {n_max_files} out of {n_objects} files"
|
|
415
494
|
directory_info = "directory" if n_directories == 1 else "directories"
|
|
416
495
|
display_suffixes = ", ".join([f"{suffix!r}" for suffix in suffixes])
|
|
417
496
|
suffix_message = f" with suffixes {display_suffixes}" if n_objects > 0 else ""
|
|
@@ -426,11 +505,12 @@ def compute_file_tree(
|
|
|
426
505
|
def view_tree(
|
|
427
506
|
path: Path,
|
|
428
507
|
*,
|
|
429
|
-
level: int =
|
|
508
|
+
level: int = 2,
|
|
430
509
|
only_dirs: bool = False,
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
510
|
+
n_max_files_per_dir_and_type: int = 100,
|
|
511
|
+
n_max_files: int = 1000,
|
|
512
|
+
include_paths: set[Any] | None = None,
|
|
513
|
+
skip_suffixes: list[str] | None = None,
|
|
434
514
|
) -> None:
|
|
435
515
|
"""Print a visual tree structure of files & directories.
|
|
436
516
|
|
|
@@ -438,7 +518,7 @@ def view_tree(
|
|
|
438
518
|
level: If `1`, only iterate through one level, if `2` iterate through 2
|
|
439
519
|
levels, if `-1` iterate through entire hierarchy.
|
|
440
520
|
only_dirs: Only iterate through directories.
|
|
441
|
-
|
|
521
|
+
n_max_files: Display limit. Will only show this many files. Doesn't affect count.
|
|
442
522
|
include_paths: Restrict to these paths.
|
|
443
523
|
skip_suffixes: Skip directories with these suffixes.
|
|
444
524
|
|
|
@@ -472,7 +552,8 @@ def view_tree(
|
|
|
472
552
|
path,
|
|
473
553
|
level=level,
|
|
474
554
|
only_dirs=only_dirs,
|
|
475
|
-
|
|
555
|
+
n_max_files=n_max_files,
|
|
556
|
+
n_max_files_per_dir_and_type=n_max_files_per_dir_and_type,
|
|
476
557
|
include_paths=include_paths,
|
|
477
558
|
skip_suffixes=skip_suffixes,
|
|
478
559
|
)
|
|
@@ -497,9 +578,10 @@ def to_url(upath):
|
|
|
497
578
|
bucket = upath._url.netloc
|
|
498
579
|
if bucket == "scverse-spatial-eu-central-1":
|
|
499
580
|
region = "eu-central-1"
|
|
500
|
-
elif f"s3://{bucket}" not in
|
|
501
|
-
|
|
502
|
-
|
|
581
|
+
elif f"s3://{bucket}" not in HOSTED_BUCKETS:
|
|
582
|
+
response = upath.fs.call_s3("head_bucket", Bucket=upath._url.netloc)
|
|
583
|
+
headers = response["ResponseMetadata"]["HTTPHeaders"]
|
|
584
|
+
region = headers.get("x-amz-bucket-region")
|
|
503
585
|
else:
|
|
504
586
|
region = bucket.replace("lamin_", "")
|
|
505
587
|
if region == "us-east-1":
|
|
@@ -578,7 +660,7 @@ def convert_pathlike(pathlike: UPathStr) -> UPath:
|
|
|
578
660
|
return path
|
|
579
661
|
|
|
580
662
|
|
|
581
|
-
|
|
663
|
+
HOSTED_REGIONS = [
|
|
582
664
|
"eu-central-1",
|
|
583
665
|
"eu-west-2",
|
|
584
666
|
"us-east-1",
|
|
@@ -588,16 +670,16 @@ hosted_regions = [
|
|
|
588
670
|
]
|
|
589
671
|
lamin_env = os.getenv("LAMIN_ENV")
|
|
590
672
|
if lamin_env is None or lamin_env == "prod":
|
|
591
|
-
hosted_buckets_list = [f"s3://lamin-{region}" for region in
|
|
673
|
+
hosted_buckets_list = [f"s3://lamin-{region}" for region in HOSTED_REGIONS]
|
|
592
674
|
hosted_buckets_list.append("s3://scverse-spatial-eu-central-1")
|
|
593
|
-
|
|
675
|
+
HOSTED_BUCKETS = tuple(hosted_buckets_list)
|
|
594
676
|
else:
|
|
595
|
-
|
|
596
|
-
credentials_cache:
|
|
677
|
+
HOSTED_BUCKETS = ("s3://lamin-hosted-test",) # type: ignore
|
|
678
|
+
credentials_cache: dict[str, dict[str, str]] = {}
|
|
597
679
|
AWS_CREDENTIALS_PRESENT = None
|
|
598
680
|
|
|
599
681
|
|
|
600
|
-
def create_path(path: UPath, access_token:
|
|
682
|
+
def create_path(path: UPath, access_token: str | None = None) -> UPath:
|
|
601
683
|
path = convert_pathlike(path)
|
|
602
684
|
# test whether we have an AWS S3 path
|
|
603
685
|
if not isinstance(path, S3Path):
|
|
@@ -611,9 +693,8 @@ def create_path(path: UPath, access_token: Optional[str] = None) -> UPath:
|
|
|
611
693
|
if path.fs.key is not None and path.fs.secret is not None:
|
|
612
694
|
anon = False
|
|
613
695
|
else:
|
|
614
|
-
# we
|
|
615
|
-
# path.fs.
|
|
616
|
-
# and check path.fs.session._credentials, but it is slower
|
|
696
|
+
# we could do path.fs.connect()
|
|
697
|
+
# and check path.fs.session._credentials, but it'd be slower
|
|
617
698
|
session = botocore.session.get_session()
|
|
618
699
|
credentials = session.get_credentials()
|
|
619
700
|
if credentials is None or credentials.access_key is None:
|
|
@@ -625,7 +706,7 @@ def create_path(path: UPath, access_token: Optional[str] = None) -> UPath:
|
|
|
625
706
|
|
|
626
707
|
# test whether we are on hosted storage or not
|
|
627
708
|
path_str = path.as_posix()
|
|
628
|
-
is_hosted_storage = path_str.startswith(
|
|
709
|
+
is_hosted_storage = path_str.startswith(HOSTED_BUCKETS)
|
|
629
710
|
|
|
630
711
|
if not is_hosted_storage:
|
|
631
712
|
# make anon request if no credentials present
|
|
@@ -652,7 +733,7 @@ def create_path(path: UPath, access_token: Optional[str] = None) -> UPath:
|
|
|
652
733
|
)
|
|
653
734
|
|
|
654
735
|
|
|
655
|
-
def get_stat_file_cloud(stat:
|
|
736
|
+
def get_stat_file_cloud(stat: dict) -> tuple[int, str, str]:
|
|
656
737
|
size = stat["size"]
|
|
657
738
|
# small files
|
|
658
739
|
if "-" not in stat["ETag"]:
|
|
@@ -669,7 +750,7 @@ def get_stat_file_cloud(stat: Dict) -> Tuple[int, str, str]:
|
|
|
669
750
|
return size, hash, hash_type
|
|
670
751
|
|
|
671
752
|
|
|
672
|
-
def get_stat_dir_cloud(path: UPath) ->
|
|
753
|
+
def get_stat_dir_cloud(path: UPath) -> tuple[int, str, str, int]:
|
|
673
754
|
sizes = []
|
|
674
755
|
md5s = []
|
|
675
756
|
objects = path.fs.find(path.as_posix(), detail=True)
|
|
@@ -701,9 +782,18 @@ def check_storage_is_empty(
|
|
|
701
782
|
# since path.fs.find raises a PermissionError on empty hosted
|
|
702
783
|
# subdirectories (see lamindb_setup/core/_settings_storage/init_storage).
|
|
703
784
|
n_offset_objects = 1 # because of touched dummy file, see mark_storage_root()
|
|
704
|
-
if
|
|
705
|
-
|
|
706
|
-
|
|
785
|
+
if root_string.startswith(HOSTED_BUCKETS):
|
|
786
|
+
# in hosted buckets, count across entire root
|
|
787
|
+
directory_string = root_string
|
|
788
|
+
# the SQLite file is not in the ".lamindb" directory
|
|
789
|
+
if account_for_sqlite_file:
|
|
790
|
+
n_offset_objects += 1 # because of SQLite file
|
|
791
|
+
else:
|
|
792
|
+
# in any other storage location, only count in .lamindb
|
|
793
|
+
if not root_string.endswith("/"):
|
|
794
|
+
root_string += "/"
|
|
795
|
+
directory_string = root_string + ".lamindb"
|
|
796
|
+
objects = root_upath.fs.find(directory_string)
|
|
707
797
|
n_objects = len(objects)
|
|
708
798
|
n_diff = n_objects - n_offset_objects
|
|
709
799
|
ask_for_deletion = (
|
|
@@ -711,9 +801,13 @@ def check_storage_is_empty(
|
|
|
711
801
|
if raise_error
|
|
712
802
|
else "consider deleting them"
|
|
713
803
|
)
|
|
804
|
+
hint = "'./lamindb/_is_initialized' "
|
|
805
|
+
if n_offset_objects == 2:
|
|
806
|
+
hint += "& SQLite file"
|
|
807
|
+
hint += " ignored"
|
|
714
808
|
message = (
|
|
715
|
-
f"Storage
|
|
716
|
-
f"({
|
|
809
|
+
f"Storage {directory_string} contains {n_objects} objects "
|
|
810
|
+
f"({hint}) - {ask_for_deletion}\n{objects}"
|
|
717
811
|
)
|
|
718
812
|
if n_diff > 0:
|
|
719
813
|
if raise_error:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: lamindb_setup
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.71.0
|
|
4
4
|
Summary: Setup & configure LaminDB.
|
|
5
5
|
Author-email: Lamin Labs <laminlabs@gmail.com>
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
@@ -14,7 +14,9 @@ Requires-Dist: requests
|
|
|
14
14
|
Requires-Dist: universal_pathlib==0.1.4
|
|
15
15
|
Requires-Dist: botocore<2.0.0
|
|
16
16
|
Requires-Dist: supabase==2.2.1
|
|
17
|
-
Requires-Dist:
|
|
17
|
+
Requires-Dist: urllib3<2 ; extra == "aws"
|
|
18
|
+
Requires-Dist: aiobotocore[boto3]>=2.5.4,<3.0.0 ; extra == "aws"
|
|
19
|
+
Requires-Dist: s3fs>=2023.12.2,<=2024.3.1 ; extra == "aws"
|
|
18
20
|
Requires-Dist: pyjwt<3.0.0 ; extra == "dev"
|
|
19
21
|
Requires-Dist: psycopg2-binary ; extra == "dev"
|
|
20
22
|
Requires-Dist: python-dotenv ; extra == "dev"
|
|
@@ -25,12 +27,12 @@ Requires-Dist: pytest-xdist ; extra == "dev"
|
|
|
25
27
|
Requires-Dist: nbproject-test>=0.4.3 ; extra == "dev"
|
|
26
28
|
Requires-Dist: pandas ; extra == "dev"
|
|
27
29
|
Requires-Dist: django-schema-graph ; extra == "erdiagram"
|
|
28
|
-
Requires-Dist:
|
|
30
|
+
Requires-Dist: gcsfs>=2023.12.2,<=2024.3.1 ; extra == "gcp"
|
|
29
31
|
Project-URL: Home, https://github.com/laminlabs/lamindb-setup
|
|
30
32
|
Provides-Extra: aws
|
|
31
33
|
Provides-Extra: dev
|
|
32
34
|
Provides-Extra: erdiagram
|
|
33
|
-
Provides-Extra:
|
|
35
|
+
Provides-Extra: gcp
|
|
34
36
|
|
|
35
37
|
[](https://codecov.io/gh/laminlabs/lamindb-setup)
|
|
36
38
|
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
lamindb_setup/__init__.py,sha256=6a3FROJtySXqU0zXTq1xIniLLPzc7_JufrXPqKJ11eI,1542
|
|
2
|
+
lamindb_setup/_add_remote_storage.py,sha256=j-p29pxrr_07Ns5X2tm6yKxa3VESPkjyt35X4Cvr-nk,1325
|
|
3
|
+
lamindb_setup/_cache.py,sha256=wA7mbysANwe8hPNbjDo9bOmXJ0xIyaS5iyxIpxSWji4,846
|
|
4
|
+
lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
|
|
5
|
+
lamindb_setup/_check_setup.py,sha256=cNEL9Q4yPpmEkGKHH8JgullWl1VUZwALJ4RHn9wZypY,2613
|
|
6
|
+
lamindb_setup/_close.py,sha256=1QS9p2SCacgovYn6xqWU4zFvwHN1RgIccvzwJgFvKgU,1186
|
|
7
|
+
lamindb_setup/_connect_instance.py,sha256=ElpbjSaAI9vq3OfmPM8TxvsqfrZyytTZSRyYUDfiv3E,11944
|
|
8
|
+
lamindb_setup/_delete.py,sha256=tOEb8N7Ga9hrMpup2zBugtl0wN_Xxv0MqqKP2CpHArQ,7267
|
|
9
|
+
lamindb_setup/_django.py,sha256=EoyWvFzH0i9wxjy4JZhcoXCTckztP_Mrl6FbYQnMmLE,1534
|
|
10
|
+
lamindb_setup/_exportdb.py,sha256=uTIZjKKTB7arzEr1j0O6lONiT2pRBKeOFdLvOV8ZwzE,2120
|
|
11
|
+
lamindb_setup/_importdb.py,sha256=yYYShzUajTsR-cTW4CZ-UNDWZY2uE5PAgNbp-wn8Ogc,1874
|
|
12
|
+
lamindb_setup/_init_instance.py,sha256=XLf-xkF_YBwvT_sGyW1wFJFbOWpTFABPRt2oeIICQNI,11825
|
|
13
|
+
lamindb_setup/_migrate.py,sha256=4nBTFg5-BK4A2gH-D3_tcFf8EtvMnIo5Mq0e_C6_9-U,8815
|
|
14
|
+
lamindb_setup/_register_instance.py,sha256=Jeu0wyvJVSVQ_n-A_7yn7xOZIP0ncJD92DRABqzPIjA,940
|
|
15
|
+
lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
|
|
16
|
+
lamindb_setup/_setup_user.py,sha256=6Oc7Rke-yRQSZbuntdUAz8QbJ6UuPzYHI9FnYlf_q-A,3670
|
|
17
|
+
lamindb_setup/_silence_loggers.py,sha256=AKF_YcHvX32eGXdsYK8MJlxEaZ-Uo2f6QDRzjKFCtws,1568
|
|
18
|
+
lamindb_setup/core/__init__.py,sha256=dV9S-rQpNK9JcBn4hiEmiLnmNqfpPFJD9pqagMCaIew,416
|
|
19
|
+
lamindb_setup/core/_aws_storage.py,sha256=nEjeUv4xUVpoV0Lx-zjjmyb9w804bDyaeiM-OqbfwM0,1799
|
|
20
|
+
lamindb_setup/core/_deprecated.py,sha256=3qxUI1dnDlSeR0BYrv7ucjqRBEojbqotPgpShXs4KF8,2520
|
|
21
|
+
lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
|
|
22
|
+
lamindb_setup/core/_hub_client.py,sha256=V0qKDsCdRn-tQy2YIk4VgXcpJFmuum6N3gcavAC7gBQ,5504
|
|
23
|
+
lamindb_setup/core/_hub_core.py,sha256=88EZA5G2JLlEZP6ol_km3Cv0dEpV2U2Z6X5vQOorGrs,15776
|
|
24
|
+
lamindb_setup/core/_hub_crud.py,sha256=m8DgbWIBZBLWB-PV_UNBldUebqnxABfJfrqvzgr3t5s,4662
|
|
25
|
+
lamindb_setup/core/_hub_utils.py,sha256=b_M1LkdCjiMWm1EOlSb9GuPdLijwVgQDtATTpeZuXI0,1875
|
|
26
|
+
lamindb_setup/core/_settings.py,sha256=jjZ_AxRXB3Y3UP6m04BAw_dhFbJbdg2-nZWmEv2LNZ8,3141
|
|
27
|
+
lamindb_setup/core/_settings_instance.py,sha256=RFUcnBBUp303dbVEHcAaIm_q7lzlWg56OrKLwdam8Pg,16588
|
|
28
|
+
lamindb_setup/core/_settings_load.py,sha256=0MkFvZa0fVrMt087GL8Y-mHH44RUDQYybKqqj-c8flU,3849
|
|
29
|
+
lamindb_setup/core/_settings_save.py,sha256=sLVNEMsrEO238Px8P8PsQjl4_RxGSKwRqB9Cffg2TrY,2637
|
|
30
|
+
lamindb_setup/core/_settings_storage.py,sha256=yN2KNd6lq1RDNYbnkaCIfoYc3uKqkSU70otUr4qALPM,12739
|
|
31
|
+
lamindb_setup/core/_settings_store.py,sha256=OMBn396BFkMXGg3came1pToBqnBUwpyqBFv2G8GARiM,2043
|
|
32
|
+
lamindb_setup/core/_settings_user.py,sha256=P2lC4WDRAFfT-Xq3MlXJ-wMKIHCoGNhMTQfRGIAyUNQ,1344
|
|
33
|
+
lamindb_setup/core/_setup_bionty_sources.py,sha256=OgPpZxN2_Wffy-ogEBz_97c_k8d2bD-DDVt89-u9GLY,3002
|
|
34
|
+
lamindb_setup/core/cloud_sqlite_locker.py,sha256=NIBNAGq7TTRrip9OzMdiQKj8QOuwhL9esyM0aehUqBA,6893
|
|
35
|
+
lamindb_setup/core/django.py,sha256=m0AKg2lJ1EYCtEtZ8frFFJbAR9qX0gnFcgqp7aeC2k0,3450
|
|
36
|
+
lamindb_setup/core/exceptions.py,sha256=eoI7AXgATgDVzgArtN7CUvpaMUC067vsBg5LHCsWzDM,305
|
|
37
|
+
lamindb_setup/core/hashing.py,sha256=mv9UCvAsSrdHYQAv3Kz7UOvjd5tIjvDTIYv_ettBuVY,2218
|
|
38
|
+
lamindb_setup/core/types.py,sha256=bcYnZ0uM_2NXKJCl94Mmc-uYrQlRUUVKG3sK2N-F-N4,532
|
|
39
|
+
lamindb_setup/core/upath.py,sha256=TcpBLVtI2aImJWn-FIMMThQt8i_ztJdETQ_Wj4Ow4qY,27965
|
|
40
|
+
lamindb_setup-0.71.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
|
|
41
|
+
lamindb_setup-0.71.0.dist-info/WHEEL,sha256=Sgu64hAMa6g5FdzHxXv9Xdse9yxpGGMeagVtPMWpJQY,99
|
|
42
|
+
lamindb_setup-0.71.0.dist-info/METADATA,sha256=yQ8BtbUkb8PMmdgno_coRXNadR3Ib4ikzV6pTW7TSqo,1620
|
|
43
|
+
lamindb_setup-0.71.0.dist-info/RECORD,,
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
lamindb_setup/__init__.py,sha256=yTrf2HNpLb9hFr44mDAEjKjXDwrKYs_n2wX6JmyjOkQ,1558
|
|
2
|
-
lamindb_setup/_add_remote_storage.py,sha256=S2rFvn-JcGaBOiBNQoAiaTPM590GQh-g4OJeNsben0Q,1802
|
|
3
|
-
lamindb_setup/_cache.py,sha256=wrwlHi2IfxcWDfOW-cHzzkQNlLvasoQPPNxihICbwew,809
|
|
4
|
-
lamindb_setup/_check.py,sha256=xgc5kF2HA4lGuAA9FJT9BJyQV5z1EwYMr92RTbGcQg4,92
|
|
5
|
-
lamindb_setup/_check_setup.py,sha256=LLdTn4JYiyWKji_sexq2eENbmvA6_gdxqMKjDnF-nJc,2543
|
|
6
|
-
lamindb_setup/_close.py,sha256=XtP7uyKRbwCGkXdSkwuiEnLYn5PEN3t3q50dPHNJRSU,1150
|
|
7
|
-
lamindb_setup/_connect_instance.py,sha256=XzHbYPmzPmAJZ1EaFtDBzw2skh1s0tppKCg2HOxx1HQ,11908
|
|
8
|
-
lamindb_setup/_delete.py,sha256=aIfQpOmb-icGsNVOd9EhHGGBQvsSJF7fBW_fzSwPhuM,5861
|
|
9
|
-
lamindb_setup/_django.py,sha256=Yv-bNj80I5Fj9AJ2BFgcv9JZg4WUrlh3UStty82_jXQ,1500
|
|
10
|
-
lamindb_setup/_exportdb.py,sha256=f5XUhRxTsU-yqfPM92OxjZKRxJo3lKaqYTRh9oytDfY,2084
|
|
11
|
-
lamindb_setup/_importdb.py,sha256=bxFZNjtVOAW4yeMQakVYZE4-WNi4QE-MdMynHShMOgI,1836
|
|
12
|
-
lamindb_setup/_init_instance.py,sha256=DlVHpz8avTYaL-IoxSO4XM1rfZmtZXsWpjtY95vYmK0,11480
|
|
13
|
-
lamindb_setup/_migrate.py,sha256=wVDfIN1z2OIJztrenpYx42XN09retbPek49TzzmFQ6I,8810
|
|
14
|
-
lamindb_setup/_register_instance.py,sha256=1cZXXuCf6cqbiDchmW5vYNTf34wMNSZ2QtHmIJbTT4c,796
|
|
15
|
-
lamindb_setup/_schema.py,sha256=edhbiUYatbO5FfBA6PYRmVmYWbtBXZBpATa70_MZiWo,642
|
|
16
|
-
lamindb_setup/_setup_user.py,sha256=5zZ924ckd8CGBiD4-5bdTN-NUNMS7N5w8Ovy6JIYPMU,3658
|
|
17
|
-
lamindb_setup/_silence_loggers.py,sha256=auXnYVSx2ErZmyXZJ7FYYA69UQgUDQ5SnuznFXzfaWs,1548
|
|
18
|
-
lamindb_setup/core/__init__.py,sha256=xfhu3cO_zPtNztmu6vlWtdbX92gPpnjyff5dOqCHTvE,369
|
|
19
|
-
lamindb_setup/core/_aws_storage.py,sha256=GIT9SagV-1oFlLpRIogFgXuTwzMZsppFjqKA4-Vp66s,1762
|
|
20
|
-
lamindb_setup/core/_deprecated.py,sha256=DNO_90uyYGM9on7rzJCu9atS7Z1qOQypsuL9PI6I-OQ,2491
|
|
21
|
-
lamindb_setup/core/_docs.py,sha256=0ha3cZqzvIQnAXVC_JlAiRrFi3VpEEVookP_oURhBr4,240
|
|
22
|
-
lamindb_setup/core/_hub_client.py,sha256=H09wJRck4d3fQ8VBSK1v8Dbj9RfqLdW9UqoTs6OCyUM,5520
|
|
23
|
-
lamindb_setup/core/_hub_core.py,sha256=Y_RnT82Y7fHgMmNvG8NTDTnx5Ybf_IwFQSMl6AFveVg,11294
|
|
24
|
-
lamindb_setup/core/_hub_crud.py,sha256=0qKKsnHlx_FceX-jahdS2vJc2IuGRzouQrtAMn6MTSA,4505
|
|
25
|
-
lamindb_setup/core/_hub_utils.py,sha256=4bjtm4QVivDwtVq3Bx9hQFZtdU2WmqrVdpupdvxMHcM,1862
|
|
26
|
-
lamindb_setup/core/_settings.py,sha256=ogeOddaJ9tiolzpGwUIu6i4vhcViXQvsREEzZ95uyvI,3241
|
|
27
|
-
lamindb_setup/core/_settings_instance.py,sha256=nAaTW4kd_wYM5hKlRwpLgBd-ATiEDvtHOX1NCnpkFyQ,13927
|
|
28
|
-
lamindb_setup/core/_settings_load.py,sha256=RkH_zfdGAlBO-4sovmFNGSP56XhSJpUYdvlZlNh-Qqs,3745
|
|
29
|
-
lamindb_setup/core/_settings_save.py,sha256=QnUQsYMAXv9xUYAsQ-5vldntSnAvNb3TMmJ61J3kU_I,2571
|
|
30
|
-
lamindb_setup/core/_settings_storage.py,sha256=Y2FsDmOFW7Kz1bh2MAMyQkWPJjoYN12ObpqVp82qRag,11750
|
|
31
|
-
lamindb_setup/core/_settings_store.py,sha256=iM3Ei-YTSQo6wbcUhOliQx_KmHo8sxDx0-elJPt9AwQ,1929
|
|
32
|
-
lamindb_setup/core/_settings_user.py,sha256=ouRhbnIN12cjvVR0Iz98hUyYyGbqcGNRb1OCeI2T6wo,1282
|
|
33
|
-
lamindb_setup/core/_setup_bionty_sources.py,sha256=xhIIJqtCzrjTONzU_68bq5kQxsNRDfAprHi2oJvY_LY,2908
|
|
34
|
-
lamindb_setup/core/cloud_sqlite_locker.py,sha256=nFwAupnbIy_sxzlJBayfD2Dh8ZMUGNxbspIbv6BGrp8,6835
|
|
35
|
-
lamindb_setup/core/django.py,sha256=bKwuv83ubKVsMeunnFtpWd1C-y29I2cuvpW10rCPpZo,3411
|
|
36
|
-
lamindb_setup/core/exceptions.py,sha256=YzkJA51ZAa2w9lawAggSdg-NA-EIYJGNRnFd0DN3_bE,275
|
|
37
|
-
lamindb_setup/core/hashing.py,sha256=nbQO_CSiX09O4bznHABft94fdFZAHPm2NIyvH5xagoo,2011
|
|
38
|
-
lamindb_setup/core/types.py,sha256=fR71SLjoN1MkCjx8TJcjYDgmgO4bPXBX5J_RKpmmy_o,497
|
|
39
|
-
lamindb_setup/core/upath.py,sha256=v8lPEA5j_SH0QzrHfw02x3HDt1ikFVc3gdoCCVfwfmw,25078
|
|
40
|
-
lamindb_setup-0.70.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
|
|
41
|
-
lamindb_setup-0.70.0.dist-info/WHEEL,sha256=Sgu64hAMa6g5FdzHxXv9Xdse9yxpGGMeagVtPMWpJQY,99
|
|
42
|
-
lamindb_setup-0.70.0.dist-info/METADATA,sha256=9hjTjaw1xv6SQR_ugZg7DMKxGu0SqgJy0b9VJuR2rVw,1469
|
|
43
|
-
lamindb_setup-0.70.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|