lamindb_setup 0.70.0__py2.py3-none-any.whl → 0.71.0__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. lamindb_setup/__init__.py +15 -15
  2. lamindb_setup/_add_remote_storage.py +22 -33
  3. lamindb_setup/_cache.py +4 -1
  4. lamindb_setup/_check.py +3 -0
  5. lamindb_setup/_check_setup.py +13 -7
  6. lamindb_setup/_close.py +2 -0
  7. lamindb_setup/_connect_instance.py +33 -26
  8. lamindb_setup/_delete.py +52 -19
  9. lamindb_setup/_django.py +4 -1
  10. lamindb_setup/_exportdb.py +4 -2
  11. lamindb_setup/_importdb.py +5 -1
  12. lamindb_setup/_init_instance.py +57 -45
  13. lamindb_setup/_migrate.py +16 -13
  14. lamindb_setup/_register_instance.py +10 -3
  15. lamindb_setup/_schema.py +6 -3
  16. lamindb_setup/_setup_user.py +7 -7
  17. lamindb_setup/_silence_loggers.py +4 -2
  18. lamindb_setup/core/__init__.py +4 -3
  19. lamindb_setup/core/_aws_storage.py +3 -0
  20. lamindb_setup/core/_deprecated.py +2 -7
  21. lamindb_setup/core/_docs.py +2 -0
  22. lamindb_setup/core/_hub_client.py +12 -10
  23. lamindb_setup/core/_hub_core.py +198 -88
  24. lamindb_setup/core/_hub_crud.py +15 -11
  25. lamindb_setup/core/_hub_utils.py +11 -8
  26. lamindb_setup/core/_settings.py +23 -26
  27. lamindb_setup/core/_settings_instance.py +149 -81
  28. lamindb_setup/core/_settings_load.py +12 -7
  29. lamindb_setup/core/_settings_save.py +11 -8
  30. lamindb_setup/core/_settings_storage.py +83 -42
  31. lamindb_setup/core/_settings_store.py +3 -2
  32. lamindb_setup/core/_settings_user.py +10 -6
  33. lamindb_setup/core/_setup_bionty_sources.py +9 -2
  34. lamindb_setup/core/cloud_sqlite_locker.py +13 -10
  35. lamindb_setup/core/django.py +3 -1
  36. lamindb_setup/core/exceptions.py +4 -2
  37. lamindb_setup/core/hashing.py +15 -5
  38. lamindb_setup/core/types.py +5 -2
  39. lamindb_setup/core/upath.py +181 -87
  40. {lamindb_setup-0.70.0.dist-info → lamindb_setup-0.71.0.dist-info}/METADATA +6 -4
  41. lamindb_setup-0.71.0.dist-info/RECORD +43 -0
  42. lamindb_setup-0.70.0.dist-info/RECORD +0 -43
  43. {lamindb_setup-0.70.0.dist-info → lamindb_setup-0.71.0.dist-info}/LICENSE +0 -0
  44. {lamindb_setup-0.70.0.dist-info → lamindb_setup-0.71.0.dist-info}/WHEEL +0 -0
@@ -4,21 +4,25 @@
4
4
  from __future__ import annotations
5
5
 
6
6
  import os
7
+ from collections import defaultdict
7
8
  from datetime import datetime, timezone
8
- import botocore.session
9
+ from functools import partial
10
+ from itertools import islice
9
11
  from pathlib import Path, PurePosixPath
10
- from typing import Literal, Dict
12
+ from typing import TYPE_CHECKING, Any, Literal
13
+
14
+ import botocore.session
11
15
  import fsspec
12
- from itertools import islice
13
- from typing import Optional, Set, Any, Tuple, List
14
- from collections import defaultdict
15
16
  from lamin_utils import logger
16
17
  from upath import UPath
17
- from upath.implementations.cloud import CloudPath, S3Path # noqa # keep CloudPath!
18
+ from upath.implementations.cloud import CloudPath, S3Path # keep CloudPath!
18
19
  from upath.implementations.local import LocalPath, PosixUPath, WindowsUPath
19
- from .types import UPathStr
20
+
20
21
  from .hashing import b16_to_b64, hash_md5s_from_dir
21
22
 
23
+ if TYPE_CHECKING:
24
+ from .types import UPathStr
25
+
22
26
  LocalPathClasses = (PosixUPath, WindowsUPath, LocalPath)
23
27
 
24
28
  # also see https://gist.github.com/securifera/e7eed730cbe1ce43d0c29d7cd2d582f4
@@ -57,7 +61,7 @@ VALID_SUFFIXES = {
57
61
  TRAILING_SEP = (os.sep, os.altsep) if os.altsep is not None else os.sep
58
62
 
59
63
 
60
- def extract_suffix_from_path(path: Path, arg_name: Optional[str] = None) -> str:
64
+ def extract_suffix_from_path(path: Path, arg_name: str | None = None) -> str:
61
65
  def process_digits(suffix: str):
62
66
  if suffix[1:].isdigit(): # :1 to skip the dot
63
67
  return "" # digits are no valid suffixes
@@ -141,44 +145,100 @@ def create_mapper(
141
145
  )
142
146
 
143
147
 
144
- def print_hook(size: int, value: int, **kwargs):
148
+ def print_hook(size: int, value: int, objectname: str, action: str):
145
149
  progress_in_percent = (value / size) * 100
146
- out = (
147
- f"... {kwargs['action']} {Path(kwargs['filepath']).name}:"
148
- f" {min(progress_in_percent, 100):4.1f}%"
149
- )
150
- if progress_in_percent >= 100:
151
- out += "\n"
150
+ out = f"... {action} {objectname}:" f" {min(progress_in_percent, 100):4.1f}%"
152
151
  if "NBPRJ_TEST_NBPATH" not in os.environ:
153
152
  print(out, end="\r")
154
153
 
155
154
 
156
155
  class ProgressCallback(fsspec.callbacks.Callback):
157
- def __init__(self, action: Literal["uploading", "downloading"]):
156
+ def __init__(
157
+ self,
158
+ objectname: str,
159
+ action: Literal["uploading", "downloading", "synchronizing"],
160
+ adjust_size: bool = False,
161
+ ):
162
+ assert action in {"uploading", "downloading", "synchronizing"}
163
+
158
164
  super().__init__()
165
+
159
166
  self.action = action
167
+ print_progress = partial(print_hook, objectname=objectname, action=action)
168
+ self.hooks = {"print_progress": print_progress}
169
+
170
+ self.adjust_size = adjust_size
171
+
172
+ def absolute_update(self, value):
173
+ pass
174
+
175
+ def relative_update(self, inc=1):
176
+ pass
177
+
178
+ def update_relative_value(self, inc=1):
179
+ self.value += inc
180
+ self.call()
160
181
 
161
182
  def branch(self, path_1, path_2, kwargs):
162
- kwargs["callback"] = fsspec.callbacks.Callback(
163
- hooks=dict(print_hook=print_hook), filepath=path_1, action=self.action
164
- )
183
+ if self.adjust_size:
184
+ if Path(path_2 if self.action != "uploading" else path_1).is_dir():
185
+ self.size -= 1
186
+ kwargs["callback"] = ChildProgressCallback(self)
187
+
188
+ def branched(self, path_1, path_2, **kwargs):
189
+ self.branch(path_1, path_2, kwargs)
190
+ return kwargs["callback"]
191
+
192
+ def wrap(self, iterable):
193
+ if self.adjust_size:
194
+ paths = []
195
+ for lpath, rpath in iterable:
196
+ paths.append((lpath, rpath))
197
+ if Path(lpath).is_dir():
198
+ self.size -= 1
199
+ self.adjust_size = False
200
+ return paths
201
+ else:
202
+ return iterable
203
+
204
+ @classmethod
205
+ def requires_progress(
206
+ cls,
207
+ maybe_callback: fsspec.callbacks.Callback | None,
208
+ print_progress: bool,
209
+ objectname: str,
210
+ action: Literal["uploading", "downloading", "synchronizing"],
211
+ **kwargs,
212
+ ):
213
+ if maybe_callback is None:
214
+ if print_progress:
215
+ return cls(objectname, action, **kwargs)
216
+ else:
217
+ return fsspec.callbacks.NoOpCallback()
218
+ return maybe_callback
165
219
 
166
- def call(self, *args, **kwargs):
167
- return None
220
+
221
+ class ChildProgressCallback(fsspec.callbacks.Callback):
222
+ def __init__(self, parent: ProgressCallback):
223
+ super().__init__()
224
+
225
+ self.parent = parent
226
+
227
+ def parent_update(self, inc=1):
228
+ self.parent.update_relative_value(inc)
229
+
230
+ def relative_update(self, inc=1):
231
+ self.parent_update(inc / self.size)
168
232
 
169
233
 
170
234
  def download_to(self, path: UPathStr, print_progress: bool = False, **kwargs):
171
235
  """Download to a path."""
172
- if print_progress:
173
- # can't do path.is_dir() because path doesn't exist
174
- # so assume any destination without a suffix is a dir
175
- # this is temporary until we have a proper progress bar for directories
176
- if os.path.splitext(path)[-1] not in {"", ".zrad", ".zarr"}:
177
- cb = ProgressCallback("downloading")
178
- else:
179
- # todo: make proper progress bar for directories
180
- cb = fsspec.callbacks.NoOpCallback()
181
- kwargs["callback"] = cb
236
+ if print_progress and "callback" not in kwargs:
237
+ callback = ProgressCallback(
238
+ PurePosixPath(path).name, "downloading", adjust_size=True
239
+ )
240
+ kwargs["callback"] = callback
241
+
182
242
  self.fs.download(str(self), str(path), **kwargs)
183
243
 
184
244
 
@@ -190,20 +250,16 @@ def upload_from(
190
250
  **kwargs,
191
251
  ):
192
252
  """Upload from a local path."""
193
- path_is_dir = os.path.isdir(path)
253
+ path = Path(path)
254
+ path_is_dir = path.is_dir()
194
255
  if not path_is_dir:
195
256
  dir_inplace = False
196
257
 
197
- if print_progress:
198
- if not path_is_dir:
199
- cb = ProgressCallback("uploading")
200
- else:
201
- # todo: make proper progress bar for directories
202
- cb = fsspec.callbacks.NoOpCallback()
203
- kwargs["callback"] = cb
258
+ if print_progress and "callback" not in kwargs:
259
+ callback = ProgressCallback(path.name, "uploading")
260
+ kwargs["callback"] = callback
204
261
 
205
262
  if dir_inplace:
206
- path = Path(path)
207
263
  source = [f for f in path.rglob("*") if f.is_file()]
208
264
  destination = [str(self / f.relative_to(path)) for f in source]
209
265
  source = [str(f) for f in source] # type: ignore
@@ -233,7 +289,14 @@ def upload_from(
233
289
  del self.fs.dircache[bucket]
234
290
 
235
291
 
236
- def synchronize(self, objectpath: Path, error_no_origin: bool = True, **kwargs):
292
+ def synchronize(
293
+ self,
294
+ objectpath: Path,
295
+ error_no_origin: bool = True,
296
+ print_progress: bool = False,
297
+ callback: fsspec.callbacks.Callback | None = None,
298
+ **kwargs,
299
+ ):
237
300
  """Sync to a local destination path."""
238
301
  # optimize the number of network requests
239
302
  if "timestamp" in kwargs:
@@ -292,15 +355,23 @@ def synchronize(self, objectpath: Path, error_no_origin: bool = True, **kwargs):
292
355
  destination_exists = False
293
356
  need_synchronize = True
294
357
  if need_synchronize:
358
+ callback = ProgressCallback.requires_progress(
359
+ callback, print_progress, objectpath.name, "synchronizing"
360
+ )
361
+ callback.set_size(len(files))
295
362
  origin_file_keys = []
296
- for file, stat in files.items():
297
- destination = PurePosixPath(file).relative_to(self.path)
298
- origin_file_keys.append(destination.as_posix())
363
+ for file, stat in callback.wrap(files.items()):
364
+ file_key = PurePosixPath(file).relative_to(self.path)
365
+ origin_file_keys.append(file_key.as_posix())
299
366
  timestamp = stat[modified_key].timestamp()
300
- origin = UPath(f"{self.protocol}://{file}", **self._kwargs)
301
- origin.synchronize(
302
- objectpath / destination, timestamp=timestamp, **kwargs
367
+
368
+ origin = f"{self.protocol}://{file}"
369
+ destination = objectpath / file_key
370
+ child = callback.branched(origin, destination.as_posix())
371
+ UPath(origin, **self._kwargs).synchronize(
372
+ destination, timestamp=timestamp, callback=child, **kwargs
303
373
  )
374
+ child.close()
304
375
  if destination_exists:
305
376
  local_files = [file for file in objectpath.rglob("*") if file.is_file()]
306
377
  if len(local_files) > len(files):
@@ -316,6 +387,10 @@ def synchronize(self, objectpath: Path, error_no_origin: bool = True, **kwargs):
316
387
  return None
317
388
 
318
389
  # synchronization logic for files
390
+ callback = ProgressCallback.requires_progress(
391
+ callback, print_progress, objectpath.name, "synchronizing"
392
+ )
393
+ kwargs["callback"] = callback
319
394
  if objectpath.exists():
320
395
  local_mts = objectpath.stat().st_mtime # type: ignore
321
396
  need_synchronize = cloud_mts > local_mts
@@ -325,9 +400,13 @@ def synchronize(self, objectpath: Path, error_no_origin: bool = True, **kwargs):
325
400
  if need_synchronize:
326
401
  self.download_to(objectpath, **kwargs)
327
402
  os.utime(objectpath, times=(cloud_mts, cloud_mts))
403
+ else:
404
+ # nothing happens if parent_update is not defined
405
+ # because of Callback.no_op
406
+ callback.parent_update()
328
407
 
329
408
 
330
- def modified(self) -> Optional[datetime]:
409
+ def modified(self) -> datetime | None:
331
410
  """Return modified time stamp."""
332
411
  mtime = self.fs.modified(str(self))
333
412
  if mtime.tzinfo is None:
@@ -340,15 +419,15 @@ def compute_file_tree(
340
419
  *,
341
420
  level: int = -1,
342
421
  only_dirs: bool = False,
343
- limit: int = 1000,
344
- include_paths: Optional[Set[Any]] = None,
345
- skip_suffixes: Optional[List[str]] = None,
346
- ) -> Tuple[str, int]:
422
+ n_max_files_per_dir_and_type: int = 100,
423
+ n_max_files: int = 1000,
424
+ include_paths: set[Any] | None = None,
425
+ skip_suffixes: list[str] | None = None,
426
+ ) -> tuple[str, int]:
347
427
  space = " "
348
428
  branch = "│ "
349
429
  tee = "├── "
350
430
  last = "└── "
351
- max_files_per_dir_per_type = 7
352
431
  if skip_suffixes is None:
353
432
  skip_suffixes_tuple = ()
354
433
  else:
@@ -382,14 +461,14 @@ def compute_file_tree(
382
461
  if only_dirs:
383
462
  contents = [d for d in contents if d.is_dir()]
384
463
  pointers = [tee] * (len(contents) - 1) + [last]
385
- n_files_per_dir_per_type = defaultdict(lambda: 0) # type: ignore
386
- for pointer, child_path in zip(pointers, contents):
464
+ n_files_per_dir_and_type = defaultdict(lambda: 0) # type: ignore
465
+ for pointer, child_path in zip(pointers, contents, strict=False): # type: ignore
387
466
  if child_path.is_dir():
388
467
  if include_dirs and child_path not in include_dirs:
389
468
  continue
390
469
  yield prefix + pointer + child_path.name
391
470
  n_directories += 1
392
- n_files_per_dir_per_type = defaultdict(lambda: 0)
471
+ n_files_per_dir_and_type = defaultdict(lambda: 0)
393
472
  extension = branch if pointer == tee else space
394
473
  yield from inner(child_path, prefix=prefix + extension, level=level - 1)
395
474
  elif not only_dirs:
@@ -397,21 +476,21 @@ def compute_file_tree(
397
476
  continue
398
477
  suffix = extract_suffix_from_path(child_path)
399
478
  suffixes.add(suffix)
400
- n_files_per_dir_per_type[suffix] += 1
479
+ n_files_per_dir_and_type[suffix] += 1
401
480
  n_objects += 1
402
- if n_files_per_dir_per_type[suffix] == max_files_per_dir_per_type:
481
+ if n_files_per_dir_and_type[suffix] == n_max_files_per_dir_and_type:
403
482
  yield prefix + "..."
404
- elif n_files_per_dir_per_type[suffix] > max_files_per_dir_per_type:
483
+ elif n_files_per_dir_and_type[suffix] > n_max_files_per_dir_and_type:
405
484
  continue
406
485
  else:
407
486
  yield prefix + pointer + child_path.name
408
487
 
409
488
  folder_tree = ""
410
489
  iterator = inner(path, level=level)
411
- for line in islice(iterator, limit):
490
+ for line in islice(iterator, n_max_files):
412
491
  folder_tree += f"\n{line}"
413
492
  if next(iterator, None):
414
- folder_tree += f"\n... only showing {limit} out of {n_objects} files"
493
+ folder_tree += f"\n... only showing {n_max_files} out of {n_objects} files"
415
494
  directory_info = "directory" if n_directories == 1 else "directories"
416
495
  display_suffixes = ", ".join([f"{suffix!r}" for suffix in suffixes])
417
496
  suffix_message = f" with suffixes {display_suffixes}" if n_objects > 0 else ""
@@ -426,11 +505,12 @@ def compute_file_tree(
426
505
  def view_tree(
427
506
  path: Path,
428
507
  *,
429
- level: int = -1,
508
+ level: int = 2,
430
509
  only_dirs: bool = False,
431
- limit: int = 1000,
432
- include_paths: Optional[Set[Any]] = None,
433
- skip_suffixes: Optional[List[str]] = None,
510
+ n_max_files_per_dir_and_type: int = 100,
511
+ n_max_files: int = 1000,
512
+ include_paths: set[Any] | None = None,
513
+ skip_suffixes: list[str] | None = None,
434
514
  ) -> None:
435
515
  """Print a visual tree structure of files & directories.
436
516
 
@@ -438,7 +518,7 @@ def view_tree(
438
518
  level: If `1`, only iterate through one level, if `2` iterate through 2
439
519
  levels, if `-1` iterate through entire hierarchy.
440
520
  only_dirs: Only iterate through directories.
441
- limit: Display limit. Will only show this many files. Doesn't affect count.
521
+ n_max_files: Display limit. Will only show this many files. Doesn't affect count.
442
522
  include_paths: Restrict to these paths.
443
523
  skip_suffixes: Skip directories with these suffixes.
444
524
 
@@ -472,7 +552,8 @@ def view_tree(
472
552
  path,
473
553
  level=level,
474
554
  only_dirs=only_dirs,
475
- limit=limit,
555
+ n_max_files=n_max_files,
556
+ n_max_files_per_dir_and_type=n_max_files_per_dir_and_type,
476
557
  include_paths=include_paths,
477
558
  skip_suffixes=skip_suffixes,
478
559
  )
@@ -497,9 +578,10 @@ def to_url(upath):
497
578
  bucket = upath._url.netloc
498
579
  if bucket == "scverse-spatial-eu-central-1":
499
580
  region = "eu-central-1"
500
- elif f"s3://{bucket}" not in hosted_buckets:
501
- metadata = upath.fs.call_s3("head_bucket", Bucket=upath._url.netloc)
502
- region = metadata["BucketRegion"]
581
+ elif f"s3://{bucket}" not in HOSTED_BUCKETS:
582
+ response = upath.fs.call_s3("head_bucket", Bucket=upath._url.netloc)
583
+ headers = response["ResponseMetadata"]["HTTPHeaders"]
584
+ region = headers.get("x-amz-bucket-region")
503
585
  else:
504
586
  region = bucket.replace("lamin_", "")
505
587
  if region == "us-east-1":
@@ -578,7 +660,7 @@ def convert_pathlike(pathlike: UPathStr) -> UPath:
578
660
  return path
579
661
 
580
662
 
581
- hosted_regions = [
663
+ HOSTED_REGIONS = [
582
664
  "eu-central-1",
583
665
  "eu-west-2",
584
666
  "us-east-1",
@@ -588,16 +670,16 @@ hosted_regions = [
588
670
  ]
589
671
  lamin_env = os.getenv("LAMIN_ENV")
590
672
  if lamin_env is None or lamin_env == "prod":
591
- hosted_buckets_list = [f"s3://lamin-{region}" for region in hosted_regions]
673
+ hosted_buckets_list = [f"s3://lamin-{region}" for region in HOSTED_REGIONS]
592
674
  hosted_buckets_list.append("s3://scverse-spatial-eu-central-1")
593
- hosted_buckets = tuple(hosted_buckets_list)
675
+ HOSTED_BUCKETS = tuple(hosted_buckets_list)
594
676
  else:
595
- hosted_buckets = ("s3://lamin-hosted-test",) # type: ignore
596
- credentials_cache: Dict[str, Dict[str, str]] = {}
677
+ HOSTED_BUCKETS = ("s3://lamin-hosted-test",) # type: ignore
678
+ credentials_cache: dict[str, dict[str, str]] = {}
597
679
  AWS_CREDENTIALS_PRESENT = None
598
680
 
599
681
 
600
- def create_path(path: UPath, access_token: Optional[str] = None) -> UPath:
682
+ def create_path(path: UPath, access_token: str | None = None) -> UPath:
601
683
  path = convert_pathlike(path)
602
684
  # test whether we have an AWS S3 path
603
685
  if not isinstance(path, S3Path):
@@ -611,9 +693,8 @@ def create_path(path: UPath, access_token: Optional[str] = None) -> UPath:
611
693
  if path.fs.key is not None and path.fs.secret is not None:
612
694
  anon = False
613
695
  else:
614
- # we can do
615
- # path.fs.connect()
616
- # and check path.fs.session._credentials, but it is slower
696
+ # we could do path.fs.connect()
697
+ # and check path.fs.session._credentials, but it'd be slower
617
698
  session = botocore.session.get_session()
618
699
  credentials = session.get_credentials()
619
700
  if credentials is None or credentials.access_key is None:
@@ -625,7 +706,7 @@ def create_path(path: UPath, access_token: Optional[str] = None) -> UPath:
625
706
 
626
707
  # test whether we are on hosted storage or not
627
708
  path_str = path.as_posix()
628
- is_hosted_storage = path_str.startswith(hosted_buckets)
709
+ is_hosted_storage = path_str.startswith(HOSTED_BUCKETS)
629
710
 
630
711
  if not is_hosted_storage:
631
712
  # make anon request if no credentials present
@@ -652,7 +733,7 @@ def create_path(path: UPath, access_token: Optional[str] = None) -> UPath:
652
733
  )
653
734
 
654
735
 
655
- def get_stat_file_cloud(stat: Dict) -> Tuple[int, str, str]:
736
+ def get_stat_file_cloud(stat: dict) -> tuple[int, str, str]:
656
737
  size = stat["size"]
657
738
  # small files
658
739
  if "-" not in stat["ETag"]:
@@ -669,7 +750,7 @@ def get_stat_file_cloud(stat: Dict) -> Tuple[int, str, str]:
669
750
  return size, hash, hash_type
670
751
 
671
752
 
672
- def get_stat_dir_cloud(path: UPath) -> Tuple[int, str, str, int]:
753
+ def get_stat_dir_cloud(path: UPath) -> tuple[int, str, str, int]:
673
754
  sizes = []
674
755
  md5s = []
675
756
  objects = path.fs.find(path.as_posix(), detail=True)
@@ -701,9 +782,18 @@ def check_storage_is_empty(
701
782
  # since path.fs.find raises a PermissionError on empty hosted
702
783
  # subdirectories (see lamindb_setup/core/_settings_storage/init_storage).
703
784
  n_offset_objects = 1 # because of touched dummy file, see mark_storage_root()
704
- if account_for_sqlite_file:
705
- n_offset_objects += 1 # because of SQLite file
706
- objects = root_upath.fs.find(root_string)
785
+ if root_string.startswith(HOSTED_BUCKETS):
786
+ # in hosted buckets, count across entire root
787
+ directory_string = root_string
788
+ # the SQLite file is not in the ".lamindb" directory
789
+ if account_for_sqlite_file:
790
+ n_offset_objects += 1 # because of SQLite file
791
+ else:
792
+ # in any other storage location, only count in .lamindb
793
+ if not root_string.endswith("/"):
794
+ root_string += "/"
795
+ directory_string = root_string + ".lamindb"
796
+ objects = root_upath.fs.find(directory_string)
707
797
  n_objects = len(objects)
708
798
  n_diff = n_objects - n_offset_objects
709
799
  ask_for_deletion = (
@@ -711,9 +801,13 @@ def check_storage_is_empty(
711
801
  if raise_error
712
802
  else "consider deleting them"
713
803
  )
804
+ hint = "'./lamindb/_is_initialized' "
805
+ if n_offset_objects == 2:
806
+ hint += "& SQLite file"
807
+ hint += " ignored"
714
808
  message = (
715
- f"Storage location contains {n_objects} objects "
716
- f"({n_offset_objects} ignored) - {ask_for_deletion}\n{objects}"
809
+ f"Storage {directory_string} contains {n_objects} objects "
810
+ f"({hint}) - {ask_for_deletion}\n{objects}"
717
811
  )
718
812
  if n_diff > 0:
719
813
  if raise_error:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: lamindb_setup
3
- Version: 0.70.0
3
+ Version: 0.71.0
4
4
  Summary: Setup & configure LaminDB.
5
5
  Author-email: Lamin Labs <laminlabs@gmail.com>
6
6
  Description-Content-Type: text/markdown
@@ -14,7 +14,9 @@ Requires-Dist: requests
14
14
  Requires-Dist: universal_pathlib==0.1.4
15
15
  Requires-Dist: botocore<2.0.0
16
16
  Requires-Dist: supabase==2.2.1
17
- Requires-Dist: s3fs ; extra == "aws"
17
+ Requires-Dist: urllib3<2 ; extra == "aws"
18
+ Requires-Dist: aiobotocore[boto3]>=2.5.4,<3.0.0 ; extra == "aws"
19
+ Requires-Dist: s3fs>=2023.12.2,<=2024.3.1 ; extra == "aws"
18
20
  Requires-Dist: pyjwt<3.0.0 ; extra == "dev"
19
21
  Requires-Dist: psycopg2-binary ; extra == "dev"
20
22
  Requires-Dist: python-dotenv ; extra == "dev"
@@ -25,12 +27,12 @@ Requires-Dist: pytest-xdist ; extra == "dev"
25
27
  Requires-Dist: nbproject-test>=0.4.3 ; extra == "dev"
26
28
  Requires-Dist: pandas ; extra == "dev"
27
29
  Requires-Dist: django-schema-graph ; extra == "erdiagram"
28
- Requires-Dist: faker ; extra == "hub"
30
+ Requires-Dist: gcsfs>=2023.12.2,<=2024.3.1 ; extra == "gcp"
29
31
  Project-URL: Home, https://github.com/laminlabs/lamindb-setup
30
32
  Provides-Extra: aws
31
33
  Provides-Extra: dev
32
34
  Provides-Extra: erdiagram
33
- Provides-Extra: hub
35
+ Provides-Extra: gcp
34
36
 
35
37
  [![codecov](https://codecov.io/gh/laminlabs/lamindb-setup/branch/main/graph/badge.svg)](https://codecov.io/gh/laminlabs/lamindb-setup)
36
38
 
@@ -0,0 +1,43 @@
1
+ lamindb_setup/__init__.py,sha256=6a3FROJtySXqU0zXTq1xIniLLPzc7_JufrXPqKJ11eI,1542
2
+ lamindb_setup/_add_remote_storage.py,sha256=j-p29pxrr_07Ns5X2tm6yKxa3VESPkjyt35X4Cvr-nk,1325
3
+ lamindb_setup/_cache.py,sha256=wA7mbysANwe8hPNbjDo9bOmXJ0xIyaS5iyxIpxSWji4,846
4
+ lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
5
+ lamindb_setup/_check_setup.py,sha256=cNEL9Q4yPpmEkGKHH8JgullWl1VUZwALJ4RHn9wZypY,2613
6
+ lamindb_setup/_close.py,sha256=1QS9p2SCacgovYn6xqWU4zFvwHN1RgIccvzwJgFvKgU,1186
7
+ lamindb_setup/_connect_instance.py,sha256=ElpbjSaAI9vq3OfmPM8TxvsqfrZyytTZSRyYUDfiv3E,11944
8
+ lamindb_setup/_delete.py,sha256=tOEb8N7Ga9hrMpup2zBugtl0wN_Xxv0MqqKP2CpHArQ,7267
9
+ lamindb_setup/_django.py,sha256=EoyWvFzH0i9wxjy4JZhcoXCTckztP_Mrl6FbYQnMmLE,1534
10
+ lamindb_setup/_exportdb.py,sha256=uTIZjKKTB7arzEr1j0O6lONiT2pRBKeOFdLvOV8ZwzE,2120
11
+ lamindb_setup/_importdb.py,sha256=yYYShzUajTsR-cTW4CZ-UNDWZY2uE5PAgNbp-wn8Ogc,1874
12
+ lamindb_setup/_init_instance.py,sha256=XLf-xkF_YBwvT_sGyW1wFJFbOWpTFABPRt2oeIICQNI,11825
13
+ lamindb_setup/_migrate.py,sha256=4nBTFg5-BK4A2gH-D3_tcFf8EtvMnIo5Mq0e_C6_9-U,8815
14
+ lamindb_setup/_register_instance.py,sha256=Jeu0wyvJVSVQ_n-A_7yn7xOZIP0ncJD92DRABqzPIjA,940
15
+ lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
16
+ lamindb_setup/_setup_user.py,sha256=6Oc7Rke-yRQSZbuntdUAz8QbJ6UuPzYHI9FnYlf_q-A,3670
17
+ lamindb_setup/_silence_loggers.py,sha256=AKF_YcHvX32eGXdsYK8MJlxEaZ-Uo2f6QDRzjKFCtws,1568
18
+ lamindb_setup/core/__init__.py,sha256=dV9S-rQpNK9JcBn4hiEmiLnmNqfpPFJD9pqagMCaIew,416
19
+ lamindb_setup/core/_aws_storage.py,sha256=nEjeUv4xUVpoV0Lx-zjjmyb9w804bDyaeiM-OqbfwM0,1799
20
+ lamindb_setup/core/_deprecated.py,sha256=3qxUI1dnDlSeR0BYrv7ucjqRBEojbqotPgpShXs4KF8,2520
21
+ lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
22
+ lamindb_setup/core/_hub_client.py,sha256=V0qKDsCdRn-tQy2YIk4VgXcpJFmuum6N3gcavAC7gBQ,5504
23
+ lamindb_setup/core/_hub_core.py,sha256=88EZA5G2JLlEZP6ol_km3Cv0dEpV2U2Z6X5vQOorGrs,15776
24
+ lamindb_setup/core/_hub_crud.py,sha256=m8DgbWIBZBLWB-PV_UNBldUebqnxABfJfrqvzgr3t5s,4662
25
+ lamindb_setup/core/_hub_utils.py,sha256=b_M1LkdCjiMWm1EOlSb9GuPdLijwVgQDtATTpeZuXI0,1875
26
+ lamindb_setup/core/_settings.py,sha256=jjZ_AxRXB3Y3UP6m04BAw_dhFbJbdg2-nZWmEv2LNZ8,3141
27
+ lamindb_setup/core/_settings_instance.py,sha256=RFUcnBBUp303dbVEHcAaIm_q7lzlWg56OrKLwdam8Pg,16588
28
+ lamindb_setup/core/_settings_load.py,sha256=0MkFvZa0fVrMt087GL8Y-mHH44RUDQYybKqqj-c8flU,3849
29
+ lamindb_setup/core/_settings_save.py,sha256=sLVNEMsrEO238Px8P8PsQjl4_RxGSKwRqB9Cffg2TrY,2637
30
+ lamindb_setup/core/_settings_storage.py,sha256=yN2KNd6lq1RDNYbnkaCIfoYc3uKqkSU70otUr4qALPM,12739
31
+ lamindb_setup/core/_settings_store.py,sha256=OMBn396BFkMXGg3came1pToBqnBUwpyqBFv2G8GARiM,2043
32
+ lamindb_setup/core/_settings_user.py,sha256=P2lC4WDRAFfT-Xq3MlXJ-wMKIHCoGNhMTQfRGIAyUNQ,1344
33
+ lamindb_setup/core/_setup_bionty_sources.py,sha256=OgPpZxN2_Wffy-ogEBz_97c_k8d2bD-DDVt89-u9GLY,3002
34
+ lamindb_setup/core/cloud_sqlite_locker.py,sha256=NIBNAGq7TTRrip9OzMdiQKj8QOuwhL9esyM0aehUqBA,6893
35
+ lamindb_setup/core/django.py,sha256=m0AKg2lJ1EYCtEtZ8frFFJbAR9qX0gnFcgqp7aeC2k0,3450
36
+ lamindb_setup/core/exceptions.py,sha256=eoI7AXgATgDVzgArtN7CUvpaMUC067vsBg5LHCsWzDM,305
37
+ lamindb_setup/core/hashing.py,sha256=mv9UCvAsSrdHYQAv3Kz7UOvjd5tIjvDTIYv_ettBuVY,2218
38
+ lamindb_setup/core/types.py,sha256=bcYnZ0uM_2NXKJCl94Mmc-uYrQlRUUVKG3sK2N-F-N4,532
39
+ lamindb_setup/core/upath.py,sha256=TcpBLVtI2aImJWn-FIMMThQt8i_ztJdETQ_Wj4Ow4qY,27965
40
+ lamindb_setup-0.71.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
41
+ lamindb_setup-0.71.0.dist-info/WHEEL,sha256=Sgu64hAMa6g5FdzHxXv9Xdse9yxpGGMeagVtPMWpJQY,99
42
+ lamindb_setup-0.71.0.dist-info/METADATA,sha256=yQ8BtbUkb8PMmdgno_coRXNadR3Ib4ikzV6pTW7TSqo,1620
43
+ lamindb_setup-0.71.0.dist-info/RECORD,,
@@ -1,43 +0,0 @@
1
- lamindb_setup/__init__.py,sha256=yTrf2HNpLb9hFr44mDAEjKjXDwrKYs_n2wX6JmyjOkQ,1558
2
- lamindb_setup/_add_remote_storage.py,sha256=S2rFvn-JcGaBOiBNQoAiaTPM590GQh-g4OJeNsben0Q,1802
3
- lamindb_setup/_cache.py,sha256=wrwlHi2IfxcWDfOW-cHzzkQNlLvasoQPPNxihICbwew,809
4
- lamindb_setup/_check.py,sha256=xgc5kF2HA4lGuAA9FJT9BJyQV5z1EwYMr92RTbGcQg4,92
5
- lamindb_setup/_check_setup.py,sha256=LLdTn4JYiyWKji_sexq2eENbmvA6_gdxqMKjDnF-nJc,2543
6
- lamindb_setup/_close.py,sha256=XtP7uyKRbwCGkXdSkwuiEnLYn5PEN3t3q50dPHNJRSU,1150
7
- lamindb_setup/_connect_instance.py,sha256=XzHbYPmzPmAJZ1EaFtDBzw2skh1s0tppKCg2HOxx1HQ,11908
8
- lamindb_setup/_delete.py,sha256=aIfQpOmb-icGsNVOd9EhHGGBQvsSJF7fBW_fzSwPhuM,5861
9
- lamindb_setup/_django.py,sha256=Yv-bNj80I5Fj9AJ2BFgcv9JZg4WUrlh3UStty82_jXQ,1500
10
- lamindb_setup/_exportdb.py,sha256=f5XUhRxTsU-yqfPM92OxjZKRxJo3lKaqYTRh9oytDfY,2084
11
- lamindb_setup/_importdb.py,sha256=bxFZNjtVOAW4yeMQakVYZE4-WNi4QE-MdMynHShMOgI,1836
12
- lamindb_setup/_init_instance.py,sha256=DlVHpz8avTYaL-IoxSO4XM1rfZmtZXsWpjtY95vYmK0,11480
13
- lamindb_setup/_migrate.py,sha256=wVDfIN1z2OIJztrenpYx42XN09retbPek49TzzmFQ6I,8810
14
- lamindb_setup/_register_instance.py,sha256=1cZXXuCf6cqbiDchmW5vYNTf34wMNSZ2QtHmIJbTT4c,796
15
- lamindb_setup/_schema.py,sha256=edhbiUYatbO5FfBA6PYRmVmYWbtBXZBpATa70_MZiWo,642
16
- lamindb_setup/_setup_user.py,sha256=5zZ924ckd8CGBiD4-5bdTN-NUNMS7N5w8Ovy6JIYPMU,3658
17
- lamindb_setup/_silence_loggers.py,sha256=auXnYVSx2ErZmyXZJ7FYYA69UQgUDQ5SnuznFXzfaWs,1548
18
- lamindb_setup/core/__init__.py,sha256=xfhu3cO_zPtNztmu6vlWtdbX92gPpnjyff5dOqCHTvE,369
19
- lamindb_setup/core/_aws_storage.py,sha256=GIT9SagV-1oFlLpRIogFgXuTwzMZsppFjqKA4-Vp66s,1762
20
- lamindb_setup/core/_deprecated.py,sha256=DNO_90uyYGM9on7rzJCu9atS7Z1qOQypsuL9PI6I-OQ,2491
21
- lamindb_setup/core/_docs.py,sha256=0ha3cZqzvIQnAXVC_JlAiRrFi3VpEEVookP_oURhBr4,240
22
- lamindb_setup/core/_hub_client.py,sha256=H09wJRck4d3fQ8VBSK1v8Dbj9RfqLdW9UqoTs6OCyUM,5520
23
- lamindb_setup/core/_hub_core.py,sha256=Y_RnT82Y7fHgMmNvG8NTDTnx5Ybf_IwFQSMl6AFveVg,11294
24
- lamindb_setup/core/_hub_crud.py,sha256=0qKKsnHlx_FceX-jahdS2vJc2IuGRzouQrtAMn6MTSA,4505
25
- lamindb_setup/core/_hub_utils.py,sha256=4bjtm4QVivDwtVq3Bx9hQFZtdU2WmqrVdpupdvxMHcM,1862
26
- lamindb_setup/core/_settings.py,sha256=ogeOddaJ9tiolzpGwUIu6i4vhcViXQvsREEzZ95uyvI,3241
27
- lamindb_setup/core/_settings_instance.py,sha256=nAaTW4kd_wYM5hKlRwpLgBd-ATiEDvtHOX1NCnpkFyQ,13927
28
- lamindb_setup/core/_settings_load.py,sha256=RkH_zfdGAlBO-4sovmFNGSP56XhSJpUYdvlZlNh-Qqs,3745
29
- lamindb_setup/core/_settings_save.py,sha256=QnUQsYMAXv9xUYAsQ-5vldntSnAvNb3TMmJ61J3kU_I,2571
30
- lamindb_setup/core/_settings_storage.py,sha256=Y2FsDmOFW7Kz1bh2MAMyQkWPJjoYN12ObpqVp82qRag,11750
31
- lamindb_setup/core/_settings_store.py,sha256=iM3Ei-YTSQo6wbcUhOliQx_KmHo8sxDx0-elJPt9AwQ,1929
32
- lamindb_setup/core/_settings_user.py,sha256=ouRhbnIN12cjvVR0Iz98hUyYyGbqcGNRb1OCeI2T6wo,1282
33
- lamindb_setup/core/_setup_bionty_sources.py,sha256=xhIIJqtCzrjTONzU_68bq5kQxsNRDfAprHi2oJvY_LY,2908
34
- lamindb_setup/core/cloud_sqlite_locker.py,sha256=nFwAupnbIy_sxzlJBayfD2Dh8ZMUGNxbspIbv6BGrp8,6835
35
- lamindb_setup/core/django.py,sha256=bKwuv83ubKVsMeunnFtpWd1C-y29I2cuvpW10rCPpZo,3411
36
- lamindb_setup/core/exceptions.py,sha256=YzkJA51ZAa2w9lawAggSdg-NA-EIYJGNRnFd0DN3_bE,275
37
- lamindb_setup/core/hashing.py,sha256=nbQO_CSiX09O4bznHABft94fdFZAHPm2NIyvH5xagoo,2011
38
- lamindb_setup/core/types.py,sha256=fR71SLjoN1MkCjx8TJcjYDgmgO4bPXBX5J_RKpmmy_o,497
39
- lamindb_setup/core/upath.py,sha256=v8lPEA5j_SH0QzrHfw02x3HDt1ikFVc3gdoCCVfwfmw,25078
40
- lamindb_setup-0.70.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
41
- lamindb_setup-0.70.0.dist-info/WHEEL,sha256=Sgu64hAMa6g5FdzHxXv9Xdse9yxpGGMeagVtPMWpJQY,99
42
- lamindb_setup-0.70.0.dist-info/METADATA,sha256=9hjTjaw1xv6SQR_ugZg7DMKxGu0SqgJy0b9VJuR2rVw,1469
43
- lamindb_setup-0.70.0.dist-info/RECORD,,