lamindb_setup 0.69.5__py2.py3-none-any.whl → 0.71.0__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. lamindb_setup/__init__.py +15 -15
  2. lamindb_setup/_add_remote_storage.py +22 -33
  3. lamindb_setup/_cache.py +4 -1
  4. lamindb_setup/_check.py +3 -0
  5. lamindb_setup/_check_setup.py +13 -7
  6. lamindb_setup/_close.py +2 -0
  7. lamindb_setup/_connect_instance.py +50 -34
  8. lamindb_setup/_delete.py +121 -22
  9. lamindb_setup/_django.py +4 -1
  10. lamindb_setup/_exportdb.py +4 -2
  11. lamindb_setup/_importdb.py +5 -1
  12. lamindb_setup/_init_instance.py +58 -46
  13. lamindb_setup/_migrate.py +20 -14
  14. lamindb_setup/_register_instance.py +10 -3
  15. lamindb_setup/_schema.py +6 -3
  16. lamindb_setup/_setup_user.py +8 -8
  17. lamindb_setup/_silence_loggers.py +4 -2
  18. lamindb_setup/core/__init__.py +4 -3
  19. lamindb_setup/core/_aws_storage.py +3 -0
  20. lamindb_setup/core/_deprecated.py +2 -7
  21. lamindb_setup/core/_docs.py +2 -0
  22. lamindb_setup/core/_hub_client.py +12 -10
  23. lamindb_setup/core/_hub_core.py +206 -85
  24. lamindb_setup/core/_hub_crud.py +15 -11
  25. lamindb_setup/core/_hub_utils.py +11 -8
  26. lamindb_setup/core/_settings.py +23 -26
  27. lamindb_setup/core/_settings_instance.py +164 -42
  28. lamindb_setup/core/_settings_load.py +13 -8
  29. lamindb_setup/core/_settings_save.py +11 -8
  30. lamindb_setup/core/_settings_storage.py +104 -95
  31. lamindb_setup/core/_settings_store.py +3 -2
  32. lamindb_setup/core/_settings_user.py +10 -6
  33. lamindb_setup/core/_setup_bionty_sources.py +9 -2
  34. lamindb_setup/core/cloud_sqlite_locker.py +13 -10
  35. lamindb_setup/core/django.py +5 -0
  36. lamindb_setup/core/exceptions.py +4 -2
  37. lamindb_setup/core/hashing.py +15 -5
  38. lamindb_setup/core/types.py +5 -2
  39. lamindb_setup/core/upath.py +202 -92
  40. {lamindb_setup-0.69.5.dist-info → lamindb_setup-0.71.0.dist-info}/METADATA +6 -4
  41. lamindb_setup-0.71.0.dist-info/RECORD +43 -0
  42. lamindb_setup-0.69.5.dist-info/RECORD +0 -43
  43. {lamindb_setup-0.69.5.dist-info → lamindb_setup-0.71.0.dist-info}/LICENSE +0 -0
  44. {lamindb_setup-0.69.5.dist-info → lamindb_setup-0.71.0.dist-info}/WHEEL +0 -0
@@ -1,22 +1,28 @@
1
1
  # we are not documenting UPath here because it's documented at lamindb.UPath
2
2
  """Paths & file systems."""
3
3
 
4
+ from __future__ import annotations
5
+
4
6
  import os
7
+ from collections import defaultdict
5
8
  from datetime import datetime, timezone
6
- import botocore.session
9
+ from functools import partial
10
+ from itertools import islice
7
11
  from pathlib import Path, PurePosixPath
8
- from typing import Literal, Dict
12
+ from typing import TYPE_CHECKING, Any, Literal
13
+
14
+ import botocore.session
9
15
  import fsspec
10
- from itertools import islice
11
- from typing import Optional, Set, Any, Tuple, List
12
- from collections import defaultdict
13
16
  from lamin_utils import logger
14
17
  from upath import UPath
15
- from upath.implementations.cloud import CloudPath, S3Path # noqa # keep CloudPath!
18
+ from upath.implementations.cloud import CloudPath, S3Path # keep CloudPath!
16
19
  from upath.implementations.local import LocalPath, PosixUPath, WindowsUPath
17
- from .types import UPathStr
20
+
18
21
  from .hashing import b16_to_b64, hash_md5s_from_dir
19
22
 
23
+ if TYPE_CHECKING:
24
+ from .types import UPathStr
25
+
20
26
  LocalPathClasses = (PosixUPath, WindowsUPath, LocalPath)
21
27
 
22
28
  # also see https://gist.github.com/securifera/e7eed730cbe1ce43d0c29d7cd2d582f4
@@ -55,7 +61,7 @@ VALID_SUFFIXES = {
55
61
  TRAILING_SEP = (os.sep, os.altsep) if os.altsep is not None else os.sep
56
62
 
57
63
 
58
- def extract_suffix_from_path(path: Path, arg_name: Optional[str] = None) -> str:
64
+ def extract_suffix_from_path(path: Path, arg_name: str | None = None) -> str:
59
65
  def process_digits(suffix: str):
60
66
  if suffix[1:].isdigit(): # :1 to skip the dot
61
67
  return "" # digits are no valid suffixes
@@ -139,44 +145,100 @@ def create_mapper(
139
145
  )
140
146
 
141
147
 
142
- def print_hook(size: int, value: int, **kwargs):
148
+ def print_hook(size: int, value: int, objectname: str, action: str):
143
149
  progress_in_percent = (value / size) * 100
144
- out = (
145
- f"... {kwargs['action']} {Path(kwargs['filepath']).name}:"
146
- f" {min(progress_in_percent, 100):4.1f}%"
147
- )
148
- if progress_in_percent >= 100:
149
- out += "\n"
150
+ out = f"... {action} {objectname}:" f" {min(progress_in_percent, 100):4.1f}%"
150
151
  if "NBPRJ_TEST_NBPATH" not in os.environ:
151
152
  print(out, end="\r")
152
153
 
153
154
 
154
155
  class ProgressCallback(fsspec.callbacks.Callback):
155
- def __init__(self, action: Literal["uploading", "downloading"]):
156
+ def __init__(
157
+ self,
158
+ objectname: str,
159
+ action: Literal["uploading", "downloading", "synchronizing"],
160
+ adjust_size: bool = False,
161
+ ):
162
+ assert action in {"uploading", "downloading", "synchronizing"}
163
+
156
164
  super().__init__()
165
+
157
166
  self.action = action
167
+ print_progress = partial(print_hook, objectname=objectname, action=action)
168
+ self.hooks = {"print_progress": print_progress}
169
+
170
+ self.adjust_size = adjust_size
171
+
172
+ def absolute_update(self, value):
173
+ pass
174
+
175
+ def relative_update(self, inc=1):
176
+ pass
177
+
178
+ def update_relative_value(self, inc=1):
179
+ self.value += inc
180
+ self.call()
158
181
 
159
182
  def branch(self, path_1, path_2, kwargs):
160
- kwargs["callback"] = fsspec.callbacks.Callback(
161
- hooks=dict(print_hook=print_hook), filepath=path_1, action=self.action
162
- )
183
+ if self.adjust_size:
184
+ if Path(path_2 if self.action != "uploading" else path_1).is_dir():
185
+ self.size -= 1
186
+ kwargs["callback"] = ChildProgressCallback(self)
187
+
188
+ def branched(self, path_1, path_2, **kwargs):
189
+ self.branch(path_1, path_2, kwargs)
190
+ return kwargs["callback"]
191
+
192
+ def wrap(self, iterable):
193
+ if self.adjust_size:
194
+ paths = []
195
+ for lpath, rpath in iterable:
196
+ paths.append((lpath, rpath))
197
+ if Path(lpath).is_dir():
198
+ self.size -= 1
199
+ self.adjust_size = False
200
+ return paths
201
+ else:
202
+ return iterable
203
+
204
+ @classmethod
205
+ def requires_progress(
206
+ cls,
207
+ maybe_callback: fsspec.callbacks.Callback | None,
208
+ print_progress: bool,
209
+ objectname: str,
210
+ action: Literal["uploading", "downloading", "synchronizing"],
211
+ **kwargs,
212
+ ):
213
+ if maybe_callback is None:
214
+ if print_progress:
215
+ return cls(objectname, action, **kwargs)
216
+ else:
217
+ return fsspec.callbacks.NoOpCallback()
218
+ return maybe_callback
163
219
 
164
- def call(self, *args, **kwargs):
165
- return None
220
+
221
+ class ChildProgressCallback(fsspec.callbacks.Callback):
222
+ def __init__(self, parent: ProgressCallback):
223
+ super().__init__()
224
+
225
+ self.parent = parent
226
+
227
+ def parent_update(self, inc=1):
228
+ self.parent.update_relative_value(inc)
229
+
230
+ def relative_update(self, inc=1):
231
+ self.parent_update(inc / self.size)
166
232
 
167
233
 
168
234
  def download_to(self, path: UPathStr, print_progress: bool = False, **kwargs):
169
235
  """Download to a path."""
170
- if print_progress:
171
- # can't do path.is_dir() because path doesn't exist
172
- # so assume any destination without a suffix is a dir
173
- # this is temporary until we have a proper progress bar for directories
174
- if os.path.splitext(path)[-1] not in {"", ".zrad", ".zarr"}:
175
- cb = ProgressCallback("downloading")
176
- else:
177
- # todo: make proper progress bar for directories
178
- cb = fsspec.callbacks.NoOpCallback()
179
- kwargs["callback"] = cb
236
+ if print_progress and "callback" not in kwargs:
237
+ callback = ProgressCallback(
238
+ PurePosixPath(path).name, "downloading", adjust_size=True
239
+ )
240
+ kwargs["callback"] = callback
241
+
180
242
  self.fs.download(str(self), str(path), **kwargs)
181
243
 
182
244
 
@@ -188,20 +250,16 @@ def upload_from(
188
250
  **kwargs,
189
251
  ):
190
252
  """Upload from a local path."""
191
- path_is_dir = os.path.isdir(path)
253
+ path = Path(path)
254
+ path_is_dir = path.is_dir()
192
255
  if not path_is_dir:
193
256
  dir_inplace = False
194
257
 
195
- if print_progress:
196
- if not path_is_dir:
197
- cb = ProgressCallback("uploading")
198
- else:
199
- # todo: make proper progress bar for directories
200
- cb = fsspec.callbacks.NoOpCallback()
201
- kwargs["callback"] = cb
258
+ if print_progress and "callback" not in kwargs:
259
+ callback = ProgressCallback(path.name, "uploading")
260
+ kwargs["callback"] = callback
202
261
 
203
262
  if dir_inplace:
204
- path = Path(path)
205
263
  source = [f for f in path.rglob("*") if f.is_file()]
206
264
  destination = [str(self / f.relative_to(path)) for f in source]
207
265
  source = [str(f) for f in source] # type: ignore
@@ -231,7 +289,14 @@ def upload_from(
231
289
  del self.fs.dircache[bucket]
232
290
 
233
291
 
234
- def synchronize(self, objectpath: Path, error_no_origin: bool = True, **kwargs):
292
+ def synchronize(
293
+ self,
294
+ objectpath: Path,
295
+ error_no_origin: bool = True,
296
+ print_progress: bool = False,
297
+ callback: fsspec.callbacks.Callback | None = None,
298
+ **kwargs,
299
+ ):
235
300
  """Sync to a local destination path."""
236
301
  # optimize the number of network requests
237
302
  if "timestamp" in kwargs:
@@ -290,15 +355,23 @@ def synchronize(self, objectpath: Path, error_no_origin: bool = True, **kwargs):
290
355
  destination_exists = False
291
356
  need_synchronize = True
292
357
  if need_synchronize:
358
+ callback = ProgressCallback.requires_progress(
359
+ callback, print_progress, objectpath.name, "synchronizing"
360
+ )
361
+ callback.set_size(len(files))
293
362
  origin_file_keys = []
294
- for file, stat in files.items():
295
- destination = PurePosixPath(file).relative_to(self.path)
296
- origin_file_keys.append(destination.as_posix())
363
+ for file, stat in callback.wrap(files.items()):
364
+ file_key = PurePosixPath(file).relative_to(self.path)
365
+ origin_file_keys.append(file_key.as_posix())
297
366
  timestamp = stat[modified_key].timestamp()
298
- origin = UPath(f"{self.protocol}://{file}", **self._kwargs)
299
- origin.synchronize(
300
- objectpath / destination, timestamp=timestamp, **kwargs
367
+
368
+ origin = f"{self.protocol}://{file}"
369
+ destination = objectpath / file_key
370
+ child = callback.branched(origin, destination.as_posix())
371
+ UPath(origin, **self._kwargs).synchronize(
372
+ destination, timestamp=timestamp, callback=child, **kwargs
301
373
  )
374
+ child.close()
302
375
  if destination_exists:
303
376
  local_files = [file for file in objectpath.rglob("*") if file.is_file()]
304
377
  if len(local_files) > len(files):
@@ -314,6 +387,10 @@ def synchronize(self, objectpath: Path, error_no_origin: bool = True, **kwargs):
314
387
  return None
315
388
 
316
389
  # synchronization logic for files
390
+ callback = ProgressCallback.requires_progress(
391
+ callback, print_progress, objectpath.name, "synchronizing"
392
+ )
393
+ kwargs["callback"] = callback
317
394
  if objectpath.exists():
318
395
  local_mts = objectpath.stat().st_mtime # type: ignore
319
396
  need_synchronize = cloud_mts > local_mts
@@ -323,9 +400,13 @@ def synchronize(self, objectpath: Path, error_no_origin: bool = True, **kwargs):
323
400
  if need_synchronize:
324
401
  self.download_to(objectpath, **kwargs)
325
402
  os.utime(objectpath, times=(cloud_mts, cloud_mts))
403
+ else:
404
+ # nothing happens if parent_update is not defined
405
+ # because of Callback.no_op
406
+ callback.parent_update()
326
407
 
327
408
 
328
- def modified(self) -> Optional[datetime]:
409
+ def modified(self) -> datetime | None:
329
410
  """Return modified time stamp."""
330
411
  mtime = self.fs.modified(str(self))
331
412
  if mtime.tzinfo is None:
@@ -338,15 +419,15 @@ def compute_file_tree(
338
419
  *,
339
420
  level: int = -1,
340
421
  only_dirs: bool = False,
341
- limit: int = 1000,
342
- include_paths: Optional[Set[Any]] = None,
343
- skip_suffixes: Optional[List[str]] = None,
344
- ) -> Tuple[str, int]:
422
+ n_max_files_per_dir_and_type: int = 100,
423
+ n_max_files: int = 1000,
424
+ include_paths: set[Any] | None = None,
425
+ skip_suffixes: list[str] | None = None,
426
+ ) -> tuple[str, int]:
345
427
  space = " "
346
428
  branch = "│ "
347
429
  tee = "├── "
348
430
  last = "└── "
349
- max_files_per_dir_per_type = 7
350
431
  if skip_suffixes is None:
351
432
  skip_suffixes_tuple = ()
352
433
  else:
@@ -380,14 +461,14 @@ def compute_file_tree(
380
461
  if only_dirs:
381
462
  contents = [d for d in contents if d.is_dir()]
382
463
  pointers = [tee] * (len(contents) - 1) + [last]
383
- n_files_per_dir_per_type = defaultdict(lambda: 0) # type: ignore
384
- for pointer, child_path in zip(pointers, contents):
464
+ n_files_per_dir_and_type = defaultdict(lambda: 0) # type: ignore
465
+ for pointer, child_path in zip(pointers, contents, strict=False): # type: ignore
385
466
  if child_path.is_dir():
386
467
  if include_dirs and child_path not in include_dirs:
387
468
  continue
388
469
  yield prefix + pointer + child_path.name
389
470
  n_directories += 1
390
- n_files_per_dir_per_type = defaultdict(lambda: 0)
471
+ n_files_per_dir_and_type = defaultdict(lambda: 0)
391
472
  extension = branch if pointer == tee else space
392
473
  yield from inner(child_path, prefix=prefix + extension, level=level - 1)
393
474
  elif not only_dirs:
@@ -395,21 +476,21 @@ def compute_file_tree(
395
476
  continue
396
477
  suffix = extract_suffix_from_path(child_path)
397
478
  suffixes.add(suffix)
398
- n_files_per_dir_per_type[suffix] += 1
479
+ n_files_per_dir_and_type[suffix] += 1
399
480
  n_objects += 1
400
- if n_files_per_dir_per_type[suffix] == max_files_per_dir_per_type:
481
+ if n_files_per_dir_and_type[suffix] == n_max_files_per_dir_and_type:
401
482
  yield prefix + "..."
402
- elif n_files_per_dir_per_type[suffix] > max_files_per_dir_per_type:
483
+ elif n_files_per_dir_and_type[suffix] > n_max_files_per_dir_and_type:
403
484
  continue
404
485
  else:
405
486
  yield prefix + pointer + child_path.name
406
487
 
407
488
  folder_tree = ""
408
489
  iterator = inner(path, level=level)
409
- for line in islice(iterator, limit):
490
+ for line in islice(iterator, n_max_files):
410
491
  folder_tree += f"\n{line}"
411
492
  if next(iterator, None):
412
- folder_tree += f"\n... only showing {limit} out of {n_objects} files"
493
+ folder_tree += f"\n... only showing {n_max_files} out of {n_objects} files"
413
494
  directory_info = "directory" if n_directories == 1 else "directories"
414
495
  display_suffixes = ", ".join([f"{suffix!r}" for suffix in suffixes])
415
496
  suffix_message = f" with suffixes {display_suffixes}" if n_objects > 0 else ""
@@ -424,11 +505,12 @@ def compute_file_tree(
424
505
  def view_tree(
425
506
  path: Path,
426
507
  *,
427
- level: int = -1,
508
+ level: int = 2,
428
509
  only_dirs: bool = False,
429
- limit: int = 1000,
430
- include_paths: Optional[Set[Any]] = None,
431
- skip_suffixes: Optional[List[str]] = None,
510
+ n_max_files_per_dir_and_type: int = 100,
511
+ n_max_files: int = 1000,
512
+ include_paths: set[Any] | None = None,
513
+ skip_suffixes: list[str] | None = None,
432
514
  ) -> None:
433
515
  """Print a visual tree structure of files & directories.
434
516
 
@@ -436,7 +518,7 @@ def view_tree(
436
518
  level: If `1`, only iterate through one level, if `2` iterate through 2
437
519
  levels, if `-1` iterate through entire hierarchy.
438
520
  only_dirs: Only iterate through directories.
439
- limit: Display limit. Will only show this many files. Doesn't affect count.
521
+ n_max_files: Display limit. Will only show this many files. Doesn't affect count.
440
522
  include_paths: Restrict to these paths.
441
523
  skip_suffixes: Skip directories with these suffixes.
442
524
 
@@ -470,7 +552,8 @@ def view_tree(
470
552
  path,
471
553
  level=level,
472
554
  only_dirs=only_dirs,
473
- limit=limit,
555
+ n_max_files=n_max_files,
556
+ n_max_files_per_dir_and_type=n_max_files_per_dir_and_type,
474
557
  include_paths=include_paths,
475
558
  skip_suffixes=skip_suffixes,
476
559
  )
@@ -495,9 +578,10 @@ def to_url(upath):
495
578
  bucket = upath._url.netloc
496
579
  if bucket == "scverse-spatial-eu-central-1":
497
580
  region = "eu-central-1"
498
- elif f"s3://{bucket}" not in hosted_buckets:
499
- metadata = upath.fs.call_s3("head_bucket", Bucket=upath._url.netloc)
500
- region = metadata["BucketRegion"]
581
+ elif f"s3://{bucket}" not in HOSTED_BUCKETS:
582
+ response = upath.fs.call_s3("head_bucket", Bucket=upath._url.netloc)
583
+ headers = response["ResponseMetadata"]["HTTPHeaders"]
584
+ region = headers.get("x-amz-bucket-region")
501
585
  else:
502
586
  region = bucket.replace("lamin_", "")
503
587
  if region == "us-east-1":
@@ -576,7 +660,7 @@ def convert_pathlike(pathlike: UPathStr) -> UPath:
576
660
  return path
577
661
 
578
662
 
579
- hosted_regions = [
663
+ HOSTED_REGIONS = [
580
664
  "eu-central-1",
581
665
  "eu-west-2",
582
666
  "us-east-1",
@@ -586,16 +670,16 @@ hosted_regions = [
586
670
  ]
587
671
  lamin_env = os.getenv("LAMIN_ENV")
588
672
  if lamin_env is None or lamin_env == "prod":
589
- hosted_buckets_list = [f"s3://lamin-{region}" for region in hosted_regions]
673
+ hosted_buckets_list = [f"s3://lamin-{region}" for region in HOSTED_REGIONS]
590
674
  hosted_buckets_list.append("s3://scverse-spatial-eu-central-1")
591
- hosted_buckets = tuple(hosted_buckets_list)
675
+ HOSTED_BUCKETS = tuple(hosted_buckets_list)
592
676
  else:
593
- hosted_buckets = ("s3://lamin-hosted-test",) # type: ignore
594
- credentials_cache: Dict[str, Dict[str, str]] = {}
677
+ HOSTED_BUCKETS = ("s3://lamin-hosted-test",) # type: ignore
678
+ credentials_cache: dict[str, dict[str, str]] = {}
595
679
  AWS_CREDENTIALS_PRESENT = None
596
680
 
597
681
 
598
- def create_path(path: UPath, access_token: Optional[str] = None) -> UPath:
682
+ def create_path(path: UPath, access_token: str | None = None) -> UPath:
599
683
  path = convert_pathlike(path)
600
684
  # test whether we have an AWS S3 path
601
685
  if not isinstance(path, S3Path):
@@ -609,9 +693,8 @@ def create_path(path: UPath, access_token: Optional[str] = None) -> UPath:
609
693
  if path.fs.key is not None and path.fs.secret is not None:
610
694
  anon = False
611
695
  else:
612
- # we can do
613
- # path.fs.connect()
614
- # and check path.fs.session._credentials, but it is slower
696
+ # we could do path.fs.connect()
697
+ # and check path.fs.session._credentials, but it'd be slower
615
698
  session = botocore.session.get_session()
616
699
  credentials = session.get_credentials()
617
700
  if credentials is None or credentials.access_key is None:
@@ -623,7 +706,7 @@ def create_path(path: UPath, access_token: Optional[str] = None) -> UPath:
623
706
 
624
707
  # test whether we are on hosted storage or not
625
708
  path_str = path.as_posix()
626
- is_hosted_storage = path_str.startswith(hosted_buckets)
709
+ is_hosted_storage = path_str.startswith(HOSTED_BUCKETS)
627
710
 
628
711
  if not is_hosted_storage:
629
712
  # make anon request if no credentials present
@@ -650,7 +733,7 @@ def create_path(path: UPath, access_token: Optional[str] = None) -> UPath:
650
733
  )
651
734
 
652
735
 
653
- def get_stat_file_cloud(stat: Dict) -> Tuple[int, str, str]:
736
+ def get_stat_file_cloud(stat: dict) -> tuple[int, str, str]:
654
737
  size = stat["size"]
655
738
  # small files
656
739
  if "-" not in stat["ETag"]:
@@ -667,7 +750,7 @@ def get_stat_file_cloud(stat: Dict) -> Tuple[int, str, str]:
667
750
  return size, hash, hash_type
668
751
 
669
752
 
670
- def get_stat_dir_cloud(path: UPath) -> Tuple[int, str, str, int]:
753
+ def get_stat_dir_cloud(path: UPath) -> tuple[int, str, str, int]:
671
754
  sizes = []
672
755
  md5s = []
673
756
  objects = path.fs.find(path.as_posix(), detail=True)
@@ -688,20 +771,47 @@ class InstanceNotEmpty(Exception):
688
771
  pass
689
772
 
690
773
 
691
- def check_s3_storage_location_empty(root: UPathStr) -> None:
774
+ # is as fast as boto3: https://lamin.ai/laminlabs/lamindata/transform/krGp3hT1f78N5zKv
775
+ def check_storage_is_empty(
776
+ root: UPathStr, *, raise_error: bool = True, account_for_sqlite_file: bool = False
777
+ ) -> int:
692
778
  root_upath = convert_pathlike(root)
693
779
  root_string = root_upath.as_posix() # type: ignore
694
780
  # we currently touch a 0-byte file in the root of a hosted storage location
695
781
  # ({storage_root}/.lamindb/_is_initialized) during storage initialization
696
782
  # since path.fs.find raises a PermissionError on empty hosted
697
783
  # subdirectories (see lamindb_setup/core/_settings_storage/init_storage).
698
- n_touched_objects = 0
699
- if root_string.startswith(hosted_buckets):
700
- n_touched_objects = 1
701
- objects = root_upath.fs.find(root_string)
784
+ n_offset_objects = 1 # because of touched dummy file, see mark_storage_root()
785
+ if root_string.startswith(HOSTED_BUCKETS):
786
+ # in hosted buckets, count across entire root
787
+ directory_string = root_string
788
+ # the SQLite file is not in the ".lamindb" directory
789
+ if account_for_sqlite_file:
790
+ n_offset_objects += 1 # because of SQLite file
791
+ else:
792
+ # in any other storage location, only count in .lamindb
793
+ if not root_string.endswith("/"):
794
+ root_string += "/"
795
+ directory_string = root_string + ".lamindb"
796
+ objects = root_upath.fs.find(directory_string)
702
797
  n_objects = len(objects)
703
- if n_objects > n_touched_objects:
704
- raise InstanceNotEmpty(
705
- f"""storage location contains objects;
706
- {compute_file_tree(root_upath)[0]}"""
707
- )
798
+ n_diff = n_objects - n_offset_objects
799
+ ask_for_deletion = (
800
+ "delete them prior to deleting the instance"
801
+ if raise_error
802
+ else "consider deleting them"
803
+ )
804
+ hint = "'./lamindb/_is_initialized' "
805
+ if n_offset_objects == 2:
806
+ hint += "& SQLite file"
807
+ hint += " ignored"
808
+ message = (
809
+ f"Storage {directory_string} contains {n_objects} objects "
810
+ f"({hint}) - {ask_for_deletion}\n{objects}"
811
+ )
812
+ if n_diff > 0:
813
+ if raise_error:
814
+ raise InstanceNotEmpty(message)
815
+ else:
816
+ logger.warning(message)
817
+ return n_diff
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: lamindb_setup
3
- Version: 0.69.5
3
+ Version: 0.71.0
4
4
  Summary: Setup & configure LaminDB.
5
5
  Author-email: Lamin Labs <laminlabs@gmail.com>
6
6
  Description-Content-Type: text/markdown
@@ -14,7 +14,9 @@ Requires-Dist: requests
14
14
  Requires-Dist: universal_pathlib==0.1.4
15
15
  Requires-Dist: botocore<2.0.0
16
16
  Requires-Dist: supabase==2.2.1
17
- Requires-Dist: s3fs ; extra == "aws"
17
+ Requires-Dist: urllib3<2 ; extra == "aws"
18
+ Requires-Dist: aiobotocore[boto3]>=2.5.4,<3.0.0 ; extra == "aws"
19
+ Requires-Dist: s3fs>=2023.12.2,<=2024.3.1 ; extra == "aws"
18
20
  Requires-Dist: pyjwt<3.0.0 ; extra == "dev"
19
21
  Requires-Dist: psycopg2-binary ; extra == "dev"
20
22
  Requires-Dist: python-dotenv ; extra == "dev"
@@ -25,12 +27,12 @@ Requires-Dist: pytest-xdist ; extra == "dev"
25
27
  Requires-Dist: nbproject-test>=0.4.3 ; extra == "dev"
26
28
  Requires-Dist: pandas ; extra == "dev"
27
29
  Requires-Dist: django-schema-graph ; extra == "erdiagram"
28
- Requires-Dist: faker ; extra == "hub"
30
+ Requires-Dist: gcsfs>=2023.12.2,<=2024.3.1 ; extra == "gcp"
29
31
  Project-URL: Home, https://github.com/laminlabs/lamindb-setup
30
32
  Provides-Extra: aws
31
33
  Provides-Extra: dev
32
34
  Provides-Extra: erdiagram
33
- Provides-Extra: hub
35
+ Provides-Extra: gcp
34
36
 
35
37
  [![codecov](https://codecov.io/gh/laminlabs/lamindb-setup/branch/main/graph/badge.svg)](https://codecov.io/gh/laminlabs/lamindb-setup)
36
38
 
@@ -0,0 +1,43 @@
1
+ lamindb_setup/__init__.py,sha256=6a3FROJtySXqU0zXTq1xIniLLPzc7_JufrXPqKJ11eI,1542
2
+ lamindb_setup/_add_remote_storage.py,sha256=j-p29pxrr_07Ns5X2tm6yKxa3VESPkjyt35X4Cvr-nk,1325
3
+ lamindb_setup/_cache.py,sha256=wA7mbysANwe8hPNbjDo9bOmXJ0xIyaS5iyxIpxSWji4,846
4
+ lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
5
+ lamindb_setup/_check_setup.py,sha256=cNEL9Q4yPpmEkGKHH8JgullWl1VUZwALJ4RHn9wZypY,2613
6
+ lamindb_setup/_close.py,sha256=1QS9p2SCacgovYn6xqWU4zFvwHN1RgIccvzwJgFvKgU,1186
7
+ lamindb_setup/_connect_instance.py,sha256=ElpbjSaAI9vq3OfmPM8TxvsqfrZyytTZSRyYUDfiv3E,11944
8
+ lamindb_setup/_delete.py,sha256=tOEb8N7Ga9hrMpup2zBugtl0wN_Xxv0MqqKP2CpHArQ,7267
9
+ lamindb_setup/_django.py,sha256=EoyWvFzH0i9wxjy4JZhcoXCTckztP_Mrl6FbYQnMmLE,1534
10
+ lamindb_setup/_exportdb.py,sha256=uTIZjKKTB7arzEr1j0O6lONiT2pRBKeOFdLvOV8ZwzE,2120
11
+ lamindb_setup/_importdb.py,sha256=yYYShzUajTsR-cTW4CZ-UNDWZY2uE5PAgNbp-wn8Ogc,1874
12
+ lamindb_setup/_init_instance.py,sha256=XLf-xkF_YBwvT_sGyW1wFJFbOWpTFABPRt2oeIICQNI,11825
13
+ lamindb_setup/_migrate.py,sha256=4nBTFg5-BK4A2gH-D3_tcFf8EtvMnIo5Mq0e_C6_9-U,8815
14
+ lamindb_setup/_register_instance.py,sha256=Jeu0wyvJVSVQ_n-A_7yn7xOZIP0ncJD92DRABqzPIjA,940
15
+ lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
16
+ lamindb_setup/_setup_user.py,sha256=6Oc7Rke-yRQSZbuntdUAz8QbJ6UuPzYHI9FnYlf_q-A,3670
17
+ lamindb_setup/_silence_loggers.py,sha256=AKF_YcHvX32eGXdsYK8MJlxEaZ-Uo2f6QDRzjKFCtws,1568
18
+ lamindb_setup/core/__init__.py,sha256=dV9S-rQpNK9JcBn4hiEmiLnmNqfpPFJD9pqagMCaIew,416
19
+ lamindb_setup/core/_aws_storage.py,sha256=nEjeUv4xUVpoV0Lx-zjjmyb9w804bDyaeiM-OqbfwM0,1799
20
+ lamindb_setup/core/_deprecated.py,sha256=3qxUI1dnDlSeR0BYrv7ucjqRBEojbqotPgpShXs4KF8,2520
21
+ lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
22
+ lamindb_setup/core/_hub_client.py,sha256=V0qKDsCdRn-tQy2YIk4VgXcpJFmuum6N3gcavAC7gBQ,5504
23
+ lamindb_setup/core/_hub_core.py,sha256=88EZA5G2JLlEZP6ol_km3Cv0dEpV2U2Z6X5vQOorGrs,15776
24
+ lamindb_setup/core/_hub_crud.py,sha256=m8DgbWIBZBLWB-PV_UNBldUebqnxABfJfrqvzgr3t5s,4662
25
+ lamindb_setup/core/_hub_utils.py,sha256=b_M1LkdCjiMWm1EOlSb9GuPdLijwVgQDtATTpeZuXI0,1875
26
+ lamindb_setup/core/_settings.py,sha256=jjZ_AxRXB3Y3UP6m04BAw_dhFbJbdg2-nZWmEv2LNZ8,3141
27
+ lamindb_setup/core/_settings_instance.py,sha256=RFUcnBBUp303dbVEHcAaIm_q7lzlWg56OrKLwdam8Pg,16588
28
+ lamindb_setup/core/_settings_load.py,sha256=0MkFvZa0fVrMt087GL8Y-mHH44RUDQYybKqqj-c8flU,3849
29
+ lamindb_setup/core/_settings_save.py,sha256=sLVNEMsrEO238Px8P8PsQjl4_RxGSKwRqB9Cffg2TrY,2637
30
+ lamindb_setup/core/_settings_storage.py,sha256=yN2KNd6lq1RDNYbnkaCIfoYc3uKqkSU70otUr4qALPM,12739
31
+ lamindb_setup/core/_settings_store.py,sha256=OMBn396BFkMXGg3came1pToBqnBUwpyqBFv2G8GARiM,2043
32
+ lamindb_setup/core/_settings_user.py,sha256=P2lC4WDRAFfT-Xq3MlXJ-wMKIHCoGNhMTQfRGIAyUNQ,1344
33
+ lamindb_setup/core/_setup_bionty_sources.py,sha256=OgPpZxN2_Wffy-ogEBz_97c_k8d2bD-DDVt89-u9GLY,3002
34
+ lamindb_setup/core/cloud_sqlite_locker.py,sha256=NIBNAGq7TTRrip9OzMdiQKj8QOuwhL9esyM0aehUqBA,6893
35
+ lamindb_setup/core/django.py,sha256=m0AKg2lJ1EYCtEtZ8frFFJbAR9qX0gnFcgqp7aeC2k0,3450
36
+ lamindb_setup/core/exceptions.py,sha256=eoI7AXgATgDVzgArtN7CUvpaMUC067vsBg5LHCsWzDM,305
37
+ lamindb_setup/core/hashing.py,sha256=mv9UCvAsSrdHYQAv3Kz7UOvjd5tIjvDTIYv_ettBuVY,2218
38
+ lamindb_setup/core/types.py,sha256=bcYnZ0uM_2NXKJCl94Mmc-uYrQlRUUVKG3sK2N-F-N4,532
39
+ lamindb_setup/core/upath.py,sha256=TcpBLVtI2aImJWn-FIMMThQt8i_ztJdETQ_Wj4Ow4qY,27965
40
+ lamindb_setup-0.71.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
41
+ lamindb_setup-0.71.0.dist-info/WHEEL,sha256=Sgu64hAMa6g5FdzHxXv9Xdse9yxpGGMeagVtPMWpJQY,99
42
+ lamindb_setup-0.71.0.dist-info/METADATA,sha256=yQ8BtbUkb8PMmdgno_coRXNadR3Ib4ikzV6pTW7TSqo,1620
43
+ lamindb_setup-0.71.0.dist-info/RECORD,,
@@ -1,43 +0,0 @@
1
- lamindb_setup/__init__.py,sha256=UtfzeMDIIg5wzz8ahXKmjc2p9pgSXvoXop6PeSM3fig,1558
2
- lamindb_setup/_add_remote_storage.py,sha256=S2rFvn-JcGaBOiBNQoAiaTPM590GQh-g4OJeNsben0Q,1802
3
- lamindb_setup/_cache.py,sha256=wrwlHi2IfxcWDfOW-cHzzkQNlLvasoQPPNxihICbwew,809
4
- lamindb_setup/_check.py,sha256=xgc5kF2HA4lGuAA9FJT9BJyQV5z1EwYMr92RTbGcQg4,92
5
- lamindb_setup/_check_setup.py,sha256=LLdTn4JYiyWKji_sexq2eENbmvA6_gdxqMKjDnF-nJc,2543
6
- lamindb_setup/_close.py,sha256=XtP7uyKRbwCGkXdSkwuiEnLYn5PEN3t3q50dPHNJRSU,1150
7
- lamindb_setup/_connect_instance.py,sha256=QICzNO3ID3JwJb2yMd6R5DfExT3q5vlkn0e8WHo2lLw,11789
8
- lamindb_setup/_delete.py,sha256=XzYaB2k6DhNfr5b-nzHZ4qGcAIxnuP8fc-hIOYXvy8E,3111
9
- lamindb_setup/_django.py,sha256=Yv-bNj80I5Fj9AJ2BFgcv9JZg4WUrlh3UStty82_jXQ,1500
10
- lamindb_setup/_exportdb.py,sha256=f5XUhRxTsU-yqfPM92OxjZKRxJo3lKaqYTRh9oytDfY,2084
11
- lamindb_setup/_importdb.py,sha256=bxFZNjtVOAW4yeMQakVYZE4-WNi4QE-MdMynHShMOgI,1836
12
- lamindb_setup/_init_instance.py,sha256=i0X9ENufdSGPog_8S1dypAvhwfGpVfsGuAIpnpfkjTc,11479
13
- lamindb_setup/_migrate.py,sha256=JlyjuhPO7qVEzaRfB5WBvkGgy-0qMh60MddM1zemSlo,8672
14
- lamindb_setup/_register_instance.py,sha256=tfdpL9NFXmtdxAGnym5Q4I5x4cT6_xVBim6pbECKlvk,795
15
- lamindb_setup/_schema.py,sha256=edhbiUYatbO5FfBA6PYRmVmYWbtBXZBpATa70_MZiWo,642
16
- lamindb_setup/_setup_user.py,sha256=xwSJpbLtl2gW-frGs5k_kBL6PZZ00pe1gfmmd-a7myc,3657
17
- lamindb_setup/_silence_loggers.py,sha256=auXnYVSx2ErZmyXZJ7FYYA69UQgUDQ5SnuznFXzfaWs,1548
18
- lamindb_setup/core/__init__.py,sha256=xfhu3cO_zPtNztmu6vlWtdbX92gPpnjyff5dOqCHTvE,369
19
- lamindb_setup/core/_aws_storage.py,sha256=GIT9SagV-1oFlLpRIogFgXuTwzMZsppFjqKA4-Vp66s,1762
20
- lamindb_setup/core/_deprecated.py,sha256=DNO_90uyYGM9on7rzJCu9atS7Z1qOQypsuL9PI6I-OQ,2491
21
- lamindb_setup/core/_docs.py,sha256=0ha3cZqzvIQnAXVC_JlAiRrFi3VpEEVookP_oURhBr4,240
22
- lamindb_setup/core/_hub_client.py,sha256=H09wJRck4d3fQ8VBSK1v8Dbj9RfqLdW9UqoTs6OCyUM,5520
23
- lamindb_setup/core/_hub_core.py,sha256=2ul3KvVorXLZfEefGbm9toRLga-rli8qzzq71vvEfw0,10990
24
- lamindb_setup/core/_hub_crud.py,sha256=0qKKsnHlx_FceX-jahdS2vJc2IuGRzouQrtAMn6MTSA,4505
25
- lamindb_setup/core/_hub_utils.py,sha256=4bjtm4QVivDwtVq3Bx9hQFZtdU2WmqrVdpupdvxMHcM,1862
26
- lamindb_setup/core/_settings.py,sha256=ogeOddaJ9tiolzpGwUIu6i4vhcViXQvsREEzZ95uyvI,3241
27
- lamindb_setup/core/_settings_instance.py,sha256=XRpHZJT8MZONpIS5Sop6Rk3BLYVmguh2PkC3TrGU7jU,11556
28
- lamindb_setup/core/_settings_load.py,sha256=65T_lW1xby__PKULufqzc1eWE30SVRNbxS2TAVSCWXM,3744
29
- lamindb_setup/core/_settings_save.py,sha256=bAnjmZuAKk4wx4EoTx-LfaCY7yyU_4bUfF7hdI71DgY,2563
30
- lamindb_setup/core/_settings_storage.py,sha256=a_Pa_nPzFuDeYxBlvFelQIkINlVGSNO_2aLvY_nxP-w,13044
31
- lamindb_setup/core/_settings_store.py,sha256=iM3Ei-YTSQo6wbcUhOliQx_KmHo8sxDx0-elJPt9AwQ,1929
32
- lamindb_setup/core/_settings_user.py,sha256=wWLrSaohWlQSLOA-7fLSRCV43rupy-ixNvkZMWT9Pgo,1281
33
- lamindb_setup/core/_setup_bionty_sources.py,sha256=xhIIJqtCzrjTONzU_68bq5kQxsNRDfAprHi2oJvY_LY,2908
34
- lamindb_setup/core/cloud_sqlite_locker.py,sha256=nFwAupnbIy_sxzlJBayfD2Dh8ZMUGNxbspIbv6BGrp8,6835
35
- lamindb_setup/core/django.py,sha256=MNlEFS8a7GX1IxmMgJymoRf3M03Eg881zQ2_MKONOQI,3335
36
- lamindb_setup/core/exceptions.py,sha256=YzkJA51ZAa2w9lawAggSdg-NA-EIYJGNRnFd0DN3_bE,275
37
- lamindb_setup/core/hashing.py,sha256=nbQO_CSiX09O4bznHABft94fdFZAHPm2NIyvH5xagoo,2011
38
- lamindb_setup/core/types.py,sha256=fR71SLjoN1MkCjx8TJcjYDgmgO4bPXBX5J_RKpmmy_o,497
39
- lamindb_setup/core/upath.py,sha256=rK1GvW_-23gE5Wr3VmAv1hw4PZCxN878P-EUxXr0RlM,24510
40
- lamindb_setup-0.69.5.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
41
- lamindb_setup-0.69.5.dist-info/WHEEL,sha256=Sgu64hAMa6g5FdzHxXv9Xdse9yxpGGMeagVtPMWpJQY,99
42
- lamindb_setup-0.69.5.dist-info/METADATA,sha256=o7IyD1LbQF4cqks-lrF4XgOh25jZcnYkrV0o9JU_9Jo,1469
43
- lamindb_setup-0.69.5.dist-info/RECORD,,