lamindb_setup 0.72.1__py2.py3-none-any.whl → 0.73.0__py2.py3-none-any.whl

lamindb_setup/__init__.py

@@ -34,7 +34,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.72.1"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.73.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import sys
 from os import name as _os_name

lamindb_setup/_init_instance.py

@@ -56,6 +56,7 @@ def register_storage_in_instance(ssettings: StorageSettings):
         "region": ssettings.region,
         "instance_uid": instance_uid,
         "created_by_id": current_user_id(),
+        "run": None,
     }
     if ssettings._uid is not None:
         defaults["uid"] = ssettings._uid

lamindb_setup/core/_aws_credentials.py

@@ -22,6 +22,10 @@ else:
     HOSTED_BUCKETS = ("s3://lamin-hosted-test",)  # type: ignore
 
 
+def _keep_trailing_slash(path_str: str):
+    return path_str if path_str[-1] == "/" else path_str + "/"
+
+
 AWS_CREDENTIALS_EXPIRATION = 11 * 60 * 60  # refresh credentials after 11 hours
 
 
@@ -81,7 +85,10 @@ class AWSCredentialsManager:
         return S3Path(path, cache_regions=cache_regions, **connection_options)
 
     def enrich_path(self, path: S3Path, access_token: str | None = None) -> S3Path:
-        path_str = path.as_posix().rstrip("/")
+        # trailing slash is needed to avoid returning incorrect results
+        # with .startswith
+        # for example s3://lamindata-eu should not receive cache for s3://lamindata
+        path_str = _keep_trailing_slash(path.as_posix())
         root = self._find_root(path_str)
 
         if root is not None:
@@ -127,7 +134,7 @@ class AWSCredentialsManager:
                         # write the bucket for everything else
                         root = path._url.netloc
                     root = "s3://" + root
-                self._set_cached_credentials(root, credentials)
+                self._set_cached_credentials(_keep_trailing_slash(root), credentials)
 
         return self._path_inject_options(path, credentials)
 

lamindb_setup/core/_settings_instance.py

@@ -5,6 +5,7 @@ import shutil
 from pathlib import Path
 from typing import TYPE_CHECKING, Literal
 
+from django.db.utils import ProgrammingError
 from lamin_utils import logger
 
 from ._hub_client import call_with_fallback
@@ -106,8 +107,15 @@ class InstanceSettings:
             local_records = Storage.objects.filter(root=local_root)
         else:
             local_records = Storage.objects.filter(type="local")
+        all_local_records = local_records.all()
+        try:
+            # trigger an error in case of a migration issue
+            all_local_records.first()
+        except ProgrammingError:
+            logger.error("not able to load Storage registry: please migrate")
+            return None
         found = False
-        for record in local_records.all():
+        for record in all_local_records:
             root_path = Path(record.root)
             if root_path.exists():
                 marker_path = root_path / ".lamindb/_is_initialized"

lamindb_setup/core/_settings_storage.py

@@ -37,33 +37,36 @@ def base62(n_char: int) -> str:
     return id
 
 
-def get_storage_region(storage_root: UPathStr) -> str | None:
-    storage_root_str = str(storage_root)
-    if storage_root_str.startswith("s3://"):
-        import botocore.session as session
+def get_storage_region(path: UPathStr) -> str | None:
+    path_str = str(path)
+    if path_str.startswith("s3://"):
+        import botocore.session
         from botocore.config import Config
-        from botocore.exceptions import NoCredentialsError
+        from botocore.exceptions import ClientError
 
         # strip the prefix and any suffixes of the bucket name
-        bucket = storage_root_str.replace("s3://", "").split("/")[0]
-        s3_session = session.get_session()
-        s3_client = s3_session.create_client("s3")
+        bucket = path_str.replace("s3://", "").split("/")[0]
+        session = botocore.session.get_session()
+        credentials = session.get_credentials()
+        if credentials is None or credentials.access_key is None:
+            config = Config(signature_version=botocore.session.UNSIGNED)
+        else:
+            config = None
+        s3_client = session.create_client("s3", config=config)
         try:
             response = s3_client.head_bucket(Bucket=bucket)
-        except NoCredentialsError:  # deal with anonymous access
-            s3_client = s3_session.create_client(
-                "s3", config=Config(signature_version=session.UNSIGNED)
-            )
-            response = s3_client.head_bucket(Bucket=bucket)
-        storage_region = response["ResponseMetadata"].get("HTTPHeaders", {})[
-            "x-amz-bucket-region"
-        ]
-        # if we want to except botcore.exceptions.ClientError to reformat an
-        # error message, this is how to do test for the "NoSuchBucket" error:
-        #     exc.response["Error"]["Code"] == "NoSuchBucket"
+        except ClientError as exc:
+            response = getattr(exc, "response", {})
+            if response.get("Error", {}).get("Code") == "404":
+                raise exc
+        region = (
+            response.get("ResponseMetadata", {})
+            .get("HTTPHeaders", {})
+            .get("x-amz-bucket-region")
+        )
     else:
-        storage_region = None
-    return storage_region
+        region = None
+    return region
 
 
 def mark_storage_root(root: UPathStr, uid: str):

lamindb_setup/core/_setup_bionty_sources.py

@@ -53,6 +53,7 @@ def write_bionty_sources(isettings: InstanceSettings) -> None:
             kwargs["species"] = kwargs.pop("organism")
         elif hasattr(PublicSource, "organism") and "species" in kwargs:
             kwargs["organism"] = kwargs.pop("species")
+        kwargs["run"] = None  # can't yet access tracking information
         record = PublicSource(**kwargs)
         all_records.append(record)
 

lamindb_setup/core/upath.py

@@ -155,7 +155,10 @@ def create_mapper(
 
 
 def print_hook(size: int, value: int, objectname: str, action: str):
-    progress_in_percent = (value / size) * 100
+    if size == 0:
+        progress_in_percent = 100.0
+    else:
+        progress_in_percent = (value / size) * 100
     out = f"... {action} {objectname}:" f" {min(progress_in_percent, 100):4.1f}%"
     if "NBPRJ_TEST_NBPATH" not in os.environ:
         end = "\n" if progress_in_percent >= 100 else "\r"
@@ -238,50 +241,75 @@ class ChildProgressCallback(fsspec.callbacks.Callback):
         self.parent.update_relative_value(inc)
 
     def relative_update(self, inc=1):
-        self.parent_update(inc / self.size)
+        if self.size != 0:
+            self.parent_update(inc / self.size)
+        else:
+            self.parent_update(1)
 
 
-def download_to(self, path: UPathStr, print_progress: bool = False, **kwargs):
-    """Download to a path."""
+def download_to(self, local_path: UPathStr, print_progress: bool = True, **kwargs):
+    """Download from self (a destination in the cloud) to the local path."""
+    if "recursive" not in kwargs:
+        kwargs["recursive"] = True
     if print_progress and "callback" not in kwargs:
         callback = ProgressCallback(
-            PurePosixPath(path).name, "downloading", adjust_size=True
+            PurePosixPath(local_path).name, "downloading", adjust_size=True
         )
         kwargs["callback"] = callback
 
-    self.fs.download(str(self), str(path), **kwargs)
+    self.fs.download(str(self), str(local_path), **kwargs)
 
 
 def upload_from(
     self,
-    path: UPathStr,
-    dir_inplace: bool = False,
-    print_progress: bool = False,
+    local_path: UPathStr,
+    create_folder: bool | None = None,
+    print_progress: bool = True,
     **kwargs,
-):
-    """Upload from a local path."""
-    path = Path(path)
-    path_is_dir = path.is_dir()
-    if not path_is_dir:
-        dir_inplace = False
+) -> UPath:
+    """Upload from the local path to `self` (a destination in the cloud).
+
+    If the local path is a directory, recursively upload its contents.
+
+    Args:
+        local_path: A local path of a file or directory.
+        create_folder: Only applies if `local_path` is a directory and then
+            defaults to `True`. If `True`, make a new folder in the destination
+            using the directory name of `local_path`. If `False`, upload the
+            contents of the directory to to the root-level of the destination.
+        print_progress: Print progress.
+
+    Returns:
+        The destination path.
+    """
+    local_path = Path(local_path)
+    local_path_is_dir = local_path.is_dir()
+    if create_folder is None:
+        create_folder = local_path_is_dir
+    if create_folder and not local_path_is_dir:
+        raise ValueError("create_folder can only be True if local_path is a directory")
 
     if print_progress and "callback" not in kwargs:
-        callback = ProgressCallback(path.name, "uploading")
+        callback = ProgressCallback(local_path.name, "uploading")
         kwargs["callback"] = callback
 
-    if dir_inplace:
-        source = [f for f in path.rglob("*") if f.is_file()]
-        destination = [str(self / f.relative_to(path)) for f in source]
+    if local_path_is_dir and not create_folder:
+        source = [f for f in local_path.rglob("*") if f.is_file()]
+        # convert_pathlike is needed to remove the trailing slash because
+        # UPath("s3://some-bucket/some-folder/") / "some-key"
+        # results in UPath("s3://some-bucket/some-folder//some-key")
+        # for upath 0.1.4
+        dest_root = convert_pathlike(self) if self._parts[-1] == "" else self
+        destination = [str(dest_root / f.relative_to(local_path)) for f in source]
         source = [str(f) for f in source]  # type: ignore
     else:
-        source = str(path)  # type: ignore
+        source = str(local_path)  # type: ignore
         destination = str(self)  # type: ignore
-    # this weird thing is to avoid s3fs triggering create_bucket in upload
-    # if dirs are present
-    # it allows to avoid permission error
-    if self.protocol != "s3" or not path_is_dir or dir_inplace:
-        cleanup_cache = False
-    else:
+
+    # the below lines are to avoid s3fs triggering create_bucket in upload if
+    # dirs are present it allows to avoid permission error
+    # would be easier to just
+    if self.protocol == "s3" and local_path_is_dir and create_folder:
         bucket = self._url.netloc
         if bucket not in self.fs.dircache:
             self.fs.dircache[bucket] = [{}]
@@ -290,14 +318,23 @@ def upload_from(
             cleanup_cache = True
         else:
             cleanup_cache = False
+    else:
+        cleanup_cache = False
 
-    self.fs.upload(source, destination, **kwargs)
+    self.fs.upload(source, destination, recursive=create_folder, **kwargs)
 
     if cleanup_cache:
         # normally this is invalidated after the upload but still better to check
         if bucket in self.fs.dircache:
             del self.fs.dircache[bucket]
 
+    if local_path_is_dir and create_folder:
+        # convert_pathlike is needed to remove the trailing slash
+        dest_root = convert_pathlike(self) if self._parts[-1] == "" else self
+        return dest_root / local_path.name
+    else:
+        return self
+
 
 def synchronize(
     self,
@@ -305,14 +342,14 @@ def synchronize(
     error_no_origin: bool = True,
     print_progress: bool = False,
     callback: fsspec.callbacks.Callback | None = None,
-    **kwargs,
+    timestamp: float | None = None,
 ):
     """Sync to a local destination path."""
     # optimize the number of network requests
-    if "timestamp" in kwargs:
+    if timestamp is not None:
         is_dir = False
         exists = True
-        cloud_mts = kwargs.pop("timestamp")
+        cloud_mts = timestamp
     else:
         # perform only one network request to check existence, type and timestamp
         try:
@@ -379,7 +416,7 @@ def synchronize(
                 destination = objectpath / file_key
                 child = callback.branched(origin, destination.as_posix())
                 UPath(origin, **self._kwargs).synchronize(
-                    destination, timestamp=timestamp, callback=child, **kwargs
+                    destination, callback=child, timestamp=timestamp
                 )
                 child.close()
             if destination_exists:
@@ -400,15 +437,16 @@ def synchronize(
     callback = ProgressCallback.requires_progress(
         callback, print_progress, objectpath.name, "synchronizing"
     )
-    kwargs["callback"] = callback
     if objectpath.exists():
-        local_mts = objectpath.stat().st_mtime  # type: ignore
-        need_synchronize = cloud_mts > local_mts
+        local_mts_obj = objectpath.stat().st_mtime  # type: ignore
+        need_synchronize = cloud_mts > local_mts_obj
     else:
         objectpath.parent.mkdir(parents=True, exist_ok=True)
         need_synchronize = True
     if need_synchronize:
-        self.download_to(objectpath, **kwargs)
+        self.download_to(
+            objectpath, recursive=False, print_progress=False, callback=callback
+        )
         os.utime(objectpath, times=(cloud_mts, cloud_mts))
     else:
         # nothing happens if parent_update is not defined
@@ -477,7 +515,7 @@ def compute_file_tree(
             if child_path.is_dir():
                 if include_dirs and child_path not in include_dirs:
                     continue
-                yield prefix + pointer + child_path.name
+                yield prefix + pointer + child_path.name + "/"
                 n_directories += 1
                 n_files_per_dir_and_type = defaultdict(lambda: 0)
                 extension = branch if pointer == tee else space

{lamindb_setup-0.72.1.dist-info → lamindb_setup-0.73.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb_setup
-Version: 0.72.1
+Version: 0.73.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <laminlabs@gmail.com>
 Description-Content-Type: text/markdown

{lamindb_setup-0.72.1.dist-info → lamindb_setup-0.73.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-lamindb_setup/__init__.py,sha256=Uw5mXYU3C6xw4fCmayp05RMBoM0iSQgj2YtJmBbzvxc,1542
+lamindb_setup/__init__.py,sha256=yI2JVyxdrqTPdboJL65uyR5-TzhBhh56tP278f4QY5s,1542
 lamindb_setup/_cache.py,sha256=wA7mbysANwe8hPNbjDo9bOmXJ0xIyaS5iyxIpxSWji4,846
 lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
 lamindb_setup/_check_setup.py,sha256=cNEL9Q4yPpmEkGKHH8JgullWl1VUZwALJ4RHn9wZypY,2613
@@ -8,7 +8,7 @@ lamindb_setup/_delete.py,sha256=Y8KSFYgY0UHAvjd7cCL6hZ_XiLeJwx50BguVATcj_Xo,5524
 lamindb_setup/_django.py,sha256=EoyWvFzH0i9wxjy4JZhcoXCTckztP_Mrl6FbYQnMmLE,1534
 lamindb_setup/_exportdb.py,sha256=uTIZjKKTB7arzEr1j0O6lONiT2pRBKeOFdLvOV8ZwzE,2120
 lamindb_setup/_importdb.py,sha256=yYYShzUajTsR-cTW4CZ-UNDWZY2uE5PAgNbp-wn8Ogc,1874
-lamindb_setup/_init_instance.py,sha256=lR-6txbf3Z2O7ki-DMZWFg36QNiUZ_B5lc6JXjScdus,11897
+lamindb_setup/_init_instance.py,sha256=phw7HW2HHtcQRYXXcHq6O_LyQgSFnotLm1mdP3EvD1U,11918
 lamindb_setup/_migrate.py,sha256=4nBTFg5-BK4A2gH-D3_tcFf8EtvMnIo5Mq0e_C6_9-U,8815
 lamindb_setup/_register_instance.py,sha256=Jeu0wyvJVSVQ_n-A_7yn7xOZIP0ncJD92DRABqzPIjA,940
 lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
@@ -16,7 +16,7 @@ lamindb_setup/_set_managed_storage.py,sha256=mNZrANn-9rwZ0oGWxxg0wS0T0VOQCWyo4nS
 lamindb_setup/_setup_user.py,sha256=6Oc7Rke-yRQSZbuntdUAz8QbJ6UuPzYHI9FnYlf_q-A,3670
 lamindb_setup/_silence_loggers.py,sha256=AKF_YcHvX32eGXdsYK8MJlxEaZ-Uo2f6QDRzjKFCtws,1568
 lamindb_setup/core/__init__.py,sha256=dV9S-rQpNK9JcBn4hiEmiLnmNqfpPFJD9pqagMCaIew,416
-lamindb_setup/core/_aws_credentials.py,sha256=84kugTtELFACJ4SGeTqT-ul0FbOPTe07P0xptv2tOPo,4972
+lamindb_setup/core/_aws_credentials.py,sha256=nK04-lNYz6MYDgD6Z56peoxCULZ82uFDRMzjwsPh25U,5293
 lamindb_setup/core/_aws_storage.py,sha256=nEjeUv4xUVpoV0Lx-zjjmyb9w804bDyaeiM-OqbfwM0,1799
 lamindb_setup/core/_deprecated.py,sha256=3qxUI1dnDlSeR0BYrv7ucjqRBEojbqotPgpShXs4KF8,2520
 lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
@@ -25,20 +25,20 @@ lamindb_setup/core/_hub_core.py,sha256=RGjTqf1owuWmkXAYy0EPaoHAaJ-0T0hAidkqa3cId
 lamindb_setup/core/_hub_crud.py,sha256=b1XF7AJpM9Q-ttm9nPG-r3OTRWHQaGzAGIyvmb83NTo,4859
 lamindb_setup/core/_hub_utils.py,sha256=b_M1LkdCjiMWm1EOlSb9GuPdLijwVgQDtATTpeZuXI0,1875
 lamindb_setup/core/_settings.py,sha256=jjZ_AxRXB3Y3UP6m04BAw_dhFbJbdg2-nZWmEv2LNZ8,3141
-lamindb_setup/core/_settings_instance.py,sha256=w5SBrp6nIJMegzNJSnfQl8HvqEtwgmR7OrayffVedLc,16612
+lamindb_setup/core/_settings_instance.py,sha256=kda3kqUtwh-XZMDmbdFIp8RGGFw0Az8T8g2SKEU65mo,16949
 lamindb_setup/core/_settings_load.py,sha256=NGgCDpN85j1EqoKlrYFIlZBMlBJm33gx2-wc96CP_ZQ,3922
 lamindb_setup/core/_settings_save.py,sha256=d1A-Ex-7H08mb8l7I0Oe0j0GilrfaDuprh_NMxhQAsQ,2704
-lamindb_setup/core/_settings_storage.py,sha256=7f0jt1zcSltpOYDPQ5CVvbBon_d7aneKTte935-2REY,13236
+lamindb_setup/core/_settings_storage.py,sha256=7IUZh0RJgVy9fQKwovXVFQbh8IkU5IJxaLzO2S-mIYk,13137
 lamindb_setup/core/_settings_store.py,sha256=dagS5c7wAMRnuZTRfCU4sKaIOyF_HwAP5Fnnn8vphno,2084
 lamindb_setup/core/_settings_user.py,sha256=P2lC4WDRAFfT-Xq3MlXJ-wMKIHCoGNhMTQfRGIAyUNQ,1344
-lamindb_setup/core/_setup_bionty_sources.py,sha256=OgPpZxN2_Wffy-ogEBz_97c_k8d2bD-DDVt89-u9GLY,3002
+lamindb_setup/core/_setup_bionty_sources.py,sha256=h_pBANsSGK6ujAFsG21mtADHVJoMLKDR4eGgRP4Fgls,3072
 lamindb_setup/core/cloud_sqlite_locker.py,sha256=NIBNAGq7TTRrip9OzMdiQKj8QOuwhL9esyM0aehUqBA,6893
 lamindb_setup/core/django.py,sha256=QUQm3zt5QIiD8uv6o9vbSm_bshqiSWzKSkgD3z2eJCg,3542
 lamindb_setup/core/exceptions.py,sha256=eoI7AXgATgDVzgArtN7CUvpaMUC067vsBg5LHCsWzDM,305
 lamindb_setup/core/hashing.py,sha256=7r96h5JBzuwfOR_gNNqTyWNPKMuiOUfBYwn6sCbZkf8,2269
 lamindb_setup/core/types.py,sha256=bcYnZ0uM_2NXKJCl94Mmc-uYrQlRUUVKG3sK2N-F-N4,532
-lamindb_setup/core/upath.py,sha256=QnAiaOZgT1TLUaX0PEs9dSJ0E4ZDD431hCfKrJIbmqQ,26339
-lamindb_setup-0.72.1.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
-lamindb_setup-0.72.1.dist-info/WHEEL,sha256=Sgu64hAMa6g5FdzHxXv9Xdse9yxpGGMeagVtPMWpJQY,99
-lamindb_setup-0.72.1.dist-info/METADATA,sha256=JPuCGY8Fy7wJzFXSh6xaBJVNBOpz2XeDvSLDzwTEXB8,1620
-lamindb_setup-0.72.1.dist-info/RECORD,,
+lamindb_setup/core/upath.py,sha256=3rCqpcQY9m09Co2aOLqOAbgdfdJlVyc2Spmyr8fBKAk,28135
+lamindb_setup-0.73.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
+lamindb_setup-0.73.0.dist-info/WHEEL,sha256=Sgu64hAMa6g5FdzHxXv9Xdse9yxpGGMeagVtPMWpJQY,99
+lamindb_setup-0.73.0.dist-info/METADATA,sha256=WFMMPhqXq6j9jl5NACjayz-MaN4ey4SVURRo-eb6Dlc,1620
+lamindb_setup-0.73.0.dist-info/RECORD,,