lamindb_setup 1.8.1__py3-none-any.whl → 1.8.3__py3-none-any.whl

lamindb_setup/__init__.py

@@ -35,7 +35,7 @@ Modules & settings:
 
 """
 
-__version__ = "1.8.1"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.8.3"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 

lamindb_setup/_check_setup.py

@@ -147,6 +147,9 @@ def _check_instance_setup(from_module: str | None = None) -> bool:
             else:
                 django_lamin.setup_django(isettings)
                 logger.important(f"connected lamindb: {isettings.slug}")
+                settings._instance_settings = (
+                    isettings  # update of local storage location
+                )
         return django_lamin.IS_SETUP
     else:
         if from_module is not None and settings.auto_connect:

lamindb_setup/_connect_instance.py

@@ -188,13 +188,32 @@ def _connect_instance(
     return isettings
 
 
+def _connect_cli(instance: str) -> None:
+    from lamindb_setup import settings as settings_
+
+    settings_.auto_connect = True
+    owner, name = get_owner_name_from_identifier(instance)
+    isettings = _connect_instance(owner, name)
+    isettings._persist(write_to_disk=True)
+    if not isettings.is_on_hub or isettings._is_cloud_sqlite:
+        # there are two reasons to call the full-blown connect
+        # (1) if the instance is not on the hub, we need to register
+        # potential users through register_user()
+        # (2) if the instance is cloud sqlite, we need to lock it
+        connect(_write_settings=False, _reload_lamindb=False)
+    else:
+        logger.important(f"connected lamindb: {isettings.slug}")
+    return None
+
+
 @unlock_cloud_sqlite_upon_exception(ignore_prev_locker=True)
 def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
     """Connect to an instance.
 
     Args:
         instance: Pass a slug (`account/name`) or URL (`https://lamin.ai/account/name`).
-            If `None`, looks for an environment variable `LAMIN_CURRENT_INSTANCE` to get the instance identifier. If it doesn't find this variable, it connects to the instance that was connected with `lamin connect` through the CLI.
+            If `None`, looks for an environment variable `LAMIN_CURRENT_INSTANCE` to get the instance identifier.
+            If it doesn't find this variable, it connects to the instance that was connected with `lamin connect` through the CLI.
     """
     # validate kwargs
     valid_kwargs = {

lamindb_setup/core/_aws_options.py

@@ -172,11 +172,7 @@ class AWSOptionsManager:
             from ._hub_core import access_aws
             from ._settings import settings
 
-            if settings.user.handle != "anonymous" or access_token is not None:
-                storage_root_info = access_aws(path_str, access_token=access_token)
-            else:
-                storage_root_info = {"credentials": {}, "accessibility": {}}
-
+            storage_root_info = access_aws(path_str, access_token=access_token)
             accessibility = storage_root_info["accessibility"]
             is_managed = accessibility.get("is_managed", False)
             if is_managed:

lamindb_setup/core/_hub_client.py

@@ -5,6 +5,7 @@ import os
 from typing import Literal
 from urllib.request import urlretrieve
 
+from httpx import HTTPTransport
 from lamin_utils import logger
 from pydantic_settings import BaseSettings
 from supabase import Client, create_client  # type: ignore
@@ -60,20 +61,29 @@ class Environment:
         self.supabase_anon_key: str = key
 
 
+DEFAULT_TIMEOUT = 20
+
+
 # runs ~0.5s
 def connect_hub(
     fallback_env: bool = False, client_options: ClientOptions | None = None
 ) -> Client:
     env = Environment(fallback=fallback_env)
     if client_options is None:
-        # function_client_timeout=5 by default
-        # increase to avoid rare timeouts for edge functions
         client_options = ClientOptions(
             auto_refresh_token=False,
-            function_client_timeout=30,
-            postgrest_client_timeout=20,
+            function_client_timeout=DEFAULT_TIMEOUT,
+            postgrest_client_timeout=DEFAULT_TIMEOUT,
         )
-    return create_client(env.supabase_api_url, env.supabase_anon_key, client_options)
+    client = create_client(env.supabase_api_url, env.supabase_anon_key, client_options)
+    # needed to enable retries for http requests in supabase
+    # these are separate clients and need separate transports
+    # retries are done only in case an httpx.ConnectError or an httpx.ConnectTimeout occurs
+    transport_kwargs = {"verify": True, "http2": True, "retries": 2}
+    client.auth._http_client._transport = HTTPTransport(**transport_kwargs)
+    client.functions._client._transport = HTTPTransport(**transport_kwargs)
+    client.postgrest.session._transport = HTTPTransport(**transport_kwargs)
+    return client
 
 
 def connect_hub_with_auth(
@@ -210,11 +220,16 @@ def request_with_auth(
     headers["Authorization"] = f"Bearer {access_token}"
 
     make_request = getattr(requests, method)
-    response = make_request(url, headers=headers, **kwargs)
-    # upate access_token and try again if failed
-    if response.status_code != 200 and renew_token:
+    timeout = kwargs.pop("timeout", DEFAULT_TIMEOUT)
+
+    response = make_request(url, headers=headers, timeout=timeout, **kwargs)
+    status_code = response.status_code
+    # update access_token and try again if failed
+    if not (200 <= status_code < 300) and renew_token:
         from lamindb_setup import settings
 
+        logger.debug(f"{method} {url} failed: {status_code} {response.text}")
+
         access_token = get_access_token(
             settings.user.email, settings.user.password, settings.user.api_key
         )
@@ -224,5 +239,5 @@ def request_with_auth(
 
         headers["Authorization"] = f"Bearer {access_token}"
 
-        response = make_request(url, headers=headers, **kwargs)
+        response = make_request(url, headers=headers, timeout=timeout, **kwargs)
     return response

lamindb_setup/core/_hub_core.py

@@ -460,9 +460,9 @@ def access_aws(storage_root: str, access_token: str | None = None) -> dict[str,
         storage_root_info = call_with_fallback_auth(
             _access_aws, storage_root=storage_root, access_token=access_token
         )
-        return storage_root_info
     else:
-        raise RuntimeError("Can only get access to AWS if authenticated.")
+        storage_root_info = call_with_fallback(_access_aws, storage_root=storage_root)
+    return storage_root_info
 
 
 def _access_aws(*, storage_root: str, client: Client) -> dict[str, dict]:
@@ -497,8 +497,8 @@ def access_db(
     instance_slug: str
     instance_api_url: str | None
     if (
-        "LAMIN_TEST_DB_TOKEN" in os.environ
-        and (env_db_token := os.environ["LAMIN_TEST_DB_TOKEN"]) != ""
+        "LAMIN_DB_TOKEN" in os.environ
+        and (env_db_token := os.environ["LAMIN_DB_TOKEN"]) != ""
     ):
         return env_db_token
 
@@ -531,11 +531,13 @@ def access_db(
         url = instance_api_url + url
 
     response = request_with_auth(url, "get", access_token, renew_token)  # type: ignore
-    response_json = response.json()
-    if response.status_code != 200:
+    status_code = response.status_code
+    if not (200 <= status_code < 300):
         raise PermissionError(
-            f"Fine-grained access to {instance_slug} failed: {response_json}"
+            f"Fine-grained access to {instance_slug} failed: {status_code} {response.text}"
         )
+
+    response_json = response.json()
     if "token" not in response_json:
         raise RuntimeError("The response of access_db does not contain a db token.")
     return response_json["token"]

lamindb_setup/core/_settings.py

@@ -350,7 +350,7 @@ class SetupPaths:
         local_filepath = SetupPaths.cloud_to_local_no_update(filepath, cache_key)
         if not isinstance(filepath, LocalPathClasses):
             local_filepath.parent.mkdir(parents=True, exist_ok=True)
-            filepath.synchronize(local_filepath, **kwargs)  # type: ignore
+            filepath.synchronize_to(local_filepath, **kwargs)  # type: ignore
         return local_filepath
 
 

lamindb_setup/core/_settings_instance.py

@@ -32,7 +32,7 @@ if TYPE_CHECKING:
 
     from ._settings_user import UserSettings
 
-LOCAL_STORAGE_MESSAGE = "No storage location found in current environment: create one via, e.g., ln.Storage(root='/dir/our_shared_dir', host='our-server-123').save()"
+LOCAL_STORAGE_MESSAGE = "No local storage location found in current environment: defaulting to cloud storage"
 
 
 def sanitize_git_repo_url(repo_url: str) -> str:
@@ -104,8 +104,20 @@ class InstanceSettings:
         for attr in attrs:
             value = getattr(self, attr)
             if attr == "storage":
-                representation += f"\n - storage root: {value.root_as_str}"
-                representation += f"\n - storage region: {value.region}"
+                if self.keep_artifacts_local:
+                    import lamindb as ln
+
+                    self._local_storage = ln.setup.settings.instance._local_storage
+                if self._local_storage is not None:
+                    value_local = self.local_storage
+                    representation += f"\n - local storage: {value_local.root_as_str} ({value_local.region})"
+                    representation += (
+                        f"\n - cloud storage: {value.root_as_str} ({value.region})"
+                    )
+                else:
+                    representation += (
+                        f"\n - storage: {value.root_as_str} ({value.region})"
+                    )
             elif attr == "db":
                 if self.dialect != "sqlite":
                     model = LaminDsnModel(db=value)
@@ -156,9 +168,17 @@ class InstanceSettings:
         found = []
         for record in all_local_records:
             root_path = Path(record.root)
-            if root_path.exists():
+            try:
+                root_path_exists = root_path.exists()
+            except PermissionError:
+                continue
+            if root_path_exists:
                 marker_path = root_path / STORAGE_UID_FILE_KEY
-                if not marker_path.exists():
+                try:
+                    marker_path_exists = marker_path.exists()
+                except PermissionError:
+                    continue
+                if not marker_path_exists:
                     legacy_filepath = root_path / LEGACY_STORAGE_UID_FILE_KEY
                     if legacy_filepath.exists():
                         logger.warning(
@@ -182,8 +202,9 @@ class InstanceSettings:
             if len(found) > 1:
                 found_display = "\n - ".join([f"{record.root}" for record in found])
                 logger.important(f"found locations:\n - {found_display}")
+            record = found[0]
             logger.important(f"defaulting to local storage: {record.root}")
-            return StorageSettings(record.root)
+            return StorageSettings(record.root, region=record.region)
         elif not mute_warning:
             start = LOCAL_STORAGE_MESSAGE[0].lower()
             logger.warning(f"{start}{LOCAL_STORAGE_MESSAGE[1:]}")
@@ -193,15 +214,19 @@ class InstanceSettings:
     def keep_artifacts_local(self) -> bool:
         """Default to keeping artifacts local.
 
-        Enable this optional setting for cloud instances on lamin.ai.
-
         Guide: :doc:`faq/keep-artifacts-local`
         """
         return self._keep_artifacts_local
 
+    @keep_artifacts_local.setter
+    def keep_artifacts_local(self, value: bool):
+        if not isinstance(value, bool):
+            raise ValueError("keep_artifacts_local must be a boolean value.")
+        self._keep_artifacts_local = value
+
     @property
     def storage(self) -> StorageSettings:
-        """Default storage.
+        """Default storage of instance.
 
         For a cloud instance, this is cloud storage. For a local instance, this
         is a local directory.
@@ -210,14 +235,16 @@ class InstanceSettings:
 
     @property
     def local_storage(self) -> StorageSettings:
-        """An additional local storage location.
+        """An alternative default local storage location in the current environment.
 
-        Is only available if :attr:`keep_artifacts_local` is enabled.
+        Serves as the default storage location if :attr:`keep_artifacts_local` is enabled.
 
         Guide: :doc:`faq/keep-artifacts-local`
         """
-        if not self._keep_artifacts_local:
-            raise ValueError("`keep_artifacts_local` is not enabled for this instance.")
+        if not self.keep_artifacts_local:
+            raise ValueError(
+                "`keep_artifacts_local` is False, switch via: ln.setup.settings.instance.keep_artifacts_local = True"
+            )
         if self._local_storage is None:
             self._local_storage = self._search_local_root()
         if self._local_storage is None:
@@ -235,7 +262,7 @@ class InstanceSettings:
             local_root, host = local_root_host
 
         local_root = Path(local_root)
-        if not self._keep_artifacts_local:
+        if not self.keep_artifacts_local:
             raise ValueError("`keep_artifacts_local` is not enabled for this instance.")
         local_storage = self._search_local_root(
             local_root=StorageSettings(local_root).root_as_str, mute_warning=True
@@ -370,7 +397,7 @@ class InstanceSettings:
                 self._check_sqlite_lock()
             sqlite_file = self._sqlite_file
             cache_file = self.storage.cloud_to_local_no_update(sqlite_file)
-            sqlite_file.synchronize(cache_file, print_progress=True)  # type: ignore
+            sqlite_file.synchronize_to(cache_file, print_progress=True)  # type: ignore
 
     def _check_sqlite_lock(self):
         if not self._cloud_sqlite_locker.has_lock:

lamindb_setup/core/_settings_storage.py

@@ -19,7 +19,13 @@ from ._aws_options import (
 from ._aws_storage import find_closest_aws_region
 from ._deprecated import deprecated
 from .hashing import hash_and_encode_as_b62
-from .upath import LocalPathClasses, UPath, _split_path_query, create_path
+from .upath import (
+    LocalPathClasses,
+    UPath,
+    _split_path_query,
+    create_path,
+    get_storage_region,
+)
 
 if TYPE_CHECKING:
     from lamindb_setup.types import StorageType, UPathStr
@@ -43,50 +49,6 @@ def instance_uid_from_uuid(instance_id: UUID) -> str:
     return hash_and_encode_as_b62(instance_id.hex)[:12]
 
 
-def get_storage_region(path: UPathStr) -> str | None:
-    path_str = str(path)
-    if path_str.startswith("s3://"):
-        import botocore.session
-        from botocore.config import Config
-        from botocore.exceptions import ClientError
-
-        # check for endpoint_url in storage options if upath
-        if isinstance(path, UPath):
-            endpoint_url = path.storage_options.get("endpoint_url", None)
-        else:
-            endpoint_url = None
-        path_part = path_str.replace("s3://", "")
-        # check for endpoint_url in the path string
-        if "?" in path_part:
-            assert endpoint_url is None
-            path_part, query = _split_path_query(path_part)
-            endpoint_url = query.get("endpoint_url", [None])[0]
-        bucket = path_part.split("/")[0]
-        session = botocore.session.get_session()
-        credentials = session.get_credentials()
-        if credentials is None or credentials.access_key is None:
-            config = Config(signature_version=botocore.session.UNSIGNED)
-        else:
-            config = None
-        s3_client = session.create_client(
-            "s3", endpoint_url=endpoint_url, config=config
-        )
-        try:
-            response = s3_client.head_bucket(Bucket=bucket)
-        except ClientError as exc:
-            response = getattr(exc, "response", {})
-            if response.get("Error", {}).get("Code") == "404":
-                raise exc
-        region = (
-            response.get("ResponseMetadata", {})
-            .get("HTTPHeaders", {})
-            .get("x-amz-bucket-region", None)
-        )
-    else:
-        region = None
-    return region
-
-
 def get_storage_type(root_as_str: str) -> StorageType:
     import fsspec
 

lamindb_setup/core/_settings_user.py

@@ -39,7 +39,7 @@ class UserSettings:
     def __repr__(self) -> str:
         """Rich string representation."""
         representation = "Current user:"
-        attrs = ["handle", "email", "uid"]
+        attrs = ["handle", "uid"]
         for attr in attrs:
             value = getattr(self, attr)
             representation += f"\n - {attr}: {value}"

lamindb_setup/core/django.py

@@ -247,7 +247,7 @@ def setup_django(
     IS_SETUP = True
 
     if isettings.keep_artifacts_local:
-        isettings._search_local_root()
+        isettings._local_storage = isettings._search_local_root()
 
 
 # THIS IS NOT SAFE

lamindb_setup/core/upath.py

@@ -23,6 +23,7 @@ from upath.registry import register_implementation
 from lamindb_setup.errors import StorageNotEmpty
 
 from ._aws_options import HOSTED_BUCKETS, get_aws_options_manager
+from ._deprecated import deprecated
 from .hashing import HASH_LENGTH, b16_to_b64, hash_from_hashes_list, hash_string
 
 if TYPE_CHECKING:
@@ -381,42 +382,29 @@ def upload_from(
         return self
 
 
-def synchronize(
-    self,
-    objectpath: Path,
+def synchronize_to(
+    origin: UPath,
+    destination: Path,
     error_no_origin: bool = True,
     print_progress: bool = False,
-    callback: fsspec.callbacks.Callback | None = None,
-    timestamp: float | None = None,
     just_check: bool = False,
+    **kwargs,
 ) -> bool:
     """Sync to a local destination path."""
-    protocol = self.protocol
-    # optimize the number of network requests
-    if timestamp is not None:
-        is_dir = False
+    destination = destination.resolve()
+    protocol = origin.protocol
+    try:
+        cloud_info = origin.stat().as_info()
         exists = True
-        cloud_mts = timestamp
-    else:
-        try:
-            cloud_stat = self.stat()
-            cloud_info = cloud_stat.as_info()
-            exists = True
-            is_dir = cloud_info["type"] == "directory"
-            if not is_dir:
-                # hf requires special treatment
-                if protocol == "hf":
-                    cloud_mts = cloud_info["last_commit"].date.timestamp()
-                else:
-                    cloud_mts = cloud_stat.st_mtime
-        except FileNotFoundError:
-            exists = False
+        is_dir = cloud_info["type"] == "directory"
+    except FileNotFoundError:
+        exists = False
 
     if not exists:
-        warn_or_error = f"The original path {self} does not exist anymore."
-        if objectpath.exists():
+        warn_or_error = f"The original path {origin} does not exist anymore."
+        if destination.exists():
             warn_or_error += (
-                f"\nHowever, the local path {objectpath} still exists, you might want"
+                f"\nHowever, the local path {destination} still exists, you might want"
                 " to reupload the object back."
             )
             logger.warning(warn_or_error)
@@ -425,113 +413,114 @@ def synchronize(
             raise FileNotFoundError(warn_or_error)
         return False
 
-    # synchronization logic for directories
-    # to synchronize directories, it should be possible to get modification times
+    use_size: bool = False
+    # use casting to int to avoid problems when the local filesystem
+    # discards fractional parts of timestamps
+    if protocol == "s3":
+        get_modified = lambda file_stat: int(file_stat["LastModified"].timestamp())
+    elif protocol == "gs":
+        get_modified = lambda file_stat: int(file_stat["mtime"].timestamp())
+    elif protocol == "hf":
+        get_modified = lambda file_stat: int(file_stat["last_commit"].date.timestamp())
+    else:  #  http etc
+        use_size = True
+        get_modified = lambda file_stat: file_stat["size"]
+
+    if use_size:
+        is_sync_needed = lambda cloud_size, local_stat: cloud_size != local_stat.st_size
+    else:
+        # no need to cast local_stat.st_mtime to int
+        # because if it has the fractional part and cloud_mtime doesn't
+        # and they have the same integer part then cloud_mtime can't be bigger
+        is_sync_needed = (
+            lambda cloud_mtime, local_stat: cloud_mtime > local_stat.st_mtime
+        )
+
+    local_paths: list[Path] = []
+    cloud_stats: dict[str, int]
     if is_dir:
-        files = self.fs.find(str(self), detail=True)
-        if protocol == "s3":
-            get_modified = lambda file_stat: file_stat["LastModified"]
-        elif protocol == "gs":
-            get_modified = lambda file_stat: file_stat["mtime"]
-        elif protocol == "hf":
-            get_modified = lambda file_stat: file_stat["last_commit"].date
-        else:
-            raise ValueError(f"Can't synchronize a directory for {protocol}.")
-        if objectpath.exists():
-            destination_exists = True
-            cloud_mts_max = max(
-                get_modified(file) for file in files.values()
-            ).timestamp()
-            local_mts = [
-                file.stat().st_mtime for file in objectpath.rglob("*") if file.is_file()
-            ]
-            n_local_files = len(local_mts)
-            local_mts_max = max(local_mts)
-            if local_mts_max == cloud_mts_max:
-                need_synchronize = n_local_files != len(files)
-            elif local_mts_max > cloud_mts_max:
-                need_synchronize = False
-            else:
-                need_synchronize = True
-        else:
-            destination_exists = False
-            need_synchronize = True
-        # just check if synchronization is needed
-        if just_check:
-            return need_synchronize
-        if need_synchronize:
-            callback = ProgressCallback.requires_progress(
-                callback, print_progress, objectpath.name, "synchronizing"
-            )
-            callback.set_size(len(files))
-            origin_file_keys = []
-            for file, stat in callback.wrap(files.items()):
-                file_key = PurePosixPath(file).relative_to(self.path).as_posix()
-                origin_file_keys.append(file_key)
-                timestamp = get_modified(stat).timestamp()
-                origin = f"{protocol}://{file}"
-                destination = objectpath / file_key
-                child = callback.branched(origin, destination.as_posix())
-                UPath(origin, **self.storage_options).synchronize(
-                    destination, callback=child, timestamp=timestamp
+        cloud_stats = {
+            file: get_modified(stat)
+            for file, stat in origin.fs.find(origin.as_posix(), detail=True).items()
+        }
+        for cloud_path in cloud_stats:
+            file_key = PurePosixPath(cloud_path).relative_to(origin.path).as_posix()
+            local_paths.append(destination / file_key)
+    else:
+        cloud_stats = {origin.path: get_modified(cloud_info)}
+        local_paths.append(destination)
+
+    local_paths_all: dict[Path, os.stat_result] = {}
+    if destination.exists():
+        if is_dir:
+            local_paths_all = {
+                path: path.stat() for path in destination.rglob("*") if path.is_file()
+            }
+            if not use_size:
+                # cast to int to remove the fractional parts
+                # there is a problem when a fractional part is allowed on one filesystem
+                # but not on the other
+                # so just normalize both to int
+                cloud_mts_max: int = max(cloud_stats.values())
+                local_mts_max: int = int(
+                    max(stat.st_mtime for stat in local_paths_all.values())
                 )
-                child.close()
-            if destination_exists:
-                local_files = [file for file in objectpath.rglob("*") if file.is_file()]
-                if len(local_files) > len(files):
-                    for file in local_files:
-                        if (
-                            file.relative_to(objectpath).as_posix()
-                            not in origin_file_keys
-                        ):
-                            file.unlink()
-                            parent = file.parent
-                            if next(parent.iterdir(), None) is None:
-                                parent.rmdir()
-        return need_synchronize
-
-    # synchronization logic for files
-    callback = ProgressCallback.requires_progress(
-        callback, print_progress, objectpath.name, "synchronizing"
-    )
-    objectpath_exists = objectpath.exists()
-    if objectpath_exists:
-        if cloud_mts != 0:
-            local_mts_obj = objectpath.stat().st_mtime
-            need_synchronize = cloud_mts > local_mts_obj
+                if local_mts_max > cloud_mts_max:
+                    return False
+                elif local_mts_max == cloud_mts_max:
+                    if len(local_paths_all) == len(cloud_stats):
+                        return False
+                    elif just_check:
+                        return True
         else:
-            # this is true for http for example
-            # where size is present but st_mtime is not
-            # we assume that any change without the change in size is unlikely
-            cloud_size = cloud_stat.st_size
-            local_size_obj = objectpath.stat().st_size
-            need_synchronize = cloud_size != local_size_obj
+            local_paths_all = {destination: destination.stat()}
+
+        cloud_files_sync = []
+        local_files_sync = []
+        for i, (cloud_file, cloud_stat) in enumerate(cloud_stats.items()):
+            local_path = local_paths[i]
+            if local_path not in local_paths_all or is_sync_needed(
+                cloud_stat, local_paths_all[local_path]
+            ):
+                cloud_files_sync.append(cloud_file)
+                local_files_sync.append(local_path.as_posix())
     else:
-        if not just_check:
-            objectpath.parent.mkdir(parents=True, exist_ok=True)
-        need_synchronize = True
-    # just check if synchronization is needed
-    if just_check:
-        return need_synchronize
-    if need_synchronize:
-        # just to be sure that overwriting an existing file doesn't corrupt it
-        # we saw some frequent corruption on some systems for unclear reasons
-        if objectpath_exists:
-            objectpath.unlink()
-        # hf has sync filesystem
-        # on sync filesystems ChildProgressCallback.branched()
-        # returns the default callback
-        # this is why a difference between s3 and hf in progress bars
-        self.download_to(
-            objectpath, recursive=False, print_progress=False, callback=callback
+        cloud_files_sync = list(cloud_stats.keys())
+        local_files_sync = [local_path.as_posix() for local_path in local_paths]
+
+    if cloud_files_sync:
+        if just_check:
+            return True
+
+        callback = ProgressCallback.requires_progress(
+            maybe_callback=kwargs.pop("callback", None),
+            print_progress=print_progress,
+            objectname=destination.name,
+            action="synchronizing",
+            adjust_size=False,
         )
-        if cloud_mts != 0:
-            os.utime(objectpath, times=(cloud_mts, cloud_mts))
+        origin.fs.download(
+            cloud_files_sync,
+            local_files_sync,
+            recursive=False,
+            callback=callback,
+            **kwargs,
+        )
+        if not use_size:
+            for i, cloud_file in enumerate(cloud_files_sync):
+                cloud_mtime = cloud_stats[cloud_file]
+                os.utime(local_files_sync[i], times=(cloud_mtime, cloud_mtime))
     else:
-        # nothing happens if parent_update is not defined
-        # because of Callback.no_op
-        callback.parent_update()
-    return need_synchronize
+        return False
+
+    if is_dir and local_paths_all:
+        for path in (path for path in local_paths_all if path not in local_paths):
+            path.unlink()
+            parent = path.parent
+            if next(parent.iterdir(), None) is None:
+                parent.rmdir()
+
+    return True
 
 
 def modified(self) -> datetime | None:
@@ -710,14 +699,7 @@ def to_url(upath):
         raise ValueError("The provided UPath must be an S3 path.")
     key = "/".join(upath.parts[1:])
     bucket = upath.drive
-    if bucket == "scverse-spatial-eu-central-1":
-        region = "eu-central-1"
-    elif f"s3://{bucket}" not in HOSTED_BUCKETS:
-        response = upath.fs.call_s3("head_bucket", Bucket=bucket)
-        headers = response["ResponseMetadata"]["HTTPHeaders"]
-        region = headers.get("x-amz-bucket-region")
-    else:
-        region = bucket.replace("lamin_", "")
+    region = get_storage_region(upath)
     if region == "us-east-1":
         return f"https://{bucket}.s3.amazonaws.com/{key}"
     else:
@@ -740,7 +722,8 @@ def to_url(upath):
 
 # add custom functions
 UPath.modified = property(modified)
-UPath.synchronize = synchronize
+UPath.synchronize = deprecated("synchronize_to")(synchronize_to)
+UPath.synchronize_to = synchronize_to
 UPath.upload_from = upload_from
 UPath.to_url = to_url
 UPath.download_to = download_to
@@ -823,6 +806,67 @@ class S3QueryPath(S3Path):
 register_implementation("s3", S3QueryPath, clobber=True)
 
 
+def get_storage_region(path: UPathStr) -> str | None:
+    upath = UPath(path)
+
+    if upath.protocol != "s3":
+        return None
+
+    bucket = upath.drive
+
+    if bucket == "scverse-spatial-eu-central-1":
+        return "eu-central-1"
+    elif f"s3://{bucket}" in HOSTED_BUCKETS:
+        return bucket.replace("lamin-", "")
+
+    from botocore.exceptions import ClientError
+
+    if isinstance(path, str):
+        import botocore.session
+        from botocore.config import Config
+
+        path_part = path.replace("s3://", "")
+        # check for endpoint_url in the path string
+        if "?" in path_part:
+            path_part, query = _split_path_query(path_part)
+            endpoint_url = query.get("endpoint_url", [None])[0]
+        else:
+            endpoint_url = None
+        session = botocore.session.get_session()
+        credentials = session.get_credentials()
+        if credentials is None or credentials.access_key is None:
+            config = Config(signature_version=botocore.session.UNSIGNED)
+        else:
+            config = None
+        s3_client = session.create_client(
+            "s3", endpoint_url=endpoint_url, config=config
+        )
+        try:
+            response = s3_client.head_bucket(Bucket=bucket)
+        except ClientError as exc:
+            response = getattr(exc, "response", {})
+            if response.get("Error", {}).get("Code") == "404":
+                raise exc
+    else:
+        upath = get_aws_options_manager()._path_inject_options(upath, {})
+        try:
+            response = upath.fs.call_s3("head_bucket", Bucket=bucket)
+        except Exception as exc:
+            cause = getattr(exc, "__cause__", None)
+            if not isinstance(cause, ClientError):
+                raise exc
+            response = getattr(cause, "response", {})
+            if response.get("Error", {}).get("Code") == "404":
+                raise exc
+
+    region = (
+        response.get("ResponseMetadata", {})
+        .get("HTTPHeaders", {})
+        .get("x-amz-bucket-region", None)
+    )
+    return region
+
+
 def create_path(path: UPathStr, access_token: str | None = None) -> UPath:
     upath = UPath(path).expanduser()
 

{lamindb_setup-1.8.1.dist-info → lamindb_setup-1.8.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 1.8.1
+Version: 1.8.3
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10

{lamindb_setup-1.8.1.dist-info → lamindb_setup-1.8.3.dist-info}/RECORD

@@ -1,8 +1,8 @@
-lamindb_setup/__init__.py,sha256=AxSsbRTefAmDAw_GNRIXygb7gedXU5pMXoRM2oXmvqY,2782
+lamindb_setup/__init__.py,sha256=Pdh2-wDgY4x06M23rDIrY4XP4QrMck8SOSU00SAoSsw,2782
 lamindb_setup/_cache.py,sha256=5o749NuW6zi6uP4rmBtwxg7ifWpAHXVngzC0tEgXLgo,2776
 lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
-lamindb_setup/_check_setup.py,sha256=bXuqx2HEc178RM7gbKZQ65PEVJFu6uSOKiHAs_xz6GI,5575
-lamindb_setup/_connect_instance.py,sha256=wQARuWyPSc9V5gYSADi_GIrwkyxNlZmsQT8-SbDHh-8,12724
+lamindb_setup/_check_setup.py,sha256=eeg7Vr7tUaTDObxq1X7J3TDPQZeitb_Uy6dxqa9xfzs,5707
+lamindb_setup/_connect_instance.py,sha256=PDvtAEHYJQVy-aMPNupN1u6PG9Rb_85JNKcjrOeHNy0,13478
 lamindb_setup/_delete.py,sha256=2KnZOqd5Kgr45XzjiDE9der35LODDUajZD6_hcurGtQ,5676
 lamindb_setup/_disconnect.py,sha256=p6tRLhixU4CuSxMKqzGTr-ovKmTRlZ8aID5dWQxOsg8,1092
 lamindb_setup/_django.py,sha256=uIQflpkp8l3axyPaKURlk3kacgpElVP5KOKmFxYSMGk,1454
@@ -21,30 +21,30 @@ lamindb_setup/errors.py,sha256=H1UM-bii0U2vPyjprOBgZK4ijZJgzgCViyGWPd8v5yU,1493
 lamindb_setup/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lamindb_setup/types.py,sha256=XlXLb4nmbc68uBj5Hp3xpDRezYGJIBZv6jAAqqN0p10,614
 lamindb_setup/core/__init__.py,sha256=5M4A6CVHBO_T5Rr9MeLaPW3WTk4-y00cgRYEgUJVU5U,410
-lamindb_setup/core/_aws_options.py,sha256=JN6fJNcotdIuT-WkBRKDPdyDri9XmorEX2unbuC0KuQ,7738
+lamindb_setup/core/_aws_options.py,sha256=b8gAk9B1kjxhf73kTJpnM-uMre1koL0MPTCq9uwpEaE,7558
 lamindb_setup/core/_aws_storage.py,sha256=nEjeUv4xUVpoV0Lx-zjjmyb9w804bDyaeiM-OqbfwM0,1799
 lamindb_setup/core/_deprecated.py,sha256=HN7iUBdEgahw5e4NHCd1VJooUfieNb6GRzS5x8jU-q8,2549
 lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
-lamindb_setup/core/_hub_client.py,sha256=iMAEwhvhsI85DWZdNT9Vv1K-AlXHw5dmTFDZB3FyAKE,7718
-lamindb_setup/core/_hub_core.py,sha256=x77WpaPMr1uA1kJBSvM5JRNyRG0vYIVlxQbhjAdOhu8,23862
+lamindb_setup/core/_hub_client.py,sha256=jICkfWW1eZoxh3ycviztBGqCJH53uVve5Xawbj8RZR4,8433
+lamindb_setup/core/_hub_core.py,sha256=S3XjVA-bBchvyzTr-jNae4oLuN2Ho8A30me0MMswln0,23917
 lamindb_setup/core/_hub_crud.py,sha256=Jz0d8wFKM1Pv9B9byyUJPlCIMkIzk56Jd-c3Awpm9Xw,5730
 lamindb_setup/core/_hub_utils.py,sha256=6dyDGyzYFgVfR_lE3VN3CP1jGp98gxPtr-T91PAP05U,2687
 lamindb_setup/core/_private_django_api.py,sha256=By63l3vIEtK1pq246FhHq3tslxsaTJGKm5VakYluWp4,2656
-lamindb_setup/core/_settings.py,sha256=0la2eYAHcdMYophheGsgQaD2nJGQrPuX4jMV5GKDiC0,12946
-lamindb_setup/core/_settings_instance.py,sha256=ML5NUZzmDGPhEJdgK-gzQ4f9zwb0NeeXe13sgQrP-1A,20732
+lamindb_setup/core/_settings.py,sha256=EtlxhtAdclS6rDRh5mrwh_q3gA7SJ1eF2rO1QtOYnnE,12949
+lamindb_setup/core/_settings_instance.py,sha256=coGnDwQaqhviVV7wJnCWpL4uMz8IvY5sFALpYdh6wTc,21869
 lamindb_setup/core/_settings_load.py,sha256=JWd0_hBy04xjKo-tH4y8C9RkaywjrmoT0PsKzVme0n4,5176
 lamindb_setup/core/_settings_save.py,sha256=XZx-vow7BT6y3JpRBB2UOJp2vwc7jOGea4wSgOPqjPU,3262
-lamindb_setup/core/_settings_storage.py,sha256=wVwDDD51UsJz5gS6juERbWHCdyucP0R2DLaUGhnLUpQ,17079
+lamindb_setup/core/_settings_storage.py,sha256=S9AvKLzJX0M_RsYcBKZB_P84CYtTY0hyeffYE3UqrQA,15478
 lamindb_setup/core/_settings_store.py,sha256=QmeWIGdIyq7UmjfHiEB_0xRD8hY-8-ZR2WntIKfwTKI,2714
-lamindb_setup/core/_settings_user.py,sha256=K2a6nQ0fhEiSb9mCY_p6ItNrHZ3J_j7EfO7CjZap9aA,1462
+lamindb_setup/core/_settings_user.py,sha256=gFfyMf-738onbh1Mf4wsmLlenQJPtjQfpUgKnOlqc2o,1453
 lamindb_setup/core/_setup_bionty_sources.py,sha256=ox3X-SHiHa2lNPSWjwZhINypbLacX6kGwH6hVVrSFZc,1505
 lamindb_setup/core/cloud_sqlite_locker.py,sha256=H_CTUCjURFXwD1cCtV_Jn0_60iztZTkaesLLXIBgIxc,7204
-lamindb_setup/core/django.py,sha256=dOt1OkUnZeYOo-LTjatQWQFh_MnjRf9IwwvRZhCwdZQ,9637
+lamindb_setup/core/django.py,sha256=MPBcapHL6mH_ebin32aales0JL3AMYqHXUut8ggVFtY,9664
 lamindb_setup/core/exceptions.py,sha256=qjMzqy_uzPA7mCOdnoWnS_fdA6OWbdZGftz-YYplrY0,84
 lamindb_setup/core/hashing.py,sha256=Y8Uc5uSGTfU6L2R_gb5w8DdHhGRog7RnkK-e9FEMjPY,3680
 lamindb_setup/core/types.py,sha256=T7NwspfRHgIIpYsXDcApks8jkOlGeGRW-YbVLB7jNIo,67
-lamindb_setup/core/upath.py,sha256=UDPcZaNp3WtZSYrX8hGQ1lq214dBn079bz0FPWFjIbA,34449
-lamindb_setup-1.8.1.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
-lamindb_setup-1.8.1.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
-lamindb_setup-1.8.1.dist-info/METADATA,sha256=jV9tU71hkGNvtGWxfr_jcw9JgOnWgHkJWlolAHORMO4,1797
-lamindb_setup-1.8.1.dist-info/RECORD,,
+lamindb_setup/core/upath.py,sha256=W47O9-Y205j29iWJ3RKKdomA587hGvoiD6_krASGFcM,35315
+lamindb_setup-1.8.3.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
+lamindb_setup-1.8.3.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
+lamindb_setup-1.8.3.dist-info/METADATA,sha256=nua4OQ80w2e2G-gik39js5-KCXteq9hINyHknrw0NGI,1797
+lamindb_setup-1.8.3.dist-info/RECORD,,