lamindb_setup 1.8.0__tar.gz → 1.8.2__tar.gz

{lamindb_setup-1.8.0 → lamindb_setup-1.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 1.8.0
+Version: 1.8.2
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10

{lamindb_setup-1.8.0 → lamindb_setup-1.8.2}/docs/hub-cloud/06-connect-hosted-instance.ipynb

@@ -16,6 +16,7 @@
     "import pytest\n",
     "import shutil\n",
     "import lamindb_setup as ln_setup\n",
+    "from lamindb_setup._connect_instance import _connect_cli\n",
     "from lamindb_setup.core.upath import UPath\n",
     "from lamindb_setup.core._hub_core import delete_instance\n",
     "from lamindb_setup.core._hub_client import connect_hub_with_auth\n",
@@ -31,7 +32,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "ln_setup.connect(f\"testuser1/{instance_name}\")"
+    "_connect_cli(f\"testuser1/{instance_name}\")  # cover here for cloud sqlite"
    ]
   },
   {
@@ -223,7 +224,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.17"
+   "version": "3.10.16"
   }
  },
  "nbformat": 4,

{lamindb_setup-1.8.0 → lamindb_setup-1.8.2}/docs/hub-cloud/07-keep-artifacts-local.ipynb

@@ -61,9 +61,8 @@
     "ln_setup.settings.instance._keep_artifacts_local = True\n",
     "with pytest.raises(ValueError) as error:\n",
     "    ln_setup.settings.instance.local_storage\n",
-    "assert (\n",
-    "    error.exconly()\n",
-    "    == \"ValueError: No storage location found in current environment: create one via, e.g., ln.Storage(root='/dir/our_shared_dir', host='our-server-123).save()\"\n",
+    "assert error.exconly().startswith(\n",
+    "    \"ValueError: No local storage location found in current environment:\"\n",
     ")"
    ]
   },

{lamindb_setup-1.8.0 → lamindb_setup-1.8.2}/docs/hub-prod/test-cloud-sync.ipynb

@@ -229,7 +229,7 @@
     ")\n",
     "assert dir_sync_local == settings.cache_dir / \"dir_cache/key\"\n",
     "\n",
-    "assert dir_sync.synchronize(dir_sync_local, just_check=True)\n",
+    "assert dir_sync.synchronize_to(dir_sync_local, just_check=True)\n",
     "assert not dir_sync_local.exists()"
    ]
   },
@@ -240,7 +240,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert dir_sync.synchronize(dir_sync_local, just_check=False)\n",
+    "assert dir_sync.synchronize_to(dir_sync_local, just_check=False)\n",
     "assert dir_sync_local.is_dir()\n",
     "assert num_files(dir_sync_local) == 2"
    ]
@@ -308,6 +308,28 @@
     "http_local.unlink()"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7e610d67",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# test download_to\n",
+    "http_path.download_to(http_local, print_progress=True)\n",
+    "assert not http_path.synchronize_to(http_local)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1e293d0a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "http_local.unlink()"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "574c3f95",
@@ -323,7 +345,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "dir_sync_local = settings.paths.cloud_to_local(dir_sync)"
+    "dir_sync_local = settings.paths.cloud_to_local(dir_sync)\n",
+    "assert not dir_sync.synchronize_to(dir_sync_local)"
    ]
   },
   {
@@ -459,7 +482,7 @@
    "source": [
     "dir_sync_local = settings.paths.cloud_to_local(dir_sync)\n",
     "\n",
-    "assert num_files(dir_sync_local) == 2\n",
+    "assert num_files(dir_sync_local) == 2, list(dir_sync_local.rglob(\"*\"))\n",
     "assert not local_file_new.exists()\n",
     "assert not local_file_new_parent.exists()\n",
     "\n",
@@ -759,7 +782,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert hf_path.synchronize(hf_path_local, just_check=True)\n",
+    "assert hf_path.synchronize_to(hf_path_local, just_check=True)\n",
     "assert not hf_path_local.exists()"
    ]
   },
@@ -770,7 +793,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "assert hf_path.synchronize(hf_path_local)\n",
+    "assert hf_path.synchronize_to(hf_path_local)\n",
     "assert hf_path_local.is_file()"
    ]
   },
@@ -793,7 +816,7 @@
    "source": [
     "hf_path = UPath(\"hf://datasets/Koncopd/lamindb-test@main/does_not_exist.file\")\n",
     "with pytest.raises(FileNotFoundError):\n",
-    "    hf_path.synchronize(UPath(\"./does_not_exist.file\"), error_no_origin=True)"
+    "    hf_path.synchronize_to(UPath(\"./does_not_exist.file\"), error_no_origin=True)"
    ]
   },
   {

{lamindb_setup-1.8.0 → lamindb_setup-1.8.2}/lamindb_setup/__init__.py

@@ -35,7 +35,7 @@ Modules & settings:
 
 """
 
-__version__ = "1.8.0"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.8.2"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 

{lamindb_setup-1.8.0 → lamindb_setup-1.8.2}/lamindb_setup/_connect_instance.py

@@ -188,13 +188,32 @@ def _connect_instance(
     return isettings
 
 
+def _connect_cli(instance: str) -> None:
+    from lamindb_setup import settings as settings_
+
+    settings_.auto_connect = True
+    owner, name = get_owner_name_from_identifier(instance)
+    isettings = _connect_instance(owner, name)
+    isettings._persist(write_to_disk=True)
+    if not isettings.is_on_hub or isettings._is_cloud_sqlite:
+        # there are two reasons to call the full-blown connect
+        # (1) if the instance is not on the hub, we need to register
+        # potential users through register_user()
+        # (2) if the instance is cloud sqlite, we need to lock it
+        connect(_write_settings=False, _reload_lamindb=False)
+    else:
+        logger.important(f"connected lamindb: {isettings.slug}")
+    return None
+
+
 @unlock_cloud_sqlite_upon_exception(ignore_prev_locker=True)
 def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
     """Connect to an instance.
 
     Args:
         instance: Pass a slug (`account/name`) or URL (`https://lamin.ai/account/name`).
-            If `None`, looks for an environment variable `LAMIN_CURRENT_INSTANCE` to get the instance identifier. If it doesn't find this variable, it connects to the instance that was connected with `lamin connect` through the CLI.
+            If `None`, looks for an environment variable `LAMIN_CURRENT_INSTANCE` to get the instance identifier.
+            If it doesn't find this variable, it connects to the instance that was connected with `lamin connect` through the CLI.
     """
     # validate kwargs
     valid_kwargs = {

{lamindb_setup-1.8.0 → lamindb_setup-1.8.2}/lamindb_setup/core/_hub_client.py

@@ -5,6 +5,7 @@ import os
 from typing import Literal
 from urllib.request import urlretrieve
 
+from httpx import HTTPTransport
 from lamin_utils import logger
 from pydantic_settings import BaseSettings
 from supabase import Client, create_client  # type: ignore
@@ -60,20 +61,29 @@ class Environment:
         self.supabase_anon_key: str = key
 
 
+DEFAULT_TIMEOUT = 20
+
+
 # runs ~0.5s
 def connect_hub(
     fallback_env: bool = False, client_options: ClientOptions | None = None
 ) -> Client:
     env = Environment(fallback=fallback_env)
     if client_options is None:
-        # function_client_timeout=5 by default
-        # increase to avoid rare timeouts for edge functions
         client_options = ClientOptions(
             auto_refresh_token=False,
-            function_client_timeout=30,
-            postgrest_client_timeout=20,
+            function_client_timeout=DEFAULT_TIMEOUT,
+            postgrest_client_timeout=DEFAULT_TIMEOUT,
         )
-    return create_client(env.supabase_api_url, env.supabase_anon_key, client_options)
+    client = create_client(env.supabase_api_url, env.supabase_anon_key, client_options)
+    # needed to enable retries for http requests in supabase
+    # these are separate clients and need separate transports
+    # retries are done only in case an httpx.ConnectError or an httpx.ConnectTimeout occurs
+    transport_kwargs = {"verify": True, "http2": True, "retries": 2}
+    client.auth._http_client._transport = HTTPTransport(**transport_kwargs)
+    client.functions._client._transport = HTTPTransport(**transport_kwargs)
+    client.postgrest.session._transport = HTTPTransport(**transport_kwargs)
+    return client
 
 
 def connect_hub_with_auth(
@@ -210,11 +220,16 @@ def request_with_auth(
     headers["Authorization"] = f"Bearer {access_token}"
 
     make_request = getattr(requests, method)
-    response = make_request(url, headers=headers, **kwargs)
-    # upate access_token and try again if failed
-    if response.status_code != 200 and renew_token:
+    timeout = kwargs.pop("timeout", DEFAULT_TIMEOUT)
+
+    response = make_request(url, headers=headers, timeout=timeout, **kwargs)
+    status_code = response.status_code
+    # update access_token and try again if failed
+    if not (200 <= status_code < 300) and renew_token:
         from lamindb_setup import settings
 
+        logger.debug(f"{method} {url} failed: {status_code} {response.text}")
+
         access_token = get_access_token(
             settings.user.email, settings.user.password, settings.user.api_key
         )
@@ -224,5 +239,5 @@ def request_with_auth(
 
         headers["Authorization"] = f"Bearer {access_token}"
 
-        response = make_request(url, headers=headers, **kwargs)
+        response = make_request(url, headers=headers, timeout=timeout, **kwargs)
     return response

{lamindb_setup-1.8.0 → lamindb_setup-1.8.2}/lamindb_setup/core/_hub_core.py

@@ -531,11 +531,13 @@ def access_db(
         url = instance_api_url + url
 
     response = request_with_auth(url, "get", access_token, renew_token)  # type: ignore
-    response_json = response.json()
-    if response.status_code != 200:
+    status_code = response.status_code
+    if not (200 <= status_code < 300):
         raise PermissionError(
-            f"Fine-grained access to {instance_slug} failed: {response_json}"
+            f"Fine-grained access to {instance_slug} failed: {status_code} {response.text}"
         )
+
+    response_json = response.json()
     if "token" not in response_json:
         raise RuntimeError("The response of access_db does not contain a db token.")
     return response_json["token"]

{lamindb_setup-1.8.0 → lamindb_setup-1.8.2}/lamindb_setup/core/_settings.py

@@ -350,7 +350,7 @@ class SetupPaths:
         local_filepath = SetupPaths.cloud_to_local_no_update(filepath, cache_key)
         if not isinstance(filepath, LocalPathClasses):
             local_filepath.parent.mkdir(parents=True, exist_ok=True)
-            filepath.synchronize(local_filepath, **kwargs)  # type: ignore
+            filepath.synchronize_to(local_filepath, **kwargs)  # type: ignore
         return local_filepath
 
 

{lamindb_setup-1.8.0 → lamindb_setup-1.8.2}/lamindb_setup/core/_settings_instance.py

@@ -32,7 +32,7 @@ if TYPE_CHECKING:
 
     from ._settings_user import UserSettings
 
-LOCAL_STORAGE_MESSAGE = "No storage location found in current environment: create one via, e.g., ln.Storage(root='/dir/our_shared_dir', host='our-server-123).save()"
+LOCAL_STORAGE_MESSAGE = "No local storage location found in current environment: defaulting to cloud storage"
 
 
 def sanitize_git_repo_url(repo_url: str) -> str:
@@ -156,9 +156,17 @@ class InstanceSettings:
         found = []
         for record in all_local_records:
             root_path = Path(record.root)
-            if root_path.exists():
+            try:
+                root_path_exists = root_path.exists()
+            except PermissionError:
+                continue
+            if root_path_exists:
                 marker_path = root_path / STORAGE_UID_FILE_KEY
-                if not marker_path.exists():
+                try:
+                    marker_path_exists = marker_path.exists()
+                except PermissionError:
+                    continue
+                if not marker_path_exists:
                     legacy_filepath = root_path / LEGACY_STORAGE_UID_FILE_KEY
                     if legacy_filepath.exists():
                         logger.warning(
@@ -193,15 +201,19 @@ class InstanceSettings:
     def keep_artifacts_local(self) -> bool:
         """Default to keeping artifacts local.
 
-        Enable this optional setting for cloud instances on lamin.ai.
-
         Guide: :doc:`faq/keep-artifacts-local`
         """
         return self._keep_artifacts_local
 
+    @keep_artifacts_local.setter
+    def keep_artifacts_local(self, value: bool):
+        if not isinstance(value, bool):
+            raise ValueError("keep_artifacts_local must be a boolean value.")
+        self._keep_artifacts_local = value
+
     @property
     def storage(self) -> StorageSettings:
-        """Default storage.
+        """Default storage of instance.
 
         For a cloud instance, this is cloud storage. For a local instance, this
         is a local directory.
@@ -210,13 +222,13 @@ class InstanceSettings:
 
     @property
     def local_storage(self) -> StorageSettings:
-        """An additional local storage location.
+        """An alternative default local storage location in the current environment.
 
-        Is only available if :attr:`keep_artifacts_local` is enabled.
+        Serves as the default storage location if :attr:`keep_artifacts_local` is enabled.
 
         Guide: :doc:`faq/keep-artifacts-local`
         """
-        if not self._keep_artifacts_local:
+        if not self.keep_artifacts_local:
             raise ValueError("`keep_artifacts_local` is not enabled for this instance.")
         if self._local_storage is None:
             self._local_storage = self._search_local_root()
@@ -235,7 +247,7 @@ class InstanceSettings:
             local_root, host = local_root_host
 
         local_root = Path(local_root)
-        if not self._keep_artifacts_local:
+        if not self.keep_artifacts_local:
             raise ValueError("`keep_artifacts_local` is not enabled for this instance.")
         local_storage = self._search_local_root(
             local_root=StorageSettings(local_root).root_as_str, mute_warning=True
@@ -264,17 +276,21 @@ class InstanceSettings:
             )
         local_root = UPath(local_root)
         assert isinstance(local_root, LocalPathClasses)
-        self._local_storage, _ = init_storage(
+        tentative_storage, hub_status = init_storage(
             local_root,
             instance_id=self._id,
             instance_slug=self.slug,
             register_hub=True,
             region=host,
         )  # type: ignore
-        register_storage_in_instance(self._local_storage)  # type: ignore
-        logger.important(
-            f"defaulting to local storage: {self._local_storage.root} on host {host}"
-        )
+        if hub_status in ["hub-record-created", "hub-record-retrieved"]:
+            register_storage_in_instance(tentative_storage)  # type: ignore
+            self._local_storage = tentative_storage
+            logger.important(
+                f"defaulting to local storage: {self._local_storage.root} on host {host}"
+            )
+        else:
+            logger.warning(f"could not set this local storage location: {local_root}")
 
     @property
     @deprecated("local_storage")
@@ -366,7 +382,7 @@ class InstanceSettings:
                 self._check_sqlite_lock()
             sqlite_file = self._sqlite_file
             cache_file = self.storage.cloud_to_local_no_update(sqlite_file)
-            sqlite_file.synchronize(cache_file, print_progress=True)  # type: ignore
+            sqlite_file.synchronize_to(cache_file, print_progress=True)  # type: ignore
 
     def _check_sqlite_lock(self):
         if not self._cloud_sqlite_locker.has_lock:

{lamindb_setup-1.8.0 → lamindb_setup-1.8.2}/lamindb_setup/core/_settings_storage.py

@@ -19,7 +19,13 @@ from ._aws_options import (
 from ._aws_storage import find_closest_aws_region
 from ._deprecated import deprecated
 from .hashing import hash_and_encode_as_b62
-from .upath import LocalPathClasses, UPath, _split_path_query, create_path
+from .upath import (
+    LocalPathClasses,
+    UPath,
+    _split_path_query,
+    create_path,
+    get_storage_region,
+)
 
 if TYPE_CHECKING:
     from lamindb_setup.types import StorageType, UPathStr
@@ -43,50 +49,6 @@ def instance_uid_from_uuid(instance_id: UUID) -> str:
     return hash_and_encode_as_b62(instance_id.hex)[:12]
 
 
-def get_storage_region(path: UPathStr) -> str | None:
-    path_str = str(path)
-    if path_str.startswith("s3://"):
-        import botocore.session
-        from botocore.config import Config
-        from botocore.exceptions import ClientError
-
-        # check for endpoint_url in storage options if upath
-        if isinstance(path, UPath):
-            endpoint_url = path.storage_options.get("endpoint_url", None)
-        else:
-            endpoint_url = None
-        path_part = path_str.replace("s3://", "")
-        # check for endpoint_url in the path string
-        if "?" in path_part:
-            assert endpoint_url is None
-            path_part, query = _split_path_query(path_part)
-            endpoint_url = query.get("endpoint_url", [None])[0]
-        bucket = path_part.split("/")[0]
-        session = botocore.session.get_session()
-        credentials = session.get_credentials()
-        if credentials is None or credentials.access_key is None:
-            config = Config(signature_version=botocore.session.UNSIGNED)
-        else:
-            config = None
-        s3_client = session.create_client(
-            "s3", endpoint_url=endpoint_url, config=config
-        )
-        try:
-            response = s3_client.head_bucket(Bucket=bucket)
-        except ClientError as exc:
-            response = getattr(exc, "response", {})
-            if response.get("Error", {}).get("Code") == "404":
-                raise exc
-        region = (
-            response.get("ResponseMetadata", {})
-            .get("HTTPHeaders", {})
-            .get("x-amz-bucket-region", None)
-        )
-    else:
-        region = None
-    return region
-
-
 def get_storage_type(root_as_str: str) -> StorageType:
     import fsspec
 

{lamindb_setup-1.8.0 → lamindb_setup-1.8.2}/lamindb_setup/core/upath.py

@@ -23,6 +23,7 @@ from upath.registry import register_implementation
 from lamindb_setup.errors import StorageNotEmpty
 
 from ._aws_options import HOSTED_BUCKETS, get_aws_options_manager
+from ._deprecated import deprecated
 from .hashing import HASH_LENGTH, b16_to_b64, hash_from_hashes_list, hash_string
 
 if TYPE_CHECKING:
@@ -381,42 +382,29 @@ def upload_from(
         return self
 
 
-def synchronize(
-    self,
-    objectpath: Path,
+def synchronize_to(
+    origin: UPath,
+    destination: Path,
     error_no_origin: bool = True,
     print_progress: bool = False,
-    callback: fsspec.callbacks.Callback | None = None,
-    timestamp: float | None = None,
     just_check: bool = False,
+    **kwargs,
 ) -> bool:
     """Sync to a local destination path."""
-    protocol = self.protocol
-    # optimize the number of network requests
-    if timestamp is not None:
-        is_dir = False
+    destination = destination.resolve()
+    protocol = origin.protocol
+    try:
+        cloud_info = origin.stat().as_info()
         exists = True
-        cloud_mts = timestamp
-    else:
-        try:
-            cloud_stat = self.stat()
-            cloud_info = cloud_stat.as_info()
-            exists = True
-            is_dir = cloud_info["type"] == "directory"
-            if not is_dir:
-                # hf requires special treatment
-                if protocol == "hf":
-                    cloud_mts = cloud_info["last_commit"].date.timestamp()
-                else:
-                    cloud_mts = cloud_stat.st_mtime
-        except FileNotFoundError:
-            exists = False
+        is_dir = cloud_info["type"] == "directory"
+    except FileNotFoundError:
+        exists = False
 
     if not exists:
-        warn_or_error = f"The original path {self} does not exist anymore."
-        if objectpath.exists():
+        warn_or_error = f"The original path {origin} does not exist anymore."
+        if destination.exists():
             warn_or_error += (
-                f"\nHowever, the local path {objectpath} still exists, you might want"
+                f"\nHowever, the local path {destination} still exists, you might want"
                 " to reupload the object back."
             )
             logger.warning(warn_or_error)
@@ -425,113 +413,114 @@ def synchronize(
             raise FileNotFoundError(warn_or_error)
         return False
 
-    # synchronization logic for directories
-    # to synchronize directories, it should be possible to get modification times
+    use_size: bool = False
+    # use casting to int to avoid problems when the local filesystem
+    # discards fractional parts of timestamps
+    if protocol == "s3":
+        get_modified = lambda file_stat: int(file_stat["LastModified"].timestamp())
+    elif protocol == "gs":
+        get_modified = lambda file_stat: int(file_stat["mtime"].timestamp())
+    elif protocol == "hf":
+        get_modified = lambda file_stat: int(file_stat["last_commit"].date.timestamp())
+    else:  #  http etc
+        use_size = True
+        get_modified = lambda file_stat: file_stat["size"]
+
+    if use_size:
+        is_sync_needed = lambda cloud_size, local_stat: cloud_size != local_stat.st_size
+    else:
+        # no need to cast local_stat.st_mtime to int
+        # because if it has the fractional part and cloud_mtime doesn't
+        # and they have the same integer part then cloud_mtime can't be bigger
+        is_sync_needed = (
+            lambda cloud_mtime, local_stat: cloud_mtime > local_stat.st_mtime
+        )
+
+    local_paths: list[Path] = []
+    cloud_stats: dict[str, int]
     if is_dir:
-        files = self.fs.find(str(self), detail=True)
-        if protocol == "s3":
-            get_modified = lambda file_stat: file_stat["LastModified"]
-        elif protocol == "gs":
-            get_modified = lambda file_stat: file_stat["mtime"]
-        elif protocol == "hf":
-            get_modified = lambda file_stat: file_stat["last_commit"].date
-        else:
-            raise ValueError(f"Can't synchronize a directory for {protocol}.")
-        if objectpath.exists():
-            destination_exists = True
-            cloud_mts_max = max(
-                get_modified(file) for file in files.values()
-            ).timestamp()
-            local_mts = [
-                file.stat().st_mtime for file in objectpath.rglob("*") if file.is_file()
-            ]
-            n_local_files = len(local_mts)
-            local_mts_max = max(local_mts)
-            if local_mts_max == cloud_mts_max:
-                need_synchronize = n_local_files != len(files)
-            elif local_mts_max > cloud_mts_max:
-                need_synchronize = False
-            else:
-                need_synchronize = True
-        else:
-            destination_exists = False
-            need_synchronize = True
-        # just check if synchronization is needed
-        if just_check:
-            return need_synchronize
-        if need_synchronize:
-            callback = ProgressCallback.requires_progress(
-                callback, print_progress, objectpath.name, "synchronizing"
-            )
-            callback.set_size(len(files))
-            origin_file_keys = []
-            for file, stat in callback.wrap(files.items()):
-                file_key = PurePosixPath(file).relative_to(self.path).as_posix()
-                origin_file_keys.append(file_key)
-                timestamp = get_modified(stat).timestamp()
-                origin = f"{protocol}://{file}"
-                destination = objectpath / file_key
-                child = callback.branched(origin, destination.as_posix())
-                UPath(origin, **self.storage_options).synchronize(
-                    destination, callback=child, timestamp=timestamp
+        cloud_stats = {
+            file: get_modified(stat)
+            for file, stat in origin.fs.find(origin.as_posix(), detail=True).items()
+        }
+        for cloud_path in cloud_stats:
+            file_key = PurePosixPath(cloud_path).relative_to(origin.path).as_posix()
+            local_paths.append(destination / file_key)
+    else:
+        cloud_stats = {origin.path: get_modified(cloud_info)}
+        local_paths.append(destination)
+
+    local_paths_all: dict[Path, os.stat_result] = {}
+    if destination.exists():
+        if is_dir:
+            local_paths_all = {
+                path: path.stat() for path in destination.rglob("*") if path.is_file()
+            }
+            if not use_size:
+                # cast to int to remove the fractional parts
+                # there is a problem when a fractional part is allowed on one filesystem
+                # but not on the other
+                # so just normalize both to int
+                cloud_mts_max: int = max(cloud_stats.values())
+                local_mts_max: int = int(
+                    max(stat.st_mtime for stat in local_paths_all.values())
                 )
-                child.close()
-            if destination_exists:
-                local_files = [file for file in objectpath.rglob("*") if file.is_file()]
-                if len(local_files) > len(files):
-                    for file in local_files:
-                        if (
-                            file.relative_to(objectpath).as_posix()
-                            not in origin_file_keys
-                        ):
-                            file.unlink()
-                            parent = file.parent
-                            if next(parent.iterdir(), None) is None:
-                                parent.rmdir()
-        return need_synchronize
-
-    # synchronization logic for files
-    callback = ProgressCallback.requires_progress(
-        callback, print_progress, objectpath.name, "synchronizing"
-    )
-    objectpath_exists = objectpath.exists()
-    if objectpath_exists:
-        if cloud_mts != 0:
-            local_mts_obj = objectpath.stat().st_mtime
-            need_synchronize = cloud_mts > local_mts_obj
+                if local_mts_max > cloud_mts_max:
+                    return False
+                elif local_mts_max == cloud_mts_max:
+                    if len(local_paths_all) == len(cloud_stats):
+                        return False
+                    elif just_check:
+                        return True
         else:
-            # this is true for http for example
-            # where size is present but st_mtime is not
-            # we assume that any change without the change in size is unlikely
-            cloud_size = cloud_stat.st_size
-            local_size_obj = objectpath.stat().st_size
-            need_synchronize = cloud_size != local_size_obj
+            local_paths_all = {destination: destination.stat()}
+
+        cloud_files_sync = []
+        local_files_sync = []
+        for i, (cloud_file, cloud_stat) in enumerate(cloud_stats.items()):
+            local_path = local_paths[i]
+            if local_path not in local_paths_all or is_sync_needed(
+                cloud_stat, local_paths_all[local_path]
+            ):
+                cloud_files_sync.append(cloud_file)
+                local_files_sync.append(local_path.as_posix())
     else:
-        if not just_check:
-            objectpath.parent.mkdir(parents=True, exist_ok=True)
-        need_synchronize = True
-    # just check if synchronization is needed
-    if just_check:
-        return need_synchronize
-    if need_synchronize:
-        # just to be sure that overwriting an existing file doesn't corrupt it
-        # we saw some frequent corruption on some systems for unclear reasons
-        if objectpath_exists:
-            objectpath.unlink()
-        # hf has sync filesystem
-        # on sync filesystems ChildProgressCallback.branched()
-        # returns the default callback
-        # this is why a difference between s3 and hf in progress bars
-        self.download_to(
-            objectpath, recursive=False, print_progress=False, callback=callback
+        cloud_files_sync = list(cloud_stats.keys())
+        local_files_sync = [local_path.as_posix() for local_path in local_paths]
+
+    if cloud_files_sync:
+        if just_check:
+            return True
+
+        callback = ProgressCallback.requires_progress(
+            maybe_callback=kwargs.pop("callback", None),
+            print_progress=print_progress,
+            objectname=destination.name,
+            action="synchronizing",
+            adjust_size=False,
         )
-        if cloud_mts != 0:
-            os.utime(objectpath, times=(cloud_mts, cloud_mts))
+        origin.fs.download(
+            cloud_files_sync,
+            local_files_sync,
+            recursive=False,
+            callback=callback,
+            **kwargs,
+        )
+        if not use_size:
+            for i, cloud_file in enumerate(cloud_files_sync):
+                cloud_mtime = cloud_stats[cloud_file]
+                os.utime(local_files_sync[i], times=(cloud_mtime, cloud_mtime))
     else:
-        # nothing happens if parent_update is not defined
-        # because of Callback.no_op
-        callback.parent_update()
-    return need_synchronize
+        return False
+
+    if is_dir and local_paths_all:
+        for path in (path for path in local_paths_all if path not in local_paths):
+            path.unlink()
+            parent = path.parent
+            if next(parent.iterdir(), None) is None:
+                parent.rmdir()
+
+    return True
 
 
 def modified(self) -> datetime | None:
@@ -710,14 +699,7 @@ def to_url(upath):
         raise ValueError("The provided UPath must be an S3 path.")
     key = "/".join(upath.parts[1:])
     bucket = upath.drive
-    if bucket == "scverse-spatial-eu-central-1":
-        region = "eu-central-1"
-    elif f"s3://{bucket}" not in HOSTED_BUCKETS:
-        response = upath.fs.call_s3("head_bucket", Bucket=bucket)
-        headers = response["ResponseMetadata"]["HTTPHeaders"]
-        region = headers.get("x-amz-bucket-region")
-    else:
-        region = bucket.replace("lamin_", "")
+    region = get_storage_region(upath)
     if region == "us-east-1":
         return f"https://{bucket}.s3.amazonaws.com/{key}"
     else:
@@ -740,7 +722,8 @@ def to_url(upath):
 
 # add custom functions
 UPath.modified = property(modified)
-UPath.synchronize = synchronize
+UPath.synchronize = deprecated("synchronize_to")(synchronize_to)
+UPath.synchronize_to = synchronize_to
 UPath.upload_from = upload_from
 UPath.to_url = to_url
 UPath.download_to = download_to
@@ -823,6 +806,67 @@ class S3QueryPath(S3Path):
 register_implementation("s3", S3QueryPath, clobber=True)
 
 
+def get_storage_region(path: UPathStr) -> str | None:
+    upath = UPath(path)
+
+    if upath.protocol != "s3":
+        return None
+
+    bucket = upath.drive
+
+    if bucket == "scverse-spatial-eu-central-1":
+        return "eu-central-1"
+    elif f"s3://{bucket}" in HOSTED_BUCKETS:
+        return bucket.replace("lamin-", "")
+
+    from botocore.exceptions import ClientError
+
+    if isinstance(path, str):
+        import botocore.session
+        from botocore.config import Config
+
+        path_part = path.replace("s3://", "")
+        # check for endpoint_url in the path string
+        if "?" in path_part:
+            path_part, query = _split_path_query(path_part)
+            endpoint_url = query.get("endpoint_url", [None])[0]
+        else:
+            endpoint_url = None
+        session = botocore.session.get_session()
+        credentials = session.get_credentials()
+        if credentials is None or credentials.access_key is None:
+            config = Config(signature_version=botocore.session.UNSIGNED)
+        else:
+            config = None
+        s3_client = session.create_client(
+            "s3", endpoint_url=endpoint_url, config=config
+        )
+        try:
+            response = s3_client.head_bucket(Bucket=bucket)
+        except ClientError as exc:
+            response = getattr(exc, "response", {})
+            if response.get("Error", {}).get("Code") == "404":
+                raise exc
+    else:
+        upath = get_aws_options_manager()._path_inject_options(upath, {})
+        try:
+            response = upath.fs.call_s3("head_bucket", Bucket=bucket)
+        except Exception as exc:
+            cause = getattr(exc, "__cause__", None)
+            if not isinstance(cause, ClientError):
+                raise exc
+            response = getattr(cause, "response", {})
+            if response.get("Error", {}).get("Code") == "404":
+                raise exc
+
+    region = (
+        response.get("ResponseMetadata", {})
+        .get("HTTPHeaders", {})
+        .get("x-amz-bucket-region", None)
+    )
+    return region
+
+
 def create_path(path: UPathStr, access_token: str | None = None) -> UPath:
     upath = UPath(path).expanduser()
 

{lamindb_setup-1.8.0 → lamindb_setup-1.8.2}/tests/storage/test_storage_stats.py

@@ -81,5 +81,12 @@ def test_get_stat_dir_cloud_hf():
 def test_get_storage_region():
     for region in HOSTED_REGIONS:
         assert get_storage_region(f"s3://lamin-{region}") == region
+    assert get_storage_region("s3://scverse-spatial-eu-central-1") == "eu-central-1"
+
     assert get_storage_region(UPath("s3://lamindata", endpoint_url=None)) == "us-east-1"
     assert get_storage_region("s3://lamindata/?endpoint_url=") == "us-east-1"
+    # private bucket
+    assert (
+        get_storage_region(UPath("s3://lamindb-setup-private-bucket/some-folder"))
+        == "us-east-1"
+    )

{lamindb_setup-1.8.0 → lamindb_setup-1.8.2}/tests/storage/test_to_url.py

@@ -11,11 +11,12 @@ def test_to_url():
         == "https://lamindata.s3.amazonaws.com/test-folder"
     )
     # private bucket
-    # next PR
-    # assert (
-    #     ln_setup.core.upath.create_path("s3://lamindb-setup-private-bucket/test-folder")
-    #     == "https://lamindb-setup-private-bucket.s3.amazonaws.com/test-folder"
-    # )
+    assert (
+        ln_setup.core.upath.create_path(
+            "s3://lamindb-setup-private-bucket/test-folder"
+        ).to_url()
+        == "https://lamindb-setup-private-bucket.s3.amazonaws.com/test-folder"
+    )
     # eu-central-1 / AWS Dev
     assert (
         ln_setup.core.upath.create_path("s3://lamindata-eu/test-folder").to_url()
@@ -27,5 +28,5 @@ def test_to_url():
         ln_setup.core.upath.create_path(
             "s3://lamin-eu-central-1/9fm7UN13/test-folder"
         ).to_url()
-        == "https://lamin-eu-central-1.s3-lamin-eu-central-1.amazonaws.com/9fm7UN13/test-folder"
+        == "https://lamin-eu-central-1.s3-eu-central-1.amazonaws.com/9fm7UN13/test-folder"
     )