lamindb_setup 1.3.2__py3-none-any.whl → 1.4.0__py3-none-any.whl

lamindb_setup/__init__.py

@@ -33,7 +33,7 @@ Modules & settings:
 
 """
 
-__version__ = "1.3.2"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.4.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 

lamindb_setup/_close.py

@@ -4,7 +4,6 @@ from lamin_utils import logger
 
 from .core._settings import settings
 from .core._settings_store import current_instance_settings_file
-from .core._setup_bionty_sources import delete_bionty_sources_yaml
 from .core.cloud_sqlite_locker import clear_locker
 
 
@@ -24,8 +23,6 @@ def close(mute: bool = False) -> None:
                 logger.warning("did not upload cache file - not enough permissions")
             else:
                 raise e
-        if "bionty" in settings.instance.modules:
-            delete_bionty_sources_yaml()
         current_instance_settings_file().unlink()
         clear_locker()
         if not mute:

lamindb_setup/_connect_instance.py

@@ -169,6 +169,7 @@ def _connect_instance(
                 if (schema_id := instance_result["schema_id"]) is None
                 else UUID(schema_id),
                 fine_grained_access=instance_result.get("fine_grained_access", False),
+                db_permissions=instance_result.get("db_permissions", None),
             )
         else:
             if hub_result != "anonymous-user":

lamindb_setup/_delete.py

@@ -131,7 +131,7 @@ def delete(slug: str, force: bool = False, require_empty: bool = True) -> int |
         # this will error if the user doesn't have permission
         delete_instance_on_hub(isettings._id, require_empty=False)
     delete_by_isettings(isettings)
-    # if .lndb file was delete, then we might count -1
+    # if lamin.db file was delete, then we might count -1
     if n_files <= 0 and isettings.storage.type == "local":
         # dir is only empty after sqlite file was delete via delete_by_isettings
         if (isettings.storage.root / ".lamindb").exists():

lamindb_setup/_init_instance.py

@@ -14,6 +14,7 @@ from lamin_utils import logger
 from ._close import close as close_instance
 from ._silence_loggers import silence_loggers
 from .core import InstanceSettings
+from .core._docs import doc_args
 from .core._settings import settings
 from .core._settings_instance import is_local_db_url
 from .core._settings_storage import StorageSettings, init_storage
@@ -95,11 +96,16 @@ def register_user(usettings):
         pass
 
 
-def register_user_and_storage_in_instance(isettings: InstanceSettings, usettings):
-    """Register user & storage in DB."""
+def register_initial_records(isettings: InstanceSettings, usettings):
+    """Register space, user & storage in DB."""
     from django.db.utils import OperationalError
+    from lamindb.models import Space
 
     try:
+        Space.objects.get_or_create(
+            name="All",
+            description="Every team & user with access to the instance has access.",
+        )
         register_user(usettings)
         register_storage_in_instance(isettings.storage)
     except OperationalError as error:
@@ -208,22 +214,32 @@ ln.Artifact.using("laminlabs/cellxgene").filter()
 Or do you want to switch off auto-connect via `lamin settings set auto-connect false`?
 """
 
+DOC_STORAGE_ARG = "A local or remote folder (`'s3://...'` or `'gs://...'`). Defaults to current working directory."
+DOC_INSTANCE_NAME = (
+    "Instance name. If not passed, it will equal the folder name passed to `storage`."
+)
+DOC_DB = "Database connection URL. Defaults to `None`, which implies an SQLite file in the storage location."
+DOC_MODULES = "Comma-separated string of schema modules."
+DOC_LOW_LEVEL_KWARGS = "Keyword arguments for low-level control."
+
 
+@doc_args(DOC_STORAGE_ARG, DOC_INSTANCE_NAME, DOC_DB, DOC_MODULES, DOC_LOW_LEVEL_KWARGS)
 def init(
     *,
-    storage: UPathStr,
+    storage: UPathStr = ".",
     name: str | None = None,
     db: PostgresDsn | None = None,
     modules: str | None = None,
     **kwargs,
 ) -> None:
-    """Create and load a LaminDB instance.
+    """Init a LaminDB instance.
 
     Args:
-        storage: Either local or remote folder (`"s3://..."` or `"gs://..."`).
-        name: Instance name.
-        db: Database connection url, do not pass for SQLite.
-        modules: Comma-separated string of modules. None if the lamindb registries are enough.
+        storage: {}
+        name: {}
+        db: {}
+        modules: {}
+        **kwargs: {}
     """
     isettings = None
     ssettings = None
@@ -359,8 +375,8 @@ def load_from_isettings(
     user = settings.user if user is None else user
 
     if init:
-        # during init both user and storage need to be registered
-        register_user_and_storage_in_instance(isettings, user)
+        # during init space, user and storage need to be registered
+        register_initial_records(isettings, user)
         write_bionty_sources(isettings)
         isettings._update_cloud_sqlite_file(unlock_cloud_sqlite=False)
     else:

lamindb_setup/_set_managed_storage.py

@@ -31,7 +31,7 @@ def set_managed_storage(root: UPathStr, **fs_kwargs):
         )
     # here the storage is registered in the hub
     # hub_record_status="hub-record-created" if a new record is created
-    # "hub-record-retireved" if the storage is in the hub already
+    # "hub-record-retrieved" if the storage is in the hub already
     ssettings, hub_record_status = init_storage(
         root=root, instance_id=settings.instance._id, register_hub=True
     )

lamindb_setup/core/_hub_client.py

@@ -69,7 +69,9 @@ def connect_hub(
         # function_client_timeout=5 by default
         # increase to avoid rare timeouts for edge functions
         client_options = ClientOptions(
-            auto_refresh_token=False, function_client_timeout=10
+            auto_refresh_token=False,
+            function_client_timeout=20,
+            postgrest_client_timeout=20,
         )
     return create_client(env.supabase_api_url, env.supabase_anon_key, client_options)
 

lamindb_setup/core/_hub_core.py

@@ -7,6 +7,7 @@ from importlib import metadata
 from typing import TYPE_CHECKING, Literal
 from uuid import UUID
 
+import jwt
 from lamin_utils import logger
 from postgrest.exceptions import APIError
 
@@ -121,7 +122,7 @@ def init_storage_hub(
     auto_populate_instance: bool = True,
     created_by: UUID | None = None,
     access_token: str | None = None,
-) -> Literal["hub-record-retireved", "hub-record-created"]:
+) -> Literal["hub-record-retrieved", "hub-record-created"]:
     if settings.user.handle != "anonymous" or access_token is not None:
         return call_with_fallback_auth(
             _init_storage_hub,
@@ -135,7 +136,7 @@ def init_storage_hub(
             _select_storage, ssettings=ssettings, update_uid=True
         )
         if storage_exists:
-            return "hub-record-retireved"
+            return "hub-record-retrieved"
         else:
             raise ValueError("Log in to create a storage location on the hub.")
 
@@ -145,7 +146,7 @@ def _init_storage_hub(
     ssettings: StorageSettings,
     auto_populate_instance: bool,
     created_by: UUID | None = None,
-) -> Literal["hub-record-retireved", "hub-record-created"]:
+) -> Literal["hub-record-retrieved", "hub-record-created"]:
     from lamindb_setup import settings
 
     created_by = settings.user._uuid if created_by is None else created_by
@@ -153,7 +154,7 @@ def _init_storage_hub(
     # database
     root = ssettings.root_as_str
     if _select_storage(ssettings, update_uid=True, client=client):
-        return "hub-record-retireved"
+        return "hub-record-retrieved"
     if ssettings.type_is_cloud:
         id = uuid.uuid5(uuid.NAMESPACE_URL, root)
     else:
@@ -548,7 +549,7 @@ def _sign_in_hub_api_key(api_key: str, client: Client):
     access_token = json.loads(response)["accessToken"]
     # probably need more info here to avoid additional queries
     # like handle, uid etc
-    account_id = client.auth._decode_jwt(access_token)["sub"]
+    account_id = jwt.decode(access_token, options={"verify_signature": False})["sub"]
     client.postgrest.auth(access_token)
     # normally public.account.id is equal to auth.user.id
     data = client.table("account").select("*").eq("id", account_id).execute().data

lamindb_setup/core/_settings_instance.py

@@ -13,7 +13,12 @@ from ._hub_client import call_with_fallback
 from ._hub_crud import select_account_handle_name_by_lnid
 from ._hub_utils import LaminDsn, LaminDsnModel
 from ._settings_save import save_instance_settings
-from ._settings_storage import StorageSettings, init_storage, mark_storage_root
+from ._settings_storage import (
+    LEGACY_STORAGE_UID_FILE_KEY,
+    STORAGE_UID_FILE_KEY,
+    StorageSettings,
+    init_storage,
+)
 from ._settings_store import current_instance_settings_file, instance_settings_file
 from .cloud_sqlite_locker import (
     EXPIRATION_TIME,
@@ -60,6 +65,7 @@ class InstanceSettings:
         api_url: str | None = None,
         schema_id: UUID | None = None,
         fine_grained_access: bool = False,
+        db_permissions: str | None = None,
         _locker_user: UserSettings | None = None,  # user to lock for if cloud sqlite
     ):
         from ._hub_utils import validate_db_arg
@@ -83,6 +89,8 @@ class InstanceSettings:
         # private, whether fine grained access is used
         # needed to be set to request jwt etc
         self._fine_grained_access = fine_grained_access
+        # permissions for db such as jwt, read, write etc.
+        self._db_permissions = db_permissions
         # if None then settings.user is used
         self._locker_user = _locker_user
 
@@ -144,32 +152,27 @@ class InstanceSettings:
         for record in all_local_records:
             root_path = Path(record.root)
             if root_path.exists():
-                marker_path = root_path / ".lamindb/_is_initialized"
-                if marker_path.exists():
-                    try:
-                        uid = marker_path.read_text()
-                    except PermissionError:
+                marker_path = root_path / STORAGE_UID_FILE_KEY
+                if not marker_path.exists():
+                    legacy_filepath = root_path / LEGACY_STORAGE_UID_FILE_KEY
+                    if legacy_filepath.exists():
                         logger.warning(
-                            f"ignoring the following location because no permission to read it: {marker_path}"
+                            f"found legacy marker file, renaming it from {legacy_filepath} to {marker_path}"
                         )
-                        continue
-                    if uid == record.uid:
-                        found = True
-                        break
-                    elif uid == "":
-                        try:
-                            # legacy instance that was not yet marked properly
-                            mark_storage_root(record.root, record.uid)
-                        except PermissionError:
-                            logger.warning(
-                                f"ignoring the following location because no permission to write to it: {marker_path}"
-                            )
-                            continue
+                        legacy_filepath.rename(marker_path)
                     else:
-                        continue
-                else:
-                    # legacy instance that was not yet marked at all
-                    mark_storage_root(record.root, record.uid)
+                        raise ValueError(
+                            f"local storage location '{root_path}' is corrupted, cannot find marker file with storage uid"
+                        )
+                try:
+                    uid = marker_path.read_text()
+                except PermissionError:
+                    logger.warning(
+                        f"ignoring the following location because no permission to read it: {marker_path}"
+                    )
+                    continue
+                if uid == record.uid:
+                    found = True
                     break
         if found:
             return StorageSettings(record.root)
@@ -295,7 +298,8 @@ class InstanceSettings:
     @property
     def _sqlite_file(self) -> UPath:
         """SQLite file."""
-        return self.storage.key_to_filepath(f"{self._id.hex}.lndb")
+        filepath = self.storage.root / ".lamindb/lamin.db"
+        return filepath
 
     @property
     def _sqlite_file_local(self) -> Path:
@@ -477,24 +481,18 @@ class InstanceSettings:
 
         disable_auto_connect(setup_django)(self, init=True)
 
-        from lamindb.models import Space
-
-        Space.objects.get_or_create(
-            name="All",
-            description="Every team & user with access to the instance has access.",
-        )
-
     def _load_db(self) -> tuple[bool, str]:
         # Is the database available and initialized as LaminDB?
         # returns a tuple of status code and message
         if self.dialect == "sqlite" and not self._sqlite_file.exists():
-            legacy_file = self.storage.key_to_filepath(f"{self.name}.lndb")
+            legacy_file = self.storage.key_to_filepath(f"{self._id.hex}.lndb")
             if legacy_file.exists():
-                raise RuntimeError(
-                    "The SQLite file has been renamed!\nPlease rename your SQLite file"
-                    f" {legacy_file} to {self._sqlite_file}"
+                logger.warning(
+                    f"The SQLite file is being renamed from {legacy_file} to {self._sqlite_file}"
                 )
-            return False, f"SQLite file {self._sqlite_file} does not exist"
+                legacy_file.rename(self._sqlite_file)
+            else:
+                return False, f"SQLite file {self._sqlite_file} does not exist"
         # we need the local sqlite to setup django
         self._update_local_sqlite_file()
         # setting up django also performs a check for migrations & prints them

lamindb_setup/core/_settings_load.py

@@ -104,6 +104,7 @@ def setup_instance_from_store(store: InstanceSettingsStore) -> InstanceSettings:
         api_url=_null_to_value(store.api_url),
         schema_id=None if store.schema_id == "null" else UUID(store.schema_id),
         fine_grained_access=store.fine_grained_access,
+        db_permissions=_null_to_value(store.db_permissions),
     )
 
 

lamindb_setup/core/_settings_save.py

@@ -61,6 +61,7 @@ def save_settings(
                         "api_url",
                         "schema_id",
                         "fine_grained_access",
+                        "db_permissions",
                     }:
                         settings_key = f"_{store_key.rstrip('_')}"
                     else:

lamindb_setup/core/_settings_storage.py

@@ -23,7 +23,8 @@ if TYPE_CHECKING:
 
     from .types import UPathStr
 
-IS_INITIALIZED_KEY = ".lamindb/_is_initialized"
+STORAGE_UID_FILE_KEY = ".lamindb/storage_uid.txt"
+LEGACY_STORAGE_UID_FILE_KEY = ".lamindb/_is_initialized"
 
 # a list of supported fsspec protocols
 # rename file to local before showing to a user
@@ -82,11 +83,15 @@ def get_storage_region(path: UPathStr) -> str | None:
 
 
 def mark_storage_root(root: UPathStr, uid: str):
-    # we need to touch a 0-byte object in folder-like storage location on S3 to avoid
+    # we need a file in folder-like storage locations on S3 to avoid
     # permission errors from leveraging s3fs on an empty hosted storage location
-    # for consistency, we write this file everywhere
+    # (path.fs.find raises a PermissionError)
+    # we also need it in case a storage location is ambiguous because a server / local environment
+    # doesn't have a globally unique identifier, then we screen for this file to map the
+    # path on a storage location in the registry
+
     root_upath = UPath(root)
-    mark_upath = root_upath / IS_INITIALIZED_KEY
+    mark_upath = root_upath / STORAGE_UID_FILE_KEY
     mark_upath.write_text(uid)
 
 
@@ -100,7 +105,7 @@ def init_storage(
     access_token: str | None = None,
 ) -> tuple[
     StorageSettings,
-    Literal["hub-record-not-created", "hub-record-retireved", "hub-record-created"],
+    Literal["hub-record-not-created", "hub-record-retrieved", "hub-record-created"],
 ]:
     assert root is not None, "`root` argument can't be `None`"
 
@@ -145,38 +150,46 @@ def init_storage(
     )
     # this stores the result of init_storage_hub
     hub_record_status: Literal[
-        "hub-record-not-created", "hub-record-retireved", "hub-record-created"
+        "hub-record-not-created", "hub-record-retrieved", "hub-record-created"
     ] = "hub-record-not-created"
     # the below might update the uid with one that's already taken on the hub
-    if not prevent_register_hub:
-        if ssettings.type_is_cloud or register_hub:
-            from ._hub_core import delete_storage_record, init_storage_hub
-
-            hub_record_status = init_storage_hub(
-                ssettings,
-                auto_populate_instance=not init_instance,
-                created_by=created_by,
-                access_token=access_token,
-            )
-    # below comes last only if everything else was successful
-    try:
-        # (federated) credentials for AWS access are provisioned under-the-hood
-        # discussion: https://laminlabs.slack.com/archives/C04FPE8V01W/p1719260587167489
-        # if access_token was passed in ssettings, it is used here
-        mark_storage_root(ssettings.root, ssettings.uid)  # type: ignore
-    except Exception:
-        logger.important(
-            f"due to lack of write access, LaminDB won't manage storage location: {ssettings.root_as_str}"
+    if not prevent_register_hub and (ssettings.type_is_cloud or register_hub):
+        from ._hub_core import delete_storage_record, init_storage_hub
+
+        # this retrieves the storage record if it exists already in the hub
+        # and updates uid and instance_id in ssettings
+        hub_record_status = init_storage_hub(
+            ssettings,
+            auto_populate_instance=not init_instance,
+            created_by=created_by,
+            access_token=access_token,
         )
-        # we have to check hub_record_status here because
-        # _select_storage inside init_storage_hub also populates ssettings._uuid
-        # and we don't want to delete an existing storage record here
-        # only newly created
-        if hub_record_status == "hub-record-created" and ssettings._uuid is not None:
-            delete_storage_record(ssettings._uuid, access_token=access_token)  # type: ignore
-            ssettings._uuid_ = None
-            hub_record_status = "hub-record-not-created"
-        ssettings._instance_id = None
+    # below comes last only if everything else was successful
+    # we check the write access here only if the storage record has been just created
+    # or if the storage is local
+    # also we have to check hub_record_status here because
+    # _select_storage inside init_storage_hub also populates ssettings._uuid
+    # and we don't want to delete an existing storage record here if no write access
+    # only newly created
+    # local storages not registered in the hub should be also marked
+    is_local_not_retrieved = not (
+        ssettings.type_is_cloud or hub_record_status == "hub-record-retrieved"
+    )
+    if hub_record_status == "hub-record-created" or is_local_not_retrieved:
+        try:
+            # (federated) credentials for AWS access are provisioned under-the-hood
+            # discussion: https://laminlabs.slack.com/archives/C04FPE8V01W/p1719260587167489
+            # if access_token was passed in ssettings, it is used here
+            mark_storage_root(ssettings.root, ssettings.uid)  # type: ignore
+        except Exception:
+            logger.important(
+                f"due to lack of write access, LaminDB won't manage storage location: {ssettings.root_as_str}"
+            )
+            if ssettings._uuid is not None:
+                delete_storage_record(ssettings._uuid, access_token=access_token)  # type: ignore
+                ssettings._uuid_ = None
+                hub_record_status = "hub-record-not-created"
+            ssettings._instance_id = None
     return ssettings, hub_record_status
 
 
@@ -234,7 +247,14 @@ class StorageSettings:
 
     @property
     def _mark_storage_root(self) -> UPath:
-        return self.root / IS_INITIALIZED_KEY
+        marker_path = self.root / STORAGE_UID_FILE_KEY
+        legacy_filepath = self.root / LEGACY_STORAGE_UID_FILE_KEY
+        if legacy_filepath.exists():
+            logger.warning(
+                f"found legacy marker file, renaming it from {legacy_filepath} to {marker_path}"
+            )
+            legacy_filepath.rename(marker_path)
+        return marker_path
 
     @property
     def record(self) -> Any:

lamindb_setup/core/_settings_store.py

@@ -58,6 +58,7 @@ class InstanceSettingsStore(BaseSettings):
     schema_str: Optional[str]
     schema_id: Optional[str] = None
     fine_grained_access: bool = False
+    db_permissions: Optional[str] = None
     id: str
     git_repo: Optional[str]
     keep_artifacts_local: Optional[bool]

lamindb_setup/core/_setup_bionty_sources.py

@@ -1,116 +1,44 @@
 from __future__ import annotations
 
-import os
 from typing import TYPE_CHECKING
 
-from django.db.utils import OperationalError, ProgrammingError
-from lamin_utils import logger
-
-from ._settings import settings as setup_settings
-
 if TYPE_CHECKING:
     from ._settings_instance import InstanceSettings
 
-# bionty.Source -> bionty.base
-RENAME = {"name": "source", "description": "source_name"}
-
 
 def write_bionty_sources(isettings: InstanceSettings) -> None:
-    """Write bionty sources to Source table."""
+    """Write public bionty sources to bt.Source table."""
     if "bionty" not in isettings.modules:
         return None
-    import shutil
 
     import bionty
     import bionty.base as bionty_base
-    from bionty._bionty import list_biorecord_models
+    from bionty._biorecord import list_biorecord_models
     from bionty.base.dev._handle_sources import parse_sources_yaml
     from bionty.models import Source
 
     bionty_models = list_biorecord_models(bionty)
 
-    shutil.copy(
-        bionty_base.settings.current_sources, bionty_base.settings.lamindb_sources
-    )
-
-    all_sources = parse_sources_yaml(bionty_base.settings.local_sources)
+    all_sources = parse_sources_yaml(bionty_base.settings.public_sources)
     all_sources_dict = all_sources.to_dict(orient="records")
 
-    def _get_currently_used(key: str):
-        return (
-            bionty_base.display_currently_used_sources()
-            .reset_index()
-            .set_index(["entity", key])
-        )
-
-    currently_used = _get_currently_used("organism")
+    currently_used = (
+        bionty_base.display_currently_used_sources(mute=True)
+        .reset_index()
+        .set_index(["entity", "organism"])
+    )
 
     all_records = []
     for kwargs in all_sources_dict:
         act = currently_used.loc[(kwargs["entity"], kwargs["organism"])].to_dict()
-        if (act["source"] == kwargs["source"]) and (
-            act["version"] == kwargs["version"]
-        ):
+        if (act["name"] == kwargs["name"]) and (act["version"] == kwargs["version"]):
             kwargs["currently_used"] = True
 
-        # when the database is not yet migrated but setup is updated
-        # won't need this once lamindb is released with the new pin
         kwargs["run"] = None  # can't yet access tracking information
         kwargs["in_db"] = False
-        for db_field, base_col in RENAME.items():
-            kwargs[db_field] = kwargs.pop(base_col)
         if kwargs["entity"] in bionty_models:
             kwargs["entity"] = f"bionty.{kwargs['entity']}"
         record = Source(**kwargs)
         all_records.append(record)
 
     Source.objects.bulk_create(all_records, ignore_conflicts=True)
-
-
-def load_bionty_sources(isettings: InstanceSettings | None = None):
-    """Write currently_used bionty sources to LAMINDB_VERSIONS_PATH in bionty."""
-    if isettings is None:
-        if setup_settings._instance_settings is not None:
-            isettings = setup_settings.instance
-        else:
-            logger.warning(
-                f"Ignoring bionty setup because running in LAMINDB_MULTI_INSTANCE mode = {os.environ.get('LAMINDB_MULTI_INSTANCE')}"
-            )
-            # not setting up bionty sources
-            return None
-    if isettings is not None:
-        if "bionty" not in isettings.modules:
-            # no need to setup anything
-            return None
-
-    import bionty.base as bionty_base
-    from bionty.base.dev._handle_sources import parse_currently_used_sources
-    from bionty.base.dev._io import write_yaml
-    from bionty.models import Source
-
-    try:
-        # need try except because of integer primary key migration
-        active_records = (
-            Source.objects.filter(currently_used=True).order_by("id").all().values()
-        )
-        for kwargs in active_records:
-            for db_field, base_col in RENAME.items():
-                kwargs[base_col] = kwargs.pop(db_field)
-            # TODO: non-bionty modules?
-            kwargs["entity"] = kwargs["entity"].replace("bionty.", "")
-        write_yaml(
-            parse_currently_used_sources(active_records),
-            bionty_base.settings.lamindb_sources,
-        )
-    except (OperationalError, ProgrammingError):
-        pass
-
-
-def delete_bionty_sources_yaml():
-    """Delete LAMINDB_SOURCES_PATH in bionty."""
-    try:
-        import bionty.base as bionty_base
-
-        bionty_base.settings.lamindb_sources.unlink(missing_ok=True)
-    except ModuleNotFoundError:
-        pass

lamindb_setup/core/django.py

@@ -125,7 +125,7 @@ def setup_django(
 
         BaseDatabaseWrapper.close_if_health_check_failed = close_if_health_check_failed
 
-        if isettings._fine_grained_access:
+        if isettings._fine_grained_access and isettings._db_permissions == "jwt":
             from ._hub_core import access_db
 
             set_db_token(access_db(isettings))

lamindb_setup/core/upath.py

@@ -316,16 +316,26 @@ def upload_from(
         callback = ProgressCallback(local_path.name, "uploading")
         kwargs["callback"] = callback
 
-    source: str | list[str]
-    destination: str | list[str]
-    if local_path_is_dir and not create_folder:
-        source = [f.as_posix() for f in local_path.rglob("*") if f.is_file()]
-        destination = fsspec.utils.other_paths(
-            source, self.as_posix(), exists=False, flatten=False
-        )
-    else:
-        source = local_path.as_posix()
-        destination = self.as_posix()
+    source: str | list[str] = local_path.as_posix()
+    destination: str | list[str] = self.as_posix()
+    if local_path_is_dir:
+        size: int = 0
+        files: list[str] = []
+        for file in (path for path in local_path.rglob("*") if path.is_file()):
+            size += file.stat().st_size
+            files.append(file.as_posix())
+        # see https://github.com/fsspec/s3fs/issues/897
+        # here we reduce batch_size for folders bigger than 8 GiB
+        # to avoid the problem in the issue
+        # the default batch size for this case is 128
+        if "batch_size" not in kwargs and size >= 8 * 2**30:
+            kwargs["batch_size"] = 64
+
+        if not create_folder:
+            source = files
+            destination = fsspec.utils.other_paths(
+                files, self.as_posix(), exists=False, flatten=False
+            )
 
     # the below lines are to avoid s3fs triggering create_bucket in upload if
     # dirs are present, it allows to avoid the permission error
@@ -899,17 +909,12 @@ def check_storage_is_empty(
 ) -> int:
     root_upath = UPath(root)
     root_string = root_upath.as_posix()  # type: ignore
-    # we currently touch a 0-byte file in the root of a hosted storage location
-    # ({storage_root}/.lamindb/_is_initialized) during storage initialization
-    # since path.fs.find raises a PermissionError on empty hosted
-    # subdirectories (see lamindb_setup/core/_settings_storage/init_storage).
-    n_offset_objects = 1  # because of touched dummy file, see mark_storage_root()
+    n_offset_objects = 1  # because of storage_uid.txt file, see mark_storage_root()
+    if account_for_sqlite_file:
+        n_offset_objects += 1  # the SQLite file is in the ".lamindb" directory
     if root_string.startswith(HOSTED_BUCKETS):
         # in hosted buckets, count across entire root
         directory_string = root_string
-        # the SQLite file is not in the ".lamindb" directory
-        if account_for_sqlite_file:
-            n_offset_objects += 1  # because of SQLite file
     else:
         # in any other storage location, only count in .lamindb
         if not root_string.endswith("/"):

{lamindb_setup-1.3.2.dist-info → lamindb_setup-1.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb_setup
-Version: 1.3.2
+Version: 1.4.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10
@@ -15,13 +15,13 @@ Requires-Dist: universal_pathlib==0.2.6
 Requires-Dist: botocore<2.0.0
 Requires-Dist: supabase>=2.8.1,<=2.11.0
 Requires-Dist: storage3!=0.11.2; python_version < '3.11'
+Requires-Dist: pyjwt<3.0.0
 Requires-Dist: psutil
 Requires-Dist: packaging
 Requires-Dist: urllib3<2 ; extra == "aws"
 Requires-Dist: aiobotocore[boto3]>=2.5.4,<3.0.0 ; extra == "aws"
 Requires-Dist: s3fs>=2023.12.2,<=2025.2.0,!=2024.10.0 ; extra == "aws"
 Requires-Dist: line_profiler ; extra == "dev"
-Requires-Dist: pyjwt<3.0.0 ; extra == "dev"
 Requires-Dist: psycopg2-binary ; extra == "dev"
 Requires-Dist: python-dotenv ; extra == "dev"
 Requires-Dist: nox ; extra == "dev"

{lamindb_setup-1.3.2.dist-info → lamindb_setup-1.4.0.dist-info}/RECORD

@@ -1,20 +1,20 @@
-lamindb_setup/__init__.py,sha256=LkDrKl5YfGD2qI3A9fPfHkesr8R0ASCPzKqvNZFtRiE,2692
+lamindb_setup/__init__.py,sha256=2h5njKI5Z4qiixV_epzJ1DfWghCpY8ao29gDHqa3guw,2692
 lamindb_setup/_cache.py,sha256=aszT-zk3S5dTLKp5g1W-S_FPh2E5YVCALwWSGPJLWBM,1493
 lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
 lamindb_setup/_check_setup.py,sha256=d1GS1Csy2G9AysfjoyVcNVY0lhHiWSwSLpfgdIQf35s,5477
-lamindb_setup/_close.py,sha256=pf8PHrtRBC6TycBtVAXzD9EZSGufoyp5M82o6zLHmt4,1240
-lamindb_setup/_connect_instance.py,sha256=IXpU43DO-DWm9twDeuQJeBbKj-oB4SzEqsCjIX4AZMo,18276
-lamindb_setup/_delete.py,sha256=4NqJkEA812VWRxTBW1o1h1YX0fF7sQwn08hj1BmvCPs,5673
+lamindb_setup/_close.py,sha256=SFfOO0OjYR3krvya66kK8RRQa7lPH8MnzUtE33yZ6sQ,1082
+lamindb_setup/_connect_instance.py,sha256=lo0EV_PA8YNlerxfPLG0u-YdmyKLAqbIIhTfzmwNDlI,18352
+lamindb_setup/_delete.py,sha256=2KnZOqd5Kgr45XzjiDE9der35LODDUajZD6_hcurGtQ,5676
 lamindb_setup/_django.py,sha256=uIQflpkp8l3axyPaKURlk3kacgpElVP5KOKmFxYSMGk,1454
 lamindb_setup/_entry_points.py,sha256=sKwXPX9xjOotoAjvgkU5LBwjjHLWVkh0ZGdiSsrch9k,522
 lamindb_setup/_exportdb.py,sha256=QLjoH4dEwqa01A12naKaDPglCCzl2_VLKWFfJRE_uSg,2113
 lamindb_setup/_importdb.py,sha256=fKv9ev5OOj_-bmzC8XZ1GxOcjIjI486yrHSHDWQrJeI,1874
-lamindb_setup/_init_instance.py,sha256=wZ1yEnbogduQOXkEbcip9Jm8mys1NtJy2W5RRWJy1lQ,14253
+lamindb_setup/_init_instance.py,sha256=LRtuXf2jiz5zTs47F7TyhCL0wfHA9YqwWHCBMadtnQs,14837
 lamindb_setup/_migrate.py,sha256=ya-15sc91i4JmEWI4j00T2892x8hdy2fSW-qz4IdxLs,9739
 lamindb_setup/_register_instance.py,sha256=X7ZGlCVOZKq4zTpi3bxML4jzo6hgN9UYmdTxxf6JLmc,1205
 lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
 lamindb_setup/_schema_metadata.py,sha256=cDkNHyFTFZq879UGY6i-rJEXfeBN59AN3Wz86caKnYI,14358
-lamindb_setup/_set_managed_storage.py,sha256=4tDxXQMt8Gw028uY3vIQxZQ7qBNXhQMc8saarNK_Z-s,2043
+lamindb_setup/_set_managed_storage.py,sha256=qC3ACD_PWG-MrzcS3fQpjDEOuxAaJBgPqCU_bDvqtXo,2043
 lamindb_setup/_setup_user.py,sha256=-g7Xj6510BDyM8kuqAsVBZFwehlhBa_uWBSV1rPeuM8,4586
 lamindb_setup/_silence_loggers.py,sha256=AKF_YcHvX32eGXdsYK8MJlxEaZ-Uo2f6QDRzjKFCtws,1568
 lamindb_setup/core/__init__.py,sha256=BxIVMX5HQq8oZ1OuY_saUEJz5Tdd7gaCPngxVu5iou4,417
@@ -22,26 +22,26 @@ lamindb_setup/core/_aws_options.py,sha256=Umxu_1AYd-0LJUElZxowvE9vNA7Z8kh_63g3pp
 lamindb_setup/core/_aws_storage.py,sha256=nEjeUv4xUVpoV0Lx-zjjmyb9w804bDyaeiM-OqbfwM0,1799
 lamindb_setup/core/_deprecated.py,sha256=HN7iUBdEgahw5e4NHCd1VJooUfieNb6GRzS5x8jU-q8,2549
 lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
-lamindb_setup/core/_hub_client.py,sha256=Wy3QRJ1mySSu7xHjLZkcQvZpBX03g8oKbxiqxRIpPgo,7456
-lamindb_setup/core/_hub_core.py,sha256=8ZdekUtJMPdWLUqiHeT2nZ_F7FV0GJkQ7ixaIOSaAr4,21309
+lamindb_setup/core/_hub_client.py,sha256=XJ9V0th11zcx87tDTDVtYf5YjnEC06_jiSRHcnbFseg,7510
+lamindb_setup/core/_hub_core.py,sha256=FCFvnQsPWUYLJ_GFsuQ8rINQ7Rqth71pDfNJHRjPY_0,21344
 lamindb_setup/core/_hub_crud.py,sha256=IAuPZes1am8OFwtcf5jSRQPGG1eKwVTEsp9Li-uq0cQ,5377
 lamindb_setup/core/_hub_utils.py,sha256=6dyDGyzYFgVfR_lE3VN3CP1jGp98gxPtr-T91PAP05U,2687
 lamindb_setup/core/_private_django_api.py,sha256=By63l3vIEtK1pq246FhHq3tslxsaTJGKm5VakYluWp4,2656
 lamindb_setup/core/_settings.py,sha256=eslFO84vb5uRRfJ3r_uu4O8677l8lU5BbpZJMSAYw6A,8244
-lamindb_setup/core/_settings_instance.py,sha256=OLpwUp4a0SvVQ_r48C7Se2O5rckOE5MC0afAUGRAYMY,19549
-lamindb_setup/core/_settings_load.py,sha256=5wzhfCqt5mA9rpgAs9EWgxifrpL7Xwjw2YWaoL4VNeU,4281
-lamindb_setup/core/_settings_save.py,sha256=P7TPIi0T7VPdyQX9Z5w9y0Uzq4Y5QHbFCuzWdN2nkfk,3185
-lamindb_setup/core/_settings_storage.py,sha256=1-Wso38WN0Pcl3QdkdZc-B01D9Jpl-bjNMmu_FRLPFU,13103
-lamindb_setup/core/_settings_store.py,sha256=NhKi7GsC4q5V280Ex7sjiSR2j6VXt1-01xPgBPtS_58,2257
+lamindb_setup/core/_settings_instance.py,sha256=iUoLJj_bl8axs4fiNA5jgLjIEvrMaJBzz-PUDWXmUwg,19395
+lamindb_setup/core/_settings_load.py,sha256=BaII7qH3hKbttHHtB5YECoz0c2SnxL_IcUEMZhuqHCo,4342
+lamindb_setup/core/_settings_save.py,sha256=P04ZnWFSN4Qq_QFWXNvJDs4yxrY9CjQBol_BDku_JEw,3227
+lamindb_setup/core/_settings_storage.py,sha256=op31t12T_fCfTG8OSfnXnIxB9VCQPg8Oo2cnZHVap8Y,14203
+lamindb_setup/core/_settings_store.py,sha256=zyybc6fJXJcFNTjBrgtgbPhriInuhpmQKNoGmy8ScFA,2298
 lamindb_setup/core/_settings_user.py,sha256=lWqV3HmZCsEq2UsU_iVNW0p9ddsNg7-B6xOaMNH1aw0,1475
-lamindb_setup/core/_setup_bionty_sources.py,sha256=qTPMV5TEbNiTB81QqG2rSs6W8j8kQ7kVQMLOXRzAxBI,4004
+lamindb_setup/core/_setup_bionty_sources.py,sha256=ox3X-SHiHa2lNPSWjwZhINypbLacX6kGwH6hVVrSFZc,1505
 lamindb_setup/core/cloud_sqlite_locker.py,sha256=i6TrT7HG0lqliPvZTlsZ_uplPaqhPBbabyfeR32SkA8,7107
-lamindb_setup/core/django.py,sha256=2-btFL9YglV90qNy3bzV6rYBer7GftGINsDbc6u4mD4,4837
+lamindb_setup/core/django.py,sha256=2qIt3t9Ra_24aKU7q1r30JY87TM_rT6Cvh8rNPlqRc8,4876
 lamindb_setup/core/exceptions.py,sha256=4NpLUNUIfXYVTFX2FvLZF8RW34exk2Vn2X3G4YhnTRg,276
 lamindb_setup/core/hashing.py,sha256=M3Q1-ywnqh4Uy5zojbQfLju19HU0ySp8Oi7FGIJXfFI,3667
 lamindb_setup/core/types.py,sha256=zJii2le38BJUmsNVvzDrbzGYr0yaeb-9Rw9IKmsBr3k,523
-lamindb_setup/core/upath.py,sha256=AvRmICPBxQx-2iQYsXDFsrJS-AAUNuw6IymLcVmkWvo,33550
-lamindb_setup-1.3.2.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
-lamindb_setup-1.3.2.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
-lamindb_setup-1.3.2.dist-info/METADATA,sha256=LIS6VdKmukplKvZNzFx1KP0bn1cXMTVDlIJ_MCtU3n8,1779
-lamindb_setup-1.3.2.dist-info/RECORD,,
+lamindb_setup/core/upath.py,sha256=bjKlQZFe-azlZZZ_VjH9bzfh93G6b88b_IE6E6Dg7cY,33662
+lamindb_setup-1.4.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
+lamindb_setup-1.4.0.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
+lamindb_setup-1.4.0.dist-info/METADATA,sha256=eirnixadBvZvh_WfrT4ZnGPcpUcUj5nTTw36NDkEi00,1762
+lamindb_setup-1.4.0.dist-info/RECORD,,