lamindb_setup 0.69.4__py2.py3-none-any.whl → 0.70.0__py2.py3-none-any.whl

lamindb_setup/__init__.py

@@ -36,7 +36,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.69.4"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.70.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import sys
 from os import name as _os_name

lamindb_setup/_connect_instance.py

@@ -27,6 +27,17 @@ from ._migrate import check_whether_migrations_in_sync
 _TEST_FAILED_LOAD = False
 
 
+INSTANCE_NOT_FOUND_MESSAGE = (
+    "'{owner}/{name}' not found:"
+    " '{hub_result}'\nCheck your permissions:"
+    " https://lamin.ai/{owner}/{name}"
+)
+
+
+class InstanceNotFoundError(SystemExit):
+    pass
+
+
 def check_db_dsn_equal_up_to_credentials(db_dsn_hub, db_dsn_local):
     return (
         db_dsn_hub.scheme == db_dsn_local.scheme
@@ -121,6 +132,8 @@ def connect(
         make_hub_request = True
         if settings_file.exists():
             isettings = load_instance_settings(settings_file)
+            # mimic instance_result from hub
+            instance_result = {"id": isettings.id.hex}
             # skip hub request for a purely local instance
             make_hub_request = isettings.is_remote
 
@@ -139,7 +152,6 @@ def connect(
                     root=storage_result["root"],
                     region=storage_result["region"],
                     uid=storage_result["lnid"],
-                    is_hybrid=instance_result["storage_mode"] == "hybrid",
                 )
                 isettings = InstanceSettings(
                     id=UUID(instance_result["id"]),
@@ -149,27 +161,24 @@ def connect(
                     db=db_updated,
                     schema=instance_result["schema_str"],
                     git_repo=instance_result["git_repo"],
+                    local_storage=instance_result["storage_mode"] == "hybrid",
                 )
                 check_whether_migrations_in_sync(instance_result["lamindb_version"])
             else:
-                error_message = (
-                    f"'{owner}/{name}' not loadable:"
-                    f" '{hub_result}'\nCheck your permissions:"
-                    f" https://lamin.ai/{owner}/{name}?tab=collaborators"
+                message = INSTANCE_NOT_FOUND_MESSAGE.format(
+                    owner=owner, name=name, hub_result=hub_result
                 )
                 if settings_file.exists():
                     isettings = load_instance_settings(settings_file)
                     if isettings.is_remote:
                         if _raise_not_reachable_error:
-                            raise SystemExit(error_message)
+                            raise InstanceNotFoundError(message)
                         return "instance-not-reachable"
 
                 else:
                     if _raise_not_reachable_error:
-                        raise SystemExit(error_message)
+                        raise InstanceNotFoundError(message)
                     return "instance-not-reachable"
-                # mimic instance_result from hub
-                instance_result = {"id": isettings.id.hex}
 
         if storage is not None:
             update_isettings_with_storage(isettings, storage)
@@ -255,7 +264,7 @@ def update_isettings_with_storage(
     isettings: InstanceSettings, storage: UPathStr
 ) -> None:
     ssettings = StorageSettings(storage)
-    if ssettings.is_cloud:
+    if ssettings.type_is_cloud:
         try:  # triggering ssettings.id makes a lookup in the storage table
             logger.success(f"loaded storage: {ssettings.id} / {ssettings.root_as_str}")
         except RuntimeError as e:

lamindb_setup/_delete.py

@@ -1,11 +1,17 @@
 import shutil
 from pathlib import Path
 from lamin_utils import logger
+from uuid import UUID
 from typing import Optional
 from .core._settings_instance import InstanceSettings
+from .core._settings_storage import StorageSettings
 from .core._settings import settings
 from .core._settings_load import load_instance_settings
 from .core._settings_store import instance_settings_file
+from .core.upath import check_storage_is_empty, hosted_buckets
+from .core._hub_core import delete_instance as delete_instance_on_hub
+from .core._hub_core import connect_instance as load_instance_from_hub
+from ._connect_instance import INSTANCE_NOT_FOUND_MESSAGE
 
 
 def delete_cache(cache_dir: Path):
@@ -13,6 +19,12 @@ def delete_cache(cache_dir: Path):
         shutil.rmtree(cache_dir)
 
 
+def delete_exclusion_dir(isettings: InstanceSettings) -> None:
+    exclusion_dir = isettings.storage.root / f".lamindb/_exclusion/{isettings.id.hex}"
+    if exclusion_dir.exists():
+        exclusion_dir.rmdir()
+
+
 def delete_by_isettings(isettings: InstanceSettings) -> None:
     settings_file = isettings._get_settings_file()
     if settings_file.exists():
@@ -22,29 +34,31 @@ def delete_by_isettings(isettings: InstanceSettings) -> None:
         try:
             if isettings._sqlite_file.exists():
                 isettings._sqlite_file.unlink()
-            exclusion_dir = (
-                isettings.storage.root / f".lamindb/_exclusion/{isettings.id.hex}"
-            )
-            if exclusion_dir.exists():
-                exclusion_dir.rmdir()
         except PermissionError:
             logger.warning(
                 "Did not have permission to delete SQLite file:"
                 f" {isettings._sqlite_file}"
             )
             pass
-    if isettings.is_remote:
-        logger.warning("manually delete your remote instance on lamin.ai")
-    logger.warning(f"manually delete your stored data: {isettings.storage.root}")
     # unset the global instance settings
     if settings._instance_exists and isettings.slug == settings.instance.slug:
         if settings._instance_settings_path.exists():
             settings._instance_settings_path.unlink()
         settings._instance_settings = None
+    if isettings.storage._mark_storage_root.exists():
+        isettings.storage._mark_storage_root.unlink()
 
 
-def delete(instance_name: str, force: bool = False) -> Optional[int]:
-    """Delete a LaminDB instance."""
+def delete(
+    instance_name: str, force: bool = False, require_empty: bool = True
+) -> Optional[int]:
+    """Delete a LaminDB instance.
+
+    Args:
+        instance_name (str): The name of the instance to delete.
+        force (bool): Whether to skip the confirmation prompt.
+        require_empty (bool): Whether to check if the instance is empty before deleting.
+    """
     if "/" in instance_name:
         logger.warning(
             "Deleting the instance of another user is currently not supported with the"
@@ -53,6 +67,44 @@ def delete(instance_name: str, force: bool = False) -> Optional[int]:
         )
         raise ValueError("Invalid instance name: '/' delimiter not allowed.")
     instance_slug = f"{settings.user.handle}/{instance_name}"
+    if settings._instance_exists and settings.instance.name == instance_name:
+        isettings = settings.instance
+    else:
+        settings_file = instance_settings_file(instance_name, settings.user.handle)
+        if not settings_file.exists():
+            hub_result = load_instance_from_hub(
+                owner=settings.user.handle, name=instance_name
+            )
+            if isinstance(hub_result, str):
+                message = INSTANCE_NOT_FOUND_MESSAGE.format(
+                    owner=settings.user.handle,
+                    name=instance_name,
+                    hub_result=hub_result,
+                )
+                logger.warning(message)
+                return None
+            instance_result, storage_result = hub_result
+            ssettings = StorageSettings(
+                root=storage_result["root"],
+                region=storage_result["region"],
+                uid=storage_result["lnid"],
+            )
+            isettings = InstanceSettings(
+                id=UUID(instance_result["id"]),
+                owner=settings.user.handle,
+                name=instance_name,
+                storage=ssettings,
+                local_storage=instance_result["storage_mode"] == "hybrid",
+                db=instance_result["db"] if "db" in instance_result else None,
+                schema=instance_result["schema_str"],
+                git_repo=instance_result["git_repo"],
+            )
+        else:
+            isettings = load_instance_settings(settings_file)
+    if isettings.dialect != "sqlite":
+        logger.warning(
+            f"delete() does not yet affect your Postgres database at {isettings.db}"
+        )
     if not force:
         valid_responses = ["y", "yes"]
         user_input = (
@@ -62,17 +114,31 @@ def delete(instance_name: str, force: bool = False) -> Optional[int]:
         )
         if user_input not in valid_responses:
             return -1
-    if settings._instance_exists and settings.instance.name == instance_name:
-        isettings = settings.instance
-    else:
-        settings_file = instance_settings_file(instance_name, settings.user.handle)
-        if not settings_file.exists():
+
+    # the actual deletion process begins here
+    if isettings.dialect == "sqlite" and isettings.is_remote:
+        # delete the exlusion dir first because it's hard to count its objects
+        delete_exclusion_dir(isettings)
+    if isettings.storage.type_is_cloud and isettings.storage.root_as_str.startswith(
+        hosted_buckets
+    ):
+        if not require_empty:
             logger.warning(
-                "could not delete as instance settings do not exist locally. did you"
-                " provide a wrong instance name? could you try loading it?"
+                "hosted storage always has to be empty, ignoring `require_empty`"
             )
-            return None
-        isettings = load_instance_settings(settings_file)
+        require_empty = True
+    n_objects = check_storage_is_empty(
+        isettings.storage.root,
+        raise_error=require_empty,
+        account_for_sqlite_file=isettings.dialect == "sqlite",
+    )
     logger.info(f"deleting instance {instance_slug}")
+    # below we can skip check_storage_is_empty() because we already called
+    # it above
+    delete_instance_on_hub(isettings.id, require_empty=False)
     delete_by_isettings(isettings)
+    if n_objects == 0 and isettings.storage.type == "local":
+        # dir is only empty after sqlite file was delete via delete_by_isettings
+        (isettings.storage.root / ".lamindb").rmdir()
+        isettings.storage.root.rmdir()
     return None

lamindb_setup/_init_instance.py

@@ -269,7 +269,7 @@ def init(
             delete_instance_record(isettings.id)
             isettings._get_settings_file().unlink(missing_ok=True)  # type: ignore
         if ssettings is not None:
-            delete_storage_record(ssettings.uuid)  # type: ignore
+            delete_storage_record(ssettings._uuid)  # type: ignore
         raise e
     return None
 

lamindb_setup/_migrate.py

@@ -16,6 +16,9 @@ def check_whether_migrations_in_sync(db_version_str: str):
         installed_version_str = metadata.version("lamindb")
     except metadata.PackageNotFoundError:
         return None
+    if db_version_str is None:
+        logger.warning("no lamindb version stored to compare with installed version")
+        return None
     installed_version = version.parse(installed_version_str)
     db_version = version.parse(db_version_str)
     if (
@@ -83,7 +86,7 @@ class migrate:
             collaborator = call_with_fallback_auth(
                 select_collaborator,
                 instance_id=instance_id_str,
-                account_id=settings.user.uuid,
+                account_id=settings.user._uuid,
             )
             if collaborator is None or collaborator["role"] != "admin":
                 raise SystemExit(

lamindb_setup/_register_instance.py

@@ -17,6 +17,6 @@ def register(_test: bool = False):
     if ssettings._uid is None and _test:
         # because django isn't up, we can't get it from the database
         ssettings._uid = base62(8)
-    ssettings._uuid = init_storage_hub(ssettings)
+    ssettings._uuid_ = init_storage_hub(ssettings)
     init_instance_hub(isettings)
     isettings._persist()

lamindb_setup/_setup_user.py

@@ -99,7 +99,7 @@ def login(
     user_settings.uid = user_id
     user_settings.handle = user_handle
     user_settings.name = user_name
-    user_settings.uuid = user_uuid
+    user_settings._uuid = user_uuid
     user_settings.access_token = access_token
     save_user_settings(user_settings)
 

lamindb_setup/core/_hub_core.py

@@ -48,6 +48,14 @@ def _delete_storage_record(storage_uuid: UUID, client: Client) -> None:
     client.table("storage").delete().eq("id", storage_uuid.hex).execute()
 
 
+def update_instance_record(instance_uuid: UUID, fields: Dict) -> None:
+    from ._hub_crud import update_instance
+
+    return call_with_fallback_auth(
+        update_instance, instance_id=instance_uuid.hex, instance_fields=fields
+    )
+
+
 def init_storage(
     ssettings: StorageSettings,
 ) -> UUID:
@@ -70,7 +78,7 @@ def _init_storage(ssettings: StorageSettings, client: Client) -> UUID:
     fields = dict(
         id=id.hex,
         lnid=ssettings.uid,
-        created_by=settings.user.uuid.hex,  # type: ignore
+        created_by=settings.user._uuid.hex,  # type: ignore
         root=root,
         region=ssettings.region,
         type=ssettings.type,
@@ -81,13 +89,13 @@ def _init_storage(ssettings: StorageSettings, client: Client) -> UUID:
     return id
 
 
-def delete_instance(identifier: Union[UUID, str]):
+def delete_instance(identifier: Union[UUID, str], require_empty: bool = True) -> None:
     """Fully delete an instance in the hub.
 
     This function deletes the relevant instance and storage records in the hub,
     conditional on the emptiness of the storage location.
     """
-    from .upath import check_s3_storage_location_empty, create_path, hosted_buckets
+    from .upath import check_storage_is_empty, create_path
     from ._settings_storage import mark_storage_root
 
     if isinstance(identifier, UUID):
@@ -103,19 +111,22 @@ def delete_instance(identifier: Union[UUID, str]):
             name=name,
         )
 
-    if instance_with_storage is not None:
+    if instance_with_storage is None:
+        logger.warning("instance not found")
+        return None
+
+    if require_empty:
         root_string = instance_with_storage["storage"]["root"]
         # gate storage and instance deletion on empty storage location for
-        # both hosted and non-hosted s3 instances
-        if root_string.startswith("s3://"):
-            root_path = create_path(root_string)
-            # only mark hosted instances
-            if root_string.startswith(hosted_buckets):
-                mark_storage_root(root_path)
-            check_s3_storage_location_empty(root_path)
-
-        delete_instance_record(UUID(instance_with_storage["id"]))
-        delete_storage_record(UUID(instance_with_storage["storage_id"]))
+        root_path = create_path(root_string)
+        mark_storage_root(root_path)  # address permission error
+        account_for_sqlite_file = instance_with_storage["db_scheme"] is None
+        check_storage_is_empty(
+            root_path, account_for_sqlite_file=account_for_sqlite_file
+        )
+    delete_instance_record(UUID(instance_with_storage["id"]))
+    delete_storage_record(UUID(instance_with_storage["storage_id"]))
+    return None
 
 
 def delete_instance_record(
@@ -140,9 +151,9 @@ def _init_instance(isettings: InstanceSettings, client: Client) -> None:
         lamindb_version = None
     fields = dict(
         id=isettings.id.hex,
-        account_id=settings.user.uuid.hex,  # type: ignore
+        account_id=settings.user._uuid.hex,  # type: ignore
         name=isettings.name,
-        storage_id=isettings.storage.uuid.hex,  # type: ignore
+        storage_id=isettings.storage._uuid.hex,  # type: ignore
         schema_str=isettings._schema_str,
         lamindb_version=lamindb_version,
         public=False,

lamindb_setup/core/_settings_instance.py

@@ -1,5 +1,8 @@
+from __future__ import annotations
+
 import os
 import shutil
+from .upath import LocalPathClasses, convert_pathlike
 from pathlib import Path
 from typing import Literal, Optional, Set, Tuple
 from uuid import UUID
@@ -16,6 +19,11 @@ from ._settings_storage import StorageSettings
 from ._settings_store import current_instance_settings_file, instance_settings_file
 from .upath import UPath
 
+LOCAL_STORAGE_ROOT_WARNING = (
+    "No local storage root found, set via `ln.setup.settings.instance.local_storage ="
+    " local_root`"
+)
+
 
 def sanitize_git_repo_url(repo_url: str) -> str:
     assert repo_url.startswith("https://")
@@ -31,6 +39,7 @@ class InstanceSettings:
         owner: str,  # owner handle
         name: str,  # instance name
         storage: StorageSettings,  # storage location
+        local_storage: bool = False,  # default to local storage
         uid: Optional[str] = None,  # instance uid/lnid
         db: Optional[str] = None,  # DB URI
         schema: Optional[str] = None,  # comma-separated string of schema names
@@ -47,6 +56,9 @@ class InstanceSettings:
         self._db: Optional[str] = db
         self._schema_str: Optional[str] = schema
         self._git_repo = None if git_repo is None else sanitize_git_repo_url(git_repo)
+        # local storage
+        self._local_storage_on = local_storage
+        self._local_storage = None
 
     def __repr__(self):
         """Rich string representation."""
@@ -85,14 +97,60 @@ class InstanceSettings:
         """Instance name."""
         return self._name
 
+    def _search_local_root(self):
+        from lnschema_core.models import Storage
+
+        records = Storage.objects.filter(type="local").all()
+        for record in records:
+            if Path(record.root).exists():
+                self._local_storage = StorageSettings(record.root)
+                logger.important(f"defaulting to local storage: {record}")
+                break
+        if self._local_storage is None:
+            logger.warning(LOCAL_STORAGE_ROOT_WARNING)
+
     @property
-    def identifier(self) -> str:
-        """Unique semantic identifier."""
-        logger.warning(
-            "InstanceSettings.identifier is deprecated and will be removed, use"
-            " InstanceSettings.slug instead"
-        )
-        return self.slug
+    def local_storage(self) -> StorageSettings:
+        """Default local storage.
+
+        Warning: Only enable if you're sure you want to use the more complicated
+        storage mode across local & cloud locations.
+
+        As an admin, enable via: `ln.setup.settings.instance.local_storage =
+        local_root`.
+
+        If enabled, you'll save artifacts to a default local storage
+        location at :attr:`~lamindb.setup.settings.InstanceSettings.local_storage`.
+
+        Upon passing `upload=True` in `artifact.save(upload=True)`, you upload the
+        artifact to the default cloud storage location:
+        :attr:`~lamindb.setup.core.InstanceSettings.storage`.
+        """
+        if not self._local_storage_on:
+            raise ValueError("Local storage is not enabled for this instance.")
+        if self._local_storage is None:
+            self._search_local_root()
+        if self._local_storage is None:
+            raise ValueError(LOCAL_STORAGE_ROOT_WARNING)
+        return self._local_storage
+
+    @local_storage.setter
+    def local_storage(self, local_root: Path):
+        from ._hub_core import update_instance_record
+        from .._init_instance import register_storage
+
+        self._search_local_root()
+        if self._local_storage is not None:
+            raise ValueError(
+                "You already configured a local storage root for this instance in this"
+                f" environment: {self.local_storage.root}"
+            )
+        local_root = convert_pathlike(local_root)
+        assert isinstance(local_root, LocalPathClasses)
+        self._local_storage = StorageSettings(local_root)  # type: ignore
+        register_storage(self._local_storage)  # type: ignore
+        self._local_storage_on = True
+        update_instance_record(self.id, {"storage_mode": "hybrid"})
 
     @property
     def slug(self) -> str:
@@ -211,16 +269,12 @@ class InstanceSettings:
             assert self._db.startswith("postgresql"), f"Unexpected DB value: {self._db}"
             return "postgresql"
 
-    @property
-    def session(self):
-        raise NotImplementedError
-
     @property
     def _is_cloud_sqlite(self) -> bool:
         # can we make this a private property, Sergei?
         # as it's not relevant to the user
         """Is this a cloud instance with sqlite db."""
-        return self.dialect == "sqlite" and self.storage.is_cloud
+        return self.dialect == "sqlite" and self.storage.type_is_cloud
 
     @property
     def _cloud_sqlite_locker(self):
@@ -244,7 +298,7 @@ class InstanceSettings:
     @property
     def is_remote(self) -> bool:
         """Boolean indicating if an instance has no local component."""
-        if not self.storage.is_cloud:
+        if not self.storage.type_is_cloud:
             return False
 
         def is_local_uri(uri: str):

lamindb_setup/core/_settings_load.py

@@ -95,5 +95,5 @@ def setup_user_from_store(store: UserSettingsStore) -> UserSettings:
     settings.uid = store.uid
     settings.handle = store.handle if store.handle != "null" else "anonymous"
     settings.name = store.name if store.name != "null" else None
-    settings.uuid = UUID(store.uuid) if store.uuid != "null" else None
+    settings._uuid = UUID(store.uuid) if store.uuid != "null" else None
     return settings

lamindb_setup/core/_settings_save.py

@@ -44,7 +44,7 @@ def save_settings(
                 elif store_key == "storage_region":
                     value = settings.storage.region
                 else:
-                    if store_key in {"db", "schema_str", "name_"}:
+                    if store_key in {"db", "schema_str", "name_", "uuid"}:
                         settings_key = f"_{store_key.rstrip('_')}"
                     else:
                         settings_key = store_key

lamindb_setup/core/_settings_storage.py

@@ -56,8 +56,9 @@ def get_storage_region(storage_root: UPathStr) -> Optional[str]:
 
 
 def mark_storage_root(root: UPathStr):
-    # we need to touch a 0-byte object in the storage location to avoid
+    # we need to touch a 0-byte object in folder-like storage location on S3 to avoid
     # permission errors from leveraging s3fs on an empty hosted storage location
+    # for consistency, we write this file everywhere
     root_upath = convert_pathlike(root)
     mark_upath = root_upath / IS_INITIALIZED_KEY
     mark_upath.touch()
@@ -92,14 +93,13 @@ def init_storage(root: UPathStr) -> "StorageSettings":
             logger.error("`storage` is not a valid local, GCP storage or AWS S3 path")
             raise e
     ssettings = StorageSettings(uid=uid, root=root_str, region=region)
-    if ssettings.is_cloud:
+    if ssettings.type_is_cloud:
         from ._hub_core import init_storage as init_storage_hub
 
         ssettings._description = f"Created as default storage for instance {uid}"
-        ssettings._uuid = init_storage_hub(ssettings)
+        ssettings._uuid_ = init_storage_hub(ssettings)
         logger.important(f"registered storage: {ssettings.root_as_str}")
-    if ssettings.is_cloud and root_str.startswith("create-s3"):
-        mark_storage_root(ssettings.root)
+    mark_storage_root(ssettings.root)
     return ssettings
 
 
@@ -121,20 +121,17 @@ class StorageSettings:
         self,
         root: UPathStr,
         region: Optional[str] = None,
-        is_hybrid: bool = False,  # refers to storage mode
         uid: Optional[str] = None,
         uuid: Optional[UUID] = None,
         access_token: Optional[str] = None,
     ):
         self._uid = uid
-        self._uuid = uuid
-        self._is_hybrid = is_hybrid
+        self._uuid_ = uuid
         self._root_init = convert_pathlike(root)
         if isinstance(self._root_init, LocalPathClasses):  # local paths
-            self._root_init.mkdir(parents=True, exist_ok=True)
+            (self._root_init / ".lamindb").mkdir(parents=True, exist_ok=True)
             self._root_init = self._root_init.resolve()
         self._root = None
-        self._remote_root = None
         self._aws_account_id: Optional[int] = None
         self._description: Optional[str] = None
         # we don't yet infer region here to make init fast
@@ -155,15 +152,19 @@ class StorageSettings:
         # save access_token here for use in self.root
         self.access_token = access_token
 
+        # local storage
+        self._has_local = False
+        self._local = None
+
     @property
     def id(self) -> int:
         """Storage id."""
         return self.record.id
 
     @property
-    def uuid(self) -> Optional[UUID]:
-        """Storage uuid."""
-        return self._uuid
+    def _uuid(self) -> Optional[UUID]:
+        """Lamin's internal storage uuid."""
+        return self._uuid_
 
     @property
     def uid(self) -> Optional[str]:
@@ -172,6 +173,10 @@ class StorageSettings:
             self._uid = self.record.uid
         return self._uid
 
+    @property
+    def _mark_storage_root(self) -> UPath:
+        return self.root / IS_INITIALIZED_KEY
+
     @property
     def record(self) -> Any:
         """Storage record."""
@@ -180,63 +185,26 @@ class StorageSettings:
             from lnschema_core.models import Storage
             from ._settings import settings
 
-            if not self.is_hybrid:
-                self._record = Storage.objects.using(settings._using_key).get(
-                    root=self.root_as_str
-                )
-            else:
-                # this has to be redone
-                records = Storage.objects.filter(type="local").all()
-                for record in records:
-                    if Path(record.root).exists():
-                        self._record = record
-                        logger.warning("found local storage location")
-                        break
+            self._record = Storage.objects.using(settings._using_key).get(
+                root=self.root_as_str
+            )
         return self._record
 
     def __repr__(self):
         """String rep."""
         s = f"root='{self.root_as_str}', uid='{self.uid}'"
-        if self.uuid is not None:
-            s += f", uuid='{self.uuid.hex}'"
+        if self._uuid is not None:
+            s += f", uuid='{self._uuid.hex}'"
         return f"StorageSettings({s})"
 
-    @property
-    def is_hybrid(self) -> bool:
-        """Qualifies storage mode.
-
-        A storage location can be local, in the cloud, or hybrid. See
-        :attr:`~lamindb.setup.core.StorageSettings.type`.
-
-        Hybrid means that a default local storage location is backed by an
-        optional cloud storage location.
-        """
-        return self._is_hybrid
-
     @property
     def root(self) -> UPath:
         """Root storage location."""
         if self._root is None:
-            if not self.is_hybrid:
-                # below makes network requests to get credentials
-                root_path = create_path(self._root_init, access_token=self.access_token)
-            else:
-                # this is a local path
-                root_path = create_path(self.record.root)
-            self._root = root_path
+            # below makes network requests to get credentials
+            self._root = create_path(self._root_init, access_token=self.access_token)
         return self._root
 
-    @property
-    def remote_root(self) -> UPath:
-        """Remote storage location. Only needed for hybrid storage."""
-        if not self.is_hybrid:
-            raise ValueError("remote_root is only defined for hybrid storage")
-        if self._remote_root is None:
-            self._remote_root = create_path(
-                self._root_init, access_token=self.access_token
-            )
-        return self._remote_root
-
     def _set_fs_kwargs(self, **kwargs):
         """Set additional fsspec arguments for cloud root.
 
@@ -299,7 +267,7 @@ class StorageSettings:
             raise e
 
     @property
-    def is_cloud(self) -> bool:
+    def type_is_cloud(self) -> bool:
         """`True` if `storage_root` is in cloud, `False` otherwise."""
         return self.type != "local"
 

lamindb_setup/core/_settings_user.py

@@ -25,7 +25,7 @@ class UserSettings:
     """User access token."""
     uid: str = "null"
     """Universal user ID."""
-    uuid: Optional[UUID] = None
+    _uuid: Optional[UUID] = None
     """Lamin's internal user ID."""
     name: Optional[str] = None
     """Full name."""

lamindb_setup/core/django.py

@@ -103,3 +103,6 @@ def setup_django(
 
     global IS_SETUP
     IS_SETUP = True
+
+    if isettings._local_storage_on:
+        isettings._search_local_root()

lamindb_setup/core/upath.py

@@ -1,6 +1,8 @@
 # we are not documenting UPath here because it's documented at lamindb.UPath
 """Paths & file systems."""
 
+from __future__ import annotations
+
 import os
 from datetime import datetime, timezone
 import botocore.session
@@ -165,7 +167,7 @@ class ProgressCallback(fsspec.callbacks.Callback):
         return None
 
 
-def download_to(self, path, print_progress: bool = False, **kwargs):
+def download_to(self, path: UPathStr, print_progress: bool = False, **kwargs):
     """Download to a path."""
     if print_progress:
         # can't do path.is_dir() because path doesn't exist
@@ -180,20 +182,38 @@ def download_to(self, path, print_progress: bool = False, **kwargs):
     self.fs.download(str(self), str(path), **kwargs)
 
 
-def upload_from(self, path, print_progress: bool = False, **kwargs):
+def upload_from(
+    self,
+    path: UPathStr,
+    dir_inplace: bool = False,
+    print_progress: bool = False,
+    **kwargs,
+):
     """Upload from a local path."""
+    path_is_dir = os.path.isdir(path)
+    if not path_is_dir:
+        dir_inplace = False
+
     if print_progress:
-        if not os.path.isdir(path):
+        if not path_is_dir:
             cb = ProgressCallback("uploading")
         else:
             # todo: make proper progress bar for directories
             cb = fsspec.callbacks.NoOpCallback()
         kwargs["callback"] = cb
+
+    if dir_inplace:
+        path = Path(path)
+        source = [f for f in path.rglob("*") if f.is_file()]
+        destination = [str(self / f.relative_to(path)) for f in source]
+        source = [str(f) for f in source]  # type: ignore
+    else:
+        source = str(path)  # type: ignore
+        destination = str(self)  # type: ignore
     # this weird thing is to avoid s3fs triggering create_bucket in upload
     # if dirs are present
     # it allows to avoid permission error
-    destination = str(self)
-    if self.protocol != "s3" or os.path.isfile(path):
+    if self.protocol != "s3" or not path_is_dir or dir_inplace:
         cleanup_cache = False
     else:
         bucket = self._url.netloc
@@ -205,7 +225,7 @@ def upload_from(self, path, print_progress: bool = False, **kwargs):
         else:
             cleanup_cache = False
 
-    self.fs.upload(str(path), destination, **kwargs)
+    self.fs.upload(source, destination, **kwargs)
 
     if cleanup_cache:
         # normally this is invalidated after the upload but still better to check
@@ -670,20 +690,34 @@ class InstanceNotEmpty(Exception):
     pass
 
 
-def check_s3_storage_location_empty(root: UPathStr) -> None:
+# is as fast as boto3: https://lamin.ai/laminlabs/lamindata/transform/krGp3hT1f78N5zKv
+def check_storage_is_empty(
+    root: UPathStr, *, raise_error: bool = True, account_for_sqlite_file: bool = False
+) -> int:
     root_upath = convert_pathlike(root)
     root_string = root_upath.as_posix()  # type: ignore
     # we currently touch a 0-byte file in the root of a hosted storage location
     # ({storage_root}/.lamindb/_is_initialized) during storage initialization
     # since path.fs.find raises a PermissionError on empty hosted
     # subdirectories (see lamindb_setup/core/_settings_storage/init_storage).
-    n_touched_objects = 0
-    if root_string.startswith(hosted_buckets):
-        n_touched_objects = 1
+    n_offset_objects = 1  # because of touched dummy file, see mark_storage_root()
+    if account_for_sqlite_file:
+        n_offset_objects += 1  # because of SQLite file
     objects = root_upath.fs.find(root_string)
     n_objects = len(objects)
-    if n_objects > n_touched_objects:
-        raise InstanceNotEmpty(
-            f"""storage location contains objects;
-            {compute_file_tree(root_upath)[0]}"""
-        )
+    n_diff = n_objects - n_offset_objects
+    ask_for_deletion = (
+        "delete them prior to deleting the instance"
+        if raise_error
+        else "consider deleting them"
+    )
+    message = (
+        f"Storage location contains {n_objects} objects "
+        f"({n_offset_objects} ignored) - {ask_for_deletion}\n{objects}"
+    )
+    if n_diff > 0:
+        if raise_error:
+            raise InstanceNotEmpty(message)
+        else:
+            logger.warning(message)
+    return n_diff

{lamindb_setup-0.69.4.dist-info → lamindb_setup-0.70.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb_setup
-Version: 0.69.4
+Version: 0.70.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <laminlabs@gmail.com>
 Description-Content-Type: text/markdown

{lamindb_setup-0.69.4.dist-info → lamindb_setup-0.70.0.dist-info}/RECORD

@@ -1,43 +1,43 @@
-lamindb_setup/__init__.py,sha256=0WvssKET9TeGkrbvOxtdFPzn4cpVXtv8JyRR4eGz7qU,1558
+lamindb_setup/__init__.py,sha256=yTrf2HNpLb9hFr44mDAEjKjXDwrKYs_n2wX6JmyjOkQ,1558
 lamindb_setup/_add_remote_storage.py,sha256=S2rFvn-JcGaBOiBNQoAiaTPM590GQh-g4OJeNsben0Q,1802
 lamindb_setup/_cache.py,sha256=wrwlHi2IfxcWDfOW-cHzzkQNlLvasoQPPNxihICbwew,809
 lamindb_setup/_check.py,sha256=xgc5kF2HA4lGuAA9FJT9BJyQV5z1EwYMr92RTbGcQg4,92
 lamindb_setup/_check_setup.py,sha256=LLdTn4JYiyWKji_sexq2eENbmvA6_gdxqMKjDnF-nJc,2543
 lamindb_setup/_close.py,sha256=XtP7uyKRbwCGkXdSkwuiEnLYn5PEN3t3q50dPHNJRSU,1150
-lamindb_setup/_connect_instance.py,sha256=QICzNO3ID3JwJb2yMd6R5DfExT3q5vlkn0e8WHo2lLw,11789
-lamindb_setup/_delete.py,sha256=XzYaB2k6DhNfr5b-nzHZ4qGcAIxnuP8fc-hIOYXvy8E,3111
+lamindb_setup/_connect_instance.py,sha256=XzHbYPmzPmAJZ1EaFtDBzw2skh1s0tppKCg2HOxx1HQ,11908
+lamindb_setup/_delete.py,sha256=aIfQpOmb-icGsNVOd9EhHGGBQvsSJF7fBW_fzSwPhuM,5861
 lamindb_setup/_django.py,sha256=Yv-bNj80I5Fj9AJ2BFgcv9JZg4WUrlh3UStty82_jXQ,1500
 lamindb_setup/_exportdb.py,sha256=f5XUhRxTsU-yqfPM92OxjZKRxJo3lKaqYTRh9oytDfY,2084
 lamindb_setup/_importdb.py,sha256=bxFZNjtVOAW4yeMQakVYZE4-WNi4QE-MdMynHShMOgI,1836
-lamindb_setup/_init_instance.py,sha256=i0X9ENufdSGPog_8S1dypAvhwfGpVfsGuAIpnpfkjTc,11479
-lamindb_setup/_migrate.py,sha256=JlyjuhPO7qVEzaRfB5WBvkGgy-0qMh60MddM1zemSlo,8672
-lamindb_setup/_register_instance.py,sha256=tfdpL9NFXmtdxAGnym5Q4I5x4cT6_xVBim6pbECKlvk,795
+lamindb_setup/_init_instance.py,sha256=DlVHpz8avTYaL-IoxSO4XM1rfZmtZXsWpjtY95vYmK0,11480
+lamindb_setup/_migrate.py,sha256=wVDfIN1z2OIJztrenpYx42XN09retbPek49TzzmFQ6I,8810
+lamindb_setup/_register_instance.py,sha256=1cZXXuCf6cqbiDchmW5vYNTf34wMNSZ2QtHmIJbTT4c,796
 lamindb_setup/_schema.py,sha256=edhbiUYatbO5FfBA6PYRmVmYWbtBXZBpATa70_MZiWo,642
-lamindb_setup/_setup_user.py,sha256=xwSJpbLtl2gW-frGs5k_kBL6PZZ00pe1gfmmd-a7myc,3657
+lamindb_setup/_setup_user.py,sha256=5zZ924ckd8CGBiD4-5bdTN-NUNMS7N5w8Ovy6JIYPMU,3658
 lamindb_setup/_silence_loggers.py,sha256=auXnYVSx2ErZmyXZJ7FYYA69UQgUDQ5SnuznFXzfaWs,1548
 lamindb_setup/core/__init__.py,sha256=xfhu3cO_zPtNztmu6vlWtdbX92gPpnjyff5dOqCHTvE,369
 lamindb_setup/core/_aws_storage.py,sha256=GIT9SagV-1oFlLpRIogFgXuTwzMZsppFjqKA4-Vp66s,1762
 lamindb_setup/core/_deprecated.py,sha256=DNO_90uyYGM9on7rzJCu9atS7Z1qOQypsuL9PI6I-OQ,2491
 lamindb_setup/core/_docs.py,sha256=0ha3cZqzvIQnAXVC_JlAiRrFi3VpEEVookP_oURhBr4,240
 lamindb_setup/core/_hub_client.py,sha256=H09wJRck4d3fQ8VBSK1v8Dbj9RfqLdW9UqoTs6OCyUM,5520
-lamindb_setup/core/_hub_core.py,sha256=2ul3KvVorXLZfEefGbm9toRLga-rli8qzzq71vvEfw0,10990
+lamindb_setup/core/_hub_core.py,sha256=Y_RnT82Y7fHgMmNvG8NTDTnx5Ybf_IwFQSMl6AFveVg,11294
 lamindb_setup/core/_hub_crud.py,sha256=0qKKsnHlx_FceX-jahdS2vJc2IuGRzouQrtAMn6MTSA,4505
 lamindb_setup/core/_hub_utils.py,sha256=4bjtm4QVivDwtVq3Bx9hQFZtdU2WmqrVdpupdvxMHcM,1862
 lamindb_setup/core/_settings.py,sha256=ogeOddaJ9tiolzpGwUIu6i4vhcViXQvsREEzZ95uyvI,3241
-lamindb_setup/core/_settings_instance.py,sha256=XRpHZJT8MZONpIS5Sop6Rk3BLYVmguh2PkC3TrGU7jU,11556
-lamindb_setup/core/_settings_load.py,sha256=65T_lW1xby__PKULufqzc1eWE30SVRNbxS2TAVSCWXM,3744
-lamindb_setup/core/_settings_save.py,sha256=bAnjmZuAKk4wx4EoTx-LfaCY7yyU_4bUfF7hdI71DgY,2563
-lamindb_setup/core/_settings_storage.py,sha256=a_Pa_nPzFuDeYxBlvFelQIkINlVGSNO_2aLvY_nxP-w,13044
+lamindb_setup/core/_settings_instance.py,sha256=nAaTW4kd_wYM5hKlRwpLgBd-ATiEDvtHOX1NCnpkFyQ,13927
+lamindb_setup/core/_settings_load.py,sha256=RkH_zfdGAlBO-4sovmFNGSP56XhSJpUYdvlZlNh-Qqs,3745
+lamindb_setup/core/_settings_save.py,sha256=QnUQsYMAXv9xUYAsQ-5vldntSnAvNb3TMmJ61J3kU_I,2571
+lamindb_setup/core/_settings_storage.py,sha256=Y2FsDmOFW7Kz1bh2MAMyQkWPJjoYN12ObpqVp82qRag,11750
 lamindb_setup/core/_settings_store.py,sha256=iM3Ei-YTSQo6wbcUhOliQx_KmHo8sxDx0-elJPt9AwQ,1929
-lamindb_setup/core/_settings_user.py,sha256=wWLrSaohWlQSLOA-7fLSRCV43rupy-ixNvkZMWT9Pgo,1281
+lamindb_setup/core/_settings_user.py,sha256=ouRhbnIN12cjvVR0Iz98hUyYyGbqcGNRb1OCeI2T6wo,1282
 lamindb_setup/core/_setup_bionty_sources.py,sha256=xhIIJqtCzrjTONzU_68bq5kQxsNRDfAprHi2oJvY_LY,2908
 lamindb_setup/core/cloud_sqlite_locker.py,sha256=nFwAupnbIy_sxzlJBayfD2Dh8ZMUGNxbspIbv6BGrp8,6835
-lamindb_setup/core/django.py,sha256=MNlEFS8a7GX1IxmMgJymoRf3M03Eg881zQ2_MKONOQI,3335
+lamindb_setup/core/django.py,sha256=bKwuv83ubKVsMeunnFtpWd1C-y29I2cuvpW10rCPpZo,3411
 lamindb_setup/core/exceptions.py,sha256=YzkJA51ZAa2w9lawAggSdg-NA-EIYJGNRnFd0DN3_bE,275
 lamindb_setup/core/hashing.py,sha256=nbQO_CSiX09O4bznHABft94fdFZAHPm2NIyvH5xagoo,2011
 lamindb_setup/core/types.py,sha256=fR71SLjoN1MkCjx8TJcjYDgmgO4bPXBX5J_RKpmmy_o,497
-lamindb_setup/core/upath.py,sha256=ahyWAQQfZRmtQBz-p_UN7R-wqsti6YZ8ZMixy-n8XlQ,24039
-lamindb_setup-0.69.4.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
-lamindb_setup-0.69.4.dist-info/WHEEL,sha256=Sgu64hAMa6g5FdzHxXv9Xdse9yxpGGMeagVtPMWpJQY,99
-lamindb_setup-0.69.4.dist-info/METADATA,sha256=HKUfuOqFrwnVrQmc-wDO3_5rc4DNJMEu25L2BjPPKbo,1469
-lamindb_setup-0.69.4.dist-info/RECORD,,
+lamindb_setup/core/upath.py,sha256=v8lPEA5j_SH0QzrHfw02x3HDt1ikFVc3gdoCCVfwfmw,25078
+lamindb_setup-0.70.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
+lamindb_setup-0.70.0.dist-info/WHEEL,sha256=Sgu64hAMa6g5FdzHxXv9Xdse9yxpGGMeagVtPMWpJQY,99
+lamindb_setup-0.70.0.dist-info/METADATA,sha256=9hjTjaw1xv6SQR_ugZg7DMKxGu0SqgJy0b9VJuR2rVw,1469
+lamindb_setup-0.70.0.dist-info/RECORD,,