lamindb_setup 1.16.0__py3-none-any.whl → 1.17.0__py3-none-any.whl

lamindb_setup/__init__.py

@@ -35,14 +35,17 @@ Migration management
 
 """
 
-__version__ = "1.16.0"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.17.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 import warnings
 
-# ignore for now, remove this after supabase upgrade in the deps
+# ignore for now, this is for timeout parameter,
+# it is more convenient to specify it directly for now
 warnings.filterwarnings("ignore", category=DeprecationWarning, module="supabase")
-warnings.filterwarnings("ignore", category=DeprecationWarning, module="supafunc")
+warnings.filterwarnings(
+    "ignore", category=DeprecationWarning, module="supabase_functions"
+)
 warnings.filterwarnings("ignore", category=DeprecationWarning, module="postgrest")
 
 from packaging import version as packaging_version

lamindb_setup/_check_setup.py

@@ -4,7 +4,6 @@ import functools
 import importlib as il
 import inspect
 import os
-from importlib.metadata import distributions
 from typing import TYPE_CHECKING
 from uuid import UUID
 
@@ -16,7 +15,6 @@ from .core._settings import settings
 from .core._settings_store import current_instance_settings_file
 from .errors import (
     MODULE_WASNT_CONFIGURED_MESSAGE_TEMPLATE,
-    InstanceNotSetupError,
     ModuleWasntConfigured,
 )
 
@@ -26,8 +24,6 @@ if TYPE_CHECKING:
     from .core._settings_instance import InstanceSettings
 
 
-CURRENT_ISETTINGS: InstanceSettings | None = None
-MODULE_CANDIDATES: set[str] | None = None
 IS_LOADING: bool = False
 
 
@@ -45,55 +41,6 @@ def disable_auto_connect(func: Callable):
     return wrapper
 
 
-def find_module_candidates():
-    """Find all local packages that depend on lamindb."""
-    global MODULE_CANDIDATES
-    if MODULE_CANDIDATES is not None:
-        return MODULE_CANDIDATES
-    all_dists = list(distributions())
-    lamindb_deps = {
-        dist.metadata["Name"].lower()
-        for dist in all_dists
-        if dist.requires and any("lamindb" in req.lower() for req in dist.requires)
-    }
-    lamindb_deps.remove("lamindb")
-    MODULE_CANDIDATES = lamindb_deps
-    return lamindb_deps
-
-
-def _get_current_instance_settings(from_module: str | None = None) -> InstanceSettings:
-    from .core._settings_instance import InstanceSettings
-
-    global CURRENT_ISETTINGS
-
-    if CURRENT_ISETTINGS is not None:
-        return CURRENT_ISETTINGS
-    if current_instance_settings_file().exists():
-        from .core._settings_load import load_instance_settings
-
-        try:
-            isettings = load_instance_settings()
-        except Exception as e:
-            # user will get more detailed traceback once they run the CLI
-            logger.error(
-                "Current instance cannot be reached, disconnect from it: `lamin disconnect`\n"
-                "Alternatively, init or load a connectable instance on the"
-                " command line: `lamin connect <instance>` or `lamin init <...>`"
-            )
-            raise e
-    else:
-        module_candidates = find_module_candidates()
-        isettings = InstanceSettings(
-            id=UUID("00000000-0000-0000-0000-000000000000"),
-            owner="none",
-            name="none",
-            storage=None,
-            modules=",".join(module_candidates),
-        )
-    CURRENT_ISETTINGS = isettings
-    return isettings
-
-
 def _normalize_module_name(module_name: str) -> str:
     return module_name.replace("lnschema_", "").replace("_", "-")
 
@@ -137,12 +84,9 @@ def _infer_callers_module_name() -> str | None:
 # users should not see it
 def _check_instance_setup(from_module: str | None = None) -> bool:
     if django_lamin.IS_SETUP:
-        # reload logic here because module might not yet have been imported
-        # upon first setup
         if from_module is not None:
             if from_module != "lamindb":
                 _check_module_in_instance_modules(from_module)
-                il.reload(il.import_module(from_module))
         else:
             infer_module = _infer_callers_module_name()
             if infer_module is not None and infer_module not in {
@@ -159,34 +103,29 @@ def _check_instance_setup(from_module: str | None = None) -> bool:
             "errors in regular lamindb usage"
         )
         return True
-    isettings = _get_current_instance_settings()
-    if isettings is not None:
-        if from_module is not None and not django_lamin.IS_SETUP and not IS_LOADING:
-            if from_module != "lamindb":
-                _check_module_in_instance_modules(from_module, isettings)
-
-                import lamindb
-
-                il.reload(il.import_module(from_module))
-            else:
-                django_lamin.setup_django(isettings)
-                if isettings.slug != "none/none":
-                    logger.important(f"connected lamindb: {isettings.slug}")
-                    # update of local storage location through search_local_root()
-                    settings._instance_settings = isettings
-                else:
-                    logger.warning("not connected, call: ln.connect('account/name')")
+
+    if IS_LOADING or from_module is None:
+        return False
+
+    if (
+        not settings._instance_exists
+        and os.environ.get("LAMIN_CURRENT_INSTANCE") is not None
+    ):
+        from ._connect_instance import connect
+
+        connect(_write_settings=False, _reload_lamindb=False)
         return django_lamin.IS_SETUP
     else:
-        if from_module is not None:
-            # the below enables users to auto-connect to an instance
-            # simply by setting an environment variable, bypassing the
-            # need of calling connect() manually
-            if os.environ.get("LAMIN_CURRENT_INSTANCE") is not None:
-                from ._connect_instance import connect
-
-                connect(_write_settings=False, _reload_lamindb=False)
-                return django_lamin.IS_SETUP
-            else:
-                logger.warning(InstanceNotSetupError.default_message)
-        return False
+        isettings = settings.instance
+        if from_module != "lamindb":
+            _check_module_in_instance_modules(from_module, isettings)
+
+            import lamindb  # connect to the instance
+        else:
+            # disable_auto_connect to avoid triggering _check_instance_setup in modules
+            disable_auto_connect(django_lamin.setup_django)(isettings)
+            if isettings.slug != "none/none":
+                logger.important(f"connected lamindb: {isettings.slug}")
+                # update of local storage location through search_local_root()
+                settings._instance_settings = isettings
+    return django_lamin.IS_SETUP

lamindb_setup/_connect_instance.py

@@ -9,10 +9,7 @@ from uuid import UUID
 
 from lamin_utils import logger
 
-from ._check_setup import (
-    _check_instance_setup,
-    _get_current_instance_settings,
-)
+from ._check_setup import _check_instance_setup
 from ._disconnect import disconnect
 from ._init_instance import load_from_isettings
 from ._silence_loggers import silence_loggers
@@ -266,13 +263,8 @@ def validate_connection_state(
     from django.db import connection
 
     if (
-        settings._instance_exists
+        settings._instance_exists  # exists only for real instances, not for none/none
         and f"{owner}/{name}" == settings.instance.slug
-        # below is to ensure that if another process interferes
-        # we don't use the in-memory mock database
-        # could be made more specific by checking whether the django
-        # configured database is the same as the one in settings
-        and connection.settings_dict["NAME"] != ":memory:"
         and not use_root_db_user  # always re-connect for root db user
     ):
         logger.important(
@@ -280,7 +272,7 @@ def validate_connection_state(
         )
         return None
     else:
-        if settings._instance_exists and settings.instance.slug != "none/none":
+        if settings._instance_exists:
             import lamindb as ln
 
             if ln.context.transform is not None:
@@ -292,7 +284,9 @@ def validate_connection_state(
 
 @unlock_cloud_sqlite_upon_exception(ignore_prev_locker=True)
 def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
-    """Connect to an instance.
+    """Connect the global default instance.
+
+    If you want to create a read-only database client, use :class:`~lamindb.DB` instead.
 
     Args:
         instance: Pass a slug (`account/name`) or URL (`https://lamin.ai/account/name`).
@@ -340,12 +334,9 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
             if settings._instance_exists:
                 isettings = settings.instance
             else:
-                isettings_or_none = _get_current_instance_settings()
-                if isettings_or_none is None:
-                    raise ValueError(
-                        "No instance was connected through the CLI, pass a value to `instance` or connect via the CLI."
-                    )
-                isettings = isettings_or_none
+                raise ValueError(
+                    "No instance was connected through the CLI, pass a value to `instance` or connect via the CLI."
+                )
             if use_root_db_user:
                 reset_django()
                 owner, name = isettings.owner, isettings.name
@@ -414,7 +405,6 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
 
         load_from_isettings(isettings, user=_user, write_settings=_write_settings)
         if _reload_lamindb:
-            importlib.reload(importlib.import_module("lamindb"))
             reset_django_module_variables()
         if isettings.slug != "none/none":
             logger.important(f"connected lamindb: {isettings.slug}")
@@ -424,10 +414,6 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
                 isettings._get_settings_file().unlink(missing_ok=True)  # type: ignore
             settings._instance_settings = None
         raise e
-    if settings.dev_dir is None:
-        logger.important_hint(
-            "to map a local dev directory, set: ln.setup.settings.dev_dir = '.'"
-        )
     return None
 
 

lamindb_setup/_delete.py

@@ -11,6 +11,7 @@ from .core._aws_options import HOSTED_BUCKETS
 from .core._hub_core import delete_instance as delete_instance_on_hub
 from .core._hub_core import get_storage_records_for_instance
 from .core._settings import settings
+from .core._settings_load import load_instance_settings
 from .core._settings_storage import StorageSettings
 from .core.upath import LocalPathClasses, check_storage_is_empty
 
@@ -38,6 +39,8 @@ def delete_exclusion_dir(isettings: InstanceSettings) -> None:
 
 
 def delete_by_isettings(isettings: InstanceSettings) -> None:
+    assert isettings.slug != "none/none"
+
     settings_file = isettings._get_settings_file()
     if settings_file.exists():
         settings_file.unlink()
@@ -51,12 +54,14 @@ def delete_by_isettings(isettings: InstanceSettings) -> None:
                 "Did not have permission to delete SQLite file:"
                 f" {isettings._sqlite_file}"
             )
-            pass
     # unset the global instance settings
-    if settings._instance_exists and isettings.slug == settings.instance.slug:
-        if settings._instance_settings_path.exists():
-            settings._instance_settings_path.unlink()
-        settings._instance_settings = None
+    isettings_on_disk = load_instance_settings()
+    if isettings_on_disk.slug == isettings.slug:
+        settings._instance_settings_path.unlink()  # current instance settings file
+        # settings.instance can differ from instance in current_settings_file()
+        # due to connect() in the same process
+        if settings.instance.slug == isettings.slug:
+            settings._instance_settings = None
 
 
 def delete(slug: str, force: bool = False, require_empty: bool = True) -> int | None:

lamindb_setup/_disconnect.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 from lamin_utils import logger
 
 from .core._settings import settings
-from .core._settings_store import current_instance_settings_file
+from .core._settings_load import load_instance_settings
 from .core.cloud_sqlite_locker import clear_locker
 
 
@@ -15,10 +15,11 @@ def disconnect(mute: bool = False) -> None:
     See Also:
         Clear default instance configuration via the CLI, see `here <https://docs.lamin.ai/cli#disconnect>`__.
     """
-    if current_instance_settings_file().exists():
-        instance = settings.instance.slug
+    # settings._instance_exists can be true due to connect even without having a file
+    if settings._instance_exists:
+        instance = settings.instance
         try:
-            settings.instance._update_cloud_sqlite_file()
+            instance._update_cloud_sqlite_file()
         except Exception as e:
             if isinstance(e, FileNotFoundError):
                 logger.warning("did not find local cache file")
@@ -26,10 +27,12 @@ def disconnect(mute: bool = False) -> None:
                 logger.warning("did not upload cache file - not enough permissions")
             else:
                 raise e
-        current_instance_settings_file().unlink()
         clear_locker()
+        # instance in current instance file can differ from instance in settings
+        if load_instance_settings().slug == instance.slug:
+            settings._instance_settings_path.unlink(missing_ok=True)
+        settings._instance_settings = None
         if not mute:
-            logger.success(f"disconnected instance: {instance}")
-    else:
-        if not mute:
-            logger.info("no instance loaded")
+            logger.success(f"disconnected instance: {instance.slug}")
+    elif not mute:
+        logger.info("no instance loaded")

lamindb_setup/_init_instance.py

@@ -340,7 +340,6 @@ def init(
             from ._schema_metadata import update_schema_in_hub
 
             update_schema_in_hub(access_token=access_token)
-        importlib.reload(importlib.import_module("lamindb"))
         reset_django_module_variables()
         logger.important(f"initialized lamindb: {isettings.slug}")
     except Exception as e:

lamindb_setup/_migrate.py

@@ -125,24 +125,10 @@ class migrate:
         from lamindb_setup._schema_metadata import update_schema_in_hub
         from lamindb_setup.core._hub_client import call_with_fallback_auth
         from lamindb_setup.core._hub_crud import (
-            select_collaborator,
             update_instance,
         )
 
         if settings.instance.is_on_hub:
-            # double check that user is an admin, otherwise will fail below
-            # due to insufficient SQL permissions with cryptic error
-            collaborator = call_with_fallback_auth(
-                select_collaborator,
-                instance_id=settings.instance._id,
-                account_id=settings.user._uuid,
-                fine_grained_access=settings.instance._fine_grained_access,
-            )
-            if collaborator is None or collaborator["role"] != "admin":
-                raise SystemExit(
-                    "❌ Only admins can deploy migrations, please ensure that you're an"
-                    f" admin: https://lamin.ai/{settings.instance.slug}/settings"
-                )
             # ensure we connect with the root user
             if "root" not in settings.instance.db:
                 connect(use_root_db_user=True)

lamindb_setup/_setup_user.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+from time import sleep
 from typing import TYPE_CHECKING
 
 from lamin_utils import logger
@@ -59,15 +60,14 @@ def login(
 
     `login()` prompts for your API key unless you set it via the `LAMIN_API_KEY` environment variable or pass it as an argument.
 
-    You can create your API key in your account settings on LaminHub (top right corner).
+    You can create your API key in your account settings on LaminHub at `lamin.ai/settings <https://lamin.ai/settings>`__.
+
+    Note that the preferred method is to use the CLI command `lamin login`: `docs.lamin.ai/cli#login <https://docs.lamin.ai/cli#login>`__.
 
     Args:
         user: User handle.
         api_key: API key.
 
-    See Also:
-        Login via the CLI command `lamin login`, see `here <https://docs.lamin.ai/cli#login>`__.
-
     Examples:
 
         Logging in the first time::
@@ -80,12 +80,15 @@ def login(
 
             ln.setup.login("myhandle")  # pass your user handle
     """
+    from getpass import getpass
+
     if user is None:
         if api_key is None:
             if "LAMIN_API_KEY" in os.environ:
                 api_key = os.environ["LAMIN_API_KEY"]
             else:
-                api_key = input("Your API key: ")
+                print("Copy your API key. To create one: https://lamin.ai/settings")
+                api_key = getpass("Paste it: ")
     elif api_key is not None:
         raise ValueError("Please provide either 'user' or 'api_key', not both.")
 

lamindb_setup/_silence_loggers.py

@@ -25,6 +25,8 @@ def silence_loggers():
         set_stream_logger(name="botocore.auth", level=logging.WARNING)
         set_stream_logger(name="botocore.endpoint", level=logging.WARNING)
         set_stream_logger(name="httpx", level=logging.WARNING)
+        set_stream_logger(name="httpcore", level=logging.WARNING)
+        set_stream_logger(name="hpack", level=logging.WARNING)
         try:
             import aiobotocore
 

lamindb_setup/core/_aws_options.py

@@ -60,6 +60,7 @@ class AWSOptionsManager:
         from aiobotocore.session import AioSession
         from s3fs import S3FileSystem
 
+        anon_env = os.getenv("LAMIN_S3_ANON") == "true"
         # this is cached so will be resued with the connection initialized
         # these options are set for paths in _path_inject_options
         # here we set the same options to cache the filesystem
@@ -68,19 +69,28 @@ class AWSOptionsManager:
             use_listings_cache=True,
             version_aware=False,
             config_kwargs={"max_pool_connections": 64},
+            anon=anon_env,
         )
 
         self._suppress_aiobotocore_traceback_logging()
 
-        try:
-            fs.connect()
-            self.anon: bool = fs.session._credentials is None
-        except Exception as e:
+        if anon_env:
+            self.anon: bool = True
             logger.warning(
-                f"There is a problem with your default AWS Credentials: {e}\n"
-                "`anon` mode will be used for all non-managed buckets."
+                "`anon` mode will be used for all non-managed buckets "
+                "because the environment variable LAMIN_S3_ANON was set to 'true'"
             )
-            self.anon = True
+        else:
+            try:
+                fs.connect()
+                self.anon = fs.session._credentials is None
+            except Exception as e:
+                logger.warning(
+                    f"There is a problem with your default AWS Credentials: {e}\n"
+                    "`anon` mode will be used for all non-managed buckets"
+                )
+                self.anon = True
+
         self.anon_public: bool | None = None
         if not self.anon:
             try:

lamindb_setup/core/_hub_client.py

@@ -11,8 +11,7 @@ import httpx
 from httpx_retries import Retry, RetryTransport
 from lamin_utils import logger
 from pydantic_settings import BaseSettings
-from supabase import Client, create_client  # type: ignore
-from supabase.lib.client_options import ClientOptions
+from supabase import Client, ClientOptions, create_client
 
 from ._settings_save import save_user_settings
 

lamindb_setup/core/_hub_core.py

@@ -383,7 +383,8 @@ def _init_instance_hub(
 ) -> None:
     from ._settings import settings
 
-    account_id = settings.user._uuid if account_id is None else account_id
+    created_by_id = settings.user._uuid.hex if account_id is None else account_id.hex  # type: ignore
+    owner_account_id = os.getenv("LAMINDB_ACCOUNT_ID_INIT", created_by_id)
 
     try:
         lamindb_version = metadata.version("lamindb")
@@ -391,13 +392,13 @@ def _init_instance_hub(
         lamindb_version = None
     fields = {
         "id": isettings._id.hex,
-        "account_id": account_id.hex,  # type: ignore
+        "account_id": owner_account_id,
         "name": isettings.name,
         "lnid": isettings.uid,
         "schema_str": isettings._schema_str,
         "lamindb_version": lamindb_version,
         "public": False,
-        "created_by_id": account_id.hex,  # type: ignore
+        "created_by_id": created_by_id,
     }
     if isettings.dialect != "sqlite":
         db_dsn = LaminDsnModel(db=isettings.db)
@@ -407,7 +408,7 @@ def _init_instance_hub(
             "db_port": db_dsn.db.port,
             "db_database": db_dsn.db.database,
         }
-        fields.update(db_fields)
+        fields.update(db_fields)  # type: ignore
     slug = isettings.slug
     # I'd like the following to be an upsert, but this seems to violate RLS
     # Similarly, if we don't specify `returning="minimal"`, we'll violate RLS
@@ -415,7 +416,9 @@ def _init_instance_hub(
     # as then init_instance is no longer idempotent
     try:
         client.table("instance").insert(fields, returning="minimal").execute()
-    except APIError:
+    except APIError as e:
+        if "new row violates row-level security policy" in str(e):
+            raise e
         logger.warning(f"instance already existed at: https://lamin.ai/{slug}")
         return None
     if isettings.dialect != "sqlite" and isettings.is_remote:
@@ -713,7 +716,7 @@ def get_lamin_site_base_url():
 
 
 def sign_up_local_hub(email) -> str | tuple[str, str, str]:
-    # raises gotrue.errors.AuthApiError: User already registered
+    # raises AuthApiError: User already registered
     password = base62(40)  # generate new password
     sign_up_kwargs = {"email": email, "password": password}
     client = connect_hub()

lamindb_setup/core/_settings.py

@@ -46,6 +46,12 @@ def _process_cache_path(cache_path: UPathStr | None) -> UPath | None:
     return cache_dir
 
 
+# returned by settings.branch for none/none instance
+class MainBranchMock:
+    id = 1
+    name = "main"
+
+
 class SetupSettings:
     """Setup settings."""
 
@@ -140,6 +146,10 @@ class SetupSettings:
     # and we never need a DB request
     def branch(self) -> Branch:
         """Default branch."""
+        # this is needed for .filter() with non-default connections
+        if not self._instance_exists:
+            return MainBranchMock()
+
         if self._branch is None:
             from lamindb import Branch
 
@@ -222,10 +232,9 @@ class SetupSettings:
         If `True`, the current instance is connected, meaning that the db and other settings
         are properly configured for use.
         """
-        if self._instance_exists:
-            return self.instance.slug != "none/none"
-        else:
-            return False
+        from . import django
+
+        return self._instance_exists and django.IS_SETUP
 
     @property
     def private_django_api(self) -> bool:
@@ -284,12 +293,7 @@ class SetupSettings:
 
     @property
     def _instance_exists(self):
-        try:
-            self.instance  # noqa
-            return True
-        # this is implicit logic that catches if no instance is loaded
-        except CurrentInstanceNotConfigured:
-            return False
+        return self.instance.slug != "none/none"
 
     @property
     def cache_dir(self) -> UPath:

lamindb_setup/core/_settings_instance.py

@@ -125,10 +125,11 @@ class InstanceSettings:
                 if self._local_storage is not None:
                     value_local = self.local_storage
                     representation += f"\n - local storage: {value_local.root_as_str} ({value_local.region})"
-                    representation += (
-                        f"\n - cloud storage: {value.root_as_str} ({value.region})"
-                    )
-                else:
+                    if value is not None:
+                        representation += (
+                            f"\n - cloud storage: {value.root_as_str} ({value.region})"
+                        )
+                elif value is not None:
                     representation += (
                         f"\n - storage: {value.root_as_str} ({value.region})"
                     )
@@ -513,17 +514,36 @@ class InstanceSettings:
 
     @property
     def dialect(self) -> Literal["sqlite", "postgresql"]:
-        """SQL dialect."""
+        """SQL dialect.
+
+        Equivalent to :attr:`vendor`.
+
+        "vendor" is the Django terminology for the type of database. "dialect" is the SQLAlchemy terminology.
+        """
         if self._db is None or self._db.startswith("sqlite://"):
             return "sqlite"
         else:
             assert self._db.startswith("postgresql"), f"Unexpected DB value: {self._db}"
             return "postgresql"
 
+    @property
+    def vendor(self) -> Literal["sqlite", "postgresql"]:
+        """Database vendor.
+
+        Equivalent to :attr:`dialect`.
+
+        "vendor" is the Django terminology for the type of database. "dialect" is the SQLAlchemy terminology.
+        """
+        return self.dialect
+
     @property
     def _is_cloud_sqlite(self) -> bool:
         """Is this a cloud instance with sqlite db."""
-        return self.dialect == "sqlite" and self.storage.type_is_cloud
+        return (
+            self.dialect == "sqlite"
+            and self.storage is not None
+            and self.storage.type_is_cloud
+        )
 
     @property
     def _cloud_sqlite_locker(self):
@@ -543,6 +563,8 @@ class InstanceSettings:
     @property
     def is_remote(self) -> bool:
         """Boolean indicating if an instance has no local component."""
+        if self.storage is None and self.db == "sqlite:///:memory:":
+            return False
         return check_is_instance_remote(self.storage.root_as_str, self.db)
 
     @property

lamindb_setup/core/_settings_load.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+from importlib.util import find_spec
 from pathlib import Path
 from typing import TYPE_CHECKING
 from uuid import UUID, uuid4
@@ -46,19 +47,36 @@ def load_cache_path_from_settings(storage_settings: Path | None = None) -> Path
         return None
 
 
+def find_module_candidates():
+    """Find all local packages that depend on lamindb."""
+    candidates = ["bionty", "wetlab"]
+    return [c for c in candidates if find_spec(c) is not None]
+
+
 def load_instance_settings(instance_settings_file: Path | None = None):
     if instance_settings_file is None:
-        instance_settings_file = current_instance_settings_file()
-    if not instance_settings_file.exists():
+        isettings_file = current_instance_settings_file()
+        if not isettings_file.exists():
+            isettings = InstanceSettings(
+                id=UUID("00000000-0000-0000-0000-000000000000"),
+                owner="none",
+                name="none",
+                storage=None,
+                modules=",".join(find_module_candidates()),
+            )
+            return isettings
+    else:
+        isettings_file = instance_settings_file
+
+    if not isettings_file.exists():
+        # this errors only if the file was explicitly provided
         raise CurrentInstanceNotConfigured
     try:
-        settings_store = InstanceSettingsStore(_env_file=instance_settings_file)
+        settings_store = InstanceSettingsStore(_env_file=isettings_file)
     except (ValidationError, TypeError) as error:
-        with open(instance_settings_file) as f:
-            content = f.read()
         raise SettingsEnvFileOutdated(
-            f"\n\n{error}\n\nYour instance settings file with\n\n{content}\nis invalid"
-            f" (likely outdated), see validation error. Please delete {instance_settings_file} &"
+            f"\n\n{error}\n\nYour instance settings file with\n\n{isettings_file.read_text()}\nis invalid"
+            f" (likely outdated), see validation error. Please delete {isettings_file} &"
             " reload (remote) or re-initialize (local) the instance with the same name & storage location."
         ) from error
     isettings = setup_instance_from_store(settings_store)

lamindb_setup/core/django.py

@@ -5,13 +5,15 @@ import builtins
 import os
 import sys
 import importlib as il
+import gzip
 import jwt
 import time
 import threading
 from pathlib import Path
+import shutil
 from packaging import version
 from ._settings_instance import InstanceSettings, is_local_db_url
-
+from ..errors import CurrentInstanceNotConfigured
 from lamin_utils import logger
 
 
@@ -21,6 +23,24 @@ IS_MIGRATING = False
 CONN_MAX_AGE = 299
 
 
+def get_connection(connection_name: str):
+    from django.db import connections
+
+    return connections[connection_name]
+
+
+def error_no_instance_wrapper(execute, sql, params, many, context):
+    connection = context["connection"]
+
+    if (
+        connection.vendor == "sqlite"
+        and connection.settings_dict.get("NAME") == ":memory:"
+    ):
+        raise CurrentInstanceNotConfigured
+
+    return execute(sql, params, many, context)
+
+
 # db token that refreshes on access if needed
 class DBToken:
     def __init__(
@@ -64,11 +84,6 @@ class DBTokenManager:
 
         self.tokens: dict[str, DBToken] = {}
 
-    def get_connection(self, connection_name: str):
-        from django.db import connections
-
-        return connections[connection_name]
-
     def set(self, token: DBToken, connection_name: str = "default"):
         if connection_name in self.tokens:
             return
@@ -77,11 +92,7 @@ class DBTokenManager:
         from django.db.backends.signals import connection_created
 
         def set_token_wrapper(execute, sql, params, many, context):
-            not_in_atomic_block = (
-                context is None
-                or "connection" not in context
-                or not context["connection"].in_atomic_block
-            )
+            not_in_atomic_block = not context["connection"].in_atomic_block
             # ignore atomic blocks
             if not_in_atomic_block:
                 sql = token.token_query + sql
@@ -98,7 +109,7 @@ class DBTokenManager:
                 result.nextset()
             return result
 
-        self.get_connection(connection_name).execute_wrappers.append(set_token_wrapper)
+        get_connection(connection_name).execute_wrappers.append(set_token_wrapper)
 
         def connection_callback(sender, connection, **kwargs):
             if (
@@ -124,7 +135,7 @@ class DBTokenManager:
                 if connection_name in self.tokens:
                     # here we don't use the connection from the closure
                     # because Atomic is a single class to manage transactions for all connections
-                    connection = self.get_connection(connection_name)
+                    connection = get_connection(connection_name)
                     if len(connection.atomic_blocks) == 1:
                         token = self.tokens[connection_name]
                         # use raw psycopg2 connection here
@@ -142,7 +153,7 @@ class DBTokenManager:
 
         from django.db.backends.signals import connection_created
 
-        connection = self.get_connection(connection_name)
+        connection = get_connection(connection_name)
 
         connection.execute_wrappers = [
             w
@@ -291,6 +302,9 @@ def setup_django(
             django.db.connections._connections = threading.local()
             logger.debug("django.db.connections._connections has been patched")
 
+        # error if trying to query with the default connection without setting up an instance
+        get_connection("default").execute_wrappers.insert(0, error_no_instance_wrapper)
+
         if isettings._fine_grained_access and isettings._db_permissions == "jwt":
             db_token = DBToken(isettings)
             db_token_manager.set(db_token)  # sets for the default connection
@@ -311,10 +325,24 @@ def setup_django(
             call_command("migrate", app_name, app_number, verbosity=2)
         isettings._update_cloud_sqlite_file(unlock_cloud_sqlite=False)
     elif init:
-        global IS_MIGRATING
-        IS_MIGRATING = True
-        call_command("migrate", verbosity=0)
-        IS_MIGRATING = False
+        modules_beyond_bionty = isettings.modules.copy()
+        compressed_sqlite_path = Path(__file__).parent / "lamin.db.gz"
+        if "bionty" in modules_beyond_bionty:
+            modules_beyond_bionty.remove("bionty")
+        if (
+            isettings.dialect == "postgresql"
+            or os.getenv("LAMINDB_INIT_FROM_SCRATCH", "false") == "true"
+            or len(modules_beyond_bionty) > 0
+            or not compressed_sqlite_path.exists()
+        ):
+            global IS_MIGRATING
+            IS_MIGRATING = True
+            call_command("migrate", verbosity=0)
+            IS_MIGRATING = False
+        else:
+            with gzip.open(compressed_sqlite_path, "rb") as f_in:
+                with open(isettings._sqlite_file_local, "wb") as f_out:
+                    shutil.copyfileobj(f_in, f_out)
 
     global IS_SETUP
     IS_SETUP = True

lamindb_setup/core/upath.py

@@ -1005,13 +1005,22 @@ def check_storage_is_empty(
         objects = [o for o in objects if "/.lamindb/_exclusion/" not in o]
     n_files = len(objects)
     n_diff = n_files - n_offset_objects
-    ask_for_deletion = (
-        "delete them prior to deleting the storage location"
-        if raise_error
-        else "consider deleting them"
-    )
-    message = f"'{directory_string}' contains {n_diff} objects - {ask_for_deletion}"
     if n_diff > 0:
+        ask_for_deletion = (
+            "delete them prior to deleting the storage location"
+            if raise_error
+            else "consider deleting them"
+        )
+        message = f"'{directory_string}' contains {n_diff} objects:\n"
+        message += "\n".join(
+            [
+                o
+                for o in objects
+                if not o.endswith(".lamindb/storage_uid.txt")
+                and not (account_for_sqlite_file and o.endswith(".lamindb/lamin.db"))
+            ]
+        )
+        message += f"\n{ask_for_deletion}"
         if raise_error:
             raise StorageNotEmpty(message) from None
         else:

lamindb_setup/errors.py

@@ -1,7 +1,6 @@
 """Errors.
 
 .. autoexception:: CurrentInstanceNotConfigured
-.. autoexception:: InstanceNotSetupError
 .. autoexception:: ModuleWasntConfigured
 .. autoexception:: StorageAlreadyManaged
 .. autoexception:: StorageNotEmpty
@@ -25,17 +24,6 @@ class DefaultMessageException(Exception):
         super().__init__(message)
 
 
-# TODO: remove this exception sooner or later because we don't have a need for it anymore
-class InstanceNotSetupError(DefaultMessageException):
-    default_message = """\
-To use lamindb, you need to connect to an instance.
-
-Connect to an instance: `ln.connect()`. Init an instance: `ln.setup.init()`.
-
-If you used the CLI to set up lamindb in a notebook, restart the Python session.
-"""
-
-
 class CurrentInstanceNotConfigured(DefaultMessageException):
     default_message = """\
 No instance is connected! Call

lamindb_setup/io.py

@@ -14,7 +14,7 @@ from django.db import models, transaction
 from rich.progress import Progress
 
 if TYPE_CHECKING:
-    from collections.abc import Iterable, Sequence
+    from collections.abc import Iterable
     from typing import Literal
 
 
@@ -22,6 +22,9 @@ def _get_registries(module_name: str) -> list[str]:
     """Get registry class names from a module."""
     schema_module = import_module(module_name)
 
+    # Ensure that models are loaded; we've observed empty exports otherwise
+    from django.db import models
+
     return [
         name
         for name in dir(schema_module.models)
@@ -142,9 +145,9 @@ def _export_full_table(
 
 
 def export_db(
-    module_names: Sequence[str] | None = None,
+    module_names: Iterable[str] | None = None,
     *,
-    output_dir: str | Path = "./lamindb_export/",
+    output_dir: str | Path | None = None,
     max_workers: int = 8,
     chunk_size: int = 500_000,
 ) -> None:
@@ -159,6 +162,11 @@ def export_db(
         max_workers: Number of parallel processes.
         chunk_size: Number of rows per chunk for large tables.
     """
+    import lamindb_setup as ln_setup
+
+    if output_dir is None:
+        output_dir = f"./{ln_setup.settings.instance.name}_export/"
+
     directory = Path(output_dir)
     directory.mkdir(parents=True, exist_ok=True)
 
@@ -332,12 +340,15 @@ def import_db(
     Temporarily disables FK constraints to allow insertion in arbitrary order.
     Requires superuser/RDS admin privileges for postgres databases.
 
+    Note: When running in a subprocess, add a short delay or explicit connection close after `import_db()`
+    to ensure all SQLite writes are flushed to disk before process termination.
+
     Args:
         input_dir: Directory containing parquet files to import.
         module_names: Module names to import (e.g., ["lamindb", "bionty", "wetlab"]).
         if_exists: How to behave if table exists: 'fail', 'replace', or 'append'.
-            If set to 'replace', existing data is deleted and new data is imported. PKs and indices are not guaranteed to be preserved which can lead to write errors.
-            If set to 'append', new data is added to existing data without clearing the table. PKs and indices are preserved but database size will greatly increase.
+            If set to 'replace', existing data is deleted and new data is imported. All PKs and indices are not guaranteed to be preserved which can lead to write errors.
+            If set to 'append', new data is added to existing data without clearing the table. All PKs and indices are preserved allowing write operations but database size will greatly increase.
             If set to 'fail', raises an error if the table contains any data.
     """
     from django.db import connection

{lamindb_setup-1.16.0.dist-info → lamindb_setup-1.17.0.dist-info}/METADATA

@@ -1,10 +1,11 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: lamindb_setup
-Version: 1.16.0
+Version: 1.17.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
+License-File: LICENSE
 Requires-Dist: lamin_utils>=0.3.3
 Requires-Dist: django>=5.2,<5.3
 Requires-Dist: dj_database_url>=1.3.0,<3.0.0
@@ -15,9 +16,7 @@ Requires-Dist: httpx_retries<1.0.0
 Requires-Dist: requests
 Requires-Dist: universal_pathlib==0.2.6
 Requires-Dist: botocore<2.0.0
-Requires-Dist: supabase>=2.8.1,<=2.16.0
-Requires-Dist: gotrue<=2.12.0
-Requires-Dist: storage3!=0.11.2; python_version < '3.11'
+Requires-Dist: supabase>=2.20.0,<=2.24.0
 Requires-Dist: pyjwt<3.0.0
 Requires-Dist: psutil
 Requires-Dist: packaging

{lamindb_setup-1.16.0.dist-info → lamindb_setup-1.17.0.dist-info}/RECORD

@@ -1,50 +1,51 @@
-lamindb_setup/__init__.py,sha256=zAgDC8o3cyH4eXcM60mbulFPECs5XZbqFRxLfGpgpc8,3215
+lamindb_setup/__init__.py,sha256=MW5TlycTeavLeMWfllbDgMo2z4r7zlNHHvMLEaJRnGE,3270
 lamindb_setup/_cache.py,sha256=pGvDNVHGx4HWr_6w5ajqEJOdysmaGc6F221qFnXkT-k,2747
 lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
-lamindb_setup/_check_setup.py,sha256=ToKMxsUq8dQBQh8baOrNVlSb1iC8h4zTg5dV8wMu0W4,6760
-lamindb_setup/_connect_instance.py,sha256=3dsaZ7LGzJYtOWDbi2RkiVJIJqdxy43suNjQ-6C96_U,17788
-lamindb_setup/_delete.py,sha256=KS3r-xGFuDmAbzPUy-9JR-YnPShYdaHjDRQrAmXQ0qM,5863
-lamindb_setup/_disconnect.py,sha256=FT8EpCm5XXDdhDH7QtAnkO3KPatq2HqT9VXGNjgJDbk,1232
+lamindb_setup/_check_setup.py,sha256=zPQho12dctJYjnZxpumq2r7XRvXwrNyWi5-UMRPSeuE,4297
+lamindb_setup/_connect_instance.py,sha256=AVJ5wDGsvPO1HAWw2KIz0R2hF0c9DMLJ_2kFsXJfwd4,17156
+lamindb_setup/_delete.py,sha256=ZMAik35NNcj4kDfOISDfzpicCi4j8gwmWAor0hEB-Jk,6123
+lamindb_setup/_disconnect.py,sha256=B0K0yTIIyhwc3MzDFL_-cKutm-_DHsiJSTrkp1MMyXA,1470
 lamindb_setup/_django.py,sha256=uIQflpkp8l3axyPaKURlk3kacgpElVP5KOKmFxYSMGk,1454
 lamindb_setup/_entry_points.py,sha256=sKwXPX9xjOotoAjvgkU5LBwjjHLWVkh0ZGdiSsrch9k,522
-lamindb_setup/_init_instance.py,sha256=zNXmZPUHYda1CfLGtsvo4gNhHprK9QVPfffUIfBlTxE,14938
-lamindb_setup/_migrate.py,sha256=SN8uphuQX-8XShH5odLyzV8-eyXATDxB5hWoxwxmgBU,11264
+lamindb_setup/_init_instance.py,sha256=EBdv0ga_nnqHU-m9fN7EkVg7YEuKOLxR8N87Ww2qEDg,14877
+lamindb_setup/_migrate.py,sha256=B7dRb4hIkuHlRxmI25N7xkdQve-fNZDZoq4GsP3TtH0,10512
 lamindb_setup/_register_instance.py,sha256=RdUZxZWHLdbvdNZWpF8e0UWROb_T0cStWbzc5yUw34I,1047
 lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
 lamindb_setup/_schema_metadata.py,sha256=Whs-e4ZMnA1niZ2l5Eu8il-33IxI4Hr5ylGEgPxx8wk,15628
 lamindb_setup/_set_managed_storage.py,sha256=xQe5DXCRiQ5VseAjVC2Bki0wB0n0tSTchvVKSx9I6eo,3094
-lamindb_setup/_setup_user.py,sha256=cjQ-Md-FkP04PnBxocbHW6wCsZsNtD2T2NB52vAOnHI,6730
-lamindb_setup/_silence_loggers.py,sha256=AKF_YcHvX32eGXdsYK8MJlxEaZ-Uo2f6QDRzjKFCtws,1568
-lamindb_setup/errors.py,sha256=lccF3X3M2mcbHVG_0HxfuJRFFpUE-42paccIxFOfefQ,1958
-lamindb_setup/io.py,sha256=9s4Itt4rrHzsUATY79r4nhGp9zVAm-9uBhiQgg60l6U,16708
+lamindb_setup/_setup_user.py,sha256=9CwKUru2jza1vXu4A5V-q-oXnbIXwR_YwjKbVnETK7Q,6931
+lamindb_setup/_silence_loggers.py,sha256=oQnvnvieqxARQQMwQkR82EJVk1TaM-Bo9o0UCOsCcXY,1697
+lamindb_setup/errors.py,sha256=hELv-tJSjyPDqT9pZ_-tmg6q_wXDtAgn8m1fIvFeMtg,1528
+lamindb_setup/io.py,sha256=stTE6VHCIpLK1ueRuWEJGh35wo9Qd1NcNN1TnLEbK-U,17156
 lamindb_setup/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lamindb_setup/types.py,sha256=fuQxZJnrGYe7a_Ju9n1RqO-HhkOAr1l1xjpAg9dmBu8,605
 lamindb_setup/core/__init__.py,sha256=adZtacDwG2C0tgx-ypp9yOAqw9qaR-IRWkgLurKpXVE,668
-lamindb_setup/core/_aws_options.py,sha256=9kQ5BB-cuJQrlJRGNqMRe1m48dP67xMbefOJP2c9OQw,9674
+lamindb_setup/core/_aws_options.py,sha256=5XbcPtRU_oVrfbuG_0kngyYfxswLLnRS3nKO4XEWepU,10054
 lamindb_setup/core/_aws_storage.py,sha256=QEtV-riQrwfivcwqHnXBbkJ-9YyNEXL4fLoCmOHZ1BI,2003
 lamindb_setup/core/_clone.py,sha256=oLTMItGxRQB1THSDP-RG2eH1qPTVcuZ7_-eElttJ518,6451
 lamindb_setup/core/_deprecated.py,sha256=M3vpM4fZPOncxY2qsXQAPeaEph28xWdv7tYaueaUyAA,2554
 lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
-lamindb_setup/core/_hub_client.py,sha256=vem145S5ppRPcWob7iclGhos8k-BfwJi9AI-l5PteDs,10481
-lamindb_setup/core/_hub_core.py,sha256=GAQK5XkHROIuqA-H8sOQZVlxvN4QIH_cmHY0TENnq2U,29090
+lamindb_setup/core/_hub_client.py,sha256=113SKT3Ki4KxyKqpYOwIB8mEG0ouy9oebLJoh_xGJZY,10426
+lamindb_setup/core/_hub_core.py,sha256=29iPGnmV1hzYIMrErgwJoHwTQ5RHNJ6-icDM81aC4ac,29255
 lamindb_setup/core/_hub_crud.py,sha256=j6516H82kLjFUNPqFGUINbDw9YbofMgjxadGzYb0OS4,6362
 lamindb_setup/core/_hub_utils.py,sha256=6dyDGyzYFgVfR_lE3VN3CP1jGp98gxPtr-T91PAP05U,2687
 lamindb_setup/core/_private_django_api.py,sha256=Z9uGL4CK0OX58rc8R_qarg9rIBp1DgjsjfP9Vj2vJHI,2629
-lamindb_setup/core/_settings.py,sha256=QbTrSkkdx0u685NJ4neNtWzhdHoaGMKcIvrfFnctTQ4,15450
-lamindb_setup/core/_settings_instance.py,sha256=eDkueLK5JZOGFhZRbGa-OffS9iBFlxMp47vF_MfmCYI,24301
-lamindb_setup/core/_settings_load.py,sha256=NQDOln8e3qyGphk8ucU7mm3HVkCv4QV4rDZro3TIwfo,5183
+lamindb_setup/core/_settings.py,sha256=LTeh2jsM01GHW2ZTrcc6rXNz6qIUT0VPEzpE9I8tg3Q,15497
+lamindb_setup/core/_settings_instance.py,sha256=EkKcT5TcGLIiMQdiTNCrf8OPp0wmj1ZmY_9kL7ovhZc,24994
+lamindb_setup/core/_settings_load.py,sha256=D6r0fiqJznHhTN_apGlrKfYzl0g21shIaV4z4zEr7Z8,5780
 lamindb_setup/core/_settings_save.py,sha256=96mWdYLyfvbnG_ok_vK4x7jm-rtqcWCD1OHEt2QSAms,3328
 lamindb_setup/core/_settings_storage.py,sha256=22EBagIr5qOZr9pqVkJsTcQtgE14en-Wh0y9rgF4FEQ,15677
 lamindb_setup/core/_settings_store.py,sha256=auZssUBb6qE5oSqdGiHhqI2B46qSpegX89VwObPQksk,2601
 lamindb_setup/core/_settings_user.py,sha256=gFfyMf-738onbh1Mf4wsmLlenQJPtjQfpUgKnOlqc2o,1453
 lamindb_setup/core/_setup_bionty_sources.py,sha256=ox3X-SHiHa2lNPSWjwZhINypbLacX6kGwH6hVVrSFZc,1505
 lamindb_setup/core/cloud_sqlite_locker.py,sha256=H_CTUCjURFXwD1cCtV_Jn0_60iztZTkaesLLXIBgIxc,7204
-lamindb_setup/core/django.py,sha256=aBdIN07ZCD8PGT04sjF1rruwlppx0cEobBnjk2Due70,12525
+lamindb_setup/core/django.py,sha256=JTlGZ2I9EGTr49NjLMTHY1FfjkOoqcoYNgMQitQRGyQ,13640
 lamindb_setup/core/exceptions.py,sha256=qjMzqy_uzPA7mCOdnoWnS_fdA6OWbdZGftz-YYplrY0,84
 lamindb_setup/core/hashing.py,sha256=Y8Uc5uSGTfU6L2R_gb5w8DdHhGRog7RnkK-e9FEMjPY,3680
+lamindb_setup/core/lamin.db.gz,sha256=cSWQ_ak6QN-SMyFA4x6zv-e2rNXr1IOrhDU5AfVAADc,81394
 lamindb_setup/core/types.py,sha256=T7NwspfRHgIIpYsXDcApks8jkOlGeGRW-YbVLB7jNIo,67
-lamindb_setup/core/upath.py,sha256=_xs6CgqQezOe6h8oQURjpOl1WT_1ctROzH3yzesVceE,36188
-lamindb_setup-1.16.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
-lamindb_setup-1.16.0.dist-info/WHEEL,sha256=CpUCUxeHQbRN5UGRQHYRJorO5Af-Qy_fHMctcQ8DSGI,82
-lamindb_setup-1.16.0.dist-info/METADATA,sha256=mkCWug_dpZRqU6Y0ajukECuIC8kezhh4bTYAIuDLAao,1830
-lamindb_setup-1.16.0.dist-info/RECORD,,
+lamindb_setup/core/upath.py,sha256=zlPZcDRHDYT7OcoOiX_w9koAzfBr9q8hB66OPwhTpBo,36504
+lamindb_setup-1.17.0.dist-info/licenses/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
+lamindb_setup-1.17.0.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
+lamindb_setup-1.17.0.dist-info/METADATA,sha256=Q8iSdBECnZFm2PKkraif0zjcwm5CqSem3NW0UdCPBHo,1766
+lamindb_setup-1.17.0.dist-info/RECORD,,

{lamindb_setup-1.16.0.dist-info → lamindb_setup-1.17.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: flit 3.10.1
+Generator: flit 3.12.0
 Root-Is-Purelib: true
 Tag: py3-none-any