lamindb_setup 1.9.0__py3-none-any.whl → 1.10.0__py3-none-any.whl

lamindb_setup/__init__.py

@@ -35,7 +35,7 @@ Modules & settings:
 
 """
 
-__version__ = "1.9.0"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.10.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 

lamindb_setup/_cache.py

@@ -8,6 +8,7 @@ from lamin_utils import logger
 
 from .core._settings_save import save_platform_user_storage_settings
 from .core._settings_store import system_settings_file
+from .errors import CurrentInstanceNotConfigured
 
 
 def clear_cache_dir():
@@ -19,9 +20,8 @@ def clear_cache_dir():
                 "disconnecting the current instance to update the cloud sqlite database."
             )
             disconnect()
-    except SystemExit as e:
-        if str(e) != "No instance connected! Call `lamin connect` or `lamin init`":
-            raise e
+    except CurrentInstanceNotConfigured:
+        pass
 
     cache_dir = settings.cache_dir
     if cache_dir.exists():

lamindb_setup/_check_setup.py

@@ -4,7 +4,9 @@ import functools
 import importlib as il
 import inspect
 import os
+from importlib.metadata import distributions
 from typing import TYPE_CHECKING
+from uuid import UUID
 
 from lamin_utils import logger
 
@@ -25,6 +27,7 @@ if TYPE_CHECKING:
 
 
 CURRENT_ISETTINGS: InstanceSettings | None = None
+MODULE_CANDIDATES: set[str] | None = None
 IS_LOADING: bool = False
 
 
@@ -42,7 +45,25 @@ def disable_auto_connect(func: Callable):
     return wrapper
 
 
-def _get_current_instance_settings() -> InstanceSettings | None:
+def find_module_candidates():
+    """Find all local packages that depend on lamindb."""
+    global MODULE_CANDIDATES
+    if MODULE_CANDIDATES is not None:
+        return MODULE_CANDIDATES
+    all_dists = list(distributions())
+    lamindb_deps = {
+        dist.metadata["Name"].lower()
+        for dist in all_dists
+        if dist.requires and any("lamindb" in req.lower() for req in dist.requires)
+    }
+    lamindb_deps.remove("lamindb")
+    MODULE_CANDIDATES = lamindb_deps
+    return lamindb_deps
+
+
+def _get_current_instance_settings(from_module: str | None = None) -> InstanceSettings:
+    from .core._settings_instance import InstanceSettings
+
     global CURRENT_ISETTINGS
 
     if CURRENT_ISETTINGS is not None:
@@ -60,9 +81,17 @@ def _get_current_instance_settings() -> InstanceSettings | None:
                 " command line: `lamin connect <instance>` or `lamin init <...>`"
             )
             raise e
-        return isettings
     else:
-        return None
+        module_candidates = find_module_candidates()
+        isettings = InstanceSettings(
+            id=UUID("00000000-0000-0000-0000-000000000000"),
+            owner="none",
+            name="none",
+            storage=None,
+            modules=",".join(module_candidates),
+        )
+    CURRENT_ISETTINGS = isettings
+    return isettings
 
 
 def _normalize_module_name(module_name: str) -> str:
@@ -132,12 +161,7 @@ def _check_instance_setup(from_module: str | None = None) -> bool:
         return True
     isettings = _get_current_instance_settings()
     if isettings is not None:
-        if (
-            from_module is not None
-            and settings.auto_connect
-            and not django_lamin.IS_SETUP
-            and not IS_LOADING
-        ):
+        if from_module is not None and not django_lamin.IS_SETUP and not IS_LOADING:
             if from_module != "lamindb":
                 _check_module_in_instance_modules(from_module, isettings)
 
@@ -146,13 +170,15 @@ def _check_instance_setup(from_module: str | None = None) -> bool:
                 il.reload(il.import_module(from_module))
             else:
                 django_lamin.setup_django(isettings)
-                logger.important(f"connected lamindb: {isettings.slug}")
-                settings._instance_settings = (
-                    isettings  # update of local storage location
-                )
+                if isettings.slug != "none/none":
+                    logger.important(f"connected lamindb: {isettings.slug}")
+                    # update of local storage location through search_local_root()
+                    settings._instance_settings = isettings
+                else:
+                    logger.warning("not connected, call: ln.connect('account/name')")
         return django_lamin.IS_SETUP
     else:
-        if from_module is not None and settings.auto_connect:
+        if from_module is not None:
             # the below enables users to auto-connect to an instance
             # simply by setting an environment variable, bypassing the
             # need of calling connect() manually

lamindb_setup/_connect_instance.py

@@ -2,23 +2,22 @@ from __future__ import annotations
 
 import importlib
 import os
+import sys
 from typing import TYPE_CHECKING, Any
 from uuid import UUID
 
 from lamin_utils import logger
 
-from ._check_setup import _check_instance_setup, _get_current_instance_settings
-from ._disconnect import disconnect
-from ._init_instance import (
-    MESSAGE_CANNOT_SWITCH_DEFAULT_INSTANCE,
-    load_from_isettings,
+from ._check_setup import (
+    _check_instance_setup,
+    _get_current_instance_settings,
+    find_module_candidates,
 )
+from ._disconnect import disconnect
+from ._init_instance import load_from_isettings
 from ._silence_loggers import silence_loggers
 from .core._hub_core import connect_instance_hub
-from .core._hub_utils import (
-    LaminDsn,
-    LaminDsnModel,
-)
+from .core._hub_utils import LaminDsnModel
 from .core._settings import settings
 from .core._settings_instance import InstanceSettings
 from .core._settings_load import load_instance_settings
@@ -69,21 +68,18 @@ def update_db_using_local(
     if hub_instance_result["db_scheme"] == "postgresql":
         if db is not None:
             # use only the provided db if it is set
-            db_dsn_hub = LaminDsnModel(db=db)
-            db_dsn_local = db_dsn_hub
-        else:
-            db_dsn_hub = LaminDsnModel(db=hub_instance_result["db"])
+            db_updated = db
+        elif (db_env := os.getenv("LAMINDB_INSTANCE_DB")) is not None:
+            logger.important("loading db URL from env variable LAMINDB_INSTANCE_DB")
             # read directly from the environment
-            if os.getenv("LAMINDB_INSTANCE_DB") is not None:
-                logger.important("loading db URL from env variable LAMINDB_INSTANCE_DB")
-                db_dsn_local = LaminDsnModel(db=os.getenv("LAMINDB_INSTANCE_DB"))
-            # read from a cached settings file in case the hub result is only
-            # read level or inexistent
-            elif settings_file.exists() and (
-                db_dsn_hub.db.user in {None, "none"} or "read" in db_dsn_hub.db.user  # type:ignore
-            ):
+            db_updated = db_env
+        else:
+            db_hub = hub_instance_result["db"]
+            db_dsn_hub = LaminDsnModel(db=db_hub)
+            # read from a cached settings file in case the hub result is inexistent
+            if db_dsn_hub.db.user in {None, "none"} and settings_file.exists():
                 isettings = load_instance_settings(settings_file)
-                db_dsn_local = LaminDsnModel(db=isettings.db)
+                db_updated = isettings.db
             else:
                 # just take the default hub result and ensure there is actually a user
                 if (
@@ -95,22 +91,7 @@ def update_db_using_local(
                         "No database access, please ask your admin to provide you with"
                         " a DB URL and pass it via --db <db_url>"
                     )
-                db_dsn_local = db_dsn_hub
-            if not check_db_dsn_equal_up_to_credentials(db_dsn_hub.db, db_dsn_local.db):
-                raise ValueError(
-                    "The local differs from the hub database information:\n"
-                    "did your database get updated by an admin?\n"
-                    "Consider deleting your cached database environment:\nrm"
-                    f" {settings_file.as_posix()}"
-                )
-        db_updated = LaminDsn.build(
-            scheme=db_dsn_hub.db.scheme,
-            user=db_dsn_local.db.user,
-            password=db_dsn_local.db.password,
-            host=db_dsn_hub.db.host,  # type: ignore
-            port=db_dsn_hub.db.port,
-            database=db_dsn_hub.db.database,
-        )
+                db_updated = db_hub
     return db_updated
 
 
@@ -127,7 +108,12 @@ def _connect_instance(
     if settings_file.exists():
         isettings = load_instance_settings(settings_file)
         # skip hub request for a purely local instance
-        make_hub_request = isettings.is_remote
+        if isettings.is_remote:
+            make_hub_request = True
+        else:
+            make_hub_request = False
+            if db is not None and isettings.dialect == "postgresql":
+                isettings._db = db
     if make_hub_request:
         # the following will return a string if the instance does not exist
         # on the hub
@@ -188,6 +174,62 @@ def _connect_instance(
     return isettings
 
 
+def reset_django_module_variables():
+    # This function updates all module-level references to Django classes
+    # But it will fail to update function level references
+    # So, if a user has
+    # def my_function():
+    #     import lamindb as ln
+    #     ...
+    #
+    # Then it will **not** work and the `ln` variable will become stale and hold a reference
+    # to the old classes
+    # There doesn't seem to be an easy way to fix this problem
+
+    import types
+
+    from django.apps import apps
+
+    app_names = {app.name for app in apps.get_app_configs()}
+
+    for name, module in sys.modules.items():
+        if (
+            module is not None
+            and (not name.startswith("__") or name == "__main__")
+            and name not in sys.builtin_module_names
+            and not (
+                hasattr(module, "__file__")
+                and module.__file__
+                and any(
+                    path in module.__file__ for path in ["/lib/python", "\\lib\\python"]
+                )
+            )
+        ):
+            try:
+                for k, v in vars(module).items():
+                    if (
+                        isinstance(v, types.ModuleType)
+                        and not k.startswith("_")
+                        and getattr(v, "__name__", None) in app_names
+                    ):
+                        if v.__name__ in sys.modules:
+                            vars(module)[k] = sys.modules[v.__name__]
+                    # Also reset classes from Django apps - but check if the class module starts with any app name
+                    elif hasattr(v, "__module__") and getattr(v, "__module__", None):
+                        class_module = v.__module__
+                        # Check if the class module starts with any of our app names
+                        if any(
+                            class_module.startswith(app_name) for app_name in app_names
+                        ):
+                            if class_module in sys.modules:
+                                fresh_module = sys.modules[class_module]
+                                attr_name = getattr(v, "__name__", k)
+                                if hasattr(fresh_module, attr_name):
+                                    vars(module)[k] = getattr(fresh_module, attr_name)
+            except (AttributeError, TypeError):
+                continue
+
+
 def _connect_cli(instance: str) -> None:
     from lamindb_setup import settings as settings_
 
@@ -214,6 +256,9 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
         instance: Pass a slug (`account/name`) or URL (`https://lamin.ai/account/name`).
             If `None`, looks for an environment variable `LAMIN_CURRENT_INSTANCE` to get the instance identifier.
             If it doesn't find this variable, it connects to the instance that was connected with `lamin connect` through the CLI.
+
+    See Also:
+        Configure an instance for auto-connect via the CLI, see `here <https://docs.lamin.ai/cli#connect>`__.
     """
     # validate kwargs
     valid_kwargs = {
@@ -250,19 +295,45 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
                     "No instance was connected through the CLI, pass a value to `instance` or connect via the CLI."
                 )
             isettings = isettings_or_none
+            if _db is not None and isettings.dialect == "postgresql":
+                isettings._db = _db
         else:
+            from django.db import connection
+
             owner, name = get_owner_name_from_identifier(instance)
             if _check_instance_setup() and not _test:
                 if (
                     settings._instance_exists
                     and f"{owner}/{name}" == settings.instance.slug
+                    # below is to ensure that if another process interferes
+                    # we don't use the in-memory mock database
+                    # could be made more specific by checking whether the django
+                    # configured database is the same as the one in settings
+                    and connection.settings_dict["NAME"] != ":memory:"
                 ):
-                    logger.important(f"connected lamindb: {settings.instance.slug}")
+                    logger.important(
+                        f"doing nothing, already connected lamindb: {settings.instance.slug}"
+                    )
                     return None
                 else:
-                    raise CannotSwitchDefaultInstance(
-                        MESSAGE_CANNOT_SWITCH_DEFAULT_INSTANCE
-                    )
+                    from lamindb_setup.core.django import reset_django
+
+                    if (
+                        settings._instance_exists
+                        and settings.instance.slug != "none/none"
+                    ):
+                        import lamindb as ln
+
+                        if ln.context.transform is not None:
+                            raise CannotSwitchDefaultInstance(
+                                "Cannot switch default instance while `ln.track()` is live: call `ln.finish()`"
+                            )
+                        else:
+                            logger.important_hint(
+                                "switching the default lamindb instance might produce unexpected side effects with function-scoped imports: "
+                                "please import lamindb at the module level instead of inside functions"
+                            )
+                    reset_django()
             elif (
                 _write_settings
                 and settings._instance_exists
@@ -315,7 +386,9 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
         load_from_isettings(isettings, user=_user, write_settings=_write_settings)
         if _reload_lamindb:
             importlib.reload(importlib.import_module("lamindb"))
-        logger.important(f"connected lamindb: {isettings.slug}")
+            reset_django_module_variables()
+        if isettings.slug != "none/none":
+            logger.important(f"connected lamindb: {isettings.slug}")
     except Exception as e:
         if isettings is not None:
             if _write_settings:

lamindb_setup/_delete.py

@@ -65,6 +65,9 @@ def delete(slug: str, force: bool = False, require_empty: bool = True) -> int |
             If the instance is owned by you, it suffices to pass the instance name.
         force: Whether to skip the confirmation prompt.
         require_empty: Whether to check if the instance is empty before deleting.
+
+    See Also:
+        Delete an instance via the CLI, see `here <https://docs.lamin.ai/cli#delete>`__.
     """
     owner, name = get_owner_name_from_identifier(slug)
     isettings = _connect_instance(owner, name, raise_permission_error=False)

lamindb_setup/_disconnect.py

@@ -8,9 +8,12 @@ from .core.cloud_sqlite_locker import clear_locker
 
 
 def disconnect(mute: bool = False) -> None:
-    """Disconnect an instance.
+    """Clear default instance configuration.
 
     Returns `None` if succeeds, otherwise an exception is raised.
+
+    See Also:
+        Clear default instance configuration via the CLI, see `here <https://docs.lamin.ai/cli#disconnect>`__.
     """
     if current_instance_settings_file().exists():
         instance = settings.instance.slug

lamindb_setup/_init_instance.py

@@ -16,7 +16,7 @@ from ._silence_loggers import silence_loggers
 from .core import InstanceSettings
 from .core._docs import doc_args
 from .core._settings import settings
-from .core._settings_instance import is_local_db_url
+from .core._settings_instance import check_is_instance_remote, is_local_db_url
 from .core._settings_storage import StorageSettings, init_storage
 from .core.upath import UPath
 from .errors import CannotSwitchDefaultInstance
@@ -215,16 +215,6 @@ def validate_init_args(
     return name_str, instance_id, instance_state, instance_slug
 
 
-MESSAGE_CANNOT_SWITCH_DEFAULT_INSTANCE = """
-You cannot write to different instances in the same Python session.
-
-Do you want to read from another instance via `SQLRecord.using()`? For example:
-
-ln.Artifact.using("laminlabs/cellxgene").filter()
-
-Or do you want to switch off auto-connect via `lamin settings set auto-connect false`?
-"""
-
 DOC_STORAGE_ARG = "A local or remote folder (`'s3://...'` or `'gs://...'`). Defaults to current working directory."
 DOC_INSTANCE_NAME = (
     "Instance name. If not passed, it will equal the folder name passed to `storage`."
@@ -251,6 +241,9 @@ def init(
         db: {}
         modules: {}
         **kwargs: {}
+
+    See Also:
+        Init an instance for via the CLI, see `here <https://docs.lamin.ai/cli#init>`__.
     """
     isettings = None
     ssettings = None
@@ -272,9 +265,16 @@ def init(
         from ._check_setup import _check_instance_setup
 
         if _check_instance_setup() and not _test:
-            raise CannotSwitchDefaultInstance(MESSAGE_CANNOT_SWITCH_DEFAULT_INSTANCE)
+            from lamindb_setup.core.django import reset_django
+
+            if settings._instance_exists:
+                raise CannotSwitchDefaultInstance(
+                    "Cannot init new instance after connecting to an existing instance."
+                )
+            reset_django()
         elif _write_settings:
             disconnect(mute=True)
+        from ._connect_instance import reset_django_module_variables
         from .core._hub_core import init_instance_hub
 
         name_str, instance_id, instance_state, _ = validate_init_args(
@@ -287,44 +287,38 @@ def init(
             _user=_user,  # will get from settings.user if _user is None
         )
         if instance_state == "connected":
-            if _write_settings:
-                settings.auto_connect = True  # we can also debate this switch here
             return None
-        prevent_register_hub = is_local_db_url(db) if db is not None else False
-        ssettings, _ = init_storage(
-            storage,
-            instance_id=instance_id,
-            instance_slug=f"{user_handle}/{name_str}",
-            init_instance=True,
-            prevent_register_hub=prevent_register_hub,
-            created_by=user__uuid,
-            access_token=access_token,
-        )
         isettings = InstanceSettings(
             id=instance_id,  # type: ignore
             owner=user_handle,
             name=name_str,
-            storage=ssettings,
             db=db,
             modules=modules,
-            uid=ssettings.uid,
             # to lock passed user in isettings._cloud_sqlite_locker.lock()
             _locker_user=_user,  # only has effect if cloud sqlite
         )
         register_on_hub = (
-            isettings.is_remote and instance_state != "instance-corrupted-or-deleted"
+            check_is_instance_remote(root=storage, db=db)
+            and instance_state != "instance-corrupted-or-deleted"
         )
         if register_on_hub:
-            # can't register the instance in the hub
-            # if storage is not in the hub
-            # raise the exception and initiate cleanups
-            if not isettings.storage.is_on_hub:
-                raise InstanceNotCreated(
-                    "Unable to create the instance because failed to register the storage."
-                )
             init_instance_hub(
                 isettings, account_id=user__uuid, access_token=access_token
             )
+        ssettings, _ = init_storage(
+            storage,
+            instance_id=instance_id,
+            instance_slug=f"{user_handle}/{name_str}",
+            init_instance=True,
+            register_hub=register_on_hub,
+            created_by=user__uuid,
+            access_token=access_token,
+        )
+        isettings._storage = ssettings
+        if register_on_hub and not ssettings.is_on_hub:
+            raise InstanceNotCreated(
+                "Unable to create the instance because failed to register the storage."
+            )
         validate_sqlite_state(isettings)
         # why call it here if it is also called in load_from_isettings?
         isettings._persist(write_to_disk=_write_settings)
@@ -344,9 +338,8 @@ def init(
             from ._schema_metadata import update_schema_in_hub
 
             update_schema_in_hub(access_token=access_token)
-        if _write_settings:
-            settings.auto_connect = True
         importlib.reload(importlib.import_module("lamindb"))
+        reset_django_module_variables()
         logger.important(f"initialized lamindb: {isettings.slug}")
     except Exception as e:
         from ._delete import delete_by_isettings
@@ -357,16 +350,10 @@ def init(
                 delete_by_isettings(isettings)
             else:
                 settings._instance_settings = None
-        if (
-            ssettings is not None
-            and (user_handle != "anonymous" or access_token is not None)
-            and ssettings.is_on_hub
-        ):
-            delete_storage_record(ssettings, access_token=access_token)  # type: ignore
-        if isettings is not None:
-            if (
-                user_handle != "anonymous" or access_token is not None
-            ) and isettings.is_on_hub:
+        if user_handle != "anonymous" or access_token is not None:
+            if ssettings is not None and ssettings.is_on_hub:
+                delete_storage_record(ssettings, access_token=access_token)
+            if isettings is not None and isettings.is_on_hub:
                 delete_instance_record(isettings._id, access_token=access_token)
         raise e
     return None
@@ -400,6 +387,9 @@ def load_from_isettings(
             # this is blocked anyways, only select and insert are allowed
             register_user(user, update_user=not isettings._fine_grained_access)
     isettings._persist(write_to_disk=write_settings)
+    # clear branch & space cache after reconnecting
+    settings._branch = None
+    settings._space = None
 
 
 def validate_sqlite_state(isettings: InstanceSettings) -> None:

lamindb_setup/_migrate.py

@@ -63,16 +63,29 @@ def check_whether_migrations_in_sync(db_version_str: str):
         # logger.important("consider migrating your database: lamin migrate deploy")
 
 
-# for tests, see lamin-cli
 class migrate:
-    """Manage migrations.
+    """Manage database migrations.
+
+    Unless you maintain your own schema modules with your own Django models, you won't need this.
 
     Examples:
 
-    >>> import lamindb as ln
-    >>> ln.setup.migrate.create()
-    >>> ln.setup.migrate.deploy()
-    >>> ln.setup.migrate.check()
+        Create a migration::
+
+            import lamindb as ln
+
+            ln.setup.migrate.create()
+
+        Deploy a migration::
+
+            ln.setup.migrate.deploy()
+
+        Check migration consistency::
+
+            ln.setup.migrate.check()
+
+    See Also:
+        Migrate an instance via the CLI, see `here <https://docs.lamin.ai/cli#migrate>`__.
 
     """
 
@@ -85,7 +98,6 @@ class migrate:
         setup_django(settings.instance, create_migrations=True)
 
     @classmethod
-    @disable_auto_connect
     def deploy(cls, package_name: str | None = None, number: int | None = None) -> None:
         """Deploy a migration."""
         from ._schema_metadata import update_schema_in_hub

lamindb_setup/_register_instance.py

@@ -21,12 +21,9 @@ def register(_test: bool = False):
     ssettings = settings.instance.storage
     if ssettings._uid is None and _test:
         # because django isn't up, we can't get it from the database
-        ssettings._uid = base62(8)
-    # cannot yet populate the instance id here
-    ssettings._instance_id = None
-    # flag auto_populate_instance can be removed once FK migration is over
-    init_storage_hub(ssettings, auto_populate_instance=False)
+        ssettings._uid = base62(12)
     init_instance_hub(isettings)
+    init_storage_hub(ssettings, is_default=True)
     isettings._is_on_hub = True
     isettings._persist()
     if isettings.dialect != "sqlite" and not _test:

lamindb_setup/_set_managed_storage.py

@@ -49,7 +49,6 @@ def set_managed_storage(root: UPathStr, host: str | None = None, **fs_kwargs):
         instance_id=settings.instance._id,
         instance_slug=settings.instance.slug,
         register_hub=settings.instance.is_on_hub,
-        prevent_register_hub=not settings.instance.is_on_hub,
         region=host,
     )
     if ssettings._instance_id is None: