lamindb_setup 1.6.1__py3-none-any.whl → 1.7.1__py3-none-any.whl

lamindb_setup/__init__.py

@@ -30,18 +30,20 @@ Modules & settings:
    settings
    core
    django
+   errors
+   types
 
 """
 
-__version__ = "1.6.1"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.7.1"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os
 
 from packaging import version as packaging_version
 
-from . import core
+from . import core, errors, types
 from ._check_setup import _check_instance_setup
-from ._connect_instance import connect, load
+from ._connect_instance import connect
 from ._delete import delete
 from ._disconnect import disconnect
 from ._django import django

lamindb_setup/_cache.py

@@ -1,10 +1,13 @@
 from __future__ import annotations
 
 import shutil
+from pathlib import Path
 
+from dotenv import dotenv_values
 from lamin_utils import logger
 
-from .core._settings_save import save_system_storage_settings
+from .core._settings_save import save_platform_user_storage_settings
+from .core._settings_store import system_settings_file
 
 
 def clear_cache_dir():
@@ -13,7 +16,7 @@ def clear_cache_dir():
     try:
         if settings.instance._is_cloud_sqlite:
             logger.warning(
-                "Disconnecting the current instance to update the cloud sqlite database."
+                "disconnecting the current instance to update the cloud sqlite database."
             )
             disconnect()
     except SystemExit as e:
@@ -21,9 +24,12 @@ def clear_cache_dir():
             raise e
 
     cache_dir = settings.cache_dir
-    shutil.rmtree(cache_dir)
-    cache_dir.mkdir()
-    logger.success("The cache directory was cleared.")
+    if cache_dir.exists():
+        shutil.rmtree(cache_dir)
+        cache_dir.mkdir()
+        logger.success("the cache directory was cleared")
+    else:
+        logger.warning("the cache directory doesn't exist")
 
 
 def get_cache_dir():
@@ -41,12 +47,41 @@ def set_cache_dir(cache_dir: str):
 
     old_cache_dir = settings.cache_dir
     new_cache_dir = _process_cache_path(cache_dir)
+
+    system_cache_dir = None
+    if (system_settings := system_settings_file()).exists():
+        system_cache_dir = dotenv_values(system_settings).get(
+            "lamindb_cache_path", None
+        )
+        system_cache_dir = (
+            Path(system_cache_dir) if system_cache_dir is not None else None
+        )
+
+    need_reset = False
     if new_cache_dir is None:
-        new_cache_dir = DEFAULT_CACHE_DIR
+        need_reset = True
+        new_cache_dir = (
+            DEFAULT_CACHE_DIR if system_cache_dir is None else system_cache_dir
+        )
+
     if new_cache_dir != old_cache_dir:
-        shutil.copytree(old_cache_dir, new_cache_dir, dirs_exist_ok=True)
-        shutil.rmtree(old_cache_dir)
-        logger.info("The current cache directory was moved to the specified location")
-    new_cache_dir = new_cache_dir.resolve()
-    save_system_storage_settings(new_cache_dir)
+        if old_cache_dir.exists():
+            shutil.copytree(old_cache_dir, new_cache_dir, dirs_exist_ok=True)
+            logger.info(
+                f"the current cache directory was copied to {new_cache_dir.as_posix()} "
+            )
+            if old_cache_dir != system_cache_dir:
+                shutil.rmtree(old_cache_dir)
+                logger.info(
+                    f"cleared the old cache directory {old_cache_dir.as_posix()}"
+                )
+            else:
+                logger.info(
+                    f"didn't clear the system cache directory {system_cache_dir.as_posix()}, "
+                    "please clear it manually if you need"
+                )
+        else:
+            new_cache_dir.mkdir(parents=True, exist_ok=True)
+        new_cache_dir = new_cache_dir.resolve()
+    save_platform_user_storage_settings(None if need_reset else new_cache_dir)
     settings._cache_dir = new_cache_dir

lamindb_setup/_check_setup.py

@@ -12,7 +12,11 @@ from ._silence_loggers import silence_loggers
 from .core import django as django_lamin
 from .core._settings import settings
 from .core._settings_store import current_instance_settings_file
-from .core.exceptions import DefaultMessageException
+from .errors import (
+    MODULE_WASNT_CONFIGURED_MESSAGE_TEMPLATE,
+    InstanceNotSetupError,
+    ModuleWasntConfigured,
+)
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -20,24 +24,10 @@ if TYPE_CHECKING:
     from .core._settings_instance import InstanceSettings
 
 
-class InstanceNotSetupError(DefaultMessageException):
-    default_message = """\
-To use lamindb, you need to connect to an instance.
-
-Connect to an instance: `ln.connect()`. Init an instance: `ln.setup.init()`.
-
-If you used the CLI to set up lamindb in a notebook, restart the Python session.
-"""
-
-
 CURRENT_ISETTINGS: InstanceSettings | None = None
 IS_LOADING: bool = False
 
 
-class ModuleWasntConfigured(SystemExit):
-    pass
-
-
 # decorator to disable auto-connect when importing a module such as lamindb
 def disable_auto_connect(func: Callable):
     @functools.wraps(func)
@@ -84,18 +74,15 @@ def _normalize_module_name(module_name: str) -> str:
 def _check_module_in_instance_modules(
     module: str, isettings: InstanceSettings | None = None
 ) -> None:
-    not_in_instance_msg = (
-        f"'{module}' is missing from this instance. "
-        "Please go to your instance settings page and add it under 'schema modules'."
-    )
-
     if isettings is not None:
         modules_raw = isettings.modules
         modules = set(modules_raw).union(
             _normalize_module_name(module) for module in modules_raw
         )
         if _normalize_module_name(module) not in modules and module not in modules:
-            raise ModuleWasntConfigured(not_in_instance_msg)
+            raise ModuleWasntConfigured(
+                MODULE_WASNT_CONFIGURED_MESSAGE_TEMPLATE.format(module)
+            )
         else:
             return
 
@@ -105,7 +92,7 @@ def _check_module_in_instance_modules(
         # app.name is always unnormalized module (python package) name
         if module == app.name or module == _normalize_module_name(app.name):
             return
-    raise ModuleWasntConfigured(not_in_instance_msg)
+    raise ModuleWasntConfigured(MODULE_WASNT_CONFIGURED_MESSAGE_TEMPLATE.format(module))
 
 
 # infer the name of the module that calls this function

lamindb_setup/_connect_instance.py

@@ -11,7 +11,6 @@ from ._check_setup import _check_instance_setup, _get_current_instance_settings
 from ._disconnect import disconnect
 from ._init_instance import (
     MESSAGE_CANNOT_SWITCH_DEFAULT_INSTANCE,
-    CannotSwitchDefaultInstance,
     load_from_isettings,
 )
 from ._silence_loggers import silence_loggers
@@ -26,12 +25,13 @@ from .core._settings_load import load_instance_settings
 from .core._settings_storage import StorageSettings
 from .core._settings_store import instance_settings_file, settings_dir
 from .core.cloud_sqlite_locker import unlock_cloud_sqlite_upon_exception
+from .errors import CannotSwitchDefaultInstance
 
 if TYPE_CHECKING:
     from pathlib import Path
 
     from .core._settings_user import UserSettings
-    from .core.types import UPathStr
+    from .types import UPathStr
 
 # this is for testing purposes only
 # set to True only to test failed load
@@ -158,7 +158,7 @@ def _connect_instance(
             isettings = InstanceSettings(
                 id=UUID(instance_result["id"]),
                 owner=owner,
-                name=name,
+                name=instance_result["name"],
                 storage=ssettings,
                 db=db_updated,
                 modules=instance_result["schema_str"],
@@ -211,7 +211,7 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
     isettings: InstanceSettings = None  # type: ignore
     # _db is still needed because it is called in init
     _db: str | None = kwargs.get("_db", None)
-    _write_settings: bool = kwargs.get("_write_settings", True)
+    _write_settings: bool = kwargs.get("_write_settings", False)
     _raise_not_found_error: bool = kwargs.get("_raise_not_found_error", True)
     _reload_lamindb: bool = kwargs.get("_reload_lamindb", True)
     _test: bool = kwargs.get("_test", False)
@@ -270,15 +270,6 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
         if _test:
             return None
         silence_loggers()
-        # migrate away from lnschema-core
-        no_lnschema_core_file = (
-            settings_dir / f"no_lnschema_core-{isettings.slug.replace('/', '--')}"
-        )
-        if not no_lnschema_core_file.exists():
-            # sqlite file for cloud sqlite instances is already updated here
-            migrate_lnschema_core(
-                isettings, no_lnschema_core_file, write_file=_write_settings
-            )
         check, msg = isettings._load_db()
         if not check:
             local_db = (
@@ -305,8 +296,7 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
         load_from_isettings(isettings, user=_user, write_settings=_write_settings)
         if _reload_lamindb:
             importlib.reload(importlib.import_module("lamindb"))
-        else:
-            logger.important(f"connected lamindb: {isettings.slug}")
+        logger.important(f"connected lamindb: {isettings.slug}")
     except Exception as e:
         if isettings is not None:
             if _write_settings:
@@ -316,141 +306,6 @@ def connect(instance: str | None = None, **kwargs: Any) -> str | tuple | None:
     return None
 
 
-def load(slug: str) -> str | tuple | None:
-    """Connect to instance and set ``auto-connect`` to true.
-
-    This is exactly the same as ``ln.connect()`` except for that
-    ``ln.connect()`` doesn't change the state of ``auto-connect``.
-    """
-    print("Warning: This is deprecated and will be removed.")
-    result = connect(slug)
-    settings.auto_connect = True
-    return result
-
-
-def migrate_lnschema_core(
-    isettings: InstanceSettings, no_lnschema_core_file: Path, write_file: bool = True
-):
-    """Migrate lnschema_core tables to lamindb tables."""
-    from urllib.parse import urlparse
-
-    # we need to do this because the sqlite file should be already synced
-    # has no effect if not cloud sqlite
-    # errors if the sqlite file is not in the cloud and doesn't exist locally
-    # isettings.db syncs but doesn't error in this case due to error_no_origin=False
-    isettings._update_local_sqlite_file()
-
-    parsed_uri = urlparse(isettings.db)
-    db_type = parsed_uri.scheme
-
-    if db_type == "sqlite":
-        import sqlite3
-
-        # maybe also use LAMINDB_DJANGO_DATABASE_URL here?
-        conn = sqlite3.connect(parsed_uri.path)
-    elif db_type in ["postgresql", "postgres"]:
-        import psycopg2
-
-        # do not ignore LAMINDB_DJANGO_DATABASE_URL if it is set
-        conn = psycopg2.connect(
-            os.environ.get("LAMINDB_DJANGO_DATABASE_URL", isettings.db)
-        )
-    else:
-        raise ValueError("Unsupported database type. Use 'sqlite' or 'postgresql' URI.")
-
-    cur = conn.cursor()
-
-    try:
-        if db_type == "sqlite":
-            cur.execute(
-                "SELECT name FROM sqlite_master WHERE type='table' AND name='lamindb_user'"
-            )
-            migrated = cur.fetchone() is not None
-
-            # tables that need to be renamed
-            cur.execute(
-                "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'lnschema_core_%'"
-            )
-            tables_to_rename = [
-                row[0][len("lnschema_core_") :] for row in cur.fetchall()
-            ]
-        else:  # postgres
-            cur.execute(
-                "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'lamindb_user')"
-            )
-            migrated = cur.fetchone()[0]
-
-            # tables that need to be renamed
-            cur.execute(
-                "SELECT table_name FROM information_schema.tables WHERE table_name LIKE 'lnschema_core_%'"
-            )
-            tables_to_rename = [
-                row[0][len("lnschema_core_") :] for row in cur.fetchall()
-            ]
-
-        if migrated:
-            if write_file:
-                no_lnschema_core_file.touch(exist_ok=True)
-        else:
-            try:
-                response = input(
-                    f"Do you want to migrate to lamindb 1.0 (integrate lnschema_core into lamindb)? (y/n) -- Will rename {tables_to_rename}"
-                )
-                if response != "y":
-                    print("Aborted.")
-                    quit()
-                if isettings.is_on_hub:
-                    from lamindb_setup.core._hub_client import call_with_fallback_auth
-                    from lamindb_setup.core._hub_crud import (
-                        select_collaborator,
-                    )
-
-                    # double check that user is an admin, otherwise will fail below
-                    # due to insufficient SQL permissions with cryptic error
-                    collaborator = call_with_fallback_auth(
-                        select_collaborator,
-                        instance_id=settings.instance._id,
-                        account_id=settings.user._uuid,
-                    )
-                    if collaborator is None or collaborator["role"] != "admin":
-                        raise SystemExit(
-                            "❌ Only admins can deploy migrations, please ensure that you're an"
-                            f" admin: https://lamin.ai/{settings.instance.slug}/settings"
-                        )
-                for table in tables_to_rename:
-                    if db_type == "sqlite":
-                        cur.execute(
-                            f"ALTER TABLE lnschema_core_{table} RENAME TO lamindb_{table}"
-                        )
-                    else:  # postgres
-                        cur.execute(
-                            f"ALTER TABLE lnschema_core_{table} RENAME TO lamindb_{table};"
-                        )
-
-                cur.execute(
-                    "UPDATE django_migrations SET app = 'lamindb' WHERE app = 'lnschema_core'"
-                )
-                print(
-                    "Renaming tables finished.\nNow, *please* call: lamin migrate deploy"
-                )
-                if write_file:
-                    no_lnschema_core_file.touch(exist_ok=True)
-            except Exception:
-                # read-only users can't rename tables
-                pass
-
-        conn.commit()
-
-    except Exception as e:
-        print(f"An error occurred: {e}")
-        conn.rollback()
-
-    finally:
-        # close the cursor and connection
-        cur.close()
-        conn.close()
-
-
 def get_owner_name_from_identifier(identifier: str):
     if "/" in identifier:
         if identifier.startswith("https://lamin.ai/"):

lamindb_setup/_init_instance.py

@@ -8,7 +8,7 @@ from uuid import UUID
 
 import click
 from django.core.exceptions import FieldError
-from django.db.utils import OperationalError, ProgrammingError
+from django.db.utils import IntegrityError, OperationalError, ProgrammingError
 from lamin_utils import logger
 
 from ._disconnect import disconnect
@@ -19,12 +19,13 @@ from .core._settings import settings
 from .core._settings_instance import is_local_db_url
 from .core._settings_storage import StorageSettings, init_storage
 from .core.upath import UPath
+from .errors import CannotSwitchDefaultInstance
 
 if TYPE_CHECKING:
     from pydantic import PostgresDsn
 
     from .core._settings_user import UserSettings
-    from .core.types import UPathStr
+    from .types import UPathStr
 
 
 class InstanceNotCreated(click.ClickException):
@@ -49,37 +50,31 @@ def get_schema_module_name(module_name, raise_import_error: bool = True) -> str
 
 
 def register_storage_in_instance(ssettings: StorageSettings):
-    from lamindb.base.users import current_user_id
     from lamindb.models import Storage
 
-    from .core.hashing import hash_and_encode_as_b62
-
-    if ssettings._instance_id is not None:
-        instance_uid = hash_and_encode_as_b62(ssettings._instance_id.hex)[:12]
-    else:
-        instance_uid = None
     # how do we ensure that this function is only called passing
     # the managing instance?
-    defaults = {
+    kwargs = {
         "root": ssettings.root_as_str,
         "type": ssettings.type,
         "region": ssettings.region,
-        "instance_uid": instance_uid,
-        "created_by_id": current_user_id(),
+        "instance_uid": ssettings.instance_uid,
         "run": None,
+        "_skip_preparation": True,
     }
     if ssettings._uid is not None:
-        defaults["uid"] = ssettings._uid
-    storage, _ = Storage.objects.update_or_create(
-        root=ssettings.root_as_str,
-        defaults=defaults,
-    )
+        kwargs["uid"] = ssettings._uid
+    # this checks if the storage already exists under the hood
+    storage = Storage(**kwargs).save()
     return storage
 
 
-def register_user(usettings):
+def register_user(usettings: UserSettings, update_user: bool = True):
     from lamindb.models import User
 
+    if not update_user and User.objects.filter(uid=usettings.uid).exists():
+        return
+
     try:
         # need to have try except because of integer primary key migration
         user, created = User.objects.update_or_create(
@@ -90,37 +85,38 @@ def register_user(usettings):
             },
         )
     # for users with only read access, except via ProgrammingError
-    # ProgrammingError: permission denied for table lnschema_core_user
-    except (OperationalError, FieldError, ProgrammingError):
+    # ProgrammingError: permission denied for table lamindb_user
+    # IntegrityError: when trying to update a user on a fine-grained access instance
+    except (OperationalError, FieldError, ProgrammingError, IntegrityError):
         pass
 
 
-def register_initial_records(isettings: InstanceSettings, usettings):
+def register_initial_records(isettings: InstanceSettings, usettings: UserSettings):
     """Register space, user & storage in DB."""
     from django.db.utils import OperationalError
     from lamindb.models import Branch, Space
 
     try:
         Space.objects.get_or_create(
-            uid="A",
-            name="All",
+            uid="a",
+            name="all",
             description="Every team & user with access to the instance has access.",
         )
         Branch.objects.get_or_create(
             id=-1,
-            uid="T",
-            name="Trash",
+            uid="t",
+            name="trash",
             description="The trash.",
         )
         Branch.objects.get_or_create(
             id=0,
-            uid="A",
-            name="Archive",
+            uid="a",
+            name="archive",
             description="The archive.",
         )
         Branch.objects.get_or_create(
-            uid="M",
-            name="Main",
+            uid="m",
+            name="main",
             description="The main & default branch of the instance.",
         )
         register_user(usettings)
@@ -178,7 +174,7 @@ def validate_init_args(
     _user: UserSettings | None = None,
 ) -> tuple[
     str,
-    UUID | None,
+    UUID,
     Literal[
         "connected",
         "instance-corrupted-or-deleted",
@@ -204,23 +200,21 @@ def validate_init_args(
         _write_settings=_write_settings,
         _user=_user,
     )
+    instance_id: UUID
     instance_state: Literal[
         "connected",
         "instance-corrupted-or-deleted",
         "account-not-exists",
         "instance-not-found",
-    ] = "connected"
-    instance_id = None
-    if response is not None:
+    ]
+    if response is None:
+        instance_state, instance_id = "connected", settings.instance._id
+    else:
         instance_id, instance_state = process_connect_response(response, instance_slug)
     modules = process_modules_arg(modules)
     return name_str, instance_id, instance_state, instance_slug
 
 
-class CannotSwitchDefaultInstance(SystemExit):
-    pass
-
-
 MESSAGE_CANNOT_SWITCH_DEFAULT_INSTANCE = """
 You cannot write to different instances in the same Python session.
 
@@ -296,13 +290,11 @@ def init(
             if _write_settings:
                 settings.auto_connect = True  # we can also debate this switch here
             return None
-        # the conditions here match `isettings.is_remote`, but I currently don't
-        # see a way of making this more elegant; should become possible if we
-        # remove the instance.storage_id FK on the hub
         prevent_register_hub = is_local_db_url(db) if db is not None else False
         ssettings, _ = init_storage(
             storage,
             instance_id=instance_id,
+            instance_slug=f"{user_handle}/{name_str}",
             init_instance=True,
             prevent_register_hub=prevent_register_hub,
             created_by=user__uuid,
@@ -370,7 +362,7 @@ def init(
             and (user_handle != "anonymous" or access_token is not None)
             and ssettings.is_on_hub
         ):
-            delete_storage_record(ssettings._uuid, access_token=access_token)  # type: ignore
+            delete_storage_record(ssettings, access_token=access_token)  # type: ignore
         if isettings is not None:
             if (
                 user_handle != "anonymous" or access_token is not None
@@ -404,7 +396,9 @@ def load_from_isettings(
         # this is our best proxy for that the user might not
         # yet be registered
         if not isettings._get_settings_file().exists():
-            register_user(user)
+            # do not try to update the user on fine grained access instances
+            # this is blocked anyways, only select and insert are allowed
+            register_user(user, update_user=not isettings._fine_grained_access)
     isettings._persist(write_to_disk=write_settings)
 
 

lamindb_setup/_register_instance.py

@@ -12,7 +12,7 @@ def register(_test: bool = False):
     from ._check_setup import _check_instance_setup
     from .core._hub_core import init_instance_hub, init_storage_hub
 
-    logger.warning("""lamin register will be removed soon""")
+    logger.warning("note that register() is only for testing purposes")
 
     isettings = settings.instance
     if not _check_instance_setup() and not _test:

lamindb_setup/_schema_metadata.py

@@ -16,6 +16,7 @@ from django.db.models import (
     OneToOneField,
     OneToOneRel,
 )
+from lamin_utils import logger
 from pydantic import BaseModel
 
 from lamindb_setup import settings
@@ -25,6 +26,9 @@ from lamindb_setup.core._hub_client import call_with_fallback_auth
 # surpress pydantic warning about `model_` namespace
 try:
     BaseModel.model_config["protected_namespaces"] = ()
+    logger.debug(
+        "pydantic.BaseModel.model_config['protected_namespaces'] has been set to ()"
+    )
 except Exception:
     pass
 

lamindb_setup/_set_managed_storage.py

@@ -10,7 +10,7 @@ from .core._settings import settings
 from .core._settings_storage import init_storage
 
 if TYPE_CHECKING:
-    from lamindb_setup.core.types import UPathStr
+    from lamindb_setup.types import UPathStr
 
 
 def set_managed_storage(root: UPathStr, **fs_kwargs):
@@ -33,7 +33,10 @@ def set_managed_storage(root: UPathStr, **fs_kwargs):
     # hub_record_status="hub-record-created" if a new record is created
     # "hub-record-retrieved" if the storage is in the hub already
     ssettings, hub_record_status = init_storage(
-        root=root, instance_id=settings.instance._id, register_hub=True
+        root=root,
+        instance_id=settings.instance._id,
+        instance_slug=settings.instance.slug,
+        register_hub=True,
     )
     if ssettings._instance_id is None:
         raise ValueError(
@@ -47,9 +50,8 @@ def set_managed_storage(root: UPathStr, **fs_kwargs):
         register_storage_in_instance(ssettings)
     except Exception as e:
         if hub_record_status == "hub-record-created" and ssettings._uuid is not None:
-            delete_storage_record(ssettings._uuid)  # type: ignore
+            delete_storage_record(ssettings)
         raise e
 
     settings.instance._storage = ssettings
-    settings.instance._persist()  # this also updates the settings object
     settings.storage._set_fs_kwargs(**fs_kwargs)

lamindb_setup/core/__init__.py

@@ -12,7 +12,7 @@ Settings:
 
 """
 
-from . import django, types, upath
+from . import django, upath
 from ._deprecated import deprecated
 from ._docs import doc_args
 from ._settings import SetupSettings

lamindb_setup/core/_aws_options.py

@@ -31,7 +31,7 @@ AWS_CREDENTIALS_EXPIRATION: int = 11 * 60 * 60  # refresh credentials after 11 h
 
 # set anon=True for these buckets if credentials fail for a public bucket
 # to be expanded
-PUBLIC_BUCKETS: tuple[str] = ("cellxgene-data-public",)
+PUBLIC_BUCKETS: tuple[str, ...] = ("cellxgene-data-public", "bionty-assets")
 
 
 # s3-comaptible endpoints managed by lamin
@@ -79,8 +79,8 @@ class AWSOptionsManager:
                 # use lamindata public bucket for this test
                 fs.call_s3("head_bucket", Bucket="lamindata")
                 self.anon_public = False
-            except Exception as e:
-                self.anon_public = isinstance(e, PermissionError)
+            except Exception:
+                self.anon_public = True
 
     def _find_root(self, path_str: str) -> str | None:
         roots = self._credentials_cache.keys()