lamindb_setup 0.70.0__py2.py3-none-any.whl → 0.71.1__py2.py3-none-any.whl

lamindb_setup/_init_instance.py

@@ -1,22 +1,28 @@
+from __future__ import annotations
+
 import importlib
+import os
 import sys
-from typing import Optional, Union
 import uuid
+from typing import TYPE_CHECKING, Literal
 from uuid import UUID
-import os
-from lamin_utils import logger
-from typing import Tuple, Literal
-from pydantic import PostgresDsn
-from django.db.utils import OperationalError, ProgrammingError
+
 from django.core.exceptions import FieldError
+from django.db.utils import OperationalError, ProgrammingError
+from lamin_utils import logger
+
 from ._close import close as close_instance
-from .core._settings import settings
 from ._silence_loggers import silence_loggers
 from .core import InstanceSettings
-from .core.types import UPathStr
+from .core._settings import settings
 from .core._settings_storage import StorageSettings, init_storage
 from .core.upath import convert_pathlike
 
+if TYPE_CHECKING:
+    from pydantic import PostgresDsn
+
+    from .core.types import UPathStr
+
 
 def get_schema_module_name(schema_name) -> str:
     import importlib.util
@@ -32,20 +38,26 @@ def get_schema_module_name(schema_name) -> str:
     )
 
 
-def register_storage(ssettings: StorageSettings):
+def register_storage_in_instance(ssettings: StorageSettings):
     from lnschema_core.models import Storage
     from lnschema_core.users import current_user_id
 
-    defaults = dict(
-        root=ssettings.root_as_str,
-        type=ssettings.type,
-        region=ssettings.region,
-        created_by_id=current_user_id(),
-    )
+    from .core.hashing import hash_and_encode_as_b62
+
+    assert ssettings._instance_id is not None
+
+    # how do we ensure that this function is only called passing
+    # the managing instance?
+    defaults = {
+        "root": ssettings.root_as_str,
+        "type": ssettings.type,
+        "region": ssettings.region,
+        "instance_uid": hash_and_encode_as_b62(ssettings._instance_id.hex)[:12],
+        "created_by_id": current_user_id(),
+    }
     if ssettings._uid is not None:
         defaults["uid"] = ssettings._uid
-
-    storage, created = Storage.objects.update_or_create(
+    storage, _ = Storage.objects.update_or_create(
         root=ssettings.root_as_str,
         defaults=defaults,
     )
@@ -60,10 +72,10 @@ def register_user(usettings):
             # need to have try except because of integer primary key migration
             user, created = User.objects.update_or_create(
                 uid=usettings.uid,
-                defaults=dict(
-                    handle=usettings.handle,
-                    name=usettings.name,
-                ),
+                defaults={
+                    "handle": usettings.handle,
+                    "name": usettings.name,
+                },
             )
         # for users with only read access, except via ProgrammingError
         # ProgrammingError: permission denied for table lnschema_core_user
@@ -71,13 +83,13 @@ def register_user(usettings):
             pass
 
 
-def register_user_and_storage(isettings: InstanceSettings, usettings):
+def register_user_and_storage_in_instance(isettings: InstanceSettings, usettings):
     """Register user & storage in DB."""
     from django.db.utils import OperationalError
 
     try:
         register_user(usettings)
-        register_storage(isettings.storage)
+        register_storage_in_instance(isettings.storage)
     except OperationalError as error:
         logger.warning(f"instance seems not set up ({error})")
 
@@ -123,11 +135,11 @@ Either delete your cache ({}) or add it back to the cloud (if delete was acciden
 
 
 def process_connect_response(
-    response: Union[Tuple, str], instance_identifier: str
-) -> Tuple[
+    response: tuple | str, instance_identifier: str
+) -> tuple[
     UUID,
     Literal[
-        "instance-corrupted-or-deleted", "account-not-exists", "instance-not-reachable"
+        "instance-corrupted-or-deleted", "account-not-exists", "instance-not-found"
     ],
 ]:
     # for internal use when creating instances through CICD
@@ -148,18 +160,18 @@ def process_connect_response(
 def validate_init_args(
     *,
     storage: UPathStr,
-    name: Optional[str] = None,
-    db: Optional[PostgresDsn] = None,
-    schema: Optional[str] = None,
+    name: str | None = None,
+    db: PostgresDsn | None = None,
+    schema: str | None = None,
     _test: bool = False,
-) -> Tuple[
+) -> tuple[
     str,
-    Optional[UUID],
+    UUID | None,
     Literal[
         "connected",
         "instance-corrupted-or-deleted",
         "account-not-exists",
-        "instance-not-reachable",
+        "instance-not-found",
     ],
     str,
 ]:
@@ -177,7 +189,7 @@ def validate_init_args(
         "connected",
         "instance-corrupted-or-deleted",
         "account-not-exists",
-        "instance-not-reachable",
+        "instance-not-found",
     ] = "connected"
     instance_id = None
     if response is not None:
@@ -196,9 +208,9 @@ Try running on the CLI: lamin set auto-connect false
 def init(
     *,
     storage: UPathStr,
-    name: Optional[str] = None,
-    db: Optional[PostgresDsn] = None,
-    schema: Optional[str] = None,
+    name: str | None = None,
+    db: PostgresDsn | None = None,
+    schema: str | None = None,
     _test: bool = False,
 ) -> None:
     """Create and load a LaminDB instance.
@@ -222,7 +234,7 @@ def init(
             close_instance(mute=True)
         from .core._hub_core import init_instance as init_instance_hub
 
-        name_str, instance_id, instance_state, instance_slug = validate_init_args(
+        name_str, instance_id, instance_state, _ = validate_init_args(
             storage=storage,
             name=name,
             db=db,
@@ -232,7 +244,7 @@ def init(
         if instance_state == "connected":
             settings.auto_connect = True  # we can also debate this switch here
             return None
-        ssettings = init_storage(storage)
+        ssettings = init_storage(storage, instance_id=instance_id)
         isettings = InstanceSettings(
             id=instance_id,  # type: ignore
             owner=settings.user.handle,
@@ -261,14 +273,18 @@ def init(
         settings.auto_connect = True
     except Exception as e:
         from ._delete import delete_by_isettings
-        from .core._hub_core import delete_storage_record as delete_storage_record
-        from .core._hub_core import delete_instance_record as delete_instance_record
+        from .core._hub_core import delete_instance_record, delete_storage_record
 
         if isettings is not None:
             delete_by_isettings(isettings)
-            delete_instance_record(isettings.id)
+            if settings.user.handle != "anonymous" and isettings.is_on_hub:
+                delete_instance_record(isettings._id)
             isettings._get_settings_file().unlink(missing_ok=True)  # type: ignore
-        if ssettings is not None:
+        if (
+            ssettings is not None
+            and settings.user.handle != "anonymous"
+            and ssettings.is_on_hub
+        ):
             delete_storage_record(ssettings._uuid)  # type: ignore
         raise e
     return None
@@ -283,7 +299,7 @@ def load_from_isettings(
 
     if init:
         # during init both user and storage need to be registered
-        register_user_and_storage(isettings, settings.user)
+        register_user_and_storage_in_instance(isettings, settings.user)
         write_bionty_sources(isettings)
         isettings._update_cloud_sqlite_file(unlock_cloud_sqlite=False)
     else:
@@ -319,8 +335,8 @@ def validate_sqlite_state(isettings: InstanceSettings) -> None:
 def infer_instance_name(
     *,
     storage: UPathStr,
-    name: Optional[str] = None,
-    db: Optional[PostgresDsn] = None,
+    name: str | None = None,
+    db: PostgresDsn | None = None,
 ) -> str:
     if name is not None:
         if "/" in name:

lamindb_setup/_migrate.py

@@ -1,11 +1,13 @@
-from typing import Optional, Dict
+from __future__ import annotations
+
+from django.db import connection
+from django.db.migrations.loader import MigrationLoader
 from lamin_utils import logger
 from packaging import version
+
 from ._check_setup import _check_instance_setup
 from .core._settings import settings
 from .core.django import setup_django
-from django.db import connection
-from django.db.migrations.loader import MigrationLoader
 
 
 # for the django-based synching code, see laminhub_rest
@@ -68,14 +70,14 @@ class migrate:
         """Deploy a migration."""
         if _check_instance_setup():
             raise RuntimeError("Restart Python session to migrate or use CLI!")
+        from lamindb_setup.core._hub_client import call_with_fallback_auth
         from lamindb_setup.core._hub_crud import (
-            update_instance,
-            select_instance_by_id,
             select_collaborator,
+            select_instance_by_id,
+            update_instance,
         )
-        from lamindb_setup.core._hub_client import call_with_fallback_auth
 
-        instance_id_str = settings.instance.id.hex
+        instance_id_str = settings.instance._id.hex
         instance = call_with_fallback_auth(
             select_instance_by_id, instance_id=instance_id_str
         )
@@ -104,7 +106,7 @@ class migrate:
             logger.important(f"updating lamindb version in hub: {lamindb.__version__}")
             call_with_fallback_auth(
                 update_instance,
-                instance_id=settings.instance.id.hex,
+                instance_id=settings.instance._id.hex,
                 instance_fields={"lamindb_version": lamindb.__version__},
             )
 
@@ -126,7 +128,7 @@ class migrate:
 
     @classmethod
     def squash(
-        cls, package_name, migration_nr, start_migration_nr: Optional[str] = None
+        cls, package_name, migration_nr, start_migration_nr: str | None = None
     ) -> None:
         """Squash migrations."""
         from django.core.management import call_command
@@ -149,9 +151,10 @@ class migrate:
 
     @classmethod
     def defined_migrations(cls, latest: bool = False):
-        from django.core.management import call_command
         from io import StringIO
 
+        from django.core.management import call_command
+
         def parse_migration_output(output):
             """Parse the output of the showmigrations command to get migration names."""
             lines = output.splitlines()
@@ -210,11 +213,11 @@ class migrate:
             # Load all migrations using Django's migration loader
             loader = MigrationLoader(connection)
             squashed_replacements = set()
-            for key, migration in loader.disk_migrations.items():
+            for _key, migration in loader.disk_migrations.items():
                 if hasattr(migration, "replaces"):
                     squashed_replacements.update(migration.replaces)
 
-            deployed_migrations: Dict = {}
+            deployed_migrations: dict = {}
             with connection.cursor() as cursor:
                 cursor.execute(
                     """
@@ -223,7 +226,7 @@ class migrate:
                     ORDER BY app, deployed DESC
                 """
                 )
-                for app, name, deployed in cursor.fetchall():
+                for app, name, _deployed in cursor.fetchall():
                     # skip migrations that are part of a squashed migration
                     if (app, name) in squashed_replacements:
                         continue

lamindb_setup/_register_instance.py

@@ -1,13 +1,19 @@
-from .core.django import setup_django
+from __future__ import annotations
+
+from lamin_utils import logger
+
 from .core._settings import settings
 from .core._settings_storage import base62
+from .core.django import setup_django
 
 
 def register(_test: bool = False):
     """Register an instance on the hub."""
+    from ._check_setup import _check_instance_setup
     from .core._hub_core import init_instance as init_instance_hub
     from .core._hub_core import init_storage as init_storage_hub
-    from ._check_setup import _check_instance_setup
+
+    logger.warning("""lamin register will be removed soon""")
 
     isettings = settings.instance
     if not _check_instance_setup() and not _test:
@@ -17,6 +23,7 @@ def register(_test: bool = False):
     if ssettings._uid is None and _test:
         # because django isn't up, we can't get it from the database
         ssettings._uid = base62(8)
-    ssettings._uuid_ = init_storage_hub(ssettings)
+    init_storage_hub(ssettings)
     init_instance_hub(isettings)
+    isettings._is_on_hub = True
     isettings._persist()

lamindb_setup/_schema.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from django.urls import path
 from lamin_utils import logger
 
@@ -13,11 +15,12 @@ urlpatterns = [
 
 
 def view():
-    from .core.django import setup_django
-    from .core._settings import settings
-    from ._check_setup import _check_instance_setup
     from django.core.management import call_command
 
+    from ._check_setup import _check_instance_setup
+    from .core._settings import settings
+    from .core.django import setup_django
+
     if _check_instance_setup():
         raise RuntimeError("Restart Python session or use CLI!")
     setup_django(settings.instance, view_schema=True)

lamindb_setup/_set_managed_storage.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from lamin_utils import logger
+
+from ._init_instance import register_storage_in_instance
+from .core._settings import settings
+from .core._settings_storage import init_storage
+
+if TYPE_CHECKING:
+    from lamindb_setup.core.types import UPathStr
+
+
+def set_managed_storage(root: UPathStr, **fs_kwargs):
+    """Add or switch to another managed storage location.
+
+    Args:
+        root: `UPathStr` - The new storage root, e.g., an S3 bucket.
+        **fs_kwargs: Additional fsspec arguments for cloud root, e.g., profile.
+
+    """
+    if settings.instance.dialect == "sqlite":
+        raise ValueError(
+            "Can't add additional managed storage locations for sqlite instances."
+        )
+    if not settings.instance.is_on_hub:
+        raise ValueError(
+            "Can't add additional managed storage locations for instances that aren't managed through the hub."
+        )
+    ssettings = init_storage(
+        root=root, instance_id=settings.instance._id, register_hub=True
+    )
+    settings.instance._storage = ssettings
+    settings.instance._persist()  # this also updates the settings object
+    register_storage_in_instance(ssettings)
+    settings.storage._set_fs_kwargs(**fs_kwargs)

lamindb_setup/_setup_user.py

@@ -1,4 +1,6 @@
-from typing import Union, Optional
+from __future__ import annotations
+
+from typing import Optional, Union
 
 from lamin_utils import logger
 
@@ -8,15 +10,13 @@ from .core._settings import settings
 from .core._settings_load import load_or_create_user_settings, load_user_settings
 from .core._settings_save import save_user_settings
 from .core._settings_store import (
+    current_user_settings_file,
     user_settings_file_email,
     user_settings_file_handle,
-    current_user_settings_file,
 )
 
 
-def load_user(
-    email: Optional[str] = None, handle: Optional[str] = None
-) -> Union[str, None]:
+def load_user(email: str | None = None, handle: str | None = None) -> str | None:
     if email is not None:
         settings_file = user_settings_file_email(email)
     if handle is not None:
@@ -46,8 +46,8 @@ def load_user(
 def login(
     user: str,
     *,
-    key: Optional[str] = None,
-    password: Optional[str] = None,  # for backward compat
+    key: str | None = None,
+    password: str | None = None,  # for backward compat
 ) -> None:
     """Log in user.
 

lamindb_setup/_silence_loggers.py

@@ -1,9 +1,11 @@
+from __future__ import annotations
+
 import logging
 
 silenced = False
 
 
-# from https://github.com/boto/boto3/blob/8c6e641bed8130a9d8cb4d97b4acbe7aa0d0657a/boto3/__init__.py#L37  # noqa
+# from https://github.com/boto/boto3/blob/8c6e641bed8130a9d8cb4d97b4acbe7aa0d0657a/boto3/__init__.py#L37
 def set_stream_logger(name, level):
     logger = logging.getLogger(name)
     logger.setLevel(level)
@@ -24,7 +26,7 @@ def silence_loggers():
         set_stream_logger(name="botocore.endpoint", level=logging.WARNING)
         set_stream_logger(name="httpx", level=logging.WARNING)
         try:
-            import aiobotocore  # noqa
+            import aiobotocore
 
             # the 7th logging message of credentials came from aiobotocore
             set_stream_logger(name="aiobotocore.credentials", level=logging.WARNING)

lamindb_setup/core/__init__.py

@@ -1,19 +1,20 @@
-"""Core library.
+"""Core setup library.
 
 Settings:
 
 .. autosummary::
    :toctree:
 
+   SetupSettings
    UserSettings
    InstanceSettings
    StorageSettings
 
 """
-from . import types
-from . import upath, django
+from . import django, types, upath
 from ._deprecated import deprecated
 from ._docs import doc_args
+from ._settings import SetupSettings
 from ._settings_instance import InstanceSettings
 from ._settings_storage import StorageSettings
 from ._settings_user import UserSettings

lamindb_setup/core/_aws_storage.py

@@ -1,3 +1,6 @@
+from __future__ import annotations
+
+
 def get_location(ip="ipinfo.io"):
     import requests  # type: ignore
 

lamindb_setup/core/_deprecated.py

@@ -1,22 +1,18 @@
-# BSD 3-Clause License
+from __future__ import annotations
 
+# BSD 3-Clause License
 # Copyright (c) 2017-2018 P. Angerer, F. Alexander Wolf, Theis Lab
 # All rights reserved.
-
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
-
 # * Redistributions of source code must retain the above copyright notice, this
 #   list of conditions and the following disclaimer.
-
 # * Redistributions in binary form must reproduce the above copyright notice,
 #   this list of conditions and the following disclaimer in the documentation
 #   and/or other materials provided with the distribution.
-
 # * Neither the name of the copyright holder nor the names of its
 #   contributors may be used to endorse or promote products derived from
 #   this software without specific prior written permission.
-
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
@@ -27,7 +23,6 @@
 # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
 import warnings
 from functools import wraps
 

lamindb_setup/core/_docs.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from textwrap import dedent
 
 

lamindb_setup/core/_hub_client.py

@@ -1,11 +1,13 @@
+from __future__ import annotations
+
 import os
-from typing import Optional
-from supabase.lib.client_options import ClientOptions
 from urllib.request import urlretrieve
-from supabase import create_client, Client
-from pydantic import BaseSettings
+
 from gotrue.errors import AuthUnknownError
 from lamin_utils import logger
+from pydantic import BaseSettings
+from supabase import Client, create_client  # type: ignore
+from supabase.lib.client_options import ClientOptions
 
 
 class Connector(BaseSettings):
@@ -21,7 +23,7 @@ def load_fallback_connector() -> Connector:
 
 
 PROD_URL = "https://hub.lamin.ai"
-PROD_ANON_KEY = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImxhZXNhdW1tZHlkbGxwcGdmY2h1Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2NTY4NDA1NTEsImV4cCI6MTk3MjQxNjU1MX0.WUeCRiun0ExUxKIv5-CtjF6878H8u26t0JmCWx3_2-c"  # noqa
+PROD_ANON_KEY = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImxhZXNhdW1tZHlkbGxwcGdmY2h1Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2NTY4NDA1NTEsImV4cCI6MTk3MjQxNjU1MX0.WUeCRiun0ExUxKIv5-CtjF6878H8u26t0JmCWx3_2-c"
 
 
 class Environment:
@@ -40,13 +42,13 @@ class Environment:
                 key = connector.key
         elif lamin_env == "staging":
             url = "https://amvrvdwndlqdzgedrqdv.supabase.co"
-            key = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImFtdnJ2ZHduZGxxZHpnZWRycWR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2NzcxNTcxMzMsImV4cCI6MTk5MjczMzEzM30.Gelt3dQEi8tT4j-JA36RbaZuUvxRnczvRr3iyRtzjY0"  # noqa
+            key = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6ImFtdnJ2ZHduZGxxZHpnZWRycWR2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2NzcxNTcxMzMsImV4cCI6MTk5MjczMzEzM30.Gelt3dQEi8tT4j-JA36RbaZuUvxRnczvRr3iyRtzjY0"
         elif lamin_env == "staging-test":
             url = "https://iugyyajllqftbpidapak.supabase.co"
-            key = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Iml1Z3l5YWpsbHFmdGJwaWRhcGFrIiwicm9sZSI6ImFub24iLCJpYXQiOjE2OTQyMjYyODMsImV4cCI6MjAwOTgwMjI4M30.s7B0gMogFhUatMSwlfuPJ95kWhdCZMn1ROhZ3t6Og90"  # noqa
+            key = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Iml1Z3l5YWpsbHFmdGJwaWRhcGFrIiwicm9sZSI6ImFub24iLCJpYXQiOjE2OTQyMjYyODMsImV4cCI6MjAwOTgwMjI4M30.s7B0gMogFhUatMSwlfuPJ95kWhdCZMn1ROhZ3t6Og90"
         elif lamin_env == "prod-test":
             url = "https://xtdacpwiqwpbxsatoyrv.supabase.co"
-            key = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Inh0ZGFjcHdpcXdwYnhzYXRveXJ2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2OTQyMjYxNDIsImV4cCI6MjAwOTgwMjE0Mn0.Dbi27qujTt8Ei9gfp9KnEWTYptE5KUbZzEK6boL46k4"  # noqa
+            key = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Inh0ZGFjcHdpcXdwYnhzYXRveXJ2Iiwicm9sZSI6ImFub24iLCJpYXQiOjE2OTQyMjYxNDIsImV4cCI6MjAwOTgwMjE0Mn0.Dbi27qujTt8Ei9gfp9KnEWTYptE5KUbZzEK6boL46k4"
         else:
             url = os.environ["SUPABASE_API_URL"]
             key = os.environ["SUPABASE_ANON_KEY"]
@@ -66,7 +68,7 @@ def connect_hub(
 def connect_hub_with_auth(
     fallback_env: bool = False,
     renew_token: bool = False,
-    access_token: Optional[str] = None,
+    access_token: str | None = None,
 ) -> Client:
     hub = connect_hub(fallback_env=fallback_env)
     if access_token is None:
@@ -83,7 +85,7 @@ def connect_hub_with_auth(
 
 
 # runs ~0.5s
-def get_access_token(email: Optional[str] = None, password: Optional[str] = None):
+def get_access_token(email: str | None = None, password: str | None = None):
     hub = connect_hub()
     try:
         auth_response = hub.auth.sign_in_with_password(