lamindb_setup 1.16.0__py3-none-any.whl → 1.18.0__py3-none-any.whl

lamindb_setup/core/_aws_options.py

@@ -20,7 +20,7 @@ lamin_env = os.getenv("LAMIN_ENV")
 if lamin_env is None or lamin_env == "prod":
     HOSTED_BUCKETS = tuple([f"s3://lamin-{region}" for region in HOSTED_REGIONS])
 else:
-    logger.warning("loaded LAMIN_ENV: staging")
+    logger.warning(f"loaded LAMIN_ENV: {lamin_env}")
     HOSTED_BUCKETS = ("s3://lamin-hosted-test",)  # type: ignore
 
 
@@ -60,6 +60,7 @@ class AWSOptionsManager:
         from aiobotocore.session import AioSession
         from s3fs import S3FileSystem
 
+        anon_env = os.getenv("LAMIN_S3_ANON") == "true"
         # this is cached so will be resued with the connection initialized
         # these options are set for paths in _path_inject_options
         # here we set the same options to cache the filesystem
@@ -68,19 +69,28 @@ class AWSOptionsManager:
             use_listings_cache=True,
             version_aware=False,
             config_kwargs={"max_pool_connections": 64},
+            anon=anon_env,
         )
 
         self._suppress_aiobotocore_traceback_logging()
 
-        try:
-            fs.connect()
-            self.anon: bool = fs.session._credentials is None
-        except Exception as e:
+        if anon_env:
+            self.anon: bool = True
             logger.warning(
-                f"There is a problem with your default AWS Credentials: {e}\n"
-                "`anon` mode will be used for all non-managed buckets."
+                "`anon` mode will be used for all non-managed buckets "
+                "because the environment variable LAMIN_S3_ANON was set to 'true'"
             )
-            self.anon = True
+        else:
+            try:
+                fs.connect()
+                self.anon = fs.session._credentials is None
+            except Exception as e:
+                logger.warning(
+                    f"There is a problem with your default AWS Credentials: {e}\n"
+                    "`anon` mode will be used for all non-managed buckets"
+                )
+                self.anon = True
+
         self.anon_public: bool | None = None
         if not self.anon:
             try:

lamindb_setup/core/_clone.py

@@ -1,142 +1,18 @@
-"""Utilities to copy, clone and load Postgres instances as local SQLite databases.
+"""Utilities to work with Postgres Snapshots.
 
 .. autosummary::
    :toctree:
 
-   init_local_sqlite
-   connect_local_sqlite
-   connect_remote_sqlite
    upload_sqlite_clone
 """
 
 import gzip
-import os
 import shutil
 from pathlib import Path
 
-from lamindb_setup.core._settings_load import load_instance_settings
-from lamindb_setup.core._settings_store import instance_settings_file
-from lamindb_setup.core.django import reset_django
 from lamindb_setup.core.upath import create_path
 
 
-def init_local_sqlite(
-    instance: str | None = None, copy_suffix: str | None = None
-) -> None:
-    """Initialize SQLite copy of an existing Postgres instance.
-
-    Creates a SQLite database with the same schema as the source Postgres instance.
-    The copy shares the same storage location as the original instance.
-
-    The copy is intended for read-only access to instance data without requiring a Postgres connection.
-    Data synchronization to complete the clone happens via a separate Lambda function.
-
-    Note that essential user, branch and storage tables are missing.
-    Therefore, it is not possible to store Artifacts without having replayed these records first.
-
-    Args:
-        instance: Pass a slug (`account/name`) or URL (`https://lamin.ai/account/name`).
-            If `None`, looks for an environment variable `LAMIN_CURRENT_INSTANCE` to get the instance identifier.
-            If it doesn't find this variable, it connects to the instance that was connected with `lamin connect` through the CLI.
-        copy_suffix: Optional suffix to append to the local clone name.
-    """
-    import lamindb_setup as ln_setup
-
-    if instance is None:  # pragma: no cover
-        instance = os.environ.get("LAMIN_CURRENT_INSTANCE")
-
-    if instance is None:
-        raise ValueError(
-            "No instance identifier provided and LAMIN_CURRENT_INSTANCE is not set"
-        )
-
-    if ln_setup.settings.instance is None:  # pragma: no cover
-        ln_setup.connect(instance)
-
-    name = (
-        f"{ln_setup.settings.instance.name}{copy_suffix}"
-        if copy_suffix is not None
-        else ln_setup.settings.instance.name
-    )
-    isettings = ln_setup._connect_instance._connect_instance(
-        owner=ln_setup.settings.instance.owner, name=name
-    )
-    isettings._db = None
-    isettings._is_on_hub = False
-    isettings._fine_grained_access = False
-    name = (
-        f"{isettings.name}{copy_suffix}" if copy_suffix is not None else isettings.name
-    )
-    isettings._name = name
-    isettings._is_clone = True
-    isettings._persist(write_to_disk=True)
-
-    if not isettings._sqlite_file_local.exists():
-        # Reset Django configuration before _init_db() because Django was already configured for the original Postgres instance.
-        # Without this reset, the `if not settings.configured`` check in `setup_django()` would skip reconfiguration,
-        # causing migrations to run against the old Postgres database instead of the new SQLite clone database.
-        reset_django()
-        isettings._init_db()
-
-
-def connect_local_sqlite(
-    instance: str,
-) -> None:
-    """Load a locally stored SQLite instance of which a remote hub Postgres instance exists.
-
-    This function bypasses the hub lookup that `lamin connect` performs, loading the SQLite clone directly from local settings files.
-    The clone must first be created via `init_local_sqlite()`.
-
-    Args:
-        instance: Instance slug in the form `account/name` (e.g., `laminlabs/privatedata-local`).
-    """
-    owner, name = instance.split("/")
-    settings_file = instance_settings_file(name=name, owner=owner)
-
-    if not settings_file.exists():
-        raise ValueError(
-            "SQLite clone not found."
-            " Run `init_local_sqlite()` to create a local copy or connect to a remote copy using `connect_remote_sqlite`."
-        )
-
-    isettings = load_instance_settings(settings_file)
-    isettings._persist(write_to_disk=False)
-
-    # Using `setup_django` instead of `_load_db` to not ping AWS RDS
-    from lamindb_setup._check_setup import disable_auto_connect
-
-    from .django import setup_django
-
-    disable_auto_connect(setup_django)(isettings)
-
-
-def connect_remote_sqlite(instance: str, *, copy_suffix: str | None = None) -> None:
-    """Load an existing SQLite copy of a hub instance.
-
-    Args:
-        instance: Instance slug in the form `account/name` (e.g., `laminlabs/privatedata-local`).
-        copy_suffix: Optional suffix of the local clone.
-    """
-    import lamindb_setup as ln_setup
-
-    owner, name = instance.split("/")
-
-    # Step 1: Create the settings file
-    isettings = ln_setup._connect_instance._connect_instance(owner=owner, name=name)
-    isettings._db = None
-    isettings._is_on_hub = False
-    isettings._fine_grained_access = False
-    isettings._db_permissions = "read"
-    name = (
-        f"{isettings.name}{copy_suffix}" if copy_suffix is not None else isettings.name
-    )
-    isettings._name = name
-    isettings._is_clone = True
-    isettings._persist(write_to_disk=True)
-
-    connect_local_sqlite(instance=instance + (copy_suffix or ""))
-
-
 def upload_sqlite_clone(
     local_sqlite_path: Path | str | None = None, compress: bool = True
 ) -> None:

lamindb_setup/core/_hub_client.py

@@ -11,8 +11,7 @@ import httpx
 from httpx_retries import Retry, RetryTransport
 from lamin_utils import logger
 from pydantic_settings import BaseSettings
-from supabase import Client, create_client  # type: ignore
-from supabase.lib.client_options import ClientOptions
+from supabase import Client, ClientOptions, create_client
 
 from ._settings_save import save_user_settings
 

lamindb_setup/core/_hub_core.py

@@ -383,7 +383,8 @@ def _init_instance_hub(
 ) -> None:
     from ._settings import settings
 
-    account_id = settings.user._uuid if account_id is None else account_id
+    created_by_id = settings.user._uuid.hex if account_id is None else account_id.hex  # type: ignore
+    owner_account_id = os.getenv("LAMINDB_ACCOUNT_ID_INIT", created_by_id)
 
     try:
         lamindb_version = metadata.version("lamindb")
@@ -391,13 +392,13 @@ def _init_instance_hub(
         lamindb_version = None
     fields = {
         "id": isettings._id.hex,
-        "account_id": account_id.hex,  # type: ignore
+        "account_id": owner_account_id,
         "name": isettings.name,
         "lnid": isettings.uid,
         "schema_str": isettings._schema_str,
         "lamindb_version": lamindb_version,
         "public": False,
-        "created_by_id": account_id.hex,  # type: ignore
+        "created_by_id": created_by_id,
     }
     if isettings.dialect != "sqlite":
         db_dsn = LaminDsnModel(db=isettings.db)
@@ -407,7 +408,7 @@ def _init_instance_hub(
             "db_port": db_dsn.db.port,
             "db_database": db_dsn.db.database,
         }
-        fields.update(db_fields)
+        fields.update(db_fields)  # type: ignore
     slug = isettings.slug
     # I'd like the following to be an upsert, but this seems to violate RLS
     # Similarly, if we don't specify `returning="minimal"`, we'll violate RLS
@@ -415,7 +416,9 @@ def _init_instance_hub(
     # as then init_instance is no longer idempotent
     try:
         client.table("instance").insert(fields, returning="minimal").execute()
-    except APIError:
+    except APIError as e:
+        if "new row violates row-level security policy" in str(e):
+            raise e
         logger.warning(f"instance already existed at: https://lamin.ai/{slug}")
         return None
     if isettings.dialect != "sqlite" and isettings.is_remote:
@@ -713,7 +716,7 @@ def get_lamin_site_base_url():
 
 
 def sign_up_local_hub(email) -> str | tuple[str, str, str]:
-    # raises gotrue.errors.AuthApiError: User already registered
+    # raises AuthApiError: User already registered
     password = base62(40)  # generate new password
     sign_up_kwargs = {"email": email, "password": password}
     client = connect_hub()

lamindb_setup/core/_private_django_api.py

@@ -34,8 +34,8 @@ def private_django_api(reverse=False):
     # the order here matters
     # changing it might break the tests
     attributes = [
-        "MultipleObjectsReturned",
         "add_to_class",
+        "arefresh_from_db",
         "adelete",
         "asave",
         "clean",
@@ -51,10 +51,8 @@ def private_django_api(reverse=False):
         "validate_unique",
     ]
     if reverse:
-        attributes.append("arefresh_from_db")
         attributes.append("full_clean")
     else:
-        attributes.append("a_refresh_from_db")
         attributes.append("full__clean")
 
     django_path = Path(db.__file__).parent.parent

lamindb_setup/core/_settings.py

@@ -46,6 +46,12 @@ def _process_cache_path(cache_path: UPathStr | None) -> UPath | None:
     return cache_dir
 
 
+# returned by settings.branch for none/none instance
+class MainBranchMock:
+    id = 1
+    name = "main"
+
+
 class SetupSettings:
     """Setup settings."""
 
@@ -140,6 +146,10 @@ class SetupSettings:
     # and we never need a DB request
     def branch(self) -> Branch:
         """Default branch."""
+        # this is needed for .filter() with non-default connections
+        if not self._instance_exists:
+            return MainBranchMock()
+
         if self._branch is None:
             from lamindb import Branch
 
@@ -222,10 +232,9 @@ class SetupSettings:
         If `True`, the current instance is connected, meaning that the db and other settings
         are properly configured for use.
         """
-        if self._instance_exists:
-            return self.instance.slug != "none/none"
-        else:
-            return False
+        from . import django
+
+        return self._instance_exists and django.IS_SETUP
 
     @property
     def private_django_api(self) -> bool:
@@ -284,12 +293,7 @@ class SetupSettings:
 
     @property
     def _instance_exists(self):
-        try:
-            self.instance  # noqa
-            return True
-        # this is implicit logic that catches if no instance is loaded
-        except CurrentInstanceNotConfigured:
-            return False
+        return self.instance.slug != "none/none"
 
     @property
     def cache_dir(self) -> UPath:

lamindb_setup/core/_settings_instance.py

@@ -125,10 +125,11 @@ class InstanceSettings:
                 if self._local_storage is not None:
                     value_local = self.local_storage
                     representation += f"\n - local storage: {value_local.root_as_str} ({value_local.region})"
-                    representation += (
-                        f"\n - cloud storage: {value.root_as_str} ({value.region})"
-                    )
-                else:
+                    if value is not None:
+                        representation += (
+                            f"\n - cloud storage: {value.root_as_str} ({value.region})"
+                        )
+                elif value is not None:
                     representation += (
                         f"\n - storage: {value.root_as_str} ({value.region})"
                     )
@@ -322,16 +323,6 @@ class InstanceSettings:
         else:
             logger.warning(f"could not set this local storage location: {local_root}")
 
-    @property
-    @deprecated("local_storage")
-    def storage_local(self) -> StorageSettings:
-        return self.local_storage
-
-    @storage_local.setter
-    @deprecated("local_storage")
-    def storage_local(self, local_root_host: tuple[Path | str, str]):
-        self.local_storage = local_root_host  # type: ignore
-
     @property
     def slug(self) -> str:
         """Unique semantic identifier of form `"{account_handle}/{instance_name}"`."""
@@ -417,11 +408,6 @@ class InstanceSettings:
         else:
             return {module for module in self._schema_str.split(",") if module != ""}
 
-    @property
-    @deprecated("modules")
-    def schema(self) -> set[str]:
-        return self.modules
-
     @property
     def _sqlite_file(self) -> UPath:
         """SQLite file."""
@@ -513,17 +499,36 @@ class InstanceSettings:
 
     @property
     def dialect(self) -> Literal["sqlite", "postgresql"]:
-        """SQL dialect."""
+        """SQL dialect.
+
+        Equivalent to :attr:`vendor`.
+
+        "vendor" is the Django terminology for the type of database. "dialect" is the SQLAlchemy terminology.
+        """
         if self._db is None or self._db.startswith("sqlite://"):
             return "sqlite"
         else:
             assert self._db.startswith("postgresql"), f"Unexpected DB value: {self._db}"
             return "postgresql"
 
+    @property
+    def vendor(self) -> Literal["sqlite", "postgresql"]:
+        """Database vendor.
+
+        Equivalent to :attr:`dialect`.
+
+        "vendor" is the Django terminology for the type of database. "dialect" is the SQLAlchemy terminology.
+        """
+        return self.dialect
+
     @property
     def _is_cloud_sqlite(self) -> bool:
         """Is this a cloud instance with sqlite db."""
-        return self.dialect == "sqlite" and self.storage.type_is_cloud
+        return (
+            self.dialect == "sqlite"
+            and self.storage is not None
+            and self.storage.type_is_cloud
+        )
 
     @property
     def _cloud_sqlite_locker(self):
@@ -543,14 +548,16 @@ class InstanceSettings:
     @property
     def is_remote(self) -> bool:
         """Boolean indicating if an instance has no local component."""
+        if self.storage is None and self.db == "sqlite:///:memory:":
+            return False
         return check_is_instance_remote(self.storage.root_as_str, self.db)
 
     @property
     def is_on_hub(self) -> bool:
-        """Is this instance on the hub?
+        """Is this instance registered on the hub?
 
-        Can only reliably establish if user has access to the instance.
-        Will return `False` in case the instance isn't found.
+        Can only establish if user has access to the instance.
+        Will return `False` in case the user token can't find the instance.
         """
         if self._is_on_hub is None:
             from ._hub_client import call_with_fallback_auth
@@ -572,6 +579,15 @@ class InstanceSettings:
                 self._is_on_hub = True
         return self._is_on_hub
 
+    @property
+    def is_managed_by_hub(self) -> bool:
+        """Is this instance managed by the hub?
+
+        Returns `True` if the instance is _managed_ by LaminHub, i.e.,
+        it was connected to LaminHub to manage access, migrations, a REST API, a UI, etc.
+        """
+        return self.api_url is not None
+
     def _get_settings_file(self) -> Path:
         return instance_settings_file(self.name, self.owner)
 

lamindb_setup/core/_settings_load.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+from importlib.util import find_spec
 from pathlib import Path
 from typing import TYPE_CHECKING
 from uuid import UUID, uuid4
@@ -46,19 +47,36 @@ def load_cache_path_from_settings(storage_settings: Path | None = None) -> Path
         return None
 
 
+def find_module_candidates():
+    """Find all local packages that depend on lamindb."""
+    candidates = ["bionty", "wetlab"]
+    return [c for c in candidates if find_spec(c) is not None]
+
+
 def load_instance_settings(instance_settings_file: Path | None = None):
     if instance_settings_file is None:
-        instance_settings_file = current_instance_settings_file()
-    if not instance_settings_file.exists():
+        isettings_file = current_instance_settings_file()
+        if not isettings_file.exists():
+            isettings = InstanceSettings(
+                id=UUID("00000000-0000-0000-0000-000000000000"),
+                owner="none",
+                name="none",
+                storage=None,
+                modules=",".join(find_module_candidates()),
+            )
+            return isettings
+    else:
+        isettings_file = instance_settings_file
+
+    if not isettings_file.exists():
+        # this errors only if the file was explicitly provided
         raise CurrentInstanceNotConfigured
     try:
-        settings_store = InstanceSettingsStore(_env_file=instance_settings_file)
+        settings_store = InstanceSettingsStore(_env_file=isettings_file)
     except (ValidationError, TypeError) as error:
-        with open(instance_settings_file) as f:
-            content = f.read()
         raise SettingsEnvFileOutdated(
-            f"\n\n{error}\n\nYour instance settings file with\n\n{content}\nis invalid"
-            f" (likely outdated), see validation error. Please delete {instance_settings_file} &"
+            f"\n\n{error}\n\nYour instance settings file with\n\n{isettings_file.read_text()}\nis invalid"
+            f" (likely outdated), see validation error. Please delete {isettings_file} &"
             " reload (remote) or re-initialize (local) the instance with the same name & storage location."
         ) from error
     isettings = setup_instance_from_store(settings_store)

lamindb_setup/core/_settings_storage.py

@@ -255,11 +255,6 @@ class StorageSettings:
         self._has_local = False
         self._local = None
 
-    @property
-    @deprecated("_id")
-    def id(self) -> int:
-        return self._id
-
     @property
     def _id(self) -> int:
         """Storage id.

lamindb_setup/core/django.py

@@ -5,13 +5,15 @@ import builtins
 import os
 import sys
 import importlib as il
+import gzip
 import jwt
 import time
 import threading
 from pathlib import Path
+import shutil
 from packaging import version
 from ._settings_instance import InstanceSettings, is_local_db_url
-
+from ..errors import CurrentInstanceNotConfigured
 from lamin_utils import logger
 
 
@@ -21,6 +23,24 @@ IS_MIGRATING = False
 CONN_MAX_AGE = 299
 
 
+def get_connection(connection_name: str):
+    from django.db import connections
+
+    return connections[connection_name]
+
+
+def error_no_instance_wrapper(execute, sql, params, many, context):
+    connection = context["connection"]
+
+    if (
+        connection.vendor == "sqlite"
+        and connection.settings_dict.get("NAME") == ":memory:"
+    ):
+        raise CurrentInstanceNotConfigured
+
+    return execute(sql, params, many, context)
+
+
 # db token that refreshes on access if needed
 class DBToken:
     def __init__(
@@ -64,11 +84,6 @@ class DBTokenManager:
 
         self.tokens: dict[str, DBToken] = {}
 
-    def get_connection(self, connection_name: str):
-        from django.db import connections
-
-        return connections[connection_name]
-
     def set(self, token: DBToken, connection_name: str = "default"):
         if connection_name in self.tokens:
             return
@@ -77,11 +92,7 @@ class DBTokenManager:
         from django.db.backends.signals import connection_created
 
         def set_token_wrapper(execute, sql, params, many, context):
-            not_in_atomic_block = (
-                context is None
-                or "connection" not in context
-                or not context["connection"].in_atomic_block
-            )
+            not_in_atomic_block = not context["connection"].in_atomic_block
             # ignore atomic blocks
             if not_in_atomic_block:
                 sql = token.token_query + sql
@@ -98,7 +109,7 @@ class DBTokenManager:
                 result.nextset()
             return result
 
-        self.get_connection(connection_name).execute_wrappers.append(set_token_wrapper)
+        get_connection(connection_name).execute_wrappers.append(set_token_wrapper)
 
         def connection_callback(sender, connection, **kwargs):
             if (
@@ -124,7 +135,7 @@ class DBTokenManager:
                 if connection_name in self.tokens:
                     # here we don't use the connection from the closure
                     # because Atomic is a single class to manage transactions for all connections
-                    connection = self.get_connection(connection_name)
+                    connection = get_connection(connection_name)
                     if len(connection.atomic_blocks) == 1:
                         token = self.tokens[connection_name]
                         # use raw psycopg2 connection here
@@ -142,7 +153,7 @@ class DBTokenManager:
 
         from django.db.backends.signals import connection_created
 
-        connection = self.get_connection(connection_name)
+        connection = get_connection(connection_name)
 
         connection.execute_wrappers = [
             w
@@ -291,6 +302,9 @@ def setup_django(
             django.db.connections._connections = threading.local()
             logger.debug("django.db.connections._connections has been patched")
 
+        # error if trying to query with the default connection without setting up an instance
+        get_connection("default").execute_wrappers.insert(0, error_no_instance_wrapper)
+
         if isettings._fine_grained_access and isettings._db_permissions == "jwt":
             db_token = DBToken(isettings)
             db_token_manager.set(db_token)  # sets for the default connection
@@ -311,6 +325,20 @@ def setup_django(
             call_command("migrate", app_name, app_number, verbosity=2)
         isettings._update_cloud_sqlite_file(unlock_cloud_sqlite=False)
     elif init:
+        modules_beyond_bionty = isettings.modules.copy()
+        compressed_sqlite_path = Path(__file__).parent / "lamin.db.gz"
+        if "bionty" in modules_beyond_bionty:
+            modules_beyond_bionty.remove("bionty")
+        # seed from compressed sqlite file
+        if (
+            isettings.dialect == "sqlite"
+            and os.getenv("LAMINDB_INIT_FROM_SCRATCH", "false") != "true"
+            and len(modules_beyond_bionty) == 0
+            and compressed_sqlite_path.exists()
+        ):
+            with gzip.open(compressed_sqlite_path, "rb") as f_in:
+                with open(isettings._sqlite_file_local, "wb") as f_out:
+                    shutil.copyfileobj(f_in, f_out)
         global IS_MIGRATING
         IS_MIGRATING = True
         call_command("migrate", verbosity=0)

lamindb_setup/core/hashing.py

@@ -88,7 +88,7 @@ def hash_file(
     file_path: Path,
     file_size: int | None = None,
     chunk_size: int | None = 50 * 1024 * 1024,
-) -> tuple[str, str]:
+) -> tuple[int, str, str]:
     with open(file_path, "rb") as fp:
         if file_size is None:
             fp.seek(0, 2)
@@ -107,15 +107,15 @@ def hash_file(
                 hashlib.sha1(first_chunk).digest() + hashlib.sha1(last_chunk).digest()
             ).digest()
             hash_type = "sha1-fl"
-    return to_b64_str(digest)[:HASH_LENGTH], hash_type
+    return file_size, to_b64_str(digest)[:HASH_LENGTH], hash_type
 
 
-def hash_dir(path: Path):
+def hash_dir(path: Path) -> tuple[int, str, str, int]:
     files = (subpath for subpath in path.rglob("*") if subpath.is_file())
 
     def hash_size(file):
-        file_size = file.stat().st_size
-        return hash_file(file, file_size)[0], file_size
+        size, hash, _ = hash_file(file)
+        return hash, size
 
     try:
         n_workers = len(psutil.Process().cpu_affinity())