lamindb_setup 0.78.0__py3-none-any.whl

lamindb_setup/_init_instance.py

@@ -0,0 +1,411 @@
+from __future__ import annotations
+
+import importlib
+import os
+import sys
+import uuid
+from typing import TYPE_CHECKING, Literal
+from uuid import UUID
+
+from django.core.exceptions import FieldError
+from django.db.utils import OperationalError, ProgrammingError
+from lamin_utils import logger
+
+from ._close import close as close_instance
+from ._silence_loggers import silence_loggers
+from .core import InstanceSettings
+from .core._settings import settings
+from .core._settings_instance import is_local_db_url
+from .core._settings_storage import StorageSettings, init_storage
+from .core.upath import UPath
+
+if TYPE_CHECKING:
+    from pydantic import PostgresDsn
+
+    from .core._settings_user import UserSettings
+    from .core.types import UPathStr
+
+
+def get_schema_module_name(schema_name) -> str:
+    import importlib.util
+
+    name_attempts = [f"lnschema_{schema_name.replace('-', '_')}", schema_name]
+    for name in name_attempts:
+        module_spec = importlib.util.find_spec(name)
+        if module_spec is not None:
+            return name
+    raise ImportError(
+        f"Python package for '{schema_name}' is not installed.\nIf your package is on PyPI, run `pip install {schema_name}`"
+    )
+
+
+def register_storage_in_instance(ssettings: StorageSettings):
+    from lnschema_core.models import Storage
+    from lnschema_core.users import current_user_id
+
+    from .core.hashing import hash_and_encode_as_b62
+
+    if ssettings._instance_id is not None:
+        instance_uid = hash_and_encode_as_b62(ssettings._instance_id.hex)[:12]
+    else:
+        instance_uid = None
+    # how do we ensure that this function is only called passing
+    # the managing instance?
+    defaults = {
+        "root": ssettings.root_as_str,
+        "type": ssettings.type,
+        "region": ssettings.region,
+        "instance_uid": instance_uid,
+        "created_by_id": current_user_id(),
+        "run": None,
+    }
+    if ssettings._uid is not None:
+        defaults["uid"] = ssettings._uid
+    storage, _ = Storage.objects.update_or_create(
+        root=ssettings.root_as_str,
+        defaults=defaults,
+    )
+    return storage
+
+
+def register_user(usettings):
+    from lnschema_core.models import User
+
+    try:
+        # need to have try except because of integer primary key migration
+        user, created = User.objects.update_or_create(
+            uid=usettings.uid,
+            defaults={
+                "handle": usettings.handle,
+                "name": usettings.name,
+            },
+        )
+    # for users with only read access, except via ProgrammingError
+    # ProgrammingError: permission denied for table lnschema_core_user
+    except (OperationalError, FieldError, ProgrammingError):
+        pass
+
+
+def register_user_and_storage_in_instance(isettings: InstanceSettings, usettings):
+    """Register user & storage in DB."""
+    from django.db.utils import OperationalError
+
+    try:
+        register_user(usettings)
+        register_storage_in_instance(isettings.storage)
+    except OperationalError as error:
+        logger.warning(f"instance seems not set up ({error})")
+
+
+def reload_schema_modules(isettings: InstanceSettings, include_core: bool = True):
+    schema_names = ["core"] if include_core else []
+    # schema_names += list(isettings.schema)
+    schema_module_names = [get_schema_module_name(n) for n in schema_names]
+
+    for schema_module_name in schema_module_names:
+        if schema_module_name in sys.modules:
+            schema_module = importlib.import_module(schema_module_name)
+            importlib.reload(schema_module)
+
+
+def reload_lamindb_itself(isettings) -> bool:
+    reloaded = False
+    if "lamindb" in sys.modules:
+        import lamindb
+
+        importlib.reload(lamindb)
+        reloaded = True
+    return reloaded
+
+
+def reload_lamindb(isettings: InstanceSettings):
+    log_message = settings.auto_connect
+    if not reload_lamindb_itself(isettings):
+        log_message = True
+    if log_message:
+        logger.important(f"connected lamindb: {isettings.slug}")
+
+
+ERROR_SQLITE_CACHE = """
+Your cached local SQLite file exists, while your cloud SQLite file ({}) doesn't.
+Either delete your cache ({}) or add it back to the cloud (if delete was accidental).
+"""
+
+
+def process_connect_response(
+    response: tuple | str, instance_identifier: str
+) -> tuple[
+    UUID,
+    Literal[
+        "instance-corrupted-or-deleted", "account-not-exists", "instance-not-found"
+    ],
+]:
+    # for internal use when creating instances through CICD
+    if isinstance(response, tuple) and response[0] == "instance-corrupted-or-deleted":
+        hub_result = response[1]
+        instance_state = response[0]
+        instance_id = UUID(hub_result["id"])
+    else:
+        instance_id_str = os.getenv("LAMINDB_INSTANCE_ID_INIT")
+        if instance_id_str is None:
+            instance_id = uuid.uuid5(uuid.NAMESPACE_URL, instance_identifier)
+        else:
+            instance_id = UUID(instance_id_str)
+        instance_state = response
+    return instance_id, instance_state
+
+
+def validate_init_args(
+    *,
+    storage: UPathStr,
+    name: str | None = None,
+    db: PostgresDsn | None = None,
+    schema: str | None = None,
+    _test: bool = False,
+    _write_settings: bool = True,
+    _user: UserSettings | None = None,
+) -> tuple[
+    str,
+    UUID | None,
+    Literal[
+        "connected",
+        "instance-corrupted-or-deleted",
+        "account-not-exists",
+        "instance-not-found",
+    ],
+    str,
+]:
+    from ._connect_instance import connect
+    from .core._hub_utils import (
+        validate_schema_arg,
+    )
+
+    # should be called as the first thing
+    name_str = infer_instance_name(storage=storage, name=name, db=db)
+    owner_str = settings.user.handle if _user is None else _user.handle
+    # test whether instance exists by trying to load it
+    instance_slug = f"{owner_str}/{name_str}"
+    response = connect(
+        instance_slug,
+        _db=db,
+        _raise_not_found_error=False,
+        _test=_test,
+        _write_settings=_write_settings,
+        _user=_user,
+    )
+    instance_state: Literal[
+        "connected",
+        "instance-corrupted-or-deleted",
+        "account-not-exists",
+        "instance-not-found",
+    ] = "connected"
+    instance_id = None
+    if response is not None:
+        instance_id, instance_state = process_connect_response(response, instance_slug)
+    schema = validate_schema_arg(schema)
+    return name_str, instance_id, instance_state, instance_slug
+
+
+MESSAGE_NO_MULTIPLE_INSTANCE = """
+Currently don't support subsequent connection to different databases in the same
+Python session.\n
+Try running on the CLI: lamin settings set auto-connect false
+"""
+
+
+def init(
+    *,
+    storage: UPathStr,
+    name: str | None = None,
+    db: PostgresDsn | None = None,
+    schema: str | None = None,
+    **kwargs,
+) -> None:
+    """Create and load a LaminDB instance.
+
+    Args:
+        storage: Either ``"create-s3"``, local or
+            remote folder (``"s3://..."`` or ``"gs://..."``).
+        name: Instance name.
+        db: Database connection url, do not pass for SQLite.
+        schema: Comma-separated string of schema modules. None if not set.
+    """
+    isettings = None
+    ssettings = None
+
+    _write_settings: bool = kwargs.get("_write_settings", True)
+    _test: bool = kwargs.get("_test", False)
+
+    # use this user instead of settings.user
+    # contains access_token
+    _user: UserSettings | None = kwargs.get("_user", None)
+    user_handle: str = settings.user.handle if _user is None else _user.handle
+    user__uuid: UUID = settings.user._uuid if _user is None else _user._uuid  # type: ignore
+    access_token: str | None = None if _user is None else _user.access_token
+
+    try:
+        silence_loggers()
+        from ._check_setup import _check_instance_setup
+
+        if _check_instance_setup() and not _test:
+            raise RuntimeError(MESSAGE_NO_MULTIPLE_INSTANCE)
+        elif _write_settings:
+            close_instance(mute=True)
+        from .core._hub_core import init_instance as init_instance_hub
+
+        name_str, instance_id, instance_state, _ = validate_init_args(
+            storage=storage,
+            name=name,
+            db=db,
+            schema=schema,
+            _test=_test,
+            _write_settings=_write_settings,
+            _user=_user,  # will get from settings.user if _user is None
+        )
+        if instance_state == "connected":
+            if _write_settings:
+                settings.auto_connect = True  # we can also debate this switch here
+            return None
+        # the conditions here match `isettings.is_remote`, but I currently don't
+        # see a way of making this more elegant; should become possible if we
+        # remove the instance.storage_id FK on the hub
+        prevent_register_hub = is_local_db_url(db) if db is not None else False
+        ssettings, _ = init_storage(
+            storage,
+            instance_id=instance_id,
+            init_instance=True,
+            prevent_register_hub=prevent_register_hub,
+            created_by=user__uuid,
+            access_token=access_token,
+        )
+        isettings = InstanceSettings(
+            id=instance_id,  # type: ignore
+            owner=user_handle,
+            name=name_str,
+            storage=ssettings,
+            db=db,
+            schema=schema,
+            uid=ssettings.uid,
+            # to lock passed user in isettings._cloud_sqlite_locker.lock()
+            _locker_user=_user,  # only has effect if cloud sqlite
+        )
+        register_on_hub = (
+            isettings.is_remote and instance_state != "instance-corrupted-or-deleted"
+        )
+        if register_on_hub:
+            init_instance_hub(
+                isettings, account_id=user__uuid, access_token=access_token
+            )
+        validate_sqlite_state(isettings)
+        # why call it here if it is also called in load_from_isettings?
+        isettings._persist(write_to_disk=_write_settings)
+        if _test:
+            return None
+        isettings._init_db()
+        load_from_isettings(
+            isettings, init=True, user=_user, write_settings=_write_settings
+        )
+        if _write_settings and isettings._is_cloud_sqlite:
+            isettings._cloud_sqlite_locker.lock()
+            logger.warning(
+                "locked instance (to unlock and push changes to the cloud SQLite file,"
+                " call: lamin disconnect)"
+            )
+        if register_on_hub and isettings.dialect != "sqlite":
+            from ._schema_metadata import update_schema_in_hub
+
+            update_schema_in_hub(access_token=access_token)
+        if _write_settings:
+            settings.auto_connect = True
+    except Exception as e:
+        from ._delete import delete_by_isettings
+        from .core._hub_core import delete_instance_record, delete_storage_record
+
+        if isettings is not None:
+            if _write_settings:
+                delete_by_isettings(isettings)
+            else:
+                settings._instance_settings = None
+            if (
+                user_handle != "anonymous" or access_token is not None
+            ) and isettings.is_on_hub:
+                delete_instance_record(isettings._id, access_token=access_token)
+        if (
+            ssettings is not None
+            and (user_handle != "anonymous" or access_token is not None)
+            and ssettings.is_on_hub
+        ):
+            delete_storage_record(ssettings._uuid, access_token=access_token)  # type: ignore
+        raise e
+    return None
+
+
+def load_from_isettings(
+    isettings: InstanceSettings,
+    *,
+    init: bool = False,
+    user: UserSettings | None = None,
+    write_settings: bool = True,
+) -> None:
+    from .core._setup_bionty_sources import write_bionty_sources
+
+    user = settings.user if user is None else user
+
+    if init:
+        # during init both user and storage need to be registered
+        register_user_and_storage_in_instance(isettings, user)
+        write_bionty_sources(isettings)
+        isettings._update_cloud_sqlite_file(unlock_cloud_sqlite=False)
+    else:
+        # when loading, django is already set up
+        #
+        # only register user if the instance is connected
+        # for the first time in an environment
+        # this is our best proxy for that the user might not
+        # yet be registered
+        if not isettings._get_settings_file().exists():
+            register_user(user)
+    isettings._persist(write_to_disk=write_settings)
+    reload_lamindb(isettings)
+
+
+def validate_sqlite_state(isettings: InstanceSettings) -> None:
+    if isettings._is_cloud_sqlite:
+        if (
+            # it's important to first evaluate the existence check
+            # for the local sqlite file because it doesn't need a network
+            # request
+            isettings._sqlite_file_local.exists()
+            and not isettings._sqlite_file.exists()
+        ):
+            raise RuntimeError(
+                ERROR_SQLITE_CACHE.format(
+                    isettings._sqlite_file, isettings._sqlite_file_local
+                )
+            )
+
+
+def infer_instance_name(
+    *,
+    storage: UPathStr,
+    name: str | None = None,
+    db: PostgresDsn | None = None,
+) -> str:
+    if name is not None:
+        if "/" in name:
+            raise ValueError("Invalid instance name: '/' delimiter not allowed.")
+        return name
+    if db is not None:
+        logger.warning("using the sql database name for the instance name")
+        # this isn't a great way to access the db name
+        # could use LaminDsn instead
+        return str(db).split("/")[-1]
+    if storage == "create-s3":
+        raise ValueError("pass name to init if storage = 'create-s3'")
+    storage_path = UPath(storage)
+    if storage_path.name != "":
+        name = storage_path.name
+    else:
+        # dedicated treatment of bucket names
+        name = storage_path._url.netloc
+    return name.lower()

lamindb_setup/_migrate.py

@@ -0,0 +1,239 @@
+from __future__ import annotations
+
+from django.db import connection
+from django.db.migrations.loader import MigrationLoader
+from lamin_utils import logger
+from packaging import version
+
+from ._check_setup import _check_instance_setup
+from .core._settings import settings
+from .core.django import setup_django
+
+
+# for the django-based synching code, see laminhub_rest
+def check_whether_migrations_in_sync(db_version_str: str):
+    from importlib import metadata
+
+    try:
+        installed_version_str = metadata.version("lamindb")
+    except metadata.PackageNotFoundError:
+        return None
+    if db_version_str is None:
+        logger.warning("no lamindb version stored to compare with installed version")
+        return None
+    installed_version = version.parse(installed_version_str)
+    db_version = version.parse(db_version_str)
+    if (
+        installed_version.major < db_version.major
+        or installed_version.minor < db_version.minor
+    ):
+        db_version_lower = f"{db_version.major}.{db_version.minor}"
+        db_version_upper = f"{db_version.major}.{db_version.minor + 1}"
+        logger.warning(
+            f"the database ({db_version_str}) is ahead of your installed lamindb"
+            f" package ({installed_version_str})"
+        )
+        logger.important(
+            "please update lamindb: pip install"
+            f' "lamindb>={db_version_lower},<{db_version_upper}"'
+        )
+    elif (
+        installed_version.major > db_version.major
+        or installed_version.minor > db_version.minor
+    ):
+        logger.warning(
+            f"the database ({db_version_str}) is behind your installed lamindb package"
+            f" ({installed_version_str})"
+        )
+        logger.important("consider migrating your database: lamin migrate deploy")
+
+
+# for tests, see lamin-cli
+class migrate:
+    """Manage migrations.
+
+    Examples:
+
+    >>> import lamindb as ln
+    >>> ln.setup.migrate.create()
+    >>> ln.setup.migrate.deploy()
+    >>> ln.setup.migrate.check()
+
+    """
+
+    @classmethod
+    def create(cls) -> None:
+        """Create a migration."""
+        if _check_instance_setup():
+            raise RuntimeError("Restart Python session to create migration or use CLI!")
+        setup_django(settings.instance, create_migrations=True)
+
+    @classmethod
+    def deploy(cls) -> None:
+        """Deploy a migration."""
+        from ._schema_metadata import update_schema_in_hub
+
+        if _check_instance_setup():
+            raise RuntimeError("Restart Python session to migrate or use CLI!")
+        from lamindb_setup.core._hub_client import call_with_fallback_auth
+        from lamindb_setup.core._hub_crud import (
+            select_collaborator,
+            update_instance,
+        )
+
+        if settings.instance.is_on_hub:
+            # double check that user is an admin, otherwise will fail below
+            # due to insufficient SQL permissions with cryptic error
+            collaborator = call_with_fallback_auth(
+                select_collaborator,
+                instance_id=settings.instance._id,
+                account_id=settings.user._uuid,
+            )
+            if collaborator is None or collaborator["role"] != "admin":
+                raise SystemExit(
+                    "❌ Only admins can deploy migrations, please ensure that you're an"
+                    f" admin: https://lamin.ai/{settings.instance.slug}/settings"
+                )
+            # we need lamindb to be installed, otherwise we can't populate the version
+            # information in the hub
+            import lamindb
+
+        # this sets up django and deploys the migrations
+        setup_django(settings.instance, deploy_migrations=True)
+        # this populates the hub
+        if settings.instance.is_on_hub:
+            logger.important(f"updating lamindb version in hub: {lamindb.__version__}")
+            # TODO: integrate update of instance table within update_schema_in_hub & below
+            if settings.instance.dialect != "sqlite":
+                update_schema_in_hub()
+            call_with_fallback_auth(
+                update_instance,
+                instance_id=settings.instance._id.hex,
+                instance_fields={"lamindb_version": lamindb.__version__},
+            )
+
+    @classmethod
+    def check(cls) -> bool:
+        """Check whether Registry definitions are in sync with migrations."""
+        from django.core.management import call_command
+
+        setup_django(settings.instance)
+        try:
+            call_command("makemigrations", check_changes=True)
+        except SystemExit:
+            logger.error(
+                "migrations are not in sync with ORMs, please create a migration: lamin"
+                " migrate create"
+            )
+            return False
+        return True
+
+    @classmethod
+    def squash(
+        cls, package_name, migration_nr, start_migration_nr: str | None = None
+    ) -> None:
+        """Squash migrations."""
+        from django.core.management import call_command
+
+        setup_django(settings.instance)
+        if start_migration_nr is not None:
+            call_command(
+                "squashmigrations", package_name, start_migration_nr, migration_nr
+            )
+        else:
+            call_command("squashmigrations", package_name, migration_nr)
+
+    @classmethod
+    def show(cls) -> None:
+        """Show migrations."""
+        from django.core.management import call_command
+
+        setup_django(settings.instance)
+        call_command("showmigrations")
+
+    @classmethod
+    def defined_migrations(cls, latest: bool = False):
+        from io import StringIO
+
+        from django.core.management import call_command
+
+        def parse_migration_output(output):
+            """Parse the output of the showmigrations command to get migration names."""
+            lines = output.splitlines()
+
+            # Initialize an empty dict to store migration names of each module
+            migration_names = {}
+
+            # Process each line
+            for line in lines:
+                if " " not in line:
+                    # CLI displays the module name in bold
+                    name = line.strip().replace("\x1b[1m", "")
+                    migration_names[name] = []
+                    continue
+                # Strip whitespace and split the line into status and migration name
+                migration_name = line.strip().split("] ")[-1].split(" ")[0]
+                # The second part is the migration name
+                migration_names[name].append(migration_name)
+
+            return migration_names
+
+        out = StringIO()
+        call_command("showmigrations", stdout=out)
+        out.seek(0)
+        output = out.getvalue()
+        if latest:
+            return {k: v[-1] for k, v in parse_migration_output(output).items()}
+        else:
+            return parse_migration_output(output)
+
+    @classmethod
+    def deployed_migrations(cls, latest: bool = False):
+        """Get the list of deployed migrations from Migration table in DB."""
+        if latest:
+            latest_migrations = {}
+            with connection.cursor() as cursor:
+                # query to get the latest migration for each app that is not squashed
+                cursor.execute(
+                    """
+                    SELECT app, name
+                    FROM django_migrations
+                    WHERE id IN (
+                        SELECT MAX(id)
+                        FROM django_migrations
+                        WHERE name NOT LIKE '%%_squashed_%%'
+                        GROUP BY app
+                    )
+                """
+                )
+                # fetch all the results
+                for app, name in cursor.fetchall():
+                    latest_migrations[app] = name
+
+            return latest_migrations
+        else:
+            # Load all migrations using Django's migration loader
+            loader = MigrationLoader(connection)
+            squashed_replacements = set()
+            for _key, migration in loader.disk_migrations.items():
+                if hasattr(migration, "replaces"):
+                    squashed_replacements.update(migration.replaces)
+
+            deployed_migrations: dict = {}
+            with connection.cursor() as cursor:
+                cursor.execute(
+                    """
+                    SELECT app, name, deployed
+                    FROM django_migrations
+                    ORDER BY app, deployed DESC
+                """
+                )
+                for app, name, _deployed in cursor.fetchall():
+                    # skip migrations that are part of a squashed migration
+                    if (app, name) in squashed_replacements:
+                        continue
+
+                    if app not in deployed_migrations:
+                        deployed_migrations[app] = []
+                    deployed_migrations[app].append(name)
+            return deployed_migrations

lamindb_setup/_register_instance.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from lamin_utils import logger
+
+from .core._settings import settings
+from .core._settings_storage import base62
+from .core.django import setup_django
+
+
+def register(_test: bool = False):
+    """Register an instance on the hub."""
+    from ._check_setup import _check_instance_setup
+    from .core._hub_core import init_instance as init_instance_hub
+    from .core._hub_core import init_storage as init_storage_hub
+
+    logger.warning("""lamin register will be removed soon""")
+
+    isettings = settings.instance
+    if not _check_instance_setup() and not _test:
+        setup_django(isettings)
+
+    ssettings = settings.instance.storage
+    if ssettings._uid is None and _test:
+        # because django isn't up, we can't get it from the database
+        ssettings._uid = base62(8)
+    # cannot yet populate the instance id here
+    ssettings._instance_id = None
+    # flag auto_populate_instance can be removed once FK migration is over
+    init_storage_hub(ssettings, auto_populate_instance=False)
+    init_instance_hub(isettings)
+    isettings._is_on_hub = True
+    isettings._persist()
+    if isettings.dialect != "sqlite" and not _test:
+        from ._schema_metadata import update_schema_in_hub
+
+        update_schema_in_hub()