lamindb_setup 0.78.0__py3-none-any.whl

lamindb_setup/_delete.py

@@ -0,0 +1,141 @@
+from __future__ import annotations
+
+import shutil
+from typing import TYPE_CHECKING
+from uuid import UUID
+
+from lamin_utils import logger
+
+from ._connect_instance import _connect_instance, get_owner_name_from_identifier
+from .core._aws_credentials import HOSTED_BUCKETS
+from .core._hub_core import delete_instance as delete_instance_on_hub
+from .core._hub_core import get_storage_records_for_instance
+from .core._settings import settings
+from .core._settings_storage import StorageSettings
+from .core.upath import LocalPathClasses, check_storage_is_empty
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from .core._settings_instance import InstanceSettings
+
+
+def delete_cache(isettings: InstanceSettings):
+    # avoid init of root
+    root = isettings.storage._root_init
+    if not isinstance(root, LocalPathClasses):
+        cache_dir = settings.cache_dir / root.path
+        if cache_dir.exists():
+            shutil.rmtree(cache_dir)
+
+
+def delete_exclusion_dir(isettings: InstanceSettings) -> None:
+    exclusion_dir = isettings.storage.root / f".lamindb/_exclusion/{isettings._id.hex}"
+    if exclusion_dir.exists():
+        exclusion_dir.rmdir()
+
+
+def delete_by_isettings(isettings: InstanceSettings) -> None:
+    settings_file = isettings._get_settings_file()
+    if settings_file.exists():
+        settings_file.unlink()
+    delete_cache(isettings)
+    if isettings.dialect == "sqlite":
+        try:
+            if isettings._sqlite_file.exists():
+                isettings._sqlite_file.unlink()
+        except PermissionError:
+            logger.warning(
+                "Did not have permission to delete SQLite file:"
+                f" {isettings._sqlite_file}"
+            )
+            pass
+    # unset the global instance settings
+    if settings._instance_exists and isettings.slug == settings.instance.slug:
+        if settings._instance_settings_path.exists():
+            settings._instance_settings_path.unlink()
+        settings._instance_settings = None
+
+
+def delete(slug: str, force: bool = False, require_empty: bool = True) -> int | None:
+    """Delete a LaminDB instance.
+
+    Args:
+        slug: The instance slug `account_handle/instance_name` or URL.
+            If the instance is owned by you, it suffices to pass the instance name.
+        force: Whether to skip the confirmation prompt.
+        require_empty: Whether to check if the instance is empty before deleting.
+    """
+    owner, name = get_owner_name_from_identifier(slug)
+    isettings = _connect_instance(owner, name, raise_permission_error=False)
+    if isettings.dialect != "sqlite":
+        logger.warning(
+            f"delete() does not yet affect your Postgres database at {isettings.db}"
+        )
+    if isettings.is_on_hub and settings.user.handle == "anonymous":
+        logger.warning(
+            "won't delete the hub component of this instance because you're not logged in"
+        )
+    if not force:
+        valid_responses = ["y", "yes"]
+        user_input = (
+            input(f"Are you sure you want to delete instance {isettings.slug}? (y/n) ")
+            .strip()
+            .lower()
+        )
+        if user_input not in valid_responses:
+            return -1
+
+    # the actual deletion process begins here
+    if isettings.dialect == "sqlite" and isettings.is_remote:
+        # delete the exlusion dir first because it's hard to count its objects
+        delete_exclusion_dir(isettings)
+    if isettings.storage.type_is_cloud and isettings.storage.root_as_str.startswith(
+        HOSTED_BUCKETS
+    ):
+        if not require_empty:
+            logger.warning(
+                "hosted storage always has to be empty, ignoring `require_empty`"
+            )
+        require_empty = True
+    # first the default storage
+    n_objects = check_storage_is_empty(
+        isettings.storage.root,
+        raise_error=require_empty,
+        account_for_sqlite_file=isettings.dialect == "sqlite",
+    )
+    if isettings.storage._mark_storage_root.exists():
+        isettings.storage._mark_storage_root.unlink(
+            missing_ok=True  # this is totally weird, but needed on Py3.11
+        )
+    # now everything that's on the hub
+    if settings.user.handle != "anonymous":
+        storage_records = get_storage_records_for_instance(isettings._id)
+        for storage_record in storage_records:
+            if storage_record["root"] == isettings.storage.root_as_str:
+                continue
+            ssettings = StorageSettings(storage_record["root"])  # type: ignore
+            check_storage_is_empty(
+                ssettings.root,  # type: ignore
+                raise_error=require_empty,
+            )
+            if ssettings._mark_storage_root.exists():
+                ssettings._mark_storage_root.unlink(
+                    missing_ok=True  # this is totally weird, but needed on Py3.11
+                )
+    logger.info(f"deleting instance {isettings.slug}")
+    # below we can skip check_storage_is_empty() because we already called
+    # it above
+    if settings.user.handle != "anonymous" and isettings.is_on_hub:
+        # start with deleting things on the hub
+        # this will error if the user doesn't have permission
+        delete_instance_on_hub(isettings._id, require_empty=False)
+    delete_by_isettings(isettings)
+    # if .lndb file was delete, then we might count -1
+    if n_objects <= 0 and isettings.storage.type == "local":
+        # dir is only empty after sqlite file was delete via delete_by_isettings
+        if (isettings.storage.root / ".lamindb").exists():
+            (isettings.storage.root / ".lamindb").rmdir()
+        if isettings.storage.root.exists():
+            isettings.storage.root.rmdir()
+    return None

lamindb_setup/_django.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from .core._settings import settings
+from .core.django import setup_django
+
+
+def django(command: str, package_name: str | None = None, **kwargs):
+    r"""Manage migrations.
+
+    Examples:
+
+    Reset auto-incrementing primary integer ids after a database import:
+
+    >>> import lamindb as ln
+    >>> ln.setup.django("sqlsequencereset", "lnschema_core")
+    BEGIN;
+    SELECT setval(pg_get_serial_sequence('"lnschema_core_user"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lnschema_core_user";  # noqa
+    SELECT setval(pg_get_serial_sequence('"lnschema_core_storage"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lnschema_core_storage";  # noqa
+    COMMIT;
+
+    You can then run the SQL output that you'll see like so:
+
+    >>> sql = \"\"\"BEGIN;
+        SELECT setval(pg_get_serial_sequence('"lnschema_core_user"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lnschema_core_user";  # noqa
+        SELECT setval(pg_get_serial_sequence('"lnschema_core_storage"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lnschema_core_storage";  # noqa
+        COMMIT;\"\"\"
+    >>> from django.db import connection
+    >>> with connection.cursor() as cursor:
+            cursor.execute(sql)
+
+    """
+    from django.core.management import call_command
+
+    setup_django(settings.instance)
+    if package_name is not None:
+        args = [package_name]
+    else:
+        args = []
+    call_command(command, *args, **kwargs)

lamindb_setup/_entry_points.py

@@ -0,0 +1,22 @@
+import sys
+import warnings
+from importlib.metadata import entry_points
+
+
+def call_registered_entry_points(group, **kwargs):
+    """load and call entry points registered under group."""
+    if sys.version_info >= (3, 10):
+        eps = entry_points(group=group)
+    else:
+        eps = entry_points().get(group, [])
+
+    for ep in eps:
+        func = ep.load()
+        try:
+            func(**kwargs)
+        except BaseException as e:
+            warnings.warn(
+                f"Error loading entry point of group {group!r}: {ep} -> {e}",
+                RuntimeWarning,
+                stacklevel=2,
+            )

lamindb_setup/_exportdb.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+from importlib import import_module
+from pathlib import Path
+
+MODELS = {
+    "core": {
+        "Collection": False,
+        "Artifact": False,
+        "Transform": False,
+        "Run": True,
+        "User": False,
+        "Storage": False,
+        "Feature": False,
+        "FeatureSet": False,
+        "ULabel": False,
+    },
+    # "bionty": {
+    #     "Organism": False,
+    #     "Gene": False,
+    #     "Protein": False,
+    #     "CellMarker": False,
+    #     "Tissue": False,
+    #     "CellType": False,
+    #     "Disease": False,
+    #     "CellLine": False,
+    #     "Phenotype": False,
+    #     "Pathway": False,
+    #     "ExperimentalFactor": False,
+    #     "DevelopmentalStage": False,
+    #     "Ethnicity": False,
+    #     "Source": False,
+    # },
+    # "wetlab": {
+    #     "ExperimentType": False,
+    #     "Experiment": False,
+    #     "Well": False,
+    #     "TreatmentTarget": False,
+    #     "Treatment": False,
+    #     "Biosample": False,
+    #     "Techsample": False,
+    # },
+}
+
+
+def exportdb() -> None:
+    directory = Path("./lamindb_export/")
+    directory.mkdir(parents=True, exist_ok=True)
+    import pandas as pd
+
+    import lamindb_setup as ln_setup
+
+    def export_registry(registry, directory):
+        table_name = registry._meta.db_table
+        df = pd.read_sql_table(table_name, ln_setup.settings.instance.db)
+        df.to_parquet(directory / f"{table_name}.parquet", compression=None)
+
+    # export data to parquet files
+    print(f"\nexporting data to parquet files in: {directory}\n")
+    for schema_name, models in MODELS.items():
+        for model_name in models.keys():
+            schema_module = import_module(f"lnschema_{schema_name}")
+            registry = getattr(schema_module, model_name)
+            export_registry(registry, directory)
+            many_to_many_names = [field.name for field in registry._meta.many_to_many]
+            for many_to_many_name in many_to_many_names:
+                link_orm = getattr(registry, many_to_many_name).through
+                export_registry(link_orm, directory)

lamindb_setup/_importdb.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from importlib import import_module
+from pathlib import Path
+
+from ._exportdb import MODELS
+
+
+def import_registry(registry, directory, connection):
+    import pandas as pd
+
+    table_name = registry._meta.db_table
+    df = pd.read_parquet(directory / f"{table_name}.parquet")
+    old_foreign_key_columns = [
+        column for column in df.columns if column.endswith("_old")
+    ]
+    for column in old_foreign_key_columns:
+        df.drop(column, axis=1, inplace=True)
+    df.to_sql(table_name, connection, if_exists="append", index=False)
+
+
+def importdb() -> None:
+    # import data from parquet files
+    directory = Path("./lamindb_export/")
+    if directory.exists():
+        response = input(
+            f"\n\nDo you want to import registries from here: {directory}? (y/n)\n"
+        )
+        if response != "y":
+            return None
+    from sqlalchemy import create_engine, text
+
+    import lamindb_setup as ln_setup
+
+    engine = create_engine(ln_setup.settings.instance.db, echo=False)
+    with engine.begin() as connection:
+        if ln_setup.settings.instance.dialect == "postgresql":
+            connection.execute(text("SET CONSTRAINTS ALL DEFERRED;"))
+        for schema_name, models in MODELS.items():
+            for model_name in models.keys():
+                print(model_name)
+                schema_module = import_module(f"lnschema_{schema_name}")
+                registry = getattr(schema_module, model_name)
+                import_registry(registry, directory, connection)
+                many_to_many_names = [
+                    field.name for field in registry._meta.many_to_many
+                ]
+                for many_to_many_name in many_to_many_names:
+                    link_orm = getattr(registry, many_to_many_name).through
+                    import_registry(link_orm, directory, connection)