lamindb_setup 0.81.3__py3-none-any.whl → 1.0a1__py3-none-any.whl

lamindb_setup/__init__.py

@@ -33,7 +33,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.81.3"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "1.0a1"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os as _os
 import sys as _sys
@@ -51,7 +51,36 @@ from ._register_instance import register
 from ._setup_user import login, logout
 from .core._settings import settings
 
-_TESTING = _os.getenv("LAMIN_TESTING") is not None
+
+def _is_CI_environment() -> bool:
+    ci_env_vars = [
+        "LAMIN_TESTING",  # Set by our nox configurations
+        "CI",  # Commonly set by many CI systems
+        "TRAVIS",  # Travis CI
+        "GITHUB_ACTIONS",  # GitHub Actions
+        "GITLAB_CI",  # GitLab CI/CD
+        "CIRCLECI",  # CircleCI
+        "JENKINS_URL",  # Jenkins
+        "TEAMCITY_VERSION",  # TeamCity
+        "BUILDKITE",  # Buildkite
+        "BITBUCKET_BUILD_NUMBER",  # Bitbucket Pipelines
+        "APPVEYOR",  # AppVeyor
+        "AZURE_HTTP_USER_AGENT",  # Azure Pipelines
+        "BUDDY",  # Buddy
+        "DRONE",  # Drone CI
+        "HUDSON_URL",  # Hudson
+        "CF_BUILD_ID",  # Codefresh
+        "WERCKER",  # Wercker
+        "NOW_BUILDER",  # ZEIT Now
+        "TASKCLUSTER_ROOT_URL",  # TaskCluster
+        "SEMAPHORE",  # Semaphore CI
+        "BUILD_ID",  # Generic build environments
+    ]
+    return any(env_var in _os.environ for env_var in ci_env_vars)
+
+
+_TESTING = _is_CI_environment()
+
 
 # hide the supabase error in a thread on windows
 if _os.name == "nt":

lamindb_setup/_check_setup.py

@@ -27,6 +27,7 @@ If you used the CLI to set up lamindb in a notebook, restart the Python session.
 
 
 CURRENT_ISETTINGS: InstanceSettings | None = None
+IS_LOADING: bool = False
 
 
 def _get_current_instance_settings() -> InstanceSettings | None:
@@ -54,16 +55,11 @@ def _get_current_instance_settings() -> InstanceSettings | None:
 
 # we make this a private function because in all the places it's used,
 # users should not see it
-def _check_instance_setup(
-    from_lamindb: bool = False, from_module: str | None = None
-) -> bool:
-    reload_module = from_lamindb or from_module is not None
-    from ._init_instance import get_schema_module_name, reload_schema_modules
-
+def _check_instance_setup(from_module: str | None = None) -> bool:
     if django.IS_SETUP:
         # reload logic here because module might not yet have been imported
         # upon first setup
-        if from_module is not None:
+        if from_module is not None and from_module != "lamindb":
             il.reload(il.import_module(from_module))
         return True
     silence_loggers()
@@ -75,18 +71,21 @@ def _check_instance_setup(
         return True
     isettings = _get_current_instance_settings()
     if isettings is not None:
-        if reload_module and settings.auto_connect:
-            if not django.IS_SETUP:
+        if (
+            from_module is not None
+            and settings.auto_connect
+            and not django.IS_SETUP
+            and not IS_LOADING
+        ):
+            if not from_module == "lamindb":
+                import lamindb
+
+                il.reload(il.import_module(from_module))
+            else:
                 django.setup_django(isettings)
-                if from_module is not None:
-                    # this only reloads `from_module`
-                    il.reload(il.import_module(from_module))
-                else:
-                    # this bulk reloads all schema modules
-                    reload_schema_modules(isettings)
                 logger.important(f"connected lamindb: {isettings.slug}")
         return django.IS_SETUP
     else:
-        if reload_module and settings.auto_connect:
+        if from_module is not None and settings.auto_connect:
             logger.warning(InstanceNotSetupError.default_message)
         return False

lamindb_setup/_connect_instance.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
+import importlib
 import os
-import sys
 from typing import TYPE_CHECKING
 from uuid import UUID
 
@@ -198,6 +198,7 @@ def connect(slug: str, **kwargs) -> str | tuple | None:
         "_db",
         "_write_settings",
         "_raise_not_found_error",
+        "_reload_lamindb",
         "_test",
         "_user",
     }
@@ -210,6 +211,7 @@ def connect(slug: str, **kwargs) -> str | tuple | None:
     _db: str | None = kwargs.get("_db", None)
     _write_settings: bool = kwargs.get("_write_settings", True)
     _raise_not_found_error: bool = kwargs.get("_raise_not_found_error", True)
+    _reload_lamindb: bool = kwargs.get("_reload_lamindb", True)
     _test: bool = kwargs.get("_test", False)
 
     access_token: str | None = None
@@ -257,6 +259,14 @@ def connect(slug: str, **kwargs) -> str | tuple | None:
         if _test:
             return None
         silence_loggers()
+        # migrate away from lnschema-core
+        no_lnschema_core_file = (
+            settings_dir / f"no_lnschema_core-{isettings.slug.replace('/', '--')}"
+        )
+        if not no_lnschema_core_file.exists():
+            migrate_lnschema_core(
+                isettings, no_lnschema_core_file, write_file=_write_settings
+            )
         check, msg = isettings._load_db()
         if not check:
             local_db = (
@@ -292,21 +302,14 @@ def connect(slug: str, **kwargs) -> str | tuple | None:
         #     except ProgrammingError:
         #         pass
         load_from_isettings(isettings, user=_user, write_settings=_write_settings)
+        if _reload_lamindb:
+            importlib.reload(importlib.import_module("lamindb"))
     except Exception as e:
         if isettings is not None:
             if _write_settings:
                 isettings._get_settings_file().unlink(missing_ok=True)  # type: ignore
             settings._instance_settings = None
         raise e
-    # rename lnschema_bionty to bionty for sql tables
-    if "bionty" in isettings.schema:
-        no_lnschema_bionty_file = (
-            settings_dir / f"no_lnschema_bionty-{isettings.slug.replace('/', '')}"
-        )
-        if not no_lnschema_bionty_file.exists():
-            migrate_lnschema_bionty(
-                isettings, no_lnschema_bionty_file, write_file=_write_settings
-            )
     return None
 
 
@@ -322,13 +325,10 @@ def load(slug: str) -> str | tuple | None:
     return result
 
 
-def migrate_lnschema_bionty(
-    isettings: InstanceSettings, no_lnschema_bionty_file: Path, write_file: bool = True
+def migrate_lnschema_core(
+    isettings: InstanceSettings, no_lnschema_core_file: Path, write_file: bool = True
 ):
-    """Migrate lnschema_bionty tables to bionty tables if bionty_source doesn't exist.
-
-    :param db_uri: str, database URI (e.g., 'sqlite:///path/to/db.sqlite' or 'postgresql://user:password@host:port/dbname')
-    """
+    """Migrate lnschema_core tables to lamindb tables."""
     from urllib.parse import urlparse
 
     parsed_uri = urlparse(isettings.db)
@@ -348,60 +348,62 @@ def migrate_lnschema_bionty(
     cur = conn.cursor()
 
     try:
-        # check if bionty_source table exists
         if db_type == "sqlite":
             cur.execute(
-                "SELECT name FROM sqlite_master WHERE type='table' AND name='bionty_source'"
+                "SELECT name FROM sqlite_master WHERE type='table' AND name='lamindb_user'"
             )
             migrated = cur.fetchone() is not None
 
             # tables that need to be renamed
             cur.execute(
-                "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'lnschema_bionty_%'"
+                "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'lnschema_core_%'"
             )
             tables_to_rename = [
-                row[0][len("lnschema_bionty_") :] for row in cur.fetchall()
+                row[0][len("lnschema_core_") :] for row in cur.fetchall()
             ]
         else:  # postgres
             cur.execute(
-                "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'bionty_source')"
+                "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'lamindb_user')"
             )
             migrated = cur.fetchone()[0]
 
             # tables that need to be renamed
             cur.execute(
-                "SELECT table_name FROM information_schema.tables WHERE table_name LIKE 'lnschema_bionty_%'"
+                "SELECT table_name FROM information_schema.tables WHERE table_name LIKE 'lnschema_core_%'"
             )
             tables_to_rename = [
-                row[0][len("lnschema_bionty_") :] for row in cur.fetchall()
+                row[0][len("lnschema_core_") :] for row in cur.fetchall()
             ]
 
         if migrated:
             if write_file:
-                no_lnschema_bionty_file.touch(exist_ok=True)
+                no_lnschema_core_file.touch(exist_ok=True)
         else:
             try:
-                # rename tables only if bionty_source doesn't exist and there are tables to rename
+                response = input(
+                    f"Do you want to migrate to lamindb 0.78 (integrate lnschema_core into lamindb)? (y/n) -- Will rename {tables_to_rename}"
+                )
+                if response != "y":
+                    print("Aborted.")
+                    quit()
                 for table in tables_to_rename:
                     if db_type == "sqlite":
                         cur.execute(
-                            f"ALTER TABLE lnschema_bionty_{table} RENAME TO bionty_{table}"
+                            f"ALTER TABLE lnschema_core_{table} RENAME TO lamindb_{table}"
                         )
                     else:  # postgres
                         cur.execute(
-                            f"ALTER TABLE lnschema_bionty_{table} RENAME TO bionty_{table};"
+                            f"ALTER TABLE lnschema_core_{table} RENAME TO lamindb_{table};"
                         )
 
-                # update django_migrations table
                 cur.execute(
-                    "UPDATE django_migrations SET app = 'bionty' WHERE app = 'lnschema_bionty'"
+                    "UPDATE django_migrations SET app = 'lamindb' WHERE app = 'lnschema_core'"
                 )
-
-                logger.warning(
-                    "Please uninstall lnschema-bionty via `pip uninstall lnschema-bionty`!"
+                print(
+                    "Renaming tables finished.\nNow, *please* call: lamin migrate deploy"
                 )
                 if write_file:
-                    no_lnschema_bionty_file.touch(exist_ok=True)
+                    no_lnschema_core_file.touch(exist_ok=True)
             except Exception:
                 # read-only users can't rename tables
                 pass

lamindb_setup/_delete.py

@@ -99,7 +99,7 @@ def delete(slug: str, force: bool = False, require_empty: bool = True) -> int |
             )
         require_empty = True
     # first the default storage
-    n_objects = check_storage_is_empty(
+    n_files = check_storage_is_empty(
         isettings.storage.root,
         raise_error=require_empty,
         account_for_sqlite_file=isettings.dialect == "sqlite",
@@ -132,7 +132,7 @@ def delete(slug: str, force: bool = False, require_empty: bool = True) -> int |
         delete_instance_on_hub(isettings._id, require_empty=False)
     delete_by_isettings(isettings)
     # if .lndb file was delete, then we might count -1
-    if n_objects <= 0 and isettings.storage.type == "local":
+    if n_files <= 0 and isettings.storage.type == "local":
         # dir is only empty after sqlite file was delete via delete_by_isettings
         if (isettings.storage.root / ".lamindb").exists():
             (isettings.storage.root / ".lamindb").rmdir()

lamindb_setup/_django.py

@@ -5,24 +5,24 @@ from .core.django import setup_django
 
 
 def django(command: str, package_name: str | None = None, **kwargs):
-    r"""Manage migrations.
+    r"""Call Django commands.
 
     Examples:
 
     Reset auto-incrementing primary integer ids after a database import:
 
     >>> import lamindb as ln
-    >>> ln.setup.django("sqlsequencereset", "lnschema_core")
+    >>> ln.setup.django("sqlsequencereset", "lamindb")
     BEGIN;
-    SELECT setval(pg_get_serial_sequence('"lnschema_core_user"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lnschema_core_user";  # noqa
-    SELECT setval(pg_get_serial_sequence('"lnschema_core_storage"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lnschema_core_storage";  # noqa
+    SELECT setval(pg_get_serial_sequence('"lamindb_user"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lamindb_user";  # noqa
+    SELECT setval(pg_get_serial_sequence('"lamindb_storage"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lamindb_storage";  # noqa
     COMMIT;
 
     You can then run the SQL output that you'll see like so:
 
     >>> sql = \"\"\"BEGIN;
-        SELECT setval(pg_get_serial_sequence('"lnschema_core_user"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lnschema_core_user";  # noqa
-        SELECT setval(pg_get_serial_sequence('"lnschema_core_storage"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lnschema_core_storage";  # noqa
+        SELECT setval(pg_get_serial_sequence('"lamindb_user"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lamindb_user";  # noqa
+        SELECT setval(pg_get_serial_sequence('"lamindb_storage"','id'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "lamindb_storage";  # noqa
         COMMIT;\"\"\"
     >>> from django.db import connection
     >>> with connection.cursor() as cursor:

lamindb_setup/_exportdb.py

@@ -4,7 +4,7 @@ from importlib import import_module
 from pathlib import Path
 
 MODELS = {
-    "core": {
+    "lamindb": {
         "Collection": False,
         "Artifact": False,
         "Transform": False,

lamindb_setup/_init_instance.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import importlib
 import os
-import sys
 import uuid
 from typing import TYPE_CHECKING, Literal
 from uuid import UUID
@@ -29,6 +28,8 @@ if TYPE_CHECKING:
 def get_schema_module_name(schema_name, raise_import_error: bool = True) -> str | None:
     import importlib.util
 
+    if schema_name == "core":
+        return "lamindb"
     name_attempts = [f"lnschema_{schema_name.replace('-', '_')}", schema_name]
     for name in name_attempts:
         module_spec = importlib.util.find_spec(name)
@@ -42,8 +43,8 @@ def get_schema_module_name(schema_name, raise_import_error: bool = True) -> str
 
 
 def register_storage_in_instance(ssettings: StorageSettings):
-    from lnschema_core.models import Storage
-    from lnschema_core.users import current_user_id
+    from lamindb.base.users import current_user_id
+    from lamindb.models import Storage
 
     from .core.hashing import hash_and_encode_as_b62
 
@@ -71,7 +72,7 @@ def register_storage_in_instance(ssettings: StorageSettings):
 
 
 def register_user(usettings):
-    from lnschema_core.models import User
+    from lamindb.models import User
 
     try:
         # need to have try except because of integer primary key migration
@@ -99,35 +100,6 @@ def register_user_and_storage_in_instance(isettings: InstanceSettings, usettings
         logger.warning(f"instance seems not set up ({error})")
 
 
-def reload_schema_modules(isettings: InstanceSettings, include_core: bool = True):
-    schema_names = ["core"] if include_core else []
-    # schema_names += list(isettings.schema)
-    schema_module_names = [get_schema_module_name(n) for n in schema_names]
-
-    for schema_module_name in schema_module_names:
-        if schema_module_name in sys.modules:
-            schema_module = importlib.import_module(schema_module_name)
-            importlib.reload(schema_module)
-
-
-def reload_lamindb_itself(isettings) -> bool:
-    reloaded = False
-    if "lamindb" in sys.modules:
-        import lamindb
-
-        importlib.reload(lamindb)
-        reloaded = True
-    return reloaded
-
-
-def reload_lamindb(isettings: InstanceSettings):
-    log_message = settings.auto_connect
-    if not reload_lamindb_itself(isettings):
-        log_message = True
-    if log_message:
-        logger.important(f"connected lamindb: {isettings.slug}")
-
-
 ERROR_SQLITE_CACHE = """
 Your cached local SQLite file exists, while your cloud SQLite file ({}) doesn't.
 Either delete your cache ({}) or add it back to the cloud (if delete was accidental).
@@ -329,6 +301,8 @@ def init(
             update_schema_in_hub(access_token=access_token)
         if _write_settings:
             settings.auto_connect = True
+        importlib.reload(importlib.import_module("lamindb"))
+        logger.important(f"initialized lamindb: {isettings.slug}")
     except Exception as e:
         from ._delete import delete_by_isettings
         from .core._hub_core import delete_instance_record, delete_storage_record
@@ -338,16 +312,17 @@ def init(
                 delete_by_isettings(isettings)
             else:
                 settings._instance_settings = None
-            if (
-                user_handle != "anonymous" or access_token is not None
-            ) and isettings.is_on_hub:
-                delete_instance_record(isettings._id, access_token=access_token)
         if (
             ssettings is not None
             and (user_handle != "anonymous" or access_token is not None)
             and ssettings.is_on_hub
         ):
             delete_storage_record(ssettings._uuid, access_token=access_token)  # type: ignore
+        if isettings is not None:
+            if (
+                user_handle != "anonymous" or access_token is not None
+            ) and isettings.is_on_hub:
+                delete_instance_record(isettings._id, access_token=access_token)
         raise e
     return None
 
@@ -378,7 +353,6 @@ def load_from_isettings(
         if not isettings._get_settings_file().exists():
             register_user(user)
     isettings._persist(write_to_disk=write_settings)
-    reload_lamindb(isettings)
 
 
 def validate_sqlite_state(isettings: InstanceSettings) -> None:
@@ -414,7 +388,7 @@ def infer_instance_name(
         return str(db).split("/")[-1]
     if storage == "create-s3":
         raise ValueError("pass name to init if storage = 'create-s3'")
-    storage_path = UPath(storage)
+    storage_path = UPath(storage).resolve()
     # not sure if name is ever ""
     if storage_path.name != "":
         name = storage_path.name

lamindb_setup/_migrate.py

@@ -5,7 +5,7 @@ from django.db.migrations.loader import MigrationLoader
 from lamin_utils import logger
 from packaging import version
 
-from ._check_setup import _check_instance_setup
+from . import _check_setup
 from .core._settings import settings
 from .core.django import setup_django
 
@@ -64,16 +64,18 @@ class migrate:
     @classmethod
     def create(cls) -> None:
         """Create a migration."""
-        if _check_instance_setup():
+        if _check_setup._check_instance_setup():
             raise RuntimeError("Restart Python session to create migration or use CLI!")
+        _check_setup.IS_LOADING = True
         setup_django(settings.instance, create_migrations=True)
+        _check_setup.IS_LOADING = False
 
     @classmethod
     def deploy(cls) -> None:
         """Deploy a migration."""
         from ._schema_metadata import update_schema_in_hub
 
-        if _check_instance_setup():
+        if _check_setup._check_instance_setup():
             raise RuntimeError("Restart Python session to migrate or use CLI!")
         from lamindb_setup.core._hub_client import call_with_fallback_auth
         from lamindb_setup.core._hub_crud import (

lamindb_setup/_schema_metadata.py

@@ -232,7 +232,7 @@ class _ModelHandler:
         return related_fields
 
     def _get_field_metadata(self, model, field: Field):
-        from lnschema_core.models import LinkORM
+        from lamindb.models import LinkORM
 
         internal_type = field.get_internal_type()
         model_name = field.model._meta.model_name
@@ -273,14 +273,16 @@ class _ModelHandler:
             through = self._get_through(field)
 
         return FieldMetadata(
-            schema_name=schema_name,
+            schema_name=schema_name if schema_name != "lamindb" else "core",
             model_name=model_name,
             field_name=field_name,
             type=internal_type,
             is_link_table=issubclass(field.model, LinkORM),
             column_name=column,
             relation_type=relation_type,
-            related_schema_name=related_schema_name,
+            related_schema_name=related_schema_name
+            if related_schema_name != "lamindb"
+            else "core",
             related_model_name=related_model_name,
             related_field_name=related_field_name,
             through=through,
@@ -288,7 +290,7 @@ class _ModelHandler:
 
     @staticmethod
     def _get_through_many_to_many(field_or_rel: ManyToManyField | ManyToManyRel):
-        from lnschema_core.models import Registry
+        from lamindb.models import Registry
 
         if isinstance(field_or_rel, ManyToManyField):
             if field_or_rel.model != Registry:
@@ -365,7 +367,7 @@ class _SchemaHandler:
 
     def to_dict(self, include_django_objects: bool = True):
         return {
-            module_name: {
+            module_name if module_name != "lamindb" else "core": {
                 model_name: model.to_dict(include_django_objects)
                 for model_name, model in module.items()
             }
@@ -376,7 +378,7 @@ class _SchemaHandler:
         return self.to_dict(include_django_objects=False)
 
     def _get_modules_metadata(self):
-        from lnschema_core.models import Record, Registry
+        from lamindb.models import Record, Registry
 
         all_models = {
             module_name: {
@@ -401,6 +403,8 @@ class _SchemaHandler:
         module_set_info = []
         for module_name in self.included_modules:
             module = self._get_schema_module(module_name)
+            if module_name == "lamindb":
+                module_name = "core"
             module_set_info.append(
                 {"id": 0, "name": module_name, "version": module.__version__}
             )

lamindb_setup/core/_aws_credentials.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import os
 import time
 
+from lamin_utils import logger
 from upath.implementations.cloud import S3Path
 
 HOSTED_REGIONS = [
@@ -40,8 +41,15 @@ class AWSCredentialsManager:
 
         # this is cached so will be resued with the connection initialized
         fs = S3FileSystem(cache_regions=True)
-        fs.connect()
-        self.anon: bool = fs.session._credentials is None
+        try:
+            fs.connect()
+            self.anon: bool = fs.session._credentials is None
+        except Exception as e:
+            logger.warning(
+                f"There is a problem with your default AWS Credentials: {e}\n"
+                "`anon` mode will be used for all non-managed buckets."
+            )
+            self.anon = True
         self.anon_public: bool | None = None
         if not self.anon:
             try:

lamindb_setup/core/_hub_core.py

@@ -233,12 +233,16 @@ def _delete_instance(
     )
     if require_empty:
         for storage_record in storage_records:
+            root_string: str = storage_record["root"]  # type: ignore
             account_for_sqlite_file = (
                 instance_with_storage["db_scheme"] is None
-                and instance_with_storage["storage"]["root"] == storage_record["root"]
+                and instance_with_storage["storage"]["root"] == root_string
             )
-            root_string = storage_record["root"]
             # gate storage and instance deletion on empty storage location for
+            # normally auth.get_session() doesn't have access_token
+            # so this block is useless i think (Sergei)
+            # the token is received from user settings inside create_path
+            # might be needed in the hub though
             if client.auth.get_session() is not None:
                 access_token = client.auth.get_session().access_token
             else:
@@ -251,7 +255,6 @@ def _delete_instance(
             check_storage_is_empty(
                 root_path, account_for_sqlite_file=account_for_sqlite_file
             )
-    _update_instance_record(instance_with_storage["id"], {"storage_id": None}, client)
     # first delete the storage records because we will turn instance_id on
     # storage into a FK soon
     for storage_record in storage_records: