lamindb_setup 0.81.3__py3-none-any.whl → 1.0.0__py3-none-any.whl

lamindb_setup/_schema_metadata.py

@@ -232,21 +232,21 @@ class _ModelHandler:
         return related_fields
 
     def _get_field_metadata(self, model, field: Field):
-        from lnschema_core.models import LinkORM
+        from lamindb.models import LinkORM
 
         internal_type = field.get_internal_type()
         model_name = field.model._meta.model_name
         relation_type = self._get_relation_type(model, field)
         if field.related_model is None:
-            schema_name = field.model.__get_schema_name__()
+            schema_name = field.model.__get_module_name__()
             related_model_name = None
             related_schema_name = None
             related_field_name = None
             field_name = field.name
         else:
             related_model_name = field.related_model._meta.model_name
-            related_schema_name = field.related_model.__get_schema_name__()
-            schema_name = field.model.__get_schema_name__()
+            related_schema_name = field.related_model.__get_module_name__()
+            schema_name = field.model.__get_module_name__()
             related_field_name = field.remote_field.name
             field_name = field.name
 
@@ -273,14 +273,16 @@ class _ModelHandler:
             through = self._get_through(field)
 
         return FieldMetadata(
-            schema_name=schema_name,
+            schema_name=schema_name if schema_name != "lamindb" else "core",
             model_name=model_name,
             field_name=field_name,
             type=internal_type,
             is_link_table=issubclass(field.model, LinkORM),
             column_name=column,
             relation_type=relation_type,
-            related_schema_name=related_schema_name,
+            related_schema_name=related_schema_name
+            if related_schema_name != "lamindb"
+            else "core",
             related_model_name=related_model_name,
             related_field_name=related_field_name,
             through=through,
@@ -288,7 +290,7 @@ class _ModelHandler:
 
     @staticmethod
     def _get_through_many_to_many(field_or_rel: ManyToManyField | ManyToManyRel):
-        from lnschema_core.models import Registry
+        from lamindb.models import Registry
 
         if isinstance(field_or_rel, ManyToManyField):
             if field_or_rel.model != Registry:
@@ -360,12 +362,12 @@ class _ModelHandler:
 
 class _SchemaHandler:
     def __init__(self) -> None:
-        self.included_modules = ["core"] + list(settings.instance.schema)
+        self.included_modules = ["core"] + list(settings.instance.modules)
         self.modules = self._get_modules_metadata()
 
     def to_dict(self, include_django_objects: bool = True):
         return {
-            module_name: {
+            module_name if module_name != "lamindb" else "core": {
                 model_name: model.to_dict(include_django_objects)
                 for model_name, model in module.items()
             }
@@ -376,7 +378,7 @@ class _SchemaHandler:
         return self.to_dict(include_django_objects=False)
 
     def _get_modules_metadata(self):
-        from lnschema_core.models import Record, Registry
+        from lamindb.models import Record, Registry
 
         all_models = {
             module_name: {
@@ -389,7 +391,7 @@ class _SchemaHandler:
                 if model.__class__ is Registry
                 and model is not Record
                 and not model._meta.abstract
-                and model.__get_schema_name__() == module_name
+                and model.__get_module_name__() == module_name
             }
             for module_name in self.included_modules
         }
@@ -401,6 +403,8 @@ class _SchemaHandler:
         module_set_info = []
         for module_name in self.included_modules:
             module = self._get_schema_module(module_name)
+            if module_name == "lamindb":
+                module_name = "core"
             module_set_info.append(
                 {"id": 0, "name": module_name, "version": module.__version__}
             )

lamindb_setup/core/_aws_credentials.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import os
 import time
 
+from lamin_utils import logger
 from upath.implementations.cloud import S3Path
 
 HOSTED_REGIONS = [
@@ -40,8 +41,15 @@ class AWSCredentialsManager:
 
         # this is cached so will be resued with the connection initialized
         fs = S3FileSystem(cache_regions=True)
-        fs.connect()
-        self.anon: bool = fs.session._credentials is None
+        try:
+            fs.connect()
+            self.anon: bool = fs.session._credentials is None
+        except Exception as e:
+            logger.warning(
+                f"There is a problem with your default AWS Credentials: {e}\n"
+                "`anon` mode will be used for all non-managed buckets."
+            )
+            self.anon = True
         self.anon_public: bool | None = None
         if not self.anon:
             try:

lamindb_setup/core/_deprecated.py

@@ -30,23 +30,30 @@ from functools import wraps
 def deprecated(new_name: str):
     """Deprecated.
 
-    This is a decorator which can be used to mark functions
+    This is a decorator which can be used to mark functions, methods and properties
     as deprecated. It will result in a warning being emitted
     when the function is used.
+
+    It will also hide the function from the docs.
+
+    Example::
+
+        @property
+        @deprecated("n_files")
+        def n_objects(self) -> int:
+            return self.n_files
+
     """
 
     def decorator(func):
         @wraps(func)
         def new_func(*args, **kwargs):
-            # turn off filter
-            warnings.simplefilter("always", DeprecationWarning)
             warnings.warn(
                 f"Use {new_name} instead of {func.__name__}, "
                 f"{func.__name__} will be removed in the future.",
-                category=DeprecationWarning,
+                category=FutureWarning,
                 stacklevel=2,
             )
-            warnings.simplefilter("default", DeprecationWarning)  # reset filter
             return func(*args, **kwargs)
 
         setattr(new_func, "__deprecated", True)

lamindb_setup/core/_hub_core.py

@@ -233,12 +233,16 @@ def _delete_instance(
     )
     if require_empty:
         for storage_record in storage_records:
+            root_string: str = storage_record["root"]  # type: ignore
             account_for_sqlite_file = (
                 instance_with_storage["db_scheme"] is None
-                and instance_with_storage["storage"]["root"] == storage_record["root"]
+                and instance_with_storage["storage"]["root"] == root_string
             )
-            root_string = storage_record["root"]
             # gate storage and instance deletion on empty storage location for
+            # normally auth.get_session() doesn't have access_token
+            # so this block is useless i think (Sergei)
+            # the token is received from user settings inside create_path
+            # might be needed in the hub though
             if client.auth.get_session() is not None:
                 access_token = client.auth.get_session().access_token
             else:
@@ -251,7 +255,6 @@ def _delete_instance(
             check_storage_is_empty(
                 root_path, account_for_sqlite_file=account_for_sqlite_file
             )
-    _update_instance_record(instance_with_storage["id"], {"storage_id": None}, client)
     # first delete the storage records because we will turn instance_id on
     # storage into a FK soon
     for storage_record in storage_records:

lamindb_setup/core/_hub_crud.py

@@ -12,25 +12,23 @@ def select_instance_by_owner_name(
     name: str,
     client: Client,
 ) -> dict | None:
-    try:
-        data = (
-            client.table("instance")
-            .select(
-                "*, account!inner!instance_account_id_28936e8f_fk_account_id(*),"
-                " storage!inner!storage_instance_id_359fca71_fk_instance_id(*)"
-            )
-            .eq("name", name)
-            .eq("account.handle", owner)
-            .eq("storage.is_default", True)
-            .execute()
-            .data
+    # this won't find an instance without the default storage
+    data = (
+        client.table("instance")
+        .select(
+            "*, account!inner!instance_account_id_28936e8f_fk_account_id(*),"
+            " storage!inner!storage_instance_id_359fca71_fk_instance_id(*)"
         )
-    except Exception:
-        return None
+        .eq("name", name)
+        .eq("account.handle", owner)
+        .eq("storage.is_default", True)
+        .execute()
+        .data
+    )
     if len(data) == 0:
         return None
     result = data[0]
-    # this is now a list
+    # this is a list
     # assume only one default storage
     result["storage"] = result["storage"][0]
     return result
@@ -89,15 +87,22 @@ def select_instance_by_id_with_storage(
     instance_id: str,
     client: Client,
 ):
-    response = (
+    # this won't find an instance without the default storage
+    data = (
         client.table("instance")
-        .select("*, storage!instance_storage_id_87963cc8_fk_storage_id(*)")
+        .select("*, storage!inner!storage_instance_id_359fca71_fk_instance_id(*)")
         .eq("id", instance_id)
+        .eq("storage.is_default", True)
         .execute()
+        .data
     )
-    if len(response.data) == 0:
+    if len(data) == 0:
         return None
-    return response.data[0]
+    result = data[0]
+    # this is a list
+    # assume only one default storage
+    result["storage"] = result["storage"][0]
+    return result
 
 
 def update_instance(instance_id: str, instance_fields: dict, client: Client):
@@ -139,17 +144,14 @@ def select_collaborator(
 def select_default_storage_by_instance_id(
     instance_id: str, client: Client
 ) -> dict | None:
-    try:
-        data = (
-            client.table("storage")
-            .select("*")
-            .eq("instance_id", instance_id)
-            .eq("is_default", True)
-            .execute()
-            .data
-        )
-    except Exception:
-        return None
+    data = (
+        client.table("storage")
+        .select("*")
+        .eq("instance_id", instance_id)
+        .eq("is_default", True)
+        .execute()
+        .data
+    )
     if len(data) == 0:
         return None
     return data[0]

lamindb_setup/core/_hub_utils.py

@@ -7,15 +7,6 @@ from pydantic import BaseModel, Field, GetCoreSchemaHandler
 from pydantic_core import CoreSchema, core_schema
 
 
-def validate_schema_arg(schema: str | None = None) -> str:
-    if schema is None or schema == "":
-        return ""
-    # currently no actual validation, can add back if we see a need
-    # the following just strips white spaces
-    to_be_validated = [s.strip() for s in schema.split(",")]
-    return ",".join(to_be_validated)
-
-
 def validate_db_arg(db: str | None) -> None:
     if db is not None:
         LaminDsnModel(db=db)

lamindb_setup/core/_settings.py

@@ -164,7 +164,7 @@ class SetupSettings:
 
     @property
     def paths(self) -> type[SetupPaths]:
-        """Convert cloud paths to lamidb local paths.
+        """Convert cloud paths to lamindb local paths.
 
         Use `settings.paths.cloud_to_local_no_update`
         or `settings.paths.cloud_to_local`.
@@ -179,7 +179,7 @@ class SetupSettings:
         repr = self.user.__repr__()
         repr += f"\nAuto-connect in Python: {self.auto_connect}\n"
         repr += f"Private Django API: {self.private_django_api}\n"
-        repr += f"Cache directory: {self.cache_dir}\n"
+        repr += f"Cache directory: {self.cache_dir.as_posix()}\n"
         if self._instance_exists:
             repr += self.instance.__repr__()
         else:
@@ -200,9 +200,14 @@ class SetupPaths:
         # cache_key is ignored if filepath is a local path
         if not isinstance(filepath, LocalPathClasses):
             # settings is defined further in this file
-            local_filepath = settings.cache_dir / (
-                filepath.path if cache_key is None else cache_key  # type: ignore
-            )
+            if cache_key is None:
+                local_key = filepath.path  # type: ignore
+                protocol = filepath.protocol  # type: ignore
+                if protocol in {"http", "https"}:
+                    local_key = local_key.removeprefix(protocol + "://")
+            else:
+                local_key = cache_key
+            local_filepath = settings.cache_dir / local_key
         else:
             local_filepath = filepath
         return local_filepath

lamindb_setup/core/_settings_instance.py

@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Literal
 from django.db.utils import ProgrammingError
 from lamin_utils import logger
 
+from ._deprecated import deprecated
 from ._hub_client import call_with_fallback
 from ._hub_crud import select_account_handle_name_by_lnid
 from ._hub_utils import LaminDsn, LaminDsnModel
@@ -53,7 +54,7 @@ class InstanceSettings:
         keep_artifacts_local: bool = False,  # default to local storage
         uid: str | None = None,  # instance uid/lnid
         db: str | None = None,  # DB URI
-        schema: str | None = None,  # comma-separated string of schema names
+        modules: str | None = None,  # comma-separated string of module names
         git_repo: str | None = None,  # a git repo URL
         is_on_hub: bool | None = None,  # initialized from hub
         api_url: str | None = None,
@@ -69,7 +70,7 @@ class InstanceSettings:
         self._storage: StorageSettings = storage
         validate_db_arg(db)
         self._db: str | None = db
-        self._schema_str: str | None = schema
+        self._schema_str: str | None = modules
         self._git_repo = None if git_repo is None else sanitize_git_repo_url(git_repo)
         # local storage
         self._keep_artifacts_local = keep_artifacts_local
@@ -84,7 +85,7 @@ class InstanceSettings:
     def __repr__(self):
         """Rich string representation."""
         representation = f"Current instance: {self.slug}"
-        attrs = ["owner", "name", "storage", "db", "schema", "git_repo"]
+        attrs = ["owner", "name", "storage", "db", "modules", "git_repo"]
         for attr in attrs:
             value = getattr(self, attr)
             if attr == "storage":
@@ -121,7 +122,7 @@ class InstanceSettings:
     def _search_local_root(
         self, local_root: str | None = None, mute_warning: bool = False
     ) -> StorageSettings | None:
-        from lnschema_core.models import Storage
+        from lamindb.models import Storage
 
         if local_root is not None:
             local_records = Storage.objects.filter(root=local_root)
@@ -271,12 +272,20 @@ class InstanceSettings:
         return hash_and_encode_as_b62(self._id.hex)[:12]
 
     @property
-    def schema(self) -> set[str]:
-        """Schema modules in addition to core schema."""
+    def modules(self) -> set[str]:
+        """The set of modules that defines the database schema.
+
+        The core schema contained in lamindb is not included in this set.
+        """
         if self._schema_str is None:
             return {}  # type: ignore
         else:
-            return {schema for schema in self._schema_str.split(",") if schema != ""}
+            return {module for module in self._schema_str.split(",") if module != ""}
+
+    @property
+    @deprecated("modules")
+    def schema(self) -> set[str]:
+        return self.modules
 
     @property
     def _sqlite_file(self) -> UPath:
@@ -358,7 +367,7 @@ class InstanceSettings:
             sqlite_filepath = self.storage.cloud_to_local(
                 self._sqlite_file, error_no_origin=False
             )
-            return f"sqlite:///{sqlite_filepath}"
+            return f"sqlite:///{sqlite_filepath.as_posix()}"
         else:
             return self._db
 
@@ -457,11 +466,24 @@ class InstanceSettings:
         settings._instance_settings = self
 
     def _init_db(self):
+        from lamindb_setup import _check_setup
+
         from .django import setup_django
 
+        _check_setup.IS_LOADING = True
         setup_django(self, init=True)
+        _check_setup.IS_LOADING = False
+
+        from lamindb.models import Space
+
+        Space.objects.get_or_create(
+            name="All",
+            description="Every team & user with access to the instance has access.",
+        )
 
     def _load_db(self) -> tuple[bool, str]:
+        from lamindb_setup import _check_setup
+
         # Is the database available and initialized as LaminDB?
         # returns a tuple of status code and message
         if self.dialect == "sqlite" and not self._sqlite_file.exists():
@@ -472,7 +494,6 @@ class InstanceSettings:
                     f" {legacy_file} to {self._sqlite_file}"
                 )
             return False, f"SQLite file {self._sqlite_file} does not exist"
-        from lamindb_setup import settings  # to check user
 
         from .django import setup_django
 
@@ -481,5 +502,7 @@ class InstanceSettings:
         # setting up django also performs a check for migrations & prints them
         # as warnings
         # this should fail, e.g., if the db is not reachable
+        _check_setup.IS_LOADING = True
         setup_django(self)
+        _check_setup.IS_LOADING = False
         return True, ""

lamindb_setup/core/_settings_load.py

@@ -98,7 +98,7 @@ def setup_instance_from_store(store: InstanceSettingsStore) -> InstanceSettings:
         name=store.name,
         storage=ssettings,
         db=_null_to_value(store.db),
-        schema=_null_to_value(store.schema_str),
+        modules=_null_to_value(store.schema_str),
         git_repo=_null_to_value(store.git_repo),
         keep_artifacts_local=store.keep_artifacts_local,  # type: ignore
     )

lamindb_setup/core/_settings_storage.py

@@ -7,6 +7,7 @@ import string
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Literal
 
+import fsspec
 from lamin_utils import logger
 
 from ._aws_credentials import HOSTED_REGIONS, get_aws_credentials_manager
@@ -24,6 +25,10 @@ if TYPE_CHECKING:
 
 IS_INITIALIZED_KEY = ".lamindb/_is_initialized"
 
+# a list of supported fsspec protocols
+# rename file to local before showing to a user
+VALID_PROTOCOLS = ("file", "gs", "s3", "hf", "http", "https")
+
 
 def base62(n_char: int) -> str:
     """Like nanoid without hyphen and underscore."""
@@ -114,16 +119,11 @@ def init_storage(
             root_str = f"s3://lamin-{region}/{uid}"
         else:
             root_str = f"s3://lamin-hosted-test/{uid}"
-    elif root_str.startswith(("gs://", "s3://", "hf://")):
-        pass
-    else:  # local path
-        try:
-            _ = Path(root_str)
-        except Exception as e:
-            logger.error(
-                "`storage` is not a valid local, GCP storage, AWS S3 path or Hugging Face path"
-            )
-            raise e
+    elif (input_protocol := fsspec.utils.get_protocol(root_str)) not in VALID_PROTOCOLS:
+        valid_protocols = ("local",) + VALID_PROTOCOLS[1:]  # show local instead of file
+        raise ValueError(
+            f"Protocol {input_protocol} is not supported, valid protocols are {', '.join(valid_protocols)}"
+        )
     ssettings = StorageSettings(
         uid=uid,
         root=root_str,
@@ -227,10 +227,10 @@ class StorageSettings:
 
     @property
     def record(self) -> Any:
-        """Storage record in current instance."""
+        """Storage record in the current instance."""
         if self._record is None:
             # dynamic import because of import order
-            from lnschema_core.models import Storage
+            from lamindb.models import Storage
 
             from ._settings import settings
 
@@ -299,14 +299,15 @@ class StorageSettings:
         return self._region
 
     @property
-    def type(self) -> Literal["local", "s3", "gs"]:
+    def type(self) -> Literal["local", "s3", "gs", "hf", "http", "https"]:
         """AWS S3 vs. Google Cloud vs. local.
 
-        Returns the protocol as a string: "local", "s3", "gs".
+        Returns the protocol as a string: "local", "s3", "gs", "http", "https".
         """
         import fsspec
 
         convert = {"file": "local"}
+        # init_storage checks that the root protocol belongs to VALID_PROTOCOLS
         protocol = fsspec.utils.get_protocol(self.root_as_str)
         return convert.get(protocol, protocol)  # type: ignore
 
@@ -345,5 +346,5 @@ class StorageSettings:
         return self.root / filekey
 
     def local_filepath(self, filekey: UPathStr) -> UPath:
-        """Local (cache) filepath from filekey: `local(filepath(...))`."""
+        """Local (cache) filepath from filekey."""
         return self.cloud_to_local(self.key_to_filepath(filekey))

lamindb_setup/core/_settings_user.py

@@ -48,7 +48,7 @@ class UserSettings:
     @property
     def id(self):
         """Integer id valid in current intance."""
-        from lnschema_core.users import current_user_id
+        from lamindb.base.users import current_user_id
 
         # there is no cache needed here because current_user_id()
         # has its own cache

lamindb_setup/core/_setup_bionty_sources.py

@@ -17,7 +17,7 @@ RENAME = {"name": "source", "description": "source_name"}
 
 def write_bionty_sources(isettings: InstanceSettings) -> None:
     """Write bionty sources to Source table."""
-    if "bionty" not in isettings.schema:
+    if "bionty" not in isettings.modules:
         return None
     import shutil
 
@@ -79,7 +79,7 @@ def load_bionty_sources(isettings: InstanceSettings | None = None):
             # not setting up bionty sources
             return None
     if isettings is not None:
-        if "bionty" not in isettings.schema:
+        if "bionty" not in isettings.modules:
             # no need to setup anything
             return None
 
@@ -96,7 +96,7 @@ def load_bionty_sources(isettings: InstanceSettings | None = None):
         for kwargs in active_records:
             for db_field, base_col in RENAME.items():
                 kwargs[base_col] = kwargs.pop(db_field)
-            # TODO: non-bionty schema?
+            # TODO: non-bionty modules?
             kwargs["entity"] = kwargs["entity"].replace("bionty.", "")
         write_yaml(
             parse_currently_used_sources(active_records),

lamindb_setup/core/django.py

@@ -5,10 +5,8 @@ import builtins
 import os
 from pathlib import Path
 import time
-from lamin_utils import logger
-from ._settings_store import current_instance_settings_file
 from ._settings_instance import InstanceSettings
-import sys
+
 
 IS_RUN_FROM_IPYTHON = getattr(builtins, "__IPYTHON__", False)
 IS_SETUP = False
@@ -54,14 +52,15 @@ def setup_django(
         }
         from .._init_instance import get_schema_module_name
 
-        schema_names = ["core"] + list(isettings.schema)
+        module_names = ["core"] + list(isettings.modules)
         raise_import_error = True if init else False
-        installed_apps = [
+        installed_apps = ["django.contrib.contenttypes"]
+        installed_apps += [
             package_name
-            for n in schema_names
+            for name in module_names
             if (
                 package_name := get_schema_module_name(
-                    n, raise_import_error=raise_import_error
+                    name, raise_import_error=raise_import_error
                 )
             )
             is not None

lamindb_setup/core/hashing.py

@@ -12,6 +12,7 @@ from __future__ import annotations
 
 import base64
 import hashlib
+import json
 from concurrent.futures import ThreadPoolExecutor
 from typing import TYPE_CHECKING
 
@@ -40,11 +41,21 @@ def b16_to_b64(s: str):
     return to_b64_str(base64.b16decode(s.strip('"'), casefold=True))
 
 
+def hash_string(string: str) -> str:
+    # as we're truncating (not here) at 22 b64, we choose md5 over sha512
+    return to_b64_str(hashlib.md5(string.encode("utf-8")).digest())
+
+
 # a lot to read about this: lamin-notes/2022/hashing
 def hash_set(s: set[str]) -> str:
-    bstr = ":".join(sorted(s)).encode("utf-8")
-    # as we're truncating at 22 b64, we choose md5 over sha512
-    return to_b64_str(hashlib.md5(bstr).digest())[:HASH_LENGTH]
+    join_s = ":".join(sorted(s))
+    return hash_string(join_s)[:HASH_LENGTH]
+
+
+def hash_dict(d: dict) -> str:
+    return to_b64_str(hashlib.md5(json.dumps(d, sort_keys=True).encode()).digest())[
+        :HASH_LENGTH
+    ]
 
 
 def hash_from_hashes_list(hashes: Iterable[str]) -> str:
@@ -111,6 +122,6 @@ def hash_dir(path: Path):
     hashes, sizes = zip(*hashes_sizes)
 
     hash, hash_type = hash_from_hashes_list(hashes), "md5-d"
-    n_objects = len(hashes)
+    n_files = len(hashes)
     size = sum(sizes)
-    return size, hash, hash_type, n_objects
+    return size, hash, hash_type, n_files