lamindb_setup 0.73.2__py2.py3-none-any.whl → 0.74.0__py2.py3-none-any.whl

lamindb_setup/__init__.py

@@ -34,7 +34,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.73.2"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.74.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import sys
 from os import name as _os_name

lamindb_setup/_init_instance.py

@@ -33,8 +33,7 @@ def get_schema_module_name(schema_name) -> str:
         if module_spec is not None:
             return name
     raise ImportError(
-        f"Python package for '{schema_name}' is not installed, tried two package names:"
-        f" {name_attempts}\nHave you installed the schema package using `pip install`?"
+        f"Python package for '{schema_name}' is not installed.\nIf your package is on PyPI, run `pip install {schema_name}`"
     )
 
 
@@ -256,7 +255,10 @@ def init(
             schema=schema,
             uid=ssettings.uid,
         )
-        if isettings.is_remote and instance_state != "instance-corrupted-or-deleted":
+        register_on_hub = (
+            isettings.is_remote and instance_state != "instance-corrupted-or-deleted"
+        )
+        if register_on_hub:
             init_instance_hub(isettings)
         validate_sqlite_state(isettings)
         isettings._persist()
@@ -270,8 +272,10 @@ def init(
                 "locked instance (to unlock and push changes to the cloud SQLite file,"
                 " call: lamin close)"
             )
-        # we can debate whether this is the right setting, but this is how
-        # things have been and we'd like to not easily break backward compat
+        if register_on_hub and isettings.dialect != "sqlite":
+            from ._schema_metadata import update_schema_in_hub
+
+            update_schema_in_hub()
         settings.auto_connect = True
     except Exception as e:
         from ._delete import delete_by_isettings

lamindb_setup/_migrate.py

@@ -75,21 +75,15 @@ class migrate:
         from lamindb_setup.core._hub_client import call_with_fallback_auth
         from lamindb_setup.core._hub_crud import (
             select_collaborator,
-            select_instance_by_id,
             update_instance,
         )
 
-        instance_id_str = settings.instance._id.hex
-        instance = call_with_fallback_auth(
-            select_instance_by_id, instance_id=instance_id_str
-        )
-        instance_is_on_hub = instance is not None
-        if instance_is_on_hub:
+        if settings.instance.is_on_hub:
             # double check that user is an admin, otherwise will fail below
-            # without idempotence
+            # due to insufficient SQL permissions with cryptic error
             collaborator = call_with_fallback_auth(
                 select_collaborator,
-                instance_id=instance_id_str,
+                instance_id=settings.instance._id,
                 account_id=settings.user._uuid,
             )
             if collaborator is None or collaborator["role"] != "admin":
@@ -104,7 +98,7 @@ class migrate:
         # this sets up django and deploys the migrations
         setup_django(settings.instance, deploy_migrations=True)
         # this populates the hub
-        if instance_is_on_hub:
+        if settings.instance.is_on_hub:
             logger.important(f"updating lamindb version in hub: {lamindb.__version__}")
             # TODO: integrate update of instance table within update_schema_in_hub & below
             if settings.instance.dialect != "sqlite":

lamindb_setup/_register_instance.py

@@ -27,3 +27,7 @@ def register(_test: bool = False):
     init_instance_hub(isettings)
     isettings._is_on_hub = True
     isettings._persist()
+    if isettings.dialect != "sqlite" and not _test:
+        from ._schema_metadata import update_schema_in_hub
+
+        update_schema_in_hub()

lamindb_setup/_schema_metadata.py

@@ -3,31 +3,26 @@ from __future__ import annotations
 import hashlib
 import importlib
 import json
-from dataclasses import dataclass
-from typing import TYPE_CHECKING, Dict
+from typing import TYPE_CHECKING, Literal
 from uuid import UUID
 
-import sqlparse
-from django.contrib.postgres.expressions import ArraySubquery
 from django.db.models import (
     Field,
+    ForeignKey,
     ForeignObjectRel,
     ManyToManyField,
     ManyToManyRel,
-    OuterRef,
-    QuerySet,
-    Subquery,
+    ManyToOneRel,
+    OneToOneField,
+    OneToOneRel,
 )
-from django.db.models.functions import JSONObject
-from sqlparse.sql import Identifier, IdentifierList
-from sqlparse.tokens import DML, Keyword
+from pydantic import BaseModel
 
 from lamindb_setup import settings
 from lamindb_setup._init_instance import get_schema_module_name
 from lamindb_setup.core._hub_client import call_with_fallback_auth
 
 if TYPE_CHECKING:
-    from lnschema_core.models import Registry
     from supabase import Client
 
 
@@ -36,7 +31,7 @@ def update_schema_in_hub() -> tuple[bool, UUID, dict]:
 
 
 def _synchronize_schema(client: Client) -> tuple[bool, UUID, dict]:
-    schema_metadata = SchemaMetadata()
+    schema_metadata = _SchemaHandler()
     schema_metadata_dict = schema_metadata.to_json()
     schema_uuid = _dict_to_uuid(schema_metadata_dict)
     schema = _get_schema_by_id(schema_uuid, client)
@@ -52,7 +47,7 @@ def _synchronize_schema(client: Client) -> tuple[bool, UUID, dict]:
                     "id": schema_uuid.hex,
                     "module_ids": module_ids,
                     "module_set_info": module_set_info,
-                    "json": schema_metadata_dict,
+                    "schema_json": schema_metadata_dict,
                 }
             )
             .execute()
@@ -90,100 +85,47 @@ def _dict_to_uuid(dict: dict):
     return uuid
 
 
-class SchemaMetadata:
-    def __init__(self) -> None:
-        self.included_modules = ["core"] + list(settings.instance.schema)
-        self.modules = self._get_modules_metadata()
-
-    def to_dict(
-        self, include_django_objects: bool = True, include_select_terms: bool = True
-    ):
-        return {
-            module_name: {
-                model_name: model.to_dict(include_django_objects, include_select_terms)
-                for model_name, model in module.items()
-            }
-            for module_name, module in self.modules.items()
-        }
-
-    def to_json(self, include_select_terms: bool = True):
-        return self.to_dict(
-            include_django_objects=False, include_select_terms=include_select_terms
-        )
-
-    def _get_modules_metadata(self):
-        return {
-            module_name: {
-                model._meta.model_name: ModelMetadata(
-                    model, module_name, self.included_modules
-                )
-                for model in self._get_schema_module(
-                    module_name
-                ).models.__dict__.values()
-                if model.__class__.__name__ == "ModelBase"
-                and model.__name__ not in ["Registry", "ORM"]
-                and not model._meta.abstract
-                and model.__get_schema_name__() == module_name
-            }
-            for module_name in self.included_modules
-        }
-
-    def _get_module_set_info(self):
-        # TODO: rely on schemamodule table for this
-        module_set_info = []
-        for module_name in self.included_modules:
-            module = self._get_schema_module(module_name)
-            module_set_info.append(
-                {"id": 0, "name": module_name, "version": module.__version__}
-            )
-        return module_set_info
-
-    @staticmethod
-    def _get_schema_module(module_name):
-        return importlib.import_module(get_schema_module_name(module_name))
-
-
-@dataclass
-class FieldMetadata:
-    schema_name: str
-    model_name: str
+RelationType = Literal["many-to-one", "one-to-many", "many-to-many", "one-to-one"]
+Type = Literal[
+    "ForeignKey",
+    "CharField",
+    "DateTimeField",
+    "AutoField",
+    "BooleanField",
+    "BigIntegerField",
+    "SmallIntegerField",
+    "TextField",
+    "BigAutoField",
+    "ManyToManyField",
+    "IntegerField",
+    "OneToOneField",
+    "JSONField",
+    "DateField",
+    "FloatField",
+]
+
+
+class Through(BaseModel):
+    left_key: str
+    right_key: str
+    link_table_name: str | None = None
+
+
+class FieldMetadata(BaseModel):
+    type: Type
+    column: str | None = None
+    through: Through | None = None
     field_name: str
-    type: str
+    model_name: str
+    schema_name: str
     is_link_table: bool
-    column: str | None = None
-    relation_type: str | None = None
-    related_schema_name: str | None = None
-    related_model_name: str | None = None
+    relation_type: RelationType | None = None
     related_field_name: str | None = None
-    through: dict | None = None
-
-
-class ModelRelations:
-    def __init__(self, fields: list[ForeignObjectRel]) -> None:
-        self.many_to_one = {}
-        self.one_to_many = {}
-        self.many_to_many = {}
-        self.one_to_one = {}
-
-        for field in fields:
-            if field.many_to_one:
-                self.many_to_one.update({field.name: field})
-            elif field.one_to_many:
-                self.one_to_many.update({field.name: field})
-            elif field.many_to_many:
-                self.many_to_many.update({field.name: field})
-            elif field.one_to_one:
-                self.one_to_one.update({field.name: field})
-
-        self.all = {
-            **self.many_to_one,
-            **self.one_to_many,
-            **self.many_to_many,
-            **self.one_to_one,
-        }
+    related_model_name: str | None = None
+    related_schema_name: str | None = None
 
 
-class ModelMetadata:
+class _ModelHandler:
     def __init__(self, model, module_name: str, included_modules: list[str]) -> None:
         self.model = model
         self.class_name = model.__name__
@@ -191,39 +133,26 @@ class ModelMetadata:
         self.model_name = model._meta.model_name
         self.table_name = model._meta.db_table
         self.included_modules = included_modules
-        self.fields, self.relations = self._get_fields_metadata(self.model)
+        self.fields = self._get_fields_metadata(self.model)
 
-    def to_dict(
-        self, include_django_objects: bool = True, include_select_terms: bool = True
-    ):
+    def to_dict(self, include_django_objects: bool = True):
         _dict = {
             "fields": self.fields.copy(),
             "class_name": self.class_name,
             "table_name": self.table_name,
         }
 
-        select_terms = self.select_terms if include_select_terms else []
-
         for field_name in self.fields.keys():
             _dict["fields"][field_name] = _dict["fields"][field_name].__dict__
-            if field_name in select_terms:
-                _dict["fields"][field_name].update(
-                    {"select_term": select_terms[field_name]}
-                )
+            through = _dict["fields"][field_name]["through"]
+            if through is not None:
+                _dict["fields"][field_name]["through"] = through.__dict__
 
         if include_django_objects:
             _dict.update({"model": self.model})
 
         return _dict
 
-    @property
-    def select_terms(self):
-        return (
-            DjangoQueryBuilder(self.module_name, self.model_name)
-            .add_all_sub_queries()
-            .extract_select_terms()
-        )
-
     def _get_fields_metadata(self, model):
         related_fields = []
         fields_metadata: dict[str, FieldMetadata] = {}
@@ -239,66 +168,37 @@ class ModelMetadata:
             ):
                 related_fields.append(field)
 
-        model_relations_metadata = ModelRelations(related_fields)
-
         related_fields_metadata = self._get_related_fields_metadata(
-            model, model_relations_metadata
+            model, related_fields
         )
 
         fields_metadata = {**fields_metadata, **related_fields_metadata}
 
-        return fields_metadata, model_relations_metadata
+        return fields_metadata
 
-    def _get_related_fields_metadata(
-        self, model, model_relations_metadata: ModelRelations
-    ):
+    def _get_related_fields_metadata(self, model, fields: list[ForeignObjectRel]):
         related_fields: dict[str, FieldMetadata] = {}
 
-        # Many to one (foreign key defined in the model)
-        for link_field_name, link_field in model_relations_metadata.many_to_one.items():
-            related_fields.update(
-                {f"{link_field_name}": self._get_field_metadata(model, link_field)}
-            )
-            for field in link_field.related_model._meta.fields:
+        for field in fields:
+            if field.many_to_one:
                 related_fields.update(
-                    {
-                        f"{link_field_name}__{field.name}": self._get_field_metadata(
-                            model, field
-                        )
-                    }
+                    {f"{field.name}": self._get_field_metadata(model, field)}
                 )
-
-        # One to many (foreign key defined in the related model)
-        for relation_name, relation in model_relations_metadata.one_to_many.items():
-            # exclude self reference as it is already included in the many to one
-            if relation.related_model == model:
-                continue
-            related_fields.update(
-                {f"{relation_name}": self._get_field_metadata(model, relation.field)}
-            )
-
-        # One to one
-        for link_field_name, link_field in model_relations_metadata.one_to_one.items():
-            related_fields.update(
-                {f"{link_field_name}": self._get_field_metadata(model, link_field)}
-            )
-            for field in link_field.related_model._meta.fields:
+            elif field.one_to_many:
+                # exclude self reference as it is already included in the many to one
+                if field.related_model == model:
+                    continue
                 related_fields.update(
-                    {
-                        f"{link_field_name}__{field.name}": self._get_field_metadata(
-                            model, field
-                        )
-                    }
+                    {f"{field.name}": self._get_field_metadata(model, field.field)}
+                )
+            elif field.many_to_many:
+                related_fields.update(
+                    {f"{field.name}": self._get_field_metadata(model, field)}
+                )
+            elif field.one_to_one:
+                related_fields.update(
+                    {f"{field.name}": self._get_field_metadata(model, field)}
                 )
-
-        # Many to many
-        for (
-            link_field_name,
-            link_field,
-        ) in model_relations_metadata.many_to_many.items():
-            related_fields.update(
-                {f"{link_field_name}": self._get_field_metadata(model, link_field)}
-            )
 
         return related_fields
 
@@ -332,45 +232,83 @@ class ModelMetadata:
             pass
 
         column = None
-        if relation_type not in ["many-to-many", "one-to-one", "one-to-many"]:
-            column = field.column
-
-        through = None
-        if relation_type == "many-to-many":
-            through = self._get_through(model, field)
+        if relation_type not in ["many-to-many", "one-to-many"]:
+            if not isinstance(field, ForeignObjectRel):
+                column = field.column
+
+        if relation_type is None:
+            through = None
+        elif relation_type == "many-to-many":
+            through = self._get_through_many_to_many(field)
+        else:
+            through = self._get_through(field)
 
         return FieldMetadata(
-            schema_name,
-            model_name,
-            field_name,
-            internal_type,
-            issubclass(field.model, LinkORM),
-            column,
-            relation_type,
-            related_schema_name,
-            related_model_name,
-            related_field_name,
-            through,
+            schema_name=schema_name,
+            model_name=model_name,
+            field_name=field_name,
+            type=internal_type,
+            is_link_table=issubclass(field.model, LinkORM),
+            column=column,
+            relation_type=relation_type,
+            related_schema_name=related_schema_name,
+            related_model_name=related_model_name,
+            related_field_name=related_field_name,
+            through=through,
         )
 
     @staticmethod
-    def _get_through(model, field_or_rel: ManyToManyField | ManyToManyRel):
-        table_name = model._meta.db_table
-        related_table_name = field_or_rel.related_model._meta.db_table
+    def _get_through_many_to_many(field_or_rel: ManyToManyField | ManyToManyRel):
+        from lnschema_core.models import Registry
 
         if isinstance(field_or_rel, ManyToManyField):
-            return {
-                "link_table_name": field_or_rel.remote_field.through._meta.db_table,
-                table_name: field_or_rel.m2m_column_name(),
-                related_table_name: field_or_rel.m2m_reverse_name(),
-            }
+            if field_or_rel.model != Registry:
+                return Through(
+                    left_key=field_or_rel.m2m_column_name(),
+                    right_key=field_or_rel.m2m_reverse_name(),
+                    link_table_name=field_or_rel.remote_field.through._meta.db_table,
+                )
+            else:
+                return Through(
+                    left_key=field_or_rel.m2m_reverse_name(),
+                    right_key=field_or_rel.m2m_column_name(),
+                    link_table_name=field_or_rel.remote_field.through._meta.db_table,
+                )
 
         if isinstance(field_or_rel, ManyToManyRel):
-            return {
-                "link_table_name": field_or_rel.through._meta.db_table,
-                table_name: field_or_rel.field.m2m_column_name(),
-                related_table_name: field_or_rel.field.m2m_reverse_name(),
-            }
+            if field_or_rel.model != Registry:
+                return Through(
+                    left_key=field_or_rel.field.m2m_reverse_name(),
+                    right_key=field_or_rel.field.m2m_column_name(),
+                    link_table_name=field_or_rel.through._meta.db_table,
+                )
+            else:
+                return Through(
+                    left_key=field_or_rel.field.m2m_column_name(),
+                    right_key=field_or_rel.field.m2m_reverse_name(),
+                    link_table_name=field_or_rel.through._meta.db_table,
+                )
+
+    def _get_through(
+        self, field_or_rel: ForeignKey | OneToOneField | ManyToOneRel | OneToOneRel
+    ):
+        if isinstance(field_or_rel, ForeignObjectRel):
+            rel_1 = field_or_rel.field.related_fields[0][0]
+            rel_2 = field_or_rel.field.related_fields[0][1]
+        else:
+            rel_1 = field_or_rel.related_fields[0][0]
+            rel_2 = field_or_rel.related_fields[0][1]
+
+        if rel_1.model._meta.model_name == self.model._meta.model_name:
+            return Through(
+                left_key=rel_1.column,
+                right_key=rel_2.column,
+            )
+        else:
+            return Through(
+                left_key=rel_2.column,
+                right_key=rel_1.column,
+            )
 
     @staticmethod
     def _get_relation_type(model, field: Field):
@@ -391,89 +329,50 @@ class ModelMetadata:
             return None
 
 
-class DjangoQueryBuilder:
-    def __init__(
-        self, module_name: str, model_name: str, query_set: QuerySet | None = None
-    ) -> None:
-        self.schema_metadata = SchemaMetadata()
-        self.module_name = module_name
-        self.model_name = model_name
-        self.model_metadata = self.schema_metadata.modules[module_name][model_name]
-        self.query_set = query_set if query_set else self.model_metadata.model.objects
-
-    def add_all_sub_queries(self):
-        all_fields = self.model_metadata.fields
-        included_relations = [
-            field_name
-            for field_name, field in all_fields.items()
-            if field.relation_type is not None
-        ]
-        self.add_sub_queries(included_relations)
-        return self
-
-    def add_sub_queries(self, included_relations: list[str]):
-        sub_queries = {
-            f"annotated_{relation_name}": self._get_sub_query(
-                self.model_metadata.fields[relation_name]
-            )
-            for relation_name in included_relations
-        }
-        self.query_set = self.query_set.annotate(**sub_queries)
-        return self
-
-    def extract_select_terms(self):
-        parsed = sqlparse.parse(self.sql_query)
-        select_found = False
-        select_terms = {}
-
-        def get_name(identifier):
-            name = identifier.get_name()
-            return name if name is not None else str(identifier).split(".")
-
-        for token in parsed[0].tokens:
-            if token.ttype is DML and token.value.upper() == "SELECT":
-                select_found = True
-            elif select_found and isinstance(token, IdentifierList):
-                for identifier in token.get_identifiers():
-                    select_terms[get_name(identifier)] = str(identifier)
-            elif select_found and isinstance(token, Identifier):
-                select_terms[get_name(token)] = str(token)
-            elif token.ttype is Keyword:
-                if token.value.upper() in ["FROM", "WHERE", "GROUP BY", "ORDER BY"]:
-                    break
-
-        return select_terms
-
-    def _get_sub_query(self, field_metadata: FieldMetadata):
-        module_name = field_metadata.related_schema_name
-        model_name = field_metadata.related_model_name
-        field_name = field_metadata.related_field_name
-        model_metadata = self.schema_metadata.modules[module_name][model_name]
-        query_set = model_metadata.model.objects.get_queryset()
-        select = {
-            field_name: field_name
-            for field_name in model_metadata.fields.keys()
-            if model_metadata.fields[field_name].relation_type is None
-            and "__" not in field_name
+class _SchemaHandler:
+    def __init__(self) -> None:
+        self.included_modules = ["core"] + list(settings.instance.schema)
+        self.modules = self._get_modules_metadata()
+
+    def to_dict(self, include_django_objects: bool = True):
+        return {
+            module_name: {
+                model_name: model.to_dict(include_django_objects)
+                for model_name, model in module.items()
+            }
+            for module_name, module in self.modules.items()
         }
 
-        if field_metadata.relation_type in ["many-to-many", "one-to-many"]:
-            return ArraySubquery(
-                Subquery(
-                    query_set.filter(**{field_name: OuterRef("pk")}).values(
-                        data=JSONObject(**select)
-                    )[:5]
+    def to_json(self):
+        return self.to_dict(include_django_objects=False)
+
+    def _get_modules_metadata(self):
+        return {
+            module_name: {
+                model._meta.model_name: _ModelHandler(
+                    model, module_name, self.included_modules
                 )
+                for model in self._get_schema_module(
+                    module_name
+                ).models.__dict__.values()
+                if model.__class__.__name__ == "RegistryMeta"
+                and model.__name__ not in ["Registry", "ORM"]
+                and not model._meta.abstract
+                and model.__get_schema_name__() == module_name
+            }
+            for module_name in self.included_modules
+        }
+
+    def _get_module_set_info(self):
+        # TODO: rely on schemamodule table for this
+        module_set_info = []
+        for module_name in self.included_modules:
+            module = self._get_schema_module(module_name)
+            module_set_info.append(
+                {"id": 0, "name": module_name, "version": module.__version__}
             )
-        if field_metadata.relation_type in ["many-to-one", "one-to-one"]:
-            return Subquery(
-                query_set.filter(**{field_name: OuterRef("pk")}).values(
-                    data=JSONObject(**select)
-                )[:5]
-            )
+        return module_set_info
 
-    @property
-    def sql_query(self):
-        sql_template, params = self.query_set.query.sql_with_params()
-        sql_query = sql_template % tuple(f"'{p}'" for p in params)
-        return sql_query.replace("annotated_", "")
+    @staticmethod
+    def _get_schema_module(module_name):
+        return importlib.import_module(get_schema_module_name(module_name))

lamindb_setup/core/_hub_client.py

@@ -119,8 +119,8 @@ def call_with_fallback_auth(
     for renew_token, fallback_env in [(False, False), (True, False), (False, True)]:
         try:
             if renew_token:
-                logger.important(
-                    "Renewing expired lamin token: call lamin login to avoid this"
+                logger.warning(
+                    "renewing expired lamin token: call `lamin login` to avoid this"
                 )
             client = connect_hub_with_auth(
                 renew_token=renew_token, fallback_env=fallback_env

lamindb_setup/core/_private_django_api.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+
+def find_vscode_stubs_folder() -> Path | None:
+    # Possible locations of VSCode extensions
+    possible_locations = [
+        Path.home() / ".vscode" / "extensions",  # Linux and macOS
+        Path.home() / ".vscode-server" / "extensions",  # Remote development
+        Path(os.environ.get("APPDATA", "")) / "Code" / "User" / "extensions",  # Windows
+        Path("/usr/share/code/resources/app/extensions"),  # Some Linux distributions
+    ]
+    for location in possible_locations:
+        if location.exists():
+            # Look for Pylance extension folder
+            pylance_folders = list(location.glob("ms-python.vscode-pylance-*"))
+            if pylance_folders:
+                # Sort to get the latest version
+                latest_pylance = sorted(pylance_folders)[-1]
+                stubs_folder = (
+                    latest_pylance / "dist" / "bundled" / "stubs" / "django-stubs"
+                )
+                if stubs_folder.exists():
+                    return stubs_folder
+
+    return None
+
+
+def private_django_api(reverse=False):
+    from django import db
+
+    # the order here matters
+    # changing it might break the tests
+    attributes = [
+        "DoesNotExist",
+        "MultipleObjectsReturned",
+        "add_to_class",
+        "adelete",
+        "refresh_from_db",
+        "asave",
+        "clean",
+        "clean_fields",
+        "date_error_message",
+        "get_constraints",
+        "get_deferred_fields",
+        "prepare_database_save",
+        "save_base",
+        "serializable_value",
+        "unique_error_message",
+        "validate_constraints",
+        "validate_unique",
+    ]
+    if reverse:
+        attributes.append("arefresh_from_db")
+        attributes.append("full_clean")
+    else:
+        attributes.append("a_refresh_from_db")
+        attributes.append("full__clean")
+
+    django_path = Path(db.__file__).parent.parent
+
+    encoding = "utf8" if os.name == "nt" else None
+
+    def prune_file(file_path):
+        content = file_path.read_text(encoding=encoding)
+        original_content = content
+
+        for attr in attributes:
+            old_name = f"_{attr}" if reverse else attr
+            new_name = attr if reverse else f"_{attr}"
+            content = content.replace(old_name, new_name)
+
+        if not reverse:
+            content = content.replace("Field_DoesNotExist", "FieldDoesNotExist")
+            content = content.replace("Object_DoesNotExist", "ObjectDoesNotExist")
+
+        if content != original_content:
+            file_path.write_text(content, encoding=encoding)
+
+    for file_path in django_path.rglob("*.py"):
+        prune_file(file_path)
+
+    pylance_path = find_vscode_stubs_folder()
+    if pylance_path is not None:
+        for file_path in pylance_path.rglob("*.pyi"):
+            prune_file(file_path)

lamindb_setup/core/_settings.py

@@ -27,6 +27,7 @@ class SetupSettings:
     _instance_settings_env: str | None = None
 
     _auto_connect_path: Path = settings_dir / "auto_connect"
+    _private_django_api_path: Path = settings_dir / "private_django_api"
 
     @property
     def _instance_settings_path(self) -> Path:
@@ -34,11 +35,17 @@ class SetupSettings:
 
     @property
     def settings_dir(self) -> Path:
+        """The directory that holds locally persisted settings."""
         return settings_dir
 
     @property
     def auto_connect(self) -> bool:
-        """Auto-connect to loaded instance upon lamindb import."""
+        """Auto-connect to loaded instance upon lamindb import.
+
+        `lamin init` and `lamin load` switch this to `True`.
+
+        `ln.connect()` doesn't change the value of this setting.
+        """
         return self._auto_connect_path.exists()
 
     @auto_connect.setter
@@ -48,10 +55,35 @@ class SetupSettings:
         else:
             self._auto_connect_path.unlink(missing_ok=True)
 
+    @property
+    def private_django_api(self) -> bool:
+        """Turn internal Django API private to clean up the API (default `False`).
+
+        This patches your local pip-installed django installation. You can undo
+        the patch by setting this back to `False`.
+        """
+        return self._private_django_api_path.exists()
+
+    @private_django_api.setter
+    def private_django_api(self, value: bool) -> None:
+        from ._private_django_api import private_django_api
+
+        # we don't want to call private_django_api() twice
+        if value and not self.private_django_api:
+            private_django_api()
+            self._private_django_api_path.touch()
+        elif not value and self.private_django_api:
+            private_django_api(reverse=True)
+            self._private_django_api_path.unlink(missing_ok=True)
+
     @property
     def user(self) -> UserSettings:
-        """:class:`~lamindb.setup.core.UserSettings`."""
-        if self._user_settings is None or self._user_settings_env != get_env_name():
+        """Settings of current user."""
+        env_changed = (
+            self._user_settings_env is not None
+            and self._user_settings_env != get_env_name()
+        )
+        if self._user_settings is None or env_changed:
             self._user_settings = load_or_create_user_settings()
             self._user_settings_env = get_env_name()
             if self._user_settings and self._user_settings.uid is None:
@@ -60,18 +92,19 @@ class SetupSettings:
 
     @property
     def instance(self) -> InstanceSettings:
-        """:class:`~lamindb.setup.core.InstanceSettings`."""
-        if (
-            self._instance_settings is None
-            or self._instance_settings_env != get_env_name()
-        ):
+        """Settings of current LaminDB instance."""
+        env_changed = (
+            self._instance_settings_env is not None
+            and self._instance_settings_env != get_env_name()
+        )
+        if self._instance_settings is None or env_changed:
             self._instance_settings = load_instance_settings()
             self._instance_settings_env = get_env_name()
         return self._instance_settings  # type: ignore
 
     @property
     def storage(self) -> StorageSettings:
-        """:class:`~lamindb.setup.core.StorageSettings`."""
+        """Settings of default storage."""
         return self.instance.storage
 
     @property
@@ -87,6 +120,7 @@ class SetupSettings:
         """Rich string representation."""
         repr = self.user.__repr__()
         repr += f"\nAuto-connect in Python: {self.auto_connect}\n"
+        repr += f"Private Django API: {self.private_django_api}\n"
         if self._instance_exists:
             repr += self.instance.__repr__()
         else:

lamindb_setup/core/_settings_instance.py

@@ -106,7 +106,8 @@ class InstanceSettings:
         if local_root is not None:
             local_records = Storage.objects.filter(root=local_root)
         else:
-            local_records = Storage.objects.filter(type="local")
+            # only search local managed storage locations (instance_uid=self.uid)
+            local_records = Storage.objects.filter(type="local", instance_uid=self.uid)
         all_local_records = local_records.all()
         try:
             # trigger an error in case of a migration issue

lamindb_setup/core/_settings_storage.py

@@ -121,6 +121,8 @@ def init_storage(
         init_storage_hub(ssettings)
     # below comes last only if everything else was successful
     try:
+        # (federated) credentials for AWS access are provisioned under-the-hood
+        # discussion: https://laminlabs.slack.com/archives/C04FPE8V01W/p1719260587167489
         mark_storage_root(ssettings.root, ssettings.uid)  # type: ignore
     except Exception:
         logger.important(

lamindb_setup/core/hashing.py

@@ -12,7 +12,10 @@ from __future__ import annotations
 
 import base64
 import hashlib
-from typing import TYPE_CHECKING
+from concurrent.futures import ThreadPoolExecutor
+from typing import TYPE_CHECKING, Iterable
+
+import psutil
 
 if TYPE_CHECKING:
     from .types import Path, UPathStr
@@ -40,7 +43,7 @@ def hash_set(s: set[str]) -> str:
     return to_b64_str(hashlib.md5(bstr).digest())[:20]
 
 
-def hash_md5s_from_dir(hashes: list[str]) -> tuple[str, str]:
+def hash_md5s_from_dir(hashes: Iterable[str]) -> tuple[str, str]:
     # need to sort below because we don't want the order of parsing the dir to
     # affect the hash
     digests = b"".join(
@@ -83,3 +86,27 @@ def hash_file(
             ).digest()
             hash_type = "sha1-fl"
     return to_b64_str(digest)[:22], hash_type
+
+
+def hash_dir(path: Path):
+    files = (subpath for subpath in path.rglob("*") if subpath.is_file())
+
+    def hash_size(file):
+        file_size = file.stat().st_size
+        return hash_file(file, file_size)[0], file_size
+
+    try:
+        n_workers = len(psutil.Process().cpu_affinity())
+    except AttributeError:
+        n_workers = psutil.cpu_count()
+    if n_workers > 1:
+        with ThreadPoolExecutor(n_workers) as pool:
+            hashes_sizes = pool.map(hash_size, files)
+    else:
+        hashes_sizes = map(hash_size, files)
+    hashes, sizes = zip(*hashes_sizes)
+
+    hash, hash_type = hash_md5s_from_dir(hashes)
+    n_objects = len(hashes)
+    size = sum(sizes)
+    return size, hash, hash_type, n_objects

{lamindb_setup-0.73.2.dist-info → lamindb_setup-0.74.0.dist-info}/METADATA

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: lamindb_setup
-Version: 0.73.2
+Version: 0.74.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <laminlabs@gmail.com>
 Description-Content-Type: text/markdown
 Requires-Dist: lnschema_core>=0.51.0
 Requires-Dist: lamin_utils>=0.3.3
-Requires-Dist: django>4.2,<5.2.0
+Requires-Dist: django>4.2,<5.3.0
 Requires-Dist: dj_database_url>=1.3.0,<3.0.0
 Requires-Dist: pydantic[dotenv]<2.0.0
 Requires-Dist: appdirs<2.0.0

{lamindb_setup-0.73.2.dist-info → lamindb_setup-0.74.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-lamindb_setup/__init__.py,sha256=HyUA-3vUuT5XnO6MHe78x0OQaPP0s4-jb6QuFXEuY3s,1542
+lamindb_setup/__init__.py,sha256=cJQ2NUW0T7g_vpNTd21VpzvTGNT733H0Kb39cuMH0Sk,1542
 lamindb_setup/_cache.py,sha256=wA7mbysANwe8hPNbjDo9bOmXJ0xIyaS5iyxIpxSWji4,846
 lamindb_setup/_check.py,sha256=28PcG8Kp6OpjSLSi1r2boL2Ryeh6xkaCL87HFbjs6GA,129
 lamindb_setup/_check_setup.py,sha256=cNEL9Q4yPpmEkGKHH8JgullWl1VUZwALJ4RHn9wZypY,2613
@@ -8,11 +8,11 @@ lamindb_setup/_delete.py,sha256=Y8KSFYgY0UHAvjd7cCL6hZ_XiLeJwx50BguVATcj_Xo,5524
 lamindb_setup/_django.py,sha256=EoyWvFzH0i9wxjy4JZhcoXCTckztP_Mrl6FbYQnMmLE,1534
 lamindb_setup/_exportdb.py,sha256=uTIZjKKTB7arzEr1j0O6lONiT2pRBKeOFdLvOV8ZwzE,2120
 lamindb_setup/_importdb.py,sha256=yYYShzUajTsR-cTW4CZ-UNDWZY2uE5PAgNbp-wn8Ogc,1874
-lamindb_setup/_init_instance.py,sha256=Cji3h2kCj8Meukkm8Btl8Lu4Jio4NiWLlzAAKukEzYY,11896
-lamindb_setup/_migrate.py,sha256=49xPkwU-QQjpq0xSUepiVfvRsMhmjfbRlhd72YIq8o8,9059
-lamindb_setup/_register_instance.py,sha256=Jeu0wyvJVSVQ_n-A_7yn7xOZIP0ncJD92DRABqzPIjA,940
+lamindb_setup/_init_instance.py,sha256=e6OJSpN2eJZd22kAfUVEUbqXF48FzNfLc5xMTS8gZ84,11919
+lamindb_setup/_migrate.py,sha256=P4n3x0SYzO9szjF2-JMa7z4mQadtWjHv5ow4HbCDZLI,8864
+lamindb_setup/_register_instance.py,sha256=LlD2n44Gmh8OXrAkL9iL9NiML8QDKnk4d869MhXoOWI,1083
 lamindb_setup/_schema.py,sha256=b3uzhhWpV5mQtDwhMINc2MabGCnGLESy51ito3yl6Wc,679
-lamindb_setup/_schema_metadata.py,sha256=G3yXJ46OkVGqHxccCgjvQnnBNKg1uhwxXB1CddlkwYw,16882
+lamindb_setup/_schema_metadata.py,sha256=QL7Ord0BnKQnqZpLAjQX2gLhLwF_9zmZFcH8EjEMrfA,12895
 lamindb_setup/_set_managed_storage.py,sha256=mNZrANn-9rwZ0oGWxxg0wS0T0VOQCWyo4nSSyNAE15Q,1419
 lamindb_setup/_setup_user.py,sha256=6Oc7Rke-yRQSZbuntdUAz8QbJ6UuPzYHI9FnYlf_q-A,3670
 lamindb_setup/_silence_loggers.py,sha256=AKF_YcHvX32eGXdsYK8MJlxEaZ-Uo2f6QDRzjKFCtws,1568
@@ -21,25 +21,26 @@ lamindb_setup/core/_aws_credentials.py,sha256=uKMQO9q42Hnepz8aj3RxwLKDWUJx8pNOYr
 lamindb_setup/core/_aws_storage.py,sha256=nEjeUv4xUVpoV0Lx-zjjmyb9w804bDyaeiM-OqbfwM0,1799
 lamindb_setup/core/_deprecated.py,sha256=3qxUI1dnDlSeR0BYrv7ucjqRBEojbqotPgpShXs4KF8,2520
 lamindb_setup/core/_docs.py,sha256=3k-YY-oVaJd_9UIY-LfBg_u8raKOCNfkZQPA73KsUhs,276
-lamindb_setup/core/_hub_client.py,sha256=V0qKDsCdRn-tQy2YIk4VgXcpJFmuum6N3gcavAC7gBQ,5504
+lamindb_setup/core/_hub_client.py,sha256=g0Lmpv19xiUWoD67SpdT6re1EDfrd7D6Bsz4gIKpx_E,5504
 lamindb_setup/core/_hub_core.py,sha256=RGjTqf1owuWmkXAYy0EPaoHAaJ-0T0hAidkqa3cIdiM,16352
 lamindb_setup/core/_hub_crud.py,sha256=b1XF7AJpM9Q-ttm9nPG-r3OTRWHQaGzAGIyvmb83NTo,4859
 lamindb_setup/core/_hub_utils.py,sha256=b_M1LkdCjiMWm1EOlSb9GuPdLijwVgQDtATTpeZuXI0,1875
-lamindb_setup/core/_settings.py,sha256=jjZ_AxRXB3Y3UP6m04BAw_dhFbJbdg2-nZWmEv2LNZ8,3141
-lamindb_setup/core/_settings_instance.py,sha256=MJ1UArq49A168uGx9nOWDiXEKeSlDqtV-dqQf1kBcZw,17518
+lamindb_setup/core/_private_django_api.py,sha256=KIn43HOhiRjkbTbddyJqv-WNTTa1bAizbM1tWXoXPBg,2869
+lamindb_setup/core/_settings.py,sha256=46axQ5HPvI0X9YuotgdpuSOfSo7qYU1DudIx3vxpFk0,4471
+lamindb_setup/core/_settings_instance.py,sha256=OMN6ZROF7iV9Juh0bgE9fOmedE7EJI_YniztwJ5-acI,17623
 lamindb_setup/core/_settings_load.py,sha256=NGgCDpN85j1EqoKlrYFIlZBMlBJm33gx2-wc96CP_ZQ,3922
 lamindb_setup/core/_settings_save.py,sha256=d1A-Ex-7H08mb8l7I0Oe0j0GilrfaDuprh_NMxhQAsQ,2704
-lamindb_setup/core/_settings_storage.py,sha256=k4XyJR6_KpUpQuBYZp4mEdABiT91gTTfbK7tAVwqZCA,13093
+lamindb_setup/core/_settings_storage.py,sha256=65aobewYX6VfOeYZjZQOOI7ZD_3b4QA9TDmrduU0m4c,13262
 lamindb_setup/core/_settings_store.py,sha256=dagS5c7wAMRnuZTRfCU4sKaIOyF_HwAP5Fnnn8vphno,2084
 lamindb_setup/core/_settings_user.py,sha256=P2lC4WDRAFfT-Xq3MlXJ-wMKIHCoGNhMTQfRGIAyUNQ,1344
 lamindb_setup/core/_setup_bionty_sources.py,sha256=h_pBANsSGK6ujAFsG21mtADHVJoMLKDR4eGgRP4Fgls,3072
 lamindb_setup/core/cloud_sqlite_locker.py,sha256=reu02M4aE2BT_A5AFqwhv48l91mOMyQ4zTd-hh-wtuU,6922
 lamindb_setup/core/django.py,sha256=QUQm3zt5QIiD8uv6o9vbSm_bshqiSWzKSkgD3z2eJCg,3542
 lamindb_setup/core/exceptions.py,sha256=eoI7AXgATgDVzgArtN7CUvpaMUC067vsBg5LHCsWzDM,305
-lamindb_setup/core/hashing.py,sha256=7r96h5JBzuwfOR_gNNqTyWNPKMuiOUfBYwn6sCbZkf8,2269
+lamindb_setup/core/hashing.py,sha256=_JliYeCcjT_foOUJ5ck1rvcCo9q7r4b4SaSclQ_w4Qo,3071
 lamindb_setup/core/types.py,sha256=bcYnZ0uM_2NXKJCl94Mmc-uYrQlRUUVKG3sK2N-F-N4,532
 lamindb_setup/core/upath.py,sha256=dwudkTVsXuyjS-2xR16WomcWtXJAEfRZ0ZzFq8_EDhE,27157
-lamindb_setup-0.73.2.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
-lamindb_setup-0.73.2.dist-info/WHEEL,sha256=Sgu64hAMa6g5FdzHxXv9Xdse9yxpGGMeagVtPMWpJQY,99
-lamindb_setup-0.73.2.dist-info/METADATA,sha256=t3AiEbJJFkA0l-DY3DIDoLQHjBW--ToZWYS8vYNxdDU,1620
-lamindb_setup-0.73.2.dist-info/RECORD,,
+lamindb_setup-0.74.0.dist-info/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
+lamindb_setup-0.74.0.dist-info/WHEEL,sha256=Sgu64hAMa6g5FdzHxXv9Xdse9yxpGGMeagVtPMWpJQY,99
+lamindb_setup-0.74.0.dist-info/METADATA,sha256=JdkBfYklKlMuRjUc1VGMGFYWM3E-v0alqmYaXI6Q2kM,1620
+lamindb_setup-0.74.0.dist-info/RECORD,,