lamindb_setup 0.73.2__tar.gz → 0.74.0__tar.gz

{lamindb_setup-0.73.2 → lamindb_setup-0.74.0}/.github/workflows/build.yml

@@ -17,14 +17,14 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.9" # run one job on 3.9
+          python-version: "3.10" # run one job on 3.9
           cache: "pip"
       - uses: aws-actions/configure-aws-credentials@v2
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: eu-central-1
-      - run: pip install -U laminci
+      - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
       - run: nox -s "install(group='hub-prod')"
       - run: nox -s "build(lamin_env='prod', group='hub-prod')"
       - uses: actions/upload-artifact@v2
@@ -42,7 +42,7 @@ jobs:
           - lamin_env: "prod"
             python-version: "3.11"
           - lamin_env: "staging"
-            python-version: "3.10" # test on 3.10
+            python-version: "3.10" # test on 3.9
     timeout-minutes: 6
     steps:
       - uses: aws-actions/configure-aws-credentials@v2
@@ -56,10 +56,9 @@ jobs:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
           cache-dependency-path: ".github/workflows/build.yml"
-      - name: checkout laminhub
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
         with:
-          repository: laminlabs/laminapp-ui
+          repository: laminlabs/laminhub
           token: ${{ secrets.GH_TOKEN_DEPLOY_LAMINAPP }}
           path: laminhub
           ref: main
@@ -76,7 +75,7 @@ jobs:
         run: docker pull postgres:latest && docker image save postgres:latest --output ~/postgres.tar
       - if: steps.cache-postgres.outputs.cache-hit == 'true'
         run: docker image load --input ~/postgres.tar
-      - run: pip install -U laminci
+      - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
       # account for in a different job
       # - uses: "google-github-actions/auth@v0"
       #   with:
@@ -98,10 +97,10 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.11" # we need to run everything for coverage on 3.11
+          python-version: "3.9"
           cache: "pip"
           cache-dependency-path: ".github/workflows/build.yml"
-      - run: pip install -U laminci
+      - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
       - run: nox -s "install(group='storage')"
       - run: nox -s lint
       - run: nox -s storage
@@ -125,10 +124,9 @@ jobs:
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: eu-central-1
       - uses: actions/checkout@v4
-      - name: checkout laminhub
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
         with:
-          repository: laminlabs/laminapp-ui
+          repository: laminlabs/laminhub
           token: ${{ secrets.GH_TOKEN_DEPLOY_LAMINAPP }}
           path: laminhub
           ref: main
@@ -143,7 +141,7 @@ jobs:
           python-version: "3.11" # we need to run everything for coverage on 3.11
           cache: "pip"
           cache-dependency-path: ".github/workflows/build.yml"
-      - run: pip install -U laminci
+      - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
       - run: nox -s "install(group='hub-local')"
       - id: cache-supabase
         uses: actions/cache@v3
@@ -206,7 +204,7 @@ jobs:
           python-version: "3.10"
           cache: "pip"
           cache-dependency-path: ".github/workflows/build.yml"
-      - run: pip install -U laminci
+      - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
       - run: nox -s "install(group='docs')"
       - uses: actions/download-artifact@v2
       - run: nox -s docs

lamindb_setup-0.74.0/.github/workflows/doc-changes.yml

@@ -0,0 +1,23 @@
+name: doc-changes
+
+on:
+  pull_request_target:
+    branches:
+      - main
+    types:
+      - closed
+
+jobs:
+  latest-changes:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - run: pip install "laminci[doc-changes]@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
+      - run: laminci doc-changes
+        env:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          docs_token: ${{ secrets.LAMIN_BUILD_DOCS }}
+          changelog_file: lamin-docs/docs/changelog/soon/lamindb.md

{lamindb_setup-0.73.2 → lamindb_setup-0.74.0}/.gitignore

@@ -108,3 +108,4 @@ lamin_sphinx
 docs/conf.py
 _docs_tmp*
 *.lndb
+_is_initialized

{lamindb_setup-0.73.2 → lamindb_setup-0.74.0}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: lamindb_setup
-Version: 0.73.2
+Version: 0.74.0
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <laminlabs@gmail.com>
 Description-Content-Type: text/markdown
 Requires-Dist: lnschema_core>=0.51.0
 Requires-Dist: lamin_utils>=0.3.3
-Requires-Dist: django>4.2,<5.2.0
+Requires-Dist: django>4.2,<5.3.0
 Requires-Dist: dj_database_url>=1.3.0,<3.0.0
 Requires-Dist: pydantic[dotenv]<2.0.0
 Requires-Dist: appdirs<2.0.0

{lamindb_setup-0.73.2 → lamindb_setup-0.74.0}/docs/changelog.md

@@ -3,6 +3,11 @@
 <!-- prettier-ignore -->
 Name | PR | Developer | Date | Version
 --- | --- | --- | --- | ---
+🐛 Only search local managed storage locations | [788](https://github.com/laminlabs/lamindb-setup/pull/788) | [falexwolf](https://github.com/falexwolf) | 2024-07-03 |
+💚 Fix test | [784](https://github.com/laminlabs/lamindb-setup/pull/784) | [falexwolf](https://github.com/falexwolf) | 2024-06-26 | 0.73.3
+♻️ Save schema to hub upon init | [782](https://github.com/laminlabs/lamindb-setup/pull/782) | [falexwolf](https://github.com/falexwolf) | 2024-06-24 |
+💚 Fix schema serialization test | [781](https://github.com/laminlabs/lamindb-setup/pull/781) | [falexwolf](https://github.com/falexwolf) | 2024-06-24 |
+Better error message | [780](https://github.com/laminlabs/lamindb-setup/pull/780) | [falexwolf](https://github.com/falexwolf) | 2024-06-20 |
 🐛 Fix permission error when screening for local storage root | [778](https://github.com/laminlabs/lamindb-setup/pull/778) | [falexwolf](https://github.com/falexwolf) | 2024-06-19 | 0.73.2
 🐛 Fix same root check in get_locker | [777](https://github.com/laminlabs/lamindb-setup/pull/777) | [Koncopd](https://github.com/Koncopd) | 2024-06-13 |
 ✨ Add .ome.zarr as composite suffix | [775](https://github.com/laminlabs/lamindb-setup/pull/775) | [sunnyosun](https://github.com/sunnyosun) | 2024-06-05 |

{lamindb_setup-0.73.2 → lamindb_setup-0.74.0}/lamindb_setup/__init__.py

@@ -34,7 +34,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.73.2"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.74.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import sys
 from os import name as _os_name

{lamindb_setup-0.73.2 → lamindb_setup-0.74.0}/lamindb_setup/_init_instance.py

@@ -33,8 +33,7 @@ def get_schema_module_name(schema_name) -> str:
         if module_spec is not None:
             return name
     raise ImportError(
-        f"Python package for '{schema_name}' is not installed, tried two package names:"
-        f" {name_attempts}\nHave you installed the schema package using `pip install`?"
+        f"Python package for '{schema_name}' is not installed.\nIf your package is on PyPI, run `pip install {schema_name}`"
     )
 
 
@@ -256,7 +255,10 @@ def init(
             schema=schema,
             uid=ssettings.uid,
         )
-        if isettings.is_remote and instance_state != "instance-corrupted-or-deleted":
+        register_on_hub = (
+            isettings.is_remote and instance_state != "instance-corrupted-or-deleted"
+        )
+        if register_on_hub:
             init_instance_hub(isettings)
         validate_sqlite_state(isettings)
         isettings._persist()
@@ -270,8 +272,10 @@ def init(
                 "locked instance (to unlock and push changes to the cloud SQLite file,"
                 " call: lamin close)"
             )
-        # we can debate whether this is the right setting, but this is how
-        # things have been and we'd like to not easily break backward compat
+        if register_on_hub and isettings.dialect != "sqlite":
+            from ._schema_metadata import update_schema_in_hub
+
+            update_schema_in_hub()
         settings.auto_connect = True
     except Exception as e:
         from ._delete import delete_by_isettings

{lamindb_setup-0.73.2 → lamindb_setup-0.74.0}/lamindb_setup/_migrate.py

@@ -75,21 +75,15 @@ class migrate:
         from lamindb_setup.core._hub_client import call_with_fallback_auth
         from lamindb_setup.core._hub_crud import (
             select_collaborator,
-            select_instance_by_id,
             update_instance,
         )
 
-        instance_id_str = settings.instance._id.hex
-        instance = call_with_fallback_auth(
-            select_instance_by_id, instance_id=instance_id_str
-        )
-        instance_is_on_hub = instance is not None
-        if instance_is_on_hub:
+        if settings.instance.is_on_hub:
             # double check that user is an admin, otherwise will fail below
-            # without idempotence
+            # due to insufficient SQL permissions with cryptic error
             collaborator = call_with_fallback_auth(
                 select_collaborator,
-                instance_id=instance_id_str,
+                instance_id=settings.instance._id,
                 account_id=settings.user._uuid,
             )
             if collaborator is None or collaborator["role"] != "admin":
@@ -104,7 +98,7 @@ class migrate:
         # this sets up django and deploys the migrations
         setup_django(settings.instance, deploy_migrations=True)
         # this populates the hub
-        if instance_is_on_hub:
+        if settings.instance.is_on_hub:
             logger.important(f"updating lamindb version in hub: {lamindb.__version__}")
             # TODO: integrate update of instance table within update_schema_in_hub & below
             if settings.instance.dialect != "sqlite":

{lamindb_setup-0.73.2 → lamindb_setup-0.74.0}/lamindb_setup/_register_instance.py

@@ -27,3 +27,7 @@ def register(_test: bool = False):
     init_instance_hub(isettings)
     isettings._is_on_hub = True
     isettings._persist()
+    if isettings.dialect != "sqlite" and not _test:
+        from ._schema_metadata import update_schema_in_hub
+
+        update_schema_in_hub()

lamindb_setup-0.74.0/lamindb_setup/_schema_metadata.py

@@ -0,0 +1,378 @@
+from __future__ import annotations
+
+import hashlib
+import importlib
+import json
+from typing import TYPE_CHECKING, Literal
+from uuid import UUID
+
+from django.db.models import (
+    Field,
+    ForeignKey,
+    ForeignObjectRel,
+    ManyToManyField,
+    ManyToManyRel,
+    ManyToOneRel,
+    OneToOneField,
+    OneToOneRel,
+)
+from pydantic import BaseModel
+
+from lamindb_setup import settings
+from lamindb_setup._init_instance import get_schema_module_name
+from lamindb_setup.core._hub_client import call_with_fallback_auth
+
+if TYPE_CHECKING:
+    from supabase import Client
+
+
+def update_schema_in_hub() -> tuple[bool, UUID, dict]:
+    return call_with_fallback_auth(_synchronize_schema)
+
+
+def _synchronize_schema(client: Client) -> tuple[bool, UUID, dict]:
+    schema_metadata = _SchemaHandler()
+    schema_metadata_dict = schema_metadata.to_json()
+    schema_uuid = _dict_to_uuid(schema_metadata_dict)
+    schema = _get_schema_by_id(schema_uuid, client)
+
+    is_new = schema is None
+    if is_new:
+        module_set_info = schema_metadata._get_module_set_info()
+        module_ids = "-".join(str(module_info["id"]) for module_info in module_set_info)
+        schema = (
+            client.table("schema")
+            .insert(
+                {
+                    "id": schema_uuid.hex,
+                    "module_ids": module_ids,
+                    "module_set_info": module_set_info,
+                    "schema_json": schema_metadata_dict,
+                }
+            )
+            .execute()
+            .data[0]
+        )
+
+    instance_response = (
+        client.table("instance")
+        .update({"schema_id": schema_uuid.hex})
+        .eq("id", settings.instance._id.hex)
+        .execute()
+    )
+    assert (
+        len(instance_response.data) == 1
+    ), f"schema of instance {settings.instance._id.hex} could not be updated with schema {schema_uuid.hex}"
+
+    return is_new, schema_uuid, schema
+
+
+def get_schema_by_id(id: UUID):
+    return call_with_fallback_auth(_get_schema_by_id, id=id)
+
+
+def _get_schema_by_id(id: UUID, client: Client):
+    response = client.table("schema").select("*").eq("id", id.hex).execute()
+    if len(response.data) == 0:
+        return None
+    return response.data[0]
+
+
+def _dict_to_uuid(dict: dict):
+    encoded = json.dumps(dict, sort_keys=True).encode("utf-8")
+    hash = hashlib.md5(encoded).digest()
+    uuid = UUID(bytes=hash[:16])
+    return uuid
+
+
+RelationType = Literal["many-to-one", "one-to-many", "many-to-many", "one-to-one"]
+Type = Literal[
+    "ForeignKey",
+    "CharField",
+    "DateTimeField",
+    "AutoField",
+    "BooleanField",
+    "BigIntegerField",
+    "SmallIntegerField",
+    "TextField",
+    "BigAutoField",
+    "ManyToManyField",
+    "IntegerField",
+    "OneToOneField",
+    "JSONField",
+    "DateField",
+    "FloatField",
+]
+
+
+class Through(BaseModel):
+    left_key: str
+    right_key: str
+    link_table_name: str | None = None
+
+
+class FieldMetadata(BaseModel):
+    type: Type
+    column: str | None = None
+    through: Through | None = None
+    field_name: str
+    model_name: str
+    schema_name: str
+    is_link_table: bool
+    relation_type: RelationType | None = None
+    related_field_name: str | None = None
+    related_model_name: str | None = None
+    related_schema_name: str | None = None
+
+
+class _ModelHandler:
+    def __init__(self, model, module_name: str, included_modules: list[str]) -> None:
+        self.model = model
+        self.class_name = model.__name__
+        self.module_name = module_name
+        self.model_name = model._meta.model_name
+        self.table_name = model._meta.db_table
+        self.included_modules = included_modules
+        self.fields = self._get_fields_metadata(self.model)
+
+    def to_dict(self, include_django_objects: bool = True):
+        _dict = {
+            "fields": self.fields.copy(),
+            "class_name": self.class_name,
+            "table_name": self.table_name,
+        }
+
+        for field_name in self.fields.keys():
+            _dict["fields"][field_name] = _dict["fields"][field_name].__dict__
+            through = _dict["fields"][field_name]["through"]
+            if through is not None:
+                _dict["fields"][field_name]["through"] = through.__dict__
+
+        if include_django_objects:
+            _dict.update({"model": self.model})
+
+        return _dict
+
+    def _get_fields_metadata(self, model):
+        related_fields = []
+        fields_metadata: dict[str, FieldMetadata] = {}
+
+        for field in model._meta.get_fields():
+            field_metadata = self._get_field_metadata(model, field)
+            if field_metadata.related_schema_name is None:
+                fields_metadata.update({field.name: field_metadata})
+
+            if (
+                field_metadata.related_schema_name in self.included_modules
+                and field_metadata.schema_name in self.included_modules
+            ):
+                related_fields.append(field)
+
+        related_fields_metadata = self._get_related_fields_metadata(
+            model, related_fields
+        )
+
+        fields_metadata = {**fields_metadata, **related_fields_metadata}
+
+        return fields_metadata
+
+    def _get_related_fields_metadata(self, model, fields: list[ForeignObjectRel]):
+        related_fields: dict[str, FieldMetadata] = {}
+
+        for field in fields:
+            if field.many_to_one:
+                related_fields.update(
+                    {f"{field.name}": self._get_field_metadata(model, field)}
+                )
+            elif field.one_to_many:
+                # exclude self reference as it is already included in the many to one
+                if field.related_model == model:
+                    continue
+                related_fields.update(
+                    {f"{field.name}": self._get_field_metadata(model, field.field)}
+                )
+            elif field.many_to_many:
+                related_fields.update(
+                    {f"{field.name}": self._get_field_metadata(model, field)}
+                )
+            elif field.one_to_one:
+                related_fields.update(
+                    {f"{field.name}": self._get_field_metadata(model, field)}
+                )
+
+        return related_fields
+
+    def _get_field_metadata(self, model, field: Field):
+        from lnschema_core.models import LinkORM
+
+        internal_type = field.get_internal_type()
+        model_name = field.model._meta.model_name
+        relation_type = self._get_relation_type(model, field)
+        if field.related_model is None:
+            schema_name = field.model.__get_schema_name__()
+            related_model_name = None
+            related_schema_name = None
+            related_field_name = None
+            field_name = field.name
+        else:
+            related_model_name = field.related_model._meta.model_name
+            related_schema_name = field.related_model.__get_schema_name__()
+            schema_name = field.model.__get_schema_name__()
+            related_field_name = field.remote_field.name
+            field_name = field.name
+
+        if relation_type in ["one-to-many"]:
+            # For a one-to-many relation, the field belong
+            # to the other model as a foreign key.
+            # To make usage similar to other relation types
+            # we need to invert model and related model.
+            schema_name, related_schema_name = related_schema_name, schema_name
+            model_name, related_model_name = related_model_name, model_name
+            field_name, related_field_name = related_field_name, field_name
+            pass
+
+        column = None
+        if relation_type not in ["many-to-many", "one-to-many"]:
+            if not isinstance(field, ForeignObjectRel):
+                column = field.column
+
+        if relation_type is None:
+            through = None
+        elif relation_type == "many-to-many":
+            through = self._get_through_many_to_many(field)
+        else:
+            through = self._get_through(field)
+
+        return FieldMetadata(
+            schema_name=schema_name,
+            model_name=model_name,
+            field_name=field_name,
+            type=internal_type,
+            is_link_table=issubclass(field.model, LinkORM),
+            column=column,
+            relation_type=relation_type,
+            related_schema_name=related_schema_name,
+            related_model_name=related_model_name,
+            related_field_name=related_field_name,
+            through=through,
+        )
+
+    @staticmethod
+    def _get_through_many_to_many(field_or_rel: ManyToManyField | ManyToManyRel):
+        from lnschema_core.models import Registry
+
+        if isinstance(field_or_rel, ManyToManyField):
+            if field_or_rel.model != Registry:
+                return Through(
+                    left_key=field_or_rel.m2m_column_name(),
+                    right_key=field_or_rel.m2m_reverse_name(),
+                    link_table_name=field_or_rel.remote_field.through._meta.db_table,
+                )
+            else:
+                return Through(
+                    left_key=field_or_rel.m2m_reverse_name(),
+                    right_key=field_or_rel.m2m_column_name(),
+                    link_table_name=field_or_rel.remote_field.through._meta.db_table,
+                )
+
+        if isinstance(field_or_rel, ManyToManyRel):
+            if field_or_rel.model != Registry:
+                return Through(
+                    left_key=field_or_rel.field.m2m_reverse_name(),
+                    right_key=field_or_rel.field.m2m_column_name(),
+                    link_table_name=field_or_rel.through._meta.db_table,
+                )
+            else:
+                return Through(
+                    left_key=field_or_rel.field.m2m_column_name(),
+                    right_key=field_or_rel.field.m2m_reverse_name(),
+                    link_table_name=field_or_rel.through._meta.db_table,
+                )
+
+    def _get_through(
+        self, field_or_rel: ForeignKey | OneToOneField | ManyToOneRel | OneToOneRel
+    ):
+        if isinstance(field_or_rel, ForeignObjectRel):
+            rel_1 = field_or_rel.field.related_fields[0][0]
+            rel_2 = field_or_rel.field.related_fields[0][1]
+        else:
+            rel_1 = field_or_rel.related_fields[0][0]
+            rel_2 = field_or_rel.related_fields[0][1]
+
+        if rel_1.model._meta.model_name == self.model._meta.model_name:
+            return Through(
+                left_key=rel_1.column,
+                right_key=rel_2.column,
+            )
+        else:
+            return Through(
+                left_key=rel_2.column,
+                right_key=rel_1.column,
+            )
+
+    @staticmethod
+    def _get_relation_type(model, field: Field):
+        if field.many_to_one:
+            # defined in the model
+            if model == field.model:
+                return "many-to-one"
+            # defined in the related model
+            else:
+                return "one-to-many"
+        elif field.one_to_many:
+            return "one-to-many"
+        elif field.many_to_many:
+            return "many-to-many"
+        elif field.one_to_one:
+            return "one-to-one"
+        else:
+            return None
+
+
+class _SchemaHandler:
+    def __init__(self) -> None:
+        self.included_modules = ["core"] + list(settings.instance.schema)
+        self.modules = self._get_modules_metadata()
+
+    def to_dict(self, include_django_objects: bool = True):
+        return {
+            module_name: {
+                model_name: model.to_dict(include_django_objects)
+                for model_name, model in module.items()
+            }
+            for module_name, module in self.modules.items()
+        }
+
+    def to_json(self):
+        return self.to_dict(include_django_objects=False)
+
+    def _get_modules_metadata(self):
+        return {
+            module_name: {
+                model._meta.model_name: _ModelHandler(
+                    model, module_name, self.included_modules
+                )
+                for model in self._get_schema_module(
+                    module_name
+                ).models.__dict__.values()
+                if model.__class__.__name__ == "RegistryMeta"
+                and model.__name__ not in ["Registry", "ORM"]
+                and not model._meta.abstract
+                and model.__get_schema_name__() == module_name
+            }
+            for module_name in self.included_modules
+        }
+
+    def _get_module_set_info(self):
+        # TODO: rely on schemamodule table for this
+        module_set_info = []
+        for module_name in self.included_modules:
+            module = self._get_schema_module(module_name)
+            module_set_info.append(
+                {"id": 0, "name": module_name, "version": module.__version__}
+            )
+        return module_set_info
+
+    @staticmethod
+    def _get_schema_module(module_name):
+        return importlib.import_module(get_schema_module_name(module_name))

{lamindb_setup-0.73.2 → lamindb_setup-0.74.0}/lamindb_setup/core/_hub_client.py

@@ -119,8 +119,8 @@ def call_with_fallback_auth(
     for renew_token, fallback_env in [(False, False), (True, False), (False, True)]:
         try:
             if renew_token:
-                logger.important(
-                    "Renewing expired lamin token: call lamin login to avoid this"
+                logger.warning(
+                    "renewing expired lamin token: call `lamin login` to avoid this"
                 )
             client = connect_hub_with_auth(
                 renew_token=renew_token, fallback_env=fallback_env