lamindb_setup 0.73.3__tar.gz → 0.74.1__tar.gz

{lamindb_setup-0.73.3 → lamindb_setup-0.74.1}/.github/workflows/build.yml

@@ -17,14 +17,14 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.9" # run one job on 3.9
+          python-version: "3.10" # run one job on 3.9
           cache: "pip"
       - uses: aws-actions/configure-aws-credentials@v2
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: eu-central-1
-      - run: pip install -U laminci
+      - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
       - run: nox -s "install(group='hub-prod')"
       - run: nox -s "build(lamin_env='prod', group='hub-prod')"
       - uses: actions/upload-artifact@v2
@@ -42,7 +42,7 @@ jobs:
           - lamin_env: "prod"
             python-version: "3.11"
           - lamin_env: "staging"
-            python-version: "3.10" # test on 3.10
+            python-version: "3.10" # test on 3.9
     timeout-minutes: 6
     steps:
       - uses: aws-actions/configure-aws-credentials@v2
@@ -56,10 +56,9 @@ jobs:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
           cache-dependency-path: ".github/workflows/build.yml"
-      - name: checkout laminhub
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
         with:
-          repository: laminlabs/laminapp-ui
+          repository: laminlabs/laminhub
           token: ${{ secrets.GH_TOKEN_DEPLOY_LAMINAPP }}
           path: laminhub
           ref: main
@@ -76,7 +75,7 @@ jobs:
         run: docker pull postgres:latest && docker image save postgres:latest --output ~/postgres.tar
       - if: steps.cache-postgres.outputs.cache-hit == 'true'
         run: docker image load --input ~/postgres.tar
-      - run: pip install -U laminci
+      - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
       # account for in a different job
       # - uses: "google-github-actions/auth@v0"
       #   with:
@@ -98,10 +97,10 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.11" # we need to run everything for coverage on 3.11
+          python-version: "3.9"
           cache: "pip"
           cache-dependency-path: ".github/workflows/build.yml"
-      - run: pip install -U laminci
+      - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
       - run: nox -s "install(group='storage')"
       - run: nox -s lint
       - run: nox -s storage
@@ -125,10 +124,9 @@ jobs:
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: eu-central-1
       - uses: actions/checkout@v4
-      - name: checkout laminhub
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
         with:
-          repository: laminlabs/laminapp-ui
+          repository: laminlabs/laminhub
           token: ${{ secrets.GH_TOKEN_DEPLOY_LAMINAPP }}
           path: laminhub
           ref: main
@@ -143,7 +141,7 @@ jobs:
           python-version: "3.11" # we need to run everything for coverage on 3.11
           cache: "pip"
           cache-dependency-path: ".github/workflows/build.yml"
-      - run: pip install -U laminci
+      - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
       - run: nox -s "install(group='hub-local')"
       - id: cache-supabase
         uses: actions/cache@v3
@@ -206,7 +204,7 @@ jobs:
           python-version: "3.10"
           cache: "pip"
           cache-dependency-path: ".github/workflows/build.yml"
-      - run: pip install -U laminci
+      - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
       - run: nox -s "install(group='docs')"
       - uses: actions/download-artifact@v2
       - run: nox -s docs

lamindb_setup-0.74.1/.github/workflows/doc-changes.yml

@@ -0,0 +1,23 @@
+name: doc-changes
+
+on:
+  pull_request_target:
+    branches:
+      - main
+    types:
+      - closed
+
+jobs:
+  latest-changes:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - run: pip install "laminci[doc-changes]@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
+      - run: laminci doc-changes
+        env:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          docs_token: ${{ secrets.LAMIN_BUILD_DOCS }}
+          changelog_file: lamin-docs/docs/changelog/soon/lamindb.md

{lamindb_setup-0.73.3 → lamindb_setup-0.74.1}/PKG-INFO

@@ -1,12 +1,12 @@
 Metadata-Version: 2.1
 Name: lamindb_setup
-Version: 0.73.3
+Version: 0.74.1
 Summary: Setup & configure LaminDB.
 Author-email: Lamin Labs <laminlabs@gmail.com>
 Description-Content-Type: text/markdown
 Requires-Dist: lnschema_core>=0.51.0
 Requires-Dist: lamin_utils>=0.3.3
-Requires-Dist: django>4.2,<5.2.0
+Requires-Dist: django>4.2,<5.3.0
 Requires-Dist: dj_database_url>=1.3.0,<3.0.0
 Requires-Dist: pydantic[dotenv]<2.0.0
 Requires-Dist: appdirs<2.0.0
@@ -14,6 +14,7 @@ Requires-Dist: requests
 Requires-Dist: universal_pathlib==0.2.2
 Requires-Dist: botocore<2.0.0
 Requires-Dist: supabase==2.2.1
+Requires-Dist: psutil
 Requires-Dist: urllib3<2 ; extra == "aws"
 Requires-Dist: aiobotocore[boto3]>=2.5.4,<3.0.0 ; extra == "aws"
 Requires-Dist: s3fs>=2023.12.2,<=2024.3.1 ; extra == "aws"

{lamindb_setup-0.73.3 → lamindb_setup-0.74.1}/docs/changelog.md

@@ -3,6 +3,7 @@
 <!-- prettier-ignore -->
 Name | PR | Developer | Date | Version
 --- | --- | --- | --- | ---
+🐛 Only search local managed storage locations | [788](https://github.com/laminlabs/lamindb-setup/pull/788) | [falexwolf](https://github.com/falexwolf) | 2024-07-03 |
 💚 Fix test | [784](https://github.com/laminlabs/lamindb-setup/pull/784) | [falexwolf](https://github.com/falexwolf) | 2024-06-26 | 0.73.3
 ♻️ Save schema to hub upon init | [782](https://github.com/laminlabs/lamindb-setup/pull/782) | [falexwolf](https://github.com/falexwolf) | 2024-06-24 |
 💚 Fix schema serialization test | [781](https://github.com/laminlabs/lamindb-setup/pull/781) | [falexwolf](https://github.com/falexwolf) | 2024-06-24 |

{lamindb_setup-0.73.3 → lamindb_setup-0.74.1}/lamindb_setup/__init__.py

@@ -34,7 +34,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.73.3"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.74.1"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import sys
 from os import name as _os_name

{lamindb_setup-0.73.3 → lamindb_setup-0.74.1}/lamindb_setup/_connect_instance.py

@@ -270,12 +270,6 @@ def load(
     This is exactly the same as ``ln.connect()`` except for that
     ``ln.connect()`` doesn't change the state of ``auto-connect``.
     """
-    # enable the message in the next release
-    logger.warning(
-        "`lamin connect` replaces `lamin load`, which will be removed in a future"
-        " version\nif you still want to auto-connect to an instance upon lamindb"
-        " import, call: `lamin load` on the CLI"
-    )
     result = connect(slug, db=db, storage=storage)
     settings.auto_connect = True
     return result

lamindb_setup-0.74.1/lamindb_setup/_schema_metadata.py

@@ -0,0 +1,378 @@
+from __future__ import annotations
+
+import hashlib
+import importlib
+import json
+from typing import TYPE_CHECKING, Literal
+from uuid import UUID
+
+from django.db.models import (
+    Field,
+    ForeignKey,
+    ForeignObjectRel,
+    ManyToManyField,
+    ManyToManyRel,
+    ManyToOneRel,
+    OneToOneField,
+    OneToOneRel,
+)
+from pydantic import BaseModel
+
+from lamindb_setup import settings
+from lamindb_setup._init_instance import get_schema_module_name
+from lamindb_setup.core._hub_client import call_with_fallback_auth
+
+if TYPE_CHECKING:
+    from supabase import Client
+
+
+def update_schema_in_hub() -> tuple[bool, UUID, dict]:
+    return call_with_fallback_auth(_synchronize_schema)
+
+
+def _synchronize_schema(client: Client) -> tuple[bool, UUID, dict]:
+    schema_metadata = _SchemaHandler()
+    schema_metadata_dict = schema_metadata.to_json()
+    schema_uuid = _dict_to_uuid(schema_metadata_dict)
+    schema = _get_schema_by_id(schema_uuid, client)
+
+    is_new = schema is None
+    if is_new:
+        module_set_info = schema_metadata._get_module_set_info()
+        module_ids = "-".join(str(module_info["id"]) for module_info in module_set_info)
+        schema = (
+            client.table("schema")
+            .insert(
+                {
+                    "id": schema_uuid.hex,
+                    "module_ids": module_ids,
+                    "module_set_info": module_set_info,
+                    "schema_json": schema_metadata_dict,
+                }
+            )
+            .execute()
+            .data[0]
+        )
+
+    instance_response = (
+        client.table("instance")
+        .update({"schema_id": schema_uuid.hex})
+        .eq("id", settings.instance._id.hex)
+        .execute()
+    )
+    assert (
+        len(instance_response.data) == 1
+    ), f"schema of instance {settings.instance._id.hex} could not be updated with schema {schema_uuid.hex}"
+
+    return is_new, schema_uuid, schema
+
+
+def get_schema_by_id(id: UUID):
+    return call_with_fallback_auth(_get_schema_by_id, id=id)
+
+
+def _get_schema_by_id(id: UUID, client: Client):
+    response = client.table("schema").select("*").eq("id", id.hex).execute()
+    if len(response.data) == 0:
+        return None
+    return response.data[0]
+
+
+def _dict_to_uuid(dict: dict):
+    encoded = json.dumps(dict, sort_keys=True).encode("utf-8")
+    hash = hashlib.md5(encoded).digest()
+    uuid = UUID(bytes=hash[:16])
+    return uuid
+
+
+RelationType = Literal["many-to-one", "one-to-many", "many-to-many", "one-to-one"]
+Type = Literal[
+    "ForeignKey",
+    "CharField",
+    "DateTimeField",
+    "AutoField",
+    "BooleanField",
+    "BigIntegerField",
+    "SmallIntegerField",
+    "TextField",
+    "BigAutoField",
+    "ManyToManyField",
+    "IntegerField",
+    "OneToOneField",
+    "JSONField",
+    "DateField",
+    "FloatField",
+]
+
+
+class Through(BaseModel):
+    left_key: str
+    right_key: str
+    link_table_name: str | None = None
+
+
+class FieldMetadata(BaseModel):
+    type: Type
+    column_name: str | None = None
+    through: Through | None = None
+    field_name: str
+    model_name: str
+    schema_name: str
+    is_link_table: bool
+    relation_type: RelationType | None = None
+    related_field_name: str | None = None
+    related_model_name: str | None = None
+    related_schema_name: str | None = None
+
+
+class _ModelHandler:
+    def __init__(self, model, module_name: str, included_modules: list[str]) -> None:
+        self.model = model
+        self.class_name = model.__name__
+        self.module_name = module_name
+        self.model_name = model._meta.model_name
+        self.table_name = model._meta.db_table
+        self.included_modules = included_modules
+        self.fields = self._get_fields_metadata(self.model)
+
+    def to_dict(self, include_django_objects: bool = True):
+        _dict = {
+            "fields": self.fields.copy(),
+            "class_name": self.class_name,
+            "table_name": self.table_name,
+        }
+
+        for field_name in self.fields.keys():
+            _dict["fields"][field_name] = _dict["fields"][field_name].__dict__
+            through = _dict["fields"][field_name]["through"]
+            if through is not None:
+                _dict["fields"][field_name]["through"] = through.__dict__
+
+        if include_django_objects:
+            _dict.update({"model": self.model})
+
+        return _dict
+
+    def _get_fields_metadata(self, model):
+        related_fields = []
+        fields_metadata: dict[str, FieldMetadata] = {}
+
+        for field in model._meta.get_fields():
+            field_metadata = self._get_field_metadata(model, field)
+            if field_metadata.related_schema_name is None:
+                fields_metadata.update({field.name: field_metadata})
+
+            if (
+                field_metadata.related_schema_name in self.included_modules
+                and field_metadata.schema_name in self.included_modules
+            ):
+                related_fields.append(field)
+
+        related_fields_metadata = self._get_related_fields_metadata(
+            model, related_fields
+        )
+
+        fields_metadata = {**fields_metadata, **related_fields_metadata}
+
+        return fields_metadata
+
+    def _get_related_fields_metadata(self, model, fields: list[ForeignObjectRel]):
+        related_fields: dict[str, FieldMetadata] = {}
+
+        for field in fields:
+            if field.many_to_one:
+                related_fields.update(
+                    {f"{field.name}": self._get_field_metadata(model, field)}
+                )
+            elif field.one_to_many:
+                # exclude self reference as it is already included in the many to one
+                if field.related_model == model:
+                    continue
+                related_fields.update(
+                    {f"{field.name}": self._get_field_metadata(model, field.field)}
+                )
+            elif field.many_to_many:
+                related_fields.update(
+                    {f"{field.name}": self._get_field_metadata(model, field)}
+                )
+            elif field.one_to_one:
+                related_fields.update(
+                    {f"{field.name}": self._get_field_metadata(model, field)}
+                )
+
+        return related_fields
+
+    def _get_field_metadata(self, model, field: Field):
+        from lnschema_core.models import LinkORM
+
+        internal_type = field.get_internal_type()
+        model_name = field.model._meta.model_name
+        relation_type = self._get_relation_type(model, field)
+        if field.related_model is None:
+            schema_name = field.model.__get_schema_name__()
+            related_model_name = None
+            related_schema_name = None
+            related_field_name = None
+            field_name = field.name
+        else:
+            related_model_name = field.related_model._meta.model_name
+            related_schema_name = field.related_model.__get_schema_name__()
+            schema_name = field.model.__get_schema_name__()
+            related_field_name = field.remote_field.name
+            field_name = field.name
+
+        if relation_type in ["one-to-many"]:
+            # For a one-to-many relation, the field belong
+            # to the other model as a foreign key.
+            # To make usage similar to other relation types
+            # we need to invert model and related model.
+            schema_name, related_schema_name = related_schema_name, schema_name
+            model_name, related_model_name = related_model_name, model_name
+            field_name, related_field_name = related_field_name, field_name
+            pass
+
+        column = None
+        if relation_type not in ["many-to-many", "one-to-many"]:
+            if not isinstance(field, ForeignObjectRel):
+                column = field.column
+
+        if relation_type is None:
+            through = None
+        elif relation_type == "many-to-many":
+            through = self._get_through_many_to_many(field)
+        else:
+            through = self._get_through(field)
+
+        return FieldMetadata(
+            schema_name=schema_name,
+            model_name=model_name,
+            field_name=field_name,
+            type=internal_type,
+            is_link_table=issubclass(field.model, LinkORM),
+            column_name=column,
+            relation_type=relation_type,
+            related_schema_name=related_schema_name,
+            related_model_name=related_model_name,
+            related_field_name=related_field_name,
+            through=through,
+        )
+
+    @staticmethod
+    def _get_through_many_to_many(field_or_rel: ManyToManyField | ManyToManyRel):
+        from lnschema_core.models import Registry
+
+        if isinstance(field_or_rel, ManyToManyField):
+            if field_or_rel.model != Registry:
+                return Through(
+                    left_key=field_or_rel.m2m_column_name(),
+                    right_key=field_or_rel.m2m_reverse_name(),
+                    link_table_name=field_or_rel.remote_field.through._meta.db_table,
+                )
+            else:
+                return Through(
+                    left_key=field_or_rel.m2m_reverse_name(),
+                    right_key=field_or_rel.m2m_column_name(),
+                    link_table_name=field_or_rel.remote_field.through._meta.db_table,
+                )
+
+        if isinstance(field_or_rel, ManyToManyRel):
+            if field_or_rel.model != Registry:
+                return Through(
+                    left_key=field_or_rel.field.m2m_reverse_name(),
+                    right_key=field_or_rel.field.m2m_column_name(),
+                    link_table_name=field_or_rel.through._meta.db_table,
+                )
+            else:
+                return Through(
+                    left_key=field_or_rel.field.m2m_column_name(),
+                    right_key=field_or_rel.field.m2m_reverse_name(),
+                    link_table_name=field_or_rel.through._meta.db_table,
+                )
+
+    def _get_through(
+        self, field_or_rel: ForeignKey | OneToOneField | ManyToOneRel | OneToOneRel
+    ):
+        if isinstance(field_or_rel, ForeignObjectRel):
+            rel_1 = field_or_rel.field.related_fields[0][0]
+            rel_2 = field_or_rel.field.related_fields[0][1]
+        else:
+            rel_1 = field_or_rel.related_fields[0][0]
+            rel_2 = field_or_rel.related_fields[0][1]
+
+        if rel_1.model._meta.model_name == self.model._meta.model_name:
+            return Through(
+                left_key=rel_1.column,
+                right_key=rel_2.column,
+            )
+        else:
+            return Through(
+                left_key=rel_2.column,
+                right_key=rel_1.column,
+            )
+
+    @staticmethod
+    def _get_relation_type(model, field: Field):
+        if field.many_to_one:
+            # defined in the model
+            if model == field.model:
+                return "many-to-one"
+            # defined in the related model
+            else:
+                return "one-to-many"
+        elif field.one_to_many:
+            return "one-to-many"
+        elif field.many_to_many:
+            return "many-to-many"
+        elif field.one_to_one:
+            return "one-to-one"
+        else:
+            return None
+
+
+class _SchemaHandler:
+    def __init__(self) -> None:
+        self.included_modules = ["core"] + list(settings.instance.schema)
+        self.modules = self._get_modules_metadata()
+
+    def to_dict(self, include_django_objects: bool = True):
+        return {
+            module_name: {
+                model_name: model.to_dict(include_django_objects)
+                for model_name, model in module.items()
+            }
+            for module_name, module in self.modules.items()
+        }
+
+    def to_json(self):
+        return self.to_dict(include_django_objects=False)
+
+    def _get_modules_metadata(self):
+        return {
+            module_name: {
+                model._meta.model_name: _ModelHandler(
+                    model, module_name, self.included_modules
+                )
+                for model in self._get_schema_module(
+                    module_name
+                ).models.__dict__.values()
+                if model.__class__.__name__ == "RecordMeta"
+                and model.__name__ not in ["Record", "ORM"]
+                and not model._meta.abstract
+                and model.__get_schema_name__() == module_name
+            }
+            for module_name in self.included_modules
+        }
+
+    def _get_module_set_info(self):
+        # TODO: rely on schemamodule table for this
+        module_set_info = []
+        for module_name in self.included_modules:
+            module = self._get_schema_module(module_name)
+            module_set_info.append(
+                {"id": 0, "name": module_name, "version": module.__version__}
+            )
+        return module_set_info
+
+    @staticmethod
+    def _get_schema_module(module_name):
+        return importlib.import_module(get_schema_module_name(module_name))

{lamindb_setup-0.73.3 → lamindb_setup-0.74.1}/lamindb_setup/core/_hub_client.py

@@ -119,8 +119,8 @@ def call_with_fallback_auth(
     for renew_token, fallback_env in [(False, False), (True, False), (False, True)]:
         try:
             if renew_token:
-                logger.important(
-                    "Renewing expired lamin token: call lamin login to avoid this"
+                logger.warning(
+                    "renewing expired lamin token: call `lamin login` to avoid this"
                 )
             client = connect_hub_with_auth(
                 renew_token=renew_token, fallback_env=fallback_env

lamindb_setup-0.74.1/lamindb_setup/core/_private_django_api.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+
+def find_vscode_stubs_folder() -> Path | None:
+    # Possible locations of VSCode extensions
+    possible_locations = [
+        Path.home() / ".vscode" / "extensions",  # Linux and macOS
+        Path.home() / ".vscode-server" / "extensions",  # Remote development
+        Path(os.environ.get("APPDATA", "")) / "Code" / "User" / "extensions",  # Windows
+        Path("/usr/share/code/resources/app/extensions"),  # Some Linux distributions
+    ]
+    for location in possible_locations:
+        if location.exists():
+            # Look for Pylance extension folder
+            pylance_folders = list(location.glob("ms-python.vscode-pylance-*"))
+            if pylance_folders:
+                # Sort to get the latest version
+                latest_pylance = sorted(pylance_folders)[-1]
+                stubs_folder = (
+                    latest_pylance / "dist" / "bundled" / "stubs" / "django-stubs"
+                )
+                if stubs_folder.exists():
+                    return stubs_folder
+
+    return None
+
+
+def private_django_api(reverse=False):
+    from django import db
+
+    # the order here matters
+    # changing it might break the tests
+    attributes = [
+        "DoesNotExist",
+        "MultipleObjectsReturned",
+        "add_to_class",
+        "adelete",
+        "refresh_from_db",
+        "asave",
+        "clean",
+        "clean_fields",
+        "date_error_message",
+        "get_constraints",
+        "get_deferred_fields",
+        "prepare_database_save",
+        "save_base",
+        "serializable_value",
+        "unique_error_message",
+        "validate_constraints",
+        "validate_unique",
+    ]
+    if reverse:
+        attributes.append("arefresh_from_db")
+        attributes.append("full_clean")
+    else:
+        attributes.append("a_refresh_from_db")
+        attributes.append("full__clean")
+
+    django_path = Path(db.__file__).parent.parent
+
+    encoding = "utf8" if os.name == "nt" else None
+
+    def prune_file(file_path):
+        content = file_path.read_text(encoding=encoding)
+        original_content = content
+
+        for attr in attributes:
+            old_name = f"_{attr}" if reverse else attr
+            new_name = attr if reverse else f"_{attr}"
+            content = content.replace(old_name, new_name)
+
+        if not reverse:
+            content = content.replace("Field_DoesNotExist", "FieldDoesNotExist")
+            content = content.replace("Object_DoesNotExist", "ObjectDoesNotExist")
+
+        if content != original_content:
+            file_path.write_text(content, encoding=encoding)
+
+    for file_path in django_path.rglob("*.py"):
+        prune_file(file_path)
+
+    pylance_path = find_vscode_stubs_folder()
+    if pylance_path is not None:
+        for file_path in pylance_path.rglob("*.pyi"):
+            prune_file(file_path)