lamindb 1.3.1__py3-none-any.whl → 1.4.0__py3-none-any.whl

lamindb/models/record.py

@@ -58,11 +58,11 @@ from lamindb_setup._connect_instance import (
     update_db_using_local,
 )
 from lamindb_setup.core._docs import doc_args
-from lamindb_setup.core._hub_core import connect_instance_hub
+from lamindb_setup.core._hub_core import access_db, connect_instance_hub
 from lamindb_setup.core._settings_store import instance_settings_file
+from lamindb_setup.core.django import db_token_manager
 from lamindb_setup.core.upath import extract_suffix_from_path
 
-from lamindb.base import deprecated
 from lamindb.base.fields import (
     CharField,
     DateTimeField,
@@ -186,8 +186,7 @@ def update_attributes(record: Record, attributes: dict[str, str]):
         if (
             getattr(record, key) != value
             and value is not None
-            and key != "dtype"
-            and key != "_aux"
+            and key not in {"dtype", "otype", "_aux"}
         ):
             logger.warning(f"updated {key} from {getattr(record, key)} to {value}")
             setattr(record, key, value)
@@ -370,6 +369,8 @@ class Registry(ModelBase):
     Note: `Registry` inherits from Django's `ModelBase`.
     """
 
+    _available_fields: set[str] = None
+
     def __new__(cls, name, bases, attrs, **kwargs):
         new_class = super().__new__(cls, name, bases, attrs, **kwargs)
         return new_class
@@ -487,10 +488,12 @@ class Registry(ModelBase):
             - Guide: :doc:`docs:registries`
             - Django documentation: `Queries <https://docs.djangoproject.com/en/stable/topics/db/queries/>`__
 
-        Examples::
+        Examples:
+
+            ::
 
-            ulabel = ln.ULabel.get("FvtpPJLJ")
-            ulabel = ln.ULabel.get(name="my-label")
+                ulabel = ln.ULabel.get("FvtpPJLJ")
+                ulabel = ln.ULabel.get(name="my-label")
         """
         from .query_set import QuerySet
 
@@ -594,17 +597,18 @@ class Registry(ModelBase):
         """
         from .query_set import QuerySet
 
-        if instance is None:
+        # we're in the default instance
+        if instance is None or instance == "default":
             return QuerySet(model=cls, using=None)
-
         owner, name = get_owner_name_from_identifier(instance)
-        if f"{owner}/{name}" == setup_settings.instance.slug:
+        if [owner, name] == setup_settings.instance.slug.split("/"):
             return QuerySet(model=cls, using=None)
 
-        settings_file = instance_settings_file(name, owner)
-        cache_filepath = (
-            ln_setup.settings.cache_dir / f"instance--{owner}--{name}--uid.txt"
+        # move on to different instances
+        cache_using_filepath = (
+            setup_settings.cache_dir / f"instance--{owner}--{name}--uid.txt"
         )
+        settings_file = instance_settings_file(name, owner)
         if not settings_file.exists():
             result = connect_instance_hub(owner=owner, name=name)
             if isinstance(result, str):
@@ -612,23 +616,47 @@ class Registry(ModelBase):
                     f"Failed to load instance {instance}, please check your permissions!"
                 )
             iresult, _ = result
-            source_module = {
-                modules for modules in iresult["schema_str"].split(",") if modules != ""
-            }  # type: ignore
-            target_module = ln_setup.settings.instance.modules
-            if not source_module.issubset(target_module):
-                missing_members = source_module - target_module
-                logger.warning(
-                    f"source modules has additional modules: {missing_members}\nconsider mounting these registry modules to transfer all metadata"
-                )
-            cache_filepath.write_text(f"{iresult['lnid']}\n{iresult['schema_str']}")  # type: ignore
-            settings_file = instance_settings_file(name, owner)
+            # do not use {} syntax below, it gives rise to a dict if the schema modules
+            # are empty and then triggers a TypeError in missing_members = source_modules - target_modules
+            source_modules = set(  # noqa
+                [mod for mod in iresult["schema_str"].split(",") if mod != ""]
+            )
+            # this just retrives the full connection string from iresult
             db = update_db_using_local(iresult, settings_file)
+            cache_using_filepath.write_text(
+                f"{iresult['lnid']}\n{iresult['schema_str']}"
+            )
+            # need to set the token if it is a fine_grained_access and the user is jwt (not public)
+            is_fine_grained_access = (
+                iresult["fine_grained_access"] and iresult["db_permissions"] == "jwt"
+            )
+            # access_db can take both: the dict from connect_instance_hub and isettings
+            into_access_db = iresult
         else:
             isettings = load_instance_settings(settings_file)
+            source_modules = isettings.modules
             db = isettings.db
-            cache_filepath.write_text(f"{isettings.uid}\n{','.join(isettings.modules)}")  # type: ignore
+            cache_using_filepath.write_text(
+                f"{isettings.uid}\n{','.join(source_modules)}"
+            )
+            # need to set the token if it is a fine_grained_access and the user is jwt (not public)
+            is_fine_grained_access = (
+                isettings._fine_grained_access and isettings._db_permissions == "jwt"
+            )
+            # access_db can take both: the dict from connect_instance_hub and isettings
+            into_access_db = isettings
+
+        target_modules = setup_settings.instance.modules
+        if not (missing_members := source_modules - target_modules):
+            logger.warning(
+                f"source modules has additional modules: {missing_members}\n"
+                "consider mounting these registry modules to transfer all metadata"
+            )
+
         add_db_connection(db, instance)
+        if is_fine_grained_access:
+            db_token = access_db(into_access_db)
+            db_token_manager.set(db_token, instance)
         return QuerySet(model=cls, using=instance)
 
     def __get_module_name__(cls) -> str:
@@ -638,10 +666,6 @@ class Registry(ModelBase):
             module_name = "core"
         return module_name
 
-    @deprecated("__get_module_name__")
-    def __get_schema_name__(cls) -> str:
-        return cls.__get_module_name__()
-
     def __get_name_with_module__(cls) -> str:
         module_name = cls.__get_module_name__()
         if module_name == "core":
@@ -650,9 +674,19 @@ class Registry(ModelBase):
             module_prefix = f"{module_name}."
         return f"{module_prefix}{cls.__name__}"
 
-    @deprecated("__get_name_with_module__")
-    def __get_name_with_schema__(cls) -> str:
-        return cls.__get_name_with_module__()
+    def __get_available_fields__(cls) -> set[str]:
+        if cls._available_fields is None:
+            cls._available_fields = {
+                f.name
+                for f in cls._meta.get_fields()
+                if not f.name.startswith("_")
+                and not f.name.startswith("links_")
+                and not f.name.endswith("_id")
+            }
+            if cls.__name__ == "Artifact":
+                cls._available_fields.add("visibility")
+                cls._available_fields.add("transform")
+        return cls._available_fields
 
 
 class BasicRecord(models.Model, metaclass=Registry):
@@ -668,75 +702,86 @@ class BasicRecord(models.Model, metaclass=Registry):
 
     def __init__(self, *args, **kwargs):
         skip_validation = kwargs.pop("_skip_validation", False)
-        if not args and skip_validation:
-            super().__init__(**kwargs)
-        elif not args and not skip_validation:
-            from ..core._settings import settings
-            from .can_curate import CanCurate
-            from .collection import Collection
-            from .schema import Schema
-            from .transform import Transform
-
-            validate_fields(self, kwargs)
-
-            # do not search for names if an id is passed; this is important
-            # e.g. when synching ids from the notebook store to lamindb
-            has_consciously_provided_uid = False
-            if "_has_consciously_provided_uid" in kwargs:
-                has_consciously_provided_uid = kwargs.pop(
-                    "_has_consciously_provided_uid"
-                )
+        if not args:
             if (
-                isinstance(self, (CanCurate, Collection, Transform))
-                and settings.creation.search_names
-                and not has_consciously_provided_uid
+                issubclass(self.__class__, Record)
+                and not self.__class__.__name__ == "Storage"
+                # do not save bionty entities in restricted spaces by default
+                and self.__class__.__module__ != "bionty.models"
             ):
-                name_field = getattr(self, "_name_field", "name")
-                exact_match = suggest_records_with_similar_names(
-                    self, name_field, kwargs
-                )
-                if exact_match is not None:
-                    if "version" in kwargs:
-                        if kwargs["version"] is not None:
-                            version_comment = " and version"
-                            existing_record = self.__class__.filter(
-                                **{
-                                    name_field: kwargs[name_field],
-                                    "version": kwargs["version"],
-                                }
-                            ).one_or_none()
+                from lamindb import context as run_context
+
+                if run_context.space is not None:
+                    kwargs["space"] = run_context.space
+            if skip_validation:
+                super().__init__(**kwargs)
+            else:
+                from ..core._settings import settings
+                from .can_curate import CanCurate
+                from .collection import Collection
+                from .schema import Schema
+                from .transform import Transform
+
+                validate_fields(self, kwargs)
+
+                # do not search for names if an id is passed; this is important
+                # e.g. when synching ids from the notebook store to lamindb
+                has_consciously_provided_uid = False
+                if "_has_consciously_provided_uid" in kwargs:
+                    has_consciously_provided_uid = kwargs.pop(
+                        "_has_consciously_provided_uid"
+                    )
+                if (
+                    isinstance(self, (CanCurate, Collection, Transform))
+                    and settings.creation.search_names
+                    and not has_consciously_provided_uid
+                ):
+                    name_field = getattr(self, "_name_field", "name")
+                    exact_match = suggest_records_with_similar_names(
+                        self, name_field, kwargs
+                    )
+                    if exact_match is not None:
+                        if "version" in kwargs:
+                            if kwargs["version"] is not None:
+                                version_comment = " and version"
+                                existing_record = self.__class__.filter(
+                                    **{
+                                        name_field: kwargs[name_field],
+                                        "version": kwargs["version"],
+                                    }
+                                ).one_or_none()
+                            else:
+                                # for a versioned record, an exact name match is not a criterion
+                                # for retrieving a record in case `version` isn't passed -
+                                # we'd always pull out many records with exactly the same name
+                                existing_record = None
                         else:
-                            # for a versioned record, an exact name match is not a criterion
-                            # for retrieving a record in case `version` isn't passed -
-                            # we'd always pull out many records with exactly the same name
-                            existing_record = None
-                    else:
-                        version_comment = ""
-                        existing_record = exact_match
-                    if existing_record is not None:
-                        logger.important(
-                            f"returning existing {self.__class__.__name__} record with same"
-                            f" {name_field}{version_comment}: '{kwargs[name_field]}'"
-                        )
-                        if isinstance(self, Schema):
-                            if existing_record.hash != kwargs["hash"]:
-                                raise ValueError(
-                                    f"Schema name is already in use by schema with uid '{existing_record.uid}', please choose a different name."
-                                )
-                        init_self_from_db(self, existing_record)
-                        update_attributes(self, kwargs)
-                        return None
-            super().__init__(**kwargs)
-            if isinstance(self, ValidateFields):
-                # this will trigger validation against django validators
-                try:
-                    if hasattr(self, "clean_fields"):
-                        self.clean_fields()
-                    else:
-                        self._Model__clean_fields()
-                except DjangoValidationError as e:
-                    message = _format_django_validation_error(self, e)
-                    raise FieldValidationError(message) from e
+                            version_comment = ""
+                            existing_record = exact_match
+                        if existing_record is not None:
+                            logger.important(
+                                f"returning existing {self.__class__.__name__} record with same"
+                                f" {name_field}{version_comment}: '{kwargs[name_field]}'"
+                            )
+                            if isinstance(self, Schema):
+                                if existing_record.hash != kwargs["hash"]:
+                                    logger.warning(
+                                        f"You're updating schema {existing_record.uid}, which might already have been used to validate datasets. Be careful."
+                                    )
+                            init_self_from_db(self, existing_record)
+                            update_attributes(self, kwargs)
+                            return None
+                super().__init__(**kwargs)
+                if isinstance(self, ValidateFields):
+                    # this will trigger validation against django validators
+                    try:
+                        if hasattr(self, "clean_fields"):
+                            self.clean_fields()
+                        else:
+                            self._Model__clean_fields()
+                    except DjangoValidationError as e:
+                        message = _format_django_validation_error(self, e)
+                        raise FieldValidationError(message) from e
         elif len(args) != len(self._meta.concrete_fields):
             raise FieldValidationError(
                 f"Use keyword arguments instead of positional arguments, e.g.: {self.__class__.__name__}(name='...')."
@@ -838,16 +883,20 @@ class BasicRecord(models.Model, metaclass=Registry):
                 if k != "run":
                     logger.important(f"{k} records: {', '.join(v)}")
 
-        if self.__class__.__name__ in {
-            "Artifact",
-            "Transform",
-            "Run",
-            "ULabel",
-            "Feature",
-            "Schema",
-            "Collection",
-            "Reference",
-        }:
+        if (
+            self.__class__.__name__
+            in {
+                "Artifact",
+                "Transform",
+                "Run",
+                "ULabel",
+                "Feature",
+                "Schema",
+                "Collection",
+                "Reference",
+            }
+            and self._branch_code >= 1
+        ):
             import lamindb as ln
 
             if ln.context.project is not None:
@@ -1313,10 +1362,12 @@ def get_transfer_run(record) -> Run:
 
     slug = record._state.db
     owner, name = get_owner_name_from_identifier(slug)
-    cache_filepath = ln_setup.settings.cache_dir / f"instance--{owner}--{name}--uid.txt"
-    if not cache_filepath.exists():
+    cache_using_filepath = (
+        ln_setup.settings.cache_dir / f"instance--{owner}--{name}--uid.txt"
+    )
+    if not cache_using_filepath.exists():
         raise SystemExit("Need to call .using() before")
-    instance_uid = cache_filepath.read_text().split("\n")[0]
+    instance_uid = cache_using_filepath.read_text().split("\n")[0]
     key = f"transfers/{instance_uid}"
     uid = instance_uid + "0000"
     transform = Transform.filter(uid=uid).one_or_none()
@@ -1399,11 +1450,13 @@ def transfer_to_default_db(
 def track_current_key_and_name_values(record: Record):
     from lamindb.models import Artifact
 
+    # below, we're using __dict__ to avoid triggering the refresh from the database
+    # which can lead to a recursion
     if isinstance(record, Artifact):
-        record._old_key = record.key
-        record._old_suffix = record.suffix
+        record._old_key = record.__dict__.get("key")
+        record._old_suffix = record.__dict__.get("suffix")
     elif hasattr(record, "_name_field"):
-        record._old_name = getattr(record, record._name_field)
+        record._old_name = record.__dict__.get(record._name_field)
 
 
 def check_name_change(record: Record):
@@ -1437,9 +1490,6 @@ def check_name_change(record: Record):
                     label_ref_is_name=True, **{f"{registry.lower()}_id": record.pk}
                 )
                 .exclude(feature_id=None)  # must have a feature
-                .exclude(
-                    feature_ref_is_name=None
-                )  # must be linked via Curator and therefore part of a schema
                 .distinct()
             )
             artifact_ids = linked_records.list("artifact__uid")

lamindb/models/run.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any, overload
 
+import numpy as np
 from django.db import models
 from django.db.models import (
     CASCADE,
@@ -19,7 +20,7 @@ from lamindb.base.fields import (
     ForeignKey,
 )
 from lamindb.base.users import current_user_id
-from lamindb.errors import ValidationError
+from lamindb.errors import InvalidArgument, ValidationError
 
 from ..base.ids import base62_20
 from .can_curate import CanCurate
@@ -33,6 +34,7 @@ if TYPE_CHECKING:
     from .artifact import Artifact
     from .collection import Collection
     from .project import Project
+    from .query_set import QuerySet
     from .schema import Schema
     from .transform import Transform
     from .ulabel import ULabel
@@ -538,6 +540,56 @@ class Run(Record):
         delete_run_artifacts(self)
         super().delete()
 
+    @classmethod
+    def filter(
+        cls,
+        *queries,
+        **expressions,
+    ) -> QuerySet:
+        """Query a set of artifacts.
+
+        Args:
+            *queries: `Q` expressions.
+            **expressions: Params, fields, and values passed via the Django query syntax.
+
+        See Also:
+            - Guide: :doc:`docs:registries`
+
+        Examples:
+
+            Query by fields::
+
+                ln.Run.filter(key="my_datasets/my_file.parquet")
+
+            Query by params::
+
+                ln.Run.filter(hyperparam_x=100)
+        """
+        from ._feature_manager import filter_base
+        from .query_set import QuerySet
+
+        if expressions:
+            keys_normalized = [key.split("__")[0] for key in expressions]
+            field_or_feature_or_param = keys_normalized[0].split("__")[0]
+            if field_or_feature_or_param in Run.__get_available_fields__():
+                return QuerySet(model=cls).filter(*queries, **expressions)
+            elif all(
+                params_validated := Param.validate(
+                    keys_normalized, field="name", mute=True
+                )
+            ):
+                return filter_base(ParamManagerRun, **expressions)
+            else:
+                params = ", ".join(sorted(np.array(keys_normalized)[~params_validated]))
+                message = f"param names: {params}"
+                fields = ", ".join(sorted(cls.__get_available_fields__()))
+                raise InvalidArgument(
+                    f"You can query either by available fields: {fields}\n"
+                    f"Or fix invalid {message}"
+                )
+        else:
+            return QuerySet(model=cls).filter(*queries, **expressions)
+
 
 def delete_run_artifacts(run: Run) -> None:
     environment = None
@@ -555,7 +607,9 @@ def delete_run_artifacts(run: Run) -> None:
         if environment._environment_of.count() == 0:
             environment.delete(permanent=True)
     if report is not None:
-        report.delete(permanent=True)
+        # only delete if there are no other runs attached to this environment
+        if report._report_of.count() == 0:
+            report.delete(permanent=True)
 
 
 class RunParamValue(BasicRecord, LinkORM):

lamindb/models/save.py

@@ -192,9 +192,7 @@ def copy_or_move_to_cache(
     # non-local storage_path further
     if local_path != cache_path:
         if cache_path.exists():
-            logger.warning(
-                f"The cache path {cache_path.as_posix()} already exists, replacing it."
-            )
+            logger.warning(f"replacing the existing cache path {cache_path.as_posix()}")
             if cache_path.is_dir():
                 shutil.rmtree(cache_path)
             else: