lamindb 1.10.2__py3-none-any.whl → 1.11.0__py3-none-any.whl

lamindb/migrations/0120_add_record_fk_constraint.py

@@ -0,0 +1,64 @@
+# Generated by Django 5.2 on 2025-08-07 18:52
+
+from django.db import migrations
+
+CREATE_FUNCTION_SQL = """
+CREATE OR REPLACE FUNCTION is_valid_record_type(record_type_id INTEGER, record_is_type BOOLEAN)
+RETURNS BOOLEAN AS $$
+BEGIN
+    -- Record with no type is valid
+    IF record_type_id IS NULL THEN
+        RETURN TRUE;
+    END IF;
+
+    -- If current record is a type, it can only reference schema-less types
+    IF record_is_type THEN
+        RETURN EXISTS (
+            SELECT 1 FROM lamindb_record r
+            WHERE r.id = record_type_id AND r.is_type AND r.schema_id IS NULL
+        );
+    END IF;
+
+    -- Regular records can reference any type
+    RETURN EXISTS (
+        SELECT 1 FROM lamindb_record r
+        WHERE r.id = record_type_id AND r.is_type
+    );
+END;
+$$ LANGUAGE plpgsql;
+"""
+
+ADD_CONSTRAINT_SQL = """
+ALTER TABLE lamindb_record
+ADD CONSTRAINT record_type_is_valid_fk
+CHECK (is_valid_record_type(type_id, is_type));
+"""
+
+DROP_CONSTRAINT_SQL = (
+    "ALTER TABLE lamindb_record DROP CONSTRAINT IF EXISTS record_type_is_valid_fk;"
+)
+DROP_FUNCTION_SQL = "DROP FUNCTION IF EXISTS is_valid_record_type(INTEGER, BOOLEAN);"
+
+
+def apply_postgres_constraint(apps, schema_editor):
+    if schema_editor.connection.vendor == "postgresql":
+        schema_editor.execute(CREATE_FUNCTION_SQL)
+        schema_editor.execute(ADD_CONSTRAINT_SQL)
+
+
+def revert_postgres_constraint(apps, schema_editor):
+    if schema_editor.connection.vendor == "postgresql":
+        schema_editor.execute(DROP_CONSTRAINT_SQL)
+        schema_editor.execute(DROP_FUNCTION_SQL)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("lamindb", "0119_squashed"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            apply_postgres_constraint, reverse_code=revert_postgres_constraint
+        ),
+    ]

lamindb/migrations/0121_recorduser.py

@@ -0,0 +1,60 @@
+# Generated by Django 5.2 on 2025-09-05 12:25
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+import lamindb.base.fields
+import lamindb.models.sqlrecord
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("lamindb", "0120_add_record_fk_constraint"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="RecordUser",
+            fields=[
+                ("id", models.BigAutoField(primary_key=True, serialize=False)),
+                (
+                    "feature",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="links_recorduser",
+                        to="lamindb.feature",
+                    ),
+                ),
+                (
+                    "record",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="values_user",
+                        to="lamindb.record",
+                    ),
+                ),
+                (
+                    "value",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="links_record",
+                        to="lamindb.user",
+                    ),
+                ),
+            ],
+            options={
+                "unique_together": {("record", "feature", "value")},
+            },
+            bases=(models.Model, lamindb.models.sqlrecord.IsLink),
+        ),
+        migrations.AddField(
+            model_name="record",
+            name="linked_users",
+            field=models.ManyToManyField(
+                related_name="records", through="lamindb.RecordUser", to="lamindb.user"
+            ),
+        ),
+    ]

lamindb/models/__init__.py

@@ -9,6 +9,7 @@
    BasicQuerySet
    QuerySet
    ArtifactSet
+   LazyArtifact
    QueryManager
    SQLRecordList
    FeatureManager
@@ -49,7 +50,7 @@ from .schema import Schema
 from .ulabel import ULabel
 
 # should come last as it needs everything else
-from .artifact import Artifact
+from .artifact import Artifact, LazyArtifact
 from ._feature_manager import FeatureManager
 from ._label_manager import LabelManager
 from .collection import Collection, CollectionArtifact
@@ -78,6 +79,7 @@ from .project import (
     PersonProject,
     RecordPerson,
     RecordReference,
+    ProjectRecord,
 )
 from .run import RunFeatureValue
 from .schema import (
@@ -94,6 +96,7 @@ from .record import (
     RecordRecord,
     RecordULabel,
     RecordRun,
+    RecordUser,
     RecordArtifact,
     ArtifactRecord,
 )

lamindb/models/_describe.py

@@ -8,8 +8,6 @@ from lamin_utils import logger
 from rich.text import Text
 from rich.tree import Tree
 
-from ..core._context import is_run_from_ipython
-
 if TYPE_CHECKING:
     from lamindb.models import Artifact, Collection, Run
 
@@ -41,6 +39,8 @@ def format_rich_tree(
 ) -> str | None:
     from rich.console import Console
 
+    from ..core._context import is_run_from_ipython
+
     # If tree has no children, return fallback
     if not tree.children:
         return fallback

lamindb/models/_feature_manager.py

@@ -23,7 +23,7 @@ from rich.table import Column, Table
 from rich.text import Text
 
 from lamindb.core.storage import LocalPathClasses
-from lamindb.errors import DoesNotExist, ValidationError
+from lamindb.errors import DoesNotExist, InvalidArgument, ValidationError
 from lamindb.models._from_values import _format_values
 from lamindb.models.feature import (
     serialize_pandas_dtype,
@@ -33,7 +33,6 @@ from lamindb.models.save import save
 from lamindb.models.schema import DICT_KEYS_TYPE, Schema
 from lamindb.models.sqlrecord import (
     REGISTRY_UNIQUE_FIELD,
-    Registry,
     get_name_field,
     transfer_fk_to_default_db_bulk,
     transfer_to_default_db,
@@ -65,7 +64,7 @@ if TYPE_CHECKING:
         Collection,
         IsLink,
     )
-    from lamindb.models.query_set import QuerySet
+    from lamindb.models.query_set import BasicQuerySet
 
     from .run import Run
 
@@ -100,7 +99,7 @@ def get_schema_by_slot_(host: Artifact) -> dict[str, Schema]:
 
 def get_label_links(
     host: Artifact | Collection, registry: str, feature: Feature
-) -> QuerySet:
+) -> BasicQuerySet:
     kwargs = {"artifact_id": host.id, "feature_id": feature.id}
     link_records = (
         getattr(host, host.features._accessor_by_registry[registry])  # type: ignore
@@ -110,7 +109,7 @@ def get_label_links(
     return link_records
 
 
-def get_schema_links(host: Artifact | Collection) -> QuerySet:
+def get_schema_links(host: Artifact | Collection) -> BasicQuerySet:
     kwargs = {"artifact_id": host.id}
     links_schema = host.feature_sets.through.objects.filter(**kwargs)
     return links_schema
@@ -496,21 +495,11 @@ def describe_features(
     return tree
 
 
-def is_valid_datetime_str(date_string: str) -> bool | str:
-    try:
-        dt = datetime.fromisoformat(date_string)
-        return dt.isoformat()
-    except ValueError:
-        return False
-
-
-def is_iterable_of_sqlrecord(value: Any):
-    return isinstance(value, Iterable) and isinstance(next(iter(value)), SQLRecord)
-
-
 def infer_feature_type_convert_json(
-    key: str, value: Any, mute: bool = False, str_as_ulabel: bool = True
+    key: str, value: Any, mute: bool = False
 ) -> tuple[str, Any, str]:
+    from lamindb.base.dtypes import is_valid_datetime_str
+
     message = ""
     if isinstance(value, bool):
         return "bool", value, message
@@ -572,21 +561,29 @@ def infer_feature_type_convert_json(
 
 
 def filter_base(
-    registry: Registry, _skip_validation: bool = True, **expression
-) -> QuerySet:
-    from .artifact import Artifact
+    queryset: BasicQuerySet,
+    _skip_validation: bool = True,
+    **expression,
+) -> BasicQuerySet:
+    from lamindb.models import Artifact, BasicQuerySet, QuerySet
+
+    # not QuerySet but only BasicQuerySet
+    assert isinstance(queryset, BasicQuerySet) and not isinstance(queryset, QuerySet)  # noqa: S101
+
+    registry = queryset.model
+    db = queryset.db
 
     model = Feature
     value_model = FeatureValue
     keys_normalized = [key.split("__")[0] for key in expression]
     if not _skip_validation:
-        validated = model.validate(keys_normalized, field="name", mute=True)
+        validated = model.using(db).validate(keys_normalized, field="name", mute=True)
         if sum(validated) != len(keys_normalized):
             raise ValidationError(
                 f"Some keys in the filter expression are not registered as features: {np.array(keys_normalized)[~validated]}"
             )
     new_expression = {}
-    features = model.filter(name__in=keys_normalized).all().distinct()
+    features = model.using(db).filter(name__in=keys_normalized).all().distinct()
     feature_param = "feature"
     for key, value in expression.items():
         split_key = key.split("__")
@@ -604,7 +601,7 @@ def filter_base(
                     from .artifact import ArtifactFeatureValue
 
                     if value:  # True
-                        return Artifact.objects.exclude(
+                        return queryset.exclude(
                             id__in=Subquery(
                                 ArtifactFeatureValue.objects.filter(
                                     featurevalue__feature=feature
@@ -612,7 +609,7 @@ def filter_base(
                             )
                         )
                     else:
-                        return Artifact.objects.exclude(
+                        return queryset.exclude(
                             id__in=Subquery(
                                 ArtifactFeatureValue.objects.filter(
                                     featurevalue__feature=feature
@@ -636,9 +633,9 @@ def filter_base(
                         f"links_{result['registry'].__name__.lower()}__feature": feature
                     }
                     if value:  # True
-                        return Artifact.objects.exclude(**kwargs)
+                        return queryset.exclude(**kwargs)
                     else:
-                        return Artifact.objects.filter(**kwargs)
+                        return queryset.filter(**kwargs)
             else:
                 # because SQL is sensitive to whether querying with __in or not
                 # and might return multiple equivalent records for the latter
@@ -652,7 +649,7 @@ def filter_base(
                     # we need the comparator here because users might query like so
                     # ln.Artifact.filter(experiment__contains="Experi")
                     expression = {f"{field_name}{comparator}": value}
-                    labels = result["registry"].filter(**expression).all()
+                    labels = result["registry"].using(db).filter(**expression).all()
                     if len(labels) == 0:
                         raise DoesNotExist(
                             f"Did not find a {label_registry.__name__} matching `{field_name}{comparator}={value}`"
@@ -678,9 +675,62 @@ def filter_base(
             # find artifacts that are annotated by all of them at the same
             # time; hence, we don't want the __in construct that we use to match strings
             # https://laminlabs.slack.com/archives/C04FPE8V01W/p1688328084810609
-    if not (new_expression):
+    if not new_expression:
         raise NotImplementedError
-    return registry.objects.filter(**new_expression)
+    return queryset.filter(**new_expression)
+
+
+def filter_with_features(
+    queryset: BasicQuerySet, *queries, **expressions
+) -> BasicQuerySet:
+    from lamindb.models import Artifact, BasicQuerySet, QuerySet
+
+    if isinstance(queryset, QuerySet):
+        # need to avoid infinite recursion because
+        # filter_with_features is called in queryset.filter otherwise
+        filter_kwargs = {"_skip_filter_with_features": True}
+    else:
+        filter_kwargs = {}
+
+    registry = queryset.model
+
+    if registry is Artifact and not any(e.startswith("kind") for e in expressions):
+        exclude_kwargs = {"kind": "__lamindb_run__"}
+    else:
+        exclude_kwargs = {}
+
+    if expressions:
+        keys_normalized = [key.split("__")[0] for key in expressions]
+        field_or_feature_or_param = keys_normalized[0].split("__")[0]
+        if field_or_feature_or_param in registry.__get_available_fields__():
+            qs = queryset.filter(*queries, **expressions, **filter_kwargs)
+        elif all(
+            features_validated := Feature.objects.using(queryset.db).validate(
+                keys_normalized, field="name", mute=True
+            )
+        ):
+            # filter_base requires qs to be BasicQuerySet
+            qs = filter_base(
+                queryset._to_class(BasicQuerySet, copy=True),
+                _skip_validation=True,
+                **expressions,
+            )._to_class(type(queryset), copy=False)
+            qs = qs.filter(*queries, **filter_kwargs)
+        else:
+            features = ", ".join(sorted(np.array(keys_normalized)[~features_validated]))
+            message = f"feature names: {features}"
+            avail_fields = registry.__get_available_fields__()
+            if "_branch_code" in avail_fields:
+                avail_fields.remove("_branch_code")  # backward compat
+            fields = ", ".join(sorted(avail_fields))
+            raise InvalidArgument(
+                f"You can query either by available fields: {fields}\n"
+                f"Or fix invalid {message}"
+            )
+    else:
+        qs = queryset.filter(*queries, **filter_kwargs)
+
+    return qs.exclude(**exclude_kwargs) if exclude_kwargs else qs
 
 
 # for deprecated functionality
@@ -719,15 +769,15 @@ def parse_staged_feature_sets_from_anndata(
             data_parse = backed_access(filepath, using_key=using_key)
         else:
             data_parse = ad.read_h5ad(filepath, backed="r")
-        type = "float"
+        dtype = "float"
     else:
-        type = "float" if adata.X is None else serialize_pandas_dtype(adata.X.dtype)
+        dtype = "float" if adata.X is None else serialize_pandas_dtype(adata.X.dtype)
     feature_sets = {}
     if var_field is not None:
         schema_var = Schema.from_values(
             data_parse.var.index,
             var_field,
-            type=type,
+            dtype=dtype,
             mute=mute,
             organism=organism,
             raise_validation_error=False,
@@ -735,7 +785,7 @@ def parse_staged_feature_sets_from_anndata(
         if schema_var is not None:
             feature_sets["var"] = schema_var
     if obs_field is not None and len(data_parse.obs.columns) > 0:
-        schema_obs = Schema.from_df(
+        schema_obs = Schema.from_dataframe(
             df=data_parse.obs,
             field=obs_field,
             mute=mute,
@@ -775,7 +825,7 @@ class FeatureManager:
         return describe_features(self._host, to_dict=True)  # type: ignore
 
     @deprecated("slots[slot].members")
-    def __getitem__(self, slot) -> QuerySet:
+    def __getitem__(self, slot) -> BasicQuerySet:
         if slot not in self.slots:
             raise ValueError(
                 f"No linked feature set for slot: {slot}\nDid you get validation"
@@ -851,16 +901,17 @@ class FeatureManager:
         self,
         values: dict[str, str | int | float | bool],
         feature_field: FieldAttr = Feature.name,
-        str_as_ulabel: bool = True,
+        schema: Schema = None,
     ) -> None:
         """Curate artifact with features & values.
 
         Args:
             values: A dictionary of keys (features) & values (labels, numbers, booleans).
-            feature_field: The field of a reference registry to map keys of the
-                dictionary.
-            str_as_ulabel: Whether to interpret string values as ulabels.
+            feature_field: The field of a reference registry to map keys of the dictionary.
+            schema: Schema to validate against.
         """
+        from lamindb.base.dtypes import is_iterable_of_sqlrecord
+
         from .._tracked import get_current_tracked_run
 
         # rename to distinguish from the values inside the dict
@@ -870,39 +921,48 @@ class FeatureManager:
             keys = list(keys)  # type: ignore
         # deal with other cases later
         assert all(isinstance(key, str) for key in keys)  # noqa: S101
+
         registry = feature_field.field.model
         value_model = FeatureValue
         model_name = "Feature"
-        records = registry.from_values(keys, field=feature_field, mute=True)
-        if len(records) != len(keys):
-            not_validated_keys = [
-                key for key in keys if key not in records.list("name")
-            ]
-            not_validated_keys_dtype_message = [
-                (key, infer_feature_type_convert_json(key, dictionary[key]))
-                for key in not_validated_keys
-            ]
-            run = get_current_tracked_run()
-            if run is not None:
-                name = f"{run.transform.type}[{run.transform.key}]"
-                type_hint = f"""  {model_name.lower()}_type = ln.{model_name}(name='{name}', is_type=True).save()"""
-                elements = [type_hint]
-                type_kwarg = f", type={model_name.lower()}_type"
-            else:
-                elements = []
-                type_kwarg = ""
-            elements += [
-                f"  ln.{model_name}(name='{key}', dtype='{dtype}'{type_kwarg}).save(){message}"
-                for key, (dtype, _, message) in not_validated_keys_dtype_message
-            ]
-            hint = "\n".join(elements)
-            msg = (
-                f"These keys could not be validated: {not_validated_keys}\n"
-                f"Here is how to create a {model_name.lower()}:\n\n{hint}"
-            )
-            raise ValidationError(msg)
 
-        # figure out which of the values go where
+        if schema is not None:
+            from lamindb.curators import DataFrameCurator
+
+            temp_df = pd.DataFrame([values])
+            curator = DataFrameCurator(temp_df, schema)
+            curator.validate()
+            records = schema.members.filter(name__in=keys)
+        else:
+            records = registry.from_values(keys, field=feature_field, mute=True)
+            if len(records) != len(keys):
+                not_validated_keys = [
+                    key for key in keys if key not in records.to_list("name")
+                ]
+                not_validated_keys_dtype_message = [
+                    (key, infer_feature_type_convert_json(key, dictionary[key]))
+                    for key in not_validated_keys
+                ]
+                run = get_current_tracked_run()
+                if run is not None:
+                    name = f"{run.transform.type}[{run.transform.key}]"
+                    type_hint = f"""  {model_name.lower()}_type = ln.{model_name}(name='{name}', is_type=True).save()"""
+                    elements = [type_hint]
+                    type_kwarg = f", type={model_name.lower()}_type"
+                else:
+                    elements = []
+                    type_kwarg = ""
+                elements += [
+                    f"  ln.{model_name}(name='{key}', dtype='{dtype}'{type_kwarg}).save(){message}"
+                    for key, (dtype, _, message) in not_validated_keys_dtype_message
+                ]
+                hint = "\n".join(elements)
+                msg = (
+                    f"These keys could not be validated: {not_validated_keys}\n"
+                    f"Here is how to create a {model_name.lower()}:\n\n{hint}"
+                )
+                raise ValidationError(msg)
+
         features_labels = defaultdict(list)
         _feature_values = []
         not_validated_values: dict[str, list[str]] = defaultdict(list)
@@ -912,7 +972,6 @@ class FeatureManager:
                 feature.name,
                 value,
                 mute=True,
-                str_as_ulabel=str_as_ulabel,
             )
             if feature.dtype == "num":
                 if inferred_type not in {"int", "float"}:
@@ -994,6 +1053,7 @@ class FeatureManager:
                 f"Here is how to create records for them:\n\n{hint}"
             )
             raise ValidationError(msg)
+
         if features_labels:
             self._add_label_feature_links(features_labels)
         if _feature_values:
@@ -1039,7 +1099,7 @@ class FeatureManager:
         feature: str | Feature,
         *,
         value: Any | None = None,
-    ):
+    ) -> None:
         """Remove value annotations for a given feature.
 
         Args:
@@ -1262,7 +1322,7 @@ class FeatureManager:
         """Add feature set corresponding to column names of DataFrame."""
         assert self._host.otype == "DataFrame"  # noqa: S101
         df = self._host.load(is_run_input=False)
-        schema = Schema.from_df(
+        schema = Schema.from_dataframe(
             df=df,
             field=field,
             mute=mute,

lamindb/models/_from_values.py

@@ -121,7 +121,7 @@ def get_existing_records(
     #     ]
     # )
     # order by causes a factor 10 in runtime
-    # records = query_set.order_by(preserved).list()
+    # records = query_set.order_by(preserved).to_list()
 
     # log validated terms
     is_validated = model.validate(
@@ -165,7 +165,7 @@ def get_existing_records(
     query = {f"{field.field.name}__in": iterable_idx.values}  # type: ignore
     if organism is not None:
         query["organism"] = organism
-    records = model.filter(**query).list()
+    records = model.filter(**query).to_list()
 
     if len(validated) == len(iterable_idx):
         return records, pd.Index([]), msg

lamindb/models/_is_versioned.py

@@ -108,12 +108,12 @@ def bump_version(
 ) -> str:
     """Bumps the version number by major or minor depending on the bump_type flag.
 
-    Parameters:
-    version (str): The current version in "MAJOR" or "MAJOR.MINOR" format.
-    bump_type (str): The type of version bump, either 'major' or 'minor'.
+    Args:
+        version: The current version in "MAJOR" or "MAJOR.MINOR" format.
+        bump_type: The type of version bump, either 'major' or 'minor'.
 
     Returns:
-    str: The new version string.
+        The new version string.
     """
     try:
         # Split the version into major and minor parts if possible

lamindb/models/_label_manager.py

@@ -268,7 +268,7 @@ class LabelManager:
                     for link in links:
                         if link.feature is not None:
                             features.add(link.feature)
-                            key = link.feature.name
+                            key = link.feature.uid
                         else:
                             key = None
                         keys.append(key)
@@ -299,9 +299,9 @@ class LabelManager:
                     )
                 save(new_features)  # type: ignore
             if hasattr(self._host, related_name):
-                for feature_name, feature_labels in labels_by_features.items():
-                    if feature_name is not None:
-                        feature_id = Feature.get(name=feature_name).id
+                for feature_uid, feature_labels in labels_by_features.items():
+                    if feature_uid is not None:
+                        feature_id = Feature.get(feature_uid).id
                     else:
                         feature_id = None
                     getattr(self._host, related_name).add(