lamindb 1.11.3__py3-none-any.whl → 1.12.1__py3-none-any.whl

lamindb/models/can_curate.py

@@ -66,7 +66,7 @@ def _inspect(
     values = _concat_lists(values)
 
     field_str = get_name_field(cls, field=field)
-    queryset = cls.all() if isinstance(cls, (QuerySet, Manager)) else cls.objects.all()
+    queryset = cls.all() if isinstance(cls, (QuerySet, Manager)) else cls.filter().all()
     registry = queryset.model
     model_name = registry._meta.model.__name__
     if isinstance(source, SQLRecord):
@@ -170,7 +170,7 @@ def _validate(
 
     field_str = get_name_field(cls, field=field)
 
-    queryset = cls.all() if isinstance(cls, (QuerySet, Manager)) else cls.objects.all()
+    queryset = cls.all() if isinstance(cls, (QuerySet, Manager)) else cls.filter().all()
     registry = queryset.model
     if isinstance(source, SQLRecord):
         _check_if_record_in_db(source, queryset.db)
@@ -191,10 +191,8 @@ def _validate(
     )
     if field_values.empty:
         if not mute:
-            msg = (
-                f"Your {cls.__name__} registry is empty, consider populating it first!"
-            )
-            if hasattr(cls, "source_id"):
+            msg = f"Your {queryset.model.__name__} registry is empty, consider populating it first!"
+            if hasattr(queryset.model, "source_id"):
                 msg += "\n   → use `.import_source()` to import records from a source, e.g. a public ontology"
             logger.warning(msg)
         return np.array([False] * len(values))
@@ -238,7 +236,7 @@ def _standardize(
     return_field_str = get_name_field(
         cls, field=field if return_field is None else return_field
     )
-    queryset = cls.all() if isinstance(cls, (QuerySet, Manager)) else cls.objects.all()
+    queryset = cls.all() if isinstance(cls, (QuerySet, Manager)) else cls.filter().all()
     registry = queryset.model
     if isinstance(source, SQLRecord):
         _check_if_record_in_db(source, queryset.db)
@@ -358,7 +356,7 @@ def _add_or_remove_synonyms(
         from IPython.display import display
 
         syns_all = (
-            record.__class__.objects.exclude(synonyms="").exclude(synonyms=None).all()  # type: ignore
+            record.__class__.filter().exclude(synonyms="").exclude(synonyms=None).all()  # type: ignore
         )
         if len(syns_all) == 0:
             return
@@ -591,7 +589,7 @@ class CanCurate:
             A list of validated records. For bionty registries. Also returns knowledge-coupled records.
 
         Notes:
-            For more info, see tutorial: :doc:`docs:bio-registries`.
+            For more info, see tutorial: :doc:`docs:manage-ontologies`.
 
         Example::
 

lamindb/models/collection.py

@@ -49,6 +49,7 @@ if TYPE_CHECKING:
     from pyarrow.dataset import Dataset as PyArrowDataset
 
     from ..core.storage import UPath
+    from .block import CollectionBlock
     from .project import Project, Reference
     from .query_set import QuerySet
     from .transform import Transform
@@ -174,7 +175,7 @@ class Collection(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
     # below is the only case in which we use a TextField
     # for description; we do so because users had descriptions exceeding 255 chars
     # in their instances
-    description: str | None = TextField(null=True, db_index=True)
+    description: str | None = TextField(null=True)
     """A description or title."""
     hash: str | None = CharField(
         max_length=HASH_LENGTH, db_index=True, null=True, unique=True
@@ -224,6 +225,8 @@ class Collection(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
     """Linked projects."""
     references: Reference
     """Linked references."""
+    blocks: CollectionBlock
+    """Blocks that annotate this collection."""
 
     @overload
     def __init__(
@@ -437,7 +440,6 @@ class Collection(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
 
         dataframe = _open_dataframe(paths, engine=engine, **kwargs)
         # track only if successful
-        # is it really needed if tracking is done in self.ordered_artifacts.all()? - Sergei
         _track_run_input(self, is_run_input)
         return dataframe
 
@@ -538,24 +540,23 @@ class Collection(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
             dtype,
         )
         # track only if successful
-        # is it really needed if tracking is done in self.ordered_artifacts.all()? - Sergei
         _track_run_input(self, is_run_input)
         return ds
 
     def cache(self, is_run_input: bool | None = None) -> list[UPath]:
         """Download cloud artifacts in collection to local cache.
 
-        Follows synching logic: only caches outdated artifacts.
+        Follows syncing logic: only downloads outdated artifacts.
 
-        Returns paths to locally cached on-disk artifacts.
+        Returns ordered paths to locally cached on-disk artifacts via `.ordered_artifacts.all()`:
 
         Args:
             is_run_input: Whether to track this collection as run input.
         """
         path_list = []
         for artifact in self.ordered_artifacts.all():
-            path_list.append(artifact.cache())
-        # is it really needed if tracking is done in self.ordered_artifacts.all()? - Sergei
+            # do not want to track data lineage on the artifact level
+            path_list.append(artifact.cache(is_run_input=False))
         _track_run_input(self, is_run_input)
         return path_list
 
@@ -573,7 +574,6 @@ class Collection(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
         artifacts = self.ordered_artifacts.all()
         concat_object = _load_concat_artifacts(artifacts, join, **kwargs)
         # only call it here because there might be errors during load or concat
-        # is it really needed if tracking is done in self.ordered_artifacts.all()? - Sergei
         _track_run_input(self, is_run_input)
         return concat_object
 
@@ -640,7 +640,7 @@ class Collection(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
         you non-deterministic order.
 
         Using the property `.ordered_artifacts` allows to iterate through a set
-        that's ordered in the order of creation.
+        that's ordered by the order of the list that created the collection.
         """
         return self.artifacts.order_by("links_collection__id")
 

lamindb/models/feature.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import importlib
+import warnings
 from typing import TYPE_CHECKING, Any, get_args, overload
 
 import numpy as np
@@ -43,6 +44,8 @@ from .sqlrecord import BaseSQLRecord, Registry, SQLRecord, _get_record_kwargs
 if TYPE_CHECKING:
     from collections.abc import Iterable
 
+    from .block import FeatureBlock
+    from .projects import Project
     from .schema import Schema
 
 FEATURE_DTYPES = set(get_args(Dtype))
@@ -538,7 +541,7 @@ class Feature(SQLRecord, CanCurate, TracksRun, TracksUpdates):
         custom registries to manage high-level derived features like gene sets.
 
     See Also:
-        :meth:`~lamindb.Feature.from_df`
+        :meth:`~lamindb.Feature.from_dataframe`
             Create feature records from DataFrame.
         :attr:`~lamindb.Artifact.features`
             Feature manager of an artifact or collection.
@@ -600,6 +603,12 @@ class Feature(SQLRecord, CanCurate, TracksRun, TracksUpdates):
     class Meta(SQLRecord.Meta, TracksRun.Meta, TracksUpdates.Meta):
         abstract = False
         app_label = "lamindb"
+        constraints = [
+            models.CheckConstraint(
+                condition=models.Q(is_type=True) | models.Q(dtype__isnull=False),
+                name="dtype_not_null_when_is_type_false",
+            )
+        ]
 
     _name_field: str = "name"
     _aux_fields: dict[str, tuple[str, type]] = {
@@ -616,7 +625,8 @@ class Feature(SQLRecord, CanCurate, TracksRun, TracksUpdates):
     """Universal id, valid across DB instances."""
     name: str = CharField(max_length=150, db_index=True)
     """Name of feature."""
-    dtype: Dtype | None = CharField(db_index=True, null=True)
+    # dtype can be null if is_type is True
+    dtype: Dtype = CharField(db_index=True, null=True)
     """Data type (:class:`~lamindb.base.types.Dtype`)."""
     type: Feature | None = ForeignKey(
         "self", PROTECT, null=True, related_name="features"
@@ -631,7 +641,7 @@ class Feature(SQLRecord, CanCurate, TracksRun, TracksUpdates):
     """Distinguish types from instances of the type."""
     unit: str | None = CharField(max_length=30, db_index=True, null=True)
     """Unit of measure, ideally SI (`m`, `s`, `kg`, etc.) or 'normalized' etc. (optional)."""
-    description: str | None = CharField(db_index=True, null=True)
+    description: str | None = TextField(null=True)
     """A description."""
     array_rank: int = models.SmallIntegerField(default=0, db_index=True)
     """Rank of feature.
@@ -684,6 +694,10 @@ class Feature(SQLRecord, CanCurate, TracksRun, TracksUpdates):
     # backward fields
     values: FeatureValue
     """Values for this feature."""
+    projects: Project
+    """Annotating projects."""
+    blocks: FeatureBlock
+    """Blocks that annotate this feature."""
 
     @overload
     def __init__(
@@ -828,71 +842,57 @@ class Feature(SQLRecord, CanCurate, TracksRun, TracksUpdates):
         field: FieldAttr | None = None,
         *,
         str_as_cat: bool | None = None,
+        type: Feature | None = None,
         mute: bool = False,
     ) -> SQLRecordList:
         """Create Feature records for dictionary keys.
 
         Args:
             dictionary: Source dictionary to extract key information from
-            field: FieldAttr for Feature model validation, defaults to Feature.name
-            str_as_cat: Whether to interpret string values as categorical
+            field: FieldAttr for Feature model validation, defaults to `Feature.name`
+            str_as_cat: Deprecated. Will be removed in LaminDB 2.0.0.
+                Create features explicitly with dtype='cat' for categorical values.
+            type: Feature type of all created features
             mute: Whether to mute dtype inference and feature creation warnings
         """
         from lamindb.models._feature_manager import infer_feature_type_convert_json
 
+        if str_as_cat is not None:
+            warnings.warn(
+                "`str_as_cat` is deprecated and will be removed in LaminDB 2.0.0. "
+                "Create features explicitly with dtype='cat' for categorical values.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+
         field = Feature.name if field is None else field
         registry = field.field.model  # type: ignore
         if registry != Feature:
             raise ValueError("field must be a Feature FieldAttr!")
 
         dtypes = {}
-        ambiguous_keys = []
         for key, value in dictionary.items():
             dtype, _, message = infer_feature_type_convert_json(key, value, mute=mute)
-
             if dtype == "cat ? str":
                 if str_as_cat is None:
-                    ambiguous_keys.append(
-                        (key, "str or cat", message.strip("# ") if message else "")
-                    )
-                    continue
-                if str_as_cat:
-                    dtype = "cat"
-                else:
                     dtype = "str"
-
+                else:
+                    dtype = "cat" if str_as_cat else "str"
             elif dtype == "list[cat ? str]":
                 if str_as_cat is None:
-                    ambiguous_keys.append(
-                        (
-                            key,
-                            "list[str] or list[cat]",
-                            message.strip("# ") if message else "",
-                        )
-                    )
-                    continue
-                if str_as_cat:
-                    dtype = "list[cat]"
-                else:
                     dtype = "list[str]"
-
+                else:
+                    dtype = "list[cat]" if str_as_cat else "list[str]"
             dtypes[key] = dtype
 
-        if ambiguous_keys:
-            error_msg = "Ambiguous dtypes detected. Please pass `str_as_cat` parameter or create features explicitly:\n"
-            for key, options, msg in ambiguous_keys:
-                error_msg += f"  '{key}': {options}"
-                if msg:
-                    error_msg += f" ({msg})"
-                error_msg += "\n"
-            error_msg += "\nUse `str_as_cat=True` to treat strings as categorical, or `str_as_cat=False` for plain strings."
-            raise ValueError(error_msg)
-
         if mute:
             original_verbosity = logger._verbosity
             logger.set_verbosity(0)
         try:
-            features = [Feature(name=key, dtype=dtype) for key, dtype in dtypes.items()]  # type: ignore
+            features = [
+                Feature(name=key, dtype=dtype, type=type)
+                for key, dtype in dtypes.items()
+            ]  # type: ignore
             assert len(features) == len(dictionary)  # noqa: S101
             return SQLRecordList(features)
         finally:

lamindb/models/has_parents.py

@@ -42,13 +42,22 @@ def _query_relatives(
     attr: str,
     cls: type[HasParents],
 ) -> QuerySet:
-    relatives = cls.objects.none()  # type: ignore
-    if len(records) == 0:
+    from .query_set import get_default_branch_ids
+
+    branch_ids = get_default_branch_ids()
+
+    def query_relatives_on_branches(records, attr, cls) -> QuerySet:
+        relatives = cls.objects.none()  # type: ignore
+        if len(records) == 0:
+            return relatives
+        for record in records:
+            relatives = relatives.union(
+                getattr(record, attr).filter(branch_id__in=branch_ids)
+            )
+        relatives = relatives.union(query_relatives_on_branches(relatives, attr, cls))
         return relatives
-    for record in records:
-        relatives = relatives.union(getattr(record, attr).all())
-    relatives = relatives.union(_query_relatives(relatives, attr, cls))
-    return relatives
+
+    return query_relatives_on_branches(records, attr, cls)
 
 
 class HasParents:

lamindb/models/project.py

@@ -12,14 +12,13 @@ from lamindb.base.fields import (
     CharField,
     DateField,
     DateTimeField,
-    EmailField,
     ForeignKey,
     TextField,
     URLField,
 )
 from lamindb.base.users import current_user_id
 
-from ..base.ids import base62_8, base62_12
+from ..base.ids import base62_12
 from .artifact import Artifact
 from .can_curate import CanCurate
 from .collection import Collection
@@ -35,59 +34,7 @@ if TYPE_CHECKING:
     from datetime import date as DateType
     from datetime import datetime
 
-
-class Person(SQLRecord, CanCurate, TracksRun, TracksUpdates, ValidateFields):
-    """People such as authors of a study or collaborators in a project.
-
-    This registry is distinct from `User` and exists for project management.
-
-    You'll soon be able to conveniently create persons from users.
-
-    Example:
-        >>> person = Person(
-        ...     name="Jane Doe",
-        ...     email="jane.doe@example.com",
-        ...     internal=True,
-        ... ).save()
-    """
-
-    class Meta(SQLRecord.Meta, TracksRun.Meta, TracksUpdates.Meta):
-        abstract = False
-        app_label = "lamindb"
-
-    id: int = models.AutoField(primary_key=True)
-    """Internal id, valid only in one DB instance."""
-    uid: str = CharField(
-        editable=False, unique=True, max_length=8, db_index=True, default=base62_8
-    )
-    """Universal id, valid across DB instances."""
-    name: str = CharField(db_index=True)
-    """Name of the person (forename(s) lastname)."""
-    email: str | None = EmailField(null=True, default=None)
-    """Email of the person."""
-    external: bool = BooleanField(default=True, db_index=True)
-    """Whether the person is external to the organization."""
-    records: Record = models.ManyToManyField(
-        Record, through="RecordPerson", related_name="linked_people"
-    )
-    """Linked records."""
-
-    @overload
-    def __init__(
-        self,
-        name: str,
-        email: str | None = None,
-        external: bool = True,
-    ): ...
-
-    @overload
-    def __init__(
-        self,
-        *db_args,
-    ): ...
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
+    from .block import ProjectBlock
 
 
 class Reference(SQLRecord, CanCurate, TracksRun, TracksUpdates, ValidateFields):
@@ -118,6 +65,8 @@ class Reference(SQLRecord, CanCurate, TracksRun, TracksUpdates, ValidateFields):
     """Universal id, valid across DB instances."""
     name: str = CharField(db_index=True)
     """Title or name of the reference document."""
+    description: str | None = TextField(null=True)
+    """A description."""
     type: Reference | None = ForeignKey(
         "self", PROTECT, null=True, related_name="references"
     )
@@ -150,30 +99,32 @@ class Reference(SQLRecord, CanCurate, TracksRun, TracksUpdates, ValidateFields):
         ],
     )
     """Digital Object Identifier (DOI) for the reference."""
-    description: str | None = CharField(null=True, db_index=True)
-    """Description of the reference."""
     text: str | None = TextField(null=True, db_index=True)
     """Abstract or full text of the reference to make it searchable."""
     date: DateType | None = DateField(null=True, default=None)
     """Date of creation or publication of the reference."""
-    authors: Person = models.ManyToManyField(Person, related_name="references")
-    """All people associated with this reference."""
     artifacts: Artifact = models.ManyToManyField(
         Artifact, through="ArtifactReference", related_name="references"
     )
-    """Artifacts associated with this reference."""
+    """Annotated artifacts."""
     transforms: Artifact = models.ManyToManyField(
         Transform, through="TransformReference", related_name="references"
     )
-    """Transforms associated with this reference."""
+    """Annotated transforms."""
     collections: Artifact = models.ManyToManyField(
         Collection, through="CollectionReference", related_name="references"
     )
-    """Collections associated with this reference."""
-    records: Record = models.ManyToManyField(
+    """Annotated collections."""
+    linked_in_records: Record = models.ManyToManyField(
         Record, through="RecordReference", related_name="linked_references"
     )
-    """Linked records."""
+    """Linked in records."""
+    records: Record = models.ManyToManyField(
+        Record, through="ReferenceRecord", related_name="references"
+    )
+    """Annotated records."""
+    projects: Project
+    """Projects that annotate this reference."""
 
     @overload
     def __init__(
@@ -227,6 +178,8 @@ class Project(SQLRecord, CanCurate, TracksRun, TracksUpdates, ValidateFields):
     """Universal id, valid across DB instances."""
     name: str = CharField(db_index=True)
     """Title or name of the Project."""
+    description: str | None = TextField(null=True)
+    """A description."""
     type: Project | None = ForeignKey(
         "self", PROTECT, null=True, related_name="projects"
     )
@@ -261,50 +214,48 @@ class Project(SQLRecord, CanCurate, TracksRun, TracksUpdates, ValidateFields):
 
     Reverse accessor for `.predecessors`.
     """
-    people: Person = models.ManyToManyField(
-        Person, through="PersonProject", related_name="projects"
-    )
-    """Linked people."""
     artifacts: Artifact = models.ManyToManyField(
         Artifact, through="ArtifactProject", related_name="projects"
     )
-    """Linked artifacts."""
+    """Annotated artifacts."""
     transforms: Transform = models.ManyToManyField(
         Transform, through="TransformProject", related_name="projects"
     )
-    """Linked transforms."""
+    """Annotated transforms."""
     runs: Run = models.ManyToManyField(
         Run, through="RunProject", related_name="projects"
     )
-    """Linked transforms."""
+    """Annotated runs."""
     ulabels: ULabel = models.ManyToManyField(
         ULabel, through="ULabelProject", related_name="projects"
     )
-    """Linked ulabels."""
+    """Annotated ulabels."""
     features: ULabel = models.ManyToManyField(
         Feature, through="FeatureProject", related_name="projects"
     )
-    """Linked features."""
+    """Annotated features."""
     schemas: ULabel = models.ManyToManyField(
         Schema, through="SchemaProject", related_name="projects"
     )
-    """Linked schemas."""
+    """Annotated schemas."""
     linked_in_records: Record = models.ManyToManyField(
         Record, through="RecordProject", related_name="linked_projects"
     )
-    """Linked records."""
+    """Linked in records."""
     records: Record = models.ManyToManyField(
         Record, through="ProjectRecord", related_name="projects"
     )
-    """Annotated record."""
+    """Annotated records."""
     collections: Collection = models.ManyToManyField(
         Collection, through="CollectionProject", related_name="projects"
     )
-    """Linked collections."""
+    """Annotated collections."""
     references: Reference = models.ManyToManyField("Reference", related_name="projects")
-    """Linked references."""
+    """Annotated references."""
     _status_code: int = models.SmallIntegerField(default=0, db_index=True)
     """Status code."""
+    blocks: ProjectBlock
+    """Blocks that annotate this project."""
 
     @overload
     def __init__(
@@ -402,17 +353,6 @@ class ULabelProject(BaseSQLRecord, IsLink, TracksRun):
         unique_together = ("ulabel", "project")
 
 
-class PersonProject(BaseSQLRecord, IsLink, TracksRun):
-    id: int = models.BigAutoField(primary_key=True)
-    person: Person = ForeignKey(Person, CASCADE, related_name="links_project")
-    project: Project = ForeignKey(Project, PROTECT, related_name="links_person")
-    role: str | None = CharField(null=True, default=None)
-
-    class Meta:
-        app_label = "lamindb"
-        unique_together = ("person", "project")
-
-
 class FeatureProject(BaseSQLRecord, IsLink, TracksRun):
     id: int = models.BigAutoField(primary_key=True)
     feature: Feature = ForeignKey(Feature, CASCADE, related_name="links_project")
@@ -433,15 +373,22 @@ class SchemaProject(BaseSQLRecord, IsLink, TracksRun):
         unique_together = ("schema", "project")
 
 
-class RecordPerson(BaseSQLRecord, IsLink):
+# for annotation of records with references, RecordReference is for storing reference values
+class ReferenceRecord(BaseSQLRecord, IsLink, TracksRun):
     id: int = models.BigAutoField(primary_key=True)
-    record: Record = ForeignKey(Record, CASCADE, related_name="values_person")
-    feature: Feature = ForeignKey(Feature, PROTECT, related_name="links_recordperson")
-    value: Person = ForeignKey(Person, PROTECT, related_name="links_record")
+    reference: Reference = ForeignKey(Reference, PROTECT, related_name="links_record")
+    feature: Feature | None = ForeignKey(
+        Feature,
+        PROTECT,
+        null=True,
+        default=None,
+        related_name="links_referencerecord",
+    )
+    record: Record = ForeignKey(Record, CASCADE, related_name="links_reference")
 
     class Meta:
         app_label = "lamindb"
-        unique_together = ("record", "feature", "value")
+        unique_together = ("reference", "feature", "record")
 
 
 class RecordReference(BaseSQLRecord, IsLink):
@@ -450,7 +397,7 @@ class RecordReference(BaseSQLRecord, IsLink):
     feature: Feature = ForeignKey(
         Feature, PROTECT, related_name="links_recordreference"
     )
-    value: Reference = ForeignKey(Reference, PROTECT, related_name="links_record")
+    value: Reference = ForeignKey(Reference, PROTECT, related_name="links_in_record")
 
     class Meta:
         app_label = "lamindb"
@@ -460,7 +407,6 @@ class RecordReference(BaseSQLRecord, IsLink):
 # for annotation of records with projects, RecordProject is for storing project values
 class ProjectRecord(BaseSQLRecord, IsLink, TracksRun):
     id: int = models.BigAutoField(primary_key=True)
-    record: Record = ForeignKey(Record, CASCADE, related_name="links_project")
     project: Project = ForeignKey(Project, PROTECT, related_name="links_record")
     feature: Feature | None = ForeignKey(
         Feature,
@@ -469,11 +415,11 @@ class ProjectRecord(BaseSQLRecord, IsLink, TracksRun):
         default=None,
         related_name="links_projectrecord",
     )
+    record: Record = ForeignKey(Record, CASCADE, related_name="links_project")
 
     class Meta:
-        # can have the same label linked to the same artifact if the feature is different
         app_label = "lamindb"
-        unique_together = ("record", "project", "feature")
+        unique_together = ("project", "feature", "record")
 
 
 class RecordProject(BaseSQLRecord, IsLink):
@@ -503,7 +449,6 @@ class ArtifactReference(BaseSQLRecord, IsLink, TracksRun):
 
     class Meta:
         app_label = "lamindb"
-        # can have the same label linked to the same artifact if the feature is different
         unique_together = ("artifact", "reference", "feature")
 
 

lamindb/models/query_manager.py

@@ -302,7 +302,7 @@ class QueryManager(Manager):
     def all(self):
         """Return QuerySet of all.
 
-        For `**kwargs`, see :meth:`lamindb.models.QuerySet.df`.
+        For `**kwargs`, see :meth:`lamindb.models.QuerySet.to_dataframe`.
         """
         self._track_run_input_manager()
         return super().all()