lamindb 1.12.1__py3-none-any.whl → 1.13.0__py3-none-any.whl

lamindb/models/artifact_set.py

@@ -11,7 +11,7 @@ from upath import UPath
 
 from ..core._mapped_collection import MappedCollection
 from ..core.storage._backed_access import _open_dataframe
-from .artifact import Artifact, _track_run_input
+from .artifact import Artifact, track_run_input
 from .collection import Collection, _load_concat_artifacts
 
 if TYPE_CHECKING:
@@ -55,7 +55,7 @@ class ArtifactSet(Iterable):
         artifacts: list[Artifact] = list(self)
         concat_object = _load_concat_artifacts(artifacts, join, **kwargs)
         # track only if successful
-        _track_run_input(artifacts, is_run_input)
+        track_run_input(artifacts, is_run_input)
         return concat_object
 
     @doc_args(Collection.open.__doc__)
@@ -74,7 +74,7 @@ class ArtifactSet(Iterable):
 
         dataframe = _open_dataframe(paths, engine=engine, **kwargs)
         # track only if successful
-        _track_run_input(artifacts, is_run_input)
+        track_run_input(artifacts, is_run_input)
         return dataframe
 
     @doc_args(Collection.mapped.__doc__)
@@ -122,7 +122,7 @@ class ArtifactSet(Iterable):
             dtype,
         )
         # track only if successful
-        _track_run_input(artifacts, is_run_input)
+        track_run_input(artifacts, is_run_input)
         return ds
 
 

lamindb/models/block.py

@@ -71,7 +71,7 @@ class RootBlock(BlockMixin, BaseSQLRecord):
 
 
 class RecordBlock(BlockMixin, BaseSQLRecord):
-    """An unstructured notes block that can be attached to an artifact."""
+    """An unstructured notes block that can be attached to a record."""
 
     class Meta:
         app_label = "lamindb"
@@ -91,7 +91,7 @@ class ArtifactBlock(BlockMixin, BaseSQLRecord):
 
 
 class TransformBlock(BlockMixin, BaseSQLRecord):
-    """An unstructured notes block that can be attached to an artifact."""
+    """An unstructured notes block that can be attached to a transform."""
 
     class Meta:
         app_label = "lamindb"
@@ -105,7 +105,7 @@ class TransformBlock(BlockMixin, BaseSQLRecord):
 
 
 class RunBlock(BlockMixin, BaseSQLRecord):
-    """An unstructured notes block that can be attached to an artifact."""
+    """An unstructured notes block that can be attached to a run."""
 
     class Meta:
         app_label = "lamindb"
@@ -115,7 +115,7 @@ class RunBlock(BlockMixin, BaseSQLRecord):
 
 
 class CollectionBlock(BlockMixin, BaseSQLRecord):
-    """An unstructured notes block that can be attached to an artifact."""
+    """An unstructured notes block that can be attached to a collection."""
 
     class Meta:
         app_label = "lamindb"
@@ -127,7 +127,7 @@ class CollectionBlock(BlockMixin, BaseSQLRecord):
 
 
 class SchemaBlock(BlockMixin, BaseSQLRecord):
-    """An unstructured notes block that can be attached to an artifact."""
+    """An unstructured notes block that can be attached to a schema."""
 
     class Meta:
         app_label = "lamindb"
@@ -137,7 +137,7 @@ class SchemaBlock(BlockMixin, BaseSQLRecord):
 
 
 class FeatureBlock(BlockMixin, BaseSQLRecord):
-    """An unstructured notes block that can be attached to an artifact."""
+    """An unstructured notes block that can be attached to a feature."""
 
     class Meta:
         app_label = "lamindb"
@@ -147,16 +147,17 @@ class FeatureBlock(BlockMixin, BaseSQLRecord):
 
 
 class ProjectBlock(BlockMixin, BaseSQLRecord):
-    """An unstructured notes block that can be attached to an artifact."""
+    """An unstructured notes block that can be attached to a project."""
 
     class Meta:
         app_label = "lamindb"
 
     project: Project = ForeignKey(Project, CASCADE, related_name="blocks")
+    """The project to which the block is attached."""
 
 
 class SpaceBlock(BlockMixin, BaseSQLRecord):
-    """An unstructured notes block that can be attached to an artifact."""
+    """An unstructured notes block that can be attached to a space."""
 
     class Meta:
         app_label = "lamindb"
@@ -166,9 +167,10 @@ class SpaceBlock(BlockMixin, BaseSQLRecord):
 
 
 class BranchBlock(BlockMixin, BaseSQLRecord):
-    """An unstructured notes block that can be attached to an artifact."""
+    """An unstructured notes block that can be attached to a branch."""
 
     class Meta:
         app_label = "lamindb"
 
     branch: Branch = ForeignKey(Branch, CASCADE, related_name="blocks")
+    """The branch to which the block is attached."""

lamindb/models/can_curate.py

@@ -356,7 +356,7 @@ def _add_or_remove_synonyms(
         from IPython.display import display
 
         syns_all = (
-            record.__class__.filter().exclude(synonyms="").exclude(synonyms=None).all()  # type: ignore
+            record.__class__.filter().exclude(synonyms="").exclude(synonyms=None)  # type: ignore
         )
         if len(syns_all) == 0:
             return

lamindb/models/collection.py

@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, Literal, overload
 import anndata as ad
 import pandas as pd
 from django.db import models
-from django.db.models import CASCADE, PROTECT
+from django.db.models import CASCADE, PROTECT, Q
 from lamin_utils import logger
 from lamindb_setup.core.hashing import HASH_LENGTH, hash_set
 
@@ -25,11 +25,11 @@ from ..models._is_versioned import process_revises
 from ._is_versioned import IsVersioned
 from .artifact import (
     Artifact,
-    _populate_subsequent_runs_,
-    _track_run_input,
     describe_artifact_collection,
     get_run,
+    populate_subsequent_run,
     save_schema_links,
+    track_run_input,
 )
 from .has_parents import view_lineage
 from .run import Run, TracksRun, TracksUpdates
@@ -317,17 +317,19 @@ class Collection(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
                     )
         # we ignore collections in trash containing the same hash
         if hash is not None:
-            existing_collection = Collection.filter(hash=hash).one_or_none()
+            existing_collection = Collection.objects.filter(
+                ~Q(branch_id=-1),
+                hash=hash,
+            ).one_or_none()
         else:
             existing_collection = None
         if existing_collection is not None:
             logger.warning(
                 f"returning existing collection with same hash: {existing_collection}; if you intended to query to track this collection as an input, use: ln.Collection.get()"
             )
-            if run is not None:
-                existing_collection._populate_subsequent_runs(run)
             init_self_from_db(self, existing_collection)
             update_attributes(self, {"description": description, "key": key})
+            populate_subsequent_run(self, run)
         else:
             _skip_validation = revises is not None and key == revises.key
             super().__init__(  # type: ignore
@@ -348,9 +350,9 @@ class Collection(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
                 _skip_validation=_skip_validation,
             )
         self._artifacts = artifacts
-        if revises is not None:
-            _track_run_input(revises, run=run)
-        _track_run_input(artifacts, run=run)
+        if revises is not None and revises.uid != self.uid:
+            track_run_input(revises, run=run)
+        track_run_input(artifacts, run=run)
 
     @classmethod
     def get(
@@ -440,7 +442,7 @@ class Collection(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
 
         dataframe = _open_dataframe(paths, engine=engine, **kwargs)
         # track only if successful
-        _track_run_input(self, is_run_input)
+        track_run_input(self, is_run_input)
         return dataframe
 
     def mapped(
@@ -540,7 +542,7 @@ class Collection(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
             dtype,
         )
         # track only if successful
-        _track_run_input(self, is_run_input)
+        track_run_input(self, is_run_input)
         return ds
 
     def cache(self, is_run_input: bool | None = None) -> list[UPath]:
@@ -557,7 +559,7 @@ class Collection(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
         for artifact in self.ordered_artifacts.all():
             # do not want to track data lineage on the artifact level
             path_list.append(artifact.cache(is_run_input=False))
-        _track_run_input(self, is_run_input)
+        track_run_input(self, is_run_input)
         return path_list
 
     def load(
@@ -570,11 +572,11 @@ class Collection(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
 
         Returns an in-memory concatenated `DataFrame` or `AnnData` object.
         """
-        # cannot call _track_run_input here, see comment further down
+        # cannot call track_run_input here, see comment further down
         artifacts = self.ordered_artifacts.all()
         concat_object = _load_concat_artifacts(artifacts, join, **kwargs)
         # only call it here because there might be errors during load or concat
-        _track_run_input(self, is_run_input)
+        track_run_input(self, is_run_input)
         return concat_object
 
     def save(self, using: str | None = None) -> Collection:
@@ -664,9 +666,6 @@ class Collection(SQLRecord, IsVersioned, TracksRun, TracksUpdates):
         """
         return describe_artifact_collection(self)
 
-    def _populate_subsequent_runs(self, run: Run) -> None:
-        _populate_subsequent_runs_(self, run)
-
 
 # internal function, not exposed to user
 def from_artifacts(artifacts: Iterable[Artifact]) -> tuple[str, dict[str, str]]:

lamindb/models/has_parents.py

@@ -550,9 +550,7 @@ def _get_all_child_runs(data: Artifact | Collection) -> list:
         runs.update(
             {
                 f.run
-                for f in data.run.output_collections.all()
-                .filter(branch_id__in=[0, 1])
-                .all()
+                for f in data.run.output_collections.all().filter(branch_id__in=[0, 1])
             }
         )
     while runs.difference(all_runs):

lamindb/models/query_manager.py

@@ -237,7 +237,7 @@ class QueryManager(Manager):
     Examples:
 
         >>> ln.save(ln.ULabel.from_values(["ULabel1", "ULabel2", "ULabel3"], field="name"))  # noqa
-        >>> labels = ln.ULabel.filter(name__icontains = "label").all()
+        >>> labels = ln.ULabel.filter(name__icontains = "label")
         >>> ln.ULabel(name="ULabel1").save()
         >>> label = ln.ULabel.get(name="ULabel1")
         >>> label.parents.set(labels)
@@ -245,7 +245,7 @@ class QueryManager(Manager):
         >>> manager.to_dataframe()
     """
 
-    def _track_run_input_manager(self):
+    def track_run_input_manager(self):
         if hasattr(self, "source_field_name") and hasattr(self, "target_field_name"):
             if (
                 self.source_field_name == "collection"
@@ -255,7 +255,7 @@ class QueryManager(Manager):
                 from lamindb.core._context import context
                 from lamindb.models.artifact import (
                     WARNING_RUN_TRANSFORM,
-                    _track_run_input,
+                    track_run_input,
                 )
 
                 if (
@@ -263,14 +263,14 @@ class QueryManager(Manager):
                     and not settings.creation.artifact_silence_missing_run_warning
                 ):
                     logger.warning(WARNING_RUN_TRANSFORM)
-                _track_run_input(self.instance)
+                track_run_input(self.instance)
 
     def to_list(self, field: str | None = None):
         """Populate a list with the results.
 
         Examples:
             >>> ln.save(ln.ULabel.from_values(["ULabel1", "ULabel2", "ULabel3"], field="name"))
-            >>> labels = ln.ULabel.filter(name__icontains="label").all()
+            >>> labels = ln.ULabel.filter(name__icontains="label")
             >>> ln.ULabel(name="ULabel1").save()
             >>> label = ln.ULabel.get(name="ULabel1")
             >>> label.parents.set(labels)
@@ -281,7 +281,7 @@ class QueryManager(Manager):
         if field is None:
             return list(self.all())
         else:
-            self._track_run_input_manager()
+            self.track_run_input_manager()
             return list(self.values_list(field, flat=True))
 
     @deprecated(new_name="to_list")
@@ -304,7 +304,7 @@ class QueryManager(Manager):
 
         For `**kwargs`, see :meth:`lamindb.models.QuerySet.to_dataframe`.
         """
-        self._track_run_input_manager()
+        self.track_run_input_manager()
         return super().all()
 
     @doc_args(_search.__doc__)

lamindb/models/query_set.py

@@ -26,6 +26,8 @@ from .sqlrecord import Registry, SQLRecord
 if TYPE_CHECKING:
     from lamindb.base.types import ListLike, StrField
 
+    from .sqlrecord import Branch
+
 T = TypeVar("T")
 
 
@@ -60,7 +62,7 @@ def get_keys_from_df(data: list, registry: SQLRecord) -> list[str]:
     return keys
 
 
-def get_default_branch_ids() -> list[int]:
+def get_default_branch_ids(branch: Branch | None = None) -> list[int]:
     """Return branch IDs to include in default queries.
 
     By default, queries include records on the main branch (branch_id=1) but exclude trashed (branch_id=-1)
@@ -72,7 +74,10 @@ def get_default_branch_ids() -> list[int]:
     Returns:
         List containing the default branch and current branch if different.
     """
-    branch_id = setup_settings.branch.id
+    if branch is None:
+        branch_id = setup_settings.branch.id
+    else:
+        branch_id = branch.id
     branch_ids = [branch_id]
     if branch_id != 1:  # add the main branch by default
         branch_ids.append(1)
@@ -142,7 +147,7 @@ def get_backward_compat_filter_kwargs(queryset, expressions):
     return list(mapped.keys()) if was_list else mapped
 
 
-def process_expressions(queryset: QuerySet, expressions: dict) -> dict:
+def process_expressions(queryset: QuerySet, queries: tuple, expressions: dict) -> dict:
     def _map_databases(value: Any, key: str, target_db: str) -> tuple[str, Any]:
         if isinstance(value, SQLRecord):
             if value._state.db != target_db:
@@ -170,6 +175,28 @@ def process_expressions(queryset: QuerySet, expressions: dict) -> dict:
 
         return key, value
 
+    branch_fields = {"branch", "branch_id"}
+    branch_prefixes = ("branch__", "branch_id__")
+
+    def queries_contain_branch(queries: tuple) -> bool:
+        """Check if any Q object in queries references branch or branch_id."""
+
+        def check_q_object(q: Q) -> bool:
+            # Q objects store their conditions in q.children
+            for child in q.children:
+                if isinstance(child, tuple) and len(child) == 2:
+                    # Normal condition: (key, value)
+                    key = child[0]
+                    if key in branch_fields or key.startswith(branch_prefixes):
+                        return True
+                elif isinstance(child, Q):
+                    # Nested Q object
+                    if check_q_object(child):
+                        return True
+            return False
+
+        return any(check_q_object(q) for q in queries if isinstance(q, Q))
+
     expressions = get_backward_compat_filter_kwargs(
         queryset,
         expressions,
@@ -179,15 +206,13 @@ def process_expressions(queryset: QuerySet, expressions: dict) -> dict:
         id_uid_hash = {"id", "uid", "hash", "id__in", "uid__in", "hash__in"}
         if not any(expression in id_uid_hash for expression in expressions):
             expressions_have_branch = False
-            branch_branch_id = {"branch", "branch_id"}
-            branch_branch_id__ = ("branch__", "branch_id__")
             for expression in expressions:
-                if expression in branch_branch_id or expression.startswith(
-                    branch_branch_id__
+                if expression in branch_fields or expression.startswith(
+                    branch_prefixes
                 ):
                     expressions_have_branch = True
                     break
-            if not expressions_have_branch:
+            if not expressions_have_branch and not queries_contain_branch(queries):
                 expressions["branch_id__in"] = get_default_branch_ids()
             else:
                 # if branch_id is None, do not apply a filter
@@ -197,6 +222,7 @@ def process_expressions(queryset: QuerySet, expressions: dict) -> dict:
                     expressions.pop("branch_id")
                 if "branch" in expressions and expressions["branch"] is None:
                     expressions.pop("branch")
+
     if queryset._db is not None:
         # only check for database mismatch if there is a defined database on the
         # queryset
@@ -254,7 +280,7 @@ def get(
             return one_helper(qs, DOESNOTEXIST_MSG)
     else:
         assert idlike is None  # noqa: S101
-        expressions = process_expressions(qs, expressions)
+        expressions = process_expressions(qs, [], expressions)
         # inject is_latest for consistency with idlike
         is_latest_was_not_in_expressions = "is_latest" not in expressions
         if issubclass(registry, IsVersioned) and is_latest_was_not_in_expressions:
@@ -1022,11 +1048,11 @@ class QuerySet(BasicQuerySet):
             raise  # pragma: no cover
 
         if is_run_input is not False:  # might be None or True or Run
-            from .artifact import Artifact, _track_run_input
+            from .artifact import Artifact, track_run_input
             from .collection import Collection
 
             if isinstance(record, (Artifact, Collection)):
-                _track_run_input(record, is_run_input)
+                track_run_input(record, is_run_input)
 
         return record
 
@@ -1059,7 +1085,7 @@ class QuerySet(BasicQuerySet):
                         f"Invalid lookup '{value}' for {field}. Did you mean {field}__name?"
                     )
 
-        expressions = process_expressions(self, expressions)
+        expressions = process_expressions(self, queries, expressions)
         # need to run a query if queries or expressions are not empty
         if queries or expressions:
             try:

lamindb/models/run.py

@@ -9,7 +9,6 @@ from django.db.models import (
 )
 from lamindb_setup import _check_instance_setup
 
-from lamindb.base import deprecated
 from lamindb.base.fields import (
     BooleanField,
     CharField,
@@ -198,31 +197,57 @@ class Run(SQLRecord):
 
     Args:
         transform: `Transform` A :class:`~lamindb.Transform` record.
-        initiated_by_run: `Run | None = None` The run that triggers this run.
+        name: `str | None = None` An optional name.
+        params: `dict | None = None` A dictionary of parameters.
         reference: `str | None = None` For instance, an external ID or a download URL.
         reference_type: `str | None = None` For instance, `redun_id`, `nextflow_id` or `url`.
+        initiated_by_run: `Run | None = None` The run that triggers this run.
 
     See Also:
         :func:`~lamindb.track`
             Globally track a script or notebook run.
+        :func:`~lamindb.tracked`
+            Track a function with this decorator.
 
     Examples:
 
-        Create a run record:
+        Create a run record::
+
+            ln.Transform(key="Cell Ranger", version="7.2.0", type="pipeline").save()
+            transform = ln.Transform.get(key="Cell Ranger", version="7.2.0")
+            run = ln.Run(transform)
+
+        Create a global run context for a custom transform::
+
+            ln.track(transform=transform)
+            ln.context.run  # global run
 
-        >>> ln.Transform(key="Cell Ranger", version="7.2.0", type="pipeline").save()
-        >>> transform = ln.Transform.get(key="Cell Ranger", version="7.2.0")
-        >>> run = ln.Run(transform)
+        Track a global run context for a notebook or script::
 
-        Create a global run context for a custom transform:
+            ln.track()
+            ln.context.run  # global run
+
+        You can pass parameters to `Run(transform, params=params)` or add them later::
+
+            run.params = {
+                "learning_rate": 0.01,
+                "input_dir": "s3://my-bucket/mydataset",
+                "downsample": True,
+                "preprocess_params": {
+                    "normalization_type": "cool",
+                    "subset_highlyvariable": True,
+                },
+            }
+            run.save()
 
-        >>> ln.track(transform=transform)
-        >>> ln.context.run  # globally available run
+        In contrast to `.params`, features are indexed in the `Feature` registry and can reference relational categorical values.
+        If you want to link feature values, use::
 
-        Track a global run context for a notebook or script:
+            run.features.add_values({
+                "experiment": "My experiment 1",
+            })
 
-        >>> ln.track()  # Jupyter notebook metadata is automatically parsed
-        >>> ln.context.run
+        Guide: :ref:`track-run-parameters`
     """
 
     class Meta:
@@ -282,7 +307,8 @@ class Run(SQLRecord):
     """The collections serving as input for this run."""
     output_records: Record
     """The collections generated by this run."""
-    """Parameter values."""
+    params: dict = models.JSONField(null=True)
+    """JSON-like parameters."""
     _feature_values: FeatureValue = models.ManyToManyField(
         "FeatureValue", through="RunFeatureValue", related_name="runs"
     )
@@ -331,6 +357,8 @@ class Run(SQLRecord):
     def __init__(
         self,
         transform: Transform,
+        name: str | None = None,
+        params: dict | None = None,
         reference: str | None = None,
         reference_type: str | None = None,
         initiated_by_run: Run | None = None,
@@ -356,6 +384,8 @@ class Run(SQLRecord):
         transform: Transform = None
         if "transform" in kwargs or len(args) == 1:
             transform = kwargs.pop("transform") if len(args) == 0 else args[0]
+        name: str | None = kwargs.pop("name", None)
+        params: dict | None = kwargs.pop("params", None)
         reference: str | None = kwargs.pop("reference", None)
         reference_type: str | None = kwargs.pop("reference_type", None)
         initiated_by_run: Run | None = kwargs.pop("initiated_by_run", None)
@@ -363,12 +393,17 @@ class Run(SQLRecord):
             raise TypeError("Pass transform parameter")
         if transform._state.adding:
             raise ValueError("Please save transform record before creating a run")
-
+        if not len(kwargs) == 0:
+            raise ValueError(
+                f"Only transform, name, params, reference, reference_type, initiated_by_run can be passed, but you passed: {kwargs}"
+            )
         super().__init__(  # type: ignore
             transform=transform,
+            name=name,
+            params=params,
             reference=reference,
-            initiated_by_run=initiated_by_run,
             reference_type=reference_type,
+            initiated_by_run=initiated_by_run,
         )
 
     @property
@@ -377,22 +412,13 @@ class Run(SQLRecord):
 
         Returns the status as a string, one of: `scheduled`, `re-started`, `started`, `completed`, `errored`, or `aborted`.
 
-        The string maps to an integer field `_status_code` of the run registry, with mapping:
-            - -3: `scheduled`
-            - -2: `re-started`
-            - -1: `started`
-            - 0: `completed`
-            - 1: `errored`
-            - 2: `aborted`
-
-        You can use this private integer field for queries.
-
         Examples:
 
-            ::
+            See the status of a run::
 
                 run.status
                 #> 'completed'
+
         """
         if self._status_code is None:
             return "unknown"
@@ -406,31 +432,9 @@ class Run(SQLRecord):
         }
         return status_dict.get(self._status_code, "unknown")
 
-    @property
-    @deprecated("features")
-    def params(self) -> FeatureManager:
-        return self.features
-
     @property
     def features(self) -> FeatureManager:
-        """Features manager.
-
-        Run parameters are tracked via the `Feature` registry, just like all other variables.
-
-        Guide: :ref:`track-run-parameters`
-
-        Example::
-
-            run.features.add_values({
-                "learning_rate": 0.01,
-                "input_dir": "s3://my-bucket/mydataset",
-                "downsample": True,
-                "preprocess_params": {
-                    "normalization_type": "cool",
-                    "subset_highlyvariable": True,
-                },
-            })
-        """
+        """Manage annotations with features."""
         from ._feature_manager import FeatureManager
 
         return FeatureManager(self)