lamindb 1.5.3__py3-none-any.whl → 1.6.0__py3-none-any.whl

lamindb/models/run.py

@@ -1,18 +1,16 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, overload
+from typing import TYPE_CHECKING, overload
 
 import numpy as np
 from django.db import models
 from django.db.models import (
     CASCADE,
     PROTECT,
-    Q,
 )
-from django.db.utils import IntegrityError
 from lamindb_setup import _check_instance_setup
-from lamindb_setup.core.hashing import HASH_LENGTH, hash_dict
 
+from lamindb.base import deprecated
 from lamindb.base.fields import (
     BooleanField,
     CharField,
@@ -20,22 +18,20 @@ from lamindb.base.fields import (
     ForeignKey,
 )
 from lamindb.base.users import current_user_id
-from lamindb.errors import InvalidArgument, ValidationError
+from lamindb.errors import InvalidArgument
 
-from ..base.ids import base62_20
+from ..base.ids import base62_16
 from .can_curate import CanCurate
-from .record import BasicRecord, LinkORM, Record, Registry
+from .sqlrecord import BaseSQLRecord, IsLink, SQLRecord
 
 if TYPE_CHECKING:
     from datetime import datetime
 
-    from lamindb.base.types import Dtype, FieldAttr
-
     from .artifact import Artifact
     from .collection import Collection
+    from .feature import FeatureValue
     from .project import Project
     from .query_set import QuerySet
-    from .schema import Schema
     from .transform import Transform
     from .ulabel import ULabel
 
@@ -43,14 +39,14 @@ if TYPE_CHECKING:
 _TRACKING_READY: bool | None = None
 
 
-class ParamManager:
-    """Param manager."""
+class FeatureManager:
+    """Feature manager."""
 
     pass
 
 
-class ParamManagerRun(ParamManager):
-    """Param manager."""
+class FeatureManagerRun(FeatureManager):
+    """Feature manager."""
 
     pass
 
@@ -140,7 +136,7 @@ class TracksUpdates(models.Model):
         super().__init__(*args, **kwargs)
 
 
-class User(BasicRecord, CanCurate):
+class User(BaseSQLRecord, CanCurate):
     """Users.
 
     All data in this registry is synced from `lamin.ai` to ensure a universal
@@ -201,152 +197,7 @@ class User(BasicRecord, CanCurate):
         super().__init__(*args, **kwargs)
 
 
-class Param(Record, CanCurate, TracksRun, TracksUpdates):
-    """Parameters of runs & models."""
-
-    class Meta(Record.Meta, TracksRun.Meta, TracksUpdates.Meta):
-        abstract = False
-
-    _name_field: str = "name"
-
-    name: str = CharField(max_length=100, db_index=True)
-    dtype: Dtype | None = CharField(db_index=True, null=True)
-    """Data type (:class:`~lamindb.base.types.Dtype`)."""
-    type: Param | None = ForeignKey("self", PROTECT, null=True, related_name="records")
-    """Type of param (e.g., 'Pipeline', 'ModelTraining', 'PostProcessing').
-
-    Allows to group features by type, e.g., all read outs, all metrics, etc.
-    """
-    records: Param
-    """Records of this type."""
-    is_type: bool = BooleanField(default=False, db_index=True, null=True)
-    """Distinguish types from instances of the type."""
-    _expect_many: bool = models.BooleanField(default=False, db_default=False)
-    """Indicates whether values for this param are expected to occur a single or multiple times for an artifact/run (default `False`).
-
-    - if it's `False` (default), the values mean artifact/run-level values and a dtype of `datetime` means `datetime`
-    - if it's `True`, the values are from an aggregation, which this seems like an edge case but when characterizing a model ensemble trained with different parameters it could be relevant
-    """
-    schemas: Schema = models.ManyToManyField(
-        "Schema", through="SchemaParam", related_name="params"
-    )
-    """Feature sets linked to this feature."""
-    # backward fields
-    values: ParamValue
-    """Values for this parameter."""
-
-    @overload
-    def __init__(
-        self,
-        name: str,
-        dtype: Dtype | Registry | list[Registry] | FieldAttr,
-        type: Param | None = None,
-        is_type: bool = False,
-    ): ...
-
-    @overload
-    def __init__(
-        self,
-        *db_args,
-    ): ...
-
-    def __init__(self, *args, **kwargs):
-        from .feature import process_init_feature_param
-
-        if len(args) == len(self._meta.concrete_fields):
-            super().__init__(*args, **kwargs)
-            return None
-
-        dtype = kwargs.get("dtype", None)
-        kwargs = process_init_feature_param(args, kwargs, is_param=True)
-        super().__init__(*args, **kwargs)
-        dtype_str = kwargs.pop("dtype", None)
-        if not self._state.adding:
-            if not (
-                self.dtype.startswith("cat")
-                if dtype == "cat"
-                else self.dtype == dtype_str
-            ):
-                raise ValidationError(
-                    f"Feature {self.name} already exists with dtype {self.dtype}, you passed {dtype_str}"
-                )
-
-
-# FeatureValue behaves in many ways like a link in a LinkORM
-# in particular, we don't want a _public field on it
-# Also, we don't inherit from TracksRun because a ParamValue
-# is typically created before a run is created and we want to
-# avoid delete cycles (for Model params though it might be helpful)
-class ParamValue(Record):
-    """Parameter values.
-
-    Is largely analogous to `FeatureValue`.
-    """
-
-    # we do not have a unique constraint on param & value because it leads to hashing errors
-    # for large dictionaries: https://lamin.ai/laminlabs/lamindata/transform/jgTrkoeuxAfs0000
-    # we do not hash values because we have `get_or_create` logic all over the place
-    # and also for checking whether the (param, value) combination exists
-    # there does not seem an issue with querying for a dict-like value
-    # https://lamin.ai/laminlabs/lamindata/transform/jgTrkoeuxAfs0001
-    _name_field: str = "value"
-
-    param: Param = ForeignKey(Param, CASCADE, related_name="values")
-    """The dimension metadata."""
-    value: Any = (
-        models.JSONField()
-    )  # stores float, integer, boolean, datetime or dictionaries
-    """The JSON-like value."""
-    # it'd be confusing and hard to populate a run here because these
-    # values are typically created upon creating a run
-    # hence, ParamValue does _not_ inherit from TracksRun but manually
-    # adds created_at & created_by
-    # because ParamValue cannot be updated, we don't need updated_at
-    created_at: datetime = DateTimeField(
-        editable=False, db_default=models.functions.Now(), db_index=True
-    )
-    """Time of creation of record."""
-    created_by: User = ForeignKey(
-        User, PROTECT, default=current_user_id, related_name="+"
-    )
-    """Creator of record."""
-    hash: str = CharField(max_length=HASH_LENGTH, null=True, db_index=True)
-
-    class Meta:
-        constraints = [
-            # For simple types, use direct value comparison
-            models.UniqueConstraint(
-                fields=["param", "value"],
-                name="unique_simple_param_value",
-                condition=Q(hash__isnull=True),
-            ),
-            # For complex types (dictionaries), use hash
-            models.UniqueConstraint(
-                fields=["param", "hash"],
-                name="unique_complex_param_value",
-                condition=Q(hash__isnull=False),
-            ),
-        ]
-
-    @classmethod
-    def get_or_create(cls, param, value):
-        # Simple types: int, float, str, bool
-        if isinstance(value, (int, float, str, bool)):
-            try:
-                return cls.objects.create(param=param, value=value, hash=None), False
-            except IntegrityError:
-                return cls.objects.get(param=param, value=value), True
-
-        # Complex types: dict, list
-        else:
-            hash = hash_dict(value)
-            try:
-                return cls.objects.create(param=param, value=value, hash=hash), False
-            except IntegrityError:
-                return cls.objects.get(param=param, hash=hash), True
-
-
-class Run(Record):
+class Run(SQLRecord):
     """Runs of transforms such as the execution of a script.
 
     A registry to store runs of transforms, such as an executation of a script.
@@ -381,14 +232,16 @@ class Run(Record):
 
     _name_field: str = "started_at"
 
-    params: ParamManager = ParamManagerRun  # type: ignore
-    """Param manager.
+    features: FeatureManager = FeatureManagerRun  # type: ignore
+    """Features manager.
+
+    Run parameters are tracked via the `Feature` registry, just like all other variables.
 
     Guide: :ref:`track-run-parameters`
 
     Example::
 
-        run.params.add_values({
+        run.features.add_values({
             "learning_rate": 0.01,
             "input_dir": "s3://my-bucket/mydataset",
             "downsample": True,
@@ -401,8 +254,9 @@ class Run(Record):
 
     id: int = models.BigAutoField(primary_key=True)
     """Internal id, valid only in one DB instance."""
+    # default uid was changed from base62_20 to base62_16 in 1.6.0
     uid: str = CharField(
-        editable=False, unique=True, db_index=True, max_length=20, default=base62_20
+        editable=False, unique=True, db_index=True, max_length=20, default=base62_16
     )
     """Universal id, valid across DB instances."""
     name: str | None = CharField(max_length=150, null=True)
@@ -446,10 +300,11 @@ class Run(Record):
     """The collections serving as input for this run."""
     output_collections: Collection
     """The collections generated by this run."""
-    _param_values: ParamValue = models.ManyToManyField(
-        ParamValue, through="RunParamValue", related_name="runs"
-    )
     """Parameter values."""
+    _feature_values: FeatureValue = models.ManyToManyField(
+        "FeatureValue", through="RunFeatureValue", related_name="runs"
+    )
+    """Feature values."""
     reference: str | None = CharField(max_length=255, db_index=True, null=True)
     """A reference like a URL or external ID (such as from a workflow manager)."""
     reference_type: str | None = CharField(max_length=25, db_index=True, null=True)
@@ -510,7 +365,7 @@ class Run(Record):
         *args,
         **kwargs,
     ):
-        self.params = ParamManager(self)  # type: ignore
+        self.features = FeatureManager(self)  # type: ignore
         if len(args) == len(self._meta.concrete_fields):
             super().__init__(*args, **kwargs)
             return None
@@ -540,6 +395,11 @@ class Run(Record):
         delete_run_artifacts(self)
         super().delete()
 
+    @property
+    @deprecated("features")
+    def params(self) -> FeatureManager:
+        return self.features
+
     @classmethod
     def filter(
         cls,
@@ -566,6 +426,7 @@ class Run(Record):
                 ln.Run.filter(hyperparam_x=100)
         """
         from ._feature_manager import filter_base
+        from .feature import Feature
         from .query_set import QuerySet
 
         if expressions:
@@ -574,14 +435,14 @@ class Run(Record):
             if field_or_feature_or_param in Run.__get_available_fields__():
                 return QuerySet(model=cls).filter(*queries, **expressions)
             elif all(
-                params_validated := Param.validate(
+                params_validated := Feature.validate(
                     keys_normalized, field="name", mute=True
                 )
             ):
-                return filter_base(ParamManagerRun, **expressions)
+                return filter_base(FeatureManagerRun, **expressions)
             else:
                 params = ", ".join(sorted(np.array(keys_normalized)[~params_validated]))
-                message = f"param names: {params}"
+                message = f"feature names: {params}"
                 fields = ", ".join(sorted(cls.__get_available_fields__()))
                 raise InvalidArgument(
                     f"You can query either by available fields: {fields}\n"
@@ -612,11 +473,13 @@ def delete_run_artifacts(run: Run) -> None:
             report.delete(permanent=True)
 
 
-class RunParamValue(BasicRecord, LinkORM):
+class RunFeatureValue(BaseSQLRecord, IsLink):
     id: int = models.BigAutoField(primary_key=True)
-    run: Run = ForeignKey(Run, CASCADE, related_name="links_paramvalue")
+    run: Run = ForeignKey(Run, CASCADE, related_name="links_featurevalue")
     # we follow the lower() case convention rather than snake case for link models
-    paramvalue: ParamValue = ForeignKey(ParamValue, PROTECT, related_name="links_run")
+    featurevalue: FeatureValue = ForeignKey(
+        "FeatureValue", PROTECT, related_name="links_run"
+    )
     created_at: datetime = DateTimeField(
         editable=False, db_default=models.functions.Now(), db_index=True
     )
@@ -627,4 +490,4 @@ class RunParamValue(BasicRecord, LinkORM):
     """Creator of record."""
 
     class Meta:
-        unique_together = ("run", "paramvalue")
+        unique_together = ("run", "featurevalue")

lamindb/models/save.py

@@ -21,7 +21,7 @@ from ..core.storage.paths import (
     delete_storage_using_key,
     store_file_or_folder,
 )
-from .record import Record
+from .sqlrecord import SQLRecord
 
 if TYPE_CHECKING:
     from collections.abc import Iterable
@@ -29,7 +29,7 @@ if TYPE_CHECKING:
     from .artifact import Artifact
 
 
-def save(records: Iterable[Record], ignore_conflicts: bool | None = False) -> None:
+def save(records: Iterable[SQLRecord], ignore_conflicts: bool | None = False) -> None:
     """Bulk save records.
 
     Note:
@@ -42,7 +42,7 @@ def save(records: Iterable[Record], ignore_conflicts: bool | None = False) -> No
         existing records! Use ``record.save()`` for these use cases.
 
     Args:
-        records: Multiple :class:`~lamindb.models.Record` objects.
+        records: Multiple :class:`~lamindb.models.SQLRecord` objects.
         ignore_conflicts: If ``True``, do not error if some records violate a
            unique or another constraint. However, it won't inplace update the id
            fields of records. If you need records with ids, you need to query
@@ -69,7 +69,7 @@ def save(records: Iterable[Record], ignore_conflicts: bool | None = False) -> No
     """
     from .artifact import Artifact
 
-    if isinstance(records, Record):
+    if isinstance(records, SQLRecord):
         raise ValueError("Please use record.save() if saving a single record.")
 
     # previously, this was all set based,
@@ -107,7 +107,7 @@ def save(records: Iterable[Record], ignore_conflicts: bool | None = False) -> No
     return None
 
 
-def bulk_create(records: Iterable[Record], ignore_conflicts: bool | None = False):
+def bulk_create(records: Iterable[SQLRecord], ignore_conflicts: bool | None = False):
     records_by_orm = defaultdict(list)
     for record in records:
         records_by_orm[record.__class__].append(record)
@@ -116,7 +116,7 @@ def bulk_create(records: Iterable[Record], ignore_conflicts: bool | None = False
         # records[:] = created  # In-place list update; does not seem to be necessary
 
 
-def bulk_update(records: Iterable[Record], ignore_conflicts: bool | None = False):
+def bulk_update(records: Iterable[SQLRecord], ignore_conflicts: bool | None = False):
     records_by_orm = defaultdict(list)
     for record in records:
         records_by_orm[record.__class__].append(record)

lamindb/models/schema.py

@@ -35,16 +35,16 @@ from .feature import (
     serialize_dtype,
     serialize_pandas_dtype,
 )
-from .record import (
-    BasicRecord,
-    LinkORM,
-    Record,
+from .run import TracksRun, TracksUpdates
+from .sqlrecord import (
+    BaseSQLRecord,
+    IsLink,
     Registry,
+    SQLRecord,
     _get_record_kwargs,
     init_self_from_db,
     update_attributes,
 )
-from .run import Param, TracksRun, TracksUpdates
 
 if TYPE_CHECKING:
     import pandas as pd
@@ -59,7 +59,7 @@ NUMBER_TYPE = "num"
 DICT_KEYS_TYPE = type({}.keys())  # type: ignore
 
 
-def validate_features(features: list[Record]) -> Record:
+def validate_features(features: list[SQLRecord]) -> SQLRecord:
     """Validate and return feature type."""
     try:
         if len(features) == 0:
@@ -70,7 +70,7 @@ def validate_features(features: list[Record]) -> Record:
         ) from None
     if not hasattr(features, "__getitem__"):
         raise TypeError("features has to be list-like")
-    if not isinstance(features[0], Record):
+    if not isinstance(features[0], SQLRecord):
         raise TypeError(
             "features has to store feature records! use .from_values() otherwise"
         )
@@ -84,8 +84,8 @@ def validate_features(features: list[Record]) -> Record:
 
 
 def get_features_config(
-    features: list[Record] | tuple[Record, dict],
-) -> tuple[list[Record], list[tuple[Record, dict]]]:
+    features: list[SQLRecord] | tuple[SQLRecord, dict],
+) -> tuple[list[SQLRecord], list[tuple[SQLRecord, dict]]]:
     """Get features and their config from the return of feature.with_config()."""
     features_list = []
     configs = []
@@ -251,13 +251,13 @@ KNOWN_SCHEMAS = {
 }
 
 
-class Schema(Record, CanCurate, TracksRun):
+class Schema(SQLRecord, CanCurate, TracksRun):
     """Schemas of a dataset such as the set of columns of a `DataFrame`.
 
     Composite schemas can have multiple slots, e.g., for an `AnnData`, one schema for slot `obs` and another one for `var`.
 
     Args:
-        features: `list[Record] | list[tuple[Feature, dict]] | None = None` Feature
+        features: `list[SQLRecord] | list[tuple[Feature, dict]] | None = None` Feature
             records, e.g., `[Feature(...), Feature(...)]` or Features with their config, e.g., `[Feature(...).with_config(optional=True)]`.
         index: `Feature | None = None` A :class:`~lamindb.Feature` record to validate an index of a `DataFrame` and therefore also, e.g., `AnnData` obs and var indices.
         slots: `dict[str, Schema] | None = None` A dictionary mapping slot names to :class:`~lamindb.Schema` objects.
@@ -350,7 +350,7 @@ class Schema(Record, CanCurate, TracksRun):
             schema = ln.Schema.from_df(df)
     """
 
-    class Meta(Record.Meta, TracksRun.Meta, TracksUpdates.Meta):
+    class Meta(SQLRecord.Meta, TracksRun.Meta, TracksUpdates.Meta):
         abstract = False
 
     _name_field: str = "name"
@@ -363,18 +363,16 @@ class Schema(Record, CanCurate, TracksRun):
 
     id: int = models.AutoField(primary_key=True)
     """Internal id, valid only in one DB instance."""
+    # Before lamindb 1.5, it was 20 char long. Since lamindb 1.5, it is 16 char long.
     uid: str = CharField(editable=False, unique=True, db_index=True, max_length=20)
-    """A universal id.
-
-    Before lamindb 1.5, it was 20 char long. Since lamindb 1.5, it is 16 char long.
-    """
+    """A universal id."""
     name: str | None = CharField(max_length=150, null=True, db_index=True)
     """A name."""
     description: str | None = CharField(null=True, db_index=True)
     """A description."""
     n: int = IntegerField()
     """Number of features in the schema."""
-    type: Schema | None = ForeignKey("self", PROTECT, null=True, related_name="records")
+    type: Schema | None = ForeignKey("self", PROTECT, null=True, related_name="schemas")
     """Type of schema.
 
     Allows to group schemas by type, e.g., all meassurements evaluating gene expression vs. protein expression vs. multi modal.
@@ -383,8 +381,8 @@ class Schema(Record, CanCurate, TracksRun):
 
     Here are a few more examples for type names: `'ExpressionPanel'`, `'ProteinPanel'`, `'Multimodal'`, `'Metadata'`, `'Embedding'`.
     """
-    records: Schema
-    """Records of this type."""
+    instances: Schema
+    """Schemas of this type (can only be non-empty if `is_type` is `True`)."""
     is_type: bool = BooleanField(default=False, db_index=True, null=True)
     """Distinguish types from instances of the type."""
     itype: str | None = CharField(
@@ -434,8 +432,6 @@ class Schema(Record, CanCurate, TracksRun):
     """
     features: Feature
     """The features contained in the schema."""
-    params: Param
-    """The params contained in the schema."""
     artifacts: Artifact
     """The artifacts that measure a feature set that matches this schema."""
     validated_artifacts: Artifact
@@ -468,7 +464,7 @@ class Schema(Record, CanCurate, TracksRun):
     @overload
     def __init__(
         self,
-        features: list[Record] | list[tuple[Feature, dict]] | None = None,
+        features: list[SQLRecord] | list[tuple[Feature, dict]] | None = None,
         index: Feature | None = None,
         slots: dict[str, Schema] | None = None,
         name: str | None = None,
@@ -503,12 +499,14 @@ class Schema(Record, CanCurate, TracksRun):
         if len(args) > 1:
             raise ValueError("Only one non-keyword arg allowed: features")
 
-        features: list[Record] | None = args[0] if args else kwargs.pop("features", [])
+        features: list[SQLRecord] | None = (
+            args[0] if args else kwargs.pop("features", [])
+        )
         index: Feature | None = kwargs.pop("index", None)
         slots: dict[str, Schema] = kwargs.pop("slots", {})
         name: str | None = kwargs.pop("name", None)
         description: str | None = kwargs.pop("description", None)
-        itype: str | Record | DeferredAttribute | None = kwargs.pop("itype", None)
+        itype: str | SQLRecord | DeferredAttribute | None = kwargs.pop("itype", None)
         flexible: bool | None = kwargs.pop("flexible", None)
         type: Feature | None = kwargs.pop("type", None)
         is_type: bool = kwargs.pop("is_type", False)
@@ -590,12 +588,12 @@ class Schema(Record, CanCurate, TracksRun):
 
     def _validate_kwargs_calculate_hash(
         self,
-        features: list[Record],
+        features: list[SQLRecord],
         index: Feature | None,
         slots: dict[str, Schema],
         name: str | None,
         description: str | None,
-        itype: str | Record | DeferredAttribute | None,
+        itype: str | SQLRecord | DeferredAttribute | None,
         flexible: bool | None,
         type: Feature | None,
         is_type: bool,
@@ -737,8 +735,8 @@ class Schema(Record, CanCurate, TracksRun):
         type: str | None = None,
         name: str | None = None,
         mute: bool = False,
-        organism: Record | str | None = None,
-        source: Record | None = None,
+        organism: SQLRecord | str | None = None,
+        source: SQLRecord | None = None,
         raise_validation_error: bool = True,
     ) -> Schema:
         """Create feature set for validated features.
@@ -772,7 +770,7 @@ class Schema(Record, CanCurate, TracksRun):
         """
         if not isinstance(field, FieldAttr):
             raise TypeError(
-                "Argument `field` must be a Record field, e.g., `Feature.name`"
+                "Argument `field` must be a SQLRecord field, e.g., `Feature.name`"
             )
         if len(values) == 0:
             raise ValueError("Provide a list of at least one value")
@@ -815,8 +813,8 @@ class Schema(Record, CanCurate, TracksRun):
         field: FieldAttr = Feature.name,
         name: str | None = None,
         mute: bool = False,
-        organism: Record | str | None = None,
-        source: Record | None = None,
+        organism: SQLRecord | str | None = None,
+        source: SQLRecord | None = None,
     ) -> Schema | None:
         """Create schema for valid columns."""
         registry = field.field.model
@@ -1147,7 +1145,7 @@ def _get_related_name(self: Schema) -> str:
     return related_name
 
 
-class SchemaFeature(BasicRecord, LinkORM):
+class SchemaFeature(BaseSQLRecord, IsLink):
     id: int = models.BigAutoField(primary_key=True)
     schema: Schema = ForeignKey(Schema, CASCADE, related_name="links_feature")
     feature: Feature = ForeignKey(Feature, PROTECT, related_name="links_schema")
@@ -1156,16 +1154,7 @@ class SchemaFeature(BasicRecord, LinkORM):
         unique_together = ("schema", "feature")
 
 
-class SchemaParam(BasicRecord, LinkORM):
-    id: int = models.BigAutoField(primary_key=True)
-    schema: Schema = ForeignKey(Schema, CASCADE, related_name="+")
-    param: Param = ForeignKey(Param, PROTECT, related_name="+")
-
-    class Meta:
-        unique_together = ("schema", "param")
-
-
-class ArtifactSchema(BasicRecord, LinkORM, TracksRun):
+class ArtifactSchema(BaseSQLRecord, IsLink, TracksRun):
     id: int = models.BigAutoField(primary_key=True)
     artifact: Artifact = ForeignKey("Artifact", CASCADE, related_name="_links_schema")
     schema: Schema = ForeignKey(Schema, PROTECT, related_name="_links_artifact")
@@ -1176,7 +1165,7 @@ class ArtifactSchema(BasicRecord, LinkORM, TracksRun):
         unique_together = (("artifact", "schema"), ("artifact", "slot"))
 
 
-class SchemaComponent(BasicRecord, LinkORM, TracksRun):
+class SchemaComponent(BaseSQLRecord, IsLink, TracksRun):
     id: int = models.BigAutoField(primary_key=True)
     composite: Schema = ForeignKey(Schema, CASCADE, related_name="links_composite")
     component: Schema = ForeignKey(Schema, PROTECT, related_name="links_component")