lamindb 1.6.2__py3-none-any.whl → 1.7.0__py3-none-any.whl

lamindb/models/record.py

@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, overload
 
 from django.db import models
 from django.db.models import CASCADE, PROTECT
+from lamin_utils import logger
 
 from lamindb.base.fields import (
     BooleanField,
@@ -13,21 +14,29 @@ from lamindb.base.fields import (
 )
 from lamindb.errors import FieldValidationError
 
-from ..base.ids import base62_12, base62_16
+from ..base.ids import base62_16
 from .artifact import Artifact
 from .can_curate import CanCurate
 from .feature import Feature
+from .has_parents import _query_relatives
+from .query_set import reorder_subset_columns_in_df
 from .run import Run, TracksRun, TracksUpdates
 from .sqlrecord import BaseSQLRecord, IsLink, SQLRecord, _get_record_kwargs
+from .transform import Transform
 from .ulabel import ULabel
 
 if TYPE_CHECKING:
-    from .project import Project
+    import pandas as pd
+
+    from .project import Person, Project, Reference
+    from .query_set import QuerySet
     from .schema import Schema
 
 
 class Record(SQLRecord, CanCurate, TracksRun, TracksUpdates):
-    """Flexible records to register, e.g., samples, donors, cells, compounds, sequences.
+    """Flexible records as you find them in Excel-like sheets.
+
+    Useful register, e.g., samples, donors, cells, compounds, sequences.
 
     This is currently more convenient to use through the UI.
 
@@ -39,12 +48,8 @@ class Record(SQLRecord, CanCurate, TracksRun, TracksUpdates):
         description: `str` A description.
 
     See Also:
-        :meth:`~lamindb.Sheet`
-            Sheets to group records.
         :meth:`~lamindb.Feature`
-            Dimensions of measurement.
-        :attr:`~lamindb.Artifact.features`
-            Feature manager for an artifact.
+            Dimensions of measurement (e.g. column of a sheet).
     """
 
     class Meta(SQLRecord.Meta, TracksRun.Meta, TracksUpdates.Meta):
@@ -67,11 +72,20 @@ class Record(SQLRecord, CanCurate, TracksRun, TracksUpdates):
     """
     records: Record
     """Records of this type (can only be non-empty if `is_type` is `True`)."""
-    is_type: bool = BooleanField(default=False, db_index=True, null=True)
-    """Distinguish types from instances of the type.
+    is_type: bool = BooleanField(default=False, db_index=True)
+    """Indicates if record is a `type`.
 
     For example, if a record "Compound" is a `type`, the actual compounds "darerinib", "tramerinib", would be instances of that `type`.
     """
+    schema: Schema | None = ForeignKey(
+        "Schema", CASCADE, null=True, related_name="records"
+    )
+    """A schema to enforce for a type (optional).
+
+    This is mostly parallel to the `schema` attribute of `Artifact`.
+
+    If `is_type` is `True`, the schema is used to enforce certain features for each records of this type.
+    """
     # naming convention in analogy with Schema
     components: Record = models.ManyToManyField(
         "Record", through="RecordRecord", symmetrical=False, related_name="composites"
@@ -79,26 +93,43 @@ class Record(SQLRecord, CanCurate, TracksRun, TracksUpdates):
     """Record-like components of this record."""
     composites: Record
     """Record-like composites of this record."""
-    sheet: Sheet | None = ForeignKey(
-        "Sheet", CASCADE, null=True, related_name="records"
-    )
-    """Group records by sheet."""
     description: str | None = CharField(null=True)
     """A description (optional)."""
-    artifacts: Artifact = models.ManyToManyField(
-        Artifact, through="RecordArtifact", related_name="records"
+    linked_artifacts: Artifact = models.ManyToManyField(
+        Artifact, through="RecordArtifact", related_name="linked_in_records"
     )
     """Linked artifacts."""
-    runs: Run = models.ManyToManyField(Run, through="RecordRun", related_name="records")
+    artifacts: Artifact = models.ManyToManyField(
+        Artifact, through="ArtifactRecord", related_name="records"
+    )
+    """Annotated artifacts."""
+    linked_runs: Run = models.ManyToManyField(
+        Run, through="RecordRun", related_name="records"
+    )
     """Linked runs."""
+    run: Run | None = ForeignKey(
+        Run,
+        PROTECT,
+        related_name="output_records",
+        null=True,
+        default=None,
+        editable=False,
+    )
+    """Run that created the record."""
+    input_of_runs: Run = models.ManyToManyField(Run, related_name="input_records")
+    """Runs that use this record as an input."""
     ulabels: ULabel = models.ManyToManyField(
         ULabel,
         through="RecordULabel",
         related_name="_records",  # in transition period
     )
     """Linked runs."""
-    projects: Project
+    linked_projects: Project
     """Linked projects."""
+    linked_references: Reference
+    """Linked references."""
+    linked_people: Person
+    """Linked people."""
 
     @overload
     def __init__(
@@ -128,10 +159,14 @@ class Record(SQLRecord, CanCurate, TracksRun, TracksUpdates):
         name: str = kwargs.pop("name", None)
         type: str | None = kwargs.pop("type", None)
         is_type: bool = kwargs.pop("is_type", False)
-        sheet: Sheet = kwargs.pop("sheet", None)
         description: str | None = kwargs.pop("description", None)
+        schema = kwargs.pop("schema", None)
+        branch = kwargs.pop("branch", None)
+        branch_id = kwargs.pop("branch_id", 1)
+        space = kwargs.pop("space", None)
+        space_id = kwargs.pop("space_id", 1)
         _skip_validation = kwargs.pop(
-            "_skip_validation", True
+            "_skip_validation", False
         )  # should not validate records
         _aux = kwargs.pop("_aux", None)
         if len(kwargs) > 0:
@@ -139,90 +174,85 @@ class Record(SQLRecord, CanCurate, TracksRun, TracksUpdates):
             raise FieldValidationError(
                 f"Only {valid_keywords} are valid keyword arguments"
             )
+        if schema and not is_type:
+            logger.important("passing schema, treating as type")
+            is_type = True
         super().__init__(
             name=name,
             type=type,
             is_type=is_type,
-            sheet=sheet,
             description=description,
+            schema=schema,
+            branch=branch,
+            branch_id=branch_id,
+            space=space,
+            space_id=space_id,
             _skip_validation=_skip_validation,
             _aux=_aux,
         )
 
-
-class Sheet(SQLRecord, TracksRun, TracksUpdates):
-    """Sheets to group records."""
-
-    class Meta(SQLRecord.Meta, TracksRun.Meta, TracksUpdates.Meta):
-        abstract = False
-
-    id: int = models.AutoField(primary_key=True)
-    uid: str = CharField(
-        editable=False, unique=True, db_index=True, max_length=12, default=base62_12
-    )
-    """A universal random id, valid across DB instances."""
-    name: str = CharField(db_index=True)
-    """Name or title of sheet."""
-    schema: Schema | None = ForeignKey(
-        "Schema", CASCADE, null=True, related_name="sheets"
-    )
-    """A schema to enforce for the sheet (optional)."""
-    description: str | None = CharField(null=True, db_index=True)
-    """A description (optional)."""
-    projects: Project
-    """Linked projects."""
-
-    @overload
-    def __init__(
-        self,
-        name: str,
-        schema: Schema | None = None,
-        description: str | None = None,
-    ): ...
-
-    @overload
-    def __init__(
-        self,
-        *db_args,
-    ): ...
-
-    def __init__(
-        self,
-        *args,
-        **kwargs,
-    ):
-        if len(args) == len(self._meta.concrete_fields):
-            super().__init__(*args, **kwargs)
-            return None
-        if len(args) > 0:
-            raise ValueError("Only one non-keyword arg allowed")
-        name: str = kwargs.pop("name", None)
-        schema: Schema | None = kwargs.pop("schema", None)
-        description: str | None = kwargs.pop("description", None)
-        _skip_validation = kwargs.pop("_skip_validation", True)
-        _aux = kwargs.pop("_aux", None)
-        if len(kwargs) > 0:
-            valid_keywords = ", ".join([val[0] for val in _get_record_kwargs(Record)])
-            raise FieldValidationError(
-                f"Only {valid_keywords} are valid keyword arguments"
-            )
-        super().__init__(
-            name=name,
-            schema=schema,
-            description=description,
-            _skip_validation=_skip_validation,
-            _aux=_aux,
+    @property
+    def is_form(self) -> bool:
+        """Check if record is a form (a record type with a validating schema)."""
+        return self.schema is not None and self.is_type
+
+    def query_children(self) -> QuerySet:
+        """Query all children of a record type recursively.
+
+        While `.records` retrieves the direct children, this method
+        retrieves all descendants of a record type.
+        """
+        return _query_relatives([self], "records", self.__class__)  # type: ignore
+
+    def to_pandas(self) -> pd.DataFrame:
+        """Export all children of a record type recursively to a pandas DataFrame."""
+        assert self.is_type, "Only types can be exported as dataframes"  # noqa: S101
+        df = self.query_children().df(features="queryset")
+        df.columns.values[0] = "__lamindb_record_uid__"
+        df.columns.values[1] = "__lamindb_record_name__"
+        if self.schema is not None:
+            desired_order = self.schema.members.list("name")  # only members is ordered!
+        else:
+            # sort alphabetically for now
+            desired_order = df.columns[2:].tolist()
+            desired_order.sort()
+        df = reorder_subset_columns_in_df(df, desired_order, position=0)  # type: ignore
+        return df.sort_index()  # order by id for now
+
+    def to_artifact(self, key: str = None) -> Artifact:
+        """Export all children of a record type as a `.csv` artifact."""
+        from lamindb.core._context import context
+
+        assert self.is_type, "Only types can be exported as artifacts"  # noqa: S101
+        if key is None:
+            file_suffix = ".csv"
+            key = f"sheet_exports/{self.name}{file_suffix}"
+        description = f": {self.description}" if self.description is not None else ""
+        format: dict[str, Any] = {"suffix": ".csv"} if key.endswith(".csv") else {}
+        format["index"] = False
+        transform, _ = Transform.objects.get_or_create(
+            key="__lamindb_record_export__", type="function"
         )
+        run = Run(transform, initiated_by_run=context.run).save()
+        run.input_records.add(self)
+        return Artifact.from_df(
+            self.to_pandas(),
+            key=key,
+            description=f"Export of sheet {self.uid}{description}",
+            schema=self.schema,
+            format=format,
+            run=run,
+        ).save()
 
 
 class RecordJson(BaseSQLRecord, IsLink):
     id: int = models.BigAutoField(primary_key=True)
     record: Record = ForeignKey(Record, CASCADE, related_name="values_json")
-    feature: Feature = ForeignKey(Feature, CASCADE, related_name="links_recordjson")
+    feature: Feature = ForeignKey(Feature, PROTECT, related_name="links_recordjson")
     value: Any = JSONField(default=None, db_default=None)
 
     class Meta:
-        unique_together = ("record", "feature")
+        unique_together = ("record", "feature")  # a list is modeled as a list in json
 
 
 class RecordRecord(SQLRecord, IsLink):
@@ -230,43 +260,59 @@ class RecordRecord(SQLRecord, IsLink):
     record: Record = ForeignKey(
         Record, CASCADE, related_name="values_record"
     )  # composite
-    feature: Feature = ForeignKey(Feature, CASCADE, related_name="links_recordrecord")
+    feature: Feature = ForeignKey(Feature, PROTECT, related_name="links_recordrecord")
     value: Record = ForeignKey(
         Record, PROTECT, related_name="links_record"
     )  # component
 
     class Meta:
-        unique_together = ("record", "feature")
+        unique_together = ("record", "feature", "value")
 
 
 class RecordULabel(BaseSQLRecord, IsLink):
     id: int = models.BigAutoField(primary_key=True)
     record: Record = ForeignKey(Record, CASCADE, related_name="values_ulabel")
-    feature: Feature = ForeignKey(Feature, CASCADE, related_name="links_recordulabel")
+    feature: Feature = ForeignKey(Feature, PROTECT, related_name="links_recordulabel")
     value: ULabel = ForeignKey(ULabel, PROTECT, related_name="links_record")
 
     class Meta:
         # allows linking exactly one record to one ulabel per feature, because we likely don't want to have Many
-        unique_together = ("record", "feature")
+        unique_together = ("record", "feature", "value")
 
 
 class RecordRun(BaseSQLRecord, IsLink):
     id: int = models.BigAutoField(primary_key=True)
     record: Record = ForeignKey(Record, CASCADE, related_name="values_run")
-    feature: Feature = ForeignKey(Feature, CASCADE, related_name="links_recordrun")
+    feature: Feature = ForeignKey(Feature, PROTECT, related_name="links_recordrun")
     value: Run = ForeignKey(Run, PROTECT, related_name="links_record")
 
     class Meta:
         # allows linking several records to a single run for the same feature because we'll likely need this
-        unique_together = ("record", "feature")
+        unique_together = ("record", "feature", "value")
 
 
 class RecordArtifact(BaseSQLRecord, IsLink):
     id: int = models.BigAutoField(primary_key=True)
     record: Record = ForeignKey(Record, CASCADE, related_name="values_artifact")
-    feature: Feature = ForeignKey(Feature, CASCADE, related_name="links_recordartifact")
-    value: Artifact = ForeignKey(Artifact, PROTECT, related_name="links_record")
+    feature: Feature = ForeignKey(Feature, PROTECT, related_name="links_recordartifact")
+    value: Artifact = ForeignKey(Artifact, PROTECT, related_name="links_in_record")
 
     class Meta:
         # allows linking several records to a single artifact for the same feature because we'll likely need this
         unique_together = ("record", "feature", "value")
+
+
+# like ArtifactULabel, for annotation
+class ArtifactRecord(BaseSQLRecord, IsLink):
+    id: int = models.BigAutoField(primary_key=True)
+    artifact: Artifact = ForeignKey(Artifact, CASCADE, related_name="links_record")
+    record: Record = ForeignKey(Record, PROTECT, related_name="links_artifact")
+    feature: Feature = ForeignKey(
+        Feature, PROTECT, null=True, related_name="links_artifactrecord"
+    )
+    label_ref_is_name: bool | None = BooleanField(null=True)
+    feature_ref_is_name: bool | None = BooleanField(null=True)
+
+    class Meta:
+        # allows linking several records to a single artifact for the same feature because we'll likely need this
+        unique_together = ("artifact", "record", "feature")

lamindb/models/run.py

@@ -27,11 +27,13 @@ from .sqlrecord import BaseSQLRecord, IsLink, SQLRecord
 if TYPE_CHECKING:
     from datetime import datetime
 
+    from ._feature_manager import FeatureManager
     from .artifact import Artifact
     from .collection import Collection
     from .feature import FeatureValue
     from .project import Project
     from .query_set import QuerySet
+    from .record import Record
     from .transform import Transform
     from .ulabel import ULabel
 
@@ -39,18 +41,6 @@ if TYPE_CHECKING:
 _TRACKING_READY: bool | None = None
 
 
-class FeatureManager:
-    """Feature manager."""
-
-    pass
-
-
-class FeatureManagerRun(FeatureManager):
-    """Feature manager."""
-
-    pass
-
-
 def current_run() -> Run | None:
     global _TRACKING_READY
 
@@ -206,12 +196,13 @@ class Run(SQLRecord):
 
     Args:
         transform: `Transform` A :class:`~lamindb.Transform` record.
+        initiated_by_run: `Run | None = None` The run that triggers this run.
         reference: `str | None = None` For instance, an external ID or a download URL.
         reference_type: `str | None = None` For instance, `redun_id`, `nextflow_id` or `url`.
 
     See Also:
-        :meth:`~lamindb.core.Context.track`
-            Track global runs & transforms for a notebook or script.
+        :func:`~lamindb.track`
+            Globally track a script or notebook run.
 
     Examples:
 
@@ -234,26 +225,6 @@ class Run(SQLRecord):
 
     _name_field: str = "started_at"
 
-    features: FeatureManager = FeatureManagerRun  # type: ignore
-    """Features manager.
-
-    Run parameters are tracked via the `Feature` registry, just like all other variables.
-
-    Guide: :ref:`track-run-parameters`
-
-    Example::
-
-        run.features.add_values({
-            "learning_rate": 0.01,
-            "input_dir": "s3://my-bucket/mydataset",
-            "downsample": True,
-            "preprocess_params": {
-                "normalization_type": "cool",
-                "subset_highlyvariable": True,
-            },
-        })
-    """
-
     id: int = models.BigAutoField(primary_key=True)
     """Internal id, valid only in one DB instance."""
     # default uid was changed from base62_20 to base62_16 in 1.6.0
@@ -302,6 +273,10 @@ class Run(SQLRecord):
     """The collections serving as input for this run."""
     output_collections: Collection
     """The collections generated by this run."""
+    input_records: Record
+    """The collections serving as input for this run."""
+    output_records: Record
+    """The collections generated by this run."""
     """Parameter values."""
     _feature_values: FeatureValue = models.ManyToManyField(
         "FeatureValue", through="RunFeatureValue", related_name="runs"
@@ -338,14 +313,15 @@ class Run(SQLRecord):
     """Linked projects."""
     _is_consecutive: bool | None = BooleanField(null=True)
     """Indicates whether code was consecutively executed. Is relevant for notebooks."""
-    _status_code: int = models.SmallIntegerField(default=0, db_index=True)
+    _status_code: int = models.SmallIntegerField(default=None, db_index=True, null=True)
     """Status code of the run.
 
-    - 0: scheduled
-    - 1: started
-    - 2: errored
-    - 3: aborted
-    - 4: completed
+    - -3: scheduled
+    - -2: re-started
+    - -1: started
+    - 0: completed
+    - 1: errored
+    - 2: aborted
     """
 
     @overload
@@ -354,6 +330,7 @@ class Run(SQLRecord):
         transform: Transform,
         reference: str | None = None,
         reference_type: str | None = None,
+        initiated_by_run: Run | None = None,
     ): ...
 
     @overload
@@ -367,7 +344,6 @@ class Run(SQLRecord):
         *args,
         **kwargs,
     ):
-        self.features = FeatureManager(self)  # type: ignore
         if len(args) == len(self._meta.concrete_fields):
             super().__init__(*args, **kwargs)
             return None
@@ -402,6 +378,30 @@ class Run(SQLRecord):
     def params(self) -> FeatureManager:
         return self.features
 
+    @property
+    def features(self) -> FeatureManager:
+        """Features manager.
+
+        Run parameters are tracked via the `Feature` registry, just like all other variables.
+
+        Guide: :ref:`track-run-parameters`
+
+        Example::
+
+            run.features.add_values({
+                "learning_rate": 0.01,
+                "input_dir": "s3://my-bucket/mydataset",
+                "downsample": True,
+                "preprocess_params": {
+                    "normalization_type": "cool",
+                    "subset_highlyvariable": True,
+                },
+            })
+        """
+        from ._feature_manager import FeatureManager
+
+        return FeatureManager(self)
+
     @classmethod
     def filter(
         cls,
@@ -441,7 +441,7 @@ class Run(SQLRecord):
                     keys_normalized, field="name", mute=True
                 )
             ):
-                return filter_base(FeatureManagerRun, **expressions)
+                return filter_base(Run, **expressions)
             else:
                 params = ", ".join(sorted(np.array(keys_normalized)[~params_validated]))
                 message = f"feature names: {params}"