lamindb 1.1.1__py3-none-any.whl → 1.2.0__py3-none-any.whl

lamindb/curators/__init__.py

@@ -9,12 +9,23 @@
    DataFrameCurator
    AnnDataCurator
 
+CatManager:
+
+.. autosummary::
+   :toctree: .
+
+   CatManager
+   DataFrameCatManager
+   AnnDataCatManager
+   MuDataCatManager
+   TiledbsomaCatManager
+   CurateLookup
+
 """
 
 from __future__ import annotations
 
 import copy
-import random
 import re
 from importlib import resources
 from itertools import chain
@@ -40,14 +51,10 @@ if TYPE_CHECKING:
 
     from lamindb.base.types import FieldAttr
     from lamindb.models import Record
-from lamindb._feature import parse_dtype, parse_dtype_single_cat
 from lamindb.base.types import FieldAttr  # noqa
-from lamindb.core._data import add_labels
-from lamindb.core._feature_manager import parse_staged_feature_sets_from_anndata
 from lamindb.core._settings import settings
 from lamindb.models import (
     Artifact,
-    CanCurate,
     Collection,
     Feature,
     Record,
@@ -55,9 +62,11 @@ from lamindb.models import (
     Schema,
     ULabel,
 )
+from lamindb.models._feature_manager import parse_staged_feature_sets_from_anndata
+from lamindb.models.artifact import add_labels, data_is_anndata
+from lamindb.models.feature import parse_dtype, parse_dtype_single_cat
+from lamindb.models._from_values import _format_values
 
-from .._artifact import data_is_anndata
-from .._from_values import _format_values
 from ..errors import InvalidArgument, ValidationError
 
 if TYPE_CHECKING:
@@ -68,7 +77,7 @@ if TYPE_CHECKING:
     from mudata import MuData
     from spatialdata import SpatialData
 
-    from lamindb._query_set import RecordList
+    from lamindb.models.query_set import RecordList
 
 
 def strip_ansi_codes(text):
@@ -141,7 +150,7 @@ class CurateLookup:
                 "    → categories.alveolar_type_1_fibroblast_cell\n\n"
                 "To look up public ontologies, use .lookup(public=True)"
             )
-        else:  # pdagma: no cover
+        else:  # pragma: no cover
             return colors.warning("No fields are found!")
 
 
@@ -199,7 +208,7 @@ class Curator:
     @doc_args(VALIDATE_DOCSTRING)
     def validate(self) -> bool | str:
         """{}"""  # noqa: D415
-        pass  # pdagma: no cover
+        pass  # pragma: no cover
 
     @doc_args(SAVE_ARTIFACT_DOCSTRING)
     def save_artifact(
@@ -453,24 +462,31 @@ class AnnDataCurator(Curator):
             raise InvalidArgument("dataset must be AnnData-like.")
         if schema.otype != "AnnData":
             raise InvalidArgument("Schema otype must be 'AnnData'.")
-        self._obs_curator = DataFrameCurator(
-            self._dataset.obs, schema._get_component("obs")
-        )
-        self._var_curator = DataFrameCurator(
-            self._dataset.var.T, schema._get_component("var")
-        )
+        # TODO: also support slots other than obs and var
+        self._slots = {
+            slot: DataFrameCurator(
+                (
+                    self._dataset.__getattribute__(slot).T
+                    if slot == "var"
+                    else self._dataset.__getattribute__(slot)
+                ),
+                slot_schema,
+            )
+            for slot, slot_schema in schema.slots.items()
+            if slot in {"obs", "var"}
+        }
 
     @property
     @doc_args(SLOTS_DOCSTRING)
     def slots(self) -> dict[str, DataFrameCurator]:
         """{}"""  # noqa: D415
-        return {"obs": self._obs_curator, "var": self._var_curator}
+        return self._slots
 
     @doc_args(VALIDATE_DOCSTRING)
     def validate(self) -> None:
         """{}"""  # noqa: D415
-        self._obs_curator.validate()
-        self._var_curator.validate()
+        for _, curator in self._slots.items():
+            curator.validate()
 
     @doc_args(SAVE_ARTIFACT_DOCSTRING)
     def save_artifact(
@@ -483,13 +499,18 @@ class AnnDataCurator(Curator):
     ):
         """{}"""  # noqa: D415
         if not self._is_validated:
-            self.validate()  # raises ValidationError if doesn't validate
-        result = parse_dtype_single_cat(self._var_curator._schema.itype, is_itype=True)
+            self.validate()
         return save_artifact(  # type: ignore
             self._dataset,
             description=description,
-            fields=self._obs_curator._cat_manager.categoricals,
-            columns_field=result["field"],
+            fields=self.slots["obs"]._cat_manager.categoricals,
+            columns_field=(
+                parse_dtype_single_cat(self.slots["var"]._schema.itype, is_itype=True)[
+                    "field"
+                ]
+                if "var" in self._slots
+                else None
+            ),
             key=key,
             artifact=self._artifact,
             revises=revises,
@@ -519,8 +540,8 @@ class CatManager:
 
     If you find non-validated values, you have several options:
 
-    - new values found in the data can be registered using :meth:`~lamindb.core.DataFrameCatManager.add_new_from`
-    - non-validated values can be accessed using :meth:`~lamindb.core.DataFrameCatManager.non_validated` and addressed manually
+    - new values found in the data can be registered using :meth:`~lamindb.curators.DataFrameCatManager.add_new_from`
+    - non-validated values can be accessed using :meth:`~lamindb.curators.DataFrameCatManager.non_validated` and addressed manually
     """
 
     def __init__(
@@ -599,7 +620,7 @@ class CatManager:
         Returns:
             None
         """
-        pass  # pdagma: no cover
+        pass  # pragma: no cover
 
     @doc_args(SAVE_ARTIFACT_DOCSTRING)
     def save_artifact(
@@ -891,7 +912,7 @@ class AnnDataCatManager(CatManager):
     def __init__(
         self,
         data: ad.AnnData | Artifact,
-        var_index: FieldAttr,
+        var_index: FieldAttr | None = None,
         categoricals: dict[str, FieldAttr] | None = None,
         obs_columns: FieldAttr = Feature.name,
         verbosity: str = "hint",
@@ -960,15 +981,16 @@ class AnnDataCatManager(CatManager):
         validated_only: bool = True,
     ):
         """Save variable records."""
-        update_registry(
-            values=list(self._adata.var.index),
-            field=self.var_index,
-            key="var_index",
-            validated_only=validated_only,
-            organism=self._organism,
-            source=self._sources.get("var_index"),
-            exclude=self._exclude.get("var_index"),
-        )
+        if self.var_index is not None:
+            update_registry(
+                values=list(self._adata.var.index),
+                field=self.var_index,
+                key="var_index",
+                validated_only=validated_only,
+                organism=self._organism,
+                source=self._sources.get("var_index"),
+                exclude=self._exclude.get("var_index"),
+            )
 
     def add_new_from(self, key: str, **kwargs):
         """Add validated & new categories.
@@ -1004,15 +1026,19 @@ class AnnDataCatManager(CatManager):
 
         # add all validated records to the current instance
         self._save_from_var_index(validated_only=True)
-        validated_var, non_validated_var = validate_categories(
-            self._adata.var.index,
-            field=self._var_field,
-            key="var_index",
-            source=self._sources.get("var_index"),
-            hint_print=".add_new_from_var_index()",
-            exclude=self._exclude.get("var_index"),
-            organism=self._organism,  # type: ignore
-        )
+        if self.var_index is not None:
+            validated_var, non_validated_var = validate_categories(
+                self._adata.var.index,
+                field=self._var_field,
+                key="var_index",
+                source=self._sources.get("var_index"),
+                hint_print=".add_new_from_var_index()",
+                exclude=self._exclude.get("var_index"),
+                organism=self._organism,  # type: ignore
+            )
+        else:
+            validated_var = True
+            non_validated_var = []
         validated_obs = self._obs_df_curator.validate()
         self._non_validated = self._obs_df_curator._non_validated  # type: ignore
         if len(non_validated_var) > 0:
@@ -1711,7 +1737,7 @@ class TiledbsomaCatManager(CatManager):
         Returns:
             A saved artifact record.
         """
-        from lamindb.core._data import add_labels
+        from lamindb.models.artifact import add_labels
 
         if not self._is_validated:
             self.validate()
@@ -1848,11 +1874,11 @@ class SpatialDataCatManager(CatManager):
             exclude=exclude,
         )
         if isinstance(sdata, Artifact):
-            # TODO: load() doesn't yet work
             self._sdata = sdata.load()
         else:
             self._sdata = self._dataset
         self._sample_metadata_key = sample_metadata_key
+        self._write_path = None
         self._var_fields = var_index
         self._verify_accessor_exists(self._var_fields.keys())
         self._categoricals = categoricals
@@ -2134,26 +2160,14 @@ class SpatialDataCatManager(CatManager):
         try:
             settings.verbosity = "warning"
 
-            if self._artifact is None:
-                # Write the SpatialData object to a random path in tmp directory
-                # The Artifact constructor will move it to the cache
-                write_path = (
-                    f"{settings.cache_dir}/{random.randint(10**7, 10**8 - 1)}.zarr"
-                )
-                self._sdata.write(write_path)
-
-                # Create the Artifact and associate Artifact metadata
-                self._artifact = Artifact(
-                    write_path,
-                    description=description,
-                    key=key,
-                    revises=revises,
-                    run=run,
-                )
-                # According to Tim it is not easy to calculate the number of observations.
-                # We would have to write custom code to iterate over labels (which might not even exist at that point)
-                self._artifact.otype = "spatialdata"
-                self._artifact.save()
+            self._artifact = Artifact.from_spatialdata(
+                self._sdata,
+                key=key,
+                description=description,
+                revises=revises,
+                run=run,
+            )
+            self._artifact.save()
 
             # Link schemas
             feature_kwargs = check_registry_organism(
@@ -2171,7 +2185,7 @@ class SpatialDataCatManager(CatManager):
                 """Add Schemas from SpatialData."""
                 if obs_fields is None:
                     obs_fields = {}
-                assert host.otype == "spatialdata"  # noqa: S101
+                assert host.otype == "SpatialData"  # noqa: S101
 
                 feature_sets = {}
 
@@ -2799,7 +2813,7 @@ class DoseHandler:
         return cls.UNIT_MAP.get(unit, unit)
 
     @classmethod
-    def validate_values(cls, values: pd.Series) -> list:
+    def validate_values(cls, values: pd.Series) -> list[str]:
         """Validate pert_dose values with strict case checking."""
         errors = []
 
@@ -2843,7 +2857,7 @@ class TimeHandler:
         return unit[0].lower()
 
     @classmethod
-    def validate_values(cls, values: pd.Series) -> list:
+    def validate_values(cls, values: pd.Series) -> list[str]:
         """Validate pert_time values."""
         errors = []
 
@@ -3197,8 +3211,8 @@ def validate_categories(
     exclude: str | list | None = None,
     hint_print: str | None = None,
     curator: CatManager | None = None,
-) -> tuple[bool, list]:
-    """Validate ontology terms in a pandas series using LaminDB registries.
+) -> tuple[bool, list[str]]:
+    """Validate ontology terms using LaminDB registries.
 
     Args:
         values: The values to validate.
@@ -3210,8 +3224,8 @@ def validate_categories(
         standardize: Whether to standardize the values.
         hint_print: The hint to print that suggests fixing non-validated values.
     """
-    from lamindb._from_values import _format_values
     from lamindb.core._settings import settings
+    from lamindb.models._from_values import _format_values
 
     model_field = f"{field.field.model.__name__}.{field.field.name}"
 
@@ -3346,7 +3360,7 @@ def validate_categories_in_df(
 def save_artifact(
     data: pd.DataFrame | ad.AnnData | MuData,
     fields: dict[str, FieldAttr] | dict[str, dict[str, FieldAttr]],
-    columns_field: FieldAttr | dict[str, FieldAttr],
+    columns_field: FieldAttr | dict[str, FieldAttr] | None = None,
     description: str | None = None,
     organism: str | None = None,
     key: str | None = None,
@@ -3372,8 +3386,7 @@ def save_artifact(
     Returns:
         The saved Artifact.
     """
-    from .._artifact import data_is_anndata, data_is_mudata
-    from ..core._data import add_labels
+    from ..models.artifact import add_labels, data_is_anndata, data_is_mudata
 
     if artifact is None:
         if data_is_anndata(data):
@@ -3395,7 +3408,7 @@ def save_artifact(
     artifact.schema = schema
     artifact.save()
 
-    if organism is not None:
+    if organism is not None and columns_field is not None:
         feature_kwargs = check_registry_organism(
             (
                 list(columns_field.values())[0].field.model
@@ -3532,8 +3545,8 @@ def update_registry(
         exclude: Values to exclude from inspect.
         kwargs: Additional keyword arguments to pass to the registry model to create new records.
     """
-    from lamindb._save import save as ln_save
     from lamindb.core._settings import settings
+    from lamindb.models.save import save as ln_save
 
     registry = field.field.model
     filter_kwargs = check_registry_organism(registry, organism)
@@ -3621,7 +3634,7 @@ def log_saved_labels(
     validated_only: bool = True,
 ) -> None:
     """Log the saved labels."""
-    from .._from_values import _format_values
+    from ..models._from_values import _format_values
 
     model_field = colors.italic(model_field)
     for k, labels in labels_saved.items():
@@ -3667,12 +3680,14 @@ def _save_organism(name: str):
     return organism
 
 
-def _ref_is_name(field: FieldAttr) -> bool | None:
+def _ref_is_name(field: FieldAttr | None) -> bool | None:
     """Check if the reference field is a name field."""
-    from .._can_curate import get_name_field
+    from ..models.can_curate import get_name_field
 
-    name_field = get_name_field(field.field.model)
-    return field.field.name == name_field
+    if field is not None:
+        name_field = get_name_field(field.field.model)
+        return field.field.name == name_field
+    return None
 
 
 # backward compat constructors ------------------
@@ -3721,7 +3736,7 @@ def from_anndata(
 @classmethod  # type: ignore
 def from_mudata(
     cls,
-    mdata: MuData,
+    mdata: MuData | UPathStr,
     var_index: dict[str, dict[str, FieldAttr]],
     categoricals: dict[str, FieldAttr] | None = None,
     verbosity: str = "hint",
@@ -3761,7 +3776,7 @@ def from_tiledbsoma(
 @classmethod  # type: ignore
 def from_spatialdata(
     cls,
-    sdata,
+    sdata: SpatialData | UPathStr,
     var_index: dict[str, FieldAttr],
     categoricals: dict[str, dict[str, FieldAttr]] | None = None,
     organism: str | None = None,

lamindb/errors.py

@@ -1,4 +1,4 @@
-"""Exceptions.
+"""Errors.
 
 .. autosummary::
    :toctree: .

lamindb/integrations/_vitessce.py

@@ -7,10 +7,10 @@ from typing import TYPE_CHECKING
 import lamindb_setup as ln_setup
 from lamin_utils import logger
 
-from lamindb._artifact import Artifact
-from lamindb._collection import Collection
-from lamindb._run import Run
-from lamindb._transform import Transform
+from lamindb.models.artifact import Artifact
+from lamindb.models.collection import Collection
+from lamindb.models.run import Run
+from lamindb.models.transform import Transform
 
 if TYPE_CHECKING:
     from vitessce import VitessceConfig

lamindb/migrations/0089_subsequent_runs.py

@@ -0,0 +1,159 @@
+# ruff: noqa: S608
+from django.db import migrations, models
+
+
+def update_model_run_relationships(apps, schema_editor, model_name):
+    vendor = schema_editor.connection.vendor
+
+    # Define table names based on model_name
+    model_table = f"lamindb_{model_name}"
+    link_table = f"lamindb_{model_name}__previous_runs"
+    model_id_field = f"{model_name}_id"
+
+    with schema_editor.connection.cursor() as cursor:
+        # Step 1: Add the current run_id to the _previous_runs table if it doesn't exist
+        cursor.execute(f"""
+            INSERT INTO {link_table} ({model_id_field}, run_id)
+            SELECT a.id, a.run_id
+            FROM {model_table} a
+            WHERE a.run_id IS NOT NULL
+            AND NOT EXISTS (
+                SELECT 1
+                FROM {link_table} apr
+                WHERE apr.{model_id_field} = a.id
+                AND apr.run_id = a.run_id
+            );
+        """)
+
+        # Step 2: For each model, find the earliest run (lowest ID) and set it as the run_id
+        if vendor == "sqlite":
+            cursor.execute(f"""
+                UPDATE {model_table}
+                SET run_id = (
+                    SELECT MIN(r.id)
+                    FROM lamindb_run r
+                    JOIN {link_table} apr ON r.id = apr.run_id
+                    WHERE apr.{model_id_field} = {model_table}.id
+                )
+                WHERE EXISTS (
+                    SELECT 1
+                    FROM {link_table} apr
+                    WHERE apr.{model_id_field} = {model_table}.id
+                );
+            """)
+        else:  # PostgreSQL
+            cursor.execute(f"""
+                UPDATE {model_table} AS a
+                SET run_id = subquery.min_run_id
+                FROM (
+                    SELECT {model_id_field}, MIN(run_id) as min_run_id
+                    FROM {link_table}
+                    GROUP BY {model_id_field}
+                ) AS subquery
+                WHERE a.id = subquery.{model_id_field}
+                AND EXISTS (
+                    SELECT 1
+                    FROM {link_table} apr
+                    WHERE apr.{model_id_field} = a.id
+                );
+            """)
+
+        # Step 3: Remove the earliest run from the link table
+        if vendor == "sqlite":
+            cursor.execute(f"""
+                DELETE FROM {link_table}
+                WHERE EXISTS (
+                    SELECT 1 FROM {model_table} a
+                    WHERE {link_table}.{model_id_field} = a.id
+                    AND {link_table}.run_id = a.run_id
+                );
+            """)
+        else:  # PostgreSQL
+            cursor.execute(f"""
+                DELETE FROM {link_table} AS apr
+                USING {model_table} AS a
+                WHERE apr.{model_id_field} = a.id
+                AND apr.run_id = a.run_id;
+            """)
+
+
+def update_artifact_run_relationships(apps, schema_editor):
+    """Migration function for artifacts."""
+    update_model_run_relationships(apps, schema_editor, "artifact")
+
+
+def update_collection_run_relationships(apps, schema_editor):
+    """Migration function for collections."""
+    update_model_run_relationships(apps, schema_editor, "collection")
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("lamindb", "0088_schema_components"),
+    ]
+
+    operations = [
+        # unrelated to subsequent runs, but related to lamindb 1.2
+        # update the otype field in the artifact table
+        migrations.RunSQL(
+            sql="""
+            UPDATE lamindb_artifact
+            SET otype = 'SpatialData'
+            WHERE otype = 'spatialdata';
+            """
+        ),
+        # Migrate artifact relationships
+        migrations.RunPython(
+            update_artifact_run_relationships, migrations.RunPython.noop
+        ),
+        # Update artifact model state
+        migrations.SeparateDatabaseAndState(
+            # Database operations (none, to keep tables intact)
+            [],
+            # State operations (to update Django model only)
+            [
+                # Remove the old field from model state
+                migrations.RemoveField(
+                    model_name="artifact",
+                    name="_previous_runs",
+                ),
+                # Add the new field with the same underlying table
+                migrations.AddField(
+                    model_name="artifact",
+                    name="_subsequent_runs",
+                    field=models.ManyToManyField(
+                        "lamindb.run",
+                        related_name="_recreated_artifacts",
+                        db_table="lamindb_artifact__previous_runs",  # Keep the original table name
+                    ),
+                ),
+            ],
+        ),
+        # Migrate collection relationships
+        migrations.RunPython(
+            update_collection_run_relationships, migrations.RunPython.noop
+        ),
+        # Update collection model state
+        migrations.SeparateDatabaseAndState(
+            # Database operations (none, to keep tables intact)
+            [],
+            # State operations (to update Django model only)
+            [
+                # Remove the old field from model state
+                migrations.RemoveField(
+                    model_name="collection",
+                    name="_previous_runs",
+                ),
+                # Add the new field with the same underlying table
+                migrations.AddField(
+                    model_name="collection",
+                    name="_subsequent_runs",
+                    field=models.ManyToManyField(
+                        "lamindb.run",
+                        related_name="_recreated_collections",
+                        db_table="lamindb_collection__previous_runs",  # Keep the original table name
+                    ),
+                ),
+            ],
+        ),
+    ]

lamindb/migrations/0090_runproject_project_runs.py

@@ -0,0 +1,73 @@
+# Generated by Django 5.2 on 2025-03-05 10:20
+
+import django.db.models.deletion
+import django.db.models.functions.datetime
+from django.db import migrations, models
+
+import lamindb.base.fields
+import lamindb.base.users
+import lamindb.models.record
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("lamindb", "0089_subsequent_runs"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="RunProject",
+            fields=[
+                ("id", models.BigAutoField(primary_key=True, serialize=False)),
+                (
+                    "created_at",
+                    lamindb.base.fields.DateTimeField(
+                        blank=True,
+                        db_default=django.db.models.functions.datetime.Now(),
+                        db_index=True,
+                        editable=False,
+                    ),
+                ),
+                (
+                    "created_by",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        default=lamindb.base.users.current_user_id,
+                        editable=False,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="+",
+                        to="lamindb.user",
+                    ),
+                ),
+                (
+                    "project",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="links_run",
+                        to="lamindb.project",
+                    ),
+                ),
+                (
+                    "run",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="links_project",
+                        to="lamindb.run",
+                    ),
+                ),
+            ],
+            options={
+                "unique_together": {("run", "project")},
+            },
+            bases=(models.Model, lamindb.models.record.LinkORM),
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="runs",
+            field=models.ManyToManyField(
+                related_name="projects", through="lamindb.RunProject", to="lamindb.run"
+            ),
+        ),
+    ]