lamindb 1.10.2__py3-none-any.whl → 1.11a1__py3-none-any.whl

lamindb/examples/cellxgene/__init__.py

@@ -3,9 +3,14 @@
 .. autosummary::
    :toctree: .
 
-   save_cxg_defaults
-   get_cxg_schema
+   save_cellxgene_defaults
+   create_cellxgene_schema
 
 """
 
-from ._cellxgene import get_cxg_schema, save_cxg_defaults
+from ._cellxgene import (
+    create_cellxgene_schema,
+    get_cxg_schema,
+    save_cellxgene_defaults,
+    save_cxg_defaults,
+)

lamindb/examples/cellxgene/_cellxgene.py

@@ -3,7 +3,9 @@ from __future__ import annotations
 from typing import TYPE_CHECKING, Collection, Literal, NamedTuple
 
 import pandas as pd
+from lamindb_setup.core import deprecated
 from lamindb_setup.core.upath import UPath
+from packaging import version
 
 from lamindb.models._from_values import _format_values
 
@@ -11,11 +13,25 @@ if TYPE_CHECKING:
     from lamindb.base.types import FieldAttr
     from lamindb.models import Schema, SQLRecord
 
-CELLxGENESchemaVersions = Literal["4.0.0", "5.0.0", "5.1.0", "5.2.0", "5.3.0"]
+CELLxGENESchemaVersions = Literal["4.0.0", "5.0.0", "5.1.0", "5.2.0", "5.3.0", "6.0.0"]
+CELLxGENEOrganisms = Literal[
+    "human",
+    "mouse",
+    "zebra danio",
+    "rhesus macaquedomestic pig",
+    "chimpanzee",
+    "white-tufted-ear marmoset",
+    "sars-2",
+]
 FieldType = Literal["ontology_id", "name"]
 
 
+@deprecated(new_name="save_cellxgene_defaults")
 def save_cxg_defaults() -> None:
+    return save_cxg_defaults()
+
+
+def save_cellxgene_defaults() -> None:
     """Save default values of the CELLxGENE schema to the instance.
 
     Adds CELLxGENE specific (control) values that are not available in the ontologies:
@@ -25,7 +41,6 @@ def save_cxg_defaults() -> None:
     - "unknown" entries for DevelopmentalStage, Phenotype, and CellType
     - "tissue", "organoid", and "cell culture" ULabels (tissue_type)
     - "cell", "nucleus", "na" ULabels (suspension_type)
-
     """
     import bionty as bt
 
@@ -47,12 +62,13 @@ def save_cxg_defaults() -> None:
     # na, unknown
     for model, name in zip(
         [
+            bt.Ethnicity,
             bt.Ethnicity,
             bt.DevelopmentalStage,
             bt.Phenotype,
             bt.CellType,
         ],
-        ["na", "unknown", "unknown", "unknown"],
+        ["na", "unknown", "unknown", "unknown", "unknown"],
     ):
         model(ontology_id=name, name=name, description="From CellxGene schema.").save()
 
@@ -76,8 +92,24 @@ def save_cxg_defaults() -> None:
             name=name, type=suspension_type, description="From CellxGene schema."
         ).save()
 
+    # organisms
+    taxonomy_ids = [
+        "NCBITaxon:9606",  # Homo sapiens (Human)
+        "NCBITaxon:10090",  # Mus musculus (House mouse)
+        "NCBITaxon:9544",  # Macaca mulatta (Rhesus monkey)
+        "NCBITaxon:9825",  # Sus scrofa domesticus (Domestic pig)
+        "NCBITaxon:9598",  # Pan troglodytes (Chimpanzee)
+        "NCBITaxon:9483",  # Callithrix jacchus (White-tufted-ear marmoset)
+        "NCBITaxon:7955",  # Danio rerio (Zebrafish)
+    ]
+    for ontology_id in taxonomy_ids:
+        bt.Organism.from_source(
+            ontology_id=ontology_id,
+            source=bt.Source.get(name="ncbitaxon", currently_used=True),
+        ).save()
+
 
-def _create_cxg_sources(
+def _create_cellxgene_sources(
     categoricals: dict[str, FieldAttr], schema_version: str, organism: str
 ) -> dict[str, SQLRecord]:
     """Create a source dictionary of CELLxGENE categoricals to Source."""
@@ -105,7 +137,7 @@ def _create_cxg_sources(
                 )
             return source
 
-    sources_df = pd.read_csv(UPath(__file__).parent / "cxg_schema_versions.csv")
+    sources_df = pd.read_csv(UPath(__file__).parent / "cellxgene_schema_versions.csv")
     sources_df = sources_df[sources_df.schema_version == schema_version]
     if sources_df.empty:
         raise ValueError(
@@ -126,11 +158,28 @@ def _create_cxg_sources(
     return key_to_source
 
 
+@deprecated(new_name="create_cellxgene_schema")
 def get_cxg_schema(
     schema_version: CELLxGENESchemaVersions,
     *,
     field_types: FieldType | Collection[FieldType] = "ontology_id",
-    organism: Literal["human", "mouse"] = "human",
+    organism: CELLxGENEOrganisms = "human",
+    spatial_library_id: str | None = None,
+) -> Schema:
+    return create_cellxgene_schema(
+        schema_version,
+        field_types=field_types,
+        organism=organism,
+        spatial_library_id=spatial_library_id,
+    )
+
+
+def create_cellxgene_schema(
+    schema_version: CELLxGENESchemaVersions,
+    *,
+    field_types: FieldType | Collection[FieldType] = "ontology_id",
+    organism: CELLxGENEOrganisms = "human",
+    spatial_library_id: str | None = None,
 ) -> Schema:
     """Generates a :class:`~lamindb.Schema` for a specific CELLxGENE schema version.
 
@@ -138,6 +187,8 @@ def get_cxg_schema(
         schema_version: The CELLxGENE Schema version.
         field_types: One or several of 'ontology_id', 'name'.
         organism: The organism of the Schema.
+        library_id: Identifier for the spatial library.
+            Specifying this value enables curation against spatial requirements.
     """
     import bionty as bt
 
@@ -168,7 +219,7 @@ def get_cxg_schema(
         "tissue": CategorySpec(bt.Tissue.name, None),
         "tissue_ontology_term_id": CategorySpec(bt.Tissue.ontology_id, None),
         "tissue_type": CategorySpec(ULabel.name, "tissue"),
-        "organism": CategorySpec(bt.Organism.name, None),
+        "organism": CategorySpec(bt.Organism.scientific_name, None),
         "organism_ontology_term_id": CategorySpec(bt.Organism.ontology_id, None),
         "donor_id": CategorySpec(str, "unknown"),
     }
@@ -195,7 +246,17 @@ def get_cxg_schema(
             f"Invalid field_types: {field_types}. Must contain 'ontology_id', 'name', or both."
         )
 
-    sources = _create_cxg_sources(
+    is_version_6_or_later = version.parse(schema_version) >= version.parse("6.0.0")
+
+    organism_fields = {"organism", "organism_ontology_term_id"}
+    if is_version_6_or_later:
+        obs_categoricals = {
+            k: v for k, v in categoricals.items() if k not in organism_fields
+        }
+    else:
+        obs_categoricals = categoricals
+
+    sources = _create_cellxgene_sources(
         categoricals=categoricals,
         schema_version=schema_version,
         organism=organism,
@@ -217,30 +278,83 @@ def get_cxg_schema(
     obs_features = [
         Feature(
             name=field,
-            dtype=categoricals[field],
+            dtype=obs_categoricals[field],
             cat_filters={"source": source},
             default_value=categoricals_to_spec[field].default,
         ).save()
         for field, source in sources.items()
-        if field != "var_index"
+        if field != "var_index" and field in obs_categoricals
     ]
     for name in ["is_primary_data", "suspension_type", "tissue_type"]:
         obs_features.append(Feature(name=name, dtype=ULabel.name).save())
 
     obs_schema = Schema(
-        name=f"obs of CELLxGENE version {schema_version}",
+        name=f"obs of CELLxGENE version {schema_version} for {organism} of {field_types}",
         features=obs_features,
         otype="DataFrame",
         minimal_set=True,
         coerce_dtype=True,
     ).save()
 
+    slots = {"var": var_schema, "obs": obs_schema}
+
+    if is_version_6_or_later:
+        uns_categoricals = {
+            k: v for k, v in categoricals.items() if k in organism_fields
+        }
+
+        uns_features = [
+            Feature(
+                name=field,
+                dtype=uns_categoricals[field],
+                cat_filters={"source": sources[field]},
+                default_value=categoricals_to_spec[field].default,
+            ).save()
+            for field in uns_categoricals
+        ]
+
+        uns_schema = Schema(
+            name=f"uns of CELLxGENE version {schema_version}",
+            features=uns_features,
+            otype="DataFrame",
+            minimal_set=True,
+            coerce_dtype=True,
+        ).save()
+
+        slots["uns"] = uns_schema
+
+        # Add spatial validation if library_id is provided
+        if spatial_library_id:
+            scalefactors_schema = Schema(
+                name=f"scalefactors of spatial {spatial_library_id}",
+                features=[
+                    Feature(name="spot_diameter_fullres", dtype=float).save(),
+                    Feature(name="tissue_hires_scalef", dtype=float).save(),
+                ],
+            ).save()
+
+            spatial_schema = Schema(
+                name="CELLxGENE spatial metadata",
+                features=[
+                    Feature(
+                        name="is_single",
+                        dtype=bool,
+                        description="True if dataset represents single spatial unit (tissue section for Visium, array for Slide-seqV2)",
+                    ).save()
+                ],
+            ).save()
+
+            slots["uns:spatial"] = spatial_schema
+            slots[f"uns:spatial:{spatial_library_id}:scalefactors"] = (
+                scalefactors_schema
+            )
+
     full_cxg_schema = Schema(
-        name=f"AnnData of CELLxGENE version {schema_version}",
+        name=f"AnnData of CELLxGENE version {schema_version} for {organism} of {', '.join(field_types) if isinstance(field_types, list) else field_types}",
         otype="AnnData",
         minimal_set=True,
         coerce_dtype=True,
-        slots={"var": var_schema, "obs": obs_schema},
+        slots=slots,
     ).save()
 
     return full_cxg_schema

lamindb/examples/cellxgene/{cxg_schema_versions.csv → cellxgene_schema_versions.csv}

@@ -52,3 +52,14 @@ schema_version,entity,organism,source,version
 5.3.0,Tissue,all,uberon,2025-01-15
 5.3.0,Gene,human,ensembl,release-110
 5.3.0,Gene,mouse,ensembl,release-110
+6.0.0,CellType,all,cl,2025-04-10
+6.0.0,ExperimentalFactor,all,efo,3.78.0
+6.0.0,Ethnicity,human,hancestro,3.0
+6.0.0,DevelopmentalStage,human,hsapdv,2025-01-23
+6.0.0,DevelopmentalStage,mouse,mmusdv,2025-01-23
+6.0.0,Disease,all,mondo,2025-05-06
+6.0.0,Organism,all,ncbitaxon,2025-03-13
+6.0.0,Phenotype,all,pato,2025-05-14
+6.0.0,Tissue,all,uberon,2025-05-28
+6.0.0,Gene,human,ensembl,release-110
+6.0.0,Gene,mouse,ensembl,release-110

lamindb/examples/croissant/__init__.py

@@ -1,6 +1,10 @@
-"""Example Croissant files.
+"""Examples for MLCommons Croissant files, which are used to store metadata about datasets.
+
+.. autosummary::
+   :toctree: .
+
+   mini_immuno
 
-Examples for MLCommons Croissant files, which are used to store metadata about datasets.
 """
 
 import json
@@ -12,6 +16,12 @@ def mini_immuno(n_files: int = 1) -> list[Path]:
 
     Args:
         n_files: Number of files inside the croissant file. Default is 1.
+
+    Example
+
+        ::
+
+            croissant_path, dataset1_path = ln.examples.croissant.mini_immuno()
     """
     from ..datasets import file_mini_csv
     from ..datasets.mini_immuno import get_dataset1

lamindb/examples/datasets/__init__.py

@@ -41,7 +41,7 @@ Dictionary, Dataframe, AnnData, MuData, SpatialData.
 .. autosummary::
    :toctree: .
 
-   dict_cxg_uns
+   dict_cellxgene_uns
    df_iris
    df_iris_in_meter
    df_iris_in_meter_study1
@@ -78,7 +78,7 @@ from ._core import (
     df_iris_in_meter,
     df_iris_in_meter_study1,
     df_iris_in_meter_study2,
-    dict_cxg_uns,
+    dict_cellxgene_uns,
     dir_iris_images,
     dir_scrnaseq_cellranger,
     file_bam,

lamindb/examples/datasets/_core.py

@@ -418,7 +418,7 @@ def mudata_papalexi21_subset() -> MuData:  # pragma: no cover
     return mdata
 
 
-def dict_cxg_uns() -> dict[str, Any]:
+def dict_cellxgene_uns() -> dict[str, Any]:
     """An example CELLxGENE AnnData `.uns` dictionary."""
     uns = {
         "organism_ontology_term_id": "NCBITaxon:9606",

lamindb/examples/datasets/_small.py

@@ -9,32 +9,36 @@ import pandas as pd
 
 def small_dataset3_cellxgene(
     otype: Literal["DataFrame", "AnnData"] = "AnnData",
+    *,
     with_obs_defaults: bool = False,
+    with_var_typo: bool = False,
     with_obs_typo: bool = False,
+    with_uns_organism: bool = False,
+    with_uns_spatial: bool = False,
 ) -> tuple[pd.DataFrame, dict[str, Any]] | ad.AnnData:
-    # TODO: consider other ids for other organisms
-    # "ENSMUSG00002076988"
-    var_ids = ["invalid_ensembl_id", "ENSG00000000419", "ENSG00000139618"]
-
+    var_id = "invalid_ensembl_id" if with_var_typo else "ENSG00000000457"
+    var_ids = [var_id, "ENSG00000000419", "ENSG00000139618"]
     lung_id = "UBERON:0002048XXX" if with_obs_typo else "UBERON:0002048"
+
+    obs_data = {
+        "disease_ontology_term_id": [
+            "MONDO:0004975",
+            "MONDO:0004980",
+            "MONDO:0004980",
+        ],
+        "development_stage_ontology_term_id": ["unknown", "unknown", "unknown"],
+        "sex_ontology_term_id": ["PATO:0000383", "PATO:0000384", "unknown"],
+        "tissue_ontology_term_id": [lung_id, lung_id, "UBERON:0000948"],
+        "cell_type": ["T cell", "B cell", "B cell"],
+        "self_reported_ethnicity": ["South Asian", "South Asian", "South Asian"],
+        "donor_id": ["-1", "1", "2"],
+        "is_primary_data": [False, False, False],
+        "suspension_type": ["cell", "cell", "cell"],
+        "tissue_type": ["tissue", "tissue", "tissue"],
+    }
+
     obs_df = pd.DataFrame(
-        {
-            "disease_ontology_term_id": [
-                "MONDO:0004975",
-                "MONDO:0004980",
-                "MONDO:0004980",
-            ],
-            "development_stage_ontology_term_id": ["unknown", "unknown", "unknown"],
-            "organism": ["human", "human", "human"],
-            "sex_ontology_term_id": ["PATO:0000383", "PATO:0000384", "unknown"],
-            "tissue_ontology_term_id": [lung_id, lung_id, "UBERON:0000948"],
-            "cell_type": ["T cell", "B cell", "B cell"],
-            "self_reported_ethnicity": ["South Asian", "South Asian", "South Asian"],
-            "donor_id": ["-1", "1", "2"],
-            "is_primary_data": [False, False, False],
-            "suspension_type": ["cell", "cell", "cell"],
-            "tissue_type": ["tissue", "tissue", "tissue"],
-        },
+        obs_data,
         index=["barcode1", "barcode2", "barcode3"],
     )
 
@@ -65,8 +69,38 @@ def small_dataset3_cellxgene(
         # CELLxGENE requires the `.raw` slot to be set - https://github.com/chanzuckerberg/single-cell-curation/issues/1304
         adata.raw = adata.copy()
         adata.raw.var.drop(columns="feature_is_filtered", inplace=True)
+
         if with_obs_defaults:
+            adata.obs["cell_type_ontology_term_id"] = [
+                "CL:0000084",
+                "CL:0000236",
+                "CL:0000236",
+            ]
+            adata.obs["self_reported_ethnicity_ontology_term_id"] = "na"
+            adata.obs["assay_ontology_term_id"] = "EFO:1001982"
             adata.obs["assay"] = "single-cell RNA sequencing"
+        if with_uns_organism:
+            adata.uns["organism_ontology_term_id"] = "NCBITaxon:9606"
+            adata.uns["organism"] = "Homo sapiens"
+        else:
+            adata.obs["organism_ontology_term_id"] = "NCBITaxon:9606"
+            obs_data["organism"] = ["Homo sapiens", "Homo sapiens", "Homo sapiens"]
+        if with_uns_spatial:
+            adata.uns["spatial"] = {
+                "is_single": True,
+                "library_123": {
+                    "scalefactors": {
+                        "spot_diameter_fullres": 165.0,
+                        "tissue_hires_scalef": 0.5,
+                    },
+                    "images": {
+                        "hires": np.random.default_rng().integers(
+                            0, 255, (2000, 2000, 3), dtype=np.uint8
+                        )
+                    },
+                },
+            }
+
         return adata
 
 
@@ -92,6 +126,16 @@ def anndata_with_obs() -> ad.AnnData:
     df.index = "obs" + df.index.astype(str)
 
     adata = ad.AnnData(X=np.zeros(shape=(40, 100), dtype=np.float32), obs=df)
-    adata.var.index = bionty_base.Gene().df().head(100)["ensembl_gene_id"].values
+    bionty_genes = bionty_base.Gene()
+    # backwards compatible
+    adata.var.index = (
+        (
+            bionty_genes.to_dataframe()
+            if hasattr(bionty_genes, "to_dataframe")
+            else bionty_genes.df()
+        )
+        .head(100)["ensembl_gene_id"]
+        .values
+    )
 
     return adata

lamindb/examples/datasets/mini_immuno.py

@@ -78,6 +78,7 @@ def get_dataset1(
     with_outdated_gene: bool = False,
     with_wrong_subtype: bool = False,
     with_index_type_mismatch: bool = False,
+    with_nested_uns: bool = False,
 ) -> pd.DataFrame | ad.AnnData:
     """A small tabular dataset measuring expression & metadata."""
     # define the data in the dataset

lamindb/migrations/0119_squashed.py

@@ -219,9 +219,8 @@ class Migration(migrations.Migration):
                     "uid",
                     lamindb.base.fields.CharField(
                         blank=True,
-                        db_default="aaaaaaaaaaaa",
                         db_index=True,
-                        default="aaaaaaaaaaaaa",
+                        default=lamindb.base.uids.base62_12,
                         editable=False,
                         max_length=12,
                         unique=True,
@@ -4582,4 +4581,8 @@ class Migration(migrations.Migration):
                 name="unique_artifact_storage_hash_null_key",
             ),
         ),
+        migrations.AlterModelOptions(
+            name="user",
+            options={},
+        ),
     ]

lamindb/migrations/0120_add_record_fk_constraint.py

@@ -0,0 +1,64 @@
+# Generated by Django 5.2 on 2025-08-07 18:52
+
+from django.db import migrations
+
+CREATE_FUNCTION_SQL = """
+CREATE OR REPLACE FUNCTION is_valid_record_type(record_type_id INTEGER, record_is_type BOOLEAN)
+RETURNS BOOLEAN AS $$
+BEGIN
+    -- Record with no type is valid
+    IF record_type_id IS NULL THEN
+        RETURN TRUE;
+    END IF;
+
+    -- If current record is a type, it can only reference schema-less types
+    IF record_is_type THEN
+        RETURN EXISTS (
+            SELECT 1 FROM lamindb_record r
+            WHERE r.id = record_type_id AND r.is_type AND r.schema_id IS NULL
+        );
+    END IF;
+
+    -- Regular records can reference any type
+    RETURN EXISTS (
+        SELECT 1 FROM lamindb_record r
+        WHERE r.id = record_type_id AND r.is_type
+    );
+END;
+$$ LANGUAGE plpgsql;
+"""
+
+ADD_CONSTRAINT_SQL = """
+ALTER TABLE lamindb_record
+ADD CONSTRAINT record_type_is_valid_fk
+CHECK (is_valid_record_type(type_id, is_type));
+"""
+
+DROP_CONSTRAINT_SQL = (
+    "ALTER TABLE lamindb_record DROP CONSTRAINT IF EXISTS record_type_is_valid_fk;"
+)
+DROP_FUNCTION_SQL = "DROP FUNCTION IF EXISTS is_valid_record_type(INTEGER, BOOLEAN);"
+
+
+def apply_postgres_constraint(apps, schema_editor):
+    if schema_editor.connection.vendor == "postgresql":
+        schema_editor.execute(CREATE_FUNCTION_SQL)
+        schema_editor.execute(ADD_CONSTRAINT_SQL)
+
+
+def revert_postgres_constraint(apps, schema_editor):
+    if schema_editor.connection.vendor == "postgresql":
+        schema_editor.execute(DROP_CONSTRAINT_SQL)
+        schema_editor.execute(DROP_FUNCTION_SQL)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("lamindb", "0119_squashed"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            apply_postgres_constraint, reverse_code=revert_postgres_constraint
+        ),
+    ]

lamindb/migrations/0121_recorduser.py

@@ -0,0 +1,53 @@
+# Generated by Django 5.2 on 2025-09-05 12:25
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+import lamindb.base.fields
+import lamindb.models.sqlrecord
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("lamindb", "0120_add_record_fk_constraint"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="RecordUser",
+            fields=[
+                ("id", models.BigAutoField(primary_key=True, serialize=False)),
+                (
+                    "feature",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="links_recorduser",
+                        to="lamindb.feature",
+                    ),
+                ),
+                (
+                    "record",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="values_user",
+                        to="lamindb.record",
+                    ),
+                ),
+                (
+                    "value",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="links_record",
+                        to="lamindb.user",
+                    ),
+                ),
+            ],
+            options={
+                "unique_together": {("record", "feature", "value")},
+            },
+            bases=(models.Model, lamindb.models.sqlrecord.IsLink),
+        ),
+    ]

lamindb/models/__init__.py

@@ -9,6 +9,7 @@
    BasicQuerySet
    QuerySet
    ArtifactSet
+   LazyArtifact
    QueryManager
    SQLRecordList
    FeatureManager
@@ -49,7 +50,7 @@ from .schema import Schema
 from .ulabel import ULabel
 
 # should come last as it needs everything else
-from .artifact import Artifact
+from .artifact import Artifact, LazyArtifact
 from ._feature_manager import FeatureManager
 from ._label_manager import LabelManager
 from .collection import Collection, CollectionArtifact
@@ -78,6 +79,7 @@ from .project import (
     PersonProject,
     RecordPerson,
     RecordReference,
+    ProjectRecord,
 )
 from .run import RunFeatureValue
 from .schema import (

lamindb/models/_describe.py

@@ -8,8 +8,6 @@ from lamin_utils import logger
 from rich.text import Text
 from rich.tree import Tree
 
-from ..core._context import is_run_from_ipython
-
 if TYPE_CHECKING:
     from lamindb.models import Artifact, Collection, Run
 
@@ -41,6 +39,8 @@ def format_rich_tree(
 ) -> str | None:
     from rich.console import Console
 
+    from ..core._context import is_run_from_ipython
+
     # If tree has no children, return fallback
     if not tree.children:
         return fallback