lamindb 1.10.1__py3-none-any.whl → 1.11a1__py3-none-any.whl

lamindb/examples/cellxgene/__init__.py

@@ -3,9 +3,14 @@
 .. autosummary::
    :toctree: .
 
-   save_cxg_defaults
-   get_cxg_schema
+   save_cellxgene_defaults
+   create_cellxgene_schema
 
 """
 
-from ._cellxgene import get_cxg_schema, save_cxg_defaults
+from ._cellxgene import (
+    create_cellxgene_schema,
+    get_cxg_schema,
+    save_cellxgene_defaults,
+    save_cxg_defaults,
+)

lamindb/examples/cellxgene/_cellxgene.py

@@ -3,7 +3,9 @@ from __future__ import annotations
 from typing import TYPE_CHECKING, Collection, Literal, NamedTuple
 
 import pandas as pd
+from lamindb_setup.core import deprecated
 from lamindb_setup.core.upath import UPath
+from packaging import version
 
 from lamindb.models._from_values import _format_values
 
@@ -11,11 +13,25 @@ if TYPE_CHECKING:
     from lamindb.base.types import FieldAttr
     from lamindb.models import Schema, SQLRecord
 
-CELLxGENESchemaVersions = Literal["4.0.0", "5.0.0", "5.1.0", "5.2.0", "5.3.0"]
+CELLxGENESchemaVersions = Literal["4.0.0", "5.0.0", "5.1.0", "5.2.0", "5.3.0", "6.0.0"]
+CELLxGENEOrganisms = Literal[
+    "human",
+    "mouse",
+    "zebra danio",
+    "rhesus macaquedomestic pig",
+    "chimpanzee",
+    "white-tufted-ear marmoset",
+    "sars-2",
+]
 FieldType = Literal["ontology_id", "name"]
 
 
+@deprecated(new_name="save_cellxgene_defaults")
 def save_cxg_defaults() -> None:
+    return save_cxg_defaults()
+
+
+def save_cellxgene_defaults() -> None:
     """Save default values of the CELLxGENE schema to the instance.
 
     Adds CELLxGENE specific (control) values that are not available in the ontologies:
@@ -25,7 +41,6 @@ def save_cxg_defaults() -> None:
     - "unknown" entries for DevelopmentalStage, Phenotype, and CellType
     - "tissue", "organoid", and "cell culture" ULabels (tissue_type)
     - "cell", "nucleus", "na" ULabels (suspension_type)
-
     """
     import bionty as bt
 
@@ -47,12 +62,13 @@ def save_cxg_defaults() -> None:
     # na, unknown
     for model, name in zip(
         [
+            bt.Ethnicity,
             bt.Ethnicity,
             bt.DevelopmentalStage,
             bt.Phenotype,
             bt.CellType,
         ],
-        ["na", "unknown", "unknown", "unknown"],
+        ["na", "unknown", "unknown", "unknown", "unknown"],
     ):
         model(ontology_id=name, name=name, description="From CellxGene schema.").save()
 
@@ -76,8 +92,24 @@ def save_cxg_defaults() -> None:
             name=name, type=suspension_type, description="From CellxGene schema."
         ).save()
 
+    # organisms
+    taxonomy_ids = [
+        "NCBITaxon:9606",  # Homo sapiens (Human)
+        "NCBITaxon:10090",  # Mus musculus (House mouse)
+        "NCBITaxon:9544",  # Macaca mulatta (Rhesus monkey)
+        "NCBITaxon:9825",  # Sus scrofa domesticus (Domestic pig)
+        "NCBITaxon:9598",  # Pan troglodytes (Chimpanzee)
+        "NCBITaxon:9483",  # Callithrix jacchus (White-tufted-ear marmoset)
+        "NCBITaxon:7955",  # Danio rerio (Zebrafish)
+    ]
+    for ontology_id in taxonomy_ids:
+        bt.Organism.from_source(
+            ontology_id=ontology_id,
+            source=bt.Source.get(name="ncbitaxon", currently_used=True),
+        ).save()
+
 
-def _create_cxg_sources(
+def _create_cellxgene_sources(
     categoricals: dict[str, FieldAttr], schema_version: str, organism: str
 ) -> dict[str, SQLRecord]:
     """Create a source dictionary of CELLxGENE categoricals to Source."""
@@ -105,7 +137,7 @@ def _create_cxg_sources(
                 )
             return source
 
-    sources_df = pd.read_csv(UPath(__file__).parent / "cxg_schema_versions.csv")
+    sources_df = pd.read_csv(UPath(__file__).parent / "cellxgene_schema_versions.csv")
     sources_df = sources_df[sources_df.schema_version == schema_version]
     if sources_df.empty:
         raise ValueError(
@@ -126,11 +158,28 @@ def _create_cxg_sources(
     return key_to_source
 
 
+@deprecated(new_name="create_cellxgene_schema")
 def get_cxg_schema(
     schema_version: CELLxGENESchemaVersions,
     *,
     field_types: FieldType | Collection[FieldType] = "ontology_id",
-    organism: Literal["human", "mouse"] = "human",
+    organism: CELLxGENEOrganisms = "human",
+    spatial_library_id: str | None = None,
+) -> Schema:
+    return create_cellxgene_schema(
+        schema_version,
+        field_types=field_types,
+        organism=organism,
+        spatial_library_id=spatial_library_id,
+    )
+
+
+def create_cellxgene_schema(
+    schema_version: CELLxGENESchemaVersions,
+    *,
+    field_types: FieldType | Collection[FieldType] = "ontology_id",
+    organism: CELLxGENEOrganisms = "human",
+    spatial_library_id: str | None = None,
 ) -> Schema:
     """Generates a :class:`~lamindb.Schema` for a specific CELLxGENE schema version.
 
@@ -138,6 +187,8 @@ def get_cxg_schema(
         schema_version: The CELLxGENE Schema version.
         field_types: One or several of 'ontology_id', 'name'.
         organism: The organism of the Schema.
+        library_id: Identifier for the spatial library.
+            Specifying this value enables curation against spatial requirements.
     """
     import bionty as bt
 
@@ -168,7 +219,7 @@ def get_cxg_schema(
         "tissue": CategorySpec(bt.Tissue.name, None),
         "tissue_ontology_term_id": CategorySpec(bt.Tissue.ontology_id, None),
         "tissue_type": CategorySpec(ULabel.name, "tissue"),
-        "organism": CategorySpec(bt.Organism.name, None),
+        "organism": CategorySpec(bt.Organism.scientific_name, None),
         "organism_ontology_term_id": CategorySpec(bt.Organism.ontology_id, None),
         "donor_id": CategorySpec(str, "unknown"),
     }
@@ -195,7 +246,17 @@ def get_cxg_schema(
             f"Invalid field_types: {field_types}. Must contain 'ontology_id', 'name', or both."
         )
 
-    sources = _create_cxg_sources(
+    is_version_6_or_later = version.parse(schema_version) >= version.parse("6.0.0")
+
+    organism_fields = {"organism", "organism_ontology_term_id"}
+    if is_version_6_or_later:
+        obs_categoricals = {
+            k: v for k, v in categoricals.items() if k not in organism_fields
+        }
+    else:
+        obs_categoricals = categoricals
+
+    sources = _create_cellxgene_sources(
         categoricals=categoricals,
         schema_version=schema_version,
         organism=organism,
@@ -217,30 +278,83 @@ def get_cxg_schema(
     obs_features = [
         Feature(
             name=field,
-            dtype=categoricals[field],
+            dtype=obs_categoricals[field],
             cat_filters={"source": source},
             default_value=categoricals_to_spec[field].default,
         ).save()
         for field, source in sources.items()
-        if field != "var_index"
+        if field != "var_index" and field in obs_categoricals
     ]
     for name in ["is_primary_data", "suspension_type", "tissue_type"]:
         obs_features.append(Feature(name=name, dtype=ULabel.name).save())
 
     obs_schema = Schema(
-        name=f"obs of CELLxGENE version {schema_version}",
+        name=f"obs of CELLxGENE version {schema_version} for {organism} of {field_types}",
         features=obs_features,
         otype="DataFrame",
         minimal_set=True,
         coerce_dtype=True,
     ).save()
 
+    slots = {"var": var_schema, "obs": obs_schema}
+
+    if is_version_6_or_later:
+        uns_categoricals = {
+            k: v for k, v in categoricals.items() if k in organism_fields
+        }
+
+        uns_features = [
+            Feature(
+                name=field,
+                dtype=uns_categoricals[field],
+                cat_filters={"source": sources[field]},
+                default_value=categoricals_to_spec[field].default,
+            ).save()
+            for field in uns_categoricals
+        ]
+
+        uns_schema = Schema(
+            name=f"uns of CELLxGENE version {schema_version}",
+            features=uns_features,
+            otype="DataFrame",
+            minimal_set=True,
+            coerce_dtype=True,
+        ).save()
+
+        slots["uns"] = uns_schema
+
+        # Add spatial validation if library_id is provided
+        if spatial_library_id:
+            scalefactors_schema = Schema(
+                name=f"scalefactors of spatial {spatial_library_id}",
+                features=[
+                    Feature(name="spot_diameter_fullres", dtype=float).save(),
+                    Feature(name="tissue_hires_scalef", dtype=float).save(),
+                ],
+            ).save()
+
+            spatial_schema = Schema(
+                name="CELLxGENE spatial metadata",
+                features=[
+                    Feature(
+                        name="is_single",
+                        dtype=bool,
+                        description="True if dataset represents single spatial unit (tissue section for Visium, array for Slide-seqV2)",
+                    ).save()
+                ],
+            ).save()
+
+            slots["uns:spatial"] = spatial_schema
+            slots[f"uns:spatial:{spatial_library_id}:scalefactors"] = (
+                scalefactors_schema
+            )
+
     full_cxg_schema = Schema(
-        name=f"AnnData of CELLxGENE version {schema_version}",
+        name=f"AnnData of CELLxGENE version {schema_version} for {organism} of {', '.join(field_types) if isinstance(field_types, list) else field_types}",
         otype="AnnData",
         minimal_set=True,
         coerce_dtype=True,
-        slots={"var": var_schema, "obs": obs_schema},
+        slots=slots,
     ).save()
 
     return full_cxg_schema

lamindb/examples/cellxgene/{cxg_schema_versions.csv → cellxgene_schema_versions.csv}

@@ -52,3 +52,14 @@ schema_version,entity,organism,source,version
 5.3.0,Tissue,all,uberon,2025-01-15
 5.3.0,Gene,human,ensembl,release-110
 5.3.0,Gene,mouse,ensembl,release-110
+6.0.0,CellType,all,cl,2025-04-10
+6.0.0,ExperimentalFactor,all,efo,3.78.0
+6.0.0,Ethnicity,human,hancestro,3.0
+6.0.0,DevelopmentalStage,human,hsapdv,2025-01-23
+6.0.0,DevelopmentalStage,mouse,mmusdv,2025-01-23
+6.0.0,Disease,all,mondo,2025-05-06
+6.0.0,Organism,all,ncbitaxon,2025-03-13
+6.0.0,Phenotype,all,pato,2025-05-14
+6.0.0,Tissue,all,uberon,2025-05-28
+6.0.0,Gene,human,ensembl,release-110
+6.0.0,Gene,mouse,ensembl,release-110

lamindb/examples/croissant/__init__.py

@@ -1,6 +1,10 @@
-"""Example Croissant files.
+"""Examples for MLCommons Croissant files, which are used to store metadata about datasets.
+
+.. autosummary::
+   :toctree: .
+
+   mini_immuno
 
-Examples for MLCommons Croissant files, which are used to store metadata about datasets.
 """
 
 import json
@@ -12,6 +16,12 @@ def mini_immuno(n_files: int = 1) -> list[Path]:
 
     Args:
         n_files: Number of files inside the croissant file. Default is 1.
+
+    Example
+
+        ::
+
+            croissant_path, dataset1_path = ln.examples.croissant.mini_immuno()
     """
     from ..datasets import file_mini_csv
     from ..datasets.mini_immuno import get_dataset1

lamindb/examples/datasets/__init__.py

@@ -41,7 +41,7 @@ Dictionary, Dataframe, AnnData, MuData, SpatialData.
 .. autosummary::
    :toctree: .
 
-   dict_cxg_uns
+   dict_cellxgene_uns
    df_iris
    df_iris_in_meter
    df_iris_in_meter_study1
@@ -78,7 +78,7 @@ from ._core import (
     df_iris_in_meter,
     df_iris_in_meter_study1,
     df_iris_in_meter_study2,
-    dict_cxg_uns,
+    dict_cellxgene_uns,
     dir_iris_images,
     dir_scrnaseq_cellranger,
     file_bam,

lamindb/examples/datasets/_core.py

@@ -418,7 +418,7 @@ def mudata_papalexi21_subset() -> MuData:  # pragma: no cover
     return mdata
 
 
-def dict_cxg_uns() -> dict[str, Any]:
+def dict_cellxgene_uns() -> dict[str, Any]:
     """An example CELLxGENE AnnData `.uns` dictionary."""
     uns = {
         "organism_ontology_term_id": "NCBITaxon:9606",

lamindb/examples/datasets/_small.py

@@ -9,32 +9,36 @@ import pandas as pd
 
 def small_dataset3_cellxgene(
     otype: Literal["DataFrame", "AnnData"] = "AnnData",
+    *,
     with_obs_defaults: bool = False,
+    with_var_typo: bool = False,
     with_obs_typo: bool = False,
+    with_uns_organism: bool = False,
+    with_uns_spatial: bool = False,
 ) -> tuple[pd.DataFrame, dict[str, Any]] | ad.AnnData:
-    # TODO: consider other ids for other organisms
-    # "ENSMUSG00002076988"
-    var_ids = ["invalid_ensembl_id", "ENSG00000000419", "ENSG00000139618"]
-
+    var_id = "invalid_ensembl_id" if with_var_typo else "ENSG00000000457"
+    var_ids = [var_id, "ENSG00000000419", "ENSG00000139618"]
     lung_id = "UBERON:0002048XXX" if with_obs_typo else "UBERON:0002048"
+
+    obs_data = {
+        "disease_ontology_term_id": [
+            "MONDO:0004975",
+            "MONDO:0004980",
+            "MONDO:0004980",
+        ],
+        "development_stage_ontology_term_id": ["unknown", "unknown", "unknown"],
+        "sex_ontology_term_id": ["PATO:0000383", "PATO:0000384", "unknown"],
+        "tissue_ontology_term_id": [lung_id, lung_id, "UBERON:0000948"],
+        "cell_type": ["T cell", "B cell", "B cell"],
+        "self_reported_ethnicity": ["South Asian", "South Asian", "South Asian"],
+        "donor_id": ["-1", "1", "2"],
+        "is_primary_data": [False, False, False],
+        "suspension_type": ["cell", "cell", "cell"],
+        "tissue_type": ["tissue", "tissue", "tissue"],
+    }
+
     obs_df = pd.DataFrame(
-        {
-            "disease_ontology_term_id": [
-                "MONDO:0004975",
-                "MONDO:0004980",
-                "MONDO:0004980",
-            ],
-            "development_stage_ontology_term_id": ["unknown", "unknown", "unknown"],
-            "organism": ["human", "human", "human"],
-            "sex_ontology_term_id": ["PATO:0000383", "PATO:0000384", "unknown"],
-            "tissue_ontology_term_id": [lung_id, lung_id, "UBERON:0000948"],
-            "cell_type": ["T cell", "B cell", "B cell"],
-            "self_reported_ethnicity": ["South Asian", "South Asian", "South Asian"],
-            "donor_id": ["-1", "1", "2"],
-            "is_primary_data": [False, False, False],
-            "suspension_type": ["cell", "cell", "cell"],
-            "tissue_type": ["tissue", "tissue", "tissue"],
-        },
+        obs_data,
         index=["barcode1", "barcode2", "barcode3"],
     )
 
@@ -65,8 +69,38 @@ def small_dataset3_cellxgene(
         # CELLxGENE requires the `.raw` slot to be set - https://github.com/chanzuckerberg/single-cell-curation/issues/1304
         adata.raw = adata.copy()
         adata.raw.var.drop(columns="feature_is_filtered", inplace=True)
+
         if with_obs_defaults:
+            adata.obs["cell_type_ontology_term_id"] = [
+                "CL:0000084",
+                "CL:0000236",
+                "CL:0000236",
+            ]
+            adata.obs["self_reported_ethnicity_ontology_term_id"] = "na"
+            adata.obs["assay_ontology_term_id"] = "EFO:1001982"
             adata.obs["assay"] = "single-cell RNA sequencing"
+        if with_uns_organism:
+            adata.uns["organism_ontology_term_id"] = "NCBITaxon:9606"
+            adata.uns["organism"] = "Homo sapiens"
+        else:
+            adata.obs["organism_ontology_term_id"] = "NCBITaxon:9606"
+            obs_data["organism"] = ["Homo sapiens", "Homo sapiens", "Homo sapiens"]
+        if with_uns_spatial:
+            adata.uns["spatial"] = {
+                "is_single": True,
+                "library_123": {
+                    "scalefactors": {
+                        "spot_diameter_fullres": 165.0,
+                        "tissue_hires_scalef": 0.5,
+                    },
+                    "images": {
+                        "hires": np.random.default_rng().integers(
+                            0, 255, (2000, 2000, 3), dtype=np.uint8
+                        )
+                    },
+                },
+            }
+
         return adata
 
 
@@ -92,6 +126,16 @@ def anndata_with_obs() -> ad.AnnData:
     df.index = "obs" + df.index.astype(str)
 
     adata = ad.AnnData(X=np.zeros(shape=(40, 100), dtype=np.float32), obs=df)
-    adata.var.index = bionty_base.Gene().df().head(100)["ensembl_gene_id"].values
+    bionty_genes = bionty_base.Gene()
+    # backwards compatible
+    adata.var.index = (
+        (
+            bionty_genes.to_dataframe()
+            if hasattr(bionty_genes, "to_dataframe")
+            else bionty_genes.df()
+        )
+        .head(100)["ensembl_gene_id"]
+        .values
+    )
 
     return adata

lamindb/examples/datasets/mini_immuno.py

@@ -78,6 +78,7 @@ def get_dataset1(
     with_outdated_gene: bool = False,
     with_wrong_subtype: bool = False,
     with_index_type_mismatch: bool = False,
+    with_nested_uns: bool = False,
 ) -> pd.DataFrame | ad.AnnData:
     """A small tabular dataset measuring expression & metadata."""
     # define the data in the dataset

lamindb/migrations/0118_alter_recordproject_value_projectrecord.py

@@ -0,0 +1,99 @@
+# Generated by Django 5.2 on 2025-08-07 15:28
+
+import django.db.models.deletion
+import django.db.models.functions.datetime
+from django.db import migrations, models
+
+import lamindb.base.fields
+import lamindb.base.users
+import lamindb.models.run
+import lamindb.models.sqlrecord
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("lamindb", "0117_fix_artifact_storage_hash_unique_constraints"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="recordproject",
+            name="value",
+            field=lamindb.base.fields.ForeignKey(
+                blank=True,
+                on_delete=django.db.models.deletion.PROTECT,
+                related_name="links_in_record",
+                to="lamindb.project",
+            ),
+        ),
+        migrations.CreateModel(
+            name="ProjectRecord",
+            fields=[
+                (
+                    "created_at",
+                    lamindb.base.fields.DateTimeField(
+                        blank=True,
+                        db_default=django.db.models.functions.datetime.Now(),
+                        db_index=True,
+                        editable=False,
+                    ),
+                ),
+                ("id", models.BigAutoField(primary_key=True, serialize=False)),
+                (
+                    "created_by",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        default=lamindb.base.users.current_user_id,
+                        editable=False,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="+",
+                        to="lamindb.user",
+                    ),
+                ),
+                (
+                    "feature",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        default=None,
+                        null=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="links_projectrecord",
+                        to="lamindb.feature",
+                    ),
+                ),
+                (
+                    "project",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="links_record",
+                        to="lamindb.project",
+                    ),
+                ),
+                (
+                    "record",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="links_project",
+                        to="lamindb.record",
+                    ),
+                ),
+                (
+                    "run",
+                    lamindb.base.fields.ForeignKey(
+                        blank=True,
+                        default=lamindb.models.run.current_run,
+                        null=True,
+                        on_delete=django.db.models.deletion.PROTECT,
+                        related_name="+",
+                        to="lamindb.run",
+                    ),
+                ),
+            ],
+            options={
+                "unique_together": {("record", "project", "feature")},
+            },
+            bases=(lamindb.models.sqlrecord.IsLink, models.Model),
+        ),
+    ]

lamindb/migrations/0119_rename_records_project_linked_in_records.py

@@ -0,0 +1,26 @@
+# Generated by Django 5.2 on 2025-08-09 13:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("lamindb", "0118_alter_recordproject_value_projectrecord"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="project",
+            old_name="records",
+            new_name="linked_in_records",
+        ),
+        migrations.AddField(
+            model_name="project",
+            name="records",
+            field=models.ManyToManyField(
+                related_name="projects",
+                through="lamindb.ProjectRecord",
+                to="lamindb.record",
+            ),
+        ),
+    ]