lamindb 1.9.1__py3-none-any.whl → 1.10.0__py3-none-any.whl

lamindb/__init__.py

@@ -108,7 +108,7 @@ Backwards compatibility.
 
 # ruff: noqa: I001
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "1.9.1"
+__version__ = "1.10.0"
 
 import warnings
 

lamindb/core/__init__.py

@@ -28,7 +28,6 @@ Modules:
 .. autosummary::
    :toctree: .
 
-   datasets
    storage
    logger
 
@@ -38,7 +37,8 @@ from lamin_utils import logger
 from lamin_utils._inspect import InspectResult
 
 from .. import errors as exceptions
-from . import datasets, loaders, subsettings, types
+from ..examples import datasets  # backward compat
+from . import loaders, subsettings, types
 from ._context import Context
 from ._mapped_collection import MappedCollection
 from ._settings import Settings

lamindb/core/storage/__init__.py

@@ -13,12 +13,13 @@ Array accessors.
    :toctree: .
 
    AnnDataAccessor
+   SpatialDataAccessor
    BackedAccessor
 """
 
 from lamindb_setup.core.upath import LocalPathClasses, UPath, infer_filesystem
 
-from ._backed_access import AnnDataAccessor, BackedAccessor
+from ._backed_access import AnnDataAccessor, BackedAccessor, SpatialDataAccessor
 from ._tiledbsoma import save_tiledbsoma_experiment
 from ._valid_suffixes import VALID_SUFFIXES
 from .objects import infer_suffix, write_to_disk

lamindb/core/storage/_anndata_accessor.py

@@ -353,7 +353,16 @@ if ZARR_INSTALLED:
         attrs_keys: dict[str, list] = {}
         obs_var_arrays = []
 
-        for path in paths:
+        prefix = storage.path
+        if prefix == "":
+            paths_iter = (path for path in paths)
+        else:
+            prefix += "/"
+            paths_iter = (
+                path.removeprefix(prefix) for path in paths if path.startswith(prefix)
+            )
+
+        for path in paths_iter:
             if path in (".zattrs", ".zgroup"):
                 continue
             parts = path.split("/")

lamindb/core/storage/_backed_access.py

@@ -9,6 +9,7 @@ from anndata._io.specs.registry import get_spec
 from ._anndata_accessor import AnnDataAccessor, StorageType, registry
 from ._polars_lazy_df import POLARS_SUFFIXES, _open_polars_lazy_df
 from ._pyarrow_dataset import PYARROW_SUFFIXES, _open_pyarrow_dataset
+from ._spatialdata_accessor import SpatialDataAccessor
 from ._tiledbsoma import _open_tiledbsoma
 from .paths import filepath_from_artifact
 
@@ -80,6 +81,7 @@ def backed_access(
     **kwargs,
 ) -> (
     AnnDataAccessor
+    | SpatialDataAccessor
     | BackedAccessor
     | SOMACollection
     | SOMAExperiment
@@ -110,6 +112,8 @@ def backed_access(
         conn, storage = registry.open("h5py", objectpath, mode=mode, **kwargs)
     elif suffix == ".zarr":
         conn, storage = registry.open("zarr", objectpath, mode=mode, **kwargs)
+        if "spatialdata_attrs" in storage.attrs:
+            return SpatialDataAccessor(storage, name)
     elif len(df_suffixes := _flat_suffixes(objectpath)) == 1 and (
         df_suffix := df_suffixes.pop()
     ) in set(PYARROW_SUFFIXES).union(POLARS_SUFFIXES):

lamindb/core/storage/_spatialdata_accessor.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from functools import cached_property
+from typing import TYPE_CHECKING
+
+from ._anndata_accessor import AnnDataAccessor
+
+if TYPE_CHECKING:
+    from zarr import Group
+
+
+class _TablesAccessor:
+    def __init__(self, tables: Group):
+        self._tables = tables
+
+    def __getitem__(self, key: str) -> AnnDataAccessor:
+        return AnnDataAccessor(connection=None, storage=self._tables[key], filename=key)
+
+    def keys(self) -> list[str]:
+        return list(self._tables.keys())
+
+    def __repr__(self) -> str:
+        """Description of the _TablesAccessor object."""
+        descr = (
+            f"Accessor for the SpatialData attribute tables\n  with keys: {self.keys()}"
+        )
+        return descr
+
+
+class SpatialDataAccessor:
+    """Cloud-backed SpatialData.
+
+    For now only allows to access `tables`.
+    """
+
+    def __init__(self, storage: Group, name: str):
+        self.storage = storage
+        self._name = name
+
+    @cached_property
+    def tables(self) -> _TablesAccessor:
+        """tables of the underlying SpatialData object."""
+        return _TablesAccessor(self.storage["tables"])
+
+    def __repr__(self):
+        """Description of the SpatialDataAccessor object."""
+        descr = (
+            "SpatialDataAccessor object"
+            f"\n  constructed for the SpatialData object {self._name}"
+            f"\n    with tables: {self.tables.keys()}"
+        )
+        return descr

lamindb/examples/__init__.py

@@ -3,27 +3,12 @@
 .. autosummary::
    :toctree: .
 
-   ingest_mini_immuno_datasets
    schemas
+   datasets
    cellxgene
+   croissant
 
 """
 
-from . import schemas
+from . import croissant, datasets, schemas
 from .cellxgene import _cellxgene
-
-
-def ingest_mini_immuno_datasets():
-    """Ingest mini immuno datasets.
-
-    .. literalinclude:: scripts/ingest_mini_immuno_datasets.py
-        :language: python
-    """
-    import sys
-    from pathlib import Path
-
-    docs_path = Path(__file__).parent.parent.parent / "docs" / "scripts"
-    if str(docs_path) not in sys.path:
-        sys.path.append(str(docs_path))
-
-    import ingest_mini_immuno_datasets  # noqa

lamindb/examples/cellxgene/_cellxgene.py

@@ -1,12 +1,16 @@
-from typing import Collection, Literal, NamedTuple
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Collection, Literal, NamedTuple
 
 import pandas as pd
 from lamindb_setup.core.upath import UPath
 
-from lamindb.base.types import FieldAttr
-from lamindb.models import Feature, Schema, SQLRecord, ULabel
 from lamindb.models._from_values import _format_values
 
+if TYPE_CHECKING:
+    from lamindb.base.types import FieldAttr
+    from lamindb.models import Schema, SQLRecord
+
 CELLxGENESchemaVersions = Literal["4.0.0", "5.0.0", "5.1.0", "5.2.0", "5.3.0"]
 FieldType = Literal["ontology_id", "name"]
 
@@ -25,6 +29,8 @@ def save_cxg_defaults() -> None:
     """
     import bionty as bt
 
+    from lamindb.models import ULabel
+
     # "normal" in Disease
     normal = bt.Phenotype.from_source(
         ontology_id="PATO:0000461",
@@ -135,6 +141,8 @@ def get_cxg_schema(
     """
     import bionty as bt
 
+    from lamindb.models import Feature, Schema, ULabel
+
     class CategorySpec(NamedTuple):
         field: str | FieldAttr
         default: str | None

lamindb/examples/croissant/__init__.py

@@ -0,0 +1,44 @@
+"""Example Croissant files.
+
+Examples for MLCommons Croissant files, which are used to store metadata about datasets.
+"""
+
+import json
+from pathlib import Path
+
+
+def mini_immuno(n_files: int = 1) -> list[Path]:
+    """Return paths to the mini immuno dataset and its metadata as a Croissant file.
+
+    Args:
+        n_files: Number of files inside the croissant file. Default is 1.
+    """
+    from ..datasets import file_mini_csv
+    from ..datasets.mini_immuno import get_dataset1
+
+    adata = get_dataset1(otype="AnnData")
+    dataset1_path = Path("mini_immuno.anndata.zarr")
+    adata.write_zarr(dataset1_path)
+    orig_croissant_path = (
+        Path(__file__).parent / "mini_immuno.anndata.zarr_metadata.json"
+    )
+    with open(orig_croissant_path, encoding="utf-8") as f:
+        data = json.load(f)
+    if n_files == 2:
+        dataset2_path = file_mini_csv()
+        data["distribution"].append(
+            {
+                "@type": "sc:FileObject",
+                "@id": "mini.csv",
+                "name": "mini.csv",
+                "encodingFormat": "text/csv",
+            }
+        )
+    croissant_path = Path("mini_immuno.anndata.zarr_metadata.json")
+    with open(croissant_path, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=2)
+    result: list[Path] = [croissant_path, dataset1_path]
+    if n_files == 1:
+        return result
+    result.append(dataset2_path)
+    return result

lamindb/examples/croissant/mini_immuno.anndata.zarr_metadata.json

@@ -0,0 +1,73 @@
+{
+  "@context": {
+    "@vocab": "https://schema.org/",
+    "cr": "https://mlcommons.org/croissant/",
+    "ml": "http://ml-schema.org/",
+    "sc": "https://schema.org/",
+    "dct": "http://purl.org/dc/terms/",
+    "data": "https://mlcommons.org/croissant/data/",
+    "rai": "https://mlcommons.org/croissant/rai/",
+    "format": "https://mlcommons.org/croissant/format/",
+    "citeAs": "https://mlcommons.org/croissant/citeAs/",
+    "conformsTo": "https://mlcommons.org/croissant/conformsTo/",
+    "@language": "en",
+    "repeated": "https://mlcommons.org/croissant/repeated/",
+    "field": "https://mlcommons.org/croissant/field/",
+    "examples": "https://mlcommons.org/croissant/examples/",
+    "recordSet": "https://mlcommons.org/croissant/recordSet/",
+    "fileObject": "https://mlcommons.org/croissant/fileObject/",
+    "fileSet": "https://mlcommons.org/croissant/fileSet/",
+    "source": "https://mlcommons.org/croissant/source/",
+    "references": "https://mlcommons.org/croissant/references/",
+    "key": "https://mlcommons.org/croissant/key/",
+    "parentField": "https://mlcommons.org/croissant/parentField/",
+    "isLiveDataset": "https://mlcommons.org/croissant/isLiveDataset/",
+    "separator": "https://mlcommons.org/croissant/separator/",
+    "extract": "https://mlcommons.org/croissant/extract/",
+    "subField": "https://mlcommons.org/croissant/subField/",
+    "regex": "https://mlcommons.org/croissant/regex/",
+    "column": "https://mlcommons.org/croissant/column/",
+    "path": "https://mlcommons.org/croissant/path/",
+    "fileProperty": "https://mlcommons.org/croissant/fileProperty/",
+    "md5": "https://mlcommons.org/croissant/md5/",
+    "jsonPath": "https://mlcommons.org/croissant/jsonPath/",
+    "transform": "https://mlcommons.org/croissant/transform/",
+    "replace": "https://mlcommons.org/croissant/replace/",
+    "dataType": "https://mlcommons.org/croissant/dataType/",
+    "includes": "https://mlcommons.org/croissant/includes/",
+    "excludes": "https://mlcommons.org/croissant/excludes/"
+  },
+  "@type": "Dataset",
+  "name": "Mini immuno dataset",
+  "description": "A few samples from the immunology dataset",
+  "url": "https://lamin.ai/laminlabs/lamindata/artifact/tCUkRcaEjTjhtozp0000",
+  "creator": {
+    "@type": "Person",
+    "name": "falexwolf"
+  },
+  "dateCreated": "2025-07-16",
+  "cr:projectName": "Mini Immuno Project",
+  "datePublished": "2025-07-16",
+  "version": "1.0",
+  "license": "https://creativecommons.org/licenses/by/4.0/",
+  "citation": "Please cite this dataset as: mini immuno (2025)",
+  "encodingFormat": "zarr",
+  "distribution": [
+    {
+      "@type": "cr:FileSet",
+      "@id": "mini_immuno.anndata.zarr",
+      "containedIn": {
+        "@id": "directory"
+      },
+      "encodingFormat": "zarr"
+    }
+  ],
+  "cr:recordSet": [
+    {
+      "@type": "cr:RecordSet",
+      "@id": "#samples",
+      "name": "samples",
+      "description": "my sample"
+    }
+  ]
+}

lamindb/{core → examples}/datasets/__init__.py

@@ -1,4 +1,4 @@
-"""Test datasets.
+"""Example datasets.
 
 The mini immuno dataset.
 

lamindb/{core → examples}/datasets/mini_immuno.py

@@ -1,16 +1,20 @@
-"""The mini immuno dataset.
+"""The two "mini immuno" datasets.
 
 .. autosummary::
    :toctree: .
 
-   define_features_labels
    get_dataset1
    get_dataset2
+   define_features_labels
+   define_mini_immuno_schema_flexible
+   save_mini_immuno_datasets
 
 """
 
 from __future__ import annotations
 
+import sys
+from pathlib import Path
 from typing import TYPE_CHECKING, Literal
 
 import anndata as ad
@@ -26,9 +30,6 @@ def define_features_labels() -> None:
     .. literalinclude:: scripts/define_mini_immuno_features_labels.py
         :language: python
     """
-    import sys
-    from pathlib import Path
-
     docs_path = Path(__file__).parent.parent.parent.parent / "docs" / "scripts"
     if str(docs_path) not in sys.path:
         sys.path.append(str(docs_path))
@@ -42,9 +43,6 @@ def define_mini_immuno_schema_flexible() -> Schema:
     .. literalinclude:: scripts/define_mini_immuno_schema_flexible.py
         :language: python
     """
-    import sys
-    from pathlib import Path
-
     from lamindb.models import Schema
 
     docs_path = Path(__file__).parent.parent.parent.parent / "docs" / "scripts"
@@ -57,6 +55,19 @@ def define_mini_immuno_schema_flexible() -> Schema:
     return Schema.get(name="Mini immuno schema")
 
 
+def save_mini_immuno_datasets():
+    """Save the two "mini immuno" datasets.
+
+    .. literalinclude:: scripts/save_mini_immuno_datasets.py
+        :language: python
+    """
+    docs_path = Path(__file__).parent.parent.parent.parent / "docs" / "scripts"
+    if str(docs_path) not in sys.path:
+        sys.path.append(str(docs_path))
+
+    import save_mini_immuno_datasets  # noqa
+
+
 def get_dataset1(
     otype: Literal["DataFrame", "AnnData"] = "DataFrame",
     gene_symbols_in_index: bool = False,

lamindb/examples/schemas/_anndata.py

@@ -1,4 +1,12 @@
-from ... import Schema
+from __future__ import annotations
+
+import importlib
+import sys
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ... import Schema
 
 
 def anndata_ensembl_gene_ids_and_valid_features_in_obs() -> Schema:
@@ -7,19 +15,21 @@ def anndata_ensembl_gene_ids_and_valid_features_in_obs() -> Schema:
     .. literalinclude:: scripts/define_schema_anndata_ensembl_gene_ids_and_valid_features_in_obs.py
         :language: python
     """
-    import subprocess
-    from pathlib import Path
+    from ... import Schema
 
     docs_path = Path(__file__).parent.parent.parent.parent / "docs" / "scripts"
-    subprocess.run(
-        [
-            "python",
-            str(
-                docs_path
-                / "define_schema_anndata_ensembl_gene_ids_and_valid_features_in_obs.py"
-            ),
-        ],
-        check=True,
-    )
-
-    return Schema.get(name="anndata_ensembl_gene_ids_and_valid_features_in_obs")
+    if str(docs_path) not in sys.path:
+        sys.path.append(str(docs_path))
+
+    try:
+        return Schema.get(name="anndata_ensembl_gene_ids_and_valid_features_in_obs")
+    except Schema.DoesNotExist:
+        import define_schema_anndata_ensembl_gene_ids_and_valid_features_in_obs  # noqa
+
+        try:
+            return Schema.get(name="anndata_ensembl_gene_ids_and_valid_features_in_obs")
+        except Schema.DoesNotExist:
+            importlib.reload(
+                define_schema_anndata_ensembl_gene_ids_and_valid_features_in_obs
+            )
+            return Schema.get(name="anndata_ensembl_gene_ids_and_valid_features_in_obs")

lamindb/examples/schemas/_simple.py

@@ -1,19 +1,33 @@
-from ... import Schema
+from __future__ import annotations
+
+import importlib
+import sys
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ... import Schema
 
 
 def valid_features() -> Schema:
     """Return a schema for an AnnData with Ensembl gene IDs and valid features in obs.
 
-    .. literalinclude:: scripts/define_schema_anndata_ensembl_gene_ids_and_valid_features_in_obs.py
+    .. literalinclude:: scripts/define_valid_features.py
         :language: python
     """
-    import subprocess
-    from pathlib import Path
+    from ... import Schema
 
     docs_path = Path(__file__).parent.parent.parent.parent / "docs" / "scripts"
-    subprocess.run(
-        ["python", str(docs_path / "define_valid_features.py")],
-        check=True,
-    )
+    if str(docs_path) not in sys.path:
+        sys.path.append(str(docs_path))
+
+    try:
+        return Schema.get(name="valid_features")
+    except Schema.DoesNotExist:
+        try:
+            import define_valid_features  # noqa
 
-    return Schema.get(name="valid_features")
+            return Schema.get(name="valid_features")
+        except Schema.DoesNotExist:
+            importlib.reload(define_valid_features)
+            return Schema.get(name="valid_features")

lamindb/integrations/__init__.py

@@ -5,8 +5,10 @@
 
    save_vitessce_config
    save_tiledbsoma_experiment
+   curate_from_croissant
 """
 
 from lamindb.core.storage import save_tiledbsoma_experiment
 
+from ._croissant import curate_from_croissant
 from ._vitessce import save_vitessce_config

lamindb/integrations/_croissant.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    import lamindb as ln
+
+
+def curate_from_croissant(
+    croissant_data: str | Path | dict[str, Any],
+    run: ln.Run | None = None,
+) -> ln.Artifact | ln.Collection:
+    """Create annotated artifacts from a CroissantML file.
+
+    Returns a collection if multiple files are found in `croissant_data`, otherwise a single artifact.
+
+    Args:
+        croissant_data: Path to CroissantML JSON file or dictionary.
+
+    Example:
+
+        ::
+
+            artifact = ln.integrations.curate_from_croissant("dataset_metadata.json")
+    """
+    import lamindb as ln
+
+    # Load CroissantML data
+    if isinstance(croissant_data, (str, Path)):
+        if not Path(croissant_data).exists():
+            raise FileNotFoundError(f"File not found: {croissant_data}")
+        with open(croissant_data, encoding="utf-8") as f:
+            data = json.load(f)
+    elif isinstance(croissant_data, dict):
+        data = croissant_data
+    else:
+        raise ValueError(
+            "croissant_data must be a file path, JSON string, or dictionary"
+        )
+
+    # Validate basic structure
+    if data.get("@type") != "Dataset":
+        raise ValueError("CroissantML @type must be 'Dataset'")
+
+    if "name" not in data:
+        raise ValueError("CroissantML must have a 'name' field")
+
+    # Extract basic metadata
+    dataset_name = data["name"]
+    description = data.get("description", "")
+    version = data.get("version", "1.0")
+    license_info = data.get("license", "")
+    project_name = data.get("cr:projectName", "")
+
+    # Create license feature and label if license info exists
+    license_label = None
+    if license_info:
+        license_label_type = ln.ULabel.filter(name="License", is_type=True).first()
+        if not license_label_type:
+            license_label_type = ln.ULabel(name="License", is_type=True).save()
+        license_label = ln.ULabel.filter(name=license_info).first()
+        if not license_label:
+            license_label = ln.ULabel(
+                name=license_info,
+                description="Dataset license",
+                type=license_label_type,
+            ).save()
+    project_label = None
+    if project_name:
+        project_label = ln.Project.filter(name=project_name).first()
+        if not project_label:
+            project_label = ln.Project(name=project_name).save()
+
+    # Extract file distributions
+    artifacts = []
+    file_distributions = data.get("distribution", [])
+    if not file_distributions:
+        raise ValueError("No file distributions found in croissant data")
+    for dist in file_distributions:
+        file_id = dist.get("@id", "")
+        if Path(file_id).exists():
+            file_path = file_id
+        else:
+            content_url = dist.get("contentUrl", "")
+            file_path = content_url or data.get("url", "")
+        if not file_path:
+            raise ValueError(
+                f"No valid file path found in croissant distribution: {dist}"
+            )
+        if len(file_distributions) == 1:
+            artifact_description = f"{dataset_name}"
+            if file_id != dataset_name:
+                artifact_description += f" ({file_id})"
+            artifact_description += f" - {description}"
+        else:
+            artifact_description = f"{file_id}"
+        artifact = ln.Artifact(  # type: ignore
+            file_path,
+            description=artifact_description,
+            version=version,
+            kind="dataset",
+            run=run,
+        ).save()
+        if license_label:
+            artifact.ulabels.add(license_label)
+        if project_label:
+            artifact.projects.add(project_label)
+        artifacts.append(artifact)
+
+    if len(artifacts) == 1:
+        return artifacts[0]
+    else:
+        collection = ln.Collection(  # type: ignore
+            artifacts, key=dataset_name, description=description, version=version
+        ).save()
+        if license_label:
+            collection.ulabels.add(license_label)
+        if project_label:
+            collection.projects.add(project_label)
+        return collection

lamindb/integrations/_vitessce.py

@@ -28,21 +28,17 @@ def save_vitessce_config(
     If the `VitessceConfig` object references multiple artifacts, automatically
     creates a `Collection` and displays the "Vitessce button" next to it.
 
+    The `VitessceConfig` artifact has `.suffix = ".vitessce.json"` and `.kind = "__lamindb_config__"`,
+    which is by default hidden on the hub UI.
+
     Guide: :doc:`docs:vitessce`.
 
     Args:
         vitessce_config: A `VitessceConfig` object.
-        key: A key for the `VitessceConfig` object. Is used as `key` for a
-            `Collection` in case the `VitessceConfig` object references
-            multiple artifacts.
-        description: A description for the `VitessceConfig` object.
-
-    .. versionchanged:: 0.76.12
-        Now assumes `vitessce-python >= 3.4.0`, which allows passing artifacts within `VitessceConfig`.
-    .. versionchanged:: 0.75.1
-        Now displays the "Vitessce button" on the hub next to the dataset. It additionally keeps displaying it next to the configuration file.
-    .. versionchanged:: 0.70.2
-        No longer saves the dataset. It only saves the `VitessceConfig` object.
+        key: A `key` for the `VitessceConfig` artifact.
+        description: A `description` for the `VitessceConfig` aritifact. Is additionally
+            used as `key` for a `Collection` in case the `VitessceConfig` object
+            references multiple artifacts.
     """
     # can only import here because vitessce is not a dependency
     from vitessce import VitessceConfig
@@ -73,6 +69,8 @@ def save_vitessce_config(
     if len(dataset_artifacts) > 1:
         # if we have more datasets, we should create a collection
         # and attach an action to the collection
+        # consicious use of description for key, see here
+        # https://github.com/laminlabs/lamindb/pull/2997
         collection = Collection(dataset_artifacts, key=description).save()
 
     # create a JSON export
@@ -80,7 +78,11 @@ def save_vitessce_config(
     with open(config_file_local_path, "w") as file:
         json.dump(vc_dict, file)
     vitessce_config_artifact = Artifact(
-        config_file_local_path, key=key, description=description, run=run
+        config_file_local_path,
+        key=key,
+        description=description,
+        run=run,
+        kind="__lamindb_config__",
     ).save()
     slug = ln_setup.settings.instance.slug
     logger.important(