lamindb 1.9.0__tar.gz → 1.10.0__tar.gz

{lamindb-1.9.0 → lamindb-1.10.0}/.gitignore

@@ -66,6 +66,7 @@ run-tests
 test-django-validation/
 curate.tiledbsoma
 small_dataset.tiledbsoma
+registered_storage
 
 # General
 .DS_Store

{lamindb-1.9.0 → lamindb-1.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb
-Version: 1.9.0
+Version: 1.10.0
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10,<3.14
@@ -10,8 +10,8 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: lamin_utils==0.15.0
-Requires-Dist: lamin_cli==1.6.0
-Requires-Dist: lamindb_setup[aws]==1.8.2
+Requires-Dist: lamin_cli==1.6.1
+Requires-Dist: lamindb_setup[aws]==1.9.0
 Requires-Dist: pyyaml
 Requires-Dist: pyarrow
 Requires-Dist: pandera>=0.24.0

{lamindb-1.9.0 → lamindb-1.10.0}/docs/arrays.ipynb

@@ -307,6 +307,112 @@
     "adata_subset.to_memory()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## SpatialData"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "It is also possible to access `AnnData` objects inside `SpatialData` `tables`:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "artifact = ln.Artifact.using(\"laminlabs/lamindata\").get(\n",
+    "    key=\"visium_aligned_guide_min.zarr\"\n",
+    ")\n",
+    "\n",
+    "access = artifact.open()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "access"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "access.tables"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This gives you the same `AnnDataAccessor` object as for a normal `AnnData`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "table = access.tables[\"table\"]\n",
+    "\n",
+    "table"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can subset it and read into memory as an actual `AnnData`:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "table_subset = table[table.obs[\"clone\"] == \"diploid\"]\n",
+    "\n",
+    "table_subset"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "```python\n",
+    "adata = table_subset.to_memory()\n",
+    "```"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

{lamindb-1.9.0 → lamindb-1.10.0}/docs/faq/keep-artifacts-local.ipynb

@@ -47,7 +47,7 @@
    },
    "outputs": [],
    "source": [
-    "# !pip install 'lamindb[jupyter]'\n",
+    "# pip install 'lamindb[jupyter]'\n",
     "import lamindb as ln\n",
     "\n",
     "ln.track(\"l9lFf83aPwRc0000\")"
@@ -132,6 +132,23 @@
     "ln.Storage.df()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can now set it as a local default storage location.\n",
+    "Next time you connect to the instance, this won't be necessary and the location will be automatically detected as the local default."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ln.settings.local_storage = \"./our_local_storage\""
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -336,7 +353,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "artifact.save(upload=True)"
@@ -352,7 +373,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "ln.settings.storage.root.view_tree()"
@@ -368,7 +393,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "assert artifact.path.exists()\n",
@@ -394,7 +423,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "filepath = ln.core.datasets.file_mini_csv()\n",
@@ -412,7 +445,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "ln.Artifact.df(include=\"storage__root\")"
@@ -435,7 +472,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "storage_record = ln.Storage.get(root__endswith=\"our_local_storage\")\n",
@@ -461,7 +502,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-ouput"
+    ]
+   },
    "outputs": [],
    "source": [
     "artifact.delete(permanent=True)\n",

{lamindb-1.9.0 → lamindb-1.10.0}/docs/registries.ipynb

@@ -45,7 +45,7 @@
    "id": "4",
    "metadata": {},
    "source": [
-    "Let's start by creating a few exemplary datasets and saving them into a LaminDB instance using, e.g., {func}`~lamindb.examples.ingest_mini_immuno_datasets`."
+    "Let's start by creating a few exemplary datasets and saving them into a LaminDB instance using, e.g., {func}`~lamindb.examples.datasets.mini_immuno.save_mini_immuno_datasets`."
    ]
   },
   {
@@ -66,7 +66,7 @@
     "ln.Artifact(ln.core.datasets.file_jpg_paradisi05(), key=\"images/my_image.jpg\").save()\n",
     "ln.Artifact(ln.core.datasets.file_fastq(), key=\"raw/my_fastq.fastq.gz\").save()\n",
     "ln.Artifact.from_df(ln.core.datasets.df_iris(), key=\"iris.parquet\").save()\n",
-    "ln.examples.ingest_mini_immuno_datasets()"
+    "ln.examples.datasets.mini_immuno.save_mini_immuno_datasets()"
    ]
   },
   {

{lamindb-1.9.0 → lamindb-1.10.0}/docs/storage/vitessce.ipynb

@@ -109,6 +109,7 @@
     "assert dataset_artifact._actions.get() == vitessce_config_artifact\n",
     "assert vitessce_config_artifact._action_targets.get() == dataset_artifact\n",
     "assert vitessce_config_artifact._actions.first() is None\n",
+    "assert vitessce_config_artifact.kind == \"__lamindb_config__\"\n",
     "assert ln.Artifact.get(_actions=vitessce_config_artifact) == dataset_artifact"
    ]
   },
@@ -163,7 +164,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "py310",
+   "display_name": "py312",
    "language": "python",
    "name": "python3"
   },
@@ -177,7 +178,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.12.8"
   }
  },
  "nbformat": 4,

{lamindb-1.9.0 → lamindb-1.10.0}/lamindb/__init__.py

@@ -108,7 +108,7 @@ Backwards compatibility.
 
 # ruff: noqa: I001
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "1.9.0"
+__version__ = "1.10.0"
 
 import warnings
 

{lamindb-1.9.0 → lamindb-1.10.0}/lamindb/core/__init__.py

@@ -28,7 +28,6 @@ Modules:
 .. autosummary::
    :toctree: .
 
-   datasets
    storage
    logger
 
@@ -38,7 +37,8 @@ from lamin_utils import logger
 from lamin_utils._inspect import InspectResult
 
 from .. import errors as exceptions
-from . import datasets, loaders, subsettings, types
+from ..examples import datasets  # backward compat
+from . import loaders, subsettings, types
 from ._context import Context
 from ._mapped_collection import MappedCollection
 from ._settings import Settings

{lamindb-1.9.0 → lamindb-1.10.0}/lamindb/core/storage/__init__.py

@@ -13,12 +13,13 @@ Array accessors.
    :toctree: .
 
    AnnDataAccessor
+   SpatialDataAccessor
    BackedAccessor
 """
 
 from lamindb_setup.core.upath import LocalPathClasses, UPath, infer_filesystem
 
-from ._backed_access import AnnDataAccessor, BackedAccessor
+from ._backed_access import AnnDataAccessor, BackedAccessor, SpatialDataAccessor
 from ._tiledbsoma import save_tiledbsoma_experiment
 from ._valid_suffixes import VALID_SUFFIXES
 from .objects import infer_suffix, write_to_disk

{lamindb-1.9.0 → lamindb-1.10.0}/lamindb/core/storage/_anndata_accessor.py

@@ -353,7 +353,16 @@ if ZARR_INSTALLED:
         attrs_keys: dict[str, list] = {}
         obs_var_arrays = []
 
-        for path in paths:
+        prefix = storage.path
+        if prefix == "":
+            paths_iter = (path for path in paths)
+        else:
+            prefix += "/"
+            paths_iter = (
+                path.removeprefix(prefix) for path in paths if path.startswith(prefix)
+            )
+
+        for path in paths_iter:
             if path in (".zattrs", ".zgroup"):
                 continue
             parts = path.split("/")

{lamindb-1.9.0 → lamindb-1.10.0}/lamindb/core/storage/_backed_access.py

@@ -9,6 +9,7 @@ from anndata._io.specs.registry import get_spec
 from ._anndata_accessor import AnnDataAccessor, StorageType, registry
 from ._polars_lazy_df import POLARS_SUFFIXES, _open_polars_lazy_df
 from ._pyarrow_dataset import PYARROW_SUFFIXES, _open_pyarrow_dataset
+from ._spatialdata_accessor import SpatialDataAccessor
 from ._tiledbsoma import _open_tiledbsoma
 from .paths import filepath_from_artifact
 
@@ -80,6 +81,7 @@ def backed_access(
     **kwargs,
 ) -> (
     AnnDataAccessor
+    | SpatialDataAccessor
     | BackedAccessor
     | SOMACollection
     | SOMAExperiment
@@ -110,6 +112,8 @@ def backed_access(
         conn, storage = registry.open("h5py", objectpath, mode=mode, **kwargs)
     elif suffix == ".zarr":
         conn, storage = registry.open("zarr", objectpath, mode=mode, **kwargs)
+        if "spatialdata_attrs" in storage.attrs:
+            return SpatialDataAccessor(storage, name)
     elif len(df_suffixes := _flat_suffixes(objectpath)) == 1 and (
         df_suffix := df_suffixes.pop()
     ) in set(PYARROW_SUFFIXES).union(POLARS_SUFFIXES):

lamindb-1.10.0/lamindb/core/storage/_spatialdata_accessor.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from functools import cached_property
+from typing import TYPE_CHECKING
+
+from ._anndata_accessor import AnnDataAccessor
+
+if TYPE_CHECKING:
+    from zarr import Group
+
+
+class _TablesAccessor:
+    def __init__(self, tables: Group):
+        self._tables = tables
+
+    def __getitem__(self, key: str) -> AnnDataAccessor:
+        return AnnDataAccessor(connection=None, storage=self._tables[key], filename=key)
+
+    def keys(self) -> list[str]:
+        return list(self._tables.keys())
+
+    def __repr__(self) -> str:
+        """Description of the _TablesAccessor object."""
+        descr = (
+            f"Accessor for the SpatialData attribute tables\n  with keys: {self.keys()}"
+        )
+        return descr
+
+
+class SpatialDataAccessor:
+    """Cloud-backed SpatialData.
+
+    For now only allows to access `tables`.
+    """
+
+    def __init__(self, storage: Group, name: str):
+        self.storage = storage
+        self._name = name
+
+    @cached_property
+    def tables(self) -> _TablesAccessor:
+        """tables of the underlying SpatialData object."""
+        return _TablesAccessor(self.storage["tables"])
+
+    def __repr__(self):
+        """Description of the SpatialDataAccessor object."""
+        descr = (
+            "SpatialDataAccessor object"
+            f"\n  constructed for the SpatialData object {self._name}"
+            f"\n    with tables: {self.tables.keys()}"
+        )
+        return descr

{lamindb-1.9.0 → lamindb-1.10.0}/lamindb/curators/core.py

@@ -1236,6 +1236,9 @@ class CatVector:
         registry = self._field.field.model
         field_name = self._field.field.name
         non_validated_records: SQLRecordList[Any] = []  # type: ignore
+        type_record = None
+        if self._subtype_str != "" and "=" not in self._subtype_str:
+            type_record = registry.get(name=self._subtype_str)
         if df is not None and registry == Feature:
             nonval_columns = Feature.inspect(df.columns, mute=True).non_validated
             non_validated_records = Feature.from_df(df.loc[:, nonval_columns])
@@ -1252,6 +1255,9 @@ class CatVector:
                 init_kwargs = {field_name: value}
                 if registry == Feature:
                     init_kwargs["dtype"] = "cat" if dtype is None else dtype
+                if type_record is not None:
+                    # if subtype_str is set, we need to set the type for new records
+                    init_kwargs["type"] = type_record
                 non_validated_records.append(registry(**init_kwargs, **create_kwargs))
         if len(non_validated_records) > 0:
             ln_save(non_validated_records)

lamindb-1.10.0/lamindb/examples/__init__.py

@@ -0,0 +1,14 @@
+"""Examples.
+
+.. autosummary::
+   :toctree: .
+
+   schemas
+   datasets
+   cellxgene
+   croissant
+
+"""
+
+from . import croissant, datasets, schemas
+from .cellxgene import _cellxgene

{lamindb-1.9.0 → lamindb-1.10.0}/lamindb/examples/cellxgene/_cellxgene.py

@@ -1,12 +1,16 @@
-from typing import Collection, Literal, NamedTuple
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Collection, Literal, NamedTuple
 
 import pandas as pd
 from lamindb_setup.core.upath import UPath
 
-from lamindb.base.types import FieldAttr
-from lamindb.models import Feature, Schema, SQLRecord, ULabel
 from lamindb.models._from_values import _format_values
 
+if TYPE_CHECKING:
+    from lamindb.base.types import FieldAttr
+    from lamindb.models import Schema, SQLRecord
+
 CELLxGENESchemaVersions = Literal["4.0.0", "5.0.0", "5.1.0", "5.2.0", "5.3.0"]
 FieldType = Literal["ontology_id", "name"]
 
@@ -25,6 +29,8 @@ def save_cxg_defaults() -> None:
     """
     import bionty as bt
 
+    from lamindb.models import ULabel
+
     # "normal" in Disease
     normal = bt.Phenotype.from_source(
         ontology_id="PATO:0000461",
@@ -135,6 +141,8 @@ def get_cxg_schema(
     """
     import bionty as bt
 
+    from lamindb.models import Feature, Schema, ULabel
+
     class CategorySpec(NamedTuple):
         field: str | FieldAttr
         default: str | None

lamindb-1.10.0/lamindb/examples/croissant/__init__.py

@@ -0,0 +1,44 @@
+"""Example Croissant files.
+
+Examples for MLCommons Croissant files, which are used to store metadata about datasets.
+"""
+
+import json
+from pathlib import Path
+
+
+def mini_immuno(n_files: int = 1) -> list[Path]:
+    """Return paths to the mini immuno dataset and its metadata as a Croissant file.
+
+    Args:
+        n_files: Number of files inside the croissant file. Default is 1.
+    """
+    from ..datasets import file_mini_csv
+    from ..datasets.mini_immuno import get_dataset1
+
+    adata = get_dataset1(otype="AnnData")
+    dataset1_path = Path("mini_immuno.anndata.zarr")
+    adata.write_zarr(dataset1_path)
+    orig_croissant_path = (
+        Path(__file__).parent / "mini_immuno.anndata.zarr_metadata.json"
+    )
+    with open(orig_croissant_path, encoding="utf-8") as f:
+        data = json.load(f)
+    if n_files == 2:
+        dataset2_path = file_mini_csv()
+        data["distribution"].append(
+            {
+                "@type": "sc:FileObject",
+                "@id": "mini.csv",
+                "name": "mini.csv",
+                "encodingFormat": "text/csv",
+            }
+        )
+    croissant_path = Path("mini_immuno.anndata.zarr_metadata.json")
+    with open(croissant_path, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=2)
+    result: list[Path] = [croissant_path, dataset1_path]
+    if n_files == 1:
+        return result
+    result.append(dataset2_path)
+    return result

lamindb-1.10.0/lamindb/examples/croissant/mini_immuno.anndata.zarr_metadata.json

@@ -0,0 +1,73 @@
+{
+  "@context": {
+    "@vocab": "https://schema.org/",
+    "cr": "https://mlcommons.org/croissant/",
+    "ml": "http://ml-schema.org/",
+    "sc": "https://schema.org/",
+    "dct": "http://purl.org/dc/terms/",
+    "data": "https://mlcommons.org/croissant/data/",
+    "rai": "https://mlcommons.org/croissant/rai/",
+    "format": "https://mlcommons.org/croissant/format/",
+    "citeAs": "https://mlcommons.org/croissant/citeAs/",
+    "conformsTo": "https://mlcommons.org/croissant/conformsTo/",
+    "@language": "en",
+    "repeated": "https://mlcommons.org/croissant/repeated/",
+    "field": "https://mlcommons.org/croissant/field/",
+    "examples": "https://mlcommons.org/croissant/examples/",
+    "recordSet": "https://mlcommons.org/croissant/recordSet/",
+    "fileObject": "https://mlcommons.org/croissant/fileObject/",
+    "fileSet": "https://mlcommons.org/croissant/fileSet/",
+    "source": "https://mlcommons.org/croissant/source/",
+    "references": "https://mlcommons.org/croissant/references/",
+    "key": "https://mlcommons.org/croissant/key/",
+    "parentField": "https://mlcommons.org/croissant/parentField/",
+    "isLiveDataset": "https://mlcommons.org/croissant/isLiveDataset/",
+    "separator": "https://mlcommons.org/croissant/separator/",
+    "extract": "https://mlcommons.org/croissant/extract/",
+    "subField": "https://mlcommons.org/croissant/subField/",
+    "regex": "https://mlcommons.org/croissant/regex/",
+    "column": "https://mlcommons.org/croissant/column/",
+    "path": "https://mlcommons.org/croissant/path/",
+    "fileProperty": "https://mlcommons.org/croissant/fileProperty/",
+    "md5": "https://mlcommons.org/croissant/md5/",
+    "jsonPath": "https://mlcommons.org/croissant/jsonPath/",
+    "transform": "https://mlcommons.org/croissant/transform/",
+    "replace": "https://mlcommons.org/croissant/replace/",
+    "dataType": "https://mlcommons.org/croissant/dataType/",
+    "includes": "https://mlcommons.org/croissant/includes/",
+    "excludes": "https://mlcommons.org/croissant/excludes/"
+  },
+  "@type": "Dataset",
+  "name": "Mini immuno dataset",
+  "description": "A few samples from the immunology dataset",
+  "url": "https://lamin.ai/laminlabs/lamindata/artifact/tCUkRcaEjTjhtozp0000",
+  "creator": {
+    "@type": "Person",
+    "name": "falexwolf"
+  },
+  "dateCreated": "2025-07-16",
+  "cr:projectName": "Mini Immuno Project",
+  "datePublished": "2025-07-16",
+  "version": "1.0",
+  "license": "https://creativecommons.org/licenses/by/4.0/",
+  "citation": "Please cite this dataset as: mini immuno (2025)",
+  "encodingFormat": "zarr",
+  "distribution": [
+    {
+      "@type": "cr:FileSet",
+      "@id": "mini_immuno.anndata.zarr",
+      "containedIn": {
+        "@id": "directory"
+      },
+      "encodingFormat": "zarr"
+    }
+  ],
+  "cr:recordSet": [
+    {
+      "@type": "cr:RecordSet",
+      "@id": "#samples",
+      "name": "samples",
+      "description": "my sample"
+    }
+  ]
+}

{lamindb-1.9.0/lamindb/core → lamindb-1.10.0/lamindb/examples}/datasets/__init__.py

@@ -1,4 +1,4 @@
-"""Test datasets.
+"""Example datasets.
 
 The mini immuno dataset.