lamindb 1.10.0__tar.gz → 1.10.2__tar.gz

{lamindb-1.10.0 → lamindb-1.10.2}/.github/workflows/build.yml

@@ -69,7 +69,7 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: |
-            ${{ github.ref == 'refs/heads/release' && '3.10' ||
+            ${{ github.ref == 'refs/heads/release' && '3.11' ||
                 '3.13'
                 }}
 

{lamindb-1.10.0 → lamindb-1.10.2}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: lamindb
-Version: 1.10.0
+Version: 1.10.2
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10,<3.14
@@ -9,9 +9,10 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+License-File: LICENSE
 Requires-Dist: lamin_utils==0.15.0
 Requires-Dist: lamin_cli==1.6.1
-Requires-Dist: lamindb_setup[aws]==1.9.0
+Requires-Dist: lamindb_setup[aws]==1.9.1
 Requires-Dist: pyyaml
 Requires-Dist: pyarrow
 Requires-Dist: pandera>=0.24.0
@@ -19,7 +20,7 @@ Requires-Dist: typing_extensions!=4.6.0
 Requires-Dist: python-dateutil
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: scipy<1.15.0
-Requires-Dist: anndata>=0.8.0,<=0.12.0
+Requires-Dist: anndata>=0.8.0,<=0.12.1
 Requires-Dist: fsspec
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary

{lamindb-1.10.0 → lamindb-1.10.2}/docs/arrays.ipynb

@@ -102,62 +102,6 @@
     "ln.Artifact(\"s3://lamindb-ci/test-arrays/testfile.hdf5\").save()"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Note that it is also possible to register Hugging Face paths. For this `huggingface_hub` package should be installed."
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We register a folder of `parquet` files as a single artifact."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "tags": [
-     "hide-output"
-    ]
-   },
-   "outputs": [],
-   "source": [
-    "ln.Artifact(\"hf://datasets/Koncopd/lamindb-test/sharded_parquet\").save()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "We also register a collection of individual `parquet` files."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "tags": [
-     "hide-output"
-    ]
-   },
-   "outputs": [],
-   "source": [
-    "artifact_shard1 = ln.Artifact(\n",
-    "    \"hf://datasets/Koncopd/lamindb-test/sharded_parquet/louvain=0/947eee0b064440c9b9910ca2eb89e608-0.parquet\"\n",
-    ").save()\n",
-    "artifact_shard2 = ln.Artifact(\n",
-    "    \"hf://datasets/Koncopd/lamindb-test/sharded_parquet/louvain=1/947eee0b064440c9b9910ca2eb89e608-0.parquet\"\n",
-    ").save()\n",
-    "\n",
-    "ln.Collection(\n",
-    "    [artifact_shard1, artifact_shard2], key=\"sharded_parquet_collection\"\n",
-    ").save()"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -495,13 +439,20 @@
     "A dataframe stored as sharded `parquet`."
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Note that it is also possible to register and access Hugging Face paths. For this `huggingface_hub` package should be installed."
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "artifact = ln.Artifact.get(key=\"sharded_parquet\")"
+    "artifact = ln.Artifact.using(\"laminlabs/lamindata\").get(key=\"sharded_parquet\")"
    ]
   },
   {
@@ -572,13 +523,19 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "collection = ln.Collection.get(key=\"sharded_parquet_collection\")"
+    "collection = ln.Collection.using(\"laminlabs/lamindata\").get(\n",
+    "    key=\"sharded_parquet_collection\"\n",
+    ")"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "backed = collection.open()"
@@ -663,7 +620,7 @@
    "outputs": [],
    "source": [
     "# clean up test instance\n",
-    "!lamin delete --force test-arrays"
+    "ln.setup.delete(\"test-arrays\", force=True)"
    ]
   }
  ],

{lamindb-1.10.0 → lamindb-1.10.2}/docs/faq/pydantic-pandera.ipynb

@@ -11,7 +11,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "This doc explains conceptual differences between data validation with `pydantic`, `pandera`, and `lamindb`."
+    "This doc explains conceptual differences between data validation with `pydantic`, `pandera`, and `LaminDB`."
    ]
   },
   {
@@ -42,10 +42,12 @@
    "source": [
     "import pandas as pd\n",
     "import pydantic\n",
-    "from typing import Literal\n",
     "import lamindb as ln\n",
     "import bionty as bt\n",
-    "import pandera\n",
+    "import pandera.pandas as pandera\n",
+    "import pprint\n",
+    "\n",
+    "from typing import Literal, Any\n",
     "\n",
     "df = ln.core.datasets.small_dataset1()\n",
     "df"
@@ -102,7 +104,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Define the Pandera schema using DataFrameSchema\n",
     "pandera_schema = pandera.DataFrameSchema(\n",
     "    {\n",
     "        \"perturbation\": pandera.Column(\n",
@@ -127,14 +128,15 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### lamindb"
+    "### LaminDB"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Features & labels are defined on the level of the database instance. You can either define a schema with required (and optional) columns."
+    "Features & labels are defined on the level of the database instance.\n",
+    "You can either define a schema with required (and optional) columns."
    ]
   },
   {
@@ -147,8 +149,8 @@
    },
    "outputs": [],
    "source": [
-    "ln.ULabel(name=\"DMSO\").save()  # define a DMSO label\n",
-    "ln.ULabel(name=\"IFNG\").save()  # define an IFNG label\n",
+    "ln.ULabel(name=\"DMSO\").save()\n",
+    "ln.ULabel(name=\"IFNG\").save()\n",
     "\n",
     "# leverage ontologies through types ln.ULabel, bt.CellType, bt.ExperimentalFactor\n",
     "lamindb_schema = ln.Schema(\n",
@@ -308,7 +310,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### lamindb"
+    "### LaminDB"
    ]
   },
   {
@@ -317,7 +319,8 @@
    "source": [
     "Because the term `\"CD8-positive, alpha-beta T cell\"` is part of the public `CellType` ontology, validation passes the first time.\n",
     "\n",
-    "If validation and not passed, we could have resolved the issue simply by adding a new term to the `CellType` registry rather than editing the code. This also puts downstream data scientists into a position to update ontologies."
+    "If validation had not passed, we could have resolved the issue simply by adding a new term to the `CellType` registry rather than editing the code.\n",
+    "This also puts downstream data scientists into a position to update ontologies."
    ]
   },
   {
@@ -380,7 +383,7 @@
    "source": [
     "## Overview of validation properties\n",
     "\n",
-    "Importantly, LaminDB offers not only a `DataFrameCurator`, but also a `AnnDataCurator`, `MuDataCurator`, `SpatialDataCurator`, `TiledbsomaCurator`.\n",
+    "Importantly, LaminDB offers not only a `DataFrameCurator`, but also a `AnnDataCurator`, `MuDataCurator`, `SpatialDataCurator`, and `TiledbsomaCurator`.\n",
     "\n",
     "The below overview only concerns validating dataframes."
    ]
@@ -464,7 +467,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "If you don't expect a need for Curator functionality for updating ontologies and standaridization, you can also use the `Artifact` constructor."
+    "If you don't expect a need for Curator functionality for updating ontologies and standardization, you can also use the `Artifact` constructor."
    ]
   },
   {
@@ -561,11 +564,242 @@
    "source": [
     "artifact.schema.features.df()"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Nested data with dynamic keys"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We will now examine another more complex example where data is nested with potentially arbitrary (dynamic) keys.\n",
+    "The example is inspired by the [CELLxGENE schema](https://github.com/chanzuckerberg/single-cell-curation/blob/main/schema/6.0.0/schema.md#uns-dataset-metadata) where annotations are stored as dictionaries in the AnnData `.uns` slot."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "uns_dict = ln.core.datasets.dict_cxg_uns()\n",
+    "pprint.pprint(uns_dict)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### pydantic"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Pydantic is primed to deal with nested data."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class Images(pydantic.BaseModel):\n",
+    "    fullres: str\n",
+    "    hires: str\n",
+    "\n",
+    "\n",
+    "class Scalefactors(pydantic.BaseModel):\n",
+    "    spot_diameter_fullres: float\n",
+    "    tissue_hires_scalef: float\n",
+    "\n",
+    "\n",
+    "class Library(pydantic.BaseModel):\n",
+    "    images: Images\n",
+    "    scalefactors: Scalefactors\n",
+    "\n",
+    "\n",
+    "class Spatial(pydantic.BaseModel):\n",
+    "    is_single: bool\n",
+    "    model_config = {\"extra\": \"allow\"}\n",
+    "\n",
+    "    def __init__(self, **data):\n",
+    "        libraries = {}\n",
+    "        other_fields = {}\n",
+    "\n",
+    "        # store all libraries under a single key for validation\n",
+    "        for key, value in data.items():\n",
+    "            if key.startswith(\"library_\"):\n",
+    "                libraries[key] = Library(**value)\n",
+    "            else:\n",
+    "                other_fields[key] = value\n",
+    "\n",
+    "        other_fields[\"libraries\"] = libraries\n",
+    "        super().__init__(**other_fields)\n",
+    "\n",
+    "\n",
+    "class SpatialDataSchema(pydantic.BaseModel):\n",
+    "    organism_ontology_term_id: str\n",
+    "    spatial: Spatial\n",
+    "\n",
+    "\n",
+    "validated_data = SpatialDataSchema(**uns_dict)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "However, pydantic either requires all dictionary keys to be known beforehand to construct the Model classes or workarounds to collect all keys for a single model."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### pandera"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Pandera cannot validate dictionaries because it is designed for structured dataframe data.\n",
+    "Therefore, we need to flatten the dictionary to transform it into a DataFrame:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def _flatten_dict(d: dict[Any, Any], parent_key: str = \"\", sep: str = \"_\"):\n",
+    "    items = []\n",
+    "    for k, v in d.items():\n",
+    "        new_key = f\"{parent_key}{sep}{k}\" if parent_key else k\n",
+    "        if isinstance(v, dict):\n",
+    "            items.extend(_flatten_dict(v, new_key, sep=sep).items())\n",
+    "        else:\n",
+    "            items.append((new_key, v))\n",
+    "    return dict(items)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def create_dynamic_schema(flattened_data: dict[str, Any]):\n",
+    "    schema_dict = {\n",
+    "        \"organism_ontology_term_id\": pandera.Column(str),\n",
+    "        \"spatial_is_single\": pandera.Column(bool),\n",
+    "    }\n",
+    "\n",
+    "    for key in flattened_data.keys():\n",
+    "        if key.startswith(\"spatial_library_\") and key.endswith(\"_images_fullres\"):\n",
+    "            lib_prefix = key.replace(\"_images_fullres\", \"\")\n",
+    "            schema_dict.update(\n",
+    "                {\n",
+    "                    f\"{lib_prefix}_images_fullres\": pandera.Column(str),\n",
+    "                    f\"{lib_prefix}_images_hires\": pandera.Column(str),\n",
+    "                    f\"{lib_prefix}_scalefactors_spot_diameter_fullres\": pandera.Column(\n",
+    "                        float\n",
+    "                    ),\n",
+    "                    f\"{lib_prefix}_scalefactors_tissue_hires_scalef\": pandera.Column(\n",
+    "                        float\n",
+    "                    ),\n",
+    "                }\n",
+    "            )\n",
+    "\n",
+    "    return pandera.DataFrameSchema(schema_dict)\n",
+    "\n",
+    "\n",
+    "flattened = _flatten_dict(uns_dict)\n",
+    "df = pd.DataFrame([flattened])\n",
+    "spatial_schema = create_dynamic_schema(flattened)\n",
+    "validated_df = spatial_schema.validate(df)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Analogously to pydantic, pandera does not have out of the box support for dynamically named keys.\n",
+    "Therefore, it is necessary to dynamically construct a pydantic schema."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### LaminDB"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Similarly, LaminDB currently requires constructing flattened dataframes to dynamically create features for the schema, which can then be used for validation with the DataFrameCurator.\n",
+    "Future improvements are expected, including support for a dictionary-specific curator."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "def create_dynamic_schema(flattened_data: dict[str, Any]) -> ln.Schema:\n",
+    "    features = []\n",
+    "\n",
+    "    for key, value in flattened_data.items():\n",
+    "        if key == \"organism_ontology_term_id\":\n",
+    "            features.append(ln.Feature(name=key, dtype=bt.Organism.ontology_id).save())\n",
+    "        elif isinstance(value, bool):\n",
+    "            features.append(ln.Feature(name=key, dtype=bool).save())\n",
+    "        elif isinstance(value, (int, float)):\n",
+    "            features.append(ln.Feature(name=key, dtype=float).save())\n",
+    "        else:\n",
+    "            features.append(ln.Feature(name=key, dtype=str).save())\n",
+    "\n",
+    "    return ln.Schema(\n",
+    "        name=\"Spatial data schema\", features=features, coerce_dtype=True\n",
+    "    ).save()\n",
+    "\n",
+    "\n",
+    "flattened = _flatten_dict(uns_dict)\n",
+    "flattened_df = pd.DataFrame([flattened])\n",
+    "spatial_schema = create_dynamic_schema(flattened)\n",
+    "curator = ln.curators.DataFrameCurator(flattened_df, spatial_schema)\n",
+    "curator.validate()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "```{note}\n",
+    "Curators for scverse data structures allow for the specification of schema slots that access and validate dataframes in nested dictionary attributes like `.attrs` or `.uns`.\n",
+    "These schema slots use colon-separated paths like `'attrs:sample'` or `'uns:spatial:images'` to to target specific dataframes for validation.\n",
+    "```"
+   ]
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "py311",
+   "display_name": "lamindb",
    "language": "python",
    "name": "python3"
   },
@@ -579,7 +813,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.11"
+   "version": "3.12.8"
   }
  },
  "nbformat": 4,

{lamindb-1.10.0 → lamindb-1.10.2}/lamindb/__init__.py

@@ -108,7 +108,7 @@ Backwards compatibility.
 
 # ruff: noqa: I001
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "1.10.0"
+__version__ = "1.10.2"
 
 import warnings
 

{lamindb-1.10.0 → lamindb-1.10.2}/lamindb/core/_settings.py

@@ -10,6 +10,7 @@ from lamindb_setup import settings as setup_settings
 from lamindb_setup._set_managed_storage import set_managed_storage
 from lamindb_setup.core import deprecated
 from lamindb_setup.core._settings_instance import sanitize_git_repo_url
+from lamindb_setup.core._settings_storage import StorageSettings
 
 from .subsettings._annotation_settings import AnnotationSettings, annotation_settings
 from .subsettings._creation_settings import CreationSettings, creation_settings
@@ -18,7 +19,6 @@ if TYPE_CHECKING:
     from collections.abc import Mapping
     from pathlib import Path
 
-    from lamindb_setup.core._settings_storage import StorageSettings
     from upath import UPath
 
 
@@ -193,13 +193,39 @@ class Settings:
 
     @storage.setter
     def storage(self, path_kwargs: str | Path | UPath | tuple[str | UPath, Mapping]):
+        import lamindb as ln
+
         if isinstance(path_kwargs, tuple):
             path, kwargs = path_kwargs
+            # we should ultimately deprecate passing host here, I think
             if isinstance(kwargs, str):
                 kwargs = {"host": kwargs}
         else:
             path, kwargs = path_kwargs, {}
-        set_managed_storage(path, **kwargs)
+        ssettings = StorageSettings(root=path)  # there is no need to pass kwargs here!
+        exists = ln.Storage.filter(root=ssettings.root_as_str).one_or_none()
+        if exists is None:
+            response = input(
+                f"Storage location {ssettings.root_as_str} does not yet exist. Do you want to continue with creating it? (y/n)"
+            )
+            # logger.warning(f"deprecated call because storage location does **not yet** exist; going forward, please create through ln.Storage(root={path}).save() going forward")
+            if response != "y":
+                return None
+            set_managed_storage(path, **kwargs)
+        else:
+            if exists.instance_uid != ln_setup.settings.instance.uid:
+                raise ValueError(
+                    f"Storage {ssettings.root_as_str} exists in another instance ({exists.instance_uid}), cannot write to it from here."
+                )
+            ssettings = StorageSettings(
+                root=exists.root,
+                region=exists.region,
+                uid=exists.uid,
+                instance_id=ln_setup.settings.instance._id,
+            )
+            ln_setup.settings.instance._storage = ssettings
+            kwargs.pop("host", None)  # host is not needed for existing storage
+            settings.storage._set_fs_kwargs(**kwargs)
 
     @property
     def instance_uid(self) -> str:
@@ -223,6 +249,23 @@ class Settings:
 
     @local_storage.setter
     def local_storage(self, local_root: Path):
+        import lamindb as ln
+
+        # note duplication with storage setter!
+        ssettings = StorageSettings(root=local_root)
+        exists = ln.Storage.filter(root=ssettings.root_as_str).one_or_none()
+        if exists is None:
+            response = input(
+                f"Storage location {ssettings.root_as_str} does not yet exist. Do you want to continue with creating it? (y/n)"
+            )
+            # logger.warning(f"deprecated call because storage location does **not yet** exist; going forward, please create through ln.Storage(root={path}).save() going forward")
+            if response != "y":
+                return None
+        else:
+            if exists.instance_uid != ln_setup.settings.instance.uid:
+                raise ValueError(
+                    f"Storage {ssettings.root_as_str} exists in another instance ({exists.instance_uid}), cannot write to it from here."
+                )
         ln_setup.settings.instance.local_storage = local_root
 
     @property