lamindb 0.76.15__tar.gz → 0.76.16__tar.gz

{lamindb-0.76.15 → lamindb-0.76.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.76.15
+Version: 0.76.16
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.9,<3.13
@@ -9,20 +9,22 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Requires-Dist: lnschema_core==0.76.1
+Requires-Dist: lnschema_core==0.76.2
 Requires-Dist: lamin_utils==0.13.7
-Requires-Dist: lamin_cli==0.20.1
+Requires-Dist: lamin_cli==0.21.2
 Requires-Dist: lamindb_setup
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
 Requires-Dist: python-dateutil
-Requires-Dist: anndata>=0.8.0,<=0.10.9
+Requires-Dist: anndata>=0.8.0,<=0.11.1
 Requires-Dist: fsspec
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
 Requires-Dist: lamindb_setup[aws] ; extra == "aws"
-Requires-Dist: bionty==0.52.0 ; extra == "bionty"
+Requires-Dist: bionty==0.52.1 ; extra == "bionty"
+Requires-Dist: cellregistry ; extra == "cellregistry"
+Requires-Dist: clinicore ; extra == "clinicore"
 Requires-Dist: line_profiler ; extra == "dev"
 Requires-Dist: pre-commit ; extra == "dev"
 Requires-Dist: nox ; extra == "dev"
@@ -35,19 +37,29 @@ Requires-Dist: nbproject_test>=0.5.1 ; extra == "dev"
 Requires-Dist: faker-biology ; extra == "dev"
 Requires-Dist: django-schema-graph ; extra == "erdiagram"
 Requires-Dist: readfcs>=1.1.9 ; extra == "fcs"
+Requires-Dist: findrefs ; extra == "findrefs"
 Requires-Dist: lamindb_setup[gcp] ; extra == "gcp"
 Requires-Dist: nbproject==0.10.5 ; extra == "jupyter"
 Requires-Dist: jupytext ; extra == "jupyter"
 Requires-Dist: nbconvert ; extra == "jupyter"
+Requires-Dist: omop ; extra == "omop"
+Requires-Dist: ourprojects ; extra == "ourprojects"
+Requires-Dist: wetlab ; extra == "wetlab"
 Requires-Dist: zarr>=2.16.0 ; extra == "zarr"
 Project-URL: Home, https://github.com/laminlabs/lamindb
 Provides-Extra: aws
 Provides-Extra: bionty
+Provides-Extra: cellregistry
+Provides-Extra: clinicore
 Provides-Extra: dev
 Provides-Extra: erdiagram
 Provides-Extra: fcs
+Provides-Extra: findrefs
 Provides-Extra: gcp
 Provides-Extra: jupyter
+Provides-Extra: omop
+Provides-Extra: ourprojects
+Provides-Extra: wetlab
 Provides-Extra: zarr
 
 [![Stars](https://img.shields.io/github/stars/laminlabs/lamindb?logo=GitHub&color=yellow)](https://github.com/laminlabs/lamindb)

{lamindb-0.76.15 → lamindb-0.76.16}/docs/arrays.ipynb

@@ -33,7 +33,9 @@
     "\n",
     "Because the artifact was validated, querying the `DataFrame` is guaranteed to succeed!\n",
     "\n",
-    "Such within-collection queries are also possible for cloud-backed collections using DuckDB, TileDB, zarr, HDF5, parquet, and other storage backends.\n",
+    "Such within-collection queries are also possible for cloud-backed collections using [DuckDB](https://duckdb.org),\n",
+    "[TileDB](https://tiledb.com), [zarr](https://zarr.readthedocs.io/en/stable), [HDF5](https://docs.h5py.org/en/stable),\n",
+    "[parquet](https://parquet.apache.org), and other storage backends.\n",
     "\n",
     "- For a use case with TileDB, see: {doc}`docs:cellxgene`\n",
     "- For a use case with DuckDB, see: {doc}`docs:rxrx`\n",
@@ -59,7 +61,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "import lamindb as ln"
@@ -92,8 +98,27 @@
    "outputs": [],
    "source": [
     "ln.Artifact(\"s3://lamindb-ci/lndb-storage/pbmc68k.h5ad\").save()\n",
-    "ln.Artifact(\"s3://lamindb-ci/lndb-storage/testfile.hdf5\").save()\n",
-    "ln.Artifact(\"s3://lamindb-ci/lndb-storage/sharded_parquet\").save()"
+    "ln.Artifact(\"s3://lamindb-ci/lndb-storage/testfile.hdf5\").save()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Note that it is also possible to register Hugging Face paths. For this `huggingface_hub` package should be installed."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "ln.Artifact(\"hf://datasets/Koncopd/lamindb-test/sharded_parquet\").save()"
    ]
   },
   {
@@ -131,7 +156,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "adata = artifact.open()"
@@ -274,7 +303,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "backed = artifact.open()"
@@ -325,7 +358,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "artifact = ln.Artifact.get(key=\"lndb-storage/sharded_parquet\")"
+    "artifact = ln.Artifact.get(key=\"sharded_parquet\")"
    ]
   },
   {
@@ -344,7 +377,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "backed = artifact.open()"
@@ -354,7 +391,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "This returns `pyarrow` `Dataset`, see [here](https://arrow.apache.org/docs/python/dataset.html)."
+    "This returns a [pyarrow dataset](https://arrow.apache.org/docs/python/dataset.html)."
    ]
   },
   {

{lamindb-0.76.15 → lamindb-0.76.16}/docs/bio-registries.ipynb

@@ -8,7 +8,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -16,7 +15,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -73,7 +71,7 @@
    "outputs": [],
    "source": [
     "# populate the database with the public ontology\n",
-    "bt.CellType.import_from_source()"
+    "bt.CellType.import_source()"
    ]
   },
   {
@@ -104,8 +102,8 @@
    "outputs": [],
    "source": [
     "# similarly, let's populate the Gene registry with human and mouse genes\n",
-    "bt.Gene.import_from_source(organism=\"human\")\n",
-    "bt.Gene.import_from_source(organism=\"mouse\")"
+    "bt.Gene.import_source(organism=\"human\")\n",
+    "bt.Gene.import_source(organism=\"mouse\")"
    ]
   },
   {
@@ -215,6 +213,39 @@
     "gdt_cell.children.df()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "It is also possible to recursively query parents or children, getting direct parents (children), their parents, and so forth."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "gdt_cell.query_parents().df()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "gdt_cell.query_children().df()"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -238,7 +269,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -246,7 +276,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -293,7 +322,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -322,7 +350,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -356,7 +383,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -446,7 +472,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -463,7 +488,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -598,7 +622,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -606,7 +629,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -627,7 +649,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -743,7 +764,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "py310",
+   "display_name": "lamindb",
    "language": "python",
    "name": "python3"
   },
@@ -757,7 +778,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.11.7"
   },
   "nbproject": {
    "id": "s7F7PezA1HdQ",

lamindb-0.76.16/docs/ehrcurator.py

@@ -0,0 +1,69 @@
+import bionty as bt
+import pandas as pd
+from lamindb.core import DataFrameCurator, Record, logger
+from lamindb.core.types import UPathStr, FieldAttr
+
+__version__ = "0.1.0"
+
+
+class EHRCurator(DataFrameCurator):
+    """Custom curation flow for electronic health record data."""
+
+    def __init__(self, data: pd.DataFrame | UPathStr):
+        # Curate these columns against the specified fields
+        DEFAULT_CATEGORICALS = {
+            "disease": bt.Disease.name,
+            "phenotype": bt.Phenotype.name,
+            "developmental_stage": bt.DevelopmentalStage.name,
+        }
+
+        # If columns or values are missing, we substitute with these defaults
+        DEFAULT_VALUES = {
+            "disease": "normal",
+            "development_stage": "unknown",
+            "phenotype": "unknown",
+        }
+
+        # Validate values onto the following ontology versions
+        DEFAULT_SOURCES = {
+            "disease": bt.Source.get(
+                entity="bionty.Disease", name="mondo", version="2023-04-04"
+            ),
+            "developmental_stage": bt.Source.get(
+                entity="bionty.DevelopmentalStage", name="hsapdv", version="2020-03-10"
+            ),
+            "phenotype": bt.Source.get(
+                entity="bionty.Phenotype",
+                name="hp",
+                version="2023-06-17",
+                organism="human",
+            ),
+        }
+
+        self.data = data
+
+        for col, default in DEFAULT_VALUES.items():
+            if col not in self.data.columns:
+                self.data[col] = default
+            else:
+                self.data[col].fillna(default, inplace=True)
+
+        super().__init__(
+            df=self.data,
+            categoricals=DEFAULT_CATEGORICALS,
+            sources=DEFAULT_SOURCES,
+            organism="human",
+        )
+
+    def validate(self, organism: str | None = None) -> bool:
+        """Validates the internal EHR standard."""
+        missing_columns = {"disease", "phenotype", "developmental_stage", "age"} - set(
+            self.data.columns
+        )
+        if missing_columns:
+            logger.error(
+                f"Columns {', '.join(map(repr, missing_columns))} are missing but required."
+            )
+            return False
+
+        return DataFrameCurator.validate(self, organism)

{lamindb-0.76.15 → lamindb-0.76.16}/docs/faq/acid.ipynb

@@ -1,7 +1,6 @@
 {
  "cells": [
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "41b33aca-1a7d-42b4-afa6-c9a5bccc060c",
    "metadata": {},
@@ -10,7 +9,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "2096971d",
    "metadata": {},
@@ -56,7 +54,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "6f448850-bc49-4f0e-ad6f-f8fd1caec4d3",
    "metadata": {},
@@ -65,7 +62,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "0c582719-42de-4e9d-bdcb-51f8aabb3ca5",
    "metadata": {},
@@ -87,7 +83,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "e4e67dda",
    "metadata": {},
@@ -116,7 +111,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "0d77b8e2",
    "metadata": {},
@@ -137,7 +131,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "cf110322-0011-4a35-9393-0465db3b562f",
    "metadata": {},
@@ -156,7 +149,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "ed3e9cfe-4d07-4d96-8443-d60b62ca82fe",
    "metadata": {},
@@ -177,7 +169,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "a098efc8",
    "metadata": {},
@@ -198,7 +189,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "825e3635",
    "metadata": {},
@@ -218,7 +208,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "5258b578-733b-4eb1-afd0-01144d2b1c47",
    "metadata": {},
@@ -238,8 +227,7 @@
    "outputs": [],
    "source": [
     "!docker stop pgtest && docker rm pgtest\n",
-    "!lamin delete --force pgtest\n",
-    "!rm -r ./test-acid"
+    "!lamin delete --force pgtest"
    ]
   }
  ],
@@ -250,16 +238,16 @@
    "name": "python3"
   },
   "language_info": {
-   "artifact_extension": ".py",
    "codemirror_mode": {
     "name": "ipython",
     "version": 3
    },
+   "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.16"
+   "version": "3.9.17"
   },
   "nbproject": {
    "id": "zz23msudiiQR",

lamindb-0.76.16/docs/faq/search.ipynb

@@ -0,0 +1,151 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "e0293a39",
+   "metadata": {},
+   "source": [
+    "# How does search work?"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "955b6253",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from laminci.db import setup_local_test_postgres"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7f312433",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pgurl = setup_local_test_postgres()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "286aeebc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!lamin init --name benchmark_search --db {pgurl} --schema bionty --storage ./benchmark_search"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "40234d47",
+   "metadata": {},
+   "source": [
+    "Here we show how to perform text search on `Record` and evaluate some search queries."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "47cef058",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import lamindb as ln\n",
+    "import bionty as bt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "83a46a9a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "bt.CellType.import_source()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7991a73f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "SEARCH_QUERIES_EXACT = (\"t cell\", \"stem cell\", \"b cell\", \"regulatory B cell\", \"Be2 cell\", \"adipocyte\")\n",
+    "SEARCH_QUERIES_CONTAINS = (\"t cel\", \"t-cel\", \"neural\", \"kidney\", \"kidne\")\n",
+    "TOP_N = 20"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "682fcd2f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for query in SEARCH_QUERIES_EXACT:\n",
+    "    print(\"Query:\", query)\n",
+    "    qs = bt.CellType.search(query)\n",
+    "    display(qs.df())\n",
+    "    \n",
+    "    assert query.lower() == qs[0].name.lower()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ee059b0d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for query in SEARCH_QUERIES_CONTAINS:\n",
+    "    print(\"Query:\", query)\n",
+    "    qs = bt.CellType.search(query)\n",
+    "    display(qs.df())\n",
+    "\n",
+    "    top_record = qs[0]\n",
+    "    query = query.lower()\n",
+    "    assert query in top_record.name.lower() or query in top_record.synonyms.lower()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "53038793",
+   "metadata": {
+    "tags": [
+     "hide-cell"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "!docker stop pgtest && docker rm pgtest\n",
+    "!lamin delete --force benchmark_search"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.17"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}