lamindb 0.76.12__tar.gz → 0.76.14__tar.gz

{lamindb-0.76.12 → lamindb-0.76.14}/.gitignore

@@ -1,8 +1,13 @@
+__MACOSX/
+
 # LaminDB
+*.db
+*.lndb
+*.jpg
+*.zarr/
 docsbuild/
 docs/lamin.md
 docs/guide/data-validation.ipynb
-docs/guide/knowledge.ipynb
 docs/guide/bionty.ipynb
 docs/guide/lnschema-core.ipynb
 docs/paradisi05_laminopathic_nuclei.jpg
@@ -15,19 +20,10 @@ lamin-tutorial/
 mytest/
 rds/
 mydb/
-bfx-output/
-scrnaseq-cellranger/
-scrnaseq-cellranger-schmidt22/
-cellranger_run_001
 docs/test-annotate-flexible/
-__MACOSX/
 docs/lamindb.*
 lamin_sphinx
 docs/conf.py
-*.db
-*.lndb
-*.jpg
-*.zarr/
 lamindb/setup/.env
 _secrets.py
 _configuration.py
@@ -56,9 +52,6 @@ test-inherit1
 test-inherit2
 test-search0
 test-search1
-test-search2
-test-search3
-test-search4
 test-search5
 default_storage
 default_storage_unit_core

{lamindb-0.76.12 → lamindb-0.76.14}/.gitmodules

@@ -19,3 +19,12 @@
 [submodule "sub/findrefs"]
 	path = sub/findrefs
 	url = https://github.com/laminlabs/findrefs
+[submodule "sub/ourprojects"]
+	path = sub/ourprojects
+	url = https://github.com/laminlabs/ourprojects
+[submodule "sub/cellregistry"]
+	path = sub/cellregistry
+	url = https://github.com/laminlabs/cellregistry
+[submodule "sub/omop"]
+	path = sub/omop
+	url = https://github.com/laminlabs/omop.git

{lamindb-0.76.12 → lamindb-0.76.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.76.12
+Version: 0.76.14
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.9
@@ -8,9 +8,9 @@ Description-Content-Type: text/markdown
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.75.0
-Requires-Dist: lamin_utils==0.13.6
-Requires-Dist: lamin_cli==0.18.0
+Requires-Dist: lnschema_core==0.76.0
+Requires-Dist: lamin_utils==0.13.7
+Requires-Dist: lamin_cli==0.20.1
 Requires-Dist: lamindb_setup
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
@@ -23,7 +23,7 @@ Requires-Dist: pandas
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
 Requires-Dist: lamindb_setup[aws] ; extra == "aws"
-Requires-Dist: bionty==0.51.2 ; extra == "bionty"
+Requires-Dist: bionty==0.52.0 ; extra == "bionty"
 Requires-Dist: line_profiler ; extra == "dev"
 Requires-Dist: pre-commit ; extra == "dev"
 Requires-Dist: nox ; extra == "dev"
@@ -37,7 +37,7 @@ Requires-Dist: faker-biology ; extra == "dev"
 Requires-Dist: django-schema-graph ; extra == "erdiagram"
 Requires-Dist: readfcs>=1.1.8 ; extra == "fcs"
 Requires-Dist: lamindb_setup[gcp] ; extra == "gcp"
-Requires-Dist: nbproject==0.10.4 ; extra == "jupyter"
+Requires-Dist: nbproject==0.10.5 ; extra == "jupyter"
 Requires-Dist: jupytext ; extra == "jupyter"
 Requires-Dist: nbconvert ; extra == "jupyter"
 Requires-Dist: zarr>=2.16.0 ; extra == "zarr"

{lamindb-0.76.12 → lamindb-0.76.14}/docs/api.md

@@ -17,9 +17,12 @@ cli
 
 lamindb
 bionty
+ourprojects
+findrefs
+cellregistry
 wetlab
 clinicore
-findrefs
+omop
 ```
 
 ```{toctree}

{lamindb-0.76.12 → lamindb-0.76.14}/docs/bio-registries.ipynb

@@ -63,7 +63,7 @@
    "outputs": [],
    "source": [
     "# check configured public ontology\n",
-    "bt.CellType.list_source(currently_used=True).one()"
+    "bt.Source.filter(entity=\"bionty.CellType\", currently_used=True).one()"
    ]
   },
   {
@@ -667,7 +667,7 @@
    },
    "outputs": [],
    "source": [
-    "bt.Phenotype.list_source(currently_used=True).df()"
+    "bt.Source.filter(entity=\"bionty.Phenotype\", currently_used=True).df()"
    ]
   },
   {
@@ -743,7 +743,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "py39",
+   "display_name": "py310",
    "language": "python",
    "name": "python3"
   },
@@ -757,7 +757,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.7"
+   "version": "3.10.13"
   },
   "nbproject": {
    "id": "s7F7PezA1HdQ",

lamindb-0.76.14/docs/cellregistry.md

@@ -0,0 +1,5 @@
+# `cellregistry`
+
+```{eval-rst}
+.. automodule:: cellregistry
+```

{lamindb-0.76.12 → lamindb-0.76.14}/docs/curate-df.ipynb

@@ -141,42 +141,7 @@
    "source": [
     "## Register new metadata values\n",
     "\n",
-    "If you see \"non-validated\" values, you'll need to decide whether to add them to your registries or \"fix\" them in your dataset.\n",
-    "\n",
-    "Because our current registries are still empty, we'll start by populating our {class}`~bionty.CellType` registry with values from a public ontology."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "bcb388d8",
-   "metadata": {
-    "tags": [
-     "hide-output"
-    ]
-   },
-   "outputs": [],
-   "source": [
-    "# this adds cell types that were validated based on a public ontology\n",
-    "curate.add_validated_from(\"cell_type\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "5bcfc09f",
-   "metadata": {},
-   "source": [
-    "If we call `validate()` again, we see that one cell type still doesn't pass validation."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "104f9932",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "curate.validate()"
+    "If you see \"non-validated\" values, you'll need to decide whether to add them to your registries or \"fix\" them in your dataset."
    ]
   },
   {
@@ -184,7 +149,7 @@
    "id": "8c2417c7",
    "metadata": {},
    "source": [
-    "Hence, let's understand which cell type in the public ontology might be the actual match."
+    "For `cell_type`, we saw that 'cerebral pyramidal neuron' is not validated, let's understand which cell type in the public ontology might be the actual match."
    ]
   },
   {
@@ -198,8 +163,8 @@
    },
    "outputs": [],
    "source": [
-    "# use a lookup object to get the correct spelling of categories from a public ontology\n",
-    "lookup = curate.lookup(\"public\")\n",
+    "# use a lookup object to get the correct spelling of categories from a public ontology via `public=True`\n",
+    "lookup = curate.lookup(public=True)\n",
     "lookup"
    ]
   },
@@ -231,32 +196,15 @@
    "outputs": [],
    "source": [
     "# fix the cell type\n",
-    "df.cell_type = df.cell_type.replace({\"cerebral pyramidal neuron\": cell_types.cerebral_cortex_pyramidal_neuron.name})\n",
-    "# now register curated and validated cell types\n",
-    "curate.add_validated_from(df.cell_type.name)"
+    "df.cell_type = df.cell_type.replace({\"cerebral pyramidal neuron\": cell_types.cerebral_cortex_pyramidal_neuron.name})"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "752142a1",
+   "id": "1dc1028a",
    "metadata": {},
    "source": [
-    "Now, do the same for `\"assay_ontology_id`\" and `\"donor\"`."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ede057e8",
-   "metadata": {
-    "tags": [
-     "hide-output"
-    ]
-   },
-   "outputs": [],
-   "source": [
-    "# this adds assays that were validated based on a public ontology\n",
-    "curate.add_validated_from(\"assay_ontology_id\")"
+    "For donor, we want to add the new donoers: 'D0001', 'D0002', 'DOOO3'"
    ]
   },
   {
@@ -362,24 +310,6 @@
     "curate.validate()"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "id": "71a917d1",
-   "metadata": {},
-   "source": [
-    "Save the validated genes following the instruction:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e31dabcc",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "curate.add_validated_from_var_index()"
-   ]
-  },
   {
    "cell_type": "markdown",
    "id": "614545ea",
@@ -521,7 +451,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "py310",
    "language": "python",
    "name": "python3"
   },

lamindb-0.76.14/docs/curate-subclass.ipynb

@@ -0,0 +1,200 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Enforce pre-defined validation constraints"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "In a [previous guide](./curate-df), you defined validation constraints ad-hoc when initializing {class}`~lamindb.Curator` objects.\n",
+    "\n",
+    "Often, you want to enforce a pre-defined set of validation constraints, like, e.g., the CELLxGENE curator ({doc}`docs:cellxgene-curate`).\n",
+    "\n",
+    "This guide shows how to subclass {class}`~lamindb.Curator` to enforce pre-defined constraints."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Define a custom curator"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Consider the example of electronic health records (EHR). We want to ensure that\n",
+    "\n",
+    "1. every record has the fields `disease`, `phenotype`, `developmental_stage`, and `age`\n",
+    "2. values for these fields map against specific versions of pre-defined ontologies"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The following implementation achieves the goal by subclassing {class}`~lamindb.core.DataFrameCurator`."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "```{eval-rst}\n",
+    ".. literalinclude:: ehrcurator.py\n",
+    "   :language: python\n",
+    "   :caption: EHR Curator\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Use the custom curator"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!lamin init --storage ./subclass-curator --schema bionty"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import lamindb as ln\n",
+    "import bionty as bt\n",
+    "import pandas as pd\n",
+    "from ehrcurator import EHRCurator\n",
+    "\n",
+    "ln.track(\"2XEr2IA4n1w40000\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "# create example DataFrame that has all mandatory columns but one ('patient_age') is wrongly named\n",
+    "data = {\n",
+    "    'disease': ['Alzheimer disease', 'Diabetes mellitus', 'Breast cancer', 'Hypertension', 'Asthma'],\n",
+    "    'phenotype': ['Cognitive decline', 'Hyperglycemia', 'Tumor growth', 'Increased blood pressure', 'Airway inflammation'],\n",
+    "    'developmental_stage': ['Adult', 'Adult', 'Adult', 'Adult', 'Child'],\n",
+    "    'patient_age': [70, 55, 60, 65, 12],\n",
+    "}\n",
+    "df = pd.DataFrame(data)\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "ehrcurator = EHRCurator(df)\n",
+    "ehrcurator.validate()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "# Fix the name of wrongly spelled column\n",
+    "df.columns = df.columns.str.replace(\"patient_age\", \"age\")\n",
+    "ehrcurator.validate()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "# Use lookup objects to curate the values\n",
+    "disease_lo = bt.Disease.public().lookup()\n",
+    "phenotype_lo = bt.Phenotype.public().lookup()\n",
+    "developmental_stage_lo = bt.DevelopmentalStage.public().lookup()\n",
+    "\n",
+    "df[\"disease\"] = df[\"disease\"].replace({\"Hypertension\": disease_lo.hypertensive_disorder.name})\n",
+    "df[\"phenotype\"] = df[\"phenotype\"].replace({\n",
+    "    \"Tumor growth\": phenotype_lo.neoplasm.name,\n",
+    "    \"Airway inflammation\": phenotype_lo.bronchitis.name}\n",
+    ")\n",
+    "df[\"developmental_stage\"] = df[\"developmental_stage\"].replace({\n",
+    "    \"Adult\": developmental_stage_lo.adolescent_stage.name,\n",
+    "    \"Child\": developmental_stage_lo.child_stage.name\n",
+    "})\n",
+    "\n",
+    "ehrcurator.validate()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-cell"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "!rm -rf subclass-curator\n",
+    "!lamin delete --force subclass-curator"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "lamindb",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

{lamindb-0.76.12 → lamindb-0.76.14}/docs/curate.md

@@ -5,4 +5,5 @@
 
 curate-df
 curate-any
+curate-subclass
 ```

lamindb-0.76.14/docs/ehrcurator.py

@@ -0,0 +1,72 @@
+import bionty as bt
+import pandas as pd
+from lamindb.core import DataFrameCurator, Record, logger
+from lamindb.core.types import UPathStr, FieldAttr
+
+__version__ = "0.1.0"
+
+# Curate these columns against the specified fields
+DEFAULT_CATEGORICALS = {
+    "disease": bt.Disease.name,
+    "phenotype": bt.Phenotype.name,
+    "developmental_stage": bt.DevelopmentalStage.name,
+}
+
+# If columns or values are missing, we substitute with these defaults
+DEFAULT_VALUES = {
+    "disease": "normal",
+    "development_stage": "unknown",
+    "phenotype": "unknown",
+}
+
+# Map values onto the following ontology versions
+DEFAULT_SOURCES = {
+    "disease": bt.Source.get(
+        entity="bionty.Disease", name="mondo", version="2023-04-04"
+    ),
+    "developmental_stage": bt.Source.get(
+        entity="bionty.DevelopmentalStage", name="hsapdv", version="2020-03-10"
+    ),
+    "phenotype": bt.Source.get(
+        entity="bionty.Phenotype", name="hp", version="2023-06-17", organism="human"
+    ),
+}
+
+
+class EHRCurator(DataFrameCurator):
+    """Custom curation flow for electronic health record data."""
+
+    def __init__(
+        self,
+        data: pd.DataFrame | UPathStr,
+        categoricals: dict[str, FieldAttr] = DEFAULT_CATEGORICALS,
+        *,
+        defaults: dict[str, str] = None,
+        sources: dict[str, Record] = DEFAULT_SOURCES,
+        organism="human",
+    ):
+        self.data = data
+
+        if defaults:
+            for col, default in defaults.items():
+                if col not in self.data.columns:
+                    self.data[col] = default
+                else:
+                    self.data[col].fillna(default, inplace=True)
+
+        super().__init__(
+            df=self.data, categoricals=categoricals, sources=sources, organism=organism
+        )
+
+    def validate(self, organism: str | None = None) -> bool:
+        """Validates the internal EHR standard."""
+        missing_columns = {"disease", "phenotype", "developmental_stage", "age"} - set(
+            self.data.columns
+        )
+        if missing_columns:
+            logger.error(
+                f"Columns {', '.join(map(repr, missing_columns))} are missing but required."
+            )
+            return False
+
+        return DataFrameCurator.validate(self, organism)

{lamindb-0.76.12 → lamindb-0.76.14}/docs/faq/idempotency.ipynb

@@ -70,7 +70,6 @@
     "import lamindb as ln\n",
     "import pytest\n",
     "\n",
-    "ln.settings.verbosity = \"hint\"\n",
     "ln.track(\"ANW20Fr4eZgM0000\")"
    ]
   },
@@ -301,8 +300,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "artifact = ln.Artifact(filepath, description=\"My fcs artifact\")\n",
-    "artifact.save()"
+    "artifact = ln.Artifact(filepath, description=\"My fcs artifact\").save()"
    ]
   },
   {
@@ -316,7 +314,9 @@
    },
    "outputs": [],
    "source": [
-    "assert artifact.hash == \"KCEXRahJ-Ui9Y6nksQ8z1A\""
+    "assert artifact.hash == \"KCEXRahJ-Ui9Y6nksQ8z1A\"\n",
+    "assert artifact.run == ln.context.run\n",
+    "assert len(artifact._previous_runs.all()) == 0"
    ]
   },
   {
@@ -355,7 +355,8 @@
    "outputs": [],
    "source": [
     "assert artifact.id == artifact2.id\n",
-    "assert artifact.run == artifact2.run"
+    "assert artifact.run == artifact2.run\n",
+    "assert len(artifact._previous_runs.all()) == 0"
    ]
   },
   {
@@ -485,8 +486,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "artifact4 = ln.Artifact(filepath, description=\"My new fcs artifact\")\n",
-    "artifact4.save()"
+    "artifact4 = ln.Artifact(filepath, description=\"My new fcs artifact\").save()"
    ]
   },
   {
@@ -543,6 +543,7 @@
    },
    "outputs": [],
    "source": [
+    "!rm -rf ./test-idempotency\n",
     "!lamin delete --force test-idempotency"
    ]
   }

{lamindb-0.76.12 → lamindb-0.76.14}/docs/faq/setup.ipynb

@@ -100,7 +100,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "with pytest.raises(RuntimeError):\n",
+    "from lamindb_setup._init_instance import CannotSwitchDefaultInstance\n",
+    "\n",
+    "with pytest.raises(CannotSwitchDefaultInstance):\n",
     "    ln.setup.init(storage=\"./testsetup2\")"
    ]
   },

lamindb-0.76.14/docs/omop.md

@@ -0,0 +1,5 @@
+# `omop`
+
+```{eval-rst}
+.. automodule:: omop
+```

lamindb-0.76.14/docs/ourprojects.md

@@ -0,0 +1,5 @@
+# `ourprojects`
+
+```{eval-rst}
+.. automodule:: ourprojects
+```

{lamindb-0.76.12 → lamindb-0.76.14}/docs/signup-login.md

@@ -1,8 +1,9 @@
 1. [Sign up](https://lamin.ai/signup) for a free account (see more [info](https://lamin.ai/docs/setup)) and copy the API key.
 2. Log in on the command line:
    ```shell
-   lamin login <email> --key <API-key>
+   lamin login
    ```
+   You will be prompted for your API key. You can create your API key in your account settings on LaminHub.
 
 ```{note}
 
@@ -16,19 +17,12 @@ For more, see [doc](inv:docs#access), [the source code](https://github.com/lamin
 
 ```
 
-On the command line, you can log in with either email or handle:
+On the command line, you can log in with your handle if you have a cached API-key:
 
 ```
-lamin login testuser1@lamin.ai
 lamin login testuser1
 ```
 
-If you don't have a cached API-key in your environment, you need to copy it from your lamin.ai account and pass it:
-
-```
-lamin login <email> --key <API-key>
-```
-
 Log out:
 
 ```

{lamindb-0.76.12 → lamindb-0.76.14}/docs/storage/vitessce.ipynb

@@ -153,7 +153,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "ln.setup.delete(\"test-vitessce\", force=True)"
+    "!rm -rf test-vitessce\n",
+    "!lamin delete --force test-vitessce"
    ]
   }
  ],