lamindb 0.76.13__tar.gz → 0.76.15__tar.gz

{lamindb-0.76.13 → lamindb-0.76.15}/.github/workflows/build.yml

@@ -32,11 +32,15 @@ jobs:
       - uses: actions/setup-python@v5
         if: ${{ matrix.group == 'storage' }}
         with:
-          python-version: "3.9" # run just one job with a different python version
+          python-version: "3.9"
       - uses: actions/setup-python@v5
-        if: ${{ matrix.group != 'storage' }}
+        if: ${{ matrix.group == 'unit-storage' }}
         with:
-          python-version: "3.11"
+          python-version: "3.11" # there are problems with installation time with python 3.12
+      - uses: actions/setup-python@v5
+        if: ${{ matrix.group != 'storage' && matrix.group != 'unit-storage'}}
+        with:
+          python-version: "3.12"
       - name: cache pre-commit
         uses: actions/cache@v4
         with:
@@ -106,7 +110,7 @@ jobs:
           aws-region: us-east-1
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.10" # use 3.10, 3.8 and 3.11 give issues
+          python-version: "3.10" # use 3.10 because 3.11+ give issues (ablog lacks 3.12 support)
       - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
       - run: nox -s "install_ci(group='docs')"
       - uses: actions/download-artifact@v4
@@ -134,7 +138,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - run: |
           python -m pip install -U uv
           uv pip install --system coverage[toml]

{lamindb-0.76.13 → lamindb-0.76.15}/.gitignore

@@ -1,8 +1,13 @@
+__MACOSX/
+
 # LaminDB
+*.db
+*.lndb
+*.jpg
+*.zarr/
 docsbuild/
 docs/lamin.md
 docs/guide/data-validation.ipynb
-docs/guide/knowledge.ipynb
 docs/guide/bionty.ipynb
 docs/guide/lnschema-core.ipynb
 docs/paradisi05_laminopathic_nuclei.jpg
@@ -15,19 +20,10 @@ lamin-tutorial/
 mytest/
 rds/
 mydb/
-bfx-output/
-scrnaseq-cellranger/
-scrnaseq-cellranger-schmidt22/
-cellranger_run_001
 docs/test-annotate-flexible/
-__MACOSX/
 docs/lamindb.*
 lamin_sphinx
 docs/conf.py
-*.db
-*.lndb
-*.jpg
-*.zarr/
 lamindb/setup/.env
 _secrets.py
 _configuration.py
@@ -56,9 +52,6 @@ test-inherit1
 test-inherit2
 test-search0
 test-search1
-test-search2
-test-search3
-test-search4
 test-search5
 default_storage
 default_storage_unit_core

{lamindb-0.76.13 → lamindb-0.76.15}/.gitmodules

@@ -19,3 +19,12 @@
 [submodule "sub/findrefs"]
 	path = sub/findrefs
 	url = https://github.com/laminlabs/findrefs
+[submodule "sub/ourprojects"]
+	path = sub/ourprojects
+	url = https://github.com/laminlabs/ourprojects
+[submodule "sub/cellregistry"]
+	path = sub/cellregistry
+	url = https://github.com/laminlabs/cellregistry
+[submodule "sub/omop"]
+	path = sub/omop
+	url = https://github.com/laminlabs/omop.git

{lamindb-0.76.13 → lamindb-0.76.15}/PKG-INFO

@@ -1,29 +1,28 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.76.13
+Version: 0.76.15
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
-Requires-Python: >=3.9
+Requires-Python: >=3.9,<3.13
 Description-Content-Type: text/markdown
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.75.0
-Requires-Dist: lamin_utils==0.13.6
-Requires-Dist: lamin_cli==0.19.0
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: lnschema_core==0.76.1
+Requires-Dist: lamin_utils==0.13.7
+Requires-Dist: lamin_cli==0.20.1
 Requires-Dist: lamindb_setup
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
 Requires-Dist: python-dateutil
 Requires-Dist: anndata>=0.8.0,<=0.10.9
-Requires-Dist: scipy<1.13.0rc1
 Requires-Dist: fsspec
-Requires-Dist: pandas
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
 Requires-Dist: lamindb_setup[aws] ; extra == "aws"
-Requires-Dist: bionty==0.51.2 ; extra == "bionty"
+Requires-Dist: bionty==0.52.0 ; extra == "bionty"
 Requires-Dist: line_profiler ; extra == "dev"
 Requires-Dist: pre-commit ; extra == "dev"
 Requires-Dist: nox ; extra == "dev"
@@ -35,9 +34,9 @@ Requires-Dist: mudata ; extra == "dev"
 Requires-Dist: nbproject_test>=0.5.1 ; extra == "dev"
 Requires-Dist: faker-biology ; extra == "dev"
 Requires-Dist: django-schema-graph ; extra == "erdiagram"
-Requires-Dist: readfcs>=1.1.8 ; extra == "fcs"
+Requires-Dist: readfcs>=1.1.9 ; extra == "fcs"
 Requires-Dist: lamindb_setup[gcp] ; extra == "gcp"
-Requires-Dist: nbproject==0.10.4 ; extra == "jupyter"
+Requires-Dist: nbproject==0.10.5 ; extra == "jupyter"
 Requires-Dist: jupytext ; extra == "jupyter"
 Requires-Dist: nbconvert ; extra == "jupyter"
 Requires-Dist: zarr>=2.16.0 ; extra == "zarr"

{lamindb-0.76.13 → lamindb-0.76.15}/docs/api.md

@@ -17,9 +17,12 @@ cli
 
 lamindb
 bionty
+ourprojects
+findrefs
+cellregistry
 wetlab
 clinicore
-findrefs
+omop
 ```
 
 ```{toctree}

{lamindb-0.76.13 → lamindb-0.76.15}/docs/arrays.ipynb

@@ -8,7 +8,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -16,7 +15,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -87,16 +85,18 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "tags": []
+    "tags": [
+     "hide-output"
+    ]
    },
    "outputs": [],
    "source": [
     "ln.Artifact(\"s3://lamindb-ci/lndb-storage/pbmc68k.h5ad\").save()\n",
-    "ln.Artifact(\"s3://lamindb-ci/lndb-storage/testfile.hdf5\").save()"
+    "ln.Artifact(\"s3://lamindb-ci/lndb-storage/testfile.hdf5\").save()\n",
+    "ln.Artifact(\"s3://lamindb-ci/lndb-storage/sharded_parquet\").save()"
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -138,7 +138,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -148,14 +147,17 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "adata"
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -165,14 +167,17 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "adata.X"
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -182,7 +187,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "obs_idx = adata.obs.cell_type.isin([\"Dendritic cells\", \"CD14+ Monocytes\"]) & (\n",
@@ -193,7 +202,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -203,14 +211,17 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "adata_subset.X"
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -220,14 +231,17 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "adata_subset.to_memory()"
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -235,7 +249,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -252,7 +265,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -269,7 +281,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -294,6 +305,80 @@
     "backed.storage"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Parquet"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "A dataframe stored as sharded `parquet`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "artifact = ln.Artifact.get(key=\"lndb-storage/sharded_parquet\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "artifact.path.view_tree()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "backed = artifact.open()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This returns `pyarrow` `Dataset`, see [here](https://arrow.apache.org/docs/python/dataset.html)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "backed"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "backed.head(5).to_pandas()"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -325,7 +410,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.7"
+   "version": "3.9.17"
   },
   "nbproject": {
    "id": "YVUCtH4GfQOy",

{lamindb-0.76.13 → lamindb-0.76.15}/docs/bio-registries.ipynb

@@ -63,7 +63,7 @@
    "outputs": [],
    "source": [
     "# check configured public ontology\n",
-    "bt.CellType.list_source(currently_used=True).one()"
+    "bt.Source.filter(entity=\"bionty.CellType\", currently_used=True).one()"
    ]
   },
   {
@@ -667,7 +667,7 @@
    },
    "outputs": [],
    "source": [
-    "bt.Phenotype.list_source(currently_used=True).df()"
+    "bt.Source.filter(entity=\"bionty.Phenotype\", currently_used=True).df()"
    ]
   },
   {
@@ -743,7 +743,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "py39",
+   "display_name": "py310",
    "language": "python",
    "name": "python3"
   },
@@ -757,7 +757,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.7"
+   "version": "3.10.13"
   },
   "nbproject": {
    "id": "s7F7PezA1HdQ",

lamindb-0.76.15/docs/cellregistry.md

@@ -0,0 +1,5 @@
+# `cellregistry`
+
+```{eval-rst}
+.. automodule:: cellregistry
+```

{lamindb-0.76.13 → lamindb-0.76.15}/docs/curate-df.ipynb

@@ -74,7 +74,7 @@
     "        \"temperature\": [37.2, 36.3, 38.2],\n",
     "        \"cell_type\": [\"cerebral pyramidal neuron\", \"astrocyte\", \"oligodendrocyte\"],\n",
     "        \"assay_ontology_id\": [\"EFO:0008913\", \"EFO:0008913\", \"EFO:0008913\"],\n",
-    "        \"donor\": [\"D0001\", \"D0002\", \"DOOO3\"]\n",
+    "        \"donor\": [\"D0001\", \"D0002\", \"D0003\"]\n",
     "    },\n",
     "    index = [\"obs1\", \"obs2\", \"obs3\"]\n",
     ")\n",
@@ -141,42 +141,7 @@
    "source": [
     "## Register new metadata values\n",
     "\n",
-    "If you see \"non-validated\" values, you'll need to decide whether to add them to your registries or \"fix\" them in your dataset.\n",
-    "\n",
-    "Because our current registries are still empty, we'll start by populating our {class}`~bionty.CellType` registry with values from a public ontology."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "bcb388d8",
-   "metadata": {
-    "tags": [
-     "hide-output"
-    ]
-   },
-   "outputs": [],
-   "source": [
-    "# this adds cell types that were validated based on a public ontology\n",
-    "curate.add_validated_from(\"cell_type\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "5bcfc09f",
-   "metadata": {},
-   "source": [
-    "If we call `validate()` again, we see that one cell type still doesn't pass validation."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "104f9932",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "curate.validate()"
+    "If you see \"non-validated\" values, you'll need to decide whether to add them to your registries or \"fix\" them in your dataset."
    ]
   },
   {
@@ -184,7 +149,7 @@
    "id": "8c2417c7",
    "metadata": {},
    "source": [
-    "Hence, let's understand which cell type in the public ontology might be the actual match."
+    "For `cell_type`, we saw that 'cerebral pyramidal neuron' is not validated, let's understand which cell type in the public ontology might be the actual match."
    ]
   },
   {
@@ -198,8 +163,9 @@
    },
    "outputs": [],
    "source": [
-    "# use a lookup object to get the correct spelling of categories from a public ontology\n",
-    "lookup = curate.lookup(\"public\")\n",
+    "# to check the correct spelling of categories, pass `public=True` to get a lookup object from public ontologies\n",
+    "# use `lookup = curate.lookup()` to get a lookup object of existing records in your instance\n",
+    "lookup = curate.lookup(public=True)\n",
     "lookup"
    ]
   },
@@ -214,7 +180,7 @@
    },
    "outputs": [],
    "source": [
-    "# here is an example for the cell_type column\n",
+    "# here is an example for the \"cell_type\" column\n",
     "cell_types = lookup[\"cell_type\"]\n",
     "cell_types.cerebral_cortex_pyramidal_neuron"
    ]
@@ -231,32 +197,15 @@
    "outputs": [],
    "source": [
     "# fix the cell type\n",
-    "df.cell_type = df.cell_type.replace({\"cerebral pyramidal neuron\": cell_types.cerebral_cortex_pyramidal_neuron.name})\n",
-    "# now register curated and validated cell types\n",
-    "curate.add_validated_from(df.cell_type.name)"
+    "df.cell_type = df.cell_type.replace({\"cerebral pyramidal neuron\": cell_types.cerebral_cortex_pyramidal_neuron.name})"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "752142a1",
+   "id": "1dc1028a",
    "metadata": {},
    "source": [
-    "Now, do the same for `\"assay_ontology_id`\" and `\"donor\"`."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ede057e8",
-   "metadata": {
-    "tags": [
-     "hide-output"
-    ]
-   },
-   "outputs": [],
-   "source": [
-    "# this adds assays that were validated based on a public ontology\n",
-    "curate.add_validated_from(\"assay_ontology_id\")"
+    "For donor, we want to add the new donors: \"D0001\", \"D0002\", \"D0003\""
    ]
   },
   {
@@ -297,7 +246,7 @@
    "source": [
     "## Validate an AnnData\n",
     "\n",
-    "Here we addtionally specify which `var_index` to validate against."
+    "Here we additionally specify which `var_index` to validate against."
    ]
   },
   {
@@ -320,7 +269,7 @@
     "        \"ENSG00000198851\": [7, 8, 9], \n",
     "        \"ENSG00000010610\": [10, 11, 12], \n",
     "        \"ENSG00000153563\": [13, 14, 15],\n",
-    "        \"corrupted\": [16, 17, 18]\n",
+    "        \"ENSGcorrupted\": [16, 17, 18]\n",
     "    }, \n",
     "    index=df.index\n",
     ")\n",
@@ -362,24 +311,6 @@
     "curate.validate()"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "id": "71a917d1",
-   "metadata": {},
-   "source": [
-    "Save the validated genes following the instruction:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e31dabcc",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "curate.add_validated_from_var_index()"
-   ]
-  },
   {
    "cell_type": "markdown",
    "id": "614545ea",
@@ -484,7 +415,7 @@
    "id": "0fa20942",
    "metadata": {},
    "source": [
-    "Validated features and labels are linked to the artifact:"
+    "Saved artifact has been annotated with validated features and labels:"
    ]
   },
   {
@@ -521,7 +452,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "py310",
    "language": "python",
    "name": "python3"
   },