lamindb 0.71.2__tar.gz → 0.71.3__tar.gz

{lamindb-0.71.2 → lamindb-0.71.3}/.github/workflows/build.yml

@@ -22,7 +22,7 @@ jobs:
           - "faq"
           - "storage"
           - "cli"
-    timeout-minutes: 10
+    timeout-minutes: 6
 
     steps:
       - uses: actions/checkout@v4

{lamindb-0.71.2 → lamindb-0.71.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.71.2
+Version: 0.71.3
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,10 +9,10 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.66.4
-Requires-Dist: lamindb_setup==0.71.3
+Requires-Dist: lnschema_core==0.66.5
+Requires-Dist: lamindb_setup==0.71.4
 Requires-Dist: lamin_utils==0.13.2
-Requires-Dist: lamin_cli==0.13.1
+Requires-Dist: lamin_cli==0.13.2
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
@@ -23,8 +23,9 @@ Requires-Dist: fsspec
 Requires-Dist: pandas
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
+Requires-Dist: psutil
 Requires-Dist: lamindb_setup[aws] ; extra == "aws"
-Requires-Dist: bionty==0.42.9 ; extra == "bionty"
+Requires-Dist: bionty==0.42.11 ; extra == "bionty"
 Requires-Dist: pandas<2 ; extra == "dev"
 Requires-Dist: pre-commit ; extra == "dev"
 Requires-Dist: nox ; extra == "dev"
@@ -37,7 +38,7 @@ Requires-Dist: faker-biology ; extra == "dev"
 Requires-Dist: django-schema-graph ; extra == "erdiagram"
 Requires-Dist: readfcs>=1.1.8 ; extra == "fcs"
 Requires-Dist: lamindb_setup[gcp] ; extra == "gcp"
-Requires-Dist: nbproject==0.10.2 ; extra == "jupyter"
+Requires-Dist: nbproject==0.10.3 ; extra == "jupyter"
 Requires-Dist: nbstripout==0.6.1 ; extra == "jupyter"
 Requires-Dist: nbconvert ; extra == "jupyter"
 Requires-Dist: zarr>=2.16.0 ; extra == "zarr"

{lamindb-0.71.2 → lamindb-0.71.3}/docs/bio-registries.ipynb

@@ -149,7 +149,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "public.search(\"gamma delta T cell\").head(3)"
@@ -166,7 +170,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "lookup = public.lookup()\n",
@@ -192,18 +200,14 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "gdt_cell = bt.CellType(lookup.gamma_delta_t_cell)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
+    "gdt_cell = bt.CellType(lookup.gamma_delta_t_cell)\n",
     "gdt_cell"
    ]
   },
@@ -217,7 +221,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "bt.CellType.from_public(ontology_id=\"CL:0000798\")"
@@ -284,7 +292,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "gdt_cell.parents.df()"
@@ -339,7 +351,7 @@
    "source": [
     "When accessing data sources, one often encounters bulk references to entities that might be corrupted or standardized using different standardization schemes.\n",
     "\n",
-    "Let's consider an example based on an `AnnData` object:"
+    "Let's consider an example based on an `AnnData` object, in the `cell_type` annotations of this `AnnData` object, we find 4 references to cell types:"
    ]
   },
   {
@@ -352,22 +364,7 @@
    },
    "outputs": [],
    "source": [
-    "adata = ln.core.datasets.anndata_with_obs()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "In the `cell_type` annotations of this `AnnData` object, we find 4 references to cell types:"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
+    "adata = ln.core.datasets.anndata_with_obs()\n",
     "adata.obs.cell_type.value_counts()"
    ]
   },
@@ -383,7 +380,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "cell_types = bt.CellType.from_values(adata.obs.cell_type)\n",
@@ -444,7 +445,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "adata.obs.cell_type_id.unique().tolist()"
@@ -453,7 +458,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "bt.CellType.from_values(adata.obs.cell_type_id, field=bt.CellType.ontology_id)"
@@ -519,10 +528,14 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
-    "bt.CellType.search(\"gamma delta T cell\").head(2)"
+    "bt.CellType.search(\"gamma delta T cell\").df().head(2)"
    ]
   },
   {
@@ -535,7 +548,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "cell_types = bt.CellType.lookup()\n",
@@ -562,7 +579,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "bt.CellType.validate([\"HSC\", \"blood forming stem cell\"])"
@@ -580,7 +601,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "bt.CellType.standardize([\"HSC\", \"blood forming stem cell\"])"
@@ -596,7 +621,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "bt.CellType.from_values([\"HSC\", \"blood forming stem cell\"])"
@@ -630,7 +659,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "bt.CellType.standardize([\"HSCs\"])"
@@ -662,7 +695,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "cell_types = bt.CellType.lookup(\"abbr\")\n",
@@ -720,13 +757,6 @@
     "bt.Gene.validate([\"TCF7\", \"ABC1\"], organism=\"human\")"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "You can also pass organism for validating features upon registering data, e.g., in `ln.Artifact.from_anndata(..., field=bionty.Gene.ensembl_gene_id, organism=...)`."
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -792,7 +822,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "hepatocyte = bt.CellType.filter(name=\"hepatocyte\").one()\n",
@@ -808,24 +842,6 @@
     "By default, records are created from the `\"currently_used\"` public sources which are configured during the instance initialization, e.g.:"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "bt.Phenotype.public()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Sometimes, the default source doesn't contain the ontology term you are looking for.\n",
-    "\n",
-    "You can then specify to create a record from a non-default source. For instance, instead of using untyped labels for iris organisms as {doc}`/tutorial2`, we can use the `ncbitaxon` ontology:"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -836,43 +852,7 @@
    },
    "outputs": [],
    "source": [
-    "public_source = bt.PublicSource.filter(entity=\"Organism\", source=\"ncbitaxon\").one()\n",
-    "iris_setosa = bt.Organism.from_public(name=\"iris setosa\", public_source=public_source)\n",
-    "iris_setosa.save()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "iris_setosa"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "iris_setosa.public_source"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "iris_setosa.parents.df()"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "Analogously, you can pass `public_source` to bulk-create records from a non-default source:"
+    "bt.Phenotype.public()"
    ]
   },
   {
@@ -885,28 +865,34 @@
    },
    "outputs": [],
    "source": [
-    "records = bt.Organism.from_values(\n",
-    "    [\"iris setosa\", \"iris versicolor\", \"iris virginica\"], public_source=public_source\n",
-    ")\n",
-    "ln.save(records)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "records"
+    "bt.Phenotype.sources(currently_used=True).df()"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
+   "cell_type": "markdown",
    "metadata": {},
-   "outputs": [],
    "source": [
-    "iris_setosa.parents.get(name=\"iris\").view_parents(with_children=True)"
+    "Sometimes, the default source doesn't contain the ontology term you are looking for.\n",
+    "\n",
+    "You can then specify to create a record from a non-default source. For instance, instead of using untyped labels for iris organisms as {doc}`/tutorial2`, we can use the `ncbitaxon` ontology:\n",
+    "\n",
+    "```python\n",
+    "\n",
+    "public_source = bt.PublicSource.filter(entity=\"Organism\", source=\"ncbitaxon\").one()\n",
+    "iris_setosa = bt.Organism.from_public(name=\"iris setosa\", public_source=public_source)\n",
+    "iris_setosa.save()\n",
+    "```\n",
+    "\n",
+    "Analogously, you can pass `public_source` to bulk-create records from a non-default source:\n",
+    "\n",
+    "```python\n",
+    "\n",
+    "records = bt.Organism.from_values(\n",
+    "    [\"iris setosa\", \"iris versicolor\", \"iris virginica\"], public_source=public_source\n",
+    ")\n",
+    "ln.save(records)\n",
+    "iris_setosa.parents.get(name=\"iris\").view_parents(with_children=True)\n",
+    "```"
    ]
   },
   {

{lamindb-0.71.2 → lamindb-0.71.3}/docs/changelog.md

@@ -14,6 +14,17 @@ LaminDB implements "migration-based versioning". When upgrading your LaminDB ins
 
 ## 0.71
 
+### 0.71.3 {small}`2024-05-14`
+
+- 🎨 Enable transfer when schema don't match [PR](https://github.com/laminlabs/lamindb/pull/1654) [@sunnyosun](https://github.com/sunnyosun)
+- ✨ Get artifacts through the CLI [PR](https://github.com/laminlabs/lamindb/pull/1642) [@falexwolf](https://github.com/falexwolf)
+- ⚡️ Improve the speed of describe [PR](https://github.com/laminlabs/lamindb/pull/1645) [@sunnyosun](https://github.com/sunnyosun)
+- ⚡️ Parallel hashing of directories [PR](https://github.com/laminlabs/lamindb/pull/1652) [@Koncopd](https://github.com/Koncopd)
+- ⚡️ Speed-up file hash [PR](https://github.com/laminlabs/lamindb/pull/1651) [@Koncopd](https://github.com/Koncopd)
+- ♻️ Refactor search [PR](https://github.com/laminlabs/lamindb/pull/1646) [@falexwolf](https://github.com/falexwolf)
+- ✨ Introduce bulk update [PR](https://github.com/laminlabs/lamindb/pull/1640) [@falexwolf](https://github.com/falexwolf)
+- 🚸 No need to pass organism if validating on ids [PR](https://github.com/laminlabs/lamindb/pull/1639) [@sunnyosun](https://github.com/sunnyosun)
+
 ### 0.71.2 {small}`2024-05-07`
 
 - ✨ Enable passing parameters to `ln.track()` [PR](https://github.com/laminlabs/lamindb/pull/1637) [@falexwolf](https://github.com/falexwolf)

{lamindb-0.71.2 → lamindb-0.71.3}/docs/introduction.ipynb

@@ -304,7 +304,7 @@
    "outputs": [],
    "source": [
     "# search in a registry\n",
-    "ln.Transform.search(\"intro\")\n",
+    "ln.Transform.search(\"intro\").df()\n",
     "\n",
     "# look up records with auto-complete\n",
     "labels = ln.ULabel.lookup()"
@@ -768,7 +768,7 @@
    "source": [
     "### Data lineage across entire projects\n",
     "\n",
-    "View the sequence of data transformations ({class}`~lamindb.Transform`) in a project (from [here](docs:project-flow), based on [Schmidt _et al._, 2022](https://pubmed.ncbi.nlm.nih.gov/35113687/)):\n",
+    "View the sequence of data transformations ({class}`~lamindb.Transform`) in a project (from [a use case](inv:docs#project-flow), based on [Schmidt _et al._, 2022](https://pubmed.ncbi.nlm.nih.gov/35113687/)):\n",
     "\n",
     "```python\n",
     "transform.view_parents()\n",

{lamindb-0.71.2 → lamindb-0.71.3}/docs/meta.ipynb

@@ -229,17 +229,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "ln.Artifact.search(\"iris\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "2aa66c4e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "ln.Artifact.search(\"iris\", return_queryset=True).first()"
+    "ln.Artifact.search(\"iris\").df()"
    ]
   },
   {
@@ -282,7 +272,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "ln.Transform.search(\"intestine\").head()"
+    "ln.Transform.search(\"intestine\").df().head()"
    ]
   },
   {

{lamindb-0.71.2 → lamindb-0.71.3}/docs/signup-login.md

@@ -12,7 +12,7 @@ Lamin does _not_ store or see any of your data, but only _basic_ metadata about
 
 If you register a LaminDB instance on LaminHub, Lamin only stores the storage location (AWS S3 or GCP bucket names, directory names).
 
-For more, see our guide on [access management & security](docs:access), [the source code](https://github.com/laminlabs/lamindb-setup), or the [privacy policy](https://lamin.ai/legal/privacy-policy).
+For more, see [doc](inv:docs#access), [the source code](https://github.com/laminlabs/lamindb-setup), or the [privacy policy](https://lamin.ai/legal/privacy-policy).
 
 ```
 

{lamindb-0.71.2 → lamindb-0.71.3}/docs/transfer.ipynb

@@ -30,9 +30,7 @@
    "outputs": [],
    "source": [
     "import lamindb as ln\n",
-    "import bionty as bt\n",
     "\n",
-    "bt.settings.organism = \"human\"\n",
     "ln.settings.transform.stem_uid = \"ITeOtm7bhtdq\"\n",
     "ln.settings.transform.version = \"1\"\n",
     "ln.track()"
@@ -42,7 +40,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "All artifacts in the `laminlabs/cellxgene` clone of CZ CELLxGENE (for more info, see [here](docs:cellxgene)):"
+    "All artifacts in the `laminlabs/cellxgene` clone of CZ CELLxGENE (for more info, see [cellxgene](inv:docs#cellxgene)):"
    ]
   },
   {

{lamindb-0.71.2 → lamindb-0.71.3}/docs/tutorial.ipynb

@@ -563,7 +563,7 @@
    },
    "outputs": [],
    "source": [
-    "ln.Artifact.search(\"meta\").head()"
+    "ln.Artifact.search(\"meta\").df().head()"
    ]
   },
   {

{lamindb-0.71.2 → lamindb-0.71.3}/lamindb/__init__.py

@@ -41,7 +41,7 @@ Modules & settings:
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.71.2"
+__version__ = "0.71.3"
 
 import os as _os
 

{lamindb-0.71.2 → lamindb-0.71.3}/lamindb/_artifact.py

@@ -1,12 +1,14 @@
 from __future__ import annotations
 
 import shutil
+from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path, PurePath, PurePosixPath
 from typing import TYPE_CHECKING, Any, Mapping
 
 import fsspec
 import lamindb_setup as ln_setup
 import pandas as pd
+import psutil
 from anndata import AnnData
 from lamin_utils import colors, logger
 from lamindb_setup import settings as setup_settings
@@ -204,15 +206,23 @@ def get_stat_or_artifact(
             return size, hash, hash_type, n_objects
     else:
         if path.is_dir():
-            md5s = []
-            size = 0
-            for subpath in path.rglob("*"):
-                if not subpath.is_file():
-                    continue
-                size += subpath.stat().st_size
-                md5s.append(hash_file(subpath)[0])
-            hash, hash_type = hash_md5s_from_dir(md5s)
-            n_objects = len(md5s)
+            files = (subpath for subpath in path.rglob("*") if subpath.is_file())
+
+            def hash_size(file):
+                file_size = file.stat().st_size
+                return hash_file(file, file_size)[0], file_size
+
+            n_workers = len(psutil.Process().cpu_affinity())
+            if n_workers > 1:
+                with ThreadPoolExecutor(n_workers) as pool:
+                    hashes_sizes = pool.map(hash_size, files)
+            else:
+                hashes_sizes = map(hash_size, files)
+            hashes, sizes = zip(*hashes_sizes)
+
+            hash, hash_type = hash_md5s_from_dir(hashes)
+            n_objects = len(hashes)
+            size = sum(sizes)
         else:
             hash, hash_type = hash_file(path)
             size = stat.st_size

{lamindb-0.71.2 → lamindb-0.71.3}/lamindb/_can_validate.py

@@ -80,7 +80,9 @@ def _inspect(
 
     # inspect in the DB
     result_db = inspect(
-        df=_filter_query_based_on_organism(queryset=queryset, organism=organism),
+        df=_filter_query_based_on_organism(
+            queryset=queryset, field=field, organism=organism
+        ),
         identifiers=values,
         field=field,
         mute=mute,
@@ -161,6 +163,7 @@ def _validate(
     field_values = pd.Series(
         _filter_query_based_on_organism(
             queryset=queryset,
+            field=field,
             organism=organism,
             values_list_field=field,
         ),
@@ -284,7 +287,9 @@ def _standardize(
 
     try:
         orm._meta.get_field(synonyms_field)
-        df = _filter_query_based_on_organism(queryset=queryset, organism=organism)
+        df = _filter_query_based_on_organism(
+            queryset=queryset, field=field, organism=organism
+        )
     except FieldDoesNotExist:
         df = pd.DataFrame()
 
@@ -439,6 +444,7 @@ def _check_synonyms_field_exist(record: Registry):
 
 def _filter_query_based_on_organism(
     queryset: QuerySet,
+    field: str,
     organism: str | Registry | None = None,
     values_list_field: str | None = None,
 ):
@@ -447,7 +453,7 @@ def _filter_query_based_on_organism(
 
     orm = queryset.model
 
-    if _has_organism_field(orm):
+    if _has_organism_field(orm) and not field.endswith("id"):
         # here, we can safely import lnschema_bionty
         from lnschema_bionty._bionty import create_or_get_organism_record
 

{lamindb-0.71.2 → lamindb-0.71.3}/lamindb/_collection.py

@@ -103,9 +103,9 @@ def __init__(
             if meta._state.adding:
                 raise ValueError("Save meta artifact before creating collection!")
             if not feature_sets:
-                feature_sets = meta.features._feature_set_by_slot
+                feature_sets = meta.features.feature_set_by_slot
             else:
-                if len(meta.features._feature_set_by_slot) > 0:
+                if len(meta.features.feature_set_by_slot) > 0:
                     logger.info("overwriting feature sets linked to artifact")
     # we ignore collections in trash containing the same hash
     if hash is not None:
@@ -129,7 +129,7 @@ def __init__(
             existing_collection.transform = run.transform
         init_self_from_db(collection, existing_collection)
         update_attributes(collection, {"description": description, "name": name})
-        for slot, feature_set in collection.features._feature_set_by_slot.items():
+        for slot, feature_set in collection.features.feature_set_by_slot.items():
             if slot in feature_sets:
                 if not feature_sets[slot] == feature_set:
                     collection.feature_sets.remove(feature_set)