lamindb 0.69.7__tar.gz → 0.69.9__tar.gz

{lamindb-0.69.7 → lamindb-0.69.9}/.github/workflows/build.yml

@@ -30,6 +30,11 @@ jobs:
           submodules: recursive
           fetch-depth: 0
       - uses: actions/setup-python@v5
+        if: ${{ matrix.group == 'storage' }}
+        with:
+          python-version: "3.9" # run just one job with a different python version
+      - uses: actions/setup-python@v5
+        if: ${{ matrix.group != 'storage' }}
         with:
           python-version: "3.11"
       - name: cache pre-commit
@@ -100,7 +105,7 @@ jobs:
           aws-region: us-east-1
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.10" # use 3.10, 3.8 and 3.11 give issues
       - run: pip install -U laminci
       - run: nox -s "install(group='docs')"
       - uses: actions/download-artifact@v2

{lamindb-0.69.7 → lamindb-0.69.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.69.7
+Version: 0.69.9
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,10 +9,10 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.64.4
-Requires-Dist: lamindb_setup==0.68.4
+Requires-Dist: lnschema_core==0.64.6
+Requires-Dist: lamindb_setup==0.69.0
 Requires-Dist: lamin_utils==0.13.1
-Requires-Dist: lamin_cli==0.12.0
+Requires-Dist: lamin_cli==0.12.1
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
@@ -27,7 +27,7 @@ Requires-Dist: urllib3<2 ; extra == "aws"
 Requires-Dist: aiobotocore[boto3]>=2.5.4,<3.0.0 ; extra == "aws"
 Requires-Dist: s3fs==2023.12.2 ; extra == "aws"
 Requires-Dist: fsspec[s3]==2023.12.2 ; extra == "aws"
-Requires-Dist: bionty==0.42.4 ; extra == "bionty"
+Requires-Dist: bionty==0.42.5 ; extra == "bionty"
 Requires-Dist: pandas<2 ; extra == "dev"
 Requires-Dist: pre-commit ; extra == "dev"
 Requires-Dist: nox ; extra == "dev"

{lamindb-0.69.7 → lamindb-0.69.9}/docs/changelog.md

@@ -1,12 +1,19 @@
 # Changelog
 
-
 ```{eval-rst}
 .. role:: small
 ```
 
 ## 0.69
 
+### 0.69.9 {small}`2024-04-04`
+
+- 🐛 Fix clashes for multiple processes [PR](https://github.com/laminlabs/lamindb/pull/1553) [@falexwolf](https://github.com/falexwolf)
+
+### 0.69.8 {small}`2024-04-04`
+
+- ♻️ Use future annotations [PR](https://github.com/laminlabs/lamindb/pull/1549) [@Zethson](https://github.com/Zethson)
+
 ### 0.69.7 {small}`2024-04-03`
 
 - ✨ Add ability to upload arbitrary files or folders from CLI [PR](https://github.com/laminlabs/lamindb/pull/1545) [@falexwolf](https://github.com/falexwolf)

{lamindb-0.69.7 → lamindb-0.69.9}/docs/introduction.ipynb

@@ -77,9 +77,9 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Provenance\n",
+    "### Data lineage\n",
     "\n",
-    "Run {meth}`~lamindb.track` to auto-generate IDs to track data lineage."
+    "Run {meth}`~lamindb.track` to track data lineage."
    ]
   },
   {
@@ -88,7 +88,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# tag your code with auto-generated identifiers\n",
+    "# tag your notebook or script with auto-generated identifiers\n",
     "ln.settings.transform.stem_uid = \"FPnfDtJz8qbE\"\n",
     "ln.settings.transform.version = \"1\"\n",
     "\n",
@@ -157,7 +157,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Label\n",
+    "### Label artifacts\n",
     "\n",
     "Label an artifact with {class}`~lamindb.ULabel`."
    ]
@@ -181,7 +181,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Query\n",
+    "### Query anything\n",
     "\n",
     "Because, under-the-hood, LaminDB is SQL, you can write arbitrary relational queries."
    ]
@@ -255,9 +255,9 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Public ontologies\n",
+    "## Validate & annotate\n",
     "\n",
-    "Every {py:mod}`bionty` registry is based on public ontologies."
+    "Let's validate a `DataFrame` by passing validation criteria in the `Annotate` constructor."
    ]
   },
   {
@@ -266,10 +266,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import bionty as bt\n",
-    "\n",
-    "cell_types = bt.CellType.public()\n",
-    "cell_types"
+    "annotate = ln.Annotate.from_df(\n",
+    "    df,\n",
+    "    columns=ln.Feature.name,  # map column names\n",
+    "    categoricals={df.perturbation.name: ln.ULabel.name},  # map categories\n",
+    ")\n",
+    "annotate.validate()"
    ]
   },
   {
@@ -278,16 +280,22 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "cell_types.search(\"gamma delta T cell\").head(2)"
+    "# add non-validated terms to registries\n",
+    "annotate.add_new_from_columns()\n",
+    "annotate.add_new_from(df.perturbation.name)\n",
+    "\n",
+    "# view the registered features\n",
+    "ln.Feature.df()"
    ]
   },
   {
-   "cell_type": "markdown",
+   "cell_type": "code",
+   "execution_count": null,
    "metadata": {},
+   "outputs": [],
    "source": [
-    "## Validate & annotate\n",
-    "\n",
-    "Let's validate a `DataFrame` by passing validation criteria in the `Annotate` constructor."
+    "# view the registered ULabels\n",
+    "ln.ULabel.df()"
    ]
   },
   {
@@ -296,12 +304,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "annotate = ln.Annotate.from_df(\n",
-    "    df,\n",
-    "    columns=ln.Feature.name,  # validate column names\n",
-    "    categoricals={df.perturbation.name: ln.ULabel.name},  # validate categories\n",
-    ")\n",
-    "annotate.validate()"
+    "# now the validation passes\n",
+    "annotate.validate()\n",
+    "\n",
+    "# annotate and save the artifact\n",
+    "artifact = annotate.save_artifact(description=\"my RNA-seq\", version=\"1\")\n",
+    "artifact.describe()"
    ]
   },
   {
@@ -310,12 +318,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# add non-validated terms to registries for column names and perturbations\n",
-    "annotate.add_new_from_columns()\n",
-    "annotate.add_new_from(df.perturbation.name)\n",
-    "\n",
-    "# view the registered features\n",
-    "ln.Feature.df()"
+    "# get lookup object for the entities of interest\n",
+    "lookups = annotate.lookup()\n",
+    "lookups"
    ]
   },
   {
@@ -324,15 +329,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# view the registered ULabels\n",
-    "ln.ULabel.df()\n",
-    "\n",
-    "# now the validation passes\n",
-    "annotate.validate()\n",
-    "\n",
-    "# annotate and save the artifact\n",
-    "artifact = annotate.save_artifact(description=\"my RNA-seq\", version=\"1\")\n",
-    "artifact.describe()"
+    "# filter artifacts with specific labels\n",
+    "perturbations = lookups[\"perturbation\"]\n",
+    "ln.Artifact.filter(ulabels=candidate_marker_study).filter(ulabels=perturbations.ifng).one()"
    ]
   },
   {
@@ -347,49 +346,49 @@
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
+   "cell_type": "markdown",
    "metadata": {},
-   "outputs": [],
    "source": [
-    "# get lookup object for the entities of interest\n",
-    "lookups = annotate.lookup()\n",
-    "lookups"
+    "## Biological registries"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
+   "cell_type": "markdown",
    "metadata": {},
-   "outputs": [],
-   "source": []
+   "source": [
+    "The generic {class}`~lamindb.Feature` and {class}`~lamindb.ULabel` registries will get you pretty far.\n",
+    "\n",
+    "Let's now look at what you do can with a dedicated biological registry like {class}`~bionty.Gene`."
+   ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
+   "cell_type": "markdown",
    "metadata": {},
-   "outputs": [],
    "source": [
-    "# filter artifacts with specific labels\n",
-    "perturbations = lookups[\"perturbation\"]\n",
-    "ln.Artifact.filter(ulabels=candidate_marker_study).filter(ulabels=perturbations.ifng).one()"
+    "### Access public ontologies\n",
+    "\n",
+    "Every {py:mod}`bionty` registry is based on public ontologies."
    ]
   },
   {
-   "cell_type": "markdown",
+   "cell_type": "code",
+   "execution_count": null,
    "metadata": {},
+   "outputs": [],
    "source": [
-    "## Biological types"
+    "import bionty as bt\n",
+    "\n",
+    "cell_types = bt.CellType.public()\n",
+    "cell_types"
    ]
   },
   {
-   "cell_type": "markdown",
+   "cell_type": "code",
+   "execution_count": null,
    "metadata": {},
+   "outputs": [],
    "source": [
-    "{class}`~lamindb.Feature` and {class}`~lamindb.ULabel` will get you pretty far.\n",
-    "However, if you frequently use a specific entity, you'll want a dedicated registry.\n",
-    "\n",
-    "Let’s look at the example of {class}`~bionty.Gene` and use it to register features."
+    "cell_types.search(\"gamma delta T cell\").head(2)"
    ]
   },
   {
@@ -446,7 +445,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "### Manage biological registries\n",
+    "### Add new entities to biological registries\n",
     "\n",
     "Create a cell type record and add a new cell state."
    ]
@@ -487,7 +486,14 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Collections of artifacts"
+    "## Scale up learning"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "How do you learn from new data batches that extend your previous data history? Leverage {class}`~lamindb.Collection`."
    ]
   },
   {
@@ -549,16 +555,9 @@
    "outputs": [],
    "source": [
     "# if it's small enough, you can load the entire collection into memory as if it was one\n",
-    "collection.load()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# you can always iterate over its artifacts\n",
+    "collection.load()\n",
+    "\n",
+    "# typically, it's too big, hence, iterate over its artifacts\n",
     "collection.artifacts.df()"
    ]
   },
@@ -567,7 +566,7 @@
    "metadata": {},
    "source": [
     "```\n",
-    "# you can also batch iterate through collections\n",
+    "# to train models, batch iterate through the collection as if it was one array\n",
     "from torch.utils.data import DataLoader, WeightedRandomSampler\n",
     "dataset = collection.mapped(label_keys=[\"perturbation\"])\n",
     "sampler = WeightedRandomSampler(\n",
@@ -579,6 +578,13 @@
     "```"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Read this [blog post](https://lamin.ai/blog/arrayloader-benchmarks) for more on training models on sharded datasets."
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -625,7 +631,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Data lineage\n",
+    "## Data lineage across entire projects\n",
     "\n",
     "View the sequence of data transformations ({class}`~lamindb.Transform`) in a project (from [here](docs:project-flow), based on [Schmidt _et al._, 2022](https://pubmed.ncbi.nlm.nih.gov/35113687/)):\n",
     "\n",
@@ -651,13 +657,19 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Loading instances\n",
+    "## Distributed databases\n",
+    "\n",
+    "LaminDB is a distributed system like git. Similar to cloning a repository, collaborators can connect to your instance via:\n",
     "\n",
-    "LaminDB is a distributed system like git. Similar to cloning a repository, collaborators can load your instance on the command-line using:\n",
+    "```python\n",
+    "ln.connect(\"account-handle/instance-name\")\n",
+    "```\n",
+    "\n",
+    "Or you can load an instance on the command line for auto-connecting in a Python session:\n",
     "\n",
     "```shell\n",
-    "lamin load myhandle/myinstance\n",
-    "```"
+    "lamin load \"account-handle/instance-name\"\n",
+    "```\n"
    ]
   },
   {

{lamindb-0.69.7 → lamindb-0.69.9}/lamindb/__init__.py

@@ -40,7 +40,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.69.7"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.69.9"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os as _os
 

{lamindb-0.69.7 → lamindb-0.69.9}/lamindb/_annotate.py

@@ -1,11 +1,15 @@
-from typing import Dict, Iterable, List, Optional, Tuple, Union
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Iterable
 
 import anndata as ad
 import lamindb_setup as ln_setup
 import pandas as pd
 from lamin_utils import colors, logger
 from lnschema_core import Artifact, Collection, Feature, Registry, Run, ULabel
-from lnschema_core.types import FieldAttr
+
+if TYPE_CHECKING:
+    from lnschema_core.types import FieldAttr
 
 
 class ValidationError(ValueError):
@@ -19,9 +23,9 @@ class AnnotateLookup:
 
     def __init__(
         self,
-        categorials: Dict[str, FieldAttr],
-        slots: Dict[str, FieldAttr] = None,
-        using: Optional[str] = None,
+        categorials: dict[str, FieldAttr],
+        slots: dict[str, FieldAttr] = None,
+        using: str | None = None,
     ) -> None:
         if slots is None:
             slots = {}
@@ -91,8 +95,8 @@ class DataFrameAnnotator:
         self,
         df: pd.DataFrame,
         columns: FieldAttr = Feature.name,
-        categoricals: Optional[Dict[str, FieldAttr]] = None,
-        using: Optional[str] = None,
+        categoricals: dict[str, FieldAttr] | None = None,
+        using: str | None = None,
         verbosity: str = "hint",
         **kwargs,
     ) -> None:
@@ -106,15 +110,15 @@ class DataFrameAnnotator:
         self._artifact = None
         self._collection = None
         self._validated = False
-        self._kwargs: Dict = kwargs
+        self._kwargs: dict = kwargs
         self._save_columns()
 
     @property
-    def fields(self) -> Dict:
+    def fields(self) -> dict:
         """Return the columns fields to validate against."""
         return self._fields
 
-    def lookup(self, using: Optional[str] = None) -> AnnotateLookup:
+    def lookup(self, using: str | None = None) -> AnnotateLookup:
         """Lookup features and labels.
 
         Args:
@@ -260,11 +264,11 @@ class DataFrameAnnotator:
 
     def save_collection(
         self,
-        artifact: Union[Artifact, Iterable[Artifact]],
+        artifact: Artifact | Iterable[Artifact],
         name: str,
-        description: Optional[str] = None,
-        reference: Optional[str] = None,
-        reference_type: Optional[str] = None,
+        description: str | None = None,
+        reference: str | None = None,
+        reference_type: str | None = None,
     ) -> Collection:
         """Save a collection from artifact/artifacts.
 
@@ -319,7 +323,7 @@ class AnnDataAnnotator(DataFrameAnnotator):
         self,
         adata: ad.AnnData,
         var_index: FieldAttr,
-        categoricals: Dict[str, FieldAttr],
+        categoricals: dict[str, FieldAttr],
         using: str = "default",
         verbosity: str = "hint",
         **kwargs,
@@ -342,11 +346,11 @@ class AnnDataAnnotator(DataFrameAnnotator):
         return self._var_field
 
     @property
-    def categoricals(self) -> Dict:
+    def categoricals(self) -> dict:
         """Return the obs fields to validate against."""
         return self._obs_fields
 
-    def lookup(self, using: Optional[str] = None) -> AnnotateLookup:
+    def lookup(self, using: str | None = None) -> AnnotateLookup:
         """Lookup features and labels."""
         return AnnotateLookup(
             categorials=self._obs_fields,
@@ -413,9 +417,9 @@ class Annotate:
     def from_df(
         cls,
         df: pd.DataFrame,
-        categoricals: Optional[Dict[str, FieldAttr]] = None,
+        categoricals: dict[str, FieldAttr] | None = None,
         columns: FieldAttr = Feature.name,
-        using: Optional[str] = None,
+        using: str | None = None,
         verbosity: str = "hint",
         **kwargs,
     ) -> DataFrameAnnotator:
@@ -433,7 +437,7 @@ class Annotate:
         cls,
         adata: ad.AnnData,
         var_index: FieldAttr,
-        categoricals: Dict[str, FieldAttr],
+        categoricals: dict[str, FieldAttr],
         using: str = "default",
         verbosity: str = "hint",
         **kwargs,
@@ -448,7 +452,7 @@ class Annotate:
         )
 
 
-def get_registry_instance(registry: Registry, using: Optional[str] = None) -> Registry:
+def get_registry_instance(registry: Registry, using: str | None = None) -> Registry:
     """Get a registry instance using a specific instance."""
     if using is not None and using != "default":
         return registry.using(using)
@@ -465,8 +469,8 @@ def standardize_and_inspect(
 
 
 def check_registry_organism(
-    registry: Registry, organism: Optional[str] = None
-) -> Optional[str]:
+    registry: Registry, organism: str | None = None
+) -> str | None:
     """Check if a registry needs an organism and return the organism name."""
     if hasattr(registry, "organism_id"):
         import bionty as bt
@@ -484,7 +488,7 @@ def validate_categories(
     values: Iterable[str],
     field: FieldAttr,
     key: str,
-    using: Optional[str] = None,
+    using: str | None = None,
     **kwargs,
 ) -> bool:
     """Validate ontology terms in a pandas series using LaminDB registries."""
@@ -560,8 +564,8 @@ def validate_categories(
 
 def validate_categories_in_df(
     df: pd.DataFrame,
-    fields: Dict[str, FieldAttr],
-    using: Optional[str] = None,
+    fields: dict[str, FieldAttr],
+    using: str | None = None,
     **kwargs,
 ) -> bool:
     """Validate categories in DataFrame columns using LaminDB registries."""
@@ -580,8 +584,8 @@ def validate_categories_in_df(
 def validate_anndata(
     adata: ad.AnnData,
     var_field: FieldAttr,
-    obs_fields: Dict[str, FieldAttr],
-    using: Optional[str] = None,
+    obs_fields: dict[str, FieldAttr],
+    using: str | None = None,
     **kwargs,
 ) -> bool:
     """Inspect metadata in an AnnData object using LaminDB registries."""
@@ -604,9 +608,9 @@ def validate_anndata(
 
 
 def save_artifact(
-    data: Union[pd.DataFrame, ad.AnnData],
+    data: pd.DataFrame | ad.AnnData,
     description: str,
-    fields: Dict[str, FieldAttr],
+    fields: dict[str, FieldAttr],
     columns_field: FieldAttr,
     **kwargs,
 ) -> Artifact:
@@ -631,7 +635,7 @@ def save_artifact(
         raise ValueError("data must be a DataFrame or AnnData object")
     artifact.save()
 
-    feature_kwargs: Dict = {}
+    feature_kwargs: dict = {}
     organism = check_registry_organism(
         columns_field.field.model, kwargs.pop("organism", None)
     )
@@ -662,14 +666,14 @@ def save_artifact(
 
 
 def update_registry(
-    values: List[str],
+    values: list[str],
     field: FieldAttr,
     key: str,
     save_function: str = "add_new_from",
-    using: Optional[str] = None,
+    using: str | None = None,
     validated_only: bool = True,
-    kwargs: Optional[Dict] = None,
-    df: Optional[pd.DataFrame] = None,
+    kwargs: dict | None = None,
+    df: pd.DataFrame | None = None,
 ) -> None:
     """Save features or labels records in the default instance from the using instance.
 
@@ -703,7 +707,7 @@ def update_registry(
             settings.verbosity = verbosity
             return
 
-        labels_saved: Dict = {"from public": [], "without reference": []}
+        labels_saved: dict = {"from public": [], "without reference": []}
 
         (
             labels_saved[f"from {using}"],
@@ -757,7 +761,7 @@ def update_registry(
 
 
 def log_saved_labels(
-    labels_saved: Dict,
+    labels_saved: dict,
     key: str,
     save_function: str,
     model_field: str,
@@ -795,7 +799,7 @@ def log_saved_labels(
             )
 
 
-def save_ulabels_with_parent(values: List[str], field: FieldAttr, key: str) -> None:
+def save_ulabels_with_parent(values: list[str], field: FieldAttr, key: str) -> None:
     """Save a parent label for the given labels."""
     registry = field.field.model
     assert registry == ULabel
@@ -808,11 +812,11 @@ def save_ulabels_with_parent(values: List[str], field: FieldAttr, key: str) -> N
 
 
 def update_registry_from_using_instance(
-    values: List[str],
+    values: list[str],
     field: FieldAttr,
-    using: Optional[str] = None,
-    kwargs: Optional[Dict] = None,
-) -> Tuple[List[str], List[str]]:
+    using: str | None = None,
+    kwargs: dict | None = None,
+) -> tuple[list[str], list[str]]:
     """Save features or labels records from the using instance.
 
     Args: