lamindb 0.76.14__tar.gz → 0.76.15__tar.gz

{lamindb-0.76.14 → lamindb-0.76.15}/.github/workflows/build.yml

@@ -32,11 +32,15 @@ jobs:
       - uses: actions/setup-python@v5
         if: ${{ matrix.group == 'storage' }}
         with:
-          python-version: "3.9" # run just one job with a different python version
+          python-version: "3.9"
       - uses: actions/setup-python@v5
-        if: ${{ matrix.group != 'storage' }}
+        if: ${{ matrix.group == 'unit-storage' }}
         with:
-          python-version: "3.11"
+          python-version: "3.11" # there are problems with installation time with python 3.12
+      - uses: actions/setup-python@v5
+        if: ${{ matrix.group != 'storage' && matrix.group != 'unit-storage'}}
+        with:
+          python-version: "3.12"
       - name: cache pre-commit
         uses: actions/cache@v4
         with:
@@ -106,7 +110,7 @@ jobs:
           aws-region: us-east-1
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.10" # use 3.10, 3.8 and 3.11 give issues
+          python-version: "3.10" # use 3.10 because 3.11+ give issues (ablog lacks 3.12 support)
       - run: pip install "laminci@git+https://x-access-token:${{ secrets.LAMIN_BUILD_DOCS }}@github.com/laminlabs/laminci"
       - run: nox -s "install_ci(group='docs')"
       - uses: actions/download-artifact@v4
@@ -134,7 +138,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - run: |
           python -m pip install -U uv
           uv pip install --system coverage[toml]

{lamindb-0.76.14 → lamindb-0.76.15}/PKG-INFO

@@ -1,14 +1,15 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.76.14
+Version: 0.76.15
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
-Requires-Python: >=3.9
+Requires-Python: >=3.9,<3.13
 Description-Content-Type: text/markdown
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.76.0
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: lnschema_core==0.76.1
 Requires-Dist: lamin_utils==0.13.7
 Requires-Dist: lamin_cli==0.20.1
 Requires-Dist: lamindb_setup
@@ -17,9 +18,7 @@ Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
 Requires-Dist: python-dateutil
 Requires-Dist: anndata>=0.8.0,<=0.10.9
-Requires-Dist: scipy<1.13.0rc1
 Requires-Dist: fsspec
-Requires-Dist: pandas
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
 Requires-Dist: lamindb_setup[aws] ; extra == "aws"
@@ -35,7 +34,7 @@ Requires-Dist: mudata ; extra == "dev"
 Requires-Dist: nbproject_test>=0.5.1 ; extra == "dev"
 Requires-Dist: faker-biology ; extra == "dev"
 Requires-Dist: django-schema-graph ; extra == "erdiagram"
-Requires-Dist: readfcs>=1.1.8 ; extra == "fcs"
+Requires-Dist: readfcs>=1.1.9 ; extra == "fcs"
 Requires-Dist: lamindb_setup[gcp] ; extra == "gcp"
 Requires-Dist: nbproject==0.10.5 ; extra == "jupyter"
 Requires-Dist: jupytext ; extra == "jupyter"

{lamindb-0.76.14 → lamindb-0.76.15}/docs/arrays.ipynb

@@ -8,7 +8,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -16,7 +15,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -87,16 +85,18 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "tags": []
+    "tags": [
+     "hide-output"
+    ]
    },
    "outputs": [],
    "source": [
     "ln.Artifact(\"s3://lamindb-ci/lndb-storage/pbmc68k.h5ad\").save()\n",
-    "ln.Artifact(\"s3://lamindb-ci/lndb-storage/testfile.hdf5\").save()"
+    "ln.Artifact(\"s3://lamindb-ci/lndb-storage/testfile.hdf5\").save()\n",
+    "ln.Artifact(\"s3://lamindb-ci/lndb-storage/sharded_parquet\").save()"
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -138,7 +138,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -148,14 +147,17 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "adata"
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -165,14 +167,17 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "adata.X"
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -182,7 +187,11 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "obs_idx = adata.obs.cell_type.isin([\"Dendritic cells\", \"CD14+ Monocytes\"]) & (\n",
@@ -193,7 +202,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -203,14 +211,17 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "adata_subset.X"
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -220,14 +231,17 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
    "outputs": [],
    "source": [
     "adata_subset.to_memory()"
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -235,7 +249,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -252,7 +265,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -269,7 +281,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -294,6 +305,80 @@
     "backed.storage"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Parquet"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "A dataframe stored as sharded `parquet`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "artifact = ln.Artifact.get(key=\"lndb-storage/sharded_parquet\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "artifact.path.view_tree()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "backed = artifact.open()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This returns `pyarrow` `Dataset`, see [here](https://arrow.apache.org/docs/python/dataset.html)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "backed"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "backed.head(5).to_pandas()"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -325,7 +410,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.7"
+   "version": "3.9.17"
   },
   "nbproject": {
    "id": "YVUCtH4GfQOy",

{lamindb-0.76.14 → lamindb-0.76.15}/docs/curate-df.ipynb

@@ -74,7 +74,7 @@
     "        \"temperature\": [37.2, 36.3, 38.2],\n",
     "        \"cell_type\": [\"cerebral pyramidal neuron\", \"astrocyte\", \"oligodendrocyte\"],\n",
     "        \"assay_ontology_id\": [\"EFO:0008913\", \"EFO:0008913\", \"EFO:0008913\"],\n",
-    "        \"donor\": [\"D0001\", \"D0002\", \"DOOO3\"]\n",
+    "        \"donor\": [\"D0001\", \"D0002\", \"D0003\"]\n",
     "    },\n",
     "    index = [\"obs1\", \"obs2\", \"obs3\"]\n",
     ")\n",
@@ -163,7 +163,8 @@
    },
    "outputs": [],
    "source": [
-    "# use a lookup object to get the correct spelling of categories from a public ontology via `public=True`\n",
+    "# to check the correct spelling of categories, pass `public=True` to get a lookup object from public ontologies\n",
+    "# use `lookup = curate.lookup()` to get a lookup object of existing records in your instance\n",
     "lookup = curate.lookup(public=True)\n",
     "lookup"
    ]
@@ -179,7 +180,7 @@
    },
    "outputs": [],
    "source": [
-    "# here is an example for the cell_type column\n",
+    "# here is an example for the \"cell_type\" column\n",
     "cell_types = lookup[\"cell_type\"]\n",
     "cell_types.cerebral_cortex_pyramidal_neuron"
    ]
@@ -204,7 +205,7 @@
    "id": "1dc1028a",
    "metadata": {},
    "source": [
-    "For donor, we want to add the new donoers: 'D0001', 'D0002', 'DOOO3'"
+    "For donor, we want to add the new donors: \"D0001\", \"D0002\", \"D0003\""
    ]
   },
   {
@@ -245,7 +246,7 @@
    "source": [
     "## Validate an AnnData\n",
     "\n",
-    "Here we addtionally specify which `var_index` to validate against."
+    "Here we additionally specify which `var_index` to validate against."
    ]
   },
   {
@@ -268,7 +269,7 @@
     "        \"ENSG00000198851\": [7, 8, 9], \n",
     "        \"ENSG00000010610\": [10, 11, 12], \n",
     "        \"ENSG00000153563\": [13, 14, 15],\n",
-    "        \"corrupted\": [16, 17, 18]\n",
+    "        \"ENSGcorrupted\": [16, 17, 18]\n",
     "    }, \n",
     "    index=df.index\n",
     ")\n",
@@ -414,7 +415,7 @@
    "id": "0fa20942",
    "metadata": {},
    "source": [
-    "Validated features and labels are linked to the artifact:"
+    "Saved artifact has been annotated with validated features and labels:"
    ]
   },
   {

{lamindb-0.76.14 → lamindb-0.76.15}/docs/tutorial.ipynb

@@ -826,7 +826,7 @@
    "source": [
     "Most functionality that you just learned about artifacts - e.g., queries & provenance - also applies to {class}`~lamindb.Collection`.\n",
     "\n",
-    "Collections become powerful if you directly leverage them for training models: {doc}`docs:scrna5`."
+    "Collections become powerful if you directly leverage them for training models: {doc}`docs:scrna-mappedcollection`."
    ]
   },
   {

{lamindb-0.76.14 → lamindb-0.76.15}/lamindb/__init__.py

@@ -43,7 +43,7 @@ Modules and settings.
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.76.14"
+__version__ = "0.76.15"
 
 import os as _os
 

{lamindb-0.76.14 → lamindb-0.76.15}/lamindb/_artifact.py

@@ -28,39 +28,41 @@ from lnschema_core.types import (
     VisibilityChoice,
 )
 
-from lamindb._utils import attach_func_to_class_method
-from lamindb.core._data import _track_run_input, describe, view_lineage
-from lamindb.core._settings import settings
-from lamindb.core.exceptions import IntegrityError, InvalidArgument
-from lamindb.core.loaders import load_to_memory
-from lamindb.core.storage import (
+from ._utils import attach_func_to_class_method
+from .core._data import (
+    _track_run_input,
+    add_transform_to_kwargs,
+    describe,
+    get_run,
+    save_feature_set_links,
+    save_feature_sets,
+    view_lineage,
+)
+from .core._settings import settings
+from .core.exceptions import IntegrityError, InvalidArgument
+from .core.loaders import load_to_memory
+from .core.storage import (
     LocalPathClasses,
     UPath,
     delete_storage,
     infer_suffix,
     write_to_disk,
 )
-from lamindb.core.storage.paths import (
+from .core.storage._pyarrow_dataset import PYARROW_SUFFIXES
+from .core.storage.objects import _mudata_is_installed
+from .core.storage.paths import (
+    AUTO_KEY_PREFIX,
     auto_storage_key_from_artifact,
     auto_storage_key_from_artifact_uid,
     check_path_is_child_of_root,
     filepath_cache_key_from_artifact,
     filepath_from_artifact,
 )
-from lamindb.core.versioning import (
+from .core.versioning import (
     create_uid,
     message_update_key_in_version_family,
 )
 
-from .core._data import (
-    add_transform_to_kwargs,
-    get_run,
-    save_feature_set_links,
-    save_feature_sets,
-)
-from .core.storage.objects import _mudata_is_installed
-from .core.storage.paths import AUTO_KEY_PREFIX
-
 try:
     from .core.storage._zarr import zarr_is_adata
 except ImportError:
@@ -72,6 +74,7 @@ except ImportError:
 if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
     from mudata import MuData
+    from pyarrow.dataset import Dataset as PyArrowDataset
     from tiledbsoma import Collection as SOMACollection
     from tiledbsoma import Experiment as SOMAExperiment
 
@@ -772,19 +775,14 @@ def from_dir(
     else:
         folder_key_path = Path(key)
 
-    # always sanitize by stripping a trailing slash
-    folder_key = folder_key_path.as_posix().rstrip("/")
-
-    # TODO: (non-local) UPath doesn't list the first level artifacts and dirs with "*"
-    pattern = "" if not isinstance(folderpath, LocalPathClasses) else "*"
-
+    folder_key = folder_key_path.as_posix()
     # silence fine-grained logging
     verbosity = settings.verbosity
     verbosity_int = settings._verbosity_int
     if verbosity_int >= 1:
         settings.verbosity = "warning"
     artifacts_dict = {}
-    for filepath in folderpath.rglob(pattern):
+    for filepath in folderpath.rglob("*"):
         if filepath.is_file():
             relative_path = get_relative_path_to_directory(filepath, folderpath)
             artifact_key = folder_key + "/" + relative_path.as_posix()
@@ -802,7 +800,8 @@ def from_dir(
         if artifact.hash is not None
     ]
     uids = artifacts_dict.keys()
-    if len(set(hashes)) == len(hashes):
+    n_unique_hashes = len(set(hashes))
+    if n_unique_hashes == len(hashes):
         artifacts = list(artifacts_dict.values())
     else:
         # consider exact duplicates (same id, same hash)
@@ -811,7 +810,7 @@ def from_dir(
         #     logger.warning("dropping duplicate records in list of artifact records")
         #     artifacts = list(set(uids))
         # consider false duplicates (different id, same hash)
-        if not len(set(uids)) == len(set(hashes)):
+        if not len(set(uids)) == n_unique_hashes:
             seen_hashes = set()
             non_unique_artifacts = {
                 hash: artifact
@@ -905,14 +904,19 @@ def replace(
 # docstring handled through attach_func_to_class_method
 def open(
     self, mode: str = "r", is_run_input: bool | None = None
-) -> AnnDataAccessor | BackedAccessor | SOMACollection | SOMAExperiment:
+) -> (
+    AnnDataAccessor | BackedAccessor | SOMACollection | SOMAExperiment | PyArrowDataset
+):
     # ignore empty suffix for now
-    suffixes = (".h5", ".hdf5", ".h5ad", ".zarr", ".tiledbsoma", "")
+    suffixes = ("", ".h5", ".hdf5", ".h5ad", ".zarr", ".tiledbsoma") + PYARROW_SUFFIXES
     if self.suffix not in suffixes:
         raise ValueError(
-            "Artifact should have a zarr, h5 or tiledbsoma object as the underlying data, please"
-            " use one of the following suffixes for the object name:"
-            f" {', '.join(suffixes[:-1])}."
+            "Artifact should have a zarr, h5, tiledbsoma object"
+            " or a compatible `pyarrow.dataset.dataset` directory"
+            " as the underlying data, please use one of the following suffixes"
+            f" for the object name: {', '.join(suffixes[1:])}."
+            f" Or no suffix for a folder with {', '.join(PYARROW_SUFFIXES)} files"
+            " (no mixing allowed)."
         )
     if self.suffix != ".tiledbsoma" and self.key != "soma" and mode != "r":
         raise ValueError("Only a tiledbsoma store can be openened with `mode!='r'`.")

{lamindb-0.76.14 → lamindb-0.76.15}/lamindb/_can_validate.py

@@ -10,10 +10,10 @@ from lamin_utils import colors, logger
 from lamindb_setup.core._docs import doc_args
 from lnschema_core import CanValidate, Record
 
-from lamindb._utils import attach_func_to_class_method
-
 from ._from_values import _has_organism_field, _print_values, get_or_create_records
 from ._record import _queryset, get_name_field
+from ._utils import attach_func_to_class_method
+from .core.exceptions import ValidationError
 
 if TYPE_CHECKING:
     from django.db.models import QuerySet
@@ -496,9 +496,9 @@ def _add_or_remove_synonyms(
                 " with the following records:\n"
             )
             display(records_df)
-            raise ValueError(
-                "cannot assigned a synonym that is already associated with a record to a different record.\n"
-                "Consider removing the synonym from existing records or using a different synonym."
+            raise ValidationError(
+                f"you are trying to assign a synonym to record: {record}\n"
+                "    → consider removing the synonym from existing records or using a different synonym."
             )
 
     # passed synonyms
@@ -516,7 +516,7 @@ def _add_or_remove_synonyms(
         return
     # because we use | as the separator
     if any("|" in i for i in syn_new_set):
-        raise ValueError("a synonym can't contain '|'!")
+        raise ValidationError("a synonym can't contain '|'!")
 
     # existing synonyms
     syns_exist = record.synonyms

{lamindb-0.76.14 → lamindb-0.76.15}/lamindb/_collection.py

@@ -20,20 +20,21 @@ from lnschema_core.models import (
 )
 from lnschema_core.types import VisibilityChoice
 
-from lamindb._utils import attach_func_to_class_method
-from lamindb.core._data import _track_run_input, describe, view_lineage
-from lamindb.core._mapped_collection import MappedCollection
-from lamindb.core.versioning import process_revises
-
 from . import Artifact, Run
 from ._record import init_self_from_db, update_attributes
+from ._utils import attach_func_to_class_method
 from .core._data import (
+    _track_run_input,
     add_transform_to_kwargs,
+    describe,
     get_run,
     save_feature_set_links,
     save_feature_sets,
+    view_lineage,
 )
+from .core._mapped_collection import MappedCollection
 from .core._settings import settings
+from .core.versioning import process_revises
 
 if TYPE_CHECKING:
     from collections.abc import Iterable