lamindb 0.69.6__tar.gz → 0.69.7__tar.gz

{lamindb-0.69.6 → lamindb-0.69.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.69.6
+Version: 0.69.7
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -12,12 +12,13 @@ Classifier: Programming Language :: Python :: 3.11
 Requires-Dist: lnschema_core==0.64.4
 Requires-Dist: lamindb_setup==0.68.4
 Requires-Dist: lamin_utils==0.13.1
-Requires-Dist: lamin_cli==0.11.0
+Requires-Dist: lamin_cli==0.12.0
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
 Requires-Dist: python-dateutil
 Requires-Dist: anndata>=0.8.0,<0.10.6
+Requires-Dist: scipy<1.13.0rc1
 Requires-Dist: fsspec
 Requires-Dist: pandas
 Requires-Dist: graphviz

{lamindb-0.69.6 → lamindb-0.69.7}/docs/changelog.md

@@ -1,11 +1,17 @@
 # Changelog
 
+
 ```{eval-rst}
 .. role:: small
 ```
 
 ## 0.69
 
+### 0.69.7 {small}`2024-04-03`
+
+- ✨ Add ability to upload arbitrary files or folders from CLI [PR](https://github.com/laminlabs/lamindb/pull/1545) [@falexwolf](https://github.com/falexwolf)
+- 🐛 Fix anndata backed mode incompatibility with scipy 1.13.0 f
+
 ### 0.69.6 {small}`2024-04-02`
 
 - 🚑️ Temp fix region for non-hosted buckets [PR](https://github.com/laminlabs/lamindb/pull/1543) [@sunnyosun](https://github.com/sunnyosun)

{lamindb-0.69.6 → lamindb-0.69.7}/lamindb/__init__.py

@@ -40,7 +40,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.69.6"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.69.7"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os as _os
 

{lamindb-0.69.6 → lamindb-0.69.7}/lamindb/_finish.py

@@ -34,7 +34,7 @@ def finish(i_saved_the_notebook: bool = False):
 
         if not i_saved_the_notebook and not ln_setup._TESTING:
             logger.error(
-                "Save the notebook, pass `i_saved_the_notebook=True`, and re-run this cell."
+                "Please save the notebook, pass `i_saved_the_notebook=True`, and re-run this cell."
             )
             return None
         notebook_content = read_notebook(run_context.path)  # type: ignore

{lamindb-0.69.6 → lamindb-0.69.7}/lamindb/core/_mapped_collection.py

@@ -45,19 +45,28 @@ class _Connect:
 class MappedCollection:
     """Map-style collection for use in data loaders.
 
-    This currently only works for collections of `AnnData` objects.
+    This class virtually concatenates `AnnData` arrays as a `pytorch map-style dataset
+    <https://pytorch.org/docs/stable/data.html#map-style-datasets>`__.
 
-    For an example, see :meth:`~lamindb.Collection.mapped`.
+    If your `AnnData` collection is in the cloud, move them into a local cache
+    first for faster access.
 
     .. note::
 
-        A similar data loader exists `here
-        <https://github.com/Genentech/scimilarity>`__.
+        For a guide, see :doc:`docs:scrna5`.
+
+        For more convenient use within :class:`~lamindb.core.MappedCollection`,
+        see :meth:`~lamindb.Collection.mapped`.
+
+        This currently only works for collections of `AnnData` objects.
+
+        The implementation was influenced by the `SCimilarity
+        <https://github.com/Genentech/scimilarity>`__ data loader.
+
 
     Args:
-        path_list: A list of paths to `AnnData` objects stored in `h5ad` or `zrad` formats.
-        label_keys: Columns of the ``.obs`` slot - the names of the metadata
-            features storing labels.
+        path_list: A list of paths to `AnnData` objects stored in `.h5ad` or `.zarr` formats.
+        label_keys: Columns of the ``.obs`` slot that store labels.
         join: `"inner"` or `"outer"` virtual joins. If ``None`` is passed,
             does not join.
         encode_labels: Encode labels into integers.

{lamindb-0.69.6 → lamindb-0.69.7}/pyproject.toml

@@ -19,13 +19,14 @@ dependencies = [
     "lnschema_core==0.64.4",
     "lamindb_setup==0.68.4",
     "lamin_utils==0.13.1",
-    "lamin_cli==0.11.0",
+    "lamin_cli==0.12.0",
     # others
     "rapidfuzz",
     "pyarrow",
     "typing_extensions!=4.6.0",
     "python-dateutil",
     "anndata>=0.8.0,<0.10.6",  # will upgrade to new anndata releases
+    "scipy<1.13.0rc1",  # anndata backed is incompatible otherwise
     "fsspec",
     "pandas",
     "graphviz",

lamindb-0.69.7/sub/lamin-cli/lamin_cli/__init__.py

@@ -0,0 +1,3 @@
+"""Lamin CLI."""
+
+__version__ = "0.12.0"

{lamindb-0.69.6 → lamindb-0.69.7}/sub/lamin-cli/lamin_cli/__main__.py

@@ -143,11 +143,13 @@ def schema(action: str):
 @click.argument(
     "filepath", type=click.Path(exists=True, dir_okay=False, file_okay=True)
 )
-def save(filepath: str):
-    """Save notebook or script."""
+@click.option("--key", type=str, default=None)
+@click.option("--description", type=str, default=None)
+def save(filepath: str, key: str, description: str):
+    """Save file or folder."""
     from lamin_cli._save import save_from_filepath_cli
 
-    if save_from_filepath_cli(filepath) is not None:
+    if save_from_filepath_cli(filepath, key, description) is not None:
         sys.exit(1)
 
 

lamindb-0.69.7/sub/lamin-cli/lamin_cli/_save.py

@@ -0,0 +1,65 @@
+from pathlib import Path
+from typing import Optional, Union
+import lamindb_setup as ln_setup
+from lamin_utils import logger
+
+
+def save_from_filepath_cli(
+    filepath: Union[str, Path], key: Optional[str], description: Optional[str]
+) -> Optional[str]:
+    if not isinstance(filepath, Path):
+        filepath = Path(filepath)
+
+    # this will be gone once we get rid of lamin load or enable loading multiple
+    # instances sequentially
+    auto_connect_state = ln_setup.settings.auto_connect
+    ln_setup.settings.auto_connect = True
+
+    import lamindb as ln
+    from lamindb.core._run_context import get_stem_uid_and_version_from_file
+    from lamindb._finish import save_run_context_core
+
+    ln_setup.settings.auto_connect = auto_connect_state
+
+    if filepath.suffix not in {".py", ".ipynb"}:
+        if key is None and description is None:
+            logger.error("Please pass a key or description via --key or --description")
+            return "missing-key-or-description"
+        artifact = ln.Artifact(filepath, key=key, description=description)
+        artifact.save()
+        slug = ln_setup.settings.instance.slug
+        logger.important(f"saved: {artifact}")
+        logger.important(f"storage path: {artifact.path}")
+        if ln_setup.settings.instance.is_remote:
+            logger.important(f"go to: https://lamin.ai/{slug}/artifact/{artifact.uid}")
+        if ln_setup.settings.storage.type == "s3":
+            logger.important(f"storage url: {artifact.path.to_url()}")
+        return None
+    else:
+        # consider notebooks & scripts a transform
+        stem_uid, transform_version = get_stem_uid_and_version_from_file(filepath)
+        # the corresponding transform family in the transform table
+        transform_family = ln.Transform.filter(uid__startswith=stem_uid).all()
+        if len(transform_family) == 0:
+            logger.error(
+                f"Did not find stem uid '{stem_uid}'"
+                " in Transform registry. Did you run ln.track()?"
+            )
+            return "not-tracked-in-transform-registry"
+        # the specific version
+        transform = transform_family.filter(version=transform_version).one()
+        # latest run of this transform by user
+        run = ln.Run.filter(transform=transform).order_by("-started_at").first()
+        if run.created_by.id != ln_setup.settings.user.id:
+            response = input(
+                "You are trying to save a transform created by another user: Source and"
+                " report files will be tagged with *your* user id. Proceed? (y/n)"
+            )
+            if response != "y":
+                return "aborted-save-notebook-created-by-different-user"
+        return save_run_context_core(
+            run=run,
+            transform=transform,
+            filepath=filepath,
+            transform_family=transform_family,
+        )

lamindb-0.69.7/sub/lamin-cli/tests/test_save_files.py

@@ -0,0 +1,33 @@
+import subprocess
+from pathlib import Path
+
+test_file = Path(__file__).parent.parent.resolve() / ".gitignore"
+
+
+def test_save_file():
+    filepath = test_file
+
+    # neither key nor description
+    result = subprocess.run(
+        f"lamin save {filepath}",
+        shell=True,
+        capture_output=True,
+    )
+    print(result.stdout.decode())
+    print(result.stderr.decode())
+    assert (
+        "Please pass a key or description via --key or --description"
+        in result.stdout.decode()
+    )
+    assert result.returncode == 1
+
+    result = subprocess.run(
+        f"lamin save {filepath} --key mytest",
+        shell=True,
+        capture_output=True,
+    )
+    print(result.stdout.decode())
+    print(result.stderr.decode())
+    assert "key='mytest'" in result.stdout.decode()
+    assert "storage path:" in result.stdout.decode()
+    assert result.returncode == 0

{lamindb-0.69.6 → lamindb-0.69.7}/sub/lnschema-core/lnschema_core/models.py

@@ -740,9 +740,9 @@ class User(Registry, CanValidate):
 class Storage(Registry):
     """Storage locations.
 
-    Is auto-managed, no need to create records.
+    A storage location can be a local or remote directory/folder or an entire S3/GCP bucket.
 
-    Can be local or remote directories or entire S3/GCP buckets.
+    This registry is auto-managed and read-only.
 
     See Also:
         Default storage: :attr:`~lamindb.core.Settings.storage`
@@ -828,10 +828,10 @@ class Storage(Registry):
 
 
 class Transform(Registry, HasParents, IsVersioned):
-    """Transforms of artifacts & collections.
+    """Transformations of artifacts & collections.
 
     A transform can refer to a script, a notebook, or a pipeline. If you execute
-    a script, a notebook, or a pipeline, you generate a run of a transform
+    a transform, you generate a run of a transform
     (:class:`~lamindb.Run`). A run has input and output data.
 
     A pipeline is typically created with a workflow tool (Nextflow, Snakemake,
@@ -839,7 +839,7 @@ class Transform(Registry, HasParents, IsVersioned):
     repository.
 
     Transforms are versioned so that a given transform maps 1:1 to a specific
-    version of a script, a notebook or a pipeline. If you switch on
+    version of code. If you switch on
     :attr:`~lamindb.core.Settings.sync_git_repo`, any script-like transform is
     synced its hashed state in a git repository.
 
@@ -1421,7 +1421,7 @@ class FeatureSet(Registry):
 
 
 class Artifact(Registry, Data, IsTree, IsVersioned):
-    """Artifacts: files, folders, or arrays.
+    """Artifacts: files, folders & arrays in storage.
 
     Args:
         path: `UPathStr` A path to a local or remote folder or file.
@@ -2033,10 +2033,20 @@ class Collection(Registry, Data, IsVersioned):
         stream: bool = False,
         is_run_input: Optional[bool] = None,
     ) -> "MappedCollection":
-        """Convert to map-style collection for data loaders.
+        """Return a map-style dataset.
 
-        Note: This currently only works for AnnData objects. The objects should
-        have the same label keys and variables.
+        Returns a `pytorch map-style dataset
+        <https://pytorch.org/docs/stable/data.html#map-style-datasets>`__ by
+        virtually concatenating `AnnData` arrays.
+
+        If your `AnnData` collection is in the cloud, move them into a local
+        cache first via :meth:`~lamindb.Collection.stage`.
+
+        .. note::
+
+            For a guide, see :doc:`docs:scrna5`.
+
+            This method currently only works for collections of `AnnData` artifacts.
 
         Args:
             label_keys: Columns of the ``.obs`` slot - the names of the metadata

{lamindb-0.69.6 → lamindb-0.69.7}/tests/test_run_context.py

@@ -93,6 +93,6 @@ def test_sync_git_repo():
     transform = ln.Transform.filter(name="initialized.py").one()
     assert (
         transform.reference
-        == "https://github.com/laminlabs/lamin-cli/blob/5d5b43879e7ad921dccb645e57d47a4147653473/tests/scripts/initialized.py"
+        == "https://github.com/laminlabs/lamin-cli/blob/5a6af4a4f4936a1ff3b11d3420aa8f6a5a72c677/tests/scripts/initialized.py"
     )
     assert transform.reference_type == "url"

lamindb-0.69.6/sub/lamin-cli/lamin_cli/__init__.py

@@ -1,3 +0,0 @@
-"""Lamin CLI."""
-
-__version__ = "0.11.0"

lamindb-0.69.6/sub/lamin-cli/lamin_cli/_save.py

@@ -1,49 +0,0 @@
-from pathlib import Path
-from typing import Optional, Union
-import lamindb_setup as ln_setup
-from lamin_utils import logger
-
-
-def save_from_filepath_cli(filepath: Union[str, Path]) -> Optional[str]:
-    if not isinstance(filepath, Path):
-        filepath = Path(filepath)
-
-    # this will be gone once we get rid of lamin load or enable loading multiple
-    # instances sequentially
-    auto_connect_state = ln_setup.settings.auto_connect
-    ln_setup.settings.auto_connect = True
-
-    import lamindb as ln
-    from lamindb.core._run_context import get_stem_uid_and_version_from_file
-    from lamindb._finish import save_run_context_core
-
-    ln_setup.settings.auto_connect = auto_connect_state
-
-    stem_uid, transform_version = get_stem_uid_and_version_from_file(filepath)
-
-    # the corresponding transform family in the transform table
-    transform_family = ln.Transform.filter(uid__startswith=stem_uid).all()
-    if len(transform_family) == 0:
-        logger.error(
-            f"Did not find stem uid '{stem_uid}'"
-            " in Transform registry. Did you run ln.track()?"
-        )
-        return "not-tracked-in-transform-registry"
-    # the specific version
-    transform = transform_family.filter(version=transform_version).one()
-    # latest run of this transform by user
-    run = ln.Run.filter(transform=transform).order_by("-started_at").first()
-    if run.created_by.id != ln_setup.settings.user.id:
-        response = input(
-            "You are trying to save a transform created by another user: Source and"
-            " report files will be tagged with *your* user id. Proceed? (y/n)"
-        )
-        if response != "y":
-            return "aborted-save-notebook-created-by-different-user"
-
-    return save_run_context_core(
-        run=run,
-        transform=transform,
-        filepath=filepath,
-        transform_family=transform_family,
-    )