lamindb 0.76.7__tar.gz → 0.76.9__tar.gz

{lamindb-0.76.7 → lamindb-0.76.9}/.gitmodules

@@ -13,3 +13,6 @@
 [submodule "sub/wetlab"]
 	path = sub/wetlab
 	url = https://github.com/laminlabs/wetlab
+[submodule "sub/clinicore"]
+	path = sub/clinicore
+	url = https://github.com/laminlabs/clinicore

{lamindb-0.76.7 → lamindb-0.76.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.76.7
+Version: 0.76.9
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,10 +9,10 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.74.3
-Requires-Dist: lamindb_setup==0.77.2
+Requires-Dist: lnschema_core==0.74.5
+Requires-Dist: lamindb_setup==0.77.5
 Requires-Dist: lamin_utils==0.13.4
-Requires-Dist: lamin_cli==0.17.3
+Requires-Dist: lamin_cli==0.17.6
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0

{lamindb-0.76.7 → lamindb-0.76.9}/docs/faq/setup.ipynb

@@ -1,7 +1,6 @@
 {
  "cells": [
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "261ca5e4-f46f-4361-9c3e-f944bbe14484",
    "metadata": {},
@@ -20,7 +19,7 @@
    },
    "outputs": [],
    "source": [
-    "!lamin close"
+    "!lamin load --unload"
    ]
   },
   {
@@ -36,7 +35,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "bbdb5cc9",
    "metadata": {},
@@ -70,7 +68,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "2d68e16d",
    "metadata": {},
@@ -90,7 +87,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "id": "c2fb4f9b",
    "metadata": {},
@@ -136,7 +132,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.9.17"
   },
   "vscode": {
    "interpreter": {

{lamindb-0.76.7 → lamindb-0.76.9}/docs/setup.ipynb

@@ -1,7 +1,6 @@
 {
  "cells": [
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -36,7 +35,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -44,7 +42,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -57,7 +54,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -69,7 +65,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -88,7 +83,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -98,7 +92,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -106,7 +99,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -116,7 +108,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -124,7 +115,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -135,7 +125,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -146,7 +135,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -154,7 +142,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -187,11 +174,12 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Settings persist in `~/.lamin/` and can also be accessed via {class}`lamindb.setup.settings`."
+    "Settings persist in `~/.lamin/` and can also be accessed via {class}`lamindb.setup.settings`. \n",
+    "\n",
+    "The settings directory can also be configured using `LAMIN_SETTINGS_DIR` environment variable."
    ]
   },
   {
@@ -222,7 +210,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -260,6 +247,46 @@
     "```"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Manage cache"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "`lamindb` mantains cache for cloud instances, i.e. instances having storage set to an AWS S3 bucket.\n",
+    "\n",
+    "Cache directory can be accessed via {class}`lamindb.settings`.\n",
+    "\n",
+    "```\n",
+    "ln.settings.storage.cache_dir\n",
+    "```\n",
+    "\n",
+    "or print the cache directory path with CLI\n",
+    "\n",
+    "```\n",
+    "lamin cache get\n",
+    "```\n",
+    "\n",
+    "It can be configured using the settings\n",
+    "\n",
+    "```\n",
+    "ln.settings.storage.cache_dir = \"some/path/to/cache\"\n",
+    "```\n",
+    "\n",
+    "or using CLI\n",
+    "\n",
+    "```\n",
+    "lamin cache set some/path/to/cache\n",
+    "```\n",
+    "\n",
+    "Cache directory can also be set using `LAMIN_CACHE_DIR` environment variable."
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -275,7 +302,7 @@
     "\n",
     "When loading a new instance, you automatically _close_ the previously loaded old instance.\n",
     "\n",
-    "If you want to close the instance without loading a new instance, use `lamin close`"
+    "If you want to close the instance without loading a new instance, use `lamin load --unload`"
    ]
   },
   {
@@ -356,7 +383,6 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -387,16 +413,16 @@
    "name": "python3"
   },
   "language_info": {
-   "artifact_extension": ".py",
    "codemirror_mode": {
     "name": "ipython",
     "version": 3
    },
+   "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.9.17"
   },
   "vscode": {
    "interpreter": {

{lamindb-0.76.7 → lamindb-0.76.9}/docs/track.ipynb

@@ -177,6 +177,35 @@
     "<br>"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Manage templates"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Save the template:\n",
+    "\n",
+    "```bash\n",
+    "lamin save template1.ipynb --key templates/template1.ipynb --description \"Template for analysis type 1\" --registry artifact\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Load the template:\n",
+    "\n",
+    "```bash\n",
+    "lamin get https://lamin.ai/account/instance/artifact/Akd7gx7Y9oVOJlei0001\n",
+    "```"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

{lamindb-0.76.7 → lamindb-0.76.9}/lamindb/__init__.py

@@ -41,7 +41,7 @@ Modules and settings.
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.76.7"
+__version__ = "0.76.9"
 
 import os as _os
 

{lamindb-0.76.7 → lamindb-0.76.9}/lamindb/_artifact.py

@@ -30,7 +30,7 @@ from lnschema_core.types import (
 from lamindb._utils import attach_func_to_class_method
 from lamindb.core._data import _track_run_input, describe, view_lineage
 from lamindb.core._settings import settings
-from lamindb.core.exceptions import IntegrityError
+from lamindb.core.exceptions import IntegrityError, InvalidArgument
 from lamindb.core.loaders import load_to_memory
 from lamindb.core.storage import (
     LocalPathClasses,
@@ -43,6 +43,7 @@ from lamindb.core.storage.paths import (
     auto_storage_key_from_artifact,
     auto_storage_key_from_artifact_uid,
     check_path_is_child_of_root,
+    filepath_cache_key_from_artifact,
     filepath_from_artifact,
 )
 from lamindb.core.versioning import (
@@ -88,8 +89,6 @@ def process_pathlike(
                 raise FileNotFoundError(filepath)
         except PermissionError:
             pass
-    if isinstance(filepath, LocalPathClasses):
-        filepath = filepath.resolve()
     if check_path_is_child_of_root(filepath, default_storage.root):
         use_existing_storage_key = True
         return default_storage, use_existing_storage_key
@@ -154,7 +153,7 @@ def process_data(
             if hasattr(default_storage, "_access_token")
             else None
         )
-        path = create_path(data, access_token=access_token)
+        path = create_path(data, access_token=access_token).resolve()
         storage, use_existing_storage_key = process_pathlike(
             path,
             default_storage=default_storage,
@@ -175,12 +174,12 @@ def process_data(
             key_suffix = None
         suffix = infer_suffix(data, format)
         if key_suffix is not None and key_suffix != suffix:
-            raise ValueError(
+            raise InvalidArgument(
                 f"The suffix '{key_suffix}' of the provided key is incorrect, it should"
                 f" be '{suffix}'."
             )
         cache_name = f"{provisional_uid}{suffix}"
-        path = settings._storage_settings.cache_dir / cache_name
+        path = settings.storage.cache_dir / cache_name
         # Alex: I don't understand the line below
         if path.suffixes == []:
             path = path.with_suffix(suffix)
@@ -344,8 +343,8 @@ def get_artifact_kwargs_from_data(
 
     if revises is not None:  # update provisional_uid
         provisional_uid, revises = create_uid(revises=revises, version=version)
-        if path.as_posix().startswith(settings._storage_settings.cache_dir.as_posix()):
-            path = path.rename(f"{provisional_uid}{suffix}")
+        if settings.storage.cache_dir in path.parents:
+            path = path.rename(path.with_name(f"{provisional_uid}{suffix}"))
 
     check_path_in_storage = False
     if use_existing_storage_key:
@@ -356,7 +355,7 @@ def get_artifact_kwargs_from_data(
             key = inferred_key
         else:
             if not key == inferred_key:
-                raise ValueError(
+                raise InvalidArgument(
                     f"The path '{data}' is already in registered storage"
                     f" '{storage.root}' with key '{inferred_key}'\nYou passed"
                     f" conflicting key '{key}': please move the file before"
@@ -399,6 +398,7 @@ def get_artifact_kwargs_from_data(
         "run_id": run.id if run is not None else None,
         "run": run,
         "_key_is_virtual": key_is_virtual,
+        "revises": revises,
     }
     if not isinstance(path, LocalPathClasses):
         local_filepath = None
@@ -614,6 +614,9 @@ def __init__(artifact: Artifact, *args, **kwargs):
     else:
         kwargs = kwargs_or_artifact
 
+    if revises is None:
+        revises = kwargs_or_artifact.pop("revises")
+
     if data is not None:
         artifact._local_filepath = privates["local_filepath"]
         artifact._cloud_filepath = privates["cloud_filepath"]
@@ -750,7 +753,7 @@ def from_dir(
         " ln.Artifact(dir_path) to get one artifact for the entire directory"
     )
     folderpath: UPath = create_path(path)  # returns Path for local
-    default_storage = settings._storage_settings.record
+    default_storage = settings.storage.record
     using_key = settings._using_key
     storage, use_existing_storage = process_pathlike(
         folderpath, default_storage, using_key
@@ -844,7 +847,7 @@ def replace(
     run: Run | None = None,
     format: str | None = None,
 ) -> None:
-    default_storage = settings._storage_settings.record
+    default_storage = settings.storage.record
     kwargs, privates = get_artifact_kwargs_from_data(
         provisional_uid=self.uid,
         data=data,
@@ -919,12 +922,14 @@ def open(
     from lamindb.core.storage._backed_access import _track_writes_factory, backed_access
 
     using_key = settings._using_key
-    filepath = filepath_from_artifact(self, using_key=using_key)
+    filepath, cache_key = filepath_cache_key_from_artifact(self, using_key=using_key)
     is_tiledbsoma_w = (
         filepath.name == "soma" or filepath.suffix == ".tiledbsoma"
     ) and mode == "w"
     # consider the case where an object is already locally cached
-    localpath = setup_settings.instance.storage.cloud_to_local_no_update(filepath)
+    localpath = setup_settings.instance.storage.cloud_to_local_no_update(
+        filepath, cache_key=cache_key
+    )
     if not is_tiledbsoma_w and localpath.exists():
         access = backed_access(localpath, mode, using_key)
     else:
@@ -956,15 +961,17 @@ def open(
 
 
 # can't really just call .cache in .load because of double tracking
-def _synchronize_cleanup_on_error(filepath: UPath) -> UPath:
+def _synchronize_cleanup_on_error(
+    filepath: UPath, cache_key: str | None = None
+) -> UPath:
     try:
         cache_path = setup_settings.instance.storage.cloud_to_local(
-            filepath, print_progress=True
+            filepath, cache_key=cache_key, print_progress=True
         )
     except Exception as e:
         if not isinstance(filepath, LocalPathClasses):
             cache_path = setup_settings.instance.storage.cloud_to_local_no_update(
-                filepath
+                filepath, cache_key=cache_key
             )
             if cache_path.is_file():
                 cache_path.unlink(missing_ok=True)
@@ -979,8 +986,11 @@ def load(self, is_run_input: bool | None = None, **kwargs) -> Any:
     if hasattr(self, "_memory_rep") and self._memory_rep is not None:
         access_memory = self._memory_rep
     else:
-        filepath = filepath_from_artifact(self, using_key=settings._using_key)
-        cache_path = _synchronize_cleanup_on_error(filepath)
+        filepath, cache_key = filepath_cache_key_from_artifact(
+            self, using_key=settings._using_key
+        )
+        cache_path = _synchronize_cleanup_on_error(filepath, cache_key=cache_key)
+        # cache_path is local so doesn't trigger any sync in load_to_memory
         access_memory = load_to_memory(cache_path, **kwargs)
     # only call if load is successfull
     _track_run_input(self, is_run_input)
@@ -989,8 +999,10 @@ def load(self, is_run_input: bool | None = None, **kwargs) -> Any:
 
 # docstring handled through attach_func_to_class_method
 def cache(self, is_run_input: bool | None = None) -> Path:
-    filepath = filepath_from_artifact(self, using_key=settings._using_key)
-    cache_path = _synchronize_cleanup_on_error(filepath)
+    filepath, cache_key = filepath_cache_key_from_artifact(
+        self, using_key=settings._using_key
+    )
+    cache_path = _synchronize_cleanup_on_error(filepath, cache_key=cache_key)
     # only call if sync is successfull
     _track_run_input(self, is_run_input)
     return cache_path
@@ -1041,7 +1053,7 @@ def delete(
     if delete_record:
         # need to grab file path before deletion
         try:
-            path = filepath_from_artifact(self, using_key)
+            path, _ = filepath_from_artifact(self, using_key)
         except OSError:
             # we can still delete the record
             logger.warning("Could not get path")
@@ -1133,8 +1145,22 @@ def _save_skip_storage(file, **kwargs) -> None:
 @doc_args(Artifact.path.__doc__)
 def path(self) -> Path | UPath:
     """{}"""  # noqa: D415
-    using_key = settings._using_key
-    return filepath_from_artifact(self, using_key)
+    # return only the path, without StorageSettings
+    filepath, _ = filepath_from_artifact(self, using_key=settings._using_key)
+    return filepath
+
+
+# get cache path without triggering sync
+@property  # type: ignore
+def _cache_path(self) -> UPath:
+    filepath, cache_key = filepath_cache_key_from_artifact(
+        self, using_key=settings._using_key
+    )
+    if isinstance(filepath, LocalPathClasses):
+        return filepath
+    return setup_settings.instance.storage.cloud_to_local_no_update(
+        filepath, cache_key=cache_key
+    )
 
 
 # docstring handled through attach_func_to_class_method
@@ -1173,6 +1199,7 @@ for name in METHOD_NAMES:
 # privates currently dealt with separately
 Artifact._delete_skip_storage = _delete_skip_storage
 Artifact._save_skip_storage = _save_skip_storage
+Artifact._cache_path = _cache_path
 Artifact.path = path
 Artifact.describe = describe
 Artifact.view_lineage = view_lineage

{lamindb-0.76.7 → lamindb-0.76.9}/lamindb/_collection.py

@@ -211,6 +211,7 @@ def mapped(
     layers_keys: str | list[str] | None = None,
     obs_keys: str | list[str] | None = None,
     obsm_keys: str | list[str] | None = None,
+    obs_filter: tuple[str, str | tuple[str, ...]] | None = None,
     join: Literal["inner", "outer"] | None = "inner",
     encode_labels: bool | list[str] = True,
     unknown_label: str | dict[str, str] | None = None,
@@ -239,6 +240,7 @@ def mapped(
         layers_keys,
         obs_keys,
         obsm_keys,
+        obs_filter,
         join,
         encode_labels,
         unknown_label,

{lamindb-0.76.7 → lamindb-0.76.9}/lamindb/_save.py

@@ -17,6 +17,7 @@ from lnschema_core.models import Artifact, Record
 
 from lamindb.core._settings import settings
 from lamindb.core.storage.paths import (
+    _cache_key_from_artifact_storage,
     attempt_accessing_path,
     auto_storage_key_from_artifact,
     delete_storage_using_key,
@@ -135,7 +136,7 @@ def check_and_attempt_upload(
     # a local env it will have a _local_filepath and needs to be uploaded
     if hasattr(artifact, "_local_filepath"):
         try:
-            storage_path = upload_artifact(
+            storage_path, cache_path = upload_artifact(
                 artifact,
                 using_key,
                 access_token=access_token,
@@ -146,7 +147,7 @@ def check_and_attempt_upload(
             return exception
         # copies (if on-disk) or moves the temporary file (if in-memory) to the cache
         if os.getenv("LAMINDB_MULTI_INSTANCE") is None:
-            copy_or_move_to_cache(artifact, storage_path)
+            copy_or_move_to_cache(artifact, storage_path, cache_path)
         # after successful upload, we should remove the attribute so that another call
         # call to save won't upload again, the user should call replace() then
         del artifact._local_filepath
@@ -154,7 +155,9 @@ def check_and_attempt_upload(
     return None
 
 
-def copy_or_move_to_cache(artifact: Artifact, storage_path: UPath):
+def copy_or_move_to_cache(
+    artifact: Artifact, storage_path: UPath, cache_path: UPath | None
+):
     local_path = artifact._local_filepath
 
     # in-memory cases
@@ -166,7 +169,7 @@ def copy_or_move_to_cache(artifact: Artifact, storage_path: UPath):
     cache_dir = settings._storage_settings.cache_dir
 
     # just delete from the cache dir if storage_path is local
-    if isinstance(storage_path, LocalPathClasses):
+    if cache_path is None:
         if (
             local_path.as_posix() != storage_path.as_posix()
             and cache_dir in local_path.parents
@@ -176,11 +179,12 @@ def copy_or_move_to_cache(artifact: Artifact, storage_path: UPath):
             else:
                 local_path.unlink()
         return None
-
-    cache_path = settings._storage_settings.cloud_to_local_no_update(storage_path)
+    # non-local storage_path further
     if local_path != cache_path:
         cache_path.parent.mkdir(parents=True, exist_ok=True)
         if cache_dir in local_path.parents:
+            if cache_path.is_dir():
+                shutil.rmtree(cache_path)
             local_path.replace(cache_path)
         else:
             if is_dir:
@@ -280,11 +284,11 @@ def upload_artifact(
     using_key: str | None = None,
     access_token: str | None = None,
     print_progress: bool = True,
-) -> UPath:
+) -> tuple[UPath, UPath | None]:
     """Store and add file and its linked entries."""
     # can't currently use  filepath_from_artifact here because it resolves to ._local_filepath
     storage_key = auto_storage_key_from_artifact(artifact)
-    storage_path = attempt_accessing_path(
+    storage_path, storage_settings = attempt_accessing_path(
         artifact, storage_key, using_key=using_key, access_token=access_token
     )
     if hasattr(artifact, "_to_store") and artifact._to_store:
@@ -292,4 +296,13 @@ def upload_artifact(
         store_file_or_folder(
             artifact._local_filepath, storage_path, print_progress=print_progress
         )
-    return storage_path
+
+    if isinstance(storage_path, LocalPathClasses):
+        cache_path = None
+    else:
+        cache_key = _cache_key_from_artifact_storage(artifact, storage_settings)
+        cache_path = storage_settings.cloud_to_local_no_update(
+            storage_path, cache_key=cache_key
+        )
+
+    return storage_path, cache_path

{lamindb-0.76.7 → lamindb-0.76.9}/lamindb/core/_context.py

@@ -450,11 +450,11 @@ class Context:
             self._logging_message += f"created Transform(uid='{transform.uid}')"
         else:
             uid = transform.uid
-            # check whether the transform file has been renamed
+            # check whether the transform.key is consistent
             if transform.key != key:
                 suid = transform.stem_uid
                 new_suid = ids.base62_12()
-                transform_type = "Notebook" if is_run_from_ipython else "Script"
+                transform_type = "notebook" if is_run_from_ipython else "script"
                 note = message_update_key_in_version_family(
                     suid=suid,
                     existing_key=transform.key,
@@ -462,7 +462,7 @@ class Context:
                     registry="Transform",
                 )
                 raise UpdateContext(
-                    f"{transform_type} filename changed.\n\nEither init a new transform family by setting:\n\n"
+                    f'\n✗ Filename "{key}" clashes with the existing key "{transform.key}" for uid "{transform.uid[:-4]}...."\n\nEither init a new transform with a new uid:\n\n'
                     f'ln.context.uid = "{new_suid}0000"\n\n{note}'
                 )
             elif transform.name != name:

{lamindb-0.76.7 → lamindb-0.76.9}/lamindb/core/_data.py

@@ -41,7 +41,7 @@ if TYPE_CHECKING:
 
 
 WARNING_RUN_TRANSFORM = (
-    "no run & transform got linked, call `ln.context.track()` & re-run`"
+    "no run & transform got linked, call `ln.context.track()` & re-run"
 )
 
 WARNING_NO_INPUT = "run input wasn't tracked, call `ln.context.track()` and re-run"