lamindb 0.76.6__tar.gz → 0.76.7__tar.gz

{lamindb-0.76.6 → lamindb-0.76.7}/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -1,6 +1,7 @@
 name: Bug report
 description: Report something that is broken or incorrect
-labels: bug
+labels:
+  - ":bug: bug"
 body:
   - type: markdown
     attributes:

{lamindb-0.76.6 → lamindb-0.76.7}/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -1,6 +1,7 @@
 name: Feature request
 description: Propose a new feature for lamindb
-labels: enhancement
+labels:
+  - ":sparkles: feature"
 body:
   - type: markdown
     attributes:

lamindb-0.76.7/.github/ISSUE_TEMPLATE/usage_question.yml

@@ -0,0 +1,20 @@
+name: Usage question
+description: Ask a usage question
+labels:
+  - ":children_crossing: ux"
+body:
+  - type: markdown
+    attributes:
+      value: |
+        **Note**: This is a public repository! Do not reveal any internal information.
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Usage question
+      description: Please ask your question.
+      placeholder: |
+        This is a public repository!
+        Do not reveal any internal information.
+    validations:
+      required: true

{lamindb-0.76.6 → lamindb-0.76.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.76.6
+Version: 0.76.7
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,10 +9,10 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.74.1
-Requires-Dist: lamindb_setup==0.77.1
+Requires-Dist: lnschema_core==0.74.3
+Requires-Dist: lamindb_setup==0.77.2
 Requires-Dist: lamin_utils==0.13.4
-Requires-Dist: lamin_cli==0.17.2
+Requires-Dist: lamin_cli==0.17.3
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
@@ -24,7 +24,7 @@ Requires-Dist: pandas
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
 Requires-Dist: lamindb_setup[aws] ; extra == "aws"
-Requires-Dist: bionty==0.50.1 ; extra == "bionty"
+Requires-Dist: bionty==0.50.2 ; extra == "bionty"
 Requires-Dist: pre-commit ; extra == "dev"
 Requires-Dist: nox ; extra == "dev"
 Requires-Dist: laminci>=0.3 ; extra == "dev"

{lamindb-0.76.6 → lamindb-0.76.7}/docs/schemas.ipynb

@@ -44,7 +44,7 @@
    "id": "7e7c43f8",
    "metadata": {},
    "source": [
-    "<img src=\"https://lamin-site-assets.s3.amazonaws.com/.lamindb/XoTQFCmmj2uU4d2xyj9t.png\" width=\"350px\" style=\"background: transparent\" align=\"right\">\n"
+    "<img src=\"https://lamin-site-assets.s3.amazonaws.com/.lamindb/XoTQFCmmj2uU4d2xyj9u.png\" width=\"350px\" style=\"background: transparent\" align=\"right\">\n"
    ]
   },
   {
@@ -64,7 +64,7 @@
    "id": "f5375b44",
    "metadata": {},
    "source": [
-    "### View the entire schema"
+    "### View the schema"
    ]
   },
   {
@@ -73,9 +73,9 @@
    "id": "de6fdc7e",
    "metadata": {},
    "source": [
-    "For your given instance, you view the registries by running `lamin schema view` on the command line.\n",
+    "For your given instance, you view the registries by running `lamin info schema` on the command line.\n",
     "\n",
-    "In your browser, navigating to `<localhost>/schema/` will show an interactive version of graph similar to the one on the right."
+    "Your browser will show an interactive version of a graph similar to the one on the right. In the left side panel, you can choose to hide the relationships you're not interested in and you'll be able to trace some relationships."
    ]
   },
   {

{lamindb-0.76.6 → lamindb-0.76.7}/docs/setup.ipynb

@@ -56,16 +56,6 @@
     "- `schema` (optional): comma-separated string of schema modules"
    ]
   },
-  {
-   "attachments": {},
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Examples\n",
-    "\n",
-    "#### Local storage + SQLite"
-   ]
-  },
   {
    "attachments": {},
    "cell_type": "markdown",
@@ -91,11 +81,10 @@
    ]
   },
   {
-   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "#### S3 + SQLite"
+    "You can also pass an AWS S3 bucket."
    ]
   },
   {
@@ -113,7 +102,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "#### GCP + Postgres"
+    "Instead of SQLite, you can pass a Postgres connection string."
    ]
   },
   {

{lamindb-0.76.6 → lamindb-0.76.7}/docs/signup-login.md

@@ -8,9 +8,9 @@
 
 An account is free & [signing up](https://lamin.ai/signup) takes 1 min.
 
-Lamin does _not_ store or see any of your data, but only _basic_ metadata about you (email address, etc.).
+Through signing up, Lamin does _not_ store or see any of your data, but only _basic_ metadata about you (email address, etc.).
 
-If you register a LaminDB instance on LaminHub, Lamin only stores the storage location (AWS S3 or GCP bucket names, directory names).
+If you register a LaminDB instance to LaminHub, we store metadata about the involved infrastructure (database server, storage locations, etc.). But we don't store any secrets when you call `lamin init` and can't see your data or metadata. If you want to connect your instance to LaminHub for exploration with read-write access: please [reach out](https://lamin.ai/contact).
 
 For more, see [doc](inv:docs#access), [the source code](https://github.com/laminlabs/lamindb-setup), or the [privacy policy](https://lamin.ai/legal/privacy-policy).
 
@@ -32,5 +32,5 @@ lamin login <email> --key <API-key>
 Log out:
 
 ```
-lamin lgout
+lamin login --logout
 ```

{lamindb-0.76.6 → lamindb-0.76.7}/docs/storage/prepare-transfer-local-to-cloud.ipynb

@@ -55,10 +55,7 @@
     "organism.save()\n",
     "artifact.labels.add(organism)\n",
     "\n",
-    "experiment_type = wl.ExperimentType(name=\"experiment-type-test-transfer-to-cloud\")\n",
-    "experiment_type.save()\n",
     "experiment = wl.Experiment(name=\"experiment-test-transfer-to-cloud\")\n",
-    "experiment.experiment_type = experiment_type\n",
     "experiment.save()\n",
     "\n",
     "artifact.experiments.add(experiment)\n",
@@ -78,7 +75,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "py310",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -92,7 +89,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.9.17"
   }
  },
  "nbformat": 4,

{lamindb-0.76.6 → lamindb-0.76.7}/docs/storage/transfer-local-to-cloud.ipynb

@@ -33,11 +33,7 @@
     "def cleanup(artifact: ln.Artifact):\n",
     "    features_sets = artifact.feature_sets.all()\n",
     "    experiments = artifact.experiments.all()\n",
-    "    try:\n",
-    "        artifact.delete(permanent=True, storage=False)\n",
-    "    except Exception as e:\n",
-    "        print(e)\n",
-    "        pass\n",
+    "    artifact.delete(permanent=True, storage=False)\n",
     "    features_sets.delete()\n",
     "    experiments.delete()\n",
     "\n",
@@ -99,7 +95,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "py310",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -113,7 +109,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.9.17"
   }
  },
  "nbformat": 4,

{lamindb-0.76.6 → lamindb-0.76.7}/docs/tutorial.ipynb

@@ -722,7 +722,25 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Are there remaining questions about storing artifacts? If so, see: {doc}`docs:faq/storage`."
+    "## File-like artifacts"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Above, we looked at a folder-like artifact. Here is a file-like artifact."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "local_filepath = ln.core.datasets.file_jpg_paradisi05()\n",
+    "artifact = ln.Artifact(local_filepath, description=\"My single image\").save()\n",
+    "artifact.load()"
    ]
   },
   {
@@ -886,7 +904,7 @@
    "metadata": {},
    "source": [
     "```\n",
-    "ln.finish()\n",
+    "ln.context.finish()\n",
     "```"
    ]
   },

{lamindb-0.76.6 → lamindb-0.76.7}/lamindb/__init__.py

@@ -41,7 +41,7 @@ Modules and settings.
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.76.6"
+__version__ = "0.76.7"
 
 import os as _os
 

{lamindb-0.76.6 → lamindb-0.76.7}/lamindb/_artifact.py

@@ -31,12 +31,12 @@ from lamindb._utils import attach_func_to_class_method
 from lamindb.core._data import _track_run_input, describe, view_lineage
 from lamindb.core._settings import settings
 from lamindb.core.exceptions import IntegrityError
+from lamindb.core.loaders import load_to_memory
 from lamindb.core.storage import (
     LocalPathClasses,
     UPath,
     delete_storage,
     infer_suffix,
-    load_to_memory,
     write_to_disk,
 )
 from lamindb.core.storage.paths import (
@@ -955,24 +955,8 @@ def open(
     return access
 
 
-# docstring handled through attach_func_to_class_method
-def load(self, is_run_input: bool | None = None, stream: bool = False, **kwargs) -> Any:
-    if hasattr(self, "_memory_rep") and self._memory_rep is not None:
-        access_memory = self._memory_rep
-    else:
-        using_key = settings._using_key
-        access_memory = load_to_memory(
-            filepath_from_artifact(self, using_key=using_key), stream=stream, **kwargs
-        )
-    # only call if load is successfull
-    _track_run_input(self, is_run_input)
-    return access_memory
-
-
-# docstring handled through attach_func_to_class_method
-def cache(self, is_run_input: bool | None = None) -> Path:
-    using_key = settings._using_key
-    filepath = filepath_from_artifact(self, using_key=using_key)
+# can't really just call .cache in .load because of double tracking
+def _synchronize_cleanup_on_error(filepath: UPath) -> UPath:
     try:
         cache_path = setup_settings.instance.storage.cloud_to_local(
             filepath, print_progress=True
@@ -987,6 +971,26 @@ def cache(self, is_run_input: bool | None = None) -> Path:
             elif cache_path.is_dir():
                 shutil.rmtree(cache_path)
         raise e
+    return cache_path
+
+
+# docstring handled through attach_func_to_class_method
+def load(self, is_run_input: bool | None = None, **kwargs) -> Any:
+    if hasattr(self, "_memory_rep") and self._memory_rep is not None:
+        access_memory = self._memory_rep
+    else:
+        filepath = filepath_from_artifact(self, using_key=settings._using_key)
+        cache_path = _synchronize_cleanup_on_error(filepath)
+        access_memory = load_to_memory(cache_path, **kwargs)
+    # only call if load is successfull
+    _track_run_input(self, is_run_input)
+    return access_memory
+
+
+# docstring handled through attach_func_to_class_method
+def cache(self, is_run_input: bool | None = None) -> Path:
+    filepath = filepath_from_artifact(self, using_key=settings._using_key)
+    cache_path = _synchronize_cleanup_on_error(filepath)
     # only call if sync is successfull
     _track_run_input(self, is_run_input)
     return cache_path

{lamindb-0.76.6 → lamindb-0.76.7}/lamindb/_collection.py

@@ -221,7 +221,12 @@ def mapped(
     is_run_input: bool | None = None,
 ) -> MappedCollection:
     path_list = []
-    for artifact in self.ordered_artifacts.all():
+    if self._state.adding:
+        artifacts = self._artifacts
+        logger.warning("The collection isn't saved, consider calling `.save()`")
+    else:
+        artifacts = self.ordered_artifacts.all()
+    for artifact in artifacts:
         if artifact.suffix not in {".h5ad", ".zarr"}:
             logger.warning(f"Ignoring artifact with suffix {artifact.suffix}")
             continue

{lamindb-0.76.6 → lamindb-0.76.7}/lamindb/_curate.py

@@ -1188,7 +1188,7 @@ def validate_categories(
         print_values = _print_values(non_validated)
         warning_message = (
             f"{colors.red(f'{n_non_validated} terms')} {are} not validated: "
-            f"{colors.red(print_values)}\n      → save terms via "
+            f"{colors.red(print_values)}\n      → fix typos, remove non-existent values, or save terms via "
             f"{colors.red(non_validated_hint_print)}"
         )
         if logger.indent == "":

{lamindb-0.76.6 → lamindb-0.76.7}/lamindb/core/__init__.py

@@ -54,6 +54,7 @@ Modules:
 .. autosummary::
    :toctree: .
 
+   loaders
    datasets
    storage
    types
@@ -87,7 +88,7 @@ from lamindb._query_set import QuerySet, RecordsList
 from lamindb.core._feature_manager import FeatureManager, ParamManager
 from lamindb.core._label_manager import LabelManager
 
-from . import _data, datasets, exceptions, fields, subsettings, types
+from . import _data, datasets, exceptions, fields, loaders, subsettings, types
 from ._context import Context
 from ._mapped_collection import MappedCollection
 from ._settings import Settings

{lamindb-0.76.6 → lamindb-0.76.7}/lamindb/core/_context.py

@@ -18,8 +18,8 @@ from ._settings import settings
 from ._sync_git import get_transform_reference_from_git_repo
 from ._track_environment import track_environment
 from .exceptions import (
-    MissingContext,
-    NotebookFileNotSavedToDisk,
+    MissingContextUID,
+    NotebookNotSaved,
     NotebookNotSavedError,
     NoTitleError,
     TrackNotCalled,
@@ -81,21 +81,30 @@ def get_notebook_name_colab() -> str:
     return name.rstrip(".ipynb")
 
 
-def raise_missing_context(transform_type: str, key: str) -> None:
+def raise_missing_context(transform_type: str, key: str) -> bool:
     transform = Transform.filter(key=key).latest_version().first()
     if transform is None:
         new_uid = f"{base62_12()}0000"
-        message = f"To track this {transform_type}, set\n\n"
+        message = f"To track this {transform_type}, copy & paste the below into the current cell and re-run it\n\n"
+        message += f'ln.context.uid = "{new_uid}"\nln.context.track()'
     else:
         uid = transform.uid
         suid, vuid = uid[: Transform._len_stem_uid], uid[Transform._len_stem_uid :]
         new_vuid = increment_base62(vuid)
         new_uid = f"{suid}{new_vuid}"
-        message = f"You already have a {transform_type} version family with key '{key}', suid '{transform.stem_uid}' & name '{transform.name}'.\n\n- to create a new {transform_type} version family, rename your file and rerun: ln.context.track()\n- to bump the version, set: "
-    message += f'ln.context.uid = "{new_uid}"'
+        message = f"You already have a version family with key '{key}' (stem_uid='{transform.stem_uid}').\n\n- to make a revision, set `ln.context.uid = '{new_uid}'`\n- to start a new version family, rename your file and rerun: `ln.context.track()`"
     if transform_type == "notebook":
-        message += "\n\nRestart your notebook if you want consecutive cell execution."
-    raise MissingContext(message)
+        print(f"→ {message}\n")
+        response = input("→ Ready to re-run? (y/n)")
+        if response == "y":
+            logger.important(
+                "Note: Restart your notebook if you want consecutive cell execution"
+            )
+            return True
+        raise MissingContextUID("Please follow the instructions.")
+    else:
+        raise MissingContextUID(message)
+    return False
 
 
 def pretty_pypackages(dependencies: dict) -> str:
@@ -280,7 +289,9 @@ class Context:
                 # if no error is raised, the transform is tracked
                 is_tracked = True
             if not is_tracked:
-                raise_missing_context(transform_type, key)
+                early_return = raise_missing_context(transform_type, key)
+                if early_return:
+                    return None
         else:
             if transform.type in {"notebook", "script"}:
                 raise ValueError(
@@ -508,7 +519,7 @@ class Context:
         When called in the last cell of a notebook:
 
         - prompts for user input if not consecutively executed
-        - requires to save the notebook in your editor
+        - requires to save the notebook in your editor right before
         - saves a run report: `run.report`
 
         Args:
@@ -530,6 +541,11 @@ class Context:
         def get_seconds_since_modified(filepath) -> float:
             return datetime.now().timestamp() - filepath.stat().st_mtime
 
+        def get_shortcut() -> str:
+            import platform
+
+            return "CMD + s" if platform.system() == "Darwin" else "CTRL + s"
+
         if context.run is None:
             raise TrackNotCalled("Please run `ln.context.track()` before `ln.finish()`")
         if context._path is None:
@@ -543,8 +559,8 @@ class Context:
             return None
         if is_run_from_ipython:  # notebooks
             if get_seconds_since_modified(context._path) > 2 and not ln_setup._TESTING:
-                raise NotebookFileNotSavedToDisk(
-                    "Please save the notebook manually in your editor right before running `ln.context.finish()`"
+                raise NotebookNotSaved(
+                    f"Please save the notebook in your editor (shortcut `{get_shortcut()}`) right before calling `ln.context.finish()`"
                 )
         save_context_core(
             run=context.run,

{lamindb-0.76.6 → lamindb-0.76.7}/lamindb/core/_data.py

@@ -39,10 +39,13 @@ from .schema import (
 if TYPE_CHECKING:
     from lnschema_core.types import StrField
 
+
 WARNING_RUN_TRANSFORM = (
-    "no run & transform get linked, consider calling ln.context.track()"
+    "no run & transform got linked, call `ln.context.track()` & re-run`"
 )
 
+WARNING_NO_INPUT = "run input wasn't tracked, call `ln.context.track()` and re-run"
+
 
 def get_run(run: Run | None) -> Run | None:
     if run is None:
@@ -384,10 +387,7 @@ def _track_run_input(
         # we don't have a run record
         if run is None:
             if settings.track_run_inputs:
-                logger.hint(
-                    "you can auto-track these data as a run input by calling"
-                    " `ln.context.track()`"
-                )
+                logger.warning(WARNING_NO_INPUT)
         # assume we have a run record
         else:
             # assume there is non-cyclic candidate input data

{lamindb-0.76.6 → lamindb-0.76.7}/lamindb/core/_feature_manager.py

@@ -808,7 +808,8 @@ def _add_from(self, data: Artifact | Collection, transfer_logs: dict = None):
         # create records from ontology_id
         if hasattr(registry, "_ontology_id_field") and len(member_uids) > 0:
             # create from bionty
-            save(registry.from_values(member_uids, field=field))
+            members_records = registry.from_values(member_uids, field=field)
+            save([r for r in members_records if r._state.adding])
         validated = registry.validate(member_uids, field=field, mute=True)
         new_members_uids = list(compress(member_uids, ~validated))
         new_members = members.filter(**{f"{field}__in": new_members_uids}).all()

{lamindb-0.76.6 → lamindb-0.76.7}/lamindb/core/_label_manager.py

@@ -98,7 +98,8 @@ def validate_labels(labels: QuerySet | list | dict):
         # save labels from ontology_ids
         if hasattr(registry, "_ontology_id_field") and len(label_uids) > 0:
             try:
-                save(registry.from_values(label_uids, field=field))
+                labels_records = registry.from_values(label_uids, field=field)
+                save([r for r in labels_records if r._state.adding])
             except Exception:  # noqa S110
                 pass
             field = "uid"

{lamindb-0.76.6 → lamindb-0.76.7}/lamindb/core/exceptions.py

@@ -9,7 +9,7 @@ The registry base class:
    ValidationError
    NotebookNotSavedError
    NoTitleError
-   MissingContext
+   MissingContextUID
    UpdateContext
    IntegrityError
 
@@ -20,7 +20,7 @@ class TrackNotCalled(SystemExit):
     pass
 
 
-class NotebookFileNotSavedToDisk(SystemExit):
+class NotebookNotSaved(SystemExit):
     pass
 
 
@@ -65,7 +65,7 @@ class NoTitleError(Exception):
     pass
 
 
-class MissingContext(SystemExit):
+class MissingContextUID(SystemExit):
     """User didn't define transform settings."""
 
     pass