lamindb 0.76.11__tar.gz → 0.76.13__tar.gz

{lamindb-0.76.11 → lamindb-0.76.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.76.11
+Version: 0.76.13
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.9
@@ -8,10 +8,10 @@ Description-Content-Type: text/markdown
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.74.6
-Requires-Dist: lamindb_setup==0.77.7
+Requires-Dist: lnschema_core==0.75.0
 Requires-Dist: lamin_utils==0.13.6
-Requires-Dist: lamin_cli==0.17.8
+Requires-Dist: lamin_cli==0.19.0
+Requires-Dist: lamindb_setup
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
@@ -23,7 +23,7 @@ Requires-Dist: pandas
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
 Requires-Dist: lamindb_setup[aws] ; extra == "aws"
-Requires-Dist: bionty==0.51.1 ; extra == "bionty"
+Requires-Dist: bionty==0.51.2 ; extra == "bionty"
 Requires-Dist: line_profiler ; extra == "dev"
 Requires-Dist: pre-commit ; extra == "dev"
 Requires-Dist: nox ; extra == "dev"

{lamindb-0.76.11 → lamindb-0.76.13}/docs/faq/idempotency.ipynb

@@ -70,7 +70,6 @@
     "import lamindb as ln\n",
     "import pytest\n",
     "\n",
-    "ln.settings.verbosity = \"hint\"\n",
     "ln.track(\"ANW20Fr4eZgM0000\")"
    ]
   },
@@ -301,8 +300,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "artifact = ln.Artifact(filepath, description=\"My fcs artifact\")\n",
-    "artifact.save()"
+    "artifact = ln.Artifact(filepath, description=\"My fcs artifact\").save()"
    ]
   },
   {
@@ -316,7 +314,9 @@
    },
    "outputs": [],
    "source": [
-    "assert artifact.hash == \"KCEXRahJ-Ui9Y6nksQ8z1A\""
+    "assert artifact.hash == \"KCEXRahJ-Ui9Y6nksQ8z1A\"\n",
+    "assert artifact.run == ln.context.run\n",
+    "assert len(artifact._previous_runs.all()) == 0"
    ]
   },
   {
@@ -355,7 +355,8 @@
    "outputs": [],
    "source": [
     "assert artifact.id == artifact2.id\n",
-    "assert artifact.run == artifact2.run"
+    "assert artifact.run == artifact2.run\n",
+    "assert len(artifact._previous_runs.all()) == 0"
    ]
   },
   {
@@ -485,8 +486,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "artifact4 = ln.Artifact(filepath, description=\"My new fcs artifact\")\n",
-    "artifact4.save()"
+    "artifact4 = ln.Artifact(filepath, description=\"My new fcs artifact\").save()"
    ]
   },
   {
@@ -543,6 +543,7 @@
    },
    "outputs": [],
    "source": [
+    "!rm -rf ./test-idempotency\n",
     "!lamin delete --force test-idempotency"
    ]
   }

{lamindb-0.76.11 → lamindb-0.76.13}/docs/faq/setup.ipynb

@@ -100,7 +100,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "with pytest.raises(RuntimeError):\n",
+    "from lamindb_setup._init_instance import CannotSwitchDefaultInstance\n",
+    "\n",
+    "with pytest.raises(CannotSwitchDefaultInstance):\n",
     "    ln.setup.init(storage=\"./testsetup2\")"
    ]
   },

{lamindb-0.76.11 → lamindb-0.76.13}/docs/scripts/run-track-with-params.py

@@ -10,8 +10,11 @@ if __name__ == "__main__":
     params = {
         "input_dir": args.input_dir,
         "learning_rate": args.learning_rate,
-        "downsample": args.downsample,
+        "preprocess_params": {
+            "downsample": args.downsample,
+            "normalization": "the_good_one",
+        },
     }
-    ln.track("JjRF4mACd9m00000", params=params)
+    ln.track("JjRF4mACd9m00001", params=params)
     # your code
     ln.finish()

{lamindb-0.76.11 → lamindb-0.76.13}/docs/setup.ipynb

@@ -254,25 +254,39 @@
    "source": [
     "`lamindb` mantains cache for cloud instances, i.e. instances having storage set to an AWS S3 bucket.\n",
     "\n",
-    "Cache directory can be accessed via {class}`lamindb.settings`.\n",
-    "\n",
-    "```\n",
-    "ln.settings.storage.cache_dir\n",
-    "```\n",
-    "\n",
-    "or print the cache directory path with CLI\n",
-    "\n",
-    "```\n",
-    "lamin cache get\n",
-    "```\n",
-    "\n",
-    "It can be configured using the settings\n",
-    "\n",
-    "```\n",
-    "ln.settings.storage.cache_dir = \"some/path/to/cache\"\n",
-    "```\n",
-    "\n",
-    "or using CLI\n",
+    "Cache directory can be accessed via {class}`lamindb.settings`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ln.settings.cache_dir"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "or print the cache directory path with CLI"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!lamin cache get"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "It can be also configured via \n",
     "\n",
     "```\n",
     "lamin cache set some/path/to/cache\n",

{lamindb-0.76.11 → lamindb-0.76.13}/docs/storage/prepare-transfer-local-to-cloud.ipynb

@@ -34,7 +34,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "ln.setup.init(storage=\"./test-transfer-to-cloud\", schema=\"bionty,wetlab\")\n",
+    "ln.setup.init(storage=\"./test-transfer-to-cloud\", schema=\"bionty,wetlab,findrefs\")\n",
     "ln.setup.settings.auto_connect = False"
    ]
   },

{lamindb-0.76.11 → lamindb-0.76.13}/docs/storage/vitessce.ipynb

@@ -82,7 +82,7 @@
     "vc = VitessceConfig(schema_version=\"1.0.15\")\n",
     "vc.add_dataset(name=\"test1\").add_object(\n",
     "    AnnDataWrapper(\n",
-    "        adata_url=dataset_artifact.path.to_url(),\n",
+    "        adata_artifact=dataset_artifact,\n",
     "        obs_embedding_paths=[\"obsm/X_umap\"],\n",
     "    ),\n",
     ")\n",
@@ -134,39 +134,17 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# use a corrupted suffix\n",
+    "# pass an artifact URL instead of the artifact object itself\n",
     "vc = VitessceConfig(schema_version=\"1.0.15\")\n",
-    "vc.add_dataset(name=\"test1\").add_object(\n",
-    "    AnnDataWrapper(\n",
-    "        adata_url=dataset_artifact.path.to_url() + \".random\",\n",
-    "        obs_embedding_paths=[\"obsm/X_umap\"],\n",
-    "    ),\n",
-    ")\n",
-    "with pytest.raises(ValueError) as error:\n",
-    "    ln.integrations.save_vitessce_config(vc)\n",
-    "print(error.exconly())\n",
-    "assert error.exconly().startswith(\"ValueError: Suffix should be '.zarr' or one of\")\n",
-    "assert error.exconly().endswith(\".random\")\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# use a corrupted stem_uid\n",
-    "vc = VitessceConfig(schema_version=\"1.0.15\")\n",
-    "vc.add_dataset(name=\"test1\").add_object(\n",
-    "    AnnDataWrapper(\n",
-    "        adata_url=dataset_artifact.path.to_url().replace(dataset_artifact.stem_uid, \"not-a-valid-uid\"),\n",
-    "        obs_embedding_paths=[\"obsm/X_umap\"],\n",
-    "    ),\n",
-    ")\n",
-    "with pytest.raises(ValueError) as error:\n",
-    "    ln.integrations.save_vitessce_config(vc)\n",
+    "with pytest.raises(AttributeError) as error:\n",
+    "    vc.add_dataset(name=\"test1\").add_object(\n",
+    "        AnnDataWrapper(\n",
+    "            adata_artifact=dataset_artifact.path.to_url(),\n",
+    "            obs_embedding_paths=[\"obsm/X_umap\"],\n",
+    "        ),\n",
+    "    )\n",
     "print(error.exconly())\n",
-    "assert error.exconly().startswith(\"ValueError: Could not find dataset with stem uid 'not-a-valid-uid' in lamindb\")"
+    "assert error.exconly().startswith(\"AttributeError: 'str' object has no attribute 'path'\")\n"
    ]
   },
   {

{lamindb-0.76.11 → lamindb-0.76.13}/docs/track.ipynb

@@ -141,6 +141,8 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
+    "(track-run-parameters)=\n",
+    "\n",
     "## Track run parameters"
    ]
   },
@@ -165,7 +167,7 @@
     "\n",
     "ln.Param(name=\"input_dir\", dtype=\"str\").save()\n",
     "ln.Param(name=\"learning_rate\", dtype=\"float\").save()\n",
-    "ln.Param(name=\"downsample\", dtype=\"bool\").save()"
+    "ln.Param(name=\"preprocess_params\", dtype=\"dict\").save()"
    ]
   },
   {
@@ -223,14 +225,24 @@
    },
    "outputs": [],
    "source": [
-    "ln.Run.params.filter(learning_rate=0.01, input_dir=\"./mydataset\").df()"
+    "ln.Run.params.filter(learning_rate=0.01, input_dir=\"./mydataset\", preprocess_params__downsample=True).df()"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Look at the parameter values that were used for a given run."
+    "Note that:\n",
+    "\n",
+    "* `preprocess_params__downsample=True` traverses the dictionary `preprocess_params` to find the key `\"downsample\"` and match it to `True`\n",
+    "* nested keys like `\"downsample\"` in a dictionary do not appear in `Param` and hence, do not get validated"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Below is how you get the parameter values that were used for a given run."
    ]
   },
   {
@@ -247,6 +259,35 @@
     "run.params.get_values()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Or [on the hub](https://lamin.ai/laminlabs/lamindata/transform/JjRF4mACd9m00001).\n",
+    "\n",
+    "<img width=\"500\" alt=\"image\" src=\"https://github.com/user-attachments/assets/d8a5df37-d585-4940-b6f0-91f99b6c436c\">"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "If you want to query all parameter values across all runs, use {class}`~lamindb.core.ParamValue`."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "tags": [
+     "hide-output"
+    ]
+   },
+   "outputs": [],
+   "source": [
+    "ln.core.ParamValue.df(include=[\"param__name\", \"created_by__handle\"])"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -297,7 +338,7 @@
    },
    "outputs": [],
    "source": [
-    "assert run.params.get_values() == {'downsample': True, 'input_dir': './mydataset', 'learning_rate': 0.01}\n",
+    "assert run.params.get_values() == {'input_dir': './mydataset', 'learning_rate': 0.01, 'preprocess_params': {'downsample': True, 'normalization': 'the_good_one'}}\n",
     "\n",
     "# clean up test instance\n",
     "!rm -r ./test-track\n",

{lamindb-0.76.11 → lamindb-0.76.13}/docs/tutorial.ipynb

@@ -197,7 +197,8 @@
    },
    "outputs": [],
    "source": [
-    "ln.UPath(\"s3://lamindata/iris_studies\").view_tree()"
+    "# we use anon=True here in case no aws credentials are configured\n",
+    "ln.UPath(\"s3://lamindata/iris_studies\", anon=True).view_tree()"
    ]
   },
   {
@@ -574,7 +575,6 @@
     "ln.Artifact.filter(\n",
     "    created_by=users.anonymous,\n",
     "    transform=transform,\n",
-    "    suffix=\".csv\",\n",
     ").df().head()"
    ]
   },
@@ -595,7 +595,7 @@
    },
    "outputs": [],
    "source": [
-    "ln.Artifact.filter(key__startswith=\"iris_studies/study0_raw_images/\").df().head()"
+    "ln.Artifact.filter(key__startswith=\"iris_studies/study0_raw_images\").df().head()"
    ]
   },
   {
@@ -947,7 +947,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.13"
+   "version": "3.9.17"
   },
   "vscode": {
    "interpreter": {

{lamindb-0.76.11 → lamindb-0.76.13}/lamindb/__init__.py

@@ -43,7 +43,7 @@ Modules and settings.
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.76.11"
+__version__ = "0.76.13"
 
 import os as _os
 

{lamindb-0.76.11 → lamindb-0.76.13}/lamindb/_artifact.py

@@ -10,7 +10,7 @@ import fsspec
 import lamindb_setup as ln_setup
 import pandas as pd
 from anndata import AnnData
-from django.db.models import Q, QuerySet
+from django.db.models import Q
 from lamin_utils import colors, logger
 from lamindb_setup import settings as setup_settings
 from lamindb_setup._init_instance import register_storage_in_instance
@@ -180,7 +180,7 @@ def process_data(
                 f" be '{suffix}'."
             )
         cache_name = f"{provisional_uid}{suffix}"
-        path = settings.storage.cache_dir / cache_name
+        path = settings.cache_dir / cache_name
         # Alex: I don't understand the line below
         if path.suffixes == []:
             path = path.with_suffix(suffix)
@@ -331,9 +331,10 @@ def get_artifact_kwargs_from_data(
         artifact = stat_or_artifact
         # update the run of the existing artifact
         if run is not None:
-            # save the information that this artifact was previously
-            # produced by another run
-            if artifact.run is not None:
+            # save the information that this artifact was previously produced by
+            # another run
+            # note: same logic exists for _output_collections_with_later_updates
+            if artifact.run is not None and artifact.run != run:
                 artifact.run._output_artifacts_with_later_updates.add(artifact)
             # update the run of the artifact with the latest run
             stat_or_artifact.run = run
@@ -344,7 +345,7 @@ def get_artifact_kwargs_from_data(
 
     if revises is not None:  # update provisional_uid
         provisional_uid, revises = create_uid(revises=revises, version=version)
-        if settings.storage.cache_dir in path.parents:
+        if settings.cache_dir in path.parents:
             path = path.rename(path.with_name(f"{provisional_uid}{suffix}"))
 
     check_path_in_storage = False
@@ -749,10 +750,6 @@ def from_dir(
     run: Run | None = None,
 ) -> list[Artifact]:
     """{}"""  # noqa: D415
-    logger.warning(
-        "this creates one artifact per file in the directory - consider"
-        " ln.Artifact(dir_path) to get one artifact for the entire directory"
-    )
     folderpath: UPath = create_path(path)  # returns Path for local
     default_storage = settings.storage.record
     using_key = settings._using_key
@@ -1125,7 +1122,7 @@ def save(self, upload: bool | None = None, **kwargs) -> Artifact:
         raise RuntimeError(exception)
     if local_path is not None and not state_was_adding:
         # only move the local artifact to cache if it was not newly created
-        local_path_cache = ln_setup.settings.storage.cache_dir / local_path.name
+        local_path_cache = ln_setup.settings.cache_dir / local_path.name
         # don't use Path.rename here because of cross-device link error
         # https://laminlabs.slack.com/archives/C04A0RMA0SC/p1710259102686969
         shutil.move(

{lamindb-0.76.11 → lamindb-0.76.13}/lamindb/_collection.py

@@ -145,15 +145,16 @@ def __init__(
         logger.warning(
             f"returning existing collection with same hash: {existing_collection}"
         )
-        # update the run of the existing artifact
+        # update the run of the existing collection
         if run is not None:
-            # save the information that this artifact was previously
-            # produced by another run
-            if existing_collection.run is not None:
+            # save the information that this collection was previously produced
+            # by another run
+            # note: same logic exists for _output_artifacts_with_later_updates
+            if existing_collection.run is not None and existing_collection.run != run:
                 existing_collection.run._output_collections_with_later_updates.add(
                     existing_collection
                 )
-            # update the run of the artifact with the latest run
+            # update the run of the collection with the latest run
             existing_collection.run = run
             existing_collection.transform = run.transform
         init_self_from_db(collection, existing_collection)

{lamindb-0.76.11 → lamindb-0.76.13}/lamindb/_feature_set.py

@@ -216,7 +216,9 @@ def members(self) -> QuerySet:
 
 
 def _get_related_name(self: FeatureSet) -> str:
-    feature_sets_related_models = dict_related_model_to_related_name(self)
+    feature_sets_related_models = dict_related_model_to_related_name(
+        self, instance=self._state.db
+    )
     related_name = feature_sets_related_models.get(self.registry)
     return related_name
 

{lamindb-0.76.11 → lamindb-0.76.13}/lamindb/_filter.py

@@ -2,8 +2,6 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
-from lnschema_core import Artifact, Collection
-
 from ._query_set import QuerySet, process_expressions
 
 if TYPE_CHECKING:

{lamindb-0.76.11 → lamindb-0.76.13}/lamindb/_finish.py

@@ -130,12 +130,12 @@ def save_context_core(
                 if response != "y":
                     return "aborted-non-consecutive"
         # write the report
-        report_path = ln_setup.settings.storage.cache_dir / filepath.name.replace(
+        report_path = ln_setup.settings.cache_dir / filepath.name.replace(
             ".ipynb", ".html"
         )
         notebook_to_report(filepath, report_path)
         # write the source code
-        source_code_path = ln_setup.settings.storage.cache_dir / filepath.name.replace(
+        source_code_path = ln_setup.settings.cache_dir / filepath.name.replace(
             ".ipynb", ".py"
         )
         notebook_to_script(transform, filepath, source_code_path)
@@ -171,7 +171,7 @@ def save_context_core(
         transform.hash = hash
 
     # track environment
-    env_path = ln_setup.settings.storage.cache_dir / f"run_env_pip_{run.uid}.txt"
+    env_path = ln_setup.settings.cache_dir / f"run_env_pip_{run.uid}.txt"
     if env_path.exists():
         overwrite_env = True
         if run.environment_id is not None and from_cli:

{lamindb-0.76.11 → lamindb-0.76.13}/lamindb/_parents.py

@@ -310,7 +310,12 @@ def _record_label(record: Record, field: str | None = None):
             rf' FACE="Monospace">uid={record.uid}<BR/>version={record.version}</FONT>>'
         )
     elif isinstance(record, Run):
-        name = f'{record.transform.name.replace("&", "&amp;")}'
+        if record.transform.name:
+            name = f'{record.transform.name.replace("&", "&amp;")}'
+        elif record.transform.key:
+            name = f'{record.transform.key.replace("&", "&amp;")}'
+        else:
+            name = f"{record.transform.uid}"
         user_display = (
             record.created_by.handle
             if record.created_by.name is None
@@ -365,7 +370,6 @@ def _get_all_parent_runs(data: Artifact | Collection) -> list:
                 inputs_run += (
                     r.input_collections.all().filter(visibility__in=[0, 1]).list()
                 )
-            run_inputs_outputs += [(inputs_run, r)]
             outputs_run = (
                 r.__getattribute__(f"output_{name}s")
                 .all()
@@ -376,7 +380,18 @@ def _get_all_parent_runs(data: Artifact | Collection) -> list:
                 outputs_run += (
                     r.output_collections.all().filter(visibility__in=[0, 1]).list()
                 )
-            run_inputs_outputs += [(r, outputs_run)]
+            # if inputs are outputs artifacts are the same, will result infinite loop
+            # so only show as outputs
+            overlap = set(inputs_run).intersection(outputs_run)
+            if overlap:
+                logger.warning(
+                    f"The following artifacts are both inputs and outputs of Run(uid={r.uid}): {overlap}\n   → Only showing as outputs."
+                )
+                inputs_run = list(set(inputs_run) - overlap)
+            if len(inputs_run) > 0:
+                run_inputs_outputs += [(inputs_run, r)]
+            if len(outputs_run) > 0:
+                run_inputs_outputs += [(r, outputs_run)]
             inputs += inputs_run
         runs = [f.run for f in inputs if f.run is not None]
     return run_inputs_outputs

{lamindb-0.76.11 → lamindb-0.76.13}/lamindb/_query_manager.py

@@ -98,26 +98,11 @@ class QueryManager(models.Manager):
 
         return _lookup(cls=self.all(), field=field, **kwargs)
 
-    def __getitem__(self, item: str):
-        try:
-            source_field_name = self.source_field_name
-            target_field_name = self.target_field_name
-
-            if (
-                source_field_name in {"artifact", "collection"}
-                and target_field_name == "feature_set"
-            ):
-                return get_feature_set_by_slot_(host=self.instance).get(item)
-
-        except Exception:  # pragma: no cover
-            return
-
 
 models.Manager.list = QueryManager.list
 models.Manager.df = QueryManager.df
 models.Manager.search = QueryManager.search
 models.Manager.lookup = QueryManager.lookup
-models.Manager.__getitem__ = QueryManager.__getitem__
 models.Manager._track_run_input_manager = QueryManager._track_run_input_manager
 # the two lines below would be easy if we could actually inherit; like this,
 # they're suboptimal

{lamindb-0.76.11 → lamindb-0.76.13}/lamindb/_query_set.py

@@ -115,7 +115,7 @@ def get(
     else:
         assert idlike is None  # noqa: S101
         expressions = process_expressions(registry, expressions)
-        return registry.objects.get(**expressions)
+        return registry.objects.using(qs.db).get(**expressions)
 
 
 class RecordsList(UserList):
@@ -139,7 +139,7 @@ class QuerySet(models.QuerySet):
 
     See Also:
 
-        `django QuerySet <https://docs.djangoproject.com/en/4.2/ref/models/querysets/>`__ # noqa
+        `django QuerySet <https://docs.djangoproject.com/en/4.2/ref/models/querysets/>`__
 
     Examples:
 
@@ -154,7 +154,7 @@ class QuerySet(models.QuerySet):
     ) -> pd.DataFrame:
         """{}"""  # noqa: D415
         # re-order the columns
-        exclude_field_names = ["created_at"]
+        exclude_field_names = ["updated_at"]
         field_names = [
             field.name
             for field in self.model._meta.fields