lamindb 0.76.3__tar.gz → 0.76.5__tar.gz

{lamindb-0.76.3 → lamindb-0.76.5}/.github/ISSUE_TEMPLATE/config.yml

@@ -1,4 +1,4 @@
-blank_issues_enabled: false
+blank_issues_enabled: true
 contact_links:
     - name: LaminHub issues
       url: https://github.com/laminlabs/laminhub-public

{lamindb-0.76.3 → lamindb-0.76.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.76.3
+Version: 0.76.5
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,22 +9,22 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.73.3
-Requires-Dist: lamindb_setup==0.76.7
+Requires-Dist: lnschema_core==0.73.5
+Requires-Dist: lamindb_setup==0.77.1
 Requires-Dist: lamin_utils==0.13.4
-Requires-Dist: lamin_cli==0.16.2
+Requires-Dist: lamin_cli==0.17.1
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
 Requires-Dist: python-dateutil
-Requires-Dist: anndata>=0.8.0,<=0.10.8
+Requires-Dist: anndata>=0.8.0,<=0.10.9
 Requires-Dist: scipy<1.13.0rc1
 Requires-Dist: fsspec
 Requires-Dist: pandas
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
 Requires-Dist: lamindb_setup[aws] ; extra == "aws"
-Requires-Dist: bionty==0.49.0 ; extra == "bionty"
+Requires-Dist: bionty==0.49.1 ; extra == "bionty"
 Requires-Dist: pre-commit ; extra == "dev"
 Requires-Dist: nox ; extra == "dev"
 Requires-Dist: laminci>=0.3 ; extra == "dev"
@@ -38,7 +38,7 @@ Requires-Dist: django-schema-graph ; extra == "erdiagram"
 Requires-Dist: readfcs>=1.1.8 ; extra == "fcs"
 Requires-Dist: lamindb_setup[gcp] ; extra == "gcp"
 Requires-Dist: nbproject==0.10.4 ; extra == "jupyter"
-Requires-Dist: nbstripout==0.6.1 ; extra == "jupyter"
+Requires-Dist: jupytext ; extra == "jupyter"
 Requires-Dist: nbconvert ; extra == "jupyter"
 Requires-Dist: zarr>=2.16.0 ; extra == "zarr"
 Project-URL: Home, https://github.com/laminlabs/lamindb

{lamindb-0.76.3 → lamindb-0.76.5}/docs/api.md

@@ -12,33 +12,19 @@ cli
 
 ```{toctree}
 :maxdepth: 1
-:caption: Core API
+:caption: Python
 :hidden:
 
 lamindb
-```
-
-```{toctree}
-:maxdepth: 1
-:caption: Plugins
-:hidden:
-
 bionty
 wetlab
 ```
 
 ```{toctree}
 :maxdepth: 1
-:caption: SDK
+:caption: R & REST
 :hidden:
 
+laminr
 rest
 ```
-
-```{toctree}
-:maxdepth: 1
-:caption: Utilities
-:hidden:
-
-readfcs
-```

lamindb-0.76.5/docs/laminr.md

@@ -0,0 +1,3 @@
+# `laminr`
+
+[github.com/laminlabs/laminr](https://github.com/laminlabs/laminr)

{lamindb-0.76.3 → lamindb-0.76.5}/docs/tutorial.ipynb

@@ -364,7 +364,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Visualize data lineage with {meth}`~lamindb.core.HasFeatures.view_lineage`:"
+    "Visualize data lineage with {meth}`~lamindb.Artifact.view_lineage`:"
    ]
   },
   {

{lamindb-0.76.3 → lamindb-0.76.5}/lamindb/__init__.py

@@ -1,7 +1,6 @@
 """A data framework for biology.
 
-Records
-=======
+Core registries.
 
 .. autosummary::
    :toctree: .
@@ -17,8 +16,7 @@ Records
    FeatureSet
    Param
 
-Key functionality
-=================
+Key functionality.
 
 .. autosummary::
    :toctree: .
@@ -29,8 +27,7 @@ Key functionality
    view
    save
 
-Modules & settings
-==================
+Modules and settings.
 
 .. autosummary::
    :toctree: .
@@ -44,7 +41,7 @@ Modules & settings
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.76.3"
+__version__ = "0.76.5"
 
 import os as _os
 
@@ -111,6 +108,6 @@ if _check_instance_setup(from_lamindb=True):
     track = context.track  # backward compat
     finish = context.finish  # backward compat
     Curate = Curator  # backward compat
-    settings.__doc__ = """Global :class:`~lamindb.core.Settings`."""
-    context.__doc__ = """Global :class:`~lamindb.core.Context`."""
+    settings.__doc__ = """Global settings (:class:`~lamindb.core.Settings`)."""
+    context.__doc__ = """Global run context (:class:`~lamindb.core.Context`)."""
     from django.db.models import Q

{lamindb-0.76.3 → lamindb-0.76.5}/lamindb/_artifact.py

@@ -366,11 +366,6 @@ def get_artifact_kwargs_from_data(
     else:
         storage = default_storage
 
-    # for now comment out this error to allow creating new versions of stores
-    # in the default folder (.lamindb)
-    #    if key is not None and key.startswith(AUTO_KEY_PREFIX):
-    #        raise ValueError(f"Key cannot start with {AUTO_KEY_PREFIX}")
-
     log_storage_hint(
         check_path_in_storage=check_path_in_storage,
         storage=storage,
@@ -542,10 +537,10 @@ def __init__(artifact: Artifact, *args, **kwargs):
         else VisibilityChoice.default.value
     )
     format = kwargs.pop("format") if "format" in kwargs else None
+    _is_internal_call = kwargs.pop("_is_internal_call", False)
     skip_check_exists = (
         kwargs.pop("skip_check_exists") if "skip_check_exists" in kwargs else False
     )
-    _uid = kwargs.pop("_uid", None)
     if "default_storage" in kwargs:
         default_storage = kwargs.pop("default_storage")
     else:
@@ -561,9 +556,6 @@ def __init__(artifact: Artifact, *args, **kwargs):
     if "is_new_version_of" in kwargs:
         logger.warning("`is_new_version_of` will be removed soon, please use `revises`")
         revises = kwargs.pop("is_new_version_of")
-    assert not (  # noqa: S101
-        revises is not None and _uid is not None
-    ), "Can not init with both `revises` and `_uid`"
     if not len(kwargs) == 0:
         raise ValueError(
             "Only data, key, run, description, version, revises, visibility"
@@ -579,15 +571,29 @@ def __init__(artifact: Artifact, *args, **kwargs):
         raise ValueError(
             f"`key` is {key}, but `revises.key` is '{revises.key}'\n\n Either do *not* pass `key`.\n\n{note}"
         )
-    if _uid is not None:
-        provisional_uid, revises = _uid, None
-    else:
-        provisional_uid, revises = create_uid(revises=revises, version=version)
     if revises is not None:
         if not isinstance(revises, Artifact):
             raise TypeError("`revises` has to be of type `Artifact`")
         if description is None:
             description = revises.description
+    if key is not None and AUTO_KEY_PREFIX in key:
+        raise ValueError(
+            f"Do not pass key that contains a managed storage path in `{AUTO_KEY_PREFIX}`"
+        )
+    # below is for internal calls that require defining the storage location
+    # ahead of constructing the Artifact
+    if isinstance(data, (str, Path)) and AUTO_KEY_PREFIX in str(data):
+        if _is_internal_call:
+            is_automanaged_path = True
+            user_provided_key = key
+            key = None
+        else:
+            raise ValueError(
+                f"Do not pass path inside the `{AUTO_KEY_PREFIX}` directory."
+            )
+    else:
+        is_automanaged_path = False
+    provisional_uid, revises = create_uid(revises=revises, version=version)
     kwargs_or_artifact, privates = get_artifact_kwargs_from_data(
         data=data,
         key=key,
@@ -615,16 +621,29 @@ def __init__(artifact: Artifact, *args, **kwargs):
     else:
         kwargs = kwargs_or_artifact
 
+    if data is not None:
+        artifact._local_filepath = privates["local_filepath"]
+        artifact._cloud_filepath = privates["cloud_filepath"]
+        artifact._memory_rep = privates["memory_rep"]
+        artifact._to_store = not privates["check_path_in_storage"]
+
+    if is_automanaged_path and _is_internal_call:
+        kwargs["_key_is_virtual"] = True
+        assert AUTO_KEY_PREFIX in kwargs["key"]  # noqa: S101
+        uid = kwargs["key"].replace(AUTO_KEY_PREFIX, "").replace(kwargs["suffix"], "")
+        kwargs["key"] = user_provided_key
+        if revises is not None:
+            assert uid.startswith(revises.stem_uid)  # noqa: S101
+        if len(uid) == 16:
+            if revises is None:
+                uid += "0000"
+            else:
+                uid, revises = create_uid(revises=revises, version=version)
+        kwargs["uid"] = uid
+
     # only set key now so that we don't do a look-up on it in case revises is passed
     if revises is not None:
         kwargs["key"] = revises.key
-    # in case we have a new version of a folder with a different hash, print a
-    # warning that the old version can't be recovered
-    if revises is not None and revises.n_objects is not None and revises.n_objects > 1:
-        logger.warning(
-            f"artifact version {version} will _update_ the state of folder {revises.path} - "
-            "to _retain_ the old state by duplicating the entire folder, do _not_ pass `revises`"
-        )
 
     kwargs["type"] = type
     kwargs["version"] = version
@@ -643,12 +662,6 @@ def __init__(artifact: Artifact, *args, **kwargs):
 
     add_transform_to_kwargs(kwargs, kwargs["run"])
 
-    if data is not None:
-        artifact._local_filepath = privates["local_filepath"]
-        artifact._cloud_filepath = privates["cloud_filepath"]
-        artifact._memory_rep = privates["memory_rep"]
-        artifact._to_store = not privates["check_path_in_storage"]
-
     super(Artifact, artifact).__init__(**kwargs)
 
 
@@ -943,10 +956,9 @@ def open(
                 if self.hash != hash:
                     from ._record import init_self_from_db
 
-                    logger.warning(
-                        "The hash of the tiledbsoma store has changed, creating a new version of the artifact."
-                    )
-                    new_version = Artifact(filepath, revises=self).save()
+                    new_version = Artifact(
+                        filepath, revises=self, _is_internal_call=True
+                    ).save()
                     init_self_from_db(self, new_version)
 
                     if localpath != filepath and localpath.exists():
@@ -1174,3 +1186,4 @@ Artifact._delete_skip_storage = _delete_skip_storage
 Artifact._save_skip_storage = _save_skip_storage
 Artifact.path = path
 Artifact.backed = backed
+Artifact.view_lineage = HasFeatures.view_lineage

{lamindb-0.76.3 → lamindb-0.76.5}/lamindb/_can_validate.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Iterable, Literal
+from typing import TYPE_CHECKING, Literal
 
 import lamindb_setup as ln_setup
 import numpy as np
@@ -79,6 +79,19 @@ def _check_organism_db(organism: Record, using_key: str | None):
                 )
 
 
+def _concat_lists(values: ListLike) -> list[str]:
+    """Concatenate a list of lists of strings into a single list."""
+    if len(values) > 0 and isinstance(values, (list, pd.Series)):
+        try:
+            if isinstance(values[0], list):
+                if isinstance(values, pd.Series):
+                    values = values.tolist()
+                values = sum([v for v in values if isinstance(v, list)], [])
+        except KeyError:
+            pass
+    return values
+
+
 def _inspect(
     cls,
     values: ListLike,
@@ -94,6 +107,7 @@ def _inspect(
 
     if isinstance(values, str):
         values = [values]
+    values = _concat_lists(values)
 
     field = get_name_field(cls, field=field)
     queryset = _queryset(cls, using_key)
@@ -184,6 +198,7 @@ def _validate(
     return_str = True if isinstance(values, str) else False
     if isinstance(values, str):
         values = [values]
+    values = _concat_lists(values)
 
     field = get_name_field(cls, field=field)
 
@@ -229,7 +244,7 @@ def _validate(
 @doc_args(CanValidate.standardize.__doc__)
 def standardize(
     cls,
-    values: Iterable,
+    values: ListLike,
     field: str | StrField | None = None,
     *,
     return_field: str = None,
@@ -295,7 +310,7 @@ def remove_synonym(self, synonym: str | ListLike):
 
 def _standardize(
     cls,
-    values: Iterable,
+    values: ListLike,
     field: str | StrField | None = None,
     *,
     return_field: str = None,
@@ -315,6 +330,7 @@ def _standardize(
     return_str = True if isinstance(values, str) else False
     if isinstance(values, str):
         values = [values]
+    values = _concat_lists(values)
 
     field = get_name_field(cls, field=field)
     return_field = get_name_field(
@@ -416,7 +432,7 @@ def _standardize(
 
 
 def _add_or_remove_synonyms(
-    synonym: str | Iterable,
+    synonym: str | ListLike,
     record: Record,
     action: Literal["add", "remove"],
     force: bool = False,

{lamindb-0.76.3 → lamindb-0.76.5}/lamindb/_curate.py

@@ -144,6 +144,7 @@ class DataFrameCurator(BaseCurator):
         organism: str | None = None,
         sources: dict[str, Record] | None = None,
         exclude: dict | None = None,
+        check_valid_keys: bool = True,
     ) -> None:
         from lamindb.core._settings import settings
 
@@ -163,6 +164,8 @@ class DataFrameCurator(BaseCurator):
             exclude = {}
         self._exclude = exclude
         self._non_validated = None
+        if check_valid_keys:
+            self._check_valid_keys()
         self._save_columns()
 
     @property
@@ -191,14 +194,25 @@ class DataFrameCurator(BaseCurator):
             using_key=using_key or self._using_key,
         )
 
+    def _check_valid_keys(self, extra: set = None) -> None:
+        if extra is None:
+            extra = set()
+        for name, d in {
+            "categoricals": self._fields,
+            "sources": self._sources,
+            "exclude": self._exclude,
+        }.items():
+            if not isinstance(d, dict):
+                raise TypeError(f"{name} must be a dictionary!")
+            valid_keys = set(self._df.columns) | {"columns"} | extra
+            nonval_keys = [key for key in d.keys() if key not in valid_keys]
+            if len(nonval_keys) > 0:
+                raise ValueError(
+                    f"the following keys passed to {name} are not allowed: {nonval_keys}"
+                )
+
     def _save_columns(self, validated_only: bool = True, **kwargs) -> None:
         """Save column name records."""
-        missing_columns = set(self.fields.keys()) - set(self._df.columns)
-        if missing_columns:
-            raise ValueError(
-                f"Columns {missing_columns} are not found in the data object!"
-            )
-
         # Always save features specified as the fields keys
         update_registry(
             values=list(self.fields.keys()),
@@ -426,8 +440,10 @@ class AnnDataCurator(DataFrameCurator):
             organism=organism,
             sources=sources,
             exclude=exclude,
+            check_valid_keys=False,
         )
         self._obs_fields = categoricals
+        self._check_valid_keys(extra={"var_index"})
 
     @property
     def var_index(self) -> FieldAttr:
@@ -633,6 +649,7 @@ class MuDataCurator:
                 verbosity=verbosity,
                 sources=self._sources.get(modality),
                 exclude=self._exclude.get(modality),
+                check_valid_keys=False,
                 **self._kwargs,
             )
             for modality in self._modalities