lamindb 0.76.0__py3-none-any.whl → 0.76.1__py3-none-any.whl

lamindb/core/_mapped_collection.py

@@ -149,7 +149,7 @@ class MappedCollection:
         self.storages = []  # type: ignore
         self.conns = []  # type: ignore
         self.parallel = parallel
-        self._path_list = path_list
+        self.path_list = path_list
         self._make_connections(path_list, parallel)
 
         self.n_obs_list = []
@@ -165,11 +165,12 @@ class MappedCollection:
         self.indices = np.hstack([np.arange(n_obs) for n_obs in self.n_obs_list])
         self.storage_idx = np.repeat(np.arange(len(self.storages)), self.n_obs_list)
 
-        self.join_vars = join
-        self.var_indices = None
-        self.var_joint = None
-        self.n_vars_list = None
-        self.n_vars = None
+        self.join_vars: Literal["inner", "outer"] | None = join
+        self.var_indices: list | None = None
+        self.var_joint: pd.Index | None = None
+        self.n_vars_list: list | None = None
+        self.var_list: list | None = None
+        self.n_vars: int | None = None
         if self.join_vars is not None:
             self._make_join_vars()
             self.n_vars = len(self.var_joint)
@@ -225,43 +226,71 @@ class MappedCollection:
             encoder.update({cat: i for i, cat in enumerate(cats)})
             self.encoders[label] = encoder
 
-    def _make_join_vars(self):
-        var_list = []
+    def _read_vars(self):
+        self.var_list = []
         self.n_vars_list = []
         for storage in self.storages:
             with _Connect(storage) as store:
                 vars = _safer_read_index(store["var"])
-                var_list.append(vars)
+                self.var_list.append(vars)
                 self.n_vars_list.append(len(vars))
 
-        vars_eq = all(var_list[0].equals(vrs) for vrs in var_list[1:])
+    def _make_join_vars(self):
+        if self.var_list is None:
+            self._read_vars()
+        vars_eq = all(self.var_list[0].equals(vrs) for vrs in self.var_list[1:])
         if vars_eq:
             self.join_vars = None
-            self.var_joint = var_list[0]
+            self.var_joint = self.var_list[0]
             return
 
         if self.join_vars == "inner":
-            self.var_joint = reduce(pd.Index.intersection, var_list)
+            self.var_joint = reduce(pd.Index.intersection, self.var_list)
             if len(self.var_joint) == 0:
                 raise ValueError(
                     "The provided AnnData objects don't have shared varibales.\n"
                     "Use join='outer'."
                 )
-            self.var_indices = [vrs.get_indexer(self.var_joint) for vrs in var_list]
+            self.var_indices = [
+                vrs.get_indexer(self.var_joint) for vrs in self.var_list
+            ]
         elif self.join_vars == "outer":
-            self.var_joint = reduce(pd.Index.union, var_list)
-            self.var_indices = [self.var_joint.get_indexer(vrs) for vrs in var_list]
+            self.var_joint = reduce(pd.Index.union, self.var_list)
+            self.var_indices = [
+                self.var_joint.get_indexer(vrs) for vrs in self.var_list
+            ]
+
+    def check_vars_sorted(self, ascending: bool = True) -> bool:
+        """Returns `True` if all variables are sorted in all objects."""
+        if self.var_list is None:
+            self._read_vars()
+        if ascending:
+            vrs_sort_status = (vrs.is_monotonic_increasing for vrs in self.var_list)
+        else:
+            vrs_sort_status = (vrs.is_monotonic_decreasing for vrs in self.var_list)
+        return all(vrs_sort_status)
+
+    def check_vars_non_aligned(self, vars: pd.Index | list) -> list[int]:
+        """Returns indices of objects with non-aligned variables.
+
+        Args:
+            vars: Check alignment against these variables.
+        """
+        if self.var_list is None:
+            self._read_vars()
+        vars = pd.Index(vars)
+        return [i for i, vrs in enumerate(self.var_list) if not vrs.equals(vars)]
 
     def __len__(self):
         return self.n_obs
 
     @property
-    def shape(self):
+    def shape(self) -> tuple[int, int]:
         """Shape of the (virtually aligned) dataset."""
         return (self.n_obs, self.n_vars)
 
     @property
-    def original_shapes(self):
+    def original_shapes(self) -> list[tuple[int, int]]:
         """Shapes of the underlying AnnData objects."""
         if self.n_vars_list is None:
             n_vars_list = [None] * len(self.n_obs_list)
@@ -374,8 +403,27 @@ class MappedCollection:
             label = label.decode("utf-8")
         return label
 
-    def get_label_weights(self, obs_keys: str | list[str]):
-        """Get all weights for the given label keys."""
+    def get_label_weights(
+        self,
+        obs_keys: str | list[str],
+        scaler: float | None = None,
+        return_categories: bool = False,
+    ):
+        """Get all weights for the given label keys.
+
+        This counts the number of labels for each label and returns
+        weights for each obs label accoding to the formula `1 / num of this label in the data`.
+        If `scaler` is provided, then `scaler / (scaler + num of this label in the data)`.
+
+        Args:
+            obs_keys: A key in the ``.obs`` slots or a list of keys. If a list is provided,
+                the labels from the obs keys will be concatenated with ``"__"`` delimeter
+            scaler: Use this number to scale the provided weights.
+            return_categories: If `False`, returns weights for each observation,
+                can be directly passed to a sampler. If `True`, returns a dictionary with
+                unique categories for labels (concatenated if `obs_keys` is a list)
+                and their weights.
+        """
         if isinstance(obs_keys, str):
             obs_keys = [obs_keys]
         labels_list = []
@@ -383,12 +431,20 @@ class MappedCollection:
             labels_to_str = self.get_merged_labels(label_key).astype(str).astype("O")
             labels_list.append(labels_to_str)
         if len(labels_list) > 1:
-            labels = reduce(lambda a, b: a + b, labels_list)
+            labels = ["__".join(labels_obs) for labels_obs in zip(*labels_list)]
         else:
             labels = labels_list[0]
-        labels = self.get_merged_labels(label_key)
-        counter = Counter(labels)  # type: ignore
-        weights = 1.0 / np.array([counter[label] for label in labels])
+        counter = Counter(labels)
+        if return_categories:
+            return {
+                k: 1.0 / v if scaler is None else scaler / (v + scaler)
+                for k, v in counter.items()
+            }
+        counts = np.array([counter[label] for label in labels])
+        if scaler is None:
+            weights = 1.0 / counts
+        else:
+            weights = scaler / (counts + scaler)
         return weights
 
     def get_merged_labels(self, label_key: str):
@@ -426,7 +482,7 @@ class MappedCollection:
                     codes = self._get_codes(store, label_key)
                     codes = decode(codes) if isinstance(codes[0], bytes) else codes
                     cats_merge.update(codes)
-        return cats_merge
+        return sorted(cats_merge)
 
     def _get_categories(self, storage: StorageType, label_key: str):  # type: ignore
         """Get categories."""
@@ -483,7 +539,7 @@ class MappedCollection:
         self._closed = True
 
     @property
-    def closed(self):
+    def closed(self) -> bool:
         """Check if connections to array streaming backend are closed.
 
         Does not matter if `parallel=True`.
@@ -508,4 +564,4 @@ class MappedCollection:
         mapped.parallel = False
         mapped.storages = []
         mapped.conns = []
-        mapped._make_connections(mapped._path_list, parallel=False)
+        mapped._make_connections(mapped.path_list, parallel=False)

lamindb/core/_settings.py

@@ -54,7 +54,7 @@ class Settings:
     track_run_inputs: bool = True
     """Track files as input upon `.load()`, `.cache()` and `.backed()`.
 
-    Requires a global run context with :func:`~lamindb.track` was created!
+    Requires a global run context with :func:`~lamindb.core.Context.track` was created!
 
     FAQ: :doc:`/faq/track-run-inputs`
     """
@@ -83,14 +83,10 @@ class Settings:
     def transform(self) -> TransformSettings:
         """Transform settings.
 
-        For example::
-
-            ln.settings.transform.stem_uid = "FPnfDtJz8qbE"  # defines version family
-            ln.settings.transform.version = "1"              # defines version
-            ln.settings.transform.name = "My good script"    # semantic name
-
-        The first two are typically auto-generated by :func:`~lamindb.track`.
+        Is deprecated since version 0.76.1.
         """
+        # enable warning soon
+        # logger.warning("Transform settings are deprecated, please instead set `ln.context.uid`")
         return transform_settings
 
     @property

lamindb/core/datasets/_core.py

@@ -432,6 +432,7 @@ def mudata_papalexi21_subset() -> MuData:  # pragma: no cover
     mdata.push_obs(["percent.mito"], mods=["rna"], drop=True)
     mdata["hto"].obs["technique"] = "cell hashing"
     mdata["hto"].obs["technique"] = mdata["hto"].obs["technique"].astype("category")
+    mdata.pull_obs(["technique"], mods="hto")
 
     return mdata
 

lamindb/core/exceptions.py

@@ -5,24 +5,41 @@ The registry base class:
 .. autosummary::
    :toctree: .
 
+   DoesNotExist
    ValidationError
    NotebookNotSavedError
    NoTitleError
-   MissingTransformSettings
-   UpdateTransformSettings
+   MissingContext
+   UpdateContext
    IntegrityError
 
 """
 
 
+class TrackNotCalled(SystemExit):
+    pass
+
+
+class NotebookNotSaved(SystemExit):
+    pass
+
+
 class ValidationError(SystemExit):
     """Validation error: not mapped in registry."""
 
     pass
 
 
+# inspired by Django's DoesNotExist
+# equivalent to SQLAlchemy's NoResultFound
+class DoesNotExist(Exception):
+    """No record found."""
+
+    pass
+
+
 # -------------------------------------------------------------------------------------
-# ln.track() AKA run_context
+# ln.context.track() AKA context
 # -------------------------------------------------------------------------------------
 
 
@@ -48,13 +65,13 @@ class NoTitleError(Exception):
     pass
 
 
-class MissingTransformSettings(SystemExit):
+class MissingContext(SystemExit):
     """User didn't define transform settings."""
 
     pass
 
 
-class UpdateTransformSettings(SystemExit):
+class UpdateContext(SystemExit):
     """Transform settings require update."""
 
     pass

lamindb/core/storage/_valid_suffixes.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from lamindb_setup.core.upath import VALID_COMPOSITE_SUFFIXES, VALID_SIMPLE_SUFFIXES
 
 # add new composite suffixes like so

lamindb/core/versioning.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from typing import TYPE_CHECKING, Literal
 
 from lamin_utils import logger
+from lamin_utils._base62 import CHARSET_DEFAULT as BASE62_CHARS
 from lamindb_setup.core.upath import LocalPathClasses, UPath
 from lnschema_core import ids
 
@@ -10,6 +11,18 @@ if TYPE_CHECKING:
     from lnschema_core.models import IsVersioned
 
 
+def increment_base62(s: str) -> str:
+    # we don't need to throw an error for zzzz because uids are enforced to be unique
+    # on the db level and have an enforced maximum length
+    value = sum(BASE62_CHARS.index(c) * (62**i) for i, c in enumerate(reversed(s)))
+    value += 1
+    result = ""
+    while value:
+        value, remainder = divmod(value, 62)
+        result = BASE62_CHARS[remainder] + result
+    return result.zfill(len(s))
+
+
 def bump_version(
     version: str,
     bump_type: str = "minor",

{lamindb-0.76.0.dist-info → lamindb-0.76.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.76.0
+Version: 0.76.1
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,10 +9,10 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.73.0
-Requires-Dist: lamindb_setup==0.76.4
+Requires-Dist: lnschema_core==0.73.1
+Requires-Dist: lamindb_setup==0.76.5
 Requires-Dist: lamin_utils==0.13.2
-Requires-Dist: lamin_cli==0.16.1
+Requires-Dist: lamin_cli==0.16.2
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
@@ -24,7 +24,7 @@ Requires-Dist: pandas
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
 Requires-Dist: lamindb_setup[aws] ; extra == "aws"
-Requires-Dist: bionty==0.48.1 ; extra == "bionty"
+Requires-Dist: bionty==0.48.2 ; extra == "bionty"
 Requires-Dist: pre-commit ; extra == "dev"
 Requires-Dist: nox ; extra == "dev"
 Requires-Dist: laminci>=0.3 ; extra == "dev"

{lamindb-0.76.0.dist-info → lamindb-0.76.1.dist-info}/RECORD

@@ -1,18 +1,18 @@
-lamindb/__init__.py,sha256=hMTLgL64GkHYHyLFHSCjR64xnvlZclrHKLlUWGuJ3i4,2249
-lamindb/_artifact.py,sha256=z3hJL0A0NQFDf0hvWYt-hZG1nX_tXog9QKtXwCIZxqU,42627
+lamindb/__init__.py,sha256=d7viqFNu34u9fGs2xJq5AAD-4-M7VS2reIRP-uEYn54,2270
+lamindb/_artifact.py,sha256=wmcofYb1-23YblpjwFo8X-dLNgB7EfwMtzS7p-CKvhs,42630
 lamindb/_can_validate.py,sha256=jPZHhtPaZJos5bL8Mk8ZNh34mItwYRwOynhofKUxwfY,17472
 lamindb/_collection.py,sha256=PIKeRZ9zttKSyzcz8UMjKNihPO1Qe-E2zfr-Ao5wtJo,14943
-lamindb/_curate.py,sha256=M42XxtcSKSo2Vh3xCio1UogVS0kCG1Dtx5cwEwnWOEM,55508
+lamindb/_curate.py,sha256=Ww_1UlHAIj_fmdxRAuhphqSEi-_qQvjJCIlNbhe8yhA,55486
 lamindb/_feature.py,sha256=27PBSJoT0yI400WwdWPFz8cmUI4KAQ58jVep-wIDsfM,7373
 lamindb/_feature_set.py,sha256=DmAy96V_RyV0yiyvWOCHgustXPsCaMwn4TrWwh2qDd8,8104
 lamindb/_filter.py,sha256=I1tSIF-izmD48YpB8NtKmau59Vpt6C54RPHWc3oinGM,1412
-lamindb/_finish.py,sha256=3rGY4RxhcKR1BIdgf6GPB03AArhlV5vYtD6wxtTHYxE,10817
+lamindb/_finish.py,sha256=VtUfnHQOxbxKUMm3IXmxGO4T0XH64w-yeyeIdwvApBw,9433
 lamindb/_from_values.py,sha256=zBOJGt_3mBKjqW5LuzuUysMwGbEcEFtYBJXAcL4Vx_0,13432
 lamindb/_is_versioned.py,sha256=Z_zVsHTnNaWvD15dEz18VocsSgka53jUzs_wDr78ltI,1344
 lamindb/_parents.py,sha256=eMavdd6IO6STOVJSlR2TzdRtx6sKYDKsMOtlR3DZlgQ,15599
-lamindb/_query_manager.py,sha256=6_ZqQr0SKTYsbMtJd5BmaMoyASIVel5v2F1KMgacWlU,4252
-lamindb/_query_set.py,sha256=f2OAE9j6l6MGAXgKp5wr8bXB1KRzkye9UCp4utyNHeI,10581
-lamindb/_record.py,sha256=PwelJVh5A0RMYa6o05wXvZU_gZyV8tPSILUW-M6m7lc,21195
+lamindb/_query_manager.py,sha256=uLg3YneNcnRP8-fV0f-_t255PMGnDO47ueQqGiDZwis,4240
+lamindb/_query_set.py,sha256=T1VmEC-rQTmOEwILKJVu5DRBLwp4DA9XnId2ndURcOo,10587
+lamindb/_record.py,sha256=mhIg7QxvWKdWW0QDAUmKwMQefDeg9k3mlw93rsVsf0Q,21422
 lamindb/_run.py,sha256=5M_r1zGDv9HlqbqRKTWCYCOtENovJ-8mQ4kY7XqcLaU,1888
 lamindb/_save.py,sha256=9lpFpQLgmPOp6JcoM9XetNkbYu2JgY70sGQZVz3Gzmw,11027
 lamindb/_storage.py,sha256=GBVChv-DHVMNEBJL5l_JT6B4RDhZ6NnwgzmUICphYKk,413
@@ -20,28 +20,28 @@ lamindb/_transform.py,sha256=Vw0Ct15nCOmL8-UwDs-hH-rgdX8PfcAadgbh_ynBzQE,3397
 lamindb/_ulabel.py,sha256=XDSdZBXX_ki5s1vOths3MjF2x5DPggBR_PV_KF4SGyg,1611
 lamindb/_utils.py,sha256=LGdiW4k3GClLz65vKAVRkL6Tw-Gkx9DWAdez1jyA5bE,428
 lamindb/_view.py,sha256=4Ln2ItTb3857PAI-70O8eJYqoTJ_NNFc7E_wds6OGns,2412
-lamindb/core/__init__.py,sha256=uCtTkcUwJ7w40cWdaVBymiyxMuKlkdX-QZMtOr3k0us,1441
-lamindb/core/_data.py,sha256=AbBVvutI7iFZW5PVEYzx0HIlaLhR9KpnoPyba9OI86k,16253
+lamindb/core/__init__.py,sha256=QePKN3dkGuNEfIc2CKwBt3Kcl39bbghX6XwXKpK3BA0,1491
+lamindb/core/_context.py,sha256=cNj_LDpA9gmkE6P6Gg1cBwp4eFukvMlggivYdrVqM8E,19407
+lamindb/core/_data.py,sha256=eocOXsZGu62LPtz6yIlvHhPSJTf3yF2ITZTffyflWYI,16269
 lamindb/core/_feature_manager.py,sha256=gJowe9-tifi0Z5IcChV-jtGYVdl6XgR2m_oswwhU17Y,31980
 lamindb/core/_label_manager.py,sha256=OO5Tl-ankQZNrF_PQ7_xfkuJ8ytMUvr-9cL9soe1Ceo,9249
-lamindb/core/_mapped_collection.py,sha256=wN2qEXnKrx0kqblVml_Kx9hYomUAZLYKckGXX4-9dr0,19621
-lamindb/core/_run_context.py,sha256=bk4hwbGbx1no-JKVpLioIOU5dyO8YCdg-srbQV7UuGE,18371
-lamindb/core/_settings.py,sha256=sDfIfq9H7H8nUE51FJF4EO_Zihlxh44S3GbaliHKJY4,6108
+lamindb/core/_mapped_collection.py,sha256=ST-cTfokIGkRadjSHEyvIK2san8cGr7WZpgbgs5neLI,22025
+lamindb/core/_settings.py,sha256=GGEB8BU5GinIfD4ktr1Smp6GPHGaInu46MhP4EecZDY,5950
 lamindb/core/_sync_git.py,sha256=qc0yfPyKeG4uuNT_3qsv-mkIMqhLFqfXNeNVO49vV00,4547
 lamindb/core/_track_environment.py,sha256=STzEVUzOeUEWdX7WDJUkKH4u08k7eupRX6AXQwoVt14,828
-lamindb/core/exceptions.py,sha256=rNLDlqDJt6stM_HGeE9qR0oWRNn4Kq0WDdIi85P53VA,1057
+lamindb/core/exceptions.py,sha256=qNFYN5Jc7Y6kw4re-jsW0AEIprsV2HB1wTcJiO-u-ks,1278
 lamindb/core/fields.py,sha256=47Jmh3efUr5ZscgimR_yckY-I3cNf8ScLutbwKCK3j4,162
 lamindb/core/schema.py,sha256=KiYQn_8fokSMztTNDe6qUocZzKXWxU32H-YChNJv51A,1877
 lamindb/core/types.py,sha256=uVBqSVLoQaTkqP9nqsJhwU6yYnx8H5e6-ZxrB6vpOOw,265
-lamindb/core/versioning.py,sha256=bDpD_UejR7Jr-DTsIvoVV3SyHNxIdjziJvCyUSskxNc,4898
+lamindb/core/versioning.py,sha256=fVzxFcJ8xPPSRuLZuUz_lfAKirI550NQTyKIowQUhhs,5404
 lamindb/core/datasets/__init__.py,sha256=zRP98oqUAaXhqWyKMiH0s_ImVIuNeziQQ2kQ_t0f-DI,1353
-lamindb/core/datasets/_core.py,sha256=DnAKMw1MzCrEcqYHoRSaqZYRnO1wwpykPS3YThMN9Es,19827
+lamindb/core/datasets/_core.py,sha256=CgVF_pXuBXLElzubDMsl1DbpYOnXCY0HleITVvBKih4,19873
 lamindb/core/datasets/_fake.py,sha256=BZF9R_1iF0HDnvtZNqL2FtsjSMuqDIfuFxnw_LJYIh4,953
 lamindb/core/storage/__init__.py,sha256=MjKg2-p8EbOYTVsgufnK93ut7HG7_MzLDAqtzXt0U2Q,501
 lamindb/core/storage/_anndata_accessor.py,sha256=jmEZeeZlt8-qBXRkU0tTA-t6dVEb_dH86wc1ok0jSRY,24030
 lamindb/core/storage/_anndata_sizes.py,sha256=aXO3OB--tF5MChenSsigW6Q-RuE8YJJOUTVukkLrv9A,1029
 lamindb/core/storage/_backed_access.py,sha256=MsSgiIccHVhqOcur2lZ4mj5LSIL5OL8nX4eqK6mloU0,4732
-lamindb/core/storage/_valid_suffixes.py,sha256=9erE4FX1ArRoTkvn5pzUmlKQSYnKeQBymOnOPCv2TD4,465
+lamindb/core/storage/_valid_suffixes.py,sha256=vUSeQ4s01rdhD_vSd6wKmFBsgMJAKkBMnL_T9Y1znMg,501
 lamindb/core/storage/_zarr.py,sha256=5ceEz6YIvgvUnVVNWhK5Z4W0WfrvyvY82Yna5jSX1_E,3661
 lamindb/core/storage/objects.py,sha256=OzvBCS-Urz5mr-O95qYt6RGBDDX5HmjfRRKWPPDn1ZE,1797
 lamindb/core/storage/paths.py,sha256=I0UjQfXfh3MptfLgpA0iSyR-X9pLOvgtXNr4B_Lwk4g,7810
@@ -52,7 +52,7 @@ lamindb/integrations/__init__.py,sha256=MoLRD_qqX5WHFUAqHL6zoY_cDkWH0zimaGT_1CyX
 lamindb/integrations/_vitessce.py,sha256=jcpLUNFq1BsDpZDkG5L3nzWNXDLuy3Eep_GfY0XqhhA,5077
 lamindb/setup/__init__.py,sha256=OwZpZzPDv5lPPGXZP7-zK6UdO4FHvvuBh439yZvIp3A,410
 lamindb/setup/core/__init__.py,sha256=SevlVrc2AZWL3uALbE5sopxBnIZPWZ1IB0NBDudiAL8,167
-lamindb-0.76.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-lamindb-0.76.0.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-lamindb-0.76.0.dist-info/METADATA,sha256=5NduEIMIlIq6hW-MADbfFtJKsuxY5JZGQwlBO5WZxUQ,2669
-lamindb-0.76.0.dist-info/RECORD,,
+lamindb-0.76.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+lamindb-0.76.1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+lamindb-0.76.1.dist-info/METADATA,sha256=_h45JtUTV8vsLXe7ePddBMIhWo4CkFD4zKvxjNq99fM,2669
+lamindb-0.76.1.dist-info/RECORD,,