lamindb 0.65.1__py3-none-any.whl → 0.66.0__py3-none-any.whl

lamindb/__init__.py

@@ -54,7 +54,7 @@ Modules & settings:
 
 """
 
-__version__ = "0.65.1"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.66.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os as _os
 

lamindb/_collection.py

@@ -5,7 +5,7 @@ import anndata as ad
 import pandas as pd
 from lamin_utils import logger
 from lamindb_setup.dev._docs import doc_args
-from lnschema_core.models import Collection, Feature, FeatureSet
+from lnschema_core.models import Collection, CollectionArtifact, Feature, FeatureSet
 from lnschema_core.types import AnnDataLike, DataLike, FieldAttr, VisibilityChoice
 
 from lamindb._utils import attach_func_to_class_method
@@ -15,6 +15,7 @@ from lamindb.dev.versioning import get_uid_from_old_version, init_uid
 
 from . import _TESTING, Artifact, Run
 from ._artifact import parse_feature_sets_from_anndata
+from ._query_set import QuerySet
 from ._registry import init_self_from_db
 from .dev._data import (
     add_transform_to_kwargs,
@@ -312,7 +313,7 @@ def from_artifacts(artifacts: Iterable[Artifact]) -> Tuple[str, Dict[str, str]]:
 def mapped(
     self,
     label_keys: Optional[Union[str, List[str]]] = None,
-    join_vars: Optional[Literal["auto", "inner"]] = "auto",
+    join: Optional[Literal["inner", "outer"]] = "inner",
     encode_labels: bool = True,
     cache_categories: bool = True,
     parallel: bool = False,
@@ -333,7 +334,7 @@ def mapped(
     return MappedCollection(
         path_list,
         label_keys,
-        join_vars,
+        join,
         encode_labels,
         cache_categories,
         parallel,
@@ -426,7 +427,14 @@ def save(self, *args, **kwargs) -> None:
     super(Collection, self).save()
     if hasattr(self, "_artifacts"):
         if self._artifacts is not None and len(self._artifacts) > 0:
-            self.artifacts.set(self._artifacts)
+            links = [
+                CollectionArtifact(collection_id=self.id, artifact_id=artifact.id)
+                for artifact in self._artifacts
+            ]
+            # the below seems to preserve the order of the list in the
+            # auto-incrementing integer primary
+            # merely using .unordered_artifacts.set(*...) doesn't achieve this
+            CollectionArtifact.objects.bulk_create(links)
     save_feature_set_links(self)
 
 
@@ -439,6 +447,14 @@ def restore(self) -> None:
         self.artifact.save()
 
 
+@property  # type: ignore
+@doc_args(Collection.artifacts.__doc__)
+def artifacts(self) -> QuerySet:
+    """{}."""
+    _track_run_input(self)
+    return self.unordered_artifacts.order_by("collectionartifact__id")
+
+
 METHOD_NAMES = [
     "__init__",
     "from_anndata",
@@ -465,3 +481,4 @@ for name in METHOD_NAMES:
 
 # this seems a Django-generated function
 delattr(Collection, "get_visibility_display")
+Collection.artifacts = artifacts

lamindb/_feature.py

@@ -9,6 +9,7 @@ from lamindb._utils import attach_func_to_class_method
 from lamindb.dev._settings import settings
 
 from . import _TESTING
+from ._query_set import RecordsList
 
 FEATURE_TYPES = {
     "int": "number",
@@ -86,7 +87,7 @@ def categoricals_from_df(df: "pd.DataFrame") -> Dict:
 
 @classmethod  # type:ignore
 @doc_args(Feature.from_df.__doc__)
-def from_df(cls, df: "pd.DataFrame") -> List["Feature"]:
+def from_df(cls, df: "pd.DataFrame") -> "RecordsList":
     """{}."""
     categoricals = categoricals_from_df(df)
 
@@ -141,7 +142,7 @@ def from_df(cls, df: "pd.DataFrame") -> List["Feature"]:
     #         f" {colors.yellow('unmapped categories')}:\n     "
     #         f" {categoricals_with_unmapped_categories_formatted}"
     #     )
-    return features
+    return RecordsList(features)
 
 
 @doc_args(Feature.save.__doc__)

lamindb/_parents.py

@@ -275,10 +275,15 @@ def _record_label(record: Registry, field: Optional[str] = None):
         )
     elif isinstance(record, Run):
         name = f'{record.transform.name.replace("&", "&")}'
+        user_display = (
+            record.created_by.handle
+            if record.created_by.name is None
+            else record.created_by.name
+        )
         return (
             rf'<{TRANSFORM_EMOJIS.get(str(record.transform.type), "💫")} {name}<BR/><FONT COLOR="GREY" POINT-SIZE="10"'
             rf' FACE="Monospace">uid={record.transform.uid}<BR/>type={record.transform.type},'
-            rf" user={record.created_by.name}<BR/>run_at={format_field_value(record.run_at)}</FONT>>"
+            rf" user={user_display}<BR/>run={format_field_value(record.run_at)}</FONT>>"
         )
     elif isinstance(record, Transform):
         name = f'{record.name.replace("&", "&amp;")}'

lamindb/_query_set.py

@@ -1,4 +1,5 @@
-from typing import Iterable, List, NamedTuple, Optional, Union
+from collections import UserList
+from typing import Dict, Iterable, List, NamedTuple, Optional, Union
 
 import pandas as pd
 from django.db import models
@@ -21,6 +22,40 @@ class MultipleResultsFound(Exception):
 #     return (series + timedelta).dt.strftime("%Y-%m-%d %H:%M:%S %Z")
 
 
+def get_keys_from_df(data: List, registry: Registry) -> List[str]:
+    if len(data) > 0:
+        if isinstance(data[0], dict):
+            keys = list(data[0].keys())
+        else:
+            keys = list(data[0].__dict__.keys())
+            if "_state" in keys:
+                keys.remove("_state")
+    else:
+        keys = [
+            field.name
+            for field in registry._meta.fields
+            if not isinstance(field, models.ForeignKey)
+        ]
+        keys += [
+            f"{field.name}_id"
+            for field in registry._meta.fields
+            if isinstance(field, models.ForeignKey)
+        ]
+    return keys
+
+
+class RecordsList(UserList):
+    """Is ordered, can't be queried, but has `.df()`."""
+
+    def __init__(self, records: List[Registry]):
+        super().__init__(record for record in records)
+
+    def df(self) -> pd.DataFrame:
+        keys = get_keys_from_df(self.data, self.data[0].__class__)
+        values = [record.__dict__ for record in self.data]
+        return pd.DataFrame(values, columns=keys)
+
+
 class QuerySet(models.QuerySet, CanValidate, IsTree):
     """Lazily loaded queried records returned by queries.
 
@@ -59,24 +94,7 @@ class QuerySet(models.QuerySet, CanValidate, IsTree):
             >>> ln.ULabel.filter().df(include=["labels__name", "labels__created_by_id"])
         """
         data = self.values()
-        if len(data) > 0:
-            keys = list(data[0].keys())
-            if "created_at" in keys:
-                keys.remove("created_at")
-        else:
-            keys = [
-                field.name
-                for field in self.model._meta.fields
-                if (
-                    not isinstance(field, models.ForeignKey)
-                    and field.name != "created_at"
-                )
-            ]
-            keys += [
-                f"{field.name}_id"
-                for field in self.model._meta.fields
-                if isinstance(field, models.ForeignKey)
-            ]
+        keys = get_keys_from_df(data, self.model)
         df = pd.DataFrame(self.values(), columns=keys)
         # if len(df) > 0 and "updated_at" in df:
         #     df.updated_at = format_and_convert_to_local_time(df.updated_at)

lamindb/dev/__init__.py

@@ -14,6 +14,7 @@ Queries of registries:
 
    QuerySet
    QueryManager
+   RecordsList
 
 Functionality of data registries:
 
@@ -62,7 +63,7 @@ from lnschema_core.models import (
 )
 
 from lamindb._query_manager import QueryManager
-from lamindb._query_set import QuerySet
+from lamindb._query_set import QuerySet, RecordsList
 from lamindb.dev._feature_manager import FeatureManager
 from lamindb.dev._label_manager import LabelManager
 

lamindb/dev/_mapped_collection.py

@@ -5,6 +5,7 @@ from typing import List, Literal, Optional, Union
 
 import numpy as np
 import pandas as pd
+from lamin_utils import logger
 from lamindb_setup.dev.upath import UPath
 
 from .storage._backed_access import (
@@ -57,13 +58,13 @@ class MappedCollection:
         self,
         path_list: List[Union[str, PathLike]],
         label_keys: Optional[Union[str, List[str]]] = None,
-        join_vars: Optional[Literal["auto", "inner", "outer"]] = "auto",
+        join: Optional[Literal["inner", "outer"]] = "outer",
         encode_labels: bool = True,
         cache_categories: bool = True,
         parallel: bool = False,
         dtype: Optional[str] = None,
     ):
-        assert join_vars in {None, "auto", "inner", "outer"}
+        assert join in {None, "inner", "outer"}
 
         self.storages = []  # type: ignore
         self.conns = []  # type: ignore
@@ -83,7 +84,7 @@ class MappedCollection:
         self.indices = np.hstack([np.arange(n_obs) for n_obs in self.n_obs_list])
         self.storage_idx = np.repeat(np.arange(len(self.storages)), self.n_obs_list)
 
-        self.join_vars = join_vars if len(path_list) > 1 else None
+        self.join_vars = join if len(path_list) > 1 else None
         self.var_indices = None
         if self.join_vars is not None:
             self._make_join_vars()
@@ -99,7 +100,6 @@ class MappedCollection:
                 self._make_encoders(self.label_keys)
 
         self._dtype = dtype
-
         self._closed = False
 
     def _make_connections(self, path_list: list, parallel: bool):
@@ -144,13 +144,20 @@ class MappedCollection:
             vars_eq = all(var_list[0].equals(vrs) for vrs in var_list[1:])
             if vars_eq:
                 self.join_vars = None
+                logger.info("The variables are same, no virtual join is performed.")
                 return
             else:
                 self.var_joint = reduce(pd.Index.intersection, var_list)
                 if len(self.var_joint) > 0:
                     self.join_vars = "inner"
+                    logger.info(
+                        "The intersection of variables is not empty, using virtual inner join."
+                    )
                 else:
                     self.join_vars = "outer"
+                    logger.info(
+                        "The intersection of variables is empty, using virtual outer join."
+                    )
 
         if self.join_vars == "inner":
             if self.var_joint is None:

{lamindb-0.65.1.dist-info → lamindb-0.66.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.65.1
+Version: 0.66.0
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,10 +9,10 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.60.0
-Requires-Dist: lamindb_setup==0.63.1
+Requires-Dist: lnschema_core==0.61.0
+Requires-Dist: lamindb_setup==0.64.0
 Requires-Dist: lamin_utils==0.13.0
-Requires-Dist: lamin_cli==0.5.0
+Requires-Dist: lamin_cli==0.6.0
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
@@ -62,14 +62,14 @@ Provides-Extra: zarr
 
 # LaminDB - A data framework for biology
 
-LaminDB is an open-source Python framework to manage biological data & analyses in generic backends:
+LaminDB is an open-source Python framework to manage biological data & analyses:
 
-- Access data & metadata across storage (files, arrays) & database (SQL) backends.
-- Track data flow across notebooks, pipelines & UI.
-- Manage registries for experimental metadata & in-house ontologies, import public ontologies.
-- Validate, standardize & annotate data using registries.
+- Access data & metadata across storage & databases.
+- Track data lineage across notebooks & pipelines.
+- Manage registries for experimental metadata & in-house ontologies.
+- Validate, standardize & annotate data.
 - Organize and share data across a mesh of LaminDB instances.
-- Manage data access with an auditable system of record.
+- Manage data access, leverage an auditable system of record.
 
 ## Documentation
 

{lamindb-0.65.1.dist-info → lamindb-0.66.0.dist-info}/RECORD

@@ -1,14 +1,14 @@
-lamindb/__init__.py,sha256=wu6XUzgFXCaSVeiVPAHKArxslewM8S2OY7-Lwk2w8EM,2691
+lamindb/__init__.py,sha256=rgCY0tETrHKyB7V5f2Y3BhY4BAJicGUUYzSRwIdRlmI,2691
 lamindb/_artifact.py,sha256=eWsLj8x6Cqy8MR7LxKyScxozM52MaqOTCK8gplloP2c,38087
-lamindb/_collection.py,sha256=0gitrRx4K1p9dWp3VSPnEczeuZcRUJPeu9_crTCthZQ,16968
+lamindb/_collection.py,sha256=gVcs3A200JZilfdYd0zrX29UrAmhP9Eovu6r_SIxXQ4,17634
 lamindb/_delete.py,sha256=jO6kcIoxY6EFgqiVF2vlbXaCaqlI25AvBo7nre3JXkQ,1968
-lamindb/_feature.py,sha256=AqQZTOL38aElT3-e7WCj8Fm2Xcso0uJO0oE72fQCScU,5989
+lamindb/_feature.py,sha256=tEcqFoEj5yp4LSJfMGyiVvxDUuLoZaik6lo05ZKcCtE,6036
 lamindb/_feature_set.py,sha256=KYgdmMdXb21pfpir1J1O21in3nJvUeznECOB38qfTvk,8654
 lamindb/_filter.py,sha256=YwWqviJ34kHTMJ8NYlrEw-vsrXkKrVIPsEZSBVvMcrI,1163
 lamindb/_from_values.py,sha256=dKz4cTUBRkXOOzFX2Ix2cKhK2Lw9PyTgi7d0PI-kh3c,11869
-lamindb/_parents.py,sha256=KU_xH5mWBEQzBi7VSemDOOfB7H1PpReh0U6268v-8Q0,14020
+lamindb/_parents.py,sha256=hyoN92YnfJFmRWmQMLLUjTKKwnIOJci5z6csMjsdYDE,14165
 lamindb/_query_manager.py,sha256=m4WUScviuNlMHeNEPZ8H8y0YsMXSbwWyfIgS4L00wBY,4332
-lamindb/_query_set.py,sha256=nacnkFaVYDmuFkpXr0fb3uNcWP6XahbMeIvJic0YCSk,9967
+lamindb/_query_set.py,sha256=tItL2YNdycpbXklYd8aW4jJX6Z-kGcNclscg0v3l8t4,10495
 lamindb/_registry.py,sha256=MxYpJUKD6Qu5eO2jO6JOcQBBGxfQpiEGPJrFaXau_jw,17421
 lamindb/_run.py,sha256=659lqY32GW7F41rFUUo37OftUa38-p8yaV9Z0oF32CE,1120
 lamindb/_save.py,sha256=UlRHJGUiHGOXv90wmawZVsOqhJIqk8f1wj8MW3Rlq_c,10535
@@ -18,11 +18,11 @@ lamindb/_ulabel.py,sha256=HALoy6HerRnehR-u8zPH-qmiFQHWxeAwkZ31jxjrfgI,1893
 lamindb/_utils.py,sha256=LGdiW4k3GClLz65vKAVRkL6Tw-Gkx9DWAdez1jyA5bE,428
 lamindb/_validate.py,sha256=fS2685MYX9h6iAWymEorJJmDYA2CGNOSmJpesbG6faU,14400
 lamindb/_view.py,sha256=yFMu4vnt0YqvN1q11boAkwigxCH1gdliDUSbzh3IuDw,2175
-lamindb/dev/__init__.py,sha256=8EwMNFJ45Ws5ApoYjaXOuca4uvyR6WtjAvnfbq7M11g,1146
+lamindb/dev/__init__.py,sha256=LLqivujL8c-oKWC15SJepAYyrTlLNvql5Vdwunc0qvE,1174
 lamindb/dev/_data.py,sha256=YPZ664qGKMl34LbZCMCEFIxQ-E81iAt_b3lvMiTe-oc,17066
 lamindb/dev/_feature_manager.py,sha256=jn8x_JbrtLFelmaFh4noOXqGSCfqVuVX0quoa7gTJtM,9366
 lamindb/dev/_label_manager.py,sha256=6E_pSQicqfTWDGEGe4WPn_3GZl_CCIMTZ6xJDh4EkC0,8740
-lamindb/dev/_mapped_collection.py,sha256=BiiVeFPs0g7TavYHtPh098cCN_8oFCo1nWguASKAAsI,14062
+lamindb/dev/_mapped_collection.py,sha256=NRjOYnC1d3IcVyqhT_Yp0xycepmeytlngYnw-5Xcnw4,14445
 lamindb/dev/_run_context.py,sha256=4eBZsbfcFpW5nqmRLbRZxuA5oeRW17XVHMzVtMH0bKA,22965
 lamindb/dev/_settings.py,sha256=nixk8lVijCbq_fRlUpkX5gvO9AdgUFjbXzFThAJhGBA,3824
 lamindb/dev/_track_environment.py,sha256=QjHWbyl2u8J4hbJG8Q_ToFaZIgS-H15Ej6syJgk-dvY,662
@@ -43,7 +43,7 @@ lamindb/dev/storage/file.py,sha256=jalzFQ8q110UUu_GGQBkU-g3M04h5g4LJ3nLjCzJ4pU,5
 lamindb/dev/storage/object.py,sha256=KGuOwwYuN2yCJxTXn9v0LanC0fjKwy_62P-WksHcf40,1140
 lamindb/setup/__init__.py,sha256=WaWKO-2XT67S65lSbS80hUojL-Mr_Wms9UxH6U54TsY,289
 lamindb/setup/dev/__init__.py,sha256=tBty426VGF2PGqqt2XuNU-WgvOrbOp1aZBDowjLuzgA,242
-lamindb-0.65.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-lamindb-0.65.1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-lamindb-0.65.1.dist-info/METADATA,sha256=yGClOnYmbvX5RHc6bWW3Jldrji95UB-TLcoaXR98BMo,3165
-lamindb-0.65.1.dist-info/RECORD,,
+lamindb-0.66.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+lamindb-0.66.0.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+lamindb-0.66.0.dist-info/METADATA,sha256=d9S5mPiFAzV1EhN7KB_VnugNCy7vdeivGqtxZsZPD60,3076
+lamindb-0.66.0.dist-info/RECORD,,