lamindb 0.75.1__tar.gz → 0.76.0__tar.gz

{lamindb-0.75.1 → lamindb-0.76.0}/.github/workflows/build.yml

@@ -21,7 +21,7 @@ jobs:
           - "faq"
           - "storage"
           - "cli"
-    timeout-minutes: 6
+    timeout-minutes: 7
 
     steps:
       - uses: actions/checkout@v4

{lamindb-0.75.1 → lamindb-0.76.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.75.1
+Version: 0.76.0
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,10 +9,10 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.72.2
-Requires-Dist: lamindb_setup==0.76.3
+Requires-Dist: lnschema_core==0.73.0
+Requires-Dist: lamindb_setup==0.76.4
 Requires-Dist: lamin_utils==0.13.2
-Requires-Dist: lamin_cli==0.16.0
+Requires-Dist: lamin_cli==0.16.1
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
@@ -24,7 +24,7 @@ Requires-Dist: pandas
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
 Requires-Dist: lamindb_setup[aws] ; extra == "aws"
-Requires-Dist: bionty==0.48.0 ; extra == "bionty"
+Requires-Dist: bionty==0.48.1 ; extra == "bionty"
 Requires-Dist: pre-commit ; extra == "dev"
 Requires-Dist: nox ; extra == "dev"
 Requires-Dist: laminci>=0.3 ; extra == "dev"

{lamindb-0.75.1 → lamindb-0.76.0}/docs/api.md

@@ -27,6 +27,14 @@ bionty
 wetlab
 ```
 
+```{toctree}
+:maxdepth: 1
+:caption: SDK
+:hidden:
+
+rest
+```
+
 ```{toctree}
 :maxdepth: 1
 :caption: Utilities

lamindb-0.76.0/docs/rest.md

@@ -0,0 +1,3 @@
+# `REST`
+
+Actual content in lamin-docs.

{lamindb-0.75.1 → lamindb-0.76.0}/lamindb/__init__.py

@@ -42,7 +42,7 @@ Modules & settings:
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.75.1"
+__version__ = "0.76.0"
 
 import os as _os
 

{lamindb-0.75.1 → lamindb-0.76.0}/lamindb/_artifact.py

@@ -594,6 +594,7 @@ def __init__(artifact: Artifact, *args, **kwargs):
     kwargs["description"] = description
     kwargs["visibility"] = visibility
     kwargs["_accessor"] = accessor
+    kwargs["is_new_version_of"] = is_new_version_of
     # this check needs to come down here because key might be populated from an
     # existing file path during get_artifact_kwargs_from_data()
     if (

{lamindb-0.75.1 → lamindb-0.76.0}/lamindb/_collection.py

@@ -158,6 +158,7 @@ def __init__(
             run=run,
             version=version,
             visibility=visibility,
+            is_new_version_of=is_new_version_of,
             **kwargs,
         )
     collection._artifacts = artifacts

{lamindb-0.75.1 → lamindb-0.76.0}/lamindb/_feature.py

@@ -42,7 +42,7 @@ def __init__(self, *args, **kwargs):
         return None
     # now we proceed with the user-facing constructor
     if len(args) != 0:
-        raise ValueError("Only non-keyword args allowed")
+        raise ValueError("Only keyword args allowed")
     dtype: type | str = kwargs.pop("dtype") if "dtype" in kwargs else None
     # cast type
     if dtype is None:

{lamindb-0.75.1 → lamindb-0.76.0}/lamindb/_from_values.py

@@ -47,15 +47,35 @@ def get_or_create_records(
 
         # new records to be created based on new values
         if len(nonexist_values) > 0:
-            if source:
-                from_source = not source.in_db
-            elif (
-                records
-                and hasattr(records[0], "source_id")
-                and records[0].source_id
-                and records[0].source.in_db
-            ):
+            source_record = None
+            if from_source:
+                if isinstance(source, Record):
+                    source_record = source
+                elif (
+                    len(records) > 0
+                    and hasattr(records[0], "source_id")
+                    and records[0].source_id
+                ):
+                    source_record = records[0].source
+            if not source_record and hasattr(Record, "public"):
+                from bionty._bionty import get_source_record
+
+                source_record = get_source_record(Record.public(organism=organism))
+            if source_record:
+                from bionty.core._add_ontology import check_source_in_db
+
+                check_source_in_db(
+                    registry=Record,
+                    source=source_record,
+                    update=True,
+                )
+
+                from_source = not source_record.in_db
+            elif hasattr(Record, "source_id"):
+                from_source = True
+            else:
                 from_source = False
+
             if from_source:
                 records_bionty, unmapped_values = create_records_from_source(
                     iterable_idx=nonexist_values,
@@ -211,10 +231,6 @@ def create_records_from_source(
         return records, iterable_idx
     # add source record to the kwargs
     source_record = get_source_record(public_ontology)
-    if source_record is not None and source_record.in_db:
-        # skips the creation of records from public if the source is already in the db
-        return records, iterable_idx
-
     kwargs.update({"source": source_record})
 
     # filter the columns in bionty df based on fields

{lamindb-0.75.1 → lamindb-0.76.0}/lamindb/_query_set.py

@@ -243,10 +243,10 @@ class QuerySet(models.QuerySet, CanValidate):
         else:
             raise MultipleResultsFound(self.all())
 
-    def latest_version(self) -> RecordsList:
+    def latest_version(self) -> QuerySet:
         """Filter every version family by latest version."""
         if issubclass(self.model, IsVersioned):
-            return filter_query_set_by_latest_version(self)
+            return self.filter(is_latest=True)
         else:
             raise ValueError("Record isn't subclass of `lamindb.core.IsVersioned`")
 
@@ -288,29 +288,6 @@ class QuerySet(models.QuerySet, CanValidate):
         return _standardize(cls=self, values=values, field=field, **kwargs)
 
 
-def filter_query_set_by_latest_version(ordered_query_set: QuerySet) -> RecordsList:
-    # evaluating length can be very costly, hence, the try-except block
-    try:
-        first_record = ordered_query_set[0]
-    except IndexError:
-        return ordered_query_set
-    records_in_view = {}
-    records_in_view[first_record.stem_uid] = first_record
-    for record in ordered_query_set:
-        # this overwrites user-provided ordering (relevant records ordered by a
-        # certain field will not show if they are not the latest version)
-        if record.stem_uid not in records_in_view:
-            records_in_view[record.stem_uid] = record
-        else:
-            if record.created_at > records_in_view[record.stem_uid].created_at:
-                # deleting the entry is needed to preserve the integrity of
-                # user-provided ordering
-                del records_in_view[record.stem_uid]
-                records_in_view[record.stem_uid] = record
-    list_records_in_view = RecordsList(records_in_view.values())
-    return list_records_in_view
-
-
 models.QuerySet.df = QuerySet.df
 models.QuerySet.list = QuerySet.list
 models.QuerySet.first = QuerySet.first

{lamindb-0.75.1 → lamindb-0.76.0}/lamindb/_record.py

@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, List, NamedTuple
 
 import dj_database_url
 import lamindb_setup as ln_setup
-from django.db import connections
+from django.db import connections, transaction
 from django.db.models import IntegerField, Manager, Q, QuerySet, Value
 from lamin_utils import logger
 from lamin_utils._lookup import Lookup
@@ -132,7 +132,10 @@ def get(cls, idlike: int | str) -> Record:
     else:
         qs = filter(cls, uid__startswith=idlike)
         if issubclass(cls, IsVersioned):
-            return qs.latest_version().one()
+            if len(idlike) <= cls._len_stem_uid:
+                return qs.latest_version().one()
+            else:
+                return qs.one()
         else:
             return qs.one()
 
@@ -165,9 +168,7 @@ def from_values(
 ) -> list[Record]:
     """{}"""  # noqa: D415
     from_source = True if cls.__module__.startswith("bionty.") else False
-    # if records from source is already saved in db, skip from_source
-    if isinstance(source, Record) and source.in_db:
-        from_source = False
+
     field_str = get_name_field(cls, field=field)
     return get_or_create_records(
         iterable=values,
@@ -528,7 +529,28 @@ def save(self, *args, **kwargs) -> Record:
     if result is not None:
         init_self_from_db(self, result)
     else:
-        super(Record, self).save(*args, **kwargs)
+        # save versioned record
+        if isinstance(self, IsVersioned) and self._is_new_version_of is not None:
+            if self._is_new_version_of.is_latest:
+                is_new_version_of = self._is_new_version_of
+            else:
+                # need one additional request
+                is_new_version_of = self.__class__.objects.get(
+                    is_latest=True, uid__startswith=self.stem_uid
+                )
+                logger.warning(
+                    f"didn't pass the latest version in `is_new_version_of`, retrieved it: {is_new_version_of}"
+                )
+            is_new_version_of.is_latest = False
+            with transaction.atomic():
+                is_new_version_of._is_new_version_of = (
+                    None  # ensure we don't start a recursion
+                )
+                is_new_version_of.save()
+                super(Record, self).save(*args, **kwargs)
+        # save unversioned record
+        else:
+            super(Record, self).save(*args, **kwargs)
     # perform transfer of many-to-many fields
     # only supported for Artifact and Collection records
     if db is not None and db != "default" and using_key is None:
@@ -553,6 +575,30 @@ def save(self, *args, **kwargs) -> Record:
     return self
 
 
+def delete(self) -> None:
+    """Delete the record."""
+    # note that the logic below does not fire if a record is moved to the trash
+    # the idea is that moving a record to the trash should move its entire version family
+    # to the trash, whereas permanently deleting should default to only deleting a single record
+    # of a version family
+    # we can consider making it easy to permanently delete entire version families as well,
+    # but that's for another time
+    if isinstance(self, IsVersioned) and self.is_latest:
+        new_latest = (
+            self.__class__.filter(is_latest=False, uid__startswith=self.stem_uid)
+            .order_by("-created_at")
+            .first()
+        )
+        if new_latest is not None:
+            new_latest.is_latest = True
+            with transaction.atomic():
+                new_latest.save()
+                super(Record, self).delete()
+            logger.warning(f"new latest version is {new_latest}")
+            return None
+    super(Record, self).delete()
+
+
 METHOD_NAMES = [
     "__init__",
     "filter",
@@ -561,6 +607,7 @@ METHOD_NAMES = [
     "search",
     "lookup",
     "save",
+    "delete",
     "from_values",
     "using",
 ]

{lamindb-0.75.1 → lamindb-0.76.0}/lamindb/_run.py

@@ -42,7 +42,7 @@ def delete_run_artifacts(run: Run) -> None:
         run.save()
     if environment is not None:
         # only delete if there are no other runs attached to this environment
-        if environment.environment_of.count() == 0:
+        if environment._environment_of.count() == 0:
             environment.delete(permanent=True)
     if report is not None:
         report.delete(permanent=True)

{lamindb-0.75.1 → lamindb-0.76.0}/lamindb/_transform.py

@@ -22,7 +22,6 @@ def __init__(transform: Transform, *args, **kwargs):
     is_new_version_of: Transform | None = (
         kwargs.pop("is_new_version_of") if "is_new_version_of" in kwargs else None
     )
-    (kwargs.pop("initial_version_id") if "initial_version_id" in kwargs else None)
     version: str | None = kwargs.pop("version") if "version" in kwargs else None
     type: TransformType | None = kwargs.pop("type") if "type" in kwargs else "pipeline"
     reference: str | None = kwargs.pop("reference") if "reference" in kwargs else None
@@ -55,6 +54,7 @@ def __init__(transform: Transform, *args, **kwargs):
         reference=reference,
         reference_type=reference_type,
         _has_consciously_provided_uid=has_consciously_provided_uid,
+        is_new_version_of=is_new_version_of,
     )
 
 

{lamindb-0.75.1 → lamindb-0.76.0}/lamindb/core/_label_manager.py

@@ -32,14 +32,15 @@ def get_labels_as_dict(self: HasFeatures, links: bool = False):
         "input_of_runs",
         "collections",
         "_source_code_artifact_of",
-        "report_of",
-        "environment_of",
+        "_report_of",
+        "_environment_of",
         "links_collection",
         "links_artifact",
         "links_feature_set",
         "previous_runs",
         "_feature_values",
-        "_lnschema_core_collection__actions_+",
+        "_action_targets",
+        "_lnschema_core_collection__actions_+",  # something seems off with this one
         "_actions",
     }
     labels = {}  # type: ignore

{lamindb-0.75.1 → lamindb-0.76.0}/lamindb/core/datasets/_core.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from pathlib import Path
+from typing import TYPE_CHECKING
 from urllib.request import urlretrieve
 
 import anndata as ad
@@ -11,6 +12,9 @@ from upath import UPath
 
 from lamindb.core._settings import settings
 
+if TYPE_CHECKING:
+    from mudata import MuData
+
 
 def file_fcs() -> Path:
     """Example FCS artifact."""
@@ -116,7 +120,7 @@ def file_mini_csv(in_storage_root=False) -> Path:
     return filepath
 
 
-def file_tiff_suo22():  # pragma: no cover
+def file_tiff_suo22() -> Path:  # pragma: no cover
     """Image file from Suo22.
 
     Pair with anndata_suo22_Visium10X
@@ -126,7 +130,7 @@ def file_tiff_suo22():  # pragma: no cover
         "F121_LP1_4LIV.tiff",
     )
     Path("suo22/").mkdir(exist_ok=True)
-    filepath = Path(filepath).rename("suo22/F121_LP1_4LIV.tiff")
+    filepath = Path(filepath).rename("suo22/F121_LP1_4LIV.tiff")  # type: ignore
     return Path(filepath)
 
 
@@ -282,17 +286,16 @@ def anndata_human_immune_cells(
 ) -> ad.AnnData:  # pragma: no cover
     """Cross-tissue immune cell analysis reveals tissue-specific features in humans.
 
-    From: https://cellxgene.cziscience.com/collections/62ef75e4-cbea-454e-a0ce-998ec40223d3  # noqa
+    From: https://cellxgene.cziscience.com/collections/62ef75e4-cbea-454e-a0ce-998ec40223d3
     Collection: Global
 
     To reproduce the subsample::
-
-        adata = sc.read('Global.h5ad')
-        adata.obs = adata.obs[['donor_id', 'tissue', 'cell_type', 'assay', 'tissue_ontology_term_id', 'cell_type_ontology_term_id', 'assay_ontology_term_id']].copy()
-        sc.pp.subsample(adata, fraction=0.005)
-        del adata.uns["development_cache_ontology_term_id_colors"]
-        del adata.uns["sex_ontology_term_id_colors"]
-        adata.write('human_immune.h5ad')
+        >>> adata = sc.read('Global.h5ad')
+        >>> adata.obs = adata.obs[['donor_id', 'tissue', 'cell_type', 'assay', 'tissue_ontology_term_id', 'cell_type_ontology_term_id', 'assay_ontology_term_id']].copy()
+        >>> sc.pp.subsample(adata, fraction=0.005)
+        >>> del adata.uns["development_cache_ontology_term_id_colors"]
+        >>> del adata.uns["sex_ontology_term_id_colors"]
+        >>> adata.write('human_immune.h5ad')
     """
     filepath, _ = urlretrieve("https://lamindb-test.s3.amazonaws.com/human_immune.h5ad")
     adata = ad.read_h5ad(filepath)
@@ -369,29 +372,34 @@ def anndata_suo22_Visium10X():  # pragma: no cover
     return ad.read_h5ad(filepath)
 
 
-def mudata_papalexi21_subset():  # pragma: no cover
+def mudata_papalexi21_subset() -> MuData:  # pragma: no cover
     """A subsetted mudata from papalexi21.
 
     To reproduce the subsetting:
-    >>> !wget https://figshare.com/ndownloader/files/36509460
-    >>> import mudata as md
-    >>> import scanpy as sc
-    >>> mdata = md.read_h5mu("36509460")
-    >>> mdata = sc.pp.subsample(mdata, n_obs=200, copy=True)[0]
-    >>> mdata[:, -300:].copy().write("papalexi21_subset_200x300_lamindb_demo_2023-07-25.h5mu")  # noqa
+        >>> !wget https://figshare.com/ndownloader/files/36509460
+        >>> import mudata as md
+        >>> import scanpy as sc
+        >>> mdata = md.read_h5mu("36509460")
+        >>> mdata = sc.pp.subsample(mdata, n_obs=200, copy=True)[0]
+        >>> mdata[:, -300:].copy().write("papalexi21_subset_200x300_lamindb_demo_2023-07-25.h5mu")
     """
     import mudata as md
 
+    md.set_options(pull_on_update=False)
+
     filepath, _ = urlretrieve(
         "https://lamindb-test.s3.amazonaws.com/papalexi21_subset_200x300_lamindb_demo_2023-07-25.h5mu",
         "papalexi21_subset.h5mu",
     )
 
     mdata = md.read_h5mu(filepath)
+
+    mdata.pull_obs()
+
+    # The MuData object is malformed with duplicated information
+    # Drop all columns for the modalities and add them again correspondingly
     for mod in ["rna", "adt", "hto", "gdo"]:
-        mdata[mod].obs.drop(
-            mdata[mod].obs.columns, axis=1, inplace=True
-        )  # Drop all columns
+        mdata[mod].obs.drop(mdata[mod].obs.columns, axis=1, inplace=True)
     for col in mdata.obs.columns:
         for mod in ["rna", "adt", "hto", "gdo"]:
             if col.endswith(f"_{mod.upper()}"):
@@ -420,12 +428,10 @@ def mudata_papalexi21_subset():  # pragma: no cover
         "HTO_classification",
     ]:
         del mdata.obs[col]
-        mdata.update()
 
-    mdata["rna"].obs["percent.mito"] = mdata.obs.pop("percent.mito")
+    mdata.push_obs(["percent.mito"], mods=["rna"], drop=True)
     mdata["hto"].obs["technique"] = "cell hashing"
     mdata["hto"].obs["technique"] = mdata["hto"].obs["technique"].astype("category")
-    mdata.update()
 
     return mdata
 

{lamindb-0.75.1 → lamindb-0.76.0}/lamindb/core/versioning.py

@@ -72,7 +72,6 @@ def set_version(version: str | None = None, previous_version: str | None = None)
     return version
 
 
-# uses `initial_version_id` to extract a stem_id that's part of id
 def init_uid(
     *,
     version: str | None = None,

{lamindb-0.75.1 → lamindb-0.76.0}/pyproject.toml

@@ -16,10 +16,10 @@ classifiers = [
 ]
 dependencies = [
     # Lamin PINNED packages
-    "lnschema_core==0.72.2",
-    "lamindb_setup==0.76.3",
+    "lnschema_core==0.73.0",
+    "lamindb_setup==0.76.4",
     "lamin_utils==0.13.2",
-    "lamin_cli==0.16.0",
+    "lamin_cli==0.16.1",
     # others
     "rapidfuzz",
     "pyarrow",
@@ -38,7 +38,7 @@ Home = "https://github.com/laminlabs/lamindb"
 
 [project.optional-dependencies]
 bionty = [
-    "bionty==0.48.0",
+    "bionty==0.48.1",
 ]
 aws = [
     "lamindb_setup[aws]",

{lamindb-0.75.1 → lamindb-0.76.0}/sub/bionty/bionty/__init__.py

@@ -128,7 +128,7 @@ Bionty base:
    base
 """
 
-__version__ = "0.48.0"
+__version__ = "0.48.1"
 
 from . import base, ids
 

{lamindb-0.75.1 → lamindb-0.76.0}/sub/bionty/bionty/base/sources.yaml

@@ -354,6 +354,9 @@ BFXPipeline:
 Drug:
   dron:
     all:
+      2024-08-05:
+        url: https://data.bioontology.org/ontologies/DRON/submissions/24/download?apikey=8b5b7825-538d-40e0-9e9e-5ab9274a9aeb
+        md5: ""
       2024-03-02:
         url: https://data.bioontology.org/ontologies/DRON/submissions/22/download?apikey=8b5b7825-538d-40e0-9e9e-5ab9274a9aeb
         md5: 84138459de4f65034e979f4e46783747