lamindb 0.55.0__py3-none-any.whl → 0.55.1__py3-none-any.whl

lamindb/__init__.py

@@ -54,7 +54,7 @@ Static classes & modules:
 
 """
 
-__version__ = "0.55.0"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.55.1"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os as _os
 

lamindb/_dataset.py

@@ -1,10 +1,15 @@
 from collections import defaultdict
+from pathlib import Path
 from typing import Dict, Iterable, Literal, Optional, Tuple, Union
 
 import anndata as ad
 import pandas as pd
 from lamin_utils import logger
+from lamindb_setup._init_instance import register_storage
+from lamindb_setup.dev import StorageSettings
 from lamindb_setup.dev._docs import doc_args
+from lamindb_setup.dev._hub_utils import get_storage_region
+from lamindb_setup.dev.upath import UPath
 from lnschema_core import Modality
 from lnschema_core.models import Dataset, Feature, FeatureSet
 from lnschema_core.types import AnnDataLike, DataLike, FieldAttr
@@ -84,11 +89,12 @@ def __init__(
             )
 
     run = get_run(run)
-    # there are exactly two ways of creating a Dataset object right now
+    # there are exactly 3 ways of creating a Dataset object right now
     # using exactly one file or using more than one file
     # init file
     if isinstance(data, (pd.DataFrame, ad.AnnData, File)):
         files = None
+        storage = None
         if isinstance(data, File):
             file = data
             if file._state.adding:
@@ -116,9 +122,23 @@ def __init__(
         if file.description is None or file.description == "tmp":
             file.description = f"See dataset {provisional_id}"  # type: ignore
         file._feature_sets = feature_sets
+        storage = None
+    # init from directory or bucket
+    elif isinstance(data, (str, Path, UPath)):
+        file = None
+        files = None
+        upath = UPath(data)
+        if not upath.is_dir:
+            raise ValueError(f"Can only pass buckets or directories, not {data}")
+        upath_str = upath.as_posix().rstrip("/")
+        region = get_storage_region(upath_str)
+        storage_settings = StorageSettings(upath_str, region)
+        storage = register_storage(storage_settings)
+        hash = None
     # init files
     else:
         file = None
+        storage = None
         if hasattr(data, "__getitem__"):
             assert isinstance(data[0], File)  # type: ignore
             files = data
@@ -144,6 +164,7 @@ def __init__(
             reference=reference,
             reference_type=reference_type,
             file=file,
+            storage=storage,
             hash=hash,
             run=run,
             version=version,
@@ -276,8 +297,9 @@ def load(
     is_run_input: Optional[bool] = None,
     **kwargs,
 ) -> DataLike:
-    _track_run_input(self, is_run_input)
+    # cannot call _track_run_input here, see comment further down
     if self.file is not None:
+        _track_run_input(self, is_run_input)
         return self.file.load()
     else:
         all_files = self.files.all()
@@ -291,9 +313,14 @@ def load(
         objects = [file.load(is_run_input=False) for file in all_files]
         file_ids = [file.id for file in all_files]
         if isinstance(objects[0], pd.DataFrame):
-            return pd.concat(objects, join=join)
+            concat_object = pd.concat(objects, join=join)
         elif isinstance(objects[0], ad.AnnData):
-            return ad.concat(objects, join=join, label="file_id", keys=file_ids)
+            concat_object = ad.concat(
+                objects, join=join, label="file_id", keys=file_ids
+            )
+        # only call it here because there might be errors during concat
+        _track_run_input(self, is_run_input)
+        return concat_object
 
 
 # docstring handled through attach_func_to_class_method
@@ -316,6 +343,14 @@ def save(self, *args, **kwargs) -> None:
     save_feature_set_links(self)
 
 
+@property  # type: ignore
+@doc_args(Dataset.path.__doc__)
+def path(self) -> Union[Path, UPath]:
+    """{}"""
+    _track_run_input(self)
+    return self.storage.path
+
+
 METHOD_NAMES = [
     "__init__",
     "from_anndata",
@@ -337,3 +372,5 @@ if _TESTING:
 
 for name in METHOD_NAMES:
     attach_func_to_class_method(name, Dataset, globals())
+
+setattr(Dataset, "path", path)

lamindb/_file.py

@@ -85,9 +85,9 @@ def process_pathlike(
                 new_root_str = new_root.as_posix().rstrip("/")
                 region = get_storage_region(new_root_str)
                 storage_settings = StorageSettings(new_root_str, region)
-                register_storage(storage_settings)
+                storage_record = register_storage(storage_settings)
                 use_existing_storage_key = True
-                return storage_settings.record, use_existing_storage_key
+                return storage_record, use_existing_storage_key
             # if the filepath is local
             else:
                 use_existing_storage_key = False

lamindb/_filter.py

@@ -3,10 +3,6 @@ from uuid import UUID
 
 import dj_database_url
 from django.db import connections
-from django.db.backends.postgresql.base import (
-    DatabaseWrapper as DatabaseWrapperPostgres,
-)
-from django.db.backends.sqlite3.base import DatabaseWrapper as DatabaseWrapperSQLite
 from lamindb_setup._init_instance import InstanceSettings
 from lamindb_setup._load_instance import get_owner_name_from_identifier
 from lamindb_setup.dev._hub_core import load_instance
@@ -22,11 +18,8 @@ def add_db_connection(isettings: InstanceSettings, using: str):
     db_config["TIME_ZONE"] = "UTC"
     db_config["OPTIONS"] = {}
     db_config["AUTOCOMMIT"] = True
-    if isettings.dialect == "sqlite":
-        db_wrapper = DatabaseWrapperSQLite(db_config, alias=using)
-    else:
-        db_wrapper = DatabaseWrapperPostgres(db_config, alias=using)
-    connections[using] = db_wrapper
+
+    connections.settings[using] = db_config
 
 
 def filter(Registry: Type[Registry], using: str = None, **expressions) -> QuerySet:

lamindb/_query_manager.py

@@ -25,6 +25,16 @@ class QueryManager(models.Manager):
         >>> manager.df()
     """
 
+    def _track_run_input_manager(self):
+        if hasattr(self, "source_field_name") and hasattr(self, "target_field_name"):
+            if self.source_field_name == "dataset" and self.target_field_name == "file":
+                from lamindb.dev._data import WARNING_RUN_TRANSFORM, _track_run_input
+                from lamindb.dev._run_context import run_context
+
+                if run_context.run is None:
+                    logger.warning(WARNING_RUN_TRANSFORM)
+                _track_run_input(self.instance)
+
     def list(self, field: Optional[str] = None):
         """Populate a list with the results.
 
@@ -41,6 +51,7 @@ class QueryManager(models.Manager):
             >>> label.parents.list("name")
             ['ULabel1', 'ULabel2', 'ULabel3']
         """
+        self._track_run_input_manager()
         if field is None:
             return [item for item in self.all()]
         else:
@@ -58,15 +69,8 @@ class QueryManager(models.Manager):
 
         For `**kwargs`, see :meth:`lamindb.dev.QuerySet.df`.
         """
-        if hasattr(self, "source_field_name") and hasattr(self, "target_field_name"):
-            if self.source_field_name == "dataset" and self.target_field_name == "file":
-                from lamindb.dev._data import WARNING_RUN_TRANSFORM, _track_run_input
-                from lamindb.dev._run_context import run_context
-
-                if run_context.run is None:
-                    logger.warning(WARNING_RUN_TRANSFORM)
-                _track_run_input(self.instance)
-        return self.all()
+        self._track_run_input_manager()
+        return self.all_base_class()
 
     def __getitem__(self, item: str):
         try:
@@ -86,3 +90,10 @@ class QueryManager(models.Manager):
 setattr(models.Manager, "list", QueryManager.list)
 setattr(models.Manager, "df", QueryManager.df)
 setattr(models.Manager, "__getitem__", QueryManager.__getitem__)
+setattr(
+    models.Manager, "_track_run_input_manager", QueryManager._track_run_input_manager
+)
+# the two lines below would be easy if we could actually inherit; like this,
+# they're suboptimal
+setattr(models.Manager, "all_base_class", models.Manager.all)
+setattr(models.Manager, "all", QueryManager.all)

lamindb/_registry.py

@@ -82,7 +82,12 @@ def __init__(orm: Registry, *args, **kwargs):
         validate_required_fields(orm, kwargs)
         from .dev._settings import settings
 
-        if settings.upon_create_search_names:
+        # do not search for names if an id is passed; this is important
+        # e.g. when synching ids from the notebook store to lamindb
+        has_consciously_provided_id = False
+        if "_has_consciously_provided_id" in kwargs:
+            has_consciously_provided_id = kwargs.pop("_has_consciously_provided_id")
+        if settings.upon_create_search_names and not has_consciously_provided_id:
             result = suggest_objects_with_same_name(orm, kwargs)
             if result == "object-with-same-name-exists":
                 if "version" in kwargs:
@@ -341,7 +346,7 @@ def transfer_to_default_db(record: Registry, save: bool = False):
                 record.transform_id = run_context.transform.id
             else:
                 record.transform_id = None
-        if hasattr(record, "storage_id"):
+        if hasattr(record, "storage_id") and record.storage_id is not None:
             record.storage.save()
         record._state.db = "default"
         if save:

lamindb/_storage.py

@@ -1,3 +1,4 @@
+from lamindb_setup.dev._docs import doc_args
 from lamindb_setup.dev.upath import UPath, create_path
 from lnschema_core import Storage
 
@@ -6,4 +7,12 @@ def root_as_path(self) -> UPath:
     return create_path(self.root)
 
 
+@property  # type: ignore
+@doc_args(Storage.path.__doc__)
+def path(self) -> UPath:
+    """{}"""
+    return create_path(self.root)
+
+
 setattr(Storage, "root_as_path", root_as_path)
+setattr(Storage, "path", path)

lamindb/_transform.py

@@ -44,8 +44,14 @@ def __init__(transform: Transform, *args, **kwargs):
         )
         if name is None:
             name = is_new_version_of.name
+
+    # this is only because the user-facing constructor allows passing an id
+    # most others don't
     if id is None:
+        has_consciously_provided_id = False
         id = new_id
+    else:
+        has_consciously_provided_id = True
     super(Transform, transform).__init__(
         id=id,
         name=name,
@@ -54,6 +60,7 @@ def __init__(transform: Transform, *args, **kwargs):
         version=version,
         initial_version_id=initial_version_id,
         reference=reference,
+        _has_consciously_provided_id=has_consciously_provided_id,
     )
 
 

lamindb/dev/_label_manager.py

@@ -127,4 +127,7 @@ class LabelManager:
             labels_list = labels.list()
             for label in labels_list:
                 transfer_to_default_db(label, save=True)
-            getattr(self._host, related_name).add(*labels_list)
+            # this should not occur as file and dataset should have the same attributes
+            # but this might not be true for custom schema
+            if hasattr(self._host, related_name):
+                getattr(self._host, related_name).add(*labels_list)

{lamindb-0.55.0.dist-info → lamindb-0.55.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.55.0
+Version: 0.55.1
 Summary: Open-source data platform for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -8,9 +8,9 @@ Description-Content-Type: text/markdown
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
-Requires-Dist: lnschema_core==0.49.2
-Requires-Dist: lamindb_setup==0.54.2
-Requires-Dist: lamin_utils==0.11.3
+Requires-Dist: lnschema_core==0.50.0
+Requires-Dist: lamindb_setup==0.54.3
+Requires-Dist: lamin_utils==0.11.4
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
@@ -35,7 +35,7 @@ Requires-Dist: pytest-cov ; extra == "dev"
 Requires-Dist: nbproject_test>=0.3.0 ; extra == "dev"
 Requires-Dist: faker-biology ; extra == "dev"
 Requires-Dist: django-schema-graph ; extra == "erdiagram"
-Requires-Dist: readfcs>=1.1.5 ; extra == "fcs"
+Requires-Dist: readfcs>=1.1.7 ; extra == "fcs"
 Requires-Dist: fsspec[gs]==2023.9.0 ; extra == "gcp"
 Requires-Dist: nbproject==0.9.2 ; extra == "jupyter"
 Requires-Dist: nbstripout ; extra == "jupyter"

{lamindb-0.55.0.dist-info → lamindb-0.55.1.dist-info}/RECORD

@@ -1,19 +1,19 @@
-lamindb/__init__.py,sha256=0JimGcpFe3pRFVTj_ZvQLPSveL_oS2Z5pcsnkH1gvpY,2945
-lamindb/_dataset.py,sha256=GINxUXm0KUBK21W6SR8187YiPQBFpK8hZxhlB5mEVxQ,12160
+lamindb/__init__.py,sha256=ZCNDGjrECrHymRvihw53TRk4QYluCJ1URlRRhKLb4vA,2945
+lamindb/_dataset.py,sha256=363qySdhwD9qO9op-fXRuTTflKmCdhGAOvf115gdjXo,13444
 lamindb/_delete.py,sha256=wiYmYnvIEHrDdmw1NiXyfCY9mBt-FI5XNFi5jyR_mkA,1968
 lamindb/_feature.py,sha256=5gsa7zsMVVtm1DID4dF3Vwo5llWyY1dH3Hg5hjaIrQk,5554
 lamindb/_feature_set.py,sha256=YyRLc76piJdh8X6B917cFBiwJTdIxvQo0n5k2-HUGro,9373
-lamindb/_file.py,sha256=JKlhUsLKzwf-dp8wiMTa7HgqfiJDAfU0t3a7u0tE_Kc,35540
-lamindb/_filter.py,sha256=iOrpREyBI8VU8nbZ5a6CH7-q2aw4ZjDGOTUXWdgj0VM,1874
+lamindb/_file.py,sha256=WotCwWGtVYLhyUARgJeInLgRnuKKRmx7E12IxS9Pe20,35548
+lamindb/_filter.py,sha256=06hUpqZTTvXPOfywR0oJ47dd281KvnejRpTvaCAIV74,1515
 lamindb/_from_values.py,sha256=OztkgQpyWy6CSrr5RcKwL1r_DTjCoYWyllR94LLd8qs,11595
 lamindb/_parents.py,sha256=GZ6eUlIDtcAh4dCjBE2CC7KkY62WkKNFP3MMLtBjW44,13048
-lamindb/_query_manager.py,sha256=JFFSGM8nCU4vlCvfnzW61JJedPjlwR-FYf1RAT1nbII,3255
+lamindb/_query_manager.py,sha256=MXueabWHqft7GWNkzmWbhfTqdk-0mKU7nWrhXG6wpYQ,3693
 lamindb/_query_set.py,sha256=Lf7vLvOsEfUWRQ3iImSj4eQPmUK1KCgeoKS_m66Lp7o,10279
-lamindb/_registry.py,sha256=vnMBHhRGFmazybaRrZ8lhRsdszOp_YwkvhQmOhIBNWk,13012
+lamindb/_registry.py,sha256=N-CzfRokoWLl1ive5aYr5mBnUQuddQ1GCjElbeUFNsY,13404
 lamindb/_run.py,sha256=659lqY32GW7F41rFUUo37OftUa38-p8yaV9Z0oF32CE,1120
 lamindb/_save.py,sha256=Lir8uDGS9XuYOaqogpT2EiDrWV_T_G9PbPvckbvUsoE,9474
-lamindb/_storage.py,sha256=vfd1yeGEalVVEJrZQBv57dlAZSHsY1kexRjPFqTxGNg,207
-lamindb/_transform.py,sha256=fWmlLBtBrplka_1SzFojdZ9EHCQ15j2yv4iB2jJAD4U,2235
+lamindb/_storage.py,sha256=HUdXGj4839C606gvxWXo0tDITbtbuyJKOgUPhagYPTI,415
+lamindb/_transform.py,sha256=O9JcOt0Khg9fF_1Rmid03XyxJms6OnxQp8rCAT3ZuDw,2499
 lamindb/_ulabel.py,sha256=lEAENh_dluNkBi8xKUH_CjJNMXldOm2liy6Rg3IH1pE,1900
 lamindb/_utils.py,sha256=LGdiW4k3GClLz65vKAVRkL6Tw-Gkx9DWAdez1jyA5bE,428
 lamindb/_validate.py,sha256=6FQIxE8ZK4HwrrmrmFSjeCFKc-mxaHej5fuBh_4y2Tk,13656
@@ -21,7 +21,7 @@ lamindb/_view.py,sha256=bzx6e-Cif2CmDQkOu6jMrq_d5rsu6g7hhdaK_sYBv_Y,2150
 lamindb/dev/__init__.py,sha256=Ja96dxb0t7raGsCr8QxqCabyEzIxeVGlL_IgmhxdsB8,1010
 lamindb/dev/_data.py,sha256=1uOGPpa1OWIM1QcgpS_M_l35sDC8Fltk-fYf_5cKxUc,13540
 lamindb/dev/_feature_manager.py,sha256=z7uWuSO5fWl0pCKdCsc86rRrcKnAmtx2wAuOTiZn5XE,5869
-lamindb/dev/_label_manager.py,sha256=MxJlB617MCP1k60lQe9rXs9LYdZjrdPSgecCDj47zU4,4686
+lamindb/dev/_label_manager.py,sha256=yW0KgM1WMYe4NOpWNCO5ymw5CY2_UQl3t9tXZER4Gdc,4887
 lamindb/dev/_priors.py,sha256=eSZEEijmeFs3zcrU27r3T2sSGdsK-cvy7vl6ukDYaU8,785
 lamindb/dev/_run_context.py,sha256=XLlEknzOe0Cabi9fvzdQl5UeLoU02fAFkCdB7S2FLZM,19109
 lamindb/dev/_settings.py,sha256=b0uq5eB6Dk7BQjBoK4a-9hy8y1DPsq0rErAQjZ4DibM,3398
@@ -41,8 +41,8 @@ lamindb/dev/storage/file.py,sha256=CrR8dhfqEWPj7jhNj7zujQE1WT4gWGCP0qKPIIta0bg,7
 lamindb/dev/storage/object.py,sha256=KGuOwwYuN2yCJxTXn9v0LanC0fjKwy_62P-WksHcf40,1140
 lamindb/setup/__init__.py,sha256=8-0F2C4Glx23-b8-D_1CBGgRBM5PppVhazhoXZYOLsg,275
 lamindb/setup/dev/__init__.py,sha256=tBty426VGF2PGqqt2XuNU-WgvOrbOp1aZBDowjLuzgA,242
-lamindb-0.55.0.dist-info/entry_points.txt,sha256=MioM8vSpKwXxY3geNBwjo1wnwy1l15WjJYlI3lpKuZI,53
-lamindb-0.55.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-lamindb-0.55.0.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-lamindb-0.55.0.dist-info/METADATA,sha256=MsDbx974Lhn4cAN_OlvzPnZky4Mo6Qpz3CW1DG9KKW4,3022
-lamindb-0.55.0.dist-info/RECORD,,
+lamindb-0.55.1.dist-info/entry_points.txt,sha256=MioM8vSpKwXxY3geNBwjo1wnwy1l15WjJYlI3lpKuZI,53
+lamindb-0.55.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+lamindb-0.55.1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+lamindb-0.55.1.dist-info/METADATA,sha256=E5NRemxlnVm8u55H11hKjH1ezGF6ZYnd4z0zHvpC-Z4,3022
+lamindb-0.55.1.dist-info/RECORD,,