lamindb 0.58.0__py3-none-any.whl → 0.58.1__py3-none-any.whl

lamindb/__init__.py

@@ -53,7 +53,7 @@ Static classes & modules:
 
 """
 
-__version__ = "0.58.0"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.58.1"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os as _os
 

lamindb/_dataset.py

@@ -63,12 +63,15 @@ def __init__(
         kwargs.pop("initial_version_id") if "initial_version_id" in kwargs else None
     )
     version: Optional[str] = kwargs.pop("version") if "version" in kwargs else None
+    visibility: Optional[int] = (
+        kwargs.pop("visibility") if "visibility" in kwargs else 0
+    )
     feature_sets: Dict[str, FeatureSet] = (
         kwargs.pop("feature_sets") if "feature_sets" in kwargs else {}
     )
     if not len(kwargs) == 0:
         raise ValueError(
-            f"Only data, name, run, description, reference, reference_type can be passed, you passed: {kwargs}"  # noqa
+            f"Only data, name, run, description, reference, reference_type, visibility can be passed, you passed: {kwargs}"  # noqa
         )
 
     if is_new_version_of is None:
@@ -145,6 +148,7 @@ def __init__(
             hash, feature_sets = from_files(files)  # type: ignore
         else:
             raise ValueError("Only DataFrame, AnnData and iterable of File is allowed")
+    # we ignore datasets in trash containing the same hash
     existing_dataset = Dataset.filter(hash=hash).one_or_none()
     if existing_dataset is not None:
         logger.warning(f"returning existing dataset with same hash: {existing_dataset}")
@@ -169,6 +173,7 @@ def __init__(
             run=run,
             version=version,
             initial_version_id=initial_version_id,
+            visibility=visibility,
             **kwargs,
         )
     dataset._files = files
@@ -344,12 +349,12 @@ def load(
         # because we're tracking data flow on the dataset-level, here, we don't
         # want to track it on the file-level
         objects = [file.load(is_run_input=False) for file in all_files]
-        file_ids = [file.id for file in all_files]
+        file_uids = [file.uid for file in all_files]
         if isinstance(objects[0], pd.DataFrame):
             concat_object = pd.concat(objects, join=join)
         elif isinstance(objects[0], ad.AnnData):
             concat_object = ad.concat(
-                objects, join=join, label="file_id", keys=file_ids
+                objects, join=join, label="file_uid", keys=file_uids
             )
         # only call it here because there might be errors during concat
         _track_run_input(self, is_run_input)
@@ -357,10 +362,32 @@ def load(
 
 
 # docstring handled through attach_func_to_class_method
-def delete(self, storage: Optional[bool] = None) -> None:
-    super(Dataset, self).delete()
+def delete(
+    self, permanent: Optional[bool] = None, storage: Optional[bool] = None
+) -> None:
+    # change visibility to 2 (trash)
+    if self.visibility < 2 and permanent is not True:
+        self.visibility = 2
+        self.save()
+        if self.file is not None:
+            self.file.visibility = 2
+            self.file.save()
+        return
+
+    # permanent delete
+    if permanent is None:
+        response = input(
+            "File record is already in trash! Are you sure to delete it from your"
+            " database? (y/n) You can't undo this action."
+        )
+        delete_record = response == "y"
+    else:
+        delete_record = permanent
+
+    if delete_record:
+        super(Dataset, self).delete()
     if self.file is not None:
-        self.file.delete(storage=storage)
+        self.file.delete(permanent=permanent, storage=storage)
 
 
 # docstring handled through attach_func_to_class_method
@@ -384,6 +411,15 @@ def path(self) -> Union[Path, UPath]:
     return self.storage.path
 
 
+# docstring handled through attach_func_to_class_method
+def restore(self) -> None:
+    self.visibility = 0
+    self.save()
+    if self.file is not None:
+        self.file.visibility = 0
+        self.file.save()
+
+
 METHOD_NAMES = [
     "__init__",
     "from_anndata",
@@ -392,6 +428,7 @@ METHOD_NAMES = [
     "load",
     "delete",
     "save",
+    "restore",
 ]
 
 if _TESTING:

lamindb/_file.py

@@ -179,6 +179,7 @@ def get_hash(
         hash, hash_type = hash_file(filepath)
     if not check_hash:
         return hash, hash_type
+    # we ignore datasets in trash containing the same hash
     result = File.filter(hash=hash).list()
     if len(result) > 0:
         if settings.upon_file_create_if_hash_exists == "error":
@@ -454,6 +455,9 @@ def __init__(file: File, *args, **kwargs):
         kwargs.pop("initial_version_id") if "initial_version_id" in kwargs else None
     )
     version: Optional[str] = kwargs.pop("version") if "version" in kwargs else None
+    visibility: Optional[int] = (
+        kwargs.pop("visibility") if "visibility" in kwargs else 0
+    )
     format = kwargs.pop("format") if "format" in kwargs else None
     log_hint = kwargs.pop("log_hint") if "log_hint" in kwargs else True
     skip_check_exists = (
@@ -462,8 +466,8 @@ def __init__(file: File, *args, **kwargs):
 
     if not len(kwargs) == 0:
         raise ValueError(
-            "Only data, key, run, description, version, is_new_version_of can be"
-            f" passed, you passed: {kwargs}"
+            "Only data, key, run, description, version, is_new_version_of, visibility"
+            f" can be passed, you passed: {kwargs}"
         )
 
     if is_new_version_of is None:
@@ -523,6 +527,7 @@ def __init__(file: File, *args, **kwargs):
     kwargs["initial_version_id"] = initial_version_id
     kwargs["version"] = version
     kwargs["description"] = description
+    kwargs["visibility"] = visibility
     # this check needs to come down here because key might be populated from an
     # existing file path during get_file_kwargs_from_data()
     if (
@@ -820,23 +825,49 @@ def stage(self, is_run_input: Optional[bool] = None) -> Path:
 
 
 # docstring handled through attach_func_to_class_method
-def delete(self, storage: Optional[bool] = None) -> None:
-    if storage is None:
-        response = input(f"Are you sure you want to delete {self} from storage? (y/n)")
-        delete_in_storage = response == "y"
+def delete(
+    self, permanent: Optional[bool] = None, storage: Optional[bool] = None
+) -> None:
+    # change visibility to 2 (trash)
+    if self.visibility < 2 and permanent is not True:
+        self.visibility = 2
+        self.save()
+        return
+
+    # if the file is already in the trash
+    # permanent delete skips the trash
+    if permanent is None:
+        response = input(
+            "File record is already in trash! Are you sure to delete it from your"
+            " database? (y/n) You can't undo this action."
+        )
+        delete_record = response == "y"
     else:
-        delete_in_storage = storage
+        delete_record = permanent
 
     # need to grab file path before deletion
     filepath = self.path
+
     # only delete in storage if DB delete is successful
     # DB delete might error because of a foreign key constraint violated etc.
-    self._delete_skip_storage()
-    # we don't yet have any way to bring back the deleted metadata record
-    # in case the storage deletion fails - this is important for ACID down the road
-    if delete_in_storage:
-        delete_storage(filepath)
-        logger.success(f"deleted stored object {colors.yellow(f'{filepath}')}")
+    if delete_record:
+        self._delete_skip_storage()
+        if self.key is None:
+            delete_in_storage = True
+        else:
+            if storage is None:
+                response = input(
+                    f"Are you sure to delete {filepath}? (y/n)  You can't undo this"
+                    " action."
+                )
+                delete_in_storage = response == "y"
+            else:
+                delete_in_storage = storage
+        # we don't yet have any way to bring back the deleted metadata record
+        # in case storage deletion fails - this is important for ACID down the road
+        if delete_in_storage:
+            delete_storage(filepath)
+            logger.success(f"deleted {colors.yellow(f'{filepath}')}")
 
 
 def _delete_skip_storage(file, *args, **kwargs) -> None:
@@ -961,6 +992,12 @@ def view_tree(
     )
 
 
+# docstring handled through attach_func_to_class_method
+def restore(self) -> None:
+    self.visibility = 0
+    self.save()
+
+
 METHOD_NAMES = [
     "__init__",
     "from_anndata",
@@ -973,6 +1010,7 @@ METHOD_NAMES = [
     "replace",
     "from_dir",
     "view_tree",
+    "restore",
 ]
 
 if _TESTING:

lamindb/_filter.py

@@ -42,6 +42,18 @@ def filter(Registry: Type[Registry], using: str = None, **expressions) -> QueryS
             id=UUID(instance_result["id"]),
         )
         add_db_connection(isettings, using)
+
+    if Registry.__name__ in {"File", "Dataset"}:
+        # visibility is set to <2 by default
+        if not any([e.startswith("visibility") for e in expressions]):
+            expressions["visibility__lt"] = 2
+        # if visibility is None, will not apply any filter for visibility
+        elif "visibility" in expressions:
+            if expressions["visibility"] is None:
+                expressions.pop("visibility")
+            elif expressions["visibility"] == "default":
+                expressions.pop("visibility")
+                expressions["visibility__lt"] = 2
     qs = QuerySet(model=Registry, using=using)
     if len(expressions) > 0:
         return qs.filter(**expressions)

lamindb/_query_set.py

@@ -210,13 +210,11 @@ class QuerySet(models.QuerySet):
         return _search(cls=self, string=string, **kwargs)
 
     @doc_args(Registry.lookup.__doc__)
-    def lookup(
-        self, field: Optional[StrField] = None, return_field: Optional[StrField] = None
-    ) -> NamedTuple:
+    def lookup(self, field: Optional[StrField] = None, **kwargs) -> NamedTuple:
         """{}"""
         from ._registry import _lookup
 
-        return _lookup(cls=self, field=field, return_field=return_field)
+        return _lookup(cls=self, field=field, **kwargs)
 
     @doc_args(CanValidate.validate.__doc__)
     def validate(

lamindb/_registry.py

@@ -147,8 +147,9 @@ def _search(
     return_queryset: bool = False,
     case_sensitive: bool = False,
     synonyms_field: Optional[StrField] = "synonyms",
+    **expressions,
 ) -> Union["pd.DataFrame", "QuerySet"]:
-    queryset = _queryset(cls)
+    queryset = _queryset(cls, **expressions)
     orm = queryset.model
 
     def _search_single_field(
@@ -229,6 +230,7 @@ def search(
     return_queryset: bool = False,
     case_sensitive: bool = False,
     synonyms_field: Optional[StrField] = "synonyms",
+    **expressions,
 ) -> Union["pd.DataFrame", "QuerySet"]:
     """{}"""
     return _search(
@@ -239,14 +241,18 @@ def search(
         limit=limit,
         case_sensitive=case_sensitive,
         synonyms_field=synonyms_field,
+        **expressions,
     )
 
 
 def _lookup(
-    cls, field: Optional[StrField] = None, return_field: Optional[StrField] = None
+    cls,
+    field: Optional[StrField] = None,
+    return_field: Optional[StrField] = None,
+    **expressions,
 ) -> NamedTuple:
     """{}"""
-    queryset = _queryset(cls)
+    queryset = _queryset(cls, **expressions)
     field = get_default_str_field(orm=queryset.model, field=field)
 
     return Lookup(
@@ -264,10 +270,13 @@ def _lookup(
 @classmethod  # type: ignore
 @doc_args(Registry.lookup.__doc__)
 def lookup(
-    cls, field: Optional[StrField] = None, return_field: Optional[StrField] = None
+    cls,
+    field: Optional[StrField] = None,
+    return_field: Optional[StrField] = None,
+    **expressions,
 ) -> NamedTuple:
     """{}"""
-    return _lookup(cls=cls, field=field, return_field=return_field)
+    return _lookup(cls=cls, field=field, return_field=return_field, **expressions)
 
 
 def get_default_str_field(
@@ -316,8 +325,12 @@ def get_default_str_field(
     return field
 
 
-def _queryset(cls: Union[Registry, QuerySet, Manager]) -> QuerySet:
-    queryset = cls.all() if isinstance(cls, QuerySet) else cls.objects.all()
+def _queryset(cls: Union[Registry, QuerySet, Manager], **expressions) -> QuerySet:
+    queryset = (
+        cls.filter(**expressions).all()
+        if isinstance(cls, QuerySet)
+        else cls.filter(**expressions).all()
+    )
     return queryset
 
 

lamindb/_save.py

@@ -252,18 +252,20 @@ def upload_data_object(file) -> None:
     """Store and add file and its linked entries."""
     # do NOT hand-craft the storage key!
     file_storage_key = auto_storage_key_from_file(file)
-    msg = f"storing file '{file.id}' at '{file_storage_key}'"
+    storage_path = lamindb_setup.settings.instance.storage.key_to_filepath(
+        file_storage_key
+    )
+    msg = f"storing file '{file.uid}' at '{storage_path}'"
     if (
         file.suffix in {".zarr", ".zrad"}
         and hasattr(file, "_memory_rep")
         and file._memory_rep is not None
     ):
         logger.save(msg)
-        storagepath = lamindb_setup.settings.storage.key_to_filepath(file_storage_key)
         print_progress = partial(
             print_hook, filepath=file_storage_key, action="uploading"
         )
-        write_adata_zarr(file._memory_rep, storagepath, callback=print_progress)
+        write_adata_zarr(file._memory_rep, storage_path, callback=print_progress)
     elif hasattr(file, "_to_store") and file._to_store:
         logger.save(msg)
         store_object(file._local_filepath, file_storage_key)

{lamindb-0.58.0.dist-info → lamindb-0.58.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.58.0
+Version: 0.58.1
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -8,8 +8,8 @@ Description-Content-Type: text/markdown
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
-Requires-Dist: lnschema_core==0.52.1
-Requires-Dist: lamindb_setup==0.55.6
+Requires-Dist: lnschema_core==0.53.0
+Requires-Dist: lamindb_setup==0.56.3
 Requires-Dist: lamin_utils==0.11.5
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow

{lamindb-0.58.0.dist-info → lamindb-0.58.1.dist-info}/RECORD

@@ -1,17 +1,17 @@
-lamindb/__init__.py,sha256=BWW5E_HjLH9kJ3Fo6sUfWV2efj0pytTAYmPxnJ-iheA,2870
-lamindb/_dataset.py,sha256=6CJHTiwe1lWLUfPHQzJpq-hj8zRjNTCWLkPmrtxpP7Q,14712
+lamindb/__init__.py,sha256=0YGJThA1KvrX4UlxWsOrnuNKxxjEe1FVhKvdZ_8KWTg,2870
+lamindb/_dataset.py,sha256=GLGtwbZLlSjy2HtJsjHgRDTOO0u0PwuarRE5qp-rGUA,15810
 lamindb/_delete.py,sha256=wiYmYnvIEHrDdmw1NiXyfCY9mBt-FI5XNFi5jyR_mkA,1968
 lamindb/_feature.py,sha256=5gsa7zsMVVtm1DID4dF3Vwo5llWyY1dH3Hg5hjaIrQk,5554
 lamindb/_feature_set.py,sha256=G63pwauDQ7jg4ydFCQLhu-lgO6tm56iQwUdRuNHeKHY,9233
-lamindb/_file.py,sha256=V85tryLD0HsZiPQV4KNmcs3Gdi0qf8dZu27rl2G9D7E,36218
-lamindb/_filter.py,sha256=fNvPbLeOxYzvNKPcFYiFz3P7bkD5_84Xh8HHAoLNdas,1716
+lamindb/_file.py,sha256=9McSL-DuhGDihfusIX1UKZ195HwhXohlWhJHV9Ki0c4,37358
+lamindb/_filter.py,sha256=JrE4tdExNkOmNf0_tnO3vo-W3tecsH6ZB74gLO_fvKE,2293
 lamindb/_from_values.py,sha256=GitpmKOqV6YHJggaCnJgGsRIHI_bnuLRVE2oo9W-SgE,11613
 lamindb/_parents.py,sha256=VT_gtomf1Erd_AKLVd1uLwigeDqMHtcaAbma3_AbQAw,13408
 lamindb/_query_manager.py,sha256=MXueabWHqft7GWNkzmWbhfTqdk-0mKU7nWrhXG6wpYQ,3693
-lamindb/_query_set.py,sha256=Lf7vLvOsEfUWRQ3iImSj4eQPmUK1KCgeoKS_m66Lp7o,10279
-lamindb/_registry.py,sha256=_pdlEvAtemiQCzpK2s14MsTKkLqE6ORDjhDs7ABs4i4,14893
+lamindb/_query_set.py,sha256=1vjTLkCCrs1GiS2KTyqmSgVRSx966UsMhApXbW7GgI0,10217
+lamindb/_registry.py,sha256=lUnHCeDDOw4mlak0_Q_EbQU1_qDrsE23l7IEbeoaV8w,15138
 lamindb/_run.py,sha256=659lqY32GW7F41rFUUo37OftUa38-p8yaV9Z0oF32CE,1120
-lamindb/_save.py,sha256=m6l5mMsxlrmlkdWhfjbwOtZ3haGEYyg63QcPG8twTMQ,10136
+lamindb/_save.py,sha256=hL34zgm-L3MFfi6P9O0AzeptFHtEnHdKheJqdOlGDM4,10154
 lamindb/_storage.py,sha256=HUdXGj4839C606gvxWXo0tDITbtbuyJKOgUPhagYPTI,415
 lamindb/_transform.py,sha256=87yUTz0RndJ_C98tBt4t2SPw8fksRgqJKwCQG_H40Kk,2515
 lamindb/_ulabel.py,sha256=lEAENh_dluNkBi8xKUH_CjJNMXldOm2liy6Rg3IH1pE,1900
@@ -41,8 +41,8 @@ lamindb/dev/storage/file.py,sha256=xfeU8X1ty80-PhnHOpupBJfibZKhp6MPLA2IjYdTBoY,7
 lamindb/dev/storage/object.py,sha256=KGuOwwYuN2yCJxTXn9v0LanC0fjKwy_62P-WksHcf40,1140
 lamindb/setup/__init__.py,sha256=8-0F2C4Glx23-b8-D_1CBGgRBM5PppVhazhoXZYOLsg,275
 lamindb/setup/dev/__init__.py,sha256=tBty426VGF2PGqqt2XuNU-WgvOrbOp1aZBDowjLuzgA,242
-lamindb-0.58.0.dist-info/entry_points.txt,sha256=MioM8vSpKwXxY3geNBwjo1wnwy1l15WjJYlI3lpKuZI,53
-lamindb-0.58.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-lamindb-0.58.0.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-lamindb-0.58.0.dist-info/METADATA,sha256=thZUko5v-kPa9zVxlYdnMZZrh2CXnZrsrC8o0z_Dsts,3030
-lamindb-0.58.0.dist-info/RECORD,,
+lamindb-0.58.1.dist-info/entry_points.txt,sha256=MioM8vSpKwXxY3geNBwjo1wnwy1l15WjJYlI3lpKuZI,53
+lamindb-0.58.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+lamindb-0.58.1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+lamindb-0.58.1.dist-info/METADATA,sha256=vsEe2aNzGIKTdXiRH07Cr6wZuFn5COOO9U1DuZRkBRM,3030
+lamindb-0.58.1.dist-info/RECORD,,