lamindb 0.59.4__tar.gz → 0.60.0__tar.gz

{lamindb-0.59.4 → lamindb-0.60.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.59.4
+Version: 0.60.0
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -8,8 +8,8 @@ Description-Content-Type: text/markdown
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
-Requires-Dist: lnschema_core==0.54.0
-Requires-Dist: lamindb_setup==0.57.2
+Requires-Dist: lnschema_core==0.54.1
+Requires-Dist: lamindb_setup==0.58.0
 Requires-Dist: lamin_utils==0.11.7
 Requires-Dist: lamin_cli==0.1.2
 Requires-Dist: rapidfuzz
@@ -27,7 +27,7 @@ Requires-Dist: boto3==1.28.17 ; extra == "aws"
 Requires-Dist: aiobotocore==2.5.4 ; extra == "aws"
 Requires-Dist: fsspec[s3]==2023.9.0 ; extra == "aws"
 Requires-Dist: s3fs>=2023.1.0 ; extra == "aws"
-Requires-Dist: lnschema_bionty==0.34.0 ; extra == "bionty"
+Requires-Dist: lnschema_bionty==0.34.2 ; extra == "bionty"
 Requires-Dist: pandas<2 ; extra == "dev"
 Requires-Dist: pre-commit ; extra == "dev"
 Requires-Dist: nox ; extra == "dev"

{lamindb-0.59.4 → lamindb-0.60.0}/docs/changelog.md

@@ -1,14 +1,20 @@
 # Changelog
 
+
 ```{eval-rst}
 .. role:: small
 ```
 
-## 0.59
+## 0.60
 
-### 0.59.4 {small}`2023-11-05`
+### 0.60.0 {small}`2023-11-09`
 
-- ⬆️ Upgrade lamindb-setup [PR1281](https://github.com/laminlabs/lamindb/pull/1281) [@fredericenard](https://github.com/fredericenard)
+- 🚸 Enable to pass metadata to dataset [PR1289](https://github.com/laminlabs/lamindb/pull/1289) [@falexwolf](https://github.com/falexwolf)
+- 🚚 Move `view_tree` to `UPath` [PR1288](https://github.com/laminlabs/lamindb/pull/1288) [@falexwolf](https://github.com/falexwolf)
+- ♻️ Move `UPath` to root-level API [PR1286](https://github.com/laminlabs/lamindb/pull/1286) [@falexwolf](https://github.com/falexwolf)
+- 🚸 Enable to add labels without a feature [PR1285](https://github.com/laminlabs/lamindb/pull/1285) [@sunnyosun](https://github.com/sunnyosun)
+
+## 0.59
 
 ### 0.59.3 {small}`2023-11-04`
 

{lamindb-0.59.4 → lamindb-0.60.0}/docs/tutorial.ipynb

@@ -232,7 +232,7 @@
    },
    "outputs": [],
    "source": [
-    "ln.File.view_tree(\"s3://lamindb-dev-datasets/iris_studies\")"
+    "ln.UPath(\"s3://lamindb-dev-datasets/iris_studies\").view_tree()"
    ]
   },
   {

{lamindb-0.59.4 → lamindb-0.60.0}/lamindb/__init__.py

@@ -1,11 +1,6 @@
 """A data framework for biology.
 
-We assume that data is stored as files or in array formats like parquet, zarr,
-HDF5, TileDB or DuckDB.
-
-LaminDB helps you manage these data with registries for metadata.
-
-The two most important are:
+LaminDB helps you manage data with registries for metadata:
 
 .. autosummary::
    :toctree: .
@@ -13,7 +8,7 @@ The two most important are:
    File
    Dataset
 
-Four registries track provenance of data batches:
+Registries to track provenance:
 
 .. autosummary::
    :toctree: .
@@ -23,7 +18,7 @@ Four registries track provenance of data batches:
    User
    Storage
 
-Four registries validate & contextualize data batches:
+Registries to validate & contextualize:
 
 .. autosummary::
    :toctree: .
@@ -33,7 +28,14 @@ Four registries validate & contextualize data batches:
    FeatureSet
    Modality
 
-Functional tools:
+You can also access data directly via paths:
+
+.. autosummary::
+   :toctree: .
+
+   UPath
+
+Functions:
 
 .. autosummary::
    :toctree: .
@@ -42,7 +44,7 @@ Functional tools:
    view
    save
 
-Static classes & modules:
+Modules & settings:
 
 .. autosummary::
    :toctree: .
@@ -53,7 +55,7 @@ Static classes & modules:
 
 """
 
-__version__ = "0.59.4"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.60.0"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os as _os
 
@@ -64,6 +66,7 @@ from lamin_utils import py_version_warning as _py_version_warning
 from lamindb_setup import _check_instance_setup
 from lamindb_setup._check_instance_setup import _INSTANCE_NOT_SETUP_WARNING
 from lamindb_setup._init_instance import reload_schema_modules as _reload_schema_modules
+from lamindb_setup.dev.upath import UPath
 
 _py_version_warning("3.8", "3.11")
 

{lamindb-0.59.4 → lamindb-0.60.0}/lamindb/_dataset.py

@@ -45,6 +45,7 @@ def __init__(
     data: Union[pd.DataFrame, ad.AnnData, File, Iterable[File]] = (
         kwargs.pop("data") if len(args) == 0 else args[0]
     )
+    meta: Optional[str] = kwargs.pop("meta") if "meta" in kwargs else None
     name: Optional[str] = kwargs.pop("name") if "name" in kwargs else None
     description: Optional[str] = (
         kwargs.pop("description") if "description" in kwargs else None
@@ -92,9 +93,26 @@ def __init__(
             )
 
     run = get_run(run)
+    data_init_complete = False
     # there are exactly 3 ways of creating a Dataset object right now
     # using exactly one file or using more than one file
     # init file
+    # init from directory or bucket
+    if isinstance(data, (str, Path, UPath)):
+        file = None
+        files = None
+        upath = UPath(data)
+        if not upath.is_dir:
+            raise ValueError(f"Can only pass buckets or directories, not {data}")
+        upath_str = upath.as_posix().rstrip("/")
+        region = get_storage_region(upath_str)
+        storage_settings = StorageSettings(upath_str, region)
+        storage = register_storage(storage_settings)
+        hash = None
+        data_init_complete = True
+    # now handle the metadata
+    if isinstance(meta, (pd.DataFrame, ad.AnnData, File)):
+        data = meta
     if isinstance(data, (pd.DataFrame, ad.AnnData, File)):
         files = None
         storage = None
@@ -126,28 +144,19 @@ def __init__(
             file.description = f"See dataset {provisional_uid}"  # type: ignore
         file._feature_sets = feature_sets
         storage = None
-    # init from directory or bucket
-    elif isinstance(data, (str, Path, UPath)):
-        file = None
-        files = None
-        upath = UPath(data)
-        if not upath.is_dir:
-            raise ValueError(f"Can only pass buckets or directories, not {data}")
-        upath_str = upath.as_posix().rstrip("/")
-        region = get_storage_region(upath_str)
-        storage_settings = StorageSettings(upath_str, region)
-        storage = register_storage(storage_settings)
-        hash = None
-    # init files
-    else:
+        data_init_complete = True
+    if not data_init_complete:
         file = None
         storage = None
         if hasattr(data, "__getitem__"):
             assert isinstance(data[0], File)  # type: ignore
             files = data
             hash, feature_sets = from_files(files)  # type: ignore
+            data_init_complete = True
         else:
-            raise ValueError("Only DataFrame, AnnData and iterable of File is allowed")
+            raise ValueError(
+                "Only DataFrame, AnnData, folder or list of File is allowed."
+            )
     # we ignore datasets in trash containing the same hash
     existing_dataset = Dataset.filter(hash=hash).one_or_none()
     if existing_dataset is not None:

{lamindb-0.59.4 → lamindb-0.60.0}/lamindb/_file.py

@@ -1,7 +1,5 @@
-from collections import defaultdict
-from itertools import islice
 from pathlib import Path, PurePath, PurePosixPath
-from typing import Any, List, Optional, Set, Tuple, Union
+from typing import Any, List, Optional, Tuple, Union
 
 import anndata as ad
 import fsspec
@@ -14,7 +12,7 @@ from lamindb_setup._init_instance import register_storage
 from lamindb_setup.dev import StorageSettings
 from lamindb_setup.dev._docs import doc_args
 from lamindb_setup.dev._hub_utils import get_storage_region
-from lamindb_setup.dev.upath import create_path
+from lamindb_setup.dev.upath import create_path, extract_suffix_from_path
 from lnschema_core import Feature, FeatureSet, File, Modality, Run, Storage
 from lnschema_core.types import AnnDataLike, DataLike, FieldAttr, PathLike
 
@@ -36,7 +34,6 @@ from lamindb.dev.storage.file import (
     ProgressCallback,
     auto_storage_key_from_file,
     auto_storage_key_from_id_suffix,
-    extract_suffix_from_path,
     filepath_from_file,
 )
 from lamindb.dev.versioning import get_ids_from_old_version, init_uid
@@ -928,7 +925,6 @@ def path(self) -> Union[Path, UPath]:
     return filepath_from_file(self)
 
 
-# adapted from: https://stackoverflow.com/questions/9727673
 @classmethod  # type: ignore
 @doc_args(File.view_tree.__doc__)
 def view_tree(
@@ -940,82 +936,18 @@ def view_tree(
     length_limit: int = 1000,
 ) -> None:
     """{}"""
-    space = "    "
-    branch = "│   "
-    tee = "├── "
-    last = "└── "
-    max_files_per_dir_per_type = 7
-
-    if path is None:
-        dir_path = settings.storage
-    else:
-        dir_path = create_path(path)  # returns Path for local
-    n_files = 0
-    n_directories = 0
-
-    # by default only including registered files
-    # need a flag and a proper implementation
-    registered_paths: Set[Any] = set()
-    registered_dirs: Set[Any] = set()
+    logger.warning("Deprecated: Please use UPath.view_tree()")
+    include_paths = None
     if path is None:
-        registered_paths = {
+        path = settings.storage
+        include_paths = {
             file.path for file in cls.filter(storage_id=setup_settings.storage.id).all()
         }
-        registered_dirs = {d for p in registered_paths for d in p.parents}
-    suffixes = set()
-
-    def inner(dir_path: Union[Path, UPath], prefix: str = "", level=-1):
-        nonlocal n_files, n_directories, suffixes
-        if not level:
-            return  # 0, stop iterating
-        stripped_dir_path = dir_path.as_posix().rstrip("/")
-        # do not iterate through zarr directories
-        if stripped_dir_path.endswith((".zarr", ".zrad")):
-            return
-        # this is needed so that the passed folder is not listed
-        contents = [
-            i
-            for i in dir_path.iterdir()
-            if i.as_posix().rstrip("/") != stripped_dir_path
-        ]
-        if limit_to_directories:
-            contents = [d for d in contents if d.is_dir()]
-        pointers = [tee] * (len(contents) - 1) + [last]
-        n_files_per_dir_per_type = defaultdict(lambda: 0)  # type: ignore
-        for pointer, path in zip(pointers, contents):
-            if path.is_dir():
-                if registered_dirs and path not in registered_dirs:
-                    continue
-                yield prefix + pointer + path.name
-                n_directories += 1
-                n_files_per_dir_per_type = defaultdict(lambda: 0)
-                extension = branch if pointer == tee else space
-                yield from inner(path, prefix=prefix + extension, level=level - 1)
-            elif not limit_to_directories:
-                if registered_paths and path not in registered_paths:
-                    continue
-                suffix = extract_suffix_from_path(path)
-                suffixes.add(suffix)
-                n_files_per_dir_per_type[suffix] += 1
-                n_files += 1
-                if n_files_per_dir_per_type[suffix] == max_files_per_dir_per_type:
-                    yield prefix + "..."
-                elif n_files_per_dir_per_type[suffix] > max_files_per_dir_per_type:
-                    continue
-                else:
-                    yield prefix + pointer + path.name
-
-    folder_tree = ""
-    iterator = inner(dir_path, level=level)
-    for line in islice(iterator, length_limit):
-        folder_tree += f"\n{line}"
-    if next(iterator, None):
-        folder_tree += f"... length_limit, {length_limit}, reached, counted:"
-    directory_info = "directory" if n_directories == 1 else "directories"
-    display_suffixes = ", ".join([f"{suffix!r}" for suffix in suffixes])
-    logger.print(
-        f"{dir_path.name} ({n_directories} sub-{directory_info} & {n_files} files with"
-        f" suffixes {display_suffixes}): {folder_tree}"
+    UPath(path).view_tree(
+        level=level,
+        limit_to_directories=limit_to_directories,
+        length_limit=length_limit,
+        include_paths=include_paths,
     )
 
 

{lamindb-0.59.4 → lamindb-0.60.0}/lamindb/_view.py

@@ -11,15 +11,15 @@ from lnschema_core import Registry
 
 def view(
     n: int = 7, schema: Optional[str] = None, registries: Optional[List[str]] = None
-):
-    """View data.
+) -> None:
+    """View latest metadata state.
 
     Args:
-        n: Display the last `n` rows of a table.
+        n: Display the last `n` rows of a registry.
         schema: Schema module to view. Default's to
             `None` and displays all schema modules.
         registries: List of Registry names. Defaults to
-            `None` and lists all ORMs.
+            `None` and lists all registries.
 
     Examples:
         >>> ln.view()

{lamindb-0.59.4 → lamindb-0.60.0}/lamindb/dev/_data.py

@@ -19,7 +19,10 @@ from lnschema_core.types import StrField
 
 from lamindb.dev._settings import settings
 
-from .._feature_set import dict_schema_name_to_model_name
+from .._feature_set import (
+    dict_related_model_to_related_name,
+    dict_schema_name_to_model_name,
+)
 from .._parents import view_flow
 from .._query_set import QuerySet
 from ._feature_manager import (
@@ -202,19 +205,29 @@ def get_labels(
 def add_labels(
     self,
     records: Union[Registry, List[Registry], QuerySet, Iterable],
-    feature: Feature,
+    feature: Optional[Feature] = None,
     *,
     field: Optional[StrField] = None,
 ) -> None:
     """{}"""
+    if self._state.adding:
+        raise ValueError("Please save the file/dataset before adding a label!")
+
     if isinstance(records, (QuerySet, QuerySet.__base__)):  # need to have both
         records = records.list()
     if isinstance(records, (str, Registry)):
         records = [records]
     if not isinstance(records, List):  # avoids warning for pd Series
         records = list(records)
+    # create records from values
     if isinstance(records[0], str):  # type: ignore
         records_validated = []
+        # feature is needed if we want to create records from values
+        if feature is None:
+            raise ValueError(
+                "Please pass a feature, e.g., via: label = ln.ULabel(name='my_label',"
+                " feature=ln.Feature(name='my_feature'))"
+            )
         if feature.registries is not None:
             orm_dict = dict_schema_name_to_model_name(File)
             for reg in feature.registries.split("|"):
@@ -231,80 +244,98 @@ def add_labels(
             )
         records = records_validated
 
-    if self._state.adding:
-        raise ValueError("Please save the file/dataset before adding a label!")
     for record in records:
         if record._state.adding:
             raise ValidationError(
                 f"{record} not validated. If it looks correct: record.save()"
             )
-    validate_feature(feature, records)  # type:ignore
-    records_by_registry = defaultdict(list)
-    for record in records:
-        records_by_registry[record.__class__.__get_name_with_schema__()].append(record)
-    for registry_name, records in records_by_registry.items():
-        getattr(self, self.features._accessor_by_orm[registry_name]).add(
-            *records, through_defaults={"feature_id": feature.id}
-        )
-    feature_set_links = get_feature_set_links(self)
-    feature_set_ids = [link.feature_set_id for link in feature_set_links.all()]
-    # get all linked features of type Feature
-    feature_sets = FeatureSet.filter(id__in=feature_set_ids).all()
-    linked_features_by_slot = {
-        feature_set_links.filter(feature_set_id=feature_set.id)
-        .one()
-        .slot: feature_set.features.all()
-        for feature_set in feature_sets
-        if "core.Feature" == feature_set.registry
-    }
-    for registry_name, records in records_by_registry.items():
-        msg = ""
-        if feature.registries is None or registry_name not in feature.registries:
+
+    if feature is None:
+        d = dict_related_model_to_related_name(self.__class__)
+        # strategy: group records by registry to reduce number of transactions
+        records_by_related_name: Dict = {}
+        for record in records:
+            related_name = d.get(record.__class__.__get_name_with_schema__())
+            if related_name is None:
+                raise ValueError(f"Can't add labels to {record.__class__} record!")
+            if related_name not in records_by_related_name:
+                records_by_related_name[related_name] = []
+            records_by_related_name[related_name].append(record)
+        for related_name, records in records_by_related_name.items():
+            print(related_name, records)
+            getattr(self, related_name).add(*records)
+    else:
+        validate_feature(feature, records)  # type:ignore
+        records_by_registry = defaultdict(list)
+        for record in records:
+            records_by_registry[record.__class__.__get_name_with_schema__()].append(
+                record
+            )
+        for registry_name, records in records_by_registry.items():
+            getattr(self, self.features._accessor_by_orm[registry_name]).add(
+                *records, through_defaults={"feature_id": feature.id}
+            )
+        feature_set_links = get_feature_set_links(self)
+        feature_set_ids = [link.feature_set_id for link in feature_set_links.all()]
+        # get all linked features of type Feature
+        feature_sets = FeatureSet.filter(id__in=feature_set_ids).all()
+        linked_features_by_slot = {
+            feature_set_links.filter(feature_set_id=feature_set.id)
+            .one()
+            .slot: feature_set.features.all()
+            for feature_set in feature_sets
+            if "core.Feature" == feature_set.registry
+        }
+        for registry_name, records in records_by_registry.items():
+            msg = ""
+            if feature.registries is None or registry_name not in feature.registries:
+                if len(msg) > 0:
+                    msg += ", "
+                msg += f"linked feature '{feature.name}' to registry '{registry_name}'"
+                if feature.registries is None:
+                    feature.registries = registry_name
+                elif registry_name not in feature.registries:
+                    feature.registries += f"|{registry_name}"
+                feature.save()
             if len(msg) > 0:
-                msg += ", "
-            msg += f"linked feature '{feature.name}' to registry '{registry_name}'"
-            if feature.registries is None:
-                feature.registries = registry_name
-            elif registry_name not in feature.registries:
-                feature.registries += f"|{registry_name}"
-            feature.save()
-        if len(msg) > 0:
-            logger.save(msg)
-        # check whether we have to update the feature set that manages labels
-        # (Feature) to account for a new feature
-        found_feature = False
-        for _, linked_features in linked_features_by_slot.items():
-            if feature in linked_features:
-                found_feature = True
-        if not found_feature:
-            if "external" in linked_features_by_slot:
-                feature_set = self.features._feature_set_by_slot["external"]
-                features_list = feature_set.features.list()
-            else:
-                features_list = []
-            features_list.append(feature)
-            feature_set = FeatureSet(features_list, modality=priors.modalities.meta)
-            feature_set.save()
-            if "external" in linked_features_by_slot:
-                old_feature_set_link = feature_set_links.filter(slot="external").one()
-                old_feature_set_link.delete()
-                remaining_links = self.feature_sets.through.objects.filter(
-                    feature_set_id=feature_set.id
-                ).all()
-                if len(remaining_links) == 0:
-                    old_feature_set = FeatureSet.filter(
-                        id=old_feature_set_link.feature_set_id
+                logger.save(msg)
+            # check whether we have to update the feature set that manages labels
+            # (Feature) to account for a new feature
+            found_feature = False
+            for _, linked_features in linked_features_by_slot.items():
+                if feature in linked_features:
+                    found_feature = True
+            if not found_feature:
+                if "external" in linked_features_by_slot:
+                    feature_set = self.features._feature_set_by_slot["external"]
+                    features_list = feature_set.features.list()
+                else:
+                    features_list = []
+                features_list.append(feature)
+                feature_set = FeatureSet(features_list, modality=priors.modalities.meta)
+                feature_set.save()
+                if "external" in linked_features_by_slot:
+                    old_feature_set_link = feature_set_links.filter(
+                        slot="external"
                     ).one()
-                    logger.info(
-                        "no file links to it anymore, deleting feature set"
-                        f" {old_feature_set}"
-                    )
-                    old_feature_set.delete()
-            self.features.add_feature_set(feature_set, slot="external")
-            logger.save(
-                f"linked new feature '{feature.name}' together with new feature set"
-                f" {feature_set}"
-            )
+                    old_feature_set_link.delete()
+                    remaining_links = self.feature_sets.through.objects.filter(
+                        feature_set_id=feature_set.id
+                    ).all()
+                    if len(remaining_links) == 0:
+                        old_feature_set = FeatureSet.filter(
+                            id=old_feature_set_link.feature_set_id
+                        ).one()
+                        logger.info(
+                            "no file links to it anymore, deleting feature set"
+                            f" {old_feature_set}"
+                        )
+                        old_feature_set.delete()
+                self.features.add_feature_set(feature_set, slot="external")
+                logger.save(
+                    f"linked new feature '{feature.name}' together with new feature set"
+                    f" {feature_set}"
+                )
 
 
 def _track_run_input(

{lamindb-0.59.4 → lamindb-0.60.0}/lamindb/dev/_label_manager.py

@@ -1,4 +1,4 @@
-from typing import Dict, List, Union
+from typing import Dict, List, Optional, Union
 
 import numpy as np
 from lamin_utils import colors, logger
@@ -116,7 +116,7 @@ class LabelManager:
     def add(
         self,
         records: Union[Registry, List[Registry], QuerySet],
-        feature: Feature,
+        feature: Optional[Feature] = None,
     ) -> None:
         """Add one or several labels and associate them with a feature.
 

{lamindb-0.59.4 → lamindb-0.60.0}/lamindb/dev/datasets/_core.py

@@ -326,9 +326,9 @@ def anndata_human_immune_cells(
         ln.save(
             lb.Gene.from_values(
                 adata.var.index, field="ensembl_gene_id", organism="human"
-            )[:-35]
+            )
         )
-        ln.save(lb.CellType.from_values(adata.obs.cell_type, field="name")[:-2])
+        ln.save(lb.CellType.from_values(adata.obs.cell_type, field="name"))
         ln.save(lb.ExperimentalFactor.from_values(adata.obs.assay, field="name"))
         ln.save(lb.Tissue.from_values(adata.obs.tissue, field="name"))
         ln.Modality(name="rna", description="RNA measurements").save()
@@ -338,7 +338,9 @@ def anndata_human_immune_cells(
         ).save()
         ln.Feature(name="tissue", type="category", registries=[lb.Tissue]).save()
         ln.Feature(name="organism", type="category", registries=[lb.Organism]).save()
+        ln.Feature(name="donor", type="category", registries=[ln.ULabel]).save()
         lb.ExperimentalFactor.from_bionty(ontology_id="EFO:0008913").save()
+        ln.save([ln.ULabel(name=name) for name in adata.obs.donor.unique()])
         ln.settings.verbosity = verbosity
         lb.settings.auto_save_parents = auto_save_parents
     return adata

{lamindb-0.59.4 → lamindb-0.60.0}/lamindb/dev/storage/__init__.py

@@ -5,8 +5,6 @@
 
    AnnDataAccessor
    BackedAccessor
-   UPath
-
 """
 from lamindb_setup.dev.upath import LocalPathClasses, UPath, infer_filesystem