lamindb 1.1.0__py3-none-any.whl → 1.1.1__py3-none-any.whl

lamindb/core/storage/_pyarrow_dataset.py

@@ -18,15 +18,30 @@ def _is_pyarrow_dataset(paths: UPath | list[UPath]) -> bool:
     # we don't check here that the filesystem is the same
     # but this is a requirement for pyarrow.dataset.dataset
     if isinstance(paths, list):
-        suffixes = {path.suffix for path in paths}
-    elif paths.is_file():
-        suffixes = {paths.suffix}
+        path_list = paths
+    elif paths.is_dir():
+        path_list = [path for path in paths.rglob("*") if path.suffix != ""]
     else:
-        suffixes = {path.suffix for path in paths.rglob("*") if path.suffix != ""}
-    return len(suffixes) == 1 and suffixes.pop() in PYARROW_SUFFIXES
-
-
-def _open_pyarrow_dataset(paths: UPath | list[UPath]) -> PyArrowDataset:
+        path_list = [paths]
+    suffix = None
+    for path in path_list:
+        path_suffixes = path.suffixes
+        # this doesn't work for externally gzipped files, REMOVE LATER
+        path_suffix = (
+            path_suffixes[-2]
+            if len(path_suffixes) > 1 and ".gz" in path_suffixes
+            else path.suffix
+        )
+        if path_suffix not in PYARROW_SUFFIXES:
+            return False
+        elif suffix is None:
+            suffix = path_suffix
+        elif path_suffix != suffix:
+            return False
+    return True
+
+
+def _open_pyarrow_dataset(paths: UPath | list[UPath], **kwargs) -> PyArrowDataset:
     if isinstance(paths, list):
         path0 = paths[0]
         if isinstance(path0, LocalPathClasses):
@@ -38,4 +53,4 @@ def _open_pyarrow_dataset(paths: UPath | list[UPath]) -> PyArrowDataset:
     else:
         paths_str, filesystem = paths.path, paths.fs
 
-    return pyarrow.dataset.dataset(paths_str, filesystem=filesystem)
+    return pyarrow.dataset.dataset(paths_str, filesystem=filesystem, **kwargs)

lamindb/core/storage/paths.py

@@ -41,24 +41,21 @@ def auto_storage_key_from_artifact_uid(uid: str, suffix: str, is_dir: bool) -> s
     return storage_key
 
 
-def _safely_resolve(upath: UPath) -> UPath:
-    if upath.protocol in {"http", "https"}:
-        resolve_kwargs = {"follow_redirects": False}
-    else:
-        resolve_kwargs = {}
-    return upath.resolve(**resolve_kwargs)
-
-
 def check_path_is_child_of_root(path: UPathStr, root: UPathStr) -> bool:
     if fsspec.utils.get_protocol(str(path)) != fsspec.utils.get_protocol(str(root)):
         return False
-    path_upath = _safely_resolve(UPath(path))
-    root_upath = _safely_resolve(UPath(root))
+    path_upath = UPath(path)
+    root_upath = UPath(root)
     if path_upath.protocol == "s3":
         endpoint_path = path_upath.storage_options.get("endpoint_url", "")
         endpoint_root = root_upath.storage_options.get("endpoint_url", "")
         if endpoint_path != endpoint_root:
             return False
+    # we don't resolve http links because they can resolve into a different domain
+    # for example into a temporary url
+    if path_upath.protocol not in {"http", "https"}:
+        path_upath = path_upath.resolve()
+        root_upath = root_upath.resolve()
     # str is needed to eliminate UPath storage_options
     # which affect equality checks
     return UPath(str(root_upath)) in UPath(str(path_upath)).parents
@@ -134,7 +131,7 @@ def filepath_cache_key_from_artifact(
 
 
 def store_file_or_folder(
-    local_path: UPathStr, storage_path: UPath, print_progress: bool = True
+    local_path: UPathStr, storage_path: UPath, print_progress: bool = True, **kwargs
 ) -> None:
     """Store file or folder (localpath) at storagepath."""
     local_path = UPath(local_path)
@@ -155,7 +152,10 @@ def store_file_or_folder(
         else:
             create_folder = None
         storage_path.upload_from(
-            local_path, create_folder=create_folder, print_progress=print_progress
+            local_path,
+            create_folder=create_folder,
+            print_progress=print_progress,
+            **kwargs,
         )
     else:  # storage path is local
         if local_path.resolve().as_posix() == storage_path.resolve().as_posix():

lamindb/curators/__init__.py

@@ -1,5 +1,7 @@
 """Curators.
 
+.. versionadded:: 1.1.0
+
 .. autosummary::
    :toctree: .
 
@@ -146,6 +148,12 @@ class CurateLookup:
 CAT_MANAGER_DOCSTRING = """Manage categoricals by updating registries."""
 
 
+SLOTS_DOCSTRING = """Curator objects by slot.
+
+.. versionadded:: 1.1.1
+"""
+
+
 VALIDATE_DOCSTRING = """Validate dataset.
 
 Raises:
@@ -170,6 +178,8 @@ class Curator:
 
     A `Curator` object makes it easy to validate, standardize & annotate datasets.
 
+    .. versionadded:: 1.1.0
+
     See:
         - :class:`~lamindb.curators.DataFrameCurator`
         - :class:`~lamindb.curators.AnnDataCurator`
@@ -212,6 +222,8 @@ class DataFrameCurator(Curator):
 
     See also :class:`~lamindb.Curator` and :class:`~lamindb.Schema`.
 
+    .. versionadded:: 1.1.0
+
     Args:
         dataset: The DataFrame-like object to validate & annotate.
         schema: A `Schema` object that defines the validation constraints.
@@ -222,9 +234,9 @@ class DataFrameCurator(Curator):
         import bionty as bt
 
         # define valid labels
-        cell_medium = ln.ULabel(name="CellMedium", is_type=True).save()
-        ln.ULabel(name="DMSO", type=cell_medium).save()
-        ln.ULabel(name="IFNG", type=cell_medium).save()
+        perturbation = ln.ULabel(name="Perturbation", is_type=True).save()
+        ln.ULabel(name="DMSO", type=perturbation).save()
+        ln.ULabel(name="IFNG", type=perturbation).save()
         bt.CellType.from_source(name="B cell").save()
         bt.CellType.from_source(name="T cell").save()
 
@@ -232,7 +244,7 @@ class DataFrameCurator(Curator):
         schema = ln.Schema(
             name="small_dataset1_obs_level_metadata",
             features=[
-                ln.Feature(name="cell_medium", dtype="cat[ULabel[CellMedium]]").save(),
+                ln.Feature(name="perturbation", dtype="cat[ULabel[Perturbation]]").save(),
                 ln.Feature(name="sample_note", dtype=str).save(),
                 ln.Feature(name="cell_type_by_expert", dtype=bt.CellType).save(),
                 ln.Feature(name="cell_type_by_model", dtype=bt.CellType).save(),
@@ -252,10 +264,10 @@ class DataFrameCurator(Curator):
         schema: Schema,
     ) -> None:
         super().__init__(dataset=dataset, schema=schema)
+        categoricals = {}
         if schema.n > 0:
             # populate features
             pandera_columns = {}
-            categoricals = {}
             for feature in schema.features.all():
                 pandera_dtype = (
                     feature.dtype if not feature.dtype.startswith("cat") else "category"
@@ -268,13 +280,13 @@ class DataFrameCurator(Curator):
             self._pandera_schema = pandera.DataFrameSchema(
                 pandera_columns, coerce=schema.coerce_dtype
             )
-            # now deal with detailed validation of categoricals
-            self._cat_manager = DataFrameCatManager(
-                self._dataset,
-                categoricals=categoricals,
-            )
         else:
             assert schema.itype is not None  # noqa: S101
+        self._cat_manager = DataFrameCatManager(
+            self._dataset,
+            columns=parse_dtype_single_cat(schema.itype, is_itype=True)["field"],
+            categoricals=categoricals,
+        )
 
     @property
     @doc_args(CAT_MANAGER_DOCSTRING)
@@ -285,16 +297,29 @@ class DataFrameCurator(Curator):
     def standardize(self) -> None:
         """Standardize the dataset.
 
-        - Adds missing columns if a default value for a feature is defined.
-        - Fills missing values with the default value if a default value for a feature is defined.
+        - Adds missing columns for features
+        - Fills missing values for features with default values
         """
         for feature in self._schema.members:
             if feature.name not in self._dataset.columns:
-                if feature.default_value is not None:
-                    self._dataset[feature.name] = feature.default_value
+                if feature.default_value is not None or feature.nullable:
+                    fill_value = (
+                        feature.default_value
+                        if feature.default_value is not None
+                        else pd.NA
+                    )
+                    if feature.dtype.startswith("cat"):
+                        self._dataset[feature.name] = pd.Categorical(
+                            [fill_value] * len(self._dataset)
+                        )
+                    else:
+                        self._dataset[feature.name] = fill_value
+                    logger.important(
+                        f"added column {feature.name} with fill value {fill_value}"
+                    )
                 else:
                     raise ValidationError(
-                        f"Missing column {feature.name} cannot be added because no default value is defined for this feature"
+                        f"Missing column {feature.name} cannot be added because is not nullable and has no default value"
                     )
             else:
                 if feature.default_value is not None:
@@ -312,46 +337,29 @@ class DataFrameCurator(Curator):
                         feature.default_value
                     )
 
+    def _cat_manager_validate(self) -> None:
+        self._cat_manager.validate()
+        if self._cat_manager._is_validated:
+            self._is_validated = True
+        else:
+            self._is_validated = False
+            raise ValidationError(self._cat_manager._validate_category_error_messages)
+
     @doc_args(VALIDATE_DOCSTRING)
     def validate(self) -> None:
         """{}"""  # noqa: D415
         if self._schema.n > 0:
-            self._cat_manager.validate()
             try:
+                # first validate through pandera
                 self._pandera_schema.validate(self._dataset)
-                if self._cat_manager._is_validated:
-                    self._is_validated = True
-                else:
-                    self._is_validated = False
-                    raise ValidationError(
-                        self._cat_manager._validate_category_error_messages
-                    )
+                # then validate lamindb categoricals
+                self._cat_manager_validate()
             except pandera.errors.SchemaError as err:
                 self._is_validated = False
                 # .exconly() doesn't exist on SchemaError
                 raise ValidationError(str(err)) from err
         else:
-            result = parse_dtype_single_cat(self._schema.itype, is_itype=True)
-            registry: CanCurate = result["registry"]
-            inspector = registry.inspect(
-                self._dataset.columns,
-                result["field"],
-                mute=True,
-            )
-            if len(inspector.non_validated) > 0:
-                # also check public ontology
-                if hasattr(registry, "public"):
-                    registry.from_values(
-                        inspector.non_validated, result["field"], mute=True
-                    ).save()
-                    inspector = registry.inspect(
-                        inspector.non_validated, result["field"], mute=True
-                    )
-                if len(inspector.non_validated) > 0:
-                    self._is_validated = False
-                    raise ValidationError(
-                        f"Invalid identifiers for {self._schema.itype}: {inspector.non_validated}"
-                    )
+            self._cat_manager_validate()
 
     @doc_args(SAVE_ARTIFACT_DOCSTRING)
     def save_artifact(
@@ -385,6 +393,8 @@ class AnnDataCurator(Curator):
 
     See also :class:`~lamindb.Curator` and :class:`~lamindb.Schema`.
 
+    .. versionadded:: 1.1.0
+
     Args:
         dataset: The AnnData-like object to validate & annotate.
         schema: A `Schema` object that defines the validation constraints.
@@ -395,9 +405,9 @@ class AnnDataCurator(Curator):
         import bionty as bt
 
         # define valid labels
-        cell_medium = ln.ULabel(name="CellMedium", is_type=True).save()
-        ln.ULabel(name="DMSO", type=cell_medium).save()
-        ln.ULabel(name="IFNG", type=cell_medium).save()
+        perturbation = ln.ULabel(name="Perturbation", is_type=True).save()
+        ln.ULabel(name="DMSO", type=perturbation).save()
+        ln.ULabel(name="IFNG", type=perturbation).save()
         bt.CellType.from_source(name="B cell").save()
         bt.CellType.from_source(name="T cell").save()
 
@@ -405,9 +415,9 @@ class AnnDataCurator(Curator):
         obs_schema = ln.Schema(
             name="small_dataset1_obs_level_metadata",
             features=[
-                ln.Feature(name="cell_medium", dtype="cat[ULabel[CellMedium]]").save(),
+                ln.Feature(name="perturbation", dtype="cat[ULabel[Perturbation]]").save(),
                 ln.Feature(name="sample_note", dtype=str).save(),
-                ln.Feature(name="cell_type_by_expert", dtype=bt.CellType").save(),
+                ln.Feature(name="cell_type_by_expert", dtype=bt.CellType).save(),
                 ln.Feature(name="cell_type_by_model", dtype=bt.CellType").save(),
             ],
         ).save()
@@ -416,7 +426,7 @@ class AnnDataCurator(Curator):
         var_schema = ln.Schema(
             name="scRNA_seq_var_schema",
             itype=bt.Gene.ensembl_gene_id,
-            dtype="num",
+            dtype=int,
         ).save()
 
         # define composite schema
@@ -450,15 +460,27 @@ class AnnDataCurator(Curator):
             self._dataset.var.T, schema._get_component("var")
         )
 
+    @property
+    @doc_args(SLOTS_DOCSTRING)
+    def slots(self) -> dict[str, DataFrameCurator]:
+        """{}"""  # noqa: D415
+        return {"obs": self._obs_curator, "var": self._var_curator}
+
     @doc_args(VALIDATE_DOCSTRING)
     def validate(self) -> None:
         """{}"""  # noqa: D415
         self._obs_curator.validate()
         self._var_curator.validate()
-        self._is_validated = True
 
     @doc_args(SAVE_ARTIFACT_DOCSTRING)
-    def save_artifact(self, *, key=None, description=None, revises=None, run=None):
+    def save_artifact(
+        self,
+        *,
+        key: str | None = None,
+        description: str | None = None,
+        revises: Artifact | None = None,
+        run: Run | None = None,
+    ):
         """{}"""  # noqa: D415
         if not self._is_validated:
             self.validate()  # raises ValidationError if doesn't validate
@@ -1031,11 +1053,6 @@ class AnnDataCatManager(CatManager):
 class MuDataCatManager(CatManager):
     """Curation flow for a ``MuData`` object.
 
-    See also :class:`~lamindb.Curator`.
-
-    Note that if genes or other measurements are removed from the MuData object,
-    the object should be recreated using :meth:`~lamindb.Curator.from_mudata`.
-
     Args:
         mdata: The MuData object to curate.
         var_index: The registry field for mapping the ``.var`` index for each modality.
@@ -1289,8 +1306,6 @@ def _maybe_curation_keys_not_present(nonval_keys: list[str], name: str):
 class TiledbsomaCatManager(CatManager):
     """Curation flow for `tiledbsoma.Experiment`.
 
-    See also :class:`~lamindb.Curator`.
-
     Args:
         experiment_uri: A local or cloud path to a `tiledbsoma.Experiment`.
         var_index: The registry fields for mapping the `.var` indices for measurements.
@@ -3168,10 +3183,7 @@ def check_registry_organism(registry: Record, organism: str | None = None) -> di
         import bionty as bt
 
         if organism is None and bt.settings.organism is None:
-            raise ValidationError(
-                f"{registry.__name__} registry requires an organism!\n"
-                "      → please pass an organism name via organism="
-            )
+            return {}
         return {"organism": organism or bt.settings.organism.name}
     return {}
 
@@ -3263,7 +3275,7 @@ def validate_categories(
             warning_message += f"    {colors.yellow(f'{len(syn_mapper)} synonym{s}')} found: {colors.yellow(syn_mapper_print)}\n    → curate synonyms via {colors.cyan(hint_msg)}"
         if n_non_validated > len(syn_mapper):
             if syn_mapper:
-                warning_message += "    for remaining terms:\n"
+                warning_message += "\n    for remaining terms:\n"
             warning_message += f"    → fix typos, remove non-existent values, or save terms via {colors.cyan(non_validated_hint_print)}"
 
         if logger.indent == "":
@@ -3422,7 +3434,7 @@ def save_artifact(
             filter_kwargs_current = get_current_filter_kwargs(registry, filter_kwargs)
             df = data if isinstance(data, pd.DataFrame) else data.obs
             # multi-value columns are separated by "|"
-            if df[key].str.contains("|").any():
+            if not df[key].isna().all() and df[key].str.contains("|").any():
                 values = df[key].str.split("|").explode().unique()
             else:
                 values = df[key].unique()

lamindb/models.py

@@ -699,7 +699,7 @@ class Registry(ModelBase):
             A record.
 
         Raises:
-            :exc:`docs:lamindb.core.exceptions.DoesNotExist`: In case no matching record is found.
+            :exc:`docs:lamindb.errors.DoesNotExist`: In case no matching record is found.
 
         See Also:
             - Guide: :doc:`docs:registries`
@@ -2015,6 +2015,8 @@ class Feature(Record, CanCurate, TracksRun, TracksUpdates):
         """A default value that overwrites missing values (default `None`).
 
         This takes effect when you call `Curator.standardize()`.
+
+        If `default_value = None`, missing values like `pd.NA` or `np.nan` are kept.
         """
         if self._aux is not None and "af" in self._aux and "0" in self._aux["af"]:
             return self._aux["af"]["0"]
@@ -2050,12 +2052,14 @@ class Feature(Record, CanCurate, TracksRun, TracksUpdates):
 
         """
         if self._aux is not None and "af" in self._aux and "1" in self._aux["af"]:
-            return self._aux["af"]["1"]
+            value = self._aux["af"]["1"]
+            return True if value is None else value
         else:
             return True
 
     @nullable.setter
     def nullable(self, value: bool) -> None:
+        assert isinstance(value, bool), value  # noqa: S101
         if self._aux is None:
             self._aux = {}
         if "af" not in self._aux:
@@ -2134,20 +2138,9 @@ class FeatureValue(Record, TracksRun):
 class Schema(Record, CanCurate, TracksRun):
     """Schemas / feature sets.
 
-    Stores references to dataset schemas: these are the sets of columns in a dataset
-    that correspond to :class:`~lamindb.Feature`, :class:`~bionty.Gene`, :class:`~bionty.Protein` or other
-    entities.
+    A simple schema is just a set of columns in a `DataFrame`, a "feature set".
 
-    .. dropdown:: Why does LaminDB model feature sets, not just features?
-
-        1. Performance: Imagine you measure the same panel of 20k transcripts in
-           1M samples. By modeling the panel as a feature set, you can link all
-           your artifacts against one feature set and only need to store 1M
-           instead of 1M x 20k = 20B links.
-        2. Interpretation: Model protein panels, gene panels, etc.
-        3. Data integration: Feature sets provide the information that determines whether two datasets can be meaningfully concatenated.
-
-        These reasons do not hold for label sets. Hence, LaminDB does not model label sets.
+    A composite schema has multiple components, e.g. for an `AnnData`, each a feature set for `obs` and `var`.
 
     Args:
         features: `Iterable[Record] | None = None` An iterable of :class:`~lamindb.Feature`
@@ -2162,7 +2155,7 @@ class Schema(Record, CanCurate, TracksRun):
         dtype: `str | None = None` The simple type. Defaults to
             `None` for sets of :class:`~lamindb.Feature` records.
             Otherwise defaults to `"num"` (e.g., for sets of :class:`~bionty.Gene`).
-        itype: `str | None = None` The schema identifier type (e.g. :class:`~lamindb.Feature`, :class:`~bionty.Gene`, ...).
+        itype: `str | None = None` The feature identifier type (e.g. :class:`~lamindb.Feature`, :class:`~bionty.Gene`, ...).
         type: `Schema | None = None` A type.
         is_type: `bool = False` Distinguish types from instances of the type.
         otype: `str | None = None` An object type to define the structure of a composite schema.
@@ -2174,6 +2167,17 @@ class Schema(Record, CanCurate, TracksRun):
         coerce_dtype: `bool = False` When True, attempts to coerce values to the specified dtype
             during validation, see :attr:`~lamindb.Schema.coerce_dtype`.
 
+    .. dropdown:: Why does LaminDB model schemas, not just features?
+
+        1. Performance: Imagine you measure the same panel of 20k transcripts in
+           1M samples. By modeling the panel as a feature set, you can link all
+           your artifacts against one feature set and only need to store 1M
+           instead of 1M x 20k = 20B links.
+        2. Interpretation: Model protein panels, gene panels, etc.
+        3. Data integration: Feature sets provide the information that determines whether two datasets can be meaningfully concatenated.
+
+        These reasons do not hold for label sets. Hence, LaminDB does not model label sets.
+
     Note:
 
         A feature set can be identified by the `hash` of its feature uids.
@@ -2211,7 +2215,10 @@ class Schema(Record, CanCurate, TracksRun):
         abstract = False
 
     _name_field: str = "name"
-    _aux_fields: dict[str, tuple[str, type]] = {"0": ("coerce_dtype", bool)}
+    _aux_fields: dict[str, tuple[str, type]] = {
+        "0": ("coerce_dtype", bool),
+        "1": ("_index_feature_uid", str),
+    }
 
     id: int = models.AutoField(primary_key=True)
     """Internal id, valid only in one DB instance."""
@@ -2427,6 +2434,39 @@ class Schema(Record, CanCurate, TracksRun):
             self._aux["af"] = {}
         self._aux["af"]["0"] = value
 
+    @coerce_dtype.setter
+    def coerce_dtype(self, value: bool) -> None:
+        if self._aux is None:
+            self._aux = {}
+        if "af" not in self._aux:
+            self._aux["af"] = {}
+        self._aux["af"]["0"] = value
+
+    # @property
+    # def index_feature(self) -> None | Feature:
+    #     # index_feature: `Record | None = None` A :class:`~lamindb.Feature` to validate the index of a `DataFrame`.
+    #     """The uid of the index feature, if `index_feature` was set."""
+    #     if self._index_feature_uid is None:
+    #         return None
+    #     else:
+    #         return self.features.get(uid=self._index_feature_uid)
+
+    # @property
+    # def _index_feature_uid(self) -> None | str:
+    #     """The uid of the index feature, if `index_feature` was set."""
+    #     if self._aux is not None and "af" in self._aux and "1" in self._aux["af"]:
+    #         return self._aux["af"]["1"]
+    #     else:
+    #         return None
+
+    # @_index_feature_uid.setter
+    # def _index_feature_uid(self, value: str) -> None:
+    #     if self._aux is None:
+    #         self._aux = {}
+    #     if "af" not in self._aux:
+    #         self._aux["af"] = {}
+    #     self._aux["af"]["1"] = value
+
     @property
     @deprecated("itype")
     def registry(self) -> str:
@@ -3057,7 +3097,7 @@ class Artifact(Record, IsVersioned, TracksRun, TracksUpdates):
         pass
 
     def open(
-        self, mode: str = "r", is_run_input: bool | None = None
+        self, mode: str = "r", is_run_input: bool | None = None, **kwargs
     ) -> (
         AnnDataAccessor
         | BackedAccessor

{lamindb-1.1.0.dist-info → lamindb-1.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lamindb
-Version: 1.1.0
+Version: 1.1.1
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.10,<3.13
@@ -10,7 +10,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: lamin_utils==0.13.10
 Requires-Dist: lamin_cli==1.1.0
-Requires-Dist: lamindb_setup[aws]==1.1.0
+Requires-Dist: lamindb_setup[aws]==1.2.0
 Requires-Dist: pyyaml
 Requires-Dist: pyarrow
 Requires-Dist: pandera