lamindb 1.3.2__py3-none-any.whl → 1.5.0__py3-none-any.whl

lamindb/core/storage/_pyarrow_dataset.py

@@ -13,41 +13,26 @@ if TYPE_CHECKING:
 PYARROW_SUFFIXES = (".parquet", ".csv", ".json", ".orc", ".arrow", ".feather", ".ipc")
 
 
-def _is_pyarrow_dataset(paths: UPath | list[UPath]) -> bool:
-    # it is assumed here that the paths exist
-    # we don't check here that the filesystem is the same
-    # but this is a requirement for pyarrow.dataset.dataset
-    if isinstance(paths, list):
-        path_list = paths
-    elif paths.is_dir():
-        path_list = [path for path in paths.rglob("*") if path.suffix != ""]
-    else:
-        path_list = [paths]
-    suffix = None
-    for path in path_list:
-        path_suffixes = path.suffixes
-        # this doesn't work for externally gzipped files, REMOVE LATER
-        path_suffix = (
-            path_suffixes[-2]
-            if len(path_suffixes) > 1 and ".gz" in path_suffixes
-            else path.suffix
-        )
-        if path_suffix not in PYARROW_SUFFIXES:
-            return False
-        elif suffix is None:
-            suffix = path_suffix
-        elif path_suffix != suffix:
-            return False
-    return True
-
-
 def _open_pyarrow_dataset(paths: UPath | list[UPath], **kwargs) -> PyArrowDataset:
     if isinstance(paths, list):
+        # a single path can be a directory, but a list of paths
+        # has to be a flat list of files
+        paths_str = []
         path0 = paths[0]
         if isinstance(path0, LocalPathClasses):
-            paths_str, filesystem = [path.as_posix() for path in paths], None
+            path_to_str = lambda p: p.as_posix()
+            filesystem = None
         else:
-            paths_str, filesystem = [path.path for path in paths], path0.fs
+            path_to_str = lambda p: p.path
+            filesystem = path0.fs
+        for path in paths:
+            if (
+                getattr(path, "protocol", None) not in {"http", "https"}
+                and path.is_dir()
+            ):
+                paths_str += [path_to_str(p) for p in path.rglob("*") if p.suffix != ""]
+            else:
+                paths_str.append(path_to_str(path))
     elif isinstance(paths, LocalPathClasses):
         paths_str, filesystem = paths.as_posix(), None
     else:

lamindb/core/storage/objects.py

@@ -21,6 +21,7 @@ def infer_suffix(dmem: SupportedDataTypes, format: str | None = None):
     """Infer LaminDB storage file suffix from a data object."""
     if isinstance(dmem, AnnData):
         if format is not None:
+            # should be `.h5ad`, `.`zarr`, or `.anndata.zarr`
             if format not in {"h5ad", "zarr", "anndata.zarr"}:
                 raise ValueError(
                     "Error when specifying AnnData storage format, it should be"
@@ -31,6 +32,8 @@ def infer_suffix(dmem: SupportedDataTypes, format: str | None = None):
         return ".h5ad"
 
     if isinstance(dmem, DataFrame):
+        if format == ".csv":
+            return ".csv"
         return ".parquet"
 
     if with_package_obj(
@@ -79,6 +82,9 @@ def write_to_disk(dmem: SupportedDataTypes, filepath: UPathStr) -> None:
             raise NotImplementedError
 
     if isinstance(dmem, DataFrame):
+        if filepath.suffix == ".csv":
+            dmem.to_csv(filepath)
+            return
         dmem.to_parquet(filepath)
         return
 

lamindb/core/subsettings/__init__.py

@@ -4,7 +4,9 @@
    :toctree: .
 
    CreationSettings
+   AnnotationSettings
 
 """
 
+from ._annotation_settings import AnnotationSettings
 from ._creation_settings import CreationSettings

lamindb/core/subsettings/_annotation_settings.py

@@ -0,0 +1,11 @@
+class AnnotationSettings:
+    n_max_records: int = 1000
+    """Maximal number of records to annotate with during automated annotation.
+
+    If the number of records to annotate exceeds this limit, print a warning and do not annotate.
+
+    The number is calculated per feature for labels, and per schema for features.
+    """
+
+
+annotation_settings = AnnotationSettings()