lamindb 1.1.1__py3-none-any.whl → 1.2.0__py3-none-any.whl

lamindb/__init__.py

@@ -16,8 +16,8 @@ Registries.
 
    Artifact
    Transform
-   ULabel
    Run
+   ULabel
    User
    Storage
    Feature
@@ -25,9 +25,18 @@ Registries.
    Param
    Collection
    Project
+   Space
    Reference
    Person
 
+Curators & integrations.
+
+.. autosummary::
+   :toctree: .
+
+   curators
+   integrations
+
 Key functionality.
 
 .. autosummary::
@@ -36,20 +45,19 @@ Key functionality.
    connect
    view
    save
+   UPath
+   settings
 
-Modules and settings.
+Low-level functionality.
 
 .. autosummary::
    :toctree: .
 
-   integrations
    context
-   curators
-   settings
    errors
    setup
-   UPath
    base
+   models
    core
 
 Backward compatibility.
@@ -62,8 +70,16 @@ Backward compatibility.
 
 """
 
+# ruff: noqa: I001
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "1.1.1"
+__version__ = "1.2.0"
+
+import warnings
+
+# through SpatialData
+warnings.filterwarnings(
+    "ignore", message="The legacy Dask DataFrame implementation is deprecated"
+)
 
 from lamindb_setup._check_setup import InstanceNotSetupError as _InstanceNotSetupError
 from lamindb_setup._check_setup import _check_instance_setup
@@ -79,23 +95,7 @@ def __getattr__(name):
 
 if _check_instance_setup(from_module="lamindb"):
     del __getattr__  # so that imports work out
-    from . import core  # isort: split
-    from . import (
-        _artifact,
-        _can_curate,
-        _collection,
-        _feature,
-        _is_versioned,
-        _parents,
-        _record,
-        _run,
-        _schema,
-        _storage,
-        _transform,
-        _ulabel,
-        integrations,
-    )
-    from ._save import save
+    from . import base
     from ._tracked import tracked
     from ._view import view
     from .core._context import context
@@ -111,12 +111,17 @@ if _check_instance_setup(from_module="lamindb"):
         Project,
         Reference,
         Run,
-        Schema,  # forward compat
+        Schema,
         Storage,
         Transform,
         ULabel,
         User,
+        Space,
     )
+    from .models.save import save
+    from . import core
+    from . import integrations
+    from . import curators
 
     track = context.track  # simple access
     finish = context.finish  # simple access

lamindb/_tracked.py

@@ -5,8 +5,8 @@ from datetime import datetime, timezone
 from typing import Callable, ParamSpec, TypeVar
 
 from .core._context import context
-from .core._feature_manager import infer_feature_type_convert_json
 from .models import Run, Transform
+from .models._feature_manager import infer_feature_type_convert_json
 
 P = ParamSpec("P")
 R = TypeVar("R")

lamindb/_view.py

@@ -9,10 +9,9 @@ from lamin_utils import colors, logger
 from lamindb_setup import settings
 from lamindb_setup._init_instance import get_schema_module_name
 
-from lamindb.core import FeatureValue, ParamValue
-from lamindb.models import Feature, Record
+from lamindb.models import Feature, FeatureValue, ParamValue, Record
 
-from ._feature import convert_pandas_dtype_to_lamin_dtype
+from .models.feature import convert_pandas_dtype_to_lamin_dtype
 
 if TYPE_CHECKING:
     import pandas as pd

lamindb/base/__init__.py

@@ -1,6 +1,6 @@
 """Base library.
 
-Is available also when no instance is connected.
+Is available also when no instance is setup.
 
 Modules:
 

lamindb/base/ids.py

@@ -1,4 +1,4 @@
-"""IDs.
+"""Universal IDs.
 
 Base generators:
 
@@ -55,15 +55,6 @@ def base62(n_char: int) -> str:
     return id
 
 
-# the following cannot be serialized by Django
-# class Base62:
-#     def __init__(self, n_char: int):
-#         self.n_char = n_char
-
-#     def __call__(self):
-#         return base62(self.n_char)
-
-
 def base26(n_char: int):
     """ASCII lowercase."""
     alphabet = string.ascii_lowercase

lamindb/core/__init__.py

@@ -1,49 +1,20 @@
 """Core library.
 
-Registries:
-
-.. autosummary::
-   :toctree: .
-
-   BasicRecord
-   Record
-   Registry
-   QuerySet
-   QueryManager
-   RecordList
-   FeatureManager
-   ParamManager
-   LabelManager
-   IsVersioned
-   CanCurate
-   HasParents
-   TracksRun
-   TracksUpdates
-   ParamValue
-   FeatureValue
-   InspectResult
-   ValidateFields
-   fields
-
-Curators:
+Settings & context:
 
 .. autosummary::
    :toctree: .
 
-   CatManager
-   DataFrameCatManager
-   AnnDataCatManager
-   MuDataCatManager
-   TiledbsomaCatManager
-   CurateLookup
+   Settings
+   subsettings
+   Context
 
-Settings & context:
+Artifact loaders:
 
 .. autosummary::
    :toctree: .
 
-   Settings
-   Context
+   loaders
 
 Data loaders:
 
@@ -57,10 +28,8 @@ Modules:
 .. autosummary::
    :toctree: .
 
-   loaders
    datasets
    storage
-   subsettings
    logger
 
 """
@@ -68,35 +37,8 @@ Modules:
 from lamin_utils import logger
 from lamin_utils._inspect import InspectResult
 
-from lamindb._query_manager import QueryManager
-from lamindb._query_set import QuerySet, RecordList
-from lamindb.core._feature_manager import FeatureManager, ParamManager
-from lamindb.core._label_manager import LabelManager
-from lamindb.curators import (
-    AnnDataCatManager,
-    CatManager,
-    CurateLookup,
-    Curator,
-    DataFrameCatManager,
-    MuDataCatManager,
-    TiledbsomaCatManager,
-)
-from lamindb.models import (
-    BasicRecord,
-    CanCurate,
-    FeatureValue,
-    HasParents,
-    IsVersioned,
-    ParamValue,
-    Record,
-    Registry,
-    TracksRun,
-    TracksUpdates,
-    ValidateFields,
-)
-
 from .. import errors as exceptions
-from . import _data, datasets, fields, loaders, subsettings, types
+from . import datasets, loaders, subsettings, types
 from ._context import Context
 from ._mapped_collection import MappedCollection
 from ._settings import Settings

lamindb/core/_compat.py

@@ -0,0 +1,60 @@
+import importlib.util
+from typing import Any, Callable, TypeVar
+
+T = TypeVar("T")
+
+
+def is_package_installed(package_name: str) -> bool:
+    spec = importlib.util.find_spec(package_name)
+    return spec is not None
+
+
+def with_package(package_name: str, operation: Callable[[Any], T]) -> T:
+    """Execute an operation that requires a specific package.
+
+    Args:
+        package_name: Package name (e.g., "mudata")
+        operation: Function that takes the imported module and returns a result
+
+    Examples:
+        # For direct package functions
+        result = with_package("mudata", lambda mod: mod.read_zarr(path))
+    """
+    try:
+        module = importlib.import_module(package_name)
+        return operation(module)
+    except ImportError:
+        raise ImportError(
+            f"Package '{package_name}' is required but not installed. "
+            f"Please install with: pip install {package_name}"
+        ) from None
+
+
+def with_package_obj(
+    obj: Any, class_name: str, package_name: str, operation: Callable[[Any], T]
+) -> tuple[bool, T | None]:
+    """Handle operations on objects that require specific packages.
+
+    Args:
+        obj: The object to operate on
+        class_name: Expected class name (e.g., "MuData")
+        package_name: Package that provides the class (e.g., "mudata")
+        operation: Function to call with the object if package is available.
+
+    Examples:
+        # For instance methods
+        handled, res = apply_class_func(dmem, "MuData", "mudata",
+                                      lambda obj: obj.write(filepath))
+    """
+    if obj.__class__.__name__ == class_name:
+        try:
+            importlib.import_module(package_name)
+            result = operation(obj)
+            return True, result
+        except ImportError:
+            raise ImportError(
+                f"Object appears to be {class_name} but '{package_name}' package is not installed. "
+                f"Please install with: pip install {package_name}"
+            ) from None
+
+    return False, None

lamindb/core/_context.py

@@ -11,7 +11,7 @@ from pathlib import Path
 from typing import TYPE_CHECKING
 
 import lamindb_setup as ln_setup
-from django.db.models import Func, IntegerField
+from django.db.models import Func, IntegerField, Q
 from lamin_utils import logger
 from lamindb_setup.core import deprecated
 from lamindb_setup.core.hashing import hash_file
@@ -20,21 +20,26 @@ from lamindb.base import ids
 from lamindb.base.ids import base62_12
 from lamindb.models import Run, Transform, format_field_value
 
+from ..core._settings import settings
 from ..errors import (
     InconsistentKey,
+    InvalidArgument,
     TrackNotCalled,
     UpdateContext,
 )
-from ._settings import settings
+from ..models._is_versioned import bump_version as bump_version_function
+from ..models._is_versioned import (
+    increment_base62,
+    message_update_key_in_version_family,
+)
 from ._sync_git import get_transform_reference_from_git_repo
 from ._track_environment import track_environment
-from .versioning import bump_version as bump_version_function
-from .versioning import increment_base62, message_update_key_in_version_family
 
 if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
 
     from lamindb.base.types import TransformType
+    from lamindb.models import Project
 
 is_run_from_ipython = getattr(builtins, "__IPYTHON__", False)
 
@@ -198,6 +203,7 @@ class Context:
         self._run: Run | None = None
         self._path: Path | None = None
         """A local path to the script that's running."""
+        self._project: Project | None = None
         self._logging_message_track: str = ""
         self._logging_message_imports: str = ""
         self._stream_tracker: LogStreamTracker = LogStreamTracker()
@@ -244,6 +250,11 @@ class Context:
     def version(self, value: str | None):
         self._version = value
 
+    @property
+    def project(self) -> Project | None:
+        """Project to label entities created during the run."""
+        return self._project
+
     @property
     def run(self) -> Run | None:
         """Managed run of context."""
@@ -253,10 +264,10 @@ class Context:
         self,
         transform: str | Transform | None = None,
         *,
+        project: str | None = None,
         params: dict | None = None,
         new_run: bool | None = None,
         path: str | None = None,
-        log_to_file: bool | None = None,
     ) -> None:
         """Track a global run of your Python session.
 
@@ -270,14 +281,12 @@ class Context:
 
         Args:
             transform: A transform `uid` or record. If `None`, creates a `uid`.
+            project: A project `name` or `uid` for labeling entities created during the run.
             params: A dictionary of parameters to track for the run.
             new_run: If `False`, loads the latest run of transform
                 (default notebook), if `True`, creates new run (default non-notebook).
             path: Filepath of notebook or script. Only needed if it can't be
                 automatically detected.
-            log_to_file: If `True`, logs stdout and stderr to a file and
-                saves the file within the current run (default non-notebook),
-                if `False`, does not log the output (default notebook).
 
         Examples:
 
@@ -290,6 +299,17 @@ class Context:
             >>> ln.track("Onv04I53OgtT0000")  # example uid, the last four characters encode the version of the transform
 
         """
+        from lamindb.models import Project
+
+        if project is not None:
+            project_record = Project.filter(
+                Q(name=project) | Q(uid=project)
+            ).one_or_none()
+            if project_record is None:
+                raise InvalidArgument(
+                    f"Project '{project}' not found, either create it with `ln.Project(name='...').save()` or fix typos."
+                )
+            self._project = project_record
         self._logging_message_track = ""
         self._logging_message_imports = ""
         if transform is not None and isinstance(transform, str):
@@ -375,6 +395,12 @@ class Context:
             )
         self._run = run
         track_environment(run)
+        if self.project is not None:
+            # to update a potential project link
+            # is only necessary if transform is loaded rather than newly created
+            # can be optimized by checking whether the transform is loaded, but it typically is
+            self.transform.save()
+        log_to_file = None
         if log_to_file is None:
             log_to_file = self.transform.type != "notebook"
         if log_to_file:
@@ -435,26 +461,23 @@ class Context:
             path_str = get_notebook_key_colab()
             path = Path(path_str)
         else:
-            import nbproject
+            from nbproject.dev import read_notebook
+            from nbproject.dev._meta_live import get_title
+            from nbproject.dev._pypackage import infer_pypackages
 
             try:
-                nbproject_title = nbproject.meta.live.title
-            except IndexError:
-                # notebook is not saved
-                pass
-            if nbproject_title is not None:
-                description = nbproject_title
-            # log imported python packages
-            try:
-                from nbproject.dev._pypackage import infer_pypackages
+                nb = read_notebook(path_str)
+
+                nbproject_title = get_title(nb)
+                if nbproject_title is not None:
+                    description = nbproject_title
 
-                nb = nbproject.dev.read_notebook(path_str)
                 self._logging_message_imports += (
                     "notebook imports:"
                     f" {pretty_pypackages(infer_pypackages(nb, pin_versions=True))}"
                 )
             except Exception:
-                logger.debug("inferring imported packages failed")
+                logger.debug("reading the notebook file failed")
                 pass
         return path, description
 

lamindb/core/_settings.py

@@ -155,12 +155,12 @@ class Settings:
     def verbosity(self) -> str:
         """Logger verbosity (default `'warning'`).
 
-        - `'error'`: ❌ only show error messages
-        - `'warning'`: ❗ also show warning messages
-        - `'success'`: ✅ also show success and save messages
-        - `'info'`: 💡 also show info messages
-        - `'hint'`: 💡 also show hint messages
-        - `'debug'`: 🐛 also show detailed debug messages
+        - `'error'`: only show error messages
+        - `'warning'`: also show warning messages
+        - `'success'`: also show success and save messages
+        - `'info'`: also show info messages
+        - `'hint'`: also show hint messages
+        - `'debug'`: also show detailed debug messages
         """
         return VERBOSITY_TO_STR[self._verbosity_int]
 

lamindb/core/_sync_git.py

@@ -7,7 +7,7 @@ from lamin_utils import logger
 from lamindb_setup import settings as setup_settings
 from lamindb_setup.core.hashing import hash_code
 
-from ._settings import sanitize_git_repo_url, settings
+from ..core._settings import sanitize_git_repo_url, settings
 
 
 class BlobHashNotFound(SystemExit):

lamindb/core/loaders.py

@@ -20,26 +20,30 @@ from __future__ import annotations
 import builtins
 import re
 from pathlib import Path
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
-import anndata as ad
 import pandas as pd
+from anndata import read_h5ad
 from lamin_utils import logger
 from lamindb_setup.core.upath import (
     create_path,
     infer_filesystem,
 )
 
-from ._settings import settings
+from ..core._settings import settings
 
 if TYPE_CHECKING:
+    from anndata import AnnData
     from lamindb_setup.core.types import UPathStr
+    from mudata import MuData
+
+    from lamindb.core.types import ScverseDataStructures
 
 try:
-    from .storage._zarr import load_anndata_zarr
+    from ..core.storage._zarr import load_zarr
 except ImportError:
 
-    def load_anndata_zarr(storepath):  # type: ignore
+    def load_zarr(storepath):  # type: ignore
         raise ImportError("Please install zarr: pip install zarr<=2.18.4")
 
 
@@ -47,7 +51,7 @@ is_run_from_ipython = getattr(builtins, "__IPYTHON__", False)
 
 
 # tested in lamin-usecases
-def load_fcs(*args, **kwargs) -> ad.AnnData:
+def load_fcs(*args, **kwargs) -> AnnData:
     """Load an `.fcs` file to `AnnData`."""
     try:
         import readfcs
@@ -62,16 +66,16 @@ def load_tsv(path: UPathStr, **kwargs) -> pd.DataFrame:
     return pd.read_csv(path_sanitized, sep="\t", **kwargs)
 
 
-def load_h5ad(filepath, **kwargs) -> ad.AnnData:
+def load_h5ad(filepath, **kwargs) -> AnnData:
     """Load an `.h5ad` file to `AnnData`."""
     fs, filepath = infer_filesystem(filepath)
     compression = kwargs.pop("compression", "infer")
     with fs.open(filepath, mode="rb", compression=compression) as file:
-        adata = ad.read_h5ad(file, backed=False, **kwargs)
+        adata = read_h5ad(file, backed=False, **kwargs)
         return adata
 
 
-def load_h5mu(filepath: UPathStr, **kwargs):
+def load_h5mu(filepath: UPathStr, **kwargs) -> MuData:
     """Load an `.h5mu` file to `MuData`."""
     import mudata as md
 
@@ -100,7 +104,7 @@ def load_html(path: UPathStr) -> None | UPathStr:
         return path
 
 
-def load_json(path: UPathStr) -> dict:
+def load_json(path: UPathStr) -> dict[str, Any] | list[Any]:
     """Load `.json` to `dict`."""
     import json
 
@@ -109,7 +113,7 @@ def load_json(path: UPathStr) -> dict:
     return data
 
 
-def load_yaml(path: UPathStr) -> dict:
+def load_yaml(path: UPathStr) -> dict[str, Any] | list[Any]:
     """Load `.yaml` to `dict`."""
     import yaml  # type: ignore
 
@@ -156,7 +160,7 @@ FILE_LOADERS = {
     ".parquet": pd.read_parquet,
     ".parquet.gz": pd.read_parquet,  # this doesn't work for externally gzipped files, REMOVE LATER
     ".fcs": load_fcs,
-    ".zarr": load_anndata_zarr,
+    ".zarr": load_zarr,
     ".html": load_html,
     ".json": load_json,
     ".yaml": load_yaml,
@@ -172,15 +176,18 @@ SUPPORTED_SUFFIXES = [sfx for sfx in FILE_LOADERS.keys() if sfx != ".rds"]
 """Suffixes with defined artifact loaders."""
 
 
-def load_to_memory(filepath: UPathStr, **kwargs):
+def load_to_memory(
+    filepath: UPathStr, **kwargs
+) -> (
+    pd.DataFrame | ScverseDataStructures | dict[str, Any] | list[Any] | UPathStr | None
+):
     """Load a file into memory.
 
     Returns the filepath if no in-memory form is found.
+    May return None in interactive sessions for images.
     """
     filepath = create_path(filepath)
 
-    filepath = settings._storage_settings.cloud_to_local(filepath, print_progress=True)
-
     # infer the correct suffix when .gz is present
     suffixes = filepath.suffixes
     suffix = (
@@ -189,8 +196,12 @@ def load_to_memory(filepath: UPathStr, **kwargs):
         else filepath.suffix
     )
 
-    loader = FILE_LOADERS.get(suffix)
+    loader = FILE_LOADERS.get(suffix, None)
     if loader is None:
-        return filepath
-    else:
-        return loader(filepath, **kwargs)
+        raise NotImplementedError(
+            f"There is no loader for {suffix} files. Use .cache() to get the path."
+        )
+
+    filepath = settings._storage_settings.cloud_to_local(filepath, print_progress=True)
+
+    return loader(filepath, **kwargs)

lamindb/core/storage/_backed_access.py

@@ -5,8 +5,6 @@ from typing import TYPE_CHECKING, Any, Callable
 
 from anndata._io.specs.registry import get_spec
 
-from lamindb.models import Artifact
-
 from ._anndata_accessor import AnnDataAccessor, StorageType, registry
 from ._pyarrow_dataset import _is_pyarrow_dataset, _open_pyarrow_dataset
 from ._tiledbsoma import _open_tiledbsoma
@@ -19,6 +17,8 @@ if TYPE_CHECKING:
     from tiledbsoma import Experiment as SOMAExperiment
     from upath import UPath
 
+    from lamindb.models.artifact import Artifact
+
 
 # this dynamically creates a subclass of a context manager class
 # and reassigns it to an instance of the superclass
@@ -74,6 +74,8 @@ def backed_access(
 ) -> (
     AnnDataAccessor | BackedAccessor | SOMACollection | SOMAExperiment | PyArrowDataset
 ):
+    from lamindb.models import Artifact
+
     if isinstance(artifact_or_filepath, Artifact):
         objectpath, _ = filepath_from_artifact(
             artifact_or_filepath, using_key=using_key