lamindb 1.1.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

lamindb/__init__.py

@@ -1,12 +1,13 @@
 """A data framework for biology.
 
-Tracking notebooks & scripts.
+Tracking notebooks, scripts & functions.
 
 .. autosummary::
    :toctree: .
 
    track
    finish
+   tracked
 
 Registries.
 
@@ -15,8 +16,8 @@ Registries.
 
    Artifact
    Transform
-   ULabel
    Run
+   ULabel
    User
    Storage
    Feature
@@ -24,9 +25,18 @@ Registries.
    Param
    Collection
    Project
+   Space
    Reference
    Person
 
+Curators & integrations.
+
+.. autosummary::
+   :toctree: .
+
+   curators
+   integrations
+
 Key functionality.
 
 .. autosummary::
@@ -35,20 +45,19 @@ Key functionality.
    connect
    view
    save
+   UPath
+   settings
 
-Modules and settings.
+Low-level functionality.
 
 .. autosummary::
    :toctree: .
 
-   integrations
    context
-   curators
-   settings
    errors
    setup
-   UPath
    base
+   models
    core
 
 Backward compatibility.
@@ -57,11 +66,20 @@ Backward compatibility.
    :toctree: .
 
    FeatureSet
+   Curator
 
 """
 
+# ruff: noqa: I001
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "1.1.0"
+__version__ = "1.2.0"
+
+import warnings
+
+# through SpatialData
+warnings.filterwarnings(
+    "ignore", message="The legacy Dask DataFrame implementation is deprecated"
+)
 
 from lamindb_setup._check_setup import InstanceNotSetupError as _InstanceNotSetupError
 from lamindb_setup._check_setup import _check_instance_setup
@@ -77,23 +95,7 @@ def __getattr__(name):
 
 if _check_instance_setup(from_module="lamindb"):
     del __getattr__  # so that imports work out
-    from . import core  # isort: split
-    from . import (
-        _artifact,
-        _can_curate,
-        _collection,
-        _feature,
-        _is_versioned,
-        _parents,
-        _record,
-        _run,
-        _schema,
-        _storage,
-        _transform,
-        _ulabel,
-        integrations,
-    )
-    from ._save import save
+    from . import base
     from ._tracked import tracked
     from ._view import view
     from .core._context import context
@@ -109,12 +111,17 @@ if _check_instance_setup(from_module="lamindb"):
         Project,
         Reference,
         Run,
-        Schema,  # forward compat
+        Schema,
         Storage,
         Transform,
         ULabel,
         User,
+        Space,
     )
+    from .models.save import save
+    from . import core
+    from . import integrations
+    from . import curators
 
     track = context.track  # simple access
     finish = context.finish  # simple access

lamindb/_finish.py

@@ -436,7 +436,15 @@ def save_context_core(
     # save both run & transform records if we arrive here
     if run is not None:
         run.save()
-    transform.save()
+    transform_id_prior_to_save = transform.id
+    transform.save()  # this in-place updates the state of transform upon hash collision
+    if transform.id != transform_id_prior_to_save:
+        # the hash existed and we're actually back to the previous version
+        # hence, this was in fact a run of the previous transform rather than of
+        # the new transform
+        # this can happen in interactive notebooks if the user makes no change to the notebook
+        run.transform = transform
+        run.save()
 
     # finalize
     if not from_cli and run is not None:

lamindb/_tracked.py

@@ -5,8 +5,8 @@ from datetime import datetime, timezone
 from typing import Callable, ParamSpec, TypeVar
 
 from .core._context import context
-from .core._feature_manager import infer_feature_type_convert_json
 from .models import Run, Transform
+from .models._feature_manager import infer_feature_type_convert_json
 
 P = ParamSpec("P")
 R = TypeVar("R")
@@ -26,10 +26,33 @@ def get_current_tracked_run() -> Run | None:
 
 
 def tracked(uid: str | None = None) -> Callable[[Callable[P, R]], Callable[P, R]]:
-    """Decorator that tracks function execution.
+    """Mark a function as tracked with this decorator.
+
+    You will be able to see inputs, outputs, and parameters of the function in the data lineage graph.
+
+    Guide: :doc:`/track`
+
+    .. versionadded:: 1.1.0
+        This is still in beta and will be refined in future releases.
 
     Args:
-        uid: Optional unique identifier for the transform
+        uid: Persist the uid to identify this transform across renames.
+
+    Example::
+
+        import lamindb as ln
+
+        @ln.tracked()
+        def subset_dataframe(
+            input_artifact_key: str,  # all arguments tracked as parameters of the function run
+            output_artifact_key: str,
+            subset_rows: int = 2,
+            subset_cols: int = 2,
+        ) -> None:
+            artifact = ln.Artifact.get(key=input_artifact_key)
+            df = artifact.load()  # auto-tracked as input
+            new_df = df.iloc[:subset_rows, :subset_cols]
+            ln.Artifact.from_df(new_df, key=output_artifact_key).save()  # auto-tracked as output
     """
 
     def decorator_tracked(func: Callable[P, R]) -> Callable[P, R]:

lamindb/_view.py

@@ -9,10 +9,9 @@ from lamin_utils import colors, logger
 from lamindb_setup import settings
 from lamindb_setup._init_instance import get_schema_module_name
 
-from lamindb.core import FeatureValue, ParamValue
-from lamindb.models import Feature, Record
+from lamindb.models import Feature, FeatureValue, ParamValue, Record
 
-from ._feature import convert_pandas_dtype_to_lamin_dtype
+from .models.feature import convert_pandas_dtype_to_lamin_dtype
 
 if TYPE_CHECKING:
     import pandas as pd

lamindb/base/__init__.py

@@ -1,6 +1,6 @@
 """Base library.
 
-Is available also when no instance is connected.
+Is available also when no instance is setup.
 
 Modules:
 

lamindb/base/ids.py

@@ -1,4 +1,4 @@
-"""IDs.
+"""Universal IDs.
 
 Base generators:
 
@@ -55,15 +55,6 @@ def base62(n_char: int) -> str:
     return id
 
 
-# the following cannot be serialized by Django
-# class Base62:
-#     def __init__(self, n_char: int):
-#         self.n_char = n_char
-
-#     def __call__(self):
-#         return base62(self.n_char)
-
-
 def base26(n_char: int):
     """ASCII lowercase."""
     alphabet = string.ascii_lowercase

lamindb/base/users.py

@@ -12,12 +12,9 @@ def current_user_id() -> int:
         if ln_setup.core.django.IS_MIGRATING:
             return 1
         else:
-            exc_attr = (
-                "DoesNotExist" if hasattr(User, "DoesNotExist") else "_DoesNotExist"
-            )
             try:
                 user_id = User.objects.get(uid=settings.user.uid).id
-            except getattr(User, exc_attr):
+            except User.DoesNotExist:
                 register_user(settings.user)
                 user_id = User.objects.get(uid=settings.user.uid).id
             return user_id

lamindb/core/__init__.py

@@ -1,49 +1,20 @@
 """Core library.
 
-Registries:
-
-.. autosummary::
-   :toctree: .
-
-   BasicRecord
-   Record
-   Registry
-   QuerySet
-   QueryManager
-   RecordList
-   FeatureManager
-   ParamManager
-   LabelManager
-   IsVersioned
-   CanCurate
-   HasParents
-   TracksRun
-   TracksUpdates
-   ParamValue
-   FeatureValue
-   InspectResult
-   ValidateFields
-   fields
-
-Curators:
+Settings & context:
 
 .. autosummary::
    :toctree: .
 
-   CatManager
-   DataFrameCatManager
-   AnnDataCatManager
-   MuDataCatManager
-   TiledbsomaCatManager
-   CurateLookup
+   Settings
+   subsettings
+   Context
 
-Settings & context:
+Artifact loaders:
 
 .. autosummary::
    :toctree: .
 
-   Settings
-   Context
+   loaders
 
 Data loaders:
 
@@ -57,10 +28,8 @@ Modules:
 .. autosummary::
    :toctree: .
 
-   loaders
    datasets
    storage
-   subsettings
    logger
 
 """
@@ -68,35 +37,8 @@ Modules:
 from lamin_utils import logger
 from lamin_utils._inspect import InspectResult
 
-from lamindb._query_manager import QueryManager
-from lamindb._query_set import QuerySet, RecordList
-from lamindb.core._feature_manager import FeatureManager, ParamManager
-from lamindb.core._label_manager import LabelManager
-from lamindb.curators import (
-    AnnDataCatManager,
-    CatManager,
-    CurateLookup,
-    Curator,
-    DataFrameCatManager,
-    MuDataCatManager,
-    TiledbsomaCatManager,
-)
-from lamindb.models import (
-    BasicRecord,
-    CanCurate,
-    FeatureValue,
-    HasParents,
-    IsVersioned,
-    ParamValue,
-    Record,
-    Registry,
-    TracksRun,
-    TracksUpdates,
-    ValidateFields,
-)
-
 from .. import errors as exceptions
-from . import _data, datasets, fields, loaders, subsettings, types
+from . import datasets, loaders, subsettings, types
 from ._context import Context
 from ._mapped_collection import MappedCollection
 from ._settings import Settings

lamindb/core/_compat.py

@@ -0,0 +1,60 @@
+import importlib.util
+from typing import Any, Callable, TypeVar
+
+T = TypeVar("T")
+
+
+def is_package_installed(package_name: str) -> bool:
+    spec = importlib.util.find_spec(package_name)
+    return spec is not None
+
+
+def with_package(package_name: str, operation: Callable[[Any], T]) -> T:
+    """Execute an operation that requires a specific package.
+
+    Args:
+        package_name: Package name (e.g., "mudata")
+        operation: Function that takes the imported module and returns a result
+
+    Examples:
+        # For direct package functions
+        result = with_package("mudata", lambda mod: mod.read_zarr(path))
+    """
+    try:
+        module = importlib.import_module(package_name)
+        return operation(module)
+    except ImportError:
+        raise ImportError(
+            f"Package '{package_name}' is required but not installed. "
+            f"Please install with: pip install {package_name}"
+        ) from None
+
+
+def with_package_obj(
+    obj: Any, class_name: str, package_name: str, operation: Callable[[Any], T]
+) -> tuple[bool, T | None]:
+    """Handle operations on objects that require specific packages.
+
+    Args:
+        obj: The object to operate on
+        class_name: Expected class name (e.g., "MuData")
+        package_name: Package that provides the class (e.g., "mudata")
+        operation: Function to call with the object if package is available.
+
+    Examples:
+        # For instance methods
+        handled, res = apply_class_func(dmem, "MuData", "mudata",
+                                      lambda obj: obj.write(filepath))
+    """
+    if obj.__class__.__name__ == class_name:
+        try:
+            importlib.import_module(package_name)
+            result = operation(obj)
+            return True, result
+        except ImportError:
+            raise ImportError(
+                f"Object appears to be {class_name} but '{package_name}' package is not installed. "
+                f"Please install with: pip install {package_name}"
+            ) from None
+
+    return False, None

lamindb/core/_context.py

@@ -11,29 +11,35 @@ from pathlib import Path
 from typing import TYPE_CHECKING
 
 import lamindb_setup as ln_setup
-from django.db.models import Func, IntegerField
+from django.db.models import Func, IntegerField, Q
 from lamin_utils import logger
+from lamindb_setup.core import deprecated
 from lamindb_setup.core.hashing import hash_file
 
 from lamindb.base import ids
 from lamindb.base.ids import base62_12
 from lamindb.models import Run, Transform, format_field_value
 
+from ..core._settings import settings
 from ..errors import (
     InconsistentKey,
+    InvalidArgument,
     TrackNotCalled,
     UpdateContext,
 )
-from ._settings import settings
+from ..models._is_versioned import bump_version as bump_version_function
+from ..models._is_versioned import (
+    increment_base62,
+    message_update_key_in_version_family,
+)
 from ._sync_git import get_transform_reference_from_git_repo
 from ._track_environment import track_environment
-from .versioning import bump_version as bump_version_function
-from .versioning import increment_base62, message_update_key_in_version_family
 
 if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
 
     from lamindb.base.types import TransformType
+    from lamindb.models import Project
 
 is_run_from_ipython = getattr(builtins, "__IPYTHON__", False)
 
@@ -197,6 +203,7 @@ class Context:
         self._run: Run | None = None
         self._path: Path | None = None
         """A local path to the script that's running."""
+        self._project: Project | None = None
         self._logging_message_track: str = ""
         self._logging_message_imports: str = ""
         self._stream_tracker: LogStreamTracker = LogStreamTracker()
@@ -217,8 +224,8 @@ class Context:
         self._description = value
 
     @property
+    @deprecated(new_name="description")
     def name(self) -> str | None:
-        """Deprecated. Populates `description` argument for `context.transform`."""
         return self._description
 
     @name.setter
@@ -243,6 +250,11 @@ class Context:
     def version(self, value: str | None):
         self._version = value
 
+    @property
+    def project(self) -> Project | None:
+        """Project to label entities created during the run."""
+        return self._project
+
     @property
     def run(self) -> Run | None:
         """Managed run of context."""
@@ -252,12 +264,12 @@ class Context:
         self,
         transform: str | Transform | None = None,
         *,
+        project: str | None = None,
         params: dict | None = None,
         new_run: bool | None = None,
         path: str | None = None,
-        log_to_file: bool | None = None,
     ) -> None:
-        """Initiate a run with tracked data lineage.
+        """Track a global run of your Python session.
 
         - sets :attr:`~lamindb.core.Context.transform` &
           :attr:`~lamindb.core.Context.run` by creating or loading `Transform` &
@@ -269,14 +281,12 @@ class Context:
 
         Args:
             transform: A transform `uid` or record. If `None`, creates a `uid`.
+            project: A project `name` or `uid` for labeling entities created during the run.
             params: A dictionary of parameters to track for the run.
             new_run: If `False`, loads the latest run of transform
                 (default notebook), if `True`, creates new run (default non-notebook).
             path: Filepath of notebook or script. Only needed if it can't be
                 automatically detected.
-            log_to_file: If `True`, logs stdout and stderr to a file and
-                saves the file within the current run (default non-notebook),
-                if `False`, does not log the output (default notebook).
 
         Examples:
 
@@ -284,7 +294,22 @@ class Context:
 
             >>> ln.track()
 
+            If you want to ensure a single version history across renames of the notebook or script, pass the auto-generated `uid` that you'll find in the logs:
+
+            >>> ln.track("Onv04I53OgtT0000")  # example uid, the last four characters encode the version of the transform
+
         """
+        from lamindb.models import Project
+
+        if project is not None:
+            project_record = Project.filter(
+                Q(name=project) | Q(uid=project)
+            ).one_or_none()
+            if project_record is None:
+                raise InvalidArgument(
+                    f"Project '{project}' not found, either create it with `ln.Project(name='...').save()` or fix typos."
+                )
+            self._project = project_record
         self._logging_message_track = ""
         self._logging_message_imports = ""
         if transform is not None and isinstance(transform, str):
@@ -370,6 +395,12 @@ class Context:
             )
         self._run = run
         track_environment(run)
+        if self.project is not None:
+            # to update a potential project link
+            # is only necessary if transform is loaded rather than newly created
+            # can be optimized by checking whether the transform is loaded, but it typically is
+            self.transform.save()
+        log_to_file = None
         if log_to_file is None:
             log_to_file = self.transform.type != "notebook"
         if log_to_file:
@@ -430,26 +461,23 @@ class Context:
             path_str = get_notebook_key_colab()
             path = Path(path_str)
         else:
-            import nbproject
+            from nbproject.dev import read_notebook
+            from nbproject.dev._meta_live import get_title
+            from nbproject.dev._pypackage import infer_pypackages
 
             try:
-                nbproject_title = nbproject.meta.live.title
-            except IndexError:
-                # notebook is not saved
-                pass
-            if nbproject_title is not None:
-                description = nbproject_title
-            # log imported python packages
-            try:
-                from nbproject.dev._pypackage import infer_pypackages
+                nb = read_notebook(path_str)
+
+                nbproject_title = get_title(nb)
+                if nbproject_title is not None:
+                    description = nbproject_title
 
-                nb = nbproject.dev.read_notebook(path_str)
                 self._logging_message_imports += (
                     "notebook imports:"
                     f" {pretty_pypackages(infer_pypackages(nb, pin_versions=True))}"
                 )
             except Exception:
-                logger.debug("inferring imported packages failed")
+                logger.debug("reading the notebook file failed")
                 pass
         return path, description
 

lamindb/core/_mapped_collection.py

@@ -27,7 +27,8 @@ if TYPE_CHECKING:
 class _Connect:
     def __init__(self, storage):
         if isinstance(storage, UPath):
-            self.conn, self.store = registry.open("h5py", storage)
+            # force no external compression even for files with .gz extension. REMOVE LATER
+            self.conn, self.store = registry.open("h5py", storage, compression=None)
             self.to_close = True
         else:
             self.conn, self.store = None, storage
@@ -246,7 +247,8 @@ class MappedCollection:
                 if parallel:
                     conn, storage = None, path
                 else:
-                    conn, storage = registry.open("h5py", path)
+                    # force no external compression even for files with .gz extension. REMOVE LATER
+                    conn, storage = registry.open("h5py", path, compression=None)
             else:
                 conn, storage = registry.open("zarr", path)
             self.conns.append(conn)

lamindb/core/_settings.py

@@ -155,12 +155,12 @@ class Settings:
     def verbosity(self) -> str:
         """Logger verbosity (default `'warning'`).
 
-        - `'error'`: ❌ only show error messages
-        - `'warning'`: ❗ also show warning messages
-        - `'success'`: ✅ also show success and save messages
-        - `'info'`: 💡 also show info messages
-        - `'hint'`: 💡 also show hint messages
-        - `'debug'`: 🐛 also show detailed debug messages
+        - `'error'`: only show error messages
+        - `'warning'`: also show warning messages
+        - `'success'`: also show success and save messages
+        - `'info'`: also show info messages
+        - `'hint'`: also show hint messages
+        - `'debug'`: also show detailed debug messages
         """
         return VERBOSITY_TO_STR[self._verbosity_int]
 

lamindb/core/_sync_git.py

@@ -7,7 +7,7 @@ from lamin_utils import logger
 from lamindb_setup import settings as setup_settings
 from lamindb_setup.core.hashing import hash_code
 
-from ._settings import sanitize_git_repo_url, settings
+from ..core._settings import sanitize_git_repo_url, settings
 
 
 class BlobHashNotFound(SystemExit):

lamindb/core/_track_environment.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import subprocess
+import sys
 from typing import TYPE_CHECKING
 
 import lamindb_setup as ln_setup
@@ -17,7 +18,7 @@ def track_environment(run: Run) -> None:
     try:
         with open(filepath, "w") as f:
             result = subprocess.run(
-                ["pip", "freeze"],
+                [sys.executable, "-m", "pip", "freeze"],
                 stdout=f,
             )
     except OSError as e:

lamindb/core/datasets/_small.py

@@ -23,7 +23,7 @@ def small_dataset1(
         var_ids[0]: [1, 2, 3],
         var_ids[1]: [3, 4, 5],
         var_ids[2]: [5, 6, 7],
-        "cell_medium": pd.Categorical(["DMSO", ifng, "DMSO"]),
+        "perturbation": pd.Categorical(["DMSO", ifng, "DMSO"]),
         "sample_note": ["was ok", "looks naah", "pretty! 🤩"],
         "cell_type_by_expert": pd.Categorical(["B cell", "T cell", "T cell"]),
         "cell_type_by_model": pd.Categorical(["B cell", "T cell", "T cell"]),
@@ -60,7 +60,7 @@ def small_dataset2(
         var_ids[0]: [2, 3, 3],
         var_ids[1]: [3, 4, 5],
         var_ids[2]: [4, 2, 3],
-        "cell_medium": pd.Categorical(["DMSO", "IFNG", "IFNG"]),
+        "perturbation": pd.Categorical(["DMSO", "IFNG", "IFNG"]),
         "cell_type_by_model": pd.Categorical(["B cell", "T cell", "T cell"]),
     }
     metadata = {
@@ -74,7 +74,7 @@ def small_dataset2(
     )
     ad.AnnData(
         dataset_df[var_ids],
-        obs=dataset_df[["cell_medium", "cell_type_by_model"]],
+        obs=dataset_df[["perturbation", "cell_type_by_model"]],
     )
     if otype == "DataFrame":
         for key, value in metadata.items():