lamindb 1.1.0__py3-none-any.whl → 1.2a2__py3-none-any.whl

lamindb/__init__.py

@@ -1,12 +1,13 @@
 """A data framework for biology.
 
-Tracking notebooks & scripts.
+Tracking notebooks, scripts & functions.
 
 .. autosummary::
    :toctree: .
 
    track
    finish
+   tracked
 
 Registries.
 
@@ -15,8 +16,8 @@ Registries.
 
    Artifact
    Transform
-   ULabel
    Run
+   ULabel
    User
    Storage
    Feature
@@ -24,9 +25,17 @@ Registries.
    Param
    Collection
    Project
+   Space
    Reference
    Person
 
+Curators & integrations.
+
+.. autosummary::
+
+   curators
+   integrations
+
 Key functionality.
 
 .. autosummary::
@@ -35,20 +44,19 @@ Key functionality.
    connect
    view
    save
+   UPath
+   settings
 
-Modules and settings.
+Low-level functionality.
 
 .. autosummary::
    :toctree: .
 
-   integrations
    context
-   curators
-   settings
    errors
    setup
-   UPath
    base
+   models
    core
 
 Backward compatibility.
@@ -57,11 +65,20 @@ Backward compatibility.
    :toctree: .
 
    FeatureSet
+   Curator
 
 """
 
+# ruff: noqa: I001
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "1.1.0"
+__version__ = "1.2a2"
+
+import warnings
+
+# through SpatialData
+warnings.filterwarnings(
+    "ignore", message="The legacy Dask DataFrame implementation is deprecated"
+)
 
 from lamindb_setup._check_setup import InstanceNotSetupError as _InstanceNotSetupError
 from lamindb_setup._check_setup import _check_instance_setup
@@ -77,23 +94,7 @@ def __getattr__(name):
 
 if _check_instance_setup(from_module="lamindb"):
     del __getattr__  # so that imports work out
-    from . import core  # isort: split
-    from . import (
-        _artifact,
-        _can_curate,
-        _collection,
-        _feature,
-        _is_versioned,
-        _parents,
-        _record,
-        _run,
-        _schema,
-        _storage,
-        _transform,
-        _ulabel,
-        integrations,
-    )
-    from ._save import save
+    from . import base
     from ._tracked import tracked
     from ._view import view
     from .core._context import context
@@ -109,12 +110,16 @@ if _check_instance_setup(from_module="lamindb"):
         Project,
         Reference,
         Run,
-        Schema,  # forward compat
+        Schema,
         Storage,
         Transform,
         ULabel,
         User,
+        Space,
     )
+    from .models.save import save
+    from . import core
+    from . import integrations
 
     track = context.track  # simple access
     finish = context.finish  # simple access

lamindb/_finish.py

@@ -436,7 +436,15 @@ def save_context_core(
     # save both run & transform records if we arrive here
     if run is not None:
         run.save()
-    transform.save()
+    transform_id_prior_to_save = transform.id
+    transform.save()  # this in-place updates the state of transform upon hash collision
+    if transform.id != transform_id_prior_to_save:
+        # the hash existed and we're actually back to the previous version
+        # hence, this was in fact a run of the previous transform rather than of
+        # the new transform
+        # this can happen in interactive notebooks if the user makes no change to the notebook
+        run.transform = transform
+        run.save()
 
     # finalize
     if not from_cli and run is not None:

lamindb/_tracked.py

@@ -5,8 +5,8 @@ from datetime import datetime, timezone
 from typing import Callable, ParamSpec, TypeVar
 
 from .core._context import context
-from .core._feature_manager import infer_feature_type_convert_json
 from .models import Run, Transform
+from .models._feature_manager import infer_feature_type_convert_json
 
 P = ParamSpec("P")
 R = TypeVar("R")
@@ -26,10 +26,33 @@ def get_current_tracked_run() -> Run | None:
 
 
 def tracked(uid: str | None = None) -> Callable[[Callable[P, R]], Callable[P, R]]:
-    """Decorator that tracks function execution.
+    """Mark a function as tracked with this decorator.
+
+    You will be able to see inputs, outputs, and parameters of the function in the data lineage graph.
+
+    Guide: :doc:`/track`
+
+    .. versionadded:: 1.1.0
+        This is still in beta and will be refined in future releases.
 
     Args:
-        uid: Optional unique identifier for the transform
+        uid: Persist the uid to identify this transform across renames.
+
+    Example::
+
+        import lamindb as ln
+
+        @ln.tracked()
+        def subset_dataframe(
+            input_artifact_key: str,  # all arguments tracked as parameters of the function run
+            output_artifact_key: str,
+            subset_rows: int = 2,
+            subset_cols: int = 2,
+        ) -> None:
+            artifact = ln.Artifact.get(key=input_artifact_key)
+            df = artifact.load()  # auto-tracked as input
+            new_df = df.iloc[:subset_rows, :subset_cols]
+            ln.Artifact.from_df(new_df, key=output_artifact_key).save()  # auto-tracked as output
     """
 
     def decorator_tracked(func: Callable[P, R]) -> Callable[P, R]:

lamindb/_view.py

@@ -9,10 +9,9 @@ from lamin_utils import colors, logger
 from lamindb_setup import settings
 from lamindb_setup._init_instance import get_schema_module_name
 
-from lamindb.core import FeatureValue, ParamValue
-from lamindb.models import Feature, Record
+from lamindb.models import Feature, FeatureValue, ParamValue, Record
 
-from ._feature import convert_pandas_dtype_to_lamin_dtype
+from .models.feature import convert_pandas_dtype_to_lamin_dtype
 
 if TYPE_CHECKING:
     import pandas as pd

lamindb/base/__init__.py

@@ -1,6 +1,6 @@
 """Base library.
 
-Is available also when no instance is connected.
+Is available also when no instance is setup.
 
 Modules:
 

lamindb/base/ids.py

@@ -1,4 +1,4 @@
-"""IDs.
+"""Universal IDs.
 
 Base generators:
 
@@ -55,15 +55,6 @@ def base62(n_char: int) -> str:
     return id
 
 
-# the following cannot be serialized by Django
-# class Base62:
-#     def __init__(self, n_char: int):
-#         self.n_char = n_char
-
-#     def __call__(self):
-#         return base62(self.n_char)
-
-
 def base26(n_char: int):
     """ASCII lowercase."""
     alphabet = string.ascii_lowercase

lamindb/base/users.py

@@ -12,12 +12,9 @@ def current_user_id() -> int:
         if ln_setup.core.django.IS_MIGRATING:
             return 1
         else:
-            exc_attr = (
-                "DoesNotExist" if hasattr(User, "DoesNotExist") else "_DoesNotExist"
-            )
             try:
                 user_id = User.objects.get(uid=settings.user.uid).id
-            except getattr(User, exc_attr):
+            except User.DoesNotExist:
                 register_user(settings.user)
                 user_id = User.objects.get(uid=settings.user.uid).id
             return user_id

lamindb/core/__init__.py

@@ -1,49 +1,20 @@
 """Core library.
 
-Registries:
-
-.. autosummary::
-   :toctree: .
-
-   BasicRecord
-   Record
-   Registry
-   QuerySet
-   QueryManager
-   RecordList
-   FeatureManager
-   ParamManager
-   LabelManager
-   IsVersioned
-   CanCurate
-   HasParents
-   TracksRun
-   TracksUpdates
-   ParamValue
-   FeatureValue
-   InspectResult
-   ValidateFields
-   fields
-
-Curators:
+Settings & context:
 
 .. autosummary::
    :toctree: .
 
-   CatManager
-   DataFrameCatManager
-   AnnDataCatManager
-   MuDataCatManager
-   TiledbsomaCatManager
-   CurateLookup
+   Settings
+   subsettings
+   Context
 
-Settings & context:
+Artifact loaders:
 
 .. autosummary::
    :toctree: .
 
-   Settings
-   Context
+   loaders
 
 Data loaders:
 
@@ -57,10 +28,8 @@ Modules:
 .. autosummary::
    :toctree: .
 
-   loaders
    datasets
    storage
-   subsettings
    logger
 
 """
@@ -68,35 +37,8 @@ Modules:
 from lamin_utils import logger
 from lamin_utils._inspect import InspectResult
 
-from lamindb._query_manager import QueryManager
-from lamindb._query_set import QuerySet, RecordList
-from lamindb.core._feature_manager import FeatureManager, ParamManager
-from lamindb.core._label_manager import LabelManager
-from lamindb.curators import (
-    AnnDataCatManager,
-    CatManager,
-    CurateLookup,
-    Curator,
-    DataFrameCatManager,
-    MuDataCatManager,
-    TiledbsomaCatManager,
-)
-from lamindb.models import (
-    BasicRecord,
-    CanCurate,
-    FeatureValue,
-    HasParents,
-    IsVersioned,
-    ParamValue,
-    Record,
-    Registry,
-    TracksRun,
-    TracksUpdates,
-    ValidateFields,
-)
-
 from .. import errors as exceptions
-from . import _data, datasets, fields, loaders, subsettings, types
+from . import datasets, loaders, subsettings, types
 from ._context import Context
 from ._mapped_collection import MappedCollection
 from ._settings import Settings

lamindb/core/_context.py

@@ -11,29 +11,35 @@ from pathlib import Path
 from typing import TYPE_CHECKING
 
 import lamindb_setup as ln_setup
-from django.db.models import Func, IntegerField
+from django.db.models import Func, IntegerField, Q
 from lamin_utils import logger
+from lamindb_setup.core import deprecated
 from lamindb_setup.core.hashing import hash_file
 
 from lamindb.base import ids
 from lamindb.base.ids import base62_12
 from lamindb.models import Run, Transform, format_field_value
 
+from ..core._settings import settings
 from ..errors import (
     InconsistentKey,
+    InvalidArgument,
     TrackNotCalled,
     UpdateContext,
 )
-from ._settings import settings
+from ..models._is_versioned import bump_version as bump_version_function
+from ..models._is_versioned import (
+    increment_base62,
+    message_update_key_in_version_family,
+)
 from ._sync_git import get_transform_reference_from_git_repo
 from ._track_environment import track_environment
-from .versioning import bump_version as bump_version_function
-from .versioning import increment_base62, message_update_key_in_version_family
 
 if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
 
     from lamindb.base.types import TransformType
+    from lamindb.models import Project
 
 is_run_from_ipython = getattr(builtins, "__IPYTHON__", False)
 
@@ -197,6 +203,7 @@ class Context:
         self._run: Run | None = None
         self._path: Path | None = None
         """A local path to the script that's running."""
+        self._project: Project | None = None
         self._logging_message_track: str = ""
         self._logging_message_imports: str = ""
         self._stream_tracker: LogStreamTracker = LogStreamTracker()
@@ -217,8 +224,8 @@ class Context:
         self._description = value
 
     @property
+    @deprecated(new_name="description")
     def name(self) -> str | None:
-        """Deprecated. Populates `description` argument for `context.transform`."""
         return self._description
 
     @name.setter
@@ -243,6 +250,11 @@ class Context:
     def version(self, value: str | None):
         self._version = value
 
+    @property
+    def project(self) -> Project | None:
+        """Project to label entities created during the run."""
+        return self._project
+
     @property
     def run(self) -> Run | None:
         """Managed run of context."""
@@ -252,12 +264,12 @@ class Context:
         self,
         transform: str | Transform | None = None,
         *,
+        project: str | None = None,
         params: dict | None = None,
         new_run: bool | None = None,
         path: str | None = None,
-        log_to_file: bool | None = None,
     ) -> None:
-        """Initiate a run with tracked data lineage.
+        """Track a global run of your Python session.
 
         - sets :attr:`~lamindb.core.Context.transform` &
           :attr:`~lamindb.core.Context.run` by creating or loading `Transform` &
@@ -269,14 +281,12 @@ class Context:
 
         Args:
             transform: A transform `uid` or record. If `None`, creates a `uid`.
+            project: A project `name` or `uid` for labeling entities created during the run.
             params: A dictionary of parameters to track for the run.
             new_run: If `False`, loads the latest run of transform
                 (default notebook), if `True`, creates new run (default non-notebook).
             path: Filepath of notebook or script. Only needed if it can't be
                 automatically detected.
-            log_to_file: If `True`, logs stdout and stderr to a file and
-                saves the file within the current run (default non-notebook),
-                if `False`, does not log the output (default notebook).
 
         Examples:
 
@@ -284,7 +294,22 @@ class Context:
 
             >>> ln.track()
 
+            If you want to ensure a single version history across renames of the notebook or script, pass the auto-generated `uid` that you'll find in the logs:
+
+            >>> ln.track("Onv04I53OgtT0000")  # example uid, the last four characters encode the version of the transform
+
         """
+        from lamindb.models import Project
+
+        if project is not None:
+            project_record = Project.filter(
+                Q(name=project) | Q(uid=project)
+            ).one_or_none()
+            if project_record is None:
+                raise InvalidArgument(
+                    f"Project '{project}' not found, either create it with `ln.Project(name='...').save()` or fix typos."
+                )
+            self._project = project_record
         self._logging_message_track = ""
         self._logging_message_imports = ""
         if transform is not None and isinstance(transform, str):
@@ -370,6 +395,12 @@ class Context:
             )
         self._run = run
         track_environment(run)
+        if self.project is not None:
+            # to update a potential project link
+            # is only necessary if transform is loaded rather than newly created
+            # can be optimized by checking whether the transform is loaded, but it typically is
+            self.transform.save()
+        log_to_file = None
         if log_to_file is None:
             log_to_file = self.transform.type != "notebook"
         if log_to_file:

lamindb/core/_mapped_collection.py

@@ -27,7 +27,8 @@ if TYPE_CHECKING:
 class _Connect:
     def __init__(self, storage):
         if isinstance(storage, UPath):
-            self.conn, self.store = registry.open("h5py", storage)
+            # force no external compression even for files with .gz extension. REMOVE LATER
+            self.conn, self.store = registry.open("h5py", storage, compression=None)
             self.to_close = True
         else:
             self.conn, self.store = None, storage
@@ -246,7 +247,8 @@ class MappedCollection:
                 if parallel:
                     conn, storage = None, path
                 else:
-                    conn, storage = registry.open("h5py", path)
+                    # force no external compression even for files with .gz extension. REMOVE LATER
+                    conn, storage = registry.open("h5py", path, compression=None)
             else:
                 conn, storage = registry.open("zarr", path)
             self.conns.append(conn)

lamindb/core/_settings.py

@@ -155,12 +155,12 @@ class Settings:
     def verbosity(self) -> str:
         """Logger verbosity (default `'warning'`).
 
-        - `'error'`: ❌ only show error messages
-        - `'warning'`: ❗ also show warning messages
-        - `'success'`: ✅ also show success and save messages
-        - `'info'`: 💡 also show info messages
-        - `'hint'`: 💡 also show hint messages
-        - `'debug'`: 🐛 also show detailed debug messages
+        - `'error'`: only show error messages
+        - `'warning'`: also show warning messages
+        - `'success'`: also show success and save messages
+        - `'info'`: also show info messages
+        - `'hint'`: also show hint messages
+        - `'debug'`: also show detailed debug messages
         """
         return VERBOSITY_TO_STR[self._verbosity_int]
 

lamindb/core/_sync_git.py

@@ -7,7 +7,7 @@ from lamin_utils import logger
 from lamindb_setup import settings as setup_settings
 from lamindb_setup.core.hashing import hash_code
 
-from ._settings import sanitize_git_repo_url, settings
+from ..core._settings import sanitize_git_repo_url, settings
 
 
 class BlobHashNotFound(SystemExit):

lamindb/core/_track_environment.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import subprocess
+import sys
 from typing import TYPE_CHECKING
 
 import lamindb_setup as ln_setup
@@ -17,7 +18,7 @@ def track_environment(run: Run) -> None:
     try:
         with open(filepath, "w") as f:
             result = subprocess.run(
-                ["pip", "freeze"],
+                [sys.executable, "-m", "pip", "freeze"],
                 stdout=f,
             )
     except OSError as e:

lamindb/core/datasets/_small.py

@@ -23,7 +23,7 @@ def small_dataset1(
         var_ids[0]: [1, 2, 3],
         var_ids[1]: [3, 4, 5],
         var_ids[2]: [5, 6, 7],
-        "cell_medium": pd.Categorical(["DMSO", ifng, "DMSO"]),
+        "perturbation": pd.Categorical(["DMSO", ifng, "DMSO"]),
         "sample_note": ["was ok", "looks naah", "pretty! 🤩"],
         "cell_type_by_expert": pd.Categorical(["B cell", "T cell", "T cell"]),
         "cell_type_by_model": pd.Categorical(["B cell", "T cell", "T cell"]),
@@ -60,7 +60,7 @@ def small_dataset2(
         var_ids[0]: [2, 3, 3],
         var_ids[1]: [3, 4, 5],
         var_ids[2]: [4, 2, 3],
-        "cell_medium": pd.Categorical(["DMSO", "IFNG", "IFNG"]),
+        "perturbation": pd.Categorical(["DMSO", "IFNG", "IFNG"]),
         "cell_type_by_model": pd.Categorical(["B cell", "T cell", "T cell"]),
     }
     metadata = {
@@ -74,7 +74,7 @@ def small_dataset2(
     )
     ad.AnnData(
         dataset_df[var_ids],
-        obs=dataset_df[["cell_medium", "cell_type_by_model"]],
+        obs=dataset_df[["perturbation", "cell_type_by_model"]],
     )
     if otype == "DataFrame":
         for key, value in metadata.items():

lamindb/core/loaders.py

@@ -30,13 +30,13 @@ from lamindb_setup.core.upath import (
     infer_filesystem,
 )
 
-from ._settings import settings
+from ..core._settings import settings
 
 if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
 
 try:
-    from .storage._zarr import load_anndata_zarr
+    from ..core.storage._zarr import load_anndata_zarr
 except ImportError:
 
     def load_anndata_zarr(storepath):  # type: ignore
@@ -65,8 +65,8 @@ def load_tsv(path: UPathStr, **kwargs) -> pd.DataFrame:
 def load_h5ad(filepath, **kwargs) -> ad.AnnData:
     """Load an `.h5ad` file to `AnnData`."""
     fs, filepath = infer_filesystem(filepath)
-
-    with fs.open(filepath, mode="rb") as file:
+    compression = kwargs.pop("compression", "infer")
+    with fs.open(filepath, mode="rb", compression=compression) as file:
         adata = ad.read_h5ad(file, backed=False, **kwargs)
         return adata
 
@@ -148,9 +148,13 @@ def load_rds(path: UPathStr) -> UPathStr:
 
 FILE_LOADERS = {
     ".csv": pd.read_csv,
+    ".csv.gz": pd.read_csv,
     ".tsv": load_tsv,
+    ".tsv.gz": load_tsv,
     ".h5ad": load_h5ad,
+    ".h5ad.gz": load_h5ad,
     ".parquet": pd.read_parquet,
+    ".parquet.gz": pd.read_parquet,  # this doesn't work for externally gzipped files, REMOVE LATER
     ".fcs": load_fcs,
     ".zarr": load_anndata_zarr,
     ".html": load_html,
@@ -175,10 +179,19 @@ def load_to_memory(filepath: UPathStr, **kwargs):
     """
     filepath = create_path(filepath)
 
-    filepath = settings._storage_settings.cloud_to_local(filepath, print_progress=True)
+    # infer the correct suffix when .gz is present
+    suffixes = filepath.suffixes
+    suffix = (
+        "".join(suffixes[-2:])
+        if len(suffixes) > 1 and ".gz" in suffixes
+        else filepath.suffix
+    )
 
-    loader = FILE_LOADERS.get(filepath.suffix)
+    loader = FILE_LOADERS.get(suffix, None)
     if loader is None:
-        return filepath
-    else:
-        return loader(filepath, **kwargs)
+        raise NotImplementedError(
+            f"There is no loader for {suffix} files. Use .cache() to get the path."
+        )
+
+    filepath = settings._storage_settings.cloud_to_local(filepath, print_progress=True)
+    return loader(filepath, **kwargs)

lamindb/core/storage/_anndata_accessor.py

@@ -16,6 +16,7 @@ from anndata._io.h5ad import read_dataframe_legacy as read_dataframe_legacy_h5
 from anndata._io.specs.registry import get_spec, read_elem, read_elem_partial
 from anndata.compat import _read_attr
 from fsspec.implementations.local import LocalFileSystem
+from fsspec.utils import infer_compression
 from lamin_utils import logger
 from lamindb_setup.core.upath import create_mapper, infer_filesystem
 from packaging import version
@@ -152,9 +153,13 @@ registry = AccessRegistry()
 
 
 @registry.register_open("h5py")
-def open(filepath: UPathStr, mode: str = "r"):
+def open(filepath: UPathStr, mode: str = "r", compression: str | None = "infer"):
     fs, file_path_str = infer_filesystem(filepath)
-    if isinstance(fs, LocalFileSystem):
+    # we don't open compressed files directly because we need fsspec to uncompress on .open
+    compression = (
+        infer_compression(file_path_str) if compression == "infer" else compression
+    )
+    if isinstance(fs, LocalFileSystem) and compression is None:
         assert mode in {"r", "r+", "a", "w", "w-"}, f"Unknown mode {mode}!"  #  noqa: S101
         return None, h5py.File(file_path_str, mode=mode)
     if mode == "r":
@@ -165,7 +170,7 @@ def open(filepath: UPathStr, mode: str = "r"):
         conn_mode = "ab"
     else:
         raise ValueError(f"Unknown mode {mode}! Should be 'r', 'w' or 'a'.")
-    conn = fs.open(file_path_str, mode=conn_mode)
+    conn = fs.open(file_path_str, mode=conn_mode, compression=compression)
     try:
         storage = h5py.File(conn, mode=mode)
     except Exception as e: