lamindb 1.1.1__py3-none-any.whl → 1.2a2__py3-none-any.whl

lamindb/__init__.py

@@ -16,8 +16,8 @@ Registries.
 
    Artifact
    Transform
-   ULabel
    Run
+   ULabel
    User
    Storage
    Feature
@@ -25,9 +25,17 @@ Registries.
    Param
    Collection
    Project
+   Space
    Reference
    Person
 
+Curators & integrations.
+
+.. autosummary::
+
+   curators
+   integrations
+
 Key functionality.
 
 .. autosummary::
@@ -36,20 +44,19 @@ Key functionality.
    connect
    view
    save
+   UPath
+   settings
 
-Modules and settings.
+Low-level functionality.
 
 .. autosummary::
    :toctree: .
 
-   integrations
    context
-   curators
-   settings
    errors
    setup
-   UPath
    base
+   models
    core
 
 Backward compatibility.
@@ -62,8 +69,16 @@ Backward compatibility.
 
 """
 
+# ruff: noqa: I001
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "1.1.1"
+__version__ = "1.2a2"
+
+import warnings
+
+# through SpatialData
+warnings.filterwarnings(
+    "ignore", message="The legacy Dask DataFrame implementation is deprecated"
+)
 
 from lamindb_setup._check_setup import InstanceNotSetupError as _InstanceNotSetupError
 from lamindb_setup._check_setup import _check_instance_setup
@@ -79,23 +94,7 @@ def __getattr__(name):
 
 if _check_instance_setup(from_module="lamindb"):
     del __getattr__  # so that imports work out
-    from . import core  # isort: split
-    from . import (
-        _artifact,
-        _can_curate,
-        _collection,
-        _feature,
-        _is_versioned,
-        _parents,
-        _record,
-        _run,
-        _schema,
-        _storage,
-        _transform,
-        _ulabel,
-        integrations,
-    )
-    from ._save import save
+    from . import base
     from ._tracked import tracked
     from ._view import view
     from .core._context import context
@@ -111,12 +110,16 @@ if _check_instance_setup(from_module="lamindb"):
         Project,
         Reference,
         Run,
-        Schema,  # forward compat
+        Schema,
         Storage,
         Transform,
         ULabel,
         User,
+        Space,
     )
+    from .models.save import save
+    from . import core
+    from . import integrations
 
     track = context.track  # simple access
     finish = context.finish  # simple access

lamindb/_tracked.py

@@ -5,8 +5,8 @@ from datetime import datetime, timezone
 from typing import Callable, ParamSpec, TypeVar
 
 from .core._context import context
-from .core._feature_manager import infer_feature_type_convert_json
 from .models import Run, Transform
+from .models._feature_manager import infer_feature_type_convert_json
 
 P = ParamSpec("P")
 R = TypeVar("R")

lamindb/_view.py

@@ -9,10 +9,9 @@ from lamin_utils import colors, logger
 from lamindb_setup import settings
 from lamindb_setup._init_instance import get_schema_module_name
 
-from lamindb.core import FeatureValue, ParamValue
-from lamindb.models import Feature, Record
+from lamindb.models import Feature, FeatureValue, ParamValue, Record
 
-from ._feature import convert_pandas_dtype_to_lamin_dtype
+from .models.feature import convert_pandas_dtype_to_lamin_dtype
 
 if TYPE_CHECKING:
     import pandas as pd

lamindb/base/__init__.py

@@ -1,6 +1,6 @@
 """Base library.
 
-Is available also when no instance is connected.
+Is available also when no instance is setup.
 
 Modules:
 

lamindb/base/ids.py

@@ -1,4 +1,4 @@
-"""IDs.
+"""Universal IDs.
 
 Base generators:
 
@@ -55,15 +55,6 @@ def base62(n_char: int) -> str:
     return id
 
 
-# the following cannot be serialized by Django
-# class Base62:
-#     def __init__(self, n_char: int):
-#         self.n_char = n_char
-
-#     def __call__(self):
-#         return base62(self.n_char)
-
-
 def base26(n_char: int):
     """ASCII lowercase."""
     alphabet = string.ascii_lowercase

lamindb/core/__init__.py

@@ -1,49 +1,20 @@
 """Core library.
 
-Registries:
-
-.. autosummary::
-   :toctree: .
-
-   BasicRecord
-   Record
-   Registry
-   QuerySet
-   QueryManager
-   RecordList
-   FeatureManager
-   ParamManager
-   LabelManager
-   IsVersioned
-   CanCurate
-   HasParents
-   TracksRun
-   TracksUpdates
-   ParamValue
-   FeatureValue
-   InspectResult
-   ValidateFields
-   fields
-
-Curators:
+Settings & context:
 
 .. autosummary::
    :toctree: .
 
-   CatManager
-   DataFrameCatManager
-   AnnDataCatManager
-   MuDataCatManager
-   TiledbsomaCatManager
-   CurateLookup
+   Settings
+   subsettings
+   Context
 
-Settings & context:
+Artifact loaders:
 
 .. autosummary::
    :toctree: .
 
-   Settings
-   Context
+   loaders
 
 Data loaders:
 
@@ -57,10 +28,8 @@ Modules:
 .. autosummary::
    :toctree: .
 
-   loaders
    datasets
    storage
-   subsettings
    logger
 
 """
@@ -68,35 +37,8 @@ Modules:
 from lamin_utils import logger
 from lamin_utils._inspect import InspectResult
 
-from lamindb._query_manager import QueryManager
-from lamindb._query_set import QuerySet, RecordList
-from lamindb.core._feature_manager import FeatureManager, ParamManager
-from lamindb.core._label_manager import LabelManager
-from lamindb.curators import (
-    AnnDataCatManager,
-    CatManager,
-    CurateLookup,
-    Curator,
-    DataFrameCatManager,
-    MuDataCatManager,
-    TiledbsomaCatManager,
-)
-from lamindb.models import (
-    BasicRecord,
-    CanCurate,
-    FeatureValue,
-    HasParents,
-    IsVersioned,
-    ParamValue,
-    Record,
-    Registry,
-    TracksRun,
-    TracksUpdates,
-    ValidateFields,
-)
-
 from .. import errors as exceptions
-from . import _data, datasets, fields, loaders, subsettings, types
+from . import datasets, loaders, subsettings, types
 from ._context import Context
 from ._mapped_collection import MappedCollection
 from ._settings import Settings

lamindb/core/_context.py

@@ -11,7 +11,7 @@ from pathlib import Path
 from typing import TYPE_CHECKING
 
 import lamindb_setup as ln_setup
-from django.db.models import Func, IntegerField
+from django.db.models import Func, IntegerField, Q
 from lamin_utils import logger
 from lamindb_setup.core import deprecated
 from lamindb_setup.core.hashing import hash_file
@@ -20,21 +20,26 @@ from lamindb.base import ids
 from lamindb.base.ids import base62_12
 from lamindb.models import Run, Transform, format_field_value
 
+from ..core._settings import settings
 from ..errors import (
     InconsistentKey,
+    InvalidArgument,
     TrackNotCalled,
     UpdateContext,
 )
-from ._settings import settings
+from ..models._is_versioned import bump_version as bump_version_function
+from ..models._is_versioned import (
+    increment_base62,
+    message_update_key_in_version_family,
+)
 from ._sync_git import get_transform_reference_from_git_repo
 from ._track_environment import track_environment
-from .versioning import bump_version as bump_version_function
-from .versioning import increment_base62, message_update_key_in_version_family
 
 if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
 
     from lamindb.base.types import TransformType
+    from lamindb.models import Project
 
 is_run_from_ipython = getattr(builtins, "__IPYTHON__", False)
 
@@ -198,6 +203,7 @@ class Context:
         self._run: Run | None = None
         self._path: Path | None = None
         """A local path to the script that's running."""
+        self._project: Project | None = None
         self._logging_message_track: str = ""
         self._logging_message_imports: str = ""
         self._stream_tracker: LogStreamTracker = LogStreamTracker()
@@ -244,6 +250,11 @@ class Context:
     def version(self, value: str | None):
         self._version = value
 
+    @property
+    def project(self) -> Project | None:
+        """Project to label entities created during the run."""
+        return self._project
+
     @property
     def run(self) -> Run | None:
         """Managed run of context."""
@@ -253,10 +264,10 @@ class Context:
         self,
         transform: str | Transform | None = None,
         *,
+        project: str | None = None,
         params: dict | None = None,
         new_run: bool | None = None,
         path: str | None = None,
-        log_to_file: bool | None = None,
     ) -> None:
         """Track a global run of your Python session.
 
@@ -270,14 +281,12 @@ class Context:
 
         Args:
             transform: A transform `uid` or record. If `None`, creates a `uid`.
+            project: A project `name` or `uid` for labeling entities created during the run.
             params: A dictionary of parameters to track for the run.
             new_run: If `False`, loads the latest run of transform
                 (default notebook), if `True`, creates new run (default non-notebook).
             path: Filepath of notebook or script. Only needed if it can't be
                 automatically detected.
-            log_to_file: If `True`, logs stdout and stderr to a file and
-                saves the file within the current run (default non-notebook),
-                if `False`, does not log the output (default notebook).
 
         Examples:
 
@@ -290,6 +299,17 @@ class Context:
             >>> ln.track("Onv04I53OgtT0000")  # example uid, the last four characters encode the version of the transform
 
         """
+        from lamindb.models import Project
+
+        if project is not None:
+            project_record = Project.filter(
+                Q(name=project) | Q(uid=project)
+            ).one_or_none()
+            if project_record is None:
+                raise InvalidArgument(
+                    f"Project '{project}' not found, either create it with `ln.Project(name='...').save()` or fix typos."
+                )
+            self._project = project_record
         self._logging_message_track = ""
         self._logging_message_imports = ""
         if transform is not None and isinstance(transform, str):
@@ -375,6 +395,12 @@ class Context:
             )
         self._run = run
         track_environment(run)
+        if self.project is not None:
+            # to update a potential project link
+            # is only necessary if transform is loaded rather than newly created
+            # can be optimized by checking whether the transform is loaded, but it typically is
+            self.transform.save()
+        log_to_file = None
         if log_to_file is None:
             log_to_file = self.transform.type != "notebook"
         if log_to_file:

lamindb/core/_settings.py

@@ -155,12 +155,12 @@ class Settings:
     def verbosity(self) -> str:
         """Logger verbosity (default `'warning'`).
 
-        - `'error'`: ❌ only show error messages
-        - `'warning'`: ❗ also show warning messages
-        - `'success'`: ✅ also show success and save messages
-        - `'info'`: 💡 also show info messages
-        - `'hint'`: 💡 also show hint messages
-        - `'debug'`: 🐛 also show detailed debug messages
+        - `'error'`: only show error messages
+        - `'warning'`: also show warning messages
+        - `'success'`: also show success and save messages
+        - `'info'`: also show info messages
+        - `'hint'`: also show hint messages
+        - `'debug'`: also show detailed debug messages
         """
         return VERBOSITY_TO_STR[self._verbosity_int]
 

lamindb/core/_sync_git.py

@@ -7,7 +7,7 @@ from lamin_utils import logger
 from lamindb_setup import settings as setup_settings
 from lamindb_setup.core.hashing import hash_code
 
-from ._settings import sanitize_git_repo_url, settings
+from ..core._settings import sanitize_git_repo_url, settings
 
 
 class BlobHashNotFound(SystemExit):

lamindb/core/loaders.py

@@ -30,13 +30,13 @@ from lamindb_setup.core.upath import (
     infer_filesystem,
 )
 
-from ._settings import settings
+from ..core._settings import settings
 
 if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
 
 try:
-    from .storage._zarr import load_anndata_zarr
+    from ..core.storage._zarr import load_anndata_zarr
 except ImportError:
 
     def load_anndata_zarr(storepath):  # type: ignore
@@ -179,8 +179,6 @@ def load_to_memory(filepath: UPathStr, **kwargs):
     """
     filepath = create_path(filepath)
 
-    filepath = settings._storage_settings.cloud_to_local(filepath, print_progress=True)
-
     # infer the correct suffix when .gz is present
     suffixes = filepath.suffixes
     suffix = (
@@ -189,8 +187,11 @@ def load_to_memory(filepath: UPathStr, **kwargs):
         else filepath.suffix
     )
 
-    loader = FILE_LOADERS.get(suffix)
+    loader = FILE_LOADERS.get(suffix, None)
     if loader is None:
-        return filepath
-    else:
-        return loader(filepath, **kwargs)
+        raise NotImplementedError(
+            f"There is no loader for {suffix} files. Use .cache() to get the path."
+        )
+
+    filepath = settings._storage_settings.cloud_to_local(filepath, print_progress=True)
+    return loader(filepath, **kwargs)

lamindb/core/storage/_backed_access.py

@@ -5,8 +5,6 @@ from typing import TYPE_CHECKING, Any, Callable
 
 from anndata._io.specs.registry import get_spec
 
-from lamindb.models import Artifact
-
 from ._anndata_accessor import AnnDataAccessor, StorageType, registry
 from ._pyarrow_dataset import _is_pyarrow_dataset, _open_pyarrow_dataset
 from ._tiledbsoma import _open_tiledbsoma
@@ -19,6 +17,8 @@ if TYPE_CHECKING:
     from tiledbsoma import Experiment as SOMAExperiment
     from upath import UPath
 
+    from lamindb.models.artifact import Artifact
+
 
 # this dynamically creates a subclass of a context manager class
 # and reassigns it to an instance of the superclass
@@ -74,6 +74,8 @@ def backed_access(
 ) -> (
     AnnDataAccessor | BackedAccessor | SOMACollection | SOMAExperiment | PyArrowDataset
 ):
+    from lamindb.models import Artifact
+
     if isinstance(artifact_or_filepath, Artifact):
         objectpath, _ = filepath_from_artifact(
             artifact_or_filepath, using_key=using_key

lamindb/core/storage/_tiledbsoma.py

@@ -12,8 +12,6 @@ from lamindb_setup.core._settings_storage import get_storage_region
 from lamindb_setup.core.upath import LocalPathClasses, create_path
 from packaging import version
 
-from lamindb.models import Artifact, Run
-
 if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
     from tiledbsoma import Collection as SOMACollection
@@ -21,6 +19,9 @@ if TYPE_CHECKING:
     from tiledbsoma import Measurement as SOMAMeasurement
     from upath import UPath
 
+    from lamindb.models.artifact import Artifact
+    from lamindb.models.run import Run
+
 
 def _load_h5ad_zarr(objpath: UPath):
     from lamindb.core.loaders import load_anndata_zarr, load_h5ad
@@ -134,9 +135,10 @@ def save_tiledbsoma_experiment(
     except ImportError as e:
         raise ImportError("Please install tiledbsoma: pip install tiledbsoma") from e
 
-    from lamindb.core._data import get_run
     from lamindb.core.storage.paths import auto_storage_key_from_artifact_uid
-    from lamindb.core.versioning import create_uid
+    from lamindb.models import Artifact
+    from lamindb.models._is_versioned import create_uid
+    from lamindb.models.artifact import get_run
 
     run = get_run(run)
 

lamindb/core/storage/_zarr.py

@@ -1,14 +1,14 @@
 from __future__ import annotations
 
 import warnings
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Literal
 
 import scipy.sparse as sparse
 import zarr
 from anndata import __version__ as anndata_version
 from anndata._io.specs import write_elem
-from anndata._io.specs.registry import get_spec
 from fsspec.implementations.local import LocalFileSystem
+from lamin_utils import logger
 from lamindb_setup.core.upath import create_mapper, infer_filesystem
 from packaging import version
 
@@ -25,15 +25,36 @@ if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
 
 
-def zarr_is_adata(storepath: UPathStr) -> bool:
+def identify_zarr_type(
+    storepath: UPathStr, *, check: bool = True
+) -> Literal["anndata", "spatialdata", "unknown"]:
+    """Identify whether a zarr store is AnnData, SpatialData, or unknown type."""
+    # we can add these cheap suffix-based-checks later
+    # also need to check whether the .spatialdata.zarr suffix
+    # actually becomes a "standard"; currently we don't recognize it
+    # unlike ".anndata.zarr" in VALID_SUFFIXES
+    # suffixes = UPath(storepath).suffixes
+    # if ".spatialdata" in suffixes:
+    #     return "spatialdata"
+    # elif ".anndata" in suffixes:
+    #     return "anndata"
+
     fs, storepath_str = infer_filesystem(storepath)
+
     if isinstance(fs, LocalFileSystem):
-        # this is faster than through an fsspec mapper for local
         open_obj = storepath_str
     else:
-        open_obj = create_mapper(fs, storepath_str, check=True)
-    storage = zarr.open(open_obj, mode="r")
-    return get_spec(storage).encoding_type == "anndata"
+        open_obj = create_mapper(fs, storepath_str, check=check)
+
+    try:
+        storage = zarr.open(open_obj, mode="r")
+        if "spatialdata_attrs" in storage.attrs:
+            return "spatialdata"
+        if storage.attrs.get("encoding-type", "") == "anndata":
+            return "anndata"
+    except Exception as error:
+        logger.warning(f"an exception occured {error}")
+    return "unknown"
 
 
 def load_anndata_zarr(storepath: UPathStr) -> AnnData:
@@ -65,7 +86,7 @@ def write_adata_zarr(
     adata_size = None
     cumulative_val = 0
 
-    def _cb(key_write: str | None = None):
+    def _report_progress(key_write: str | None = None):
         nonlocal adata_size
         nonlocal cumulative_val
 
@@ -91,9 +112,9 @@ def write_adata_zarr(
 
     def _write_elem_cb(f, k, elem, dataset_kwargs):
         write_elem(f, k, elem, dataset_kwargs=dataset_kwargs)
-        _cb(k)
+        _report_progress(k)
 
-    _cb(None)
+    _report_progress(None)
     with warnings.catch_warnings():
         warnings.filterwarnings("ignore", category=UserWarning, module="zarr")
 
@@ -114,4 +135,4 @@ def write_adata_zarr(
             )
         _write_elem_cb(f, "raw", adata.raw, dataset_kwargs=dataset_kwargs)
     # todo: fix size less than total at the end
-    _cb(None)
+    _report_progress(None)