lamindb 1.0.5__py3-none-any.whl → 1.1.1__py3-none-any.whl

lamindb/_schema.py

@@ -10,21 +10,23 @@ from lamindb_setup.core.hashing import hash_set
 
 from lamindb.base import ids
 from lamindb.base.types import FieldAttr, ListLike
+from lamindb.errors import InvalidArgument
 from lamindb.models import Feature, Record, Schema
 
-from ._feature import convert_pandas_dtype_to_lamin_dtype
-from ._record import init_self_from_db
+from ._feature import convert_pandas_dtype_to_lamin_dtype, get_dtype_str_from_dtype
+from ._record import init_self_from_db, update_attributes
 from ._utils import attach_func_to_class_method
-from .core.exceptions import ValidationError
 from .core.relations import (
     dict_related_model_to_related_name,
     get_related_name,
 )
+from .errors import ValidationError
 
 if TYPE_CHECKING:
     from collections.abc import Iterable
 
     import pandas as pd
+    from django.db.models.query_utils import DeferredAttribute
 
     from ._query_set import QuerySet
 
@@ -60,46 +62,136 @@ def __init__(self, *args, **kwargs):
     if len(args) == len(self._meta.concrete_fields):
         super(Schema, self).__init__(*args, **kwargs)
         return None
-    # now we proceed with the user-facing constructor
     if len(args) > 1:
         raise ValueError("Only one non-keyword arg allowed: features")
-    features: Iterable[Record] = kwargs.pop("features") if len(args) == 0 else args[0]
-    dtype: str | None = kwargs.pop("dtype") if "dtype" in kwargs else None
-    name: str | None = kwargs.pop("name") if "name" in kwargs else None
-    if len(kwargs) > 0:
-        raise ValueError("Only features, dtype, name are valid keyword arguments")
-    # now code
-    features_registry = validate_features(features)
+
+    features: Iterable[Record] | None = args[0] if args else kwargs.pop("features", [])
+    # typing here anticipates transitioning to a ManyToMany
+    # between composites and components similar to feature_sets
+    # in lamindb v2
+    components: dict[str, Schema] = kwargs.pop("components", {})
+    name: str | None = kwargs.pop("name", None)
+    description: str | None = kwargs.pop("description", None)
+    dtype: str | None = kwargs.pop("dtype", None)
+    itype: str | Record | DeferredAttribute | None = kwargs.pop("itype", None)
+    type: Feature | None = kwargs.pop("type", None)
+    is_type: bool = kwargs.pop("is_type", False)
+    otype: str | None = kwargs.pop("otype", None)
+    minimal_set: bool = kwargs.pop("minimal_set", True)
+    ordered_set: bool = kwargs.pop("ordered_set", False)
+    maximal_set: bool = kwargs.pop("maximal_set", False)
+    slot: str | None = kwargs.pop("slot", None)
+    coerce_dtype: bool | None = kwargs.pop("coerce_dtype", None)
+
+    if kwargs:
+        raise ValueError(
+            f"Unexpected keyword arguments: {', '.join(kwargs.keys())}\n"
+            "Valid arguments are: features, description, dtype, itype, type, "
+            "is_type, otype, minimal_set, ordered_set, maximal_set, "
+            "slot, validated_by, coerce_dtype"
+        )
+
+    if features:
+        features_registry = validate_features(features)
+        itype_compare = features_registry.__get_name_with_module__()
+        if itype is not None:
+            assert itype == itype_compare, str(itype_compare)  # noqa: S101
+        else:
+            itype = itype_compare
+        n_features = len(features)
+    else:
+        n_features = -1
     if dtype is None:
-        dtype = None if features_registry == Feature else NUMBER_TYPE
-    n_features = len(features)
-    features_hash = hash_set({feature.uid for feature in features})
-    schema = Schema.filter(hash=features_hash).one_or_none()
+        dtype = None if itype is not None and itype == "Feature" else NUMBER_TYPE
+    else:
+        dtype = get_type_str(dtype)
+    components: dict[str, Schema]
+    if components:
+        itype = "Composite"
+        if otype is None:
+            raise InvalidArgument("Please pass otype != None for composite schemas")
+    if itype is not None and not isinstance(itype, str):
+        itype_str = get_dtype_str_from_dtype(itype, is_itype=True)
+    else:
+        itype_str = itype
+    validated_kwargs = {
+        "name": name,
+        "description": description,
+        "type": type,
+        "dtype": dtype,
+        "is_type": is_type,
+        "otype": otype,
+        "n": n_features,
+        "itype": itype_str,
+        "minimal_set": minimal_set,
+        "ordered_set": ordered_set,
+        "maximal_set": maximal_set,
+    }
+    if coerce_dtype:
+        validated_kwargs["_aux"] = {"af": {"0": coerce_dtype}}
+    if features:
+        hash = hash_set({feature.uid for feature in features})
+    elif components:
+        hash = hash_set({component.hash for component in components.values()})
+    else:
+        hash = hash_set({str(value) for value in validated_kwargs.values()})
+    validated_kwargs["hash"] = hash
+    validated_kwargs["slot"] = slot
+    schema = Schema.filter(hash=hash).one_or_none()
     if schema is not None:
-        logger.debug(f"loaded: {schema}")
+        logger.important(f"returning existing schema with same hash: {schema}")
         init_self_from_db(self, schema)
+        update_attributes(self, validated_kwargs)
         return None
-    else:
-        hash = features_hash
-    self._features = (get_related_name(features_registry), features)
-
-    super(Schema, self).__init__(
-        uid=ids.base62_20(),
-        name=name,
-        dtype=get_type_str(dtype),
-        n=n_features,
-        registry=features_registry.__get_name_with_module__(),
-        hash=hash,
-    )
+    if features:
+        self._features = (get_related_name(features_registry), features)
+    elif components:
+        for slot, component in components.items():
+            if component._state.adding:
+                raise InvalidArgument(
+                    f"component {slot} {component} must be saved before use"
+                )
+        self._components = components
+    validated_kwargs["uid"] = ids.base62_20()
+    super(Schema, self).__init__(**validated_kwargs)
 
 
 @doc_args(Schema.save.__doc__)
 def save(self, *args, **kwargs) -> Schema:
     """{}"""  # noqa: D415
+    from lamindb._save import bulk_create
+
     super(Schema, self).save(*args, **kwargs)
+    if hasattr(self, "_components"):
+        # analogous to save_schema_links in core._data.py
+        # which is called to save feature sets in artifact.save()
+        links = []
+        for slot, component in self._components.items():
+            kwargs = {
+                "composite_id": self.id,
+                "component_id": component.id,
+                "slot": slot,
+            }
+            links.append(Schema.components.through(**kwargs))
+        bulk_create(links, ignore_conflicts=True)
     if hasattr(self, "_features"):
+        assert self.n > 0  # noqa: S101
         related_name, records = self._features
-        getattr(self, related_name).set(records)
+        # only the following method preserves the order
+        # .set() does not preserve the order but orders by
+        # the feature primary key
+        through_model = getattr(self, related_name).through
+        related_model_split = self.itype.split(".")
+        if len(related_model_split) == 1:
+            related_field = related_model_split[0].lower()
+        else:
+            related_field = related_model_split[1].lower()
+        related_field_id = f"{related_field}_id"
+        links = [
+            through_model(**{"schema_id": self.id, related_field_id: record.id})
+            for record in records
+        ]
+        through_model.objects.bulk_create(links, ignore_conflicts=True)
     return self
 
 
@@ -181,10 +273,10 @@ def from_df(
             logger.warning("no validated features, skip creating feature set")
         return None
     if registry == Feature:
-        validated_features = Feature.from_values(
+        validated_features = Feature.from_values(  # type: ignore
             df.columns, field=field, organism=organism
         )
-        schema = Schema(validated_features, name=name, dtype=None)
+        schema = Schema(validated_features, name=name, dtype=None, otype="DataFrame")
     else:
         dtypes = [col.dtype for (_, col) in df.loc[:, validated].items()]
         if len(set(dtypes)) != 1:
@@ -200,6 +292,7 @@ def from_df(
             features=validated_features,
             name=name,
             dtype=get_type_str(dtype),
+            otype="DataFrame",
         )
     return schema
 
@@ -215,14 +308,12 @@ def members(self) -> QuerySet:
     related_name = self._get_related_name()
     if related_name is None:
         related_name = "features"
-    return self.__getattribute__(related_name).all()
+    return self.__getattribute__(related_name).order_by("links_schema__id")
 
 
 def _get_related_name(self: Schema) -> str:
-    _schemas_m2m_related_models = dict_related_model_to_related_name(
-        self, instance=self._state.db
-    )
-    related_name = _schemas_m2m_related_models.get(self.itype)
+    related_models = dict_related_model_to_related_name(self, instance=self._state.db)
+    related_name = related_models.get(self.itype)
     return related_name
 
 
@@ -245,6 +336,12 @@ if ln_setup._TESTING:
 for name in METHOD_NAMES:
     attach_func_to_class_method(name, Schema, globals())
 
-Schema.members = members
+Schema.members = members  # type: ignore
 Schema._get_related_name = _get_related_name
-Schema.feature_sets = Schema._artifacts_m2m  # backward compat
+# excluded on docs via
+# https://github.com/laminlabs/lndocs/blob/8c1963de65445107ea69b3fd59354c3828e067d1/lndocs/lamin_sphinx/__init__.py#L584-L588
+delattr(Schema, "validated_by")  # we don't want to expose these
+delattr(Schema, "validated_by_id")  # we don't want to expose these
+delattr(Schema, "validated_schemas")  # we don't want to expose these
+delattr(Schema, "composite")  # we don't want to expose these
+delattr(Schema, "composite_id")  # we don't want to expose these

lamindb/_storage.py

@@ -12,4 +12,4 @@ def path(self) -> UPath:
     return create_path(self.root, access_token=access_token)
 
 
-Storage.path = path
+Storage.path = path  # type: ignore

lamindb/_tracked.py

@@ -0,0 +1,129 @@
+import functools
+import inspect
+from contextvars import ContextVar
+from datetime import datetime, timezone
+from typing import Callable, ParamSpec, TypeVar
+
+from .core._context import context
+from .core._feature_manager import infer_feature_type_convert_json
+from .models import Run, Transform
+
+P = ParamSpec("P")
+R = TypeVar("R")
+
+# Create a context variable to store the current tracked run
+current_tracked_run: ContextVar[Run | None] = ContextVar(
+    "current_tracked_run", default=None
+)
+
+
+def get_current_tracked_run() -> Run | None:
+    """Get the run object."""
+    run = current_tracked_run.get()
+    if run is None:
+        run = context.run
+    return run
+
+
+def tracked(uid: str | None = None) -> Callable[[Callable[P, R]], Callable[P, R]]:
+    """Mark a function as tracked with this decorator.
+
+    You will be able to see inputs, outputs, and parameters of the function in the data lineage graph.
+
+    Guide: :doc:`/track`
+
+    .. versionadded:: 1.1.0
+        This is still in beta and will be refined in future releases.
+
+    Args:
+        uid: Persist the uid to identify this transform across renames.
+
+    Example::
+
+        import lamindb as ln
+
+        @ln.tracked()
+        def subset_dataframe(
+            input_artifact_key: str,  # all arguments tracked as parameters of the function run
+            output_artifact_key: str,
+            subset_rows: int = 2,
+            subset_cols: int = 2,
+        ) -> None:
+            artifact = ln.Artifact.get(key=input_artifact_key)
+            df = artifact.load()  # auto-tracked as input
+            new_df = df.iloc[:subset_rows, :subset_cols]
+            ln.Artifact.from_df(new_df, key=output_artifact_key).save()  # auto-tracked as output
+    """
+
+    def decorator_tracked(func: Callable[P, R]) -> Callable[P, R]:
+        # Get the original signature
+        sig = inspect.signature(func)
+
+        @functools.wraps(func)
+        def wrapper_tracked(*args: P.args, **kwargs: P.kwargs) -> R:
+            # Get function metadata
+            source_code = inspect.getsource(func)
+
+            initiated_by_run = get_current_tracked_run()
+            if initiated_by_run is None:
+                if context.run is None:
+                    raise RuntimeError(
+                        "Please track the global run context before using @ln.tracked(): ln.track()"
+                    )
+                initiated_by_run = context.run
+
+            # Get fully qualified function name
+            module_name = func.__module__
+            if module_name in {"__main__", "__mp_main__"}:
+                qualified_name = (
+                    f"{initiated_by_run.transform.key}/{func.__qualname__}.py"
+                )
+            else:
+                qualified_name = f"{module_name}.{func.__qualname__}.py"
+
+            # Create transform and run objects
+            transform = Transform(  # type: ignore
+                uid=uid,
+                key=qualified_name,
+                type="function",
+                source_code=source_code,
+            ).save()
+
+            run = Run(transform=transform, initiated_by_run=initiated_by_run)  # type: ignore
+            run.started_at = datetime.now(timezone.utc)
+            run.save()
+
+            # Bind arguments to get a mapping of parameter names to values
+            bound_args = sig.bind(*args, **kwargs)
+            bound_args.apply_defaults()
+            params = dict(bound_args.arguments)
+
+            # Remove the run parameter if it exists (we'll inject our own)
+            params.pop("run", None)
+
+            # Deal with non-trivial parameter values
+            filtered_params = {}
+            for key, value in params.items():
+                dtype, _, _ = infer_feature_type_convert_json(
+                    key, value, str_as_ulabel=False
+                )
+                if (dtype == "?" or dtype.startswith("cat")) and dtype != "cat ? str":
+                    continue
+                filtered_params[key] = value
+
+            # Add parameters to the run
+            run.params.add_values(filtered_params)
+
+            # Set the run in context and execute function
+            token = current_tracked_run.set(run)
+            try:
+                result = func(*args, **kwargs)
+                run.finished_at = datetime.now(timezone.utc)
+                run.save()
+                return result
+            finally:
+                current_tracked_run.reset(token)
+
+        return wrapper_tracked
+
+    return decorator_tracked

lamindb/_transform.py

@@ -5,14 +5,16 @@ from typing import TYPE_CHECKING
 
 from lamin_utils import logger
 from lamindb_setup.core._docs import doc_args
+from lamindb_setup.core.hashing import hash_string
 
 from lamindb.models import Run, Transform
 
 from ._parents import _view_parents
+from ._record import init_self_from_db, update_attributes
 from ._run import delete_run_artifacts
 from .core._settings import settings
-from .core.exceptions import InconsistentKey
 from .core.versioning import message_update_key_in_version_family, process_revises
+from .errors import InconsistentKey
 
 if TYPE_CHECKING:
     from lamindb.base.types import TransformType
@@ -54,6 +56,9 @@ def __init__(transform: Transform, *args, **kwargs):
             )
     # below is internal use that we'll hopefully be able to eliminate
     uid: str | None = kwargs.pop("uid") if "uid" in kwargs else None
+    source_code: str | None = (
+        kwargs.pop("source_code") if "source_code" in kwargs else None
+    )
     if not len(kwargs) == 0:
         raise ValueError(
             "Only key, description, version, type, revises, reference, "
@@ -84,8 +89,6 @@ def __init__(transform: Transform, *args, **kwargs):
                     )
                     uid = revises.uid
     if revises is not None and uid is not None and uid == revises.uid:
-        from ._record import init_self_from_db, update_attributes
-
         if revises.key != key:
             logger.warning("ignoring inconsistent key")
         init_self_from_db(transform, revises)
@@ -111,7 +114,15 @@ def __init__(transform: Transform, *args, **kwargs):
         uid = new_uid
     else:
         has_consciously_provided_uid = True
-    super(Transform, transform).__init__(
+    hash = None
+    if source_code is not None:
+        hash = hash_string(source_code)
+        transform_candidate = Transform.filter(hash=hash, is_latest=True).one_or_none()
+        if transform_candidate is not None:
+            init_self_from_db(transform, transform_candidate)
+            update_attributes(transform, {"key": key, "description": description})
+            return None
+    super(Transform, transform).__init__(  # type: ignore
         uid=uid,
         description=description,
         key=key,
@@ -119,6 +130,8 @@ def __init__(transform: Transform, *args, **kwargs):
         version=version,
         reference=reference,
         reference_type=reference_type,
+        source_code=source_code,
+        hash=hash,
         _has_consciously_provided_uid=has_consciously_provided_uid,
         revises=revises,
     )
@@ -151,7 +164,7 @@ def view_lineage(self, with_successors: bool = False, distance: int = 5):
     )
 
 
-Transform.__init__ = __init__
-Transform.delete = delete
-Transform.latest_run = latest_run
-Transform.view_lineage = view_lineage
+Transform.__init__ = __init__  # type: ignore
+Transform.delete = delete  # type: ignore
+Transform.latest_run = latest_run  # type: ignore
+Transform.view_lineage = view_lineage  # type: ignore

lamindb/_ulabel.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
 import lamindb_setup as ln_setup
-from lamin_utils import logger
 
+from lamindb._record import _get_record_kwargs
+from lamindb.errors import FieldValidationError
 from lamindb.models import ULabel
 
 from ._utils import attach_func_to_class_method
@@ -17,7 +18,7 @@ def __init__(self, *args, **kwargs):
         raise ValueError("Only one non-keyword arg allowed")
     name: str = kwargs.pop("name") if "name" in kwargs else None
     type: str | None = kwargs.pop("type") if "type" in kwargs else None
-    is_type: str | None = kwargs.pop("is_type") if "is_type" in kwargs else None
+    is_type: bool = kwargs.pop("is_type") if "is_type" in kwargs else False
     description: str | None = (
         kwargs.pop("description") if "description" in kwargs else None
     )
@@ -26,18 +27,8 @@ def __init__(self, *args, **kwargs):
         kwargs.pop("reference_type") if "reference_type" in kwargs else None
     )
     if len(kwargs) > 0:
-        raise ValueError(
-            "Only name, description, reference, reference_type are valid keyword arguments"
-        )
-    if is_type:
-        if name.endswith("s"):
-            logger.warning(
-                "`name` ends with 's', in case you're naming with plural, consider the singular for a type name"
-            )
-        if name[0].islower():
-            logger.warning(
-                "`name` starts with lowercase, in case you're naming a type, consider starting with uppercase"
-            )
+        valid_keywords = ", ".join([val[0] for val in _get_record_kwargs(ULabel)])
+        raise FieldValidationError(f"Only {valid_keywords} are valid keyword arguments")
     super(ULabel, self).__init__(
         name=name,
         type=type,

lamindb/base/users.py

@@ -12,12 +12,9 @@ def current_user_id() -> int:
         if ln_setup.core.django.IS_MIGRATING:
             return 1
         else:
-            exc_attr = (
-                "DoesNotExist" if hasattr(User, "DoesNotExist") else "_DoesNotExist"
-            )
             try:
                 user_id = User.objects.get(uid=settings.user.uid).id
-            except getattr(User, exc_attr):
+            except User.DoesNotExist:
                 register_user(settings.user)
                 user_id = User.objects.get(uid=settings.user.uid).id
             return user_id

lamindb/base/validation.py

@@ -2,16 +2,12 @@ from typing import TYPE_CHECKING, Literal, Union, get_args, get_origin, get_type
 
 from lamin_utils import colors
 
+from lamindb.errors import FieldValidationError
+
 if TYPE_CHECKING:
     from .models import Record
 
 
-class FieldValidationError(SystemExit):
-    """Field validation error."""
-
-    pass
-
-
 def validate_literal_fields(record: "Record", kwargs) -> None:
     """Validate all Literal type fields in a record.
 

lamindb/core/__init__.py

@@ -10,7 +10,6 @@ Registries:
    Registry
    QuerySet
    QueryManager
-   Schema
    RecordList
    FeatureManager
    ParamManager
@@ -31,11 +30,11 @@ Curators:
 .. autosummary::
    :toctree: .
 
-   BaseCurator
-   DataFrameCurator
-   AnnDataCurator
-   MuDataCurator
-   SOMACurator
+   CatManager
+   DataFrameCatManager
+   AnnDataCatManager
+   MuDataCatManager
+   TiledbsomaCatManager
    CurateLookup
 
 Settings & context:
@@ -61,7 +60,6 @@ Modules:
    loaders
    datasets
    storage
-   exceptions
    subsettings
    logger
 
@@ -75,12 +73,13 @@ from lamindb._query_set import QuerySet, RecordList
 from lamindb.core._feature_manager import FeatureManager, ParamManager
 from lamindb.core._label_manager import LabelManager
 from lamindb.curators import (
-    AnnDataCurator,
-    BaseCurator,
+    AnnDataCatManager,
+    CatManager,
     CurateLookup,
-    DataFrameCurator,
-    MuDataCurator,
-    SOMACurator,
+    Curator,
+    DataFrameCatManager,
+    MuDataCatManager,
+    TiledbsomaCatManager,
 )
 from lamindb.models import (
     BasicRecord,
@@ -91,13 +90,13 @@ from lamindb.models import (
     ParamValue,
     Record,
     Registry,
-    Schema,
     TracksRun,
     TracksUpdates,
     ValidateFields,
 )
 
-from . import _data, datasets, exceptions, fields, loaders, subsettings, types
+from .. import errors as exceptions
+from . import _data, datasets, fields, loaders, subsettings, types
 from ._context import Context
 from ._mapped_collection import MappedCollection
 from ._settings import Settings

lamindb/core/_context.py

@@ -13,20 +13,21 @@ from typing import TYPE_CHECKING
 import lamindb_setup as ln_setup
 from django.db.models import Func, IntegerField
 from lamin_utils import logger
+from lamindb_setup.core import deprecated
 from lamindb_setup.core.hashing import hash_file
 
 from lamindb.base import ids
 from lamindb.base.ids import base62_12
 from lamindb.models import Run, Transform, format_field_value
 
-from ._settings import settings
-from ._sync_git import get_transform_reference_from_git_repo
-from ._track_environment import track_environment
-from .exceptions import (
+from ..errors import (
     InconsistentKey,
     TrackNotCalled,
     UpdateContext,
 )
+from ._settings import settings
+from ._sync_git import get_transform_reference_from_git_repo
+from ._track_environment import track_environment
 from .versioning import bump_version as bump_version_function
 from .versioning import increment_base62, message_update_key_in_version_family
 
@@ -217,8 +218,8 @@ class Context:
         self._description = value
 
     @property
+    @deprecated(new_name="description")
     def name(self) -> str | None:
-        """Deprecated. Populates `description` argument for `context.transform`."""
         return self._description
 
     @name.setter
@@ -257,7 +258,7 @@ class Context:
         path: str | None = None,
         log_to_file: bool | None = None,
     ) -> None:
-        """Initiate a run with tracked data lineage.
+        """Track a global run of your Python session.
 
         - sets :attr:`~lamindb.core.Context.transform` &
           :attr:`~lamindb.core.Context.run` by creating or loading `Transform` &
@@ -284,6 +285,10 @@ class Context:
 
             >>> ln.track()
 
+            If you want to ensure a single version history across renames of the notebook or script, pass the auto-generated `uid` that you'll find in the logs:
+
+            >>> ln.track("Onv04I53OgtT0000")  # example uid, the last four characters encode the version of the transform
+
         """
         self._logging_message_track = ""
         self._logging_message_imports = ""
@@ -315,7 +320,7 @@ class Context:
                 description=description,
                 transform_ref=transform_ref,
                 transform_ref_type=transform_ref_type,
-                transform_type=transform_type,
+                transform_type=transform_type,  # type: ignore
             )
         else:
             if transform.type in {"notebook", "script"}:
@@ -352,7 +357,7 @@ class Context:
                 self._logging_message_track += f", re-started Run('{run.uid[:8]}...') at {format_field_value(run.started_at)}"
 
         if run is None:  # create new run
-            run = Run(
+            run = Run(  # type: ignore
                 transform=self._transform,
                 params=params,
             )
@@ -587,7 +592,7 @@ class Context:
             assert key is not None  # noqa: S101
             raise_update_context = False
             try:
-                transform = Transform(
+                transform = Transform(  # type: ignore
                     uid=self.uid,
                     version=self.version,
                     description=description,