lamindb 0.73.1__py3-none-any.whl → 0.74.0__py3-none-any.whl

lamindb/__init__.py

@@ -14,6 +14,7 @@ Registries:
    ULabel
    Feature
    FeatureSet
+   Param
 
 Key functionality:
 
@@ -41,12 +42,13 @@ Modules & settings:
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.73.1"
+__version__ = "0.74.0"
 
 import os as _os
 
 import lamindb_setup as _lamindb_setup
-from lamindb_setup._check_setup import InstanceNotSetupError, _check_instance_setup
+from lamindb_setup._check_setup import InstanceNotSetupError as _InstanceNotSetupError
+from lamindb_setup._check_setup import _check_instance_setup
 from lamindb_setup._connect_instance import connect
 from lamindb_setup.core.upath import UPath
 
@@ -54,17 +56,18 @@ from . import setup
 
 
 def __getattr__(name):
-    raise InstanceNotSetupError()
+    raise _InstanceNotSetupError()
 
 
 if _check_instance_setup(from_lamindb=True):
-    del InstanceNotSetupError
+    del _InstanceNotSetupError
     del __getattr__  # delete so that imports work out
     from lnschema_core.models import (
         Artifact,
         Collection,
         Feature,
         FeatureSet,
+        Param,
         Run,
         Storage,
         Transform,

lamindb/_artifact.py

@@ -23,7 +23,7 @@ from lamindb_setup.core.upath import (
     get_stat_dir_cloud,
     get_stat_file_cloud,
 )
-from lnschema_core.models import Artifact, FeatureManager, Run, Storage
+from lnschema_core.models import Artifact, FeatureManager, ParamManager, Run, Storage
 from lnschema_core.types import (
     VisibilityChoice,
 )
@@ -84,7 +84,7 @@ def process_pathlike(
             pass
     if isinstance(filepath, LocalPathClasses):
         filepath = filepath.resolve()
-    if check_path_is_child_of_root(filepath, default_storage.root_as_path()):
+    if check_path_is_child_of_root(filepath, default_storage.root):
         use_existing_storage_key = True
         return default_storage, use_existing_storage_key
     else:
@@ -272,28 +272,17 @@ def check_path_in_existing_storage(
 ) -> Storage | bool:
     for storage in Storage.objects.using(using_key).filter().all():
         # if path is part of storage, return it
-        if check_path_is_child_of_root(path, root=create_path(storage.root)):
+        if check_path_is_child_of_root(path, root=storage.root):
             return storage
     return False
 
 
 def check_path_is_child_of_root(path: Path | UPath, root: Path | UPath | None) -> bool:
-    path = UPath(str(path)) if not isinstance(path, UPath) else path
-    root = UPath(str(root)) if not isinstance(root, UPath) else root
-
-    # the following comparisons can fail if types aren't comparable
-    if not isinstance(path, LocalPathClasses) and not isinstance(
-        root, LocalPathClasses
-    ):
-        # the following tests equivalency of two UPath objects
-        # via string representations; otherwise
-        # S3Path('s3://lndb-storage/') and S3Path('s3://lamindb-ci/')
-        # test as equivalent
-        return list(path.parents)[-1].as_posix() == root.as_posix()
-    elif isinstance(path, LocalPathClasses) and isinstance(root, LocalPathClasses):
-        return root.resolve() in path.resolve().parents
-    else:
-        return False
+    # str is needed to eliminate UPath storage_options
+    # from the equality checks below
+    path = UPath(str(path))
+    root = UPath(str(root))
+    return root.resolve() in path.resolve().parents
 
 
 def get_relative_path_to_directory(
@@ -358,7 +347,7 @@ def get_artifact_kwargs_from_data(
     check_path_in_storage = False
     if use_existing_storage_key:
         inferred_key = get_relative_path_to_directory(
-            path=path, directory=storage.root_as_path()
+            path=path, directory=UPath(storage.root)
         ).as_posix()
         if key is None:
             key = inferred_key
@@ -516,6 +505,7 @@ def update_attributes(data: HasFeatures, attributes: Mapping[str, str]):
 
 def __init__(artifact: Artifact, *args, **kwargs):
     artifact.features = FeatureManager(artifact)
+    artifact.params = ParamManager(artifact)
     # Below checks for the Django-internal call in from_db()
     # it'd be better if we could avoid this, but not being able to create a Artifact
     # from data with the default constructor renders the central class of the API
@@ -532,6 +522,7 @@ def __init__(artifact: Artifact, *args, **kwargs):
         raise ValueError("Only one non-keyword arg allowed: data")
 
     data: str | Path = kwargs.pop("data") if len(args) == 0 else args[0]
+    type: str = kwargs.pop("type") if "type" in kwargs else "dataset"
     key: str | None = kwargs.pop("key") if "key" in kwargs else None
     run: Run | None = kwargs.pop("run") if "run" in kwargs else None
     description: str | None = (
@@ -616,6 +607,7 @@ def __init__(artifact: Artifact, *args, **kwargs):
             "to _retain_ the old state by duplicating the entire folder, do _not_ pass `is_new_version_of`"
         )
 
+    kwargs["type"] = type
     kwargs["uid"] = provisional_uid
     kwargs["version"] = version
     kwargs["description"] = description
@@ -662,6 +654,7 @@ def from_df(
         version=version,
         is_new_version_of=is_new_version_of,
         accessor="DataFrame",
+        type="dataset",
         **kwargs,
     )
     return artifact
@@ -690,6 +683,7 @@ def from_anndata(
         version=version,
         is_new_version_of=is_new_version_of,
         accessor="AnnData",
+        type="dataset",
         **kwargs,
     )
     return artifact
@@ -716,6 +710,7 @@ def from_mudata(
         version=version,
         is_new_version_of=is_new_version_of,
         accessor="MuData",
+        type="dataset",
         **kwargs,
     )
     return artifact
@@ -752,7 +747,7 @@ def from_dir(
         else:
             # maintain the hierachy within an existing storage location
             folder_key_path = get_relative_path_to_directory(
-                folderpath, storage.root_as_path()
+                folderpath, UPath(storage.root)
             )
     else:
         folder_key_path = Path(key)

lamindb/_feature.py

@@ -27,12 +27,12 @@ FEATURE_TYPES = {
 }
 
 
-def convert_numpy_dtype_to_lamin_feature_type(dtype) -> str:
+def convert_numpy_dtype_to_lamin_feature_type(dtype, str_as_cat: bool = True) -> str:
     orig_type = dtype.name
     # strip precision qualifiers
     type = "".join(i for i in orig_type if not i.isdigit())
     if type == "object" or type == "str":
-        type = "cat"
+        type = "cat" if str_as_cat else "str"
     return type
 
 

lamindb/_finish.py

@@ -33,13 +33,19 @@ def get_seconds_since_modified(filepath) -> float:
     return datetime.now().timestamp() - filepath.stat().st_mtime
 
 
-def finish():
+def finish() -> None:
     """Mark a tracked run as finished.
 
     Saves source code and, for notebooks, a run report to your default storage location.
     """
-    if run_context.path is None:
+    if run_context.run is None:
         raise TrackNotCalled("Please run `ln.track()` before `ln.finish()`")
+    if run_context.path is None:
+        assert run_context.transform.type not in {"script", "notebook"}
+        run_context.run.finished_at = datetime.now(timezone.utc)
+        run_context.run.save()
+        # nothing else to do
+        return None
     if is_run_from_ipython:  # notebooks
         if (
             get_seconds_since_modified(run_context.path) > 3

lamindb/_from_values.py

@@ -278,14 +278,26 @@ def index_iterable(iterable: Iterable) -> pd.Index:
 
 
 def _print_values(names: Iterable, n: int = 20, quotes: bool = True) -> str:
-    names = (name for name in names if name != "None")
-    unique_names = list(dict.fromkeys(names))[:n]
-    if quotes:
-        print_values = ", ".join(f"'{name}'" for name in unique_names)
+    if isinstance(names, dict):
+        items = {
+            f"{key}: {value}": None
+            for key, value in names.items()
+            if key != "None" and value != "None"
+        }
     else:
-        print_values = ", ".join(f"{name}" for name in unique_names)
-    if len(unique_names) > n:
+        # Use a dictionary instead of a list to have unique values and preserve order
+        items = {str(name): None for name in names if name != "None"}
+
+    unique_items = list(items.keys())
+
+    if quotes:
+        unique_items = [f"'{item}'" for item in unique_items]
+
+    print_values = ", ".join(unique_items[:n])
+
+    if len(unique_items) > n:
         print_values += ", ..."
+
     return print_values
 
 

lamindb/_registry.py

@@ -54,7 +54,7 @@ def suggest_records_with_similar_names(record: Registry, kwargs) -> bool:
     """
     if kwargs.get("name") is None:
         return False
-    queryset = _search(record.__class__, kwargs["name"], truncate_words=True, limit=5)
+    queryset = _search(record.__class__, kwargs["name"], truncate_words=True, limit=20)
     if not queryset.exists():  # empty queryset
         return False
     for alternative_record in queryset:

lamindb/_run.py

@@ -1,9 +1,10 @@
 from __future__ import annotations
 
-from lnschema_core.models import Run, Transform
+from lnschema_core.models import ParamManager, Run, Transform
 
 
 def __init__(run: Run, *args, **kwargs):
+    run.params = ParamManager(run)
     if len(args) == len(run._meta.concrete_fields):
         super(Run, run).__init__(*args, **kwargs)
         return None

lamindb/_storage.py

@@ -3,11 +3,6 @@ from lamindb_setup.core.upath import UPath, create_path
 from lnschema_core import Storage
 
 
-def root_as_path(self) -> UPath:
-    access_token = self._access_token if hasattr(self, "_access_token") else None
-    return create_path(self.root, access_token=access_token)
-
-
 @property  # type: ignore
 @doc_args(Storage.path.__doc__)
 def path(self) -> UPath:
@@ -16,5 +11,4 @@ def path(self) -> UPath:
     return create_path(self.root, access_token=access_token)
 
 
-Storage.root_as_path = root_as_path
 Storage.path = path

lamindb/_view.py

@@ -1,14 +1,16 @@
 from __future__ import annotations
 
+import builtins
 import importlib
 import inspect
 
-from IPython.display import display
 from lamin_utils import colors, logger
 from lamindb_setup import settings
 from lamindb_setup._init_instance import get_schema_module_name
 from lnschema_core import Registry
 
+is_run_from_ipython = getattr(builtins, "__IPYTHON__", False)
+
 
 def view(
     n: int = 7, schema: str | None = None, registries: list[str] | None = None
@@ -25,6 +27,11 @@ def view(
     Examples:
         >>> ln.view()
     """
+    if is_run_from_ipython:
+        from IPython.display import display as show
+    else:
+        show = logger.print
+
     if schema is not None:
         schema_names = [schema]
     else:
@@ -60,4 +67,4 @@ def view(
                 df = orm.df().iloc[-n:]
             if df.shape[0] > 0:
                 logger.print(colors.blue(colors.bold(orm.__name__)))
-                display(df)
+                show(df)

lamindb/core/__init__.py

@@ -10,7 +10,9 @@ Registries:
    QueryManager
    RecordsList
    HasFeatures
+   HasParams
    FeatureManager
+   ParamManager
    LabelManager
    IsVersioned
    CanValidate
@@ -56,6 +58,7 @@ from lamin_utils._inspect import InspectResult
 from lnschema_core.models import (
     CanValidate,
     HasFeatures,
+    HasParams,
     HasParents,
     IsVersioned,
     Registry,
@@ -71,7 +74,7 @@ from lamindb._annotate import (
 )
 from lamindb._query_manager import QueryManager
 from lamindb._query_set import QuerySet, RecordsList
-from lamindb.core._feature_manager import FeatureManager
+from lamindb.core._feature_manager import FeatureManager, ParamManager
 from lamindb.core._label_manager import LabelManager
 
 from . import _data, datasets, exceptions, fields, types

lamindb/core/_data.py

@@ -24,13 +24,12 @@ from lamindb._registry import get_default_str_field
 from lamindb.core._settings import settings
 
 from ._feature_manager import (
-    FeatureManager,
     get_feature_set_links,
     get_host_id_field,
     get_label_links,
     print_features,
 )
-from ._label_manager import LabelManager, print_labels
+from ._label_manager import print_labels
 from ._run_context import run_context
 from .exceptions import ValidationError
 from .schema import (
@@ -159,7 +158,11 @@ def describe(self: HasFeatures, print_types: bool = False):
         msg += f"  {colors.italic('Provenance')}\n"
         msg += prov_msg
     msg += print_labels(self, print_types=print_types)
-    msg += print_features(self, print_types=print_types)  # type: ignore
+    msg += print_features(  # type: ignore
+        self,
+        print_types=print_types,
+        print_params=hasattr(self, "type") and self.type == "model",
+    )
     logger.print(msg)
 
 

lamindb/core/_feature_manager.py

@@ -10,9 +10,8 @@ import numpy as np
 import pandas as pd
 from anndata import AnnData
 from django.contrib.postgres.aggregates import ArrayAgg
-from django.db import connections, models
-from django.db.models import Aggregate, CharField, F, Value
-from django.db.models.functions import Concat
+from django.db import connections
+from django.db.models import Aggregate
 from lamin_utils import colors, logger
 from lamindb_setup.core.upath import create_path
 from lnschema_core.models import (
@@ -24,8 +23,15 @@ from lnschema_core.models import (
     FeatureManagerCollection,
     FeatureValue,
     HasFeatures,
+    HasParams,
     LinkORM,
+    Param,
+    ParamManager,
+    ParamManagerArtifact,
+    ParamManagerRun,
+    ParamValue,
     Registry,
+    Run,
     ULabel,
 )
 
@@ -43,6 +49,9 @@ from lamindb.core.storage import LocalPathClasses
 
 from ._label_manager import get_labels_as_dict
 from ._settings import settings
+from .schema import (
+    dict_related_model_to_related_name,
+)
 
 if TYPE_CHECKING:
     from lnschema_core.types import FieldAttr
@@ -107,8 +116,10 @@ def get_feature_set_links(host: Artifact | Collection) -> QuerySet:
     return feature_set_links
 
 
-def get_link_attr(link: LinkORM, data: HasFeatures) -> str:
+def get_link_attr(link: LinkORM | type[LinkORM], data: HasFeatures) -> str:
     link_model_name = link.__class__.__name__
+    if link_model_name == "ModelBase":  # we passed the type of the link
+        link_model_name = link.__name__
     link_attr = link_model_name.replace(data.__class__.__name__, "")
     if link_attr == "ExperimentalFactor":
         link_attr = "experimental_factor"
@@ -131,42 +142,51 @@ def custom_aggregate(field, using: str):
 
 
 def print_features(
-    self: HasFeatures, print_types: bool = False, to_dict: bool = False
+    self: HasFeatures | HasParams,
+    print_types: bool = False,
+    to_dict: bool = False,
+    print_params: bool = False,
 ) -> str | dict[str, Any]:
     from lamindb._from_values import _print_values
 
     msg = ""
     dictionary = {}
     # categorical feature values
-    labels_msg = ""
-    labels_by_feature = defaultdict(list)
-    for _, (_, links) in get_labels_as_dict(self, links=True).items():
-        for link in links:
-            if link.feature_id is not None:
-                link_attr = get_link_attr(link, self)
-                labels_by_feature[link.feature_id].append(getattr(link, link_attr).name)
-    for feature_id, labels_list in labels_by_feature.items():
-        feature = Feature.objects.using(self._state.db).get(id=feature_id)
-        print_values = _print_values(labels_list, n=10)
-        type_str = f": {feature.dtype}" if print_types else ""
-        if to_dict:
-            dictionary[feature.name] = (
-                labels_list if len(labels_list) > 1 else labels_list[0]
-            )
-        labels_msg += f"    '{feature.name}'{type_str} = {print_values}\n"
-    if labels_msg:
-        msg += labels_msg
+    if not print_params:
+        labels_msg = ""
+        labels_by_feature = defaultdict(list)
+        for _, (_, links) in get_labels_as_dict(self, links=True).items():
+            for link in links:
+                if link.feature_id is not None:
+                    link_attr = get_link_attr(link, self)
+                    labels_by_feature[link.feature_id].append(
+                        getattr(link, link_attr).name
+                    )
+        for feature_id, labels_list in labels_by_feature.items():
+            feature = Feature.objects.using(self._state.db).get(id=feature_id)
+            print_values = _print_values(labels_list, n=10)
+            type_str = f": {feature.dtype}" if print_types else ""
+            if to_dict:
+                dictionary[feature.name] = (
+                    labels_list if len(labels_list) > 1 else labels_list[0]
+                )
+            labels_msg += f"    '{feature.name}'{type_str} = {print_values}\n"
+        if labels_msg:
+            msg += labels_msg
 
     # non-categorical feature values
     non_labels_msg = ""
-    if self.id is not None and self.__class__ == Artifact:
-        feature_values = self.feature_values.values(
-            "feature__name", "feature__dtype"
-        ).annotate(values=custom_aggregate("value", self._state.db))
+    if self.id is not None and self.__class__ == Artifact or self.__class__ == Run:
+        attr_name = "param" if print_params else "feature"
+        feature_values = (
+            getattr(self, f"{attr_name}_values")
+            .values(f"{attr_name}__name", f"{attr_name}__dtype")
+            .annotate(values=custom_aggregate("value", self._state.db))
+        )
         if len(feature_values) > 0:
             for fv in feature_values:
-                feature_name = fv["feature__name"]
-                feature_dtype = fv["feature__dtype"]
+                feature_name = fv[f"{attr_name}__name"]
+                feature_dtype = fv[f"{attr_name}__dtype"]
                 values = fv["values"]
                 # TODO: understand why the below is necessary
                 if not isinstance(values, list):
@@ -174,24 +194,33 @@ def print_features(
                 if to_dict:
                     dictionary[feature_name] = values if len(values) > 1 else values[0]
                 type_str = f": {feature_dtype}" if print_types else ""
-                non_labels_msg += f"    '{feature_name}'{type_str} = {_print_values(values, n=10, quotes=False)}\n"
+                printed_values = (
+                    _print_values(values, n=10, quotes=False)
+                    if not feature_dtype.startswith("list")
+                    else values
+                )
+                non_labels_msg += f"    '{feature_name}'{type_str} = {printed_values}\n"
             msg += non_labels_msg
 
     if msg != "":
-        msg = f"  {colors.italic('Features')}\n" + msg
+        header = "Features" if not print_params else "Params"
+        msg = f"  {colors.italic(header)}\n" + msg
 
     # feature sets
-    feature_set_msg = ""
-    for slot, feature_set in get_feature_set_by_slot_(self).items():
-        features = feature_set.members
-        # features.first() is a lot slower than features[0] here
-        name_field = get_default_str_field(features[0])
-        feature_names = list(features.values_list(name_field, flat=True)[:20])
-        type_str = f": {feature_set.registry}" if print_types else ""
-        feature_set_msg += f"    '{slot}'{type_str} = {_print_values(feature_names)}\n"
-    if feature_set_msg:
-        msg += f"  {colors.italic('Feature sets')}\n"
-        msg += feature_set_msg
+    if not print_params:
+        feature_set_msg = ""
+        for slot, feature_set in get_feature_set_by_slot_(self).items():
+            features = feature_set.members
+            # features.first() is a lot slower than features[0] here
+            name_field = get_default_str_field(features[0])
+            feature_names = list(features.values_list(name_field, flat=True)[:20])
+            type_str = f": {feature_set.registry}" if print_types else ""
+            feature_set_msg += (
+                f"    '{slot}'{type_str} = {_print_values(feature_names)}\n"
+            )
+        if feature_set_msg:
+            msg += f"  {colors.italic('Feature sets')}\n"
+            msg += feature_set_msg
     if to_dict:
         return dictionary
     else:
@@ -258,7 +287,9 @@ def parse_feature_sets_from_anndata(
     return feature_sets
 
 
-def infer_feature_type_convert_json(value: Any, mute: bool = False) -> tuple[str, Any]:
+def infer_feature_type_convert_json(
+    value: Any, mute: bool = False, str_as_ulabel: bool = True
+) -> tuple[str, Any]:
     if isinstance(value, bool):
         return FEATURE_TYPES["bool"], value
     elif isinstance(value, int):
@@ -266,39 +297,53 @@ def infer_feature_type_convert_json(value: Any, mute: bool = False) -> tuple[str
     elif isinstance(value, float):
         return FEATURE_TYPES["float"], value
     elif isinstance(value, str):
-        return FEATURE_TYPES["str"] + "[ULabel]", value
+        if str_as_ulabel:
+            return FEATURE_TYPES["str"] + "[ULabel]", value
+        else:
+            return "str", value
     elif isinstance(value, Iterable) and not isinstance(value, (str, bytes)):
         if isinstance(value, (pd.Series, np.ndarray)):
-            return convert_numpy_dtype_to_lamin_feature_type(value.dtype), list(value)
+            return convert_numpy_dtype_to_lamin_feature_type(
+                value.dtype, str_as_cat=str_as_ulabel
+            ), list(value)
+        if isinstance(value, dict):
+            return "dict", value
         if len(value) > 0:  # type: ignore
             first_element_type = type(next(iter(value)))
             if all(isinstance(elem, first_element_type) for elem in value):
                 if first_element_type == bool:
-                    return FEATURE_TYPES["bool"], value
+                    return f"list[{FEATURE_TYPES['bool']}]", value
                 elif first_element_type == int:
-                    return FEATURE_TYPES["int"], value
+                    return f"list[{FEATURE_TYPES['int']}]", value
                 elif first_element_type == float:
-                    return FEATURE_TYPES["float"], value
+                    return f"list[{FEATURE_TYPES['float']}]", value
                 elif first_element_type == str:
-                    return FEATURE_TYPES["str"] + "[ULabel]", value
+                    if str_as_ulabel:
+                        return FEATURE_TYPES["str"] + "[ULabel]", value
+                    else:
+                        return "list[str]", value
+    elif isinstance(value, Registry):
+        return (f"cat[{value.__class__.__get_name_with_schema__()}]", value)
     if not mute:
         logger.warning(f"cannot infer feature type of: {value}, returning '?")
     return ("?", value)
 
 
-def __init__(self, host: Artifact | Collection):
+def __init__(self, host: Artifact | Collection | Run):
     self._host = host
     self._feature_set_by_slot_ = None
     self._accessor_by_registry_ = None
 
 
 def __repr__(self) -> str:
-    return print_features(self._host)  # type: ignore
+    return print_features(self._host, print_params=(self.__class__ == ParamManager))  # type: ignore
 
 
 def get_values(self) -> dict[str, Any]:
     """Get feature values as a dictionary."""
-    return print_features(self._host, to_dict=True)  # type: ignore
+    return print_features(
+        self._host, to_dict=True, print_params=(self.__class__ == ParamManager)
+    )  # type: ignore
 
 
 def __getitem__(self, slot) -> QuerySet:
@@ -316,19 +361,25 @@ def __getitem__(self, slot) -> QuerySet:
 @classmethod  # type: ignore
 def filter(cls, **expression) -> QuerySet:
     """Filter features."""
+    if cls in {FeatureManagerArtifact, FeatureManagerCollection}:
+        model = Feature
+        value_model = FeatureValue
+    else:
+        model = Param
+        value_model = ParamValue
     keys_normalized = [key.split("__")[0] for key in expression]
-    validated = Feature.validate(keys_normalized, field="name", mute=True)
+    validated = model.validate(keys_normalized, field="name", mute=True)
     if sum(validated) != len(keys_normalized):
         raise ValidationError(
             f"Some keys in the filter expression are not registered as features: {np.array(keys_normalized)[~validated]}"
         )
     new_expression = {}
-    features = Feature.filter(name__in=keys_normalized).all().distinct()
+    features = model.filter(name__in=keys_normalized).all().distinct()
     for key, value in expression.items():
         normalized_key = key.split("__")[0]
         feature = features.get(name=normalized_key)
         if not feature.dtype.startswith("cat"):
-            feature_value = FeatureValue.filter(feature=feature, value=value).one()
+            feature_value = value_model.filter(feature=feature, value=value).one()
             new_expression["feature_values"] = feature_value
         else:
             if isinstance(value, str):
@@ -336,10 +387,12 @@ def filter(cls, **expression) -> QuerySet:
                 new_expression["ulabels"] = label
             else:
                 raise NotImplementedError
-    if cls == FeatureManagerArtifact:
+    if cls == FeatureManagerArtifact or cls == ParamManagerArtifact:
         return Artifact.filter(**new_expression)
-    else:
+    elif cls == FeatureManagerCollection:
         return Collection.filter(**new_expression)
+    elif cls == ParamManagerRun:
+        return Run.filter(**new_expression)
 
 
 @property  # type: ignore
@@ -358,16 +411,17 @@ def _accessor_by_registry(self):
     return self._accessor_by_registry_
 
 
-def add_values(
+def _add_values(
     self,
     values: dict[str, str | int | float | bool],
-    feature_field: FieldAttr = Feature.name,
+    feature_param_field: FieldAttr,
+    str_as_ulabel: bool = True,
 ) -> None:
     """Annotate artifact with features & values.
 
     Args:
         values: A dictionary of keys (features) & values (labels, numbers, booleans).
-        feature_field: The field of a reference registry to map keys of the
+        feature_param_field: The field of a reference registry to map keys of the
             dictionary.
     """
     # rename to distinguish from the values inside the dict
@@ -377,62 +431,83 @@ def add_values(
         keys = list(keys)  # type: ignore
     # deal with other cases later
     assert all(isinstance(key, str) for key in keys)
-    registry = feature_field.field.model
-    validated = registry.validate(keys, field=feature_field, mute=True)
+    registry = feature_param_field.field.model
+    is_param = registry == Param
+    model = Param if is_param else Feature
+    value_model = ParamValue if is_param else FeatureValue
+    model_name = "Param" if is_param else "Feature"
+    if is_param:
+        if self._host.__class__ == Artifact:
+            if self._host.type != "model":
+                raise ValidationError("Can only set params for model-like artifacts.")
+    else:
+        if self._host.__class__ == Artifact:
+            if self._host.type != "dataset" and self._host.type is not None:
+                raise ValidationError(
+                    "Can only set features for dataset-like artifacts."
+                )
+    validated = registry.validate(keys, field=feature_param_field, mute=True)
     keys_array = np.array(keys)
     validated_keys = keys_array[validated]
     if validated.sum() != len(keys):
         not_validated_keys = keys_array[~validated]
         hint = "\n".join(
             [
-                f"  ln.Feature(name='{key}', dtype='{infer_feature_type_convert_json(features_values[key])[0]}').save()"
+                f"  ln.{model_name}(name='{key}', dtype='{infer_feature_type_convert_json(features_values[key], str_as_ulabel=str_as_ulabel)[0]}').save()"
                 for key in not_validated_keys
             ]
         )
         msg = (
             f"These keys could not be validated: {not_validated_keys.tolist()}\n"
-            f"If there are no typos, create features for them:\n\n{hint}"
+            f"Here is how to create a {model_name.lower()}:\n\n{hint}"
         )
         raise ValidationError(msg)
     registry.from_values(
         validated_keys,
-        field=feature_field,
+        field=feature_param_field,
     )
     # figure out which of the values go where
-    features_labels = []
+    features_labels = defaultdict(list)
     feature_values = []
     not_validated_values = []
     for key, value in features_values.items():
-        feature = Feature.filter(name=key).one()
+        feature = model.filter(name=key).one()
         inferred_type, converted_value = infer_feature_type_convert_json(
-            value, mute=True
+            value,
+            mute=True,
+            str_as_ulabel=str_as_ulabel,
         )
         if feature.dtype == "number":
             if inferred_type not in {"int", "float"}:
                 raise TypeError(
                     f"Value for feature '{key}' with type {feature.dtype} must be a number"
                 )
-        elif feature.dtype == "cat":
+        elif feature.dtype.startswith("cat"):
             if not (inferred_type.startswith("cat") or isinstance(value, Registry)):
                 raise TypeError(
                     f"Value for feature '{key}' with type '{feature.dtype}' must be a string or record."
                 )
-        elif feature.dtype == "bool":
-            assert isinstance(value, bool)
+        elif not inferred_type == feature.dtype:
+            raise ValidationError(
+                f"Expected dtype for '{key}' is '{feature.dtype}', got '{inferred_type}'"
+            )
         if not feature.dtype.startswith("cat"):
             # can remove the query once we have the unique constraint
-            feature_value = FeatureValue.filter(
-                feature=feature, value=converted_value
-            ).one_or_none()
+            filter_kwargs = {model_name.lower(): feature, "value": converted_value}
+            feature_value = value_model.filter(**filter_kwargs).one_or_none()
             if feature_value is None:
-                feature_value = FeatureValue(feature=feature, value=converted_value)
+                feature_value = value_model(**filter_kwargs)
             feature_values.append(feature_value)
         else:
             if isinstance(value, Registry):
-                assert not value._state.adding
+                if value._state.adding:
+                    raise ValidationError(
+                        "Please save your label record before annotation."
+                    )
                 label_record = value
-                assert isinstance(label_record, ULabel)
-                features_labels.append((feature, label_record))
+                features_labels[
+                    label_record.__class__.__get_name_with_schema__()
+                ].append((feature, label_record))
             else:
                 if isinstance(value, str):
                     values = [value]  # type: ignore
@@ -447,7 +522,7 @@ def add_values(
                 if validated.sum() != len(values):
                     not_validated_values += values_array[~validated].tolist()
                 label_records = ULabel.from_values(validated_values, field="name")
-                features_labels += [
+                features_labels["ULabel"] += [
                     (feature, label_record) for label_record in label_records
                 ]
     if not_validated_values:
@@ -457,40 +532,67 @@ def add_values(
         )
         msg = (
             f"These values could not be validated: {not_validated_values}\n"
-            f"If there are no typos, create ulabels for them:\n\n{hint}"
+            f"Here is how to create ulabels for them:\n\n{hint}"
         )
         raise ValidationError(msg)
     # bulk add all links to ArtifactULabel
     if features_labels:
-        LinkORM = self._host.ulabels.through
-        links = [
-            LinkORM(
-                artifact_id=self._host.id, feature_id=feature.id, ulabel_id=label.id
-            )
-            for (feature, label) in features_labels
-        ]
-        # a link might already exist
-        try:
-            save(links, ignore_conflicts=False)
-        except Exception:
-            save(links, ignore_conflicts=True)
-            # now deal with links that were previously saved without a feature_id
-            saved_links = LinkORM.filter(
-                artifact_id=self._host.id,
-                ulabel_id__in=[l.id for _, l in features_labels],
-            )
-            for link in saved_links.all():
-                # TODO: also check for inconsistent features
-                if link.feature_id is None:
-                    link.feature_id = [
-                        f.id for f, l in features_labels if l.id == link.ulabel_id
-                    ][0]
-                    link.save()
+        if list(features_labels.keys()) != ["ULabel"]:
+            related_names = dict_related_model_to_related_name(self._host.__class__)
+        else:
+            related_names = {"ULabel": "ulabels"}
+        for class_name, registry_features_labels in features_labels.items():
+            related_name = related_names[class_name]  # e.g., "ulabels"
+            LinkORM = getattr(self._host, related_name).through
+            field_name = f"{get_link_attr(LinkORM, self._host)}_id"  # e.g., ulabel_id
+            links = [
+                LinkORM(
+                    **{
+                        "artifact_id": self._host.id,
+                        "feature_id": feature.id,
+                        field_name: label.id,
+                    }
+                )
+                for (feature, label) in registry_features_labels
+            ]
+            # a link might already exist
+            try:
+                save(links, ignore_conflicts=False)
+            except Exception:
+                save(links, ignore_conflicts=True)
+                # now deal with links that were previously saved without a feature_id
+                saved_links = LinkORM.filter(
+                    **{
+                        "artifact_id": self._host.id,
+                        f"{field_name}__in": [
+                            l.id for _, l in registry_features_labels
+                        ],
+                    }
+                )
+                for link in saved_links.all():
+                    # TODO: also check for inconsistent features
+                    if link.feature_id is None:
+                        link.feature_id = [
+                            f.id
+                            for f, l in registry_features_labels
+                            if l.id == getattr(link, field_name)
+                        ][0]
+                        link.save()
     if feature_values:
         save(feature_values)
-        LinkORM = self._host.feature_values.through
+        if is_param:
+            LinkORM = self._host.param_values.through
+            valuefield_id = "paramvalue_id"
+        else:
+            LinkORM = self._host.feature_values.through
+            valuefield_id = "featurevalue_id"
         links = [
-            LinkORM(artifact_id=self._host.id, featurevalue_id=feature_value.id)
+            LinkORM(
+                **{
+                    f"{self._host.__get_name_with_schema__().lower()}_id": self._host.id,
+                    valuefield_id: feature_value.id,
+                }
+            )
             for feature_value in feature_values
         ]
         # a link might already exist, to avoid raising a unique constraint
@@ -498,6 +600,35 @@ def add_values(
         save(links, ignore_conflicts=True)
 
 
+def add_values_features(
+    self,
+    values: dict[str, str | int | float | bool],
+    feature_field: FieldAttr = Feature.name,
+    str_as_ulabel: bool = True,
+) -> None:
+    """Annotate artifact with features & values.
+
+    Args:
+        values: A dictionary of keys (features) & values (labels, numbers, booleans).
+        feature_field: The field of a reference registry to map keys of the
+            dictionary.
+        str_as_ulabel: Whether to interpret string values as ulabels.
+    """
+    _add_values(self, values, feature_field, str_as_ulabel=str_as_ulabel)
+
+
+def add_values_params(
+    self,
+    values: dict[str, str | int | float | bool],
+) -> None:
+    """Annotate artifact with features & values.
+
+    Args:
+        values: A dictionary of keys (features) & values (labels, numbers, booleans).
+    """
+    _add_values(self, values, Param.name, str_as_ulabel=False)
+
+
 def add_feature_set(self, feature_set: FeatureSet, slot: str) -> None:
     """Annotate artifact with a feature set.
 
@@ -686,15 +817,19 @@ def _add_from(self, data: HasFeatures, parents: bool = True):
 
 
 FeatureManager.__init__ = __init__
+ParamManager.__init__ = __init__
 FeatureManager.__repr__ = __repr__
+ParamManager.__repr__ = __repr__
 FeatureManager.__getitem__ = __getitem__
 FeatureManager.get_values = get_values
 FeatureManager._feature_set_by_slot = _feature_set_by_slot
 FeatureManager._accessor_by_registry = _accessor_by_registry
-FeatureManager.add_values = add_values
+FeatureManager.add_values = add_values_features
 FeatureManager.add_feature_set = add_feature_set
 FeatureManager._add_set_from_df = _add_set_from_df
 FeatureManager._add_set_from_anndata = _add_set_from_anndata
 FeatureManager._add_set_from_mudata = _add_set_from_mudata
 FeatureManager._add_from = _add_from
 FeatureManager.filter = filter
+ParamManager.add_values = add_values_params
+ParamManager.get_values = get_values

lamindb/core/_run_context.py

@@ -211,26 +211,6 @@ def pretty_pypackages(dependencies: dict) -> str:
     return " ".join(deps_list)
 
 
-def parse_and_link_params(run: Run, params: dict) -> None:
-    param_values = []
-    for key, value in params.items():
-        param = Param.filter(name=key).one_or_none()
-        if param is None:
-            dtype = type(value).__name__
-            logger.warning(
-                f"param '{key}' does not yet exist, creating it with dtype '{dtype}'"
-            )
-            param = Param(name=key, dtype=dtype).save()
-        param_value, _ = ParamValue.objects.get_or_create(param=param, value=value)
-        param_values.append(param_value)
-    if param_values:
-        links = [
-            RunParamValue(run_id=run.id, paramvalue_id=param_value.id)
-            for param_value in param_values
-        ]
-        RunParamValue.objects.bulk_create(links)
-
-
 class run_context:
     """Global run context."""
 
@@ -370,7 +350,7 @@ class run_context:
         # need to save in all cases
         run.save()
         if params is not None:
-            parse_and_link_params(run, params)
+            run.params.add_values(params)
         cls.run = run
 
         from ._track_environment import track_environment

{lamindb-0.73.1.dist-info → lamindb-0.74.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.73.1
+Version: 0.74.0
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,8 +9,8 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.68.1
-Requires-Dist: lamindb_setup==0.73.1
+Requires-Dist: lnschema_core==0.70.0
+Requires-Dist: lamindb_setup==0.73.2
 Requires-Dist: lamin_utils==0.13.2
 Requires-Dist: lamin_cli==0.14.0
 Requires-Dist: rapidfuzz
@@ -25,7 +25,7 @@ Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
 Requires-Dist: psutil
 Requires-Dist: lamindb_setup[aws] ; extra == "aws"
-Requires-Dist: bionty==0.43.1 ; extra == "bionty"
+Requires-Dist: bionty==0.44.0 ; extra == "bionty"
 Requires-Dist: pandas<2 ; extra == "dev"
 Requires-Dist: pre-commit ; extra == "dev"
 Requires-Dist: nox ; extra == "dev"

{lamindb-0.73.1.dist-info → lamindb-0.74.0.dist-info}/RECORD

@@ -1,31 +1,31 @@
-lamindb/__init__.py,sha256=gdITn70-3HoUMrn3iMBej3cwEYPot50lGkLAkm4WgY4,2207
+lamindb/__init__.py,sha256=GHl_dP7bu9sy6o-6q5GDDm41RPzoXQ9_CTwpOU4HOLU,2297
 lamindb/_annotate.py,sha256=CScyKVB3k_x5p0pihxO1UrTFBCvPcSSjBXvGY3YgTLs,44381
-lamindb/_artifact.py,sha256=uMUlXCGU1JYSnOYMxCwEe_1MslbcnZACP9SYL-a4zRk,40816
+lamindb/_artifact.py,sha256=OpHmffSsmUMjF3P_1qjtf8m0QDuLWsAcp5pJSC7scdA,40454
 lamindb/_can_validate.py,sha256=s1q0lxplqnhytrVgArBTm05XKMMmpreK0ZlVCsd2jjk,14849
 lamindb/_collection.py,sha256=AGiztgM_OIet617aF86muL6lIv6GEDcmXorize6V62U,14657
-lamindb/_feature.py,sha256=Aawz4jCgjhuneZnmoty7JkZI3zjA_50vr_0dnFjSN3U,7348
+lamindb/_feature.py,sha256=Z_Awtsj183SA1egZgNmJI_dH04rqXAj72od3H6n0uBk,7398
 lamindb/_feature_set.py,sha256=ZAFLyIiWC6yPOFTF7K03olE2Rl3KthsV-1ttqJySzqQ,8106
 lamindb/_filter.py,sha256=jEA1n1Hi6lEPaD0JXiTSh9K4joGzWU6Yxy0LCLTiOdY,1422
-lamindb/_finish.py,sha256=CP52X5Wr-96DdXVGqz_Ls4uum3OErk1Twt9e74Q4grk,9945
-lamindb/_from_values.py,sha256=qQTeIjA8RrW5Dpv6ZdzTeLU3WS1NEsftP0igesLJUqw,13649
+lamindb/_finish.py,sha256=CqilMKpmkocX5jVnLDpqbQ1SwkEuV-RFdd1CtnopUBk,10202
+lamindb/_from_values.py,sha256=p0DFK7oovz5DkzutUMQNybjCe-uFIxqtP91TuZ_Myu0,13901
 lamindb/_is_versioned.py,sha256=0PgRCmxEmYDcAjllLSOYZm132B1lW6QgmBBERhRyFt0,1341
 lamindb/_parents.py,sha256=kb5AHkntpTP5g2lk1aPL0FmIilYZYVZvj6stddFOI40,15075
 lamindb/_query_manager.py,sha256=qxwrBM8UmNQnUiNOfe84YN6NpfJBg2wQ2JqUfH6HHhc,4225
 lamindb/_query_set.py,sha256=JgdJY2pACCqjC4AHjFiMAdq_qjeuhEo2NUX68f6ODwk,11597
-lamindb/_registry.py,sha256=VRyWgDEYtuaV_krMzLDfg2uBwSCMCBPwelISGLOHuQw,18755
-lamindb/_run.py,sha256=b7A52M1On3QzFgIYyfQoz5Kk7V3wcu9p_Prq5bzd8v8,1838
+lamindb/_registry.py,sha256=hoBkVl8D4yLQKMoOhVIDTNkO_401yCoVHI4MlNHDZgI,18756
+lamindb/_run.py,sha256=xj3ER4F_yWvuNw1mr0XU-QuIPi5hBO7Ue0ygBgJQ6mc,1887
 lamindb/_save.py,sha256=It4XO448D8NG2cReo9Xy0lQBQdkMm_rCx_TGD1qZWWc,11864
-lamindb/_storage.py,sha256=VW8xq3VRv58-ciholvOdlcgvp_OIlLxx5GxLt-e2Irs,614
+lamindb/_storage.py,sha256=8wRefV-Klu6VBVtwcwppvTojeXnxRThaBdFniA0AEIw,400
 lamindb/_transform.py,sha256=E9C7psuOnsNrUQpWRuGgEUM8_pc7YhDn7n4ieHzB4X0,3169
 lamindb/_ulabel.py,sha256=XDSdZBXX_ki5s1vOths3MjF2x5DPggBR_PV_KF4SGyg,1611
 lamindb/_utils.py,sha256=LGdiW4k3GClLz65vKAVRkL6Tw-Gkx9DWAdez1jyA5bE,428
-lamindb/_view.py,sha256=GV1FrqIMmdooEkA-5zvcTWgV1nqx1sehi6WdWEaFpxM,2171
-lamindb/core/__init__.py,sha256=RkE5BrdOWNHQIz60z1npYoRim-Ot24-L7JTmLXd2WTU,1363
-lamindb/core/_data.py,sha256=xkjEvAhWeEEYRIZIi9Qbb0aEWiUPttq-qnCLx_2xy48,16172
-lamindb/core/_feature_manager.py,sha256=h7puH83LjaiJrBlmw3wSkUVO5c_LVfubCJKklPryP84,26205
+lamindb/_view.py,sha256=NdWWF75rmkCHefqBViUnsHLu3OViJ1z_bEvfcedydsw,2330
+lamindb/core/__init__.py,sha256=948oRi2FsLY1H4quaDHGSApwkM218I8scciUovJwsfs,1421
+lamindb/core/_data.py,sha256=olA2-_CueezxmFB2757lGv5g3P8ystHkhGwJECloWcY,16230
+lamindb/core/_feature_manager.py,sha256=jFp6Sd9SEX_bQEWmAf957U3T7UrGdsHg5qfQ5_odHp4,31367
 lamindb/core/_label_manager.py,sha256=d9r3tiNDFbpZQmxE1jmgdgnMYPOfTPpAKGKtRRLmVj8,9640
 lamindb/core/_mapped_collection.py,sha256=_OwFZh5SePDUD70XIK5kngv3we_Z5-YdGHNfpUSatSQ,19469
-lamindb/core/_run_context.py,sha256=aGdZv4IeYo7QpXl9H9U7RNwpgzqGeCDKPuoSdtP-CAE,18988
+lamindb/core/_run_context.py,sha256=A-hJmj56p8ggw8ADYjJiN_hLftG_aAtE8Y5leaUW6Tk,18220
 lamindb/core/_settings.py,sha256=rW1KfEXfT56XErwcnSuQxaCytpOy1kJ-u7tVmkmNmxY,6131
 lamindb/core/_sync_git.py,sha256=5Fb82eG1WYyBlfMRCMlcUZNTVk70UU_SPnfvYEBL-A8,4124
 lamindb/core/_track_environment.py,sha256=xLZ6kgzxWS6MWZ5LQ_wkbJX99vmYOT8iQ-Fz4OHCgWw,754
@@ -49,7 +49,7 @@ lamindb/integrations/__init__.py,sha256=aH2PmO2m4-vwIifMYTB0Fyyr_gZWtVnV71jT0tVW
 lamindb/integrations/_vitessce.py,sha256=Qd39OuNsL0GXU7nMvEWLRRESFQ0mwGf_ePMJE_FDYm8,2639
 lamindb/setup/__init__.py,sha256=OwZpZzPDv5lPPGXZP7-zK6UdO4FHvvuBh439yZvIp3A,410
 lamindb/setup/core/__init__.py,sha256=SevlVrc2AZWL3uALbE5sopxBnIZPWZ1IB0NBDudiAL8,167
-lamindb-0.73.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-lamindb-0.73.1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-lamindb-0.73.1.dist-info/METADATA,sha256=wJj1zRW8leb93Gj0ouXl-ujvE3UzjfNZ4i_oXQEZEvE,2735
-lamindb-0.73.1.dist-info/RECORD,,
+lamindb-0.74.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+lamindb-0.74.0.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+lamindb-0.74.0.dist-info/METADATA,sha256=2r0FMsKG8RAIaPZLLMO8FIUj13IeW87jDSIVHzOizIs,2735
+lamindb-0.74.0.dist-info/RECORD,,