lamindb 0.74.1__py3-none-any.whl → 0.74.2__py3-none-any.whl

lamindb/_view.py

@@ -7,7 +7,7 @@ import inspect
 from lamin_utils import colors, logger
 from lamindb_setup import settings
 from lamindb_setup._init_instance import get_schema_module_name
-from lnschema_core import Registry
+from lnschema_core import Record
 
 is_run_from_ipython = getattr(builtins, "__IPYTHON__", False)
 
@@ -21,7 +21,7 @@ def view(
         n: Display the last `n` rows of a registry.
         schema: Schema module to view. Default's to
             `None` and displays all schema modules.
-        registries: List of Registry names. Defaults to
+        registries: List of Record names. Defaults to
             `None` and lists all registries.
 
     Examples:
@@ -44,8 +44,8 @@ def view(
             orm
             for orm in schema_module.__dict__.values()
             if inspect.isclass(orm)
-            and issubclass(orm, Registry)
-            and orm.__name__ != "Registry"
+            and issubclass(orm, Record)
+            and orm.__name__ != "Record"
         }
         if registries is not None:
             filtered_registries = {

lamindb/core/__init__.py

@@ -5,7 +5,7 @@ Registries:
 .. autosummary::
    :toctree: .
 
-   Registry
+   Record
    QuerySet
    QueryManager
    RecordsList
@@ -19,18 +19,20 @@ Registries:
    HasParents
    TracksRun
    TracksUpdates
+   ParamValue
+   FeatureValue
    InspectResult
    fields
 
-Annotators:
+Curators:
 
 .. autosummary::
    :toctree: .
 
-   DataFrameAnnotator
-   AnnDataAnnotator
-   MuDataAnnotator
-   AnnotateLookup
+   DataFrameCurator
+   AnnDataCurator
+   MuDataCurator
+   CurateLookup
 
 Other:
 
@@ -57,20 +59,22 @@ Modules:
 from lamin_utils._inspect import InspectResult
 from lnschema_core.models import (
     CanValidate,
+    FeatureValue,
     HasFeatures,
     HasParams,
     HasParents,
     IsVersioned,
-    Registry,
+    ParamValue,
+    Record,
     TracksRun,
     TracksUpdates,
 )
 
-from lamindb._annotate import (
-    AnnDataAnnotator,
-    AnnotateLookup,
-    DataFrameAnnotator,
-    MuDataAnnotator,
+from lamindb._curate import (
+    AnnDataCurator,
+    CurateLookup,
+    DataFrameCurator,
+    MuDataCurator,
 )
 from lamindb._query_manager import QueryManager
 from lamindb._query_set import QuerySet, RecordsList

lamindb/core/_data.py

@@ -11,7 +11,7 @@ from lnschema_core.models import (
     Feature,
     FeatureSet,
     HasFeatures,
-    Registry,
+    Record,
     Run,
     ULabel,
     __repr__,
@@ -20,7 +20,7 @@ from lnschema_core.models import (
 
 from lamindb._parents import view_lineage
 from lamindb._query_set import QuerySet
-from lamindb._registry import get_default_str_field
+from lamindb._record import get_default_str_field
 from lamindb.core._settings import settings
 
 from ._feature_manager import (
@@ -96,7 +96,7 @@ def save_feature_set_links(self: Artifact | Collection) -> None:
 
 @doc_args(HasFeatures.describe.__doc__)
 def describe(self: HasFeatures, print_types: bool = False):
-    """{}."""
+    """{}"""  # noqa: D415
     # prefetch all many-to-many relationships
     # doesn't work for describing using artifact
     # self = (
@@ -166,7 +166,7 @@ def describe(self: HasFeatures, print_types: bool = False):
     logger.print(msg)
 
 
-def validate_feature(feature: Feature, records: list[Registry]) -> None:
+def validate_feature(feature: Feature, records: list[Record]) -> None:
     """Validate feature record, adjust feature.dtype based on labels records."""
     if not isinstance(feature, Feature):
         raise TypeError("feature has to be of type Feature")
@@ -183,7 +183,7 @@ def get_labels(
     mute: bool = False,
     flat_names: bool = False,
 ) -> QuerySet | dict[str, QuerySet] | list:
-    """{}."""
+    """{}"""  # noqa: D415
     if not isinstance(feature, Feature):
         raise TypeError("feature has to be of type Feature")
     if feature.dtype is None or not feature.dtype.startswith("cat["):
@@ -210,7 +210,7 @@ def get_labels(
             ).all()
     if flat_names:
         # returns a flat list of names
-        from lamindb._registry import get_default_str_field
+        from lamindb._record import get_default_str_field
 
         values = []
         for v in qs_by_registry.values():
@@ -224,18 +224,18 @@ def get_labels(
 
 def add_labels(
     self,
-    records: Registry | list[Registry] | QuerySet | Iterable,
+    records: Record | list[Record] | QuerySet | Iterable,
     feature: Feature | None = None,
     *,
     field: StrField | None = None,
 ) -> None:
-    """{}."""
+    """{}"""  # noqa: D415
     if self._state.adding:
         raise ValueError("Please save the artifact/collection before adding a label!")
 
     if isinstance(records, (QuerySet, QuerySet.__base__)):  # need to have both
         records = records.list()
-    if isinstance(records, (str, Registry)):
+    if isinstance(records, (str, Record)):
         records = [records]
     if not isinstance(records, List):  # avoids warning for pd Series
         records = list(records)
@@ -260,7 +260,7 @@ def add_labels(
         # ask users to pass records
         if len(records_validated) == 0:
             raise ValueError(
-                "Please pass a record (a `Registry` object), not a string, e.g., via:"
+                "Please pass a record (a `Record` object), not a string, e.g., via:"
                 " label"
                 f" = ln.ULabel(name='{records[0]}')"  # type: ignore
             )

lamindb/core/_feature_manager.py

@@ -30,14 +30,14 @@ from lnschema_core.models import (
     ParamManagerArtifact,
     ParamManagerRun,
     ParamValue,
-    Registry,
+    Record,
     Run,
     ULabel,
 )
 
 from lamindb._feature import FEATURE_TYPES, convert_numpy_dtype_to_lamin_feature_type
 from lamindb._feature_set import DICT_KEYS_TYPE, FeatureSet
-from lamindb._registry import (
+from lamindb._record import (
     REGISTRY_UNIQUE_FIELD,
     get_default_str_field,
     transfer_fk_to_default_db_bulk,
@@ -118,7 +118,9 @@ def get_feature_set_links(host: Artifact | Collection) -> QuerySet:
 
 def get_link_attr(link: LinkORM | type[LinkORM], data: HasFeatures) -> str:
     link_model_name = link.__class__.__name__
-    if link_model_name == "ModelBase":  # we passed the type of the link
+    if (
+        link_model_name == "ModelBase" or link_model_name == "RecordMeta"
+    ):  # we passed the type of the link
         link_model_name = link.__name__
     link_attr = link_model_name.replace(data.__class__.__name__, "")
     if link_attr == "ExperimentalFactor":
@@ -162,6 +164,7 @@ def print_features(
                     labels_by_feature[link.feature_id].append(
                         getattr(link, link_attr).name
                     )
+        labels_msgs = []
         for feature_id, labels_list in labels_by_feature.items():
             feature = Feature.objects.using(self._state.db).get(id=feature_id)
             print_values = _print_values(labels_list, n=10)
@@ -170,8 +173,9 @@ def print_features(
                 dictionary[feature.name] = (
                     labels_list if len(labels_list) > 1 else labels_list[0]
                 )
-            labels_msg += f"    '{feature.name}'{type_str} = {print_values}\n"
-        if labels_msg:
+            labels_msgs.append(f"    '{feature.name}'{type_str} = {print_values}")
+        if len(labels_msgs) > 0:
+            labels_msg = "\n".join(sorted(labels_msgs)) + "\n"
             msg += labels_msg
 
     # non-categorical feature values
@@ -182,6 +186,7 @@ def print_features(
             getattr(self, f"{attr_name}_values")
             .values(f"{attr_name}__name", f"{attr_name}__dtype")
             .annotate(values=custom_aggregate("value", self._state.db))
+            .order_by(f"{attr_name}__name")
         )
         if len(feature_values) > 0:
             for fv in feature_values:
@@ -232,7 +237,7 @@ def parse_feature_sets_from_anndata(
     var_field: FieldAttr | None = None,
     obs_field: FieldAttr = Feature.name,
     mute: bool = False,
-    organism: str | Registry | None = None,
+    organism: str | Record | None = None,
 ) -> dict:
     data_parse = adata
     if not isinstance(adata, AnnData):  # is a path
@@ -322,7 +327,12 @@ def infer_feature_type_convert_json(
                         return FEATURE_TYPES["str"] + "[ULabel]", value
                     else:
                         return "list[str]", value
-    elif isinstance(value, Registry):
+                elif first_element_type == Record:
+                    return (
+                        f"cat[{first_element_type.__get_name_with_schema__()}]",
+                        value,
+                    )
+    elif isinstance(value, Record):
         return (f"cat[{value.__class__.__get_name_with_schema__()}]", value)
     if not mute:
         logger.warning(f"cannot infer feature type of: {value}, returning '?")
@@ -417,7 +427,7 @@ def _add_values(
     feature_param_field: FieldAttr,
     str_as_ulabel: bool = True,
 ) -> None:
-    """Annotate artifact with features & values.
+    """Curate artifact with features & values.
 
     Args:
         values: A dictionary of keys (features) & values (labels, numbers, booleans).
@@ -430,7 +440,7 @@ def _add_values(
     if isinstance(keys, DICT_KEYS_TYPE):
         keys = list(keys)  # type: ignore
     # deal with other cases later
-    assert all(isinstance(key, str) for key in keys)
+    assert all(isinstance(key, str) for key in keys)  # noqa: S101
     registry = feature_param_field.field.model
     is_param = registry == Param
     model = Param if is_param else Feature
@@ -483,10 +493,11 @@ def _add_values(
                     f"Value for feature '{key}' with type {feature.dtype} must be a number"
                 )
         elif feature.dtype.startswith("cat"):
-            if not (inferred_type.startswith("cat") or isinstance(value, Registry)):
-                raise TypeError(
-                    f"Value for feature '{key}' with type '{feature.dtype}' must be a string or record."
-                )
+            if inferred_type != "?":
+                if not (inferred_type.startswith("cat") or isinstance(value, Record)):
+                    raise TypeError(
+                        f"Value for feature '{key}' with type '{feature.dtype}' must be a string or record."
+                    )
         elif not inferred_type == feature.dtype:
             raise ValidationError(
                 f"Expected dtype for '{key}' is '{feature.dtype}', got '{inferred_type}'"
@@ -499,15 +510,21 @@ def _add_values(
                 feature_value = value_model(**filter_kwargs)
             feature_values.append(feature_value)
         else:
-            if isinstance(value, Registry):
-                if value._state.adding:
-                    raise ValidationError(
-                        "Please save your label record before annotation."
+            if isinstance(value, Record) or (
+                isinstance(value, Iterable) and isinstance(next(iter(value)), Record)
+            ):
+                if isinstance(value, Record):
+                    label_records = [value]
+                else:
+                    label_records = value  # type: ignore
+                for record in label_records:
+                    if record._state.adding:
+                        raise ValidationError(
+                            f"Please save {record} before annotation."
+                        )
+                    features_labels[record.__class__.__get_name_with_schema__()].append(
+                        (feature, record)
                     )
-                label_record = value
-                features_labels[
-                    label_record.__class__.__get_name_with_schema__()
-                ].append((feature, label_record))
             else:
                 if isinstance(value, str):
                     values = [value]  # type: ignore
@@ -589,7 +606,7 @@ def _add_values(
         links = [
             LinkORM(
                 **{
-                    f"{self._host.__get_name_with_schema__().lower()}_id": self._host.id,
+                    f"{self._host.__class__.__get_name_with_schema__().lower()}_id": self._host.id,
                     valuefield_id: feature_value.id,
                 }
             )
@@ -606,7 +623,7 @@ def add_values_features(
     feature_field: FieldAttr = Feature.name,
     str_as_ulabel: bool = True,
 ) -> None:
-    """Annotate artifact with features & values.
+    """Curate artifact with features & values.
 
     Args:
         values: A dictionary of keys (features) & values (labels, numbers, booleans).
@@ -621,7 +638,7 @@ def add_values_params(
     self,
     values: dict[str, str | int | float | bool],
 ) -> None:
-    """Annotate artifact with features & values.
+    """Curate artifact with features & values.
 
     Args:
         values: A dictionary of keys (features) & values (labels, numbers, booleans).
@@ -630,7 +647,7 @@ def add_values_params(
 
 
 def add_feature_set(self, feature_set: FeatureSet, slot: str) -> None:
-    """Annotate artifact with a feature set.
+    """Curate artifact with a feature set.
 
     Args:
         feature_set: `FeatureSet` A feature set record.
@@ -666,10 +683,10 @@ def _add_set_from_df(
 ):
     """Add feature set corresponding to column names of DataFrame."""
     if isinstance(self._host, Artifact):
-        assert self._host.accessor == "DataFrame"
+        assert self._host.accessor == "DataFrame"  # noqa: S101
     else:
         # Collection
-        assert self._host.artifact.accessor == "DataFrame"
+        assert self._host.artifact.accessor == "DataFrame"  # noqa: S101
 
     # parse and register features
     registry = field.field.model
@@ -693,11 +710,11 @@ def _add_set_from_anndata(
     var_field: FieldAttr,
     obs_field: FieldAttr | None = Feature.name,
     mute: bool = False,
-    organism: str | Registry | None = None,
+    organism: str | Record | None = None,
 ):
     """Add features from AnnData."""
     if isinstance(self._host, Artifact):
-        assert self._host.accessor == "AnnData"
+        assert self._host.accessor == "AnnData"  # noqa: S101
     else:
         raise NotImplementedError()
 
@@ -721,13 +738,13 @@ def _add_set_from_mudata(
     var_fields: dict[str, FieldAttr],
     obs_fields: dict[str, FieldAttr] = None,
     mute: bool = False,
-    organism: str | Registry | None = None,
+    organism: str | Record | None = None,
 ):
     """Add features from MuData."""
     if obs_fields is None:
         obs_fields = {}
     if isinstance(self._host, Artifact):
-        assert self._host.accessor == "MuData"
+        assert self._host.accessor == "MuData"  # noqa: S101
     else:
         raise NotImplementedError()
 

lamindb/core/_label_manager.py

@@ -8,7 +8,7 @@ from lamin_utils import colors, logger
 from lnschema_core.models import Feature
 
 from lamindb._from_values import _print_values
-from lamindb._registry import (
+from lamindb._record import (
     REGISTRY_UNIQUE_FIELD,
     get_default_str_field,
     transfer_fk_to_default_db_bulk,
@@ -20,7 +20,7 @@ from ._settings import settings
 from .schema import dict_related_model_to_related_name
 
 if TYPE_CHECKING:
-    from lnschema_core.models import Artifact, Collection, HasFeatures, Registry
+    from lnschema_core.models import Artifact, Collection, HasFeatures, Record
 
     from lamindb._query_set import QuerySet
 
@@ -66,7 +66,7 @@ def print_labels(self: HasFeatures, field: str = "name", print_types: bool = Fal
                 print_values = _print_values(labels_list, n=10)
                 type_str = f": {related_model}" if print_types else ""
                 labels_msg += f"    .{related_name}{type_str} = {print_values}\n"
-        except Exception:
+        except Exception:  # noqa: S112
             continue
     msg = ""
     if labels_msg:
@@ -102,7 +102,7 @@ def validate_labels(labels: QuerySet | list | dict, parents: bool = True):
                 records = registry.from_values(label_uids, field=field)
                 if len(records) > 0:
                     save(records, parents=parents)
-            except Exception:
+            except Exception:  # noqa S110
                 pass
             field = "uid"
             label_uids = np.array(
@@ -146,7 +146,7 @@ class LabelManager:
 
     def add(
         self,
-        records: Registry | list[Registry] | QuerySet,
+        records: Record | list[Record] | QuerySet,
         feature: Feature | None = None,
     ) -> None:
         """Add one or several labels and associate them with a feature.

lamindb/core/_mapped_collection.py

@@ -107,7 +107,10 @@ class MappedCollection:
         parallel: bool = False,
         dtype: str | None = None,
     ):
-        assert join in {None, "inner", "outer"}
+        if join not in {None, "inner", "outer"}:  # pragma: nocover
+            raise ValueError(
+                f"join must be one of None, 'inner, or 'outer' but was {type(join)}"
+            )
 
         if layers_keys is None:
             self.layers_keys = ["X"]

lamindb/core/_run_context.py

@@ -43,7 +43,7 @@ def get_uid_ext(version: str) -> str:
     # merely zero-padding the nbproject version such that the base62 encoding is
     # at least 4 characters long doesn't yields sufficiently diverse hashes and
     # leads to collisions; it'd be nice because the uid_ext would be ordered
-    return encodebytes(hashlib.md5(version.encode()).digest())[:4]
+    return encodebytes(hashlib.md5(version.encode()).digest())[:4]  # noqa: S324
 
 
 def update_stem_uid_or_version(
@@ -113,7 +113,7 @@ def get_notebook_name_colab() -> str:
 
     ip = gethostbyname(gethostname())  # 172.28.0.12
     try:
-        name = get(f"http://{ip}:9000/api/sessions").json()[0]["name"]
+        name = get(f"http://{ip}:9000/api/sessions").json()[0]["name"]  # noqa: S113
     except Exception:
         logger.warning(
             "could not get notebook name from Google Colab, using: notebook.ipynb"

lamindb/core/_settings.py

@@ -108,7 +108,8 @@ class Settings:
         For example: `ln.sync_git_repo = https://github.com/laminlabs/redun-lamin`
         """
         self._sync_git_repo = sanitize_git_repo_url(value)
-        assert self._sync_git_repo.startswith("https://")
+        if not self._sync_git_repo.startswith("https://"):  # pragma: nocover
+            raise ValueError("git repository URL must start with 'https://'.")
 
     @property
     def storage(self) -> StorageSettings:

lamindb/core/_sync_git.py

@@ -24,8 +24,7 @@ def get_git_repo_from_remote() -> Path:
         f"running outside of synched git repo, cloning {repo_url} into {repo_dir}"
     )
     result = subprocess.run(
-        f"git clone --depth 10 {repo_url}.git",
-        shell=True,
+        ["git", "clone", "--depth", "10", f"{repo_url}.git"],
         capture_output=True,
         cwd=setup_settings.storage.cache_dir,
     )
@@ -36,8 +35,7 @@ def get_git_repo_from_remote() -> Path:
 
 def check_local_git_repo() -> bool:
     result = subprocess.run(
-        "git config --get remote.origin.url",
-        shell=True,
+        ["git", "config", "--get remote.origin.url"],
         capture_output=True,
     )
     result_str = result.stdout.decode().strip()
@@ -55,10 +53,9 @@ def check_local_git_repo() -> bool:
 
 
 def get_git_commit_hash(blob_hash: str, repo_dir: Path | None = None) -> str | None:
-    command = f"git log --find-object={blob_hash} --pretty=format:%H"
+    command = ["git", "log", f"--find-object={blob_hash}", "--pretty=format:%H"]
     result = subprocess.run(
         command,
-        shell=True,
         capture_output=True,
         cwd=repo_dir,
     )
@@ -68,7 +65,7 @@ def get_git_commit_hash(blob_hash: str, repo_dir: Path | None = None) -> str | N
     if commit_hash == "" or result.returncode == 1:
         return None
     else:
-        assert (
+        assert (  # noqa: S101
             len(commit_hash) == 40
         ), f"commit hash |{commit_hash}| is not 40 characters long"
         return commit_hash
@@ -82,21 +79,34 @@ def get_filepath_within_git_repo(
     # from anywhere in the repo, hence, let's get the root
     repo_root = (
         subprocess.run(
-            "git rev-parse --show-toplevel",
-            shell=True,
+            ["git", "rev-parse", "--show-toplevel"],
             capture_output=True,
             cwd=repo_dir,
         )
         .stdout.decode()
         .strip()
     )
-    command = f"git ls-tree -r {commit_hash} | grep -E {blob_hash}"
+    # Run the git commands separately to circumvent spawning a shell
+    git_command = ["git", "ls-tree", "-r", commit_hash]
+    git_process = subprocess.Popen(
+        git_command,
+        stdout=subprocess.PIPE,
+        cwd=repo_root,
+    )
+
+    grep_command = ["grep", "-E", blob_hash]
     result = subprocess.run(
-        command,
-        shell=True,
+        grep_command,
+        stdin=git_process.stdout,
         capture_output=True,
         cwd=repo_root,
     )
+
+    # Close the stdout to allow git_process to receive a SIGPIPE if grep_command exits
+    git_process.stdout.close()
+    git_process.wait()
+
+    command = " ".join(git_command) + " | " + " ".join(grep_command)
     if result.returncode != 0 and result.stderr.decode() != "":
         raise RuntimeError(f"{command}\n{result.stderr.decode()}")
     if len(result.stdout.decode()) == 0:

lamindb/core/_track_environment.py

@@ -15,7 +15,11 @@ def track_environment(run: Run) -> None:
     # create a requirements.txt
     # we don't create a conda environment.yml mostly for its slowness
     try:
-        result = subprocess.run(f"pip freeze > {str(filepath)}", shell=True)
+        with open(filepath, "w") as f:
+            result = subprocess.run(
+                ["pip", "freeze"],
+                stdout=f,
+            )
     except OSError as e:
         result = None
         logger.warning(f"could not run pip freeze with error {e}")

lamindb/core/fields.py

@@ -1,6 +1,6 @@
 """Fields.
 
-The field accessor of a :class:`~lamindb.core.Registry`:
+The field accessor of a :class:`~lamindb.core.Record`:
 
 .. autosummary::
    :toctree: .

lamindb/core/schema.py

@@ -1,10 +1,10 @@
 from __future__ import annotations
 
 from django.db.models import ManyToManyField
-from lnschema_core.models import Feature, FeatureSet, LinkORM, Registry
+from lnschema_core.models import Feature, FeatureSet, LinkORM, Record
 
 
-def dict_schema_name_to_model_name(orm: type[Registry]) -> dict[str, Registry]:
+def dict_schema_name_to_model_name(orm: type[Record]) -> dict[str, Record]:
     d: dict = {
         i.related_model.__get_name_with_schema__(): i.related_model
         for i in orm._meta.related_objects
@@ -21,9 +21,9 @@ def dict_schema_name_to_model_name(orm: type[Registry]) -> dict[str, Registry]:
 
 
 def dict_related_model_to_related_name(
-    orm: type[Registry], links: bool = False
+    orm: type[Record], links: bool = False
 ) -> dict[str, str]:
-    def include(model: Registry):
+    def include(model: Record):
         return not links != issubclass(model, LinkORM)
 
     related_objects = orm._meta.related_objects + orm._meta.many_to_many
@@ -39,7 +39,7 @@ def dict_related_model_to_related_name(
     return d
 
 
-def get_related_name(features_type: type[Registry]) -> str:
+def get_related_name(features_type: type[Record]) -> str:
     candidates = [
         field.related_name
         for field in FeatureSet._meta.related_objects
@@ -49,7 +49,7 @@ def get_related_name(features_type: type[Registry]) -> str:
         raise ValueError(
             f"Can't create feature sets from {features_type.__name__} because it's not"
             " related to it!\nYou need to create a link model between FeatureSet and"
-            " your Registry in your custom schema.\nTo do so, add a"
+            " your Record in your custom schema.\nTo do so, add a"
             " line:\nfeature_sets = models.ManyToMany(FeatureSet,"
             " related_name='mythings')\n"
         )