lamindb 0.72.1__py3-none-any.whl → 0.73.0__py3-none-any.whl

lamindb/core/_label_manager.py

@@ -1,10 +1,11 @@
 from __future__ import annotations
 
+from collections import defaultdict
 from typing import TYPE_CHECKING, Dict
 
 import numpy as np
 from lamin_utils import colors, logger
-from lnschema_core.models import Artifact, Collection, Data, Feature, LinkORM, Registry
+from lnschema_core.models import Feature
 
 from lamindb._from_values import _print_values
 from lamindb._registry import (
@@ -19,77 +20,62 @@ from ._settings import settings
 from .schema import dict_related_model_to_related_name
 
 if TYPE_CHECKING:
+    from lnschema_core.models import Artifact, Collection, HasFeatures, Registry
+
     from lamindb._query_set import QuerySet
 
 
-def get_labels_as_dict(self: Data):
+def get_labels_as_dict(self: HasFeatures, links: bool = False):
+    exclude_set = {
+        "feature_sets",
+        "unordered_artifacts",
+        "input_of",
+        "collections",
+        "source_code_of",
+        "report_of",
+        "environment_of",
+        "collection_links",
+        "artifact_links",
+        "feature_set_links",
+        "previous_runs",
+        "feature_values",
+    }
     labels = {}  # type: ignore
     if self.id is None:
         return labels
     for related_model_name, related_name in dict_related_model_to_related_name(
-        self.__class__
+        self.__class__, links=links
     ).items():
-        if related_name in {
-            "feature_sets",
-            "unordered_artifacts",
-            "input_of",
-            "collections",
-            "source_of",
-            "report_of",
-            "environment_of",
-        }:
-            continue
-        labels[related_name] = (related_model_name, self.__getattribute__(related_name))
+        if related_name not in exclude_set:
+            labels[related_name] = (
+                related_model_name,
+                getattr(self, related_name).all(),
+            )
     return labels
 
 
-def print_labels(
-    self: Data, field: str = "name", ignore_labels_with_feature: bool = True
-):
+def print_labels(self: HasFeatures, field: str = "name", print_types: bool = False):
     labels_msg = ""
     for related_name, (related_model, labels) in get_labels_as_dict(self).items():
+        # there is a try except block here to deal with schema inconsistencies
+        # during transfer between instances
         try:
             labels_list = list(labels.values_list(field, flat=True))
             if len(labels_list) > 0:
                 get_default_str_field(labels)
-                print_values = _print_values(labels_list[:20], n=10)
-                labels_msg += f"  📎 {related_name} ({len(labels_list)}, {colors.italic(related_model)}): {print_values}\n"
+                print_values = _print_values(labels_list, n=10)
+                type_str = f": {related_model}" if print_types else ""
+                labels_msg += f"    .{related_name}{type_str} = {print_values}\n"
         except Exception:
             continue
-    if len(labels_msg) > 0:
-        return f"{colors.green('Labels')}:\n{labels_msg}"
-    else:
-        return ""
-
-
-def transfer_add_labels(labels, features_lookup_self, self, row, parents: bool = True):
-    def transfer_single_registry(validated_labels, new_labels):
-        # here the new labels are transferred to the self db
-        if len(new_labels) > 0:
-            transfer_fk_to_default_db_bulk(new_labels, using_key=None)
-            for label in new_labels:
-                transfer_to_default_db(
-                    label, using_key=None, mute=True, transfer_fk=False
-                )
-            # not saving parents for Organism during transfer
-            registry = new_labels[0].__class__
-            logger.info(f"saving {len(new_labels)} new {registry.__name__} records")
-            save(new_labels)
-        # link labels records from self db
-        self._host.labels.add(
-            validated_labels + new_labels,
-            feature=features_lookup_self.get(row["name"]),
-        )
-
-    # validate labels on the default db
-    result = validate_labels(labels, parents=parents)
-    if isinstance(result, Dict):
-        for _, (validated_labels, new_labels) in result.items():
-            transfer_single_registry(validated_labels, new_labels)
-    else:
-        transfer_single_registry(*result)
+    msg = ""
+    if labels_msg:
+        msg += f"  {colors.italic('Labels')}\n"
+        msg += labels_msg
+    return msg
 
 
+# Alex: is this a label transfer function?
 def validate_labels(labels: QuerySet | list | dict, parents: bool = True):
     def validate_labels_registry(
         labels: QuerySet | list | dict, parents: bool = True
@@ -168,7 +154,6 @@ class LabelManager:
         Args:
             records: Label records to add.
             feature: Feature under which to group the labels.
-            field: Field to parse iterable with from_values.
         """
         from ._data import add_labels
 
@@ -191,8 +176,8 @@ class LabelManager:
 
         return get_labels(self._host, feature=feature, mute=mute, flat_names=flat_names)
 
-    def add_from(self, data: Data, parents: bool = True):
-        """Transfer labels from a file or collection.
+    def add_from(self, data: HasFeatures, parents: bool = True) -> None:
+        """Add labels from an artifact or collection to another artifact or collection.
 
         Examples:
             >>> file1 = ln.Artifact(pd.DataFrame(index=[0, 1]))
@@ -207,45 +192,61 @@ class LabelManager:
         """
         from django.db.utils import ProgrammingError
 
-        features_lookup_self = {f.name: f for f in Feature.objects.filter().all()}
-        features_lookup_data = {
-            f.name: f for f in Feature.objects.using(data._state.db).filter().all()
-        }
-        for _, feature_set in data.features.feature_set_by_slot.items():
-            # add labels stratified by feature
-            if feature_set.registry == "Feature":
-                # df_slot is the Feature table with type
-                df_slot = feature_set.features.df()
-                for _, row in df_slot.iterrows():
-                    if row["dtype"].startswith("cat["):
-                        logger.info(f"transferring {row['name']}")
-                        # labels records from data db
-                        labels = data.labels.get(
-                            features_lookup_data.get(row["name"]), mute=True
-                        )
-                        transfer_add_labels(
-                            labels, features_lookup_self, self, row, parents=parents
-                        )
-        # TODO: for now, has to be duplicated
         using_key = settings._using_key
         for related_name, (_, labels) in get_labels_as_dict(data).items():
             labels = labels.all()
             try:
-                if len(labels) == 0:
+                if not labels.exists():
                     continue
-                validated_labels, new_labels = validate_labels(labels, parents=parents)
+                # look for features
+                data_name_lower = data.__class__.__name__.lower()
+                labels_by_features = defaultdict(list)
+                features = set()
+                _, new_labels = validate_labels(labels, parents=parents)
                 if len(new_labels) > 0:
                     transfer_fk_to_default_db_bulk(new_labels, using_key)
-                    for label in new_labels:
+                for label in labels:
+                    # if the link table doesn't follow this convention, we'll ignore it
+                    if not hasattr(label, f"{data_name_lower}_links"):
+                        key = None
+                    else:
+                        link = getattr(label, f"{data_name_lower}_links").get(
+                            **{f"{data_name_lower}_id": data.id}
+                        )
+                        if link.feature is not None:
+                            features.add(link.feature)
+                            key = link.feature.name
+                        else:
+                            key = None
+                    label_returned = transfer_to_default_db(
+                        label,
+                        using_key,
+                        mute=True,
+                        transfer_fk=False,
+                        save=True,
+                    )
+                    # TODO: refactor return value of transfer to default db
+                    if label_returned is not None:
+                        label = label_returned
+                    labels_by_features[key].append(label)
+                # treat features
+                _, new_features = validate_labels(list(features))
+                if len(new_features) > 0:
+                    transfer_fk_to_default_db_bulk(new_features, using_key)
+                    for feature in new_features:
                         transfer_to_default_db(
-                            label, using_key, mute=True, transfer_fk=False
+                            feature, using_key, mute=True, transfer_fk=False
                         )
-                    save(new_labels, parents=parents)
-                # this should not occur as file and collection should have the same attributes
-                # but this might not be true for custom schema
-                labels_list = validated_labels + new_labels
+                    save(new_features, parents=parents)
                 if hasattr(self._host, related_name):
-                    getattr(self._host, related_name).add(*labels_list)
+                    for feature_name, labels in labels_by_features.items():
+                        if feature_name is not None:
+                            feature_id = Feature.filter(name=feature_name).one().id
+                        else:
+                            feature_id = None
+                        getattr(self._host, related_name).add(
+                            *labels, through_defaults={"feature_id": feature_id}
+                        )
             # ProgrammingError is raised when schemas don't match between source and target instances
             except ProgrammingError:
                 continue

lamindb/core/_run_context.py

@@ -14,11 +14,17 @@ from lnschema_core.models import Param, ParamValue, RunParamValue
 from lnschema_core.types import TransformType
 from lnschema_core.users import current_user_id
 
-from lamindb._save import save
 from lamindb.core._transform_settings import transform as transform_settings
 
 from ._settings import settings
 from ._sync_git import get_transform_reference_from_git_repo
+from .exceptions import (
+    MissingTransformSettings,
+    NotebookNotSavedError,
+    NoTitleError,
+    UpdateTransformSettings,
+)
+from .versioning import bump_version as bump_version_function
 
 if TYPE_CHECKING:
     from lamindb_setup.core.types import UPathStr
@@ -32,22 +38,6 @@ msg_path_failed = (
 )
 
 
-class NotebookNotSavedError(Exception):
-    pass
-
-
-class NoTitleError(Exception):
-    pass
-
-
-class MissingTransformSettings(SystemExit):
-    pass
-
-
-class UpdateTransformSettings(SystemExit):
-    pass
-
-
 def get_uid_ext(version: str) -> str:
     from lamin_utils._base62 import encodebytes
 
@@ -86,13 +76,7 @@ def update_stem_uid_or_version(
     if bump_version:
         new_stem_uid = stem_uid
         if response == "bump":
-            try:
-                new_version = str(int(version) + 1)
-            except ValueError:
-                new_version = input(
-                    f"The current version is '{version}' - please type the new"
-                    " version: "
-                )
+            new_version = bump_version_function(version, behavior="prompt")
         else:
             new_version = response
         updated = new_version != version
@@ -145,6 +129,44 @@ ln.settings.transform.stem_uid = "{stem_uid}"
 ln.settings.transform.version = "{version}"
 """
 
+MESSAGE_UPDATE = """You updated your {transform_type}.
+
+If this is a minor update, bump your version from {old_version} to:
+
+ln.settings.transform.version = "{new_version_minor_bump}"
+
+If this is a major update, bump it to:
+
+ln.settings.transform.version = "{new_version_major_bump}"
+
+If this is a new {transform_type}, set:
+
+ln.settings.transform.stem_uid = "{new_stem_uid}"
+ln.settings.transform.version = "1"
+
+"""
+
+
+def raise_transform_settings_error_needs_update(old_version: str) -> None:
+    from lnschema_core.ids import base62_12
+
+    transform_type = "notebook" if is_run_from_ipython else "script"
+    new_stem_uid = base62_12()
+
+    raise UpdateTransformSettings(
+        MESSAGE_UPDATE.format(
+            transform_type=transform_type,
+            new_stem_uid=new_stem_uid,
+            old_version=old_version,
+            new_version_major_bump=bump_version_function(
+                old_version, bump_type="major", behavior="ignore"
+            ),
+            new_version_minor_bump=bump_version_function(
+                old_version, bump_type="minor", behavior="ignore"
+            ),
+        )
+    )
+
 
 def raise_transform_settings_error() -> None:
     from lnschema_core.ids import base62_12
@@ -227,7 +249,7 @@ class run_context:
         transform: Transform | None = None,
         new_run: bool | None = None,
         path: str | None = None,
-    ) -> None:
+    ) -> Run:
         """Track notebook or script run.
 
         Creates or loads a global :class:`~lamindb.Run` that enables data
@@ -354,7 +376,7 @@ class run_context:
         from ._track_environment import track_environment
 
         track_environment(run)
-        return None
+        return run
 
     @classmethod
     def _track_script(
@@ -474,14 +496,14 @@ class run_context:
                     transform.key = key
                     transform.save()
                     logger.important(f"updated: {transform}")
-            # check whether the notebook source code was already saved
+            # check whether transform source code was already saved
             if transform.source_code_id is not None:
                 response = None
                 if is_run_from_ipython:
                     if os.getenv("LAMIN_TESTING") is None:
                         response = input(
                             "You already saved source code for this notebook."
-                            " Bump the version before a new run? (y/n)"
+                            " Auto-bump the version before a new run? (y/n)"
                         )
                     else:
                         response = "y"
@@ -492,7 +514,7 @@ class run_context:
                         if os.getenv("LAMIN_TESTING") is None:
                             response = input(
                                 "You already saved source code for this script and meanwhile modified it without bumping a version."
-                                " Bump the version before a new run? (y/n)"
+                                " Auto-bump the version before a new run? (y/n)"
                             )
                         else:
                             response = "y"
@@ -503,8 +525,10 @@ class run_context:
                     if response == "y":
                         update_stem_uid_or_version(stem_uid, version, bump_version=True)
                     else:
-                        # we want a new stem_uid in this case, hence raise the error
-                        raise_transform_settings_error()
+                        # the user didn't agree to auto-bump, hence treat manually
+                        raise_transform_settings_error_needs_update(
+                            old_version=transform.version
+                        )
             else:
                 logger.important(f"loaded: {transform}")
         cls.transform = transform

lamindb/core/_sync_git.py

@@ -20,7 +20,9 @@ def get_git_repo_from_remote() -> Path:
     if repo_dir.exists():
         logger.warning(f"git repo {repo_dir} already exists locally")
         return repo_dir
-    logger.important(f"cloning {repo_url} into {repo_dir}")
+    logger.important(
+        f"running outside of synched git repo, cloning {repo_url} into {repo_dir}"
+    )
     result = subprocess.run(
         f"git clone --depth 10 {repo_url}.git",
         shell=True,
@@ -48,8 +50,7 @@ def check_local_git_repo() -> bool:
             # running-in-correct-git-repo
             return True
         else:
-            # running-in-wrong-git-repo
-            logger.warning("running in wrong git repo")
+            # running-outside-of-correct-git-repo
             return False
 
 

lamindb/core/datasets/_core.py

@@ -135,7 +135,7 @@ def dir_iris_images() -> UPath:  # pragma: no cover
 
     Based on: https://github.com/laminlabs/lamindb-dev-datasets/pull/2
     """
-    return UPath("s3://lamindb-dev-datasets/iris_studies")
+    return UPath("s3://lamindata/iris_studies")
 
 
 def anndata_mouse_sc_lymph_node(

lamindb/core/exceptions.py

@@ -6,11 +6,44 @@ The registry base class:
    :toctree: .
 
    ValidationError
+   NotebookNotSavedError
+   NoTitleError
+   MissingTransformSettings
+   UpdateTransformSettings
 
 """
 
 
-class ValidationError(Exception):
+class ValidationError(SystemExit):
     """Validation error: not mapped in registry."""
 
     pass
+
+
+# -------------------------------------------------------------------------------------
+# ln.track() AKA run_context
+# -------------------------------------------------------------------------------------
+
+
+class NotebookNotSavedError(Exception):
+    """Notebook wasn't saved."""
+
+    pass
+
+
+class NoTitleError(Exception):
+    """Notebook has no title."""
+
+    pass
+
+
+class MissingTransformSettings(SystemExit):
+    """User didn't define transform settings."""
+
+    pass
+
+
+class UpdateTransformSettings(SystemExit):
+    """Transform settings require update."""
+
+    pass

lamindb/core/schema.py

@@ -1,5 +1,6 @@
 from typing import Type
 
+from django.db.models import ManyToManyField
 from lnschema_core.models import Feature, FeatureSet, LinkORM, Registry
 
 
@@ -19,20 +20,22 @@ def dict_schema_name_to_model_name(orm: Type[Registry]) -> dict[str, Registry]:
     return d
 
 
-def dict_related_model_to_related_name(orm: Type[Registry]) -> dict[str, str]:
+def dict_related_model_to_related_name(
+    orm: Type[Registry], links: bool = False
+) -> dict[str, str]:
+    def include(model: Registry):
+        return not links != issubclass(model, LinkORM)
+
+    related_objects = orm._meta.related_objects + orm._meta.many_to_many
     d: dict = {
-        i.related_model.__get_name_with_schema__(): i.related_name
-        for i in orm._meta.related_objects
-        if (i.name is not None and not issubclass(i.related_model, LinkORM))
+        record.related_model.__get_name_with_schema__(): (
+            record.related_name
+            if not isinstance(record, ManyToManyField)
+            else record.name
+        )
+        for record in related_objects
+        if (record.name is not None and include(record.related_model))
     }
-    d.update(
-        {
-            i.related_model.__get_name_with_schema__(): i.name
-            for i in orm._meta.many_to_many
-            if (i.name is not None and not issubclass(i.related_model, LinkORM))
-        }
-    )
-
     return d
 
 

lamindb/core/storage/paths.py

@@ -106,12 +106,17 @@ def read_adata_h5ad(filepath, **kwargs) -> ad.AnnData:
         return adata
 
 
-def store_file_or_folder(local_path: UPathStr, storage_path: UPath) -> None:
+def store_file_or_folder(
+    local_path: UPathStr, storage_path: UPath, print_progress: bool = True
+) -> None:
     """Store file or folder (localpath) at storagepath."""
     local_path = Path(local_path)
     if not isinstance(storage_path, LocalPathClasses):
         # this uploads files and directories
-        storage_path.upload_from(local_path, dir_inplace=True, print_progress=True)
+        create_folder = False if local_path.is_dir() else None
+        storage_path.upload_from(
+            local_path, create_folder=create_folder, print_progress=print_progress
+        )
     else:  # storage path is local
         storage_path.parent.mkdir(parents=True, exist_ok=True)
         if local_path.is_file():
@@ -132,7 +137,9 @@ def delete_storage_using_key(
     delete_storage(filepath)
 
 
-def delete_storage(storagepath: Path):
+def delete_storage(
+    storagepath: Path, raise_file_not_found_error: bool = True
+) -> None | str:
     """Delete arbitrary artifact."""
     # TODO is_relative_to is not available in 3.8 and deprecated since 3.12
     # replace with check_path_is_child_of_root but this needs to first be debugged
@@ -156,8 +163,11 @@ def delete_storage(storagepath: Path):
             shutil.rmtree(storagepath)
         else:
             storagepath.rmdir()
-    else:
+    elif raise_file_not_found_error:
         raise FileNotFoundError(f"{storagepath} is not an existing path!")
+    else:
+        logger.warning(f"{storagepath} is not an existing path!")
+    return None
 
 
 # tested in lamin-usecases

lamindb/core/versioning.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Optional, Tuple
+from typing import TYPE_CHECKING, Literal
 
+from lamin_utils import logger
 from lamindb_setup.core.upath import LocalPathClasses, UPath
 from lnschema_core import ids
 
@@ -9,6 +10,51 @@ if TYPE_CHECKING:
     from lnschema_core.models import IsVersioned
 
 
+def bump_version(
+    version: str,
+    bump_type: str = "minor",
+    behavior: Literal["prompt", "error", "ignore"] = "error",
+) -> str:
+    """Bumps the version number by major or minor depending on the bump_type flag.
+
+    Parameters:
+    version (str): The current version in "MAJOR" or "MAJOR.MINOR" format.
+    bump_type (str): The type of version bump, either 'major' or 'minor'.
+
+    Returns:
+    str: The new version string.
+    """
+    try:
+        # Split the version into major and minor parts if possible
+        parts = version.split(".")
+        major = int(parts[0])
+        minor = int(parts[1]) if len(parts) > 1 else 0
+
+        if bump_type == "major":
+            # Bump the major version and reset the minor version
+            new_version = f"{major + 1}"
+        elif bump_type == "minor":
+            # Bump the minor version
+            new_version = f"{major}.{minor + 1}"
+        else:
+            raise ValueError("bump_type must be 'major' or 'minor'")
+
+    except (ValueError, IndexError):
+        if behavior == "prompt":
+            new_version = input(
+                f"The current version is '{version}' - please type the new version: "
+            )
+        elif behavior == "error":
+            raise ValueError(
+                "Cannot auto-increment non-integer castable version, please provide"
+                " manually"
+            ) from None
+        else:
+            logger.warning("could not auto-increment version, fix '?' manually")
+            new_version = "?"
+    return new_version
+
+
 def set_version(version: str | None = None, previous_version: str | None = None):
     """(Auto-) set version.
 
@@ -22,13 +68,7 @@ def set_version(version: str | None = None, previous_version: str | None = None)
     if version == previous_version:
         raise ValueError(f"Please increment the previous version: '{previous_version}'")
     if version is None and previous_version is not None:
-        try:
-            version = str(int(previous_version) + 1)  # increment version by 1
-        except ValueError:
-            raise ValueError(
-                "Cannot auto-increment non-integer castable version, please provide"
-                " manually"
-            ) from None
+        version = bump_version(previous_version, bump_type="major")
     return version
 
 

{lamindb-0.72.1.dist-info → lamindb-0.73.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.72.1
+Version: 0.73.0
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,10 +9,10 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.67.1
-Requires-Dist: lamindb_setup==0.72.2
+Requires-Dist: lnschema_core==0.68.0
+Requires-Dist: lamindb_setup==0.73.0
 Requires-Dist: lamin_utils==0.13.2
-Requires-Dist: lamin_cli==0.13.2
+Requires-Dist: lamin_cli==0.14.0
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
@@ -33,6 +33,7 @@ Requires-Dist: laminci>=0.3 ; extra == "dev"
 Requires-Dist: pytest>=6.0 ; extra == "dev"
 Requires-Dist: coverage ; extra == "dev"
 Requires-Dist: pytest-cov ; extra == "dev"
+Requires-Dist: mudata ; extra == "dev"
 Requires-Dist: nbproject_test>=0.5.1 ; extra == "dev"
 Requires-Dist: faker-biology ; extra == "dev"
 Requires-Dist: django-schema-graph ; extra == "erdiagram"