lamindb 0.77.0__py3-none-any.whl → 0.77.2__py3-none-any.whl

lamindb/core/__init__.py

@@ -9,7 +9,7 @@ Registries:
    Registry
    QuerySet
    QueryManager
-   RecordsList
+   RecordList
    FeatureManager
    ParamManager
    LabelManager
@@ -88,7 +88,7 @@ from lamindb._curate import (
     MuDataCurator,
 )
 from lamindb._query_manager import QueryManager
-from lamindb._query_set import QuerySet, RecordsList
+from lamindb._query_set import QuerySet, RecordList
 from lamindb.core._feature_manager import FeatureManager, ParamManager
 from lamindb.core._label_manager import LabelManager
 

lamindb/core/_context.py

@@ -90,7 +90,7 @@ def raise_missing_context(transform_type: str, key: str) -> bool:
             f"you already have a transform with key '{key}': Transform('{transform.uid[:8]}')\n"
             f'  (1) to make a revision, run: ln.track("{new_uid}")\n  (2) to create a new transform, rename your {transform_type} file and re-run: ln.track()'
         )
-    if transform_type == "notebook":
+    if is_run_from_ipython:
         print(f"→ {message}")
         response = input("→ Ready to re-run? (y/n)")
         if response == "y":
@@ -343,7 +343,7 @@ class Context:
             )
             if run is not None:  # loaded latest run
                 run.started_at = datetime.now(timezone.utc)  # update run time
-                self._logging_message_track += f", started Run('{run.uid[:8]}') at {format_field_value(run.started_at)}"
+                self._logging_message_track += f", re-started Run('{run.uid[:8]}') at {format_field_value(run.started_at)}"
 
         if run is None:  # create new run
             run = Run(
@@ -433,7 +433,7 @@ class Context:
                 nb = nbproject.dev.read_notebook(path_str)
                 self._logging_message_imports += (
                     "notebook imports:"
-                    f" {pretty_pypackages(infer_pypackages(nb, pin_versions=True))}\n"
+                    f" {pretty_pypackages(infer_pypackages(nb, pin_versions=True))}"
                 )
             except Exception:
                 logger.debug("inferring imported packages failed")
@@ -579,15 +579,11 @@ class Context:
             `lamin save script.py` or `lamin save notebook.ipynb` → `docs </cli#lamin-save>`__
 
         """
-        from lamindb._finish import save_context_core
-
-        def get_seconds_since_modified(filepath) -> float:
-            return datetime.now().timestamp() - filepath.stat().st_mtime
-
-        def get_shortcut() -> str:
-            import platform
-
-            return "CMD + s" if platform.system() == "Darwin" else "CTRL + s"
+        from lamindb._finish import (
+            get_seconds_since_modified,
+            get_shortcut,
+            save_context_core,
+        )
 
         if self.run is None:
             raise TrackNotCalled("Please run `ln.track()` before `ln.finish()`")
@@ -609,7 +605,7 @@ class Context:
                 self.transform.save()
             if get_seconds_since_modified(self._path) > 2 and not ln_setup._TESTING:
                 raise NotebookNotSaved(
-                    f"Please save the notebook in your editor (shortcut `{get_shortcut()}`) right before calling `ln.finish()`"
+                    f"Please save the notebook in your editor (shortcut `{get_shortcut()}`) within 2 sec before calling `ln.finish()`"
                 )
         save_context_core(
             run=self.run,

lamindb/core/_data.py

@@ -18,21 +18,16 @@ from lnschema_core.models import (
     record_repr,
 )
 
-from lamindb._parents import view_lineage
 from lamindb._query_set import QuerySet
-from lamindb._record import get_name_field
 from lamindb.core._settings import settings
 
 from ._context import context
 from ._django import get_artifact_with_related, get_related_model
 from ._feature_manager import (
     add_label_feature_links,
-    get_feature_set_links,
     get_host_id_field,
     get_label_links,
-    print_features,
 )
-from ._label_manager import print_labels
 from .exceptions import ValidationError
 from .schema import (
     dict_related_model_to_related_name,
@@ -129,21 +124,10 @@ def format_input_of_runs(self, print_types):
     return ""
 
 
-def format_labels_and_features(self, related_data, print_types):
-    msg = print_labels(
-        self, m2m_data=related_data.get("m2m", {}), print_types=print_types
-    )
-    if isinstance(self, Artifact):
-        msg += print_features(  # type: ignore
-            self,
-            related_data=related_data,
-            print_types=print_types,
-            print_params=hasattr(self, "type") and self.type == "model",
-        )
-    return msg
-
-
 def _describe_postgres(self: Artifact | Collection, print_types: bool = False):
+    from ._describe import describe_general
+    from ._feature_manager import describe_features
+
     model_name = self.__class__.__name__
     msg = f"{colors.green(model_name)}{record_repr(self, include_foreign_keys=False).lstrip(model_name)}\n"
     if self._state.db is not None and self._state.db != "default":
@@ -161,30 +145,21 @@ def _describe_postgres(self: Artifact | Collection, print_types: bool = False):
     else:
         result = get_artifact_with_related(self, include_fk=True, include_m2m=True)
     related_data = result.get("related_data", {})
-    fk_data = related_data.get("fk", {})
-
-    # Provenance
-    prov_msg = format_provenance(self, fk_data, print_types)
-    if prov_msg:
-        msg += f"  {colors.italic('Provenance')}\n{prov_msg}"
-
-    # Input of runs
-    input_of_message = format_input_of_runs(self, print_types)
-    if input_of_message:
-        msg += f"  {colors.italic('Usage')}\n{input_of_message}"
-
-    # Labels and features
-    msg += format_labels_and_features(self, related_data, print_types)
+    # TODO: fk_data = related_data.get("fk", {})
+
+    tree = describe_general(self)
+    return describe_features(
+        self,
+        tree=tree,
+        related_data=related_data,
+        with_labels=True,
+        print_params=hasattr(self, "type") and self.type == "model",
+    )
 
-    # Print entire message
-    logger.print(msg)
 
-
-@doc_args(Artifact.describe.__doc__)
-def describe(self: Artifact | Collection, print_types: bool = False):
-    """{}"""  # noqa: D415
-    if not self._state.adding and connections[self._state.db].vendor == "postgresql":
-        return _describe_postgres(self, print_types=print_types)
+def _describe_sqlite(self: Artifact | Collection, print_types: bool = False):
+    from ._describe import describe_general
+    from ._feature_manager import describe_features
 
     model_name = self.__class__.__name__
     msg = f"{colors.green(model_name)}{record_repr(self, include_foreign_keys=False).lstrip(model_name)}\n"
@@ -192,7 +167,6 @@ def describe(self: Artifact | Collection, print_types: bool = False):
         msg += f"  {colors.italic('Database instance')}\n"
         msg += f"    slug: {self._state.db}\n"
 
-    prov_msg = ""
     fields = self._meta.fields
     direct_fields = []
     foreign_key_fields = []
@@ -219,35 +193,26 @@ def describe(self: Artifact | Collection, print_types: bool = False):
             .prefetch_related(*many_to_many_fields)
             .get(id=self.id)
         )
+    tree = describe_general(self)
+    return describe_features(
+        self,
+        tree=tree,
+        with_labels=True,
+        print_params=hasattr(self, "type") and self.type == "model",
+    )
 
-    # provenance
-    if len(foreign_key_fields) > 0:  # always True for Artifact and Collection
-        fields_values = [(field, getattr(self, field)) for field in foreign_key_fields]
-        type_str = lambda attr: (
-            f": {attr.__class__.__get_name_with_schema__()}" if print_types else ""
-        )
-        related_msg = "".join(
-            [
-                f"    .{field_name}{type_str(attr)} = {format_field_value(getattr(attr, get_name_field(attr)))}\n"
-                for (field_name, attr) in fields_values
-                if attr is not None
-            ]
-        )
-        prov_msg += related_msg
-    if prov_msg:
-        msg += f"  {colors.italic('Provenance')}\n"
-        msg += prov_msg
 
-    # Input of runs
-    input_of_message = format_input_of_runs(self, print_types)
-    if input_of_message:
-        msg += f"  {colors.italic('Usage')}\n{input_of_message}"
+@doc_args(Artifact.describe.__doc__)
+def describe(self: Artifact | Collection, print_types: bool = False):
+    """{}"""  # noqa: D415
+    from ._describe import print_rich_tree
 
-    # Labels and features
-    msg += format_labels_and_features(self, {}, print_types)
+    if not self._state.adding and connections[self._state.db].vendor == "postgresql":
+        tree = _describe_postgres(self, print_types=print_types)
+    else:
+        tree = _describe_sqlite(self, print_types=print_types)
 
-    # Print entire message
-    logger.print(msg)
+    print_rich_tree(tree)
 
 
 def validate_feature(feature: Feature, records: list[Record]) -> None:
@@ -314,6 +279,7 @@ def add_labels(
     field: StrField | None = None,
     feature_ref_is_name: bool | None = None,
     label_ref_is_name: bool | None = None,
+    from_curator: bool = False,
 ) -> None:
     """{}"""  # noqa: D415
     if self._state.adding:
@@ -374,11 +340,36 @@ def add_labels(
     else:
         validate_feature(feature, records)  # type:ignore
         records_by_registry = defaultdict(list)
+        feature_sets = self.feature_sets.filter(registry="Feature").all()
+        internal_features = set()  # type: ignore
+        if len(feature_sets) > 0:
+            for feature_set in feature_sets:
+                internal_features = internal_features.union(
+                    set(feature_set.members.values_list("name", flat=True))
+                )  # type: ignore
         for record in records:
             records_by_registry[record.__class__.__get_name_with_schema__()].append(
                 record
             )
         for registry_name, records in records_by_registry.items():
+            if not from_curator and feature.name in internal_features:
+                raise ValidationError(
+                    "Cannot manually annotate internal feature with label. Please use ln.Curator"
+                )
+            if registry_name not in feature.dtype:
+                if not feature.dtype.startswith("cat"):
+                    raise ValidationError(
+                        f"Feature {feature.name} needs dtype='cat' for label annotation, currently has dtype='{feature.dtype}'"
+                    )
+                if feature.dtype == "cat":
+                    feature.dtype = f"cat[{registry_name}]"
+                    feature.save()
+                elif registry_name not in feature.dtype:
+                    new_dtype = feature.dtype.rstrip("]") + f"|{registry_name}]"
+                    raise ValidationError(
+                        f"Label type {registry_name} is not valid for Feature(name='{feature.name}', dtype='{feature.dtype}'), consider updating to dtype='{new_dtype}'"
+                    )
+
             if registry_name not in self.features._accessor_by_registry:
                 logger.warning(f"skipping {registry_name}")
                 continue
@@ -393,27 +384,6 @@ def add_labels(
                 feature_ref_is_name=feature_ref_is_name,
                 label_ref_is_name=label_ref_is_name,
             )
-        links_feature_set = get_feature_set_links(self)
-        feature_set_ids = [link.featureset_id for link in links_feature_set.all()]
-        # get all linked features of type Feature
-        feature_sets = FeatureSet.filter(id__in=feature_set_ids).all()
-        {
-            links_feature_set.filter(featureset_id=feature_set.id)
-            .one()
-            .slot: feature_set.features.all()
-            for feature_set in feature_sets
-            if "Feature" == feature_set.registry
-        }
-        for registry_name, _ in records_by_registry.items():
-            if registry_name not in feature.dtype:
-                logger.debug(
-                    f"updated categorical feature '{feature.name}' type with registry '{registry_name}'"
-                )
-                if not feature.dtype.startswith("cat["):
-                    feature.dtype = f"cat[{registry_name}]"
-                elif registry_name not in feature.dtype:
-                    feature.dtype = feature.dtype.rstrip("]") + f"|{registry_name}]"
-                feature.save()
 
 
 def _track_run_input(

lamindb/core/_describe.py

@@ -0,0 +1,139 @@
+from __future__ import annotations
+
+import datetime
+from typing import TYPE_CHECKING
+
+from lamin_utils import logger
+from rich.text import Text
+from rich.tree import Tree
+
+if TYPE_CHECKING:
+    from lnschema_core.models import Artifact, Collection, Run
+
+
+def highlight_time(iso: str):
+    tz = datetime.datetime.now().astimezone().tzinfo
+    res = (
+        datetime.datetime.fromisoformat(iso)
+        .replace(tzinfo=datetime.timezone.utc)
+        .astimezone(tz)
+        .strftime("%Y-%m-%d %H:%M:%S")
+    )
+    return Text(res, style="dim")
+
+
+# Define consistent column widths
+NAME_WIDTH = 25
+TYPE_WIDTH = 25
+VALUES_WIDTH = 40
+
+
+def print_rich_tree(tree: Tree, fallback=str):
+    from rich.console import Console
+
+    console = Console(force_terminal=True)
+
+    if tree.children:
+        try:
+            from IPython import get_ipython
+            from IPython.core.interactiveshell import InteractiveShell
+            from IPython.display import display
+
+            shell = get_ipython()
+            if isinstance(shell, InteractiveShell):  # Covers all interactive shells
+                display(tree)
+                return ""
+            else:
+                with console.capture() as capture:
+                    console.print(tree)
+                return capture.get()
+        except (ImportError, NameError):
+            with console.capture() as capture:
+                console.print(tree)
+            return capture.get()
+    else:
+        return fallback
+
+
+def describe_header(self: Artifact | Collection | Run) -> Tree:
+    if hasattr(self, "is_latest") and not self.is_latest:
+        logger.warning(
+            f"This is not the latest version of the {self.__class__.__name__}."
+        )
+    if hasattr(self, "visibility"):
+        if self.visibility == 0:
+            logger.warning("This artifact is hidden.")
+        elif self.visibility == -1:
+            logger.warning("This artifact is the trash.")
+    # initialize tree
+    suffix = self.suffix if hasattr(self, "suffix") and self.suffix else ""
+    accessor = self._accessor if hasattr(self, "_accessor") and self._accessor else ""
+    suffix_accessor = (
+        f"{suffix}/{accessor}" if suffix and accessor else suffix or accessor or ""
+    )
+
+    tree = Tree(
+        Text.assemble(
+            (self.__class__.__name__, "bold"), (f" {suffix_accessor}", "bold dim")
+        ),
+        guide_style="dim",  # dim the connecting lines
+    )
+    return tree
+
+
+def describe_general(self: Artifact | Collection, tree: Tree | None = None) -> Tree:
+    if tree is None:
+        tree = describe_header(self)
+
+    # add general information (order is the same as in API docs)
+    general = tree.add(Text("General", style="bold bright_cyan"))
+    general.add(f".uid = '{self.uid}'")
+    if hasattr(self, "key") and self.key:
+        general.add(
+            f".key = '{self.key}'" if self._key_is_virtual else f".key = {self.key}"
+        )
+    if hasattr(self, "size") and self.size:
+        general.add(f".size = {self.size}")
+    if hasattr(self, "hash") and self.hash:
+        general.add(f".hash = '{self.hash}'")
+    if hasattr(self, "n_objects") and self.n_objects:
+        general.add(f".n_objects = {self.n_objects}")
+    if hasattr(self, "n_observations") and self.n_observations:
+        general.add(Text(f".n_observations = {self.n_observations}"))
+    if hasattr(self, "version") and self.version:
+        general.add(Text(f".version = '{self.version}'"))
+
+    if hasattr(self, "storage"):
+        storage_root = self.storage.root
+        # general.add(f".storage = {storage_root}")
+        general.add(
+            Text.assemble(
+                ".path = ",
+                (storage_root, "dim"),
+                f"{str(self.path).removeprefix(storage_root)}",
+            )
+        )
+    if hasattr(self, "created_by") and self.created_by:
+        general.add(
+            Text.assemble(
+                ".created_by = ",
+                (
+                    self.created_by.handle
+                    if self.created_by.name is None
+                    else f"{self.created_by.handle} ({self.created_by.name})"
+                ),
+            )
+        )
+    if hasattr(self, "created_at") and self.created_at:
+        general.add(
+            Text.assemble(".created_at = ", highlight_time(str(self.created_at)))
+        )
+    if hasattr(self, "transform") and self.transform:
+        general.add(
+            Text(
+                f".transform = '{self.transform.name}'",
+                style="cyan3",
+            )
+        )
+
+    return tree

lamindb/core/_django.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from django.contrib.postgres.aggregates import ArrayAgg
 from django.db import connection
 from django.db.models import F, OuterRef, Q, Subquery
@@ -35,7 +37,7 @@ def get_artifact_with_related(
     """Fetch an artifact with its related data."""
     from lamindb._can_curate import get_name_field
 
-    from ._label_manager import LABELS_EXCLUDE_SET
+    from ._label_manager import EXCLUDE_LABELS
 
     model = artifact.__class__
     schema_modules = get_schemas_modules(artifact._state.db)
@@ -54,7 +56,7 @@ def get_artifact_with_related(
             for v in dict_related_model_to_related_name(
                 model, instance=artifact._state.db
             ).values()
-            if not v.startswith("_") and v not in LABELS_EXCLUDE_SET
+            if not v.startswith("_") and v not in EXCLUDE_LABELS
         ]
     )
     link_tables = (
@@ -179,14 +181,11 @@ def get_featureset_m2m_relations(
 
         # Get the correct field names for the through table
         through_model = getattr(FeatureSet, name).through
-        related_field = (
-            through_model.__name__.replace("FeatureSet", "").lower().replace("_", "")
-        )
 
         # Subquery to get limited related records
         limited_related = Subquery(
             through_model.objects.filter(featureset=OuterRef("pk")).values(
-                related_field
+                related_model.__name__.lower()
             )[:limit]
         )