lamindb 1.0.5__py3-none-any.whl → 1.1.1__py3-none-any.whl

lamindb/_finish.py

@@ -96,7 +96,7 @@ def save_run_logs(run: Run, save_run: bool = False) -> None:
     if logs_path.exists():
         if run.report is not None:
             logger.important("overwriting run.report")
-        artifact = Artifact(
+        artifact = Artifact(  # type: ignore
             logs_path,
             description=f"log streams of run {run.uid}",
             _branch_code=0,
@@ -159,7 +159,7 @@ def notebook_to_report(notebook_path: Path, output_path: Path) -> None:
     output_path.write_text(html, encoding="utf-8")
 
 
-def notebook_to_script(
+def notebook_to_script(  # type: ignore
     transform: Transform, notebook_path: Path, script_path: Path | None = None
 ) -> None | str:
     import jupytext
@@ -207,8 +207,13 @@ def clean_r_notebook_html(file_path: Path) -> tuple[str | None, Path]:
 
 
 def check_filepath_recently_saved(filepath: Path, is_finish_retry: bool) -> bool:
-    recently_saved_time = 3 if not is_finish_retry else 20
+    # the recently_saved_time needs to be very low for the first check
+    # because an accidental save (e.g. via auto-save) might otherwise lead
+    # to upload of an outdated notebook
+    # also see implementation for R notebooks below
+    offset_saved_time = 0.3 if not is_finish_retry else 20
     for retry in range(30):
+        recently_saved_time = offset_saved_time + retry  # sleep time is 1 sec
         if get_seconds_since_modified(filepath) > recently_saved_time:
             if retry == 0:
                 prefix = f"{LEVEL_TO_COLORS[20]}{LEVEL_TO_ICONS[20]}{RESET_COLOR}"
@@ -316,7 +321,8 @@ def save_context_core(
                 f"no html report found; to attach one, create an .html export for your {filepath.suffix} file and then run: lamin save {filepath}"
             )
     if report_path is not None and is_r_notebook and not from_cli:  # R notebooks
-        recently_saved_time = 3 if not is_retry else 20
+        # see comment above in check_filepath_recently_saved
+        recently_saved_time = 0.3 if not is_retry else 20
         if get_seconds_since_modified(report_path) > recently_saved_time:
             # the automated retry solution of Jupyter notebooks does not work in RStudio because the execution of the notebook cell
             # seems to block the event loop of the frontend
@@ -365,7 +371,7 @@ def save_context_core(
                 artifact = ln.Artifact.filter(hash=hash, _branch_code=0).one_or_none()
                 new_env_artifact = artifact is None
                 if new_env_artifact:
-                    artifact = ln.Artifact(
+                    artifact = ln.Artifact(  # type: ignore
                         env_path,
                         description="requirements.txt",
                         _branch_code=0,
@@ -411,7 +417,7 @@ def save_context_core(
                     else:
                         logger.important("report is already saved")
                 else:
-                    report_file = ln.Artifact(
+                    report_file = ln.Artifact(  # type: ignore
                         report_path,
                         description=f"Report of run {run.uid}",
                         _branch_code=0,  # hidden file
@@ -430,7 +436,15 @@ def save_context_core(
     # save both run & transform records if we arrive here
     if run is not None:
         run.save()
-    transform.save()
+    transform_id_prior_to_save = transform.id
+    transform.save()  # this in-place updates the state of transform upon hash collision
+    if transform.id != transform_id_prior_to_save:
+        # the hash existed and we're actually back to the previous version
+        # hence, this was in fact a run of the previous transform rather than of
+        # the new transform
+        # this can happen in interactive notebooks if the user makes no change to the notebook
+        run.transform = transform
+        run.save()
 
     # finalize
     if not from_cli and run is not None:

lamindb/_from_values.py

@@ -9,8 +9,6 @@ from lamin_utils import colors, logger
 from lamindb._query_set import RecordList
 from lamindb.models import Record
 
-from .core._settings import settings
-
 if TYPE_CHECKING:
     from collections.abc import Iterable
 
@@ -29,88 +27,72 @@ def get_or_create_records(
     mute: bool = False,
 ) -> RecordList:
     """Get or create records from iterables."""
-    registry = field.field.model
+    registry = field.field.model  # type: ignore
     if create:
-        return RecordList([registry(**{field.field.name: value}) for value in iterable])
-    creation_search_names = settings.creation.search_names
+        return RecordList([registry(**{field.field.name: value}) for value in iterable])  # type: ignore
     organism = _get_organism_record(field, organism)
-    settings.creation.search_names = False
-    try:
-        iterable_idx = index_iterable(iterable)
-
-        # returns existing records & non-existing values
-        records, nonexist_values, msg = get_existing_records(
-            iterable_idx=iterable_idx,
-            field=field,
-            organism=organism,
-            mute=mute,
-        )
+    iterable_idx = index_iterable(iterable)
+
+    # returns existing records & non-existing values
+    records, nonexist_values, msg = get_existing_records(
+        iterable_idx=iterable_idx,
+        field=field,
+        organism=organism,
+        mute=mute,
+    )
 
-        # new records to be created based on new values
-        if len(nonexist_values) > 0:
-            source_record = None
-            if from_source:
-                if isinstance(source, Record):
-                    source_record = source
-            if not source_record and hasattr(registry, "public"):
-                if organism is None:
-                    organism = _ensembl_prefix(nonexist_values[0], field, organism)
-                    organism = _get_organism_record(field, organism, force=True)
-
-            if source_record:
-                from bionty.core._add_ontology import check_source_in_db
-
-                check_source_in_db(registry=registry, source=source_record)
-
-                from_source = not source_record.in_db
-            elif hasattr(registry, "source_id"):
-                from_source = True
-            else:
-                from_source = False
-
-            if from_source:
-                records_bionty, unmapped_values = create_records_from_source(
-                    iterable_idx=nonexist_values,
-                    field=field,
-                    organism=organism,
-                    source=source_record,
-                    msg=msg,
-                    mute=mute,
+    # new records to be created based on new values
+    if len(nonexist_values) > 0:
+        source_record = None
+        if from_source:
+            if isinstance(source, Record):
+                source_record = source
+        if not source_record and hasattr(registry, "public"):
+            if organism is None:
+                organism = _ensembl_prefix(nonexist_values[0], field, organism)
+                organism = _get_organism_record(field, organism, force=True)
+
+        if source_record:
+            from bionty.core._add_ontology import check_source_in_db
+
+            check_source_in_db(registry=registry, source=source_record)
+
+            from_source = not source_record.in_db
+        elif hasattr(registry, "source_id"):
+            from_source = True
+        else:
+            from_source = False
+
+        if from_source:
+            records_bionty, unmapped_values = create_records_from_source(
+                iterable_idx=nonexist_values,
+                field=field,
+                organism=organism,
+                source=source_record,
+                msg=msg,
+                mute=mute,
+            )
+            if len(records_bionty) > 0:
+                msg = ""
+            for record in records_bionty:
+                record._from_source = True
+            records += records_bionty
+        else:
+            unmapped_values = nonexist_values
+        # unmapped new_ids will NOT create records
+        if len(unmapped_values) > 0:
+            if len(msg) > 0 and not mute:
+                logger.success(msg)
+            s = "" if len(unmapped_values) == 1 else "s"
+            print_values = colors.yellow(_format_values(unmapped_values))
+            name = registry.__name__
+            n_nonval = colors.yellow(f"{len(unmapped_values)} non-validated")
+            if not mute:
+                logger.warning(
+                    f"{colors.red('did not create')} {name} record{s} for "
+                    f"{n_nonval} {colors.italic(f'{field.field.name}{s}')}: {print_values}"  # type: ignore
                 )
-                if len(records_bionty) > 0:
-                    msg = ""
-                for record in records_bionty:
-                    record._from_source = True
-                records += records_bionty
-            else:
-                unmapped_values = nonexist_values
-            # unmapped new_ids will NOT create records
-            if len(unmapped_values) > 0:
-                if len(msg) > 0 and not mute:
-                    logger.success(msg)
-                s = "" if len(unmapped_values) == 1 else "s"
-                print_values = colors.yellow(_format_values(unmapped_values))
-                name = registry.__name__
-                n_nonval = colors.yellow(f"{len(unmapped_values)} non-validated")
-                if not mute:
-                    logger.warning(
-                        f"{colors.red('did not create')} {name} record{s} for "
-                        f"{n_nonval} {colors.italic(f'{field.field.name}{s}')}: {print_values}"
-                    )
-        # if registry.__get_module_name__() == "bionty" or registry == ULabel:
-        #     if isinstance(iterable, pd.Series):
-        #         feature = iterable.name
-        #     feature_name = None
-        #     if isinstance(feature, str):
-        #         feature_name = feature
-        #     if feature_name is not None:
-        #         if feature_name is not None:
-        #             for record in records:
-        #                 record._feature = feature_name
-        #         logger.debug(f"added default feature '{feature_name}'")
-        return RecordList(records)
-    finally:
-        settings.creation.search_names = creation_search_names
+    return RecordList(records)
 
 
 def get_existing_records(
@@ -120,10 +102,10 @@ def get_existing_records(
     mute: bool = False,
 ):
     # NOTE: existing records matching is agnostic to the source
-    model = field.field.model
-    if organism is None and field.field.name == "ensembl_gene_id":
+    model = field.field.model  # type: ignore
+    if organism is None and field.field.name == "ensembl_gene_id":  # type: ignore
         if len(iterable_idx) > 0:
-            organism = _ensembl_prefix(iterable_idx[0], field, organism)
+            organism = _ensembl_prefix(iterable_idx[0], field, organism)  # type: ignore
             organism = _get_organism_record(field, organism, force=True)
 
     # standardize based on the DB reference
@@ -152,6 +134,7 @@ def get_existing_records(
     is_validated = model.validate(
         iterable_idx, field=field, organism=organism, mute=True
     )
+
     if len(is_validated) > 0:
         validated = iterable_idx[is_validated]
     else:
@@ -165,7 +148,7 @@ def get_existing_records(
             msg = (
                 "loaded"
                 f" {colors.green(f'{len(validated)} {model.__name__} record{s}')}"
-                f" matching {colors.italic(f'{field.field.name}')}: {print_values}"
+                f" matching {colors.italic(f'{field.field.name}')}: {print_values}"  # type: ignore
             )
         if len(syn_mapper) > 0:
             s = "" if len(syn_mapper) == 1 else "s"
@@ -189,7 +172,7 @@ def get_existing_records(
     # get all existing records in the db
     # if necessary, create records for the values in kwargs
     # k:v -> k:v_record
-    query = {f"{field.field.name}__in": iterable_idx.values}
+    query = {f"{field.field.name}__in": iterable_idx.values}  # type: ignore
     if organism is not None:
         query["organism"] = organism
     records = model.filter(**query).list()
@@ -209,7 +192,7 @@ def create_records_from_source(
     msg: str = "",
     mute: bool = False,
 ):
-    model = field.field.model
+    model = field.field.model  # type: ignore
     records: list = []
     # populate additional fields from bionty
     from bionty._bionty import get_source_record
@@ -232,11 +215,11 @@ def create_records_from_source(
     # standardize in the bionty reference
     # do not inspect synonyms if the field is not name field
     inspect_synonyms = True
-    if hasattr(model, "_name_field") and field.field.name != model._name_field:
+    if hasattr(model, "_name_field") and field.field.name != model._name_field:  # type: ignore
         inspect_synonyms = False
     result = public_ontology.inspect(
         iterable_idx,
-        field=field.field.name,
+        field=field.field.name,  # type: ignore
         mute=True,
         inspect_synonyms=inspect_synonyms,
     )
@@ -257,12 +240,14 @@ def create_records_from_source(
 
     # create records for values that are found in the bionty reference
     # matching either field or synonyms
-    mapped_values = iterable_idx.intersection(bionty_df[field.field.name])
+    mapped_values = iterable_idx.intersection(bionty_df[field.field.name])  # type: ignore
 
     multi_msg = ""
     if len(mapped_values) > 0:
         bionty_kwargs, multi_msg = _bulk_create_dicts_from_df(
-            keys=mapped_values, column_name=field.field.name, df=bionty_df
+            keys=mapped_values,
+            column_name=field.field.name,  # type: ignore
+            df=bionty_df,
         )
 
         if hasattr(model, "organism_id") and organism is None:
@@ -274,7 +259,7 @@ def create_records_from_source(
             else {"source": source}
         )
         for bk in bionty_kwargs:
-            records.append(model(**bk, **create_kwargs))
+            records.append(model(**bk, **create_kwargs, _skip_validation=True))
 
         # number of records that matches field (not synonyms)
         validated = result.validated
@@ -288,7 +273,7 @@ def create_records_from_source(
                 logger.success(
                     "created"
                     f" {colors.purple(f'{len(validated)} {model.__name__} record{s} from Bionty')}"
-                    f" matching {colors.italic(f'{field.field.name}')}: {print_values}"
+                    f" matching {colors.italic(f'{field.field.name}')}: {print_values}"  # type: ignore
                 )
 
     # make sure that synonyms logging appears after the field logging
@@ -365,7 +350,7 @@ def _has_organism_field(registry: type[Record]) -> bool:
         return False
 
 
-def _get_organism_record(
+def _get_organism_record(  # type: ignore
     field: StrField, organism: str | Record, force: bool = False
 ) -> Record:
     """Get organism record.
@@ -375,10 +360,10 @@ def _get_organism_record(
         organism: the organism to get the record for
         force: whether to force fetching the organism record
     """
-    registry = field.field.model
+    registry = field.field.model  # type: ignore
     check = True
     if not force and hasattr(registry, "_ontology_id_field"):
-        check = field.field.name != registry._ontology_id_field
+        check = field.field.name != registry._ontology_id_field  # type: ignore
         # e.g. bionty.CellMarker has "name" as _ontology_id_field
         if not registry._ontology_id_field.endswith("id"):
             check = True
@@ -397,10 +382,10 @@ def _get_organism_record(
 
 
 def _ensembl_prefix(id: str, field: StrField, organism: Record | None) -> str | None:
-    if field.field.name == "ensembl_gene_id" and organism is None:
+    if field.field.name == "ensembl_gene_id" and organism is None:  # type: ignore
         if id.startswith("ENSG"):
-            organism = "human"
+            organism = "human"  # type: ignore
         elif id.startswith("ENSMUSG"):
-            organism = "mouse"
+            organism = "mouse"  # type: ignore
 
     return organism

lamindb/_parents.py

@@ -44,7 +44,7 @@ def _query_relatives(
     kind: Literal["parents", "children"],
     cls: type[HasParents],
 ) -> QuerySet:
-    relatives = cls.objects.none()
+    relatives = cls.objects.none()  # type: ignore
     if len(records) == 0:
         return relatives
     for record in records:
@@ -350,9 +350,9 @@ def _record_label(record: Record, field: str | None = None):
         )
     elif isinstance(record, Run):
         if record.transform.description:
-            name = f'{record.transform.description.replace("&", "&")}'
+            name = f"{record.transform.description.replace('&', '&')}"
         elif record.transform.key:
-            name = f'{record.transform.key.replace("&", "&")}'
+            name = f"{record.transform.key.replace('&', '&')}"
         else:
             name = f"{record.transform.uid}"
         user_display = (
@@ -366,7 +366,7 @@ def _record_label(record: Record, field: str | None = None):
             rf" user={user_display}<BR/>run={format_field_value(record.started_at)}</FONT>>"
         )
     elif isinstance(record, Transform):
-        name = f'{record.name.replace("&", "&amp;")}'
+        name = f"{record.name.replace('&', '&amp;')}"
         return (
             rf'<{TRANSFORM_EMOJIS.get(str(record.type), "💫")} {name}<BR/><FONT COLOR="GREY" POINT-SIZE="10"'
             rf' FACE="Monospace">uid={record.uid}<BR/>type={record.type},'

lamindb/_query_set.py

@@ -8,6 +8,7 @@ from collections.abc import Iterable as IterableType
 from typing import TYPE_CHECKING, Any, Generic, NamedTuple, TypeVar
 
 import pandas as pd
+from django.core.exceptions import FieldError
 from django.db import models
 from django.db.models import F, ForeignKey, ManyToManyField
 from django.db.models.fields.related import ForeignObjectRel
@@ -26,7 +27,7 @@ from lamindb.models import (
     Transform,
 )
 
-from .core.exceptions import DoesNotExist
+from .errors import DoesNotExist
 
 T = TypeVar("T")
 
@@ -91,14 +92,12 @@ def get_backward_compat_filter_kwargs(queryset, expressions):
             "n_objects": "n_files",
             "visibility": "_branch_code",  # for convenience (and backward compat <1.0)
             "transform": "run__transform",  # for convenience (and backward compat <1.0)
-            "feature_sets": "_schemas_m2m",
             "type": "kind",
             "_accessor": "otype",
         }
     elif queryset.model == Schema:
         name_mappings = {
             "registry": "itype",
-            "artifacts": "_artifacts_m2m",  # will raise warning when we start to migrate over
         }
     else:
         return expressions
@@ -114,7 +113,6 @@ def get_backward_compat_filter_kwargs(queryset, expressions):
             if parts[0] not in {
                 "transform",
                 "visibility",
-                "feature_sets",
                 "schemas",
                 "artifacts",
             }:
@@ -203,7 +201,7 @@ def get(
         qs = QuerySet(model=registry_or_queryset)
         registry = registry_or_queryset
     if isinstance(idlike, int):
-        return super(QuerySet, qs).get(id=idlike)
+        return super(QuerySet, qs).get(id=idlike)  # type: ignore
     elif isinstance(idlike, str):
         qs = qs.filter(uid__startswith=idlike)
         if issubclass(registry, IsVersioned):
@@ -216,7 +214,27 @@ def get(
     else:
         assert idlike is None  # noqa: S101
         expressions = process_expressions(qs, expressions)
-        return registry.objects.using(qs.db).get(**expressions)
+        # don't want _branch_code here in .get(), only in .filter()
+        expressions.pop("_branch_code", None)
+        # inject is_latest for consistency with idlike
+        is_latest_was_not_in_expressions = "is_latest" not in expressions
+        if issubclass(registry, IsVersioned) and is_latest_was_not_in_expressions:
+            expressions["is_latest"] = True
+        try:
+            return registry.objects.using(qs.db).get(**expressions)
+        except registry.DoesNotExist:
+            # handle the case in which the is_latest injection led to a missed query
+            if "is_latest" in expressions and is_latest_was_not_in_expressions:
+                expressions.pop("is_latest")
+                result = (
+                    registry.objects.using(qs.db)
+                    .filter(**expressions)
+                    .order_by("-created_at")
+                    .first()
+                )
+                if result is not None:
+                    return result
+            raise registry.DoesNotExist from registry.DoesNotExist
 
 
 class RecordList(UserList, Generic[T]):
@@ -537,13 +555,13 @@ class QuerySet(models.QuerySet):
         elif isinstance(include, str):
             include = [include]
         include = get_backward_compat_filter_kwargs(self, include)
-        field_names = get_basic_field_names(self, include, features)
+        field_names = get_basic_field_names(self, include, features)  # type: ignore
 
         annotate_kwargs = {}
         if features:
             annotate_kwargs.update(get_feature_annotate_kwargs(features))
         if include:
-            include = include.copy()[::-1]
+            include = include.copy()[::-1]  # type: ignore
             include_kwargs = {s: F(s) for s in include if s not in field_names}
             annotate_kwargs.update(include_kwargs)
         if annotate_kwargs:
@@ -561,12 +579,6 @@ class QuerySet(models.QuerySet):
         pk_column_name = pk_name if pk_name in df.columns else f"{pk_name}_id"
         if pk_column_name in df_reshaped.columns:
             df_reshaped = df_reshaped.set_index(pk_column_name)
-
-        # Compatibility code
-        df_reshaped.columns = df_reshaped.columns.str.replace(
-            r"_schemas_m2m", "feature_sets", regex=True
-        )
-
         return df_reshaped
 
     def delete(self, *args, **kwargs):
@@ -601,17 +613,65 @@ class QuerySet(models.QuerySet):
             return None
         return self[0]
 
+    def _handle_unknown_field(self, error: FieldError) -> None:
+        """Suggest available fields if an unknown field was passed."""
+        if "Cannot resolve keyword" in str(error):
+            field = str(error).split("'")[1]
+            fields = ", ".join(
+                sorted(
+                    f.name
+                    for f in self.model._meta.get_fields()
+                    if not f.name.startswith("_")
+                    and not f.name.startswith("links_")
+                    and not f.name.endswith("_id")
+                )
+            )
+            raise FieldError(
+                f"Unknown field '{field}'. Available fields: {fields}"
+            ) from None
+        raise error  # pragma: no cover
+
     def get(self, idlike: int | str | None = None, **expressions) -> Record:
         """Query a single record. Raises error if there are more or none."""
-        return get(self, idlike, **expressions)
+        try:
+            return get(self, idlike, **expressions)
+        except ValueError as e:
+            # Pass through original error for explicit id lookups
+            if "Field 'id' expected a number" in str(e):
+                if "id" in expressions:
+                    raise
+                field = next(iter(expressions))
+                raise FieldError(
+                    f"Invalid lookup '{expressions[field]}' for {field}. Did you mean {field}__name?"
+                ) from None
+            raise  # pragma: no cover
+        except FieldError as e:
+            self._handle_unknown_field(e)
+            raise  # pragma: no cover
 
     def filter(self, *queries, **expressions) -> QuerySet:
         """Query a set of records."""
+        # Suggest to use __name for related fields such as id when not passed
+        for field, value in expressions.items():
+            if (
+                isinstance(value, str)
+                and value.strip("-").isalpha()
+                and "__" not in field
+                and hasattr(self.model, field)
+            ):
+                field_attr = getattr(self.model, field)
+                if hasattr(field_attr, "field") and field_attr.field.related_model:
+                    raise FieldError(
+                        f"Invalid lookup '{value}' for {field}. Did you mean {field}__name?"
+                    )
+
         expressions = process_expressions(self, expressions)
         if len(expressions) > 0:
-            return super().filter(*queries, **expressions)
-        else:
-            return self
+            try:
+                return super().filter(*queries, **expressions)
+            except FieldError as e:
+                self._handle_unknown_field(e)
+        return self
 
     def one(self) -> Record:
         """Exactly one result. Raises error if there are more or none."""