lamindb 0.76.15__py3-none-any.whl → 0.76.16__py3-none-any.whl

lamindb/__init__.py

@@ -43,7 +43,7 @@ Modules and settings.
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.76.15"
+__version__ = "0.76.16"
 
 import os as _os
 

lamindb/_artifact.py

@@ -111,7 +111,12 @@ def process_pathlike(
             # for the storage root: the bucket
             if not isinstance(filepath, LocalPathClasses):
                 # for a cloud path, new_root is always the bucket name
-                new_root = list(filepath.parents)[-1]
+                if filepath.protocol == "hf":
+                    hf_path = filepath.fs.resolve_path(filepath.as_posix())
+                    hf_path.path_in_repo = ""
+                    new_root = "hf://" + hf_path.unresolve()
+                else:
+                    new_root = list(filepath.parents)[-1]
                 # do not register remote storage locations on hub if the current instance
                 # is not managed on the hub
                 storage_settings, _ = init_storage(
@@ -213,9 +218,9 @@ def get_stat_or_artifact(
         if stat is not None:
             # convert UPathStatResult to fsspec info dict
             stat = stat.as_info()
-            if "ETag" in stat:  # is file
+            if (store_type := stat["type"]) == "file":
                 size, hash, hash_type = get_stat_file_cloud(stat)
-            elif stat["type"] == "directory":
+            elif store_type == "directory":
                 size, hash, hash_type, n_objects = get_stat_dir_cloud(path)
         if hash is None:
             logger.warning(f"did not add hash for {path}")
@@ -240,7 +245,7 @@ def get_stat_or_artifact(
             .order_by("-created_at")
             .all()
         )
-        artifact_with_same_hash_exists = len(result.filter(hash=hash).all()) > 0
+        artifact_with_same_hash_exists = result.filter(hash=hash).count() > 0
         if not artifact_with_same_hash_exists and len(result) > 0:
             logger.important(
                 f"creating new artifact version for key='{key}' (storage: '{settings.storage.root_as_str}')"

lamindb/_can_validate.py

@@ -108,14 +108,14 @@ def _check_organism_db(organism: Record, using_key: str | None):
 
 def _concat_lists(values: ListLike) -> list[str]:
     """Concatenate a list of lists of strings into a single list."""
-    if len(values) > 0 and isinstance(values, (list, pd.Series)):
-        try:
-            if isinstance(values[0], list):
-                if isinstance(values, pd.Series):
-                    values = values.tolist()
-                values = sum([v for v in values if isinstance(v, list)], [])
-        except KeyError:
-            pass
+    if isinstance(values, (list, pd.Series)) and len(values) > 0:
+        first_item = values[0] if isinstance(values, list) else values.iloc[0]
+        if isinstance(first_item, list):
+            if isinstance(values, pd.Series):
+                values = values.tolist()
+            values = [
+                v for sublist in values if isinstance(sublist, list) for v in sublist
+            ]
     return values
 
 
@@ -250,7 +250,7 @@ def _validate(
                 f"Your {cls.__name__} registry is empty, consider populating it first!"
             )
             if hasattr(cls, "source_id"):
-                msg += "\n   → use `.import_from_source()` to import records from a source, e.g. a public ontology"
+                msg += "\n   → use `.import_source()` to import records from a source, e.g. a public ontology"
             logger.warning(msg)
         return np.array([False] * len(values))
 
@@ -388,7 +388,11 @@ def _standardize(
 
     try:
         registry._meta.get_field(synonyms_field)
-        fields = {i for i in [field, return_field, synonyms_field] if i is not None}
+        fields = {
+            field_name
+            for field_name in [field, return_field, synonyms_field]
+            if field_name is not None
+        }
         df = _filter_query_based_on_organism(
             queryset=queryset,
             field=field,
@@ -445,14 +449,19 @@ def _standardize(
         if len(std_names_bt_mapper) > 0 and not mute:
             s = "" if len(std_names_bt_mapper) == 1 else "s"
             field_print = "synonym" if field == return_field else field
-            warn_msg = (
-                f"found {len(std_names_bt_mapper)} {field_print}{s} in Bionty:"
-                f" {list(std_names_bt_mapper.keys())}"
+
+            reduced_mapped_keys_str = f"{list(std_names_bt_mapper.keys())[:10] + ['...'] if len(std_names_bt_mapper) > 10 else list(std_names_bt_mapper.keys())}"
+            truncated_note = (
+                " (output truncated)" if len(std_names_bt_mapper) > 10 else ""
             )
-            warn_msg += (
-                f"\n   please add corresponding {registry._meta.model.__name__} records via"
-                f" `.from_values({list(set(std_names_bt_mapper.values()))})`"
+
+            warn_msg = (
+                f"found {len(std_names_bt_mapper)} {field_print}{s} in Bionty{truncated_note}:"
+                f" {reduced_mapped_keys_str}\n"
+                f"  please add corresponding {registry._meta.model.__name__} records via{truncated_note}:"
+                f" `.from_values({reduced_mapped_keys_str})`"
             )
+
             logger.warning(warn_msg)
 
         mapper.update(std_names_bt_mapper)

lamindb/_curate.py

@@ -20,6 +20,7 @@ from .core.exceptions import ValidationError
 
 if TYPE_CHECKING:
     from collections.abc import Iterable
+    from typing import Any
 
     from lamindb_setup.core.types import UPathStr
     from lnschema_core.types import FieldAttr
@@ -226,7 +227,7 @@ class DataFrameCurator(BaseCurator):
                     f"the following keys passed to {name} are not allowed: {nonval_keys}"
                 )
 
-    def _save_columns(self, validated_only: bool = True, **kwargs) -> None:
+    def _save_columns(self, validated_only: bool = True) -> None:
         """Save column name records."""
         # Always save features specified as the fields keys
         update_registry(
@@ -238,7 +239,7 @@ class DataFrameCurator(BaseCurator):
             validated_only=False,
             source=self._sources.get("columns"),
             exclude=self._exclude.get("columns"),
-            **kwargs,
+            **self._kwargs,  # type: ignore
         )
 
         # Save the rest of the columns based on validated_only
@@ -255,7 +256,7 @@ class DataFrameCurator(BaseCurator):
                 source=self._sources.get("columns"),
                 exclude=self._exclude.get("columns"),
                 warning=False,  # Do not warn about missing columns, just an info message
-                **kwargs,
+                **self._kwargs,  # type: ignore
             )
 
     def add_new_from(self, key: str, organism: str | None = None, **kwargs):
@@ -292,7 +293,7 @@ class DataFrameCurator(BaseCurator):
                     f"Feature {categorical} is not part of the fields!"
                 )
             update_registry(
-                values=flatten_unique(self._df[categorical]),
+                values=_flatten_unique(self._df[categorical]),
                 field=self.fields[categorical],
                 key=categorical,
                 using_key=self._using_key,
@@ -305,7 +306,6 @@ class DataFrameCurator(BaseCurator):
     def _update_registry_all(self, validated_only: bool = True, **kwargs):
         """Save labels for all features."""
         for name in self.fields.keys():
-            logger.info(f"saving validated records of '{name}'")
             self._update_registry(name, validated_only=validated_only, **kwargs)
 
     def validate(self, organism: str | None = None) -> bool:
@@ -436,12 +436,15 @@ class AnnDataCurator(DataFrameCurator):
     ) -> None:
         from lamindb_setup.core import upath
 
+        if isinstance(var_index, str):
+            raise TypeError("var_index parameter has to be a bionty field")
+
         from ._artifact import data_is_anndata
 
         if sources is None:
             sources = {}
         if not data_is_anndata(data):
-            raise ValueError(
+            raise TypeError(
                 "data has to be an AnnData object or a path to AnnData-like"
             )
         if isinstance(data, ad.AnnData):
@@ -451,6 +454,11 @@ class AnnDataCurator(DataFrameCurator):
 
             self._adata = backed_access(upath.create_path(data))
 
+        if "symbol" in str(var_index):
+            logger.warning(
+                "Curating gene symbols is discouraged. See FAQ for more details."
+            )
+
         self._data = data
         self._var_field = var_index
         super().__init__(
@@ -512,10 +520,8 @@ class AnnDataCurator(DataFrameCurator):
 
     def _update_registry_all(self, validated_only: bool = True, **kwargs):
         """Save labels for all features."""
-        logger.info("saving validated records of 'var_index'")
         self._save_from_var_index(validated_only=validated_only, **self._kwargs)
         for name in self._obs_fields.keys():
-            logger.info(f"saving validated terms of '{name}'")
             self._update_registry(name, validated_only=validated_only, **self._kwargs)
 
     def add_new_from_var_index(self, organism: str | None = None, **kwargs):
@@ -1229,7 +1235,7 @@ def validate_categories(
     if n_non_validated == 0:
         if n_validated == 0:
             logger.indent = ""
-            logger.success(f"{key} is validated against {colors.italic(model_field)}")
+            logger.success(f"'{key}' is validated against {colors.italic(model_field)}")
             return True, []
         else:
             # validated values still need to be saved to the current instance
@@ -1434,8 +1440,8 @@ def save_artifact(
     return artifact
 
 
-def flatten_unique(series):
-    """Flatten a pandas series if it contains lists."""
+def _flatten_unique(series: pd.Series[list[Any] | Any]) -> list[Any]:
+    """Flatten a Pandas series containing lists or single items into a unique list of elements."""
     result = set()
 
     for item in series:
@@ -1505,9 +1511,14 @@ def update_registry(
 
         public_records = [r for r in existing_and_public_records if r._state.adding]
         # here we check to only save the public records if they are from the specified source
-        # we check the uid because r.source and soruce can be from different instances
+        # we check the uid because r.source and source can be from different instances
         if source:
             public_records = [r for r in public_records if r.source.uid == source.uid]
+
+        if public_records:
+            settings.verbosity = "info"
+            logger.info(f"saving validated records of '{key}'")
+            settings.verbosity = "error"
         ln_save(public_records)
         labels_saved["from public"] = [
             getattr(r, field.field.name) for r in public_records

lamindb/_finish.py

@@ -103,10 +103,10 @@ def save_context_core(
 
     # for scripts, things are easy
     is_consecutive = True
-    is_notebook = transform.type == "notebook"
+    is_ipynb = filepath.suffix == ".ipynb"
     source_code_path = filepath
     # for notebooks, we need more work
-    if is_notebook:
+    if is_ipynb:
         try:
             import jupytext
             from nbproject.dev import (
@@ -198,7 +198,7 @@ def save_context_core(
         run.finished_at = datetime.now(timezone.utc)
 
     # track report and set is_consecutive
-    if not is_notebook:
+    if not is_ipynb:
         run.is_consecutive = True
         run.save()
     else:
@@ -234,8 +234,15 @@ def save_context_core(
     # finalize
     if not from_cli:
         run_time = run.finished_at - run.started_at
+        days = run_time.days
+        seconds = run_time.seconds
+        hours = seconds // 3600
+        minutes = (seconds % 3600) // 60
+        secs = seconds % 60
+        formatted_run_time = f"{days}d {hours}h {minutes}m {secs}s"
+
         logger.important(
-            f"finished Run('{run.uid[:8]}') after {run_time} at {format_field_value(run.finished_at)}"
+            f"finished Run('{run.uid[:8]}') after {formatted_run_time} at {format_field_value(run.finished_at)}"
         )
     if ln_setup.settings.instance.is_on_hub:
         identifier = ln_setup.settings.instance.slug
@@ -244,9 +251,7 @@ def save_context_core(
         )
         if not from_cli:
             thing, name = (
-                ("notebook", "notebook.ipynb")
-                if is_notebook
-                else ("script", "script.py")
+                ("notebook", "notebook.ipynb") if is_ipynb else ("script", "script.py")
             )
             logger.important(
                 f"if you want to update your {thing} without re-running it, use `lamin save {name}`"

lamindb/_query_set.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
 from collections import UserList
-from typing import TYPE_CHECKING, NamedTuple
+from collections.abc import Iterable
+from collections.abc import Iterable as IterableType
+from typing import TYPE_CHECKING, Any, NamedTuple
 
 import pandas as pd
 from django.db import models
@@ -69,8 +71,33 @@ def one_helper(self):
         return self[0]
 
 
-def process_expressions(registry: Registry, expressions: dict) -> dict:
-    if registry in {Artifact, Collection}:
+def process_expressions(queryset: QuerySet, expressions: dict) -> dict:
+    def _map_databases(value: Any, key: str, target_db: str) -> tuple[str, Any]:
+        if isinstance(value, Record):
+            if value._state.db != target_db:
+                logger.warning(
+                    f"passing record from database {value._state.db} to query {target_db}, matching on uid '{value.uid}'"
+                )
+                return f"{key}__uid", value.uid
+            return key, value
+
+        if (
+            key.endswith("__in")
+            and isinstance(value, IterableType)
+            and not isinstance(value, str)
+        ):
+            if any(isinstance(v, Record) and v._state.db != target_db for v in value):
+                logger.warning(
+                    f"passing records from another database to query {target_db}, matching on uids"
+                )
+                return key.replace("__in", "__uid__in"), [
+                    v.uid if isinstance(v, Record) else v for v in value
+                ]
+            return key, value
+
+        return key, value
+
+    if queryset.model in {Artifact, Collection}:
         # visibility is set to 0 unless expressions contains id or uid equality
         if not (
             "id" in expressions
@@ -87,7 +114,17 @@ def process_expressions(registry: Registry, expressions: dict) -> dict:
             # sense for a non-NULLABLE column
             elif visibility in expressions and expressions[visibility] is None:
                 expressions.pop(visibility)
-    return expressions
+    if queryset._db is not None:
+        # only check for database mismatch if there is a defined database on the
+        # queryset
+        return dict(
+            (
+                _map_databases(value, key, queryset._db)
+                for key, value in expressions.items()
+            )
+        )
+    else:
+        return expressions
 
 
 def get(
@@ -114,7 +151,7 @@ def get(
             return qs.one()
     else:
         assert idlike is None  # noqa: S101
-        expressions = process_expressions(registry, expressions)
+        expressions = process_expressions(qs, expressions)
         return registry.objects.using(qs.db).get(**expressions)
 
 
@@ -282,6 +319,14 @@ class QuerySet(models.QuerySet):
         """Query a single record. Raises error if there are more or none."""
         return get(self, idlike, **expressions)
 
+    def filter(self, *queries, **expressions) -> QuerySet:
+        """Query a set of records."""
+        expressions = process_expressions(self, expressions)
+        if len(expressions) > 0:
+            return super().filter(*queries, **expressions)
+        else:
+            return self
+
     def one(self) -> Record:
         """Exactly one result. Raises error if there are more or none."""
         return one_helper(self)

lamindb/_record.py

@@ -1,12 +1,16 @@
 from __future__ import annotations
 
 import builtins
+from functools import reduce
 from typing import TYPE_CHECKING, NamedTuple
 
 import dj_database_url
 import lamindb_setup as ln_setup
+from django.core.exceptions import FieldDoesNotExist
 from django.db import connections, transaction
-from django.db.models import IntegerField, Manager, Q, QuerySet, Value
+from django.db.models import F, IntegerField, Manager, Q, QuerySet, TextField, Value
+from django.db.models.functions import Cast, Coalesce
+from django.db.models.lookups import Contains, Exact, IContains, IExact, IRegex, Regex
 from lamin_utils import colors, logger
 from lamin_utils._lookup import Lookup
 from lamindb_setup._connect_instance import (
@@ -17,11 +21,22 @@ from lamindb_setup._connect_instance import (
 from lamindb_setup.core._docs import doc_args
 from lamindb_setup.core._hub_core import connect_instance_hub
 from lamindb_setup.core._settings_store import instance_settings_file
-from lnschema_core.models import Artifact, Feature, IsVersioned, Record, Run, Transform
+from lnschema_core.models import (
+    Artifact,
+    Collection,
+    Feature,
+    FeatureSet,
+    IsVersioned,
+    Param,
+    Record,
+    Run,
+    Transform,
+    ULabel,
+)
 
 from ._utils import attach_func_to_class_method
 from .core._settings import settings
-from .core.exceptions import RecordNameChangeIntegrityError
+from .core.exceptions import RecordNameChangeIntegrityError, ValidationError
 
 if TYPE_CHECKING:
     import pandas as pd
@@ -48,6 +63,7 @@ def update_attributes(record: Record, attributes: dict[str, str]):
 
 
 def validate_required_fields(record: Record, kwargs):
+    # a "required field" is a Django field that has `null=True, default=None`
     required_fields = {
         k.name for k in record._meta.fields if not k.null and k.default is None
     }
@@ -58,25 +74,47 @@ def validate_required_fields(record: Record, kwargs):
     ]
     if missing_fields:
         raise TypeError(f"{missing_fields} are required.")
+    # ensure the exact length of the internal uid for core entities
+    if "uid" in kwargs and record.__class__ in {
+        Artifact,
+        Collection,
+        Transform,
+        Run,
+        ULabel,
+        Feature,
+        FeatureSet,
+        Param,
+    }:
+        uid_max_length = record.__class__._meta.get_field(
+            "uid"
+        ).max_length  # triggers FieldDoesNotExist
+        if len(kwargs["uid"]) != uid_max_length:  # triggers KeyError
+            raise ValidationError(
+                f'`uid` must be exactly {uid_max_length} characters long, got {len(kwargs["uid"])}.'
+            )
 
 
-def suggest_records_with_similar_names(record: Record, kwargs) -> bool:
+def suggest_records_with_similar_names(record: Record, name_field: str, kwargs) -> bool:
     """Returns True if found exact match, otherwise False.
 
     Logs similar matches if found.
     """
-    if kwargs.get("name") is None:
+    if kwargs.get(name_field) is None or not isinstance(kwargs.get(name_field), str):
         return False
     queryset = _search(
-        record.__class__, kwargs["name"], field="name", truncate_words=True, limit=3
+        record.__class__,
+        kwargs[name_field],
+        field=name_field,
+        truncate_string=True,
+        limit=3,
     )
     if not queryset.exists():  # empty queryset
         return False
     for alternative_record in queryset:
-        if alternative_record.name == kwargs["name"]:
+        if getattr(alternative_record, name_field) == kwargs[name_field]:
             return True
     s, it, nots = ("", "it", "s") if len(queryset) == 1 else ("s", "one of them", "")
-    msg = f"record{s} with similar name{s} exist{nots}! did you mean to load {it}?"
+    msg = f"record{s} with similar {name_field}{s} exist{nots}! did you mean to load {it}?"
     if IPYTHON:
         from IPython.display import display
 
@@ -98,13 +136,19 @@ def __init__(record: Record, *args, **kwargs):
         if "_has_consciously_provided_uid" in kwargs:
             has_consciously_provided_uid = kwargs.pop("_has_consciously_provided_uid")
         if settings.creation.search_names and not has_consciously_provided_uid:
-            match = suggest_records_with_similar_names(record, kwargs)
+            name_field = (
+                "name" if not hasattr(record, "_name_field") else record._name_field
+            )
+            match = suggest_records_with_similar_names(record, name_field, kwargs)
             if match:
                 if "version" in kwargs:
                     if kwargs["version"] is not None:
                         version_comment = " and version"
                         existing_record = record.__class__.filter(
-                            name=kwargs["name"], version=kwargs["version"]
+                            **{
+                                name_field: kwargs[name_field],
+                                "version": kwargs["version"],
+                            }
                         ).one_or_none()
                     else:
                         # for a versioned record, an exact name match is not a
@@ -115,12 +159,12 @@ def __init__(record: Record, *args, **kwargs):
                 else:
                     version_comment = ""
                     existing_record = record.__class__.filter(
-                        name=kwargs["name"]
+                        **{name_field: kwargs[name_field]}
                     ).one_or_none()
                 if existing_record is not None:
                     logger.important(
                         f"returning existing {record.__class__.__name__} record with same"
-                        f" name{version_comment}: '{kwargs['name']}'"
+                        f" {name_field}{version_comment}: '{kwargs[name_field]}'"
                     )
                     init_self_from_db(record, existing_record)
                     return None
@@ -137,9 +181,13 @@ def __init__(record: Record, *args, **kwargs):
 @doc_args(Record.filter.__doc__)
 def filter(cls, *queries, **expressions) -> QuerySet:
     """{}"""  # noqa: D415
-    from lamindb._filter import filter
+    from lamindb._query_set import QuerySet
 
-    return filter(cls, *queries, **expressions)
+    _using_key = None
+    if "_using_key" in expressions:
+        _using_key = expressions.pop("_using_key")
+
+    return QuerySet(model=cls, using=_using_key).filter(*queries, **expressions)
 
 
 @classmethod  # type:ignore
@@ -150,8 +198,6 @@ def get(
     **expressions,
 ) -> Record:
     """{}"""  # noqa: D415
-    # this is the only place in which we need the lamindb queryset
-    # in this file; everywhere else it should be Django's
     from lamindb._query_set import QuerySet
 
     return QuerySet(model=cls).get(idlike, **expressions)
@@ -166,9 +212,7 @@ def df(
     limit: int = 100,
 ) -> pd.DataFrame:
     """{}"""  # noqa: D415
-    from lamindb._filter import filter
-
-    query_set = filter(cls)
+    query_set = cls.filter()
     if hasattr(cls, "updated_at"):
         query_set = query_set.order_by("-updated_at")
     return query_set[:limit].df(include=include, join=join)
@@ -182,7 +226,7 @@ def _search(
     limit: int | None = 20,
     case_sensitive: bool = False,
     using_key: str | None = None,
-    truncate_words: bool = False,
+    truncate_string: bool = False,
 ) -> QuerySet:
     input_queryset = _queryset(cls, using_key=using_key)
     registry = input_queryset.model
@@ -209,48 +253,67 @@ def _search(
             else:
                 fields.append(field)
 
-    # decompose search string
-    def truncate_word(word) -> str:
-        if len(word) > 5:
-            n_80_pct = int(len(word) * 0.8)
-            return word[:n_80_pct]
-        elif len(word) > 3:
-            return word[:3]
-        else:
-            return word
-
-    decomposed_string = str(string).split()
-    # add the entire string back
-    decomposed_string += [string]
-    for word in decomposed_string:
-        # will not search against words with 3 or fewer characters
-        if len(word) <= 3:
-            decomposed_string.remove(word)
-    if truncate_words:
-        decomposed_string = [truncate_word(word) for word in decomposed_string]
-    # construct the query
-    expression = Q()
-    case_sensitive_i = "" if case_sensitive else "i"
-    for field in fields:
-        for word in decomposed_string:
-            query = {f"{field}__{case_sensitive_i}contains": word}
-            expression |= Q(**query)
-    output_queryset = input_queryset.filter(expression)
-    # ensure exact matches are at the top
-    narrow_expression = Q()
+    if truncate_string:
+        if (len_string := len(string)) > 5:
+            n_80_pct = int(len_string * 0.8)
+            string = string[:n_80_pct]
+
+    string = string.strip()
+
+    exact_lookup = Exact if case_sensitive else IExact
+    regex_lookup = Regex if case_sensitive else IRegex
+    contains_lookup = Contains if case_sensitive else IContains
+
+    ranks = []
+    contains_filters = []
     for field in fields:
-        query = {f"{field}__{case_sensitive_i}contains": string}
-        narrow_expression |= Q(**query)
-    refined_output_queryset = output_queryset.filter(narrow_expression).annotate(
-        ordering=Value(1, output_field=IntegerField())
-    )
-    remaining_output_queryset = output_queryset.exclude(narrow_expression).annotate(
-        ordering=Value(2, output_field=IntegerField())
+        field_expr = Coalesce(
+            Cast(field, output_field=TextField()),
+            Value(""),
+            output_field=TextField(),
+        )
+        # exact rank
+        exact_expr = exact_lookup(field_expr, string)
+        exact_rank = Cast(exact_expr, output_field=IntegerField()) * 200
+        ranks.append(exact_rank)
+        # exact synonym
+        synonym_expr = regex_lookup(field_expr, rf"(?:^|.*\|){string}(?:\|.*|$)")
+        synonym_rank = Cast(synonym_expr, output_field=IntegerField()) * 200
+        ranks.append(synonym_rank)
+        # match as sub-phrase
+        sub_expr = regex_lookup(
+            field_expr, rf"(?:^|.*[ \|\.,;:]){string}(?:[ \|\.,;:].*|$)"
+        )
+        sub_rank = Cast(sub_expr, output_field=IntegerField()) * 10
+        ranks.append(sub_rank)
+        # startswith and avoid matching string with " " on the right
+        # mostly for truncated
+        startswith_expr = regex_lookup(field_expr, rf"(?:^|\|){string}[^ ]*(\||$)")
+        startswith_rank = Cast(startswith_expr, output_field=IntegerField()) * 8
+        ranks.append(startswith_rank)
+        # match as sub-phrase from the left, mostly for truncated
+        right_expr = regex_lookup(field_expr, rf"(?:^|.*[ \|]){string}.*")
+        right_rank = Cast(right_expr, output_field=IntegerField()) * 2
+        ranks.append(right_rank)
+        # match as sub-phrase from the right
+        left_expr = regex_lookup(field_expr, rf".*{string}(?:$|[ \|\.,;:].*)")
+        left_rank = Cast(left_expr, output_field=IntegerField()) * 2
+        ranks.append(left_rank)
+        # simple contains filter
+        contains_expr = contains_lookup(field_expr, string)
+        contains_filter = Q(contains_expr)
+        contains_filters.append(contains_filter)
+        # also rank by contains
+        contains_rank = Cast(contains_expr, output_field=IntegerField())
+        ranks.append(contains_rank)
+
+    ranked_queryset = (
+        input_queryset.filter(reduce(lambda a, b: a | b, contains_filters))
+        .alias(rank=sum(ranks))
+        .order_by("-rank")
     )
-    combined_queryset = refined_output_queryset.union(
-        remaining_output_queryset
-    ).order_by("ordering")[:limit]
-    return combined_queryset
+
+    return ranked_queryset[:limit]
 
 
 @classmethod  # type: ignore

lamindb/core/_context.py

@@ -82,11 +82,14 @@ def raise_missing_context(transform_type: str, key: str) -> bool:
     transform = Transform.filter(key=key).latest_version().first()
     if transform is None:
         new_uid = f"{base62_12()}0000"
-        message = f'to track this {transform_type}, copy & paste `ln.track("{new_uid}")` and re-run'
+        message = f'to track this {transform_type}, run: ln.track("{new_uid}")'
     else:
         uid = transform.uid
         new_uid = f"{uid[:-4]}{increment_base62(uid[-4:])}"
-        message = f"you already have a transform with key '{key}' ('{transform.uid}')\n  - to make a revision, call `ln.track('{new_uid}')`\n  - to create a new transform, rename your file and run: `ln.track()`"
+        message = (
+            f"you already have a transform with key '{key}': Transform('{transform.uid[:8]}')\n"
+            f'  (1) to make a revision, run: ln.track("{new_uid}")\n  (2) to create a new transform, rename your {transform_type} file and re-run: ln.track()'
+        )
     if transform_type == "notebook":
         print(f"→ {message}")
         response = input("→ Ready to re-run? (y/n)")
@@ -118,6 +121,8 @@ class Context:
     Enables convenient data lineage tracking by managing a transform & run
     upon :meth:`~lamindb.core.Context.track` & :meth:`~lamindb.core.Context.finish`.
 
+    Guide: :doc:`/track`
+
     Examples:
 
         Is typically used via the global :class:`~lamindb.context` object via `ln.track()` and `ln.finish()`:
@@ -137,7 +142,8 @@ class Context:
         self._run: Run | None = None
         self._path: Path | None = None
         """A local path to the script that's running."""
-        self._logging_message: str = ""
+        self._logging_message_track: str = ""
+        self._logging_message_imports: str = ""
 
     @property
     def transform(self) -> Transform | None:
@@ -178,12 +184,11 @@ class Context:
 
     def track(
         self,
-        uid: str | None = None,
+        transform: str | Transform | None = None,
         *,
         params: dict | None = None,
         new_run: bool | None = None,
         path: str | None = None,
-        transform: Transform | None = None,
     ) -> None:
         """Initiate a run with tracked data lineage.
 
@@ -196,24 +201,31 @@ class Context:
         script-like transform exists in a git repository and links it.
 
         Args:
-            uid: A `uid` to create or load a transform.
+            transform: A transform `uid` or record. If `None`, creates a `uid`.
             params: A dictionary of parameters to track for the run.
             new_run: If `False`, loads latest run of transform
                 (default notebook), if `True`, creates new run (default pipeline).
             path: Filepath of notebook or script. Only needed if it can't be
                 automatically detected.
-            transform: Useful to track an abstract pipeline.
 
         Examples:
 
-            To track the run of a notebook or script, call:
+            To create a transform `uid` for tracking a script or notebook, call:
 
-            >>> import lamindb as ln
             >>> ln.track()
 
+            To track the run of a notebook or script, call:
+
+            >>> ln.track("FPnfDtJz8qbE0000")  # replace with your uid
+
         """
-        if uid is not None:
+        self._logging_message_track = ""
+        self._logging_message_imports = ""
+        uid = None
+        if transform is not None and isinstance(transform, str):
+            uid = transform
             self.uid = uid
+            transform = None
         self._path = None
         if transform is None:
             is_tracked = False
@@ -223,17 +235,20 @@ class Context:
             )
             transform = None
             stem_uid = None
+            # you can set ln.context.uid and then call ln.track() without passing anythin
+            # that has been the preferred syntax for a while; we'll likely
+            # deprecate it at some point
             if uid is not None or self.uid is not None:
                 transform = Transform.filter(uid=self.uid).one_or_none()
                 if self.version is not None:
                     # test inconsistent version passed
                     if (
                         transform is not None
-                        and transform.version is not None
-                        and self.version != transform.version
+                        and transform.version is not None  # type: ignore
+                        and self.version != transform.version  # type: ignore
                     ):
                         raise SystemExit(
-                            f"Please pass consistent version: ln.context.version = '{transform.version}'"
+                            f"Please pass consistent version: ln.context.version = '{transform.version}'"  # type: ignore
                         )
                     # test whether version was already used for another member of the family
                     suid, vuid = (
@@ -302,10 +317,14 @@ class Context:
                 transform_exists = Transform.filter(id=transform.id).first()
             if transform_exists is None:
                 transform.save()
-                self._logging_message += f"created Transform('{transform.uid[:8]}')"
+                self._logging_message_track += (
+                    f"created Transform('{transform.uid[:8]}')"
+                )
                 transform_exists = transform
             else:
-                self._logging_message += f"loaded Transform('{transform.uid[:8]}')"
+                self._logging_message_track += (
+                    f"loaded Transform('{transform.uid[:8]}')"
+                )
             self._transform = transform_exists
 
         if new_run is None:  # for notebooks, default to loading latest runs
@@ -322,7 +341,7 @@ class Context:
             )
             if run is not None:  # loaded latest run
                 run.started_at = datetime.now(timezone.utc)  # update run time
-                self._logging_message += f", started Run('{run.uid[:8]}') at {format_field_value(run.started_at)}"
+                self._logging_message_track += f", started Run('{run.uid[:8]}') at {format_field_value(run.started_at)}"
 
         if run is None:  # create new run
             run = Run(
@@ -330,7 +349,7 @@ class Context:
                 params=params,
             )
             run.started_at = datetime.now(timezone.utc)
-            self._logging_message += f", started new Run('{run.uid[:8]}') at {format_field_value(run.started_at)}"
+            self._logging_message_track += f", started new Run('{run.uid[:8]}') at {format_field_value(run.started_at)}"
         # can only determine at ln.finish() if run was consecutive in
         # interactive session, otherwise, is consecutive
         run.is_consecutive = True if is_run_from_ipython else None
@@ -338,13 +357,14 @@ class Context:
         run.save()
         if params is not None:
             run.params.add_values(params)
-            self._logging_message += "\n→ params: " + " ".join(
+            self._logging_message_track += "\n→ params: " + " ".join(
                 f"{key}='{value}'" for key, value in params.items()
             )
         self._run = run
         track_environment(run)
-        logger.important(self._logging_message)
-        self._logging_message = ""
+        logger.important(self._logging_message_track)
+        if self._logging_message_imports:
+            logger.important(self._logging_message_imports)
 
     def _track_script(
         self,
@@ -406,9 +426,9 @@ class Context:
                 from nbproject.dev._pypackage import infer_pypackages
 
                 nb = nbproject.dev.read_notebook(path_str)
-                logger.important(
+                self._logging_message_imports += (
                     "notebook imports:"
-                    f" {pretty_pypackages(infer_pypackages(nb, pin_versions=True))}"
+                    f" {pretty_pypackages(infer_pypackages(nb, pin_versions=True))}\n"
                 )
             except Exception:
                 logger.debug("inferring imported packages failed")
@@ -471,7 +491,7 @@ class Context:
                 raise_update_context = True
             if raise_update_context:
                 raise UpdateContext(get_key_clashing_message(revises, key))
-            self._logging_message += f"created Transform('{transform.uid[:8]}')"
+            self._logging_message_track += f"created Transform('{transform.uid[:8]}')"
         else:
             uid = transform.uid
             # transform was already saved via `finish()`
@@ -485,7 +505,7 @@ class Context:
             elif transform.name != name:
                 transform.name = name
                 transform.save()
-                self._logging_message += (
+                self._logging_message_track += (
                     "updated transform name, "  # white space on purpose
                 )
             elif (
@@ -509,7 +529,7 @@ class Context:
                     if condition:
                         bump_revision = True
                     else:
-                        self._logging_message += (
+                        self._logging_message_track += (
                             f"loaded Transform('{transform.uid[:8]}')"
                         )
                 if bump_revision:
@@ -523,7 +543,9 @@ class Context:
                         f'ln.track("{uid[:-4]}{increment_base62(uid[-4:])}")'
                     )
             else:
-                self._logging_message += f"loaded Transform('{transform.uid[:8]}')"
+                self._logging_message_track += (
+                    f"loaded Transform('{transform.uid[:8]}')"
+                )
         self._transform = transform
 
     def finish(self, ignore_non_consecutive: None | bool = None) -> None:

lamindb/core/_label_manager.py

@@ -202,7 +202,7 @@ class LabelManager:
             transfer_logs = {"mapped": [], "transferred": [], "run": None}
         using_key = settings._using_key
         for related_name, (_, labels) in get_labels_as_dict(
-            data, instance=self._host._state.db
+            data, instance=data._state.db
         ).items():
             labels = labels.all()
             if not labels.exists():

lamindb/core/storage/_anndata_accessor.py

@@ -54,13 +54,16 @@ if anndata_version_parse < version.parse("0.10.0"):
         return SparseDataset(group)
 
 else:
+    if anndata_version_parse >= version.parse("0.11.0"):
+        from anndata._core.sparse_dataset import (  # type: ignore
+            _CSRDataset as CSRDataset,
+        )
+    else:
+        from anndata._core.sparse_dataset import CSRDataset  # type: ignore
     from anndata._core.sparse_dataset import (
         BaseCompressedSparseDataset as SparseDataset,
     )
-    from anndata._core.sparse_dataset import (  # type: ignore
-        CSRDataset,
-        sparse_dataset,
-    )
+    from anndata._core.sparse_dataset import sparse_dataset  # type: ignore
 
     def _check_group_format(*args):
         pass

{lamindb-0.76.15.dist-info → lamindb-0.76.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.76.15
+Version: 0.76.16
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.9,<3.13
@@ -9,20 +9,22 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Requires-Dist: lnschema_core==0.76.1
+Requires-Dist: lnschema_core==0.76.2
 Requires-Dist: lamin_utils==0.13.7
-Requires-Dist: lamin_cli==0.20.1
+Requires-Dist: lamin_cli==0.21.2
 Requires-Dist: lamindb_setup
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
 Requires-Dist: python-dateutil
-Requires-Dist: anndata>=0.8.0,<=0.10.9
+Requires-Dist: anndata>=0.8.0,<=0.11.1
 Requires-Dist: fsspec
 Requires-Dist: graphviz
 Requires-Dist: psycopg2-binary
 Requires-Dist: lamindb_setup[aws] ; extra == "aws"
-Requires-Dist: bionty==0.52.0 ; extra == "bionty"
+Requires-Dist: bionty==0.52.1 ; extra == "bionty"
+Requires-Dist: cellregistry ; extra == "cellregistry"
+Requires-Dist: clinicore ; extra == "clinicore"
 Requires-Dist: line_profiler ; extra == "dev"
 Requires-Dist: pre-commit ; extra == "dev"
 Requires-Dist: nox ; extra == "dev"
@@ -35,19 +37,29 @@ Requires-Dist: nbproject_test>=0.5.1 ; extra == "dev"
 Requires-Dist: faker-biology ; extra == "dev"
 Requires-Dist: django-schema-graph ; extra == "erdiagram"
 Requires-Dist: readfcs>=1.1.9 ; extra == "fcs"
+Requires-Dist: findrefs ; extra == "findrefs"
 Requires-Dist: lamindb_setup[gcp] ; extra == "gcp"
 Requires-Dist: nbproject==0.10.5 ; extra == "jupyter"
 Requires-Dist: jupytext ; extra == "jupyter"
 Requires-Dist: nbconvert ; extra == "jupyter"
+Requires-Dist: omop ; extra == "omop"
+Requires-Dist: ourprojects ; extra == "ourprojects"
+Requires-Dist: wetlab ; extra == "wetlab"
 Requires-Dist: zarr>=2.16.0 ; extra == "zarr"
 Project-URL: Home, https://github.com/laminlabs/lamindb
 Provides-Extra: aws
 Provides-Extra: bionty
+Provides-Extra: cellregistry
+Provides-Extra: clinicore
 Provides-Extra: dev
 Provides-Extra: erdiagram
 Provides-Extra: fcs
+Provides-Extra: findrefs
 Provides-Extra: gcp
 Provides-Extra: jupyter
+Provides-Extra: omop
+Provides-Extra: ourprojects
+Provides-Extra: wetlab
 Provides-Extra: zarr
 
 [![Stars](https://img.shields.io/github/stars/laminlabs/lamindb?logo=GitHub&color=yellow)](https://github.com/laminlabs/lamindb)

{lamindb-0.76.15.dist-info → lamindb-0.76.16.dist-info}/RECORD

@@ -1,18 +1,17 @@
-lamindb/__init__.py,sha256=ZoRboX4PQc0sbVajeZ1fd4GoLN3YDj5U74zVUpcSd9I,2278
-lamindb/_artifact.py,sha256=WNdKAJFu3sFgQ_Qe1JflDHiTP4EGhAIkzaHFwTthAjY,44903
-lamindb/_can_validate.py,sha256=TKfkHgkPl1bwuJrQCp6pHgQbi7m2pc_zjFhZfUAUU20,19573
+lamindb/__init__.py,sha256=kJTUjuddkfht0Wb1zhHebfdwK5q1CGpluwZwj8cvYfs,2278
+lamindb/_artifact.py,sha256=4sHLnJQdWAjplB-dk0JXzhrHaaDn2ri5SeyvA7yzajQ,45166
+lamindb/_can_validate.py,sha256=zRmSOJqPwwHcmRHDH18Dbn6eklY62vgiLpyKgvXpbLU,19956
 lamindb/_collection.py,sha256=MLOEoOgTu7rTlRD7zkm1k0YIk_gVhQDO17JbmZCptOs,14573
-lamindb/_curate.py,sha256=pLsdtnnTn0qQbsUB4hrU3yuoVnVNh-BQe10EpMNZ4Ns,64083
+lamindb/_curate.py,sha256=aBpw0RdzDeFd4NdMzr5d4ZbpwOinONwcELnEy_odSNw,64493
 lamindb/_feature.py,sha256=9cgrcHoyOa1jpON-9KiUfFSHcxiGECiefUAqAx4cVvU,5325
 lamindb/_feature_set.py,sha256=WdXw_YGlMXCs8l0WVHOrqvvrH2hsQLqCiho8LFDYwhI,8161
-lamindb/_filter.py,sha256=Pf9NHV4gm7NOC0Frtvx4W7nvwt2EowOP74DwppyXAZs,635
-lamindb/_finish.py,sha256=VMAmxCUFmTKIMSCx7LEh4QAnWDeue6MeUAAzkMVEYMU,9546
+lamindb/_finish.py,sha256=r8q6rhml2vHiDlojCnfSpS8pbfnq7u6MwIcBIwuVq2o,9745
 lamindb/_from_values.py,sha256=uRtZLaMWKoANMMXm1hrADHfckRCTiK8_d02Yp07nLkw,14119
 lamindb/_is_versioned.py,sha256=GWZk-usV6aB33Cl9AlrnEGE5nxUkZic7QJzOW_DrwQA,1298
 lamindb/_parents.py,sha256=INhbqh6IaUjuYVUOp-6rnOGN-8kGZirNqqW9XQ1qz_M,17119
 lamindb/_query_manager.py,sha256=noc05Ad-aADxckOVBVDAiErFB7gL8XTgckELvI4rGmM,3702
-lamindb/_query_set.py,sha256=6vHOvB_uXzKVVIw8AAVw7EYOIAGuw3TYcUzkpNlFLdE,12973
-lamindb/_record.py,sha256=isR9GMQFwUUwVSmPNABvEzcJS38TbjhD7Cc6kygPsTA,26819
+lamindb/_query_set.py,sha256=xcGQZvWIGM9gQPtES3jQjZL_wCaUAp-ZNeVb1Kv-Ado,14682
+lamindb/_record.py,sha256=nVkVnOcVFxVyE5a1boXtKp4i4idlIJHJevXfHiwoxSk,29149
 lamindb/_run.py,sha256=K_5drpLn3D7y3XtZ3vtAw35rG5RCSvB4bXQZx4ESSI0,1964
 lamindb/_save.py,sha256=OD052Qr_hiMyAonHTktKETe_Bhnp1RY810y0rwZqpBQ,11352
 lamindb/_storage.py,sha256=GBVChv-DHVMNEBJL5l_JT6B4RDhZ6NnwgzmUICphYKk,413
@@ -21,11 +20,11 @@ lamindb/_ulabel.py,sha256=DQQzAPkrOg8W9I77BJ5umajR8MQcFSvXYUy53YNN2HA,1604
 lamindb/_utils.py,sha256=LGdiW4k3GClLz65vKAVRkL6Tw-Gkx9DWAdez1jyA5bE,428
 lamindb/_view.py,sha256=4Ln2ItTb3857PAI-70O8eJYqoTJ_NNFc7E_wds6OGns,2412
 lamindb/core/__init__.py,sha256=y87MCP1BEC2qHNVDIOwqVteIP_2hPCdIoa9JXr0EG8U,1524
-lamindb/core/_context.py,sha256=dI3z7fCMRPC3IMb7-EIaQYhacSZBA4HfLVFyoJtVL7I,22900
+lamindb/core/_context.py,sha256=6TJzCA88F4LUxLXBVpJ0UAeKhHgL8WieKgsbXmVtlnU,23803
 lamindb/core/_data.py,sha256=BVZkxK8aloSecH25LivbwnjcG1fz7Gs2UDceO5pWd3I,20049
 lamindb/core/_django.py,sha256=yeMPp1n9WrFmEjVRdavfpVqAolPLd24RseTQlvsK67w,7157
 lamindb/core/_feature_manager.py,sha256=q4WmzJvFLL_fAs-vNRgV2klanAoU6Wu8_g0O2dyIjVg,40027
-lamindb/core/_label_manager.py,sha256=yh-r4KbtOArMUKPJL75yOxJc8HUKqsik8pExBVKyDlA,10949
+lamindb/core/_label_manager.py,sha256=Y0NdvXLGB3RTYPnNJiGfnhnvYWSS7XsOEj9UattJw3c,10943
 lamindb/core/_mapped_collection.py,sha256=EDS0xzOdCc_iGE_Iqv5COTVHNm4jWue7Jtcd8DdXkJU,24591
 lamindb/core/_settings.py,sha256=6jNadlQdimxCsKR2ZyUD0YJYzOdubTnKktki-MqEWqQ,6137
 lamindb/core/_sync_git.py,sha256=lIgl6YfpH4rCFT1WILAp7zlemZfxog1d0zp3cX0KIZw,4531
@@ -40,7 +39,7 @@ lamindb/core/datasets/__init__.py,sha256=zRP98oqUAaXhqWyKMiH0s_ImVIuNeziQQ2kQ_t0
 lamindb/core/datasets/_core.py,sha256=JGP_q-OQibDCEaI54jZ2F6fSbSW9Yg6oYOqgOCXM0v4,20414
 lamindb/core/datasets/_fake.py,sha256=BZF9R_1iF0HDnvtZNqL2FtsjSMuqDIfuFxnw_LJYIh4,953
 lamindb/core/storage/__init__.py,sha256=JOIMu_7unbyhndtH1j0Q-9AvY8knSuc1IJO9sQnyBAQ,498
-lamindb/core/storage/_anndata_accessor.py,sha256=2p1HjoatmZjQ1u94tjgmXgiv8MKowrQH5xInDmiLCw4,24191
+lamindb/core/storage/_anndata_accessor.py,sha256=C321qng00vMmugukxv5dX8z3oJeRxq869DgAGaEd5rg,24413
 lamindb/core/storage/_anndata_sizes.py,sha256=aXO3OB--tF5MChenSsigW6Q-RuE8YJJOUTVukkLrv9A,1029
 lamindb/core/storage/_backed_access.py,sha256=t9iS9mlZQBy1FfIS-Twt-96npYiShbPwEo2y9_3b6jY,3517
 lamindb/core/storage/_pyarrow_dataset.py,sha256=wuLsEvdblqMdUdDfMtis8AWrE3igzvFWTSTbxuD1Oc8,926
@@ -56,7 +55,7 @@ lamindb/integrations/__init__.py,sha256=RWGMYYIzr8zvmNPyVB4m-p4gMDhxdRbjES2Ed23O
 lamindb/integrations/_vitessce.py,sha256=uPl45_w4Uu9_BhpBDDVonC1nDOuAnB7DAnzi5w5bZAE,4032
 lamindb/setup/__init__.py,sha256=OwZpZzPDv5lPPGXZP7-zK6UdO4FHvvuBh439yZvIp3A,410
 lamindb/setup/core/__init__.py,sha256=SevlVrc2AZWL3uALbE5sopxBnIZPWZ1IB0NBDudiAL8,167
-lamindb-0.76.15.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-lamindb-0.76.15.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-lamindb-0.76.15.dist-info/METADATA,sha256=31K0SoWfAxBpWR3x_7PLFfwpu-mnsQ8IHFqsvjdg_cw,2365
-lamindb-0.76.15.dist-info/RECORD,,
+lamindb-0.76.16.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+lamindb-0.76.16.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+lamindb-0.76.16.dist-info/METADATA,sha256=kAhqI1VyLBx0iXLsQIt31re6xPcJEG2YiXS0N1BaO_M,2797
+lamindb-0.76.16.dist-info/RECORD,,

lamindb/_filter.py

@@ -1,21 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-from ._query_set import QuerySet, process_expressions
-
-if TYPE_CHECKING:
-    from lnschema_core import Record
-
-
-def filter(registry: type[Record], *queries, **expressions) -> QuerySet:
-    """See :meth:`~lamindb.core.Record.filter`."""
-    _using_key = None
-    if "_using_key" in expressions:
-        _using_key = expressions.pop("_using_key")
-    expressions = process_expressions(registry, expressions)
-    qs = QuerySet(model=registry, using=_using_key)
-    if len(expressions) > 0:
-        return qs.filter(*queries, **expressions)
-    else:
-        return qs