lamindb 0.77.2__py3-none-any.whl → 1.0rc1__py3-none-any.whl

lamindb/_from_values.py

@@ -5,16 +5,16 @@ from typing import TYPE_CHECKING
 import pandas as pd
 from django.core.exceptions import FieldDoesNotExist
 from lamin_utils import colors, logger
-from lnschema_core.models import Record
 
 from lamindb._query_set import RecordList
+from lamindb.models import Record
 
 from .core._settings import settings
 
 if TYPE_CHECKING:
     from collections.abc import Iterable
 
-    from lnschema_core.types import ListLike, StrField
+    from lamindb.base.types import ListLike, StrField
 
 
 # The base function for `from_values`
@@ -52,12 +52,6 @@ def get_or_create_records(
             if from_source:
                 if isinstance(source, Record):
                     source_record = source
-                elif (
-                    len(records) > 0
-                    and hasattr(records[0], "source_id")
-                    and records[0].source_id
-                ):
-                    source_record = records[0].source
             if not source_record and hasattr(registry, "public"):
                 if organism is None:
                     organism = _ensembl_prefix(nonexist_values[0], field, organism)
@@ -95,7 +89,7 @@ def get_or_create_records(
                 if len(msg) > 0 and not mute:
                     logger.success(msg)
                 s = "" if len(unmapped_values) == 1 else "s"
-                print_values = colors.yellow(_print_values(unmapped_values))
+                print_values = colors.yellow(_format_values(unmapped_values))
                 name = registry.__name__
                 n_nonval = colors.yellow(f"{len(unmapped_values)} non-validated")
                 if not mute:
@@ -103,7 +97,7 @@ def get_or_create_records(
                         f"{colors.red('did not create')} {name} record{s} for "
                         f"{n_nonval} {colors.italic(f'{field.field.name}{s}')}: {print_values}"
                     )
-        # if registry.__get_schema_name__() == "bionty" or registry == ULabel:
+        # if registry.__get_module_name__() == "bionty" or registry == ULabel:
         #     if isinstance(iterable, pd.Series):
         #         feature = iterable.name
         #     feature_name = None
@@ -167,7 +161,7 @@ def get_existing_records(
     if not mute:
         if len(validated) > 0:
             s = "" if len(validated) == 1 else "s"
-            print_values = colors.green(_print_values(validated))
+            print_values = colors.green(_format_values(validated))
             msg = (
                 "loaded"
                 f" {colors.green(f'{len(validated)} {model.__name__} record{s}')}"
@@ -176,7 +170,7 @@ def get_existing_records(
         if len(syn_mapper) > 0:
             s = "" if len(syn_mapper) == 1 else "s"
             names = list(syn_mapper.keys())
-            print_values = colors.green(_print_values(names))
+            print_values = colors.green(_format_values(names))
             syn_msg = (
                 "loaded"
                 f" {colors.green(f'{len(syn_mapper)} {model.__name__} record{s}')}"
@@ -236,14 +230,23 @@ def create_records_from_source(
     bionty_df = filter_bionty_df_columns(model=model, public_ontology=public_ontology)
 
     # standardize in the bionty reference
-    result = public_ontology.inspect(iterable_idx, field=field.field.name, mute=True)
+    # do not inspect synonyms if the field is not name field
+    inspect_synonyms = True
+    if hasattr(model, "_name_field") and field.field.name != model._name_field:
+        inspect_synonyms = False
+    result = public_ontology.inspect(
+        iterable_idx,
+        field=field.field.name,
+        mute=True,
+        inspect_synonyms=inspect_synonyms,
+    )
     syn_mapper = result.synonyms_mapper
 
     msg_syn: str = ""
     if len(syn_mapper) > 0:
         s = "" if len(syn_mapper) == 1 else "s"
         names = list(syn_mapper.keys())
-        print_values = colors.purple(_print_values(names))
+        print_values = colors.purple(_format_values(names))
         msg_syn = (
             "created"
             f" {colors.purple(f'{len(syn_mapper)} {model.__name__} record{s} from Bionty')}"
@@ -277,7 +280,7 @@ def create_records_from_source(
         validated = result.validated
         if len(validated) > 0:
             s = "" if len(validated) == 1 else "s"
-            print_values = colors.purple(_print_values(validated))
+            print_values = colors.purple(_format_values(validated))
             # this is the success msg for existing records in the DB
             if len(msg) > 0 and not mute:
                 logger.success(msg)
@@ -307,7 +310,7 @@ def index_iterable(iterable: Iterable) -> pd.Index:
     return idx[(idx != "") & (~idx.isnull())]
 
 
-def _print_values(
+def _format_values(
     names: Iterable, n: int = 20, quotes: bool = True, sep: str = "'"
 ) -> str:
     if isinstance(names, dict):
@@ -345,7 +348,7 @@ def _bulk_create_dicts_from_df(
         dup = df.index[df.index.duplicated()].unique().tolist()
         if len(dup) > 0:
             s = "" if len(dup) == 1 else "s"
-            print_values = _print_values(dup)
+            print_values = _format_values(dup)
             multi_msg = (
                 f"ambiguous validation in Bionty for {len(dup)} record{s}:"
                 f" {print_values}"

lamindb/_is_versioned.py

@@ -3,7 +3,8 @@ from __future__ import annotations
 import lamindb_setup as ln_setup
 from lamin_utils import logger
 from lamindb_setup.core.upath import UPath
-from lnschema_core.models import IsVersioned
+
+from lamindb.models import IsVersioned
 
 from ._utils import attach_func_to_class_method
 from .core.versioning import create_uid, get_new_path_from_uid

lamindb/_parents.py

@@ -5,15 +5,22 @@ from typing import TYPE_CHECKING, Literal
 
 import lamindb_setup as ln_setup
 from lamin_utils import logger
-from lnschema_core import Artifact, Collection, Record, Run, Transform
-from lnschema_core.models import HasParents, format_field_value
+
+from lamindb.models import (
+    Artifact,
+    Collection,
+    HasParents,
+    Record,
+    Run,
+    Transform,
+    format_field_value,
+)
 
 from ._record import get_name_field
 from ._utils import attach_func_to_class_method
 
 if TYPE_CHECKING:
-    from lnschema_core.types import StrField
-
+    from lamindb.base.types import StrField
     from lamindb.core import QuerySet
 
 LAMIN_GREEN_LIGHTER = "#10b981"
@@ -25,7 +32,7 @@ TRANSFORM_EMOJIS = {
     "pipeline": "🧩",
     "script": "📝",
     "function": "🔧",
-    "glue": "🧲",
+    "linker": "🧲",
 }
 is_run_from_ipython = getattr(builtins, "__IPYTHON__", False)
 
@@ -342,8 +349,8 @@ def _record_label(record: Record, field: str | None = None):
             rf' FACE="Monospace">uid={record.uid}<BR/>version={record.version}</FONT>>'
         )
     elif isinstance(record, Run):
-        if record.transform.name:
-            name = f'{record.transform.name.replace("&", "&amp;")}'
+        if record.transform.description:
+            name = f'{record.transform.description.replace("&", "&amp;")}'
         elif record.transform.key:
             name = f'{record.transform.key.replace("&", "&amp;")}'
         else:
@@ -395,22 +402,22 @@ def _get_all_parent_runs(data: Artifact | Collection) -> list:
             inputs_run = (
                 r.__getattribute__(f"input_{name}s")
                 .all()
-                .filter(visibility__in=[0, 1])
+                .filter(_branch_code__in=[0, 1])
                 .list()
             )
             if name == "artifact":
                 inputs_run += (
-                    r.input_collections.all().filter(visibility__in=[0, 1]).list()
+                    r.input_collections.all().filter(_branch_code__in=[0, 1]).list()
                 )
             outputs_run = (
                 r.__getattribute__(f"output_{name}s")
                 .all()
-                .filter(visibility__in=[0, 1])
+                .filter(_branch_code__in=[0, 1])
                 .list()
             )
             if name == "artifact":
                 outputs_run += (
-                    r.output_collections.all().filter(visibility__in=[0, 1]).list()
+                    r.output_collections.all().filter(_branch_code__in=[0, 1]).list()
                 )
             # if inputs are outputs artifacts are the same, will result infinite loop
             # so only show as outputs
@@ -444,7 +451,7 @@ def _get_all_child_runs(data: Artifact | Collection) -> list:
             {
                 f.run
                 for f in data.run.output_collections.all()
-                .filter(visibility__in=[0, 1])
+                .filter(_branch_code__in=[0, 1])
                 .all()
             }
         )
@@ -455,24 +462,24 @@ def _get_all_child_runs(data: Artifact | Collection) -> list:
             inputs_run = (
                 r.__getattribute__(f"input_{name}s")
                 .all()
-                .filter(visibility__in=[0, 1])
+                .filter(_branch_code__in=[0, 1])
                 .list()
             )
             if name == "artifact":
                 inputs_run += (
-                    r.input_collections.all().filter(visibility__in=[0, 1]).list()
+                    r.input_collections.all().filter(_branch_code__in=[0, 1]).list()
                 )
             run_inputs_outputs += [(inputs_run, r)]
 
             outputs_run = (
                 r.__getattribute__(f"output_{name}s")
                 .all()
-                .filter(visibility__in=[0, 1])
+                .filter(_branch_code__in=[0, 1])
                 .list()
             )
             if name == "artifact":
                 outputs_run += (
-                    r.output_collections.all().filter(visibility__in=[0, 1]).list()
+                    r.output_collections.all().filter(_branch_code__in=[0, 1]).list()
                 )
             run_inputs_outputs += [(r, outputs_run)]
 

lamindb/_query_manager.py

@@ -5,13 +5,13 @@ from typing import TYPE_CHECKING, NamedTuple
 from django.db import models
 from lamin_utils import logger
 from lamindb_setup.core._docs import doc_args
-from lnschema_core.models import Record
 
-from .core._feature_manager import get_feature_set_by_slot_
+from lamindb.models import Record
+
 from .core._settings import settings
 
 if TYPE_CHECKING:
-    from lnschema_core.types import StrField
+    from lamindb.base.types import StrField
 
 
 class QueryManager(models.Manager):

lamindb/_query_set.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import re
+import warnings
 from collections import UserList
 from collections.abc import Iterable
 from collections.abc import Iterable as IterableType
@@ -12,17 +13,17 @@ from django.db.models import F, ForeignKey, ManyToManyField
 from django.db.models.fields.related import ForeignObjectRel
 from lamin_utils import logger
 from lamindb_setup.core._docs import doc_args
-from lnschema_core.models import (
+
+from lamindb.models import (
     Artifact,
     CanCurate,
     Collection,
     Feature,
     IsVersioned,
     Record,
-    Registry,
     Run,
+    Schema,
     Transform,
-    VisibilityChoice,
 )
 
 from .core.exceptions import DoesNotExist
@@ -32,7 +33,7 @@ T = TypeVar("T")
 if TYPE_CHECKING:
     from collections.abc import Iterable
 
-    from lnschema_core.types import ListLike, StrField
+    from lamindb.base.types import ListLike, StrField
 
 
 class MultipleResultsFound(Exception):
@@ -79,6 +80,58 @@ def one_helper(self):
         return self[0]
 
 
+def get_backward_compat_filter_kwargs(queryset, expressions):
+    if queryset.model in {Collection, Transform}:
+        name_mappings = {
+            "name": "key",
+            "visibility": "_branch_code",  # for convenience (and backward compat <1.0)
+        }
+    elif queryset.model == Artifact:
+        name_mappings = {
+            "n_objects": "n_files",
+            "visibility": "_branch_code",  # for convenience (and backward compat <1.0)
+            "transform": "run__transform",  # for convenience (and backward compat <1.0)
+            "feature_sets": "_schemas_m2m",
+            "type": "kind",
+            "_accessor": "otype",
+        }
+    elif queryset.model == Schema:
+        name_mappings = {
+            "registry": "itype",
+            "artifacts": "_artifacts_m2m",  # will raise warning when we start to migrate over
+        }
+    else:
+        return expressions
+    was_list = False
+    if isinstance(expressions, list):
+        # make a dummy dictionary
+        was_list = True
+        expressions = {field: True for field in expressions}
+    mapped = {}
+    for field, value in expressions.items():
+        parts = field.split("__")
+        if parts[0] in name_mappings:
+            if parts[0] not in {
+                "transform",
+                "visibility",
+                "feature_sets",
+                "schemas",
+                "artifacts",
+            }:
+                warnings.warn(
+                    f"{name_mappings[parts[0]]} is deprecated, please query for {parts[0]} instead",
+                    DeprecationWarning,
+                    stacklevel=2,
+                )
+            new_field = name_mappings[parts[0]] + (
+                "__" + "__".join(parts[1:]) if len(parts) > 1 else ""
+            )
+            mapped[new_field] = value
+        else:
+            mapped[field] = value
+    return list(mapped.keys()) if was_list else mapped
+
+
 def process_expressions(queryset: QuerySet, expressions: dict) -> dict:
     def _map_databases(value: Any, key: str, target_db: str) -> tuple[str, Any]:
         if isinstance(value, Record):
@@ -105,23 +158,26 @@ def process_expressions(queryset: QuerySet, expressions: dict) -> dict:
 
         return key, value
 
-    if queryset.model in {Artifact, Collection}:
-        # visibility is set to 0 unless expressions contains id or uid equality
+    expressions = get_backward_compat_filter_kwargs(
+        queryset,
+        expressions,
+    )
+
+    if issubclass(queryset.model, Record):
+        # _branch_code is set to 0 unless expressions contains id or uid
         if not (
             "id" in expressions
             or "uid" in expressions
             or "uid__startswith" in expressions
         ):
-            visibility = "visibility"
-            if not any(e.startswith(visibility) for e in expressions):
-                expressions[visibility] = (
-                    VisibilityChoice.default.value
-                )  # default visibility
-            # if visibility is None, do not apply a filter
+            _branch_code = "_branch_code"
+            if not any(e.startswith(_branch_code) for e in expressions):
+                expressions[_branch_code] = 1  # default _branch_code
+            # if _branch_code is None, do not apply a filter
             # otherwise, it would mean filtering for NULL values, which doesn't make
             # sense for a non-NULLABLE column
-            elif visibility in expressions and expressions[visibility] is None:
-                expressions.pop(visibility)
+            elif _branch_code in expressions and expressions[_branch_code] is None:
+                expressions.pop(_branch_code)
     if queryset._db is not None:
         # only check for database mismatch if there is a defined database on the
         # queryset
@@ -213,6 +269,8 @@ def get_basic_field_names(
         "created_at",
         "created_by_id",
         "updated_at",
+        "_aux",
+        "_branch_code",
     ]:
         if field_name in field_names:
             field_names.remove(field_name)
@@ -242,17 +300,17 @@ def get_feature_annotate_kwargs(show_features: bool | list[str]) -> dict[str, An
     link_models_on_models = {
         getattr(
             Artifact, obj.related_name
-        ).through.__get_name_with_schema__(): obj.related_model.__get_name_with_schema__()
+        ).through.__get_name_with_module__(): obj.related_model.__get_name_with_module__()
         for obj in Artifact._meta.related_objects
-        if obj.related_model.__get_name_with_schema__() in cat_feature_types
+        if obj.related_model.__get_name_with_module__() in cat_feature_types
     }
     link_models_on_models["ArtifactULabel"] = "ULabel"
     link_attributes_on_models = {
         obj.related_name: link_models_on_models[
-            obj.related_model.__get_name_with_schema__()
+            obj.related_model.__get_name_with_module__()
         ]
         for obj in Artifact._meta.related_objects
-        if obj.related_model.__get_name_with_schema__() in link_models_on_models
+        if obj.related_model.__get_name_with_module__() in link_models_on_models
     }
     # Prepare Django's annotate for features
     annotate_kwargs = {}
@@ -478,7 +536,9 @@ class QuerySet(models.QuerySet):
             include = []
         elif isinstance(include, str):
             include = [include]
+        include = get_backward_compat_filter_kwargs(self, include)
         field_names = get_basic_field_names(self, include, features)
+
         annotate_kwargs = {}
         if features:
             annotate_kwargs.update(get_feature_annotate_kwargs(features))
@@ -490,6 +550,7 @@ class QuerySet(models.QuerySet):
             queryset = self.annotate(**annotate_kwargs)
         else:
             queryset = self
+
         df = pd.DataFrame(queryset.values(*field_names, *list(annotate_kwargs.keys())))
         if len(df) == 0:
             df = pd.DataFrame({}, columns=field_names)
@@ -500,6 +561,12 @@ class QuerySet(models.QuerySet):
         pk_column_name = pk_name if pk_name in df.columns else f"{pk_name}_id"
         if pk_column_name in df_reshaped.columns:
             df_reshaped = df_reshaped.set_index(pk_column_name)
+
+        # Compatibility code
+        df_reshaped.columns = df_reshaped.columns.str.replace(
+            r"_schemas_m2m", "feature_sets", regex=True
+        )
+
         return df_reshaped
 
     def delete(self, *args, **kwargs):