lamindb 0.54.4__py3-none-any.whl → 0.55.1__py3-none-any.whl

lamindb/__init__.py

@@ -50,12 +50,11 @@ Static classes & modules:
 
    settings
    setup
-   schema
    dev
 
 """
 
-__version__ = "0.54.4"  # denote a release candidate for 0.1.0 with 0.1rc1
+__version__ = "0.55.1"  # denote a release candidate for 0.1.0 with 0.1rc1
 
 import os as _os
 
@@ -65,6 +64,7 @@ import lamindb_setup as _lamindb_setup
 from lamin_utils import py_version_warning as _py_version_warning
 from lamindb_setup import _check_instance_setup
 from lamindb_setup._check_instance_setup import _INSTANCE_NOT_SETUP_WARNING
+from lamindb_setup._init_instance import reload_schema_modules as _reload_schema_modules
 
 _py_version_warning("3.8", "3.11")
 
@@ -114,15 +114,15 @@ if _INSTANCE_SETUP:
     from . import _ulabel  # noqa
     from . import _validate  # noqa
     from . import dev  # noqa
-    from . import schema  # noqa
     from ._delete import delete  # noqa
-    from ._registry import select_backward as select  # noqa
     from ._save import save  # noqa
     from ._view import view  # noqa
     from .dev import _priors  # noqa
     from .dev._run_context import run_context  # noqa
     from .dev._settings import settings
 
+    # schema modules
+    _reload_schema_modules(_lamindb_setup.settings.instance)
+
     track = run_context._track  # noqa
-    add = save  # backward compat
     settings.__doc__ = """Global :class:`~lamindb.dev.Settings`."""

lamindb/_dataset.py

@@ -1,10 +1,15 @@
 from collections import defaultdict
+from pathlib import Path
 from typing import Dict, Iterable, Literal, Optional, Tuple, Union
 
 import anndata as ad
 import pandas as pd
 from lamin_utils import logger
+from lamindb_setup._init_instance import register_storage
+from lamindb_setup.dev import StorageSettings
 from lamindb_setup.dev._docs import doc_args
+from lamindb_setup.dev._hub_utils import get_storage_region
+from lamindb_setup.dev.upath import UPath
 from lnschema_core import Modality
 from lnschema_core.models import Dataset, Feature, FeatureSet
 from lnschema_core.types import AnnDataLike, DataLike, FieldAttr
@@ -21,7 +26,7 @@ from .dev._data import (
     add_transform_to_kwargs,
     get_run,
     save_feature_set_links,
-    save_transform_run_feature_sets,
+    save_feature_sets,
 )
 from .dev.hashing import hash_set
 
@@ -84,11 +89,12 @@ def __init__(
             )
 
     run = get_run(run)
-    # there are exactly two ways of creating a Dataset object right now
+    # there are exactly 3 ways of creating a Dataset object right now
     # using exactly one file or using more than one file
     # init file
     if isinstance(data, (pd.DataFrame, ad.AnnData, File)):
         files = None
+        storage = None
         if isinstance(data, File):
             file = data
             if file._state.adding:
@@ -116,9 +122,23 @@ def __init__(
         if file.description is None or file.description == "tmp":
             file.description = f"See dataset {provisional_id}"  # type: ignore
         file._feature_sets = feature_sets
+        storage = None
+    # init from directory or bucket
+    elif isinstance(data, (str, Path, UPath)):
+        file = None
+        files = None
+        upath = UPath(data)
+        if not upath.is_dir:
+            raise ValueError(f"Can only pass buckets or directories, not {data}")
+        upath_str = upath.as_posix().rstrip("/")
+        region = get_storage_region(upath_str)
+        storage_settings = StorageSettings(upath_str, region)
+        storage = register_storage(storage_settings)
+        hash = None
     # init files
     else:
         file = None
+        storage = None
         if hasattr(data, "__getitem__"):
             assert isinstance(data[0], File)  # type: ignore
             files = data
@@ -144,6 +164,7 @@ def __init__(
             reference=reference,
             reference_type=reference_type,
             file=file,
+            storage=storage,
             hash=hash,
             run=run,
             version=version,
@@ -276,8 +297,9 @@ def load(
     is_run_input: Optional[bool] = None,
     **kwargs,
 ) -> DataLike:
-    _track_run_input(self, is_run_input)
+    # cannot call _track_run_input here, see comment further down
     if self.file is not None:
+        _track_run_input(self, is_run_input)
         return self.file.load()
     else:
         all_files = self.files.all()
@@ -291,9 +313,14 @@ def load(
         objects = [file.load(is_run_input=False) for file in all_files]
         file_ids = [file.id for file in all_files]
         if isinstance(objects[0], pd.DataFrame):
-            return pd.concat(objects, join=join)
+            concat_object = pd.concat(objects, join=join)
         elif isinstance(objects[0], ad.AnnData):
-            return ad.concat(objects, join=join, label="file_id", keys=file_ids)
+            concat_object = ad.concat(
+                objects, join=join, label="file_id", keys=file_ids
+            )
+        # only call it here because there might be errors during concat
+        _track_run_input(self, is_run_input)
+        return concat_object
 
 
 # docstring handled through attach_func_to_class_method
@@ -308,7 +335,7 @@ def save(self, *args, **kwargs) -> None:
     if self.file is not None:
         self.file.save()
     # we don't need to save feature sets again
-    save_transform_run_feature_sets(self)
+    save_feature_sets(self)
     super(Dataset, self).save()
     if hasattr(self, "_files"):
         if self._files is not None and len(self._files) > 0:
@@ -316,6 +343,14 @@ def save(self, *args, **kwargs) -> None:
     save_feature_set_links(self)
 
 
+@property  # type: ignore
+@doc_args(Dataset.path.__doc__)
+def path(self) -> Union[Path, UPath]:
+    """{}"""
+    _track_run_input(self)
+    return self.storage.path
+
+
 METHOD_NAMES = [
     "__init__",
     "from_anndata",
@@ -337,3 +372,5 @@ if _TESTING:
 
 for name in METHOD_NAMES:
     attach_func_to_class_method(name, Dataset, globals())
+
+setattr(Dataset, "path", path)

lamindb/_file.py

@@ -47,7 +47,7 @@ from .dev._data import (
     add_transform_to_kwargs,
     get_run,
     save_feature_set_links,
-    save_transform_run_feature_sets,
+    save_feature_sets,
 )
 from .dev._priors import priors
 from .dev.storage.file import AUTO_KEY_PREFIX
@@ -85,9 +85,9 @@ def process_pathlike(
                 new_root_str = new_root.as_posix().rstrip("/")
                 region = get_storage_region(new_root_str)
                 storage_settings = StorageSettings(new_root_str, region)
-                register_storage(storage_settings)
+                storage_record = register_storage(storage_settings)
                 use_existing_storage_key = True
-                return storage_settings.record, use_existing_storage_key
+                return storage_record, use_existing_storage_key
             # if the filepath is local
             else:
                 use_existing_storage_key = False
@@ -833,7 +833,7 @@ def save(self, *args, **kwargs) -> None:
 
 
 def _save_skip_storage(file, *args, **kwargs) -> None:
-    save_transform_run_feature_sets(file)
+    save_feature_sets(file)
     super(File, file).save(*args, **kwargs)
     save_feature_set_links(file)
 

lamindb/_filter.py

@@ -1,13 +1,43 @@
 from typing import Type
+from uuid import UUID
 
+import dj_database_url
+from django.db import connections
+from lamindb_setup._init_instance import InstanceSettings
+from lamindb_setup._load_instance import get_owner_name_from_identifier
+from lamindb_setup.dev._hub_core import load_instance
 from lnschema_core import Registry
 
 from lamindb._query_set import QuerySet
 
 
-def filter(Registry: Type[Registry], **expressions) -> QuerySet:
+def add_db_connection(isettings: InstanceSettings, using: str):
+    db_config = dj_database_url.config(
+        default=isettings.db, conn_max_age=600, conn_health_checks=True
+    )
+    db_config["TIME_ZONE"] = "UTC"
+    db_config["OPTIONS"] = {}
+    db_config["AUTOCOMMIT"] = True
+
+    connections.settings[using] = db_config
+
+
+def filter(Registry: Type[Registry], using: str = None, **expressions) -> QuerySet:
     """See :meth:`~lamindb.dev.Registry.filter`."""
-    qs = QuerySet(model=Registry)
+    if using is not None:
+        owner, name = get_owner_name_from_identifier(using)
+        instance_result, storage_result = load_instance(owner=owner, name=name)
+        isettings = InstanceSettings(
+            owner=owner,
+            name=name,
+            storage_root=storage_result["root"],
+            storage_region=storage_result["region"],
+            db=instance_result["db"],
+            schema=instance_result["schema_str"],
+            id=UUID(instance_result["id"]),
+        )
+        add_db_connection(isettings, using)
+    qs = QuerySet(model=Registry, using=using)
     if len(expressions) > 0:
         return qs.filter(**expressions)
     else:

lamindb/_parents.py

@@ -78,7 +78,7 @@ def view_flow(data: Union[File, Dataset], with_children: bool = True) -> None:
         df_values += _get_all_child_runs(data)
     df_edges = _df_edges_from_runs(df_values)
 
-    data_label = _label_data_run_transform(data)
+    data_label = _record_label(data)
 
     def add_node(
         record: Union[Run, File, Dataset],
@@ -156,7 +156,7 @@ def _view_parents(
             else:
                 df_edges = df_edges_children
 
-    record_label = record.__getattribute__(field)
+    record_label = _record_label(record, field)
 
     u = graphviz.Digraph(
         record.id,
@@ -172,7 +172,7 @@ def _view_parents(
     )
     u.node(
         record.id,
-        label=_label_data_run_transform(record)
+        label=_record_label(record)
         if record.__class__.__name__ == "Transform"
         else _add_emoji(record, record_label),
         fillcolor=LAMIN_GREEN_LIGHTER,
@@ -237,22 +237,57 @@ def _df_edges_from_parents(
     df_edges["source_record"] = df_edges["source"].apply(lambda x: all.get(id=x))
     df_edges["target_record"] = df_edges["target"].apply(lambda x: all.get(id=x))
     if record.__class__.__name__ == "Transform":
-        df_edges["source_label"] = df_edges["source_record"].apply(
-            _label_data_run_transform
-        )
-        df_edges["target_label"] = df_edges["target_record"].apply(
-            _label_data_run_transform
-        )
+        df_edges["source_label"] = df_edges["source_record"].apply(_record_label)
+        df_edges["target_label"] = df_edges["target_record"].apply(_record_label)
     else:
         df_edges["source_label"] = df_edges["source_record"].apply(
-            lambda x: x.__getattribute__(field)
+            lambda x: _record_label(x, field)
         )
         df_edges["target_label"] = df_edges["target_record"].apply(
-            lambda x: x.__getattribute__(field)
+            lambda x: _record_label(x, field)
         )
     return df_edges
 
 
+def _record_label(record: Registry, field: Optional[str] = None):
+    if isinstance(record, File):
+        if record.description is None:
+            name = record.key
+        else:
+            name = record.description.replace("&", "&")
+
+        return (
+            rf'<📄 {name}<BR/><FONT COLOR="GREY" POINT-SIZE="10"'
+            rf' FACE="Monospace">id={record.id}<BR/>suffix={record.suffix}</FONT>>'
+        )
+    elif isinstance(record, Dataset):
+        name = record.name.replace("&", "&amp;")
+        return (
+            rf'<🍱 {name}<BR/><FONT COLOR="GREY" POINT-SIZE="10"'
+            rf' FACE="Monospace">id={record.id}<BR/>version={record.version}</FONT>>'
+        )
+    elif isinstance(record, Run):
+        name = f'{record.transform.name.replace("&", "&amp;")}'
+        return (
+            rf'<{TRANSFORM_EMOJIS.get(str(record.transform.type), "💫")} {name}<BR/><FONT COLOR="GREY" POINT-SIZE="10"'  # noqa
+            rf' FACE="Monospace">id={record.id}<BR/>type={record.transform.type},'
+            rf" user={record.created_by.name}<BR/>run_at={format_field_value(record.run_at)}</FONT>>"  # noqa
+        )
+    elif isinstance(record, Transform):
+        name = f'{record.name.replace("&", "&amp;")}'
+        return (
+            rf'<{TRANSFORM_EMOJIS.get(str(record.type), "💫")} {name}<BR/><FONT COLOR="GREY" POINT-SIZE="10"'  # noqa
+            rf' FACE="Monospace">id={record.id}<BR/>type={record.type},'
+            rf" user={record.created_by.name}<BR/>updated_at={format_field_value(record.updated_at)}</FONT>>"  # noqa
+        )
+    else:
+        name = record.__getattribute__(field)
+        return (
+            rf'<{name}<BR/><FONT COLOR="GREY" POINT-SIZE="10"'
+            rf' FACE="Monospace">id={record.id}</FONT>>'
+        )
+
+
 def _add_emoji(record: Registry, label: str):
     if record.__class__.__name__ == "Transform":
         emoji = TRANSFORM_EMOJIS.get(record.type, "💫")
@@ -321,39 +356,6 @@ def _get_all_child_runs(data: Union[File, Dataset]) -> List:
     return run_inputs_outputs
 
 
-def _label_data_run_transform(record: Union[File, Run, Transform]):
-    if isinstance(record, File):
-        if record.description is None:
-            name = record.key
-        else:
-            name = record.description.replace("&", "&amp;")
-
-        return (
-            rf'<📄 {name}<BR/><FONT COLOR="GREY" POINT-SIZE="10"'
-            rf' FACE="Monospace">id={record.id}<BR/>suffix={record.suffix}</FONT>>'
-        )
-    elif isinstance(record, Dataset):
-        name = record.name.replace("&", "&amp;")
-        return (
-            rf'<🍱 {name}<BR/><FONT COLOR="GREY" POINT-SIZE="10"'
-            rf' FACE="Monospace">id={record.id}<BR/>version={record.version}</FONT>>'
-        )
-    elif isinstance(record, Run):
-        name = f'{record.transform.name.replace("&", "&amp;")}'
-        return (
-            rf'<{TRANSFORM_EMOJIS.get(str(record.transform.type), "💫")} {name}<BR/><FONT COLOR="GREY" POINT-SIZE="10"'  # noqa
-            rf' FACE="Monospace">id={record.id}<BR/>type={record.transform.type},'
-            rf" user={record.created_by.name}<BR/>run_at={format_field_value(record.run_at)}</FONT>>"  # noqa
-        )
-    elif isinstance(record, Transform):
-        name = f'{record.name.replace("&", "&amp;")}'
-        return (
-            rf'<{TRANSFORM_EMOJIS.get(str(record.type), "💫")} {name}<BR/><FONT COLOR="GREY" POINT-SIZE="10"'  # noqa
-            rf' FACE="Monospace">id={record.id}<BR/>type={record.type},'
-            rf" user={record.created_by.name}<BR/>updated_at={format_field_value(record.updated_at)}</FONT>>"  # noqa
-        )
-
-
 def _df_edges_from_runs(df_values: List):
     import pandas as pd
 
@@ -363,8 +365,8 @@ def _df_edges_from_runs(df_values: List):
     df = df.drop_duplicates().dropna()
     df["source"] = [f"{i._meta.model_name}_{i.id}" for i in df["source_record"]]
     df["target"] = [f"{i._meta.model_name}_{i.id}" for i in df["target_record"]]
-    df["source_label"] = df["source_record"].apply(_label_data_run_transform)
-    df["target_label"] = df["target_record"].apply(_label_data_run_transform)
+    df["source_label"] = df["source_record"].apply(_record_label)
+    df["target_label"] = df["target_record"].apply(_record_label)
     return df
 
 

lamindb/_query_manager.py

@@ -1,6 +1,7 @@
 from typing import Optional
 
 from django.db import models
+from lamin_utils import logger
 
 from .dev._feature_manager import get_feature_set_by_slot
 
@@ -24,6 +25,16 @@ class QueryManager(models.Manager):
         >>> manager.df()
     """
 
+    def _track_run_input_manager(self):
+        if hasattr(self, "source_field_name") and hasattr(self, "target_field_name"):
+            if self.source_field_name == "dataset" and self.target_field_name == "file":
+                from lamindb.dev._data import WARNING_RUN_TRANSFORM, _track_run_input
+                from lamindb.dev._run_context import run_context
+
+                if run_context.run is None:
+                    logger.warning(WARNING_RUN_TRANSFORM)
+                _track_run_input(self.instance)
+
     def list(self, field: Optional[str] = None):
         """Populate a list with the results.
 
@@ -40,6 +51,7 @@ class QueryManager(models.Manager):
             >>> label.parents.list("name")
             ['ULabel1', 'ULabel2', 'ULabel3']
         """
+        self._track_run_input_manager()
         if field is None:
             return [item for item in self.all()]
         else:
@@ -52,6 +64,14 @@ class QueryManager(models.Manager):
         """
         return self.all().df(**kwargs)
 
+    def all(self):
+        """Return QuerySet of all.
+
+        For `**kwargs`, see :meth:`lamindb.dev.QuerySet.df`.
+        """
+        self._track_run_input_manager()
+        return self.all_base_class()
+
     def __getitem__(self, item: str):
         try:
             source_field_name = self.source_field_name
@@ -70,3 +90,10 @@ class QueryManager(models.Manager):
 setattr(models.Manager, "list", QueryManager.list)
 setattr(models.Manager, "df", QueryManager.df)
 setattr(models.Manager, "__getitem__", QueryManager.__getitem__)
+setattr(
+    models.Manager, "_track_run_input_manager", QueryManager._track_run_input_manager
+)
+# the two lines below would be easy if we could actually inherit; like this,
+# they're suboptimal
+setattr(models.Manager, "all_base_class", models.Manager.all)
+setattr(models.Manager, "all", QueryManager.all)

lamindb/_registry.py

@@ -1,6 +1,7 @@
 import builtins
 from typing import Iterable, List, NamedTuple, Optional, Union
 
+import lamindb_setup as ln_setup
 import pandas as pd
 from django.core.exceptions import FieldDoesNotExist
 from django.db.models import Manager, QuerySet
@@ -81,7 +82,12 @@ def __init__(orm: Registry, *args, **kwargs):
         validate_required_fields(orm, kwargs)
         from .dev._settings import settings
 
-        if settings.upon_create_search_names:
+        # do not search for names if an id is passed; this is important
+        # e.g. when synching ids from the notebook store to lamindb
+        has_consciously_provided_id = False
+        if "_has_consciously_provided_id" in kwargs:
+            has_consciously_provided_id = kwargs.pop("_has_consciously_provided_id")
+        if settings.upon_create_search_names and not has_consciously_provided_id:
             result = suggest_objects_with_same_name(orm, kwargs)
             if result == "object-with-same-name-exists":
                 if "version" in kwargs:
@@ -314,10 +320,59 @@ def _queryset(cls: Union[Registry, QuerySet, Manager]) -> QuerySet:
     return queryset
 
 
+def transfer_to_default_db(record: Registry, save: bool = False):
+    db = record._state.db
+    if db is not None and db != "default":
+        logger.info(f"saving from instance {db} to default instance: {record}")
+        from lamindb.dev._data import WARNING_RUN_TRANSFORM
+        from lamindb.dev._run_context import run_context
+
+        logger.hint("saving to default instance")
+        if (
+            hasattr(record, "created_by_id")
+            and record.created_by_id != ln_setup.settings.user.id
+        ):
+            logger.info(f"updating created_by_id with {ln_setup.settings.user.id}")
+            record.created_by_id = ln_setup.settings.user.id
+        if hasattr(record, "run_id"):
+            if run_context.run is not None:
+                logger.info("updating run & transform to current run & transform")
+                record.run_id = run_context.run.id
+            else:
+                logger.warning(WARNING_RUN_TRANSFORM)
+                record.run_id = None
+        if hasattr(record, "transform_id"):
+            if run_context.transform is not None:
+                record.transform_id = run_context.transform.id
+            else:
+                record.transform_id = None
+        if hasattr(record, "storage_id") and record.storage_id is not None:
+            record.storage.save()
+        record._state.db = "default"
+        if save:
+            record.save()
+
+
+# docstring handled through attach_func_to_class_method
+def save(self, *args, **kwargs) -> None:
+    db = self._state.db
+    transfer_to_default_db(self)
+    super(Registry, self).save(*args, **kwargs)
+    if db is not None and db != "default":
+        if hasattr(self, "labels"):
+            logger.info("transfer labels")
+            from copy import copy
+
+            self_on_db = copy(self)
+            self_on_db._state.db = db
+            self.labels.add_from(self_on_db)
+
+
 METHOD_NAMES = [
     "__init__",
     "search",
     "lookup",
+    "save",
     "from_values",
 ]
 
@@ -347,16 +402,5 @@ def __get_name_with_schema__(cls) -> str:
     return f"{schema_name}.{cls.__name__}"
 
 
-def select_backward(cls, **expressions):
-    logger.warning("select() is deprecated! please use: Registry.filter()")
-    return cls.filter(**expressions)
-
-
-@classmethod  # type: ignore
-def select(cls, **expressions):
-    return select_backward(cls, **expressions)
-
-
 setattr(Registry, "__get_schema_name__", __get_schema_name__)
 setattr(Registry, "__get_name_with_schema__", __get_name_with_schema__)
-setattr(Registry, "select", select)  # backward compat

lamindb/_storage.py

@@ -1,3 +1,4 @@
+from lamindb_setup.dev._docs import doc_args
 from lamindb_setup.dev.upath import UPath, create_path
 from lnschema_core import Storage
 
@@ -6,4 +7,12 @@ def root_as_path(self) -> UPath:
     return create_path(self.root)
 
 
+@property  # type: ignore
+@doc_args(Storage.path.__doc__)
+def path(self) -> UPath:
+    """{}"""
+    return create_path(self.root)
+
+
 setattr(Storage, "root_as_path", root_as_path)
+setattr(Storage, "path", path)

lamindb/_transform.py

@@ -44,8 +44,14 @@ def __init__(transform: Transform, *args, **kwargs):
         )
         if name is None:
             name = is_new_version_of.name
+
+    # this is only because the user-facing constructor allows passing an id
+    # most others don't
     if id is None:
+        has_consciously_provided_id = False
         id = new_id
+    else:
+        has_consciously_provided_id = True
     super(Transform, transform).__init__(
         id=id,
         name=name,
@@ -54,6 +60,7 @@ def __init__(transform: Transform, *args, **kwargs):
         version=version,
         initial_version_id=initial_version_id,
         reference=reference,
+        _has_consciously_provided_id=has_consciously_provided_id,
     )
 
 

lamindb/dev/_data.py

@@ -34,15 +34,16 @@ from ._priors import priors
 from ._run_context import run_context
 from .exceptions import ValidationError
 
+WARNING_RUN_TRANSFORM = (
+    "no run & transform get linked, consider passing a `run` or calling ln.track()"
+)
+
 
 def get_run(run: Optional[Run]) -> Optional[Run]:
     if run is None:
         run = run_context.run
-        if run is None:
-            logger.warning(
-                "no run & transform get linked, consider passing a `run` or calling"
-                " ln.track()"
-            )
+        if run is None and not settings.silence_file_run_transform_warning:
+            logger.warning(WARNING_RUN_TRANSFORM)
     return run
 
 
@@ -51,11 +52,7 @@ def add_transform_to_kwargs(kwargs: Dict[str, Any], run: Run):
         kwargs["transform"] = run.transform
 
 
-def save_transform_run_feature_sets(self: Union[File, Dataset]) -> None:
-    if self.transform is not None:
-        self.transform.save()
-    if self.run is not None:
-        self.run.save()
+def save_feature_sets(self: Union[File, Dataset]) -> None:
     if hasattr(self, "_feature_sets"):
         saved_feature_sets = {}
         for key, feature_set in self._feature_sets.items():

lamindb/dev/_label_manager.py

@@ -6,7 +6,7 @@ from lnschema_core.models import Data, Dataset, Feature, File, Registry
 from .._feature_set import dict_related_model_to_related_name
 from .._from_values import _print_values
 from .._query_set import QuerySet
-from .._registry import get_default_str_field
+from .._registry import get_default_str_field, transfer_to_default_db
 
 
 def get_labels_as_dict(self: Data):
@@ -14,7 +14,14 @@ def get_labels_as_dict(self: Data):
     for related_model, related_name in dict_related_model_to_related_name(
         self.__class__
     ).items():
-        if related_name in {"feature_sets", "files", "input_of", "datasets"}:
+        if related_name in {
+            "feature_sets",
+            "files",
+            "input_of",
+            "datasets",
+            "source_of",
+            "report_of",
+        }:
             continue
         labels[related_name] = (related_model, self.__getattribute__(related_name))
     return labels
@@ -110,9 +117,17 @@ class LabelManager:
                         labels = data.labels.get(
                             getattr(features_lookup, row["name"]), mute=True
                         )
+                        for label in labels:
+                            transfer_to_default_db(label, save=True)
                         self._host.labels.add(
                             labels, feature=getattr(features_lookup, row["name"])
                         )
         # for now, have this be duplicated, need to disentangle above
         for related_name, (_, labels) in get_labels_as_dict(data).items():
-            getattr(self._host, related_name).add(*labels.all())
+            labels_list = labels.list()
+            for label in labels_list:
+                transfer_to_default_db(label, save=True)
+            # this should not occur as file and dataset should have the same attributes
+            # but this might not be true for custom schema
+            if hasattr(self._host, related_name):
+                getattr(self._host, related_name).add(*labels_list)

lamindb/dev/_settings.py

@@ -56,6 +56,8 @@ class Settings:
 
     FAQ: :doc:`/faq/track-run-inputs`
     """
+    silence_file_run_transform_warning: bool = False
+    """Silence warning about missing run & transform during file creation."""
 
     @property
     def storage(self) -> Union[Path, UPath]:

{lamindb-0.54.4.dist-info → lamindb-0.55.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.54.4
+Version: 0.55.1
 Summary: Open-source data platform for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -8,9 +8,9 @@ Description-Content-Type: text/markdown
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
-Requires-Dist: lnschema_core==0.48.4
-Requires-Dist: lamindb_setup==0.53.2
-Requires-Dist: lamin_utils==0.11.3
+Requires-Dist: lnschema_core==0.50.0
+Requires-Dist: lamindb_setup==0.54.3
+Requires-Dist: lamin_utils==0.11.4
 Requires-Dist: rapidfuzz
 Requires-Dist: pyarrow
 Requires-Dist: typing_extensions!=4.6.0
@@ -34,11 +34,14 @@ Requires-Dist: coverage ; extra == "dev"
 Requires-Dist: pytest-cov ; extra == "dev"
 Requires-Dist: nbproject_test>=0.3.0 ; extra == "dev"
 Requires-Dist: faker-biology ; extra == "dev"
-Requires-Dist: erdiagram>=0.1.3 ; extra == "erdiagram"
-Requires-Dist: readfcs>=1.1.5 ; extra == "fcs"
+Requires-Dist: django-schema-graph ; extra == "erdiagram"
+Requires-Dist: readfcs>=1.1.7 ; extra == "fcs"
 Requires-Dist: fsspec[gs]==2023.9.0 ; extra == "gcp"
 Requires-Dist: nbproject==0.9.2 ; extra == "jupyter"
+Requires-Dist: nbstripout ; extra == "jupyter"
+Requires-Dist: nbconvert ; extra == "jupyter"
 Requires-Dist: psycopg2-binary ; extra == "postgres"
+Requires-Dist: lamin_vault ; extra == "postgres"
 Requires-Dist: zarr>=2.16.0 ; extra == "zarr"
 Project-URL: Home, https://github.com/laminlabs/lamindb
 Provides-Extra: aws

{lamindb-0.54.4.dist-info → lamindb-0.55.1.dist-info}/RECORD

@@ -1,30 +1,30 @@
-lamindb/__init__.py,sha256=dgz0a8-2DKOnshq03qxCrI9HH_ksnRxCz_WamVZ5gZ8,2911
-lamindb/_dataset.py,sha256=I6RiJ9HP1U-hi_2Xy9utDnWHw8L0N8aM-vB9_w4pRUE,12188
+lamindb/__init__.py,sha256=ZCNDGjrECrHymRvihw53TRk4QYluCJ1URlRRhKLb4vA,2945
+lamindb/_dataset.py,sha256=363qySdhwD9qO9op-fXRuTTflKmCdhGAOvf115gdjXo,13444
 lamindb/_delete.py,sha256=wiYmYnvIEHrDdmw1NiXyfCY9mBt-FI5XNFi5jyR_mkA,1968
 lamindb/_feature.py,sha256=5gsa7zsMVVtm1DID4dF3Vwo5llWyY1dH3Hg5hjaIrQk,5554
 lamindb/_feature_set.py,sha256=YyRLc76piJdh8X6B917cFBiwJTdIxvQo0n5k2-HUGro,9373
-lamindb/_file.py,sha256=yrjP-nAEJ8P8dC1mGpB7-LqYVbNdwz0KY68tFZgm4PU,35568
-lamindb/_filter.py,sha256=Rf5RSkglIhJhSdk3AIPfc83F5NIfZrdCZDpgcYNj5KY,351
+lamindb/_file.py,sha256=WotCwWGtVYLhyUARgJeInLgRnuKKRmx7E12IxS9Pe20,35548
+lamindb/_filter.py,sha256=06hUpqZTTvXPOfywR0oJ47dd281KvnejRpTvaCAIV74,1515
 lamindb/_from_values.py,sha256=OztkgQpyWy6CSrr5RcKwL1r_DTjCoYWyllR94LLd8qs,11595
-lamindb/_parents.py,sha256=-BMhDfOQ8ORC5jbh09u4WVh-1G4vm509Bv4cDYh54OE,12969
-lamindb/_query_manager.py,sha256=F3ACAjy6UF7Vvdf9rnHRCLx8XvZAgpvTQLCqWpCH6Xk,2592
+lamindb/_parents.py,sha256=GZ6eUlIDtcAh4dCjBE2CC7KkY62WkKNFP3MMLtBjW44,13048
+lamindb/_query_manager.py,sha256=MXueabWHqft7GWNkzmWbhfTqdk-0mKU7nWrhXG6wpYQ,3693
 lamindb/_query_set.py,sha256=Lf7vLvOsEfUWRQ3iImSj4eQPmUK1KCgeoKS_m66Lp7o,10279
-lamindb/_registry.py,sha256=HvgT1kDUbUOo513nhBlKexI_C3lH3DJEyig-FIyHq2A,11444
+lamindb/_registry.py,sha256=N-CzfRokoWLl1ive5aYr5mBnUQuddQ1GCjElbeUFNsY,13404
 lamindb/_run.py,sha256=659lqY32GW7F41rFUUo37OftUa38-p8yaV9Z0oF32CE,1120
 lamindb/_save.py,sha256=Lir8uDGS9XuYOaqogpT2EiDrWV_T_G9PbPvckbvUsoE,9474
-lamindb/_storage.py,sha256=vfd1yeGEalVVEJrZQBv57dlAZSHsY1kexRjPFqTxGNg,207
-lamindb/_transform.py,sha256=fWmlLBtBrplka_1SzFojdZ9EHCQ15j2yv4iB2jJAD4U,2235
+lamindb/_storage.py,sha256=HUdXGj4839C606gvxWXo0tDITbtbuyJKOgUPhagYPTI,415
+lamindb/_transform.py,sha256=O9JcOt0Khg9fF_1Rmid03XyxJms6OnxQp8rCAT3ZuDw,2499
 lamindb/_ulabel.py,sha256=lEAENh_dluNkBi8xKUH_CjJNMXldOm2liy6Rg3IH1pE,1900
 lamindb/_utils.py,sha256=LGdiW4k3GClLz65vKAVRkL6Tw-Gkx9DWAdez1jyA5bE,428
 lamindb/_validate.py,sha256=6FQIxE8ZK4HwrrmrmFSjeCFKc-mxaHej5fuBh_4y2Tk,13656
 lamindb/_view.py,sha256=bzx6e-Cif2CmDQkOu6jMrq_d5rsu6g7hhdaK_sYBv_Y,2150
 lamindb/dev/__init__.py,sha256=Ja96dxb0t7raGsCr8QxqCabyEzIxeVGlL_IgmhxdsB8,1010
-lamindb/dev/_data.py,sha256=hNhs6YsnjwC6JI09EuMfgpsF4GT6_xxBafp5iJ7-5cs,13614
+lamindb/dev/_data.py,sha256=1uOGPpa1OWIM1QcgpS_M_l35sDC8Fltk-fYf_5cKxUc,13540
 lamindb/dev/_feature_manager.py,sha256=z7uWuSO5fWl0pCKdCsc86rRrcKnAmtx2wAuOTiZn5XE,5869
-lamindb/dev/_label_manager.py,sha256=HCYZaUP1u4v8L0NemJjmm7kz5fA_Q-aISUJCMzqX4YU,4305
+lamindb/dev/_label_manager.py,sha256=yW0KgM1WMYe4NOpWNCO5ymw5CY2_UQl3t9tXZER4Gdc,4887
 lamindb/dev/_priors.py,sha256=eSZEEijmeFs3zcrU27r3T2sSGdsK-cvy7vl6ukDYaU8,785
 lamindb/dev/_run_context.py,sha256=XLlEknzOe0Cabi9fvzdQl5UeLoU02fAFkCdB7S2FLZM,19109
-lamindb/dev/_settings.py,sha256=AqPrB3ij3uIzlXDmH7rl8HsqIgaDUNReafwxjJDQphw,3267
+lamindb/dev/_settings.py,sha256=b0uq5eB6Dk7BQjBoK4a-9hy8y1DPsq0rErAQjZ4DibM,3398
 lamindb/dev/exceptions.py,sha256=PHk5lyBdJPrrEQcid3ItfdNzz3fgiQsUmsEDdz063F0,197
 lamindb/dev/fields.py,sha256=Yzdk2qF2ILNYktyswLTgHNrjCN8-McGsv5pqRdijIZ8,171
 lamindb/dev/hashing.py,sha256=IlNrHy-a9NqB0vfqiwIh4sjt40CvaiZIvfK6gMnkxDo,1381
@@ -39,12 +39,10 @@ lamindb/dev/storage/_backed_access.py,sha256=xbeXMFIXzDv7_fjR384omxdpSy3pF-T91rB
 lamindb/dev/storage/_zarr.py,sha256=7W1Jos1QOOF3f41uML_arQoDTNPZVpRyP2m3SLWaCAo,2766
 lamindb/dev/storage/file.py,sha256=CrR8dhfqEWPj7jhNj7zujQE1WT4gWGCP0qKPIIta0bg,7978
 lamindb/dev/storage/object.py,sha256=KGuOwwYuN2yCJxTXn9v0LanC0fjKwy_62P-WksHcf40,1140
-lamindb/schema/__init__.py,sha256=dtMCkfxDdYe6FHdqvySRZApyqgf_D1pkqTytyOX1dvw,619
-lamindb/schema/_core.py,sha256=nWR3X_rNd1AbWw3naMiBi8ppAEpqIDyEYqM54feRB_s,766
 lamindb/setup/__init__.py,sha256=8-0F2C4Glx23-b8-D_1CBGgRBM5PppVhazhoXZYOLsg,275
 lamindb/setup/dev/__init__.py,sha256=tBty426VGF2PGqqt2XuNU-WgvOrbOp1aZBDowjLuzgA,242
-lamindb-0.54.4.dist-info/entry_points.txt,sha256=MioM8vSpKwXxY3geNBwjo1wnwy1l15WjJYlI3lpKuZI,53
-lamindb-0.54.4.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-lamindb-0.54.4.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
-lamindb-0.54.4.dist-info/METADATA,sha256=fvMRsYeapWFpUlnv8wIyG07hT0chub9HQyx0EGuj240,2877
-lamindb-0.54.4.dist-info/RECORD,,
+lamindb-0.55.1.dist-info/entry_points.txt,sha256=MioM8vSpKwXxY3geNBwjo1wnwy1l15WjJYlI3lpKuZI,53
+lamindb-0.55.1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+lamindb-0.55.1.dist-info/WHEEL,sha256=EZbGkh7Ie4PoZfRQ8I0ZuP9VklN_TvcZ6DSE5Uar4z4,81
+lamindb-0.55.1.dist-info/METADATA,sha256=E5NRemxlnVm8u55H11hKjH1ezGF6ZYnd4z0zHvpC-Z4,3022
+lamindb-0.55.1.dist-info/RECORD,,

lamindb/schema/__init__.py

@@ -1,28 +0,0 @@
-"""Schema tools & overview.
-
-Guide: :doc:`/schemas`
-
-You can access mounted schema modules with domain-specific entities via
-available via `ln.schema.<module>.<entity>`.
-
-However, we recommend to import schema modules, e.g., like `import
-lnschema_bionty as bt`.
-
-.. autosummary::
-   :toctree: .
-
-   graph
-   view
-
-"""
-import importlib as _importlib
-
-from lamindb_setup import settings as _settings
-from lamindb_setup._init_instance import reload_schema_modules as _reload_schema_modules
-
-from .. import _INSTANCE_SETUP
-
-if _INSTANCE_SETUP:
-    _reload_schema_modules(_settings.instance)
-
-from ._core import graph, view

lamindb/schema/_core.py

@@ -1,37 +0,0 @@
-import lamindb_setup as setup
-
-
-def graph():
-    """Get diagram of entity relationships as `pydot.Dot` graph object.
-
-    It uses `erdiagram.create_schema_graph`
-    """
-    import erdiagram
-
-    metadata = get_db_metadata()
-    return erdiagram.create_schema_graph(
-        metadata=metadata,
-        show_datatypes=False,
-        show_indexes=False,
-        rankdir="TB",
-        concentrate=True,
-    )
-
-
-def view():
-    """View diagram of entity relationships.
-
-    It displays :func:`~lamindb.schema.graph`.
-    """
-    import erdiagram
-
-    erdiagram.view(graph())
-
-
-def get_db_metadata():
-    import sqlalchemy as sa
-
-    engine = sa.create_engine(setup.settings.instance.db)
-    metadata = sa.MetaData(bind=engine)
-    metadata.reflect()
-    return metadata