lamindb 0.77.2__py3-none-any.whl → 1.0rc1__py3-none-any.whl

lamindb/_collection.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from collections import defaultdict
 from typing import (
     TYPE_CHECKING,
@@ -13,28 +14,27 @@ import pandas as pd
 from lamin_utils import logger
 from lamindb_setup.core._docs import doc_args
 from lamindb_setup.core.hashing import hash_set
-from lnschema_core.models import (
+
+from lamindb.models import (
     Collection,
     CollectionArtifact,
-    FeatureSet,
+    Schema,
 )
-from lnschema_core.types import VisibilityChoice
 
-from . import Artifact, Run
 from ._parents import view_lineage
 from ._record import init_self_from_db, update_attributes
 from ._utils import attach_func_to_class_method
 from .core._data import (
     _track_run_input,
-    add_transform_to_kwargs,
     describe,
     get_run,
-    save_feature_set_links,
-    save_feature_sets,
+    save_schema_links,
+    save_staged__schemas_m2m,
 )
 from .core._mapped_collection import MappedCollection
 from .core._settings import settings
 from .core.versioning import process_revises
+from .models import Artifact, Run
 
 if TYPE_CHECKING:
     from collections.abc import Iterable
@@ -50,31 +50,31 @@ class CollectionFeatureManager:
     def __init__(self, collection: Collection):
         self._collection = collection
 
-    def get_feature_sets_union(self) -> dict[str, FeatureSet]:
-        links_feature_set_artifact = Artifact.feature_sets.through.objects.filter(
+    def _get_staged__schemas_m2m_union(self) -> dict[str, Schema]:
+        links_schema_artifact = Artifact._schemas_m2m.through.objects.filter(
             artifact_id__in=self._collection.artifacts.values_list("id", flat=True)
         )
-        feature_sets_by_slots = defaultdict(list)
-        for link in links_feature_set_artifact:
-            feature_sets_by_slots[link.slot].append(link.featureset_id)
-        feature_sets_union = {}
-        for slot, feature_set_ids_slot in feature_sets_by_slots.items():
-            feature_set_1 = FeatureSet.get(id=feature_set_ids_slot[0])
-            related_name = feature_set_1._get_related_name()
-            features_registry = getattr(FeatureSet, related_name).field.model
+        _schemas_m2m_by_slots = defaultdict(list)
+        for link in links_schema_artifact:
+            _schemas_m2m_by_slots[link.slot].append(link.schema_id)
+        _schemas_m2m_union = {}
+        for slot, schema_ids_slot in _schemas_m2m_by_slots.items():
+            schema_1 = Schema.get(id=schema_ids_slot[0])
+            related_name = schema_1._get_related_name()
+            features_registry = getattr(Schema, related_name).field.model
             # this way of writing the __in statement turned out to be the fastest
             # evaluated on a link table with 16M entries connecting 500 feature sets with
             # 60k genes
             feature_ids = (
-                features_registry.feature_sets.through.objects.filter(
-                    featureset_id__in=feature_set_ids_slot
+                features_registry.schemas.through.objects.filter(
+                    schema_id__in=schema_ids_slot
                 )
                 .values(f"{features_registry.__name__.lower()}_id")
                 .distinct()
             )
             features = features_registry.filter(id__in=feature_ids)
-            feature_sets_union[slot] = FeatureSet(features, dtype=feature_set_1.dtype)
-        return feature_sets_union
+            _schemas_m2m_union[slot] = Schema(features, dtype=schema_1.dtype)
+        return _schemas_m2m_union
 
 
 def __init__(
@@ -95,7 +95,7 @@ def __init__(
     meta_artifact: Artifact | None = (
         kwargs.pop("meta_artifact") if "meta_artifact" in kwargs else None
     )
-    name: str | None = kwargs.pop("name") if "name" in kwargs else None
+    key: str | None = kwargs.pop("key") if "key" in kwargs else None
     description: str | None = (
         kwargs.pop("description") if "description" in kwargs else None
     )
@@ -106,27 +106,29 @@ def __init__(
     run: Run | None = kwargs.pop("run") if "run" in kwargs else None
     revises: Collection | None = kwargs.pop("revises") if "revises" in kwargs else None
     version: str | None = kwargs.pop("version") if "version" in kwargs else None
-    visibility: int | None = (
-        kwargs.pop("visibility")
-        if "visibility" in kwargs
-        else VisibilityChoice.default.value
+    _branch_code: int | None = (
+        kwargs.pop("_branch_code") if "_branch_code" in kwargs else 1
     )
-    if "is_new_version_of" in kwargs:
-        logger.warning("`is_new_version_of` will be removed soon, please use `revises`")
-        revises = kwargs.pop("is_new_version_of")
+    if "name" in kwargs:
+        key = kwargs.pop("name")
+        warnings.warn(
+            f"argument `name` will be removed, please pass {key} to `key` instead",
+            FutureWarning,
+            stacklevel=2,
+        )
     if not len(kwargs) == 0:
         raise ValueError(
-            f"Only artifacts, name, run, description, reference, reference_type, visibility can be passed, you passed: {kwargs}"
+            f"Only artifacts, key, run, description, reference, reference_type can be passed, you passed: {kwargs}"
         )
-    provisional_uid, version, name, revises = process_revises(
-        revises, version, name, Collection
+    provisional_uid, version, key, description, revises = process_revises(
+        revises, version, key, description, Collection
     )
     run = get_run(run)
     if isinstance(artifacts, Artifact):
         artifacts = [artifacts]
     else:
         if not hasattr(artifacts, "__getitem__"):
-            raise ValueError("Artifact or List[Artifact] is allowed.")
+            raise ValueError("Artifact or list[Artifact] is allowed.")
         assert isinstance(artifacts[0], Artifact)  # type: ignore  # noqa: S101
     hash = from_artifacts(artifacts)  # type: ignore
     if meta_artifact is not None:
@@ -144,7 +146,7 @@ def __init__(
         existing_collection = None
     if existing_collection is not None:
         logger.warning(
-            f"returning existing collection with same hash: {existing_collection}"
+            f"returning existing collection with same hash: {existing_collection}; if you intended to query to track this collection as an input, use: ln.Collection.get()"
         )
         # update the run of the existing collection
         if run is not None:
@@ -157,18 +159,16 @@ def __init__(
                 )
             # update the run of the collection with the latest run
             existing_collection.run = run
-            existing_collection.transform = run.transform
         init_self_from_db(collection, existing_collection)
-        update_attributes(collection, {"description": description, "name": name})
+        update_attributes(collection, {"description": description, "key": key})
     else:
         kwargs = {}
-        add_transform_to_kwargs(kwargs, run)
         search_names_setting = settings.creation.search_names
-        if revises is not None and name == revises.name:
+        if revises is not None and key == revises.key:
             settings.creation.search_names = False
         super(Collection, collection).__init__(
             uid=provisional_uid,
-            name=name,
+            key=key,
             description=description,
             reference=reference,
             reference_type=reference_type,
@@ -176,7 +176,7 @@ def __init__(
             hash=hash,
             run=run,
             version=version,
-            visibility=visibility,
+            _branch_code=_branch_code,
             revises=revises,
             **kwargs,
         )
@@ -224,7 +224,7 @@ def mapped(
     layers_keys: str | list[str] | None = None,
     obs_keys: str | list[str] | None = None,
     obsm_keys: str | list[str] | None = None,
-    obs_filter: tuple[str, str | tuple[str, ...]] | None = None,
+    obs_filter: dict[str, str | tuple[str, ...]] | None = None,
     join: Literal["inner", "outer"] | None = "inner",
     encode_labels: bool | list[str] = True,
     unknown_label: str | dict[str, str] | None = None,
@@ -306,12 +306,14 @@ def load(
 
 # docstring handled through attach_func_to_class_method
 def delete(self, permanent: bool | None = None) -> None:
-    # change visibility to trash
-    trash_visibility = VisibilityChoice.trash.value
-    if self.visibility > trash_visibility and permanent is not True:
-        self.visibility = trash_visibility
+    # change _branch_code to trash
+    trash__branch_code = -1
+    if self._branch_code > trash__branch_code and permanent is not True:
+        self._branch_code = trash__branch_code
         self.save()
-        logger.warning(f"moved collection to trash (visibility = {trash_visibility})")
+        logger.warning(
+            f"moved collection to trash (_branch_code = {trash__branch_code})"
+        )
         return
 
     # permanent delete
@@ -333,7 +335,7 @@ def save(self, using: str | None = None) -> Collection:
     if self.meta_artifact is not None:
         self.meta_artifact.save()
     # we don't need to save feature sets again
-    save_feature_sets(self)
+    save_staged__schemas_m2m(self)
     super(Collection, self).save()
     # we don't allow updating the collection of artifacts
     # if users want to update the set of artifacts, they
@@ -348,7 +350,7 @@ def save(self, using: str | None = None) -> Collection:
         # merely using .artifacts.set(*...) doesn't achieve this
         # we need ignore_conflicts=True so that this won't error if links already exist
         CollectionArtifact.objects.bulk_create(links, ignore_conflicts=True)
-    save_feature_set_links(self)
+    save_schema_links(self)
     if using is not None:
         logger.warning("using argument is ignored")
     return self
@@ -356,7 +358,7 @@ def save(self, using: str | None = None) -> Collection:
 
 # docstring handled through attach_func_to_class_method
 def restore(self) -> None:
-    self.visibility = VisibilityChoice.default.value
+    self._branch_code = 1
     self.save()
 
 

lamindb/_feature.py

@@ -1,28 +1,28 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, Literal, get_args
+from typing import TYPE_CHECKING, Any, get_args
 
 import lamindb_setup as ln_setup
 import pandas as pd
 from lamin_utils import logger
 from lamindb_setup.core._docs import doc_args
-from lnschema_core.models import Artifact, Feature, Record
-from lnschema_core.types import FeatureDtype
 from pandas.api.types import CategoricalDtype, is_string_dtype
 
+from lamindb.base.types import FeatureDtype
 from lamindb.core.exceptions import ValidationError
+from lamindb.models import Artifact, Feature, Record
 
 from ._query_set import RecordList
 from ._utils import attach_func_to_class_method
-from .core._settings import settings
-from .core.schema import dict_schema_name_to_model_name
+from .core.relations import dict_module_name_to_model_name
 
 if TYPE_CHECKING:
     from collections.abc import Iterable
 
-    from lnschema_core.types import FieldAttr
     from pandas.core.dtypes.base import ExtensionDtype
 
+    from lamindb.base.types import FieldAttr
+
 
 FEATURE_DTYPES = set(get_args(FeatureDtype))
 
@@ -38,9 +38,9 @@ def get_dtype_str_from_dtype(dtype: Any) -> str:
             raise ValueError(error_message)
         registries_str = ""
         for registry in dtype:
-            if not hasattr(registry, "__get_name_with_schema__"):
+            if not hasattr(registry, "__get_name_with_module__"):
                 raise ValueError(error_message)
-            registries_str += registry.__get_name_with_schema__() + "|"
+            registries_str += registry.__get_name_with_module__() + "|"
         dtype_str = f'cat[{registries_str.rstrip("|")}]'
     return dtype_str
 
@@ -89,7 +89,7 @@ def __init__(self, *args, **kwargs):
                 if registries_str != "":
                     registry_str_list = registries_str.split("|")
                     for registry_str in registry_str_list:
-                        if registry_str not in dict_schema_name_to_model_name(Artifact):
+                        if registry_str not in dict_module_name_to_model_name(Artifact):
                             raise ValueError(
                                 f"'{registry_str}' is an invalid dtype, pass, e.g. `[ln.ULabel, bt.CellType]` or similar"
                             )

lamindb/_finish.py

@@ -9,17 +9,14 @@ from lamin_utils import logger
 from lamindb_setup.core.hashing import hash_file
 
 from lamindb.core.exceptions import NotebookNotSaved
+from lamindb.models import Artifact, Run, Transform
 
 if TYPE_CHECKING:
     from pathlib import Path
 
-    from lnschema_core import Run, Transform
 
-    from ._query_set import QuerySet
-
-
-def get_r_save_notebook_message() -> str:
-    return f"Please save the notebook in RStudio (shortcut `{get_shortcut()}`) within 2 sec before calling `db$finish()`"
+def get_save_notebook_message() -> str:
+    return f"Please save the notebook in your editor (shortcut `{get_shortcut()}`) within 2 sec before calling `finish()`"
 
 
 def get_shortcut() -> str:
@@ -32,13 +29,29 @@ def get_seconds_since_modified(filepath) -> float:
     return datetime.now().timestamp() - filepath.stat().st_mtime
 
 
+def save_run_logs(run: Run, save_run: bool = False) -> None:
+    logs_path = ln_setup.settings.cache_dir / f"run_logs_{run.uid}.txt"
+    if logs_path.exists():
+        if run.report is not None:
+            logger.important("overwriting run.report")
+        artifact = Artifact(
+            logs_path,
+            description=f"log streams of run {run.uid}",
+            _branch_code=0,
+            run=False,
+        )
+        artifact.save(upload=True, print_progress=False)
+        run.report = artifact
+        if save_run:  # defaults to false because is slow
+            run.save()
+
+
 # this is from the get_title function in nbproject
 # should be moved into lamindb sooner or later
 def prepare_notebook(
     nb,
     strip_title: bool = False,
 ) -> str | None:
-    """Strip title from the notebook if requested."""
     title_found = False
     for cell in nb.cells:
         cell.metadata.clear()  # strip cell metadata
@@ -85,8 +98,8 @@ def notebook_to_report(notebook_path: Path, output_path: Path) -> None:
 
 
 def notebook_to_script(
-    transform: Transform, notebook_path: Path, script_path: Path
-) -> None:
+    transform: Transform, notebook_path: Path, script_path: Path | None = None
+) -> None | str:
     import jupytext
 
     notebook = jupytext.read(notebook_path)
@@ -94,8 +107,11 @@ def notebook_to_script(
     # remove global metadata header
     py_content = re.sub(r"^# ---\n.*?# ---\n\n", "", py_content, flags=re.DOTALL)
     # replace title
-    py_content = py_content.replace(f"# # {transform.name}", "# # transform.name")
-    script_path.write_text(py_content)
+    py_content = py_content.replace(f"# # {transform.description}", "#")
+    if script_path is None:
+        return py_content
+    else:
+        script_path.write_text(py_content)
 
 
 # removes NotebookNotSaved error message from notebook html
@@ -114,7 +130,7 @@ def clean_r_notebook_html(file_path: Path) -> tuple[str | None, Path]:
         cleaned_content = re.sub(pattern_title, "", cleaned_content)
         cleaned_content = re.sub(pattern_h1, "", cleaned_content)
     cleaned_content = cleaned_content.replace(
-        f"NotebookNotSaved: {get_r_save_notebook_message()}", ""
+        f"NotebookNotSaved: {get_save_notebook_message()}", ""
     )
     cleaned_path = file_path.parent / (f"{file_path.stem}.cleaned{file_path.suffix}")
     cleaned_path.write_text(cleaned_content)
@@ -123,20 +139,19 @@ def clean_r_notebook_html(file_path: Path) -> tuple[str | None, Path]:
 
 def save_context_core(
     *,
-    run: Run,
+    run: Run | None,
     transform: Transform,
     filepath: Path,
     finished_at: bool = False,
     ignore_non_consecutive: bool | None = None,
     from_cli: bool = False,
 ) -> str | None:
-    from lnschema_core.models import (
+    import lamindb as ln
+    from lamindb.models import (
         format_field_value,  # needs to come after lamindb was imported because of CLI use
     )
 
-    import lamindb as ln
-
-    from .core._context import context, is_run_from_ipython
+    from .core._context import context
 
     ln.settings.verbosity = "success"
 
@@ -149,7 +164,7 @@ def save_context_core(
     # for notebooks, we need more work
     if is_ipynb:
         try:
-            import jupytext
+            import jupytext  # noqa: F401
             from nbproject.dev import (
                 check_consecutiveness,
                 read_notebook,
@@ -189,23 +204,19 @@ def save_context_core(
             logger.warning(
                 f"no {filepath.with_suffix('.nb.html')} found, save your manually rendered .html report via the CLI: lamin save {filepath}"
             )
+    if report_path is not None and not from_cli:
+        if get_seconds_since_modified(report_path) > 2 and not ln_setup._TESTING:
+            # this can happen when auto-knitting an html with RStudio
+            raise NotebookNotSaved(get_save_notebook_message())
     ln.settings.creation.artifact_silence_missing_run_warning = True
     # track source code
     hash, _ = hash_file(source_code_path)  # ignore hash_type for now
-    if (
-        transform._source_code_artifact_id is not None
-        or transform.hash is not None  # .hash is equivalent to .transform
-    ):
+    if transform.hash is not None:
         # check if the hash of the transform source code matches
         # (for scripts, we already run the same logic in track() - we can deduplicate the call at some point)
-        ref_hash = (
-            transform.hash
-            if transform.hash is not None
-            else transform._source_code_artifact.hash
-        )
-        if hash != ref_hash:
+        if hash != transform.hash:
             response = input(
-                f"You are about to overwrite existing source code (hash '{ref_hash}') for Transform('{transform.uid}')."
+                f"You are about to overwrite existing source code (hash '{transform.hash}') for Transform('{transform.uid}')."
                 f" Proceed? (y/n)"
             )
             if response == "y":
@@ -221,75 +232,77 @@ def save_context_core(
         transform.hash = hash
 
     # track environment
-    env_path = ln_setup.settings.cache_dir / f"run_env_pip_{run.uid}.txt"
-    if env_path.exists():
-        overwrite_env = True
-        if run.environment_id is not None and from_cli:
-            logger.important("run.environment is already saved")
-            overwrite_env = False
-        if overwrite_env:
-            hash, _ = hash_file(env_path)
-            artifact = ln.Artifact.filter(hash=hash, visibility=0).one_or_none()
-            new_env_artifact = artifact is None
-            if new_env_artifact:
-                artifact = ln.Artifact(
-                    env_path,
-                    description="requirements.txt",
-                    visibility=0,
-                    run=False,
-                )
-                artifact.save(upload=True, print_progress=False)
-            run.environment = artifact
-            if new_env_artifact:
-                logger.debug(f"saved run.environment: {run.environment}")
+    if run is not None:
+        env_path = ln_setup.settings.cache_dir / f"run_env_pip_{run.uid}.txt"
+        if env_path.exists():
+            overwrite_env = True
+            if run.environment_id is not None and from_cli:
+                logger.important("run.environment is already saved, ignoring")
+                overwrite_env = False
+            if overwrite_env:
+                hash, _ = hash_file(env_path)
+                artifact = ln.Artifact.filter(hash=hash, _branch_code=0).one_or_none()
+                new_env_artifact = artifact is None
+                if new_env_artifact:
+                    artifact = ln.Artifact(
+                        env_path,
+                        description="requirements.txt",
+                        _branch_code=0,
+                        run=False,
+                    )
+                    artifact.save(upload=True, print_progress=False)
+                run.environment = artifact
+                if new_env_artifact:
+                    logger.debug(f"saved run.environment: {run.environment}")
 
     # set finished_at
-    if finished_at:
+    if finished_at and run is not None:
         run.finished_at = datetime.now(timezone.utc)
 
+    # track logs
+    if run is not None and not from_cli and not is_ipynb and not is_r_notebook:
+        save_run_logs(run)
+
     # track report and set is_consecutive
-    if report_path is not None:
-        if not from_cli:
-            if get_seconds_since_modified(report_path) > 2 and not ln_setup._TESTING:
-                # this can happen when auto-knitting an html with RStudio
-                raise NotebookNotSaved(get_r_save_notebook_message())
-        if is_r_notebook:
-            title_text, report_path = clean_r_notebook_html(report_path)
-            if title_text is not None:
-                transform.name = title_text
-        if run.report_id is not None:
-            hash, _ = hash_file(report_path)  # ignore hash_type for now
-            if hash != run.report.hash:
-                response = input(
-                    f"You are about to overwrite an existing report (hash '{run.report.hash}') for Run('{run.uid}'). Proceed? (y/n)"
-                )
-                if response == "y":
-                    run.report.replace(report_path)
-                    run.report.save(upload=True, print_progress=False)
+    if run is not None:
+        if report_path is not None:
+            if is_r_notebook:
+                title_text, report_path = clean_r_notebook_html(report_path)
+                if title_text is not None:
+                    transform.description = title_text
+            if run.report_id is not None:
+                hash, _ = hash_file(report_path)  # ignore hash_type for now
+                if hash != run.report.hash:
+                    response = input(
+                        f"You are about to overwrite an existing report (hash '{run.report.hash}') for Run('{run.uid}'). Proceed? (y/n)"
+                    )
+                    if response == "y":
+                        run.report.replace(report_path)
+                        run.report.save(upload=True, print_progress=False)
+                    else:
+                        logger.important("keeping old report")
                 else:
-                    logger.important("keeping old report")
+                    logger.important("report is already saved")
             else:
-                logger.important("report is already saved")
-        else:
-            report_file = ln.Artifact(
-                report_path,
-                description=f"Report of run {run.uid}",
-                visibility=0,  # hidden file
-                run=False,
+                report_file = ln.Artifact(
+                    report_path,
+                    description=f"Report of run {run.uid}",
+                    _branch_code=0,  # hidden file
+                    run=False,
+                )
+                report_file.save(upload=True, print_progress=False)
+                run.report = report_file
+            logger.debug(
+                f"saved transform.latest_run.report: {transform.latest_run.report}"
             )
-            report_file.save(upload=True, print_progress=False)
-            run.report = report_file
-        logger.debug(
-            f"saved transform.latest_run.report: {transform.latest_run.report}"
-        )
-    run.is_consecutive = is_consecutive
+        run.is_consecutive = is_consecutive
 
-    # save both run & transform records if we arrive here
-    run.save()
+        # save both run & transform records if we arrive here
+        run.save()
     transform.save()
 
     # finalize
-    if not from_cli:
+    if not from_cli and run is not None:
         run_time = run.finished_at - run.started_at
         days = run_time.days
         seconds = run_time.seconds