lamindb 0.73.2__py3-none-any.whl → 0.74.1__py3-none-any.whl

lamindb/__init__.py

@@ -14,6 +14,7 @@ Registries:
    ULabel
    Feature
    FeatureSet
+   Param
 
 Key functionality:
 
@@ -41,12 +42,13 @@ Modules & settings:
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.73.2"
+__version__ = "0.74.1"
 
 import os as _os
 
 import lamindb_setup as _lamindb_setup
-from lamindb_setup._check_setup import InstanceNotSetupError, _check_instance_setup
+from lamindb_setup._check_setup import InstanceNotSetupError as _InstanceNotSetupError
+from lamindb_setup._check_setup import _check_instance_setup
 from lamindb_setup._connect_instance import connect
 from lamindb_setup.core.upath import UPath
 
@@ -54,17 +56,18 @@ from . import setup
 
 
 def __getattr__(name):
-    raise InstanceNotSetupError()
+    raise _InstanceNotSetupError()
 
 
 if _check_instance_setup(from_lamindb=True):
-    del InstanceNotSetupError
+    del _InstanceNotSetupError
     del __getattr__  # delete so that imports work out
     from lnschema_core.models import (
         Artifact,
         Collection,
         Feature,
         FeatureSet,
+        Param,
         Run,
         Storage,
         Transform,
@@ -87,17 +90,14 @@ if _check_instance_setup(from_lamindb=True):
         _storage,
         _transform,
         _ulabel,
+        integrations,
     )
-
-    dev = core  # backward compat
-    from . import integrations
     from ._annotate import Annotate
     from ._finish import finish
     from ._save import save
     from ._view import view
     from .core._run_context import run_context as _run_context
     from .core._settings import settings
-    from .core._transform_settings import transform  # backward compat
 
     # schema modules
     if not _os.environ.get("LAMINDB_MULTI_INSTANCE") == "true":

lamindb/_artifact.py

@@ -23,7 +23,7 @@ from lamindb_setup.core.upath import (
     get_stat_dir_cloud,
     get_stat_file_cloud,
 )
-from lnschema_core.models import Artifact, FeatureManager, Run, Storage
+from lnschema_core.models import Artifact, FeatureManager, ParamManager, Run, Storage
 from lnschema_core.types import (
     VisibilityChoice,
 )
@@ -84,7 +84,7 @@ def process_pathlike(
             pass
     if isinstance(filepath, LocalPathClasses):
         filepath = filepath.resolve()
-    if check_path_is_child_of_root(filepath, default_storage.root_as_path()):
+    if check_path_is_child_of_root(filepath, default_storage.root):
         use_existing_storage_key = True
         return default_storage, use_existing_storage_key
     else:
@@ -190,7 +190,7 @@ def get_stat_or_artifact(
     using_key: str | None = None,
 ) -> tuple[int, str | None, str | None, int | None] | Artifact:
     n_objects = None
-    if settings.upon_file_create_skip_size_hash:
+    if settings.creation.artifact_skip_size_hash:
         return None, None, None, n_objects
     stat = path.stat()  # one network request
     if not isinstance(path, LocalPathClasses):
@@ -242,14 +242,14 @@ def get_stat_or_artifact(
             Artifact.objects.using(using_key).filter(hash=hash, visibility=None).all()
         )
     if len(result) > 0:
-        if settings.upon_artifact_create_if_hash_exists == "error":
+        if settings.creation.artifact_if_hash_exists == "error":
             msg = f"artifact with same hash exists: {result[0]}"
             hint = (
                 "💡 you can make this error a warning:\n"
-                "    ln.settings.upon_artifact_create_if_hash_exists"
+                "    ln.settings.creation.artifact_if_hash_exists"
             )
             raise FileExistsError(f"{msg}\n{hint}")
-        elif settings.upon_artifact_create_if_hash_exists == "warn_create_new":
+        elif settings.creation.artifact_if_hash_exists == "warn_create_new":
             logger.warning(
                 "creating new Artifact object despite existing artifact with same hash:"
                 f" {result[0]}"
@@ -272,28 +272,17 @@ def check_path_in_existing_storage(
 ) -> Storage | bool:
     for storage in Storage.objects.using(using_key).filter().all():
         # if path is part of storage, return it
-        if check_path_is_child_of_root(path, root=create_path(storage.root)):
+        if check_path_is_child_of_root(path, root=storage.root):
             return storage
     return False
 
 
 def check_path_is_child_of_root(path: Path | UPath, root: Path | UPath | None) -> bool:
-    path = UPath(str(path)) if not isinstance(path, UPath) else path
-    root = UPath(str(root)) if not isinstance(root, UPath) else root
-
-    # the following comparisons can fail if types aren't comparable
-    if not isinstance(path, LocalPathClasses) and not isinstance(
-        root, LocalPathClasses
-    ):
-        # the following tests equivalency of two UPath objects
-        # via string representations; otherwise
-        # S3Path('s3://lndb-storage/') and S3Path('s3://lamindb-ci/')
-        # test as equivalent
-        return list(path.parents)[-1].as_posix() == root.as_posix()
-    elif isinstance(path, LocalPathClasses) and isinstance(root, LocalPathClasses):
-        return root.resolve() in path.resolve().parents
-    else:
-        return False
+    # str is needed to eliminate UPath storage_options
+    # from the equality checks below
+    path = UPath(str(path))
+    root = UPath(str(root))
+    return root.resolve() in path.resolve().parents
 
 
 def get_relative_path_to_directory(
@@ -358,7 +347,7 @@ def get_artifact_kwargs_from_data(
     check_path_in_storage = False
     if use_existing_storage_key:
         inferred_key = get_relative_path_to_directory(
-            path=path, directory=storage.root_as_path()
+            path=path, directory=UPath(storage.root)
         ).as_posix()
         if key is None:
             key = inferred_key
@@ -387,7 +376,7 @@ def get_artifact_kwargs_from_data(
     )
 
     # do we use a virtual or an actual storage key?
-    key_is_virtual = settings.artifact_use_virtual_keys
+    key_is_virtual = settings.creation._artifact_use_virtual_keys
 
     # if the file is already in storage, independent of the default
     # we use an actual storage key
@@ -516,6 +505,7 @@ def update_attributes(data: HasFeatures, attributes: Mapping[str, str]):
 
 def __init__(artifact: Artifact, *args, **kwargs):
     artifact.features = FeatureManager(artifact)
+    artifact.params = ParamManager(artifact)
     # Below checks for the Django-internal call in from_db()
     # it'd be better if we could avoid this, but not being able to create a Artifact
     # from data with the default constructor renders the central class of the API
@@ -532,6 +522,7 @@ def __init__(artifact: Artifact, *args, **kwargs):
         raise ValueError("Only one non-keyword arg allowed: data")
 
     data: str | Path = kwargs.pop("data") if len(args) == 0 else args[0]
+    type: str = kwargs.pop("type") if "type" in kwargs else "dataset"
     key: str | None = kwargs.pop("key") if "key" in kwargs else None
     run: Run | None = kwargs.pop("run") if "run" in kwargs else None
     description: str | None = (
@@ -616,6 +607,7 @@ def __init__(artifact: Artifact, *args, **kwargs):
             "to _retain_ the old state by duplicating the entire folder, do _not_ pass `is_new_version_of`"
         )
 
+    kwargs["type"] = type
     kwargs["uid"] = provisional_uid
     kwargs["version"] = version
     kwargs["description"] = description
@@ -662,6 +654,7 @@ def from_df(
         version=version,
         is_new_version_of=is_new_version_of,
         accessor="DataFrame",
+        type="dataset",
         **kwargs,
     )
     return artifact
@@ -690,6 +683,7 @@ def from_anndata(
         version=version,
         is_new_version_of=is_new_version_of,
         accessor="AnnData",
+        type="dataset",
         **kwargs,
     )
     return artifact
@@ -716,6 +710,7 @@ def from_mudata(
         version=version,
         is_new_version_of=is_new_version_of,
         accessor="MuData",
+        type="dataset",
         **kwargs,
     )
     return artifact
@@ -752,7 +747,7 @@ def from_dir(
         else:
             # maintain the hierachy within an existing storage location
             folder_key_path = get_relative_path_to_directory(
-                folderpath, storage.root_as_path()
+                folderpath, UPath(storage.root)
             )
     else:
         folder_key_path = Path(key)

lamindb/_feature.py

@@ -27,12 +27,12 @@ FEATURE_TYPES = {
 }
 
 
-def convert_numpy_dtype_to_lamin_feature_type(dtype) -> str:
+def convert_numpy_dtype_to_lamin_feature_type(dtype, str_as_cat: bool = True) -> str:
     orig_type = dtype.name
     # strip precision qualifiers
     type = "".join(i for i in orig_type if not i.isdigit())
     if type == "object" or type == "str":
-        type = "cat"
+        type = "cat" if str_as_cat else "str"
     return type
 
 

lamindb/_finish.py

@@ -33,13 +33,19 @@ def get_seconds_since_modified(filepath) -> float:
     return datetime.now().timestamp() - filepath.stat().st_mtime
 
 
-def finish():
+def finish() -> None:
     """Mark a tracked run as finished.
 
     Saves source code and, for notebooks, a run report to your default storage location.
     """
-    if run_context.path is None:
+    if run_context.run is None:
         raise TrackNotCalled("Please run `ln.track()` before `ln.finish()`")
+    if run_context.path is None:
+        assert run_context.transform.type not in {"script", "notebook"}
+        run_context.run.finished_at = datetime.now(timezone.utc)
+        run_context.run.save()
+        # nothing else to do
+        return None
     if is_run_from_ipython:  # notebooks
         if (
             get_seconds_since_modified(run_context.path) > 3
@@ -138,7 +144,7 @@ def save_run_context_core(
                 prev_report = prev_transform.latest_report
             if prev_transform.source_code_id is not None:
                 prev_source = prev_transform.source_code
-    ln.settings.silence_file_run_transform_warning = True
+    ln.settings.creation.artifact_silence_missing_run_warning = True
 
     # track source code
     if transform.source_code_id is not None:

lamindb/_from_values.py

@@ -28,7 +28,7 @@ def get_or_create_records(
     Registry = field.field.model
     if create:
         return [Registry(**{field.field.name: value}) for value in iterable]
-    upon_create_search_names = settings.upon_create_search_names
+    creation_search_names = settings.creation.search_names
     feature: Feature = None
     organism = _get_organism_record(field, organism)
     kwargs: dict = {}
@@ -36,7 +36,7 @@ def get_or_create_records(
         kwargs["organism"] = organism
     if public_source is not None:
         kwargs["public_source"] = public_source
-    settings.upon_create_search_names = False
+    settings.creation.search_names = False
     try:
         iterable_idx = index_iterable(iterable)
 
@@ -88,7 +88,7 @@ def get_or_create_records(
                 logger.debug(f"added default feature '{feature_name}'")
         return records
     finally:
-        settings.upon_create_search_names = upon_create_search_names
+        settings.creation.search_names = creation_search_names
 
 
 def get_existing_records(
@@ -278,14 +278,26 @@ def index_iterable(iterable: Iterable) -> pd.Index:
 
 
 def _print_values(names: Iterable, n: int = 20, quotes: bool = True) -> str:
-    names = (name for name in names if name != "None")
-    unique_names = list(dict.fromkeys(names))[:n]
-    if quotes:
-        print_values = ", ".join(f"'{name}'" for name in unique_names)
+    if isinstance(names, dict):
+        items = {
+            f"{key}: {value}": None
+            for key, value in names.items()
+            if key != "None" and value != "None"
+        }
     else:
-        print_values = ", ".join(f"{name}" for name in unique_names)
-    if len(unique_names) > n:
+        # Use a dictionary instead of a list to have unique values and preserve order
+        items = {str(name): None for name in names if name != "None"}
+
+    unique_items = list(items.keys())
+
+    if quotes:
+        unique_items = [f"'{item}'" for item in unique_items]
+
+    print_values = ", ".join(unique_items[:n])
+
+    if len(unique_items) > n:
         print_values += ", ..."
+
     return print_values
 
 

lamindb/_query_manager.py

@@ -45,7 +45,7 @@ class QueryManager(models.Manager):
 
                 if (
                     run_context.run is None
-                    and not settings.silence_file_run_transform_warning
+                    and not settings.creation.artifact_silence_missing_run_warning
                 ):
                     logger.warning(WARNING_RUN_TRANSFORM)
                 _track_run_input(self.instance)

lamindb/_query_set.py

@@ -97,7 +97,9 @@ class QuerySet(models.QuerySet, CanValidate):
     """
 
     @doc_args(Registry.df.__doc__)
-    def df(self, include: str | list[str] | None = None) -> pd.DataFrame:
+    def df(
+        self, include: str | list[str] | None = None, join: str = "inner"
+    ) -> pd.DataFrame:
         """{}."""
         # re-order the columns
         exclude_field_names = ["created_at"]
@@ -173,7 +175,7 @@ class QuerySet(models.QuerySet, CanValidate):
                     link_groupby = link_df.groupby(left_side_link_model)[
                         values_expression
                     ].apply(list)
-                    df = pd.concat((link_groupby, df), axis=1, join="inner")
+                    df = pd.concat((link_groupby, df), axis=1, join=join)
                     df.rename(columns={values_expression: expression}, inplace=True)
                 else:
                     # the F() based implementation could also work for many-to-many,
@@ -185,7 +187,7 @@ class QuerySet(models.QuerySet, CanValidate):
                     )
                     df_anno = df_anno.set_index(pk_column_name)
                     df_anno.rename(columns={"expression": expression}, inplace=True)
-                    df = pd.concat((df_anno, df), axis=1, join="inner")
+                    df = pd.concat((df_anno, df), axis=1, join=join)
         return df
 
     def delete(self, *args, **kwargs):

lamindb/_registry.py

@@ -82,7 +82,7 @@ def __init__(orm: Registry, *args, **kwargs):
         has_consciously_provided_uid = False
         if "_has_consciously_provided_uid" in kwargs:
             has_consciously_provided_uid = kwargs.pop("_has_consciously_provided_uid")
-        if settings.upon_create_search_names and not has_consciously_provided_uid:
+        if settings.creation.search_names and not has_consciously_provided_uid:
             match = suggest_records_with_similar_names(orm, kwargs)
             if match:
                 if "version" in kwargs:
@@ -447,7 +447,7 @@ def transfer_to_default_db(
             if run_context.run is not None:
                 record.run_id = run_context.run.id
             else:
-                if not settings.silence_file_run_transform_warning:
+                if not settings.creation.artifact_silence_missing_run_warning:
                     logger.warning(WARNING_RUN_TRANSFORM)
                 record.run_id = None
         if hasattr(record, "transform_id") and record._meta.model_name != "run":

lamindb/_run.py

@@ -1,9 +1,10 @@
 from __future__ import annotations
 
-from lnschema_core.models import Run, Transform
+from lnschema_core.models import ParamManager, Run, Transform
 
 
 def __init__(run: Run, *args, **kwargs):
+    run.params = ParamManager(run)
     if len(args) == len(run._meta.concrete_fields):
         super(Run, run).__init__(*args, **kwargs)
         return None

lamindb/_storage.py

@@ -3,11 +3,6 @@ from lamindb_setup.core.upath import UPath, create_path
 from lnschema_core import Storage
 
 
-def root_as_path(self) -> UPath:
-    access_token = self._access_token if hasattr(self, "_access_token") else None
-    return create_path(self.root, access_token=access_token)
-
-
 @property  # type: ignore
 @doc_args(Storage.path.__doc__)
 def path(self) -> UPath:
@@ -16,5 +11,4 @@ def path(self) -> UPath:
     return create_path(self.root, access_token=access_token)
 
 
-Storage.root_as_path = root_as_path
 Storage.path = path

lamindb/core/__init__.py

@@ -10,7 +10,9 @@ Registries:
    QueryManager
    RecordsList
    HasFeatures
+   HasParams
    FeatureManager
+   ParamManager
    LabelManager
    IsVersioned
    CanValidate
@@ -30,13 +32,12 @@ Annotators:
    MuDataAnnotator
    AnnotateLookup
 
-Classes:
+Other:
 
 .. autosummary::
    :toctree: .
 
    Settings
-   TransformSettings
    MappedCollection
    run_context
 
@@ -49,6 +50,7 @@ Modules:
    storage
    types
    exceptions
+   subsettings
 
 """
 
@@ -56,6 +58,7 @@ from lamin_utils._inspect import InspectResult
 from lnschema_core.models import (
     CanValidate,
     HasFeatures,
+    HasParams,
     HasParents,
     IsVersioned,
     Registry,
@@ -71,11 +74,10 @@ from lamindb._annotate import (
 )
 from lamindb._query_manager import QueryManager
 from lamindb._query_set import QuerySet, RecordsList
-from lamindb.core._feature_manager import FeatureManager
+from lamindb.core._feature_manager import FeatureManager, ParamManager
 from lamindb.core._label_manager import LabelManager
 
-from . import _data, datasets, exceptions, fields, types
+from . import _data, datasets, exceptions, fields, subsettings, types
 from ._mapped_collection import MappedCollection
 from ._run_context import run_context
 from ._settings import Settings
-from ._transform_settings import TransformSettings

lamindb/core/_data.py

@@ -24,13 +24,12 @@ from lamindb._registry import get_default_str_field
 from lamindb.core._settings import settings
 
 from ._feature_manager import (
-    FeatureManager,
     get_feature_set_links,
     get_host_id_field,
     get_label_links,
     print_features,
 )
-from ._label_manager import LabelManager, print_labels
+from ._label_manager import print_labels
 from ._run_context import run_context
 from .exceptions import ValidationError
 from .schema import (
@@ -47,7 +46,7 @@ WARNING_RUN_TRANSFORM = "no run & transform get linked, consider calling ln.trac
 def get_run(run: Run | None) -> Run | None:
     if run is None:
         run = run_context.run
-        if run is None and not settings.silence_file_run_transform_warning:
+        if run is None and not settings.creation.artifact_silence_missing_run_warning:
             logger.warning(WARNING_RUN_TRANSFORM)
     # suppress run by passing False
     elif not run:
@@ -159,7 +158,11 @@ def describe(self: HasFeatures, print_types: bool = False):
         msg += f"  {colors.italic('Provenance')}\n"
         msg += prov_msg
     msg += print_labels(self, print_types=print_types)
-    msg += print_features(self, print_types=print_types)  # type: ignore
+    msg += print_features(  # type: ignore
+        self,
+        print_types=print_types,
+        print_params=hasattr(self, "type") and self.type == "model",
+    )
     logger.print(msg)