lamindb 0.73.2__tar.gz → 0.74.0__tar.gz

{lamindb-0.73.2 → lamindb-0.74.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lamindb
-Version: 0.73.2
+Version: 0.74.0
 Summary: A data framework for biology.
 Author-email: Lamin Labs <open-source@lamin.ai>
 Requires-Python: >=3.8
@@ -9,8 +9,8 @@ Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Dist: lnschema_core==0.69.0
-Requires-Dist: lamindb_setup==0.73.1
+Requires-Dist: lnschema_core==0.70.0
+Requires-Dist: lamindb_setup==0.73.2
 Requires-Dist: lamin_utils==0.13.2
 Requires-Dist: lamin_cli==0.14.0
 Requires-Dist: rapidfuzz

{lamindb-0.73.2 → lamindb-0.74.0}/docs/changelog.md

@@ -12,6 +12,18 @@ LaminDB implements "migration-based versioning". When upgrading your LaminDB ins
 .. role:: small
 ```
 
+## 0.74
+
+### 0.74.0 {small}`2024-06-19`
+
+You can now distinguish model-like and dataset-like artifacts via a `type` field in the `Artifact` registry.
+
+Leverage `artifact.params.add_values()` to annotate model-like artifacts like you leverage `artifact.features.add_values()` to annotate dataset-like artifacts.
+
+- 🏗️ Add `type` field to `Artifact`, allow linking model-like artifacts against params, validate params akin to validating features, enable features-based annotation with non-ulabels [PR](https://github.com/laminlabs/lamindb/pull/1690) [@falexwolf](https://github.com/falexwolf)
+- 🚸 Support dict in add_values [PR](https://github.com/laminlabs/lamindb/pull/1705) [@Zethson](https://github.com/Zethson)
+- ♻️ Refactor after upath upgrade [PR](https://github.com/laminlabs/lamindb/pull/1699) [PR](https://github.com/laminlabs/lamindb/pull/1700) [@Koncopd](https://github.com/Koncopd)
+
 ## 0.73
 
 ### 0.73.2 {small}`2024-06-13`

{lamindb-0.73.2 → lamindb-0.74.0}/docs/faq/setup.ipynb

@@ -54,7 +54,7 @@
    },
    "outputs": [],
    "source": [
-    "with pytest.raises(ln.InstanceNotSetupError):\n",
+    "with pytest.raises(ln._InstanceNotSetupError):\n",
     "    ln.track()"
    ]
   },

{lamindb-0.73.2 → lamindb-0.74.0}/lamindb/__init__.py

@@ -14,6 +14,7 @@ Registries:
    ULabel
    Feature
    FeatureSet
+   Param
 
 Key functionality:
 
@@ -41,12 +42,13 @@ Modules & settings:
 """
 
 # denote a release candidate for 0.1.0 with 0.1rc1, 0.1a1, 0.1b1, etc.
-__version__ = "0.73.2"
+__version__ = "0.74.0"
 
 import os as _os
 
 import lamindb_setup as _lamindb_setup
-from lamindb_setup._check_setup import InstanceNotSetupError, _check_instance_setup
+from lamindb_setup._check_setup import InstanceNotSetupError as _InstanceNotSetupError
+from lamindb_setup._check_setup import _check_instance_setup
 from lamindb_setup._connect_instance import connect
 from lamindb_setup.core.upath import UPath
 
@@ -54,17 +56,18 @@ from . import setup
 
 
 def __getattr__(name):
-    raise InstanceNotSetupError()
+    raise _InstanceNotSetupError()
 
 
 if _check_instance_setup(from_lamindb=True):
-    del InstanceNotSetupError
+    del _InstanceNotSetupError
     del __getattr__  # delete so that imports work out
     from lnschema_core.models import (
         Artifact,
         Collection,
         Feature,
         FeatureSet,
+        Param,
         Run,
         Storage,
         Transform,

{lamindb-0.73.2 → lamindb-0.74.0}/lamindb/_artifact.py

@@ -23,7 +23,7 @@ from lamindb_setup.core.upath import (
     get_stat_dir_cloud,
     get_stat_file_cloud,
 )
-from lnschema_core.models import Artifact, FeatureManager, Run, Storage
+from lnschema_core.models import Artifact, FeatureManager, ParamManager, Run, Storage
 from lnschema_core.types import (
     VisibilityChoice,
 )
@@ -84,7 +84,7 @@ def process_pathlike(
             pass
     if isinstance(filepath, LocalPathClasses):
         filepath = filepath.resolve()
-    if check_path_is_child_of_root(filepath, default_storage.root_as_path()):
+    if check_path_is_child_of_root(filepath, default_storage.root):
         use_existing_storage_key = True
         return default_storage, use_existing_storage_key
     else:
@@ -272,28 +272,17 @@ def check_path_in_existing_storage(
 ) -> Storage | bool:
     for storage in Storage.objects.using(using_key).filter().all():
         # if path is part of storage, return it
-        if check_path_is_child_of_root(path, root=create_path(storage.root)):
+        if check_path_is_child_of_root(path, root=storage.root):
             return storage
     return False
 
 
 def check_path_is_child_of_root(path: Path | UPath, root: Path | UPath | None) -> bool:
-    path = UPath(str(path)) if not isinstance(path, UPath) else path
-    root = UPath(str(root)) if not isinstance(root, UPath) else root
-
-    # the following comparisons can fail if types aren't comparable
-    if not isinstance(path, LocalPathClasses) and not isinstance(
-        root, LocalPathClasses
-    ):
-        # the following tests equivalency of two UPath objects
-        # via string representations; otherwise
-        # S3Path('s3://lndb-storage/') and S3Path('s3://lamindb-ci/')
-        # test as equivalent
-        return list(path.parents)[-1].as_posix() == root.as_posix()
-    elif isinstance(path, LocalPathClasses) and isinstance(root, LocalPathClasses):
-        return root.resolve() in path.resolve().parents
-    else:
-        return False
+    # str is needed to eliminate UPath storage_options
+    # from the equality checks below
+    path = UPath(str(path))
+    root = UPath(str(root))
+    return root.resolve() in path.resolve().parents
 
 
 def get_relative_path_to_directory(
@@ -358,7 +347,7 @@ def get_artifact_kwargs_from_data(
     check_path_in_storage = False
     if use_existing_storage_key:
         inferred_key = get_relative_path_to_directory(
-            path=path, directory=storage.root_as_path()
+            path=path, directory=UPath(storage.root)
         ).as_posix()
         if key is None:
             key = inferred_key
@@ -516,6 +505,7 @@ def update_attributes(data: HasFeatures, attributes: Mapping[str, str]):
 
 def __init__(artifact: Artifact, *args, **kwargs):
     artifact.features = FeatureManager(artifact)
+    artifact.params = ParamManager(artifact)
     # Below checks for the Django-internal call in from_db()
     # it'd be better if we could avoid this, but not being able to create a Artifact
     # from data with the default constructor renders the central class of the API
@@ -532,6 +522,7 @@ def __init__(artifact: Artifact, *args, **kwargs):
         raise ValueError("Only one non-keyword arg allowed: data")
 
     data: str | Path = kwargs.pop("data") if len(args) == 0 else args[0]
+    type: str = kwargs.pop("type") if "type" in kwargs else "dataset"
     key: str | None = kwargs.pop("key") if "key" in kwargs else None
     run: Run | None = kwargs.pop("run") if "run" in kwargs else None
     description: str | None = (
@@ -616,6 +607,7 @@ def __init__(artifact: Artifact, *args, **kwargs):
             "to _retain_ the old state by duplicating the entire folder, do _not_ pass `is_new_version_of`"
         )
 
+    kwargs["type"] = type
     kwargs["uid"] = provisional_uid
     kwargs["version"] = version
     kwargs["description"] = description
@@ -662,6 +654,7 @@ def from_df(
         version=version,
         is_new_version_of=is_new_version_of,
         accessor="DataFrame",
+        type="dataset",
         **kwargs,
     )
     return artifact
@@ -690,6 +683,7 @@ def from_anndata(
         version=version,
         is_new_version_of=is_new_version_of,
         accessor="AnnData",
+        type="dataset",
         **kwargs,
     )
     return artifact
@@ -716,6 +710,7 @@ def from_mudata(
         version=version,
         is_new_version_of=is_new_version_of,
         accessor="MuData",
+        type="dataset",
         **kwargs,
     )
     return artifact
@@ -752,7 +747,7 @@ def from_dir(
         else:
             # maintain the hierachy within an existing storage location
             folder_key_path = get_relative_path_to_directory(
-                folderpath, storage.root_as_path()
+                folderpath, UPath(storage.root)
             )
     else:
         folder_key_path = Path(key)

{lamindb-0.73.2 → lamindb-0.74.0}/lamindb/_feature.py

@@ -27,12 +27,12 @@ FEATURE_TYPES = {
 }
 
 
-def convert_numpy_dtype_to_lamin_feature_type(dtype) -> str:
+def convert_numpy_dtype_to_lamin_feature_type(dtype, str_as_cat: bool = True) -> str:
     orig_type = dtype.name
     # strip precision qualifiers
     type = "".join(i for i in orig_type if not i.isdigit())
     if type == "object" or type == "str":
-        type = "cat"
+        type = "cat" if str_as_cat else "str"
     return type
 
 

{lamindb-0.73.2 → lamindb-0.74.0}/lamindb/_finish.py

@@ -33,13 +33,19 @@ def get_seconds_since_modified(filepath) -> float:
     return datetime.now().timestamp() - filepath.stat().st_mtime
 
 
-def finish():
+def finish() -> None:
     """Mark a tracked run as finished.
 
     Saves source code and, for notebooks, a run report to your default storage location.
     """
-    if run_context.path is None:
+    if run_context.run is None:
         raise TrackNotCalled("Please run `ln.track()` before `ln.finish()`")
+    if run_context.path is None:
+        assert run_context.transform.type not in {"script", "notebook"}
+        run_context.run.finished_at = datetime.now(timezone.utc)
+        run_context.run.save()
+        # nothing else to do
+        return None
     if is_run_from_ipython:  # notebooks
         if (
             get_seconds_since_modified(run_context.path) > 3

{lamindb-0.73.2 → lamindb-0.74.0}/lamindb/_from_values.py

@@ -278,14 +278,26 @@ def index_iterable(iterable: Iterable) -> pd.Index:
 
 
 def _print_values(names: Iterable, n: int = 20, quotes: bool = True) -> str:
-    names = (name for name in names if name != "None")
-    unique_names = list(dict.fromkeys(names))[:n]
-    if quotes:
-        print_values = ", ".join(f"'{name}'" for name in unique_names)
+    if isinstance(names, dict):
+        items = {
+            f"{key}: {value}": None
+            for key, value in names.items()
+            if key != "None" and value != "None"
+        }
     else:
-        print_values = ", ".join(f"{name}" for name in unique_names)
-    if len(unique_names) > n:
+        # Use a dictionary instead of a list to have unique values and preserve order
+        items = {str(name): None for name in names if name != "None"}
+
+    unique_items = list(items.keys())
+
+    if quotes:
+        unique_items = [f"'{item}'" for item in unique_items]
+
+    print_values = ", ".join(unique_items[:n])
+
+    if len(unique_items) > n:
         print_values += ", ..."
+
     return print_values
 
 

{lamindb-0.73.2 → lamindb-0.74.0}/lamindb/_run.py

@@ -1,9 +1,10 @@
 from __future__ import annotations
 
-from lnschema_core.models import Run, Transform
+from lnschema_core.models import ParamManager, Run, Transform
 
 
 def __init__(run: Run, *args, **kwargs):
+    run.params = ParamManager(run)
     if len(args) == len(run._meta.concrete_fields):
         super(Run, run).__init__(*args, **kwargs)
         return None

{lamindb-0.73.2 → lamindb-0.74.0}/lamindb/_storage.py

@@ -3,11 +3,6 @@ from lamindb_setup.core.upath import UPath, create_path
 from lnschema_core import Storage
 
 
-def root_as_path(self) -> UPath:
-    access_token = self._access_token if hasattr(self, "_access_token") else None
-    return create_path(self.root, access_token=access_token)
-
-
 @property  # type: ignore
 @doc_args(Storage.path.__doc__)
 def path(self) -> UPath:
@@ -16,5 +11,4 @@ def path(self) -> UPath:
     return create_path(self.root, access_token=access_token)
 
 
-Storage.root_as_path = root_as_path
 Storage.path = path

{lamindb-0.73.2 → lamindb-0.74.0}/lamindb/core/__init__.py

@@ -10,7 +10,9 @@ Registries:
    QueryManager
    RecordsList
    HasFeatures
+   HasParams
    FeatureManager
+   ParamManager
    LabelManager
    IsVersioned
    CanValidate
@@ -56,6 +58,7 @@ from lamin_utils._inspect import InspectResult
 from lnschema_core.models import (
     CanValidate,
     HasFeatures,
+    HasParams,
     HasParents,
     IsVersioned,
     Registry,
@@ -71,7 +74,7 @@ from lamindb._annotate import (
 )
 from lamindb._query_manager import QueryManager
 from lamindb._query_set import QuerySet, RecordsList
-from lamindb.core._feature_manager import FeatureManager
+from lamindb.core._feature_manager import FeatureManager, ParamManager
 from lamindb.core._label_manager import LabelManager
 
 from . import _data, datasets, exceptions, fields, types

{lamindb-0.73.2 → lamindb-0.74.0}/lamindb/core/_data.py

@@ -24,13 +24,12 @@ from lamindb._registry import get_default_str_field
 from lamindb.core._settings import settings
 
 from ._feature_manager import (
-    FeatureManager,
     get_feature_set_links,
     get_host_id_field,
     get_label_links,
     print_features,
 )
-from ._label_manager import LabelManager, print_labels
+from ._label_manager import print_labels
 from ._run_context import run_context
 from .exceptions import ValidationError
 from .schema import (
@@ -159,7 +158,11 @@ def describe(self: HasFeatures, print_types: bool = False):
         msg += f"  {colors.italic('Provenance')}\n"
         msg += prov_msg
     msg += print_labels(self, print_types=print_types)
-    msg += print_features(self, print_types=print_types)  # type: ignore
+    msg += print_features(  # type: ignore
+        self,
+        print_types=print_types,
+        print_params=hasattr(self, "type") and self.type == "model",
+    )
     logger.print(msg)