hydraflow 0.3.1__py3-none-any.whl → 0.3.3__py3-none-any.whl

hydraflow/__init__.py

@@ -4,7 +4,13 @@ from .context import chdir_artifact, chdir_hydra, log_run, start_run, watch
 from .mlflow import list_runs, search_runs, set_experiment
 from .progress import multi_tasks_progress, parallel_progress
 from .run_collection import RunCollection
-from .utils import get_artifact_dir, get_hydra_output_dir, load_config
+from .utils import (
+    get_artifact_dir,
+    get_hydra_output_dir,
+    get_overrides,
+    load_config,
+    load_overrides,
+)
 
 __all__ = [
     "RunCollection",
@@ -12,8 +18,10 @@ __all__ = [
     "chdir_hydra",
     "get_artifact_dir",
     "get_hydra_output_dir",
+    "get_overrides",
     "list_runs",
     "load_config",
+    "load_overrides",
     "log_run",
     "multi_tasks_progress",
     "parallel_progress",

hydraflow/param.py

@@ -72,4 +72,4 @@ def _match_tuple(param: str, value: tuple) -> bool | None:
     if type(value[0]) is not type(value[1]):
         return None
 
-    return value[0] <= type(value[0])(param) < value[1]  # type: ignore
+    return value[0] <= type(value[0])(param) <= value[1]  # type: ignore

hydraflow/run_collection.py

@@ -24,12 +24,12 @@ from itertools import chain
 from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar, overload
 
 from mlflow.entities import RunStatus
-from polars.dataframe import DataFrame
 
 import hydraflow.param
-from hydraflow.config import collect_params, iter_params
+from hydraflow.config import iter_params
 from hydraflow.run_data import RunCollectionData
 from hydraflow.run_info import RunCollectionInfo
+from hydraflow.utils import load_config
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Iterator
@@ -239,8 +239,8 @@ class RunCollection:
         The filtering supports:
         - Exact matches for single values.
         - Membership checks for lists of values.
-        - Range checks for tuples of two values (inclusive of the lower bound
-          and exclusive of the upper bound).
+        - Range checks for tuples of two values (inclusive of both the lower
+          and upper bound).
 
         Args:
             config (object | None): The configuration object to filter the runs.
@@ -476,7 +476,7 @@ class RunCollection:
         """
         return (func(run, *args, **kwargs) for run in self)
 
-    def map_run_id(
+    def map_id(
         self,
         func: Callable[Concatenate[str, P], T],
         *args: P.args,
@@ -516,7 +516,7 @@ class RunCollection:
             in the collection.
 
         """
-        return (func(config, *args, **kwargs) for config in self.data.config)
+        return (func(load_config(run), *args, **kwargs) for run in self)
 
     def map_uri(
         self,
@@ -569,8 +569,8 @@ class RunCollection:
 
     def group_by(
         self,
-        *names: str | list[str],
-    ) -> dict[tuple[str | None, ...], RunCollection]:
+        names: str | list[str],
+    ) -> dict[str | None | tuple[str | None, ...], RunCollection]:
         """Group runs by specified parameter names.
 
         Group the runs in the collection based on the values of the
@@ -578,33 +578,27 @@ class RunCollection:
         form a key in the returned dictionary.
 
         Args:
-            *names (str | list[str]): The names of the parameters to group by.
+            names (str | list[str]): The names of the parameters to group by.
                 This can be a single parameter name or multiple names provided
                 as separate arguments or as a list.
 
         Returns:
-            dict[tuple[str | None, ...], RunCollection]: A dictionary where the keys
-            are tuples of parameter values and the values are RunCollection objects
-            containing the runs that match those parameter values.
+            dict[str | None | tuple[str | None, ...], RunCollection]: A
+            dictionary where the keys are tuples of parameter values and the
+            values are `RunCollection` objects containing the runs that match
+            those parameter values.
 
         """
-        grouped_runs: dict[tuple[str | None, ...], list[Run]] = {}
+        grouped_runs: dict[str | None | tuple[str | None, ...], list[Run]] = {}
+        is_list = isinstance(names, list)
         for run in self._runs:
-            key = get_params(run, *names)
+            key = get_params(run, names)
+            if not is_list:
+                key = key[0]
             grouped_runs.setdefault(key, []).append(run)
 
         return {key: RunCollection(runs) for key, runs in grouped_runs.items()}
 
-    @property
-    def config(self) -> DataFrame:
-        """Get the runs' configurations as a polars DataFrame.
-
-        Returns:
-            A polars DataFrame containing the runs' configurations.
-
-        """
-        return DataFrame(self.map_config(collect_params))
-
 
 def _param_matches(run: Run, key: str, value: Any) -> bool:
     params = run.data.params
@@ -637,8 +631,8 @@ def filter_runs(
     The filtering supports:
     - Exact matches for single values.
     - Membership checks for lists of values.
-    - Range checks for tuples of two values (inclusive of the lower bound and
-      exclusive of the upper bound).
+    - Range checks for tuples of two values (inclusive of both the lower and
+      upper bound).
 
     Args:
         runs (list[Run]): The list of runs to filter.

hydraflow/run_data.py

@@ -4,10 +4,13 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
-from hydraflow.utils import load_config
+from polars.dataframe import DataFrame
+
+from hydraflow.config import collect_params
 
 if TYPE_CHECKING:
-    from omegaconf import DictConfig
+    from collections.abc import Iterable
+    from typing import Any
 
     from hydraflow.run_collection import RunCollection
 
@@ -19,16 +22,36 @@ class RunCollectionData:
         self._runs = runs
 
     @property
-    def params(self) -> list[dict[str, str]]:
+    def params(self) -> dict[str, list[str]]:
         """Get the parameters for each run in the collection."""
-        return [run.data.params for run in self._runs]
+        return _to_dict(run.data.params for run in self._runs)
 
     @property
-    def metrics(self) -> list[dict[str, float]]:
+    def metrics(self) -> dict[str, list[float]]:
         """Get the metrics for each run in the collection."""
-        return [run.data.metrics for run in self._runs]
+        return _to_dict(run.data.metrics for run in self._runs)
 
     @property
-    def config(self) -> list[DictConfig]:
-        """Get the configuration for each run in the collection."""
-        return [load_config(run) for run in self._runs]
+    def config(self) -> DataFrame:
+        """Get the runs' configurations as a polars DataFrame.
+
+        Returns:
+            A polars DataFrame containing the runs' configurations.
+
+        """
+        return DataFrame(self._runs.map_config(collect_params))
+
+
+def _to_dict(it: Iterable[dict[str, Any]]) -> dict[str, list[Any]]:
+    """Convert an iterable of dictionaries to a dictionary of lists."""
+    data = list(it)
+    if not data:
+        return {}
+
+    keys = []
+    for d in data:
+        for key in d:
+            if key not in keys:
+                keys.append(key)
+
+    return {key: [x.get(key) for x in data] for key in keys}

hydraflow/utils.py

@@ -68,6 +68,11 @@ def get_hydra_output_dir(run: Run | None = None) -> Path:
     raise FileNotFoundError
 
 
+def get_overrides() -> list[str]:
+    """Retrieve the overrides for the current run."""
+    return HydraConfig.get().overrides.task
+
+
 def load_config(run: Run) -> DictConfig:
     """Load the configuration for a given run.
 
@@ -86,3 +91,23 @@ def load_config(run: Run) -> DictConfig:
     """
     path = get_artifact_dir(run) / ".hydra/config.yaml"
     return OmegaConf.load(path)  # type: ignore
+
+
+def load_overrides(run: Run) -> list[str]:
+    """Load the overrides for a given run.
+
+    This function loads the overrides for the provided Run instance
+    by downloading the overrides file from the MLflow artifacts and
+    loading it using OmegaConf. It returns an empty config if
+    `.hydra/overrides.yaml` is not found in the run's artifact directory.
+
+    Args:
+        run (Run): The Run instance for which to load the overrides.
+
+    Returns:
+        The loaded overrides as a list of strings. Returns an empty list
+        if the overrides file is not found.
+
+    """
+    path = get_artifact_dir(run) / ".hydra/overrides.yaml"
+    return [str(x) for x in OmegaConf.load(path)]

{hydraflow-0.3.1.dist-info → hydraflow-0.3.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: hydraflow
-Version: 0.3.1
+Version: 0.3.3
 Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
 Project-URL: Documentation, https://github.com/daizutabi/hydraflow
 Project-URL: Source, https://github.com/daizutabi/hydraflow

hydraflow-0.3.3.dist-info/RECORD

@@ -0,0 +1,16 @@
+hydraflow/__init__.py,sha256=aZWgotgg9BAYGOk6WAocOdo1M9ydAg_i7SThFeuLNo8,797
+hydraflow/asyncio.py,sha256=-i1C8KAmNDImrjHnk92Csaa1mpjdK8Vp4ZVaQV-l94s,6634
+hydraflow/config.py,sha256=6V5omJ3-h9-ZwVpM5rTA4FqE_mu8urTy9OqV4zG79gw,2671
+hydraflow/context.py,sha256=412884e84qIEYtbxJT4roYsKfldGaTKzgo6Q1FAsT5U,8733
+hydraflow/mlflow.py,sha256=JELqXFCJ9MsEJaQWT5dyleTFk8BHL7cQwW_gzhkPoIg,8729
+hydraflow/param.py,sha256=QkLeQvt5ZF3GyRGnhP66o0GElc1ZOOCxCL7PdyfIUbA,1939
+hydraflow/progress.py,sha256=zvKX1HCN8_xDOsgYOEcLLhkhdPdep-U8vHrc0XZ-6SQ,6163
+hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+hydraflow/run_collection.py,sha256=OiPibp8zM4gpsvqpHr9sBh4_1zmHRuXfPcmen7xND-s,24792
+hydraflow/run_data.py,sha256=qeFX1iRvNAorXA9QQIjzr0o2_82TI44eZKp7llKG8GI,1549
+hydraflow/run_info.py,sha256=sMXOo20ClaRIommMEzuAbO_OrcXx7M1Yt4FMV7spxz0,998
+hydraflow/utils.py,sha256=XFZkUNQ6amYrlSJHIBoQvrxmDXwQG-M7T9BPpqid9Bc,3500
+hydraflow-0.3.3.dist-info/METADATA,sha256=IH_V71WNJTLRItKdVASVm99ZmfXWQYj365RL_qDC_aY,3840
+hydraflow-0.3.3.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+hydraflow-0.3.3.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
+hydraflow-0.3.3.dist-info/RECORD,,

hydraflow-0.3.1.dist-info/RECORD

@@ -1,16 +0,0 @@
-hydraflow/__init__.py,sha256=6sfM1ashUkfrNf7lOR7raFYhG8YdOAJR-JgRNL_IVo8,698
-hydraflow/asyncio.py,sha256=-i1C8KAmNDImrjHnk92Csaa1mpjdK8Vp4ZVaQV-l94s,6634
-hydraflow/config.py,sha256=6V5omJ3-h9-ZwVpM5rTA4FqE_mu8urTy9OqV4zG79gw,2671
-hydraflow/context.py,sha256=412884e84qIEYtbxJT4roYsKfldGaTKzgo6Q1FAsT5U,8733
-hydraflow/mlflow.py,sha256=JELqXFCJ9MsEJaQWT5dyleTFk8BHL7cQwW_gzhkPoIg,8729
-hydraflow/param.py,sha256=dvIXcKgc_MPiju3WEk9qz5FOUeK5qSj-YWN2ophCpUM,1938
-hydraflow/progress.py,sha256=zvKX1HCN8_xDOsgYOEcLLhkhdPdep-U8vHrc0XZ-6SQ,6163
-hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hydraflow/run_collection.py,sha256=Xv6-KD5ac-vv-4Q3PZrzJy1x84H_g7UoP7ZqZ8_DQeQ,24973
-hydraflow/run_data.py,sha256=ZXVr0PHyufH9wwyQYWtpE4_MheAC2ArTW_J1TTMQ4iI,983
-hydraflow/run_info.py,sha256=sMXOo20ClaRIommMEzuAbO_OrcXx7M1Yt4FMV7spxz0,998
-hydraflow/utils.py,sha256=aRdBdToKfvHhN2qFiRzPHIdQxS7cTpZREQeP8HreAfI,2676
-hydraflow-0.3.1.dist-info/METADATA,sha256=W38pNcCNy7Kmx1t9dwFoANsRjCk40-KBJUWux_BvHqA,3840
-hydraflow-0.3.1.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-hydraflow-0.3.1.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
-hydraflow-0.3.1.dist-info/RECORD,,