hydraflow 0.3.0__py3-none-any.whl → 0.3.2__py3-none-any.whl

hydraflow/__init__.py

@@ -1,19 +1,15 @@
-"""Provide a collection of MLflow runs."""
+"""Integrate Hydra and MLflow to manage and track machine learning experiments."""
 
-from .context import chdir_artifact, log_run, start_run, watch
-from .mlflow import (
-    list_runs,
-    search_runs,
-    set_experiment,
-)
+from .context import chdir_artifact, chdir_hydra, log_run, start_run, watch
+from .mlflow import list_runs, search_runs, set_experiment
 from .progress import multi_tasks_progress, parallel_progress
 from .run_collection import RunCollection
-from .run_data import load_config
-from .run_info import get_artifact_dir, get_hydra_output_dir
+from .utils import get_artifact_dir, get_hydra_output_dir, load_config
 
 __all__ = [
     "RunCollection",
     "chdir_artifact",
+    "chdir_hydra",
     "get_artifact_dir",
     "get_hydra_output_dir",
     "list_runs",

hydraflow/config.py

@@ -54,7 +54,7 @@ def _iter_params(config: object, prefix: str = "") -> Iterator[tuple[str, Any]]:
     if isinstance(config, DictConfig):
         for key, value in config.items():
             if _is_param(value):
-                yield f"{prefix}{key}", value
+                yield f"{prefix}{key}", _convert(value)
 
             else:
                 yield from _iter_params(value, f"{prefix}{key}.")
@@ -62,7 +62,7 @@ def _iter_params(config: object, prefix: str = "") -> Iterator[tuple[str, Any]]:
     elif isinstance(config, ListConfig):
         for index, value in enumerate(config):
             if _is_param(value):
-                yield f"{prefix}{index}", value
+                yield f"{prefix}{index}", _convert(value)
 
             else:
                 yield from _iter_params(value, f"{prefix}{index}.")
@@ -78,3 +78,11 @@ def _is_param(value: object) -> bool:
             return False
 
     return True
+
+
+def _convert(value: Any) -> Any:
+    """Convert the given value to a Python object."""
+    if isinstance(value, ListConfig):
+        return list(value)
+
+    return value

hydraflow/context.py

@@ -238,6 +238,25 @@ class Handler(PatternMatchingEventHandler):
             self.func(file)
 
 
+@contextmanager
+def chdir_hydra() -> Iterator[Path]:
+    """Change the current working directory to the hydra output directory.
+
+    This context manager changes the current working directory to the hydra output
+    directory. It ensures that the directory is changed back to the original
+    directory after the context is exited.
+    """
+    curdir = Path.cwd()
+    path = HydraConfig.get().runtime.output_dir
+
+    os.chdir(path)
+    try:
+        yield Path(path)
+
+    finally:
+        os.chdir(curdir)
+
+
 @contextmanager
 def chdir_artifact(
     run: Run,

hydraflow/mlflow.py

@@ -207,8 +207,14 @@ def _list_runs(
         if experiment := mlflow.get_experiment_by_name(name):
             loc = experiment.artifact_location
 
-            if isinstance(loc, str) and loc.startswith("file://"):
-                path = Path(mlflow.artifacts.download_artifacts(loc))
+            if isinstance(loc, str):
+                if loc.startswith("file://"):
+                    path = Path(mlflow.artifacts.download_artifacts(loc))
+                elif Path(loc).is_dir():
+                    path = Path(loc)
+                else:
+                    continue
+
                 run_ids.extend(file.stem for file in path.iterdir() if file.is_dir())
 
     it = (joblib.delayed(mlflow.get_run)(run_id) for run_id in run_ids)

hydraflow/param.py

@@ -72,4 +72,4 @@ def _match_tuple(param: str, value: tuple) -> bool | None:
     if type(value[0]) is not type(value[1]):
         return None
 
-    return value[0] <= type(value[0])(param) < value[1]  # type: ignore
+    return value[0] <= type(value[0])(param) <= value[1]  # type: ignore

hydraflow/run_collection.py

@@ -239,8 +239,8 @@ class RunCollection:
         The filtering supports:
         - Exact matches for single values.
         - Membership checks for lists of values.
-        - Range checks for tuples of two values (inclusive of the lower bound
-          and exclusive of the upper bound).
+        - Range checks for tuples of two values (inclusive of both the lower
+          and upper bound).
 
         Args:
             config (object | None): The configuration object to filter the runs.
@@ -476,7 +476,7 @@ class RunCollection:
         """
         return (func(run, *args, **kwargs) for run in self)
 
-    def map_run_id(
+    def map_id(
         self,
         func: Callable[Concatenate[str, P], T],
         *args: P.args,
@@ -569,8 +569,8 @@ class RunCollection:
 
     def group_by(
         self,
-        *names: str | list[str],
-    ) -> dict[tuple[str | None, ...], RunCollection]:
+        names: str | list[str],
+    ) -> dict[str | None | tuple[str | None, ...], RunCollection]:
         """Group runs by specified parameter names.
 
         Group the runs in the collection based on the values of the
@@ -578,19 +578,23 @@ class RunCollection:
         form a key in the returned dictionary.
 
         Args:
-            *names (str | list[str]): The names of the parameters to group by.
+            names (str | list[str]): The names of the parameters to group by.
                 This can be a single parameter name or multiple names provided
                 as separate arguments or as a list.
 
         Returns:
-            dict[tuple[str | None, ...], RunCollection]: A dictionary where the keys
-            are tuples of parameter values and the values are RunCollection objects
-            containing the runs that match those parameter values.
+            dict[str | None | tuple[str | None, ...], RunCollection]: A
+            dictionary where the keys are tuples of parameter values and the
+            values are `RunCollection` objects containing the runs that match
+            those parameter values.
 
         """
-        grouped_runs: dict[tuple[str | None, ...], list[Run]] = {}
+        grouped_runs: dict[str | None | tuple[str | None, ...], list[Run]] = {}
+        is_list = isinstance(names, list)
         for run in self._runs:
-            key = get_params(run, *names)
+            key = get_params(run, names)
+            if not is_list:
+                key = key[0]
             grouped_runs.setdefault(key, []).append(run)
 
         return {key: RunCollection(runs) for key, runs in grouped_runs.items()}
@@ -637,8 +641,8 @@ def filter_runs(
     The filtering supports:
     - Exact matches for single values.
     - Membership checks for lists of values.
-    - Range checks for tuples of two values (inclusive of the lower bound and
-      exclusive of the upper bound).
+    - Range checks for tuples of two values (inclusive of both the lower and
+      upper bound).
 
     Args:
         runs (list[Run]): The list of runs to filter.

hydraflow/run_data.py

@@ -1,21 +1,19 @@
-"""Provide information about MLflow runs."""
+"""Provide data about `RunCollection` instances."""
 
 from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
-from omegaconf import DictConfig, OmegaConf
-
-from hydraflow.run_info import get_artifact_dir
+from hydraflow.utils import load_config
 
 if TYPE_CHECKING:
-    from mlflow.entities import Run
+    from omegaconf import DictConfig
 
     from hydraflow.run_collection import RunCollection
 
 
 class RunCollectionData:
-    """Provide information about MLflow runs."""
+    """Provide data about a `RunCollection` instance."""
 
     def __init__(self, runs: RunCollection) -> None:
         self._runs = runs
@@ -34,23 +32,3 @@ class RunCollectionData:
     def config(self) -> list[DictConfig]:
         """Get the configuration for each run in the collection."""
         return [load_config(run) for run in self._runs]
-
-
-def load_config(run: Run) -> DictConfig:
-    """Load the configuration for a given run.
-
-    This function loads the configuration for the provided Run instance
-    by downloading the configuration file from the MLflow artifacts and
-    loading it using OmegaConf. It returns an empty config if
-    `.hydra/config.yaml` is not found in the run's artifact directory.
-
-    Args:
-        run (Run): The Run instance for which to load the configuration.
-
-    Returns:
-        The loaded configuration as a DictConfig object. Returns an empty
-        DictConfig if the configuration file is not found.
-
-    """
-    path = get_artifact_dir(run) / ".hydra/config.yaml"
-    return OmegaConf.load(path)  # type: ignore

hydraflow/run_info.py

@@ -1,23 +1,19 @@
-"""Provide information about MLflow runs."""
+"""Provide information about `RunCollection` instances."""
 
 from __future__ import annotations
 
-from pathlib import Path
 from typing import TYPE_CHECKING
 
-import mlflow
-from hydra.core.hydra_config import HydraConfig
-from mlflow.tracking import artifact_utils
-from omegaconf import OmegaConf
+from hydraflow.utils import get_artifact_dir
 
 if TYPE_CHECKING:
-    from mlflow.entities import Run
+    from pathlib import Path
 
     from hydraflow.run_collection import RunCollection
 
 
 class RunCollectionInfo:
-    """Provide information about MLflow runs."""
+    """Provide information about a `RunCollection` instance."""
 
     def __init__(self, runs: RunCollection) -> None:
         self._runs = runs
@@ -36,56 +32,3 @@ class RunCollectionInfo:
     def artifact_dir(self) -> list[Path]:
         """Get the artifact directory for each run in the collection."""
         return [get_artifact_dir(run) for run in self._runs]
-
-
-def get_artifact_dir(run: Run | None = None) -> Path:
-    """Retrieve the artifact directory for the given run.
-
-    This function uses MLflow to get the artifact directory for the given run.
-
-    Args:
-        run (Run | None): The run object. Defaults to None.
-
-    Returns:
-        The local path to the directory where the artifacts are downloaded.
-
-    """
-    if run is None:
-        uri = mlflow.get_artifact_uri()
-    else:
-        uri = artifact_utils.get_artifact_uri(run.info.run_id)
-
-    return Path(mlflow.artifacts.download_artifacts(uri))
-
-
-def get_hydra_output_dir(run: Run | None = None) -> Path:
-    """Retrieve the Hydra output directory for the given run.
-
-    This function returns the Hydra output directory. If no run is provided,
-    it retrieves the output directory from the current Hydra configuration.
-    If a run is provided, it retrieves the artifact path for the run, loads
-    the Hydra configuration from the downloaded artifacts, and returns the
-    output directory specified in that configuration.
-
-    Args:
-        run (Run | None): The run object. Defaults to None.
-
-    Returns:
-        Path: The path to the Hydra output directory.
-
-    Raises:
-        FileNotFoundError: If the Hydra configuration file is not found
-            in the artifacts.
-
-    """
-    if run is None:
-        hc = HydraConfig.get()
-        return Path(hc.runtime.output_dir)
-
-    path = get_artifact_dir(run) / ".hydra/hydra.yaml"
-
-    if path.exists():
-        hc = OmegaConf.load(path)
-        return Path(hc.hydra.runtime.output_dir)
-
-    raise FileNotFoundError

hydraflow/utils.py

@@ -0,0 +1,88 @@
+"""Provide utility functions for HydraFlow."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import mlflow
+from hydra.core.hydra_config import HydraConfig
+from mlflow.entities import Run
+from mlflow.tracking import artifact_utils
+from omegaconf import DictConfig, OmegaConf
+
+if TYPE_CHECKING:
+    from mlflow.entities import Run
+
+
+def get_artifact_dir(run: Run | None = None) -> Path:
+    """Retrieve the artifact directory for the given run.
+
+    This function uses MLflow to get the artifact directory for the given run.
+
+    Args:
+        run (Run | None): The run object. Defaults to None.
+
+    Returns:
+        The local path to the directory where the artifacts are downloaded.
+
+    """
+    if run is None:
+        uri = mlflow.get_artifact_uri()
+    else:
+        uri = artifact_utils.get_artifact_uri(run.info.run_id)
+
+    return Path(mlflow.artifacts.download_artifacts(uri))
+
+
+def get_hydra_output_dir(run: Run | None = None) -> Path:
+    """Retrieve the Hydra output directory for the given run.
+
+    This function returns the Hydra output directory. If no run is provided,
+    it retrieves the output directory from the current Hydra configuration.
+    If a run is provided, it retrieves the artifact path for the run, loads
+    the Hydra configuration from the downloaded artifacts, and returns the
+    output directory specified in that configuration.
+
+    Args:
+        run (Run | None): The run object. Defaults to None.
+
+    Returns:
+        Path: The path to the Hydra output directory.
+
+    Raises:
+        FileNotFoundError: If the Hydra configuration file is not found
+            in the artifacts.
+
+    """
+    if run is None:
+        hc = HydraConfig.get()
+        return Path(hc.runtime.output_dir)
+
+    path = get_artifact_dir(run) / ".hydra/hydra.yaml"
+
+    if path.exists():
+        hc = OmegaConf.load(path)
+        return Path(hc.hydra.runtime.output_dir)
+
+    raise FileNotFoundError
+
+
+def load_config(run: Run) -> DictConfig:
+    """Load the configuration for a given run.
+
+    This function loads the configuration for the provided Run instance
+    by downloading the configuration file from the MLflow artifacts and
+    loading it using OmegaConf. It returns an empty config if
+    `.hydra/config.yaml` is not found in the run's artifact directory.
+
+    Args:
+        run (Run): The Run instance for which to load the configuration.
+
+    Returns:
+        The loaded configuration as a DictConfig object. Returns an empty
+        DictConfig if the configuration file is not found.
+
+    """
+    path = get_artifact_dir(run) / ".hydra/config.yaml"
+    return OmegaConf.load(path)  # type: ignore

{hydraflow-0.3.0.dist-info → hydraflow-0.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: hydraflow
-Version: 0.3.0
+Version: 0.3.2
 Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
 Project-URL: Documentation, https://github.com/daizutabi/hydraflow
 Project-URL: Source, https://github.com/daizutabi/hydraflow

hydraflow-0.3.2.dist-info/RECORD

@@ -0,0 +1,16 @@
+hydraflow/__init__.py,sha256=6sfM1ashUkfrNf7lOR7raFYhG8YdOAJR-JgRNL_IVo8,698
+hydraflow/asyncio.py,sha256=-i1C8KAmNDImrjHnk92Csaa1mpjdK8Vp4ZVaQV-l94s,6634
+hydraflow/config.py,sha256=6V5omJ3-h9-ZwVpM5rTA4FqE_mu8urTy9OqV4zG79gw,2671
+hydraflow/context.py,sha256=412884e84qIEYtbxJT4roYsKfldGaTKzgo6Q1FAsT5U,8733
+hydraflow/mlflow.py,sha256=JELqXFCJ9MsEJaQWT5dyleTFk8BHL7cQwW_gzhkPoIg,8729
+hydraflow/param.py,sha256=QkLeQvt5ZF3GyRGnhP66o0GElc1ZOOCxCL7PdyfIUbA,1939
+hydraflow/progress.py,sha256=zvKX1HCN8_xDOsgYOEcLLhkhdPdep-U8vHrc0XZ-6SQ,6163
+hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+hydraflow/run_collection.py,sha256=0SCParNkSoXw7Ule-FH3az578Hhu-1TKGyrFNXxTciU,25082
+hydraflow/run_data.py,sha256=ZXVr0PHyufH9wwyQYWtpE4_MheAC2ArTW_J1TTMQ4iI,983
+hydraflow/run_info.py,sha256=sMXOo20ClaRIommMEzuAbO_OrcXx7M1Yt4FMV7spxz0,998
+hydraflow/utils.py,sha256=aRdBdToKfvHhN2qFiRzPHIdQxS7cTpZREQeP8HreAfI,2676
+hydraflow-0.3.2.dist-info/METADATA,sha256=RktfWMufNqrMU2CBSbmVxwE_e7VPbLhVYQHgBKXuh9Q,3840
+hydraflow-0.3.2.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+hydraflow-0.3.2.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
+hydraflow-0.3.2.dist-info/RECORD,,

hydraflow-0.3.0.dist-info/RECORD

@@ -1,15 +0,0 @@
-hydraflow/__init__.py,sha256=zlLTztJPXyBFJC5Z8G7_OnlfzAHJPRrfE1c2OoDvlTg,667
-hydraflow/asyncio.py,sha256=-i1C8KAmNDImrjHnk92Csaa1mpjdK8Vp4ZVaQV-l94s,6634
-hydraflow/config.py,sha256=Wx7jymwLVr5EfpzXBpvv3Ax3VhGhvWyA7Yy6EzsPYWk,2479
-hydraflow/context.py,sha256=IaDy-ZCdCfWwv95S-gyQNp062oBdtSVaz6dxGmO6Y8w,8226
-hydraflow/mlflow.py,sha256=GkOr_pXfpfY5USYBLrCigHcP13VgrAK_e9kheR1Wke4,8579
-hydraflow/param.py,sha256=dvIXcKgc_MPiju3WEk9qz5FOUeK5qSj-YWN2ophCpUM,1938
-hydraflow/progress.py,sha256=zvKX1HCN8_xDOsgYOEcLLhkhdPdep-U8vHrc0XZ-6SQ,6163
-hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hydraflow/run_collection.py,sha256=Xv6-KD5ac-vv-4Q3PZrzJy1x84H_g7UoP7ZqZ8_DQeQ,24973
-hydraflow/run_data.py,sha256=HgXGjV5oN6VxOAhrFRjubWz5ZiRqT1a2VdS5OcH2UQQ,1732
-hydraflow/run_info.py,sha256=4QrTmyPEQ_PVn7JKXJIa9NkXGAdqh8k5Sue1ggQS5aQ,2678
-hydraflow-0.3.0.dist-info/METADATA,sha256=DmC1Yjwuc3snUQiePCr5xvdtbfIevOapiA2sg8w6Aho,3840
-hydraflow-0.3.0.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-hydraflow-0.3.0.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
-hydraflow-0.3.0.dist-info/RECORD,,