hydraflow 0.5.4__py3-none-any.whl → 0.6.1__py3-none-any.whl

hydraflow/__init__.py

@@ -1,7 +1,7 @@
 """Integrate Hydra and MLflow to manage and track machine learning experiments."""
 
 from .config import select_config, select_overrides
-from .context import chdir_artifact, chdir_hydra_output, log_run, start_run
+from .context import chdir_artifact, log_run, start_run
 from .mlflow import list_runs, search_runs, set_experiment
 from .run_collection import RunCollection
 from .utils import (
@@ -17,7 +17,6 @@ from .utils import (
 __all__ = [
     "RunCollection",
     "chdir_artifact",
-    "chdir_hydra_output",
     "get_artifact_dir",
     "get_artifact_path",
     "get_hydra_output_dir",

hydraflow/config.py

@@ -22,6 +22,7 @@ def collect_params(config: object) -> dict[str, Any]:
 
     Returns:
         dict[str, Any]: A dictionary of collected parameters.
+
     """
     return dict(iter_params(config))
 
@@ -40,6 +41,7 @@ def iter_params(config: object, prefix: str = "") -> Iterator[tuple[str, Any]]:
 
     Yields:
         Key-value pairs representing the parameters in the configuration object.
+
     """
     if config is None:
         return
@@ -113,6 +115,7 @@ def select_config(config: object, names: list[str]) -> dict[str, Any]:
 
     Returns:
         DictConfig: A new configuration object containing only the selected parameters.
+
     """
     if not isinstance(config, DictConfig):
         config = OmegaConf.structured(config)

hydraflow/context.py

@@ -13,6 +13,7 @@ import mlflow.artifacts
 from hydra.core.hydra_config import HydraConfig
 
 from hydraflow.mlflow import log_params
+from hydraflow.utils import get_artifact_dir
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
@@ -48,6 +49,7 @@ def log_run(
             # Perform operations within the MLflow run context
             pass
         ```
+
     """
     if config:
         log_params(config, synchronous=synchronous)
@@ -55,7 +57,7 @@ def log_run(
     hc = HydraConfig.get()
     output_dir = Path(hc.runtime.output_dir)
 
-    # Save '.hydra' config directory first.
+    # Save '.hydra' config directory.
     output_subdir = output_dir / (hc.output_subdir or "")
     mlflow.log_artifacts(output_subdir.as_posix(), hc.output_subdir)
 
@@ -68,14 +70,43 @@ def log_run(
         raise
 
     finally:
-        # Save output_dir including '.hydra' config directory.
-        mlflow.log_artifacts(output_dir.as_posix())
+        log_text(output_dir)
+
+
+def log_text(directory: Path, pattern: str = "*.log") -> None:
+    """Log text files in the given directory as artifacts.
+
+    Append the text files to the existing text file in the artifact directory.
+
+    Args:
+        directory (Path): The directory to find the logs in.
+        pattern (str): The pattern to match the logs.
+
+    """
+    artifact_dir = get_artifact_dir()
+
+    for file in directory.glob(pattern):
+        if not file.is_file():
+            continue
+
+        file_artifact = artifact_dir / file.name
+        if file_artifact.exists():
+            text = file_artifact.read_text()
+            if not text.endswith("\n"):
+                text += "\n"
+        else:
+            text = ""
+
+        text += file.read_text()
+        mlflow.log_text(text, file.name)
 
 
 @contextmanager
 def start_run(  # noqa: PLR0913
     config: object,
     *,
+    chdir: bool = False,
+    run: Run | None = None,
     run_id: str | None = None,
     experiment_id: str | None = None,
     run_name: str | None = None,
@@ -93,6 +124,9 @@ def start_run(  # noqa: PLR0913
 
     Args:
         config (object): The configuration object to log parameters from.
+        chdir (bool): Whether to change the current working directory to the
+            artifact directory of the current run. Defaults to False.
+        run (Run | None): The existing run. Defaults to None.
         run_id (str | None): The existing run ID. Defaults to None.
         experiment_id (str | None): The experiment ID. Defaults to None.
         run_name (str | None): The name of the run. Defaults to None.
@@ -117,7 +151,11 @@ def start_run(  # noqa: PLR0913
         - `mlflow.start_run`: The MLflow function to start a run directly.
         - `log_run`: A context manager to log parameters and manage the MLflow
            run context.
+
     """
+    if run:
+        run_id = run.info.run_id
+
     with (
         mlflow.start_run(
             run_id=run_id,
@@ -131,33 +169,15 @@ def start_run(  # noqa: PLR0913
         ) as run,
         log_run(config if run_id is None else None, synchronous=synchronous),
     ):
-        yield run
+        if chdir:
+            with chdir_artifact(run):
+                yield run
+        else:
+            yield run
 
 
 @contextmanager
-def chdir_hydra_output() -> Iterator[Path]:
-    """Change the current working directory to the hydra output directory.
-
-    This context manager changes the current working directory to the hydra output
-    directory. It ensures that the directory is changed back to the original
-    directory after the context is exited.
-    """
-    curdir = Path.cwd()
-    path = HydraConfig.get().runtime.output_dir
-
-    os.chdir(path)
-    try:
-        yield Path(path)
-
-    finally:
-        os.chdir(curdir)
-
-
-@contextmanager
-def chdir_artifact(
-    run: Run,
-    artifact_path: str | None = None,
-) -> Iterator[Path]:
+def chdir_artifact(run: Run | None = None) -> Iterator[Path]:
     """Change the current working directory to the artifact directory of the given run.
 
     This context manager changes the current working directory to the artifact
@@ -165,18 +185,16 @@ def chdir_artifact(
     to the original directory after the context is exited.
 
     Args:
-        run (Run): The run to get the artifact directory from.
-        artifact_path (str | None): The artifact path.
+        run (Run | None): The run to get the artifact directory from.
+
     """
     curdir = Path.cwd()
-    path = mlflow.artifacts.download_artifacts(
-        run_id=run.info.run_id,
-        artifact_path=artifact_path,
-    )
+    artifact_dir = get_artifact_dir(run)
+
+    os.chdir(artifact_dir)
 
-    os.chdir(path)
     try:
-        yield Path(path)
+        yield artifact_dir
 
     finally:
         os.chdir(curdir)

hydraflow/mlflow.py

@@ -16,7 +16,6 @@ Key Features:
 
 from __future__ import annotations
 
-from pathlib import Path
 from typing import TYPE_CHECKING
 
 import joblib
@@ -28,8 +27,11 @@ from mlflow.tracking.fluent import SEARCH_MAX_RESULTS_PANDAS, _get_experiment_id
 
 from hydraflow.config import iter_params
 from hydraflow.run_collection import RunCollection
+from hydraflow.utils import get_artifact_dir
 
 if TYPE_CHECKING:
+    from pathlib import Path
+
     from mlflow.entities.experiment import Experiment
 
 
@@ -54,6 +56,7 @@ def set_experiment(
     Returns:
         Experiment: An instance of `mlflow.entities.Experiment` representing
         the new active experiment.
+
     """
     if uri is not None:
         mlflow.set_tracking_uri(uri)
@@ -77,6 +80,7 @@ def log_params(config: object, *, synchronous: bool | None = None) -> None:
         config (object): The configuration object to log the parameters from.
         synchronous (bool | None): Whether to log the parameters synchronously.
             Defaults to None.
+
     """
     for key, value in iter_params(config):
         mlflow.log_param(key, value, synchronous=synchronous)
@@ -133,6 +137,7 @@ def search_runs(  # noqa: PLR0913
 
     Returns:
         A `RunCollection` object containing the search results.
+
     """
     runs = mlflow.search_runs(
         experiment_ids=experiment_ids,
@@ -177,6 +182,7 @@ def list_runs(
     Returns:
         RunCollection: A `RunCollection` instance containing the runs for the
         specified experiments.
+
     """
     rc = _list_runs(experiment_names, n_jobs)
     if status is None:
@@ -207,16 +213,10 @@ def _list_runs(
 
     for name in experiment_names:
         if experiment := mlflow.get_experiment_by_name(name):
-            loc = experiment.artifact_location
-
-            if isinstance(loc, str):
-                if loc.startswith("file:"):
-                    path = Path(mlflow.artifacts.download_artifacts(loc))
-                elif Path(loc).is_dir():
-                    path = Path(loc)
-                else:
-                    continue  # no cov
+            uri = experiment.artifact_location
 
+            if isinstance(uri, str):
+                path = get_artifact_dir(uri=uri)
                 run_ids.extend(file.stem for file in path.iterdir() if file.is_dir())
 
     it = (joblib.delayed(mlflow.get_run)(run_id) for run_id in run_ids)

hydraflow/param.py

@@ -28,6 +28,7 @@ def match(param: str, value: Any) -> bool:  # noqa: PLR0911
     Returns:
         True if the parameter matches the specified value,
         False otherwise.
+
     """
     if callable(value):
         return value(param)
@@ -94,6 +95,7 @@ def to_value(param: str | None, type_: type) -> Any:
 
     Returns:
         The converted value.
+
     """
     if param is None or param == "None":
         return None
@@ -129,6 +131,7 @@ def get_params(run: Run, *names: str | list[str]) -> tuple[str | None, ...]:
     Returns:
         tuple[str | None, ...]: A tuple containing the values of the specified
         parameters in the order they were provided.
+
     """
     names_ = []
     for name in names:
@@ -155,6 +158,7 @@ def get_values(run: Run, names: list[str], types: list[type]) -> tuple[Any, ...]
     Returns:
         tuple[Any, ...]: A tuple containing the values of the specified
         parameters in the order they were provided.
+
     """
     params = get_params(run, names)
     it = zip(params, types, strict=True)

hydraflow/run_collection.py

@@ -106,6 +106,7 @@ class RunCollection:
 
         Returns:
             A new `RunCollection` instance with the runs from both collections.
+
         """
         return self.__class__(self._runs + other._runs)
 
@@ -118,6 +119,7 @@ class RunCollection:
         Returns:
             A new `RunCollection` instance with the runs that are in this collection
             but not in the other.
+
         """
         runs = [run for run in self._runs if run not in other._runs]  # noqa: SLF001
         return self.__class__(runs)
@@ -150,6 +152,7 @@ class RunCollection:
         Returns:
             A new `RunCollection` instance containing the first n runs if n is
             positive, or the last n runs if n is negative.
+
         """
         if n < 0:
             return self.__class__(self._runs[n:])
@@ -164,6 +167,7 @@ class RunCollection:
 
         Raises:
             ValueError: If the collection does not contain exactly one run.
+
         """
         if len(self._runs) != 1:
             raise ValueError("The collection does not contain exactly one run.")
@@ -176,6 +180,7 @@ class RunCollection:
         Returns:
             The only `Run` instance in the collection, or None if the collection
             does not contain exactly one run.
+
         """
         return self._runs[0] if len(self._runs) == 1 else None
 
@@ -187,6 +192,7 @@ class RunCollection:
 
         Raises:
             ValueError: If the collection is empty.
+
         """
         if not self._runs:
             raise ValueError("The collection is empty.")
@@ -199,6 +205,7 @@ class RunCollection:
         Returns:
             The first `Run` instance in the collection, or None if the collection
             is empty.
+
         """
         return self._runs[0] if self._runs else None
 
@@ -210,6 +217,7 @@ class RunCollection:
 
         Raises:
             ValueError: If the collection is empty.
+
         """
         if not self._runs:
             raise ValueError("The collection is empty.")
@@ -222,6 +230,7 @@ class RunCollection:
         Returns:
             The last `Run` instance in the collection, or None if the collection
             is empty.
+
         """
         return self._runs[-1] if self._runs else None
 
@@ -262,6 +271,7 @@ class RunCollection:
 
         Returns:
             A new `RunCollection` object containing the filtered runs.
+
         """
         return RunCollection(
             filter_runs(
@@ -294,6 +304,7 @@ class RunCollection:
 
         See Also:
             `filter`: Perform the actual filtering logic.
+
         """
         try:
             return self.filter(config, **kwargs).first()
@@ -318,6 +329,7 @@ class RunCollection:
 
         See Also:
             `filter`: Perform the actual filtering logic.
+
         """
         return self.filter(config, **kwargs).try_first()
 
@@ -341,6 +353,7 @@ class RunCollection:
 
         See Also:
             `filter`: Perform the actual filtering logic.
+
         """
         try:
             return self.filter(config, **kwargs).last()
@@ -365,6 +378,7 @@ class RunCollection:
 
         See Also:
             `filter`: Perform the actual filtering logic.
+
         """
         return self.filter(config, **kwargs).try_last()
 
@@ -389,6 +403,7 @@ class RunCollection:
 
         See Also:
             `filter`: Perform the actual filtering logic.
+
         """
         try:
             return self.filter(config, **kwargs).one()
@@ -417,6 +432,7 @@ class RunCollection:
 
         See Also:
             `filter`: Perform the actual filtering logic.
+
         """
         return self.filter(config, **kwargs).try_one()
 
@@ -429,6 +445,7 @@ class RunCollection:
 
         Returns:
             A list of unique parameter names.
+
         """
         param_names = set()
 
@@ -453,6 +470,7 @@ class RunCollection:
         Returns:
             A dictionary where the keys are parameter names and the values are
             lists of parameter values.
+
         """
         params = {}
 
@@ -484,6 +502,7 @@ class RunCollection:
 
         Yields:
             Results obtained by applying the function to each run in the collection.
+
         """
         return (func(run, *args, **kwargs) for run in self)
 
@@ -504,6 +523,7 @@ class RunCollection:
         Yields:
             Results obtained by applying the function to each run id in the
             collection.
+
         """
         return (func(run_id, *args, **kwargs) for run_id in self.info.run_id)
 
@@ -524,6 +544,7 @@ class RunCollection:
         Yields:
             Results obtained by applying the function to each run configuration
             in the collection.
+
         """
         return (func(load_config(run), *args, **kwargs) for run in self)
 
@@ -548,6 +569,7 @@ class RunCollection:
         Yields:
             Results obtained by applying the function to each artifact URI in the
             collection.
+
         """
         return (func(uri, *args, **kwargs) for uri in self.info.artifact_uri)
 
@@ -571,6 +593,7 @@ class RunCollection:
         Yields:
             Results obtained by applying the function to each artifact directory
             in the collection.
+
         """
         return (func(dir, *args, **kwargs) for dir in self.info.artifact_dir)  # noqa: A001
 
@@ -594,6 +617,7 @@ class RunCollection:
             dictionary where the keys are tuples of parameter values and the
             values are `RunCollection` objects containing the runs that match
             those parameter values.
+
         """
         grouped_runs: dict[str | None | tuple[str | None, ...], list[Run]] = {}
         is_list = isinstance(names, list)
@@ -620,6 +644,7 @@ class RunCollection:
             key (Callable[[Run], Any] | None): A function that takes a run and returns
                 a value to sort by.
             reverse (bool): If True, sort in descending order.
+
         """
         self._runs.sort(key=key or (lambda x: x.info.start_time), reverse=reverse)
 
@@ -633,6 +658,7 @@ class RunCollection:
 
         Returns:
             A list of values for the specified parameters.
+
         """
         is_list = isinstance(names, list)
 
@@ -664,6 +690,7 @@ class RunCollection:
                 This can be a single parameter name or multiple names provided
                 as separate arguments or as a list.
             reverse (bool): If True, sort in descending order.
+
         """
         values = self.values(names)
         index = sorted(range(len(self)), key=lambda i: values[i], reverse=reverse)
@@ -721,6 +748,7 @@ def filter_runs(
 
     Returns:
         A list of runs that match the specified configuration and key-value pairs.
+
     """
     if override:
         config = select_overrides(config)
@@ -751,6 +779,7 @@ def filter_runs_by_status(
 
     Returns:
         A list of runs that match the specified status.
+
     """
     if isinstance(status, str):
         if status.startswith("!"):

hydraflow/run_data.py

@@ -37,6 +37,7 @@ class RunCollectionData:
 
         Returns:
             A DataFrame containing the runs' configurations.
+
         """
         return DataFrame(self._runs.map_config(collect_params))
 

hydraflow/utils.py

@@ -3,6 +3,8 @@
 from __future__ import annotations
 
 import shutil
+import urllib.parse
+import urllib.request
 from pathlib import Path
 from typing import TYPE_CHECKING
 
@@ -16,29 +18,40 @@ if TYPE_CHECKING:
     from collections.abc import Iterable
 
 
-def get_artifact_dir(run: Run | None = None) -> Path:
+def get_artifact_dir(run: Run | None = None, uri: str | None = None) -> Path:
     """Retrieve the artifact directory for the given run.
 
     This function uses MLflow to get the artifact directory for the given run.
 
     Args:
         run (Run | None): The run object. Defaults to None.
+        uri (str | None): The URI of the artifact. Defaults to None.
 
     Returns:
         The local path to the directory where the artifacts are downloaded.
+
     """
-    uri = mlflow.get_artifact_uri() if run is None else run.info.artifact_uri
+    if run is not None and uri is not None:
+        raise ValueError("Cannot provide both run and uri")
+
+    if run is None and uri is None:
+        uri = mlflow.get_artifact_uri()
+    elif run:
+        uri = run.info.artifact_uri
 
     if not isinstance(uri, str):
         raise NotImplementedError
 
     if uri.startswith("file:"):
-        return Path(mlflow.artifacts.download_artifacts(uri))
+        return file_uri_to_path(uri)
 
-    if Path(uri).is_dir():
-        return Path(uri)
+    return Path(uri)
 
-    raise NotImplementedError
+
+def file_uri_to_path(uri: str) -> Path:
+    """Convert a file URI to a local path."""
+    path = urllib.parse.urlparse(uri).path
+    return Path(urllib.request.url2pathname(path))  # for Windows
 
 
 def get_artifact_path(run: Run | None, path: str) -> Path:
@@ -52,6 +65,7 @@ def get_artifact_path(run: Run | None, path: str) -> Path:
 
     Returns:
         The local path to the artifact.
+
     """
     return get_artifact_dir(run) / path
 
@@ -74,6 +88,7 @@ def get_hydra_output_dir(run: Run | None = None) -> Path:
     Raises:
         FileNotFoundError: If the Hydra configuration file is not found
             in the artifacts.
+
     """
     if run is None:
         hc = HydraConfig.get()
@@ -102,6 +117,7 @@ def load_config(run: Run) -> DictConfig:
     Returns:
         The loaded configuration as a DictConfig object. Returns an empty
         DictConfig if the configuration file is not found.
+
     """
     path = get_artifact_dir(run) / ".hydra/config.yaml"
     return OmegaConf.load(path)  # type: ignore
@@ -126,6 +142,7 @@ def load_overrides(run: Run) -> list[str]:
     Returns:
         The loaded overrides as a list of strings. Returns an empty list
         if the overrides file is not found.
+
     """
     path = get_artifact_dir(run) / ".hydra/overrides.yaml"
     return [str(x) for x in OmegaConf.load(path)]

{hydraflow-0.5.4.dist-info → hydraflow-0.6.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydraflow
-Version: 0.5.4
+Version: 0.6.1
 Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
 Project-URL: Documentation, https://daizutabi.github.io/hydraflow/
 Project-URL: Source, https://github.com/daizutabi/hydraflow

hydraflow-0.6.1.dist-info/RECORD

@@ -0,0 +1,14 @@
+hydraflow/__init__.py,sha256=VPIPNNCyjMAkWBbdvB7Ltwe3QWoc2FwuqkV8uJM5JoM,809
+hydraflow/config.py,sha256=MNX9da5bPVDcjnpji7Cm9ndK6ura92pt361m4PRh6_E,4326
+hydraflow/context.py,sha256=3xfKhMozkKFqtWeOp9Gie0A5o5URMta4US6iVD5TcLU,6002
+hydraflow/mlflow.py,sha256=imD3XL0RTlpnKrkyvO8FNy_Bv6hwSfLiOu1yJuL40ck,8773
+hydraflow/param.py,sha256=yu1aMNXRLegXGDL-68vwIkfeDF9CaU784WZENGLwl7Q,4572
+hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+hydraflow/run_collection.py,sha256=2GRVOy87_2SPjHuCzzUvRNugO_grtFUVjtTfhznwBAc,27444
+hydraflow/run_data.py,sha256=dpyyfnuH9mCtIZeigMo1iFQo9bafMdEL4i4uI2l0UqY,1525
+hydraflow/run_info.py,sha256=Jf5wrIjRLIV1-k-obHDqwKHa6j_ZonrY8od-rXlbtMo,1024
+hydraflow/utils.py,sha256=a9i5PEJn8Ssowv9dqHadAihZXlsqtVjHZ9MZvkPq1bY,4747
+hydraflow-0.6.1.dist-info/METADATA,sha256=FzLPE3L9VlZdrYOicnzTda7BhHuLTU0-WRz02TLgt6c,4700
+hydraflow-0.6.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+hydraflow-0.6.1.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
+hydraflow-0.6.1.dist-info/RECORD,,

hydraflow-0.5.4.dist-info/RECORD

@@ -1,14 +0,0 @@
-hydraflow/__init__.py,sha256=9XO9FD3uiTTPN6X6UAC9FtkJjEqUQZNqpoAmSrjUHfI,855
-hydraflow/config.py,sha256=k6pwneDoC4mb7cVkafMfITn8QemICPnFbhIRGLsmF4w,4323
-hydraflow/context.py,sha256=KukvVexCF32myBOryYMVhINknbAgPFcanmr4cT2EXog,5521
-hydraflow/mlflow.py,sha256=TVp6O9rYNYcdqnlssstXQlI_P_N6inUG-F01X6uJNv4,8928
-hydraflow/param.py,sha256=kuVlxEfgoKL35rNu49M-PkdNr41djK0YZ5IO7fHTSSU,4568
-hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hydraflow/run_collection.py,sha256=mMGpr6cAh9Bf2SeRsYNHySOQJNrWNMI7esDWzyMxnJg,27415
-hydraflow/run_data.py,sha256=V_yhkBsJlDTkLbh1TShkvDxUQYUYTpXFahXJGfdmgro,1524
-hydraflow/run_info.py,sha256=Jf5wrIjRLIV1-k-obHDqwKHa6j_ZonrY8od-rXlbtMo,1024
-hydraflow/utils.py,sha256=ha-gSq3TkWWOlKmnDuyLNCuKNQdEG08CXRzKtb3C4mk,4344
-hydraflow-0.5.4.dist-info/METADATA,sha256=ijJv5powqpMjZ1adDCWdLxRLeEaosmKF3gG74L25fCs,4700
-hydraflow-0.5.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-hydraflow-0.5.4.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
-hydraflow-0.5.4.dist-info/RECORD,,