hydraflow 0.4.5__tar.gz → 0.5.0__tar.gz

{hydraflow-0.4.5 → hydraflow-0.5.0}/.devcontainer/devcontainer.json

@@ -8,6 +8,7 @@
       "extensions": [
         "charliermarsh.ruff",
         "fill-labs.dependi",
+        "markis.code-coverage",
         "ms-python.python",
         "ms-python.vscode-pylance",
         "tamasfe.even-better-toml"

{hydraflow-0.4.5 → hydraflow-0.5.0}/.devcontainer/postCreate.sh

@@ -1,10 +1,11 @@
 #!/bin/bash
 
 echo 'eval "$(starship init bash)"' >> ~/.bashrc
-echo "alias ll='ls -alF'" >> ~/.bashrc
 mkdir -p ~/.config
 cp .devcontainer/starship.toml ~/.config
 
 curl -LsSf https://astral.sh/uv/install.sh | sh
-source $HOME/.cargo/env
-echo 'eval "$(uv generate-shell-completion bash)"' >> ~/.bashrc
+echo 'eval "$(uv generate-shell-completion bash)"' >> ~/.bashrc
+uv tool install ruff@latest
+uv python install
+uv sync -U

{hydraflow-0.4.5 → hydraflow-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: hydraflow
-Version: 0.4.5
+Version: 0.5.0
 Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
 Project-URL: Documentation, https://github.com/daizutabi/hydraflow
 Project-URL: Source, https://github.com/daizutabi/hydraflow
@@ -27,6 +27,7 @@ License: MIT License
         LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
         OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
         SOFTWARE.
+License-File: LICENSE
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python
@@ -36,12 +37,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.10
 Requires-Dist: hydra-core>=1.3
-Requires-Dist: joblib
 Requires-Dist: mlflow>=2.15
-Requires-Dist: polars
-Requires-Dist: rich
-Requires-Dist: watchdog
-Requires-Dist: watchfiles
 Description-Content-Type: text/markdown
 
 # Hydraflow
@@ -118,16 +114,6 @@ def my_app(cfg: MySQLConfig) -> None:
     with hydraflow.start_run():
         # Your app code below.
 
-        with hydraflow.watch(callback):
-            # Watch files in the MLflow artifact directory.
-            # You can update metrics or log other artifacts
-            # according to the watched files in your callback
-            # function.
-            pass
-
-# Your callback function here.
-def callback(file: Path) -> None:
-    pass
 
 if __name__ == "__main__":
     my_app()

{hydraflow-0.4.5 → hydraflow-0.5.0}/README.md

@@ -72,16 +72,6 @@ def my_app(cfg: MySQLConfig) -> None:
     with hydraflow.start_run():
         # Your app code below.
 
-        with hydraflow.watch(callback):
-            # Watch files in the MLflow artifact directory.
-            # You can update metrics or log other artifacts
-            # according to the watched files in your callback
-            # function.
-            pass
-
-# Your callback function here.
-def callback(file: Path) -> None:
-    pass
 
 if __name__ == "__main__":
     my_app()

{hydraflow-0.4.5 → hydraflow-0.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hydraflow"
-version = "0.4.5"
+version = "0.5.0"
 description = "Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments."
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -19,15 +19,7 @@ classifiers = [
   "Programming Language :: Python :: 3.13",
 ]
 requires-python = ">=3.10"
-dependencies = [
-  "hydra-core>=1.3",
-  "joblib",
-  "mlflow>=2.15",
-  "polars",
-  "rich",
-  "watchdog",
-  "watchfiles",
-]
+dependencies = ["hydra-core>=1.3", "mlflow>=2.15"]
 
 [project.urls]
 Documentation = "https://github.com/daizutabi/hydraflow"
@@ -40,12 +32,10 @@ dev-dependencies = [
   "mkapi",
   "mkdocs-material",
   "mkdocs>=1.6",
-  "pytest-asyncio",
   "pytest-clarity",
   "pytest-cov",
   "pytest-randomly",
   "pytest-xdist",
-  "ruff",
 ]
 
 [tool.hatch.build.targets.sdist]
@@ -61,11 +51,15 @@ addopts = [
   "--cov-report=lcov:lcov.info",
 ]
 doctest_optionflags = ["NORMALIZE_WHITESPACE", "IGNORE_EXCEPTION_DETAIL"]
-filterwarnings = ['ignore:pkg_resources is deprecated:DeprecationWarning']
-asyncio_default_fixture_loop_scope = "function"
+filterwarnings = [
+  "ignore:pkg_resources is deprecated:DeprecationWarning",
+  "ignore:Support for class-based `config` is deprecated",
+  "ignore:Pydantic V1 style",
+]
 
 [tool.coverage.report]
 exclude_lines = ["no cov", "raise NotImplementedError", "if TYPE_CHECKING:"]
+skip_covered = true
 
 [tool.ruff]
 line-length = 88
@@ -75,6 +69,7 @@ target-version = "py310"
 select = ["ALL"]
 unfixable = ["F401"]
 ignore = [
+  "A005",
   "ANN003",
   "ANN401",
   "ARG002",

{hydraflow-0.4.5 → hydraflow-0.5.0}/src/hydraflow/__init__.py

@@ -1,9 +1,8 @@
 """Integrate Hydra and MLflow to manage and track machine learning experiments."""
 
 from .config import select_config, select_overrides
-from .context import chdir_artifact, chdir_hydra_output, log_run, start_run, watch
+from .context import chdir_artifact, chdir_hydra_output, log_run, start_run
 from .mlflow import list_runs, search_runs, set_experiment
-from .progress import multi_tasks_progress, parallel_progress
 from .run_collection import RunCollection
 from .utils import (
     get_artifact_dir,
@@ -11,6 +10,7 @@ from .utils import (
     get_overrides,
     load_config,
     load_overrides,
+    remove_run,
 )
 
 __all__ = [
@@ -24,12 +24,10 @@ __all__ = [
     "load_config",
     "load_overrides",
     "log_run",
-    "multi_tasks_progress",
-    "parallel_progress",
+    "remove_run",
     "search_runs",
     "select_config",
     "select_overrides",
     "set_experiment",
     "start_run",
-    "watch",
 ]

{hydraflow-0.4.5 → hydraflow-0.5.0}/src/hydraflow/context.py

@@ -4,7 +4,6 @@ from __future__ import annotations
 
 import logging
 import os
-import time
 from contextlib import contextmanager
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -12,14 +11,11 @@ from typing import TYPE_CHECKING
 import mlflow
 import mlflow.artifacts
 from hydra.core.hydra_config import HydraConfig
-from watchdog.events import FileModifiedEvent, PatternMatchingEventHandler
-from watchdog.observers import Observer
 
 from hydraflow.mlflow import log_params
-from hydraflow.run_info import get_artifact_dir
 
 if TYPE_CHECKING:
-    from collections.abc import Callable, Iterator
+    from collections.abc import Iterator
 
     from mlflow.entities.run import Run
 
@@ -64,14 +60,8 @@ def log_run(
     output_subdir = output_dir / (hc.output_subdir or "")
     mlflow.log_artifacts(output_subdir.as_posix(), hc.output_subdir)
 
-    def log_artifact(path: Path) -> None:
-        local_path = (output_dir / path).as_posix()
-        mlflow.log_artifact(local_path)
-
     try:
         yield
-        # with watch(log_artifact, output_dir, ignore_log=False):
-        #     yield
 
     except Exception as e:
         msg = f"Error during log_run: {e}"
@@ -146,101 +136,6 @@ def start_run(  # noqa: PLR0913
         yield run
 
 
-@contextmanager
-def watch(
-    callback: Callable[[Path], None],
-    dir: Path | str = "",  # noqa: A002
-    *,
-    timeout: int = 60,
-    ignore_patterns: list[str] | None = None,
-    ignore_log: bool = True,
-) -> Iterator[None]:
-    """Watch the given directory for changes.
-
-    This context manager sets up a file system watcher on the specified directory.
-    When a file modification is detected, the provided function is called with
-    the path of the modified file. The watcher runs for the specified timeout
-    period or until the context is exited.
-
-    Args:
-        callback (Callable[[Path], None]): The function to call when a change is
-            detected. It should accept a single argument of type `Path`,
-            which is the path of the modified file.
-        dir (Path | str): The directory to watch. If not specified,
-            the current MLflow artifact URI is used. Defaults to "".
-        timeout (int): The timeout period in seconds for the watcher
-            to run after the context is exited. Defaults to 60.
-        ignore_patterns (list[str] | None): A list of glob patterns to ignore.
-            Defaults to None.
-        ignore_log (bool): Whether to ignore log files. Defaults to True.
-
-    Yields:
-        None
-
-    Example:
-        ```python
-        with watch(log_artifact, "/path/to/dir"):
-            # Perform operations while watching the directory for changes
-            pass
-        ```
-
-    """
-    dir = dir or get_artifact_dir()  # noqa: A001
-    if isinstance(dir, Path):
-        dir = dir.as_posix()  # noqa: A001
-
-    handler = Handler(callback, ignore_patterns=ignore_patterns, ignore_log=ignore_log)
-    observer = Observer()
-    observer.schedule(handler, dir, recursive=True)
-    observer.start()
-
-    try:
-        yield
-
-    except Exception as e:
-        msg = f"Error during watch: {e}"
-        log.exception(msg)
-        raise
-
-    finally:
-        elapsed = 0
-        while not observer.event_queue.empty():
-            time.sleep(0.2)
-            elapsed += 0.2
-            if elapsed > timeout:
-                break
-
-        observer.stop()
-        observer.join()
-
-
-class Handler(PatternMatchingEventHandler):
-    """Monitor file changes and call the given function when a change is detected."""
-
-    def __init__(
-        self,
-        func: Callable[[Path], None],
-        *,
-        ignore_patterns: list[str] | None = None,
-        ignore_log: bool = True,
-    ) -> None:
-        self.func = func
-
-        if ignore_log:
-            if ignore_patterns:
-                ignore_patterns.append("*.log")
-            else:
-                ignore_patterns = ["*.log"]
-
-        super().__init__(ignore_patterns=ignore_patterns)
-
-    def on_modified(self, event: FileModifiedEvent) -> None:
-        """Modify when a file is modified."""
-        file = Path(str(event.src_path))
-        if file.is_file():
-            self.func(file)
-
-
 @contextmanager
 def chdir_hydra_output() -> Iterator[Path]:
     """Change the current working directory to the hydra output directory.

{hydraflow-0.4.5 → hydraflow-0.5.0}/src/hydraflow/run_collection.py

@@ -575,7 +575,7 @@ class RunCollection:
         """
         return (func(dir, *args, **kwargs) for dir in self.info.artifact_dir)  # noqa: A001
 
-    def group_by(
+    def groupby(
         self,
         names: str | list[str],
     ) -> dict[str | None | tuple[str | None, ...], RunCollection]:

{hydraflow-0.4.5 → hydraflow-0.5.0}/src/hydraflow/run_data.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
-from polars.dataframe import DataFrame
+from pandas import DataFrame
 
 from hydraflow.config import collect_params
 
@@ -33,10 +33,10 @@ class RunCollectionData:
 
     @property
     def config(self) -> DataFrame:
-        """Get the runs' configurations as a polars DataFrame.
+        """Get the runs' configurations as a DataFrame.
 
         Returns:
-            A polars DataFrame containing the runs' configurations.
+            A DataFrame containing the runs' configurations.
 
         """
         return DataFrame(self._runs.map_config(collect_params))

{hydraflow-0.4.5 → hydraflow-0.5.0}/src/hydraflow/utils.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import shutil
 from pathlib import Path
 from typing import TYPE_CHECKING
 
@@ -9,11 +10,10 @@ import mlflow
 import mlflow.artifacts
 from hydra.core.hydra_config import HydraConfig
 from mlflow.entities import Run
-from mlflow.tracking import artifact_utils
 from omegaconf import DictConfig, OmegaConf
 
 if TYPE_CHECKING:
-    from mlflow.entities import Run
+    from collections.abc import Iterable
 
 
 def get_artifact_dir(run: Run | None = None) -> Path:
@@ -28,10 +28,10 @@ def get_artifact_dir(run: Run | None = None) -> Path:
         The local path to the directory where the artifacts are downloaded.
 
     """
-    if run is None:
-        uri = mlflow.get_artifact_uri()
-    else:
-        uri = artifact_utils.get_artifact_uri(run.info.run_id)
+    uri = mlflow.get_artifact_uri() if run is None else run.info.artifact_uri
+
+    if not (isinstance(uri, str) and uri.startswith("file://")):
+        raise NotImplementedError
 
     return Path(mlflow.artifacts.download_artifacts(uri))
 
@@ -112,3 +112,13 @@ def load_overrides(run: Run) -> list[str]:
     """
     path = get_artifact_dir(run) / ".hydra/overrides.yaml"
     return [str(x) for x in OmegaConf.load(path)]
+
+
+def remove_run(run: Run | Iterable[Run]) -> None:
+    """Remove the given run from the MLflow tracking server."""
+    if not isinstance(run, Run):
+        for r in run:
+            remove_run(r)
+        return
+
+    shutil.rmtree(get_artifact_dir(run).parent)

{hydraflow-0.4.5/tests/scripts → hydraflow-0.5.0/tests/apps}/app.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import logging
-import time
 from dataclasses import dataclass, field
 from pathlib import Path
 
@@ -41,8 +40,9 @@ def app(cfg: MySQLConfig):
         assert cfg.get("values") == [1, 2, 3]  # type: ignore
 
     hydraflow.set_experiment(prefix="_", suffix="_")
-    with hydraflow.start_run(cfg) as run:
-        log.info(f"START, {cfg.host}, {cfg.port} ")
+    with hydraflow.start_run(cfg, synchronous=True) as run:
+        msg = f"START, {cfg.host}, {cfg.port} "
+        log.info(msg)
 
         artifact_dir = hydraflow.get_artifact_dir()
         output_dir = hydraflow.get_hydra_output_dir()
@@ -52,10 +52,6 @@ def app(cfg: MySQLConfig):
         mlflow.log_text("A " + artifact_dir.as_posix(), "artifact_dir.txt")
         mlflow.log_text("B " + output_dir.as_posix(), "output_dir.txt")
 
-        # with hydraflow.watch(callback, ignore_patterns=["b.txt"]):
-        #     (artifact_dir / "a.txt").write_text("abc")
-        #     time.sleep(0.1)
-
         (artifact_dir / "a.txt").write_text("abc")
 
         mlflow.log_metric("m", cfg.port + 1, 1)
@@ -65,16 +61,10 @@ def app(cfg: MySQLConfig):
         assert hydraflow.get_overrides() == hydraflow.load_overrides(run)
 
         if cfg.host == "error":
-            raise Exception("error")
+            raise Exception("error")  # noqa: TRY002
 
         log.info("END")
 
 
-def callback(path: Path):
-    log.info(f"WATCH, {path.as_posix()}")
-    m = len(path.read_text())  # len("abc") == 3
-    # mlflow.log_metric("watch", m, 1, synchronous=True)
-
-
 if __name__ == "__main__":
     app()

hydraflow-0.5.0/tests/conftest.py

@@ -0,0 +1,16 @@
+import os
+import uuid
+from pathlib import Path
+
+import mlflow
+import pytest
+
+
+@pytest.fixture(scope="module")
+def experiment_name(tmp_path_factory: pytest.TempPathFactory):
+    cwd = Path.cwd()
+    name = str(uuid.uuid4())
+    os.chdir(tmp_path_factory.mktemp(name))
+    mlflow.set_experiment(name)
+    yield name
+    os.chdir(cwd)

{hydraflow-0.4.5 → hydraflow-0.5.0}/tests/test_app.py

@@ -1,27 +1,24 @@
-from __future__ import annotations
-
+import os
 import subprocess
 import sys
 from pathlib import Path
-from typing import TYPE_CHECKING
 
 import mlflow
 import pytest
 from mlflow.entities import RunStatus
-from omegaconf import OmegaConf
-
-if TYPE_CHECKING:
-    from omegaconf import DictConfig
+from omegaconf import DictConfig, OmegaConf
 
-    from hydraflow.run_collection import RunCollection
+from hydraflow.run_collection import RunCollection
 
 
-@pytest.fixture
-def rc(monkeypatch, tmp_path):
+@pytest.fixture(scope="module")
+def rc(tmp_path_factory: pytest.TempPathFactory):
     import hydraflow
 
-    file = Path("tests/scripts/app.py").absolute()
-    monkeypatch.chdir(tmp_path)
+    cwd = Path.cwd()
+
+    file = Path("tests/apps/app.py").absolute()
+    os.chdir(tmp_path_factory.mktemp("test_app"))
 
     args = [sys.executable, file.as_posix(), "-m"]
     args += ["host=x,y", "port=1,2", "hydra.job.name=info"]
@@ -30,6 +27,8 @@ def rc(monkeypatch, tmp_path):
     mlflow.set_experiment("_info_")
     yield hydraflow.list_runs()
 
+    os.chdir(cwd)
+
 
 def test_list_runs_all(rc: RunCollection):
     from hydraflow.mlflow import list_runs
@@ -112,7 +111,7 @@ def test_app_data_config(rc: RunCollection):
 def test_app_data_config_list(rc: RunCollection):
     config = rc.data.config
     values = config["values"].to_list()
-    assert str(config.select("values").dtypes) == "[List(Int64)]"
+    assert str(config["values"].dtypes) == "object"
     for x in values:
         assert isinstance(x, list)
         assert x == [1, 2, 3]
@@ -154,8 +153,8 @@ def test_app_map_config(rc: RunCollection):
     assert ports == [2, 3, 2, 3]
 
 
-def test_app_group_by(rc: RunCollection):
-    grouped = rc.group_by("host")
+def test_app_groupby(rc: RunCollection):
+    grouped = rc.groupby("host")
     assert len(grouped) == 2
     assert grouped["x"].data.params["port"] == ["1", "2"]
     assert grouped["x"].data.params["host"] == ["x", "x"]
@@ -165,8 +164,8 @@ def test_app_group_by(rc: RunCollection):
     assert grouped["y"].data.params["values"] == ["[1, 2, 3]", "[1, 2, 3]"]
 
 
-def test_app_group_by_list(rc: RunCollection):
-    grouped = rc.group_by(["host"])
+def test_app_groupby_list(rc: RunCollection):
+    grouped = rc.groupby(["host"])
     assert len(grouped) == 2
     assert ("x",) in grouped
     assert ("y",) in grouped
@@ -203,8 +202,7 @@ def test_log_run_error(monkeypatch, tmp_path):
     args = [sys.executable, file.as_posix()]
     args += ["host=error", "hydra.job.name=error"]
     cp = subprocess.run(args, check=False, capture_output=True)
-    assert cp.returncode == 1
-    assert b"Error during log_run: error" in cp.stdout
+    assert cp.returncode
 
 
 def test_chdir_artifact(rc: RunCollection):

{hydraflow-0.4.5 → hydraflow-0.5.0}/tests/test_context.py

@@ -1,16 +1,15 @@
-import time
 from pathlib import Path
 from unittest.mock import MagicMock, patch
 
 import mlflow
 import pytest
 
-from hydraflow.context import log_run, start_run, watch
+from hydraflow.context import log_run, start_run
 from hydraflow.run_collection import RunCollection
 
 
 @pytest.fixture
-def runs(monkeypatch, tmp_path):
+def runs(monkeypatch: pytest.MonkeyPatch, tmp_path: Path):
     from hydraflow.mlflow import list_runs
 
     monkeypatch.chdir(tmp_path)
@@ -67,22 +66,3 @@ def test_log_run_error_handling(tmp_path: Path):
         with pytest.raises(Exception, match="Test exception"):
             with log_run(config):
                 pass
-
-
-def test_watch_context_manager(tmp_path: Path):
-    test_dir = tmp_path / "test_watch"
-    test_dir.mkdir(parents=True, exist_ok=True)
-    test_file = test_dir / "test_file.txt"
-
-    called = []
-
-    def mock_func(path: Path):
-        assert path == test_file
-        called.append(path)
-
-    with watch(mock_func, test_dir):
-        test_file.write_text("new content")
-        time.sleep(1)
-
-    assert len(called) == 1
-    assert called[0] == test_file

{hydraflow-0.4.5 → hydraflow-0.5.0}/tests/test_log_run.py

@@ -1,5 +1,4 @@
-from __future__ import annotations
-
+import os
 import subprocess
 import sys
 from pathlib import Path
@@ -10,10 +9,12 @@ from mlflow.artifacts import download_artifacts
 from mlflow.entities.run import Run
 
 
-@pytest.fixture
-def runs(monkeypatch, tmp_path):
-    file = Path("tests/scripts/app.py").absolute()
-    monkeypatch.chdir(tmp_path)
+@pytest.fixture(scope="module")
+def runs(tmp_path_factory: pytest.TempPathFactory):
+    file = Path("tests/apps/app.py").absolute()
+
+    cwd = Path.cwd()
+    os.chdir(tmp_path_factory.mktemp("test_log_run"))
 
     args = [sys.executable, file.as_posix(), "-m"]
     args += ["host=x,y", "port=1,2", "hydra.job.name=log_run"]
@@ -21,16 +22,17 @@ def runs(monkeypatch, tmp_path):
 
     mlflow.set_experiment("_log_run_")
     runs = mlflow.search_runs(output_format="list")
+
     assert len(runs) == 4
     assert isinstance(runs, list)
     yield runs
 
+    os.chdir(cwd)
+
 
-@pytest.fixture(params=range(4))
-def run(runs, request):
-    run = runs[request.param]  # type: ignore
-    assert isinstance(run, Run)
-    return run
+@pytest.fixture(scope="module", params=range(4))
+def run(runs: list[Run], request: pytest.FixtureRequest):
+    return runs[request.param]
 
 
 @pytest.fixture

{hydraflow-0.4.5 → hydraflow-0.5.0}/tests/test_mlflow.py

@@ -5,7 +5,7 @@ from hydra.core.hydra_config import HydraConfig
 
 
 @pytest.fixture
-def hydra_config(monkeypatch):
+def hydra_config(monkeypatch: pytest.MonkeyPatch):
     class MockJob:
         name = "test_job"
 

{hydraflow-0.4.5 → hydraflow-0.5.0}/tests/test_param.py

@@ -1,9 +1,11 @@
+from pathlib import Path
+
 import mlflow
 import pytest
 
 
 @pytest.fixture
-def param(monkeypatch, tmp_path):
+def param(monkeypatch: pytest.MonkeyPatch, tmp_path: Path):
     def param(value):
         monkeypatch.chdir(tmp_path)
         mlflow.set_experiment("test_param")