hydraflow 0.4.6__tar.gz → 0.5.0__tar.gz

{hydraflow-0.4.6 → hydraflow-0.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydraflow
-Version: 0.4.6
+Version: 0.5.0
 Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
 Project-URL: Documentation, https://github.com/daizutabi/hydraflow
 Project-URL: Source, https://github.com/daizutabi/hydraflow
@@ -37,12 +37,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.10
 Requires-Dist: hydra-core>=1.3
-Requires-Dist: joblib
 Requires-Dist: mlflow>=2.15
-Requires-Dist: polars
-Requires-Dist: rich
-Requires-Dist: watchdog
-Requires-Dist: watchfiles
 Description-Content-Type: text/markdown
 
 # Hydraflow
@@ -119,16 +114,6 @@ def my_app(cfg: MySQLConfig) -> None:
     with hydraflow.start_run():
         # Your app code below.
 
-        with hydraflow.watch(callback):
-            # Watch files in the MLflow artifact directory.
-            # You can update metrics or log other artifacts
-            # according to the watched files in your callback
-            # function.
-            pass
-
-# Your callback function here.
-def callback(file: Path) -> None:
-    pass
 
 if __name__ == "__main__":
     my_app()

{hydraflow-0.4.6 → hydraflow-0.5.0}/README.md

@@ -72,16 +72,6 @@ def my_app(cfg: MySQLConfig) -> None:
     with hydraflow.start_run():
         # Your app code below.
 
-        with hydraflow.watch(callback):
-            # Watch files in the MLflow artifact directory.
-            # You can update metrics or log other artifacts
-            # according to the watched files in your callback
-            # function.
-            pass
-
-# Your callback function here.
-def callback(file: Path) -> None:
-    pass
 
 if __name__ == "__main__":
     my_app()

{hydraflow-0.4.6 → hydraflow-0.5.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hydraflow"
-version = "0.4.6"
+version = "0.5.0"
 description = "Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments."
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -19,15 +19,7 @@ classifiers = [
   "Programming Language :: Python :: 3.13",
 ]
 requires-python = ">=3.10"
-dependencies = [
-  "hydra-core>=1.3",
-  "joblib",
-  "mlflow>=2.15",
-  "polars",
-  "rich",
-  "watchdog",
-  "watchfiles",
-]
+dependencies = ["hydra-core>=1.3", "mlflow>=2.15"]
 
 [project.urls]
 Documentation = "https://github.com/daizutabi/hydraflow"
@@ -40,7 +32,6 @@ dev-dependencies = [
   "mkapi",
   "mkdocs-material",
   "mkdocs>=1.6",
-  "pytest-asyncio",
   "pytest-clarity",
   "pytest-cov",
   "pytest-randomly",
@@ -65,7 +56,6 @@ filterwarnings = [
   "ignore:Support for class-based `config` is deprecated",
   "ignore:Pydantic V1 style",
 ]
-asyncio_default_fixture_loop_scope = "function"
 
 [tool.coverage.report]
 exclude_lines = ["no cov", "raise NotImplementedError", "if TYPE_CHECKING:"]

{hydraflow-0.4.6 → hydraflow-0.5.0}/src/hydraflow/__init__.py

@@ -1,9 +1,8 @@
 """Integrate Hydra and MLflow to manage and track machine learning experiments."""
 
 from .config import select_config, select_overrides
-from .context import chdir_artifact, chdir_hydra_output, log_run, start_run, watch
+from .context import chdir_artifact, chdir_hydra_output, log_run, start_run
 from .mlflow import list_runs, search_runs, set_experiment
-from .progress import multi_tasks_progress, parallel_progress
 from .run_collection import RunCollection
 from .utils import (
     get_artifact_dir,
@@ -25,13 +24,10 @@ __all__ = [
     "load_config",
     "load_overrides",
     "log_run",
-    "multi_tasks_progress",
-    "parallel_progress",
     "remove_run",
     "search_runs",
     "select_config",
     "select_overrides",
     "set_experiment",
     "start_run",
-    "watch",
 ]

{hydraflow-0.4.6 → hydraflow-0.5.0}/src/hydraflow/context.py

@@ -4,7 +4,6 @@ from __future__ import annotations
 
 import logging
 import os
-import time
 from contextlib import contextmanager
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -12,14 +11,11 @@ from typing import TYPE_CHECKING
 import mlflow
 import mlflow.artifacts
 from hydra.core.hydra_config import HydraConfig
-from watchdog.events import FileModifiedEvent, PatternMatchingEventHandler
-from watchdog.observers import Observer
 
 from hydraflow.mlflow import log_params
-from hydraflow.run_info import get_artifact_dir
 
 if TYPE_CHECKING:
-    from collections.abc import Callable, Iterator
+    from collections.abc import Iterator
 
     from mlflow.entities.run import Run
 
@@ -64,14 +60,8 @@ def log_run(
     output_subdir = output_dir / (hc.output_subdir or "")
     mlflow.log_artifacts(output_subdir.as_posix(), hc.output_subdir)
 
-    def log_artifact(path: Path) -> None:
-        local_path = (output_dir / path).as_posix()
-        mlflow.log_artifact(local_path)
-
     try:
         yield
-        # with watch(log_artifact, output_dir, ignore_log=False):
-        #     yield
 
     except Exception as e:
         msg = f"Error during log_run: {e}"
@@ -146,101 +136,6 @@ def start_run(  # noqa: PLR0913
         yield run
 
 
-@contextmanager
-def watch(
-    callback: Callable[[Path], None],
-    dir: Path | str = "",  # noqa: A002
-    *,
-    timeout: int = 60,
-    ignore_patterns: list[str] | None = None,
-    ignore_log: bool = True,
-) -> Iterator[None]:
-    """Watch the given directory for changes.
-
-    This context manager sets up a file system watcher on the specified directory.
-    When a file modification is detected, the provided function is called with
-    the path of the modified file. The watcher runs for the specified timeout
-    period or until the context is exited.
-
-    Args:
-        callback (Callable[[Path], None]): The function to call when a change is
-            detected. It should accept a single argument of type `Path`,
-            which is the path of the modified file.
-        dir (Path | str): The directory to watch. If not specified,
-            the current MLflow artifact URI is used. Defaults to "".
-        timeout (int): The timeout period in seconds for the watcher
-            to run after the context is exited. Defaults to 60.
-        ignore_patterns (list[str] | None): A list of glob patterns to ignore.
-            Defaults to None.
-        ignore_log (bool): Whether to ignore log files. Defaults to True.
-
-    Yields:
-        None
-
-    Example:
-        ```python
-        with watch(log_artifact, "/path/to/dir"):
-            # Perform operations while watching the directory for changes
-            pass
-        ```
-
-    """
-    dir = dir or get_artifact_dir()  # noqa: A001
-    if isinstance(dir, Path):
-        dir = dir.as_posix()  # noqa: A001
-
-    handler = Handler(callback, ignore_patterns=ignore_patterns, ignore_log=ignore_log)
-    observer = Observer()
-    observer.schedule(handler, dir, recursive=True)
-    observer.start()
-
-    try:
-        yield
-
-    except Exception as e:
-        msg = f"Error during watch: {e}"
-        log.exception(msg)
-        raise
-
-    finally:
-        elapsed = 0
-        while not observer.event_queue.empty():
-            time.sleep(0.2)
-            elapsed += 0.2
-            if elapsed > timeout:
-                break
-
-        observer.stop()
-        observer.join()
-
-
-class Handler(PatternMatchingEventHandler):
-    """Monitor file changes and call the given function when a change is detected."""
-
-    def __init__(
-        self,
-        func: Callable[[Path], None],
-        *,
-        ignore_patterns: list[str] | None = None,
-        ignore_log: bool = True,
-    ) -> None:
-        self.func = func
-
-        if ignore_log:
-            if ignore_patterns:
-                ignore_patterns.append("*.log")
-            else:
-                ignore_patterns = ["*.log"]
-
-        super().__init__(ignore_patterns=ignore_patterns)
-
-    def on_modified(self, event: FileModifiedEvent) -> None:
-        """Modify when a file is modified."""
-        file = Path(str(event.src_path))
-        if file.is_file():
-            self.func(file)
-
-
 @contextmanager
 def chdir_hydra_output() -> Iterator[Path]:
     """Change the current working directory to the hydra output directory.

{hydraflow-0.4.6 → hydraflow-0.5.0}/src/hydraflow/run_collection.py

@@ -575,7 +575,7 @@ class RunCollection:
         """
         return (func(dir, *args, **kwargs) for dir in self.info.artifact_dir)  # noqa: A001
 
-    def group_by(
+    def groupby(
         self,
         names: str | list[str],
     ) -> dict[str | None | tuple[str | None, ...], RunCollection]:

{hydraflow-0.4.6 → hydraflow-0.5.0}/src/hydraflow/run_data.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
-from polars.dataframe import DataFrame
+from pandas import DataFrame
 
 from hydraflow.config import collect_params
 
@@ -33,10 +33,10 @@ class RunCollectionData:
 
     @property
     def config(self) -> DataFrame:
-        """Get the runs' configurations as a polars DataFrame.
+        """Get the runs' configurations as a DataFrame.
 
         Returns:
-            A polars DataFrame containing the runs' configurations.
+            A DataFrame containing the runs' configurations.
 
         """
         return DataFrame(self._runs.map_config(collect_params))

{hydraflow-0.4.6/tests/scripts → hydraflow-0.5.0/tests/apps}/app.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import logging
-import time
 from dataclasses import dataclass, field
 from pathlib import Path
 
@@ -41,8 +40,9 @@ def app(cfg: MySQLConfig):
         assert cfg.get("values") == [1, 2, 3]  # type: ignore
 
     hydraflow.set_experiment(prefix="_", suffix="_")
-    with hydraflow.start_run(cfg) as run:
-        log.info(f"START, {cfg.host}, {cfg.port} ")
+    with hydraflow.start_run(cfg, synchronous=True) as run:
+        msg = f"START, {cfg.host}, {cfg.port} "
+        log.info(msg)
 
         artifact_dir = hydraflow.get_artifact_dir()
         output_dir = hydraflow.get_hydra_output_dir()
@@ -52,10 +52,6 @@ def app(cfg: MySQLConfig):
         mlflow.log_text("A " + artifact_dir.as_posix(), "artifact_dir.txt")
         mlflow.log_text("B " + output_dir.as_posix(), "output_dir.txt")
 
-        # with hydraflow.watch(callback, ignore_patterns=["b.txt"]):
-        #     (artifact_dir / "a.txt").write_text("abc")
-        #     time.sleep(0.1)
-
         (artifact_dir / "a.txt").write_text("abc")
 
         mlflow.log_metric("m", cfg.port + 1, 1)
@@ -65,16 +61,10 @@ def app(cfg: MySQLConfig):
         assert hydraflow.get_overrides() == hydraflow.load_overrides(run)
 
         if cfg.host == "error":
-            raise Exception("error")
+            raise Exception("error")  # noqa: TRY002
 
         log.info("END")
 
 
-def callback(path: Path):
-    log.info(f"WATCH, {path.as_posix()}")
-    m = len(path.read_text())  # len("abc") == 3
-    # mlflow.log_metric("watch", m, 1, synchronous=True)
-
-
 if __name__ == "__main__":
     app()

hydraflow-0.5.0/tests/conftest.py

@@ -0,0 +1,16 @@
+import os
+import uuid
+from pathlib import Path
+
+import mlflow
+import pytest
+
+
+@pytest.fixture(scope="module")
+def experiment_name(tmp_path_factory: pytest.TempPathFactory):
+    cwd = Path.cwd()
+    name = str(uuid.uuid4())
+    os.chdir(tmp_path_factory.mktemp(name))
+    mlflow.set_experiment(name)
+    yield name
+    os.chdir(cwd)

{hydraflow-0.4.6 → hydraflow-0.5.0}/tests/test_app.py

@@ -1,20 +1,14 @@
-from __future__ import annotations
-
 import os
 import subprocess
 import sys
 from pathlib import Path
-from typing import TYPE_CHECKING
 
 import mlflow
 import pytest
 from mlflow.entities import RunStatus
-from omegaconf import OmegaConf
-
-if TYPE_CHECKING:
-    from omegaconf import DictConfig
+from omegaconf import DictConfig, OmegaConf
 
-    from hydraflow.run_collection import RunCollection
+from hydraflow.run_collection import RunCollection
 
 
 @pytest.fixture(scope="module")
@@ -23,7 +17,7 @@ def rc(tmp_path_factory: pytest.TempPathFactory):
 
     cwd = Path.cwd()
 
-    file = Path("tests/scripts/app.py").absolute()
+    file = Path("tests/apps/app.py").absolute()
     os.chdir(tmp_path_factory.mktemp("test_app"))
 
     args = [sys.executable, file.as_posix(), "-m"]
@@ -117,7 +111,7 @@ def test_app_data_config(rc: RunCollection):
 def test_app_data_config_list(rc: RunCollection):
     config = rc.data.config
     values = config["values"].to_list()
-    assert str(config.select("values").dtypes) == "[List(Int64)]"
+    assert str(config["values"].dtypes) == "object"
     for x in values:
         assert isinstance(x, list)
         assert x == [1, 2, 3]
@@ -159,8 +153,8 @@ def test_app_map_config(rc: RunCollection):
     assert ports == [2, 3, 2, 3]
 
 
-def test_app_group_by(rc: RunCollection):
-    grouped = rc.group_by("host")
+def test_app_groupby(rc: RunCollection):
+    grouped = rc.groupby("host")
     assert len(grouped) == 2
     assert grouped["x"].data.params["port"] == ["1", "2"]
     assert grouped["x"].data.params["host"] == ["x", "x"]
@@ -170,8 +164,8 @@ def test_app_group_by(rc: RunCollection):
     assert grouped["y"].data.params["values"] == ["[1, 2, 3]", "[1, 2, 3]"]
 
 
-def test_app_group_by_list(rc: RunCollection):
-    grouped = rc.group_by(["host"])
+def test_app_groupby_list(rc: RunCollection):
+    grouped = rc.groupby(["host"])
     assert len(grouped) == 2
     assert ("x",) in grouped
     assert ("y",) in grouped

{hydraflow-0.4.6 → hydraflow-0.5.0}/tests/test_context.py

@@ -1,16 +1,15 @@
-import time
 from pathlib import Path
 from unittest.mock import MagicMock, patch
 
 import mlflow
 import pytest
 
-from hydraflow.context import log_run, start_run, watch
+from hydraflow.context import log_run, start_run
 from hydraflow.run_collection import RunCollection
 
 
 @pytest.fixture
-def runs(monkeypatch, tmp_path):
+def runs(monkeypatch: pytest.MonkeyPatch, tmp_path: Path):
     from hydraflow.mlflow import list_runs
 
     monkeypatch.chdir(tmp_path)
@@ -67,22 +66,3 @@ def test_log_run_error_handling(tmp_path: Path):
         with pytest.raises(Exception, match="Test exception"):
             with log_run(config):
                 pass
-
-
-def test_watch_context_manager(tmp_path: Path):
-    test_dir = tmp_path / "test_watch"
-    test_dir.mkdir(parents=True, exist_ok=True)
-    test_file = test_dir / "test_file.txt"
-
-    called = []
-
-    def mock_func(path: Path):
-        assert path == test_file
-        called.append(path)
-
-    with watch(mock_func, test_dir):
-        test_file.write_text("new content")
-        time.sleep(1)
-
-    assert len(called) == 1
-    assert called[0] == test_file

{hydraflow-0.4.6 → hydraflow-0.5.0}/tests/test_log_run.py

@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 import os
 import subprocess
 import sys
@@ -13,7 +11,7 @@ from mlflow.entities.run import Run
 
 @pytest.fixture(scope="module")
 def runs(tmp_path_factory: pytest.TempPathFactory):
-    file = Path("tests/scripts/app.py").absolute()
+    file = Path("tests/apps/app.py").absolute()
 
     cwd = Path.cwd()
     os.chdir(tmp_path_factory.mktemp("test_log_run"))
@@ -32,11 +30,9 @@ def runs(tmp_path_factory: pytest.TempPathFactory):
     os.chdir(cwd)
 
 
-@pytest.fixture(params=range(4))
-def run(runs, request):
-    run = runs[request.param]  # type: ignore
-    assert isinstance(run, Run)
-    return run
+@pytest.fixture(scope="module", params=range(4))
+def run(runs: list[Run], request: pytest.FixtureRequest):
+    return runs[request.param]
 
 
 @pytest.fixture

{hydraflow-0.4.6 → hydraflow-0.5.0}/tests/test_mlflow.py

@@ -5,7 +5,7 @@ from hydra.core.hydra_config import HydraConfig
 
 
 @pytest.fixture
-def hydra_config(monkeypatch):
+def hydra_config(monkeypatch: pytest.MonkeyPatch):
     class MockJob:
         name = "test_job"
 

{hydraflow-0.4.6 → hydraflow-0.5.0}/tests/test_param.py

@@ -1,9 +1,11 @@
+from pathlib import Path
+
 import mlflow
 import pytest
 
 
 @pytest.fixture
-def param(monkeypatch, tmp_path):
+def param(monkeypatch: pytest.MonkeyPatch, tmp_path: Path):
     def param(value):
         monkeypatch.chdir(tmp_path)
         mlflow.set_experiment("test_param")