hydraflow 0.2.1__tar.gz → 0.2.3__tar.gz

{hydraflow-0.2.1 → hydraflow-0.2.3}/.devcontainer/devcontainer.json

@@ -7,7 +7,12 @@
   },
   "customizations": {
     "vscode": {
-      "extensions": ["charliermarsh.ruff"]
+      "extensions": [
+        "charliermarsh.ruff",
+        "henriiik.vscode-sort",
+        "ms-python.python",
+        "ms-python.vscode-pylance"
+      ]
     }
   },
   "postCreateCommand": ".devcontainer/postCreate.sh"

{hydraflow-0.2.1 → hydraflow-0.2.3}/.gitignore

@@ -1,5 +1,6 @@
 .coverage
+.env
 .venv/
 __pycache__/
-lcov.info
-dist/
+dist/
+lcov.info

{hydraflow-0.2.1 → hydraflow-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: hydraflow
-Version: 0.2.1
+Version: 0.2.3
 Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
 Project-URL: Documentation, https://github.com/daizutabi/hydraflow
 Project-URL: Source, https://github.com/daizutabi/hydraflow
@@ -20,7 +20,9 @@ Requires-Dist: hydra-core>1.3
 Requires-Dist: mlflow>2.15
 Requires-Dist: setuptools
 Requires-Dist: watchdog
+Requires-Dist: watchfiles
 Provides-Extra: dev
+Requires-Dist: pytest-asyncio; extra == 'dev'
 Requires-Dist: pytest-clarity; extra == 'dev'
 Requires-Dist: pytest-cov; extra == 'dev'
 Requires-Dist: pytest-randomly; extra == 'dev'
@@ -46,14 +48,23 @@ Description-Content-Type: text/markdown
 
 ## Overview
 
-Hydraflow is a powerful library designed to seamlessly integrate [Hydra](https://hydra.cc/) and [MLflow](https://mlflow.org/), making it easier to manage and track machine learning experiments. By combining the flexibility of Hydra's configuration management with the robust experiment tracking capabilities of MLflow, Hydraflow provides a comprehensive solution for managing complex machine learning workflows.
+Hydraflow is a powerful library designed to seamlessly integrate
+[Hydra](https://hydra.cc/) and [MLflow](https://mlflow.org/), making it easier to
+manage and track machine learning experiments. By combining the flexibility of
+Hydra's configuration management with the robust experiment tracking capabilities
+of MLflow, Hydraflow provides a comprehensive solution for managing complex
+machine learning workflows.
 
 ## Key Features
 
-- **Configuration Management**: Utilize Hydra's advanced configuration management to handle complex parameter sweeps and experiment setups.
-- **Experiment Tracking**: Leverage MLflow's tracking capabilities to log parameters, metrics, and artifacts for each run.
-- **Artifact Management**: Automatically log and manage artifacts, such as model checkpoints and configuration files, with MLflow.
-- **Seamless Integration**: Easily integrate Hydra and MLflow in your machine learning projects with minimal setup.
+- **Configuration Management**: Utilize Hydra's advanced configuration management
+  to handle complex parameter sweeps and experiment setups.
+- **Experiment Tracking**: Leverage MLflow's tracking capabilities to log parameters,
+  metrics, and artifacts for each run.
+- **Artifact Management**: Automatically log and manage artifacts, such as model
+  checkpoints and configuration files, with MLflow.
+- **Seamless Integration**: Easily integrate Hydra and MLflow in your machine learning
+  projects with minimal setup.
 
 ## Installation
 

{hydraflow-0.2.1 → hydraflow-0.2.3}/README.md

@@ -17,14 +17,23 @@
 
 ## Overview
 
-Hydraflow is a powerful library designed to seamlessly integrate [Hydra](https://hydra.cc/) and [MLflow](https://mlflow.org/), making it easier to manage and track machine learning experiments. By combining the flexibility of Hydra's configuration management with the robust experiment tracking capabilities of MLflow, Hydraflow provides a comprehensive solution for managing complex machine learning workflows.
+Hydraflow is a powerful library designed to seamlessly integrate
+[Hydra](https://hydra.cc/) and [MLflow](https://mlflow.org/), making it easier to
+manage and track machine learning experiments. By combining the flexibility of
+Hydra's configuration management with the robust experiment tracking capabilities
+of MLflow, Hydraflow provides a comprehensive solution for managing complex
+machine learning workflows.
 
 ## Key Features
 
-- **Configuration Management**: Utilize Hydra's advanced configuration management to handle complex parameter sweeps and experiment setups.
-- **Experiment Tracking**: Leverage MLflow's tracking capabilities to log parameters, metrics, and artifacts for each run.
-- **Artifact Management**: Automatically log and manage artifacts, such as model checkpoints and configuration files, with MLflow.
-- **Seamless Integration**: Easily integrate Hydra and MLflow in your machine learning projects with minimal setup.
+- **Configuration Management**: Utilize Hydra's advanced configuration management
+  to handle complex parameter sweeps and experiment setups.
+- **Experiment Tracking**: Leverage MLflow's tracking capabilities to log parameters,
+  metrics, and artifacts for each run.
+- **Artifact Management**: Automatically log and manage artifacts, such as model
+  checkpoints and configuration files, with MLflow.
+- **Seamless Integration**: Easily integrate Hydra and MLflow in your machine learning
+  projects with minimal setup.
 
 ## Installation
 

{hydraflow-0.2.1 → hydraflow-0.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hydraflow"
-version = "0.2.1"
+version = "0.2.3"
 description = "Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments."
 readme = "README.md"
 license = "MIT"
@@ -19,10 +19,22 @@ classifiers = [
   "Topic :: Software Development :: Documentation",
 ]
 requires-python = ">=3.10"
-dependencies = ["hydra-core>1.3", "mlflow>2.15", "setuptools", "watchdog"]
+dependencies = [
+  "hydra-core>1.3",
+  "mlflow>2.15",
+  "setuptools",
+  "watchdog",
+  "watchfiles",
+]
 
 [project.optional-dependencies]
-dev = ["pytest-clarity", "pytest-cov", "pytest-randomly", "pytest-xdist"]
+dev = [
+  "pytest-asyncio",
+  "pytest-clarity",
+  "pytest-cov",
+  "pytest-randomly",
+  "pytest-xdist",
+]
 
 [project.urls]
 Documentation = "https://github.com/daizutabi/hydraflow"
@@ -41,18 +53,13 @@ addopts = [
   "--cov=hydraflow",
   "--cov-report=lcov:lcov.info",
 ]
-
 doctest_optionflags = ["NORMALIZE_WHITESPACE", "IGNORE_EXCEPTION_DETAIL"]
 filterwarnings = ['ignore:pkg_resources is deprecated:DeprecationWarning']
+asyncio_default_fixture_loop_scope = "function"
 
 [tool.coverage.report]
 exclude_lines = ["no cov", "raise NotImplementedError", "if TYPE_CHECKING:"]
 
-[tool.hatch.envs.docs.scripts]
-build = "mkdocs build --clean --strict {args}"
-serve = "mkdocs serve --dev-addr localhost:8000 {args}"
-deploy = "mkdocs gh-deploy --force"
-
 [tool.ruff]
 line-length = 100
 target-version = "py312"

{hydraflow-0.2.1 → hydraflow-0.2.3}/src/hydraflow/__init__.py

@@ -2,25 +2,29 @@ from .context import Info, chdir_artifact, log_run, watch
 from .mlflow import set_experiment
 from .runs import (
     Run,
-    Runs,
+    RunCollection,
     filter_runs,
     get_param_dict,
     get_param_names,
     get_run,
+    list_runs,
     load_config,
+    search_runs,
 )
 
 __all__ = [
     "Info",
     "Run",
-    "Runs",
+    "RunCollection",
     "chdir_artifact",
     "filter_runs",
     "get_param_dict",
     "get_param_names",
     "get_run",
+    "list_runs",
     "load_config",
     "log_run",
+    "search_runs",
     "set_experiment",
     "watch",
 ]

hydraflow-0.2.3/src/hydraflow/asyncio.py

@@ -0,0 +1,199 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+from asyncio.subprocess import PIPE
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import watchfiles
+
+if TYPE_CHECKING:
+    from asyncio.streams import StreamReader
+    from collections.abc import Callable
+
+    from watchfiles import Change
+
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+async def execute_command(
+    program: str,
+    *args: str,
+    stdout: Callable[[str], None] | None = None,
+    stderr: Callable[[str], None] | None = None,
+    stop_event: asyncio.Event,
+) -> int:
+    """
+    Runs a command asynchronously and pass the output to callback functions.
+
+    Args:
+        program (str): The program to run.
+        *args (str): Arguments for the program.
+        stdout (Callable[[str], None] | None): Callback for standard output.
+        stderr (Callable[[str], None] | None): Callback for standard error.
+        stop_event (asyncio.Event): Event to signal when the process is done.
+
+    Returns:
+        int: The return code of the process.
+    """
+    try:
+        process = await asyncio.create_subprocess_exec(program, *args, stdout=PIPE, stderr=PIPE)
+        await asyncio.gather(
+            process_stream(process.stdout, stdout),
+            process_stream(process.stderr, stderr),
+        )
+        returncode = await process.wait()
+
+    except Exception as e:
+        logger.error(f"Error running command: {e}")
+        returncode = 1
+
+    finally:
+        stop_event.set()
+
+    return returncode
+
+
+async def process_stream(
+    stream: StreamReader | None,
+    callback: Callable[[str], None] | None,
+) -> None:
+    """
+    Reads a stream asynchronously and pass each line to a callback function.
+
+    Args:
+        stream (StreamReader | None): The stream to read from.
+        callback (Callable[[str], None] | None): The callback function to handle
+        each line.
+    """
+    if stream is None or callback is None:
+        return
+
+    while True:
+        line = await stream.readline()
+        if line:
+            callback(line.decode().strip())
+        else:
+            break
+
+
+async def monitor_file_changes(
+    paths: list[str | Path],
+    callback: Callable[[set[tuple[Change, str]]], None],
+    stop_event: asyncio.Event,
+    **awatch_kwargs,
+) -> None:
+    """
+    Watches for file changes in specified paths and pass the changes to a
+    callback function.
+
+    Args:
+        paths (list[str | Path]): List of paths to monitor for changes.
+        callback (Callable[[set[tuple[Change, str]]], None]): The callback
+        function to handle file changes.
+        stop_event (asyncio.Event): Event to signal when to stop watching.
+        **awatch_kwargs: Additional keyword arguments to pass to watchfiles.awatch.
+    """
+    str_paths = [str(path) for path in paths]
+    try:
+        async for changes in watchfiles.awatch(*str_paths, stop_event=stop_event, **awatch_kwargs):
+            callback(changes)
+    except Exception as e:
+        logger.error(f"Error watching files: {e}")
+
+
+async def run_and_monitor(
+    program: str,
+    *args: str,
+    stdout: Callable[[str], None] | None = None,
+    stderr: Callable[[str], None] | None = None,
+    watch: Callable[[set[tuple[Change, str]]], None] | None = None,
+    paths: list[str | Path] | None = None,
+    **awatch_kwargs,
+) -> int:
+    """
+    Runs a command and optionally watch for file changes concurrently.
+
+    Args:
+        program (str): The program to run.
+        *args (str): Arguments for the program.
+        stdout (Callable[[str], None] | None): Callback for standard output.
+        stderr (Callable[[str], None] | None): Callback for standard error.
+        watch (Callable[[set[tuple[Change, str]]], None] | None): Callback for
+        file changes.
+        paths (list[str | Path] | None): List of paths to monitor for changes.
+    """
+    stop_event = asyncio.Event()
+    run_task = asyncio.create_task(
+        execute_command(program, *args, stop_event=stop_event, stdout=stdout, stderr=stderr)
+    )
+    if watch and paths:
+        monitor_task = asyncio.create_task(
+            monitor_file_changes(paths, watch, stop_event, **awatch_kwargs)
+        )
+    else:
+        monitor_task = None
+
+    try:
+        if monitor_task:
+            await asyncio.gather(run_task, monitor_task)
+        else:
+            await run_task
+
+    except Exception as e:
+        logger.error(f"Error in run_and_monitor: {e}")
+    finally:
+        stop_event.set()
+        await run_task
+        if monitor_task:
+            await monitor_task
+
+    return run_task.result()
+
+
+def run(
+    program: str,
+    *args: str,
+    stdout: Callable[[str], None] | None = None,
+    stderr: Callable[[str], None] | None = None,
+    watch: Callable[[set[tuple[Change, str]]], None] | None = None,
+    paths: list[str | Path] | None = None,
+    **awatch_kwargs,
+) -> int:
+    """
+    Run a command synchronously and optionally watch for file changes.
+
+    This function is a synchronous wrapper around the asynchronous `run_and_monitor` function.
+    It runs a specified command and optionally monitors specified paths for file changes,
+    invoking the provided callbacks for standard output, standard error, and file changes.
+
+    Args:
+        program (str): The program to run.
+        *args (str): Arguments for the program.
+        stdout (Callable[[str], None] | None): Callback for handling standard output lines.
+        stderr (Callable[[str], None] | None): Callback for handling standard error lines.
+        watch (Callable[[set[tuple[Change, str]]], None] | None): Callback for handling file changes.
+        paths (list[str | Path] | None): List of paths to monitor for file changes.
+        **awatch_kwargs: Additional keyword arguments to pass to `watchfiles.awatch`.
+
+    Returns:
+        int: The return code of the process.
+    """
+    if watch and not paths:
+        paths = [Path.cwd()]
+
+    return asyncio.run(
+        run_and_monitor(
+            program,
+            *args,
+            stdout=stdout,
+            stderr=stderr,
+            watch=watch,
+            paths=paths,
+            **awatch_kwargs,
+        )
+    )

{hydraflow-0.2.1 → hydraflow-0.2.3}/src/hydraflow/config.py

@@ -30,6 +30,9 @@ def iter_params(config: object, prefix: str = "") -> Iterator[tuple[str, Any]]:
     Yields:
         Key-value pairs representing the parameters in the configuration object.
     """
+    if config is None:
+        return
+
     if not isinstance(config, (DictConfig, ListConfig)):
         config = OmegaConf.create(config)  # type: ignore