hydraflow 0.7.5__tar.gz → 0.9.0__tar.gz

{hydraflow-0.7.5 → hydraflow-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydraflow
-Version: 0.7.5
+Version: 0.9.0
 Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
 Project-URL: Documentation, https://daizutabi.github.io/hydraflow/
 Project-URL: Source, https://github.com/daizutabi/hydraflow
@@ -41,6 +41,7 @@ Requires-Dist: mlflow>=2.15
 Requires-Dist: omegaconf
 Requires-Dist: rich
 Requires-Dist: typer
+Requires-Dist: ulid
 Description-Content-Type: text/markdown
 
 # Hydraflow
@@ -93,31 +94,29 @@ pip install hydraflow
 Here is a simple example to get you started with Hydraflow:
 
 ```python
-import hydra
-import hydraflow
-import mlflow
+from __future__ import annotations
+
 from dataclasses import dataclass
-from hydra.core.config_store import ConfigStore
 from pathlib import Path
+from typing import TYPE_CHECKING
 
-@dataclass
-class MySQLConfig:
-    host: str = "localhost"
-    port: int = 3306
+import hydraflow
 
-cs = ConfigStore.instance()
-cs.store(name="config", node=MySQLConfig)
+if TYPE_CHECKING:
+    from mlflow.entities import Run
+
+
+@dataclass
+class Config:
+    count: int = 1
+    name: str = "a"
 
-@hydra.main(version_base=None, config_name="config")
-def my_app(cfg: MySQLConfig) -> None:
-    # Set experiment by Hydra job name.
-    hydraflow.set_experiment()
 
-    # Automatically log Hydra config as params.
-    with hydraflow.start_run(cfg):
-        # Your app code below.
+@hydraflow.main(Config)
+def app(run: Run, cfg: Config):
+    """Your app code here."""
 
 
 if __name__ == "__main__":
-    my_app()
+    app()
 ```

{hydraflow-0.7.5 → hydraflow-0.9.0}/README.md

@@ -48,31 +48,29 @@ pip install hydraflow
 Here is a simple example to get you started with Hydraflow:
 
 ```python
-import hydra
-import hydraflow
-import mlflow
+from __future__ import annotations
+
 from dataclasses import dataclass
-from hydra.core.config_store import ConfigStore
 from pathlib import Path
+from typing import TYPE_CHECKING
 
-@dataclass
-class MySQLConfig:
-    host: str = "localhost"
-    port: int = 3306
+import hydraflow
 
-cs = ConfigStore.instance()
-cs.store(name="config", node=MySQLConfig)
+if TYPE_CHECKING:
+    from mlflow.entities import Run
+
+
+@dataclass
+class Config:
+    count: int = 1
+    name: str = "a"
 
-@hydra.main(version_base=None, config_name="config")
-def my_app(cfg: MySQLConfig) -> None:
-    # Set experiment by Hydra job name.
-    hydraflow.set_experiment()
 
-    # Automatically log Hydra config as params.
-    with hydraflow.start_run(cfg):
-        # Your app code below.
+@hydraflow.main(Config)
+def app(run: Run, cfg: Config):
+    """Your app code here."""
 
 
 if __name__ == "__main__":
-    my_app()
+    app()
 ```

{hydraflow-0.7.5 → hydraflow-0.9.0}/apps/quickstart.py

@@ -2,7 +2,9 @@ import logging
 from dataclasses import dataclass
 
 import hydra
+import mlflow
 from hydra.core.config_store import ConfigStore
+from hydra.core.hydra_config import HydraConfig
 
 import hydraflow
 
@@ -19,9 +21,10 @@ cs = ConfigStore.instance()
 cs.store(name="config", node=Config)
 
 
-@hydra.main(version_base=None, config_name="config")
+@hydra.main(config_name="config", version_base=None)
 def app(cfg: Config) -> None:
-    hydraflow.set_experiment()
+    hc = HydraConfig.get()
+    mlflow.set_experiment(hc.job.name)
 
     with hydraflow.start_run(cfg):
         log.info(f"{cfg.width=}, {cfg.height=}")

{hydraflow-0.7.5 → hydraflow-0.9.0}/docs/usage/quickstart.md

@@ -12,16 +12,6 @@ There are two main steps to using Hydraflow:
 --8<-- "apps/quickstart.py"
 ```
 
-### Set the MLflow experiment
-
-[`hydraflow.set_experiment`][] sets the MLflow experiment using the Hydra job name.
-Optionally, it can also set the tracking URI with `uri` argument.
-For example,
-
-```python
-hydraflow.set_experiment(uri="sqlite:///mlruns.db")
-```
-
 ### Start a new MLflow run
 
 [`hydraflow.start_run`][] starts a new MLflow run that logs the Hydra configuration.
@@ -64,10 +54,8 @@ $ python apps/quickstart.py -m width=400,600 height=100,200,300
 ### Run collection
 
 ```pycon exec="1" source="console" session="quickstart"
->>> import mlflow
->>> mlflow.set_experiment("quickstart")
 >>> import hydraflow
->>> rc = hydraflow.list_runs()
+>>> rc = hydraflow.list_runs("quickstart")
 >>> print(rc)
 ```
 
@@ -107,28 +95,6 @@ $ python apps/quickstart.py -m width=400,600 height=100,200,300
 >>> print(filtered)
 ```
 
-```pycon exec="1" source="console" session="quickstart"
->>> run = rc.find(height=100)
->>> print(run.data.params)
-```
-
-```pycon exec="1" source="console" session="quickstart"
->>> run = rc.find_last(height=100)
->>> print(run.data.params)
-```
-
-### Map runs
-
-```pycon exec="1" source="console" session="quickstart"
->>> params = rc.map(lambda x: x.data.params)
->>> for p in params:
-...     print(p)
-```
-
-```pycon exec="1" source="console" session="quickstart"
->>> list(rc.map_id(print))
-```
-
 ### Group runs
 
 ```pycon exec="1" source="console" session="quickstart"

{hydraflow-0.7.5 → hydraflow-0.9.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hydraflow"
-version = "0.7.5"
+version = "0.9.0"
 description = "Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments."
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -19,7 +19,14 @@ classifiers = [
   "Programming Language :: Python :: 3.13",
 ]
 requires-python = ">=3.10"
-dependencies = ["hydra-core>=1.3", "mlflow>=2.15", "omegaconf", "rich", "typer"]
+dependencies = [
+  "hydra-core>=1.3",
+  "mlflow>=2.15",
+  "omegaconf",
+  "rich",
+  "typer",
+  "ulid",
+]
 
 [project.urls]
 Documentation = "https://daizutabi.github.io/hydraflow/"
@@ -44,6 +51,7 @@ addopts = [
   "--cov=hydraflow",
   "--cov-report=lcov:lcov.info",
   "--dist=loadgroup",
+  "--doctest-modules",
   "-n8",
 ]
 filterwarnings = [
@@ -67,29 +75,28 @@ ignore = [
   "ANN003",
   "ANN401",
   "B904",
+  "D104",
   "D105",
   "D107",
   "D203",
   "D213",
   "EM101",
+  "FBT001",
+  "FBT002",
   "PGH003",
+  "PLR0911",
+  "PLR0913",
   "PLR1704",
+  "PLR2004",
+  "S603",
+  "SIM102",
+  "SIM108",
   "TRY003",
 ]
 
 [tool.ruff.lint.per-file-ignores]
-"tests/*" = [
-  "A001",
-  "ANN",
-  "ARG",
-  "D",
-  "FBT",
-  "PLR",
-  "PT",
-  "S",
-  "SIM108",
-  "SLF",
-]
 "apps/*.py" = ["D", "G", "INP"]
-"src/hydraflow/main.py" = ["ANN201", "D401", "PLR0913"]
 "src/hydraflow/cli.py" = ["ANN", "D"]
+"src/hydraflow/core/main.py" = ["ANN201", "D401"]
+"src/hydraflow/executor/conf.py" = ["ANN", "D"]
+"tests/*" = ["A001", "ANN", "ARG", "D", "FBT", "PD", "PLR", "PT", "S", "SLF"]

hydraflow-0.9.0/src/hydraflow/__init__.py

@@ -0,0 +1,29 @@
+"""Integrate Hydra and MLflow to manage and track machine learning experiments."""
+
+from hydraflow.core.context import chdir_artifact, log_run, start_run
+from hydraflow.core.io import (
+    get_artifact_dir,
+    get_artifact_path,
+    get_hydra_output_dir,
+    load_config,
+    remove_run,
+)
+from hydraflow.core.main import main
+from hydraflow.core.mlflow import list_run_ids, list_run_paths, list_runs
+from hydraflow.entities.run_collection import RunCollection
+
+__all__ = [
+    "RunCollection",
+    "chdir_artifact",
+    "get_artifact_dir",
+    "get_artifact_path",
+    "get_hydra_output_dir",
+    "list_run_ids",
+    "list_run_paths",
+    "list_runs",
+    "load_config",
+    "log_run",
+    "main",
+    "remove_run",
+    "start_run",
+]

hydraflow-0.9.0/src/hydraflow/cli.py

@@ -0,0 +1,67 @@
+"""Hydraflow CLI."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Annotated
+
+import typer
+from rich.console import Console
+from typer import Argument, Option
+
+from hydraflow.executor.io import load_config
+
+if TYPE_CHECKING:
+    from hydraflow.executor.job import Job
+
+app = typer.Typer(add_completion=False)
+console = Console()
+
+
+def get_job(name: str) -> Job:
+    cfg = load_config()
+    job = cfg.jobs[name]
+
+    if not job.name:
+        job.name = name
+
+    return job
+
+
+@app.command()
+def run(
+    name: Annotated[str, Argument(help="Job name.", show_default=False)],
+) -> None:
+    """Run a job."""
+    import mlflow
+
+    from hydraflow.executor.job import multirun
+
+    job = get_job(name)
+    mlflow.set_experiment(job.name)
+    multirun(job)
+
+
+@app.command()
+def show(
+    name: Annotated[str, Argument(help="Job name.", show_default=False)],
+) -> None:
+    """Show a job."""
+    from hydraflow.executor.job import show
+
+    job = get_job(name)
+    show(job)
+
+
+@app.callback(invoke_without_command=True)
+def callback(
+    *,
+    version: Annotated[
+        bool,
+        Option("--version", help="Show the version and exit."),
+    ] = False,
+) -> None:
+    if version:
+        import importlib.metadata
+
+        typer.echo(f"hydraflow {importlib.metadata.version('hydraflow')}")
+        raise typer.Exit

{hydraflow-0.7.5/src/hydraflow → hydraflow-0.9.0/src/hydraflow/core}/config.py

@@ -6,35 +6,19 @@ from typing import TYPE_CHECKING
 
 from omegaconf import DictConfig, ListConfig, OmegaConf
 
-from hydraflow.utils import get_overrides
-
 if TYPE_CHECKING:
     from collections.abc import Iterator
     from typing import Any
 
 
-def collect_params(config: object) -> dict[str, Any]:
-    """Iterate over parameters and collect them into a dictionary.
-
-    Args:
-        config (object): The configuration object to iterate over.
-        prefix (str): The prefix to prepend to the parameter keys.
-
-    Returns:
-        dict[str, Any]: A dictionary of collected parameters.
-
-    """
-    return dict(iter_params(config))
-
-
-def iter_params(config: object, prefix: str = "") -> Iterator[tuple[str, Any]]:
+def iter_params(config: Any, prefix: str = "") -> Iterator[tuple[str, Any]]:
     """Recursively iterate over the parameters in the given configuration object.
 
     This function traverses the configuration object and yields key-value pairs
     representing the parameters. The keys are prefixed with the provided prefix.
 
     Args:
-        config (object): The configuration object to iterate over. This can be a
+        config (Any): The configuration object to iterate over. This can be a
             dictionary, list, DictConfig, or ListConfig.
         prefix (str): The prefix to prepend to the parameter keys.
             Defaults to an empty string.
@@ -50,7 +34,7 @@ def iter_params(config: object, prefix: str = "") -> Iterator[tuple[str, Any]]:
         config = _from_dotlist(config)
 
     if not isinstance(config, DictConfig | ListConfig):
-        config = OmegaConf.create(config)  # type: ignore
+        config = OmegaConf.create(config)
 
     yield from _iter_params(config, prefix)
 
@@ -65,7 +49,7 @@ def _from_dotlist(config: list[str]) -> dict[str, str]:
     return result
 
 
-def _iter_params(config: object, prefix: str = "") -> Iterator[tuple[str, Any]]:
+def _iter_params(config: Any, prefix: str = "") -> Iterator[tuple[str, Any]]:
     if isinstance(config, DictConfig):
         for key, value in config.items():
             if _is_param(value):
@@ -83,12 +67,12 @@ def _iter_params(config: object, prefix: str = "") -> Iterator[tuple[str, Any]]:
                 yield from _iter_params(value, f"{prefix}{index}.")
 
 
-def _is_param(value: object) -> bool:
+def _is_param(value: Any) -> bool:
     """Check if the given value is a parameter."""
     if isinstance(value, DictConfig):
         return False
 
-    if isinstance(value, ListConfig):  # noqa: SIM102
+    if isinstance(value, ListConfig):
         if any(isinstance(v, DictConfig | ListConfig) for v in value):
             return False
 
@@ -103,14 +87,14 @@ def _convert(value: Any) -> Any:
     return value
 
 
-def select_config(config: object, names: list[str]) -> dict[str, Any]:
+def select_config(config: Any, names: list[str]) -> dict[str, Any]:
     """Select the given parameters from the configuration object.
 
     This function selects the given parameters from the configuration object
     and returns a new configuration object containing only the selected parameters.
 
     Args:
-        config (object): The configuration object to select parameters from.
+        config (Any): The configuration object to select parameters from.
         names (list[str]): The names of the parameters to select.
 
     Returns:
@@ -120,7 +104,7 @@ def select_config(config: object, names: list[str]) -> dict[str, Any]:
     if not isinstance(config, DictConfig):
         config = OmegaConf.structured(config)
 
-    return {name: _get(config, name) for name in names}  # type: ignore
+    return {name: _get(config, name) for name in names}
 
 
 def _get(config: DictConfig, name: str) -> Any:
@@ -132,8 +116,7 @@ def _get(config: DictConfig, name: str) -> Any:
     return _get(config.get(prefix), name)
 
 
-def select_overrides(config: object) -> dict[str, Any]:
+def select_overrides(config: object, overrides: list[str]) -> dict[str, Any]:
     """Select the given overrides from the configuration object."""
-    overrides = get_overrides()
     names = [override.split("=")[0].strip() for override in overrides]
     return select_config(config, names)

{hydraflow-0.7.5/src/hydraflow → hydraflow-0.9.0/src/hydraflow/core}/context.py

@@ -12,8 +12,9 @@ import mlflow
 import mlflow.artifacts
 from hydra.core.hydra_config import HydraConfig
 
-from hydraflow.mlflow import log_params
-from hydraflow.utils import get_artifact_dir
+from hydraflow.core.io import get_artifact_dir
+
+from .mlflow import log_params, log_text
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
@@ -55,11 +56,11 @@ def log_run(
         log_params(config, synchronous=synchronous)
 
     hc = HydraConfig.get()
-    output_dir = Path(hc.runtime.output_dir)
+    hydra_dir = Path(hc.runtime.output_dir)
 
     # Save '.hydra' config directory.
-    output_subdir = output_dir / (hc.output_subdir or "")
-    mlflow.log_artifacts(output_subdir.as_posix(), hc.output_subdir)
+    hydra_subdir = hydra_dir / (hc.output_subdir or "")
+    mlflow.log_artifacts(hydra_subdir.as_posix(), hc.output_subdir)
 
     try:
         yield
@@ -70,43 +71,14 @@ def log_run(
         raise
 
     finally:
-        log_text(output_dir)
-
-
-def log_text(directory: Path, pattern: str = "*.log") -> None:
-    """Log text files in the given directory as artifacts.
-
-    Append the text files to the existing text file in the artifact directory.
-
-    Args:
-        directory (Path): The directory to find the logs in.
-        pattern (str): The pattern to match the logs.
-
-    """
-    artifact_dir = get_artifact_dir()
-
-    for file in directory.glob(pattern):
-        if not file.is_file():
-            continue
-
-        file_artifact = artifact_dir / file.name
-        if file_artifact.exists():
-            text = file_artifact.read_text()
-            if not text.endswith("\n"):
-                text += "\n"
-        else:
-            text = ""
-
-        text += file.read_text()
-        mlflow.log_text(text, file.name)
+        log_text(hydra_dir)
 
 
 @contextmanager
-def start_run(  # noqa: PLR0913
+def start_run(
     config: object,
     *,
     chdir: bool = False,
-    run: Run | None = None,
     run_id: str | None = None,
     experiment_id: str | None = None,
     run_name: str | None = None,
@@ -126,7 +98,6 @@ def start_run(  # noqa: PLR0913
         config (object): The configuration object to log parameters from.
         chdir (bool): Whether to change the current working directory to the
             artifact directory of the current run. Defaults to False.
-        run (Run | None): The existing run. Defaults to None.
         run_id (str | None): The existing run ID. Defaults to None.
         experiment_id (str | None): The experiment ID. Defaults to None.
         run_name (str | None): The name of the run. Defaults to None.
@@ -142,20 +113,7 @@ def start_run(  # noqa: PLR0913
     Yields:
         Run: An MLflow Run object representing the started run.
 
-    Example:
-        with start_run(config) as run:
-            # Perform operations within the MLflow run context
-            pass
-
-    See Also:
-        - `mlflow.start_run`: The MLflow function to start a run directly.
-        - `log_run`: A context manager to log parameters and manage the MLflow
-           run context.
-
     """
-    if run:
-        run_id = run.info.run_id
-
     with (
         mlflow.start_run(
             run_id=run_id,

hydraflow-0.7.5/src/hydraflow/utils.py → hydraflow-0.9.0/src/hydraflow/core/io.py

@@ -12,46 +12,42 @@ import mlflow
 import mlflow.artifacts
 from hydra.core.hydra_config import HydraConfig
 from mlflow.entities import Run
-from omegaconf import DictConfig, OmegaConf
+from omegaconf import DictConfig, ListConfig, OmegaConf
 
 if TYPE_CHECKING:
     from collections.abc import Iterable
 
 
-def get_artifact_dir(run: Run | None = None, uri: str | None = None) -> Path:
+def file_uri_to_path(uri: str) -> Path:
+    """Convert a file URI to a local path."""
+    if not uri.startswith("file:"):
+        return Path(uri)
+
+    path = urllib.parse.urlparse(uri).path
+    return Path(urllib.request.url2pathname(path))  # for Windows
+
+
+def get_artifact_dir(run: Run | None = None) -> Path:
     """Retrieve the artifact directory for the given run.
 
     This function uses MLflow to get the artifact directory for the given run.
 
     Args:
         run (Run | None): The run object. Defaults to None.
-        uri (str | None): The URI of the artifact. Defaults to None.
 
     Returns:
         The local path to the directory where the artifacts are downloaded.
 
     """
-    if run is not None and uri is not None:
-        raise ValueError("Cannot provide both run and uri")
-
-    if run is None and uri is None:
+    if run is None:
         uri = mlflow.get_artifact_uri()
-    elif run:
+    else:
         uri = run.info.artifact_uri
 
     if not isinstance(uri, str):
         raise NotImplementedError
 
-    if uri.startswith("file:"):
-        return file_uri_to_path(uri)
-
-    return Path(uri)
-
-
-def file_uri_to_path(uri: str) -> Path:
-    """Convert a file URI to a local path."""
-    path = urllib.parse.urlparse(uri).path
-    return Path(urllib.request.url2pathname(path))  # for Windows
+    return file_uri_to_path(uri)
 
 
 def get_artifact_path(run: Run | None, path: str) -> Path:
@@ -123,12 +119,7 @@ def load_config(run: Run) -> DictConfig:
     return OmegaConf.load(path)  # type: ignore
 
 
-def get_overrides() -> list[str]:
-    """Retrieve the overrides for the current run."""
-    return list(HydraConfig.get().overrides.task)  # ListConifg -> list
-
-
-def load_overrides(run: Run) -> list[str]:
+def load_overrides(run: Run) -> ListConfig:
     """Load the overrides for a given run.
 
     This function loads the overrides for the provided Run instance
@@ -137,15 +128,15 @@ def load_overrides(run: Run) -> list[str]:
     `.hydra/overrides.yaml` is not found in the run's artifact directory.
 
     Args:
-        run (Run): The Run instance for which to load the overrides.
+        run (Run): The Run instance for which to load the configuration.
 
     Returns:
-        The loaded overrides as a list of strings. Returns an empty list
-        if the overrides file is not found.
+        The loaded configuration as a DictConfig object. Returns an empty
+        DictConfig if the configuration file is not found.
 
     """
     path = get_artifact_dir(run) / ".hydra/overrides.yaml"
-    return [str(x) for x in OmegaConf.load(path)]
+    return OmegaConf.load(path)  # type: ignore
 
 
 def remove_run(run: Run | Iterable[Run]) -> None: