hydraflow 0.8.0__py3-none-any.whl → 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hydraflow/__init__.py +5 -5
- hydraflow/cli.py +31 -39
- hydraflow/core/__init__.py +0 -0
- hydraflow/{context.py → core/context.py} +3 -2
- hydraflow/{main.py → core/main.py} +5 -3
- hydraflow/{mlflow.py → core/mlflow.py} +4 -3
- hydraflow/entities/__init__.py +0 -0
- hydraflow/{run_collection.py → entities/run_collection.py} +8 -7
- hydraflow/{run_data.py → entities/run_data.py} +3 -3
- hydraflow/{run_info.py → entities/run_info.py} +2 -2
- hydraflow/executor/__init__.py +0 -0
- hydraflow/executor/conf.py +23 -0
- hydraflow/executor/io.py +34 -0
- hydraflow/executor/job.py +152 -0
- hydraflow/executor/parser.py +397 -0
- {hydraflow-0.8.0.dist-info → hydraflow-0.9.0.dist-info}/METADATA +18 -19
- hydraflow-0.9.0.dist-info/RECORD +24 -0
- hydraflow-0.8.0.dist-info/RECORD +0 -17
- /hydraflow/{config.py → core/config.py} +0 -0
- /hydraflow/{utils.py → core/io.py} +0 -0
- /hydraflow/{param.py → core/param.py} +0 -0
- {hydraflow-0.8.0.dist-info → hydraflow-0.9.0.dist-info}/WHEEL +0 -0
- {hydraflow-0.8.0.dist-info → hydraflow-0.9.0.dist-info}/entry_points.txt +0 -0
- {hydraflow-0.8.0.dist-info → hydraflow-0.9.0.dist-info}/licenses/LICENSE +0 -0
hydraflow/__init__.py
CHANGED
@@ -1,16 +1,16 @@
|
|
1
1
|
"""Integrate Hydra and MLflow to manage and track machine learning experiments."""
|
2
2
|
|
3
|
-
from hydraflow.context import chdir_artifact, log_run, start_run
|
4
|
-
from hydraflow.
|
5
|
-
from hydraflow.mlflow import list_run_ids, list_run_paths, list_runs
|
6
|
-
from hydraflow.run_collection import RunCollection
|
7
|
-
from hydraflow.utils import (
|
3
|
+
from hydraflow.core.context import chdir_artifact, log_run, start_run
|
4
|
+
from hydraflow.core.io import (
|
8
5
|
get_artifact_dir,
|
9
6
|
get_artifact_path,
|
10
7
|
get_hydra_output_dir,
|
11
8
|
load_config,
|
12
9
|
remove_run,
|
13
10
|
)
|
11
|
+
from hydraflow.core.main import main
|
12
|
+
from hydraflow.core.mlflow import list_run_ids, list_run_paths, list_runs
|
13
|
+
from hydraflow.entities.run_collection import RunCollection
|
14
14
|
|
15
15
|
__all__ = [
|
16
16
|
"RunCollection",
|
hydraflow/cli.py
CHANGED
@@ -2,41 +2,54 @@
|
|
2
2
|
|
3
3
|
from __future__ import annotations
|
4
4
|
|
5
|
-
from
|
6
|
-
from typing import Annotated
|
5
|
+
from typing import TYPE_CHECKING, Annotated
|
7
6
|
|
8
7
|
import typer
|
9
|
-
from omegaconf import DictConfig, OmegaConf
|
10
8
|
from rich.console import Console
|
11
9
|
from typer import Argument, Option
|
12
10
|
|
11
|
+
from hydraflow.executor.io import load_config
|
12
|
+
|
13
|
+
if TYPE_CHECKING:
|
14
|
+
from hydraflow.executor.job import Job
|
15
|
+
|
13
16
|
app = typer.Typer(add_completion=False)
|
14
17
|
console = Console()
|
15
18
|
|
16
19
|
|
20
|
+
def get_job(name: str) -> Job:
|
21
|
+
cfg = load_config()
|
22
|
+
job = cfg.jobs[name]
|
23
|
+
|
24
|
+
if not job.name:
|
25
|
+
job.name = name
|
26
|
+
|
27
|
+
return job
|
28
|
+
|
29
|
+
|
17
30
|
@app.command()
|
18
31
|
def run(
|
19
|
-
|
20
|
-
list[str] | None,
|
21
|
-
Argument(help="Job names.", show_default=False),
|
22
|
-
] = None,
|
32
|
+
name: Annotated[str, Argument(help="Job name.", show_default=False)],
|
23
33
|
) -> None:
|
24
|
-
"""Run
|
25
|
-
|
34
|
+
"""Run a job."""
|
35
|
+
import mlflow
|
26
36
|
|
27
|
-
|
28
|
-
|
37
|
+
from hydraflow.executor.job import multirun
|
38
|
+
|
39
|
+
job = get_job(name)
|
40
|
+
mlflow.set_experiment(job.name)
|
41
|
+
multirun(job)
|
29
42
|
|
30
43
|
|
31
44
|
@app.command()
|
32
|
-
def show(
|
33
|
-
"
|
34
|
-
|
45
|
+
def show(
|
46
|
+
name: Annotated[str, Argument(help="Job name.", show_default=False)],
|
47
|
+
) -> None:
|
48
|
+
"""Show a job."""
|
49
|
+
from hydraflow.executor.job import show
|
35
50
|
|
36
|
-
|
37
|
-
|
38
|
-
syntax = Syntax(code, "yaml")
|
39
|
-
console.print(syntax)
|
51
|
+
job = get_job(name)
|
52
|
+
show(job)
|
40
53
|
|
41
54
|
|
42
55
|
@app.callback(invoke_without_command=True)
|
@@ -52,24 +65,3 @@ def callback(
|
|
52
65
|
|
53
66
|
typer.echo(f"hydraflow {importlib.metadata.version('hydraflow')}")
|
54
67
|
raise typer.Exit
|
55
|
-
|
56
|
-
|
57
|
-
def find_config() -> Path:
|
58
|
-
if Path("hydraflow.yaml").exists():
|
59
|
-
return Path("hydraflow.yaml")
|
60
|
-
|
61
|
-
if Path("hydraflow.yml").exists():
|
62
|
-
return Path("hydraflow.yml")
|
63
|
-
|
64
|
-
typer.echo("No config file found.")
|
65
|
-
raise typer.Exit(code=1)
|
66
|
-
|
67
|
-
|
68
|
-
def load_config() -> DictConfig:
|
69
|
-
cfg = OmegaConf.load(find_config())
|
70
|
-
|
71
|
-
if isinstance(cfg, DictConfig):
|
72
|
-
return cfg
|
73
|
-
|
74
|
-
typer.echo("Invalid config file.")
|
75
|
-
raise typer.Exit(code=1)
|
File without changes
|
@@ -12,8 +12,9 @@ import mlflow
|
|
12
12
|
import mlflow.artifacts
|
13
13
|
from hydra.core.hydra_config import HydraConfig
|
14
14
|
|
15
|
-
from hydraflow.
|
16
|
-
|
15
|
+
from hydraflow.core.io import get_artifact_dir
|
16
|
+
|
17
|
+
from .mlflow import log_params, log_text
|
17
18
|
|
18
19
|
if TYPE_CHECKING:
|
19
20
|
from collections.abc import Iterator
|
@@ -7,6 +7,7 @@ management.
|
|
7
7
|
|
8
8
|
The main functionality is provided through the `main` decorator, which can be
|
9
9
|
used to wrap experiment entry points. This decorator handles:
|
10
|
+
|
10
11
|
- Configuration management via Hydra
|
11
12
|
- Experiment tracking via MLflow
|
12
13
|
- Run deduplication based on configurations
|
@@ -44,11 +45,12 @@ from mlflow.entities import RunStatus
|
|
44
45
|
from omegaconf import OmegaConf
|
45
46
|
|
46
47
|
import hydraflow
|
47
|
-
from hydraflow.
|
48
|
+
from hydraflow.core.io import file_uri_to_path
|
48
49
|
|
49
50
|
if TYPE_CHECKING:
|
50
51
|
from collections.abc import Callable
|
51
52
|
from pathlib import Path
|
53
|
+
from typing import Any
|
52
54
|
|
53
55
|
from mlflow.entities import Run
|
54
56
|
|
@@ -115,7 +117,7 @@ def main(
|
|
115
117
|
return decorator
|
116
118
|
|
117
119
|
|
118
|
-
def get_run_id(uri: str, config:
|
120
|
+
def get_run_id(uri: str, config: Any, overrides: list[str] | None) -> str | None:
|
119
121
|
"""Try to get the run ID for the given configuration.
|
120
122
|
|
121
123
|
If the run is not found, the function will return None.
|
@@ -137,7 +139,7 @@ def get_run_id(uri: str, config: object, overrides: list[str] | None) -> str | N
|
|
137
139
|
return None
|
138
140
|
|
139
141
|
|
140
|
-
def equals(run_dir: Path, config:
|
142
|
+
def equals(run_dir: Path, config: Any, overrides: list[str] | None) -> bool:
|
141
143
|
"""Check if the run directory matches the given configuration or overrides.
|
142
144
|
|
143
145
|
Args:
|
@@ -13,9 +13,10 @@ import joblib
|
|
13
13
|
import mlflow
|
14
14
|
import mlflow.artifacts
|
15
15
|
|
16
|
-
from hydraflow.
|
17
|
-
from hydraflow.run_collection import RunCollection
|
18
|
-
|
16
|
+
from hydraflow.core.io import file_uri_to_path, get_artifact_dir
|
17
|
+
from hydraflow.entities.run_collection import RunCollection
|
18
|
+
|
19
|
+
from .config import iter_params
|
19
20
|
|
20
21
|
if TYPE_CHECKING:
|
21
22
|
from pathlib import Path
|
File without changes
|
@@ -25,12 +25,13 @@ from typing import TYPE_CHECKING, Any, overload
|
|
25
25
|
|
26
26
|
from mlflow.entities import RunStatus
|
27
27
|
|
28
|
-
import hydraflow.param
|
29
|
-
from hydraflow.config import iter_params, select_config, select_overrides
|
30
|
-
from hydraflow.
|
31
|
-
from hydraflow.
|
32
|
-
|
33
|
-
from
|
28
|
+
import hydraflow.core.param
|
29
|
+
from hydraflow.core.config import iter_params, select_config, select_overrides
|
30
|
+
from hydraflow.core.io import load_config
|
31
|
+
from hydraflow.core.param import get_params, get_values
|
32
|
+
|
33
|
+
from .run_data import RunCollectionData
|
34
|
+
from .run_info import RunCollectionInfo
|
34
35
|
|
35
36
|
if TYPE_CHECKING:
|
36
37
|
from collections.abc import Callable, Iterator
|
@@ -478,7 +479,7 @@ def _param_matches(run: Run, key: str, value: Any) -> bool:
|
|
478
479
|
if param == "None":
|
479
480
|
return value is None or value == "None"
|
480
481
|
|
481
|
-
return hydraflow.param.match(param, value)
|
482
|
+
return hydraflow.core.param.match(param, value)
|
482
483
|
|
483
484
|
|
484
485
|
def filter_runs(
|
@@ -6,14 +6,14 @@ from typing import TYPE_CHECKING
|
|
6
6
|
|
7
7
|
from pandas import DataFrame
|
8
8
|
|
9
|
-
from hydraflow.config import iter_params
|
10
|
-
from hydraflow.
|
9
|
+
from hydraflow.core.config import iter_params
|
10
|
+
from hydraflow.core.io import load_config
|
11
11
|
|
12
12
|
if TYPE_CHECKING:
|
13
13
|
from collections.abc import Iterable
|
14
14
|
from typing import Any
|
15
15
|
|
16
|
-
from
|
16
|
+
from .run_collection import RunCollection
|
17
17
|
|
18
18
|
|
19
19
|
class RunCollectionData:
|
@@ -4,12 +4,12 @@ from __future__ import annotations
|
|
4
4
|
|
5
5
|
from typing import TYPE_CHECKING
|
6
6
|
|
7
|
-
from hydraflow.
|
7
|
+
from hydraflow.core.io import get_artifact_dir
|
8
8
|
|
9
9
|
if TYPE_CHECKING:
|
10
10
|
from pathlib import Path
|
11
11
|
|
12
|
-
from
|
12
|
+
from .run_collection import RunCollection
|
13
13
|
|
14
14
|
|
15
15
|
class RunCollectionInfo:
|
File without changes
|
@@ -0,0 +1,23 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
from dataclasses import dataclass, field
|
4
|
+
|
5
|
+
|
6
|
+
@dataclass
|
7
|
+
class Step:
|
8
|
+
args: str = ""
|
9
|
+
batch: str = ""
|
10
|
+
options: str = ""
|
11
|
+
|
12
|
+
|
13
|
+
@dataclass
|
14
|
+
class Job:
|
15
|
+
name: str = ""
|
16
|
+
run: str = ""
|
17
|
+
call: str = ""
|
18
|
+
steps: list[Step] = field(default_factory=list)
|
19
|
+
|
20
|
+
|
21
|
+
@dataclass
|
22
|
+
class HydraflowConf:
|
23
|
+
jobs: dict[str, Job] = field(default_factory=dict)
|
hydraflow/executor/io.py
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
"""Hydraflow jobs IO."""
|
2
|
+
|
3
|
+
from __future__ import annotations
|
4
|
+
|
5
|
+
from pathlib import Path
|
6
|
+
|
7
|
+
from omegaconf import OmegaConf
|
8
|
+
|
9
|
+
from .conf import HydraflowConf
|
10
|
+
|
11
|
+
|
12
|
+
def find_config_file() -> Path | None:
|
13
|
+
"""Find the hydraflow config file."""
|
14
|
+
if Path("hydraflow.yaml").exists():
|
15
|
+
return Path("hydraflow.yaml")
|
16
|
+
|
17
|
+
if Path("hydraflow.yml").exists():
|
18
|
+
return Path("hydraflow.yml")
|
19
|
+
|
20
|
+
return None
|
21
|
+
|
22
|
+
|
23
|
+
def load_config() -> HydraflowConf:
|
24
|
+
"""Load the hydraflow config."""
|
25
|
+
schema = OmegaConf.structured(HydraflowConf)
|
26
|
+
|
27
|
+
path = find_config_file()
|
28
|
+
|
29
|
+
if path is None:
|
30
|
+
return schema
|
31
|
+
|
32
|
+
cfg = OmegaConf.load(path)
|
33
|
+
|
34
|
+
return OmegaConf.merge(schema, cfg) # type: ignore
|
@@ -0,0 +1,152 @@
|
|
1
|
+
"""Job execution and argument handling for HydraFlow.
|
2
|
+
|
3
|
+
This module provides functionality for executing jobs in HydraFlow, including:
|
4
|
+
|
5
|
+
- Argument parsing and expansion for job steps
|
6
|
+
- Batch processing of Hydra configurations
|
7
|
+
- Execution of jobs via shell commands or Python functions
|
8
|
+
|
9
|
+
The module supports two execution modes:
|
10
|
+
|
11
|
+
1. Shell command execution
|
12
|
+
2. Python function calls
|
13
|
+
|
14
|
+
Each job can consist of multiple steps, and each step can have its own
|
15
|
+
arguments and options that will be expanded into multiple runs.
|
16
|
+
"""
|
17
|
+
|
18
|
+
from __future__ import annotations
|
19
|
+
|
20
|
+
import importlib
|
21
|
+
import shlex
|
22
|
+
import subprocess
|
23
|
+
from subprocess import CalledProcessError
|
24
|
+
from typing import TYPE_CHECKING
|
25
|
+
|
26
|
+
import ulid
|
27
|
+
|
28
|
+
from .parser import collect, expand
|
29
|
+
|
30
|
+
if TYPE_CHECKING:
|
31
|
+
from collections.abc import Iterator
|
32
|
+
|
33
|
+
from .conf import Job, Step
|
34
|
+
|
35
|
+
|
36
|
+
def iter_args(step: Step) -> Iterator[list[str]]:
|
37
|
+
"""Iterate over combinations generated from parsed arguments.
|
38
|
+
|
39
|
+
Generate all possible combinations of arguments by parsing and
|
40
|
+
expanding each one, yielding them as an iterator.
|
41
|
+
|
42
|
+
Args:
|
43
|
+
step (Step): The step to parse.
|
44
|
+
|
45
|
+
Yields:
|
46
|
+
list[str]: a list of the parsed argument combinations.
|
47
|
+
|
48
|
+
"""
|
49
|
+
args = collect(step.args)
|
50
|
+
options = [o for o in step.options.split(" ") if o]
|
51
|
+
|
52
|
+
for batch in expand(step.batch):
|
53
|
+
yield [*options, *sorted([*batch, *args])]
|
54
|
+
|
55
|
+
|
56
|
+
def iter_batches(job: Job) -> Iterator[list[str]]:
|
57
|
+
"""Generate Hydra application arguments for a job.
|
58
|
+
|
59
|
+
This function generates a list of Hydra application arguments
|
60
|
+
for a given job, including the job name and the root directory
|
61
|
+
for the sweep.
|
62
|
+
|
63
|
+
Args:
|
64
|
+
job (Job): The job to generate the Hydra configuration for.
|
65
|
+
|
66
|
+
Returns:
|
67
|
+
list[str]: A list of Hydra configuration strings.
|
68
|
+
|
69
|
+
"""
|
70
|
+
job_name = f"hydra.job.name={job.name}"
|
71
|
+
|
72
|
+
for step in job.steps:
|
73
|
+
for args in iter_args(step):
|
74
|
+
sweep_dir = f"hydra.sweep.dir=multirun/{ulid.ulid()}"
|
75
|
+
yield ["--multirun", sweep_dir, job_name, *args]
|
76
|
+
|
77
|
+
|
78
|
+
def multirun(job: Job) -> None:
|
79
|
+
"""Execute multiple runs of a job using either shell commands or Python functions.
|
80
|
+
|
81
|
+
This function processes a job configuration and executes it in one of two modes:
|
82
|
+
|
83
|
+
1. Shell command mode (job.run): Executes shell commands with the generated
|
84
|
+
arguments
|
85
|
+
2. Python function mode (job.call): Calls a Python function with the generated
|
86
|
+
arguments
|
87
|
+
|
88
|
+
Args:
|
89
|
+
job (Job): The job configuration containing run parameters and steps.
|
90
|
+
|
91
|
+
Raises:
|
92
|
+
RuntimeError: If a shell command fails or if a function call encounters
|
93
|
+
an error.
|
94
|
+
ValueError: If the Python function path is invalid or the function cannot
|
95
|
+
be imported.
|
96
|
+
|
97
|
+
"""
|
98
|
+
it = iter_batches(job)
|
99
|
+
|
100
|
+
if job.run:
|
101
|
+
base_cmds = shlex.split(job.run)
|
102
|
+
for args in it:
|
103
|
+
cmds = [*base_cmds, *args]
|
104
|
+
try:
|
105
|
+
subprocess.run(cmds, check=True)
|
106
|
+
except CalledProcessError as e:
|
107
|
+
msg = f"Command failed with exit code {e.returncode}"
|
108
|
+
raise RuntimeError(msg) from e
|
109
|
+
|
110
|
+
elif job.call:
|
111
|
+
if "." not in job.call:
|
112
|
+
msg = f"Invalid function path: {job.call}."
|
113
|
+
msg += " Expected format: 'package.module.function'"
|
114
|
+
raise ValueError(msg)
|
115
|
+
|
116
|
+
try:
|
117
|
+
module_name, func_name = job.call.rsplit(".", 1)
|
118
|
+
module = importlib.import_module(module_name)
|
119
|
+
func = getattr(module, func_name)
|
120
|
+
except (ImportError, AttributeError, ModuleNotFoundError) as e:
|
121
|
+
msg = f"Failed to import or find function: {job.call}"
|
122
|
+
raise ValueError(msg) from e
|
123
|
+
|
124
|
+
for args in it:
|
125
|
+
try:
|
126
|
+
func(*args)
|
127
|
+
except Exception as e: # noqa: PERF203
|
128
|
+
msg = f"Function call '{job.call}' failed with args: {args}"
|
129
|
+
raise RuntimeError(msg) from e
|
130
|
+
|
131
|
+
|
132
|
+
def show(job: Job) -> None:
|
133
|
+
"""Show the job configuration.
|
134
|
+
|
135
|
+
This function shows the job configuration for a given job.
|
136
|
+
|
137
|
+
Args:
|
138
|
+
job (Job): The job configuration to show.
|
139
|
+
|
140
|
+
"""
|
141
|
+
it = iter_batches(job)
|
142
|
+
|
143
|
+
if job.run:
|
144
|
+
base_cmds = shlex.split(job.run)
|
145
|
+
for args in it:
|
146
|
+
cmds = " ".join([*base_cmds, *args])
|
147
|
+
print(cmds) # noqa: T201
|
148
|
+
|
149
|
+
elif job.call:
|
150
|
+
print(f"call: {job.call}") # noqa: T201
|
151
|
+
for args in it:
|
152
|
+
print(f"args: {args}") # noqa: T201
|
@@ -0,0 +1,397 @@
|
|
1
|
+
"""Parse and convert string representations of numbers and ranges.
|
2
|
+
|
3
|
+
This module provides utility functions for parsing and converting
|
4
|
+
string representations of numbers and ranges. It includes functions
|
5
|
+
to convert strings to numbers, count decimal places, handle numeric
|
6
|
+
ranges, and expand values from string arguments.
|
7
|
+
"""
|
8
|
+
|
9
|
+
from __future__ import annotations
|
10
|
+
|
11
|
+
import re
|
12
|
+
from itertools import chain, product
|
13
|
+
from typing import TYPE_CHECKING
|
14
|
+
|
15
|
+
if TYPE_CHECKING:
|
16
|
+
from collections.abc import Iterator
|
17
|
+
|
18
|
+
|
19
|
+
def to_number(x: str) -> int | float:
|
20
|
+
"""Convert a string to an integer or float.
|
21
|
+
|
22
|
+
Attempts to convert a string to an integer or a float,
|
23
|
+
returning 0 if the string is empty or cannot be converted.
|
24
|
+
|
25
|
+
Args:
|
26
|
+
x (str): The string to convert.
|
27
|
+
|
28
|
+
Returns:
|
29
|
+
int | float: The converted number as an integer or float.
|
30
|
+
|
31
|
+
Examples:
|
32
|
+
>>> type(to_number("1"))
|
33
|
+
<class 'int'>
|
34
|
+
>>> type(to_number("1.2"))
|
35
|
+
<class 'float'>
|
36
|
+
>>> to_number("")
|
37
|
+
0
|
38
|
+
>>> to_number("1e-3")
|
39
|
+
0.001
|
40
|
+
|
41
|
+
"""
|
42
|
+
if not x:
|
43
|
+
return 0
|
44
|
+
|
45
|
+
if "." in x or "e" in x.lower():
|
46
|
+
return float(x)
|
47
|
+
|
48
|
+
return int(x)
|
49
|
+
|
50
|
+
|
51
|
+
def count_decimal_places(x: str) -> int:
|
52
|
+
"""Count decimal places in a string.
|
53
|
+
|
54
|
+
Examine a string representing a number and returns the count
|
55
|
+
of decimal places present after the decimal point.
|
56
|
+
Return 0 if no decimal point is found.
|
57
|
+
|
58
|
+
Args:
|
59
|
+
x (str): The string to check.
|
60
|
+
|
61
|
+
Returns:
|
62
|
+
int: The number of decimal places.
|
63
|
+
|
64
|
+
Examples:
|
65
|
+
>>> count_decimal_places("1")
|
66
|
+
0
|
67
|
+
>>> count_decimal_places("-1.2")
|
68
|
+
1
|
69
|
+
>>> count_decimal_places("1.234")
|
70
|
+
3
|
71
|
+
>>> count_decimal_places("-1.234e-10")
|
72
|
+
3
|
73
|
+
|
74
|
+
"""
|
75
|
+
if "." not in x:
|
76
|
+
return 0
|
77
|
+
|
78
|
+
decimal_part = x.split(".")[1]
|
79
|
+
if "e" in decimal_part.lower():
|
80
|
+
decimal_part = decimal_part.split("e")[0]
|
81
|
+
|
82
|
+
return len(decimal_part)
|
83
|
+
|
84
|
+
|
85
|
+
def is_number(x: str) -> bool:
|
86
|
+
"""Check if a string is a number.
|
87
|
+
|
88
|
+
Args:
|
89
|
+
x (str): The string to check.
|
90
|
+
|
91
|
+
Returns:
|
92
|
+
bool: True if the string is a number, False otherwise.
|
93
|
+
|
94
|
+
Examples:
|
95
|
+
>>> is_number("1")
|
96
|
+
True
|
97
|
+
>>> is_number("-1.2")
|
98
|
+
True
|
99
|
+
>>> is_number("1.2.3")
|
100
|
+
False
|
101
|
+
|
102
|
+
"""
|
103
|
+
try:
|
104
|
+
float(x)
|
105
|
+
except ValueError:
|
106
|
+
return False
|
107
|
+
return True
|
108
|
+
|
109
|
+
|
110
|
+
SUFFIX_EXPONENT = {
|
111
|
+
"T": "e12",
|
112
|
+
"G": "e9",
|
113
|
+
"M": "e6",
|
114
|
+
"k": "e3",
|
115
|
+
"m": "e-3",
|
116
|
+
"u": "e-6",
|
117
|
+
"n": "e-9",
|
118
|
+
"p": "e-12",
|
119
|
+
"f": "e-15",
|
120
|
+
}
|
121
|
+
|
122
|
+
|
123
|
+
def _get_range(arg: str) -> tuple[float, float, float]:
|
124
|
+
args = [to_number(x) for x in arg.split(":")]
|
125
|
+
|
126
|
+
if len(args) == 2:
|
127
|
+
if args[0] > args[1]:
|
128
|
+
raise ValueError("start cannot be greater than stop")
|
129
|
+
|
130
|
+
return (args[0], 1, args[1])
|
131
|
+
|
132
|
+
if args[1] == 0:
|
133
|
+
raise ValueError("step cannot be zero")
|
134
|
+
if args[1] > 0 and args[0] > args[2]:
|
135
|
+
raise ValueError("start cannot be greater than stop")
|
136
|
+
if args[1] < 0 and args[0] < args[2]:
|
137
|
+
raise ValueError("start cannot be less than stop")
|
138
|
+
|
139
|
+
return args[0], args[1], args[2]
|
140
|
+
|
141
|
+
|
142
|
+
def _arange(start: float, step: float, stop: float) -> list[float]:
|
143
|
+
result = []
|
144
|
+
current = start
|
145
|
+
|
146
|
+
while current <= stop if step > 0 else current >= stop:
|
147
|
+
result.append(current)
|
148
|
+
current += step
|
149
|
+
|
150
|
+
return result
|
151
|
+
|
152
|
+
|
153
|
+
def split_suffix(arg: str) -> tuple[str, str]:
|
154
|
+
"""Split a string into prefix and suffix.
|
155
|
+
|
156
|
+
Args:
|
157
|
+
arg (str): The string to split.
|
158
|
+
|
159
|
+
Returns:
|
160
|
+
tuple[str, str]: A tuple containing the prefix and suffix.
|
161
|
+
|
162
|
+
Examples:
|
163
|
+
>>> split_suffix("1:k")
|
164
|
+
('1', 'e3')
|
165
|
+
>>> split_suffix("1:2:k")
|
166
|
+
('1:2', 'e3')
|
167
|
+
>>> split_suffix("1:2:M")
|
168
|
+
('1:2', 'e6')
|
169
|
+
>>> split_suffix(":1:2:M")
|
170
|
+
(':1:2', 'e6')
|
171
|
+
|
172
|
+
"""
|
173
|
+
if len(arg) < 3 or ":" not in arg:
|
174
|
+
return arg, ""
|
175
|
+
|
176
|
+
prefix, suffix = arg.rsplit(":", 1)
|
177
|
+
|
178
|
+
if suffix.lower().startswith("e"):
|
179
|
+
return prefix, suffix
|
180
|
+
|
181
|
+
if suffix not in SUFFIX_EXPONENT:
|
182
|
+
return arg, ""
|
183
|
+
|
184
|
+
return prefix, SUFFIX_EXPONENT[suffix]
|
185
|
+
|
186
|
+
|
187
|
+
def add_exponent(value: str, exponent: str) -> str:
|
188
|
+
"""Append an exponent to a value string.
|
189
|
+
|
190
|
+
Args:
|
191
|
+
value (str): The value to modify.
|
192
|
+
exponent (str): The exponent to append.
|
193
|
+
|
194
|
+
Returns:
|
195
|
+
str: The value with the exponent added.
|
196
|
+
|
197
|
+
"""
|
198
|
+
if value in ["0", "0.", "0.0"] or not exponent:
|
199
|
+
return value
|
200
|
+
|
201
|
+
return f"{value}{exponent}"
|
202
|
+
|
203
|
+
|
204
|
+
def collect_values(arg: str) -> list[str]:
|
205
|
+
"""Collect a list of values from a range argument.
|
206
|
+
|
207
|
+
Collect all individual values within a numeric range
|
208
|
+
represented by a string (e.g., `1:4`) and return them
|
209
|
+
as a list of strings.
|
210
|
+
Support both integer and floating-point ranges.
|
211
|
+
|
212
|
+
Args:
|
213
|
+
arg (str): The argument to collect.
|
214
|
+
|
215
|
+
Returns:
|
216
|
+
list[str]: A list of the collected values.
|
217
|
+
|
218
|
+
"""
|
219
|
+
if ":" not in arg:
|
220
|
+
return [arg]
|
221
|
+
|
222
|
+
arg, exponent = split_suffix(arg)
|
223
|
+
|
224
|
+
if ":" not in arg:
|
225
|
+
return [f"{arg}{exponent}"]
|
226
|
+
|
227
|
+
rng = _get_range(arg)
|
228
|
+
|
229
|
+
if all(isinstance(x, int) for x in rng):
|
230
|
+
values = [str(x) for x in _arange(*rng)]
|
231
|
+
else:
|
232
|
+
n = max(*(count_decimal_places(x) for x in arg.split(":")))
|
233
|
+
values = [str(round(x, n)) for x in _arange(*rng)]
|
234
|
+
|
235
|
+
return [add_exponent(x, exponent) for x in values]
|
236
|
+
|
237
|
+
|
238
|
+
def split(arg: str) -> list[str]:
|
239
|
+
r"""Split a string by top-level commas.
|
240
|
+
|
241
|
+
Splits a string by commas while respecting nested structures.
|
242
|
+
Commas inside brackets and quotes are ignored, only splitting
|
243
|
+
at the top-level commas.
|
244
|
+
|
245
|
+
Args:
|
246
|
+
arg (str): The string to split.
|
247
|
+
|
248
|
+
Returns:
|
249
|
+
list[str]: A list of split strings.
|
250
|
+
|
251
|
+
Examples:
|
252
|
+
>>> split("[a,1],[b,2]")
|
253
|
+
['[a,1]', '[b,2]']
|
254
|
+
>>> split('"x,y",z')
|
255
|
+
['"x,y"', 'z']
|
256
|
+
>>> split("'p,q',r")
|
257
|
+
["'p,q'", 'r']
|
258
|
+
|
259
|
+
"""
|
260
|
+
result = []
|
261
|
+
current = []
|
262
|
+
bracket_count = 0
|
263
|
+
in_single_quote = False
|
264
|
+
in_double_quote = False
|
265
|
+
|
266
|
+
for char in arg:
|
267
|
+
if char == "'" and not in_double_quote:
|
268
|
+
in_single_quote = not in_single_quote
|
269
|
+
elif char == '"' and not in_single_quote:
|
270
|
+
in_double_quote = not in_double_quote
|
271
|
+
elif char == "[" and not (in_single_quote or in_double_quote):
|
272
|
+
bracket_count += 1
|
273
|
+
elif char == "]" and not (in_single_quote or in_double_quote):
|
274
|
+
bracket_count -= 1
|
275
|
+
elif (
|
276
|
+
char == ","
|
277
|
+
and bracket_count == 0
|
278
|
+
and not in_single_quote
|
279
|
+
and not in_double_quote
|
280
|
+
):
|
281
|
+
result.append("".join(current))
|
282
|
+
current = []
|
283
|
+
continue
|
284
|
+
current.append(char)
|
285
|
+
|
286
|
+
if current:
|
287
|
+
result.append("".join(current))
|
288
|
+
|
289
|
+
return result
|
290
|
+
|
291
|
+
|
292
|
+
def expand_values(arg: str) -> list[str]:
|
293
|
+
"""Expand a string argument into a list of values.
|
294
|
+
|
295
|
+
Take a string containing comma-separated values or ranges and return a list
|
296
|
+
of all individual values. Handle numeric ranges and special characters.
|
297
|
+
|
298
|
+
Args:
|
299
|
+
arg (str): The argument to expand.
|
300
|
+
|
301
|
+
Returns:
|
302
|
+
list[str]: A list of the expanded values.
|
303
|
+
|
304
|
+
"""
|
305
|
+
return list(chain.from_iterable(collect_values(x) for x in split(arg)))
|
306
|
+
|
307
|
+
|
308
|
+
def collect_arg(arg: str) -> str:
|
309
|
+
"""Collect a string of expanded key-value pairs.
|
310
|
+
|
311
|
+
Take a key-value pair argument and concatenates all expanded values with commas,
|
312
|
+
returning a single string suitable for command-line usage.
|
313
|
+
|
314
|
+
Args:
|
315
|
+
arg (str): The argument to collect.
|
316
|
+
|
317
|
+
Returns:
|
318
|
+
str: A string of the collected key and values.
|
319
|
+
|
320
|
+
"""
|
321
|
+
key, arg = arg.split("=")
|
322
|
+
arg = ",".join(expand_values(arg))
|
323
|
+
return f"{key}={arg}"
|
324
|
+
|
325
|
+
|
326
|
+
def expand_arg(arg: str) -> Iterator[str]:
|
327
|
+
"""Parse a string argument into a list of values.
|
328
|
+
|
329
|
+
Responsible for parsing a string that may contain multiple
|
330
|
+
arguments separated by pipes ("|") and returns a list of all
|
331
|
+
expanded arguments.
|
332
|
+
|
333
|
+
Args:
|
334
|
+
arg (str): The argument to parse.
|
335
|
+
|
336
|
+
Returns:
|
337
|
+
list[str]: A list of the parsed arguments.
|
338
|
+
|
339
|
+
"""
|
340
|
+
if "|" not in arg:
|
341
|
+
key, value = arg.split("=")
|
342
|
+
|
343
|
+
for v in expand_values(value):
|
344
|
+
yield f"{key}={v}"
|
345
|
+
|
346
|
+
return
|
347
|
+
|
348
|
+
args = arg.split("|")
|
349
|
+
key = ""
|
350
|
+
|
351
|
+
for arg_ in args:
|
352
|
+
if "=" in arg_:
|
353
|
+
key, value = arg_.split("=")
|
354
|
+
elif key:
|
355
|
+
value = arg_
|
356
|
+
else:
|
357
|
+
msg = f"Invalid argument: {arg_}"
|
358
|
+
raise ValueError(msg)
|
359
|
+
|
360
|
+
value = ",".join(expand_values(value))
|
361
|
+
yield f"{key}={value}"
|
362
|
+
|
363
|
+
|
364
|
+
def collect(args: str | list[str]) -> list[str]:
|
365
|
+
"""Collect a list of arguments into a list of strings.
|
366
|
+
|
367
|
+
Args:
|
368
|
+
args (list[str]): The arguments to collect.
|
369
|
+
|
370
|
+
Returns:
|
371
|
+
list[str]: A list of the collected arguments.
|
372
|
+
|
373
|
+
"""
|
374
|
+
if isinstance(args, str):
|
375
|
+
args = re.split(r"\s+", args.strip())
|
376
|
+
|
377
|
+
args = [arg for arg in args if "=" in arg]
|
378
|
+
|
379
|
+
return [collect_arg(arg) for arg in args]
|
380
|
+
|
381
|
+
|
382
|
+
def expand(args: str | list[str]) -> list[list[str]]:
|
383
|
+
"""Expand a list of arguments into a list of lists of strings.
|
384
|
+
|
385
|
+
Args:
|
386
|
+
args (list[str]): The arguments to expand.
|
387
|
+
|
388
|
+
Returns:
|
389
|
+
list[list[str]]: A list of the expanded arguments.
|
390
|
+
|
391
|
+
"""
|
392
|
+
if isinstance(args, str):
|
393
|
+
args = re.split(r"\s+", args.strip())
|
394
|
+
|
395
|
+
args = [arg for arg in args if "=" in arg]
|
396
|
+
|
397
|
+
return [list(x) for x in product(*(expand_arg(arg) for arg in args))]
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: hydraflow
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.9.0
|
4
4
|
Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
|
5
5
|
Project-URL: Documentation, https://daizutabi.github.io/hydraflow/
|
6
6
|
Project-URL: Source, https://github.com/daizutabi/hydraflow
|
@@ -41,6 +41,7 @@ Requires-Dist: mlflow>=2.15
|
|
41
41
|
Requires-Dist: omegaconf
|
42
42
|
Requires-Dist: rich
|
43
43
|
Requires-Dist: typer
|
44
|
+
Requires-Dist: ulid
|
44
45
|
Description-Content-Type: text/markdown
|
45
46
|
|
46
47
|
# Hydraflow
|
@@ -93,31 +94,29 @@ pip install hydraflow
|
|
93
94
|
Here is a simple example to get you started with Hydraflow:
|
94
95
|
|
95
96
|
```python
|
96
|
-
import
|
97
|
-
|
98
|
-
import mlflow
|
97
|
+
from __future__ import annotations
|
98
|
+
|
99
99
|
from dataclasses import dataclass
|
100
|
-
from hydra.core.config_store import ConfigStore
|
101
100
|
from pathlib import Path
|
101
|
+
from typing import TYPE_CHECKING
|
102
102
|
|
103
|
-
|
104
|
-
class MySQLConfig:
|
105
|
-
host: str = "localhost"
|
106
|
-
port: int = 3306
|
103
|
+
import hydraflow
|
107
104
|
|
108
|
-
|
109
|
-
|
105
|
+
if TYPE_CHECKING:
|
106
|
+
from mlflow.entities import Run
|
107
|
+
|
108
|
+
|
109
|
+
@dataclass
|
110
|
+
class Config:
|
111
|
+
count: int = 1
|
112
|
+
name: str = "a"
|
110
113
|
|
111
|
-
@hydra.main(config_name="config", version_base=None)
|
112
|
-
def my_app(cfg: MySQLConfig) -> None:
|
113
|
-
# Set experiment by Hydra job name.
|
114
|
-
hydraflow.set_experiment()
|
115
114
|
|
116
|
-
|
117
|
-
|
118
|
-
|
115
|
+
@hydraflow.main(Config)
|
116
|
+
def app(run: Run, cfg: Config):
|
117
|
+
"""Your app code here."""
|
119
118
|
|
120
119
|
|
121
120
|
if __name__ == "__main__":
|
122
|
-
|
121
|
+
app()
|
123
122
|
```
|
@@ -0,0 +1,24 @@
|
|
1
|
+
hydraflow/__init__.py,sha256=WnReG-E2paQSMLJm42N-KaQyHYsfA4OX-WWcst610PI,738
|
2
|
+
hydraflow/cli.py,sha256=gbDPj49azP8CCGxkxU0rksh1-gCyjP0VkVYH34ktcsA,1338
|
3
|
+
hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
4
|
+
hydraflow/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
5
|
+
hydraflow/core/config.py,sha256=SJzjgsO_kzB78_whJ3lmy7GlZvTvwZONH1BJBn8zCuI,3817
|
6
|
+
hydraflow/core/context.py,sha256=QPyPg1xrTlmhviKNn-0nDY9bXcVky1zInqRqPN-VNhc,4741
|
7
|
+
hydraflow/core/io.py,sha256=T4ESiepEcqR-FZlo_m7VTBEFMwalrqPI8eFKPagvv3Q,4402
|
8
|
+
hydraflow/core/main.py,sha256=gYb1OOVH0CL4385Dm-06Mqi1Mr9-24URwLUiW86pGNs,5018
|
9
|
+
hydraflow/core/mlflow.py,sha256=M3MhiChnMzKnKRmjBl4h_SRGkAZKL7GAmFr3DdzwRuQ,5666
|
10
|
+
hydraflow/core/param.py,sha256=LHU9j9_7oA99igasoOyKofKClVr9FmGA3UABJ-KmyS0,4538
|
11
|
+
hydraflow/entities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
12
|
+
hydraflow/entities/run_collection.py,sha256=BHbMvYB4WyG5qSNYpIzwmHySOtmGNb-jg2OrCaTPr2I,19651
|
13
|
+
hydraflow/entities/run_data.py,sha256=Y2_Lc-BdQ7nXhcEIjdHGHIkLrXsmAktOftESEwYOY8o,1602
|
14
|
+
hydraflow/entities/run_info.py,sha256=FRC6ICOlzB2u_xi_33Qs-YZLt677UotuNbYqI7XSmHY,1017
|
15
|
+
hydraflow/executor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
16
|
+
hydraflow/executor/conf.py,sha256=q_FrPXQJCVGKS1FYnGRGqTUgMQeMBkaVPW2mtQc8oxk,384
|
17
|
+
hydraflow/executor/io.py,sha256=4nafwge6vHanYFuEHxd0LRv_3ZLgMpV50qSbssZNe3Q,696
|
18
|
+
hydraflow/executor/job.py,sha256=Vp2IZOuIC25Gqo9A_5MkllWl1T1QBfUnI5ksMvKwakg,4479
|
19
|
+
hydraflow/executor/parser.py,sha256=y4C9wVdUnazJDxdWrT5y3yWFIo0zAGzO-cS9x1MTK_8,9486
|
20
|
+
hydraflow-0.9.0.dist-info/METADATA,sha256=p8-vmPIYV-uU9BJbSB1roqrhNO7nK2HRdrpj8Tip1AA,4559
|
21
|
+
hydraflow-0.9.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
22
|
+
hydraflow-0.9.0.dist-info/entry_points.txt,sha256=XI0khPbpCIUo9UPqkNEpgh-kqK3Jy8T7L2VCWOdkbSM,48
|
23
|
+
hydraflow-0.9.0.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
|
24
|
+
hydraflow-0.9.0.dist-info/RECORD,,
|
hydraflow-0.8.0.dist-info/RECORD
DELETED
@@ -1,17 +0,0 @@
|
|
1
|
-
hydraflow/__init__.py,sha256=yp4LT1FDYPIduR6PqJNuSm9kztVCpL1P0zcPHWGvaJU,712
|
2
|
-
hydraflow/cli.py,sha256=jxqFppNeJWAr2Tb-C_MQXEJtegJ6TXcd3C1CT7Jdb1A,1559
|
3
|
-
hydraflow/config.py,sha256=SJzjgsO_kzB78_whJ3lmy7GlZvTvwZONH1BJBn8zCuI,3817
|
4
|
-
hydraflow/context.py,sha256=H5xeNbhMS23U-epsucprl5G3lbOR1aO9nDES4QGLWNk,4747
|
5
|
-
hydraflow/main.py,sha256=O5ETCMCg12zXoaYlZMHcM4IYAs6GVTkADrmEssrtjkk,4994
|
6
|
-
hydraflow/mlflow.py,sha256=pRRsBaBBH4cfzSko-8mmo5bV04GGklxoO0kORkInypM,5663
|
7
|
-
hydraflow/param.py,sha256=LHU9j9_7oA99igasoOyKofKClVr9FmGA3UABJ-KmyS0,4538
|
8
|
-
hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
9
|
-
hydraflow/run_collection.py,sha256=rtH1cglSlK3QFg9hhifo9lzjDa9veHpoyYxEOmIEM84,19646
|
10
|
-
hydraflow/run_data.py,sha256=S2NNFtA1TleqpgeK4mIn1YY8YbWJFyhF7wXR5NWeYLk,1604
|
11
|
-
hydraflow/run_info.py,sha256=Jf5wrIjRLIV1-k-obHDqwKHa6j_ZonrY8od-rXlbtMo,1024
|
12
|
-
hydraflow/utils.py,sha256=T4ESiepEcqR-FZlo_m7VTBEFMwalrqPI8eFKPagvv3Q,4402
|
13
|
-
hydraflow-0.8.0.dist-info/METADATA,sha256=J1ilgG7L4A8OvzgZSNycp0YgyHk5e8_gwTr9NN82Ejk,4767
|
14
|
-
hydraflow-0.8.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
15
|
-
hydraflow-0.8.0.dist-info/entry_points.txt,sha256=XI0khPbpCIUo9UPqkNEpgh-kqK3Jy8T7L2VCWOdkbSM,48
|
16
|
-
hydraflow-0.8.0.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
|
17
|
-
hydraflow-0.8.0.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|