hydraflow 0.8.0__py3-none-any.whl → 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
hydraflow/__init__.py CHANGED
@@ -1,16 +1,16 @@
1
1
  """Integrate Hydra and MLflow to manage and track machine learning experiments."""
2
2
 
3
- from hydraflow.context import chdir_artifact, log_run, start_run
4
- from hydraflow.main import main
5
- from hydraflow.mlflow import list_run_ids, list_run_paths, list_runs
6
- from hydraflow.run_collection import RunCollection
7
- from hydraflow.utils import (
3
+ from hydraflow.core.context import chdir_artifact, log_run, start_run
4
+ from hydraflow.core.io import (
8
5
  get_artifact_dir,
9
6
  get_artifact_path,
10
7
  get_hydra_output_dir,
11
8
  load_config,
12
9
  remove_run,
13
10
  )
11
+ from hydraflow.core.main import main
12
+ from hydraflow.core.mlflow import list_run_ids, list_run_paths, list_runs
13
+ from hydraflow.entities.run_collection import RunCollection
14
14
 
15
15
  __all__ = [
16
16
  "RunCollection",
hydraflow/cli.py CHANGED
@@ -2,41 +2,54 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
- from pathlib import Path
6
- from typing import Annotated
5
+ from typing import TYPE_CHECKING, Annotated
7
6
 
8
7
  import typer
9
- from omegaconf import DictConfig, OmegaConf
10
8
  from rich.console import Console
11
9
  from typer import Argument, Option
12
10
 
11
+ from hydraflow.executor.io import load_config
12
+
13
+ if TYPE_CHECKING:
14
+ from hydraflow.executor.job import Job
15
+
13
16
  app = typer.Typer(add_completion=False)
14
17
  console = Console()
15
18
 
16
19
 
20
+ def get_job(name: str) -> Job:
21
+ cfg = load_config()
22
+ job = cfg.jobs[name]
23
+
24
+ if not job.name:
25
+ job.name = name
26
+
27
+ return job
28
+
29
+
17
30
  @app.command()
18
31
  def run(
19
- names: Annotated[
20
- list[str] | None,
21
- Argument(help="Job names.", show_default=False),
22
- ] = None,
32
+ name: Annotated[str, Argument(help="Job name.", show_default=False)],
23
33
  ) -> None:
24
- """Run jobs."""
25
- typer.echo(names)
34
+ """Run a job."""
35
+ import mlflow
26
36
 
27
- cfg = load_config()
28
- typer.echo(cfg)
37
+ from hydraflow.executor.job import multirun
38
+
39
+ job = get_job(name)
40
+ mlflow.set_experiment(job.name)
41
+ multirun(job)
29
42
 
30
43
 
31
44
  @app.command()
32
- def show() -> None:
33
- """Show the config."""
34
- from rich.syntax import Syntax
45
+ def show(
46
+ name: Annotated[str, Argument(help="Job name.", show_default=False)],
47
+ ) -> None:
48
+ """Show a job."""
49
+ from hydraflow.executor.job import show
35
50
 
36
- cfg = load_config()
37
- code = OmegaConf.to_yaml(cfg)
38
- syntax = Syntax(code, "yaml")
39
- console.print(syntax)
51
+ job = get_job(name)
52
+ show(job)
40
53
 
41
54
 
42
55
  @app.callback(invoke_without_command=True)
@@ -52,24 +65,3 @@ def callback(
52
65
 
53
66
  typer.echo(f"hydraflow {importlib.metadata.version('hydraflow')}")
54
67
  raise typer.Exit
55
-
56
-
57
- def find_config() -> Path:
58
- if Path("hydraflow.yaml").exists():
59
- return Path("hydraflow.yaml")
60
-
61
- if Path("hydraflow.yml").exists():
62
- return Path("hydraflow.yml")
63
-
64
- typer.echo("No config file found.")
65
- raise typer.Exit(code=1)
66
-
67
-
68
- def load_config() -> DictConfig:
69
- cfg = OmegaConf.load(find_config())
70
-
71
- if isinstance(cfg, DictConfig):
72
- return cfg
73
-
74
- typer.echo("Invalid config file.")
75
- raise typer.Exit(code=1)
File without changes
@@ -12,8 +12,9 @@ import mlflow
12
12
  import mlflow.artifacts
13
13
  from hydra.core.hydra_config import HydraConfig
14
14
 
15
- from hydraflow.mlflow import log_params, log_text
16
- from hydraflow.utils import get_artifact_dir
15
+ from hydraflow.core.io import get_artifact_dir
16
+
17
+ from .mlflow import log_params, log_text
17
18
 
18
19
  if TYPE_CHECKING:
19
20
  from collections.abc import Iterator
@@ -7,6 +7,7 @@ management.
7
7
 
8
8
  The main functionality is provided through the `main` decorator, which can be
9
9
  used to wrap experiment entry points. This decorator handles:
10
+
10
11
  - Configuration management via Hydra
11
12
  - Experiment tracking via MLflow
12
13
  - Run deduplication based on configurations
@@ -44,11 +45,12 @@ from mlflow.entities import RunStatus
44
45
  from omegaconf import OmegaConf
45
46
 
46
47
  import hydraflow
47
- from hydraflow.utils import file_uri_to_path
48
+ from hydraflow.core.io import file_uri_to_path
48
49
 
49
50
  if TYPE_CHECKING:
50
51
  from collections.abc import Callable
51
52
  from pathlib import Path
53
+ from typing import Any
52
54
 
53
55
  from mlflow.entities import Run
54
56
 
@@ -115,7 +117,7 @@ def main(
115
117
  return decorator
116
118
 
117
119
 
118
- def get_run_id(uri: str, config: object, overrides: list[str] | None) -> str | None:
120
+ def get_run_id(uri: str, config: Any, overrides: list[str] | None) -> str | None:
119
121
  """Try to get the run ID for the given configuration.
120
122
 
121
123
  If the run is not found, the function will return None.
@@ -137,7 +139,7 @@ def get_run_id(uri: str, config: object, overrides: list[str] | None) -> str | N
137
139
  return None
138
140
 
139
141
 
140
- def equals(run_dir: Path, config: object, overrides: list[str] | None) -> bool:
142
+ def equals(run_dir: Path, config: Any, overrides: list[str] | None) -> bool:
141
143
  """Check if the run directory matches the given configuration or overrides.
142
144
 
143
145
  Args:
@@ -13,9 +13,10 @@ import joblib
13
13
  import mlflow
14
14
  import mlflow.artifacts
15
15
 
16
- from hydraflow.config import iter_params
17
- from hydraflow.run_collection import RunCollection
18
- from hydraflow.utils import file_uri_to_path, get_artifact_dir
16
+ from hydraflow.core.io import file_uri_to_path, get_artifact_dir
17
+ from hydraflow.entities.run_collection import RunCollection
18
+
19
+ from .config import iter_params
19
20
 
20
21
  if TYPE_CHECKING:
21
22
  from pathlib import Path
File without changes
@@ -25,12 +25,13 @@ from typing import TYPE_CHECKING, Any, overload
25
25
 
26
26
  from mlflow.entities import RunStatus
27
27
 
28
- import hydraflow.param
29
- from hydraflow.config import iter_params, select_config, select_overrides
30
- from hydraflow.param import get_params, get_values
31
- from hydraflow.run_data import RunCollectionData
32
- from hydraflow.run_info import RunCollectionInfo
33
- from hydraflow.utils import load_config
28
+ import hydraflow.core.param
29
+ from hydraflow.core.config import iter_params, select_config, select_overrides
30
+ from hydraflow.core.io import load_config
31
+ from hydraflow.core.param import get_params, get_values
32
+
33
+ from .run_data import RunCollectionData
34
+ from .run_info import RunCollectionInfo
34
35
 
35
36
  if TYPE_CHECKING:
36
37
  from collections.abc import Callable, Iterator
@@ -478,7 +479,7 @@ def _param_matches(run: Run, key: str, value: Any) -> bool:
478
479
  if param == "None":
479
480
  return value is None or value == "None"
480
481
 
481
- return hydraflow.param.match(param, value)
482
+ return hydraflow.core.param.match(param, value)
482
483
 
483
484
 
484
485
  def filter_runs(
@@ -6,14 +6,14 @@ from typing import TYPE_CHECKING
6
6
 
7
7
  from pandas import DataFrame
8
8
 
9
- from hydraflow.config import iter_params
10
- from hydraflow.utils import load_config
9
+ from hydraflow.core.config import iter_params
10
+ from hydraflow.core.io import load_config
11
11
 
12
12
  if TYPE_CHECKING:
13
13
  from collections.abc import Iterable
14
14
  from typing import Any
15
15
 
16
- from hydraflow.run_collection import RunCollection
16
+ from .run_collection import RunCollection
17
17
 
18
18
 
19
19
  class RunCollectionData:
@@ -4,12 +4,12 @@ from __future__ import annotations
4
4
 
5
5
  from typing import TYPE_CHECKING
6
6
 
7
- from hydraflow.utils import get_artifact_dir
7
+ from hydraflow.core.io import get_artifact_dir
8
8
 
9
9
  if TYPE_CHECKING:
10
10
  from pathlib import Path
11
11
 
12
- from hydraflow.run_collection import RunCollection
12
+ from .run_collection import RunCollection
13
13
 
14
14
 
15
15
  class RunCollectionInfo:
File without changes
@@ -0,0 +1,23 @@
1
+ from __future__ import annotations
2
+
3
+ from dataclasses import dataclass, field
4
+
5
+
6
+ @dataclass
7
+ class Step:
8
+ args: str = ""
9
+ batch: str = ""
10
+ options: str = ""
11
+
12
+
13
+ @dataclass
14
+ class Job:
15
+ name: str = ""
16
+ run: str = ""
17
+ call: str = ""
18
+ steps: list[Step] = field(default_factory=list)
19
+
20
+
21
+ @dataclass
22
+ class HydraflowConf:
23
+ jobs: dict[str, Job] = field(default_factory=dict)
@@ -0,0 +1,34 @@
1
+ """Hydraflow jobs IO."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from pathlib import Path
6
+
7
+ from omegaconf import OmegaConf
8
+
9
+ from .conf import HydraflowConf
10
+
11
+
12
+ def find_config_file() -> Path | None:
13
+ """Find the hydraflow config file."""
14
+ if Path("hydraflow.yaml").exists():
15
+ return Path("hydraflow.yaml")
16
+
17
+ if Path("hydraflow.yml").exists():
18
+ return Path("hydraflow.yml")
19
+
20
+ return None
21
+
22
+
23
+ def load_config() -> HydraflowConf:
24
+ """Load the hydraflow config."""
25
+ schema = OmegaConf.structured(HydraflowConf)
26
+
27
+ path = find_config_file()
28
+
29
+ if path is None:
30
+ return schema
31
+
32
+ cfg = OmegaConf.load(path)
33
+
34
+ return OmegaConf.merge(schema, cfg) # type: ignore
@@ -0,0 +1,152 @@
1
+ """Job execution and argument handling for HydraFlow.
2
+
3
+ This module provides functionality for executing jobs in HydraFlow, including:
4
+
5
+ - Argument parsing and expansion for job steps
6
+ - Batch processing of Hydra configurations
7
+ - Execution of jobs via shell commands or Python functions
8
+
9
+ The module supports two execution modes:
10
+
11
+ 1. Shell command execution
12
+ 2. Python function calls
13
+
14
+ Each job can consist of multiple steps, and each step can have its own
15
+ arguments and options that will be expanded into multiple runs.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import importlib
21
+ import shlex
22
+ import subprocess
23
+ from subprocess import CalledProcessError
24
+ from typing import TYPE_CHECKING
25
+
26
+ import ulid
27
+
28
+ from .parser import collect, expand
29
+
30
+ if TYPE_CHECKING:
31
+ from collections.abc import Iterator
32
+
33
+ from .conf import Job, Step
34
+
35
+
36
+ def iter_args(step: Step) -> Iterator[list[str]]:
37
+ """Iterate over combinations generated from parsed arguments.
38
+
39
+ Generate all possible combinations of arguments by parsing and
40
+ expanding each one, yielding them as an iterator.
41
+
42
+ Args:
43
+ step (Step): The step to parse.
44
+
45
+ Yields:
46
+ list[str]: a list of the parsed argument combinations.
47
+
48
+ """
49
+ args = collect(step.args)
50
+ options = [o for o in step.options.split(" ") if o]
51
+
52
+ for batch in expand(step.batch):
53
+ yield [*options, *sorted([*batch, *args])]
54
+
55
+
56
+ def iter_batches(job: Job) -> Iterator[list[str]]:
57
+ """Generate Hydra application arguments for a job.
58
+
59
+ This function generates a list of Hydra application arguments
60
+ for a given job, including the job name and the root directory
61
+ for the sweep.
62
+
63
+ Args:
64
+ job (Job): The job to generate the Hydra configuration for.
65
+
66
+ Returns:
67
+ list[str]: A list of Hydra configuration strings.
68
+
69
+ """
70
+ job_name = f"hydra.job.name={job.name}"
71
+
72
+ for step in job.steps:
73
+ for args in iter_args(step):
74
+ sweep_dir = f"hydra.sweep.dir=multirun/{ulid.ulid()}"
75
+ yield ["--multirun", sweep_dir, job_name, *args]
76
+
77
+
78
+ def multirun(job: Job) -> None:
79
+ """Execute multiple runs of a job using either shell commands or Python functions.
80
+
81
+ This function processes a job configuration and executes it in one of two modes:
82
+
83
+ 1. Shell command mode (job.run): Executes shell commands with the generated
84
+ arguments
85
+ 2. Python function mode (job.call): Calls a Python function with the generated
86
+ arguments
87
+
88
+ Args:
89
+ job (Job): The job configuration containing run parameters and steps.
90
+
91
+ Raises:
92
+ RuntimeError: If a shell command fails or if a function call encounters
93
+ an error.
94
+ ValueError: If the Python function path is invalid or the function cannot
95
+ be imported.
96
+
97
+ """
98
+ it = iter_batches(job)
99
+
100
+ if job.run:
101
+ base_cmds = shlex.split(job.run)
102
+ for args in it:
103
+ cmds = [*base_cmds, *args]
104
+ try:
105
+ subprocess.run(cmds, check=True)
106
+ except CalledProcessError as e:
107
+ msg = f"Command failed with exit code {e.returncode}"
108
+ raise RuntimeError(msg) from e
109
+
110
+ elif job.call:
111
+ if "." not in job.call:
112
+ msg = f"Invalid function path: {job.call}."
113
+ msg += " Expected format: 'package.module.function'"
114
+ raise ValueError(msg)
115
+
116
+ try:
117
+ module_name, func_name = job.call.rsplit(".", 1)
118
+ module = importlib.import_module(module_name)
119
+ func = getattr(module, func_name)
120
+ except (ImportError, AttributeError, ModuleNotFoundError) as e:
121
+ msg = f"Failed to import or find function: {job.call}"
122
+ raise ValueError(msg) from e
123
+
124
+ for args in it:
125
+ try:
126
+ func(*args)
127
+ except Exception as e: # noqa: PERF203
128
+ msg = f"Function call '{job.call}' failed with args: {args}"
129
+ raise RuntimeError(msg) from e
130
+
131
+
132
+ def show(job: Job) -> None:
133
+ """Show the job configuration.
134
+
135
+ This function shows the job configuration for a given job.
136
+
137
+ Args:
138
+ job (Job): The job configuration to show.
139
+
140
+ """
141
+ it = iter_batches(job)
142
+
143
+ if job.run:
144
+ base_cmds = shlex.split(job.run)
145
+ for args in it:
146
+ cmds = " ".join([*base_cmds, *args])
147
+ print(cmds) # noqa: T201
148
+
149
+ elif job.call:
150
+ print(f"call: {job.call}") # noqa: T201
151
+ for args in it:
152
+ print(f"args: {args}") # noqa: T201
@@ -0,0 +1,397 @@
1
+ """Parse and convert string representations of numbers and ranges.
2
+
3
+ This module provides utility functions for parsing and converting
4
+ string representations of numbers and ranges. It includes functions
5
+ to convert strings to numbers, count decimal places, handle numeric
6
+ ranges, and expand values from string arguments.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import re
12
+ from itertools import chain, product
13
+ from typing import TYPE_CHECKING
14
+
15
+ if TYPE_CHECKING:
16
+ from collections.abc import Iterator
17
+
18
+
19
+ def to_number(x: str) -> int | float:
20
+ """Convert a string to an integer or float.
21
+
22
+ Attempts to convert a string to an integer or a float,
23
+ returning 0 if the string is empty or cannot be converted.
24
+
25
+ Args:
26
+ x (str): The string to convert.
27
+
28
+ Returns:
29
+ int | float: The converted number as an integer or float.
30
+
31
+ Examples:
32
+ >>> type(to_number("1"))
33
+ <class 'int'>
34
+ >>> type(to_number("1.2"))
35
+ <class 'float'>
36
+ >>> to_number("")
37
+ 0
38
+ >>> to_number("1e-3")
39
+ 0.001
40
+
41
+ """
42
+ if not x:
43
+ return 0
44
+
45
+ if "." in x or "e" in x.lower():
46
+ return float(x)
47
+
48
+ return int(x)
49
+
50
+
51
+ def count_decimal_places(x: str) -> int:
52
+ """Count decimal places in a string.
53
+
54
+ Examine a string representing a number and returns the count
55
+ of decimal places present after the decimal point.
56
+ Return 0 if no decimal point is found.
57
+
58
+ Args:
59
+ x (str): The string to check.
60
+
61
+ Returns:
62
+ int: The number of decimal places.
63
+
64
+ Examples:
65
+ >>> count_decimal_places("1")
66
+ 0
67
+ >>> count_decimal_places("-1.2")
68
+ 1
69
+ >>> count_decimal_places("1.234")
70
+ 3
71
+ >>> count_decimal_places("-1.234e-10")
72
+ 3
73
+
74
+ """
75
+ if "." not in x:
76
+ return 0
77
+
78
+ decimal_part = x.split(".")[1]
79
+ if "e" in decimal_part.lower():
80
+ decimal_part = decimal_part.split("e")[0]
81
+
82
+ return len(decimal_part)
83
+
84
+
85
+ def is_number(x: str) -> bool:
86
+ """Check if a string is a number.
87
+
88
+ Args:
89
+ x (str): The string to check.
90
+
91
+ Returns:
92
+ bool: True if the string is a number, False otherwise.
93
+
94
+ Examples:
95
+ >>> is_number("1")
96
+ True
97
+ >>> is_number("-1.2")
98
+ True
99
+ >>> is_number("1.2.3")
100
+ False
101
+
102
+ """
103
+ try:
104
+ float(x)
105
+ except ValueError:
106
+ return False
107
+ return True
108
+
109
+
110
+ SUFFIX_EXPONENT = {
111
+ "T": "e12",
112
+ "G": "e9",
113
+ "M": "e6",
114
+ "k": "e3",
115
+ "m": "e-3",
116
+ "u": "e-6",
117
+ "n": "e-9",
118
+ "p": "e-12",
119
+ "f": "e-15",
120
+ }
121
+
122
+
123
+ def _get_range(arg: str) -> tuple[float, float, float]:
124
+ args = [to_number(x) for x in arg.split(":")]
125
+
126
+ if len(args) == 2:
127
+ if args[0] > args[1]:
128
+ raise ValueError("start cannot be greater than stop")
129
+
130
+ return (args[0], 1, args[1])
131
+
132
+ if args[1] == 0:
133
+ raise ValueError("step cannot be zero")
134
+ if args[1] > 0 and args[0] > args[2]:
135
+ raise ValueError("start cannot be greater than stop")
136
+ if args[1] < 0 and args[0] < args[2]:
137
+ raise ValueError("start cannot be less than stop")
138
+
139
+ return args[0], args[1], args[2]
140
+
141
+
142
+ def _arange(start: float, step: float, stop: float) -> list[float]:
143
+ result = []
144
+ current = start
145
+
146
+ while current <= stop if step > 0 else current >= stop:
147
+ result.append(current)
148
+ current += step
149
+
150
+ return result
151
+
152
+
153
+ def split_suffix(arg: str) -> tuple[str, str]:
154
+ """Split a string into prefix and suffix.
155
+
156
+ Args:
157
+ arg (str): The string to split.
158
+
159
+ Returns:
160
+ tuple[str, str]: A tuple containing the prefix and suffix.
161
+
162
+ Examples:
163
+ >>> split_suffix("1:k")
164
+ ('1', 'e3')
165
+ >>> split_suffix("1:2:k")
166
+ ('1:2', 'e3')
167
+ >>> split_suffix("1:2:M")
168
+ ('1:2', 'e6')
169
+ >>> split_suffix(":1:2:M")
170
+ (':1:2', 'e6')
171
+
172
+ """
173
+ if len(arg) < 3 or ":" not in arg:
174
+ return arg, ""
175
+
176
+ prefix, suffix = arg.rsplit(":", 1)
177
+
178
+ if suffix.lower().startswith("e"):
179
+ return prefix, suffix
180
+
181
+ if suffix not in SUFFIX_EXPONENT:
182
+ return arg, ""
183
+
184
+ return prefix, SUFFIX_EXPONENT[suffix]
185
+
186
+
187
+ def add_exponent(value: str, exponent: str) -> str:
188
+ """Append an exponent to a value string.
189
+
190
+ Args:
191
+ value (str): The value to modify.
192
+ exponent (str): The exponent to append.
193
+
194
+ Returns:
195
+ str: The value with the exponent added.
196
+
197
+ """
198
+ if value in ["0", "0.", "0.0"] or not exponent:
199
+ return value
200
+
201
+ return f"{value}{exponent}"
202
+
203
+
204
+ def collect_values(arg: str) -> list[str]:
205
+ """Collect a list of values from a range argument.
206
+
207
+ Collect all individual values within a numeric range
208
+ represented by a string (e.g., `1:4`) and return them
209
+ as a list of strings.
210
+ Support both integer and floating-point ranges.
211
+
212
+ Args:
213
+ arg (str): The argument to collect.
214
+
215
+ Returns:
216
+ list[str]: A list of the collected values.
217
+
218
+ """
219
+ if ":" not in arg:
220
+ return [arg]
221
+
222
+ arg, exponent = split_suffix(arg)
223
+
224
+ if ":" not in arg:
225
+ return [f"{arg}{exponent}"]
226
+
227
+ rng = _get_range(arg)
228
+
229
+ if all(isinstance(x, int) for x in rng):
230
+ values = [str(x) for x in _arange(*rng)]
231
+ else:
232
+ n = max(*(count_decimal_places(x) for x in arg.split(":")))
233
+ values = [str(round(x, n)) for x in _arange(*rng)]
234
+
235
+ return [add_exponent(x, exponent) for x in values]
236
+
237
+
238
+ def split(arg: str) -> list[str]:
239
+ r"""Split a string by top-level commas.
240
+
241
+ Splits a string by commas while respecting nested structures.
242
+ Commas inside brackets and quotes are ignored, only splitting
243
+ at the top-level commas.
244
+
245
+ Args:
246
+ arg (str): The string to split.
247
+
248
+ Returns:
249
+ list[str]: A list of split strings.
250
+
251
+ Examples:
252
+ >>> split("[a,1],[b,2]")
253
+ ['[a,1]', '[b,2]']
254
+ >>> split('"x,y",z')
255
+ ['"x,y"', 'z']
256
+ >>> split("'p,q',r")
257
+ ["'p,q'", 'r']
258
+
259
+ """
260
+ result = []
261
+ current = []
262
+ bracket_count = 0
263
+ in_single_quote = False
264
+ in_double_quote = False
265
+
266
+ for char in arg:
267
+ if char == "'" and not in_double_quote:
268
+ in_single_quote = not in_single_quote
269
+ elif char == '"' and not in_single_quote:
270
+ in_double_quote = not in_double_quote
271
+ elif char == "[" and not (in_single_quote or in_double_quote):
272
+ bracket_count += 1
273
+ elif char == "]" and not (in_single_quote or in_double_quote):
274
+ bracket_count -= 1
275
+ elif (
276
+ char == ","
277
+ and bracket_count == 0
278
+ and not in_single_quote
279
+ and not in_double_quote
280
+ ):
281
+ result.append("".join(current))
282
+ current = []
283
+ continue
284
+ current.append(char)
285
+
286
+ if current:
287
+ result.append("".join(current))
288
+
289
+ return result
290
+
291
+
292
+ def expand_values(arg: str) -> list[str]:
293
+ """Expand a string argument into a list of values.
294
+
295
+ Take a string containing comma-separated values or ranges and return a list
296
+ of all individual values. Handle numeric ranges and special characters.
297
+
298
+ Args:
299
+ arg (str): The argument to expand.
300
+
301
+ Returns:
302
+ list[str]: A list of the expanded values.
303
+
304
+ """
305
+ return list(chain.from_iterable(collect_values(x) for x in split(arg)))
306
+
307
+
308
+ def collect_arg(arg: str) -> str:
309
+ """Collect a string of expanded key-value pairs.
310
+
311
+ Take a key-value pair argument and concatenates all expanded values with commas,
312
+ returning a single string suitable for command-line usage.
313
+
314
+ Args:
315
+ arg (str): The argument to collect.
316
+
317
+ Returns:
318
+ str: A string of the collected key and values.
319
+
320
+ """
321
+ key, arg = arg.split("=")
322
+ arg = ",".join(expand_values(arg))
323
+ return f"{key}={arg}"
324
+
325
+
326
+ def expand_arg(arg: str) -> Iterator[str]:
327
+ """Parse a string argument into a list of values.
328
+
329
+ Responsible for parsing a string that may contain multiple
330
+ arguments separated by pipes ("|") and returns a list of all
331
+ expanded arguments.
332
+
333
+ Args:
334
+ arg (str): The argument to parse.
335
+
336
+ Returns:
337
+ list[str]: A list of the parsed arguments.
338
+
339
+ """
340
+ if "|" not in arg:
341
+ key, value = arg.split("=")
342
+
343
+ for v in expand_values(value):
344
+ yield f"{key}={v}"
345
+
346
+ return
347
+
348
+ args = arg.split("|")
349
+ key = ""
350
+
351
+ for arg_ in args:
352
+ if "=" in arg_:
353
+ key, value = arg_.split("=")
354
+ elif key:
355
+ value = arg_
356
+ else:
357
+ msg = f"Invalid argument: {arg_}"
358
+ raise ValueError(msg)
359
+
360
+ value = ",".join(expand_values(value))
361
+ yield f"{key}={value}"
362
+
363
+
364
+ def collect(args: str | list[str]) -> list[str]:
365
+ """Collect a list of arguments into a list of strings.
366
+
367
+ Args:
368
+ args (list[str]): The arguments to collect.
369
+
370
+ Returns:
371
+ list[str]: A list of the collected arguments.
372
+
373
+ """
374
+ if isinstance(args, str):
375
+ args = re.split(r"\s+", args.strip())
376
+
377
+ args = [arg for arg in args if "=" in arg]
378
+
379
+ return [collect_arg(arg) for arg in args]
380
+
381
+
382
+ def expand(args: str | list[str]) -> list[list[str]]:
383
+ """Expand a list of arguments into a list of lists of strings.
384
+
385
+ Args:
386
+ args (list[str]): The arguments to expand.
387
+
388
+ Returns:
389
+ list[list[str]]: A list of the expanded arguments.
390
+
391
+ """
392
+ if isinstance(args, str):
393
+ args = re.split(r"\s+", args.strip())
394
+
395
+ args = [arg for arg in args if "=" in arg]
396
+
397
+ return [list(x) for x in product(*(expand_arg(arg) for arg in args))]
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: hydraflow
3
- Version: 0.8.0
3
+ Version: 0.9.0
4
4
  Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
5
5
  Project-URL: Documentation, https://daizutabi.github.io/hydraflow/
6
6
  Project-URL: Source, https://github.com/daizutabi/hydraflow
@@ -41,6 +41,7 @@ Requires-Dist: mlflow>=2.15
41
41
  Requires-Dist: omegaconf
42
42
  Requires-Dist: rich
43
43
  Requires-Dist: typer
44
+ Requires-Dist: ulid
44
45
  Description-Content-Type: text/markdown
45
46
 
46
47
  # Hydraflow
@@ -93,31 +94,29 @@ pip install hydraflow
93
94
  Here is a simple example to get you started with Hydraflow:
94
95
 
95
96
  ```python
96
- import hydra
97
- import hydraflow
98
- import mlflow
97
+ from __future__ import annotations
98
+
99
99
  from dataclasses import dataclass
100
- from hydra.core.config_store import ConfigStore
101
100
  from pathlib import Path
101
+ from typing import TYPE_CHECKING
102
102
 
103
- @dataclass
104
- class MySQLConfig:
105
- host: str = "localhost"
106
- port: int = 3306
103
+ import hydraflow
107
104
 
108
- cs = ConfigStore.instance()
109
- cs.store(name="config", node=MySQLConfig)
105
+ if TYPE_CHECKING:
106
+ from mlflow.entities import Run
107
+
108
+
109
+ @dataclass
110
+ class Config:
111
+ count: int = 1
112
+ name: str = "a"
110
113
 
111
- @hydra.main(config_name="config", version_base=None)
112
- def my_app(cfg: MySQLConfig) -> None:
113
- # Set experiment by Hydra job name.
114
- hydraflow.set_experiment()
115
114
 
116
- # Automatically log Hydra config as params.
117
- with hydraflow.start_run(cfg):
118
- # Your app code below.
115
+ @hydraflow.main(Config)
116
+ def app(run: Run, cfg: Config):
117
+ """Your app code here."""
119
118
 
120
119
 
121
120
  if __name__ == "__main__":
122
- my_app()
121
+ app()
123
122
  ```
@@ -0,0 +1,24 @@
1
+ hydraflow/__init__.py,sha256=WnReG-E2paQSMLJm42N-KaQyHYsfA4OX-WWcst610PI,738
2
+ hydraflow/cli.py,sha256=gbDPj49azP8CCGxkxU0rksh1-gCyjP0VkVYH34ktcsA,1338
3
+ hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
4
+ hydraflow/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
5
+ hydraflow/core/config.py,sha256=SJzjgsO_kzB78_whJ3lmy7GlZvTvwZONH1BJBn8zCuI,3817
6
+ hydraflow/core/context.py,sha256=QPyPg1xrTlmhviKNn-0nDY9bXcVky1zInqRqPN-VNhc,4741
7
+ hydraflow/core/io.py,sha256=T4ESiepEcqR-FZlo_m7VTBEFMwalrqPI8eFKPagvv3Q,4402
8
+ hydraflow/core/main.py,sha256=gYb1OOVH0CL4385Dm-06Mqi1Mr9-24URwLUiW86pGNs,5018
9
+ hydraflow/core/mlflow.py,sha256=M3MhiChnMzKnKRmjBl4h_SRGkAZKL7GAmFr3DdzwRuQ,5666
10
+ hydraflow/core/param.py,sha256=LHU9j9_7oA99igasoOyKofKClVr9FmGA3UABJ-KmyS0,4538
11
+ hydraflow/entities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
12
+ hydraflow/entities/run_collection.py,sha256=BHbMvYB4WyG5qSNYpIzwmHySOtmGNb-jg2OrCaTPr2I,19651
13
+ hydraflow/entities/run_data.py,sha256=Y2_Lc-BdQ7nXhcEIjdHGHIkLrXsmAktOftESEwYOY8o,1602
14
+ hydraflow/entities/run_info.py,sha256=FRC6ICOlzB2u_xi_33Qs-YZLt677UotuNbYqI7XSmHY,1017
15
+ hydraflow/executor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
16
+ hydraflow/executor/conf.py,sha256=q_FrPXQJCVGKS1FYnGRGqTUgMQeMBkaVPW2mtQc8oxk,384
17
+ hydraflow/executor/io.py,sha256=4nafwge6vHanYFuEHxd0LRv_3ZLgMpV50qSbssZNe3Q,696
18
+ hydraflow/executor/job.py,sha256=Vp2IZOuIC25Gqo9A_5MkllWl1T1QBfUnI5ksMvKwakg,4479
19
+ hydraflow/executor/parser.py,sha256=y4C9wVdUnazJDxdWrT5y3yWFIo0zAGzO-cS9x1MTK_8,9486
20
+ hydraflow-0.9.0.dist-info/METADATA,sha256=p8-vmPIYV-uU9BJbSB1roqrhNO7nK2HRdrpj8Tip1AA,4559
21
+ hydraflow-0.9.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
22
+ hydraflow-0.9.0.dist-info/entry_points.txt,sha256=XI0khPbpCIUo9UPqkNEpgh-kqK3Jy8T7L2VCWOdkbSM,48
23
+ hydraflow-0.9.0.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
24
+ hydraflow-0.9.0.dist-info/RECORD,,
@@ -1,17 +0,0 @@
1
- hydraflow/__init__.py,sha256=yp4LT1FDYPIduR6PqJNuSm9kztVCpL1P0zcPHWGvaJU,712
2
- hydraflow/cli.py,sha256=jxqFppNeJWAr2Tb-C_MQXEJtegJ6TXcd3C1CT7Jdb1A,1559
3
- hydraflow/config.py,sha256=SJzjgsO_kzB78_whJ3lmy7GlZvTvwZONH1BJBn8zCuI,3817
4
- hydraflow/context.py,sha256=H5xeNbhMS23U-epsucprl5G3lbOR1aO9nDES4QGLWNk,4747
5
- hydraflow/main.py,sha256=O5ETCMCg12zXoaYlZMHcM4IYAs6GVTkADrmEssrtjkk,4994
6
- hydraflow/mlflow.py,sha256=pRRsBaBBH4cfzSko-8mmo5bV04GGklxoO0kORkInypM,5663
7
- hydraflow/param.py,sha256=LHU9j9_7oA99igasoOyKofKClVr9FmGA3UABJ-KmyS0,4538
8
- hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
9
- hydraflow/run_collection.py,sha256=rtH1cglSlK3QFg9hhifo9lzjDa9veHpoyYxEOmIEM84,19646
10
- hydraflow/run_data.py,sha256=S2NNFtA1TleqpgeK4mIn1YY8YbWJFyhF7wXR5NWeYLk,1604
11
- hydraflow/run_info.py,sha256=Jf5wrIjRLIV1-k-obHDqwKHa6j_ZonrY8od-rXlbtMo,1024
12
- hydraflow/utils.py,sha256=T4ESiepEcqR-FZlo_m7VTBEFMwalrqPI8eFKPagvv3Q,4402
13
- hydraflow-0.8.0.dist-info/METADATA,sha256=J1ilgG7L4A8OvzgZSNycp0YgyHk5e8_gwTr9NN82Ejk,4767
14
- hydraflow-0.8.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
15
- hydraflow-0.8.0.dist-info/entry_points.txt,sha256=XI0khPbpCIUo9UPqkNEpgh-kqK3Jy8T7L2VCWOdkbSM,48
16
- hydraflow-0.8.0.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
17
- hydraflow-0.8.0.dist-info/RECORD,,
File without changes
File without changes
File without changes