hydraflow 0.10.2__py3-none-any.whl → 0.11.1__py3-none-any.whl

hydraflow/executor/conf.py

@@ -5,9 +5,9 @@ from dataclasses import dataclass, field
 
 @dataclass
 class Step:
-    args: str = ""
     batch: str = ""
-    options: str = ""
+    args: str = ""
+    configs: str = ""
 
 
 @dataclass
@@ -15,6 +15,7 @@ class Job:
     name: str = ""
     run: str = ""
     call: str = ""
+    configs: str = ""
     steps: list[Step] = field(default_factory=list)
 
 

hydraflow/executor/job.py

@@ -12,7 +12,7 @@ The module supports two execution modes:
 2. Python function calls
 
 Each job can consist of multiple steps, and each step can have its own
-arguments and options that will be expanded into multiple runs.
+arguments and configurations that will be expanded into multiple runs.
 """
 
 from __future__ import annotations
@@ -31,27 +31,27 @@ from .parser import collect, expand
 if TYPE_CHECKING:
     from collections.abc import Iterator
 
-    from .conf import Job, Step
+    from .conf import Job
 
 
-def iter_args(step: Step) -> Iterator[list[str]]:
+def iter_args(batch: str, args: str) -> Iterator[list[str]]:
     """Iterate over combinations generated from parsed arguments.
 
     Generate all possible combinations of arguments by parsing and
     expanding each one, yielding them as an iterator.
 
     Args:
-        step (Step): The step to parse.
+        batch (str): The batch to parse.
+        args (str): The arguments to parse.
 
     Yields:
         list[str]: a list of the parsed argument combinations.
 
     """
-    args = collect(step.args)
-    options = [o for o in step.options.split(" ") if o]
+    args_ = collect(args)
 
-    for batch in expand(step.batch):
-        yield [*options, *sorted([*batch, *args])]
+    for batch_ in expand(batch):
+        yield [*batch_, *args_]
 
 
 def iter_batches(job: Job) -> Iterator[list[str]]:
@@ -69,11 +69,14 @@ def iter_batches(job: Job) -> Iterator[list[str]]:
 
     """
     job_name = f"hydra.job.name={job.name}"
+    job_configs = shlex.split(job.configs)
 
     for step in job.steps:
-        for args in iter_args(step):
+        configs = shlex.split(step.configs) or job_configs
+
+        for args in iter_args(step.batch, step.args):
             sweep_dir = f"hydra.sweep.dir=multirun/{ulid.ULID()}"
-            yield ["--multirun", sweep_dir, job_name, *args]
+            yield ["--multirun", *args, job_name, sweep_dir, *configs]
 
 
 def multirun(job: Job) -> None:
@@ -162,4 +165,4 @@ def to_text(job: Job) -> str:
         for args in it:
             text += f"args: {args}\n"
 
-    return text
+    return text.rstrip()

hydraflow/executor/parser.py

@@ -216,6 +216,9 @@ def collect_values(arg: str) -> list[str]:
         list[str]: A list of the collected values.
 
     """
+    if "(" in arg:
+        return collect_parentheses(arg)
+
     if ":" not in arg:
         return [arg]
 
@@ -235,6 +238,58 @@ def collect_values(arg: str) -> list[str]:
     return [add_exponent(x, exponent) for x in values]
 
 
+def split_parentheses(arg: str) -> Iterator[str]:
+    """Split a string with parentheses into a list of strings.
+
+    Args:
+        arg (str): The string to split.
+
+    Returns:
+        Iterator[str]: An iterator of the split strings.
+
+    Examples:
+        >>> list(split_parentheses("a(b,c)m(e:f)k"))
+        ['a', 'b,c', 'e-3', 'e:f', 'e3']
+        >>> list(split_parentheses("(b,c)d(e:f)"))
+        ['b,c', 'd', 'e:f']
+
+    """
+    current = ""
+
+    for char in arg:
+        if char in ("(", ")"):
+            if current:
+                yield SUFFIX_EXPONENT.get(current, current)
+                current = ""
+        else:
+            current += char
+
+    if current:
+        yield SUFFIX_EXPONENT.get(current, current)
+
+
+def collect_parentheses(arg: str) -> list[str]:
+    """Collect values from a string with parentheses.
+
+    Args:
+        arg (str): The string to collect values from.
+
+    Returns:
+        list[str]: A list of the collected values.
+
+    Examples:
+        >>> collect_parentheses("(1:3,5:2:9,20)k")
+        ['1e3', '2e3', '3e3', '5e3', '7e3', '9e3', '20e3']
+        >>> collect_parentheses("2e(-1,-2,-3)")
+        ['2e-1', '2e-2', '2e-3']
+        >>> collect_parentheses("(1:3)e(3,5)")
+        ['1e3', '2e3', '3e3', '1e5', '2e5', '3e5']
+
+    """
+    it = [expand_values(x) for x in split_parentheses(arg)]
+    return ["".join(x[::-1]) for x in product(*it[::-1])]
+
+
 def split(arg: str) -> list[str]:
     r"""Split a string by top-level commas.
 
@@ -255,11 +310,14 @@ def split(arg: str) -> list[str]:
         ['"x,y"', 'z']
         >>> split("'p,q',r")
         ["'p,q'", 'r']
+        >>> split("(a,b)m,(1,2:4)k")
+        ['(a,b)m', '(1,2:4)k']
 
     """
     result = []
     current = []
     bracket_count = 0
+    paren_count = 0
     in_single_quote = False
     in_double_quote = False
 
@@ -272,9 +330,14 @@ def split(arg: str) -> list[str]:
             bracket_count += 1
         elif char == "]" and not (in_single_quote or in_double_quote):
             bracket_count -= 1
+        elif char == "(" and not (in_single_quote or in_double_quote):
+            paren_count += 1
+        elif char == ")" and not (in_single_quote or in_double_quote):
+            paren_count -= 1
         elif (
             char == ","
             and bracket_count == 0
+            and paren_count == 0
             and not in_single_quote
             and not in_double_quote
         ):
@@ -303,8 +366,7 @@ def expand_values(arg: str, suffix: str = "") -> Iterator[str]:
         Iterator[str]: An iterator of the expanded values.
 
     """
-    if suffix in SUFFIX_EXPONENT:
-        suffix = SUFFIX_EXPONENT[suffix]
+    suffix = SUFFIX_EXPONENT.get(suffix, suffix)
 
     for value in chain.from_iterable(collect_values(x) for x in split(arg)):
         yield f"{value}{suffix}"
@@ -320,6 +382,10 @@ def split_arg(arg: str) -> tuple[str, str, str]:
         tuple[str, str, str]: A tuple containing the key, suffix, and value.
 
     """
+    if "=" not in arg:
+        msg = f"Invalid argument: {arg}"
+        raise ValueError(msg)
+
     key, value = arg.split("=")
 
     if "/" in key:
@@ -399,8 +465,6 @@ def collect(args: str | list[str]) -> list[str]:
     if isinstance(args, str):
         args = shlex.split(args)
 
-    args = [arg for arg in args if "=" in arg]
-
     return [collect_arg(arg) for arg in args]
 
 
@@ -417,6 +481,4 @@ def expand(args: str | list[str]) -> list[list[str]]:
     if isinstance(args, str):
         args = shlex.split(args)
 
-    args = [arg for arg in args if "=" in arg]
-
     return [list(x) for x in product(*(expand_arg(arg) for arg in args))]

{hydraflow-0.10.2.dist-info → hydraflow-0.11.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydraflow
-Version: 0.10.2
+Version: 0.11.1
 Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
 Project-URL: Documentation, https://daizutabi.github.io/hydraflow/
 Project-URL: Source, https://github.com/daizutabi/hydraflow
@@ -97,7 +97,6 @@ Here is a simple example to get you started with Hydraflow:
 from __future__ import annotations
 
 from dataclasses import dataclass
-from pathlib import Path
 from typing import TYPE_CHECKING
 
 import hydraflow

{hydraflow-0.10.2.dist-info → hydraflow-0.11.1.dist-info}/RECORD

@@ -13,12 +13,12 @@ hydraflow/entities/run_collection.py,sha256=E8IRBgxCnJE_IPCaSmS2mc9GtDXXLBfc7GHv
 hydraflow/entities/run_data.py,sha256=Y2_Lc-BdQ7nXhcEIjdHGHIkLrXsmAktOftESEwYOY8o,1602
 hydraflow/entities/run_info.py,sha256=FRC6ICOlzB2u_xi_33Qs-YZLt677UotuNbYqI7XSmHY,1017
 hydraflow/executor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-hydraflow/executor/conf.py,sha256=q_FrPXQJCVGKS1FYnGRGqTUgMQeMBkaVPW2mtQc8oxk,384
+hydraflow/executor/conf.py,sha256=SJNiQ87MXMlpDfdm0POcv55MY3GS5FUh5wT7u3XU3oU,406
 hydraflow/executor/io.py,sha256=xV3m-nV9eKbu9Fb7u04J2bfmR_Ky3jTEJjq4QC2m6V4,954
-hydraflow/executor/job.py,sha256=YZ9JuVrdAcG3_HvNngHy9NL4-3bjJTaColE_5efF8TU,4743
-hydraflow/executor/parser.py,sha256=MO8VU0uVQZeku6kbw8Urid_5QEcnR8atd5h-yDP5OhQ,10147
-hydraflow-0.10.2.dist-info/METADATA,sha256=BidbFHVOZjuNytZBhYAg9_T5OBkmu4VNprJJFxnCqPI,4574
-hydraflow-0.10.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-hydraflow-0.10.2.dist-info/entry_points.txt,sha256=XI0khPbpCIUo9UPqkNEpgh-kqK3Jy8T7L2VCWOdkbSM,48
-hydraflow-0.10.2.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
-hydraflow-0.10.2.dist-info/RECORD,,
+hydraflow/executor/job.py,sha256=yiVbAYgsZZtSTRER-H5pUopULeygzPNzKlSkl27yFI4,4856
+hydraflow/executor/parser.py,sha256=6oqMlAZWykukZWLpRuFUaXXqROxIyqSEacDtNOxBCjw,11894
+hydraflow-0.11.1.dist-info/METADATA,sha256=iPCvp7q_NK3wwRSfvE3HzxBUfJ5kQW9r9kAd3hdazGc,4549
+hydraflow-0.11.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+hydraflow-0.11.1.dist-info/entry_points.txt,sha256=XI0khPbpCIUo9UPqkNEpgh-kqK3Jy8T7L2VCWOdkbSM,48
+hydraflow-0.11.1.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
+hydraflow-0.11.1.dist-info/RECORD,,