hydraflow 0.7.5__py3-none-any.whl → 0.9.0__py3-none-any.whl

hydraflow/executor/job.py

@@ -0,0 +1,152 @@
+"""Job execution and argument handling for HydraFlow.
+
+This module provides functionality for executing jobs in HydraFlow, including:
+
+- Argument parsing and expansion for job steps
+- Batch processing of Hydra configurations
+- Execution of jobs via shell commands or Python functions
+
+The module supports two execution modes:
+
+1. Shell command execution
+2. Python function calls
+
+Each job can consist of multiple steps, and each step can have its own
+arguments and options that will be expanded into multiple runs.
+"""
+
+from __future__ import annotations
+
+import importlib
+import shlex
+import subprocess
+from subprocess import CalledProcessError
+from typing import TYPE_CHECKING
+
+import ulid
+
+from .parser import collect, expand
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+    from .conf import Job, Step
+
+
+def iter_args(step: Step) -> Iterator[list[str]]:
+    """Iterate over combinations generated from parsed arguments.
+
+    Generate all possible combinations of arguments by parsing and
+    expanding each one, yielding them as an iterator.
+
+    Args:
+        step (Step): The step to parse.
+
+    Yields:
+        list[str]: a list of the parsed argument combinations.
+
+    """
+    args = collect(step.args)
+    options = [o for o in step.options.split(" ") if o]
+
+    for batch in expand(step.batch):
+        yield [*options, *sorted([*batch, *args])]
+
+
+def iter_batches(job: Job) -> Iterator[list[str]]:
+    """Generate Hydra application arguments for a job.
+
+    This function generates a list of Hydra application arguments
+    for a given job, including the job name and the root directory
+    for the sweep.
+
+    Args:
+        job (Job): The job to generate the Hydra configuration for.
+
+    Returns:
+        list[str]: A list of Hydra configuration strings.
+
+    """
+    job_name = f"hydra.job.name={job.name}"
+
+    for step in job.steps:
+        for args in iter_args(step):
+            sweep_dir = f"hydra.sweep.dir=multirun/{ulid.ulid()}"
+            yield ["--multirun", sweep_dir, job_name, *args]
+
+
+def multirun(job: Job) -> None:
+    """Execute multiple runs of a job using either shell commands or Python functions.
+
+    This function processes a job configuration and executes it in one of two modes:
+
+    1. Shell command mode (job.run): Executes shell commands with the generated
+       arguments
+    2. Python function mode (job.call): Calls a Python function with the generated
+       arguments
+
+    Args:
+        job (Job): The job configuration containing run parameters and steps.
+
+    Raises:
+        RuntimeError: If a shell command fails or if a function call encounters
+            an error.
+        ValueError: If the Python function path is invalid or the function cannot
+            be imported.
+
+    """
+    it = iter_batches(job)
+
+    if job.run:
+        base_cmds = shlex.split(job.run)
+        for args in it:
+            cmds = [*base_cmds, *args]
+            try:
+                subprocess.run(cmds, check=True)
+            except CalledProcessError as e:
+                msg = f"Command failed with exit code {e.returncode}"
+                raise RuntimeError(msg) from e
+
+    elif job.call:
+        if "." not in job.call:
+            msg = f"Invalid function path: {job.call}."
+            msg += " Expected format: 'package.module.function'"
+            raise ValueError(msg)
+
+        try:
+            module_name, func_name = job.call.rsplit(".", 1)
+            module = importlib.import_module(module_name)
+            func = getattr(module, func_name)
+        except (ImportError, AttributeError, ModuleNotFoundError) as e:
+            msg = f"Failed to import or find function: {job.call}"
+            raise ValueError(msg) from e
+
+        for args in it:
+            try:
+                func(*args)
+            except Exception as e:  # noqa: PERF203
+                msg = f"Function call '{job.call}' failed with args: {args}"
+                raise RuntimeError(msg) from e
+
+
+def show(job: Job) -> None:
+    """Show the job configuration.
+
+    This function shows the job configuration for a given job.
+
+    Args:
+        job (Job): The job configuration to show.
+
+    """
+    it = iter_batches(job)
+
+    if job.run:
+        base_cmds = shlex.split(job.run)
+        for args in it:
+            cmds = " ".join([*base_cmds, *args])
+            print(cmds)  # noqa: T201
+
+    elif job.call:
+        print(f"call: {job.call}")  # noqa: T201
+        for args in it:
+            print(f"args: {args}")  # noqa: T201

hydraflow/executor/parser.py

@@ -0,0 +1,397 @@
+"""Parse and convert string representations of numbers and ranges.
+
+This module provides utility functions for parsing and converting
+string representations of numbers and ranges. It includes functions
+to convert strings to numbers, count decimal places, handle numeric
+ranges, and expand values from string arguments.
+"""
+
+from __future__ import annotations
+
+import re
+from itertools import chain, product
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+
+
+def to_number(x: str) -> int | float:
+    """Convert a string to an integer or float.
+
+    Attempts to convert a string to an integer or a float,
+    returning 0 if the string is empty or cannot be converted.
+
+    Args:
+        x (str): The string to convert.
+
+    Returns:
+        int | float: The converted number as an integer or float.
+
+    Examples:
+        >>> type(to_number("1"))
+        <class 'int'>
+        >>> type(to_number("1.2"))
+        <class 'float'>
+        >>> to_number("")
+        0
+        >>> to_number("1e-3")
+        0.001
+
+    """
+    if not x:
+        return 0
+
+    if "." in x or "e" in x.lower():
+        return float(x)
+
+    return int(x)
+
+
+def count_decimal_places(x: str) -> int:
+    """Count decimal places in a string.
+
+    Examine a string representing a number and returns the count
+    of decimal places present after the decimal point.
+    Return 0 if no decimal point is found.
+
+    Args:
+        x (str): The string to check.
+
+    Returns:
+        int: The number of decimal places.
+
+    Examples:
+        >>> count_decimal_places("1")
+        0
+        >>> count_decimal_places("-1.2")
+        1
+        >>> count_decimal_places("1.234")
+        3
+        >>> count_decimal_places("-1.234e-10")
+        3
+
+    """
+    if "." not in x:
+        return 0
+
+    decimal_part = x.split(".")[1]
+    if "e" in decimal_part.lower():
+        decimal_part = decimal_part.split("e")[0]
+
+    return len(decimal_part)
+
+
+def is_number(x: str) -> bool:
+    """Check if a string is a number.
+
+    Args:
+        x (str): The string to check.
+
+    Returns:
+        bool: True if the string is a number, False otherwise.
+
+    Examples:
+        >>> is_number("1")
+        True
+        >>> is_number("-1.2")
+        True
+        >>> is_number("1.2.3")
+        False
+
+    """
+    try:
+        float(x)
+    except ValueError:
+        return False
+    return True
+
+
+SUFFIX_EXPONENT = {
+    "T": "e12",
+    "G": "e9",
+    "M": "e6",
+    "k": "e3",
+    "m": "e-3",
+    "u": "e-6",
+    "n": "e-9",
+    "p": "e-12",
+    "f": "e-15",
+}
+
+
+def _get_range(arg: str) -> tuple[float, float, float]:
+    args = [to_number(x) for x in arg.split(":")]
+
+    if len(args) == 2:
+        if args[0] > args[1]:
+            raise ValueError("start cannot be greater than stop")
+
+        return (args[0], 1, args[1])
+
+    if args[1] == 0:
+        raise ValueError("step cannot be zero")
+    if args[1] > 0 and args[0] > args[2]:
+        raise ValueError("start cannot be greater than stop")
+    if args[1] < 0 and args[0] < args[2]:
+        raise ValueError("start cannot be less than stop")
+
+    return args[0], args[1], args[2]
+
+
+def _arange(start: float, step: float, stop: float) -> list[float]:
+    result = []
+    current = start
+
+    while current <= stop if step > 0 else current >= stop:
+        result.append(current)
+        current += step
+
+    return result
+
+
+def split_suffix(arg: str) -> tuple[str, str]:
+    """Split a string into prefix and suffix.
+
+    Args:
+        arg (str): The string to split.
+
+    Returns:
+        tuple[str, str]: A tuple containing the prefix and suffix.
+
+    Examples:
+        >>> split_suffix("1:k")
+        ('1', 'e3')
+        >>> split_suffix("1:2:k")
+        ('1:2', 'e3')
+        >>> split_suffix("1:2:M")
+        ('1:2', 'e6')
+        >>> split_suffix(":1:2:M")
+        (':1:2', 'e6')
+
+    """
+    if len(arg) < 3 or ":" not in arg:
+        return arg, ""
+
+    prefix, suffix = arg.rsplit(":", 1)
+
+    if suffix.lower().startswith("e"):
+        return prefix, suffix
+
+    if suffix not in SUFFIX_EXPONENT:
+        return arg, ""
+
+    return prefix, SUFFIX_EXPONENT[suffix]
+
+
+def add_exponent(value: str, exponent: str) -> str:
+    """Append an exponent to a value string.
+
+    Args:
+        value (str): The value to modify.
+        exponent (str): The exponent to append.
+
+    Returns:
+        str: The value with the exponent added.
+
+    """
+    if value in ["0", "0.", "0.0"] or not exponent:
+        return value
+
+    return f"{value}{exponent}"
+
+
+def collect_values(arg: str) -> list[str]:
+    """Collect a list of values from a range argument.
+
+    Collect all individual values within a numeric range
+    represented by a string (e.g., `1:4`) and return them
+    as a list of strings.
+    Support both integer and floating-point ranges.
+
+    Args:
+        arg (str): The argument to collect.
+
+    Returns:
+        list[str]: A list of the collected values.
+
+    """
+    if ":" not in arg:
+        return [arg]
+
+    arg, exponent = split_suffix(arg)
+
+    if ":" not in arg:
+        return [f"{arg}{exponent}"]
+
+    rng = _get_range(arg)
+
+    if all(isinstance(x, int) for x in rng):
+        values = [str(x) for x in _arange(*rng)]
+    else:
+        n = max(*(count_decimal_places(x) for x in arg.split(":")))
+        values = [str(round(x, n)) for x in _arange(*rng)]
+
+    return [add_exponent(x, exponent) for x in values]
+
+
+def split(arg: str) -> list[str]:
+    r"""Split a string by top-level commas.
+
+    Splits a string by commas while respecting nested structures.
+    Commas inside brackets and quotes are ignored, only splitting
+    at the top-level commas.
+
+    Args:
+        arg (str): The string to split.
+
+    Returns:
+        list[str]: A list of split strings.
+
+    Examples:
+        >>> split("[a,1],[b,2]")
+        ['[a,1]', '[b,2]']
+        >>> split('"x,y",z')
+        ['"x,y"', 'z']
+        >>> split("'p,q',r")
+        ["'p,q'", 'r']
+
+    """
+    result = []
+    current = []
+    bracket_count = 0
+    in_single_quote = False
+    in_double_quote = False
+
+    for char in arg:
+        if char == "'" and not in_double_quote:
+            in_single_quote = not in_single_quote
+        elif char == '"' and not in_single_quote:
+            in_double_quote = not in_double_quote
+        elif char == "[" and not (in_single_quote or in_double_quote):
+            bracket_count += 1
+        elif char == "]" and not (in_single_quote or in_double_quote):
+            bracket_count -= 1
+        elif (
+            char == ","
+            and bracket_count == 0
+            and not in_single_quote
+            and not in_double_quote
+        ):
+            result.append("".join(current))
+            current = []
+            continue
+        current.append(char)
+
+    if current:
+        result.append("".join(current))
+
+    return result
+
+
+def expand_values(arg: str) -> list[str]:
+    """Expand a string argument into a list of values.
+
+    Take a string containing comma-separated values or ranges and return a list
+    of all individual values. Handle numeric ranges and special characters.
+
+    Args:
+        arg (str): The argument to expand.
+
+    Returns:
+        list[str]: A list of the expanded values.
+
+    """
+    return list(chain.from_iterable(collect_values(x) for x in split(arg)))
+
+
+def collect_arg(arg: str) -> str:
+    """Collect a string of expanded key-value pairs.
+
+    Take a key-value pair argument and concatenates all expanded values with commas,
+    returning a single string suitable for command-line usage.
+
+    Args:
+        arg (str): The argument to collect.
+
+    Returns:
+        str: A string of the collected key and values.
+
+    """
+    key, arg = arg.split("=")
+    arg = ",".join(expand_values(arg))
+    return f"{key}={arg}"
+
+
+def expand_arg(arg: str) -> Iterator[str]:
+    """Parse a string argument into a list of values.
+
+    Responsible for parsing a string that may contain multiple
+    arguments separated by pipes ("|") and returns a list of all
+    expanded arguments.
+
+    Args:
+        arg (str): The argument to parse.
+
+    Returns:
+       list[str]: A list of the parsed arguments.
+
+    """
+    if "|" not in arg:
+        key, value = arg.split("=")
+
+        for v in expand_values(value):
+            yield f"{key}={v}"
+
+        return
+
+    args = arg.split("|")
+    key = ""
+
+    for arg_ in args:
+        if "=" in arg_:
+            key, value = arg_.split("=")
+        elif key:
+            value = arg_
+        else:
+            msg = f"Invalid argument: {arg_}"
+            raise ValueError(msg)
+
+        value = ",".join(expand_values(value))
+        yield f"{key}={value}"
+
+
+def collect(args: str | list[str]) -> list[str]:
+    """Collect a list of arguments into a list of strings.
+
+    Args:
+        args (list[str]): The arguments to collect.
+
+    Returns:
+        list[str]: A list of the collected arguments.
+
+    """
+    if isinstance(args, str):
+        args = re.split(r"\s+", args.strip())
+
+    args = [arg for arg in args if "=" in arg]
+
+    return [collect_arg(arg) for arg in args]
+
+
+def expand(args: str | list[str]) -> list[list[str]]:
+    """Expand a list of arguments into a list of lists of strings.
+
+    Args:
+        args (list[str]): The arguments to expand.
+
+    Returns:
+        list[list[str]]: A list of the expanded arguments.
+
+    """
+    if isinstance(args, str):
+        args = re.split(r"\s+", args.strip())
+
+    args = [arg for arg in args if "=" in arg]
+
+    return [list(x) for x in product(*(expand_arg(arg) for arg in args))]

{hydraflow-0.7.5.dist-info → hydraflow-0.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hydraflow
-Version: 0.7.5
+Version: 0.9.0
 Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
 Project-URL: Documentation, https://daizutabi.github.io/hydraflow/
 Project-URL: Source, https://github.com/daizutabi/hydraflow
@@ -41,6 +41,7 @@ Requires-Dist: mlflow>=2.15
 Requires-Dist: omegaconf
 Requires-Dist: rich
 Requires-Dist: typer
+Requires-Dist: ulid
 Description-Content-Type: text/markdown
 
 # Hydraflow
@@ -93,31 +94,29 @@ pip install hydraflow
 Here is a simple example to get you started with Hydraflow:
 
 ```python
-import hydra
-import hydraflow
-import mlflow
+from __future__ import annotations
+
 from dataclasses import dataclass
-from hydra.core.config_store import ConfigStore
 from pathlib import Path
+from typing import TYPE_CHECKING
 
-@dataclass
-class MySQLConfig:
-    host: str = "localhost"
-    port: int = 3306
+import hydraflow
 
-cs = ConfigStore.instance()
-cs.store(name="config", node=MySQLConfig)
+if TYPE_CHECKING:
+    from mlflow.entities import Run
+
+
+@dataclass
+class Config:
+    count: int = 1
+    name: str = "a"
 
-@hydra.main(version_base=None, config_name="config")
-def my_app(cfg: MySQLConfig) -> None:
-    # Set experiment by Hydra job name.
-    hydraflow.set_experiment()
 
-    # Automatically log Hydra config as params.
-    with hydraflow.start_run(cfg):
-        # Your app code below.
+@hydraflow.main(Config)
+def app(run: Run, cfg: Config):
+    """Your app code here."""
 
 
 if __name__ == "__main__":
-    my_app()
+    app()
 ```

hydraflow-0.9.0.dist-info/RECORD

@@ -0,0 +1,24 @@
+hydraflow/__init__.py,sha256=WnReG-E2paQSMLJm42N-KaQyHYsfA4OX-WWcst610PI,738
+hydraflow/cli.py,sha256=gbDPj49azP8CCGxkxU0rksh1-gCyjP0VkVYH34ktcsA,1338
+hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+hydraflow/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+hydraflow/core/config.py,sha256=SJzjgsO_kzB78_whJ3lmy7GlZvTvwZONH1BJBn8zCuI,3817
+hydraflow/core/context.py,sha256=QPyPg1xrTlmhviKNn-0nDY9bXcVky1zInqRqPN-VNhc,4741
+hydraflow/core/io.py,sha256=T4ESiepEcqR-FZlo_m7VTBEFMwalrqPI8eFKPagvv3Q,4402
+hydraflow/core/main.py,sha256=gYb1OOVH0CL4385Dm-06Mqi1Mr9-24URwLUiW86pGNs,5018
+hydraflow/core/mlflow.py,sha256=M3MhiChnMzKnKRmjBl4h_SRGkAZKL7GAmFr3DdzwRuQ,5666
+hydraflow/core/param.py,sha256=LHU9j9_7oA99igasoOyKofKClVr9FmGA3UABJ-KmyS0,4538
+hydraflow/entities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+hydraflow/entities/run_collection.py,sha256=BHbMvYB4WyG5qSNYpIzwmHySOtmGNb-jg2OrCaTPr2I,19651
+hydraflow/entities/run_data.py,sha256=Y2_Lc-BdQ7nXhcEIjdHGHIkLrXsmAktOftESEwYOY8o,1602
+hydraflow/entities/run_info.py,sha256=FRC6ICOlzB2u_xi_33Qs-YZLt677UotuNbYqI7XSmHY,1017
+hydraflow/executor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+hydraflow/executor/conf.py,sha256=q_FrPXQJCVGKS1FYnGRGqTUgMQeMBkaVPW2mtQc8oxk,384
+hydraflow/executor/io.py,sha256=4nafwge6vHanYFuEHxd0LRv_3ZLgMpV50qSbssZNe3Q,696
+hydraflow/executor/job.py,sha256=Vp2IZOuIC25Gqo9A_5MkllWl1T1QBfUnI5ksMvKwakg,4479
+hydraflow/executor/parser.py,sha256=y4C9wVdUnazJDxdWrT5y3yWFIo0zAGzO-cS9x1MTK_8,9486
+hydraflow-0.9.0.dist-info/METADATA,sha256=p8-vmPIYV-uU9BJbSB1roqrhNO7nK2HRdrpj8Tip1AA,4559
+hydraflow-0.9.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+hydraflow-0.9.0.dist-info/entry_points.txt,sha256=XI0khPbpCIUo9UPqkNEpgh-kqK3Jy8T7L2VCWOdkbSM,48
+hydraflow-0.9.0.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
+hydraflow-0.9.0.dist-info/RECORD,,

hydraflow/main.py

@@ -1,54 +0,0 @@
-"""main decorator."""
-
-from __future__ import annotations
-
-from functools import wraps
-from typing import TYPE_CHECKING, Any
-
-import hydra
-from hydra.core.config_store import ConfigStore
-from mlflow.entities import RunStatus
-
-import hydraflow
-
-if TYPE_CHECKING:
-    from collections.abc import Callable
-
-    from mlflow.entities import Run
-
-FINISHED = RunStatus.to_string(RunStatus.FINISHED)
-
-
-def main(
-    node: Any,
-    config_name: str = "config",
-    *,
-    chdir: bool = False,
-    force_new_run: bool = False,
-    skip_finished: bool = True,
-):
-    """Main decorator."""
-
-    def decorator(app: Callable[[Run, Any], None]) -> Callable[[], None]:
-        ConfigStore.instance().store(name=config_name, node=node)
-
-        @wraps(app)
-        @hydra.main(version_base=None, config_name=config_name)
-        def inner_app(cfg: object) -> None:
-            hydraflow.set_experiment()
-
-            if force_new_run:
-                run = None
-            else:
-                rc = hydraflow.search_runs()
-                run = rc.try_get(cfg, override=True)
-
-                if skip_finished and run and run.info.status == FINISHED:
-                    return
-
-            with hydraflow.start_run(cfg, run=run, chdir=chdir) as run:
-                app(run, cfg)
-
-        return inner_app
-
-    return decorator