hydraflow 0.2.8__py3-none-any.whl → 0.2.9__py3-none-any.whl

hydraflow/__init__.py

@@ -5,6 +5,7 @@ from .mlflow import (
     search_runs,
     set_experiment,
 )
+from .progress import multi_tasks_progress, parallel_progress
 from .run_collection import RunCollection
 
 __all__ = [
@@ -15,6 +16,8 @@ __all__ = [
     "list_runs",
     "load_config",
     "log_run",
+    "multi_tasks_progress",
+    "parallel_progress",
     "search_runs",
     "set_experiment",
     "start_run",

hydraflow/asyncio.py

@@ -41,7 +41,9 @@ async def execute_command(
         int: The return code of the process.
     """
     try:
-        process = await asyncio.create_subprocess_exec(program, *args, stdout=PIPE, stderr=PIPE)
+        process = await asyncio.create_subprocess_exec(
+            program, *args, stdout=PIPE, stderr=PIPE
+        )
         await asyncio.gather(
             process_stream(process.stdout, stdout),
             process_stream(process.stderr, stderr),
@@ -100,7 +102,9 @@ async def monitor_file_changes(
     """
     str_paths = [str(path) for path in paths]
     try:
-        async for changes in watchfiles.awatch(*str_paths, stop_event=stop_event, **awatch_kwargs):
+        async for changes in watchfiles.awatch(
+            *str_paths, stop_event=stop_event, **awatch_kwargs
+        ):
             callback(changes)
     except Exception as e:
         logger.error(f"Error watching files: {e}")
@@ -129,7 +133,9 @@ async def run_and_monitor(
     """
     stop_event = asyncio.Event()
     run_task = asyncio.create_task(
-        execute_command(program, *args, stop_event=stop_event, stdout=stdout, stderr=stderr)
+        execute_command(
+            program, *args, stop_event=stop_event, stdout=stdout, stderr=stderr
+        )
     )
     if watch and paths:
         monitor_task = asyncio.create_task(

hydraflow/progress.py

@@ -1,17 +1,129 @@
+"""
+Module for managing progress tracking in parallel processing using Joblib
+and Rich's Progress bar.
+
+Provide context managers and functions to facilitate the execution
+of tasks in parallel while displaying progress updates.
+
+The following key components are provided:
+
+- JoblibProgress: A context manager for tracking progress with Rich's Progress
+    bar.
+- parallel_progress: A function to execute a given function in parallel over
+    an iterable with progress tracking.
+- multi_tasks_progress: A function to render auto-updating progress bars for
+    multiple tasks concurrently.
+
+Usage:
+    Import the necessary functions and use them to manage progress in your
+    parallel processing tasks.
+"""
+
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from contextlib import contextmanager
+from typing import TYPE_CHECKING, TypeVar
 
 import joblib
 from rich.progress import Progress
 
 if TYPE_CHECKING:
-    from collections.abc import Iterable
+    from collections.abc import Callable, Iterable, Iterator
 
     from rich.progress import ProgressColumn
 
 
-def multi_task_progress(
+# https://github.com/jonghwanhyeon/joblib-progress/blob/main/joblib_progress/__init__.py
+@contextmanager
+def JoblibProgress(
+    *columns: ProgressColumn | str,
+    description: str | None = None,
+    total: int | None = None,
+    **kwargs,
+) -> Iterator[Progress]:
+    """
+    Context manager for tracking progress using Joblib with Rich's Progress bar.
+
+    Args:
+        *columns (ProgressColumn | str): Columns to display in the progress bar.
+        description (str | None, optional): A description for the progress task.
+            Defaults to None.
+        total (int | None, optional): The total number of tasks. If None, it will
+            be determined automatically.
+        **kwargs: Additional keyword arguments passed to the Progress instance.
+
+    Yields:
+        Progress: A Progress instance for managing the progress bar.
+
+    Example:
+        with JoblibProgress("task", total=100) as progress:
+            # Your parallel processing code here
+    """
+    if not columns:
+        columns = Progress.get_default_columns()
+
+    progress = Progress(*columns, **kwargs)
+
+    if description is None:
+        description = "Processing..."
+
+    task_id = progress.add_task(description, total=total)
+    print_progress = joblib.parallel.Parallel.print_progress
+
+    def update_progress(self: joblib.parallel.Parallel):
+        progress.update(task_id, completed=self.n_completed_tasks, refresh=True)
+        return print_progress(self)
+
+    try:
+        joblib.parallel.Parallel.print_progress = update_progress
+        progress.start()
+        yield progress
+
+    finally:
+        progress.stop()
+        joblib.parallel.Parallel.print_progress = print_progress
+
+
+T = TypeVar("T")
+U = TypeVar("U")
+
+
+def parallel_progress(
+    func: Callable[[T], U],
+    iterable: Iterable[T],
+    *columns: ProgressColumn | str,
+    n_jobs: int = -1,
+    description: str | None = None,
+    **kwargs,
+) -> list[U]:
+    """
+    Execute a function in parallel over an iterable with progress tracking.
+
+    Args:
+        func (Callable[[T], U]): The function to execute on each item in the
+            iterable.
+        iterable (Iterable[T]): An iterable of items to process.
+        *columns (ProgressColumn | str): Additional columns to display in the
+            progress bar.
+        n_jobs (int, optional): The number of jobs to run in parallel.
+            Defaults to -1 (all processors).
+        description (str | None, optional): A description for the progress bar.
+            Defaults to None.
+        **kwargs: Additional keyword arguments passed to the Progress instance.
+
+    Returns:
+        list[U]: A list of results from applying the function to each item in
+        the iterable.
+    """
+    iterable = list(iterable)
+    total = len(iterable)
+
+    with JoblibProgress(*columns, description=description, total=total, **kwargs):
+        it = (joblib.delayed(func)(x) for x in iterable)
+        return joblib.Parallel(n_jobs=n_jobs)(it)  # type: ignore
+
+
+def multi_tasks_progress(
     iterables: Iterable[Iterable[int | tuple[int, int]]],
     *columns: ProgressColumn | str,
     n_jobs: int = -1,
@@ -52,7 +164,8 @@ def multi_task_progress(
 
         task_main = progress.add_task(main_description, total=None) if n > 1 else None
         tasks = [
-            progress.add_task(description.format(i), start=False, total=None) for i in range(n)
+            progress.add_task(description.format(i), start=False, total=None)
+            for i in range(n)
         ]
 
         total = {}
@@ -87,45 +200,3 @@ def multi_task_progress(
 
         else:
             func(0)
-
-
-if __name__ == "__main__":
-    import random
-    import time
-
-    from rich.progress import MofNCompleteColumn, Progress, SpinnerColumn, TimeElapsedColumn
-
-    from hydraflow.progress import multi_task_progress
-
-    def task(total):
-        for i in range(total or 90):
-            if total is None:
-                yield i
-            else:
-                yield i, total
-            time.sleep(random.random() / 30)
-
-    def multi_task_progress_test(unknown_total: bool):
-        tasks = [task(random.randint(80, 100)) for _ in range(4)]
-        if unknown_total:
-            tasks = [task(None), *tasks, task(None)]
-
-        columns = [
-            SpinnerColumn(),
-            *Progress.get_default_columns(),
-            MofNCompleteColumn(),
-            TimeElapsedColumn(),
-        ]
-
-        kwargs = {}
-        if unknown_total:
-            kwargs["main_description"] = "unknown"
-
-        multi_task_progress(tasks, *columns, n_jobs=4, **kwargs)
-
-    multi_task_progress_test(False)
-    multi_task_progress_test(True)
-    multi_task_progress([task(100)])
-    multi_task_progress([task(None)], description="unknown")
-    multi_task_progress([task(100), task(None)], main_description="transient", transient=True)
-    multi_task_progress([task(100)], description="transient", transient=True)

hydraflow/run_collection.py

@@ -468,7 +468,9 @@ class RunCollection:
         """
         return (func(dir, *args, **kwargs) for dir in self.info.artifact_dir)
 
-    def group_by(self, *names: str | list[str]) -> dict[tuple[str | None, ...], RunCollection]:
+    def group_by(
+        self, *names: str | list[str]
+    ) -> dict[tuple[str | None, ...], RunCollection]:
         """
         Group runs by specified parameter names.
 
@@ -493,25 +495,6 @@ class RunCollection:
 
         return {key: RunCollection(runs) for key, runs in grouped_runs.items()}
 
-    def group_by_values(self, *names: str | list[str]) -> list[RunCollection]:
-        """
-        Group runs by specified parameter names.
-
-        This method groups the runs in the collection based on the values of the
-        specified parameters. Each unique combination of parameter values will
-        form a separate RunCollection in the returned list.
-
-        Args:
-            *names (str | list[str]): The names of the parameters to group by.
-                This can be a single parameter name or multiple names provided
-                as separate arguments or as a list.
-
-        Returns:
-            list[RunCollection]: A list of RunCollection objects, where each
-            object contains runs that match the specified parameter values.
-        """
-        return list(self.group_by(*names).values())
-
 
 def _param_matches(run: Run, key: str, value: Any) -> bool:
     """
@@ -671,7 +654,9 @@ def find_last_run(runs: list[Run], config: object | None = None, **kwargs) -> Ru
     return filtered_runs[-1]
 
 
-def try_find_last_run(runs: list[Run], config: object | None = None, **kwargs) -> Run | None:
+def try_find_last_run(
+    runs: list[Run], config: object | None = None, **kwargs
+) -> Run | None:
     """
     Find the last run based on the provided configuration.
 

{hydraflow-0.2.8.dist-info → hydraflow-0.2.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: hydraflow
-Version: 0.2.8
+Version: 0.2.9
 Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
 Project-URL: Documentation, https://github.com/daizutabi/hydraflow
 Project-URL: Source, https://github.com/daizutabi/hydraflow

hydraflow-0.2.9.dist-info/RECORD

@@ -0,0 +1,12 @@
+hydraflow/__init__.py,sha256=B7rWSiGP5WwWjijcb41Bv9uuo5MQ6gbBbVWGAWYtK-k,598
+hydraflow/asyncio.py,sha256=jdXuEFC6f7L_Dq6beASFZPQSvCnGimVxU-PRFsNc5U0,6241
+hydraflow/config.py,sha256=6TCKNQZ3sSrIEvl245T2udwFuknejyN1dMcIVmOHdrQ,2102
+hydraflow/context.py,sha256=G7JMrG70sgBH2qILXl5nkGWNUoRggj518JWUq0ZiJ9E,7776
+hydraflow/info.py,sha256=Vj2sT66Ric63mmaq7Yu8nDFhsGQYO3MCHrxFpapDufc,3458
+hydraflow/mlflow.py,sha256=Q8RGijSURTjRkEDxzi_2Tk9KOx3QK__al5aArGQriHA,7249
+hydraflow/progress.py,sha256=UIIKlweji3L0uRi4hZ_DrtRcnayHPlsMoug7hVEKq8k,6753
+hydraflow/run_collection.py,sha256=V5lGdGHYgsSpBOYGaVEL1mpKJvdiEshBL0KmmZ8qeZo,29161
+hydraflow-0.2.9.dist-info/METADATA,sha256=ZjJQz_4MogGkcs16dOwnsp_J0icg9ypgQdXOYxVdxJg,4181
+hydraflow-0.2.9.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+hydraflow-0.2.9.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
+hydraflow-0.2.9.dist-info/RECORD,,

hydraflow-0.2.8.dist-info/RECORD

@@ -1,12 +0,0 @@
-hydraflow/__init__.py,sha256=K2xXk5Za_9OkiRmbsgkuWn7EMaTcQOVCPFs5oTP_QFw,483
-hydraflow/asyncio.py,sha256=yh851L315QHzRBwq6r-uwO2oZKgz1JawHp-fswfxT1E,6175
-hydraflow/config.py,sha256=6TCKNQZ3sSrIEvl245T2udwFuknejyN1dMcIVmOHdrQ,2102
-hydraflow/context.py,sha256=G7JMrG70sgBH2qILXl5nkGWNUoRggj518JWUq0ZiJ9E,7776
-hydraflow/info.py,sha256=Vj2sT66Ric63mmaq7Yu8nDFhsGQYO3MCHrxFpapDufc,3458
-hydraflow/mlflow.py,sha256=Q8RGijSURTjRkEDxzi_2Tk9KOx3QK__al5aArGQriHA,7249
-hydraflow/progress.py,sha256=0GJfKnnY_SAHVWpGvLdgOBsogGs8vVofjLuphuUEy2g,4296
-hydraflow/run_collection.py,sha256=Ge-PAsoQBbn7cuow0DYMf5SoBmIXUfZ9ftufN_75Pw8,29963
-hydraflow-0.2.8.dist-info/METADATA,sha256=9CF5S8LdmDUx4sihDqVRvwLLk34FNBmy_Vv6BVoahoc,4181
-hydraflow-0.2.8.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-hydraflow-0.2.8.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
-hydraflow-0.2.8.dist-info/RECORD,,