hydraflow 0.4.5__py3-none-any.whl → 0.5.0__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
hydraflow/__init__.py CHANGED
@@ -1,9 +1,8 @@
1
1
  """Integrate Hydra and MLflow to manage and track machine learning experiments."""
2
2
 
3
3
  from .config import select_config, select_overrides
4
- from .context import chdir_artifact, chdir_hydra_output, log_run, start_run, watch
4
+ from .context import chdir_artifact, chdir_hydra_output, log_run, start_run
5
5
  from .mlflow import list_runs, search_runs, set_experiment
6
- from .progress import multi_tasks_progress, parallel_progress
7
6
  from .run_collection import RunCollection
8
7
  from .utils import (
9
8
  get_artifact_dir,
@@ -11,6 +10,7 @@ from .utils import (
11
10
  get_overrides,
12
11
  load_config,
13
12
  load_overrides,
13
+ remove_run,
14
14
  )
15
15
 
16
16
  __all__ = [
@@ -24,12 +24,10 @@ __all__ = [
24
24
  "load_config",
25
25
  "load_overrides",
26
26
  "log_run",
27
- "multi_tasks_progress",
28
- "parallel_progress",
27
+ "remove_run",
29
28
  "search_runs",
30
29
  "select_config",
31
30
  "select_overrides",
32
31
  "set_experiment",
33
32
  "start_run",
34
- "watch",
35
33
  ]
hydraflow/context.py CHANGED
@@ -4,7 +4,6 @@ from __future__ import annotations
4
4
 
5
5
  import logging
6
6
  import os
7
- import time
8
7
  from contextlib import contextmanager
9
8
  from pathlib import Path
10
9
  from typing import TYPE_CHECKING
@@ -12,14 +11,11 @@ from typing import TYPE_CHECKING
12
11
  import mlflow
13
12
  import mlflow.artifacts
14
13
  from hydra.core.hydra_config import HydraConfig
15
- from watchdog.events import FileModifiedEvent, PatternMatchingEventHandler
16
- from watchdog.observers import Observer
17
14
 
18
15
  from hydraflow.mlflow import log_params
19
- from hydraflow.run_info import get_artifact_dir
20
16
 
21
17
  if TYPE_CHECKING:
22
- from collections.abc import Callable, Iterator
18
+ from collections.abc import Iterator
23
19
 
24
20
  from mlflow.entities.run import Run
25
21
 
@@ -64,14 +60,8 @@ def log_run(
64
60
  output_subdir = output_dir / (hc.output_subdir or "")
65
61
  mlflow.log_artifacts(output_subdir.as_posix(), hc.output_subdir)
66
62
 
67
- def log_artifact(path: Path) -> None:
68
- local_path = (output_dir / path).as_posix()
69
- mlflow.log_artifact(local_path)
70
-
71
63
  try:
72
64
  yield
73
- # with watch(log_artifact, output_dir, ignore_log=False):
74
- # yield
75
65
 
76
66
  except Exception as e:
77
67
  msg = f"Error during log_run: {e}"
@@ -146,101 +136,6 @@ def start_run( # noqa: PLR0913
146
136
  yield run
147
137
 
148
138
 
149
- @contextmanager
150
- def watch(
151
- callback: Callable[[Path], None],
152
- dir: Path | str = "", # noqa: A002
153
- *,
154
- timeout: int = 60,
155
- ignore_patterns: list[str] | None = None,
156
- ignore_log: bool = True,
157
- ) -> Iterator[None]:
158
- """Watch the given directory for changes.
159
-
160
- This context manager sets up a file system watcher on the specified directory.
161
- When a file modification is detected, the provided function is called with
162
- the path of the modified file. The watcher runs for the specified timeout
163
- period or until the context is exited.
164
-
165
- Args:
166
- callback (Callable[[Path], None]): The function to call when a change is
167
- detected. It should accept a single argument of type `Path`,
168
- which is the path of the modified file.
169
- dir (Path | str): The directory to watch. If not specified,
170
- the current MLflow artifact URI is used. Defaults to "".
171
- timeout (int): The timeout period in seconds for the watcher
172
- to run after the context is exited. Defaults to 60.
173
- ignore_patterns (list[str] | None): A list of glob patterns to ignore.
174
- Defaults to None.
175
- ignore_log (bool): Whether to ignore log files. Defaults to True.
176
-
177
- Yields:
178
- None
179
-
180
- Example:
181
- ```python
182
- with watch(log_artifact, "/path/to/dir"):
183
- # Perform operations while watching the directory for changes
184
- pass
185
- ```
186
-
187
- """
188
- dir = dir or get_artifact_dir() # noqa: A001
189
- if isinstance(dir, Path):
190
- dir = dir.as_posix() # noqa: A001
191
-
192
- handler = Handler(callback, ignore_patterns=ignore_patterns, ignore_log=ignore_log)
193
- observer = Observer()
194
- observer.schedule(handler, dir, recursive=True)
195
- observer.start()
196
-
197
- try:
198
- yield
199
-
200
- except Exception as e:
201
- msg = f"Error during watch: {e}"
202
- log.exception(msg)
203
- raise
204
-
205
- finally:
206
- elapsed = 0
207
- while not observer.event_queue.empty():
208
- time.sleep(0.2)
209
- elapsed += 0.2
210
- if elapsed > timeout:
211
- break
212
-
213
- observer.stop()
214
- observer.join()
215
-
216
-
217
- class Handler(PatternMatchingEventHandler):
218
- """Monitor file changes and call the given function when a change is detected."""
219
-
220
- def __init__(
221
- self,
222
- func: Callable[[Path], None],
223
- *,
224
- ignore_patterns: list[str] | None = None,
225
- ignore_log: bool = True,
226
- ) -> None:
227
- self.func = func
228
-
229
- if ignore_log:
230
- if ignore_patterns:
231
- ignore_patterns.append("*.log")
232
- else:
233
- ignore_patterns = ["*.log"]
234
-
235
- super().__init__(ignore_patterns=ignore_patterns)
236
-
237
- def on_modified(self, event: FileModifiedEvent) -> None:
238
- """Modify when a file is modified."""
239
- file = Path(str(event.src_path))
240
- if file.is_file():
241
- self.func(file)
242
-
243
-
244
139
  @contextmanager
245
140
  def chdir_hydra_output() -> Iterator[Path]:
246
141
  """Change the current working directory to the hydra output directory.
@@ -575,7 +575,7 @@ class RunCollection:
575
575
  """
576
576
  return (func(dir, *args, **kwargs) for dir in self.info.artifact_dir) # noqa: A001
577
577
 
578
- def group_by(
578
+ def groupby(
579
579
  self,
580
580
  names: str | list[str],
581
581
  ) -> dict[str | None | tuple[str | None, ...], RunCollection]:
hydraflow/run_data.py CHANGED
@@ -4,7 +4,7 @@ from __future__ import annotations
4
4
 
5
5
  from typing import TYPE_CHECKING
6
6
 
7
- from polars.dataframe import DataFrame
7
+ from pandas import DataFrame
8
8
 
9
9
  from hydraflow.config import collect_params
10
10
 
@@ -33,10 +33,10 @@ class RunCollectionData:
33
33
 
34
34
  @property
35
35
  def config(self) -> DataFrame:
36
- """Get the runs' configurations as a polars DataFrame.
36
+ """Get the runs' configurations as a DataFrame.
37
37
 
38
38
  Returns:
39
- A polars DataFrame containing the runs' configurations.
39
+ A DataFrame containing the runs' configurations.
40
40
 
41
41
  """
42
42
  return DataFrame(self._runs.map_config(collect_params))
hydraflow/utils.py CHANGED
@@ -2,6 +2,7 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
+ import shutil
5
6
  from pathlib import Path
6
7
  from typing import TYPE_CHECKING
7
8
 
@@ -9,11 +10,10 @@ import mlflow
9
10
  import mlflow.artifacts
10
11
  from hydra.core.hydra_config import HydraConfig
11
12
  from mlflow.entities import Run
12
- from mlflow.tracking import artifact_utils
13
13
  from omegaconf import DictConfig, OmegaConf
14
14
 
15
15
  if TYPE_CHECKING:
16
- from mlflow.entities import Run
16
+ from collections.abc import Iterable
17
17
 
18
18
 
19
19
  def get_artifact_dir(run: Run | None = None) -> Path:
@@ -28,10 +28,10 @@ def get_artifact_dir(run: Run | None = None) -> Path:
28
28
  The local path to the directory where the artifacts are downloaded.
29
29
 
30
30
  """
31
- if run is None:
32
- uri = mlflow.get_artifact_uri()
33
- else:
34
- uri = artifact_utils.get_artifact_uri(run.info.run_id)
31
+ uri = mlflow.get_artifact_uri() if run is None else run.info.artifact_uri
32
+
33
+ if not (isinstance(uri, str) and uri.startswith("file://")):
34
+ raise NotImplementedError
35
35
 
36
36
  return Path(mlflow.artifacts.download_artifacts(uri))
37
37
 
@@ -112,3 +112,13 @@ def load_overrides(run: Run) -> list[str]:
112
112
  """
113
113
  path = get_artifact_dir(run) / ".hydra/overrides.yaml"
114
114
  return [str(x) for x in OmegaConf.load(path)]
115
+
116
+
117
+ def remove_run(run: Run | Iterable[Run]) -> None:
118
+ """Remove the given run from the MLflow tracking server."""
119
+ if not isinstance(run, Run):
120
+ for r in run:
121
+ remove_run(r)
122
+ return
123
+
124
+ shutil.rmtree(get_artifact_dir(run).parent)
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.3
1
+ Metadata-Version: 2.4
2
2
  Name: hydraflow
3
- Version: 0.4.5
3
+ Version: 0.5.0
4
4
  Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
5
5
  Project-URL: Documentation, https://github.com/daizutabi/hydraflow
6
6
  Project-URL: Source, https://github.com/daizutabi/hydraflow
@@ -27,6 +27,7 @@ License: MIT License
27
27
  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
28
28
  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
29
29
  SOFTWARE.
30
+ License-File: LICENSE
30
31
  Classifier: Development Status :: 4 - Beta
31
32
  Classifier: License :: OSI Approved :: MIT License
32
33
  Classifier: Programming Language :: Python
@@ -36,12 +37,7 @@ Classifier: Programming Language :: Python :: 3.12
36
37
  Classifier: Programming Language :: Python :: 3.13
37
38
  Requires-Python: >=3.10
38
39
  Requires-Dist: hydra-core>=1.3
39
- Requires-Dist: joblib
40
40
  Requires-Dist: mlflow>=2.15
41
- Requires-Dist: polars
42
- Requires-Dist: rich
43
- Requires-Dist: watchdog
44
- Requires-Dist: watchfiles
45
41
  Description-Content-Type: text/markdown
46
42
 
47
43
  # Hydraflow
@@ -118,16 +114,6 @@ def my_app(cfg: MySQLConfig) -> None:
118
114
  with hydraflow.start_run():
119
115
  # Your app code below.
120
116
 
121
- with hydraflow.watch(callback):
122
- # Watch files in the MLflow artifact directory.
123
- # You can update metrics or log other artifacts
124
- # according to the watched files in your callback
125
- # function.
126
- pass
127
-
128
- # Your callback function here.
129
- def callback(file: Path) -> None:
130
- pass
131
117
 
132
118
  if __name__ == "__main__":
133
119
  my_app()
@@ -0,0 +1,14 @@
1
+ hydraflow/__init__.py,sha256=DKtFjTXHTgceX7rpWHiKqhcpG5xtGIseFvN28f7iwYo,807
2
+ hydraflow/config.py,sha256=MNX9da5bPVDcjnpji7Cm9ndK6ura92pt361m4PRh6_E,4326
3
+ hydraflow/context.py,sha256=3g7OQXWcFvK6PVVbXpQg7Hr8nsJkF9pLFrXNi_3aV5A,5524
4
+ hydraflow/mlflow.py,sha256=kWVK_Xw2hkRnTg33jSP3VW13UZF6_hBGhN52mPmLgvk,8753
5
+ hydraflow/param.py,sha256=c5sc6NwD6DKwZzVwprXzZD5FSi6qRgSHkc6TXBKQEdg,4502
6
+ hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
7
+ hydraflow/run_collection.py,sha256=RQRN50k5m_bDWn4vzOProbnjzVejI5hsr5WSd4owbhQ,26773
8
+ hydraflow/run_data.py,sha256=dpyyfnuH9mCtIZeigMo1iFQo9bafMdEL4i4uI2l0UqY,1525
9
+ hydraflow/run_info.py,sha256=sMXOo20ClaRIommMEzuAbO_OrcXx7M1Yt4FMV7spxz0,998
10
+ hydraflow/utils.py,sha256=jbNrbtIfMqxE4LrdTNd1g7sF68XgAvydGqW5iAZ6n-c,3834
11
+ hydraflow-0.5.0.dist-info/METADATA,sha256=pv7eNdU5mT05N936AWyuLzZfzYSgVrsGZ5d354bD_-8,4697
12
+ hydraflow-0.5.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
13
+ hydraflow-0.5.0.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
14
+ hydraflow-0.5.0.dist-info/RECORD,,
@@ -1,4 +1,4 @@
1
1
  Wheel-Version: 1.0
2
- Generator: hatchling 1.26.3
2
+ Generator: hatchling 1.27.0
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
hydraflow/asyncio.py DELETED
@@ -1,227 +0,0 @@
1
- """Provide functionality for running commands and monitoring file changes."""
2
-
3
- from __future__ import annotations
4
-
5
- import asyncio
6
- import logging
7
- from asyncio.subprocess import PIPE
8
- from pathlib import Path
9
- from typing import TYPE_CHECKING
10
-
11
- import watchfiles
12
-
13
- if TYPE_CHECKING:
14
- from asyncio.streams import StreamReader
15
- from collections.abc import Callable
16
-
17
- from watchfiles import Change
18
-
19
-
20
- # Set up logging
21
- logging.basicConfig(level=logging.INFO)
22
- logger = logging.getLogger(__name__)
23
-
24
-
25
- async def execute_command(
26
- program: str,
27
- *args: str,
28
- stdout: Callable[[str], None] | None = None,
29
- stderr: Callable[[str], None] | None = None,
30
- stop_event: asyncio.Event,
31
- ) -> int:
32
- """Run a command asynchronously and pass the output to callback functions.
33
-
34
- Args:
35
- program (str): The program to run.
36
- *args (str): Arguments for the program.
37
- stdout (Callable[[str], None] | None): Callback for standard output.
38
- stderr (Callable[[str], None] | None): Callback for standard error.
39
- stop_event (asyncio.Event): Event to signal when the process is done.
40
-
41
- Returns:
42
- int: The return code of the process.
43
-
44
- """
45
- try:
46
- process = await asyncio.create_subprocess_exec(
47
- program,
48
- *args,
49
- stdout=PIPE,
50
- stderr=PIPE,
51
- )
52
- await asyncio.gather(
53
- process_stream(process.stdout, stdout),
54
- process_stream(process.stderr, stderr),
55
- )
56
- returncode = await process.wait()
57
-
58
- except Exception as e:
59
- msg = f"Error running command: {e}"
60
- logger.exception(msg)
61
- returncode = 1
62
-
63
- finally:
64
- stop_event.set()
65
-
66
- return returncode
67
-
68
-
69
- async def process_stream(
70
- stream: StreamReader | None,
71
- callback: Callable[[str], None] | None,
72
- ) -> None:
73
- """Read a stream asynchronously and pass each line to a callback function.
74
-
75
- Args:
76
- stream (StreamReader | None): The stream to read from.
77
- callback (Callable[[str], None] | None): The callback function to handle
78
- each line.
79
-
80
- """
81
- if stream is None or callback is None:
82
- return
83
-
84
- while True:
85
- line = await stream.readline()
86
- if line:
87
- callback(line.decode().strip())
88
- else:
89
- break
90
-
91
-
92
- async def monitor_file_changes(
93
- paths: list[str | Path],
94
- callback: Callable[[set[tuple[Change, str]]], None],
95
- stop_event: asyncio.Event,
96
- **awatch_kwargs,
97
- ) -> None:
98
- """Watch file changes in specified paths and pass the changes to a callback.
99
-
100
- Args:
101
- paths (list[str | Path]): List of paths to monitor for changes.
102
- callback (Callable[[set[tuple[Change, str]]], None]): The callback
103
- function to handle file changes.
104
- stop_event (asyncio.Event): Event to signal when to stop watching.
105
- **awatch_kwargs: Additional keyword arguments to pass to watchfiles.awatch.
106
-
107
- """
108
- str_paths = [str(path) for path in paths]
109
- try:
110
- async for changes in watchfiles.awatch(
111
- *str_paths,
112
- stop_event=stop_event,
113
- **awatch_kwargs,
114
- ):
115
- callback(changes)
116
- except Exception as e:
117
- msg = f"Error watching files: {e}"
118
- logger.exception(msg)
119
- raise
120
-
121
-
122
- async def run_and_monitor(
123
- program: str,
124
- *args: str,
125
- stdout: Callable[[str], None] | None = None,
126
- stderr: Callable[[str], None] | None = None,
127
- watch: Callable[[set[tuple[Change, str]]], None] | None = None,
128
- paths: list[str | Path] | None = None,
129
- **awatch_kwargs,
130
- ) -> int:
131
- """Run a command and optionally watch for file changes concurrently.
132
-
133
- Args:
134
- program (str): The program to run.
135
- *args (str): Arguments for the program.
136
- stdout (Callable[[str], None] | None): Callback for standard output.
137
- stderr (Callable[[str], None] | None): Callback for standard error.
138
- watch (Callable[[set[tuple[Change, str]]], None] | None): Callback for
139
- file changes.
140
- paths (list[str | Path] | None): List of paths to monitor for changes.
141
- **awatch_kwargs: Additional keyword arguments to pass to `watchfiles.awatch`.
142
-
143
- """
144
- stop_event = asyncio.Event()
145
- run_task = asyncio.create_task(
146
- execute_command(
147
- program,
148
- *args,
149
- stop_event=stop_event,
150
- stdout=stdout,
151
- stderr=stderr,
152
- ),
153
- )
154
- if watch and paths:
155
- coro = monitor_file_changes(paths, watch, stop_event, **awatch_kwargs)
156
- monitor_task = asyncio.create_task(coro)
157
- else:
158
- monitor_task = None
159
-
160
- try:
161
- if monitor_task:
162
- await asyncio.gather(run_task, monitor_task)
163
- else:
164
- await run_task
165
-
166
- except Exception as e:
167
- msg = f"Error in run_and_monitor: {e}"
168
- logger.exception(msg)
169
- raise
170
-
171
- finally:
172
- stop_event.set()
173
- await run_task
174
- if monitor_task:
175
- await monitor_task
176
-
177
- return run_task.result()
178
-
179
-
180
- def run(
181
- program: str,
182
- *args: str,
183
- stdout: Callable[[str], None] | None = None,
184
- stderr: Callable[[str], None] | None = None,
185
- watch: Callable[[set[tuple[Change, str]]], None] | None = None,
186
- paths: list[str | Path] | None = None,
187
- **awatch_kwargs,
188
- ) -> int:
189
- """Run a command synchronously and optionally watch for file changes.
190
-
191
- This function is a synchronous wrapper around the asynchronous
192
- `run_and_monitor` function. It runs a specified command and optionally
193
- monitors specified paths for file changes, invoking the provided callbacks for
194
- standard output, standard error, and file changes.
195
-
196
- Args:
197
- program (str): The program to run.
198
- *args (str): Arguments for the program.
199
- stdout (Callable[[str], None] | None): Callback for handling standard
200
- output lines.
201
- stderr (Callable[[str], None] | None): Callback for handling standard
202
- error lines.
203
- watch (Callable[[set[tuple[Change, str]]], None] | None): Callback for
204
- handling file changes.
205
- paths (list[str | Path] | None): List of paths to monitor for file
206
- changes.
207
- **awatch_kwargs: Additional keyword arguments to pass to
208
- `watchfiles.awatch`.
209
-
210
- Returns:
211
- int: The return code of the process.
212
-
213
- """
214
- if watch and not paths:
215
- paths = [Path.cwd()]
216
-
217
- return asyncio.run(
218
- run_and_monitor(
219
- program,
220
- *args,
221
- stdout=stdout,
222
- stderr=stderr,
223
- watch=watch,
224
- paths=paths,
225
- **awatch_kwargs,
226
- ),
227
- )
hydraflow/progress.py DELETED
@@ -1,184 +0,0 @@
1
- """Context managers and functions for parallel task execution with progress.
2
-
3
- Provide context managers and functions to facilitate the execution
4
- of tasks in parallel while displaying progress updates.
5
- """
6
-
7
- from __future__ import annotations
8
-
9
- from contextlib import contextmanager
10
- from typing import TYPE_CHECKING, TypeVar
11
-
12
- import joblib
13
- from rich.progress import Progress
14
-
15
- if TYPE_CHECKING:
16
- from collections.abc import Callable, Iterable, Iterator
17
-
18
- from rich.progress import ProgressColumn
19
-
20
-
21
- # https://github.com/jonghwanhyeon/joblib-progress/blob/main/joblib_progress/__init__.py
22
- @contextmanager
23
- def JoblibProgress( # noqa: N802
24
- *columns: ProgressColumn | str,
25
- description: str | None = None,
26
- total: int | None = None,
27
- **kwargs,
28
- ) -> Iterator[Progress]:
29
- """Context manager for tracking progress using Joblib with Rich's Progress bar.
30
-
31
- Args:
32
- *columns (ProgressColumn | str): Columns to display in the progress bar.
33
- description (str | None, optional): A description for the progress task.
34
- Defaults to None.
35
- total (int | None, optional): The total number of tasks. If None, it will
36
- be determined automatically.
37
- **kwargs: Additional keyword arguments passed to the Progress instance.
38
-
39
- Yields:
40
- Progress: A Progress instance for managing the progress bar.
41
-
42
- Example:
43
- ```python
44
- with JoblibProgress("task", total=100) as progress:
45
- # Your parallel processing code here
46
- ```
47
-
48
- """
49
- if not columns:
50
- columns = Progress.get_default_columns()
51
-
52
- progress = Progress(*columns, **kwargs)
53
-
54
- if description is None:
55
- description = "Processing..."
56
-
57
- task_id = progress.add_task(description, total=total)
58
- print_progress = joblib.parallel.Parallel.print_progress
59
-
60
- def update_progress(self: joblib.parallel.Parallel) -> None:
61
- progress.update(task_id, completed=self.n_completed_tasks, refresh=True)
62
- return print_progress(self)
63
-
64
- try:
65
- joblib.parallel.Parallel.print_progress = update_progress
66
- progress.start()
67
- yield progress
68
-
69
- finally:
70
- progress.stop()
71
- joblib.parallel.Parallel.print_progress = print_progress
72
-
73
-
74
- T = TypeVar("T")
75
- U = TypeVar("U")
76
-
77
-
78
- def parallel_progress(
79
- func: Callable[[T], U],
80
- iterable: Iterable[T],
81
- *columns: ProgressColumn | str,
82
- n_jobs: int = -1,
83
- description: str | None = None,
84
- **kwargs,
85
- ) -> list[U]:
86
- """Execute a function in parallel over an iterable with progress tracking.
87
-
88
- Args:
89
- func (Callable[[T], U]): The function to execute on each item in the
90
- iterable.
91
- iterable (Iterable[T]): An iterable of items to process.
92
- *columns (ProgressColumn | str): Additional columns to display in the
93
- progress bar.
94
- n_jobs (int, optional): The number of jobs to run in parallel.
95
- Defaults to -1 (all processors).
96
- description (str | None, optional): A description for the progress bar.
97
- Defaults to None.
98
- **kwargs: Additional keyword arguments passed to the Progress instance.
99
-
100
- Returns:
101
- list[U]: A list of results from applying the function to each item in
102
- the iterable.
103
-
104
- """
105
- iterable = list(iterable)
106
- total = len(iterable)
107
-
108
- with JoblibProgress(*columns, description=description, total=total, **kwargs):
109
- it = (joblib.delayed(func)(x) for x in iterable)
110
- return joblib.Parallel(n_jobs=n_jobs)(it) # type: ignore
111
-
112
-
113
- def multi_tasks_progress(
114
- iterables: Iterable[Iterable[int | tuple[int, int]]],
115
- *columns: ProgressColumn | str,
116
- n_jobs: int = -1,
117
- description: str = "#{:0>3}",
118
- main_description: str = "main",
119
- transient: bool | None = None,
120
- **kwargs,
121
- ) -> None:
122
- """Render auto-updating progress bars for multiple tasks concurrently.
123
-
124
- Args:
125
- iterables (Iterable[Iterable[int | tuple[int, int]]]): A collection of
126
- iterables, each representing a task. Each iterable can yield
127
- integers (completed) or tuples of integers (completed, total).
128
- *columns (ProgressColumn | str): Additional columns to display in the
129
- progress bars.
130
- n_jobs (int, optional): Number of jobs to run in parallel. Defaults to
131
- -1, which means using all processors.
132
- description (str, optional): Format string for describing tasks. Defaults to
133
- "#{:0>3}".
134
- main_description (str, optional): Description for the main task.
135
- Defaults to "main".
136
- transient (bool | None, optional): Whether to remove the progress bar
137
- after completion. Defaults to None.
138
- **kwargs: Additional keyword arguments passed to the Progress instance.
139
-
140
- Returns:
141
- None
142
-
143
- """
144
- if not columns:
145
- columns = Progress.get_default_columns()
146
-
147
- iterables = list(iterables)
148
-
149
- with Progress(*columns, transient=transient or False, **kwargs) as progress:
150
- task_main = progress.add_task(main_description, total=None)
151
-
152
- task_ids = [
153
- progress.add_task(description.format(i), start=False, total=None)
154
- for i in range(len(iterables))
155
- ]
156
-
157
- total = {}
158
- completed = {}
159
-
160
- def func(i: int) -> None:
161
- completed[i] = 0
162
- total[i] = None
163
- progress.start_task(task_ids[i])
164
-
165
- for index in iterables[i]:
166
- if isinstance(index, tuple):
167
- completed[i], total[i] = index[0] + 1, index[1]
168
- else:
169
- completed[i] = index + 1
170
-
171
- progress.update(task_ids[i], total=total[i], completed=completed[i])
172
-
173
- if all(t is not None for t in total.values()):
174
- t = sum(total.values())
175
- else:
176
- t = None
177
- c = sum(completed.values())
178
- progress.update(task_main, total=t, completed=c)
179
-
180
- if transient is not False:
181
- progress.remove_task(task_ids[i])
182
-
183
- it = (joblib.delayed(func)(i) for i in range(len(iterables)))
184
- joblib.Parallel(n_jobs, prefer="threads")(it)
@@ -1,16 +0,0 @@
1
- hydraflow/__init__.py,sha256=VbrHKs2Cg93QJ8K9WHYxkXmzOpb8o9ugiwV-mXDT0JE,908
2
- hydraflow/asyncio.py,sha256=-i1C8KAmNDImrjHnk92Csaa1mpjdK8Vp4ZVaQV-l94s,6634
3
- hydraflow/config.py,sha256=MNX9da5bPVDcjnpji7Cm9ndK6ura92pt361m4PRh6_E,4326
4
- hydraflow/context.py,sha256=kz5SvjvjN7Z_2WjHYpO9SWwDfsPT_UeZcsm8pDymhjs,8836
5
- hydraflow/mlflow.py,sha256=kWVK_Xw2hkRnTg33jSP3VW13UZF6_hBGhN52mPmLgvk,8753
6
- hydraflow/param.py,sha256=c5sc6NwD6DKwZzVwprXzZD5FSi6qRgSHkc6TXBKQEdg,4502
7
- hydraflow/progress.py,sha256=zvKX1HCN8_xDOsgYOEcLLhkhdPdep-U8vHrc0XZ-6SQ,6163
8
- hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
9
- hydraflow/run_collection.py,sha256=eBNGwtvkRKpOEqcwDUS1tkIFuxY_PVi6SEzzd1PwG5s,26774
10
- hydraflow/run_data.py,sha256=qeFX1iRvNAorXA9QQIjzr0o2_82TI44eZKp7llKG8GI,1549
11
- hydraflow/run_info.py,sha256=sMXOo20ClaRIommMEzuAbO_OrcXx7M1Yt4FMV7spxz0,998
12
- hydraflow/utils.py,sha256=Xq78F2iOkgi9JnCYfX1reQw_Y9K6o8oNYBDEwrf18cI,3552
13
- hydraflow-0.4.5.dist-info/METADATA,sha256=SOxec-saE-PWA4ViuKI1wAMI7EARLfmOPZv9t6XzInI,5127
14
- hydraflow-0.4.5.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
15
- hydraflow-0.4.5.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
16
- hydraflow-0.4.5.dist-info/RECORD,,