hydraflow 0.4.6__py3-none-any.whl → 0.5.1__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- hydraflow/__init__.py +1 -5
- hydraflow/context.py +1 -106
- hydraflow/run_collection.py +3 -3
- hydraflow/run_data.py +3 -3
- {hydraflow-0.4.6.dist-info → hydraflow-0.5.1.dist-info}/METADATA +2 -17
- hydraflow-0.5.1.dist-info/RECORD +14 -0
- hydraflow/asyncio.py +0 -227
- hydraflow/progress.py +0 -184
- hydraflow-0.4.6.dist-info/RECORD +0 -16
- {hydraflow-0.4.6.dist-info → hydraflow-0.5.1.dist-info}/WHEEL +0 -0
- {hydraflow-0.4.6.dist-info → hydraflow-0.5.1.dist-info}/licenses/LICENSE +0 -0
hydraflow/__init__.py
CHANGED
@@ -1,9 +1,8 @@
|
|
1
1
|
"""Integrate Hydra and MLflow to manage and track machine learning experiments."""
|
2
2
|
|
3
3
|
from .config import select_config, select_overrides
|
4
|
-
from .context import chdir_artifact, chdir_hydra_output, log_run, start_run
|
4
|
+
from .context import chdir_artifact, chdir_hydra_output, log_run, start_run
|
5
5
|
from .mlflow import list_runs, search_runs, set_experiment
|
6
|
-
from .progress import multi_tasks_progress, parallel_progress
|
7
6
|
from .run_collection import RunCollection
|
8
7
|
from .utils import (
|
9
8
|
get_artifact_dir,
|
@@ -25,13 +24,10 @@ __all__ = [
|
|
25
24
|
"load_config",
|
26
25
|
"load_overrides",
|
27
26
|
"log_run",
|
28
|
-
"multi_tasks_progress",
|
29
|
-
"parallel_progress",
|
30
27
|
"remove_run",
|
31
28
|
"search_runs",
|
32
29
|
"select_config",
|
33
30
|
"select_overrides",
|
34
31
|
"set_experiment",
|
35
32
|
"start_run",
|
36
|
-
"watch",
|
37
33
|
]
|
hydraflow/context.py
CHANGED
@@ -4,7 +4,6 @@ from __future__ import annotations
|
|
4
4
|
|
5
5
|
import logging
|
6
6
|
import os
|
7
|
-
import time
|
8
7
|
from contextlib import contextmanager
|
9
8
|
from pathlib import Path
|
10
9
|
from typing import TYPE_CHECKING
|
@@ -12,14 +11,11 @@ from typing import TYPE_CHECKING
|
|
12
11
|
import mlflow
|
13
12
|
import mlflow.artifacts
|
14
13
|
from hydra.core.hydra_config import HydraConfig
|
15
|
-
from watchdog.events import FileModifiedEvent, PatternMatchingEventHandler
|
16
|
-
from watchdog.observers import Observer
|
17
14
|
|
18
15
|
from hydraflow.mlflow import log_params
|
19
|
-
from hydraflow.run_info import get_artifact_dir
|
20
16
|
|
21
17
|
if TYPE_CHECKING:
|
22
|
-
from collections.abc import
|
18
|
+
from collections.abc import Iterator
|
23
19
|
|
24
20
|
from mlflow.entities.run import Run
|
25
21
|
|
@@ -64,14 +60,8 @@ def log_run(
|
|
64
60
|
output_subdir = output_dir / (hc.output_subdir or "")
|
65
61
|
mlflow.log_artifacts(output_subdir.as_posix(), hc.output_subdir)
|
66
62
|
|
67
|
-
def log_artifact(path: Path) -> None:
|
68
|
-
local_path = (output_dir / path).as_posix()
|
69
|
-
mlflow.log_artifact(local_path)
|
70
|
-
|
71
63
|
try:
|
72
64
|
yield
|
73
|
-
# with watch(log_artifact, output_dir, ignore_log=False):
|
74
|
-
# yield
|
75
65
|
|
76
66
|
except Exception as e:
|
77
67
|
msg = f"Error during log_run: {e}"
|
@@ -146,101 +136,6 @@ def start_run( # noqa: PLR0913
|
|
146
136
|
yield run
|
147
137
|
|
148
138
|
|
149
|
-
@contextmanager
|
150
|
-
def watch(
|
151
|
-
callback: Callable[[Path], None],
|
152
|
-
dir: Path | str = "", # noqa: A002
|
153
|
-
*,
|
154
|
-
timeout: int = 60,
|
155
|
-
ignore_patterns: list[str] | None = None,
|
156
|
-
ignore_log: bool = True,
|
157
|
-
) -> Iterator[None]:
|
158
|
-
"""Watch the given directory for changes.
|
159
|
-
|
160
|
-
This context manager sets up a file system watcher on the specified directory.
|
161
|
-
When a file modification is detected, the provided function is called with
|
162
|
-
the path of the modified file. The watcher runs for the specified timeout
|
163
|
-
period or until the context is exited.
|
164
|
-
|
165
|
-
Args:
|
166
|
-
callback (Callable[[Path], None]): The function to call when a change is
|
167
|
-
detected. It should accept a single argument of type `Path`,
|
168
|
-
which is the path of the modified file.
|
169
|
-
dir (Path | str): The directory to watch. If not specified,
|
170
|
-
the current MLflow artifact URI is used. Defaults to "".
|
171
|
-
timeout (int): The timeout period in seconds for the watcher
|
172
|
-
to run after the context is exited. Defaults to 60.
|
173
|
-
ignore_patterns (list[str] | None): A list of glob patterns to ignore.
|
174
|
-
Defaults to None.
|
175
|
-
ignore_log (bool): Whether to ignore log files. Defaults to True.
|
176
|
-
|
177
|
-
Yields:
|
178
|
-
None
|
179
|
-
|
180
|
-
Example:
|
181
|
-
```python
|
182
|
-
with watch(log_artifact, "/path/to/dir"):
|
183
|
-
# Perform operations while watching the directory for changes
|
184
|
-
pass
|
185
|
-
```
|
186
|
-
|
187
|
-
"""
|
188
|
-
dir = dir or get_artifact_dir() # noqa: A001
|
189
|
-
if isinstance(dir, Path):
|
190
|
-
dir = dir.as_posix() # noqa: A001
|
191
|
-
|
192
|
-
handler = Handler(callback, ignore_patterns=ignore_patterns, ignore_log=ignore_log)
|
193
|
-
observer = Observer()
|
194
|
-
observer.schedule(handler, dir, recursive=True)
|
195
|
-
observer.start()
|
196
|
-
|
197
|
-
try:
|
198
|
-
yield
|
199
|
-
|
200
|
-
except Exception as e:
|
201
|
-
msg = f"Error during watch: {e}"
|
202
|
-
log.exception(msg)
|
203
|
-
raise
|
204
|
-
|
205
|
-
finally:
|
206
|
-
elapsed = 0
|
207
|
-
while not observer.event_queue.empty():
|
208
|
-
time.sleep(0.2)
|
209
|
-
elapsed += 0.2
|
210
|
-
if elapsed > timeout:
|
211
|
-
break
|
212
|
-
|
213
|
-
observer.stop()
|
214
|
-
observer.join()
|
215
|
-
|
216
|
-
|
217
|
-
class Handler(PatternMatchingEventHandler):
|
218
|
-
"""Monitor file changes and call the given function when a change is detected."""
|
219
|
-
|
220
|
-
def __init__(
|
221
|
-
self,
|
222
|
-
func: Callable[[Path], None],
|
223
|
-
*,
|
224
|
-
ignore_patterns: list[str] | None = None,
|
225
|
-
ignore_log: bool = True,
|
226
|
-
) -> None:
|
227
|
-
self.func = func
|
228
|
-
|
229
|
-
if ignore_log:
|
230
|
-
if ignore_patterns:
|
231
|
-
ignore_patterns.append("*.log")
|
232
|
-
else:
|
233
|
-
ignore_patterns = ["*.log"]
|
234
|
-
|
235
|
-
super().__init__(ignore_patterns=ignore_patterns)
|
236
|
-
|
237
|
-
def on_modified(self, event: FileModifiedEvent) -> None:
|
238
|
-
"""Modify when a file is modified."""
|
239
|
-
file = Path(str(event.src_path))
|
240
|
-
if file.is_file():
|
241
|
-
self.func(file)
|
242
|
-
|
243
|
-
|
244
139
|
@contextmanager
|
245
140
|
def chdir_hydra_output() -> Iterator[Path]:
|
246
141
|
"""Change the current working directory to the hydra output directory.
|
hydraflow/run_collection.py
CHANGED
@@ -575,7 +575,7 @@ class RunCollection:
|
|
575
575
|
"""
|
576
576
|
return (func(dir, *args, **kwargs) for dir in self.info.artifact_dir) # noqa: A001
|
577
577
|
|
578
|
-
def
|
578
|
+
def groupby(
|
579
579
|
self,
|
580
580
|
names: str | list[str],
|
581
581
|
) -> dict[str | None | tuple[str | None, ...], RunCollection]:
|
@@ -609,8 +609,8 @@ class RunCollection:
|
|
609
609
|
|
610
610
|
def sort(
|
611
611
|
self,
|
612
|
-
key: Callable[[Run], Any] | None = None,
|
613
612
|
*,
|
613
|
+
key: Callable[[Run], Any] | None = None,
|
614
614
|
reverse: bool = False,
|
615
615
|
) -> None:
|
616
616
|
"""Sort the runs in the collection.
|
@@ -652,7 +652,7 @@ class RunCollection:
|
|
652
652
|
|
653
653
|
return [v[0] for v in values]
|
654
654
|
|
655
|
-
def
|
655
|
+
def sorted(
|
656
656
|
self,
|
657
657
|
names: str | list[str],
|
658
658
|
*,
|
hydraflow/run_data.py
CHANGED
@@ -4,7 +4,7 @@ from __future__ import annotations
|
|
4
4
|
|
5
5
|
from typing import TYPE_CHECKING
|
6
6
|
|
7
|
-
from
|
7
|
+
from pandas import DataFrame
|
8
8
|
|
9
9
|
from hydraflow.config import collect_params
|
10
10
|
|
@@ -33,10 +33,10 @@ class RunCollectionData:
|
|
33
33
|
|
34
34
|
@property
|
35
35
|
def config(self) -> DataFrame:
|
36
|
-
"""Get the runs' configurations as a
|
36
|
+
"""Get the runs' configurations as a DataFrame.
|
37
37
|
|
38
38
|
Returns:
|
39
|
-
A
|
39
|
+
A DataFrame containing the runs' configurations.
|
40
40
|
|
41
41
|
"""
|
42
42
|
return DataFrame(self._runs.map_config(collect_params))
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: hydraflow
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.5.1
|
4
4
|
Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
|
5
5
|
Project-URL: Documentation, https://github.com/daizutabi/hydraflow
|
6
6
|
Project-URL: Source, https://github.com/daizutabi/hydraflow
|
@@ -37,12 +37,7 @@ Classifier: Programming Language :: Python :: 3.12
|
|
37
37
|
Classifier: Programming Language :: Python :: 3.13
|
38
38
|
Requires-Python: >=3.10
|
39
39
|
Requires-Dist: hydra-core>=1.3
|
40
|
-
Requires-Dist: joblib
|
41
40
|
Requires-Dist: mlflow>=2.15
|
42
|
-
Requires-Dist: polars
|
43
|
-
Requires-Dist: rich
|
44
|
-
Requires-Dist: watchdog
|
45
|
-
Requires-Dist: watchfiles
|
46
41
|
Description-Content-Type: text/markdown
|
47
42
|
|
48
43
|
# Hydraflow
|
@@ -116,19 +111,9 @@ def my_app(cfg: MySQLConfig) -> None:
|
|
116
111
|
hydraflow.set_experiment()
|
117
112
|
|
118
113
|
# Automatically log Hydra config as params.
|
119
|
-
with hydraflow.start_run():
|
114
|
+
with hydraflow.start_run(cfg):
|
120
115
|
# Your app code below.
|
121
116
|
|
122
|
-
with hydraflow.watch(callback):
|
123
|
-
# Watch files in the MLflow artifact directory.
|
124
|
-
# You can update metrics or log other artifacts
|
125
|
-
# according to the watched files in your callback
|
126
|
-
# function.
|
127
|
-
pass
|
128
|
-
|
129
|
-
# Your callback function here.
|
130
|
-
def callback(file: Path) -> None:
|
131
|
-
pass
|
132
117
|
|
133
118
|
if __name__ == "__main__":
|
134
119
|
my_app()
|
@@ -0,0 +1,14 @@
|
|
1
|
+
hydraflow/__init__.py,sha256=DKtFjTXHTgceX7rpWHiKqhcpG5xtGIseFvN28f7iwYo,807
|
2
|
+
hydraflow/config.py,sha256=MNX9da5bPVDcjnpji7Cm9ndK6ura92pt361m4PRh6_E,4326
|
3
|
+
hydraflow/context.py,sha256=3g7OQXWcFvK6PVVbXpQg7Hr8nsJkF9pLFrXNi_3aV5A,5524
|
4
|
+
hydraflow/mlflow.py,sha256=kWVK_Xw2hkRnTg33jSP3VW13UZF6_hBGhN52mPmLgvk,8753
|
5
|
+
hydraflow/param.py,sha256=c5sc6NwD6DKwZzVwprXzZD5FSi6qRgSHkc6TXBKQEdg,4502
|
6
|
+
hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
7
|
+
hydraflow/run_collection.py,sha256=zPrlKwLuzqePj57pXbgKWrE03S_kjxTaxY9trItf6Gc,26772
|
8
|
+
hydraflow/run_data.py,sha256=dpyyfnuH9mCtIZeigMo1iFQo9bafMdEL4i4uI2l0UqY,1525
|
9
|
+
hydraflow/run_info.py,sha256=sMXOo20ClaRIommMEzuAbO_OrcXx7M1Yt4FMV7spxz0,998
|
10
|
+
hydraflow/utils.py,sha256=jbNrbtIfMqxE4LrdTNd1g7sF68XgAvydGqW5iAZ6n-c,3834
|
11
|
+
hydraflow-0.5.1.dist-info/METADATA,sha256=ZRMGo-8y9JiUSCijXQMeVkG9Gw0edDRgBdkptMDK0IU,4700
|
12
|
+
hydraflow-0.5.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
13
|
+
hydraflow-0.5.1.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
|
14
|
+
hydraflow-0.5.1.dist-info/RECORD,,
|
hydraflow/asyncio.py
DELETED
@@ -1,227 +0,0 @@
|
|
1
|
-
"""Provide functionality for running commands and monitoring file changes."""
|
2
|
-
|
3
|
-
from __future__ import annotations
|
4
|
-
|
5
|
-
import asyncio
|
6
|
-
import logging
|
7
|
-
from asyncio.subprocess import PIPE
|
8
|
-
from pathlib import Path
|
9
|
-
from typing import TYPE_CHECKING
|
10
|
-
|
11
|
-
import watchfiles
|
12
|
-
|
13
|
-
if TYPE_CHECKING:
|
14
|
-
from asyncio.streams import StreamReader
|
15
|
-
from collections.abc import Callable
|
16
|
-
|
17
|
-
from watchfiles import Change
|
18
|
-
|
19
|
-
|
20
|
-
# Set up logging
|
21
|
-
logging.basicConfig(level=logging.INFO)
|
22
|
-
logger = logging.getLogger(__name__)
|
23
|
-
|
24
|
-
|
25
|
-
async def execute_command(
|
26
|
-
program: str,
|
27
|
-
*args: str,
|
28
|
-
stdout: Callable[[str], None] | None = None,
|
29
|
-
stderr: Callable[[str], None] | None = None,
|
30
|
-
stop_event: asyncio.Event,
|
31
|
-
) -> int:
|
32
|
-
"""Run a command asynchronously and pass the output to callback functions.
|
33
|
-
|
34
|
-
Args:
|
35
|
-
program (str): The program to run.
|
36
|
-
*args (str): Arguments for the program.
|
37
|
-
stdout (Callable[[str], None] | None): Callback for standard output.
|
38
|
-
stderr (Callable[[str], None] | None): Callback for standard error.
|
39
|
-
stop_event (asyncio.Event): Event to signal when the process is done.
|
40
|
-
|
41
|
-
Returns:
|
42
|
-
int: The return code of the process.
|
43
|
-
|
44
|
-
"""
|
45
|
-
try:
|
46
|
-
process = await asyncio.create_subprocess_exec(
|
47
|
-
program,
|
48
|
-
*args,
|
49
|
-
stdout=PIPE,
|
50
|
-
stderr=PIPE,
|
51
|
-
)
|
52
|
-
await asyncio.gather(
|
53
|
-
process_stream(process.stdout, stdout),
|
54
|
-
process_stream(process.stderr, stderr),
|
55
|
-
)
|
56
|
-
returncode = await process.wait()
|
57
|
-
|
58
|
-
except Exception as e:
|
59
|
-
msg = f"Error running command: {e}"
|
60
|
-
logger.exception(msg)
|
61
|
-
returncode = 1
|
62
|
-
|
63
|
-
finally:
|
64
|
-
stop_event.set()
|
65
|
-
|
66
|
-
return returncode
|
67
|
-
|
68
|
-
|
69
|
-
async def process_stream(
|
70
|
-
stream: StreamReader | None,
|
71
|
-
callback: Callable[[str], None] | None,
|
72
|
-
) -> None:
|
73
|
-
"""Read a stream asynchronously and pass each line to a callback function.
|
74
|
-
|
75
|
-
Args:
|
76
|
-
stream (StreamReader | None): The stream to read from.
|
77
|
-
callback (Callable[[str], None] | None): The callback function to handle
|
78
|
-
each line.
|
79
|
-
|
80
|
-
"""
|
81
|
-
if stream is None or callback is None:
|
82
|
-
return
|
83
|
-
|
84
|
-
while True:
|
85
|
-
line = await stream.readline()
|
86
|
-
if line:
|
87
|
-
callback(line.decode().strip())
|
88
|
-
else:
|
89
|
-
break
|
90
|
-
|
91
|
-
|
92
|
-
async def monitor_file_changes(
|
93
|
-
paths: list[str | Path],
|
94
|
-
callback: Callable[[set[tuple[Change, str]]], None],
|
95
|
-
stop_event: asyncio.Event,
|
96
|
-
**awatch_kwargs,
|
97
|
-
) -> None:
|
98
|
-
"""Watch file changes in specified paths and pass the changes to a callback.
|
99
|
-
|
100
|
-
Args:
|
101
|
-
paths (list[str | Path]): List of paths to monitor for changes.
|
102
|
-
callback (Callable[[set[tuple[Change, str]]], None]): The callback
|
103
|
-
function to handle file changes.
|
104
|
-
stop_event (asyncio.Event): Event to signal when to stop watching.
|
105
|
-
**awatch_kwargs: Additional keyword arguments to pass to watchfiles.awatch.
|
106
|
-
|
107
|
-
"""
|
108
|
-
str_paths = [str(path) for path in paths]
|
109
|
-
try:
|
110
|
-
async for changes in watchfiles.awatch(
|
111
|
-
*str_paths,
|
112
|
-
stop_event=stop_event,
|
113
|
-
**awatch_kwargs,
|
114
|
-
):
|
115
|
-
callback(changes)
|
116
|
-
except Exception as e:
|
117
|
-
msg = f"Error watching files: {e}"
|
118
|
-
logger.exception(msg)
|
119
|
-
raise
|
120
|
-
|
121
|
-
|
122
|
-
async def run_and_monitor(
|
123
|
-
program: str,
|
124
|
-
*args: str,
|
125
|
-
stdout: Callable[[str], None] | None = None,
|
126
|
-
stderr: Callable[[str], None] | None = None,
|
127
|
-
watch: Callable[[set[tuple[Change, str]]], None] | None = None,
|
128
|
-
paths: list[str | Path] | None = None,
|
129
|
-
**awatch_kwargs,
|
130
|
-
) -> int:
|
131
|
-
"""Run a command and optionally watch for file changes concurrently.
|
132
|
-
|
133
|
-
Args:
|
134
|
-
program (str): The program to run.
|
135
|
-
*args (str): Arguments for the program.
|
136
|
-
stdout (Callable[[str], None] | None): Callback for standard output.
|
137
|
-
stderr (Callable[[str], None] | None): Callback for standard error.
|
138
|
-
watch (Callable[[set[tuple[Change, str]]], None] | None): Callback for
|
139
|
-
file changes.
|
140
|
-
paths (list[str | Path] | None): List of paths to monitor for changes.
|
141
|
-
**awatch_kwargs: Additional keyword arguments to pass to `watchfiles.awatch`.
|
142
|
-
|
143
|
-
"""
|
144
|
-
stop_event = asyncio.Event()
|
145
|
-
run_task = asyncio.create_task(
|
146
|
-
execute_command(
|
147
|
-
program,
|
148
|
-
*args,
|
149
|
-
stop_event=stop_event,
|
150
|
-
stdout=stdout,
|
151
|
-
stderr=stderr,
|
152
|
-
),
|
153
|
-
)
|
154
|
-
if watch and paths:
|
155
|
-
coro = monitor_file_changes(paths, watch, stop_event, **awatch_kwargs)
|
156
|
-
monitor_task = asyncio.create_task(coro)
|
157
|
-
else:
|
158
|
-
monitor_task = None
|
159
|
-
|
160
|
-
try:
|
161
|
-
if monitor_task:
|
162
|
-
await asyncio.gather(run_task, monitor_task)
|
163
|
-
else:
|
164
|
-
await run_task
|
165
|
-
|
166
|
-
except Exception as e:
|
167
|
-
msg = f"Error in run_and_monitor: {e}"
|
168
|
-
logger.exception(msg)
|
169
|
-
raise
|
170
|
-
|
171
|
-
finally:
|
172
|
-
stop_event.set()
|
173
|
-
await run_task
|
174
|
-
if monitor_task:
|
175
|
-
await monitor_task
|
176
|
-
|
177
|
-
return run_task.result()
|
178
|
-
|
179
|
-
|
180
|
-
def run(
|
181
|
-
program: str,
|
182
|
-
*args: str,
|
183
|
-
stdout: Callable[[str], None] | None = None,
|
184
|
-
stderr: Callable[[str], None] | None = None,
|
185
|
-
watch: Callable[[set[tuple[Change, str]]], None] | None = None,
|
186
|
-
paths: list[str | Path] | None = None,
|
187
|
-
**awatch_kwargs,
|
188
|
-
) -> int:
|
189
|
-
"""Run a command synchronously and optionally watch for file changes.
|
190
|
-
|
191
|
-
This function is a synchronous wrapper around the asynchronous
|
192
|
-
`run_and_monitor` function. It runs a specified command and optionally
|
193
|
-
monitors specified paths for file changes, invoking the provided callbacks for
|
194
|
-
standard output, standard error, and file changes.
|
195
|
-
|
196
|
-
Args:
|
197
|
-
program (str): The program to run.
|
198
|
-
*args (str): Arguments for the program.
|
199
|
-
stdout (Callable[[str], None] | None): Callback for handling standard
|
200
|
-
output lines.
|
201
|
-
stderr (Callable[[str], None] | None): Callback for handling standard
|
202
|
-
error lines.
|
203
|
-
watch (Callable[[set[tuple[Change, str]]], None] | None): Callback for
|
204
|
-
handling file changes.
|
205
|
-
paths (list[str | Path] | None): List of paths to monitor for file
|
206
|
-
changes.
|
207
|
-
**awatch_kwargs: Additional keyword arguments to pass to
|
208
|
-
`watchfiles.awatch`.
|
209
|
-
|
210
|
-
Returns:
|
211
|
-
int: The return code of the process.
|
212
|
-
|
213
|
-
"""
|
214
|
-
if watch and not paths:
|
215
|
-
paths = [Path.cwd()]
|
216
|
-
|
217
|
-
return asyncio.run(
|
218
|
-
run_and_monitor(
|
219
|
-
program,
|
220
|
-
*args,
|
221
|
-
stdout=stdout,
|
222
|
-
stderr=stderr,
|
223
|
-
watch=watch,
|
224
|
-
paths=paths,
|
225
|
-
**awatch_kwargs,
|
226
|
-
),
|
227
|
-
)
|
hydraflow/progress.py
DELETED
@@ -1,184 +0,0 @@
|
|
1
|
-
"""Context managers and functions for parallel task execution with progress.
|
2
|
-
|
3
|
-
Provide context managers and functions to facilitate the execution
|
4
|
-
of tasks in parallel while displaying progress updates.
|
5
|
-
"""
|
6
|
-
|
7
|
-
from __future__ import annotations
|
8
|
-
|
9
|
-
from contextlib import contextmanager
|
10
|
-
from typing import TYPE_CHECKING, TypeVar
|
11
|
-
|
12
|
-
import joblib
|
13
|
-
from rich.progress import Progress
|
14
|
-
|
15
|
-
if TYPE_CHECKING:
|
16
|
-
from collections.abc import Callable, Iterable, Iterator
|
17
|
-
|
18
|
-
from rich.progress import ProgressColumn
|
19
|
-
|
20
|
-
|
21
|
-
# https://github.com/jonghwanhyeon/joblib-progress/blob/main/joblib_progress/__init__.py
|
22
|
-
@contextmanager
|
23
|
-
def JoblibProgress( # noqa: N802
|
24
|
-
*columns: ProgressColumn | str,
|
25
|
-
description: str | None = None,
|
26
|
-
total: int | None = None,
|
27
|
-
**kwargs,
|
28
|
-
) -> Iterator[Progress]:
|
29
|
-
"""Context manager for tracking progress using Joblib with Rich's Progress bar.
|
30
|
-
|
31
|
-
Args:
|
32
|
-
*columns (ProgressColumn | str): Columns to display in the progress bar.
|
33
|
-
description (str | None, optional): A description for the progress task.
|
34
|
-
Defaults to None.
|
35
|
-
total (int | None, optional): The total number of tasks. If None, it will
|
36
|
-
be determined automatically.
|
37
|
-
**kwargs: Additional keyword arguments passed to the Progress instance.
|
38
|
-
|
39
|
-
Yields:
|
40
|
-
Progress: A Progress instance for managing the progress bar.
|
41
|
-
|
42
|
-
Example:
|
43
|
-
```python
|
44
|
-
with JoblibProgress("task", total=100) as progress:
|
45
|
-
# Your parallel processing code here
|
46
|
-
```
|
47
|
-
|
48
|
-
"""
|
49
|
-
if not columns:
|
50
|
-
columns = Progress.get_default_columns()
|
51
|
-
|
52
|
-
progress = Progress(*columns, **kwargs)
|
53
|
-
|
54
|
-
if description is None:
|
55
|
-
description = "Processing..."
|
56
|
-
|
57
|
-
task_id = progress.add_task(description, total=total)
|
58
|
-
print_progress = joblib.parallel.Parallel.print_progress
|
59
|
-
|
60
|
-
def update_progress(self: joblib.parallel.Parallel) -> None:
|
61
|
-
progress.update(task_id, completed=self.n_completed_tasks, refresh=True)
|
62
|
-
return print_progress(self)
|
63
|
-
|
64
|
-
try:
|
65
|
-
joblib.parallel.Parallel.print_progress = update_progress
|
66
|
-
progress.start()
|
67
|
-
yield progress
|
68
|
-
|
69
|
-
finally:
|
70
|
-
progress.stop()
|
71
|
-
joblib.parallel.Parallel.print_progress = print_progress
|
72
|
-
|
73
|
-
|
74
|
-
T = TypeVar("T")
|
75
|
-
U = TypeVar("U")
|
76
|
-
|
77
|
-
|
78
|
-
def parallel_progress(
|
79
|
-
func: Callable[[T], U],
|
80
|
-
iterable: Iterable[T],
|
81
|
-
*columns: ProgressColumn | str,
|
82
|
-
n_jobs: int = -1,
|
83
|
-
description: str | None = None,
|
84
|
-
**kwargs,
|
85
|
-
) -> list[U]:
|
86
|
-
"""Execute a function in parallel over an iterable with progress tracking.
|
87
|
-
|
88
|
-
Args:
|
89
|
-
func (Callable[[T], U]): The function to execute on each item in the
|
90
|
-
iterable.
|
91
|
-
iterable (Iterable[T]): An iterable of items to process.
|
92
|
-
*columns (ProgressColumn | str): Additional columns to display in the
|
93
|
-
progress bar.
|
94
|
-
n_jobs (int, optional): The number of jobs to run in parallel.
|
95
|
-
Defaults to -1 (all processors).
|
96
|
-
description (str | None, optional): A description for the progress bar.
|
97
|
-
Defaults to None.
|
98
|
-
**kwargs: Additional keyword arguments passed to the Progress instance.
|
99
|
-
|
100
|
-
Returns:
|
101
|
-
list[U]: A list of results from applying the function to each item in
|
102
|
-
the iterable.
|
103
|
-
|
104
|
-
"""
|
105
|
-
iterable = list(iterable)
|
106
|
-
total = len(iterable)
|
107
|
-
|
108
|
-
with JoblibProgress(*columns, description=description, total=total, **kwargs):
|
109
|
-
it = (joblib.delayed(func)(x) for x in iterable)
|
110
|
-
return joblib.Parallel(n_jobs=n_jobs)(it) # type: ignore
|
111
|
-
|
112
|
-
|
113
|
-
def multi_tasks_progress(
|
114
|
-
iterables: Iterable[Iterable[int | tuple[int, int]]],
|
115
|
-
*columns: ProgressColumn | str,
|
116
|
-
n_jobs: int = -1,
|
117
|
-
description: str = "#{:0>3}",
|
118
|
-
main_description: str = "main",
|
119
|
-
transient: bool | None = None,
|
120
|
-
**kwargs,
|
121
|
-
) -> None:
|
122
|
-
"""Render auto-updating progress bars for multiple tasks concurrently.
|
123
|
-
|
124
|
-
Args:
|
125
|
-
iterables (Iterable[Iterable[int | tuple[int, int]]]): A collection of
|
126
|
-
iterables, each representing a task. Each iterable can yield
|
127
|
-
integers (completed) or tuples of integers (completed, total).
|
128
|
-
*columns (ProgressColumn | str): Additional columns to display in the
|
129
|
-
progress bars.
|
130
|
-
n_jobs (int, optional): Number of jobs to run in parallel. Defaults to
|
131
|
-
-1, which means using all processors.
|
132
|
-
description (str, optional): Format string for describing tasks. Defaults to
|
133
|
-
"#{:0>3}".
|
134
|
-
main_description (str, optional): Description for the main task.
|
135
|
-
Defaults to "main".
|
136
|
-
transient (bool | None, optional): Whether to remove the progress bar
|
137
|
-
after completion. Defaults to None.
|
138
|
-
**kwargs: Additional keyword arguments passed to the Progress instance.
|
139
|
-
|
140
|
-
Returns:
|
141
|
-
None
|
142
|
-
|
143
|
-
"""
|
144
|
-
if not columns:
|
145
|
-
columns = Progress.get_default_columns()
|
146
|
-
|
147
|
-
iterables = list(iterables)
|
148
|
-
|
149
|
-
with Progress(*columns, transient=transient or False, **kwargs) as progress:
|
150
|
-
task_main = progress.add_task(main_description, total=None)
|
151
|
-
|
152
|
-
task_ids = [
|
153
|
-
progress.add_task(description.format(i), start=False, total=None)
|
154
|
-
for i in range(len(iterables))
|
155
|
-
]
|
156
|
-
|
157
|
-
total = {}
|
158
|
-
completed = {}
|
159
|
-
|
160
|
-
def func(i: int) -> None:
|
161
|
-
completed[i] = 0
|
162
|
-
total[i] = None
|
163
|
-
progress.start_task(task_ids[i])
|
164
|
-
|
165
|
-
for index in iterables[i]:
|
166
|
-
if isinstance(index, tuple):
|
167
|
-
completed[i], total[i] = index[0] + 1, index[1]
|
168
|
-
else:
|
169
|
-
completed[i] = index + 1
|
170
|
-
|
171
|
-
progress.update(task_ids[i], total=total[i], completed=completed[i])
|
172
|
-
|
173
|
-
if all(t is not None for t in total.values()):
|
174
|
-
t = sum(total.values())
|
175
|
-
else:
|
176
|
-
t = None
|
177
|
-
c = sum(completed.values())
|
178
|
-
progress.update(task_main, total=t, completed=c)
|
179
|
-
|
180
|
-
if transient is not False:
|
181
|
-
progress.remove_task(task_ids[i])
|
182
|
-
|
183
|
-
it = (joblib.delayed(func)(i) for i in range(len(iterables)))
|
184
|
-
joblib.Parallel(n_jobs, prefer="threads")(it)
|
hydraflow-0.4.6.dist-info/RECORD
DELETED
@@ -1,16 +0,0 @@
|
|
1
|
-
hydraflow/__init__.py,sha256=4wUu8HR__oM0lTUiIqxO7iP6ubLSfKI6y7_P9_RuYtA,942
|
2
|
-
hydraflow/asyncio.py,sha256=-i1C8KAmNDImrjHnk92Csaa1mpjdK8Vp4ZVaQV-l94s,6634
|
3
|
-
hydraflow/config.py,sha256=MNX9da5bPVDcjnpji7Cm9ndK6ura92pt361m4PRh6_E,4326
|
4
|
-
hydraflow/context.py,sha256=kz5SvjvjN7Z_2WjHYpO9SWwDfsPT_UeZcsm8pDymhjs,8836
|
5
|
-
hydraflow/mlflow.py,sha256=kWVK_Xw2hkRnTg33jSP3VW13UZF6_hBGhN52mPmLgvk,8753
|
6
|
-
hydraflow/param.py,sha256=c5sc6NwD6DKwZzVwprXzZD5FSi6qRgSHkc6TXBKQEdg,4502
|
7
|
-
hydraflow/progress.py,sha256=zvKX1HCN8_xDOsgYOEcLLhkhdPdep-U8vHrc0XZ-6SQ,6163
|
8
|
-
hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
9
|
-
hydraflow/run_collection.py,sha256=eBNGwtvkRKpOEqcwDUS1tkIFuxY_PVi6SEzzd1PwG5s,26774
|
10
|
-
hydraflow/run_data.py,sha256=qeFX1iRvNAorXA9QQIjzr0o2_82TI44eZKp7llKG8GI,1549
|
11
|
-
hydraflow/run_info.py,sha256=sMXOo20ClaRIommMEzuAbO_OrcXx7M1Yt4FMV7spxz0,998
|
12
|
-
hydraflow/utils.py,sha256=jbNrbtIfMqxE4LrdTNd1g7sF68XgAvydGqW5iAZ6n-c,3834
|
13
|
-
hydraflow-0.4.6.dist-info/METADATA,sha256=s3eXM1oDcJVWj930b3c_9iKATXLoYZMMDmA9THkCdu8,5149
|
14
|
-
hydraflow-0.4.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
15
|
-
hydraflow-0.4.6.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
|
16
|
-
hydraflow-0.4.6.dist-info/RECORD,,
|
File without changes
|
File without changes
|