hydraflow 0.4.6__py3-none-any.whl → 0.5.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hydraflow/__init__.py +1 -5
- hydraflow/context.py +1 -106
- hydraflow/run_collection.py +3 -3
- hydraflow/run_data.py +3 -3
- {hydraflow-0.4.6.dist-info → hydraflow-0.5.1.dist-info}/METADATA +2 -17
- hydraflow-0.5.1.dist-info/RECORD +14 -0
- hydraflow/asyncio.py +0 -227
- hydraflow/progress.py +0 -184
- hydraflow-0.4.6.dist-info/RECORD +0 -16
- {hydraflow-0.4.6.dist-info → hydraflow-0.5.1.dist-info}/WHEEL +0 -0
- {hydraflow-0.4.6.dist-info → hydraflow-0.5.1.dist-info}/licenses/LICENSE +0 -0
hydraflow/__init__.py
CHANGED
@@ -1,9 +1,8 @@
|
|
1
1
|
"""Integrate Hydra and MLflow to manage and track machine learning experiments."""
|
2
2
|
|
3
3
|
from .config import select_config, select_overrides
|
4
|
-
from .context import chdir_artifact, chdir_hydra_output, log_run, start_run
|
4
|
+
from .context import chdir_artifact, chdir_hydra_output, log_run, start_run
|
5
5
|
from .mlflow import list_runs, search_runs, set_experiment
|
6
|
-
from .progress import multi_tasks_progress, parallel_progress
|
7
6
|
from .run_collection import RunCollection
|
8
7
|
from .utils import (
|
9
8
|
get_artifact_dir,
|
@@ -25,13 +24,10 @@ __all__ = [
|
|
25
24
|
"load_config",
|
26
25
|
"load_overrides",
|
27
26
|
"log_run",
|
28
|
-
"multi_tasks_progress",
|
29
|
-
"parallel_progress",
|
30
27
|
"remove_run",
|
31
28
|
"search_runs",
|
32
29
|
"select_config",
|
33
30
|
"select_overrides",
|
34
31
|
"set_experiment",
|
35
32
|
"start_run",
|
36
|
-
"watch",
|
37
33
|
]
|
hydraflow/context.py
CHANGED
@@ -4,7 +4,6 @@ from __future__ import annotations
|
|
4
4
|
|
5
5
|
import logging
|
6
6
|
import os
|
7
|
-
import time
|
8
7
|
from contextlib import contextmanager
|
9
8
|
from pathlib import Path
|
10
9
|
from typing import TYPE_CHECKING
|
@@ -12,14 +11,11 @@ from typing import TYPE_CHECKING
|
|
12
11
|
import mlflow
|
13
12
|
import mlflow.artifacts
|
14
13
|
from hydra.core.hydra_config import HydraConfig
|
15
|
-
from watchdog.events import FileModifiedEvent, PatternMatchingEventHandler
|
16
|
-
from watchdog.observers import Observer
|
17
14
|
|
18
15
|
from hydraflow.mlflow import log_params
|
19
|
-
from hydraflow.run_info import get_artifact_dir
|
20
16
|
|
21
17
|
if TYPE_CHECKING:
|
22
|
-
from collections.abc import
|
18
|
+
from collections.abc import Iterator
|
23
19
|
|
24
20
|
from mlflow.entities.run import Run
|
25
21
|
|
@@ -64,14 +60,8 @@ def log_run(
|
|
64
60
|
output_subdir = output_dir / (hc.output_subdir or "")
|
65
61
|
mlflow.log_artifacts(output_subdir.as_posix(), hc.output_subdir)
|
66
62
|
|
67
|
-
def log_artifact(path: Path) -> None:
|
68
|
-
local_path = (output_dir / path).as_posix()
|
69
|
-
mlflow.log_artifact(local_path)
|
70
|
-
|
71
63
|
try:
|
72
64
|
yield
|
73
|
-
# with watch(log_artifact, output_dir, ignore_log=False):
|
74
|
-
# yield
|
75
65
|
|
76
66
|
except Exception as e:
|
77
67
|
msg = f"Error during log_run: {e}"
|
@@ -146,101 +136,6 @@ def start_run( # noqa: PLR0913
|
|
146
136
|
yield run
|
147
137
|
|
148
138
|
|
149
|
-
@contextmanager
|
150
|
-
def watch(
|
151
|
-
callback: Callable[[Path], None],
|
152
|
-
dir: Path | str = "", # noqa: A002
|
153
|
-
*,
|
154
|
-
timeout: int = 60,
|
155
|
-
ignore_patterns: list[str] | None = None,
|
156
|
-
ignore_log: bool = True,
|
157
|
-
) -> Iterator[None]:
|
158
|
-
"""Watch the given directory for changes.
|
159
|
-
|
160
|
-
This context manager sets up a file system watcher on the specified directory.
|
161
|
-
When a file modification is detected, the provided function is called with
|
162
|
-
the path of the modified file. The watcher runs for the specified timeout
|
163
|
-
period or until the context is exited.
|
164
|
-
|
165
|
-
Args:
|
166
|
-
callback (Callable[[Path], None]): The function to call when a change is
|
167
|
-
detected. It should accept a single argument of type `Path`,
|
168
|
-
which is the path of the modified file.
|
169
|
-
dir (Path | str): The directory to watch. If not specified,
|
170
|
-
the current MLflow artifact URI is used. Defaults to "".
|
171
|
-
timeout (int): The timeout period in seconds for the watcher
|
172
|
-
to run after the context is exited. Defaults to 60.
|
173
|
-
ignore_patterns (list[str] | None): A list of glob patterns to ignore.
|
174
|
-
Defaults to None.
|
175
|
-
ignore_log (bool): Whether to ignore log files. Defaults to True.
|
176
|
-
|
177
|
-
Yields:
|
178
|
-
None
|
179
|
-
|
180
|
-
Example:
|
181
|
-
```python
|
182
|
-
with watch(log_artifact, "/path/to/dir"):
|
183
|
-
# Perform operations while watching the directory for changes
|
184
|
-
pass
|
185
|
-
```
|
186
|
-
|
187
|
-
"""
|
188
|
-
dir = dir or get_artifact_dir() # noqa: A001
|
189
|
-
if isinstance(dir, Path):
|
190
|
-
dir = dir.as_posix() # noqa: A001
|
191
|
-
|
192
|
-
handler = Handler(callback, ignore_patterns=ignore_patterns, ignore_log=ignore_log)
|
193
|
-
observer = Observer()
|
194
|
-
observer.schedule(handler, dir, recursive=True)
|
195
|
-
observer.start()
|
196
|
-
|
197
|
-
try:
|
198
|
-
yield
|
199
|
-
|
200
|
-
except Exception as e:
|
201
|
-
msg = f"Error during watch: {e}"
|
202
|
-
log.exception(msg)
|
203
|
-
raise
|
204
|
-
|
205
|
-
finally:
|
206
|
-
elapsed = 0
|
207
|
-
while not observer.event_queue.empty():
|
208
|
-
time.sleep(0.2)
|
209
|
-
elapsed += 0.2
|
210
|
-
if elapsed > timeout:
|
211
|
-
break
|
212
|
-
|
213
|
-
observer.stop()
|
214
|
-
observer.join()
|
215
|
-
|
216
|
-
|
217
|
-
class Handler(PatternMatchingEventHandler):
|
218
|
-
"""Monitor file changes and call the given function when a change is detected."""
|
219
|
-
|
220
|
-
def __init__(
|
221
|
-
self,
|
222
|
-
func: Callable[[Path], None],
|
223
|
-
*,
|
224
|
-
ignore_patterns: list[str] | None = None,
|
225
|
-
ignore_log: bool = True,
|
226
|
-
) -> None:
|
227
|
-
self.func = func
|
228
|
-
|
229
|
-
if ignore_log:
|
230
|
-
if ignore_patterns:
|
231
|
-
ignore_patterns.append("*.log")
|
232
|
-
else:
|
233
|
-
ignore_patterns = ["*.log"]
|
234
|
-
|
235
|
-
super().__init__(ignore_patterns=ignore_patterns)
|
236
|
-
|
237
|
-
def on_modified(self, event: FileModifiedEvent) -> None:
|
238
|
-
"""Modify when a file is modified."""
|
239
|
-
file = Path(str(event.src_path))
|
240
|
-
if file.is_file():
|
241
|
-
self.func(file)
|
242
|
-
|
243
|
-
|
244
139
|
@contextmanager
|
245
140
|
def chdir_hydra_output() -> Iterator[Path]:
|
246
141
|
"""Change the current working directory to the hydra output directory.
|
hydraflow/run_collection.py
CHANGED
@@ -575,7 +575,7 @@ class RunCollection:
|
|
575
575
|
"""
|
576
576
|
return (func(dir, *args, **kwargs) for dir in self.info.artifact_dir) # noqa: A001
|
577
577
|
|
578
|
-
def
|
578
|
+
def groupby(
|
579
579
|
self,
|
580
580
|
names: str | list[str],
|
581
581
|
) -> dict[str | None | tuple[str | None, ...], RunCollection]:
|
@@ -609,8 +609,8 @@ class RunCollection:
|
|
609
609
|
|
610
610
|
def sort(
|
611
611
|
self,
|
612
|
-
key: Callable[[Run], Any] | None = None,
|
613
612
|
*,
|
613
|
+
key: Callable[[Run], Any] | None = None,
|
614
614
|
reverse: bool = False,
|
615
615
|
) -> None:
|
616
616
|
"""Sort the runs in the collection.
|
@@ -652,7 +652,7 @@ class RunCollection:
|
|
652
652
|
|
653
653
|
return [v[0] for v in values]
|
654
654
|
|
655
|
-
def
|
655
|
+
def sorted(
|
656
656
|
self,
|
657
657
|
names: str | list[str],
|
658
658
|
*,
|
hydraflow/run_data.py
CHANGED
@@ -4,7 +4,7 @@ from __future__ import annotations
|
|
4
4
|
|
5
5
|
from typing import TYPE_CHECKING
|
6
6
|
|
7
|
-
from
|
7
|
+
from pandas import DataFrame
|
8
8
|
|
9
9
|
from hydraflow.config import collect_params
|
10
10
|
|
@@ -33,10 +33,10 @@ class RunCollectionData:
|
|
33
33
|
|
34
34
|
@property
|
35
35
|
def config(self) -> DataFrame:
|
36
|
-
"""Get the runs' configurations as a
|
36
|
+
"""Get the runs' configurations as a DataFrame.
|
37
37
|
|
38
38
|
Returns:
|
39
|
-
A
|
39
|
+
A DataFrame containing the runs' configurations.
|
40
40
|
|
41
41
|
"""
|
42
42
|
return DataFrame(self._runs.map_config(collect_params))
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: hydraflow
|
3
|
-
Version: 0.
|
3
|
+
Version: 0.5.1
|
4
4
|
Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
|
5
5
|
Project-URL: Documentation, https://github.com/daizutabi/hydraflow
|
6
6
|
Project-URL: Source, https://github.com/daizutabi/hydraflow
|
@@ -37,12 +37,7 @@ Classifier: Programming Language :: Python :: 3.12
|
|
37
37
|
Classifier: Programming Language :: Python :: 3.13
|
38
38
|
Requires-Python: >=3.10
|
39
39
|
Requires-Dist: hydra-core>=1.3
|
40
|
-
Requires-Dist: joblib
|
41
40
|
Requires-Dist: mlflow>=2.15
|
42
|
-
Requires-Dist: polars
|
43
|
-
Requires-Dist: rich
|
44
|
-
Requires-Dist: watchdog
|
45
|
-
Requires-Dist: watchfiles
|
46
41
|
Description-Content-Type: text/markdown
|
47
42
|
|
48
43
|
# Hydraflow
|
@@ -116,19 +111,9 @@ def my_app(cfg: MySQLConfig) -> None:
|
|
116
111
|
hydraflow.set_experiment()
|
117
112
|
|
118
113
|
# Automatically log Hydra config as params.
|
119
|
-
with hydraflow.start_run():
|
114
|
+
with hydraflow.start_run(cfg):
|
120
115
|
# Your app code below.
|
121
116
|
|
122
|
-
with hydraflow.watch(callback):
|
123
|
-
# Watch files in the MLflow artifact directory.
|
124
|
-
# You can update metrics or log other artifacts
|
125
|
-
# according to the watched files in your callback
|
126
|
-
# function.
|
127
|
-
pass
|
128
|
-
|
129
|
-
# Your callback function here.
|
130
|
-
def callback(file: Path) -> None:
|
131
|
-
pass
|
132
117
|
|
133
118
|
if __name__ == "__main__":
|
134
119
|
my_app()
|
@@ -0,0 +1,14 @@
|
|
1
|
+
hydraflow/__init__.py,sha256=DKtFjTXHTgceX7rpWHiKqhcpG5xtGIseFvN28f7iwYo,807
|
2
|
+
hydraflow/config.py,sha256=MNX9da5bPVDcjnpji7Cm9ndK6ura92pt361m4PRh6_E,4326
|
3
|
+
hydraflow/context.py,sha256=3g7OQXWcFvK6PVVbXpQg7Hr8nsJkF9pLFrXNi_3aV5A,5524
|
4
|
+
hydraflow/mlflow.py,sha256=kWVK_Xw2hkRnTg33jSP3VW13UZF6_hBGhN52mPmLgvk,8753
|
5
|
+
hydraflow/param.py,sha256=c5sc6NwD6DKwZzVwprXzZD5FSi6qRgSHkc6TXBKQEdg,4502
|
6
|
+
hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
7
|
+
hydraflow/run_collection.py,sha256=zPrlKwLuzqePj57pXbgKWrE03S_kjxTaxY9trItf6Gc,26772
|
8
|
+
hydraflow/run_data.py,sha256=dpyyfnuH9mCtIZeigMo1iFQo9bafMdEL4i4uI2l0UqY,1525
|
9
|
+
hydraflow/run_info.py,sha256=sMXOo20ClaRIommMEzuAbO_OrcXx7M1Yt4FMV7spxz0,998
|
10
|
+
hydraflow/utils.py,sha256=jbNrbtIfMqxE4LrdTNd1g7sF68XgAvydGqW5iAZ6n-c,3834
|
11
|
+
hydraflow-0.5.1.dist-info/METADATA,sha256=ZRMGo-8y9JiUSCijXQMeVkG9Gw0edDRgBdkptMDK0IU,4700
|
12
|
+
hydraflow-0.5.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
13
|
+
hydraflow-0.5.1.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
|
14
|
+
hydraflow-0.5.1.dist-info/RECORD,,
|
hydraflow/asyncio.py
DELETED
@@ -1,227 +0,0 @@
|
|
1
|
-
"""Provide functionality for running commands and monitoring file changes."""
|
2
|
-
|
3
|
-
from __future__ import annotations
|
4
|
-
|
5
|
-
import asyncio
|
6
|
-
import logging
|
7
|
-
from asyncio.subprocess import PIPE
|
8
|
-
from pathlib import Path
|
9
|
-
from typing import TYPE_CHECKING
|
10
|
-
|
11
|
-
import watchfiles
|
12
|
-
|
13
|
-
if TYPE_CHECKING:
|
14
|
-
from asyncio.streams import StreamReader
|
15
|
-
from collections.abc import Callable
|
16
|
-
|
17
|
-
from watchfiles import Change
|
18
|
-
|
19
|
-
|
20
|
-
# Set up logging
|
21
|
-
logging.basicConfig(level=logging.INFO)
|
22
|
-
logger = logging.getLogger(__name__)
|
23
|
-
|
24
|
-
|
25
|
-
async def execute_command(
|
26
|
-
program: str,
|
27
|
-
*args: str,
|
28
|
-
stdout: Callable[[str], None] | None = None,
|
29
|
-
stderr: Callable[[str], None] | None = None,
|
30
|
-
stop_event: asyncio.Event,
|
31
|
-
) -> int:
|
32
|
-
"""Run a command asynchronously and pass the output to callback functions.
|
33
|
-
|
34
|
-
Args:
|
35
|
-
program (str): The program to run.
|
36
|
-
*args (str): Arguments for the program.
|
37
|
-
stdout (Callable[[str], None] | None): Callback for standard output.
|
38
|
-
stderr (Callable[[str], None] | None): Callback for standard error.
|
39
|
-
stop_event (asyncio.Event): Event to signal when the process is done.
|
40
|
-
|
41
|
-
Returns:
|
42
|
-
int: The return code of the process.
|
43
|
-
|
44
|
-
"""
|
45
|
-
try:
|
46
|
-
process = await asyncio.create_subprocess_exec(
|
47
|
-
program,
|
48
|
-
*args,
|
49
|
-
stdout=PIPE,
|
50
|
-
stderr=PIPE,
|
51
|
-
)
|
52
|
-
await asyncio.gather(
|
53
|
-
process_stream(process.stdout, stdout),
|
54
|
-
process_stream(process.stderr, stderr),
|
55
|
-
)
|
56
|
-
returncode = await process.wait()
|
57
|
-
|
58
|
-
except Exception as e:
|
59
|
-
msg = f"Error running command: {e}"
|
60
|
-
logger.exception(msg)
|
61
|
-
returncode = 1
|
62
|
-
|
63
|
-
finally:
|
64
|
-
stop_event.set()
|
65
|
-
|
66
|
-
return returncode
|
67
|
-
|
68
|
-
|
69
|
-
async def process_stream(
|
70
|
-
stream: StreamReader | None,
|
71
|
-
callback: Callable[[str], None] | None,
|
72
|
-
) -> None:
|
73
|
-
"""Read a stream asynchronously and pass each line to a callback function.
|
74
|
-
|
75
|
-
Args:
|
76
|
-
stream (StreamReader | None): The stream to read from.
|
77
|
-
callback (Callable[[str], None] | None): The callback function to handle
|
78
|
-
each line.
|
79
|
-
|
80
|
-
"""
|
81
|
-
if stream is None or callback is None:
|
82
|
-
return
|
83
|
-
|
84
|
-
while True:
|
85
|
-
line = await stream.readline()
|
86
|
-
if line:
|
87
|
-
callback(line.decode().strip())
|
88
|
-
else:
|
89
|
-
break
|
90
|
-
|
91
|
-
|
92
|
-
async def monitor_file_changes(
|
93
|
-
paths: list[str | Path],
|
94
|
-
callback: Callable[[set[tuple[Change, str]]], None],
|
95
|
-
stop_event: asyncio.Event,
|
96
|
-
**awatch_kwargs,
|
97
|
-
) -> None:
|
98
|
-
"""Watch file changes in specified paths and pass the changes to a callback.
|
99
|
-
|
100
|
-
Args:
|
101
|
-
paths (list[str | Path]): List of paths to monitor for changes.
|
102
|
-
callback (Callable[[set[tuple[Change, str]]], None]): The callback
|
103
|
-
function to handle file changes.
|
104
|
-
stop_event (asyncio.Event): Event to signal when to stop watching.
|
105
|
-
**awatch_kwargs: Additional keyword arguments to pass to watchfiles.awatch.
|
106
|
-
|
107
|
-
"""
|
108
|
-
str_paths = [str(path) for path in paths]
|
109
|
-
try:
|
110
|
-
async for changes in watchfiles.awatch(
|
111
|
-
*str_paths,
|
112
|
-
stop_event=stop_event,
|
113
|
-
**awatch_kwargs,
|
114
|
-
):
|
115
|
-
callback(changes)
|
116
|
-
except Exception as e:
|
117
|
-
msg = f"Error watching files: {e}"
|
118
|
-
logger.exception(msg)
|
119
|
-
raise
|
120
|
-
|
121
|
-
|
122
|
-
async def run_and_monitor(
|
123
|
-
program: str,
|
124
|
-
*args: str,
|
125
|
-
stdout: Callable[[str], None] | None = None,
|
126
|
-
stderr: Callable[[str], None] | None = None,
|
127
|
-
watch: Callable[[set[tuple[Change, str]]], None] | None = None,
|
128
|
-
paths: list[str | Path] | None = None,
|
129
|
-
**awatch_kwargs,
|
130
|
-
) -> int:
|
131
|
-
"""Run a command and optionally watch for file changes concurrently.
|
132
|
-
|
133
|
-
Args:
|
134
|
-
program (str): The program to run.
|
135
|
-
*args (str): Arguments for the program.
|
136
|
-
stdout (Callable[[str], None] | None): Callback for standard output.
|
137
|
-
stderr (Callable[[str], None] | None): Callback for standard error.
|
138
|
-
watch (Callable[[set[tuple[Change, str]]], None] | None): Callback for
|
139
|
-
file changes.
|
140
|
-
paths (list[str | Path] | None): List of paths to monitor for changes.
|
141
|
-
**awatch_kwargs: Additional keyword arguments to pass to `watchfiles.awatch`.
|
142
|
-
|
143
|
-
"""
|
144
|
-
stop_event = asyncio.Event()
|
145
|
-
run_task = asyncio.create_task(
|
146
|
-
execute_command(
|
147
|
-
program,
|
148
|
-
*args,
|
149
|
-
stop_event=stop_event,
|
150
|
-
stdout=stdout,
|
151
|
-
stderr=stderr,
|
152
|
-
),
|
153
|
-
)
|
154
|
-
if watch and paths:
|
155
|
-
coro = monitor_file_changes(paths, watch, stop_event, **awatch_kwargs)
|
156
|
-
monitor_task = asyncio.create_task(coro)
|
157
|
-
else:
|
158
|
-
monitor_task = None
|
159
|
-
|
160
|
-
try:
|
161
|
-
if monitor_task:
|
162
|
-
await asyncio.gather(run_task, monitor_task)
|
163
|
-
else:
|
164
|
-
await run_task
|
165
|
-
|
166
|
-
except Exception as e:
|
167
|
-
msg = f"Error in run_and_monitor: {e}"
|
168
|
-
logger.exception(msg)
|
169
|
-
raise
|
170
|
-
|
171
|
-
finally:
|
172
|
-
stop_event.set()
|
173
|
-
await run_task
|
174
|
-
if monitor_task:
|
175
|
-
await monitor_task
|
176
|
-
|
177
|
-
return run_task.result()
|
178
|
-
|
179
|
-
|
180
|
-
def run(
|
181
|
-
program: str,
|
182
|
-
*args: str,
|
183
|
-
stdout: Callable[[str], None] | None = None,
|
184
|
-
stderr: Callable[[str], None] | None = None,
|
185
|
-
watch: Callable[[set[tuple[Change, str]]], None] | None = None,
|
186
|
-
paths: list[str | Path] | None = None,
|
187
|
-
**awatch_kwargs,
|
188
|
-
) -> int:
|
189
|
-
"""Run a command synchronously and optionally watch for file changes.
|
190
|
-
|
191
|
-
This function is a synchronous wrapper around the asynchronous
|
192
|
-
`run_and_monitor` function. It runs a specified command and optionally
|
193
|
-
monitors specified paths for file changes, invoking the provided callbacks for
|
194
|
-
standard output, standard error, and file changes.
|
195
|
-
|
196
|
-
Args:
|
197
|
-
program (str): The program to run.
|
198
|
-
*args (str): Arguments for the program.
|
199
|
-
stdout (Callable[[str], None] | None): Callback for handling standard
|
200
|
-
output lines.
|
201
|
-
stderr (Callable[[str], None] | None): Callback for handling standard
|
202
|
-
error lines.
|
203
|
-
watch (Callable[[set[tuple[Change, str]]], None] | None): Callback for
|
204
|
-
handling file changes.
|
205
|
-
paths (list[str | Path] | None): List of paths to monitor for file
|
206
|
-
changes.
|
207
|
-
**awatch_kwargs: Additional keyword arguments to pass to
|
208
|
-
`watchfiles.awatch`.
|
209
|
-
|
210
|
-
Returns:
|
211
|
-
int: The return code of the process.
|
212
|
-
|
213
|
-
"""
|
214
|
-
if watch and not paths:
|
215
|
-
paths = [Path.cwd()]
|
216
|
-
|
217
|
-
return asyncio.run(
|
218
|
-
run_and_monitor(
|
219
|
-
program,
|
220
|
-
*args,
|
221
|
-
stdout=stdout,
|
222
|
-
stderr=stderr,
|
223
|
-
watch=watch,
|
224
|
-
paths=paths,
|
225
|
-
**awatch_kwargs,
|
226
|
-
),
|
227
|
-
)
|
hydraflow/progress.py
DELETED
@@ -1,184 +0,0 @@
|
|
1
|
-
"""Context managers and functions for parallel task execution with progress.
|
2
|
-
|
3
|
-
Provide context managers and functions to facilitate the execution
|
4
|
-
of tasks in parallel while displaying progress updates.
|
5
|
-
"""
|
6
|
-
|
7
|
-
from __future__ import annotations
|
8
|
-
|
9
|
-
from contextlib import contextmanager
|
10
|
-
from typing import TYPE_CHECKING, TypeVar
|
11
|
-
|
12
|
-
import joblib
|
13
|
-
from rich.progress import Progress
|
14
|
-
|
15
|
-
if TYPE_CHECKING:
|
16
|
-
from collections.abc import Callable, Iterable, Iterator
|
17
|
-
|
18
|
-
from rich.progress import ProgressColumn
|
19
|
-
|
20
|
-
|
21
|
-
# https://github.com/jonghwanhyeon/joblib-progress/blob/main/joblib_progress/__init__.py
|
22
|
-
@contextmanager
|
23
|
-
def JoblibProgress( # noqa: N802
|
24
|
-
*columns: ProgressColumn | str,
|
25
|
-
description: str | None = None,
|
26
|
-
total: int | None = None,
|
27
|
-
**kwargs,
|
28
|
-
) -> Iterator[Progress]:
|
29
|
-
"""Context manager for tracking progress using Joblib with Rich's Progress bar.
|
30
|
-
|
31
|
-
Args:
|
32
|
-
*columns (ProgressColumn | str): Columns to display in the progress bar.
|
33
|
-
description (str | None, optional): A description for the progress task.
|
34
|
-
Defaults to None.
|
35
|
-
total (int | None, optional): The total number of tasks. If None, it will
|
36
|
-
be determined automatically.
|
37
|
-
**kwargs: Additional keyword arguments passed to the Progress instance.
|
38
|
-
|
39
|
-
Yields:
|
40
|
-
Progress: A Progress instance for managing the progress bar.
|
41
|
-
|
42
|
-
Example:
|
43
|
-
```python
|
44
|
-
with JoblibProgress("task", total=100) as progress:
|
45
|
-
# Your parallel processing code here
|
46
|
-
```
|
47
|
-
|
48
|
-
"""
|
49
|
-
if not columns:
|
50
|
-
columns = Progress.get_default_columns()
|
51
|
-
|
52
|
-
progress = Progress(*columns, **kwargs)
|
53
|
-
|
54
|
-
if description is None:
|
55
|
-
description = "Processing..."
|
56
|
-
|
57
|
-
task_id = progress.add_task(description, total=total)
|
58
|
-
print_progress = joblib.parallel.Parallel.print_progress
|
59
|
-
|
60
|
-
def update_progress(self: joblib.parallel.Parallel) -> None:
|
61
|
-
progress.update(task_id, completed=self.n_completed_tasks, refresh=True)
|
62
|
-
return print_progress(self)
|
63
|
-
|
64
|
-
try:
|
65
|
-
joblib.parallel.Parallel.print_progress = update_progress
|
66
|
-
progress.start()
|
67
|
-
yield progress
|
68
|
-
|
69
|
-
finally:
|
70
|
-
progress.stop()
|
71
|
-
joblib.parallel.Parallel.print_progress = print_progress
|
72
|
-
|
73
|
-
|
74
|
-
T = TypeVar("T")
|
75
|
-
U = TypeVar("U")
|
76
|
-
|
77
|
-
|
78
|
-
def parallel_progress(
|
79
|
-
func: Callable[[T], U],
|
80
|
-
iterable: Iterable[T],
|
81
|
-
*columns: ProgressColumn | str,
|
82
|
-
n_jobs: int = -1,
|
83
|
-
description: str | None = None,
|
84
|
-
**kwargs,
|
85
|
-
) -> list[U]:
|
86
|
-
"""Execute a function in parallel over an iterable with progress tracking.
|
87
|
-
|
88
|
-
Args:
|
89
|
-
func (Callable[[T], U]): The function to execute on each item in the
|
90
|
-
iterable.
|
91
|
-
iterable (Iterable[T]): An iterable of items to process.
|
92
|
-
*columns (ProgressColumn | str): Additional columns to display in the
|
93
|
-
progress bar.
|
94
|
-
n_jobs (int, optional): The number of jobs to run in parallel.
|
95
|
-
Defaults to -1 (all processors).
|
96
|
-
description (str | None, optional): A description for the progress bar.
|
97
|
-
Defaults to None.
|
98
|
-
**kwargs: Additional keyword arguments passed to the Progress instance.
|
99
|
-
|
100
|
-
Returns:
|
101
|
-
list[U]: A list of results from applying the function to each item in
|
102
|
-
the iterable.
|
103
|
-
|
104
|
-
"""
|
105
|
-
iterable = list(iterable)
|
106
|
-
total = len(iterable)
|
107
|
-
|
108
|
-
with JoblibProgress(*columns, description=description, total=total, **kwargs):
|
109
|
-
it = (joblib.delayed(func)(x) for x in iterable)
|
110
|
-
return joblib.Parallel(n_jobs=n_jobs)(it) # type: ignore
|
111
|
-
|
112
|
-
|
113
|
-
def multi_tasks_progress(
|
114
|
-
iterables: Iterable[Iterable[int | tuple[int, int]]],
|
115
|
-
*columns: ProgressColumn | str,
|
116
|
-
n_jobs: int = -1,
|
117
|
-
description: str = "#{:0>3}",
|
118
|
-
main_description: str = "main",
|
119
|
-
transient: bool | None = None,
|
120
|
-
**kwargs,
|
121
|
-
) -> None:
|
122
|
-
"""Render auto-updating progress bars for multiple tasks concurrently.
|
123
|
-
|
124
|
-
Args:
|
125
|
-
iterables (Iterable[Iterable[int | tuple[int, int]]]): A collection of
|
126
|
-
iterables, each representing a task. Each iterable can yield
|
127
|
-
integers (completed) or tuples of integers (completed, total).
|
128
|
-
*columns (ProgressColumn | str): Additional columns to display in the
|
129
|
-
progress bars.
|
130
|
-
n_jobs (int, optional): Number of jobs to run in parallel. Defaults to
|
131
|
-
-1, which means using all processors.
|
132
|
-
description (str, optional): Format string for describing tasks. Defaults to
|
133
|
-
"#{:0>3}".
|
134
|
-
main_description (str, optional): Description for the main task.
|
135
|
-
Defaults to "main".
|
136
|
-
transient (bool | None, optional): Whether to remove the progress bar
|
137
|
-
after completion. Defaults to None.
|
138
|
-
**kwargs: Additional keyword arguments passed to the Progress instance.
|
139
|
-
|
140
|
-
Returns:
|
141
|
-
None
|
142
|
-
|
143
|
-
"""
|
144
|
-
if not columns:
|
145
|
-
columns = Progress.get_default_columns()
|
146
|
-
|
147
|
-
iterables = list(iterables)
|
148
|
-
|
149
|
-
with Progress(*columns, transient=transient or False, **kwargs) as progress:
|
150
|
-
task_main = progress.add_task(main_description, total=None)
|
151
|
-
|
152
|
-
task_ids = [
|
153
|
-
progress.add_task(description.format(i), start=False, total=None)
|
154
|
-
for i in range(len(iterables))
|
155
|
-
]
|
156
|
-
|
157
|
-
total = {}
|
158
|
-
completed = {}
|
159
|
-
|
160
|
-
def func(i: int) -> None:
|
161
|
-
completed[i] = 0
|
162
|
-
total[i] = None
|
163
|
-
progress.start_task(task_ids[i])
|
164
|
-
|
165
|
-
for index in iterables[i]:
|
166
|
-
if isinstance(index, tuple):
|
167
|
-
completed[i], total[i] = index[0] + 1, index[1]
|
168
|
-
else:
|
169
|
-
completed[i] = index + 1
|
170
|
-
|
171
|
-
progress.update(task_ids[i], total=total[i], completed=completed[i])
|
172
|
-
|
173
|
-
if all(t is not None for t in total.values()):
|
174
|
-
t = sum(total.values())
|
175
|
-
else:
|
176
|
-
t = None
|
177
|
-
c = sum(completed.values())
|
178
|
-
progress.update(task_main, total=t, completed=c)
|
179
|
-
|
180
|
-
if transient is not False:
|
181
|
-
progress.remove_task(task_ids[i])
|
182
|
-
|
183
|
-
it = (joblib.delayed(func)(i) for i in range(len(iterables)))
|
184
|
-
joblib.Parallel(n_jobs, prefer="threads")(it)
|
hydraflow-0.4.6.dist-info/RECORD
DELETED
@@ -1,16 +0,0 @@
|
|
1
|
-
hydraflow/__init__.py,sha256=4wUu8HR__oM0lTUiIqxO7iP6ubLSfKI6y7_P9_RuYtA,942
|
2
|
-
hydraflow/asyncio.py,sha256=-i1C8KAmNDImrjHnk92Csaa1mpjdK8Vp4ZVaQV-l94s,6634
|
3
|
-
hydraflow/config.py,sha256=MNX9da5bPVDcjnpji7Cm9ndK6ura92pt361m4PRh6_E,4326
|
4
|
-
hydraflow/context.py,sha256=kz5SvjvjN7Z_2WjHYpO9SWwDfsPT_UeZcsm8pDymhjs,8836
|
5
|
-
hydraflow/mlflow.py,sha256=kWVK_Xw2hkRnTg33jSP3VW13UZF6_hBGhN52mPmLgvk,8753
|
6
|
-
hydraflow/param.py,sha256=c5sc6NwD6DKwZzVwprXzZD5FSi6qRgSHkc6TXBKQEdg,4502
|
7
|
-
hydraflow/progress.py,sha256=zvKX1HCN8_xDOsgYOEcLLhkhdPdep-U8vHrc0XZ-6SQ,6163
|
8
|
-
hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
9
|
-
hydraflow/run_collection.py,sha256=eBNGwtvkRKpOEqcwDUS1tkIFuxY_PVi6SEzzd1PwG5s,26774
|
10
|
-
hydraflow/run_data.py,sha256=qeFX1iRvNAorXA9QQIjzr0o2_82TI44eZKp7llKG8GI,1549
|
11
|
-
hydraflow/run_info.py,sha256=sMXOo20ClaRIommMEzuAbO_OrcXx7M1Yt4FMV7spxz0,998
|
12
|
-
hydraflow/utils.py,sha256=jbNrbtIfMqxE4LrdTNd1g7sF68XgAvydGqW5iAZ6n-c,3834
|
13
|
-
hydraflow-0.4.6.dist-info/METADATA,sha256=s3eXM1oDcJVWj930b3c_9iKATXLoYZMMDmA9THkCdu8,5149
|
14
|
-
hydraflow-0.4.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
15
|
-
hydraflow-0.4.6.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
|
16
|
-
hydraflow-0.4.6.dist-info/RECORD,,
|
File without changes
|
File without changes
|