hydraflow 0.2.17__py3-none-any.whl → 0.2.18__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hydraflow/asyncio.py +13 -11
- hydraflow/config.py +3 -6
- hydraflow/context.py +15 -15
- hydraflow/info.py +16 -6
- hydraflow/mlflow.py +36 -23
- hydraflow/param.py +11 -0
- hydraflow/progress.py +7 -18
- hydraflow/run_collection.py +112 -71
- {hydraflow-0.2.17.dist-info → hydraflow-0.2.18.dist-info}/METADATA +1 -1
- hydraflow-0.2.18.dist-info/RECORD +14 -0
- hydraflow-0.2.17.dist-info/RECORD +0 -14
- {hydraflow-0.2.17.dist-info → hydraflow-0.2.18.dist-info}/WHEEL +0 -0
- {hydraflow-0.2.17.dist-info → hydraflow-0.2.18.dist-info}/licenses/LICENSE +0 -0
hydraflow/asyncio.py
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
"""Provide functionality for running commands and monitoring file changes."""
|
2
|
+
|
1
3
|
from __future__ import annotations
|
2
4
|
|
3
5
|
import asyncio
|
@@ -27,8 +29,7 @@ async def execute_command(
|
|
27
29
|
stderr: Callable[[str], None] | None = None,
|
28
30
|
stop_event: asyncio.Event,
|
29
31
|
) -> int:
|
30
|
-
"""
|
31
|
-
Runs a command asynchronously and pass the output to callback functions.
|
32
|
+
"""Run a command asynchronously and pass the output to callback functions.
|
32
33
|
|
33
34
|
Args:
|
34
35
|
program (str): The program to run.
|
@@ -39,6 +40,7 @@ async def execute_command(
|
|
39
40
|
|
40
41
|
Returns:
|
41
42
|
int: The return code of the process.
|
43
|
+
|
42
44
|
"""
|
43
45
|
try:
|
44
46
|
process = await asyncio.create_subprocess_exec(
|
@@ -68,13 +70,13 @@ async def process_stream(
|
|
68
70
|
stream: StreamReader | None,
|
69
71
|
callback: Callable[[str], None] | None,
|
70
72
|
) -> None:
|
71
|
-
"""
|
72
|
-
Reads a stream asynchronously and pass each line to a callback function.
|
73
|
+
"""Read a stream asynchronously and pass each line to a callback function.
|
73
74
|
|
74
75
|
Args:
|
75
76
|
stream (StreamReader | None): The stream to read from.
|
76
77
|
callback (Callable[[str], None] | None): The callback function to handle
|
77
78
|
each line.
|
79
|
+
|
78
80
|
"""
|
79
81
|
if stream is None or callback is None:
|
80
82
|
return
|
@@ -93,9 +95,7 @@ async def monitor_file_changes(
|
|
93
95
|
stop_event: asyncio.Event,
|
94
96
|
**awatch_kwargs,
|
95
97
|
) -> None:
|
96
|
-
"""
|
97
|
-
Watches for file changes in specified paths and pass the changes to a
|
98
|
-
callback function.
|
98
|
+
"""Watch file changes in specified paths and pass the changes to a callback.
|
99
99
|
|
100
100
|
Args:
|
101
101
|
paths (list[str | Path]): List of paths to monitor for changes.
|
@@ -103,6 +103,7 @@ async def monitor_file_changes(
|
|
103
103
|
function to handle file changes.
|
104
104
|
stop_event (asyncio.Event): Event to signal when to stop watching.
|
105
105
|
**awatch_kwargs: Additional keyword arguments to pass to watchfiles.awatch.
|
106
|
+
|
106
107
|
"""
|
107
108
|
str_paths = [str(path) for path in paths]
|
108
109
|
try:
|
@@ -127,8 +128,7 @@ async def run_and_monitor(
|
|
127
128
|
paths: list[str | Path] | None = None,
|
128
129
|
**awatch_kwargs,
|
129
130
|
) -> int:
|
130
|
-
"""
|
131
|
-
Runs a command and optionally watch for file changes concurrently.
|
131
|
+
"""Run a command and optionally watch for file changes concurrently.
|
132
132
|
|
133
133
|
Args:
|
134
134
|
program (str): The program to run.
|
@@ -138,6 +138,8 @@ async def run_and_monitor(
|
|
138
138
|
watch (Callable[[set[tuple[Change, str]]], None] | None): Callback for
|
139
139
|
file changes.
|
140
140
|
paths (list[str | Path] | None): List of paths to monitor for changes.
|
141
|
+
**awatch_kwargs: Additional keyword arguments to pass to `watchfiles.awatch`.
|
142
|
+
|
141
143
|
"""
|
142
144
|
stop_event = asyncio.Event()
|
143
145
|
run_task = asyncio.create_task(
|
@@ -184,8 +186,7 @@ def run(
|
|
184
186
|
paths: list[str | Path] | None = None,
|
185
187
|
**awatch_kwargs,
|
186
188
|
) -> int:
|
187
|
-
"""
|
188
|
-
Run a command synchronously and optionally watch for file changes.
|
189
|
+
"""Run a command synchronously and optionally watch for file changes.
|
189
190
|
|
190
191
|
This function is a synchronous wrapper around the asynchronous
|
191
192
|
`run_and_monitor` function. It runs a specified command and optionally
|
@@ -208,6 +209,7 @@ def run(
|
|
208
209
|
|
209
210
|
Returns:
|
210
211
|
int: The return code of the process.
|
212
|
+
|
211
213
|
"""
|
212
214
|
if watch and not paths:
|
213
215
|
paths = [Path.cwd()]
|
hydraflow/config.py
CHANGED
@@ -1,7 +1,4 @@
|
|
1
|
-
"""
|
2
|
-
This module provides functionality for working with configuration
|
3
|
-
objects using the OmegaConf library.
|
4
|
-
"""
|
1
|
+
"""Provide functionality for working with configuration objects using the OmegaConf."""
|
5
2
|
|
6
3
|
from __future__ import annotations
|
7
4
|
|
@@ -15,8 +12,7 @@ if TYPE_CHECKING:
|
|
15
12
|
|
16
13
|
|
17
14
|
def iter_params(config: object, prefix: str = "") -> Iterator[tuple[str, Any]]:
|
18
|
-
"""
|
19
|
-
Recursively iterate over the parameters in the given configuration object.
|
15
|
+
"""Recursively iterate over the parameters in the given configuration object.
|
20
16
|
|
21
17
|
This function traverses the configuration object and yields key-value pairs
|
22
18
|
representing the parameters. The keys are prefixed with the provided prefix.
|
@@ -29,6 +25,7 @@ def iter_params(config: object, prefix: str = "") -> Iterator[tuple[str, Any]]:
|
|
29
25
|
|
30
26
|
Yields:
|
31
27
|
Key-value pairs representing the parameters in the configuration object.
|
28
|
+
|
32
29
|
"""
|
33
30
|
if config is None:
|
34
31
|
return
|
hydraflow/context.py
CHANGED
@@ -1,7 +1,4 @@
|
|
1
|
-
"""
|
2
|
-
This module provides context managers to log parameters and manage the MLflow
|
3
|
-
run context.
|
4
|
-
"""
|
1
|
+
"""Provide context managers to log parameters and manage the MLflow run context."""
|
5
2
|
|
6
3
|
from __future__ import annotations
|
7
4
|
|
@@ -34,9 +31,7 @@ def log_run(
|
|
34
31
|
*,
|
35
32
|
synchronous: bool | None = None,
|
36
33
|
) -> Iterator[None]:
|
37
|
-
"""
|
38
|
-
Log the parameters from the given configuration object and manage the MLflow
|
39
|
-
run context.
|
34
|
+
"""Log the parameters from the given configuration object.
|
40
35
|
|
41
36
|
This context manager logs the parameters from the provided configuration object
|
42
37
|
using MLflow. It also manages the MLflow run context, ensuring that artifacts
|
@@ -56,6 +51,7 @@ def log_run(
|
|
56
51
|
# Perform operations within the MLflow run context
|
57
52
|
pass
|
58
53
|
```
|
54
|
+
|
59
55
|
"""
|
60
56
|
log_params(config, synchronous=synchronous)
|
61
57
|
|
@@ -98,8 +94,7 @@ def start_run( # noqa: PLR0913
|
|
98
94
|
log_system_metrics: bool | None = None,
|
99
95
|
synchronous: bool | None = None,
|
100
96
|
) -> Iterator[Run]:
|
101
|
-
"""
|
102
|
-
Start an MLflow run and log parameters using the provided configuration object.
|
97
|
+
"""Start an MLflow run and log parameters using the provided configuration object.
|
103
98
|
|
104
99
|
This context manager starts an MLflow run and logs parameters using the specified
|
105
100
|
configuration object. It ensures that the run is properly closed after completion.
|
@@ -130,6 +125,7 @@ def start_run( # noqa: PLR0913
|
|
130
125
|
- `mlflow.start_run`: The MLflow function to start a run directly.
|
131
126
|
- `log_run`: A context manager to log parameters and manage the MLflow
|
132
127
|
run context.
|
128
|
+
|
133
129
|
"""
|
134
130
|
with (
|
135
131
|
mlflow.start_run(
|
@@ -156,9 +152,7 @@ def watch(
|
|
156
152
|
ignore_patterns: list[str] | None = None,
|
157
153
|
ignore_log: bool = True,
|
158
154
|
) -> Iterator[None]:
|
159
|
-
"""
|
160
|
-
Watch the given directory for changes and call the provided function
|
161
|
-
when a change is detected.
|
155
|
+
"""Watch the given directory for changes.
|
162
156
|
|
163
157
|
This context manager sets up a file system watcher on the specified directory.
|
164
158
|
When a file modification is detected, the provided function is called with
|
@@ -173,6 +167,9 @@ def watch(
|
|
173
167
|
the current MLflow artifact URI is used. Defaults to "".
|
174
168
|
timeout (int): The timeout period in seconds for the watcher
|
175
169
|
to run after the context is exited. Defaults to 60.
|
170
|
+
ignore_patterns (list[str] | None): A list of glob patterns to ignore.
|
171
|
+
Defaults to None.
|
172
|
+
ignore_log (bool): Whether to ignore log files. Defaults to True.
|
176
173
|
|
177
174
|
Yields:
|
178
175
|
None
|
@@ -183,6 +180,7 @@ def watch(
|
|
183
180
|
# Perform operations while watching the directory for changes
|
184
181
|
pass
|
185
182
|
```
|
183
|
+
|
186
184
|
"""
|
187
185
|
dir = dir or get_artifact_dir() # noqa: A001
|
188
186
|
if isinstance(dir, Path):
|
@@ -214,6 +212,8 @@ def watch(
|
|
214
212
|
|
215
213
|
|
216
214
|
class Handler(PatternMatchingEventHandler):
|
215
|
+
"""Monitor file changes and call the given function when a change is detected."""
|
216
|
+
|
217
217
|
def __init__(
|
218
218
|
self,
|
219
219
|
func: Callable[[Path], None],
|
@@ -232,6 +232,7 @@ class Handler(PatternMatchingEventHandler):
|
|
232
232
|
super().__init__(ignore_patterns=ignore_patterns)
|
233
233
|
|
234
234
|
def on_modified(self, event: FileModifiedEvent) -> None:
|
235
|
+
"""Modify when a file is modified."""
|
235
236
|
file = Path(str(event.src_path))
|
236
237
|
if file.is_file():
|
237
238
|
self.func(file)
|
@@ -242,9 +243,7 @@ def chdir_artifact(
|
|
242
243
|
run: Run,
|
243
244
|
artifact_path: str | None = None,
|
244
245
|
) -> Iterator[Path]:
|
245
|
-
"""
|
246
|
-
Change the current working directory to the artifact directory of the
|
247
|
-
given run.
|
246
|
+
"""Change the current working directory to the artifact directory of the given run.
|
248
247
|
|
249
248
|
This context manager changes the current working directory to the artifact
|
250
249
|
directory of the given run. It ensures that the directory is changed back
|
@@ -253,6 +252,7 @@ def chdir_artifact(
|
|
253
252
|
Args:
|
254
253
|
run (Run): The run to get the artifact directory from.
|
255
254
|
artifact_path (str | None): The artifact path.
|
255
|
+
|
256
256
|
"""
|
257
257
|
curdir = Path.cwd()
|
258
258
|
path = mlflow.artifacts.download_artifacts(
|
hydraflow/info.py
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
"""Provide information about MLflow runs."""
|
2
|
+
|
1
3
|
from __future__ import annotations
|
2
4
|
|
3
5
|
from pathlib import Path
|
@@ -15,37 +17,44 @@ if TYPE_CHECKING:
|
|
15
17
|
|
16
18
|
|
17
19
|
class RunCollectionInfo:
|
20
|
+
"""Provide information about MLflow runs."""
|
21
|
+
|
18
22
|
def __init__(self, runs: RunCollection) -> None:
|
19
23
|
self._runs = runs
|
20
24
|
|
21
25
|
@property
|
22
26
|
def run_id(self) -> list[str]:
|
27
|
+
"""Get the run ID for each run in the collection."""
|
23
28
|
return [run.info.run_id for run in self._runs]
|
24
29
|
|
25
30
|
@property
|
26
31
|
def params(self) -> list[dict[str, str]]:
|
32
|
+
"""Get the parameters for each run in the collection."""
|
27
33
|
return [run.data.params for run in self._runs]
|
28
34
|
|
29
35
|
@property
|
30
36
|
def metrics(self) -> list[dict[str, float]]:
|
37
|
+
"""Get the metrics for each run in the collection."""
|
31
38
|
return [run.data.metrics for run in self._runs]
|
32
39
|
|
33
40
|
@property
|
34
41
|
def artifact_uri(self) -> list[str | None]:
|
42
|
+
"""Get the artifact URI for each run in the collection."""
|
35
43
|
return [run.info.artifact_uri for run in self._runs]
|
36
44
|
|
37
45
|
@property
|
38
46
|
def artifact_dir(self) -> list[Path]:
|
47
|
+
"""Get the artifact directory for each run in the collection."""
|
39
48
|
return [get_artifact_dir(run) for run in self._runs]
|
40
49
|
|
41
50
|
@property
|
42
51
|
def config(self) -> list[DictConfig]:
|
52
|
+
"""Get the configuration for each run in the collection."""
|
43
53
|
return [load_config(run) for run in self._runs]
|
44
54
|
|
45
55
|
|
46
56
|
def get_artifact_dir(run: Run | None = None) -> Path:
|
47
|
-
"""
|
48
|
-
Retrieve the artifact directory for the given run.
|
57
|
+
"""Retrieve the artifact directory for the given run.
|
49
58
|
|
50
59
|
This function uses MLflow to get the artifact directory for the given run.
|
51
60
|
|
@@ -54,6 +63,7 @@ def get_artifact_dir(run: Run | None = None) -> Path:
|
|
54
63
|
|
55
64
|
Returns:
|
56
65
|
The local path to the directory where the artifacts are downloaded.
|
66
|
+
|
57
67
|
"""
|
58
68
|
if run is None:
|
59
69
|
uri = mlflow.get_artifact_uri()
|
@@ -64,8 +74,7 @@ def get_artifact_dir(run: Run | None = None) -> Path:
|
|
64
74
|
|
65
75
|
|
66
76
|
def get_hydra_output_dir(run: Run | None = None) -> Path:
|
67
|
-
"""
|
68
|
-
Retrieve the Hydra output directory for the given run.
|
77
|
+
"""Retrieve the Hydra output directory for the given run.
|
69
78
|
|
70
79
|
This function returns the Hydra output directory. If no run is provided,
|
71
80
|
it retrieves the output directory from the current Hydra configuration.
|
@@ -82,6 +91,7 @@ def get_hydra_output_dir(run: Run | None = None) -> Path:
|
|
82
91
|
Raises:
|
83
92
|
FileNotFoundError: If the Hydra configuration file is not found
|
84
93
|
in the artifacts.
|
94
|
+
|
85
95
|
"""
|
86
96
|
if run is None:
|
87
97
|
hc = HydraConfig.get()
|
@@ -97,8 +107,7 @@ def get_hydra_output_dir(run: Run | None = None) -> Path:
|
|
97
107
|
|
98
108
|
|
99
109
|
def load_config(run: Run) -> DictConfig:
|
100
|
-
"""
|
101
|
-
Load the configuration for a given run.
|
110
|
+
"""Load the configuration for a given run.
|
102
111
|
|
103
112
|
This function loads the configuration for the provided Run instance
|
104
113
|
by downloading the configuration file from the MLflow artifacts and
|
@@ -111,6 +120,7 @@ def load_config(run: Run) -> DictConfig:
|
|
111
120
|
Returns:
|
112
121
|
The loaded configuration as a DictConfig object. Returns an empty
|
113
122
|
DictConfig if the configuration file is not found.
|
123
|
+
|
114
124
|
"""
|
115
125
|
path = get_artifact_dir(run) / ".hydra/config.yaml"
|
116
126
|
return OmegaConf.load(path) # type: ignore
|
hydraflow/mlflow.py
CHANGED
@@ -1,20 +1,17 @@
|
|
1
|
-
"""
|
2
|
-
|
3
|
-
|
4
|
-
|
1
|
+
"""Provide functionality to log parameters from Hydra configuration objects.
|
2
|
+
|
3
|
+
This module provides functions to log parameters from Hydra configuration objects
|
4
|
+
to MLflow, set experiments, and manage tracking URIs. It integrates Hydra's
|
5
|
+
configuration management with MLflow's experiment tracking capabilities.
|
5
6
|
|
6
7
|
Key Features:
|
7
|
-
- **Experiment Management**: Set
|
8
|
-
|
9
|
-
- **
|
10
|
-
|
11
|
-
- **Run
|
12
|
-
|
13
|
-
|
14
|
-
easy access to outputs generated during experiments.
|
15
|
-
|
16
|
-
This module is designed to integrate seamlessly with Hydra, providing a robust
|
17
|
-
solution for tracking machine learning experiments and their associated metadata.
|
8
|
+
- **Experiment Management**: Set experiment names and tracking URIs using Hydra
|
9
|
+
configuration details.
|
10
|
+
- **Parameter Logging**: Log parameters from Hydra configuration objects to MLflow,
|
11
|
+
supporting both synchronous and asynchronous logging.
|
12
|
+
- **Run Collection**: Utilize the `RunCollection` class to manage and interact with
|
13
|
+
multiple MLflow runs, providing methods to filter and retrieve runs based on
|
14
|
+
various criteria.
|
18
15
|
"""
|
19
16
|
|
20
17
|
from __future__ import annotations
|
@@ -40,8 +37,7 @@ def set_experiment(
|
|
40
37
|
suffix: str = "",
|
41
38
|
uri: str | Path | None = None,
|
42
39
|
) -> Experiment:
|
43
|
-
"""
|
44
|
-
Sets the experiment name and tracking URI optionally.
|
40
|
+
"""Set the experiment name and tracking URI optionally.
|
45
41
|
|
46
42
|
This function sets the experiment name by combining the given prefix,
|
47
43
|
the job name from HydraConfig, and the given suffix. Optionally, it can
|
@@ -55,6 +51,7 @@ def set_experiment(
|
|
55
51
|
Returns:
|
56
52
|
Experiment: An instance of `mlflow.entities.Experiment` representing
|
57
53
|
the new active experiment.
|
54
|
+
|
58
55
|
"""
|
59
56
|
if uri is not None:
|
60
57
|
mlflow.set_tracking_uri(uri)
|
@@ -65,8 +62,7 @@ def set_experiment(
|
|
65
62
|
|
66
63
|
|
67
64
|
def log_params(config: object, *, synchronous: bool | None = None) -> None:
|
68
|
-
"""
|
69
|
-
Log the parameters from the given configuration object.
|
65
|
+
"""Log the parameters from the given configuration object.
|
70
66
|
|
71
67
|
This method logs the parameters from the provided configuration object
|
72
68
|
using MLflow. It iterates over the parameters and logs them using the
|
@@ -76,6 +72,7 @@ def log_params(config: object, *, synchronous: bool | None = None) -> None:
|
|
76
72
|
config (object): The configuration object to log the parameters from.
|
77
73
|
synchronous (bool | None): Whether to log the parameters synchronously.
|
78
74
|
Defaults to None.
|
75
|
+
|
79
76
|
"""
|
80
77
|
for key, value in iter_params(config):
|
81
78
|
mlflow.log_param(key, value, synchronous=synchronous)
|
@@ -91,8 +88,7 @@ def search_runs( # noqa: PLR0913
|
|
91
88
|
search_all_experiments: bool = False,
|
92
89
|
experiment_names: list[str] | None = None,
|
93
90
|
) -> RunCollection:
|
94
|
-
"""
|
95
|
-
Search for Runs that fit the specified criteria.
|
91
|
+
"""Search for Runs that fit the specified criteria.
|
96
92
|
|
97
93
|
This function wraps the `mlflow.search_runs` function and returns the
|
98
94
|
results as a `RunCollection` object. It allows for flexible searching of
|
@@ -133,6 +129,7 @@ def search_runs( # noqa: PLR0913
|
|
133
129
|
|
134
130
|
Returns:
|
135
131
|
A `RunCollection` object containing the search results.
|
132
|
+
|
136
133
|
"""
|
137
134
|
runs = mlflow.search_runs(
|
138
135
|
experiment_ids=experiment_ids,
|
@@ -151,9 +148,9 @@ def search_runs( # noqa: PLR0913
|
|
151
148
|
def list_runs(
|
152
149
|
experiment_names: str | list[str] | None = None,
|
153
150
|
n_jobs: int = 0,
|
151
|
+
status: str | list[str] | int | list[int] | None = None,
|
154
152
|
) -> RunCollection:
|
155
|
-
"""
|
156
|
-
List all runs for the specified experiments.
|
153
|
+
"""List all runs for the specified experiments.
|
157
154
|
|
158
155
|
This function retrieves all runs for the given list of experiment names.
|
159
156
|
If no experiment names are provided (None), it defaults to searching all runs
|
@@ -169,11 +166,27 @@ def list_runs(
|
|
169
166
|
for runs. If None or an empty list is provided, the function will
|
170
167
|
search the currently active experiment or all experiments except
|
171
168
|
the "Default" experiment.
|
169
|
+
n_jobs (int): The number of jobs to run in parallel. If 0, the function
|
170
|
+
will search runs sequentially.
|
171
|
+
status (str | list[str] | int | list[int] | None): The status of the runs
|
172
|
+
to filter.
|
172
173
|
|
173
174
|
Returns:
|
174
175
|
RunCollection: A `RunCollection` instance containing the runs for the
|
175
176
|
specified experiments.
|
177
|
+
|
176
178
|
"""
|
179
|
+
rc = _list_runs(experiment_names, n_jobs)
|
180
|
+
if status is None:
|
181
|
+
return rc
|
182
|
+
|
183
|
+
return rc.filter(status=status)
|
184
|
+
|
185
|
+
|
186
|
+
def _list_runs(
|
187
|
+
experiment_names: str | list[str] | None = None,
|
188
|
+
n_jobs: int = 0,
|
189
|
+
) -> RunCollection:
|
177
190
|
if isinstance(experiment_names, str):
|
178
191
|
experiment_names = [experiment_names]
|
179
192
|
|
hydraflow/param.py
CHANGED
@@ -1,3 +1,13 @@
|
|
1
|
+
"""Provide utility functions for parameter matching.
|
2
|
+
|
3
|
+
The main function `match` checks if a given parameter matches a specified value.
|
4
|
+
It supports various types of values including None, boolean, list, tuple, int,
|
5
|
+
float, and str.
|
6
|
+
|
7
|
+
Helper functions `_match_list` and `_match_tuple` are used internally to handle
|
8
|
+
matching for list and tuple types respectively.
|
9
|
+
"""
|
10
|
+
|
1
11
|
from __future__ import annotations
|
2
12
|
|
3
13
|
from typing import Any
|
@@ -13,6 +23,7 @@ def match(param: str, value: Any) -> bool:
|
|
13
23
|
Returns:
|
14
24
|
True if the parameter matches the specified value,
|
15
25
|
False otherwise.
|
26
|
+
|
16
27
|
"""
|
17
28
|
if value in [None, True, False]:
|
18
29
|
return param == str(value)
|
hydraflow/progress.py
CHANGED
@@ -1,18 +1,7 @@
|
|
1
|
-
"""
|
2
|
-
Module for managing progress tracking in parallel processing using Joblib
|
3
|
-
and Rich's Progress bar.
|
1
|
+
"""Context managers and functions for parallel task execution with progress.
|
4
2
|
|
5
3
|
Provide context managers and functions to facilitate the execution
|
6
4
|
of tasks in parallel while displaying progress updates.
|
7
|
-
|
8
|
-
The following key components are provided:
|
9
|
-
|
10
|
-
- JoblibProgress: A context manager for tracking progress with Rich's progress
|
11
|
-
bar.
|
12
|
-
- parallel_progress: A function to execute a given function in parallel over
|
13
|
-
an iterable with progress tracking.
|
14
|
-
- multi_tasks_progress: A function to render auto-updating progress bars for
|
15
|
-
multiple tasks concurrently.
|
16
5
|
"""
|
17
6
|
|
18
7
|
from __future__ import annotations
|
@@ -37,8 +26,7 @@ def JoblibProgress( # noqa: N802
|
|
37
26
|
total: int | None = None,
|
38
27
|
**kwargs,
|
39
28
|
) -> Iterator[Progress]:
|
40
|
-
"""
|
41
|
-
Context manager for tracking progress using Joblib with Rich's Progress bar.
|
29
|
+
"""Context manager for tracking progress using Joblib with Rich's Progress bar.
|
42
30
|
|
43
31
|
Args:
|
44
32
|
*columns (ProgressColumn | str): Columns to display in the progress bar.
|
@@ -56,6 +44,7 @@ def JoblibProgress( # noqa: N802
|
|
56
44
|
with JoblibProgress("task", total=100) as progress:
|
57
45
|
# Your parallel processing code here
|
58
46
|
```
|
47
|
+
|
59
48
|
"""
|
60
49
|
if not columns:
|
61
50
|
columns = Progress.get_default_columns()
|
@@ -94,8 +83,7 @@ def parallel_progress(
|
|
94
83
|
description: str | None = None,
|
95
84
|
**kwargs,
|
96
85
|
) -> list[U]:
|
97
|
-
"""
|
98
|
-
Execute a function in parallel over an iterable with progress tracking.
|
86
|
+
"""Execute a function in parallel over an iterable with progress tracking.
|
99
87
|
|
100
88
|
Args:
|
101
89
|
func (Callable[[T], U]): The function to execute on each item in the
|
@@ -112,6 +100,7 @@ def parallel_progress(
|
|
112
100
|
Returns:
|
113
101
|
list[U]: A list of results from applying the function to each item in
|
114
102
|
the iterable.
|
103
|
+
|
115
104
|
"""
|
116
105
|
iterable = list(iterable)
|
117
106
|
total = len(iterable)
|
@@ -130,8 +119,7 @@ def multi_tasks_progress(
|
|
130
119
|
transient: bool | None = None,
|
131
120
|
**kwargs,
|
132
121
|
) -> None:
|
133
|
-
"""
|
134
|
-
Render auto-updating progress bars for multiple tasks concurrently.
|
122
|
+
"""Render auto-updating progress bars for multiple tasks concurrently.
|
135
123
|
|
136
124
|
Args:
|
137
125
|
iterables (Iterable[Iterable[int | tuple[int, int]]]): A collection of
|
@@ -151,6 +139,7 @@ def multi_tasks_progress(
|
|
151
139
|
|
152
140
|
Returns:
|
153
141
|
None
|
142
|
+
|
154
143
|
"""
|
155
144
|
if not columns:
|
156
145
|
columns = Progress.get_default_columns()
|
hydraflow/run_collection.py
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
-
"""
|
2
|
-
|
3
|
-
|
1
|
+
"""Provide a collection of MLflow runs.
|
2
|
+
|
3
|
+
This module includes the `RunCollection` class, which serves as a container
|
4
4
|
for multiple MLflow `Run` instances, and various methods to filter and
|
5
5
|
retrieve these runs.
|
6
6
|
|
@@ -23,6 +23,8 @@ from dataclasses import dataclass, field
|
|
23
23
|
from itertools import chain
|
24
24
|
from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar, overload
|
25
25
|
|
26
|
+
from mlflow.entities import RunStatus
|
27
|
+
|
26
28
|
import hydraflow.param
|
27
29
|
from hydraflow.config import iter_params
|
28
30
|
from hydraflow.info import RunCollectionInfo
|
@@ -42,8 +44,7 @@ P = ParamSpec("P")
|
|
42
44
|
|
43
45
|
@dataclass
|
44
46
|
class RunCollection:
|
45
|
-
"""
|
46
|
-
Represent a collection of MLflow runs.
|
47
|
+
"""Represent a collection of MLflow runs.
|
47
48
|
|
48
49
|
Provide methods to interact with the runs, such as filtering,
|
49
50
|
retrieving specific runs, and accessing run information.
|
@@ -93,7 +94,6 @@ class RunCollection:
|
|
93
94
|
@classmethod
|
94
95
|
def from_list(cls, runs: list[Run]) -> RunCollection:
|
95
96
|
"""Create a `RunCollection` instance from a list of MLflow `Run` instances."""
|
96
|
-
|
97
97
|
return cls(runs)
|
98
98
|
|
99
99
|
@property
|
@@ -114,6 +114,7 @@ class RunCollection:
|
|
114
114
|
Returns:
|
115
115
|
A new `RunCollection` instance containing the first n runs if n is
|
116
116
|
positive, or the last n runs if n is negative.
|
117
|
+
|
117
118
|
"""
|
118
119
|
if n < 0:
|
119
120
|
return self.__class__(self._runs[n:])
|
@@ -126,17 +127,28 @@ class RunCollection:
|
|
126
127
|
*,
|
127
128
|
reverse: bool = False,
|
128
129
|
) -> None:
|
130
|
+
"""Sort the runs in the collection.
|
131
|
+
|
132
|
+
Sort the runs in the collection according to the provided key function
|
133
|
+
and optional reverse flag.
|
134
|
+
|
135
|
+
Args:
|
136
|
+
key (Callable[[Run], Any] | None): A function that takes a run and returns
|
137
|
+
a value to sort by.
|
138
|
+
reverse (bool): If True, sort in descending order.
|
139
|
+
|
140
|
+
"""
|
129
141
|
self._runs.sort(key=key or (lambda x: x.info.start_time), reverse=reverse)
|
130
142
|
|
131
143
|
def one(self) -> Run:
|
132
|
-
"""
|
133
|
-
Get the only `Run` instance in the collection.
|
144
|
+
"""Get the only `Run` instance in the collection.
|
134
145
|
|
135
146
|
Returns:
|
136
147
|
The only `Run` instance in the collection.
|
137
148
|
|
138
149
|
Raises:
|
139
150
|
ValueError: If the collection does not contain exactly one run.
|
151
|
+
|
140
152
|
"""
|
141
153
|
if len(self._runs) != 1:
|
142
154
|
raise ValueError("The collection does not contain exactly one run.")
|
@@ -144,24 +156,24 @@ class RunCollection:
|
|
144
156
|
return self._runs[0]
|
145
157
|
|
146
158
|
def try_one(self) -> Run | None:
|
147
|
-
"""
|
148
|
-
Try to get the only `Run` instance in the collection.
|
159
|
+
"""Try to get the only `Run` instance in the collection.
|
149
160
|
|
150
161
|
Returns:
|
151
162
|
The only `Run` instance in the collection, or None if the collection
|
152
163
|
does not contain exactly one run.
|
164
|
+
|
153
165
|
"""
|
154
166
|
return self._runs[0] if len(self._runs) == 1 else None
|
155
167
|
|
156
168
|
def first(self) -> Run:
|
157
|
-
"""
|
158
|
-
Get the first `Run` instance in the collection.
|
169
|
+
"""Get the first `Run` instance in the collection.
|
159
170
|
|
160
171
|
Returns:
|
161
172
|
The first `Run` instance in the collection.
|
162
173
|
|
163
174
|
Raises:
|
164
175
|
ValueError: If the collection is empty.
|
176
|
+
|
165
177
|
"""
|
166
178
|
if not self._runs:
|
167
179
|
raise ValueError("The collection is empty.")
|
@@ -169,24 +181,24 @@ class RunCollection:
|
|
169
181
|
return self._runs[0]
|
170
182
|
|
171
183
|
def try_first(self) -> Run | None:
|
172
|
-
"""
|
173
|
-
Try to get the first `Run` instance in the collection.
|
184
|
+
"""Try to get the first `Run` instance in the collection.
|
174
185
|
|
175
186
|
Returns:
|
176
187
|
The first `Run` instance in the collection, or None if the collection
|
177
188
|
is empty.
|
189
|
+
|
178
190
|
"""
|
179
191
|
return self._runs[0] if self._runs else None
|
180
192
|
|
181
193
|
def last(self) -> Run:
|
182
|
-
"""
|
183
|
-
Get the last `Run` instance in the collection.
|
194
|
+
"""Get the last `Run` instance in the collection.
|
184
195
|
|
185
196
|
Returns:
|
186
197
|
The last `Run` instance in the collection.
|
187
198
|
|
188
199
|
Raises:
|
189
200
|
ValueError: If the collection is empty.
|
201
|
+
|
190
202
|
"""
|
191
203
|
if not self._runs:
|
192
204
|
raise ValueError("The collection is empty.")
|
@@ -194,18 +206,17 @@ class RunCollection:
|
|
194
206
|
return self._runs[-1]
|
195
207
|
|
196
208
|
def try_last(self) -> Run | None:
|
197
|
-
"""
|
198
|
-
Try to get the last `Run` instance in the collection.
|
209
|
+
"""Try to get the last `Run` instance in the collection.
|
199
210
|
|
200
211
|
Returns:
|
201
212
|
The last `Run` instance in the collection, or None if the collection
|
202
213
|
is empty.
|
214
|
+
|
203
215
|
"""
|
204
216
|
return self._runs[-1] if self._runs else None
|
205
217
|
|
206
218
|
def filter(self, config: object | None = None, **kwargs) -> RunCollection:
|
207
|
-
"""
|
208
|
-
Filter the `Run` instances based on the provided configuration.
|
219
|
+
"""Filter the `Run` instances based on the provided configuration.
|
209
220
|
|
210
221
|
This method filters the runs in the collection according to the
|
211
222
|
specified configuration object and additional key-value pairs. The
|
@@ -228,12 +239,12 @@ class RunCollection:
|
|
228
239
|
|
229
240
|
Returns:
|
230
241
|
A new `RunCollection` object containing the filtered runs.
|
242
|
+
|
231
243
|
"""
|
232
244
|
return RunCollection(filter_runs(self._runs, config, **kwargs))
|
233
245
|
|
234
246
|
def find(self, config: object | None = None, **kwargs) -> Run:
|
235
|
-
"""
|
236
|
-
Find the first `Run` instance based on the provided configuration.
|
247
|
+
"""Find the first `Run` instance based on the provided configuration.
|
237
248
|
|
238
249
|
This method filters the runs in the collection according to the
|
239
250
|
specified configuration object and returns the first run that matches
|
@@ -252,6 +263,7 @@ class RunCollection:
|
|
252
263
|
|
253
264
|
See Also:
|
254
265
|
`filter`: Perform the actual filtering logic.
|
266
|
+
|
255
267
|
"""
|
256
268
|
try:
|
257
269
|
return self.filter(config, **kwargs).first()
|
@@ -259,8 +271,7 @@ class RunCollection:
|
|
259
271
|
raise ValueError("No run matches the provided configuration.")
|
260
272
|
|
261
273
|
def try_find(self, config: object | None = None, **kwargs) -> Run | None:
|
262
|
-
"""
|
263
|
-
Try to find the first `Run` instance based on the provided configuration.
|
274
|
+
"""Try to find the first `Run` instance based on the provided configuration.
|
264
275
|
|
265
276
|
This method filters the runs in the collection according to the
|
266
277
|
specified configuration object and returns the first run that matches
|
@@ -277,12 +288,12 @@ class RunCollection:
|
|
277
288
|
|
278
289
|
See Also:
|
279
290
|
`filter`: Perform the actual filtering logic.
|
291
|
+
|
280
292
|
"""
|
281
293
|
return self.filter(config, **kwargs).try_first()
|
282
294
|
|
283
295
|
def find_last(self, config: object | None = None, **kwargs) -> Run:
|
284
|
-
"""
|
285
|
-
Find the last `Run` instance based on the provided configuration.
|
296
|
+
"""Find the last `Run` instance based on the provided configuration.
|
286
297
|
|
287
298
|
This method filters the runs in the collection according to the
|
288
299
|
specified configuration object and returns the last run that matches
|
@@ -301,6 +312,7 @@ class RunCollection:
|
|
301
312
|
|
302
313
|
See Also:
|
303
314
|
`filter`: Perform the actual filtering logic.
|
315
|
+
|
304
316
|
"""
|
305
317
|
try:
|
306
318
|
return self.filter(config, **kwargs).last()
|
@@ -308,8 +320,7 @@ class RunCollection:
|
|
308
320
|
raise ValueError("No run matches the provided configuration.")
|
309
321
|
|
310
322
|
def try_find_last(self, config: object | None = None, **kwargs) -> Run | None:
|
311
|
-
"""
|
312
|
-
Try to find the last `Run` instance based on the provided configuration.
|
323
|
+
"""Try to find the last `Run` instance based on the provided configuration.
|
313
324
|
|
314
325
|
This method filters the runs in the collection according to the
|
315
326
|
specified configuration object and returns the last run that matches
|
@@ -326,12 +337,12 @@ class RunCollection:
|
|
326
337
|
|
327
338
|
See Also:
|
328
339
|
`filter`: Perform the actual filtering logic.
|
340
|
+
|
329
341
|
"""
|
330
342
|
return self.filter(config, **kwargs).try_last()
|
331
343
|
|
332
344
|
def get(self, config: object | None = None, **kwargs) -> Run:
|
333
|
-
"""
|
334
|
-
Retrieve a specific `Run` instance based on the provided configuration.
|
345
|
+
"""Retrieve a specific `Run` instance based on the provided configuration.
|
335
346
|
|
336
347
|
This method filters the runs in the collection according to the
|
337
348
|
specified configuration object and returns the run that matches the
|
@@ -351,6 +362,7 @@ class RunCollection:
|
|
351
362
|
|
352
363
|
See Also:
|
353
364
|
`filter`: Perform the actual filtering logic.
|
365
|
+
|
354
366
|
"""
|
355
367
|
try:
|
356
368
|
return self.filter(config, **kwargs).one()
|
@@ -359,8 +371,7 @@ class RunCollection:
|
|
359
371
|
raise ValueError(msg)
|
360
372
|
|
361
373
|
def try_get(self, config: object | None = None, **kwargs) -> Run | None:
|
362
|
-
"""
|
363
|
-
Try to retrieve a specific `Run` instance based on the provided configuration.
|
374
|
+
"""Try to retrieve a specific `Run` instance based on the provided config.
|
364
375
|
|
365
376
|
This method filters the runs in the collection according to the
|
366
377
|
specified configuration object and returns the run that matches the
|
@@ -380,12 +391,12 @@ class RunCollection:
|
|
380
391
|
|
381
392
|
See Also:
|
382
393
|
`filter`: Perform the actual filtering logic.
|
394
|
+
|
383
395
|
"""
|
384
396
|
return self.filter(config, **kwargs).try_one()
|
385
397
|
|
386
398
|
def get_param_names(self) -> list[str]:
|
387
|
-
"""
|
388
|
-
Get the parameter names from the runs.
|
399
|
+
"""Get the parameter names from the runs.
|
389
400
|
|
390
401
|
This method extracts the unique parameter names from the provided list
|
391
402
|
of runs. It iterates through each run and collects the parameter names
|
@@ -393,6 +404,7 @@ class RunCollection:
|
|
393
404
|
|
394
405
|
Returns:
|
395
406
|
A list of unique parameter names.
|
407
|
+
|
396
408
|
"""
|
397
409
|
param_names = set()
|
398
410
|
|
@@ -402,24 +414,30 @@ class RunCollection:
|
|
402
414
|
|
403
415
|
return list(param_names)
|
404
416
|
|
405
|
-
def get_param_dict(self) -> dict[str, list[str]]:
|
406
|
-
"""
|
407
|
-
Get the parameter dictionary from the list of runs.
|
417
|
+
def get_param_dict(self, *, drop_const: bool = False) -> dict[str, list[str]]:
|
418
|
+
"""Get the parameter dictionary from the list of runs.
|
408
419
|
|
409
420
|
This method extracts the parameter names and their corresponding values
|
410
421
|
from the provided list of runs. It iterates through each run and
|
411
422
|
collects the parameter values into a dictionary where the keys are
|
412
423
|
parameter names and the values are lists of parameter values.
|
413
424
|
|
425
|
+
Args:
|
426
|
+
drop_const (bool): If True, drop the parameter values that are constant
|
427
|
+
across all runs.
|
428
|
+
|
414
429
|
Returns:
|
415
430
|
A dictionary where the keys are parameter names and the values are
|
416
431
|
lists of parameter values.
|
432
|
+
|
417
433
|
"""
|
418
434
|
params = {}
|
419
435
|
|
420
436
|
for name in self.get_param_names():
|
421
437
|
it = (run.data.params[name] for run in self if name in run.data.params)
|
422
|
-
|
438
|
+
unique_values = sorted(set(it))
|
439
|
+
if not drop_const or len(unique_values) > 1:
|
440
|
+
params[name] = unique_values
|
423
441
|
|
424
442
|
return params
|
425
443
|
|
@@ -429,9 +447,7 @@ class RunCollection:
|
|
429
447
|
*args: P.args,
|
430
448
|
**kwargs: P.kwargs,
|
431
449
|
) -> Iterator[T]:
|
432
|
-
"""
|
433
|
-
Apply a function to each run in the collection and return an iterator of
|
434
|
-
results.
|
450
|
+
"""Return an iterator of results by applying a function to each run.
|
435
451
|
|
436
452
|
This method iterates over each run in the collection and applies the
|
437
453
|
provided function to it, along with any additional arguments and
|
@@ -445,6 +461,7 @@ class RunCollection:
|
|
445
461
|
|
446
462
|
Yields:
|
447
463
|
Results obtained by applying the function to each run in the collection.
|
464
|
+
|
448
465
|
"""
|
449
466
|
return (func(run, *args, **kwargs) for run in self)
|
450
467
|
|
@@ -454,9 +471,7 @@ class RunCollection:
|
|
454
471
|
*args: P.args,
|
455
472
|
**kwargs: P.kwargs,
|
456
473
|
) -> Iterator[T]:
|
457
|
-
"""
|
458
|
-
Apply a function to each run id in the collection and return an iterator
|
459
|
-
of results.
|
474
|
+
"""Return an iterator of results by applying a function to each run id.
|
460
475
|
|
461
476
|
Args:
|
462
477
|
func (Callable[[str, P], T]): A function that takes a run id and returns a
|
@@ -467,6 +482,7 @@ class RunCollection:
|
|
467
482
|
Yields:
|
468
483
|
Results obtained by applying the function to each run id in the
|
469
484
|
collection.
|
485
|
+
|
470
486
|
"""
|
471
487
|
return (func(run_id, *args, **kwargs) for run_id in self.info.run_id)
|
472
488
|
|
@@ -476,9 +492,7 @@ class RunCollection:
|
|
476
492
|
*args: P.args,
|
477
493
|
**kwargs: P.kwargs,
|
478
494
|
) -> Iterator[T]:
|
479
|
-
"""
|
480
|
-
Apply a function to each run configuration in the collection and return
|
481
|
-
an iterator of results.
|
495
|
+
"""Return an iterator of results by applying a function to each run config.
|
482
496
|
|
483
497
|
Args:
|
484
498
|
func (Callable[[DictConfig, P], T]): A function that takes a run
|
@@ -489,6 +503,7 @@ class RunCollection:
|
|
489
503
|
Yields:
|
490
504
|
Results obtained by applying the function to each run configuration
|
491
505
|
in the collection.
|
506
|
+
|
492
507
|
"""
|
493
508
|
return (func(config, *args, **kwargs) for config in self.info.config)
|
494
509
|
|
@@ -498,9 +513,7 @@ class RunCollection:
|
|
498
513
|
*args: P.args,
|
499
514
|
**kwargs: P.kwargs,
|
500
515
|
) -> Iterator[T]:
|
501
|
-
"""
|
502
|
-
Apply a function to each artifact URI in the collection and return an
|
503
|
-
iterator of results.
|
516
|
+
"""Return an iterator of results by applying a function to each artifact URI.
|
504
517
|
|
505
518
|
Iterate over each run in the collection, retrieves the artifact URI, and
|
506
519
|
apply the provided function to it. If a run does not have an artifact
|
@@ -515,6 +528,7 @@ class RunCollection:
|
|
515
528
|
Yields:
|
516
529
|
Results obtained by applying the function to each artifact URI in the
|
517
530
|
collection.
|
531
|
+
|
518
532
|
"""
|
519
533
|
return (func(uri, *args, **kwargs) for uri in self.info.artifact_uri)
|
520
534
|
|
@@ -524,9 +538,7 @@ class RunCollection:
|
|
524
538
|
*args: P.args,
|
525
539
|
**kwargs: P.kwargs,
|
526
540
|
) -> Iterator[T]:
|
527
|
-
"""
|
528
|
-
Apply a function to each artifact directory in the collection and return
|
529
|
-
an iterator of results.
|
541
|
+
"""Return an iterator of results by applying a function to each artifact dir.
|
530
542
|
|
531
543
|
Iterate over each run in the collection, downloads the artifact
|
532
544
|
directory, and apply the provided function to the directory path.
|
@@ -540,6 +552,7 @@ class RunCollection:
|
|
540
552
|
Yields:
|
541
553
|
Results obtained by applying the function to each artifact directory
|
542
554
|
in the collection.
|
555
|
+
|
543
556
|
"""
|
544
557
|
return (func(dir, *args, **kwargs) for dir in self.info.artifact_dir) # noqa: A001
|
545
558
|
|
@@ -547,8 +560,7 @@ class RunCollection:
|
|
547
560
|
self,
|
548
561
|
*names: str | list[str],
|
549
562
|
) -> dict[tuple[str | None, ...], RunCollection]:
|
550
|
-
"""
|
551
|
-
Group runs by specified parameter names.
|
563
|
+
"""Group runs by specified parameter names.
|
552
564
|
|
553
565
|
Group the runs in the collection based on the values of the
|
554
566
|
specified parameters. Each unique combination of parameter values will
|
@@ -563,6 +575,7 @@ class RunCollection:
|
|
563
575
|
dict[tuple[str | None, ...], RunCollection]: A dictionary where the keys
|
564
576
|
are tuples of parameter values and the values are RunCollection objects
|
565
577
|
containing the runs that match those parameter values.
|
578
|
+
|
566
579
|
"""
|
567
580
|
grouped_runs: dict[tuple[str | None, ...], list[Run]] = {}
|
568
581
|
for run in self._runs:
|
@@ -588,11 +601,10 @@ def filter_runs(
|
|
588
601
|
runs: list[Run],
|
589
602
|
config: object | None = None,
|
590
603
|
*,
|
591
|
-
status: str | list[str] | None = None,
|
604
|
+
status: str | list[str] | int | list[int] | None = None,
|
592
605
|
**kwargs,
|
593
606
|
) -> list[Run]:
|
594
|
-
"""
|
595
|
-
Filter the runs based on the provided configuration.
|
607
|
+
"""Filter the runs based on the provided configuration.
|
596
608
|
|
597
609
|
Filter the runs in the collection according to the
|
598
610
|
specified configuration object and additional key-value pairs.
|
@@ -612,33 +624,61 @@ def filter_runs(
|
|
612
624
|
config (object | None): The configuration object to filter the runs.
|
613
625
|
This can be any object that provides key-value pairs through the
|
614
626
|
`iter_params` function.
|
615
|
-
status (str | list[str] | None): The status of
|
627
|
+
status (str | list[str] | RunStatus | list[RunStatus] | None): The status of
|
628
|
+
the runs to filter.
|
616
629
|
**kwargs: Additional key-value pairs to filter the runs.
|
617
630
|
|
618
631
|
Returns:
|
619
632
|
A list of runs that match the specified configuration and key-value pairs.
|
633
|
+
|
620
634
|
"""
|
621
635
|
for key, value in chain(iter_params(config), kwargs.items()):
|
622
636
|
runs = [run for run in runs if _param_matches(run, key, value)]
|
623
637
|
|
624
|
-
|
625
|
-
|
638
|
+
if len(runs) == 0 or status is None:
|
639
|
+
return runs
|
626
640
|
|
627
|
-
|
628
|
-
status = status[1:].lower()
|
629
|
-
return [run for run in runs if run.info.status.lower() != status]
|
641
|
+
return filter_runs_by_status(runs, status)
|
630
642
|
|
631
|
-
if status:
|
632
|
-
status = [status] if isinstance(status, str) else status
|
633
|
-
status = [s.lower() for s in status]
|
634
|
-
return [run for run in runs if run.info.status.lower() in status]
|
635
643
|
|
636
|
-
|
644
|
+
def filter_runs_by_status(
|
645
|
+
runs: list[Run],
|
646
|
+
status: str | list[str] | int | list[int],
|
647
|
+
) -> list[Run]:
|
648
|
+
"""Filter the runs based on the provided status.
|
637
649
|
|
650
|
+
Args:
|
651
|
+
runs (list[Run]): The list of runs to filter.
|
652
|
+
status (str | list[str] | int | list[int]): The status of the runs
|
653
|
+
to filter.
|
654
|
+
|
655
|
+
Returns:
|
656
|
+
A list of runs that match the specified status.
|
638
657
|
|
639
|
-
def get_params(run: Run, *names: str | list[str]) -> tuple[str | None, ...]:
|
640
658
|
"""
|
641
|
-
|
659
|
+
if isinstance(status, str):
|
660
|
+
if status.startswith("!"):
|
661
|
+
status = status[1:].lower()
|
662
|
+
return [run for run in runs if run.info.status.lower() != status]
|
663
|
+
|
664
|
+
status = [status]
|
665
|
+
|
666
|
+
elif isinstance(status, int):
|
667
|
+
status = [RunStatus.to_string(status)]
|
668
|
+
|
669
|
+
status = [_to_lower(s) for s in status]
|
670
|
+
return [run for run in runs if run.info.status.lower() in status]
|
671
|
+
|
672
|
+
|
673
|
+
def _to_lower(status: str | int) -> str:
|
674
|
+
if isinstance(status, str):
|
675
|
+
return status.lower()
|
676
|
+
|
677
|
+
return RunStatus.to_string(status).lower()
|
678
|
+
|
679
|
+
|
680
|
+
def get_params(run: Run, *names: str | list[str]) -> tuple[str | None, ...]:
|
681
|
+
"""Retrieve the values of specified parameters from the given run.
|
642
682
|
|
643
683
|
This function extracts the values of the parameters identified by the
|
644
684
|
provided names from the specified run. It can accept both individual
|
@@ -653,6 +693,7 @@ def get_params(run: Run, *names: str | list[str]) -> tuple[str | None, ...]:
|
|
653
693
|
Returns:
|
654
694
|
tuple[str | None, ...]: A tuple containing the values of the specified
|
655
695
|
parameters in the order they were provided.
|
696
|
+
|
656
697
|
"""
|
657
698
|
names_ = []
|
658
699
|
for name in names:
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.3
|
2
2
|
Name: hydraflow
|
3
|
-
Version: 0.2.
|
3
|
+
Version: 0.2.18
|
4
4
|
Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
|
5
5
|
Project-URL: Documentation, https://github.com/daizutabi/hydraflow
|
6
6
|
Project-URL: Source, https://github.com/daizutabi/hydraflow
|
@@ -0,0 +1,14 @@
|
|
1
|
+
hydraflow/__init__.py,sha256=B7rWSiGP5WwWjijcb41Bv9uuo5MQ6gbBbVWGAWYtK-k,598
|
2
|
+
hydraflow/asyncio.py,sha256=-i1C8KAmNDImrjHnk92Csaa1mpjdK8Vp4ZVaQV-l94s,6634
|
3
|
+
hydraflow/config.py,sha256=sBaEYPMAGSIOc_wdDsWm0k4y3AZyWIET8gqa_o95SDA,2089
|
4
|
+
hydraflow/context.py,sha256=ih_jnexaHoToNq1dZ6sBzhJWFluPiQluOlYTYOzNEgk,8222
|
5
|
+
hydraflow/info.py,sha256=Vzyz9dEWcU9ovRG3JWshxIazzod1cZoHF74bHhHL3AI,3946
|
6
|
+
hydraflow/mlflow.py,sha256=GkOr_pXfpfY5USYBLrCigHcP13VgrAK_e9kheR1Wke4,8579
|
7
|
+
hydraflow/param.py,sha256=dvIXcKgc_MPiju3WEk9qz5FOUeK5qSj-YWN2ophCpUM,1938
|
8
|
+
hydraflow/progress.py,sha256=zvKX1HCN8_xDOsgYOEcLLhkhdPdep-U8vHrc0XZ-6SQ,6163
|
9
|
+
hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
10
|
+
hydraflow/run_collection.py,sha256=gsseBQ6a2YolNanISgEgkjei7o9U6ZGV-Tk50UYH850,24295
|
11
|
+
hydraflow-0.2.18.dist-info/METADATA,sha256=roL3lGtlIibF6rHbCp4aXrCphhq-OkNe0JwLxM1xtBY,3819
|
12
|
+
hydraflow-0.2.18.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
|
13
|
+
hydraflow-0.2.18.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
|
14
|
+
hydraflow-0.2.18.dist-info/RECORD,,
|
@@ -1,14 +0,0 @@
|
|
1
|
-
hydraflow/__init__.py,sha256=B7rWSiGP5WwWjijcb41Bv9uuo5MQ6gbBbVWGAWYtK-k,598
|
2
|
-
hydraflow/asyncio.py,sha256=eFnDbNOQ5Hmjdforr8rTW6i_rr-zFIVY3xSQQ45gMPA,6511
|
3
|
-
hydraflow/config.py,sha256=YU6xYLinxq-Iqw1R3Zy7s3_u8nfpvnvXlGIkPXJTNLc,2116
|
4
|
-
hydraflow/context.py,sha256=4UDaWGoVmeF36UqsKoh6dd_cS_YVRfz80gFr28ouNlo,8040
|
5
|
-
hydraflow/info.py,sha256=7EsCMEH6LJZB3FZiQ3IpPFTD3Meaz7G3M-HvDQeo1rw,3466
|
6
|
-
hydraflow/mlflow.py,sha256=irD1INrVaI_1RIzUCjI36voBqgZszZ4dkSLo4aT1_FM,8271
|
7
|
-
hydraflow/param.py,sha256=W71zJH39s8cJcy3qV-PFQHJYyQnfa1GbnHOIqCMG3Jc,1573
|
8
|
-
hydraflow/progress.py,sha256=b5LvLm3d0eW3WsaidZAZotJNTTN3OwSY3XwxXXsJV9A,6561
|
9
|
-
hydraflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
10
|
-
hydraflow/run_collection.py,sha256=ym3M5ApEZVwJ1rYgOs4aYluTBfJeOECD6Z9SLFhv5O8,23260
|
11
|
-
hydraflow-0.2.17.dist-info/METADATA,sha256=uD6q000C_h2JsuFh0mkf1YmpTYxVDI1RLaAUKzZ6fDw,3819
|
12
|
-
hydraflow-0.2.17.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
|
13
|
-
hydraflow-0.2.17.dist-info/licenses/LICENSE,sha256=IGdDrBPqz1O0v_UwCW-NJlbX9Hy9b3uJ11t28y2srmY,1062
|
14
|
-
hydraflow-0.2.17.dist-info/RECORD,,
|
File without changes
|
File without changes
|