hydraflow 0.4.1__tar.gz → 0.4.2__tar.gz
Sign up to get free protection for your applications and to get access to all the features.
- hydraflow-0.4.2/.devcontainer/devcontainer.json +18 -0
- hydraflow-0.4.2/.devcontainer/postCreate.sh +10 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/PKG-INFO +1 -1
- {hydraflow-0.4.1 → hydraflow-0.4.2}/pyproject.toml +1 -1
- {hydraflow-0.4.1 → hydraflow-0.4.2}/src/hydraflow/context.py +1 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/src/hydraflow/mlflow.py +1 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/src/hydraflow/param.py +4 -4
- {hydraflow-0.4.1 → hydraflow-0.4.2}/src/hydraflow/utils.py +1 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/tests/scripts/app.py +13 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/tests/test_app.py +19 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/tests/test_asyncio.py +55 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/tests/test_param.py +16 -0
- hydraflow-0.4.1/.devcontainer/devcontainer.json +0 -17
- hydraflow-0.4.1/.devcontainer/postCreate.sh +0 -5
- {hydraflow-0.4.1 → hydraflow-0.4.2}/.devcontainer/starship.toml +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/.gitattributes +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/.gitignore +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/LICENSE +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/README.md +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/apps/quickstart.py +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/mkdocs.yml +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/src/hydraflow/__init__.py +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/src/hydraflow/asyncio.py +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/src/hydraflow/config.py +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/src/hydraflow/progress.py +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/src/hydraflow/py.typed +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/src/hydraflow/run_collection.py +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/src/hydraflow/run_data.py +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/src/hydraflow/run_info.py +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/tests/__init__.py +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/tests/integ/__init__.py +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/tests/integ/app.py +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/tests/scripts/__init__.py +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/tests/scripts/progress.py +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/tests/scripts/watch.py +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/tests/test_config.py +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/tests/test_context.py +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/tests/test_log_run.py +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/tests/test_mlflow.py +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/tests/test_progress.py +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/tests/test_run_collection.py +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/tests/test_run_data.py +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/tests/test_run_info.py +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/tests/test_version.py +0 -0
- {hydraflow-0.4.1 → hydraflow-0.4.2}/tests/test_watch.py +0 -0
@@ -0,0 +1,18 @@
|
|
1
|
+
{
|
2
|
+
"image": "mcr.microsoft.com/vscode/devcontainers/base:ubuntu24.04",
|
3
|
+
"features": {
|
4
|
+
"ghcr.io/devcontainers-contrib/features/starship:1": {}
|
5
|
+
},
|
6
|
+
"customizations": {
|
7
|
+
"vscode": {
|
8
|
+
"extensions": [
|
9
|
+
"charliermarsh.ruff",
|
10
|
+
"fill-labs.dependi",
|
11
|
+
"ms-python.python",
|
12
|
+
"ms-python.vscode-pylance",
|
13
|
+
"tamasfe.even-better-toml"
|
14
|
+
]
|
15
|
+
}
|
16
|
+
},
|
17
|
+
"postCreateCommand": ".devcontainer/postCreate.sh"
|
18
|
+
}
|
@@ -0,0 +1,10 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
|
3
|
+
echo 'eval "$(starship init bash)"' >> ~/.bashrc
|
4
|
+
echo "alias ll='ls -alF'" >> ~/.bashrc
|
5
|
+
mkdir -p ~/.config
|
6
|
+
cp .devcontainer/starship.toml ~/.config
|
7
|
+
|
8
|
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
9
|
+
source $HOME/.cargo/env
|
10
|
+
echo 'eval "$(uv generate-shell-completion bash)"' >> ~/.bashrc
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.3
|
2
2
|
Name: hydraflow
|
3
|
-
Version: 0.4.
|
3
|
+
Version: 0.4.2
|
4
4
|
Summary: Hydraflow integrates Hydra and MLflow to manage and track machine learning experiments.
|
5
5
|
Project-URL: Documentation, https://github.com/daizutabi/hydraflow
|
6
6
|
Project-URL: Source, https://github.com/daizutabi/hydraflow
|
@@ -10,6 +10,7 @@ from pathlib import Path
|
|
10
10
|
from typing import TYPE_CHECKING
|
11
11
|
|
12
12
|
import mlflow
|
13
|
+
import mlflow.artifacts
|
13
14
|
from hydra.core.hydra_config import HydraConfig
|
14
15
|
from watchdog.events import FileModifiedEvent, PatternMatchingEventHandler
|
15
16
|
from watchdog.observers import Observer
|
@@ -21,6 +21,7 @@ from typing import TYPE_CHECKING
|
|
21
21
|
|
22
22
|
import joblib
|
23
23
|
import mlflow
|
24
|
+
import mlflow.artifacts
|
24
25
|
from hydra.core.hydra_config import HydraConfig
|
25
26
|
from mlflow.entities import ViewType
|
26
27
|
from mlflow.tracking.fluent import SEARCH_MAX_RESULTS_PANDAS, _get_experiment_id
|
@@ -30,7 +30,7 @@ def match(param: str, value: Any) -> bool:
|
|
30
30
|
False otherwise.
|
31
31
|
|
32
32
|
"""
|
33
|
-
if value in [None, True, False]:
|
33
|
+
if any(value is x for x in [None, True, False]):
|
34
34
|
return param == str(value)
|
35
35
|
|
36
36
|
if isinstance(value, list) and (m := _match_list(param, value)) is not None:
|
@@ -39,12 +39,12 @@ def match(param: str, value: Any) -> bool:
|
|
39
39
|
if isinstance(value, tuple) and (m := _match_tuple(param, value)) is not None:
|
40
40
|
return m
|
41
41
|
|
42
|
+
if isinstance(value, int | float):
|
43
|
+
return float(param) == value
|
44
|
+
|
42
45
|
if isinstance(value, str):
|
43
46
|
return param == value
|
44
47
|
|
45
|
-
if isinstance(value, int | float):
|
46
|
-
return type(value)(param) == value
|
47
|
-
|
48
48
|
return param == str(value)
|
49
49
|
|
50
50
|
|
@@ -30,6 +30,16 @@ def app(cfg: MySQLConfig):
|
|
30
30
|
with hydraflow.chdir_hydra_output() as path:
|
31
31
|
Path("chdir_hydra.txt").write_text(path.as_posix())
|
32
32
|
|
33
|
+
o = hydraflow.select_overrides(cfg)
|
34
|
+
if "host" in o:
|
35
|
+
assert o["host"] == cfg.host
|
36
|
+
|
37
|
+
if "port" not in o:
|
38
|
+
assert cfg.port == 3306
|
39
|
+
|
40
|
+
if "values" not in o:
|
41
|
+
assert cfg.get("values") == [1, 2, 3] # type: ignore
|
42
|
+
|
33
43
|
hydraflow.set_experiment(prefix="_", suffix="_")
|
34
44
|
with hydraflow.start_run(cfg) as run:
|
35
45
|
log.info(f"START, {cfg.host}, {cfg.port} ")
|
@@ -52,6 +62,9 @@ def app(cfg: MySQLConfig):
|
|
52
62
|
|
53
63
|
assert hydraflow.get_overrides() == hydraflow.load_overrides(run)
|
54
64
|
|
65
|
+
if cfg.host == "error":
|
66
|
+
raise Exception("error")
|
67
|
+
|
55
68
|
log.info("END")
|
56
69
|
|
57
70
|
|
@@ -194,3 +194,22 @@ def test_sort_by(rc: RunCollection):
|
|
194
194
|
|
195
195
|
sorted = rc.sort_by(["host", "port"], reverse=True)
|
196
196
|
assert sorted.values(["host", "port"]) == [("y", 2), ("y", 1), ("x", 2), ("x", 1)]
|
197
|
+
|
198
|
+
|
199
|
+
def test_log_run_error(monkeypatch, tmp_path):
|
200
|
+
file = Path("tests/scripts/app.py").absolute()
|
201
|
+
monkeypatch.chdir(tmp_path)
|
202
|
+
|
203
|
+
args = [sys.executable, file.as_posix()]
|
204
|
+
args += ["host=error", "hydra.job.name=error"]
|
205
|
+
cp = subprocess.run(args, check=False, capture_output=True)
|
206
|
+
assert cp.returncode == 1
|
207
|
+
assert b"Error during log_run: error" in cp.stdout
|
208
|
+
|
209
|
+
|
210
|
+
def test_chdir_artifact(rc: RunCollection):
|
211
|
+
from hydraflow.context import chdir_artifact
|
212
|
+
|
213
|
+
with chdir_artifact(rc[0]):
|
214
|
+
assert Path.cwd().stem == "artifacts"
|
215
|
+
assert Path.cwd().parent.stem == rc[0].info.run_id
|
@@ -164,3 +164,58 @@ def test_run(tmp_path: Path):
|
|
164
164
|
assert stderr_lines == ["world"]
|
165
165
|
assert Path(path).read_text() == "hello world"
|
166
166
|
assert len(changes_detected) >= 2
|
167
|
+
|
168
|
+
|
169
|
+
@pytest.mark.asyncio
|
170
|
+
async def test_execute_command_nonexistent():
|
171
|
+
from hydraflow.asyncio import execute_command
|
172
|
+
|
173
|
+
stop_event = asyncio.Event()
|
174
|
+
|
175
|
+
rc = await execute_command("nonexistent_command", stop_event=stop_event)
|
176
|
+
assert rc == 1
|
177
|
+
assert stop_event.is_set()
|
178
|
+
|
179
|
+
|
180
|
+
@pytest.mark.asyncio
|
181
|
+
async def test_process_stream_none():
|
182
|
+
from hydraflow.asyncio import process_stream
|
183
|
+
|
184
|
+
assert await process_stream(None, None) is None
|
185
|
+
|
186
|
+
|
187
|
+
@pytest.mark.asyncio
|
188
|
+
async def test_monitor_file_changes_error():
|
189
|
+
from hydraflow.asyncio import monitor_file_changes
|
190
|
+
|
191
|
+
stop_event = asyncio.Event()
|
192
|
+
|
193
|
+
with pytest.raises(FileNotFoundError):
|
194
|
+
await monitor_file_changes(["nonexistent_path"], lambda _: None, stop_event)
|
195
|
+
|
196
|
+
|
197
|
+
@pytest.mark.asyncio
|
198
|
+
async def test_run_and_monitor_none():
|
199
|
+
from hydraflow.asyncio import run_and_monitor
|
200
|
+
|
201
|
+
assert await run_and_monitor("echo", "hello") == 0
|
202
|
+
|
203
|
+
|
204
|
+
@pytest.mark.asyncio
|
205
|
+
async def test_run_and_monitor_error():
|
206
|
+
from hydraflow.asyncio import run_and_monitor
|
207
|
+
|
208
|
+
with pytest.raises(FileNotFoundError):
|
209
|
+
await run_and_monitor(
|
210
|
+
"nonexistent_command",
|
211
|
+
watch=lambda _: None,
|
212
|
+
paths=["nonexistent_path"],
|
213
|
+
)
|
214
|
+
|
215
|
+
|
216
|
+
def test_run_cwd():
|
217
|
+
from hydraflow.asyncio import run
|
218
|
+
|
219
|
+
return_code = run(sys.executable, "--version", watch=lambda _: None)
|
220
|
+
|
221
|
+
assert return_code == 0
|
@@ -49,6 +49,22 @@ def test_param(param, x, y):
|
|
49
49
|
assert match(p, x)
|
50
50
|
|
51
51
|
|
52
|
+
def test_param_float():
|
53
|
+
from hydraflow.param import match
|
54
|
+
|
55
|
+
assert match("1.0", 1.0)
|
56
|
+
assert match("1.0", 1)
|
57
|
+
assert match("0.0", 0)
|
58
|
+
assert match("0.0", 0.0)
|
59
|
+
|
60
|
+
|
61
|
+
def test_param_bool():
|
62
|
+
from hydraflow.param import match
|
63
|
+
|
64
|
+
assert not match("1", True)
|
65
|
+
assert not match("0", False)
|
66
|
+
|
67
|
+
|
52
68
|
def test_match_list():
|
53
69
|
from hydraflow.param import _match_list
|
54
70
|
|
@@ -1,17 +0,0 @@
|
|
1
|
-
{
|
2
|
-
"image": "mcr.microsoft.com/vscode/devcontainers/python:3.12",
|
3
|
-
"features": {
|
4
|
-
"ghcr.io/devcontainers-contrib/features/starship:1": {},
|
5
|
-
"ghcr.io/va-h/devcontainers-features/uv:1": {}
|
6
|
-
},
|
7
|
-
"customizations": {
|
8
|
-
"vscode": {
|
9
|
-
"extensions": [
|
10
|
-
"charliermarsh.ruff",
|
11
|
-
"ms-python.python",
|
12
|
-
"ms-python.vscode-pylance"
|
13
|
-
]
|
14
|
-
}
|
15
|
-
},
|
16
|
-
"postCreateCommand": ".devcontainer/postCreate.sh"
|
17
|
-
}
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|