furu 0.0.5__py3-none-any.whl → 0.0.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- furu/__init__.py +3 -15
- furu/aliases.py +53 -0
- furu/core/furu.py +162 -5
- furu/dashboard/api/models.py +18 -4
- furu/dashboard/api/routes.py +8 -0
- furu/dashboard/frontend/dist/assets/{index-BjyrY-Zz.js → index-NiDdQnqO.js} +15 -15
- furu/dashboard/frontend/dist/index.html +1 -1
- furu/dashboard/scanner.py +173 -147
- furu/migration.py +491 -763
- furu/runtime/overrides.py +37 -0
- furu/schema.py +46 -0
- furu/storage/metadata.py +19 -2
- furu/testing.py +232 -0
- {furu-0.0.5.dist-info → furu-0.0.7.dist-info}/METADATA +84 -1
- {furu-0.0.5.dist-info → furu-0.0.7.dist-info}/RECORD +17 -14
- {furu-0.0.5.dist-info → furu-0.0.7.dist-info}/WHEEL +1 -1
- furu/migrate.py +0 -48
- {furu-0.0.5.dist-info → furu-0.0.7.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
from collections.abc import Generator, Mapping
|
|
2
|
+
from contextlib import contextmanager
|
|
3
|
+
from contextvars import ContextVar
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
OverrideValue = Any
|
|
8
|
+
|
|
9
|
+
_OVERRIDES: ContextVar[dict[str, OverrideValue]] = ContextVar(
|
|
10
|
+
"FURU_RESULT_OVERRIDES",
|
|
11
|
+
default={},
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def has_override(furu_hash: str) -> bool:
|
|
16
|
+
return furu_hash in _OVERRIDES.get()
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def lookup_override(furu_hash: str) -> tuple[bool, OverrideValue]:
|
|
20
|
+
overrides = _OVERRIDES.get()
|
|
21
|
+
if furu_hash in overrides:
|
|
22
|
+
return True, overrides[furu_hash]
|
|
23
|
+
return False, None
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@contextmanager
|
|
27
|
+
def override_furu_hashes(
|
|
28
|
+
overrides: Mapping[str, OverrideValue],
|
|
29
|
+
) -> Generator[None, None, None]:
|
|
30
|
+
current = _OVERRIDES.get()
|
|
31
|
+
merged = dict(current)
|
|
32
|
+
merged.update(overrides)
|
|
33
|
+
token = _OVERRIDES.set(merged)
|
|
34
|
+
try:
|
|
35
|
+
yield
|
|
36
|
+
finally:
|
|
37
|
+
_OVERRIDES.reset(token)
|
furu/schema.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import chz
|
|
4
|
+
|
|
5
|
+
from .serialization.serializer import JsonValue
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def schema_key_from_furu_obj(furu_obj: dict[str, JsonValue]) -> tuple[str, ...]:
|
|
9
|
+
if not isinstance(furu_obj, dict):
|
|
10
|
+
raise TypeError(f"schema_key requires dict furu_obj, got {type(furu_obj)}")
|
|
11
|
+
keys: set[str] = set()
|
|
12
|
+
for key in furu_obj:
|
|
13
|
+
if not isinstance(key, str):
|
|
14
|
+
raise TypeError(f"schema_key requires string keys, got {type(key)}")
|
|
15
|
+
if key.startswith("_"):
|
|
16
|
+
continue
|
|
17
|
+
keys.add(key)
|
|
18
|
+
return tuple(sorted(keys))
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def schema_key_from_metadata_raw(metadata: dict[str, JsonValue]) -> tuple[str, ...]:
|
|
22
|
+
raw = metadata.get("schema_key")
|
|
23
|
+
if raw is None:
|
|
24
|
+
raise ValueError("metadata missing schema_key")
|
|
25
|
+
if isinstance(raw, tuple):
|
|
26
|
+
items = raw
|
|
27
|
+
elif isinstance(raw, list):
|
|
28
|
+
items = raw
|
|
29
|
+
else:
|
|
30
|
+
raise TypeError(f"metadata schema_key must be list or tuple, got {type(raw)}")
|
|
31
|
+
keys: list[str] = []
|
|
32
|
+
for item in items:
|
|
33
|
+
if not isinstance(item, str):
|
|
34
|
+
raise TypeError(f"metadata schema_key must be strings, got {type(item)}")
|
|
35
|
+
keys.append(item)
|
|
36
|
+
return tuple(keys)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def schema_key_from_cls(cls: type) -> tuple[str, ...]:
|
|
40
|
+
fields = chz.chz_fields(cls)
|
|
41
|
+
keys = {
|
|
42
|
+
field.logical_name
|
|
43
|
+
for field in fields.values()
|
|
44
|
+
if not field.logical_name.startswith("_")
|
|
45
|
+
}
|
|
46
|
+
return tuple(sorted(keys))
|
furu/storage/metadata.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import datetime
|
|
2
2
|
import getpass
|
|
3
|
+
import importlib.metadata
|
|
3
4
|
import json
|
|
4
5
|
import os
|
|
5
6
|
import platform
|
|
@@ -9,9 +10,10 @@ import sys
|
|
|
9
10
|
from pathlib import Path
|
|
10
11
|
from typing import TYPE_CHECKING
|
|
11
12
|
|
|
12
|
-
from pydantic import BaseModel, ConfigDict
|
|
13
|
+
from pydantic import BaseModel, ConfigDict, field_validator
|
|
13
14
|
|
|
14
15
|
from ..config import FURU_CONFIG
|
|
16
|
+
from ..schema import schema_key_from_furu_obj
|
|
15
17
|
from ..serialization import BaseModel as PydanticBaseModel
|
|
16
18
|
from ..serialization import FuruSerializer
|
|
17
19
|
from ..serialization.serializer import JsonValue
|
|
@@ -50,6 +52,7 @@ class EnvironmentInfo(BaseModel):
|
|
|
50
52
|
|
|
51
53
|
timestamp: str
|
|
52
54
|
command: str
|
|
55
|
+
furu_package_version: str | None = None
|
|
53
56
|
python_version: str
|
|
54
57
|
executable: str
|
|
55
58
|
platform: str
|
|
@@ -66,6 +69,7 @@ class FuruMetadata(BaseModel):
|
|
|
66
69
|
# Furu-specific fields
|
|
67
70
|
furu_python_def: str
|
|
68
71
|
furu_obj: JsonValue # Serialized Furu object from FuruSerializer.to_dict()
|
|
72
|
+
schema_key: tuple[str, ...]
|
|
69
73
|
furu_hash: str
|
|
70
74
|
furu_path: str
|
|
71
75
|
|
|
@@ -79,6 +83,7 @@ class FuruMetadata(BaseModel):
|
|
|
79
83
|
# Environment info
|
|
80
84
|
timestamp: str
|
|
81
85
|
command: str
|
|
86
|
+
furu_package_version: str | None = None
|
|
82
87
|
python_version: str
|
|
83
88
|
executable: str
|
|
84
89
|
platform: str
|
|
@@ -86,6 +91,13 @@ class FuruMetadata(BaseModel):
|
|
|
86
91
|
user: str
|
|
87
92
|
pid: int
|
|
88
93
|
|
|
94
|
+
@field_validator("schema_key", mode="before")
|
|
95
|
+
@classmethod
|
|
96
|
+
def _coerce_schema_key(cls, value: object) -> object:
|
|
97
|
+
if isinstance(value, list):
|
|
98
|
+
return tuple(value)
|
|
99
|
+
return value
|
|
100
|
+
|
|
89
101
|
|
|
90
102
|
class MetadataManager:
|
|
91
103
|
"""Handles metadata collection and storage."""
|
|
@@ -150,7 +162,8 @@ class MetadataManager:
|
|
|
150
162
|
except (subprocess.CalledProcessError, FileNotFoundError) as e:
|
|
151
163
|
raise RuntimeError(
|
|
152
164
|
"Git remote 'origin' is required for provenance but was not found. "
|
|
153
|
-
"Set FURU_ALLOW_NO_GIT_ORIGIN=1 to allow missing origin
|
|
165
|
+
"Set FURU_ALLOW_NO_GIT_ORIGIN=1 to allow missing origin, "
|
|
166
|
+
"or set FURU_RECORD_GIT=ignore to disable git metadata."
|
|
154
167
|
) from e
|
|
155
168
|
|
|
156
169
|
if ignore_diff:
|
|
@@ -216,6 +229,7 @@ class MetadataManager:
|
|
|
216
229
|
timespec="microseconds"
|
|
217
230
|
),
|
|
218
231
|
command=" ".join(sys.argv) if sys.argv else "<unknown>",
|
|
232
|
+
furu_package_version=importlib.metadata.version("furu"),
|
|
219
233
|
python_version=sys.version,
|
|
220
234
|
executable=sys.executable,
|
|
221
235
|
platform=platform.platform(),
|
|
@@ -237,10 +251,12 @@ class MetadataManager:
|
|
|
237
251
|
raise TypeError(
|
|
238
252
|
f"Expected FuruSerializer.to_dict to return dict, got {type(serialized_obj)}"
|
|
239
253
|
)
|
|
254
|
+
schema_key = schema_key_from_furu_obj(serialized_obj)
|
|
240
255
|
|
|
241
256
|
return FuruMetadata(
|
|
242
257
|
furu_python_def=FuruSerializer.to_python(furu_obj, multiline=False),
|
|
243
258
|
furu_obj=serialized_obj,
|
|
259
|
+
schema_key=schema_key,
|
|
244
260
|
furu_hash=FuruSerializer.compute_hash(furu_obj),
|
|
245
261
|
furu_path=str(directory.resolve()),
|
|
246
262
|
git_commit=git_info.git_commit,
|
|
@@ -250,6 +266,7 @@ class MetadataManager:
|
|
|
250
266
|
git_submodules=git_info.git_submodules,
|
|
251
267
|
timestamp=env_info.timestamp,
|
|
252
268
|
command=env_info.command,
|
|
269
|
+
furu_package_version=env_info.furu_package_version,
|
|
253
270
|
python_version=env_info.python_version,
|
|
254
271
|
executable=env_info.executable,
|
|
255
272
|
platform=env_info.platform,
|
furu/testing.py
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
from collections.abc import Generator, Mapping, Sequence
|
|
2
|
+
from contextlib import contextmanager
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import cast, overload
|
|
6
|
+
|
|
7
|
+
import chz
|
|
8
|
+
import pytest
|
|
9
|
+
|
|
10
|
+
from .config import FURU_CONFIG, RecordGitMode
|
|
11
|
+
from .core import Furu
|
|
12
|
+
from .runtime.overrides import OverrideValue, override_furu_hashes
|
|
13
|
+
|
|
14
|
+
OverrideKey = Furu | str
|
|
15
|
+
_PATH_MISSING = object()
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class _FuruConfigSnapshot:
|
|
20
|
+
base_root: Path
|
|
21
|
+
version_controlled_root_override: Path | None
|
|
22
|
+
record_git: RecordGitMode
|
|
23
|
+
allow_no_git_origin: bool
|
|
24
|
+
poll_interval: float
|
|
25
|
+
stale_timeout: float
|
|
26
|
+
max_wait_time_sec: float | None
|
|
27
|
+
lease_duration_sec: float
|
|
28
|
+
heartbeat_interval_sec: float
|
|
29
|
+
cancelled_is_preempted: bool
|
|
30
|
+
retry_failed: bool
|
|
31
|
+
submitit_root: Path
|
|
32
|
+
|
|
33
|
+
@classmethod
|
|
34
|
+
def capture(cls) -> "_FuruConfigSnapshot":
|
|
35
|
+
return cls(
|
|
36
|
+
base_root=FURU_CONFIG.base_root,
|
|
37
|
+
version_controlled_root_override=FURU_CONFIG.version_controlled_root_override,
|
|
38
|
+
record_git=FURU_CONFIG.record_git,
|
|
39
|
+
allow_no_git_origin=FURU_CONFIG.allow_no_git_origin,
|
|
40
|
+
poll_interval=FURU_CONFIG.poll_interval,
|
|
41
|
+
stale_timeout=FURU_CONFIG.stale_timeout,
|
|
42
|
+
max_wait_time_sec=FURU_CONFIG.max_wait_time_sec,
|
|
43
|
+
lease_duration_sec=FURU_CONFIG.lease_duration_sec,
|
|
44
|
+
heartbeat_interval_sec=FURU_CONFIG.heartbeat_interval_sec,
|
|
45
|
+
cancelled_is_preempted=FURU_CONFIG.cancelled_is_preempted,
|
|
46
|
+
retry_failed=FURU_CONFIG.retry_failed,
|
|
47
|
+
submitit_root=FURU_CONFIG.submitit_root,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
def restore(self) -> None:
|
|
51
|
+
FURU_CONFIG.base_root = self.base_root
|
|
52
|
+
FURU_CONFIG.version_controlled_root_override = (
|
|
53
|
+
self.version_controlled_root_override
|
|
54
|
+
)
|
|
55
|
+
FURU_CONFIG.record_git = self.record_git
|
|
56
|
+
FURU_CONFIG.allow_no_git_origin = self.allow_no_git_origin
|
|
57
|
+
FURU_CONFIG.poll_interval = self.poll_interval
|
|
58
|
+
FURU_CONFIG.stale_timeout = self.stale_timeout
|
|
59
|
+
FURU_CONFIG.max_wait_time_sec = self.max_wait_time_sec
|
|
60
|
+
FURU_CONFIG.lease_duration_sec = self.lease_duration_sec
|
|
61
|
+
FURU_CONFIG.heartbeat_interval_sec = self.heartbeat_interval_sec
|
|
62
|
+
FURU_CONFIG.cancelled_is_preempted = self.cancelled_is_preempted
|
|
63
|
+
FURU_CONFIG.retry_failed = self.retry_failed
|
|
64
|
+
FURU_CONFIG.submitit_root = self.submitit_root
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _apply_test_config(base_root: Path) -> Path:
|
|
68
|
+
root = base_root.resolve()
|
|
69
|
+
FURU_CONFIG.base_root = root
|
|
70
|
+
FURU_CONFIG.version_controlled_root_override = root / "furu-data" / "artifacts"
|
|
71
|
+
FURU_CONFIG.record_git = "ignore"
|
|
72
|
+
FURU_CONFIG.allow_no_git_origin = False
|
|
73
|
+
FURU_CONFIG.poll_interval = 0.01
|
|
74
|
+
FURU_CONFIG.stale_timeout = 0.1
|
|
75
|
+
FURU_CONFIG.max_wait_time_sec = None
|
|
76
|
+
FURU_CONFIG.lease_duration_sec = 0.05
|
|
77
|
+
FURU_CONFIG.heartbeat_interval_sec = 0.01
|
|
78
|
+
FURU_CONFIG.cancelled_is_preempted = True
|
|
79
|
+
FURU_CONFIG.retry_failed = True
|
|
80
|
+
FURU_CONFIG.submitit_root = root / "submitit"
|
|
81
|
+
return root
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@contextmanager
|
|
85
|
+
def furu_test_env(base_root: Path) -> Generator[Path, None, None]:
|
|
86
|
+
snapshot = _FuruConfigSnapshot.capture()
|
|
87
|
+
root = _apply_test_config(base_root)
|
|
88
|
+
try:
|
|
89
|
+
yield root
|
|
90
|
+
finally:
|
|
91
|
+
snapshot.restore()
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@overload
|
|
95
|
+
def override_results(
|
|
96
|
+
overrides: Mapping[Furu, OverrideValue],
|
|
97
|
+
) -> Generator[None, None, None]: ...
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@overload
|
|
101
|
+
def override_results(
|
|
102
|
+
overrides: Mapping[str, OverrideValue],
|
|
103
|
+
) -> Generator[None, None, None]: ...
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@contextmanager
|
|
107
|
+
def override_results(
|
|
108
|
+
overrides: Mapping[OverrideKey, OverrideValue],
|
|
109
|
+
) -> Generator[None, None, None]:
|
|
110
|
+
"""Override specific Furu results within the context.
|
|
111
|
+
|
|
112
|
+
Overrides are keyed by furu_hash, so identical configs share a stub.
|
|
113
|
+
Keys may be a Furu instance or a furu_hash string.
|
|
114
|
+
"""
|
|
115
|
+
hash_overrides: dict[str, OverrideValue] = {}
|
|
116
|
+
for key, value in overrides.items():
|
|
117
|
+
hash_key: str
|
|
118
|
+
if isinstance(key, Furu):
|
|
119
|
+
hash_key = cast(str, key.furu_hash)
|
|
120
|
+
elif isinstance(key, str):
|
|
121
|
+
if not key:
|
|
122
|
+
raise ValueError("override furu_hash must be non-empty")
|
|
123
|
+
hash_key = key
|
|
124
|
+
else:
|
|
125
|
+
raise TypeError(
|
|
126
|
+
"override_results keys must be Furu instances or furu_hash strings"
|
|
127
|
+
)
|
|
128
|
+
hash_overrides[hash_key] = value
|
|
129
|
+
with override_furu_hashes(hash_overrides):
|
|
130
|
+
yield
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@overload
|
|
134
|
+
def override_results_for(
|
|
135
|
+
root: Furu,
|
|
136
|
+
overrides: Mapping[str, OverrideValue],
|
|
137
|
+
) -> Generator[None, None, None]: ...
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
@contextmanager
|
|
141
|
+
def override_results_for(
|
|
142
|
+
root: Furu,
|
|
143
|
+
overrides: Mapping[str, OverrideValue],
|
|
144
|
+
) -> Generator[None, None, None]:
|
|
145
|
+
"""Override results by dotted field path relative to a root Furu object.
|
|
146
|
+
|
|
147
|
+
Paths follow chz traversal: use numeric segments for list indices and
|
|
148
|
+
key segments for mappings (e.g. "deps.0" or "deps.key").
|
|
149
|
+
"""
|
|
150
|
+
hash_overrides: dict[str, OverrideValue] = {}
|
|
151
|
+
for path, value in overrides.items():
|
|
152
|
+
target = _resolve_override_path(root, path)
|
|
153
|
+
hash_overrides[cast(str, target.furu_hash)] = value
|
|
154
|
+
with override_furu_hashes(hash_overrides):
|
|
155
|
+
yield
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _resolve_override_path(root: Furu, path: str) -> Furu:
|
|
159
|
+
if not path:
|
|
160
|
+
raise ValueError("override path must be non-empty")
|
|
161
|
+
current: object = root
|
|
162
|
+
for segment in path.split("."):
|
|
163
|
+
if not segment:
|
|
164
|
+
raise ValueError(f"override path has empty segment: {path!r}")
|
|
165
|
+
current = _resolve_path_segment(current, segment, path)
|
|
166
|
+
if not isinstance(current, Furu):
|
|
167
|
+
raise TypeError(
|
|
168
|
+
f"override path {path!r} must resolve to a Furu instance; got {type(current).__name__}"
|
|
169
|
+
)
|
|
170
|
+
return current
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _resolve_path_segment(current: object, segment: str, path: str) -> object:
|
|
174
|
+
if chz.is_chz(current):
|
|
175
|
+
value = getattr(current, segment, _PATH_MISSING)
|
|
176
|
+
if value is _PATH_MISSING:
|
|
177
|
+
raise AttributeError(f"override path {path!r} has no attribute {segment!r}")
|
|
178
|
+
return value
|
|
179
|
+
if isinstance(current, Mapping):
|
|
180
|
+
mapping = cast(Mapping[object, OverrideValue], current)
|
|
181
|
+
return _resolve_mapping_segment(mapping, segment, path)
|
|
182
|
+
if _is_indexable_sequence(current):
|
|
183
|
+
sequence = cast(Sequence[OverrideValue], current)
|
|
184
|
+
return _resolve_sequence_segment(sequence, segment, path)
|
|
185
|
+
raise TypeError(
|
|
186
|
+
f"override path {path!r} cannot traverse into {type(current).__name__}"
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _resolve_mapping_segment(
|
|
191
|
+
mapping: Mapping[object, OverrideValue],
|
|
192
|
+
segment: str,
|
|
193
|
+
path: str,
|
|
194
|
+
) -> OverrideValue:
|
|
195
|
+
if segment in mapping:
|
|
196
|
+
return mapping[segment]
|
|
197
|
+
index = _parse_index_segment(segment)
|
|
198
|
+
if index is not None and index in mapping:
|
|
199
|
+
return mapping[index]
|
|
200
|
+
raise KeyError(f"override path {path!r} has no key {segment!r}")
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _resolve_sequence_segment(
|
|
204
|
+
sequence: Sequence[OverrideValue],
|
|
205
|
+
segment: str,
|
|
206
|
+
path: str,
|
|
207
|
+
) -> OverrideValue:
|
|
208
|
+
index = _parse_index_segment(segment)
|
|
209
|
+
if index is None:
|
|
210
|
+
raise TypeError(f"override path {path!r} index {segment!r} is not an integer")
|
|
211
|
+
if index < 0:
|
|
212
|
+
raise ValueError(f"override path {path!r} index must be non-negative")
|
|
213
|
+
return sequence[index]
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _parse_index_segment(segment: str) -> int | None:
|
|
217
|
+
if not segment.isdigit():
|
|
218
|
+
return None
|
|
219
|
+
return int(segment)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _is_indexable_sequence(value: object) -> bool:
|
|
223
|
+
if isinstance(value, (str, bytes, bytearray)):
|
|
224
|
+
return False
|
|
225
|
+
return isinstance(value, Sequence)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
@pytest.fixture()
|
|
229
|
+
def furu_tmp_root(tmp_path: Path) -> Generator[Path, None, None]:
|
|
230
|
+
"""Configure furu to use a temporary root for the test."""
|
|
231
|
+
with furu_test_env(tmp_path) as root:
|
|
232
|
+
yield root
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: furu
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.7
|
|
4
4
|
Summary: Cacheable, nested pipelines for Python. Define computations as configs; furu handles caching, state tracking, and result reuse across runs.
|
|
5
5
|
Author: Herman Brunborg
|
|
6
6
|
Author-email: Herman Brunborg <herman@brunborg.com>
|
|
@@ -503,6 +503,89 @@ furu.FURU_CONFIG.record_git = "uncached"
|
|
|
503
503
|
furu.FURU_CONFIG.poll_interval = 5.0
|
|
504
504
|
```
|
|
505
505
|
|
|
506
|
+
### Testing with pytest
|
|
507
|
+
|
|
508
|
+
Use the built-in pytest fixture to isolate Furu storage in tests (each test gets
|
|
509
|
+
its own temp root, so identical configs in separate tests will not collide):
|
|
510
|
+
|
|
511
|
+
```python
|
|
512
|
+
# conftest.py
|
|
513
|
+
pytest_plugins = ["furu.testing"]
|
|
514
|
+
```
|
|
515
|
+
|
|
516
|
+
```python
|
|
517
|
+
# test_pipeline.py
|
|
518
|
+
import json
|
|
519
|
+
from pathlib import Path
|
|
520
|
+
|
|
521
|
+
import furu
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
class TrainModel(furu.Furu[Path]):
|
|
525
|
+
lr: float = furu.chz.field(default=1e-3)
|
|
526
|
+
|
|
527
|
+
def _create(self) -> Path:
|
|
528
|
+
path = self.furu_dir / "metrics.json"
|
|
529
|
+
path.write_text(json.dumps({"lr": self.lr}))
|
|
530
|
+
return path
|
|
531
|
+
|
|
532
|
+
def _load(self) -> Path:
|
|
533
|
+
return self.furu_dir / "metrics.json"
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def test_create_and_reload(furu_tmp_root):
|
|
537
|
+
obj = TrainModel(lr=1e-3)
|
|
538
|
+
first = obj.get()
|
|
539
|
+
second = obj.get()
|
|
540
|
+
assert first.read_text() == second.read_text()
|
|
541
|
+
assert (furu_tmp_root / "data").exists()
|
|
542
|
+
```
|
|
543
|
+
|
|
544
|
+
Override specific dependencies when you want to skip deeper chains:
|
|
545
|
+
|
|
546
|
+
```python
|
|
547
|
+
from furu.testing import override_results
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
class Normalize(furu.Furu[str]):
|
|
551
|
+
def _create(self) -> str:
|
|
552
|
+
return "normalized"
|
|
553
|
+
|
|
554
|
+
def _load(self) -> str:
|
|
555
|
+
return "normalized"
|
|
556
|
+
|
|
557
|
+
|
|
558
|
+
class TrainModel(furu.Furu[str]):
|
|
559
|
+
normalizer: Normalize = furu.chz.field(default_factory=Normalize)
|
|
560
|
+
|
|
561
|
+
def _create(self) -> str:
|
|
562
|
+
return f"trained:{self.normalizer.get()}"
|
|
563
|
+
|
|
564
|
+
def _load(self) -> str:
|
|
565
|
+
return "trained"
|
|
566
|
+
|
|
567
|
+
|
|
568
|
+
def test_override_dependency(furu_tmp_root):
|
|
569
|
+
normalizer = Normalize()
|
|
570
|
+
model = TrainModel(normalizer=normalizer)
|
|
571
|
+
with override_results({normalizer: "stub"}):
|
|
572
|
+
assert model.get() == "trained:stub"
|
|
573
|
+
```
|
|
574
|
+
|
|
575
|
+
If you want to override without instantiating the dependency directly, target it
|
|
576
|
+
by dotted path from the root object (chz-style paths, e.g. `deps.0` for lists and
|
|
577
|
+
`deps.key` for mappings):
|
|
578
|
+
|
|
579
|
+
```python
|
|
580
|
+
from furu.testing import override_results_for
|
|
581
|
+
|
|
582
|
+
|
|
583
|
+
def test_override_by_path(furu_tmp_root):
|
|
584
|
+
model = TrainModel()
|
|
585
|
+
with override_results_for(model, {"normalizer": "stub"}):
|
|
586
|
+
assert model.get() == "trained:stub"
|
|
587
|
+
```
|
|
588
|
+
|
|
506
589
|
### Class-Level Options
|
|
507
590
|
|
|
508
591
|
```python
|
|
@@ -1,21 +1,22 @@
|
|
|
1
|
-
furu/__init__.py,sha256=
|
|
1
|
+
furu/__init__.py,sha256=umlKxb3zB5KYsx3meqLFzqNRduGfEVUplTZSelX-p4E,1729
|
|
2
2
|
furu/adapters/__init__.py,sha256=onLzEj9hccPK15g8a8va2T19nqQXoxb9rQlJIjKSKnE,69
|
|
3
3
|
furu/adapters/submitit.py,sha256=FV3XEUSQuS5vIyzkW-Iuqtf8SRL-fsokPG67u7tMF5I,7276
|
|
4
|
+
furu/aliases.py,sha256=C31Oe2w8gTAuVjo-eKCs_Xp3Ir16HRyn327CeczuO6c,1755
|
|
4
5
|
furu/config.py,sha256=UGnH8QAKMUgrGMGNkfBgLXideXEpDlozUSsX9iNN8Lw,6844
|
|
5
6
|
furu/core/__init__.py,sha256=6hH7i6r627c0FZn6eQVsSG7LD4QmTta6iQw0AiPQPTM,156
|
|
6
|
-
furu/core/furu.py,sha256=
|
|
7
|
+
furu/core/furu.py,sha256=PBz9_gykog9J-QY45pt99DR5f03ssFys1DhiEPBRAQA,66692
|
|
7
8
|
furu/core/list.py,sha256=QaGSh8NFg1K2WFncM8duOYQ6KLZ6EW2pRLArN_e5Juw,3662
|
|
8
9
|
furu/dashboard/__init__.py,sha256=ziAordJfkbbXNIM7iA9O7vR2gsCq34AInYiMYOCfWOc,362
|
|
9
10
|
furu/dashboard/__main__.py,sha256=cNs65IMl4kwZFpxa9xLXmFSy4-M5D1X1ZBfTDxW11vo,144
|
|
10
11
|
furu/dashboard/api/__init__.py,sha256=9-WyWOt-VQJJBIsdW29D-7JvR-BivJd9G_SRaRptCz0,80
|
|
11
|
-
furu/dashboard/api/models.py,sha256=
|
|
12
|
-
furu/dashboard/api/routes.py,sha256=
|
|
12
|
+
furu/dashboard/api/models.py,sha256=ptC1NKMl2zGb8ERR7JY_EQBi-Zxc4NvNhnFxcOsBawk,5133
|
|
13
|
+
furu/dashboard/api/routes.py,sha256=TCGTceH-i7dv1Y6JE4JFBcmyOs6QTTDRp5zdjwuBpPQ,5035
|
|
13
14
|
furu/dashboard/frontend/dist/assets/index-BXAIKNNr.css,sha256=qhsN0Td3mM-GAR8mZ0CtocynABLKa1ncl9ioDrTKOIQ,34768
|
|
14
|
-
furu/dashboard/frontend/dist/assets/index-
|
|
15
|
+
furu/dashboard/frontend/dist/assets/index-NiDdQnqO.js,sha256=Ns1BosYU5xckMAn_b4iUtD6-m7tB9r3GY7_rwFgzumI,545103
|
|
15
16
|
furu/dashboard/frontend/dist/favicon.svg,sha256=3TSLHNZITFe3JTPoYHZnDgiGsJxIzf39v97l2A1Hodo,369
|
|
16
|
-
furu/dashboard/frontend/dist/index.html,sha256=
|
|
17
|
+
furu/dashboard/frontend/dist/index.html,sha256=i7eDqiisdL86W0ZldsUg3z5wsX1NfH98adPxgh0p06Q,810
|
|
17
18
|
furu/dashboard/main.py,sha256=gj9Cdj2qyaSCEkmfNHUMQXlXv6GpWTQ9IZEi7WzlCSo,4463
|
|
18
|
-
furu/dashboard/scanner.py,sha256=
|
|
19
|
+
furu/dashboard/scanner.py,sha256=CMLcUTZ2mvaigpMM1qbonpPwvG8Om_FIVd11PruSZFo,35467
|
|
19
20
|
furu/errors.py,sha256=FFbV4M0-ipVGizv5ee80L-NZFVjaRjy8i19mClr6R0g,3959
|
|
20
21
|
furu/execution/__init__.py,sha256=ixVw1Shvg2ulS597OYYeGgSSTwv25j_McuQdDXIiEL8,625
|
|
21
22
|
furu/execution/context.py,sha256=0tAbM0azqEus8hknf_A9-Zs9Sq99bnUkFyV4RO4ZMRU,666
|
|
@@ -27,20 +28,22 @@ furu/execution/slurm_dag.py,sha256=xh9EUGdPZaAH3UfcRqo6MsKYBIV-UW3_7owY8kLOwz4,9
|
|
|
27
28
|
furu/execution/slurm_pool.py,sha256=ft76Gp-HgFWWjGvDclUChLOjY1rvhhfkP5mxhK3ViQk,30395
|
|
28
29
|
furu/execution/slurm_spec.py,sha256=DG8BF4FCga2ZXsqGUvfNibk6II40JcShVZ4jTwxTdec,977
|
|
29
30
|
furu/execution/submitit_factory.py,sha256=B2vkDtmscuAX0sBaj9V5pNlgOtkkV35yJ1fZ7A-DSvU,1119
|
|
30
|
-
furu/
|
|
31
|
-
furu/migration.py,sha256=EYWULuH8lEVvESthO2qEF95WJTo1Uj6d4L6VU2zmWpw,31350
|
|
31
|
+
furu/migration.py,sha256=OG-GvDscZYNY0Mz8nQJ-F7tWW9Ev6iUwTKOyc6uwBpc,21058
|
|
32
32
|
furu/runtime/__init__.py,sha256=fQqE7wUuWunLD73Vm3lss7BFSij3UVxXOKQXBAOS8zw,504
|
|
33
33
|
furu/runtime/env.py,sha256=lb-LWl-1EM_CP8sy0z3HAY20NXQ-v3QdOgte1i0HYVA,214
|
|
34
34
|
furu/runtime/logging.py,sha256=Xni1hWyH21bKc6D2owBZzThsj6q8yQOBD9zUrDS4jtI,10760
|
|
35
|
+
furu/runtime/overrides.py,sha256=E3fsZ0ReNOnC9xioHHFlmudm5K2DZLFFcEIvrnA6t2o,871
|
|
35
36
|
furu/runtime/tracebacks.py,sha256=PGCuOq8QkWSoun791gjUXM8frOP2wWV8IBlqaA4nuGE,1631
|
|
37
|
+
furu/schema.py,sha256=t-XyapYJpYQEmLFo-PGwrCUzfAG-WefTi3OYbveRp0Y,1453
|
|
36
38
|
furu/serialization/__init__.py,sha256=L7oHuIbxdSh7GCY3thMQnDwlt_ERH-TMy0YKEAZLrPs,341
|
|
37
39
|
furu/serialization/migrations.py,sha256=HD5g8JCBdH3Y0rHJYc4Ug1IXBVcUDxLE7nfiXZnXcUE,7772
|
|
38
40
|
furu/serialization/serializer.py,sha256=_nfUaAOy_KHegvfXlpPh4rCuvkzalJva75OvDg5nXiI,10114
|
|
39
41
|
furu/storage/__init__.py,sha256=cLLL-GPpSu9C72Mdk5S6TGu3g-SnBfEuxzfpx5ZJPtw,616
|
|
40
|
-
furu/storage/metadata.py,sha256=
|
|
42
|
+
furu/storage/metadata.py,sha256=EYT4sRSFz2KY3nlc4DAJBfxuvi2rQedPjPK_NGJlQ3k,10313
|
|
41
43
|
furu/storage/migration.py,sha256=FNExLdPu1ekKZR2XJkAgags9U8pV2FfkKAECSXkSra8,2585
|
|
42
44
|
furu/storage/state.py,sha256=kcIfAwdKWT8Q2ElbC5qofQC6noS_k6eNSPkNAdYXoaY,43707
|
|
43
|
-
furu
|
|
44
|
-
furu-0.0.
|
|
45
|
-
furu-0.0.
|
|
46
|
-
furu-0.0.
|
|
45
|
+
furu/testing.py,sha256=lS-30bOu_RI1l4OV4lGWNpx5HOAwX2JYHHqakOkz8so,7804
|
|
46
|
+
furu-0.0.7.dist-info/WHEEL,sha256=fAguSjoiATBe7TNBkJwOjyL1Tt4wwiaQGtNtjRPNMQA,80
|
|
47
|
+
furu-0.0.7.dist-info/entry_points.txt,sha256=hZkjtFzNlb33Zk-aUfLMRj-XgVDxdT82-JXG9d4bu2E,60
|
|
48
|
+
furu-0.0.7.dist-info/METADATA,sha256=v6gNqvq19H4H9WPMl2PcaDIOZVP1Qtz1lQfzBMzN0Yc,19117
|
|
49
|
+
furu-0.0.7.dist-info/RECORD,,
|
furu/migrate.py
DELETED
|
@@ -1,48 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from typing import Literal, Mapping
|
|
4
|
-
|
|
5
|
-
from .core.furu import Furu
|
|
6
|
-
from .migration import (
|
|
7
|
-
MigrationValue,
|
|
8
|
-
apply_migration,
|
|
9
|
-
find_migration_candidates_initialized_target,
|
|
10
|
-
)
|
|
11
|
-
from .storage import MigrationRecord
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
MigrationPolicy = Literal["alias", "move", "copy"]
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
def migrate(
|
|
18
|
-
from_obj: Furu,
|
|
19
|
-
to_obj: Furu,
|
|
20
|
-
*,
|
|
21
|
-
policy: MigrationPolicy = "alias",
|
|
22
|
-
origin: str | None = None,
|
|
23
|
-
note: str | None = None,
|
|
24
|
-
default_values: Mapping[str, MigrationValue] | None = None,
|
|
25
|
-
) -> MigrationRecord:
|
|
26
|
-
from_namespace = ".".join(from_obj._namespace().parts)
|
|
27
|
-
candidates = find_migration_candidates_initialized_target(
|
|
28
|
-
to_obj=to_obj,
|
|
29
|
-
from_namespace=from_namespace,
|
|
30
|
-
default_fields=None,
|
|
31
|
-
drop_fields=None,
|
|
32
|
-
)
|
|
33
|
-
if not candidates:
|
|
34
|
-
raise ValueError("migration: no candidates found for initialized target")
|
|
35
|
-
if len(candidates) != 1:
|
|
36
|
-
raise ValueError("migration: expected exactly one candidate")
|
|
37
|
-
candidate = candidates[0]
|
|
38
|
-
if default_values:
|
|
39
|
-
candidate = candidate.with_default_values(default_values)
|
|
40
|
-
records = apply_migration(
|
|
41
|
-
candidate,
|
|
42
|
-
policy=policy,
|
|
43
|
-
cascade=True,
|
|
44
|
-
origin=origin,
|
|
45
|
-
note=note,
|
|
46
|
-
conflict="throw",
|
|
47
|
-
)
|
|
48
|
-
return records[0]
|
|
File without changes
|