furu 0.0.5__py3-none-any.whl → 0.0.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- furu/__init__.py +3 -15
- furu/aliases.py +53 -0
- furu/core/furu.py +162 -5
- furu/dashboard/api/models.py +18 -4
- furu/dashboard/api/routes.py +8 -0
- furu/dashboard/frontend/dist/assets/{index-BjyrY-Zz.js → index-NiDdQnqO.js} +15 -15
- furu/dashboard/frontend/dist/index.html +1 -1
- furu/dashboard/scanner.py +173 -147
- furu/migration.py +491 -763
- furu/runtime/overrides.py +37 -0
- furu/schema.py +46 -0
- furu/storage/metadata.py +19 -2
- furu/testing.py +232 -0
- {furu-0.0.5.dist-info → furu-0.0.7.dist-info}/METADATA +84 -1
- {furu-0.0.5.dist-info → furu-0.0.7.dist-info}/RECORD +17 -14
- {furu-0.0.5.dist-info → furu-0.0.7.dist-info}/WHEEL +1 -1
- furu/migrate.py +0 -48
- {furu-0.0.5.dist-info → furu-0.0.7.dist-info}/entry_points.txt +0 -0
furu/__init__.py
CHANGED
|
@@ -36,15 +36,7 @@ from .runtime import (
|
|
|
36
36
|
log,
|
|
37
37
|
write_separator,
|
|
38
38
|
)
|
|
39
|
-
from .
|
|
40
|
-
from .migration import (
|
|
41
|
-
NamespacePair,
|
|
42
|
-
MigrationCandidate,
|
|
43
|
-
MigrationSkip,
|
|
44
|
-
apply_migration,
|
|
45
|
-
find_migration_candidates,
|
|
46
|
-
find_migration_candidates_initialized_target,
|
|
47
|
-
)
|
|
39
|
+
from .migration import FuruRef, MigrationReport, MigrationSkip
|
|
48
40
|
from .serialization import FuruSerializer
|
|
49
41
|
from .storage import MetadataManager, StateManager
|
|
50
42
|
|
|
@@ -67,13 +59,9 @@ __all__ = [
|
|
|
67
59
|
"DependencyChzSpec",
|
|
68
60
|
"DependencySpec",
|
|
69
61
|
"MISSING",
|
|
70
|
-
"
|
|
71
|
-
"
|
|
72
|
-
"MigrationCandidate",
|
|
62
|
+
"FuruRef",
|
|
63
|
+
"MigrationReport",
|
|
73
64
|
"MigrationSkip",
|
|
74
|
-
"apply_migration",
|
|
75
|
-
"find_migration_candidates",
|
|
76
|
-
"find_migration_candidates_initialized_target",
|
|
77
65
|
"MetadataManager",
|
|
78
66
|
"StateManager",
|
|
79
67
|
"SubmititAdapter",
|
furu/aliases.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections import defaultdict
|
|
4
|
+
from collections.abc import Iterator
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from .config import FURU_CONFIG
|
|
8
|
+
from .storage import MigrationManager, MigrationRecord
|
|
9
|
+
from .storage.migration import RootKind
|
|
10
|
+
from .storage.state import StateManager
|
|
11
|
+
|
|
12
|
+
AliasKey = tuple[str, str, RootKind]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def iter_roots() -> Iterator[Path]:
|
|
16
|
+
"""Iterate over all existing Furu storage roots."""
|
|
17
|
+
for version_controlled in (False, True):
|
|
18
|
+
root = FURU_CONFIG.get_root(version_controlled)
|
|
19
|
+
if root.exists():
|
|
20
|
+
yield root
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def find_experiment_dirs(root: Path) -> list[Path]:
|
|
24
|
+
"""Find all directories containing .furu/state.json files."""
|
|
25
|
+
experiments: list[Path] = []
|
|
26
|
+
|
|
27
|
+
for furu_dir in root.rglob(StateManager.INTERNAL_DIR):
|
|
28
|
+
if not furu_dir.is_dir():
|
|
29
|
+
continue
|
|
30
|
+
state_file = furu_dir / StateManager.STATE_FILE
|
|
31
|
+
if state_file.is_file():
|
|
32
|
+
experiments.append(furu_dir.parent)
|
|
33
|
+
|
|
34
|
+
return experiments
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def alias_key(migration: MigrationRecord) -> AliasKey:
|
|
38
|
+
return (migration.from_namespace, migration.from_hash, migration.from_root)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def collect_aliases(
|
|
42
|
+
*, include_inactive: bool = True
|
|
43
|
+
) -> dict[AliasKey, list[MigrationRecord]]:
|
|
44
|
+
aliases: dict[AliasKey, list[MigrationRecord]] = defaultdict(list)
|
|
45
|
+
for root in iter_roots():
|
|
46
|
+
for experiment_dir in find_experiment_dirs(root):
|
|
47
|
+
migration = MigrationManager.read_migration(experiment_dir)
|
|
48
|
+
if migration is None or migration.kind != "alias":
|
|
49
|
+
continue
|
|
50
|
+
if not include_inactive and migration.overwritten_at is not None:
|
|
51
|
+
continue
|
|
52
|
+
aliases[alias_key(migration)].append(migration)
|
|
53
|
+
return aliases
|
furu/core/furu.py
CHANGED
|
@@ -18,13 +18,14 @@ from typing import (
|
|
|
18
18
|
Callable,
|
|
19
19
|
ClassVar,
|
|
20
20
|
Hashable,
|
|
21
|
+
Iterable,
|
|
22
|
+
Literal,
|
|
21
23
|
Mapping,
|
|
22
24
|
Protocol,
|
|
23
25
|
Self,
|
|
24
26
|
Sequence,
|
|
25
27
|
TypeAlias,
|
|
26
28
|
TypedDict,
|
|
27
|
-
TypeVar,
|
|
28
29
|
cast,
|
|
29
30
|
)
|
|
30
31
|
|
|
@@ -46,8 +47,20 @@ from ..errors import (
|
|
|
46
47
|
from ..runtime import current_holder
|
|
47
48
|
from ..runtime.logging import enter_holder, get_logger, log, write_separator
|
|
48
49
|
from ..runtime.tracebacks import format_traceback
|
|
50
|
+
from ..runtime.overrides import has_override, lookup_override
|
|
51
|
+
from ..schema import schema_key_from_cls
|
|
49
52
|
from ..serialization import FuruSerializer
|
|
50
53
|
from ..serialization.serializer import JsonValue
|
|
54
|
+
from ..aliases import AliasKey, collect_aliases
|
|
55
|
+
from ..migration import (
|
|
56
|
+
FuruRef,
|
|
57
|
+
MigrationReport,
|
|
58
|
+
current as _current_refs,
|
|
59
|
+
migrate as _migrate,
|
|
60
|
+
migrate_one as _migrate_one,
|
|
61
|
+
resolve_original_ref,
|
|
62
|
+
stale as _stale_refs,
|
|
63
|
+
)
|
|
51
64
|
from ..storage import (
|
|
52
65
|
FuruMetadata,
|
|
53
66
|
MetadataManager,
|
|
@@ -311,11 +324,113 @@ class Furu[T](ABC):
|
|
|
311
324
|
"""Convert to Python code."""
|
|
312
325
|
return FuruSerializer.to_python(self, multiline=multiline)
|
|
313
326
|
|
|
327
|
+
@classmethod
|
|
328
|
+
def schema_key(cls) -> tuple[str, ...]:
|
|
329
|
+
"""
|
|
330
|
+
Return the stable schema key for this class.
|
|
331
|
+
|
|
332
|
+
Returns:
|
|
333
|
+
tuple[str, ...]: Sorted field names for the class schema.
|
|
334
|
+
"""
|
|
335
|
+
return schema_key_from_cls(cls)
|
|
336
|
+
|
|
337
|
+
@classmethod
|
|
338
|
+
def migrate(
|
|
339
|
+
cls,
|
|
340
|
+
*,
|
|
341
|
+
from_schema: Iterable[str] | None = None,
|
|
342
|
+
from_drop: Iterable[str] | None = None,
|
|
343
|
+
from_add: Iterable[str] | None = None,
|
|
344
|
+
from_hash: str | None = None,
|
|
345
|
+
from_furu_obj: dict[str, JsonValue] | None = None,
|
|
346
|
+
from_namespace: str | None = None,
|
|
347
|
+
default_field: Iterable[str] | None = None,
|
|
348
|
+
set_field: Mapping[str, JsonValue] | None = None,
|
|
349
|
+
drop_field: Iterable[str] | None = None,
|
|
350
|
+
rename_field: Mapping[str, str] | None = None,
|
|
351
|
+
include_alias_sources: bool = False,
|
|
352
|
+
conflict: Literal["throw", "skip"] = "throw",
|
|
353
|
+
origin: str | None = None,
|
|
354
|
+
note: str | None = None,
|
|
355
|
+
) -> MigrationReport:
|
|
356
|
+
"""
|
|
357
|
+
Create alias-only migrations for stored objects matching the selector.
|
|
358
|
+
|
|
359
|
+
Notes:
|
|
360
|
+
set_field only adds fields that are not already present. To replace
|
|
361
|
+
a field, drop it first and then set it.
|
|
362
|
+
"""
|
|
363
|
+
return _migrate(
|
|
364
|
+
cls,
|
|
365
|
+
from_schema=from_schema,
|
|
366
|
+
from_drop=from_drop,
|
|
367
|
+
from_add=from_add,
|
|
368
|
+
from_hash=from_hash,
|
|
369
|
+
from_furu_obj=from_furu_obj,
|
|
370
|
+
from_namespace=from_namespace,
|
|
371
|
+
default_field=default_field,
|
|
372
|
+
set_field=set_field,
|
|
373
|
+
drop_field=drop_field,
|
|
374
|
+
rename_field=rename_field,
|
|
375
|
+
include_alias_sources=include_alias_sources,
|
|
376
|
+
conflict=conflict,
|
|
377
|
+
origin=origin,
|
|
378
|
+
note=note,
|
|
379
|
+
)
|
|
380
|
+
|
|
381
|
+
@classmethod
|
|
382
|
+
def migrate_one(
|
|
383
|
+
cls,
|
|
384
|
+
*,
|
|
385
|
+
from_hash: str,
|
|
386
|
+
from_namespace: str | None = None,
|
|
387
|
+
default_field: Iterable[str] | None = None,
|
|
388
|
+
set_field: Mapping[str, JsonValue] | None = None,
|
|
389
|
+
drop_field: Iterable[str] | None = None,
|
|
390
|
+
rename_field: Mapping[str, str] | None = None,
|
|
391
|
+
include_alias_sources: bool = False,
|
|
392
|
+
conflict: Literal["throw", "skip"] = "throw",
|
|
393
|
+
origin: str | None = None,
|
|
394
|
+
note: str | None = None,
|
|
395
|
+
) -> MigrationRecord | None:
|
|
396
|
+
return _migrate_one(
|
|
397
|
+
cls,
|
|
398
|
+
from_hash=from_hash,
|
|
399
|
+
from_namespace=from_namespace,
|
|
400
|
+
default_field=default_field,
|
|
401
|
+
set_field=set_field,
|
|
402
|
+
drop_field=drop_field,
|
|
403
|
+
rename_field=rename_field,
|
|
404
|
+
include_alias_sources=include_alias_sources,
|
|
405
|
+
conflict=conflict,
|
|
406
|
+
origin=origin,
|
|
407
|
+
note=note,
|
|
408
|
+
)
|
|
409
|
+
|
|
410
|
+
@classmethod
|
|
411
|
+
def stale(cls, *, namespace: str | None = None) -> list[FuruRef]:
|
|
412
|
+
return _stale_refs(cls, namespace=namespace)
|
|
413
|
+
|
|
414
|
+
@classmethod
|
|
415
|
+
def current(cls, *, namespace: str | None = None) -> list[FuruRef]:
|
|
416
|
+
return _current_refs(cls, namespace=namespace)
|
|
417
|
+
|
|
314
418
|
def log(self: Self, message: str, *, level: str = "INFO") -> Path:
|
|
315
|
-
"""
|
|
419
|
+
"""
|
|
420
|
+
Log a message to the current holder's `furu.log`.
|
|
421
|
+
|
|
422
|
+
Parameters:
|
|
423
|
+
message (str): The message to log.
|
|
424
|
+
level (str): Logging level name (e.g., "INFO", "ERROR").
|
|
425
|
+
|
|
426
|
+
Returns:
|
|
427
|
+
Path: Path to the log file where the message was written.
|
|
428
|
+
"""
|
|
316
429
|
return log(message, level=level)
|
|
317
430
|
|
|
318
431
|
def _exists_quiet(self: Self) -> bool:
|
|
432
|
+
if has_override(self.furu_hash):
|
|
433
|
+
return True
|
|
319
434
|
directory = self._base_furu_dir()
|
|
320
435
|
success_dir = self._success_marker_dir(directory)
|
|
321
436
|
if success_dir is None:
|
|
@@ -345,6 +460,9 @@ class Furu[T](ABC):
|
|
|
345
460
|
"""Check if result exists and is valid."""
|
|
346
461
|
logger = get_logger()
|
|
347
462
|
directory = self._base_furu_dir()
|
|
463
|
+
if has_override(self.furu_hash):
|
|
464
|
+
logger.info("exists %s -> true (override)", directory)
|
|
465
|
+
return True
|
|
348
466
|
success_dir = self._success_marker_dir(directory)
|
|
349
467
|
if success_dir is None:
|
|
350
468
|
logger.info("exists %s -> false", directory)
|
|
@@ -363,6 +481,45 @@ class Furu[T](ABC):
|
|
|
363
481
|
"""Get migration record for this object."""
|
|
364
482
|
return MigrationManager.read_migration(self._base_furu_dir())
|
|
365
483
|
|
|
484
|
+
def is_alias(self: Self) -> bool:
|
|
485
|
+
record = self._alias_record(self._base_furu_dir())
|
|
486
|
+
return record is not None
|
|
487
|
+
|
|
488
|
+
def original(self: Self) -> FuruRef:
|
|
489
|
+
base_dir = self._base_furu_dir()
|
|
490
|
+
ref = FuruRef(
|
|
491
|
+
namespace=".".join(self.__class__._namespace().parts),
|
|
492
|
+
furu_hash=self.furu_hash,
|
|
493
|
+
root=MigrationManager.root_kind_for_dir(base_dir),
|
|
494
|
+
directory=base_dir,
|
|
495
|
+
)
|
|
496
|
+
return resolve_original_ref(ref)
|
|
497
|
+
|
|
498
|
+
def aliases(
|
|
499
|
+
self: Self,
|
|
500
|
+
*,
|
|
501
|
+
include_inactive: bool = True,
|
|
502
|
+
alias_index: dict[AliasKey, list[MigrationRecord]] | None = None,
|
|
503
|
+
) -> list[FuruRef]:
|
|
504
|
+
original_ref = self.original()
|
|
505
|
+
if alias_index is None:
|
|
506
|
+
alias_index = collect_aliases(include_inactive=include_inactive)
|
|
507
|
+
records = alias_index.get(
|
|
508
|
+
(original_ref.namespace, original_ref.furu_hash, original_ref.root),
|
|
509
|
+
[],
|
|
510
|
+
)
|
|
511
|
+
refs = [
|
|
512
|
+
FuruRef(
|
|
513
|
+
namespace=record.to_namespace,
|
|
514
|
+
furu_hash=record.to_hash,
|
|
515
|
+
root=record.to_root,
|
|
516
|
+
directory=MigrationManager.resolve_dir(record, target="to"),
|
|
517
|
+
)
|
|
518
|
+
for record in records
|
|
519
|
+
]
|
|
520
|
+
refs.sort(key=lambda item: item.furu_hash)
|
|
521
|
+
return refs
|
|
522
|
+
|
|
366
523
|
def get(self: Self, *, force: bool = False) -> T:
|
|
367
524
|
"""
|
|
368
525
|
Load result if it exists, computing if necessary.
|
|
@@ -376,6 +533,9 @@ class Furu[T](ABC):
|
|
|
376
533
|
Raises:
|
|
377
534
|
FuruComputeError: If computation fails with detailed error information
|
|
378
535
|
"""
|
|
536
|
+
has_override_value, override_value = lookup_override(self.furu_hash)
|
|
537
|
+
if has_override_value:
|
|
538
|
+
return cast(T, override_value)
|
|
379
539
|
from furu.errors import (
|
|
380
540
|
FuruExecutionError,
|
|
381
541
|
FuruMissingArtifact,
|
|
@@ -1584,6 +1744,3 @@ def _dependency_type_error(
|
|
|
1584
1744
|
f"{path} must be a Furu instance or a collection of Furu instances; "
|
|
1585
1745
|
f"got {type(value).__name__}"
|
|
1586
1746
|
)
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
_H = TypeVar("_H", bound=Furu, covariant=True)
|
furu/dashboard/api/models.py
CHANGED
|
@@ -19,6 +19,17 @@ class HealthCheck(BaseModel):
|
|
|
19
19
|
version: str
|
|
20
20
|
|
|
21
21
|
|
|
22
|
+
class AliasInfo(BaseModel):
|
|
23
|
+
"""Alias reference information for an experiment."""
|
|
24
|
+
|
|
25
|
+
namespace: str
|
|
26
|
+
furu_hash: str
|
|
27
|
+
migrated_at: str | None = None
|
|
28
|
+
overwritten_at: str | None = None
|
|
29
|
+
origin: str | None = None
|
|
30
|
+
note: str | None = None
|
|
31
|
+
|
|
32
|
+
|
|
22
33
|
class ExperimentSummary(BaseModel):
|
|
23
34
|
"""Summary of an experiment for list views."""
|
|
24
35
|
|
|
@@ -46,6 +57,13 @@ class ExperimentSummary(BaseModel):
|
|
|
46
57
|
to_namespace: str | None = None
|
|
47
58
|
to_hash: str | None = None
|
|
48
59
|
original_result_status: str | None = None
|
|
60
|
+
original_namespace: str | None = None
|
|
61
|
+
original_hash: str | None = None
|
|
62
|
+
schema_key: list[str] | None = None
|
|
63
|
+
current_schema_key: list[str] | None = None
|
|
64
|
+
is_stale: bool | None = None
|
|
65
|
+
is_alias: bool | None = None
|
|
66
|
+
aliases: list[AliasInfo] | None = None
|
|
49
67
|
|
|
50
68
|
|
|
51
69
|
class ExperimentDetail(ExperimentSummary):
|
|
@@ -55,10 +73,6 @@ class ExperimentDetail(ExperimentSummary):
|
|
|
55
73
|
state: JsonDict
|
|
56
74
|
metadata: JsonDict | None = None
|
|
57
75
|
attempt: StateAttempt | None = None
|
|
58
|
-
original_namespace: str | None = None
|
|
59
|
-
original_hash: str | None = None
|
|
60
|
-
alias_namespaces: list[str] | None = None
|
|
61
|
-
alias_hashes: list[str] | None = None
|
|
62
76
|
|
|
63
77
|
|
|
64
78
|
class ExperimentList(BaseModel):
|
furu/dashboard/api/routes.py
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
"""API route definitions for the Furu Dashboard."""
|
|
2
2
|
|
|
3
|
+
from typing import Literal
|
|
4
|
+
|
|
3
5
|
from fastapi import APIRouter, HTTPException, Query
|
|
4
6
|
|
|
5
7
|
from .. import __version__
|
|
@@ -21,6 +23,8 @@ from .models import (
|
|
|
21
23
|
|
|
22
24
|
router = APIRouter(prefix="/api", tags=["api"])
|
|
23
25
|
|
|
26
|
+
SchemaFilter = Literal["current", "stale", "any"]
|
|
27
|
+
|
|
24
28
|
|
|
25
29
|
@router.get("/health", response_model=HealthCheck)
|
|
26
30
|
async def health_check() -> HealthCheck:
|
|
@@ -57,6 +61,9 @@ async def list_experiments(
|
|
|
57
61
|
migration_policy: str | None = Query(
|
|
58
62
|
None, description="Filter by migration policy"
|
|
59
63
|
),
|
|
64
|
+
schema: SchemaFilter = Query(
|
|
65
|
+
"current", description="Filter by schema status (current, stale, any)"
|
|
66
|
+
),
|
|
60
67
|
view: str = Query("resolved", description="View mode: resolved or original"),
|
|
61
68
|
limit: int = Query(100, ge=1, le=1000, description="Maximum number of results"),
|
|
62
69
|
offset: int = Query(0, ge=0, description="Offset for pagination"),
|
|
@@ -76,6 +83,7 @@ async def list_experiments(
|
|
|
76
83
|
config_filter=config_filter,
|
|
77
84
|
migration_kind=migration_kind,
|
|
78
85
|
migration_policy=migration_policy,
|
|
86
|
+
schema=schema,
|
|
79
87
|
view=view,
|
|
80
88
|
)
|
|
81
89
|
|