furu 0.0.3__py3-none-any.whl → 0.0.4__py3-none-any.whl

furu/core/list.py

@@ -116,7 +116,7 @@ class FuruList(Generic[_H], metaclass=_FuruListMeta):
 
         # Use the collection
         for exp in MyExperiments:
-            result = exp.load_or_create()
+            result = exp.get()
             print(result)
     """
 

furu/dashboard/__init__.py

@@ -5,5 +5,14 @@ Install with: uv add furu[dashboard]
 Run with: furu-dashboard serve
 """
 
-__version__ = "0.1.0"
+from importlib import metadata
 
+
+def _resolve_version() -> str:
+    try:
+        return metadata.version("furu")
+    except metadata.PackageNotFoundError:
+        return "0.0.0"
+
+
+__version__ = _resolve_version()

furu/dashboard/main.py

@@ -5,11 +5,12 @@ from pathlib import Path
 
 import typer
 import uvicorn
-from fastapi import FastAPI
+from fastapi import FastAPI, HTTPException
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import FileResponse
 from fastapi.staticfiles import StaticFiles
 
+from . import __version__
 from .api.routes import router as api_router
 
 
@@ -36,7 +37,7 @@ def create_app(*, serve_frontend: bool = False) -> FastAPI:
     app = FastAPI(
         title="Furu Dashboard",
         description="Monitoring dashboard for Furu experiments",
-        version="0.1.0",
+        version=__version__,
     )
 
     # CORS middleware for development
@@ -64,7 +65,13 @@ def create_app(*, serve_frontend: bool = False) -> FastAPI:
         @app.get("/{full_path:path}")
         async def serve_spa(full_path: str) -> FileResponse:
             """Serve the React SPA for all non-API routes."""
-            file_path = frontend_dir / full_path
+            requested = Path(full_path)
+            if ".." in requested.parts:
+                raise HTTPException(status_code=404, detail="Not found")
+            frontend_root = frontend_dir.resolve()
+            file_path = (frontend_dir / requested).resolve()
+            if not file_path.is_relative_to(frontend_root):
+                raise HTTPException(status_code=404, detail="Not found")
             if file_path.is_file() and not full_path.startswith("api"):
                 return FileResponse(file_path)
             return FileResponse(frontend_dir / "index.html")

furu/errors.py

@@ -30,6 +30,14 @@ class FuruError(Exception):
         return "\n".join(lines)
 
 
+class FuruExecutionError(FuruError):
+    """Raised when executor wiring or scheduling fails."""
+
+
+class FuruValidationError(FuruError):
+    """Raised by `_validate()` to indicate an invalid or missing artifact."""
+
+
 class FuruWaitTimeout(FuruError):
     """Raised when waiting for a result exceeds _max_wait_time_sec."""
 
@@ -70,11 +78,8 @@ class FuruComputeError(FuruError):
         msg = super().__str__()  # ty: ignore[invalid-super-argument]
         internal_dir = self.state_path.parent
         furu_dir = internal_dir.parent
-        log_path = internal_dir / "furu.log"
 
-        msg += f"\n\nDirectory: {furu_dir}"
-        msg += f"\nState file: {self.state_path}"
-        msg += f"\nLog file: {log_path}"
+        msg += f"\n\nFuru dir: {furu_dir}"
 
         if self.recorded_error_type or self.recorded_error_message:
             msg += "\n\nRecorded error (from state.json):"
@@ -116,3 +121,11 @@ class FuruMigrationRequired(FuruError):
         if self.state_path is not None:
             msg += f"\n\nState file: {self.state_path}"
         return msg
+
+
+class FuruMissingArtifact(FuruError):
+    """Raised when a dependency is missing in executor mode."""
+
+
+class FuruSpecMismatch(FuruError):
+    """Raised when executor spec keys do not match."""

furu/execution/__init__.py

@@ -0,0 +1,22 @@
+"""Execution helpers for Furu."""
+
+from .local import run_local
+from .paths import submitit_logs_dir, submitit_root_dir
+from .slurm_dag import SlurmDagSubmission, submit_slurm_dag
+from .slurm_pool import SlurmPoolRun, run_slurm_pool
+from .slurm_spec import SlurmSpec, SlurmSpecValue, resolve_slurm_spec
+from .submitit_factory import make_executor_for_spec
+
+__all__ = [
+    "SlurmSpec",
+    "SlurmSpecValue",
+    "resolve_slurm_spec",
+    "SlurmDagSubmission",
+    "submit_slurm_dag",
+    "make_executor_for_spec",
+    "SlurmPoolRun",
+    "run_slurm_pool",
+    "run_local",
+    "submitit_logs_dir",
+    "submitit_root_dir",
+]

furu/execution/context.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from contextvars import ContextVar
+from dataclasses import dataclass
+from typing import Literal
+
+Backend = Literal["local", "submitit"]
+
+
+@dataclass(frozen=True)
+class ExecContext:
+    mode: Literal["interactive", "executor"]
+    spec_key: str | None = None
+    backend: Backend | None = None
+    current_node_hash: str | None = None
+
+
+EXEC_CONTEXT: ContextVar[ExecContext] = ContextVar(
+    "FURU_EXEC_CONTEXT",
+    default=ExecContext(
+        mode="interactive",
+        spec_key=None,
+        backend=None,
+        current_node_hash=None,
+    ),
+)
+
+
+def in_executor() -> bool:
+    return EXEC_CONTEXT.get().mode == "executor"

furu/execution/local.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+import time
+from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait
+
+from ..config import FURU_CONFIG
+from ..core import Furu
+from ..errors import FuruComputeError, FuruError
+from ..storage.state import StateManager
+from .context import EXEC_CONTEXT, ExecContext
+from .plan import PlanNode, build_plan, ready_todo
+from .plan_utils import reconcile_or_timeout_in_progress
+
+
+def _normalize_window_size(window_size: str | int, root_count: int) -> int:
+    if root_count == 0:
+        return 0
+    if isinstance(window_size, str):
+        match window_size:
+            case "dfs":
+                return 1
+            case "bfs":
+                return root_count
+            case _:
+                raise ValueError(
+                    "window_size must be 'dfs', 'bfs', or a positive integer"
+                )
+    if isinstance(window_size, bool) or not isinstance(window_size, int):
+        raise TypeError("window_size must be 'dfs', 'bfs', or a positive integer")
+    if window_size < 1:
+        raise ValueError("window_size must be >= 1")
+    return min(window_size, root_count)
+
+
+def _run_node(node: PlanNode) -> None:
+    token = EXEC_CONTEXT.set(
+        ExecContext(
+            mode="executor",
+            spec_key=node.spec_key,
+            backend="local",
+            current_node_hash=node.obj._furu_hash,
+        )
+    )
+    try:
+        node.obj.get(force=True)
+    finally:
+        EXEC_CONTEXT.reset(token)
+
+
+def run_local(
+    roots: list[Furu],
+    *,
+    max_workers: int = 8,
+    window_size: str | int = "bfs",
+    poll_interval_sec: float = 0.25,
+) -> None:
+    if not roots:
+        return
+    if max_workers < 1:
+        raise ValueError("max_workers must be >= 1")
+
+    window = _normalize_window_size(window_size, len(roots))
+    active_indices = list(range(min(window, len(roots))))
+    next_index = len(active_indices)
+    inflight: dict[str, Future[None]] = {}
+    completed_hashes: set[str] = set()
+    retry_attempts: dict[str, int] = {}
+
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        while True:
+            active_roots = [roots[index] for index in active_indices]
+            plan = build_plan(active_roots, completed_hashes=completed_hashes)
+
+            ready = [digest for digest in ready_todo(plan) if digest not in inflight]
+            available = max_workers - len(inflight)
+            for digest in ready[:available]:
+                node = plan.nodes[digest]
+                inflight[digest] = executor.submit(_run_node, node)
+
+            completed = [digest for digest, future in inflight.items() if future.done()]
+            for digest in completed:
+                future = inflight.pop(digest)
+                try:
+                    future.result()
+                except Exception as exc:
+                    if isinstance(exc, FuruComputeError):
+                        compute_error = exc
+                        wrapped_exc: Exception | None = None
+                    elif isinstance(exc, FuruError):
+                        raise
+                    else:
+                        node = plan.nodes.get(digest)
+                        if node is None:
+                            raise
+                        state_path = StateManager.get_state_path(
+                            node.obj._base_furu_dir()
+                        )
+                        compute_error = FuruComputeError(
+                            "local executor failed for "
+                            f"{node.obj.__class__.__name__}({node.obj._furu_hash})",
+                            state_path,
+                            original_error=exc,
+                        )
+                        wrapped_exc = exc
+                    if not FURU_CONFIG.retry_failed:
+                        if wrapped_exc is not None:
+                            raise compute_error from wrapped_exc
+                        raise compute_error
+                    attempt = retry_attempts.get(digest, 0) + 1
+                    retry_attempts[digest] = attempt
+                    if attempt <= FURU_CONFIG.max_compute_retries:
+                        continue
+                    if wrapped_exc is not None:
+                        raise compute_error from wrapped_exc
+                    raise compute_error
+                completed_hashes.add(digest)
+                retry_attempts.pop(digest, None)
+
+            if not FURU_CONFIG.retry_failed:
+                failed = [
+                    node
+                    for digest, node in plan.nodes.items()
+                    if node.status == "FAILED" and digest not in inflight
+                ]
+                if failed:
+                    names = ", ".join(
+                        f"{node.obj.__class__.__name__}({node.obj._furu_hash})"
+                        for node in failed
+                    )
+                    raise RuntimeError(
+                        f"Cannot run local executor with failed dependencies: {names}"
+                    )
+
+            if completed:
+                continue
+
+            # Avoid a busy-spin loop while waiting for long-running tasks.
+            if inflight and not completed:
+                wait(
+                    inflight.values(),
+                    timeout=poll_interval_sec,
+                    return_when=FIRST_COMPLETED,
+                )
+                continue
+
+            finished_indices = [
+                index
+                for index in active_indices
+                if plan.nodes.get(roots[index]._furu_hash) is not None
+                and plan.nodes[roots[index]._furu_hash].status == "DONE"
+            ]
+            for index in finished_indices:
+                active_indices.remove(index)
+
+            while len(active_indices) < window and next_index < len(roots):
+                active_indices.append(next_index)
+                next_index += 1
+
+            if not active_indices and not inflight and next_index >= len(roots):
+                return
+
+            if not inflight and not ready:
+                if any(node.status == "IN_PROGRESS" for node in plan.nodes.values()):
+                    stale_detected = reconcile_or_timeout_in_progress(
+                        plan,
+                        stale_timeout_sec=FURU_CONFIG.stale_timeout,
+                    )
+                    if stale_detected:
+                        continue
+                    time.sleep(poll_interval_sec)
+                    continue
+                todo_nodes = [
+                    node for node in plan.nodes.values() if node.status == "TODO"
+                ]
+                if todo_nodes:
+                    sample = ", ".join(
+                        f"{node.obj.__class__.__name__}({node.obj._furu_hash})"
+                        for node in todo_nodes[:3]
+                    )
+                    raise RuntimeError(
+                        "run_local stalled with no progress; "
+                        f"remaining TODO nodes: {sample}"
+                    )
+                time.sleep(poll_interval_sec)

furu/execution/paths.py

@@ -0,0 +1,20 @@
+from pathlib import Path
+
+from furu.config import FURU_CONFIG
+
+
+def submitit_root_dir(override: Path | None = None) -> Path:
+    return (override or FURU_CONFIG.get_submitit_root()).resolve()
+
+
+def submitit_logs_dir(
+    kind: str,
+    spec_key: str,
+    override: Path | None = None,
+    run_id: str | None = None,
+) -> Path:
+    root = submitit_root_dir(override)
+    path = root / kind / spec_key
+    if run_id:
+        path = path / run_id
+    return path

furu/execution/plan.py

@@ -0,0 +1,238 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+from ..config import FURU_CONFIG
+from ..core import Furu
+from ..runtime.logging import get_logger
+from ..storage.state import (
+    StateManager,
+    _StateAttemptFailed,
+    _StateAttemptQueued,
+    _StateAttemptRunning,
+    _StateResultFailed,
+)
+
+Status = Literal["DONE", "IN_PROGRESS", "TODO", "FAILED"]
+
+_MISSING_TIMESTAMP_SEEN: dict[str, float] = {}
+
+
+@dataclass
+class PlanNode:
+    obj: Furu
+    status: Status
+    spec_key: str
+    deps_all: set[str]
+    deps_pending: set[str]
+    dependents: set[str]
+
+
+@dataclass
+class DependencyPlan:
+    roots: list[Furu]
+    nodes: dict[str, PlanNode]
+
+
+def _classify(obj: Furu, completed_hashes: set[str] | None) -> Status:
+    if completed_hashes is not None and obj._furu_hash in completed_hashes:
+        return "DONE"
+    if obj._exists_quiet() and not obj._always_rerun():
+        return "DONE"
+
+    state = obj.get_state()
+    attempt = state.attempt
+    if isinstance(attempt, (_StateAttemptQueued, _StateAttemptRunning)):
+        return "IN_PROGRESS"
+    if isinstance(state.result, _StateResultFailed) or isinstance(
+        attempt, _StateAttemptFailed
+    ):
+        if FURU_CONFIG.retry_failed:
+            return "TODO"
+        return "FAILED"
+    return "TODO"
+
+
+def build_plan(
+    roots: list[Furu],
+    *,
+    completed_hashes: set[str] | None = None,
+) -> DependencyPlan:
+    nodes: dict[str, PlanNode] = {}
+    stack = list(roots)
+    seen: set[str] = set()
+
+    while stack:
+        obj = stack.pop()
+        digest = obj._furu_hash
+        if digest in seen:
+            continue
+        seen.add(digest)
+
+        status = _classify(obj, completed_hashes)
+        node = PlanNode(
+            obj=obj,
+            status=status,
+            spec_key=obj._executor_spec_key(),
+            deps_all=set(),
+            deps_pending=set(),
+            dependents=set(),
+        )
+        nodes[digest] = node
+
+        if status != "TODO":
+            continue
+
+        deps = obj._get_dependencies(recursive=False)
+        node.deps_all = {dep._furu_hash for dep in deps}
+        for dep in deps:
+            stack.append(dep)
+
+    for digest, node in nodes.items():
+        if node.status != "TODO":
+            continue
+        node.deps_pending = {
+            dep for dep in node.deps_all if dep in nodes and nodes[dep].status != "DONE"
+        }
+
+    for digest, node in nodes.items():
+        for dep in node.deps_pending:
+            nodes[dep].dependents.add(digest)
+
+    return DependencyPlan(roots=roots, nodes=nodes)
+
+
+def topo_order_todo(plan: DependencyPlan) -> list[str]:
+    todo = {digest for digest, node in plan.nodes.items() if node.status == "TODO"}
+    indeg = {digest: 0 for digest in todo}
+
+    for digest in todo:
+        node = plan.nodes[digest]
+        for dep in node.deps_pending:
+            if dep in todo:
+                indeg[digest] += 1
+
+    ready = sorted([digest for digest, deg in indeg.items() if deg == 0])
+    out: list[str] = []
+
+    while ready:
+        digest = ready.pop(0)
+        out.append(digest)
+        for dep in plan.nodes[digest].dependents:
+            if dep not in todo:
+                continue
+            indeg[dep] -= 1
+            if indeg[dep] == 0:
+                ready.append(dep)
+                ready.sort()
+
+    if len(out) != len(todo):
+        raise ValueError("Cycle detected in TODO dependency graph")
+    return out
+
+
+def ready_todo(plan: DependencyPlan) -> list[str]:
+    return sorted(
+        [
+            digest
+            for digest, node in plan.nodes.items()
+            if node.status == "TODO"
+            and all(plan.nodes[dep].status == "DONE" for dep in node.deps_pending)
+        ]
+    )
+
+
+def _attempt_age_sec(
+    attempt: _StateAttemptQueued | _StateAttemptRunning,
+    *,
+    updated_at: str | None,
+    stale_timeout_sec: float,
+    digest: str,
+    name: str,
+) -> float | None:
+    timestamp = attempt.heartbeat_at
+    if attempt.status == "queued":
+        timestamp = attempt.started_at
+    parsed = StateManager._parse_time(timestamp)
+    if parsed is None:
+        parsed = StateManager._parse_time(updated_at)
+    if parsed is not None:
+        _MISSING_TIMESTAMP_SEEN.pop(digest, None)
+        return (StateManager._utcnow() - parsed).total_seconds()
+    if stale_timeout_sec <= 0:
+        return None
+    now = StateManager._utcnow().timestamp()
+    first_seen = _MISSING_TIMESTAMP_SEEN.get(digest)
+    if first_seen is None:
+        _MISSING_TIMESTAMP_SEEN[digest] = now
+        logger = get_logger()
+        logger.warning(
+            "IN_PROGRESS attempt missing heartbeat/started timestamps for %s; "
+            "deferring stale timeout check.",
+            name,
+        )
+        return None
+    return now - first_seen
+
+
+def reconcile_in_progress(
+    plan: DependencyPlan,
+    *,
+    stale_timeout_sec: float,
+) -> bool:
+    stale_attempts: list[
+        tuple[PlanNode, _StateAttemptQueued | _StateAttemptRunning]
+    ] = []
+    for node in plan.nodes.values():
+        if node.status != "IN_PROGRESS":
+            _MISSING_TIMESTAMP_SEEN.pop(node.obj._furu_hash, None)
+            continue
+        state = StateManager.reconcile(node.obj._base_furu_dir())
+        attempt = state.attempt
+        if not isinstance(attempt, (_StateAttemptQueued, _StateAttemptRunning)):
+            _MISSING_TIMESTAMP_SEEN.pop(node.obj._furu_hash, None)
+            continue
+        if stale_timeout_sec <= 0:
+            continue
+        name = f"{node.obj.__class__.__name__}({node.obj._furu_hash})"
+        age = _attempt_age_sec(
+            attempt,
+            updated_at=state.updated_at,
+            stale_timeout_sec=stale_timeout_sec,
+            digest=node.obj._furu_hash,
+            name=name,
+        )
+        if age is None or age < stale_timeout_sec:
+            continue
+        stale_attempts.append((node, attempt))
+
+    if not stale_attempts:
+        return False
+
+    names = ", ".join(
+        f"{node.obj.__class__.__name__}({node.obj._furu_hash})"
+        for node, _attempt in stale_attempts
+    )
+    if not FURU_CONFIG.retry_failed:
+        raise RuntimeError(
+            "Stale IN_PROGRESS dependencies detected: "
+            f"{names} exceeded {stale_timeout_sec:.1f}s without heartbeat."
+        )
+
+    stale_detected = False
+    for node, attempt in stale_attempts:
+        stale_detected = True
+        StateManager.finish_attempt_preempted(
+            node.obj._base_furu_dir(),
+            attempt_id=attempt.id,
+            error={
+                "type": "StaleHeartbeat",
+                "message": (
+                    f"Attempt stale after {stale_timeout_sec:.1f}s without heartbeat."
+                ),
+            },
+            reason="stale_timeout",
+        )
+        _MISSING_TIMESTAMP_SEEN.pop(node.obj._furu_hash, None)
+    return stale_detected

furu/execution/plan_utils.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from .plan import DependencyPlan, reconcile_in_progress
+
+
+def reconcile_or_timeout_in_progress(
+    plan: DependencyPlan,
+    *,
+    stale_timeout_sec: float,
+) -> bool:
+    if not any(node.status == "IN_PROGRESS" for node in plan.nodes.values()):
+        return False
+    return reconcile_in_progress(plan, stale_timeout_sec=stale_timeout_sec)