furu 0.0.3__py3-none-any.whl → 0.0.5__py3-none-any.whl

furu/execution/local.py

@@ -0,0 +1,186 @@
+from __future__ import annotations
+
+import time
+from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait
+
+from ..config import FURU_CONFIG
+from ..core import Furu
+from ..errors import FuruComputeError, FuruError
+from ..runtime.logging import enter_holder
+from ..storage.state import StateManager
+from .context import EXEC_CONTEXT, ExecContext
+from .plan import PlanNode, build_plan, ready_todo
+from .plan_utils import reconcile_or_timeout_in_progress
+
+
+def _normalize_window_size(window_size: str | int, root_count: int) -> int:
+    if root_count == 0:
+        return 0
+    if isinstance(window_size, str):
+        match window_size:
+            case "dfs":
+                return 1
+            case "bfs":
+                return root_count
+            case _:
+                raise ValueError(
+                    "window_size must be 'dfs', 'bfs', or a positive integer"
+                )
+    if isinstance(window_size, bool) or not isinstance(window_size, int):
+        raise TypeError("window_size must be 'dfs', 'bfs', or a positive integer")
+    if window_size < 1:
+        raise ValueError("window_size must be >= 1")
+    return min(window_size, root_count)
+
+
+def _run_node(node: PlanNode) -> None:
+    token = EXEC_CONTEXT.set(
+        ExecContext(
+            mode="executor",
+            spec_key=node.spec_key,
+            backend="local",
+            current_node_hash=node.obj.furu_hash,
+        )
+    )
+    try:
+        with enter_holder(node.obj):
+            node.obj.get(force=True)
+    finally:
+        EXEC_CONTEXT.reset(token)
+
+
+def run_local(
+    roots: list[Furu],
+    *,
+    max_workers: int = 8,
+    window_size: str | int = "bfs",
+    poll_interval_sec: float = 0.25,
+) -> None:
+    if not roots:
+        return
+    if max_workers < 1:
+        raise ValueError("max_workers must be >= 1")
+
+    window = _normalize_window_size(window_size, len(roots))
+    active_indices = list(range(min(window, len(roots))))
+    next_index = len(active_indices)
+    inflight: dict[str, Future[None]] = {}
+    completed_hashes: set[str] = set()
+    retry_attempts: dict[str, int] = {}
+
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        while True:
+            active_roots = [roots[index] for index in active_indices]
+            plan = build_plan(active_roots, completed_hashes=completed_hashes)
+
+            ready = [digest for digest in ready_todo(plan) if digest not in inflight]
+            available = max_workers - len(inflight)
+            for digest in ready[:available]:
+                node = plan.nodes[digest]
+                inflight[digest] = executor.submit(_run_node, node)
+
+            completed = [digest for digest, future in inflight.items() if future.done()]
+            for digest in completed:
+                future = inflight.pop(digest)
+                try:
+                    future.result()
+                except Exception as exc:
+                    if isinstance(exc, FuruComputeError):
+                        compute_error = exc
+                        wrapped_exc: Exception | None = None
+                    elif isinstance(exc, FuruError):
+                        raise
+                    else:
+                        node = plan.nodes.get(digest)
+                        if node is None:
+                            raise
+                        state_path = StateManager.get_state_path(
+                            node.obj._base_furu_dir()
+                        )
+                        compute_error = FuruComputeError(
+                            "local executor failed for "
+                            f"{node.obj.__class__.__name__}({node.obj.furu_hash})",
+                            state_path,
+                            original_error=exc,
+                        )
+                        wrapped_exc = exc
+                    if not FURU_CONFIG.retry_failed:
+                        if wrapped_exc is not None:
+                            raise compute_error from wrapped_exc
+                        raise compute_error
+                    attempt = retry_attempts.get(digest, 0) + 1
+                    retry_attempts[digest] = attempt
+                    if attempt <= FURU_CONFIG.max_compute_retries:
+                        continue
+                    if wrapped_exc is not None:
+                        raise compute_error from wrapped_exc
+                    raise compute_error
+                completed_hashes.add(digest)
+                retry_attempts.pop(digest, None)
+
+            if not FURU_CONFIG.retry_failed:
+                failed = [
+                    node
+                    for digest, node in plan.nodes.items()
+                    if node.status == "FAILED" and digest not in inflight
+                ]
+                if failed:
+                    names = ", ".join(
+                        f"{node.obj.__class__.__name__}({node.obj.furu_hash})"
+                        for node in failed
+                    )
+                    raise RuntimeError(
+                        f"Cannot run local executor with failed dependencies: {names}"
+                    )
+
+            if completed:
+                continue
+
+            # Avoid a busy-spin loop while waiting for long-running tasks.
+            if inflight and not completed:
+                wait(
+                    inflight.values(),
+                    timeout=poll_interval_sec,
+                    return_when=FIRST_COMPLETED,
+                )
+                continue
+
+            finished_indices = [
+                index
+                for index in active_indices
+                if plan.nodes.get(roots[index].furu_hash) is not None
+                and plan.nodes[roots[index].furu_hash].status == "DONE"
+            ]
+            for index in finished_indices:
+                active_indices.remove(index)
+
+            while len(active_indices) < window and next_index < len(roots):
+                active_indices.append(next_index)
+                next_index += 1
+
+            if not active_indices and not inflight and next_index >= len(roots):
+                return
+
+            if not inflight and not ready:
+                if any(node.status == "IN_PROGRESS" for node in plan.nodes.values()):
+                    stale_detected = reconcile_or_timeout_in_progress(
+                        plan,
+                        stale_timeout_sec=FURU_CONFIG.stale_timeout,
+                    )
+                    if stale_detected:
+                        continue
+                    time.sleep(poll_interval_sec)
+                    continue
+                todo_nodes = [
+                    node for node in plan.nodes.values() if node.status == "TODO"
+                ]
+                if todo_nodes:
+                    sample = ", ".join(
+                        f"{node.obj.__class__.__name__}({node.obj.furu_hash})"
+                        for node in todo_nodes[:3]
+                    )
+                    raise RuntimeError(
+                        "run_local stalled with no progress; "
+                        f"remaining TODO nodes: {sample}"
+                    )
+                time.sleep(poll_interval_sec)

furu/execution/paths.py

@@ -0,0 +1,20 @@
+from pathlib import Path
+
+from furu.config import FURU_CONFIG
+
+
+def submitit_root_dir(override: Path | None = None) -> Path:
+    return (override or FURU_CONFIG.get_submitit_root()).resolve()
+
+
+def submitit_logs_dir(
+    kind: str,
+    spec_key: str,
+    override: Path | None = None,
+    run_id: str | None = None,
+) -> Path:
+    root = submitit_root_dir(override)
+    path = root / kind / spec_key
+    if run_id:
+        path = path / run_id
+    return path

furu/execution/plan.py

@@ -0,0 +1,330 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+import time
+from pathlib import Path
+from typing import Literal
+
+from ..config import FURU_CONFIG
+from ..core import Furu
+from ..errors import FuruValidationError
+from ..runtime.logging import get_logger
+from ..storage.migration import MigrationManager, MigrationRecord
+from ..storage.state import (
+    StateManager,
+    _StateAttemptFailed,
+    _StateAttemptQueued,
+    _StateAttemptRunning,
+    _FuruState,
+    _StateResultFailed,
+)
+
+Status = Literal["DONE", "IN_PROGRESS", "TODO", "FAILED"]
+
+_MISSING_TIMESTAMP_SEEN: dict[str, float] = {}
+
+
+@dataclass
+class PlanNode:
+    obj: Furu
+    status: Status
+    spec_key: str
+    deps_all: set[str]
+    deps_pending: set[str]
+    dependents: set[str]
+
+
+@dataclass
+class DependencyPlan:
+    roots: list[Furu]
+    nodes: dict[str, PlanNode]
+
+
+@dataclass
+class _PlanCache:
+    migration_records: dict[Path, MigrationRecord | None]
+    alias_targets: dict[Path, Path | None]
+    marker_exists: dict[Path, bool]
+    states: dict[Path, _FuruState]
+
+
+def _marker_exists(directory: Path, cache: _PlanCache) -> bool:
+    if directory in cache.marker_exists:
+        return cache.marker_exists[directory]
+    exists = StateManager.success_marker_exists(directory)
+    cache.marker_exists[directory] = exists
+    return exists
+
+
+def _migration_record(directory: Path, cache: _PlanCache) -> MigrationRecord | None:
+    if directory not in cache.migration_records:
+        cache.migration_records[directory] = MigrationManager.read_migration(directory)
+    return cache.migration_records[directory]
+
+
+def _alias_target_dir(base_dir: Path, cache: _PlanCache) -> Path | None:
+    if base_dir in cache.alias_targets:
+        return cache.alias_targets[base_dir]
+    record = _migration_record(base_dir, cache)
+    if record is None or record.kind != "alias" or record.overwritten_at is not None:
+        cache.alias_targets[base_dir] = None
+        return None
+    if _marker_exists(base_dir, cache):
+        cache.alias_targets[base_dir] = None
+        return None
+    target_dir = MigrationManager.resolve_dir(record, target="from")
+    if _marker_exists(target_dir, cache):
+        cache.alias_targets[base_dir] = target_dir
+        return target_dir
+    cache.alias_targets[base_dir] = None
+    return None
+
+
+def _state_for(directory: Path, cache: _PlanCache) -> _FuruState:
+    if directory not in cache.states:
+        cache.states[directory] = StateManager.read_state(directory)
+    return cache.states[directory]
+
+
+def _validate_cached(obj: Furu, *, directory: Path) -> bool:
+    try:
+        return obj._validate()
+    except FuruValidationError as exc:
+        logger = get_logger()
+        logger.warning(
+            "exists %s -> false (validate invalid for %s: %s)",
+            directory,
+            f"{obj.__class__.__name__}({obj.furu_hash})",
+            exc,
+        )
+        return False
+    except Exception as exc:
+        logger = get_logger()
+        logger.exception(
+            "exists %s -> false (validate crashed for %s: %s)",
+            directory,
+            f"{obj.__class__.__name__}({obj.furu_hash})",
+            exc,
+        )
+        return False
+
+
+def _classify(
+    obj: Furu,
+    completed_hashes: set[str] | None,
+    cache: _PlanCache,
+) -> Status:
+    if completed_hashes is not None and obj.furu_hash in completed_hashes:
+        return "DONE"
+    base_dir = obj._base_furu_dir()
+    alias_target = None
+    if not obj._always_rerun():
+        alias_target = _alias_target_dir(base_dir, cache)
+        success_dir = alias_target or base_dir
+        if _marker_exists(success_dir, cache):
+            if _validate_cached(obj, directory=base_dir):
+                return "DONE"
+
+    state_dir = alias_target or base_dir
+    state = _state_for(state_dir, cache)
+    attempt = state.attempt
+    if isinstance(attempt, (_StateAttemptQueued, _StateAttemptRunning)):
+        return "IN_PROGRESS"
+    if isinstance(state.result, _StateResultFailed) or isinstance(
+        attempt, _StateAttemptFailed
+    ):
+        if FURU_CONFIG.retry_failed:
+            return "TODO"
+        return "FAILED"
+    return "TODO"
+
+
+def build_plan(
+    roots: list[Furu],
+    *,
+    completed_hashes: set[str] | None = None,
+) -> DependencyPlan:
+    cache = _PlanCache(
+        migration_records={},
+        alias_targets={},
+        marker_exists={},
+        states={},
+    )
+    nodes: dict[str, PlanNode] = {}
+    stack = list(roots)
+    seen: set[str] = set()
+
+    while stack:
+        obj = stack.pop()
+        digest = obj.furu_hash
+        if digest in seen:
+            continue
+        seen.add(digest)
+
+        status = _classify(obj, completed_hashes, cache)
+        node = PlanNode(
+            obj=obj,
+            status=status,
+            spec_key=obj._executor_spec_key(),
+            deps_all=set(),
+            deps_pending=set(),
+            dependents=set(),
+        )
+        nodes[digest] = node
+
+        if status != "TODO":
+            continue
+
+        deps = obj._get_dependencies(recursive=False)
+        node.deps_all = {dep.furu_hash for dep in deps}
+        for dep in deps:
+            stack.append(dep)
+
+    for digest, node in nodes.items():
+        if node.status != "TODO":
+            continue
+        node.deps_pending = {
+            dep for dep in node.deps_all if dep in nodes and nodes[dep].status != "DONE"
+        }
+
+    for digest, node in nodes.items():
+        for dep in node.deps_pending:
+            nodes[dep].dependents.add(digest)
+
+    return DependencyPlan(roots=roots, nodes=nodes)
+
+
+def topo_order_todo(plan: DependencyPlan) -> list[str]:
+    todo = {digest for digest, node in plan.nodes.items() if node.status == "TODO"}
+    indeg = {digest: 0 for digest in todo}
+
+    for digest in todo:
+        node = plan.nodes[digest]
+        for dep in node.deps_pending:
+            if dep in todo:
+                indeg[digest] += 1
+
+    ready = sorted([digest for digest, deg in indeg.items() if deg == 0])
+    out: list[str] = []
+
+    while ready:
+        digest = ready.pop(0)
+        out.append(digest)
+        for dep in plan.nodes[digest].dependents:
+            if dep not in todo:
+                continue
+            indeg[dep] -= 1
+            if indeg[dep] == 0:
+                ready.append(dep)
+                ready.sort()
+
+    if len(out) != len(todo):
+        raise ValueError("Cycle detected in TODO dependency graph")
+    return out
+
+
+def ready_todo(plan: DependencyPlan) -> list[str]:
+    return sorted(
+        [
+            digest
+            for digest, node in plan.nodes.items()
+            if node.status == "TODO"
+            and all(plan.nodes[dep].status == "DONE" for dep in node.deps_pending)
+        ]
+    )
+
+
+def _attempt_age_sec(
+    attempt: _StateAttemptQueued | _StateAttemptRunning,
+    *,
+    directory: Path,
+    stale_timeout_sec: float,
+    digest: str,
+    name: str,
+) -> float | None:
+    if attempt.status == "queued":
+        parsed = StateManager._parse_time(attempt.started_at)
+        if parsed is not None:
+            _MISSING_TIMESTAMP_SEEN.pop(digest, None)
+            return (StateManager._utcnow() - parsed).total_seconds()
+    else:
+        last_heartbeat = StateManager.last_heartbeat_mtime(directory)
+        if last_heartbeat is not None:
+            _MISSING_TIMESTAMP_SEEN.pop(digest, None)
+            return max(0.0, time.time() - last_heartbeat)
+    if stale_timeout_sec <= 0:
+        return None
+    now = StateManager._utcnow().timestamp()
+    first_seen = _MISSING_TIMESTAMP_SEEN.get(digest)
+    if first_seen is None:
+        _MISSING_TIMESTAMP_SEEN[digest] = now
+        logger = get_logger()
+        logger.warning(
+            "IN_PROGRESS attempt missing heartbeat/started timestamps for %s; "
+            "deferring stale timeout check.",
+            name,
+        )
+        return None
+    return now - first_seen
+
+
+def reconcile_in_progress(
+    plan: DependencyPlan,
+    *,
+    stale_timeout_sec: float,
+) -> bool:
+    stale_attempts: list[
+        tuple[PlanNode, _StateAttemptQueued | _StateAttemptRunning]
+    ] = []
+    for node in plan.nodes.values():
+        if node.status != "IN_PROGRESS":
+            _MISSING_TIMESTAMP_SEEN.pop(node.obj.furu_hash, None)
+            continue
+        state = StateManager.reconcile(node.obj._base_furu_dir())
+        attempt = state.attempt
+        if not isinstance(attempt, (_StateAttemptQueued, _StateAttemptRunning)):
+            _MISSING_TIMESTAMP_SEEN.pop(node.obj.furu_hash, None)
+            continue
+        if stale_timeout_sec <= 0:
+            continue
+        name = f"{node.obj.__class__.__name__}({node.obj.furu_hash})"
+        age = _attempt_age_sec(
+            attempt,
+            directory=node.obj._base_furu_dir(),
+            stale_timeout_sec=stale_timeout_sec,
+            digest=node.obj.furu_hash,
+            name=name,
+        )
+        if age is None or age < stale_timeout_sec:
+            continue
+        stale_attempts.append((node, attempt))
+
+    if not stale_attempts:
+        return False
+
+    names = ", ".join(
+        f"{node.obj.__class__.__name__}({node.obj.furu_hash})"
+        for node, _attempt in stale_attempts
+    )
+    if not FURU_CONFIG.retry_failed:
+        raise RuntimeError(
+            "Stale IN_PROGRESS dependencies detected: "
+            f"{names} exceeded {stale_timeout_sec:.1f}s without heartbeat."
+        )
+
+    stale_detected = False
+    for node, attempt in stale_attempts:
+        stale_detected = True
+        StateManager.finish_attempt_preempted(
+            node.obj._base_furu_dir(),
+            attempt_id=attempt.id,
+            error={
+                "type": "StaleHeartbeat",
+                "message": (
+                    f"Attempt stale after {stale_timeout_sec:.1f}s without heartbeat."
+                ),
+            },
+            reason="stale_timeout",
+        )
+        _MISSING_TIMESTAMP_SEEN.pop(node.obj.furu_hash, None)
+    return stale_detected

furu/execution/plan_utils.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from .plan import DependencyPlan, reconcile_in_progress
+
+
+def reconcile_or_timeout_in_progress(
+    plan: DependencyPlan,
+    *,
+    stale_timeout_sec: float,
+) -> bool:
+    if not any(node.status == "IN_PROGRESS" for node in plan.nodes.values()):
+        return False
+    return reconcile_in_progress(plan, stale_timeout_sec=stale_timeout_sec)