furu 0.0.4__py3-none-any.whl → 0.0.5__py3-none-any.whl

furu/execution/plan.py

@@ -1,16 +1,21 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
+import time
+from pathlib import Path
 from typing import Literal
 
 from ..config import FURU_CONFIG
 from ..core import Furu
+from ..errors import FuruValidationError
 from ..runtime.logging import get_logger
+from ..storage.migration import MigrationManager, MigrationRecord
 from ..storage.state import (
     StateManager,
     _StateAttemptFailed,
     _StateAttemptQueued,
     _StateAttemptRunning,
+    _FuruState,
     _StateResultFailed,
 )
 
@@ -35,13 +40,93 @@ class DependencyPlan:
     nodes: dict[str, PlanNode]
 
 
-def _classify(obj: Furu, completed_hashes: set[str] | None) -> Status:
-    if completed_hashes is not None and obj._furu_hash in completed_hashes:
-        return "DONE"
-    if obj._exists_quiet() and not obj._always_rerun():
-        return "DONE"
+@dataclass
+class _PlanCache:
+    migration_records: dict[Path, MigrationRecord | None]
+    alias_targets: dict[Path, Path | None]
+    marker_exists: dict[Path, bool]
+    states: dict[Path, _FuruState]
+
+
+def _marker_exists(directory: Path, cache: _PlanCache) -> bool:
+    if directory in cache.marker_exists:
+        return cache.marker_exists[directory]
+    exists = StateManager.success_marker_exists(directory)
+    cache.marker_exists[directory] = exists
+    return exists
+
+
+def _migration_record(directory: Path, cache: _PlanCache) -> MigrationRecord | None:
+    if directory not in cache.migration_records:
+        cache.migration_records[directory] = MigrationManager.read_migration(directory)
+    return cache.migration_records[directory]
+
+
+def _alias_target_dir(base_dir: Path, cache: _PlanCache) -> Path | None:
+    if base_dir in cache.alias_targets:
+        return cache.alias_targets[base_dir]
+    record = _migration_record(base_dir, cache)
+    if record is None or record.kind != "alias" or record.overwritten_at is not None:
+        cache.alias_targets[base_dir] = None
+        return None
+    if _marker_exists(base_dir, cache):
+        cache.alias_targets[base_dir] = None
+        return None
+    target_dir = MigrationManager.resolve_dir(record, target="from")
+    if _marker_exists(target_dir, cache):
+        cache.alias_targets[base_dir] = target_dir
+        return target_dir
+    cache.alias_targets[base_dir] = None
+    return None
+
+
+def _state_for(directory: Path, cache: _PlanCache) -> _FuruState:
+    if directory not in cache.states:
+        cache.states[directory] = StateManager.read_state(directory)
+    return cache.states[directory]
 
-    state = obj.get_state()
+
+def _validate_cached(obj: Furu, *, directory: Path) -> bool:
+    try:
+        return obj._validate()
+    except FuruValidationError as exc:
+        logger = get_logger()
+        logger.warning(
+            "exists %s -> false (validate invalid for %s: %s)",
+            directory,
+            f"{obj.__class__.__name__}({obj.furu_hash})",
+            exc,
+        )
+        return False
+    except Exception as exc:
+        logger = get_logger()
+        logger.exception(
+            "exists %s -> false (validate crashed for %s: %s)",
+            directory,
+            f"{obj.__class__.__name__}({obj.furu_hash})",
+            exc,
+        )
+        return False
+
+
+def _classify(
+    obj: Furu,
+    completed_hashes: set[str] | None,
+    cache: _PlanCache,
+) -> Status:
+    if completed_hashes is not None and obj.furu_hash in completed_hashes:
+        return "DONE"
+    base_dir = obj._base_furu_dir()
+    alias_target = None
+    if not obj._always_rerun():
+        alias_target = _alias_target_dir(base_dir, cache)
+        success_dir = alias_target or base_dir
+        if _marker_exists(success_dir, cache):
+            if _validate_cached(obj, directory=base_dir):
+                return "DONE"
+
+    state_dir = alias_target or base_dir
+    state = _state_for(state_dir, cache)
     attempt = state.attempt
     if isinstance(attempt, (_StateAttemptQueued, _StateAttemptRunning)):
         return "IN_PROGRESS"
@@ -59,18 +144,24 @@ def build_plan(
     *,
     completed_hashes: set[str] | None = None,
 ) -> DependencyPlan:
+    cache = _PlanCache(
+        migration_records={},
+        alias_targets={},
+        marker_exists={},
+        states={},
+    )
     nodes: dict[str, PlanNode] = {}
     stack = list(roots)
     seen: set[str] = set()
 
     while stack:
         obj = stack.pop()
-        digest = obj._furu_hash
+        digest = obj.furu_hash
         if digest in seen:
             continue
         seen.add(digest)
 
-        status = _classify(obj, completed_hashes)
+        status = _classify(obj, completed_hashes, cache)
         node = PlanNode(
             obj=obj,
             status=status,
@@ -85,7 +176,7 @@ def build_plan(
             continue
 
         deps = obj._get_dependencies(recursive=False)
-        node.deps_all = {dep._furu_hash for dep in deps}
+        node.deps_all = {dep.furu_hash for dep in deps}
         for dep in deps:
             stack.append(dep)
 
@@ -146,20 +237,21 @@ def ready_todo(plan: DependencyPlan) -> list[str]:
 def _attempt_age_sec(
     attempt: _StateAttemptQueued | _StateAttemptRunning,
     *,
-    updated_at: str | None,
+    directory: Path,
     stale_timeout_sec: float,
     digest: str,
     name: str,
 ) -> float | None:
-    timestamp = attempt.heartbeat_at
     if attempt.status == "queued":
-        timestamp = attempt.started_at
-    parsed = StateManager._parse_time(timestamp)
-    if parsed is None:
-        parsed = StateManager._parse_time(updated_at)
-    if parsed is not None:
-        _MISSING_TIMESTAMP_SEEN.pop(digest, None)
-        return (StateManager._utcnow() - parsed).total_seconds()
+        parsed = StateManager._parse_time(attempt.started_at)
+        if parsed is not None:
+            _MISSING_TIMESTAMP_SEEN.pop(digest, None)
+            return (StateManager._utcnow() - parsed).total_seconds()
+    else:
+        last_heartbeat = StateManager.last_heartbeat_mtime(directory)
+        if last_heartbeat is not None:
+            _MISSING_TIMESTAMP_SEEN.pop(digest, None)
+            return max(0.0, time.time() - last_heartbeat)
     if stale_timeout_sec <= 0:
         return None
     now = StateManager._utcnow().timestamp()
@@ -186,21 +278,21 @@ def reconcile_in_progress(
     ] = []
     for node in plan.nodes.values():
         if node.status != "IN_PROGRESS":
-            _MISSING_TIMESTAMP_SEEN.pop(node.obj._furu_hash, None)
+            _MISSING_TIMESTAMP_SEEN.pop(node.obj.furu_hash, None)
             continue
         state = StateManager.reconcile(node.obj._base_furu_dir())
         attempt = state.attempt
         if not isinstance(attempt, (_StateAttemptQueued, _StateAttemptRunning)):
-            _MISSING_TIMESTAMP_SEEN.pop(node.obj._furu_hash, None)
+            _MISSING_TIMESTAMP_SEEN.pop(node.obj.furu_hash, None)
             continue
         if stale_timeout_sec <= 0:
             continue
-        name = f"{node.obj.__class__.__name__}({node.obj._furu_hash})"
+        name = f"{node.obj.__class__.__name__}({node.obj.furu_hash})"
         age = _attempt_age_sec(
             attempt,
-            updated_at=state.updated_at,
+            directory=node.obj._base_furu_dir(),
             stale_timeout_sec=stale_timeout_sec,
-            digest=node.obj._furu_hash,
+            digest=node.obj.furu_hash,
             name=name,
         )
         if age is None or age < stale_timeout_sec:
@@ -211,7 +303,7 @@ def reconcile_in_progress(
         return False
 
     names = ", ".join(
-        f"{node.obj.__class__.__name__}({node.obj._furu_hash})"
+        f"{node.obj.__class__.__name__}({node.obj.furu_hash})"
         for node, _attempt in stale_attempts
     )
     if not FURU_CONFIG.retry_failed:
@@ -234,5 +326,5 @@ def reconcile_in_progress(
             },
             reason="stale_timeout",
         )
-        _MISSING_TIMESTAMP_SEEN.pop(node.obj._furu_hash, None)
+        _MISSING_TIMESTAMP_SEEN.pop(node.obj.furu_hash, None)
     return stale_detected

furu/execution/slurm_dag.py

@@ -48,24 +48,24 @@ def _attempt_is_terminal(obj: Furu, directory: Path | None = None) -> bool:
 def _set_submitit_job_id(directory: Path, job_id: str) -> bool:
     updated = False
 
-    def mutate(state: _FuruState) -> None:
+    def mutate(state: _FuruState) -> bool:
         nonlocal updated
         attempt = state.attempt
         if attempt is None:
-            return
+            return False
         if attempt.backend != "submitit":
-            return
+            return False
         if (
             attempt.status not in {"queued", "running"}
             and attempt.status not in StateManager.TERMINAL_STATUSES
         ):
-            return
+            return False
         existing = attempt.scheduler.get("job_id")
         if existing == job_id:
-            updated = True
-            return
+            return False
         attempt.scheduler["job_id"] = job_id
         updated = True
+        return True
 
     StateManager.update_state(directory, mutate)
     return updated
@@ -117,7 +117,7 @@ def _wait_for_job_id(
             suffix = f" Last seen job_id={last_job_id}." if last_job_id else ""
             raise TimeoutError(
                 "Timed out waiting for submitit job_id for "
-                f"{obj.__class__.__name__} ({obj._furu_hash}).{suffix}"
+                f"{obj.__class__.__name__} ({obj.furu_hash}).{suffix}"
             )
 
         time.sleep(poll_interval_sec)
@@ -129,7 +129,7 @@ def _job_id_for_in_progress(obj: Furu) -> str:
     if attempt is None:
         raise RuntimeError(
             "Cannot wire Slurm DAG dependency for IN_PROGRESS "
-            f"{obj.__class__.__name__} ({obj._furu_hash}) without an attempt."
+            f"{obj.__class__.__name__} ({obj.furu_hash}) without an attempt."
         )
     if attempt.backend != "submitit":
         raise FuruExecutionError(
@@ -144,7 +144,7 @@ def _job_id_for_in_progress(obj: Furu) -> str:
     ):
         raise FuruExecutionError(
             "Cannot wire afterok dependency to a terminal non-success dependency. "
-            f"Dependency {obj.__class__.__name__} ({obj._furu_hash}) status={attempt.status}."
+            f"Dependency {obj.__class__.__name__} ({obj.furu_hash}) status={attempt.status}."
         )
 
     job_id = attempt.scheduler.get("job_id")
@@ -160,10 +160,12 @@ def _job_id_for_in_progress(obj: Furu) -> str:
     state2 = obj.get_state()
     attempt2 = state2.attempt
     if attempt2 is not None and attempt2.status in StateManager.TERMINAL_STATUSES:
-        if attempt2.status != "success" or isinstance(state2.result, _StateResultFailed):
+        if attempt2.status != "success" or isinstance(
+            state2.result, _StateResultFailed
+        ):
             raise FuruExecutionError(
                 "Cannot wire afterok dependency: dependency became terminal and did not succeed. "
-                f"Dependency {obj.__class__.__name__} ({obj._furu_hash}) status={attempt2.status} "
+                f"Dependency {obj.__class__.__name__} ({obj.furu_hash}) status={attempt2.status} "
                 f"job_id={resolved}."
             )
 
@@ -190,7 +192,7 @@ def submit_slurm_dag(
     failed = [node for node in plan.nodes.values() if node.status == "FAILED"]
     if failed:
         names = ", ".join(
-            f"{node.obj.__class__.__name__}({node.obj._furu_hash})" for node in failed
+            f"{node.obj.__class__.__name__}({node.obj.furu_hash})" for node in failed
         )
         raise RuntimeError(f"Cannot submit slurm DAG with failed dependencies: {names}")
 
@@ -198,7 +200,7 @@ def submit_slurm_dag(
     job_id_by_hash: dict[str, str] = {}
     root_job_ids: dict[str, str] = {}
 
-    root_hashes = {root._furu_hash for root in roots}
+    root_hashes = {root.furu_hash for root in roots}
 
     for digest in order:
         node = plan.nodes[digest]
@@ -254,7 +256,7 @@ def submit_slurm_dag(
             root_job_ids[digest] = job_id
 
     for root in roots:
-        digest = root._furu_hash
+        digest = root.furu_hash
         if digest in root_job_ids:
             continue
         node = plan.nodes.get(digest)

furu/execution/slurm_pool.py

@@ -299,7 +299,7 @@ def _missing_spec_keys(
         if node.spec_key in specs:
             continue
         missing.setdefault(node.spec_key, []).append(
-            f"{node.obj.__class__.__name__}({node.obj._furu_hash})"
+            f"{node.obj.__class__.__name__}({node.obj.furu_hash})"
         )
     return missing
 
@@ -777,7 +777,7 @@ def run_slurm_pool(
             failed = [node for node in plan.nodes.values() if node.status == "FAILED"]
             if failed:
                 names = ", ".join(
-                    f"{node.obj.__class__.__name__}({node.obj._furu_hash})"
+                    f"{node.obj.__class__.__name__}({node.obj.furu_hash})"
                     for node in failed
                 )
                 raise RuntimeError(
@@ -834,8 +834,8 @@ def run_slurm_pool(
         finished_indices = [
             index
             for index in active_indices
-            if plan.nodes.get(roots[index]._furu_hash) is not None
-            and plan.nodes[roots[index]._furu_hash].status == "DONE"
+            if plan.nodes.get(roots[index].furu_hash) is not None
+            and plan.nodes[roots[index].furu_hash].status == "DONE"
         ]
         for index in finished_indices:
             active_indices.remove(index)
@@ -859,7 +859,7 @@ def run_slurm_pool(
             todo_nodes = [node for node in plan.nodes.values() if node.status == "TODO"]
             if todo_nodes:
                 sample = ", ".join(
-                    f"{node.obj.__class__.__name__}({node.obj._furu_hash})"
+                    f"{node.obj.__class__.__name__}({node.obj.furu_hash})"
                     for node in todo_nodes[:3]
                 )
                 raise RuntimeError(

furu/execution/slurm_spec.py

@@ -19,7 +19,7 @@ class SlurmSpec:
 
 
 class _SpecNode(Protocol):
-    _furu_hash: str
+    furu_hash: str
 
     def _executor_spec_key(self) -> str: ...
 
@@ -32,7 +32,7 @@ def resolve_slurm_spec(specs: Mapping[str, SlurmSpec], node: _SpecNode) -> Slurm
     if spec_key not in specs:
         raise KeyError(
             "Missing slurm spec for key "
-            f"'{spec_key}' for node {node.__class__.__name__} ({node._furu_hash})."
+            f"'{spec_key}' for node {node.__class__.__name__} ({node.furu_hash})."
         )
 
     return specs[spec_key]

furu/migration.py

@@ -456,6 +456,7 @@ def _apply_single_migration(
         shutil.rmtree(to_dir)
 
     to_dir.mkdir(parents=True, exist_ok=True)
+    StateManager.ensure_internal_dir(to_dir)
     now = _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds")
 
     if policy in {"move", "copy"}:
@@ -557,8 +558,6 @@ def _copy_state(from_dir: Path, to_dir: Path, *, clear_source: bool) -> None:
     src_internal = from_dir / StateManager.INTERNAL_DIR
     if not src_internal.exists():
         return
-    dst_internal = to_dir / StateManager.INTERNAL_DIR
-    dst_internal.mkdir(parents=True, exist_ok=True)
     state_path = StateManager.get_state_path(from_dir)
     if state_path.is_file():
         shutil.copy2(state_path, StateManager.get_state_path(to_dir))

furu/runtime/env.py

@@ -5,4 +5,4 @@ def load_env() -> None:
 
 
 # Preserve previous behavior: attempt to load `.env` at import-time.
-load_env()
+# load_env() # TODO: find a nice way to auto load .env if needed

furu/runtime/logging.py

@@ -4,6 +4,7 @@ import datetime
 import logging
 import os
 import threading
+import sys
 from pathlib import Path
 from typing import Generator, Protocol
 
@@ -102,6 +103,16 @@ class _FuruLogFormatter(logging.Formatter):
         dt = datetime.datetime.fromtimestamp(record.created, tz=datetime.timezone.utc)
         return dt.isoformat(timespec="seconds")
 
+    def format(self, record: logging.LogRecord) -> str:
+        caller_file = getattr(record, "furu_caller_file", None)
+        caller_line = getattr(record, "furu_caller_line", None)
+        if isinstance(caller_file, str) and isinstance(caller_line, int):
+            location = f"{Path(caller_file).name}:{caller_line}"
+        else:
+            location = f"{record.filename}:{record.lineno}"
+        record.furu_location = location  # type: ignore[attr-defined]
+        return super().format(record)
+
 
 class _FuruContextFileHandler(logging.Handler):
     """
@@ -112,7 +123,8 @@ class _FuruContextFileHandler(logging.Handler):
         message = self.format(record)
 
         directory = current_log_dir()
-        directory.mkdir(parents=True, exist_ok=True)
+        if directory.name != ".furu":
+            directory.mkdir(parents=True, exist_ok=True)
 
         log_path = directory / "furu.log"
         with _FURU_LOG_LOCK:
@@ -241,7 +253,7 @@ def configure_logging() -> None:
         handler.addFilter(_FuruFileFilter())
         handler.setFormatter(
             _FuruLogFormatter(
-                "%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)d %(message)s"
+                "%(asctime)s [%(levelname)s] %(name)s %(furu_location)s %(message)s"
             )
         )
         root.addHandler(handler)
@@ -280,7 +292,20 @@ def log(message: str, *, level: str = "INFO") -> Path:
         raise ValueError(f"Unknown log level: {level!r}")
 
     configure_logging()
-    get_logger().log(level_no, message)
+    caller_info: dict[str, object] = {}
+    frame = sys._getframe(1)
+    if frame is not None:
+        furu_pkg_dir = str(Path(__file__).parent.parent)
+        while frame is not None:
+            filename = frame.f_code.co_filename
+            if not filename.startswith(furu_pkg_dir):
+                caller_info = {
+                    "furu_caller_file": filename,
+                    "furu_caller_line": frame.f_lineno,
+                }
+                break
+            frame = frame.f_back
+    get_logger().log(level_no, message, extra=caller_info)
     return log_path
 
 
@@ -293,7 +318,8 @@ def write_separator(line: str = "------------------") -> Path:
     directory = current_log_dir()
     log_path = directory / "furu.log"
 
-    directory.mkdir(parents=True, exist_ok=True)
+    if directory.name != ".furu":
+        directory.mkdir(parents=True, exist_ok=True)
 
     with _FURU_LOG_LOCK:
         with log_path.open("a", encoding="utf-8") as fp:

furu/storage/metadata.py

@@ -19,7 +19,7 @@ from ..serialization.serializer import JsonValue
 if TYPE_CHECKING:
     from ..core.furu import Furu
 
-# Module-level cache for metadata (controlled via FURU_CACHE_METADATA)
+# Module-level cache for metadata (controlled via FURU_RECORD_GIT=cached)
 _cached_git_info: "GitInfo | None" = None
 _cached_git_info_time: float = 0.0
 
@@ -113,6 +113,16 @@ class MetadataManager:
         global _cached_git_info, _cached_git_info_time
         import time
 
+        record_git = FURU_CONFIG.record_git
+        if record_git == "ignore":
+            return GitInfo(
+                git_commit="<ignored>",
+                git_branch="<ignored>",
+                git_remote=None,
+                git_patch="<ignored>",
+                git_submodules={},
+            )
+
         ttl = FURU_CONFIG.cache_metadata_ttl_sec
         # Return cached result if caching is enabled and not expired
         if ttl is not None and _cached_git_info is not None:
@@ -120,41 +130,28 @@ class MetadataManager:
             if age < ttl:
                 return _cached_git_info
 
-        if not FURU_CONFIG.require_git:
+        try:
+            head = cls.run_git_command(["rev-parse", "HEAD"])
+            branch = cls.run_git_command(["rev-parse", "--abbrev-ref", "HEAD"])
+        except (subprocess.CalledProcessError, FileNotFoundError) as e:
+            raise RuntimeError(
+                "Failed to read git commit/branch for provenance. "
+                "If this is expected, set FURU_RECORD_GIT=ignore."
+            ) from e
+
+        if FURU_CONFIG.allow_no_git_origin:
             try:
-                head = cls.run_git_command(["rev-parse", "HEAD"])
-                branch = cls.run_git_command(["rev-parse", "--abbrev-ref", "HEAD"])
+                remote = cls.run_git_command(["remote", "get-url", "origin"])
             except (subprocess.CalledProcessError, FileNotFoundError):
-                return GitInfo(
-                    git_commit="<no-git>",
-                    git_branch="<no-git>",
-                    git_remote=None,
-                    git_patch="<no-git>",
-                    git_submodules={},
-                )
+                remote = None
         else:
-            try:
-                head = cls.run_git_command(["rev-parse", "HEAD"])
-                branch = cls.run_git_command(["rev-parse", "--abbrev-ref", "HEAD"])
-            except (subprocess.CalledProcessError, FileNotFoundError) as e:
-                raise RuntimeError(
-                    "Failed to read git commit/branch for provenance. "
-                    "If this is expected, set FURU_REQUIRE_GIT=0."
-                ) from e
-
-        if FURU_CONFIG.require_git_remote:
             try:
                 remote = cls.run_git_command(["remote", "get-url", "origin"])
             except (subprocess.CalledProcessError, FileNotFoundError) as e:
                 raise RuntimeError(
                     "Git remote 'origin' is required for provenance but was not found. "
-                    "Set FURU_REQUIRE_GIT_REMOTE=0 to allow missing origin."
+                    "Set FURU_ALLOW_NO_GIT_ORIGIN=1 to allow missing origin."
                 ) from e
-        else:
-            try:
-                remote = cls.run_git_command(["remote", "get-url", "origin"])
-            except (subprocess.CalledProcessError, FileNotFoundError):
-                remote = None
 
         if ignore_diff:
             patch = "<ignored-diff>"
@@ -187,7 +184,7 @@ class MetadataManager:
             if len(patch) > 50_000:
                 raise ValueError(
                     f"Git diff too large ({len(patch):,} bytes). "
-                    "Use ignore_diff=True or FURU_IGNORE_DIFF=1"
+                    "Set FURU_RECORD_GIT=ignore to skip git metadata."
                 )
 
         submodules: dict[str, str] = {}
@@ -265,7 +262,6 @@ class MetadataManager:
     def write_metadata(cls, metadata: FuruMetadata, directory: Path) -> None:
         """Write metadata to file."""
         metadata_path = cls.get_metadata_path(directory)
-        metadata_path.parent.mkdir(parents=True, exist_ok=True)
         metadata_path.write_text(
             json.dumps(
                 metadata.model_dump(mode="json"),

furu/storage/migration.py

@@ -52,7 +52,6 @@ class MigrationManager:
     @classmethod
     def write_migration(cls, record: MigrationRecord, directory: Path) -> None:
         path = cls.get_migration_path(directory)
-        path.parent.mkdir(parents=True, exist_ok=True)
         tmp = path.with_suffix(".tmp")
         tmp.write_text(json.dumps(record.model_dump(mode="json"), indent=2))
         tmp.replace(path)