furu 0.0.1__py3-none-any.whl → 0.0.2__py3-none-any.whl

furu/config.py

@@ -1,18 +1,24 @@
 import os
+from importlib import import_module
 from pathlib import Path
 
 
 class FuruConfig:
     """Central configuration for Furu behavior."""
 
+    DEFAULT_ROOT_DIR = Path("furu-data")
+    VERSION_CONTROLLED_SUBDIR = DEFAULT_ROOT_DIR / "artifacts"
+
     def __init__(self):
         def _get_base_root() -> Path:
             env = os.getenv("FURU_PATH")
             if env:
                 return Path(env).expanduser().resolve()
-            return Path("data-furu").resolve()
+            project_root = self._find_project_root(fallback_to_cwd=True)
+            return (project_root / self.DEFAULT_ROOT_DIR).resolve()
 
         self.base_root = _get_base_root()
+        self.version_controlled_root_override = self._get_version_controlled_override()
         self.poll_interval = float(os.getenv("FURU_POLL_INTERVAL_SECS", "10"))
         self.wait_log_every_sec = float(os.getenv("FURU_WAIT_LOG_EVERY_SECS", "10"))
         self.stale_timeout = float(os.getenv("FURU_STALE_AFTER_SECS", str(30 * 60)))
@@ -37,11 +43,23 @@ class FuruConfig:
             "true",
             "yes",
         }
-        self.force_recompute = {
+        always_rerun_items = {
             item.strip()
-            for item in os.getenv("FURU_FORCE_RECOMPUTE", "").split(",")
+            for item in os.getenv("FURU_ALWAYS_RERUN", "").split(",")
             if item.strip()
         }
+        all_entries = {item for item in always_rerun_items if item.lower() == "all"}
+        if all_entries and len(always_rerun_items) > len(all_entries):
+            raise ValueError(
+                "FURU_ALWAYS_RERUN cannot combine 'ALL' with specific entries"
+            )
+        self.always_rerun_all = bool(all_entries)
+        if self.always_rerun_all:
+            always_rerun_items = {
+                item for item in always_rerun_items if item.lower() != "all"
+            }
+        self._require_namespaces_exist(always_rerun_items)
+        self.always_rerun = always_rerun_items
         self.cancelled_is_preempted = os.getenv(
             "FURU_CANCELLED_IS_PREEMPTED", "false"
         ).lower() in {"1", "true", "yes"}
@@ -77,11 +95,67 @@ class FuruConfig:
         return num * multipliers[unit]
 
     def get_root(self, version_controlled: bool = False) -> Path:
-        """Get root directory for storage (version_controlled determines subdirectory)."""
+        """Get root directory for storage (version_controlled uses its own root)."""
         if version_controlled:
-            return self.base_root / "git"
+            if self.version_controlled_root_override is not None:
+                return self.version_controlled_root_override
+            return self._resolve_version_controlled_root()
         return self.base_root / "data"
 
+    @classmethod
+    def _get_version_controlled_override(cls) -> Path | None:
+        env = os.getenv("FURU_VERSION_CONTROLLED_PATH")
+        if env:
+            return Path(env).expanduser().resolve()
+        return None
+
+    @classmethod
+    def _resolve_version_controlled_root(cls) -> Path:
+        project_root = cls._find_project_root()
+        return (project_root / cls.VERSION_CONTROLLED_SUBDIR).resolve()
+
+    @staticmethod
+    def _find_project_root(
+        start: Path | None = None, *, fallback_to_cwd: bool = False
+    ) -> Path:
+        base = (start or Path.cwd()).resolve()
+        git_root: Path | None = None
+        for path in (base, *base.parents):
+            if (path / "pyproject.toml").is_file():
+                return path
+            if git_root is None and (path / ".git").exists():
+                git_root = path
+        if git_root is not None:
+            return git_root
+        if fallback_to_cwd:
+            return base
+        raise ValueError(
+            "Cannot locate pyproject.toml or .git to determine version-controlled root. "
+            "Set FURU_VERSION_CONTROLLED_PATH to override."
+        )
+
+    @staticmethod
+    def _require_namespaces_exist(namespaces: set[str]) -> None:
+        if not namespaces:
+            return
+        missing_sentinel = object()
+        for namespace in namespaces:
+            module_name, _, qualname = namespace.rpartition(".")
+            if not module_name or not qualname:
+                raise ValueError(
+                    "FURU_ALWAYS_RERUN entries must be 'module.QualifiedName', "
+                    f"got {namespace!r}"
+                )
+            target: object = import_module(module_name)
+            for attr in qualname.split("."):
+                value = getattr(target, attr, missing_sentinel)
+                if value is missing_sentinel:
+                    raise ValueError(
+                        "FURU_ALWAYS_RERUN entry does not exist: "
+                        f"{namespace!r}"
+                    )
+                target = value
+
     @property
     def raw_dir(self) -> Path:
         return self.base_root / "raw"

furu/core/furu.py

@@ -204,11 +204,13 @@ class Furu[T](ABC):
         """Compute hash of this object's content for storage identification."""
         return FuruSerializer.compute_hash(self)
 
-    def _force_recompute(self: Self) -> bool:
-        if not FURU_CONFIG.force_recompute:
+    def _always_rerun(self: Self) -> bool:
+        if FURU_CONFIG.always_rerun_all:
+            return True
+        if not FURU_CONFIG.always_rerun:
             return False
         qualname = f"{self.__class__.__module__}.{self.__class__.__qualname__}"
-        return qualname in FURU_CONFIG.force_recompute
+        return qualname in FURU_CONFIG.always_rerun
 
     def _base_furu_dir(self: Self) -> Path:
         root = FURU_CONFIG.get_root(self.version_controlled)
@@ -333,12 +335,12 @@ class Furu[T](ABC):
                         )
                         migration = MigrationManager.read_migration(base_dir)
 
-                if alias_active and self._force_recompute():
+                if alias_active and self._always_rerun():
                     if migration is not None:
                         self._maybe_detach_alias(
                             directory=base_dir,
                             record=migration,
-                            reason="force_recompute",
+                            reason="always_rerun",
                         )
                     migration = MigrationManager.read_migration(base_dir)
                     alias_active = False
@@ -350,9 +352,9 @@ class Furu[T](ABC):
                 success_marker = StateManager.get_success_marker_path(directory)
                 if success_marker.is_file():
                     # We have a success marker. Check if we can use it.
-                    if self._force_recompute():
+                    if self._always_rerun():
                         self._invalidate_cached_success(
-                            directory, reason="force_recompute enabled"
+                            directory, reason="always_rerun enabled"
                         )
                         # Fall through to normal load
                     else:
@@ -381,9 +383,9 @@ class Furu[T](ABC):
                 needs_reconcile = True
                 if isinstance(state0.result, _StateResultSuccess):
                     # Double check logic if we fell through to here (e.g. race condition or invalidation above)
-                    if self._force_recompute():
+                    if self._always_rerun():
                         self._invalidate_cached_success(
-                            directory, reason="force_recompute enabled"
+                            directory, reason="always_rerun enabled"
                         )
                         state0 = StateManager.read_state(directory)
                     else:

furu/storage/state.py

@@ -1008,6 +1008,35 @@ def compute_lock(
         FuruLockNotAcquired: If lock cannot be acquired (after waiting)
         FuruWaitTimeout: If max_wait_time_sec is exceeded
     """
+    def _format_wait_duration(seconds: float) -> str:
+        if seconds < 60.0:
+            return f"{seconds:.1f}s"
+        minutes = seconds / 60.0
+        if minutes < 60.0:
+            return f"{minutes:.1f}m"
+        hours = minutes / 60.0
+        if hours < 24.0:
+            return f"{hours:.1f}h"
+        days = hours / 24.0
+        return f"{days:.1f}d"
+
+    def _describe_wait(attempt: _StateAttempt, waited_sec: float) -> str:
+        label = "last heartbeat"
+        timestamp = attempt.heartbeat_at
+        if attempt.status == "queued":
+            label = "queued at"
+            timestamp = attempt.started_at
+        parsed = StateManager._parse_time(timestamp)
+        timestamp_info = timestamp
+        if parsed is not None:
+            age = (StateManager._utcnow() - parsed).total_seconds()
+            timestamp_info = f"{timestamp} ({_format_wait_duration(age)} ago)"
+        return (
+            "waited "
+            f"{_format_wait_duration(waited_sec)}, {label} {timestamp_info}, "
+            f"status {attempt.status}, backend {attempt.backend}"
+        )
+
     lock_path = StateManager.get_lock_path(directory, StateManager.COMPUTE_LOCK)
 
     lock_fd: int | None = None
@@ -1031,6 +1060,49 @@ def compute_lock(
 
         lock_fd = StateManager.try_lock(lock_path)
         if lock_fd is not None:
+            state = StateManager.read_state(directory)
+            if isinstance(state.result, _StateResultSuccess):
+                StateManager.release_lock(lock_fd, lock_path)
+                raise FuruLockNotAcquired(
+                    "Cannot acquire lock: experiment already succeeded"
+                )
+            if isinstance(state.result, _StateResultFailed):
+                StateManager.release_lock(lock_fd, lock_path)
+                raise FuruLockNotAcquired("Cannot acquire lock: experiment already failed")
+            attempt = state.attempt
+            if (
+                isinstance(attempt, (_StateAttemptQueued, _StateAttemptRunning))
+                and attempt.backend != backend
+            ):
+                StateManager.release_lock(lock_fd, lock_path)
+                lock_fd = None
+                if reconcile_fn is not None:
+                    reconcile_fn(directory)
+                    state = StateManager.read_state(directory)
+                if isinstance(state.result, _StateResultSuccess):
+                    raise FuruLockNotAcquired(
+                        "Cannot acquire lock: experiment already succeeded"
+                    )
+                if isinstance(state.result, _StateResultFailed):
+                    raise FuruLockNotAcquired(
+                        "Cannot acquire lock: experiment already failed"
+                    )
+                attempt = state.attempt
+                if not isinstance(attempt, (_StateAttemptQueued, _StateAttemptRunning)):
+                    continue
+                if attempt.backend == backend:
+                    continue
+                now = time.time()
+                if now >= next_wait_log_at:
+                    waited_sec = now - start_time
+                    logger.info(
+                        "compute_lock: waiting for lock creation %s (%s)",
+                        directory,
+                        _describe_wait(attempt, waited_sec),
+                    )
+                    next_wait_log_at = now + wait_log_every_sec
+                time.sleep(poll_interval_sec)
+                continue
             break
 
         # Lock held by someone else - reconcile and check state
@@ -1064,9 +1136,11 @@ def compute_lock(
         # Active attempt exists - wait for it
         now = time.time()
         if now >= next_wait_log_at:
+            waited_sec = now - start_time
             logger.info(
-                "compute_lock: waiting for lock %s",
+                "compute_lock: waiting for lock %s (%s)",
                 directory,
+                _describe_wait(attempt, waited_sec),
             )
             next_wait_log_at = now + wait_log_every_sec
         time.sleep(poll_interval_sec)

{furu-0.0.1.dist-info → furu-0.0.2.dist-info}/METADATA

@@ -1,19 +1,20 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.3
 Name: furu
-Version: 0.0.1
+Version: 0.0.2
 Summary: Cacheable, nested pipelines for Python. Define computations as configs; furu handles caching, state tracking, and result reuse across runs.
+Author: Herman Brunborg
 Author-email: Herman Brunborg <herman@brunborg.com>
-Requires-Python: >=3.12
 Requires-Dist: chz>=0.4.0
 Requires-Dist: cloudpickle>=3.1.1
 Requires-Dist: pydantic>=2.12.5
 Requires-Dist: python-dotenv>=1.0.0
 Requires-Dist: rich>=14.2.0
 Requires-Dist: submitit>=1.5.3
+Requires-Dist: fastapi>=0.109.0 ; extra == 'dashboard'
+Requires-Dist: uvicorn[standard]>=0.27.0 ; extra == 'dashboard'
+Requires-Dist: typer>=0.9.0 ; extra == 'dashboard'
+Requires-Python: >=3.12
 Provides-Extra: dashboard
-Requires-Dist: fastapi>=0.109.0; extra == 'dashboard'
-Requires-Dist: typer>=0.9.0; extra == 'dashboard'
-Requires-Dist: uvicorn[standard]>=0.27.0; extra == 'dashboard'
 Description-Content-Type: text/markdown
 
 # furu
@@ -132,20 +133,25 @@ class TrainTextModel(furu.Furu[str]):
 
 ### Storage Structure
 
+Furu uses two roots: `FURU_PATH` for `data/` + `raw/`, and
+`FURU_VERSION_CONTROLLED_PATH` for `artifacts/`. Defaults:
+
+```
+FURU_PATH=<project>/furu-data
+FURU_VERSION_CONTROLLED_PATH=<project>/furu-data/artifacts
+```
+
+`<project>` is the nearest directory containing `pyproject.toml` (falling back to
+the git root). This means you can move `FURU_PATH` without relocating artifacts.
+
 ```
 $FURU_PATH/
-├── data/                         # Default storage (version_controlled=False)
-│   └── <module>/<Class>/
-│       └── <hash>/
-│           ├── .furu/
-│           │   ├── metadata.json # Config, git info, environment
-│           │   ├── state.json    # Status and timestamps
-│           │   ├── furu.log    # Captured logs
-│           │   └── SUCCESS.json  # Marker file
-│           └── <your outputs>    # Files from _create()
-├── git/                          # For version_controlled=True
-│   └── <same structure>
-└── raw/                          # Shared directory for large files
+├── data/                         # version_controlled=False
+│   └── <module>/<Class>/<hash>/
+└── raw/
+
+$FURU_VERSION_CONTROLLED_PATH/    # version_controlled=True
+└── <module>/<Class>/<hash>/
 ```
 
 ## Features
@@ -259,10 +265,17 @@ For artifacts that should be stored separately (e.g., checked into git):
 
 ```python
 class VersionedConfig(furu.Furu[dict], version_controlled=True):
-    # Stored under $FURU_PATH/git/ instead of $FURU_PATH/data/
+    # Stored under $FURU_VERSION_CONTROLLED_PATH
+    # Default: <project>/furu-data/artifacts
     ...
 ```
 
+`<project>` is the nearest directory containing `pyproject.toml`, or the git root
+if `pyproject.toml` is missing.
+
+It is typical to keep `furu-data/data/` and `furu-data/raw/` in `.gitignore` while
+committing `furu-data/artifacts/`.
+
 ## Logging
 
 Furu installs stdlib `logging` handlers that capture logs to per-artifact files.
@@ -397,9 +410,11 @@ The `/api/experiments` endpoint supports:
 
 | Variable | Default | Description |
 |----------|---------|-------------|
-| `FURU_PATH` | `./data-furu/` | Base storage directory |
+| `FURU_PATH` | `<project>/furu-data` | Base storage directory for non-versioned artifacts |
+| `FURU_VERSION_CONTROLLED_PATH` | `<project>/furu-data/artifacts` | Override version-controlled storage root |
 | `FURU_LOG_LEVEL` | `INFO` | Console verbosity (`DEBUG`, `INFO`, `WARNING`, `ERROR`) |
 | `FURU_IGNORE_DIFF` | `false` | Skip embedding git diff in metadata |
+| `FURU_ALWAYS_RERUN` | `""` | Comma-separated class qualnames to always rerun (use `ALL` to bypass cache globally; cannot combine with other entries; entries must be importable) |
 | `FURU_POLL_INTERVAL_SECS` | `10` | Polling interval for queued/running jobs |
 | `FURU_WAIT_LOG_EVERY_SECS` | `10` | Interval between "waiting" log messages |
 | `FURU_STALE_AFTER_SECS` | `1800` | Consider running jobs stale after this duration |

{furu-0.0.1.dist-info → furu-0.0.2.dist-info}/RECORD

@@ -1,24 +1,24 @@
 furu/__init__.py,sha256=fhSViHOJ9W-64swuaBFdZOfq0ZMuSj6LSiX2ZfcjhD8,1736
-furu/config.py,sha256=F_Bh9vs0Dq5-3fXMylEBbm7F9-Q2n9aLt1iTb-RAl-4,3538
-furu/errors.py,sha256=d1Kp5O9cVoQwXmQeZC-35u7xldw_c3ryYXrbVfv-Lws,2001
-furu/migrate.py,sha256=x_Uh7oXAv40L5ZAHJhdnw-o7ct56rWUSZLbHHfRObeY,1313
-furu/migration.py,sha256=A91dng1XRn1N_xJrmBhh-OvU22GlseqOh6PmVhNZh3w,31307
 furu/adapters/__init__.py,sha256=onLzEj9hccPK15g8a8va2T19nqQXoxb9rQlJIjKSKnE,69
 furu/adapters/submitit.py,sha256=OuCP0pEkO1kI4WLcSUvMqXwVCCy-8uwUE7v1qvkLZnU,6214
+furu/config.py,sha256=C9mYQLgP4ciPmONCpQUu2YVV8adscCkfLsiyjXZVcpQ,6636
 furu/core/__init__.py,sha256=gzFMgaAYnffofQksR6E1NegiwBF99h0ysn_QeD5wIhw,82
-furu/core/furu.py,sha256=MjwpJtS0T8aRtLsFiiVTB8oh5UtIQrF3ohzYbD9XFIc,39047
+furu/core/furu.py,sha256=7swlMfGXBB_jmGABgMSl28v_qiE8Ot4vuDSos42cweQ,39085
 furu/core/list.py,sha256=hwwlvqaKB1grPBGKXc15scF1RCqDvWc0AoDbhKlN4W0,3625
 furu/dashboard/__init__.py,sha256=zNVddterfpjQtcpihIl3TRJdgdjOHYR0uO0cOSaGABg,172
 furu/dashboard/__main__.py,sha256=cNs65IMl4kwZFpxa9xLXmFSy4-M5D1X1ZBfTDxW11vo,144
-furu/dashboard/main.py,sha256=8JYc79gbJ9MjvIRdGDuAcR2Mme9kyY4ryZb11ZZ4uVA,4069
-furu/dashboard/scanner.py,sha256=qXCvkvFByBc09TUdth5Js67rS8zpRBlRkVQ9dJ7YbdE,34696
 furu/dashboard/api/__init__.py,sha256=9-WyWOt-VQJJBIsdW29D-7JvR-BivJd9G_SRaRptCz0,80
 furu/dashboard/api/models.py,sha256=SCu-kLJyW7dwSKswdgQNS3wQuj25ORs0pHkvX9xBbo4,4767
 furu/dashboard/api/routes.py,sha256=iZez0khIUvbgfeSoy1BJvmoEEbgUrdSQA8SN8iAIkM8,4813
-furu/dashboard/frontend/dist/favicon.svg,sha256=3TSLHNZITFe3JTPoYHZnDgiGsJxIzf39v97l2A1Hodo,369
-furu/dashboard/frontend/dist/index.html,sha256=o3XhvegC9rBpUiWNfXdCHqf_tg2795nob1NI0nBpFS4,810
 furu/dashboard/frontend/dist/assets/index-CbdDfSOZ.css,sha256=k3kxCuCqyxKgIv4M9itoAImMU8NMzkzAdTNQ4v_4fMU,34612
 furu/dashboard/frontend/dist/assets/index-DDv_TYB_.js,sha256=FH0uqY7P7vm3rikvDaJ504FZh0Z97nCkVcIglK-ElAY,543928
+furu/dashboard/frontend/dist/favicon.svg,sha256=3TSLHNZITFe3JTPoYHZnDgiGsJxIzf39v97l2A1Hodo,369
+furu/dashboard/frontend/dist/index.html,sha256=o3XhvegC9rBpUiWNfXdCHqf_tg2795nob1NI0nBpFS4,810
+furu/dashboard/main.py,sha256=8JYc79gbJ9MjvIRdGDuAcR2Mme9kyY4ryZb11ZZ4uVA,4069
+furu/dashboard/scanner.py,sha256=qXCvkvFByBc09TUdth5Js67rS8zpRBlRkVQ9dJ7YbdE,34696
+furu/errors.py,sha256=d1Kp5O9cVoQwXmQeZC-35u7xldw_c3ryYXrbVfv-Lws,2001
+furu/migrate.py,sha256=x_Uh7oXAv40L5ZAHJhdnw-o7ct56rWUSZLbHHfRObeY,1313
+furu/migration.py,sha256=A91dng1XRn1N_xJrmBhh-OvU22GlseqOh6PmVhNZh3w,31307
 furu/runtime/__init__.py,sha256=fQqE7wUuWunLD73Vm3lss7BFSij3UVxXOKQXBAOS8zw,504
 furu/runtime/env.py,sha256=o1phhoTDhOnhALr3Ozf1ldrdvk2ClyEvBWbebHM6BXg,160
 furu/runtime/logging.py,sha256=JkuTFtbv6dYk088P6_Bga46bnKSDt-ElAqmiY86hMys,9773
@@ -29,8 +29,8 @@ furu/serialization/serializer.py,sha256=THWqHzpSwXj3Nj3PZ3JhwlWJ8sgvVyGrwBEDB_EW
 furu/storage/__init__.py,sha256=cLLL-GPpSu9C72Mdk5S6TGu3g-SnBfEuxzfpx5ZJPtw,616
 furu/storage/metadata.py,sha256=u4F4V1dDZtsiniO5xDCy8YxJZxGnreriYnJ1fOvQ2Bg,9232
 furu/storage/migration.py,sha256=Ars9aYwvhXpIBDf6L9ojGjp_l656-RfdtEAFKN0sZZY,2640
-furu/storage/state.py,sha256=tbVX74P6nVHhL1EBztgKp9BCe0UHpW0nyGkSeJXPejs,37581
-furu-0.0.1.dist-info/METADATA,sha256=mGC5hO68kGPxMUepH1Cnws-TDowOyCi1cgJ36pgTTOA,13294
-furu-0.0.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-furu-0.0.1.dist-info/entry_points.txt,sha256=pIkNLYq-gaxYbh_lATWl31BHTrKBg1jN6jK1AgN6-QY,59
-furu-0.0.1.dist-info/RECORD,,
+furu/storage/state.py,sha256=q8wWJnGMWzx56PfsRMAMRB62p5vVw-iZ5rnUPfw2-js,40878
+furu-0.0.2.dist-info/WHEEL,sha256=XV0cjMrO7zXhVAIyyc8aFf1VjZ33Fen4IiJk5zFlC3g,80
+furu-0.0.2.dist-info/entry_points.txt,sha256=hZkjtFzNlb33Zk-aUfLMRj-XgVDxdT82-JXG9d4bu2E,60
+furu-0.0.2.dist-info/METADATA,sha256=8Zvp5E5XHn11a8YedVhAOi9KAvH9LwKvxiW4Jn7-Hsg,13833
+furu-0.0.2.dist-info/RECORD,,

{furu-0.0.1.dist-info → furu-0.0.2.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: hatchling 1.28.0
+Generator: uv 0.9.26
 Root-Is-Purelib: true
 Tag: py3-none-any

{furu-0.0.1.dist-info → furu-0.0.2.dist-info}/entry_points.txt

@@ -1,2 +1,3 @@
 [console_scripts]
 furu-dashboard = furu.dashboard.main:cli
+