furu 0.0.2__py3-none-any.whl → 0.0.4__py3-none-any.whl

{furu-0.0.2.dist-info → furu-0.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: furu
-Version: 0.0.2
+Version: 0.0.4
 Summary: Cacheable, nested pipelines for Python. Define computations as configs; furu handles caching, state tracking, and result reuse across runs.
 Author: Herman Brunborg
 Author-email: Herman Brunborg <herman@brunborg.com>
@@ -44,7 +44,7 @@ The `[dashboard]` extra includes the web dashboard. Omit it for the core library
 1. Subclass `furu.Furu[T]`
 2. Implement `_create(self) -> T` (compute and write to `self.furu_dir`)
 3. Implement `_load(self) -> T` (load from `self.furu_dir`)
-4. Call `load_or_create()`
+4. Call `get()`
 
 ```python
 # my_project/pipelines.py
@@ -75,10 +75,10 @@ class TrainModel(furu.Furu[Path]):
 from my_project.pipelines import TrainModel
 
 # First call: runs _create(), caches result
-artifact = TrainModel(lr=3e-4, steps=5000).load_or_create()
+artifact = TrainModel(lr=3e-4, steps=5000).get()
 
 # Second call with same config: loads from cache via _load()
-artifact = TrainModel(lr=3e-4, steps=5000).load_or_create()
+artifact = TrainModel(lr=3e-4, steps=5000).get()
 ```
 
 > **Tip:** Define Furu classes in importable modules (not `__main__`); the artifact namespace is derived from the class's module + qualified name.
@@ -96,7 +96,7 @@ Each `Furu` instance maps deterministically to a directory based on its config:
 - **namespace**: Derived from the class's module + qualified name (e.g., `my_project.pipelines/TrainModel`)
 - **hash**: Computed from the object's config values using Blake2s
 
-When you call `load_or_create()`:
+When you call `get()`:
 1. If no cached result exists → run `_create()`, save state as "success"
 2. If cached result exists → run `_load()` to retrieve it
 3. If another process is running → wait for it to finish, then load
@@ -123,7 +123,7 @@ class TrainTextModel(furu.Furu[str]):
     dataset: Dataset = furu.chz.field(default_factory=Dataset)
 
     def _create(self) -> str:
-        data = self.dataset.load_or_create()  # Triggers Dataset cache
+        data = self.dataset.get()  # Triggers Dataset cache
         (self.furu_dir / "model.txt").write_text(f"trained on:\n{data}")
         return "trained"
 
@@ -131,6 +131,58 @@ class TrainTextModel(furu.Furu[str]):
         return (self.furu_dir / "model.txt").read_text()
 ```
 
+### Executors (Local + Slurm)
+
+Use the execution helpers for batch runs and cluster scheduling:
+
+```python
+from furu.execution import run_local
+
+run_local(
+    [TrainModel(lr=3e-4, steps=5000), TrainModel(lr=1e-3, steps=2000)],
+    max_workers=8,
+    window_size="bfs",
+)
+```
+
+```python
+from furu.execution import SlurmSpec, submit_slurm_dag
+
+specs = {
+    "default": SlurmSpec(partition="cpu", cpus=8, mem_gb=32, time_min=120),
+    "gpu": SlurmSpec(partition="gpu", gpus=1, cpus=8, mem_gb=64, time_min=720),
+}
+
+submit_slurm_dag([TrainModel(lr=3e-4, steps=5000)], specs=specs)
+```
+
+```python
+from furu.execution import run_slurm_pool
+
+run_slurm_pool(
+    [TrainModel(lr=3e-4, steps=5000)],
+    specs=specs,
+    max_workers_total=50,
+    window_size="bfs",
+)
+```
+
+Submitit logs are stored under `<FURU_PATH>/submitit` by default. Override with
+`FURU_SUBMITIT_PATH` when you want a different logs root.
+
+### Breaking Changes and Executor Semantics
+
+- `load_or_create()` is removed; use `get()` exclusively.
+- `get()` no longer accepts per-call `retry_failed` overrides. Configure retries via
+  `FURU_RETRY_FAILED` or `FURU_CONFIG.retry_failed`.
+- Executor runs (`run_local`, `run_slurm_pool`, `submit_slurm_dag`) fail fast if a
+  dependency is FAILED while `retry_failed` is disabled; with retries enabled, failed
+  compute nodes are retried (bounded by `FURU_MAX_COMPUTE_RETRIES` retries).
+- Pool protocol/queue failures (invalid payloads, spec mismatch, missing artifacts) are
+  fatal even when `retry_failed` is enabled; only compute failures are retried.
+- `FURU_ALWAYS_RERUN` causes matching nodes to recompute once per executor run, but
+  repeated references in the same run reuse that result.
+
 ### Storage Structure
 
 Furu uses two roots: `FURU_PATH` for `data/` + `raw/`, and
@@ -176,7 +228,7 @@ class MyExperiments(furu.FuruList[TrainModel]):
 
 # Iterate over all experiments
 for exp in MyExperiments:
-    exp.load_or_create()
+    exp.get()
 
 # Access by name
 exp = MyExperiments.by_name("baseline")
@@ -191,14 +243,17 @@ for name, exp in MyExperiments.items():
 
 ### Custom Validation
 
-Override `_validate()` to add custom cache invalidation logic:
+Override `_validate()` to add custom cache invalidation logic. Return False or
+raise `furu.FuruValidationError` to force re-computation. In executor planning,
+any other exception is logged and treated as invalid (no crash); in interactive
+`exists()` calls, exceptions still surface:
 
 ```python
 class ModelWithValidation(furu.Furu[Path]):
     checkpoint_name: str = "model.pt"
 
     def _validate(self) -> bool:
-        # Return False to force re-computation
+        # Return False (or raise FuruValidationError) to force re-computation
         ckpt = self.furu_dir / self.checkpoint_name
         return ckpt.exists() and ckpt.stat().st_size > 0
 
@@ -220,7 +275,7 @@ if obj.exists():
 
 # Get metadata without triggering computation
 metadata = obj.get_metadata()
-print(f"Hash: {obj._furu_hash}")
+print(f"Hash: {obj.furu_hash}")
 print(f"Dir: {obj.furu_dir}")
 ```
 
@@ -251,7 +306,7 @@ class LargeDataProcessor(furu.Furu[Path]):
     def _create(self) -> Path:
         # self.raw_dir is shared across all configs
         # Create a subfolder for isolation if needed
-        my_raw = self.raw_dir / self._furu_hash
+        my_raw = self.raw_dir / self.furu_hash
         my_raw.mkdir(exist_ok=True)
         
         large_file = my_raw / "huge_dataset.bin"
@@ -303,8 +358,8 @@ HHMMSS file.py:line message
 
 Furu emits status messages like:
 ```
-load_or_create TrainModel abc123def (missing->create)
-load_or_create TrainModel abc123def (success->load)
+get TrainModel abc123def (missing->create)
+get TrainModel abc123def (success->load)
 ```
 
 ### Explicit Setup
@@ -325,7 +380,7 @@ logger = furu.get_logger()
 from furu import FuruComputeError, FuruWaitTimeout, FuruLockNotAcquired
 
 try:
-    result = obj.load_or_create()
+    result = obj.get()
 except FuruComputeError as e:
     print(f"Computation failed: {e}")
     print(f"State file: {e.state_path}")
@@ -336,29 +391,21 @@ except FuruLockNotAcquired:
     print("Could not acquire lock")
 ```
 
-## Submitit Integration
-
-Run computations on SLURM clusters via [submitit](https://github.com/facebookincubator/submitit):
+By default, failed artifacts are retried on the next `get()` call. Set
+`FURU_RETRY_FAILED=0` to keep failures sticky.
 
-```python
-import submitit
-import furu
+`FURU_MAX_WAIT_SECS` overrides the per-class `_max_wait_time_sec` (default 600s)
+timeout used when waiting for compute locks before raising `FuruWaitTimeout`.
 
-executor = submitit.AutoExecutor(folder="submitit_logs")
-executor.update_parameters(
-    timeout_min=60,
-    slurm_partition="gpu",
-    gpus_per_node=1,
-)
+Failures during metadata collection or signal handler setup (before `_create()`
+runs) raise `FuruComputeError` with the original exception attached. These
+failures still mark the attempt as failed and record details in `state.json`
+and `furu.log`.
 
-# Submit job and return immediately
-job = my_furu_obj.load_or_create(executor=executor)
-
-# Job ID is tracked in .furu/state.json
-print(job.job_id)
-```
+## Submitit Integration
 
-Furu handles preemption, requeuing, and state tracking automatically.
+Furu includes a `SubmititAdapter` for integrating submitit executors with the
+state system. Executor helpers in `furu.execution` handle submission workflows.
 
 ## Dashboard
 
@@ -415,7 +462,10 @@ The `/api/experiments` endpoint supports:
 | `FURU_LOG_LEVEL` | `INFO` | Console verbosity (`DEBUG`, `INFO`, `WARNING`, `ERROR`) |
 | `FURU_IGNORE_DIFF` | `false` | Skip embedding git diff in metadata |
 | `FURU_ALWAYS_RERUN` | `""` | Comma-separated class qualnames to always rerun (use `ALL` to bypass cache globally; cannot combine with other entries; entries must be importable) |
+| `FURU_RETRY_FAILED` | `true` | Retry failed artifacts by default (set to `0` to keep failures sticky) |
+| `FURU_MAX_COMPUTE_RETRIES` | `3` | Maximum compute retries per node after the first failure |
 | `FURU_POLL_INTERVAL_SECS` | `10` | Polling interval for queued/running jobs |
+| `FURU_MAX_WAIT_SECS` | unset | Override wait timeout (falls back to `_max_wait_time_sec`, default 600s) |
 | `FURU_WAIT_LOG_EVERY_SECS` | `10` | Interval between "waiting" log messages |
 | `FURU_STALE_AFTER_SECS` | `1800` | Consider running jobs stale after this duration |
 | `FURU_LEASE_SECS` | `120` | Compute lock lease duration |

furu-0.0.4.dist-info/RECORD

@@ -0,0 +1,46 @@
+furu/__init__.py,sha256=Z8VssTuQm2nH7bgB8SQc8pXsNGc-H1QGHFffKzNzqk8,2018
+furu/adapters/__init__.py,sha256=onLzEj9hccPK15g8a8va2T19nqQXoxb9rQlJIjKSKnE,69
+furu/adapters/submitit.py,sha256=FV3XEUSQuS5vIyzkW-Iuqtf8SRL-fsokPG67u7tMF5I,7276
+furu/config.py,sha256=1nlJff4KNrWDvLhmnuLrsc7FJIxFLFhz3eOXZ8-ngX4,7349
+furu/core/__init__.py,sha256=6hH7i6r627c0FZn6eQVsSG7LD4QmTta6iQw0AiPQPTM,156
+furu/core/furu.py,sha256=Cy2cOnM5vsQoSk9nIVYj2Fx017wOQFPbxhnvYQsh7nI,58881
+furu/core/list.py,sha256=xSuBT35p1anJ2fKQPxb-3cRTONUamFjfzkreVaI9Jo4,3614
+furu/dashboard/__init__.py,sha256=ziAordJfkbbXNIM7iA9O7vR2gsCq34AInYiMYOCfWOc,362
+furu/dashboard/__main__.py,sha256=cNs65IMl4kwZFpxa9xLXmFSy4-M5D1X1ZBfTDxW11vo,144
+furu/dashboard/api/__init__.py,sha256=9-WyWOt-VQJJBIsdW29D-7JvR-BivJd9G_SRaRptCz0,80
+furu/dashboard/api/models.py,sha256=SCu-kLJyW7dwSKswdgQNS3wQuj25ORs0pHkvX9xBbo4,4767
+furu/dashboard/api/routes.py,sha256=iZez0khIUvbgfeSoy1BJvmoEEbgUrdSQA8SN8iAIkM8,4813
+furu/dashboard/frontend/dist/assets/index-BXAIKNNr.css,sha256=qhsN0Td3mM-GAR8mZ0CtocynABLKa1ncl9ioDrTKOIQ,34768
+furu/dashboard/frontend/dist/assets/index-DS3FsqcY.js,sha256=nfrKjhWThPtL8n5iTd9_1W-bsyMGwg2O8Iq2jkjj9Lg,544699
+furu/dashboard/frontend/dist/favicon.svg,sha256=3TSLHNZITFe3JTPoYHZnDgiGsJxIzf39v97l2A1Hodo,369
+furu/dashboard/frontend/dist/index.html,sha256=d9a8ZFKZ5uDtN3urqVNmS8LWMBhOC0eW7X0noT0RcYQ,810
+furu/dashboard/main.py,sha256=gj9Cdj2qyaSCEkmfNHUMQXlXv6GpWTQ9IZEi7WzlCSo,4463
+furu/dashboard/scanner.py,sha256=qXCvkvFByBc09TUdth5Js67rS8zpRBlRkVQ9dJ7YbdE,34696
+furu/errors.py,sha256=FFbV4M0-ipVGizv5ee80L-NZFVjaRjy8i19mClr6R0g,3959
+furu/execution/__init__.py,sha256=ixVw1Shvg2ulS597OYYeGgSSTwv25j_McuQdDXIiEL8,625
+furu/execution/context.py,sha256=0tAbM0azqEus8hknf_A9-Zs9Sq99bnUkFyV4RO4ZMRU,666
+furu/execution/local.py,sha256=TkKrRdmaQrN7i7Sxe87eHibRJOnz5OxU0Oj8qL_xP4I,7059
+furu/execution/paths.py,sha256=0MfQk5Kh7bxvJiWvG40TJe7RF5Q5Na6uvi6qV0OT3Vc,460
+furu/execution/plan.py,sha256=fM7CkXm_M0lL3vqdiNnWzbvMJAoSYKDBAnC82Af_rYM,6860
+furu/execution/plan_utils.py,sha256=TAQqlPeJfOdH2MT-X7g3j1Se_0e4oKvG0tJaWC1kM40,381
+furu/execution/slurm_dag.py,sha256=FOJcPKmIzRyrbJIq7heqGjKN0EFRMyOcV-yP7Ci87Qs,9360
+furu/execution/slurm_pool.py,sha256=bi90fzZXAnoWHSPQba8Z3tk4_QMaqikWxCCzRfvDMvk,30400
+furu/execution/slurm_spec.py,sha256=A1VX5K6aG8Ricg4fhnkz3Alkw_fx1bx53D0p4Ms3FqA,979
+furu/execution/submitit_factory.py,sha256=B2vkDtmscuAX0sBaj9V5pNlgOtkkV35yJ1fZ7A-DSvU,1119
+furu/migrate.py,sha256=x_Uh7oXAv40L5ZAHJhdnw-o7ct56rWUSZLbHHfRObeY,1313
+furu/migration.py,sha256=R2-tARMx4VKryiqJ7WHia_dPVxRbTqofPpCFVE9zQ8U,31411
+furu/runtime/__init__.py,sha256=fQqE7wUuWunLD73Vm3lss7BFSij3UVxXOKQXBAOS8zw,504
+furu/runtime/env.py,sha256=o1phhoTDhOnhALr3Ozf1ldrdvk2ClyEvBWbebHM6BXg,160
+furu/runtime/logging.py,sha256=WS3mB8VqMYUxPPI0yv1K-LnzVBj84Mnu1Qf9P2hCUUE,9652
+furu/runtime/tracebacks.py,sha256=PGCuOq8QkWSoun791gjUXM8frOP2wWV8IBlqaA4nuGE,1631
+furu/serialization/__init__.py,sha256=L7oHuIbxdSh7GCY3thMQnDwlt_ERH-TMy0YKEAZLrPs,341
+furu/serialization/migrations.py,sha256=HD5g8JCBdH3Y0rHJYc4Ug1IXBVcUDxLE7nfiXZnXcUE,7772
+furu/serialization/serializer.py,sha256=_nfUaAOy_KHegvfXlpPh4rCuvkzalJva75OvDg5nXiI,10114
+furu/storage/__init__.py,sha256=cLLL-GPpSu9C72Mdk5S6TGu3g-SnBfEuxzfpx5ZJPtw,616
+furu/storage/metadata.py,sha256=MH6w5hs-2rwHD6G9erMPM5pE3hm0h5Pk_G3Z6eyyGB0,9899
+furu/storage/migration.py,sha256=Ars9aYwvhXpIBDf6L9ojGjp_l656-RfdtEAFKN0sZZY,2640
+furu/storage/state.py,sha256=SFonqragT2eMCZbBKIvcA4JVe78rVmDRvo4Ky2IcNgc,43632
+furu-0.0.4.dist-info/WHEEL,sha256=XV0cjMrO7zXhVAIyyc8aFf1VjZ33Fen4IiJk5zFlC3g,80
+furu-0.0.4.dist-info/entry_points.txt,sha256=hZkjtFzNlb33Zk-aUfLMRj-XgVDxdT82-JXG9d4bu2E,60
+furu-0.0.4.dist-info/METADATA,sha256=fdUBvn-vEnVim9V5hAamE1sFuaKzWdwWPI17VU2Vyfc,16162
+furu-0.0.4.dist-info/RECORD,,

furu-0.0.2.dist-info/RECORD

@@ -1,36 +0,0 @@
-furu/__init__.py,sha256=fhSViHOJ9W-64swuaBFdZOfq0ZMuSj6LSiX2ZfcjhD8,1736
-furu/adapters/__init__.py,sha256=onLzEj9hccPK15g8a8va2T19nqQXoxb9rQlJIjKSKnE,69
-furu/adapters/submitit.py,sha256=OuCP0pEkO1kI4WLcSUvMqXwVCCy-8uwUE7v1qvkLZnU,6214
-furu/config.py,sha256=C9mYQLgP4ciPmONCpQUu2YVV8adscCkfLsiyjXZVcpQ,6636
-furu/core/__init__.py,sha256=gzFMgaAYnffofQksR6E1NegiwBF99h0ysn_QeD5wIhw,82
-furu/core/furu.py,sha256=7swlMfGXBB_jmGABgMSl28v_qiE8Ot4vuDSos42cweQ,39085
-furu/core/list.py,sha256=hwwlvqaKB1grPBGKXc15scF1RCqDvWc0AoDbhKlN4W0,3625
-furu/dashboard/__init__.py,sha256=zNVddterfpjQtcpihIl3TRJdgdjOHYR0uO0cOSaGABg,172
-furu/dashboard/__main__.py,sha256=cNs65IMl4kwZFpxa9xLXmFSy4-M5D1X1ZBfTDxW11vo,144
-furu/dashboard/api/__init__.py,sha256=9-WyWOt-VQJJBIsdW29D-7JvR-BivJd9G_SRaRptCz0,80
-furu/dashboard/api/models.py,sha256=SCu-kLJyW7dwSKswdgQNS3wQuj25ORs0pHkvX9xBbo4,4767
-furu/dashboard/api/routes.py,sha256=iZez0khIUvbgfeSoy1BJvmoEEbgUrdSQA8SN8iAIkM8,4813
-furu/dashboard/frontend/dist/assets/index-CbdDfSOZ.css,sha256=k3kxCuCqyxKgIv4M9itoAImMU8NMzkzAdTNQ4v_4fMU,34612
-furu/dashboard/frontend/dist/assets/index-DDv_TYB_.js,sha256=FH0uqY7P7vm3rikvDaJ504FZh0Z97nCkVcIglK-ElAY,543928
-furu/dashboard/frontend/dist/favicon.svg,sha256=3TSLHNZITFe3JTPoYHZnDgiGsJxIzf39v97l2A1Hodo,369
-furu/dashboard/frontend/dist/index.html,sha256=o3XhvegC9rBpUiWNfXdCHqf_tg2795nob1NI0nBpFS4,810
-furu/dashboard/main.py,sha256=8JYc79gbJ9MjvIRdGDuAcR2Mme9kyY4ryZb11ZZ4uVA,4069
-furu/dashboard/scanner.py,sha256=qXCvkvFByBc09TUdth5Js67rS8zpRBlRkVQ9dJ7YbdE,34696
-furu/errors.py,sha256=d1Kp5O9cVoQwXmQeZC-35u7xldw_c3ryYXrbVfv-Lws,2001
-furu/migrate.py,sha256=x_Uh7oXAv40L5ZAHJhdnw-o7ct56rWUSZLbHHfRObeY,1313
-furu/migration.py,sha256=A91dng1XRn1N_xJrmBhh-OvU22GlseqOh6PmVhNZh3w,31307
-furu/runtime/__init__.py,sha256=fQqE7wUuWunLD73Vm3lss7BFSij3UVxXOKQXBAOS8zw,504
-furu/runtime/env.py,sha256=o1phhoTDhOnhALr3Ozf1ldrdvk2ClyEvBWbebHM6BXg,160
-furu/runtime/logging.py,sha256=JkuTFtbv6dYk088P6_Bga46bnKSDt-ElAqmiY86hMys,9773
-furu/runtime/tracebacks.py,sha256=PGCuOq8QkWSoun791gjUXM8frOP2wWV8IBlqaA4nuGE,1631
-furu/serialization/__init__.py,sha256=L7oHuIbxdSh7GCY3thMQnDwlt_ERH-TMy0YKEAZLrPs,341
-furu/serialization/migrations.py,sha256=HD5g8JCBdH3Y0rHJYc4Ug1IXBVcUDxLE7nfiXZnXcUE,7772
-furu/serialization/serializer.py,sha256=THWqHzpSwXj3Nj3PZ3JhwlWJ8sgvVyGrwBEDB_EWuAE,8355
-furu/storage/__init__.py,sha256=cLLL-GPpSu9C72Mdk5S6TGu3g-SnBfEuxzfpx5ZJPtw,616
-furu/storage/metadata.py,sha256=u4F4V1dDZtsiniO5xDCy8YxJZxGnreriYnJ1fOvQ2Bg,9232
-furu/storage/migration.py,sha256=Ars9aYwvhXpIBDf6L9ojGjp_l656-RfdtEAFKN0sZZY,2640
-furu/storage/state.py,sha256=q8wWJnGMWzx56PfsRMAMRB62p5vVw-iZ5rnUPfw2-js,40878
-furu-0.0.2.dist-info/WHEEL,sha256=XV0cjMrO7zXhVAIyyc8aFf1VjZ33Fen4IiJk5zFlC3g,80
-furu-0.0.2.dist-info/entry_points.txt,sha256=hZkjtFzNlb33Zk-aUfLMRj-XgVDxdT82-JXG9d4bu2E,60
-furu-0.0.2.dist-info/METADATA,sha256=8Zvp5E5XHn11a8YedVhAOi9KAvH9LwKvxiW4Jn7-Hsg,13833
-furu-0.0.2.dist-info/RECORD,,