furu 0.0.1__tar.gz → 0.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {furu-0.0.1 → furu-0.0.2}/PKG-INFO +35 -20
- {furu-0.0.1 → furu-0.0.2}/README.md +28 -14
- {furu-0.0.1 → furu-0.0.2}/pyproject.toml +3 -9
- furu-0.0.2/src/furu/config.py +172 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/core/furu.py +11 -9
- {furu-0.0.1 → furu-0.0.2}/src/furu/storage/state.py +75 -1
- furu-0.0.1/.github/workflows/ci.yml +0 -33
- furu-0.0.1/.github/workflows/release.yml +0 -99
- furu-0.0.1/.gitignore +0 -231
- furu-0.0.1/.vscode/settings.json +0 -3
- furu-0.0.1/AGENTS.md +0 -265
- furu-0.0.1/CHANGELOG.md +0 -5
- furu-0.0.1/Makefile +0 -156
- furu-0.0.1/TODO.md +0 -131
- furu-0.0.1/dashboard-frontend/bun.lock +0 -1234
- furu-0.0.1/dashboard-frontend/index.html +0 -21
- furu-0.0.1/dashboard-frontend/orval.config.ts +0 -34
- furu-0.0.1/dashboard-frontend/package-lock.json +0 -7628
- furu-0.0.1/dashboard-frontend/package.json +0 -40
- furu-0.0.1/dashboard-frontend/postcss.config.js +0 -9
- furu-0.0.1/dashboard-frontend/src/api.test.ts +0 -314
- furu-0.0.1/dashboard-frontend/src/components/DAGVisualization.tsx +0 -392
- furu-0.0.1/dashboard-frontend/src/components/EmptyState.tsx +0 -27
- furu-0.0.1/dashboard-frontend/src/components/StatsCard.tsx +0 -63
- furu-0.0.1/dashboard-frontend/src/components/StatusBadge.tsx +0 -40
- furu-0.0.1/dashboard-frontend/src/components/ui/badge.tsx +0 -35
- furu-0.0.1/dashboard-frontend/src/components/ui/button.tsx +0 -56
- furu-0.0.1/dashboard-frontend/src/components/ui/card.tsx +0 -70
- furu-0.0.1/dashboard-frontend/src/components/ui/input.tsx +0 -20
- furu-0.0.1/dashboard-frontend/src/components/ui/table.tsx +0 -96
- furu-0.0.1/dashboard-frontend/src/index.css +0 -78
- furu-0.0.1/dashboard-frontend/src/lib/api-client.ts +0 -26
- furu-0.0.1/dashboard-frontend/src/lib/utils.ts +0 -6
- furu-0.0.1/dashboard-frontend/src/main.tsx +0 -52
- furu-0.0.1/dashboard-frontend/src/routes/__root.tsx +0 -68
- furu-0.0.1/dashboard-frontend/src/routes/dag.tsx +0 -113
- furu-0.0.1/dashboard-frontend/src/routes/experiments.tsx +0 -444
- furu-0.0.1/dashboard-frontend/src/routes/experiments_.$namespace.$furu_hash.tsx +0 -804
- furu-0.0.1/dashboard-frontend/src/routes/index.tsx +0 -195
- furu-0.0.1/dashboard-frontend/tailwind.config.js +0 -41
- furu-0.0.1/dashboard-frontend/tsconfig.json +0 -24
- furu-0.0.1/dashboard-frontend/tsconfig.node.json +0 -14
- furu-0.0.1/dashboard-frontend/vite.config.ts +0 -21
- furu-0.0.1/e2e/bun.lock +0 -21
- furu-0.0.1/e2e/generate_data.py +0 -347
- furu-0.0.1/e2e/global-setup.ts +0 -37
- furu-0.0.1/e2e/global-teardown.ts +0 -14
- furu-0.0.1/e2e/package.json +0 -15
- furu-0.0.1/e2e/playwright.config.ts +0 -66
- furu-0.0.1/e2e/tests/api.spec.ts +0 -182
- furu-0.0.1/e2e/tests/experiments.spec.ts +0 -163
- furu-0.0.1/e2e/tests/home.spec.ts +0 -36
- furu-0.0.1/e2e/tests/navigation.spec.ts +0 -435
- furu-0.0.1/examples/README.md +0 -21
- furu-0.0.1/examples/my_project/__init__.py +0 -3
- furu-0.0.1/examples/my_project/pipelines.py +0 -65
- furu-0.0.1/examples/run_logging.py +0 -26
- furu-0.0.1/examples/run_nested.py +0 -26
- furu-0.0.1/examples/run_train.py +0 -26
- furu-0.0.1/src/furu/config.py +0 -98
- furu-0.0.1/src/furu/dashboard/frontend/dist/favicon.svg +0 -10
- furu-0.0.1/tasks/migration.md +0 -360
- furu-0.0.1/tests/conftest.py +0 -25
- furu-0.0.1/tests/dashboard/__init__.py +0 -4
- furu-0.0.1/tests/dashboard/conftest.py +0 -448
- furu-0.0.1/tests/dashboard/pipelines.py +0 -192
- furu-0.0.1/tests/dashboard/test_api.py +0 -865
- furu-0.0.1/tests/dashboard/test_scanner.py +0 -786
- furu-0.0.1/tests/test_config.py +0 -10
- furu-0.0.1/tests/test_errors.py +0 -83
- furu-0.0.1/tests/test_furu_core.py +0 -158
- furu-0.0.1/tests/test_furu_inheritance.py +0 -44
- furu-0.0.1/tests/test_furu_inheritance_polymorphic.py +0 -118
- furu-0.0.1/tests/test_furu_list.py +0 -30
- furu-0.0.1/tests/test_furu_typing.py +0 -208
- furu-0.0.1/tests/test_logger.py +0 -141
- furu-0.0.1/tests/test_metadata.py +0 -46
- furu-0.0.1/tests/test_migrations.py +0 -848
- furu-0.0.1/tests/test_raw_dir.py +0 -15
- furu-0.0.1/tests/test_serializer.py +0 -71
- furu-0.0.1/tests/test_state_manager.py +0 -464
- furu-0.0.1/tests/test_submitit_path.py +0 -71
- furu-0.0.1/tests/test_tracebacks.py +0 -5
- furu-0.0.1/uv.lock +0 -745
- furu-0.0.1/version.txt +0 -1
- {furu-0.0.1 → furu-0.0.2}/src/furu/__init__.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/adapters/__init__.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/adapters/submitit.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/core/__init__.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/core/list.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/dashboard/__init__.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/dashboard/__main__.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/dashboard/api/__init__.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/dashboard/api/models.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/dashboard/api/routes.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/dashboard/frontend/dist/assets/index-CbdDfSOZ.css +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/dashboard/frontend/dist/assets/index-DDv_TYB_.js +0 -0
- {furu-0.0.1/dashboard-frontend/public → furu-0.0.2/src/furu/dashboard/frontend/dist}/favicon.svg +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/dashboard/frontend/dist/index.html +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/dashboard/main.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/dashboard/scanner.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/errors.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/migrate.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/migration.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/runtime/__init__.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/runtime/env.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/runtime/logging.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/runtime/tracebacks.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/serialization/__init__.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/serialization/migrations.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/serialization/serializer.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/storage/__init__.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/storage/metadata.py +0 -0
- {furu-0.0.1 → furu-0.0.2}/src/furu/storage/migration.py +0 -0
|
@@ -1,19 +1,20 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
2
|
Name: furu
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.2
|
|
4
4
|
Summary: Cacheable, nested pipelines for Python. Define computations as configs; furu handles caching, state tracking, and result reuse across runs.
|
|
5
|
+
Author: Herman Brunborg
|
|
5
6
|
Author-email: Herman Brunborg <herman@brunborg.com>
|
|
6
|
-
Requires-Python: >=3.12
|
|
7
7
|
Requires-Dist: chz>=0.4.0
|
|
8
8
|
Requires-Dist: cloudpickle>=3.1.1
|
|
9
9
|
Requires-Dist: pydantic>=2.12.5
|
|
10
10
|
Requires-Dist: python-dotenv>=1.0.0
|
|
11
11
|
Requires-Dist: rich>=14.2.0
|
|
12
12
|
Requires-Dist: submitit>=1.5.3
|
|
13
|
+
Requires-Dist: fastapi>=0.109.0 ; extra == 'dashboard'
|
|
14
|
+
Requires-Dist: uvicorn[standard]>=0.27.0 ; extra == 'dashboard'
|
|
15
|
+
Requires-Dist: typer>=0.9.0 ; extra == 'dashboard'
|
|
16
|
+
Requires-Python: >=3.12
|
|
13
17
|
Provides-Extra: dashboard
|
|
14
|
-
Requires-Dist: fastapi>=0.109.0; extra == 'dashboard'
|
|
15
|
-
Requires-Dist: typer>=0.9.0; extra == 'dashboard'
|
|
16
|
-
Requires-Dist: uvicorn[standard]>=0.27.0; extra == 'dashboard'
|
|
17
18
|
Description-Content-Type: text/markdown
|
|
18
19
|
|
|
19
20
|
# furu
|
|
@@ -132,20 +133,25 @@ class TrainTextModel(furu.Furu[str]):
|
|
|
132
133
|
|
|
133
134
|
### Storage Structure
|
|
134
135
|
|
|
136
|
+
Furu uses two roots: `FURU_PATH` for `data/` + `raw/`, and
|
|
137
|
+
`FURU_VERSION_CONTROLLED_PATH` for `artifacts/`. Defaults:
|
|
138
|
+
|
|
139
|
+
```
|
|
140
|
+
FURU_PATH=<project>/furu-data
|
|
141
|
+
FURU_VERSION_CONTROLLED_PATH=<project>/furu-data/artifacts
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
`<project>` is the nearest directory containing `pyproject.toml` (falling back to
|
|
145
|
+
the git root). This means you can move `FURU_PATH` without relocating artifacts.
|
|
146
|
+
|
|
135
147
|
```
|
|
136
148
|
$FURU_PATH/
|
|
137
|
-
├── data/ #
|
|
138
|
-
│ └── <module>/<Class>/
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
│ │ ├── furu.log # Captured logs
|
|
144
|
-
│ │ └── SUCCESS.json # Marker file
|
|
145
|
-
│ └── <your outputs> # Files from _create()
|
|
146
|
-
├── git/ # For version_controlled=True
|
|
147
|
-
│ └── <same structure>
|
|
148
|
-
└── raw/ # Shared directory for large files
|
|
149
|
+
├── data/ # version_controlled=False
|
|
150
|
+
│ └── <module>/<Class>/<hash>/
|
|
151
|
+
└── raw/
|
|
152
|
+
|
|
153
|
+
$FURU_VERSION_CONTROLLED_PATH/ # version_controlled=True
|
|
154
|
+
└── <module>/<Class>/<hash>/
|
|
149
155
|
```
|
|
150
156
|
|
|
151
157
|
## Features
|
|
@@ -259,10 +265,17 @@ For artifacts that should be stored separately (e.g., checked into git):
|
|
|
259
265
|
|
|
260
266
|
```python
|
|
261
267
|
class VersionedConfig(furu.Furu[dict], version_controlled=True):
|
|
262
|
-
# Stored under $
|
|
268
|
+
# Stored under $FURU_VERSION_CONTROLLED_PATH
|
|
269
|
+
# Default: <project>/furu-data/artifacts
|
|
263
270
|
...
|
|
264
271
|
```
|
|
265
272
|
|
|
273
|
+
`<project>` is the nearest directory containing `pyproject.toml`, or the git root
|
|
274
|
+
if `pyproject.toml` is missing.
|
|
275
|
+
|
|
276
|
+
It is typical to keep `furu-data/data/` and `furu-data/raw/` in `.gitignore` while
|
|
277
|
+
committing `furu-data/artifacts/`.
|
|
278
|
+
|
|
266
279
|
## Logging
|
|
267
280
|
|
|
268
281
|
Furu installs stdlib `logging` handlers that capture logs to per-artifact files.
|
|
@@ -397,9 +410,11 @@ The `/api/experiments` endpoint supports:
|
|
|
397
410
|
|
|
398
411
|
| Variable | Default | Description |
|
|
399
412
|
|----------|---------|-------------|
|
|
400
|
-
| `FURU_PATH` |
|
|
413
|
+
| `FURU_PATH` | `<project>/furu-data` | Base storage directory for non-versioned artifacts |
|
|
414
|
+
| `FURU_VERSION_CONTROLLED_PATH` | `<project>/furu-data/artifacts` | Override version-controlled storage root |
|
|
401
415
|
| `FURU_LOG_LEVEL` | `INFO` | Console verbosity (`DEBUG`, `INFO`, `WARNING`, `ERROR`) |
|
|
402
416
|
| `FURU_IGNORE_DIFF` | `false` | Skip embedding git diff in metadata |
|
|
417
|
+
| `FURU_ALWAYS_RERUN` | `""` | Comma-separated class qualnames to always rerun (use `ALL` to bypass cache globally; cannot combine with other entries; entries must be importable) |
|
|
403
418
|
| `FURU_POLL_INTERVAL_SECS` | `10` | Polling interval for queued/running jobs |
|
|
404
419
|
| `FURU_WAIT_LOG_EVERY_SECS` | `10` | Interval between "waiting" log messages |
|
|
405
420
|
| `FURU_STALE_AFTER_SECS` | `1800` | Consider running jobs stale after this duration |
|
|
@@ -114,20 +114,25 @@ class TrainTextModel(furu.Furu[str]):
|
|
|
114
114
|
|
|
115
115
|
### Storage Structure
|
|
116
116
|
|
|
117
|
+
Furu uses two roots: `FURU_PATH` for `data/` + `raw/`, and
|
|
118
|
+
`FURU_VERSION_CONTROLLED_PATH` for `artifacts/`. Defaults:
|
|
119
|
+
|
|
120
|
+
```
|
|
121
|
+
FURU_PATH=<project>/furu-data
|
|
122
|
+
FURU_VERSION_CONTROLLED_PATH=<project>/furu-data/artifacts
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
`<project>` is the nearest directory containing `pyproject.toml` (falling back to
|
|
126
|
+
the git root). This means you can move `FURU_PATH` without relocating artifacts.
|
|
127
|
+
|
|
117
128
|
```
|
|
118
129
|
$FURU_PATH/
|
|
119
|
-
├── data/ #
|
|
120
|
-
│ └── <module>/<Class>/
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
│ │ ├── furu.log # Captured logs
|
|
126
|
-
│ │ └── SUCCESS.json # Marker file
|
|
127
|
-
│ └── <your outputs> # Files from _create()
|
|
128
|
-
├── git/ # For version_controlled=True
|
|
129
|
-
│ └── <same structure>
|
|
130
|
-
└── raw/ # Shared directory for large files
|
|
130
|
+
├── data/ # version_controlled=False
|
|
131
|
+
│ └── <module>/<Class>/<hash>/
|
|
132
|
+
└── raw/
|
|
133
|
+
|
|
134
|
+
$FURU_VERSION_CONTROLLED_PATH/ # version_controlled=True
|
|
135
|
+
└── <module>/<Class>/<hash>/
|
|
131
136
|
```
|
|
132
137
|
|
|
133
138
|
## Features
|
|
@@ -241,10 +246,17 @@ For artifacts that should be stored separately (e.g., checked into git):
|
|
|
241
246
|
|
|
242
247
|
```python
|
|
243
248
|
class VersionedConfig(furu.Furu[dict], version_controlled=True):
|
|
244
|
-
# Stored under $
|
|
249
|
+
# Stored under $FURU_VERSION_CONTROLLED_PATH
|
|
250
|
+
# Default: <project>/furu-data/artifacts
|
|
245
251
|
...
|
|
246
252
|
```
|
|
247
253
|
|
|
254
|
+
`<project>` is the nearest directory containing `pyproject.toml`, or the git root
|
|
255
|
+
if `pyproject.toml` is missing.
|
|
256
|
+
|
|
257
|
+
It is typical to keep `furu-data/data/` and `furu-data/raw/` in `.gitignore` while
|
|
258
|
+
committing `furu-data/artifacts/`.
|
|
259
|
+
|
|
248
260
|
## Logging
|
|
249
261
|
|
|
250
262
|
Furu installs stdlib `logging` handlers that capture logs to per-artifact files.
|
|
@@ -379,9 +391,11 @@ The `/api/experiments` endpoint supports:
|
|
|
379
391
|
|
|
380
392
|
| Variable | Default | Description |
|
|
381
393
|
|----------|---------|-------------|
|
|
382
|
-
| `FURU_PATH` |
|
|
394
|
+
| `FURU_PATH` | `<project>/furu-data` | Base storage directory for non-versioned artifacts |
|
|
395
|
+
| `FURU_VERSION_CONTROLLED_PATH` | `<project>/furu-data/artifacts` | Override version-controlled storage root |
|
|
383
396
|
| `FURU_LOG_LEVEL` | `INFO` | Console verbosity (`DEBUG`, `INFO`, `WARNING`, `ERROR`) |
|
|
384
397
|
| `FURU_IGNORE_DIFF` | `false` | Skip embedding git diff in metadata |
|
|
398
|
+
| `FURU_ALWAYS_RERUN` | `""` | Comma-separated class qualnames to always rerun (use `ALL` to bypass cache globally; cannot combine with other entries; entries must be importable) |
|
|
385
399
|
| `FURU_POLL_INTERVAL_SECS` | `10` | Polling interval for queued/running jobs |
|
|
386
400
|
| `FURU_WAIT_LOG_EVERY_SECS` | `10` | Interval between "waiting" log messages |
|
|
387
401
|
| `FURU_STALE_AFTER_SECS` | `1800` | Consider running jobs stale after this duration |
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "furu"
|
|
3
|
-
version = "0.0.
|
|
3
|
+
version = "0.0.2"
|
|
4
4
|
description = "Cacheable, nested pipelines for Python. Define computations as configs; furu handles caching, state tracking, and result reuse across runs."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
authors = [
|
|
@@ -42,14 +42,8 @@ test = [
|
|
|
42
42
|
]
|
|
43
43
|
|
|
44
44
|
[build-system]
|
|
45
|
-
requires = ["
|
|
46
|
-
build-backend = "
|
|
47
|
-
|
|
48
|
-
[tool.hatch.build]
|
|
49
|
-
artifacts = ["src/furu/dashboard/frontend/dist/**/*"]
|
|
50
|
-
|
|
51
|
-
[tool.hatch.build.targets.wheel]
|
|
52
|
-
packages = ["src/furu"]
|
|
45
|
+
requires = ["uv_build>=0.9.26,<0.10.0"]
|
|
46
|
+
build-backend = "uv_build"
|
|
53
47
|
|
|
54
48
|
[tool.pytest.ini_options]
|
|
55
49
|
testpaths = ["tests"]
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from importlib import import_module
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class FuruConfig:
|
|
7
|
+
"""Central configuration for Furu behavior."""
|
|
8
|
+
|
|
9
|
+
DEFAULT_ROOT_DIR = Path("furu-data")
|
|
10
|
+
VERSION_CONTROLLED_SUBDIR = DEFAULT_ROOT_DIR / "artifacts"
|
|
11
|
+
|
|
12
|
+
def __init__(self):
|
|
13
|
+
def _get_base_root() -> Path:
|
|
14
|
+
env = os.getenv("FURU_PATH")
|
|
15
|
+
if env:
|
|
16
|
+
return Path(env).expanduser().resolve()
|
|
17
|
+
project_root = self._find_project_root(fallback_to_cwd=True)
|
|
18
|
+
return (project_root / self.DEFAULT_ROOT_DIR).resolve()
|
|
19
|
+
|
|
20
|
+
self.base_root = _get_base_root()
|
|
21
|
+
self.version_controlled_root_override = self._get_version_controlled_override()
|
|
22
|
+
self.poll_interval = float(os.getenv("FURU_POLL_INTERVAL_SECS", "10"))
|
|
23
|
+
self.wait_log_every_sec = float(os.getenv("FURU_WAIT_LOG_EVERY_SECS", "10"))
|
|
24
|
+
self.stale_timeout = float(os.getenv("FURU_STALE_AFTER_SECS", str(30 * 60)))
|
|
25
|
+
self.lease_duration_sec = float(os.getenv("FURU_LEASE_SECS", "120"))
|
|
26
|
+
hb = os.getenv("FURU_HEARTBEAT_SECS")
|
|
27
|
+
self.heartbeat_interval_sec = (
|
|
28
|
+
float(hb) if hb is not None else max(1.0, self.lease_duration_sec / 3.0)
|
|
29
|
+
)
|
|
30
|
+
self.max_requeues = int(os.getenv("FURU_PREEMPT_MAX", "5"))
|
|
31
|
+
self.ignore_git_diff = os.getenv("FURU_IGNORE_DIFF", "0").lower() in {
|
|
32
|
+
"1",
|
|
33
|
+
"true",
|
|
34
|
+
"yes",
|
|
35
|
+
}
|
|
36
|
+
self.require_git = os.getenv("FURU_REQUIRE_GIT", "1").lower() in {
|
|
37
|
+
"1",
|
|
38
|
+
"true",
|
|
39
|
+
"yes",
|
|
40
|
+
}
|
|
41
|
+
self.require_git_remote = os.getenv("FURU_REQUIRE_GIT_REMOTE", "1").lower() in {
|
|
42
|
+
"1",
|
|
43
|
+
"true",
|
|
44
|
+
"yes",
|
|
45
|
+
}
|
|
46
|
+
always_rerun_items = {
|
|
47
|
+
item.strip()
|
|
48
|
+
for item in os.getenv("FURU_ALWAYS_RERUN", "").split(",")
|
|
49
|
+
if item.strip()
|
|
50
|
+
}
|
|
51
|
+
all_entries = {item for item in always_rerun_items if item.lower() == "all"}
|
|
52
|
+
if all_entries and len(always_rerun_items) > len(all_entries):
|
|
53
|
+
raise ValueError(
|
|
54
|
+
"FURU_ALWAYS_RERUN cannot combine 'ALL' with specific entries"
|
|
55
|
+
)
|
|
56
|
+
self.always_rerun_all = bool(all_entries)
|
|
57
|
+
if self.always_rerun_all:
|
|
58
|
+
always_rerun_items = {
|
|
59
|
+
item for item in always_rerun_items if item.lower() != "all"
|
|
60
|
+
}
|
|
61
|
+
self._require_namespaces_exist(always_rerun_items)
|
|
62
|
+
self.always_rerun = always_rerun_items
|
|
63
|
+
self.cancelled_is_preempted = os.getenv(
|
|
64
|
+
"FURU_CANCELLED_IS_PREEMPTED", "false"
|
|
65
|
+
).lower() in {"1", "true", "yes"}
|
|
66
|
+
|
|
67
|
+
# Parse FURU_CACHE_METADATA: "never", "forever", or duration like "5m", "1h"
|
|
68
|
+
# Default: "5m" (5 minutes) - balances performance with freshness
|
|
69
|
+
self.cache_metadata_ttl_sec: float | None = self._parse_cache_duration(
|
|
70
|
+
os.getenv("FURU_CACHE_METADATA", "5m")
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
@staticmethod
|
|
74
|
+
def _parse_cache_duration(value: str) -> float | None:
|
|
75
|
+
"""Parse cache duration string into seconds. Returns None for 'never', float('inf') for 'forever'."""
|
|
76
|
+
value = value.strip().lower()
|
|
77
|
+
if value in {"never", "0", "false", "no"}:
|
|
78
|
+
return None # No caching
|
|
79
|
+
if value in {"forever", "inf", "true", "yes", "1"}:
|
|
80
|
+
return float("inf") # Cache forever
|
|
81
|
+
|
|
82
|
+
# Parse duration like "5m", "1h", "30s"
|
|
83
|
+
import re
|
|
84
|
+
|
|
85
|
+
match = re.match(r"^(\d+(?:\.\d+)?)\s*([smh]?)$", value)
|
|
86
|
+
if not match:
|
|
87
|
+
raise ValueError(
|
|
88
|
+
f"Invalid FURU_CACHE_METADATA value: {value!r}. "
|
|
89
|
+
"Use 'never', 'forever', or duration like '5m', '1h', '30s'"
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
num = float(match.group(1))
|
|
93
|
+
unit = match.group(2) or "s"
|
|
94
|
+
multipliers = {"s": 1, "m": 60, "h": 3600}
|
|
95
|
+
return num * multipliers[unit]
|
|
96
|
+
|
|
97
|
+
def get_root(self, version_controlled: bool = False) -> Path:
|
|
98
|
+
"""Get root directory for storage (version_controlled uses its own root)."""
|
|
99
|
+
if version_controlled:
|
|
100
|
+
if self.version_controlled_root_override is not None:
|
|
101
|
+
return self.version_controlled_root_override
|
|
102
|
+
return self._resolve_version_controlled_root()
|
|
103
|
+
return self.base_root / "data"
|
|
104
|
+
|
|
105
|
+
@classmethod
|
|
106
|
+
def _get_version_controlled_override(cls) -> Path | None:
|
|
107
|
+
env = os.getenv("FURU_VERSION_CONTROLLED_PATH")
|
|
108
|
+
if env:
|
|
109
|
+
return Path(env).expanduser().resolve()
|
|
110
|
+
return None
|
|
111
|
+
|
|
112
|
+
@classmethod
|
|
113
|
+
def _resolve_version_controlled_root(cls) -> Path:
|
|
114
|
+
project_root = cls._find_project_root()
|
|
115
|
+
return (project_root / cls.VERSION_CONTROLLED_SUBDIR).resolve()
|
|
116
|
+
|
|
117
|
+
@staticmethod
|
|
118
|
+
def _find_project_root(
|
|
119
|
+
start: Path | None = None, *, fallback_to_cwd: bool = False
|
|
120
|
+
) -> Path:
|
|
121
|
+
base = (start or Path.cwd()).resolve()
|
|
122
|
+
git_root: Path | None = None
|
|
123
|
+
for path in (base, *base.parents):
|
|
124
|
+
if (path / "pyproject.toml").is_file():
|
|
125
|
+
return path
|
|
126
|
+
if git_root is None and (path / ".git").exists():
|
|
127
|
+
git_root = path
|
|
128
|
+
if git_root is not None:
|
|
129
|
+
return git_root
|
|
130
|
+
if fallback_to_cwd:
|
|
131
|
+
return base
|
|
132
|
+
raise ValueError(
|
|
133
|
+
"Cannot locate pyproject.toml or .git to determine version-controlled root. "
|
|
134
|
+
"Set FURU_VERSION_CONTROLLED_PATH to override."
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
@staticmethod
|
|
138
|
+
def _require_namespaces_exist(namespaces: set[str]) -> None:
|
|
139
|
+
if not namespaces:
|
|
140
|
+
return
|
|
141
|
+
missing_sentinel = object()
|
|
142
|
+
for namespace in namespaces:
|
|
143
|
+
module_name, _, qualname = namespace.rpartition(".")
|
|
144
|
+
if not module_name or not qualname:
|
|
145
|
+
raise ValueError(
|
|
146
|
+
"FURU_ALWAYS_RERUN entries must be 'module.QualifiedName', "
|
|
147
|
+
f"got {namespace!r}"
|
|
148
|
+
)
|
|
149
|
+
target: object = import_module(module_name)
|
|
150
|
+
for attr in qualname.split("."):
|
|
151
|
+
value = getattr(target, attr, missing_sentinel)
|
|
152
|
+
if value is missing_sentinel:
|
|
153
|
+
raise ValueError(
|
|
154
|
+
"FURU_ALWAYS_RERUN entry does not exist: "
|
|
155
|
+
f"{namespace!r}"
|
|
156
|
+
)
|
|
157
|
+
target = value
|
|
158
|
+
|
|
159
|
+
@property
|
|
160
|
+
def raw_dir(self) -> Path:
|
|
161
|
+
return self.base_root / "raw"
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
FURU_CONFIG = FuruConfig()
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def get_furu_root(*, version_controlled: bool = False) -> Path:
|
|
168
|
+
return FURU_CONFIG.get_root(version_controlled=version_controlled)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def set_furu_root(path: Path) -> None:
|
|
172
|
+
FURU_CONFIG.base_root = path.resolve()
|
|
@@ -204,11 +204,13 @@ class Furu[T](ABC):
|
|
|
204
204
|
"""Compute hash of this object's content for storage identification."""
|
|
205
205
|
return FuruSerializer.compute_hash(self)
|
|
206
206
|
|
|
207
|
-
def
|
|
208
|
-
if
|
|
207
|
+
def _always_rerun(self: Self) -> bool:
|
|
208
|
+
if FURU_CONFIG.always_rerun_all:
|
|
209
|
+
return True
|
|
210
|
+
if not FURU_CONFIG.always_rerun:
|
|
209
211
|
return False
|
|
210
212
|
qualname = f"{self.__class__.__module__}.{self.__class__.__qualname__}"
|
|
211
|
-
return qualname in FURU_CONFIG.
|
|
213
|
+
return qualname in FURU_CONFIG.always_rerun
|
|
212
214
|
|
|
213
215
|
def _base_furu_dir(self: Self) -> Path:
|
|
214
216
|
root = FURU_CONFIG.get_root(self.version_controlled)
|
|
@@ -333,12 +335,12 @@ class Furu[T](ABC):
|
|
|
333
335
|
)
|
|
334
336
|
migration = MigrationManager.read_migration(base_dir)
|
|
335
337
|
|
|
336
|
-
if alias_active and self.
|
|
338
|
+
if alias_active and self._always_rerun():
|
|
337
339
|
if migration is not None:
|
|
338
340
|
self._maybe_detach_alias(
|
|
339
341
|
directory=base_dir,
|
|
340
342
|
record=migration,
|
|
341
|
-
reason="
|
|
343
|
+
reason="always_rerun",
|
|
342
344
|
)
|
|
343
345
|
migration = MigrationManager.read_migration(base_dir)
|
|
344
346
|
alias_active = False
|
|
@@ -350,9 +352,9 @@ class Furu[T](ABC):
|
|
|
350
352
|
success_marker = StateManager.get_success_marker_path(directory)
|
|
351
353
|
if success_marker.is_file():
|
|
352
354
|
# We have a success marker. Check if we can use it.
|
|
353
|
-
if self.
|
|
355
|
+
if self._always_rerun():
|
|
354
356
|
self._invalidate_cached_success(
|
|
355
|
-
directory, reason="
|
|
357
|
+
directory, reason="always_rerun enabled"
|
|
356
358
|
)
|
|
357
359
|
# Fall through to normal load
|
|
358
360
|
else:
|
|
@@ -381,9 +383,9 @@ class Furu[T](ABC):
|
|
|
381
383
|
needs_reconcile = True
|
|
382
384
|
if isinstance(state0.result, _StateResultSuccess):
|
|
383
385
|
# Double check logic if we fell through to here (e.g. race condition or invalidation above)
|
|
384
|
-
if self.
|
|
386
|
+
if self._always_rerun():
|
|
385
387
|
self._invalidate_cached_success(
|
|
386
|
-
directory, reason="
|
|
388
|
+
directory, reason="always_rerun enabled"
|
|
387
389
|
)
|
|
388
390
|
state0 = StateManager.read_state(directory)
|
|
389
391
|
else:
|
|
@@ -1008,6 +1008,35 @@ def compute_lock(
|
|
|
1008
1008
|
FuruLockNotAcquired: If lock cannot be acquired (after waiting)
|
|
1009
1009
|
FuruWaitTimeout: If max_wait_time_sec is exceeded
|
|
1010
1010
|
"""
|
|
1011
|
+
def _format_wait_duration(seconds: float) -> str:
|
|
1012
|
+
if seconds < 60.0:
|
|
1013
|
+
return f"{seconds:.1f}s"
|
|
1014
|
+
minutes = seconds / 60.0
|
|
1015
|
+
if minutes < 60.0:
|
|
1016
|
+
return f"{minutes:.1f}m"
|
|
1017
|
+
hours = minutes / 60.0
|
|
1018
|
+
if hours < 24.0:
|
|
1019
|
+
return f"{hours:.1f}h"
|
|
1020
|
+
days = hours / 24.0
|
|
1021
|
+
return f"{days:.1f}d"
|
|
1022
|
+
|
|
1023
|
+
def _describe_wait(attempt: _StateAttempt, waited_sec: float) -> str:
|
|
1024
|
+
label = "last heartbeat"
|
|
1025
|
+
timestamp = attempt.heartbeat_at
|
|
1026
|
+
if attempt.status == "queued":
|
|
1027
|
+
label = "queued at"
|
|
1028
|
+
timestamp = attempt.started_at
|
|
1029
|
+
parsed = StateManager._parse_time(timestamp)
|
|
1030
|
+
timestamp_info = timestamp
|
|
1031
|
+
if parsed is not None:
|
|
1032
|
+
age = (StateManager._utcnow() - parsed).total_seconds()
|
|
1033
|
+
timestamp_info = f"{timestamp} ({_format_wait_duration(age)} ago)"
|
|
1034
|
+
return (
|
|
1035
|
+
"waited "
|
|
1036
|
+
f"{_format_wait_duration(waited_sec)}, {label} {timestamp_info}, "
|
|
1037
|
+
f"status {attempt.status}, backend {attempt.backend}"
|
|
1038
|
+
)
|
|
1039
|
+
|
|
1011
1040
|
lock_path = StateManager.get_lock_path(directory, StateManager.COMPUTE_LOCK)
|
|
1012
1041
|
|
|
1013
1042
|
lock_fd: int | None = None
|
|
@@ -1031,6 +1060,49 @@ def compute_lock(
|
|
|
1031
1060
|
|
|
1032
1061
|
lock_fd = StateManager.try_lock(lock_path)
|
|
1033
1062
|
if lock_fd is not None:
|
|
1063
|
+
state = StateManager.read_state(directory)
|
|
1064
|
+
if isinstance(state.result, _StateResultSuccess):
|
|
1065
|
+
StateManager.release_lock(lock_fd, lock_path)
|
|
1066
|
+
raise FuruLockNotAcquired(
|
|
1067
|
+
"Cannot acquire lock: experiment already succeeded"
|
|
1068
|
+
)
|
|
1069
|
+
if isinstance(state.result, _StateResultFailed):
|
|
1070
|
+
StateManager.release_lock(lock_fd, lock_path)
|
|
1071
|
+
raise FuruLockNotAcquired("Cannot acquire lock: experiment already failed")
|
|
1072
|
+
attempt = state.attempt
|
|
1073
|
+
if (
|
|
1074
|
+
isinstance(attempt, (_StateAttemptQueued, _StateAttemptRunning))
|
|
1075
|
+
and attempt.backend != backend
|
|
1076
|
+
):
|
|
1077
|
+
StateManager.release_lock(lock_fd, lock_path)
|
|
1078
|
+
lock_fd = None
|
|
1079
|
+
if reconcile_fn is not None:
|
|
1080
|
+
reconcile_fn(directory)
|
|
1081
|
+
state = StateManager.read_state(directory)
|
|
1082
|
+
if isinstance(state.result, _StateResultSuccess):
|
|
1083
|
+
raise FuruLockNotAcquired(
|
|
1084
|
+
"Cannot acquire lock: experiment already succeeded"
|
|
1085
|
+
)
|
|
1086
|
+
if isinstance(state.result, _StateResultFailed):
|
|
1087
|
+
raise FuruLockNotAcquired(
|
|
1088
|
+
"Cannot acquire lock: experiment already failed"
|
|
1089
|
+
)
|
|
1090
|
+
attempt = state.attempt
|
|
1091
|
+
if not isinstance(attempt, (_StateAttemptQueued, _StateAttemptRunning)):
|
|
1092
|
+
continue
|
|
1093
|
+
if attempt.backend == backend:
|
|
1094
|
+
continue
|
|
1095
|
+
now = time.time()
|
|
1096
|
+
if now >= next_wait_log_at:
|
|
1097
|
+
waited_sec = now - start_time
|
|
1098
|
+
logger.info(
|
|
1099
|
+
"compute_lock: waiting for lock creation %s (%s)",
|
|
1100
|
+
directory,
|
|
1101
|
+
_describe_wait(attempt, waited_sec),
|
|
1102
|
+
)
|
|
1103
|
+
next_wait_log_at = now + wait_log_every_sec
|
|
1104
|
+
time.sleep(poll_interval_sec)
|
|
1105
|
+
continue
|
|
1034
1106
|
break
|
|
1035
1107
|
|
|
1036
1108
|
# Lock held by someone else - reconcile and check state
|
|
@@ -1064,9 +1136,11 @@ def compute_lock(
|
|
|
1064
1136
|
# Active attempt exists - wait for it
|
|
1065
1137
|
now = time.time()
|
|
1066
1138
|
if now >= next_wait_log_at:
|
|
1139
|
+
waited_sec = now - start_time
|
|
1067
1140
|
logger.info(
|
|
1068
|
-
"compute_lock: waiting for lock %s",
|
|
1141
|
+
"compute_lock: waiting for lock %s (%s)",
|
|
1069
1142
|
directory,
|
|
1143
|
+
_describe_wait(attempt, waited_sec),
|
|
1070
1144
|
)
|
|
1071
1145
|
next_wait_log_at = now + wait_log_every_sec
|
|
1072
1146
|
time.sleep(poll_interval_sec)
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
name: CI
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
pull_request:
|
|
5
|
-
|
|
6
|
-
jobs:
|
|
7
|
-
test:
|
|
8
|
-
runs-on: ubuntu-latest
|
|
9
|
-
steps:
|
|
10
|
-
- uses: actions/checkout@v4
|
|
11
|
-
|
|
12
|
-
- name: Install uv
|
|
13
|
-
uses: astral-sh/setup-uv@v4
|
|
14
|
-
|
|
15
|
-
- name: Set up Python
|
|
16
|
-
run: uv python install
|
|
17
|
-
|
|
18
|
-
- name: Install bun
|
|
19
|
-
uses: oven-sh/setup-bun@v2
|
|
20
|
-
|
|
21
|
-
- name: Install dependencies
|
|
22
|
-
run: |
|
|
23
|
-
uv sync --all-extras
|
|
24
|
-
cd dashboard-frontend && bun install
|
|
25
|
-
|
|
26
|
-
- name: Install e2e dependencies
|
|
27
|
-
run: make dashboard-install-e2e
|
|
28
|
-
|
|
29
|
-
- name: Build frontend artifacts
|
|
30
|
-
run: make frontend-build
|
|
31
|
-
|
|
32
|
-
- name: Run tests
|
|
33
|
-
run: make test-all
|
|
@@ -1,99 +0,0 @@
|
|
|
1
|
-
name: Release
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
push:
|
|
5
|
-
branches:
|
|
6
|
-
- main
|
|
7
|
-
paths:
|
|
8
|
-
- pyproject.toml
|
|
9
|
-
|
|
10
|
-
permissions:
|
|
11
|
-
contents: write
|
|
12
|
-
id-token: write
|
|
13
|
-
|
|
14
|
-
jobs:
|
|
15
|
-
release:
|
|
16
|
-
runs-on: ubuntu-latest
|
|
17
|
-
environment: pypi
|
|
18
|
-
steps:
|
|
19
|
-
- uses: actions/checkout@v4
|
|
20
|
-
with:
|
|
21
|
-
fetch-depth: 0
|
|
22
|
-
|
|
23
|
-
- name: Install uv
|
|
24
|
-
uses: astral-sh/setup-uv@v4
|
|
25
|
-
|
|
26
|
-
- name: Set up Python
|
|
27
|
-
run: uv python install
|
|
28
|
-
|
|
29
|
-
- name: Install bun
|
|
30
|
-
uses: oven-sh/setup-bun@v2
|
|
31
|
-
|
|
32
|
-
- name: Read version
|
|
33
|
-
id: version
|
|
34
|
-
run: |
|
|
35
|
-
python - <<'PY'
|
|
36
|
-
import tomllib
|
|
37
|
-
from pathlib import Path
|
|
38
|
-
|
|
39
|
-
data = tomllib.loads(Path("pyproject.toml").read_text())
|
|
40
|
-
version = data["project"]["version"]
|
|
41
|
-
print(f"version={version}")
|
|
42
|
-
Path("version.txt").write_text(version)
|
|
43
|
-
PY
|
|
44
|
-
echo "version=$(cat version.txt)" >> $GITHUB_OUTPUT
|
|
45
|
-
|
|
46
|
-
- name: Check release
|
|
47
|
-
id: tag
|
|
48
|
-
run: |
|
|
49
|
-
git fetch --tags
|
|
50
|
-
if git rev-parse "v${{ steps.version.outputs.version }}" >/dev/null 2>&1; then
|
|
51
|
-
echo "should_release=false" >> $GITHUB_OUTPUT
|
|
52
|
-
exit 0
|
|
53
|
-
fi
|
|
54
|
-
|
|
55
|
-
if git diff -U0 HEAD^ HEAD -- pyproject.toml | grep -E '^[+-]version = "' >/dev/null; then
|
|
56
|
-
echo "should_release=true" >> $GITHUB_OUTPUT
|
|
57
|
-
else
|
|
58
|
-
echo "should_release=false" >> $GITHUB_OUTPUT
|
|
59
|
-
fi
|
|
60
|
-
|
|
61
|
-
- name: Install dependencies
|
|
62
|
-
if: steps.tag.outputs.should_release == 'true'
|
|
63
|
-
run: |
|
|
64
|
-
uv sync --all-extras
|
|
65
|
-
cd dashboard-frontend && bun install
|
|
66
|
-
|
|
67
|
-
- name: Install e2e dependencies
|
|
68
|
-
if: steps.tag.outputs.should_release == 'true'
|
|
69
|
-
run: make dashboard-install-e2e
|
|
70
|
-
|
|
71
|
-
- name: Build frontend
|
|
72
|
-
if: steps.tag.outputs.should_release == 'true'
|
|
73
|
-
run: make frontend-build
|
|
74
|
-
|
|
75
|
-
- name: Run tests
|
|
76
|
-
if: steps.tag.outputs.should_release == 'true'
|
|
77
|
-
run: make test-all
|
|
78
|
-
|
|
79
|
-
- name: Build package
|
|
80
|
-
if: steps.tag.outputs.should_release == 'true'
|
|
81
|
-
run: uv build
|
|
82
|
-
|
|
83
|
-
- name: Create tag
|
|
84
|
-
if: steps.tag.outputs.should_release == 'true'
|
|
85
|
-
run: |
|
|
86
|
-
git tag "v${{ steps.version.outputs.version }}"
|
|
87
|
-
git push origin "v${{ steps.version.outputs.version }}"
|
|
88
|
-
|
|
89
|
-
- name: Publish to PyPI
|
|
90
|
-
if: steps.tag.outputs.should_release == 'true'
|
|
91
|
-
uses: pypa/gh-action-pypi-publish@release/v1
|
|
92
|
-
|
|
93
|
-
- name: Create GitHub Release
|
|
94
|
-
if: steps.tag.outputs.should_release == 'true'
|
|
95
|
-
uses: softprops/action-gh-release@v2
|
|
96
|
-
with:
|
|
97
|
-
tag_name: "v${{ steps.version.outputs.version }}"
|
|
98
|
-
files: dist/*
|
|
99
|
-
generate_release_notes: true
|