flexlock 0.8.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. flexlock-0.8.2/.claude/settings.local.json +7 -0
  2. flexlock-0.8.2/.claude/worktrees/dedup-deps/.claude/settings.local.json +7 -0
  3. flexlock-0.8.2/.claude/worktrees/dedup-deps/.gitattributes +2 -0
  4. flexlock-0.8.2/.claude/worktrees/dedup-deps/.gitignore +16 -0
  5. flexlock-0.8.2/.claude/worktrees/dedup-deps/FABLE_IMPLEMENTATION_PLAN.md +368 -0
  6. flexlock-0.8.2/.claude/worktrees/dedup-deps/README.md +341 -0
  7. flexlock-0.8.2/.claude/worktrees/dedup-deps/docs/README.md +23 -0
  8. flexlock-0.8.2/.claude/worktrees/dedup-deps/docs/agentic_workflows.md +131 -0
  9. flexlock-0.8.2/.claude/worktrees/dedup-deps/docs/api.rst +127 -0
  10. flexlock-0.8.2/.claude/worktrees/dedup-deps/docs/cli_reference.md +811 -0
  11. flexlock-0.8.2/.claude/worktrees/dedup-deps/docs/conf.py +64 -0
  12. flexlock-0.8.2/.claude/worktrees/dedup-deps/docs/debugging.md +352 -0
  13. flexlock-0.8.2/.claude/worktrees/dedup-deps/docs/hpc_integration.md +712 -0
  14. flexlock-0.8.2/.claude/worktrees/dedup-deps/docs/index.rst +40 -0
  15. flexlock-0.8.2/.claude/worktrees/dedup-deps/docs/mlflow_integration.md +960 -0
  16. flexlock-0.8.2/.claude/worktrees/dedup-deps/docs/philosophy.md +171 -0
  17. flexlock-0.8.2/.claude/worktrees/dedup-deps/docs/python_api.md +905 -0
  18. flexlock-0.8.2/.claude/worktrees/dedup-deps/docs/quickstart.md +175 -0
  19. flexlock-0.8.2/.claude/worktrees/dedup-deps/docs/reference.md +305 -0
  20. flexlock-0.8.2/.claude/worktrees/dedup-deps/docs/requirements.txt +10 -0
  21. flexlock-0.8.2/.claude/worktrees/dedup-deps/docs/resolution_simplification_plan.md +226 -0
  22. flexlock-0.8.2/.claude/worktrees/dedup-deps/docs/resolvers.md +445 -0
  23. flexlock-0.8.2/.claude/worktrees/dedup-deps/docs/upload_docs.sh +17 -0
  24. flexlock-0.8.2/.claude/worktrees/dedup-deps/docs/usage_guide.md +1075 -0
  25. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/__init__.py +87 -0
  26. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/api.py +1362 -0
  27. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/backends/__init__.py +11 -0
  28. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/backends/base.py +41 -0
  29. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/backends/pbs.py +219 -0
  30. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/backends/slurm.py +333 -0
  31. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/cli.py +953 -0
  32. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/config.py +107 -0
  33. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/context.py +7 -0
  34. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/data_hash.py +310 -0
  35. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/debug.py +473 -0
  36. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/diff.py +253 -0
  37. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/diff_cli.py +155 -0
  38. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/exceptions.py +50 -0
  39. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/export.py +134 -0
  40. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/fingerprint.py +129 -0
  41. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/flexcli.py +163 -0
  42. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/freeze.py +291 -0
  43. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/git_utils.py +196 -0
  44. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/index.py +300 -0
  45. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/load_stage.py +60 -0
  46. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/mlflow.py +192 -0
  47. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/parallel.py +378 -0
  48. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/query.py +739 -0
  49. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/report.py +78 -0
  50. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/report_template.html +246 -0
  51. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/resolvers.py +248 -0
  52. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/run_cli.py +44 -0
  53. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/run_record.py +173 -0
  54. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/runner.py +560 -0
  55. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/save_dir.py +124 -0
  56. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/skills/flexlock-new-stage/SKILL.md +56 -0
  57. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/skills/flexlock-report/SKILL.md +50 -0
  58. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/skills/flexlock-run-and-watch/SKILL.md +58 -0
  59. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/skills/flexlock-survey/SKILL.md +61 -0
  60. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/snapshot.py +245 -0
  61. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/status_cli.py +299 -0
  62. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/taskdb.py +589 -0
  63. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/utils.py +710 -0
  64. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/worker.py +199 -0
  65. flexlock-0.8.2/.claude/worktrees/dedup-deps/flexlock/worker_cli.py +138 -0
  66. flexlock-0.8.2/.claude/worktrees/dedup-deps/pixi.lock +5793 -0
  67. flexlock-0.8.2/.claude/worktrees/dedup-deps/pyproject.toml +103 -0
  68. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_api.py +425 -0
  69. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_chained_and_paths.py +227 -0
  70. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_check.py +62 -0
  71. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_cli.py +488 -0
  72. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_completion_marker.py +154 -0
  73. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_data_hash.py +244 -0
  74. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_debug_enhanced.py +403 -0
  75. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_debug_integration.py +105 -0
  76. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_deferred.py +98 -0
  77. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_diff.py +559 -0
  78. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_diff_cli.py +66 -0
  79. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_dump_enqueue.py +274 -0
  80. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_fingerprint.py +112 -0
  81. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_flexcli.py +89 -0
  82. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_flexcli_jupyter.py +201 -0
  83. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_git_utils.py +181 -0
  84. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_index.py +202 -0
  85. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_load_stage.py +176 -0
  86. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_mlflowlink.py +246 -0
  87. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_multistage_config.py +347 -0
  88. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_node_selection.py +524 -0
  89. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_parallel.py +584 -0
  90. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_progressive_framework.py +136 -0
  91. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_query.py +371 -0
  92. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_report.py +104 -0
  93. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_resolvers.py +421 -0
  94. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_run_record.py +120 -0
  95. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_runner.py +293 -0
  96. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_save_dir.py +125 -0
  97. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_skills.py +72 -0
  98. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_slurm_validation.py +250 -0
  99. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_snapshot.py +437 -0
  100. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_submit_params.py +377 -0
  101. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_sweep_semantics.py +312 -0
  102. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_target_includes.py +135 -0
  103. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_utils.py +923 -0
  104. flexlock-0.8.2/.claude/worktrees/dedup-deps/tests/test_worker_preflight.py +97 -0
  105. flexlock-0.8.2/.gitattributes +2 -0
  106. flexlock-0.8.2/.gitignore +16 -0
  107. flexlock-0.8.2/FABLE_IMPLEMENTATION_PLAN.md +368 -0
  108. flexlock-0.8.2/PKG-INFO +357 -0
  109. flexlock-0.8.2/README.md +341 -0
  110. flexlock-0.8.2/bench/taskdb_contention.py +126 -0
  111. flexlock-0.8.2/docs/README.md +23 -0
  112. flexlock-0.8.2/docs/agentic_workflows.md +131 -0
  113. flexlock-0.8.2/docs/api.rst +127 -0
  114. flexlock-0.8.2/docs/cli_reference.md +811 -0
  115. flexlock-0.8.2/docs/conf.py +64 -0
  116. flexlock-0.8.2/docs/debugging.md +352 -0
  117. flexlock-0.8.2/docs/dev_plan_task_tags.md +181 -0
  118. flexlock-0.8.2/docs/hpc_integration.md +712 -0
  119. flexlock-0.8.2/docs/index.rst +40 -0
  120. flexlock-0.8.2/docs/mlflow_integration.md +960 -0
  121. flexlock-0.8.2/docs/philosophy.md +171 -0
  122. flexlock-0.8.2/docs/python_api.md +905 -0
  123. flexlock-0.8.2/docs/quickstart.md +175 -0
  124. flexlock-0.8.2/docs/reference.md +305 -0
  125. flexlock-0.8.2/docs/requirements.txt +10 -0
  126. flexlock-0.8.2/docs/resolution_simplification_plan.md +226 -0
  127. flexlock-0.8.2/docs/resolvers.md +466 -0
  128. flexlock-0.8.2/docs/upload_docs.sh +17 -0
  129. flexlock-0.8.2/docs/usage_guide.md +1083 -0
  130. flexlock-0.8.2/flexlock/__init__.py +87 -0
  131. flexlock-0.8.2/flexlock/api.py +1444 -0
  132. flexlock-0.8.2/flexlock/backends/__init__.py +11 -0
  133. flexlock-0.8.2/flexlock/backends/base.py +41 -0
  134. flexlock-0.8.2/flexlock/backends/pbs.py +219 -0
  135. flexlock-0.8.2/flexlock/backends/slurm.py +333 -0
  136. flexlock-0.8.2/flexlock/cli.py +953 -0
  137. flexlock-0.8.2/flexlock/config.py +121 -0
  138. flexlock-0.8.2/flexlock/context.py +7 -0
  139. flexlock-0.8.2/flexlock/data_hash.py +310 -0
  140. flexlock-0.8.2/flexlock/debug.py +473 -0
  141. flexlock-0.8.2/flexlock/diff.py +253 -0
  142. flexlock-0.8.2/flexlock/diff_cli.py +155 -0
  143. flexlock-0.8.2/flexlock/exceptions.py +50 -0
  144. flexlock-0.8.2/flexlock/export.py +134 -0
  145. flexlock-0.8.2/flexlock/fingerprint.py +129 -0
  146. flexlock-0.8.2/flexlock/flexcli.py +163 -0
  147. flexlock-0.8.2/flexlock/freeze.py +291 -0
  148. flexlock-0.8.2/flexlock/git_utils.py +196 -0
  149. flexlock-0.8.2/flexlock/index.py +300 -0
  150. flexlock-0.8.2/flexlock/load_stage.py +60 -0
  151. flexlock-0.8.2/flexlock/mlflow.py +192 -0
  152. flexlock-0.8.2/flexlock/parallel.py +378 -0
  153. flexlock-0.8.2/flexlock/query.py +739 -0
  154. flexlock-0.8.2/flexlock/report.py +78 -0
  155. flexlock-0.8.2/flexlock/report_template.html +246 -0
  156. flexlock-0.8.2/flexlock/resolvers.py +248 -0
  157. flexlock-0.8.2/flexlock/run_cli.py +44 -0
  158. flexlock-0.8.2/flexlock/run_record.py +173 -0
  159. flexlock-0.8.2/flexlock/runner.py +571 -0
  160. flexlock-0.8.2/flexlock/save_dir.py +232 -0
  161. flexlock-0.8.2/flexlock/skills/flexlock-new-stage/SKILL.md +57 -0
  162. flexlock-0.8.2/flexlock/skills/flexlock-report/SKILL.md +50 -0
  163. flexlock-0.8.2/flexlock/skills/flexlock-run-and-watch/SKILL.md +60 -0
  164. flexlock-0.8.2/flexlock/skills/flexlock-survey/SKILL.md +61 -0
  165. flexlock-0.8.2/flexlock/snapshot.py +245 -0
  166. flexlock-0.8.2/flexlock/status_cli.py +299 -0
  167. flexlock-0.8.2/flexlock/taskdb.py +685 -0
  168. flexlock-0.8.2/flexlock/utils.py +710 -0
  169. flexlock-0.8.2/flexlock/worker.py +228 -0
  170. flexlock-0.8.2/flexlock/worker_cli.py +143 -0
  171. flexlock-0.8.2/pixi.lock +5793 -0
  172. flexlock-0.8.2/plan_ergonomics_fable.md +200 -0
  173. flexlock-0.8.2/pyproject.toml +103 -0
  174. flexlock-0.8.2/suggested_improvements.md +126 -0
  175. flexlock-0.8.2/tests/test_api.py +462 -0
  176. flexlock-0.8.2/tests/test_chained_and_paths.py +227 -0
  177. flexlock-0.8.2/tests/test_check.py +62 -0
  178. flexlock-0.8.2/tests/test_cli.py +488 -0
  179. flexlock-0.8.2/tests/test_completion_marker.py +154 -0
  180. flexlock-0.8.2/tests/test_data_hash.py +244 -0
  181. flexlock-0.8.2/tests/test_debug_enhanced.py +403 -0
  182. flexlock-0.8.2/tests/test_debug_integration.py +105 -0
  183. flexlock-0.8.2/tests/test_deferred.py +98 -0
  184. flexlock-0.8.2/tests/test_diff.py +559 -0
  185. flexlock-0.8.2/tests/test_diff_cli.py +66 -0
  186. flexlock-0.8.2/tests/test_dump_enqueue.py +274 -0
  187. flexlock-0.8.2/tests/test_fingerprint.py +112 -0
  188. flexlock-0.8.2/tests/test_flexcli.py +89 -0
  189. flexlock-0.8.2/tests/test_flexcli_jupyter.py +201 -0
  190. flexlock-0.8.2/tests/test_git_utils.py +181 -0
  191. flexlock-0.8.2/tests/test_index.py +202 -0
  192. flexlock-0.8.2/tests/test_load_stage.py +176 -0
  193. flexlock-0.8.2/tests/test_mlflowlink.py +246 -0
  194. flexlock-0.8.2/tests/test_multistage_config.py +362 -0
  195. flexlock-0.8.2/tests/test_node_selection.py +524 -0
  196. flexlock-0.8.2/tests/test_parallel.py +584 -0
  197. flexlock-0.8.2/tests/test_progressive_framework.py +136 -0
  198. flexlock-0.8.2/tests/test_query.py +371 -0
  199. flexlock-0.8.2/tests/test_report.py +104 -0
  200. flexlock-0.8.2/tests/test_resolvers.py +421 -0
  201. flexlock-0.8.2/tests/test_run_record.py +120 -0
  202. flexlock-0.8.2/tests/test_runner.py +293 -0
  203. flexlock-0.8.2/tests/test_save_dir.py +210 -0
  204. flexlock-0.8.2/tests/test_save_dir_guard.py +145 -0
  205. flexlock-0.8.2/tests/test_skills.py +72 -0
  206. flexlock-0.8.2/tests/test_slurm_validation.py +250 -0
  207. flexlock-0.8.2/tests/test_snapshot.py +437 -0
  208. flexlock-0.8.2/tests/test_submit_params.py +377 -0
  209. flexlock-0.8.2/tests/test_sweep_semantics.py +312 -0
  210. flexlock-0.8.2/tests/test_target_includes.py +135 -0
  211. flexlock-0.8.2/tests/test_utils.py +973 -0
  212. flexlock-0.8.2/tests/test_worker_preflight.py +97 -0
@@ -0,0 +1,7 @@
1
+ {
2
+ "permissions": {
3
+ "allow": [
4
+ "Bash(python -m pytest:*)"
5
+ ]
6
+ }
7
+ }
@@ -0,0 +1,7 @@
1
+ {
2
+ "permissions": {
3
+ "allow": [
4
+ "Bash(python -m pytest:*)"
5
+ ]
6
+ }
7
+ }
@@ -0,0 +1,2 @@
1
+ # SCM syntax highlighting & preventing 3-way merges
2
+ pixi.lock merge=binary linguist-language=YAML linguist-generated=true
@@ -0,0 +1,16 @@
1
+ # pixi environments
2
+ .pixi/*
3
+ !.pixi/config.toml
4
+ **/__pycache__
5
+ .env
6
+ .pytest_cache
7
+ flexlock.egg-info
8
+ trash
9
+ *.conda
10
+ mlruns
11
+ .env
12
+ dist
13
+ doc/_build
14
+ docs/_build
15
+ docs/_build
16
+ outputs
@@ -0,0 +1,368 @@
1
+ # FlexLock — Implementation plan for `fable_notes.md`
2
+
3
+ Worktree: `/scale/user/qfebvre/projects/2509_flexlock/flexlock_fable`
4
+ Branch: `fable-improvements` (off `main` @ a6d25d3)
5
+ Test runner: `pixi run test` (pytest, suite under `tests/`)
6
+
7
+ All findings in `fable_notes.md` were re-verified against current `main`; every issue
8
+ is still present. Work is grouped into 5 phases ordered by **risk-adjusted value**:
9
+ surgical correctness fixes first (small, well-tested, high impact), then the two
10
+ structural refactors that eliminate whole bug classes, then docs, then the optional
11
+ deep redesigns.
12
+
13
+ Each item lists: **change**, **files**, **test**. Commit per item (or per tight
14
+ group) so review and bisection stay clean.
15
+
16
+ ---
17
+
18
+ ## Phase 0 — Guardrails (do first, ~30 min)
19
+
20
+ Before touching behaviour, lock in the current contract so regressions are visible.
21
+
22
+ 1. **Baseline the suite.** `pixi run test` on the fresh worktree; record pass/fail so
23
+ we distinguish pre-existing failures from ones we introduce.
24
+ 2. **Add characterization tests for the caching path** — a serial single-run cache-hit
25
+ (via `_find_matching_run`), and a `n_jobs=2` sweep re-run. The sweep re-run **already
26
+ works** via the task DB (`INSERT OR IGNORE` + `pending_count`); the test pins that
27
+ behaviour so the Phase 2.2 re-scoping doesn't regress it. (This corrects the earlier
28
+ "issue 1 = resume broken" premise — resume is fine; see 2.2.)
29
+
30
+ No production code changes in this phase.
31
+
32
+ ---
33
+
34
+ ## Phase 1 — Surgical correctness fixes (low risk, isolated)
35
+
36
+ These are localized, each independently testable, no cross-dependencies. Land them
37
+ first to bank value and shrink the diff the refactors sit on top of.
38
+
39
+ ### 1.1 `hash_data(use_cache=False)` ignored — **issue 4**
40
+ - **Change:** replace the hand-rolled `os.environ.get("FLEXLOCK_NO_CACHE", use_cache)
41
+ not in (...)` with: honour the `use_cache` argument, and let the env var only *force
42
+ off* — `use_cache = use_cache and not config.get_env_bool("FLEXLOCK_NO_CACHE", False)`.
43
+ - **Files:** `flexlock/data_hash.py` (import `from . import config`).
44
+ - **Test:** `tests/test_data_hash.py` — assert `use_cache=False` recomputes (patch
45
+ `_get_db` to fail if opened); assert `FLEXLOCK_NO_CACHE=1` disables even when
46
+ `use_cache=True`; assert `yes`/`on` accepted.
47
+
48
+ ### 1.2 `dirhash` ignores file paths — **issue 5**
49
+ - **Change:** hash `(relative_path, content_hash)` pairs, not bare content hashes.
50
+ Build `rel = f.relative_to(base_path).as_posix()`, feed `f"{rel}\0{content_hash}"`
51
+ into the final hasher over the sorted list.
52
+ - **Files:** `flexlock/data_hash.py` (`dirhash`).
53
+ - **Compat note:** this changes directory hash values → **invalidates existing data
54
+ caches and any stored `run.lock` data hashes**. Acceptable (hashes are opaque), but
55
+ call it out in CHANGELOG; bump an internal hash-version constant so stale cache rows
56
+ are ignored rather than mismatched. Clear `~/.cache/flexlock/hashes.db` is the manual
57
+ fallback.
58
+ - **Test:** `tests/test_data_hash.py` — two dirs with identical file contents under
59
+ different names must now hash differently; swapping contents between two files must
60
+ change the hash.
61
+
62
+ ### 1.3 `git_utils` returns error strings — **issue 13**
63
+ - **Change:** `get_git_tree_hash` / `get_git_commit` raise (wrap in
64
+ `FlexLockSnapshotError`) instead of returning `f"Error ..."`. Grep callers first —
65
+ if any rely on the string-return being truthy, adjust to try/except.
66
+ - **Files:** `flexlock/git_utils.py`, `flexlock/exceptions.py` (already has the class),
67
+ callers found via grep.
68
+ - **Test:** `tests/test_git_utils.py` — calling on a non-repo path raises, not returns
69
+ a string.
70
+
71
+ ### 1.4 `instantiate` mutates its input — **issue 11**
72
+ - **Change:** copy the config **before** `del cfg["_snapshot_"]`. Move the defensive
73
+ copy to the top of `instantiate` so the caller's config keeps `_snapshot_`.
74
+ - **Files:** `flexlock/utils.py` (`instantiate`, ~line 838).
75
+ - **Test:** `tests/test_utils.py` — instantiate a cfg with `_snapshot_`, assert the
76
+ original still has the key afterward.
77
+
78
+ ### 1.5 Env-var parsing drift — **issue 10**
79
+ - **Change:** route `FLEXLOCK_DEBUG` parsing in `flexcli.py` and `runner.py` through
80
+ `config.get_env_bool`; delete the ad-hoc `in ("1","true")` checks.
81
+ - **Files:** `flexlock/flexcli.py`, `flexlock/runner.py`, `flexlock/debug.py` (make it
82
+ read `config.DEBUG_STRATEGY` instead of `os.environ.get` directly).
83
+ - **Test:** `tests/test_flexcli.py` / `test_debug_*` — `FLEXLOCK_DEBUG=yes` enables.
84
+
85
+ ### 1.6 `latest:` resolver returns the pattern on no-match — **issue 20**
86
+ - **Change:** raise `FileNotFoundError` when no match; add optional `default` arg
87
+ mirroring `run_lock:`. Replace `run_lock:`'s `default=None` sentinel with a distinct
88
+ `_MISSING` sentinel so an explicit `null` default is honoured.
89
+ - **Files:** `flexlock/resolvers.py`.
90
+ - **Test:** `tests/test_resolvers.py` — no-match raises; with default returns default;
91
+ explicit `null` default distinguishable from absent.
92
+
93
+ ### 1.7 `vinc:` concurrency — **issue 19**
94
+ - **REVISED after implementation:** the mkdir-in-resolver claim is **incompatible**
95
+ with load-bearing behaviour. `${vinc:}` must resolve *idempotently* for multiple
96
+ references within one `submit()` (save_dir, logger_dir, dirpath all frozen to the
97
+ same `${vinc:}` node by `select_and_freeze_root_refs`); the codebase coordinates
98
+ versioning purely through filesystem scans, and the counter only advances once the
99
+ previous run's dir exists. Claiming a dir on *every* resolver call makes the 2nd/3rd
100
+ reference advance (breaks `test_vinc_stable_with_cross_tree_refs`).
101
+ - **Done instead:** keep pure-scan; document the residual cross-process race in the
102
+ docstring. The atomic claim is **deferred to the run-commit path** (Phase 3.1
103
+ `RunRecord`: create the dir with `exist_ok=False` + retry), which is the only place
104
+ a claim can be taken without breaking within-submit idempotency.
105
+ - **Files:** `flexlock/resolvers.py` (docstring).
106
+
107
+ ### 1.8 Dead / broken code — **issue 17**
108
+ - **Change:** delete `FlexLockRunner.check_if_exists` (broken, uncalled); remove the
109
+ unreachable `if self.defaults is None` in `Project.get`; simplify
110
+ `return None if return_snapshot else None` in `snapshot.py`. For `STRICT_VALIDATION`
111
+ / `MAX_DISPLAY_ITEMS`: **delete** from `config.py` and the env-var doc table (nothing
112
+ reads them; implementing them is out of scope).
113
+ - **Files:** `flexlock/runner.py`, `flexlock/api.py`, `flexlock/snapshot.py`,
114
+ `flexlock/config.py`, `docs/reference.md`.
115
+ - **Test:** existing suite must stay green; grep confirms no references.
116
+
117
+ ### 1.9 Logging stack split — **issue 18**
118
+ - **Change:** in `taskdb.py` drop the `logging.getLogger(__name__)` shadow; use the
119
+ loguru `logger` already imported, consistent with the rest of the package.
120
+ - **Files:** `flexlock/taskdb.py`.
121
+ - **Test:** smoke — a `dump_to_yaml` collision path emits via loguru (capture with
122
+ `caplog`/loguru sink).
123
+
124
+ **Phase 1 exit:** full suite green; each fix has a dedicated test.
125
+
126
+ ---
127
+
128
+ ## Phase 2 — Cache & result correctness (the core-promise fixes)
129
+
130
+ These are the issues the notes call out as most important (1–3), plus the diff-quality
131
+ issues that share the same code. Slightly higher risk; sequence matters.
132
+
133
+ ### 2.1 Fingerprint as a pure, stable digest — **issue 3 + redesign 3 (part 1)**
134
+ - **Change:** add `create_shadow_tree(repo_path, ignore_patterns)` in `git_utils.py`
135
+ that stages into a shadow index and runs **`write-tree` only** (no `commit-tree`, no
136
+ `update-ref`), returning `{tree, is_dirty}`. Keep `create_shadow_snapshot` (commit+ref)
137
+ for the real execution path. Add a `persist: bool` flag to `RunTracker.record_env`
138
+ (default `True`); fingerprinting calls it with `persist=False` -> tree-only, so
139
+ smart-run leaves no refs/commits behind.
140
+ - **Produce a `Fingerprint` value:** `fingerprint(cfg) -> str` = stable digest (sha) over
141
+ (a) canonicalized config with `save_dir` **prefix-normalized at fingerprint time**,
142
+ (b) per-repo tree hashes, (c) data hashes. Where include/exclude patterns are set,
143
+ hash the **filtered subtree** (git tree restricted to the pathspec) so today's
144
+ "relevant-files-unchanged still matches" becomes plain digest equality — no special
145
+ case in the matcher.
146
+ - **Files:** `flexlock/git_utils.py`, `flexlock/snapshot.py`, new
147
+ `flexlock/fingerprint.py`, `flexlock/api.py`.
148
+ - **Test:** `tests/test_git_utils.py` — `create_shadow_tree` adds **no** `refs/flexlock/
149
+ runs/*` and no commit object, same `tree` as `create_shadow_snapshot`.
150
+ `tests/test_fingerprint.py` (new) — same cfg -> same digest; changing an
151
+ include-relevant file changes it; changing an excluded file does not; save_dir change
152
+ alone does not.
153
+
154
+ ### 2.2 Project-wide fingerprint index — sweep items first-class — **issue 1 (DECIDED: first-class) + redesign 3 (part 2)**
155
+ - **Decision recorded:** sweep items **must be first-class** — a config first run as a
156
+ sweep task must cache-hit when re-run serially or from a different sweep, and be
157
+ discoverable as `prevs` lineage and in `flexlock ls/diff`. (Correction to the note:
158
+ same-*sweep* resume already works via the task DB's `INSERT OR IGNORE` + `pending_count`
159
+ early-return; that is **not** the gap. The gap is that a sweep task's snapshot lives
160
+ only in the DB, invisible to every `run.lock` reader.)
161
+ - **Change:** introduce a derived, project-wide index (SQLite,
162
+ `<results_root>/.flexlock/index.db` — location resolution below):
163
+ ```
164
+ runs(fingerprint PK, status, location_kind, run_lock_path,
165
+ task_db_path, task_id, save_dir, ts)
166
+ ```
167
+ - On **any** successful run — serial *or* sweep task — upsert a row keyed by
168
+ `fingerprint` with `status='done'` and a location pointer:
169
+ `location_kind='run_lock'` -> `run_lock_path` (+ `save_dir`), or
170
+ `location_kind='task'` -> `task_db_path` + `task_id` (+ `save_dir`).
171
+ Serial/isolated/single-HPC writes go through `api.py`; sweep tasks write from
172
+ `worker.py` right after `write_complete_marker`.
173
+ - `_find_matching_run` becomes: `fp = fingerprint(cfg)` -> `SELECT ... WHERE
174
+ fingerprint=? AND status='done'`. On hit, **verify** the pointed-to location still
175
+ exists and is complete (`run.complete` for run_lock; task row still `done` +
176
+ `results.json` present for task); if stale, **prune the row and treat as miss**
177
+ (self-healing). This subsumes the current `run.complete`-exists check and, because
178
+ only `status='done'` rows are returned, failed/interrupted runs are never served
179
+ from cache (ties into 2.3).
180
+ - `get_result` resolves either location kind to the right `results.json` (or the task
181
+ row's `result_info`) uniformly.
182
+ - **Index-location resolution:** the index must be shared across the runs it should
183
+ match. Resolve in order: `FLEXLOCK_INDEX` env -> nearest `.flexlock/index.db` walking
184
+ up from each `search_dir` -> a per-`search_root` `.flexlock/index.db`. Document that
185
+ `search_dirs` and the index scope must agree (a project-level wrapper should set both).
186
+ - **Legacy backfill & fallback:** the index is a **derived cache**; `run.lock` stays
187
+ authoritative. Add `flexlock reindex <dir>` to walk existing `**/run.lock` once and
188
+ populate the index. On an index **miss**, optionally fall back to the old glob scan
189
+ (behind `FLEXLOCK_INDEX_FALLBACK=1`, default on for one release) and backfill any hit
190
+ so the slow path self-eliminates. Removing the index file is always safe.
191
+ - **Concurrency:** many sweep workers upsert concurrently — reuse the taskdb SQLite
192
+ conventions (`busy_timeout`, `INSERT OR REPLACE` by PK). Upsert-to-`done` is
193
+ idempotent; a later success for the same fingerprint just refreshes the pointer.
194
+ - **RunDiff:** demoted to the **explainer** for `flexlock-diff` only (human-readable
195
+ "why different"), no longer the matcher.
196
+ - **Files:** new `flexlock/index.py`, `flexlock/api.py` (`_find_matching_run`,
197
+ `get_result`, write-on-success), `flexlock/worker.py` (write-on-success),
198
+ `flexlock/cli.py` (`reindex`), and `RunRecord` (Phase 3.1) as the natural writer.
199
+ - **Test:** `tests/test_index.py` (new) — a config run as a sweep task is a cache hit
200
+ when re-submitted serially; a failed task is **not** a hit; deleting the pointed-to
201
+ dir makes the stale row self-prune to a miss; `reindex` backfills legacy `run.lock`
202
+ runs; a 100-item sweep re-run does **not** re-parse every old `run.lock` (assert the
203
+ glob path isn't taken when the index is warm).
204
+
205
+ ### 2.3 Failures reported as SUCCESS — **issue 2** (+ **14**, **16** structurally)
206
+ - **Change:** introduce a single `collect_results(indices, configs, db_path, tag)` that
207
+ reads per-task terminal status from the task DB (`get_all_tasks`/`get_status_counts`
208
+ by task id) and builds `ExecutionResult` with real statuses
209
+ (`SUCCESS`/`CACHED`/`FAILED`/`INTERRUPTED`/`SUBMITTED`) and `error`. Replace the four
210
+ hand-built `status="SUCCESS"` sites (single-HPC, isolated, sweep-parallel,
211
+ and the missing-`results.json`→`None` case) with calls to it.
212
+ - **Also (issue 14):** add `timeout: int | None = None` param to `Project.submit`
213
+ (default `None`); thread it to both the single-HPC and sweep waits so behaviour is
214
+ consistent and user-controllable. Remove the asymmetric `DEFAULT_TIMEOUT`-vs-`None`
215
+ split.
216
+ - **Also (issue 16):** forward `isolated` and `debug` into `_submit_sweep`; where a
217
+ path genuinely can't honour a kwarg, raise `FlexLockConfigError` rather than silently
218
+ dropping it.
219
+ - **Files:** `flexlock/api.py` (new helper + call sites), `flexlock/taskdb.py`
220
+ (ensure a `get_all_tasks`/status-by-id accessor exists).
221
+ - **Test:** `tests/test_parallel.py` / `test_api.py` — a sweep where one task raises
222
+ returns that item with `status="FAILED"` and a non-empty `error`;
223
+ `max(results, key=...)` can skip failures. HPC path mocked.
224
+
225
+ ### 2.4 `force=True` reaches sweep items — **issue 15**
226
+ - **Change:** move the `run.complete` invalidation to *after* per-item merge — unlink
227
+ each item's `save_dir/run.complete` inside `_submit_sweep` when `force`. Thread a
228
+ `force` flag into `_submit_sweep` (currently not passed).
229
+ - **Files:** `flexlock/api.py`.
230
+ - **Test:** `tests/test_submit_params.py` — a forced sweep re-executes every item even
231
+ when markers exist.
232
+
233
+ ### 2.5 RunDiff correctness — **issues 7, 8, 9**
234
+ - **7 (ignore list eats user keys):** stop ignoring `date/time/system/cwd/job_id/
235
+ work_dir/datetime` at *every* nesting level. Apply the ignore set **only at the
236
+ snapshot top level** (the keys FlexLock itself injects), not inside the user
237
+ `config` subtree. Implement by passing `depth`/`path` to `_recursive_diff` and only
238
+ honouring the FlexLock-injected keys at `path == ""`. (`save_dir`/`_snapshot_` stay
239
+ normalized/ignored as today.)
240
+ - **8 (substring normalization):** in `_normalize_val`, replace only a path *prefix*:
241
+ `if val == root or val.startswith(root + os.sep): val = "<SAVE_DIR>" + val[len(root):]`.
242
+ - **9 (asymmetry / thin messages):** iterate the union of `c_repos`/`t_repos` keys in
243
+ `compare_git` (flag repos present in target but not current); make `compare_data`
244
+ report which keys/hashes differ instead of `"Data differs"`.
245
+ - **Files:** `flexlock/diff.py`.
246
+ - **Test:** `tests/test_diff.py` — user key `train.time: 100` vs `200` is a mismatch
247
+ (not ignored); `outputs/other/data.csv` is not spuriously normalized when `save_dir`
248
+ is `outputs`; a repo only in target is flagged; data diff names the key.
249
+
250
+ **Phase 2 exit:** the sweep-resume characterization test passes; failed sweep items are
251
+ visible; smart-run leaves no git refs behind; diff tests cover the false-hit cases.
252
+
253
+ ---
254
+
255
+ ## Phase 3 — Structural refactors that kill bug classes
256
+
257
+ Two focused refactors from the "deeper redesign" list. These are worth doing because
258
+ Phase 2 fixes (esp. 2.2, 2.3) are cleaner on top of them; do them if the Phase 2 interim
259
+ versions feel fragile.
260
+
261
+ ### 3.1 `RunRecord` — one owner of the on-disk contract — **redesign 2**
262
+ - **Change:** a class encapsulating a run directory: `write_lock(snapshot)`,
263
+ `mark_complete(result)`, `write_results(result)`, `load()`, `status` property,
264
+ `is_complete`, and — on `mark_complete` — **upsert the fingerprint index row** (2.2)
265
+ so the index write happens in exactly one place for serial runs *and* sweep tasks.
266
+ `RunTracker.save`, `worker.py`, and the `api.py` single-run path all route through it.
267
+ This makes driver and worker structurally identical (the reason issue 1's fix is
268
+ clean here) and gives `flexlock ls/gc/diff` and the index a single loader/writer.
269
+ - **Files:** new `flexlock/run_record.py`; refactor `snapshot.py`, `worker.py`,
270
+ `api.py`, `cli.py`.
271
+ - **Test:** new `tests/test_run_record.py` (round-trip lock/complete/results/status);
272
+ existing snapshot/cli tests must stay green.
273
+
274
+ ### 3.2 `ExecutionResult` as a typed result — **redesign 5** (+ fixes **issue 6**)
275
+ - **Change:** frozen dataclass with a `Status` enum, `error: str | None`,
276
+ `metrics: dict` (the return payload), `.get`/`__getitem__` delegating to `metrics`.
277
+ **Remove** the `setattr(self, key, value)` dynamic-attribute injection (issue 6 —
278
+ clobbering). Add `raise_on_failure()`. Keep `.result`/attribute-style access via an
279
+ explicit `__getattr__` over `metrics` for backward compat (documented, and guarded
280
+ against reserved names).
281
+ - **Compat risk:** `submit_chained` reads `getattr(parent, attr)` (e.g. `save_dir`) —
282
+ ensure those remain real attributes. `runner.run` unwraps `.result`.
283
+ - **Files:** `flexlock/api.py`.
284
+ - **Test:** `tests/test_api.py` — a function returning `{"status": "...", "get": 1}`
285
+ no longer clobbers `ExecutionResult.status`/`.get`; `raise_on_failure` raises on
286
+ `FAILED`.
287
+
288
+ > **Deferred (document as follow-up issues, do not implement now):**
289
+ > - **redesign 1** (split the `submit` god-method into `ConfigPipeline` +
290
+ > `ExecutionBackend` protocol + `collect_results`): large; Phase 2.3 already extracts
291
+ > `collect_results`, which is the highest-value slice. Full backend-protocol refactor
292
+ > is a separate PR.
293
+ > - **redesign 3** (project-wide fingerprint index): **promoted into Phase 2.1/2.2** —
294
+ > it is the mechanism that makes sweep items first-class, so it is no longer deferred.
295
+ > - **redesign 4** (replace hand-rolled interpolation parser with OmegaConf grammar):
296
+ > high-risk, needs property-based tests; only worth it if freeze bugs surface.
297
+ > - **redesign 6** (functional `submit_chained` without `self.defaults` mutation) and
298
+ > **redesign 7/8** (exception wrapping, `Settings` dataclass): schedule after the
299
+ > above land.
300
+
301
+ ---
302
+
303
+ ## Phase 4 — Documentation & guidelines
304
+
305
+ Land alongside the code so docs match reality.
306
+
307
+ 1. **Fix every doc↔code mismatch** from the notes table: CLI caching is opt-in
308
+ (`--check-exists`); `flexlock diff` is the separate `flexlock-diff` entry point;
309
+ default save_dir fallback is `outputs/<name>/<timestamp>`; `sweep_dir_suffix`
310
+ *nests* `sweep_{i:04d}`; `ExecutionResult` statuses; unify the
311
+ `FLEXLOCK_DIR_FILE_LIMIT` vs `FLEXLOCK_CACHE_DIR_FILE_LIMIT` knob to one name;
312
+ correct `FLEXLOCK_CACHE` default to `~/.cache/flexlock`; either implement or delete
313
+ the `FLEXLOCK_CONFIGURE_LOGGING` behaviour and the exception-wrapping claim in §12.
314
+ - **Files:** `README.md`, `docs/quickstart.md`, `docs/reference.md`,
315
+ `docs/usage_guide.md`, `Project.submit` docstring, `ExecutionResult` docstring.
316
+ 2. **Add the 10 usage guidelines** from the notes to `docs/usage_guide.md` (canonical
317
+ results root + explicit `search_dirs`; spell caching intent; check `run.complete`
318
+ before trusting results; keep targets importable / not in notebooks; `.gitignore`
319
+ big files + periodic `flexlock gc`; relative-interp-only configs; one sweep = one
320
+ dir + `tag=`; `dry_run` before HPC; small JSON-serializable returns avoiding reserved
321
+ keys; avoid reserved config key names).
322
+ 3. **Document the shadow-snapshot storage cost** (`git add --all` commits un-ignored
323
+ files into `.git`) prominently in `docs/philosophy.md` / snapshot docs.
324
+
325
+ Some guideline items become *unnecessary* once code is fixed: guideline 3 ("don't trust
326
+ result.status") is handled once the index serves only `status='done'` runs (2.2) and
327
+ statuses are real (2.3); guideline 10 (reserved config key names) after issue 7. Reframe
328
+ those as "now handled" rather than warnings.
329
+
330
+ ---
331
+
332
+ ## Sequencing summary
333
+
334
+ ```
335
+ Phase 0 baseline (DONE: 407 green)
336
+ Phase 1 1.1 1.2 1.3 1.4 1.5 1.6 1.7 1.8 1.9 (DONE, 415 green)
337
+ Phase 2/3 interleaved [DECIDED]:
338
+ 2.1 pure Fingerprint digest (DONE, 428 green)
339
+ 3.1 RunRecord (on-disk-contract owner) (DONE, 434 green)
340
+ 2.2 project-wide index (sweep first-class) (DONE, 445 green)
341
+ 2.5 RunDiff correctness (explainer only) (DONE, 451 green)
342
+ 2.4 force reaches sweep items (DONE)
343
+ 2.3 real statuses (collect_results) (DONE, 453 green)
344
+ 3.2 ExecutionResult typed (Status enum) (DONE, 455 green)
345
+ Phase 4 docs + guidelines (TODO)
346
+ ```
347
+
348
+ Each phase ends green on `pixi run test`. Recommend a PR per phase (Phase 1 as one PR
349
+ of small commits; Phase 2 as its own; Phase 3 optional separate PR).
350
+
351
+ ## Explicit decisions — RESOLVED
352
+ - **2.2 sweep items first-class** via the project-wide fingerprint index (redesign 3).
353
+ - **Index location scope [DECIDED]:** resolve in order (1) `FLEXLOCK_INDEX` env,
354
+ (2) nearest `.flexlock/index.db` walking up from each `search_dir`, (3) per
355
+ results-root `<results_root>/.flexlock/index.db`. A project wrapper should set both
356
+ `search_dirs` and the index so they agree.
357
+ - **Sequencing [DECIDED]:** interleave — **Phase 3.1 `RunRecord` lands before 2.2**
358
+ so the index has exactly one writer for serial runs *and* sweep tasks. Order:
359
+ `2.1 fingerprint -> 3.1 RunRecord -> 2.2 index -> 2.3 statuses -> 2.4 -> 2.5 -> 3.2`.
360
+ - **Glob fallback lifetime:** keep `FLEXLOCK_INDEX_FALLBACK` (default on) for one
361
+ release so legacy runs still hit + backfill, then default off.
362
+ - **Task-row git identity:** include the master snapshot's `repos` in a task's
363
+ fingerprint (else `parent_lock` short-circuits git and the fingerprint
364
+ under-specifies code identity).
365
+ - **1.2 hash format change [DONE]:** accepted invalidating existing data-hash caches;
366
+ implemented via `HASH_VERSION` bump + versioned cache file (`hashes_v2.db`).
367
+ - **1.7 vinc claim [REVISED]:** kept pure-scan; atomic claim deferred to 3.1 RunRecord
368
+ (see 1.7 above) because an in-resolver claim breaks within-submit idempotency.