strata-forge 0.2.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. {strata_forge-0.2.0 → strata_forge-0.3.0}/PKG-INFO +6 -4
  2. {strata_forge-0.2.0 → strata_forge-0.3.0}/README.md +1 -1
  3. {strata_forge-0.2.0 → strata_forge-0.3.0}/pyproject.toml +16 -3
  4. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/__init__.py +1 -1
  5. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/compute.py +7 -1
  6. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/helpers.py +3 -1
  7. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/train.py +12 -3
  8. strata_forge-0.3.0/src/strata_forge/compute/README.md +38 -0
  9. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/compute/__init__.py +7 -1
  10. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/compute/backends/__init__.py +4 -0
  11. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/compute/backends/base.py +97 -3
  12. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/compute/backends/local.py +98 -7
  13. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/compute/backends/skypilot.py +39 -4
  14. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/compute/backends/ssh.py +276 -42
  15. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/compute/serving.py +10 -2
  16. strata_forge-0.3.0/src/strata_forge/compute/task.py +314 -0
  17. strata_forge-0.3.0/src/strata_forge/core/README.md +25 -0
  18. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/core/__init__.py +13 -1
  19. strata_forge-0.3.0/src/strata_forge/core/redact.py +540 -0
  20. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/__init__.py +22 -1
  21. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/client.py +284 -30
  22. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/errors.py +27 -2
  23. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/messages.py +88 -3
  24. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/multimodal.py +5 -0
  25. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/providers/__init__.py +6 -1
  26. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/providers/azure.py +11 -0
  27. strata_forge-0.3.0/src/strata_forge/llm/providers/base.py +216 -0
  28. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/providers/config.py +6 -4
  29. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/providers/openai.py +37 -1
  30. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/providers/openai_compat.py +19 -2
  31. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/registry.py +37 -3
  32. strata_forge-0.3.0/src/strata_forge/llm/registry_data.yaml +524 -0
  33. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/responses.py +15 -5
  34. strata_forge-0.3.0/src/strata_forge/llm/responses_wire.py +719 -0
  35. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/routing.py +27 -2
  36. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/tools.py +16 -1
  37. strata_forge-0.3.0/src/strata_forge/pipelines/__init__.py +29 -0
  38. strata_forge-0.3.0/src/strata_forge/pipelines/_common.py +755 -0
  39. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/pipelines/finetune_runner.py +118 -22
  40. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/pipelines/inference_runner.py +155 -19
  41. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/storage/hf_hub.py +7 -4
  42. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/training/__init__.py +16 -3
  43. strata_forge-0.3.0/src/strata_forge/training/loading.py +39 -0
  44. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/training/peft.py +65 -3
  45. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/training/preference.py +36 -6
  46. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/training/sft.py +22 -6
  47. strata_forge-0.2.0/src/strata_forge/compute/README.md +0 -19
  48. strata_forge-0.2.0/src/strata_forge/compute/task.py +0 -167
  49. strata_forge-0.2.0/src/strata_forge/core/README.md +0 -5
  50. strata_forge-0.2.0/src/strata_forge/llm/providers/base.py +0 -111
  51. strata_forge-0.2.0/src/strata_forge/llm/registry_data.yaml +0 -273
  52. strata_forge-0.2.0/src/strata_forge/pipelines/__init__.py +0 -7
  53. strata_forge-0.2.0/src/strata_forge/pipelines/_common.py +0 -304
  54. {strata_forge-0.2.0 → strata_forge-0.3.0}/.gitignore +0 -0
  55. {strata_forge-0.2.0 → strata_forge-0.3.0}/LICENSE +0 -0
  56. {strata_forge-0.2.0 → strata_forge-0.3.0}/NOTICE +0 -0
  57. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/README.md +0 -0
  58. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/__init__.py +0 -0
  59. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/agent.py +0 -0
  60. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/memory/__init__.py +0 -0
  61. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/memory/conversation.py +0 -0
  62. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/memory/episodic.py +0 -0
  63. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/memory/vector_store.py +0 -0
  64. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/multi_agent.py +0 -0
  65. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/tools/__init__.py +0 -0
  66. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/tools/calculator.py +0 -0
  67. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/tools/fetch_url.py +0 -0
  68. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/tools/fs_read.py +0 -0
  69. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/tools/web_search.py +0 -0
  70. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/README.md +0 -0
  71. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/__init__.py +0 -0
  72. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/chat.py +0 -0
  73. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/datasets.py +0 -0
  74. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/doctor.py +0 -0
  75. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/eval_cmd.py +0 -0
  76. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/experiments.py +0 -0
  77. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/main.py +0 -0
  78. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/prompts.py +0 -0
  79. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/serve.py +0 -0
  80. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/compute/batch.py +0 -0
  81. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/compute/job.py +0 -0
  82. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/config/README.md +0 -0
  83. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/config/__init__.py +0 -0
  84. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/config/env.py +0 -0
  85. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/config/overlays.py +0 -0
  86. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/config/settings.py +0 -0
  87. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/core/budget.py +0 -0
  88. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/core/errors.py +0 -0
  89. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/core/ids.py +0 -0
  90. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/core/logging.py +0 -0
  91. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/core/repro.py +0 -0
  92. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/core/retry.py +0 -0
  93. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/core/types.py +0 -0
  94. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/README.md +0 -0
  95. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/__init__.py +0 -0
  96. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/hf_bridge.py +0 -0
  97. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/schema.py +0 -0
  98. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/store.py +0 -0
  99. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/stores/__init__.py +0 -0
  100. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/stores/langfuse.py +0 -0
  101. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/stores/memory.py +0 -0
  102. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/synthetic/__init__.py +0 -0
  103. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/synthetic/distillation.py +0 -0
  104. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/synthetic/self_instruct.py +0 -0
  105. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/versioning.py +0 -0
  106. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/README.md +0 -0
  107. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/__init__.py +0 -0
  108. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/ci_gate.py +0 -0
  109. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/experiment.py +0 -0
  110. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/graders/__init__.py +0 -0
  111. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/graders/base.py +0 -0
  112. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/graders/exact.py +0 -0
  113. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/graders/json_grader.py +0 -0
  114. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/graders/llm_judge.py +0 -0
  115. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/graders/pairwise.py +0 -0
  116. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/graders/semantic.py +0 -0
  117. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/metrics.py +0 -0
  118. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/reports/__init__.py +0 -0
  119. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/reports/html.py +0 -0
  120. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/reports/markdown.py +0 -0
  121. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/runner.py +0 -0
  122. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/sweeps.py +0 -0
  123. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/trace_replay.py +0 -0
  124. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/README.md +0 -0
  125. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/cache.py +0 -0
  126. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/cost.py +0 -0
  127. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/diagnostic.py +0 -0
  128. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/fallback.py +0 -0
  129. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/loop_events.py +0 -0
  130. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/providers/anthropic.py +0 -0
  131. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/providers/bedrock.py +0 -0
  132. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/providers/vertex.py +0 -0
  133. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/schemas.py +0 -0
  134. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/streaming.py +0 -0
  135. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/tokens.py +0 -0
  136. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/prompts/README.md +0 -0
  137. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/prompts/__init__.py +0 -0
  138. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/prompts/cache_aware.py +0 -0
  139. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/prompts/registry.py +0 -0
  140. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/prompts/rendering.py +0 -0
  141. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/prompts/stores/__init__.py +0 -0
  142. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/prompts/stores/langfuse.py +0 -0
  143. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/prompts/stores/memory.py +0 -0
  144. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/prompts/template.py +0 -0
  145. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/prompts/variables.py +0 -0
  146. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/py.typed +0 -0
  147. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/README.md +0 -0
  148. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/__init__.py +0 -0
  149. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/bm25.py +0 -0
  150. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/chunking.py +0 -0
  151. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/dense.py +0 -0
  152. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/embedding.py +0 -0
  153. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/hybrid.py +0 -0
  154. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/pipeline.py +0 -0
  155. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/qdrant.py +0 -0
  156. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/rerankers.py +0 -0
  157. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/retrieval.py +0 -0
  158. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/vector_store.py +0 -0
  159. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/storage/README.md +0 -0
  160. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/storage/__init__.py +0 -0
  161. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/storage/gateway.py +0 -0
  162. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/sync.py +0 -0
  163. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/tracing/README.md +0 -0
  164. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/tracing/__init__.py +0 -0
  165. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/tracing/client.py +0 -0
  166. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/tracing/decorator.py +0 -0
  167. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/tracing/litellm_callback.py +0 -0
  168. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/tracing/metrics.py +0 -0
  169. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/tracing/score.py +0 -0
  170. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/tracing/span.py +0 -0
  171. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/training/README.md +0 -0
  172. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/training/chat_template.py +0 -0
  173. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/training/dataset_format.py +0 -0
  174. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/training/hardware.py +0 -0
  175. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/training/methods.py +0 -0
  176. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/training/packing.py +0 -0
  177. {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/training/progress.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: strata-forge
3
- Version: 0.2.0
3
+ Version: 0.3.0
4
4
  Summary: Typed, async-first baseline for AI/LLM experiments across major foundation-model providers.
5
5
  Project-URL: Homepage, https://github.com/GemovicNemanja/strata-forge
6
6
  Project-URL: Repository, https://github.com/GemovicNemanja/strata-forge
@@ -21,8 +21,8 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
21
21
  Classifier: Typing :: Typed
22
22
  Requires-Python: >=3.14
23
23
  Requires-Dist: anyio<5,>=4.6
24
- Requires-Dist: jinja2<4,>=3.1
25
- Requires-Dist: litellm<2,>=1.55
24
+ Requires-Dist: jinja2<4,>=3.1.6
25
+ Requires-Dist: litellm<2,>=1.83
26
26
  Requires-Dist: pydantic-settings<3,>=2.6
27
27
  Requires-Dist: pydantic<3,>=2.10
28
28
  Requires-Dist: python-dotenv<2,>=1.0
@@ -35,6 +35,7 @@ Provides-Extra: all
35
35
  Requires-Dist: accelerate<2,>=1.4; extra == 'all'
36
36
  Requires-Dist: adlfs<2027,>=2024.7; extra == 'all'
37
37
  Requires-Dist: asyncssh<3,>=2.18; extra == 'all'
38
+ Requires-Dist: bitsandbytes<1,>=0.49; extra == 'all'
38
39
  Requires-Dist: boto3<2,>=1.35; extra == 'all'
39
40
  Requires-Dist: cohere<8,>=5.13; extra == 'all'
40
41
  Requires-Dist: datasets<6,>=4.0; extra == 'all'
@@ -66,6 +67,7 @@ Requires-Dist: nltk<4,>=3.9; extra == 'evals'
66
67
  Requires-Dist: rouge-score<0.2,>=0.1.2; extra == 'evals'
67
68
  Provides-Extra: finetuning
68
69
  Requires-Dist: accelerate<2,>=1.4; extra == 'finetuning'
70
+ Requires-Dist: bitsandbytes<1,>=0.49; extra == 'finetuning'
69
71
  Requires-Dist: datasets<6,>=4.0; extra == 'finetuning'
70
72
  Requires-Dist: peft<1,>=0.18; extra == 'finetuning'
71
73
  Requires-Dist: torch<3,>=2.9; extra == 'finetuning'
@@ -118,7 +120,7 @@ Optional extras pull in the heavy dependencies only when you need them:
118
120
  pip install "strata-forge[rag]" # embeddings, vector store, reranking
119
121
  pip install "strata-forge[compute]" # SkyPilot + SSH remote execution
120
122
  pip install "strata-forge[serving]" # vLLM
121
- pip install "strata-forge[finetuning]" # torch, transformers, trl, peft
123
+ pip install "strata-forge[finetuning]" # torch, transformers, trl, peft, bitsandbytes
122
124
  pip install "strata-forge[all]" # everything
123
125
  ```
124
126
 
@@ -21,7 +21,7 @@ Optional extras pull in the heavy dependencies only when you need them:
21
21
  pip install "strata-forge[rag]" # embeddings, vector store, reranking
22
22
  pip install "strata-forge[compute]" # SkyPilot + SSH remote execution
23
23
  pip install "strata-forge[serving]" # vLLM
24
- pip install "strata-forge[finetuning]" # torch, transformers, trl, peft
24
+ pip install "strata-forge[finetuning]" # torch, transformers, trl, peft, bitsandbytes
25
25
  pip install "strata-forge[all]" # everything
26
26
  ```
27
27
 
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "strata-forge"
3
- version = "0.2.0"
3
+ version = "0.3.0"
4
4
  description = "Typed, async-first baseline for AI/LLM experiments across major foundation-model providers."
5
5
  requires-python = ">=3.14"
6
6
  readme = "README.md"
@@ -34,10 +34,14 @@ classifiers = [
34
34
  # shipped that morning and a breaking upstream release lands as a mid-run crash on the user's
35
35
  # hardware rather than as a red build here. Raise a ceiling deliberately, with the call sites
36
36
  # checked against the new major -- never to make a resolver stop complaining.
37
+ # .github/workflows/clean-install.yml performs that fresh install every night (and on any branch
38
+ # push touching this file) and builds every config a run builds.
37
39
  dependencies = [
38
40
  "pydantic>=2.10,<3",
39
41
  "pydantic-settings>=2.6,<3",
40
- "litellm>=1.55,<2",
42
+ # The floor is the release the Responses API path (`litellm.aresponses`) is tested against;
43
+ # releases before 1.66 have no `aresponses` at all.
44
+ "litellm>=1.83,<2",
41
45
  "structlog>=24.4,<27",
42
46
  "tenacity>=9.0,<10",
43
47
  "typer>=0.15,<1",
@@ -45,7 +49,7 @@ dependencies = [
45
49
  "pyyaml>=6.0,<7",
46
50
  "python-dotenv>=1.0,<2",
47
51
  "rich>=13.9,<16",
48
- "jinja2>=3.1,<4",
52
+ "jinja2>=3.1.6,<4",
49
53
  ]
50
54
 
51
55
  [project.urls]
@@ -95,6 +99,15 @@ finetuning = [
95
99
  "peft>=0.18,<1",
96
100
  "datasets>=4.0,<6",
97
101
  "accelerate>=1.4,<2",
102
+ # QLoRA quantises the base model inside `from_pretrained`, which refuses to load without this.
103
+ # 0.49 is the first release with both a CUDA 13.0 kernel (the build the torch above resolves
104
+ # to on Linux; 0.48 added it) and an Apple Silicon wheel, so it installs everywhere torch>=2.9
105
+ # publishes a cp314 wheel: Linux x86_64/aarch64, Windows x64, macOS arm64. That is why there
106
+ # is no platform marker. Transformers 5 asks for >=0.46.1. The macOS wheel needs macOS 14 and
107
+ # there is no sdist, so an Apple Silicon Mac on 12 or 13 cannot install this extra (torch would
108
+ # settle on an older macOS 11 wheel; bitsandbytes has none). `require_bitsandbytes` in
109
+ # training/peft.py holds the same floor; a test keeps the two equal.
110
+ "bitsandbytes>=0.49,<1",
98
111
  ]
99
112
  all = [
100
113
  "strata-forge[redis,multimodal,inspect,compute,serving,storage,finetuning,langfuse,hf,evals,rag,bedrock]",
@@ -1,3 +1,3 @@
1
1
  """AI Forge — typed, async-first baseline for AI/LLM experiments."""
2
2
 
3
- __version__ = "0.2.0"
3
+ __version__ = "0.3.0"
@@ -209,9 +209,15 @@ async def _cancel(job_id: str) -> None:
209
209
 
210
210
 
211
211
  async def _cleanup(job_id: str) -> None:
212
+ from strata_forge.compute.backends.base import CleanupError
213
+
212
214
  job, backend_name, backend_kwargs = _load_job(job_id)
213
215
  backend = _make_backend(backend_name, backend_kwargs)
214
- await backend.cleanup(job)
216
+ try:
217
+ await backend.cleanup(job)
218
+ except CleanupError as exc:
219
+ # The saved job is the only handle on what survived, so it stays for the retry.
220
+ error_exit(f"cleanup failed, job kept for a retry: {exc}")
215
221
  _state_path(job_id).unlink(missing_ok=True)
216
222
  Console().print(f"[bold green]cleaned up[/] {job.id}")
217
223
 
@@ -47,7 +47,9 @@ def run_async[T](coro: Awaitable[T]) -> T:
47
47
 
48
48
  def error_exit(message: str, *, code: int = 1) -> NoReturn:
49
49
  """Print ``message`` to stderr and exit with ``code``."""
50
- _ERROR_CONSOLE.print(f"error: {message}")
50
+ # Not markup: messages name extras (``strata-forge[finetuning]``) and carry exception text,
51
+ # and rich would read either as a style tag and drop it from the line.
52
+ _ERROR_CONSOLE.print(f"error: {message}", markup=False)
51
53
  raise typer.Exit(code=code)
52
54
 
53
55
 
@@ -157,11 +157,20 @@ def dpo_cmd(
157
157
  def _build_peft(adapter: str, rank: int) -> Any:
158
158
  if adapter == "none":
159
159
  return None
160
- from strata_forge.training.peft import LoRAConfig, QLoRAConfig
160
+ from strata_forge.training.peft import (
161
+ LoRAConfig,
162
+ MissingBitsAndBytesError,
163
+ QLoRAConfig,
164
+ require_bitsandbytes,
165
+ )
161
166
 
162
167
  if adapter == "lora":
163
168
  return LoRAConfig(r=rank)
164
169
  if adapter == "qlora":
170
+ try:
171
+ require_bitsandbytes()
172
+ except MissingBitsAndBytesError as exc:
173
+ error_exit(str(exc))
165
174
  return QLoRAConfig(lora=LoRAConfig(r=rank))
166
175
  error_exit(f"unknown adapter {adapter!r}")
167
176
 
@@ -205,8 +214,8 @@ async def _run_sft(
205
214
  ) -> None:
206
215
  from strata_forge.training.sft import SFTConfig, SFTRunner
207
216
 
208
- dataset = await _resolve_dataset(dataset_name, dataset_version, dataset_file)
209
217
  peft_cfg = _build_peft(adapter, adapter_rank)
218
+ dataset = await _resolve_dataset(dataset_name, dataset_version, dataset_file)
210
219
 
211
220
  config = SFTConfig(
212
221
  model_id=model_id,
@@ -257,8 +266,8 @@ async def _run_dpo(
257
266
  ) -> None:
258
267
  from strata_forge.training.preference import DPOConfig, PreferenceRunner
259
268
 
260
- dataset = await _resolve_dataset(dataset_name, dataset_version, dataset_file)
261
269
  peft_cfg = _build_peft(adapter, adapter_rank)
270
+ dataset = await _resolve_dataset(dataset_name, dataset_version, dataset_file)
262
271
 
263
272
  config = DPOConfig(
264
273
  model_id=model_id,
@@ -0,0 +1,38 @@
1
+ # strata_forge.compute
2
+
3
+ Remote compute orchestration. The module ships typed `Task` /
4
+ `ResourceSpec` / `Job` / `JobStatus` shapes, a YAML task loader for
5
+ the SkyPilot-task-YAML subset, a `Backend` Protocol that every
6
+ implementation satisfies, and an in-process `LocalBackend` for
7
+ tests and ad-hoc local runs.
8
+
9
+ The SSH and SkyPilot backends sit behind the lazy `[compute]` extra,
10
+ alongside a batch inference runner that consumes any `Backend`, and
11
+ vLLM / TGI / SGLang serving adapters that present as
12
+ `strata_forge.llm` `openai_compat` providers. The training runners
13
+ (SFT, DPO/ORPO/KTO, PEFT/LoRA/QLoRA) live under
14
+ `strata_forge.training`.
15
+
16
+ Credentials a job needs travel beside the task, never in it:
17
+ `Task.secrets` holds them as `SecretStr` values that are never
18
+ serialised, rendered or logged, and every backend delivers them as a
19
+ private file the job finds through `FORGE_SECRETS_FILE`, never as an
20
+ environment variable or on a command line. A backend with no private
21
+ channel (SkyPilot) refuses a task that carries secrets.
22
+
23
+ Cleanup is verified: `cleanup` returns only once a job's backend-side
24
+ state (its remote workdir, and any secrets file left in it) is gone,
25
+ raises `CleanupError` when that state survived, and treats state that
26
+ is already gone as success, so an orchestrator can retry it until it
27
+ succeeds.
28
+
29
+ A model server holds no credential at all: the batch-inference runner
30
+ downloads the model's safetensors weights, configs and tokenizer files
31
+ itself, with the run's own token, and serves that local snapshot with
32
+ the Hub switched off. `build_vllm_task(served_model_name=...)` keeps
33
+ the served model addressable by its Hub id.
34
+
35
+ See [ADR 0013](../../../docs/architecture/adr/0013-compute-task-and-backend-shapes.md)
36
+ for the task-as-data + Protocol design rationale,
37
+ [ADR 0019](../../../docs/architecture/adr/0019-secrets-travel-beside-the-task.md)
38
+ for secret delivery, and `docs/roadmap.md` for the current status.
@@ -15,9 +15,11 @@ the batch runner are dep-free.
15
15
  from strata_forge.compute.backends import (
16
16
  MAX_CONSOLE_CHUNK_BYTES,
17
17
  Backend,
18
+ CleanupError,
18
19
  LocalBackend,
19
20
  SkyPilotBackend,
20
21
  SSHBackend,
22
+ SubmitCleanupError,
21
23
  safe_workdir_relpath,
22
24
  )
23
25
  from strata_forge.compute.batch import BatchInferenceResult, BatchInferenceRunner
@@ -30,13 +32,16 @@ from strata_forge.compute.serving import (
30
32
  serving_endpoint,
31
33
  wait_for_endpoint,
32
34
  )
33
- from strata_forge.compute.task import ResourceSpec, Task
35
+ from strata_forge.compute.task import SECRETS_FILE_ENV, SECRETS_FILE_NAME, ResourceSpec, Task
34
36
 
35
37
  __all__ = [
36
38
  "MAX_CONSOLE_CHUNK_BYTES",
39
+ "SECRETS_FILE_ENV",
40
+ "SECRETS_FILE_NAME",
37
41
  "Backend",
38
42
  "BatchInferenceResult",
39
43
  "BatchInferenceRunner",
44
+ "CleanupError",
40
45
  "ConsoleChunk",
41
46
  "Job",
42
47
  "JobState",
@@ -46,6 +51,7 @@ __all__ = [
46
51
  "SSHBackend",
47
52
  "ServingEndpoint",
48
53
  "SkyPilotBackend",
54
+ "SubmitCleanupError",
49
55
  "Task",
50
56
  "build_sglang_task",
51
57
  "build_tgi_task",
@@ -9,6 +9,8 @@ and :class:`SkyPilotBackend` lazy-import their SDKs behind the
9
9
  from strata_forge.compute.backends.base import (
10
10
  MAX_CONSOLE_CHUNK_BYTES,
11
11
  Backend,
12
+ CleanupError,
13
+ SubmitCleanupError,
12
14
  safe_workdir_relpath,
13
15
  )
14
16
  from strata_forge.compute.backends.local import LocalBackend
@@ -18,8 +20,10 @@ from strata_forge.compute.backends.ssh import SSHBackend
18
20
  __all__ = [
19
21
  "MAX_CONSOLE_CHUNK_BYTES",
20
22
  "Backend",
23
+ "CleanupError",
21
24
  "LocalBackend",
22
25
  "SSHBackend",
23
26
  "SkyPilotBackend",
27
+ "SubmitCleanupError",
24
28
  "safe_workdir_relpath",
25
29
  ]
@@ -13,7 +13,12 @@ from __future__ import annotations
13
13
  import posixpath
14
14
  from typing import TYPE_CHECKING, Protocol, runtime_checkable
15
15
 
16
+ from strata_forge.compute.task import SECRETS_FILE_ENV
17
+ from strata_forge.core.errors import ForgeError
18
+
16
19
  if TYPE_CHECKING:
20
+ from collections.abc import Sequence
21
+
17
22
  from strata_forge.compute.job import ConsoleChunk, Job, JobStatus
18
23
  from strata_forge.compute.task import Task
19
24
 
@@ -21,9 +26,16 @@ __all__ = [
21
26
  "MAX_CONSOLE_CHUNK_BYTES",
22
27
  "MAX_READ_FILE_BYTES",
23
28
  "Backend",
29
+ "CleanupError",
30
+ "SubmitCleanupError",
24
31
  "safe_workdir_relpath",
32
+ "secrets_guarded_script",
25
33
  ]
26
34
 
35
+ # The unexported shell variable that holds the secrets file's path for the trap. Unexported so
36
+ # no child of the job (the setup step's package installs above all) inherits a pointer to it.
37
+ _SECRETS_PATH_VAR = "_forge_secrets_file"
38
+
27
39
  # Hard cap on a single :meth:`Backend.read_file` result. A control plane polls
28
40
  # read_file on an interval from a SHARED process; without a cap a runaway/malicious
29
41
  # file (e.g. a huge progress.jsonl) could exhaust that process's memory and degrade
@@ -37,6 +49,71 @@ MAX_READ_FILE_BYTES = 8 * 1024 * 1024
37
49
  MAX_CONSOLE_CHUNK_BYTES = 64 * 1024
38
50
 
39
51
 
52
+ class SubmitCleanupError(RuntimeError):
53
+ """A submit failed after creating remote state that it then could not remove.
54
+
55
+ Raised from the submit's own failure (its ``__cause__``) when that failure is an ordinary
56
+ exception; a cancellation is re-raised as itself. That state may hold the job's secrets file,
57
+ and a submit that raises hands back no job, so :attr:`job` is the handle to remove it with:
58
+ pass it to :meth:`Backend.cleanup` once the host is reachable again. It names no running
59
+ process, so ``status``, ``cancel`` and the log readers do not apply to it.
60
+ """
61
+
62
+ def __init__(self, message: str, job: Job) -> None:
63
+ super().__init__(message)
64
+ self.job = job
65
+
66
+
67
+ class CleanupError(ForgeError, OSError):
68
+ """:meth:`Backend.cleanup` ran, and the job's backend-side state is still there.
69
+
70
+ The state may hold the job's secrets file, so a caller that sees this has NOT cleaned up and
71
+ should try again later: the method is idempotent, and a retry against state that has since
72
+ gone is a success. The message names the job and how the removal failed (an exit status,
73
+ an exception type), never what the remote printed or what the state contains, so it is safe
74
+ to store and show. A transport failure (the host is unreachable, the command timed out) is
75
+ raised as itself, not as this: it says nothing about whether the state survived.
76
+
77
+ It is also an :class:`OSError`, as the failure of :func:`shutil.rmtree` it stands for would
78
+ be, so an orchestrator that already treats an ``OSError`` from ``cleanup`` as "not cleaned"
79
+ keeps doing so rather than letting this one escape.
80
+ """
81
+
82
+
83
+ def secrets_guarded_script(
84
+ path_expr: str, setup: str, run: str, *, prelude: Sequence[str] = ()
85
+ ) -> str:
86
+ """The shell text that runs ``setup`` then ``run`` for a job whose secrets are in a file.
87
+
88
+ ``path_expr`` is a shell word that evaluates to the file's absolute path. The script:
89
+
90
+ - removes the file on exit through a trap in the OUTER shell, and runs ``prelude``, ``setup``
91
+ and ``run`` in a subshell below it. A task's own commands may install an ``EXIT`` trap of
92
+ their own (an orchestrator's bootstrap does, to stop a progress ticker), and a second
93
+ ``trap ... EXIT`` in one shell REPLACES the first: in the same shell, that would silently
94
+ drop the removal, and the file would outlive a failed setup or a cancellation. A trap set
95
+ in the subshell cannot reach the outer one.
96
+ - exports :data:`~strata_forge.compute.task.SECRETS_FILE_ENV` only between ``setup`` and
97
+ ``run``, so the setup step's package installs never hold even the path to the file.
98
+
99
+ The outer shell exits with the subshell's status, which the trap leaves untouched. bash runs
100
+ an ``EXIT`` trap on a normal exit and on a fatal signal such as the SIGTERM a cancel sends;
101
+ only SIGKILL skips it.
102
+ """
103
+ run_step = f'export {SECRETS_FILE_ENV}="${_SECRETS_PATH_VAR}" && {run}'
104
+ body = f"{setup} && {run_step}" if setup else run_step
105
+ return "\n".join(
106
+ [
107
+ f"{_SECRETS_PATH_VAR}={path_expr}",
108
+ f"trap 'rm -f \"${_SECRETS_PATH_VAR}\"' EXIT",
109
+ "(",
110
+ *prelude,
111
+ body,
112
+ ")",
113
+ ]
114
+ )
115
+
116
+
40
117
  def safe_workdir_relpath(path: str) -> str:
41
118
  """Validate ``path`` as a job-workdir-relative path and return it normalized.
42
119
 
@@ -72,7 +149,15 @@ class Backend(Protocol):
72
149
  ... # pragma: no cover — Protocol body
73
150
 
74
151
  async def submit(self, task: Task) -> Job:
75
- """Submit ``task`` for execution; return its :class:`Job` handle."""
152
+ """Submit ``task`` for execution; return its :class:`Job` handle.
153
+
154
+ ``task.secrets`` are written to a private (0600) file whose absolute path the job's
155
+ ``run`` step finds in ``FORGE_SECRETS_FILE``, never into the job's environment, a command
156
+ line, a script or the returned job's metadata. A backend with no such channel raises
157
+ rather than falling back to the environment. A submit that fails after creating remote
158
+ state it then cannot remove raises :class:`SubmitCleanupError`, whose ``job`` names that
159
+ state for :meth:`cleanup`.
160
+ """
76
161
  ... # pragma: no cover — Protocol body
77
162
 
78
163
  async def status(self, job: Job) -> JobStatus:
@@ -140,11 +225,20 @@ class Backend(Protocol):
140
225
  ... # pragma: no cover — Protocol body
141
226
 
142
227
  async def cleanup(self, job: Job) -> None:
143
- """Tear down any backend-side state associated with ``job``.
228
+ """Tear down any backend-side state associated with ``job``, and prove it is gone.
144
229
 
145
230
  Examples: remote temporary directories (SSH), SkyPilot
146
231
  clusters launched specifically for this job, captured log
147
232
  buffers. Idempotent — callers can invoke it repeatedly or
148
- on already-cleaned jobs.
233
+ on already-cleaned jobs; state that is already absent is a
234
+ success.
235
+
236
+ Returning means the state is gone, not that a removal was
237
+ attempted: that state may hold the job's secrets file, and
238
+ a caller that retries until cleanup succeeds can only do so
239
+ if a failed one says so. When the backend could run the
240
+ removal but the state survived it, this raises
241
+ :class:`CleanupError`; a transport failure propagates as
242
+ itself.
149
243
  """
150
244
  ... # pragma: no cover — Protocol body
@@ -23,6 +23,14 @@ Those files outlive ``cleanup`` and the backend object, which is the
23
23
  point: when the process that owned the buffers is gone, the file is
24
24
  the only remaining evidence. It is opt-in so that a library user who
25
25
  never asks for it never finds log files appearing beside their code.
26
+
27
+ ``Task.secrets`` are written to a 0600 file in a private (0700)
28
+ temporary directory of the job's own, and the task's ``run`` step
29
+ finds its path in ``FORGE_SECRETS_FILE`` (``setup`` does not); the
30
+ values are never in the child's environment. The child's outer shell
31
+ removes the file on exit, below a subshell the task's own traps
32
+ cannot displace, and the backend removes the directory once the
33
+ child is gone.
26
34
  """
27
35
 
28
36
  from __future__ import annotations
@@ -30,7 +38,10 @@ from __future__ import annotations
30
38
  import asyncio
31
39
  import contextlib
32
40
  import os
41
+ import shlex
42
+ import shutil
33
43
  import signal
44
+ import tempfile
34
45
  import uuid
35
46
  from datetime import UTC, datetime
36
47
  from pathlib import Path
@@ -39,9 +50,12 @@ from typing import TYPE_CHECKING
39
50
  from strata_forge.compute.backends.base import (
40
51
  MAX_CONSOLE_CHUNK_BYTES,
41
52
  MAX_READ_FILE_BYTES,
53
+ CleanupError,
42
54
  safe_workdir_relpath,
55
+ secrets_guarded_script,
43
56
  )
44
57
  from strata_forge.compute.job import ConsoleChunk, Job, JobStatus
58
+ from strata_forge.compute.task import SECRETS_FILE_ENV, SECRETS_FILE_NAME, render_secrets_payload
45
59
 
46
60
  if TYPE_CHECKING:
47
61
  from strata_forge.compute.task import Task
@@ -186,6 +200,8 @@ class _JobState:
186
200
  self.workdir: str | None = None
187
201
  # Where this job's streams were teed, when a log_dir was configured.
188
202
  self.log_paths: tuple[Path, Path] | None = None
203
+ # The private directory holding the job's secrets file, while there is one.
204
+ self.secrets_dir: Path | None = None
189
205
 
190
206
 
191
207
  class LocalBackend:
@@ -195,8 +211,12 @@ class LocalBackend:
195
211
  name: Optional override for the backend's reported name.
196
212
  Default ``"local"``.
197
213
  env_inherit: When ``True`` (default), child processes
198
- inherit the parent's ``os.environ``. When ``False``,
199
- the child sees only ``Task.env``.
214
+ inherit the parent's ``os.environ`` (minus any
215
+ ``FORGE_SECRETS_FILE``, which names the parent's file,
216
+ not the child's). When ``False``, the child sees only
217
+ ``Task.env``. A task with secrets also exports
218
+ ``FORGE_SECRETS_FILE`` to its ``run`` step, never to
219
+ ``setup``.
200
220
  log_dir: When set, every job also tees its stdout/stderr
201
221
  to ``serve.stdout.log`` / ``serve.stderr.log`` under
202
222
  this directory, written as the bytes arrive and left
@@ -229,15 +249,26 @@ class LocalBackend:
229
249
  if task.num_nodes != 1:
230
250
  err = f"LocalBackend can only run single-node tasks; got num_nodes={task.num_nodes}"
231
251
  raise ValueError(err)
252
+ # A task built with `model_copy(update=...)` skipped every validator; nothing is
253
+ # delivered until its secrets have been through them.
254
+ task = task.revalidated()
232
255
 
233
256
  job_id = uuid.uuid4().hex
234
257
  state = _JobState()
235
258
  state.workdir = task.workdir
236
259
  state.log_paths = self._prepare_log_paths(state)
237
- self._jobs[job_id] = state
238
260
 
239
261
  env = self._build_env(task)
240
262
  script = self._build_script(task)
263
+ if task.secrets:
264
+ state.secrets_dir = _write_secrets_file(task)
265
+ # The child's outer shell removes the file however it exits, so it does not outlive
266
+ # a setup that failed before anything read it, and only `run` is told where it is.
267
+ # The directory goes with the job.
268
+ script = secrets_guarded_script(
269
+ shlex.quote(str(state.secrets_dir / SECRETS_FILE_NAME)), task.setup, task.run
270
+ )
271
+ self._jobs[job_id] = state
241
272
 
242
273
  async def _runner() -> None:
243
274
  try:
@@ -288,6 +319,7 @@ class LocalBackend:
288
319
  state.exit_code = -1
289
320
  state.stderr_buffer.extend(f"local backend exception: {exc!r}".encode())
290
321
  finally:
322
+ _remove_secrets_dir(state)
291
323
  state.finished_at = datetime.now(UTC)
292
324
 
293
325
  # Fire-and-forget; status / logs / cancel observe state directly.
@@ -321,6 +353,9 @@ class LocalBackend:
321
353
  env: dict[str, str] = {}
322
354
  if self._env_inherit:
323
355
  env.update(os.environ)
356
+ # A path to the PARENT's secrets file is not the child's to read, and a child that
357
+ # read it would also delete it. Only this task's own file is ever named.
358
+ env.pop(SECRETS_FILE_ENV, None)
324
359
  env.update(task.env)
325
360
  return env
326
361
 
@@ -472,14 +507,17 @@ class LocalBackend:
472
507
  state.cancelled = True
473
508
 
474
509
  async def cleanup(self, job: Job) -> None:
475
- """Drop the job's in-memory state and make sure its process is gone.
510
+ """Make sure the job's process is gone, remove its secrets directory, drop its state.
476
511
 
477
- Any ``log_dir`` files are left on disk on purpose: they exist to be read AFTER the
478
- buffers they mirror have been discarded.
512
+ Raises :class:`CleanupError` when the secrets directory survives the removal, and keeps
513
+ the job's state so a later call can retry it: dropping the state first would leave the
514
+ directory with nothing that names it. Any ``log_dir`` files are left on disk on purpose:
515
+ they exist to be read AFTER the buffers they mirror have been discarded.
479
516
  """
480
- state = self._jobs.pop(job.id, None)
517
+ state = self._jobs.get(job.id)
481
518
  if state is None:
482
519
  return # already cleaned up — idempotent
520
+ removed = _remove_secrets_dir(state)
483
521
  # Make sure any lingering process is gone before we drop the state. Once this returns,
484
522
  # the state is dropped and nothing can name the group again, so it sweeps unconditionally
485
523
  # for the same reason cancel does — a job whose own process exited may still have left
@@ -487,6 +525,59 @@ class LocalBackend:
487
525
  process = state.process
488
526
  if process is not None:
489
527
  await _reap_group(process)
528
+ # Once more after the reap: the tree that was just stopped may have been writing there.
529
+ if not removed and not _remove_secrets_dir(state):
530
+ err = f"LocalBackend: job {job.id!r}'s secrets directory survived cleanup"
531
+ raise CleanupError(err)
532
+ self._jobs.pop(job.id, None)
533
+
534
+
535
+ def _write_secrets_file(task: Task) -> Path:
536
+ """Write ``task.secrets`` to a 0600 file in a fresh 0700 directory; return the directory.
537
+
538
+ Modes are set explicitly rather than requested through ``mkdir``/``open``: those only ever
539
+ narrow by the umask, so a restrictive umask would leave the file unreadable by the child and
540
+ the explicit ``chmod`` is what makes the mode exact. ``O_EXCL | O_NOFOLLOW`` refuses a file
541
+ or symlink already at the path rather than writing through it.
542
+ """
543
+ directory = Path(tempfile.mkdtemp(prefix="forge-job-"))
544
+ try:
545
+ directory.chmod(0o700)
546
+ fd = os.open(
547
+ directory / SECRETS_FILE_NAME,
548
+ os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
549
+ 0o600,
550
+ )
551
+ with os.fdopen(fd, "w", encoding="utf-8") as fh:
552
+ os.fchmod(fh.fileno(), 0o600)
553
+ fh.write(render_secrets_payload(task.secrets))
554
+ except BaseException:
555
+ shutil.rmtree(directory, ignore_errors=True)
556
+ raise
557
+ return directory
558
+
559
+
560
+ def _remove_secrets_dir(state: _JobState) -> bool:
561
+ """Remove the job's secrets directory, if it still has one; return whether it is gone.
562
+
563
+ Idempotent. The job forgets the directory only once nothing is left at its path, so a removal
564
+ that failed is retried by the next call rather than recorded as done.
565
+ """
566
+ directory = state.secrets_dir
567
+ if directory is None:
568
+ return True
569
+ shutil.rmtree(directory, ignore_errors=True)
570
+ # `lstat`, not `exists`: `exists` returns False on ANY error, so a parent that cannot be
571
+ # searched (EACCES) would read as "gone" while the file is still there. Only "no such
572
+ # entry" proves it.
573
+ try:
574
+ directory.lstat()
575
+ except FileNotFoundError, NotADirectoryError:
576
+ state.secrets_dir = None
577
+ return True
578
+ except OSError:
579
+ return False
580
+ return False
490
581
 
491
582
 
492
583
  def _slice_stream(