strata-forge 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {strata_forge-0.2.0 → strata_forge-0.3.0}/PKG-INFO +6 -4
- {strata_forge-0.2.0 → strata_forge-0.3.0}/README.md +1 -1
- {strata_forge-0.2.0 → strata_forge-0.3.0}/pyproject.toml +16 -3
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/__init__.py +1 -1
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/compute.py +7 -1
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/helpers.py +3 -1
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/train.py +12 -3
- strata_forge-0.3.0/src/strata_forge/compute/README.md +38 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/compute/__init__.py +7 -1
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/compute/backends/__init__.py +4 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/compute/backends/base.py +97 -3
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/compute/backends/local.py +98 -7
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/compute/backends/skypilot.py +39 -4
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/compute/backends/ssh.py +276 -42
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/compute/serving.py +10 -2
- strata_forge-0.3.0/src/strata_forge/compute/task.py +314 -0
- strata_forge-0.3.0/src/strata_forge/core/README.md +25 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/core/__init__.py +13 -1
- strata_forge-0.3.0/src/strata_forge/core/redact.py +540 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/__init__.py +22 -1
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/client.py +284 -30
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/errors.py +27 -2
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/messages.py +88 -3
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/multimodal.py +5 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/providers/__init__.py +6 -1
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/providers/azure.py +11 -0
- strata_forge-0.3.0/src/strata_forge/llm/providers/base.py +216 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/providers/config.py +6 -4
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/providers/openai.py +37 -1
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/providers/openai_compat.py +19 -2
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/registry.py +37 -3
- strata_forge-0.3.0/src/strata_forge/llm/registry_data.yaml +524 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/responses.py +15 -5
- strata_forge-0.3.0/src/strata_forge/llm/responses_wire.py +719 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/routing.py +27 -2
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/tools.py +16 -1
- strata_forge-0.3.0/src/strata_forge/pipelines/__init__.py +29 -0
- strata_forge-0.3.0/src/strata_forge/pipelines/_common.py +755 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/pipelines/finetune_runner.py +118 -22
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/pipelines/inference_runner.py +155 -19
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/storage/hf_hub.py +7 -4
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/training/__init__.py +16 -3
- strata_forge-0.3.0/src/strata_forge/training/loading.py +39 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/training/peft.py +65 -3
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/training/preference.py +36 -6
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/training/sft.py +22 -6
- strata_forge-0.2.0/src/strata_forge/compute/README.md +0 -19
- strata_forge-0.2.0/src/strata_forge/compute/task.py +0 -167
- strata_forge-0.2.0/src/strata_forge/core/README.md +0 -5
- strata_forge-0.2.0/src/strata_forge/llm/providers/base.py +0 -111
- strata_forge-0.2.0/src/strata_forge/llm/registry_data.yaml +0 -273
- strata_forge-0.2.0/src/strata_forge/pipelines/__init__.py +0 -7
- strata_forge-0.2.0/src/strata_forge/pipelines/_common.py +0 -304
- {strata_forge-0.2.0 → strata_forge-0.3.0}/.gitignore +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/LICENSE +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/NOTICE +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/README.md +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/__init__.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/agent.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/memory/__init__.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/memory/conversation.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/memory/episodic.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/memory/vector_store.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/multi_agent.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/tools/__init__.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/tools/calculator.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/tools/fetch_url.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/tools/fs_read.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/agents/tools/web_search.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/README.md +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/__init__.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/chat.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/datasets.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/doctor.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/eval_cmd.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/experiments.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/main.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/prompts.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/cli/serve.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/compute/batch.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/compute/job.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/config/README.md +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/config/__init__.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/config/env.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/config/overlays.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/config/settings.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/core/budget.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/core/errors.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/core/ids.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/core/logging.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/core/repro.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/core/retry.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/core/types.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/README.md +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/__init__.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/hf_bridge.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/schema.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/store.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/stores/__init__.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/stores/langfuse.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/stores/memory.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/synthetic/__init__.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/synthetic/distillation.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/synthetic/self_instruct.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/datasets/versioning.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/README.md +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/__init__.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/ci_gate.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/experiment.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/graders/__init__.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/graders/base.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/graders/exact.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/graders/json_grader.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/graders/llm_judge.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/graders/pairwise.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/graders/semantic.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/metrics.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/reports/__init__.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/reports/html.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/reports/markdown.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/runner.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/sweeps.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/evals/trace_replay.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/README.md +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/cache.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/cost.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/diagnostic.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/fallback.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/loop_events.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/providers/anthropic.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/providers/bedrock.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/providers/vertex.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/schemas.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/streaming.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/llm/tokens.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/prompts/README.md +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/prompts/__init__.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/prompts/cache_aware.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/prompts/registry.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/prompts/rendering.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/prompts/stores/__init__.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/prompts/stores/langfuse.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/prompts/stores/memory.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/prompts/template.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/prompts/variables.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/py.typed +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/README.md +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/__init__.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/bm25.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/chunking.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/dense.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/embedding.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/hybrid.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/pipeline.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/qdrant.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/rerankers.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/retrieval.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/rag/vector_store.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/storage/README.md +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/storage/__init__.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/storage/gateway.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/sync.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/tracing/README.md +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/tracing/__init__.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/tracing/client.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/tracing/decorator.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/tracing/litellm_callback.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/tracing/metrics.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/tracing/score.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/tracing/span.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/training/README.md +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/training/chat_template.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/training/dataset_format.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/training/hardware.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/training/methods.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/training/packing.py +0 -0
- {strata_forge-0.2.0 → strata_forge-0.3.0}/src/strata_forge/training/progress.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: strata-forge
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Typed, async-first baseline for AI/LLM experiments across major foundation-model providers.
|
|
5
5
|
Project-URL: Homepage, https://github.com/GemovicNemanja/strata-forge
|
|
6
6
|
Project-URL: Repository, https://github.com/GemovicNemanja/strata-forge
|
|
@@ -21,8 +21,8 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
|
21
21
|
Classifier: Typing :: Typed
|
|
22
22
|
Requires-Python: >=3.14
|
|
23
23
|
Requires-Dist: anyio<5,>=4.6
|
|
24
|
-
Requires-Dist: jinja2<4,>=3.1
|
|
25
|
-
Requires-Dist: litellm<2,>=1.
|
|
24
|
+
Requires-Dist: jinja2<4,>=3.1.6
|
|
25
|
+
Requires-Dist: litellm<2,>=1.83
|
|
26
26
|
Requires-Dist: pydantic-settings<3,>=2.6
|
|
27
27
|
Requires-Dist: pydantic<3,>=2.10
|
|
28
28
|
Requires-Dist: python-dotenv<2,>=1.0
|
|
@@ -35,6 +35,7 @@ Provides-Extra: all
|
|
|
35
35
|
Requires-Dist: accelerate<2,>=1.4; extra == 'all'
|
|
36
36
|
Requires-Dist: adlfs<2027,>=2024.7; extra == 'all'
|
|
37
37
|
Requires-Dist: asyncssh<3,>=2.18; extra == 'all'
|
|
38
|
+
Requires-Dist: bitsandbytes<1,>=0.49; extra == 'all'
|
|
38
39
|
Requires-Dist: boto3<2,>=1.35; extra == 'all'
|
|
39
40
|
Requires-Dist: cohere<8,>=5.13; extra == 'all'
|
|
40
41
|
Requires-Dist: datasets<6,>=4.0; extra == 'all'
|
|
@@ -66,6 +67,7 @@ Requires-Dist: nltk<4,>=3.9; extra == 'evals'
|
|
|
66
67
|
Requires-Dist: rouge-score<0.2,>=0.1.2; extra == 'evals'
|
|
67
68
|
Provides-Extra: finetuning
|
|
68
69
|
Requires-Dist: accelerate<2,>=1.4; extra == 'finetuning'
|
|
70
|
+
Requires-Dist: bitsandbytes<1,>=0.49; extra == 'finetuning'
|
|
69
71
|
Requires-Dist: datasets<6,>=4.0; extra == 'finetuning'
|
|
70
72
|
Requires-Dist: peft<1,>=0.18; extra == 'finetuning'
|
|
71
73
|
Requires-Dist: torch<3,>=2.9; extra == 'finetuning'
|
|
@@ -118,7 +120,7 @@ Optional extras pull in the heavy dependencies only when you need them:
|
|
|
118
120
|
pip install "strata-forge[rag]" # embeddings, vector store, reranking
|
|
119
121
|
pip install "strata-forge[compute]" # SkyPilot + SSH remote execution
|
|
120
122
|
pip install "strata-forge[serving]" # vLLM
|
|
121
|
-
pip install "strata-forge[finetuning]" # torch, transformers, trl, peft
|
|
123
|
+
pip install "strata-forge[finetuning]" # torch, transformers, trl, peft, bitsandbytes
|
|
122
124
|
pip install "strata-forge[all]" # everything
|
|
123
125
|
```
|
|
124
126
|
|
|
@@ -21,7 +21,7 @@ Optional extras pull in the heavy dependencies only when you need them:
|
|
|
21
21
|
pip install "strata-forge[rag]" # embeddings, vector store, reranking
|
|
22
22
|
pip install "strata-forge[compute]" # SkyPilot + SSH remote execution
|
|
23
23
|
pip install "strata-forge[serving]" # vLLM
|
|
24
|
-
pip install "strata-forge[finetuning]" # torch, transformers, trl, peft
|
|
24
|
+
pip install "strata-forge[finetuning]" # torch, transformers, trl, peft, bitsandbytes
|
|
25
25
|
pip install "strata-forge[all]" # everything
|
|
26
26
|
```
|
|
27
27
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "strata-forge"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.3.0"
|
|
4
4
|
description = "Typed, async-first baseline for AI/LLM experiments across major foundation-model providers."
|
|
5
5
|
requires-python = ">=3.14"
|
|
6
6
|
readme = "README.md"
|
|
@@ -34,10 +34,14 @@ classifiers = [
|
|
|
34
34
|
# shipped that morning and a breaking upstream release lands as a mid-run crash on the user's
|
|
35
35
|
# hardware rather than as a red build here. Raise a ceiling deliberately, with the call sites
|
|
36
36
|
# checked against the new major -- never to make a resolver stop complaining.
|
|
37
|
+
# .github/workflows/clean-install.yml performs that fresh install every night (and on any branch
|
|
38
|
+
# push touching this file) and builds every config a run builds.
|
|
37
39
|
dependencies = [
|
|
38
40
|
"pydantic>=2.10,<3",
|
|
39
41
|
"pydantic-settings>=2.6,<3",
|
|
40
|
-
|
|
42
|
+
# The floor is the release the Responses API path (`litellm.aresponses`) is tested against;
|
|
43
|
+
# releases before 1.66 have no `aresponses` at all.
|
|
44
|
+
"litellm>=1.83,<2",
|
|
41
45
|
"structlog>=24.4,<27",
|
|
42
46
|
"tenacity>=9.0,<10",
|
|
43
47
|
"typer>=0.15,<1",
|
|
@@ -45,7 +49,7 @@ dependencies = [
|
|
|
45
49
|
"pyyaml>=6.0,<7",
|
|
46
50
|
"python-dotenv>=1.0,<2",
|
|
47
51
|
"rich>=13.9,<16",
|
|
48
|
-
"jinja2>=3.1,<4",
|
|
52
|
+
"jinja2>=3.1.6,<4",
|
|
49
53
|
]
|
|
50
54
|
|
|
51
55
|
[project.urls]
|
|
@@ -95,6 +99,15 @@ finetuning = [
|
|
|
95
99
|
"peft>=0.18,<1",
|
|
96
100
|
"datasets>=4.0,<6",
|
|
97
101
|
"accelerate>=1.4,<2",
|
|
102
|
+
# QLoRA quantises the base model inside `from_pretrained`, which refuses to load without this.
|
|
103
|
+
# 0.49 is the first release with both a CUDA 13.0 kernel (the build the torch above resolves
|
|
104
|
+
# to on Linux; 0.48 added it) and an Apple Silicon wheel, so it installs everywhere torch>=2.9
|
|
105
|
+
# publishes a cp314 wheel: Linux x86_64/aarch64, Windows x64, macOS arm64. That is why there
|
|
106
|
+
# is no platform marker. Transformers 5 asks for >=0.46.1. The macOS wheel needs macOS 14 and
|
|
107
|
+
# there is no sdist, so an Apple Silicon Mac on 12 or 13 cannot install this extra (torch would
|
|
108
|
+
# settle on an older macOS 11 wheel; bitsandbytes has none). `require_bitsandbytes` in
|
|
109
|
+
# training/peft.py holds the same floor; a test keeps the two equal.
|
|
110
|
+
"bitsandbytes>=0.49,<1",
|
|
98
111
|
]
|
|
99
112
|
all = [
|
|
100
113
|
"strata-forge[redis,multimodal,inspect,compute,serving,storage,finetuning,langfuse,hf,evals,rag,bedrock]",
|
|
@@ -209,9 +209,15 @@ async def _cancel(job_id: str) -> None:
|
|
|
209
209
|
|
|
210
210
|
|
|
211
211
|
async def _cleanup(job_id: str) -> None:
|
|
212
|
+
from strata_forge.compute.backends.base import CleanupError
|
|
213
|
+
|
|
212
214
|
job, backend_name, backend_kwargs = _load_job(job_id)
|
|
213
215
|
backend = _make_backend(backend_name, backend_kwargs)
|
|
214
|
-
|
|
216
|
+
try:
|
|
217
|
+
await backend.cleanup(job)
|
|
218
|
+
except CleanupError as exc:
|
|
219
|
+
# The saved job is the only handle on what survived, so it stays for the retry.
|
|
220
|
+
error_exit(f"cleanup failed, job kept for a retry: {exc}")
|
|
215
221
|
_state_path(job_id).unlink(missing_ok=True)
|
|
216
222
|
Console().print(f"[bold green]cleaned up[/] {job.id}")
|
|
217
223
|
|
|
@@ -47,7 +47,9 @@ def run_async[T](coro: Awaitable[T]) -> T:
|
|
|
47
47
|
|
|
48
48
|
def error_exit(message: str, *, code: int = 1) -> NoReturn:
|
|
49
49
|
"""Print ``message`` to stderr and exit with ``code``."""
|
|
50
|
-
|
|
50
|
+
# Not markup: messages name extras (``strata-forge[finetuning]``) and carry exception text,
|
|
51
|
+
# and rich would read either as a style tag and drop it from the line.
|
|
52
|
+
_ERROR_CONSOLE.print(f"error: {message}", markup=False)
|
|
51
53
|
raise typer.Exit(code=code)
|
|
52
54
|
|
|
53
55
|
|
|
@@ -157,11 +157,20 @@ def dpo_cmd(
|
|
|
157
157
|
def _build_peft(adapter: str, rank: int) -> Any:
|
|
158
158
|
if adapter == "none":
|
|
159
159
|
return None
|
|
160
|
-
from strata_forge.training.peft import
|
|
160
|
+
from strata_forge.training.peft import (
|
|
161
|
+
LoRAConfig,
|
|
162
|
+
MissingBitsAndBytesError,
|
|
163
|
+
QLoRAConfig,
|
|
164
|
+
require_bitsandbytes,
|
|
165
|
+
)
|
|
161
166
|
|
|
162
167
|
if adapter == "lora":
|
|
163
168
|
return LoRAConfig(r=rank)
|
|
164
169
|
if adapter == "qlora":
|
|
170
|
+
try:
|
|
171
|
+
require_bitsandbytes()
|
|
172
|
+
except MissingBitsAndBytesError as exc:
|
|
173
|
+
error_exit(str(exc))
|
|
165
174
|
return QLoRAConfig(lora=LoRAConfig(r=rank))
|
|
166
175
|
error_exit(f"unknown adapter {adapter!r}")
|
|
167
176
|
|
|
@@ -205,8 +214,8 @@ async def _run_sft(
|
|
|
205
214
|
) -> None:
|
|
206
215
|
from strata_forge.training.sft import SFTConfig, SFTRunner
|
|
207
216
|
|
|
208
|
-
dataset = await _resolve_dataset(dataset_name, dataset_version, dataset_file)
|
|
209
217
|
peft_cfg = _build_peft(adapter, adapter_rank)
|
|
218
|
+
dataset = await _resolve_dataset(dataset_name, dataset_version, dataset_file)
|
|
210
219
|
|
|
211
220
|
config = SFTConfig(
|
|
212
221
|
model_id=model_id,
|
|
@@ -257,8 +266,8 @@ async def _run_dpo(
|
|
|
257
266
|
) -> None:
|
|
258
267
|
from strata_forge.training.preference import DPOConfig, PreferenceRunner
|
|
259
268
|
|
|
260
|
-
dataset = await _resolve_dataset(dataset_name, dataset_version, dataset_file)
|
|
261
269
|
peft_cfg = _build_peft(adapter, adapter_rank)
|
|
270
|
+
dataset = await _resolve_dataset(dataset_name, dataset_version, dataset_file)
|
|
262
271
|
|
|
263
272
|
config = DPOConfig(
|
|
264
273
|
model_id=model_id,
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# strata_forge.compute
|
|
2
|
+
|
|
3
|
+
Remote compute orchestration. The module ships typed `Task` /
|
|
4
|
+
`ResourceSpec` / `Job` / `JobStatus` shapes, a YAML task loader for
|
|
5
|
+
the SkyPilot-task-YAML subset, a `Backend` Protocol that every
|
|
6
|
+
implementation satisfies, and an in-process `LocalBackend` for
|
|
7
|
+
tests and ad-hoc local runs.
|
|
8
|
+
|
|
9
|
+
The SSH and SkyPilot backends sit behind the lazy `[compute]` extra,
|
|
10
|
+
alongside a batch inference runner that consumes any `Backend`, and
|
|
11
|
+
vLLM / TGI / SGLang serving adapters that present as
|
|
12
|
+
`strata_forge.llm` `openai_compat` providers. The training runners
|
|
13
|
+
(SFT, DPO/ORPO/KTO, PEFT/LoRA/QLoRA) live under
|
|
14
|
+
`strata_forge.training`.
|
|
15
|
+
|
|
16
|
+
Credentials a job needs travel beside the task, never in it:
|
|
17
|
+
`Task.secrets` holds them as `SecretStr` values that are never
|
|
18
|
+
serialised, rendered or logged, and every backend delivers them as a
|
|
19
|
+
private file the job finds through `FORGE_SECRETS_FILE`, never as an
|
|
20
|
+
environment variable or on a command line. A backend with no private
|
|
21
|
+
channel (SkyPilot) refuses a task that carries secrets.
|
|
22
|
+
|
|
23
|
+
Cleanup is verified: `cleanup` returns only once a job's backend-side
|
|
24
|
+
state (its remote workdir, and any secrets file left in it) is gone,
|
|
25
|
+
raises `CleanupError` when that state survived, and treats state that
|
|
26
|
+
is already gone as success, so an orchestrator can retry it until it
|
|
27
|
+
succeeds.
|
|
28
|
+
|
|
29
|
+
A model server holds no credential at all: the batch-inference runner
|
|
30
|
+
downloads the model's safetensors weights, configs and tokenizer files
|
|
31
|
+
itself, with the run's own token, and serves that local snapshot with
|
|
32
|
+
the Hub switched off. `build_vllm_task(served_model_name=...)` keeps
|
|
33
|
+
the served model addressable by its Hub id.
|
|
34
|
+
|
|
35
|
+
See [ADR 0013](../../../docs/architecture/adr/0013-compute-task-and-backend-shapes.md)
|
|
36
|
+
for the task-as-data + Protocol design rationale,
|
|
37
|
+
[ADR 0019](../../../docs/architecture/adr/0019-secrets-travel-beside-the-task.md)
|
|
38
|
+
for secret delivery, and `docs/roadmap.md` for the current status.
|
|
@@ -15,9 +15,11 @@ the batch runner are dep-free.
|
|
|
15
15
|
from strata_forge.compute.backends import (
|
|
16
16
|
MAX_CONSOLE_CHUNK_BYTES,
|
|
17
17
|
Backend,
|
|
18
|
+
CleanupError,
|
|
18
19
|
LocalBackend,
|
|
19
20
|
SkyPilotBackend,
|
|
20
21
|
SSHBackend,
|
|
22
|
+
SubmitCleanupError,
|
|
21
23
|
safe_workdir_relpath,
|
|
22
24
|
)
|
|
23
25
|
from strata_forge.compute.batch import BatchInferenceResult, BatchInferenceRunner
|
|
@@ -30,13 +32,16 @@ from strata_forge.compute.serving import (
|
|
|
30
32
|
serving_endpoint,
|
|
31
33
|
wait_for_endpoint,
|
|
32
34
|
)
|
|
33
|
-
from strata_forge.compute.task import ResourceSpec, Task
|
|
35
|
+
from strata_forge.compute.task import SECRETS_FILE_ENV, SECRETS_FILE_NAME, ResourceSpec, Task
|
|
34
36
|
|
|
35
37
|
__all__ = [
|
|
36
38
|
"MAX_CONSOLE_CHUNK_BYTES",
|
|
39
|
+
"SECRETS_FILE_ENV",
|
|
40
|
+
"SECRETS_FILE_NAME",
|
|
37
41
|
"Backend",
|
|
38
42
|
"BatchInferenceResult",
|
|
39
43
|
"BatchInferenceRunner",
|
|
44
|
+
"CleanupError",
|
|
40
45
|
"ConsoleChunk",
|
|
41
46
|
"Job",
|
|
42
47
|
"JobState",
|
|
@@ -46,6 +51,7 @@ __all__ = [
|
|
|
46
51
|
"SSHBackend",
|
|
47
52
|
"ServingEndpoint",
|
|
48
53
|
"SkyPilotBackend",
|
|
54
|
+
"SubmitCleanupError",
|
|
49
55
|
"Task",
|
|
50
56
|
"build_sglang_task",
|
|
51
57
|
"build_tgi_task",
|
|
@@ -9,6 +9,8 @@ and :class:`SkyPilotBackend` lazy-import their SDKs behind the
|
|
|
9
9
|
from strata_forge.compute.backends.base import (
|
|
10
10
|
MAX_CONSOLE_CHUNK_BYTES,
|
|
11
11
|
Backend,
|
|
12
|
+
CleanupError,
|
|
13
|
+
SubmitCleanupError,
|
|
12
14
|
safe_workdir_relpath,
|
|
13
15
|
)
|
|
14
16
|
from strata_forge.compute.backends.local import LocalBackend
|
|
@@ -18,8 +20,10 @@ from strata_forge.compute.backends.ssh import SSHBackend
|
|
|
18
20
|
__all__ = [
|
|
19
21
|
"MAX_CONSOLE_CHUNK_BYTES",
|
|
20
22
|
"Backend",
|
|
23
|
+
"CleanupError",
|
|
21
24
|
"LocalBackend",
|
|
22
25
|
"SSHBackend",
|
|
23
26
|
"SkyPilotBackend",
|
|
27
|
+
"SubmitCleanupError",
|
|
24
28
|
"safe_workdir_relpath",
|
|
25
29
|
]
|
|
@@ -13,7 +13,12 @@ from __future__ import annotations
|
|
|
13
13
|
import posixpath
|
|
14
14
|
from typing import TYPE_CHECKING, Protocol, runtime_checkable
|
|
15
15
|
|
|
16
|
+
from strata_forge.compute.task import SECRETS_FILE_ENV
|
|
17
|
+
from strata_forge.core.errors import ForgeError
|
|
18
|
+
|
|
16
19
|
if TYPE_CHECKING:
|
|
20
|
+
from collections.abc import Sequence
|
|
21
|
+
|
|
17
22
|
from strata_forge.compute.job import ConsoleChunk, Job, JobStatus
|
|
18
23
|
from strata_forge.compute.task import Task
|
|
19
24
|
|
|
@@ -21,9 +26,16 @@ __all__ = [
|
|
|
21
26
|
"MAX_CONSOLE_CHUNK_BYTES",
|
|
22
27
|
"MAX_READ_FILE_BYTES",
|
|
23
28
|
"Backend",
|
|
29
|
+
"CleanupError",
|
|
30
|
+
"SubmitCleanupError",
|
|
24
31
|
"safe_workdir_relpath",
|
|
32
|
+
"secrets_guarded_script",
|
|
25
33
|
]
|
|
26
34
|
|
|
35
|
+
# The unexported shell variable that holds the secrets file's path for the trap. Unexported so
|
|
36
|
+
# no child of the job (the setup step's package installs above all) inherits a pointer to it.
|
|
37
|
+
_SECRETS_PATH_VAR = "_forge_secrets_file"
|
|
38
|
+
|
|
27
39
|
# Hard cap on a single :meth:`Backend.read_file` result. A control plane polls
|
|
28
40
|
# read_file on an interval from a SHARED process; without a cap a runaway/malicious
|
|
29
41
|
# file (e.g. a huge progress.jsonl) could exhaust that process's memory and degrade
|
|
@@ -37,6 +49,71 @@ MAX_READ_FILE_BYTES = 8 * 1024 * 1024
|
|
|
37
49
|
MAX_CONSOLE_CHUNK_BYTES = 64 * 1024
|
|
38
50
|
|
|
39
51
|
|
|
52
|
+
class SubmitCleanupError(RuntimeError):
|
|
53
|
+
"""A submit failed after creating remote state that it then could not remove.
|
|
54
|
+
|
|
55
|
+
Raised from the submit's own failure (its ``__cause__``) when that failure is an ordinary
|
|
56
|
+
exception; a cancellation is re-raised as itself. That state may hold the job's secrets file,
|
|
57
|
+
and a submit that raises hands back no job, so :attr:`job` is the handle to remove it with:
|
|
58
|
+
pass it to :meth:`Backend.cleanup` once the host is reachable again. It names no running
|
|
59
|
+
process, so ``status``, ``cancel`` and the log readers do not apply to it.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
def __init__(self, message: str, job: Job) -> None:
|
|
63
|
+
super().__init__(message)
|
|
64
|
+
self.job = job
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class CleanupError(ForgeError, OSError):
|
|
68
|
+
""":meth:`Backend.cleanup` ran, and the job's backend-side state is still there.
|
|
69
|
+
|
|
70
|
+
The state may hold the job's secrets file, so a caller that sees this has NOT cleaned up and
|
|
71
|
+
should try again later: the method is idempotent, and a retry against state that has since
|
|
72
|
+
gone is a success. The message names the job and how the removal failed (an exit status,
|
|
73
|
+
an exception type), never what the remote printed or what the state contains, so it is safe
|
|
74
|
+
to store and show. A transport failure (the host is unreachable, the command timed out) is
|
|
75
|
+
raised as itself, not as this: it says nothing about whether the state survived.
|
|
76
|
+
|
|
77
|
+
It is also an :class:`OSError`, as the failure of :func:`shutil.rmtree` it stands for would
|
|
78
|
+
be, so an orchestrator that already treats an ``OSError`` from ``cleanup`` as "not cleaned"
|
|
79
|
+
keeps doing so rather than letting this one escape.
|
|
80
|
+
"""
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def secrets_guarded_script(
|
|
84
|
+
path_expr: str, setup: str, run: str, *, prelude: Sequence[str] = ()
|
|
85
|
+
) -> str:
|
|
86
|
+
"""The shell text that runs ``setup`` then ``run`` for a job whose secrets are in a file.
|
|
87
|
+
|
|
88
|
+
``path_expr`` is a shell word that evaluates to the file's absolute path. The script:
|
|
89
|
+
|
|
90
|
+
- removes the file on exit through a trap in the OUTER shell, and runs ``prelude``, ``setup``
|
|
91
|
+
and ``run`` in a subshell below it. A task's own commands may install an ``EXIT`` trap of
|
|
92
|
+
their own (an orchestrator's bootstrap does, to stop a progress ticker), and a second
|
|
93
|
+
``trap ... EXIT`` in one shell REPLACES the first: in the same shell, that would silently
|
|
94
|
+
drop the removal, and the file would outlive a failed setup or a cancellation. A trap set
|
|
95
|
+
in the subshell cannot reach the outer one.
|
|
96
|
+
- exports :data:`~strata_forge.compute.task.SECRETS_FILE_ENV` only between ``setup`` and
|
|
97
|
+
``run``, so the setup step's package installs never hold even the path to the file.
|
|
98
|
+
|
|
99
|
+
The outer shell exits with the subshell's status, which the trap leaves untouched. bash runs
|
|
100
|
+
an ``EXIT`` trap on a normal exit and on a fatal signal such as the SIGTERM a cancel sends;
|
|
101
|
+
only SIGKILL skips it.
|
|
102
|
+
"""
|
|
103
|
+
run_step = f'export {SECRETS_FILE_ENV}="${_SECRETS_PATH_VAR}" && {run}'
|
|
104
|
+
body = f"{setup} && {run_step}" if setup else run_step
|
|
105
|
+
return "\n".join(
|
|
106
|
+
[
|
|
107
|
+
f"{_SECRETS_PATH_VAR}={path_expr}",
|
|
108
|
+
f"trap 'rm -f \"${_SECRETS_PATH_VAR}\"' EXIT",
|
|
109
|
+
"(",
|
|
110
|
+
*prelude,
|
|
111
|
+
body,
|
|
112
|
+
")",
|
|
113
|
+
]
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
|
|
40
117
|
def safe_workdir_relpath(path: str) -> str:
|
|
41
118
|
"""Validate ``path`` as a job-workdir-relative path and return it normalized.
|
|
42
119
|
|
|
@@ -72,7 +149,15 @@ class Backend(Protocol):
|
|
|
72
149
|
... # pragma: no cover — Protocol body
|
|
73
150
|
|
|
74
151
|
async def submit(self, task: Task) -> Job:
|
|
75
|
-
"""Submit ``task`` for execution; return its :class:`Job` handle.
|
|
152
|
+
"""Submit ``task`` for execution; return its :class:`Job` handle.
|
|
153
|
+
|
|
154
|
+
``task.secrets`` are written to a private (0600) file whose absolute path the job's
|
|
155
|
+
``run`` step finds in ``FORGE_SECRETS_FILE``, never into the job's environment, a command
|
|
156
|
+
line, a script or the returned job's metadata. A backend with no such channel raises
|
|
157
|
+
rather than falling back to the environment. A submit that fails after creating remote
|
|
158
|
+
state it then cannot remove raises :class:`SubmitCleanupError`, whose ``job`` names that
|
|
159
|
+
state for :meth:`cleanup`.
|
|
160
|
+
"""
|
|
76
161
|
... # pragma: no cover — Protocol body
|
|
77
162
|
|
|
78
163
|
async def status(self, job: Job) -> JobStatus:
|
|
@@ -140,11 +225,20 @@ class Backend(Protocol):
|
|
|
140
225
|
... # pragma: no cover — Protocol body
|
|
141
226
|
|
|
142
227
|
async def cleanup(self, job: Job) -> None:
|
|
143
|
-
"""Tear down any backend-side state associated with ``job
|
|
228
|
+
"""Tear down any backend-side state associated with ``job``, and prove it is gone.
|
|
144
229
|
|
|
145
230
|
Examples: remote temporary directories (SSH), SkyPilot
|
|
146
231
|
clusters launched specifically for this job, captured log
|
|
147
232
|
buffers. Idempotent — callers can invoke it repeatedly or
|
|
148
|
-
on already-cleaned jobs
|
|
233
|
+
on already-cleaned jobs; state that is already absent is a
|
|
234
|
+
success.
|
|
235
|
+
|
|
236
|
+
Returning means the state is gone, not that a removal was
|
|
237
|
+
attempted: that state may hold the job's secrets file, and
|
|
238
|
+
a caller that retries until cleanup succeeds can only do so
|
|
239
|
+
if a failed one says so. When the backend could run the
|
|
240
|
+
removal but the state survived it, this raises
|
|
241
|
+
:class:`CleanupError`; a transport failure propagates as
|
|
242
|
+
itself.
|
|
149
243
|
"""
|
|
150
244
|
... # pragma: no cover — Protocol body
|
|
@@ -23,6 +23,14 @@ Those files outlive ``cleanup`` and the backend object, which is the
|
|
|
23
23
|
point: when the process that owned the buffers is gone, the file is
|
|
24
24
|
the only remaining evidence. It is opt-in so that a library user who
|
|
25
25
|
never asks for it never finds log files appearing beside their code.
|
|
26
|
+
|
|
27
|
+
``Task.secrets`` are written to a 0600 file in a private (0700)
|
|
28
|
+
temporary directory of the job's own, and the task's ``run`` step
|
|
29
|
+
finds its path in ``FORGE_SECRETS_FILE`` (``setup`` does not); the
|
|
30
|
+
values are never in the child's environment. The child's outer shell
|
|
31
|
+
removes the file on exit, below a subshell the task's own traps
|
|
32
|
+
cannot displace, and the backend removes the directory once the
|
|
33
|
+
child is gone.
|
|
26
34
|
"""
|
|
27
35
|
|
|
28
36
|
from __future__ import annotations
|
|
@@ -30,7 +38,10 @@ from __future__ import annotations
|
|
|
30
38
|
import asyncio
|
|
31
39
|
import contextlib
|
|
32
40
|
import os
|
|
41
|
+
import shlex
|
|
42
|
+
import shutil
|
|
33
43
|
import signal
|
|
44
|
+
import tempfile
|
|
34
45
|
import uuid
|
|
35
46
|
from datetime import UTC, datetime
|
|
36
47
|
from pathlib import Path
|
|
@@ -39,9 +50,12 @@ from typing import TYPE_CHECKING
|
|
|
39
50
|
from strata_forge.compute.backends.base import (
|
|
40
51
|
MAX_CONSOLE_CHUNK_BYTES,
|
|
41
52
|
MAX_READ_FILE_BYTES,
|
|
53
|
+
CleanupError,
|
|
42
54
|
safe_workdir_relpath,
|
|
55
|
+
secrets_guarded_script,
|
|
43
56
|
)
|
|
44
57
|
from strata_forge.compute.job import ConsoleChunk, Job, JobStatus
|
|
58
|
+
from strata_forge.compute.task import SECRETS_FILE_ENV, SECRETS_FILE_NAME, render_secrets_payload
|
|
45
59
|
|
|
46
60
|
if TYPE_CHECKING:
|
|
47
61
|
from strata_forge.compute.task import Task
|
|
@@ -186,6 +200,8 @@ class _JobState:
|
|
|
186
200
|
self.workdir: str | None = None
|
|
187
201
|
# Where this job's streams were teed, when a log_dir was configured.
|
|
188
202
|
self.log_paths: tuple[Path, Path] | None = None
|
|
203
|
+
# The private directory holding the job's secrets file, while there is one.
|
|
204
|
+
self.secrets_dir: Path | None = None
|
|
189
205
|
|
|
190
206
|
|
|
191
207
|
class LocalBackend:
|
|
@@ -195,8 +211,12 @@ class LocalBackend:
|
|
|
195
211
|
name: Optional override for the backend's reported name.
|
|
196
212
|
Default ``"local"``.
|
|
197
213
|
env_inherit: When ``True`` (default), child processes
|
|
198
|
-
inherit the parent's ``os.environ
|
|
199
|
-
|
|
214
|
+
inherit the parent's ``os.environ`` (minus any
|
|
215
|
+
``FORGE_SECRETS_FILE``, which names the parent's file,
|
|
216
|
+
not the child's). When ``False``, the child sees only
|
|
217
|
+
``Task.env``. A task with secrets also exports
|
|
218
|
+
``FORGE_SECRETS_FILE`` to its ``run`` step, never to
|
|
219
|
+
``setup``.
|
|
200
220
|
log_dir: When set, every job also tees its stdout/stderr
|
|
201
221
|
to ``serve.stdout.log`` / ``serve.stderr.log`` under
|
|
202
222
|
this directory, written as the bytes arrive and left
|
|
@@ -229,15 +249,26 @@ class LocalBackend:
|
|
|
229
249
|
if task.num_nodes != 1:
|
|
230
250
|
err = f"LocalBackend can only run single-node tasks; got num_nodes={task.num_nodes}"
|
|
231
251
|
raise ValueError(err)
|
|
252
|
+
# A task built with `model_copy(update=...)` skipped every validator; nothing is
|
|
253
|
+
# delivered until its secrets have been through them.
|
|
254
|
+
task = task.revalidated()
|
|
232
255
|
|
|
233
256
|
job_id = uuid.uuid4().hex
|
|
234
257
|
state = _JobState()
|
|
235
258
|
state.workdir = task.workdir
|
|
236
259
|
state.log_paths = self._prepare_log_paths(state)
|
|
237
|
-
self._jobs[job_id] = state
|
|
238
260
|
|
|
239
261
|
env = self._build_env(task)
|
|
240
262
|
script = self._build_script(task)
|
|
263
|
+
if task.secrets:
|
|
264
|
+
state.secrets_dir = _write_secrets_file(task)
|
|
265
|
+
# The child's outer shell removes the file however it exits, so it does not outlive
|
|
266
|
+
# a setup that failed before anything read it, and only `run` is told where it is.
|
|
267
|
+
# The directory goes with the job.
|
|
268
|
+
script = secrets_guarded_script(
|
|
269
|
+
shlex.quote(str(state.secrets_dir / SECRETS_FILE_NAME)), task.setup, task.run
|
|
270
|
+
)
|
|
271
|
+
self._jobs[job_id] = state
|
|
241
272
|
|
|
242
273
|
async def _runner() -> None:
|
|
243
274
|
try:
|
|
@@ -288,6 +319,7 @@ class LocalBackend:
|
|
|
288
319
|
state.exit_code = -1
|
|
289
320
|
state.stderr_buffer.extend(f"local backend exception: {exc!r}".encode())
|
|
290
321
|
finally:
|
|
322
|
+
_remove_secrets_dir(state)
|
|
291
323
|
state.finished_at = datetime.now(UTC)
|
|
292
324
|
|
|
293
325
|
# Fire-and-forget; status / logs / cancel observe state directly.
|
|
@@ -321,6 +353,9 @@ class LocalBackend:
|
|
|
321
353
|
env: dict[str, str] = {}
|
|
322
354
|
if self._env_inherit:
|
|
323
355
|
env.update(os.environ)
|
|
356
|
+
# A path to the PARENT's secrets file is not the child's to read, and a child that
|
|
357
|
+
# read it would also delete it. Only this task's own file is ever named.
|
|
358
|
+
env.pop(SECRETS_FILE_ENV, None)
|
|
324
359
|
env.update(task.env)
|
|
325
360
|
return env
|
|
326
361
|
|
|
@@ -472,14 +507,17 @@ class LocalBackend:
|
|
|
472
507
|
state.cancelled = True
|
|
473
508
|
|
|
474
509
|
async def cleanup(self, job: Job) -> None:
|
|
475
|
-
"""
|
|
510
|
+
"""Make sure the job's process is gone, remove its secrets directory, drop its state.
|
|
476
511
|
|
|
477
|
-
|
|
478
|
-
|
|
512
|
+
Raises :class:`CleanupError` when the secrets directory survives the removal, and keeps
|
|
513
|
+
the job's state so a later call can retry it: dropping the state first would leave the
|
|
514
|
+
directory with nothing that names it. Any ``log_dir`` files are left on disk on purpose:
|
|
515
|
+
they exist to be read AFTER the buffers they mirror have been discarded.
|
|
479
516
|
"""
|
|
480
|
-
state = self._jobs.
|
|
517
|
+
state = self._jobs.get(job.id)
|
|
481
518
|
if state is None:
|
|
482
519
|
return # already cleaned up — idempotent
|
|
520
|
+
removed = _remove_secrets_dir(state)
|
|
483
521
|
# Make sure any lingering process is gone before we drop the state. Once this returns,
|
|
484
522
|
# the state is dropped and nothing can name the group again, so it sweeps unconditionally
|
|
485
523
|
# for the same reason cancel does — a job whose own process exited may still have left
|
|
@@ -487,6 +525,59 @@ class LocalBackend:
|
|
|
487
525
|
process = state.process
|
|
488
526
|
if process is not None:
|
|
489
527
|
await _reap_group(process)
|
|
528
|
+
# Once more after the reap: the tree that was just stopped may have been writing there.
|
|
529
|
+
if not removed and not _remove_secrets_dir(state):
|
|
530
|
+
err = f"LocalBackend: job {job.id!r}'s secrets directory survived cleanup"
|
|
531
|
+
raise CleanupError(err)
|
|
532
|
+
self._jobs.pop(job.id, None)
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def _write_secrets_file(task: Task) -> Path:
|
|
536
|
+
"""Write ``task.secrets`` to a 0600 file in a fresh 0700 directory; return the directory.
|
|
537
|
+
|
|
538
|
+
Modes are set explicitly rather than requested through ``mkdir``/``open``: those only ever
|
|
539
|
+
narrow by the umask, so a restrictive umask would leave the file unreadable by the child and
|
|
540
|
+
the explicit ``chmod`` is what makes the mode exact. ``O_EXCL | O_NOFOLLOW`` refuses a file
|
|
541
|
+
or symlink already at the path rather than writing through it.
|
|
542
|
+
"""
|
|
543
|
+
directory = Path(tempfile.mkdtemp(prefix="forge-job-"))
|
|
544
|
+
try:
|
|
545
|
+
directory.chmod(0o700)
|
|
546
|
+
fd = os.open(
|
|
547
|
+
directory / SECRETS_FILE_NAME,
|
|
548
|
+
os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW,
|
|
549
|
+
0o600,
|
|
550
|
+
)
|
|
551
|
+
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
|
552
|
+
os.fchmod(fh.fileno(), 0o600)
|
|
553
|
+
fh.write(render_secrets_payload(task.secrets))
|
|
554
|
+
except BaseException:
|
|
555
|
+
shutil.rmtree(directory, ignore_errors=True)
|
|
556
|
+
raise
|
|
557
|
+
return directory
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def _remove_secrets_dir(state: _JobState) -> bool:
|
|
561
|
+
"""Remove the job's secrets directory, if it still has one; return whether it is gone.
|
|
562
|
+
|
|
563
|
+
Idempotent. The job forgets the directory only once nothing is left at its path, so a removal
|
|
564
|
+
that failed is retried by the next call rather than recorded as done.
|
|
565
|
+
"""
|
|
566
|
+
directory = state.secrets_dir
|
|
567
|
+
if directory is None:
|
|
568
|
+
return True
|
|
569
|
+
shutil.rmtree(directory, ignore_errors=True)
|
|
570
|
+
# `lstat`, not `exists`: `exists` returns False on ANY error, so a parent that cannot be
|
|
571
|
+
# searched (EACCES) would read as "gone" while the file is still there. Only "no such
|
|
572
|
+
# entry" proves it.
|
|
573
|
+
try:
|
|
574
|
+
directory.lstat()
|
|
575
|
+
except FileNotFoundError, NotADirectoryError:
|
|
576
|
+
state.secrets_dir = None
|
|
577
|
+
return True
|
|
578
|
+
except OSError:
|
|
579
|
+
return False
|
|
580
|
+
return False
|
|
490
581
|
|
|
491
582
|
|
|
492
583
|
def _slice_stream(
|