agent-learning-kit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_learning_kit-0.1.0.dist-info/METADATA +381 -0
- agent_learning_kit-0.1.0.dist-info/RECORD +642 -0
- agent_learning_kit-0.1.0.dist-info/WHEEL +4 -0
- agent_learning_kit-0.1.0.dist-info/entry_points.txt +5 -0
- agent_learning_kit-0.1.0.dist-info/licenses/LICENSE +173 -0
- agent_learning_kit-0.1.0.dist-info/licenses/NOTICE +7 -0
- fi/__init__.py +5 -0
- fi/alk/__init__.py +57 -0
- fi/alk/_facade.py +31 -0
- fi/alk/_module_alias.py +68 -0
- fi/alk/_paths.py +14 -0
- fi/alk/_schema.py +522 -0
- fi/alk/actions.py +727 -0
- fi/alk/bench/__init__.py +517 -0
- fi/alk/bench/_codeexec.py +213 -0
- fi/alk/bench/_coding.py +215 -0
- fi/alk/bench/_docker.py +237 -0
- fi/alk/bench/_grader.py +286 -0
- fi/alk/bench/_pull.py +212 -0
- fi/alk/bench/_voice.py +147 -0
- fi/alk/capabilities.py +627 -0
- fi/alk/cli.py +6396 -0
- fi/alk/config.py +130 -0
- fi/alk/cua_loop.py +562 -0
- fi/alk/evals.py +2351 -0
- fi/alk/extensions.py +163 -0
- fi/alk/harness/ARCHITECTURE.md +231 -0
- fi/alk/harness/DESIGN.md +246 -0
- fi/alk/harness/ENVIRONMENT_CONFORMANCE.md +127 -0
- fi/alk/harness/HOW-IT-WORKS.md +297 -0
- fi/alk/harness/IMPLEMENTATION_AND_VALIDATION_STATUS.md +229 -0
- fi/alk/harness/README.md +417 -0
- fi/alk/harness/__init__.py +77 -0
- fi/alk/harness/__main__.py +3 -0
- fi/alk/harness/amend.py +312 -0
- fi/alk/harness/artifacts.py +319 -0
- fi/alk/harness/authoring_entrypoint.py +189 -0
- fi/alk/harness/authoring_runtime_validation.py +267 -0
- fi/alk/harness/backends/README.md +43 -0
- fi/alk/harness/backends/__init__.py +122 -0
- fi/alk/harness/backends/base.py +241 -0
- fi/alk/harness/backends/claude.py +211 -0
- fi/alk/harness/backends/files.py +182 -0
- fi/alk/harness/backends/vertex_gemini.py +457 -0
- fi/alk/harness/background_noise.py +95 -0
- fi/alk/harness/build.py +385 -0
- fi/alk/harness/bundle.py +593 -0
- fi/alk/harness/bundle_author_v2.py +1831 -0
- fi/alk/harness/bundle_v2.py +719 -0
- fi/alk/harness/call_runner.py +1440 -0
- fi/alk/harness/callback_http_adapter.py +111 -0
- fi/alk/harness/catalogue.py +287 -0
- fi/alk/harness/chat.py +428 -0
- fi/alk/harness/chat_call_runner.py +506 -0
- fi/alk/harness/checks.py +136 -0
- fi/alk/harness/cli.py +1354 -0
- fi/alk/harness/config.py +338 -0
- fi/alk/harness/contract.py +718 -0
- fi/alk/harness/credentials.py +674 -0
- fi/alk/harness/data/persona_vocabulary.json +111 -0
- fi/alk/harness/environment.py +99 -0
- fi/alk/harness/environment_plan.py +168 -0
- fi/alk/harness/events.py +125 -0
- fi/alk/harness/executor.py +304 -0
- fi/alk/harness/folder.py +234 -0
- fi/alk/harness/generated_runtime.py +815 -0
- fi/alk/harness/github.py +72 -0
- fi/alk/harness/hosted_authoring_entrypoint.py +183 -0
- fi/alk/harness/hosted_entrypoint.py +2402 -0
- fi/alk/harness/hosted_scheduler.py +2218 -0
- fi/alk/harness/job.py +426 -0
- fi/alk/harness/judge.py +184 -0
- fi/alk/harness/livekit_source.py +50 -0
- fi/alk/harness/livekit_tool_trace_bootstrap.py +71 -0
- fi/alk/harness/observability.py +208 -0
- fi/alk/harness/outbound.py +3252 -0
- fi/alk/harness/packaging.py +515 -0
- fi/alk/harness/persona_guides.py +157 -0
- fi/alk/harness/platform.py +692 -0
- fi/alk/harness/process_preflight.py +764 -0
- fi/alk/harness/process_runtime.py +5670 -0
- fi/alk/harness/prove.py +425 -0
- fi/alk/harness/provider_import.py +703 -0
- fi/alk/harness/provider_lifecycle.py +392 -0
- fi/alk/harness/provision.py +2896 -0
- fi/alk/harness/reception.py +147 -0
- fi/alk/harness/retell_chat_call_runner.py +373 -0
- fi/alk/harness/run/__init__.py +296 -0
- fi/alk/harness/run/alk.py +184 -0
- fi/alk/harness/run/call.py +162 -0
- fi/alk/harness/run/conversation.py +264 -0
- fi/alk/harness/run/data/voices_by_language_and_gender.json +693 -0
- fi/alk/harness/run/evidence.py +195 -0
- fi/alk/harness/run/grade.py +598 -0
- fi/alk/harness/run/live.py +297 -0
- fi/alk/harness/run/models.py +56 -0
- fi/alk/harness/run/platform_evals.py +227 -0
- fi/alk/harness/run/sdk_voice.py +130 -0
- fi/alk/harness/run/simulation.py +1209 -0
- fi/alk/harness/run/stage.py +91 -0
- fi/alk/harness/run/targets.py +508 -0
- fi/alk/harness/run/tools.py +601 -0
- fi/alk/harness/run/voice.py +340 -0
- fi/alk/harness/runtime.py +172 -0
- fi/alk/harness/sandbox_server.py +2011 -0
- fi/alk/harness/sandbox_worker.py +44 -0
- fi/alk/harness/scenario.py +1048 -0
- fi/alk/harness/scenario_source.py +879 -0
- fi/alk/harness/scenario_tools.py +1143 -0
- fi/alk/harness/scenarios.py +915 -0
- fi/alk/harness/secrets.py +168 -0
- fi/alk/harness/service_catalog.py +97 -0
- fi/alk/harness/session.py +391 -0
- fi/alk/harness/sessions.py +372 -0
- fi/alk/harness/simulator.py +76 -0
- fi/alk/harness/simulator_voice.py +928 -0
- fi/alk/harness/skills/build-environment/SKILL.md +538 -0
- fi/alk/harness/skills/harness.md +131 -0
- fi/alk/harness/skills/kinds/chat.md +48 -0
- fi/alk/harness/skills/kinds/voice-voicemail.md +63 -0
- fi/alk/harness/skills/kinds/voice.md +59 -0
- fi/alk/harness/skills/plan-suite/SKILL.md +103 -0
- fi/alk/harness/skills/provision-environment/SKILL.md +136 -0
- fi/alk/harness/skills/run-scenarios/SKILL.md +112 -0
- fi/alk/harness/skills/understand-agent/SKILL.md +251 -0
- fi/alk/harness/skills/write-scenarios/SKILL.md +606 -0
- fi/alk/harness/skills/write-scenarios/references/refusals.md +28 -0
- fi/alk/harness/skills/write-scenarios/references/world-api.md +92 -0
- fi/alk/harness/source_data_invariants.py +444 -0
- fi/alk/harness/source_tool_evidence.py +79 -0
- fi/alk/harness/sources.py +253 -0
- fi/alk/harness/spend.py +140 -0
- fi/alk/harness/tool_trace_proxy.py +104 -0
- fi/alk/harness/tools.py +1018 -0
- fi/alk/harness/understand.py +169 -0
- fi/alk/harness/voicemail_audio.py +74 -0
- fi/alk/harness/world/__init__.py +33 -0
- fi/alk/harness/world/errors.py +68 -0
- fi/alk/harness/world/expectations.py +91 -0
- fi/alk/harness/world/handle.py +538 -0
- fi/alk/harness/world/kinds.py +196 -0
- fi/alk/harness/world/mutate.py +186 -0
- fi/alk/harness/world/probe.py +413 -0
- fi/alk/harness/world/provision.py +511 -0
- fi/alk/harness/world/provisioned.py +191 -0
- fi/alk/harness/world/runtime.py +616 -0
- fi/alk/harness/world/snapshot.py +288 -0
- fi/alk/harness/world/stores/__init__.py +305 -0
- fi/alk/harness/world/stores/container.py +215 -0
- fi/alk/harness/world/stores/inprocess.py +346 -0
- fi/alk/harness/world/stores/postgres.py +481 -0
- fi/alk/harness/world/stores/prove.py +202 -0
- fi/alk/harness/world/stores/sqlite.py +245 -0
- fi/alk/harness/world/stores/written.py +182 -0
- fi/alk/harness/world/tools.py +1516 -0
- fi/alk/harness/world/workspace.py +144 -0
- fi/alk/image_loop.py +453 -0
- fi/alk/image_perturb.py +241 -0
- fi/alk/improve.py +274 -0
- fi/alk/live/__init__.py +154 -0
- fi/alk/live/_attribution.py +184 -0
- fi/alk/live/_capture.py +264 -0
- fi/alk/live/_codec.py +391 -0
- fi/alk/live/_contract.py +134 -0
- fi/alk/live/_loopback.py +316 -0
- fi/alk/live/_perturb.py +449 -0
- fi/alk/live/_runner.py +386 -0
- fi/alk/live/_stats.py +561 -0
- fi/alk/live/_transcript.py +240 -0
- fi/alk/live/_workers/__init__.py +9 -0
- fi/alk/live/_workers/a2a_worker.py +316 -0
- fi/alk/live/_workers/langgraph_worker.py +217 -0
- fi/alk/live/_workers/livekit_worker.py +207 -0
- fi/alk/live/_workers/mcp_loopback_server.py +46 -0
- fi/alk/live/_workers/mcp_worker.py +158 -0
- fi/alk/live/_workers/pipecat_worker.py +189 -0
- fi/alk/live/a2a_lane.py +138 -0
- fi/alk/live/langgraph_lane.py +339 -0
- fi/alk/live/livekit_lane.py +376 -0
- fi/alk/live/mcp_lane.py +172 -0
- fi/alk/live/pipecat_lane.py +341 -0
- fi/alk/live/voice_redteam.py +494 -0
- fi/alk/loss.py +306 -0
- fi/alk/optimize.py +36260 -0
- fi/alk/practice/__init__.py +51 -0
- fi/alk/practice/_assess.py +103 -0
- fi/alk/practice/_budget.py +81 -0
- fi/alk/practice/_calibrate.py +69 -0
- fi/alk/practice/_capstone.py +86 -0
- fi/alk/practice/_contract.py +91 -0
- fi/alk/practice/_diagnose.py +79 -0
- fi/alk/practice/_drill.py +196 -0
- fi/alk/practice/_experiment.py +720 -0
- fi/alk/practice/_schedule.py +102 -0
- fi/alk/practice/_store.py +194 -0
- fi/alk/practice/_trainer.py +245 -0
- fi/alk/practice/_update.py +125 -0
- fi/alk/redteam.py +2621 -0
- fi/alk/rewardhack.py +237 -0
- fi/alk/simulate.py +10351 -0
- fi/alk/studio/__init__.py +82 -0
- fi/alk/studio/_bias.py +314 -0
- fi/alk/studio/_calibration.py +522 -0
- fi/alk/studio/_coverage.py +262 -0
- fi/alk/studio/_download.py +665 -0
- fi/alk/studio/_fidelity_attack.py +114 -0
- fi/alk/studio/_generate.py +652 -0
- fi/alk/studio/_library.py +370 -0
- fi/alk/studio/_scan.py +134 -0
- fi/alk/studio/_upgrade.py +42 -0
- fi/alk/studio/_vendor.py +172 -0
- fi/alk/suite.py +4200 -0
- fi/alk/tasks.py +828 -0
- fi/alk/telemetry/__init__.py +149 -0
- fi/alk/telemetry/_contract.py +141 -0
- fi/alk/telemetry/_emit.py +182 -0
- fi/alk/telemetry/_ledger.py +296 -0
- fi/alk/telemetry/_queue.py +127 -0
- fi/alk/telemetry/_row.py +294 -0
- fi/alk/telemetry/_run.py +233 -0
- fi/alk/telemetry/_sync.py +193 -0
- fi/alk/telemetry/_url.py +119 -0
- fi/alk/trinity.py +49397 -0
- fi/alk/voice_loop.py +174 -0
- fi/api/__init__.py +1 -0
- fi/api/auth.py +137 -0
- fi/api/types.py +29 -0
- fi/cli/__init__.py +9 -0
- fi/cli/assertions/__init__.py +25 -0
- fi/cli/assertions/conditions.py +76 -0
- fi/cli/assertions/evaluator.py +286 -0
- fi/cli/assertions/exit_codes.py +20 -0
- fi/cli/assertions/parser.py +131 -0
- fi/cli/assertions/reporter.py +194 -0
- fi/cli/commands/__init__.py +9 -0
- fi/cli/commands/config.py +165 -0
- fi/cli/commands/export.py +208 -0
- fi/cli/commands/init.py +112 -0
- fi/cli/commands/list_cmd.py +213 -0
- fi/cli/commands/run.py +486 -0
- fi/cli/commands/validate.py +173 -0
- fi/cli/commands/view.py +424 -0
- fi/cli/config/__init__.py +6 -0
- fi/cli/config/defaults.py +206 -0
- fi/cli/config/loader.py +155 -0
- fi/cli/config/schema.py +174 -0
- fi/cli/main.py +78 -0
- fi/cli/output/__init__.py +6 -0
- fi/cli/output/formatters.py +106 -0
- fi/cli/output/reporters.py +46 -0
- fi/cli/storage/__init__.py +5 -0
- fi/cli/storage/run_history.py +249 -0
- fi/cli/utils/__init__.py +5 -0
- fi/cli/utils/console.py +44 -0
- fi/evals/__init__.py +131 -0
- fi/evals/autoeval/__init__.py +137 -0
- fi/evals/autoeval/analyzer.py +211 -0
- fi/evals/autoeval/config.py +244 -0
- fi/evals/autoeval/export.py +213 -0
- fi/evals/autoeval/interactive.py +283 -0
- fi/evals/autoeval/pipeline.py +625 -0
- fi/evals/autoeval/prompts.py +139 -0
- fi/evals/autoeval/recommender.py +242 -0
- fi/evals/autoeval/rules.py +589 -0
- fi/evals/autoeval/templates.py +299 -0
- fi/evals/autoeval/types.py +232 -0
- fi/evals/core/__init__.py +16 -0
- fi/evals/core/cloud_registry.py +184 -0
- fi/evals/core/engines.py +368 -0
- fi/evals/core/evaluate.py +319 -0
- fi/evals/core/judge_prompt.py +90 -0
- fi/evals/core/prompt_generator.py +83 -0
- fi/evals/core/registry.py +57 -0
- fi/evals/core/result.py +55 -0
- fi/evals/evaluator.py +721 -0
- fi/evals/execution.py +168 -0
- fi/evals/feedback/__init__.py +32 -0
- fi/evals/feedback/calibrator.py +160 -0
- fi/evals/feedback/collector.py +214 -0
- fi/evals/feedback/hooks.py +81 -0
- fi/evals/feedback/retriever.py +128 -0
- fi/evals/feedback/store.py +272 -0
- fi/evals/feedback/types.py +99 -0
- fi/evals/framework/README.md +79 -0
- fi/evals/framework/__init__.py +267 -0
- fi/evals/framework/backends/Dockerfile.eval-runner +33 -0
- fi/evals/framework/backends/__init__.py +99 -0
- fi/evals/framework/backends/_container.py +141 -0
- fi/evals/framework/backends/_utils.py +145 -0
- fi/evals/framework/backends/base.py +223 -0
- fi/evals/framework/backends/celery_backend.py +417 -0
- fi/evals/framework/backends/celery_worker.py +78 -0
- fi/evals/framework/backends/kubernetes_backend.py +665 -0
- fi/evals/framework/backends/ray_backend.py +521 -0
- fi/evals/framework/backends/temporal.py +350 -0
- fi/evals/framework/backends/temporal_worker.py +126 -0
- fi/evals/framework/backends/thread_pool.py +286 -0
- fi/evals/framework/context.py +258 -0
- fi/evals/framework/enrichment.py +306 -0
- fi/evals/framework/evals/__init__.py +68 -0
- fi/evals/framework/evals/agentic.py +399 -0
- fi/evals/framework/evals/builder.py +609 -0
- fi/evals/framework/evals/semantic.py +142 -0
- fi/evals/framework/evaluator.py +647 -0
- fi/evals/framework/evaluators/__init__.py +22 -0
- fi/evals/framework/evaluators/blocking.py +347 -0
- fi/evals/framework/evaluators/non_blocking.py +577 -0
- fi/evals/framework/propagation.py +421 -0
- fi/evals/framework/protocols.py +385 -0
- fi/evals/framework/registry.py +370 -0
- fi/evals/framework/resilience/__init__.py +150 -0
- fi/evals/framework/resilience/circuit_breaker.py +309 -0
- fi/evals/framework/resilience/degradation.py +355 -0
- fi/evals/framework/resilience/health.py +505 -0
- fi/evals/framework/resilience/rate_limiter.py +228 -0
- fi/evals/framework/resilience/retry.py +274 -0
- fi/evals/framework/resilience/types.py +288 -0
- fi/evals/framework/resilience/wrapper.py +433 -0
- fi/evals/framework/types.py +218 -0
- fi/evals/guardrails/README.md +915 -0
- fi/evals/guardrails/__init__.py +96 -0
- fi/evals/guardrails/backends/__init__.py +43 -0
- fi/evals/guardrails/backends/azure.py +361 -0
- fi/evals/guardrails/backends/base.py +88 -0
- fi/evals/guardrails/backends/generic_llm.py +163 -0
- fi/evals/guardrails/backends/granite.py +216 -0
- fi/evals/guardrails/backends/llamaguard.py +221 -0
- fi/evals/guardrails/backends/local_base.py +479 -0
- fi/evals/guardrails/backends/openai.py +365 -0
- fi/evals/guardrails/backends/qwen.py +170 -0
- fi/evals/guardrails/backends/shieldgemma.py +154 -0
- fi/evals/guardrails/backends/turing.py +235 -0
- fi/evals/guardrails/backends/vllm_client.py +321 -0
- fi/evals/guardrails/backends/wildguard.py +188 -0
- fi/evals/guardrails/base.py +888 -0
- fi/evals/guardrails/config.py +221 -0
- fi/evals/guardrails/discovery.py +243 -0
- fi/evals/guardrails/gateway.py +437 -0
- fi/evals/guardrails/registry.py +231 -0
- fi/evals/guardrails/scanners/__init__.py +127 -0
- fi/evals/guardrails/scanners/base.py +191 -0
- fi/evals/guardrails/scanners/code_injection.py +243 -0
- fi/evals/guardrails/scanners/eval_delegate.py +574 -0
- fi/evals/guardrails/scanners/invisible_chars.py +351 -0
- fi/evals/guardrails/scanners/jailbreak.py +412 -0
- fi/evals/guardrails/scanners/language.py +288 -0
- fi/evals/guardrails/scanners/pipeline.py +260 -0
- fi/evals/guardrails/scanners/regex.py +311 -0
- fi/evals/guardrails/scanners/secrets.py +274 -0
- fi/evals/guardrails/scanners/topics.py +649 -0
- fi/evals/guardrails/scanners/urls.py +341 -0
- fi/evals/guardrails/types.py +96 -0
- fi/evals/llm/__init__.py +3 -0
- fi/evals/llm/base_llm_provider.py +35 -0
- fi/evals/llm/providers/litellm.py +70 -0
- fi/evals/local/__init__.py +90 -0
- fi/evals/local/evaluator.py +690 -0
- fi/evals/local/execution_mode.py +121 -0
- fi/evals/local/llm.py +489 -0
- fi/evals/local/metrics/__init__.py +19 -0
- fi/evals/local/registry.py +360 -0
- fi/evals/manager.py +1018 -0
- fi/evals/manager_types.py +362 -0
- fi/evals/metrics/__init__.py +185 -0
- fi/evals/metrics/agents/__init__.py +74 -0
- fi/evals/metrics/agents/metrics.py +693 -0
- fi/evals/metrics/agents/report.py +36463 -0
- fi/evals/metrics/agents/types.py +160 -0
- fi/evals/metrics/base_llm_metric.py +111 -0
- fi/evals/metrics/base_metric.py +138 -0
- fi/evals/metrics/code_security/__init__.py +305 -0
- fi/evals/metrics/code_security/analyzer.py +985 -0
- fi/evals/metrics/code_security/benchmarks/__init__.py +73 -0
- fi/evals/metrics/code_security/benchmarks/builtin.py +750 -0
- fi/evals/metrics/code_security/benchmarks/loader.py +580 -0
- fi/evals/metrics/code_security/benchmarks/types.py +308 -0
- fi/evals/metrics/code_security/detectors/__init__.py +186 -0
- fi/evals/metrics/code_security/detectors/base.py +394 -0
- fi/evals/metrics/code_security/detectors/cryptography.py +345 -0
- fi/evals/metrics/code_security/detectors/injection.py +744 -0
- fi/evals/metrics/code_security/detectors/secrets.py +287 -0
- fi/evals/metrics/code_security/detectors/serialization.py +192 -0
- fi/evals/metrics/code_security/joint_metrics.py +588 -0
- fi/evals/metrics/code_security/judges/__init__.py +83 -0
- fi/evals/metrics/code_security/judges/base.py +238 -0
- fi/evals/metrics/code_security/judges/dual_judge.py +534 -0
- fi/evals/metrics/code_security/judges/llm_judge.py +301 -0
- fi/evals/metrics/code_security/judges/pattern_judge.py +515 -0
- fi/evals/metrics/code_security/metrics.py +388 -0
- fi/evals/metrics/code_security/modes/__init__.py +63 -0
- fi/evals/metrics/code_security/modes/adversarial.py +284 -0
- fi/evals/metrics/code_security/modes/autocomplete.py +198 -0
- fi/evals/metrics/code_security/modes/base.py +283 -0
- fi/evals/metrics/code_security/modes/instruct.py +253 -0
- fi/evals/metrics/code_security/modes/repair.py +230 -0
- fi/evals/metrics/code_security/reports/__init__.py +57 -0
- fi/evals/metrics/code_security/reports/generator.py +404 -0
- fi/evals/metrics/code_security/reports/leaderboard.py +509 -0
- fi/evals/metrics/code_security/types.py +534 -0
- fi/evals/metrics/function_calling/__init__.py +34 -0
- fi/evals/metrics/function_calling/metrics.py +573 -0
- fi/evals/metrics/function_calling/types.py +87 -0
- fi/evals/metrics/hallucination/__init__.py +54 -0
- fi/evals/metrics/hallucination/detector.py +149 -0
- fi/evals/metrics/hallucination/metrics.py +390 -0
- fi/evals/metrics/hallucination/nli.py +253 -0
- fi/evals/metrics/hallucination/sentinel.py +106 -0
- fi/evals/metrics/hallucination/types.py +132 -0
- fi/evals/metrics/heuristics/aggregation_metrics.py +85 -0
- fi/evals/metrics/heuristics/json_metrics.py +87 -0
- fi/evals/metrics/heuristics/similarity_metrics.py +375 -0
- fi/evals/metrics/heuristics/string_metrics.py +391 -0
- fi/evals/metrics/llm_as_judges/__init__.py +17 -0
- fi/evals/metrics/llm_as_judges/custom_judge/metric.py +112 -0
- fi/evals/metrics/llm_as_judges/custom_judge/prompts.py +26 -0
- fi/evals/metrics/llm_as_judges/types.py +48 -0
- fi/evals/metrics/rag/__init__.py +111 -0
- fi/evals/metrics/rag/advanced/__init__.py +14 -0
- fi/evals/metrics/rag/advanced/multi_hop.py +283 -0
- fi/evals/metrics/rag/advanced/source_attribution.py +344 -0
- fi/evals/metrics/rag/generation/__init__.py +17 -0
- fi/evals/metrics/rag/generation/answer_relevancy.py +176 -0
- fi/evals/metrics/rag/generation/context_utilization.py +245 -0
- fi/evals/metrics/rag/generation/faithfulness.py +241 -0
- fi/evals/metrics/rag/generation/groundedness.py +131 -0
- fi/evals/metrics/rag/rag_score.py +277 -0
- fi/evals/metrics/rag/retrieval/__init__.py +20 -0
- fi/evals/metrics/rag/retrieval/context_entity_recall.py +124 -0
- fi/evals/metrics/rag/retrieval/context_precision.py +158 -0
- fi/evals/metrics/rag/retrieval/context_recall.py +106 -0
- fi/evals/metrics/rag/retrieval/noise_sensitivity.py +163 -0
- fi/evals/metrics/rag/retrieval/ranking.py +261 -0
- fi/evals/metrics/rag/types.py +100 -0
- fi/evals/metrics/rag/utils/__init__.py +62 -0
- fi/evals/metrics/rag/utils/claims.py +189 -0
- fi/evals/metrics/rag/utils/entities.py +244 -0
- fi/evals/metrics/rag/utils/nli.py +92 -0
- fi/evals/metrics/rag/utils/similarity.py +345 -0
- fi/evals/metrics/structured/__init__.py +114 -0
- fi/evals/metrics/structured/field_completeness.py +313 -0
- fi/evals/metrics/structured/hierarchy_score.py +366 -0
- fi/evals/metrics/structured/json_validation.py +190 -0
- fi/evals/metrics/structured/schema_compliance.py +280 -0
- fi/evals/metrics/structured/structured_output_score.py +298 -0
- fi/evals/metrics/structured/types.py +108 -0
- fi/evals/metrics/structured/validators/__init__.py +30 -0
- fi/evals/metrics/structured/validators/base.py +189 -0
- fi/evals/metrics/structured/validators/json_validator.py +196 -0
- fi/evals/metrics/structured/validators/pydantic_validator.py +178 -0
- fi/evals/metrics/structured/validators/yaml_validator.py +248 -0
- fi/evals/otel/__init__.py +266 -0
- fi/evals/otel/config.py +400 -0
- fi/evals/otel/conventions.py +463 -0
- fi/evals/otel/enrichment.py +371 -0
- fi/evals/otel/instrumentors/__init__.py +140 -0
- fi/evals/otel/instrumentors/anthropic.py +517 -0
- fi/evals/otel/instrumentors/base.py +382 -0
- fi/evals/otel/instrumentors/openai.py +673 -0
- fi/evals/otel/processors/__init__.py +36 -0
- fi/evals/otel/processors/base.py +473 -0
- fi/evals/otel/processors/cost.py +445 -0
- fi/evals/otel/processors/evaluation.py +559 -0
- fi/evals/otel/processors/llm.py +462 -0
- fi/evals/otel/tracer.py +506 -0
- fi/evals/otel/types.py +232 -0
- fi/evals/otel_utils.py +23 -0
- fi/evals/protect.py +671 -0
- fi/evals/protect_input_adapter.py +154 -0
- fi/evals/streaming/__init__.py +88 -0
- fi/evals/streaming/buffer.py +213 -0
- fi/evals/streaming/evaluator.py +551 -0
- fi/evals/streaming/policy.py +307 -0
- fi/evals/streaming/scorers.py +368 -0
- fi/evals/streaming/types.py +238 -0
- fi/evals/templates.py +472 -0
- fi/evals/types.py +156 -0
- fi/opt/__init__.py +221 -0
- fi/opt/_objective_scoring.py +85 -0
- fi/opt/base/__init__.py +11 -0
- fi/opt/base/base_generator.py +33 -0
- fi/opt/base/base_mapper.py +26 -0
- fi/opt/base/base_optimizer.py +45 -0
- fi/opt/base/evaluator.py +211 -0
- fi/opt/components.py +3095 -0
- fi/opt/datamappers/__init__.py +3 -0
- fi/opt/datamappers/basic_mapper.py +40 -0
- fi/opt/deployment.py +1021 -0
- fi/opt/evidence.py +4332 -0
- fi/opt/generators/__init__.py +3 -0
- fi/opt/generators/litellm.py +66 -0
- fi/opt/integrations/__init__.py +23 -0
- fi/opt/integrations/generative_suite.py +410 -0
- fi/opt/integrations/simulate.py +1313 -0
- fi/opt/mutations.py +771 -0
- fi/opt/observability.py +4639 -0
- fi/opt/optimizer_trace.py +889 -0
- fi/opt/optimizers/__init__.py +80 -0
- fi/opt/optimizers/agent.py +331 -0
- fi/opt/optimizers/agent_bandit.py +392 -0
- fi/opt/optimizers/agent_curriculum.py +635 -0
- fi/opt/optimizers/agent_evolution.py +894 -0
- fi/opt/optimizers/agent_feedback.py +1863 -0
- fi/opt/optimizers/agent_pareto.py +547 -0
- fi/opt/optimizers/agent_social_memory.py +1113 -0
- fi/opt/optimizers/agent_tpe.py +321 -0
- fi/opt/optimizers/bayesian_search.py +449 -0
- fi/opt/optimizers/council.py +2075 -0
- fi/opt/optimizers/futureagi_replay.py +799 -0
- fi/opt/optimizers/gepa.py +322 -0
- fi/opt/optimizers/metaprompt.py +243 -0
- fi/opt/optimizers/promptwizard.py +417 -0
- fi/opt/optimizers/protegi.py +329 -0
- fi/opt/optimizers/random_search.py +224 -0
- fi/opt/research.py +518 -0
- fi/opt/simulation.py +260 -0
- fi/opt/targets.py +232 -0
- fi/opt/types.py +66 -0
- fi/opt/utils/__init__.py +4 -0
- fi/opt/utils/early_stopping.py +266 -0
- fi/opt/utils/setup_logging.py +82 -0
- fi/simulate/__init__.py +540 -0
- fi/simulate/_hashing.py +35 -0
- fi/simulate/_logging.py +10 -0
- fi/simulate/adapters.py +87 -0
- fi/simulate/agent/__init__.py +120 -0
- fi/simulate/agent/browser.py +658 -0
- fi/simulate/agent/definition.py +587 -0
- fi/simulate/agent/frameworks.py +3528 -0
- fi/simulate/agent/generic.py +8286 -0
- fi/simulate/agent/import_probe.py +227 -0
- fi/simulate/agent/memory.py +905 -0
- fi/simulate/agent/mocks.py +101 -0
- fi/simulate/agent/multi_agent.py +361 -0
- fi/simulate/agent/orchestration.py +903 -0
- fi/simulate/agent/realtime.py +665 -0
- fi/simulate/agent/wrapper.py +99 -0
- fi/simulate/agent/wrappers/__init__.py +18 -0
- fi/simulate/agent/wrappers/anthropic.py +62 -0
- fi/simulate/agent/wrappers/gemini.py +65 -0
- fi/simulate/agent/wrappers/http.py +404 -0
- fi/simulate/agent/wrappers/langchain.py +80 -0
- fi/simulate/agent/wrappers/openai.py +75 -0
- fi/simulate/agent/wrappers/websocket.py +326 -0
- fi/simulate/artifacts/__init__.py +11 -0
- fi/simulate/artifacts/manifest.py +62 -0
- fi/simulate/cli.py +20560 -0
- fi/simulate/endpoints/__init__.py +45 -0
- fi/simulate/endpoints/_http_actor.py +73 -0
- fi/simulate/endpoints/actor_sources.py +243 -0
- fi/simulate/endpoints/base.py +107 -0
- fi/simulate/endpoints/builtins.py +10 -0
- fi/simulate/endpoints/callable.py +95 -0
- fi/simulate/endpoints/http.py +76 -0
- fi/simulate/endpoints/livekit.py +138 -0
- fi/simulate/endpoints/originators.py +132 -0
- fi/simulate/endpoints/profiles.py +348 -0
- fi/simulate/endpoints/retell.py +633 -0
- fi/simulate/endpoints/vapi.py +205 -0
- fi/simulate/endpoints/websocket.py +76 -0
- fi/simulate/environment.py +33026 -0
- fi/simulate/environments/__init__.py +11 -0
- fi/simulate/environments/base.py +73 -0
- fi/simulate/environments/chat.py +697 -0
- fi/simulate/environments/voice.py +212 -0
- fi/simulate/evaluation/__init__.py +4 -0
- fi/simulate/evaluation/ai_eval.py +227 -0
- fi/simulate/evidence/__init__.py +35 -0
- fi/simulate/evidence/base.py +59 -0
- fi/simulate/evidence/caller_observed.py +50 -0
- fi/simulate/evidence/livekit_instrumentation.py +51 -0
- fi/simulate/evidence/livekit_room.py +50 -0
- fi/simulate/evidence/otel.py +49 -0
- fi/simulate/evidence/providers/__init__.py +24 -0
- fi/simulate/evidence/providers/base.py +61 -0
- fi/simulate/evidence/providers/retell.py +376 -0
- fi/simulate/evidence/providers/vapi.py +426 -0
- fi/simulate/hosted/__init__.py +32 -0
- fi/simulate/hosted/child_entrypoint.py +306 -0
- fi/simulate/hosted/job.py +150 -0
- fi/simulate/hosted/targets.py +53 -0
- fi/simulate/instrumentation/__init__.py +5 -0
- fi/simulate/instrumentation/livekit/__init__.py +122 -0
- fi/simulate/manifest.py +1033 -0
- fi/simulate/matrix_cli.py +165 -0
- fi/simulate/realtime/__init__.py +40 -0
- fi/simulate/realtime/events.py +107 -0
- fi/simulate/realtime/media.py +61 -0
- fi/simulate/realtime/session.py +91 -0
- fi/simulate/recording/__init__.py +5 -0
- fi/simulate/recording/room_recorder.py +326 -0
- fi/simulate/registry.py +185 -0
- fi/simulate/results/__init__.py +9 -0
- fi/simulate/results/base.py +18 -0
- fi/simulate/results/filesystem.py +71 -0
- fi/simulate/results/futureagi.py +1340 -0
- fi/simulate/runtime/__init__.py +85 -0
- fi/simulate/runtime/capabilities.py +40 -0
- fi/simulate/runtime/events.py +63 -0
- fi/simulate/runtime/failures.py +25 -0
- fi/simulate/runtime/ids.py +34 -0
- fi/simulate/runtime/plan.py +70 -0
- fi/simulate/runtime/planner.py +102 -0
- fi/simulate/runtime/report.py +174 -0
- fi/simulate/runtime/run.py +75 -0
- fi/simulate/runtime/runner.py +333 -0
- fi/simulate/runtime/spec.py +186 -0
- fi/simulate/simulation/__init__.py +30 -0
- fi/simulate/simulation/behavior_policy.py +425 -0
- fi/simulate/simulation/bridge/__init__.py +9 -0
- fi/simulate/simulation/bridge/audio.py +29 -0
- fi/simulate/simulation/bridge/connector.py +46 -0
- fi/simulate/simulation/bridge/livekit.py +252 -0
- fi/simulate/simulation/bridge/retell.py +188 -0
- fi/simulate/simulation/bridge/vapi.py +177 -0
- fi/simulate/simulation/contract.py +419 -0
- fi/simulate/simulation/engines/__init__.py +12 -0
- fi/simulate/simulation/engines/base.py +21 -0
- fi/simulate/simulation/engines/cloud.py +517 -0
- fi/simulate/simulation/engines/livekit.py +4167 -0
- fi/simulate/simulation/engines/local_text.py +89 -0
- fi/simulate/simulation/fidelity.py +374 -0
- fi/simulate/simulation/gemini_tts_stream.py +110 -0
- fi/simulate/simulation/generator.py +91 -0
- fi/simulate/simulation/goal_machine.py +185 -0
- fi/simulate/simulation/livekit_models.py +467 -0
- fi/simulate/simulation/matrix.py +170 -0
- fi/simulate/simulation/models.py +279 -0
- fi/simulate/simulation/runner.py +153 -0
- fi/simulate/simulation/synthetic.py +880 -0
- fi/simulate/simulation/voice_prompt.py +502 -0
- fi/simulate/simulator/__init__.py +55 -0
- fi/simulate/simulator/builtins.py +53 -0
- fi/simulate/suite.py +1288 -0
- fi/simulate/utils/routes.py +164 -0
- fi/simulate/voice.py +225 -0
- fi/simulate/voice_cli.py +182 -0
- fi/utils/__init__.py +1 -0
- fi/utils/constants.py +14 -0
- fi/utils/errors.py +200 -0
- fi/utils/executor.py +26 -0
- fi/utils/routes.py +119 -0
- fi/utils/utils.py +17 -0
|
@@ -0,0 +1,1831 @@
|
|
|
1
|
+
"""Deterministic producer for hosted ``EnvironmentBundleV2`` directories.
|
|
2
|
+
|
|
3
|
+
This module is deliberately a compiler, not a second execution engine. It converts the
|
|
4
|
+
packaging already present in a submitted repository into the process vocabulary consumed by the
|
|
5
|
+
Daytona guest, adds the harness-owned world database, adopts frozen scenario artifacts, seals the
|
|
6
|
+
result, and runs the guest's exact preflight before publishing it.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
import ast
|
|
13
|
+
import hashlib
|
|
14
|
+
import json
|
|
15
|
+
import logging
|
|
16
|
+
import re
|
|
17
|
+
import shutil
|
|
18
|
+
import sqlite3
|
|
19
|
+
import tempfile
|
|
20
|
+
from dataclasses import dataclass
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import Any
|
|
23
|
+
|
|
24
|
+
import yaml
|
|
25
|
+
|
|
26
|
+
from .bundle import CapabilityProtocol
|
|
27
|
+
from .catalogue import CATALOGUE
|
|
28
|
+
from .bundle_v2 import (
|
|
29
|
+
BUNDLE_V2_MANIFEST,
|
|
30
|
+
BUNDLE_V2_SCHEMA_VERSION,
|
|
31
|
+
BaselineStrategy,
|
|
32
|
+
BundleFileV2,
|
|
33
|
+
BundleProvenanceV2,
|
|
34
|
+
BundleRuntimeV2,
|
|
35
|
+
CapabilityV2,
|
|
36
|
+
EnvironmentBundleV2,
|
|
37
|
+
EvidenceSeam,
|
|
38
|
+
ManagedEngine,
|
|
39
|
+
ManagedProcess,
|
|
40
|
+
ProcessUser,
|
|
41
|
+
ReadinessProbeV2,
|
|
42
|
+
RuntimeKindV2,
|
|
43
|
+
SecretPurpose,
|
|
44
|
+
Seed,
|
|
45
|
+
Sentinel,
|
|
46
|
+
SourceProcess,
|
|
47
|
+
StartedCheck,
|
|
48
|
+
StoreBaseline,
|
|
49
|
+
StoreEntry,
|
|
50
|
+
compute_inputs_digest,
|
|
51
|
+
load_bundle_v2,
|
|
52
|
+
seal_bundle_v2,
|
|
53
|
+
)
|
|
54
|
+
from .contract import ToolEntry
|
|
55
|
+
from .credentials import discover_credentials
|
|
56
|
+
from .job import HarnessJob
|
|
57
|
+
from .job import ProviderExecutionMode, SourceKind
|
|
58
|
+
from .process_preflight import preflight_bundle
|
|
59
|
+
from .provision import source_fingerprint
|
|
60
|
+
from .provider_lifecycle import ProviderRepositoryManifest, load_provider_manifest
|
|
61
|
+
from .provider_import import ProviderImportSpec
|
|
62
|
+
from .world.tools import _binding
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class BundleAuthorError(RuntimeError):
|
|
66
|
+
"""A source cannot be compiled into an honest hosted process bundle."""
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
logger = logging.getLogger(__name__)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass(frozen=True)
|
|
73
|
+
class EnvironmentPlanV2:
|
|
74
|
+
packaging: str
|
|
75
|
+
control_service: str | None
|
|
76
|
+
processes: tuple[ManagedProcess | SourceProcess, ...]
|
|
77
|
+
capabilities: dict[str, CapabilityV2]
|
|
78
|
+
readiness: tuple[ReadinessProbeV2, ...]
|
|
79
|
+
|
|
80
|
+
def __post_init__(self) -> None:
|
|
81
|
+
names = [process.name for process in self.processes]
|
|
82
|
+
if len(names) != len(set(names)):
|
|
83
|
+
raise BundleAuthorError("environment_plan_process_names_not_unique")
|
|
84
|
+
if self.control_service is not None and self.control_service not in names:
|
|
85
|
+
raise BundleAuthorError("environment_plan_control_service_missing")
|
|
86
|
+
known = set(names)
|
|
87
|
+
for process in self.processes:
|
|
88
|
+
missing = sorted(set(process.depends_on) - known)
|
|
89
|
+
if missing:
|
|
90
|
+
raise BundleAuthorError(
|
|
91
|
+
f"environment_plan_dependency_missing: {process.name}: {', '.join(missing)}"
|
|
92
|
+
)
|
|
93
|
+
for slug, capability in self.capabilities.items():
|
|
94
|
+
if capability.service not in known:
|
|
95
|
+
raise BundleAuthorError(
|
|
96
|
+
f"environment_plan_capability_service_missing: {slug}: {capability.service}"
|
|
97
|
+
)
|
|
98
|
+
missing_probes = sorted(
|
|
99
|
+
{
|
|
100
|
+
probe.capability
|
|
101
|
+
for probe in self.readiness
|
|
102
|
+
if probe.capability not in self.capabilities
|
|
103
|
+
}
|
|
104
|
+
)
|
|
105
|
+
if missing_probes:
|
|
106
|
+
raise BundleAuthorError(
|
|
107
|
+
"environment_plan_readiness_capability_missing: "
|
|
108
|
+
+ ", ".join(missing_probes)
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
_COMPOSE_NAMES = (
|
|
113
|
+
"compose.yml",
|
|
114
|
+
"compose.yaml",
|
|
115
|
+
"docker-compose.yml",
|
|
116
|
+
"docker-compose.yaml",
|
|
117
|
+
)
|
|
118
|
+
_IGNORED_ARTIFACT_PARTS = {".git", ".venv", "__pycache__", "node_modules"}
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _sql_literal(value: Any) -> str:
|
|
122
|
+
if value is None:
|
|
123
|
+
return "NULL"
|
|
124
|
+
if isinstance(value, bool):
|
|
125
|
+
return "TRUE" if value else "FALSE"
|
|
126
|
+
if isinstance(value, (int, float)):
|
|
127
|
+
return str(value)
|
|
128
|
+
if isinstance(value, (dict, list)):
|
|
129
|
+
value = json.dumps(value, sort_keys=True)
|
|
130
|
+
return "'" + str(value).replace("'", "''") + "'"
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _identifier(value: str) -> str:
|
|
134
|
+
return '"' + value.replace('"', '""') + '"'
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _json_type(values: list[Any]) -> str:
|
|
138
|
+
present = [value for value in values if value is not None]
|
|
139
|
+
if present and all(isinstance(value, bool) for value in present):
|
|
140
|
+
return "boolean"
|
|
141
|
+
if present and all(
|
|
142
|
+
isinstance(value, int) and not isinstance(value, bool) for value in present
|
|
143
|
+
):
|
|
144
|
+
return "bigint"
|
|
145
|
+
if present and all(
|
|
146
|
+
isinstance(value, (int, float)) and not isinstance(value, bool)
|
|
147
|
+
for value in present
|
|
148
|
+
):
|
|
149
|
+
return "double precision"
|
|
150
|
+
if present and all(isinstance(value, (dict, list)) for value in present):
|
|
151
|
+
return "jsonb"
|
|
152
|
+
return "text"
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _constraint_checked_seed_sql(statements: list[str]) -> str:
|
|
156
|
+
"""Load generated rows in dependency order without disabling source constraints.
|
|
157
|
+
|
|
158
|
+
Retry only foreign-key failures after other rows have been inserted. Each failed
|
|
159
|
+
insert rolls back in its PL/pgSQL subtransaction. A pass with no progress rejects
|
|
160
|
+
missing references/cycles instead of silently producing an invalid world.
|
|
161
|
+
"""
|
|
162
|
+
if not statements:
|
|
163
|
+
return ""
|
|
164
|
+
commands = ",\n".join(_sql_literal(statement) for statement in statements)
|
|
165
|
+
body = (
|
|
166
|
+
"DECLARE\n"
|
|
167
|
+
f" pending text[] := ARRAY[{commands}];\n"
|
|
168
|
+
" remaining text[]; command text; progress boolean; failure_detail text;\n"
|
|
169
|
+
"BEGIN\n"
|
|
170
|
+
" WHILE cardinality(pending) > 0 LOOP\n"
|
|
171
|
+
" remaining := ARRAY[]::text[]; progress := false;\n"
|
|
172
|
+
" FOREACH command IN ARRAY pending LOOP\n"
|
|
173
|
+
" BEGIN\n"
|
|
174
|
+
" EXECUTE command; progress := true;\n"
|
|
175
|
+
" EXCEPTION WHEN foreign_key_violation THEN\n"
|
|
176
|
+
" GET STACKED DIAGNOSTICS failure_detail = MESSAGE_TEXT;\n"
|
|
177
|
+
" remaining := array_append(remaining, command);\n"
|
|
178
|
+
" END;\n"
|
|
179
|
+
" END LOOP;\n"
|
|
180
|
+
" IF cardinality(remaining) > 0 AND NOT progress THEN\n"
|
|
181
|
+
" RAISE EXCEPTION 'seed_dependency_unresolved: % statements; %', "
|
|
182
|
+
"cardinality(remaining), failure_detail USING ERRCODE = '23503';\n"
|
|
183
|
+
" END IF;\n"
|
|
184
|
+
" pending := remaining;\n"
|
|
185
|
+
" END LOOP;\n"
|
|
186
|
+
"END"
|
|
187
|
+
)
|
|
188
|
+
return "DO " + _sql_literal(body) + ";\n"
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _collections_sql(path: Path, *, include_schema: bool = True) -> str:
|
|
192
|
+
body = json.loads(path.read_text(encoding="utf-8"))
|
|
193
|
+
if not isinstance(body, dict):
|
|
194
|
+
raise BundleAuthorError("collections_invalid: expected an object")
|
|
195
|
+
statements: list[str] = []
|
|
196
|
+
for table, raw_rows in body.items():
|
|
197
|
+
rows = raw_rows if isinstance(raw_rows, list) else []
|
|
198
|
+
records = [row for row in rows if isinstance(row, dict)]
|
|
199
|
+
columns = sorted({str(column) for row in records for column in row})
|
|
200
|
+
if not columns:
|
|
201
|
+
columns = ["id"]
|
|
202
|
+
definitions = [
|
|
203
|
+
f"{_identifier(column)} {_json_type([row.get(column) for row in records])}"
|
|
204
|
+
for column in columns
|
|
205
|
+
]
|
|
206
|
+
if include_schema:
|
|
207
|
+
statements.append(
|
|
208
|
+
f"CREATE TABLE IF NOT EXISTS {_identifier(str(table))} "
|
|
209
|
+
f"({', '.join(definitions)});"
|
|
210
|
+
)
|
|
211
|
+
for row in records:
|
|
212
|
+
values = ", ".join(_sql_literal(row.get(column)) for column in columns)
|
|
213
|
+
names = ", ".join(_identifier(column) for column in columns)
|
|
214
|
+
statements.append(
|
|
215
|
+
f"INSERT INTO {_identifier(str(table))} ({names}) VALUES ({values});"
|
|
216
|
+
)
|
|
217
|
+
if not include_schema:
|
|
218
|
+
return _constraint_checked_seed_sql(statements)
|
|
219
|
+
return "\n".join(statements) + "\n"
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _sqlite_type(declared: str) -> str:
|
|
223
|
+
normalized = declared.upper()
|
|
224
|
+
if "BOOL" in normalized:
|
|
225
|
+
return "boolean"
|
|
226
|
+
if "INT" in normalized:
|
|
227
|
+
return "bigint"
|
|
228
|
+
if any(mark in normalized for mark in ("REAL", "FLOA", "DOUB")):
|
|
229
|
+
return "double precision"
|
|
230
|
+
if any(mark in normalized for mark in ("NUMERIC", "DECIMAL")):
|
|
231
|
+
return "numeric"
|
|
232
|
+
if "BLOB" in normalized:
|
|
233
|
+
return "bytea"
|
|
234
|
+
return "text"
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _sqlite_json_type(values: list[Any], sql_type: str) -> str:
|
|
238
|
+
"""Preserve structured SQLite TEXT values when moving a world to Postgres.
|
|
239
|
+
|
|
240
|
+
SQLite has no native JSON/array storage class, so generated worlds store lists and
|
|
241
|
+
objects as JSON text. Treating those columns as Postgres ``text`` changes the tool
|
|
242
|
+
contract (``[]`` becomes the string ``"[]"``). Only promote a column when every
|
|
243
|
+
non-null value is a JSON object or array; ordinary strings remain text.
|
|
244
|
+
"""
|
|
245
|
+
if sql_type != "text":
|
|
246
|
+
return sql_type
|
|
247
|
+
present = [value for value in values if value is not None]
|
|
248
|
+
if not present or not all(isinstance(value, str) for value in present):
|
|
249
|
+
return sql_type
|
|
250
|
+
try:
|
|
251
|
+
decoded = [json.loads(value) for value in present]
|
|
252
|
+
except (TypeError, ValueError, json.JSONDecodeError):
|
|
253
|
+
return sql_type
|
|
254
|
+
return (
|
|
255
|
+
"jsonb"
|
|
256
|
+
if all(isinstance(value, (dict, list)) for value in decoded)
|
|
257
|
+
else sql_type
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def _postgres_text_array_literal(value: Any) -> str:
|
|
262
|
+
decoded = json.loads(value) if isinstance(value, str) else value
|
|
263
|
+
if not isinstance(decoded, list) or any(
|
|
264
|
+
isinstance(item, (dict, list)) for item in decoded
|
|
265
|
+
):
|
|
266
|
+
raise BundleAuthorError("sqlite_text_array_invalid: expected scalar JSON array")
|
|
267
|
+
escaped = [
|
|
268
|
+
'"' + str(item).replace("\\", "\\\\").replace('"', '\\"') + '"'
|
|
269
|
+
for item in decoded
|
|
270
|
+
]
|
|
271
|
+
return "{" + ",".join(escaped) + "}"
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _sqlite_value(value: Any, sql_type: str) -> Any:
|
|
275
|
+
if value is not None and sql_type == "boolean":
|
|
276
|
+
return bool(value)
|
|
277
|
+
if value is not None and sql_type == "jsonb" and isinstance(value, str):
|
|
278
|
+
return json.loads(value)
|
|
279
|
+
if value is not None and sql_type == "text[]":
|
|
280
|
+
return _postgres_text_array_literal(value)
|
|
281
|
+
return value
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _contract_column_declarations(
|
|
285
|
+
contract: dict[str, Any],
|
|
286
|
+
) -> dict[tuple[str, str], str]:
|
|
287
|
+
"""Return authored SQL declarations keyed by table and column.
|
|
288
|
+
|
|
289
|
+
SQLite affinity erases semantic types (notably BOOLEAN -> INTEGER and
|
|
290
|
+
TIMESTAMPTZ -> TEXT). The contract is the authoritative schema description,
|
|
291
|
+
so retain its safe type/default hints while still deriving keys and indexes
|
|
292
|
+
from the executable SQLite world.
|
|
293
|
+
"""
|
|
294
|
+
|
|
295
|
+
schema = contract.get("data_schema")
|
|
296
|
+
if not isinstance(schema, dict):
|
|
297
|
+
return {}
|
|
298
|
+
return {
|
|
299
|
+
(str(table), str(column)): str(declaration).strip()
|
|
300
|
+
for table, raw_columns in schema.items()
|
|
301
|
+
if isinstance(raw_columns, dict)
|
|
302
|
+
for column, declaration in raw_columns.items()
|
|
303
|
+
if str(declaration).strip()
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def _contract_sql_type(declaration: str) -> str | None:
|
|
308
|
+
normalized = declaration.strip().upper()
|
|
309
|
+
patterns = (
|
|
310
|
+
(r"^BOOLEAN\b", "boolean"),
|
|
311
|
+
(r"^(?:BIGINT|INTEGER|INT|SMALLINT)\b", "bigint"),
|
|
312
|
+
(r"^(?:DOUBLE PRECISION|REAL|FLOAT)\b", "double precision"),
|
|
313
|
+
(
|
|
314
|
+
r"^(?:NUMERIC|DECIMAL)(?:\s*\(\s*\d+\s*(?:,\s*\d+\s*)?\))?\b",
|
|
315
|
+
"numeric",
|
|
316
|
+
),
|
|
317
|
+
(r"^TIMESTAMPTZ\b", "timestamptz"),
|
|
318
|
+
(r"^TIMESTAMP\b", "timestamp"),
|
|
319
|
+
(r"^JSONB?\b", "jsonb"),
|
|
320
|
+
(r"^TEXT\s*\[\s*\]", "text[]"),
|
|
321
|
+
(r"^(?:TEXT|VARCHAR|CHAR)\b", "text"),
|
|
322
|
+
)
|
|
323
|
+
for pattern, sql_type in patterns:
|
|
324
|
+
if re.match(pattern, normalized):
|
|
325
|
+
return sql_type
|
|
326
|
+
return None
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def _safe_sql_default(raw: Any, *, sql_type: str) -> str | None:
|
|
330
|
+
"""Translate a small, non-executable default grammar to PostgreSQL."""
|
|
331
|
+
|
|
332
|
+
if raw is None:
|
|
333
|
+
return None
|
|
334
|
+
value = str(raw).strip()
|
|
335
|
+
while len(value) >= 2 and value[0] == "(" and value[-1] == ")":
|
|
336
|
+
value = value[1:-1].strip()
|
|
337
|
+
upper = value.upper()
|
|
338
|
+
if sql_type == "text[]" and re.fullmatch(r"'(?:[^']|'')*'", value):
|
|
339
|
+
inner = value[1:-1].replace("''", "'")
|
|
340
|
+
if inner.startswith("["):
|
|
341
|
+
return _sql_literal(_postgres_text_array_literal(inner))
|
|
342
|
+
if sql_type == "boolean" and upper in {"TRUE", "FALSE", "1", "0"}:
|
|
343
|
+
return "TRUE" if upper in {"TRUE", "1"} else "FALSE"
|
|
344
|
+
if upper in {"CURRENT_TIMESTAMP", "NOW()"}:
|
|
345
|
+
return "now()"
|
|
346
|
+
if re.fullmatch(r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)", value):
|
|
347
|
+
return value
|
|
348
|
+
if re.fullmatch(r"'(?:[^']|'')*'", value):
|
|
349
|
+
return value
|
|
350
|
+
return None
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def _contract_default(declaration: str, *, sql_type: str) -> str | None:
|
|
354
|
+
match = re.search(
|
|
355
|
+
r"\bDEFAULT\s+(NOW\(\)|CURRENT_TIMESTAMP|TRUE|FALSE|"
|
|
356
|
+
r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)|'(?:[^']|'')*')",
|
|
357
|
+
declaration,
|
|
358
|
+
flags=re.IGNORECASE,
|
|
359
|
+
)
|
|
360
|
+
return _safe_sql_default(match.group(1), sql_type=sql_type) if match else None
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def _sqlite_sql(
|
|
364
|
+
path: Path,
|
|
365
|
+
*,
|
|
366
|
+
contract_declarations: dict[tuple[str, str], str] | None = None,
|
|
367
|
+
include_schema: bool = True,
|
|
368
|
+
) -> str:
|
|
369
|
+
statements: list[str] = []
|
|
370
|
+
contract_declarations = contract_declarations or {}
|
|
371
|
+
connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
|
|
372
|
+
connection.row_factory = sqlite3.Row
|
|
373
|
+
try:
|
|
374
|
+
tables = [
|
|
375
|
+
row[0]
|
|
376
|
+
for row in connection.execute(
|
|
377
|
+
"SELECT name FROM sqlite_master WHERE type='table' "
|
|
378
|
+
"AND name NOT LIKE 'sqlite_%' ORDER BY name"
|
|
379
|
+
)
|
|
380
|
+
]
|
|
381
|
+
for table in tables:
|
|
382
|
+
info = list(connection.execute(f"PRAGMA table_info({_identifier(table)})"))
|
|
383
|
+
selected = connection.execute(
|
|
384
|
+
f"SELECT * FROM {_identifier(table)}"
|
|
385
|
+
).fetchall()
|
|
386
|
+
definitions: list[str] = []
|
|
387
|
+
columns: list[str] = []
|
|
388
|
+
column_types: list[str] = []
|
|
389
|
+
primary_key_columns = [
|
|
390
|
+
str(row[1])
|
|
391
|
+
for row in sorted(info, key=lambda item: int(item[5] or 0))
|
|
392
|
+
if int(row[5] or 0)
|
|
393
|
+
]
|
|
394
|
+
for row in info:
|
|
395
|
+
name = str(row[1])
|
|
396
|
+
declaration = contract_declarations.get((table, name), "")
|
|
397
|
+
sql_type = _contract_sql_type(declaration) or _sqlite_json_type(
|
|
398
|
+
[record[name] for record in selected],
|
|
399
|
+
_sqlite_type(str(row[2] or "")),
|
|
400
|
+
)
|
|
401
|
+
# SQLite reports the ordinal of every column in a composite key. Marking each
|
|
402
|
+
# such column as an inline PostgreSQL primary key creates multiple conflicting
|
|
403
|
+
# constraints. Only a single-column key is emitted inline; composite keys are
|
|
404
|
+
# emitted once as a table constraint below.
|
|
405
|
+
suffix = (
|
|
406
|
+
" PRIMARY KEY"
|
|
407
|
+
if int(row[5] or 0) and len(primary_key_columns) == 1
|
|
408
|
+
else ""
|
|
409
|
+
)
|
|
410
|
+
if int(row[3] or 0) and not int(row[5] or 0):
|
|
411
|
+
suffix += " NOT NULL"
|
|
412
|
+
default = _safe_sql_default(row[4], sql_type=sql_type)
|
|
413
|
+
if default is None and declaration:
|
|
414
|
+
default = _contract_default(declaration, sql_type=sql_type)
|
|
415
|
+
if default is not None:
|
|
416
|
+
suffix += f" DEFAULT {default}"
|
|
417
|
+
definitions.append(f"{_identifier(name)} {sql_type}{suffix}")
|
|
418
|
+
columns.append(name)
|
|
419
|
+
column_types.append(sql_type)
|
|
420
|
+
if len(primary_key_columns) > 1:
|
|
421
|
+
definitions.append(
|
|
422
|
+
"PRIMARY KEY ("
|
|
423
|
+
+ ", ".join(_identifier(column) for column in primary_key_columns)
|
|
424
|
+
+ ")"
|
|
425
|
+
)
|
|
426
|
+
# ``PRAGMA table_info`` exposes primary keys but not UNIQUE constraints. Dropping
|
|
427
|
+
# those constraints during the SQLite -> Postgres compilation changes executable
|
|
428
|
+
# tool semantics: a source statement such as ``ON CONFLICT (phone)`` becomes invalid
|
|
429
|
+
# even though it worked against the authored world. Preserve every concrete,
|
|
430
|
+
# non-partial unique index except the primary-key index already represented above.
|
|
431
|
+
for index in connection.execute(f"PRAGMA index_list({_identifier(table)})"):
|
|
432
|
+
unique = bool(index[2])
|
|
433
|
+
origin = str(index[3] or "")
|
|
434
|
+
partial = bool(index[4])
|
|
435
|
+
if not unique or origin == "pk" or partial:
|
|
436
|
+
continue
|
|
437
|
+
index_name = str(index[1])
|
|
438
|
+
index_columns = [
|
|
439
|
+
str(column[2])
|
|
440
|
+
for column in connection.execute(
|
|
441
|
+
f"PRAGMA index_info({_identifier(index_name)})"
|
|
442
|
+
)
|
|
443
|
+
if column[2] is not None
|
|
444
|
+
]
|
|
445
|
+
if index_columns:
|
|
446
|
+
definitions.append(
|
|
447
|
+
"UNIQUE ("
|
|
448
|
+
+ ", ".join(_identifier(column) for column in index_columns)
|
|
449
|
+
+ ")"
|
|
450
|
+
)
|
|
451
|
+
if include_schema:
|
|
452
|
+
statements.append(
|
|
453
|
+
f"CREATE TABLE IF NOT EXISTS {_identifier(table)} "
|
|
454
|
+
f"({', '.join(definitions)});"
|
|
455
|
+
)
|
|
456
|
+
for record in selected:
|
|
457
|
+
# An authored SQLite world cannot retain the distinction between an
|
|
458
|
+
# omitted source column and an explicitly stored NULL: every row is
|
|
459
|
+
# read back with every column present. When the real source schema is
|
|
460
|
+
# adopted below, sending those NULLs explicitly suppresses PostgreSQL
|
|
461
|
+
# defaults and can violate source NOT NULL constraints. Treat NULL in
|
|
462
|
+
# the generated world as "unspecified" and omit it from this row. On
|
|
463
|
+
# PostgreSQL that produces exactly the source-schema behaviour: its
|
|
464
|
+
# default is applied when one exists, otherwise the value remains NULL.
|
|
465
|
+
populated = [
|
|
466
|
+
(column, sql_type)
|
|
467
|
+
for column, sql_type in zip(columns, column_types, strict=True)
|
|
468
|
+
if record[column] is not None
|
|
469
|
+
]
|
|
470
|
+
if not populated:
|
|
471
|
+
statements.append(
|
|
472
|
+
f"INSERT INTO {_identifier(table)} DEFAULT VALUES;"
|
|
473
|
+
)
|
|
474
|
+
continue
|
|
475
|
+
names = ", ".join(
|
|
476
|
+
_identifier(column) for column, _sql_type in populated
|
|
477
|
+
)
|
|
478
|
+
values = ", ".join(
|
|
479
|
+
_sql_literal(_sqlite_value(record[column], sql_type))
|
|
480
|
+
for column, sql_type in populated
|
|
481
|
+
)
|
|
482
|
+
statements.append(
|
|
483
|
+
f"INSERT INTO {_identifier(table)} ({names}) VALUES ({values});"
|
|
484
|
+
)
|
|
485
|
+
finally:
|
|
486
|
+
connection.close()
|
|
487
|
+
if not include_schema:
|
|
488
|
+
return _constraint_checked_seed_sql(statements)
|
|
489
|
+
return "\n".join(statements) + "\n"
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
def _store_json_seed_sql(path: Path) -> str:
|
|
493
|
+
"""Restore rows exported by the existing ALK world store into adopted PostgreSQL tables.
|
|
494
|
+
|
|
495
|
+
``schema.sql`` is deliberately schema-only in several established authoring outputs. The
|
|
496
|
+
matching ``store.json`` carries the frozen rows under ``rows``. PostgreSQL's
|
|
497
|
+
``jsonb_populate_recordset`` performs the type-aware conversion (including arrays, numerics,
|
|
498
|
+
timestamps and JSON) against the adopted table definition instead of guessing SQL types.
|
|
499
|
+
"""
|
|
500
|
+
body = json.loads(path.read_text(encoding="utf-8"))
|
|
501
|
+
rows = body.get("rows") if isinstance(body, dict) else None
|
|
502
|
+
if not isinstance(rows, dict):
|
|
503
|
+
raise BundleAuthorError("store_invalid: expected an object with a rows object")
|
|
504
|
+
statements: list[str] = []
|
|
505
|
+
for table in sorted(rows):
|
|
506
|
+
raw_rows = rows[table]
|
|
507
|
+
if not isinstance(raw_rows, list):
|
|
508
|
+
raise BundleAuthorError(f"store_invalid: rows.{table} must be an array")
|
|
509
|
+
records = [row for row in raw_rows if isinstance(row, dict)]
|
|
510
|
+
if len(records) != len(raw_rows):
|
|
511
|
+
raise BundleAuthorError(
|
|
512
|
+
f"store_invalid: rows.{table} contains a non-object row"
|
|
513
|
+
)
|
|
514
|
+
if not records:
|
|
515
|
+
continue
|
|
516
|
+
payload = json.dumps(records, sort_keys=True, separators=(",", ":"))
|
|
517
|
+
statements.append(
|
|
518
|
+
f"INSERT INTO public.{_identifier(str(table))} "
|
|
519
|
+
f"SELECT * FROM jsonb_populate_recordset(NULL::public.{_identifier(str(table))}, "
|
|
520
|
+
f"{_sql_literal(payload)}::jsonb);"
|
|
521
|
+
)
|
|
522
|
+
if not statements:
|
|
523
|
+
return ""
|
|
524
|
+
return _constraint_checked_seed_sql(statements)
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
def _contained_source_path(source: Path, raw_path: str) -> Path | None:
|
|
528
|
+
"""Resolve a submitted path without ever following it outside the checkout."""
|
|
529
|
+
|
|
530
|
+
try:
|
|
531
|
+
candidate = (source / raw_path).resolve()
|
|
532
|
+
root = source.resolve()
|
|
533
|
+
except (OSError, RuntimeError, ValueError):
|
|
534
|
+
return None
|
|
535
|
+
if not candidate.is_relative_to(root) or not candidate.exists():
|
|
536
|
+
return None
|
|
537
|
+
return candidate
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def _schema_like(path: Path) -> bool:
|
|
541
|
+
name = path.name.lower()
|
|
542
|
+
return path.suffix.lower() == ".sql" and any(
|
|
543
|
+
marker in name for marker in ("schema", "migration", "migrate", "ddl")
|
|
544
|
+
)
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
def _compose_source_schema_paths(source: Path) -> list[Path]:
|
|
548
|
+
"""Discover repository-owned DDL mounted into a database init directory.
|
|
549
|
+
|
|
550
|
+
Compose is only evidence here; it is never executed by the hosted guest. Restricting this
|
|
551
|
+
to schema/migration-named SQL files avoids adopting fixture/seed data, which must come from
|
|
552
|
+
the freshly authored scenario world instead.
|
|
553
|
+
"""
|
|
554
|
+
|
|
555
|
+
compose_path = _compose_path(source)
|
|
556
|
+
if compose_path is None:
|
|
557
|
+
return []
|
|
558
|
+
compose = _load_compose(compose_path)
|
|
559
|
+
discovered: list[Path] = []
|
|
560
|
+
for service in compose["services"].values():
|
|
561
|
+
if not isinstance(service, dict):
|
|
562
|
+
continue
|
|
563
|
+
for volume in service.get("volumes") or []:
|
|
564
|
+
raw_source = ""
|
|
565
|
+
target = ""
|
|
566
|
+
if isinstance(volume, str):
|
|
567
|
+
pieces = volume.split(":")
|
|
568
|
+
if len(pieces) >= 2:
|
|
569
|
+
raw_source, target = pieces[0], pieces[1]
|
|
570
|
+
elif isinstance(volume, dict):
|
|
571
|
+
raw_source = str(volume.get("source") or "")
|
|
572
|
+
target = str(volume.get("target") or "")
|
|
573
|
+
if "docker-entrypoint-initdb.d" not in target or not raw_source:
|
|
574
|
+
continue
|
|
575
|
+
path = _contained_source_path(source, raw_source)
|
|
576
|
+
if path is None:
|
|
577
|
+
continue
|
|
578
|
+
if path.is_file() and _schema_like(path):
|
|
579
|
+
discovered.append(path)
|
|
580
|
+
elif path.is_dir():
|
|
581
|
+
discovered.extend(
|
|
582
|
+
candidate
|
|
583
|
+
for candidate in sorted(path.rglob("*.sql"))
|
|
584
|
+
if candidate.is_file() and _schema_like(candidate)
|
|
585
|
+
)
|
|
586
|
+
return discovered
|
|
587
|
+
|
|
588
|
+
|
|
589
|
+
def _source_schema_paths(
|
|
590
|
+
source: Path, *, contract: dict[str, Any] | None = None
|
|
591
|
+
) -> list[Path]:
|
|
592
|
+
"""Return deterministic source-owned schema artifacts in precedence order.
|
|
593
|
+
|
|
594
|
+
Executable repository evidence is authoritative. The generated contract may point at that
|
|
595
|
+
evidence, but it cannot replace or truncate it. This is intentionally independent of model
|
|
596
|
+
output so two fresh authoring runs compile the same source schema.
|
|
597
|
+
"""
|
|
598
|
+
|
|
599
|
+
candidates = _compose_source_schema_paths(source)
|
|
600
|
+
for conventional in ("db/schema.sql", "schema.sql"):
|
|
601
|
+
path = _contained_source_path(source, conventional)
|
|
602
|
+
if path is not None and path.is_file():
|
|
603
|
+
candidates.append(path)
|
|
604
|
+
|
|
605
|
+
store = (contract or {}).get("data_store")
|
|
606
|
+
declared = (
|
|
607
|
+
str(store.get("schema_from") or "").strip() if isinstance(store, dict) else ""
|
|
608
|
+
)
|
|
609
|
+
if declared:
|
|
610
|
+
path = _contained_source_path(source, declared)
|
|
611
|
+
if path is not None:
|
|
612
|
+
if path.is_file() and path.suffix.lower() == ".sql":
|
|
613
|
+
candidates.append(path)
|
|
614
|
+
elif path.is_dir():
|
|
615
|
+
candidates.extend(
|
|
616
|
+
candidate
|
|
617
|
+
for candidate in sorted(path.rglob("*.sql"))
|
|
618
|
+
if candidate.is_file() and _schema_like(candidate)
|
|
619
|
+
)
|
|
620
|
+
elif declared.lower().endswith(".sql") and not candidates:
|
|
621
|
+
raise BundleAuthorError(f"source_schema_missing: {declared}")
|
|
622
|
+
|
|
623
|
+
unique: dict[str, Path] = {}
|
|
624
|
+
for path in candidates:
|
|
625
|
+
relative = path.relative_to(source.resolve()).as_posix()
|
|
626
|
+
unique.setdefault(relative, path)
|
|
627
|
+
return [unique[key] for key in sorted(unique)]
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
def _adopted_seed_sql(
|
|
631
|
+
authoring: Path,
|
|
632
|
+
*,
|
|
633
|
+
source: Path | None = None,
|
|
634
|
+
contract: dict[str, Any] | None = None,
|
|
635
|
+
) -> tuple[str, list[str]]:
|
|
636
|
+
source_schemas = (
|
|
637
|
+
_source_schema_paths(source, contract=contract) if source is not None else []
|
|
638
|
+
)
|
|
639
|
+
if source_schemas:
|
|
640
|
+
schema_sql = "\n".join(
|
|
641
|
+
path.read_text(encoding="utf-8") for path in source_schemas
|
|
642
|
+
)
|
|
643
|
+
adopted = [
|
|
644
|
+
f"source/{path.relative_to(source.resolve()).as_posix()}"
|
|
645
|
+
for path in source_schemas
|
|
646
|
+
]
|
|
647
|
+
store = authoring / "store.json"
|
|
648
|
+
if store.is_file():
|
|
649
|
+
return (
|
|
650
|
+
schema_sql + "\n" + _store_json_seed_sql(store),
|
|
651
|
+
adopted + ["store.json"],
|
|
652
|
+
)
|
|
653
|
+
sqlite = authoring / "world.sqlite"
|
|
654
|
+
if sqlite.is_file():
|
|
655
|
+
rows = _sqlite_sql(
|
|
656
|
+
sqlite,
|
|
657
|
+
contract_declarations=_contract_column_declarations(contract or {}),
|
|
658
|
+
include_schema=False,
|
|
659
|
+
)
|
|
660
|
+
return schema_sql + "\n" + rows, adopted + ["world.sqlite"]
|
|
661
|
+
collections = authoring / "collections.json"
|
|
662
|
+
if collections.is_file():
|
|
663
|
+
rows = _collections_sql(collections, include_schema=False)
|
|
664
|
+
return schema_sql + "\n" + rows, adopted + ["collections.json"]
|
|
665
|
+
return schema_sql, adopted
|
|
666
|
+
|
|
667
|
+
schema = authoring / "schema.sql"
|
|
668
|
+
if schema.is_file():
|
|
669
|
+
sql = schema.read_text(encoding="utf-8")
|
|
670
|
+
adopted = ["schema.sql"]
|
|
671
|
+
store = authoring / "store.json"
|
|
672
|
+
if store.is_file():
|
|
673
|
+
sql += "\n" + _store_json_seed_sql(store)
|
|
674
|
+
adopted.append("store.json")
|
|
675
|
+
return sql, adopted
|
|
676
|
+
sqlite = authoring / "world.sqlite"
|
|
677
|
+
if sqlite.is_file():
|
|
678
|
+
return _sqlite_sql(
|
|
679
|
+
sqlite,
|
|
680
|
+
contract_declarations=_contract_column_declarations(contract or {}),
|
|
681
|
+
), ["world.sqlite"]
|
|
682
|
+
collections = authoring / "collections.json"
|
|
683
|
+
if collections.is_file():
|
|
684
|
+
return _collections_sql(collections), ["collections.json"]
|
|
685
|
+
return "", []
|
|
686
|
+
|
|
687
|
+
|
|
688
|
+
def _compose_path(source: Path) -> Path | None:
|
|
689
|
+
matches = [source / name for name in _COMPOSE_NAMES if (source / name).is_file()]
|
|
690
|
+
if len(matches) > 1:
|
|
691
|
+
raise BundleAuthorError(
|
|
692
|
+
"compose_ambiguous: " + ", ".join(path.name for path in matches)
|
|
693
|
+
)
|
|
694
|
+
return matches[0] if matches else None
|
|
695
|
+
|
|
696
|
+
|
|
697
|
+
def _load_compose(path: Path) -> dict[str, Any]:
|
|
698
|
+
try:
|
|
699
|
+
body = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
|
700
|
+
except (OSError, yaml.YAMLError) as exc:
|
|
701
|
+
raise BundleAuthorError(f"compose_invalid: {exc}") from exc
|
|
702
|
+
if not isinstance(body, dict) or not isinstance(body.get("services"), dict):
|
|
703
|
+
raise BundleAuthorError("compose_invalid: services must be an object")
|
|
704
|
+
return body
|
|
705
|
+
|
|
706
|
+
|
|
707
|
+
_RUNTIME_ENVIRONMENT_NAME = re.compile(r"[A-Z_][A-Z0-9_]*")
|
|
708
|
+
_SECRET_ENVIRONMENT_NAME = re.compile(
|
|
709
|
+
r"(?:API_?KEY|SECRET|TOKEN|PASSWORD|CREDENTIAL|PRIVATE_?KEY)", re.IGNORECASE
|
|
710
|
+
)
|
|
711
|
+
|
|
712
|
+
|
|
713
|
+
def _declared_runtime_environment(source: Path) -> dict[str, str]:
|
|
714
|
+
"""Load public, non-secret process defaults declared by the repository.
|
|
715
|
+
|
|
716
|
+
Bundle V2 processes do not execute a Docker image and therefore cannot inherit image-level
|
|
717
|
+
``ENV`` values. Repositories that need deterministic runtime knobs can declare them in
|
|
718
|
+
``alk.yaml`` under ``runtime.environment``. Values are sealed into the bundle manifest, so
|
|
719
|
+
credential-shaped names and shell-style interpolation are rejected; secrets must continue
|
|
720
|
+
to travel through purpose-scoped refs.
|
|
721
|
+
"""
|
|
722
|
+
path = source / "alk.yaml"
|
|
723
|
+
if not path.is_file():
|
|
724
|
+
return {}
|
|
725
|
+
try:
|
|
726
|
+
body = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
|
727
|
+
except (OSError, yaml.YAMLError) as exc:
|
|
728
|
+
raise BundleAuthorError(f"runtime_manifest_invalid: {exc}") from exc
|
|
729
|
+
if not isinstance(body, dict):
|
|
730
|
+
raise BundleAuthorError("runtime_manifest_invalid: root must be an object")
|
|
731
|
+
runtime = body.get("runtime") or {}
|
|
732
|
+
if not isinstance(runtime, dict):
|
|
733
|
+
raise BundleAuthorError("runtime_manifest_invalid: runtime must be an object")
|
|
734
|
+
raw_environment = runtime.get("environment") or {}
|
|
735
|
+
if not isinstance(raw_environment, dict):
|
|
736
|
+
raise BundleAuthorError(
|
|
737
|
+
"runtime_manifest_invalid: runtime.environment must be an object"
|
|
738
|
+
)
|
|
739
|
+
environment: dict[str, str] = {}
|
|
740
|
+
for raw_name, raw_value in raw_environment.items():
|
|
741
|
+
name = str(raw_name)
|
|
742
|
+
if not _RUNTIME_ENVIRONMENT_NAME.fullmatch(name):
|
|
743
|
+
raise BundleAuthorError(f"runtime_environment_name_invalid: {name}")
|
|
744
|
+
if _SECRET_ENVIRONMENT_NAME.search(name):
|
|
745
|
+
raise BundleAuthorError(f"runtime_environment_secret_forbidden: {name}")
|
|
746
|
+
if not isinstance(raw_value, (str, int, float, bool)):
|
|
747
|
+
raise BundleAuthorError(f"runtime_environment_value_invalid: {name}")
|
|
748
|
+
value = str(raw_value)
|
|
749
|
+
if "${" in value or "{{" in value:
|
|
750
|
+
raise BundleAuthorError(
|
|
751
|
+
f"runtime_environment_interpolation_forbidden: {name}"
|
|
752
|
+
)
|
|
753
|
+
environment[name] = value
|
|
754
|
+
return environment
|
|
755
|
+
|
|
756
|
+
|
|
757
|
+
def _python_process(
|
|
758
|
+
*,
|
|
759
|
+
name: str,
|
|
760
|
+
working_directory: str,
|
|
761
|
+
entry: str,
|
|
762
|
+
control: bool,
|
|
763
|
+
needs_secrets: bool,
|
|
764
|
+
port: int | None = None,
|
|
765
|
+
environment: dict[str, str] | None = None,
|
|
766
|
+
depends_on: list[str] | None = None,
|
|
767
|
+
) -> SourceProcess:
|
|
768
|
+
# The build tree is writable; the submitted source remains read-only. ``uv sync`` creates a
|
|
769
|
+
# project-local venv for pyproject repositories, while requirements/stdlib sources get the
|
|
770
|
+
# same explicit venv boundary. No dependency is installed into the immutable snapshot.
|
|
771
|
+
build: list[list[str]]
|
|
772
|
+
run: list[str]
|
|
773
|
+
relative_root = Path(working_directory)
|
|
774
|
+
# Discovery happens at the caller's source root; these placeholders are resolved below by
|
|
775
|
+
# `_plan_python`, which replaces this conservative default where necessary.
|
|
776
|
+
build = [["python3.12", "-m", "venv", ".venv"]]
|
|
777
|
+
run = [".venv/bin/python", entry]
|
|
778
|
+
del relative_root
|
|
779
|
+
return SourceProcess(
|
|
780
|
+
name=name,
|
|
781
|
+
working_directory=working_directory,
|
|
782
|
+
build_commands=build,
|
|
783
|
+
run_command=run,
|
|
784
|
+
# Match the established Compose harness lane: submitted processes may adapt
|
|
785
|
+
# deterministic test-only provider seams without receiving an extra credential or
|
|
786
|
+
# control-plane capability.
|
|
787
|
+
environment={"HARNESS_MODE": "1", **(environment or {})},
|
|
788
|
+
fixed_port=port,
|
|
789
|
+
started_check=StartedCheck(port=True, timeout_seconds=180) if port else None,
|
|
790
|
+
secret_purposes=[SecretPurpose.TARGET_PROVIDER] if needs_secrets else [],
|
|
791
|
+
user=ProcessUser.SVC_AGENT if control else ProcessUser.SVC_TOOLS,
|
|
792
|
+
depends_on=depends_on or [],
|
|
793
|
+
)
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
def _plan_python(
|
|
797
|
+
source: Path,
|
|
798
|
+
*,
|
|
799
|
+
name: str,
|
|
800
|
+
root: Path,
|
|
801
|
+
entry: str,
|
|
802
|
+
control: bool,
|
|
803
|
+
needs_secrets: bool,
|
|
804
|
+
port: int | None = None,
|
|
805
|
+
environment: dict[str, str] | None = None,
|
|
806
|
+
depends_on: list[str] | None = None,
|
|
807
|
+
livekit_download: bool = False,
|
|
808
|
+
run_override: list[str] | None = None,
|
|
809
|
+
) -> SourceProcess:
|
|
810
|
+
relative = root.relative_to(source).as_posix() or "."
|
|
811
|
+
process = _python_process(
|
|
812
|
+
name=name,
|
|
813
|
+
working_directory=relative,
|
|
814
|
+
entry=entry,
|
|
815
|
+
control=control,
|
|
816
|
+
needs_secrets=needs_secrets,
|
|
817
|
+
port=port,
|
|
818
|
+
environment=environment,
|
|
819
|
+
depends_on=depends_on,
|
|
820
|
+
)
|
|
821
|
+
python = _docker_python(root)
|
|
822
|
+
if (root / "pyproject.toml").is_file():
|
|
823
|
+
commands = [["uv", "sync", "--no-cache", "--python", python]]
|
|
824
|
+
if (root / "uv.lock").is_file():
|
|
825
|
+
commands[0].append("--locked")
|
|
826
|
+
if livekit_download:
|
|
827
|
+
commands.append(
|
|
828
|
+
[
|
|
829
|
+
"uv",
|
|
830
|
+
"run",
|
|
831
|
+
"--no-sync",
|
|
832
|
+
"python",
|
|
833
|
+
"-m",
|
|
834
|
+
"livekit.agents",
|
|
835
|
+
"download-files",
|
|
836
|
+
]
|
|
837
|
+
)
|
|
838
|
+
run = ["uv", "run", "--no-sync", "python", entry]
|
|
839
|
+
elif (root / "requirements.txt").is_file():
|
|
840
|
+
commands = [
|
|
841
|
+
[python, "-m", "venv", ".venv"],
|
|
842
|
+
[
|
|
843
|
+
".venv/bin/python",
|
|
844
|
+
"-m",
|
|
845
|
+
"pip",
|
|
846
|
+
"install",
|
|
847
|
+
"--requirement",
|
|
848
|
+
"requirements.txt",
|
|
849
|
+
],
|
|
850
|
+
]
|
|
851
|
+
run = [".venv/bin/python", entry]
|
|
852
|
+
else:
|
|
853
|
+
commands = []
|
|
854
|
+
run = [python, entry]
|
|
855
|
+
return process.model_copy(
|
|
856
|
+
update={"build_commands": commands, "run_command": run_override or run}
|
|
857
|
+
)
|
|
858
|
+
|
|
859
|
+
|
|
860
|
+
def _docker_python(root: Path) -> str:
|
|
861
|
+
dockerfile = root / "Dockerfile"
|
|
862
|
+
if not dockerfile.is_file():
|
|
863
|
+
return "python3.12"
|
|
864
|
+
text = dockerfile.read_text(encoding="utf-8", errors="replace")
|
|
865
|
+
argument = re.search(r"(?mi)^ARG\s+PYTHON_VERSION\s*=\s*([0-9]+\.[0-9]+)\s*$", text)
|
|
866
|
+
if argument:
|
|
867
|
+
return f"python{argument.group(1)}"
|
|
868
|
+
direct = re.search(r"(?mi)^FROM\s+(?:[^/\s]+/)*python:([0-9]+\.[0-9]+)", text)
|
|
869
|
+
return f"python{direct.group(1)}" if direct else "python3.12"
|
|
870
|
+
|
|
871
|
+
|
|
872
|
+
def _dockerfile_run(root: Path) -> list[str] | None:
|
|
873
|
+
dockerfile = root / "Dockerfile"
|
|
874
|
+
if not dockerfile.is_file():
|
|
875
|
+
return None
|
|
876
|
+
commands = []
|
|
877
|
+
for line in dockerfile.read_text(encoding="utf-8", errors="replace").splitlines():
|
|
878
|
+
stripped = line.strip()
|
|
879
|
+
if stripped.upper().startswith("CMD "):
|
|
880
|
+
commands.append(stripped[4:].strip())
|
|
881
|
+
if not commands:
|
|
882
|
+
return None
|
|
883
|
+
raw = commands[-1]
|
|
884
|
+
if not raw.startswith("["):
|
|
885
|
+
raise BundleAuthorError(
|
|
886
|
+
f"dockerfile_command_unsupported: {dockerfile} uses shell-form CMD"
|
|
887
|
+
)
|
|
888
|
+
try:
|
|
889
|
+
argv = json.loads(raw)
|
|
890
|
+
except ValueError as exc:
|
|
891
|
+
raise BundleAuthorError(f"dockerfile_command_invalid: {dockerfile}") from exc
|
|
892
|
+
if (
|
|
893
|
+
not isinstance(argv, list)
|
|
894
|
+
or not argv
|
|
895
|
+
or not all(isinstance(item, str) for item in argv)
|
|
896
|
+
):
|
|
897
|
+
raise BundleAuthorError(f"dockerfile_command_invalid: {dockerfile}")
|
|
898
|
+
if argv[0] == "python":
|
|
899
|
+
argv[0] = ".venv/bin/python" if (root / "requirements.txt").is_file() else "uv"
|
|
900
|
+
if argv[0] == "uv":
|
|
901
|
+
argv[1:1] = ["run", "--no-sync", "python"]
|
|
902
|
+
elif (
|
|
903
|
+
argv[0] in {"uvicorn", "gunicorn", "flask"}
|
|
904
|
+
and (root / "requirements.txt").is_file()
|
|
905
|
+
):
|
|
906
|
+
argv[0] = f".venv/bin/{argv[0]}"
|
|
907
|
+
return argv
|
|
908
|
+
|
|
909
|
+
|
|
910
|
+
# LiveKit's CLI needs a subcommand: `agent.py` alone prints usage and exits without registering.
|
|
911
|
+
_LIVEKIT_WORKER_SUBCOMMANDS = frozenset({"start", "dev", "connect", "console"})
|
|
912
|
+
|
|
913
|
+
|
|
914
|
+
def _hands_off_to_livekit_cli(root: Path, entry: str) -> bool:
|
|
915
|
+
"""Whether the entry delegates to LiveKit's CLI. An agent that runs its own worker must not."""
|
|
916
|
+
path = root / entry
|
|
917
|
+
if not path.is_file():
|
|
918
|
+
return False
|
|
919
|
+
return "cli.run_app" in path.read_text(encoding="utf-8", errors="replace")
|
|
920
|
+
|
|
921
|
+
|
|
922
|
+
def _discover_callback_entrypoint(root: Path) -> str | None:
|
|
923
|
+
"""Return the repository's unique module-level ``agent_callback``, if present.
|
|
924
|
+
|
|
925
|
+
Callback support is a source property, not an LLM-authored contract property. Contract
|
|
926
|
+
authoring can legitimately omit ``runtime.interface`` even when the repository exports the
|
|
927
|
+
canonical callback. Treating that omission as authoritative used to compile such agents as
|
|
928
|
+
``python agent.py`` HTTP services, which can never pass the generated port readiness probe.
|
|
929
|
+
"""
|
|
930
|
+
candidates: list[str] = []
|
|
931
|
+
for path in sorted(root.rglob("*.py")):
|
|
932
|
+
relative = path.relative_to(root)
|
|
933
|
+
if any(part in _IGNORED_ARTIFACT_PARTS for part in relative.parts):
|
|
934
|
+
continue
|
|
935
|
+
try:
|
|
936
|
+
tree = ast.parse(path.read_text(encoding="utf-8", errors="replace"))
|
|
937
|
+
except SyntaxError:
|
|
938
|
+
continue
|
|
939
|
+
if any(
|
|
940
|
+
isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
|
|
941
|
+
and node.name == "agent_callback"
|
|
942
|
+
for node in tree.body
|
|
943
|
+
):
|
|
944
|
+
module = ".".join(relative.with_suffix("").parts)
|
|
945
|
+
candidates.append(f"{module}:agent_callback")
|
|
946
|
+
if not candidates:
|
|
947
|
+
return None
|
|
948
|
+
if len(candidates) != 1:
|
|
949
|
+
raise BundleAuthorError(
|
|
950
|
+
"callback_entrypoint_ambiguous: " + ", ".join(candidates)
|
|
951
|
+
)
|
|
952
|
+
return candidates[0]
|
|
953
|
+
|
|
954
|
+
|
|
955
|
+
def _callback_entrypoint(root: Path) -> str:
|
|
956
|
+
"""Find the callback promised by an explicitly callable runtime contract."""
|
|
957
|
+
candidate = _discover_callback_entrypoint(root)
|
|
958
|
+
if candidate is None:
|
|
959
|
+
raise BundleAuthorError(
|
|
960
|
+
"callback_entrypoint_missing: callable runtime requires one module-level "
|
|
961
|
+
"agent_callback"
|
|
962
|
+
)
|
|
963
|
+
return candidate
|
|
964
|
+
|
|
965
|
+
|
|
966
|
+
def _callback_adapter_source() -> str:
|
|
967
|
+
return (
|
|
968
|
+
Path(__file__).with_name("callback_http_adapter.py").read_text(encoding="utf-8")
|
|
969
|
+
)
|
|
970
|
+
|
|
971
|
+
|
|
972
|
+
def _managed_world_db() -> ManagedProcess:
|
|
973
|
+
return ManagedProcess(
|
|
974
|
+
name="world-db",
|
|
975
|
+
engine=ManagedEngine.POSTGRES,
|
|
976
|
+
version="16",
|
|
977
|
+
user=ProcessUser.SVC_DATA,
|
|
978
|
+
)
|
|
979
|
+
|
|
980
|
+
|
|
981
|
+
def _tool_proxy_process() -> SourceProcess:
|
|
982
|
+
return SourceProcess(
|
|
983
|
+
name="tool-proxy",
|
|
984
|
+
working_directory="generated/tool-proxy",
|
|
985
|
+
source_origin="bundle",
|
|
986
|
+
run_command=["/opt/alk-venv/bin/python", "proxy.py"],
|
|
987
|
+
environment={
|
|
988
|
+
"PORT": "{{PORT_tool-proxy}}",
|
|
989
|
+
"UPSTREAM_URL": "{{TOOLS_UPSTREAM_URL}}",
|
|
990
|
+
"DATABASE_URL": "{{WORLD_DATABASE_URL}}",
|
|
991
|
+
},
|
|
992
|
+
started_check=StartedCheck(port=True, timeout_seconds=180),
|
|
993
|
+
user=ProcessUser.SVC_TOOLS,
|
|
994
|
+
depends_on=["tools-api", "world-db"],
|
|
995
|
+
)
|
|
996
|
+
|
|
997
|
+
|
|
998
|
+
def resolve_environment_plan(
|
|
999
|
+
source: str | Path,
|
|
1000
|
+
job: HarnessJob,
|
|
1001
|
+
*,
|
|
1002
|
+
contract_modality: str | None = None,
|
|
1003
|
+
contract_interface_kind: str | None = None,
|
|
1004
|
+
) -> EnvironmentPlanV2:
|
|
1005
|
+
"""Resolve packaging once. Authoring and provisioning consume this same immutable plan."""
|
|
1006
|
+
root = Path(source).resolve()
|
|
1007
|
+
if not root.is_dir():
|
|
1008
|
+
raise BundleAuthorError(f"source_unavailable: {root}")
|
|
1009
|
+
connector = job.agent.connector.lower()
|
|
1010
|
+
if job.agent.mode is ProviderExecutionMode.ENVIRONMENT_BACKED:
|
|
1011
|
+
declaration = load_provider_manifest(
|
|
1012
|
+
root, str(job.agent.config.get("lifecycle_manifest") or "alk.yaml")
|
|
1013
|
+
)
|
|
1014
|
+
if declaration.provider.type.value != connector:
|
|
1015
|
+
raise BundleAuthorError(
|
|
1016
|
+
"provider_lifecycle_connector_mismatch: "
|
|
1017
|
+
f"job={connector}, manifest={declaration.provider.type.value}"
|
|
1018
|
+
)
|
|
1019
|
+
# Hosted repository submissions normally arrive as ``connector=auto``. In the unified
|
|
1020
|
+
# Daytona lane the contract is authored *after* dispatch, so the control plane cannot rewrite
|
|
1021
|
+
# that field before this compiler runs. The frozen contract is therefore the authoritative
|
|
1022
|
+
# late-bound modality signal. Voice is routed through LiveKit because that is the hosted
|
|
1023
|
+
# repository voice connector implemented by the guest; explicit vapi/retell values never
|
|
1024
|
+
# enter this path.
|
|
1025
|
+
is_livekit = connector == "livekit" or (
|
|
1026
|
+
connector == "auto" and (contract_modality or "").strip().lower() == "voice"
|
|
1027
|
+
)
|
|
1028
|
+
needs_target_secrets = any(
|
|
1029
|
+
reference.purpose == SecretPurpose.TARGET_PROVIDER.value
|
|
1030
|
+
for reference in job.agent.secret_refs.values()
|
|
1031
|
+
)
|
|
1032
|
+
compose = _compose_path(root)
|
|
1033
|
+
processes: list[ManagedProcess | SourceProcess] = [_managed_world_db()]
|
|
1034
|
+
capabilities: dict[str, CapabilityV2] = {
|
|
1035
|
+
"world_db": CapabilityV2(
|
|
1036
|
+
protocol=CapabilityProtocol.POSTGRES,
|
|
1037
|
+
service="world-db",
|
|
1038
|
+
container_port=5432,
|
|
1039
|
+
configuration_name="WORLD_DATABASE_URL",
|
|
1040
|
+
)
|
|
1041
|
+
}
|
|
1042
|
+
readiness = [ReadinessProbeV2(capability="world_db", timeout_seconds=180)]
|
|
1043
|
+
declared_runtime_environment = _declared_runtime_environment(root)
|
|
1044
|
+
|
|
1045
|
+
# A connect-only provider target is hosted by Vapi/Retell and is addressed by the
|
|
1046
|
+
# provider ID in the job. When no repository was submitted there is deliberately no
|
|
1047
|
+
# customer process to discover or launch; the local runtime only owns the isolated world.
|
|
1048
|
+
# Keep repository-backed connect-only jobs on the normal path so uploaded tool/backend
|
|
1049
|
+
# implementations are still compiled and exercised.
|
|
1050
|
+
if (
|
|
1051
|
+
job.agent.mode is ProviderExecutionMode.CONNECT_ONLY
|
|
1052
|
+
and job.source.kind is SourceKind.PROVIDER
|
|
1053
|
+
):
|
|
1054
|
+
return EnvironmentPlanV2(
|
|
1055
|
+
packaging="provider_connect_only",
|
|
1056
|
+
control_service=None,
|
|
1057
|
+
processes=tuple(processes),
|
|
1058
|
+
capabilities=capabilities,
|
|
1059
|
+
readiness=tuple(readiness),
|
|
1060
|
+
)
|
|
1061
|
+
|
|
1062
|
+
if compose is not None:
|
|
1063
|
+
body = _load_compose(compose)
|
|
1064
|
+
services = body["services"]
|
|
1065
|
+
# Compile the submitted topology. Supported managed services become snapshot engines;
|
|
1066
|
+
# source services remain source processes. Unknown image-only dependencies are rejected
|
|
1067
|
+
# explicitly instead of being silently emulated.
|
|
1068
|
+
managed_names: set[str] = set()
|
|
1069
|
+
for service_name, raw in services.items():
|
|
1070
|
+
service = raw if isinstance(raw, dict) else {}
|
|
1071
|
+
image = str(service.get("image") or "")
|
|
1072
|
+
if image.startswith("postgres:"):
|
|
1073
|
+
if service_name != "postgres":
|
|
1074
|
+
raise BundleAuthorError(
|
|
1075
|
+
f"managed_name_unsupported: postgres service must be named postgres, got {service_name}"
|
|
1076
|
+
)
|
|
1077
|
+
managed_names.add(service_name)
|
|
1078
|
+
continue
|
|
1079
|
+
if image and "redis" in image:
|
|
1080
|
+
processes.append(
|
|
1081
|
+
ManagedProcess(
|
|
1082
|
+
name=service_name,
|
|
1083
|
+
engine=ManagedEngine.REDIS,
|
|
1084
|
+
version=image.split(":", 1)[1].split("-", 1)[0]
|
|
1085
|
+
if ":" in image
|
|
1086
|
+
else "7",
|
|
1087
|
+
user=ProcessUser.SVC_DATA,
|
|
1088
|
+
)
|
|
1089
|
+
)
|
|
1090
|
+
managed_names.add(service_name)
|
|
1091
|
+
capabilities[f"{service_name}_redis"] = CapabilityV2(
|
|
1092
|
+
protocol=CapabilityProtocol.REDIS,
|
|
1093
|
+
service=service_name,
|
|
1094
|
+
container_port=6379,
|
|
1095
|
+
configuration_name=f"{service_name.upper().replace('-', '_')}_URL",
|
|
1096
|
+
)
|
|
1097
|
+
readiness.append(ReadinessProbeV2(capability=f"{service_name}_redis"))
|
|
1098
|
+
continue
|
|
1099
|
+
if image and not service.get("build"):
|
|
1100
|
+
raise BundleAuthorError(
|
|
1101
|
+
f"engine_unsupported: image-only service {service_name!r} ({image!r}) is not in the snapshot catalog"
|
|
1102
|
+
)
|
|
1103
|
+
|
|
1104
|
+
source_services = [name for name in services if name not in managed_names]
|
|
1105
|
+
control_name = (
|
|
1106
|
+
"agent"
|
|
1107
|
+
if "agent" in source_services
|
|
1108
|
+
else (
|
|
1109
|
+
"api"
|
|
1110
|
+
if "api" in source_services
|
|
1111
|
+
else source_services[-1]
|
|
1112
|
+
if source_services
|
|
1113
|
+
else ""
|
|
1114
|
+
)
|
|
1115
|
+
)
|
|
1116
|
+
if not control_name:
|
|
1117
|
+
raise BundleAuthorError(
|
|
1118
|
+
"control_service_missing: compose has no source-built service"
|
|
1119
|
+
)
|
|
1120
|
+
for service_name in source_services:
|
|
1121
|
+
service = services[service_name]
|
|
1122
|
+
build = service.get("build", ".")
|
|
1123
|
+
if isinstance(build, dict):
|
|
1124
|
+
context = str(build.get("context") or ".")
|
|
1125
|
+
else:
|
|
1126
|
+
context = str(build)
|
|
1127
|
+
service_root = (root / context).resolve()
|
|
1128
|
+
if not service_root.is_relative_to(root):
|
|
1129
|
+
raise BundleAuthorError(f"build_context_escape: {service_name}")
|
|
1130
|
+
environment: dict[str, str] = {}
|
|
1131
|
+
raw_env = service.get("environment") or {}
|
|
1132
|
+
if isinstance(raw_env, dict):
|
|
1133
|
+
environment = {
|
|
1134
|
+
str(k): str(v) for k, v in raw_env.items() if v is not None
|
|
1135
|
+
}
|
|
1136
|
+
depends = (
|
|
1137
|
+
list((service.get("depends_on") or {}).keys())
|
|
1138
|
+
if isinstance(service.get("depends_on"), dict)
|
|
1139
|
+
else list(service.get("depends_on") or [])
|
|
1140
|
+
)
|
|
1141
|
+
if service_name == "tools-api" and "postgres" in managed_names:
|
|
1142
|
+
# The target DB is intentionally a distinct per-world logical DB on the same
|
|
1143
|
+
# harness-owned Postgres engine. This preserves reset/isolation without another
|
|
1144
|
+
# daemon per call.
|
|
1145
|
+
environment["DATABASE_URL"] = "{{WORLD_DATABASE_URL}}"
|
|
1146
|
+
depends = [
|
|
1147
|
+
"world-db" if item == "postgres" else item for item in depends
|
|
1148
|
+
]
|
|
1149
|
+
if service_name == control_name and "tools-api" in source_services:
|
|
1150
|
+
environment["TOOLS_API_URL"] = "{{TOOLS_API_URL}}"
|
|
1151
|
+
if is_livekit and service_name == control_name:
|
|
1152
|
+
environment = {**declared_runtime_environment, **environment}
|
|
1153
|
+
environment.setdefault(
|
|
1154
|
+
"LIVEKIT_AGENT_NAME",
|
|
1155
|
+
"uber-voice-booking-{{JOB_ID}}-w{{WORLD_INDEX}}",
|
|
1156
|
+
)
|
|
1157
|
+
environment.setdefault(
|
|
1158
|
+
"HARNESS_TOOL_TRACE",
|
|
1159
|
+
"{{WORLD_DIR}}/agent-tool-calls.jsonl",
|
|
1160
|
+
)
|
|
1161
|
+
entry = (
|
|
1162
|
+
"agent/agent.py"
|
|
1163
|
+
if (service_root / "agent" / "agent.py").is_file()
|
|
1164
|
+
else "agent.py"
|
|
1165
|
+
)
|
|
1166
|
+
port = 8080 if service_name in {"api", "tools-api"} else None
|
|
1167
|
+
process = _plan_python(
|
|
1168
|
+
root,
|
|
1169
|
+
name=service_name,
|
|
1170
|
+
root=service_root,
|
|
1171
|
+
entry=entry,
|
|
1172
|
+
control=service_name == control_name,
|
|
1173
|
+
needs_secrets=needs_target_secrets and service_name == control_name,
|
|
1174
|
+
port=port,
|
|
1175
|
+
environment=environment,
|
|
1176
|
+
depends_on=[item for item in depends if item != "postgres"],
|
|
1177
|
+
livekit_download=is_livekit and service_name == control_name,
|
|
1178
|
+
run_override=_dockerfile_run(service_root),
|
|
1179
|
+
)
|
|
1180
|
+
if is_livekit and service_name == control_name:
|
|
1181
|
+
# The LiveKit worker opens its HTTP health port before it has registered with
|
|
1182
|
+
# the dispatch service. Treating the port as readiness creates a race where a
|
|
1183
|
+
# named dispatch is submitted in that gap; self-hosted LiveKit leaves that
|
|
1184
|
+
# dispatch unassigned even after the worker subsequently registers. The worker
|
|
1185
|
+
# log is the first observable signal that it can actually accept the call.
|
|
1186
|
+
process = process.model_copy(
|
|
1187
|
+
update={
|
|
1188
|
+
"started_check": StartedCheck(
|
|
1189
|
+
log_marker="registered worker", timeout_seconds=180
|
|
1190
|
+
)
|
|
1191
|
+
}
|
|
1192
|
+
)
|
|
1193
|
+
processes.append(process)
|
|
1194
|
+
if port:
|
|
1195
|
+
slug = "target_http" if service_name == control_name else "tools_api"
|
|
1196
|
+
config = (
|
|
1197
|
+
"TARGET_HTTP_URL"
|
|
1198
|
+
if service_name == control_name
|
|
1199
|
+
else "TOOLS_UPSTREAM_URL"
|
|
1200
|
+
)
|
|
1201
|
+
capabilities[slug] = CapabilityV2(
|
|
1202
|
+
protocol=CapabilityProtocol.HTTP,
|
|
1203
|
+
service=service_name,
|
|
1204
|
+
container_port=port,
|
|
1205
|
+
configuration_name=config,
|
|
1206
|
+
)
|
|
1207
|
+
readiness.append(
|
|
1208
|
+
ReadinessProbeV2(
|
|
1209
|
+
capability=slug, path="/health", timeout_seconds=180
|
|
1210
|
+
)
|
|
1211
|
+
)
|
|
1212
|
+
if "tools-api" in source_services:
|
|
1213
|
+
processes.append(_tool_proxy_process())
|
|
1214
|
+
# The target must not become eligible to start until the evidence proxy is ready.
|
|
1215
|
+
# Depending only on the upstream tools process leaves a race where the agent starts
|
|
1216
|
+
# with TOOLS_API_URL pointing at a port that has not been bound yet.
|
|
1217
|
+
rewritten: list[ManagedProcess | SourceProcess] = []
|
|
1218
|
+
for process in processes:
|
|
1219
|
+
if isinstance(process, SourceProcess) and process.name == control_name:
|
|
1220
|
+
dependencies = [
|
|
1221
|
+
"tool-proxy" if item == "tools-api" else item
|
|
1222
|
+
for item in process.depends_on
|
|
1223
|
+
]
|
|
1224
|
+
if "tool-proxy" not in dependencies:
|
|
1225
|
+
dependencies.append("tool-proxy")
|
|
1226
|
+
process = process.model_copy(update={"depends_on": dependencies})
|
|
1227
|
+
rewritten.append(process)
|
|
1228
|
+
processes = rewritten
|
|
1229
|
+
capabilities["tool_proxy"] = CapabilityV2(
|
|
1230
|
+
protocol=CapabilityProtocol.HTTP,
|
|
1231
|
+
service="tool-proxy",
|
|
1232
|
+
container_port=8080,
|
|
1233
|
+
configuration_name="TOOLS_API_URL",
|
|
1234
|
+
)
|
|
1235
|
+
readiness.append(
|
|
1236
|
+
ReadinessProbeV2(
|
|
1237
|
+
capability="tool_proxy", path="/health", timeout_seconds=180
|
|
1238
|
+
)
|
|
1239
|
+
)
|
|
1240
|
+
packaging = "compose"
|
|
1241
|
+
else:
|
|
1242
|
+
contract_is_callback = (contract_interface_kind or "").strip().lower().replace(
|
|
1243
|
+
"-", "_"
|
|
1244
|
+
) == "callable"
|
|
1245
|
+
discovered_callback = (
|
|
1246
|
+
None if is_livekit else _discover_callback_entrypoint(root)
|
|
1247
|
+
)
|
|
1248
|
+
is_callback = not is_livekit and (
|
|
1249
|
+
contract_is_callback or discovered_callback is not None
|
|
1250
|
+
)
|
|
1251
|
+
entry = "agent.py"
|
|
1252
|
+
if not is_callback and not (root / entry).is_file():
|
|
1253
|
+
candidates = sorted(root.glob("**/agent.py"))
|
|
1254
|
+
if len(candidates) != 1:
|
|
1255
|
+
raise BundleAuthorError(
|
|
1256
|
+
"component_ambiguous: expected exactly one agent.py"
|
|
1257
|
+
)
|
|
1258
|
+
component = candidates[0].parent
|
|
1259
|
+
# An entrypoint directory is not necessarily its Python project root. Preserve
|
|
1260
|
+
# the nearest enclosing manifest and its sibling packages instead of flattening
|
|
1261
|
+
# src/ and silently running without the repository's dependencies.
|
|
1262
|
+
for parent in (component, *component.parents):
|
|
1263
|
+
if not parent.is_relative_to(root):
|
|
1264
|
+
break
|
|
1265
|
+
if any(
|
|
1266
|
+
(parent / name).is_file()
|
|
1267
|
+
for name in ("pyproject.toml", "requirements.txt")
|
|
1268
|
+
):
|
|
1269
|
+
component = parent
|
|
1270
|
+
break
|
|
1271
|
+
entry = candidates[0].relative_to(component).as_posix()
|
|
1272
|
+
else:
|
|
1273
|
+
component = root
|
|
1274
|
+
control_name = "agent"
|
|
1275
|
+
port = None if is_livekit else 8080
|
|
1276
|
+
environment = (
|
|
1277
|
+
{
|
|
1278
|
+
**declared_runtime_environment,
|
|
1279
|
+
"LIVEKIT_AGENT_NAME": (
|
|
1280
|
+
root.name.replace("_", "-") + "-{{JOB_ID}}-w{{WORLD_INDEX}}"
|
|
1281
|
+
),
|
|
1282
|
+
"HARNESS_TOOL_TRACE": "{{WORLD_DIR}}/agent-tool-calls.jsonl",
|
|
1283
|
+
}
|
|
1284
|
+
if is_livekit
|
|
1285
|
+
else dict(declared_runtime_environment)
|
|
1286
|
+
)
|
|
1287
|
+
callback_entrypoint = (
|
|
1288
|
+
discovered_callback or _callback_entrypoint(root) if is_callback else None
|
|
1289
|
+
)
|
|
1290
|
+
if callback_entrypoint:
|
|
1291
|
+
environment.update(
|
|
1292
|
+
{
|
|
1293
|
+
"PORT": "{{PORT_agent}}",
|
|
1294
|
+
"ALK_CALLBACK_ENTRYPOINT": callback_entrypoint,
|
|
1295
|
+
}
|
|
1296
|
+
)
|
|
1297
|
+
process = _plan_python(
|
|
1298
|
+
root,
|
|
1299
|
+
name=control_name,
|
|
1300
|
+
root=root if is_callback else component,
|
|
1301
|
+
entry=entry,
|
|
1302
|
+
control=True,
|
|
1303
|
+
needs_secrets=needs_target_secrets,
|
|
1304
|
+
port=port,
|
|
1305
|
+
environment=environment,
|
|
1306
|
+
livekit_download=is_livekit,
|
|
1307
|
+
run_override=(None if is_callback else _dockerfile_run(component)),
|
|
1308
|
+
)
|
|
1309
|
+
if is_callback:
|
|
1310
|
+
python_command = process.run_command[:-1]
|
|
1311
|
+
process = process.model_copy(
|
|
1312
|
+
update={
|
|
1313
|
+
"run_command": python_command + ["-c", _callback_adapter_source()],
|
|
1314
|
+
"started_check": StartedCheck(port=True, timeout_seconds=180),
|
|
1315
|
+
}
|
|
1316
|
+
)
|
|
1317
|
+
if is_livekit:
|
|
1318
|
+
update: dict[str, Any] = {
|
|
1319
|
+
"started_check": StartedCheck(
|
|
1320
|
+
log_marker="registered worker", timeout_seconds=180
|
|
1321
|
+
)
|
|
1322
|
+
}
|
|
1323
|
+
# Only a Dockerfile CMD carries the subcommand today, so a repository without one
|
|
1324
|
+
# starts `agent.py` bare and never reaches the registration this check waits for.
|
|
1325
|
+
if _hands_off_to_livekit_cli(component, entry) and not (
|
|
1326
|
+
set(process.run_command) & _LIVEKIT_WORKER_SUBCOMMANDS
|
|
1327
|
+
):
|
|
1328
|
+
update["run_command"] = [*process.run_command, "start"]
|
|
1329
|
+
process = process.model_copy(update=update)
|
|
1330
|
+
processes.append(process)
|
|
1331
|
+
if port:
|
|
1332
|
+
capabilities["target_http"] = CapabilityV2(
|
|
1333
|
+
protocol=CapabilityProtocol.HTTP,
|
|
1334
|
+
service=control_name,
|
|
1335
|
+
container_port=port,
|
|
1336
|
+
configuration_name="TARGET_HTTP_URL",
|
|
1337
|
+
)
|
|
1338
|
+
readiness.append(
|
|
1339
|
+
ReadinessProbeV2(
|
|
1340
|
+
capability="target_http", path="/health", timeout_seconds=180
|
|
1341
|
+
)
|
|
1342
|
+
)
|
|
1343
|
+
packaging = (
|
|
1344
|
+
"dockerfile" if (root / "Dockerfile").is_file() else "generated_python"
|
|
1345
|
+
)
|
|
1346
|
+
|
|
1347
|
+
return EnvironmentPlanV2(
|
|
1348
|
+
packaging=packaging,
|
|
1349
|
+
control_service=control_name,
|
|
1350
|
+
processes=tuple(processes),
|
|
1351
|
+
capabilities=capabilities,
|
|
1352
|
+
readiness=tuple(readiness),
|
|
1353
|
+
)
|
|
1354
|
+
|
|
1355
|
+
|
|
1356
|
+
def _copy_scenarios(authoring: Path, staging: Path, *, count: int) -> None:
|
|
1357
|
+
source = authoring / "scenarios"
|
|
1358
|
+
if not source.is_dir():
|
|
1359
|
+
raise BundleAuthorError(f"scenario_artifacts_missing: {source}")
|
|
1360
|
+
target = staging / "scenarios"
|
|
1361
|
+
target.mkdir()
|
|
1362
|
+
folders = sorted(path for path in source.iterdir() if path.is_dir())
|
|
1363
|
+
if len(folders) < count:
|
|
1364
|
+
raise BundleAuthorError(
|
|
1365
|
+
f"scenario_artifacts_insufficient: requested {count}, found {len(folders)}"
|
|
1366
|
+
)
|
|
1367
|
+
for source_folder in folders[:count]:
|
|
1368
|
+
shutil.copytree(source_folder, target / source_folder.name)
|
|
1369
|
+
for folder in sorted(path for path in target.iterdir() if path.is_dir()):
|
|
1370
|
+
document = folder / "scenario.json"
|
|
1371
|
+
if not document.is_file():
|
|
1372
|
+
continue
|
|
1373
|
+
body = json.loads(document.read_text(encoding="utf-8"))
|
|
1374
|
+
body["scenario_key"] = str(
|
|
1375
|
+
body.get("scenario_key") or body.get("name") or folder.name
|
|
1376
|
+
)
|
|
1377
|
+
body["scenario_id"] = str(body.get("scenario_id") or "")
|
|
1378
|
+
document.write_text(
|
|
1379
|
+
json.dumps(body, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
|
1380
|
+
)
|
|
1381
|
+
|
|
1382
|
+
|
|
1383
|
+
def _copy_sub_goal_catalogue(authoring: Path, staging: Path) -> list[str]:
|
|
1384
|
+
"""Put the sub-goal catalogue beside the scenarios that name its entries.
|
|
1385
|
+
|
|
1386
|
+
Scenarios reference sub-goals by name only, so without the catalogue a description, a judged
|
|
1387
|
+
sub-goal's claim and `_deterministic_names` all come back empty, each silently. A warning
|
|
1388
|
+
rather than an error, since failing the run is worse than the degraded reporting.
|
|
1389
|
+
"""
|
|
1390
|
+
catalogue = authoring / CATALOGUE
|
|
1391
|
+
if not catalogue.is_file():
|
|
1392
|
+
logger.warning(
|
|
1393
|
+
"no %s in %s: sub-goals will reach the platform without their descriptions or claims",
|
|
1394
|
+
CATALOGUE,
|
|
1395
|
+
authoring,
|
|
1396
|
+
)
|
|
1397
|
+
return []
|
|
1398
|
+
shutil.copy2(catalogue, staging / CATALOGUE)
|
|
1399
|
+
return [CATALOGUE]
|
|
1400
|
+
|
|
1401
|
+
|
|
1402
|
+
def _copy_chat_authoring(authoring: Path, staging: Path) -> list[str]:
|
|
1403
|
+
"""Adopt the frozen target/tool contract needed by response-carried HTTP tools.
|
|
1404
|
+
|
|
1405
|
+
These are authoring outputs, not repository inference performed by the hosted consumer. The
|
|
1406
|
+
producer validates and seals them exactly like scenario code. Voice bundles legitimately have
|
|
1407
|
+
none; HTTP chat bundles require a contract at pre-dial time and fail there with a typed error.
|
|
1408
|
+
"""
|
|
1409
|
+
adopted: list[str] = []
|
|
1410
|
+
contract = authoring / "contract.json"
|
|
1411
|
+
if contract.is_file():
|
|
1412
|
+
shutil.copy2(contract, staging / "contract.json")
|
|
1413
|
+
adopted.append("contract.json")
|
|
1414
|
+
handlers = authoring / "handlers"
|
|
1415
|
+
if handlers.is_dir():
|
|
1416
|
+
shutil.copytree(handlers, staging / "handlers")
|
|
1417
|
+
adopted.append("handlers/")
|
|
1418
|
+
prompt = authoring / "simulator_prompt.md"
|
|
1419
|
+
if prompt.is_file():
|
|
1420
|
+
shutil.copy2(prompt, staging / "simulator_prompt.md")
|
|
1421
|
+
adopted.append("simulator_prompt.md")
|
|
1422
|
+
return adopted
|
|
1423
|
+
|
|
1424
|
+
|
|
1425
|
+
def _compile_source_tool_handlers(contract: dict[str, Any], staging: Path) -> list[str]:
|
|
1426
|
+
"""Seal bindings for caller-executed tools that live in the submitted source.
|
|
1427
|
+
|
|
1428
|
+
HTTP/chat agents can return a tool request for the harness caller to execute. Contract
|
|
1429
|
+
discovery already records the repository's real import/construct entrypoint; hosted bundle
|
|
1430
|
+
authoring must carry that binding into the guest just as local world authoring does. This
|
|
1431
|
+
compiles only recorded source entrypoints and never supplies a replacement implementation.
|
|
1432
|
+
Explicit authoring handlers win, which preserves bindings that needed custom invocation code.
|
|
1433
|
+
"""
|
|
1434
|
+
raw_entries = contract.get("tool_entrypoints")
|
|
1435
|
+
if not isinstance(raw_entries, list):
|
|
1436
|
+
return []
|
|
1437
|
+
handlers = staging / "handlers"
|
|
1438
|
+
written: list[str] = []
|
|
1439
|
+
for raw in raw_entries:
|
|
1440
|
+
if not isinstance(raw, dict):
|
|
1441
|
+
continue
|
|
1442
|
+
try:
|
|
1443
|
+
entry = ToolEntry.model_validate(raw)
|
|
1444
|
+
except ValueError as exc:
|
|
1445
|
+
raise BundleAuthorError(f"contract_tool_entry_invalid: {exc}") from exc
|
|
1446
|
+
if entry.mode not in {"import", "construct"}:
|
|
1447
|
+
continue
|
|
1448
|
+
if not entry.module or not entry.callable:
|
|
1449
|
+
raise BundleAuthorError(
|
|
1450
|
+
f"contract_tool_entry_incomplete: {entry.tool}: "
|
|
1451
|
+
f"{entry.mode} requires module and callable"
|
|
1452
|
+
)
|
|
1453
|
+
if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", entry.tool):
|
|
1454
|
+
raise BundleAuthorError(f"contract_tool_name_unsafe: {entry.tool!r}")
|
|
1455
|
+
handlers.mkdir(parents=True, exist_ok=True)
|
|
1456
|
+
destination = handlers / f"{entry.tool}.py"
|
|
1457
|
+
if destination.exists():
|
|
1458
|
+
continue
|
|
1459
|
+
destination.write_text(
|
|
1460
|
+
_binding(
|
|
1461
|
+
module=entry.module,
|
|
1462
|
+
called=entry.callable,
|
|
1463
|
+
style="method" if entry.mode == "construct" else "function",
|
|
1464
|
+
first_arg=entry.first_arg,
|
|
1465
|
+
factory=entry.factory,
|
|
1466
|
+
),
|
|
1467
|
+
encoding="utf-8",
|
|
1468
|
+
)
|
|
1469
|
+
written.append(f"handlers/{entry.tool}.py")
|
|
1470
|
+
return written
|
|
1471
|
+
|
|
1472
|
+
|
|
1473
|
+
def _files(root: Path) -> list[BundleFileV2]:
|
|
1474
|
+
records: list[BundleFileV2] = []
|
|
1475
|
+
for path in sorted(root.rglob("*")):
|
|
1476
|
+
if path.is_dir() or path.name == BUNDLE_V2_MANIFEST:
|
|
1477
|
+
continue
|
|
1478
|
+
if path.is_symlink():
|
|
1479
|
+
raise BundleAuthorError(
|
|
1480
|
+
f"bundle_symlink_forbidden: {path.relative_to(root)}"
|
|
1481
|
+
)
|
|
1482
|
+
relative = path.relative_to(root)
|
|
1483
|
+
if any(part in _IGNORED_ARTIFACT_PARTS for part in relative.parts):
|
|
1484
|
+
continue
|
|
1485
|
+
content = path.read_bytes()
|
|
1486
|
+
records.append(
|
|
1487
|
+
BundleFileV2(
|
|
1488
|
+
path=relative.as_posix(),
|
|
1489
|
+
sha256=hashlib.sha256(content).hexdigest(),
|
|
1490
|
+
size=len(content),
|
|
1491
|
+
)
|
|
1492
|
+
)
|
|
1493
|
+
return records
|
|
1494
|
+
|
|
1495
|
+
|
|
1496
|
+
def author_bundle_v2(
|
|
1497
|
+
*,
|
|
1498
|
+
source: str | Path,
|
|
1499
|
+
job: HarnessJob,
|
|
1500
|
+
authoring: str | Path,
|
|
1501
|
+
output: str | Path,
|
|
1502
|
+
) -> EnvironmentBundleV2:
|
|
1503
|
+
source_root = Path(source).resolve()
|
|
1504
|
+
authoring_root = Path(authoring).resolve()
|
|
1505
|
+
output_root = Path(output).resolve()
|
|
1506
|
+
contract_modality: str | None = None
|
|
1507
|
+
contract_interface_kind: str | None = None
|
|
1508
|
+
contract_body: dict[str, Any] = {}
|
|
1509
|
+
contract_path = authoring_root / "contract.json"
|
|
1510
|
+
if contract_path.is_file():
|
|
1511
|
+
try:
|
|
1512
|
+
contract_body = json.loads(contract_path.read_text(encoding="utf-8"))
|
|
1513
|
+
except (OSError, ValueError) as exc:
|
|
1514
|
+
raise BundleAuthorError(
|
|
1515
|
+
f"contract_invalid: cannot read {contract_path}: {exc}"
|
|
1516
|
+
) from exc
|
|
1517
|
+
if not isinstance(contract_body, dict):
|
|
1518
|
+
raise BundleAuthorError("contract_invalid: contract.json must be an object")
|
|
1519
|
+
contract_modality = str(contract_body.get("modality") or "").strip().lower()
|
|
1520
|
+
runtime = contract_body.get("runtime")
|
|
1521
|
+
interface = runtime.get("interface") if isinstance(runtime, dict) else None
|
|
1522
|
+
if isinstance(interface, dict):
|
|
1523
|
+
contract_interface_kind = str(interface.get("kind") or "").strip().lower()
|
|
1524
|
+
elif (
|
|
1525
|
+
contract_modality == "chat"
|
|
1526
|
+
and _discover_callback_entrypoint(source_root) is not None
|
|
1527
|
+
):
|
|
1528
|
+
# The callback is a deterministic source property. Do not let a stochastic
|
|
1529
|
+
# authoring omission make the compiled adapter unreachable at call time: the
|
|
1530
|
+
# environment plan already discovers and exposes this same callback, so seal the
|
|
1531
|
+
# matching interface into the bundle's contract as part of compilation.
|
|
1532
|
+
runtime = dict(runtime) if isinstance(runtime, dict) else {}
|
|
1533
|
+
runtime["interface"] = {
|
|
1534
|
+
"kind": "callable",
|
|
1535
|
+
"protocol": "fi.alk",
|
|
1536
|
+
"path": "",
|
|
1537
|
+
"health_path": "",
|
|
1538
|
+
"include_tools": True,
|
|
1539
|
+
}
|
|
1540
|
+
contract_body = {**contract_body, "runtime": runtime}
|
|
1541
|
+
contract_interface_kind = "callable"
|
|
1542
|
+
plan = resolve_environment_plan(
|
|
1543
|
+
source_root,
|
|
1544
|
+
job,
|
|
1545
|
+
contract_modality=contract_modality,
|
|
1546
|
+
contract_interface_kind=contract_interface_kind,
|
|
1547
|
+
)
|
|
1548
|
+
provided_environment = {
|
|
1549
|
+
str(name).upper()
|
|
1550
|
+
for name in (job.metadata.get("environment_value_names", []) or [])
|
|
1551
|
+
}
|
|
1552
|
+
provided_environment.update(_declared_runtime_environment(source_root))
|
|
1553
|
+
for process in plan.processes:
|
|
1554
|
+
provided_environment.update(
|
|
1555
|
+
str(name).upper() for name in (getattr(process, "environment", None) or {})
|
|
1556
|
+
)
|
|
1557
|
+
credential_manifest = discover_credentials(
|
|
1558
|
+
source_root,
|
|
1559
|
+
secret_refs=job.agent.secret_refs,
|
|
1560
|
+
provided_environment=provided_environment,
|
|
1561
|
+
scan_paths={
|
|
1562
|
+
str(getattr(process, "working_directory", ".") or ".")
|
|
1563
|
+
for process in plan.processes
|
|
1564
|
+
if isinstance(process, SourceProcess)
|
|
1565
|
+
},
|
|
1566
|
+
)
|
|
1567
|
+
if not credential_manifest.ready:
|
|
1568
|
+
missing = sorted(
|
|
1569
|
+
item.environment_name for item in credential_manifest.missing_required
|
|
1570
|
+
)
|
|
1571
|
+
unsatisfied = sorted(
|
|
1572
|
+
choice.id
|
|
1573
|
+
for choice in credential_manifest.credential_choices
|
|
1574
|
+
if not choice.satisfied
|
|
1575
|
+
)
|
|
1576
|
+
details = [*(f"environment:{name}" for name in missing)]
|
|
1577
|
+
details.extend(f"credential_choice:{name}" for name in unsatisfied)
|
|
1578
|
+
raise BundleAuthorError(
|
|
1579
|
+
"target_runtime_configuration_missing: " + ", ".join(details)
|
|
1580
|
+
)
|
|
1581
|
+
output_root.parent.mkdir(parents=True, exist_ok=True)
|
|
1582
|
+
temporary = Path(
|
|
1583
|
+
tempfile.mkdtemp(prefix=f".{output_root.name}.", dir=output_root.parent)
|
|
1584
|
+
)
|
|
1585
|
+
try:
|
|
1586
|
+
_copy_scenarios(authoring_root, temporary, count=job.scenario_count)
|
|
1587
|
+
adopted_catalogue = _copy_sub_goal_catalogue(authoring_root, temporary)
|
|
1588
|
+
adopted_chat_files = _copy_chat_authoring(authoring_root, temporary)
|
|
1589
|
+
if "contract.json" in adopted_chat_files and contract_body:
|
|
1590
|
+
(temporary / "contract.json").write_text(
|
|
1591
|
+
json.dumps(contract_body, indent=2, sort_keys=True) + "\n",
|
|
1592
|
+
encoding="utf-8",
|
|
1593
|
+
)
|
|
1594
|
+
adopted_chat_files.extend(
|
|
1595
|
+
_compile_source_tool_handlers(contract_body, temporary)
|
|
1596
|
+
)
|
|
1597
|
+
if any(process.name == "tool-proxy" for process in plan.processes):
|
|
1598
|
+
generated = temporary / "generated" / "tool-proxy"
|
|
1599
|
+
generated.mkdir(parents=True)
|
|
1600
|
+
shutil.copy2(
|
|
1601
|
+
Path(__file__).with_name("tool_trace_proxy.py"),
|
|
1602
|
+
generated / "proxy.py",
|
|
1603
|
+
)
|
|
1604
|
+
seed_dir = temporary / "seed"
|
|
1605
|
+
seed_dir.mkdir()
|
|
1606
|
+
seed_path = seed_dir / "world.sql"
|
|
1607
|
+
prefix = (
|
|
1608
|
+
"CREATE TABLE IF NOT EXISTS harness_seed_sentinel (id text PRIMARY KEY);\n"
|
|
1609
|
+
"INSERT INTO harness_seed_sentinel(id) VALUES ('ready') ON CONFLICT DO NOTHING;\n"
|
|
1610
|
+
"CREATE TABLE IF NOT EXISTS _alk_tool_trace ("
|
|
1611
|
+
"id bigserial PRIMARY KEY, name text NOT NULL, arguments jsonb NOT NULL, "
|
|
1612
|
+
"result jsonb, ok boolean NOT NULL, error text, at double precision NOT NULL);\n"
|
|
1613
|
+
)
|
|
1614
|
+
schema, adopted_seed = _adopted_seed_sql(
|
|
1615
|
+
authoring_root,
|
|
1616
|
+
source=source_root,
|
|
1617
|
+
contract=contract_body,
|
|
1618
|
+
)
|
|
1619
|
+
seed_path.write_text(prefix + schema, encoding="utf-8")
|
|
1620
|
+
migrations = ["seed/world.sql"]
|
|
1621
|
+
store = StoreEntry(
|
|
1622
|
+
capability="world_db",
|
|
1623
|
+
migrations=migrations,
|
|
1624
|
+
seed_files=[],
|
|
1625
|
+
baseline=StoreBaseline(
|
|
1626
|
+
strategy=BaselineStrategy.TEMPLATE_DATABASE,
|
|
1627
|
+
inputs_digest=compute_inputs_digest(
|
|
1628
|
+
temporary,
|
|
1629
|
+
migrations,
|
|
1630
|
+
[],
|
|
1631
|
+
engine=ManagedEngine.POSTGRES,
|
|
1632
|
+
version="16",
|
|
1633
|
+
),
|
|
1634
|
+
),
|
|
1635
|
+
sentinel=Sentinel(
|
|
1636
|
+
query="SELECT id FROM harness_seed_sentinel WHERE id='ready'",
|
|
1637
|
+
expected="ready",
|
|
1638
|
+
),
|
|
1639
|
+
)
|
|
1640
|
+
provider_manifest: ProviderRepositoryManifest | None = None
|
|
1641
|
+
provider_import: ProviderImportSpec | None = None
|
|
1642
|
+
if job.agent.mode is ProviderExecutionMode.ENVIRONMENT_BACKED:
|
|
1643
|
+
provider_manifest = load_provider_manifest(
|
|
1644
|
+
source_root,
|
|
1645
|
+
str(job.agent.config.get("lifecycle_manifest") or "alk.yaml"),
|
|
1646
|
+
)
|
|
1647
|
+
declared = set(provider_manifest.provider.required_secrets)
|
|
1648
|
+
supplied = set(job.agent.secret_refs)
|
|
1649
|
+
missing = sorted(declared - supplied)
|
|
1650
|
+
if missing:
|
|
1651
|
+
raise BundleAuthorError(
|
|
1652
|
+
"provider_lifecycle_secrets_missing: " + ", ".join(missing)
|
|
1653
|
+
)
|
|
1654
|
+
elif job.agent.mode is ProviderExecutionMode.PROVIDER_IMPORT:
|
|
1655
|
+
connector = job.agent.connector.strip().lower()
|
|
1656
|
+
provider = "retell" if connector == "retell_chat" else connector
|
|
1657
|
+
secret_name = "VAPI_API_KEY" if provider == "vapi" else "RETELL_API_KEY"
|
|
1658
|
+
if secret_name not in job.agent.secret_refs:
|
|
1659
|
+
raise BundleAuthorError(
|
|
1660
|
+
f"provider_import_secret_missing: {secret_name}"
|
|
1661
|
+
)
|
|
1662
|
+
configured_capability = str(
|
|
1663
|
+
job.agent.config.get("public_capability") or ""
|
|
1664
|
+
).strip()
|
|
1665
|
+
http_capabilities = sorted(
|
|
1666
|
+
name
|
|
1667
|
+
for name, capability in plan.capabilities.items()
|
|
1668
|
+
if capability.protocol.value == "http"
|
|
1669
|
+
)
|
|
1670
|
+
if configured_capability:
|
|
1671
|
+
if configured_capability not in http_capabilities:
|
|
1672
|
+
raise BundleAuthorError(
|
|
1673
|
+
"provider_import_public_capability_invalid: "
|
|
1674
|
+
f"{configured_capability!r} is not an HTTP capability"
|
|
1675
|
+
)
|
|
1676
|
+
public_capability = configured_capability
|
|
1677
|
+
elif len(http_capabilities) == 1:
|
|
1678
|
+
public_capability = http_capabilities[0]
|
|
1679
|
+
else:
|
|
1680
|
+
raise BundleAuthorError(
|
|
1681
|
+
"provider_import_public_capability_ambiguous: configure public_capability; "
|
|
1682
|
+
f"found {http_capabilities}"
|
|
1683
|
+
)
|
|
1684
|
+
target_key = "assistant_id" if provider == "vapi" else "agent_id"
|
|
1685
|
+
provider_import = ProviderImportSpec(
|
|
1686
|
+
type=provider,
|
|
1687
|
+
source_target_id=str(job.agent.config[target_key]),
|
|
1688
|
+
public_capability=public_capability,
|
|
1689
|
+
environment_tools=sorted(
|
|
1690
|
+
{
|
|
1691
|
+
str(tool.get("name") or "").strip()
|
|
1692
|
+
for tool in contract_body.get("tools", [])
|
|
1693
|
+
if isinstance(tool, dict)
|
|
1694
|
+
and str(tool.get("name") or "").strip()
|
|
1695
|
+
}
|
|
1696
|
+
),
|
|
1697
|
+
event_path=str(
|
|
1698
|
+
job.agent.config.get("event_path") or "/provider/events"
|
|
1699
|
+
),
|
|
1700
|
+
tool_path=str(job.agent.config.get("tool_path") or "/provider/tools"),
|
|
1701
|
+
api_base_url=str(job.agent.config.get("provider_api_base_url") or "")
|
|
1702
|
+
or None,
|
|
1703
|
+
target_modality="chat" if connector == "retell_chat" else "voice",
|
|
1704
|
+
)
|
|
1705
|
+
|
|
1706
|
+
manifest = EnvironmentBundleV2(
|
|
1707
|
+
schema_version=BUNDLE_V2_SCHEMA_VERSION,
|
|
1708
|
+
digest="sha256:" + "0" * 64,
|
|
1709
|
+
name=str(job.metadata.get("name") or source_root.name),
|
|
1710
|
+
runtime=BundleRuntimeV2(
|
|
1711
|
+
kind=RuntimeKindV2.PROCESS,
|
|
1712
|
+
control_service=plan.control_service,
|
|
1713
|
+
evidence_seam=EvidenceSeam.TOOL_TRACE,
|
|
1714
|
+
),
|
|
1715
|
+
processes=list(plan.processes),
|
|
1716
|
+
seed=Seed(stores=[store]),
|
|
1717
|
+
capabilities=plan.capabilities,
|
|
1718
|
+
readiness=list(plan.readiness),
|
|
1719
|
+
files=_files(temporary),
|
|
1720
|
+
provenance=BundleProvenanceV2(
|
|
1721
|
+
source_kind=job.source.kind.value,
|
|
1722
|
+
repository=job.source.repository,
|
|
1723
|
+
commit=job.source.commit_sha,
|
|
1724
|
+
source_digest=source_fingerprint(source_root),
|
|
1725
|
+
generator="fi.alk.harness.bundle_author_v2",
|
|
1726
|
+
generator_version="2",
|
|
1727
|
+
adopted_files=["scenarios/"]
|
|
1728
|
+
+ adopted_catalogue
|
|
1729
|
+
+ adopted_seed
|
|
1730
|
+
+ adopted_chat_files,
|
|
1731
|
+
generated_files=["manifest.json", "seed/world.sql"],
|
|
1732
|
+
),
|
|
1733
|
+
metadata={
|
|
1734
|
+
"packaging": plan.packaging,
|
|
1735
|
+
"environment_plan_version": "2",
|
|
1736
|
+
**(
|
|
1737
|
+
{
|
|
1738
|
+
"provider_connect_only": {
|
|
1739
|
+
"connector": job.agent.connector.strip().lower()
|
|
1740
|
+
}
|
|
1741
|
+
}
|
|
1742
|
+
if job.agent.mode is ProviderExecutionMode.CONNECT_ONLY
|
|
1743
|
+
and job.source.kind is SourceKind.PROVIDER
|
|
1744
|
+
else {}
|
|
1745
|
+
),
|
|
1746
|
+
**(
|
|
1747
|
+
{
|
|
1748
|
+
"provider_lifecycle": provider_manifest.provider.model_dump(
|
|
1749
|
+
mode="json"
|
|
1750
|
+
)
|
|
1751
|
+
}
|
|
1752
|
+
if provider_manifest is not None
|
|
1753
|
+
else {}
|
|
1754
|
+
),
|
|
1755
|
+
**(
|
|
1756
|
+
{"provider_import": provider_import.model_dump(mode="json")}
|
|
1757
|
+
if provider_import is not None
|
|
1758
|
+
else {}
|
|
1759
|
+
),
|
|
1760
|
+
"environment_plan_hash": hashlib.sha256(
|
|
1761
|
+
json.dumps(
|
|
1762
|
+
{
|
|
1763
|
+
"packaging": plan.packaging,
|
|
1764
|
+
"control_service": plan.control_service,
|
|
1765
|
+
"processes": [
|
|
1766
|
+
item.model_dump(mode="json") for item in plan.processes
|
|
1767
|
+
],
|
|
1768
|
+
"capabilities": {
|
|
1769
|
+
key: value.model_dump(mode="json")
|
|
1770
|
+
for key, value in plan.capabilities.items()
|
|
1771
|
+
},
|
|
1772
|
+
},
|
|
1773
|
+
sort_keys=True,
|
|
1774
|
+
separators=(",", ":"),
|
|
1775
|
+
).encode()
|
|
1776
|
+
).hexdigest(),
|
|
1777
|
+
},
|
|
1778
|
+
)
|
|
1779
|
+
manifest = manifest.model_copy(update={"digest": seal_bundle_v2(manifest)})
|
|
1780
|
+
(temporary / BUNDLE_V2_MANIFEST).write_text(
|
|
1781
|
+
json.dumps(manifest.model_dump(mode="json"), indent=2, sort_keys=True)
|
|
1782
|
+
+ "\n",
|
|
1783
|
+
encoding="utf-8",
|
|
1784
|
+
)
|
|
1785
|
+
loaded = load_bundle_v2(temporary)
|
|
1786
|
+
preflight_bundle(
|
|
1787
|
+
temporary,
|
|
1788
|
+
loaded,
|
|
1789
|
+
parallelism=job.runtime.parallelism,
|
|
1790
|
+
secret_refs={
|
|
1791
|
+
alias: reference.purpose
|
|
1792
|
+
for alias, reference in job.agent.secret_refs.items()
|
|
1793
|
+
},
|
|
1794
|
+
)
|
|
1795
|
+
if output_root.exists():
|
|
1796
|
+
backup = output_root.with_name(output_root.name + ".previous")
|
|
1797
|
+
if backup.exists():
|
|
1798
|
+
shutil.rmtree(backup)
|
|
1799
|
+
output_root.rename(backup)
|
|
1800
|
+
temporary.rename(output_root)
|
|
1801
|
+
shutil.rmtree(backup)
|
|
1802
|
+
else:
|
|
1803
|
+
temporary.rename(output_root)
|
|
1804
|
+
return loaded
|
|
1805
|
+
except Exception:
|
|
1806
|
+
shutil.rmtree(temporary, ignore_errors=True)
|
|
1807
|
+
raise
|
|
1808
|
+
|
|
1809
|
+
|
|
1810
|
+
def _load_job(path: Path) -> HarnessJob:
|
|
1811
|
+
return HarnessJob.model_validate_json(path.read_text(encoding="utf-8"))
|
|
1812
|
+
|
|
1813
|
+
|
|
1814
|
+
def main(argv: list[str] | None = None) -> int:
|
|
1815
|
+
parser = argparse.ArgumentParser(prog="alk-bundle-author-v2")
|
|
1816
|
+
parser.add_argument("--job", type=Path, required=True)
|
|
1817
|
+
parser.add_argument("--source", type=Path, required=True)
|
|
1818
|
+
parser.add_argument("--authoring", type=Path, required=True)
|
|
1819
|
+
parser.add_argument("--output", type=Path, required=True)
|
|
1820
|
+
args = parser.parse_args(argv)
|
|
1821
|
+
author_bundle_v2(
|
|
1822
|
+
source=args.source,
|
|
1823
|
+
job=_load_job(args.job),
|
|
1824
|
+
authoring=args.authoring,
|
|
1825
|
+
output=args.output,
|
|
1826
|
+
)
|
|
1827
|
+
return 0
|
|
1828
|
+
|
|
1829
|
+
|
|
1830
|
+
if __name__ == "__main__": # pragma: no cover
|
|
1831
|
+
raise SystemExit(main())
|