agent-learning-kit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_learning_kit-0.1.0.dist-info/METADATA +381 -0
- agent_learning_kit-0.1.0.dist-info/RECORD +642 -0
- agent_learning_kit-0.1.0.dist-info/WHEEL +4 -0
- agent_learning_kit-0.1.0.dist-info/entry_points.txt +5 -0
- agent_learning_kit-0.1.0.dist-info/licenses/LICENSE +173 -0
- agent_learning_kit-0.1.0.dist-info/licenses/NOTICE +7 -0
- fi/__init__.py +5 -0
- fi/alk/__init__.py +57 -0
- fi/alk/_facade.py +31 -0
- fi/alk/_module_alias.py +68 -0
- fi/alk/_paths.py +14 -0
- fi/alk/_schema.py +522 -0
- fi/alk/actions.py +727 -0
- fi/alk/bench/__init__.py +517 -0
- fi/alk/bench/_codeexec.py +213 -0
- fi/alk/bench/_coding.py +215 -0
- fi/alk/bench/_docker.py +237 -0
- fi/alk/bench/_grader.py +286 -0
- fi/alk/bench/_pull.py +212 -0
- fi/alk/bench/_voice.py +147 -0
- fi/alk/capabilities.py +627 -0
- fi/alk/cli.py +6396 -0
- fi/alk/config.py +130 -0
- fi/alk/cua_loop.py +562 -0
- fi/alk/evals.py +2351 -0
- fi/alk/extensions.py +163 -0
- fi/alk/harness/ARCHITECTURE.md +231 -0
- fi/alk/harness/DESIGN.md +246 -0
- fi/alk/harness/ENVIRONMENT_CONFORMANCE.md +127 -0
- fi/alk/harness/HOW-IT-WORKS.md +297 -0
- fi/alk/harness/IMPLEMENTATION_AND_VALIDATION_STATUS.md +229 -0
- fi/alk/harness/README.md +417 -0
- fi/alk/harness/__init__.py +77 -0
- fi/alk/harness/__main__.py +3 -0
- fi/alk/harness/amend.py +312 -0
- fi/alk/harness/artifacts.py +319 -0
- fi/alk/harness/authoring_entrypoint.py +189 -0
- fi/alk/harness/authoring_runtime_validation.py +267 -0
- fi/alk/harness/backends/README.md +43 -0
- fi/alk/harness/backends/__init__.py +122 -0
- fi/alk/harness/backends/base.py +241 -0
- fi/alk/harness/backends/claude.py +211 -0
- fi/alk/harness/backends/files.py +182 -0
- fi/alk/harness/backends/vertex_gemini.py +457 -0
- fi/alk/harness/background_noise.py +95 -0
- fi/alk/harness/build.py +385 -0
- fi/alk/harness/bundle.py +593 -0
- fi/alk/harness/bundle_author_v2.py +1831 -0
- fi/alk/harness/bundle_v2.py +719 -0
- fi/alk/harness/call_runner.py +1440 -0
- fi/alk/harness/callback_http_adapter.py +111 -0
- fi/alk/harness/catalogue.py +287 -0
- fi/alk/harness/chat.py +428 -0
- fi/alk/harness/chat_call_runner.py +506 -0
- fi/alk/harness/checks.py +136 -0
- fi/alk/harness/cli.py +1354 -0
- fi/alk/harness/config.py +338 -0
- fi/alk/harness/contract.py +718 -0
- fi/alk/harness/credentials.py +674 -0
- fi/alk/harness/data/persona_vocabulary.json +111 -0
- fi/alk/harness/environment.py +99 -0
- fi/alk/harness/environment_plan.py +168 -0
- fi/alk/harness/events.py +125 -0
- fi/alk/harness/executor.py +304 -0
- fi/alk/harness/folder.py +234 -0
- fi/alk/harness/generated_runtime.py +815 -0
- fi/alk/harness/github.py +72 -0
- fi/alk/harness/hosted_authoring_entrypoint.py +183 -0
- fi/alk/harness/hosted_entrypoint.py +2402 -0
- fi/alk/harness/hosted_scheduler.py +2218 -0
- fi/alk/harness/job.py +426 -0
- fi/alk/harness/judge.py +184 -0
- fi/alk/harness/livekit_source.py +50 -0
- fi/alk/harness/livekit_tool_trace_bootstrap.py +71 -0
- fi/alk/harness/observability.py +208 -0
- fi/alk/harness/outbound.py +3252 -0
- fi/alk/harness/packaging.py +515 -0
- fi/alk/harness/persona_guides.py +157 -0
- fi/alk/harness/platform.py +692 -0
- fi/alk/harness/process_preflight.py +764 -0
- fi/alk/harness/process_runtime.py +5670 -0
- fi/alk/harness/prove.py +425 -0
- fi/alk/harness/provider_import.py +703 -0
- fi/alk/harness/provider_lifecycle.py +392 -0
- fi/alk/harness/provision.py +2896 -0
- fi/alk/harness/reception.py +147 -0
- fi/alk/harness/retell_chat_call_runner.py +373 -0
- fi/alk/harness/run/__init__.py +296 -0
- fi/alk/harness/run/alk.py +184 -0
- fi/alk/harness/run/call.py +162 -0
- fi/alk/harness/run/conversation.py +264 -0
- fi/alk/harness/run/data/voices_by_language_and_gender.json +693 -0
- fi/alk/harness/run/evidence.py +195 -0
- fi/alk/harness/run/grade.py +598 -0
- fi/alk/harness/run/live.py +297 -0
- fi/alk/harness/run/models.py +56 -0
- fi/alk/harness/run/platform_evals.py +227 -0
- fi/alk/harness/run/sdk_voice.py +130 -0
- fi/alk/harness/run/simulation.py +1209 -0
- fi/alk/harness/run/stage.py +91 -0
- fi/alk/harness/run/targets.py +508 -0
- fi/alk/harness/run/tools.py +601 -0
- fi/alk/harness/run/voice.py +340 -0
- fi/alk/harness/runtime.py +172 -0
- fi/alk/harness/sandbox_server.py +2011 -0
- fi/alk/harness/sandbox_worker.py +44 -0
- fi/alk/harness/scenario.py +1048 -0
- fi/alk/harness/scenario_source.py +879 -0
- fi/alk/harness/scenario_tools.py +1143 -0
- fi/alk/harness/scenarios.py +915 -0
- fi/alk/harness/secrets.py +168 -0
- fi/alk/harness/service_catalog.py +97 -0
- fi/alk/harness/session.py +391 -0
- fi/alk/harness/sessions.py +372 -0
- fi/alk/harness/simulator.py +76 -0
- fi/alk/harness/simulator_voice.py +928 -0
- fi/alk/harness/skills/build-environment/SKILL.md +538 -0
- fi/alk/harness/skills/harness.md +131 -0
- fi/alk/harness/skills/kinds/chat.md +48 -0
- fi/alk/harness/skills/kinds/voice-voicemail.md +63 -0
- fi/alk/harness/skills/kinds/voice.md +59 -0
- fi/alk/harness/skills/plan-suite/SKILL.md +103 -0
- fi/alk/harness/skills/provision-environment/SKILL.md +136 -0
- fi/alk/harness/skills/run-scenarios/SKILL.md +112 -0
- fi/alk/harness/skills/understand-agent/SKILL.md +251 -0
- fi/alk/harness/skills/write-scenarios/SKILL.md +606 -0
- fi/alk/harness/skills/write-scenarios/references/refusals.md +28 -0
- fi/alk/harness/skills/write-scenarios/references/world-api.md +92 -0
- fi/alk/harness/source_data_invariants.py +444 -0
- fi/alk/harness/source_tool_evidence.py +79 -0
- fi/alk/harness/sources.py +253 -0
- fi/alk/harness/spend.py +140 -0
- fi/alk/harness/tool_trace_proxy.py +104 -0
- fi/alk/harness/tools.py +1018 -0
- fi/alk/harness/understand.py +169 -0
- fi/alk/harness/voicemail_audio.py +74 -0
- fi/alk/harness/world/__init__.py +33 -0
- fi/alk/harness/world/errors.py +68 -0
- fi/alk/harness/world/expectations.py +91 -0
- fi/alk/harness/world/handle.py +538 -0
- fi/alk/harness/world/kinds.py +196 -0
- fi/alk/harness/world/mutate.py +186 -0
- fi/alk/harness/world/probe.py +413 -0
- fi/alk/harness/world/provision.py +511 -0
- fi/alk/harness/world/provisioned.py +191 -0
- fi/alk/harness/world/runtime.py +616 -0
- fi/alk/harness/world/snapshot.py +288 -0
- fi/alk/harness/world/stores/__init__.py +305 -0
- fi/alk/harness/world/stores/container.py +215 -0
- fi/alk/harness/world/stores/inprocess.py +346 -0
- fi/alk/harness/world/stores/postgres.py +481 -0
- fi/alk/harness/world/stores/prove.py +202 -0
- fi/alk/harness/world/stores/sqlite.py +245 -0
- fi/alk/harness/world/stores/written.py +182 -0
- fi/alk/harness/world/tools.py +1516 -0
- fi/alk/harness/world/workspace.py +144 -0
- fi/alk/image_loop.py +453 -0
- fi/alk/image_perturb.py +241 -0
- fi/alk/improve.py +274 -0
- fi/alk/live/__init__.py +154 -0
- fi/alk/live/_attribution.py +184 -0
- fi/alk/live/_capture.py +264 -0
- fi/alk/live/_codec.py +391 -0
- fi/alk/live/_contract.py +134 -0
- fi/alk/live/_loopback.py +316 -0
- fi/alk/live/_perturb.py +449 -0
- fi/alk/live/_runner.py +386 -0
- fi/alk/live/_stats.py +561 -0
- fi/alk/live/_transcript.py +240 -0
- fi/alk/live/_workers/__init__.py +9 -0
- fi/alk/live/_workers/a2a_worker.py +316 -0
- fi/alk/live/_workers/langgraph_worker.py +217 -0
- fi/alk/live/_workers/livekit_worker.py +207 -0
- fi/alk/live/_workers/mcp_loopback_server.py +46 -0
- fi/alk/live/_workers/mcp_worker.py +158 -0
- fi/alk/live/_workers/pipecat_worker.py +189 -0
- fi/alk/live/a2a_lane.py +138 -0
- fi/alk/live/langgraph_lane.py +339 -0
- fi/alk/live/livekit_lane.py +376 -0
- fi/alk/live/mcp_lane.py +172 -0
- fi/alk/live/pipecat_lane.py +341 -0
- fi/alk/live/voice_redteam.py +494 -0
- fi/alk/loss.py +306 -0
- fi/alk/optimize.py +36260 -0
- fi/alk/practice/__init__.py +51 -0
- fi/alk/practice/_assess.py +103 -0
- fi/alk/practice/_budget.py +81 -0
- fi/alk/practice/_calibrate.py +69 -0
- fi/alk/practice/_capstone.py +86 -0
- fi/alk/practice/_contract.py +91 -0
- fi/alk/practice/_diagnose.py +79 -0
- fi/alk/practice/_drill.py +196 -0
- fi/alk/practice/_experiment.py +720 -0
- fi/alk/practice/_schedule.py +102 -0
- fi/alk/practice/_store.py +194 -0
- fi/alk/practice/_trainer.py +245 -0
- fi/alk/practice/_update.py +125 -0
- fi/alk/redteam.py +2621 -0
- fi/alk/rewardhack.py +237 -0
- fi/alk/simulate.py +10351 -0
- fi/alk/studio/__init__.py +82 -0
- fi/alk/studio/_bias.py +314 -0
- fi/alk/studio/_calibration.py +522 -0
- fi/alk/studio/_coverage.py +262 -0
- fi/alk/studio/_download.py +665 -0
- fi/alk/studio/_fidelity_attack.py +114 -0
- fi/alk/studio/_generate.py +652 -0
- fi/alk/studio/_library.py +370 -0
- fi/alk/studio/_scan.py +134 -0
- fi/alk/studio/_upgrade.py +42 -0
- fi/alk/studio/_vendor.py +172 -0
- fi/alk/suite.py +4200 -0
- fi/alk/tasks.py +828 -0
- fi/alk/telemetry/__init__.py +149 -0
- fi/alk/telemetry/_contract.py +141 -0
- fi/alk/telemetry/_emit.py +182 -0
- fi/alk/telemetry/_ledger.py +296 -0
- fi/alk/telemetry/_queue.py +127 -0
- fi/alk/telemetry/_row.py +294 -0
- fi/alk/telemetry/_run.py +233 -0
- fi/alk/telemetry/_sync.py +193 -0
- fi/alk/telemetry/_url.py +119 -0
- fi/alk/trinity.py +49397 -0
- fi/alk/voice_loop.py +174 -0
- fi/api/__init__.py +1 -0
- fi/api/auth.py +137 -0
- fi/api/types.py +29 -0
- fi/cli/__init__.py +9 -0
- fi/cli/assertions/__init__.py +25 -0
- fi/cli/assertions/conditions.py +76 -0
- fi/cli/assertions/evaluator.py +286 -0
- fi/cli/assertions/exit_codes.py +20 -0
- fi/cli/assertions/parser.py +131 -0
- fi/cli/assertions/reporter.py +194 -0
- fi/cli/commands/__init__.py +9 -0
- fi/cli/commands/config.py +165 -0
- fi/cli/commands/export.py +208 -0
- fi/cli/commands/init.py +112 -0
- fi/cli/commands/list_cmd.py +213 -0
- fi/cli/commands/run.py +486 -0
- fi/cli/commands/validate.py +173 -0
- fi/cli/commands/view.py +424 -0
- fi/cli/config/__init__.py +6 -0
- fi/cli/config/defaults.py +206 -0
- fi/cli/config/loader.py +155 -0
- fi/cli/config/schema.py +174 -0
- fi/cli/main.py +78 -0
- fi/cli/output/__init__.py +6 -0
- fi/cli/output/formatters.py +106 -0
- fi/cli/output/reporters.py +46 -0
- fi/cli/storage/__init__.py +5 -0
- fi/cli/storage/run_history.py +249 -0
- fi/cli/utils/__init__.py +5 -0
- fi/cli/utils/console.py +44 -0
- fi/evals/__init__.py +131 -0
- fi/evals/autoeval/__init__.py +137 -0
- fi/evals/autoeval/analyzer.py +211 -0
- fi/evals/autoeval/config.py +244 -0
- fi/evals/autoeval/export.py +213 -0
- fi/evals/autoeval/interactive.py +283 -0
- fi/evals/autoeval/pipeline.py +625 -0
- fi/evals/autoeval/prompts.py +139 -0
- fi/evals/autoeval/recommender.py +242 -0
- fi/evals/autoeval/rules.py +589 -0
- fi/evals/autoeval/templates.py +299 -0
- fi/evals/autoeval/types.py +232 -0
- fi/evals/core/__init__.py +16 -0
- fi/evals/core/cloud_registry.py +184 -0
- fi/evals/core/engines.py +368 -0
- fi/evals/core/evaluate.py +319 -0
- fi/evals/core/judge_prompt.py +90 -0
- fi/evals/core/prompt_generator.py +83 -0
- fi/evals/core/registry.py +57 -0
- fi/evals/core/result.py +55 -0
- fi/evals/evaluator.py +721 -0
- fi/evals/execution.py +168 -0
- fi/evals/feedback/__init__.py +32 -0
- fi/evals/feedback/calibrator.py +160 -0
- fi/evals/feedback/collector.py +214 -0
- fi/evals/feedback/hooks.py +81 -0
- fi/evals/feedback/retriever.py +128 -0
- fi/evals/feedback/store.py +272 -0
- fi/evals/feedback/types.py +99 -0
- fi/evals/framework/README.md +79 -0
- fi/evals/framework/__init__.py +267 -0
- fi/evals/framework/backends/Dockerfile.eval-runner +33 -0
- fi/evals/framework/backends/__init__.py +99 -0
- fi/evals/framework/backends/_container.py +141 -0
- fi/evals/framework/backends/_utils.py +145 -0
- fi/evals/framework/backends/base.py +223 -0
- fi/evals/framework/backends/celery_backend.py +417 -0
- fi/evals/framework/backends/celery_worker.py +78 -0
- fi/evals/framework/backends/kubernetes_backend.py +665 -0
- fi/evals/framework/backends/ray_backend.py +521 -0
- fi/evals/framework/backends/temporal.py +350 -0
- fi/evals/framework/backends/temporal_worker.py +126 -0
- fi/evals/framework/backends/thread_pool.py +286 -0
- fi/evals/framework/context.py +258 -0
- fi/evals/framework/enrichment.py +306 -0
- fi/evals/framework/evals/__init__.py +68 -0
- fi/evals/framework/evals/agentic.py +399 -0
- fi/evals/framework/evals/builder.py +609 -0
- fi/evals/framework/evals/semantic.py +142 -0
- fi/evals/framework/evaluator.py +647 -0
- fi/evals/framework/evaluators/__init__.py +22 -0
- fi/evals/framework/evaluators/blocking.py +347 -0
- fi/evals/framework/evaluators/non_blocking.py +577 -0
- fi/evals/framework/propagation.py +421 -0
- fi/evals/framework/protocols.py +385 -0
- fi/evals/framework/registry.py +370 -0
- fi/evals/framework/resilience/__init__.py +150 -0
- fi/evals/framework/resilience/circuit_breaker.py +309 -0
- fi/evals/framework/resilience/degradation.py +355 -0
- fi/evals/framework/resilience/health.py +505 -0
- fi/evals/framework/resilience/rate_limiter.py +228 -0
- fi/evals/framework/resilience/retry.py +274 -0
- fi/evals/framework/resilience/types.py +288 -0
- fi/evals/framework/resilience/wrapper.py +433 -0
- fi/evals/framework/types.py +218 -0
- fi/evals/guardrails/README.md +915 -0
- fi/evals/guardrails/__init__.py +96 -0
- fi/evals/guardrails/backends/__init__.py +43 -0
- fi/evals/guardrails/backends/azure.py +361 -0
- fi/evals/guardrails/backends/base.py +88 -0
- fi/evals/guardrails/backends/generic_llm.py +163 -0
- fi/evals/guardrails/backends/granite.py +216 -0
- fi/evals/guardrails/backends/llamaguard.py +221 -0
- fi/evals/guardrails/backends/local_base.py +479 -0
- fi/evals/guardrails/backends/openai.py +365 -0
- fi/evals/guardrails/backends/qwen.py +170 -0
- fi/evals/guardrails/backends/shieldgemma.py +154 -0
- fi/evals/guardrails/backends/turing.py +235 -0
- fi/evals/guardrails/backends/vllm_client.py +321 -0
- fi/evals/guardrails/backends/wildguard.py +188 -0
- fi/evals/guardrails/base.py +888 -0
- fi/evals/guardrails/config.py +221 -0
- fi/evals/guardrails/discovery.py +243 -0
- fi/evals/guardrails/gateway.py +437 -0
- fi/evals/guardrails/registry.py +231 -0
- fi/evals/guardrails/scanners/__init__.py +127 -0
- fi/evals/guardrails/scanners/base.py +191 -0
- fi/evals/guardrails/scanners/code_injection.py +243 -0
- fi/evals/guardrails/scanners/eval_delegate.py +574 -0
- fi/evals/guardrails/scanners/invisible_chars.py +351 -0
- fi/evals/guardrails/scanners/jailbreak.py +412 -0
- fi/evals/guardrails/scanners/language.py +288 -0
- fi/evals/guardrails/scanners/pipeline.py +260 -0
- fi/evals/guardrails/scanners/regex.py +311 -0
- fi/evals/guardrails/scanners/secrets.py +274 -0
- fi/evals/guardrails/scanners/topics.py +649 -0
- fi/evals/guardrails/scanners/urls.py +341 -0
- fi/evals/guardrails/types.py +96 -0
- fi/evals/llm/__init__.py +3 -0
- fi/evals/llm/base_llm_provider.py +35 -0
- fi/evals/llm/providers/litellm.py +70 -0
- fi/evals/local/__init__.py +90 -0
- fi/evals/local/evaluator.py +690 -0
- fi/evals/local/execution_mode.py +121 -0
- fi/evals/local/llm.py +489 -0
- fi/evals/local/metrics/__init__.py +19 -0
- fi/evals/local/registry.py +360 -0
- fi/evals/manager.py +1018 -0
- fi/evals/manager_types.py +362 -0
- fi/evals/metrics/__init__.py +185 -0
- fi/evals/metrics/agents/__init__.py +74 -0
- fi/evals/metrics/agents/metrics.py +693 -0
- fi/evals/metrics/agents/report.py +36463 -0
- fi/evals/metrics/agents/types.py +160 -0
- fi/evals/metrics/base_llm_metric.py +111 -0
- fi/evals/metrics/base_metric.py +138 -0
- fi/evals/metrics/code_security/__init__.py +305 -0
- fi/evals/metrics/code_security/analyzer.py +985 -0
- fi/evals/metrics/code_security/benchmarks/__init__.py +73 -0
- fi/evals/metrics/code_security/benchmarks/builtin.py +750 -0
- fi/evals/metrics/code_security/benchmarks/loader.py +580 -0
- fi/evals/metrics/code_security/benchmarks/types.py +308 -0
- fi/evals/metrics/code_security/detectors/__init__.py +186 -0
- fi/evals/metrics/code_security/detectors/base.py +394 -0
- fi/evals/metrics/code_security/detectors/cryptography.py +345 -0
- fi/evals/metrics/code_security/detectors/injection.py +744 -0
- fi/evals/metrics/code_security/detectors/secrets.py +287 -0
- fi/evals/metrics/code_security/detectors/serialization.py +192 -0
- fi/evals/metrics/code_security/joint_metrics.py +588 -0
- fi/evals/metrics/code_security/judges/__init__.py +83 -0
- fi/evals/metrics/code_security/judges/base.py +238 -0
- fi/evals/metrics/code_security/judges/dual_judge.py +534 -0
- fi/evals/metrics/code_security/judges/llm_judge.py +301 -0
- fi/evals/metrics/code_security/judges/pattern_judge.py +515 -0
- fi/evals/metrics/code_security/metrics.py +388 -0
- fi/evals/metrics/code_security/modes/__init__.py +63 -0
- fi/evals/metrics/code_security/modes/adversarial.py +284 -0
- fi/evals/metrics/code_security/modes/autocomplete.py +198 -0
- fi/evals/metrics/code_security/modes/base.py +283 -0
- fi/evals/metrics/code_security/modes/instruct.py +253 -0
- fi/evals/metrics/code_security/modes/repair.py +230 -0
- fi/evals/metrics/code_security/reports/__init__.py +57 -0
- fi/evals/metrics/code_security/reports/generator.py +404 -0
- fi/evals/metrics/code_security/reports/leaderboard.py +509 -0
- fi/evals/metrics/code_security/types.py +534 -0
- fi/evals/metrics/function_calling/__init__.py +34 -0
- fi/evals/metrics/function_calling/metrics.py +573 -0
- fi/evals/metrics/function_calling/types.py +87 -0
- fi/evals/metrics/hallucination/__init__.py +54 -0
- fi/evals/metrics/hallucination/detector.py +149 -0
- fi/evals/metrics/hallucination/metrics.py +390 -0
- fi/evals/metrics/hallucination/nli.py +253 -0
- fi/evals/metrics/hallucination/sentinel.py +106 -0
- fi/evals/metrics/hallucination/types.py +132 -0
- fi/evals/metrics/heuristics/aggregation_metrics.py +85 -0
- fi/evals/metrics/heuristics/json_metrics.py +87 -0
- fi/evals/metrics/heuristics/similarity_metrics.py +375 -0
- fi/evals/metrics/heuristics/string_metrics.py +391 -0
- fi/evals/metrics/llm_as_judges/__init__.py +17 -0
- fi/evals/metrics/llm_as_judges/custom_judge/metric.py +112 -0
- fi/evals/metrics/llm_as_judges/custom_judge/prompts.py +26 -0
- fi/evals/metrics/llm_as_judges/types.py +48 -0
- fi/evals/metrics/rag/__init__.py +111 -0
- fi/evals/metrics/rag/advanced/__init__.py +14 -0
- fi/evals/metrics/rag/advanced/multi_hop.py +283 -0
- fi/evals/metrics/rag/advanced/source_attribution.py +344 -0
- fi/evals/metrics/rag/generation/__init__.py +17 -0
- fi/evals/metrics/rag/generation/answer_relevancy.py +176 -0
- fi/evals/metrics/rag/generation/context_utilization.py +245 -0
- fi/evals/metrics/rag/generation/faithfulness.py +241 -0
- fi/evals/metrics/rag/generation/groundedness.py +131 -0
- fi/evals/metrics/rag/rag_score.py +277 -0
- fi/evals/metrics/rag/retrieval/__init__.py +20 -0
- fi/evals/metrics/rag/retrieval/context_entity_recall.py +124 -0
- fi/evals/metrics/rag/retrieval/context_precision.py +158 -0
- fi/evals/metrics/rag/retrieval/context_recall.py +106 -0
- fi/evals/metrics/rag/retrieval/noise_sensitivity.py +163 -0
- fi/evals/metrics/rag/retrieval/ranking.py +261 -0
- fi/evals/metrics/rag/types.py +100 -0
- fi/evals/metrics/rag/utils/__init__.py +62 -0
- fi/evals/metrics/rag/utils/claims.py +189 -0
- fi/evals/metrics/rag/utils/entities.py +244 -0
- fi/evals/metrics/rag/utils/nli.py +92 -0
- fi/evals/metrics/rag/utils/similarity.py +345 -0
- fi/evals/metrics/structured/__init__.py +114 -0
- fi/evals/metrics/structured/field_completeness.py +313 -0
- fi/evals/metrics/structured/hierarchy_score.py +366 -0
- fi/evals/metrics/structured/json_validation.py +190 -0
- fi/evals/metrics/structured/schema_compliance.py +280 -0
- fi/evals/metrics/structured/structured_output_score.py +298 -0
- fi/evals/metrics/structured/types.py +108 -0
- fi/evals/metrics/structured/validators/__init__.py +30 -0
- fi/evals/metrics/structured/validators/base.py +189 -0
- fi/evals/metrics/structured/validators/json_validator.py +196 -0
- fi/evals/metrics/structured/validators/pydantic_validator.py +178 -0
- fi/evals/metrics/structured/validators/yaml_validator.py +248 -0
- fi/evals/otel/__init__.py +266 -0
- fi/evals/otel/config.py +400 -0
- fi/evals/otel/conventions.py +463 -0
- fi/evals/otel/enrichment.py +371 -0
- fi/evals/otel/instrumentors/__init__.py +140 -0
- fi/evals/otel/instrumentors/anthropic.py +517 -0
- fi/evals/otel/instrumentors/base.py +382 -0
- fi/evals/otel/instrumentors/openai.py +673 -0
- fi/evals/otel/processors/__init__.py +36 -0
- fi/evals/otel/processors/base.py +473 -0
- fi/evals/otel/processors/cost.py +445 -0
- fi/evals/otel/processors/evaluation.py +559 -0
- fi/evals/otel/processors/llm.py +462 -0
- fi/evals/otel/tracer.py +506 -0
- fi/evals/otel/types.py +232 -0
- fi/evals/otel_utils.py +23 -0
- fi/evals/protect.py +671 -0
- fi/evals/protect_input_adapter.py +154 -0
- fi/evals/streaming/__init__.py +88 -0
- fi/evals/streaming/buffer.py +213 -0
- fi/evals/streaming/evaluator.py +551 -0
- fi/evals/streaming/policy.py +307 -0
- fi/evals/streaming/scorers.py +368 -0
- fi/evals/streaming/types.py +238 -0
- fi/evals/templates.py +472 -0
- fi/evals/types.py +156 -0
- fi/opt/__init__.py +221 -0
- fi/opt/_objective_scoring.py +85 -0
- fi/opt/base/__init__.py +11 -0
- fi/opt/base/base_generator.py +33 -0
- fi/opt/base/base_mapper.py +26 -0
- fi/opt/base/base_optimizer.py +45 -0
- fi/opt/base/evaluator.py +211 -0
- fi/opt/components.py +3095 -0
- fi/opt/datamappers/__init__.py +3 -0
- fi/opt/datamappers/basic_mapper.py +40 -0
- fi/opt/deployment.py +1021 -0
- fi/opt/evidence.py +4332 -0
- fi/opt/generators/__init__.py +3 -0
- fi/opt/generators/litellm.py +66 -0
- fi/opt/integrations/__init__.py +23 -0
- fi/opt/integrations/generative_suite.py +410 -0
- fi/opt/integrations/simulate.py +1313 -0
- fi/opt/mutations.py +771 -0
- fi/opt/observability.py +4639 -0
- fi/opt/optimizer_trace.py +889 -0
- fi/opt/optimizers/__init__.py +80 -0
- fi/opt/optimizers/agent.py +331 -0
- fi/opt/optimizers/agent_bandit.py +392 -0
- fi/opt/optimizers/agent_curriculum.py +635 -0
- fi/opt/optimizers/agent_evolution.py +894 -0
- fi/opt/optimizers/agent_feedback.py +1863 -0
- fi/opt/optimizers/agent_pareto.py +547 -0
- fi/opt/optimizers/agent_social_memory.py +1113 -0
- fi/opt/optimizers/agent_tpe.py +321 -0
- fi/opt/optimizers/bayesian_search.py +449 -0
- fi/opt/optimizers/council.py +2075 -0
- fi/opt/optimizers/futureagi_replay.py +799 -0
- fi/opt/optimizers/gepa.py +322 -0
- fi/opt/optimizers/metaprompt.py +243 -0
- fi/opt/optimizers/promptwizard.py +417 -0
- fi/opt/optimizers/protegi.py +329 -0
- fi/opt/optimizers/random_search.py +224 -0
- fi/opt/research.py +518 -0
- fi/opt/simulation.py +260 -0
- fi/opt/targets.py +232 -0
- fi/opt/types.py +66 -0
- fi/opt/utils/__init__.py +4 -0
- fi/opt/utils/early_stopping.py +266 -0
- fi/opt/utils/setup_logging.py +82 -0
- fi/simulate/__init__.py +540 -0
- fi/simulate/_hashing.py +35 -0
- fi/simulate/_logging.py +10 -0
- fi/simulate/adapters.py +87 -0
- fi/simulate/agent/__init__.py +120 -0
- fi/simulate/agent/browser.py +658 -0
- fi/simulate/agent/definition.py +587 -0
- fi/simulate/agent/frameworks.py +3528 -0
- fi/simulate/agent/generic.py +8286 -0
- fi/simulate/agent/import_probe.py +227 -0
- fi/simulate/agent/memory.py +905 -0
- fi/simulate/agent/mocks.py +101 -0
- fi/simulate/agent/multi_agent.py +361 -0
- fi/simulate/agent/orchestration.py +903 -0
- fi/simulate/agent/realtime.py +665 -0
- fi/simulate/agent/wrapper.py +99 -0
- fi/simulate/agent/wrappers/__init__.py +18 -0
- fi/simulate/agent/wrappers/anthropic.py +62 -0
- fi/simulate/agent/wrappers/gemini.py +65 -0
- fi/simulate/agent/wrappers/http.py +404 -0
- fi/simulate/agent/wrappers/langchain.py +80 -0
- fi/simulate/agent/wrappers/openai.py +75 -0
- fi/simulate/agent/wrappers/websocket.py +326 -0
- fi/simulate/artifacts/__init__.py +11 -0
- fi/simulate/artifacts/manifest.py +62 -0
- fi/simulate/cli.py +20560 -0
- fi/simulate/endpoints/__init__.py +45 -0
- fi/simulate/endpoints/_http_actor.py +73 -0
- fi/simulate/endpoints/actor_sources.py +243 -0
- fi/simulate/endpoints/base.py +107 -0
- fi/simulate/endpoints/builtins.py +10 -0
- fi/simulate/endpoints/callable.py +95 -0
- fi/simulate/endpoints/http.py +76 -0
- fi/simulate/endpoints/livekit.py +138 -0
- fi/simulate/endpoints/originators.py +132 -0
- fi/simulate/endpoints/profiles.py +348 -0
- fi/simulate/endpoints/retell.py +633 -0
- fi/simulate/endpoints/vapi.py +205 -0
- fi/simulate/endpoints/websocket.py +76 -0
- fi/simulate/environment.py +33026 -0
- fi/simulate/environments/__init__.py +11 -0
- fi/simulate/environments/base.py +73 -0
- fi/simulate/environments/chat.py +697 -0
- fi/simulate/environments/voice.py +212 -0
- fi/simulate/evaluation/__init__.py +4 -0
- fi/simulate/evaluation/ai_eval.py +227 -0
- fi/simulate/evidence/__init__.py +35 -0
- fi/simulate/evidence/base.py +59 -0
- fi/simulate/evidence/caller_observed.py +50 -0
- fi/simulate/evidence/livekit_instrumentation.py +51 -0
- fi/simulate/evidence/livekit_room.py +50 -0
- fi/simulate/evidence/otel.py +49 -0
- fi/simulate/evidence/providers/__init__.py +24 -0
- fi/simulate/evidence/providers/base.py +61 -0
- fi/simulate/evidence/providers/retell.py +376 -0
- fi/simulate/evidence/providers/vapi.py +426 -0
- fi/simulate/hosted/__init__.py +32 -0
- fi/simulate/hosted/child_entrypoint.py +306 -0
- fi/simulate/hosted/job.py +150 -0
- fi/simulate/hosted/targets.py +53 -0
- fi/simulate/instrumentation/__init__.py +5 -0
- fi/simulate/instrumentation/livekit/__init__.py +122 -0
- fi/simulate/manifest.py +1033 -0
- fi/simulate/matrix_cli.py +165 -0
- fi/simulate/realtime/__init__.py +40 -0
- fi/simulate/realtime/events.py +107 -0
- fi/simulate/realtime/media.py +61 -0
- fi/simulate/realtime/session.py +91 -0
- fi/simulate/recording/__init__.py +5 -0
- fi/simulate/recording/room_recorder.py +326 -0
- fi/simulate/registry.py +185 -0
- fi/simulate/results/__init__.py +9 -0
- fi/simulate/results/base.py +18 -0
- fi/simulate/results/filesystem.py +71 -0
- fi/simulate/results/futureagi.py +1340 -0
- fi/simulate/runtime/__init__.py +85 -0
- fi/simulate/runtime/capabilities.py +40 -0
- fi/simulate/runtime/events.py +63 -0
- fi/simulate/runtime/failures.py +25 -0
- fi/simulate/runtime/ids.py +34 -0
- fi/simulate/runtime/plan.py +70 -0
- fi/simulate/runtime/planner.py +102 -0
- fi/simulate/runtime/report.py +174 -0
- fi/simulate/runtime/run.py +75 -0
- fi/simulate/runtime/runner.py +333 -0
- fi/simulate/runtime/spec.py +186 -0
- fi/simulate/simulation/__init__.py +30 -0
- fi/simulate/simulation/behavior_policy.py +425 -0
- fi/simulate/simulation/bridge/__init__.py +9 -0
- fi/simulate/simulation/bridge/audio.py +29 -0
- fi/simulate/simulation/bridge/connector.py +46 -0
- fi/simulate/simulation/bridge/livekit.py +252 -0
- fi/simulate/simulation/bridge/retell.py +188 -0
- fi/simulate/simulation/bridge/vapi.py +177 -0
- fi/simulate/simulation/contract.py +419 -0
- fi/simulate/simulation/engines/__init__.py +12 -0
- fi/simulate/simulation/engines/base.py +21 -0
- fi/simulate/simulation/engines/cloud.py +517 -0
- fi/simulate/simulation/engines/livekit.py +4167 -0
- fi/simulate/simulation/engines/local_text.py +89 -0
- fi/simulate/simulation/fidelity.py +374 -0
- fi/simulate/simulation/gemini_tts_stream.py +110 -0
- fi/simulate/simulation/generator.py +91 -0
- fi/simulate/simulation/goal_machine.py +185 -0
- fi/simulate/simulation/livekit_models.py +467 -0
- fi/simulate/simulation/matrix.py +170 -0
- fi/simulate/simulation/models.py +279 -0
- fi/simulate/simulation/runner.py +153 -0
- fi/simulate/simulation/synthetic.py +880 -0
- fi/simulate/simulation/voice_prompt.py +502 -0
- fi/simulate/simulator/__init__.py +55 -0
- fi/simulate/simulator/builtins.py +53 -0
- fi/simulate/suite.py +1288 -0
- fi/simulate/utils/routes.py +164 -0
- fi/simulate/voice.py +225 -0
- fi/simulate/voice_cli.py +182 -0
- fi/utils/__init__.py +1 -0
- fi/utils/constants.py +14 -0
- fi/utils/errors.py +200 -0
- fi/utils/executor.py +26 -0
- fi/utils/routes.py +119 -0
- fi/utils/utils.py +17 -0
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
{
|
|
2
|
+
"GenderChoices": [
|
|
3
|
+
"male",
|
|
4
|
+
"female"
|
|
5
|
+
],
|
|
6
|
+
"AgeGroupChoices": [
|
|
7
|
+
"18-25",
|
|
8
|
+
"25-32",
|
|
9
|
+
"32-40",
|
|
10
|
+
"40-50",
|
|
11
|
+
"50-60",
|
|
12
|
+
"60+"
|
|
13
|
+
],
|
|
14
|
+
"LocationChoices": [
|
|
15
|
+
"United States",
|
|
16
|
+
"Canada",
|
|
17
|
+
"United Kingdom",
|
|
18
|
+
"Australia",
|
|
19
|
+
"India"
|
|
20
|
+
],
|
|
21
|
+
"ProfessionChoices": [
|
|
22
|
+
"Student",
|
|
23
|
+
"Teacher",
|
|
24
|
+
"Engineer",
|
|
25
|
+
"Doctor",
|
|
26
|
+
"Nurse",
|
|
27
|
+
"Business Owner",
|
|
28
|
+
"Manager",
|
|
29
|
+
"Sales Representative",
|
|
30
|
+
"Customer Service",
|
|
31
|
+
"Technician",
|
|
32
|
+
"Consultant",
|
|
33
|
+
"Accountant",
|
|
34
|
+
"Marketing Professional",
|
|
35
|
+
"Retired",
|
|
36
|
+
"Homemaker",
|
|
37
|
+
"Freelancer",
|
|
38
|
+
"Other"
|
|
39
|
+
],
|
|
40
|
+
"PersonalityChoices": [
|
|
41
|
+
"Friendly and cooperative",
|
|
42
|
+
"Professional and formal",
|
|
43
|
+
"Cautious and skeptical",
|
|
44
|
+
"Impatient and direct",
|
|
45
|
+
"Detail-oriented",
|
|
46
|
+
"Easy-going",
|
|
47
|
+
"Anxious",
|
|
48
|
+
"Confident",
|
|
49
|
+
"Analytical",
|
|
50
|
+
"Emotional",
|
|
51
|
+
"Reserved",
|
|
52
|
+
"Talkative"
|
|
53
|
+
],
|
|
54
|
+
"CommunicationStyleChoices": [
|
|
55
|
+
"Direct and concise",
|
|
56
|
+
"Detailed and elaborate",
|
|
57
|
+
"Casual and friendly",
|
|
58
|
+
"Formal and polite",
|
|
59
|
+
"Technical",
|
|
60
|
+
"Simple and clear",
|
|
61
|
+
"Questioning",
|
|
62
|
+
"Assertive",
|
|
63
|
+
"Passive",
|
|
64
|
+
"Collaborative"
|
|
65
|
+
],
|
|
66
|
+
"AccentChoices": [
|
|
67
|
+
"American",
|
|
68
|
+
"Australian",
|
|
69
|
+
"Indian",
|
|
70
|
+
"Canadian",
|
|
71
|
+
"Neutral"
|
|
72
|
+
],
|
|
73
|
+
"LanguageChoices": [
|
|
74
|
+
"Arabic",
|
|
75
|
+
"Bulgarian",
|
|
76
|
+
"Chinese Simplified",
|
|
77
|
+
"Czech",
|
|
78
|
+
"Danish",
|
|
79
|
+
"Dutch",
|
|
80
|
+
"English",
|
|
81
|
+
"Finnish",
|
|
82
|
+
"French",
|
|
83
|
+
"German",
|
|
84
|
+
"Greek",
|
|
85
|
+
"Hindi",
|
|
86
|
+
"Hungarian",
|
|
87
|
+
"Indonesian",
|
|
88
|
+
"Italian",
|
|
89
|
+
"Japanese",
|
|
90
|
+
"Korean",
|
|
91
|
+
"Malay",
|
|
92
|
+
"Norwegian",
|
|
93
|
+
"Polish",
|
|
94
|
+
"Portuguese",
|
|
95
|
+
"Romanian",
|
|
96
|
+
"Russian",
|
|
97
|
+
"Slovak",
|
|
98
|
+
"Spanish",
|
|
99
|
+
"Swedish",
|
|
100
|
+
"Turkish",
|
|
101
|
+
"Ukrainian",
|
|
102
|
+
"Vietnamese"
|
|
103
|
+
],
|
|
104
|
+
"ConversationSpeedChoices": [
|
|
105
|
+
"0.5",
|
|
106
|
+
"0.75",
|
|
107
|
+
"1.0",
|
|
108
|
+
"1.25",
|
|
109
|
+
"1.5"
|
|
110
|
+
]
|
|
111
|
+
}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"""The small ``fi.simulate.environment`` contract a generated world implements.
|
|
2
|
+
|
|
3
|
+
The parent package currently imports optional LiveKit code from ``fi.simulate.__init__`` before a
|
|
4
|
+
submodule can be imported. Keep the normal integration when that import is available, while
|
|
5
|
+
letting the standalone harness run its deterministic world and scenario gates from the base
|
|
6
|
+
editable dependency alone.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from abc import ABC
|
|
12
|
+
from typing import Any, Mapping
|
|
13
|
+
|
|
14
|
+
try:
|
|
15
|
+
from fi.simulate.environment import (
|
|
16
|
+
EnvironmentAdapter,
|
|
17
|
+
EnvironmentSnapshot,
|
|
18
|
+
ToolExecutionResult,
|
|
19
|
+
)
|
|
20
|
+
except (
|
|
21
|
+
ImportError
|
|
22
|
+
): # pragma: no cover - exercised only without the parent's optional extras
|
|
23
|
+
from pydantic import BaseModel, Field
|
|
24
|
+
|
|
25
|
+
class EnvironmentSnapshot(BaseModel):
|
|
26
|
+
"""State and tool definitions published by a local environment."""
|
|
27
|
+
|
|
28
|
+
tools: list[dict[str, Any]] = Field(default_factory=list)
|
|
29
|
+
artifacts: list[Any] = Field(default_factory=list)
|
|
30
|
+
events: list[Any] = Field(default_factory=list)
|
|
31
|
+
state: dict[str, Any] = Field(default_factory=dict)
|
|
32
|
+
metadata: dict[str, Any] = Field(default_factory=dict)
|
|
33
|
+
|
|
34
|
+
class ToolExecutionResult(BaseModel):
|
|
35
|
+
"""Result from executing one local tool call."""
|
|
36
|
+
|
|
37
|
+
tool_call_id: str | None = None
|
|
38
|
+
tool_name: str
|
|
39
|
+
content: str
|
|
40
|
+
result: Any = None
|
|
41
|
+
success: bool = True
|
|
42
|
+
error: str | None = None
|
|
43
|
+
state_updates: dict[str, Any] = Field(default_factory=dict)
|
|
44
|
+
artifacts: list[Any] = Field(default_factory=list)
|
|
45
|
+
events: list[Any] = Field(default_factory=list)
|
|
46
|
+
metadata: dict[str, Any] = Field(default_factory=dict)
|
|
47
|
+
|
|
48
|
+
def to_tool_message(self) -> dict[str, Any]:
|
|
49
|
+
return {
|
|
50
|
+
"role": "tool",
|
|
51
|
+
"tool_call_id": self.tool_call_id or self.tool_name,
|
|
52
|
+
"content": self.content,
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
class EnvironmentAdapter(ABC):
|
|
56
|
+
"""The subset of the parent environment protocol the harness needs."""
|
|
57
|
+
|
|
58
|
+
name = "environment"
|
|
59
|
+
|
|
60
|
+
def reset(self, **_context: Any) -> EnvironmentSnapshot:
|
|
61
|
+
return EnvironmentSnapshot()
|
|
62
|
+
|
|
63
|
+
def observe(self, **_context: Any) -> EnvironmentSnapshot:
|
|
64
|
+
return EnvironmentSnapshot()
|
|
65
|
+
|
|
66
|
+
def handle_tool_call(
|
|
67
|
+
self, tool_call: Mapping[str, Any], **_context: Any
|
|
68
|
+
) -> ToolExecutionResult | None:
|
|
69
|
+
return None
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
# Compatibility exports: the environment artifact was split into focused catalogue and
|
|
73
|
+
# simulator modules. Existing ALK integrations may keep importing the original surface.
|
|
74
|
+
from .catalogue import ( # noqa: E402
|
|
75
|
+
Catalogue,
|
|
76
|
+
SubGoal,
|
|
77
|
+
load_catalogue,
|
|
78
|
+
save_catalogue,
|
|
79
|
+
validate_sub_goal,
|
|
80
|
+
)
|
|
81
|
+
from .simulator import ( # noqa: E402
|
|
82
|
+
load_simulator_prompt,
|
|
83
|
+
save_simulator_prompt,
|
|
84
|
+
validate_simulator_prompt,
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
__all__ = [
|
|
88
|
+
"Catalogue",
|
|
89
|
+
"EnvironmentAdapter",
|
|
90
|
+
"EnvironmentSnapshot",
|
|
91
|
+
"SubGoal",
|
|
92
|
+
"ToolExecutionResult",
|
|
93
|
+
"load_catalogue",
|
|
94
|
+
"load_simulator_prompt",
|
|
95
|
+
"save_catalogue",
|
|
96
|
+
"save_simulator_prompt",
|
|
97
|
+
"validate_simulator_prompt",
|
|
98
|
+
"validate_sub_goal",
|
|
99
|
+
]
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""Canonical, immutable description of an ALK environment.
|
|
2
|
+
|
|
3
|
+
The bundle is the portable payload. The environment plan is the stable decision record inside
|
|
4
|
+
that payload: which source revision was admitted, which runtime document will be used, which
|
|
5
|
+
services/capabilities exist, and how readiness is proven. It deliberately excludes ephemeral
|
|
6
|
+
project names, allocated host ports, and resolved secret values.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import hashlib
|
|
12
|
+
import json
|
|
13
|
+
import re
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
from pydantic import BaseModel, Field, JsonValue, model_validator
|
|
17
|
+
|
|
18
|
+
from .bundle import (
|
|
19
|
+
BundleRuntime,
|
|
20
|
+
Capability,
|
|
21
|
+
EnvironmentBundle,
|
|
22
|
+
ReadinessProbe,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
ENVIRONMENT_PLAN_SCHEMA_VERSION = "futureagi.environment-plan.v1"
|
|
26
|
+
ENVIRONMENT_PLAN_FILE = "environment-plan.json"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class EnvironmentPlanError(RuntimeError):
|
|
30
|
+
"""The frozen plan is absent, corrupt, or inconsistent with its bundle."""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class PlanSource(BaseModel):
|
|
34
|
+
kind: str
|
|
35
|
+
repository: str | None = None
|
|
36
|
+
commit_sha: str | None = None
|
|
37
|
+
source_digest: str
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class EnvironmentPlan(BaseModel):
|
|
41
|
+
schema_version: str = ENVIRONMENT_PLAN_SCHEMA_VERSION
|
|
42
|
+
digest: str
|
|
43
|
+
source: PlanSource
|
|
44
|
+
runtime: BundleRuntime
|
|
45
|
+
services: list[str] = Field(default_factory=list)
|
|
46
|
+
capabilities: dict[str, Capability] = Field(default_factory=dict)
|
|
47
|
+
readiness: list[ReadinessProbe] = Field(default_factory=list)
|
|
48
|
+
configuration_names: list[str] = Field(default_factory=list)
|
|
49
|
+
metadata: dict[str, JsonValue] = Field(default_factory=dict)
|
|
50
|
+
|
|
51
|
+
@model_validator(mode="after")
|
|
52
|
+
def _valid_plan(self) -> "EnvironmentPlan":
|
|
53
|
+
if self.schema_version != ENVIRONMENT_PLAN_SCHEMA_VERSION:
|
|
54
|
+
raise ValueError(
|
|
55
|
+
f"environment_plan_schema_unsupported: {self.schema_version}"
|
|
56
|
+
)
|
|
57
|
+
if not re.fullmatch(r"sha256:[0-9a-f]{64}", self.digest):
|
|
58
|
+
raise ValueError("environment_plan_digest_invalid")
|
|
59
|
+
missing = [
|
|
60
|
+
probe.capability
|
|
61
|
+
for probe in self.readiness
|
|
62
|
+
if probe.capability not in self.capabilities
|
|
63
|
+
]
|
|
64
|
+
if missing:
|
|
65
|
+
raise ValueError(
|
|
66
|
+
"environment_plan_readiness_capability_missing: "
|
|
67
|
+
+ ", ".join(sorted(set(missing)))
|
|
68
|
+
)
|
|
69
|
+
if self.configuration_names != sorted(set(self.configuration_names)):
|
|
70
|
+
raise ValueError("environment_plan_configuration_names_not_canonical")
|
|
71
|
+
return self
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _digest(raw: dict) -> str:
|
|
75
|
+
canonical = dict(raw)
|
|
76
|
+
canonical.pop("digest", None)
|
|
77
|
+
encoded = json.dumps(
|
|
78
|
+
canonical, sort_keys=True, separators=(",", ":"), ensure_ascii=False
|
|
79
|
+
).encode("utf-8")
|
|
80
|
+
return "sha256:" + hashlib.sha256(encoded).hexdigest()
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def create_environment_plan(
|
|
84
|
+
*,
|
|
85
|
+
source: PlanSource,
|
|
86
|
+
runtime: BundleRuntime,
|
|
87
|
+
services: list[str],
|
|
88
|
+
capabilities: dict[str, Capability],
|
|
89
|
+
readiness: list[ReadinessProbe],
|
|
90
|
+
metadata: dict[str, JsonValue] | None = None,
|
|
91
|
+
) -> EnvironmentPlan:
|
|
92
|
+
"""Create the same plan digest for the same admitted environment decision."""
|
|
93
|
+
names = sorted(
|
|
94
|
+
{
|
|
95
|
+
capability.configuration_name
|
|
96
|
+
for capability in capabilities.values()
|
|
97
|
+
if capability.configuration_name
|
|
98
|
+
}
|
|
99
|
+
)
|
|
100
|
+
raw = {
|
|
101
|
+
"schema_version": ENVIRONMENT_PLAN_SCHEMA_VERSION,
|
|
102
|
+
"digest": "sha256:" + "0" * 64,
|
|
103
|
+
"source": source.model_dump(mode="json"),
|
|
104
|
+
"runtime": runtime.model_dump(mode="json"),
|
|
105
|
+
"services": sorted(set(services)),
|
|
106
|
+
"capabilities": {
|
|
107
|
+
key: capabilities[key].model_dump(mode="json")
|
|
108
|
+
for key in sorted(capabilities)
|
|
109
|
+
},
|
|
110
|
+
"readiness": [
|
|
111
|
+
probe.model_dump(mode="json") for probe in readiness
|
|
112
|
+
],
|
|
113
|
+
"configuration_names": names,
|
|
114
|
+
"metadata": metadata or {},
|
|
115
|
+
}
|
|
116
|
+
raw["digest"] = _digest(raw)
|
|
117
|
+
return EnvironmentPlan.model_validate(raw)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def write_environment_plan(root: str | Path, plan: EnvironmentPlan) -> Path:
|
|
121
|
+
target = Path(root) / ENVIRONMENT_PLAN_FILE
|
|
122
|
+
temporary = target.with_name(f".{target.name}.tmp")
|
|
123
|
+
temporary.write_text(plan.model_dump_json(indent=2) + "\n", encoding="utf-8")
|
|
124
|
+
temporary.replace(target)
|
|
125
|
+
return target
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def load_environment_plan(
|
|
129
|
+
root: str | Path, *, bundle: EnvironmentBundle | None = None
|
|
130
|
+
) -> EnvironmentPlan:
|
|
131
|
+
target = Path(root) / ENVIRONMENT_PLAN_FILE
|
|
132
|
+
if not target.is_file():
|
|
133
|
+
raise EnvironmentPlanError(f"environment_plan_missing: {target}")
|
|
134
|
+
try:
|
|
135
|
+
plan = EnvironmentPlan.model_validate_json(target.read_text(encoding="utf-8"))
|
|
136
|
+
except (OSError, ValueError) as exc:
|
|
137
|
+
raise EnvironmentPlanError(f"environment_plan_invalid: {exc}") from exc
|
|
138
|
+
if _digest(plan.model_dump(mode="json")) != plan.digest:
|
|
139
|
+
raise EnvironmentPlanError("environment_plan_digest_mismatch")
|
|
140
|
+
if bundle is not None:
|
|
141
|
+
expected = create_environment_plan(
|
|
142
|
+
source=PlanSource(
|
|
143
|
+
kind=bundle.provenance.source_kind,
|
|
144
|
+
repository=bundle.provenance.repository,
|
|
145
|
+
commit_sha=bundle.provenance.commit,
|
|
146
|
+
source_digest=bundle.provenance.source_digest,
|
|
147
|
+
),
|
|
148
|
+
runtime=bundle.runtime,
|
|
149
|
+
services=bundle.services,
|
|
150
|
+
capabilities=bundle.capabilities,
|
|
151
|
+
readiness=bundle.readiness,
|
|
152
|
+
metadata=plan.metadata,
|
|
153
|
+
)
|
|
154
|
+
if expected.digest != plan.digest:
|
|
155
|
+
raise EnvironmentPlanError("environment_plan_bundle_mismatch")
|
|
156
|
+
return plan
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
__all__ = [
|
|
160
|
+
"ENVIRONMENT_PLAN_FILE",
|
|
161
|
+
"ENVIRONMENT_PLAN_SCHEMA_VERSION",
|
|
162
|
+
"EnvironmentPlan",
|
|
163
|
+
"EnvironmentPlanError",
|
|
164
|
+
"PlanSource",
|
|
165
|
+
"create_environment_plan",
|
|
166
|
+
"load_environment_plan",
|
|
167
|
+
"write_environment_plan",
|
|
168
|
+
]
|
fi/alk/harness/events.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""Durable harness progress delivery for both local and hosted execution."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from threading import RLock
|
|
9
|
+
from typing import Protocol
|
|
10
|
+
|
|
11
|
+
from fi.simulate.runtime.events import CanonicalEvent
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class EventTransport(Protocol):
|
|
15
|
+
"""Platform boundary. Implementations must be idempotent by ``event_id``."""
|
|
16
|
+
|
|
17
|
+
def send(self, run_id: str, events: list[CanonicalEvent]) -> set[str]: ...
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class EventOutbox:
|
|
21
|
+
"""Append-only event journal with a durable acknowledgement cursor.
|
|
22
|
+
|
|
23
|
+
Every event lands locally before a network attempt. A killed local CLI or hosted sandbox can
|
|
24
|
+
reopen the outbox and retry without losing progress; the platform deduplicates event IDs.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
def __init__(self, root: str | Path, run_id: str) -> None:
|
|
28
|
+
self.root = Path(root).expanduser().resolve() / run_id
|
|
29
|
+
self.root.mkdir(parents=True, exist_ok=True)
|
|
30
|
+
self.events_path = self.root / "harness-events.jsonl"
|
|
31
|
+
self.acked_path = self.root / "harness-events.acked.json"
|
|
32
|
+
self._lock = RLock()
|
|
33
|
+
|
|
34
|
+
def append(self, event: CanonicalEvent) -> None:
|
|
35
|
+
with self._lock, self.events_path.open("a", encoding="utf-8") as stream:
|
|
36
|
+
stream.write(event.model_dump_json(exclude_none=True) + "\n")
|
|
37
|
+
stream.flush()
|
|
38
|
+
os.fsync(stream.fileno())
|
|
39
|
+
|
|
40
|
+
def pending(self) -> list[CanonicalEvent]:
|
|
41
|
+
acknowledged = self.acknowledged()
|
|
42
|
+
if not self.events_path.exists():
|
|
43
|
+
return []
|
|
44
|
+
events: list[CanonicalEvent] = []
|
|
45
|
+
with self._lock, self.events_path.open(encoding="utf-8") as stream:
|
|
46
|
+
for number, line in enumerate(stream, start=1):
|
|
47
|
+
if not line.strip():
|
|
48
|
+
continue
|
|
49
|
+
try:
|
|
50
|
+
event = CanonicalEvent.model_validate_json(line)
|
|
51
|
+
except ValueError as exc:
|
|
52
|
+
raise ValueError(
|
|
53
|
+
f"event_outbox_corrupt: line {number}: {exc}"
|
|
54
|
+
) from exc
|
|
55
|
+
if event.event_id not in acknowledged:
|
|
56
|
+
events.append(event)
|
|
57
|
+
return events
|
|
58
|
+
|
|
59
|
+
def acknowledged(self) -> set[str]:
|
|
60
|
+
if not self.acked_path.exists():
|
|
61
|
+
return set()
|
|
62
|
+
try:
|
|
63
|
+
value = json.loads(self.acked_path.read_text(encoding="utf-8"))
|
|
64
|
+
except (OSError, ValueError) as exc:
|
|
65
|
+
raise ValueError(f"event_acknowledgements_corrupt: {exc}") from exc
|
|
66
|
+
return {str(item) for item in value.get("event_ids", [])}
|
|
67
|
+
|
|
68
|
+
def acknowledge(self, event_ids: set[str]) -> None:
|
|
69
|
+
if not event_ids:
|
|
70
|
+
return
|
|
71
|
+
with self._lock:
|
|
72
|
+
all_ids = self.acknowledged() | event_ids
|
|
73
|
+
temporary = self.acked_path.with_suffix(".tmp")
|
|
74
|
+
temporary.write_text(
|
|
75
|
+
json.dumps({"event_ids": sorted(all_ids)}, separators=(",", ":"))
|
|
76
|
+
+ "\n",
|
|
77
|
+
encoding="utf-8",
|
|
78
|
+
)
|
|
79
|
+
temporary.replace(self.acked_path)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class BufferedEventSink:
|
|
83
|
+
"""Write-through local sink with best-effort, retryable platform delivery."""
|
|
84
|
+
|
|
85
|
+
def __init__(
|
|
86
|
+
self,
|
|
87
|
+
outbox: EventOutbox,
|
|
88
|
+
transport: EventTransport | None = None,
|
|
89
|
+
*,
|
|
90
|
+
batch_size: int = 100,
|
|
91
|
+
) -> None:
|
|
92
|
+
self.outbox = outbox
|
|
93
|
+
self.transport = transport
|
|
94
|
+
self.batch_size = max(1, batch_size)
|
|
95
|
+
|
|
96
|
+
def write(self, event: CanonicalEvent) -> None:
|
|
97
|
+
self.outbox.append(event)
|
|
98
|
+
if self.transport is not None:
|
|
99
|
+
try:
|
|
100
|
+
self.flush(limit=self.batch_size)
|
|
101
|
+
except Exception:
|
|
102
|
+
# Execution is authoritative; telemetry is retried from the durable outbox.
|
|
103
|
+
pass
|
|
104
|
+
|
|
105
|
+
def flush(self, *, limit: int | None = None) -> int:
|
|
106
|
+
if self.transport is None:
|
|
107
|
+
return 0
|
|
108
|
+
pending = self.outbox.pending()
|
|
109
|
+
if limit is not None:
|
|
110
|
+
pending = pending[:limit]
|
|
111
|
+
delivered = 0
|
|
112
|
+
for start in range(0, len(pending), self.batch_size):
|
|
113
|
+
batch = pending[start : start + self.batch_size]
|
|
114
|
+
accepted = self.transport.send(batch[0].run_id, batch)
|
|
115
|
+
expected = {event.event_id for event in batch}
|
|
116
|
+
# A transport may acknowledge a subset. Everything else remains pending.
|
|
117
|
+
acknowledged = expected & set(accepted)
|
|
118
|
+
self.outbox.acknowledge(acknowledged)
|
|
119
|
+
delivered += len(acknowledged)
|
|
120
|
+
if acknowledged != expected:
|
|
121
|
+
break
|
|
122
|
+
return delivered
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
__all__ = ["BufferedEventSink", "EventOutbox", "EventTransport"]
|