agent-learning-kit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_learning_kit-0.1.0.dist-info/METADATA +381 -0
- agent_learning_kit-0.1.0.dist-info/RECORD +642 -0
- agent_learning_kit-0.1.0.dist-info/WHEEL +4 -0
- agent_learning_kit-0.1.0.dist-info/entry_points.txt +5 -0
- agent_learning_kit-0.1.0.dist-info/licenses/LICENSE +173 -0
- agent_learning_kit-0.1.0.dist-info/licenses/NOTICE +7 -0
- fi/__init__.py +5 -0
- fi/alk/__init__.py +57 -0
- fi/alk/_facade.py +31 -0
- fi/alk/_module_alias.py +68 -0
- fi/alk/_paths.py +14 -0
- fi/alk/_schema.py +522 -0
- fi/alk/actions.py +727 -0
- fi/alk/bench/__init__.py +517 -0
- fi/alk/bench/_codeexec.py +213 -0
- fi/alk/bench/_coding.py +215 -0
- fi/alk/bench/_docker.py +237 -0
- fi/alk/bench/_grader.py +286 -0
- fi/alk/bench/_pull.py +212 -0
- fi/alk/bench/_voice.py +147 -0
- fi/alk/capabilities.py +627 -0
- fi/alk/cli.py +6396 -0
- fi/alk/config.py +130 -0
- fi/alk/cua_loop.py +562 -0
- fi/alk/evals.py +2351 -0
- fi/alk/extensions.py +163 -0
- fi/alk/harness/ARCHITECTURE.md +231 -0
- fi/alk/harness/DESIGN.md +246 -0
- fi/alk/harness/ENVIRONMENT_CONFORMANCE.md +127 -0
- fi/alk/harness/HOW-IT-WORKS.md +297 -0
- fi/alk/harness/IMPLEMENTATION_AND_VALIDATION_STATUS.md +229 -0
- fi/alk/harness/README.md +417 -0
- fi/alk/harness/__init__.py +77 -0
- fi/alk/harness/__main__.py +3 -0
- fi/alk/harness/amend.py +312 -0
- fi/alk/harness/artifacts.py +319 -0
- fi/alk/harness/authoring_entrypoint.py +189 -0
- fi/alk/harness/authoring_runtime_validation.py +267 -0
- fi/alk/harness/backends/README.md +43 -0
- fi/alk/harness/backends/__init__.py +122 -0
- fi/alk/harness/backends/base.py +241 -0
- fi/alk/harness/backends/claude.py +211 -0
- fi/alk/harness/backends/files.py +182 -0
- fi/alk/harness/backends/vertex_gemini.py +457 -0
- fi/alk/harness/background_noise.py +95 -0
- fi/alk/harness/build.py +385 -0
- fi/alk/harness/bundle.py +593 -0
- fi/alk/harness/bundle_author_v2.py +1831 -0
- fi/alk/harness/bundle_v2.py +719 -0
- fi/alk/harness/call_runner.py +1440 -0
- fi/alk/harness/callback_http_adapter.py +111 -0
- fi/alk/harness/catalogue.py +287 -0
- fi/alk/harness/chat.py +428 -0
- fi/alk/harness/chat_call_runner.py +506 -0
- fi/alk/harness/checks.py +136 -0
- fi/alk/harness/cli.py +1354 -0
- fi/alk/harness/config.py +338 -0
- fi/alk/harness/contract.py +718 -0
- fi/alk/harness/credentials.py +674 -0
- fi/alk/harness/data/persona_vocabulary.json +111 -0
- fi/alk/harness/environment.py +99 -0
- fi/alk/harness/environment_plan.py +168 -0
- fi/alk/harness/events.py +125 -0
- fi/alk/harness/executor.py +304 -0
- fi/alk/harness/folder.py +234 -0
- fi/alk/harness/generated_runtime.py +815 -0
- fi/alk/harness/github.py +72 -0
- fi/alk/harness/hosted_authoring_entrypoint.py +183 -0
- fi/alk/harness/hosted_entrypoint.py +2402 -0
- fi/alk/harness/hosted_scheduler.py +2218 -0
- fi/alk/harness/job.py +426 -0
- fi/alk/harness/judge.py +184 -0
- fi/alk/harness/livekit_source.py +50 -0
- fi/alk/harness/livekit_tool_trace_bootstrap.py +71 -0
- fi/alk/harness/observability.py +208 -0
- fi/alk/harness/outbound.py +3252 -0
- fi/alk/harness/packaging.py +515 -0
- fi/alk/harness/persona_guides.py +157 -0
- fi/alk/harness/platform.py +692 -0
- fi/alk/harness/process_preflight.py +764 -0
- fi/alk/harness/process_runtime.py +5670 -0
- fi/alk/harness/prove.py +425 -0
- fi/alk/harness/provider_import.py +703 -0
- fi/alk/harness/provider_lifecycle.py +392 -0
- fi/alk/harness/provision.py +2896 -0
- fi/alk/harness/reception.py +147 -0
- fi/alk/harness/retell_chat_call_runner.py +373 -0
- fi/alk/harness/run/__init__.py +296 -0
- fi/alk/harness/run/alk.py +184 -0
- fi/alk/harness/run/call.py +162 -0
- fi/alk/harness/run/conversation.py +264 -0
- fi/alk/harness/run/data/voices_by_language_and_gender.json +693 -0
- fi/alk/harness/run/evidence.py +195 -0
- fi/alk/harness/run/grade.py +598 -0
- fi/alk/harness/run/live.py +297 -0
- fi/alk/harness/run/models.py +56 -0
- fi/alk/harness/run/platform_evals.py +227 -0
- fi/alk/harness/run/sdk_voice.py +130 -0
- fi/alk/harness/run/simulation.py +1209 -0
- fi/alk/harness/run/stage.py +91 -0
- fi/alk/harness/run/targets.py +508 -0
- fi/alk/harness/run/tools.py +601 -0
- fi/alk/harness/run/voice.py +340 -0
- fi/alk/harness/runtime.py +172 -0
- fi/alk/harness/sandbox_server.py +2011 -0
- fi/alk/harness/sandbox_worker.py +44 -0
- fi/alk/harness/scenario.py +1048 -0
- fi/alk/harness/scenario_source.py +879 -0
- fi/alk/harness/scenario_tools.py +1143 -0
- fi/alk/harness/scenarios.py +915 -0
- fi/alk/harness/secrets.py +168 -0
- fi/alk/harness/service_catalog.py +97 -0
- fi/alk/harness/session.py +391 -0
- fi/alk/harness/sessions.py +372 -0
- fi/alk/harness/simulator.py +76 -0
- fi/alk/harness/simulator_voice.py +928 -0
- fi/alk/harness/skills/build-environment/SKILL.md +538 -0
- fi/alk/harness/skills/harness.md +131 -0
- fi/alk/harness/skills/kinds/chat.md +48 -0
- fi/alk/harness/skills/kinds/voice-voicemail.md +63 -0
- fi/alk/harness/skills/kinds/voice.md +59 -0
- fi/alk/harness/skills/plan-suite/SKILL.md +103 -0
- fi/alk/harness/skills/provision-environment/SKILL.md +136 -0
- fi/alk/harness/skills/run-scenarios/SKILL.md +112 -0
- fi/alk/harness/skills/understand-agent/SKILL.md +251 -0
- fi/alk/harness/skills/write-scenarios/SKILL.md +606 -0
- fi/alk/harness/skills/write-scenarios/references/refusals.md +28 -0
- fi/alk/harness/skills/write-scenarios/references/world-api.md +92 -0
- fi/alk/harness/source_data_invariants.py +444 -0
- fi/alk/harness/source_tool_evidence.py +79 -0
- fi/alk/harness/sources.py +253 -0
- fi/alk/harness/spend.py +140 -0
- fi/alk/harness/tool_trace_proxy.py +104 -0
- fi/alk/harness/tools.py +1018 -0
- fi/alk/harness/understand.py +169 -0
- fi/alk/harness/voicemail_audio.py +74 -0
- fi/alk/harness/world/__init__.py +33 -0
- fi/alk/harness/world/errors.py +68 -0
- fi/alk/harness/world/expectations.py +91 -0
- fi/alk/harness/world/handle.py +538 -0
- fi/alk/harness/world/kinds.py +196 -0
- fi/alk/harness/world/mutate.py +186 -0
- fi/alk/harness/world/probe.py +413 -0
- fi/alk/harness/world/provision.py +511 -0
- fi/alk/harness/world/provisioned.py +191 -0
- fi/alk/harness/world/runtime.py +616 -0
- fi/alk/harness/world/snapshot.py +288 -0
- fi/alk/harness/world/stores/__init__.py +305 -0
- fi/alk/harness/world/stores/container.py +215 -0
- fi/alk/harness/world/stores/inprocess.py +346 -0
- fi/alk/harness/world/stores/postgres.py +481 -0
- fi/alk/harness/world/stores/prove.py +202 -0
- fi/alk/harness/world/stores/sqlite.py +245 -0
- fi/alk/harness/world/stores/written.py +182 -0
- fi/alk/harness/world/tools.py +1516 -0
- fi/alk/harness/world/workspace.py +144 -0
- fi/alk/image_loop.py +453 -0
- fi/alk/image_perturb.py +241 -0
- fi/alk/improve.py +274 -0
- fi/alk/live/__init__.py +154 -0
- fi/alk/live/_attribution.py +184 -0
- fi/alk/live/_capture.py +264 -0
- fi/alk/live/_codec.py +391 -0
- fi/alk/live/_contract.py +134 -0
- fi/alk/live/_loopback.py +316 -0
- fi/alk/live/_perturb.py +449 -0
- fi/alk/live/_runner.py +386 -0
- fi/alk/live/_stats.py +561 -0
- fi/alk/live/_transcript.py +240 -0
- fi/alk/live/_workers/__init__.py +9 -0
- fi/alk/live/_workers/a2a_worker.py +316 -0
- fi/alk/live/_workers/langgraph_worker.py +217 -0
- fi/alk/live/_workers/livekit_worker.py +207 -0
- fi/alk/live/_workers/mcp_loopback_server.py +46 -0
- fi/alk/live/_workers/mcp_worker.py +158 -0
- fi/alk/live/_workers/pipecat_worker.py +189 -0
- fi/alk/live/a2a_lane.py +138 -0
- fi/alk/live/langgraph_lane.py +339 -0
- fi/alk/live/livekit_lane.py +376 -0
- fi/alk/live/mcp_lane.py +172 -0
- fi/alk/live/pipecat_lane.py +341 -0
- fi/alk/live/voice_redteam.py +494 -0
- fi/alk/loss.py +306 -0
- fi/alk/optimize.py +36260 -0
- fi/alk/practice/__init__.py +51 -0
- fi/alk/practice/_assess.py +103 -0
- fi/alk/practice/_budget.py +81 -0
- fi/alk/practice/_calibrate.py +69 -0
- fi/alk/practice/_capstone.py +86 -0
- fi/alk/practice/_contract.py +91 -0
- fi/alk/practice/_diagnose.py +79 -0
- fi/alk/practice/_drill.py +196 -0
- fi/alk/practice/_experiment.py +720 -0
- fi/alk/practice/_schedule.py +102 -0
- fi/alk/practice/_store.py +194 -0
- fi/alk/practice/_trainer.py +245 -0
- fi/alk/practice/_update.py +125 -0
- fi/alk/redteam.py +2621 -0
- fi/alk/rewardhack.py +237 -0
- fi/alk/simulate.py +10351 -0
- fi/alk/studio/__init__.py +82 -0
- fi/alk/studio/_bias.py +314 -0
- fi/alk/studio/_calibration.py +522 -0
- fi/alk/studio/_coverage.py +262 -0
- fi/alk/studio/_download.py +665 -0
- fi/alk/studio/_fidelity_attack.py +114 -0
- fi/alk/studio/_generate.py +652 -0
- fi/alk/studio/_library.py +370 -0
- fi/alk/studio/_scan.py +134 -0
- fi/alk/studio/_upgrade.py +42 -0
- fi/alk/studio/_vendor.py +172 -0
- fi/alk/suite.py +4200 -0
- fi/alk/tasks.py +828 -0
- fi/alk/telemetry/__init__.py +149 -0
- fi/alk/telemetry/_contract.py +141 -0
- fi/alk/telemetry/_emit.py +182 -0
- fi/alk/telemetry/_ledger.py +296 -0
- fi/alk/telemetry/_queue.py +127 -0
- fi/alk/telemetry/_row.py +294 -0
- fi/alk/telemetry/_run.py +233 -0
- fi/alk/telemetry/_sync.py +193 -0
- fi/alk/telemetry/_url.py +119 -0
- fi/alk/trinity.py +49397 -0
- fi/alk/voice_loop.py +174 -0
- fi/api/__init__.py +1 -0
- fi/api/auth.py +137 -0
- fi/api/types.py +29 -0
- fi/cli/__init__.py +9 -0
- fi/cli/assertions/__init__.py +25 -0
- fi/cli/assertions/conditions.py +76 -0
- fi/cli/assertions/evaluator.py +286 -0
- fi/cli/assertions/exit_codes.py +20 -0
- fi/cli/assertions/parser.py +131 -0
- fi/cli/assertions/reporter.py +194 -0
- fi/cli/commands/__init__.py +9 -0
- fi/cli/commands/config.py +165 -0
- fi/cli/commands/export.py +208 -0
- fi/cli/commands/init.py +112 -0
- fi/cli/commands/list_cmd.py +213 -0
- fi/cli/commands/run.py +486 -0
- fi/cli/commands/validate.py +173 -0
- fi/cli/commands/view.py +424 -0
- fi/cli/config/__init__.py +6 -0
- fi/cli/config/defaults.py +206 -0
- fi/cli/config/loader.py +155 -0
- fi/cli/config/schema.py +174 -0
- fi/cli/main.py +78 -0
- fi/cli/output/__init__.py +6 -0
- fi/cli/output/formatters.py +106 -0
- fi/cli/output/reporters.py +46 -0
- fi/cli/storage/__init__.py +5 -0
- fi/cli/storage/run_history.py +249 -0
- fi/cli/utils/__init__.py +5 -0
- fi/cli/utils/console.py +44 -0
- fi/evals/__init__.py +131 -0
- fi/evals/autoeval/__init__.py +137 -0
- fi/evals/autoeval/analyzer.py +211 -0
- fi/evals/autoeval/config.py +244 -0
- fi/evals/autoeval/export.py +213 -0
- fi/evals/autoeval/interactive.py +283 -0
- fi/evals/autoeval/pipeline.py +625 -0
- fi/evals/autoeval/prompts.py +139 -0
- fi/evals/autoeval/recommender.py +242 -0
- fi/evals/autoeval/rules.py +589 -0
- fi/evals/autoeval/templates.py +299 -0
- fi/evals/autoeval/types.py +232 -0
- fi/evals/core/__init__.py +16 -0
- fi/evals/core/cloud_registry.py +184 -0
- fi/evals/core/engines.py +368 -0
- fi/evals/core/evaluate.py +319 -0
- fi/evals/core/judge_prompt.py +90 -0
- fi/evals/core/prompt_generator.py +83 -0
- fi/evals/core/registry.py +57 -0
- fi/evals/core/result.py +55 -0
- fi/evals/evaluator.py +721 -0
- fi/evals/execution.py +168 -0
- fi/evals/feedback/__init__.py +32 -0
- fi/evals/feedback/calibrator.py +160 -0
- fi/evals/feedback/collector.py +214 -0
- fi/evals/feedback/hooks.py +81 -0
- fi/evals/feedback/retriever.py +128 -0
- fi/evals/feedback/store.py +272 -0
- fi/evals/feedback/types.py +99 -0
- fi/evals/framework/README.md +79 -0
- fi/evals/framework/__init__.py +267 -0
- fi/evals/framework/backends/Dockerfile.eval-runner +33 -0
- fi/evals/framework/backends/__init__.py +99 -0
- fi/evals/framework/backends/_container.py +141 -0
- fi/evals/framework/backends/_utils.py +145 -0
- fi/evals/framework/backends/base.py +223 -0
- fi/evals/framework/backends/celery_backend.py +417 -0
- fi/evals/framework/backends/celery_worker.py +78 -0
- fi/evals/framework/backends/kubernetes_backend.py +665 -0
- fi/evals/framework/backends/ray_backend.py +521 -0
- fi/evals/framework/backends/temporal.py +350 -0
- fi/evals/framework/backends/temporal_worker.py +126 -0
- fi/evals/framework/backends/thread_pool.py +286 -0
- fi/evals/framework/context.py +258 -0
- fi/evals/framework/enrichment.py +306 -0
- fi/evals/framework/evals/__init__.py +68 -0
- fi/evals/framework/evals/agentic.py +399 -0
- fi/evals/framework/evals/builder.py +609 -0
- fi/evals/framework/evals/semantic.py +142 -0
- fi/evals/framework/evaluator.py +647 -0
- fi/evals/framework/evaluators/__init__.py +22 -0
- fi/evals/framework/evaluators/blocking.py +347 -0
- fi/evals/framework/evaluators/non_blocking.py +577 -0
- fi/evals/framework/propagation.py +421 -0
- fi/evals/framework/protocols.py +385 -0
- fi/evals/framework/registry.py +370 -0
- fi/evals/framework/resilience/__init__.py +150 -0
- fi/evals/framework/resilience/circuit_breaker.py +309 -0
- fi/evals/framework/resilience/degradation.py +355 -0
- fi/evals/framework/resilience/health.py +505 -0
- fi/evals/framework/resilience/rate_limiter.py +228 -0
- fi/evals/framework/resilience/retry.py +274 -0
- fi/evals/framework/resilience/types.py +288 -0
- fi/evals/framework/resilience/wrapper.py +433 -0
- fi/evals/framework/types.py +218 -0
- fi/evals/guardrails/README.md +915 -0
- fi/evals/guardrails/__init__.py +96 -0
- fi/evals/guardrails/backends/__init__.py +43 -0
- fi/evals/guardrails/backends/azure.py +361 -0
- fi/evals/guardrails/backends/base.py +88 -0
- fi/evals/guardrails/backends/generic_llm.py +163 -0
- fi/evals/guardrails/backends/granite.py +216 -0
- fi/evals/guardrails/backends/llamaguard.py +221 -0
- fi/evals/guardrails/backends/local_base.py +479 -0
- fi/evals/guardrails/backends/openai.py +365 -0
- fi/evals/guardrails/backends/qwen.py +170 -0
- fi/evals/guardrails/backends/shieldgemma.py +154 -0
- fi/evals/guardrails/backends/turing.py +235 -0
- fi/evals/guardrails/backends/vllm_client.py +321 -0
- fi/evals/guardrails/backends/wildguard.py +188 -0
- fi/evals/guardrails/base.py +888 -0
- fi/evals/guardrails/config.py +221 -0
- fi/evals/guardrails/discovery.py +243 -0
- fi/evals/guardrails/gateway.py +437 -0
- fi/evals/guardrails/registry.py +231 -0
- fi/evals/guardrails/scanners/__init__.py +127 -0
- fi/evals/guardrails/scanners/base.py +191 -0
- fi/evals/guardrails/scanners/code_injection.py +243 -0
- fi/evals/guardrails/scanners/eval_delegate.py +574 -0
- fi/evals/guardrails/scanners/invisible_chars.py +351 -0
- fi/evals/guardrails/scanners/jailbreak.py +412 -0
- fi/evals/guardrails/scanners/language.py +288 -0
- fi/evals/guardrails/scanners/pipeline.py +260 -0
- fi/evals/guardrails/scanners/regex.py +311 -0
- fi/evals/guardrails/scanners/secrets.py +274 -0
- fi/evals/guardrails/scanners/topics.py +649 -0
- fi/evals/guardrails/scanners/urls.py +341 -0
- fi/evals/guardrails/types.py +96 -0
- fi/evals/llm/__init__.py +3 -0
- fi/evals/llm/base_llm_provider.py +35 -0
- fi/evals/llm/providers/litellm.py +70 -0
- fi/evals/local/__init__.py +90 -0
- fi/evals/local/evaluator.py +690 -0
- fi/evals/local/execution_mode.py +121 -0
- fi/evals/local/llm.py +489 -0
- fi/evals/local/metrics/__init__.py +19 -0
- fi/evals/local/registry.py +360 -0
- fi/evals/manager.py +1018 -0
- fi/evals/manager_types.py +362 -0
- fi/evals/metrics/__init__.py +185 -0
- fi/evals/metrics/agents/__init__.py +74 -0
- fi/evals/metrics/agents/metrics.py +693 -0
- fi/evals/metrics/agents/report.py +36463 -0
- fi/evals/metrics/agents/types.py +160 -0
- fi/evals/metrics/base_llm_metric.py +111 -0
- fi/evals/metrics/base_metric.py +138 -0
- fi/evals/metrics/code_security/__init__.py +305 -0
- fi/evals/metrics/code_security/analyzer.py +985 -0
- fi/evals/metrics/code_security/benchmarks/__init__.py +73 -0
- fi/evals/metrics/code_security/benchmarks/builtin.py +750 -0
- fi/evals/metrics/code_security/benchmarks/loader.py +580 -0
- fi/evals/metrics/code_security/benchmarks/types.py +308 -0
- fi/evals/metrics/code_security/detectors/__init__.py +186 -0
- fi/evals/metrics/code_security/detectors/base.py +394 -0
- fi/evals/metrics/code_security/detectors/cryptography.py +345 -0
- fi/evals/metrics/code_security/detectors/injection.py +744 -0
- fi/evals/metrics/code_security/detectors/secrets.py +287 -0
- fi/evals/metrics/code_security/detectors/serialization.py +192 -0
- fi/evals/metrics/code_security/joint_metrics.py +588 -0
- fi/evals/metrics/code_security/judges/__init__.py +83 -0
- fi/evals/metrics/code_security/judges/base.py +238 -0
- fi/evals/metrics/code_security/judges/dual_judge.py +534 -0
- fi/evals/metrics/code_security/judges/llm_judge.py +301 -0
- fi/evals/metrics/code_security/judges/pattern_judge.py +515 -0
- fi/evals/metrics/code_security/metrics.py +388 -0
- fi/evals/metrics/code_security/modes/__init__.py +63 -0
- fi/evals/metrics/code_security/modes/adversarial.py +284 -0
- fi/evals/metrics/code_security/modes/autocomplete.py +198 -0
- fi/evals/metrics/code_security/modes/base.py +283 -0
- fi/evals/metrics/code_security/modes/instruct.py +253 -0
- fi/evals/metrics/code_security/modes/repair.py +230 -0
- fi/evals/metrics/code_security/reports/__init__.py +57 -0
- fi/evals/metrics/code_security/reports/generator.py +404 -0
- fi/evals/metrics/code_security/reports/leaderboard.py +509 -0
- fi/evals/metrics/code_security/types.py +534 -0
- fi/evals/metrics/function_calling/__init__.py +34 -0
- fi/evals/metrics/function_calling/metrics.py +573 -0
- fi/evals/metrics/function_calling/types.py +87 -0
- fi/evals/metrics/hallucination/__init__.py +54 -0
- fi/evals/metrics/hallucination/detector.py +149 -0
- fi/evals/metrics/hallucination/metrics.py +390 -0
- fi/evals/metrics/hallucination/nli.py +253 -0
- fi/evals/metrics/hallucination/sentinel.py +106 -0
- fi/evals/metrics/hallucination/types.py +132 -0
- fi/evals/metrics/heuristics/aggregation_metrics.py +85 -0
- fi/evals/metrics/heuristics/json_metrics.py +87 -0
- fi/evals/metrics/heuristics/similarity_metrics.py +375 -0
- fi/evals/metrics/heuristics/string_metrics.py +391 -0
- fi/evals/metrics/llm_as_judges/__init__.py +17 -0
- fi/evals/metrics/llm_as_judges/custom_judge/metric.py +112 -0
- fi/evals/metrics/llm_as_judges/custom_judge/prompts.py +26 -0
- fi/evals/metrics/llm_as_judges/types.py +48 -0
- fi/evals/metrics/rag/__init__.py +111 -0
- fi/evals/metrics/rag/advanced/__init__.py +14 -0
- fi/evals/metrics/rag/advanced/multi_hop.py +283 -0
- fi/evals/metrics/rag/advanced/source_attribution.py +344 -0
- fi/evals/metrics/rag/generation/__init__.py +17 -0
- fi/evals/metrics/rag/generation/answer_relevancy.py +176 -0
- fi/evals/metrics/rag/generation/context_utilization.py +245 -0
- fi/evals/metrics/rag/generation/faithfulness.py +241 -0
- fi/evals/metrics/rag/generation/groundedness.py +131 -0
- fi/evals/metrics/rag/rag_score.py +277 -0
- fi/evals/metrics/rag/retrieval/__init__.py +20 -0
- fi/evals/metrics/rag/retrieval/context_entity_recall.py +124 -0
- fi/evals/metrics/rag/retrieval/context_precision.py +158 -0
- fi/evals/metrics/rag/retrieval/context_recall.py +106 -0
- fi/evals/metrics/rag/retrieval/noise_sensitivity.py +163 -0
- fi/evals/metrics/rag/retrieval/ranking.py +261 -0
- fi/evals/metrics/rag/types.py +100 -0
- fi/evals/metrics/rag/utils/__init__.py +62 -0
- fi/evals/metrics/rag/utils/claims.py +189 -0
- fi/evals/metrics/rag/utils/entities.py +244 -0
- fi/evals/metrics/rag/utils/nli.py +92 -0
- fi/evals/metrics/rag/utils/similarity.py +345 -0
- fi/evals/metrics/structured/__init__.py +114 -0
- fi/evals/metrics/structured/field_completeness.py +313 -0
- fi/evals/metrics/structured/hierarchy_score.py +366 -0
- fi/evals/metrics/structured/json_validation.py +190 -0
- fi/evals/metrics/structured/schema_compliance.py +280 -0
- fi/evals/metrics/structured/structured_output_score.py +298 -0
- fi/evals/metrics/structured/types.py +108 -0
- fi/evals/metrics/structured/validators/__init__.py +30 -0
- fi/evals/metrics/structured/validators/base.py +189 -0
- fi/evals/metrics/structured/validators/json_validator.py +196 -0
- fi/evals/metrics/structured/validators/pydantic_validator.py +178 -0
- fi/evals/metrics/structured/validators/yaml_validator.py +248 -0
- fi/evals/otel/__init__.py +266 -0
- fi/evals/otel/config.py +400 -0
- fi/evals/otel/conventions.py +463 -0
- fi/evals/otel/enrichment.py +371 -0
- fi/evals/otel/instrumentors/__init__.py +140 -0
- fi/evals/otel/instrumentors/anthropic.py +517 -0
- fi/evals/otel/instrumentors/base.py +382 -0
- fi/evals/otel/instrumentors/openai.py +673 -0
- fi/evals/otel/processors/__init__.py +36 -0
- fi/evals/otel/processors/base.py +473 -0
- fi/evals/otel/processors/cost.py +445 -0
- fi/evals/otel/processors/evaluation.py +559 -0
- fi/evals/otel/processors/llm.py +462 -0
- fi/evals/otel/tracer.py +506 -0
- fi/evals/otel/types.py +232 -0
- fi/evals/otel_utils.py +23 -0
- fi/evals/protect.py +671 -0
- fi/evals/protect_input_adapter.py +154 -0
- fi/evals/streaming/__init__.py +88 -0
- fi/evals/streaming/buffer.py +213 -0
- fi/evals/streaming/evaluator.py +551 -0
- fi/evals/streaming/policy.py +307 -0
- fi/evals/streaming/scorers.py +368 -0
- fi/evals/streaming/types.py +238 -0
- fi/evals/templates.py +472 -0
- fi/evals/types.py +156 -0
- fi/opt/__init__.py +221 -0
- fi/opt/_objective_scoring.py +85 -0
- fi/opt/base/__init__.py +11 -0
- fi/opt/base/base_generator.py +33 -0
- fi/opt/base/base_mapper.py +26 -0
- fi/opt/base/base_optimizer.py +45 -0
- fi/opt/base/evaluator.py +211 -0
- fi/opt/components.py +3095 -0
- fi/opt/datamappers/__init__.py +3 -0
- fi/opt/datamappers/basic_mapper.py +40 -0
- fi/opt/deployment.py +1021 -0
- fi/opt/evidence.py +4332 -0
- fi/opt/generators/__init__.py +3 -0
- fi/opt/generators/litellm.py +66 -0
- fi/opt/integrations/__init__.py +23 -0
- fi/opt/integrations/generative_suite.py +410 -0
- fi/opt/integrations/simulate.py +1313 -0
- fi/opt/mutations.py +771 -0
- fi/opt/observability.py +4639 -0
- fi/opt/optimizer_trace.py +889 -0
- fi/opt/optimizers/__init__.py +80 -0
- fi/opt/optimizers/agent.py +331 -0
- fi/opt/optimizers/agent_bandit.py +392 -0
- fi/opt/optimizers/agent_curriculum.py +635 -0
- fi/opt/optimizers/agent_evolution.py +894 -0
- fi/opt/optimizers/agent_feedback.py +1863 -0
- fi/opt/optimizers/agent_pareto.py +547 -0
- fi/opt/optimizers/agent_social_memory.py +1113 -0
- fi/opt/optimizers/agent_tpe.py +321 -0
- fi/opt/optimizers/bayesian_search.py +449 -0
- fi/opt/optimizers/council.py +2075 -0
- fi/opt/optimizers/futureagi_replay.py +799 -0
- fi/opt/optimizers/gepa.py +322 -0
- fi/opt/optimizers/metaprompt.py +243 -0
- fi/opt/optimizers/promptwizard.py +417 -0
- fi/opt/optimizers/protegi.py +329 -0
- fi/opt/optimizers/random_search.py +224 -0
- fi/opt/research.py +518 -0
- fi/opt/simulation.py +260 -0
- fi/opt/targets.py +232 -0
- fi/opt/types.py +66 -0
- fi/opt/utils/__init__.py +4 -0
- fi/opt/utils/early_stopping.py +266 -0
- fi/opt/utils/setup_logging.py +82 -0
- fi/simulate/__init__.py +540 -0
- fi/simulate/_hashing.py +35 -0
- fi/simulate/_logging.py +10 -0
- fi/simulate/adapters.py +87 -0
- fi/simulate/agent/__init__.py +120 -0
- fi/simulate/agent/browser.py +658 -0
- fi/simulate/agent/definition.py +587 -0
- fi/simulate/agent/frameworks.py +3528 -0
- fi/simulate/agent/generic.py +8286 -0
- fi/simulate/agent/import_probe.py +227 -0
- fi/simulate/agent/memory.py +905 -0
- fi/simulate/agent/mocks.py +101 -0
- fi/simulate/agent/multi_agent.py +361 -0
- fi/simulate/agent/orchestration.py +903 -0
- fi/simulate/agent/realtime.py +665 -0
- fi/simulate/agent/wrapper.py +99 -0
- fi/simulate/agent/wrappers/__init__.py +18 -0
- fi/simulate/agent/wrappers/anthropic.py +62 -0
- fi/simulate/agent/wrappers/gemini.py +65 -0
- fi/simulate/agent/wrappers/http.py +404 -0
- fi/simulate/agent/wrappers/langchain.py +80 -0
- fi/simulate/agent/wrappers/openai.py +75 -0
- fi/simulate/agent/wrappers/websocket.py +326 -0
- fi/simulate/artifacts/__init__.py +11 -0
- fi/simulate/artifacts/manifest.py +62 -0
- fi/simulate/cli.py +20560 -0
- fi/simulate/endpoints/__init__.py +45 -0
- fi/simulate/endpoints/_http_actor.py +73 -0
- fi/simulate/endpoints/actor_sources.py +243 -0
- fi/simulate/endpoints/base.py +107 -0
- fi/simulate/endpoints/builtins.py +10 -0
- fi/simulate/endpoints/callable.py +95 -0
- fi/simulate/endpoints/http.py +76 -0
- fi/simulate/endpoints/livekit.py +138 -0
- fi/simulate/endpoints/originators.py +132 -0
- fi/simulate/endpoints/profiles.py +348 -0
- fi/simulate/endpoints/retell.py +633 -0
- fi/simulate/endpoints/vapi.py +205 -0
- fi/simulate/endpoints/websocket.py +76 -0
- fi/simulate/environment.py +33026 -0
- fi/simulate/environments/__init__.py +11 -0
- fi/simulate/environments/base.py +73 -0
- fi/simulate/environments/chat.py +697 -0
- fi/simulate/environments/voice.py +212 -0
- fi/simulate/evaluation/__init__.py +4 -0
- fi/simulate/evaluation/ai_eval.py +227 -0
- fi/simulate/evidence/__init__.py +35 -0
- fi/simulate/evidence/base.py +59 -0
- fi/simulate/evidence/caller_observed.py +50 -0
- fi/simulate/evidence/livekit_instrumentation.py +51 -0
- fi/simulate/evidence/livekit_room.py +50 -0
- fi/simulate/evidence/otel.py +49 -0
- fi/simulate/evidence/providers/__init__.py +24 -0
- fi/simulate/evidence/providers/base.py +61 -0
- fi/simulate/evidence/providers/retell.py +376 -0
- fi/simulate/evidence/providers/vapi.py +426 -0
- fi/simulate/hosted/__init__.py +32 -0
- fi/simulate/hosted/child_entrypoint.py +306 -0
- fi/simulate/hosted/job.py +150 -0
- fi/simulate/hosted/targets.py +53 -0
- fi/simulate/instrumentation/__init__.py +5 -0
- fi/simulate/instrumentation/livekit/__init__.py +122 -0
- fi/simulate/manifest.py +1033 -0
- fi/simulate/matrix_cli.py +165 -0
- fi/simulate/realtime/__init__.py +40 -0
- fi/simulate/realtime/events.py +107 -0
- fi/simulate/realtime/media.py +61 -0
- fi/simulate/realtime/session.py +91 -0
- fi/simulate/recording/__init__.py +5 -0
- fi/simulate/recording/room_recorder.py +326 -0
- fi/simulate/registry.py +185 -0
- fi/simulate/results/__init__.py +9 -0
- fi/simulate/results/base.py +18 -0
- fi/simulate/results/filesystem.py +71 -0
- fi/simulate/results/futureagi.py +1340 -0
- fi/simulate/runtime/__init__.py +85 -0
- fi/simulate/runtime/capabilities.py +40 -0
- fi/simulate/runtime/events.py +63 -0
- fi/simulate/runtime/failures.py +25 -0
- fi/simulate/runtime/ids.py +34 -0
- fi/simulate/runtime/plan.py +70 -0
- fi/simulate/runtime/planner.py +102 -0
- fi/simulate/runtime/report.py +174 -0
- fi/simulate/runtime/run.py +75 -0
- fi/simulate/runtime/runner.py +333 -0
- fi/simulate/runtime/spec.py +186 -0
- fi/simulate/simulation/__init__.py +30 -0
- fi/simulate/simulation/behavior_policy.py +425 -0
- fi/simulate/simulation/bridge/__init__.py +9 -0
- fi/simulate/simulation/bridge/audio.py +29 -0
- fi/simulate/simulation/bridge/connector.py +46 -0
- fi/simulate/simulation/bridge/livekit.py +252 -0
- fi/simulate/simulation/bridge/retell.py +188 -0
- fi/simulate/simulation/bridge/vapi.py +177 -0
- fi/simulate/simulation/contract.py +419 -0
- fi/simulate/simulation/engines/__init__.py +12 -0
- fi/simulate/simulation/engines/base.py +21 -0
- fi/simulate/simulation/engines/cloud.py +517 -0
- fi/simulate/simulation/engines/livekit.py +4167 -0
- fi/simulate/simulation/engines/local_text.py +89 -0
- fi/simulate/simulation/fidelity.py +374 -0
- fi/simulate/simulation/gemini_tts_stream.py +110 -0
- fi/simulate/simulation/generator.py +91 -0
- fi/simulate/simulation/goal_machine.py +185 -0
- fi/simulate/simulation/livekit_models.py +467 -0
- fi/simulate/simulation/matrix.py +170 -0
- fi/simulate/simulation/models.py +279 -0
- fi/simulate/simulation/runner.py +153 -0
- fi/simulate/simulation/synthetic.py +880 -0
- fi/simulate/simulation/voice_prompt.py +502 -0
- fi/simulate/simulator/__init__.py +55 -0
- fi/simulate/simulator/builtins.py +53 -0
- fi/simulate/suite.py +1288 -0
- fi/simulate/utils/routes.py +164 -0
- fi/simulate/voice.py +225 -0
- fi/simulate/voice_cli.py +182 -0
- fi/utils/__init__.py +1 -0
- fi/utils/constants.py +14 -0
- fi/utils/errors.py +200 -0
- fi/utils/executor.py +26 -0
- fi/utils/routes.py +119 -0
- fi/utils/utils.py +17 -0
|
@@ -0,0 +1,3528 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import copy
|
|
5
|
+
import inspect
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from typing import Any, Callable, Dict, Mapping, Optional, Sequence
|
|
8
|
+
from urllib.parse import urlparse
|
|
9
|
+
|
|
10
|
+
from fi.simulate.agent.generic import GenericAgentWrapper, InputMode
|
|
11
|
+
from fi.simulate.agent.wrapper import AgentInput, AgentResponse, AgentWrapper
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True)
|
|
15
|
+
class FrameworkAdapterSpec:
|
|
16
|
+
"""Import-free adapter preset for a common agent/orchestration framework."""
|
|
17
|
+
|
|
18
|
+
name: str
|
|
19
|
+
method: Optional[str]
|
|
20
|
+
input_mode: InputMode
|
|
21
|
+
modality: str = "text"
|
|
22
|
+
transport: str = "in_process"
|
|
23
|
+
lifecycle_hooks: tuple[str, ...] = ()
|
|
24
|
+
capabilities: tuple[str, ...] = ()
|
|
25
|
+
notes: str = ""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
FRAMEWORK_PRESETS: Dict[str, FrameworkAdapterSpec] = {
|
|
29
|
+
# Text/chat orchestration
|
|
30
|
+
"custom": FrameworkAdapterSpec("custom", None, "auto", notes="Bring-your-own framework adapter."),
|
|
31
|
+
"callable": FrameworkAdapterSpec("callable", None, "agent_input", notes="Plain Python callable."),
|
|
32
|
+
"a2a": FrameworkAdapterSpec("a2a", "send_message", "dict", notes="Agent2Agent protocol client/server session."),
|
|
33
|
+
"openenv": FrameworkAdapterSpec(
|
|
34
|
+
"openenv",
|
|
35
|
+
"run",
|
|
36
|
+
"dict",
|
|
37
|
+
capabilities=(
|
|
38
|
+
"environment_replay",
|
|
39
|
+
"reset_step_trace",
|
|
40
|
+
"runtime_trace",
|
|
41
|
+
"state",
|
|
42
|
+
"artifacts",
|
|
43
|
+
),
|
|
44
|
+
notes="OpenEnv/Gymnasium-style environment replay adapter.",
|
|
45
|
+
),
|
|
46
|
+
"gymnasium": FrameworkAdapterSpec(
|
|
47
|
+
"gymnasium",
|
|
48
|
+
"run",
|
|
49
|
+
"dict",
|
|
50
|
+
capabilities=(
|
|
51
|
+
"environment_replay",
|
|
52
|
+
"reset_step_trace",
|
|
53
|
+
"runtime_trace",
|
|
54
|
+
"state",
|
|
55
|
+
"artifacts",
|
|
56
|
+
),
|
|
57
|
+
notes="Gymnasium Env reset/step replay adapter.",
|
|
58
|
+
),
|
|
59
|
+
"langchain": FrameworkAdapterSpec("langchain", "ainvoke", "dict", notes="LangChain Runnable/Chain."),
|
|
60
|
+
"langgraph": FrameworkAdapterSpec("langgraph", "ainvoke", "dict", notes="LangGraph compiled graph."),
|
|
61
|
+
"llamaindex": FrameworkAdapterSpec("llamaindex", "achat", "text", notes="LlamaIndex chat/query engines."),
|
|
62
|
+
"crewai": FrameworkAdapterSpec("crewai", "kickoff", "dict", notes="CrewAI Crew kickoff."),
|
|
63
|
+
"autogen": FrameworkAdapterSpec("autogen", "run", "text", notes="AutoGen AgentChat style task run."),
|
|
64
|
+
"semantic_kernel": FrameworkAdapterSpec("semantic_kernel", "invoke", "dict", notes="Semantic Kernel function/agent."),
|
|
65
|
+
"openai_agents": FrameworkAdapterSpec("openai_agents", "run", "text", notes="OpenAI Agents SDK runner/agent."),
|
|
66
|
+
"pydantic_ai": FrameworkAdapterSpec("pydantic_ai", "run", "text", notes="PydanticAI agent."),
|
|
67
|
+
"haystack": FrameworkAdapterSpec("haystack", "run", "dict", notes="Haystack pipeline."),
|
|
68
|
+
"agno": FrameworkAdapterSpec("agno", "run", "dict", notes="Agno agent/team runner."),
|
|
69
|
+
"beeai": FrameworkAdapterSpec("beeai", "run", "dict", notes="BeeAI agent runner."),
|
|
70
|
+
"claude_agent_sdk": FrameworkAdapterSpec("claude_agent_sdk", "query", "text", notes="Claude Agent SDK query runner."),
|
|
71
|
+
"dspy": FrameworkAdapterSpec("dspy", "__call__", "dict", notes="DSPy module/program."),
|
|
72
|
+
"google_adk": FrameworkAdapterSpec("google_adk", "run", "dict", notes="Google ADK runner/agent."),
|
|
73
|
+
"guardrails": FrameworkAdapterSpec("guardrails", "__call__", "text", notes="Guardrails validation wrapper."),
|
|
74
|
+
"litellm": FrameworkAdapterSpec("litellm", "completion", "dict", notes="LiteLLM completion shim."),
|
|
75
|
+
"mcp": FrameworkAdapterSpec("mcp", "call_tool", "dict", notes="MCP client/server tool session."),
|
|
76
|
+
"portkey": FrameworkAdapterSpec("portkey", "chat", "dict", notes="Portkey gateway client."),
|
|
77
|
+
"smolagents": FrameworkAdapterSpec("smolagents", "run", "text", notes="SmolAgents runner."),
|
|
78
|
+
"strands": FrameworkAdapterSpec("strands", "__call__", "text", notes="Strands agent callable."),
|
|
79
|
+
# Voice and realtime
|
|
80
|
+
"livekit": FrameworkAdapterSpec("livekit", "respond", "text", modality="voice", notes="LiveKit agent/session shim."),
|
|
81
|
+
"pipecat": FrameworkAdapterSpec("pipecat", "process", "dict", modality="voice", notes="Pipecat pipeline/processor shim."),
|
|
82
|
+
"vapi": FrameworkAdapterSpec("vapi", "respond", "dict", modality="voice", notes="Webhook/local adapter shim."),
|
|
83
|
+
"retell": FrameworkAdapterSpec("retell", "respond", "dict", modality="voice", notes="Webhook/local adapter shim."),
|
|
84
|
+
"elevenlabs": FrameworkAdapterSpec("elevenlabs", "respond", "dict", modality="voice", notes="ElevenLabs conversational agent shim."),
|
|
85
|
+
"deepgram": FrameworkAdapterSpec("deepgram", "respond", "dict", modality="voice", notes="Deepgram voice agent shim."),
|
|
86
|
+
"agora": FrameworkAdapterSpec("agora", "respond", "dict", modality="voice", notes="Agora conversational AI shim."),
|
|
87
|
+
"twilio": FrameworkAdapterSpec("twilio", "respond", "dict", modality="voice", notes="Twilio voice/media stream webhook shim."),
|
|
88
|
+
# Model/provider clients commonly instrumented by TraceAI
|
|
89
|
+
"anthropic": FrameworkAdapterSpec("anthropic", "messages.create", "messages", notes="Anthropic messages client shim."),
|
|
90
|
+
"bedrock": FrameworkAdapterSpec("bedrock", "invoke_model", "dict", notes="AWS Bedrock client shim."),
|
|
91
|
+
"cerebras": FrameworkAdapterSpec("cerebras", "chat", "dict", notes="Cerebras client shim."),
|
|
92
|
+
"cohere": FrameworkAdapterSpec("cohere", "chat", "dict", notes="Cohere client shim."),
|
|
93
|
+
"deepseek": FrameworkAdapterSpec("deepseek", "chat", "dict", notes="DeepSeek OpenAI-compatible client shim."),
|
|
94
|
+
"fireworks": FrameworkAdapterSpec("fireworks", "chat", "dict", notes="Fireworks client shim."),
|
|
95
|
+
"google_genai": FrameworkAdapterSpec("google_genai", "generate_content", "dict", notes="Google GenAI client shim."),
|
|
96
|
+
"groq": FrameworkAdapterSpec("groq", "chat", "dict", notes="Groq client shim."),
|
|
97
|
+
"huggingface": FrameworkAdapterSpec("huggingface", "__call__", "dict", notes="Hugging Face pipeline/client shim."),
|
|
98
|
+
"instructor": FrameworkAdapterSpec("instructor", "chat", "dict", notes="Instructor structured output client shim."),
|
|
99
|
+
"mistralai": FrameworkAdapterSpec("mistralai", "chat", "dict", notes="Mistral AI client shim."),
|
|
100
|
+
"ollama": FrameworkAdapterSpec("ollama", "chat", "dict", notes="Ollama client shim."),
|
|
101
|
+
"openai": FrameworkAdapterSpec("openai", "chat.completions.create", "messages", notes="OpenAI chat client shim."),
|
|
102
|
+
"together": FrameworkAdapterSpec("together", "chat", "dict", notes="Together AI client shim."),
|
|
103
|
+
"vertexai": FrameworkAdapterSpec("vertexai", "generate_content", "dict", notes="Vertex AI client shim."),
|
|
104
|
+
"vllm": FrameworkAdapterSpec("vllm", "generate", "dict", notes="vLLM server/client shim."),
|
|
105
|
+
"xai": FrameworkAdapterSpec("xai", "chat", "dict", notes="xAI client shim."),
|
|
106
|
+
# Computer-use / browser / multimodal
|
|
107
|
+
"computer_use": FrameworkAdapterSpec("computer_use", "run", "dict", modality="cua", notes="Browser or desktop CUA runner."),
|
|
108
|
+
"browser_use": FrameworkAdapterSpec("browser_use", "run", "dict", modality="cua", notes="Browser automation agent."),
|
|
109
|
+
"playwright": FrameworkAdapterSpec("playwright", "run", "dict", modality="cua", notes="Playwright-backed agent harness."),
|
|
110
|
+
"vision_agent": FrameworkAdapterSpec("vision_agent", "run", "dict", modality="image", notes="Image or multimodal agent."),
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
_DISCOVERY_METHOD_ORDER = (
|
|
115
|
+
"ainvoke",
|
|
116
|
+
"invoke",
|
|
117
|
+
"astream",
|
|
118
|
+
"stream",
|
|
119
|
+
"stream_events",
|
|
120
|
+
"execute_task",
|
|
121
|
+
"process_frame",
|
|
122
|
+
"send_message",
|
|
123
|
+
"message_send",
|
|
124
|
+
"call",
|
|
125
|
+
"achat",
|
|
126
|
+
"chat",
|
|
127
|
+
"responses.create",
|
|
128
|
+
"chat.completions.create",
|
|
129
|
+
"messages.create",
|
|
130
|
+
"kickoff",
|
|
131
|
+
"query",
|
|
132
|
+
"process",
|
|
133
|
+
"respond",
|
|
134
|
+
"run",
|
|
135
|
+
"run_stream",
|
|
136
|
+
"arun",
|
|
137
|
+
"send",
|
|
138
|
+
"completion",
|
|
139
|
+
"call_tool",
|
|
140
|
+
"invoke_model",
|
|
141
|
+
"generate_content",
|
|
142
|
+
"generate",
|
|
143
|
+
"__call__",
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
_DISCOVERY_METHOD_INPUT_MODES: dict[str, InputMode] = {
|
|
147
|
+
"ainvoke": "dict",
|
|
148
|
+
"invoke": "dict",
|
|
149
|
+
"astream": "dict",
|
|
150
|
+
"stream": "dict",
|
|
151
|
+
"stream_events": "dict",
|
|
152
|
+
"execute_task": "dict",
|
|
153
|
+
"process_frame": "dict",
|
|
154
|
+
"send_message": "dict",
|
|
155
|
+
"message_send": "dict",
|
|
156
|
+
"responses.create": "text",
|
|
157
|
+
"chat.completions.create": "messages",
|
|
158
|
+
"messages.create": "messages",
|
|
159
|
+
"kickoff": "dict",
|
|
160
|
+
"process": "dict",
|
|
161
|
+
"completion": "dict",
|
|
162
|
+
"call_tool": "dict",
|
|
163
|
+
"invoke_model": "dict",
|
|
164
|
+
"generate_content": "dict",
|
|
165
|
+
"generate": "dict",
|
|
166
|
+
"call": "agent_input",
|
|
167
|
+
"achat": "text",
|
|
168
|
+
"chat": "text",
|
|
169
|
+
"query": "text",
|
|
170
|
+
"respond": "text",
|
|
171
|
+
"run": "text",
|
|
172
|
+
"run_stream": "text",
|
|
173
|
+
"arun": "text",
|
|
174
|
+
"send": "text",
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
_STREAMING_METHODS = {"astream", "stream", "stream_events", "run_stream"}
|
|
178
|
+
|
|
179
|
+
_KEYWORD_INPUT_NAMES = (
|
|
180
|
+
"inputs",
|
|
181
|
+
"input",
|
|
182
|
+
"payload",
|
|
183
|
+
"frame",
|
|
184
|
+
"request",
|
|
185
|
+
"contents",
|
|
186
|
+
"arguments",
|
|
187
|
+
"task",
|
|
188
|
+
"user_prompt",
|
|
189
|
+
"prompt",
|
|
190
|
+
"message",
|
|
191
|
+
"messages",
|
|
192
|
+
"query",
|
|
193
|
+
"data",
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
_METHOD_INPUT_KEY_PREFERENCES = {
|
|
197
|
+
"execute_task": ("task", "input", "payload"),
|
|
198
|
+
"kickoff": ("inputs", "input", "payload"),
|
|
199
|
+
"run": ("task", "user_prompt", "prompt", "input"),
|
|
200
|
+
"arun": ("task", "user_prompt", "prompt", "input"),
|
|
201
|
+
"run_stream": ("task", "user_prompt", "prompt", "input"),
|
|
202
|
+
"send_message": ("message", "payload", "input"),
|
|
203
|
+
"message_send": ("message", "payload", "input"),
|
|
204
|
+
"send": ("message", "messages", "input"),
|
|
205
|
+
"achat": ("message", "messages", "input"),
|
|
206
|
+
"chat": ("message", "messages", "input"),
|
|
207
|
+
"query": ("query", "input", "message"),
|
|
208
|
+
"respond": ("message", "input", "payload"),
|
|
209
|
+
"process": ("frame", "payload", "input", "data"),
|
|
210
|
+
"process_frame": ("frame", "payload", "input", "data"),
|
|
211
|
+
"responses.create": ("input", "messages", "payload"),
|
|
212
|
+
"chat.completions.create": ("messages", "input", "payload"),
|
|
213
|
+
"messages.create": ("messages", "input", "payload"),
|
|
214
|
+
"completion": ("request", "payload", "input"),
|
|
215
|
+
"call_tool": ("payload", "input", "arguments"),
|
|
216
|
+
"invoke_model": ("payload", "input", "request"),
|
|
217
|
+
"generate_content": ("contents", "input", "payload"),
|
|
218
|
+
"generate": ("prompt", "input", "payload"),
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
_DISCOVERY_INPUT_MODE_ORDER: tuple[InputMode, ...] = (
|
|
222
|
+
"dict",
|
|
223
|
+
"text",
|
|
224
|
+
"agent_input",
|
|
225
|
+
"messages",
|
|
226
|
+
"auto",
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def supported_frameworks() -> list[str]:
|
|
231
|
+
"""Return built-in framework preset names.
|
|
232
|
+
|
|
233
|
+
``wrap_framework`` also accepts unknown framework names as custom adapters
|
|
234
|
+
when the caller supplies method/input-mode overrides or the generic wrapper
|
|
235
|
+
can infer a callable method.
|
|
236
|
+
"""
|
|
237
|
+
|
|
238
|
+
return sorted(FRAMEWORK_PRESETS)
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def framework_adapter_contract(
|
|
242
|
+
framework: str,
|
|
243
|
+
*,
|
|
244
|
+
target: str | None = None,
|
|
245
|
+
method: str | None = None,
|
|
246
|
+
input_mode: InputMode | None = None,
|
|
247
|
+
input_key: str | None = None,
|
|
248
|
+
input_kwargs: Mapping[str, Any] | None = None,
|
|
249
|
+
modality: str | None = None,
|
|
250
|
+
trace_runtime: bool = True,
|
|
251
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
252
|
+
) -> dict[str, Any]:
|
|
253
|
+
"""Return the native adapter contract used for framework simulation.
|
|
254
|
+
|
|
255
|
+
The contract is import-free and local: it describes the framework shim,
|
|
256
|
+
lifecycle, transport, capabilities, schemas, and replay requirements without
|
|
257
|
+
pulling in LangGraph, LiveKit, Pipecat, or any other framework package.
|
|
258
|
+
"""
|
|
259
|
+
|
|
260
|
+
meta = dict(metadata or {})
|
|
261
|
+
key = _framework_key(framework)
|
|
262
|
+
spec = FRAMEWORK_PRESETS.get(key)
|
|
263
|
+
adapter_kind = "preset" if spec is not None else "custom"
|
|
264
|
+
resolved_method = str(method or (spec.method if spec else "") or "auto")
|
|
265
|
+
resolved_input_mode = str(input_mode or (spec.input_mode if spec else "") or "auto")
|
|
266
|
+
resolved_modality = str(modality or meta.get("modality") or (spec.modality if spec else "text"))
|
|
267
|
+
transport = str((spec.transport if spec else "") or _default_transport(resolved_modality))
|
|
268
|
+
lifecycle_hooks = list(
|
|
269
|
+
spec.lifecycle_hooks
|
|
270
|
+
if spec and spec.lifecycle_hooks
|
|
271
|
+
else _default_lifecycle_hooks(resolved_modality)
|
|
272
|
+
)
|
|
273
|
+
capabilities = list(
|
|
274
|
+
spec.capabilities
|
|
275
|
+
if spec and spec.capabilities
|
|
276
|
+
else _default_capabilities(resolved_modality, resolved_input_mode)
|
|
277
|
+
)
|
|
278
|
+
target_scheme = urlparse(str(target or "")).scheme.lower()
|
|
279
|
+
target_is_external = target_scheme in {"http", "https"}
|
|
280
|
+
local_fixture = not target_is_external
|
|
281
|
+
|
|
282
|
+
contract: dict[str, Any] = {
|
|
283
|
+
"kind": "agent-learning.framework-adapter-contract.v1",
|
|
284
|
+
"framework": key,
|
|
285
|
+
"adapter": adapter_kind,
|
|
286
|
+
"method": resolved_method,
|
|
287
|
+
"input_mode": resolved_input_mode,
|
|
288
|
+
"modality": resolved_modality,
|
|
289
|
+
"transport": transport,
|
|
290
|
+
"lifecycle_hooks": lifecycle_hooks,
|
|
291
|
+
"capabilities": capabilities,
|
|
292
|
+
"schemas": {
|
|
293
|
+
"input": _input_schema(resolved_input_mode),
|
|
294
|
+
"output": _output_schema(),
|
|
295
|
+
},
|
|
296
|
+
"trace_runtime": bool(trace_runtime),
|
|
297
|
+
"requires_external_service": False,
|
|
298
|
+
"local_executable_fixture": local_fixture,
|
|
299
|
+
"evidence_requirements": [
|
|
300
|
+
"framework_runtime",
|
|
301
|
+
"framework_trace",
|
|
302
|
+
"tool_calls",
|
|
303
|
+
"adapter_conformance",
|
|
304
|
+
"metric_evidence",
|
|
305
|
+
],
|
|
306
|
+
}
|
|
307
|
+
if input_key:
|
|
308
|
+
contract["input_key"] = str(input_key)
|
|
309
|
+
input_kwargs_keys = sorted(str(key) for key in dict(input_kwargs or {}))
|
|
310
|
+
if input_kwargs_keys:
|
|
311
|
+
contract["input_kwargs_keys"] = input_kwargs_keys
|
|
312
|
+
if target:
|
|
313
|
+
contract["target"] = str(target)
|
|
314
|
+
contract["target_scheme"] = target_scheme
|
|
315
|
+
if spec and spec.notes:
|
|
316
|
+
contract["notes"] = spec.notes
|
|
317
|
+
if key in {"openenv", "gymnasium", "gymnasium_env", "environment_replay"}:
|
|
318
|
+
contract["evidence_requirements"] = [
|
|
319
|
+
*contract["evidence_requirements"],
|
|
320
|
+
"openenv",
|
|
321
|
+
]
|
|
322
|
+
return contract
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def framework_adapter_contract_matrix(
|
|
326
|
+
frameworks: Sequence[str] | str | None = None,
|
|
327
|
+
*,
|
|
328
|
+
targets: Mapping[str, str] | None = None,
|
|
329
|
+
methods: Mapping[str, str] | None = None,
|
|
330
|
+
input_modes: Mapping[str, InputMode] | None = None,
|
|
331
|
+
modalities: Mapping[str, str] | None = None,
|
|
332
|
+
trace_runtime: bool = True,
|
|
333
|
+
allow_external_targets: bool = False,
|
|
334
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
335
|
+
) -> dict[str, Any]:
|
|
336
|
+
"""Return a native, import-free adapter contract matrix.
|
|
337
|
+
|
|
338
|
+
The matrix is a first-party certification artifact: it proves what the
|
|
339
|
+
Agent Learning simulator can run through local adapter fixtures without
|
|
340
|
+
importing or calling LangGraph, LiveKit, Pipecat, or other frameworks.
|
|
341
|
+
HTTP/HTTPS targets are rejected by default so this path stays native unless
|
|
342
|
+
a caller explicitly opts into external target documentation.
|
|
343
|
+
"""
|
|
344
|
+
|
|
345
|
+
framework_keys = _framework_matrix_keys(frameworks)
|
|
346
|
+
target_map = {
|
|
347
|
+
_framework_key(key): str(value)
|
|
348
|
+
for key, value in dict(targets or {}).items()
|
|
349
|
+
}
|
|
350
|
+
method_map = {
|
|
351
|
+
_framework_key(key): str(value)
|
|
352
|
+
for key, value in dict(methods or {}).items()
|
|
353
|
+
}
|
|
354
|
+
input_mode_map = {
|
|
355
|
+
_framework_key(key): value for key, value in dict(input_modes or {}).items()
|
|
356
|
+
}
|
|
357
|
+
modality_map = {
|
|
358
|
+
_framework_key(key): str(value)
|
|
359
|
+
for key, value in dict(modalities or {}).items()
|
|
360
|
+
}
|
|
361
|
+
external_targets = {
|
|
362
|
+
key: target
|
|
363
|
+
for key, target in target_map.items()
|
|
364
|
+
if _is_external_target(target)
|
|
365
|
+
}
|
|
366
|
+
if external_targets and not allow_external_targets:
|
|
367
|
+
blocked = ", ".join(
|
|
368
|
+
f"{key}={target}" for key, target in sorted(external_targets.items())
|
|
369
|
+
)
|
|
370
|
+
raise ValueError(
|
|
371
|
+
"external targets are disabled for native framework adapter matrices: "
|
|
372
|
+
f"{blocked}"
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
contracts = [
|
|
376
|
+
framework_adapter_contract(
|
|
377
|
+
key,
|
|
378
|
+
target=target_map.get(key) or _local_fixture_target(key),
|
|
379
|
+
method=method_map.get(key),
|
|
380
|
+
input_mode=input_mode_map.get(key),
|
|
381
|
+
modality=modality_map.get(key),
|
|
382
|
+
trace_runtime=trace_runtime,
|
|
383
|
+
metadata=copy_metadata,
|
|
384
|
+
)
|
|
385
|
+
for key in framework_keys
|
|
386
|
+
for copy_metadata in [dict(metadata or {})]
|
|
387
|
+
]
|
|
388
|
+
profiles = [
|
|
389
|
+
_framework_adapter_capability_profile_from_contract(contract)
|
|
390
|
+
for contract in contracts
|
|
391
|
+
]
|
|
392
|
+
findings = _framework_matrix_findings(contracts)
|
|
393
|
+
summary = _framework_matrix_summary(contracts)
|
|
394
|
+
return {
|
|
395
|
+
"kind": "agent-learning.framework-adapter-contract-matrix.v1",
|
|
396
|
+
"status": "passed" if not findings else "failed",
|
|
397
|
+
"requires_external_service": False,
|
|
398
|
+
"runtime": "in_process",
|
|
399
|
+
"allow_external_targets": bool(allow_external_targets),
|
|
400
|
+
"framework_count": len(framework_keys),
|
|
401
|
+
"frameworks": framework_keys,
|
|
402
|
+
"contracts": contracts,
|
|
403
|
+
"profiles": profiles,
|
|
404
|
+
"summary": summary,
|
|
405
|
+
"profile_summary": _framework_profile_collection_summary(profiles),
|
|
406
|
+
"findings": findings,
|
|
407
|
+
"contract_quality_gate": {
|
|
408
|
+
"kind": "agent-learning.framework-adapter-contract.v1",
|
|
409
|
+
"required_frameworks": framework_keys,
|
|
410
|
+
"require_trace_runtime": bool(trace_runtime),
|
|
411
|
+
"require_local_executable_fixture": not bool(allow_external_targets),
|
|
412
|
+
"require_no_external_service": True,
|
|
413
|
+
"require_target": True,
|
|
414
|
+
"forbidden_target_schemes": (
|
|
415
|
+
[] if allow_external_targets else ["http", "https"]
|
|
416
|
+
),
|
|
417
|
+
"required_schema_sections": ["input", "output"],
|
|
418
|
+
"required_lifecycle_hooks": ["setup", "teardown"],
|
|
419
|
+
"required_capabilities": ["messages", "tool_calls", "runtime_trace"],
|
|
420
|
+
"required_evidence_requirements": [
|
|
421
|
+
"framework_runtime",
|
|
422
|
+
"framework_trace",
|
|
423
|
+
"tool_calls",
|
|
424
|
+
"adapter_conformance",
|
|
425
|
+
"metric_evidence",
|
|
426
|
+
],
|
|
427
|
+
},
|
|
428
|
+
"evidence_requirements": [
|
|
429
|
+
"framework_runtime",
|
|
430
|
+
"framework_trace",
|
|
431
|
+
"adapter_conformance",
|
|
432
|
+
"metric_evidence",
|
|
433
|
+
"matrix_coverage",
|
|
434
|
+
],
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
def framework_adapter_capability_profile(
|
|
439
|
+
framework: str,
|
|
440
|
+
*,
|
|
441
|
+
target: str | None = None,
|
|
442
|
+
method: str | None = None,
|
|
443
|
+
input_mode: InputMode | None = None,
|
|
444
|
+
input_key: str | None = None,
|
|
445
|
+
input_kwargs: Mapping[str, Any] | None = None,
|
|
446
|
+
modality: str | None = None,
|
|
447
|
+
trace_runtime: bool = True,
|
|
448
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
449
|
+
contract: Mapping[str, Any] | None = None,
|
|
450
|
+
) -> dict[str, Any]:
|
|
451
|
+
"""Return a portable trinity profile for one framework adapter.
|
|
452
|
+
|
|
453
|
+
Profiles are derived from the native adapter contract and intentionally do
|
|
454
|
+
not import LangChain, LiveKit, Pipecat, or any other framework. The payload
|
|
455
|
+
is the shared handshake between simulate-sdk, ai-evaluation, and agent-opt:
|
|
456
|
+
it tells each library which local contract, metric, and optimization layer
|
|
457
|
+
represent the framework surface.
|
|
458
|
+
"""
|
|
459
|
+
|
|
460
|
+
selected_contract = (
|
|
461
|
+
copy.deepcopy(dict(contract))
|
|
462
|
+
if contract is not None
|
|
463
|
+
else framework_adapter_contract(
|
|
464
|
+
framework,
|
|
465
|
+
target=target,
|
|
466
|
+
method=method,
|
|
467
|
+
input_mode=input_mode,
|
|
468
|
+
input_key=input_key,
|
|
469
|
+
input_kwargs=input_kwargs,
|
|
470
|
+
modality=modality,
|
|
471
|
+
trace_runtime=trace_runtime,
|
|
472
|
+
metadata=metadata,
|
|
473
|
+
)
|
|
474
|
+
)
|
|
475
|
+
return _framework_adapter_capability_profile_from_contract(selected_contract)
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def framework_adapter_capability_profiles(
|
|
479
|
+
frameworks: Sequence[str] | str | None = None,
|
|
480
|
+
*,
|
|
481
|
+
matrix: Mapping[str, Any] | None = None,
|
|
482
|
+
targets: Mapping[str, str] | None = None,
|
|
483
|
+
methods: Mapping[str, str] | None = None,
|
|
484
|
+
input_modes: Mapping[str, InputMode] | None = None,
|
|
485
|
+
modalities: Mapping[str, str] | None = None,
|
|
486
|
+
trace_runtime: bool = True,
|
|
487
|
+
allow_external_targets: bool = False,
|
|
488
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
489
|
+
) -> dict[str, Any]:
|
|
490
|
+
"""Return framework capability profiles for a matrix or framework list."""
|
|
491
|
+
|
|
492
|
+
matrix_payload = (
|
|
493
|
+
copy.deepcopy(dict(matrix))
|
|
494
|
+
if matrix is not None
|
|
495
|
+
else framework_adapter_contract_matrix(
|
|
496
|
+
frameworks,
|
|
497
|
+
targets=targets,
|
|
498
|
+
methods=methods,
|
|
499
|
+
input_modes=input_modes,
|
|
500
|
+
modalities=modalities,
|
|
501
|
+
trace_runtime=trace_runtime,
|
|
502
|
+
allow_external_targets=allow_external_targets,
|
|
503
|
+
metadata=metadata,
|
|
504
|
+
)
|
|
505
|
+
)
|
|
506
|
+
contracts = [
|
|
507
|
+
copy.deepcopy(dict(contract))
|
|
508
|
+
for contract in matrix_payload.get("contracts", []) or []
|
|
509
|
+
if isinstance(contract, Mapping)
|
|
510
|
+
]
|
|
511
|
+
if not contracts:
|
|
512
|
+
framework_keys = _framework_matrix_keys(
|
|
513
|
+
matrix_payload.get("frameworks") or frameworks
|
|
514
|
+
)
|
|
515
|
+
contracts = [
|
|
516
|
+
framework_adapter_contract(
|
|
517
|
+
key,
|
|
518
|
+
target=(dict(targets or {}).get(key) if targets else None),
|
|
519
|
+
method=(dict(methods or {}).get(key) if methods else None),
|
|
520
|
+
input_mode=(dict(input_modes or {}).get(key) if input_modes else None),
|
|
521
|
+
modality=(dict(modalities or {}).get(key) if modalities else None),
|
|
522
|
+
trace_runtime=trace_runtime,
|
|
523
|
+
metadata=metadata,
|
|
524
|
+
)
|
|
525
|
+
for key in framework_keys
|
|
526
|
+
]
|
|
527
|
+
|
|
528
|
+
profiles = [
|
|
529
|
+
_framework_adapter_capability_profile_from_contract(contract)
|
|
530
|
+
for contract in contracts
|
|
531
|
+
]
|
|
532
|
+
required_frameworks = _framework_matrix_keys(
|
|
533
|
+
matrix_payload.get("frameworks") or frameworks
|
|
534
|
+
)
|
|
535
|
+
findings = _framework_profile_collection_findings(
|
|
536
|
+
profiles,
|
|
537
|
+
required_frameworks=required_frameworks,
|
|
538
|
+
)
|
|
539
|
+
return {
|
|
540
|
+
"kind": "agent-learning.framework-adapter-capability-profiles.v1",
|
|
541
|
+
"status": "passed" if not findings else "failed",
|
|
542
|
+
"passed": not findings,
|
|
543
|
+
"requires_external_service": False,
|
|
544
|
+
"framework_count": len(required_frameworks),
|
|
545
|
+
"profile_count": len(profiles),
|
|
546
|
+
"frameworks": required_frameworks,
|
|
547
|
+
"profiles": profiles,
|
|
548
|
+
"summary": _framework_profile_collection_summary(profiles),
|
|
549
|
+
"findings": findings,
|
|
550
|
+
"source_matrix_kind": matrix_payload.get("kind"),
|
|
551
|
+
"evidence_requirements": [
|
|
552
|
+
"framework_adapter_contract",
|
|
553
|
+
"framework_adapter_profile",
|
|
554
|
+
"framework_runtime",
|
|
555
|
+
"framework_trace",
|
|
556
|
+
"metric_evidence",
|
|
557
|
+
"optimization_lineage",
|
|
558
|
+
],
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
|
|
562
|
+
def wrap_framework(
|
|
563
|
+
framework: str,
|
|
564
|
+
agent: Any,
|
|
565
|
+
*,
|
|
566
|
+
target: str | None = None,
|
|
567
|
+
method: str | None = None,
|
|
568
|
+
input_mode: InputMode | None = None,
|
|
569
|
+
input_key: str | None = None,
|
|
570
|
+
input_kwargs: Mapping[str, Any] | None = None,
|
|
571
|
+
system_prompt: str | None = None,
|
|
572
|
+
output_key: str | None = None,
|
|
573
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
574
|
+
trace_runtime: bool = False,
|
|
575
|
+
runtime_metadata: Optional[Dict[str, Any]] = None,
|
|
576
|
+
) -> AgentWrapper:
|
|
577
|
+
"""
|
|
578
|
+
Wrap a known or custom framework by name without importing that framework.
|
|
579
|
+
|
|
580
|
+
Presets are intentionally thin. They encode the most common method/payload
|
|
581
|
+
shape while leaving escape hatches for custom method, input_mode, and
|
|
582
|
+
output_key.
|
|
583
|
+
"""
|
|
584
|
+
|
|
585
|
+
key = framework.lower().replace("-", "_")
|
|
586
|
+
spec = FRAMEWORK_PRESETS.get(key)
|
|
587
|
+
raw_metadata = dict(metadata or {})
|
|
588
|
+
contract = raw_metadata.get("framework_adapter_contract")
|
|
589
|
+
if not isinstance(contract, dict):
|
|
590
|
+
contract = framework_adapter_contract(
|
|
591
|
+
key,
|
|
592
|
+
target=target,
|
|
593
|
+
method=method,
|
|
594
|
+
input_mode=input_mode,
|
|
595
|
+
input_key=input_key,
|
|
596
|
+
input_kwargs=input_kwargs,
|
|
597
|
+
trace_runtime=trace_runtime,
|
|
598
|
+
metadata=raw_metadata,
|
|
599
|
+
)
|
|
600
|
+
runtime = dict(runtime_metadata or {})
|
|
601
|
+
runtime.setdefault("framework_adapter_contract", contract)
|
|
602
|
+
if spec is None:
|
|
603
|
+
return GenericAgentWrapper(
|
|
604
|
+
agent,
|
|
605
|
+
method=method,
|
|
606
|
+
input_mode=input_mode or "auto",
|
|
607
|
+
input_key=input_key,
|
|
608
|
+
input_kwargs=input_kwargs,
|
|
609
|
+
output_key=output_key,
|
|
610
|
+
system_prompt=system_prompt,
|
|
611
|
+
metadata={
|
|
612
|
+
"framework": key,
|
|
613
|
+
"modality": str(raw_metadata.get("modality") or "text"),
|
|
614
|
+
"adapter": "custom",
|
|
615
|
+
"framework_adapter_contract": contract,
|
|
616
|
+
**raw_metadata,
|
|
617
|
+
},
|
|
618
|
+
trace_runtime=trace_runtime,
|
|
619
|
+
runtime_metadata=runtime,
|
|
620
|
+
)
|
|
621
|
+
|
|
622
|
+
return GenericAgentWrapper(
|
|
623
|
+
agent,
|
|
624
|
+
method=method or spec.method,
|
|
625
|
+
input_mode=input_mode or spec.input_mode,
|
|
626
|
+
input_key=input_key,
|
|
627
|
+
input_kwargs=input_kwargs,
|
|
628
|
+
output_key=output_key,
|
|
629
|
+
system_prompt=system_prompt,
|
|
630
|
+
metadata={
|
|
631
|
+
"framework": spec.name,
|
|
632
|
+
"modality": spec.modality,
|
|
633
|
+
"framework_adapter_contract": contract,
|
|
634
|
+
**raw_metadata,
|
|
635
|
+
},
|
|
636
|
+
trace_runtime=trace_runtime,
|
|
637
|
+
runtime_metadata=runtime,
|
|
638
|
+
)
|
|
639
|
+
|
|
640
|
+
|
|
641
|
+
async def probe_framework_adapter(
|
|
642
|
+
framework: str,
|
|
643
|
+
agent: Any,
|
|
644
|
+
*,
|
|
645
|
+
cases: Sequence[Mapping[str, Any]] | None = None,
|
|
646
|
+
target: str | None = None,
|
|
647
|
+
method: str | Callable[..., Any] | None = None,
|
|
648
|
+
input_mode: InputMode | None = None,
|
|
649
|
+
input_key: str | None = None,
|
|
650
|
+
input_kwargs: Mapping[str, Any] | None = None,
|
|
651
|
+
system_prompt: str | None = None,
|
|
652
|
+
output_key: str | None = None,
|
|
653
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
654
|
+
trace_runtime: bool = True,
|
|
655
|
+
allow_external_target: bool = False,
|
|
656
|
+
) -> dict[str, Any]:
|
|
657
|
+
"""Run a local adapter conformance probe for any framework shim.
|
|
658
|
+
|
|
659
|
+
The probe is intentionally import-free: callers pass an already-created
|
|
660
|
+
LangChain/LangGraph/LiveKit/Pipecat/custom object or plain callable. The
|
|
661
|
+
function wraps it with the same generic adapter used by manifests, executes
|
|
662
|
+
representative cases, and returns runtime evidence that can feed evals,
|
|
663
|
+
reports, optimizer proofs, or Future AGI UI cards.
|
|
664
|
+
"""
|
|
665
|
+
|
|
666
|
+
if target and _is_external_target(target) and not allow_external_target:
|
|
667
|
+
raise ValueError(
|
|
668
|
+
"external targets are disabled for framework adapter probes; "
|
|
669
|
+
"set allow_external_target=True only when the user explicitly "
|
|
670
|
+
"wants to test that live workload"
|
|
671
|
+
)
|
|
672
|
+
|
|
673
|
+
key = _framework_key(framework)
|
|
674
|
+
method_name = _probe_method_name(method)
|
|
675
|
+
selected_metadata = dict(metadata or {})
|
|
676
|
+
contract = framework_adapter_contract(
|
|
677
|
+
key,
|
|
678
|
+
target=target,
|
|
679
|
+
method=method_name,
|
|
680
|
+
input_mode=input_mode,
|
|
681
|
+
input_key=input_key,
|
|
682
|
+
input_kwargs=input_kwargs,
|
|
683
|
+
trace_runtime=trace_runtime,
|
|
684
|
+
metadata=selected_metadata,
|
|
685
|
+
)
|
|
686
|
+
callable_signature = _adapter_callable_signature(
|
|
687
|
+
agent,
|
|
688
|
+
method=method,
|
|
689
|
+
method_name=str(contract.get("method") or method_name or ""),
|
|
690
|
+
)
|
|
691
|
+
if callable_signature:
|
|
692
|
+
contract["callable_signature"] = callable_signature
|
|
693
|
+
selected_metadata["framework_adapter_contract"] = contract
|
|
694
|
+
wrapper = wrap_framework(
|
|
695
|
+
key,
|
|
696
|
+
agent,
|
|
697
|
+
target=target,
|
|
698
|
+
method=method,
|
|
699
|
+
input_mode=input_mode,
|
|
700
|
+
input_key=input_key,
|
|
701
|
+
input_kwargs=input_kwargs,
|
|
702
|
+
system_prompt=system_prompt,
|
|
703
|
+
output_key=output_key,
|
|
704
|
+
metadata=selected_metadata,
|
|
705
|
+
trace_runtime=trace_runtime,
|
|
706
|
+
runtime_metadata={"framework_adapter_contract": contract},
|
|
707
|
+
)
|
|
708
|
+
|
|
709
|
+
probe_cases = _probe_cases(cases)
|
|
710
|
+
case_results: list[dict[str, Any]] = []
|
|
711
|
+
findings: list[dict[str, Any]] = []
|
|
712
|
+
|
|
713
|
+
for index, case in enumerate(probe_cases, start=1):
|
|
714
|
+
case_result = await _run_probe_case(
|
|
715
|
+
wrapper,
|
|
716
|
+
key,
|
|
717
|
+
case,
|
|
718
|
+
index=index,
|
|
719
|
+
trace_runtime=trace_runtime,
|
|
720
|
+
contract=contract,
|
|
721
|
+
)
|
|
722
|
+
case_results.append(case_result)
|
|
723
|
+
findings.extend(case_result.get("findings", []))
|
|
724
|
+
|
|
725
|
+
summary = _probe_summary(case_results, contract)
|
|
726
|
+
status = "passed" if not findings else "failed"
|
|
727
|
+
return {
|
|
728
|
+
"kind": "agent-learning.framework-adapter-probe.v1",
|
|
729
|
+
"status": status,
|
|
730
|
+
"passed": status == "passed",
|
|
731
|
+
"framework": key,
|
|
732
|
+
"method": method_name or str(contract.get("method") or "auto"),
|
|
733
|
+
"input_mode": str(input_mode or contract.get("input_mode") or "auto"),
|
|
734
|
+
"input_key": str(input_key or contract.get("input_key") or "")
|
|
735
|
+
or None,
|
|
736
|
+
"input_kwargs_keys": sorted(str(key) for key in dict(input_kwargs or {})),
|
|
737
|
+
"requires_external_service": bool(contract.get("requires_external_service")),
|
|
738
|
+
"allow_external_target": bool(allow_external_target),
|
|
739
|
+
"contract": contract,
|
|
740
|
+
"summary": summary,
|
|
741
|
+
"cases": case_results,
|
|
742
|
+
"findings": findings,
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
|
|
746
|
+
def run_framework_adapter_probe(
|
|
747
|
+
framework: str,
|
|
748
|
+
agent: Any,
|
|
749
|
+
**kwargs: Any,
|
|
750
|
+
) -> dict[str, Any]:
|
|
751
|
+
"""Synchronous wrapper for :func:`probe_framework_adapter`.
|
|
752
|
+
|
|
753
|
+
Use ``await probe_framework_adapter(...)`` when already inside an event loop.
|
|
754
|
+
"""
|
|
755
|
+
|
|
756
|
+
try:
|
|
757
|
+
asyncio.get_running_loop()
|
|
758
|
+
except RuntimeError:
|
|
759
|
+
return asyncio.run(probe_framework_adapter(framework, agent, **kwargs))
|
|
760
|
+
raise RuntimeError(
|
|
761
|
+
"run_framework_adapter_probe cannot run inside an active event loop; "
|
|
762
|
+
"await probe_framework_adapter(...) instead"
|
|
763
|
+
)
|
|
764
|
+
|
|
765
|
+
|
|
766
|
+
def discover_framework_adapter(
|
|
767
|
+
framework: str,
|
|
768
|
+
agent: Any = None,
|
|
769
|
+
*,
|
|
770
|
+
target: str | None = None,
|
|
771
|
+
method_candidates: Sequence[str | None] | None = None,
|
|
772
|
+
input_mode_candidates: Sequence[InputMode] | None = None,
|
|
773
|
+
modality: str | None = None,
|
|
774
|
+
trace_runtime: bool = True,
|
|
775
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
776
|
+
allow_external_target: bool = False,
|
|
777
|
+
max_candidates: int | None = 24,
|
|
778
|
+
) -> dict[str, Any]:
|
|
779
|
+
"""Discover local adapter candidates for an arbitrary framework object.
|
|
780
|
+
|
|
781
|
+
Discovery never imports optional framework packages or calls the supplied
|
|
782
|
+
agent. It inspects callable attributes, combines them with the built-in
|
|
783
|
+
framework presets, and returns ranked method/input-mode contracts that can
|
|
784
|
+
be passed directly to ``optimize_framework_adapter_probe``.
|
|
785
|
+
"""
|
|
786
|
+
|
|
787
|
+
if target and _is_external_target(target) and not allow_external_target:
|
|
788
|
+
raise ValueError(
|
|
789
|
+
"external targets are disabled for framework adapter discovery; "
|
|
790
|
+
"set allow_external_target=True only when the user explicitly "
|
|
791
|
+
"wants to document that live workload"
|
|
792
|
+
)
|
|
793
|
+
if max_candidates is not None and int(max_candidates) <= 0:
|
|
794
|
+
raise ValueError("max_candidates must be greater than zero")
|
|
795
|
+
|
|
796
|
+
key = _framework_key(framework)
|
|
797
|
+
spec = FRAMEWORK_PRESETS.get(key)
|
|
798
|
+
selected_metadata = dict(metadata or {})
|
|
799
|
+
inventory = _adapter_discovery_inventory(agent)
|
|
800
|
+
methods = _adapter_discovery_methods(
|
|
801
|
+
agent,
|
|
802
|
+
spec=spec,
|
|
803
|
+
method_candidates=method_candidates,
|
|
804
|
+
)
|
|
805
|
+
input_modes = _adapter_discovery_input_modes(
|
|
806
|
+
spec=spec,
|
|
807
|
+
input_mode_candidates=input_mode_candidates,
|
|
808
|
+
)
|
|
809
|
+
|
|
810
|
+
candidates: list[dict[str, Any]] = []
|
|
811
|
+
seen_pairs: set[tuple[str, str]] = set()
|
|
812
|
+
explicit_methods = {
|
|
813
|
+
str(method)
|
|
814
|
+
for method in method_candidates or []
|
|
815
|
+
if method is not None and str(method)
|
|
816
|
+
}
|
|
817
|
+
|
|
818
|
+
for method_name in methods:
|
|
819
|
+
for input_mode in _adapter_discovery_modes_for_method(
|
|
820
|
+
method_name,
|
|
821
|
+
input_modes,
|
|
822
|
+
spec=spec,
|
|
823
|
+
):
|
|
824
|
+
pair = (str(method_name or ""), str(input_mode))
|
|
825
|
+
if pair in seen_pairs:
|
|
826
|
+
continue
|
|
827
|
+
seen_pairs.add(pair)
|
|
828
|
+
input_key = _adapter_candidate_input_key(
|
|
829
|
+
agent,
|
|
830
|
+
method_name,
|
|
831
|
+
input_mode,
|
|
832
|
+
)
|
|
833
|
+
contract = framework_adapter_contract(
|
|
834
|
+
key,
|
|
835
|
+
target=target,
|
|
836
|
+
method=method_name,
|
|
837
|
+
input_mode=input_mode,
|
|
838
|
+
input_key=input_key,
|
|
839
|
+
modality=modality,
|
|
840
|
+
trace_runtime=trace_runtime,
|
|
841
|
+
metadata=selected_metadata,
|
|
842
|
+
)
|
|
843
|
+
scoring = _adapter_discovery_score(
|
|
844
|
+
method_name,
|
|
845
|
+
input_mode,
|
|
846
|
+
spec=spec,
|
|
847
|
+
inventory=inventory,
|
|
848
|
+
explicit_methods=explicit_methods,
|
|
849
|
+
)
|
|
850
|
+
adapter_candidate: dict[str, Any] = {
|
|
851
|
+
"input_mode": input_mode,
|
|
852
|
+
"trace_runtime": bool(trace_runtime),
|
|
853
|
+
}
|
|
854
|
+
if method_name:
|
|
855
|
+
adapter_candidate["method"] = method_name
|
|
856
|
+
if input_key:
|
|
857
|
+
adapter_candidate["input_key"] = input_key
|
|
858
|
+
if target:
|
|
859
|
+
adapter_candidate["target"] = str(target)
|
|
860
|
+
candidates.append(
|
|
861
|
+
{
|
|
862
|
+
"rank": 0,
|
|
863
|
+
"framework": key,
|
|
864
|
+
"method": method_name or "auto",
|
|
865
|
+
"input_mode": input_mode,
|
|
866
|
+
"score": scoring["score"],
|
|
867
|
+
"reasons": scoring["reasons"],
|
|
868
|
+
"agent_method_present": _adapter_method_present(
|
|
869
|
+
inventory,
|
|
870
|
+
method_name,
|
|
871
|
+
),
|
|
872
|
+
"contract": contract,
|
|
873
|
+
"adapter_candidate": adapter_candidate,
|
|
874
|
+
}
|
|
875
|
+
)
|
|
876
|
+
|
|
877
|
+
candidates.sort(
|
|
878
|
+
key=lambda item: (
|
|
879
|
+
-float(item["score"]),
|
|
880
|
+
_adapter_method_rank(str(item.get("method") or "")),
|
|
881
|
+
_adapter_input_mode_rank(str(item.get("input_mode") or "")),
|
|
882
|
+
)
|
|
883
|
+
)
|
|
884
|
+
if max_candidates is not None:
|
|
885
|
+
candidates = candidates[: int(max_candidates)]
|
|
886
|
+
for rank, candidate in enumerate(candidates, start=1):
|
|
887
|
+
candidate["rank"] = rank
|
|
888
|
+
|
|
889
|
+
adapter_candidates = [
|
|
890
|
+
dict(candidate["adapter_candidate"]) for candidate in candidates
|
|
891
|
+
]
|
|
892
|
+
findings = _adapter_discovery_findings(inventory, candidates)
|
|
893
|
+
status = "passed" if candidates else "failed"
|
|
894
|
+
top = candidates[0] if candidates else {}
|
|
895
|
+
return {
|
|
896
|
+
"kind": "agent-learning.framework-adapter-discovery.v1",
|
|
897
|
+
"status": status,
|
|
898
|
+
"passed": status == "passed",
|
|
899
|
+
"framework": key,
|
|
900
|
+
"target": str(target) if target else None,
|
|
901
|
+
"requires_external_service": False,
|
|
902
|
+
"allow_external_target": bool(allow_external_target),
|
|
903
|
+
"trace_runtime": bool(trace_runtime),
|
|
904
|
+
"agent": inventory,
|
|
905
|
+
"summary": {
|
|
906
|
+
"framework": key,
|
|
907
|
+
"candidate_count": len(candidates),
|
|
908
|
+
"adapter_candidate_count": len(adapter_candidates),
|
|
909
|
+
"max_candidates": max_candidates,
|
|
910
|
+
"top_method": top.get("method"),
|
|
911
|
+
"top_input_mode": top.get("input_mode"),
|
|
912
|
+
"top_score": top.get("score"),
|
|
913
|
+
"agent_provided": bool(inventory.get("provided")),
|
|
914
|
+
"agent_callable": bool(inventory.get("callable")),
|
|
915
|
+
"method_count": len(inventory.get("exposed_methods", [])),
|
|
916
|
+
"local_executable_fixture": not bool(target and _is_external_target(target)),
|
|
917
|
+
},
|
|
918
|
+
"candidates": candidates,
|
|
919
|
+
"adapter_candidates": adapter_candidates,
|
|
920
|
+
"findings": findings,
|
|
921
|
+
"evidence_requirements": [
|
|
922
|
+
"local_introspection",
|
|
923
|
+
"framework_adapter_contract",
|
|
924
|
+
"adapter_candidates",
|
|
925
|
+
"framework_adapter_probe",
|
|
926
|
+
"metric_evidence",
|
|
927
|
+
],
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
|
|
931
|
+
def _framework_key(value: str) -> str:
|
|
932
|
+
return str(value or "custom").strip().lower().replace("-", "_") or "custom"
|
|
933
|
+
|
|
934
|
+
|
|
935
|
+
def _framework_matrix_keys(frameworks: Sequence[str] | str | None) -> list[str]:
|
|
936
|
+
default_frameworks = (
|
|
937
|
+
"langchain",
|
|
938
|
+
"langgraph",
|
|
939
|
+
"llamaindex",
|
|
940
|
+
"crewai",
|
|
941
|
+
"autogen",
|
|
942
|
+
"openai_agents",
|
|
943
|
+
"livekit",
|
|
944
|
+
"pipecat",
|
|
945
|
+
)
|
|
946
|
+
values: Sequence[str] | str = frameworks or default_frameworks
|
|
947
|
+
if isinstance(values, str):
|
|
948
|
+
values = [values]
|
|
949
|
+
keys: list[str] = []
|
|
950
|
+
seen: set[str] = set()
|
|
951
|
+
for value in values:
|
|
952
|
+
key = _framework_key(value)
|
|
953
|
+
if key in seen:
|
|
954
|
+
continue
|
|
955
|
+
seen.add(key)
|
|
956
|
+
keys.append(key)
|
|
957
|
+
if not keys:
|
|
958
|
+
raise ValueError("frameworks must contain at least one framework")
|
|
959
|
+
return keys
|
|
960
|
+
|
|
961
|
+
|
|
962
|
+
def _local_fixture_target(framework: str) -> str:
|
|
963
|
+
return f"agent-learning-fixture://framework/{_framework_key(framework)}"
|
|
964
|
+
|
|
965
|
+
|
|
966
|
+
def _is_external_target(target: str) -> bool:
|
|
967
|
+
return urlparse(str(target or "")).scheme.lower() in {"http", "https"}
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
def _adapter_discovery_inventory(agent: Any) -> dict[str, Any]:
|
|
971
|
+
if agent is None:
|
|
972
|
+
return {
|
|
973
|
+
"provided": False,
|
|
974
|
+
"callable": False,
|
|
975
|
+
"type": None,
|
|
976
|
+
"exposed_methods": [],
|
|
977
|
+
"wrapper": False,
|
|
978
|
+
}
|
|
979
|
+
|
|
980
|
+
exposed_methods = _adapter_public_callable_names(agent)
|
|
981
|
+
return {
|
|
982
|
+
"provided": True,
|
|
983
|
+
"callable": callable(agent),
|
|
984
|
+
"type": _adapter_agent_type(agent),
|
|
985
|
+
"exposed_methods": exposed_methods,
|
|
986
|
+
"wrapper": isinstance(agent, AgentWrapper),
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
|
|
990
|
+
def _adapter_agent_type(agent: Any) -> str:
|
|
991
|
+
if inspect.isfunction(agent) or inspect.ismethod(agent):
|
|
992
|
+
module = getattr(agent, "__module__", "")
|
|
993
|
+
qualname = getattr(agent, "__qualname__", getattr(agent, "__name__", "callable"))
|
|
994
|
+
return f"{module}.{qualname}".strip(".")
|
|
995
|
+
cls = type(agent)
|
|
996
|
+
module = getattr(cls, "__module__", "")
|
|
997
|
+
qualname = getattr(cls, "__qualname__", getattr(cls, "__name__", "object"))
|
|
998
|
+
return f"{module}.{qualname}".strip(".")
|
|
999
|
+
|
|
1000
|
+
|
|
1001
|
+
def _adapter_public_callable_names(agent: Any) -> list[str]:
|
|
1002
|
+
names: list[str] = []
|
|
1003
|
+
seen: set[str] = set()
|
|
1004
|
+
|
|
1005
|
+
for name in _DISCOVERY_METHOD_ORDER:
|
|
1006
|
+
if name == "__call__" and not callable(agent):
|
|
1007
|
+
continue
|
|
1008
|
+
if name != "__call__" and not _adapter_has_callable_method(agent, name):
|
|
1009
|
+
continue
|
|
1010
|
+
seen.add(name)
|
|
1011
|
+
names.append(name)
|
|
1012
|
+
|
|
1013
|
+
try:
|
|
1014
|
+
members = inspect.getmembers_static(agent)
|
|
1015
|
+
except Exception:
|
|
1016
|
+
members = []
|
|
1017
|
+
|
|
1018
|
+
public_names: list[str] = []
|
|
1019
|
+
for name, value in members:
|
|
1020
|
+
if name in seen:
|
|
1021
|
+
continue
|
|
1022
|
+
if name.startswith("_"):
|
|
1023
|
+
continue
|
|
1024
|
+
if not (
|
|
1025
|
+
inspect.isroutine(value)
|
|
1026
|
+
or isinstance(value, (classmethod, staticmethod))
|
|
1027
|
+
or _adapter_has_callable_method(agent, name)
|
|
1028
|
+
):
|
|
1029
|
+
continue
|
|
1030
|
+
if not _adapter_has_callable_method(agent, name):
|
|
1031
|
+
continue
|
|
1032
|
+
public_names.append(name)
|
|
1033
|
+
|
|
1034
|
+
public_names.sort(key=lambda item: (_adapter_method_rank(item), item))
|
|
1035
|
+
names.extend(public_names[:24])
|
|
1036
|
+
return names
|
|
1037
|
+
|
|
1038
|
+
|
|
1039
|
+
def _adapter_resolve_callable_attr_path(agent: Any, method_name: str | None) -> Callable[..., Any] | None:
|
|
1040
|
+
if not method_name:
|
|
1041
|
+
return agent if callable(agent) else None
|
|
1042
|
+
value = agent
|
|
1043
|
+
for raw_part in str(method_name).split("."):
|
|
1044
|
+
part = raw_part.strip()
|
|
1045
|
+
if not part:
|
|
1046
|
+
return None
|
|
1047
|
+
try:
|
|
1048
|
+
value = getattr(value, part)
|
|
1049
|
+
except Exception:
|
|
1050
|
+
return None
|
|
1051
|
+
return value if callable(value) else None
|
|
1052
|
+
|
|
1053
|
+
|
|
1054
|
+
def _adapter_method_leaf(method_name: str | None) -> str:
|
|
1055
|
+
return str(method_name or "").rsplit(".", 1)[-1]
|
|
1056
|
+
|
|
1057
|
+
|
|
1058
|
+
def _adapter_has_callable_method(agent: Any, method_name: str) -> bool:
|
|
1059
|
+
if not method_name:
|
|
1060
|
+
return callable(agent)
|
|
1061
|
+
return _adapter_resolve_callable_attr_path(agent, method_name) is not None
|
|
1062
|
+
|
|
1063
|
+
|
|
1064
|
+
def _adapter_discovery_methods(
|
|
1065
|
+
agent: Any,
|
|
1066
|
+
*,
|
|
1067
|
+
spec: FrameworkAdapterSpec | None,
|
|
1068
|
+
method_candidates: Sequence[str | None] | None,
|
|
1069
|
+
) -> list[str | None]:
|
|
1070
|
+
methods: list[str | None] = []
|
|
1071
|
+
|
|
1072
|
+
for method in method_candidates or []:
|
|
1073
|
+
methods.append(str(method) if method is not None and str(method) else None)
|
|
1074
|
+
if spec and spec.method:
|
|
1075
|
+
methods.append(spec.method)
|
|
1076
|
+
methods.extend(_adapter_public_callable_names(agent) if agent is not None else [])
|
|
1077
|
+
if agent is not None and callable(agent):
|
|
1078
|
+
methods.append(None)
|
|
1079
|
+
methods.extend(_DISCOVERY_METHOD_ORDER)
|
|
1080
|
+
if spec is None:
|
|
1081
|
+
methods.append(None)
|
|
1082
|
+
|
|
1083
|
+
unique: list[str | None] = []
|
|
1084
|
+
seen: set[str] = set()
|
|
1085
|
+
for method in methods:
|
|
1086
|
+
key = str(method or "")
|
|
1087
|
+
if key in seen:
|
|
1088
|
+
continue
|
|
1089
|
+
seen.add(key)
|
|
1090
|
+
unique.append(method)
|
|
1091
|
+
return unique or [None]
|
|
1092
|
+
|
|
1093
|
+
|
|
1094
|
+
def _adapter_discovery_input_modes(
|
|
1095
|
+
*,
|
|
1096
|
+
spec: FrameworkAdapterSpec | None,
|
|
1097
|
+
input_mode_candidates: Sequence[InputMode] | None,
|
|
1098
|
+
) -> list[InputMode]:
|
|
1099
|
+
modes: list[InputMode] = []
|
|
1100
|
+
if input_mode_candidates is not None:
|
|
1101
|
+
modes.extend(input_mode_candidates)
|
|
1102
|
+
elif spec is not None and spec.input_mode != "auto":
|
|
1103
|
+
modes.append(spec.input_mode)
|
|
1104
|
+
modes.extend(_DISCOVERY_INPUT_MODE_ORDER)
|
|
1105
|
+
|
|
1106
|
+
unique: list[InputMode] = []
|
|
1107
|
+
seen: set[str] = set()
|
|
1108
|
+
for mode in modes:
|
|
1109
|
+
normalized = str(mode or "auto")
|
|
1110
|
+
if normalized not in _DISCOVERY_INPUT_MODE_ORDER:
|
|
1111
|
+
continue
|
|
1112
|
+
if normalized in seen:
|
|
1113
|
+
continue
|
|
1114
|
+
seen.add(normalized)
|
|
1115
|
+
unique.append(normalized) # type: ignore[arg-type]
|
|
1116
|
+
return unique or ["auto"]
|
|
1117
|
+
|
|
1118
|
+
|
|
1119
|
+
def _adapter_discovery_modes_for_method(
|
|
1120
|
+
method_name: str | None,
|
|
1121
|
+
input_modes: Sequence[InputMode],
|
|
1122
|
+
*,
|
|
1123
|
+
spec: FrameworkAdapterSpec | None,
|
|
1124
|
+
) -> list[InputMode]:
|
|
1125
|
+
modes: list[InputMode] = []
|
|
1126
|
+
inferred = _adapter_inferred_input_mode(method_name)
|
|
1127
|
+
if inferred:
|
|
1128
|
+
modes.append(inferred)
|
|
1129
|
+
elif method_name is None and spec is not None:
|
|
1130
|
+
modes.append(spec.input_mode)
|
|
1131
|
+
modes.extend(input_modes)
|
|
1132
|
+
|
|
1133
|
+
unique: list[InputMode] = []
|
|
1134
|
+
seen: set[str] = set()
|
|
1135
|
+
for mode in modes:
|
|
1136
|
+
normalized = str(mode or "auto")
|
|
1137
|
+
if normalized not in _DISCOVERY_INPUT_MODE_ORDER:
|
|
1138
|
+
continue
|
|
1139
|
+
if normalized in seen:
|
|
1140
|
+
continue
|
|
1141
|
+
seen.add(normalized)
|
|
1142
|
+
unique.append(normalized) # type: ignore[arg-type]
|
|
1143
|
+
return unique or ["auto"]
|
|
1144
|
+
|
|
1145
|
+
|
|
1146
|
+
def _adapter_inferred_input_mode(method_name: str | None) -> InputMode | None:
|
|
1147
|
+
if method_name is None:
|
|
1148
|
+
return "agent_input"
|
|
1149
|
+
return (
|
|
1150
|
+
_DISCOVERY_METHOD_INPUT_MODES.get(method_name)
|
|
1151
|
+
or _DISCOVERY_METHOD_INPUT_MODES.get(_adapter_method_leaf(method_name))
|
|
1152
|
+
)
|
|
1153
|
+
|
|
1154
|
+
|
|
1155
|
+
def _adapter_candidate_input_key(
|
|
1156
|
+
agent: Any,
|
|
1157
|
+
method_name: str | None,
|
|
1158
|
+
input_mode: InputMode,
|
|
1159
|
+
) -> str | None:
|
|
1160
|
+
if agent is None or not method_name:
|
|
1161
|
+
return None
|
|
1162
|
+
method = _adapter_resolve_callable_attr_path(agent, method_name)
|
|
1163
|
+
if method is None:
|
|
1164
|
+
return None
|
|
1165
|
+
try:
|
|
1166
|
+
signature = inspect.signature(method)
|
|
1167
|
+
except (TypeError, ValueError):
|
|
1168
|
+
return None
|
|
1169
|
+
params = list(signature.parameters.values())
|
|
1170
|
+
names = {param.name: param for param in params}
|
|
1171
|
+
preferred_names = (
|
|
1172
|
+
(
|
|
1173
|
+
_METHOD_INPUT_KEY_PREFERENCES.get(str(method_name or ""))
|
|
1174
|
+
or _METHOD_INPUT_KEY_PREFERENCES.get(_adapter_method_leaf(method_name), ())
|
|
1175
|
+
)
|
|
1176
|
+
+ _KEYWORD_INPUT_NAMES
|
|
1177
|
+
)
|
|
1178
|
+
accepts_positional = any(
|
|
1179
|
+
param.kind
|
|
1180
|
+
in {
|
|
1181
|
+
inspect.Parameter.POSITIONAL_ONLY,
|
|
1182
|
+
inspect.Parameter.POSITIONAL_OR_KEYWORD,
|
|
1183
|
+
inspect.Parameter.VAR_POSITIONAL,
|
|
1184
|
+
}
|
|
1185
|
+
for param in params
|
|
1186
|
+
)
|
|
1187
|
+
for name in preferred_names:
|
|
1188
|
+
param = names.get(name)
|
|
1189
|
+
if param is None or param.kind == inspect.Parameter.POSITIONAL_ONLY:
|
|
1190
|
+
continue
|
|
1191
|
+
if name == "inputs" or not accepts_positional or param.kind == inspect.Parameter.KEYWORD_ONLY:
|
|
1192
|
+
return name
|
|
1193
|
+
if not accepts_positional:
|
|
1194
|
+
for param in params:
|
|
1195
|
+
if param.kind == inspect.Parameter.KEYWORD_ONLY:
|
|
1196
|
+
return param.name
|
|
1197
|
+
if any(param.kind == inspect.Parameter.VAR_KEYWORD for param in params):
|
|
1198
|
+
if input_mode == "dict":
|
|
1199
|
+
if method_name == "kickoff":
|
|
1200
|
+
return "inputs"
|
|
1201
|
+
if method_name in {"process", "process_frame"}:
|
|
1202
|
+
return "frame"
|
|
1203
|
+
return "payload"
|
|
1204
|
+
return "task" if method_name in {"run", "arun", "run_stream"} else "input"
|
|
1205
|
+
return None
|
|
1206
|
+
|
|
1207
|
+
|
|
1208
|
+
def _adapter_callable_signature(
|
|
1209
|
+
agent: Any,
|
|
1210
|
+
*,
|
|
1211
|
+
method: str | Callable[..., Any] | None,
|
|
1212
|
+
method_name: str | None,
|
|
1213
|
+
) -> dict[str, Any]:
|
|
1214
|
+
callable_method = method if callable(method) else None
|
|
1215
|
+
resolved_method = str(method_name or "").strip()
|
|
1216
|
+
if callable_method is None:
|
|
1217
|
+
if resolved_method == "auto":
|
|
1218
|
+
resolved_method = ""
|
|
1219
|
+
callable_method = _adapter_resolve_callable_attr_path(agent, resolved_method)
|
|
1220
|
+
if callable_method is None and callable(agent):
|
|
1221
|
+
callable_method = agent
|
|
1222
|
+
if callable_method is None:
|
|
1223
|
+
return {}
|
|
1224
|
+
|
|
1225
|
+
try:
|
|
1226
|
+
signature = inspect.signature(callable_method)
|
|
1227
|
+
except (TypeError, ValueError):
|
|
1228
|
+
return {}
|
|
1229
|
+
|
|
1230
|
+
params = list(signature.parameters.values())
|
|
1231
|
+
parameter_rows = [
|
|
1232
|
+
{
|
|
1233
|
+
"name": param.name,
|
|
1234
|
+
"kind": str(param.kind).rsplit(".", 1)[-1].lower(),
|
|
1235
|
+
"required": param.default is inspect.Parameter.empty
|
|
1236
|
+
and param.kind
|
|
1237
|
+
not in {
|
|
1238
|
+
inspect.Parameter.VAR_POSITIONAL,
|
|
1239
|
+
inspect.Parameter.VAR_KEYWORD,
|
|
1240
|
+
},
|
|
1241
|
+
"has_default": param.default is not inspect.Parameter.empty,
|
|
1242
|
+
"annotation": _adapter_annotation_name(param.annotation),
|
|
1243
|
+
}
|
|
1244
|
+
for param in params
|
|
1245
|
+
]
|
|
1246
|
+
selected_method = (
|
|
1247
|
+
str(method)
|
|
1248
|
+
if isinstance(method, str) and str(method)
|
|
1249
|
+
else resolved_method
|
|
1250
|
+
or getattr(callable_method, "__name__", None)
|
|
1251
|
+
or "callable"
|
|
1252
|
+
)
|
|
1253
|
+
method_leaf = _adapter_method_leaf(selected_method)
|
|
1254
|
+
preferred_input_key = _adapter_candidate_input_key(
|
|
1255
|
+
agent,
|
|
1256
|
+
selected_method if isinstance(method, str) else method_leaf,
|
|
1257
|
+
"auto",
|
|
1258
|
+
)
|
|
1259
|
+
return {
|
|
1260
|
+
"kind": "agent-learning.framework-adapter-callable-signature.v1",
|
|
1261
|
+
"inspectable": True,
|
|
1262
|
+
"method": selected_method,
|
|
1263
|
+
"method_leaf": method_leaf,
|
|
1264
|
+
"callable_type": _adapter_agent_type(callable_method),
|
|
1265
|
+
"parameters": parameter_rows,
|
|
1266
|
+
"parameter_names": [row["name"] for row in parameter_rows],
|
|
1267
|
+
"required_parameters": [
|
|
1268
|
+
row["name"] for row in parameter_rows if row["required"]
|
|
1269
|
+
],
|
|
1270
|
+
"required_parameter_count": sum(1 for row in parameter_rows if row["required"]),
|
|
1271
|
+
"accepts_positional": _params_accept_positional(params),
|
|
1272
|
+
"accepts_var_positional": any(
|
|
1273
|
+
param.kind == inspect.Parameter.VAR_POSITIONAL for param in params
|
|
1274
|
+
),
|
|
1275
|
+
"accepts_var_keyword": any(
|
|
1276
|
+
param.kind == inspect.Parameter.VAR_KEYWORD for param in params
|
|
1277
|
+
),
|
|
1278
|
+
"keyword_only_parameters": [
|
|
1279
|
+
param.name
|
|
1280
|
+
for param in params
|
|
1281
|
+
if param.kind == inspect.Parameter.KEYWORD_ONLY
|
|
1282
|
+
],
|
|
1283
|
+
"positional_parameters": [
|
|
1284
|
+
param.name
|
|
1285
|
+
for param in params
|
|
1286
|
+
if param.kind
|
|
1287
|
+
in {
|
|
1288
|
+
inspect.Parameter.POSITIONAL_ONLY,
|
|
1289
|
+
inspect.Parameter.POSITIONAL_OR_KEYWORD,
|
|
1290
|
+
}
|
|
1291
|
+
],
|
|
1292
|
+
"preferred_input_key": preferred_input_key,
|
|
1293
|
+
"is_async": inspect.iscoroutinefunction(callable_method),
|
|
1294
|
+
"is_generator": inspect.isgeneratorfunction(callable_method),
|
|
1295
|
+
"is_async_generator": inspect.isasyncgenfunction(callable_method),
|
|
1296
|
+
"return_annotation": _adapter_annotation_name(signature.return_annotation),
|
|
1297
|
+
}
|
|
1298
|
+
|
|
1299
|
+
|
|
1300
|
+
def _adapter_annotation_name(annotation: Any) -> str | None:
|
|
1301
|
+
if annotation is inspect.Signature.empty or annotation is inspect.Parameter.empty:
|
|
1302
|
+
return None
|
|
1303
|
+
if isinstance(annotation, type):
|
|
1304
|
+
return annotation.__name__
|
|
1305
|
+
return str(annotation)
|
|
1306
|
+
|
|
1307
|
+
|
|
1308
|
+
def _params_accept_positional(params: Sequence[inspect.Parameter]) -> bool:
|
|
1309
|
+
return any(
|
|
1310
|
+
param.kind
|
|
1311
|
+
in {
|
|
1312
|
+
inspect.Parameter.POSITIONAL_ONLY,
|
|
1313
|
+
inspect.Parameter.POSITIONAL_OR_KEYWORD,
|
|
1314
|
+
inspect.Parameter.VAR_POSITIONAL,
|
|
1315
|
+
}
|
|
1316
|
+
for param in params
|
|
1317
|
+
)
|
|
1318
|
+
|
|
1319
|
+
|
|
1320
|
+
def _adapter_discovery_score(
|
|
1321
|
+
method_name: str | None,
|
|
1322
|
+
input_mode: InputMode,
|
|
1323
|
+
*,
|
|
1324
|
+
spec: FrameworkAdapterSpec | None,
|
|
1325
|
+
inventory: Mapping[str, Any],
|
|
1326
|
+
explicit_methods: set[str],
|
|
1327
|
+
) -> dict[str, Any]:
|
|
1328
|
+
score = 0.15
|
|
1329
|
+
reasons: list[str] = ["local_contract_candidate"]
|
|
1330
|
+
method_present = _adapter_method_present(inventory, method_name)
|
|
1331
|
+
inferred_mode = _adapter_inferred_input_mode(method_name)
|
|
1332
|
+
|
|
1333
|
+
if method_name and method_name in explicit_methods:
|
|
1334
|
+
score += 0.15
|
|
1335
|
+
reasons.append("explicit_method_candidate")
|
|
1336
|
+
if method_present:
|
|
1337
|
+
score += 0.35
|
|
1338
|
+
reasons.append("agent_exposes_method")
|
|
1339
|
+
elif method_name is None and inventory.get("callable"):
|
|
1340
|
+
score += 0.35
|
|
1341
|
+
reasons.append("agent_is_direct_callable")
|
|
1342
|
+
elif inventory.get("provided") and method_name:
|
|
1343
|
+
score -= 0.15
|
|
1344
|
+
reasons.append("method_not_found_on_agent")
|
|
1345
|
+
|
|
1346
|
+
if spec and method_name and method_name == spec.method:
|
|
1347
|
+
score += 0.2
|
|
1348
|
+
reasons.append("matches_framework_preset_method")
|
|
1349
|
+
if spec and input_mode == spec.input_mode:
|
|
1350
|
+
score += 0.15
|
|
1351
|
+
reasons.append("matches_framework_preset_input_mode")
|
|
1352
|
+
if inferred_mode and input_mode == inferred_mode:
|
|
1353
|
+
score += 0.15
|
|
1354
|
+
reasons.append("matches_inferred_input_mode")
|
|
1355
|
+
if method_name == "execute_task" and input_mode == "dict":
|
|
1356
|
+
score += 0.1
|
|
1357
|
+
reasons.append("task_payload_adapter")
|
|
1358
|
+
if method_name == "process_frame" and input_mode == "dict":
|
|
1359
|
+
score += 0.1
|
|
1360
|
+
reasons.append("frame_payload_adapter")
|
|
1361
|
+
if method_name in _STREAMING_METHODS:
|
|
1362
|
+
score += 0.1
|
|
1363
|
+
reasons.append("streaming_adapter_surface")
|
|
1364
|
+
if input_mode == "auto":
|
|
1365
|
+
score -= 0.05
|
|
1366
|
+
reasons.append("auto_input_mode_requires_runtime_inference")
|
|
1367
|
+
|
|
1368
|
+
normalized = max(0.0, min(1.0, score))
|
|
1369
|
+
return {"score": round(normalized, 3), "reasons": reasons}
|
|
1370
|
+
|
|
1371
|
+
|
|
1372
|
+
def _adapter_method_present(
|
|
1373
|
+
inventory: Mapping[str, Any],
|
|
1374
|
+
method_name: str | None,
|
|
1375
|
+
) -> bool:
|
|
1376
|
+
if method_name is None:
|
|
1377
|
+
return bool(inventory.get("callable"))
|
|
1378
|
+
return method_name in set(inventory.get("exposed_methods", []) or [])
|
|
1379
|
+
|
|
1380
|
+
|
|
1381
|
+
def _adapter_method_rank(method_name: str) -> int:
|
|
1382
|
+
normalized = str(method_name or "")
|
|
1383
|
+
if normalized == "auto":
|
|
1384
|
+
normalized = ""
|
|
1385
|
+
try:
|
|
1386
|
+
return _DISCOVERY_METHOD_ORDER.index(normalized)
|
|
1387
|
+
except ValueError:
|
|
1388
|
+
return len(_DISCOVERY_METHOD_ORDER)
|
|
1389
|
+
|
|
1390
|
+
|
|
1391
|
+
def _adapter_input_mode_rank(input_mode: str) -> int:
|
|
1392
|
+
try:
|
|
1393
|
+
return _DISCOVERY_INPUT_MODE_ORDER.index(input_mode) # type: ignore[arg-type]
|
|
1394
|
+
except ValueError:
|
|
1395
|
+
return len(_DISCOVERY_INPUT_MODE_ORDER)
|
|
1396
|
+
|
|
1397
|
+
|
|
1398
|
+
def _adapter_discovery_findings(
|
|
1399
|
+
inventory: Mapping[str, Any],
|
|
1400
|
+
candidates: Sequence[Mapping[str, Any]],
|
|
1401
|
+
) -> list[dict[str, Any]]:
|
|
1402
|
+
findings: list[dict[str, Any]] = []
|
|
1403
|
+
if not inventory.get("provided"):
|
|
1404
|
+
findings.append(
|
|
1405
|
+
{
|
|
1406
|
+
"level": "info",
|
|
1407
|
+
"type": "agent_not_provided",
|
|
1408
|
+
"message": "Discovery used framework presets without inspecting an agent.",
|
|
1409
|
+
}
|
|
1410
|
+
)
|
|
1411
|
+
elif not inventory.get("callable") and not inventory.get("exposed_methods"):
|
|
1412
|
+
findings.append(
|
|
1413
|
+
{
|
|
1414
|
+
"level": "warning",
|
|
1415
|
+
"type": "no_callable_adapter_surface",
|
|
1416
|
+
"message": "Agent does not expose a discovered callable adapter method.",
|
|
1417
|
+
}
|
|
1418
|
+
)
|
|
1419
|
+
if not candidates:
|
|
1420
|
+
findings.append(
|
|
1421
|
+
{
|
|
1422
|
+
"level": "error",
|
|
1423
|
+
"type": "adapter_candidates_missing",
|
|
1424
|
+
"message": "No framework adapter candidates were discovered.",
|
|
1425
|
+
}
|
|
1426
|
+
)
|
|
1427
|
+
return findings
|
|
1428
|
+
|
|
1429
|
+
|
|
1430
|
+
def _framework_matrix_summary(contracts: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
|
|
1431
|
+
frameworks: list[str] = []
|
|
1432
|
+
methods: dict[str, str] = {}
|
|
1433
|
+
input_modes: dict[str, str] = {}
|
|
1434
|
+
modalities: dict[str, str] = {}
|
|
1435
|
+
transports: dict[str, str] = {}
|
|
1436
|
+
capabilities: set[str] = set()
|
|
1437
|
+
evidence_requirements: set[str] = set()
|
|
1438
|
+
target_schemes: set[str] = set()
|
|
1439
|
+
|
|
1440
|
+
for contract in contracts:
|
|
1441
|
+
framework = _framework_key(str(contract.get("framework") or "custom"))
|
|
1442
|
+
frameworks.append(framework)
|
|
1443
|
+
methods[framework] = str(contract.get("method") or "")
|
|
1444
|
+
input_modes[framework] = str(contract.get("input_mode") or "")
|
|
1445
|
+
modalities[framework] = str(contract.get("modality") or "")
|
|
1446
|
+
transports[framework] = str(contract.get("transport") or "")
|
|
1447
|
+
capabilities.update(
|
|
1448
|
+
str(item) for item in contract.get("capabilities", []) or []
|
|
1449
|
+
)
|
|
1450
|
+
evidence_requirements.update(
|
|
1451
|
+
str(item) for item in contract.get("evidence_requirements", []) or []
|
|
1452
|
+
)
|
|
1453
|
+
target_scheme = str(contract.get("target_scheme") or "")
|
|
1454
|
+
if target_scheme:
|
|
1455
|
+
target_schemes.add(target_scheme)
|
|
1456
|
+
|
|
1457
|
+
return {
|
|
1458
|
+
"frameworks": frameworks,
|
|
1459
|
+
"modalities": sorted(set(modalities.values())),
|
|
1460
|
+
"transports": sorted(set(transports.values())),
|
|
1461
|
+
"methods": methods,
|
|
1462
|
+
"input_modes": input_modes,
|
|
1463
|
+
"framework_modalities": modalities,
|
|
1464
|
+
"framework_transports": transports,
|
|
1465
|
+
"capabilities": sorted(capabilities),
|
|
1466
|
+
"evidence_requirements": sorted(evidence_requirements),
|
|
1467
|
+
"target_schemes": sorted(target_schemes),
|
|
1468
|
+
"contract_count": len(contracts),
|
|
1469
|
+
"local_executable_fixture_count": sum(
|
|
1470
|
+
1 for contract in contracts if bool(contract.get("local_executable_fixture"))
|
|
1471
|
+
),
|
|
1472
|
+
"trace_runtime_count": sum(
|
|
1473
|
+
1 for contract in contracts if bool(contract.get("trace_runtime"))
|
|
1474
|
+
),
|
|
1475
|
+
"requires_external_service_count": sum(
|
|
1476
|
+
1 for contract in contracts if bool(contract.get("requires_external_service"))
|
|
1477
|
+
),
|
|
1478
|
+
"external_target_count": sum(
|
|
1479
|
+
1
|
|
1480
|
+
for contract in contracts
|
|
1481
|
+
if _is_external_target(str(contract.get("target") or ""))
|
|
1482
|
+
),
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1485
|
+
|
|
1486
|
+
def _framework_matrix_findings(
|
|
1487
|
+
contracts: Sequence[Mapping[str, Any]],
|
|
1488
|
+
) -> list[dict[str, Any]]:
|
|
1489
|
+
findings: list[dict[str, Any]] = []
|
|
1490
|
+
for contract in contracts:
|
|
1491
|
+
framework = _framework_key(str(contract.get("framework") or "custom"))
|
|
1492
|
+
if contract.get("kind") != "agent-learning.framework-adapter-contract.v1":
|
|
1493
|
+
findings.append({"framework": framework, "type": "contract_kind_mismatch"})
|
|
1494
|
+
if bool(contract.get("requires_external_service")):
|
|
1495
|
+
findings.append(
|
|
1496
|
+
{"framework": framework, "type": "external_service_required"}
|
|
1497
|
+
)
|
|
1498
|
+
if not bool(contract.get("local_executable_fixture")):
|
|
1499
|
+
findings.append({"framework": framework, "type": "local_fixture_missing"})
|
|
1500
|
+
if _is_external_target(str(contract.get("target") or "")):
|
|
1501
|
+
findings.append({"framework": framework, "type": "external_target_scheme"})
|
|
1502
|
+
return findings
|
|
1503
|
+
|
|
1504
|
+
|
|
1505
|
+
def _framework_adapter_capability_profile_from_contract(
|
|
1506
|
+
contract: Mapping[str, Any],
|
|
1507
|
+
) -> dict[str, Any]:
|
|
1508
|
+
payload = copy.deepcopy(dict(contract))
|
|
1509
|
+
framework = _framework_key(str(payload.get("framework") or "custom"))
|
|
1510
|
+
capabilities = _framework_profile_capabilities(payload)
|
|
1511
|
+
task_surfaces = _framework_profile_task_surfaces(payload)
|
|
1512
|
+
bindings = _framework_profile_bindings(payload)
|
|
1513
|
+
findings = _framework_profile_findings(payload, bindings)
|
|
1514
|
+
summary = _framework_profile_summary(
|
|
1515
|
+
payload,
|
|
1516
|
+
capabilities=capabilities,
|
|
1517
|
+
task_surfaces=task_surfaces,
|
|
1518
|
+
bindings=bindings,
|
|
1519
|
+
)
|
|
1520
|
+
status = "passed" if not findings else "failed"
|
|
1521
|
+
return {
|
|
1522
|
+
"kind": "agent-learning.framework-adapter-capability-profile.v1",
|
|
1523
|
+
"status": status,
|
|
1524
|
+
"passed": status == "passed",
|
|
1525
|
+
"framework": framework,
|
|
1526
|
+
"method": str(payload.get("method") or "auto"),
|
|
1527
|
+
"input_mode": str(payload.get("input_mode") or "auto"),
|
|
1528
|
+
"modality": str(payload.get("modality") or "text"),
|
|
1529
|
+
"transport": str(payload.get("transport") or "in_process"),
|
|
1530
|
+
"requires_external_service": bool(payload.get("requires_external_service")),
|
|
1531
|
+
"local_executable_fixture": bool(payload.get("local_executable_fixture")),
|
|
1532
|
+
"trace_runtime": bool(payload.get("trace_runtime")),
|
|
1533
|
+
"contract": payload,
|
|
1534
|
+
"capabilities": capabilities,
|
|
1535
|
+
"task_surfaces": task_surfaces,
|
|
1536
|
+
"bindings": bindings,
|
|
1537
|
+
"summary": summary,
|
|
1538
|
+
"findings": findings,
|
|
1539
|
+
"evidence_requirements": sorted(
|
|
1540
|
+
{
|
|
1541
|
+
"framework_adapter_profile",
|
|
1542
|
+
"framework_adapter_contract",
|
|
1543
|
+
*[
|
|
1544
|
+
str(item)
|
|
1545
|
+
for item in payload.get("evidence_requirements", []) or []
|
|
1546
|
+
if str(item)
|
|
1547
|
+
],
|
|
1548
|
+
}
|
|
1549
|
+
),
|
|
1550
|
+
}
|
|
1551
|
+
|
|
1552
|
+
|
|
1553
|
+
def _framework_profile_capabilities(
|
|
1554
|
+
contract: Mapping[str, Any],
|
|
1555
|
+
) -> list[dict[str, Any]]:
|
|
1556
|
+
names = [
|
|
1557
|
+
*[str(item) for item in contract.get("capabilities", []) or []],
|
|
1558
|
+
"adapter_contract",
|
|
1559
|
+
"local_fixture",
|
|
1560
|
+
"metric_evidence",
|
|
1561
|
+
"optimization_search",
|
|
1562
|
+
]
|
|
1563
|
+
if bool(contract.get("trace_runtime")):
|
|
1564
|
+
names.append("trace_runtime")
|
|
1565
|
+
if str(contract.get("modality") or "") == "voice":
|
|
1566
|
+
names.extend(["voice", "realtime"])
|
|
1567
|
+
if str(contract.get("modality") or "") == "cua":
|
|
1568
|
+
names.extend(["browser", "computer_use"])
|
|
1569
|
+
|
|
1570
|
+
capabilities: list[dict[str, Any]] = []
|
|
1571
|
+
seen: set[str] = set()
|
|
1572
|
+
for raw_name in names:
|
|
1573
|
+
name = _framework_key(raw_name)
|
|
1574
|
+
if not name or name in seen:
|
|
1575
|
+
continue
|
|
1576
|
+
seen.add(name)
|
|
1577
|
+
capabilities.append(
|
|
1578
|
+
{
|
|
1579
|
+
"name": name,
|
|
1580
|
+
"category": _framework_profile_capability_category(name),
|
|
1581
|
+
"status": "supported",
|
|
1582
|
+
"source": "framework_adapter_contract",
|
|
1583
|
+
}
|
|
1584
|
+
)
|
|
1585
|
+
return capabilities
|
|
1586
|
+
|
|
1587
|
+
|
|
1588
|
+
def _framework_profile_capability_category(name: str) -> str:
|
|
1589
|
+
normalized = _framework_key(name)
|
|
1590
|
+
if normalized in {"tool_calls", "tools", "call_tool"}:
|
|
1591
|
+
return "tools"
|
|
1592
|
+
if normalized in {"runtime_trace", "trace_runtime", "metric_evidence"}:
|
|
1593
|
+
return "observability"
|
|
1594
|
+
if normalized in {"structured_input", "messages", "artifacts", "state"}:
|
|
1595
|
+
return "io"
|
|
1596
|
+
if normalized in {"voice", "realtime", "streaming"}:
|
|
1597
|
+
return "realtime"
|
|
1598
|
+
if normalized in {"browser", "computer_use"}:
|
|
1599
|
+
return "computer_use"
|
|
1600
|
+
if normalized in {"environment_replay", "reset_step_trace"}:
|
|
1601
|
+
return "world"
|
|
1602
|
+
if normalized in {"optimization_search"}:
|
|
1603
|
+
return "optimization"
|
|
1604
|
+
if normalized in {"adapter_contract", "local_fixture"}:
|
|
1605
|
+
return "adapter"
|
|
1606
|
+
return "framework"
|
|
1607
|
+
|
|
1608
|
+
|
|
1609
|
+
def _framework_profile_task_surfaces(
|
|
1610
|
+
contract: Mapping[str, Any],
|
|
1611
|
+
) -> list[dict[str, Any]]:
|
|
1612
|
+
framework = _framework_key(str(contract.get("framework") or "custom"))
|
|
1613
|
+
method = str(contract.get("method") or "auto")
|
|
1614
|
+
input_mode = str(contract.get("input_mode") or "auto")
|
|
1615
|
+
return [
|
|
1616
|
+
{
|
|
1617
|
+
"name": "framework_adapter_simulation",
|
|
1618
|
+
"library": "simulate-sdk",
|
|
1619
|
+
"framework": framework,
|
|
1620
|
+
"method": method,
|
|
1621
|
+
"input_mode": input_mode,
|
|
1622
|
+
"evidence": [
|
|
1623
|
+
"framework_adapter_contract",
|
|
1624
|
+
"framework_runtime",
|
|
1625
|
+
"framework_trace",
|
|
1626
|
+
],
|
|
1627
|
+
},
|
|
1628
|
+
{
|
|
1629
|
+
"name": "framework_adapter_evaluation",
|
|
1630
|
+
"library": "ai-evaluation",
|
|
1631
|
+
"metric": "framework_adapter_contract_quality",
|
|
1632
|
+
"evidence": [
|
|
1633
|
+
"adapter_conformance",
|
|
1634
|
+
"metric_evidence",
|
|
1635
|
+
"tool_calls",
|
|
1636
|
+
],
|
|
1637
|
+
},
|
|
1638
|
+
{
|
|
1639
|
+
"name": "framework_adapter_optimization",
|
|
1640
|
+
"library": "agent-opt",
|
|
1641
|
+
"layers": ["framework", "integration", "harness", "evaluator"],
|
|
1642
|
+
"search_paths": [
|
|
1643
|
+
"agent.method",
|
|
1644
|
+
"agent.input_mode",
|
|
1645
|
+
"simulation.environments",
|
|
1646
|
+
],
|
|
1647
|
+
},
|
|
1648
|
+
]
|
|
1649
|
+
|
|
1650
|
+
|
|
1651
|
+
def _framework_profile_bindings(
|
|
1652
|
+
contract: Mapping[str, Any],
|
|
1653
|
+
) -> dict[str, Any]:
|
|
1654
|
+
method = str(contract.get("method") or "auto")
|
|
1655
|
+
input_mode = str(contract.get("input_mode") or "auto")
|
|
1656
|
+
framework = _framework_key(str(contract.get("framework") or "custom"))
|
|
1657
|
+
return {
|
|
1658
|
+
"simulate-sdk": {
|
|
1659
|
+
"adapter": "wrap_framework",
|
|
1660
|
+
"contract": "framework_adapter_contract",
|
|
1661
|
+
"matrix": "framework_adapter_contract_matrix",
|
|
1662
|
+
"probe": "probe_framework_adapter",
|
|
1663
|
+
"framework": framework,
|
|
1664
|
+
"method": method,
|
|
1665
|
+
"input_mode": input_mode,
|
|
1666
|
+
"local_executable_fixture": bool(
|
|
1667
|
+
contract.get("local_executable_fixture")
|
|
1668
|
+
),
|
|
1669
|
+
},
|
|
1670
|
+
"ai-evaluation": {
|
|
1671
|
+
"metric": "framework_adapter_contract_quality",
|
|
1672
|
+
"contract_kind": "agent-learning.framework-adapter-contract.v1",
|
|
1673
|
+
"required_frameworks": [framework],
|
|
1674
|
+
"required_methods": [method] if method != "auto" else [],
|
|
1675
|
+
"required_input_modes": [input_mode] if input_mode != "auto" else [],
|
|
1676
|
+
"required_capabilities": [
|
|
1677
|
+
str(item)
|
|
1678
|
+
for item in contract.get("capabilities", []) or []
|
|
1679
|
+
if str(item)
|
|
1680
|
+
],
|
|
1681
|
+
},
|
|
1682
|
+
"agent-opt": {
|
|
1683
|
+
"target": "OptimizationTarget",
|
|
1684
|
+
"candidate": "AgentCandidate",
|
|
1685
|
+
"optimizer": "AgentOptimizer",
|
|
1686
|
+
"layers": ["framework", "integration", "harness", "evaluator"],
|
|
1687
|
+
"search_paths": [
|
|
1688
|
+
"agent.method",
|
|
1689
|
+
"agent.input_mode",
|
|
1690
|
+
"simulation.environments",
|
|
1691
|
+
],
|
|
1692
|
+
},
|
|
1693
|
+
}
|
|
1694
|
+
|
|
1695
|
+
|
|
1696
|
+
def _framework_profile_findings(
|
|
1697
|
+
contract: Mapping[str, Any],
|
|
1698
|
+
bindings: Mapping[str, Any],
|
|
1699
|
+
) -> list[dict[str, Any]]:
|
|
1700
|
+
findings: list[dict[str, Any]] = []
|
|
1701
|
+
framework = _framework_key(str(contract.get("framework") or "custom"))
|
|
1702
|
+
if contract.get("kind") != "agent-learning.framework-adapter-contract.v1":
|
|
1703
|
+
findings.append(
|
|
1704
|
+
{
|
|
1705
|
+
"level": "error",
|
|
1706
|
+
"framework": framework,
|
|
1707
|
+
"type": "contract_kind_mismatch",
|
|
1708
|
+
}
|
|
1709
|
+
)
|
|
1710
|
+
if bool(contract.get("requires_external_service")):
|
|
1711
|
+
findings.append(
|
|
1712
|
+
{
|
|
1713
|
+
"level": "error",
|
|
1714
|
+
"framework": framework,
|
|
1715
|
+
"type": "external_service_required",
|
|
1716
|
+
}
|
|
1717
|
+
)
|
|
1718
|
+
if not bool(contract.get("local_executable_fixture")):
|
|
1719
|
+
findings.append(
|
|
1720
|
+
{
|
|
1721
|
+
"level": "error",
|
|
1722
|
+
"framework": framework,
|
|
1723
|
+
"type": "local_fixture_missing",
|
|
1724
|
+
}
|
|
1725
|
+
)
|
|
1726
|
+
if _is_external_target(str(contract.get("target") or "")):
|
|
1727
|
+
findings.append(
|
|
1728
|
+
{
|
|
1729
|
+
"level": "error",
|
|
1730
|
+
"framework": framework,
|
|
1731
|
+
"type": "external_target_scheme",
|
|
1732
|
+
}
|
|
1733
|
+
)
|
|
1734
|
+
for library in ("simulate-sdk", "ai-evaluation", "agent-opt"):
|
|
1735
|
+
if not isinstance(bindings.get(library), Mapping):
|
|
1736
|
+
findings.append(
|
|
1737
|
+
{
|
|
1738
|
+
"level": "error",
|
|
1739
|
+
"framework": framework,
|
|
1740
|
+
"type": "trinity_binding_missing",
|
|
1741
|
+
"library": library,
|
|
1742
|
+
}
|
|
1743
|
+
)
|
|
1744
|
+
return findings
|
|
1745
|
+
|
|
1746
|
+
|
|
1747
|
+
def _framework_profile_summary(
|
|
1748
|
+
contract: Mapping[str, Any],
|
|
1749
|
+
*,
|
|
1750
|
+
capabilities: Sequence[Mapping[str, Any]],
|
|
1751
|
+
task_surfaces: Sequence[Mapping[str, Any]],
|
|
1752
|
+
bindings: Mapping[str, Any],
|
|
1753
|
+
) -> dict[str, Any]:
|
|
1754
|
+
return {
|
|
1755
|
+
"framework": _framework_key(str(contract.get("framework") or "custom")),
|
|
1756
|
+
"method": str(contract.get("method") or "auto"),
|
|
1757
|
+
"input_mode": str(contract.get("input_mode") or "auto"),
|
|
1758
|
+
"modality": str(contract.get("modality") or "text"),
|
|
1759
|
+
"transport": str(contract.get("transport") or "in_process"),
|
|
1760
|
+
"capability_count": len(capabilities),
|
|
1761
|
+
"task_surface_count": len(task_surfaces),
|
|
1762
|
+
"binding_count": len(bindings),
|
|
1763
|
+
"libraries": sorted(str(key) for key in bindings),
|
|
1764
|
+
"local_executable_fixture": bool(contract.get("local_executable_fixture")),
|
|
1765
|
+
"trace_runtime": bool(contract.get("trace_runtime")),
|
|
1766
|
+
"requires_external_service": bool(contract.get("requires_external_service")),
|
|
1767
|
+
}
|
|
1768
|
+
|
|
1769
|
+
|
|
1770
|
+
def _framework_profile_collection_summary(
|
|
1771
|
+
profiles: Sequence[Mapping[str, Any]],
|
|
1772
|
+
) -> dict[str, Any]:
|
|
1773
|
+
frameworks = [
|
|
1774
|
+
_framework_key(str(profile.get("framework") or "custom"))
|
|
1775
|
+
for profile in profiles
|
|
1776
|
+
]
|
|
1777
|
+
libraries = sorted(
|
|
1778
|
+
{
|
|
1779
|
+
str(library)
|
|
1780
|
+
for profile in profiles
|
|
1781
|
+
for library in (profile.get("bindings") or {})
|
|
1782
|
+
}
|
|
1783
|
+
)
|
|
1784
|
+
capabilities = sorted(
|
|
1785
|
+
{
|
|
1786
|
+
_framework_key(str(capability.get("name") or ""))
|
|
1787
|
+
for profile in profiles
|
|
1788
|
+
for capability in profile.get("capabilities", []) or []
|
|
1789
|
+
if isinstance(capability, Mapping)
|
|
1790
|
+
}
|
|
1791
|
+
)
|
|
1792
|
+
return {
|
|
1793
|
+
"frameworks": frameworks,
|
|
1794
|
+
"profile_count": len(profiles),
|
|
1795
|
+
"passed_profile_count": sum(
|
|
1796
|
+
1 for profile in profiles if str(profile.get("status")) == "passed"
|
|
1797
|
+
),
|
|
1798
|
+
"failed_profile_count": sum(
|
|
1799
|
+
1 for profile in profiles if str(profile.get("status")) != "passed"
|
|
1800
|
+
),
|
|
1801
|
+
"libraries": libraries,
|
|
1802
|
+
"capabilities": capabilities,
|
|
1803
|
+
"local_executable_fixture_count": sum(
|
|
1804
|
+
1 for profile in profiles if bool(profile.get("local_executable_fixture"))
|
|
1805
|
+
),
|
|
1806
|
+
"requires_external_service_count": sum(
|
|
1807
|
+
1 for profile in profiles if bool(profile.get("requires_external_service"))
|
|
1808
|
+
),
|
|
1809
|
+
}
|
|
1810
|
+
|
|
1811
|
+
|
|
1812
|
+
def _framework_profile_collection_findings(
|
|
1813
|
+
profiles: Sequence[Mapping[str, Any]],
|
|
1814
|
+
*,
|
|
1815
|
+
required_frameworks: Sequence[str],
|
|
1816
|
+
) -> list[dict[str, Any]]:
|
|
1817
|
+
findings: list[dict[str, Any]] = []
|
|
1818
|
+
observed = {
|
|
1819
|
+
_framework_key(str(profile.get("framework") or "custom"))
|
|
1820
|
+
for profile in profiles
|
|
1821
|
+
}
|
|
1822
|
+
for framework in required_frameworks:
|
|
1823
|
+
key = _framework_key(framework)
|
|
1824
|
+
if key not in observed:
|
|
1825
|
+
findings.append(
|
|
1826
|
+
{
|
|
1827
|
+
"level": "error",
|
|
1828
|
+
"framework": key,
|
|
1829
|
+
"type": "framework_profile_missing",
|
|
1830
|
+
}
|
|
1831
|
+
)
|
|
1832
|
+
for profile in profiles:
|
|
1833
|
+
for finding in profile.get("findings", []) or []:
|
|
1834
|
+
if isinstance(finding, Mapping):
|
|
1835
|
+
findings.append(copy.deepcopy(dict(finding)))
|
|
1836
|
+
return findings
|
|
1837
|
+
|
|
1838
|
+
|
|
1839
|
+
def _default_transport(modality: str) -> str:
|
|
1840
|
+
if modality == "voice":
|
|
1841
|
+
return "realtime_adapter"
|
|
1842
|
+
if modality == "cua":
|
|
1843
|
+
return "browser_adapter"
|
|
1844
|
+
if modality == "image":
|
|
1845
|
+
return "multimodal_adapter"
|
|
1846
|
+
return "in_process"
|
|
1847
|
+
|
|
1848
|
+
|
|
1849
|
+
def _default_lifecycle_hooks(modality: str) -> tuple[str, ...]:
|
|
1850
|
+
if modality == "voice":
|
|
1851
|
+
return ("setup", "connect", "stream", "respond", "teardown")
|
|
1852
|
+
if modality == "cua":
|
|
1853
|
+
return ("setup", "observe", "act", "verify", "teardown")
|
|
1854
|
+
return ("setup", "invoke", "observe", "teardown")
|
|
1855
|
+
|
|
1856
|
+
|
|
1857
|
+
def _default_capabilities(modality: str, input_mode: str) -> tuple[str, ...]:
|
|
1858
|
+
capabilities = [
|
|
1859
|
+
"messages",
|
|
1860
|
+
"tool_calls",
|
|
1861
|
+
"runtime_trace",
|
|
1862
|
+
"state",
|
|
1863
|
+
"artifacts",
|
|
1864
|
+
]
|
|
1865
|
+
if input_mode == "dict":
|
|
1866
|
+
capabilities.append("structured_input")
|
|
1867
|
+
if input_mode in {"dict", "messages"}:
|
|
1868
|
+
capabilities.append("streaming_trace")
|
|
1869
|
+
if modality == "voice":
|
|
1870
|
+
capabilities.extend(["voice_frames", "realtime_events"])
|
|
1871
|
+
elif modality == "cua":
|
|
1872
|
+
capabilities.extend(["browser_actions", "visual_grounding"])
|
|
1873
|
+
elif modality == "image":
|
|
1874
|
+
capabilities.extend(["image_context", "multimodal_grounding"])
|
|
1875
|
+
return tuple(capabilities)
|
|
1876
|
+
|
|
1877
|
+
|
|
1878
|
+
def _input_schema(input_mode: str) -> dict[str, Any]:
|
|
1879
|
+
if input_mode == "dict":
|
|
1880
|
+
return {
|
|
1881
|
+
"type": "object",
|
|
1882
|
+
"required": ["messages", "scenario"],
|
|
1883
|
+
"additionalProperties": True,
|
|
1884
|
+
}
|
|
1885
|
+
if input_mode == "messages":
|
|
1886
|
+
return {
|
|
1887
|
+
"type": "array",
|
|
1888
|
+
"items": {"type": "object", "required": ["role", "content"]},
|
|
1889
|
+
}
|
|
1890
|
+
if input_mode == "agent_input":
|
|
1891
|
+
return {"type": "object", "class": "AgentInput"}
|
|
1892
|
+
if input_mode == "text":
|
|
1893
|
+
return {"type": "string"}
|
|
1894
|
+
return {"type": "any"}
|
|
1895
|
+
|
|
1896
|
+
|
|
1897
|
+
def _output_schema() -> dict[str, Any]:
|
|
1898
|
+
return {
|
|
1899
|
+
"oneOf": [
|
|
1900
|
+
{"type": "string"},
|
|
1901
|
+
{"type": "object", "class": "AgentResponse"},
|
|
1902
|
+
],
|
|
1903
|
+
"required_trace_state": ["framework_runtime"],
|
|
1904
|
+
}
|
|
1905
|
+
|
|
1906
|
+
|
|
1907
|
+
def _probe_method_name(method: str | Callable[..., Any] | None) -> str | None:
|
|
1908
|
+
if method is None:
|
|
1909
|
+
return None
|
|
1910
|
+
if isinstance(method, str):
|
|
1911
|
+
return method
|
|
1912
|
+
return getattr(method, "__name__", None) or "callable"
|
|
1913
|
+
|
|
1914
|
+
|
|
1915
|
+
def _probe_cases(cases: Sequence[Mapping[str, Any]] | None) -> list[dict[str, Any]]:
|
|
1916
|
+
if not cases:
|
|
1917
|
+
return [
|
|
1918
|
+
{
|
|
1919
|
+
"id": "default",
|
|
1920
|
+
"input": "Return a short adapter probe result.",
|
|
1921
|
+
}
|
|
1922
|
+
]
|
|
1923
|
+
return [dict(case) for case in cases]
|
|
1924
|
+
|
|
1925
|
+
|
|
1926
|
+
async def _run_probe_case(
|
|
1927
|
+
wrapper: AgentWrapper,
|
|
1928
|
+
framework: str,
|
|
1929
|
+
case: Mapping[str, Any],
|
|
1930
|
+
*,
|
|
1931
|
+
index: int,
|
|
1932
|
+
trace_runtime: bool,
|
|
1933
|
+
contract: Mapping[str, Any],
|
|
1934
|
+
) -> dict[str, Any]:
|
|
1935
|
+
case_id = str(case.get("id") or f"case_{index}")
|
|
1936
|
+
agent_input = _probe_agent_input(framework, case, index=index)
|
|
1937
|
+
raw_response = await wrapper.call(agent_input)
|
|
1938
|
+
response = (
|
|
1939
|
+
raw_response
|
|
1940
|
+
if isinstance(raw_response, AgentResponse)
|
|
1941
|
+
else AgentResponse(content=str(raw_response))
|
|
1942
|
+
)
|
|
1943
|
+
response_payload = _probe_response_payload(response)
|
|
1944
|
+
runtime_trace = dict((response.state or {}).get("framework_runtime") or {})
|
|
1945
|
+
observed_io_contract = _probe_observed_io_contract(
|
|
1946
|
+
framework,
|
|
1947
|
+
case_id=case_id,
|
|
1948
|
+
runtime_trace=runtime_trace,
|
|
1949
|
+
response_payload=response_payload,
|
|
1950
|
+
contract=contract,
|
|
1951
|
+
)
|
|
1952
|
+
checks = _probe_case_checks(
|
|
1953
|
+
case,
|
|
1954
|
+
response_payload,
|
|
1955
|
+
runtime_trace=runtime_trace,
|
|
1956
|
+
observed_io_contract=observed_io_contract,
|
|
1957
|
+
trace_runtime=trace_runtime,
|
|
1958
|
+
contract=contract,
|
|
1959
|
+
)
|
|
1960
|
+
findings = [
|
|
1961
|
+
{
|
|
1962
|
+
"case_id": case_id,
|
|
1963
|
+
"check": check["id"],
|
|
1964
|
+
"level": "error",
|
|
1965
|
+
"message": check["message"],
|
|
1966
|
+
"expected": check.get("expected"),
|
|
1967
|
+
"observed": check.get("observed"),
|
|
1968
|
+
}
|
|
1969
|
+
for check in checks
|
|
1970
|
+
if not check["passed"]
|
|
1971
|
+
]
|
|
1972
|
+
return {
|
|
1973
|
+
"id": case_id,
|
|
1974
|
+
"status": "passed" if not findings else "failed",
|
|
1975
|
+
"passed": not findings,
|
|
1976
|
+
"input": {
|
|
1977
|
+
"message_count": len(agent_input.messages),
|
|
1978
|
+
"tool_count": len(agent_input.tools),
|
|
1979
|
+
"artifact_count": len(agent_input.artifacts),
|
|
1980
|
+
"event_count": len(agent_input.events),
|
|
1981
|
+
"modality": agent_input.modality,
|
|
1982
|
+
},
|
|
1983
|
+
"response": response_payload,
|
|
1984
|
+
"runtime_trace": runtime_trace,
|
|
1985
|
+
"observed_io_contract": observed_io_contract,
|
|
1986
|
+
"checks": checks,
|
|
1987
|
+
"findings": findings,
|
|
1988
|
+
}
|
|
1989
|
+
|
|
1990
|
+
|
|
1991
|
+
def _probe_agent_input(
|
|
1992
|
+
framework: str,
|
|
1993
|
+
case: Mapping[str, Any],
|
|
1994
|
+
*,
|
|
1995
|
+
index: int,
|
|
1996
|
+
) -> AgentInput:
|
|
1997
|
+
messages = _probe_messages(case)
|
|
1998
|
+
new_message = dict(case.get("new_message") or messages[-1])
|
|
1999
|
+
metadata = {
|
|
2000
|
+
"framework": framework,
|
|
2001
|
+
"probe_case_id": str(case.get("id") or f"case_{index}"),
|
|
2002
|
+
**dict(case.get("metadata") or {}),
|
|
2003
|
+
}
|
|
2004
|
+
return AgentInput(
|
|
2005
|
+
thread_id=str(case.get("thread_id") or f"{framework}-probe-{index}"),
|
|
2006
|
+
messages=messages,
|
|
2007
|
+
new_message=new_message,
|
|
2008
|
+
execution_id=str(case.get("execution_id") or f"{framework}-probe"),
|
|
2009
|
+
turn_index=int(case.get("turn_index") or index - 1),
|
|
2010
|
+
scenario_name=str(case.get("scenario_name") or "framework-adapter-probe"),
|
|
2011
|
+
persona=dict(case.get("persona") or {}),
|
|
2012
|
+
situation=str(case.get("situation") or ""),
|
|
2013
|
+
expected_outcome=str(case.get("expected_outcome") or ""),
|
|
2014
|
+
modality=str(case.get("modality") or ""),
|
|
2015
|
+
artifacts=list(case.get("artifacts") or []),
|
|
2016
|
+
events=list(case.get("events") or []),
|
|
2017
|
+
memory=dict(case.get("memory") or {}),
|
|
2018
|
+
tools=[dict(tool) for tool in case.get("tools", []) if isinstance(tool, Mapping)],
|
|
2019
|
+
metadata=metadata,
|
|
2020
|
+
)
|
|
2021
|
+
|
|
2022
|
+
|
|
2023
|
+
def _probe_messages(case: Mapping[str, Any]) -> list[dict[str, Any]]:
|
|
2024
|
+
raw_messages = case.get("messages")
|
|
2025
|
+
if isinstance(raw_messages, Sequence) and not isinstance(raw_messages, (str, bytes)):
|
|
2026
|
+
messages = [
|
|
2027
|
+
dict(message)
|
|
2028
|
+
for message in raw_messages
|
|
2029
|
+
if isinstance(message, Mapping)
|
|
2030
|
+
]
|
|
2031
|
+
if messages:
|
|
2032
|
+
return messages
|
|
2033
|
+
message = case.get("input", case.get("message", "Run the adapter probe."))
|
|
2034
|
+
return [{"role": "user", "content": str(message)}]
|
|
2035
|
+
|
|
2036
|
+
|
|
2037
|
+
def _probe_case_checks(
|
|
2038
|
+
case: Mapping[str, Any],
|
|
2039
|
+
response: Mapping[str, Any],
|
|
2040
|
+
*,
|
|
2041
|
+
runtime_trace: Mapping[str, Any],
|
|
2042
|
+
observed_io_contract: Mapping[str, Any],
|
|
2043
|
+
trace_runtime: bool,
|
|
2044
|
+
contract: Mapping[str, Any],
|
|
2045
|
+
) -> list[dict[str, Any]]:
|
|
2046
|
+
checks: list[dict[str, Any]] = []
|
|
2047
|
+
content = str(response.get("content") or "")
|
|
2048
|
+
tool_names = set(response.get("tool_names") or [])
|
|
2049
|
+
event_types = set(response.get("event_types") or [])
|
|
2050
|
+
state_keys = set(response.get("state_keys") or [])
|
|
2051
|
+
|
|
2052
|
+
for term in _probe_strings(case.get("expected_contains")):
|
|
2053
|
+
checks.append(
|
|
2054
|
+
_probe_check(
|
|
2055
|
+
f"content_contains_{_framework_key(term)}",
|
|
2056
|
+
term.lower() in content.lower(),
|
|
2057
|
+
f"response content should contain {term!r}",
|
|
2058
|
+
expected=term,
|
|
2059
|
+
observed=content,
|
|
2060
|
+
)
|
|
2061
|
+
)
|
|
2062
|
+
for tool in _probe_strings(case.get("required_tools")):
|
|
2063
|
+
checks.append(
|
|
2064
|
+
_probe_check(
|
|
2065
|
+
f"required_tool_{_framework_key(tool)}",
|
|
2066
|
+
tool in tool_names,
|
|
2067
|
+
f"response should emit required tool {tool!r}",
|
|
2068
|
+
expected=tool,
|
|
2069
|
+
observed=sorted(tool_names),
|
|
2070
|
+
)
|
|
2071
|
+
)
|
|
2072
|
+
for event_type in _probe_strings(case.get("required_events")):
|
|
2073
|
+
checks.append(
|
|
2074
|
+
_probe_check(
|
|
2075
|
+
f"required_event_{_framework_key(event_type)}",
|
|
2076
|
+
event_type in event_types,
|
|
2077
|
+
f"response should emit required event {event_type!r}",
|
|
2078
|
+
expected=event_type,
|
|
2079
|
+
observed=sorted(event_types),
|
|
2080
|
+
)
|
|
2081
|
+
)
|
|
2082
|
+
for state_key in _probe_strings(case.get("required_state_keys")):
|
|
2083
|
+
checks.append(
|
|
2084
|
+
_probe_check(
|
|
2085
|
+
f"required_state_{_framework_key(state_key)}",
|
|
2086
|
+
state_key in state_keys,
|
|
2087
|
+
f"response should include required state key {state_key!r}",
|
|
2088
|
+
expected=state_key,
|
|
2089
|
+
observed=sorted(state_keys),
|
|
2090
|
+
)
|
|
2091
|
+
)
|
|
2092
|
+
if trace_runtime:
|
|
2093
|
+
runtime_summary = dict(runtime_trace.get("summary") or {})
|
|
2094
|
+
runtime_contract = dict(
|
|
2095
|
+
dict(runtime_trace.get("metadata") or {}).get("framework_adapter_contract")
|
|
2096
|
+
or {}
|
|
2097
|
+
)
|
|
2098
|
+
signature = dict(contract.get("callable_signature") or {})
|
|
2099
|
+
io_summary = dict(observed_io_contract.get("summary") or {})
|
|
2100
|
+
checks.extend(
|
|
2101
|
+
[
|
|
2102
|
+
_probe_check(
|
|
2103
|
+
"framework_runtime_trace_present",
|
|
2104
|
+
bool(runtime_trace),
|
|
2105
|
+
"trace_runtime=True should attach framework_runtime state",
|
|
2106
|
+
expected=True,
|
|
2107
|
+
observed=bool(runtime_trace),
|
|
2108
|
+
),
|
|
2109
|
+
_probe_check(
|
|
2110
|
+
"framework_runtime_contract_present",
|
|
2111
|
+
runtime_contract.get("kind")
|
|
2112
|
+
== "agent-learning.framework-adapter-contract.v1",
|
|
2113
|
+
"runtime trace should carry the adapter contract",
|
|
2114
|
+
expected="agent-learning.framework-adapter-contract.v1",
|
|
2115
|
+
observed=runtime_contract.get("kind"),
|
|
2116
|
+
),
|
|
2117
|
+
_probe_check(
|
|
2118
|
+
"framework_runtime_invocation_present",
|
|
2119
|
+
int(runtime_summary.get("invocation_count") or 0) >= 1,
|
|
2120
|
+
"runtime trace should record at least one invocation",
|
|
2121
|
+
expected=">=1",
|
|
2122
|
+
observed=runtime_summary.get("invocation_count"),
|
|
2123
|
+
),
|
|
2124
|
+
_probe_check(
|
|
2125
|
+
"framework_adapter_callable_signature_present",
|
|
2126
|
+
signature.get("kind")
|
|
2127
|
+
== "agent-learning.framework-adapter-callable-signature.v1",
|
|
2128
|
+
"probe contract should carry deterministic callable signature evidence",
|
|
2129
|
+
expected="agent-learning.framework-adapter-callable-signature.v1",
|
|
2130
|
+
observed=signature.get("kind"),
|
|
2131
|
+
),
|
|
2132
|
+
_probe_check(
|
|
2133
|
+
"framework_adapter_observed_io_contract_present",
|
|
2134
|
+
observed_io_contract.get("kind")
|
|
2135
|
+
== "agent-learning.framework-adapter-observed-io-contract.v1"
|
|
2136
|
+
and int(io_summary.get("invocation_count") or 0) >= 1,
|
|
2137
|
+
"probe case should carry observed input/output contract evidence",
|
|
2138
|
+
expected="agent-learning.framework-adapter-observed-io-contract.v1",
|
|
2139
|
+
observed={
|
|
2140
|
+
"kind": observed_io_contract.get("kind"),
|
|
2141
|
+
"invocation_count": io_summary.get("invocation_count"),
|
|
2142
|
+
},
|
|
2143
|
+
),
|
|
2144
|
+
_probe_check(
|
|
2145
|
+
"framework_adapter_observed_io_matches_signature",
|
|
2146
|
+
io_summary.get("signature_bound") is True,
|
|
2147
|
+
"observed adapter invocation should bind to the callable signature",
|
|
2148
|
+
expected=True,
|
|
2149
|
+
observed=io_summary.get("signature_bound"),
|
|
2150
|
+
),
|
|
2151
|
+
]
|
|
2152
|
+
)
|
|
2153
|
+
checks.append(
|
|
2154
|
+
_probe_check(
|
|
2155
|
+
"adapter_contract_local_first",
|
|
2156
|
+
contract.get("requires_external_service") is False,
|
|
2157
|
+
"adapter contract should not require a hosted service",
|
|
2158
|
+
expected=False,
|
|
2159
|
+
observed=contract.get("requires_external_service"),
|
|
2160
|
+
)
|
|
2161
|
+
)
|
|
2162
|
+
return checks
|
|
2163
|
+
|
|
2164
|
+
|
|
2165
|
+
def _probe_check(
|
|
2166
|
+
check_id: str,
|
|
2167
|
+
passed: bool,
|
|
2168
|
+
message: str,
|
|
2169
|
+
*,
|
|
2170
|
+
expected: Any = None,
|
|
2171
|
+
observed: Any = None,
|
|
2172
|
+
) -> dict[str, Any]:
|
|
2173
|
+
return {
|
|
2174
|
+
"id": check_id,
|
|
2175
|
+
"passed": bool(passed),
|
|
2176
|
+
"message": message,
|
|
2177
|
+
"expected": expected,
|
|
2178
|
+
"observed": observed,
|
|
2179
|
+
}
|
|
2180
|
+
|
|
2181
|
+
|
|
2182
|
+
def _probe_strings(value: Any) -> list[str]:
|
|
2183
|
+
if value is None:
|
|
2184
|
+
return []
|
|
2185
|
+
if isinstance(value, str):
|
|
2186
|
+
return [value]
|
|
2187
|
+
if isinstance(value, Sequence):
|
|
2188
|
+
return [str(item) for item in value if str(item)]
|
|
2189
|
+
return [str(value)]
|
|
2190
|
+
|
|
2191
|
+
|
|
2192
|
+
def _unique_probe_strings(values: Sequence[Any]) -> list[str]:
|
|
2193
|
+
seen: set[str] = set()
|
|
2194
|
+
unique: list[str] = []
|
|
2195
|
+
for value in values:
|
|
2196
|
+
text = str(value or "")
|
|
2197
|
+
if not text or text in seen:
|
|
2198
|
+
continue
|
|
2199
|
+
seen.add(text)
|
|
2200
|
+
unique.append(text)
|
|
2201
|
+
return unique
|
|
2202
|
+
|
|
2203
|
+
|
|
2204
|
+
def _probe_observed_io_contract(
|
|
2205
|
+
framework: str,
|
|
2206
|
+
*,
|
|
2207
|
+
case_id: str,
|
|
2208
|
+
runtime_trace: Mapping[str, Any],
|
|
2209
|
+
response_payload: Mapping[str, Any],
|
|
2210
|
+
contract: Mapping[str, Any],
|
|
2211
|
+
) -> dict[str, Any]:
|
|
2212
|
+
invocations = [
|
|
2213
|
+
dict(invocation)
|
|
2214
|
+
for invocation in list(runtime_trace.get("invocations") or [])
|
|
2215
|
+
if isinstance(invocation, Mapping)
|
|
2216
|
+
]
|
|
2217
|
+
signature = (
|
|
2218
|
+
dict(contract.get("callable_signature") or {})
|
|
2219
|
+
if isinstance(contract.get("callable_signature"), Mapping)
|
|
2220
|
+
else {}
|
|
2221
|
+
)
|
|
2222
|
+
observed_invocations: list[dict[str, Any]] = []
|
|
2223
|
+
for index, invocation in enumerate(invocations, start=1):
|
|
2224
|
+
input_shape = dict(invocation.get("input") or {})
|
|
2225
|
+
output_shape = dict(invocation.get("output") or {})
|
|
2226
|
+
observed_invocations.append(
|
|
2227
|
+
{
|
|
2228
|
+
"id": str(invocation.get("id") or f"{case_id}_invocation_{index}"),
|
|
2229
|
+
"framework": str(invocation.get("framework") or framework),
|
|
2230
|
+
"method": str(invocation.get("method") or ""),
|
|
2231
|
+
"input_mode": str(invocation.get("input_mode") or ""),
|
|
2232
|
+
"call_style": str(invocation.get("call_style") or ""),
|
|
2233
|
+
"input_key": invocation.get("input_key"),
|
|
2234
|
+
"input_kwargs_keys": _unique_probe_strings(
|
|
2235
|
+
list(invocation.get("input_kwargs_keys") or [])
|
|
2236
|
+
),
|
|
2237
|
+
"input_shape": input_shape,
|
|
2238
|
+
"output_shape": output_shape,
|
|
2239
|
+
"duration_ms": int(invocation.get("duration_ms") or 0),
|
|
2240
|
+
"signals": _unique_probe_strings(list(invocation.get("signals") or [])),
|
|
2241
|
+
}
|
|
2242
|
+
)
|
|
2243
|
+
signature_bound = bool(signature) and bool(observed_invocations) and all(
|
|
2244
|
+
_probe_invocation_matches_signature(invocation, signature)
|
|
2245
|
+
for invocation in observed_invocations
|
|
2246
|
+
)
|
|
2247
|
+
input_keys = _unique_probe_strings(
|
|
2248
|
+
invocation.get("input_key")
|
|
2249
|
+
for invocation in observed_invocations
|
|
2250
|
+
if invocation.get("input_key") not in (None, "", [], {})
|
|
2251
|
+
)
|
|
2252
|
+
output_shapes = [dict(item.get("output_shape") or {}) for item in observed_invocations]
|
|
2253
|
+
return {
|
|
2254
|
+
"kind": "agent-learning.framework-adapter-observed-io-contract.v1",
|
|
2255
|
+
"framework": framework,
|
|
2256
|
+
"case_id": case_id,
|
|
2257
|
+
"method": contract.get("method"),
|
|
2258
|
+
"input_mode": contract.get("input_mode"),
|
|
2259
|
+
"signature_method": signature.get("method"),
|
|
2260
|
+
"signature_bound": signature_bound,
|
|
2261
|
+
"invocations": observed_invocations,
|
|
2262
|
+
"summary": {
|
|
2263
|
+
"invocation_count": len(observed_invocations),
|
|
2264
|
+
"methods": _unique_probe_strings(
|
|
2265
|
+
invocation.get("method") for invocation in observed_invocations
|
|
2266
|
+
),
|
|
2267
|
+
"input_modes": _unique_probe_strings(
|
|
2268
|
+
invocation.get("input_mode") for invocation in observed_invocations
|
|
2269
|
+
),
|
|
2270
|
+
"call_styles": _unique_probe_strings(
|
|
2271
|
+
invocation.get("call_style") for invocation in observed_invocations
|
|
2272
|
+
),
|
|
2273
|
+
"input_keys": input_keys,
|
|
2274
|
+
"input_kwargs_keys": _unique_probe_strings(
|
|
2275
|
+
key
|
|
2276
|
+
for invocation in observed_invocations
|
|
2277
|
+
for key in list(invocation.get("input_kwargs_keys") or [])
|
|
2278
|
+
),
|
|
2279
|
+
"input_types": _unique_probe_strings(
|
|
2280
|
+
dict(invocation.get("input_shape") or {}).get("type")
|
|
2281
|
+
for invocation in observed_invocations
|
|
2282
|
+
),
|
|
2283
|
+
"output_types": _unique_probe_strings(
|
|
2284
|
+
output.get("type") for output in output_shapes
|
|
2285
|
+
),
|
|
2286
|
+
"output_state_keys": _unique_probe_strings(
|
|
2287
|
+
key for output in output_shapes for key in list(output.get("state_keys") or [])
|
|
2288
|
+
),
|
|
2289
|
+
"output_metadata_keys": _unique_probe_strings(
|
|
2290
|
+
key
|
|
2291
|
+
for output in output_shapes
|
|
2292
|
+
for key in list(output.get("metadata_keys") or [])
|
|
2293
|
+
),
|
|
2294
|
+
"output_tool_names": _unique_probe_strings(
|
|
2295
|
+
key for output in output_shapes for key in list(output.get("tool_names") or [])
|
|
2296
|
+
),
|
|
2297
|
+
"output_event_types": _unique_probe_strings(
|
|
2298
|
+
key for output in output_shapes for key in list(output.get("event_types") or [])
|
|
2299
|
+
),
|
|
2300
|
+
"output_artifact_types": _unique_probe_strings(
|
|
2301
|
+
key
|
|
2302
|
+
for output in output_shapes
|
|
2303
|
+
for key in list(output.get("artifact_types") or [])
|
|
2304
|
+
),
|
|
2305
|
+
"content_observed": bool(response_payload.get("content")),
|
|
2306
|
+
"signature_bound": signature_bound,
|
|
2307
|
+
},
|
|
2308
|
+
}
|
|
2309
|
+
|
|
2310
|
+
|
|
2311
|
+
def _probe_invocation_matches_signature(
|
|
2312
|
+
invocation: Mapping[str, Any],
|
|
2313
|
+
signature: Mapping[str, Any],
|
|
2314
|
+
) -> bool:
|
|
2315
|
+
if not signature:
|
|
2316
|
+
return False
|
|
2317
|
+
method = str(invocation.get("method") or "")
|
|
2318
|
+
signature_method = str(signature.get("method") or "")
|
|
2319
|
+
signature_leaf = str(signature.get("method_leaf") or "")
|
|
2320
|
+
method_matches = method in {signature_method, signature_leaf, "callable"} or (
|
|
2321
|
+
signature_method in {"", "auto"} and bool(method)
|
|
2322
|
+
)
|
|
2323
|
+
if not method_matches:
|
|
2324
|
+
return False
|
|
2325
|
+
|
|
2326
|
+
call_style = str(invocation.get("call_style") or "")
|
|
2327
|
+
input_key = invocation.get("input_key")
|
|
2328
|
+
parameter_names = set(str(name) for name in list(signature.get("parameter_names") or []))
|
|
2329
|
+
input_kwargs_keys = set(
|
|
2330
|
+
str(key) for key in list(invocation.get("input_kwargs_keys") or []) if str(key)
|
|
2331
|
+
)
|
|
2332
|
+
required_parameters = set(
|
|
2333
|
+
str(name) for name in list(signature.get("required_parameters") or []) if str(name)
|
|
2334
|
+
)
|
|
2335
|
+
accepts_var_keyword = bool(signature.get("accepts_var_keyword"))
|
|
2336
|
+
|
|
2337
|
+
if call_style in {"keyword", "positional_with_kwargs"} and input_key:
|
|
2338
|
+
return (
|
|
2339
|
+
str(input_key) in parameter_names
|
|
2340
|
+
or accepts_var_keyword
|
|
2341
|
+
or str(input_key) == str(signature.get("preferred_input_key") or "")
|
|
2342
|
+
)
|
|
2343
|
+
if call_style == "expanded_kwargs":
|
|
2344
|
+
return accepts_var_keyword
|
|
2345
|
+
if call_style in {"positional", "positional_with_kwargs"}:
|
|
2346
|
+
return bool(signature.get("accepts_positional"))
|
|
2347
|
+
if call_style == "none":
|
|
2348
|
+
return required_parameters <= input_kwargs_keys
|
|
2349
|
+
return False
|
|
2350
|
+
|
|
2351
|
+
|
|
2352
|
+
def _probe_response_payload(response: AgentResponse) -> dict[str, Any]:
|
|
2353
|
+
tool_calls = [dict(call) for call in response.tool_calls or []]
|
|
2354
|
+
events = [event.model_dump() for event in response.events]
|
|
2355
|
+
artifacts = [artifact.model_dump() for artifact in response.artifacts]
|
|
2356
|
+
state = dict(response.state or {})
|
|
2357
|
+
metadata = dict(response.metadata or {})
|
|
2358
|
+
streaming_trace = (
|
|
2359
|
+
state.get("streaming_trace")
|
|
2360
|
+
if isinstance(state.get("streaming_trace"), Mapping)
|
|
2361
|
+
else {}
|
|
2362
|
+
)
|
|
2363
|
+
return {
|
|
2364
|
+
"content": response.content,
|
|
2365
|
+
"tool_call_count": len(tool_calls),
|
|
2366
|
+
"tool_names": sorted(
|
|
2367
|
+
{
|
|
2368
|
+
str(
|
|
2369
|
+
call.get("name")
|
|
2370
|
+
or call.get("tool")
|
|
2371
|
+
or dict(call.get("function") or {}).get("name")
|
|
2372
|
+
or ""
|
|
2373
|
+
)
|
|
2374
|
+
for call in tool_calls
|
|
2375
|
+
if isinstance(call, Mapping)
|
|
2376
|
+
}
|
|
2377
|
+
),
|
|
2378
|
+
"event_count": len(events),
|
|
2379
|
+
"event_types": sorted({str(event.get("type") or "") for event in events}),
|
|
2380
|
+
"artifact_count": len(artifacts),
|
|
2381
|
+
"artifact_types": sorted({str(artifact.get("type") or "") for artifact in artifacts}),
|
|
2382
|
+
"artifact_evidence": [
|
|
2383
|
+
_probe_artifact_evidence(artifact) for artifact in artifacts
|
|
2384
|
+
],
|
|
2385
|
+
"state_keys": sorted(str(key) for key in state),
|
|
2386
|
+
"framework_lifecycle_summary": _probe_framework_lifecycle_summary(
|
|
2387
|
+
state.get("framework_lifecycle_trace")
|
|
2388
|
+
),
|
|
2389
|
+
"framework_trace_summary": _probe_framework_trace_summary(
|
|
2390
|
+
state.get("framework_trace")
|
|
2391
|
+
),
|
|
2392
|
+
"message_history_summary": _probe_message_history_summary(
|
|
2393
|
+
state.get("message_history")
|
|
2394
|
+
),
|
|
2395
|
+
"framework_handoff_summary": _probe_framework_handoff_summary(
|
|
2396
|
+
state.get("framework_handoffs")
|
|
2397
|
+
),
|
|
2398
|
+
"orchestration_trace_summary": _probe_orchestration_trace_summary(
|
|
2399
|
+
state.get("orchestration_trace")
|
|
2400
|
+
),
|
|
2401
|
+
"workflow_trace_summary": _probe_workflow_trace_summary(
|
|
2402
|
+
state.get("workflow_trace")
|
|
2403
|
+
),
|
|
2404
|
+
"realtime_trace_summary": _probe_realtime_trace_summary(
|
|
2405
|
+
state.get("realtime_trace")
|
|
2406
|
+
),
|
|
2407
|
+
"framework_memory_summary": _probe_framework_memory_summary(
|
|
2408
|
+
state.get("framework_memory")
|
|
2409
|
+
),
|
|
2410
|
+
"retrieval_memory_summary": _probe_retrieval_memory_summary(
|
|
2411
|
+
state.get("retrieval_memory")
|
|
2412
|
+
),
|
|
2413
|
+
"agent_memory_lineage_summary": _probe_agent_memory_lineage_summary(
|
|
2414
|
+
state.get("agent_memory_lineage")
|
|
2415
|
+
),
|
|
2416
|
+
"browser_cua_summary": _probe_browser_cua_summary(
|
|
2417
|
+
state.get("browser_cua")
|
|
2418
|
+
),
|
|
2419
|
+
"mcp_tool_session_summary": _probe_mcp_tool_session_summary(
|
|
2420
|
+
state.get("mcp_tool_session")
|
|
2421
|
+
),
|
|
2422
|
+
"a2a_protocol_summary": _probe_a2a_protocol_summary(
|
|
2423
|
+
state.get("a2a_protocol_trace")
|
|
2424
|
+
),
|
|
2425
|
+
"openenv_summary": _probe_openenv_summary(state.get("openenv")),
|
|
2426
|
+
"agent_trust_boundary_summary": _probe_agent_trust_boundary_summary(
|
|
2427
|
+
state.get("agent_trust_boundary_model")
|
|
2428
|
+
),
|
|
2429
|
+
"agent_control_plane_summary": _probe_agent_control_plane_summary(
|
|
2430
|
+
state.get("agent_control_plane")
|
|
2431
|
+
),
|
|
2432
|
+
"metadata_keys": sorted(str(key) for key in metadata),
|
|
2433
|
+
"streaming": bool(streaming_trace or metadata.get("streaming")),
|
|
2434
|
+
"streaming_trace_signals": sorted(
|
|
2435
|
+
str(signal)
|
|
2436
|
+
for signal in (dict(streaming_trace).get("signals") or [])
|
|
2437
|
+
if str(signal)
|
|
2438
|
+
),
|
|
2439
|
+
"streaming_trace_summary": dict(
|
|
2440
|
+
dict(streaming_trace).get("summary") or {}
|
|
2441
|
+
),
|
|
2442
|
+
}
|
|
2443
|
+
|
|
2444
|
+
|
|
2445
|
+
def _probe_framework_lifecycle_summary(value: Any) -> dict[str, Any]:
|
|
2446
|
+
trace = dict(value or {}) if isinstance(value, Mapping) else {}
|
|
2447
|
+
summary = trace.get("summary")
|
|
2448
|
+
return dict(summary) if isinstance(summary, Mapping) else {}
|
|
2449
|
+
|
|
2450
|
+
|
|
2451
|
+
def _probe_agent_trust_boundary_summary(value: Any) -> dict[str, Any]:
|
|
2452
|
+
payload = dict(value or {}) if isinstance(value, Mapping) else {}
|
|
2453
|
+
summary = dict(payload.get("summary") or {}) if payload else {}
|
|
2454
|
+
if not payload and not summary:
|
|
2455
|
+
return {}
|
|
2456
|
+
return {
|
|
2457
|
+
"framework": payload.get("framework"),
|
|
2458
|
+
"control_count": summary.get("control_count"),
|
|
2459
|
+
"required_control_rate": summary.get("required_control_rate"),
|
|
2460
|
+
"high_risk_unmitigated_count": summary.get(
|
|
2461
|
+
"high_risk_unmitigated_count"
|
|
2462
|
+
),
|
|
2463
|
+
"evidence_count": summary.get("evidence_count"),
|
|
2464
|
+
"gaps": list(summary.get("gaps") or []),
|
|
2465
|
+
"present_controls": list(summary.get("present_controls") or []),
|
|
2466
|
+
"present_categories": list(summary.get("present_categories") or []),
|
|
2467
|
+
"assets": [
|
|
2468
|
+
str(item.get("id") or item.get("name") or "")
|
|
2469
|
+
for item in _probe_mappings(payload.get("assets"))
|
|
2470
|
+
if item.get("id") or item.get("name")
|
|
2471
|
+
],
|
|
2472
|
+
"tools": [
|
|
2473
|
+
str(item.get("id") or item.get("name") or "")
|
|
2474
|
+
for item in _probe_mappings(payload.get("tools"))
|
|
2475
|
+
if item.get("id") or item.get("name")
|
|
2476
|
+
],
|
|
2477
|
+
"surfaces": [
|
|
2478
|
+
str(item.get("id") or item.get("name") or "")
|
|
2479
|
+
for item in _probe_mappings(payload.get("surfaces"))
|
|
2480
|
+
if item.get("id") or item.get("name")
|
|
2481
|
+
],
|
|
2482
|
+
"threats": list(summary.get("threats") or []),
|
|
2483
|
+
"mitigated_threats": list(summary.get("mitigated_threats") or []),
|
|
2484
|
+
"signals": list(payload.get("signals") or []),
|
|
2485
|
+
"has_identity": bool(summary.get("has_identity")),
|
|
2486
|
+
"has_permissions": bool(summary.get("has_permissions")),
|
|
2487
|
+
"has_sandbox": bool(summary.get("has_sandbox")),
|
|
2488
|
+
"has_audit": bool(summary.get("has_audit")),
|
|
2489
|
+
"has_canaries": bool(summary.get("has_canaries")),
|
|
2490
|
+
"has_human_approval": bool(summary.get("has_human_approval")),
|
|
2491
|
+
"has_memory_isolation": bool(summary.get("has_memory_isolation")),
|
|
2492
|
+
"has_network_egress_controls": bool(
|
|
2493
|
+
summary.get("has_network_egress_controls")
|
|
2494
|
+
),
|
|
2495
|
+
"has_tool_allowlist": bool(summary.get("has_tool_allowlist")),
|
|
2496
|
+
"has_data_boundary": bool(summary.get("has_data_boundary")),
|
|
2497
|
+
"has_secret_handling": bool(summary.get("has_secret_handling")),
|
|
2498
|
+
}
|
|
2499
|
+
|
|
2500
|
+
|
|
2501
|
+
def _probe_agent_control_plane_summary(value: Any) -> dict[str, Any]:
|
|
2502
|
+
payload = dict(value or {}) if isinstance(value, Mapping) else {}
|
|
2503
|
+
summary = dict(payload.get("summary") or {}) if payload else {}
|
|
2504
|
+
if not payload and not summary:
|
|
2505
|
+
return {}
|
|
2506
|
+
return {
|
|
2507
|
+
"framework": payload.get("framework"),
|
|
2508
|
+
"control_count": summary.get("control_count"),
|
|
2509
|
+
"required_control_rate": summary.get("required_control_rate"),
|
|
2510
|
+
"exceeded_budget_count": summary.get("exceeded_budget_count"),
|
|
2511
|
+
"high_risk_uncontained_count": summary.get(
|
|
2512
|
+
"high_risk_uncontained_count"
|
|
2513
|
+
),
|
|
2514
|
+
"approval_required_action_count": summary.get(
|
|
2515
|
+
"approval_required_action_count"
|
|
2516
|
+
),
|
|
2517
|
+
"approved_action_count": summary.get("approved_action_count"),
|
|
2518
|
+
"blocked_action_count": summary.get("blocked_action_count"),
|
|
2519
|
+
"rolled_back_action_count": summary.get("rolled_back_action_count"),
|
|
2520
|
+
"contained_incident_count": summary.get("contained_incident_count"),
|
|
2521
|
+
"within_budget_count": summary.get("within_budget_count"),
|
|
2522
|
+
"evidence_count": summary.get("evidence_count"),
|
|
2523
|
+
"gaps": list(summary.get("gaps") or []),
|
|
2524
|
+
"present_controls": list(summary.get("present_controls") or []),
|
|
2525
|
+
"present_categories": list(summary.get("present_categories") or []),
|
|
2526
|
+
"actions": list(summary.get("actions") or []),
|
|
2527
|
+
"budgets": list(summary.get("budgets") or []),
|
|
2528
|
+
"incidents": list(summary.get("incidents") or []),
|
|
2529
|
+
"signals": list(payload.get("signals") or []),
|
|
2530
|
+
"has_risk_scoring": bool(summary.get("has_risk_scoring")),
|
|
2531
|
+
"has_action_policy": bool(summary.get("has_action_policy")),
|
|
2532
|
+
"has_approval_gates": bool(summary.get("has_approval_gates")),
|
|
2533
|
+
"has_rollback": bool(summary.get("has_rollback")),
|
|
2534
|
+
"has_kill_switch": bool(summary.get("has_kill_switch")),
|
|
2535
|
+
"has_circuit_breakers": bool(summary.get("has_circuit_breakers")),
|
|
2536
|
+
"has_rate_limits": bool(summary.get("has_rate_limits")),
|
|
2537
|
+
"has_budgets": bool(summary.get("has_budgets")),
|
|
2538
|
+
"has_audit": bool(summary.get("has_audit")),
|
|
2539
|
+
"has_containment": bool(summary.get("has_containment")),
|
|
2540
|
+
"has_drift_detection": bool(summary.get("has_drift_detection")),
|
|
2541
|
+
}
|
|
2542
|
+
|
|
2543
|
+
|
|
2544
|
+
def _probe_framework_trace_summary(value: Any) -> dict[str, Any]:
|
|
2545
|
+
trace = dict(value or {}) if isinstance(value, Mapping) else {}
|
|
2546
|
+
if not trace:
|
|
2547
|
+
return {}
|
|
2548
|
+
summary = (
|
|
2549
|
+
dict(trace.get("summary") or {})
|
|
2550
|
+
if isinstance(trace.get("summary"), Mapping)
|
|
2551
|
+
else {}
|
|
2552
|
+
)
|
|
2553
|
+
for count_key, trace_key in (
|
|
2554
|
+
("span_count", "spans"),
|
|
2555
|
+
("event_count", "events"),
|
|
2556
|
+
("checkpoint_count", "checkpoints"),
|
|
2557
|
+
("session_count", "sessions"),
|
|
2558
|
+
):
|
|
2559
|
+
if count_key not in summary and isinstance(trace.get(trace_key), list):
|
|
2560
|
+
summary[count_key] = len(trace.get(trace_key, []))
|
|
2561
|
+
elif count_key not in summary and trace.get(count_key) is not None:
|
|
2562
|
+
summary[count_key] = trace.get(count_key)
|
|
2563
|
+
for key in ("signals", "tool_names"):
|
|
2564
|
+
if trace.get(key):
|
|
2565
|
+
summary[key] = sorted(str(item) for item in trace.get(key, []) if str(item))
|
|
2566
|
+
adapter_conformance = (
|
|
2567
|
+
dict(trace.get("adapter_conformance"))
|
|
2568
|
+
if isinstance(trace.get("adapter_conformance"), Mapping)
|
|
2569
|
+
else {}
|
|
2570
|
+
)
|
|
2571
|
+
if adapter_conformance:
|
|
2572
|
+
summary["adapter_conformance_passed"] = bool(
|
|
2573
|
+
adapter_conformance.get("passed")
|
|
2574
|
+
)
|
|
2575
|
+
findings = _probe_mappings(adapter_conformance.get("findings"))
|
|
2576
|
+
summary["adapter_conformance_finding_count"] = len(findings)
|
|
2577
|
+
required_signals = adapter_conformance.get("required_signals")
|
|
2578
|
+
if required_signals:
|
|
2579
|
+
summary["adapter_required_signals"] = sorted(
|
|
2580
|
+
str(item) for item in _probe_list(required_signals) if str(item)
|
|
2581
|
+
)
|
|
2582
|
+
spans = _probe_mappings(trace.get("spans"))
|
|
2583
|
+
events = _probe_mappings(trace.get("events"))
|
|
2584
|
+
if spans and "span_names" not in summary:
|
|
2585
|
+
summary["span_names"] = sorted(
|
|
2586
|
+
{
|
|
2587
|
+
str(span.get("name") or span.get("id") or "")
|
|
2588
|
+
for span in spans
|
|
2589
|
+
if span.get("name") or span.get("id")
|
|
2590
|
+
}
|
|
2591
|
+
)
|
|
2592
|
+
if events and "event_names" not in summary:
|
|
2593
|
+
summary["event_names"] = sorted(
|
|
2594
|
+
{
|
|
2595
|
+
str(event.get("name") or event.get("id") or event.get("type") or "")
|
|
2596
|
+
for event in events
|
|
2597
|
+
if event.get("name") or event.get("id") or event.get("type")
|
|
2598
|
+
}
|
|
2599
|
+
)
|
|
2600
|
+
return summary
|
|
2601
|
+
|
|
2602
|
+
|
|
2603
|
+
def _probe_message_history_summary(value: Any) -> dict[str, Any]:
|
|
2604
|
+
history = dict(value or {}) if isinstance(value, Mapping) else {}
|
|
2605
|
+
if not history:
|
|
2606
|
+
return {}
|
|
2607
|
+
messages = _probe_mappings(history.get("messages"))
|
|
2608
|
+
summary: dict[str, Any] = {}
|
|
2609
|
+
for key in (
|
|
2610
|
+
"message_count",
|
|
2611
|
+
"tool_call_count",
|
|
2612
|
+
"tool_response_count",
|
|
2613
|
+
"handoff_count",
|
|
2614
|
+
):
|
|
2615
|
+
if history.get(key) not in (None, "", [], {}):
|
|
2616
|
+
summary[key] = history.get(key)
|
|
2617
|
+
for key in ("roles", "sources", "types", "tool_names"):
|
|
2618
|
+
values = _probe_list(history.get(key))
|
|
2619
|
+
if values:
|
|
2620
|
+
summary[key] = sorted(str(item) for item in values if str(item))
|
|
2621
|
+
speaker_sequence = [
|
|
2622
|
+
str(
|
|
2623
|
+
message.get("source")
|
|
2624
|
+
or message.get("speaker")
|
|
2625
|
+
or message.get("role")
|
|
2626
|
+
or ""
|
|
2627
|
+
)
|
|
2628
|
+
for message in messages
|
|
2629
|
+
if message.get("source") or message.get("speaker") or message.get("role")
|
|
2630
|
+
]
|
|
2631
|
+
if speaker_sequence:
|
|
2632
|
+
summary["speaker_sequence"] = speaker_sequence
|
|
2633
|
+
message_types = [
|
|
2634
|
+
str(message.get("type") or message.get("message_type") or "")
|
|
2635
|
+
for message in messages
|
|
2636
|
+
if message.get("type") or message.get("message_type")
|
|
2637
|
+
]
|
|
2638
|
+
if message_types:
|
|
2639
|
+
summary["message_types"] = sorted(set(message_types))
|
|
2640
|
+
stop_reason = str(history.get("stop_reason") or "")
|
|
2641
|
+
if stop_reason:
|
|
2642
|
+
summary["stop_reason"] = stop_reason
|
|
2643
|
+
last_content = str(history.get("last_content") or "")
|
|
2644
|
+
if last_content:
|
|
2645
|
+
summary["last_content"] = last_content
|
|
2646
|
+
handoffs = _probe_mappings(history.get("handoffs"))
|
|
2647
|
+
if handoffs:
|
|
2648
|
+
summary["handoffs"] = [
|
|
2649
|
+
{
|
|
2650
|
+
key: str(handoff.get(key) or "")
|
|
2651
|
+
for key in ("from", "to", "task")
|
|
2652
|
+
if handoff.get(key) not in (None, "", [], {})
|
|
2653
|
+
}
|
|
2654
|
+
for handoff in handoffs
|
|
2655
|
+
]
|
|
2656
|
+
return summary
|
|
2657
|
+
|
|
2658
|
+
|
|
2659
|
+
def _probe_framework_handoff_summary(value: Any) -> dict[str, Any]:
|
|
2660
|
+
coordination = dict(value or {}) if isinstance(value, Mapping) else {}
|
|
2661
|
+
if not coordination:
|
|
2662
|
+
return {}
|
|
2663
|
+
summary: dict[str, Any] = {}
|
|
2664
|
+
for key in ("handoff_count", "review_count", "reconciliation_count"):
|
|
2665
|
+
if coordination.get(key) not in (None, "", [], {}):
|
|
2666
|
+
summary[key] = coordination.get(key)
|
|
2667
|
+
participants = _probe_list(coordination.get("participants"))
|
|
2668
|
+
if participants:
|
|
2669
|
+
summary["participants"] = sorted(str(item) for item in participants if str(item))
|
|
2670
|
+
handoffs = _probe_mappings(coordination.get("handoffs"))
|
|
2671
|
+
if handoffs:
|
|
2672
|
+
summary["handoffs"] = [
|
|
2673
|
+
{
|
|
2674
|
+
key: str(handoff.get(key) or "")
|
|
2675
|
+
for key in ("from", "to", "task", "reason", "message_type")
|
|
2676
|
+
if handoff.get(key) not in (None, "", [], {})
|
|
2677
|
+
}
|
|
2678
|
+
for handoff in handoffs
|
|
2679
|
+
]
|
|
2680
|
+
reviews = _probe_mappings(coordination.get("reviews"))
|
|
2681
|
+
if reviews:
|
|
2682
|
+
summary["reviews"] = [
|
|
2683
|
+
{
|
|
2684
|
+
key: str(review.get(key) or "")
|
|
2685
|
+
for key in ("reviewer", "target", "status", "message_type")
|
|
2686
|
+
if review.get(key) not in (None, "", [], {})
|
|
2687
|
+
}
|
|
2688
|
+
for review in reviews
|
|
2689
|
+
]
|
|
2690
|
+
reconciliations = _probe_mappings(coordination.get("reconciliations"))
|
|
2691
|
+
if reconciliations:
|
|
2692
|
+
summary["reconciliations"] = [
|
|
2693
|
+
{
|
|
2694
|
+
key: str(reconciliation.get(key) or "")
|
|
2695
|
+
for key in (
|
|
2696
|
+
"source",
|
|
2697
|
+
"accepted_source",
|
|
2698
|
+
"status",
|
|
2699
|
+
"message_type",
|
|
2700
|
+
)
|
|
2701
|
+
if reconciliation.get(key) not in (None, "", [], {})
|
|
2702
|
+
}
|
|
2703
|
+
for reconciliation in reconciliations
|
|
2704
|
+
]
|
|
2705
|
+
summary["has_handoffs"] = bool(handoffs)
|
|
2706
|
+
summary["has_reviews"] = bool(reviews)
|
|
2707
|
+
summary["has_reconciliation"] = bool(reconciliations)
|
|
2708
|
+
return summary
|
|
2709
|
+
|
|
2710
|
+
|
|
2711
|
+
def _probe_orchestration_trace_summary(value: Any) -> dict[str, Any]:
|
|
2712
|
+
trace = dict(value or {}) if isinstance(value, Mapping) else {}
|
|
2713
|
+
summary = dict(trace.get("summary") or {}) if isinstance(trace.get("summary"), Mapping) else {}
|
|
2714
|
+
if trace.get("signals"):
|
|
2715
|
+
summary["signals"] = sorted(str(signal) for signal in trace.get("signals", []) if str(signal))
|
|
2716
|
+
if trace.get("nodes"):
|
|
2717
|
+
summary["node_names"] = sorted(
|
|
2718
|
+
{
|
|
2719
|
+
str(dict(node).get("name") or dict(node).get("id") or "")
|
|
2720
|
+
for node in trace.get("nodes", [])
|
|
2721
|
+
if isinstance(node, Mapping)
|
|
2722
|
+
and (dict(node).get("name") or dict(node).get("id"))
|
|
2723
|
+
}
|
|
2724
|
+
)
|
|
2725
|
+
for key, trace_key in (
|
|
2726
|
+
("node_count", "nodes"),
|
|
2727
|
+
("edge_count", "edges"),
|
|
2728
|
+
("step_count", "steps"),
|
|
2729
|
+
):
|
|
2730
|
+
if key not in summary and isinstance(trace.get(trace_key), list):
|
|
2731
|
+
summary[key] = len(trace.get(trace_key, []))
|
|
2732
|
+
return summary
|
|
2733
|
+
|
|
2734
|
+
|
|
2735
|
+
def _probe_workflow_trace_summary(value: Any) -> dict[str, Any]:
|
|
2736
|
+
trace = dict(value or {}) if isinstance(value, Mapping) else {}
|
|
2737
|
+
if not trace:
|
|
2738
|
+
return {}
|
|
2739
|
+
summary = (
|
|
2740
|
+
dict(trace.get("summary") or {})
|
|
2741
|
+
if isinstance(trace.get("summary"), Mapping)
|
|
2742
|
+
else {}
|
|
2743
|
+
)
|
|
2744
|
+
for count_key, trace_key in (
|
|
2745
|
+
("node_count", "nodes"),
|
|
2746
|
+
("edge_count", "edges"),
|
|
2747
|
+
("step_count", "steps"),
|
|
2748
|
+
("checkpoint_count", "checkpoints"),
|
|
2749
|
+
("route_decision_count", "route_decisions"),
|
|
2750
|
+
("interrupt_count", "interrupts"),
|
|
2751
|
+
("replay_count", "replay"),
|
|
2752
|
+
("write_count", "writes"),
|
|
2753
|
+
("state_snapshot_count", "state_snapshots"),
|
|
2754
|
+
):
|
|
2755
|
+
if count_key not in summary and isinstance(trace.get(trace_key), list):
|
|
2756
|
+
summary[count_key] = len(trace.get(trace_key, []))
|
|
2757
|
+
elif count_key not in summary and trace.get(count_key) is not None:
|
|
2758
|
+
summary[count_key] = trace.get(count_key)
|
|
2759
|
+
if "tool_call_count" not in summary and trace.get("tool_call_count") is not None:
|
|
2760
|
+
summary["tool_call_count"] = trace.get("tool_call_count")
|
|
2761
|
+
|
|
2762
|
+
nodes = _probe_mappings(trace.get("nodes"))
|
|
2763
|
+
steps = _probe_mappings(trace.get("steps"))
|
|
2764
|
+
checkpoints = _probe_mappings(trace.get("checkpoints"))
|
|
2765
|
+
routes = _probe_mappings(trace.get("route_decisions"))
|
|
2766
|
+
interrupts = _probe_mappings(trace.get("interrupts"))
|
|
2767
|
+
replay = _probe_mappings(trace.get("replay"))
|
|
2768
|
+
topology = dict(trace.get("topology") or {}) if isinstance(trace.get("topology"), Mapping) else {}
|
|
2769
|
+
final_state = (
|
|
2770
|
+
dict(trace.get("final_state"))
|
|
2771
|
+
if isinstance(trace.get("final_state"), Mapping)
|
|
2772
|
+
else {}
|
|
2773
|
+
)
|
|
2774
|
+
|
|
2775
|
+
node_names = [
|
|
2776
|
+
node.get("name") or node.get("id")
|
|
2777
|
+
for node in nodes
|
|
2778
|
+
if node.get("name") or node.get("id")
|
|
2779
|
+
]
|
|
2780
|
+
step_names = [
|
|
2781
|
+
step.get("name") or step.get("node") or step.get("id")
|
|
2782
|
+
for step in steps
|
|
2783
|
+
if step.get("name") or step.get("node") or step.get("id")
|
|
2784
|
+
]
|
|
2785
|
+
tool_names = [
|
|
2786
|
+
*[str(item) for item in _probe_list(trace.get("tool_names")) if str(item)],
|
|
2787
|
+
*[
|
|
2788
|
+
str(call.get("name") or call.get("tool") or "")
|
|
2789
|
+
for step in steps
|
|
2790
|
+
for call in _probe_mappings(step.get("tool_calls"))
|
|
2791
|
+
if call.get("name") or call.get("tool")
|
|
2792
|
+
],
|
|
2793
|
+
]
|
|
2794
|
+
final_state_keys = trace.get("final_state_keys") or list(final_state.keys())
|
|
2795
|
+
|
|
2796
|
+
for key, values in (
|
|
2797
|
+
("node_names", node_names),
|
|
2798
|
+
("step_names", step_names),
|
|
2799
|
+
("checkpoint_ids", [item.get("id") or item.get("checkpoint_id") for item in checkpoints]),
|
|
2800
|
+
("route_targets", [item.get("target") or item.get("selected") for item in routes]),
|
|
2801
|
+
("interrupt_nodes", [item.get("node") or item.get("id") for item in interrupts]),
|
|
2802
|
+
("replay_ids", [item.get("id") or item.get("replay_id") for item in replay]),
|
|
2803
|
+
("tool_names", tool_names),
|
|
2804
|
+
("step_statuses", trace.get("step_statuses")),
|
|
2805
|
+
("final_state_keys", final_state_keys),
|
|
2806
|
+
):
|
|
2807
|
+
cleaned = sorted({str(item) for item in _probe_list(values) if str(item)})
|
|
2808
|
+
if cleaned:
|
|
2809
|
+
summary[key] = cleaned
|
|
2810
|
+
|
|
2811
|
+
for key in ("entry_nodes", "terminal_nodes"):
|
|
2812
|
+
values = sorted(str(item) for item in _probe_list(topology.get(key)) if str(item))
|
|
2813
|
+
if values:
|
|
2814
|
+
summary[key] = values
|
|
2815
|
+
for key in ("has_replay", "has_interrupts", "has_routes"):
|
|
2816
|
+
if trace.get(key) is not None:
|
|
2817
|
+
summary[key] = bool(trace.get(key))
|
|
2818
|
+
if topology:
|
|
2819
|
+
summary["has_topology"] = True
|
|
2820
|
+
if trace.get("framework"):
|
|
2821
|
+
summary["framework"] = str(trace.get("framework"))
|
|
2822
|
+
return summary
|
|
2823
|
+
|
|
2824
|
+
|
|
2825
|
+
def _probe_realtime_trace_summary(value: Any) -> dict[str, Any]:
|
|
2826
|
+
trace = dict(value or {}) if isinstance(value, Mapping) else {}
|
|
2827
|
+
summary = (
|
|
2828
|
+
dict(trace.get("summary") or {})
|
|
2829
|
+
if isinstance(trace.get("summary"), Mapping)
|
|
2830
|
+
else {}
|
|
2831
|
+
)
|
|
2832
|
+
for key in (
|
|
2833
|
+
"signals",
|
|
2834
|
+
"tool_names",
|
|
2835
|
+
"frame_types",
|
|
2836
|
+
"event_types",
|
|
2837
|
+
"categories",
|
|
2838
|
+
"directions",
|
|
2839
|
+
"modalities",
|
|
2840
|
+
):
|
|
2841
|
+
if trace.get(key):
|
|
2842
|
+
summary[key] = sorted(str(item) for item in trace.get(key, []) if str(item))
|
|
2843
|
+
for count_key, value_key in (
|
|
2844
|
+
("frame_count", "frames"),
|
|
2845
|
+
("event_count", "events"),
|
|
2846
|
+
):
|
|
2847
|
+
if count_key not in summary and isinstance(trace.get(value_key), list):
|
|
2848
|
+
summary[count_key] = len(trace.get(value_key, []))
|
|
2849
|
+
return summary
|
|
2850
|
+
|
|
2851
|
+
|
|
2852
|
+
def _probe_framework_memory_summary(value: Any) -> dict[str, Any]:
|
|
2853
|
+
trace = dict(value or {}) if isinstance(value, Mapping) else {}
|
|
2854
|
+
if not trace:
|
|
2855
|
+
return {}
|
|
2856
|
+
summary = (
|
|
2857
|
+
dict(trace.get("summary") or {})
|
|
2858
|
+
if isinstance(trace.get("summary"), Mapping)
|
|
2859
|
+
else {}
|
|
2860
|
+
)
|
|
2861
|
+
for count_key, value_key in (
|
|
2862
|
+
("operation_count", "operations"),
|
|
2863
|
+
("checkpoint_count", "checkpoints"),
|
|
2864
|
+
("memory_count", "memories"),
|
|
2865
|
+
("retrieval_count", "retrievals"),
|
|
2866
|
+
("store_count", "stores"),
|
|
2867
|
+
):
|
|
2868
|
+
if count_key not in summary and isinstance(trace.get(value_key), list):
|
|
2869
|
+
summary[count_key] = len(trace.get(value_key, []))
|
|
2870
|
+
elif count_key not in summary and trace.get(count_key) is not None:
|
|
2871
|
+
summary[count_key] = trace.get(count_key)
|
|
2872
|
+
policies = dict(trace.get("policies") or {}) if isinstance(trace.get("policies"), Mapping) else {}
|
|
2873
|
+
if "policy_count" not in summary:
|
|
2874
|
+
summary["policy_count"] = len(policies) or trace.get("policy_count", 0)
|
|
2875
|
+
for key in ("operation_types", "source_ids", "namespaces", "retrieval_doc_ids"):
|
|
2876
|
+
if trace.get(key):
|
|
2877
|
+
summary[key] = sorted(str(item) for item in trace.get(key, []) if str(item))
|
|
2878
|
+
policy_keys = trace.get("policy_keys") or list(policies.keys())
|
|
2879
|
+
if policy_keys:
|
|
2880
|
+
summary["policy_keys"] = sorted(str(item) for item in policy_keys if str(item))
|
|
2881
|
+
return summary
|
|
2882
|
+
|
|
2883
|
+
|
|
2884
|
+
def _probe_retrieval_memory_summary(value: Any) -> dict[str, Any]:
|
|
2885
|
+
trace = dict(value or {}) if isinstance(value, Mapping) else {}
|
|
2886
|
+
if not trace:
|
|
2887
|
+
return {}
|
|
2888
|
+
summary = (
|
|
2889
|
+
dict(trace.get("summary") or {})
|
|
2890
|
+
if isinstance(trace.get("summary"), Mapping)
|
|
2891
|
+
else {}
|
|
2892
|
+
)
|
|
2893
|
+
documents = [
|
|
2894
|
+
dict(item)
|
|
2895
|
+
for item in (trace.get("documents") or [])
|
|
2896
|
+
if isinstance(item, Mapping)
|
|
2897
|
+
]
|
|
2898
|
+
queries = [
|
|
2899
|
+
dict(item)
|
|
2900
|
+
for item in (trace.get("queries") or [])
|
|
2901
|
+
if isinstance(item, Mapping)
|
|
2902
|
+
]
|
|
2903
|
+
citations = [
|
|
2904
|
+
dict(item)
|
|
2905
|
+
for item in (trace.get("citations") or [])
|
|
2906
|
+
if isinstance(item, Mapping)
|
|
2907
|
+
]
|
|
2908
|
+
memory_writes = [
|
|
2909
|
+
dict(item)
|
|
2910
|
+
for item in (trace.get("memory_writes") or [])
|
|
2911
|
+
if isinstance(item, Mapping)
|
|
2912
|
+
]
|
|
2913
|
+
summary.setdefault("document_count", len(documents))
|
|
2914
|
+
summary.setdefault("query_count", len(queries))
|
|
2915
|
+
summary.setdefault("citation_count", len(citations))
|
|
2916
|
+
summary.setdefault("memory_write_count", len(memory_writes))
|
|
2917
|
+
summary.setdefault(
|
|
2918
|
+
"current_document_count",
|
|
2919
|
+
len([document for document in documents if document.get("current") is not False]),
|
|
2920
|
+
)
|
|
2921
|
+
doc_ids = sorted(
|
|
2922
|
+
{
|
|
2923
|
+
str(
|
|
2924
|
+
document.get("id")
|
|
2925
|
+
or document.get("doc_id")
|
|
2926
|
+
or document.get("source")
|
|
2927
|
+
or ""
|
|
2928
|
+
)
|
|
2929
|
+
for document in documents
|
|
2930
|
+
if document.get("id") or document.get("doc_id") or document.get("source")
|
|
2931
|
+
}
|
|
2932
|
+
)
|
|
2933
|
+
cited_doc_ids = sorted(
|
|
2934
|
+
{
|
|
2935
|
+
str(doc_id)
|
|
2936
|
+
for citation in citations
|
|
2937
|
+
for doc_id in _probe_list(citation.get("doc_ids"))
|
|
2938
|
+
if str(doc_id)
|
|
2939
|
+
}
|
|
2940
|
+
)
|
|
2941
|
+
if doc_ids:
|
|
2942
|
+
summary["document_ids"] = doc_ids
|
|
2943
|
+
if cited_doc_ids:
|
|
2944
|
+
summary["citation_doc_ids"] = cited_doc_ids
|
|
2945
|
+
if trace.get("require_current") is not None:
|
|
2946
|
+
summary["require_current"] = bool(trace.get("require_current"))
|
|
2947
|
+
return summary
|
|
2948
|
+
|
|
2949
|
+
|
|
2950
|
+
def _probe_agent_memory_lineage_summary(value: Any) -> dict[str, Any]:
|
|
2951
|
+
trace = dict(value or {}) if isinstance(value, Mapping) else {}
|
|
2952
|
+
if not trace:
|
|
2953
|
+
return {}
|
|
2954
|
+
summary = (
|
|
2955
|
+
dict(trace.get("summary") or {})
|
|
2956
|
+
if isinstance(trace.get("summary"), Mapping)
|
|
2957
|
+
else {}
|
|
2958
|
+
)
|
|
2959
|
+
stores = _probe_mappings(trace.get("stores"))
|
|
2960
|
+
memories = _probe_mappings(trace.get("memories"))
|
|
2961
|
+
operations = _probe_mappings(trace.get("operations"))
|
|
2962
|
+
lineage = _probe_mappings(trace.get("lineage"))
|
|
2963
|
+
policies = dict(trace.get("policies") or {}) if isinstance(trace.get("policies"), Mapping) else {}
|
|
2964
|
+
poison_tests = _probe_mappings(trace.get("poison_tests") or trace.get("poisoning_tests"))
|
|
2965
|
+
isolation_tests = _probe_mappings(trace.get("isolation_tests"))
|
|
2966
|
+
retention_tests = _probe_mappings(trace.get("retention_tests") or trace.get("deletion_tests"))
|
|
2967
|
+
observability = (
|
|
2968
|
+
dict(trace.get("observability"))
|
|
2969
|
+
if isinstance(trace.get("observability"), Mapping)
|
|
2970
|
+
else {}
|
|
2971
|
+
)
|
|
2972
|
+
artifacts = _probe_mappings(trace.get("artifacts"))
|
|
2973
|
+
operation_type_values = [
|
|
2974
|
+
operation_type
|
|
2975
|
+
for operation_type in (
|
|
2976
|
+
_probe_memory_key(
|
|
2977
|
+
operation.get("operation")
|
|
2978
|
+
or operation.get("type")
|
|
2979
|
+
or operation.get("op")
|
|
2980
|
+
)
|
|
2981
|
+
for operation in operations
|
|
2982
|
+
)
|
|
2983
|
+
if operation_type
|
|
2984
|
+
]
|
|
2985
|
+
operation_types = sorted(set(operation_type_values))
|
|
2986
|
+
policy_keys = sorted(_probe_memory_key(key) for key in policies if _probe_memory_key(key))
|
|
2987
|
+
attributed_memories = [
|
|
2988
|
+
memory
|
|
2989
|
+
for memory in memories
|
|
2990
|
+
if _probe_list(
|
|
2991
|
+
memory.get("source_ids")
|
|
2992
|
+
or memory.get("sources")
|
|
2993
|
+
or memory.get("doc_ids")
|
|
2994
|
+
)
|
|
2995
|
+
]
|
|
2996
|
+
unattributed_memories = [
|
|
2997
|
+
str(memory.get("id") or memory.get("key") or index)
|
|
2998
|
+
for index, memory in enumerate(memories, start=1)
|
|
2999
|
+
if not _probe_list(
|
|
3000
|
+
memory.get("source_ids")
|
|
3001
|
+
or memory.get("sources")
|
|
3002
|
+
or memory.get("doc_ids")
|
|
3003
|
+
)
|
|
3004
|
+
and str(memory.get("status") or "active").lower() not in {"deleted", "expired", "blocked"}
|
|
3005
|
+
]
|
|
3006
|
+
poison_good = {"passed", "blocked", "mitigated", "contained", "accepted"}
|
|
3007
|
+
isolation_good = {"passed", "blocked", "mitigated", "contained"}
|
|
3008
|
+
retention_good = {"passed", "deleted", "expired", "purged", "mitigated"}
|
|
3009
|
+
poisoning_failures = [
|
|
3010
|
+
test
|
|
3011
|
+
for test in poison_tests
|
|
3012
|
+
if _probe_memory_key(test.get("status")) not in poison_good
|
|
3013
|
+
]
|
|
3014
|
+
isolation_violations = [
|
|
3015
|
+
test
|
|
3016
|
+
for test in isolation_tests
|
|
3017
|
+
if _probe_memory_key(test.get("status")) not in isolation_good
|
|
3018
|
+
]
|
|
3019
|
+
retention_violations = [
|
|
3020
|
+
test
|
|
3021
|
+
for test in retention_tests
|
|
3022
|
+
if _probe_memory_key(test.get("status")) not in retention_good
|
|
3023
|
+
]
|
|
3024
|
+
policy_violations = [
|
|
3025
|
+
operation
|
|
3026
|
+
for operation in operations
|
|
3027
|
+
if _probe_memory_key(operation.get("status")) in {"policy_violation", "violation", "failed_policy"}
|
|
3028
|
+
or _probe_memory_key(operation.get("policy_decision")) in {"violation", "failed", "bypassed"}
|
|
3029
|
+
]
|
|
3030
|
+
summary.setdefault("store_count", len(stores))
|
|
3031
|
+
summary.setdefault("memory_count", len(memories))
|
|
3032
|
+
summary.setdefault("operation_count", len(operations))
|
|
3033
|
+
summary.setdefault("lineage_count", len(lineage))
|
|
3034
|
+
summary.setdefault("policy_count", len(policies))
|
|
3035
|
+
summary.setdefault("artifact_count", len(artifacts))
|
|
3036
|
+
summary.setdefault("observability_hook_count", _probe_observability_count(observability))
|
|
3037
|
+
summary.setdefault("attributed_memory_count", len(attributed_memories))
|
|
3038
|
+
summary.setdefault("unattributed_memory_count", len(unattributed_memories))
|
|
3039
|
+
summary.setdefault("poisoned_memory_count", 0)
|
|
3040
|
+
summary.setdefault("open_poisoning_count", len(poisoning_failures))
|
|
3041
|
+
summary.setdefault("isolation_violation_count", len(isolation_violations))
|
|
3042
|
+
summary.setdefault("retention_violation_count", len(retention_violations))
|
|
3043
|
+
summary.setdefault("policy_violation_count", len(policy_violations))
|
|
3044
|
+
for operation_type in ("read", "write", "update", "delete", "recall"):
|
|
3045
|
+
summary.setdefault(
|
|
3046
|
+
f"{operation_type}_operation_count",
|
|
3047
|
+
operation_type_values.count(operation_type),
|
|
3048
|
+
)
|
|
3049
|
+
summary.setdefault("has_target", bool(trace.get("target")))
|
|
3050
|
+
summary.setdefault("has_stores", bool(stores))
|
|
3051
|
+
summary.setdefault("has_memory_records", bool(memories))
|
|
3052
|
+
summary.setdefault("has_operations", bool(operations))
|
|
3053
|
+
summary.setdefault("has_lineage", bool(lineage))
|
|
3054
|
+
summary.setdefault("has_source_attribution", bool(attributed_memories) and not unattributed_memories)
|
|
3055
|
+
summary.setdefault(
|
|
3056
|
+
"has_tenant_isolation",
|
|
3057
|
+
"tenant_isolation" in policy_keys or bool(isolation_tests),
|
|
3058
|
+
)
|
|
3059
|
+
summary.setdefault("has_audit", "audit" in policy_keys)
|
|
3060
|
+
summary.setdefault(
|
|
3061
|
+
"has_retention_policy",
|
|
3062
|
+
any(key in policy_keys for key in ("retention", "retention_policy")),
|
|
3063
|
+
)
|
|
3064
|
+
summary.setdefault(
|
|
3065
|
+
"has_deletion_policy",
|
|
3066
|
+
any(key in policy_keys for key in ("deletion", "deletion_policy")),
|
|
3067
|
+
)
|
|
3068
|
+
summary.setdefault("has_redaction", "redaction" in policy_keys)
|
|
3069
|
+
summary.setdefault("has_canaries", "canary" in policy_keys or bool(poison_tests))
|
|
3070
|
+
summary.setdefault("has_observability", bool(observability))
|
|
3071
|
+
summary.setdefault("has_artifacts", bool(artifacts))
|
|
3072
|
+
if operation_types:
|
|
3073
|
+
summary["operation_types"] = operation_types
|
|
3074
|
+
if policy_keys:
|
|
3075
|
+
summary["policy_keys"] = policy_keys
|
|
3076
|
+
observed_evidence = {
|
|
3077
|
+
signal
|
|
3078
|
+
for flag, signal in (
|
|
3079
|
+
("has_target", "target"),
|
|
3080
|
+
("has_stores", "store"),
|
|
3081
|
+
("has_memory_records", "memory_record"),
|
|
3082
|
+
("has_operations", "operation"),
|
|
3083
|
+
("has_lineage", "lineage"),
|
|
3084
|
+
("has_source_attribution", "source_attribution"),
|
|
3085
|
+
("has_tenant_isolation", "tenant_isolation"),
|
|
3086
|
+
("has_audit", "audit"),
|
|
3087
|
+
("has_retention_policy", "retention_policy"),
|
|
3088
|
+
("has_deletion_policy", "deletion_policy"),
|
|
3089
|
+
("has_redaction", "redaction"),
|
|
3090
|
+
("has_canaries", "canary"),
|
|
3091
|
+
("has_observability", "observability"),
|
|
3092
|
+
("has_artifacts", "artifact"),
|
|
3093
|
+
)
|
|
3094
|
+
if summary.get(flag)
|
|
3095
|
+
}
|
|
3096
|
+
observed_evidence.update(f"{operation_type}_operation" for operation_type in operation_types)
|
|
3097
|
+
observed_signals = {
|
|
3098
|
+
*observed_evidence,
|
|
3099
|
+
*operation_types,
|
|
3100
|
+
*policy_keys,
|
|
3101
|
+
"agent_memory_lineage",
|
|
3102
|
+
"memory_lineage",
|
|
3103
|
+
"memory_provenance",
|
|
3104
|
+
"memory",
|
|
3105
|
+
"provenance",
|
|
3106
|
+
}
|
|
3107
|
+
summary.setdefault("observed_evidence", sorted(observed_evidence))
|
|
3108
|
+
summary.setdefault("observed_signals", sorted(observed_signals))
|
|
3109
|
+
summary.setdefault("blocking_gap_count", 0)
|
|
3110
|
+
return summary
|
|
3111
|
+
|
|
3112
|
+
|
|
3113
|
+
def _probe_browser_cua_summary(value: Any) -> dict[str, Any]:
|
|
3114
|
+
trace = dict(value or {}) if isinstance(value, Mapping) else {}
|
|
3115
|
+
if not trace:
|
|
3116
|
+
return {}
|
|
3117
|
+
summary = (
|
|
3118
|
+
dict(trace.get("summary") or {})
|
|
3119
|
+
if isinstance(trace.get("summary"), Mapping)
|
|
3120
|
+
else {}
|
|
3121
|
+
)
|
|
3122
|
+
snapshots = _probe_mappings(trace.get("snapshots"))
|
|
3123
|
+
actions = _probe_mappings(trace.get("action_replay") or trace.get("actions"))
|
|
3124
|
+
screenshots = _probe_mappings(trace.get("screenshots"))
|
|
3125
|
+
regions = dict(trace.get("regions") or {}) if isinstance(trace.get("regions"), Mapping) else {}
|
|
3126
|
+
network_log = _probe_mappings(trace.get("network_log"))
|
|
3127
|
+
runtime_events = _probe_mappings(trace.get("runtime_events"))
|
|
3128
|
+
performance_entries = _probe_mappings(trace.get("performance_entries"))
|
|
3129
|
+
prompt_injections = _probe_mappings(trace.get("prompt_injections"))
|
|
3130
|
+
mutation_pack = (
|
|
3131
|
+
dict(trace.get("mutation_pack"))
|
|
3132
|
+
if isinstance(trace.get("mutation_pack"), Mapping)
|
|
3133
|
+
else {}
|
|
3134
|
+
)
|
|
3135
|
+
mutations = _probe_mappings(
|
|
3136
|
+
mutation_pack.get("mutations")
|
|
3137
|
+
or trace.get("browser_mutations")
|
|
3138
|
+
or trace.get("mutations")
|
|
3139
|
+
)
|
|
3140
|
+
for count_key, count in (
|
|
3141
|
+
("snapshot_count", len(snapshots)),
|
|
3142
|
+
("action_count", len(actions)),
|
|
3143
|
+
("screenshot_count", len(screenshots)),
|
|
3144
|
+
("region_count", len(regions)),
|
|
3145
|
+
("network_request_count", len(network_log)),
|
|
3146
|
+
("runtime_event_count", len(runtime_events)),
|
|
3147
|
+
("performance_entry_count", len(performance_entries)),
|
|
3148
|
+
("prompt_injection_surface_count", len(prompt_injections)),
|
|
3149
|
+
("mutation_count", len(mutations)),
|
|
3150
|
+
):
|
|
3151
|
+
summary.setdefault(count_key, trace.get(count_key, count))
|
|
3152
|
+
for count_key in (
|
|
3153
|
+
"successful_action_count",
|
|
3154
|
+
"blocked_action_count",
|
|
3155
|
+
"matched_action_count",
|
|
3156
|
+
"prompt_injection_touched_count",
|
|
3157
|
+
"screenshot_diff_count",
|
|
3158
|
+
):
|
|
3159
|
+
if trace.get(count_key) is not None:
|
|
3160
|
+
summary[count_key] = trace.get(count_key)
|
|
3161
|
+
summary.setdefault(
|
|
3162
|
+
"stale_action_count",
|
|
3163
|
+
len([action for action in actions if action.get("stale_screenshot")]),
|
|
3164
|
+
)
|
|
3165
|
+
summary.setdefault(
|
|
3166
|
+
"dom_snapshot_count",
|
|
3167
|
+
len([snapshot for snapshot in snapshots if snapshot.get("dom")]),
|
|
3168
|
+
)
|
|
3169
|
+
summary.setdefault(
|
|
3170
|
+
"screenshot_snapshot_count",
|
|
3171
|
+
len(
|
|
3172
|
+
[
|
|
3173
|
+
snapshot
|
|
3174
|
+
for snapshot in snapshots
|
|
3175
|
+
if snapshot.get("screenshot_uri") or snapshot.get("screenshot_path")
|
|
3176
|
+
]
|
|
3177
|
+
),
|
|
3178
|
+
)
|
|
3179
|
+
summary["layout_shift_present"] = bool(
|
|
3180
|
+
trace.get("layout_shift_present")
|
|
3181
|
+
or trace.get("layout_shift_distribution")
|
|
3182
|
+
)
|
|
3183
|
+
summary["storage_present"] = bool(trace.get("storage_present"))
|
|
3184
|
+
action_types = trace.get("action_types") or [
|
|
3185
|
+
action.get("action") or action.get("type")
|
|
3186
|
+
for action in actions
|
|
3187
|
+
if action.get("action") or action.get("type")
|
|
3188
|
+
]
|
|
3189
|
+
tool_names = trace.get("tool_names") or [
|
|
3190
|
+
action.get("tool") or action.get("tool_name")
|
|
3191
|
+
for action in actions
|
|
3192
|
+
if action.get("tool") or action.get("tool_name")
|
|
3193
|
+
]
|
|
3194
|
+
mutation_ids = [
|
|
3195
|
+
mutation.get("id") or mutation.get("name")
|
|
3196
|
+
for mutation in mutations
|
|
3197
|
+
if mutation.get("id") or mutation.get("name")
|
|
3198
|
+
]
|
|
3199
|
+
mutation_types = [
|
|
3200
|
+
mutation.get("type") or mutation.get("kind")
|
|
3201
|
+
for mutation in mutations
|
|
3202
|
+
if mutation.get("type") or mutation.get("kind")
|
|
3203
|
+
]
|
|
3204
|
+
region_ids = [
|
|
3205
|
+
*list(regions.keys()),
|
|
3206
|
+
*[
|
|
3207
|
+
region.get("id") or region.get("name")
|
|
3208
|
+
for region in regions.values()
|
|
3209
|
+
if isinstance(region, Mapping) and (region.get("id") or region.get("name"))
|
|
3210
|
+
],
|
|
3211
|
+
]
|
|
3212
|
+
prompt_injection_ids = [
|
|
3213
|
+
injection.get("id") or injection.get("selector")
|
|
3214
|
+
for injection in prompt_injections
|
|
3215
|
+
if injection.get("id") or injection.get("selector")
|
|
3216
|
+
]
|
|
3217
|
+
summary["action_types"] = sorted(str(item) for item in action_types if str(item))
|
|
3218
|
+
summary["tool_names"] = sorted(str(item) for item in tool_names if str(item))
|
|
3219
|
+
summary["mutation_ids"] = sorted(str(item) for item in mutation_ids if str(item))
|
|
3220
|
+
summary["mutation_types"] = sorted(str(item) for item in mutation_types if str(item))
|
|
3221
|
+
summary["region_ids"] = sorted(str(item) for item in region_ids if str(item))
|
|
3222
|
+
summary["prompt_injection_ids"] = sorted(
|
|
3223
|
+
str(item) for item in prompt_injection_ids if str(item)
|
|
3224
|
+
)
|
|
3225
|
+
layout_distribution = (
|
|
3226
|
+
dict(trace.get("layout_shift_distribution"))
|
|
3227
|
+
if isinstance(trace.get("layout_shift_distribution"), Mapping)
|
|
3228
|
+
else {}
|
|
3229
|
+
)
|
|
3230
|
+
layout_values = [
|
|
3231
|
+
_probe_float(layout_distribution.get(key))
|
|
3232
|
+
for key in ("max", "p95", "score", "value")
|
|
3233
|
+
]
|
|
3234
|
+
layout_values = [value for value in layout_values if value is not None]
|
|
3235
|
+
if layout_values:
|
|
3236
|
+
summary["max_layout_shift_score"] = max(layout_values)
|
|
3237
|
+
performance_durations = [
|
|
3238
|
+
_probe_float(entry.get("duration_ms") or entry.get("duration"))
|
|
3239
|
+
for entry in performance_entries
|
|
3240
|
+
]
|
|
3241
|
+
performance_durations = [value for value in performance_durations if value is not None]
|
|
3242
|
+
if performance_durations:
|
|
3243
|
+
summary["max_performance_duration_ms"] = max(performance_durations)
|
|
3244
|
+
compact_actions = []
|
|
3245
|
+
for action in actions[:5]:
|
|
3246
|
+
compact_action = {
|
|
3247
|
+
key: action.get(key)
|
|
3248
|
+
for key in (
|
|
3249
|
+
"id",
|
|
3250
|
+
"tool",
|
|
3251
|
+
"tool_name",
|
|
3252
|
+
"action",
|
|
3253
|
+
"selector",
|
|
3254
|
+
"success",
|
|
3255
|
+
"matched",
|
|
3256
|
+
"blocked",
|
|
3257
|
+
"mutation_id",
|
|
3258
|
+
"mutation_type",
|
|
3259
|
+
)
|
|
3260
|
+
if action.get(key) not in (None, "", [], {})
|
|
3261
|
+
}
|
|
3262
|
+
region = action.get("region")
|
|
3263
|
+
if isinstance(region, Mapping):
|
|
3264
|
+
compact_action["region"] = {
|
|
3265
|
+
key: region.get(key)
|
|
3266
|
+
for key in ("id", "name", "selector", "x", "y", "width", "height")
|
|
3267
|
+
if region.get(key) not in (None, "", [], {})
|
|
3268
|
+
}
|
|
3269
|
+
if compact_action:
|
|
3270
|
+
compact_actions.append(compact_action)
|
|
3271
|
+
if compact_actions:
|
|
3272
|
+
summary["actions"] = compact_actions
|
|
3273
|
+
if runtime_events:
|
|
3274
|
+
summary["runtime_events"] = [
|
|
3275
|
+
{
|
|
3276
|
+
key: event.get(key)
|
|
3277
|
+
for key in ("id", "name", "type", "level", "message")
|
|
3278
|
+
if event.get(key) not in (None, "", [], {})
|
|
3279
|
+
}
|
|
3280
|
+
for event in runtime_events[:5]
|
|
3281
|
+
]
|
|
3282
|
+
return summary
|
|
3283
|
+
|
|
3284
|
+
|
|
3285
|
+
def _probe_mcp_tool_session_summary(value: Any) -> dict[str, Any]:
|
|
3286
|
+
trace = dict(value or {}) if isinstance(value, Mapping) else {}
|
|
3287
|
+
summary = trace.get("summary")
|
|
3288
|
+
return dict(summary) if isinstance(summary, Mapping) else {}
|
|
3289
|
+
|
|
3290
|
+
|
|
3291
|
+
def _probe_a2a_protocol_summary(value: Any) -> dict[str, Any]:
|
|
3292
|
+
trace = dict(value or {}) if isinstance(value, Mapping) else {}
|
|
3293
|
+
summary = trace.get("summary")
|
|
3294
|
+
return dict(summary) if isinstance(summary, Mapping) else {}
|
|
3295
|
+
|
|
3296
|
+
|
|
3297
|
+
def _probe_openenv_summary(value: Any) -> dict[str, Any]:
|
|
3298
|
+
trace = dict(value or {}) if isinstance(value, Mapping) else {}
|
|
3299
|
+
if not trace:
|
|
3300
|
+
return {}
|
|
3301
|
+
summary = (
|
|
3302
|
+
dict(trace.get("summary") or {})
|
|
3303
|
+
if isinstance(trace.get("summary"), Mapping)
|
|
3304
|
+
else {}
|
|
3305
|
+
)
|
|
3306
|
+
trajectory = _probe_mappings(trace.get("trajectory") or trace.get("steps"))
|
|
3307
|
+
action_log = _probe_mappings(trace.get("action_log") or trace.get("actions"))
|
|
3308
|
+
error_log = _probe_mappings(trace.get("error_log") or trace.get("errors"))
|
|
3309
|
+
sandbox = (
|
|
3310
|
+
dict(trace.get("sandbox"))
|
|
3311
|
+
if isinstance(trace.get("sandbox"), Mapping)
|
|
3312
|
+
else {}
|
|
3313
|
+
)
|
|
3314
|
+
for count_key, fallback in (
|
|
3315
|
+
("reset_count", 1 if trace.get("initial_observation") is not None else 0),
|
|
3316
|
+
("step_count", len(trajectory)),
|
|
3317
|
+
("action_route_count", len(action_log) or len(trajectory)),
|
|
3318
|
+
(
|
|
3319
|
+
"failure_count",
|
|
3320
|
+
max(
|
|
3321
|
+
len(_probe_mappings(trace.get("failure_injections") or trace.get("faults"))),
|
|
3322
|
+
sum(
|
|
3323
|
+
1
|
|
3324
|
+
for step in trajectory
|
|
3325
|
+
if step.get("failure_injected") or step.get("failure")
|
|
3326
|
+
),
|
|
3327
|
+
),
|
|
3328
|
+
),
|
|
3329
|
+
(
|
|
3330
|
+
"metadata_capture_count",
|
|
3331
|
+
sum(
|
|
3332
|
+
1
|
|
3333
|
+
for step in trajectory
|
|
3334
|
+
if step.get("metadata") not in (None, "", [], {})
|
|
3335
|
+
or step.get("info") not in (None, "", [], {})
|
|
3336
|
+
)
|
|
3337
|
+
+ (1 if trace.get("reset_info") else 0),
|
|
3338
|
+
),
|
|
3339
|
+
("error_count", len(error_log)),
|
|
3340
|
+
):
|
|
3341
|
+
if summary.get(count_key) in (None, "", [], {}):
|
|
3342
|
+
summary[count_key] = fallback
|
|
3343
|
+
if summary.get("reward_total") in (None, "", [], {}):
|
|
3344
|
+
summary["reward_total"] = round(
|
|
3345
|
+
sum(_probe_float(step.get("reward")) or 0.0 for step in trajectory),
|
|
3346
|
+
4,
|
|
3347
|
+
)
|
|
3348
|
+
for key in ("done", "terminated", "truncated"):
|
|
3349
|
+
if summary.get(key) in (None, "", [], {}):
|
|
3350
|
+
summary[key] = any(bool(step.get(key)) for step in trajectory)
|
|
3351
|
+
if summary.get("sandbox_enabled") in (None, "", [], {}):
|
|
3352
|
+
summary["sandbox_enabled"] = bool(sandbox.get("enabled", bool(sandbox)))
|
|
3353
|
+
if summary.get("isolation") in (None, "", [], {}):
|
|
3354
|
+
summary["isolation"] = str(sandbox.get("isolation") or "process")
|
|
3355
|
+
for key in (
|
|
3356
|
+
"runtime",
|
|
3357
|
+
"transport",
|
|
3358
|
+
"requires_external_service",
|
|
3359
|
+
"deterministic_reset",
|
|
3360
|
+
):
|
|
3361
|
+
if summary.get(key) in (None, "", [], {}) and trace.get(key) not in (
|
|
3362
|
+
None,
|
|
3363
|
+
"",
|
|
3364
|
+
[],
|
|
3365
|
+
{},
|
|
3366
|
+
):
|
|
3367
|
+
summary[key] = trace.get(key)
|
|
3368
|
+
signals = trace.get("signals")
|
|
3369
|
+
if signals and summary.get("signals") in (None, "", [], {}):
|
|
3370
|
+
summary["signals"] = sorted(str(signal) for signal in signals if str(signal))
|
|
3371
|
+
return summary
|
|
3372
|
+
|
|
3373
|
+
|
|
3374
|
+
def _probe_artifact_evidence(artifact: Mapping[str, Any]) -> dict[str, Any]:
|
|
3375
|
+
metadata = artifact.get("metadata")
|
|
3376
|
+
return {
|
|
3377
|
+
"type": str(artifact.get("type") or ""),
|
|
3378
|
+
"uri": str(artifact.get("uri") or ""),
|
|
3379
|
+
"path": str(artifact.get("path") or ""),
|
|
3380
|
+
"mime_type": str(artifact.get("mime_type") or ""),
|
|
3381
|
+
"role": str(artifact.get("role") or ""),
|
|
3382
|
+
"metadata": dict(metadata) if isinstance(metadata, Mapping) else {},
|
|
3383
|
+
}
|
|
3384
|
+
|
|
3385
|
+
|
|
3386
|
+
def _probe_list(value: Any) -> list[Any]:
|
|
3387
|
+
if value is None:
|
|
3388
|
+
return []
|
|
3389
|
+
if isinstance(value, list):
|
|
3390
|
+
return value
|
|
3391
|
+
if isinstance(value, tuple):
|
|
3392
|
+
return list(value)
|
|
3393
|
+
return [value]
|
|
3394
|
+
|
|
3395
|
+
|
|
3396
|
+
def _probe_mappings(value: Any) -> list[dict[str, Any]]:
|
|
3397
|
+
return [dict(item) for item in _probe_list(value) if isinstance(item, Mapping)]
|
|
3398
|
+
|
|
3399
|
+
|
|
3400
|
+
def _probe_memory_key(value: Any) -> str:
|
|
3401
|
+
return str(value or "").strip().lower().replace("-", "_").replace(" ", "_")
|
|
3402
|
+
|
|
3403
|
+
|
|
3404
|
+
def _probe_observability_count(observability: Mapping[str, Any]) -> int:
|
|
3405
|
+
count = 0
|
|
3406
|
+
for value in observability.values():
|
|
3407
|
+
if isinstance(value, Mapping):
|
|
3408
|
+
count += len(value)
|
|
3409
|
+
elif isinstance(value, (list, tuple, set)):
|
|
3410
|
+
count += len([item for item in value if item])
|
|
3411
|
+
elif value:
|
|
3412
|
+
count += 1
|
|
3413
|
+
return count
|
|
3414
|
+
|
|
3415
|
+
|
|
3416
|
+
def _probe_float(value: Any) -> float | None:
|
|
3417
|
+
if isinstance(value, bool):
|
|
3418
|
+
return None
|
|
3419
|
+
if isinstance(value, (int, float)):
|
|
3420
|
+
return float(value)
|
|
3421
|
+
if isinstance(value, str):
|
|
3422
|
+
try:
|
|
3423
|
+
return float(value.strip())
|
|
3424
|
+
except ValueError:
|
|
3425
|
+
return None
|
|
3426
|
+
return None
|
|
3427
|
+
|
|
3428
|
+
|
|
3429
|
+
def _probe_summary(
|
|
3430
|
+
cases: Sequence[Mapping[str, Any]],
|
|
3431
|
+
contract: Mapping[str, Any],
|
|
3432
|
+
) -> dict[str, Any]:
|
|
3433
|
+
passed = sum(1 for case in cases if case.get("passed"))
|
|
3434
|
+
failed = len(cases) - passed
|
|
3435
|
+
response_tool_count = sum(
|
|
3436
|
+
int(dict(case.get("response") or {}).get("tool_call_count") or 0)
|
|
3437
|
+
for case in cases
|
|
3438
|
+
)
|
|
3439
|
+
runtime_trace_count = sum(1 for case in cases if case.get("runtime_trace"))
|
|
3440
|
+
streaming_trace_count = sum(
|
|
3441
|
+
1 for case in cases if dict(case.get("response") or {}).get("streaming")
|
|
3442
|
+
)
|
|
3443
|
+
observed_io_contracts = [
|
|
3444
|
+
dict(case.get("observed_io_contract") or {})
|
|
3445
|
+
for case in cases
|
|
3446
|
+
if isinstance(case.get("observed_io_contract"), Mapping)
|
|
3447
|
+
]
|
|
3448
|
+
observed_io_contract_count = sum(
|
|
3449
|
+
1
|
|
3450
|
+
for item in observed_io_contracts
|
|
3451
|
+
if item.get("kind")
|
|
3452
|
+
== "agent-learning.framework-adapter-observed-io-contract.v1"
|
|
3453
|
+
and int(dict(item.get("summary") or {}).get("invocation_count") or 0) >= 1
|
|
3454
|
+
)
|
|
3455
|
+
signature_bound_count = sum(
|
|
3456
|
+
1
|
|
3457
|
+
for item in observed_io_contracts
|
|
3458
|
+
if dict(item.get("summary") or {}).get("signature_bound") is True
|
|
3459
|
+
)
|
|
3460
|
+
call_contract_count = sum(
|
|
3461
|
+
1
|
|
3462
|
+
for case in cases
|
|
3463
|
+
for invocation in dict(case.get("runtime_trace") or {}).get("invocations", [])
|
|
3464
|
+
if isinstance(invocation, Mapping)
|
|
3465
|
+
and dict(invocation.get("call_contract") or {}).get("kind")
|
|
3466
|
+
== "agent-learning.framework-adapter-call-contract.v1"
|
|
3467
|
+
)
|
|
3468
|
+
input_keys = sorted(
|
|
3469
|
+
{
|
|
3470
|
+
str(invocation.get("input_key"))
|
|
3471
|
+
for case in cases
|
|
3472
|
+
for invocation in dict(case.get("runtime_trace") or {}).get("invocations", [])
|
|
3473
|
+
if isinstance(invocation, Mapping)
|
|
3474
|
+
and invocation.get("input_key") not in (None, "", [], {})
|
|
3475
|
+
}
|
|
3476
|
+
)
|
|
3477
|
+
call_styles = sorted(
|
|
3478
|
+
{
|
|
3479
|
+
str(invocation.get("call_style"))
|
|
3480
|
+
for case in cases
|
|
3481
|
+
for invocation in dict(case.get("runtime_trace") or {}).get("invocations", [])
|
|
3482
|
+
if isinstance(invocation, Mapping)
|
|
3483
|
+
and invocation.get("call_style") not in (None, "", [], {})
|
|
3484
|
+
}
|
|
3485
|
+
)
|
|
3486
|
+
input_kwargs_keys = sorted(
|
|
3487
|
+
{
|
|
3488
|
+
str(key)
|
|
3489
|
+
for case in cases
|
|
3490
|
+
for invocation in dict(case.get("runtime_trace") or {}).get("invocations", [])
|
|
3491
|
+
if isinstance(invocation, Mapping)
|
|
3492
|
+
for key in invocation.get("input_kwargs_keys", [])
|
|
3493
|
+
if key not in (None, "", [], {})
|
|
3494
|
+
}
|
|
3495
|
+
)
|
|
3496
|
+
input_types = _unique_probe_strings(
|
|
3497
|
+
input_type
|
|
3498
|
+
for item in observed_io_contracts
|
|
3499
|
+
for input_type in list(dict(item.get("summary") or {}).get("input_types") or [])
|
|
3500
|
+
)
|
|
3501
|
+
output_types = _unique_probe_strings(
|
|
3502
|
+
output_type
|
|
3503
|
+
for item in observed_io_contracts
|
|
3504
|
+
for output_type in list(dict(item.get("summary") or {}).get("output_types") or [])
|
|
3505
|
+
)
|
|
3506
|
+
return {
|
|
3507
|
+
"case_count": len(cases),
|
|
3508
|
+
"passed_case_count": passed,
|
|
3509
|
+
"failed_case_count": failed,
|
|
3510
|
+
"runtime_trace_count": runtime_trace_count,
|
|
3511
|
+
"streaming_trace_count": streaming_trace_count,
|
|
3512
|
+
"call_contract_count": call_contract_count,
|
|
3513
|
+
"callable_signature_present": bool(contract.get("callable_signature")),
|
|
3514
|
+
"observed_io_contract_count": observed_io_contract_count,
|
|
3515
|
+
"signature_bound_count": signature_bound_count,
|
|
3516
|
+
"tool_call_count": response_tool_count,
|
|
3517
|
+
"framework": contract.get("framework"),
|
|
3518
|
+
"method": contract.get("method"),
|
|
3519
|
+
"input_mode": contract.get("input_mode"),
|
|
3520
|
+
"input_keys": input_keys,
|
|
3521
|
+
"input_types": input_types,
|
|
3522
|
+
"output_types": output_types,
|
|
3523
|
+
"call_styles": call_styles,
|
|
3524
|
+
"input_kwargs_keys": input_kwargs_keys,
|
|
3525
|
+
"local_executable_fixture": bool(contract.get("local_executable_fixture")),
|
|
3526
|
+
"requires_external_service": bool(contract.get("requires_external_service")),
|
|
3527
|
+
"trace_runtime": bool(contract.get("trace_runtime")),
|
|
3528
|
+
}
|