agent-learning-kit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent_learning_kit-0.1.0.dist-info/METADATA +381 -0
- agent_learning_kit-0.1.0.dist-info/RECORD +642 -0
- agent_learning_kit-0.1.0.dist-info/WHEEL +4 -0
- agent_learning_kit-0.1.0.dist-info/entry_points.txt +5 -0
- agent_learning_kit-0.1.0.dist-info/licenses/LICENSE +173 -0
- agent_learning_kit-0.1.0.dist-info/licenses/NOTICE +7 -0
- fi/__init__.py +5 -0
- fi/alk/__init__.py +57 -0
- fi/alk/_facade.py +31 -0
- fi/alk/_module_alias.py +68 -0
- fi/alk/_paths.py +14 -0
- fi/alk/_schema.py +522 -0
- fi/alk/actions.py +727 -0
- fi/alk/bench/__init__.py +517 -0
- fi/alk/bench/_codeexec.py +213 -0
- fi/alk/bench/_coding.py +215 -0
- fi/alk/bench/_docker.py +237 -0
- fi/alk/bench/_grader.py +286 -0
- fi/alk/bench/_pull.py +212 -0
- fi/alk/bench/_voice.py +147 -0
- fi/alk/capabilities.py +627 -0
- fi/alk/cli.py +6396 -0
- fi/alk/config.py +130 -0
- fi/alk/cua_loop.py +562 -0
- fi/alk/evals.py +2351 -0
- fi/alk/extensions.py +163 -0
- fi/alk/harness/ARCHITECTURE.md +231 -0
- fi/alk/harness/DESIGN.md +246 -0
- fi/alk/harness/ENVIRONMENT_CONFORMANCE.md +127 -0
- fi/alk/harness/HOW-IT-WORKS.md +297 -0
- fi/alk/harness/IMPLEMENTATION_AND_VALIDATION_STATUS.md +229 -0
- fi/alk/harness/README.md +417 -0
- fi/alk/harness/__init__.py +77 -0
- fi/alk/harness/__main__.py +3 -0
- fi/alk/harness/amend.py +312 -0
- fi/alk/harness/artifacts.py +319 -0
- fi/alk/harness/authoring_entrypoint.py +189 -0
- fi/alk/harness/authoring_runtime_validation.py +267 -0
- fi/alk/harness/backends/README.md +43 -0
- fi/alk/harness/backends/__init__.py +122 -0
- fi/alk/harness/backends/base.py +241 -0
- fi/alk/harness/backends/claude.py +211 -0
- fi/alk/harness/backends/files.py +182 -0
- fi/alk/harness/backends/vertex_gemini.py +457 -0
- fi/alk/harness/background_noise.py +95 -0
- fi/alk/harness/build.py +385 -0
- fi/alk/harness/bundle.py +593 -0
- fi/alk/harness/bundle_author_v2.py +1831 -0
- fi/alk/harness/bundle_v2.py +719 -0
- fi/alk/harness/call_runner.py +1440 -0
- fi/alk/harness/callback_http_adapter.py +111 -0
- fi/alk/harness/catalogue.py +287 -0
- fi/alk/harness/chat.py +428 -0
- fi/alk/harness/chat_call_runner.py +506 -0
- fi/alk/harness/checks.py +136 -0
- fi/alk/harness/cli.py +1354 -0
- fi/alk/harness/config.py +338 -0
- fi/alk/harness/contract.py +718 -0
- fi/alk/harness/credentials.py +674 -0
- fi/alk/harness/data/persona_vocabulary.json +111 -0
- fi/alk/harness/environment.py +99 -0
- fi/alk/harness/environment_plan.py +168 -0
- fi/alk/harness/events.py +125 -0
- fi/alk/harness/executor.py +304 -0
- fi/alk/harness/folder.py +234 -0
- fi/alk/harness/generated_runtime.py +815 -0
- fi/alk/harness/github.py +72 -0
- fi/alk/harness/hosted_authoring_entrypoint.py +183 -0
- fi/alk/harness/hosted_entrypoint.py +2402 -0
- fi/alk/harness/hosted_scheduler.py +2218 -0
- fi/alk/harness/job.py +426 -0
- fi/alk/harness/judge.py +184 -0
- fi/alk/harness/livekit_source.py +50 -0
- fi/alk/harness/livekit_tool_trace_bootstrap.py +71 -0
- fi/alk/harness/observability.py +208 -0
- fi/alk/harness/outbound.py +3252 -0
- fi/alk/harness/packaging.py +515 -0
- fi/alk/harness/persona_guides.py +157 -0
- fi/alk/harness/platform.py +692 -0
- fi/alk/harness/process_preflight.py +764 -0
- fi/alk/harness/process_runtime.py +5670 -0
- fi/alk/harness/prove.py +425 -0
- fi/alk/harness/provider_import.py +703 -0
- fi/alk/harness/provider_lifecycle.py +392 -0
- fi/alk/harness/provision.py +2896 -0
- fi/alk/harness/reception.py +147 -0
- fi/alk/harness/retell_chat_call_runner.py +373 -0
- fi/alk/harness/run/__init__.py +296 -0
- fi/alk/harness/run/alk.py +184 -0
- fi/alk/harness/run/call.py +162 -0
- fi/alk/harness/run/conversation.py +264 -0
- fi/alk/harness/run/data/voices_by_language_and_gender.json +693 -0
- fi/alk/harness/run/evidence.py +195 -0
- fi/alk/harness/run/grade.py +598 -0
- fi/alk/harness/run/live.py +297 -0
- fi/alk/harness/run/models.py +56 -0
- fi/alk/harness/run/platform_evals.py +227 -0
- fi/alk/harness/run/sdk_voice.py +130 -0
- fi/alk/harness/run/simulation.py +1209 -0
- fi/alk/harness/run/stage.py +91 -0
- fi/alk/harness/run/targets.py +508 -0
- fi/alk/harness/run/tools.py +601 -0
- fi/alk/harness/run/voice.py +340 -0
- fi/alk/harness/runtime.py +172 -0
- fi/alk/harness/sandbox_server.py +2011 -0
- fi/alk/harness/sandbox_worker.py +44 -0
- fi/alk/harness/scenario.py +1048 -0
- fi/alk/harness/scenario_source.py +879 -0
- fi/alk/harness/scenario_tools.py +1143 -0
- fi/alk/harness/scenarios.py +915 -0
- fi/alk/harness/secrets.py +168 -0
- fi/alk/harness/service_catalog.py +97 -0
- fi/alk/harness/session.py +391 -0
- fi/alk/harness/sessions.py +372 -0
- fi/alk/harness/simulator.py +76 -0
- fi/alk/harness/simulator_voice.py +928 -0
- fi/alk/harness/skills/build-environment/SKILL.md +538 -0
- fi/alk/harness/skills/harness.md +131 -0
- fi/alk/harness/skills/kinds/chat.md +48 -0
- fi/alk/harness/skills/kinds/voice-voicemail.md +63 -0
- fi/alk/harness/skills/kinds/voice.md +59 -0
- fi/alk/harness/skills/plan-suite/SKILL.md +103 -0
- fi/alk/harness/skills/provision-environment/SKILL.md +136 -0
- fi/alk/harness/skills/run-scenarios/SKILL.md +112 -0
- fi/alk/harness/skills/understand-agent/SKILL.md +251 -0
- fi/alk/harness/skills/write-scenarios/SKILL.md +606 -0
- fi/alk/harness/skills/write-scenarios/references/refusals.md +28 -0
- fi/alk/harness/skills/write-scenarios/references/world-api.md +92 -0
- fi/alk/harness/source_data_invariants.py +444 -0
- fi/alk/harness/source_tool_evidence.py +79 -0
- fi/alk/harness/sources.py +253 -0
- fi/alk/harness/spend.py +140 -0
- fi/alk/harness/tool_trace_proxy.py +104 -0
- fi/alk/harness/tools.py +1018 -0
- fi/alk/harness/understand.py +169 -0
- fi/alk/harness/voicemail_audio.py +74 -0
- fi/alk/harness/world/__init__.py +33 -0
- fi/alk/harness/world/errors.py +68 -0
- fi/alk/harness/world/expectations.py +91 -0
- fi/alk/harness/world/handle.py +538 -0
- fi/alk/harness/world/kinds.py +196 -0
- fi/alk/harness/world/mutate.py +186 -0
- fi/alk/harness/world/probe.py +413 -0
- fi/alk/harness/world/provision.py +511 -0
- fi/alk/harness/world/provisioned.py +191 -0
- fi/alk/harness/world/runtime.py +616 -0
- fi/alk/harness/world/snapshot.py +288 -0
- fi/alk/harness/world/stores/__init__.py +305 -0
- fi/alk/harness/world/stores/container.py +215 -0
- fi/alk/harness/world/stores/inprocess.py +346 -0
- fi/alk/harness/world/stores/postgres.py +481 -0
- fi/alk/harness/world/stores/prove.py +202 -0
- fi/alk/harness/world/stores/sqlite.py +245 -0
- fi/alk/harness/world/stores/written.py +182 -0
- fi/alk/harness/world/tools.py +1516 -0
- fi/alk/harness/world/workspace.py +144 -0
- fi/alk/image_loop.py +453 -0
- fi/alk/image_perturb.py +241 -0
- fi/alk/improve.py +274 -0
- fi/alk/live/__init__.py +154 -0
- fi/alk/live/_attribution.py +184 -0
- fi/alk/live/_capture.py +264 -0
- fi/alk/live/_codec.py +391 -0
- fi/alk/live/_contract.py +134 -0
- fi/alk/live/_loopback.py +316 -0
- fi/alk/live/_perturb.py +449 -0
- fi/alk/live/_runner.py +386 -0
- fi/alk/live/_stats.py +561 -0
- fi/alk/live/_transcript.py +240 -0
- fi/alk/live/_workers/__init__.py +9 -0
- fi/alk/live/_workers/a2a_worker.py +316 -0
- fi/alk/live/_workers/langgraph_worker.py +217 -0
- fi/alk/live/_workers/livekit_worker.py +207 -0
- fi/alk/live/_workers/mcp_loopback_server.py +46 -0
- fi/alk/live/_workers/mcp_worker.py +158 -0
- fi/alk/live/_workers/pipecat_worker.py +189 -0
- fi/alk/live/a2a_lane.py +138 -0
- fi/alk/live/langgraph_lane.py +339 -0
- fi/alk/live/livekit_lane.py +376 -0
- fi/alk/live/mcp_lane.py +172 -0
- fi/alk/live/pipecat_lane.py +341 -0
- fi/alk/live/voice_redteam.py +494 -0
- fi/alk/loss.py +306 -0
- fi/alk/optimize.py +36260 -0
- fi/alk/practice/__init__.py +51 -0
- fi/alk/practice/_assess.py +103 -0
- fi/alk/practice/_budget.py +81 -0
- fi/alk/practice/_calibrate.py +69 -0
- fi/alk/practice/_capstone.py +86 -0
- fi/alk/practice/_contract.py +91 -0
- fi/alk/practice/_diagnose.py +79 -0
- fi/alk/practice/_drill.py +196 -0
- fi/alk/practice/_experiment.py +720 -0
- fi/alk/practice/_schedule.py +102 -0
- fi/alk/practice/_store.py +194 -0
- fi/alk/practice/_trainer.py +245 -0
- fi/alk/practice/_update.py +125 -0
- fi/alk/redteam.py +2621 -0
- fi/alk/rewardhack.py +237 -0
- fi/alk/simulate.py +10351 -0
- fi/alk/studio/__init__.py +82 -0
- fi/alk/studio/_bias.py +314 -0
- fi/alk/studio/_calibration.py +522 -0
- fi/alk/studio/_coverage.py +262 -0
- fi/alk/studio/_download.py +665 -0
- fi/alk/studio/_fidelity_attack.py +114 -0
- fi/alk/studio/_generate.py +652 -0
- fi/alk/studio/_library.py +370 -0
- fi/alk/studio/_scan.py +134 -0
- fi/alk/studio/_upgrade.py +42 -0
- fi/alk/studio/_vendor.py +172 -0
- fi/alk/suite.py +4200 -0
- fi/alk/tasks.py +828 -0
- fi/alk/telemetry/__init__.py +149 -0
- fi/alk/telemetry/_contract.py +141 -0
- fi/alk/telemetry/_emit.py +182 -0
- fi/alk/telemetry/_ledger.py +296 -0
- fi/alk/telemetry/_queue.py +127 -0
- fi/alk/telemetry/_row.py +294 -0
- fi/alk/telemetry/_run.py +233 -0
- fi/alk/telemetry/_sync.py +193 -0
- fi/alk/telemetry/_url.py +119 -0
- fi/alk/trinity.py +49397 -0
- fi/alk/voice_loop.py +174 -0
- fi/api/__init__.py +1 -0
- fi/api/auth.py +137 -0
- fi/api/types.py +29 -0
- fi/cli/__init__.py +9 -0
- fi/cli/assertions/__init__.py +25 -0
- fi/cli/assertions/conditions.py +76 -0
- fi/cli/assertions/evaluator.py +286 -0
- fi/cli/assertions/exit_codes.py +20 -0
- fi/cli/assertions/parser.py +131 -0
- fi/cli/assertions/reporter.py +194 -0
- fi/cli/commands/__init__.py +9 -0
- fi/cli/commands/config.py +165 -0
- fi/cli/commands/export.py +208 -0
- fi/cli/commands/init.py +112 -0
- fi/cli/commands/list_cmd.py +213 -0
- fi/cli/commands/run.py +486 -0
- fi/cli/commands/validate.py +173 -0
- fi/cli/commands/view.py +424 -0
- fi/cli/config/__init__.py +6 -0
- fi/cli/config/defaults.py +206 -0
- fi/cli/config/loader.py +155 -0
- fi/cli/config/schema.py +174 -0
- fi/cli/main.py +78 -0
- fi/cli/output/__init__.py +6 -0
- fi/cli/output/formatters.py +106 -0
- fi/cli/output/reporters.py +46 -0
- fi/cli/storage/__init__.py +5 -0
- fi/cli/storage/run_history.py +249 -0
- fi/cli/utils/__init__.py +5 -0
- fi/cli/utils/console.py +44 -0
- fi/evals/__init__.py +131 -0
- fi/evals/autoeval/__init__.py +137 -0
- fi/evals/autoeval/analyzer.py +211 -0
- fi/evals/autoeval/config.py +244 -0
- fi/evals/autoeval/export.py +213 -0
- fi/evals/autoeval/interactive.py +283 -0
- fi/evals/autoeval/pipeline.py +625 -0
- fi/evals/autoeval/prompts.py +139 -0
- fi/evals/autoeval/recommender.py +242 -0
- fi/evals/autoeval/rules.py +589 -0
- fi/evals/autoeval/templates.py +299 -0
- fi/evals/autoeval/types.py +232 -0
- fi/evals/core/__init__.py +16 -0
- fi/evals/core/cloud_registry.py +184 -0
- fi/evals/core/engines.py +368 -0
- fi/evals/core/evaluate.py +319 -0
- fi/evals/core/judge_prompt.py +90 -0
- fi/evals/core/prompt_generator.py +83 -0
- fi/evals/core/registry.py +57 -0
- fi/evals/core/result.py +55 -0
- fi/evals/evaluator.py +721 -0
- fi/evals/execution.py +168 -0
- fi/evals/feedback/__init__.py +32 -0
- fi/evals/feedback/calibrator.py +160 -0
- fi/evals/feedback/collector.py +214 -0
- fi/evals/feedback/hooks.py +81 -0
- fi/evals/feedback/retriever.py +128 -0
- fi/evals/feedback/store.py +272 -0
- fi/evals/feedback/types.py +99 -0
- fi/evals/framework/README.md +79 -0
- fi/evals/framework/__init__.py +267 -0
- fi/evals/framework/backends/Dockerfile.eval-runner +33 -0
- fi/evals/framework/backends/__init__.py +99 -0
- fi/evals/framework/backends/_container.py +141 -0
- fi/evals/framework/backends/_utils.py +145 -0
- fi/evals/framework/backends/base.py +223 -0
- fi/evals/framework/backends/celery_backend.py +417 -0
- fi/evals/framework/backends/celery_worker.py +78 -0
- fi/evals/framework/backends/kubernetes_backend.py +665 -0
- fi/evals/framework/backends/ray_backend.py +521 -0
- fi/evals/framework/backends/temporal.py +350 -0
- fi/evals/framework/backends/temporal_worker.py +126 -0
- fi/evals/framework/backends/thread_pool.py +286 -0
- fi/evals/framework/context.py +258 -0
- fi/evals/framework/enrichment.py +306 -0
- fi/evals/framework/evals/__init__.py +68 -0
- fi/evals/framework/evals/agentic.py +399 -0
- fi/evals/framework/evals/builder.py +609 -0
- fi/evals/framework/evals/semantic.py +142 -0
- fi/evals/framework/evaluator.py +647 -0
- fi/evals/framework/evaluators/__init__.py +22 -0
- fi/evals/framework/evaluators/blocking.py +347 -0
- fi/evals/framework/evaluators/non_blocking.py +577 -0
- fi/evals/framework/propagation.py +421 -0
- fi/evals/framework/protocols.py +385 -0
- fi/evals/framework/registry.py +370 -0
- fi/evals/framework/resilience/__init__.py +150 -0
- fi/evals/framework/resilience/circuit_breaker.py +309 -0
- fi/evals/framework/resilience/degradation.py +355 -0
- fi/evals/framework/resilience/health.py +505 -0
- fi/evals/framework/resilience/rate_limiter.py +228 -0
- fi/evals/framework/resilience/retry.py +274 -0
- fi/evals/framework/resilience/types.py +288 -0
- fi/evals/framework/resilience/wrapper.py +433 -0
- fi/evals/framework/types.py +218 -0
- fi/evals/guardrails/README.md +915 -0
- fi/evals/guardrails/__init__.py +96 -0
- fi/evals/guardrails/backends/__init__.py +43 -0
- fi/evals/guardrails/backends/azure.py +361 -0
- fi/evals/guardrails/backends/base.py +88 -0
- fi/evals/guardrails/backends/generic_llm.py +163 -0
- fi/evals/guardrails/backends/granite.py +216 -0
- fi/evals/guardrails/backends/llamaguard.py +221 -0
- fi/evals/guardrails/backends/local_base.py +479 -0
- fi/evals/guardrails/backends/openai.py +365 -0
- fi/evals/guardrails/backends/qwen.py +170 -0
- fi/evals/guardrails/backends/shieldgemma.py +154 -0
- fi/evals/guardrails/backends/turing.py +235 -0
- fi/evals/guardrails/backends/vllm_client.py +321 -0
- fi/evals/guardrails/backends/wildguard.py +188 -0
- fi/evals/guardrails/base.py +888 -0
- fi/evals/guardrails/config.py +221 -0
- fi/evals/guardrails/discovery.py +243 -0
- fi/evals/guardrails/gateway.py +437 -0
- fi/evals/guardrails/registry.py +231 -0
- fi/evals/guardrails/scanners/__init__.py +127 -0
- fi/evals/guardrails/scanners/base.py +191 -0
- fi/evals/guardrails/scanners/code_injection.py +243 -0
- fi/evals/guardrails/scanners/eval_delegate.py +574 -0
- fi/evals/guardrails/scanners/invisible_chars.py +351 -0
- fi/evals/guardrails/scanners/jailbreak.py +412 -0
- fi/evals/guardrails/scanners/language.py +288 -0
- fi/evals/guardrails/scanners/pipeline.py +260 -0
- fi/evals/guardrails/scanners/regex.py +311 -0
- fi/evals/guardrails/scanners/secrets.py +274 -0
- fi/evals/guardrails/scanners/topics.py +649 -0
- fi/evals/guardrails/scanners/urls.py +341 -0
- fi/evals/guardrails/types.py +96 -0
- fi/evals/llm/__init__.py +3 -0
- fi/evals/llm/base_llm_provider.py +35 -0
- fi/evals/llm/providers/litellm.py +70 -0
- fi/evals/local/__init__.py +90 -0
- fi/evals/local/evaluator.py +690 -0
- fi/evals/local/execution_mode.py +121 -0
- fi/evals/local/llm.py +489 -0
- fi/evals/local/metrics/__init__.py +19 -0
- fi/evals/local/registry.py +360 -0
- fi/evals/manager.py +1018 -0
- fi/evals/manager_types.py +362 -0
- fi/evals/metrics/__init__.py +185 -0
- fi/evals/metrics/agents/__init__.py +74 -0
- fi/evals/metrics/agents/metrics.py +693 -0
- fi/evals/metrics/agents/report.py +36463 -0
- fi/evals/metrics/agents/types.py +160 -0
- fi/evals/metrics/base_llm_metric.py +111 -0
- fi/evals/metrics/base_metric.py +138 -0
- fi/evals/metrics/code_security/__init__.py +305 -0
- fi/evals/metrics/code_security/analyzer.py +985 -0
- fi/evals/metrics/code_security/benchmarks/__init__.py +73 -0
- fi/evals/metrics/code_security/benchmarks/builtin.py +750 -0
- fi/evals/metrics/code_security/benchmarks/loader.py +580 -0
- fi/evals/metrics/code_security/benchmarks/types.py +308 -0
- fi/evals/metrics/code_security/detectors/__init__.py +186 -0
- fi/evals/metrics/code_security/detectors/base.py +394 -0
- fi/evals/metrics/code_security/detectors/cryptography.py +345 -0
- fi/evals/metrics/code_security/detectors/injection.py +744 -0
- fi/evals/metrics/code_security/detectors/secrets.py +287 -0
- fi/evals/metrics/code_security/detectors/serialization.py +192 -0
- fi/evals/metrics/code_security/joint_metrics.py +588 -0
- fi/evals/metrics/code_security/judges/__init__.py +83 -0
- fi/evals/metrics/code_security/judges/base.py +238 -0
- fi/evals/metrics/code_security/judges/dual_judge.py +534 -0
- fi/evals/metrics/code_security/judges/llm_judge.py +301 -0
- fi/evals/metrics/code_security/judges/pattern_judge.py +515 -0
- fi/evals/metrics/code_security/metrics.py +388 -0
- fi/evals/metrics/code_security/modes/__init__.py +63 -0
- fi/evals/metrics/code_security/modes/adversarial.py +284 -0
- fi/evals/metrics/code_security/modes/autocomplete.py +198 -0
- fi/evals/metrics/code_security/modes/base.py +283 -0
- fi/evals/metrics/code_security/modes/instruct.py +253 -0
- fi/evals/metrics/code_security/modes/repair.py +230 -0
- fi/evals/metrics/code_security/reports/__init__.py +57 -0
- fi/evals/metrics/code_security/reports/generator.py +404 -0
- fi/evals/metrics/code_security/reports/leaderboard.py +509 -0
- fi/evals/metrics/code_security/types.py +534 -0
- fi/evals/metrics/function_calling/__init__.py +34 -0
- fi/evals/metrics/function_calling/metrics.py +573 -0
- fi/evals/metrics/function_calling/types.py +87 -0
- fi/evals/metrics/hallucination/__init__.py +54 -0
- fi/evals/metrics/hallucination/detector.py +149 -0
- fi/evals/metrics/hallucination/metrics.py +390 -0
- fi/evals/metrics/hallucination/nli.py +253 -0
- fi/evals/metrics/hallucination/sentinel.py +106 -0
- fi/evals/metrics/hallucination/types.py +132 -0
- fi/evals/metrics/heuristics/aggregation_metrics.py +85 -0
- fi/evals/metrics/heuristics/json_metrics.py +87 -0
- fi/evals/metrics/heuristics/similarity_metrics.py +375 -0
- fi/evals/metrics/heuristics/string_metrics.py +391 -0
- fi/evals/metrics/llm_as_judges/__init__.py +17 -0
- fi/evals/metrics/llm_as_judges/custom_judge/metric.py +112 -0
- fi/evals/metrics/llm_as_judges/custom_judge/prompts.py +26 -0
- fi/evals/metrics/llm_as_judges/types.py +48 -0
- fi/evals/metrics/rag/__init__.py +111 -0
- fi/evals/metrics/rag/advanced/__init__.py +14 -0
- fi/evals/metrics/rag/advanced/multi_hop.py +283 -0
- fi/evals/metrics/rag/advanced/source_attribution.py +344 -0
- fi/evals/metrics/rag/generation/__init__.py +17 -0
- fi/evals/metrics/rag/generation/answer_relevancy.py +176 -0
- fi/evals/metrics/rag/generation/context_utilization.py +245 -0
- fi/evals/metrics/rag/generation/faithfulness.py +241 -0
- fi/evals/metrics/rag/generation/groundedness.py +131 -0
- fi/evals/metrics/rag/rag_score.py +277 -0
- fi/evals/metrics/rag/retrieval/__init__.py +20 -0
- fi/evals/metrics/rag/retrieval/context_entity_recall.py +124 -0
- fi/evals/metrics/rag/retrieval/context_precision.py +158 -0
- fi/evals/metrics/rag/retrieval/context_recall.py +106 -0
- fi/evals/metrics/rag/retrieval/noise_sensitivity.py +163 -0
- fi/evals/metrics/rag/retrieval/ranking.py +261 -0
- fi/evals/metrics/rag/types.py +100 -0
- fi/evals/metrics/rag/utils/__init__.py +62 -0
- fi/evals/metrics/rag/utils/claims.py +189 -0
- fi/evals/metrics/rag/utils/entities.py +244 -0
- fi/evals/metrics/rag/utils/nli.py +92 -0
- fi/evals/metrics/rag/utils/similarity.py +345 -0
- fi/evals/metrics/structured/__init__.py +114 -0
- fi/evals/metrics/structured/field_completeness.py +313 -0
- fi/evals/metrics/structured/hierarchy_score.py +366 -0
- fi/evals/metrics/structured/json_validation.py +190 -0
- fi/evals/metrics/structured/schema_compliance.py +280 -0
- fi/evals/metrics/structured/structured_output_score.py +298 -0
- fi/evals/metrics/structured/types.py +108 -0
- fi/evals/metrics/structured/validators/__init__.py +30 -0
- fi/evals/metrics/structured/validators/base.py +189 -0
- fi/evals/metrics/structured/validators/json_validator.py +196 -0
- fi/evals/metrics/structured/validators/pydantic_validator.py +178 -0
- fi/evals/metrics/structured/validators/yaml_validator.py +248 -0
- fi/evals/otel/__init__.py +266 -0
- fi/evals/otel/config.py +400 -0
- fi/evals/otel/conventions.py +463 -0
- fi/evals/otel/enrichment.py +371 -0
- fi/evals/otel/instrumentors/__init__.py +140 -0
- fi/evals/otel/instrumentors/anthropic.py +517 -0
- fi/evals/otel/instrumentors/base.py +382 -0
- fi/evals/otel/instrumentors/openai.py +673 -0
- fi/evals/otel/processors/__init__.py +36 -0
- fi/evals/otel/processors/base.py +473 -0
- fi/evals/otel/processors/cost.py +445 -0
- fi/evals/otel/processors/evaluation.py +559 -0
- fi/evals/otel/processors/llm.py +462 -0
- fi/evals/otel/tracer.py +506 -0
- fi/evals/otel/types.py +232 -0
- fi/evals/otel_utils.py +23 -0
- fi/evals/protect.py +671 -0
- fi/evals/protect_input_adapter.py +154 -0
- fi/evals/streaming/__init__.py +88 -0
- fi/evals/streaming/buffer.py +213 -0
- fi/evals/streaming/evaluator.py +551 -0
- fi/evals/streaming/policy.py +307 -0
- fi/evals/streaming/scorers.py +368 -0
- fi/evals/streaming/types.py +238 -0
- fi/evals/templates.py +472 -0
- fi/evals/types.py +156 -0
- fi/opt/__init__.py +221 -0
- fi/opt/_objective_scoring.py +85 -0
- fi/opt/base/__init__.py +11 -0
- fi/opt/base/base_generator.py +33 -0
- fi/opt/base/base_mapper.py +26 -0
- fi/opt/base/base_optimizer.py +45 -0
- fi/opt/base/evaluator.py +211 -0
- fi/opt/components.py +3095 -0
- fi/opt/datamappers/__init__.py +3 -0
- fi/opt/datamappers/basic_mapper.py +40 -0
- fi/opt/deployment.py +1021 -0
- fi/opt/evidence.py +4332 -0
- fi/opt/generators/__init__.py +3 -0
- fi/opt/generators/litellm.py +66 -0
- fi/opt/integrations/__init__.py +23 -0
- fi/opt/integrations/generative_suite.py +410 -0
- fi/opt/integrations/simulate.py +1313 -0
- fi/opt/mutations.py +771 -0
- fi/opt/observability.py +4639 -0
- fi/opt/optimizer_trace.py +889 -0
- fi/opt/optimizers/__init__.py +80 -0
- fi/opt/optimizers/agent.py +331 -0
- fi/opt/optimizers/agent_bandit.py +392 -0
- fi/opt/optimizers/agent_curriculum.py +635 -0
- fi/opt/optimizers/agent_evolution.py +894 -0
- fi/opt/optimizers/agent_feedback.py +1863 -0
- fi/opt/optimizers/agent_pareto.py +547 -0
- fi/opt/optimizers/agent_social_memory.py +1113 -0
- fi/opt/optimizers/agent_tpe.py +321 -0
- fi/opt/optimizers/bayesian_search.py +449 -0
- fi/opt/optimizers/council.py +2075 -0
- fi/opt/optimizers/futureagi_replay.py +799 -0
- fi/opt/optimizers/gepa.py +322 -0
- fi/opt/optimizers/metaprompt.py +243 -0
- fi/opt/optimizers/promptwizard.py +417 -0
- fi/opt/optimizers/protegi.py +329 -0
- fi/opt/optimizers/random_search.py +224 -0
- fi/opt/research.py +518 -0
- fi/opt/simulation.py +260 -0
- fi/opt/targets.py +232 -0
- fi/opt/types.py +66 -0
- fi/opt/utils/__init__.py +4 -0
- fi/opt/utils/early_stopping.py +266 -0
- fi/opt/utils/setup_logging.py +82 -0
- fi/simulate/__init__.py +540 -0
- fi/simulate/_hashing.py +35 -0
- fi/simulate/_logging.py +10 -0
- fi/simulate/adapters.py +87 -0
- fi/simulate/agent/__init__.py +120 -0
- fi/simulate/agent/browser.py +658 -0
- fi/simulate/agent/definition.py +587 -0
- fi/simulate/agent/frameworks.py +3528 -0
- fi/simulate/agent/generic.py +8286 -0
- fi/simulate/agent/import_probe.py +227 -0
- fi/simulate/agent/memory.py +905 -0
- fi/simulate/agent/mocks.py +101 -0
- fi/simulate/agent/multi_agent.py +361 -0
- fi/simulate/agent/orchestration.py +903 -0
- fi/simulate/agent/realtime.py +665 -0
- fi/simulate/agent/wrapper.py +99 -0
- fi/simulate/agent/wrappers/__init__.py +18 -0
- fi/simulate/agent/wrappers/anthropic.py +62 -0
- fi/simulate/agent/wrappers/gemini.py +65 -0
- fi/simulate/agent/wrappers/http.py +404 -0
- fi/simulate/agent/wrappers/langchain.py +80 -0
- fi/simulate/agent/wrappers/openai.py +75 -0
- fi/simulate/agent/wrappers/websocket.py +326 -0
- fi/simulate/artifacts/__init__.py +11 -0
- fi/simulate/artifacts/manifest.py +62 -0
- fi/simulate/cli.py +20560 -0
- fi/simulate/endpoints/__init__.py +45 -0
- fi/simulate/endpoints/_http_actor.py +73 -0
- fi/simulate/endpoints/actor_sources.py +243 -0
- fi/simulate/endpoints/base.py +107 -0
- fi/simulate/endpoints/builtins.py +10 -0
- fi/simulate/endpoints/callable.py +95 -0
- fi/simulate/endpoints/http.py +76 -0
- fi/simulate/endpoints/livekit.py +138 -0
- fi/simulate/endpoints/originators.py +132 -0
- fi/simulate/endpoints/profiles.py +348 -0
- fi/simulate/endpoints/retell.py +633 -0
- fi/simulate/endpoints/vapi.py +205 -0
- fi/simulate/endpoints/websocket.py +76 -0
- fi/simulate/environment.py +33026 -0
- fi/simulate/environments/__init__.py +11 -0
- fi/simulate/environments/base.py +73 -0
- fi/simulate/environments/chat.py +697 -0
- fi/simulate/environments/voice.py +212 -0
- fi/simulate/evaluation/__init__.py +4 -0
- fi/simulate/evaluation/ai_eval.py +227 -0
- fi/simulate/evidence/__init__.py +35 -0
- fi/simulate/evidence/base.py +59 -0
- fi/simulate/evidence/caller_observed.py +50 -0
- fi/simulate/evidence/livekit_instrumentation.py +51 -0
- fi/simulate/evidence/livekit_room.py +50 -0
- fi/simulate/evidence/otel.py +49 -0
- fi/simulate/evidence/providers/__init__.py +24 -0
- fi/simulate/evidence/providers/base.py +61 -0
- fi/simulate/evidence/providers/retell.py +376 -0
- fi/simulate/evidence/providers/vapi.py +426 -0
- fi/simulate/hosted/__init__.py +32 -0
- fi/simulate/hosted/child_entrypoint.py +306 -0
- fi/simulate/hosted/job.py +150 -0
- fi/simulate/hosted/targets.py +53 -0
- fi/simulate/instrumentation/__init__.py +5 -0
- fi/simulate/instrumentation/livekit/__init__.py +122 -0
- fi/simulate/manifest.py +1033 -0
- fi/simulate/matrix_cli.py +165 -0
- fi/simulate/realtime/__init__.py +40 -0
- fi/simulate/realtime/events.py +107 -0
- fi/simulate/realtime/media.py +61 -0
- fi/simulate/realtime/session.py +91 -0
- fi/simulate/recording/__init__.py +5 -0
- fi/simulate/recording/room_recorder.py +326 -0
- fi/simulate/registry.py +185 -0
- fi/simulate/results/__init__.py +9 -0
- fi/simulate/results/base.py +18 -0
- fi/simulate/results/filesystem.py +71 -0
- fi/simulate/results/futureagi.py +1340 -0
- fi/simulate/runtime/__init__.py +85 -0
- fi/simulate/runtime/capabilities.py +40 -0
- fi/simulate/runtime/events.py +63 -0
- fi/simulate/runtime/failures.py +25 -0
- fi/simulate/runtime/ids.py +34 -0
- fi/simulate/runtime/plan.py +70 -0
- fi/simulate/runtime/planner.py +102 -0
- fi/simulate/runtime/report.py +174 -0
- fi/simulate/runtime/run.py +75 -0
- fi/simulate/runtime/runner.py +333 -0
- fi/simulate/runtime/spec.py +186 -0
- fi/simulate/simulation/__init__.py +30 -0
- fi/simulate/simulation/behavior_policy.py +425 -0
- fi/simulate/simulation/bridge/__init__.py +9 -0
- fi/simulate/simulation/bridge/audio.py +29 -0
- fi/simulate/simulation/bridge/connector.py +46 -0
- fi/simulate/simulation/bridge/livekit.py +252 -0
- fi/simulate/simulation/bridge/retell.py +188 -0
- fi/simulate/simulation/bridge/vapi.py +177 -0
- fi/simulate/simulation/contract.py +419 -0
- fi/simulate/simulation/engines/__init__.py +12 -0
- fi/simulate/simulation/engines/base.py +21 -0
- fi/simulate/simulation/engines/cloud.py +517 -0
- fi/simulate/simulation/engines/livekit.py +4167 -0
- fi/simulate/simulation/engines/local_text.py +89 -0
- fi/simulate/simulation/fidelity.py +374 -0
- fi/simulate/simulation/gemini_tts_stream.py +110 -0
- fi/simulate/simulation/generator.py +91 -0
- fi/simulate/simulation/goal_machine.py +185 -0
- fi/simulate/simulation/livekit_models.py +467 -0
- fi/simulate/simulation/matrix.py +170 -0
- fi/simulate/simulation/models.py +279 -0
- fi/simulate/simulation/runner.py +153 -0
- fi/simulate/simulation/synthetic.py +880 -0
- fi/simulate/simulation/voice_prompt.py +502 -0
- fi/simulate/simulator/__init__.py +55 -0
- fi/simulate/simulator/builtins.py +53 -0
- fi/simulate/suite.py +1288 -0
- fi/simulate/utils/routes.py +164 -0
- fi/simulate/voice.py +225 -0
- fi/simulate/voice_cli.py +182 -0
- fi/utils/__init__.py +1 -0
- fi/utils/constants.py +14 -0
- fi/utils/errors.py +200 -0
- fi/utils/executor.py +26 -0
- fi/utils/routes.py +119 -0
- fi/utils/utils.py +17 -0
|
@@ -0,0 +1,674 @@
|
|
|
1
|
+
"""Safe, deterministic credential discovery for submitted agent repositories.
|
|
2
|
+
|
|
3
|
+
The reasoning model may later explain a requirement, but it never needs to see a resolved
|
|
4
|
+
credential. This module performs the cheap preflight first and emits a structured manifest the
|
|
5
|
+
platform can use to ask for missing secrets before a build or provider call starts.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import ast
|
|
11
|
+
import hashlib
|
|
12
|
+
import json
|
|
13
|
+
import re
|
|
14
|
+
from enum import Enum
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Iterable
|
|
17
|
+
|
|
18
|
+
from pydantic import BaseModel, Field
|
|
19
|
+
|
|
20
|
+
from fi.simulate.runtime.spec import SecretRef
|
|
21
|
+
|
|
22
|
+
CREDENTIAL_MANIFEST_VERSION = "futureagi.credential-requirements.v1"
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class RequirementKind(str, Enum):
|
|
26
|
+
SECRET = "secret"
|
|
27
|
+
CONFIGURATION = "configuration"
|
|
28
|
+
HARNESS_INFRASTRUCTURE = "harness_infrastructure"
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class RequirementStatus(str, Enum):
|
|
32
|
+
MISSING = "missing"
|
|
33
|
+
CONFIGURED = "configured"
|
|
34
|
+
OPTIONAL = "optional"
|
|
35
|
+
HARNESS_PROVIDED = "harness_provided"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class CredentialRequirement(BaseModel):
|
|
39
|
+
id: str
|
|
40
|
+
environment_name: str
|
|
41
|
+
provider: str
|
|
42
|
+
purpose: str
|
|
43
|
+
kind: RequirementKind
|
|
44
|
+
required: bool = True
|
|
45
|
+
status: RequirementStatus
|
|
46
|
+
detected_from: list[str] = Field(default_factory=list)
|
|
47
|
+
accepted_secret_types: list[str] = Field(default_factory=list)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class CredentialChoice(BaseModel):
|
|
51
|
+
id: str
|
|
52
|
+
purpose: str
|
|
53
|
+
options: list[list[str]]
|
|
54
|
+
satisfied: bool = False
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class CredentialManifest(BaseModel):
|
|
58
|
+
schema_version: str = CREDENTIAL_MANIFEST_VERSION
|
|
59
|
+
source_digest: str
|
|
60
|
+
detected_connectors: list[str] = Field(default_factory=list)
|
|
61
|
+
requirements: list[CredentialRequirement] = Field(default_factory=list)
|
|
62
|
+
credential_choices: list[CredentialChoice] = Field(default_factory=list)
|
|
63
|
+
scanned_files: int = Field(ge=0)
|
|
64
|
+
truncated: bool = False
|
|
65
|
+
|
|
66
|
+
@property
|
|
67
|
+
def missing_required(self) -> list[CredentialRequirement]:
|
|
68
|
+
choice_members = {
|
|
69
|
+
name
|
|
70
|
+
for choice in self.credential_choices
|
|
71
|
+
for option in choice.options
|
|
72
|
+
for name in option
|
|
73
|
+
}
|
|
74
|
+
return [
|
|
75
|
+
item
|
|
76
|
+
for item in self.requirements
|
|
77
|
+
if item.required
|
|
78
|
+
and item.status is RequirementStatus.MISSING
|
|
79
|
+
and item.environment_name not in choice_members
|
|
80
|
+
]
|
|
81
|
+
|
|
82
|
+
@property
|
|
83
|
+
def ready(self) -> bool:
|
|
84
|
+
return not self.missing_required and all(
|
|
85
|
+
choice.satisfied for choice in self.credential_choices
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
_IGNORED_PARTS = {
|
|
90
|
+
".git",
|
|
91
|
+
".venv",
|
|
92
|
+
"venv",
|
|
93
|
+
".tox",
|
|
94
|
+
".nox",
|
|
95
|
+
"node_modules",
|
|
96
|
+
"test",
|
|
97
|
+
"tests",
|
|
98
|
+
"__pycache__",
|
|
99
|
+
"artifacts",
|
|
100
|
+
"build",
|
|
101
|
+
"dist",
|
|
102
|
+
"vendor",
|
|
103
|
+
}
|
|
104
|
+
_SOURCE_SUFFIXES = {
|
|
105
|
+
".py",
|
|
106
|
+
".js",
|
|
107
|
+
".jsx",
|
|
108
|
+
".ts",
|
|
109
|
+
".tsx",
|
|
110
|
+
".mjs",
|
|
111
|
+
".cjs",
|
|
112
|
+
".go",
|
|
113
|
+
".rb",
|
|
114
|
+
".toml",
|
|
115
|
+
".yaml",
|
|
116
|
+
".yml",
|
|
117
|
+
".json",
|
|
118
|
+
}
|
|
119
|
+
_CONFIG_NAMES = {
|
|
120
|
+
".env.example",
|
|
121
|
+
".env.sample",
|
|
122
|
+
".env.template",
|
|
123
|
+
"env.example",
|
|
124
|
+
"env.sample",
|
|
125
|
+
"env.template",
|
|
126
|
+
"docker-compose.yml",
|
|
127
|
+
"docker-compose.yaml",
|
|
128
|
+
"compose.yml",
|
|
129
|
+
"compose.yaml",
|
|
130
|
+
"pyproject.toml",
|
|
131
|
+
"package.json",
|
|
132
|
+
}
|
|
133
|
+
_MAX_FILES = 2_000
|
|
134
|
+
_MAX_FILE_BYTES = 1_000_000
|
|
135
|
+
|
|
136
|
+
_ENV_DECLARATION = re.compile(
|
|
137
|
+
r"(?m)^(?:export[ \t]+)?([A-Z][A-Z0-9_]{2,})[ \t]*=[ \t]*([^\r\n#]*)"
|
|
138
|
+
)
|
|
139
|
+
_ENV_TEMPLATE_NAMES = {
|
|
140
|
+
".env.example",
|
|
141
|
+
".env.sample",
|
|
142
|
+
".env.template",
|
|
143
|
+
"env.example",
|
|
144
|
+
"env.sample",
|
|
145
|
+
"env.template",
|
|
146
|
+
}
|
|
147
|
+
_COMPOSE_NAMES = {
|
|
148
|
+
"docker-compose.yml",
|
|
149
|
+
"docker-compose.yaml",
|
|
150
|
+
"compose.yml",
|
|
151
|
+
"compose.yaml",
|
|
152
|
+
}
|
|
153
|
+
_PLACEHOLDER_VALUE = re.compile(
|
|
154
|
+
r"^(?:"
|
|
155
|
+
r"\[[^\]]*\]|<[^>]*>|\{[^}]*\}|"
|
|
156
|
+
r"(?:your|replace|insert|change)[-_ ]?(?:me|this|.*)|"
|
|
157
|
+
r"example|placeholder|todo|xxx+"
|
|
158
|
+
r")$",
|
|
159
|
+
re.IGNORECASE,
|
|
160
|
+
)
|
|
161
|
+
_COMPOSE_VARIABLE = re.compile(r"\$\{([A-Z][A-Z0-9_]{2,})(?:(:?[-?])([^}]*))?\}")
|
|
162
|
+
_PYTHON_REQUIRED = re.compile(r"os\.environ\s*\[\s*['\"]([A-Z][A-Z0-9_]{2,})['\"]\s*\]")
|
|
163
|
+
_PYTHON_GETENV = re.compile(
|
|
164
|
+
r"(?:os\.getenv|os\.environ\.get)\s*\(\s*['\"]([A-Z][A-Z0-9_]{2,})['\"](?:\s*,\s*([^\)]+))?"
|
|
165
|
+
)
|
|
166
|
+
_JS_ENV = re.compile(r"process\.env\.([A-Z][A-Z0-9_]{2,})")
|
|
167
|
+
|
|
168
|
+
_SECRET_NAME = re.compile(
|
|
169
|
+
r"(?:^|_)(?:API_?KEY|API_?SECRET|AUTHORIZATION|CREDENTIALS?|PASSWORD|PRIVATE_?KEY|SECRET|TOKEN)(?:_|$)"
|
|
170
|
+
)
|
|
171
|
+
_HARNESS_PROVIDED = {
|
|
172
|
+
"DATABASE_URL",
|
|
173
|
+
"POSTGRES_HOST",
|
|
174
|
+
"POSTGRES_PORT",
|
|
175
|
+
"POSTGRES_DB",
|
|
176
|
+
"POSTGRES_USER",
|
|
177
|
+
"POSTGRES_PASSWORD",
|
|
178
|
+
"REDIS_URL",
|
|
179
|
+
"TOOLS_API_URL",
|
|
180
|
+
"MCP_URL",
|
|
181
|
+
}
|
|
182
|
+
_NON_USER_CONFIGURATION = {
|
|
183
|
+
"PORT",
|
|
184
|
+
"HOST",
|
|
185
|
+
"LOG_LEVEL",
|
|
186
|
+
"ENVIRONMENT",
|
|
187
|
+
"NODE_ENV",
|
|
188
|
+
"PYTHONPATH",
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
_PROVIDERS: tuple[tuple[tuple[str, ...], str, str], ...] = (
|
|
192
|
+
(("LIVEKIT_", "ACCEPTANCE_LIVEKIT_"), "livekit", "connect to the voice room"),
|
|
193
|
+
(("DEEPGRAM_",), "deepgram", "speech-to-text"),
|
|
194
|
+
(("CARTESIA_",), "cartesia", "text-to-speech"),
|
|
195
|
+
(("ELEVENLABS_",), "elevenlabs", "text-to-speech"),
|
|
196
|
+
(("OPENAI_",), "openai", "language or realtime model"),
|
|
197
|
+
(("ANTHROPIC_",), "anthropic", "language model"),
|
|
198
|
+
(("GOOGLE_", "VERTEX_", "CLOUD_ML_"), "google_cloud", "model or cloud runtime"),
|
|
199
|
+
(("VAPI_",), "vapi", "hosted voice agent connection"),
|
|
200
|
+
(("RETELL_",), "retell", "hosted voice agent connection"),
|
|
201
|
+
(("TWILIO_",), "twilio", "telephony connection"),
|
|
202
|
+
(("AWS_",), "aws", "cloud service"),
|
|
203
|
+
(("AZURE_",), "azure", "cloud service"),
|
|
204
|
+
(("MONGODB_", "MONGO_"), "mongodb", "database connection"),
|
|
205
|
+
(("PINECONE_",), "pinecone", "vector database"),
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
_CONNECTOR_SIGNALS = {
|
|
209
|
+
"livekit": ("livekit", "@livekit/", "livekit-agents"),
|
|
210
|
+
"vapi": ("vapi", "@vapi-ai"),
|
|
211
|
+
"retell": ("retell", "retell-sdk"),
|
|
212
|
+
"twilio": ("twilio",),
|
|
213
|
+
"pipecat": ("pipecat",),
|
|
214
|
+
"mcp": ("mcp", "modelcontextprotocol"),
|
|
215
|
+
"http": ("fastapi", "flask", "express", "axios", "httpx"),
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
# Some SDKs consume credentials internally, so the submitted source never calls ``getenv``.
|
|
219
|
+
# Connector detection must still make those requirements visible before the expensive build or
|
|
220
|
+
# worker-start phase. Keep this list to credentials inherent to the connector itself; model and
|
|
221
|
+
# optional application integrations continue to be discovered from explicit source declarations.
|
|
222
|
+
_CONNECTOR_REQUIREMENTS: dict[str, tuple[str, ...]] = {
|
|
223
|
+
"livekit": ("LIVEKIT_URL", "LIVEKIT_API_KEY", "LIVEKIT_API_SECRET"),
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
_LIVEKIT_PROVIDER_CREDENTIALS = {
|
|
227
|
+
"deepgram": ("DEEPGRAM_API_KEY",),
|
|
228
|
+
"cartesia": ("CARTESIA_API_KEY",),
|
|
229
|
+
"elevenlabs": ("ELEVENLABS_API_KEY",),
|
|
230
|
+
"openai": ("OPENAI_API_KEY",),
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _dotted_name(node: ast.AST) -> str:
|
|
235
|
+
if isinstance(node, ast.Name):
|
|
236
|
+
return node.id
|
|
237
|
+
if isinstance(node, ast.Attribute):
|
|
238
|
+
prefix = _dotted_name(node.value)
|
|
239
|
+
return f"{prefix}.{node.attr}" if prefix else node.attr
|
|
240
|
+
return ""
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _python_sdk_requirements(content: str) -> list[tuple[str, str]]:
|
|
244
|
+
"""Discover credentials consumed internally by constructors in submitted Python.
|
|
245
|
+
|
|
246
|
+
This is AST-based because model constructors routinely span multiple lines and contain
|
|
247
|
+
nested calls; a regex ending at the first ``)`` silently misses valid ``vertexai=True``
|
|
248
|
+
configurations. Parsing is read-only and never imports or executes customer code.
|
|
249
|
+
"""
|
|
250
|
+
try:
|
|
251
|
+
tree = ast.parse(content)
|
|
252
|
+
except SyntaxError:
|
|
253
|
+
return []
|
|
254
|
+
aliases: dict[str, str] = {}
|
|
255
|
+
imports: dict[str, str] = {}
|
|
256
|
+
for node in ast.walk(tree):
|
|
257
|
+
if isinstance(node, ast.ImportFrom):
|
|
258
|
+
module = node.module or ""
|
|
259
|
+
for item in node.names:
|
|
260
|
+
local = item.asname or item.name
|
|
261
|
+
imports[local] = f"{module}.{item.name}" if module else item.name
|
|
262
|
+
if module == "livekit.plugins":
|
|
263
|
+
aliases[local] = item.name
|
|
264
|
+
elif isinstance(node, ast.Import):
|
|
265
|
+
for item in node.names:
|
|
266
|
+
local = item.asname or item.name.split(".")[0]
|
|
267
|
+
imports[local] = item.name
|
|
268
|
+
prefix = "livekit.plugins."
|
|
269
|
+
if item.name.startswith(prefix):
|
|
270
|
+
aliases[item.asname or item.name] = item.name[len(prefix) :].split(
|
|
271
|
+
"."
|
|
272
|
+
)[0]
|
|
273
|
+
|
|
274
|
+
found: set[tuple[str, str]] = set()
|
|
275
|
+
for node in ast.walk(tree):
|
|
276
|
+
if not isinstance(node, ast.Call):
|
|
277
|
+
continue
|
|
278
|
+
path = _dotted_name(node.func)
|
|
279
|
+
if not path:
|
|
280
|
+
continue
|
|
281
|
+
parts = path.split(".")
|
|
282
|
+
provider = aliases.get(parts[0], parts[-2] if len(parts) > 1 else "")
|
|
283
|
+
constructor = parts[-1]
|
|
284
|
+
imported_path = imports.get(parts[0], parts[0])
|
|
285
|
+
resolved_path = ".".join([imported_path, *parts[1:]])
|
|
286
|
+
if provider in _LIVEKIT_PROVIDER_CREDENTIALS and constructor in {
|
|
287
|
+
"LLM",
|
|
288
|
+
"STT",
|
|
289
|
+
"TTS",
|
|
290
|
+
"RealtimeModel",
|
|
291
|
+
}:
|
|
292
|
+
for name in _LIVEKIT_PROVIDER_CREDENTIALS[provider]:
|
|
293
|
+
found.add((name, f"sdk:livekit.plugins.{provider}"))
|
|
294
|
+
if provider == "google" and constructor in {"LLM", "STT", "TTS"}:
|
|
295
|
+
vertex = next(
|
|
296
|
+
(
|
|
297
|
+
keyword.value
|
|
298
|
+
for keyword in node.keywords
|
|
299
|
+
if keyword.arg == "vertexai"
|
|
300
|
+
),
|
|
301
|
+
None,
|
|
302
|
+
)
|
|
303
|
+
if isinstance(vertex, ast.Constant) and vertex.value is True:
|
|
304
|
+
for name in (
|
|
305
|
+
"GOOGLE_APPLICATION_CREDENTIALS",
|
|
306
|
+
"GOOGLE_APPLICATION_CREDENTIALS_JSON",
|
|
307
|
+
"GOOGLE_CLOUD_PROJECT",
|
|
308
|
+
):
|
|
309
|
+
found.add((name, "sdk:livekit.plugins.google.vertex"))
|
|
310
|
+
# LangChain and Google's first-party SDKs resolve Application Default Credentials
|
|
311
|
+
# internally, so customer code commonly reads only project/location and never calls
|
|
312
|
+
# getenv for the credential file itself. Recognize the constructor and explicit Vertex
|
|
313
|
+
# mode without importing or executing submitted code.
|
|
314
|
+
vertex = next(
|
|
315
|
+
(keyword.value for keyword in node.keywords if keyword.arg == "vertexai"),
|
|
316
|
+
None,
|
|
317
|
+
)
|
|
318
|
+
is_explicit_vertex_client = (
|
|
319
|
+
resolved_path == "langchain_google_genai.ChatGoogleGenerativeAI"
|
|
320
|
+
and isinstance(vertex, ast.Constant)
|
|
321
|
+
and vertex.value is True
|
|
322
|
+
) or (
|
|
323
|
+
resolved_path in {"google.genai.Client", "genai.Client"}
|
|
324
|
+
and isinstance(vertex, ast.Constant)
|
|
325
|
+
and vertex.value is True
|
|
326
|
+
)
|
|
327
|
+
if is_explicit_vertex_client:
|
|
328
|
+
for name in (
|
|
329
|
+
"GOOGLE_APPLICATION_CREDENTIALS",
|
|
330
|
+
"GOOGLE_APPLICATION_CREDENTIALS_JSON",
|
|
331
|
+
"GOOGLE_CLOUD_PROJECT",
|
|
332
|
+
):
|
|
333
|
+
found.add((name, f"sdk:{resolved_path}.vertex"))
|
|
334
|
+
return sorted(found)
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def discover_credentials(
|
|
338
|
+
root: str | Path,
|
|
339
|
+
*,
|
|
340
|
+
secret_refs: dict[str, SecretRef] | None = None,
|
|
341
|
+
provided_environment: Iterable[str] = (),
|
|
342
|
+
scan_paths: Iterable[str | Path] | None = None,
|
|
343
|
+
) -> CredentialManifest:
|
|
344
|
+
"""Inspect declarations and environment reads without executing submitted code."""
|
|
345
|
+
root = Path(root).expanduser().resolve()
|
|
346
|
+
if not root.is_dir():
|
|
347
|
+
raise ValueError(f"credential_source_missing: {root}")
|
|
348
|
+
configured = {str(name).upper() for name in provided_environment}
|
|
349
|
+
for alias, ref in (secret_refs or {}).items():
|
|
350
|
+
configured.add(str(alias).upper())
|
|
351
|
+
configured.add(str(ref.key).upper())
|
|
352
|
+
configured.add(_identifier(ref.purpose).upper())
|
|
353
|
+
# Hosted process runtime materializes this vault value to a private file and exports the
|
|
354
|
+
# conventional Google variable to the child. Discovery deals in names only; no credential
|
|
355
|
+
# value is read or persisted here.
|
|
356
|
+
if "GOOGLE_APPLICATION_CREDENTIALS_JSON" in configured:
|
|
357
|
+
configured.add("GOOGLE_APPLICATION_CREDENTIALS")
|
|
358
|
+
|
|
359
|
+
findings: dict[str, dict[str, object]] = {}
|
|
360
|
+
connector_hits: set[str] = set()
|
|
361
|
+
scanned = 0
|
|
362
|
+
truncated = False
|
|
363
|
+
digest = hashlib.sha256()
|
|
364
|
+
|
|
365
|
+
for path in _candidate_files(root, scan_paths=scan_paths):
|
|
366
|
+
if scanned >= _MAX_FILES:
|
|
367
|
+
truncated = True
|
|
368
|
+
break
|
|
369
|
+
try:
|
|
370
|
+
size = path.stat().st_size
|
|
371
|
+
except OSError:
|
|
372
|
+
continue
|
|
373
|
+
if size > _MAX_FILE_BYTES:
|
|
374
|
+
continue
|
|
375
|
+
try:
|
|
376
|
+
content = path.read_text(encoding="utf-8", errors="replace")
|
|
377
|
+
except OSError:
|
|
378
|
+
continue
|
|
379
|
+
scanned += 1
|
|
380
|
+
relative = path.relative_to(root).as_posix()
|
|
381
|
+
digest.update(relative.encode())
|
|
382
|
+
digest.update(hashlib.sha256(content.encode()).digest())
|
|
383
|
+
lowered = content.lower()
|
|
384
|
+
for connector, signals in _CONNECTOR_SIGNALS.items():
|
|
385
|
+
if any(signal in lowered for signal in signals):
|
|
386
|
+
connector_hits.add(connector)
|
|
387
|
+
|
|
388
|
+
def record(
|
|
389
|
+
name: str, *, required: bool, declared_default: bool = False
|
|
390
|
+
) -> None:
|
|
391
|
+
# ALK_* is a reserved control-plane namespace. Provider adapters and the
|
|
392
|
+
# hosted lifecycle legitimately read these values from their process
|
|
393
|
+
# environment, but they are injected at launch and must never be admitted as
|
|
394
|
+
# customer-supplied target configuration.
|
|
395
|
+
if name in _NON_USER_CONFIGURATION or name.startswith("ALK_"):
|
|
396
|
+
return
|
|
397
|
+
item = findings.setdefault(
|
|
398
|
+
name,
|
|
399
|
+
{
|
|
400
|
+
"required": False,
|
|
401
|
+
"declared_default": False,
|
|
402
|
+
"detected_from": set(),
|
|
403
|
+
},
|
|
404
|
+
)
|
|
405
|
+
item["required"] = bool(item["required"]) or required
|
|
406
|
+
item["declared_default"] = (
|
|
407
|
+
bool(item["declared_default"]) or declared_default
|
|
408
|
+
)
|
|
409
|
+
detected = item["detected_from"]
|
|
410
|
+
assert isinstance(detected, set)
|
|
411
|
+
detected.add(relative)
|
|
412
|
+
|
|
413
|
+
# An uppercase assignment in source is usually a constant, not an
|
|
414
|
+
# environment declaration. Only env templates use NAME=value syntax.
|
|
415
|
+
if path.name.lower() in _ENV_TEMPLATE_NAMES:
|
|
416
|
+
for match in _ENV_DECLARATION.finditer(content):
|
|
417
|
+
name = match.group(1)
|
|
418
|
+
value = match.group(2).strip().strip("\"'")
|
|
419
|
+
usable_default = bool(value) and not _PLACEHOLDER_VALUE.fullmatch(value)
|
|
420
|
+
record(
|
|
421
|
+
name,
|
|
422
|
+
# A blank non-secret setting in an example file documents a knob; it
|
|
423
|
+
# does not prove that the selected runtime path needs a value. Strict
|
|
424
|
+
# source reads and Compose's :? operator remain authoritative. Secret
|
|
425
|
+
# placeholders stay required because SDKs commonly consume them without
|
|
426
|
+
# an explicit getenv call in customer code.
|
|
427
|
+
required=(
|
|
428
|
+
not usable_default and _kind(name) is RequirementKind.SECRET
|
|
429
|
+
),
|
|
430
|
+
declared_default=usable_default,
|
|
431
|
+
)
|
|
432
|
+
if path.name.lower() in _COMPOSE_NAMES:
|
|
433
|
+
for match in _COMPOSE_VARIABLE.finditer(content):
|
|
434
|
+
operator, fallback = match.group(2), (match.group(3) or "").strip()
|
|
435
|
+
# Compose substitutes an unset plain ${NAME} with an empty string. Only its
|
|
436
|
+
# explicit error operators are admission requirements; source-level strict
|
|
437
|
+
# reads can still make the same name required when a built runtime is scanned.
|
|
438
|
+
required = operator in {"?", ":?"}
|
|
439
|
+
record(
|
|
440
|
+
match.group(1),
|
|
441
|
+
required=required,
|
|
442
|
+
declared_default=bool(fallback) and operator not in {"?", ":?"},
|
|
443
|
+
)
|
|
444
|
+
for match in _PYTHON_REQUIRED.finditer(content):
|
|
445
|
+
# ``os.environ[NAME] if os.environ.get(NAME) else default`` is a common guarded
|
|
446
|
+
# access idiom. The indexed read alone looks mandatory, but the same-line guard
|
|
447
|
+
# proves the application has a fallback and should not block job submission.
|
|
448
|
+
line_start = content.rfind("\n", 0, match.start()) + 1
|
|
449
|
+
line_end = content.find("\n", match.end())
|
|
450
|
+
line = content[line_start : line_end if line_end >= 0 else len(content)]
|
|
451
|
+
name = match.group(1)
|
|
452
|
+
guarded = bool(
|
|
453
|
+
re.search(
|
|
454
|
+
rf"(?:os\.getenv|os\.environ\.get)\s*\(\s*['\"]{re.escape(name)}['\"]",
|
|
455
|
+
line,
|
|
456
|
+
)
|
|
457
|
+
)
|
|
458
|
+
record(name, required=not guarded, declared_default=guarded)
|
|
459
|
+
for match in _PYTHON_GETENV.finditer(content):
|
|
460
|
+
default = (match.group(2) or "").strip()
|
|
461
|
+
record(
|
|
462
|
+
match.group(1),
|
|
463
|
+
# getenv/get is explicitly nullable. Treat it as optional unless
|
|
464
|
+
# a blank env template or a strict indexed read says otherwise.
|
|
465
|
+
required=False,
|
|
466
|
+
declared_default=bool(default),
|
|
467
|
+
)
|
|
468
|
+
for match in _JS_ENV.finditer(content):
|
|
469
|
+
record(match.group(1), required=True)
|
|
470
|
+
if path.suffix.lower() == ".py":
|
|
471
|
+
for name, origin in _python_sdk_requirements(content):
|
|
472
|
+
record(name, required=True)
|
|
473
|
+
detected = findings[name]["detected_from"]
|
|
474
|
+
assert isinstance(detected, set)
|
|
475
|
+
detected.add(origin)
|
|
476
|
+
|
|
477
|
+
for connector in sorted(connector_hits):
|
|
478
|
+
for name in _CONNECTOR_REQUIREMENTS.get(connector, ()):
|
|
479
|
+
item = findings.setdefault(
|
|
480
|
+
name,
|
|
481
|
+
{
|
|
482
|
+
"required": True,
|
|
483
|
+
"declared_default": False,
|
|
484
|
+
"detected_from": set(),
|
|
485
|
+
},
|
|
486
|
+
)
|
|
487
|
+
item["required"] = True
|
|
488
|
+
detected = item["detected_from"]
|
|
489
|
+
assert isinstance(detected, set)
|
|
490
|
+
detected.add(f"connector:{connector}")
|
|
491
|
+
|
|
492
|
+
requirements = []
|
|
493
|
+
for name, finding in sorted(findings.items()):
|
|
494
|
+
kind = _kind(name)
|
|
495
|
+
required = bool(finding["required"])
|
|
496
|
+
if kind is RequirementKind.CONFIGURATION and finding["declared_default"]:
|
|
497
|
+
required = False
|
|
498
|
+
if kind is RequirementKind.HARNESS_INFRASTRUCTURE:
|
|
499
|
+
status = RequirementStatus.HARNESS_PROVIDED
|
|
500
|
+
elif name in configured:
|
|
501
|
+
status = RequirementStatus.CONFIGURED
|
|
502
|
+
elif not required:
|
|
503
|
+
status = RequirementStatus.OPTIONAL
|
|
504
|
+
else:
|
|
505
|
+
status = RequirementStatus.MISSING
|
|
506
|
+
provider, purpose = _provider(name)
|
|
507
|
+
requirements.append(
|
|
508
|
+
CredentialRequirement(
|
|
509
|
+
id=_identifier(name),
|
|
510
|
+
environment_name=name,
|
|
511
|
+
provider=provider,
|
|
512
|
+
purpose=purpose,
|
|
513
|
+
kind=kind,
|
|
514
|
+
required=required,
|
|
515
|
+
status=status,
|
|
516
|
+
detected_from=sorted(finding["detected_from"]),
|
|
517
|
+
accepted_secret_types=(
|
|
518
|
+
[_secret_type(name)] if kind is RequirementKind.SECRET else []
|
|
519
|
+
),
|
|
520
|
+
)
|
|
521
|
+
)
|
|
522
|
+
|
|
523
|
+
choices = _credential_choices(requirements)
|
|
524
|
+
manifest_core = {
|
|
525
|
+
"files": digest.hexdigest(),
|
|
526
|
+
"connectors": sorted(connector_hits),
|
|
527
|
+
"requirements": [item.model_dump(mode="json") for item in requirements],
|
|
528
|
+
"credential_choices": [item.model_dump(mode="json") for item in choices],
|
|
529
|
+
}
|
|
530
|
+
return CredentialManifest(
|
|
531
|
+
source_digest=hashlib.sha256(
|
|
532
|
+
json.dumps(manifest_core, sort_keys=True, separators=(",", ":")).encode()
|
|
533
|
+
).hexdigest(),
|
|
534
|
+
detected_connectors=sorted(connector_hits),
|
|
535
|
+
requirements=requirements,
|
|
536
|
+
credential_choices=choices,
|
|
537
|
+
scanned_files=scanned,
|
|
538
|
+
truncated=truncated,
|
|
539
|
+
)
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
def _candidate_files(
|
|
543
|
+
root: Path, *, scan_paths: Iterable[str | Path] | None = None
|
|
544
|
+
) -> list[Path]:
|
|
545
|
+
scopes: list[Path] | None = None
|
|
546
|
+
if scan_paths is not None:
|
|
547
|
+
scopes = []
|
|
548
|
+
for value in scan_paths:
|
|
549
|
+
candidate = (root / value).resolve()
|
|
550
|
+
try:
|
|
551
|
+
candidate.relative_to(root)
|
|
552
|
+
except ValueError:
|
|
553
|
+
continue
|
|
554
|
+
if candidate.exists():
|
|
555
|
+
scopes.append(candidate)
|
|
556
|
+
result: list[Path] = []
|
|
557
|
+
for path in sorted(root.rglob("*")):
|
|
558
|
+
try:
|
|
559
|
+
relative = path.relative_to(root)
|
|
560
|
+
except ValueError:
|
|
561
|
+
continue
|
|
562
|
+
if any(part in _IGNORED_PARTS for part in relative.parts):
|
|
563
|
+
continue
|
|
564
|
+
if path.is_symlink() or not path.is_file():
|
|
565
|
+
continue
|
|
566
|
+
if scopes is not None and not any(
|
|
567
|
+
path == scope or (scope.is_dir() and scope in path.parents)
|
|
568
|
+
for scope in scopes
|
|
569
|
+
):
|
|
570
|
+
continue
|
|
571
|
+
if path.name in _CONFIG_NAMES or path.suffix.lower() in _SOURCE_SUFFIXES:
|
|
572
|
+
result.append(path)
|
|
573
|
+
return result
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def _kind(name: str) -> RequirementKind:
|
|
577
|
+
if name in _HARNESS_PROVIDED:
|
|
578
|
+
return RequirementKind.HARNESS_INFRASTRUCTURE
|
|
579
|
+
if _SECRET_NAME.search(name):
|
|
580
|
+
return RequirementKind.SECRET
|
|
581
|
+
return RequirementKind.CONFIGURATION
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
def _provider(name: str) -> tuple[str, str]:
|
|
585
|
+
for prefixes, provider, purpose in _PROVIDERS:
|
|
586
|
+
if name.startswith(prefixes):
|
|
587
|
+
return provider, purpose
|
|
588
|
+
if name in _HARNESS_PROVIDED:
|
|
589
|
+
return "harness", "generated test infrastructure"
|
|
590
|
+
return "agent", "agent runtime configuration"
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
def _secret_type(name: str) -> str:
|
|
594
|
+
if "PRIVATE_KEY" in name or name.endswith("CREDENTIALS"):
|
|
595
|
+
return "credential_file"
|
|
596
|
+
if "TOKEN" in name:
|
|
597
|
+
return "token"
|
|
598
|
+
if "SECRET" in name or "PASSWORD" in name:
|
|
599
|
+
return "secret"
|
|
600
|
+
return "api_key"
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
def _credential_choices(
|
|
604
|
+
requirements: list[CredentialRequirement],
|
|
605
|
+
) -> list[CredentialChoice]:
|
|
606
|
+
"""Describe common provider auth alternatives without reading secret values."""
|
|
607
|
+
by_name = {item.environment_name: item for item in requirements}
|
|
608
|
+
google_options = [
|
|
609
|
+
["GEMINI_API_KEY"],
|
|
610
|
+
["GOOGLE_API_KEY"],
|
|
611
|
+
["GOOGLE_APPLICATION_CREDENTIALS", "GOOGLE_CLOUD_PROJECT"],
|
|
612
|
+
["GOOGLE_APPLICATION_CREDENTIALS_JSON", "GOOGLE_CLOUD_PROJECT"],
|
|
613
|
+
]
|
|
614
|
+
# Repositories often keep more than one model backend behind LLM_PROVIDER. A strict
|
|
615
|
+
# credential read inside one branch (or a standalone bakeoff utility in the same runtime
|
|
616
|
+
# tree) must not make every backend credential mandatory. The selected provider's one
|
|
617
|
+
# complete authentication route is the requirement. Without an explicit provider selector
|
|
618
|
+
# we remain conservative and do not merge independent integrations into one choice.
|
|
619
|
+
if "LLM_PROVIDER" in by_name:
|
|
620
|
+
google_options.extend(
|
|
621
|
+
[option for option in (["OPENAI_API_KEY"], ["AGENTCC_API_KEY"])]
|
|
622
|
+
)
|
|
623
|
+
present_options = [
|
|
624
|
+
option for option in google_options if all(name in by_name for name in option)
|
|
625
|
+
]
|
|
626
|
+
google_names = {
|
|
627
|
+
"GEMINI_API_KEY",
|
|
628
|
+
"GOOGLE_API_KEY",
|
|
629
|
+
"GOOGLE_APPLICATION_CREDENTIALS",
|
|
630
|
+
"GOOGLE_APPLICATION_CREDENTIALS_JSON",
|
|
631
|
+
}
|
|
632
|
+
if len(present_options) < 2 or not any(
|
|
633
|
+
any(name in google_names for name in option) for option in present_options
|
|
634
|
+
):
|
|
635
|
+
return []
|
|
636
|
+
configured = {
|
|
637
|
+
RequirementStatus.CONFIGURED,
|
|
638
|
+
RequirementStatus.HARNESS_PROVIDED,
|
|
639
|
+
}
|
|
640
|
+
satisfied = any(
|
|
641
|
+
all(by_name[name].status in configured for name in option)
|
|
642
|
+
for option in present_options
|
|
643
|
+
)
|
|
644
|
+
if not satisfied:
|
|
645
|
+
# These variables are optional individually but one complete route is
|
|
646
|
+
# mandatory. Mark the members input-capable; the choice prevents the UI
|
|
647
|
+
# from requiring every alternative.
|
|
648
|
+
for name in {name for option in present_options for name in option}:
|
|
649
|
+
if by_name[name].status not in configured:
|
|
650
|
+
by_name[name].required = True
|
|
651
|
+
by_name[name].status = RequirementStatus.MISSING
|
|
652
|
+
return [
|
|
653
|
+
CredentialChoice(
|
|
654
|
+
id="google_model_auth",
|
|
655
|
+
purpose="authenticate the Google/Gemini model runtime",
|
|
656
|
+
options=present_options,
|
|
657
|
+
satisfied=satisfied,
|
|
658
|
+
)
|
|
659
|
+
]
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
def _identifier(value: str) -> str:
|
|
663
|
+
return re.sub(r"[^a-z0-9]+", "_", value.lower()).strip("_")
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
__all__ = [
|
|
667
|
+
"CREDENTIAL_MANIFEST_VERSION",
|
|
668
|
+
"CredentialManifest",
|
|
669
|
+
"CredentialChoice",
|
|
670
|
+
"CredentialRequirement",
|
|
671
|
+
"RequirementKind",
|
|
672
|
+
"RequirementStatus",
|
|
673
|
+
"discover_credentials",
|
|
674
|
+
]
|