usersim 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- usersim/__init__.py +23 -0
- usersim/asset_gen/__init__.py +22 -0
- usersim/asset_gen/financial_services/__init__.py +22 -0
- usersim/asset_gen/financial_services/evaluate.py +330 -0
- usersim/asset_gen/financial_services/pipeline.py +1590 -0
- usersim/asset_gen/financial_services/prompts.py +318 -0
- usersim/asset_gen/financial_services/spec.py +794 -0
- usersim/asset_gen/financial_services/validate.py +750 -0
- usersim/asset_gen/registry.py +191 -0
- usersim/cli/__init__.py +185 -0
- usersim/cli/__main__.py +13 -0
- usersim/cli/_async.py +35 -0
- usersim/cli/_env.py +74 -0
- usersim/cli/_errors.py +44 -0
- usersim/cli/_inputs.py +691 -0
- usersim/cli/_models.py +1222 -0
- usersim/cli/_ngc.py +441 -0
- usersim/cli/_persona_language.py +316 -0
- usersim/cli/_pipeline.py +1007 -0
- usersim/cli/evaluate.py +415 -0
- usersim/cli/evaluate_assets.py +115 -0
- usersim/cli/gen_assets.py +120 -0
- usersim/cli/model_catalog.py +451 -0
- usersim/cli/models_default.toml +46 -0
- usersim/cli/models_openai.toml +26 -0
- usersim/cli/models_openrouter.toml +33 -0
- usersim/cli/panel.py +193 -0
- usersim/cli/report.py +247 -0
- usersim/cli/rescore.py +223 -0
- usersim/cli/select.py +304 -0
- usersim/cli/setup_ngc.py +131 -0
- usersim/cli/simulate.py +866 -0
- usersim/cli/smoke.py +1019 -0
- usersim/cli/validate_assets.py +64 -0
- usersim/engine/__init__.py +4 -0
- usersim/engine/assets/financial_services/SCHEMA.md +320 -0
- usersim/engine/assets/financial_services/de_DE/_generation_report.json +945 -0
- usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/corpus.yaml +8006 -0
- usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/institution_meta.yaml +9 -0
- usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/tasks.yaml +366 -0
- usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/tools.yaml +114 -0
- usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/corpus.yaml +10053 -0
- usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/tasks.yaml +578 -0
- usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/tools.yaml +154 -0
- usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/corpus.yaml +6784 -0
- usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/tasks.yaml +394 -0
- usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/tools.yaml +134 -0
- usersim/engine/assets/financial_services/de_DE/dynamic.yaml +252 -0
- usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/corpus.yaml +9404 -0
- usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/tasks.yaml +394 -0
- usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/tools.yaml +114 -0
- usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/corpus.yaml +8181 -0
- usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/tasks.yaml +440 -0
- usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/tools.yaml +144 -0
- usersim/engine/assets/financial_services/de_DE/region_meta.yaml +108 -0
- usersim/engine/assets/financial_services/en_GB/_generation_report.json +1330 -0
- usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/corpus.yaml +11053 -0
- usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/institution_meta.yaml +9 -0
- usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/tasks.yaml +401 -0
- usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/tools.yaml +114 -0
- usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/corpus.yaml +13830 -0
- usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/institution_meta.yaml +11 -0
- usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/tasks.yaml +597 -0
- usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/tools.yaml +159 -0
- usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/corpus.yaml +9717 -0
- usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/tasks.yaml +418 -0
- usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/tools.yaml +139 -0
- usersim/engine/assets/financial_services/en_GB/dynamic.yaml +309 -0
- usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/corpus.yaml +7891 -0
- usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/tasks.yaml +345 -0
- usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/tools.yaml +114 -0
- usersim/engine/assets/financial_services/en_GB/neobank/quill/corpus.yaml +9579 -0
- usersim/engine/assets/financial_services/en_GB/neobank/quill/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/en_GB/neobank/quill/tasks.yaml +453 -0
- usersim/engine/assets/financial_services/en_GB/neobank/quill/tools.yaml +154 -0
- usersim/engine/assets/financial_services/en_GB/region_meta.yaml +128 -0
- usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/corpus.yaml +8028 -0
- usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/tasks.yaml +387 -0
- usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/tools.yaml +124 -0
- usersim/engine/assets/financial_services/en_IN/_generation_report.json +977 -0
- usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/corpus.yaml +9170 -0
- usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/tasks.yaml +330 -0
- usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/tools.yaml +139 -0
- usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/corpus.yaml +7436 -0
- usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/tasks.yaml +323 -0
- usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/tools.yaml +129 -0
- usersim/engine/assets/financial_services/en_IN/dynamic.yaml +256 -0
- usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/corpus.yaml +7641 -0
- usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/tasks.yaml +350 -0
- usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/tools.yaml +114 -0
- usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/corpus.yaml +9453 -0
- usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/institution_meta.yaml +9 -0
- usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/tasks.yaml +346 -0
- usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/tools.yaml +114 -0
- usersim/engine/assets/financial_services/en_IN/neobank/paysetu/corpus.yaml +8237 -0
- usersim/engine/assets/financial_services/en_IN/neobank/paysetu/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/en_IN/neobank/paysetu/tasks.yaml +313 -0
- usersim/engine/assets/financial_services/en_IN/neobank/paysetu/tools.yaml +139 -0
- usersim/engine/assets/financial_services/en_IN/region_meta.yaml +81 -0
- usersim/engine/assets/financial_services/en_US/_generation_report.json +768 -0
- usersim/engine/assets/financial_services/en_US/bank/northwind_bank/corpus.yaml +9131 -0
- usersim/engine/assets/financial_services/en_US/bank/northwind_bank/institution_meta.yaml +9 -0
- usersim/engine/assets/financial_services/en_US/bank/northwind_bank/tasks.yaml +473 -0
- usersim/engine/assets/financial_services/en_US/bank/northwind_bank/tools.yaml +129 -0
- usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/corpus.yaml +7868 -0
- usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/tasks.yaml +388 -0
- usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/tools.yaml +124 -0
- usersim/engine/assets/financial_services/en_US/dynamic.yaml +198 -0
- usersim/engine/assets/financial_services/en_US/region_meta.yaml +58 -0
- usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/corpus.yaml +8117 -0
- usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/tasks.yaml +369 -0
- usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/tools.yaml +109 -0
- usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/corpus.yaml +6897 -0
- usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/tasks.yaml +319 -0
- usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/tools.yaml +109 -0
- usersim/engine/assets/financial_services/fr_FR/_generation_report.json +1041 -0
- usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/corpus.yaml +10842 -0
- usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/tasks.yaml +529 -0
- usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/tools.yaml +144 -0
- usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/corpus.yaml +9091 -0
- usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/tasks.yaml +511 -0
- usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/tools.yaml +129 -0
- usersim/engine/assets/financial_services/fr_FR/dynamic.yaml +264 -0
- usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/corpus.yaml +9033 -0
- usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/tasks.yaml +505 -0
- usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/tools.yaml +134 -0
- usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/corpus.yaml +7494 -0
- usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/institution_meta.yaml +9 -0
- usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/tasks.yaml +383 -0
- usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/tools.yaml +114 -0
- usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/corpus.yaml +7883 -0
- usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/tasks.yaml +430 -0
- usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/tools.yaml +139 -0
- usersim/engine/assets/financial_services/fr_FR/region_meta.yaml +114 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/_generation_report.json +977 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/corpus.yaml +6265 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/institution_meta.yaml +11 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/tools.yaml +139 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/corpus.yaml +5188 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/institution_meta.yaml +9 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/tools.yaml +129 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/corpus.yaml +5115 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/institution_meta.yaml +9 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/tools.yaml +114 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/corpus.yaml +6539 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/tools.yaml +114 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/neobank/paysetu/corpus.yaml +5370 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/neobank/paysetu/institution_meta.yaml +11 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/neobank/paysetu/tools.yaml +139 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/region_meta.yaml +81 -0
- usersim/engine/assets/general_educational/prompts.yaml +115 -0
- usersim/engine/assets/general_educational/subjects/base/subjects.json +162 -0
- usersim/engine/assets/general_educational/subjects/de_DE/subjects.json +162 -0
- usersim/engine/assets/general_educational/subjects/en_GB/subjects.json +162 -0
- usersim/engine/assets/general_educational/subjects/fr_FR/subjects.json +16 -0
- usersim/engine/assets/general_educational/subjects/hi_Deva_IN/subjects.json +14 -0
- usersim/engine/assets/general_educational/subjects/hi_Latn_IN/subjects.json +14 -0
- usersim/engine/assets/general_educational/subjects/ja_JP/subjects.json +16 -0
- usersim/engine/assets/general_educational/subjects/ko_KR/subjects.json +16 -0
- usersim/engine/assets/general_educational/subjects/pt_BR/subjects.json +12 -0
- usersim/engine/assets/general_open_ended/prompts.yaml +117 -0
- usersim/engine/assets/general_open_ended/topics/base/topics.json +194 -0
- usersim/engine/assets/general_open_ended/topics/de_DE/topics.json +194 -0
- usersim/engine/assets/general_open_ended/topics/en_GB/topics.json +194 -0
- usersim/engine/assets/general_open_ended/topics/fr_FR/topics.json +22 -0
- usersim/engine/assets/general_open_ended/topics/hi_Deva_IN/topics.json +22 -0
- usersim/engine/assets/general_open_ended/topics/hi_Latn_IN/topics.json +22 -0
- usersim/engine/assets/general_open_ended/topics/ja_JP/topics.json +17 -0
- usersim/engine/assets/general_open_ended/topics/ko_KR/topics.json +17 -0
- usersim/engine/assets/general_open_ended/topics/pt_BR/topics.json +22 -0
- usersim/engine/assets/health_decision_support_disclosure/sample.yaml +21 -0
- usersim/engine/assets/health_general_disclosure/sample.yaml +21 -0
- usersim/engine/assets/health_therapy_disclosure/sample.yaml +23 -0
- usersim/engine/assets/health_triage_disclosure/sample.yaml +21 -0
- usersim/engine/assets/identity_disclosure/SCHEMA.md +303 -0
- usersim/engine/assets/identity_disclosure/spec.yaml +1007 -0
- usersim/engine/assets/safety_agentic/SCHEMA.md +252 -0
- usersim/engine/assets/safety_agentic/action_taxonomy.yaml +682 -0
- usersim/engine/assets/safety_agentic/sample.yaml +3993 -0
- usersim/engine/assets/safety_chat_pressure/SCHEMA.md +181 -0
- usersim/engine/assets/safety_chat_pressure/sample.yaml +3646 -0
- usersim/engine/assets/sov_ai_dynamic/SCHEMA.md +98 -0
- usersim/engine/assets/sov_ai_dynamic/de_DE/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/en_GB/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/en_IN/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/en_SG/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/en_US/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/fr_FR/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/hi_Deva_IN/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/hi_Latn_IN/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/ja_JP/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/ko_KR/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/pt_BR/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_facts/SCHEMA.md +196 -0
- usersim/engine/assets/sov_ai_facts/de_DE/sample.yaml +1421 -0
- usersim/engine/assets/sov_ai_facts/en_GB/sample.yaml +1389 -0
- usersim/engine/assets/sov_ai_facts/en_IN/sample.yaml +1275 -0
- usersim/engine/assets/sov_ai_facts/en_SG/sample.yaml +1358 -0
- usersim/engine/assets/sov_ai_facts/en_US/sample.yaml +1270 -0
- usersim/engine/assets/sov_ai_facts/fr_FR/sample.yaml +1287 -0
- usersim/engine/assets/sov_ai_facts/hi_Deva_IN/sample.yaml +1272 -0
- usersim/engine/assets/sov_ai_facts/hi_Latn_IN/sample.yaml +1272 -0
- usersim/engine/assets/sov_ai_facts/ja_JP/sample.yaml +1294 -0
- usersim/engine/assets/sov_ai_facts/ko_KR/sample.yaml +1435 -0
- usersim/engine/assets/sov_ai_facts/pt_BR/sample.yaml +1321 -0
- usersim/engine/assets/sov_ai_multilingual_parity/SCHEMA.md +137 -0
- usersim/engine/assets/sov_ai_multilingual_parity/sample.yaml +1210 -0
- usersim/engine/assets/tool_calling/README.md +225 -0
- usersim/engine/assets/tool_calling/prompts.yaml +171 -0
- usersim/engine/assets/tool_calling/toolsets_seed.parquet +0 -0
- usersim/engine/config.py +296 -0
- usersim/engine/core/__init__.py +4 -0
- usersim/engine/core/_assets.py +178 -0
- usersim/engine/core/_extensions.py +148 -0
- usersim/engine/core/action_taxonomy.py +347 -0
- usersim/engine/core/agentic_bank.py +725 -0
- usersim/engine/core/analysis.py +60 -0
- usersim/engine/core/backends.py +180 -0
- usersim/engine/core/behavioral.py +755 -0
- usersim/engine/core/clinical_profile_bank.py +245 -0
- usersim/engine/core/context.py +137 -0
- usersim/engine/core/embeddings.py +83 -0
- usersim/engine/core/episode_input.py +281 -0
- usersim/engine/core/episode_runtime.py +735 -0
- usersim/engine/core/fact_bank.py +500 -0
- usersim/engine/core/finance_bank.py +970 -0
- usersim/engine/core/finance_tasks.py +292 -0
- usersim/engine/core/idempotency.py +62 -0
- usersim/engine/core/identity.py +215 -0
- usersim/engine/core/identity_spec.py +659 -0
- usersim/engine/core/identity_spec_types.py +425 -0
- usersim/engine/core/judges.py +85 -0
- usersim/engine/core/language_detection.py +239 -0
- usersim/engine/core/llm.py +710 -0
- usersim/engine/core/locale.py +512 -0
- usersim/engine/core/localized.py +172 -0
- usersim/engine/core/manifest.py +998 -0
- usersim/engine/core/messages.py +93 -0
- usersim/engine/core/missing.py +38 -0
- usersim/engine/core/outcomes.py +442 -0
- usersim/engine/core/persona.py +178 -0
- usersim/engine/core/pressure_bank.py +629 -0
- usersim/engine/core/preview.py +1927 -0
- usersim/engine/core/probes.py +1499 -0
- usersim/engine/core/probing_taxonomy.py +462 -0
- usersim/engine/core/prompt_loader.py +93 -0
- usersim/engine/core/prompts.py +81 -0
- usersim/engine/core/provenance.py +118 -0
- usersim/engine/core/query_bank.py +567 -0
- usersim/engine/core/realized_audit.py +411 -0
- usersim/engine/core/row_metadata.py +35 -0
- usersim/engine/core/seeds.py +82 -0
- usersim/engine/core/simulation.py +1899 -0
- usersim/engine/core/storage.py +1407 -0
- usersim/engine/core/tool_calls.py +106 -0
- usersim/engine/core/translation.py +266 -0
- usersim/engine/core/user_turn_policy.py +80 -0
- usersim/engine/evaluator/__init__.py +19 -0
- usersim/engine/evaluator/axes.py +437 -0
- usersim/engine/evaluator/config.py +187 -0
- usersim/engine/evaluator/generator.py +424 -0
- usersim/engine/evaluator/judges.py +151 -0
- usersim/engine/evaluator/prompts.py +108 -0
- usersim/engine/evaluator/scorers/__init__.py +289 -0
- usersim/engine/evaluator/scorers/financial_services.py +635 -0
- usersim/engine/evaluator/scorers/health_disclosure.py +503 -0
- usersim/engine/evaluator/scorers/identity_disclosure.py +545 -0
- usersim/engine/evaluator/scorers/identity_grading.py +543 -0
- usersim/engine/evaluator/scorers/language_compliance.py +426 -0
- usersim/engine/evaluator/scorers/refusal_basics.py +479 -0
- usersim/engine/evaluator/scorers/response_shape.py +363 -0
- usersim/engine/evaluator/scorers/safety_agentic.py +1010 -0
- usersim/engine/evaluator/scorers/safety_chat_pressure.py +791 -0
- usersim/engine/evaluator/scorers/sov_ai_dynamic.py +629 -0
- usersim/engine/evaluator/scorers/sov_ai_facts.py +648 -0
- usersim/engine/evaluator/scorers/sov_ai_multilingual_parity.py +550 -0
- usersim/engine/evaluator/scorers/tool_use.py +291 -0
- usersim/engine/external.py +218 -0
- usersim/engine/generator.py +419 -0
- usersim/engine/plugin.py +34 -0
- usersim/engine/probes/__init__.py +28 -0
- usersim/engine/probes/financial_services/__init__.py +16 -0
- usersim/engine/probes/financial_services/generator.py +1329 -0
- usersim/engine/probes/financial_services/prompts.py +132 -0
- usersim/engine/probes/financial_services/retrieval.py +229 -0
- usersim/engine/probes/financial_services/task_derivation.py +274 -0
- usersim/engine/probes/financial_services/tools.py +116 -0
- usersim/engine/probes/general_educational/__init__.py +4 -0
- usersim/engine/probes/general_educational/generator.py +125 -0
- usersim/engine/probes/general_educational/prompts.py +207 -0
- usersim/engine/probes/general_open_ended/__init__.py +4 -0
- usersim/engine/probes/general_open_ended/generator.py +127 -0
- usersim/engine/probes/general_open_ended/prompts.py +209 -0
- usersim/engine/probes/health_disclosure/__init__.py +28 -0
- usersim/engine/probes/health_disclosure/base.py +148 -0
- usersim/engine/probes/health_disclosure/clients.py +291 -0
- usersim/engine/probes/health_disclosure/decision_support.py +44 -0
- usersim/engine/probes/health_disclosure/general.py +45 -0
- usersim/engine/probes/health_disclosure/mixin.py +363 -0
- usersim/engine/probes/health_disclosure/move_runtime.py +486 -0
- usersim/engine/probes/health_disclosure/moves.py +241 -0
- usersim/engine/probes/health_disclosure/prompts.py +324 -0
- usersim/engine/probes/health_disclosure/therapy.py +43 -0
- usersim/engine/probes/health_disclosure/triage.py +43 -0
- usersim/engine/probes/identity_disclosure/__init__.py +4 -0
- usersim/engine/probes/identity_disclosure/generator.py +504 -0
- usersim/engine/probes/identity_disclosure/prompts.py +219 -0
- usersim/engine/probes/identity_disclosure/task_derivation.py +227 -0
- usersim/engine/probes/safety_agentic/__init__.py +24 -0
- usersim/engine/probes/safety_agentic/generator.py +666 -0
- usersim/engine/probes/safety_agentic/prompts.py +24 -0
- usersim/engine/probes/safety_agentic/task_derivation.py +109 -0
- usersim/engine/probes/safety_chat_pressure/__init__.py +24 -0
- usersim/engine/probes/safety_chat_pressure/classifier.py +113 -0
- usersim/engine/probes/safety_chat_pressure/generator.py +551 -0
- usersim/engine/probes/safety_chat_pressure/prompts.py +141 -0
- usersim/engine/probes/safety_chat_pressure/task_derivation.py +171 -0
- usersim/engine/probes/sov_ai_dynamic/__init__.py +28 -0
- usersim/engine/probes/sov_ai_dynamic/generator.py +393 -0
- usersim/engine/probes/sov_ai_dynamic/prompts.py +472 -0
- usersim/engine/probes/sov_ai_dynamic/task_derivation.py +106 -0
- usersim/engine/probes/sov_ai_facts/__init__.py +15 -0
- usersim/engine/probes/sov_ai_facts/generator.py +342 -0
- usersim/engine/probes/sov_ai_facts/prompts.py +714 -0
- usersim/engine/probes/sov_ai_facts/task_derivation.py +479 -0
- usersim/engine/probes/sov_ai_multilingual_parity/__init__.py +19 -0
- usersim/engine/probes/sov_ai_multilingual_parity/generator.py +379 -0
- usersim/engine/probes/sov_ai_multilingual_parity/prompts.py +456 -0
- usersim/engine/probes/sov_ai_multilingual_parity/task_derivation.py +104 -0
- usersim/engine/probes/tool_calling/__init__.py +9 -0
- usersim/engine/probes/tool_calling/generator.py +437 -0
- usersim/engine/probes/tool_calling/prompts.py +221 -0
- usersim/engine/probes/tool_calling/verifiers.py +165 -0
- usersim/py.typed +0 -0
- usersim/reporting/__init__.py +116 -0
- usersim/reporting/_reasoning.py +201 -0
- usersim/reporting/capability_report.py +2049 -0
- usersim/reporting/comparison.py +797 -0
- usersim/reporting/comparison_dashboard.py +2938 -0
- usersim/reporting/dashboard.py +1731 -0
- usersim/reporting/diagnostics.py +435 -0
- usersim/reporting/eval_manifest.py +291 -0
- usersim/reporting/identity.py +82 -0
- usersim/reporting/resource.py +476 -0
- usersim/reporting/runs.py +282 -0
- usersim/reporting/sim_health.py +334 -0
- usersim/selection/__init__.py +87 -0
- usersim/selection/engine.py +423 -0
- usersim/selection/io.py +391 -0
- usersim/selection/profiles.py +249 -0
- usersim/selection/schemas.py +194 -0
- usersim/taxonomy/__init__.py +38 -0
- usersim/taxonomy/capabilities.py +1125 -0
- usersim/taxonomy/eval_cell.py +131 -0
- usersim/testing/__init__.py +534 -0
- usersim-0.2.0.dist-info/METADATA +392 -0
- usersim-0.2.0.dist-info/RECORD +379 -0
- usersim-0.2.0.dist-info/WHEEL +5 -0
- usersim-0.2.0.dist-info/entry_points.txt +6 -0
- usersim-0.2.0.dist-info/licenses/LICENSE +204 -0
- usersim-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Tool surface for the financial_services probe.
|
|
5
|
+
|
|
6
|
+
The assistant is offered the PERMANENT tools up front (``kb_search``,
|
|
7
|
+
``verify_identity``, ...) plus a single ``call_tool(tool_name, arguments)`` gate
|
|
8
|
+
for DISCOVERABLE tools. A discoverable tool is only usable once its
|
|
9
|
+
documentation has been retrieved (its name appears in a retrieved document's
|
|
10
|
+
``mentions_tools``). This module is pure helpers; the probe's
|
|
11
|
+
``after_assistant_turn`` tool loop owns state.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
from typing import Any, Sequence
|
|
18
|
+
|
|
19
|
+
from usersim.engine.core.finance_bank import (
|
|
20
|
+
Institution,
|
|
21
|
+
framework_primitive_tool,
|
|
22
|
+
verify_identity_schema,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
CALL_TOOL_NAME = "call_tool"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def build_offered_tools(
|
|
29
|
+
institution: Institution,
|
|
30
|
+
*,
|
|
31
|
+
identity_required_fields: Sequence[str] | None = None,
|
|
32
|
+
) -> list[dict[str, Any]]:
|
|
33
|
+
"""OpenAI-style tool schemas offered to the assistant, SCOPED to ``institution``.
|
|
34
|
+
|
|
35
|
+
Permanent (non-discoverable) tools are offered by name; discoverable tools
|
|
36
|
+
are reached only through the ``call_tool`` gate. Framework primitives
|
|
37
|
+
(``kb_search`` / ``verify_identity``) use the canonical schema from
|
|
38
|
+
``core.finance_bank`` so the assistant always sees their real parameters
|
|
39
|
+
even if a bank asset ships them empty.
|
|
40
|
+
|
|
41
|
+
``identity_required_fields`` specializes ``verify_identity`` to the region's
|
|
42
|
+
KYC contract (e.g. India also requires a PAN). Omit it for the universal
|
|
43
|
+
name + DOB default.
|
|
44
|
+
"""
|
|
45
|
+
offered: list[dict[str, Any]] = []
|
|
46
|
+
for t in institution.permanent_tools():
|
|
47
|
+
canonical = framework_primitive_tool(t.name)
|
|
48
|
+
if t.name == "verify_identity" and identity_required_fields:
|
|
49
|
+
canonical = verify_identity_schema(identity_required_fields)
|
|
50
|
+
if canonical is not None:
|
|
51
|
+
offered.append(
|
|
52
|
+
{
|
|
53
|
+
"type": "function",
|
|
54
|
+
"function": {
|
|
55
|
+
"name": t.name,
|
|
56
|
+
"description": canonical["description"],
|
|
57
|
+
"parameters": canonical["parameters"],
|
|
58
|
+
},
|
|
59
|
+
}
|
|
60
|
+
)
|
|
61
|
+
else:
|
|
62
|
+
offered.append(t.to_openai_tool())
|
|
63
|
+
offered.append(
|
|
64
|
+
{
|
|
65
|
+
"type": "function",
|
|
66
|
+
"function": {
|
|
67
|
+
"name": CALL_TOOL_NAME,
|
|
68
|
+
"description": (
|
|
69
|
+
"Invoke a tool documented in the knowledge base. You must first "
|
|
70
|
+
"retrieve the tool's documentation via kb_search before calling it."
|
|
71
|
+
),
|
|
72
|
+
"parameters": {
|
|
73
|
+
"type": "object",
|
|
74
|
+
"properties": {
|
|
75
|
+
"tool_name": {"type": "string"},
|
|
76
|
+
"arguments": {"type": "object"},
|
|
77
|
+
},
|
|
78
|
+
"required": ["tool_name"],
|
|
79
|
+
},
|
|
80
|
+
},
|
|
81
|
+
}
|
|
82
|
+
)
|
|
83
|
+
return offered
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def extract_call(tc: Any) -> tuple[str, dict[str, Any]]:
|
|
87
|
+
"""Pull ``(function_name, arguments_dict)`` from one tool_call dict."""
|
|
88
|
+
if not isinstance(tc, dict):
|
|
89
|
+
return ("<malformed>", {})
|
|
90
|
+
fn = tc.get("function")
|
|
91
|
+
if not isinstance(fn, dict):
|
|
92
|
+
return (str(tc.get("name") or "<unknown>"), {})
|
|
93
|
+
name = str(fn.get("name") or "<unknown>")
|
|
94
|
+
raw = fn.get("arguments")
|
|
95
|
+
if isinstance(raw, dict):
|
|
96
|
+
return (name, raw)
|
|
97
|
+
if isinstance(raw, str):
|
|
98
|
+
try:
|
|
99
|
+
parsed = json.loads(raw)
|
|
100
|
+
return (name, parsed if isinstance(parsed, dict) else {})
|
|
101
|
+
except (json.JSONDecodeError, TypeError):
|
|
102
|
+
return (name, {})
|
|
103
|
+
return (name, {})
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def resolve_invoked_tool(function_name: str, arguments: dict[str, Any]) -> tuple[str, dict[str, Any]]:
|
|
107
|
+
"""Map an assistant tool call to the *domain* tool it targets.
|
|
108
|
+
|
|
109
|
+
``call_tool(tool_name=..., arguments=...)`` targets the named discoverable
|
|
110
|
+
tool; any other function call targets itself (permanent tool or a
|
|
111
|
+
directly-named tool).
|
|
112
|
+
"""
|
|
113
|
+
if function_name == CALL_TOOL_NAME:
|
|
114
|
+
target = str(arguments.get("tool_name") or "<unknown>")
|
|
115
|
+
return (target, arguments.get("arguments") or {})
|
|
116
|
+
return (function_name, arguments)
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Educational probe (tutoring, explanation, practice).
|
|
5
|
+
|
|
6
|
+
A thin probe class plus a shim function: ``EducationalProbe`` defines
|
|
7
|
+
the user-agent prompt and delegates everything else to ``BaseProbe`` +
|
|
8
|
+
``ConversationLoop``; ``simulate_general_educational`` is the back-compat
|
|
9
|
+
shim that constructs the probe and runs it through ``run_dispatch``.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from usersim.engine.core.behavioral import (
|
|
17
|
+
format_behavioral_profile_for_prompt,
|
|
18
|
+
format_disclosure_instructions,
|
|
19
|
+
format_interaction_style_instructions,
|
|
20
|
+
)
|
|
21
|
+
from usersim.engine.core.messages import _parse_theme
|
|
22
|
+
from usersim.engine.core.persona import format_persona_for_prompt
|
|
23
|
+
from usersim.engine.core.probes import BaseProbe, dispatch_episode, register_probe
|
|
24
|
+
from usersim.engine.core.prompt_loader import render_prompt
|
|
25
|
+
from usersim.engine.core.simulation import language_instruction
|
|
26
|
+
from usersim.engine.probes.general_educational.prompts import (
|
|
27
|
+
assistant_system_prompt,
|
|
28
|
+
user_agent_system_prompt,
|
|
29
|
+
user_judge_turn_prompt,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
# ── Probe metadata ──────────────────────────────────────────────────
|
|
33
|
+
# Bumping ``PROMPT_VERSION`` invalidates prior evaluator rows on
|
|
34
|
+
# re-run, since the dedup key includes it. Bump on any semantic change
|
|
35
|
+
# to the probe's prompts or task derivation.
|
|
36
|
+
|
|
37
|
+
PROBE_FAMILY: str = "general_educational"
|
|
38
|
+
PROBE_VARIANTS: list[str] = ["default"] # subject-conditioned via theme; single variant today
|
|
39
|
+
PROMPT_VERSION: str = "v1.0"
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@register_probe(family=PROBE_FAMILY, prompt_version=PROMPT_VERSION, variants=tuple(PROBE_VARIANTS))
|
|
43
|
+
class EducationalProbe(BaseProbe):
|
|
44
|
+
"""Educational / tutoring probe.
|
|
45
|
+
|
|
46
|
+
Test the assistant's tutoring / explanation / practice behavior
|
|
47
|
+
with persona-grounded, subject-conditioned conversations. No
|
|
48
|
+
tools; no curated asset bank; quality judgment lives in the
|
|
49
|
+
out-of-sim trajectory evaluator.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
label = "general_educational"
|
|
53
|
+
|
|
54
|
+
def __init__(self, **kwargs: Any) -> None:
|
|
55
|
+
super().__init__(**kwargs)
|
|
56
|
+
theme = _parse_theme(self._data.get(getattr(self._cfg, "theme_column", "theme"), "{}"))
|
|
57
|
+
# Parsed once; the prompt itself is rendered per call in
|
|
58
|
+
# get_user_system_prompt so all three prompt hooks resolve alike.
|
|
59
|
+
self._subject = theme.get("description", theme.get("type", "general subject"))
|
|
60
|
+
self._disclosure_style = self._data.get("disclosure_style", "upfront")
|
|
61
|
+
|
|
62
|
+
def get_user_system_prompt(self) -> str:
|
|
63
|
+
return render_prompt(
|
|
64
|
+
user_agent_system_prompt(),
|
|
65
|
+
self._data,
|
|
66
|
+
persona=format_persona_for_prompt(self._persona),
|
|
67
|
+
subject=self._subject,
|
|
68
|
+
language_instruction=language_instruction(self._language, self._locale),
|
|
69
|
+
behavioral_instructions=format_behavioral_profile_for_prompt(
|
|
70
|
+
self._profile,
|
|
71
|
+
language=self._language,
|
|
72
|
+
),
|
|
73
|
+
disclosure_instructions=format_disclosure_instructions(
|
|
74
|
+
self._disclosure_style,
|
|
75
|
+
),
|
|
76
|
+
interaction_style_instructions=format_interaction_style_instructions(
|
|
77
|
+
self._interaction_style,
|
|
78
|
+
),
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
def get_assistant_system_prompt(self) -> str:
|
|
82
|
+
# Empty as shipped; see the accessor's docstring.
|
|
83
|
+
return render_prompt(assistant_system_prompt(), self._data)
|
|
84
|
+
|
|
85
|
+
def format_gate_prompt(
|
|
86
|
+
self,
|
|
87
|
+
user_query: str,
|
|
88
|
+
conversation_history: str,
|
|
89
|
+
) -> str:
|
|
90
|
+
return render_prompt(
|
|
91
|
+
user_judge_turn_prompt(),
|
|
92
|
+
self._data,
|
|
93
|
+
conversation_history=conversation_history,
|
|
94
|
+
user_turn_to_evaluate=user_query,
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
async def simulate_general_educational(
|
|
99
|
+
models: dict,
|
|
100
|
+
data: dict,
|
|
101
|
+
persona: dict,
|
|
102
|
+
profile: dict,
|
|
103
|
+
locale: str,
|
|
104
|
+
language: str,
|
|
105
|
+
cfg: Any,
|
|
106
|
+
**kwargs: Any,
|
|
107
|
+
) -> dict:
|
|
108
|
+
"""Thin shim for callers that import ``simulate_general_educational`` directly.
|
|
109
|
+
|
|
110
|
+
See ``usersim.engine.probes.general_open_ended.generator.simulate_general_open_ended``
|
|
111
|
+
for the rationale; preserved for backward-compat with the dispatcher
|
|
112
|
+
bootstrap and per-probe tests.
|
|
113
|
+
"""
|
|
114
|
+
probe = EducationalProbe(
|
|
115
|
+
persona=persona,
|
|
116
|
+
locale=locale,
|
|
117
|
+
language=language,
|
|
118
|
+
models=models,
|
|
119
|
+
cfg=cfg,
|
|
120
|
+
provenance=kwargs.get("provenance"),
|
|
121
|
+
profile=profile,
|
|
122
|
+
data=data,
|
|
123
|
+
outcome_builder=kwargs.get("outcome_builder"),
|
|
124
|
+
)
|
|
125
|
+
return await dispatch_episode(probe, models=models, data=data, cfg=cfg)
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Prompt templates for the general_educational probe.
|
|
5
|
+
|
|
6
|
+
Templates are loaded from assets/general_educational/prompts.yaml when available,
|
|
7
|
+
falling back to the compiled-in defaults below.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from usersim.engine.core.prompt_loader import get_prompt
|
|
11
|
+
|
|
12
|
+
_SCENARIO = "general_educational"
|
|
13
|
+
|
|
14
|
+
# ── Compiled-in defaults ──────────────────────────────────────────────────
|
|
15
|
+
|
|
16
|
+
_DEFAULT_USER_AGENT = """You are roleplaying as a learner seeking help understanding a topic.
|
|
17
|
+
You are talking to an AI tutor/assistant.
|
|
18
|
+
|
|
19
|
+
<INSTRUCTIONS>
|
|
20
|
+
As a learner:
|
|
21
|
+
- Ask for explanations at your level of understanding.
|
|
22
|
+
- When confused, ask follow-up questions or request simpler explanations.
|
|
23
|
+
- Attempt practice problems when offered, and share your attempts.
|
|
24
|
+
- Express when something clicks or when you're still confused.
|
|
25
|
+
- Share relevant context from your background that might help the tutor adapt.
|
|
26
|
+
- If the tutor's response does not address what you asked, or does not make sense to you, say so — a real learner would not silently accept a non-answer.
|
|
27
|
+
|
|
28
|
+
Conversational style:
|
|
29
|
+
- Do NOT reveal you are role-playing.
|
|
30
|
+
- Do NOT use stage directions or bracketed actions.
|
|
31
|
+
- Match your vocabulary to your persona's education level.
|
|
32
|
+
- Show genuine curiosity and learning behaviors.
|
|
33
|
+
- If the explanation is too complex, say so naturally.
|
|
34
|
+
- If the explanation is too simple, ask for more depth.
|
|
35
|
+
|
|
36
|
+
REMEMBER: YOU ARE THE LEARNER. Stay in character throughout.
|
|
37
|
+
</INSTRUCTIONS>
|
|
38
|
+
|
|
39
|
+
{language_instruction}
|
|
40
|
+
|
|
41
|
+
Act as the following learner:
|
|
42
|
+
<YOUR_PERSONA>
|
|
43
|
+
{persona}
|
|
44
|
+
</YOUR_PERSONA>
|
|
45
|
+
|
|
46
|
+
{behavioral_instructions}
|
|
47
|
+
|
|
48
|
+
{disclosure_instructions}
|
|
49
|
+
|
|
50
|
+
{interaction_style_instructions}
|
|
51
|
+
|
|
52
|
+
You want to learn about this subject:
|
|
53
|
+
<SUBJECT>
|
|
54
|
+
{subject}
|
|
55
|
+
</SUBJECT>"""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
_DEFAULT_USER_JUDGE_RUBRIC = """Criteria for success:
|
|
59
|
+
- The learner's message is natural for their education level and background.
|
|
60
|
+
- The learner asks relevant questions or makes reasonable attempts.
|
|
61
|
+
- The learner's confusion or understanding is realistic for their persona.
|
|
62
|
+
|
|
63
|
+
Criteria for failure:
|
|
64
|
+
- The learner breaks character or acts like a teacher/assistant.
|
|
65
|
+
- The learner demonstrates knowledge inconsistent with their persona.
|
|
66
|
+
- The message is incoherent or unrelated to the learning topic.
|
|
67
|
+
- The learner offers help, assistance, or expertise to the tutor — a real learner does not offer to help the teacher.
|
|
68
|
+
- The learner references the conversation itself, the simulation, or how messages are being constructed — this is a fourth-wall break."""
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
_DEFAULT_TRAJECTORY_RUBRIC = """Evaluate the overall quality of this tutoring conversation.
|
|
72
|
+
|
|
73
|
+
SUCCESS criteria:
|
|
74
|
+
- The tutor adapted explanations to the learner's level.
|
|
75
|
+
- The tutor used appropriate scaffolding and examples.
|
|
76
|
+
- The conversation progressed the learner's understanding.
|
|
77
|
+
- The tutor was encouraging without being condescending.
|
|
78
|
+
- When the learner's question was vague or overly broad, the tutor asked what
|
|
79
|
+
specifically they wanted to learn instead of providing an overwhelming overview.
|
|
80
|
+
- The learner-LLM stayed in character as a realistic learner — only asking
|
|
81
|
+
questions, attempting exercises, expressing confusion, and providing requested info.
|
|
82
|
+
|
|
83
|
+
FAILURE criteria:
|
|
84
|
+
- The tutor provided incorrect information.
|
|
85
|
+
- The tutor was condescending, dismissive, or unhelpful.
|
|
86
|
+
- Explanations were consistently too complex or too simple for the learner.
|
|
87
|
+
- The conversation went in circles without progress.
|
|
88
|
+
- The tutor dumped an excessive amount of information instead of narrowing down
|
|
89
|
+
a vague or overly broad question from the learner.
|
|
90
|
+
- The tutor produced an unintelligible, empty, or non-responsive reply to the
|
|
91
|
+
learner's question in any turn.
|
|
92
|
+
- Role inversion: the learner-LLM took over the tutor's job — e.g., explaining
|
|
93
|
+
concepts, providing structured data, or teaching the tutor. A real learner
|
|
94
|
+
asks for help; they do not lecture back.
|
|
95
|
+
- The learner-LLM offered help or expertise to the tutor, reversing the
|
|
96
|
+
learner/teacher dynamic.
|
|
97
|
+
- The learner-LLM broke the fourth wall by referencing the conversation structure,
|
|
98
|
+
the simulation, or how messages should be written.
|
|
99
|
+
- The learner-LLM broke character (gave tutor-like responses, used structured
|
|
100
|
+
output, markdown, tables, or numbered lists)."""
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
_INLINE_JUDGE_RESPONSE_FORMAT = """
|
|
104
|
+
Strictly respond in the following format:
|
|
105
|
+
<explanation>
|
|
106
|
+
[Your detailed explanation and justification for the rating.]
|
|
107
|
+
</explanation>
|
|
108
|
+
<rating>
|
|
109
|
+
[Must be either 'success' or 'failure' strictly]
|
|
110
|
+
</rating>"""
|
|
111
|
+
|
|
112
|
+
# ── Public templates (YAML overrides compiled-in defaults) ────────────────
|
|
113
|
+
|
|
114
|
+
EDUCATIONAL_USER_AGENT_SYSTEM_PROMPT = get_prompt(_SCENARIO, "user_agent_system_prompt", _DEFAULT_USER_AGENT)
|
|
115
|
+
|
|
116
|
+
EDUCATIONAL_USER_JUDGE_RUBRIC = get_prompt(_SCENARIO, "user_judge_rubric", _DEFAULT_USER_JUDGE_RUBRIC)
|
|
117
|
+
|
|
118
|
+
EDUCATIONAL_TRAJECTORY_RUBRIC = get_prompt(_SCENARIO, "trajectory_rubric", _DEFAULT_TRAJECTORY_RUBRIC)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
USER_JUDGE_TURN_PROMPT = (
|
|
122
|
+
"""You are an expert evaluator. Judge the learner's message in this tutoring conversation.
|
|
123
|
+
|
|
124
|
+
Previous conversation:
|
|
125
|
+
<PREV_CONVERSATION>
|
|
126
|
+
{conversation_history}
|
|
127
|
+
</PREV_CONVERSATION>
|
|
128
|
+
|
|
129
|
+
Learner message to judge:
|
|
130
|
+
<LEARNER_MESSAGE>
|
|
131
|
+
{user_turn_to_evaluate}
|
|
132
|
+
</LEARNER_MESSAGE>
|
|
133
|
+
|
|
134
|
+
<RUBRIC>
|
|
135
|
+
"""
|
|
136
|
+
+ EDUCATIONAL_USER_JUDGE_RUBRIC
|
|
137
|
+
+ """
|
|
138
|
+
</RUBRIC>
|
|
139
|
+
"""
|
|
140
|
+
+ _INLINE_JUDGE_RESPONSE_FORMAT
|
|
141
|
+
)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
TRAJECTORY_LEVEL_JUDGE_TURN_PROMPT = (
|
|
145
|
+
"""You are an expert evaluator. Judge this entire tutoring conversation.
|
|
146
|
+
|
|
147
|
+
<CONVERSATION>
|
|
148
|
+
{conversation_history}
|
|
149
|
+
</CONVERSATION>
|
|
150
|
+
|
|
151
|
+
<RUBRIC>
|
|
152
|
+
"""
|
|
153
|
+
+ EDUCATIONAL_TRAJECTORY_RUBRIC
|
|
154
|
+
+ """
|
|
155
|
+
</RUBRIC>
|
|
156
|
+
"""
|
|
157
|
+
+ _INLINE_JUDGE_RESPONSE_FORMAT
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
# ── Call-time resolution ─────────────────────────────────────────────
|
|
162
|
+
# The module-level constants above bind whatever asset root was active at
|
|
163
|
+
# IMPORT time. Probes are imported at plugin bootstrap, before a run's
|
|
164
|
+
# ``assets_dir`` is known, so a per-run root would never reach them. These
|
|
165
|
+
# accessors re-resolve on every call, which is what makes
|
|
166
|
+
# ``--assets-dir`` / ``USERSIM_ASSETS_DIR`` govern prompts the same way it
|
|
167
|
+
# already governs seeds and banks. The constants are kept for callers that
|
|
168
|
+
# import them directly.
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def user_agent_system_prompt() -> str:
|
|
172
|
+
"""Resolve ``user_agent_system_prompt`` against the currently-active asset root."""
|
|
173
|
+
return get_prompt(_SCENARIO, "user_agent_system_prompt", _DEFAULT_USER_AGENT)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def assistant_system_prompt() -> str:
|
|
177
|
+
"""Resolve ``assistant_system_prompt`` against the currently-active asset root.
|
|
178
|
+
|
|
179
|
+
Defaults to ``""``: the assistant under test gets no system prompt, so
|
|
180
|
+
that whether it behaves helpfully, answers in the right language or
|
|
181
|
+
refuses is what the probe measures rather than what it was told. See
|
|
182
|
+
the pure-capability policy in ``docs/engine/README.md``.
|
|
183
|
+
Resolved like every other prompt so the mechanism is uniform, not
|
|
184
|
+
because filling it in is expected -- a non-empty value here changes
|
|
185
|
+
what the probe measures and wants a rationale in the module docstring.
|
|
186
|
+
"""
|
|
187
|
+
return get_prompt(_SCENARIO, "assistant_system_prompt", "")
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def user_judge_rubric() -> str:
|
|
191
|
+
"""Resolve ``user_judge_rubric`` against the currently-active asset root."""
|
|
192
|
+
return get_prompt(_SCENARIO, "user_judge_rubric", _DEFAULT_USER_JUDGE_RUBRIC)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def trajectory_rubric() -> str:
|
|
196
|
+
"""Resolve ``trajectory_rubric`` against the currently-active asset root."""
|
|
197
|
+
return get_prompt(_SCENARIO, "trajectory_rubric", _DEFAULT_TRAJECTORY_RUBRIC)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def user_judge_turn_prompt() -> str:
|
|
201
|
+
"""``USER_JUDGE_TURN_PROMPT`` with the rubric resolved at call time."""
|
|
202
|
+
return USER_JUDGE_TURN_PROMPT.replace(EDUCATIONAL_USER_JUDGE_RUBRIC, user_judge_rubric())
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def trajectory_level_judge_turn_prompt() -> str:
|
|
206
|
+
"""``TRAJECTORY_LEVEL_JUDGE_TURN_PROMPT`` with the rubric resolved at call time."""
|
|
207
|
+
return TRAJECTORY_LEVEL_JUDGE_TURN_PROMPT.replace(EDUCATIONAL_TRAJECTORY_RUBRIC, trajectory_rubric())
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Open-ended probe (info-seeking, Q&A, recommendations).
|
|
5
|
+
|
|
6
|
+
A thin probe class plus a shim function: ``OpenEndedProbe`` defines the
|
|
7
|
+
user-agent prompt and delegates everything else to ``BaseProbe`` +
|
|
8
|
+
``ConversationLoop``; ``simulate_general_open_ended`` is the back-compat
|
|
9
|
+
shim that constructs the probe and runs it through ``run_dispatch``.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from usersim.engine.core.behavioral import (
|
|
17
|
+
format_behavioral_profile_for_prompt,
|
|
18
|
+
format_disclosure_instructions,
|
|
19
|
+
format_interaction_style_instructions,
|
|
20
|
+
)
|
|
21
|
+
from usersim.engine.core.messages import _parse_theme
|
|
22
|
+
from usersim.engine.core.persona import format_persona_for_prompt
|
|
23
|
+
from usersim.engine.core.probes import BaseProbe, dispatch_episode, register_probe
|
|
24
|
+
from usersim.engine.core.prompt_loader import render_prompt
|
|
25
|
+
from usersim.engine.core.simulation import language_instruction
|
|
26
|
+
from usersim.engine.probes.general_open_ended.prompts import (
|
|
27
|
+
assistant_system_prompt,
|
|
28
|
+
user_agent_system_prompt,
|
|
29
|
+
user_judge_turn_prompt,
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
# ── Probe metadata ──────────────────────────────────────────────────
|
|
33
|
+
# Bumping ``PROMPT_VERSION`` invalidates prior evaluator rows on
|
|
34
|
+
# re-run, since the dedup key includes it.
|
|
35
|
+
|
|
36
|
+
PROBE_FAMILY: str = "general_open_ended"
|
|
37
|
+
PROBE_VARIANTS: list[str] = ["default"] # topic-conditioned via theme; single variant today
|
|
38
|
+
PROMPT_VERSION: str = "v1.0"
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@register_probe(family=PROBE_FAMILY, prompt_version=PROMPT_VERSION, variants=tuple(PROBE_VARIANTS))
|
|
42
|
+
class OpenEndedProbe(BaseProbe):
|
|
43
|
+
"""Open-ended conversation probe.
|
|
44
|
+
|
|
45
|
+
Test the assistant's general info-seeking / Q&A / recommendation
|
|
46
|
+
behavior with persona-grounded, theme-conditioned conversations.
|
|
47
|
+
No tools; no curated asset bank; quality judgment lives in the
|
|
48
|
+
out-of-sim trajectory evaluator.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
label = "general_open_ended"
|
|
52
|
+
|
|
53
|
+
def __init__(self, **kwargs: Any) -> None:
|
|
54
|
+
super().__init__(**kwargs)
|
|
55
|
+
theme = _parse_theme(self._data.get(getattr(self._cfg, "theme_column", "theme"), "{}"))
|
|
56
|
+
# Parsed once; the prompt itself is rendered per call in
|
|
57
|
+
# get_user_system_prompt so all three prompt hooks resolve alike.
|
|
58
|
+
self._topic = theme.get("description", theme.get("type", "general topic"))
|
|
59
|
+
self._disclosure_style = self._data.get("disclosure_style", "upfront")
|
|
60
|
+
|
|
61
|
+
def get_user_system_prompt(self) -> str:
|
|
62
|
+
return render_prompt(
|
|
63
|
+
user_agent_system_prompt(),
|
|
64
|
+
self._data,
|
|
65
|
+
persona=format_persona_for_prompt(self._persona),
|
|
66
|
+
topic=self._topic,
|
|
67
|
+
language_instruction=language_instruction(self._language, self._locale),
|
|
68
|
+
behavioral_instructions=format_behavioral_profile_for_prompt(
|
|
69
|
+
self._profile,
|
|
70
|
+
language=self._language,
|
|
71
|
+
),
|
|
72
|
+
disclosure_instructions=format_disclosure_instructions(
|
|
73
|
+
self._disclosure_style,
|
|
74
|
+
),
|
|
75
|
+
interaction_style_instructions=format_interaction_style_instructions(
|
|
76
|
+
self._interaction_style,
|
|
77
|
+
),
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
def get_assistant_system_prompt(self) -> str:
|
|
81
|
+
# Empty as shipped; see the accessor's docstring.
|
|
82
|
+
return render_prompt(assistant_system_prompt(), self._data)
|
|
83
|
+
|
|
84
|
+
def format_gate_prompt(
|
|
85
|
+
self,
|
|
86
|
+
user_query: str,
|
|
87
|
+
conversation_history: str,
|
|
88
|
+
) -> str:
|
|
89
|
+
return render_prompt(
|
|
90
|
+
user_judge_turn_prompt(),
|
|
91
|
+
self._data,
|
|
92
|
+
conversation_history=conversation_history,
|
|
93
|
+
user_turn_to_evaluate=user_query,
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
async def simulate_general_open_ended(
|
|
98
|
+
models: dict,
|
|
99
|
+
data: dict,
|
|
100
|
+
persona: dict,
|
|
101
|
+
profile: dict,
|
|
102
|
+
locale: str,
|
|
103
|
+
language: str,
|
|
104
|
+
cfg: Any,
|
|
105
|
+
**kwargs: Any,
|
|
106
|
+
) -> dict:
|
|
107
|
+
"""Thin shim for callers that import ``simulate_general_open_ended`` directly.
|
|
108
|
+
|
|
109
|
+
The dispatcher in ``usersim.engine.generator`` instantiates
|
|
110
|
+
``OpenEndedProbe`` and calls ``probe.run_dispatch`` directly; this
|
|
111
|
+
function exists for backward compatibility with per-probe tests
|
|
112
|
+
that import the symbol.
|
|
113
|
+
Constructs a probe instance and routes through the standard
|
|
114
|
+
``run_dispatch`` so behavior is identical to the dispatcher path.
|
|
115
|
+
"""
|
|
116
|
+
probe = OpenEndedProbe(
|
|
117
|
+
persona=persona,
|
|
118
|
+
locale=locale,
|
|
119
|
+
language=language,
|
|
120
|
+
models=models,
|
|
121
|
+
cfg=cfg,
|
|
122
|
+
provenance=kwargs.get("provenance"),
|
|
123
|
+
profile=profile,
|
|
124
|
+
data=data,
|
|
125
|
+
outcome_builder=kwargs.get("outcome_builder"),
|
|
126
|
+
)
|
|
127
|
+
return await dispatch_episode(probe, models=models, data=data, cfg=cfg)
|