usersim 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (379) hide show
  1. usersim/__init__.py +23 -0
  2. usersim/asset_gen/__init__.py +22 -0
  3. usersim/asset_gen/financial_services/__init__.py +22 -0
  4. usersim/asset_gen/financial_services/evaluate.py +330 -0
  5. usersim/asset_gen/financial_services/pipeline.py +1590 -0
  6. usersim/asset_gen/financial_services/prompts.py +318 -0
  7. usersim/asset_gen/financial_services/spec.py +794 -0
  8. usersim/asset_gen/financial_services/validate.py +750 -0
  9. usersim/asset_gen/registry.py +191 -0
  10. usersim/cli/__init__.py +185 -0
  11. usersim/cli/__main__.py +13 -0
  12. usersim/cli/_async.py +35 -0
  13. usersim/cli/_env.py +74 -0
  14. usersim/cli/_errors.py +44 -0
  15. usersim/cli/_inputs.py +691 -0
  16. usersim/cli/_models.py +1222 -0
  17. usersim/cli/_ngc.py +441 -0
  18. usersim/cli/_persona_language.py +316 -0
  19. usersim/cli/_pipeline.py +1007 -0
  20. usersim/cli/evaluate.py +415 -0
  21. usersim/cli/evaluate_assets.py +115 -0
  22. usersim/cli/gen_assets.py +120 -0
  23. usersim/cli/model_catalog.py +451 -0
  24. usersim/cli/models_default.toml +46 -0
  25. usersim/cli/models_openai.toml +26 -0
  26. usersim/cli/models_openrouter.toml +33 -0
  27. usersim/cli/panel.py +193 -0
  28. usersim/cli/report.py +247 -0
  29. usersim/cli/rescore.py +223 -0
  30. usersim/cli/select.py +304 -0
  31. usersim/cli/setup_ngc.py +131 -0
  32. usersim/cli/simulate.py +866 -0
  33. usersim/cli/smoke.py +1019 -0
  34. usersim/cli/validate_assets.py +64 -0
  35. usersim/engine/__init__.py +4 -0
  36. usersim/engine/assets/financial_services/SCHEMA.md +320 -0
  37. usersim/engine/assets/financial_services/de_DE/_generation_report.json +945 -0
  38. usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/corpus.yaml +8006 -0
  39. usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/institution_meta.yaml +9 -0
  40. usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/tasks.yaml +366 -0
  41. usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/tools.yaml +114 -0
  42. usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/corpus.yaml +10053 -0
  43. usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/institution_meta.yaml +10 -0
  44. usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/tasks.yaml +578 -0
  45. usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/tools.yaml +154 -0
  46. usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/corpus.yaml +6784 -0
  47. usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/institution_meta.yaml +8 -0
  48. usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/tasks.yaml +394 -0
  49. usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/tools.yaml +134 -0
  50. usersim/engine/assets/financial_services/de_DE/dynamic.yaml +252 -0
  51. usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/corpus.yaml +9404 -0
  52. usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/institution_meta.yaml +8 -0
  53. usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/tasks.yaml +394 -0
  54. usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/tools.yaml +114 -0
  55. usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/corpus.yaml +8181 -0
  56. usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/institution_meta.yaml +10 -0
  57. usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/tasks.yaml +440 -0
  58. usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/tools.yaml +144 -0
  59. usersim/engine/assets/financial_services/de_DE/region_meta.yaml +108 -0
  60. usersim/engine/assets/financial_services/en_GB/_generation_report.json +1330 -0
  61. usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/corpus.yaml +11053 -0
  62. usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/institution_meta.yaml +9 -0
  63. usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/tasks.yaml +401 -0
  64. usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/tools.yaml +114 -0
  65. usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/corpus.yaml +13830 -0
  66. usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/institution_meta.yaml +11 -0
  67. usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/tasks.yaml +597 -0
  68. usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/tools.yaml +159 -0
  69. usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/corpus.yaml +9717 -0
  70. usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/institution_meta.yaml +8 -0
  71. usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/tasks.yaml +418 -0
  72. usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/tools.yaml +139 -0
  73. usersim/engine/assets/financial_services/en_GB/dynamic.yaml +309 -0
  74. usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/corpus.yaml +7891 -0
  75. usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/institution_meta.yaml +8 -0
  76. usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/tasks.yaml +345 -0
  77. usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/tools.yaml +114 -0
  78. usersim/engine/assets/financial_services/en_GB/neobank/quill/corpus.yaml +9579 -0
  79. usersim/engine/assets/financial_services/en_GB/neobank/quill/institution_meta.yaml +10 -0
  80. usersim/engine/assets/financial_services/en_GB/neobank/quill/tasks.yaml +453 -0
  81. usersim/engine/assets/financial_services/en_GB/neobank/quill/tools.yaml +154 -0
  82. usersim/engine/assets/financial_services/en_GB/region_meta.yaml +128 -0
  83. usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/corpus.yaml +8028 -0
  84. usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/institution_meta.yaml +8 -0
  85. usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/tasks.yaml +387 -0
  86. usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/tools.yaml +124 -0
  87. usersim/engine/assets/financial_services/en_IN/_generation_report.json +977 -0
  88. usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/corpus.yaml +9170 -0
  89. usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/institution_meta.yaml +10 -0
  90. usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/tasks.yaml +330 -0
  91. usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/tools.yaml +139 -0
  92. usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/corpus.yaml +7436 -0
  93. usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/institution_meta.yaml +8 -0
  94. usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/tasks.yaml +323 -0
  95. usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/tools.yaml +129 -0
  96. usersim/engine/assets/financial_services/en_IN/dynamic.yaml +256 -0
  97. usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/corpus.yaml +7641 -0
  98. usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/institution_meta.yaml +8 -0
  99. usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/tasks.yaml +350 -0
  100. usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/tools.yaml +114 -0
  101. usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/corpus.yaml +9453 -0
  102. usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/institution_meta.yaml +9 -0
  103. usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/tasks.yaml +346 -0
  104. usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/tools.yaml +114 -0
  105. usersim/engine/assets/financial_services/en_IN/neobank/paysetu/corpus.yaml +8237 -0
  106. usersim/engine/assets/financial_services/en_IN/neobank/paysetu/institution_meta.yaml +10 -0
  107. usersim/engine/assets/financial_services/en_IN/neobank/paysetu/tasks.yaml +313 -0
  108. usersim/engine/assets/financial_services/en_IN/neobank/paysetu/tools.yaml +139 -0
  109. usersim/engine/assets/financial_services/en_IN/region_meta.yaml +81 -0
  110. usersim/engine/assets/financial_services/en_US/_generation_report.json +768 -0
  111. usersim/engine/assets/financial_services/en_US/bank/northwind_bank/corpus.yaml +9131 -0
  112. usersim/engine/assets/financial_services/en_US/bank/northwind_bank/institution_meta.yaml +9 -0
  113. usersim/engine/assets/financial_services/en_US/bank/northwind_bank/tasks.yaml +473 -0
  114. usersim/engine/assets/financial_services/en_US/bank/northwind_bank/tools.yaml +129 -0
  115. usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/corpus.yaml +7868 -0
  116. usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/institution_meta.yaml +8 -0
  117. usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/tasks.yaml +388 -0
  118. usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/tools.yaml +124 -0
  119. usersim/engine/assets/financial_services/en_US/dynamic.yaml +198 -0
  120. usersim/engine/assets/financial_services/en_US/region_meta.yaml +58 -0
  121. usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/corpus.yaml +8117 -0
  122. usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/institution_meta.yaml +8 -0
  123. usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/tasks.yaml +369 -0
  124. usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/tools.yaml +109 -0
  125. usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/corpus.yaml +6897 -0
  126. usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/institution_meta.yaml +8 -0
  127. usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/tasks.yaml +319 -0
  128. usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/tools.yaml +109 -0
  129. usersim/engine/assets/financial_services/fr_FR/_generation_report.json +1041 -0
  130. usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/corpus.yaml +10842 -0
  131. usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/institution_meta.yaml +10 -0
  132. usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/tasks.yaml +529 -0
  133. usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/tools.yaml +144 -0
  134. usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/corpus.yaml +9091 -0
  135. usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/institution_meta.yaml +8 -0
  136. usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/tasks.yaml +511 -0
  137. usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/tools.yaml +129 -0
  138. usersim/engine/assets/financial_services/fr_FR/dynamic.yaml +264 -0
  139. usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/corpus.yaml +9033 -0
  140. usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/institution_meta.yaml +8 -0
  141. usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/tasks.yaml +505 -0
  142. usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/tools.yaml +134 -0
  143. usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/corpus.yaml +7494 -0
  144. usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/institution_meta.yaml +9 -0
  145. usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/tasks.yaml +383 -0
  146. usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/tools.yaml +114 -0
  147. usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/corpus.yaml +7883 -0
  148. usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/institution_meta.yaml +10 -0
  149. usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/tasks.yaml +430 -0
  150. usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/tools.yaml +139 -0
  151. usersim/engine/assets/financial_services/fr_FR/region_meta.yaml +114 -0
  152. usersim/engine/assets/financial_services/hi_Deva_IN/_generation_report.json +977 -0
  153. usersim/engine/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/corpus.yaml +6265 -0
  154. usersim/engine/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/institution_meta.yaml +11 -0
  155. usersim/engine/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/tools.yaml +139 -0
  156. usersim/engine/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/corpus.yaml +5188 -0
  157. usersim/engine/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/institution_meta.yaml +9 -0
  158. usersim/engine/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/tools.yaml +129 -0
  159. usersim/engine/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/corpus.yaml +5115 -0
  160. usersim/engine/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/institution_meta.yaml +9 -0
  161. usersim/engine/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/tools.yaml +114 -0
  162. usersim/engine/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/corpus.yaml +6539 -0
  163. usersim/engine/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/institution_meta.yaml +10 -0
  164. usersim/engine/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/tools.yaml +114 -0
  165. usersim/engine/assets/financial_services/hi_Deva_IN/neobank/paysetu/corpus.yaml +5370 -0
  166. usersim/engine/assets/financial_services/hi_Deva_IN/neobank/paysetu/institution_meta.yaml +11 -0
  167. usersim/engine/assets/financial_services/hi_Deva_IN/neobank/paysetu/tools.yaml +139 -0
  168. usersim/engine/assets/financial_services/hi_Deva_IN/region_meta.yaml +81 -0
  169. usersim/engine/assets/general_educational/prompts.yaml +115 -0
  170. usersim/engine/assets/general_educational/subjects/base/subjects.json +162 -0
  171. usersim/engine/assets/general_educational/subjects/de_DE/subjects.json +162 -0
  172. usersim/engine/assets/general_educational/subjects/en_GB/subjects.json +162 -0
  173. usersim/engine/assets/general_educational/subjects/fr_FR/subjects.json +16 -0
  174. usersim/engine/assets/general_educational/subjects/hi_Deva_IN/subjects.json +14 -0
  175. usersim/engine/assets/general_educational/subjects/hi_Latn_IN/subjects.json +14 -0
  176. usersim/engine/assets/general_educational/subjects/ja_JP/subjects.json +16 -0
  177. usersim/engine/assets/general_educational/subjects/ko_KR/subjects.json +16 -0
  178. usersim/engine/assets/general_educational/subjects/pt_BR/subjects.json +12 -0
  179. usersim/engine/assets/general_open_ended/prompts.yaml +117 -0
  180. usersim/engine/assets/general_open_ended/topics/base/topics.json +194 -0
  181. usersim/engine/assets/general_open_ended/topics/de_DE/topics.json +194 -0
  182. usersim/engine/assets/general_open_ended/topics/en_GB/topics.json +194 -0
  183. usersim/engine/assets/general_open_ended/topics/fr_FR/topics.json +22 -0
  184. usersim/engine/assets/general_open_ended/topics/hi_Deva_IN/topics.json +22 -0
  185. usersim/engine/assets/general_open_ended/topics/hi_Latn_IN/topics.json +22 -0
  186. usersim/engine/assets/general_open_ended/topics/ja_JP/topics.json +17 -0
  187. usersim/engine/assets/general_open_ended/topics/ko_KR/topics.json +17 -0
  188. usersim/engine/assets/general_open_ended/topics/pt_BR/topics.json +22 -0
  189. usersim/engine/assets/health_decision_support_disclosure/sample.yaml +21 -0
  190. usersim/engine/assets/health_general_disclosure/sample.yaml +21 -0
  191. usersim/engine/assets/health_therapy_disclosure/sample.yaml +23 -0
  192. usersim/engine/assets/health_triage_disclosure/sample.yaml +21 -0
  193. usersim/engine/assets/identity_disclosure/SCHEMA.md +303 -0
  194. usersim/engine/assets/identity_disclosure/spec.yaml +1007 -0
  195. usersim/engine/assets/safety_agentic/SCHEMA.md +252 -0
  196. usersim/engine/assets/safety_agentic/action_taxonomy.yaml +682 -0
  197. usersim/engine/assets/safety_agentic/sample.yaml +3993 -0
  198. usersim/engine/assets/safety_chat_pressure/SCHEMA.md +181 -0
  199. usersim/engine/assets/safety_chat_pressure/sample.yaml +3646 -0
  200. usersim/engine/assets/sov_ai_dynamic/SCHEMA.md +98 -0
  201. usersim/engine/assets/sov_ai_dynamic/de_DE/categories.yaml +116 -0
  202. usersim/engine/assets/sov_ai_dynamic/en_GB/categories.yaml +116 -0
  203. usersim/engine/assets/sov_ai_dynamic/en_IN/categories.yaml +116 -0
  204. usersim/engine/assets/sov_ai_dynamic/en_SG/categories.yaml +116 -0
  205. usersim/engine/assets/sov_ai_dynamic/en_US/categories.yaml +116 -0
  206. usersim/engine/assets/sov_ai_dynamic/fr_FR/categories.yaml +116 -0
  207. usersim/engine/assets/sov_ai_dynamic/hi_Deva_IN/categories.yaml +116 -0
  208. usersim/engine/assets/sov_ai_dynamic/hi_Latn_IN/categories.yaml +116 -0
  209. usersim/engine/assets/sov_ai_dynamic/ja_JP/categories.yaml +116 -0
  210. usersim/engine/assets/sov_ai_dynamic/ko_KR/categories.yaml +116 -0
  211. usersim/engine/assets/sov_ai_dynamic/pt_BR/categories.yaml +116 -0
  212. usersim/engine/assets/sov_ai_facts/SCHEMA.md +196 -0
  213. usersim/engine/assets/sov_ai_facts/de_DE/sample.yaml +1421 -0
  214. usersim/engine/assets/sov_ai_facts/en_GB/sample.yaml +1389 -0
  215. usersim/engine/assets/sov_ai_facts/en_IN/sample.yaml +1275 -0
  216. usersim/engine/assets/sov_ai_facts/en_SG/sample.yaml +1358 -0
  217. usersim/engine/assets/sov_ai_facts/en_US/sample.yaml +1270 -0
  218. usersim/engine/assets/sov_ai_facts/fr_FR/sample.yaml +1287 -0
  219. usersim/engine/assets/sov_ai_facts/hi_Deva_IN/sample.yaml +1272 -0
  220. usersim/engine/assets/sov_ai_facts/hi_Latn_IN/sample.yaml +1272 -0
  221. usersim/engine/assets/sov_ai_facts/ja_JP/sample.yaml +1294 -0
  222. usersim/engine/assets/sov_ai_facts/ko_KR/sample.yaml +1435 -0
  223. usersim/engine/assets/sov_ai_facts/pt_BR/sample.yaml +1321 -0
  224. usersim/engine/assets/sov_ai_multilingual_parity/SCHEMA.md +137 -0
  225. usersim/engine/assets/sov_ai_multilingual_parity/sample.yaml +1210 -0
  226. usersim/engine/assets/tool_calling/README.md +225 -0
  227. usersim/engine/assets/tool_calling/prompts.yaml +171 -0
  228. usersim/engine/assets/tool_calling/toolsets_seed.parquet +0 -0
  229. usersim/engine/config.py +296 -0
  230. usersim/engine/core/__init__.py +4 -0
  231. usersim/engine/core/_assets.py +178 -0
  232. usersim/engine/core/_extensions.py +148 -0
  233. usersim/engine/core/action_taxonomy.py +347 -0
  234. usersim/engine/core/agentic_bank.py +725 -0
  235. usersim/engine/core/analysis.py +60 -0
  236. usersim/engine/core/backends.py +180 -0
  237. usersim/engine/core/behavioral.py +755 -0
  238. usersim/engine/core/clinical_profile_bank.py +245 -0
  239. usersim/engine/core/context.py +137 -0
  240. usersim/engine/core/embeddings.py +83 -0
  241. usersim/engine/core/episode_input.py +281 -0
  242. usersim/engine/core/episode_runtime.py +735 -0
  243. usersim/engine/core/fact_bank.py +500 -0
  244. usersim/engine/core/finance_bank.py +970 -0
  245. usersim/engine/core/finance_tasks.py +292 -0
  246. usersim/engine/core/idempotency.py +62 -0
  247. usersim/engine/core/identity.py +215 -0
  248. usersim/engine/core/identity_spec.py +659 -0
  249. usersim/engine/core/identity_spec_types.py +425 -0
  250. usersim/engine/core/judges.py +85 -0
  251. usersim/engine/core/language_detection.py +239 -0
  252. usersim/engine/core/llm.py +710 -0
  253. usersim/engine/core/locale.py +512 -0
  254. usersim/engine/core/localized.py +172 -0
  255. usersim/engine/core/manifest.py +998 -0
  256. usersim/engine/core/messages.py +93 -0
  257. usersim/engine/core/missing.py +38 -0
  258. usersim/engine/core/outcomes.py +442 -0
  259. usersim/engine/core/persona.py +178 -0
  260. usersim/engine/core/pressure_bank.py +629 -0
  261. usersim/engine/core/preview.py +1927 -0
  262. usersim/engine/core/probes.py +1499 -0
  263. usersim/engine/core/probing_taxonomy.py +462 -0
  264. usersim/engine/core/prompt_loader.py +93 -0
  265. usersim/engine/core/prompts.py +81 -0
  266. usersim/engine/core/provenance.py +118 -0
  267. usersim/engine/core/query_bank.py +567 -0
  268. usersim/engine/core/realized_audit.py +411 -0
  269. usersim/engine/core/row_metadata.py +35 -0
  270. usersim/engine/core/seeds.py +82 -0
  271. usersim/engine/core/simulation.py +1899 -0
  272. usersim/engine/core/storage.py +1407 -0
  273. usersim/engine/core/tool_calls.py +106 -0
  274. usersim/engine/core/translation.py +266 -0
  275. usersim/engine/core/user_turn_policy.py +80 -0
  276. usersim/engine/evaluator/__init__.py +19 -0
  277. usersim/engine/evaluator/axes.py +437 -0
  278. usersim/engine/evaluator/config.py +187 -0
  279. usersim/engine/evaluator/generator.py +424 -0
  280. usersim/engine/evaluator/judges.py +151 -0
  281. usersim/engine/evaluator/prompts.py +108 -0
  282. usersim/engine/evaluator/scorers/__init__.py +289 -0
  283. usersim/engine/evaluator/scorers/financial_services.py +635 -0
  284. usersim/engine/evaluator/scorers/health_disclosure.py +503 -0
  285. usersim/engine/evaluator/scorers/identity_disclosure.py +545 -0
  286. usersim/engine/evaluator/scorers/identity_grading.py +543 -0
  287. usersim/engine/evaluator/scorers/language_compliance.py +426 -0
  288. usersim/engine/evaluator/scorers/refusal_basics.py +479 -0
  289. usersim/engine/evaluator/scorers/response_shape.py +363 -0
  290. usersim/engine/evaluator/scorers/safety_agentic.py +1010 -0
  291. usersim/engine/evaluator/scorers/safety_chat_pressure.py +791 -0
  292. usersim/engine/evaluator/scorers/sov_ai_dynamic.py +629 -0
  293. usersim/engine/evaluator/scorers/sov_ai_facts.py +648 -0
  294. usersim/engine/evaluator/scorers/sov_ai_multilingual_parity.py +550 -0
  295. usersim/engine/evaluator/scorers/tool_use.py +291 -0
  296. usersim/engine/external.py +218 -0
  297. usersim/engine/generator.py +419 -0
  298. usersim/engine/plugin.py +34 -0
  299. usersim/engine/probes/__init__.py +28 -0
  300. usersim/engine/probes/financial_services/__init__.py +16 -0
  301. usersim/engine/probes/financial_services/generator.py +1329 -0
  302. usersim/engine/probes/financial_services/prompts.py +132 -0
  303. usersim/engine/probes/financial_services/retrieval.py +229 -0
  304. usersim/engine/probes/financial_services/task_derivation.py +274 -0
  305. usersim/engine/probes/financial_services/tools.py +116 -0
  306. usersim/engine/probes/general_educational/__init__.py +4 -0
  307. usersim/engine/probes/general_educational/generator.py +125 -0
  308. usersim/engine/probes/general_educational/prompts.py +207 -0
  309. usersim/engine/probes/general_open_ended/__init__.py +4 -0
  310. usersim/engine/probes/general_open_ended/generator.py +127 -0
  311. usersim/engine/probes/general_open_ended/prompts.py +209 -0
  312. usersim/engine/probes/health_disclosure/__init__.py +28 -0
  313. usersim/engine/probes/health_disclosure/base.py +148 -0
  314. usersim/engine/probes/health_disclosure/clients.py +291 -0
  315. usersim/engine/probes/health_disclosure/decision_support.py +44 -0
  316. usersim/engine/probes/health_disclosure/general.py +45 -0
  317. usersim/engine/probes/health_disclosure/mixin.py +363 -0
  318. usersim/engine/probes/health_disclosure/move_runtime.py +486 -0
  319. usersim/engine/probes/health_disclosure/moves.py +241 -0
  320. usersim/engine/probes/health_disclosure/prompts.py +324 -0
  321. usersim/engine/probes/health_disclosure/therapy.py +43 -0
  322. usersim/engine/probes/health_disclosure/triage.py +43 -0
  323. usersim/engine/probes/identity_disclosure/__init__.py +4 -0
  324. usersim/engine/probes/identity_disclosure/generator.py +504 -0
  325. usersim/engine/probes/identity_disclosure/prompts.py +219 -0
  326. usersim/engine/probes/identity_disclosure/task_derivation.py +227 -0
  327. usersim/engine/probes/safety_agentic/__init__.py +24 -0
  328. usersim/engine/probes/safety_agentic/generator.py +666 -0
  329. usersim/engine/probes/safety_agentic/prompts.py +24 -0
  330. usersim/engine/probes/safety_agentic/task_derivation.py +109 -0
  331. usersim/engine/probes/safety_chat_pressure/__init__.py +24 -0
  332. usersim/engine/probes/safety_chat_pressure/classifier.py +113 -0
  333. usersim/engine/probes/safety_chat_pressure/generator.py +551 -0
  334. usersim/engine/probes/safety_chat_pressure/prompts.py +141 -0
  335. usersim/engine/probes/safety_chat_pressure/task_derivation.py +171 -0
  336. usersim/engine/probes/sov_ai_dynamic/__init__.py +28 -0
  337. usersim/engine/probes/sov_ai_dynamic/generator.py +393 -0
  338. usersim/engine/probes/sov_ai_dynamic/prompts.py +472 -0
  339. usersim/engine/probes/sov_ai_dynamic/task_derivation.py +106 -0
  340. usersim/engine/probes/sov_ai_facts/__init__.py +15 -0
  341. usersim/engine/probes/sov_ai_facts/generator.py +342 -0
  342. usersim/engine/probes/sov_ai_facts/prompts.py +714 -0
  343. usersim/engine/probes/sov_ai_facts/task_derivation.py +479 -0
  344. usersim/engine/probes/sov_ai_multilingual_parity/__init__.py +19 -0
  345. usersim/engine/probes/sov_ai_multilingual_parity/generator.py +379 -0
  346. usersim/engine/probes/sov_ai_multilingual_parity/prompts.py +456 -0
  347. usersim/engine/probes/sov_ai_multilingual_parity/task_derivation.py +104 -0
  348. usersim/engine/probes/tool_calling/__init__.py +9 -0
  349. usersim/engine/probes/tool_calling/generator.py +437 -0
  350. usersim/engine/probes/tool_calling/prompts.py +221 -0
  351. usersim/engine/probes/tool_calling/verifiers.py +165 -0
  352. usersim/py.typed +0 -0
  353. usersim/reporting/__init__.py +116 -0
  354. usersim/reporting/_reasoning.py +201 -0
  355. usersim/reporting/capability_report.py +2049 -0
  356. usersim/reporting/comparison.py +797 -0
  357. usersim/reporting/comparison_dashboard.py +2938 -0
  358. usersim/reporting/dashboard.py +1731 -0
  359. usersim/reporting/diagnostics.py +435 -0
  360. usersim/reporting/eval_manifest.py +291 -0
  361. usersim/reporting/identity.py +82 -0
  362. usersim/reporting/resource.py +476 -0
  363. usersim/reporting/runs.py +282 -0
  364. usersim/reporting/sim_health.py +334 -0
  365. usersim/selection/__init__.py +87 -0
  366. usersim/selection/engine.py +423 -0
  367. usersim/selection/io.py +391 -0
  368. usersim/selection/profiles.py +249 -0
  369. usersim/selection/schemas.py +194 -0
  370. usersim/taxonomy/__init__.py +38 -0
  371. usersim/taxonomy/capabilities.py +1125 -0
  372. usersim/taxonomy/eval_cell.py +131 -0
  373. usersim/testing/__init__.py +534 -0
  374. usersim-0.2.0.dist-info/METADATA +392 -0
  375. usersim-0.2.0.dist-info/RECORD +379 -0
  376. usersim-0.2.0.dist-info/WHEEL +5 -0
  377. usersim-0.2.0.dist-info/entry_points.txt +6 -0
  378. usersim-0.2.0.dist-info/licenses/LICENSE +204 -0
  379. usersim-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,116 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Tool surface for the financial_services probe.
5
+
6
+ The assistant is offered the PERMANENT tools up front (``kb_search``,
7
+ ``verify_identity``, ...) plus a single ``call_tool(tool_name, arguments)`` gate
8
+ for DISCOVERABLE tools. A discoverable tool is only usable once its
9
+ documentation has been retrieved (its name appears in a retrieved document's
10
+ ``mentions_tools``). This module is pure helpers; the probe's
11
+ ``after_assistant_turn`` tool loop owns state.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import json
17
+ from typing import Any, Sequence
18
+
19
+ from usersim.engine.core.finance_bank import (
20
+ Institution,
21
+ framework_primitive_tool,
22
+ verify_identity_schema,
23
+ )
24
+
25
+ CALL_TOOL_NAME = "call_tool"
26
+
27
+
28
+ def build_offered_tools(
29
+ institution: Institution,
30
+ *,
31
+ identity_required_fields: Sequence[str] | None = None,
32
+ ) -> list[dict[str, Any]]:
33
+ """OpenAI-style tool schemas offered to the assistant, SCOPED to ``institution``.
34
+
35
+ Permanent (non-discoverable) tools are offered by name; discoverable tools
36
+ are reached only through the ``call_tool`` gate. Framework primitives
37
+ (``kb_search`` / ``verify_identity``) use the canonical schema from
38
+ ``core.finance_bank`` so the assistant always sees their real parameters
39
+ even if a bank asset ships them empty.
40
+
41
+ ``identity_required_fields`` specializes ``verify_identity`` to the region's
42
+ KYC contract (e.g. India also requires a PAN). Omit it for the universal
43
+ name + DOB default.
44
+ """
45
+ offered: list[dict[str, Any]] = []
46
+ for t in institution.permanent_tools():
47
+ canonical = framework_primitive_tool(t.name)
48
+ if t.name == "verify_identity" and identity_required_fields:
49
+ canonical = verify_identity_schema(identity_required_fields)
50
+ if canonical is not None:
51
+ offered.append(
52
+ {
53
+ "type": "function",
54
+ "function": {
55
+ "name": t.name,
56
+ "description": canonical["description"],
57
+ "parameters": canonical["parameters"],
58
+ },
59
+ }
60
+ )
61
+ else:
62
+ offered.append(t.to_openai_tool())
63
+ offered.append(
64
+ {
65
+ "type": "function",
66
+ "function": {
67
+ "name": CALL_TOOL_NAME,
68
+ "description": (
69
+ "Invoke a tool documented in the knowledge base. You must first "
70
+ "retrieve the tool's documentation via kb_search before calling it."
71
+ ),
72
+ "parameters": {
73
+ "type": "object",
74
+ "properties": {
75
+ "tool_name": {"type": "string"},
76
+ "arguments": {"type": "object"},
77
+ },
78
+ "required": ["tool_name"],
79
+ },
80
+ },
81
+ }
82
+ )
83
+ return offered
84
+
85
+
86
+ def extract_call(tc: Any) -> tuple[str, dict[str, Any]]:
87
+ """Pull ``(function_name, arguments_dict)`` from one tool_call dict."""
88
+ if not isinstance(tc, dict):
89
+ return ("<malformed>", {})
90
+ fn = tc.get("function")
91
+ if not isinstance(fn, dict):
92
+ return (str(tc.get("name") or "<unknown>"), {})
93
+ name = str(fn.get("name") or "<unknown>")
94
+ raw = fn.get("arguments")
95
+ if isinstance(raw, dict):
96
+ return (name, raw)
97
+ if isinstance(raw, str):
98
+ try:
99
+ parsed = json.loads(raw)
100
+ return (name, parsed if isinstance(parsed, dict) else {})
101
+ except (json.JSONDecodeError, TypeError):
102
+ return (name, {})
103
+ return (name, {})
104
+
105
+
106
+ def resolve_invoked_tool(function_name: str, arguments: dict[str, Any]) -> tuple[str, dict[str, Any]]:
107
+ """Map an assistant tool call to the *domain* tool it targets.
108
+
109
+ ``call_tool(tool_name=..., arguments=...)`` targets the named discoverable
110
+ tool; any other function call targets itself (permanent tool or a
111
+ directly-named tool).
112
+ """
113
+ if function_name == CALL_TOOL_NAME:
114
+ target = str(arguments.get("tool_name") or "<unknown>")
115
+ return (target, arguments.get("arguments") or {})
116
+ return (function_name, arguments)
@@ -0,0 +1,4 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Educational conversation simulation probe (tutoring, explanation, practice)."""
@@ -0,0 +1,125 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Educational probe (tutoring, explanation, practice).
5
+
6
+ A thin probe class plus a shim function: ``EducationalProbe`` defines
7
+ the user-agent prompt and delegates everything else to ``BaseProbe`` +
8
+ ``ConversationLoop``; ``simulate_general_educational`` is the back-compat
9
+ shim that constructs the probe and runs it through ``run_dispatch``.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ from typing import Any
15
+
16
+ from usersim.engine.core.behavioral import (
17
+ format_behavioral_profile_for_prompt,
18
+ format_disclosure_instructions,
19
+ format_interaction_style_instructions,
20
+ )
21
+ from usersim.engine.core.messages import _parse_theme
22
+ from usersim.engine.core.persona import format_persona_for_prompt
23
+ from usersim.engine.core.probes import BaseProbe, dispatch_episode, register_probe
24
+ from usersim.engine.core.prompt_loader import render_prompt
25
+ from usersim.engine.core.simulation import language_instruction
26
+ from usersim.engine.probes.general_educational.prompts import (
27
+ assistant_system_prompt,
28
+ user_agent_system_prompt,
29
+ user_judge_turn_prompt,
30
+ )
31
+
32
+ # ── Probe metadata ──────────────────────────────────────────────────
33
+ # Bumping ``PROMPT_VERSION`` invalidates prior evaluator rows on
34
+ # re-run, since the dedup key includes it. Bump on any semantic change
35
+ # to the probe's prompts or task derivation.
36
+
37
+ PROBE_FAMILY: str = "general_educational"
38
+ PROBE_VARIANTS: list[str] = ["default"] # subject-conditioned via theme; single variant today
39
+ PROMPT_VERSION: str = "v1.0"
40
+
41
+
42
+ @register_probe(family=PROBE_FAMILY, prompt_version=PROMPT_VERSION, variants=tuple(PROBE_VARIANTS))
43
+ class EducationalProbe(BaseProbe):
44
+ """Educational / tutoring probe.
45
+
46
+ Test the assistant's tutoring / explanation / practice behavior
47
+ with persona-grounded, subject-conditioned conversations. No
48
+ tools; no curated asset bank; quality judgment lives in the
49
+ out-of-sim trajectory evaluator.
50
+ """
51
+
52
+ label = "general_educational"
53
+
54
+ def __init__(self, **kwargs: Any) -> None:
55
+ super().__init__(**kwargs)
56
+ theme = _parse_theme(self._data.get(getattr(self._cfg, "theme_column", "theme"), "{}"))
57
+ # Parsed once; the prompt itself is rendered per call in
58
+ # get_user_system_prompt so all three prompt hooks resolve alike.
59
+ self._subject = theme.get("description", theme.get("type", "general subject"))
60
+ self._disclosure_style = self._data.get("disclosure_style", "upfront")
61
+
62
+ def get_user_system_prompt(self) -> str:
63
+ return render_prompt(
64
+ user_agent_system_prompt(),
65
+ self._data,
66
+ persona=format_persona_for_prompt(self._persona),
67
+ subject=self._subject,
68
+ language_instruction=language_instruction(self._language, self._locale),
69
+ behavioral_instructions=format_behavioral_profile_for_prompt(
70
+ self._profile,
71
+ language=self._language,
72
+ ),
73
+ disclosure_instructions=format_disclosure_instructions(
74
+ self._disclosure_style,
75
+ ),
76
+ interaction_style_instructions=format_interaction_style_instructions(
77
+ self._interaction_style,
78
+ ),
79
+ )
80
+
81
+ def get_assistant_system_prompt(self) -> str:
82
+ # Empty as shipped; see the accessor's docstring.
83
+ return render_prompt(assistant_system_prompt(), self._data)
84
+
85
+ def format_gate_prompt(
86
+ self,
87
+ user_query: str,
88
+ conversation_history: str,
89
+ ) -> str:
90
+ return render_prompt(
91
+ user_judge_turn_prompt(),
92
+ self._data,
93
+ conversation_history=conversation_history,
94
+ user_turn_to_evaluate=user_query,
95
+ )
96
+
97
+
98
+ async def simulate_general_educational(
99
+ models: dict,
100
+ data: dict,
101
+ persona: dict,
102
+ profile: dict,
103
+ locale: str,
104
+ language: str,
105
+ cfg: Any,
106
+ **kwargs: Any,
107
+ ) -> dict:
108
+ """Thin shim for callers that import ``simulate_general_educational`` directly.
109
+
110
+ See ``usersim.engine.probes.general_open_ended.generator.simulate_general_open_ended``
111
+ for the rationale; preserved for backward-compat with the dispatcher
112
+ bootstrap and per-probe tests.
113
+ """
114
+ probe = EducationalProbe(
115
+ persona=persona,
116
+ locale=locale,
117
+ language=language,
118
+ models=models,
119
+ cfg=cfg,
120
+ provenance=kwargs.get("provenance"),
121
+ profile=profile,
122
+ data=data,
123
+ outcome_builder=kwargs.get("outcome_builder"),
124
+ )
125
+ return await dispatch_episode(probe, models=models, data=data, cfg=cfg)
@@ -0,0 +1,207 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Prompt templates for the general_educational probe.
5
+
6
+ Templates are loaded from assets/general_educational/prompts.yaml when available,
7
+ falling back to the compiled-in defaults below.
8
+ """
9
+
10
+ from usersim.engine.core.prompt_loader import get_prompt
11
+
12
+ _SCENARIO = "general_educational"
13
+
14
+ # ── Compiled-in defaults ──────────────────────────────────────────────────
15
+
16
+ _DEFAULT_USER_AGENT = """You are roleplaying as a learner seeking help understanding a topic.
17
+ You are talking to an AI tutor/assistant.
18
+
19
+ <INSTRUCTIONS>
20
+ As a learner:
21
+ - Ask for explanations at your level of understanding.
22
+ - When confused, ask follow-up questions or request simpler explanations.
23
+ - Attempt practice problems when offered, and share your attempts.
24
+ - Express when something clicks or when you're still confused.
25
+ - Share relevant context from your background that might help the tutor adapt.
26
+ - If the tutor's response does not address what you asked, or does not make sense to you, say so — a real learner would not silently accept a non-answer.
27
+
28
+ Conversational style:
29
+ - Do NOT reveal you are role-playing.
30
+ - Do NOT use stage directions or bracketed actions.
31
+ - Match your vocabulary to your persona's education level.
32
+ - Show genuine curiosity and learning behaviors.
33
+ - If the explanation is too complex, say so naturally.
34
+ - If the explanation is too simple, ask for more depth.
35
+
36
+ REMEMBER: YOU ARE THE LEARNER. Stay in character throughout.
37
+ </INSTRUCTIONS>
38
+
39
+ {language_instruction}
40
+
41
+ Act as the following learner:
42
+ <YOUR_PERSONA>
43
+ {persona}
44
+ </YOUR_PERSONA>
45
+
46
+ {behavioral_instructions}
47
+
48
+ {disclosure_instructions}
49
+
50
+ {interaction_style_instructions}
51
+
52
+ You want to learn about this subject:
53
+ <SUBJECT>
54
+ {subject}
55
+ </SUBJECT>"""
56
+
57
+
58
+ _DEFAULT_USER_JUDGE_RUBRIC = """Criteria for success:
59
+ - The learner's message is natural for their education level and background.
60
+ - The learner asks relevant questions or makes reasonable attempts.
61
+ - The learner's confusion or understanding is realistic for their persona.
62
+
63
+ Criteria for failure:
64
+ - The learner breaks character or acts like a teacher/assistant.
65
+ - The learner demonstrates knowledge inconsistent with their persona.
66
+ - The message is incoherent or unrelated to the learning topic.
67
+ - The learner offers help, assistance, or expertise to the tutor — a real learner does not offer to help the teacher.
68
+ - The learner references the conversation itself, the simulation, or how messages are being constructed — this is a fourth-wall break."""
69
+
70
+
71
+ _DEFAULT_TRAJECTORY_RUBRIC = """Evaluate the overall quality of this tutoring conversation.
72
+
73
+ SUCCESS criteria:
74
+ - The tutor adapted explanations to the learner's level.
75
+ - The tutor used appropriate scaffolding and examples.
76
+ - The conversation progressed the learner's understanding.
77
+ - The tutor was encouraging without being condescending.
78
+ - When the learner's question was vague or overly broad, the tutor asked what
79
+ specifically they wanted to learn instead of providing an overwhelming overview.
80
+ - The learner-LLM stayed in character as a realistic learner — only asking
81
+ questions, attempting exercises, expressing confusion, and providing requested info.
82
+
83
+ FAILURE criteria:
84
+ - The tutor provided incorrect information.
85
+ - The tutor was condescending, dismissive, or unhelpful.
86
+ - Explanations were consistently too complex or too simple for the learner.
87
+ - The conversation went in circles without progress.
88
+ - The tutor dumped an excessive amount of information instead of narrowing down
89
+ a vague or overly broad question from the learner.
90
+ - The tutor produced an unintelligible, empty, or non-responsive reply to the
91
+ learner's question in any turn.
92
+ - Role inversion: the learner-LLM took over the tutor's job — e.g., explaining
93
+ concepts, providing structured data, or teaching the tutor. A real learner
94
+ asks for help; they do not lecture back.
95
+ - The learner-LLM offered help or expertise to the tutor, reversing the
96
+ learner/teacher dynamic.
97
+ - The learner-LLM broke the fourth wall by referencing the conversation structure,
98
+ the simulation, or how messages should be written.
99
+ - The learner-LLM broke character (gave tutor-like responses, used structured
100
+ output, markdown, tables, or numbered lists)."""
101
+
102
+
103
+ _INLINE_JUDGE_RESPONSE_FORMAT = """
104
+ Strictly respond in the following format:
105
+ <explanation>
106
+ [Your detailed explanation and justification for the rating.]
107
+ </explanation>
108
+ <rating>
109
+ [Must be either 'success' or 'failure' strictly]
110
+ </rating>"""
111
+
112
+ # ── Public templates (YAML overrides compiled-in defaults) ────────────────
113
+
114
+ EDUCATIONAL_USER_AGENT_SYSTEM_PROMPT = get_prompt(_SCENARIO, "user_agent_system_prompt", _DEFAULT_USER_AGENT)
115
+
116
+ EDUCATIONAL_USER_JUDGE_RUBRIC = get_prompt(_SCENARIO, "user_judge_rubric", _DEFAULT_USER_JUDGE_RUBRIC)
117
+
118
+ EDUCATIONAL_TRAJECTORY_RUBRIC = get_prompt(_SCENARIO, "trajectory_rubric", _DEFAULT_TRAJECTORY_RUBRIC)
119
+
120
+
121
+ USER_JUDGE_TURN_PROMPT = (
122
+ """You are an expert evaluator. Judge the learner's message in this tutoring conversation.
123
+
124
+ Previous conversation:
125
+ <PREV_CONVERSATION>
126
+ {conversation_history}
127
+ </PREV_CONVERSATION>
128
+
129
+ Learner message to judge:
130
+ <LEARNER_MESSAGE>
131
+ {user_turn_to_evaluate}
132
+ </LEARNER_MESSAGE>
133
+
134
+ <RUBRIC>
135
+ """
136
+ + EDUCATIONAL_USER_JUDGE_RUBRIC
137
+ + """
138
+ </RUBRIC>
139
+ """
140
+ + _INLINE_JUDGE_RESPONSE_FORMAT
141
+ )
142
+
143
+
144
+ TRAJECTORY_LEVEL_JUDGE_TURN_PROMPT = (
145
+ """You are an expert evaluator. Judge this entire tutoring conversation.
146
+
147
+ <CONVERSATION>
148
+ {conversation_history}
149
+ </CONVERSATION>
150
+
151
+ <RUBRIC>
152
+ """
153
+ + EDUCATIONAL_TRAJECTORY_RUBRIC
154
+ + """
155
+ </RUBRIC>
156
+ """
157
+ + _INLINE_JUDGE_RESPONSE_FORMAT
158
+ )
159
+
160
+
161
+ # ── Call-time resolution ─────────────────────────────────────────────
162
+ # The module-level constants above bind whatever asset root was active at
163
+ # IMPORT time. Probes are imported at plugin bootstrap, before a run's
164
+ # ``assets_dir`` is known, so a per-run root would never reach them. These
165
+ # accessors re-resolve on every call, which is what makes
166
+ # ``--assets-dir`` / ``USERSIM_ASSETS_DIR`` govern prompts the same way it
167
+ # already governs seeds and banks. The constants are kept for callers that
168
+ # import them directly.
169
+
170
+
171
+ def user_agent_system_prompt() -> str:
172
+ """Resolve ``user_agent_system_prompt`` against the currently-active asset root."""
173
+ return get_prompt(_SCENARIO, "user_agent_system_prompt", _DEFAULT_USER_AGENT)
174
+
175
+
176
+ def assistant_system_prompt() -> str:
177
+ """Resolve ``assistant_system_prompt`` against the currently-active asset root.
178
+
179
+ Defaults to ``""``: the assistant under test gets no system prompt, so
180
+ that whether it behaves helpfully, answers in the right language or
181
+ refuses is what the probe measures rather than what it was told. See
182
+ the pure-capability policy in ``docs/engine/README.md``.
183
+ Resolved like every other prompt so the mechanism is uniform, not
184
+ because filling it in is expected -- a non-empty value here changes
185
+ what the probe measures and wants a rationale in the module docstring.
186
+ """
187
+ return get_prompt(_SCENARIO, "assistant_system_prompt", "")
188
+
189
+
190
+ def user_judge_rubric() -> str:
191
+ """Resolve ``user_judge_rubric`` against the currently-active asset root."""
192
+ return get_prompt(_SCENARIO, "user_judge_rubric", _DEFAULT_USER_JUDGE_RUBRIC)
193
+
194
+
195
+ def trajectory_rubric() -> str:
196
+ """Resolve ``trajectory_rubric`` against the currently-active asset root."""
197
+ return get_prompt(_SCENARIO, "trajectory_rubric", _DEFAULT_TRAJECTORY_RUBRIC)
198
+
199
+
200
+ def user_judge_turn_prompt() -> str:
201
+ """``USER_JUDGE_TURN_PROMPT`` with the rubric resolved at call time."""
202
+ return USER_JUDGE_TURN_PROMPT.replace(EDUCATIONAL_USER_JUDGE_RUBRIC, user_judge_rubric())
203
+
204
+
205
+ def trajectory_level_judge_turn_prompt() -> str:
206
+ """``TRAJECTORY_LEVEL_JUDGE_TURN_PROMPT`` with the rubric resolved at call time."""
207
+ return TRAJECTORY_LEVEL_JUDGE_TURN_PROMPT.replace(EDUCATIONAL_TRAJECTORY_RUBRIC, trajectory_rubric())
@@ -0,0 +1,4 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Open-ended conversation simulation probe (info-seeking, Q&A, recommendations)."""
@@ -0,0 +1,127 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Open-ended probe (info-seeking, Q&A, recommendations).
5
+
6
+ A thin probe class plus a shim function: ``OpenEndedProbe`` defines the
7
+ user-agent prompt and delegates everything else to ``BaseProbe`` +
8
+ ``ConversationLoop``; ``simulate_general_open_ended`` is the back-compat
9
+ shim that constructs the probe and runs it through ``run_dispatch``.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ from typing import Any
15
+
16
+ from usersim.engine.core.behavioral import (
17
+ format_behavioral_profile_for_prompt,
18
+ format_disclosure_instructions,
19
+ format_interaction_style_instructions,
20
+ )
21
+ from usersim.engine.core.messages import _parse_theme
22
+ from usersim.engine.core.persona import format_persona_for_prompt
23
+ from usersim.engine.core.probes import BaseProbe, dispatch_episode, register_probe
24
+ from usersim.engine.core.prompt_loader import render_prompt
25
+ from usersim.engine.core.simulation import language_instruction
26
+ from usersim.engine.probes.general_open_ended.prompts import (
27
+ assistant_system_prompt,
28
+ user_agent_system_prompt,
29
+ user_judge_turn_prompt,
30
+ )
31
+
32
+ # ── Probe metadata ──────────────────────────────────────────────────
33
+ # Bumping ``PROMPT_VERSION`` invalidates prior evaluator rows on
34
+ # re-run, since the dedup key includes it.
35
+
36
+ PROBE_FAMILY: str = "general_open_ended"
37
+ PROBE_VARIANTS: list[str] = ["default"] # topic-conditioned via theme; single variant today
38
+ PROMPT_VERSION: str = "v1.0"
39
+
40
+
41
+ @register_probe(family=PROBE_FAMILY, prompt_version=PROMPT_VERSION, variants=tuple(PROBE_VARIANTS))
42
+ class OpenEndedProbe(BaseProbe):
43
+ """Open-ended conversation probe.
44
+
45
+ Test the assistant's general info-seeking / Q&A / recommendation
46
+ behavior with persona-grounded, theme-conditioned conversations.
47
+ No tools; no curated asset bank; quality judgment lives in the
48
+ out-of-sim trajectory evaluator.
49
+ """
50
+
51
+ label = "general_open_ended"
52
+
53
+ def __init__(self, **kwargs: Any) -> None:
54
+ super().__init__(**kwargs)
55
+ theme = _parse_theme(self._data.get(getattr(self._cfg, "theme_column", "theme"), "{}"))
56
+ # Parsed once; the prompt itself is rendered per call in
57
+ # get_user_system_prompt so all three prompt hooks resolve alike.
58
+ self._topic = theme.get("description", theme.get("type", "general topic"))
59
+ self._disclosure_style = self._data.get("disclosure_style", "upfront")
60
+
61
+ def get_user_system_prompt(self) -> str:
62
+ return render_prompt(
63
+ user_agent_system_prompt(),
64
+ self._data,
65
+ persona=format_persona_for_prompt(self._persona),
66
+ topic=self._topic,
67
+ language_instruction=language_instruction(self._language, self._locale),
68
+ behavioral_instructions=format_behavioral_profile_for_prompt(
69
+ self._profile,
70
+ language=self._language,
71
+ ),
72
+ disclosure_instructions=format_disclosure_instructions(
73
+ self._disclosure_style,
74
+ ),
75
+ interaction_style_instructions=format_interaction_style_instructions(
76
+ self._interaction_style,
77
+ ),
78
+ )
79
+
80
+ def get_assistant_system_prompt(self) -> str:
81
+ # Empty as shipped; see the accessor's docstring.
82
+ return render_prompt(assistant_system_prompt(), self._data)
83
+
84
+ def format_gate_prompt(
85
+ self,
86
+ user_query: str,
87
+ conversation_history: str,
88
+ ) -> str:
89
+ return render_prompt(
90
+ user_judge_turn_prompt(),
91
+ self._data,
92
+ conversation_history=conversation_history,
93
+ user_turn_to_evaluate=user_query,
94
+ )
95
+
96
+
97
+ async def simulate_general_open_ended(
98
+ models: dict,
99
+ data: dict,
100
+ persona: dict,
101
+ profile: dict,
102
+ locale: str,
103
+ language: str,
104
+ cfg: Any,
105
+ **kwargs: Any,
106
+ ) -> dict:
107
+ """Thin shim for callers that import ``simulate_general_open_ended`` directly.
108
+
109
+ The dispatcher in ``usersim.engine.generator`` instantiates
110
+ ``OpenEndedProbe`` and calls ``probe.run_dispatch`` directly; this
111
+ function exists for backward compatibility with per-probe tests
112
+ that import the symbol.
113
+ Constructs a probe instance and routes through the standard
114
+ ``run_dispatch`` so behavior is identical to the dispatcher path.
115
+ """
116
+ probe = OpenEndedProbe(
117
+ persona=persona,
118
+ locale=locale,
119
+ language=language,
120
+ models=models,
121
+ cfg=cfg,
122
+ provenance=kwargs.get("provenance"),
123
+ profile=profile,
124
+ data=data,
125
+ outcome_builder=kwargs.get("outcome_builder"),
126
+ )
127
+ return await dispatch_episode(probe, models=models, data=data, cfg=cfg)