usersim 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (379) hide show
  1. usersim/__init__.py +23 -0
  2. usersim/asset_gen/__init__.py +22 -0
  3. usersim/asset_gen/financial_services/__init__.py +22 -0
  4. usersim/asset_gen/financial_services/evaluate.py +330 -0
  5. usersim/asset_gen/financial_services/pipeline.py +1590 -0
  6. usersim/asset_gen/financial_services/prompts.py +318 -0
  7. usersim/asset_gen/financial_services/spec.py +794 -0
  8. usersim/asset_gen/financial_services/validate.py +750 -0
  9. usersim/asset_gen/registry.py +191 -0
  10. usersim/cli/__init__.py +185 -0
  11. usersim/cli/__main__.py +13 -0
  12. usersim/cli/_async.py +35 -0
  13. usersim/cli/_env.py +74 -0
  14. usersim/cli/_errors.py +44 -0
  15. usersim/cli/_inputs.py +691 -0
  16. usersim/cli/_models.py +1222 -0
  17. usersim/cli/_ngc.py +441 -0
  18. usersim/cli/_persona_language.py +316 -0
  19. usersim/cli/_pipeline.py +1007 -0
  20. usersim/cli/evaluate.py +415 -0
  21. usersim/cli/evaluate_assets.py +115 -0
  22. usersim/cli/gen_assets.py +120 -0
  23. usersim/cli/model_catalog.py +451 -0
  24. usersim/cli/models_default.toml +46 -0
  25. usersim/cli/models_openai.toml +26 -0
  26. usersim/cli/models_openrouter.toml +33 -0
  27. usersim/cli/panel.py +193 -0
  28. usersim/cli/report.py +247 -0
  29. usersim/cli/rescore.py +223 -0
  30. usersim/cli/select.py +304 -0
  31. usersim/cli/setup_ngc.py +131 -0
  32. usersim/cli/simulate.py +866 -0
  33. usersim/cli/smoke.py +1019 -0
  34. usersim/cli/validate_assets.py +64 -0
  35. usersim/engine/__init__.py +4 -0
  36. usersim/engine/assets/financial_services/SCHEMA.md +320 -0
  37. usersim/engine/assets/financial_services/de_DE/_generation_report.json +945 -0
  38. usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/corpus.yaml +8006 -0
  39. usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/institution_meta.yaml +9 -0
  40. usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/tasks.yaml +366 -0
  41. usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/tools.yaml +114 -0
  42. usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/corpus.yaml +10053 -0
  43. usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/institution_meta.yaml +10 -0
  44. usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/tasks.yaml +578 -0
  45. usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/tools.yaml +154 -0
  46. usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/corpus.yaml +6784 -0
  47. usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/institution_meta.yaml +8 -0
  48. usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/tasks.yaml +394 -0
  49. usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/tools.yaml +134 -0
  50. usersim/engine/assets/financial_services/de_DE/dynamic.yaml +252 -0
  51. usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/corpus.yaml +9404 -0
  52. usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/institution_meta.yaml +8 -0
  53. usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/tasks.yaml +394 -0
  54. usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/tools.yaml +114 -0
  55. usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/corpus.yaml +8181 -0
  56. usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/institution_meta.yaml +10 -0
  57. usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/tasks.yaml +440 -0
  58. usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/tools.yaml +144 -0
  59. usersim/engine/assets/financial_services/de_DE/region_meta.yaml +108 -0
  60. usersim/engine/assets/financial_services/en_GB/_generation_report.json +1330 -0
  61. usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/corpus.yaml +11053 -0
  62. usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/institution_meta.yaml +9 -0
  63. usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/tasks.yaml +401 -0
  64. usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/tools.yaml +114 -0
  65. usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/corpus.yaml +13830 -0
  66. usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/institution_meta.yaml +11 -0
  67. usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/tasks.yaml +597 -0
  68. usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/tools.yaml +159 -0
  69. usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/corpus.yaml +9717 -0
  70. usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/institution_meta.yaml +8 -0
  71. usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/tasks.yaml +418 -0
  72. usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/tools.yaml +139 -0
  73. usersim/engine/assets/financial_services/en_GB/dynamic.yaml +309 -0
  74. usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/corpus.yaml +7891 -0
  75. usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/institution_meta.yaml +8 -0
  76. usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/tasks.yaml +345 -0
  77. usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/tools.yaml +114 -0
  78. usersim/engine/assets/financial_services/en_GB/neobank/quill/corpus.yaml +9579 -0
  79. usersim/engine/assets/financial_services/en_GB/neobank/quill/institution_meta.yaml +10 -0
  80. usersim/engine/assets/financial_services/en_GB/neobank/quill/tasks.yaml +453 -0
  81. usersim/engine/assets/financial_services/en_GB/neobank/quill/tools.yaml +154 -0
  82. usersim/engine/assets/financial_services/en_GB/region_meta.yaml +128 -0
  83. usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/corpus.yaml +8028 -0
  84. usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/institution_meta.yaml +8 -0
  85. usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/tasks.yaml +387 -0
  86. usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/tools.yaml +124 -0
  87. usersim/engine/assets/financial_services/en_IN/_generation_report.json +977 -0
  88. usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/corpus.yaml +9170 -0
  89. usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/institution_meta.yaml +10 -0
  90. usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/tasks.yaml +330 -0
  91. usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/tools.yaml +139 -0
  92. usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/corpus.yaml +7436 -0
  93. usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/institution_meta.yaml +8 -0
  94. usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/tasks.yaml +323 -0
  95. usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/tools.yaml +129 -0
  96. usersim/engine/assets/financial_services/en_IN/dynamic.yaml +256 -0
  97. usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/corpus.yaml +7641 -0
  98. usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/institution_meta.yaml +8 -0
  99. usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/tasks.yaml +350 -0
  100. usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/tools.yaml +114 -0
  101. usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/corpus.yaml +9453 -0
  102. usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/institution_meta.yaml +9 -0
  103. usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/tasks.yaml +346 -0
  104. usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/tools.yaml +114 -0
  105. usersim/engine/assets/financial_services/en_IN/neobank/paysetu/corpus.yaml +8237 -0
  106. usersim/engine/assets/financial_services/en_IN/neobank/paysetu/institution_meta.yaml +10 -0
  107. usersim/engine/assets/financial_services/en_IN/neobank/paysetu/tasks.yaml +313 -0
  108. usersim/engine/assets/financial_services/en_IN/neobank/paysetu/tools.yaml +139 -0
  109. usersim/engine/assets/financial_services/en_IN/region_meta.yaml +81 -0
  110. usersim/engine/assets/financial_services/en_US/_generation_report.json +768 -0
  111. usersim/engine/assets/financial_services/en_US/bank/northwind_bank/corpus.yaml +9131 -0
  112. usersim/engine/assets/financial_services/en_US/bank/northwind_bank/institution_meta.yaml +9 -0
  113. usersim/engine/assets/financial_services/en_US/bank/northwind_bank/tasks.yaml +473 -0
  114. usersim/engine/assets/financial_services/en_US/bank/northwind_bank/tools.yaml +129 -0
  115. usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/corpus.yaml +7868 -0
  116. usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/institution_meta.yaml +8 -0
  117. usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/tasks.yaml +388 -0
  118. usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/tools.yaml +124 -0
  119. usersim/engine/assets/financial_services/en_US/dynamic.yaml +198 -0
  120. usersim/engine/assets/financial_services/en_US/region_meta.yaml +58 -0
  121. usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/corpus.yaml +8117 -0
  122. usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/institution_meta.yaml +8 -0
  123. usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/tasks.yaml +369 -0
  124. usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/tools.yaml +109 -0
  125. usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/corpus.yaml +6897 -0
  126. usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/institution_meta.yaml +8 -0
  127. usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/tasks.yaml +319 -0
  128. usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/tools.yaml +109 -0
  129. usersim/engine/assets/financial_services/fr_FR/_generation_report.json +1041 -0
  130. usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/corpus.yaml +10842 -0
  131. usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/institution_meta.yaml +10 -0
  132. usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/tasks.yaml +529 -0
  133. usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/tools.yaml +144 -0
  134. usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/corpus.yaml +9091 -0
  135. usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/institution_meta.yaml +8 -0
  136. usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/tasks.yaml +511 -0
  137. usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/tools.yaml +129 -0
  138. usersim/engine/assets/financial_services/fr_FR/dynamic.yaml +264 -0
  139. usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/corpus.yaml +9033 -0
  140. usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/institution_meta.yaml +8 -0
  141. usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/tasks.yaml +505 -0
  142. usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/tools.yaml +134 -0
  143. usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/corpus.yaml +7494 -0
  144. usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/institution_meta.yaml +9 -0
  145. usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/tasks.yaml +383 -0
  146. usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/tools.yaml +114 -0
  147. usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/corpus.yaml +7883 -0
  148. usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/institution_meta.yaml +10 -0
  149. usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/tasks.yaml +430 -0
  150. usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/tools.yaml +139 -0
  151. usersim/engine/assets/financial_services/fr_FR/region_meta.yaml +114 -0
  152. usersim/engine/assets/financial_services/hi_Deva_IN/_generation_report.json +977 -0
  153. usersim/engine/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/corpus.yaml +6265 -0
  154. usersim/engine/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/institution_meta.yaml +11 -0
  155. usersim/engine/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/tools.yaml +139 -0
  156. usersim/engine/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/corpus.yaml +5188 -0
  157. usersim/engine/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/institution_meta.yaml +9 -0
  158. usersim/engine/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/tools.yaml +129 -0
  159. usersim/engine/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/corpus.yaml +5115 -0
  160. usersim/engine/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/institution_meta.yaml +9 -0
  161. usersim/engine/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/tools.yaml +114 -0
  162. usersim/engine/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/corpus.yaml +6539 -0
  163. usersim/engine/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/institution_meta.yaml +10 -0
  164. usersim/engine/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/tools.yaml +114 -0
  165. usersim/engine/assets/financial_services/hi_Deva_IN/neobank/paysetu/corpus.yaml +5370 -0
  166. usersim/engine/assets/financial_services/hi_Deva_IN/neobank/paysetu/institution_meta.yaml +11 -0
  167. usersim/engine/assets/financial_services/hi_Deva_IN/neobank/paysetu/tools.yaml +139 -0
  168. usersim/engine/assets/financial_services/hi_Deva_IN/region_meta.yaml +81 -0
  169. usersim/engine/assets/general_educational/prompts.yaml +115 -0
  170. usersim/engine/assets/general_educational/subjects/base/subjects.json +162 -0
  171. usersim/engine/assets/general_educational/subjects/de_DE/subjects.json +162 -0
  172. usersim/engine/assets/general_educational/subjects/en_GB/subjects.json +162 -0
  173. usersim/engine/assets/general_educational/subjects/fr_FR/subjects.json +16 -0
  174. usersim/engine/assets/general_educational/subjects/hi_Deva_IN/subjects.json +14 -0
  175. usersim/engine/assets/general_educational/subjects/hi_Latn_IN/subjects.json +14 -0
  176. usersim/engine/assets/general_educational/subjects/ja_JP/subjects.json +16 -0
  177. usersim/engine/assets/general_educational/subjects/ko_KR/subjects.json +16 -0
  178. usersim/engine/assets/general_educational/subjects/pt_BR/subjects.json +12 -0
  179. usersim/engine/assets/general_open_ended/prompts.yaml +117 -0
  180. usersim/engine/assets/general_open_ended/topics/base/topics.json +194 -0
  181. usersim/engine/assets/general_open_ended/topics/de_DE/topics.json +194 -0
  182. usersim/engine/assets/general_open_ended/topics/en_GB/topics.json +194 -0
  183. usersim/engine/assets/general_open_ended/topics/fr_FR/topics.json +22 -0
  184. usersim/engine/assets/general_open_ended/topics/hi_Deva_IN/topics.json +22 -0
  185. usersim/engine/assets/general_open_ended/topics/hi_Latn_IN/topics.json +22 -0
  186. usersim/engine/assets/general_open_ended/topics/ja_JP/topics.json +17 -0
  187. usersim/engine/assets/general_open_ended/topics/ko_KR/topics.json +17 -0
  188. usersim/engine/assets/general_open_ended/topics/pt_BR/topics.json +22 -0
  189. usersim/engine/assets/health_decision_support_disclosure/sample.yaml +21 -0
  190. usersim/engine/assets/health_general_disclosure/sample.yaml +21 -0
  191. usersim/engine/assets/health_therapy_disclosure/sample.yaml +23 -0
  192. usersim/engine/assets/health_triage_disclosure/sample.yaml +21 -0
  193. usersim/engine/assets/identity_disclosure/SCHEMA.md +303 -0
  194. usersim/engine/assets/identity_disclosure/spec.yaml +1007 -0
  195. usersim/engine/assets/safety_agentic/SCHEMA.md +252 -0
  196. usersim/engine/assets/safety_agentic/action_taxonomy.yaml +682 -0
  197. usersim/engine/assets/safety_agentic/sample.yaml +3993 -0
  198. usersim/engine/assets/safety_chat_pressure/SCHEMA.md +181 -0
  199. usersim/engine/assets/safety_chat_pressure/sample.yaml +3646 -0
  200. usersim/engine/assets/sov_ai_dynamic/SCHEMA.md +98 -0
  201. usersim/engine/assets/sov_ai_dynamic/de_DE/categories.yaml +116 -0
  202. usersim/engine/assets/sov_ai_dynamic/en_GB/categories.yaml +116 -0
  203. usersim/engine/assets/sov_ai_dynamic/en_IN/categories.yaml +116 -0
  204. usersim/engine/assets/sov_ai_dynamic/en_SG/categories.yaml +116 -0
  205. usersim/engine/assets/sov_ai_dynamic/en_US/categories.yaml +116 -0
  206. usersim/engine/assets/sov_ai_dynamic/fr_FR/categories.yaml +116 -0
  207. usersim/engine/assets/sov_ai_dynamic/hi_Deva_IN/categories.yaml +116 -0
  208. usersim/engine/assets/sov_ai_dynamic/hi_Latn_IN/categories.yaml +116 -0
  209. usersim/engine/assets/sov_ai_dynamic/ja_JP/categories.yaml +116 -0
  210. usersim/engine/assets/sov_ai_dynamic/ko_KR/categories.yaml +116 -0
  211. usersim/engine/assets/sov_ai_dynamic/pt_BR/categories.yaml +116 -0
  212. usersim/engine/assets/sov_ai_facts/SCHEMA.md +196 -0
  213. usersim/engine/assets/sov_ai_facts/de_DE/sample.yaml +1421 -0
  214. usersim/engine/assets/sov_ai_facts/en_GB/sample.yaml +1389 -0
  215. usersim/engine/assets/sov_ai_facts/en_IN/sample.yaml +1275 -0
  216. usersim/engine/assets/sov_ai_facts/en_SG/sample.yaml +1358 -0
  217. usersim/engine/assets/sov_ai_facts/en_US/sample.yaml +1270 -0
  218. usersim/engine/assets/sov_ai_facts/fr_FR/sample.yaml +1287 -0
  219. usersim/engine/assets/sov_ai_facts/hi_Deva_IN/sample.yaml +1272 -0
  220. usersim/engine/assets/sov_ai_facts/hi_Latn_IN/sample.yaml +1272 -0
  221. usersim/engine/assets/sov_ai_facts/ja_JP/sample.yaml +1294 -0
  222. usersim/engine/assets/sov_ai_facts/ko_KR/sample.yaml +1435 -0
  223. usersim/engine/assets/sov_ai_facts/pt_BR/sample.yaml +1321 -0
  224. usersim/engine/assets/sov_ai_multilingual_parity/SCHEMA.md +137 -0
  225. usersim/engine/assets/sov_ai_multilingual_parity/sample.yaml +1210 -0
  226. usersim/engine/assets/tool_calling/README.md +225 -0
  227. usersim/engine/assets/tool_calling/prompts.yaml +171 -0
  228. usersim/engine/assets/tool_calling/toolsets_seed.parquet +0 -0
  229. usersim/engine/config.py +296 -0
  230. usersim/engine/core/__init__.py +4 -0
  231. usersim/engine/core/_assets.py +178 -0
  232. usersim/engine/core/_extensions.py +148 -0
  233. usersim/engine/core/action_taxonomy.py +347 -0
  234. usersim/engine/core/agentic_bank.py +725 -0
  235. usersim/engine/core/analysis.py +60 -0
  236. usersim/engine/core/backends.py +180 -0
  237. usersim/engine/core/behavioral.py +755 -0
  238. usersim/engine/core/clinical_profile_bank.py +245 -0
  239. usersim/engine/core/context.py +137 -0
  240. usersim/engine/core/embeddings.py +83 -0
  241. usersim/engine/core/episode_input.py +281 -0
  242. usersim/engine/core/episode_runtime.py +735 -0
  243. usersim/engine/core/fact_bank.py +500 -0
  244. usersim/engine/core/finance_bank.py +970 -0
  245. usersim/engine/core/finance_tasks.py +292 -0
  246. usersim/engine/core/idempotency.py +62 -0
  247. usersim/engine/core/identity.py +215 -0
  248. usersim/engine/core/identity_spec.py +659 -0
  249. usersim/engine/core/identity_spec_types.py +425 -0
  250. usersim/engine/core/judges.py +85 -0
  251. usersim/engine/core/language_detection.py +239 -0
  252. usersim/engine/core/llm.py +710 -0
  253. usersim/engine/core/locale.py +512 -0
  254. usersim/engine/core/localized.py +172 -0
  255. usersim/engine/core/manifest.py +998 -0
  256. usersim/engine/core/messages.py +93 -0
  257. usersim/engine/core/missing.py +38 -0
  258. usersim/engine/core/outcomes.py +442 -0
  259. usersim/engine/core/persona.py +178 -0
  260. usersim/engine/core/pressure_bank.py +629 -0
  261. usersim/engine/core/preview.py +1927 -0
  262. usersim/engine/core/probes.py +1499 -0
  263. usersim/engine/core/probing_taxonomy.py +462 -0
  264. usersim/engine/core/prompt_loader.py +93 -0
  265. usersim/engine/core/prompts.py +81 -0
  266. usersim/engine/core/provenance.py +118 -0
  267. usersim/engine/core/query_bank.py +567 -0
  268. usersim/engine/core/realized_audit.py +411 -0
  269. usersim/engine/core/row_metadata.py +35 -0
  270. usersim/engine/core/seeds.py +82 -0
  271. usersim/engine/core/simulation.py +1899 -0
  272. usersim/engine/core/storage.py +1407 -0
  273. usersim/engine/core/tool_calls.py +106 -0
  274. usersim/engine/core/translation.py +266 -0
  275. usersim/engine/core/user_turn_policy.py +80 -0
  276. usersim/engine/evaluator/__init__.py +19 -0
  277. usersim/engine/evaluator/axes.py +437 -0
  278. usersim/engine/evaluator/config.py +187 -0
  279. usersim/engine/evaluator/generator.py +424 -0
  280. usersim/engine/evaluator/judges.py +151 -0
  281. usersim/engine/evaluator/prompts.py +108 -0
  282. usersim/engine/evaluator/scorers/__init__.py +289 -0
  283. usersim/engine/evaluator/scorers/financial_services.py +635 -0
  284. usersim/engine/evaluator/scorers/health_disclosure.py +503 -0
  285. usersim/engine/evaluator/scorers/identity_disclosure.py +545 -0
  286. usersim/engine/evaluator/scorers/identity_grading.py +543 -0
  287. usersim/engine/evaluator/scorers/language_compliance.py +426 -0
  288. usersim/engine/evaluator/scorers/refusal_basics.py +479 -0
  289. usersim/engine/evaluator/scorers/response_shape.py +363 -0
  290. usersim/engine/evaluator/scorers/safety_agentic.py +1010 -0
  291. usersim/engine/evaluator/scorers/safety_chat_pressure.py +791 -0
  292. usersim/engine/evaluator/scorers/sov_ai_dynamic.py +629 -0
  293. usersim/engine/evaluator/scorers/sov_ai_facts.py +648 -0
  294. usersim/engine/evaluator/scorers/sov_ai_multilingual_parity.py +550 -0
  295. usersim/engine/evaluator/scorers/tool_use.py +291 -0
  296. usersim/engine/external.py +218 -0
  297. usersim/engine/generator.py +419 -0
  298. usersim/engine/plugin.py +34 -0
  299. usersim/engine/probes/__init__.py +28 -0
  300. usersim/engine/probes/financial_services/__init__.py +16 -0
  301. usersim/engine/probes/financial_services/generator.py +1329 -0
  302. usersim/engine/probes/financial_services/prompts.py +132 -0
  303. usersim/engine/probes/financial_services/retrieval.py +229 -0
  304. usersim/engine/probes/financial_services/task_derivation.py +274 -0
  305. usersim/engine/probes/financial_services/tools.py +116 -0
  306. usersim/engine/probes/general_educational/__init__.py +4 -0
  307. usersim/engine/probes/general_educational/generator.py +125 -0
  308. usersim/engine/probes/general_educational/prompts.py +207 -0
  309. usersim/engine/probes/general_open_ended/__init__.py +4 -0
  310. usersim/engine/probes/general_open_ended/generator.py +127 -0
  311. usersim/engine/probes/general_open_ended/prompts.py +209 -0
  312. usersim/engine/probes/health_disclosure/__init__.py +28 -0
  313. usersim/engine/probes/health_disclosure/base.py +148 -0
  314. usersim/engine/probes/health_disclosure/clients.py +291 -0
  315. usersim/engine/probes/health_disclosure/decision_support.py +44 -0
  316. usersim/engine/probes/health_disclosure/general.py +45 -0
  317. usersim/engine/probes/health_disclosure/mixin.py +363 -0
  318. usersim/engine/probes/health_disclosure/move_runtime.py +486 -0
  319. usersim/engine/probes/health_disclosure/moves.py +241 -0
  320. usersim/engine/probes/health_disclosure/prompts.py +324 -0
  321. usersim/engine/probes/health_disclosure/therapy.py +43 -0
  322. usersim/engine/probes/health_disclosure/triage.py +43 -0
  323. usersim/engine/probes/identity_disclosure/__init__.py +4 -0
  324. usersim/engine/probes/identity_disclosure/generator.py +504 -0
  325. usersim/engine/probes/identity_disclosure/prompts.py +219 -0
  326. usersim/engine/probes/identity_disclosure/task_derivation.py +227 -0
  327. usersim/engine/probes/safety_agentic/__init__.py +24 -0
  328. usersim/engine/probes/safety_agentic/generator.py +666 -0
  329. usersim/engine/probes/safety_agentic/prompts.py +24 -0
  330. usersim/engine/probes/safety_agentic/task_derivation.py +109 -0
  331. usersim/engine/probes/safety_chat_pressure/__init__.py +24 -0
  332. usersim/engine/probes/safety_chat_pressure/classifier.py +113 -0
  333. usersim/engine/probes/safety_chat_pressure/generator.py +551 -0
  334. usersim/engine/probes/safety_chat_pressure/prompts.py +141 -0
  335. usersim/engine/probes/safety_chat_pressure/task_derivation.py +171 -0
  336. usersim/engine/probes/sov_ai_dynamic/__init__.py +28 -0
  337. usersim/engine/probes/sov_ai_dynamic/generator.py +393 -0
  338. usersim/engine/probes/sov_ai_dynamic/prompts.py +472 -0
  339. usersim/engine/probes/sov_ai_dynamic/task_derivation.py +106 -0
  340. usersim/engine/probes/sov_ai_facts/__init__.py +15 -0
  341. usersim/engine/probes/sov_ai_facts/generator.py +342 -0
  342. usersim/engine/probes/sov_ai_facts/prompts.py +714 -0
  343. usersim/engine/probes/sov_ai_facts/task_derivation.py +479 -0
  344. usersim/engine/probes/sov_ai_multilingual_parity/__init__.py +19 -0
  345. usersim/engine/probes/sov_ai_multilingual_parity/generator.py +379 -0
  346. usersim/engine/probes/sov_ai_multilingual_parity/prompts.py +456 -0
  347. usersim/engine/probes/sov_ai_multilingual_parity/task_derivation.py +104 -0
  348. usersim/engine/probes/tool_calling/__init__.py +9 -0
  349. usersim/engine/probes/tool_calling/generator.py +437 -0
  350. usersim/engine/probes/tool_calling/prompts.py +221 -0
  351. usersim/engine/probes/tool_calling/verifiers.py +165 -0
  352. usersim/py.typed +0 -0
  353. usersim/reporting/__init__.py +116 -0
  354. usersim/reporting/_reasoning.py +201 -0
  355. usersim/reporting/capability_report.py +2049 -0
  356. usersim/reporting/comparison.py +797 -0
  357. usersim/reporting/comparison_dashboard.py +2938 -0
  358. usersim/reporting/dashboard.py +1731 -0
  359. usersim/reporting/diagnostics.py +435 -0
  360. usersim/reporting/eval_manifest.py +291 -0
  361. usersim/reporting/identity.py +82 -0
  362. usersim/reporting/resource.py +476 -0
  363. usersim/reporting/runs.py +282 -0
  364. usersim/reporting/sim_health.py +334 -0
  365. usersim/selection/__init__.py +87 -0
  366. usersim/selection/engine.py +423 -0
  367. usersim/selection/io.py +391 -0
  368. usersim/selection/profiles.py +249 -0
  369. usersim/selection/schemas.py +194 -0
  370. usersim/taxonomy/__init__.py +38 -0
  371. usersim/taxonomy/capabilities.py +1125 -0
  372. usersim/taxonomy/eval_cell.py +131 -0
  373. usersim/testing/__init__.py +534 -0
  374. usersim-0.2.0.dist-info/METADATA +392 -0
  375. usersim-0.2.0.dist-info/RECORD +379 -0
  376. usersim-0.2.0.dist-info/WHEEL +5 -0
  377. usersim-0.2.0.dist-info/entry_points.txt +6 -0
  378. usersim-0.2.0.dist-info/licenses/LICENSE +204 -0
  379. usersim-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,93 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Message and data formatting helpers for conversation simulation."""
5
+
6
+ from __future__ import annotations
7
+
8
+ import json
9
+ from typing import Any
10
+
11
+
12
+ def format_tools_for_prompt(tools: list[dict[str, Any]]) -> str:
13
+ """Format a list of tool definitions into a human-readable prompt block."""
14
+ lines = []
15
+ for tool_data in tools:
16
+ tool = tool_data.get("tool", {}) if "tool" in tool_data else tool_data
17
+ name = tool.get("function", {}).get("name", "Unknown")
18
+ description = tool.get("function", {}).get("description", "No description")
19
+ lines.append(f"- {name}: {description}")
20
+ return "\n".join(lines)
21
+
22
+
23
+ def format_themes_for_prompt(themes: list[dict[str, Any]]) -> str:
24
+ """Format a list of theme dicts into a human-readable prompt block."""
25
+ lines = []
26
+ for theme in themes:
27
+ lines.append(f"- {theme.get('type', 'Unknown')}: {theme.get('description', 'No description')}")
28
+ return "\n".join(lines)
29
+
30
+
31
+ def format_conversation_history_for_prompt(messages: list[dict[str, Any]]) -> str:
32
+ """Format conversation messages into a text block for judge prompts."""
33
+ lines = []
34
+ for msg in messages:
35
+ role = msg.get("role", "unknown")
36
+ content = msg.get("content", "")
37
+ tool_calls = msg.get("tool_calls")
38
+
39
+ if role == "system":
40
+ continue
41
+ elif role == "user":
42
+ lines.append(f"User: {content}")
43
+ elif role == "assistant":
44
+ if tool_calls:
45
+ tc_summary = []
46
+ for tc in tool_calls:
47
+ fn = tc.get("function", {})
48
+ tc_summary.append(f"{fn.get('name', '?')}({fn.get('arguments', '{}')})")
49
+ lines.append(f"Assistant [tool calls]: {'; '.join(tc_summary)}")
50
+ if content:
51
+ lines.append(f"Assistant: {content}")
52
+ elif role == "tool":
53
+ lines.append(f"Tool response: {content[:500]}")
54
+
55
+ return "\n".join(lines) if lines else "N/A"
56
+
57
+
58
+ def project_public_dialogue(
59
+ messages: list[dict[str, Any]],
60
+ ) -> list[dict[str, Any]]:
61
+ """Return only text a real user could observe.
62
+
63
+ Assistant text remains public even when its source message also contains a
64
+ tool call. Protocol envelopes, arguments, and raw tool results do not.
65
+ """
66
+ result: list[dict[str, Any]] = []
67
+ for message in messages:
68
+ role = message.get("role")
69
+ content = message.get("content", "")
70
+ if role == "user":
71
+ result.append({"role": "user", "content": content})
72
+ elif role == "assistant" and content:
73
+ result.append({"role": "assistant", "content": content})
74
+ return result
75
+
76
+
77
+ def _parse_theme(raw: Any) -> dict[str, Any]:
78
+ """Normalise a theme value into a dict with 'type' and 'description' keys."""
79
+ if isinstance(raw, dict):
80
+ return raw
81
+ if isinstance(raw, str):
82
+ try:
83
+ parsed = json.loads(raw)
84
+ if isinstance(parsed, list):
85
+ return parsed[0] if parsed else {"type": "general", "description": "General conversation"}
86
+ if isinstance(parsed, dict):
87
+ return parsed
88
+ except (json.JSONDecodeError, TypeError):
89
+ pass
90
+ return {"type": raw, "description": raw}
91
+ if isinstance(raw, list):
92
+ return raw[0] if raw else {"type": "general", "description": "General conversation"}
93
+ return {"type": "general", "description": "General conversation"}
@@ -0,0 +1,38 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Empty table cells, however pandas represents them.
5
+
6
+ pandas holds an empty cell as None, a float NaN or ``pd.NA``, depending on the
7
+ column's dtype, how the frame was built and the installed pandas version. Code
8
+ that falls back on a falsy value (``row.get(key) or default``) behaves the same
9
+ for all three only once they are None.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import math
15
+ import sys
16
+ from collections.abc import Mapping
17
+ from typing import Any
18
+
19
+
20
+ def is_missing(value: Any) -> bool:
21
+ """True for an empty cell: None, a float NaN, ``pd.NA`` or ``pd.NaT``.
22
+
23
+ A container is never missing, whatever it holds, so a list or array cell is
24
+ never truth-tested element by element.
25
+ """
26
+ if value is None:
27
+ return True
28
+ if isinstance(value, float):
29
+ return math.isnan(value)
30
+ # pandas' own markers (NA, NaT) exist only once pandas has been imported, and
31
+ # importing it here would make every importer of this module pay for pandas.
32
+ pandas = sys.modules.get("pandas")
33
+ return pandas is not None and pandas.api.types.is_scalar(value) and bool(pandas.isna(value))
34
+
35
+
36
+ def none_for_missing(row: Mapping[str, Any]) -> dict[str, Any]:
37
+ """Copy ``row`` with every empty cell replaced by None."""
38
+ return {key: None if is_missing(value) else value for key, value in row.items()}
@@ -0,0 +1,442 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Structured outcome records emitted by the simulator.
5
+
6
+ The simulator's contract with every downstream consumer (reporting,
7
+ evaluator, training-data extraction) rests on two artifacts:
8
+
9
+ - ``SimulationOutcome`` — one record per trajectory. The row-level
10
+ aggregate: status, failure attribution, counters, per-model tokens
11
+ and calls, wall-clock, provenance, warnings.
12
+ - ``SimulationTrace`` — zero-or-more records per trajectory, keyed
13
+ (``trajectory_id`` x ``turn_idx`` x ``call_idx``), discriminated by
14
+ ``TraceKind``. Per-turn / per-call telemetry lives here: in-sim judge
15
+ traces, tool-call timings, API-response reroll details, etc. This is
16
+ the debugging surface, not the assistant-evaluation surface.
17
+
18
+ The principle: the simulator **emits** structured outcomes; the evaluator
19
+ **interprets** them. Anything that requires a model call to determine
20
+ its value is the simulator's responsibility (it cannot be reconstructed
21
+ later). Anything that is a derivation, aggregation, or judgment over
22
+ what's already captured belongs to the evaluator.
23
+
24
+ In-sim judge raw outputs (user-query gate ratings, assistant-quality
25
+ judge ratings, early-stop completion verdicts) are written to the
26
+ ``simulation_traces`` sidecar with the appropriate ``TraceKind`` — never
27
+ to ``simulation_outcome`` and never to the evaluator's ``eval_store``.
28
+
29
+ The load-bearing decisions around this
30
+ separation.
31
+ """
32
+
33
+ from __future__ import annotations
34
+
35
+ import json
36
+ from dataclasses import dataclass, field
37
+ from enum import Enum
38
+ from typing import Any
39
+
40
+ # ---------------------------------------------------------------------------
41
+ # Enums
42
+ # ---------------------------------------------------------------------------
43
+
44
+
45
+ class OutcomeStatus(str, Enum):
46
+ """Row-level outcome status."""
47
+
48
+ OK = "ok"
49
+ COMPLETED_WITH_WARNINGS = "completed_with_warnings"
50
+ FAILED = "failed"
51
+
52
+
53
+ class FailureClass(str, Enum):
54
+ """Structured classification of how a trajectory failed (if it did)."""
55
+
56
+ USER_QUERY_GATE_EXHAUSTED = "user_query_gate_exhausted"
57
+ USER_FOLLOWUP_GATE_EXHAUSTED = "user_followup_gate_exhausted"
58
+ USER_LANGUAGE_GATE_EXHAUSTED = "user_language_gate_exhausted"
59
+ API_RESPONSE_INVALID = "api_response_invalid"
60
+ TOOL_CALL_SCHEMA_ERROR = "tool_call_schema_error"
61
+ JUDGE_PARSE_ERROR = "judge_parse_error"
62
+ INFRASTRUCTURE_ERROR = "infrastructure_error"
63
+ SCENARIO_ABORTED = "scenario_aborted"
64
+ TURN_LIMIT_REACHED = "turn_limit_reached"
65
+
66
+
67
+ class FailureAttribution(str, Enum):
68
+ """Which actor caused the failure. Orthogonal to FailureClass."""
69
+
70
+ USER_MODEL = "user_model"
71
+ ASSISTANT_MODEL = "assistant_model"
72
+ API_RESPONSE_MODEL = "api_response_model"
73
+ JUDGE_MODEL = "judge_model"
74
+ SCENARIO_LOGIC = "scenario_logic"
75
+ INFRASTRUCTURE = "infrastructure"
76
+
77
+
78
+ class TraceKind(str, Enum):
79
+ """Discriminator for ``SimulationTrace`` entries.
80
+
81
+ These are *simulation-internal* events — they drive behavioral
82
+ primitives (frustration escalation, gate retries, flow control) or
83
+ capture per-call telemetry. They are NEVER read by the evaluator as
84
+ assistant-quality signal.
85
+ """
86
+
87
+ USER_QUERY_GATE = "user_query_gate"
88
+ USER_FOLLOWUP_GATE = "user_followup_gate"
89
+ FOURTH_WALL_PREFILTER = "fourth_wall_prefilter"
90
+ # User turn regenerated because its script did not match the
91
+ # locale's expected script (e.g. romanized Hindi in an hi_Deva_IN
92
+ # run). Deterministic, no model call. See the script-compliance
93
+ # gate in ``core/simulation.py``.
94
+ LANGUAGE_PREFILTER = "language_prefilter"
95
+ ASSISTANT_QUALITY = "assistant_quality"
96
+ EARLY_STOP = "early_stop"
97
+ API_RESPONSE_REROLL = "api_response_reroll"
98
+ TOOL_CALL_VERIFIER = "tool_call_verifier"
99
+ USER_ROLE_VIOLATION = "user_role_violation"
100
+ SUMMARIZATION = "summarization"
101
+ # Per-assistant-turn capitulation classifier check (yes/no
102
+ # decision made by the probe's ``is_capitulation_detected`` hook).
103
+ # Emitted regardless of result so the trace log shows the
104
+ # classifier's signal at every turn for audit purposes.
105
+ CAPITULATION_CHECK = "capitulation_check"
106
+ # A retrieval query was machine-translated into the borrowed corpus's
107
+ # language before searching (India variants, where the conversation
108
+ # language and the asset language differ). Carries both strings so a
109
+ # retrieval miss can be attributed to the translator rather than the
110
+ # assistant — the query the assistant wrote is not the query that ranked.
111
+ KB_QUERY_TRANSLATION = "kb_query_translation"
112
+
113
+
114
+ class WarningKind(str, Enum):
115
+ """Structured warning kinds for ``SimulationOutcome.warnings``."""
116
+
117
+ ID_FABRICATION = "id_fabrication"
118
+ COMPRESSION_FALLBACK = "compression_fallback"
119
+ POOR_ASSISTANT_RESPONSE = "poor_assistant_response"
120
+ # Flagged when a trajectory probed a fact that was marked
121
+ # ``placeholder: true`` in its bank. Like every ``USED_PLACEHOLDER_*``
122
+ # kind below, it is recorded on the outcome so a reader can tell which
123
+ # findings rest on placeholder content.
124
+ USED_PLACEHOLDER_FACT = "used_placeholder_fact"
125
+ # Flagged when a trajectory probed against a sov_ai_dynamic
126
+ # taxonomy that was marked ``placeholder: true`` at the file level.
127
+ # The flag is per taxonomy rather than per category, because the
128
+ # invitations and sub-topic hints share one reviewer audit.
129
+ USED_PLACEHOLDER_TAXONOMY = "used_placeholder_taxonomy"
130
+ # Flagged when a trajectory probed against a query that was marked
131
+ # ``placeholder: true`` in its query bank, for the cross-locale
132
+ # ``sov_ai_multilingual_parity`` probe. The flag is per query, because
133
+ # an entry's N locale renderings share one reviewer audit.
134
+ USED_PLACEHOLDER_QUERY = "used_placeholder_query"
135
+ # Flagged when a safety_chat_pressure trajectory probed against a
136
+ # target_request marked ``placeholder: true`` in its
137
+ # pressure-strategy bank.
138
+ USED_PLACEHOLDER_TARGET = "used_placeholder_target"
139
+ # Flagged when a safety_agentic trajectory probed against an
140
+ # action_request marked ``placeholder: true`` in its agentic-action
141
+ # bank.
142
+ USED_PLACEHOLDER_AGENTIC_ACTION = "used_placeholder_agentic_action"
143
+ # Flagged when a financial_services trajectory ran against a task
144
+ # template (verifiable tier), or an institution or taxonomy (dynamic
145
+ # tier), marked ``placeholder: true`` (no domain expert has verified it).
146
+ USED_PLACEHOLDER_FINANCE = "used_placeholder_finance"
147
+ # Flagged when a health_disclosure trajectory used a clinical profile
148
+ # marked ``placeholder: true`` in its clinical-profile bank (the bundled
149
+ # synthetic banks are all placeholders until a customer ships real ground
150
+ # truth).
151
+ USED_PLACEHOLDER_CLINICAL_PROFILE = "used_placeholder_clinical_profile"
152
+ # Flagged when an identity_disclosure trajectory drew on a spec, tactic
153
+ # or strategy marked ``placeholder: true`` (phrasings, vocabulary and
154
+ # expected identities no domain expert has verified).
155
+ USED_PLACEHOLDER_IDENTITY = "used_placeholder_identity"
156
+ # Flagged when a trajectory's verbatim turn-1 was machine-translated
157
+ # from an India base-locale (en_IN) bank into the conversation
158
+ # language, because that language has no native bank of its own;
159
+ # financial_services also flags it when kb_search queries are
160
+ # translated to search the base-locale corpus. Recorded on the outcome
161
+ # so a reader can tell which findings rest on machine-translated
162
+ # content. A locale with its own native asset never translates, so its
163
+ # rows never carry this warning.
164
+ USED_MACHINE_TRANSLATION = "used_machine_translation"
165
+
166
+
167
+ PLACEHOLDER_WARNING_KINDS: frozenset[str] = frozenset(
168
+ kind.value for kind in WarningKind if kind.name.startswith("USED_PLACEHOLDER_")
169
+ )
170
+
171
+
172
+ # ---------------------------------------------------------------------------
173
+ # Records
174
+ # ---------------------------------------------------------------------------
175
+
176
+
177
+ @dataclass
178
+ class Provenance:
179
+ """Everything needed to audit what a trajectory was built against.
180
+
181
+ Recorded on every row, so rows built against different persona
182
+ dataset, prompt, code or bank versions can be told apart.
183
+ """
184
+
185
+ nemotron_personas_version: str | None = None
186
+ scenario_prompt_version: str | None = None
187
+ code_sha: str | None = None
188
+ bank_version: dict[str, str] = field(default_factory=dict)
189
+
190
+ def to_dict(self) -> dict[str, Any]:
191
+ return {
192
+ "nemotron_personas_version": self.nemotron_personas_version,
193
+ "scenario_prompt_version": self.scenario_prompt_version,
194
+ "code_sha": self.code_sha,
195
+ "bank_version": dict(self.bank_version),
196
+ }
197
+
198
+
199
+ @dataclass
200
+ class SimulationWarning:
201
+ """Structured warning entry. Stored in ``SimulationOutcome.warnings``."""
202
+
203
+ kind: WarningKind
204
+ turn_idx: int | None = None
205
+ detail: str = ""
206
+
207
+ def to_dict(self) -> dict[str, Any]:
208
+ return {
209
+ "kind": self.kind.value,
210
+ "turn_idx": self.turn_idx,
211
+ "detail": self.detail,
212
+ }
213
+
214
+
215
+ @dataclass
216
+ class SimulationOutcome:
217
+ """The canonical per-trajectory row-level outcome record.
218
+
219
+ Everything a downstream consumer needs to answer "what happened with
220
+ this trajectory at an aggregate level?" — without re-parsing
221
+ conversation messages or invoking any model.
222
+ """
223
+
224
+ status: OutcomeStatus = OutcomeStatus.OK
225
+ failure_class: FailureClass | None = None
226
+ failure_attribution: FailureAttribution | None = None
227
+ failure_detail: str = ""
228
+
229
+ n_turns: int = 0
230
+ n_tool_calls: int = 0
231
+ n_user_query_attempts: int = 0
232
+ n_user_followup_retries: int = 0
233
+ n_assistant_inline_failures: int = 0
234
+ # Judge-rejected assistant candidates that were discarded and
235
+ # regenerated (``max_assistant_attempts > 1``). Distinct from
236
+ # ``n_assistant_inline_failures``, which counts turns whose FINAL
237
+ # candidate is flagged in the delivered transcript.
238
+ n_assistant_retries: int = 0
239
+ n_api_response_rerolls: int = 0
240
+ n_fourth_wall_triggers: int = 0
241
+ n_user_role_violations: int = 0
242
+ n_user_language_violations: int = 0
243
+
244
+ warnings: list[SimulationWarning] = field(default_factory=list)
245
+ early_stop: bool = False
246
+
247
+ per_model_input_tokens: dict[str, int] = field(default_factory=dict)
248
+ per_model_output_tokens: dict[str, int] = field(default_factory=dict)
249
+ per_model_calls: dict[str, int] = field(default_factory=dict)
250
+ wall_clock_s_by_alias: dict[str, float] = field(default_factory=dict)
251
+ wall_clock_s: float = 0.0
252
+
253
+ provenance: Provenance = field(default_factory=Provenance)
254
+
255
+ def to_dict(self) -> dict[str, Any]:
256
+ return {
257
+ "status": self.status.value,
258
+ "failure_class": self.failure_class.value if self.failure_class else None,
259
+ "failure_attribution": (self.failure_attribution.value if self.failure_attribution else None),
260
+ "failure_detail": self.failure_detail,
261
+ "n_turns": self.n_turns,
262
+ "n_tool_calls": self.n_tool_calls,
263
+ "n_user_query_attempts": self.n_user_query_attempts,
264
+ "n_user_followup_retries": self.n_user_followup_retries,
265
+ "n_assistant_inline_failures": self.n_assistant_inline_failures,
266
+ "n_assistant_retries": self.n_assistant_retries,
267
+ "n_api_response_rerolls": self.n_api_response_rerolls,
268
+ "n_fourth_wall_triggers": self.n_fourth_wall_triggers,
269
+ "n_user_role_violations": self.n_user_role_violations,
270
+ "n_user_language_violations": self.n_user_language_violations,
271
+ "warnings": [w.to_dict() for w in self.warnings],
272
+ "early_stop": self.early_stop,
273
+ "per_model_input_tokens": dict(self.per_model_input_tokens),
274
+ "per_model_output_tokens": dict(self.per_model_output_tokens),
275
+ "per_model_calls": dict(self.per_model_calls),
276
+ "wall_clock_s_by_alias": dict(self.wall_clock_s_by_alias),
277
+ "wall_clock_s": round(self.wall_clock_s, 3),
278
+ "provenance": self.provenance.to_dict(),
279
+ }
280
+
281
+ def to_json(self) -> str:
282
+ return json.dumps(self.to_dict(), ensure_ascii=False, default=str)
283
+
284
+
285
+ @dataclass
286
+ class SimulationTrace:
287
+ """A single per-turn / per-call telemetry entry.
288
+
289
+ Stored in the sidecar table keyed (``trajectory_id``, ``turn_idx``,
290
+ ``call_idx``). ``kind`` discriminates which event it captures.
291
+ """
292
+
293
+ kind: TraceKind
294
+ turn_idx: int
295
+ call_idx: int = 0
296
+ model_alias: str | None = None
297
+ input_tokens: int = 0
298
+ output_tokens: int = 0
299
+ latency_s: float = 0.0
300
+ prompt_version: str | None = None
301
+ rating: str | None = None # for judge kinds: "success" / "warn" / "failure"
302
+ detail: str | None = None # free-text: short explanation, error, etc.
303
+ extra: dict[str, Any] = field(default_factory=dict)
304
+
305
+ def to_dict(self) -> dict[str, Any]:
306
+ return {
307
+ "kind": self.kind.value,
308
+ "turn_idx": self.turn_idx,
309
+ "call_idx": self.call_idx,
310
+ "model_alias": self.model_alias,
311
+ "input_tokens": self.input_tokens,
312
+ "output_tokens": self.output_tokens,
313
+ "latency_s": round(self.latency_s, 3),
314
+ "prompt_version": self.prompt_version,
315
+ "rating": self.rating,
316
+ "detail": self.detail,
317
+ "extra": dict(self.extra),
318
+ }
319
+
320
+
321
+ # ---------------------------------------------------------------------------
322
+ # Builder — mutated during simulation; frozen into SimulationOutcome at end
323
+ # ---------------------------------------------------------------------------
324
+
325
+
326
+ class OutcomeBuilder:
327
+ """Mutable accumulator for the sim's structured outcome record.
328
+
329
+ Owned by ``ConversationState``. Mutated by the simulator as it
330
+ progresses. ``finalize(status, ...)`` returns an immutable
331
+ ``SimulationOutcome``; ``traces()`` returns the list of
332
+ ``SimulationTrace`` entries for the sidecar.
333
+ """
334
+
335
+ def __init__(self, provenance: Provenance | None = None) -> None:
336
+ self._outcome = SimulationOutcome(provenance=provenance or Provenance())
337
+ self._traces: list[SimulationTrace] = []
338
+
339
+ # ── counters ───────────────────────────────────────────────────
340
+
341
+ def inc_user_query_attempts(self) -> None:
342
+ self._outcome.n_user_query_attempts += 1
343
+
344
+ def inc_user_followup_retries(self, n: int = 1) -> None:
345
+ self._outcome.n_user_followup_retries += n
346
+
347
+ def inc_assistant_inline_failures(self) -> None:
348
+ self._outcome.n_assistant_inline_failures += 1
349
+
350
+ def inc_assistant_retries(self) -> None:
351
+ self._outcome.n_assistant_retries += 1
352
+
353
+ def inc_api_response_rerolls(self) -> None:
354
+ self._outcome.n_api_response_rerolls += 1
355
+
356
+ def inc_fourth_wall_triggers(self) -> None:
357
+ self._outcome.n_fourth_wall_triggers += 1
358
+
359
+ def inc_user_role_violations(self) -> None:
360
+ self._outcome.n_user_role_violations += 1
361
+
362
+ def inc_user_language_violations(self) -> None:
363
+ self._outcome.n_user_language_violations += 1
364
+
365
+ # ── warnings ──────────────────────────────────────────────────
366
+
367
+ def add_warning(
368
+ self,
369
+ kind: WarningKind,
370
+ turn_idx: int | None = None,
371
+ detail: str = "",
372
+ ) -> None:
373
+ self._outcome.warnings.append(SimulationWarning(kind=kind, turn_idx=turn_idx, detail=detail))
374
+
375
+ # ── traces ────────────────────────────────────────────────────
376
+
377
+ def add_trace(self, trace: SimulationTrace) -> None:
378
+ self._traces.append(trace)
379
+
380
+ def traces(self) -> list[SimulationTrace]:
381
+ return list(self._traces)
382
+
383
+ # ── per-model aggregation (called from within the LLM layer) ──
384
+
385
+ def record_call(
386
+ self,
387
+ alias: str,
388
+ input_tokens: int,
389
+ output_tokens: int,
390
+ elapsed_s: float,
391
+ ) -> None:
392
+ self._outcome.per_model_calls[alias] = self._outcome.per_model_calls.get(alias, 0) + 1
393
+ self._outcome.per_model_input_tokens[alias] = self._outcome.per_model_input_tokens.get(alias, 0) + input_tokens
394
+ self._outcome.per_model_output_tokens[alias] = (
395
+ self._outcome.per_model_output_tokens.get(alias, 0) + output_tokens
396
+ )
397
+ self._outcome.wall_clock_s_by_alias[alias] = self._outcome.wall_clock_s_by_alias.get(alias, 0.0) + elapsed_s
398
+
399
+ # ── finalization ──────────────────────────────────────────────
400
+
401
+ def set_early_stop(self, value: bool = True) -> None:
402
+ self._outcome.early_stop = value
403
+
404
+ def set_n_turns(self, n: int) -> None:
405
+ self._outcome.n_turns = n
406
+
407
+ def set_n_tool_calls(self, n: int) -> None:
408
+ self._outcome.n_tool_calls = n
409
+
410
+ def set_wall_clock_s(self, s: float) -> None:
411
+ self._outcome.wall_clock_s = s
412
+
413
+ def finalize(
414
+ self,
415
+ status: OutcomeStatus,
416
+ failure_class: FailureClass | None = None,
417
+ failure_attribution: FailureAttribution | None = None,
418
+ failure_detail: str = "",
419
+ ) -> SimulationOutcome:
420
+ self._outcome.status = status
421
+ self._outcome.failure_class = failure_class
422
+ self._outcome.failure_attribution = failure_attribution
423
+ self._outcome.failure_detail = failure_detail
424
+ # Promote to COMPLETED_WITH_WARNINGS if we finished OK but
425
+ # accumulated warnings (identifier fabrication, etc).
426
+ if status == OutcomeStatus.OK and (
427
+ self._outcome.warnings
428
+ or self._outcome.n_user_role_violations > 0
429
+ or self._outcome.n_user_language_violations > 0
430
+ ):
431
+ self._outcome.status = OutcomeStatus.COMPLETED_WITH_WARNINGS
432
+ return self._outcome
433
+
434
+
435
+ # ---------------------------------------------------------------------------
436
+ # Convenience
437
+ # ---------------------------------------------------------------------------
438
+
439
+
440
+ def serialize_traces(traces: list[SimulationTrace]) -> str:
441
+ """JSON-encode a list of traces for storage as a string column."""
442
+ return json.dumps([t.to_dict() for t in traces], ensure_ascii=False, default=str)