usersim 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (379) hide show
  1. usersim/__init__.py +23 -0
  2. usersim/asset_gen/__init__.py +22 -0
  3. usersim/asset_gen/financial_services/__init__.py +22 -0
  4. usersim/asset_gen/financial_services/evaluate.py +330 -0
  5. usersim/asset_gen/financial_services/pipeline.py +1590 -0
  6. usersim/asset_gen/financial_services/prompts.py +318 -0
  7. usersim/asset_gen/financial_services/spec.py +794 -0
  8. usersim/asset_gen/financial_services/validate.py +750 -0
  9. usersim/asset_gen/registry.py +191 -0
  10. usersim/cli/__init__.py +185 -0
  11. usersim/cli/__main__.py +13 -0
  12. usersim/cli/_async.py +35 -0
  13. usersim/cli/_env.py +74 -0
  14. usersim/cli/_errors.py +44 -0
  15. usersim/cli/_inputs.py +691 -0
  16. usersim/cli/_models.py +1222 -0
  17. usersim/cli/_ngc.py +441 -0
  18. usersim/cli/_persona_language.py +316 -0
  19. usersim/cli/_pipeline.py +1007 -0
  20. usersim/cli/evaluate.py +415 -0
  21. usersim/cli/evaluate_assets.py +115 -0
  22. usersim/cli/gen_assets.py +120 -0
  23. usersim/cli/model_catalog.py +451 -0
  24. usersim/cli/models_default.toml +46 -0
  25. usersim/cli/models_openai.toml +26 -0
  26. usersim/cli/models_openrouter.toml +33 -0
  27. usersim/cli/panel.py +193 -0
  28. usersim/cli/report.py +247 -0
  29. usersim/cli/rescore.py +223 -0
  30. usersim/cli/select.py +304 -0
  31. usersim/cli/setup_ngc.py +131 -0
  32. usersim/cli/simulate.py +866 -0
  33. usersim/cli/smoke.py +1019 -0
  34. usersim/cli/validate_assets.py +64 -0
  35. usersim/engine/__init__.py +4 -0
  36. usersim/engine/assets/financial_services/SCHEMA.md +320 -0
  37. usersim/engine/assets/financial_services/de_DE/_generation_report.json +945 -0
  38. usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/corpus.yaml +8006 -0
  39. usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/institution_meta.yaml +9 -0
  40. usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/tasks.yaml +366 -0
  41. usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/tools.yaml +114 -0
  42. usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/corpus.yaml +10053 -0
  43. usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/institution_meta.yaml +10 -0
  44. usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/tasks.yaml +578 -0
  45. usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/tools.yaml +154 -0
  46. usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/corpus.yaml +6784 -0
  47. usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/institution_meta.yaml +8 -0
  48. usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/tasks.yaml +394 -0
  49. usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/tools.yaml +134 -0
  50. usersim/engine/assets/financial_services/de_DE/dynamic.yaml +252 -0
  51. usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/corpus.yaml +9404 -0
  52. usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/institution_meta.yaml +8 -0
  53. usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/tasks.yaml +394 -0
  54. usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/tools.yaml +114 -0
  55. usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/corpus.yaml +8181 -0
  56. usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/institution_meta.yaml +10 -0
  57. usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/tasks.yaml +440 -0
  58. usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/tools.yaml +144 -0
  59. usersim/engine/assets/financial_services/de_DE/region_meta.yaml +108 -0
  60. usersim/engine/assets/financial_services/en_GB/_generation_report.json +1330 -0
  61. usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/corpus.yaml +11053 -0
  62. usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/institution_meta.yaml +9 -0
  63. usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/tasks.yaml +401 -0
  64. usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/tools.yaml +114 -0
  65. usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/corpus.yaml +13830 -0
  66. usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/institution_meta.yaml +11 -0
  67. usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/tasks.yaml +597 -0
  68. usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/tools.yaml +159 -0
  69. usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/corpus.yaml +9717 -0
  70. usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/institution_meta.yaml +8 -0
  71. usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/tasks.yaml +418 -0
  72. usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/tools.yaml +139 -0
  73. usersim/engine/assets/financial_services/en_GB/dynamic.yaml +309 -0
  74. usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/corpus.yaml +7891 -0
  75. usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/institution_meta.yaml +8 -0
  76. usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/tasks.yaml +345 -0
  77. usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/tools.yaml +114 -0
  78. usersim/engine/assets/financial_services/en_GB/neobank/quill/corpus.yaml +9579 -0
  79. usersim/engine/assets/financial_services/en_GB/neobank/quill/institution_meta.yaml +10 -0
  80. usersim/engine/assets/financial_services/en_GB/neobank/quill/tasks.yaml +453 -0
  81. usersim/engine/assets/financial_services/en_GB/neobank/quill/tools.yaml +154 -0
  82. usersim/engine/assets/financial_services/en_GB/region_meta.yaml +128 -0
  83. usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/corpus.yaml +8028 -0
  84. usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/institution_meta.yaml +8 -0
  85. usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/tasks.yaml +387 -0
  86. usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/tools.yaml +124 -0
  87. usersim/engine/assets/financial_services/en_IN/_generation_report.json +977 -0
  88. usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/corpus.yaml +9170 -0
  89. usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/institution_meta.yaml +10 -0
  90. usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/tasks.yaml +330 -0
  91. usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/tools.yaml +139 -0
  92. usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/corpus.yaml +7436 -0
  93. usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/institution_meta.yaml +8 -0
  94. usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/tasks.yaml +323 -0
  95. usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/tools.yaml +129 -0
  96. usersim/engine/assets/financial_services/en_IN/dynamic.yaml +256 -0
  97. usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/corpus.yaml +7641 -0
  98. usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/institution_meta.yaml +8 -0
  99. usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/tasks.yaml +350 -0
  100. usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/tools.yaml +114 -0
  101. usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/corpus.yaml +9453 -0
  102. usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/institution_meta.yaml +9 -0
  103. usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/tasks.yaml +346 -0
  104. usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/tools.yaml +114 -0
  105. usersim/engine/assets/financial_services/en_IN/neobank/paysetu/corpus.yaml +8237 -0
  106. usersim/engine/assets/financial_services/en_IN/neobank/paysetu/institution_meta.yaml +10 -0
  107. usersim/engine/assets/financial_services/en_IN/neobank/paysetu/tasks.yaml +313 -0
  108. usersim/engine/assets/financial_services/en_IN/neobank/paysetu/tools.yaml +139 -0
  109. usersim/engine/assets/financial_services/en_IN/region_meta.yaml +81 -0
  110. usersim/engine/assets/financial_services/en_US/_generation_report.json +768 -0
  111. usersim/engine/assets/financial_services/en_US/bank/northwind_bank/corpus.yaml +9131 -0
  112. usersim/engine/assets/financial_services/en_US/bank/northwind_bank/institution_meta.yaml +9 -0
  113. usersim/engine/assets/financial_services/en_US/bank/northwind_bank/tasks.yaml +473 -0
  114. usersim/engine/assets/financial_services/en_US/bank/northwind_bank/tools.yaml +129 -0
  115. usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/corpus.yaml +7868 -0
  116. usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/institution_meta.yaml +8 -0
  117. usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/tasks.yaml +388 -0
  118. usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/tools.yaml +124 -0
  119. usersim/engine/assets/financial_services/en_US/dynamic.yaml +198 -0
  120. usersim/engine/assets/financial_services/en_US/region_meta.yaml +58 -0
  121. usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/corpus.yaml +8117 -0
  122. usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/institution_meta.yaml +8 -0
  123. usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/tasks.yaml +369 -0
  124. usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/tools.yaml +109 -0
  125. usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/corpus.yaml +6897 -0
  126. usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/institution_meta.yaml +8 -0
  127. usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/tasks.yaml +319 -0
  128. usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/tools.yaml +109 -0
  129. usersim/engine/assets/financial_services/fr_FR/_generation_report.json +1041 -0
  130. usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/corpus.yaml +10842 -0
  131. usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/institution_meta.yaml +10 -0
  132. usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/tasks.yaml +529 -0
  133. usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/tools.yaml +144 -0
  134. usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/corpus.yaml +9091 -0
  135. usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/institution_meta.yaml +8 -0
  136. usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/tasks.yaml +511 -0
  137. usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/tools.yaml +129 -0
  138. usersim/engine/assets/financial_services/fr_FR/dynamic.yaml +264 -0
  139. usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/corpus.yaml +9033 -0
  140. usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/institution_meta.yaml +8 -0
  141. usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/tasks.yaml +505 -0
  142. usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/tools.yaml +134 -0
  143. usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/corpus.yaml +7494 -0
  144. usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/institution_meta.yaml +9 -0
  145. usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/tasks.yaml +383 -0
  146. usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/tools.yaml +114 -0
  147. usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/corpus.yaml +7883 -0
  148. usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/institution_meta.yaml +10 -0
  149. usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/tasks.yaml +430 -0
  150. usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/tools.yaml +139 -0
  151. usersim/engine/assets/financial_services/fr_FR/region_meta.yaml +114 -0
  152. usersim/engine/assets/financial_services/hi_Deva_IN/_generation_report.json +977 -0
  153. usersim/engine/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/corpus.yaml +6265 -0
  154. usersim/engine/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/institution_meta.yaml +11 -0
  155. usersim/engine/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/tools.yaml +139 -0
  156. usersim/engine/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/corpus.yaml +5188 -0
  157. usersim/engine/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/institution_meta.yaml +9 -0
  158. usersim/engine/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/tools.yaml +129 -0
  159. usersim/engine/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/corpus.yaml +5115 -0
  160. usersim/engine/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/institution_meta.yaml +9 -0
  161. usersim/engine/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/tools.yaml +114 -0
  162. usersim/engine/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/corpus.yaml +6539 -0
  163. usersim/engine/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/institution_meta.yaml +10 -0
  164. usersim/engine/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/tools.yaml +114 -0
  165. usersim/engine/assets/financial_services/hi_Deva_IN/neobank/paysetu/corpus.yaml +5370 -0
  166. usersim/engine/assets/financial_services/hi_Deva_IN/neobank/paysetu/institution_meta.yaml +11 -0
  167. usersim/engine/assets/financial_services/hi_Deva_IN/neobank/paysetu/tools.yaml +139 -0
  168. usersim/engine/assets/financial_services/hi_Deva_IN/region_meta.yaml +81 -0
  169. usersim/engine/assets/general_educational/prompts.yaml +115 -0
  170. usersim/engine/assets/general_educational/subjects/base/subjects.json +162 -0
  171. usersim/engine/assets/general_educational/subjects/de_DE/subjects.json +162 -0
  172. usersim/engine/assets/general_educational/subjects/en_GB/subjects.json +162 -0
  173. usersim/engine/assets/general_educational/subjects/fr_FR/subjects.json +16 -0
  174. usersim/engine/assets/general_educational/subjects/hi_Deva_IN/subjects.json +14 -0
  175. usersim/engine/assets/general_educational/subjects/hi_Latn_IN/subjects.json +14 -0
  176. usersim/engine/assets/general_educational/subjects/ja_JP/subjects.json +16 -0
  177. usersim/engine/assets/general_educational/subjects/ko_KR/subjects.json +16 -0
  178. usersim/engine/assets/general_educational/subjects/pt_BR/subjects.json +12 -0
  179. usersim/engine/assets/general_open_ended/prompts.yaml +117 -0
  180. usersim/engine/assets/general_open_ended/topics/base/topics.json +194 -0
  181. usersim/engine/assets/general_open_ended/topics/de_DE/topics.json +194 -0
  182. usersim/engine/assets/general_open_ended/topics/en_GB/topics.json +194 -0
  183. usersim/engine/assets/general_open_ended/topics/fr_FR/topics.json +22 -0
  184. usersim/engine/assets/general_open_ended/topics/hi_Deva_IN/topics.json +22 -0
  185. usersim/engine/assets/general_open_ended/topics/hi_Latn_IN/topics.json +22 -0
  186. usersim/engine/assets/general_open_ended/topics/ja_JP/topics.json +17 -0
  187. usersim/engine/assets/general_open_ended/topics/ko_KR/topics.json +17 -0
  188. usersim/engine/assets/general_open_ended/topics/pt_BR/topics.json +22 -0
  189. usersim/engine/assets/health_decision_support_disclosure/sample.yaml +21 -0
  190. usersim/engine/assets/health_general_disclosure/sample.yaml +21 -0
  191. usersim/engine/assets/health_therapy_disclosure/sample.yaml +23 -0
  192. usersim/engine/assets/health_triage_disclosure/sample.yaml +21 -0
  193. usersim/engine/assets/identity_disclosure/SCHEMA.md +303 -0
  194. usersim/engine/assets/identity_disclosure/spec.yaml +1007 -0
  195. usersim/engine/assets/safety_agentic/SCHEMA.md +252 -0
  196. usersim/engine/assets/safety_agentic/action_taxonomy.yaml +682 -0
  197. usersim/engine/assets/safety_agentic/sample.yaml +3993 -0
  198. usersim/engine/assets/safety_chat_pressure/SCHEMA.md +181 -0
  199. usersim/engine/assets/safety_chat_pressure/sample.yaml +3646 -0
  200. usersim/engine/assets/sov_ai_dynamic/SCHEMA.md +98 -0
  201. usersim/engine/assets/sov_ai_dynamic/de_DE/categories.yaml +116 -0
  202. usersim/engine/assets/sov_ai_dynamic/en_GB/categories.yaml +116 -0
  203. usersim/engine/assets/sov_ai_dynamic/en_IN/categories.yaml +116 -0
  204. usersim/engine/assets/sov_ai_dynamic/en_SG/categories.yaml +116 -0
  205. usersim/engine/assets/sov_ai_dynamic/en_US/categories.yaml +116 -0
  206. usersim/engine/assets/sov_ai_dynamic/fr_FR/categories.yaml +116 -0
  207. usersim/engine/assets/sov_ai_dynamic/hi_Deva_IN/categories.yaml +116 -0
  208. usersim/engine/assets/sov_ai_dynamic/hi_Latn_IN/categories.yaml +116 -0
  209. usersim/engine/assets/sov_ai_dynamic/ja_JP/categories.yaml +116 -0
  210. usersim/engine/assets/sov_ai_dynamic/ko_KR/categories.yaml +116 -0
  211. usersim/engine/assets/sov_ai_dynamic/pt_BR/categories.yaml +116 -0
  212. usersim/engine/assets/sov_ai_facts/SCHEMA.md +196 -0
  213. usersim/engine/assets/sov_ai_facts/de_DE/sample.yaml +1421 -0
  214. usersim/engine/assets/sov_ai_facts/en_GB/sample.yaml +1389 -0
  215. usersim/engine/assets/sov_ai_facts/en_IN/sample.yaml +1275 -0
  216. usersim/engine/assets/sov_ai_facts/en_SG/sample.yaml +1358 -0
  217. usersim/engine/assets/sov_ai_facts/en_US/sample.yaml +1270 -0
  218. usersim/engine/assets/sov_ai_facts/fr_FR/sample.yaml +1287 -0
  219. usersim/engine/assets/sov_ai_facts/hi_Deva_IN/sample.yaml +1272 -0
  220. usersim/engine/assets/sov_ai_facts/hi_Latn_IN/sample.yaml +1272 -0
  221. usersim/engine/assets/sov_ai_facts/ja_JP/sample.yaml +1294 -0
  222. usersim/engine/assets/sov_ai_facts/ko_KR/sample.yaml +1435 -0
  223. usersim/engine/assets/sov_ai_facts/pt_BR/sample.yaml +1321 -0
  224. usersim/engine/assets/sov_ai_multilingual_parity/SCHEMA.md +137 -0
  225. usersim/engine/assets/sov_ai_multilingual_parity/sample.yaml +1210 -0
  226. usersim/engine/assets/tool_calling/README.md +225 -0
  227. usersim/engine/assets/tool_calling/prompts.yaml +171 -0
  228. usersim/engine/assets/tool_calling/toolsets_seed.parquet +0 -0
  229. usersim/engine/config.py +296 -0
  230. usersim/engine/core/__init__.py +4 -0
  231. usersim/engine/core/_assets.py +178 -0
  232. usersim/engine/core/_extensions.py +148 -0
  233. usersim/engine/core/action_taxonomy.py +347 -0
  234. usersim/engine/core/agentic_bank.py +725 -0
  235. usersim/engine/core/analysis.py +60 -0
  236. usersim/engine/core/backends.py +180 -0
  237. usersim/engine/core/behavioral.py +755 -0
  238. usersim/engine/core/clinical_profile_bank.py +245 -0
  239. usersim/engine/core/context.py +137 -0
  240. usersim/engine/core/embeddings.py +83 -0
  241. usersim/engine/core/episode_input.py +281 -0
  242. usersim/engine/core/episode_runtime.py +735 -0
  243. usersim/engine/core/fact_bank.py +500 -0
  244. usersim/engine/core/finance_bank.py +970 -0
  245. usersim/engine/core/finance_tasks.py +292 -0
  246. usersim/engine/core/idempotency.py +62 -0
  247. usersim/engine/core/identity.py +215 -0
  248. usersim/engine/core/identity_spec.py +659 -0
  249. usersim/engine/core/identity_spec_types.py +425 -0
  250. usersim/engine/core/judges.py +85 -0
  251. usersim/engine/core/language_detection.py +239 -0
  252. usersim/engine/core/llm.py +710 -0
  253. usersim/engine/core/locale.py +512 -0
  254. usersim/engine/core/localized.py +172 -0
  255. usersim/engine/core/manifest.py +998 -0
  256. usersim/engine/core/messages.py +93 -0
  257. usersim/engine/core/missing.py +38 -0
  258. usersim/engine/core/outcomes.py +442 -0
  259. usersim/engine/core/persona.py +178 -0
  260. usersim/engine/core/pressure_bank.py +629 -0
  261. usersim/engine/core/preview.py +1927 -0
  262. usersim/engine/core/probes.py +1499 -0
  263. usersim/engine/core/probing_taxonomy.py +462 -0
  264. usersim/engine/core/prompt_loader.py +93 -0
  265. usersim/engine/core/prompts.py +81 -0
  266. usersim/engine/core/provenance.py +118 -0
  267. usersim/engine/core/query_bank.py +567 -0
  268. usersim/engine/core/realized_audit.py +411 -0
  269. usersim/engine/core/row_metadata.py +35 -0
  270. usersim/engine/core/seeds.py +82 -0
  271. usersim/engine/core/simulation.py +1899 -0
  272. usersim/engine/core/storage.py +1407 -0
  273. usersim/engine/core/tool_calls.py +106 -0
  274. usersim/engine/core/translation.py +266 -0
  275. usersim/engine/core/user_turn_policy.py +80 -0
  276. usersim/engine/evaluator/__init__.py +19 -0
  277. usersim/engine/evaluator/axes.py +437 -0
  278. usersim/engine/evaluator/config.py +187 -0
  279. usersim/engine/evaluator/generator.py +424 -0
  280. usersim/engine/evaluator/judges.py +151 -0
  281. usersim/engine/evaluator/prompts.py +108 -0
  282. usersim/engine/evaluator/scorers/__init__.py +289 -0
  283. usersim/engine/evaluator/scorers/financial_services.py +635 -0
  284. usersim/engine/evaluator/scorers/health_disclosure.py +503 -0
  285. usersim/engine/evaluator/scorers/identity_disclosure.py +545 -0
  286. usersim/engine/evaluator/scorers/identity_grading.py +543 -0
  287. usersim/engine/evaluator/scorers/language_compliance.py +426 -0
  288. usersim/engine/evaluator/scorers/refusal_basics.py +479 -0
  289. usersim/engine/evaluator/scorers/response_shape.py +363 -0
  290. usersim/engine/evaluator/scorers/safety_agentic.py +1010 -0
  291. usersim/engine/evaluator/scorers/safety_chat_pressure.py +791 -0
  292. usersim/engine/evaluator/scorers/sov_ai_dynamic.py +629 -0
  293. usersim/engine/evaluator/scorers/sov_ai_facts.py +648 -0
  294. usersim/engine/evaluator/scorers/sov_ai_multilingual_parity.py +550 -0
  295. usersim/engine/evaluator/scorers/tool_use.py +291 -0
  296. usersim/engine/external.py +218 -0
  297. usersim/engine/generator.py +419 -0
  298. usersim/engine/plugin.py +34 -0
  299. usersim/engine/probes/__init__.py +28 -0
  300. usersim/engine/probes/financial_services/__init__.py +16 -0
  301. usersim/engine/probes/financial_services/generator.py +1329 -0
  302. usersim/engine/probes/financial_services/prompts.py +132 -0
  303. usersim/engine/probes/financial_services/retrieval.py +229 -0
  304. usersim/engine/probes/financial_services/task_derivation.py +274 -0
  305. usersim/engine/probes/financial_services/tools.py +116 -0
  306. usersim/engine/probes/general_educational/__init__.py +4 -0
  307. usersim/engine/probes/general_educational/generator.py +125 -0
  308. usersim/engine/probes/general_educational/prompts.py +207 -0
  309. usersim/engine/probes/general_open_ended/__init__.py +4 -0
  310. usersim/engine/probes/general_open_ended/generator.py +127 -0
  311. usersim/engine/probes/general_open_ended/prompts.py +209 -0
  312. usersim/engine/probes/health_disclosure/__init__.py +28 -0
  313. usersim/engine/probes/health_disclosure/base.py +148 -0
  314. usersim/engine/probes/health_disclosure/clients.py +291 -0
  315. usersim/engine/probes/health_disclosure/decision_support.py +44 -0
  316. usersim/engine/probes/health_disclosure/general.py +45 -0
  317. usersim/engine/probes/health_disclosure/mixin.py +363 -0
  318. usersim/engine/probes/health_disclosure/move_runtime.py +486 -0
  319. usersim/engine/probes/health_disclosure/moves.py +241 -0
  320. usersim/engine/probes/health_disclosure/prompts.py +324 -0
  321. usersim/engine/probes/health_disclosure/therapy.py +43 -0
  322. usersim/engine/probes/health_disclosure/triage.py +43 -0
  323. usersim/engine/probes/identity_disclosure/__init__.py +4 -0
  324. usersim/engine/probes/identity_disclosure/generator.py +504 -0
  325. usersim/engine/probes/identity_disclosure/prompts.py +219 -0
  326. usersim/engine/probes/identity_disclosure/task_derivation.py +227 -0
  327. usersim/engine/probes/safety_agentic/__init__.py +24 -0
  328. usersim/engine/probes/safety_agentic/generator.py +666 -0
  329. usersim/engine/probes/safety_agentic/prompts.py +24 -0
  330. usersim/engine/probes/safety_agentic/task_derivation.py +109 -0
  331. usersim/engine/probes/safety_chat_pressure/__init__.py +24 -0
  332. usersim/engine/probes/safety_chat_pressure/classifier.py +113 -0
  333. usersim/engine/probes/safety_chat_pressure/generator.py +551 -0
  334. usersim/engine/probes/safety_chat_pressure/prompts.py +141 -0
  335. usersim/engine/probes/safety_chat_pressure/task_derivation.py +171 -0
  336. usersim/engine/probes/sov_ai_dynamic/__init__.py +28 -0
  337. usersim/engine/probes/sov_ai_dynamic/generator.py +393 -0
  338. usersim/engine/probes/sov_ai_dynamic/prompts.py +472 -0
  339. usersim/engine/probes/sov_ai_dynamic/task_derivation.py +106 -0
  340. usersim/engine/probes/sov_ai_facts/__init__.py +15 -0
  341. usersim/engine/probes/sov_ai_facts/generator.py +342 -0
  342. usersim/engine/probes/sov_ai_facts/prompts.py +714 -0
  343. usersim/engine/probes/sov_ai_facts/task_derivation.py +479 -0
  344. usersim/engine/probes/sov_ai_multilingual_parity/__init__.py +19 -0
  345. usersim/engine/probes/sov_ai_multilingual_parity/generator.py +379 -0
  346. usersim/engine/probes/sov_ai_multilingual_parity/prompts.py +456 -0
  347. usersim/engine/probes/sov_ai_multilingual_parity/task_derivation.py +104 -0
  348. usersim/engine/probes/tool_calling/__init__.py +9 -0
  349. usersim/engine/probes/tool_calling/generator.py +437 -0
  350. usersim/engine/probes/tool_calling/prompts.py +221 -0
  351. usersim/engine/probes/tool_calling/verifiers.py +165 -0
  352. usersim/py.typed +0 -0
  353. usersim/reporting/__init__.py +116 -0
  354. usersim/reporting/_reasoning.py +201 -0
  355. usersim/reporting/capability_report.py +2049 -0
  356. usersim/reporting/comparison.py +797 -0
  357. usersim/reporting/comparison_dashboard.py +2938 -0
  358. usersim/reporting/dashboard.py +1731 -0
  359. usersim/reporting/diagnostics.py +435 -0
  360. usersim/reporting/eval_manifest.py +291 -0
  361. usersim/reporting/identity.py +82 -0
  362. usersim/reporting/resource.py +476 -0
  363. usersim/reporting/runs.py +282 -0
  364. usersim/reporting/sim_health.py +334 -0
  365. usersim/selection/__init__.py +87 -0
  366. usersim/selection/engine.py +423 -0
  367. usersim/selection/io.py +391 -0
  368. usersim/selection/profiles.py +249 -0
  369. usersim/selection/schemas.py +194 -0
  370. usersim/taxonomy/__init__.py +38 -0
  371. usersim/taxonomy/capabilities.py +1125 -0
  372. usersim/taxonomy/eval_cell.py +131 -0
  373. usersim/testing/__init__.py +534 -0
  374. usersim-0.2.0.dist-info/METADATA +392 -0
  375. usersim-0.2.0.dist-info/RECORD +379 -0
  376. usersim-0.2.0.dist-info/WHEEL +5 -0
  377. usersim-0.2.0.dist-info/entry_points.txt +6 -0
  378. usersim-0.2.0.dist-info/licenses/LICENSE +204 -0
  379. usersim-0.2.0.dist-info/top_level.txt +1 -0
usersim/cli/_inputs.py ADDED
@@ -0,0 +1,691 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Running stored episode rows: ``usersim simulate --inputs``.
5
+
6
+ Rows written by ``usersim simulate --materialize-inputs`` run exactly as they
7
+ were stored, so two setups can be compared on the same simulated users.
8
+
9
+ - **Identity.** Each run of a row gets its own ``trajectory_id``: a hash of the
10
+ stored row and of everything about the setup that can change the
11
+ conversation (see :func:`setup_fingerprint`). The stored row's own id is
12
+ saved as ``input_id``, the column that pairs two runs of the same rows.
13
+ - **The conversation.** It runs with the stored id as ``trajectory_id``, as
14
+ the hosted runtime does, so whatever a probe derives from its id (the tools
15
+ ``tool_calling`` offers, say) is the same for every setup. The new id is
16
+ written only when the rows are saved.
17
+ - **Runs.** A run holds one setup. A resume reopens the latest run of stored
18
+ rows whose setup fingerprint matches; otherwise the first saved rows claim
19
+ a new run. Every save happens under the root's write lock, after checking
20
+ the run's saved rows again.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import hashlib
26
+ import json
27
+ import logging
28
+ import os
29
+ import sys
30
+ import tempfile
31
+ import time
32
+ from collections import Counter
33
+ from collections.abc import Mapping
34
+ from dataclasses import dataclass
35
+ from pathlib import Path
36
+ from typing import Any
37
+
38
+ logger = logging.getLogger("usersim.cli.simulate")
39
+
40
+ #: The saved row's copy of the stored row's ``trajectory_id``.
41
+ INPUT_ID_COLUMN = "input_id"
42
+
43
+ #: Names the id formula, so a later formula never collides with this one.
44
+ _ID_SCHEME = "usersim-inputs-v1"
45
+
46
+ #: ``USERSIM_*`` variables that cannot change a conversation: debug logging,
47
+ #: where the models file is (the models themselves are fingerprinted),
48
+ #: provenance labels, and the evaluator's audit settings. Every other
49
+ #: ``USERSIM_*`` variable is part of the setup, including ones added later.
50
+ _OPERATIONAL_ENV = frozenset(
51
+ {
52
+ "USERSIM_DEBUG_LOG",
53
+ "USERSIM_MODELS_CONFIG",
54
+ "USERSIM_CODE_SHA",
55
+ "USERSIM_NEMOTRON_PERSONAS_VERSION",
56
+ "USERSIM_AUDIT_MODEL",
57
+ "USERSIM_AUDIT_REASONING_EFFORT",
58
+ }
59
+ )
60
+
61
+ #: Settings that may differ between the rows of one invocation.
62
+ _PER_ROW_SETTINGS = ("locale", "assets_dir")
63
+ #: Settings the run's models file decides, whatever the rows were stored with:
64
+ #: replaying the assistant's reasoning belongs to the assistant under test.
65
+ _RUN_SETTINGS = ("replay_assistant_reasoning",)
66
+
67
+
68
+ @dataclass(frozen=True)
69
+ class EpisodeInputs:
70
+ """Stored rows checked and ready to run."""
71
+
72
+ rows: list[dict[str, Any]]
73
+ #: The ``usersim_config`` every row shares, apart from ``_PER_ROW_SETTINGS``
74
+ #: and ``_RUN_SETTINGS``, which the run settles (see :func:`_run_config`).
75
+ config: dict[str, Any]
76
+ #: The asset root the rows run against.
77
+ assets_dir: Path
78
+ #: The file the rows came from, when they came from one.
79
+ source: Path | None = None
80
+
81
+
82
+ @dataclass(frozen=True)
83
+ class InputsRun:
84
+ """What one call to :func:`simulate_inputs_to_dataset` did."""
85
+
86
+ run_id: str | None
87
+ resumed: bool
88
+ written: int
89
+ skipped: int
90
+
91
+
92
+ # ── loading and checking ────────────────────────────────────────────────
93
+
94
+
95
+ def load_episode_inputs(path: str | Path) -> list[dict[str, Any]]:
96
+ """Read stored rows from a ``.jsonl`` or ``.parquet`` file, values as stored.
97
+
98
+ Parquet is read through pyarrow, not pandas, so no value is re-typed on
99
+ the way in.
100
+ """
101
+ path = Path(path)
102
+ if path.suffix == ".jsonl":
103
+ with path.open(encoding="utf-8") as handle:
104
+ return [json.loads(line) for line in handle if line.strip()]
105
+ if path.suffix == ".parquet":
106
+ import pyarrow.parquet as pq
107
+
108
+ return pq.read_table(path).to_pylist()
109
+ raise ValueError(f"expected a .jsonl or .parquet file, got {path}")
110
+
111
+
112
+ def _plain(value: Any) -> Any:
113
+ """Plain Python for a row built in memory, which may hold numpy values."""
114
+ if isinstance(value, Mapping):
115
+ return {key: _plain(item) for key, item in value.items()}
116
+ if isinstance(value, (list, tuple)):
117
+ return [_plain(item) for item in value]
118
+ if isinstance(value, str):
119
+ return str(value)
120
+ if hasattr(value, "tolist"):
121
+ return _plain(value.tolist())
122
+ return value
123
+
124
+
125
+ def prepare_episode_inputs(
126
+ rows: list[Mapping[str, Any]],
127
+ *,
128
+ assets_dir: str | Path | None = None,
129
+ source: str | Path | None = None,
130
+ ) -> EpisodeInputs:
131
+ """Check stored rows before any model call, and settle what they run against.
132
+
133
+ Raises ``ValueError`` naming the row and the problem. The rows must share
134
+ one setup: their ``usersim_config`` may differ only in locale, asset
135
+ root and the settings the run's models file decides, so that one run
136
+ holds one setup.
137
+ """
138
+ from pydantic import ValidationError
139
+
140
+ from usersim.engine.config import ConversationSimulatorConfig
141
+ from usersim.engine.core.episode_input import EPISODE_INPUT_COLUMN
142
+ from usersim.engine.core.probes import resolve_probe
143
+
144
+ if not rows:
145
+ raise ValueError("no rows to run")
146
+ checked: list[dict[str, Any]] = []
147
+ seen: set[str] = set()
148
+ for i, raw in enumerate(rows):
149
+ if not isinstance(raw, Mapping):
150
+ raise ValueError(f"row {i} is not a mapping")
151
+ row = _plain(dict(raw))
152
+ stored_id = row.get("trajectory_id")
153
+ if not isinstance(stored_id, str) or not stored_id:
154
+ raise ValueError(f"row {i} has no trajectory_id; each stored row needs its own")
155
+ where = f"row {i} ({stored_id})"
156
+ if stored_id in seen:
157
+ raise ValueError(f"{where}: trajectory_id appears in more than one row; each stored row needs its own")
158
+ seen.add(stored_id)
159
+ for reserved in (INPUT_ID_COLUMN, EPISODE_INPUT_COLUMN):
160
+ if reserved in row:
161
+ raise ValueError(f"{where} already has {reserved!r}; that is a run's output, not a stored row")
162
+ if not isinstance(row.get("usersim_config"), Mapping):
163
+ raise ValueError(f"{where} has no usersim_config; run rows written by --materialize-inputs")
164
+ try:
165
+ config = ConversationSimulatorConfig.model_validate(dict(row["usersim_config"]))
166
+ except ValidationError as error:
167
+ raise ValueError(f"{where}: usersim_config does not load: {error}") from None
168
+ if not row.get(config.persona_column):
169
+ raise ValueError(f"{where} has no {config.persona_column}")
170
+ try:
171
+ resolve_probe(str(row.get(config.probe_type_column)))
172
+ except KeyError as error:
173
+ raise ValueError(f"{where}: {error.args[0]}") from None
174
+ if "locale" in row and row["locale"] != config.locale:
175
+ raise ValueError(f"{where}: locale {row['locale']!r} differs from its config's {config.locale!r}")
176
+ try:
177
+ json.dumps(row, allow_nan=True)
178
+ except (TypeError, ValueError) as error:
179
+ raise ValueError(f"{where} holds a value JSON cannot represent: {error}") from None
180
+ checked.append(row)
181
+ return EpisodeInputs(
182
+ rows=checked,
183
+ config=_shared_config(checked),
184
+ assets_dir=_resolve_assets_dir(checked, assets_dir),
185
+ source=Path(source).resolve() if source is not None else None,
186
+ )
187
+
188
+
189
+ def _shared_config(rows: list[dict[str, Any]]) -> dict[str, Any]:
190
+ """The setup every row shares; ``ValueError`` naming the settings that differ."""
191
+
192
+ def setup(row: dict[str, Any]) -> dict[str, Any]:
193
+ return {
194
+ key: value
195
+ for key, value in row["usersim_config"].items()
196
+ if key not in _PER_ROW_SETTINGS and key not in _RUN_SETTINGS
197
+ }
198
+
199
+ first = setup(rows[0])
200
+ for row in rows[1:]:
201
+ other = setup(row)
202
+ differing = sorted(key for key in first.keys() | other.keys() if first.get(key) != other.get(key))
203
+ if differing:
204
+ shown = ", ".join(f"{key}: {first.get(key)!r} vs {other.get(key)!r}" for key in differing)
205
+ raise ValueError(
206
+ f"the rows were stored with different settings ({shown}); one run holds one setup, "
207
+ "so run them separately"
208
+ )
209
+ return dict(rows[0]["usersim_config"])
210
+
211
+
212
+ def _resolve_assets_dir(rows: list[dict[str, Any]], given: str | Path | None) -> Path:
213
+ """``--assets-dir`` when given, else the one asset root the rows were stored with."""
214
+ from usersim.engine.core._assets import packaged_assets_dir
215
+
216
+ if given is not None:
217
+ path = Path(given).resolve()
218
+ if not path.is_dir():
219
+ raise ValueError(f"--assets-dir {path} is not a directory")
220
+ return path
221
+ stored = {row["usersim_config"].get("assets_dir") for row in rows}
222
+ hint = f"pass --assets-dir (the assets that ship with User Sim are at {packaged_assets_dir()})"
223
+ if len(stored) != 1 or None in stored:
224
+ raise ValueError(f"the rows do not name one asset folder; {hint}")
225
+ path = Path(stored.pop())
226
+ if not path.is_dir():
227
+ raise ValueError(f"the asset folder the rows were stored with, {path}, does not exist here; {hint}")
228
+ return path
229
+
230
+
231
+ # ── identity ────────────────────────────────────────────────────────────
232
+
233
+
234
+ def _digest(value: Any) -> str:
235
+ """SHA-256 of ``value``'s canonical JSON form."""
236
+ text = json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":"), default=str)
237
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()
238
+
239
+
240
+ def _contents_digest(path: Path) -> str:
241
+ """SHA-256 over a file's bytes, or over every file under a folder by relative path.
242
+
243
+ Hidden files and folders (``.DS_Store``, ``.ipynb_checkpoints``) are left
244
+ out: no loader reads them, and they change without the assets changing.
245
+ """
246
+ if path.is_file():
247
+ return hashlib.sha256(path.read_bytes()).hexdigest()
248
+ entries = sorted(
249
+ item
250
+ for item in path.rglob("*")
251
+ if item.is_file() and not any(part.startswith(".") for part in item.relative_to(path).parts)
252
+ )
253
+ return _digest(
254
+ [[item.relative_to(path).as_posix(), hashlib.sha256(item.read_bytes()).hexdigest()] for item in entries]
255
+ )
256
+
257
+
258
+ def _model_settings(models: Any, aliases: list[str]) -> tuple[dict[str, Any], dict[str, Any]]:
259
+ """Each alias's settings as Data Designer sends them: readable, and as hashed.
260
+
261
+ The readable form names a provider's extra headers without their values,
262
+ which may be credentials; the hashed form carries a digest of them.
263
+ """
264
+ from usersim.cli._models import resolved_model_providers, to_model_configs
265
+
266
+ configs = {config.alias: config for config in to_model_configs(models)}
267
+ providers = {provider.name: provider for provider in resolved_model_providers(models)}
268
+ readable: dict[str, Any] = {}
269
+ hashed: dict[str, Any] = {}
270
+ for alias in sorted(set(aliases)):
271
+ config = configs.get(alias)
272
+ if config is None:
273
+ readable[alias] = hashed[alias] = None
274
+ continue
275
+ params = config.inference_parameters
276
+ settings = {"model": config.model, "request": dict(params.generate_kwargs), "timeout": params.timeout}
277
+ provider = providers.get(config.provider)
278
+ if provider is None:
279
+ readable[alias] = hashed[alias] = {**settings, "provider": None}
280
+ continue
281
+ served = {
282
+ "endpoint": provider.endpoint,
283
+ "provider_type": provider.provider_type,
284
+ "extra_body": provider.extra_body or {},
285
+ }
286
+ headers = provider.extra_headers or {}
287
+ readable[alias] = {**settings, "provider": {**served, "extra_headers": sorted(headers)}}
288
+ hashed[alias] = {**settings, "provider": {**served, "extra_headers": _digest(headers)}}
289
+ return readable, hashed
290
+
291
+
292
+ def _asset_roots_digest(assets_dir: Path) -> list[str]:
293
+ """The contents of every asset root a run searches, in priority order.
294
+
295
+ Every root counts, not just the first copy of each file: an identity spec
296
+ that ``extends`` its own probe reads the copy below it on the path. Where a
297
+ root is does not count, so the same assets in another folder are the same
298
+ setup.
299
+ """
300
+ from usersim.engine.core._assets import asset_search_path, reset_runtime_assets_dir, set_runtime_assets_dir
301
+
302
+ token = set_runtime_assets_dir(assets_dir)
303
+ try:
304
+ roots = asset_search_path()
305
+ finally:
306
+ reset_runtime_assets_dir(token)
307
+ return [_contents_digest(root) if root.exists() else "missing" for root in roots]
308
+
309
+
310
+ def _environment_settings() -> dict[str, list[str]]:
311
+ """Every ``USERSIM_*`` variable that can change a conversation.
312
+
313
+ A part of a value (split on ``os.pathsep``) that names an existing file or
314
+ folder counts by its contents, so the same bank in another place is the
315
+ same setup, and an edited bank is a different one.
316
+ """
317
+ settings: dict[str, list[str]] = {}
318
+ for name in sorted(os.environ):
319
+ if not name.startswith("USERSIM_") or name in _OPERATIONAL_ENV:
320
+ continue
321
+ parts = os.environ[name].split(os.pathsep)
322
+ settings[name] = [_contents_digest(Path(part)) if part and Path(part).exists() else part for part in parts]
323
+ return settings
324
+
325
+
326
+ def _prompt_versions() -> dict[str, str]:
327
+ """``PROMPT_VERSION`` of every registered probe, read as the episode preamble reads it."""
328
+ from usersim.engine.core.probes import known_probes, resolve_probe
329
+
330
+ return {
331
+ label: str(getattr(sys.modules.get(resolve_probe(label).__module__), "PROMPT_VERSION", "v1.0"))
332
+ for label in known_probes()
333
+ }
334
+
335
+
336
+ def _extension_identities() -> list[dict[str, Any]]:
337
+ """Every entry point User Sim discovers, with the package release behind it."""
338
+ from importlib.metadata import entry_points
339
+
340
+ from usersim.engine.core._extensions import ENTRY_POINT_GROUPS, extensions_disabled
341
+
342
+ if extensions_disabled():
343
+ return []
344
+ identities: list[dict[str, Any]] = []
345
+ for group in ENTRY_POINT_GROUPS:
346
+ for entry in sorted(entry_points(group=group), key=lambda e: (e.name, e.value)):
347
+ dist = getattr(entry, "dist", None)
348
+ identities.append(
349
+ {
350
+ "group": group,
351
+ "name": entry.name,
352
+ "value": entry.value,
353
+ "distribution": dist.name if dist else None,
354
+ "version": dist.version if dist else None,
355
+ }
356
+ )
357
+ return identities
358
+
359
+
360
+ def _data_designer_version() -> str | None:
361
+ """The installed Data Designer release, which shapes every model request."""
362
+ from usersim.engine.core.manifest import _pkg_version
363
+
364
+ return _pkg_version("data_designer")
365
+
366
+
367
+ def setup_fingerprint(models: Any, inputs: EpisodeInputs) -> tuple[str, dict[str, Any]]:
368
+ """The fingerprint of everything about a setup that can change a conversation.
369
+
370
+ Returns the fingerprint and the run record that names it. Covered:
371
+
372
+ - every simulator model alias as Data Designer sends it, provider settings
373
+ included (``api_key``, a provider's name and ``max_parallel_requests``
374
+ are left out: they don't change what a model says);
375
+ - the shared ``usersim_config`` as loaded, defaults applied, with whether
376
+ the assistant's reasoning is replayed as the run's models file sets it;
377
+ - the contents of every asset root, and every non-operational
378
+ ``USERSIM_*`` variable;
379
+ - every probe's ``PROMPT_VERSION``, every extension's package release, and
380
+ the User Sim and Data Designer releases.
381
+
382
+ A code change that bumps none of these versions is not seen.
383
+ """
384
+ from usersim.engine.config import MODEL_ALIASES, ConversationSimulatorConfig
385
+ from usersim.engine.core.manifest import _own_version
386
+ from usersim.engine.core.storage import RUN_KIND_STORED
387
+
388
+ simulation = ConversationSimulatorConfig.model_validate(_run_config(inputs, models)).model_dump(mode="json")
389
+ for key in _PER_ROW_SETTINGS:
390
+ simulation.pop(key, None)
391
+ aliases = [*MODEL_ALIASES, simulation.get("finance_embedding_model_alias") or ""]
392
+ readable, hashed = _model_settings(models, [alias for alias in aliases if alias])
393
+ fingerprint = _digest(
394
+ {
395
+ "models": hashed,
396
+ "simulation": simulation,
397
+ "assets": _asset_roots_digest(inputs.assets_dir),
398
+ "environment": _environment_settings(),
399
+ "prompts": _prompt_versions(),
400
+ "extensions": _extension_identities(),
401
+ "usersim": _own_version(),
402
+ "data_designer": _data_designer_version(),
403
+ }
404
+ )
405
+ record = {"kind": RUN_KIND_STORED, "fingerprint": fingerprint, "models": readable, "simulation": simulation}
406
+ return fingerprint, record
407
+
408
+
409
+ def replay_trajectory_id(row: Mapping[str, Any], fingerprint: str) -> str:
410
+ """The ``trajectory_id`` of ``row`` run under the setup ``fingerprint`` names.
411
+
412
+ The stored row is hashed whole, apart from its machine-specific asset
413
+ path, so a row whose contents changed under the same stored id is a
414
+ different conversation.
415
+ """
416
+ stored = dict(row)
417
+ stored["usersim_config"] = {key: value for key, value in row["usersim_config"].items() if key != "assets_dir"}
418
+ composite = f"{_ID_SCHEME}|{_digest(stored)}|{fingerprint}"
419
+ return hashlib.sha256(composite.encode("utf-8")).hexdigest()[:16]
420
+
421
+
422
+ # ── running ─────────────────────────────────────────────────────────────
423
+
424
+
425
+ def _matching_run(out: Path, fingerprint: str) -> str | None:
426
+ """The latest run of stored rows made with this setup."""
427
+ from usersim.engine.core.storage import RUN_KIND_STORED, latest_run_where
428
+
429
+ def matches(run_id: str, record: dict[str, Any] | None) -> bool:
430
+ return bool(record) and record.get("kind") == RUN_KIND_STORED and record.get("fingerprint") == fingerprint
431
+
432
+ return latest_run_where(out, matches)
433
+
434
+
435
+ def _saved_inputs(out: Path, run_id: str) -> dict[str, set[str]]:
436
+ """Each ``input_id`` saved in ``run_id``, with the trajectory ids saved under it."""
437
+ from usersim.engine.core.storage import read_partitioned_dataset
438
+
439
+ saved = read_partitioned_dataset(out, run=run_id, columns=["trajectory_id", INPUT_ID_COLUMN])
440
+ pairs: dict[str, set[str]] = {}
441
+ if not saved.empty:
442
+ for input_id, trajectory_id in zip(saved[INPUT_ID_COLUMN].astype(str), saved["trajectory_id"].astype(str)):
443
+ pairs.setdefault(input_id, set()).add(trajectory_id)
444
+ return pairs
445
+
446
+
447
+ def _unsaved(planned: list[tuple[str, str]], saved: dict[str, set[str]], run_id: str) -> tuple[set[str], int]:
448
+ """The ``(input_id, trajectory_id)`` pairs not yet saved in ``run_id``, and how many were.
449
+
450
+ Raises ``ValueError`` for an input saved there under a different id: its
451
+ stored contents changed, and one run must not hold two versions of it.
452
+ """
453
+ todo: set[str] = set()
454
+ done = 0
455
+ for input_id, trajectory_id in planned:
456
+ prior = saved.get(input_id)
457
+ if not prior:
458
+ todo.add(input_id)
459
+ elif trajectory_id in prior:
460
+ done += 1
461
+ else:
462
+ raise ValueError(
463
+ f"stored row {input_id!r} already ran in run {run_id} with different contents; give the "
464
+ "changed row its own trajectory_id, or write to another --out"
465
+ )
466
+ return todo, done
467
+
468
+
469
+ @dataclass(frozen=True)
470
+ class InputsPlan:
471
+ """What a run of stored rows would do, worked out before any model call."""
472
+
473
+ fingerprint: str
474
+ #: The record a new run with this setup is published with.
475
+ record: dict[str, Any]
476
+ #: The run of stored rows with this setup that a resume reopens, if any.
477
+ matching_run: str | None
478
+ #: Stored ids not yet saved in that run.
479
+ pending: set[str]
480
+ #: Rows already saved there.
481
+ saved: int
482
+
483
+
484
+ def plan_inputs_run(inputs: EpisodeInputs, models: Any, out: str | Path, *, skip_existing: bool = True) -> InputsPlan:
485
+ """Fingerprint the setup and find what the matching run already holds.
486
+
487
+ Raises ``ValueError`` if that run holds one of the stored rows with
488
+ different contents.
489
+ """
490
+ fingerprint, record = setup_fingerprint(models, inputs)
491
+ stored_ids = {row["trajectory_id"] for row in inputs.rows}
492
+ matching = _matching_run(Path(out).resolve(), fingerprint) if skip_existing else None
493
+ if matching is None:
494
+ return InputsPlan(fingerprint, record, matching_run=None, pending=stored_ids, saved=0)
495
+ planned = [(row["trajectory_id"], replay_trajectory_id(row, fingerprint)) for row in inputs.rows]
496
+ pending, saved = _unsaved(planned, _saved_inputs(Path(out).resolve(), matching), matching)
497
+ return InputsPlan(fingerprint, record, matching_run=matching, pending=pending, saved=saved)
498
+
499
+
500
+ def _run_config(inputs: EpisodeInputs, models: Any) -> dict[str, Any]:
501
+ """The rows' shared ``usersim_config``, with ``_RUN_SETTINGS`` as this run sets them.
502
+
503
+ The assistant under test comes from the run's models file, so whether its
504
+ earlier reasoning is replayed does too, as in a plain run. Raises
505
+ ``ConfigError`` for a models file that cannot replay.
506
+ """
507
+ from usersim.cli._models import assistant_replays_reasoning
508
+ from usersim.engine.config import ConversationSimulatorConfig
509
+
510
+ store_reasoning = ConversationSimulatorConfig.model_validate(dict(inputs.config)).store_reasoning
511
+ return {
512
+ **inputs.config,
513
+ "replay_assistant_reasoning": assistant_replays_reasoning(models, store_reasoning=store_reasoning),
514
+ }
515
+
516
+
517
+ def _column_config(inputs: EpisodeInputs, locale: str, models: Any):
518
+ """The simulator column for ``locale``'s rows, on the run's asset root."""
519
+ from usersim.engine.config import ConversationSimulatorConfig
520
+
521
+ return ConversationSimulatorConfig.model_validate(
522
+ {**_run_config(inputs, models), "locale": locale, "assets_dir": str(inputs.assets_dir)}
523
+ )
524
+
525
+
526
+ #: Value types a stored column added back to the saved rows keeps as they are.
527
+ _PLAIN_TYPES = frozenset({str, bool, int, float})
528
+
529
+
530
+ def _json_columns(rows: list[dict[str, Any]]) -> set[str]:
531
+ """The stored columns a run saves as JSON text, chosen from all of an invocation's rows.
532
+
533
+ A column is saved as it is when every value present is of one plain type
534
+ (``str``, ``bool``, ``float``, or an ``int`` within 64 bits). Anything else,
535
+ a nested value or a mix such as an ``int`` beside a ``float``, would be
536
+ converted or refused by Arrow, so it is saved as JSON text, which keeps
537
+ every value exactly. The choice depends only on the stored rows, so every
538
+ invocation over the same rows, a resume included, saves a column the same
539
+ way.
540
+ """
541
+ kinds: dict[str, set[type]] = {}
542
+ oversized: set[str] = set()
543
+ for row in rows:
544
+ for key, value in row.items():
545
+ if value is None:
546
+ continue
547
+ kinds.setdefault(key, set()).add(type(value))
548
+ if type(value) is int and not -(2**63) <= value < 2**63:
549
+ oversized.add(key)
550
+ return {key for key, types in kinds.items() if key in oversized or len(types) != 1 or not types <= _PLAIN_TYPES}
551
+
552
+
553
+ def _as_saved_rows(df, stored: dict[str, tuple[dict[str, Any], str]], json_columns: set[str]):
554
+ """The rows a run saves.
555
+
556
+ The rows get the new ids, the stored id as ``input_id``, and a plain run's
557
+ columns. ``df`` comes back from Data Designer with each stored row's own
558
+ id as ``trajectory_id``. The carrier column and the full persona are
559
+ dropped, as a plain run drops the persona, and a stored column the run did
560
+ not keep is added back from the stored rows, as JSON text when it is one
561
+ of ``json_columns``.
562
+ """
563
+ import pandas as pd
564
+
565
+ from usersim.engine.core.episode_input import EPISODE_INPUT_COLUMN
566
+
567
+ df = df.copy()
568
+ input_ids = df["trajectory_id"].astype(str).tolist()
569
+ df[INPUT_ID_COLUMN] = input_ids
570
+ df["trajectory_id"] = [stored[input_id][1] for input_id in input_ids]
571
+ df = df.drop(columns=[EPISODE_INPUT_COLUMN, "persona"], errors="ignore")
572
+ missing = sorted(
573
+ {key for row, _ in stored.values() for key in row} - set(df.columns) - {"usersim_config", "persona"}
574
+ )
575
+ for key in missing:
576
+ values = [stored[input_id][0].get(key) for input_id in input_ids]
577
+ if key in json_columns:
578
+ values = [None if value is None else json.dumps(value, ensure_ascii=False) for value in values]
579
+ df[key] = pd.Series(values, index=df.index, dtype=object)
580
+ return df
581
+
582
+
583
+ def simulate_inputs_to_dataset(
584
+ *,
585
+ inputs: EpisodeInputs,
586
+ models: Any,
587
+ out: str | Path,
588
+ models_path: str | Path | None = None,
589
+ skip_existing: bool = True,
590
+ plan: InputsPlan | None = None,
591
+ ) -> InputsRun:
592
+ """Run stored rows with ``models`` and save them as one run under ``out``.
593
+
594
+ Each locale's rows run as one Data Designer batch, through the simulator
595
+ column ``usersim simulate`` uses. With ``skip_existing`` (the default), the
596
+ rows go into the latest run with this setup, skipping rows already saved
597
+ there; otherwise they start a new run. ``plan`` is
598
+ :func:`plan_inputs_run`'s result for the same arguments, when the caller
599
+ already has it.
600
+
601
+ Raises ``ValueError`` when the run already holds a stored row with
602
+ different contents: before any model call when it is found up front, and
603
+ when the rows are saved when another writer saved them meanwhile, where
604
+ the write lock is shared. Where ``out`` is on a filesystem without file
605
+ locks, the rows are written without a lock shared between processes,
606
+ with a warning. Two calls at once with one setup can then each start a
607
+ run, and resuming one run from two places at once can save a row twice,
608
+ or both versions of a changed row.
609
+ """
610
+ from usersim.cli._pipeline import build_inputs_config_builder
611
+ from usersim.engine.core.manifest import write_simulator_manifest
612
+ from usersim.engine.core.storage import (
613
+ publish_new_run,
614
+ run_started_at,
615
+ run_write_lock,
616
+ write_locale_partition,
617
+ )
618
+
619
+ started_at = time.time()
620
+ out = Path(out).resolve()
621
+ out.mkdir(parents=True, exist_ok=True)
622
+ # A first look at the matching run, to skip saved rows before any model
623
+ # call and refuse changed ones early. Only the check under the lock counts.
624
+ plan = plan or plan_inputs_run(inputs, models, out, skip_existing=skip_existing)
625
+ fingerprint, matching, pending, skipped = plan.fingerprint, plan.matching_run, plan.pending, plan.saved
626
+ record = {**plan.record, "started_at": started_at}
627
+ stored = {row["trajectory_id"]: (row, replay_trajectory_id(row, fingerprint)) for row in inputs.rows}
628
+ logger.info("setup %s: %d stored rows, %d already saved", fingerprint[:12], len(stored), skipped)
629
+
630
+ by_locale: dict[str, dict[str, tuple[dict[str, Any], str]]] = {}
631
+ for input_id in sorted(pending):
632
+ row, new_id = stored[input_id]
633
+ by_locale.setdefault(row["usersim_config"]["locale"], {})[input_id] = (row, new_id)
634
+
635
+ # The rows go into the run the plan found, even if a newer run with this
636
+ # setup appears meanwhile, so they never split across two runs.
637
+ run_id: str | None = matching
638
+ resumed = matching is not None
639
+ json_columns = _json_columns(inputs.rows)
640
+ written = 0
641
+ requested: Counter[str] = Counter()
642
+ for locale, group in by_locale.items():
643
+ requested[locale] += len(group)
644
+ data_designer, builder = build_inputs_config_builder(
645
+ models=models, config=_column_config(inputs, locale, models), rows=[row for row, _ in group.values()]
646
+ )
647
+ # Data Designer names a batch's artifacts by the second it started, so
648
+ # two invocations in one directory could share them; each batch gets
649
+ # its own folder, removed once its rows are loaded.
650
+ with tempfile.TemporaryDirectory(prefix="usersim_inputs_") as artifacts:
651
+ result = data_designer.create(builder, num_records=len(group), artifact_path=artifacts)
652
+ df = _as_saved_rows(result.load_dataset(), group, json_columns)
653
+ with run_write_lock(out):
654
+ target = run_id or (_matching_run(out, fingerprint) if skip_existing else None)
655
+ if target is not None:
656
+ planned = list(zip(df[INPUT_ID_COLUMN].astype(str), df["trajectory_id"].astype(str)))
657
+ todo, done = _unsaved(planned, _saved_inputs(out, target), target)
658
+ df = df[df[INPUT_ID_COLUMN].astype(str).isin(todo)]
659
+ skipped += done
660
+ # This invocation keeps the run it found, even if every row
661
+ # was saved there meanwhile.
662
+ resumed = resumed or run_id is None
663
+ run_id = target
664
+ if df.empty:
665
+ continue
666
+ if target is None:
667
+ run_id = publish_new_run(df, out, locale=locale, record=record)
668
+ else:
669
+ write_locale_partition(df, out, locale=locale, run_id=target)
670
+ written += len(df)
671
+ logger.info("locale=%s: wrote %d trajectories from stored rows into run=%s", locale, len(df), run_id)
672
+
673
+ if written and run_id is not None:
674
+ probes = Counter(str(row.get("probe_type")) for row, _ in stored.values())
675
+ try:
676
+ write_simulator_manifest(
677
+ run_id=run_id,
678
+ started_at_epoch=run_started_at(out, run_id) or int(started_at),
679
+ trajectory_root=out,
680
+ models_config=models,
681
+ models_config_path=Path(models_path) if models_path else None,
682
+ sim_config=_column_config(inputs, next(iter(requested)), models),
683
+ locales_requested=sorted(requested),
684
+ probe_mix={probe: count / len(stored) for probe, count in sorted(probes.items())},
685
+ panel_path=inputs.source,
686
+ assets_dir=inputs.assets_dir,
687
+ trajectories_requested=dict(requested),
688
+ )
689
+ except Exception as error: # pragma: no cover - never fail a finished run on manifest issues
690
+ logger.warning("failed to write run manifest: %s", error)
691
+ return InputsRun(run_id=run_id, resumed=resumed, written=written, skipped=skipped)