usersim 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- usersim/__init__.py +23 -0
- usersim/asset_gen/__init__.py +22 -0
- usersim/asset_gen/financial_services/__init__.py +22 -0
- usersim/asset_gen/financial_services/evaluate.py +330 -0
- usersim/asset_gen/financial_services/pipeline.py +1590 -0
- usersim/asset_gen/financial_services/prompts.py +318 -0
- usersim/asset_gen/financial_services/spec.py +794 -0
- usersim/asset_gen/financial_services/validate.py +750 -0
- usersim/asset_gen/registry.py +191 -0
- usersim/cli/__init__.py +185 -0
- usersim/cli/__main__.py +13 -0
- usersim/cli/_async.py +35 -0
- usersim/cli/_env.py +74 -0
- usersim/cli/_errors.py +44 -0
- usersim/cli/_inputs.py +691 -0
- usersim/cli/_models.py +1222 -0
- usersim/cli/_ngc.py +441 -0
- usersim/cli/_persona_language.py +316 -0
- usersim/cli/_pipeline.py +1007 -0
- usersim/cli/evaluate.py +415 -0
- usersim/cli/evaluate_assets.py +115 -0
- usersim/cli/gen_assets.py +120 -0
- usersim/cli/model_catalog.py +451 -0
- usersim/cli/models_default.toml +46 -0
- usersim/cli/models_openai.toml +26 -0
- usersim/cli/models_openrouter.toml +33 -0
- usersim/cli/panel.py +193 -0
- usersim/cli/report.py +247 -0
- usersim/cli/rescore.py +223 -0
- usersim/cli/select.py +304 -0
- usersim/cli/setup_ngc.py +131 -0
- usersim/cli/simulate.py +866 -0
- usersim/cli/smoke.py +1019 -0
- usersim/cli/validate_assets.py +64 -0
- usersim/engine/__init__.py +4 -0
- usersim/engine/assets/financial_services/SCHEMA.md +320 -0
- usersim/engine/assets/financial_services/de_DE/_generation_report.json +945 -0
- usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/corpus.yaml +8006 -0
- usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/institution_meta.yaml +9 -0
- usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/tasks.yaml +366 -0
- usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/tools.yaml +114 -0
- usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/corpus.yaml +10053 -0
- usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/tasks.yaml +578 -0
- usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/tools.yaml +154 -0
- usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/corpus.yaml +6784 -0
- usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/tasks.yaml +394 -0
- usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/tools.yaml +134 -0
- usersim/engine/assets/financial_services/de_DE/dynamic.yaml +252 -0
- usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/corpus.yaml +9404 -0
- usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/tasks.yaml +394 -0
- usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/tools.yaml +114 -0
- usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/corpus.yaml +8181 -0
- usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/tasks.yaml +440 -0
- usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/tools.yaml +144 -0
- usersim/engine/assets/financial_services/de_DE/region_meta.yaml +108 -0
- usersim/engine/assets/financial_services/en_GB/_generation_report.json +1330 -0
- usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/corpus.yaml +11053 -0
- usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/institution_meta.yaml +9 -0
- usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/tasks.yaml +401 -0
- usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/tools.yaml +114 -0
- usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/corpus.yaml +13830 -0
- usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/institution_meta.yaml +11 -0
- usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/tasks.yaml +597 -0
- usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/tools.yaml +159 -0
- usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/corpus.yaml +9717 -0
- usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/tasks.yaml +418 -0
- usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/tools.yaml +139 -0
- usersim/engine/assets/financial_services/en_GB/dynamic.yaml +309 -0
- usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/corpus.yaml +7891 -0
- usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/tasks.yaml +345 -0
- usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/tools.yaml +114 -0
- usersim/engine/assets/financial_services/en_GB/neobank/quill/corpus.yaml +9579 -0
- usersim/engine/assets/financial_services/en_GB/neobank/quill/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/en_GB/neobank/quill/tasks.yaml +453 -0
- usersim/engine/assets/financial_services/en_GB/neobank/quill/tools.yaml +154 -0
- usersim/engine/assets/financial_services/en_GB/region_meta.yaml +128 -0
- usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/corpus.yaml +8028 -0
- usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/tasks.yaml +387 -0
- usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/tools.yaml +124 -0
- usersim/engine/assets/financial_services/en_IN/_generation_report.json +977 -0
- usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/corpus.yaml +9170 -0
- usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/tasks.yaml +330 -0
- usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/tools.yaml +139 -0
- usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/corpus.yaml +7436 -0
- usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/tasks.yaml +323 -0
- usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/tools.yaml +129 -0
- usersim/engine/assets/financial_services/en_IN/dynamic.yaml +256 -0
- usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/corpus.yaml +7641 -0
- usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/tasks.yaml +350 -0
- usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/tools.yaml +114 -0
- usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/corpus.yaml +9453 -0
- usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/institution_meta.yaml +9 -0
- usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/tasks.yaml +346 -0
- usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/tools.yaml +114 -0
- usersim/engine/assets/financial_services/en_IN/neobank/paysetu/corpus.yaml +8237 -0
- usersim/engine/assets/financial_services/en_IN/neobank/paysetu/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/en_IN/neobank/paysetu/tasks.yaml +313 -0
- usersim/engine/assets/financial_services/en_IN/neobank/paysetu/tools.yaml +139 -0
- usersim/engine/assets/financial_services/en_IN/region_meta.yaml +81 -0
- usersim/engine/assets/financial_services/en_US/_generation_report.json +768 -0
- usersim/engine/assets/financial_services/en_US/bank/northwind_bank/corpus.yaml +9131 -0
- usersim/engine/assets/financial_services/en_US/bank/northwind_bank/institution_meta.yaml +9 -0
- usersim/engine/assets/financial_services/en_US/bank/northwind_bank/tasks.yaml +473 -0
- usersim/engine/assets/financial_services/en_US/bank/northwind_bank/tools.yaml +129 -0
- usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/corpus.yaml +7868 -0
- usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/tasks.yaml +388 -0
- usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/tools.yaml +124 -0
- usersim/engine/assets/financial_services/en_US/dynamic.yaml +198 -0
- usersim/engine/assets/financial_services/en_US/region_meta.yaml +58 -0
- usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/corpus.yaml +8117 -0
- usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/tasks.yaml +369 -0
- usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/tools.yaml +109 -0
- usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/corpus.yaml +6897 -0
- usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/tasks.yaml +319 -0
- usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/tools.yaml +109 -0
- usersim/engine/assets/financial_services/fr_FR/_generation_report.json +1041 -0
- usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/corpus.yaml +10842 -0
- usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/tasks.yaml +529 -0
- usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/tools.yaml +144 -0
- usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/corpus.yaml +9091 -0
- usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/tasks.yaml +511 -0
- usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/tools.yaml +129 -0
- usersim/engine/assets/financial_services/fr_FR/dynamic.yaml +264 -0
- usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/corpus.yaml +9033 -0
- usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/tasks.yaml +505 -0
- usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/tools.yaml +134 -0
- usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/corpus.yaml +7494 -0
- usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/institution_meta.yaml +9 -0
- usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/tasks.yaml +383 -0
- usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/tools.yaml +114 -0
- usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/corpus.yaml +7883 -0
- usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/tasks.yaml +430 -0
- usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/tools.yaml +139 -0
- usersim/engine/assets/financial_services/fr_FR/region_meta.yaml +114 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/_generation_report.json +977 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/corpus.yaml +6265 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/institution_meta.yaml +11 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/tools.yaml +139 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/corpus.yaml +5188 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/institution_meta.yaml +9 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/tools.yaml +129 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/corpus.yaml +5115 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/institution_meta.yaml +9 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/tools.yaml +114 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/corpus.yaml +6539 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/tools.yaml +114 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/neobank/paysetu/corpus.yaml +5370 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/neobank/paysetu/institution_meta.yaml +11 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/neobank/paysetu/tools.yaml +139 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/region_meta.yaml +81 -0
- usersim/engine/assets/general_educational/prompts.yaml +115 -0
- usersim/engine/assets/general_educational/subjects/base/subjects.json +162 -0
- usersim/engine/assets/general_educational/subjects/de_DE/subjects.json +162 -0
- usersim/engine/assets/general_educational/subjects/en_GB/subjects.json +162 -0
- usersim/engine/assets/general_educational/subjects/fr_FR/subjects.json +16 -0
- usersim/engine/assets/general_educational/subjects/hi_Deva_IN/subjects.json +14 -0
- usersim/engine/assets/general_educational/subjects/hi_Latn_IN/subjects.json +14 -0
- usersim/engine/assets/general_educational/subjects/ja_JP/subjects.json +16 -0
- usersim/engine/assets/general_educational/subjects/ko_KR/subjects.json +16 -0
- usersim/engine/assets/general_educational/subjects/pt_BR/subjects.json +12 -0
- usersim/engine/assets/general_open_ended/prompts.yaml +117 -0
- usersim/engine/assets/general_open_ended/topics/base/topics.json +194 -0
- usersim/engine/assets/general_open_ended/topics/de_DE/topics.json +194 -0
- usersim/engine/assets/general_open_ended/topics/en_GB/topics.json +194 -0
- usersim/engine/assets/general_open_ended/topics/fr_FR/topics.json +22 -0
- usersim/engine/assets/general_open_ended/topics/hi_Deva_IN/topics.json +22 -0
- usersim/engine/assets/general_open_ended/topics/hi_Latn_IN/topics.json +22 -0
- usersim/engine/assets/general_open_ended/topics/ja_JP/topics.json +17 -0
- usersim/engine/assets/general_open_ended/topics/ko_KR/topics.json +17 -0
- usersim/engine/assets/general_open_ended/topics/pt_BR/topics.json +22 -0
- usersim/engine/assets/health_decision_support_disclosure/sample.yaml +21 -0
- usersim/engine/assets/health_general_disclosure/sample.yaml +21 -0
- usersim/engine/assets/health_therapy_disclosure/sample.yaml +23 -0
- usersim/engine/assets/health_triage_disclosure/sample.yaml +21 -0
- usersim/engine/assets/identity_disclosure/SCHEMA.md +303 -0
- usersim/engine/assets/identity_disclosure/spec.yaml +1007 -0
- usersim/engine/assets/safety_agentic/SCHEMA.md +252 -0
- usersim/engine/assets/safety_agentic/action_taxonomy.yaml +682 -0
- usersim/engine/assets/safety_agentic/sample.yaml +3993 -0
- usersim/engine/assets/safety_chat_pressure/SCHEMA.md +181 -0
- usersim/engine/assets/safety_chat_pressure/sample.yaml +3646 -0
- usersim/engine/assets/sov_ai_dynamic/SCHEMA.md +98 -0
- usersim/engine/assets/sov_ai_dynamic/de_DE/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/en_GB/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/en_IN/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/en_SG/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/en_US/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/fr_FR/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/hi_Deva_IN/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/hi_Latn_IN/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/ja_JP/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/ko_KR/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/pt_BR/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_facts/SCHEMA.md +196 -0
- usersim/engine/assets/sov_ai_facts/de_DE/sample.yaml +1421 -0
- usersim/engine/assets/sov_ai_facts/en_GB/sample.yaml +1389 -0
- usersim/engine/assets/sov_ai_facts/en_IN/sample.yaml +1275 -0
- usersim/engine/assets/sov_ai_facts/en_SG/sample.yaml +1358 -0
- usersim/engine/assets/sov_ai_facts/en_US/sample.yaml +1270 -0
- usersim/engine/assets/sov_ai_facts/fr_FR/sample.yaml +1287 -0
- usersim/engine/assets/sov_ai_facts/hi_Deva_IN/sample.yaml +1272 -0
- usersim/engine/assets/sov_ai_facts/hi_Latn_IN/sample.yaml +1272 -0
- usersim/engine/assets/sov_ai_facts/ja_JP/sample.yaml +1294 -0
- usersim/engine/assets/sov_ai_facts/ko_KR/sample.yaml +1435 -0
- usersim/engine/assets/sov_ai_facts/pt_BR/sample.yaml +1321 -0
- usersim/engine/assets/sov_ai_multilingual_parity/SCHEMA.md +137 -0
- usersim/engine/assets/sov_ai_multilingual_parity/sample.yaml +1210 -0
- usersim/engine/assets/tool_calling/README.md +225 -0
- usersim/engine/assets/tool_calling/prompts.yaml +171 -0
- usersim/engine/assets/tool_calling/toolsets_seed.parquet +0 -0
- usersim/engine/config.py +296 -0
- usersim/engine/core/__init__.py +4 -0
- usersim/engine/core/_assets.py +178 -0
- usersim/engine/core/_extensions.py +148 -0
- usersim/engine/core/action_taxonomy.py +347 -0
- usersim/engine/core/agentic_bank.py +725 -0
- usersim/engine/core/analysis.py +60 -0
- usersim/engine/core/backends.py +180 -0
- usersim/engine/core/behavioral.py +755 -0
- usersim/engine/core/clinical_profile_bank.py +245 -0
- usersim/engine/core/context.py +137 -0
- usersim/engine/core/embeddings.py +83 -0
- usersim/engine/core/episode_input.py +281 -0
- usersim/engine/core/episode_runtime.py +735 -0
- usersim/engine/core/fact_bank.py +500 -0
- usersim/engine/core/finance_bank.py +970 -0
- usersim/engine/core/finance_tasks.py +292 -0
- usersim/engine/core/idempotency.py +62 -0
- usersim/engine/core/identity.py +215 -0
- usersim/engine/core/identity_spec.py +659 -0
- usersim/engine/core/identity_spec_types.py +425 -0
- usersim/engine/core/judges.py +85 -0
- usersim/engine/core/language_detection.py +239 -0
- usersim/engine/core/llm.py +710 -0
- usersim/engine/core/locale.py +512 -0
- usersim/engine/core/localized.py +172 -0
- usersim/engine/core/manifest.py +998 -0
- usersim/engine/core/messages.py +93 -0
- usersim/engine/core/missing.py +38 -0
- usersim/engine/core/outcomes.py +442 -0
- usersim/engine/core/persona.py +178 -0
- usersim/engine/core/pressure_bank.py +629 -0
- usersim/engine/core/preview.py +1927 -0
- usersim/engine/core/probes.py +1499 -0
- usersim/engine/core/probing_taxonomy.py +462 -0
- usersim/engine/core/prompt_loader.py +93 -0
- usersim/engine/core/prompts.py +81 -0
- usersim/engine/core/provenance.py +118 -0
- usersim/engine/core/query_bank.py +567 -0
- usersim/engine/core/realized_audit.py +411 -0
- usersim/engine/core/row_metadata.py +35 -0
- usersim/engine/core/seeds.py +82 -0
- usersim/engine/core/simulation.py +1899 -0
- usersim/engine/core/storage.py +1407 -0
- usersim/engine/core/tool_calls.py +106 -0
- usersim/engine/core/translation.py +266 -0
- usersim/engine/core/user_turn_policy.py +80 -0
- usersim/engine/evaluator/__init__.py +19 -0
- usersim/engine/evaluator/axes.py +437 -0
- usersim/engine/evaluator/config.py +187 -0
- usersim/engine/evaluator/generator.py +424 -0
- usersim/engine/evaluator/judges.py +151 -0
- usersim/engine/evaluator/prompts.py +108 -0
- usersim/engine/evaluator/scorers/__init__.py +289 -0
- usersim/engine/evaluator/scorers/financial_services.py +635 -0
- usersim/engine/evaluator/scorers/health_disclosure.py +503 -0
- usersim/engine/evaluator/scorers/identity_disclosure.py +545 -0
- usersim/engine/evaluator/scorers/identity_grading.py +543 -0
- usersim/engine/evaluator/scorers/language_compliance.py +426 -0
- usersim/engine/evaluator/scorers/refusal_basics.py +479 -0
- usersim/engine/evaluator/scorers/response_shape.py +363 -0
- usersim/engine/evaluator/scorers/safety_agentic.py +1010 -0
- usersim/engine/evaluator/scorers/safety_chat_pressure.py +791 -0
- usersim/engine/evaluator/scorers/sov_ai_dynamic.py +629 -0
- usersim/engine/evaluator/scorers/sov_ai_facts.py +648 -0
- usersim/engine/evaluator/scorers/sov_ai_multilingual_parity.py +550 -0
- usersim/engine/evaluator/scorers/tool_use.py +291 -0
- usersim/engine/external.py +218 -0
- usersim/engine/generator.py +419 -0
- usersim/engine/plugin.py +34 -0
- usersim/engine/probes/__init__.py +28 -0
- usersim/engine/probes/financial_services/__init__.py +16 -0
- usersim/engine/probes/financial_services/generator.py +1329 -0
- usersim/engine/probes/financial_services/prompts.py +132 -0
- usersim/engine/probes/financial_services/retrieval.py +229 -0
- usersim/engine/probes/financial_services/task_derivation.py +274 -0
- usersim/engine/probes/financial_services/tools.py +116 -0
- usersim/engine/probes/general_educational/__init__.py +4 -0
- usersim/engine/probes/general_educational/generator.py +125 -0
- usersim/engine/probes/general_educational/prompts.py +207 -0
- usersim/engine/probes/general_open_ended/__init__.py +4 -0
- usersim/engine/probes/general_open_ended/generator.py +127 -0
- usersim/engine/probes/general_open_ended/prompts.py +209 -0
- usersim/engine/probes/health_disclosure/__init__.py +28 -0
- usersim/engine/probes/health_disclosure/base.py +148 -0
- usersim/engine/probes/health_disclosure/clients.py +291 -0
- usersim/engine/probes/health_disclosure/decision_support.py +44 -0
- usersim/engine/probes/health_disclosure/general.py +45 -0
- usersim/engine/probes/health_disclosure/mixin.py +363 -0
- usersim/engine/probes/health_disclosure/move_runtime.py +486 -0
- usersim/engine/probes/health_disclosure/moves.py +241 -0
- usersim/engine/probes/health_disclosure/prompts.py +324 -0
- usersim/engine/probes/health_disclosure/therapy.py +43 -0
- usersim/engine/probes/health_disclosure/triage.py +43 -0
- usersim/engine/probes/identity_disclosure/__init__.py +4 -0
- usersim/engine/probes/identity_disclosure/generator.py +504 -0
- usersim/engine/probes/identity_disclosure/prompts.py +219 -0
- usersim/engine/probes/identity_disclosure/task_derivation.py +227 -0
- usersim/engine/probes/safety_agentic/__init__.py +24 -0
- usersim/engine/probes/safety_agentic/generator.py +666 -0
- usersim/engine/probes/safety_agentic/prompts.py +24 -0
- usersim/engine/probes/safety_agentic/task_derivation.py +109 -0
- usersim/engine/probes/safety_chat_pressure/__init__.py +24 -0
- usersim/engine/probes/safety_chat_pressure/classifier.py +113 -0
- usersim/engine/probes/safety_chat_pressure/generator.py +551 -0
- usersim/engine/probes/safety_chat_pressure/prompts.py +141 -0
- usersim/engine/probes/safety_chat_pressure/task_derivation.py +171 -0
- usersim/engine/probes/sov_ai_dynamic/__init__.py +28 -0
- usersim/engine/probes/sov_ai_dynamic/generator.py +393 -0
- usersim/engine/probes/sov_ai_dynamic/prompts.py +472 -0
- usersim/engine/probes/sov_ai_dynamic/task_derivation.py +106 -0
- usersim/engine/probes/sov_ai_facts/__init__.py +15 -0
- usersim/engine/probes/sov_ai_facts/generator.py +342 -0
- usersim/engine/probes/sov_ai_facts/prompts.py +714 -0
- usersim/engine/probes/sov_ai_facts/task_derivation.py +479 -0
- usersim/engine/probes/sov_ai_multilingual_parity/__init__.py +19 -0
- usersim/engine/probes/sov_ai_multilingual_parity/generator.py +379 -0
- usersim/engine/probes/sov_ai_multilingual_parity/prompts.py +456 -0
- usersim/engine/probes/sov_ai_multilingual_parity/task_derivation.py +104 -0
- usersim/engine/probes/tool_calling/__init__.py +9 -0
- usersim/engine/probes/tool_calling/generator.py +437 -0
- usersim/engine/probes/tool_calling/prompts.py +221 -0
- usersim/engine/probes/tool_calling/verifiers.py +165 -0
- usersim/py.typed +0 -0
- usersim/reporting/__init__.py +116 -0
- usersim/reporting/_reasoning.py +201 -0
- usersim/reporting/capability_report.py +2049 -0
- usersim/reporting/comparison.py +797 -0
- usersim/reporting/comparison_dashboard.py +2938 -0
- usersim/reporting/dashboard.py +1731 -0
- usersim/reporting/diagnostics.py +435 -0
- usersim/reporting/eval_manifest.py +291 -0
- usersim/reporting/identity.py +82 -0
- usersim/reporting/resource.py +476 -0
- usersim/reporting/runs.py +282 -0
- usersim/reporting/sim_health.py +334 -0
- usersim/selection/__init__.py +87 -0
- usersim/selection/engine.py +423 -0
- usersim/selection/io.py +391 -0
- usersim/selection/profiles.py +249 -0
- usersim/selection/schemas.py +194 -0
- usersim/taxonomy/__init__.py +38 -0
- usersim/taxonomy/capabilities.py +1125 -0
- usersim/taxonomy/eval_cell.py +131 -0
- usersim/testing/__init__.py +534 -0
- usersim-0.2.0.dist-info/METADATA +392 -0
- usersim-0.2.0.dist-info/RECORD +379 -0
- usersim-0.2.0.dist-info/WHEEL +5 -0
- usersim-0.2.0.dist-info/entry_points.txt +6 -0
- usersim-0.2.0.dist-info/licenses/LICENSE +204 -0
- usersim-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Message and data formatting helpers for conversation simulation."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def format_tools_for_prompt(tools: list[dict[str, Any]]) -> str:
|
|
13
|
+
"""Format a list of tool definitions into a human-readable prompt block."""
|
|
14
|
+
lines = []
|
|
15
|
+
for tool_data in tools:
|
|
16
|
+
tool = tool_data.get("tool", {}) if "tool" in tool_data else tool_data
|
|
17
|
+
name = tool.get("function", {}).get("name", "Unknown")
|
|
18
|
+
description = tool.get("function", {}).get("description", "No description")
|
|
19
|
+
lines.append(f"- {name}: {description}")
|
|
20
|
+
return "\n".join(lines)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def format_themes_for_prompt(themes: list[dict[str, Any]]) -> str:
|
|
24
|
+
"""Format a list of theme dicts into a human-readable prompt block."""
|
|
25
|
+
lines = []
|
|
26
|
+
for theme in themes:
|
|
27
|
+
lines.append(f"- {theme.get('type', 'Unknown')}: {theme.get('description', 'No description')}")
|
|
28
|
+
return "\n".join(lines)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def format_conversation_history_for_prompt(messages: list[dict[str, Any]]) -> str:
|
|
32
|
+
"""Format conversation messages into a text block for judge prompts."""
|
|
33
|
+
lines = []
|
|
34
|
+
for msg in messages:
|
|
35
|
+
role = msg.get("role", "unknown")
|
|
36
|
+
content = msg.get("content", "")
|
|
37
|
+
tool_calls = msg.get("tool_calls")
|
|
38
|
+
|
|
39
|
+
if role == "system":
|
|
40
|
+
continue
|
|
41
|
+
elif role == "user":
|
|
42
|
+
lines.append(f"User: {content}")
|
|
43
|
+
elif role == "assistant":
|
|
44
|
+
if tool_calls:
|
|
45
|
+
tc_summary = []
|
|
46
|
+
for tc in tool_calls:
|
|
47
|
+
fn = tc.get("function", {})
|
|
48
|
+
tc_summary.append(f"{fn.get('name', '?')}({fn.get('arguments', '{}')})")
|
|
49
|
+
lines.append(f"Assistant [tool calls]: {'; '.join(tc_summary)}")
|
|
50
|
+
if content:
|
|
51
|
+
lines.append(f"Assistant: {content}")
|
|
52
|
+
elif role == "tool":
|
|
53
|
+
lines.append(f"Tool response: {content[:500]}")
|
|
54
|
+
|
|
55
|
+
return "\n".join(lines) if lines else "N/A"
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def project_public_dialogue(
|
|
59
|
+
messages: list[dict[str, Any]],
|
|
60
|
+
) -> list[dict[str, Any]]:
|
|
61
|
+
"""Return only text a real user could observe.
|
|
62
|
+
|
|
63
|
+
Assistant text remains public even when its source message also contains a
|
|
64
|
+
tool call. Protocol envelopes, arguments, and raw tool results do not.
|
|
65
|
+
"""
|
|
66
|
+
result: list[dict[str, Any]] = []
|
|
67
|
+
for message in messages:
|
|
68
|
+
role = message.get("role")
|
|
69
|
+
content = message.get("content", "")
|
|
70
|
+
if role == "user":
|
|
71
|
+
result.append({"role": "user", "content": content})
|
|
72
|
+
elif role == "assistant" and content:
|
|
73
|
+
result.append({"role": "assistant", "content": content})
|
|
74
|
+
return result
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _parse_theme(raw: Any) -> dict[str, Any]:
|
|
78
|
+
"""Normalise a theme value into a dict with 'type' and 'description' keys."""
|
|
79
|
+
if isinstance(raw, dict):
|
|
80
|
+
return raw
|
|
81
|
+
if isinstance(raw, str):
|
|
82
|
+
try:
|
|
83
|
+
parsed = json.loads(raw)
|
|
84
|
+
if isinstance(parsed, list):
|
|
85
|
+
return parsed[0] if parsed else {"type": "general", "description": "General conversation"}
|
|
86
|
+
if isinstance(parsed, dict):
|
|
87
|
+
return parsed
|
|
88
|
+
except (json.JSONDecodeError, TypeError):
|
|
89
|
+
pass
|
|
90
|
+
return {"type": raw, "description": raw}
|
|
91
|
+
if isinstance(raw, list):
|
|
92
|
+
return raw[0] if raw else {"type": "general", "description": "General conversation"}
|
|
93
|
+
return {"type": "general", "description": "General conversation"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Empty table cells, however pandas represents them.
|
|
5
|
+
|
|
6
|
+
pandas holds an empty cell as None, a float NaN or ``pd.NA``, depending on the
|
|
7
|
+
column's dtype, how the frame was built and the installed pandas version. Code
|
|
8
|
+
that falls back on a falsy value (``row.get(key) or default``) behaves the same
|
|
9
|
+
for all three only once they are None.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import math
|
|
15
|
+
import sys
|
|
16
|
+
from collections.abc import Mapping
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def is_missing(value: Any) -> bool:
|
|
21
|
+
"""True for an empty cell: None, a float NaN, ``pd.NA`` or ``pd.NaT``.
|
|
22
|
+
|
|
23
|
+
A container is never missing, whatever it holds, so a list or array cell is
|
|
24
|
+
never truth-tested element by element.
|
|
25
|
+
"""
|
|
26
|
+
if value is None:
|
|
27
|
+
return True
|
|
28
|
+
if isinstance(value, float):
|
|
29
|
+
return math.isnan(value)
|
|
30
|
+
# pandas' own markers (NA, NaT) exist only once pandas has been imported, and
|
|
31
|
+
# importing it here would make every importer of this module pay for pandas.
|
|
32
|
+
pandas = sys.modules.get("pandas")
|
|
33
|
+
return pandas is not None and pandas.api.types.is_scalar(value) and bool(pandas.isna(value))
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def none_for_missing(row: Mapping[str, Any]) -> dict[str, Any]:
|
|
37
|
+
"""Copy ``row`` with every empty cell replaced by None."""
|
|
38
|
+
return {key: None if is_missing(value) else value for key, value in row.items()}
|
|
@@ -0,0 +1,442 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Structured outcome records emitted by the simulator.
|
|
5
|
+
|
|
6
|
+
The simulator's contract with every downstream consumer (reporting,
|
|
7
|
+
evaluator, training-data extraction) rests on two artifacts:
|
|
8
|
+
|
|
9
|
+
- ``SimulationOutcome`` — one record per trajectory. The row-level
|
|
10
|
+
aggregate: status, failure attribution, counters, per-model tokens
|
|
11
|
+
and calls, wall-clock, provenance, warnings.
|
|
12
|
+
- ``SimulationTrace`` — zero-or-more records per trajectory, keyed
|
|
13
|
+
(``trajectory_id`` x ``turn_idx`` x ``call_idx``), discriminated by
|
|
14
|
+
``TraceKind``. Per-turn / per-call telemetry lives here: in-sim judge
|
|
15
|
+
traces, tool-call timings, API-response reroll details, etc. This is
|
|
16
|
+
the debugging surface, not the assistant-evaluation surface.
|
|
17
|
+
|
|
18
|
+
The principle: the simulator **emits** structured outcomes; the evaluator
|
|
19
|
+
**interprets** them. Anything that requires a model call to determine
|
|
20
|
+
its value is the simulator's responsibility (it cannot be reconstructed
|
|
21
|
+
later). Anything that is a derivation, aggregation, or judgment over
|
|
22
|
+
what's already captured belongs to the evaluator.
|
|
23
|
+
|
|
24
|
+
In-sim judge raw outputs (user-query gate ratings, assistant-quality
|
|
25
|
+
judge ratings, early-stop completion verdicts) are written to the
|
|
26
|
+
``simulation_traces`` sidecar with the appropriate ``TraceKind`` — never
|
|
27
|
+
to ``simulation_outcome`` and never to the evaluator's ``eval_store``.
|
|
28
|
+
|
|
29
|
+
The load-bearing decisions around this
|
|
30
|
+
separation.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import json
|
|
36
|
+
from dataclasses import dataclass, field
|
|
37
|
+
from enum import Enum
|
|
38
|
+
from typing import Any
|
|
39
|
+
|
|
40
|
+
# ---------------------------------------------------------------------------
|
|
41
|
+
# Enums
|
|
42
|
+
# ---------------------------------------------------------------------------
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class OutcomeStatus(str, Enum):
|
|
46
|
+
"""Row-level outcome status."""
|
|
47
|
+
|
|
48
|
+
OK = "ok"
|
|
49
|
+
COMPLETED_WITH_WARNINGS = "completed_with_warnings"
|
|
50
|
+
FAILED = "failed"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class FailureClass(str, Enum):
|
|
54
|
+
"""Structured classification of how a trajectory failed (if it did)."""
|
|
55
|
+
|
|
56
|
+
USER_QUERY_GATE_EXHAUSTED = "user_query_gate_exhausted"
|
|
57
|
+
USER_FOLLOWUP_GATE_EXHAUSTED = "user_followup_gate_exhausted"
|
|
58
|
+
USER_LANGUAGE_GATE_EXHAUSTED = "user_language_gate_exhausted"
|
|
59
|
+
API_RESPONSE_INVALID = "api_response_invalid"
|
|
60
|
+
TOOL_CALL_SCHEMA_ERROR = "tool_call_schema_error"
|
|
61
|
+
JUDGE_PARSE_ERROR = "judge_parse_error"
|
|
62
|
+
INFRASTRUCTURE_ERROR = "infrastructure_error"
|
|
63
|
+
SCENARIO_ABORTED = "scenario_aborted"
|
|
64
|
+
TURN_LIMIT_REACHED = "turn_limit_reached"
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class FailureAttribution(str, Enum):
|
|
68
|
+
"""Which actor caused the failure. Orthogonal to FailureClass."""
|
|
69
|
+
|
|
70
|
+
USER_MODEL = "user_model"
|
|
71
|
+
ASSISTANT_MODEL = "assistant_model"
|
|
72
|
+
API_RESPONSE_MODEL = "api_response_model"
|
|
73
|
+
JUDGE_MODEL = "judge_model"
|
|
74
|
+
SCENARIO_LOGIC = "scenario_logic"
|
|
75
|
+
INFRASTRUCTURE = "infrastructure"
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class TraceKind(str, Enum):
|
|
79
|
+
"""Discriminator for ``SimulationTrace`` entries.
|
|
80
|
+
|
|
81
|
+
These are *simulation-internal* events — they drive behavioral
|
|
82
|
+
primitives (frustration escalation, gate retries, flow control) or
|
|
83
|
+
capture per-call telemetry. They are NEVER read by the evaluator as
|
|
84
|
+
assistant-quality signal.
|
|
85
|
+
"""
|
|
86
|
+
|
|
87
|
+
USER_QUERY_GATE = "user_query_gate"
|
|
88
|
+
USER_FOLLOWUP_GATE = "user_followup_gate"
|
|
89
|
+
FOURTH_WALL_PREFILTER = "fourth_wall_prefilter"
|
|
90
|
+
# User turn regenerated because its script did not match the
|
|
91
|
+
# locale's expected script (e.g. romanized Hindi in an hi_Deva_IN
|
|
92
|
+
# run). Deterministic, no model call. See the script-compliance
|
|
93
|
+
# gate in ``core/simulation.py``.
|
|
94
|
+
LANGUAGE_PREFILTER = "language_prefilter"
|
|
95
|
+
ASSISTANT_QUALITY = "assistant_quality"
|
|
96
|
+
EARLY_STOP = "early_stop"
|
|
97
|
+
API_RESPONSE_REROLL = "api_response_reroll"
|
|
98
|
+
TOOL_CALL_VERIFIER = "tool_call_verifier"
|
|
99
|
+
USER_ROLE_VIOLATION = "user_role_violation"
|
|
100
|
+
SUMMARIZATION = "summarization"
|
|
101
|
+
# Per-assistant-turn capitulation classifier check (yes/no
|
|
102
|
+
# decision made by the probe's ``is_capitulation_detected`` hook).
|
|
103
|
+
# Emitted regardless of result so the trace log shows the
|
|
104
|
+
# classifier's signal at every turn for audit purposes.
|
|
105
|
+
CAPITULATION_CHECK = "capitulation_check"
|
|
106
|
+
# A retrieval query was machine-translated into the borrowed corpus's
|
|
107
|
+
# language before searching (India variants, where the conversation
|
|
108
|
+
# language and the asset language differ). Carries both strings so a
|
|
109
|
+
# retrieval miss can be attributed to the translator rather than the
|
|
110
|
+
# assistant — the query the assistant wrote is not the query that ranked.
|
|
111
|
+
KB_QUERY_TRANSLATION = "kb_query_translation"
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
class WarningKind(str, Enum):
|
|
115
|
+
"""Structured warning kinds for ``SimulationOutcome.warnings``."""
|
|
116
|
+
|
|
117
|
+
ID_FABRICATION = "id_fabrication"
|
|
118
|
+
COMPRESSION_FALLBACK = "compression_fallback"
|
|
119
|
+
POOR_ASSISTANT_RESPONSE = "poor_assistant_response"
|
|
120
|
+
# Flagged when a trajectory probed a fact that was marked
|
|
121
|
+
# ``placeholder: true`` in its bank. Like every ``USED_PLACEHOLDER_*``
|
|
122
|
+
# kind below, it is recorded on the outcome so a reader can tell which
|
|
123
|
+
# findings rest on placeholder content.
|
|
124
|
+
USED_PLACEHOLDER_FACT = "used_placeholder_fact"
|
|
125
|
+
# Flagged when a trajectory probed against a sov_ai_dynamic
|
|
126
|
+
# taxonomy that was marked ``placeholder: true`` at the file level.
|
|
127
|
+
# The flag is per taxonomy rather than per category, because the
|
|
128
|
+
# invitations and sub-topic hints share one reviewer audit.
|
|
129
|
+
USED_PLACEHOLDER_TAXONOMY = "used_placeholder_taxonomy"
|
|
130
|
+
# Flagged when a trajectory probed against a query that was marked
|
|
131
|
+
# ``placeholder: true`` in its query bank, for the cross-locale
|
|
132
|
+
# ``sov_ai_multilingual_parity`` probe. The flag is per query, because
|
|
133
|
+
# an entry's N locale renderings share one reviewer audit.
|
|
134
|
+
USED_PLACEHOLDER_QUERY = "used_placeholder_query"
|
|
135
|
+
# Flagged when a safety_chat_pressure trajectory probed against a
|
|
136
|
+
# target_request marked ``placeholder: true`` in its
|
|
137
|
+
# pressure-strategy bank.
|
|
138
|
+
USED_PLACEHOLDER_TARGET = "used_placeholder_target"
|
|
139
|
+
# Flagged when a safety_agentic trajectory probed against an
|
|
140
|
+
# action_request marked ``placeholder: true`` in its agentic-action
|
|
141
|
+
# bank.
|
|
142
|
+
USED_PLACEHOLDER_AGENTIC_ACTION = "used_placeholder_agentic_action"
|
|
143
|
+
# Flagged when a financial_services trajectory ran against a task
|
|
144
|
+
# template (verifiable tier), or an institution or taxonomy (dynamic
|
|
145
|
+
# tier), marked ``placeholder: true`` (no domain expert has verified it).
|
|
146
|
+
USED_PLACEHOLDER_FINANCE = "used_placeholder_finance"
|
|
147
|
+
# Flagged when a health_disclosure trajectory used a clinical profile
|
|
148
|
+
# marked ``placeholder: true`` in its clinical-profile bank (the bundled
|
|
149
|
+
# synthetic banks are all placeholders until a customer ships real ground
|
|
150
|
+
# truth).
|
|
151
|
+
USED_PLACEHOLDER_CLINICAL_PROFILE = "used_placeholder_clinical_profile"
|
|
152
|
+
# Flagged when an identity_disclosure trajectory drew on a spec, tactic
|
|
153
|
+
# or strategy marked ``placeholder: true`` (phrasings, vocabulary and
|
|
154
|
+
# expected identities no domain expert has verified).
|
|
155
|
+
USED_PLACEHOLDER_IDENTITY = "used_placeholder_identity"
|
|
156
|
+
# Flagged when a trajectory's verbatim turn-1 was machine-translated
|
|
157
|
+
# from an India base-locale (en_IN) bank into the conversation
|
|
158
|
+
# language, because that language has no native bank of its own;
|
|
159
|
+
# financial_services also flags it when kb_search queries are
|
|
160
|
+
# translated to search the base-locale corpus. Recorded on the outcome
|
|
161
|
+
# so a reader can tell which findings rest on machine-translated
|
|
162
|
+
# content. A locale with its own native asset never translates, so its
|
|
163
|
+
# rows never carry this warning.
|
|
164
|
+
USED_MACHINE_TRANSLATION = "used_machine_translation"
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
PLACEHOLDER_WARNING_KINDS: frozenset[str] = frozenset(
|
|
168
|
+
kind.value for kind in WarningKind if kind.name.startswith("USED_PLACEHOLDER_")
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
# ---------------------------------------------------------------------------
|
|
173
|
+
# Records
|
|
174
|
+
# ---------------------------------------------------------------------------
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
@dataclass
|
|
178
|
+
class Provenance:
|
|
179
|
+
"""Everything needed to audit what a trajectory was built against.
|
|
180
|
+
|
|
181
|
+
Recorded on every row, so rows built against different persona
|
|
182
|
+
dataset, prompt, code or bank versions can be told apart.
|
|
183
|
+
"""
|
|
184
|
+
|
|
185
|
+
nemotron_personas_version: str | None = None
|
|
186
|
+
scenario_prompt_version: str | None = None
|
|
187
|
+
code_sha: str | None = None
|
|
188
|
+
bank_version: dict[str, str] = field(default_factory=dict)
|
|
189
|
+
|
|
190
|
+
def to_dict(self) -> dict[str, Any]:
|
|
191
|
+
return {
|
|
192
|
+
"nemotron_personas_version": self.nemotron_personas_version,
|
|
193
|
+
"scenario_prompt_version": self.scenario_prompt_version,
|
|
194
|
+
"code_sha": self.code_sha,
|
|
195
|
+
"bank_version": dict(self.bank_version),
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
@dataclass
|
|
200
|
+
class SimulationWarning:
|
|
201
|
+
"""Structured warning entry. Stored in ``SimulationOutcome.warnings``."""
|
|
202
|
+
|
|
203
|
+
kind: WarningKind
|
|
204
|
+
turn_idx: int | None = None
|
|
205
|
+
detail: str = ""
|
|
206
|
+
|
|
207
|
+
def to_dict(self) -> dict[str, Any]:
|
|
208
|
+
return {
|
|
209
|
+
"kind": self.kind.value,
|
|
210
|
+
"turn_idx": self.turn_idx,
|
|
211
|
+
"detail": self.detail,
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
@dataclass
|
|
216
|
+
class SimulationOutcome:
|
|
217
|
+
"""The canonical per-trajectory row-level outcome record.
|
|
218
|
+
|
|
219
|
+
Everything a downstream consumer needs to answer "what happened with
|
|
220
|
+
this trajectory at an aggregate level?" — without re-parsing
|
|
221
|
+
conversation messages or invoking any model.
|
|
222
|
+
"""
|
|
223
|
+
|
|
224
|
+
status: OutcomeStatus = OutcomeStatus.OK
|
|
225
|
+
failure_class: FailureClass | None = None
|
|
226
|
+
failure_attribution: FailureAttribution | None = None
|
|
227
|
+
failure_detail: str = ""
|
|
228
|
+
|
|
229
|
+
n_turns: int = 0
|
|
230
|
+
n_tool_calls: int = 0
|
|
231
|
+
n_user_query_attempts: int = 0
|
|
232
|
+
n_user_followup_retries: int = 0
|
|
233
|
+
n_assistant_inline_failures: int = 0
|
|
234
|
+
# Judge-rejected assistant candidates that were discarded and
|
|
235
|
+
# regenerated (``max_assistant_attempts > 1``). Distinct from
|
|
236
|
+
# ``n_assistant_inline_failures``, which counts turns whose FINAL
|
|
237
|
+
# candidate is flagged in the delivered transcript.
|
|
238
|
+
n_assistant_retries: int = 0
|
|
239
|
+
n_api_response_rerolls: int = 0
|
|
240
|
+
n_fourth_wall_triggers: int = 0
|
|
241
|
+
n_user_role_violations: int = 0
|
|
242
|
+
n_user_language_violations: int = 0
|
|
243
|
+
|
|
244
|
+
warnings: list[SimulationWarning] = field(default_factory=list)
|
|
245
|
+
early_stop: bool = False
|
|
246
|
+
|
|
247
|
+
per_model_input_tokens: dict[str, int] = field(default_factory=dict)
|
|
248
|
+
per_model_output_tokens: dict[str, int] = field(default_factory=dict)
|
|
249
|
+
per_model_calls: dict[str, int] = field(default_factory=dict)
|
|
250
|
+
wall_clock_s_by_alias: dict[str, float] = field(default_factory=dict)
|
|
251
|
+
wall_clock_s: float = 0.0
|
|
252
|
+
|
|
253
|
+
provenance: Provenance = field(default_factory=Provenance)
|
|
254
|
+
|
|
255
|
+
def to_dict(self) -> dict[str, Any]:
|
|
256
|
+
return {
|
|
257
|
+
"status": self.status.value,
|
|
258
|
+
"failure_class": self.failure_class.value if self.failure_class else None,
|
|
259
|
+
"failure_attribution": (self.failure_attribution.value if self.failure_attribution else None),
|
|
260
|
+
"failure_detail": self.failure_detail,
|
|
261
|
+
"n_turns": self.n_turns,
|
|
262
|
+
"n_tool_calls": self.n_tool_calls,
|
|
263
|
+
"n_user_query_attempts": self.n_user_query_attempts,
|
|
264
|
+
"n_user_followup_retries": self.n_user_followup_retries,
|
|
265
|
+
"n_assistant_inline_failures": self.n_assistant_inline_failures,
|
|
266
|
+
"n_assistant_retries": self.n_assistant_retries,
|
|
267
|
+
"n_api_response_rerolls": self.n_api_response_rerolls,
|
|
268
|
+
"n_fourth_wall_triggers": self.n_fourth_wall_triggers,
|
|
269
|
+
"n_user_role_violations": self.n_user_role_violations,
|
|
270
|
+
"n_user_language_violations": self.n_user_language_violations,
|
|
271
|
+
"warnings": [w.to_dict() for w in self.warnings],
|
|
272
|
+
"early_stop": self.early_stop,
|
|
273
|
+
"per_model_input_tokens": dict(self.per_model_input_tokens),
|
|
274
|
+
"per_model_output_tokens": dict(self.per_model_output_tokens),
|
|
275
|
+
"per_model_calls": dict(self.per_model_calls),
|
|
276
|
+
"wall_clock_s_by_alias": dict(self.wall_clock_s_by_alias),
|
|
277
|
+
"wall_clock_s": round(self.wall_clock_s, 3),
|
|
278
|
+
"provenance": self.provenance.to_dict(),
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
def to_json(self) -> str:
|
|
282
|
+
return json.dumps(self.to_dict(), ensure_ascii=False, default=str)
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
@dataclass
|
|
286
|
+
class SimulationTrace:
|
|
287
|
+
"""A single per-turn / per-call telemetry entry.
|
|
288
|
+
|
|
289
|
+
Stored in the sidecar table keyed (``trajectory_id``, ``turn_idx``,
|
|
290
|
+
``call_idx``). ``kind`` discriminates which event it captures.
|
|
291
|
+
"""
|
|
292
|
+
|
|
293
|
+
kind: TraceKind
|
|
294
|
+
turn_idx: int
|
|
295
|
+
call_idx: int = 0
|
|
296
|
+
model_alias: str | None = None
|
|
297
|
+
input_tokens: int = 0
|
|
298
|
+
output_tokens: int = 0
|
|
299
|
+
latency_s: float = 0.0
|
|
300
|
+
prompt_version: str | None = None
|
|
301
|
+
rating: str | None = None # for judge kinds: "success" / "warn" / "failure"
|
|
302
|
+
detail: str | None = None # free-text: short explanation, error, etc.
|
|
303
|
+
extra: dict[str, Any] = field(default_factory=dict)
|
|
304
|
+
|
|
305
|
+
def to_dict(self) -> dict[str, Any]:
|
|
306
|
+
return {
|
|
307
|
+
"kind": self.kind.value,
|
|
308
|
+
"turn_idx": self.turn_idx,
|
|
309
|
+
"call_idx": self.call_idx,
|
|
310
|
+
"model_alias": self.model_alias,
|
|
311
|
+
"input_tokens": self.input_tokens,
|
|
312
|
+
"output_tokens": self.output_tokens,
|
|
313
|
+
"latency_s": round(self.latency_s, 3),
|
|
314
|
+
"prompt_version": self.prompt_version,
|
|
315
|
+
"rating": self.rating,
|
|
316
|
+
"detail": self.detail,
|
|
317
|
+
"extra": dict(self.extra),
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
# ---------------------------------------------------------------------------
|
|
322
|
+
# Builder — mutated during simulation; frozen into SimulationOutcome at end
|
|
323
|
+
# ---------------------------------------------------------------------------
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
class OutcomeBuilder:
|
|
327
|
+
"""Mutable accumulator for the sim's structured outcome record.
|
|
328
|
+
|
|
329
|
+
Owned by ``ConversationState``. Mutated by the simulator as it
|
|
330
|
+
progresses. ``finalize(status, ...)`` returns an immutable
|
|
331
|
+
``SimulationOutcome``; ``traces()`` returns the list of
|
|
332
|
+
``SimulationTrace`` entries for the sidecar.
|
|
333
|
+
"""
|
|
334
|
+
|
|
335
|
+
def __init__(self, provenance: Provenance | None = None) -> None:
|
|
336
|
+
self._outcome = SimulationOutcome(provenance=provenance or Provenance())
|
|
337
|
+
self._traces: list[SimulationTrace] = []
|
|
338
|
+
|
|
339
|
+
# ── counters ───────────────────────────────────────────────────
|
|
340
|
+
|
|
341
|
+
def inc_user_query_attempts(self) -> None:
|
|
342
|
+
self._outcome.n_user_query_attempts += 1
|
|
343
|
+
|
|
344
|
+
def inc_user_followup_retries(self, n: int = 1) -> None:
|
|
345
|
+
self._outcome.n_user_followup_retries += n
|
|
346
|
+
|
|
347
|
+
def inc_assistant_inline_failures(self) -> None:
|
|
348
|
+
self._outcome.n_assistant_inline_failures += 1
|
|
349
|
+
|
|
350
|
+
def inc_assistant_retries(self) -> None:
|
|
351
|
+
self._outcome.n_assistant_retries += 1
|
|
352
|
+
|
|
353
|
+
def inc_api_response_rerolls(self) -> None:
|
|
354
|
+
self._outcome.n_api_response_rerolls += 1
|
|
355
|
+
|
|
356
|
+
def inc_fourth_wall_triggers(self) -> None:
|
|
357
|
+
self._outcome.n_fourth_wall_triggers += 1
|
|
358
|
+
|
|
359
|
+
def inc_user_role_violations(self) -> None:
|
|
360
|
+
self._outcome.n_user_role_violations += 1
|
|
361
|
+
|
|
362
|
+
def inc_user_language_violations(self) -> None:
|
|
363
|
+
self._outcome.n_user_language_violations += 1
|
|
364
|
+
|
|
365
|
+
# ── warnings ──────────────────────────────────────────────────
|
|
366
|
+
|
|
367
|
+
def add_warning(
|
|
368
|
+
self,
|
|
369
|
+
kind: WarningKind,
|
|
370
|
+
turn_idx: int | None = None,
|
|
371
|
+
detail: str = "",
|
|
372
|
+
) -> None:
|
|
373
|
+
self._outcome.warnings.append(SimulationWarning(kind=kind, turn_idx=turn_idx, detail=detail))
|
|
374
|
+
|
|
375
|
+
# ── traces ────────────────────────────────────────────────────
|
|
376
|
+
|
|
377
|
+
def add_trace(self, trace: SimulationTrace) -> None:
|
|
378
|
+
self._traces.append(trace)
|
|
379
|
+
|
|
380
|
+
def traces(self) -> list[SimulationTrace]:
|
|
381
|
+
return list(self._traces)
|
|
382
|
+
|
|
383
|
+
# ── per-model aggregation (called from within the LLM layer) ──
|
|
384
|
+
|
|
385
|
+
def record_call(
|
|
386
|
+
self,
|
|
387
|
+
alias: str,
|
|
388
|
+
input_tokens: int,
|
|
389
|
+
output_tokens: int,
|
|
390
|
+
elapsed_s: float,
|
|
391
|
+
) -> None:
|
|
392
|
+
self._outcome.per_model_calls[alias] = self._outcome.per_model_calls.get(alias, 0) + 1
|
|
393
|
+
self._outcome.per_model_input_tokens[alias] = self._outcome.per_model_input_tokens.get(alias, 0) + input_tokens
|
|
394
|
+
self._outcome.per_model_output_tokens[alias] = (
|
|
395
|
+
self._outcome.per_model_output_tokens.get(alias, 0) + output_tokens
|
|
396
|
+
)
|
|
397
|
+
self._outcome.wall_clock_s_by_alias[alias] = self._outcome.wall_clock_s_by_alias.get(alias, 0.0) + elapsed_s
|
|
398
|
+
|
|
399
|
+
# ── finalization ──────────────────────────────────────────────
|
|
400
|
+
|
|
401
|
+
def set_early_stop(self, value: bool = True) -> None:
|
|
402
|
+
self._outcome.early_stop = value
|
|
403
|
+
|
|
404
|
+
def set_n_turns(self, n: int) -> None:
|
|
405
|
+
self._outcome.n_turns = n
|
|
406
|
+
|
|
407
|
+
def set_n_tool_calls(self, n: int) -> None:
|
|
408
|
+
self._outcome.n_tool_calls = n
|
|
409
|
+
|
|
410
|
+
def set_wall_clock_s(self, s: float) -> None:
|
|
411
|
+
self._outcome.wall_clock_s = s
|
|
412
|
+
|
|
413
|
+
def finalize(
|
|
414
|
+
self,
|
|
415
|
+
status: OutcomeStatus,
|
|
416
|
+
failure_class: FailureClass | None = None,
|
|
417
|
+
failure_attribution: FailureAttribution | None = None,
|
|
418
|
+
failure_detail: str = "",
|
|
419
|
+
) -> SimulationOutcome:
|
|
420
|
+
self._outcome.status = status
|
|
421
|
+
self._outcome.failure_class = failure_class
|
|
422
|
+
self._outcome.failure_attribution = failure_attribution
|
|
423
|
+
self._outcome.failure_detail = failure_detail
|
|
424
|
+
# Promote to COMPLETED_WITH_WARNINGS if we finished OK but
|
|
425
|
+
# accumulated warnings (identifier fabrication, etc).
|
|
426
|
+
if status == OutcomeStatus.OK and (
|
|
427
|
+
self._outcome.warnings
|
|
428
|
+
or self._outcome.n_user_role_violations > 0
|
|
429
|
+
or self._outcome.n_user_language_violations > 0
|
|
430
|
+
):
|
|
431
|
+
self._outcome.status = OutcomeStatus.COMPLETED_WITH_WARNINGS
|
|
432
|
+
return self._outcome
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
# ---------------------------------------------------------------------------
|
|
436
|
+
# Convenience
|
|
437
|
+
# ---------------------------------------------------------------------------
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def serialize_traces(traces: list[SimulationTrace]) -> str:
|
|
441
|
+
"""JSON-encode a list of traces for storage as a string column."""
|
|
442
|
+
return json.dumps([t.to_dict() for t in traces], ensure_ascii=False, default=str)
|