usersim 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- usersim/__init__.py +23 -0
- usersim/asset_gen/__init__.py +22 -0
- usersim/asset_gen/financial_services/__init__.py +22 -0
- usersim/asset_gen/financial_services/evaluate.py +330 -0
- usersim/asset_gen/financial_services/pipeline.py +1590 -0
- usersim/asset_gen/financial_services/prompts.py +318 -0
- usersim/asset_gen/financial_services/spec.py +794 -0
- usersim/asset_gen/financial_services/validate.py +750 -0
- usersim/asset_gen/registry.py +191 -0
- usersim/cli/__init__.py +185 -0
- usersim/cli/__main__.py +13 -0
- usersim/cli/_async.py +35 -0
- usersim/cli/_env.py +74 -0
- usersim/cli/_errors.py +44 -0
- usersim/cli/_inputs.py +691 -0
- usersim/cli/_models.py +1222 -0
- usersim/cli/_ngc.py +441 -0
- usersim/cli/_persona_language.py +316 -0
- usersim/cli/_pipeline.py +1007 -0
- usersim/cli/evaluate.py +415 -0
- usersim/cli/evaluate_assets.py +115 -0
- usersim/cli/gen_assets.py +120 -0
- usersim/cli/model_catalog.py +451 -0
- usersim/cli/models_default.toml +46 -0
- usersim/cli/models_openai.toml +26 -0
- usersim/cli/models_openrouter.toml +33 -0
- usersim/cli/panel.py +193 -0
- usersim/cli/report.py +247 -0
- usersim/cli/rescore.py +223 -0
- usersim/cli/select.py +304 -0
- usersim/cli/setup_ngc.py +131 -0
- usersim/cli/simulate.py +866 -0
- usersim/cli/smoke.py +1019 -0
- usersim/cli/validate_assets.py +64 -0
- usersim/engine/__init__.py +4 -0
- usersim/engine/assets/financial_services/SCHEMA.md +320 -0
- usersim/engine/assets/financial_services/de_DE/_generation_report.json +945 -0
- usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/corpus.yaml +8006 -0
- usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/institution_meta.yaml +9 -0
- usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/tasks.yaml +366 -0
- usersim/engine/assets/financial_services/de_DE/bank/bausparkasse_heimstatt/tools.yaml +114 -0
- usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/corpus.yaml +10053 -0
- usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/tasks.yaml +578 -0
- usersim/engine/assets/financial_services/de_DE/bank/sparkasse_lindenhain/tools.yaml +154 -0
- usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/corpus.yaml +6784 -0
- usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/tasks.yaml +394 -0
- usersim/engine/assets/financial_services/de_DE/brokerage/kranich_invest/tools.yaml +134 -0
- usersim/engine/assets/financial_services/de_DE/dynamic.yaml +252 -0
- usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/corpus.yaml +9404 -0
- usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/tasks.yaml +394 -0
- usersim/engine/assets/financial_services/de_DE/insurer/hanseatica_versicherung/tools.yaml +114 -0
- usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/corpus.yaml +8181 -0
- usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/tasks.yaml +440 -0
- usersim/engine/assets/financial_services/de_DE/neobank/kompass_direktbank/tools.yaml +144 -0
- usersim/engine/assets/financial_services/de_DE/region_meta.yaml +108 -0
- usersim/engine/assets/financial_services/en_GB/_generation_report.json +1330 -0
- usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/corpus.yaml +11053 -0
- usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/institution_meta.yaml +9 -0
- usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/tasks.yaml +401 -0
- usersim/engine/assets/financial_services/en_GB/bank/fernbridge_building_society/tools.yaml +114 -0
- usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/corpus.yaml +13830 -0
- usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/institution_meta.yaml +11 -0
- usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/tasks.yaml +597 -0
- usersim/engine/assets/financial_services/en_GB/bank/wrenfield_bank/tools.yaml +159 -0
- usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/corpus.yaml +9717 -0
- usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/tasks.yaml +418 -0
- usersim/engine/assets/financial_services/en_GB/brokerage/copperleaf_invest/tools.yaml +139 -0
- usersim/engine/assets/financial_services/en_GB/dynamic.yaml +309 -0
- usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/corpus.yaml +7891 -0
- usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/tasks.yaml +345 -0
- usersim/engine/assets/financial_services/en_GB/insurer/kingsmere_insurance/tools.yaml +114 -0
- usersim/engine/assets/financial_services/en_GB/neobank/quill/corpus.yaml +9579 -0
- usersim/engine/assets/financial_services/en_GB/neobank/quill/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/en_GB/neobank/quill/tasks.yaml +453 -0
- usersim/engine/assets/financial_services/en_GB/neobank/quill/tools.yaml +154 -0
- usersim/engine/assets/financial_services/en_GB/region_meta.yaml +128 -0
- usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/corpus.yaml +8028 -0
- usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/tasks.yaml +387 -0
- usersim/engine/assets/financial_services/en_GB/retirement_provider/oakhaven_pensions/tools.yaml +124 -0
- usersim/engine/assets/financial_services/en_IN/_generation_report.json +977 -0
- usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/corpus.yaml +9170 -0
- usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/tasks.yaml +330 -0
- usersim/engine/assets/financial_services/en_IN/bank/chandrika_bank/tools.yaml +139 -0
- usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/corpus.yaml +7436 -0
- usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/tasks.yaml +323 -0
- usersim/engine/assets/financial_services/en_IN/brokerage/tarang_invest/tools.yaml +129 -0
- usersim/engine/assets/financial_services/en_IN/dynamic.yaml +256 -0
- usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/corpus.yaml +7641 -0
- usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/tasks.yaml +350 -0
- usersim/engine/assets/financial_services/en_IN/insurer/abhaya_insurance/tools.yaml +114 -0
- usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/corpus.yaml +9453 -0
- usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/institution_meta.yaml +9 -0
- usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/tasks.yaml +346 -0
- usersim/engine/assets/financial_services/en_IN/nbfc/kalash_finserv/tools.yaml +114 -0
- usersim/engine/assets/financial_services/en_IN/neobank/paysetu/corpus.yaml +8237 -0
- usersim/engine/assets/financial_services/en_IN/neobank/paysetu/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/en_IN/neobank/paysetu/tasks.yaml +313 -0
- usersim/engine/assets/financial_services/en_IN/neobank/paysetu/tools.yaml +139 -0
- usersim/engine/assets/financial_services/en_IN/region_meta.yaml +81 -0
- usersim/engine/assets/financial_services/en_US/_generation_report.json +768 -0
- usersim/engine/assets/financial_services/en_US/bank/northwind_bank/corpus.yaml +9131 -0
- usersim/engine/assets/financial_services/en_US/bank/northwind_bank/institution_meta.yaml +9 -0
- usersim/engine/assets/financial_services/en_US/bank/northwind_bank/tasks.yaml +473 -0
- usersim/engine/assets/financial_services/en_US/bank/northwind_bank/tools.yaml +129 -0
- usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/corpus.yaml +7868 -0
- usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/tasks.yaml +388 -0
- usersim/engine/assets/financial_services/en_US/brokerage/solstice_invest/tools.yaml +124 -0
- usersim/engine/assets/financial_services/en_US/dynamic.yaml +198 -0
- usersim/engine/assets/financial_services/en_US/region_meta.yaml +58 -0
- usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/corpus.yaml +8117 -0
- usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/tasks.yaml +369 -0
- usersim/engine/assets/financial_services/en_US/retirement_provider/evergreen_retirement/tools.yaml +109 -0
- usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/corpus.yaml +6897 -0
- usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/tasks.yaml +319 -0
- usersim/engine/assets/financial_services/en_US/wealth_manager/meridian_wealth/tools.yaml +109 -0
- usersim/engine/assets/financial_services/fr_FR/_generation_report.json +1041 -0
- usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/corpus.yaml +10842 -0
- usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/tasks.yaml +529 -0
- usersim/engine/assets/financial_services/fr_FR/bank/credit_mirabelle/tools.yaml +144 -0
- usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/corpus.yaml +9091 -0
- usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/tasks.yaml +511 -0
- usersim/engine/assets/financial_services/fr_FR/brokerage/valbreuil_bourse/tools.yaml +129 -0
- usersim/engine/assets/financial_services/fr_FR/dynamic.yaml +264 -0
- usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/corpus.yaml +9033 -0
- usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/institution_meta.yaml +8 -0
- usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/tasks.yaml +505 -0
- usersim/engine/assets/financial_services/fr_FR/insurer/egide_assurances/tools.yaml +134 -0
- usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/corpus.yaml +7494 -0
- usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/institution_meta.yaml +9 -0
- usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/tasks.yaml +383 -0
- usersim/engine/assets/financial_services/fr_FR/nbfc/ondine_credit/tools.yaml +114 -0
- usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/corpus.yaml +7883 -0
- usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/tasks.yaml +430 -0
- usersim/engine/assets/financial_services/fr_FR/neobank/kiosque/tools.yaml +139 -0
- usersim/engine/assets/financial_services/fr_FR/region_meta.yaml +114 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/_generation_report.json +977 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/corpus.yaml +6265 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/institution_meta.yaml +11 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/bank/chandrika_bank/tools.yaml +139 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/corpus.yaml +5188 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/institution_meta.yaml +9 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/brokerage/tarang_invest/tools.yaml +129 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/corpus.yaml +5115 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/institution_meta.yaml +9 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/insurer/abhaya_insurance/tools.yaml +114 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/corpus.yaml +6539 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/institution_meta.yaml +10 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/nbfc/kalash_finserv/tools.yaml +114 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/neobank/paysetu/corpus.yaml +5370 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/neobank/paysetu/institution_meta.yaml +11 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/neobank/paysetu/tools.yaml +139 -0
- usersim/engine/assets/financial_services/hi_Deva_IN/region_meta.yaml +81 -0
- usersim/engine/assets/general_educational/prompts.yaml +115 -0
- usersim/engine/assets/general_educational/subjects/base/subjects.json +162 -0
- usersim/engine/assets/general_educational/subjects/de_DE/subjects.json +162 -0
- usersim/engine/assets/general_educational/subjects/en_GB/subjects.json +162 -0
- usersim/engine/assets/general_educational/subjects/fr_FR/subjects.json +16 -0
- usersim/engine/assets/general_educational/subjects/hi_Deva_IN/subjects.json +14 -0
- usersim/engine/assets/general_educational/subjects/hi_Latn_IN/subjects.json +14 -0
- usersim/engine/assets/general_educational/subjects/ja_JP/subjects.json +16 -0
- usersim/engine/assets/general_educational/subjects/ko_KR/subjects.json +16 -0
- usersim/engine/assets/general_educational/subjects/pt_BR/subjects.json +12 -0
- usersim/engine/assets/general_open_ended/prompts.yaml +117 -0
- usersim/engine/assets/general_open_ended/topics/base/topics.json +194 -0
- usersim/engine/assets/general_open_ended/topics/de_DE/topics.json +194 -0
- usersim/engine/assets/general_open_ended/topics/en_GB/topics.json +194 -0
- usersim/engine/assets/general_open_ended/topics/fr_FR/topics.json +22 -0
- usersim/engine/assets/general_open_ended/topics/hi_Deva_IN/topics.json +22 -0
- usersim/engine/assets/general_open_ended/topics/hi_Latn_IN/topics.json +22 -0
- usersim/engine/assets/general_open_ended/topics/ja_JP/topics.json +17 -0
- usersim/engine/assets/general_open_ended/topics/ko_KR/topics.json +17 -0
- usersim/engine/assets/general_open_ended/topics/pt_BR/topics.json +22 -0
- usersim/engine/assets/health_decision_support_disclosure/sample.yaml +21 -0
- usersim/engine/assets/health_general_disclosure/sample.yaml +21 -0
- usersim/engine/assets/health_therapy_disclosure/sample.yaml +23 -0
- usersim/engine/assets/health_triage_disclosure/sample.yaml +21 -0
- usersim/engine/assets/identity_disclosure/SCHEMA.md +303 -0
- usersim/engine/assets/identity_disclosure/spec.yaml +1007 -0
- usersim/engine/assets/safety_agentic/SCHEMA.md +252 -0
- usersim/engine/assets/safety_agentic/action_taxonomy.yaml +682 -0
- usersim/engine/assets/safety_agentic/sample.yaml +3993 -0
- usersim/engine/assets/safety_chat_pressure/SCHEMA.md +181 -0
- usersim/engine/assets/safety_chat_pressure/sample.yaml +3646 -0
- usersim/engine/assets/sov_ai_dynamic/SCHEMA.md +98 -0
- usersim/engine/assets/sov_ai_dynamic/de_DE/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/en_GB/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/en_IN/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/en_SG/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/en_US/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/fr_FR/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/hi_Deva_IN/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/hi_Latn_IN/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/ja_JP/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/ko_KR/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_dynamic/pt_BR/categories.yaml +116 -0
- usersim/engine/assets/sov_ai_facts/SCHEMA.md +196 -0
- usersim/engine/assets/sov_ai_facts/de_DE/sample.yaml +1421 -0
- usersim/engine/assets/sov_ai_facts/en_GB/sample.yaml +1389 -0
- usersim/engine/assets/sov_ai_facts/en_IN/sample.yaml +1275 -0
- usersim/engine/assets/sov_ai_facts/en_SG/sample.yaml +1358 -0
- usersim/engine/assets/sov_ai_facts/en_US/sample.yaml +1270 -0
- usersim/engine/assets/sov_ai_facts/fr_FR/sample.yaml +1287 -0
- usersim/engine/assets/sov_ai_facts/hi_Deva_IN/sample.yaml +1272 -0
- usersim/engine/assets/sov_ai_facts/hi_Latn_IN/sample.yaml +1272 -0
- usersim/engine/assets/sov_ai_facts/ja_JP/sample.yaml +1294 -0
- usersim/engine/assets/sov_ai_facts/ko_KR/sample.yaml +1435 -0
- usersim/engine/assets/sov_ai_facts/pt_BR/sample.yaml +1321 -0
- usersim/engine/assets/sov_ai_multilingual_parity/SCHEMA.md +137 -0
- usersim/engine/assets/sov_ai_multilingual_parity/sample.yaml +1210 -0
- usersim/engine/assets/tool_calling/README.md +225 -0
- usersim/engine/assets/tool_calling/prompts.yaml +171 -0
- usersim/engine/assets/tool_calling/toolsets_seed.parquet +0 -0
- usersim/engine/config.py +296 -0
- usersim/engine/core/__init__.py +4 -0
- usersim/engine/core/_assets.py +178 -0
- usersim/engine/core/_extensions.py +148 -0
- usersim/engine/core/action_taxonomy.py +347 -0
- usersim/engine/core/agentic_bank.py +725 -0
- usersim/engine/core/analysis.py +60 -0
- usersim/engine/core/backends.py +180 -0
- usersim/engine/core/behavioral.py +755 -0
- usersim/engine/core/clinical_profile_bank.py +245 -0
- usersim/engine/core/context.py +137 -0
- usersim/engine/core/embeddings.py +83 -0
- usersim/engine/core/episode_input.py +281 -0
- usersim/engine/core/episode_runtime.py +735 -0
- usersim/engine/core/fact_bank.py +500 -0
- usersim/engine/core/finance_bank.py +970 -0
- usersim/engine/core/finance_tasks.py +292 -0
- usersim/engine/core/idempotency.py +62 -0
- usersim/engine/core/identity.py +215 -0
- usersim/engine/core/identity_spec.py +659 -0
- usersim/engine/core/identity_spec_types.py +425 -0
- usersim/engine/core/judges.py +85 -0
- usersim/engine/core/language_detection.py +239 -0
- usersim/engine/core/llm.py +710 -0
- usersim/engine/core/locale.py +512 -0
- usersim/engine/core/localized.py +172 -0
- usersim/engine/core/manifest.py +998 -0
- usersim/engine/core/messages.py +93 -0
- usersim/engine/core/missing.py +38 -0
- usersim/engine/core/outcomes.py +442 -0
- usersim/engine/core/persona.py +178 -0
- usersim/engine/core/pressure_bank.py +629 -0
- usersim/engine/core/preview.py +1927 -0
- usersim/engine/core/probes.py +1499 -0
- usersim/engine/core/probing_taxonomy.py +462 -0
- usersim/engine/core/prompt_loader.py +93 -0
- usersim/engine/core/prompts.py +81 -0
- usersim/engine/core/provenance.py +118 -0
- usersim/engine/core/query_bank.py +567 -0
- usersim/engine/core/realized_audit.py +411 -0
- usersim/engine/core/row_metadata.py +35 -0
- usersim/engine/core/seeds.py +82 -0
- usersim/engine/core/simulation.py +1899 -0
- usersim/engine/core/storage.py +1407 -0
- usersim/engine/core/tool_calls.py +106 -0
- usersim/engine/core/translation.py +266 -0
- usersim/engine/core/user_turn_policy.py +80 -0
- usersim/engine/evaluator/__init__.py +19 -0
- usersim/engine/evaluator/axes.py +437 -0
- usersim/engine/evaluator/config.py +187 -0
- usersim/engine/evaluator/generator.py +424 -0
- usersim/engine/evaluator/judges.py +151 -0
- usersim/engine/evaluator/prompts.py +108 -0
- usersim/engine/evaluator/scorers/__init__.py +289 -0
- usersim/engine/evaluator/scorers/financial_services.py +635 -0
- usersim/engine/evaluator/scorers/health_disclosure.py +503 -0
- usersim/engine/evaluator/scorers/identity_disclosure.py +545 -0
- usersim/engine/evaluator/scorers/identity_grading.py +543 -0
- usersim/engine/evaluator/scorers/language_compliance.py +426 -0
- usersim/engine/evaluator/scorers/refusal_basics.py +479 -0
- usersim/engine/evaluator/scorers/response_shape.py +363 -0
- usersim/engine/evaluator/scorers/safety_agentic.py +1010 -0
- usersim/engine/evaluator/scorers/safety_chat_pressure.py +791 -0
- usersim/engine/evaluator/scorers/sov_ai_dynamic.py +629 -0
- usersim/engine/evaluator/scorers/sov_ai_facts.py +648 -0
- usersim/engine/evaluator/scorers/sov_ai_multilingual_parity.py +550 -0
- usersim/engine/evaluator/scorers/tool_use.py +291 -0
- usersim/engine/external.py +218 -0
- usersim/engine/generator.py +419 -0
- usersim/engine/plugin.py +34 -0
- usersim/engine/probes/__init__.py +28 -0
- usersim/engine/probes/financial_services/__init__.py +16 -0
- usersim/engine/probes/financial_services/generator.py +1329 -0
- usersim/engine/probes/financial_services/prompts.py +132 -0
- usersim/engine/probes/financial_services/retrieval.py +229 -0
- usersim/engine/probes/financial_services/task_derivation.py +274 -0
- usersim/engine/probes/financial_services/tools.py +116 -0
- usersim/engine/probes/general_educational/__init__.py +4 -0
- usersim/engine/probes/general_educational/generator.py +125 -0
- usersim/engine/probes/general_educational/prompts.py +207 -0
- usersim/engine/probes/general_open_ended/__init__.py +4 -0
- usersim/engine/probes/general_open_ended/generator.py +127 -0
- usersim/engine/probes/general_open_ended/prompts.py +209 -0
- usersim/engine/probes/health_disclosure/__init__.py +28 -0
- usersim/engine/probes/health_disclosure/base.py +148 -0
- usersim/engine/probes/health_disclosure/clients.py +291 -0
- usersim/engine/probes/health_disclosure/decision_support.py +44 -0
- usersim/engine/probes/health_disclosure/general.py +45 -0
- usersim/engine/probes/health_disclosure/mixin.py +363 -0
- usersim/engine/probes/health_disclosure/move_runtime.py +486 -0
- usersim/engine/probes/health_disclosure/moves.py +241 -0
- usersim/engine/probes/health_disclosure/prompts.py +324 -0
- usersim/engine/probes/health_disclosure/therapy.py +43 -0
- usersim/engine/probes/health_disclosure/triage.py +43 -0
- usersim/engine/probes/identity_disclosure/__init__.py +4 -0
- usersim/engine/probes/identity_disclosure/generator.py +504 -0
- usersim/engine/probes/identity_disclosure/prompts.py +219 -0
- usersim/engine/probes/identity_disclosure/task_derivation.py +227 -0
- usersim/engine/probes/safety_agentic/__init__.py +24 -0
- usersim/engine/probes/safety_agentic/generator.py +666 -0
- usersim/engine/probes/safety_agentic/prompts.py +24 -0
- usersim/engine/probes/safety_agentic/task_derivation.py +109 -0
- usersim/engine/probes/safety_chat_pressure/__init__.py +24 -0
- usersim/engine/probes/safety_chat_pressure/classifier.py +113 -0
- usersim/engine/probes/safety_chat_pressure/generator.py +551 -0
- usersim/engine/probes/safety_chat_pressure/prompts.py +141 -0
- usersim/engine/probes/safety_chat_pressure/task_derivation.py +171 -0
- usersim/engine/probes/sov_ai_dynamic/__init__.py +28 -0
- usersim/engine/probes/sov_ai_dynamic/generator.py +393 -0
- usersim/engine/probes/sov_ai_dynamic/prompts.py +472 -0
- usersim/engine/probes/sov_ai_dynamic/task_derivation.py +106 -0
- usersim/engine/probes/sov_ai_facts/__init__.py +15 -0
- usersim/engine/probes/sov_ai_facts/generator.py +342 -0
- usersim/engine/probes/sov_ai_facts/prompts.py +714 -0
- usersim/engine/probes/sov_ai_facts/task_derivation.py +479 -0
- usersim/engine/probes/sov_ai_multilingual_parity/__init__.py +19 -0
- usersim/engine/probes/sov_ai_multilingual_parity/generator.py +379 -0
- usersim/engine/probes/sov_ai_multilingual_parity/prompts.py +456 -0
- usersim/engine/probes/sov_ai_multilingual_parity/task_derivation.py +104 -0
- usersim/engine/probes/tool_calling/__init__.py +9 -0
- usersim/engine/probes/tool_calling/generator.py +437 -0
- usersim/engine/probes/tool_calling/prompts.py +221 -0
- usersim/engine/probes/tool_calling/verifiers.py +165 -0
- usersim/py.typed +0 -0
- usersim/reporting/__init__.py +116 -0
- usersim/reporting/_reasoning.py +201 -0
- usersim/reporting/capability_report.py +2049 -0
- usersim/reporting/comparison.py +797 -0
- usersim/reporting/comparison_dashboard.py +2938 -0
- usersim/reporting/dashboard.py +1731 -0
- usersim/reporting/diagnostics.py +435 -0
- usersim/reporting/eval_manifest.py +291 -0
- usersim/reporting/identity.py +82 -0
- usersim/reporting/resource.py +476 -0
- usersim/reporting/runs.py +282 -0
- usersim/reporting/sim_health.py +334 -0
- usersim/selection/__init__.py +87 -0
- usersim/selection/engine.py +423 -0
- usersim/selection/io.py +391 -0
- usersim/selection/profiles.py +249 -0
- usersim/selection/schemas.py +194 -0
- usersim/taxonomy/__init__.py +38 -0
- usersim/taxonomy/capabilities.py +1125 -0
- usersim/taxonomy/eval_cell.py +131 -0
- usersim/testing/__init__.py +534 -0
- usersim-0.2.0.dist-info/METADATA +392 -0
- usersim-0.2.0.dist-info/RECORD +379 -0
- usersim-0.2.0.dist-info/WHEEL +5 -0
- usersim-0.2.0.dist-info/entry_points.txt +6 -0
- usersim-0.2.0.dist-info/licenses/LICENSE +204 -0
- usersim-0.2.0.dist-info/top_level.txt +1 -0
usersim/cli/_inputs.py
ADDED
|
@@ -0,0 +1,691 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Running stored episode rows: ``usersim simulate --inputs``.
|
|
5
|
+
|
|
6
|
+
Rows written by ``usersim simulate --materialize-inputs`` run exactly as they
|
|
7
|
+
were stored, so two setups can be compared on the same simulated users.
|
|
8
|
+
|
|
9
|
+
- **Identity.** Each run of a row gets its own ``trajectory_id``: a hash of the
|
|
10
|
+
stored row and of everything about the setup that can change the
|
|
11
|
+
conversation (see :func:`setup_fingerprint`). The stored row's own id is
|
|
12
|
+
saved as ``input_id``, the column that pairs two runs of the same rows.
|
|
13
|
+
- **The conversation.** It runs with the stored id as ``trajectory_id``, as
|
|
14
|
+
the hosted runtime does, so whatever a probe derives from its id (the tools
|
|
15
|
+
``tool_calling`` offers, say) is the same for every setup. The new id is
|
|
16
|
+
written only when the rows are saved.
|
|
17
|
+
- **Runs.** A run holds one setup. A resume reopens the latest run of stored
|
|
18
|
+
rows whose setup fingerprint matches; otherwise the first saved rows claim
|
|
19
|
+
a new run. Every save happens under the root's write lock, after checking
|
|
20
|
+
the run's saved rows again.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import hashlib
|
|
26
|
+
import json
|
|
27
|
+
import logging
|
|
28
|
+
import os
|
|
29
|
+
import sys
|
|
30
|
+
import tempfile
|
|
31
|
+
import time
|
|
32
|
+
from collections import Counter
|
|
33
|
+
from collections.abc import Mapping
|
|
34
|
+
from dataclasses import dataclass
|
|
35
|
+
from pathlib import Path
|
|
36
|
+
from typing import Any
|
|
37
|
+
|
|
38
|
+
logger = logging.getLogger("usersim.cli.simulate")
|
|
39
|
+
|
|
40
|
+
#: The saved row's copy of the stored row's ``trajectory_id``.
|
|
41
|
+
INPUT_ID_COLUMN = "input_id"
|
|
42
|
+
|
|
43
|
+
#: Names the id formula, so a later formula never collides with this one.
|
|
44
|
+
_ID_SCHEME = "usersim-inputs-v1"
|
|
45
|
+
|
|
46
|
+
#: ``USERSIM_*`` variables that cannot change a conversation: debug logging,
|
|
47
|
+
#: where the models file is (the models themselves are fingerprinted),
|
|
48
|
+
#: provenance labels, and the evaluator's audit settings. Every other
|
|
49
|
+
#: ``USERSIM_*`` variable is part of the setup, including ones added later.
|
|
50
|
+
_OPERATIONAL_ENV = frozenset(
|
|
51
|
+
{
|
|
52
|
+
"USERSIM_DEBUG_LOG",
|
|
53
|
+
"USERSIM_MODELS_CONFIG",
|
|
54
|
+
"USERSIM_CODE_SHA",
|
|
55
|
+
"USERSIM_NEMOTRON_PERSONAS_VERSION",
|
|
56
|
+
"USERSIM_AUDIT_MODEL",
|
|
57
|
+
"USERSIM_AUDIT_REASONING_EFFORT",
|
|
58
|
+
}
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
#: Settings that may differ between the rows of one invocation.
|
|
62
|
+
_PER_ROW_SETTINGS = ("locale", "assets_dir")
|
|
63
|
+
#: Settings the run's models file decides, whatever the rows were stored with:
|
|
64
|
+
#: replaying the assistant's reasoning belongs to the assistant under test.
|
|
65
|
+
_RUN_SETTINGS = ("replay_assistant_reasoning",)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass(frozen=True)
|
|
69
|
+
class EpisodeInputs:
|
|
70
|
+
"""Stored rows checked and ready to run."""
|
|
71
|
+
|
|
72
|
+
rows: list[dict[str, Any]]
|
|
73
|
+
#: The ``usersim_config`` every row shares, apart from ``_PER_ROW_SETTINGS``
|
|
74
|
+
#: and ``_RUN_SETTINGS``, which the run settles (see :func:`_run_config`).
|
|
75
|
+
config: dict[str, Any]
|
|
76
|
+
#: The asset root the rows run against.
|
|
77
|
+
assets_dir: Path
|
|
78
|
+
#: The file the rows came from, when they came from one.
|
|
79
|
+
source: Path | None = None
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@dataclass(frozen=True)
|
|
83
|
+
class InputsRun:
|
|
84
|
+
"""What one call to :func:`simulate_inputs_to_dataset` did."""
|
|
85
|
+
|
|
86
|
+
run_id: str | None
|
|
87
|
+
resumed: bool
|
|
88
|
+
written: int
|
|
89
|
+
skipped: int
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
# ── loading and checking ────────────────────────────────────────────────
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def load_episode_inputs(path: str | Path) -> list[dict[str, Any]]:
|
|
96
|
+
"""Read stored rows from a ``.jsonl`` or ``.parquet`` file, values as stored.
|
|
97
|
+
|
|
98
|
+
Parquet is read through pyarrow, not pandas, so no value is re-typed on
|
|
99
|
+
the way in.
|
|
100
|
+
"""
|
|
101
|
+
path = Path(path)
|
|
102
|
+
if path.suffix == ".jsonl":
|
|
103
|
+
with path.open(encoding="utf-8") as handle:
|
|
104
|
+
return [json.loads(line) for line in handle if line.strip()]
|
|
105
|
+
if path.suffix == ".parquet":
|
|
106
|
+
import pyarrow.parquet as pq
|
|
107
|
+
|
|
108
|
+
return pq.read_table(path).to_pylist()
|
|
109
|
+
raise ValueError(f"expected a .jsonl or .parquet file, got {path}")
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _plain(value: Any) -> Any:
|
|
113
|
+
"""Plain Python for a row built in memory, which may hold numpy values."""
|
|
114
|
+
if isinstance(value, Mapping):
|
|
115
|
+
return {key: _plain(item) for key, item in value.items()}
|
|
116
|
+
if isinstance(value, (list, tuple)):
|
|
117
|
+
return [_plain(item) for item in value]
|
|
118
|
+
if isinstance(value, str):
|
|
119
|
+
return str(value)
|
|
120
|
+
if hasattr(value, "tolist"):
|
|
121
|
+
return _plain(value.tolist())
|
|
122
|
+
return value
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def prepare_episode_inputs(
|
|
126
|
+
rows: list[Mapping[str, Any]],
|
|
127
|
+
*,
|
|
128
|
+
assets_dir: str | Path | None = None,
|
|
129
|
+
source: str | Path | None = None,
|
|
130
|
+
) -> EpisodeInputs:
|
|
131
|
+
"""Check stored rows before any model call, and settle what they run against.
|
|
132
|
+
|
|
133
|
+
Raises ``ValueError`` naming the row and the problem. The rows must share
|
|
134
|
+
one setup: their ``usersim_config`` may differ only in locale, asset
|
|
135
|
+
root and the settings the run's models file decides, so that one run
|
|
136
|
+
holds one setup.
|
|
137
|
+
"""
|
|
138
|
+
from pydantic import ValidationError
|
|
139
|
+
|
|
140
|
+
from usersim.engine.config import ConversationSimulatorConfig
|
|
141
|
+
from usersim.engine.core.episode_input import EPISODE_INPUT_COLUMN
|
|
142
|
+
from usersim.engine.core.probes import resolve_probe
|
|
143
|
+
|
|
144
|
+
if not rows:
|
|
145
|
+
raise ValueError("no rows to run")
|
|
146
|
+
checked: list[dict[str, Any]] = []
|
|
147
|
+
seen: set[str] = set()
|
|
148
|
+
for i, raw in enumerate(rows):
|
|
149
|
+
if not isinstance(raw, Mapping):
|
|
150
|
+
raise ValueError(f"row {i} is not a mapping")
|
|
151
|
+
row = _plain(dict(raw))
|
|
152
|
+
stored_id = row.get("trajectory_id")
|
|
153
|
+
if not isinstance(stored_id, str) or not stored_id:
|
|
154
|
+
raise ValueError(f"row {i} has no trajectory_id; each stored row needs its own")
|
|
155
|
+
where = f"row {i} ({stored_id})"
|
|
156
|
+
if stored_id in seen:
|
|
157
|
+
raise ValueError(f"{where}: trajectory_id appears in more than one row; each stored row needs its own")
|
|
158
|
+
seen.add(stored_id)
|
|
159
|
+
for reserved in (INPUT_ID_COLUMN, EPISODE_INPUT_COLUMN):
|
|
160
|
+
if reserved in row:
|
|
161
|
+
raise ValueError(f"{where} already has {reserved!r}; that is a run's output, not a stored row")
|
|
162
|
+
if not isinstance(row.get("usersim_config"), Mapping):
|
|
163
|
+
raise ValueError(f"{where} has no usersim_config; run rows written by --materialize-inputs")
|
|
164
|
+
try:
|
|
165
|
+
config = ConversationSimulatorConfig.model_validate(dict(row["usersim_config"]))
|
|
166
|
+
except ValidationError as error:
|
|
167
|
+
raise ValueError(f"{where}: usersim_config does not load: {error}") from None
|
|
168
|
+
if not row.get(config.persona_column):
|
|
169
|
+
raise ValueError(f"{where} has no {config.persona_column}")
|
|
170
|
+
try:
|
|
171
|
+
resolve_probe(str(row.get(config.probe_type_column)))
|
|
172
|
+
except KeyError as error:
|
|
173
|
+
raise ValueError(f"{where}: {error.args[0]}") from None
|
|
174
|
+
if "locale" in row and row["locale"] != config.locale:
|
|
175
|
+
raise ValueError(f"{where}: locale {row['locale']!r} differs from its config's {config.locale!r}")
|
|
176
|
+
try:
|
|
177
|
+
json.dumps(row, allow_nan=True)
|
|
178
|
+
except (TypeError, ValueError) as error:
|
|
179
|
+
raise ValueError(f"{where} holds a value JSON cannot represent: {error}") from None
|
|
180
|
+
checked.append(row)
|
|
181
|
+
return EpisodeInputs(
|
|
182
|
+
rows=checked,
|
|
183
|
+
config=_shared_config(checked),
|
|
184
|
+
assets_dir=_resolve_assets_dir(checked, assets_dir),
|
|
185
|
+
source=Path(source).resolve() if source is not None else None,
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _shared_config(rows: list[dict[str, Any]]) -> dict[str, Any]:
|
|
190
|
+
"""The setup every row shares; ``ValueError`` naming the settings that differ."""
|
|
191
|
+
|
|
192
|
+
def setup(row: dict[str, Any]) -> dict[str, Any]:
|
|
193
|
+
return {
|
|
194
|
+
key: value
|
|
195
|
+
for key, value in row["usersim_config"].items()
|
|
196
|
+
if key not in _PER_ROW_SETTINGS and key not in _RUN_SETTINGS
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
first = setup(rows[0])
|
|
200
|
+
for row in rows[1:]:
|
|
201
|
+
other = setup(row)
|
|
202
|
+
differing = sorted(key for key in first.keys() | other.keys() if first.get(key) != other.get(key))
|
|
203
|
+
if differing:
|
|
204
|
+
shown = ", ".join(f"{key}: {first.get(key)!r} vs {other.get(key)!r}" for key in differing)
|
|
205
|
+
raise ValueError(
|
|
206
|
+
f"the rows were stored with different settings ({shown}); one run holds one setup, "
|
|
207
|
+
"so run them separately"
|
|
208
|
+
)
|
|
209
|
+
return dict(rows[0]["usersim_config"])
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _resolve_assets_dir(rows: list[dict[str, Any]], given: str | Path | None) -> Path:
|
|
213
|
+
"""``--assets-dir`` when given, else the one asset root the rows were stored with."""
|
|
214
|
+
from usersim.engine.core._assets import packaged_assets_dir
|
|
215
|
+
|
|
216
|
+
if given is not None:
|
|
217
|
+
path = Path(given).resolve()
|
|
218
|
+
if not path.is_dir():
|
|
219
|
+
raise ValueError(f"--assets-dir {path} is not a directory")
|
|
220
|
+
return path
|
|
221
|
+
stored = {row["usersim_config"].get("assets_dir") for row in rows}
|
|
222
|
+
hint = f"pass --assets-dir (the assets that ship with User Sim are at {packaged_assets_dir()})"
|
|
223
|
+
if len(stored) != 1 or None in stored:
|
|
224
|
+
raise ValueError(f"the rows do not name one asset folder; {hint}")
|
|
225
|
+
path = Path(stored.pop())
|
|
226
|
+
if not path.is_dir():
|
|
227
|
+
raise ValueError(f"the asset folder the rows were stored with, {path}, does not exist here; {hint}")
|
|
228
|
+
return path
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
# ── identity ────────────────────────────────────────────────────────────
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _digest(value: Any) -> str:
|
|
235
|
+
"""SHA-256 of ``value``'s canonical JSON form."""
|
|
236
|
+
text = json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":"), default=str)
|
|
237
|
+
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _contents_digest(path: Path) -> str:
|
|
241
|
+
"""SHA-256 over a file's bytes, or over every file under a folder by relative path.
|
|
242
|
+
|
|
243
|
+
Hidden files and folders (``.DS_Store``, ``.ipynb_checkpoints``) are left
|
|
244
|
+
out: no loader reads them, and they change without the assets changing.
|
|
245
|
+
"""
|
|
246
|
+
if path.is_file():
|
|
247
|
+
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
248
|
+
entries = sorted(
|
|
249
|
+
item
|
|
250
|
+
for item in path.rglob("*")
|
|
251
|
+
if item.is_file() and not any(part.startswith(".") for part in item.relative_to(path).parts)
|
|
252
|
+
)
|
|
253
|
+
return _digest(
|
|
254
|
+
[[item.relative_to(path).as_posix(), hashlib.sha256(item.read_bytes()).hexdigest()] for item in entries]
|
|
255
|
+
)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _model_settings(models: Any, aliases: list[str]) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
259
|
+
"""Each alias's settings as Data Designer sends them: readable, and as hashed.
|
|
260
|
+
|
|
261
|
+
The readable form names a provider's extra headers without their values,
|
|
262
|
+
which may be credentials; the hashed form carries a digest of them.
|
|
263
|
+
"""
|
|
264
|
+
from usersim.cli._models import resolved_model_providers, to_model_configs
|
|
265
|
+
|
|
266
|
+
configs = {config.alias: config for config in to_model_configs(models)}
|
|
267
|
+
providers = {provider.name: provider for provider in resolved_model_providers(models)}
|
|
268
|
+
readable: dict[str, Any] = {}
|
|
269
|
+
hashed: dict[str, Any] = {}
|
|
270
|
+
for alias in sorted(set(aliases)):
|
|
271
|
+
config = configs.get(alias)
|
|
272
|
+
if config is None:
|
|
273
|
+
readable[alias] = hashed[alias] = None
|
|
274
|
+
continue
|
|
275
|
+
params = config.inference_parameters
|
|
276
|
+
settings = {"model": config.model, "request": dict(params.generate_kwargs), "timeout": params.timeout}
|
|
277
|
+
provider = providers.get(config.provider)
|
|
278
|
+
if provider is None:
|
|
279
|
+
readable[alias] = hashed[alias] = {**settings, "provider": None}
|
|
280
|
+
continue
|
|
281
|
+
served = {
|
|
282
|
+
"endpoint": provider.endpoint,
|
|
283
|
+
"provider_type": provider.provider_type,
|
|
284
|
+
"extra_body": provider.extra_body or {},
|
|
285
|
+
}
|
|
286
|
+
headers = provider.extra_headers or {}
|
|
287
|
+
readable[alias] = {**settings, "provider": {**served, "extra_headers": sorted(headers)}}
|
|
288
|
+
hashed[alias] = {**settings, "provider": {**served, "extra_headers": _digest(headers)}}
|
|
289
|
+
return readable, hashed
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _asset_roots_digest(assets_dir: Path) -> list[str]:
|
|
293
|
+
"""The contents of every asset root a run searches, in priority order.
|
|
294
|
+
|
|
295
|
+
Every root counts, not just the first copy of each file: an identity spec
|
|
296
|
+
that ``extends`` its own probe reads the copy below it on the path. Where a
|
|
297
|
+
root is does not count, so the same assets in another folder are the same
|
|
298
|
+
setup.
|
|
299
|
+
"""
|
|
300
|
+
from usersim.engine.core._assets import asset_search_path, reset_runtime_assets_dir, set_runtime_assets_dir
|
|
301
|
+
|
|
302
|
+
token = set_runtime_assets_dir(assets_dir)
|
|
303
|
+
try:
|
|
304
|
+
roots = asset_search_path()
|
|
305
|
+
finally:
|
|
306
|
+
reset_runtime_assets_dir(token)
|
|
307
|
+
return [_contents_digest(root) if root.exists() else "missing" for root in roots]
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def _environment_settings() -> dict[str, list[str]]:
|
|
311
|
+
"""Every ``USERSIM_*`` variable that can change a conversation.
|
|
312
|
+
|
|
313
|
+
A part of a value (split on ``os.pathsep``) that names an existing file or
|
|
314
|
+
folder counts by its contents, so the same bank in another place is the
|
|
315
|
+
same setup, and an edited bank is a different one.
|
|
316
|
+
"""
|
|
317
|
+
settings: dict[str, list[str]] = {}
|
|
318
|
+
for name in sorted(os.environ):
|
|
319
|
+
if not name.startswith("USERSIM_") or name in _OPERATIONAL_ENV:
|
|
320
|
+
continue
|
|
321
|
+
parts = os.environ[name].split(os.pathsep)
|
|
322
|
+
settings[name] = [_contents_digest(Path(part)) if part and Path(part).exists() else part for part in parts]
|
|
323
|
+
return settings
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def _prompt_versions() -> dict[str, str]:
|
|
327
|
+
"""``PROMPT_VERSION`` of every registered probe, read as the episode preamble reads it."""
|
|
328
|
+
from usersim.engine.core.probes import known_probes, resolve_probe
|
|
329
|
+
|
|
330
|
+
return {
|
|
331
|
+
label: str(getattr(sys.modules.get(resolve_probe(label).__module__), "PROMPT_VERSION", "v1.0"))
|
|
332
|
+
for label in known_probes()
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def _extension_identities() -> list[dict[str, Any]]:
|
|
337
|
+
"""Every entry point User Sim discovers, with the package release behind it."""
|
|
338
|
+
from importlib.metadata import entry_points
|
|
339
|
+
|
|
340
|
+
from usersim.engine.core._extensions import ENTRY_POINT_GROUPS, extensions_disabled
|
|
341
|
+
|
|
342
|
+
if extensions_disabled():
|
|
343
|
+
return []
|
|
344
|
+
identities: list[dict[str, Any]] = []
|
|
345
|
+
for group in ENTRY_POINT_GROUPS:
|
|
346
|
+
for entry in sorted(entry_points(group=group), key=lambda e: (e.name, e.value)):
|
|
347
|
+
dist = getattr(entry, "dist", None)
|
|
348
|
+
identities.append(
|
|
349
|
+
{
|
|
350
|
+
"group": group,
|
|
351
|
+
"name": entry.name,
|
|
352
|
+
"value": entry.value,
|
|
353
|
+
"distribution": dist.name if dist else None,
|
|
354
|
+
"version": dist.version if dist else None,
|
|
355
|
+
}
|
|
356
|
+
)
|
|
357
|
+
return identities
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def _data_designer_version() -> str | None:
|
|
361
|
+
"""The installed Data Designer release, which shapes every model request."""
|
|
362
|
+
from usersim.engine.core.manifest import _pkg_version
|
|
363
|
+
|
|
364
|
+
return _pkg_version("data_designer")
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def setup_fingerprint(models: Any, inputs: EpisodeInputs) -> tuple[str, dict[str, Any]]:
|
|
368
|
+
"""The fingerprint of everything about a setup that can change a conversation.
|
|
369
|
+
|
|
370
|
+
Returns the fingerprint and the run record that names it. Covered:
|
|
371
|
+
|
|
372
|
+
- every simulator model alias as Data Designer sends it, provider settings
|
|
373
|
+
included (``api_key``, a provider's name and ``max_parallel_requests``
|
|
374
|
+
are left out: they don't change what a model says);
|
|
375
|
+
- the shared ``usersim_config`` as loaded, defaults applied, with whether
|
|
376
|
+
the assistant's reasoning is replayed as the run's models file sets it;
|
|
377
|
+
- the contents of every asset root, and every non-operational
|
|
378
|
+
``USERSIM_*`` variable;
|
|
379
|
+
- every probe's ``PROMPT_VERSION``, every extension's package release, and
|
|
380
|
+
the User Sim and Data Designer releases.
|
|
381
|
+
|
|
382
|
+
A code change that bumps none of these versions is not seen.
|
|
383
|
+
"""
|
|
384
|
+
from usersim.engine.config import MODEL_ALIASES, ConversationSimulatorConfig
|
|
385
|
+
from usersim.engine.core.manifest import _own_version
|
|
386
|
+
from usersim.engine.core.storage import RUN_KIND_STORED
|
|
387
|
+
|
|
388
|
+
simulation = ConversationSimulatorConfig.model_validate(_run_config(inputs, models)).model_dump(mode="json")
|
|
389
|
+
for key in _PER_ROW_SETTINGS:
|
|
390
|
+
simulation.pop(key, None)
|
|
391
|
+
aliases = [*MODEL_ALIASES, simulation.get("finance_embedding_model_alias") or ""]
|
|
392
|
+
readable, hashed = _model_settings(models, [alias for alias in aliases if alias])
|
|
393
|
+
fingerprint = _digest(
|
|
394
|
+
{
|
|
395
|
+
"models": hashed,
|
|
396
|
+
"simulation": simulation,
|
|
397
|
+
"assets": _asset_roots_digest(inputs.assets_dir),
|
|
398
|
+
"environment": _environment_settings(),
|
|
399
|
+
"prompts": _prompt_versions(),
|
|
400
|
+
"extensions": _extension_identities(),
|
|
401
|
+
"usersim": _own_version(),
|
|
402
|
+
"data_designer": _data_designer_version(),
|
|
403
|
+
}
|
|
404
|
+
)
|
|
405
|
+
record = {"kind": RUN_KIND_STORED, "fingerprint": fingerprint, "models": readable, "simulation": simulation}
|
|
406
|
+
return fingerprint, record
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def replay_trajectory_id(row: Mapping[str, Any], fingerprint: str) -> str:
|
|
410
|
+
"""The ``trajectory_id`` of ``row`` run under the setup ``fingerprint`` names.
|
|
411
|
+
|
|
412
|
+
The stored row is hashed whole, apart from its machine-specific asset
|
|
413
|
+
path, so a row whose contents changed under the same stored id is a
|
|
414
|
+
different conversation.
|
|
415
|
+
"""
|
|
416
|
+
stored = dict(row)
|
|
417
|
+
stored["usersim_config"] = {key: value for key, value in row["usersim_config"].items() if key != "assets_dir"}
|
|
418
|
+
composite = f"{_ID_SCHEME}|{_digest(stored)}|{fingerprint}"
|
|
419
|
+
return hashlib.sha256(composite.encode("utf-8")).hexdigest()[:16]
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
# ── running ─────────────────────────────────────────────────────────────
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def _matching_run(out: Path, fingerprint: str) -> str | None:
|
|
426
|
+
"""The latest run of stored rows made with this setup."""
|
|
427
|
+
from usersim.engine.core.storage import RUN_KIND_STORED, latest_run_where
|
|
428
|
+
|
|
429
|
+
def matches(run_id: str, record: dict[str, Any] | None) -> bool:
|
|
430
|
+
return bool(record) and record.get("kind") == RUN_KIND_STORED and record.get("fingerprint") == fingerprint
|
|
431
|
+
|
|
432
|
+
return latest_run_where(out, matches)
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
def _saved_inputs(out: Path, run_id: str) -> dict[str, set[str]]:
|
|
436
|
+
"""Each ``input_id`` saved in ``run_id``, with the trajectory ids saved under it."""
|
|
437
|
+
from usersim.engine.core.storage import read_partitioned_dataset
|
|
438
|
+
|
|
439
|
+
saved = read_partitioned_dataset(out, run=run_id, columns=["trajectory_id", INPUT_ID_COLUMN])
|
|
440
|
+
pairs: dict[str, set[str]] = {}
|
|
441
|
+
if not saved.empty:
|
|
442
|
+
for input_id, trajectory_id in zip(saved[INPUT_ID_COLUMN].astype(str), saved["trajectory_id"].astype(str)):
|
|
443
|
+
pairs.setdefault(input_id, set()).add(trajectory_id)
|
|
444
|
+
return pairs
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
def _unsaved(planned: list[tuple[str, str]], saved: dict[str, set[str]], run_id: str) -> tuple[set[str], int]:
|
|
448
|
+
"""The ``(input_id, trajectory_id)`` pairs not yet saved in ``run_id``, and how many were.
|
|
449
|
+
|
|
450
|
+
Raises ``ValueError`` for an input saved there under a different id: its
|
|
451
|
+
stored contents changed, and one run must not hold two versions of it.
|
|
452
|
+
"""
|
|
453
|
+
todo: set[str] = set()
|
|
454
|
+
done = 0
|
|
455
|
+
for input_id, trajectory_id in planned:
|
|
456
|
+
prior = saved.get(input_id)
|
|
457
|
+
if not prior:
|
|
458
|
+
todo.add(input_id)
|
|
459
|
+
elif trajectory_id in prior:
|
|
460
|
+
done += 1
|
|
461
|
+
else:
|
|
462
|
+
raise ValueError(
|
|
463
|
+
f"stored row {input_id!r} already ran in run {run_id} with different contents; give the "
|
|
464
|
+
"changed row its own trajectory_id, or write to another --out"
|
|
465
|
+
)
|
|
466
|
+
return todo, done
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
@dataclass(frozen=True)
|
|
470
|
+
class InputsPlan:
|
|
471
|
+
"""What a run of stored rows would do, worked out before any model call."""
|
|
472
|
+
|
|
473
|
+
fingerprint: str
|
|
474
|
+
#: The record a new run with this setup is published with.
|
|
475
|
+
record: dict[str, Any]
|
|
476
|
+
#: The run of stored rows with this setup that a resume reopens, if any.
|
|
477
|
+
matching_run: str | None
|
|
478
|
+
#: Stored ids not yet saved in that run.
|
|
479
|
+
pending: set[str]
|
|
480
|
+
#: Rows already saved there.
|
|
481
|
+
saved: int
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def plan_inputs_run(inputs: EpisodeInputs, models: Any, out: str | Path, *, skip_existing: bool = True) -> InputsPlan:
|
|
485
|
+
"""Fingerprint the setup and find what the matching run already holds.
|
|
486
|
+
|
|
487
|
+
Raises ``ValueError`` if that run holds one of the stored rows with
|
|
488
|
+
different contents.
|
|
489
|
+
"""
|
|
490
|
+
fingerprint, record = setup_fingerprint(models, inputs)
|
|
491
|
+
stored_ids = {row["trajectory_id"] for row in inputs.rows}
|
|
492
|
+
matching = _matching_run(Path(out).resolve(), fingerprint) if skip_existing else None
|
|
493
|
+
if matching is None:
|
|
494
|
+
return InputsPlan(fingerprint, record, matching_run=None, pending=stored_ids, saved=0)
|
|
495
|
+
planned = [(row["trajectory_id"], replay_trajectory_id(row, fingerprint)) for row in inputs.rows]
|
|
496
|
+
pending, saved = _unsaved(planned, _saved_inputs(Path(out).resolve(), matching), matching)
|
|
497
|
+
return InputsPlan(fingerprint, record, matching_run=matching, pending=pending, saved=saved)
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
def _run_config(inputs: EpisodeInputs, models: Any) -> dict[str, Any]:
|
|
501
|
+
"""The rows' shared ``usersim_config``, with ``_RUN_SETTINGS`` as this run sets them.
|
|
502
|
+
|
|
503
|
+
The assistant under test comes from the run's models file, so whether its
|
|
504
|
+
earlier reasoning is replayed does too, as in a plain run. Raises
|
|
505
|
+
``ConfigError`` for a models file that cannot replay.
|
|
506
|
+
"""
|
|
507
|
+
from usersim.cli._models import assistant_replays_reasoning
|
|
508
|
+
from usersim.engine.config import ConversationSimulatorConfig
|
|
509
|
+
|
|
510
|
+
store_reasoning = ConversationSimulatorConfig.model_validate(dict(inputs.config)).store_reasoning
|
|
511
|
+
return {
|
|
512
|
+
**inputs.config,
|
|
513
|
+
"replay_assistant_reasoning": assistant_replays_reasoning(models, store_reasoning=store_reasoning),
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
def _column_config(inputs: EpisodeInputs, locale: str, models: Any):
|
|
518
|
+
"""The simulator column for ``locale``'s rows, on the run's asset root."""
|
|
519
|
+
from usersim.engine.config import ConversationSimulatorConfig
|
|
520
|
+
|
|
521
|
+
return ConversationSimulatorConfig.model_validate(
|
|
522
|
+
{**_run_config(inputs, models), "locale": locale, "assets_dir": str(inputs.assets_dir)}
|
|
523
|
+
)
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
#: Value types a stored column added back to the saved rows keeps as they are.
|
|
527
|
+
_PLAIN_TYPES = frozenset({str, bool, int, float})
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def _json_columns(rows: list[dict[str, Any]]) -> set[str]:
|
|
531
|
+
"""The stored columns a run saves as JSON text, chosen from all of an invocation's rows.
|
|
532
|
+
|
|
533
|
+
A column is saved as it is when every value present is of one plain type
|
|
534
|
+
(``str``, ``bool``, ``float``, or an ``int`` within 64 bits). Anything else,
|
|
535
|
+
a nested value or a mix such as an ``int`` beside a ``float``, would be
|
|
536
|
+
converted or refused by Arrow, so it is saved as JSON text, which keeps
|
|
537
|
+
every value exactly. The choice depends only on the stored rows, so every
|
|
538
|
+
invocation over the same rows, a resume included, saves a column the same
|
|
539
|
+
way.
|
|
540
|
+
"""
|
|
541
|
+
kinds: dict[str, set[type]] = {}
|
|
542
|
+
oversized: set[str] = set()
|
|
543
|
+
for row in rows:
|
|
544
|
+
for key, value in row.items():
|
|
545
|
+
if value is None:
|
|
546
|
+
continue
|
|
547
|
+
kinds.setdefault(key, set()).add(type(value))
|
|
548
|
+
if type(value) is int and not -(2**63) <= value < 2**63:
|
|
549
|
+
oversized.add(key)
|
|
550
|
+
return {key for key, types in kinds.items() if key in oversized or len(types) != 1 or not types <= _PLAIN_TYPES}
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
def _as_saved_rows(df, stored: dict[str, tuple[dict[str, Any], str]], json_columns: set[str]):
|
|
554
|
+
"""The rows a run saves.
|
|
555
|
+
|
|
556
|
+
The rows get the new ids, the stored id as ``input_id``, and a plain run's
|
|
557
|
+
columns. ``df`` comes back from Data Designer with each stored row's own
|
|
558
|
+
id as ``trajectory_id``. The carrier column and the full persona are
|
|
559
|
+
dropped, as a plain run drops the persona, and a stored column the run did
|
|
560
|
+
not keep is added back from the stored rows, as JSON text when it is one
|
|
561
|
+
of ``json_columns``.
|
|
562
|
+
"""
|
|
563
|
+
import pandas as pd
|
|
564
|
+
|
|
565
|
+
from usersim.engine.core.episode_input import EPISODE_INPUT_COLUMN
|
|
566
|
+
|
|
567
|
+
df = df.copy()
|
|
568
|
+
input_ids = df["trajectory_id"].astype(str).tolist()
|
|
569
|
+
df[INPUT_ID_COLUMN] = input_ids
|
|
570
|
+
df["trajectory_id"] = [stored[input_id][1] for input_id in input_ids]
|
|
571
|
+
df = df.drop(columns=[EPISODE_INPUT_COLUMN, "persona"], errors="ignore")
|
|
572
|
+
missing = sorted(
|
|
573
|
+
{key for row, _ in stored.values() for key in row} - set(df.columns) - {"usersim_config", "persona"}
|
|
574
|
+
)
|
|
575
|
+
for key in missing:
|
|
576
|
+
values = [stored[input_id][0].get(key) for input_id in input_ids]
|
|
577
|
+
if key in json_columns:
|
|
578
|
+
values = [None if value is None else json.dumps(value, ensure_ascii=False) for value in values]
|
|
579
|
+
df[key] = pd.Series(values, index=df.index, dtype=object)
|
|
580
|
+
return df
|
|
581
|
+
|
|
582
|
+
|
|
583
|
+
def simulate_inputs_to_dataset(
|
|
584
|
+
*,
|
|
585
|
+
inputs: EpisodeInputs,
|
|
586
|
+
models: Any,
|
|
587
|
+
out: str | Path,
|
|
588
|
+
models_path: str | Path | None = None,
|
|
589
|
+
skip_existing: bool = True,
|
|
590
|
+
plan: InputsPlan | None = None,
|
|
591
|
+
) -> InputsRun:
|
|
592
|
+
"""Run stored rows with ``models`` and save them as one run under ``out``.
|
|
593
|
+
|
|
594
|
+
Each locale's rows run as one Data Designer batch, through the simulator
|
|
595
|
+
column ``usersim simulate`` uses. With ``skip_existing`` (the default), the
|
|
596
|
+
rows go into the latest run with this setup, skipping rows already saved
|
|
597
|
+
there; otherwise they start a new run. ``plan`` is
|
|
598
|
+
:func:`plan_inputs_run`'s result for the same arguments, when the caller
|
|
599
|
+
already has it.
|
|
600
|
+
|
|
601
|
+
Raises ``ValueError`` when the run already holds a stored row with
|
|
602
|
+
different contents: before any model call when it is found up front, and
|
|
603
|
+
when the rows are saved when another writer saved them meanwhile, where
|
|
604
|
+
the write lock is shared. Where ``out`` is on a filesystem without file
|
|
605
|
+
locks, the rows are written without a lock shared between processes,
|
|
606
|
+
with a warning. Two calls at once with one setup can then each start a
|
|
607
|
+
run, and resuming one run from two places at once can save a row twice,
|
|
608
|
+
or both versions of a changed row.
|
|
609
|
+
"""
|
|
610
|
+
from usersim.cli._pipeline import build_inputs_config_builder
|
|
611
|
+
from usersim.engine.core.manifest import write_simulator_manifest
|
|
612
|
+
from usersim.engine.core.storage import (
|
|
613
|
+
publish_new_run,
|
|
614
|
+
run_started_at,
|
|
615
|
+
run_write_lock,
|
|
616
|
+
write_locale_partition,
|
|
617
|
+
)
|
|
618
|
+
|
|
619
|
+
started_at = time.time()
|
|
620
|
+
out = Path(out).resolve()
|
|
621
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
622
|
+
# A first look at the matching run, to skip saved rows before any model
|
|
623
|
+
# call and refuse changed ones early. Only the check under the lock counts.
|
|
624
|
+
plan = plan or plan_inputs_run(inputs, models, out, skip_existing=skip_existing)
|
|
625
|
+
fingerprint, matching, pending, skipped = plan.fingerprint, plan.matching_run, plan.pending, plan.saved
|
|
626
|
+
record = {**plan.record, "started_at": started_at}
|
|
627
|
+
stored = {row["trajectory_id"]: (row, replay_trajectory_id(row, fingerprint)) for row in inputs.rows}
|
|
628
|
+
logger.info("setup %s: %d stored rows, %d already saved", fingerprint[:12], len(stored), skipped)
|
|
629
|
+
|
|
630
|
+
by_locale: dict[str, dict[str, tuple[dict[str, Any], str]]] = {}
|
|
631
|
+
for input_id in sorted(pending):
|
|
632
|
+
row, new_id = stored[input_id]
|
|
633
|
+
by_locale.setdefault(row["usersim_config"]["locale"], {})[input_id] = (row, new_id)
|
|
634
|
+
|
|
635
|
+
# The rows go into the run the plan found, even if a newer run with this
|
|
636
|
+
# setup appears meanwhile, so they never split across two runs.
|
|
637
|
+
run_id: str | None = matching
|
|
638
|
+
resumed = matching is not None
|
|
639
|
+
json_columns = _json_columns(inputs.rows)
|
|
640
|
+
written = 0
|
|
641
|
+
requested: Counter[str] = Counter()
|
|
642
|
+
for locale, group in by_locale.items():
|
|
643
|
+
requested[locale] += len(group)
|
|
644
|
+
data_designer, builder = build_inputs_config_builder(
|
|
645
|
+
models=models, config=_column_config(inputs, locale, models), rows=[row for row, _ in group.values()]
|
|
646
|
+
)
|
|
647
|
+
# Data Designer names a batch's artifacts by the second it started, so
|
|
648
|
+
# two invocations in one directory could share them; each batch gets
|
|
649
|
+
# its own folder, removed once its rows are loaded.
|
|
650
|
+
with tempfile.TemporaryDirectory(prefix="usersim_inputs_") as artifacts:
|
|
651
|
+
result = data_designer.create(builder, num_records=len(group), artifact_path=artifacts)
|
|
652
|
+
df = _as_saved_rows(result.load_dataset(), group, json_columns)
|
|
653
|
+
with run_write_lock(out):
|
|
654
|
+
target = run_id or (_matching_run(out, fingerprint) if skip_existing else None)
|
|
655
|
+
if target is not None:
|
|
656
|
+
planned = list(zip(df[INPUT_ID_COLUMN].astype(str), df["trajectory_id"].astype(str)))
|
|
657
|
+
todo, done = _unsaved(planned, _saved_inputs(out, target), target)
|
|
658
|
+
df = df[df[INPUT_ID_COLUMN].astype(str).isin(todo)]
|
|
659
|
+
skipped += done
|
|
660
|
+
# This invocation keeps the run it found, even if every row
|
|
661
|
+
# was saved there meanwhile.
|
|
662
|
+
resumed = resumed or run_id is None
|
|
663
|
+
run_id = target
|
|
664
|
+
if df.empty:
|
|
665
|
+
continue
|
|
666
|
+
if target is None:
|
|
667
|
+
run_id = publish_new_run(df, out, locale=locale, record=record)
|
|
668
|
+
else:
|
|
669
|
+
write_locale_partition(df, out, locale=locale, run_id=target)
|
|
670
|
+
written += len(df)
|
|
671
|
+
logger.info("locale=%s: wrote %d trajectories from stored rows into run=%s", locale, len(df), run_id)
|
|
672
|
+
|
|
673
|
+
if written and run_id is not None:
|
|
674
|
+
probes = Counter(str(row.get("probe_type")) for row, _ in stored.values())
|
|
675
|
+
try:
|
|
676
|
+
write_simulator_manifest(
|
|
677
|
+
run_id=run_id,
|
|
678
|
+
started_at_epoch=run_started_at(out, run_id) or int(started_at),
|
|
679
|
+
trajectory_root=out,
|
|
680
|
+
models_config=models,
|
|
681
|
+
models_config_path=Path(models_path) if models_path else None,
|
|
682
|
+
sim_config=_column_config(inputs, next(iter(requested)), models),
|
|
683
|
+
locales_requested=sorted(requested),
|
|
684
|
+
probe_mix={probe: count / len(stored) for probe, count in sorted(probes.items())},
|
|
685
|
+
panel_path=inputs.source,
|
|
686
|
+
assets_dir=inputs.assets_dir,
|
|
687
|
+
trajectories_requested=dict(requested),
|
|
688
|
+
)
|
|
689
|
+
except Exception as error: # pragma: no cover - never fail a finished run on manifest issues
|
|
690
|
+
logger.warning("failed to write run manifest: %s", error)
|
|
691
|
+
return InputsRun(run_id=run_id, resumed=resumed, written=written, skipped=skipped)
|