edsl 0.1.39.dev3__py3-none-any.whl → 0.1.39.dev4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edsl/Base.py +413 -332
- edsl/BaseDiff.py +260 -260
- edsl/TemplateLoader.py +24 -24
- edsl/__init__.py +57 -49
- edsl/__version__.py +1 -1
- edsl/agents/Agent.py +1071 -867
- edsl/agents/AgentList.py +551 -413
- edsl/agents/Invigilator.py +284 -233
- edsl/agents/InvigilatorBase.py +257 -270
- edsl/agents/PromptConstructor.py +272 -354
- edsl/agents/QuestionInstructionPromptBuilder.py +128 -0
- edsl/agents/QuestionTemplateReplacementsBuilder.py +137 -0
- edsl/agents/__init__.py +2 -3
- edsl/agents/descriptors.py +99 -99
- edsl/agents/prompt_helpers.py +129 -129
- edsl/agents/question_option_processor.py +172 -0
- edsl/auto/AutoStudy.py +130 -117
- edsl/auto/StageBase.py +243 -230
- edsl/auto/StageGenerateSurvey.py +178 -178
- edsl/auto/StageLabelQuestions.py +125 -125
- edsl/auto/StagePersona.py +61 -61
- edsl/auto/StagePersonaDimensionValueRanges.py +88 -88
- edsl/auto/StagePersonaDimensionValues.py +74 -74
- edsl/auto/StagePersonaDimensions.py +69 -69
- edsl/auto/StageQuestions.py +74 -73
- edsl/auto/SurveyCreatorPipeline.py +21 -21
- edsl/auto/utilities.py +218 -224
- edsl/base/Base.py +279 -279
- edsl/config.py +177 -157
- edsl/conversation/Conversation.py +290 -290
- edsl/conversation/car_buying.py +59 -58
- edsl/conversation/chips.py +95 -95
- edsl/conversation/mug_negotiation.py +81 -81
- edsl/conversation/next_speaker_utilities.py +93 -93
- edsl/coop/CoopFunctionsMixin.py +15 -0
- edsl/coop/ExpectedParrotKeyHandler.py +125 -0
- edsl/coop/PriceFetcher.py +54 -54
- edsl/coop/__init__.py +2 -2
- edsl/coop/coop.py +1106 -1028
- edsl/coop/utils.py +131 -131
- edsl/data/Cache.py +573 -555
- edsl/data/CacheEntry.py +230 -233
- edsl/data/CacheHandler.py +168 -149
- edsl/data/RemoteCacheSync.py +186 -78
- edsl/data/SQLiteDict.py +292 -292
- edsl/data/__init__.py +5 -4
- edsl/data/hack.py +10 -0
- edsl/data/orm.py +10 -10
- edsl/data_transfer_models.py +74 -73
- edsl/enums.py +202 -175
- edsl/exceptions/BaseException.py +21 -21
- edsl/exceptions/__init__.py +54 -54
- edsl/exceptions/agents.py +54 -42
- edsl/exceptions/cache.py +5 -5
- edsl/exceptions/configuration.py +16 -16
- edsl/exceptions/coop.py +10 -10
- edsl/exceptions/data.py +14 -14
- edsl/exceptions/general.py +34 -34
- edsl/exceptions/inference_services.py +5 -0
- edsl/exceptions/jobs.py +33 -33
- edsl/exceptions/language_models.py +63 -63
- edsl/exceptions/prompts.py +15 -15
- edsl/exceptions/questions.py +109 -91
- edsl/exceptions/results.py +29 -29
- edsl/exceptions/scenarios.py +29 -22
- edsl/exceptions/surveys.py +37 -37
- edsl/inference_services/AnthropicService.py +106 -87
- edsl/inference_services/AvailableModelCacheHandler.py +184 -0
- edsl/inference_services/AvailableModelFetcher.py +215 -0
- edsl/inference_services/AwsBedrock.py +118 -120
- edsl/inference_services/AzureAI.py +215 -217
- edsl/inference_services/DeepInfraService.py +18 -18
- edsl/inference_services/GoogleService.py +143 -148
- edsl/inference_services/GroqService.py +20 -20
- edsl/inference_services/InferenceServiceABC.py +80 -147
- edsl/inference_services/InferenceServicesCollection.py +138 -97
- edsl/inference_services/MistralAIService.py +120 -123
- edsl/inference_services/OllamaService.py +18 -18
- edsl/inference_services/OpenAIService.py +236 -224
- edsl/inference_services/PerplexityService.py +160 -163
- edsl/inference_services/ServiceAvailability.py +135 -0
- edsl/inference_services/TestService.py +90 -89
- edsl/inference_services/TogetherAIService.py +172 -170
- edsl/inference_services/data_structures.py +134 -0
- edsl/inference_services/models_available_cache.py +118 -118
- edsl/inference_services/rate_limits_cache.py +25 -25
- edsl/inference_services/registry.py +41 -41
- edsl/inference_services/write_available.py +10 -10
- edsl/jobs/AnswerQuestionFunctionConstructor.py +223 -0
- edsl/jobs/Answers.py +43 -56
- edsl/jobs/FetchInvigilator.py +47 -0
- edsl/jobs/InterviewTaskManager.py +98 -0
- edsl/jobs/InterviewsConstructor.py +50 -0
- edsl/jobs/Jobs.py +823 -898
- edsl/jobs/JobsChecks.py +172 -147
- edsl/jobs/JobsComponentConstructor.py +189 -0
- edsl/jobs/JobsPrompts.py +270 -268
- edsl/jobs/JobsRemoteInferenceHandler.py +311 -239
- edsl/jobs/JobsRemoteInferenceLogger.py +239 -0
- edsl/jobs/RequestTokenEstimator.py +30 -0
- edsl/jobs/__init__.py +1 -1
- edsl/jobs/async_interview_runner.py +138 -0
- edsl/jobs/buckets/BucketCollection.py +104 -63
- edsl/jobs/buckets/ModelBuckets.py +65 -65
- edsl/jobs/buckets/TokenBucket.py +283 -251
- edsl/jobs/buckets/TokenBucketAPI.py +211 -0
- edsl/jobs/buckets/TokenBucketClient.py +191 -0
- edsl/jobs/check_survey_scenario_compatibility.py +85 -0
- edsl/jobs/data_structures.py +120 -0
- edsl/jobs/decorators.py +35 -0
- edsl/jobs/interviews/Interview.py +396 -661
- edsl/jobs/interviews/InterviewExceptionCollection.py +99 -99
- edsl/jobs/interviews/InterviewExceptionEntry.py +186 -186
- edsl/jobs/interviews/InterviewStatistic.py +63 -63
- edsl/jobs/interviews/InterviewStatisticsCollection.py +25 -25
- edsl/jobs/interviews/InterviewStatusDictionary.py +78 -78
- edsl/jobs/interviews/InterviewStatusLog.py +92 -92
- edsl/jobs/interviews/ReportErrors.py +66 -66
- edsl/jobs/interviews/interview_status_enum.py +9 -9
- edsl/jobs/jobs_status_enums.py +9 -0
- edsl/jobs/loggers/HTMLTableJobLogger.py +304 -0
- edsl/jobs/results_exceptions_handler.py +98 -0
- edsl/jobs/runners/JobsRunnerAsyncio.py +151 -466
- edsl/jobs/runners/JobsRunnerStatus.py +297 -330
- edsl/jobs/tasks/QuestionTaskCreator.py +244 -242
- edsl/jobs/tasks/TaskCreators.py +64 -64
- edsl/jobs/tasks/TaskHistory.py +470 -450
- edsl/jobs/tasks/TaskStatusLog.py +23 -23
- edsl/jobs/tasks/task_status_enum.py +161 -163
- edsl/jobs/tokens/InterviewTokenUsage.py +27 -27
- edsl/jobs/tokens/TokenUsage.py +34 -34
- edsl/language_models/ComputeCost.py +63 -0
- edsl/language_models/LanguageModel.py +626 -668
- edsl/language_models/ModelList.py +164 -155
- edsl/language_models/PriceManager.py +127 -0
- edsl/language_models/RawResponseHandler.py +106 -0
- edsl/language_models/RegisterLanguageModelsMeta.py +184 -184
- edsl/language_models/ServiceDataSources.py +0 -0
- edsl/language_models/__init__.py +2 -3
- edsl/language_models/fake_openai_call.py +15 -15
- edsl/language_models/fake_openai_service.py +61 -61
- edsl/language_models/key_management/KeyLookup.py +63 -0
- edsl/language_models/key_management/KeyLookupBuilder.py +273 -0
- edsl/language_models/key_management/KeyLookupCollection.py +38 -0
- edsl/language_models/key_management/__init__.py +0 -0
- edsl/language_models/key_management/models.py +131 -0
- edsl/language_models/model.py +256 -0
- edsl/language_models/repair.py +156 -156
- edsl/language_models/utilities.py +65 -64
- edsl/notebooks/Notebook.py +263 -258
- edsl/notebooks/NotebookToLaTeX.py +142 -0
- edsl/notebooks/__init__.py +1 -1
- edsl/prompts/Prompt.py +352 -362
- edsl/prompts/__init__.py +2 -2
- edsl/questions/ExceptionExplainer.py +77 -0
- edsl/questions/HTMLQuestion.py +103 -0
- edsl/questions/QuestionBase.py +518 -664
- edsl/questions/QuestionBasePromptsMixin.py +221 -217
- edsl/questions/QuestionBudget.py +227 -227
- edsl/questions/QuestionCheckBox.py +359 -359
- edsl/questions/QuestionExtract.py +180 -182
- edsl/questions/QuestionFreeText.py +113 -114
- edsl/questions/QuestionFunctional.py +166 -166
- edsl/questions/QuestionList.py +223 -231
- edsl/questions/QuestionMatrix.py +265 -0
- edsl/questions/QuestionMultipleChoice.py +330 -286
- edsl/questions/QuestionNumerical.py +151 -153
- edsl/questions/QuestionRank.py +314 -324
- edsl/questions/Quick.py +41 -41
- edsl/questions/SimpleAskMixin.py +74 -73
- edsl/questions/__init__.py +27 -26
- edsl/questions/{AnswerValidatorMixin.py → answer_validator_mixin.py} +334 -289
- edsl/questions/compose_questions.py +98 -98
- edsl/questions/data_structures.py +20 -0
- edsl/questions/decorators.py +21 -21
- edsl/questions/derived/QuestionLikertFive.py +76 -76
- edsl/questions/derived/QuestionLinearScale.py +90 -87
- edsl/questions/derived/QuestionTopK.py +93 -93
- edsl/questions/derived/QuestionYesNo.py +82 -82
- edsl/questions/descriptors.py +427 -413
- edsl/questions/loop_processor.py +149 -0
- edsl/questions/prompt_templates/question_budget.jinja +13 -13
- edsl/questions/prompt_templates/question_checkbox.jinja +32 -32
- edsl/questions/prompt_templates/question_extract.jinja +11 -11
- edsl/questions/prompt_templates/question_free_text.jinja +3 -3
- edsl/questions/prompt_templates/question_linear_scale.jinja +11 -11
- edsl/questions/prompt_templates/question_list.jinja +17 -17
- edsl/questions/prompt_templates/question_multiple_choice.jinja +33 -33
- edsl/questions/prompt_templates/question_numerical.jinja +36 -36
- edsl/questions/{QuestionBaseGenMixin.py → question_base_gen_mixin.py} +168 -161
- edsl/questions/question_registry.py +177 -177
- edsl/questions/{RegisterQuestionsMeta.py → register_questions_meta.py} +71 -71
- edsl/questions/{ResponseValidatorABC.py → response_validator_abc.py} +188 -174
- edsl/questions/response_validator_factory.py +34 -0
- edsl/questions/settings.py +12 -12
- edsl/questions/templates/budget/answering_instructions.jinja +7 -7
- edsl/questions/templates/budget/question_presentation.jinja +7 -7
- edsl/questions/templates/checkbox/answering_instructions.jinja +10 -10
- edsl/questions/templates/checkbox/question_presentation.jinja +22 -22
- edsl/questions/templates/extract/answering_instructions.jinja +7 -7
- edsl/questions/templates/likert_five/answering_instructions.jinja +10 -10
- edsl/questions/templates/likert_five/question_presentation.jinja +11 -11
- edsl/questions/templates/linear_scale/answering_instructions.jinja +5 -5
- edsl/questions/templates/linear_scale/question_presentation.jinja +5 -5
- edsl/questions/templates/list/answering_instructions.jinja +3 -3
- edsl/questions/templates/list/question_presentation.jinja +5 -5
- edsl/questions/templates/matrix/__init__.py +1 -0
- edsl/questions/templates/matrix/answering_instructions.jinja +5 -0
- edsl/questions/templates/matrix/question_presentation.jinja +20 -0
- edsl/questions/templates/multiple_choice/answering_instructions.jinja +9 -9
- edsl/questions/templates/multiple_choice/question_presentation.jinja +11 -11
- edsl/questions/templates/numerical/answering_instructions.jinja +6 -6
- edsl/questions/templates/numerical/question_presentation.jinja +6 -6
- edsl/questions/templates/rank/answering_instructions.jinja +11 -11
- edsl/questions/templates/rank/question_presentation.jinja +15 -15
- edsl/questions/templates/top_k/answering_instructions.jinja +8 -8
- edsl/questions/templates/top_k/question_presentation.jinja +22 -22
- edsl/questions/templates/yes_no/answering_instructions.jinja +6 -6
- edsl/questions/templates/yes_no/question_presentation.jinja +11 -11
- edsl/results/CSSParameterizer.py +108 -108
- edsl/results/Dataset.py +587 -424
- edsl/results/DatasetExportMixin.py +594 -731
- edsl/results/DatasetTree.py +295 -275
- edsl/results/MarkdownToDocx.py +122 -0
- edsl/results/MarkdownToPDF.py +111 -0
- edsl/results/Result.py +557 -465
- edsl/results/Results.py +1183 -1165
- edsl/results/ResultsExportMixin.py +45 -43
- edsl/results/ResultsGGMixin.py +121 -121
- edsl/results/TableDisplay.py +125 -198
- edsl/results/TextEditor.py +50 -0
- edsl/results/__init__.py +2 -2
- edsl/results/file_exports.py +252 -0
- edsl/results/{ResultsFetchMixin.py → results_fetch_mixin.py} +33 -33
- edsl/results/{Selector.py → results_selector.py} +145 -135
- edsl/results/{ResultsToolsMixin.py → results_tools_mixin.py} +98 -98
- edsl/results/smart_objects.py +96 -0
- edsl/results/table_data_class.py +12 -0
- edsl/results/table_display.css +77 -77
- edsl/results/table_renderers.py +118 -0
- edsl/results/tree_explore.py +115 -115
- edsl/scenarios/ConstructDownloadLink.py +109 -0
- edsl/scenarios/DocumentChunker.py +102 -0
- edsl/scenarios/DocxScenario.py +16 -0
- edsl/scenarios/FileStore.py +511 -632
- edsl/scenarios/PdfExtractor.py +40 -0
- edsl/scenarios/Scenario.py +498 -601
- edsl/scenarios/ScenarioHtmlMixin.py +65 -64
- edsl/scenarios/ScenarioList.py +1458 -1287
- edsl/scenarios/ScenarioListExportMixin.py +45 -52
- edsl/scenarios/ScenarioListPdfMixin.py +239 -261
- edsl/scenarios/__init__.py +3 -4
- edsl/scenarios/directory_scanner.py +96 -0
- edsl/scenarios/file_methods.py +85 -0
- edsl/scenarios/handlers/__init__.py +13 -0
- edsl/scenarios/handlers/csv.py +38 -0
- edsl/scenarios/handlers/docx.py +76 -0
- edsl/scenarios/handlers/html.py +37 -0
- edsl/scenarios/handlers/json.py +111 -0
- edsl/scenarios/handlers/latex.py +5 -0
- edsl/scenarios/handlers/md.py +51 -0
- edsl/scenarios/handlers/pdf.py +68 -0
- edsl/scenarios/handlers/png.py +39 -0
- edsl/scenarios/handlers/pptx.py +105 -0
- edsl/scenarios/handlers/py.py +294 -0
- edsl/scenarios/handlers/sql.py +313 -0
- edsl/scenarios/handlers/sqlite.py +149 -0
- edsl/scenarios/handlers/txt.py +33 -0
- edsl/scenarios/{ScenarioJoin.py → scenario_join.py} +131 -127
- edsl/scenarios/scenario_selector.py +156 -0
- edsl/shared.py +1 -1
- edsl/study/ObjectEntry.py +173 -173
- edsl/study/ProofOfWork.py +113 -113
- edsl/study/SnapShot.py +80 -80
- edsl/study/Study.py +521 -528
- edsl/study/__init__.py +4 -4
- edsl/surveys/ConstructDAG.py +92 -0
- edsl/surveys/DAG.py +148 -148
- edsl/surveys/EditSurvey.py +221 -0
- edsl/surveys/InstructionHandler.py +100 -0
- edsl/surveys/Memory.py +31 -31
- edsl/surveys/MemoryManagement.py +72 -0
- edsl/surveys/MemoryPlan.py +244 -244
- edsl/surveys/Rule.py +327 -326
- edsl/surveys/RuleCollection.py +385 -387
- edsl/surveys/RuleManager.py +172 -0
- edsl/surveys/Simulator.py +75 -0
- edsl/surveys/Survey.py +1280 -1801
- edsl/surveys/SurveyCSS.py +273 -261
- edsl/surveys/SurveyExportMixin.py +259 -259
- edsl/surveys/{SurveyFlowVisualizationMixin.py → SurveyFlowVisualization.py} +181 -179
- edsl/surveys/SurveyQualtricsImport.py +284 -284
- edsl/surveys/SurveyToApp.py +141 -0
- edsl/surveys/__init__.py +5 -3
- edsl/surveys/base.py +53 -53
- edsl/surveys/descriptors.py +60 -56
- edsl/surveys/instructions/ChangeInstruction.py +48 -49
- edsl/surveys/instructions/Instruction.py +56 -65
- edsl/surveys/instructions/InstructionCollection.py +82 -77
- edsl/templates/error_reporting/base.html +23 -23
- edsl/templates/error_reporting/exceptions_by_model.html +34 -34
- edsl/templates/error_reporting/exceptions_by_question_name.html +16 -16
- edsl/templates/error_reporting/exceptions_by_type.html +16 -16
- edsl/templates/error_reporting/interview_details.html +115 -115
- edsl/templates/error_reporting/interviews.html +19 -19
- edsl/templates/error_reporting/overview.html +4 -4
- edsl/templates/error_reporting/performance_plot.html +1 -1
- edsl/templates/error_reporting/report.css +73 -73
- edsl/templates/error_reporting/report.html +117 -117
- edsl/templates/error_reporting/report.js +25 -25
- edsl/test_h +1 -0
- edsl/tools/__init__.py +1 -1
- edsl/tools/clusters.py +192 -192
- edsl/tools/embeddings.py +27 -27
- edsl/tools/embeddings_plotting.py +118 -118
- edsl/tools/plotting.py +112 -112
- edsl/tools/summarize.py +18 -18
- edsl/utilities/PrettyList.py +56 -0
- edsl/utilities/SystemInfo.py +28 -28
- edsl/utilities/__init__.py +22 -22
- edsl/utilities/ast_utilities.py +25 -25
- edsl/utilities/data/Registry.py +6 -6
- edsl/utilities/data/__init__.py +1 -1
- edsl/utilities/data/scooter_results.json +1 -1
- edsl/utilities/decorators.py +77 -77
- edsl/utilities/gcp_bucket/cloud_storage.py +96 -96
- edsl/utilities/gcp_bucket/example.py +50 -0
- edsl/utilities/interface.py +627 -627
- edsl/utilities/is_notebook.py +18 -0
- edsl/utilities/is_valid_variable_name.py +11 -0
- edsl/utilities/naming_utilities.py +263 -263
- edsl/utilities/remove_edsl_version.py +24 -0
- edsl/utilities/repair_functions.py +28 -28
- edsl/utilities/restricted_python.py +70 -70
- edsl/utilities/utilities.py +436 -424
- {edsl-0.1.39.dev3.dist-info → edsl-0.1.39.dev4.dist-info}/LICENSE +21 -21
- {edsl-0.1.39.dev3.dist-info → edsl-0.1.39.dev4.dist-info}/METADATA +13 -11
- edsl-0.1.39.dev4.dist-info/RECORD +361 -0
- edsl/language_models/KeyLookup.py +0 -30
- edsl/language_models/registry.py +0 -190
- edsl/language_models/unused/ReplicateBase.py +0 -83
- edsl/results/ResultsDBMixin.py +0 -238
- edsl-0.1.39.dev3.dist-info/RECORD +0 -277
- {edsl-0.1.39.dev3.dist-info → edsl-0.1.39.dev4.dist-info}/WHEEL +0 -0
edsl/questions/QuestionBudget.py
CHANGED
@@ -1,227 +1,227 @@
|
|
1
|
-
from __future__ import annotations
|
2
|
-
from typing import Any, Optional, Union, List
|
3
|
-
|
4
|
-
from pydantic import Field, BaseModel, validator
|
5
|
-
|
6
|
-
from edsl.questions.QuestionBase import QuestionBase
|
7
|
-
from edsl.questions.descriptors import IntegerDescriptor, QuestionOptionsDescriptor
|
8
|
-
from edsl.questions.
|
9
|
-
|
10
|
-
|
11
|
-
class BudgewResponseValidator(ResponseValidatorABC):
|
12
|
-
valid_examples = []
|
13
|
-
|
14
|
-
invalid_examples = []
|
15
|
-
|
16
|
-
def fix(self, response, verbose=False):
|
17
|
-
if verbose:
|
18
|
-
print(f"Fixing list response: {response}")
|
19
|
-
answer = str(response.get("answer") or response.get("generated_tokens", ""))
|
20
|
-
if len(answer.split(",")) > 0:
|
21
|
-
return (
|
22
|
-
{"answer": answer.split(",")} | {"comment": response.get("comment")}
|
23
|
-
if "comment" in response
|
24
|
-
else {}
|
25
|
-
)
|
26
|
-
|
27
|
-
|
28
|
-
def create_budget_model(
|
29
|
-
budget_sum: float, permissive: bool, question_options: List[str]
|
30
|
-
):
|
31
|
-
class BudgetResponse(BaseModel):
|
32
|
-
answer: List[float] = Field(
|
33
|
-
...,
|
34
|
-
description="List of non-negative numbers representing budget allocation",
|
35
|
-
min_items=len(question_options),
|
36
|
-
max_items=len(question_options),
|
37
|
-
)
|
38
|
-
comment: Optional[str] = None
|
39
|
-
generated_tokens: Optional[str] = None
|
40
|
-
|
41
|
-
@validator("answer")
|
42
|
-
def validate_answer(cls, v):
|
43
|
-
if len(v) != len(question_options):
|
44
|
-
raise ValueError(f"Must provide {len(question_options)} values")
|
45
|
-
if any(x < 0 for x in v):
|
46
|
-
raise ValueError("All values must be non-negative")
|
47
|
-
total = sum(v)
|
48
|
-
if not permissive and total != budget_sum:
|
49
|
-
raise ValueError(f"Sum of numbers must equal {budget_sum}")
|
50
|
-
elif permissive and total > budget_sum:
|
51
|
-
raise ValueError(f"Sum of numbers cannot exceed {budget_sum}")
|
52
|
-
return v
|
53
|
-
|
54
|
-
class Config:
|
55
|
-
extra = "forbid"
|
56
|
-
|
57
|
-
return BudgetResponse
|
58
|
-
|
59
|
-
|
60
|
-
class QuestionBudget(QuestionBase):
|
61
|
-
"""This question prompts the agent to allocate a budget among options."""
|
62
|
-
|
63
|
-
question_type = "budget"
|
64
|
-
budget_sum: int = IntegerDescriptor(none_allowed=False)
|
65
|
-
question_options: list[str] = QuestionOptionsDescriptor(q_budget=True)
|
66
|
-
_response_model = None
|
67
|
-
response_validator_class = BudgewResponseValidator
|
68
|
-
|
69
|
-
def __init__(
|
70
|
-
self,
|
71
|
-
question_name: str,
|
72
|
-
question_text: str,
|
73
|
-
question_options: list[str],
|
74
|
-
budget_sum: int,
|
75
|
-
include_comment: bool = True,
|
76
|
-
question_presentation: Optional[str] = None,
|
77
|
-
answering_instructions: Optional[str] = None,
|
78
|
-
permissive: bool = False,
|
79
|
-
):
|
80
|
-
"""Instantiate a new QuestionBudget.
|
81
|
-
|
82
|
-
:param question_name: The name of the question.
|
83
|
-
:param question_text: The text of the question.
|
84
|
-
:param question_options: The options for allocation of the budget sum.
|
85
|
-
:param budget_sum: The total amount of the budget to be allocated among the options.
|
86
|
-
"""
|
87
|
-
self.question_name = question_name
|
88
|
-
self.question_text = question_text
|
89
|
-
self.question_options = question_options
|
90
|
-
self.budget_sum = budget_sum
|
91
|
-
self.question_presentation = question_presentation
|
92
|
-
self.answering_instructions = answering_instructions
|
93
|
-
self.permissive = permissive
|
94
|
-
self.include_comment = include_comment
|
95
|
-
|
96
|
-
def create_response_model(self):
|
97
|
-
return create_budget_model(
|
98
|
-
self.budget_sum, self.permissive, self.question_options
|
99
|
-
)
|
100
|
-
|
101
|
-
def _translate_answer_code_to_answer(
|
102
|
-
self, answer_code, combined_dict
|
103
|
-
) -> list[dict]:
|
104
|
-
"""
|
105
|
-
Translate the answer codes to the actual answers.
|
106
|
-
|
107
|
-
For example, for a budget question with options ["a", "b", "c"],
|
108
|
-
the answer codes are 0, 1, and 2. The LLM will respond with 0.
|
109
|
-
This code will translate that to "a".
|
110
|
-
"""
|
111
|
-
translated_codes = []
|
112
|
-
for answer_code, question_option in zip(answer_code, self.question_options):
|
113
|
-
translated_codes.append({question_option: answer_code})
|
114
|
-
|
115
|
-
return translated_codes
|
116
|
-
|
117
|
-
# def _simulate_answer(self, human_readable=True):
|
118
|
-
# """Simulate a valid answer for debugging purposes (what the validator expects)."""
|
119
|
-
# from edsl.utilities.utilities import random_string
|
120
|
-
|
121
|
-
# if human_readable:
|
122
|
-
# keys = self.question_options
|
123
|
-
# else:
|
124
|
-
# keys = range(len(self.question_options))
|
125
|
-
# remaining_budget = self.budget_sum
|
126
|
-
# values = []
|
127
|
-
# for _ in range(len(self.question_options)):
|
128
|
-
# if _ == len(self.question_options) - 1:
|
129
|
-
# # Assign remaining budget to the last value
|
130
|
-
# values.append(remaining_budget)
|
131
|
-
# else:
|
132
|
-
# # Generate a random value between 0 and remaining budget
|
133
|
-
# value = random.randint(0, remaining_budget)
|
134
|
-
# values.append(value)
|
135
|
-
# remaining_budget -= value
|
136
|
-
# answer = dict(zip(keys, values))
|
137
|
-
# return {
|
138
|
-
# "answer": answer,
|
139
|
-
# "comment": random_string(),
|
140
|
-
# }
|
141
|
-
|
142
|
-
@property
|
143
|
-
def question_html_content(self) -> str:
|
144
|
-
from jinja2 import Template
|
145
|
-
|
146
|
-
question_html_content = Template(
|
147
|
-
"""
|
148
|
-
<form id="budgetForm">
|
149
|
-
<p>Total Budget: {{ budget_sum }}</p>
|
150
|
-
<p>Remaining Budget: <span id="remainingBudget">{{ budget_sum }}</span></p>
|
151
|
-
{% for option in question_options %}
|
152
|
-
<div>
|
153
|
-
<label for="{{ option }}">{{ option }}</label>
|
154
|
-
<input type="number" id="{{ option }}" name="{{ question_name }}[{{ option }}]" value="0" min="0" max="{{ budget_sum }}" oninput="updateRemainingBudget()">
|
155
|
-
</div>
|
156
|
-
{% endfor %}
|
157
|
-
</form>
|
158
|
-
<script>
|
159
|
-
function updateRemainingBudget() {
|
160
|
-
let totalBudget = {{ budget_sum }};
|
161
|
-
let allocated = 0;
|
162
|
-
|
163
|
-
{% for option in question_options %}
|
164
|
-
allocated += parseInt(document.getElementById("{{ option }}").value) || 0;
|
165
|
-
{% endfor %}
|
166
|
-
|
167
|
-
let remaining = totalBudget - allocated;
|
168
|
-
document.getElementById('remainingBudget').innerText = remaining;
|
169
|
-
|
170
|
-
{% for option in question_options %}
|
171
|
-
document.getElementById("{{ option }}").max = remaining + parseInt(document.getElementById("{{ option }}").value);
|
172
|
-
{% endfor %}
|
173
|
-
}
|
174
|
-
</script>
|
175
|
-
"""
|
176
|
-
).render(
|
177
|
-
question_name=self.question_name,
|
178
|
-
budget_sum=self.budget_sum,
|
179
|
-
question_options=self.question_options,
|
180
|
-
)
|
181
|
-
return question_html_content
|
182
|
-
|
183
|
-
################
|
184
|
-
# Helpful methods
|
185
|
-
################
|
186
|
-
@classmethod
|
187
|
-
def example(cls, include_comment: bool = True) -> QuestionBudget:
|
188
|
-
"""Return an example of a budget question."""
|
189
|
-
return cls(
|
190
|
-
question_name="food_budget",
|
191
|
-
question_text="How would you allocate $100?",
|
192
|
-
question_options=["Pizza", "Ice Cream", "Burgers", "Salad"],
|
193
|
-
budget_sum=100,
|
194
|
-
include_comment=include_comment,
|
195
|
-
)
|
196
|
-
|
197
|
-
|
198
|
-
def main():
|
199
|
-
"""Create an example of a budget question and demonstrate its functionality."""
|
200
|
-
from edsl.questions.QuestionBudget import QuestionBudget
|
201
|
-
|
202
|
-
q = QuestionBudget.example()
|
203
|
-
q.question_text
|
204
|
-
q.question_options
|
205
|
-
q.question_name
|
206
|
-
# validate an answer
|
207
|
-
q._validate_answer(
|
208
|
-
{"answer": {0: 100, 1: 0, 2: 0, 3: 0}, "comment": "I like custard"}
|
209
|
-
)
|
210
|
-
# translate answer code
|
211
|
-
q._translate_answer_code_to_answer({0: 100, 1: 0, 2: 0, 3: 0})
|
212
|
-
# simulate answer
|
213
|
-
q._simulate_answer()
|
214
|
-
q._simulate_answer(human_readable=False)
|
215
|
-
q._validate_answer(q._simulate_answer(human_readable=False))
|
216
|
-
# serialization (inherits from Question)
|
217
|
-
q.to_dict()
|
218
|
-
assert q.from_dict(q.to_dict()) == q
|
219
|
-
|
220
|
-
|
221
|
-
if __name__ == "__main__":
|
222
|
-
# q = QuestionBudget.example()
|
223
|
-
# results = q.run()
|
224
|
-
|
225
|
-
import doctest
|
226
|
-
|
227
|
-
doctest.testmod(optionflags=doctest.ELLIPSIS)
|
1
|
+
from __future__ import annotations
|
2
|
+
from typing import Any, Optional, Union, List
|
3
|
+
|
4
|
+
from pydantic import Field, BaseModel, validator
|
5
|
+
|
6
|
+
from edsl.questions.QuestionBase import QuestionBase
|
7
|
+
from edsl.questions.descriptors import IntegerDescriptor, QuestionOptionsDescriptor
|
8
|
+
from edsl.questions.response_validator_abc import ResponseValidatorABC
|
9
|
+
|
10
|
+
|
11
|
+
class BudgewResponseValidator(ResponseValidatorABC):
|
12
|
+
valid_examples = []
|
13
|
+
|
14
|
+
invalid_examples = []
|
15
|
+
|
16
|
+
def fix(self, response, verbose=False):
|
17
|
+
if verbose:
|
18
|
+
print(f"Fixing list response: {response}")
|
19
|
+
answer = str(response.get("answer") or response.get("generated_tokens", ""))
|
20
|
+
if len(answer.split(",")) > 0:
|
21
|
+
return (
|
22
|
+
{"answer": answer.split(",")} | {"comment": response.get("comment")}
|
23
|
+
if "comment" in response
|
24
|
+
else {}
|
25
|
+
)
|
26
|
+
|
27
|
+
|
28
|
+
def create_budget_model(
|
29
|
+
budget_sum: float, permissive: bool, question_options: List[str]
|
30
|
+
):
|
31
|
+
class BudgetResponse(BaseModel):
|
32
|
+
answer: List[float] = Field(
|
33
|
+
...,
|
34
|
+
description="List of non-negative numbers representing budget allocation",
|
35
|
+
min_items=len(question_options),
|
36
|
+
max_items=len(question_options),
|
37
|
+
)
|
38
|
+
comment: Optional[str] = None
|
39
|
+
generated_tokens: Optional[str] = None
|
40
|
+
|
41
|
+
@validator("answer")
|
42
|
+
def validate_answer(cls, v):
|
43
|
+
if len(v) != len(question_options):
|
44
|
+
raise ValueError(f"Must provide {len(question_options)} values")
|
45
|
+
if any(x < 0 for x in v):
|
46
|
+
raise ValueError("All values must be non-negative")
|
47
|
+
total = sum(v)
|
48
|
+
if not permissive and total != budget_sum:
|
49
|
+
raise ValueError(f"Sum of numbers must equal {budget_sum}")
|
50
|
+
elif permissive and total > budget_sum:
|
51
|
+
raise ValueError(f"Sum of numbers cannot exceed {budget_sum}")
|
52
|
+
return v
|
53
|
+
|
54
|
+
class Config:
|
55
|
+
extra = "forbid"
|
56
|
+
|
57
|
+
return BudgetResponse
|
58
|
+
|
59
|
+
|
60
|
+
class QuestionBudget(QuestionBase):
|
61
|
+
"""This question prompts the agent to allocate a budget among options."""
|
62
|
+
|
63
|
+
question_type = "budget"
|
64
|
+
budget_sum: int = IntegerDescriptor(none_allowed=False)
|
65
|
+
question_options: list[str] = QuestionOptionsDescriptor(q_budget=True)
|
66
|
+
_response_model = None
|
67
|
+
response_validator_class = BudgewResponseValidator
|
68
|
+
|
69
|
+
def __init__(
|
70
|
+
self,
|
71
|
+
question_name: str,
|
72
|
+
question_text: str,
|
73
|
+
question_options: list[str],
|
74
|
+
budget_sum: int,
|
75
|
+
include_comment: bool = True,
|
76
|
+
question_presentation: Optional[str] = None,
|
77
|
+
answering_instructions: Optional[str] = None,
|
78
|
+
permissive: bool = False,
|
79
|
+
):
|
80
|
+
"""Instantiate a new QuestionBudget.
|
81
|
+
|
82
|
+
:param question_name: The name of the question.
|
83
|
+
:param question_text: The text of the question.
|
84
|
+
:param question_options: The options for allocation of the budget sum.
|
85
|
+
:param budget_sum: The total amount of the budget to be allocated among the options.
|
86
|
+
"""
|
87
|
+
self.question_name = question_name
|
88
|
+
self.question_text = question_text
|
89
|
+
self.question_options = question_options
|
90
|
+
self.budget_sum = budget_sum
|
91
|
+
self.question_presentation = question_presentation
|
92
|
+
self.answering_instructions = answering_instructions
|
93
|
+
self.permissive = permissive
|
94
|
+
self.include_comment = include_comment
|
95
|
+
|
96
|
+
def create_response_model(self):
|
97
|
+
return create_budget_model(
|
98
|
+
self.budget_sum, self.permissive, self.question_options
|
99
|
+
)
|
100
|
+
|
101
|
+
def _translate_answer_code_to_answer(
|
102
|
+
self, answer_code, combined_dict
|
103
|
+
) -> list[dict]:
|
104
|
+
"""
|
105
|
+
Translate the answer codes to the actual answers.
|
106
|
+
|
107
|
+
For example, for a budget question with options ["a", "b", "c"],
|
108
|
+
the answer codes are 0, 1, and 2. The LLM will respond with 0.
|
109
|
+
This code will translate that to "a".
|
110
|
+
"""
|
111
|
+
translated_codes = []
|
112
|
+
for answer_code, question_option in zip(answer_code, self.question_options):
|
113
|
+
translated_codes.append({question_option: answer_code})
|
114
|
+
|
115
|
+
return translated_codes
|
116
|
+
|
117
|
+
# def _simulate_answer(self, human_readable=True):
|
118
|
+
# """Simulate a valid answer for debugging purposes (what the validator expects)."""
|
119
|
+
# from edsl.utilities.utilities import random_string
|
120
|
+
|
121
|
+
# if human_readable:
|
122
|
+
# keys = self.question_options
|
123
|
+
# else:
|
124
|
+
# keys = range(len(self.question_options))
|
125
|
+
# remaining_budget = self.budget_sum
|
126
|
+
# values = []
|
127
|
+
# for _ in range(len(self.question_options)):
|
128
|
+
# if _ == len(self.question_options) - 1:
|
129
|
+
# # Assign remaining budget to the last value
|
130
|
+
# values.append(remaining_budget)
|
131
|
+
# else:
|
132
|
+
# # Generate a random value between 0 and remaining budget
|
133
|
+
# value = random.randint(0, remaining_budget)
|
134
|
+
# values.append(value)
|
135
|
+
# remaining_budget -= value
|
136
|
+
# answer = dict(zip(keys, values))
|
137
|
+
# return {
|
138
|
+
# "answer": answer,
|
139
|
+
# "comment": random_string(),
|
140
|
+
# }
|
141
|
+
|
142
|
+
@property
|
143
|
+
def question_html_content(self) -> str:
|
144
|
+
from jinja2 import Template
|
145
|
+
|
146
|
+
question_html_content = Template(
|
147
|
+
"""
|
148
|
+
<form id="budgetForm">
|
149
|
+
<p>Total Budget: {{ budget_sum }}</p>
|
150
|
+
<p>Remaining Budget: <span id="remainingBudget">{{ budget_sum }}</span></p>
|
151
|
+
{% for option in question_options %}
|
152
|
+
<div>
|
153
|
+
<label for="{{ option }}">{{ option }}</label>
|
154
|
+
<input type="number" id="{{ option }}" name="{{ question_name }}[{{ option }}]" value="0" min="0" max="{{ budget_sum }}" oninput="updateRemainingBudget()">
|
155
|
+
</div>
|
156
|
+
{% endfor %}
|
157
|
+
</form>
|
158
|
+
<script>
|
159
|
+
function updateRemainingBudget() {
|
160
|
+
let totalBudget = {{ budget_sum }};
|
161
|
+
let allocated = 0;
|
162
|
+
|
163
|
+
{% for option in question_options %}
|
164
|
+
allocated += parseInt(document.getElementById("{{ option }}").value) || 0;
|
165
|
+
{% endfor %}
|
166
|
+
|
167
|
+
let remaining = totalBudget - allocated;
|
168
|
+
document.getElementById('remainingBudget').innerText = remaining;
|
169
|
+
|
170
|
+
{% for option in question_options %}
|
171
|
+
document.getElementById("{{ option }}").max = remaining + parseInt(document.getElementById("{{ option }}").value);
|
172
|
+
{% endfor %}
|
173
|
+
}
|
174
|
+
</script>
|
175
|
+
"""
|
176
|
+
).render(
|
177
|
+
question_name=self.question_name,
|
178
|
+
budget_sum=self.budget_sum,
|
179
|
+
question_options=self.question_options,
|
180
|
+
)
|
181
|
+
return question_html_content
|
182
|
+
|
183
|
+
################
|
184
|
+
# Helpful methods
|
185
|
+
################
|
186
|
+
@classmethod
|
187
|
+
def example(cls, include_comment: bool = True) -> QuestionBudget:
|
188
|
+
"""Return an example of a budget question."""
|
189
|
+
return cls(
|
190
|
+
question_name="food_budget",
|
191
|
+
question_text="How would you allocate $100?",
|
192
|
+
question_options=["Pizza", "Ice Cream", "Burgers", "Salad"],
|
193
|
+
budget_sum=100,
|
194
|
+
include_comment=include_comment,
|
195
|
+
)
|
196
|
+
|
197
|
+
|
198
|
+
def main():
|
199
|
+
"""Create an example of a budget question and demonstrate its functionality."""
|
200
|
+
from edsl.questions.QuestionBudget import QuestionBudget
|
201
|
+
|
202
|
+
q = QuestionBudget.example()
|
203
|
+
q.question_text
|
204
|
+
q.question_options
|
205
|
+
q.question_name
|
206
|
+
# validate an answer
|
207
|
+
q._validate_answer(
|
208
|
+
{"answer": {0: 100, 1: 0, 2: 0, 3: 0}, "comment": "I like custard"}
|
209
|
+
)
|
210
|
+
# translate answer code
|
211
|
+
q._translate_answer_code_to_answer({0: 100, 1: 0, 2: 0, 3: 0})
|
212
|
+
# simulate answer
|
213
|
+
q._simulate_answer()
|
214
|
+
q._simulate_answer(human_readable=False)
|
215
|
+
q._validate_answer(q._simulate_answer(human_readable=False))
|
216
|
+
# serialization (inherits from Question)
|
217
|
+
q.to_dict()
|
218
|
+
assert q.from_dict(q.to_dict()) == q
|
219
|
+
|
220
|
+
|
221
|
+
if __name__ == "__main__":
|
222
|
+
# q = QuestionBudget.example()
|
223
|
+
# results = q.run()
|
224
|
+
|
225
|
+
import doctest
|
226
|
+
|
227
|
+
doctest.testmod(optionflags=doctest.ELLIPSIS)
|