edsl 0.1.33__py3-none-any.whl → 0.1.33.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. edsl/Base.py +3 -9
  2. edsl/__init__.py +3 -8
  3. edsl/__version__.py +1 -1
  4. edsl/agents/Agent.py +8 -40
  5. edsl/agents/AgentList.py +0 -43
  6. edsl/agents/Invigilator.py +219 -135
  7. edsl/agents/InvigilatorBase.py +59 -148
  8. edsl/agents/{PromptConstructor.py → PromptConstructionMixin.py} +89 -138
  9. edsl/agents/__init__.py +0 -1
  10. edsl/config.py +56 -47
  11. edsl/coop/coop.py +7 -50
  12. edsl/data/Cache.py +1 -35
  13. edsl/data_transfer_models.py +38 -73
  14. edsl/enums.py +0 -4
  15. edsl/exceptions/language_models.py +1 -25
  16. edsl/exceptions/questions.py +5 -62
  17. edsl/exceptions/results.py +0 -4
  18. edsl/inference_services/AnthropicService.py +11 -13
  19. edsl/inference_services/AwsBedrock.py +17 -19
  20. edsl/inference_services/AzureAI.py +20 -37
  21. edsl/inference_services/GoogleService.py +12 -16
  22. edsl/inference_services/GroqService.py +0 -2
  23. edsl/inference_services/InferenceServiceABC.py +3 -58
  24. edsl/inference_services/OpenAIService.py +54 -48
  25. edsl/inference_services/models_available_cache.py +6 -0
  26. edsl/inference_services/registry.py +0 -6
  27. edsl/jobs/Answers.py +12 -10
  28. edsl/jobs/Jobs.py +21 -36
  29. edsl/jobs/buckets/BucketCollection.py +15 -24
  30. edsl/jobs/buckets/TokenBucket.py +14 -93
  31. edsl/jobs/interviews/Interview.py +78 -366
  32. edsl/jobs/interviews/InterviewExceptionEntry.py +19 -85
  33. edsl/jobs/interviews/InterviewTaskBuildingMixin.py +286 -0
  34. edsl/jobs/interviews/{InterviewExceptionCollection.py → interview_exception_tracking.py} +68 -14
  35. edsl/jobs/interviews/retry_management.py +37 -0
  36. edsl/jobs/runners/JobsRunnerAsyncio.py +175 -146
  37. edsl/jobs/runners/JobsRunnerStatusMixin.py +333 -0
  38. edsl/jobs/tasks/QuestionTaskCreator.py +23 -30
  39. edsl/jobs/tasks/TaskHistory.py +213 -148
  40. edsl/language_models/LanguageModel.py +156 -261
  41. edsl/language_models/ModelList.py +2 -2
  42. edsl/language_models/RegisterLanguageModelsMeta.py +29 -14
  43. edsl/language_models/registry.py +6 -23
  44. edsl/language_models/repair.py +19 -0
  45. edsl/prompts/Prompt.py +2 -52
  46. edsl/questions/AnswerValidatorMixin.py +26 -23
  47. edsl/questions/QuestionBase.py +249 -329
  48. edsl/questions/QuestionBudget.py +41 -99
  49. edsl/questions/QuestionCheckBox.py +35 -227
  50. edsl/questions/QuestionExtract.py +27 -98
  51. edsl/questions/QuestionFreeText.py +29 -52
  52. edsl/questions/QuestionFunctional.py +0 -7
  53. edsl/questions/QuestionList.py +22 -141
  54. edsl/questions/QuestionMultipleChoice.py +65 -159
  55. edsl/questions/QuestionNumerical.py +46 -88
  56. edsl/questions/QuestionRank.py +24 -182
  57. edsl/questions/RegisterQuestionsMeta.py +12 -31
  58. edsl/questions/__init__.py +4 -3
  59. edsl/questions/derived/QuestionLikertFive.py +5 -10
  60. edsl/questions/derived/QuestionLinearScale.py +2 -15
  61. edsl/questions/derived/QuestionTopK.py +1 -10
  62. edsl/questions/derived/QuestionYesNo.py +3 -24
  63. edsl/questions/descriptors.py +7 -43
  64. edsl/questions/question_registry.py +2 -6
  65. edsl/results/Dataset.py +0 -20
  66. edsl/results/DatasetExportMixin.py +48 -46
  67. edsl/results/Result.py +5 -32
  68. edsl/results/Results.py +46 -135
  69. edsl/results/ResultsDBMixin.py +3 -3
  70. edsl/scenarios/FileStore.py +10 -71
  71. edsl/scenarios/Scenario.py +25 -96
  72. edsl/scenarios/ScenarioImageMixin.py +2 -2
  73. edsl/scenarios/ScenarioList.py +39 -361
  74. edsl/scenarios/ScenarioListExportMixin.py +0 -9
  75. edsl/scenarios/ScenarioListPdfMixin.py +4 -150
  76. edsl/study/SnapShot.py +1 -8
  77. edsl/study/Study.py +0 -32
  78. edsl/surveys/Rule.py +1 -10
  79. edsl/surveys/RuleCollection.py +5 -21
  80. edsl/surveys/Survey.py +310 -636
  81. edsl/surveys/SurveyExportMixin.py +9 -71
  82. edsl/surveys/SurveyFlowVisualizationMixin.py +1 -2
  83. edsl/surveys/SurveyQualtricsImport.py +4 -75
  84. edsl/utilities/gcp_bucket/simple_example.py +9 -0
  85. edsl/utilities/utilities.py +1 -9
  86. {edsl-0.1.33.dist-info → edsl-0.1.33.dev1.dist-info}/METADATA +2 -5
  87. edsl-0.1.33.dev1.dist-info/RECORD +209 -0
  88. edsl/TemplateLoader.py +0 -24
  89. edsl/auto/AutoStudy.py +0 -117
  90. edsl/auto/StageBase.py +0 -230
  91. edsl/auto/StageGenerateSurvey.py +0 -178
  92. edsl/auto/StageLabelQuestions.py +0 -125
  93. edsl/auto/StagePersona.py +0 -61
  94. edsl/auto/StagePersonaDimensionValueRanges.py +0 -88
  95. edsl/auto/StagePersonaDimensionValues.py +0 -74
  96. edsl/auto/StagePersonaDimensions.py +0 -69
  97. edsl/auto/StageQuestions.py +0 -73
  98. edsl/auto/SurveyCreatorPipeline.py +0 -21
  99. edsl/auto/utilities.py +0 -224
  100. edsl/coop/PriceFetcher.py +0 -58
  101. edsl/inference_services/MistralAIService.py +0 -120
  102. edsl/inference_services/TestService.py +0 -80
  103. edsl/inference_services/TogetherAIService.py +0 -170
  104. edsl/jobs/FailedQuestion.py +0 -78
  105. edsl/jobs/runners/JobsRunnerStatus.py +0 -331
  106. edsl/language_models/fake_openai_call.py +0 -15
  107. edsl/language_models/fake_openai_service.py +0 -61
  108. edsl/language_models/utilities.py +0 -61
  109. edsl/questions/QuestionBaseGenMixin.py +0 -133
  110. edsl/questions/QuestionBasePromptsMixin.py +0 -266
  111. edsl/questions/Quick.py +0 -41
  112. edsl/questions/ResponseValidatorABC.py +0 -170
  113. edsl/questions/decorators.py +0 -21
  114. edsl/questions/prompt_templates/question_budget.jinja +0 -13
  115. edsl/questions/prompt_templates/question_checkbox.jinja +0 -32
  116. edsl/questions/prompt_templates/question_extract.jinja +0 -11
  117. edsl/questions/prompt_templates/question_free_text.jinja +0 -3
  118. edsl/questions/prompt_templates/question_linear_scale.jinja +0 -11
  119. edsl/questions/prompt_templates/question_list.jinja +0 -17
  120. edsl/questions/prompt_templates/question_multiple_choice.jinja +0 -33
  121. edsl/questions/prompt_templates/question_numerical.jinja +0 -37
  122. edsl/questions/templates/__init__.py +0 -0
  123. edsl/questions/templates/budget/__init__.py +0 -0
  124. edsl/questions/templates/budget/answering_instructions.jinja +0 -7
  125. edsl/questions/templates/budget/question_presentation.jinja +0 -7
  126. edsl/questions/templates/checkbox/__init__.py +0 -0
  127. edsl/questions/templates/checkbox/answering_instructions.jinja +0 -10
  128. edsl/questions/templates/checkbox/question_presentation.jinja +0 -22
  129. edsl/questions/templates/extract/__init__.py +0 -0
  130. edsl/questions/templates/extract/answering_instructions.jinja +0 -7
  131. edsl/questions/templates/extract/question_presentation.jinja +0 -1
  132. edsl/questions/templates/free_text/__init__.py +0 -0
  133. edsl/questions/templates/free_text/answering_instructions.jinja +0 -0
  134. edsl/questions/templates/free_text/question_presentation.jinja +0 -1
  135. edsl/questions/templates/likert_five/__init__.py +0 -0
  136. edsl/questions/templates/likert_five/answering_instructions.jinja +0 -10
  137. edsl/questions/templates/likert_five/question_presentation.jinja +0 -12
  138. edsl/questions/templates/linear_scale/__init__.py +0 -0
  139. edsl/questions/templates/linear_scale/answering_instructions.jinja +0 -5
  140. edsl/questions/templates/linear_scale/question_presentation.jinja +0 -5
  141. edsl/questions/templates/list/__init__.py +0 -0
  142. edsl/questions/templates/list/answering_instructions.jinja +0 -4
  143. edsl/questions/templates/list/question_presentation.jinja +0 -5
  144. edsl/questions/templates/multiple_choice/__init__.py +0 -0
  145. edsl/questions/templates/multiple_choice/answering_instructions.jinja +0 -9
  146. edsl/questions/templates/multiple_choice/html.jinja +0 -0
  147. edsl/questions/templates/multiple_choice/question_presentation.jinja +0 -12
  148. edsl/questions/templates/numerical/__init__.py +0 -0
  149. edsl/questions/templates/numerical/answering_instructions.jinja +0 -8
  150. edsl/questions/templates/numerical/question_presentation.jinja +0 -7
  151. edsl/questions/templates/rank/__init__.py +0 -0
  152. edsl/questions/templates/rank/answering_instructions.jinja +0 -11
  153. edsl/questions/templates/rank/question_presentation.jinja +0 -15
  154. edsl/questions/templates/top_k/__init__.py +0 -0
  155. edsl/questions/templates/top_k/answering_instructions.jinja +0 -8
  156. edsl/questions/templates/top_k/question_presentation.jinja +0 -22
  157. edsl/questions/templates/yes_no/__init__.py +0 -0
  158. edsl/questions/templates/yes_no/answering_instructions.jinja +0 -6
  159. edsl/questions/templates/yes_no/question_presentation.jinja +0 -12
  160. edsl/results/DatasetTree.py +0 -145
  161. edsl/results/Selector.py +0 -118
  162. edsl/results/tree_explore.py +0 -115
  163. edsl/surveys/instructions/ChangeInstruction.py +0 -47
  164. edsl/surveys/instructions/Instruction.py +0 -34
  165. edsl/surveys/instructions/InstructionCollection.py +0 -77
  166. edsl/surveys/instructions/__init__.py +0 -0
  167. edsl/templates/error_reporting/base.html +0 -24
  168. edsl/templates/error_reporting/exceptions_by_model.html +0 -35
  169. edsl/templates/error_reporting/exceptions_by_question_name.html +0 -17
  170. edsl/templates/error_reporting/exceptions_by_type.html +0 -17
  171. edsl/templates/error_reporting/interview_details.html +0 -116
  172. edsl/templates/error_reporting/interviews.html +0 -10
  173. edsl/templates/error_reporting/overview.html +0 -5
  174. edsl/templates/error_reporting/performance_plot.html +0 -2
  175. edsl/templates/error_reporting/report.css +0 -74
  176. edsl/templates/error_reporting/report.html +0 -118
  177. edsl/templates/error_reporting/report.js +0 -25
  178. edsl-0.1.33.dist-info/RECORD +0 -295
  179. {edsl-0.1.33.dist-info → edsl-0.1.33.dev1.dist-info}/LICENSE +0 -0
  180. {edsl-0.1.33.dist-info → edsl-0.1.33.dev1.dist-info}/WHEEL +0 -0
@@ -1,116 +0,0 @@
1
- <div class="question">question_name: {{ question }}</div>
2
-
3
-
4
- <h2>Exception details</h2>
5
-
6
- {% for exception_message in exceptions %}
7
- <div class="exception-detail">
8
- <div class="exception-header">
9
- <span class="exception-exception">Exception: {{ exception_message.name }}</span>
10
- <button class="toggle-btn">▼</button>
11
- </div>
12
- <div class="exception-content">
13
- <table border="1">
14
- <tr>
15
- <th>Key</th>
16
- <th>Value</th>
17
- </tr>
18
- <tr>
19
- <td>Interview ID (index in results)</td>
20
- <td>{{ index }}</td>
21
- </tr>
22
- <tr>
23
- <td>Question name (question_name)</td>
24
- <td>{{ question }}</td>
25
- </tr>
26
-
27
- <tr>
28
- <td>Question type (question_type)</td>
29
- <td>{{ exception_message.question_type }}</td>
30
- </tr>
31
-
32
- <tr>
33
- <td>Human-readable question</td>
34
- <td>{{ interview.survey.get_question(question).html(
35
- scenario = interview.scenario,
36
- agent = interview.agent,
37
- answers = exception_message.answers)
38
-
39
- }}</td>
40
- </tr>
41
- <tr>
42
- <td>Scenario</td>
43
- <td>{{ interview.scenario._repr_html_() }}</td>
44
- </tr>
45
- <tr>
46
- <td>Agent</td>
47
- <td>{{ interview.agent._repr_html_() }}</td>
48
- </tr>
49
- <tr>
50
- <td>Model name</td>
51
- <td>{{ interview.model.model }}</td>
52
- </tr>
53
- <tr>
54
- <td>Inference service</td>
55
- <td>{{ interview.model._inference_service_ }}</td>
56
- </tr>
57
- <tr>
58
- <td>Model parameters</td>
59
- <td>{{ interview.model._repr_html_() }}</td>
60
- </tr>
61
- <tr>
62
- <td>User Prompt</td>
63
- <td><pre>{{ exception_message.rendered_prompts['user_prompt'] }}</pre></td>
64
- </tr>
65
- <tr>
66
- <td>System Prompt</td>
67
- <td><pre>{{ exception_message.rendered_prompts['system_prompt'] }}</pre></td>
68
- </tr>
69
- <tr>
70
- <td>Raw model response</td>
71
- <td><pre>{{ exception_message.raw_model_response }}</pre>
72
- </td>
73
- </tr>
74
- <tr>
75
- <td>Generated token string (at {{ exception_message.key_sequence }}) in raw response</td>
76
- <td><pre>{{ exception_message.generated_token_string }}</pre>
77
- </td>
78
- </tr>
79
- <tr>
80
- <td>Code to (likely) reproduce the error</td>
81
- <td>
82
- <textarea id="codeToCopy" rows="10" cols="90">{{ exception_message.code_to_reproduce }}</textarea>
83
- <button onclick="copyCode()">Copy</button>
84
- </td>
85
- </tr>
86
-
87
- </table>
88
-
89
-
90
- {% if exception_message.exception.__class__.__name__ == 'QuestionAnswerValidationError' %}
91
- <h3>Answer validation details</h3>
92
- <table border="1">
93
- <tr>
94
- <th>Field</th>
95
- <th>Value</th>
96
- </tr>
97
- {% for field, (explanation, open_tag, close_tag, value) in exception_message.exception.to_html_dict().items() %}
98
-
99
- <tr>
100
- <td>{{ field }}: ({{ explanation }})</td>
101
- <td><{{open_tag}}> {{ value | escape }} <{{close_tag}}></td>
102
- </tr>
103
- {% endfor %}
104
- </table>
105
- {% endif %}
106
-
107
- <div class="exception-time">Time: {{ exception_message.time }}</div>
108
- <div class="exception-traceback">Traceback:
109
- <text>
110
- <pre>{{ exception_message.traceback }}</pre>
111
- </text>
112
- </div>
113
- </div>
114
- </div>
115
-
116
- {% endfor %}
@@ -1,10 +0,0 @@
1
- {% for index, interview in interviews.items() %}
2
- {% if interview.exceptions != {} %}
3
- <div class="interview">Interview: {{ index }} </div>
4
- Model: {{ interview.model.model }}
5
- <h1>Failing questions</h1>
6
- {% endif %}
7
- {% for question, exceptions in interview.exceptions.items() %}
8
- {% include 'interview_details.html' %}
9
- {% endfor %}
10
- {% endfor %}
@@ -1,5 +0,0 @@
1
- <h1>Overview</h1>
2
- <p>There were {{ interviews|length }} total interview(s). An 'interview' is the result of one survey, taken by one agent, with one model, with one scenario.</p>
3
- The number of interviews with any exceptions was {{ num_exceptions }}.</p>
4
- <p>For advice on dealing with exceptions on Expected Parrot,
5
- see <a href="https://docs.expectedparrot.com/en/latest/exceptions.html">here</a>.</p>
@@ -1,2 +0,0 @@
1
- <h1>Performance Plot</h1>
2
- {{ performance_plot_html }}
@@ -1,74 +0,0 @@
1
- body {
2
- font-family: Arial, sans-serif;
3
- line-height: 1.6;
4
- background-color: #f9f9f9;
5
- color: #333;
6
- margin: 20px;
7
- }
8
-
9
- .interview {
10
- font-size: 1.5em;
11
- margin-bottom: 10px;
12
- padding: 10px;
13
- background-color: #e3f2fd;
14
- border-left: 5px solid #2196f3;
15
- }
16
-
17
- .question {
18
- font-size: 1.2em;
19
- margin-bottom: 10px;
20
- padding: 10px;
21
- background-color: #fff9c4;
22
- border-left: 5px solid #ffeb3b;
23
- }
24
-
25
- .exception-detail {
26
- margin-bottom: 10px;
27
- background-color: #ffebee;
28
- border-left: 5px solid #f44336;
29
- }
30
-
31
- .exception-header {
32
- padding: 10px;
33
- cursor: pointer;
34
- display: flex;
35
- justify-content: space-between;
36
- align-items: center;
37
- }
38
-
39
- .exception-content {
40
- padding: 10px;
41
- display: none;
42
- }
43
-
44
- .exception-content.show {
45
- display: block;
46
- }
47
-
48
- .question-detail {
49
- border: 3px solid black;
50
- padding: 10px;
51
- }
52
-
53
- .exception-exception {
54
- font-weight: bold;
55
- color: #d32f2f;
56
- }
57
-
58
- .exception-time,
59
- .exception-traceback {
60
- font-style: italic;
61
- color: #555;
62
- }
63
-
64
- .toggle-btn {
65
- background: none;
66
- border: none;
67
- font-size: 1.2em;
68
- cursor: pointer;
69
- transition: transform 0.3s;
70
- }
71
-
72
- .toggle-btn.rotated {
73
- transform: rotate(180deg);
74
- }
@@ -1,118 +0,0 @@
1
- <!DOCTYPE html>
2
- <html lang="en">
3
- <head>
4
- <meta charset="UTF-8">
5
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
6
- <title>Exception Details</title>
7
- <style>
8
- {{ css }}
9
- </style>
10
- </head>
11
- <body>
12
- <h1>Overview</h1>
13
- <p>There were {{ interviews|length }} total interviews. The number of interviews with exceptions was {{ num_exceptions }}.</p>
14
- <p>The models used were: {{ models_used }}.</p>
15
- <p>For documentation on dealing with exceptions on Expected Parrot,
16
- see <a href="https://docs.expectedparrot.com/en/latest/exceptions.html">here</a>.</p>
17
-
18
- <h2>Exceptions by Type</h2>
19
- <table>
20
- <thead>
21
- <tr>
22
- <th>Exception Type</th>
23
- <th>Number</th>
24
- </tr>
25
- </thead>
26
- <tbody>
27
- {% for exception_type, exceptions in exceptions_by_type.items() %}
28
- <tr>
29
- <td>{{ exception_type }}</td>
30
- <td>{{ exceptions }}</td>
31
- </tr>
32
- {% endfor %}
33
- </tbody>
34
- </table>
35
-
36
-
37
- <h2>Exceptions by Model</h2>
38
- <table>
39
- <thead>
40
- <tr>
41
- <th>Model</th>
42
- <th>Number</th>
43
- </tr>
44
- </thead>
45
- <tbody>
46
- {% for model, exceptions in exceptions_by_model.items() %}
47
- <tr>
48
- <td>{{ model }}</td>
49
- <td>{{ exceptions }}</td>
50
- </tr>
51
- {% endfor %}
52
- </tbody>
53
- </table>
54
-
55
-
56
- <h2>Exceptions by Question Name</h2>
57
- <table>
58
- <thead>
59
- <tr>
60
- <th>Question Name</th>
61
- <th>Number of Exceptions</th>
62
- </tr>
63
- </thead>
64
- <tbody>
65
- {% for question_name, exception_count in exceptions_by_question_name.items() %}
66
- <tr>
67
- <td>{{ question_name }}</td>
68
- <td>{{ exception_count }}</td>
69
- </tr>
70
- {% endfor %}
71
- </tbody>
72
- </table>
73
-
74
-
75
- {% for index, interview in interviews.items() %}
76
- {% if interview.exceptions != {} %}
77
- <div class="interview">Interview: {{ index }} </div>
78
- <h1>Failing questions</h1>
79
- {% endif %}
80
- {% for question, exceptions in interview.exceptions.items() %}
81
- <div class="question">question_name: {{ question }}</div>
82
-
83
- <h2>Question</h2>
84
- <div class="question-detail">
85
- {{ interview.survey.get_question(question).html() }}
86
- </div>
87
-
88
- <h2>Scenario</h2>
89
- <div class="scenario">
90
- {{ interview.scenario._repr_html_() }}
91
- </div>
92
-
93
- <h2>Agent</h2>
94
- <div class="agent">
95
- {{ interview.agent._repr_html_() }}
96
- </div>
97
-
98
- <h2>Model</h2>
99
- <div class="model">
100
- {{ interview.model._repr_html_() }}
101
- </div>
102
-
103
- <h2>Exception details</h2>
104
-
105
- {% for exception_message in exceptions %}
106
- <div class="exception-detail">
107
- <div class="exception-exception">Exception: {{ exception_message.exception }}</div>
108
- <div class="exception-time">Time: {{ exception_message.time }}</div>
109
- <div class="exception-traceback">Traceback: <pre>{{ exception_message.traceback }} </pre></div>
110
- </div>
111
- {% endfor %}
112
- {% endfor %}
113
- {% endfor %}
114
-
115
- <h1>Performance Plot</h1>
116
- {{ performance_plot_html }}
117
- </body>
118
- </html>
@@ -1,25 +0,0 @@
1
- document.addEventListener('DOMContentLoaded', function() {
2
- const collapsibleSections = document.querySelectorAll('.exception-detail, .raw-model-response');
3
-
4
- collapsibleSections.forEach(section => {
5
- const header = section.querySelector('.exception-header, .response-header');
6
- const content = section.querySelector('.exception-content, .response-content');
7
- const toggleBtn = section.querySelector('.toggle-btn');
8
-
9
- header.addEventListener('click', function() {
10
- content.classList.toggle('show');
11
- toggleBtn.classList.toggle('rotated');
12
- });
13
- });
14
-
15
- });
16
-
17
- function copyCode() {
18
- const textarea = document.getElementById('codeToCopy');
19
- textarea.select();
20
- textarea.setSelectionRange(0, 99999); // For mobile devices
21
- document.execCommand("copy");
22
-
23
- // Optionally, you can display an alert or change the button text to indicate success
24
- alert("Code copied to clipboard!");
25
- }