edsl 0.1.15__py3-none-any.whl → 0.1.40__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edsl/Base.py +348 -38
- edsl/BaseDiff.py +260 -0
- edsl/TemplateLoader.py +24 -0
- edsl/__init__.py +45 -10
- edsl/__version__.py +1 -1
- edsl/agents/Agent.py +842 -144
- edsl/agents/AgentList.py +521 -25
- edsl/agents/Invigilator.py +250 -374
- edsl/agents/InvigilatorBase.py +257 -0
- edsl/agents/PromptConstructor.py +272 -0
- edsl/agents/QuestionInstructionPromptBuilder.py +128 -0
- edsl/agents/QuestionTemplateReplacementsBuilder.py +137 -0
- edsl/agents/descriptors.py +43 -13
- edsl/agents/prompt_helpers.py +129 -0
- edsl/agents/question_option_processor.py +172 -0
- edsl/auto/AutoStudy.py +130 -0
- edsl/auto/StageBase.py +243 -0
- edsl/auto/StageGenerateSurvey.py +178 -0
- edsl/auto/StageLabelQuestions.py +125 -0
- edsl/auto/StagePersona.py +61 -0
- edsl/auto/StagePersonaDimensionValueRanges.py +88 -0
- edsl/auto/StagePersonaDimensionValues.py +74 -0
- edsl/auto/StagePersonaDimensions.py +69 -0
- edsl/auto/StageQuestions.py +74 -0
- edsl/auto/SurveyCreatorPipeline.py +21 -0
- edsl/auto/utilities.py +218 -0
- edsl/base/Base.py +279 -0
- edsl/config.py +115 -113
- edsl/conversation/Conversation.py +290 -0
- edsl/conversation/car_buying.py +59 -0
- edsl/conversation/chips.py +95 -0
- edsl/conversation/mug_negotiation.py +81 -0
- edsl/conversation/next_speaker_utilities.py +93 -0
- edsl/coop/CoopFunctionsMixin.py +15 -0
- edsl/coop/ExpectedParrotKeyHandler.py +125 -0
- edsl/coop/PriceFetcher.py +54 -0
- edsl/coop/__init__.py +1 -0
- edsl/coop/coop.py +1029 -134
- edsl/coop/utils.py +131 -0
- edsl/data/Cache.py +560 -89
- edsl/data/CacheEntry.py +230 -0
- edsl/data/CacheHandler.py +168 -0
- edsl/data/RemoteCacheSync.py +186 -0
- edsl/data/SQLiteDict.py +292 -0
- edsl/data/__init__.py +5 -3
- edsl/data/orm.py +6 -33
- edsl/data_transfer_models.py +74 -27
- edsl/enums.py +165 -8
- edsl/exceptions/BaseException.py +21 -0
- edsl/exceptions/__init__.py +52 -46
- edsl/exceptions/agents.py +33 -15
- edsl/exceptions/cache.py +5 -0
- edsl/exceptions/coop.py +8 -0
- edsl/exceptions/general.py +34 -0
- edsl/exceptions/inference_services.py +5 -0
- edsl/exceptions/jobs.py +15 -0
- edsl/exceptions/language_models.py +46 -1
- edsl/exceptions/questions.py +80 -5
- edsl/exceptions/results.py +16 -5
- edsl/exceptions/scenarios.py +29 -0
- edsl/exceptions/surveys.py +13 -10
- edsl/inference_services/AnthropicService.py +106 -0
- edsl/inference_services/AvailableModelCacheHandler.py +184 -0
- edsl/inference_services/AvailableModelFetcher.py +215 -0
- edsl/inference_services/AwsBedrock.py +118 -0
- edsl/inference_services/AzureAI.py +215 -0
- edsl/inference_services/DeepInfraService.py +18 -0
- edsl/inference_services/GoogleService.py +143 -0
- edsl/inference_services/GroqService.py +20 -0
- edsl/inference_services/InferenceServiceABC.py +80 -0
- edsl/inference_services/InferenceServicesCollection.py +138 -0
- edsl/inference_services/MistralAIService.py +120 -0
- edsl/inference_services/OllamaService.py +18 -0
- edsl/inference_services/OpenAIService.py +236 -0
- edsl/inference_services/PerplexityService.py +160 -0
- edsl/inference_services/ServiceAvailability.py +135 -0
- edsl/inference_services/TestService.py +90 -0
- edsl/inference_services/TogetherAIService.py +172 -0
- edsl/inference_services/data_structures.py +134 -0
- edsl/inference_services/models_available_cache.py +118 -0
- edsl/inference_services/rate_limits_cache.py +25 -0
- edsl/inference_services/registry.py +41 -0
- edsl/inference_services/write_available.py +10 -0
- edsl/jobs/AnswerQuestionFunctionConstructor.py +223 -0
- edsl/jobs/Answers.py +21 -20
- edsl/jobs/FetchInvigilator.py +47 -0
- edsl/jobs/InterviewTaskManager.py +98 -0
- edsl/jobs/InterviewsConstructor.py +50 -0
- edsl/jobs/Jobs.py +684 -206
- edsl/jobs/JobsChecks.py +172 -0
- edsl/jobs/JobsComponentConstructor.py +189 -0
- edsl/jobs/JobsPrompts.py +270 -0
- edsl/jobs/JobsRemoteInferenceHandler.py +311 -0
- edsl/jobs/JobsRemoteInferenceLogger.py +239 -0
- edsl/jobs/RequestTokenEstimator.py +30 -0
- edsl/jobs/async_interview_runner.py +138 -0
- edsl/jobs/buckets/BucketCollection.py +104 -0
- edsl/jobs/buckets/ModelBuckets.py +65 -0
- edsl/jobs/buckets/TokenBucket.py +283 -0
- edsl/jobs/buckets/TokenBucketAPI.py +211 -0
- edsl/jobs/buckets/TokenBucketClient.py +191 -0
- edsl/jobs/check_survey_scenario_compatibility.py +85 -0
- edsl/jobs/data_structures.py +120 -0
- edsl/jobs/decorators.py +35 -0
- edsl/jobs/interviews/Interview.py +392 -0
- edsl/jobs/interviews/InterviewExceptionCollection.py +99 -0
- edsl/jobs/interviews/InterviewExceptionEntry.py +186 -0
- edsl/jobs/interviews/InterviewStatistic.py +63 -0
- edsl/jobs/interviews/InterviewStatisticsCollection.py +25 -0
- edsl/jobs/interviews/InterviewStatusDictionary.py +78 -0
- edsl/jobs/interviews/InterviewStatusLog.py +92 -0
- edsl/jobs/interviews/ReportErrors.py +66 -0
- edsl/jobs/interviews/interview_status_enum.py +9 -0
- edsl/jobs/jobs_status_enums.py +9 -0
- edsl/jobs/loggers/HTMLTableJobLogger.py +304 -0
- edsl/jobs/results_exceptions_handler.py +98 -0
- edsl/jobs/runners/JobsRunnerAsyncio.py +151 -110
- edsl/jobs/runners/JobsRunnerStatus.py +298 -0
- edsl/jobs/tasks/QuestionTaskCreator.py +244 -0
- edsl/jobs/tasks/TaskCreators.py +64 -0
- edsl/jobs/tasks/TaskHistory.py +470 -0
- edsl/jobs/tasks/TaskStatusLog.py +23 -0
- edsl/jobs/tasks/task_status_enum.py +161 -0
- edsl/jobs/tokens/InterviewTokenUsage.py +27 -0
- edsl/jobs/tokens/TokenUsage.py +34 -0
- edsl/language_models/ComputeCost.py +63 -0
- edsl/language_models/LanguageModel.py +507 -386
- edsl/language_models/ModelList.py +164 -0
- edsl/language_models/PriceManager.py +127 -0
- edsl/language_models/RawResponseHandler.py +106 -0
- edsl/language_models/RegisterLanguageModelsMeta.py +184 -0
- edsl/language_models/__init__.py +1 -8
- edsl/language_models/fake_openai_call.py +15 -0
- edsl/language_models/fake_openai_service.py +61 -0
- edsl/language_models/key_management/KeyLookup.py +63 -0
- edsl/language_models/key_management/KeyLookupBuilder.py +273 -0
- edsl/language_models/key_management/KeyLookupCollection.py +38 -0
- edsl/language_models/key_management/__init__.py +0 -0
- edsl/language_models/key_management/models.py +131 -0
- edsl/language_models/model.py +256 -0
- edsl/language_models/repair.py +109 -41
- edsl/language_models/utilities.py +65 -0
- edsl/notebooks/Notebook.py +263 -0
- edsl/notebooks/NotebookToLaTeX.py +142 -0
- edsl/notebooks/__init__.py +1 -0
- edsl/prompts/Prompt.py +222 -93
- edsl/prompts/__init__.py +1 -1
- edsl/questions/ExceptionExplainer.py +77 -0
- edsl/questions/HTMLQuestion.py +103 -0
- edsl/questions/QuestionBase.py +518 -0
- edsl/questions/QuestionBasePromptsMixin.py +221 -0
- edsl/questions/QuestionBudget.py +164 -67
- edsl/questions/QuestionCheckBox.py +281 -62
- edsl/questions/QuestionDict.py +343 -0
- edsl/questions/QuestionExtract.py +136 -50
- edsl/questions/QuestionFreeText.py +79 -55
- edsl/questions/QuestionFunctional.py +138 -41
- edsl/questions/QuestionList.py +184 -57
- edsl/questions/QuestionMatrix.py +265 -0
- edsl/questions/QuestionMultipleChoice.py +293 -69
- edsl/questions/QuestionNumerical.py +109 -56
- edsl/questions/QuestionRank.py +244 -49
- edsl/questions/Quick.py +41 -0
- edsl/questions/SimpleAskMixin.py +74 -0
- edsl/questions/__init__.py +9 -6
- edsl/questions/{AnswerValidatorMixin.py → answer_validator_mixin.py} +153 -38
- edsl/questions/compose_questions.py +13 -7
- edsl/questions/data_structures.py +20 -0
- edsl/questions/decorators.py +21 -0
- edsl/questions/derived/QuestionLikertFive.py +28 -26
- edsl/questions/derived/QuestionLinearScale.py +41 -28
- edsl/questions/derived/QuestionTopK.py +34 -26
- edsl/questions/derived/QuestionYesNo.py +40 -27
- edsl/questions/descriptors.py +228 -74
- edsl/questions/loop_processor.py +149 -0
- edsl/questions/prompt_templates/question_budget.jinja +13 -0
- edsl/questions/prompt_templates/question_checkbox.jinja +32 -0
- edsl/questions/prompt_templates/question_extract.jinja +11 -0
- edsl/questions/prompt_templates/question_free_text.jinja +3 -0
- edsl/questions/prompt_templates/question_linear_scale.jinja +11 -0
- edsl/questions/prompt_templates/question_list.jinja +17 -0
- edsl/questions/prompt_templates/question_multiple_choice.jinja +33 -0
- edsl/questions/prompt_templates/question_numerical.jinja +37 -0
- edsl/questions/question_base_gen_mixin.py +168 -0
- edsl/questions/question_registry.py +130 -46
- edsl/questions/register_questions_meta.py +71 -0
- edsl/questions/response_validator_abc.py +188 -0
- edsl/questions/response_validator_factory.py +34 -0
- edsl/questions/settings.py +5 -2
- edsl/questions/templates/__init__.py +0 -0
- edsl/questions/templates/budget/__init__.py +0 -0
- edsl/questions/templates/budget/answering_instructions.jinja +7 -0
- edsl/questions/templates/budget/question_presentation.jinja +7 -0
- edsl/questions/templates/checkbox/__init__.py +0 -0
- edsl/questions/templates/checkbox/answering_instructions.jinja +10 -0
- edsl/questions/templates/checkbox/question_presentation.jinja +22 -0
- edsl/questions/templates/dict/__init__.py +0 -0
- edsl/questions/templates/dict/answering_instructions.jinja +21 -0
- edsl/questions/templates/dict/question_presentation.jinja +1 -0
- edsl/questions/templates/extract/__init__.py +0 -0
- edsl/questions/templates/extract/answering_instructions.jinja +7 -0
- edsl/questions/templates/extract/question_presentation.jinja +1 -0
- edsl/questions/templates/free_text/__init__.py +0 -0
- edsl/questions/templates/free_text/answering_instructions.jinja +0 -0
- edsl/questions/templates/free_text/question_presentation.jinja +1 -0
- edsl/questions/templates/likert_five/__init__.py +0 -0
- edsl/questions/templates/likert_five/answering_instructions.jinja +10 -0
- edsl/questions/templates/likert_five/question_presentation.jinja +12 -0
- edsl/questions/templates/linear_scale/__init__.py +0 -0
- edsl/questions/templates/linear_scale/answering_instructions.jinja +5 -0
- edsl/questions/templates/linear_scale/question_presentation.jinja +5 -0
- edsl/questions/templates/list/__init__.py +0 -0
- edsl/questions/templates/list/answering_instructions.jinja +4 -0
- edsl/questions/templates/list/question_presentation.jinja +5 -0
- edsl/questions/templates/matrix/__init__.py +1 -0
- edsl/questions/templates/matrix/answering_instructions.jinja +5 -0
- edsl/questions/templates/matrix/question_presentation.jinja +20 -0
- edsl/questions/templates/multiple_choice/__init__.py +0 -0
- edsl/questions/templates/multiple_choice/answering_instructions.jinja +9 -0
- edsl/questions/templates/multiple_choice/html.jinja +0 -0
- edsl/questions/templates/multiple_choice/question_presentation.jinja +12 -0
- edsl/questions/templates/numerical/__init__.py +0 -0
- edsl/questions/templates/numerical/answering_instructions.jinja +7 -0
- edsl/questions/templates/numerical/question_presentation.jinja +7 -0
- edsl/questions/templates/rank/__init__.py +0 -0
- edsl/questions/templates/rank/answering_instructions.jinja +11 -0
- edsl/questions/templates/rank/question_presentation.jinja +15 -0
- edsl/questions/templates/top_k/__init__.py +0 -0
- edsl/questions/templates/top_k/answering_instructions.jinja +8 -0
- edsl/questions/templates/top_k/question_presentation.jinja +22 -0
- edsl/questions/templates/yes_no/__init__.py +0 -0
- edsl/questions/templates/yes_no/answering_instructions.jinja +6 -0
- edsl/questions/templates/yes_no/question_presentation.jinja +12 -0
- edsl/results/CSSParameterizer.py +108 -0
- edsl/results/Dataset.py +550 -19
- edsl/results/DatasetExportMixin.py +594 -0
- edsl/results/DatasetTree.py +295 -0
- edsl/results/MarkdownToDocx.py +122 -0
- edsl/results/MarkdownToPDF.py +111 -0
- edsl/results/Result.py +477 -173
- edsl/results/Results.py +987 -269
- edsl/results/ResultsExportMixin.py +28 -125
- edsl/results/ResultsGGMixin.py +83 -15
- edsl/results/TableDisplay.py +125 -0
- edsl/results/TextEditor.py +50 -0
- edsl/results/__init__.py +1 -1
- edsl/results/file_exports.py +252 -0
- edsl/results/results_fetch_mixin.py +33 -0
- edsl/results/results_selector.py +145 -0
- edsl/results/results_tools_mixin.py +98 -0
- edsl/results/smart_objects.py +96 -0
- edsl/results/table_data_class.py +12 -0
- edsl/results/table_display.css +78 -0
- edsl/results/table_renderers.py +118 -0
- edsl/results/tree_explore.py +115 -0
- edsl/scenarios/ConstructDownloadLink.py +109 -0
- edsl/scenarios/DocumentChunker.py +102 -0
- edsl/scenarios/DocxScenario.py +16 -0
- edsl/scenarios/FileStore.py +543 -0
- edsl/scenarios/PdfExtractor.py +40 -0
- edsl/scenarios/Scenario.py +431 -62
- edsl/scenarios/ScenarioHtmlMixin.py +65 -0
- edsl/scenarios/ScenarioList.py +1415 -45
- edsl/scenarios/ScenarioListExportMixin.py +45 -0
- edsl/scenarios/ScenarioListPdfMixin.py +239 -0
- edsl/scenarios/__init__.py +2 -0
- edsl/scenarios/directory_scanner.py +96 -0
- edsl/scenarios/file_methods.py +85 -0
- edsl/scenarios/handlers/__init__.py +13 -0
- edsl/scenarios/handlers/csv.py +49 -0
- edsl/scenarios/handlers/docx.py +76 -0
- edsl/scenarios/handlers/html.py +37 -0
- edsl/scenarios/handlers/json.py +111 -0
- edsl/scenarios/handlers/latex.py +5 -0
- edsl/scenarios/handlers/md.py +51 -0
- edsl/scenarios/handlers/pdf.py +68 -0
- edsl/scenarios/handlers/png.py +39 -0
- edsl/scenarios/handlers/pptx.py +105 -0
- edsl/scenarios/handlers/py.py +294 -0
- edsl/scenarios/handlers/sql.py +313 -0
- edsl/scenarios/handlers/sqlite.py +149 -0
- edsl/scenarios/handlers/txt.py +33 -0
- edsl/scenarios/scenario_join.py +131 -0
- edsl/scenarios/scenario_selector.py +156 -0
- edsl/shared.py +1 -0
- edsl/study/ObjectEntry.py +173 -0
- edsl/study/ProofOfWork.py +113 -0
- edsl/study/SnapShot.py +80 -0
- edsl/study/Study.py +521 -0
- edsl/study/__init__.py +4 -0
- edsl/surveys/ConstructDAG.py +92 -0
- edsl/surveys/DAG.py +92 -11
- edsl/surveys/EditSurvey.py +221 -0
- edsl/surveys/InstructionHandler.py +100 -0
- edsl/surveys/Memory.py +9 -4
- edsl/surveys/MemoryManagement.py +72 -0
- edsl/surveys/MemoryPlan.py +156 -35
- edsl/surveys/Rule.py +221 -74
- edsl/surveys/RuleCollection.py +241 -61
- edsl/surveys/RuleManager.py +172 -0
- edsl/surveys/Simulator.py +75 -0
- edsl/surveys/Survey.py +1079 -339
- edsl/surveys/SurveyCSS.py +273 -0
- edsl/surveys/SurveyExportMixin.py +235 -40
- edsl/surveys/SurveyFlowVisualization.py +181 -0
- edsl/surveys/SurveyQualtricsImport.py +284 -0
- edsl/surveys/SurveyToApp.py +141 -0
- edsl/surveys/__init__.py +4 -2
- edsl/surveys/base.py +19 -3
- edsl/surveys/descriptors.py +17 -6
- edsl/surveys/instructions/ChangeInstruction.py +48 -0
- edsl/surveys/instructions/Instruction.py +56 -0
- edsl/surveys/instructions/InstructionCollection.py +82 -0
- edsl/surveys/instructions/__init__.py +0 -0
- edsl/templates/error_reporting/base.html +24 -0
- edsl/templates/error_reporting/exceptions_by_model.html +35 -0
- edsl/templates/error_reporting/exceptions_by_question_name.html +17 -0
- edsl/templates/error_reporting/exceptions_by_type.html +17 -0
- edsl/templates/error_reporting/interview_details.html +116 -0
- edsl/templates/error_reporting/interviews.html +19 -0
- edsl/templates/error_reporting/overview.html +5 -0
- edsl/templates/error_reporting/performance_plot.html +2 -0
- edsl/templates/error_reporting/report.css +74 -0
- edsl/templates/error_reporting/report.html +118 -0
- edsl/templates/error_reporting/report.js +25 -0
- edsl/tools/__init__.py +1 -0
- edsl/tools/clusters.py +192 -0
- edsl/tools/embeddings.py +27 -0
- edsl/tools/embeddings_plotting.py +118 -0
- edsl/tools/plotting.py +112 -0
- edsl/tools/summarize.py +18 -0
- edsl/utilities/PrettyList.py +56 -0
- edsl/utilities/SystemInfo.py +5 -0
- edsl/utilities/__init__.py +21 -20
- edsl/utilities/ast_utilities.py +3 -0
- edsl/utilities/data/Registry.py +2 -0
- edsl/utilities/decorators.py +41 -0
- edsl/utilities/gcp_bucket/__init__.py +0 -0
- edsl/utilities/gcp_bucket/cloud_storage.py +96 -0
- edsl/utilities/interface.py +310 -60
- edsl/utilities/is_notebook.py +18 -0
- edsl/utilities/is_valid_variable_name.py +11 -0
- edsl/utilities/naming_utilities.py +263 -0
- edsl/utilities/remove_edsl_version.py +24 -0
- edsl/utilities/repair_functions.py +28 -0
- edsl/utilities/restricted_python.py +70 -0
- edsl/utilities/utilities.py +203 -13
- edsl-0.1.40.dist-info/METADATA +111 -0
- edsl-0.1.40.dist-info/RECORD +362 -0
- {edsl-0.1.15.dist-info → edsl-0.1.40.dist-info}/WHEEL +1 -1
- edsl/agents/AgentListExportMixin.py +0 -24
- edsl/coop/old.py +0 -31
- edsl/data/Database.py +0 -141
- edsl/data/crud.py +0 -121
- edsl/jobs/Interview.py +0 -435
- edsl/jobs/JobsRunner.py +0 -63
- edsl/jobs/JobsRunnerStatusMixin.py +0 -115
- edsl/jobs/base.py +0 -47
- edsl/jobs/buckets.py +0 -178
- edsl/jobs/runners/JobsRunnerDryRun.py +0 -19
- edsl/jobs/runners/JobsRunnerStreaming.py +0 -54
- edsl/jobs/task_management.py +0 -215
- edsl/jobs/token_tracking.py +0 -78
- edsl/language_models/DeepInfra.py +0 -69
- edsl/language_models/OpenAI.py +0 -98
- edsl/language_models/model_interfaces/GeminiPro.py +0 -66
- edsl/language_models/model_interfaces/LanguageModelOpenAIFour.py +0 -8
- edsl/language_models/model_interfaces/LanguageModelOpenAIThreeFiveTurbo.py +0 -8
- edsl/language_models/model_interfaces/LlamaTwo13B.py +0 -21
- edsl/language_models/model_interfaces/LlamaTwo70B.py +0 -21
- edsl/language_models/model_interfaces/Mixtral8x7B.py +0 -24
- edsl/language_models/registry.py +0 -81
- edsl/language_models/schemas.py +0 -15
- edsl/language_models/unused/ReplicateBase.py +0 -83
- edsl/prompts/QuestionInstructionsBase.py +0 -6
- edsl/prompts/library/agent_instructions.py +0 -29
- edsl/prompts/library/agent_persona.py +0 -17
- edsl/prompts/library/question_budget.py +0 -26
- edsl/prompts/library/question_checkbox.py +0 -32
- edsl/prompts/library/question_extract.py +0 -19
- edsl/prompts/library/question_freetext.py +0 -14
- edsl/prompts/library/question_linear_scale.py +0 -20
- edsl/prompts/library/question_list.py +0 -22
- edsl/prompts/library/question_multiple_choice.py +0 -44
- edsl/prompts/library/question_numerical.py +0 -31
- edsl/prompts/library/question_rank.py +0 -21
- edsl/prompts/prompt_config.py +0 -33
- edsl/prompts/registry.py +0 -185
- edsl/questions/Question.py +0 -240
- edsl/report/InputOutputDataTypes.py +0 -134
- edsl/report/RegressionMixin.py +0 -28
- edsl/report/ReportOutputs.py +0 -1228
- edsl/report/ResultsFetchMixin.py +0 -106
- edsl/report/ResultsOutputMixin.py +0 -14
- edsl/report/demo.ipynb +0 -645
- edsl/results/ResultsDBMixin.py +0 -184
- edsl/surveys/SurveyFlowVisualizationMixin.py +0 -92
- edsl/trackers/Tracker.py +0 -91
- edsl/trackers/TrackerAPI.py +0 -196
- edsl/trackers/TrackerTasks.py +0 -70
- edsl/utilities/pastebin.py +0 -141
- edsl-0.1.15.dist-info/METADATA +0 -69
- edsl-0.1.15.dist-info/RECORD +0 -142
- /edsl/{language_models/model_interfaces → inference_services}/__init__.py +0 -0
- /edsl/{report/__init__.py → jobs/runners/JobsRunnerStatusData.py} +0 -0
- /edsl/{trackers/__init__.py → language_models/ServiceDataSources.py} +0 -0
- {edsl-0.1.15.dist-info → edsl-0.1.40.dist-info}/LICENSE +0 -0
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html lang="en">
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
6
|
+
<title>Exception Details</title>
|
|
7
|
+
<style>
|
|
8
|
+
{{ css }}
|
|
9
|
+
</style>
|
|
10
|
+
</head>
|
|
11
|
+
<body>
|
|
12
|
+
<h1>Overview</h1>
|
|
13
|
+
<p>There were {{ interviews|length }} total interviews. The number of interviews with exceptions was {{ num_exceptions }}.</p>
|
|
14
|
+
<p>The models used were: {{ models_used }}.</p>
|
|
15
|
+
<p>For documentation on dealing with exceptions on Expected Parrot,
|
|
16
|
+
see <a href="https://docs.expectedparrot.com/en/latest/exceptions.html">here</a>.</p>
|
|
17
|
+
|
|
18
|
+
<h2>Exceptions by Type</h2>
|
|
19
|
+
<table>
|
|
20
|
+
<thead>
|
|
21
|
+
<tr>
|
|
22
|
+
<th>Exception Type</th>
|
|
23
|
+
<th>Number</th>
|
|
24
|
+
</tr>
|
|
25
|
+
</thead>
|
|
26
|
+
<tbody>
|
|
27
|
+
{% for exception_type, exceptions in exceptions_by_type.items() %}
|
|
28
|
+
<tr>
|
|
29
|
+
<td>{{ exception_type }}</td>
|
|
30
|
+
<td>{{ exceptions }}</td>
|
|
31
|
+
</tr>
|
|
32
|
+
{% endfor %}
|
|
33
|
+
</tbody>
|
|
34
|
+
</table>
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
<h2>Exceptions by Model</h2>
|
|
38
|
+
<table>
|
|
39
|
+
<thead>
|
|
40
|
+
<tr>
|
|
41
|
+
<th>Model</th>
|
|
42
|
+
<th>Number</th>
|
|
43
|
+
</tr>
|
|
44
|
+
</thead>
|
|
45
|
+
<tbody>
|
|
46
|
+
{% for model, exceptions in exceptions_by_model.items() %}
|
|
47
|
+
<tr>
|
|
48
|
+
<td>{{ model }}</td>
|
|
49
|
+
<td>{{ exceptions }}</td>
|
|
50
|
+
</tr>
|
|
51
|
+
{% endfor %}
|
|
52
|
+
</tbody>
|
|
53
|
+
</table>
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
<h2>Exceptions by Question Name</h2>
|
|
57
|
+
<table>
|
|
58
|
+
<thead>
|
|
59
|
+
<tr>
|
|
60
|
+
<th>Question Name</th>
|
|
61
|
+
<th>Number of Exceptions</th>
|
|
62
|
+
</tr>
|
|
63
|
+
</thead>
|
|
64
|
+
<tbody>
|
|
65
|
+
{% for question_name, exception_count in exceptions_by_question_name.items() %}
|
|
66
|
+
<tr>
|
|
67
|
+
<td>{{ question_name }}</td>
|
|
68
|
+
<td>{{ exception_count }}</td>
|
|
69
|
+
</tr>
|
|
70
|
+
{% endfor %}
|
|
71
|
+
</tbody>
|
|
72
|
+
</table>
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
{% for index, interview in interviews.items() %}
|
|
76
|
+
{% if interview.exceptions != {} %}
|
|
77
|
+
<div class="interview">Interview: {{ index }} </div>
|
|
78
|
+
<h1>Failing questions</h1>
|
|
79
|
+
{% endif %}
|
|
80
|
+
{% for question, exceptions in interview.exceptions.items() %}
|
|
81
|
+
<div class="question">question_name: {{ question }}</div>
|
|
82
|
+
|
|
83
|
+
<h2>Question</h2>
|
|
84
|
+
<div class="question-detail">
|
|
85
|
+
{{ interview.survey._get_question_by_name(question).html() }}
|
|
86
|
+
</div>
|
|
87
|
+
|
|
88
|
+
<h2>Scenario</h2>
|
|
89
|
+
<div class="scenario">
|
|
90
|
+
{{ interview.scenario._repr_html_() }}
|
|
91
|
+
</div>
|
|
92
|
+
|
|
93
|
+
<h2>Agent</h2>
|
|
94
|
+
<div class="agent">
|
|
95
|
+
{{ interview.agent._repr_html_() }}
|
|
96
|
+
</div>
|
|
97
|
+
|
|
98
|
+
<h2>Model</h2>
|
|
99
|
+
<div class="model">
|
|
100
|
+
{{ interview.model._repr_html_() }}
|
|
101
|
+
</div>
|
|
102
|
+
|
|
103
|
+
<h2>Exception details</h2>
|
|
104
|
+
|
|
105
|
+
{% for exception_message in exceptions %}
|
|
106
|
+
<div class="exception-detail">
|
|
107
|
+
<div class="exception-exception">Exception: {{ exception_message.exception }}</div>
|
|
108
|
+
<div class="exception-time">Time: {{ exception_message.time }}</div>
|
|
109
|
+
<div class="exception-traceback">Traceback: <pre>{{ exception_message.traceback }} </pre></div>
|
|
110
|
+
</div>
|
|
111
|
+
{% endfor %}
|
|
112
|
+
{% endfor %}
|
|
113
|
+
{% endfor %}
|
|
114
|
+
|
|
115
|
+
<h1>Performance Plot</h1>
|
|
116
|
+
{{ performance_plot_html }}
|
|
117
|
+
</body>
|
|
118
|
+
</html>
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
document.addEventListener('DOMContentLoaded', function() {
|
|
2
|
+
const collapsibleSections = document.querySelectorAll('.exception-detail, .raw-model-response');
|
|
3
|
+
|
|
4
|
+
collapsibleSections.forEach(section => {
|
|
5
|
+
const header = section.querySelector('.exception-header, .response-header');
|
|
6
|
+
const content = section.querySelector('.exception-content, .response-content');
|
|
7
|
+
const toggleBtn = section.querySelector('.toggle-btn');
|
|
8
|
+
|
|
9
|
+
header.addEventListener('click', function() {
|
|
10
|
+
content.classList.toggle('show');
|
|
11
|
+
toggleBtn.classList.toggle('rotated');
|
|
12
|
+
});
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
function copyCode() {
|
|
18
|
+
const textarea = document.getElementById('codeToCopy');
|
|
19
|
+
textarea.select();
|
|
20
|
+
textarea.setSelectionRange(0, 99999); // For mobile devices
|
|
21
|
+
document.execCommand("copy");
|
|
22
|
+
|
|
23
|
+
// Optionally, you can display an alert or change the button text to indicate success
|
|
24
|
+
alert("Code copied to clipboard!");
|
|
25
|
+
}
|
edsl/tools/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from edsl.tools.plotting import barchart
|
edsl/tools/clusters.py
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import numpy as np
|
|
3
|
+
from sklearn.cluster import KMeans
|
|
4
|
+
from sklearn.manifold import TSNE
|
|
5
|
+
from sklearn.decomposition import PCA
|
|
6
|
+
from IPython.display import display_html
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def compute_tsne(embeddings, cluster_labels, text_labels):
|
|
10
|
+
"""
|
|
11
|
+
Compute t-SNE on embedding vectors.
|
|
12
|
+
|
|
13
|
+
Parameters:
|
|
14
|
+
embeddings (np.ndarray): The embedding vectors.
|
|
15
|
+
cluster_labels (np.ndarray): Cluster labels for each embedding.
|
|
16
|
+
text_labels (list): Text labels for each embedding.
|
|
17
|
+
|
|
18
|
+
Returns:
|
|
19
|
+
list: List of dictionaries with x, y coordinates, cluster labels, and text labels.
|
|
20
|
+
"""
|
|
21
|
+
tsne = TSNE(n_components=2, random_state=42)
|
|
22
|
+
tsne_results = tsne.fit_transform(embeddings)
|
|
23
|
+
data = [
|
|
24
|
+
{
|
|
25
|
+
"x": float(tsne_results[i, 0]),
|
|
26
|
+
"y": float(tsne_results[i, 1]),
|
|
27
|
+
"cluster_label": str(cluster_labels[i]),
|
|
28
|
+
"text_label": text_labels[i],
|
|
29
|
+
}
|
|
30
|
+
for i in range(len(cluster_labels))
|
|
31
|
+
]
|
|
32
|
+
return data
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def compute_pca(embeddings, cluster_labels, text_labels):
|
|
36
|
+
"""
|
|
37
|
+
Compute PCA on embedding vectors.
|
|
38
|
+
|
|
39
|
+
Parameters:
|
|
40
|
+
embeddings (np.ndarray): The embedding vectors.
|
|
41
|
+
cluster_labels (np.ndarray): Cluster labels for each embedding.
|
|
42
|
+
text_labels (list): Text labels for each embedding.
|
|
43
|
+
|
|
44
|
+
Returns:
|
|
45
|
+
list: List of dictionaries with x, y coordinates, cluster labels, and text labels.
|
|
46
|
+
"""
|
|
47
|
+
pca = PCA(n_components=2)
|
|
48
|
+
pca_results = pca.fit_transform(embeddings)
|
|
49
|
+
data = [
|
|
50
|
+
{
|
|
51
|
+
"x": float(pca_results[i, 0]),
|
|
52
|
+
"y": float(pca_results[i, 1]),
|
|
53
|
+
"cluster_label": str(cluster_labels[i]),
|
|
54
|
+
"text_label": text_labels[i],
|
|
55
|
+
}
|
|
56
|
+
for i in range(len(cluster_labels))
|
|
57
|
+
]
|
|
58
|
+
return data
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def plot(embeddings, text_labels, n_clusters=5, method="tsne"):
|
|
62
|
+
"""
|
|
63
|
+
Perform k-means clustering and plot results in a Jupyter notebook using D3.js.
|
|
64
|
+
|
|
65
|
+
Parameters:
|
|
66
|
+
embeddings (np.ndarray): The embedding vectors.
|
|
67
|
+
text_labels (list): Text labels for each embedding.
|
|
68
|
+
n_clusters (int): The number of clusters to form.
|
|
69
|
+
method (str): The dimensionality reduction method to use ('tsne' or 'pca').
|
|
70
|
+
"""
|
|
71
|
+
# Perform k-means clustering
|
|
72
|
+
kmeans = KMeans(n_clusters=n_clusters, random_state=42)
|
|
73
|
+
cluster_labels = kmeans.fit_predict(embeddings)
|
|
74
|
+
|
|
75
|
+
# Compute dimensionality reduction
|
|
76
|
+
if method == "tsne":
|
|
77
|
+
data = compute_tsne(embeddings, cluster_labels, text_labels)
|
|
78
|
+
elif method == "pca":
|
|
79
|
+
data = compute_pca(embeddings, cluster_labels, text_labels)
|
|
80
|
+
else:
|
|
81
|
+
raise ValueError("Invalid method. Choose 'tsne' or 'pca'.")
|
|
82
|
+
|
|
83
|
+
# Convert data to JSON
|
|
84
|
+
data_json = json.dumps(data)
|
|
85
|
+
|
|
86
|
+
# HTML content with embedded data
|
|
87
|
+
html_content = f"""
|
|
88
|
+
<!DOCTYPE html>
|
|
89
|
+
<html lang="en">
|
|
90
|
+
<head>
|
|
91
|
+
<meta charset="UTF-8">
|
|
92
|
+
<title>t-SNE/PCA Plot with D3.js</title>
|
|
93
|
+
<script src="https://d3js.org/d3.v6.min.js"></script>
|
|
94
|
+
<style>
|
|
95
|
+
.tooltip {{
|
|
96
|
+
position: absolute;
|
|
97
|
+
text-align: center;
|
|
98
|
+
width: auto;
|
|
99
|
+
height: auto;
|
|
100
|
+
padding: 2px;
|
|
101
|
+
font: 12px sans-serif;
|
|
102
|
+
background: lightsteelblue;
|
|
103
|
+
border: 0px;
|
|
104
|
+
border-radius: 8px;
|
|
105
|
+
pointer-events: none;
|
|
106
|
+
}}
|
|
107
|
+
.dot {{
|
|
108
|
+
stroke: #000;
|
|
109
|
+
stroke-width: 0.5;
|
|
110
|
+
}}
|
|
111
|
+
</style>
|
|
112
|
+
</head>
|
|
113
|
+
<body>
|
|
114
|
+
<svg width="600" height="600"></svg>
|
|
115
|
+
|
|
116
|
+
<script>
|
|
117
|
+
// Embedded data
|
|
118
|
+
const data = {data_json};
|
|
119
|
+
|
|
120
|
+
const svg = d3.select("svg"),
|
|
121
|
+
width = +svg.attr("width"),
|
|
122
|
+
height = +svg.attr("height");
|
|
123
|
+
|
|
124
|
+
// Set up scales
|
|
125
|
+
const x = d3.scaleLinear()
|
|
126
|
+
.domain(d3.extent(data, d => d.x))
|
|
127
|
+
.range([0, width]);
|
|
128
|
+
|
|
129
|
+
const y = d3.scaleLinear()
|
|
130
|
+
.domain(d3.extent(data, d => d.y))
|
|
131
|
+
.range([height, 0]);
|
|
132
|
+
|
|
133
|
+
// Set up color scale
|
|
134
|
+
const color = d3.scaleOrdinal(d3.schemeCategory10);
|
|
135
|
+
|
|
136
|
+
// Create tooltip
|
|
137
|
+
const tooltip = d3.select("body").append("div")
|
|
138
|
+
.attr("class", "tooltip")
|
|
139
|
+
.style("opacity", 0);
|
|
140
|
+
|
|
141
|
+
// Create circles for each point
|
|
142
|
+
svg.selectAll("circle")
|
|
143
|
+
.data(data)
|
|
144
|
+
.enter().append("circle")
|
|
145
|
+
.attr("cx", d => x(d.x))
|
|
146
|
+
.attr("cy", d => y(d.y))
|
|
147
|
+
.attr("r", 5)
|
|
148
|
+
.attr("class", "dot")
|
|
149
|
+
.style("fill", d => color(d.cluster_label))
|
|
150
|
+
.on("mouseover", function(event, d) {{
|
|
151
|
+
tooltip.transition()
|
|
152
|
+
.duration(200)
|
|
153
|
+
.style("opacity", .9);
|
|
154
|
+
tooltip.html(d.text_label)
|
|
155
|
+
.style("left", (event.pageX + 5) + "px")
|
|
156
|
+
.style("top", (event.pageY - 28) + "px");
|
|
157
|
+
}})
|
|
158
|
+
.on("mouseout", function(d) {{
|
|
159
|
+
tooltip.transition()
|
|
160
|
+
.duration(500)
|
|
161
|
+
.style("opacity", 0);
|
|
162
|
+
}});
|
|
163
|
+
</script>
|
|
164
|
+
</body>
|
|
165
|
+
</html>
|
|
166
|
+
"""
|
|
167
|
+
|
|
168
|
+
# Write HTML content to a temporary file
|
|
169
|
+
html_file = "tsne_pca_plot.html"
|
|
170
|
+
with open(html_file, "w") as file:
|
|
171
|
+
file.write(html_content)
|
|
172
|
+
|
|
173
|
+
# Display the HTML content in an iframe within a Jupyter notebook
|
|
174
|
+
display_html(
|
|
175
|
+
f'<iframe src="{html_file}" width="600" height="600"></iframe>', raw=True
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
# Example usage
|
|
180
|
+
if __name__ == "__main__":
|
|
181
|
+
# Generate some sample data (embedding vectors)
|
|
182
|
+
np.random.seed(42)
|
|
183
|
+
embedding_vectors = np.random.rand(
|
|
184
|
+
100, 50
|
|
185
|
+
) # 100 samples with 50-dimensional embeddings
|
|
186
|
+
text_labels = [f"Text {i}" for i in range(100)] # Sample text labels
|
|
187
|
+
|
|
188
|
+
# Plot the clusters using t-SNE
|
|
189
|
+
plot(embedding_vectors, text_labels, n_clusters=5, method="tsne")
|
|
190
|
+
|
|
191
|
+
# Plot the clusters using PCA
|
|
192
|
+
plot(embedding_vectors, text_labels, n_clusters=5, method="pca")
|
edsl/tools/embeddings.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import openai
|
|
2
|
+
from dotenv import load_dotenv
|
|
3
|
+
import os
|
|
4
|
+
|
|
5
|
+
# Load environment variables from .env file
|
|
6
|
+
load_dotenv()
|
|
7
|
+
|
|
8
|
+
# Get the OpenAI API key from the environment variable
|
|
9
|
+
openai.api_key = os.getenv("OPENAI_API_KEY")
|
|
10
|
+
|
|
11
|
+
from openai import Client
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def get_embeddings(texts):
|
|
15
|
+
"""
|
|
16
|
+
Get embeddings for a list of texts using OpenAI API.
|
|
17
|
+
|
|
18
|
+
Args:
|
|
19
|
+
texts (list of str): List of strings to get embeddings for.
|
|
20
|
+
|
|
21
|
+
Returns:
|
|
22
|
+
list of list of float: List of embeddings.
|
|
23
|
+
"""
|
|
24
|
+
client = Client()
|
|
25
|
+
response = client.embeddings.create(input=texts, model="text-embedding-ada-002")
|
|
26
|
+
embeddings = [item.embedding for item in response.data]
|
|
27
|
+
return embeddings
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import numpy as np
|
|
3
|
+
from sklearn.manifold import TSNE
|
|
4
|
+
from IPython.display import display_html
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def compute_tsne(embeddings, labels):
|
|
8
|
+
embeddings_np = np.array(embeddings)
|
|
9
|
+
tsne = TSNE(n_components=2, random_state=42)
|
|
10
|
+
tsne_results = tsne.fit_transform(embeddings_np)
|
|
11
|
+
data = [
|
|
12
|
+
{
|
|
13
|
+
"x": float(tsne_results[i, 0]),
|
|
14
|
+
"y": float(tsne_results[i, 1]),
|
|
15
|
+
"label": labels[i],
|
|
16
|
+
}
|
|
17
|
+
for i in range(len(labels))
|
|
18
|
+
]
|
|
19
|
+
return data
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def plot_tsne_in_notebook(embeddings, labels):
|
|
23
|
+
# Compute t-SNE
|
|
24
|
+
data = compute_tsne(embeddings, labels)
|
|
25
|
+
|
|
26
|
+
# Convert data to JSON
|
|
27
|
+
data_json = json.dumps(data)
|
|
28
|
+
|
|
29
|
+
# HTML content with embedded data
|
|
30
|
+
html_content = f"""
|
|
31
|
+
<!DOCTYPE html>
|
|
32
|
+
<html lang="en">
|
|
33
|
+
<head>
|
|
34
|
+
<meta charset="UTF-8">
|
|
35
|
+
<title>t-SNE Plot with D3.js</title>
|
|
36
|
+
<script src="https://d3js.org/d3.v6.min.js"></script>
|
|
37
|
+
<style>
|
|
38
|
+
.tooltip {{
|
|
39
|
+
position: absolute;
|
|
40
|
+
text-align: center;
|
|
41
|
+
width: 80px;
|
|
42
|
+
height: 28px;
|
|
43
|
+
padding: 2px;
|
|
44
|
+
font: 12px sans-serif;
|
|
45
|
+
background: lightsteelblue;
|
|
46
|
+
border: 0px;
|
|
47
|
+
border-radius: 8px;
|
|
48
|
+
pointer-events: none;
|
|
49
|
+
}}
|
|
50
|
+
</style>
|
|
51
|
+
</head>
|
|
52
|
+
<body>
|
|
53
|
+
<svg width="600" height="600"></svg>
|
|
54
|
+
|
|
55
|
+
<script>
|
|
56
|
+
// Embedded data
|
|
57
|
+
const data = {data_json};
|
|
58
|
+
|
|
59
|
+
const svg = d3.select("svg"),
|
|
60
|
+
width = +svg.attr("width"),
|
|
61
|
+
height = +svg.attr("height");
|
|
62
|
+
|
|
63
|
+
// Set up scales
|
|
64
|
+
const x = d3.scaleLinear()
|
|
65
|
+
.domain(d3.extent(data, d => d.x))
|
|
66
|
+
.range([0, width]);
|
|
67
|
+
|
|
68
|
+
const y = d3.scaleLinear()
|
|
69
|
+
.domain(d3.extent(data, d => d.y))
|
|
70
|
+
.range([height, 0]);
|
|
71
|
+
|
|
72
|
+
// Create tooltip
|
|
73
|
+
const tooltip = d3.select("body").append("div")
|
|
74
|
+
.attr("class", "tooltip")
|
|
75
|
+
.style("opacity", 0);
|
|
76
|
+
|
|
77
|
+
// Create circles for each point
|
|
78
|
+
svg.selectAll("circle")
|
|
79
|
+
.data(data)
|
|
80
|
+
.enter().append("circle")
|
|
81
|
+
.attr("cx", d => x(d.x))
|
|
82
|
+
.attr("cy", d => y(d.y))
|
|
83
|
+
.attr("r", 5)
|
|
84
|
+
.style("fill", "steelblue")
|
|
85
|
+
.on("mouseover", function(event, d) {{
|
|
86
|
+
tooltip.transition()
|
|
87
|
+
.duration(200)
|
|
88
|
+
.style("opacity", .9);
|
|
89
|
+
tooltip.html(d.label)
|
|
90
|
+
.style("left", (event.pageX + 5) + "px")
|
|
91
|
+
.style("top", (event.pageY - 28) + "px");
|
|
92
|
+
}})
|
|
93
|
+
.on("mouseout", function(d) {{
|
|
94
|
+
tooltip.transition()
|
|
95
|
+
.duration(500)
|
|
96
|
+
.style("opacity", 0);
|
|
97
|
+
}});
|
|
98
|
+
</script>
|
|
99
|
+
</body>
|
|
100
|
+
</html>
|
|
101
|
+
"""
|
|
102
|
+
|
|
103
|
+
# Write HTML content to a temporary file
|
|
104
|
+
html_file = "tsne_plot.html"
|
|
105
|
+
with open(html_file, "w") as file:
|
|
106
|
+
file.write(html_content)
|
|
107
|
+
|
|
108
|
+
# Display the HTML content in an iframe within a Jupyter notebook
|
|
109
|
+
display_html(
|
|
110
|
+
f'<iframe src="{html_file}" width="600" height="600"></iframe>', raw=True
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
# Example usage
|
|
115
|
+
if __name__ == "__main__":
|
|
116
|
+
embeddings = [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6], [0.7, 0.8, 0.9]]
|
|
117
|
+
labels = ["String 1", "String 2", "String 3"]
|
|
118
|
+
plot_tsne_in_notebook(embeddings, labels)
|
edsl/tools/plotting.py
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
from typing import Optional
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def count_query(field):
|
|
5
|
+
return f"""SELECT
|
|
6
|
+
{field},
|
|
7
|
+
COUNT(*) as number
|
|
8
|
+
FROM self
|
|
9
|
+
GROUP BY {field}
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def get_options(results, field):
|
|
14
|
+
question_type = results.survey._get_question_by_name(field).question_type
|
|
15
|
+
if question_type in ["multiple_choice", "checkbox"]:
|
|
16
|
+
return results.select(f"{field}_question_options").first()
|
|
17
|
+
else:
|
|
18
|
+
return None
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def interpret_image(path, analysis):
|
|
22
|
+
from edsl import QuestionFreeText
|
|
23
|
+
from edsl import Model
|
|
24
|
+
from edsl import Scenario
|
|
25
|
+
|
|
26
|
+
s = Scenario.from_image(path)
|
|
27
|
+
if isinstance(analysis, str):
|
|
28
|
+
plot_question_texts = [analysis]
|
|
29
|
+
elif isinstance(analysis, list):
|
|
30
|
+
plot_question_texts = analysis
|
|
31
|
+
|
|
32
|
+
scenario_list = s.replicate(len(plot_question_texts))
|
|
33
|
+
scenario_list.add_list("plot_question_text", plot_question_texts)
|
|
34
|
+
|
|
35
|
+
m = Model("gpt-4o")
|
|
36
|
+
q = QuestionFreeText(
|
|
37
|
+
question_text="{{ plot_question_text }}", question_name="interpretation"
|
|
38
|
+
)
|
|
39
|
+
results = q.by(m).by(scenario_list).run()
|
|
40
|
+
return results.select("plot_question_text", "interpretation").print(
|
|
41
|
+
format="rich",
|
|
42
|
+
pretty_labels={
|
|
43
|
+
"scenario.plot_question_text": "Question to the model",
|
|
44
|
+
"answer.interpretation": "Model answer",
|
|
45
|
+
},
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def barchart(
|
|
50
|
+
results,
|
|
51
|
+
field: str,
|
|
52
|
+
fetch_options=True,
|
|
53
|
+
xlab: Optional[str] = None,
|
|
54
|
+
ylab: Optional[str] = None,
|
|
55
|
+
analysis: Optional[str] = None,
|
|
56
|
+
format: str = "png",
|
|
57
|
+
):
|
|
58
|
+
labels = ""
|
|
59
|
+
if xlab:
|
|
60
|
+
labels += f"+ xlab('{xlab}')"
|
|
61
|
+
if ylab:
|
|
62
|
+
labels += f"+ ylab('{ylab}')"
|
|
63
|
+
|
|
64
|
+
if fetch_options:
|
|
65
|
+
factor_orders = {field: get_options(results, field)}
|
|
66
|
+
else:
|
|
67
|
+
factor_orders = None
|
|
68
|
+
|
|
69
|
+
plot = results.ggplot2(
|
|
70
|
+
f"""ggplot(data = self, aes(x = {field}, y = number)) +
|
|
71
|
+
geom_bar(stat = "identity") +
|
|
72
|
+
theme_bw() +
|
|
73
|
+
theme(axis.text.x = element_text(angle = 45, hjust = 1)) {labels}""",
|
|
74
|
+
sql=count_query(field),
|
|
75
|
+
factor_orders=factor_orders,
|
|
76
|
+
format=format,
|
|
77
|
+
filename=f"barchart_{field}.{format}",
|
|
78
|
+
)
|
|
79
|
+
if analysis:
|
|
80
|
+
interpret_image(f"barchart_{field}.{format}", analysis)
|
|
81
|
+
|
|
82
|
+
return plot
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def theme_plot(results, field, context, themes=None, progress_bar=False):
|
|
86
|
+
_, themes = results.auto_theme(
|
|
87
|
+
field=field, context=context, themes=themes, progress_bar=progress_bar
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
themes_query = f"""
|
|
91
|
+
SELECT theme, COUNT(*) AS mentions
|
|
92
|
+
FROM (
|
|
93
|
+
SELECT json_each.value AS theme
|
|
94
|
+
FROM self,
|
|
95
|
+
json_each({ field }_themes)
|
|
96
|
+
)
|
|
97
|
+
GROUP BY theme
|
|
98
|
+
HAVING theme <> 'Other'
|
|
99
|
+
ORDER BY mentions DESC
|
|
100
|
+
"""
|
|
101
|
+
themes = results.sql(themes_query, to_list=True)
|
|
102
|
+
|
|
103
|
+
(
|
|
104
|
+
results.filter(f"{field} != ''").ggplot2(
|
|
105
|
+
"""ggplot(data = self, aes(x = theme, y = mentions)) +
|
|
106
|
+
geom_bar(stat = "identity") +
|
|
107
|
+
coord_flip() +
|
|
108
|
+
theme_bw()""",
|
|
109
|
+
sql=themes_query,
|
|
110
|
+
factor_orders={"theme": [t[0] for t in themes]},
|
|
111
|
+
)
|
|
112
|
+
)
|
edsl/tools/summarize.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from edsl import QuestionList, Scenario, Model
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def summarize(texts, seed_phrase, n_bullets, n_words, models=None):
|
|
5
|
+
if models is None:
|
|
6
|
+
models = Model()
|
|
7
|
+
s = Scenario(
|
|
8
|
+
text=texts, seed_phrase=seed_phrase, n_bullets=n_bullets, n_words=n_words
|
|
9
|
+
).expand("text")
|
|
10
|
+
QuestionList(
|
|
11
|
+
question_text="""
|
|
12
|
+
I have the following TEXT EXAMPLE :
|
|
13
|
+
{{ text_example_json }}
|
|
14
|
+
Please summarize the main point of this EXAMPLE {{seed_phrase }} into {{ n_bullets }} bullet points, where
|
|
15
|
+
each bullet point is a {{ n_words }} word phrase.
|
|
16
|
+
""",
|
|
17
|
+
question_name="summarize",
|
|
18
|
+
).by(s).by(models).run()
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
from collections import UserList
|
|
2
|
+
from edsl.results.Dataset import Dataset
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class PrettyList(UserList):
|
|
6
|
+
def __init__(self, data=None, columns=None):
|
|
7
|
+
super().__init__(data)
|
|
8
|
+
self.columns = columns
|
|
9
|
+
|
|
10
|
+
def _repr_html_(self):
|
|
11
|
+
if isinstance(self[0], list) or isinstance(self[0], tuple):
|
|
12
|
+
num_cols = len(self[0])
|
|
13
|
+
else:
|
|
14
|
+
num_cols = 1
|
|
15
|
+
|
|
16
|
+
if self.columns:
|
|
17
|
+
columns = self.columns
|
|
18
|
+
else:
|
|
19
|
+
columns = list(range(num_cols))
|
|
20
|
+
|
|
21
|
+
d = {}
|
|
22
|
+
for column in columns:
|
|
23
|
+
d[column] = []
|
|
24
|
+
|
|
25
|
+
for row in self:
|
|
26
|
+
for index, column in enumerate(columns):
|
|
27
|
+
if isinstance(row, list) or isinstance(row, tuple):
|
|
28
|
+
d[column].append(row[index])
|
|
29
|
+
else:
|
|
30
|
+
d[column].append(row)
|
|
31
|
+
# raise ValueError(d)
|
|
32
|
+
return Dataset([{key: entry} for key, entry in d.items()])._repr_html_()
|
|
33
|
+
|
|
34
|
+
if num_cols > 1:
|
|
35
|
+
return (
|
|
36
|
+
"<pre><table>"
|
|
37
|
+
+ "".join(["<th>" + str(column) + "</th>" for column in columns])
|
|
38
|
+
+ "".join(
|
|
39
|
+
[
|
|
40
|
+
"<tr>"
|
|
41
|
+
+ "".join(["<td>" + str(x) + "</td>" for x in row])
|
|
42
|
+
+ "</tr>"
|
|
43
|
+
for row in self
|
|
44
|
+
]
|
|
45
|
+
)
|
|
46
|
+
+ "</table></pre>"
|
|
47
|
+
)
|
|
48
|
+
else:
|
|
49
|
+
return (
|
|
50
|
+
"<pre><table>"
|
|
51
|
+
+ "".join(["<th>" + str(index) + "</th>" for index in columns])
|
|
52
|
+
+ "".join(
|
|
53
|
+
["<tr>" + "<td>" + str(row) + "</td>" + "</tr>" for row in self]
|
|
54
|
+
)
|
|
55
|
+
+ "</table></pre>"
|
|
56
|
+
)
|