edsl 0.1.39.dev3__py3-none-any.whl → 0.1.39.dev5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edsl/Base.py +413 -332
- edsl/BaseDiff.py +260 -260
- edsl/TemplateLoader.py +24 -24
- edsl/__init__.py +57 -49
- edsl/__version__.py +1 -1
- edsl/agents/Agent.py +1071 -867
- edsl/agents/AgentList.py +551 -413
- edsl/agents/Invigilator.py +284 -233
- edsl/agents/InvigilatorBase.py +257 -270
- edsl/agents/PromptConstructor.py +272 -354
- edsl/agents/QuestionInstructionPromptBuilder.py +128 -0
- edsl/agents/QuestionTemplateReplacementsBuilder.py +137 -0
- edsl/agents/__init__.py +2 -3
- edsl/agents/descriptors.py +99 -99
- edsl/agents/prompt_helpers.py +129 -129
- edsl/agents/question_option_processor.py +172 -0
- edsl/auto/AutoStudy.py +130 -117
- edsl/auto/StageBase.py +243 -230
- edsl/auto/StageGenerateSurvey.py +178 -178
- edsl/auto/StageLabelQuestions.py +125 -125
- edsl/auto/StagePersona.py +61 -61
- edsl/auto/StagePersonaDimensionValueRanges.py +88 -88
- edsl/auto/StagePersonaDimensionValues.py +74 -74
- edsl/auto/StagePersonaDimensions.py +69 -69
- edsl/auto/StageQuestions.py +74 -73
- edsl/auto/SurveyCreatorPipeline.py +21 -21
- edsl/auto/utilities.py +218 -224
- edsl/base/Base.py +279 -279
- edsl/config.py +177 -157
- edsl/conversation/Conversation.py +290 -290
- edsl/conversation/car_buying.py +59 -58
- edsl/conversation/chips.py +95 -95
- edsl/conversation/mug_negotiation.py +81 -81
- edsl/conversation/next_speaker_utilities.py +93 -93
- edsl/coop/CoopFunctionsMixin.py +15 -0
- edsl/coop/ExpectedParrotKeyHandler.py +125 -0
- edsl/coop/PriceFetcher.py +54 -54
- edsl/coop/__init__.py +2 -2
- edsl/coop/coop.py +1106 -1028
- edsl/coop/utils.py +131 -131
- edsl/data/Cache.py +573 -555
- edsl/data/CacheEntry.py +230 -233
- edsl/data/CacheHandler.py +168 -149
- edsl/data/RemoteCacheSync.py +186 -78
- edsl/data/SQLiteDict.py +292 -292
- edsl/data/__init__.py +5 -4
- edsl/data/orm.py +10 -10
- edsl/data_transfer_models.py +74 -73
- edsl/enums.py +202 -175
- edsl/exceptions/BaseException.py +21 -21
- edsl/exceptions/__init__.py +54 -54
- edsl/exceptions/agents.py +54 -42
- edsl/exceptions/cache.py +5 -5
- edsl/exceptions/configuration.py +16 -16
- edsl/exceptions/coop.py +10 -10
- edsl/exceptions/data.py +14 -14
- edsl/exceptions/general.py +34 -34
- edsl/exceptions/inference_services.py +5 -0
- edsl/exceptions/jobs.py +33 -33
- edsl/exceptions/language_models.py +63 -63
- edsl/exceptions/prompts.py +15 -15
- edsl/exceptions/questions.py +109 -91
- edsl/exceptions/results.py +29 -29
- edsl/exceptions/scenarios.py +29 -22
- edsl/exceptions/surveys.py +37 -37
- edsl/inference_services/AnthropicService.py +106 -87
- edsl/inference_services/AvailableModelCacheHandler.py +184 -0
- edsl/inference_services/AvailableModelFetcher.py +215 -0
- edsl/inference_services/AwsBedrock.py +118 -120
- edsl/inference_services/AzureAI.py +215 -217
- edsl/inference_services/DeepInfraService.py +18 -18
- edsl/inference_services/GoogleService.py +143 -148
- edsl/inference_services/GroqService.py +20 -20
- edsl/inference_services/InferenceServiceABC.py +80 -147
- edsl/inference_services/InferenceServicesCollection.py +138 -97
- edsl/inference_services/MistralAIService.py +120 -123
- edsl/inference_services/OllamaService.py +18 -18
- edsl/inference_services/OpenAIService.py +236 -224
- edsl/inference_services/PerplexityService.py +160 -163
- edsl/inference_services/ServiceAvailability.py +135 -0
- edsl/inference_services/TestService.py +90 -89
- edsl/inference_services/TogetherAIService.py +172 -170
- edsl/inference_services/data_structures.py +134 -0
- edsl/inference_services/models_available_cache.py +118 -118
- edsl/inference_services/rate_limits_cache.py +25 -25
- edsl/inference_services/registry.py +41 -41
- edsl/inference_services/write_available.py +10 -10
- edsl/jobs/AnswerQuestionFunctionConstructor.py +223 -0
- edsl/jobs/Answers.py +43 -56
- edsl/jobs/FetchInvigilator.py +47 -0
- edsl/jobs/InterviewTaskManager.py +98 -0
- edsl/jobs/InterviewsConstructor.py +50 -0
- edsl/jobs/Jobs.py +823 -898
- edsl/jobs/JobsChecks.py +172 -147
- edsl/jobs/JobsComponentConstructor.py +189 -0
- edsl/jobs/JobsPrompts.py +270 -268
- edsl/jobs/JobsRemoteInferenceHandler.py +311 -239
- edsl/jobs/JobsRemoteInferenceLogger.py +239 -0
- edsl/jobs/RequestTokenEstimator.py +30 -0
- edsl/jobs/__init__.py +1 -1
- edsl/jobs/async_interview_runner.py +138 -0
- edsl/jobs/buckets/BucketCollection.py +104 -63
- edsl/jobs/buckets/ModelBuckets.py +65 -65
- edsl/jobs/buckets/TokenBucket.py +283 -251
- edsl/jobs/buckets/TokenBucketAPI.py +211 -0
- edsl/jobs/buckets/TokenBucketClient.py +191 -0
- edsl/jobs/check_survey_scenario_compatibility.py +85 -0
- edsl/jobs/data_structures.py +120 -0
- edsl/jobs/decorators.py +35 -0
- edsl/jobs/interviews/Interview.py +396 -661
- edsl/jobs/interviews/InterviewExceptionCollection.py +99 -99
- edsl/jobs/interviews/InterviewExceptionEntry.py +186 -186
- edsl/jobs/interviews/InterviewStatistic.py +63 -63
- edsl/jobs/interviews/InterviewStatisticsCollection.py +25 -25
- edsl/jobs/interviews/InterviewStatusDictionary.py +78 -78
- edsl/jobs/interviews/InterviewStatusLog.py +92 -92
- edsl/jobs/interviews/ReportErrors.py +66 -66
- edsl/jobs/interviews/interview_status_enum.py +9 -9
- edsl/jobs/jobs_status_enums.py +9 -0
- edsl/jobs/loggers/HTMLTableJobLogger.py +304 -0
- edsl/jobs/results_exceptions_handler.py +98 -0
- edsl/jobs/runners/JobsRunnerAsyncio.py +151 -466
- edsl/jobs/runners/JobsRunnerStatus.py +297 -330
- edsl/jobs/tasks/QuestionTaskCreator.py +244 -242
- edsl/jobs/tasks/TaskCreators.py +64 -64
- edsl/jobs/tasks/TaskHistory.py +470 -450
- edsl/jobs/tasks/TaskStatusLog.py +23 -23
- edsl/jobs/tasks/task_status_enum.py +161 -163
- edsl/jobs/tokens/InterviewTokenUsage.py +27 -27
- edsl/jobs/tokens/TokenUsage.py +34 -34
- edsl/language_models/ComputeCost.py +63 -0
- edsl/language_models/LanguageModel.py +626 -668
- edsl/language_models/ModelList.py +164 -155
- edsl/language_models/PriceManager.py +127 -0
- edsl/language_models/RawResponseHandler.py +106 -0
- edsl/language_models/RegisterLanguageModelsMeta.py +184 -184
- edsl/language_models/ServiceDataSources.py +0 -0
- edsl/language_models/__init__.py +2 -3
- edsl/language_models/fake_openai_call.py +15 -15
- edsl/language_models/fake_openai_service.py +61 -61
- edsl/language_models/key_management/KeyLookup.py +63 -0
- edsl/language_models/key_management/KeyLookupBuilder.py +273 -0
- edsl/language_models/key_management/KeyLookupCollection.py +38 -0
- edsl/language_models/key_management/__init__.py +0 -0
- edsl/language_models/key_management/models.py +131 -0
- edsl/language_models/model.py +256 -0
- edsl/language_models/repair.py +156 -156
- edsl/language_models/utilities.py +65 -64
- edsl/notebooks/Notebook.py +263 -258
- edsl/notebooks/NotebookToLaTeX.py +142 -0
- edsl/notebooks/__init__.py +1 -1
- edsl/prompts/Prompt.py +352 -362
- edsl/prompts/__init__.py +2 -2
- edsl/questions/ExceptionExplainer.py +77 -0
- edsl/questions/HTMLQuestion.py +103 -0
- edsl/questions/QuestionBase.py +518 -664
- edsl/questions/QuestionBasePromptsMixin.py +221 -217
- edsl/questions/QuestionBudget.py +227 -227
- edsl/questions/QuestionCheckBox.py +359 -359
- edsl/questions/QuestionExtract.py +180 -182
- edsl/questions/QuestionFreeText.py +113 -114
- edsl/questions/QuestionFunctional.py +166 -166
- edsl/questions/QuestionList.py +223 -231
- edsl/questions/QuestionMatrix.py +265 -0
- edsl/questions/QuestionMultipleChoice.py +330 -286
- edsl/questions/QuestionNumerical.py +151 -153
- edsl/questions/QuestionRank.py +314 -324
- edsl/questions/Quick.py +41 -41
- edsl/questions/SimpleAskMixin.py +74 -73
- edsl/questions/__init__.py +27 -26
- edsl/questions/{AnswerValidatorMixin.py → answer_validator_mixin.py} +334 -289
- edsl/questions/compose_questions.py +98 -98
- edsl/questions/data_structures.py +20 -0
- edsl/questions/decorators.py +21 -21
- edsl/questions/derived/QuestionLikertFive.py +76 -76
- edsl/questions/derived/QuestionLinearScale.py +90 -87
- edsl/questions/derived/QuestionTopK.py +93 -93
- edsl/questions/derived/QuestionYesNo.py +82 -82
- edsl/questions/descriptors.py +427 -413
- edsl/questions/loop_processor.py +149 -0
- edsl/questions/prompt_templates/question_budget.jinja +13 -13
- edsl/questions/prompt_templates/question_checkbox.jinja +32 -32
- edsl/questions/prompt_templates/question_extract.jinja +11 -11
- edsl/questions/prompt_templates/question_free_text.jinja +3 -3
- edsl/questions/prompt_templates/question_linear_scale.jinja +11 -11
- edsl/questions/prompt_templates/question_list.jinja +17 -17
- edsl/questions/prompt_templates/question_multiple_choice.jinja +33 -33
- edsl/questions/prompt_templates/question_numerical.jinja +36 -36
- edsl/questions/{QuestionBaseGenMixin.py → question_base_gen_mixin.py} +168 -161
- edsl/questions/question_registry.py +177 -177
- edsl/questions/{RegisterQuestionsMeta.py → register_questions_meta.py} +71 -71
- edsl/questions/{ResponseValidatorABC.py → response_validator_abc.py} +188 -174
- edsl/questions/response_validator_factory.py +34 -0
- edsl/questions/settings.py +12 -12
- edsl/questions/templates/budget/answering_instructions.jinja +7 -7
- edsl/questions/templates/budget/question_presentation.jinja +7 -7
- edsl/questions/templates/checkbox/answering_instructions.jinja +10 -10
- edsl/questions/templates/checkbox/question_presentation.jinja +22 -22
- edsl/questions/templates/extract/answering_instructions.jinja +7 -7
- edsl/questions/templates/likert_five/answering_instructions.jinja +10 -10
- edsl/questions/templates/likert_five/question_presentation.jinja +11 -11
- edsl/questions/templates/linear_scale/answering_instructions.jinja +5 -5
- edsl/questions/templates/linear_scale/question_presentation.jinja +5 -5
- edsl/questions/templates/list/answering_instructions.jinja +3 -3
- edsl/questions/templates/list/question_presentation.jinja +5 -5
- edsl/questions/templates/matrix/__init__.py +1 -0
- edsl/questions/templates/matrix/answering_instructions.jinja +5 -0
- edsl/questions/templates/matrix/question_presentation.jinja +20 -0
- edsl/questions/templates/multiple_choice/answering_instructions.jinja +9 -9
- edsl/questions/templates/multiple_choice/question_presentation.jinja +11 -11
- edsl/questions/templates/numerical/answering_instructions.jinja +6 -6
- edsl/questions/templates/numerical/question_presentation.jinja +6 -6
- edsl/questions/templates/rank/answering_instructions.jinja +11 -11
- edsl/questions/templates/rank/question_presentation.jinja +15 -15
- edsl/questions/templates/top_k/answering_instructions.jinja +8 -8
- edsl/questions/templates/top_k/question_presentation.jinja +22 -22
- edsl/questions/templates/yes_no/answering_instructions.jinja +6 -6
- edsl/questions/templates/yes_no/question_presentation.jinja +11 -11
- edsl/results/CSSParameterizer.py +108 -108
- edsl/results/Dataset.py +587 -424
- edsl/results/DatasetExportMixin.py +594 -731
- edsl/results/DatasetTree.py +295 -275
- edsl/results/MarkdownToDocx.py +122 -0
- edsl/results/MarkdownToPDF.py +111 -0
- edsl/results/Result.py +557 -465
- edsl/results/Results.py +1183 -1165
- edsl/results/ResultsExportMixin.py +45 -43
- edsl/results/ResultsGGMixin.py +121 -121
- edsl/results/TableDisplay.py +125 -198
- edsl/results/TextEditor.py +50 -0
- edsl/results/__init__.py +2 -2
- edsl/results/file_exports.py +252 -0
- edsl/results/{ResultsFetchMixin.py → results_fetch_mixin.py} +33 -33
- edsl/results/{Selector.py → results_selector.py} +145 -135
- edsl/results/{ResultsToolsMixin.py → results_tools_mixin.py} +98 -98
- edsl/results/smart_objects.py +96 -0
- edsl/results/table_data_class.py +12 -0
- edsl/results/table_display.css +77 -77
- edsl/results/table_renderers.py +118 -0
- edsl/results/tree_explore.py +115 -115
- edsl/scenarios/ConstructDownloadLink.py +109 -0
- edsl/scenarios/DocumentChunker.py +102 -0
- edsl/scenarios/DocxScenario.py +16 -0
- edsl/scenarios/FileStore.py +511 -632
- edsl/scenarios/PdfExtractor.py +40 -0
- edsl/scenarios/Scenario.py +498 -601
- edsl/scenarios/ScenarioHtmlMixin.py +65 -64
- edsl/scenarios/ScenarioList.py +1458 -1287
- edsl/scenarios/ScenarioListExportMixin.py +45 -52
- edsl/scenarios/ScenarioListPdfMixin.py +239 -261
- edsl/scenarios/__init__.py +3 -4
- edsl/scenarios/directory_scanner.py +96 -0
- edsl/scenarios/file_methods.py +85 -0
- edsl/scenarios/handlers/__init__.py +13 -0
- edsl/scenarios/handlers/csv.py +38 -0
- edsl/scenarios/handlers/docx.py +76 -0
- edsl/scenarios/handlers/html.py +37 -0
- edsl/scenarios/handlers/json.py +111 -0
- edsl/scenarios/handlers/latex.py +5 -0
- edsl/scenarios/handlers/md.py +51 -0
- edsl/scenarios/handlers/pdf.py +68 -0
- edsl/scenarios/handlers/png.py +39 -0
- edsl/scenarios/handlers/pptx.py +105 -0
- edsl/scenarios/handlers/py.py +294 -0
- edsl/scenarios/handlers/sql.py +313 -0
- edsl/scenarios/handlers/sqlite.py +149 -0
- edsl/scenarios/handlers/txt.py +33 -0
- edsl/scenarios/{ScenarioJoin.py → scenario_join.py} +131 -127
- edsl/scenarios/scenario_selector.py +156 -0
- edsl/shared.py +1 -1
- edsl/study/ObjectEntry.py +173 -173
- edsl/study/ProofOfWork.py +113 -113
- edsl/study/SnapShot.py +80 -80
- edsl/study/Study.py +521 -528
- edsl/study/__init__.py +4 -4
- edsl/surveys/ConstructDAG.py +92 -0
- edsl/surveys/DAG.py +148 -148
- edsl/surveys/EditSurvey.py +221 -0
- edsl/surveys/InstructionHandler.py +100 -0
- edsl/surveys/Memory.py +31 -31
- edsl/surveys/MemoryManagement.py +72 -0
- edsl/surveys/MemoryPlan.py +244 -244
- edsl/surveys/Rule.py +327 -326
- edsl/surveys/RuleCollection.py +385 -387
- edsl/surveys/RuleManager.py +172 -0
- edsl/surveys/Simulator.py +75 -0
- edsl/surveys/Survey.py +1280 -1801
- edsl/surveys/SurveyCSS.py +273 -261
- edsl/surveys/SurveyExportMixin.py +259 -259
- edsl/surveys/{SurveyFlowVisualizationMixin.py → SurveyFlowVisualization.py} +181 -179
- edsl/surveys/SurveyQualtricsImport.py +284 -284
- edsl/surveys/SurveyToApp.py +141 -0
- edsl/surveys/__init__.py +5 -3
- edsl/surveys/base.py +53 -53
- edsl/surveys/descriptors.py +60 -56
- edsl/surveys/instructions/ChangeInstruction.py +48 -49
- edsl/surveys/instructions/Instruction.py +56 -65
- edsl/surveys/instructions/InstructionCollection.py +82 -77
- edsl/templates/error_reporting/base.html +23 -23
- edsl/templates/error_reporting/exceptions_by_model.html +34 -34
- edsl/templates/error_reporting/exceptions_by_question_name.html +16 -16
- edsl/templates/error_reporting/exceptions_by_type.html +16 -16
- edsl/templates/error_reporting/interview_details.html +115 -115
- edsl/templates/error_reporting/interviews.html +19 -19
- edsl/templates/error_reporting/overview.html +4 -4
- edsl/templates/error_reporting/performance_plot.html +1 -1
- edsl/templates/error_reporting/report.css +73 -73
- edsl/templates/error_reporting/report.html +117 -117
- edsl/templates/error_reporting/report.js +25 -25
- edsl/tools/__init__.py +1 -1
- edsl/tools/clusters.py +192 -192
- edsl/tools/embeddings.py +27 -27
- edsl/tools/embeddings_plotting.py +118 -118
- edsl/tools/plotting.py +112 -112
- edsl/tools/summarize.py +18 -18
- edsl/utilities/PrettyList.py +56 -0
- edsl/utilities/SystemInfo.py +28 -28
- edsl/utilities/__init__.py +22 -22
- edsl/utilities/ast_utilities.py +25 -25
- edsl/utilities/data/Registry.py +6 -6
- edsl/utilities/data/__init__.py +1 -1
- edsl/utilities/data/scooter_results.json +1 -1
- edsl/utilities/decorators.py +77 -77
- edsl/utilities/gcp_bucket/cloud_storage.py +96 -96
- edsl/utilities/interface.py +627 -627
- edsl/utilities/is_notebook.py +18 -0
- edsl/utilities/is_valid_variable_name.py +11 -0
- edsl/utilities/naming_utilities.py +263 -263
- edsl/utilities/remove_edsl_version.py +24 -0
- edsl/utilities/repair_functions.py +28 -28
- edsl/utilities/restricted_python.py +70 -70
- edsl/utilities/utilities.py +436 -424
- {edsl-0.1.39.dev3.dist-info → edsl-0.1.39.dev5.dist-info}/LICENSE +21 -21
- {edsl-0.1.39.dev3.dist-info → edsl-0.1.39.dev5.dist-info}/METADATA +13 -11
- edsl-0.1.39.dev5.dist-info/RECORD +358 -0
- {edsl-0.1.39.dev3.dist-info → edsl-0.1.39.dev5.dist-info}/WHEEL +1 -1
- edsl/language_models/KeyLookup.py +0 -30
- edsl/language_models/registry.py +0 -190
- edsl/language_models/unused/ReplicateBase.py +0 -83
- edsl/results/ResultsDBMixin.py +0 -238
- edsl-0.1.39.dev3.dist-info/RECORD +0 -277
@@ -0,0 +1,18 @@
|
|
1
|
+
def is_notebook() -> bool:
|
2
|
+
"""Check if the code is running in a Jupyter notebook or Google Colab."""
|
3
|
+
try:
|
4
|
+
shell = get_ipython().__class__.__name__
|
5
|
+
if shell == "ZMQInteractiveShell":
|
6
|
+
return True # Jupyter notebook or qtconsole
|
7
|
+
elif shell == "Shell": # Google Colab's shell class
|
8
|
+
import sys
|
9
|
+
|
10
|
+
if "google.colab" in sys.modules:
|
11
|
+
return True # Running in Google Colab
|
12
|
+
return False
|
13
|
+
elif shell == "TerminalInteractiveShell":
|
14
|
+
return False # Terminal running IPython
|
15
|
+
else:
|
16
|
+
return False # Other type
|
17
|
+
except NameError:
|
18
|
+
return False # Probably standard Python interpreter
|
@@ -0,0 +1,11 @@
|
|
1
|
+
import keyword
|
2
|
+
|
3
|
+
|
4
|
+
def is_valid_variable_name(name, allow_name=True):
|
5
|
+
"""Check if a string is a valid variable name."""
|
6
|
+
if allow_name:
|
7
|
+
return name.isidentifier() and not keyword.iskeyword(name)
|
8
|
+
else:
|
9
|
+
return (
|
10
|
+
name.isidentifier() and not keyword.iskeyword(name) and not name == "name"
|
11
|
+
)
|
@@ -1,263 +1,263 @@
|
|
1
|
-
import re
|
2
|
-
import keyword
|
3
|
-
|
4
|
-
stop_words = {
|
5
|
-
"into",
|
6
|
-
"mustn't",
|
7
|
-
"there",
|
8
|
-
"you'll",
|
9
|
-
"don",
|
10
|
-
"have",
|
11
|
-
"at",
|
12
|
-
"if",
|
13
|
-
"on",
|
14
|
-
"some",
|
15
|
-
"with",
|
16
|
-
"in",
|
17
|
-
"can",
|
18
|
-
"mightn",
|
19
|
-
"off",
|
20
|
-
"few",
|
21
|
-
"not",
|
22
|
-
"d",
|
23
|
-
"hadn",
|
24
|
-
"shan't",
|
25
|
-
"t",
|
26
|
-
"re",
|
27
|
-
"where",
|
28
|
-
"s",
|
29
|
-
"won't",
|
30
|
-
"mustn",
|
31
|
-
"wasn't",
|
32
|
-
"didn't",
|
33
|
-
"has",
|
34
|
-
"same",
|
35
|
-
"too",
|
36
|
-
"will",
|
37
|
-
"you've",
|
38
|
-
"all",
|
39
|
-
"haven't",
|
40
|
-
"isn't",
|
41
|
-
"over",
|
42
|
-
"of",
|
43
|
-
"about",
|
44
|
-
"its",
|
45
|
-
"being",
|
46
|
-
"it",
|
47
|
-
"her",
|
48
|
-
"should",
|
49
|
-
"himself",
|
50
|
-
"wasn",
|
51
|
-
"out",
|
52
|
-
"theirs",
|
53
|
-
"aren",
|
54
|
-
"that",
|
55
|
-
"our",
|
56
|
-
"shouldn't",
|
57
|
-
"you'd",
|
58
|
-
"such",
|
59
|
-
"above",
|
60
|
-
"my",
|
61
|
-
"the",
|
62
|
-
"any",
|
63
|
-
"been",
|
64
|
-
"as",
|
65
|
-
"very",
|
66
|
-
"herself",
|
67
|
-
"o",
|
68
|
-
"weren",
|
69
|
-
"until",
|
70
|
-
"their",
|
71
|
-
"shouldn",
|
72
|
-
"up",
|
73
|
-
"wouldn",
|
74
|
-
"couldn't",
|
75
|
-
"hasn't",
|
76
|
-
"no",
|
77
|
-
"than",
|
78
|
-
"hadn't",
|
79
|
-
"had",
|
80
|
-
"you",
|
81
|
-
"here",
|
82
|
-
"yourself",
|
83
|
-
"yourselves",
|
84
|
-
"during",
|
85
|
-
"ain",
|
86
|
-
"once",
|
87
|
-
"aren't",
|
88
|
-
"what",
|
89
|
-
"so",
|
90
|
-
"hers",
|
91
|
-
"that'll",
|
92
|
-
"other",
|
93
|
-
"ours",
|
94
|
-
"yours",
|
95
|
-
"nor",
|
96
|
-
"him",
|
97
|
-
"doesn",
|
98
|
-
"doesn't",
|
99
|
-
"he",
|
100
|
-
"them",
|
101
|
-
"for",
|
102
|
-
"ll",
|
103
|
-
"isn",
|
104
|
-
"this",
|
105
|
-
"or",
|
106
|
-
"who",
|
107
|
-
"only",
|
108
|
-
"itself",
|
109
|
-
"they",
|
110
|
-
"between",
|
111
|
-
"against",
|
112
|
-
"under",
|
113
|
-
"me",
|
114
|
-
"now",
|
115
|
-
"mightn't",
|
116
|
-
"those",
|
117
|
-
"needn't",
|
118
|
-
"these",
|
119
|
-
"when",
|
120
|
-
"before",
|
121
|
-
"his",
|
122
|
-
"she's",
|
123
|
-
"having",
|
124
|
-
"be",
|
125
|
-
"don't",
|
126
|
-
"haven",
|
127
|
-
"won",
|
128
|
-
"while",
|
129
|
-
"both",
|
130
|
-
"didn",
|
131
|
-
"by",
|
132
|
-
"ourselves",
|
133
|
-
"m",
|
134
|
-
"your",
|
135
|
-
"then",
|
136
|
-
"myself",
|
137
|
-
"we",
|
138
|
-
"it's",
|
139
|
-
"should've",
|
140
|
-
"through",
|
141
|
-
"why",
|
142
|
-
"from",
|
143
|
-
"and",
|
144
|
-
"hasn",
|
145
|
-
"more",
|
146
|
-
"how",
|
147
|
-
"ve",
|
148
|
-
"most",
|
149
|
-
"because",
|
150
|
-
"did",
|
151
|
-
"y",
|
152
|
-
"i",
|
153
|
-
"an",
|
154
|
-
"but",
|
155
|
-
"whom",
|
156
|
-
"below",
|
157
|
-
"further",
|
158
|
-
"am",
|
159
|
-
"which",
|
160
|
-
"just",
|
161
|
-
"ma",
|
162
|
-
"you're",
|
163
|
-
"couldn",
|
164
|
-
"do",
|
165
|
-
"shan",
|
166
|
-
"own",
|
167
|
-
"again",
|
168
|
-
"are",
|
169
|
-
"weren't",
|
170
|
-
"down",
|
171
|
-
"is",
|
172
|
-
"were",
|
173
|
-
"each",
|
174
|
-
"needn",
|
175
|
-
"themselves",
|
176
|
-
"she",
|
177
|
-
"after",
|
178
|
-
"does",
|
179
|
-
"wouldn't",
|
180
|
-
"to",
|
181
|
-
"a",
|
182
|
-
"was",
|
183
|
-
"doing",
|
184
|
-
}
|
185
|
-
|
186
|
-
|
187
|
-
def sanitize_string(input_string, max_length=35):
|
188
|
-
"""Return a sanitized version of the input string that can be used as a variable name.
|
189
|
-
|
190
|
-
>>> candidate_names = ["How are you doing this morning, Dave? What is your favorite kind of coffee?", "class", "def", "here_is_some_text"]
|
191
|
-
>>> [sanitize_string(name) for name in candidate_names]
|
192
|
-
['morning_dave_favorite_kind_coffee', 'class_modified', 'def_modified', 'here_is_some_text']
|
193
|
-
"""
|
194
|
-
|
195
|
-
# Ensure nltk stopwords are downloaded
|
196
|
-
# try:
|
197
|
-
# from nltk.corpus import stopwords
|
198
|
-
# except ImportError or ModuleNotFoundError:
|
199
|
-
# print(
|
200
|
-
# "nltk is not installed. Please install it using 'pip install nltk' to use these features."
|
201
|
-
# )
|
202
|
-
# raise
|
203
|
-
|
204
|
-
# try:
|
205
|
-
# stop_words = set(stopwords.words("english"))
|
206
|
-
# except LookupError:
|
207
|
-
# nltk.download("stopwords")
|
208
|
-
# stop_words = set(stopwords.words("english"))
|
209
|
-
# # raise LookupError("Stopwords not found. Please download them using nltk.download('stopwords')")
|
210
|
-
|
211
|
-
# # Define the list of stopwords
|
212
|
-
|
213
|
-
# Replace special characters with spaces and split into words
|
214
|
-
words = re.sub(r"\W+", " ", input_string).split()
|
215
|
-
|
216
|
-
# Remove stopwords
|
217
|
-
important_words = [word for word in words if word.lower() not in stop_words]
|
218
|
-
|
219
|
-
# Join words with underscores
|
220
|
-
sanitized_string = "_".join(important_words)
|
221
|
-
|
222
|
-
# Ensure the length is less than 25 characters
|
223
|
-
if len(sanitized_string) > max_length:
|
224
|
-
# split off the last word and remove it
|
225
|
-
words = sanitized_string[:max_length].split("_")
|
226
|
-
if len(words) == 1:
|
227
|
-
sanitized_string = words[0]
|
228
|
-
else:
|
229
|
-
sanitized_string = "_".join(words[:-1])
|
230
|
-
|
231
|
-
# Remove leading and trailing underscores
|
232
|
-
sanitized_string = sanitized_string.strip("_")
|
233
|
-
|
234
|
-
# Check if the string is a Python keyword
|
235
|
-
if keyword.iskeyword(sanitized_string):
|
236
|
-
sanitized_string += "_modified"
|
237
|
-
|
238
|
-
result = sanitized_string.lower()
|
239
|
-
return result
|
240
|
-
|
241
|
-
|
242
|
-
# Example usage
|
243
|
-
# input_string = "This is a sample variable-name@123 for testing"
|
244
|
-
# sanitized_string = sanitize_string(input_string)
|
245
|
-
# print(sanitized_string) # Output might be: sample_variable_name_123
|
246
|
-
|
247
|
-
# if __name__ == "__main__":
|
248
|
-
# candidate_names = [
|
249
|
-
# "How are you doing this morning, Dave? What is your favorite kind of coffee?",
|
250
|
-
# "class",
|
251
|
-
# "def",
|
252
|
-
# "here_is_some_text",
|
253
|
-
# ]
|
254
|
-
# for name in candidate_names:
|
255
|
-
# print(f"Original: {name}")
|
256
|
-
# print(f"Sanitized: {sanitize_string(name)}")
|
257
|
-
# print()
|
258
|
-
|
259
|
-
if __name__ == "__main__":
|
260
|
-
# from edsl.conjure.InputData import InputDataABC
|
261
|
-
import doctest
|
262
|
-
|
263
|
-
doctest.testmod(optionflags=doctest.ELLIPSIS)
|
1
|
+
import re
|
2
|
+
import keyword
|
3
|
+
|
4
|
+
stop_words = {
|
5
|
+
"into",
|
6
|
+
"mustn't",
|
7
|
+
"there",
|
8
|
+
"you'll",
|
9
|
+
"don",
|
10
|
+
"have",
|
11
|
+
"at",
|
12
|
+
"if",
|
13
|
+
"on",
|
14
|
+
"some",
|
15
|
+
"with",
|
16
|
+
"in",
|
17
|
+
"can",
|
18
|
+
"mightn",
|
19
|
+
"off",
|
20
|
+
"few",
|
21
|
+
"not",
|
22
|
+
"d",
|
23
|
+
"hadn",
|
24
|
+
"shan't",
|
25
|
+
"t",
|
26
|
+
"re",
|
27
|
+
"where",
|
28
|
+
"s",
|
29
|
+
"won't",
|
30
|
+
"mustn",
|
31
|
+
"wasn't",
|
32
|
+
"didn't",
|
33
|
+
"has",
|
34
|
+
"same",
|
35
|
+
"too",
|
36
|
+
"will",
|
37
|
+
"you've",
|
38
|
+
"all",
|
39
|
+
"haven't",
|
40
|
+
"isn't",
|
41
|
+
"over",
|
42
|
+
"of",
|
43
|
+
"about",
|
44
|
+
"its",
|
45
|
+
"being",
|
46
|
+
"it",
|
47
|
+
"her",
|
48
|
+
"should",
|
49
|
+
"himself",
|
50
|
+
"wasn",
|
51
|
+
"out",
|
52
|
+
"theirs",
|
53
|
+
"aren",
|
54
|
+
"that",
|
55
|
+
"our",
|
56
|
+
"shouldn't",
|
57
|
+
"you'd",
|
58
|
+
"such",
|
59
|
+
"above",
|
60
|
+
"my",
|
61
|
+
"the",
|
62
|
+
"any",
|
63
|
+
"been",
|
64
|
+
"as",
|
65
|
+
"very",
|
66
|
+
"herself",
|
67
|
+
"o",
|
68
|
+
"weren",
|
69
|
+
"until",
|
70
|
+
"their",
|
71
|
+
"shouldn",
|
72
|
+
"up",
|
73
|
+
"wouldn",
|
74
|
+
"couldn't",
|
75
|
+
"hasn't",
|
76
|
+
"no",
|
77
|
+
"than",
|
78
|
+
"hadn't",
|
79
|
+
"had",
|
80
|
+
"you",
|
81
|
+
"here",
|
82
|
+
"yourself",
|
83
|
+
"yourselves",
|
84
|
+
"during",
|
85
|
+
"ain",
|
86
|
+
"once",
|
87
|
+
"aren't",
|
88
|
+
"what",
|
89
|
+
"so",
|
90
|
+
"hers",
|
91
|
+
"that'll",
|
92
|
+
"other",
|
93
|
+
"ours",
|
94
|
+
"yours",
|
95
|
+
"nor",
|
96
|
+
"him",
|
97
|
+
"doesn",
|
98
|
+
"doesn't",
|
99
|
+
"he",
|
100
|
+
"them",
|
101
|
+
"for",
|
102
|
+
"ll",
|
103
|
+
"isn",
|
104
|
+
"this",
|
105
|
+
"or",
|
106
|
+
"who",
|
107
|
+
"only",
|
108
|
+
"itself",
|
109
|
+
"they",
|
110
|
+
"between",
|
111
|
+
"against",
|
112
|
+
"under",
|
113
|
+
"me",
|
114
|
+
"now",
|
115
|
+
"mightn't",
|
116
|
+
"those",
|
117
|
+
"needn't",
|
118
|
+
"these",
|
119
|
+
"when",
|
120
|
+
"before",
|
121
|
+
"his",
|
122
|
+
"she's",
|
123
|
+
"having",
|
124
|
+
"be",
|
125
|
+
"don't",
|
126
|
+
"haven",
|
127
|
+
"won",
|
128
|
+
"while",
|
129
|
+
"both",
|
130
|
+
"didn",
|
131
|
+
"by",
|
132
|
+
"ourselves",
|
133
|
+
"m",
|
134
|
+
"your",
|
135
|
+
"then",
|
136
|
+
"myself",
|
137
|
+
"we",
|
138
|
+
"it's",
|
139
|
+
"should've",
|
140
|
+
"through",
|
141
|
+
"why",
|
142
|
+
"from",
|
143
|
+
"and",
|
144
|
+
"hasn",
|
145
|
+
"more",
|
146
|
+
"how",
|
147
|
+
"ve",
|
148
|
+
"most",
|
149
|
+
"because",
|
150
|
+
"did",
|
151
|
+
"y",
|
152
|
+
"i",
|
153
|
+
"an",
|
154
|
+
"but",
|
155
|
+
"whom",
|
156
|
+
"below",
|
157
|
+
"further",
|
158
|
+
"am",
|
159
|
+
"which",
|
160
|
+
"just",
|
161
|
+
"ma",
|
162
|
+
"you're",
|
163
|
+
"couldn",
|
164
|
+
"do",
|
165
|
+
"shan",
|
166
|
+
"own",
|
167
|
+
"again",
|
168
|
+
"are",
|
169
|
+
"weren't",
|
170
|
+
"down",
|
171
|
+
"is",
|
172
|
+
"were",
|
173
|
+
"each",
|
174
|
+
"needn",
|
175
|
+
"themselves",
|
176
|
+
"she",
|
177
|
+
"after",
|
178
|
+
"does",
|
179
|
+
"wouldn't",
|
180
|
+
"to",
|
181
|
+
"a",
|
182
|
+
"was",
|
183
|
+
"doing",
|
184
|
+
}
|
185
|
+
|
186
|
+
|
187
|
+
def sanitize_string(input_string, max_length=35):
|
188
|
+
"""Return a sanitized version of the input string that can be used as a variable name.
|
189
|
+
|
190
|
+
>>> candidate_names = ["How are you doing this morning, Dave? What is your favorite kind of coffee?", "class", "def", "here_is_some_text"]
|
191
|
+
>>> [sanitize_string(name) for name in candidate_names]
|
192
|
+
['morning_dave_favorite_kind_coffee', 'class_modified', 'def_modified', 'here_is_some_text']
|
193
|
+
"""
|
194
|
+
|
195
|
+
# Ensure nltk stopwords are downloaded
|
196
|
+
# try:
|
197
|
+
# from nltk.corpus import stopwords
|
198
|
+
# except ImportError or ModuleNotFoundError:
|
199
|
+
# print(
|
200
|
+
# "nltk is not installed. Please install it using 'pip install nltk' to use these features."
|
201
|
+
# )
|
202
|
+
# raise
|
203
|
+
|
204
|
+
# try:
|
205
|
+
# stop_words = set(stopwords.words("english"))
|
206
|
+
# except LookupError:
|
207
|
+
# nltk.download("stopwords")
|
208
|
+
# stop_words = set(stopwords.words("english"))
|
209
|
+
# # raise LookupError("Stopwords not found. Please download them using nltk.download('stopwords')")
|
210
|
+
|
211
|
+
# # Define the list of stopwords
|
212
|
+
|
213
|
+
# Replace special characters with spaces and split into words
|
214
|
+
words = re.sub(r"\W+", " ", input_string).split()
|
215
|
+
|
216
|
+
# Remove stopwords
|
217
|
+
important_words = [word for word in words if word.lower() not in stop_words]
|
218
|
+
|
219
|
+
# Join words with underscores
|
220
|
+
sanitized_string = "_".join(important_words)
|
221
|
+
|
222
|
+
# Ensure the length is less than 25 characters
|
223
|
+
if len(sanitized_string) > max_length:
|
224
|
+
# split off the last word and remove it
|
225
|
+
words = sanitized_string[:max_length].split("_")
|
226
|
+
if len(words) == 1:
|
227
|
+
sanitized_string = words[0]
|
228
|
+
else:
|
229
|
+
sanitized_string = "_".join(words[:-1])
|
230
|
+
|
231
|
+
# Remove leading and trailing underscores
|
232
|
+
sanitized_string = sanitized_string.strip("_")
|
233
|
+
|
234
|
+
# Check if the string is a Python keyword
|
235
|
+
if keyword.iskeyword(sanitized_string):
|
236
|
+
sanitized_string += "_modified"
|
237
|
+
|
238
|
+
result = sanitized_string.lower()
|
239
|
+
return result
|
240
|
+
|
241
|
+
|
242
|
+
# Example usage
|
243
|
+
# input_string = "This is a sample variable-name@123 for testing"
|
244
|
+
# sanitized_string = sanitize_string(input_string)
|
245
|
+
# print(sanitized_string) # Output might be: sample_variable_name_123
|
246
|
+
|
247
|
+
# if __name__ == "__main__":
|
248
|
+
# candidate_names = [
|
249
|
+
# "How are you doing this morning, Dave? What is your favorite kind of coffee?",
|
250
|
+
# "class",
|
251
|
+
# "def",
|
252
|
+
# "here_is_some_text",
|
253
|
+
# ]
|
254
|
+
# for name in candidate_names:
|
255
|
+
# print(f"Original: {name}")
|
256
|
+
# print(f"Sanitized: {sanitize_string(name)}")
|
257
|
+
# print()
|
258
|
+
|
259
|
+
if __name__ == "__main__":
|
260
|
+
# from edsl.conjure.InputData import InputDataABC
|
261
|
+
import doctest
|
262
|
+
|
263
|
+
doctest.testmod(optionflags=doctest.ELLIPSIS)
|
@@ -0,0 +1,24 @@
|
|
1
|
+
from functools import wraps
|
2
|
+
|
3
|
+
|
4
|
+
def remove_edsl_version(func):
|
5
|
+
"""
|
6
|
+
Decorator for the EDSL objects' `from_dict` method.
|
7
|
+
- Removes the EDSL version and class name from the dictionary.
|
8
|
+
- Ensures backwards compatibility with older versions of EDSL.
|
9
|
+
"""
|
10
|
+
|
11
|
+
@wraps(func)
|
12
|
+
def wrapper(cls, data, *args, **kwargs):
|
13
|
+
data_copy = dict(data)
|
14
|
+
edsl_version = data_copy.pop("edsl_version", None)
|
15
|
+
edsl_classname = data_copy.pop("edsl_class_name", None)
|
16
|
+
|
17
|
+
# Version- and class-specific logic here
|
18
|
+
if edsl_classname == "Survey":
|
19
|
+
if edsl_version is None or edsl_version <= "0.1.20":
|
20
|
+
data_copy["question_groups"] = {}
|
21
|
+
|
22
|
+
return func(cls, data_copy, *args, **kwargs)
|
23
|
+
|
24
|
+
return wrapper
|
@@ -1,28 +1,28 @@
|
|
1
|
-
import json
|
2
|
-
from edsl.utilities.utilities import valid_json
|
3
|
-
|
4
|
-
|
5
|
-
def extract_json_from_string(s):
|
6
|
-
"""Extract a JSON string from a string."""
|
7
|
-
# Find the first occurrence of '{'
|
8
|
-
start_idx = s.find("{")
|
9
|
-
# Find the last occurrence of '}'
|
10
|
-
end_idx = s.rfind("}")
|
11
|
-
# If both '{' and '}' are found in the string
|
12
|
-
if start_idx != -1 and end_idx != -1 and start_idx < end_idx:
|
13
|
-
# Extract the substring from start_idx to end_idx (inclusive)
|
14
|
-
json_str = s[start_idx : end_idx + 1]
|
15
|
-
try:
|
16
|
-
return json.loads(json_str)
|
17
|
-
except json.JSONDecodeError:
|
18
|
-
raise ValueError("Invalid JSON string")
|
19
|
-
else:
|
20
|
-
raise ValueError("No JSON object found in string")
|
21
|
-
|
22
|
-
|
23
|
-
if __name__ == "__main__":
|
24
|
-
text = (
|
25
|
-
'Sure - here is some JSON { "key": "value", "number": 123, "array": [1, 2, 3] }'
|
26
|
-
)
|
27
|
-
extracted_json = extract_json_from_string(text)
|
28
|
-
d = extracted_json
|
1
|
+
import json
|
2
|
+
from edsl.utilities.utilities import valid_json
|
3
|
+
|
4
|
+
|
5
|
+
def extract_json_from_string(s):
|
6
|
+
"""Extract a JSON string from a string."""
|
7
|
+
# Find the first occurrence of '{'
|
8
|
+
start_idx = s.find("{")
|
9
|
+
# Find the last occurrence of '}'
|
10
|
+
end_idx = s.rfind("}")
|
11
|
+
# If both '{' and '}' are found in the string
|
12
|
+
if start_idx != -1 and end_idx != -1 and start_idx < end_idx:
|
13
|
+
# Extract the substring from start_idx to end_idx (inclusive)
|
14
|
+
json_str = s[start_idx : end_idx + 1]
|
15
|
+
try:
|
16
|
+
return json.loads(json_str)
|
17
|
+
except json.JSONDecodeError:
|
18
|
+
raise ValueError("Invalid JSON string")
|
19
|
+
else:
|
20
|
+
raise ValueError("No JSON object found in string")
|
21
|
+
|
22
|
+
|
23
|
+
if __name__ == "__main__":
|
24
|
+
text = (
|
25
|
+
'Sure - here is some JSON { "key": "value", "number": 123, "array": [1, 2, 3] }'
|
26
|
+
)
|
27
|
+
extracted_json = extract_json_from_string(text)
|
28
|
+
d = extracted_json
|