edsl 0.1.37.dev6__py3-none-any.whl → 0.1.38.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (261) hide show
  1. edsl/Base.py +303 -303
  2. edsl/BaseDiff.py +260 -260
  3. edsl/TemplateLoader.py +24 -24
  4. edsl/__init__.py +48 -48
  5. edsl/__version__.py +1 -1
  6. edsl/agents/Agent.py +855 -855
  7. edsl/agents/AgentList.py +350 -350
  8. edsl/agents/Invigilator.py +222 -222
  9. edsl/agents/InvigilatorBase.py +284 -284
  10. edsl/agents/PromptConstructor.py +353 -353
  11. edsl/agents/__init__.py +3 -3
  12. edsl/agents/descriptors.py +99 -99
  13. edsl/agents/prompt_helpers.py +129 -129
  14. edsl/auto/AutoStudy.py +117 -117
  15. edsl/auto/StageBase.py +230 -230
  16. edsl/auto/StageGenerateSurvey.py +178 -178
  17. edsl/auto/StageLabelQuestions.py +125 -125
  18. edsl/auto/StagePersona.py +61 -61
  19. edsl/auto/StagePersonaDimensionValueRanges.py +88 -88
  20. edsl/auto/StagePersonaDimensionValues.py +74 -74
  21. edsl/auto/StagePersonaDimensions.py +69 -69
  22. edsl/auto/StageQuestions.py +73 -73
  23. edsl/auto/SurveyCreatorPipeline.py +21 -21
  24. edsl/auto/utilities.py +224 -224
  25. edsl/base/Base.py +289 -289
  26. edsl/config.py +149 -149
  27. edsl/conjure/AgentConstructionMixin.py +160 -160
  28. edsl/conjure/Conjure.py +62 -62
  29. edsl/conjure/InputData.py +659 -659
  30. edsl/conjure/InputDataCSV.py +48 -48
  31. edsl/conjure/InputDataMixinQuestionStats.py +182 -182
  32. edsl/conjure/InputDataPyRead.py +91 -91
  33. edsl/conjure/InputDataSPSS.py +8 -8
  34. edsl/conjure/InputDataStata.py +8 -8
  35. edsl/conjure/QuestionOptionMixin.py +76 -76
  36. edsl/conjure/QuestionTypeMixin.py +23 -23
  37. edsl/conjure/RawQuestion.py +65 -65
  38. edsl/conjure/SurveyResponses.py +7 -7
  39. edsl/conjure/__init__.py +9 -9
  40. edsl/conjure/naming_utilities.py +263 -263
  41. edsl/conjure/utilities.py +201 -201
  42. edsl/conversation/Conversation.py +290 -290
  43. edsl/conversation/car_buying.py +58 -58
  44. edsl/conversation/chips.py +95 -95
  45. edsl/conversation/mug_negotiation.py +81 -81
  46. edsl/conversation/next_speaker_utilities.py +93 -93
  47. edsl/coop/PriceFetcher.py +54 -54
  48. edsl/coop/__init__.py +2 -2
  49. edsl/coop/coop.py +958 -958
  50. edsl/coop/utils.py +131 -131
  51. edsl/data/Cache.py +527 -527
  52. edsl/data/CacheEntry.py +228 -228
  53. edsl/data/CacheHandler.py +149 -149
  54. edsl/data/RemoteCacheSync.py +97 -97
  55. edsl/data/SQLiteDict.py +292 -292
  56. edsl/data/__init__.py +4 -4
  57. edsl/data/orm.py +10 -10
  58. edsl/data_transfer_models.py +73 -73
  59. edsl/enums.py +173 -173
  60. edsl/exceptions/BaseException.py +21 -21
  61. edsl/exceptions/__init__.py +54 -54
  62. edsl/exceptions/agents.py +38 -38
  63. edsl/exceptions/configuration.py +16 -16
  64. edsl/exceptions/coop.py +10 -10
  65. edsl/exceptions/data.py +14 -14
  66. edsl/exceptions/general.py +34 -34
  67. edsl/exceptions/jobs.py +33 -33
  68. edsl/exceptions/language_models.py +63 -63
  69. edsl/exceptions/prompts.py +15 -15
  70. edsl/exceptions/questions.py +91 -91
  71. edsl/exceptions/results.py +29 -29
  72. edsl/exceptions/scenarios.py +22 -22
  73. edsl/exceptions/surveys.py +37 -37
  74. edsl/inference_services/AnthropicService.py +87 -87
  75. edsl/inference_services/AwsBedrock.py +120 -120
  76. edsl/inference_services/AzureAI.py +217 -217
  77. edsl/inference_services/DeepInfraService.py +18 -18
  78. edsl/inference_services/GoogleService.py +156 -156
  79. edsl/inference_services/GroqService.py +20 -20
  80. edsl/inference_services/InferenceServiceABC.py +147 -147
  81. edsl/inference_services/InferenceServicesCollection.py +97 -97
  82. edsl/inference_services/MistralAIService.py +123 -123
  83. edsl/inference_services/OllamaService.py +18 -18
  84. edsl/inference_services/OpenAIService.py +224 -224
  85. edsl/inference_services/TestService.py +89 -89
  86. edsl/inference_services/TogetherAIService.py +170 -170
  87. edsl/inference_services/models_available_cache.py +118 -118
  88. edsl/inference_services/rate_limits_cache.py +25 -25
  89. edsl/inference_services/registry.py +39 -39
  90. edsl/inference_services/write_available.py +10 -10
  91. edsl/jobs/Answers.py +56 -56
  92. edsl/jobs/Jobs.py +1347 -1347
  93. edsl/jobs/__init__.py +1 -1
  94. edsl/jobs/buckets/BucketCollection.py +63 -63
  95. edsl/jobs/buckets/ModelBuckets.py +65 -65
  96. edsl/jobs/buckets/TokenBucket.py +248 -248
  97. edsl/jobs/interviews/Interview.py +661 -661
  98. edsl/jobs/interviews/InterviewExceptionCollection.py +99 -99
  99. edsl/jobs/interviews/InterviewExceptionEntry.py +186 -186
  100. edsl/jobs/interviews/InterviewStatistic.py +63 -63
  101. edsl/jobs/interviews/InterviewStatisticsCollection.py +25 -25
  102. edsl/jobs/interviews/InterviewStatusDictionary.py +78 -78
  103. edsl/jobs/interviews/InterviewStatusLog.py +92 -92
  104. edsl/jobs/interviews/ReportErrors.py +66 -66
  105. edsl/jobs/interviews/interview_status_enum.py +9 -9
  106. edsl/jobs/runners/JobsRunnerAsyncio.py +338 -338
  107. edsl/jobs/runners/JobsRunnerStatus.py +332 -332
  108. edsl/jobs/tasks/QuestionTaskCreator.py +242 -242
  109. edsl/jobs/tasks/TaskCreators.py +64 -64
  110. edsl/jobs/tasks/TaskHistory.py +442 -442
  111. edsl/jobs/tasks/TaskStatusLog.py +23 -23
  112. edsl/jobs/tasks/task_status_enum.py +163 -163
  113. edsl/jobs/tokens/InterviewTokenUsage.py +27 -27
  114. edsl/jobs/tokens/TokenUsage.py +34 -34
  115. edsl/language_models/KeyLookup.py +30 -30
  116. edsl/language_models/LanguageModel.py +706 -706
  117. edsl/language_models/ModelList.py +102 -102
  118. edsl/language_models/RegisterLanguageModelsMeta.py +184 -184
  119. edsl/language_models/__init__.py +3 -3
  120. edsl/language_models/fake_openai_call.py +15 -15
  121. edsl/language_models/fake_openai_service.py +61 -61
  122. edsl/language_models/registry.py +137 -137
  123. edsl/language_models/repair.py +156 -156
  124. edsl/language_models/unused/ReplicateBase.py +83 -83
  125. edsl/language_models/utilities.py +64 -64
  126. edsl/notebooks/Notebook.py +259 -259
  127. edsl/notebooks/__init__.py +1 -1
  128. edsl/prompts/Prompt.py +357 -357
  129. edsl/prompts/__init__.py +2 -2
  130. edsl/questions/AnswerValidatorMixin.py +289 -289
  131. edsl/questions/QuestionBase.py +656 -656
  132. edsl/questions/QuestionBaseGenMixin.py +161 -161
  133. edsl/questions/QuestionBasePromptsMixin.py +234 -234
  134. edsl/questions/QuestionBudget.py +227 -227
  135. edsl/questions/QuestionCheckBox.py +359 -359
  136. edsl/questions/QuestionExtract.py +183 -183
  137. edsl/questions/QuestionFreeText.py +114 -114
  138. edsl/questions/QuestionFunctional.py +159 -159
  139. edsl/questions/QuestionList.py +231 -231
  140. edsl/questions/QuestionMultipleChoice.py +286 -286
  141. edsl/questions/QuestionNumerical.py +153 -153
  142. edsl/questions/QuestionRank.py +324 -324
  143. edsl/questions/Quick.py +41 -41
  144. edsl/questions/RegisterQuestionsMeta.py +71 -71
  145. edsl/questions/ResponseValidatorABC.py +174 -174
  146. edsl/questions/SimpleAskMixin.py +73 -73
  147. edsl/questions/__init__.py +26 -26
  148. edsl/questions/compose_questions.py +98 -98
  149. edsl/questions/decorators.py +21 -21
  150. edsl/questions/derived/QuestionLikertFive.py +76 -76
  151. edsl/questions/derived/QuestionLinearScale.py +87 -87
  152. edsl/questions/derived/QuestionTopK.py +91 -91
  153. edsl/questions/derived/QuestionYesNo.py +82 -82
  154. edsl/questions/descriptors.py +413 -413
  155. edsl/questions/prompt_templates/question_budget.jinja +13 -13
  156. edsl/questions/prompt_templates/question_checkbox.jinja +32 -32
  157. edsl/questions/prompt_templates/question_extract.jinja +11 -11
  158. edsl/questions/prompt_templates/question_free_text.jinja +3 -3
  159. edsl/questions/prompt_templates/question_linear_scale.jinja +11 -11
  160. edsl/questions/prompt_templates/question_list.jinja +17 -17
  161. edsl/questions/prompt_templates/question_multiple_choice.jinja +33 -33
  162. edsl/questions/prompt_templates/question_numerical.jinja +36 -36
  163. edsl/questions/question_registry.py +147 -147
  164. edsl/questions/settings.py +12 -12
  165. edsl/questions/templates/budget/answering_instructions.jinja +7 -7
  166. edsl/questions/templates/budget/question_presentation.jinja +7 -7
  167. edsl/questions/templates/checkbox/answering_instructions.jinja +10 -10
  168. edsl/questions/templates/checkbox/question_presentation.jinja +22 -22
  169. edsl/questions/templates/extract/answering_instructions.jinja +7 -7
  170. edsl/questions/templates/likert_five/answering_instructions.jinja +10 -10
  171. edsl/questions/templates/likert_five/question_presentation.jinja +11 -11
  172. edsl/questions/templates/linear_scale/answering_instructions.jinja +5 -5
  173. edsl/questions/templates/linear_scale/question_presentation.jinja +5 -5
  174. edsl/questions/templates/list/answering_instructions.jinja +3 -3
  175. edsl/questions/templates/list/question_presentation.jinja +5 -5
  176. edsl/questions/templates/multiple_choice/answering_instructions.jinja +9 -9
  177. edsl/questions/templates/multiple_choice/question_presentation.jinja +11 -11
  178. edsl/questions/templates/numerical/answering_instructions.jinja +6 -6
  179. edsl/questions/templates/numerical/question_presentation.jinja +6 -6
  180. edsl/questions/templates/rank/answering_instructions.jinja +11 -11
  181. edsl/questions/templates/rank/question_presentation.jinja +15 -15
  182. edsl/questions/templates/top_k/answering_instructions.jinja +8 -8
  183. edsl/questions/templates/top_k/question_presentation.jinja +22 -22
  184. edsl/questions/templates/yes_no/answering_instructions.jinja +6 -6
  185. edsl/questions/templates/yes_no/question_presentation.jinja +11 -11
  186. edsl/results/Dataset.py +293 -293
  187. edsl/results/DatasetExportMixin.py +717 -717
  188. edsl/results/DatasetTree.py +145 -145
  189. edsl/results/Result.py +450 -450
  190. edsl/results/Results.py +1071 -1071
  191. edsl/results/ResultsDBMixin.py +238 -238
  192. edsl/results/ResultsExportMixin.py +43 -43
  193. edsl/results/ResultsFetchMixin.py +33 -33
  194. edsl/results/ResultsGGMixin.py +121 -121
  195. edsl/results/ResultsToolsMixin.py +98 -98
  196. edsl/results/Selector.py +135 -135
  197. edsl/results/__init__.py +2 -2
  198. edsl/results/tree_explore.py +115 -115
  199. edsl/scenarios/FileStore.py +458 -458
  200. edsl/scenarios/Scenario.py +546 -546
  201. edsl/scenarios/ScenarioHtmlMixin.py +64 -64
  202. edsl/scenarios/ScenarioList.py +1112 -1112
  203. edsl/scenarios/ScenarioListExportMixin.py +52 -52
  204. edsl/scenarios/ScenarioListPdfMixin.py +261 -261
  205. edsl/scenarios/__init__.py +4 -4
  206. edsl/shared.py +1 -1
  207. edsl/study/ObjectEntry.py +173 -173
  208. edsl/study/ProofOfWork.py +113 -113
  209. edsl/study/SnapShot.py +80 -80
  210. edsl/study/Study.py +528 -528
  211. edsl/study/__init__.py +4 -4
  212. edsl/surveys/DAG.py +148 -148
  213. edsl/surveys/Memory.py +31 -31
  214. edsl/surveys/MemoryPlan.py +244 -244
  215. edsl/surveys/Rule.py +330 -330
  216. edsl/surveys/RuleCollection.py +387 -387
  217. edsl/surveys/Survey.py +1795 -1795
  218. edsl/surveys/SurveyCSS.py +261 -261
  219. edsl/surveys/SurveyExportMixin.py +259 -259
  220. edsl/surveys/SurveyFlowVisualizationMixin.py +121 -121
  221. edsl/surveys/SurveyQualtricsImport.py +284 -284
  222. edsl/surveys/__init__.py +3 -3
  223. edsl/surveys/base.py +53 -53
  224. edsl/surveys/descriptors.py +56 -56
  225. edsl/surveys/instructions/ChangeInstruction.py +47 -47
  226. edsl/surveys/instructions/Instruction.py +51 -51
  227. edsl/surveys/instructions/InstructionCollection.py +77 -77
  228. edsl/templates/error_reporting/base.html +23 -23
  229. edsl/templates/error_reporting/exceptions_by_model.html +34 -34
  230. edsl/templates/error_reporting/exceptions_by_question_name.html +16 -16
  231. edsl/templates/error_reporting/exceptions_by_type.html +16 -16
  232. edsl/templates/error_reporting/interview_details.html +115 -115
  233. edsl/templates/error_reporting/interviews.html +9 -9
  234. edsl/templates/error_reporting/overview.html +4 -4
  235. edsl/templates/error_reporting/performance_plot.html +1 -1
  236. edsl/templates/error_reporting/report.css +73 -73
  237. edsl/templates/error_reporting/report.html +117 -117
  238. edsl/templates/error_reporting/report.js +25 -25
  239. edsl/tools/__init__.py +1 -1
  240. edsl/tools/clusters.py +192 -192
  241. edsl/tools/embeddings.py +27 -27
  242. edsl/tools/embeddings_plotting.py +118 -118
  243. edsl/tools/plotting.py +112 -112
  244. edsl/tools/summarize.py +18 -18
  245. edsl/utilities/SystemInfo.py +28 -28
  246. edsl/utilities/__init__.py +22 -22
  247. edsl/utilities/ast_utilities.py +25 -25
  248. edsl/utilities/data/Registry.py +6 -6
  249. edsl/utilities/data/__init__.py +1 -1
  250. edsl/utilities/data/scooter_results.json +1 -1
  251. edsl/utilities/decorators.py +77 -77
  252. edsl/utilities/gcp_bucket/cloud_storage.py +96 -96
  253. edsl/utilities/interface.py +627 -627
  254. edsl/utilities/repair_functions.py +28 -28
  255. edsl/utilities/restricted_python.py +70 -70
  256. edsl/utilities/utilities.py +409 -409
  257. {edsl-0.1.37.dev6.dist-info → edsl-0.1.38.dev1.dist-info}/LICENSE +21 -21
  258. {edsl-0.1.37.dev6.dist-info → edsl-0.1.38.dev1.dist-info}/METADATA +1 -1
  259. edsl-0.1.38.dev1.dist-info/RECORD +283 -0
  260. edsl-0.1.37.dev6.dist-info/RECORD +0 -283
  261. {edsl-0.1.37.dev6.dist-info → edsl-0.1.38.dev1.dist-info}/WHEEL +0 -0
@@ -1,458 +1,458 @@
1
- import base64
2
- import io
3
- import tempfile
4
- import mimetypes
5
- import os
6
- from typing import Dict, Any, IO, Optional
7
- import requests
8
- from urllib.parse import urlparse
9
-
10
- import google.generativeai as genai
11
-
12
- from edsl import Scenario
13
- from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
14
- from edsl.utilities.utilities import is_notebook
15
-
16
-
17
- def view_pdf(pdf_path):
18
- import os
19
- import subprocess
20
-
21
- if is_notebook():
22
- from IPython.display import IFrame
23
- from IPython.display import display, HTML
24
-
25
- # Replace 'path/to/your/file.pdf' with the actual path to your PDF file
26
- IFrame(pdf_path, width=700, height=600)
27
- display(HTML(f'<a href="{pdf_path}" target="_blank">Open PDF</a>'))
28
- return
29
-
30
- if os.path.exists(pdf_path):
31
- try:
32
- if (os_name := os.name) == "posix":
33
- # for cool kids
34
- subprocess.run(["open", pdf_path], check=True) # macOS
35
- elif os_name == "nt":
36
- os.startfile(pdf_path) # Windows
37
- else:
38
- subprocess.run(["xdg-open", pdf_path], check=True) # Linux
39
- except Exception as e:
40
- print(f"Error opening PDF: {e}")
41
- else:
42
- print("PDF file was not created successfully.")
43
-
44
-
45
- class FileStore(Scenario):
46
- def __init__(
47
- self,
48
- path: Optional[str] = None,
49
- mime_type: Optional[str] = None,
50
- binary: Optional[bool] = None,
51
- suffix: Optional[str] = None,
52
- base64_string: Optional[str] = None,
53
- external_locations: Optional[Dict[str, str]] = None,
54
- **kwargs,
55
- ):
56
- if path is None and "filename" in kwargs:
57
- path = kwargs["filename"]
58
- self.path = path
59
- self.suffix = suffix or path.split(".")[-1]
60
- self.binary = binary or False
61
- self.mime_type = (
62
- mime_type or mimetypes.guess_type(path)[0] or "application/octet-stream"
63
- )
64
- self.base64_string = base64_string or self.encode_file_to_base64_string(path)
65
- self.external_locations = external_locations or {}
66
- super().__init__(
67
- {
68
- "path": self.path,
69
- "base64_string": self.base64_string,
70
- "binary": self.binary,
71
- "suffix": self.suffix,
72
- "mime_type": self.mime_type,
73
- "external_locations": self.external_locations,
74
- }
75
- )
76
-
77
- def __str__(self):
78
- return "FileStore: self.path"
79
-
80
- @classmethod
81
- def example(self):
82
- import tempfile
83
-
84
- with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as f:
85
- f.write(b"Hello, World!")
86
-
87
- return self(path=f.name)
88
-
89
- @property
90
- def size(self) -> int:
91
- if self.base64_string != None:
92
- return (len(self.base64_string) / 4.0) * 3 # from base64 to char size
93
- return os.path.getsize(self.path)
94
-
95
- def upload_google(self, refresh: bool = False) -> None:
96
- genai.configure(api_key=os.getenv("GOOGLE_API_KEY"))
97
- google_info = genai.upload_file(self.path, mime_type=self.mime_type)
98
- self.external_locations["google"] = google_info.to_dict()
99
-
100
- @classmethod
101
- @remove_edsl_version
102
- def from_dict(cls, d):
103
- # return cls(d["filename"], d["binary"], d["suffix"], d["base64_string"])
104
- return cls(**d)
105
-
106
- def __repr__(self):
107
- return f"FileStore(path='{self.path}')"
108
-
109
- def encode_file_to_base64_string(self, file_path: str):
110
- try:
111
- # Attempt to open the file in text mode
112
- with open(file_path, "r") as text_file:
113
- # Read the text data
114
- text_data = text_file.read()
115
- # Encode the text data to a base64 string
116
- base64_encoded_data = base64.b64encode(text_data.encode("utf-8"))
117
- except UnicodeDecodeError:
118
- # If reading as text fails, open the file in binary mode
119
- with open(file_path, "rb") as binary_file:
120
- # Read the binary data
121
- binary_data = binary_file.read()
122
- # Encode the binary data to a base64 string
123
- base64_encoded_data = base64.b64encode(binary_data)
124
- self.binary = True
125
- # Convert the base64 bytes to a string
126
- base64_string = base64_encoded_data.decode("utf-8")
127
-
128
- return base64_string
129
-
130
- def open(self) -> "IO":
131
- if self.binary:
132
- return self.base64_to_file(self["base64_string"], is_binary=True)
133
- else:
134
- return self.base64_to_text_file(self["base64_string"])
135
-
136
- @staticmethod
137
- def base64_to_text_file(base64_string) -> "IO":
138
- # Decode the base64 string to bytes
139
- text_data_bytes = base64.b64decode(base64_string)
140
-
141
- # Convert bytes to string
142
- text_data = text_data_bytes.decode("utf-8")
143
-
144
- # Create a StringIO object from the text data
145
- text_file = io.StringIO(text_data)
146
-
147
- return text_file
148
-
149
- @staticmethod
150
- def base64_to_file(base64_string, is_binary=True):
151
- # Decode the base64 string to bytes
152
- file_data = base64.b64decode(base64_string)
153
-
154
- if is_binary:
155
- # Create a BytesIO object for binary data
156
- return io.BytesIO(file_data)
157
- else:
158
- # Convert bytes to string for text data
159
- text_data = file_data.decode("utf-8")
160
- # Create a StringIO object for text data
161
- return io.StringIO(text_data)
162
-
163
- def to_tempfile(self, suffix=None):
164
- if suffix is None:
165
- suffix = self.suffix
166
- if self.binary:
167
- file_like_object = self.base64_to_file(
168
- self["base64_string"], is_binary=True
169
- )
170
- else:
171
- file_like_object = self.base64_to_text_file(self["base64_string"])
172
-
173
- # Create a named temporary file
174
- mode = "wb" if self.binary else "w"
175
- temp_file = tempfile.NamedTemporaryFile(
176
- delete=False, suffix="." + suffix, mode=mode
177
- )
178
-
179
- if self.binary:
180
- temp_file.write(file_like_object.read())
181
- else:
182
- temp_file.write(file_like_object.read())
183
-
184
- temp_file.close()
185
-
186
- return temp_file.name
187
-
188
- def view(self, max_size: int = 300) -> None:
189
- if self.suffix == "pdf":
190
- view_pdf(self.path)
191
-
192
- if self.suffix == "png" or self.suffix == "jpg" or self.suffix == "jpeg":
193
- if is_notebook():
194
- from IPython.display import Image
195
- from PIL import Image as PILImage
196
-
197
- if max_size:
198
- # Open the image using Pillow
199
- with PILImage.open(self.path) as img:
200
- # Get original width and height
201
- original_width, original_height = img.size
202
-
203
- # Calculate the scaling factor
204
- scale = min(
205
- max_size / original_width, max_size / original_height
206
- )
207
-
208
- # Calculate new dimensions
209
- new_width = int(original_width * scale)
210
- new_height = int(original_height * scale)
211
-
212
- return Image(self.path, width=new_width, height=new_height)
213
- else:
214
- return Image(self.path)
215
-
216
- def push(
217
- self, description: Optional[str] = None, visibility: str = "unlisted"
218
- ) -> dict:
219
- """
220
- Push the object to Coop.
221
- :param description: The description of the object to push.
222
- :param visibility: The visibility of the object to push.
223
- """
224
- scenario_version = Scenario.from_dict(self.to_dict())
225
- if description is None:
226
- description = "File: " + self.path
227
- info = scenario_version.push(description=description, visibility=visibility)
228
- return info
229
-
230
- @classmethod
231
- def pull(cls, uuid: str, expected_parrot_url: Optional[str] = None) -> "FileStore":
232
- """
233
- :param uuid: The UUID of the object to pull.
234
- :param expected_parrot_url: The URL of the Parrot server to use.
235
- :return: The object pulled from the Parrot server.
236
- """
237
- scenario_version = Scenario.pull(uuid, expected_parrot_url=expected_parrot_url)
238
- return cls.from_dict(scenario_version.to_dict())
239
-
240
- @classmethod
241
- def from_url(
242
- cls,
243
- url: str,
244
- download_path: Optional[str] = None,
245
- mime_type: Optional[str] = None,
246
- ) -> "FileStore":
247
- """
248
- :param url: The URL of the file to download.
249
- :param download_path: The path to save the downloaded file.
250
- :param mime_type: The MIME type of the file. If None, it will be guessed from the file extension.
251
- """
252
-
253
- response = requests.get(url, stream=True)
254
- response.raise_for_status() # Raises an HTTPError for bad responses
255
-
256
- # Get the filename from the URL if download_path is not provided
257
- if download_path is None:
258
- filename = os.path.basename(urlparse(url).path)
259
- if not filename:
260
- filename = "downloaded_file"
261
- # download_path = filename
262
- download_path = os.path.join(os.getcwd(), filename)
263
-
264
- # Ensure the directory exists
265
- os.makedirs(os.path.dirname(download_path), exist_ok=True)
266
-
267
- # Write the file
268
- with open(download_path, "wb") as file:
269
- for chunk in response.iter_content(chunk_size=8192):
270
- file.write(chunk)
271
-
272
- # Create and return a new File instance
273
- return cls(download_path, mime_type=mime_type)
274
-
275
-
276
- class CSVFileStore(FileStore):
277
- @classmethod
278
- def example(cls):
279
- from edsl.results.Results import Results
280
-
281
- r = Results.example()
282
- import tempfile
283
-
284
- with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as f:
285
- r.to_csv(filename=f.name)
286
-
287
- return cls(f.name)
288
-
289
- def view(self):
290
- import pandas as pd
291
-
292
- return pd.read_csv(self.to_tempfile())
293
-
294
-
295
- class PDFFileStore(FileStore):
296
- def view(self):
297
- pdf_path = self.to_tempfile()
298
- print(f"PDF path: {pdf_path}") # Print the path to ensure it exists
299
- import os
300
- import subprocess
301
-
302
- if os.path.exists(pdf_path):
303
- try:
304
- if os.name == "posix":
305
- # for cool kids
306
- subprocess.run(["open", pdf_path], check=True) # macOS
307
- elif os.name == "nt":
308
- os.startfile(pdf_path) # Windows
309
- else:
310
- subprocess.run(["xdg-open", pdf_path], check=True) # Linux
311
- except Exception as e:
312
- print(f"Error opening PDF: {e}")
313
- else:
314
- print("PDF file was not created successfully.")
315
-
316
- @classmethod
317
- def example(cls):
318
- import textwrap
319
-
320
- pdf_string = textwrap.dedent(
321
- """\
322
- %PDF-1.4
323
- 1 0 obj
324
- << /Type /Catalog /Pages 2 0 R >>
325
- endobj
326
- 2 0 obj
327
- << /Type /Pages /Kids [3 0 R] /Count 1 >>
328
- endobj
329
- 3 0 obj
330
- << /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R >>
331
- endobj
332
- 4 0 obj
333
- << /Length 44 >>
334
- stream
335
- BT
336
- /F1 24 Tf
337
- 100 700 Td
338
- (Hello, World!) Tj
339
- ET
340
- endstream
341
- endobj
342
- 5 0 obj
343
- << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>
344
- endobj
345
- 6 0 obj
346
- << /ProcSet [/PDF /Text] /Font << /F1 5 0 R >> >>
347
- endobj
348
- xref
349
- 0 7
350
- 0000000000 65535 f
351
- 0000000010 00000 n
352
- 0000000053 00000 n
353
- 0000000100 00000 n
354
- 0000000173 00000 n
355
- 0000000232 00000 n
356
- 0000000272 00000 n
357
- trailer
358
- << /Size 7 /Root 1 0 R >>
359
- startxref
360
- 318
361
- %%EOF"""
362
- )
363
- import tempfile
364
-
365
- with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f:
366
- f.write(pdf_string.encode())
367
-
368
- return cls(f.name)
369
-
370
-
371
- class PNGFileStore(FileStore):
372
- @classmethod
373
- def example(cls):
374
- import textwrap
375
-
376
- png_string = textwrap.dedent(
377
- """\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x01\x00\x00\x00\x01\x00\x08\x06\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\x0cIDAT\x08\xd7c\x00\x01"""
378
- )
379
- import tempfile
380
-
381
- with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as f:
382
- f.write(png_string.encode())
383
-
384
- return cls(f.name)
385
-
386
- def view(self):
387
- import matplotlib.pyplot as plt
388
- import matplotlib.image as mpimg
389
-
390
- img = mpimg.imread(self.to_tempfile())
391
- plt.imshow(img)
392
- plt.show()
393
-
394
-
395
- class SQLiteFileStore(FileStore):
396
- @classmethod
397
- def example(cls):
398
- import sqlite3
399
- import tempfile
400
-
401
- with tempfile.NamedTemporaryFile(suffix=".sqlite", delete=False) as f:
402
- conn = sqlite3.connect(f.name)
403
- c = conn.cursor()
404
- c.execute("""CREATE TABLE stocks (date text)""")
405
- conn.commit()
406
-
407
- return cls(f.name)
408
-
409
- def view(self):
410
- import subprocess
411
- import os
412
-
413
- sqlite_path = self.to_tempfile()
414
- os.system(f"sqlite3 {sqlite_path}")
415
-
416
-
417
- class HTMLFileStore(FileStore):
418
- @classmethod
419
- def example(cls):
420
- import tempfile
421
-
422
- with tempfile.NamedTemporaryFile(suffix=".html", delete=False) as f:
423
- f.write("<html><body><h1>Test</h1></body></html>".encode())
424
-
425
- return cls(f.name)
426
-
427
- def view(self):
428
- import webbrowser
429
-
430
- html_path = self.to_tempfile()
431
- webbrowser.open("file://" + html_path)
432
-
433
-
434
- if __name__ == "__main__":
435
- # file_path = "../conjure/examples/Ex11-2.sav"
436
- # fs = FileStore(file_path)
437
- # info = fs.push()
438
- # print(info)
439
-
440
- # fs = CSVFileStore.example()
441
- # fs.to_tempfile()
442
- # print(fs.view())
443
-
444
- # fs = PDFFileStore.example()
445
- # fs.view()
446
-
447
- # fs = PDFFileStore("paper.pdf")
448
- # fs.view()
449
- # from edsl import Conjure
450
- pass
451
- # fs = PNGFileStore("logo.png")
452
- # fs.view()
453
- # fs.upload_google()
454
-
455
- # c = Conjure(datafile_name=fs.to_tempfile())
456
- # f = PDFFileStore("paper.pdf")
457
- # print(f.to_tempfile())
458
- # f.push()
1
+ import base64
2
+ import io
3
+ import tempfile
4
+ import mimetypes
5
+ import os
6
+ from typing import Dict, Any, IO, Optional
7
+ import requests
8
+ from urllib.parse import urlparse
9
+
10
+ import google.generativeai as genai
11
+
12
+ from edsl import Scenario
13
+ from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
14
+ from edsl.utilities.utilities import is_notebook
15
+
16
+
17
+ def view_pdf(pdf_path):
18
+ import os
19
+ import subprocess
20
+
21
+ if is_notebook():
22
+ from IPython.display import IFrame
23
+ from IPython.display import display, HTML
24
+
25
+ # Replace 'path/to/your/file.pdf' with the actual path to your PDF file
26
+ IFrame(pdf_path, width=700, height=600)
27
+ display(HTML(f'<a href="{pdf_path}" target="_blank">Open PDF</a>'))
28
+ return
29
+
30
+ if os.path.exists(pdf_path):
31
+ try:
32
+ if (os_name := os.name) == "posix":
33
+ # for cool kids
34
+ subprocess.run(["open", pdf_path], check=True) # macOS
35
+ elif os_name == "nt":
36
+ os.startfile(pdf_path) # Windows
37
+ else:
38
+ subprocess.run(["xdg-open", pdf_path], check=True) # Linux
39
+ except Exception as e:
40
+ print(f"Error opening PDF: {e}")
41
+ else:
42
+ print("PDF file was not created successfully.")
43
+
44
+
45
+ class FileStore(Scenario):
46
+ def __init__(
47
+ self,
48
+ path: Optional[str] = None,
49
+ mime_type: Optional[str] = None,
50
+ binary: Optional[bool] = None,
51
+ suffix: Optional[str] = None,
52
+ base64_string: Optional[str] = None,
53
+ external_locations: Optional[Dict[str, str]] = None,
54
+ **kwargs,
55
+ ):
56
+ if path is None and "filename" in kwargs:
57
+ path = kwargs["filename"]
58
+ self.path = path
59
+ self.suffix = suffix or path.split(".")[-1]
60
+ self.binary = binary or False
61
+ self.mime_type = (
62
+ mime_type or mimetypes.guess_type(path)[0] or "application/octet-stream"
63
+ )
64
+ self.base64_string = base64_string or self.encode_file_to_base64_string(path)
65
+ self.external_locations = external_locations or {}
66
+ super().__init__(
67
+ {
68
+ "path": self.path,
69
+ "base64_string": self.base64_string,
70
+ "binary": self.binary,
71
+ "suffix": self.suffix,
72
+ "mime_type": self.mime_type,
73
+ "external_locations": self.external_locations,
74
+ }
75
+ )
76
+
77
+ def __str__(self):
78
+ return "FileStore: self.path"
79
+
80
+ @classmethod
81
+ def example(self):
82
+ import tempfile
83
+
84
+ with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as f:
85
+ f.write(b"Hello, World!")
86
+
87
+ return self(path=f.name)
88
+
89
+ @property
90
+ def size(self) -> int:
91
+ if self.base64_string != None:
92
+ return (len(self.base64_string) / 4.0) * 3 # from base64 to char size
93
+ return os.path.getsize(self.path)
94
+
95
+ def upload_google(self, refresh: bool = False) -> None:
96
+ genai.configure(api_key=os.getenv("GOOGLE_API_KEY"))
97
+ google_info = genai.upload_file(self.path, mime_type=self.mime_type)
98
+ self.external_locations["google"] = google_info.to_dict()
99
+
100
+ @classmethod
101
+ @remove_edsl_version
102
+ def from_dict(cls, d):
103
+ # return cls(d["filename"], d["binary"], d["suffix"], d["base64_string"])
104
+ return cls(**d)
105
+
106
+ def __repr__(self):
107
+ return f"FileStore(path='{self.path}')"
108
+
109
+ def encode_file_to_base64_string(self, file_path: str):
110
+ try:
111
+ # Attempt to open the file in text mode
112
+ with open(file_path, "r") as text_file:
113
+ # Read the text data
114
+ text_data = text_file.read()
115
+ # Encode the text data to a base64 string
116
+ base64_encoded_data = base64.b64encode(text_data.encode("utf-8"))
117
+ except UnicodeDecodeError:
118
+ # If reading as text fails, open the file in binary mode
119
+ with open(file_path, "rb") as binary_file:
120
+ # Read the binary data
121
+ binary_data = binary_file.read()
122
+ # Encode the binary data to a base64 string
123
+ base64_encoded_data = base64.b64encode(binary_data)
124
+ self.binary = True
125
+ # Convert the base64 bytes to a string
126
+ base64_string = base64_encoded_data.decode("utf-8")
127
+
128
+ return base64_string
129
+
130
+ def open(self) -> "IO":
131
+ if self.binary:
132
+ return self.base64_to_file(self["base64_string"], is_binary=True)
133
+ else:
134
+ return self.base64_to_text_file(self["base64_string"])
135
+
136
+ @staticmethod
137
+ def base64_to_text_file(base64_string) -> "IO":
138
+ # Decode the base64 string to bytes
139
+ text_data_bytes = base64.b64decode(base64_string)
140
+
141
+ # Convert bytes to string
142
+ text_data = text_data_bytes.decode("utf-8")
143
+
144
+ # Create a StringIO object from the text data
145
+ text_file = io.StringIO(text_data)
146
+
147
+ return text_file
148
+
149
+ @staticmethod
150
+ def base64_to_file(base64_string, is_binary=True):
151
+ # Decode the base64 string to bytes
152
+ file_data = base64.b64decode(base64_string)
153
+
154
+ if is_binary:
155
+ # Create a BytesIO object for binary data
156
+ return io.BytesIO(file_data)
157
+ else:
158
+ # Convert bytes to string for text data
159
+ text_data = file_data.decode("utf-8")
160
+ # Create a StringIO object for text data
161
+ return io.StringIO(text_data)
162
+
163
+ def to_tempfile(self, suffix=None):
164
+ if suffix is None:
165
+ suffix = self.suffix
166
+ if self.binary:
167
+ file_like_object = self.base64_to_file(
168
+ self["base64_string"], is_binary=True
169
+ )
170
+ else:
171
+ file_like_object = self.base64_to_text_file(self["base64_string"])
172
+
173
+ # Create a named temporary file
174
+ mode = "wb" if self.binary else "w"
175
+ temp_file = tempfile.NamedTemporaryFile(
176
+ delete=False, suffix="." + suffix, mode=mode
177
+ )
178
+
179
+ if self.binary:
180
+ temp_file.write(file_like_object.read())
181
+ else:
182
+ temp_file.write(file_like_object.read())
183
+
184
+ temp_file.close()
185
+
186
+ return temp_file.name
187
+
188
+ def view(self, max_size: int = 300) -> None:
189
+ if self.suffix == "pdf":
190
+ view_pdf(self.path)
191
+
192
+ if self.suffix == "png" or self.suffix == "jpg" or self.suffix == "jpeg":
193
+ if is_notebook():
194
+ from IPython.display import Image
195
+ from PIL import Image as PILImage
196
+
197
+ if max_size:
198
+ # Open the image using Pillow
199
+ with PILImage.open(self.path) as img:
200
+ # Get original width and height
201
+ original_width, original_height = img.size
202
+
203
+ # Calculate the scaling factor
204
+ scale = min(
205
+ max_size / original_width, max_size / original_height
206
+ )
207
+
208
+ # Calculate new dimensions
209
+ new_width = int(original_width * scale)
210
+ new_height = int(original_height * scale)
211
+
212
+ return Image(self.path, width=new_width, height=new_height)
213
+ else:
214
+ return Image(self.path)
215
+
216
+ def push(
217
+ self, description: Optional[str] = None, visibility: str = "unlisted"
218
+ ) -> dict:
219
+ """
220
+ Push the object to Coop.
221
+ :param description: The description of the object to push.
222
+ :param visibility: The visibility of the object to push.
223
+ """
224
+ scenario_version = Scenario.from_dict(self.to_dict())
225
+ if description is None:
226
+ description = "File: " + self.path
227
+ info = scenario_version.push(description=description, visibility=visibility)
228
+ return info
229
+
230
+ @classmethod
231
+ def pull(cls, uuid: str, expected_parrot_url: Optional[str] = None) -> "FileStore":
232
+ """
233
+ :param uuid: The UUID of the object to pull.
234
+ :param expected_parrot_url: The URL of the Parrot server to use.
235
+ :return: The object pulled from the Parrot server.
236
+ """
237
+ scenario_version = Scenario.pull(uuid, expected_parrot_url=expected_parrot_url)
238
+ return cls.from_dict(scenario_version.to_dict())
239
+
240
+ @classmethod
241
+ def from_url(
242
+ cls,
243
+ url: str,
244
+ download_path: Optional[str] = None,
245
+ mime_type: Optional[str] = None,
246
+ ) -> "FileStore":
247
+ """
248
+ :param url: The URL of the file to download.
249
+ :param download_path: The path to save the downloaded file.
250
+ :param mime_type: The MIME type of the file. If None, it will be guessed from the file extension.
251
+ """
252
+
253
+ response = requests.get(url, stream=True)
254
+ response.raise_for_status() # Raises an HTTPError for bad responses
255
+
256
+ # Get the filename from the URL if download_path is not provided
257
+ if download_path is None:
258
+ filename = os.path.basename(urlparse(url).path)
259
+ if not filename:
260
+ filename = "downloaded_file"
261
+ # download_path = filename
262
+ download_path = os.path.join(os.getcwd(), filename)
263
+
264
+ # Ensure the directory exists
265
+ os.makedirs(os.path.dirname(download_path), exist_ok=True)
266
+
267
+ # Write the file
268
+ with open(download_path, "wb") as file:
269
+ for chunk in response.iter_content(chunk_size=8192):
270
+ file.write(chunk)
271
+
272
+ # Create and return a new File instance
273
+ return cls(download_path, mime_type=mime_type)
274
+
275
+
276
+ class CSVFileStore(FileStore):
277
+ @classmethod
278
+ def example(cls):
279
+ from edsl.results.Results import Results
280
+
281
+ r = Results.example()
282
+ import tempfile
283
+
284
+ with tempfile.NamedTemporaryFile(suffix=".csv", delete=False) as f:
285
+ r.to_csv(filename=f.name)
286
+
287
+ return cls(f.name)
288
+
289
+ def view(self):
290
+ import pandas as pd
291
+
292
+ return pd.read_csv(self.to_tempfile())
293
+
294
+
295
+ class PDFFileStore(FileStore):
296
+ def view(self):
297
+ pdf_path = self.to_tempfile()
298
+ print(f"PDF path: {pdf_path}") # Print the path to ensure it exists
299
+ import os
300
+ import subprocess
301
+
302
+ if os.path.exists(pdf_path):
303
+ try:
304
+ if os.name == "posix":
305
+ # for cool kids
306
+ subprocess.run(["open", pdf_path], check=True) # macOS
307
+ elif os.name == "nt":
308
+ os.startfile(pdf_path) # Windows
309
+ else:
310
+ subprocess.run(["xdg-open", pdf_path], check=True) # Linux
311
+ except Exception as e:
312
+ print(f"Error opening PDF: {e}")
313
+ else:
314
+ print("PDF file was not created successfully.")
315
+
316
+ @classmethod
317
+ def example(cls):
318
+ import textwrap
319
+
320
+ pdf_string = textwrap.dedent(
321
+ """\
322
+ %PDF-1.4
323
+ 1 0 obj
324
+ << /Type /Catalog /Pages 2 0 R >>
325
+ endobj
326
+ 2 0 obj
327
+ << /Type /Pages /Kids [3 0 R] /Count 1 >>
328
+ endobj
329
+ 3 0 obj
330
+ << /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R >>
331
+ endobj
332
+ 4 0 obj
333
+ << /Length 44 >>
334
+ stream
335
+ BT
336
+ /F1 24 Tf
337
+ 100 700 Td
338
+ (Hello, World!) Tj
339
+ ET
340
+ endstream
341
+ endobj
342
+ 5 0 obj
343
+ << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>
344
+ endobj
345
+ 6 0 obj
346
+ << /ProcSet [/PDF /Text] /Font << /F1 5 0 R >> >>
347
+ endobj
348
+ xref
349
+ 0 7
350
+ 0000000000 65535 f
351
+ 0000000010 00000 n
352
+ 0000000053 00000 n
353
+ 0000000100 00000 n
354
+ 0000000173 00000 n
355
+ 0000000232 00000 n
356
+ 0000000272 00000 n
357
+ trailer
358
+ << /Size 7 /Root 1 0 R >>
359
+ startxref
360
+ 318
361
+ %%EOF"""
362
+ )
363
+ import tempfile
364
+
365
+ with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f:
366
+ f.write(pdf_string.encode())
367
+
368
+ return cls(f.name)
369
+
370
+
371
+ class PNGFileStore(FileStore):
372
+ @classmethod
373
+ def example(cls):
374
+ import textwrap
375
+
376
+ png_string = textwrap.dedent(
377
+ """\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x01\x00\x00\x00\x01\x00\x08\x06\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\x0cIDAT\x08\xd7c\x00\x01"""
378
+ )
379
+ import tempfile
380
+
381
+ with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as f:
382
+ f.write(png_string.encode())
383
+
384
+ return cls(f.name)
385
+
386
+ def view(self):
387
+ import matplotlib.pyplot as plt
388
+ import matplotlib.image as mpimg
389
+
390
+ img = mpimg.imread(self.to_tempfile())
391
+ plt.imshow(img)
392
+ plt.show()
393
+
394
+
395
+ class SQLiteFileStore(FileStore):
396
+ @classmethod
397
+ def example(cls):
398
+ import sqlite3
399
+ import tempfile
400
+
401
+ with tempfile.NamedTemporaryFile(suffix=".sqlite", delete=False) as f:
402
+ conn = sqlite3.connect(f.name)
403
+ c = conn.cursor()
404
+ c.execute("""CREATE TABLE stocks (date text)""")
405
+ conn.commit()
406
+
407
+ return cls(f.name)
408
+
409
+ def view(self):
410
+ import subprocess
411
+ import os
412
+
413
+ sqlite_path = self.to_tempfile()
414
+ os.system(f"sqlite3 {sqlite_path}")
415
+
416
+
417
+ class HTMLFileStore(FileStore):
418
+ @classmethod
419
+ def example(cls):
420
+ import tempfile
421
+
422
+ with tempfile.NamedTemporaryFile(suffix=".html", delete=False) as f:
423
+ f.write("<html><body><h1>Test</h1></body></html>".encode())
424
+
425
+ return cls(f.name)
426
+
427
+ def view(self):
428
+ import webbrowser
429
+
430
+ html_path = self.to_tempfile()
431
+ webbrowser.open("file://" + html_path)
432
+
433
+
434
+ if __name__ == "__main__":
435
+ # file_path = "../conjure/examples/Ex11-2.sav"
436
+ # fs = FileStore(file_path)
437
+ # info = fs.push()
438
+ # print(info)
439
+
440
+ # fs = CSVFileStore.example()
441
+ # fs.to_tempfile()
442
+ # print(fs.view())
443
+
444
+ # fs = PDFFileStore.example()
445
+ # fs.view()
446
+
447
+ # fs = PDFFileStore("paper.pdf")
448
+ # fs.view()
449
+ # from edsl import Conjure
450
+ pass
451
+ # fs = PNGFileStore("logo.png")
452
+ # fs.view()
453
+ # fs.upload_google()
454
+
455
+ # c = Conjure(datafile_name=fs.to_tempfile())
456
+ # f = PDFFileStore("paper.pdf")
457
+ # print(f.to_tempfile())
458
+ # f.push()