webscout 6.3__py3-none-any.whl → 6.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of webscout might be problematic. Click here for more details.

Files changed (85) hide show
  1. webscout/AIauto.py +191 -176
  2. webscout/AIbase.py +0 -197
  3. webscout/AIutel.py +488 -1130
  4. webscout/Bing_search.py +250 -153
  5. webscout/DWEBS.py +151 -19
  6. webscout/Extra/__init__.py +2 -1
  7. webscout/Extra/autocoder/__init__.py +9 -0
  8. webscout/Extra/autocoder/autocoder_utiles.py +121 -0
  9. webscout/Extra/autocoder/rawdog.py +681 -0
  10. webscout/Extra/autollama.py +246 -195
  11. webscout/Extra/gguf.py +441 -416
  12. webscout/LLM.py +206 -43
  13. webscout/Litlogger/__init__.py +681 -0
  14. webscout/Provider/DARKAI.py +1 -1
  15. webscout/Provider/EDITEE.py +1 -1
  16. webscout/Provider/NinjaChat.py +1 -1
  17. webscout/Provider/PI.py +221 -207
  18. webscout/Provider/Perplexity.py +598 -598
  19. webscout/Provider/RoboCoders.py +206 -0
  20. webscout/Provider/TTI/AiForce/__init__.py +22 -0
  21. webscout/Provider/TTI/AiForce/async_aiforce.py +257 -0
  22. webscout/Provider/TTI/AiForce/sync_aiforce.py +242 -0
  23. webscout/Provider/TTI/Nexra/__init__.py +22 -0
  24. webscout/Provider/TTI/Nexra/async_nexra.py +286 -0
  25. webscout/Provider/TTI/Nexra/sync_nexra.py +258 -0
  26. webscout/Provider/TTI/PollinationsAI/__init__.py +23 -0
  27. webscout/Provider/TTI/PollinationsAI/async_pollinations.py +330 -0
  28. webscout/Provider/TTI/PollinationsAI/sync_pollinations.py +285 -0
  29. webscout/Provider/TTI/__init__.py +2 -4
  30. webscout/Provider/TTI/artbit/__init__.py +22 -0
  31. webscout/Provider/TTI/artbit/async_artbit.py +184 -0
  32. webscout/Provider/TTI/artbit/sync_artbit.py +176 -0
  33. webscout/Provider/TTI/blackbox/__init__.py +4 -0
  34. webscout/Provider/TTI/blackbox/async_blackbox.py +212 -0
  35. webscout/Provider/TTI/{blackboximage.py → blackbox/sync_blackbox.py} +199 -153
  36. webscout/Provider/TTI/deepinfra/__init__.py +4 -0
  37. webscout/Provider/TTI/deepinfra/async_deepinfra.py +227 -0
  38. webscout/Provider/TTI/deepinfra/sync_deepinfra.py +199 -0
  39. webscout/Provider/TTI/huggingface/__init__.py +22 -0
  40. webscout/Provider/TTI/huggingface/async_huggingface.py +199 -0
  41. webscout/Provider/TTI/huggingface/sync_huggingface.py +195 -0
  42. webscout/Provider/TTI/imgninza/__init__.py +4 -0
  43. webscout/Provider/TTI/imgninza/async_ninza.py +214 -0
  44. webscout/Provider/TTI/{imgninza.py → imgninza/sync_ninza.py} +209 -136
  45. webscout/Provider/TTI/talkai/__init__.py +4 -0
  46. webscout/Provider/TTI/talkai/async_talkai.py +229 -0
  47. webscout/Provider/TTI/talkai/sync_talkai.py +207 -0
  48. webscout/Provider/__init__.py +146 -139
  49. webscout/Provider/askmyai.py +2 -2
  50. webscout/Provider/cerebras.py +227 -219
  51. webscout/Provider/llama3mitril.py +0 -1
  52. webscout/Provider/mhystical.py +176 -0
  53. webscout/Provider/perplexitylabs.py +265 -0
  54. webscout/Provider/twitterclone.py +251 -245
  55. webscout/Provider/typegpt.py +359 -0
  56. webscout/__init__.py +28 -23
  57. webscout/__main__.py +5 -5
  58. webscout/cli.py +252 -280
  59. webscout/conversation.py +227 -0
  60. webscout/exceptions.py +161 -29
  61. webscout/litagent/__init__.py +172 -0
  62. webscout/litprinter/__init__.py +831 -0
  63. webscout/optimizers.py +270 -0
  64. webscout/prompt_manager.py +279 -0
  65. webscout/swiftcli/__init__.py +810 -0
  66. webscout/transcriber.py +479 -551
  67. webscout/update_checker.py +125 -0
  68. webscout/version.py +1 -1
  69. {webscout-6.3.dist-info → webscout-6.4.dist-info}/METADATA +26 -45
  70. {webscout-6.3.dist-info → webscout-6.4.dist-info}/RECORD +75 -45
  71. webscout/Provider/TTI/AIuncensoredimage.py +0 -103
  72. webscout/Provider/TTI/Nexra.py +0 -120
  73. webscout/Provider/TTI/PollinationsAI.py +0 -138
  74. webscout/Provider/TTI/WebSimAI.py +0 -142
  75. webscout/Provider/TTI/aiforce.py +0 -160
  76. webscout/Provider/TTI/artbit.py +0 -141
  77. webscout/Provider/TTI/deepinfra.py +0 -148
  78. webscout/Provider/TTI/huggingface.py +0 -155
  79. webscout/Provider/TTI/talkai.py +0 -116
  80. webscout/models.py +0 -23
  81. /webscout/{g4f.py → gpt4free.py} +0 -0
  82. {webscout-6.3.dist-info → webscout-6.4.dist-info}/LICENSE.md +0 -0
  83. {webscout-6.3.dist-info → webscout-6.4.dist-info}/WHEEL +0 -0
  84. {webscout-6.3.dist-info → webscout-6.4.dist-info}/entry_points.txt +0 -0
  85. {webscout-6.3.dist-info → webscout-6.4.dist-info}/top_level.txt +0 -0
@@ -1,599 +1,599 @@
1
- import json
2
- import time
3
- from typing import Iterable, Dict, Any, Generator
4
-
5
- from os import listdir
6
- from uuid import uuid4
7
- from time import sleep, time
8
- from threading import Thread
9
- from json import loads, dumps
10
- from random import getrandbits
11
- from websocket import WebSocketApp
12
- from requests import Session, get, post
13
-
14
-
15
- from webscout.AIutel import Optimizers
16
- from webscout.AIutel import Conversation
17
- from webscout.AIutel import AwesomePrompts, sanitize_stream
18
- from webscout.AIbase import Provider, AsyncProvider
19
- from webscout import exceptions
20
-
21
-
22
- class Perplexity(Provider):
23
- def __init__(
24
- self,
25
- email: str = None,
26
- is_conversation: bool = True,
27
- max_tokens: int = 600,
28
- timeout: int = 30,
29
- intro: str = None,
30
- filepath: str = None,
31
- update_file: bool = True,
32
- proxies: dict = {},
33
- history_offset: int = 10250,
34
- act: str = None,
35
- quiet: bool = False,
36
- ) -> None:
37
- """Instantiates PERPLEXITY
38
-
39
- Args:
40
- email (str, optional): Your perplexity.ai email. Defaults to None.
41
- is_conversation (bool, optional): Flag for chatting conversationally. Defaults to True.
42
- max_tokens (int, optional): Maximum number of tokens to be generated upon completion. Defaults to 600.
43
- timeout (int, optional): Http request timeout. Defaults to 30.
44
- intro (str, optional): Conversation introductory prompt. Defaults to None.
45
- filepath (str, optional): Path to file containing conversation history. Defaults to None.
46
- update_file (bool, optional): Add new prompts and responses to the file. Defaults to True.
47
- proxies (dict, optional): Http request proxies. Defaults to {}.
48
- history_offset (int, optional): Limit conversation history to this number of last texts. Defaults to 10250.
49
- act (str|int, optional): Awesome prompt key or index. (Used as intro). Defaults to None.
50
- quiet (bool, optional): Ignore web search-results and yield final response only. Defaults to False.
51
- """
52
- self.max_tokens_to_sample = max_tokens
53
- self.is_conversation = is_conversation
54
- self.last_response = {}
55
- self.web_results: dict = {}
56
- self.quiet = quiet
57
-
58
- self.session: Session = Session()
59
- self.user_agent: dict = {
60
- "User-Agent": "Ask/2.9.1/2406 (iOS; iPhone; Version 17.1) isiOSOnMac/false",
61
- "X-Client-Name": "Perplexity-iOS",
62
- "X-App-ApiClient": "ios",
63
- }
64
- self.session.headers.update(self.user_agent)
65
-
66
- if email and ".perplexity_session" in listdir():
67
- self._recover_session(email)
68
- else:
69
- self._init_session_without_login()
70
-
71
- if email:
72
- self._login(email)
73
-
74
- self.email: str = email
75
- self.t: str = self._get_t()
76
- self.sid: str = self._get_sid()
77
-
78
- self.n: int = 1
79
- self.base: int = 420
80
- self.queue: list = []
81
- self.finished: bool = True
82
- self.last_uuid: str = None
83
- self.backend_uuid: str = (
84
- None # unused because we can't yet follow-up questions
85
- )
86
- self.frontend_session_id: str = str(uuid4())
87
-
88
- assert self._ask_anonymous_user(), "failed to ask anonymous user"
89
- self.ws: WebSocketApp = self._init_websocket()
90
- self.ws_thread: Thread = Thread(target=self.ws.run_forever).start()
91
- self._auth_session()
92
-
93
- while not (self.ws.sock and self.ws.sock.connected):
94
- sleep(0.01)
95
-
96
- self.__available_optimizers = (
97
- method
98
- for method in dir(Optimizers)
99
- if callable(getattr(Optimizers, method)) and not method.startswith("__")
100
- )
101
- Conversation.intro = (
102
- AwesomePrompts().get_act(
103
- act, raise_not_found=True, default=None, case_insensitive=True
104
- )
105
- if act
106
- else intro or Conversation.intro
107
- )
108
- self.conversation = Conversation(
109
- is_conversation, self.max_tokens_to_sample, filepath, update_file
110
- )
111
- self.conversation.history_offset = history_offset
112
- self.session.proxies = proxies
113
-
114
- def _recover_session(self, email: str) -> None:
115
- with open(".perplexity_session", "r") as f:
116
- perplexity_session: dict = loads(f.read())
117
-
118
- if email in perplexity_session:
119
- self.session.cookies.update(perplexity_session[email])
120
- else:
121
- self._login(email, perplexity_session)
122
-
123
- def _login(self, email: str, ps: dict = None) -> None:
124
- self.session.post(
125
- url="https://www.perplexity.ai/api/auth/signin-email",
126
- data={"email": email},
127
- )
128
-
129
- email_link: str = str(input("paste the link you received by email: "))
130
- self.session.get(email_link)
131
-
132
- if ps:
133
- ps[email] = self.session.cookies.get_dict()
134
- else:
135
- ps = {email: self.session.cookies.get_dict()}
136
-
137
- with open(".perplexity_session", "w") as f:
138
- f.write(dumps(ps))
139
-
140
- def _init_session_without_login(self) -> None:
141
- self.session.get(url=f"https://www.perplexity.ai/search/{str(uuid4())}")
142
- self.session.headers.update(self.user_agent)
143
-
144
- def _auth_session(self) -> None:
145
- self.session.get(url="https://www.perplexity.ai/api/auth/session")
146
-
147
- def _get_t(self) -> str:
148
- return format(getrandbits(32), "08x")
149
-
150
- def _get_sid(self) -> str:
151
- return loads(
152
- self.session.get(
153
- url=f"https://www.perplexity.ai/socket.io/?EIO=4&transport=polling&t={self.t}"
154
- ).text[1:]
155
- )["sid"]
156
-
157
- def _ask_anonymous_user(self) -> bool:
158
- response = self.session.post(
159
- url=f"https://www.perplexity.ai/socket.io/?EIO=4&transport=polling&t={self.t}&sid={self.sid}",
160
- data='40{"jwt":"anonymous-ask-user"}',
161
- ).text
162
-
163
- return response == "OK"
164
-
165
- def _start_interaction(self) -> None:
166
- self.finished = False
167
-
168
- if self.n == 9:
169
- self.n = 0
170
- self.base *= 10
171
- else:
172
- self.n += 1
173
-
174
- self.queue = []
175
-
176
- def _get_cookies_str(self) -> str:
177
- cookies = ""
178
- for key, value in self.session.cookies.get_dict().items():
179
- cookies += f"{key}={value}; "
180
- return cookies[:-2]
181
-
182
- def _write_file_url(self, filename: str, file_url: str) -> None:
183
- if ".perplexity_files_url" in listdir():
184
- with open(".perplexity_files_url", "r") as f:
185
- perplexity_files_url: dict = loads(f.read())
186
- else:
187
- perplexity_files_url: dict = {}
188
-
189
- perplexity_files_url[filename] = file_url
190
-
191
- with open(".perplexity_files_url", "w") as f:
192
- f.write(dumps(perplexity_files_url))
193
-
194
- def _init_websocket(self) -> WebSocketApp:
195
- def on_open(ws: WebSocketApp) -> None:
196
- ws.send("2probe")
197
- ws.send("5")
198
-
199
- def on_message(ws: WebSocketApp, message: str) -> None:
200
- if message == "2":
201
- ws.send("3")
202
- elif not self.finished:
203
- if message.startswith("42"):
204
- message: list = loads(message[2:])
205
- content: dict = message[1]
206
- if "mode" in content and content["mode"] == "copilot":
207
- content["copilot_answer"] = loads(content["text"])
208
- elif "mode" in content:
209
- content.update(loads(content["text"]))
210
- content.pop("text")
211
- if (
212
- not ("final" in content and content["final"])
213
- ) or ("status" in content and content["status"] == "completed"):
214
- self.queue.append(content)
215
- if message[0] == "query_answered":
216
- self.last_uuid = content["uuid"]
217
- self.finished = True
218
- elif message.startswith("43"):
219
- message: dict = loads(message[3:])[0]
220
- if (
221
- "uuid" in message and message["uuid"] != self.last_uuid
222
- ) or "uuid" not in message:
223
- self.queue.append(message)
224
- self.finished = True
225
-
226
- return WebSocketApp(
227
- url=f"wss://www.perplexity.ai/socket.io/?EIO=4&transport=websocket&sid={self.sid}",
228
- header=self.user_agent,
229
- cookie=self._get_cookies_str(),
230
- on_open=on_open,
231
- on_message=on_message,
232
- on_error=lambda ws, err: print(f"websocket error: {err}"),
233
- )
234
-
235
- def _s(
236
- self,
237
- query: str,
238
- mode: str = "concise",
239
- search_focus: str = "internet",
240
- attachments: list[str] = [],
241
- language: str = "en-GB",
242
- in_page: str = None,
243
- in_domain: str = None,
244
- ) -> None:
245
- assert self.finished, "already searching"
246
- assert mode in ["concise", "copilot"], "invalid mode"
247
- assert len(attachments) <= 4, "too many attachments: max 4"
248
- assert (
249
- search_focus
250
- in [
251
- "internet",
252
- "scholar",
253
- "writing",
254
- "wolfram",
255
- "youtube",
256
- "reddit",
257
- ]
258
- ), "invalid search focus"
259
-
260
- if in_page:
261
- search_focus = "in_page"
262
- if in_domain:
263
- search_focus = "in_domain"
264
-
265
- self._start_interaction()
266
- ws_message: str = (
267
- f"{self.base + self.n}"
268
- + dumps(
269
- [
270
- "perplexity_ask",
271
- query,
272
- {
273
- "version": "2.1",
274
- "source": "default", # "ios"
275
- "frontend_session_id": self.frontend_session_id,
276
- "language": language,
277
- "timezone": "CET",
278
- "attachments": attachments,
279
- "search_focus": search_focus,
280
- "frontend_uuid": str(uuid4()),
281
- "mode": mode,
282
- # "use_inhouse_model": True
283
- "in_page": in_page,
284
- "in_domain": in_domain,
285
- },
286
- ]
287
- )
288
- )
289
-
290
- self.ws.send(ws_message)
291
-
292
- def search(
293
- self,
294
- query: str,
295
- mode: str = "concise",
296
- search_focus: str = "internet",
297
- attachments: list[str] = [],
298
- language: str = "en-GB",
299
- timeout: float = 30,
300
- in_page: str = None,
301
- in_domain: str = None,
302
- ) -> Iterable[Dict]:
303
- self._s(query, mode, search_focus, attachments, language, in_page, in_domain)
304
-
305
- start_time: float = time()
306
- while (not self.finished) or len(self.queue) != 0:
307
- if timeout and time() - start_time > timeout:
308
- self.finished = True
309
- return {"error": "timeout"}
310
- if len(self.queue) != 0:
311
- yield self.queue.pop(0)
312
-
313
- def search_sync(
314
- self,
315
- query: str,
316
- mode: str = "concise",
317
- search_focus: str = "internet",
318
- attachments: list[str] = [],
319
- language: str = "en-GB",
320
- timeout: float = 30,
321
- in_page: str = None,
322
- in_domain: str = None,
323
- ) -> dict:
324
- self._s(query, mode, search_focus, attachments, language, in_page, in_domain)
325
-
326
- start_time: float = time()
327
- while not self.finished:
328
- if timeout and time() - start_time > timeout:
329
- self.finished = True
330
- return {"error": "timeout"}
331
-
332
- return self.queue.pop(-1)
333
-
334
- def upload(self, filename: str) -> str:
335
- assert self.finished, "already searching"
336
- assert filename.split(".")[-1] in [
337
- "txt",
338
- "pdf",
339
- ], "invalid file format"
340
-
341
- if filename.startswith("http"):
342
- file = get(filename).content
343
- else:
344
- with open(filename, "rb") as f:
345
- file = f.read()
346
-
347
- self._start_interaction()
348
- ws_message: str = (
349
- f"{self.base + self.n}"
350
- + dumps(
351
- [
352
- "get_upload_url",
353
- {
354
- "version": "2.1",
355
- "source": "default",
356
- "content_type": "text/plain"
357
- if filename.split(".")[-1] == "txt"
358
- else "application/pdf",
359
- },
360
- ]
361
- )
362
- )
363
-
364
- self.ws.send(ws_message)
365
-
366
- while not self.finished or len(self.queue) != 0:
367
- if len(self.queue) != 0:
368
- upload_data = self.queue.pop(0)
369
-
370
- assert not upload_data["rate_limited"], "rate limited"
371
-
372
- post(
373
- url=upload_data["url"],
374
- files={
375
- "acl": (None, upload_data["fields"]["acl"]),
376
- "Content-Type": (None, upload_data["fields"]["Content-Type"]),
377
- "key": (None, upload_data["fields"]["key"]),
378
- "AWSAccessKeyId": (None, upload_data["fields"]["AWSAccessKeyId"]),
379
- "x-amz-security-token": (
380
- None,
381
- upload_data["fields"]["x-amz-security-token"],
382
- ),
383
- "policy": (None, upload_data["fields"]["policy"]),
384
- "signature": (None, upload_data["fields"]["signature"]),
385
- "file": (filename, file),
386
- },
387
- )
388
-
389
- file_url: str = (
390
- upload_data["url"] + upload_data["fields"]["key"].split("$")[0] + filename
391
- )
392
-
393
- self._write_file_url(filename, file_url)
394
-
395
- return file_url
396
-
397
- def threads(self, query: str = None, limit: int = None) -> list[dict]:
398
- assert self.email, "not logged in"
399
- assert self.finished, "already searching"
400
-
401
- if not limit:
402
- limit = 20
403
- data: dict = {"version": "2.1", "source": "default", "limit": limit, "offset": 0}
404
- if query:
405
- data["search_term"] = query
406
-
407
- self._start_interaction()
408
- ws_message: str = f"{self.base + self.n}" + dumps(["list_ask_threads", data])
409
-
410
- self.ws.send(ws_message)
411
-
412
- while not self.finished or len(self.queue) != 0:
413
- if len(self.queue) != 0:
414
- return self.queue.pop(0)
415
-
416
- def list_autosuggest(self, query: str = "", search_focus: str = "internet") -> list[dict]:
417
- assert self.finished, "already searching"
418
-
419
- self._start_interaction()
420
- ws_message: str = (
421
- f"{self.base + self.n}"
422
- + dumps(
423
- [
424
- "list_autosuggest",
425
- query,
426
- {
427
- "has_attachment": False,
428
- "search_focus": search_focus,
429
- "source": "default",
430
- "version": "2.1",
431
- },
432
- ]
433
- )
434
- )
435
-
436
- self.ws.send(ws_message)
437
-
438
- while not self.finished or len(self.queue) != 0:
439
- if len(self.queue) != 0:
440
- return self.queue.pop(0)
441
-
442
- def close(self) -> None:
443
- self.ws.close()
444
-
445
- if self.email:
446
- with open(".perplexity_session", "r") as f:
447
- perplexity_session: dict = loads(f.read())
448
-
449
- perplexity_session[self.email] = self.session.cookies.get_dict()
450
-
451
- with open(".perplexity_session", "w") as f:
452
- f.write(dumps(perplexity_session))
453
-
454
- def ask(
455
- self,
456
- prompt: str,
457
- stream: bool = False,
458
- raw: bool = False,
459
- optimizer: str = None,
460
- conversationally: bool = False,
461
- ) -> dict | Generator:
462
- """Chat with AI
463
-
464
- Args:
465
- prompt (str): Prompt to be send.
466
- stream (bool, optional): Flag for streaming response. Defaults to False.
467
- raw (bool, optional): Stream back raw response as received. Defaults to False.
468
- optimizer (str, optional): Prompt optimizer name - `[code, shell_command]`. Defaults to None.
469
- conversationally (bool, optional): Chat conversationally when using optimizer. Defaults to False.
470
- Returns:
471
- dict : {}
472
- ```json
473
- {
474
- "status": "pending",
475
- "uuid": "3604dfcc-611f-4b7d-989d-edca2a7233c7",
476
- "read_write_token": null,
477
- "frontend_context_uuid": "f6d43119-5231-481d-b692-f52e1f52d2c6",
478
- "final": false,
479
- "backend_uuid": "a6d6ec9e-da69-4841-af74-0de0409267a8",
480
- "media_items": [],
481
- "widget_data": [],
482
- "knowledge_cards": [],
483
- "expect_search_results": "false",
484
- "mode": "concise",
485
- "search_focus": "internet",
486
- "gpt4": false,
487
- "display_model": "turbo",
488
- "attachments": null,
489
- "answer": "",
490
- "web_results": [],
491
- "chunks": [],
492
- "extra_web_results": []
493
- }
494
- ```
495
- """
496
- conversation_prompt = self.conversation.gen_complete_prompt(prompt)
497
- if optimizer:
498
- if optimizer in self.__available_optimizers:
499
- conversation_prompt = getattr(Optimizers, optimizer)(
500
- conversation_prompt if conversationally else prompt
501
- )
502
- else:
503
- raise Exception(
504
- f"Optimizer is not one of {self.__available_optimizers}"
505
- )
506
-
507
- def for_stream():
508
- for response in self.search(conversation_prompt):
509
- yield dumps(response) if raw else response
510
- self.last_response.update(response)
511
-
512
- self.conversation.update_chat_history(
513
- prompt, self.get_message(self.last_response)
514
- )
515
-
516
- def for_non_stream():
517
- self.last_response.update(self.search_sync(conversation_prompt))
518
- self.conversation.update_chat_history(
519
- prompt, self.get_message(self.last_response)
520
- )
521
- return self.last_response
522
-
523
- return for_stream() if stream else for_non_stream()
524
-
525
- def chat(
526
- self,
527
- prompt: str,
528
- stream: bool = False,
529
- optimizer: str = None,
530
- conversationally: bool = False,
531
- ) -> str | Generator:
532
- """Generate response `str`
533
- Args:
534
- prompt (str): Prompt to be send.
535
- stream (bool, optional): Flag for streaming response. Defaults to False.
536
- optimizer (str, optional): Prompt optimizer name - `[code, shell_command]`. Defaults to None.
537
- conversationally (bool, optional): Chat conversationally when using optimizer. Defaults to False.
538
- Returns:
539
- str: Response generated
540
- """
541
-
542
- def for_stream():
543
- for response in self.ask(
544
- prompt, True, optimizer=optimizer, conversationally=conversationally
545
- ):
546
- yield self.get_message(response)
547
-
548
- def for_non_stream():
549
- return self.get_message(
550
- self.ask(
551
- prompt,
552
- False,
553
- optimizer=optimizer,
554
- conversationally=conversationally,
555
- )
556
- )
557
-
558
- return for_stream() if stream else for_non_stream()
559
-
560
- def get_message(self, response: dict) -> str:
561
- """Retrieves message only from response
562
-
563
- Args:
564
- response (dict): Response generated by `self.ask`
565
-
566
- Returns:
567
- str: Message extracted
568
- """
569
- assert isinstance(response, dict), "Response should be of dict data-type only"
570
- text_str: str = response.get("answer", "")
571
-
572
- def update_web_results(web_results: list) -> None:
573
- for index, results in enumerate(web_results, start=1):
574
- self.web_results[str(index) + ". " + results["name"]] = dict(
575
- url=results.get("url"), snippet=results.get("snippet")
576
- )
577
-
578
- if response.get("text"):
579
- # last chunk
580
- target: dict[str, Any] = json.loads(response.get("text"))
581
- text_str = target.get("answer")
582
- web_results: list[dict] = target.get("web_results")
583
- self.web_results.clear()
584
- update_web_results(web_results)
585
-
586
- return text_str
587
-
588
- else:
589
- return text_str
590
-
591
-
592
- if __name__ == "__main__":
593
- perplexity = Perplexity()
594
- # Stream the response
595
- response = perplexity.chat("tell me about Abhay koul, HelpingAI ")
596
- for chunk in response:
597
- print(chunk, end="", flush=True)
598
-
1
+ import json
2
+ import time
3
+ from typing import Iterable, Dict, Any, Generator
4
+
5
+ from os import listdir
6
+ from uuid import uuid4
7
+ from time import sleep, time
8
+ from threading import Thread
9
+ from json import loads, dumps
10
+ from random import getrandbits
11
+ from websocket import WebSocketApp
12
+ from requests import Session, get, post
13
+
14
+
15
+ from webscout.AIutel import Optimizers
16
+ from webscout.AIutel import Conversation
17
+ from webscout.AIutel import AwesomePrompts, sanitize_stream
18
+ from webscout.AIbase import Provider, AsyncProvider
19
+ from webscout import exceptions
20
+
21
+
22
+ class Perplexity(Provider):
23
+ def __init__(
24
+ self,
25
+ email: str = None,
26
+ is_conversation: bool = True,
27
+ max_tokens: int = 600,
28
+ timeout: int = 30,
29
+ intro: str = None,
30
+ filepath: str = None,
31
+ update_file: bool = True,
32
+ proxies: dict = {},
33
+ history_offset: int = 10250,
34
+ act: str = None,
35
+ quiet: bool = False,
36
+ ) -> None:
37
+ """Instantiates PERPLEXITY
38
+
39
+ Args:
40
+ email (str, optional): Your perplexity.ai email. Defaults to None.
41
+ is_conversation (bool, optional): Flag for chatting conversationally. Defaults to True.
42
+ max_tokens (int, optional): Maximum number of tokens to be generated upon completion. Defaults to 600.
43
+ timeout (int, optional): Http request timeout. Defaults to 30.
44
+ intro (str, optional): Conversation introductory prompt. Defaults to None.
45
+ filepath (str, optional): Path to file containing conversation history. Defaults to None.
46
+ update_file (bool, optional): Add new prompts and responses to the file. Defaults to True.
47
+ proxies (dict, optional): Http request proxies. Defaults to {}.
48
+ history_offset (int, optional): Limit conversation history to this number of last texts. Defaults to 10250.
49
+ act (str|int, optional): Awesome prompt key or index. (Used as intro). Defaults to None.
50
+ quiet (bool, optional): Ignore web search-results and yield final response only. Defaults to False.
51
+ """
52
+ self.max_tokens_to_sample = max_tokens
53
+ self.is_conversation = is_conversation
54
+ self.last_response = {}
55
+ self.web_results: dict = {}
56
+ self.quiet = quiet
57
+
58
+ self.session: Session = Session()
59
+ self.user_agent: dict = {
60
+ "User-Agent": "Ask/2.9.1/2406 (iOS; iPhone; Version 17.1) isiOSOnMac/false",
61
+ "X-Client-Name": "Perplexity-iOS",
62
+ "X-App-ApiClient": "ios",
63
+ }
64
+ self.session.headers.update(self.user_agent)
65
+
66
+ if email and ".perplexity_session" in listdir():
67
+ self._recover_session(email)
68
+ else:
69
+ self._init_session_without_login()
70
+
71
+ if email:
72
+ self._login(email)
73
+
74
+ self.email: str = email
75
+ self.t: str = self._get_t()
76
+ self.sid: str = self._get_sid()
77
+
78
+ self.n: int = 1
79
+ self.base: int = 420
80
+ self.queue: list = []
81
+ self.finished: bool = True
82
+ self.last_uuid: str = None
83
+ self.backend_uuid: str = (
84
+ None # unused because we can't yet follow-up questions
85
+ )
86
+ self.frontend_session_id: str = str(uuid4())
87
+
88
+ assert self._ask_anonymous_user(), "failed to ask anonymous user"
89
+ self.ws: WebSocketApp = self._init_websocket()
90
+ self.ws_thread: Thread = Thread(target=self.ws.run_forever).start()
91
+ self._auth_session()
92
+
93
+ while not (self.ws.sock and self.ws.sock.connected):
94
+ sleep(0.01)
95
+
96
+ self.__available_optimizers = (
97
+ method
98
+ for method in dir(Optimizers)
99
+ if callable(getattr(Optimizers, method)) and not method.startswith("__")
100
+ )
101
+ Conversation.intro = (
102
+ AwesomePrompts().get_act(
103
+ act, raise_not_found=True, default=None, case_insensitive=True
104
+ )
105
+ if act
106
+ else intro or Conversation.intro
107
+ )
108
+ self.conversation = Conversation(
109
+ is_conversation, self.max_tokens_to_sample, filepath, update_file
110
+ )
111
+ self.conversation.history_offset = history_offset
112
+ self.session.proxies = proxies
113
+
114
+ def _recover_session(self, email: str) -> None:
115
+ with open(".perplexity_session", "r") as f:
116
+ perplexity_session: dict = loads(f.read())
117
+
118
+ if email in perplexity_session:
119
+ self.session.cookies.update(perplexity_session[email])
120
+ else:
121
+ self._login(email, perplexity_session)
122
+
123
+ def _login(self, email: str, ps: dict = None) -> None:
124
+ self.session.post(
125
+ url="https://www.perplexity.ai/api/auth/signin-email",
126
+ data={"email": email},
127
+ )
128
+
129
+ email_link: str = str(input("paste the link you received by email: "))
130
+ self.session.get(email_link)
131
+
132
+ if ps:
133
+ ps[email] = self.session.cookies.get_dict()
134
+ else:
135
+ ps = {email: self.session.cookies.get_dict()}
136
+
137
+ with open(".perplexity_session", "w") as f:
138
+ f.write(dumps(ps))
139
+
140
+ def _init_session_without_login(self) -> None:
141
+ self.session.get(url=f"https://www.perplexity.ai/search/{str(uuid4())}")
142
+ self.session.headers.update(self.user_agent)
143
+
144
+ def _auth_session(self) -> None:
145
+ self.session.get(url="https://www.perplexity.ai/api/auth/session")
146
+
147
+ def _get_t(self) -> str:
148
+ return format(getrandbits(32), "08x")
149
+
150
+ def _get_sid(self) -> str:
151
+ return loads(
152
+ self.session.get(
153
+ url=f"https://www.perplexity.ai/socket.io/?EIO=4&transport=polling&t={self.t}"
154
+ ).text[1:]
155
+ )["sid"]
156
+
157
+ def _ask_anonymous_user(self) -> bool:
158
+ response = self.session.post(
159
+ url=f"https://www.perplexity.ai/socket.io/?EIO=4&transport=polling&t={self.t}&sid={self.sid}",
160
+ data='40{"jwt":"anonymous-ask-user"}',
161
+ ).text
162
+
163
+ return response == "OK"
164
+
165
+ def _start_interaction(self) -> None:
166
+ self.finished = False
167
+
168
+ if self.n == 9:
169
+ self.n = 0
170
+ self.base *= 10
171
+ else:
172
+ self.n += 1
173
+
174
+ self.queue = []
175
+
176
+ def _get_cookies_str(self) -> str:
177
+ cookies = ""
178
+ for key, value in self.session.cookies.get_dict().items():
179
+ cookies += f"{key}={value}; "
180
+ return cookies[:-2]
181
+
182
+ def _write_file_url(self, filename: str, file_url: str) -> None:
183
+ if ".perplexity_files_url" in listdir():
184
+ with open(".perplexity_files_url", "r") as f:
185
+ perplexity_files_url: dict = loads(f.read())
186
+ else:
187
+ perplexity_files_url: dict = {}
188
+
189
+ perplexity_files_url[filename] = file_url
190
+
191
+ with open(".perplexity_files_url", "w") as f:
192
+ f.write(dumps(perplexity_files_url))
193
+
194
+ def _init_websocket(self) -> WebSocketApp:
195
+ def on_open(ws: WebSocketApp) -> None:
196
+ ws.send("2probe")
197
+ ws.send("5")
198
+
199
+ def on_message(ws: WebSocketApp, message: str) -> None:
200
+ if message == "2":
201
+ ws.send("3")
202
+ elif not self.finished:
203
+ if message.startswith("42"):
204
+ message: list = loads(message[2:])
205
+ content: dict = message[1]
206
+ if "mode" in content and content["mode"] == "copilot":
207
+ content["copilot_answer"] = loads(content["text"])
208
+ elif "mode" in content:
209
+ content.update(loads(content["text"]))
210
+ content.pop("text")
211
+ if (
212
+ not ("final" in content and content["final"])
213
+ ) or ("status" in content and content["status"] == "completed"):
214
+ self.queue.append(content)
215
+ if message[0] == "query_answered":
216
+ self.last_uuid = content["uuid"]
217
+ self.finished = True
218
+ elif message.startswith("43"):
219
+ message: dict = loads(message[3:])[0]
220
+ if (
221
+ "uuid" in message and message["uuid"] != self.last_uuid
222
+ ) or "uuid" not in message:
223
+ self.queue.append(message)
224
+ self.finished = True
225
+
226
+ return WebSocketApp(
227
+ url=f"wss://www.perplexity.ai/socket.io/?EIO=4&transport=websocket&sid={self.sid}",
228
+ header=self.user_agent,
229
+ cookie=self._get_cookies_str(),
230
+ on_open=on_open,
231
+ on_message=on_message,
232
+ on_error=lambda ws, err: print(f"websocket error: {err}"),
233
+ )
234
+
235
+ def _s(
236
+ self,
237
+ query: str,
238
+ mode: str = "concise",
239
+ search_focus: str = "internet",
240
+ attachments: list[str] = [],
241
+ language: str = "en-GB",
242
+ in_page: str = None,
243
+ in_domain: str = None,
244
+ ) -> None:
245
+ assert self.finished, "already searching"
246
+ assert mode in ["concise", "copilot"], "invalid mode"
247
+ assert len(attachments) <= 4, "too many attachments: max 4"
248
+ assert (
249
+ search_focus
250
+ in [
251
+ "internet",
252
+ "scholar",
253
+ "writing",
254
+ "wolfram",
255
+ "youtube",
256
+ "reddit",
257
+ ]
258
+ ), "invalid search focus"
259
+
260
+ if in_page:
261
+ search_focus = "in_page"
262
+ if in_domain:
263
+ search_focus = "in_domain"
264
+
265
+ self._start_interaction()
266
+ ws_message: str = (
267
+ f"{self.base + self.n}"
268
+ + dumps(
269
+ [
270
+ "perplexity_ask",
271
+ query,
272
+ {
273
+ "version": "2.1",
274
+ "source": "default", # "ios"
275
+ "frontend_session_id": self.frontend_session_id,
276
+ "language": language,
277
+ "timezone": "CET",
278
+ "attachments": attachments,
279
+ "search_focus": search_focus,
280
+ "frontend_uuid": str(uuid4()),
281
+ "mode": mode,
282
+ # "use_inhouse_model": True
283
+ "in_page": in_page,
284
+ "in_domain": in_domain,
285
+ },
286
+ ]
287
+ )
288
+ )
289
+
290
+ self.ws.send(ws_message)
291
+
292
+ def search(
293
+ self,
294
+ query: str,
295
+ mode: str = "concise",
296
+ search_focus: str = "internet",
297
+ attachments: list[str] = [],
298
+ language: str = "en-GB",
299
+ timeout: float = 30,
300
+ in_page: str = None,
301
+ in_domain: str = None,
302
+ ) -> Iterable[Dict]:
303
+ self._s(query, mode, search_focus, attachments, language, in_page, in_domain)
304
+
305
+ start_time: float = time()
306
+ while (not self.finished) or len(self.queue) != 0:
307
+ if timeout and time() - start_time > timeout:
308
+ self.finished = True
309
+ return {"error": "timeout"}
310
+ if len(self.queue) != 0:
311
+ yield self.queue.pop(0)
312
+
313
+ def search_sync(
314
+ self,
315
+ query: str,
316
+ mode: str = "concise",
317
+ search_focus: str = "internet",
318
+ attachments: list[str] = [],
319
+ language: str = "en-GB",
320
+ timeout: float = 30,
321
+ in_page: str = None,
322
+ in_domain: str = None,
323
+ ) -> dict:
324
+ self._s(query, mode, search_focus, attachments, language, in_page, in_domain)
325
+
326
+ start_time: float = time()
327
+ while not self.finished:
328
+ if timeout and time() - start_time > timeout:
329
+ self.finished = True
330
+ return {"error": "timeout"}
331
+
332
+ return self.queue.pop(-1)
333
+
334
+ def upload(self, filename: str) -> str:
335
+ assert self.finished, "already searching"
336
+ assert filename.split(".")[-1] in [
337
+ "txt",
338
+ "pdf",
339
+ ], "invalid file format"
340
+
341
+ if filename.startswith("http"):
342
+ file = get(filename).content
343
+ else:
344
+ with open(filename, "rb") as f:
345
+ file = f.read()
346
+
347
+ self._start_interaction()
348
+ ws_message: str = (
349
+ f"{self.base + self.n}"
350
+ + dumps(
351
+ [
352
+ "get_upload_url",
353
+ {
354
+ "version": "2.1",
355
+ "source": "default",
356
+ "content_type": "text/plain"
357
+ if filename.split(".")[-1] == "txt"
358
+ else "application/pdf",
359
+ },
360
+ ]
361
+ )
362
+ )
363
+
364
+ self.ws.send(ws_message)
365
+
366
+ while not self.finished or len(self.queue) != 0:
367
+ if len(self.queue) != 0:
368
+ upload_data = self.queue.pop(0)
369
+
370
+ assert not upload_data["rate_limited"], "rate limited"
371
+
372
+ post(
373
+ url=upload_data["url"],
374
+ files={
375
+ "acl": (None, upload_data["fields"]["acl"]),
376
+ "Content-Type": (None, upload_data["fields"]["Content-Type"]),
377
+ "key": (None, upload_data["fields"]["key"]),
378
+ "AWSAccessKeyId": (None, upload_data["fields"]["AWSAccessKeyId"]),
379
+ "x-amz-security-token": (
380
+ None,
381
+ upload_data["fields"]["x-amz-security-token"],
382
+ ),
383
+ "policy": (None, upload_data["fields"]["policy"]),
384
+ "signature": (None, upload_data["fields"]["signature"]),
385
+ "file": (filename, file),
386
+ },
387
+ )
388
+
389
+ file_url: str = (
390
+ upload_data["url"] + upload_data["fields"]["key"].split("$")[0] + filename
391
+ )
392
+
393
+ self._write_file_url(filename, file_url)
394
+
395
+ return file_url
396
+
397
+ def threads(self, query: str = None, limit: int = None) -> list[dict]:
398
+ assert self.email, "not logged in"
399
+ assert self.finished, "already searching"
400
+
401
+ if not limit:
402
+ limit = 20
403
+ data: dict = {"version": "2.1", "source": "default", "limit": limit, "offset": 0}
404
+ if query:
405
+ data["search_term"] = query
406
+
407
+ self._start_interaction()
408
+ ws_message: str = f"{self.base + self.n}" + dumps(["list_ask_threads", data])
409
+
410
+ self.ws.send(ws_message)
411
+
412
+ while not self.finished or len(self.queue) != 0:
413
+ if len(self.queue) != 0:
414
+ return self.queue.pop(0)
415
+
416
+ def list_autosuggest(self, query: str = "", search_focus: str = "internet") -> list[dict]:
417
+ assert self.finished, "already searching"
418
+
419
+ self._start_interaction()
420
+ ws_message: str = (
421
+ f"{self.base + self.n}"
422
+ + dumps(
423
+ [
424
+ "list_autosuggest",
425
+ query,
426
+ {
427
+ "has_attachment": False,
428
+ "search_focus": search_focus,
429
+ "source": "default",
430
+ "version": "2.1",
431
+ },
432
+ ]
433
+ )
434
+ )
435
+
436
+ self.ws.send(ws_message)
437
+
438
+ while not self.finished or len(self.queue) != 0:
439
+ if len(self.queue) != 0:
440
+ return self.queue.pop(0)
441
+
442
+ def close(self) -> None:
443
+ self.ws.close()
444
+
445
+ if self.email:
446
+ with open(".perplexity_session", "r") as f:
447
+ perplexity_session: dict = loads(f.read())
448
+
449
+ perplexity_session[self.email] = self.session.cookies.get_dict()
450
+
451
+ with open(".perplexity_session", "w") as f:
452
+ f.write(dumps(perplexity_session))
453
+
454
+ def ask(
455
+ self,
456
+ prompt: str,
457
+ stream: bool = False,
458
+ raw: bool = False,
459
+ optimizer: str = None,
460
+ conversationally: bool = False,
461
+ ) -> dict | Generator:
462
+ """Chat with AI
463
+
464
+ Args:
465
+ prompt (str): Prompt to be send.
466
+ stream (bool, optional): Flag for streaming response. Defaults to False.
467
+ raw (bool, optional): Stream back raw response as received. Defaults to False.
468
+ optimizer (str, optional): Prompt optimizer name - `[code, shell_command]`. Defaults to None.
469
+ conversationally (bool, optional): Chat conversationally when using optimizer. Defaults to False.
470
+ Returns:
471
+ dict : {}
472
+ ```json
473
+ {
474
+ "status": "pending",
475
+ "uuid": "3604dfcc-611f-4b7d-989d-edca2a7233c7",
476
+ "read_write_token": null,
477
+ "frontend_context_uuid": "f6d43119-5231-481d-b692-f52e1f52d2c6",
478
+ "final": false,
479
+ "backend_uuid": "a6d6ec9e-da69-4841-af74-0de0409267a8",
480
+ "media_items": [],
481
+ "widget_data": [],
482
+ "knowledge_cards": [],
483
+ "expect_search_results": "false",
484
+ "mode": "concise",
485
+ "search_focus": "internet",
486
+ "gpt4": false,
487
+ "display_model": "turbo",
488
+ "attachments": null,
489
+ "answer": "",
490
+ "web_results": [],
491
+ "chunks": [],
492
+ "extra_web_results": []
493
+ }
494
+ ```
495
+ """
496
+ conversation_prompt = self.conversation.gen_complete_prompt(prompt)
497
+ if optimizer:
498
+ if optimizer in self.__available_optimizers:
499
+ conversation_prompt = getattr(Optimizers, optimizer)(
500
+ conversation_prompt if conversationally else prompt
501
+ )
502
+ else:
503
+ raise Exception(
504
+ f"Optimizer is not one of {self.__available_optimizers}"
505
+ )
506
+
507
+ def for_stream():
508
+ for response in self.search(conversation_prompt):
509
+ yield dumps(response) if raw else response
510
+ self.last_response.update(response)
511
+
512
+ self.conversation.update_chat_history(
513
+ prompt, self.get_message(self.last_response)
514
+ )
515
+
516
+ def for_non_stream():
517
+ self.last_response.update(self.search_sync(conversation_prompt))
518
+ self.conversation.update_chat_history(
519
+ prompt, self.get_message(self.last_response)
520
+ )
521
+ return self.last_response
522
+
523
+ return for_stream() if stream else for_non_stream()
524
+
525
+ def chat(
526
+ self,
527
+ prompt: str,
528
+ stream: bool = False,
529
+ optimizer: str = None,
530
+ conversationally: bool = False,
531
+ ) -> str | Generator:
532
+ """Generate response `str`
533
+ Args:
534
+ prompt (str): Prompt to be send.
535
+ stream (bool, optional): Flag for streaming response. Defaults to False.
536
+ optimizer (str, optional): Prompt optimizer name - `[code, shell_command]`. Defaults to None.
537
+ conversationally (bool, optional): Chat conversationally when using optimizer. Defaults to False.
538
+ Returns:
539
+ str: Response generated
540
+ """
541
+
542
+ def for_stream():
543
+ for response in self.ask(
544
+ prompt, True, optimizer=optimizer, conversationally=conversationally
545
+ ):
546
+ yield self.get_message(response)
547
+
548
+ def for_non_stream():
549
+ return self.get_message(
550
+ self.ask(
551
+ prompt,
552
+ False,
553
+ optimizer=optimizer,
554
+ conversationally=conversationally,
555
+ )
556
+ )
557
+
558
+ return for_stream() if stream else for_non_stream()
559
+
560
+ def get_message(self, response: dict) -> str:
561
+ """Retrieves message only from response
562
+
563
+ Args:
564
+ response (dict): Response generated by `self.ask`
565
+
566
+ Returns:
567
+ str: Message extracted
568
+ """
569
+ assert isinstance(response, dict), "Response should be of dict data-type only"
570
+ text_str: str = response.get("answer", "")
571
+
572
+ def update_web_results(web_results: list) -> None:
573
+ for index, results in enumerate(web_results, start=1):
574
+ self.web_results[str(index) + ". " + results["name"]] = dict(
575
+ url=results.get("url"), snippet=results.get("snippet")
576
+ )
577
+
578
+ if response.get("text"):
579
+ # last chunk
580
+ target: dict[str, Any] = json.loads(response.get("text"))
581
+ text_str = target.get("answer")
582
+ web_results: list[dict] = target.get("web_results")
583
+ self.web_results.clear()
584
+ update_web_results(web_results)
585
+
586
+ return text_str
587
+
588
+ else:
589
+ return text_str
590
+
591
+
592
+ if __name__ == "__main__":
593
+ perplexity = Perplexity()
594
+ # Stream the response
595
+ response = perplexity.chat("tell me about Abhay koul, HelpingAI ")
596
+ for chunk in response:
597
+ print(chunk, end="", flush=True)
598
+
599
599
  perplexity.close()