symbolicai 0.21.0__py3-none-any.whl → 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- symai/__init__.py +269 -173
- symai/backend/base.py +123 -110
- symai/backend/engines/drawing/engine_bfl.py +45 -44
- symai/backend/engines/drawing/engine_gpt_image.py +112 -97
- symai/backend/engines/embedding/engine_llama_cpp.py +63 -52
- symai/backend/engines/embedding/engine_openai.py +25 -21
- symai/backend/engines/execute/engine_python.py +19 -18
- symai/backend/engines/files/engine_io.py +104 -95
- symai/backend/engines/imagecaptioning/engine_blip2.py +28 -24
- symai/backend/engines/imagecaptioning/engine_llavacpp_client.py +102 -79
- symai/backend/engines/index/engine_pinecone.py +124 -97
- symai/backend/engines/index/engine_qdrant.py +1011 -0
- symai/backend/engines/index/engine_vectordb.py +84 -56
- symai/backend/engines/lean/engine_lean4.py +96 -52
- symai/backend/engines/neurosymbolic/__init__.py +41 -13
- symai/backend/engines/neurosymbolic/engine_anthropic_claudeX_chat.py +330 -248
- symai/backend/engines/neurosymbolic/engine_anthropic_claudeX_reasoning.py +329 -264
- symai/backend/engines/neurosymbolic/engine_cerebras.py +328 -0
- symai/backend/engines/neurosymbolic/engine_deepseekX_reasoning.py +118 -88
- symai/backend/engines/neurosymbolic/engine_google_geminiX_reasoning.py +344 -299
- symai/backend/engines/neurosymbolic/engine_groq.py +173 -115
- symai/backend/engines/neurosymbolic/engine_huggingface.py +114 -84
- symai/backend/engines/neurosymbolic/engine_llama_cpp.py +144 -118
- symai/backend/engines/neurosymbolic/engine_openai_gptX_chat.py +415 -307
- symai/backend/engines/neurosymbolic/engine_openai_gptX_reasoning.py +394 -231
- symai/backend/engines/ocr/engine_apilayer.py +23 -27
- symai/backend/engines/output/engine_stdout.py +10 -13
- symai/backend/engines/{webscraping → scrape}/engine_requests.py +101 -54
- symai/backend/engines/search/engine_openai.py +100 -88
- symai/backend/engines/search/engine_parallel.py +665 -0
- symai/backend/engines/search/engine_perplexity.py +44 -45
- symai/backend/engines/search/engine_serpapi.py +37 -34
- symai/backend/engines/speech_to_text/engine_local_whisper.py +54 -51
- symai/backend/engines/symbolic/engine_wolframalpha.py +15 -9
- symai/backend/engines/text_to_speech/engine_openai.py +20 -26
- symai/backend/engines/text_vision/engine_clip.py +39 -37
- symai/backend/engines/userinput/engine_console.py +5 -6
- symai/backend/mixin/__init__.py +13 -0
- symai/backend/mixin/anthropic.py +48 -38
- symai/backend/mixin/deepseek.py +6 -5
- symai/backend/mixin/google.py +7 -4
- symai/backend/mixin/groq.py +2 -4
- symai/backend/mixin/openai.py +140 -110
- symai/backend/settings.py +87 -20
- symai/chat.py +216 -123
- symai/collect/__init__.py +7 -1
- symai/collect/dynamic.py +80 -70
- symai/collect/pipeline.py +67 -51
- symai/collect/stats.py +161 -109
- symai/components.py +707 -360
- symai/constraints.py +24 -12
- symai/core.py +1857 -1233
- symai/core_ext.py +83 -80
- symai/endpoints/api.py +166 -104
- symai/extended/.DS_Store +0 -0
- symai/extended/__init__.py +46 -12
- symai/extended/api_builder.py +29 -21
- symai/extended/arxiv_pdf_parser.py +23 -14
- symai/extended/bibtex_parser.py +9 -6
- symai/extended/conversation.py +156 -126
- symai/extended/document.py +50 -30
- symai/extended/file_merger.py +57 -14
- symai/extended/graph.py +51 -32
- symai/extended/html_style_template.py +18 -14
- symai/extended/interfaces/blip_2.py +2 -3
- symai/extended/interfaces/clip.py +4 -3
- symai/extended/interfaces/console.py +9 -1
- symai/extended/interfaces/dall_e.py +4 -2
- symai/extended/interfaces/file.py +2 -0
- symai/extended/interfaces/flux.py +4 -2
- symai/extended/interfaces/gpt_image.py +16 -7
- symai/extended/interfaces/input.py +2 -1
- symai/extended/interfaces/llava.py +1 -2
- symai/extended/interfaces/{naive_webscraping.py → naive_scrape.py} +4 -3
- symai/extended/interfaces/naive_vectordb.py +9 -10
- symai/extended/interfaces/ocr.py +5 -3
- symai/extended/interfaces/openai_search.py +2 -0
- symai/extended/interfaces/parallel.py +30 -0
- symai/extended/interfaces/perplexity.py +2 -0
- symai/extended/interfaces/pinecone.py +12 -9
- symai/extended/interfaces/python.py +2 -0
- symai/extended/interfaces/serpapi.py +3 -1
- symai/extended/interfaces/terminal.py +2 -4
- symai/extended/interfaces/tts.py +3 -2
- symai/extended/interfaces/whisper.py +3 -2
- symai/extended/interfaces/wolframalpha.py +2 -1
- symai/extended/metrics/__init__.py +11 -1
- symai/extended/metrics/similarity.py +14 -13
- symai/extended/os_command.py +39 -29
- symai/extended/packages/__init__.py +29 -3
- symai/extended/packages/symdev.py +51 -43
- symai/extended/packages/sympkg.py +41 -35
- symai/extended/packages/symrun.py +63 -50
- symai/extended/repo_cloner.py +14 -12
- symai/extended/seo_query_optimizer.py +15 -13
- symai/extended/solver.py +116 -91
- symai/extended/summarizer.py +12 -10
- symai/extended/taypan_interpreter.py +17 -18
- symai/extended/vectordb.py +122 -92
- symai/formatter/__init__.py +9 -1
- symai/formatter/formatter.py +51 -47
- symai/formatter/regex.py +70 -69
- symai/functional.py +325 -176
- symai/imports.py +190 -147
- symai/interfaces.py +57 -28
- symai/memory.py +45 -35
- symai/menu/screen.py +28 -19
- symai/misc/console.py +66 -56
- symai/misc/loader.py +8 -5
- symai/models/__init__.py +17 -1
- symai/models/base.py +395 -236
- symai/models/errors.py +1 -2
- symai/ops/__init__.py +32 -22
- symai/ops/measures.py +24 -25
- symai/ops/primitives.py +1149 -731
- symai/post_processors.py +58 -50
- symai/pre_processors.py +86 -82
- symai/processor.py +21 -13
- symai/prompts.py +764 -685
- symai/server/huggingface_server.py +135 -49
- symai/server/llama_cpp_server.py +21 -11
- symai/server/qdrant_server.py +206 -0
- symai/shell.py +100 -42
- symai/shellsv.py +700 -492
- symai/strategy.py +630 -346
- symai/symbol.py +368 -322
- symai/utils.py +100 -78
- {symbolicai-0.21.0.dist-info → symbolicai-1.1.0.dist-info}/METADATA +22 -10
- symbolicai-1.1.0.dist-info/RECORD +168 -0
- symbolicai-0.21.0.dist-info/RECORD +0 -162
- {symbolicai-0.21.0.dist-info → symbolicai-1.1.0.dist-info}/WHEEL +0 -0
- {symbolicai-0.21.0.dist-info → symbolicai-1.1.0.dist-info}/entry_points.txt +0 -0
- {symbolicai-0.21.0.dist-info → symbolicai-1.1.0.dist-info}/licenses/LICENSE +0 -0
- {symbolicai-0.21.0.dist-info → symbolicai-1.1.0.dist-info}/top_level.txt +0 -0
|
@@ -4,7 +4,7 @@ import logging
|
|
|
4
4
|
import requests
|
|
5
5
|
|
|
6
6
|
from ....symbol import Result
|
|
7
|
-
from ....utils import
|
|
7
|
+
from ....utils import UserMessage
|
|
8
8
|
from ...base import Engine
|
|
9
9
|
from ...settings import SYMAI_CONFIG
|
|
10
10
|
|
|
@@ -17,13 +17,13 @@ logging.getLogger("httpcore").setLevel(logging.ERROR)
|
|
|
17
17
|
class SearchResult(Result):
|
|
18
18
|
def __init__(self, value, **kwargs) -> None:
|
|
19
19
|
super().__init__(value, **kwargs)
|
|
20
|
-
if value.get(
|
|
21
|
-
|
|
20
|
+
if value.get("error"):
|
|
21
|
+
UserMessage(value["error"], raise_with=ValueError)
|
|
22
22
|
try:
|
|
23
|
-
self._value = value[
|
|
23
|
+
self._value = value["choices"][0]["message"]["content"]
|
|
24
24
|
except Exception as e:
|
|
25
25
|
self._value = None
|
|
26
|
-
|
|
26
|
+
UserMessage(f"Failed to parse response: {e}", raise_with=ValueError)
|
|
27
27
|
|
|
28
28
|
def __str__(self) -> str:
|
|
29
29
|
try:
|
|
@@ -35,80 +35,79 @@ class SearchResult(Result):
|
|
|
35
35
|
try:
|
|
36
36
|
return f"<pre>{json.dumps(self.raw, indent=2)}</pre>"
|
|
37
37
|
except TypeError:
|
|
38
|
-
return f"<pre>{
|
|
38
|
+
return f"<pre>{self.raw!s}</pre>"
|
|
39
39
|
|
|
40
40
|
|
|
41
41
|
class PerplexityEngine(Engine):
|
|
42
42
|
def __init__(self):
|
|
43
43
|
super().__init__()
|
|
44
44
|
self.config = SYMAI_CONFIG
|
|
45
|
-
self.api_key = self.config[
|
|
46
|
-
self.model = self.config[
|
|
45
|
+
self.api_key = self.config["SEARCH_ENGINE_API_KEY"]
|
|
46
|
+
self.model = self.config["SEARCH_ENGINE_MODEL"]
|
|
47
47
|
self.name = self.__class__.__name__
|
|
48
48
|
|
|
49
49
|
def id(self) -> str:
|
|
50
|
-
if self.config.get(
|
|
51
|
-
|
|
50
|
+
if self.config.get("SEARCH_ENGINE_API_KEY") and self.config.get(
|
|
51
|
+
"SEARCH_ENGINE_MODEL"
|
|
52
|
+
).startswith("sonar"):
|
|
53
|
+
return "search"
|
|
52
54
|
return super().id() # default to unregistered
|
|
53
55
|
|
|
54
56
|
def command(self, *args, **kwargs):
|
|
55
57
|
super().command(*args, **kwargs)
|
|
56
|
-
if
|
|
57
|
-
self.api_key = kwargs[
|
|
58
|
-
if
|
|
59
|
-
self.model = kwargs[
|
|
58
|
+
if "SEARCH_ENGINE_API_KEY" in kwargs:
|
|
59
|
+
self.api_key = kwargs["SEARCH_ENGINE_API_KEY"]
|
|
60
|
+
if "SEARCH_ENGINE_MODEL" in kwargs:
|
|
61
|
+
self.model = kwargs["SEARCH_ENGINE_MODEL"]
|
|
60
62
|
|
|
61
63
|
def forward(self, argument):
|
|
62
|
-
messages
|
|
63
|
-
kwargs
|
|
64
|
+
messages = argument.prop.prepared_input
|
|
65
|
+
kwargs = argument.kwargs
|
|
64
66
|
|
|
65
67
|
payload = {
|
|
66
68
|
"model": self.model,
|
|
67
69
|
"messages": messages,
|
|
68
|
-
"max_tokens": kwargs.get(
|
|
69
|
-
"temperature": kwargs.get(
|
|
70
|
-
"top_p": kwargs.get(
|
|
71
|
-
"search_domain_filter": kwargs.get(
|
|
72
|
-
"return_images": kwargs.get(
|
|
73
|
-
"return_related_questions": kwargs.get(
|
|
74
|
-
"search_recency_filter": kwargs.get(
|
|
75
|
-
"top_k": kwargs.get(
|
|
76
|
-
"stream": kwargs.get(
|
|
77
|
-
"presence_penalty": kwargs.get(
|
|
78
|
-
"frequency_penalty": kwargs.get(
|
|
79
|
-
"response_format": kwargs.get(
|
|
70
|
+
"max_tokens": kwargs.get("max_tokens", None),
|
|
71
|
+
"temperature": kwargs.get("temperature", 0.2),
|
|
72
|
+
"top_p": kwargs.get("top_p", 0.9),
|
|
73
|
+
"search_domain_filter": kwargs.get("search_domain_filter", []),
|
|
74
|
+
"return_images": kwargs.get("return_images", False),
|
|
75
|
+
"return_related_questions": kwargs.get("return_related_questions", False),
|
|
76
|
+
"search_recency_filter": kwargs.get("search_recency_filter", "month"),
|
|
77
|
+
"top_k": kwargs.get("top_k", 0),
|
|
78
|
+
"stream": kwargs.get("stream", False),
|
|
79
|
+
"presence_penalty": kwargs.get("presence_penalty", 0),
|
|
80
|
+
"frequency_penalty": kwargs.get("frequency_penalty", 1),
|
|
81
|
+
"response_format": kwargs.get("response_format", None),
|
|
80
82
|
}
|
|
81
|
-
web_search_options = kwargs.get(
|
|
83
|
+
web_search_options = kwargs.get("web_search_options", None)
|
|
82
84
|
if web_search_options is not None:
|
|
83
85
|
payload["web_search_options"] = web_search_options
|
|
84
86
|
|
|
85
|
-
headers = {
|
|
86
|
-
"Authorization": f"Bearer {self.api_key}",
|
|
87
|
-
"Content-Type": "application/json"
|
|
88
|
-
}
|
|
87
|
+
headers = {"Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json"}
|
|
89
88
|
|
|
90
89
|
try:
|
|
91
|
-
res = requests.post(
|
|
90
|
+
res = requests.post(
|
|
91
|
+
"https://api.perplexity.ai/chat/completions", json=payload, headers=headers
|
|
92
|
+
)
|
|
92
93
|
res = SearchResult(res.json())
|
|
93
94
|
except Exception as e:
|
|
94
|
-
|
|
95
|
+
UserMessage(f"Failed to make request: {e}", raise_with=ValueError)
|
|
95
96
|
|
|
96
97
|
metadata = {"raw_output": res.raw}
|
|
97
|
-
output
|
|
98
|
+
output = [res]
|
|
98
99
|
|
|
99
100
|
return output, metadata
|
|
100
101
|
|
|
101
102
|
def prepare(self, argument):
|
|
102
|
-
system_message =
|
|
103
|
+
system_message = (
|
|
104
|
+
"You are a helpful AI assistant. Be precise and informative."
|
|
105
|
+
if argument.kwargs.get("system_message") is None
|
|
106
|
+
else argument.kwargs.get("system_message")
|
|
107
|
+
)
|
|
103
108
|
|
|
104
109
|
res = [
|
|
105
|
-
{
|
|
106
|
-
|
|
107
|
-
"content": system_message
|
|
108
|
-
},
|
|
109
|
-
{
|
|
110
|
-
"role": "user",
|
|
111
|
-
"content": f"{argument.prop.query}"
|
|
112
|
-
}
|
|
110
|
+
{"role": "system", "content": system_message},
|
|
111
|
+
{"role": "user", "content": f"{argument.prop.query}"},
|
|
113
112
|
]
|
|
114
113
|
argument.prop.prepared_input = res
|
|
@@ -3,34 +3,34 @@ import json
|
|
|
3
3
|
from IPython.utils import io
|
|
4
4
|
|
|
5
5
|
from ....symbol import Result
|
|
6
|
-
from ....utils import
|
|
6
|
+
from ....utils import UserMessage
|
|
7
7
|
from ...base import Engine
|
|
8
8
|
from ...settings import SYMAI_CONFIG
|
|
9
9
|
|
|
10
10
|
try:
|
|
11
11
|
from serpapi import GoogleSearch
|
|
12
|
-
except:
|
|
12
|
+
except ImportError:
|
|
13
13
|
GoogleSearch = None
|
|
14
14
|
|
|
15
15
|
|
|
16
16
|
class SearchResult(Result):
|
|
17
17
|
def __init__(self, value, **kwargs) -> None:
|
|
18
18
|
super().__init__(value, **kwargs)
|
|
19
|
-
if
|
|
20
|
-
self._value = value[
|
|
21
|
-
elif
|
|
22
|
-
self._value = value[
|
|
23
|
-
elif
|
|
24
|
-
self._value = value[
|
|
25
|
-
elif
|
|
26
|
-
self._value = value["organic_results"][0][
|
|
19
|
+
if "answer_box" in value and "answer" in value["answer_box"]:
|
|
20
|
+
self._value = value["answer_box"]["answer"]
|
|
21
|
+
elif "answer_box" in value and "snippet" in value["answer_box"]:
|
|
22
|
+
self._value = value["answer_box"]["snippet"]
|
|
23
|
+
elif "answer_box" in value and "snippet_highlighted_words" in value["answer_box"]:
|
|
24
|
+
self._value = value["answer_box"]["snippet_highlighted_words"][0]
|
|
25
|
+
elif "organic_results" in value and "snippet" in value["organic_results"][0]:
|
|
26
|
+
self._value = value["organic_results"][0]["snippet"]
|
|
27
27
|
else:
|
|
28
28
|
self._value = value
|
|
29
29
|
|
|
30
|
-
if
|
|
31
|
-
self.results = value[
|
|
30
|
+
if "organic_results" in value:
|
|
31
|
+
self.results = value["organic_results"]
|
|
32
32
|
if len(self.results) > 0:
|
|
33
|
-
self.links = [r[
|
|
33
|
+
self.links = [r["link"] for r in self.results]
|
|
34
34
|
else:
|
|
35
35
|
self.links = []
|
|
36
36
|
else:
|
|
@@ -38,42 +38,45 @@ class SearchResult(Result):
|
|
|
38
38
|
self.links = []
|
|
39
39
|
|
|
40
40
|
def __str__(self) -> str:
|
|
41
|
-
|
|
42
|
-
return json_str
|
|
41
|
+
return json.dumps(self.raw, indent=2)
|
|
43
42
|
|
|
44
43
|
def _repr_html_(self) -> str:
|
|
45
|
-
|
|
46
|
-
return json_str
|
|
44
|
+
return json.dumps(self.raw, indent=2)
|
|
47
45
|
|
|
48
46
|
|
|
49
47
|
class SerpApiEngine(Engine):
|
|
50
48
|
def __init__(self):
|
|
51
49
|
super().__init__()
|
|
52
50
|
self.config = SYMAI_CONFIG
|
|
53
|
-
self.api_key = self.config[
|
|
54
|
-
self.engine = self.config[
|
|
51
|
+
self.api_key = self.config["SEARCH_ENGINE_API_KEY"]
|
|
52
|
+
self.engine = self.config["SEARCH_ENGINE_MODEL"]
|
|
55
53
|
self.name = self.__class__.__name__
|
|
56
54
|
|
|
57
55
|
def id(self) -> str:
|
|
58
|
-
if
|
|
56
|
+
if (
|
|
57
|
+
self.config.get("SEARCH_ENGINE_API_KEY")
|
|
58
|
+
and self.config.get("SEARCH_ENGINE_MODEL") == "google"
|
|
59
|
+
): # only support Google for now
|
|
59
60
|
if GoogleSearch is None:
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
61
|
+
UserMessage(
|
|
62
|
+
"SerpApi is not installed. Please install it with `pip install symbolicai[serpapi]`"
|
|
63
|
+
)
|
|
64
|
+
return "search"
|
|
65
|
+
return super().id() # default to unregistered
|
|
63
66
|
|
|
64
67
|
def command(self, *args, **kwargs):
|
|
65
68
|
super().command(*args, **kwargs)
|
|
66
|
-
if
|
|
67
|
-
self.api_key = kwargs[
|
|
68
|
-
if
|
|
69
|
-
self.engine
|
|
69
|
+
if "SEARCH_ENGINE_API_KEY" in kwargs:
|
|
70
|
+
self.api_key = kwargs["SEARCH_ENGINE_API_KEY"]
|
|
71
|
+
if "SEARCH_ENGINE_MODEL" in kwargs:
|
|
72
|
+
self.engine = kwargs["SEARCH_ENGINE_MODEL"]
|
|
70
73
|
|
|
71
74
|
def forward(self, argument):
|
|
72
|
-
queries
|
|
73
|
-
kwargs
|
|
75
|
+
queries = argument.prop.prepared_input
|
|
76
|
+
kwargs = argument.kwargs
|
|
74
77
|
queries_ = queries if isinstance(queries, list) else [queries]
|
|
75
|
-
rsp
|
|
76
|
-
engine
|
|
78
|
+
rsp = []
|
|
79
|
+
engine = kwargs.get("engine", self.engine)
|
|
77
80
|
|
|
78
81
|
for q in queries_:
|
|
79
82
|
query = {
|
|
@@ -82,11 +85,11 @@ class SerpApiEngine(Engine):
|
|
|
82
85
|
"q": q,
|
|
83
86
|
"google_domain": "google.com",
|
|
84
87
|
"gl": "us",
|
|
85
|
-
"hl": "en"
|
|
88
|
+
"hl": "en",
|
|
86
89
|
}
|
|
87
90
|
|
|
88
91
|
# send to Google
|
|
89
|
-
with io.capture_output()
|
|
92
|
+
with io.capture_output(): # disables prints from GoogleSearch
|
|
90
93
|
search = GoogleSearch(query)
|
|
91
94
|
res = search.get_dict()
|
|
92
95
|
|
|
@@ -100,7 +103,7 @@ class SerpApiEngine(Engine):
|
|
|
100
103
|
return output, metadata
|
|
101
104
|
|
|
102
105
|
def prepare(self, argument):
|
|
103
|
-
res
|
|
106
|
+
res = ""
|
|
104
107
|
res += str(argument.prop.query)
|
|
105
108
|
res += str(argument.prop.processed_input)
|
|
106
109
|
argument.prop.prepared_input = res
|
|
@@ -1,22 +1,21 @@
|
|
|
1
|
-
import
|
|
1
|
+
import contextlib
|
|
2
2
|
import re
|
|
3
|
+
from collections.abc import Iterable
|
|
3
4
|
from itertools import takewhile
|
|
4
|
-
from typing import Iterable
|
|
5
5
|
|
|
6
6
|
import torch
|
|
7
7
|
|
|
8
8
|
from ....symbol import Expression, Result
|
|
9
|
-
from ....utils import
|
|
9
|
+
from ....utils import UserMessage
|
|
10
10
|
from ...base import Engine
|
|
11
11
|
from ...settings import SYMAI_CONFIG
|
|
12
12
|
|
|
13
13
|
try:
|
|
14
14
|
import whisper
|
|
15
|
-
from whisper.audio import
|
|
16
|
-
N_SAMPLES # @NOTE: sample_rate (16_000) * chunk_length (30) = 480_000
|
|
15
|
+
from whisper.audio import N_SAMPLES # @NOTE: sample_rate (16_000) * chunk_length (30) = 480_000
|
|
17
16
|
from whisper.tokenizer import get_tokenizer
|
|
18
17
|
except ImportError:
|
|
19
|
-
whisper
|
|
18
|
+
whisper = None
|
|
20
19
|
N_SAMPLES = 16_000 * 30
|
|
21
20
|
|
|
22
21
|
|
|
@@ -27,17 +26,19 @@ class WhisperTimestampsFormatter(Expression):
|
|
|
27
26
|
def forward(self, response: list[str]) -> str:
|
|
28
27
|
result = []
|
|
29
28
|
for i, interval in enumerate(response):
|
|
30
|
-
|
|
29
|
+
interval_tokens = self._filter_empty_string(interval)
|
|
31
30
|
prev_end = 0.0
|
|
32
31
|
prev_start = 0.0
|
|
33
|
-
for head, tail in zip(
|
|
32
|
+
for head, tail in zip(interval_tokens[::2], interval_tokens[1::2], strict=False):
|
|
34
33
|
start = self._get_timestamp(head)
|
|
35
34
|
end = self._get_timestamp(tail)
|
|
36
35
|
if start >= prev_end:
|
|
37
36
|
start = prev_end
|
|
38
37
|
prev_end = end
|
|
39
38
|
prev_start = start
|
|
40
|
-
result.append(
|
|
39
|
+
result.append(
|
|
40
|
+
f"{self._format_to_hours(start + (i * 30))} {self._get_sentence(head)}"
|
|
41
|
+
)
|
|
41
42
|
continue
|
|
42
43
|
if start < prev_start:
|
|
43
44
|
continue
|
|
@@ -46,12 +47,11 @@ class WhisperTimestampsFormatter(Expression):
|
|
|
46
47
|
start = prev_end
|
|
47
48
|
else:
|
|
48
49
|
start += prev_end
|
|
49
|
-
if start + delta > 30
|
|
50
|
-
end = 30
|
|
51
|
-
else:
|
|
52
|
-
end = start + delta
|
|
50
|
+
end = 30 if start + delta > 30 else start + delta
|
|
53
51
|
prev_end = end
|
|
54
|
-
result.append(
|
|
52
|
+
result.append(
|
|
53
|
+
f"{self._format_to_hours(start + (i * 30))} {self._get_sentence(head)}"
|
|
54
|
+
)
|
|
55
55
|
return "\n".join(result)
|
|
56
56
|
|
|
57
57
|
def _filter_empty_string(self, s: str) -> list[str]:
|
|
@@ -68,8 +68,7 @@ class WhisperTimestampsFormatter(Expression):
|
|
|
68
68
|
seconds %= 3600
|
|
69
69
|
minutes = int(seconds // 60)
|
|
70
70
|
seconds %= 60
|
|
71
|
-
|
|
72
|
-
return formatted_time
|
|
71
|
+
return f"{hours:02d}:{minutes:02d}:{int(seconds):02d}"
|
|
73
72
|
|
|
74
73
|
|
|
75
74
|
class WhisperResult(Result):
|
|
@@ -80,20 +79,20 @@ class WhisperResult(Result):
|
|
|
80
79
|
|
|
81
80
|
def get_bins(self, bin_size_s: int = 5 * 60) -> list[str]:
|
|
82
81
|
tmps = list(map(self._seconds, re.findall(r"\b\d{2}:\d{2}:\d{2}\b", self._value)))
|
|
83
|
-
value_pairs = list(zip(tmps, self._value.split("\n")))
|
|
84
|
-
|
|
82
|
+
value_pairs = list(zip(tmps, self._value.split("\n"), strict=False))
|
|
83
|
+
bin_segments = []
|
|
85
84
|
result = []
|
|
86
85
|
for tmp, seg in value_pairs:
|
|
87
|
-
|
|
86
|
+
bin_segments.append(seg)
|
|
88
87
|
if tmp == 0 or (tmp - bin_size_s) % bin_size_s != 0:
|
|
89
88
|
continue
|
|
90
|
-
result.append("\n".join(
|
|
91
|
-
|
|
92
|
-
result.append("\n".join(
|
|
89
|
+
result.append("\n".join(bin_segments))
|
|
90
|
+
bin_segments = []
|
|
91
|
+
result.append("\n".join(bin_segments))
|
|
93
92
|
return result
|
|
94
93
|
|
|
95
94
|
def _seconds(self, tmp: str) -> int:
|
|
96
|
-
h, m
|
|
95
|
+
h, m, s = tmp.split(":")
|
|
97
96
|
return int(h) * 3600 + int(m) * 60 + int(s)
|
|
98
97
|
|
|
99
98
|
|
|
@@ -101,37 +100,44 @@ class WhisperEngine(Engine):
|
|
|
101
100
|
def __init__(self, model: str | None = None, to_device: str | None = None):
|
|
102
101
|
super().__init__()
|
|
103
102
|
self.config = SYMAI_CONFIG
|
|
104
|
-
self.model = None
|
|
105
|
-
self.model_id = self.config[
|
|
106
|
-
self.old_model_id = self.config[
|
|
103
|
+
self.model = None # lazy loading
|
|
104
|
+
self.model_id = self.config["SPEECH_TO_TEXT_ENGINE_MODEL"] if model is None else model
|
|
105
|
+
self.old_model_id = self.config["SPEECH_TO_TEXT_ENGINE_MODEL"] if model is None else model
|
|
107
106
|
self.tokens = []
|
|
108
107
|
self.text = []
|
|
109
108
|
self.formatter = WhisperTimestampsFormatter()
|
|
110
109
|
self.name = self.__class__.__name__
|
|
111
110
|
if self.model is None or self.model_id != self.old_model_id:
|
|
112
|
-
device_fallback =
|
|
111
|
+
device_fallback = "cpu"
|
|
113
112
|
device = "cuda" if torch.cuda.is_available() else device_fallback
|
|
114
|
-
device =
|
|
113
|
+
device = (
|
|
114
|
+
to_device if to_device is not None else device_fallback
|
|
115
|
+
) # user preference over auto detection
|
|
115
116
|
try:
|
|
116
117
|
self.model = whisper.load_model(self.model_id, device=device)
|
|
117
118
|
except RuntimeError:
|
|
118
|
-
|
|
119
|
+
UserMessage(
|
|
120
|
+
f"Whisper failed to load model on device {device}. Fallback to {device_fallback}."
|
|
121
|
+
)
|
|
119
122
|
self.model = whisper.load_model(self.model_id, device=device_fallback)
|
|
120
123
|
self.old_model_id = self.model_id
|
|
121
124
|
|
|
122
125
|
self._try_compile()
|
|
123
126
|
|
|
124
127
|
def id(self) -> str:
|
|
125
|
-
if self.config[
|
|
128
|
+
if self.config["SPEECH_TO_TEXT_ENGINE_MODEL"]:
|
|
126
129
|
if whisper is None:
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
+
UserMessage(
|
|
131
|
+
"Whisper is not installed. Please install it with `pip install symbolicai[whisper]`",
|
|
132
|
+
raise_with=ImportError,
|
|
133
|
+
)
|
|
134
|
+
return "speech-to-text"
|
|
135
|
+
return super().id() # default to unregistered
|
|
130
136
|
|
|
131
137
|
def command(self, *args, **kwargs):
|
|
132
138
|
super().command(*args, **kwargs)
|
|
133
|
-
if
|
|
134
|
-
self.model_id = kwargs[
|
|
139
|
+
if "SPEECH_TO_TEXT_ENGINE_MODEL" in kwargs:
|
|
140
|
+
self.model_id = kwargs["SPEECH_TO_TEXT_ENGINE_MODEL"]
|
|
135
141
|
|
|
136
142
|
def forward(self, argument):
|
|
137
143
|
assert whisper is not None, "Whisper is not installed. Please install it first."
|
|
@@ -145,16 +151,17 @@ class WhisperEngine(Engine):
|
|
|
145
151
|
without_timestamps = kwargs.get("without_timestamps", False)
|
|
146
152
|
|
|
147
153
|
raw_result = []
|
|
148
|
-
if prompt ==
|
|
149
|
-
|
|
154
|
+
if prompt == "detect_language":
|
|
155
|
+
# @NOTE: the accuracy of mel spectrogram is not good enough; don't use it to transcribe
|
|
150
156
|
audio = whisper.pad_or_trim(audio)
|
|
151
157
|
mel = whisper.log_mel_spectrogram(audio).to(self.model.device)
|
|
152
158
|
_, probs = self.model.detect_language(mel)
|
|
153
159
|
rsp = max(probs, key=probs.get)
|
|
154
|
-
elif prompt ==
|
|
160
|
+
elif prompt == "decode":
|
|
155
161
|
if show_pbar:
|
|
156
|
-
#
|
|
157
|
-
from tqdm import tqdm
|
|
162
|
+
# Suppress tqdm warning; keep optional dependency lazy.
|
|
163
|
+
from tqdm import tqdm # noqa
|
|
164
|
+
|
|
158
165
|
pbar = tqdm(self._get_chunks(audio))
|
|
159
166
|
else:
|
|
160
167
|
pbar = self._get_chunks(audio)
|
|
@@ -168,11 +175,9 @@ class WhisperEngine(Engine):
|
|
|
168
175
|
)
|
|
169
176
|
raw_result.append(result)
|
|
170
177
|
self.text.append(result["text"])
|
|
171
|
-
self.tokens.append(
|
|
172
|
-
token
|
|
173
|
-
|
|
174
|
-
for token in segment["tokens"]
|
|
175
|
-
])
|
|
178
|
+
self.tokens.append(
|
|
179
|
+
[token for segment in result["segments"] for token in segment["tokens"]]
|
|
180
|
+
)
|
|
176
181
|
if without_timestamps is not None:
|
|
177
182
|
tokenizer = get_tokenizer(self.model.is_multilingual)
|
|
178
183
|
tokens = [tokenizer.decode_with_timestamps(t) for t in self.tokens]
|
|
@@ -180,7 +185,7 @@ class WhisperEngine(Engine):
|
|
|
180
185
|
else:
|
|
181
186
|
rsp = " ".join(self.text)
|
|
182
187
|
else:
|
|
183
|
-
|
|
188
|
+
UserMessage(f"Unknown whisper command prompt: {prompt}", raise_with=ValueError)
|
|
184
189
|
|
|
185
190
|
metadata = {}
|
|
186
191
|
rsp = WhisperResult(rsp)
|
|
@@ -191,7 +196,7 @@ class WhisperEngine(Engine):
|
|
|
191
196
|
def prepare(self, argument):
|
|
192
197
|
assert not argument.prop.processed_input, "Whisper does not support processed_input."
|
|
193
198
|
assert argument.prop.audio, "Whisper requires audio input."
|
|
194
|
-
audio_file
|
|
199
|
+
audio_file = str(argument.prop.audio)
|
|
195
200
|
audio = whisper.load_audio(audio_file)
|
|
196
201
|
argument.prop.prepared_input = (audio_file, audio)
|
|
197
202
|
|
|
@@ -201,10 +206,8 @@ class WhisperEngine(Engine):
|
|
|
201
206
|
"""
|
|
202
207
|
size = len(it)
|
|
203
208
|
for i in range(0, size, batch):
|
|
204
|
-
yield torch.tensor(it[i:min(i + batch, size)]).to(self.model.device)
|
|
209
|
+
yield torch.tensor(it[i : min(i + batch, size)]).to(self.model.device)
|
|
205
210
|
|
|
206
211
|
def _try_compile(self):
|
|
207
|
-
|
|
212
|
+
with contextlib.suppress(Exception):
|
|
208
213
|
self.model = torch.compile(self.model)
|
|
209
|
-
except Exception:
|
|
210
|
-
pass
|
|
@@ -1,14 +1,17 @@
|
|
|
1
1
|
from copy import deepcopy
|
|
2
2
|
|
|
3
3
|
from ....symbol import Result
|
|
4
|
-
from ....utils import
|
|
4
|
+
from ....utils import UserMessage
|
|
5
5
|
from ...base import Engine
|
|
6
6
|
from ...settings import SYMAI_CONFIG
|
|
7
7
|
|
|
8
8
|
try:
|
|
9
9
|
import wolframalpha as wa
|
|
10
10
|
except ImportError:
|
|
11
|
-
|
|
11
|
+
UserMessage(
|
|
12
|
+
"WolframAlpha is not installed. Please install it with `pip install symbolicai[wolframalpha]`",
|
|
13
|
+
raise_with=ImportError,
|
|
14
|
+
)
|
|
12
15
|
|
|
13
16
|
|
|
14
17
|
class WolframResult(Result):
|
|
@@ -22,19 +25,19 @@ class WolframAlphaEngine(Engine):
|
|
|
22
25
|
def __init__(self, api_key: str | None = None):
|
|
23
26
|
super().__init__()
|
|
24
27
|
self.config = deepcopy(SYMAI_CONFIG)
|
|
25
|
-
self.api_key = self.config[
|
|
28
|
+
self.api_key = self.config["SYMBOLIC_ENGINE_API_KEY"] if api_key is None else api_key
|
|
26
29
|
self.client = wa.Client(self.api_key)
|
|
27
30
|
self.name = self.__class__.__name__
|
|
28
31
|
|
|
29
32
|
def id(self) -> str:
|
|
30
|
-
if self.config[
|
|
31
|
-
return
|
|
32
|
-
return super().id()
|
|
33
|
+
if self.config["SYMBOLIC_ENGINE_API_KEY"]:
|
|
34
|
+
return "symbolic"
|
|
35
|
+
return super().id() # default to unregistered
|
|
33
36
|
|
|
34
37
|
def command(self, *args, **kwargs):
|
|
35
38
|
super().command(*args, **kwargs)
|
|
36
|
-
if
|
|
37
|
-
self.api_key = kwargs[
|
|
39
|
+
if "SYMBOLIC_ENGINE_API_KEY" in kwargs:
|
|
40
|
+
self.api_key = kwargs["SYMBOLIC_ENGINE_API_KEY"]
|
|
38
41
|
self.client = wa.Client(self.api_key)
|
|
39
42
|
|
|
40
43
|
def forward(self, argument):
|
|
@@ -45,7 +48,10 @@ class WolframAlphaEngine(Engine):
|
|
|
45
48
|
rsp = self.client.query(queries)
|
|
46
49
|
rsp = WolframResult(rsp)
|
|
47
50
|
except Exception as e:
|
|
48
|
-
|
|
51
|
+
UserMessage(
|
|
52
|
+
f'Failed to interact with WolframAlpha: {e}.\n\n If you are getting an error related to "assert", that is a well-known issue with WolframAlpha. There is a manual fix for this issue: https://github.com/jaraco/wolframalpha/pull/34/commits/6eb3828ee812f65592e00629710fc027d40e7bd1',
|
|
53
|
+
raise_with=ValueError,
|
|
54
|
+
)
|
|
49
55
|
|
|
50
56
|
metadata = {}
|
|
51
57
|
|
|
@@ -1,48 +1,42 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
|
|
3
3
|
from openai import OpenAI
|
|
4
|
-
from typing import Optional
|
|
5
|
-
|
|
6
|
-
# suppress openai logging
|
|
7
|
-
logging.getLogger("openai").setLevel(logging.WARNING)
|
|
8
4
|
|
|
5
|
+
from ....symbol import Result
|
|
9
6
|
from ...base import Engine
|
|
10
7
|
from ...settings import SYMAI_CONFIG
|
|
11
|
-
|
|
8
|
+
|
|
9
|
+
# suppress openai logging
|
|
10
|
+
logging.getLogger("openai").setLevel(logging.WARNING)
|
|
12
11
|
|
|
13
12
|
|
|
14
13
|
class TTSEngine(Engine):
|
|
15
|
-
def __init__(self, api_key:
|
|
14
|
+
def __init__(self, api_key: str | None = None, model: str | None = None):
|
|
16
15
|
super().__init__()
|
|
17
16
|
self.config = SYMAI_CONFIG
|
|
18
|
-
self.api_key = self.config[
|
|
19
|
-
self.model_id = self.config[
|
|
17
|
+
self.api_key = self.config["TEXT_TO_SPEECH_ENGINE_API_KEY"] if api_key is None else api_key
|
|
18
|
+
self.model_id = self.config["TEXT_TO_SPEECH_ENGINE_MODEL"] if model is None else model
|
|
20
19
|
self.tokens = []
|
|
21
20
|
self.text = []
|
|
22
21
|
self.client = OpenAI(api_key=self.api_key)
|
|
23
22
|
self.name = self.__class__.__name__
|
|
24
23
|
|
|
25
24
|
def id(self) -> str:
|
|
26
|
-
if self.config[
|
|
27
|
-
return
|
|
28
|
-
return super().id()
|
|
25
|
+
if self.config["TEXT_TO_SPEECH_ENGINE_API_KEY"]:
|
|
26
|
+
return "text-to-speech"
|
|
27
|
+
return super().id() # default to unregistered
|
|
29
28
|
|
|
30
29
|
def command(self, *args, **kwargs):
|
|
31
30
|
super().command(*args, **kwargs)
|
|
32
|
-
if
|
|
33
|
-
self.api_key = kwargs[
|
|
34
|
-
if
|
|
35
|
-
self.model_id = kwargs[
|
|
31
|
+
if "TEXT_TO_SPEECH_ENGINE_API_KEY" in kwargs:
|
|
32
|
+
self.api_key = kwargs["TEXT_TO_SPEECH_ENGINE_API_KEY"]
|
|
33
|
+
if "TEXT_TO_SPEECH_ENGINE_MODEL" in kwargs:
|
|
34
|
+
self.model_id = kwargs["TEXT_TO_SPEECH_ENGINE_MODEL"]
|
|
36
35
|
|
|
37
36
|
def forward(self, argument):
|
|
38
|
-
kwargs = argument.kwargs
|
|
39
37
|
voice, path, prompt = argument.prop.prepared_input
|
|
40
38
|
|
|
41
|
-
rsp = self.client.audio.speech.create(
|
|
42
|
-
model=self.model_id,
|
|
43
|
-
voice=voice,
|
|
44
|
-
input=prompt
|
|
45
|
-
)
|
|
39
|
+
rsp = self.client.audio.speech.create(model=self.model_id, voice=voice, input=prompt)
|
|
46
40
|
|
|
47
41
|
metadata = {}
|
|
48
42
|
|
|
@@ -53,9 +47,9 @@ class TTSEngine(Engine):
|
|
|
53
47
|
|
|
54
48
|
def prepare(self, argument):
|
|
55
49
|
assert not argument.prop.processed_input, "TTSEngine does not support processed_input."
|
|
56
|
-
assert
|
|
57
|
-
assert
|
|
58
|
-
voice
|
|
59
|
-
audio_file
|
|
60
|
-
prompt
|
|
50
|
+
assert "voice" in argument.kwargs, "TTS requires voice selection."
|
|
51
|
+
assert "path" in argument.kwargs, "TTS requires path selection."
|
|
52
|
+
voice = str(argument.kwargs["voice"]).lower()
|
|
53
|
+
audio_file = str(argument.kwargs["path"])
|
|
54
|
+
prompt = str(argument.prop.prompt)
|
|
61
55
|
argument.prop.prepared_input = (voice, audio_file, prompt)
|