symbolicai 0.21.0__py3-none-any.whl → 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- symai/__init__.py +269 -173
- symai/backend/base.py +123 -110
- symai/backend/engines/drawing/engine_bfl.py +45 -44
- symai/backend/engines/drawing/engine_gpt_image.py +112 -97
- symai/backend/engines/embedding/engine_llama_cpp.py +63 -52
- symai/backend/engines/embedding/engine_openai.py +25 -21
- symai/backend/engines/execute/engine_python.py +19 -18
- symai/backend/engines/files/engine_io.py +104 -95
- symai/backend/engines/imagecaptioning/engine_blip2.py +28 -24
- symai/backend/engines/imagecaptioning/engine_llavacpp_client.py +102 -79
- symai/backend/engines/index/engine_pinecone.py +124 -97
- symai/backend/engines/index/engine_qdrant.py +1011 -0
- symai/backend/engines/index/engine_vectordb.py +84 -56
- symai/backend/engines/lean/engine_lean4.py +96 -52
- symai/backend/engines/neurosymbolic/__init__.py +41 -13
- symai/backend/engines/neurosymbolic/engine_anthropic_claudeX_chat.py +330 -248
- symai/backend/engines/neurosymbolic/engine_anthropic_claudeX_reasoning.py +329 -264
- symai/backend/engines/neurosymbolic/engine_cerebras.py +328 -0
- symai/backend/engines/neurosymbolic/engine_deepseekX_reasoning.py +118 -88
- symai/backend/engines/neurosymbolic/engine_google_geminiX_reasoning.py +344 -299
- symai/backend/engines/neurosymbolic/engine_groq.py +173 -115
- symai/backend/engines/neurosymbolic/engine_huggingface.py +114 -84
- symai/backend/engines/neurosymbolic/engine_llama_cpp.py +144 -118
- symai/backend/engines/neurosymbolic/engine_openai_gptX_chat.py +415 -307
- symai/backend/engines/neurosymbolic/engine_openai_gptX_reasoning.py +394 -231
- symai/backend/engines/ocr/engine_apilayer.py +23 -27
- symai/backend/engines/output/engine_stdout.py +10 -13
- symai/backend/engines/{webscraping → scrape}/engine_requests.py +101 -54
- symai/backend/engines/search/engine_openai.py +100 -88
- symai/backend/engines/search/engine_parallel.py +665 -0
- symai/backend/engines/search/engine_perplexity.py +44 -45
- symai/backend/engines/search/engine_serpapi.py +37 -34
- symai/backend/engines/speech_to_text/engine_local_whisper.py +54 -51
- symai/backend/engines/symbolic/engine_wolframalpha.py +15 -9
- symai/backend/engines/text_to_speech/engine_openai.py +20 -26
- symai/backend/engines/text_vision/engine_clip.py +39 -37
- symai/backend/engines/userinput/engine_console.py +5 -6
- symai/backend/mixin/__init__.py +13 -0
- symai/backend/mixin/anthropic.py +48 -38
- symai/backend/mixin/deepseek.py +6 -5
- symai/backend/mixin/google.py +7 -4
- symai/backend/mixin/groq.py +2 -4
- symai/backend/mixin/openai.py +140 -110
- symai/backend/settings.py +87 -20
- symai/chat.py +216 -123
- symai/collect/__init__.py +7 -1
- symai/collect/dynamic.py +80 -70
- symai/collect/pipeline.py +67 -51
- symai/collect/stats.py +161 -109
- symai/components.py +707 -360
- symai/constraints.py +24 -12
- symai/core.py +1857 -1233
- symai/core_ext.py +83 -80
- symai/endpoints/api.py +166 -104
- symai/extended/.DS_Store +0 -0
- symai/extended/__init__.py +46 -12
- symai/extended/api_builder.py +29 -21
- symai/extended/arxiv_pdf_parser.py +23 -14
- symai/extended/bibtex_parser.py +9 -6
- symai/extended/conversation.py +156 -126
- symai/extended/document.py +50 -30
- symai/extended/file_merger.py +57 -14
- symai/extended/graph.py +51 -32
- symai/extended/html_style_template.py +18 -14
- symai/extended/interfaces/blip_2.py +2 -3
- symai/extended/interfaces/clip.py +4 -3
- symai/extended/interfaces/console.py +9 -1
- symai/extended/interfaces/dall_e.py +4 -2
- symai/extended/interfaces/file.py +2 -0
- symai/extended/interfaces/flux.py +4 -2
- symai/extended/interfaces/gpt_image.py +16 -7
- symai/extended/interfaces/input.py +2 -1
- symai/extended/interfaces/llava.py +1 -2
- symai/extended/interfaces/{naive_webscraping.py → naive_scrape.py} +4 -3
- symai/extended/interfaces/naive_vectordb.py +9 -10
- symai/extended/interfaces/ocr.py +5 -3
- symai/extended/interfaces/openai_search.py +2 -0
- symai/extended/interfaces/parallel.py +30 -0
- symai/extended/interfaces/perplexity.py +2 -0
- symai/extended/interfaces/pinecone.py +12 -9
- symai/extended/interfaces/python.py +2 -0
- symai/extended/interfaces/serpapi.py +3 -1
- symai/extended/interfaces/terminal.py +2 -4
- symai/extended/interfaces/tts.py +3 -2
- symai/extended/interfaces/whisper.py +3 -2
- symai/extended/interfaces/wolframalpha.py +2 -1
- symai/extended/metrics/__init__.py +11 -1
- symai/extended/metrics/similarity.py +14 -13
- symai/extended/os_command.py +39 -29
- symai/extended/packages/__init__.py +29 -3
- symai/extended/packages/symdev.py +51 -43
- symai/extended/packages/sympkg.py +41 -35
- symai/extended/packages/symrun.py +63 -50
- symai/extended/repo_cloner.py +14 -12
- symai/extended/seo_query_optimizer.py +15 -13
- symai/extended/solver.py +116 -91
- symai/extended/summarizer.py +12 -10
- symai/extended/taypan_interpreter.py +17 -18
- symai/extended/vectordb.py +122 -92
- symai/formatter/__init__.py +9 -1
- symai/formatter/formatter.py +51 -47
- symai/formatter/regex.py +70 -69
- symai/functional.py +325 -176
- symai/imports.py +190 -147
- symai/interfaces.py +57 -28
- symai/memory.py +45 -35
- symai/menu/screen.py +28 -19
- symai/misc/console.py +66 -56
- symai/misc/loader.py +8 -5
- symai/models/__init__.py +17 -1
- symai/models/base.py +395 -236
- symai/models/errors.py +1 -2
- symai/ops/__init__.py +32 -22
- symai/ops/measures.py +24 -25
- symai/ops/primitives.py +1149 -731
- symai/post_processors.py +58 -50
- symai/pre_processors.py +86 -82
- symai/processor.py +21 -13
- symai/prompts.py +764 -685
- symai/server/huggingface_server.py +135 -49
- symai/server/llama_cpp_server.py +21 -11
- symai/server/qdrant_server.py +206 -0
- symai/shell.py +100 -42
- symai/shellsv.py +700 -492
- symai/strategy.py +630 -346
- symai/symbol.py +368 -322
- symai/utils.py +100 -78
- {symbolicai-0.21.0.dist-info → symbolicai-1.1.0.dist-info}/METADATA +22 -10
- symbolicai-1.1.0.dist-info/RECORD +168 -0
- symbolicai-0.21.0.dist-info/RECORD +0 -162
- {symbolicai-0.21.0.dist-info → symbolicai-1.1.0.dist-info}/WHEEL +0 -0
- {symbolicai-0.21.0.dist-info → symbolicai-1.1.0.dist-info}/entry_points.txt +0 -0
- {symbolicai-0.21.0.dist-info → symbolicai-1.1.0.dist-info}/licenses/LICENSE +0 -0
- {symbolicai-0.21.0.dist-info → symbolicai-1.1.0.dist-info}/top_level.txt +0 -0
symai/imports.py
CHANGED
|
@@ -1,30 +1,31 @@
|
|
|
1
1
|
import importlib
|
|
2
2
|
import json
|
|
3
3
|
import logging
|
|
4
|
-
import os
|
|
5
4
|
import shutil
|
|
6
5
|
import stat
|
|
7
6
|
import subprocess
|
|
8
7
|
import sys
|
|
9
8
|
from pathlib import Path
|
|
10
|
-
from typing import List, Tuple, Union
|
|
11
9
|
|
|
12
10
|
from loguru import logger
|
|
13
11
|
|
|
14
12
|
from .backend.settings import HOME_PATH
|
|
15
13
|
from .symbol import Expression
|
|
14
|
+
from .utils import UserMessage
|
|
16
15
|
|
|
17
16
|
logging.getLogger("subprocess").setLevel(logging.ERROR)
|
|
18
17
|
|
|
19
18
|
|
|
20
|
-
__root_dir__
|
|
21
|
-
BASE_PACKAGE_MODULE =
|
|
22
|
-
BASE_PACKAGE_PATH =
|
|
19
|
+
__root_dir__ = HOME_PATH / "packages"
|
|
20
|
+
BASE_PACKAGE_MODULE = "" # use relative path
|
|
21
|
+
BASE_PACKAGE_PATH = __root_dir__
|
|
23
22
|
sys.path.append(str(__root_dir__))
|
|
24
23
|
|
|
25
24
|
|
|
26
25
|
class Import(Expression):
|
|
27
|
-
def __init__(
|
|
26
|
+
def __init__(
|
|
27
|
+
self, module: str, local_path: str | None = None, submodules: bool = False, *args, **kwargs
|
|
28
|
+
):
|
|
28
29
|
super(self).__init__(*args, **kwargs)
|
|
29
30
|
self.module = module
|
|
30
31
|
self.local_path = local_path
|
|
@@ -33,10 +34,10 @@ class Import(Expression):
|
|
|
33
34
|
@staticmethod
|
|
34
35
|
def exists(module):
|
|
35
36
|
# Check if module is a local path or a GitHub repo reference
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
37
|
+
module_path = Path(module)
|
|
38
|
+
if module_path.exists() and module_path.is_dir():
|
|
39
|
+
return (module_path / "package.json").exists()
|
|
40
|
+
return (BASE_PACKAGE_PATH / module / "package.json").exists()
|
|
40
41
|
|
|
41
42
|
@staticmethod
|
|
42
43
|
def get_from_local(module, local_path):
|
|
@@ -47,37 +48,43 @@ class Import(Expression):
|
|
|
47
48
|
local_path: Path to local package directory
|
|
48
49
|
"""
|
|
49
50
|
# If base package does not exist, create it
|
|
50
|
-
|
|
51
|
-
os.makedirs(BASE_PACKAGE_PATH)
|
|
51
|
+
BASE_PACKAGE_PATH.mkdir(parents=True, exist_ok=True)
|
|
52
52
|
|
|
53
53
|
# Create module directory
|
|
54
|
-
module_path =
|
|
55
|
-
|
|
54
|
+
module_path = BASE_PACKAGE_PATH / module
|
|
55
|
+
module_path.parent.mkdir(parents=True, exist_ok=True)
|
|
56
56
|
|
|
57
57
|
# Copy files from local path to package directory
|
|
58
58
|
try:
|
|
59
|
-
if
|
|
59
|
+
if module_path.exists():
|
|
60
60
|
shutil.rmtree(module_path)
|
|
61
61
|
shutil.copytree(local_path, module_path)
|
|
62
62
|
logger.info(f"Copied local package from {local_path} to {module_path}")
|
|
63
63
|
|
|
64
64
|
# Install dependencies
|
|
65
|
-
|
|
66
|
-
|
|
65
|
+
package_json = module_path / "package.json"
|
|
66
|
+
if package_json.exists():
|
|
67
|
+
with package_json.open() as f:
|
|
67
68
|
pkg = json.load(f)
|
|
68
|
-
for dependency in pkg.get(
|
|
69
|
+
for dependency in pkg.get("dependencies", []):
|
|
69
70
|
# Update git_url for the dependency
|
|
70
|
-
git_url_dependency = f
|
|
71
|
-
|
|
72
|
-
|
|
71
|
+
git_url_dependency = f"git@github.com:{dependency}.git"
|
|
72
|
+
dependency_path = BASE_PACKAGE_PATH / dependency
|
|
73
|
+
if not dependency_path.exists():
|
|
74
|
+
subprocess.check_call(
|
|
75
|
+
["git", "clone", git_url_dependency, str(dependency_path)]
|
|
76
|
+
)
|
|
73
77
|
|
|
74
78
|
# Install requirements
|
|
75
|
-
|
|
76
|
-
|
|
79
|
+
requirements_file = module_path / "requirements.txt"
|
|
80
|
+
if requirements_file.exists():
|
|
81
|
+
with requirements_file.open() as f:
|
|
77
82
|
for dependency in f.readlines():
|
|
78
|
-
|
|
79
|
-
if
|
|
80
|
-
subprocess.check_call(
|
|
83
|
+
dependency_name = dependency.strip()
|
|
84
|
+
if dependency_name:
|
|
85
|
+
subprocess.check_call(
|
|
86
|
+
[sys.executable, "-m", "pip", "install", dependency_name]
|
|
87
|
+
)
|
|
81
88
|
except Exception as e:
|
|
82
89
|
logger.error(f"Error installing from local path: {e}")
|
|
83
90
|
raise
|
|
@@ -85,64 +92,66 @@ class Import(Expression):
|
|
|
85
92
|
@staticmethod
|
|
86
93
|
def get_from_github(module, submodules=False):
|
|
87
94
|
# if base package does not exist, create it
|
|
88
|
-
|
|
89
|
-
os.makedirs(BASE_PACKAGE_PATH)
|
|
95
|
+
BASE_PACKAGE_PATH.mkdir(parents=True, exist_ok=True)
|
|
90
96
|
|
|
91
97
|
# Clone repository
|
|
92
|
-
git_url = f
|
|
93
|
-
clone_cmd = [
|
|
98
|
+
git_url = f"git@github.com:{module}.git"
|
|
99
|
+
clone_cmd = ["git", "clone"]
|
|
94
100
|
if submodules:
|
|
95
|
-
clone_cmd.extend([
|
|
96
|
-
clone_cmd.extend([git_url,
|
|
101
|
+
clone_cmd.extend(["--recurse-submodules"])
|
|
102
|
+
clone_cmd.extend([git_url, str(BASE_PACKAGE_PATH / module)])
|
|
97
103
|
subprocess.check_call(clone_cmd)
|
|
98
104
|
|
|
99
105
|
# Install dependencies
|
|
100
|
-
|
|
106
|
+
package_json = BASE_PACKAGE_PATH / module / "package.json"
|
|
107
|
+
with package_json.open() as f:
|
|
101
108
|
pkg = json.load(f)
|
|
102
|
-
for dependency in pkg[
|
|
109
|
+
for dependency in pkg["dependencies"]:
|
|
103
110
|
# Update git_url for the dependency
|
|
104
|
-
git_url_dependency = f
|
|
105
|
-
|
|
106
|
-
|
|
111
|
+
git_url_dependency = f"git@github.com:{dependency}.git"
|
|
112
|
+
dependency_path = BASE_PACKAGE_PATH / dependency
|
|
113
|
+
if not dependency_path.exists():
|
|
114
|
+
subprocess.check_call(
|
|
115
|
+
["git", "clone", git_url_dependency, str(dependency_path)]
|
|
116
|
+
)
|
|
107
117
|
|
|
108
118
|
# Install requirements
|
|
109
|
-
|
|
110
|
-
|
|
119
|
+
requirements_file = BASE_PACKAGE_PATH / module / "requirements.txt"
|
|
120
|
+
if requirements_file.exists():
|
|
121
|
+
with requirements_file.open() as f:
|
|
111
122
|
for dependency in f.readlines():
|
|
112
|
-
subprocess.check_call([
|
|
123
|
+
subprocess.check_call(["pip", "install", dependency])
|
|
113
124
|
|
|
114
125
|
@staticmethod
|
|
115
126
|
def load_module_class(module):
|
|
116
127
|
module_classes = []
|
|
117
128
|
# Detect if module is a local path
|
|
118
|
-
|
|
129
|
+
module_path_obj = Path(module)
|
|
130
|
+
is_local_path = module_path_obj.exists() and module_path_obj.is_dir()
|
|
119
131
|
|
|
120
|
-
if is_local_path
|
|
121
|
-
package_path = module
|
|
122
|
-
else:
|
|
123
|
-
package_path = f'{BASE_PACKAGE_PATH}/{module}'
|
|
132
|
+
package_path = module_path_obj if is_local_path else BASE_PACKAGE_PATH / module
|
|
124
133
|
|
|
125
|
-
with
|
|
134
|
+
with (package_path / "package.json").open() as f:
|
|
126
135
|
pkg = json.load(f)
|
|
127
|
-
for expr in pkg[
|
|
136
|
+
for expr in pkg["expressions"]:
|
|
128
137
|
if is_local_path:
|
|
129
|
-
module_path = f'{package_path}/{expr["module"].replace("/", ".")}'
|
|
130
138
|
# For local modules, we need to add the path to sys.path
|
|
131
|
-
parent_dir =
|
|
132
|
-
if parent_dir not in sys.path:
|
|
133
|
-
sys.path.append(parent_dir)
|
|
139
|
+
parent_dir = package_path.parent
|
|
140
|
+
if str(parent_dir) not in sys.path:
|
|
141
|
+
sys.path.append(str(parent_dir))
|
|
134
142
|
# Use local module's name from the directory structure
|
|
135
|
-
module_name =
|
|
143
|
+
module_name = package_path.name
|
|
136
144
|
relative_module_path = f"{module_name}.{expr['module'].replace('/', '.')}"
|
|
137
145
|
else:
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
relative_module_path = module.split('/')[0] + '.' + module.split('/')[1] + '.' + relative_module_path
|
|
146
|
+
module_parts = module.split("/")
|
|
147
|
+
relative_module_path = ".".join(
|
|
148
|
+
[*module_parts, expr["module"].replace("/", ".")]
|
|
149
|
+
)
|
|
143
150
|
|
|
144
151
|
try:
|
|
145
|
-
module_class = getattr(
|
|
152
|
+
module_class = getattr(
|
|
153
|
+
importlib.import_module(relative_module_path), expr["type"]
|
|
154
|
+
)
|
|
146
155
|
module_classes.append(module_class)
|
|
147
156
|
except (ImportError, ModuleNotFoundError) as e:
|
|
148
157
|
logger.error(f"Error importing module {relative_module_path}: {e}")
|
|
@@ -150,66 +159,79 @@ class Import(Expression):
|
|
|
150
159
|
return module_classes
|
|
151
160
|
|
|
152
161
|
@staticmethod
|
|
153
|
-
def
|
|
162
|
+
def _normalize_expressions(expressions: list[str] | tuple[str] | str) -> tuple[list[str], bool]:
|
|
163
|
+
if isinstance(expressions, str):
|
|
164
|
+
return [expressions], True
|
|
165
|
+
if isinstance(expressions, (list, tuple)):
|
|
166
|
+
expression_list = list(expressions)
|
|
167
|
+
return expression_list, len(expression_list) == 1
|
|
168
|
+
msg = "Invalid type for 'expressions'. Must be str, list or tuple."
|
|
169
|
+
UserMessage(msg)
|
|
170
|
+
raise Exception(msg)
|
|
171
|
+
|
|
172
|
+
@staticmethod
|
|
173
|
+
def load_expression(
|
|
174
|
+
module, expressions: list[str] | tuple[str] | str
|
|
175
|
+
) -> list[Expression] | Expression:
|
|
176
|
+
expression_list, return_single = Import._normalize_expressions(expressions)
|
|
177
|
+
expected_count = len(expression_list)
|
|
178
|
+
expression_targets = set(expression_list)
|
|
154
179
|
module_classes = []
|
|
155
180
|
# Detect if module is a local path
|
|
156
|
-
|
|
181
|
+
module_path_obj = Path(module)
|
|
182
|
+
is_local_path = module_path_obj.exists() and module_path_obj.is_dir()
|
|
183
|
+
|
|
184
|
+
package_path = module_path_obj if is_local_path else BASE_PACKAGE_PATH / module
|
|
157
185
|
|
|
158
186
|
if is_local_path:
|
|
159
|
-
|
|
187
|
+
parent_dir = package_path.parent
|
|
188
|
+
if str(parent_dir) not in sys.path:
|
|
189
|
+
sys.path.append(str(parent_dir))
|
|
190
|
+
module_name = package_path.name
|
|
160
191
|
else:
|
|
161
|
-
|
|
192
|
+
module_parts = module.split("/")
|
|
162
193
|
|
|
163
|
-
with
|
|
194
|
+
with (package_path / "package.json").open() as f:
|
|
164
195
|
pkg = json.load(f)
|
|
165
|
-
for expr in pkg[
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
196
|
+
for expr in pkg["expressions"]:
|
|
197
|
+
relative_module_path = (
|
|
198
|
+
f"{module_name}.{expr['module'].replace('/', '.')}"
|
|
199
|
+
if is_local_path
|
|
200
|
+
else ".".join([*module_parts, expr["module"].replace("/", ".")])
|
|
201
|
+
)
|
|
171
202
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
try:
|
|
186
|
-
module_obj = importlib.import_module(relative_module_path)
|
|
187
|
-
module_class = getattr(module_obj, expr['type'])
|
|
188
|
-
return module_class
|
|
189
|
-
except (ImportError, ModuleNotFoundError) as e:
|
|
190
|
-
logger.error(f"Error importing module {relative_module_path}: {e}")
|
|
191
|
-
raise
|
|
192
|
-
elif isinstance(expressions, list) or isinstance(expressions, tuple):
|
|
193
|
-
if expr['type'] in expressions:
|
|
194
|
-
try:
|
|
195
|
-
module_obj = importlib.import_module(relative_module_path)
|
|
196
|
-
module_class = getattr(module_obj, expr['type'])
|
|
197
|
-
if len(expressions) == 1:
|
|
198
|
-
return module_class
|
|
199
|
-
module_classes.append(module_class)
|
|
200
|
-
except (ImportError, ModuleNotFoundError) as e:
|
|
201
|
-
logger.error(f"Error importing module {relative_module_path}: {e}")
|
|
202
|
-
raise
|
|
203
|
-
else:
|
|
204
|
-
raise Exception("Invalid type for 'expressions'. Must be str, list or tuple.")
|
|
203
|
+
if expr["type"] not in expression_targets:
|
|
204
|
+
continue
|
|
205
|
+
|
|
206
|
+
try:
|
|
207
|
+
module_obj = importlib.import_module(relative_module_path)
|
|
208
|
+
module_class = getattr(module_obj, expr["type"])
|
|
209
|
+
except (ImportError, ModuleNotFoundError) as e:
|
|
210
|
+
logger.error(f"Error importing module {relative_module_path}: {e}")
|
|
211
|
+
raise
|
|
212
|
+
|
|
213
|
+
if return_single:
|
|
214
|
+
return module_class
|
|
215
|
+
module_classes.append(module_class)
|
|
205
216
|
|
|
206
217
|
assert len(module_classes) > 0, f"Expression '{expressions}' not found in module '{module}'"
|
|
207
218
|
module_classes_names = [str(class_.__name__) for class_ in module_classes]
|
|
208
|
-
|
|
219
|
+
missing_expressions = [expr for expr in expression_list if expr not in module_classes_names]
|
|
220
|
+
assert len(module_classes) == expected_count, (
|
|
221
|
+
f"Not all expressions found in module '{module}'. Could not load {missing_expressions}"
|
|
222
|
+
)
|
|
209
223
|
return module_classes
|
|
210
224
|
|
|
211
|
-
def __new__(
|
|
212
|
-
|
|
225
|
+
def __new__(
|
|
226
|
+
cls,
|
|
227
|
+
module,
|
|
228
|
+
auto_clone: bool = True,
|
|
229
|
+
verbose: bool = False,
|
|
230
|
+
local_path: str | None = None,
|
|
231
|
+
submodules: bool = False,
|
|
232
|
+
*args,
|
|
233
|
+
**kwargs,
|
|
234
|
+
):
|
|
213
235
|
"""
|
|
214
236
|
Import a module from GitHub or local path.
|
|
215
237
|
|
|
@@ -222,15 +244,18 @@ class Import(Expression):
|
|
|
222
244
|
*args, **kwargs: Additional arguments to pass to the module constructor
|
|
223
245
|
"""
|
|
224
246
|
# Detect if module is a local path
|
|
225
|
-
|
|
247
|
+
module_path_obj = Path(module)
|
|
248
|
+
is_local_path = module_path_obj.exists() and module_path_obj.is_dir()
|
|
226
249
|
|
|
227
250
|
if is_local_path:
|
|
228
251
|
# If module is a local path
|
|
229
|
-
package_path =
|
|
230
|
-
if not
|
|
231
|
-
|
|
252
|
+
package_path = module_path_obj
|
|
253
|
+
if not (package_path / "package.json").exists():
|
|
254
|
+
msg = f"No package.json found in {module}"
|
|
255
|
+
UserMessage(msg)
|
|
256
|
+
raise ValueError(msg)
|
|
232
257
|
|
|
233
|
-
with
|
|
258
|
+
with (package_path / "package.json").open() as f:
|
|
234
259
|
pkg = json.load(f)
|
|
235
260
|
else:
|
|
236
261
|
# Module is a GitHub reference
|
|
@@ -240,27 +265,29 @@ class Import(Expression):
|
|
|
240
265
|
else:
|
|
241
266
|
Import.get_from_github(module, submodules)
|
|
242
267
|
|
|
243
|
-
with
|
|
268
|
+
with (BASE_PACKAGE_PATH / module / "package.json").open() as f:
|
|
244
269
|
pkg = json.load(f)
|
|
245
|
-
if
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
relative_module_path =
|
|
253
|
-
class_ = expr[
|
|
270
|
+
if "run" not in pkg:
|
|
271
|
+
msg = f"Module '{module}' has no 'run' expression defined."
|
|
272
|
+
UserMessage(msg)
|
|
273
|
+
raise Exception(msg)
|
|
274
|
+
expr = pkg["run"]
|
|
275
|
+
module_rel = expr["module"].replace("/", ".")
|
|
276
|
+
module_parts = module.split("/")
|
|
277
|
+
relative_module_path = ".".join([*module_parts, module_rel])
|
|
278
|
+
class_ = expr["type"]
|
|
254
279
|
if verbose:
|
|
255
280
|
logger.info(f"Loading module '{relative_module_path}.{expr['type']}'")
|
|
256
281
|
module_class = getattr(importlib.import_module(relative_module_path), class_)
|
|
257
282
|
return module_class(*args, **kwargs)
|
|
258
283
|
|
|
259
|
-
def __call__(self, *
|
|
260
|
-
|
|
284
|
+
def __call__(self, *_args, **_kwargs):
|
|
285
|
+
msg = "Cannot call Import class directly. Use Import.load_module_class(module) instead."
|
|
286
|
+
UserMessage(msg)
|
|
287
|
+
raise Exception(msg)
|
|
261
288
|
|
|
262
289
|
@staticmethod
|
|
263
|
-
def install(module: str, local_path: str = None, submodules: bool = False):
|
|
290
|
+
def install(module: str, local_path: str | None = None, submodules: bool = False):
|
|
264
291
|
"""Install a package from GitHub or a local path.
|
|
265
292
|
|
|
266
293
|
Args:
|
|
@@ -269,11 +296,14 @@ class Import(Expression):
|
|
|
269
296
|
submodules: Whether to initialize submodules for GitHub repos
|
|
270
297
|
"""
|
|
271
298
|
# Determine if module is a local path
|
|
272
|
-
|
|
299
|
+
local_path_obj = Path(local_path) if local_path is not None else None
|
|
300
|
+
is_local_path = (
|
|
301
|
+
local_path_obj is not None and local_path_obj.exists() and local_path_obj.is_dir()
|
|
302
|
+
)
|
|
273
303
|
|
|
274
304
|
if not Import.exists(module):
|
|
275
305
|
if is_local_path:
|
|
276
|
-
Import.get_from_local(module,
|
|
306
|
+
Import.get_from_local(module, str(local_path_obj))
|
|
277
307
|
logger.success(f"Module '{module}' installed from local path.")
|
|
278
308
|
else:
|
|
279
309
|
Import.get_from_github(module, submodules)
|
|
@@ -284,44 +314,54 @@ class Import(Expression):
|
|
|
284
314
|
@staticmethod
|
|
285
315
|
def remove(module: str):
|
|
286
316
|
# Determine if module is a local path
|
|
287
|
-
|
|
317
|
+
module_path_obj = Path(module)
|
|
318
|
+
is_local_path = module_path_obj.exists() and module_path_obj.is_dir()
|
|
288
319
|
|
|
289
320
|
if is_local_path:
|
|
290
321
|
# For local path, remove directly
|
|
291
|
-
if
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
322
|
+
if module_path_obj.exists():
|
|
323
|
+
|
|
324
|
+
def del_rw(_action, name, _exc):
|
|
325
|
+
path_obj = Path(name)
|
|
326
|
+
path_obj.chmod(stat.S_IWRITE)
|
|
327
|
+
path_obj.unlink()
|
|
328
|
+
|
|
329
|
+
shutil.rmtree(module_path_obj, onerror=del_rw)
|
|
296
330
|
logger.success(f"Removed local module at '{module}'")
|
|
297
331
|
else:
|
|
298
332
|
logger.error(f"Local module '{module}' not found.")
|
|
299
333
|
else:
|
|
300
334
|
# For GitHub modules, remove from packages directory
|
|
301
|
-
module_path =
|
|
302
|
-
if
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
335
|
+
module_path = BASE_PACKAGE_PATH / module
|
|
336
|
+
if module_path.exists():
|
|
337
|
+
|
|
338
|
+
def del_rw(_action, name, _exc):
|
|
339
|
+
path_obj = Path(name)
|
|
340
|
+
path_obj.chmod(stat.S_IWRITE)
|
|
341
|
+
path_obj.unlink()
|
|
342
|
+
|
|
306
343
|
shutil.rmtree(module_path, onerror=del_rw)
|
|
307
344
|
logger.success(f"Removed module '{module}'")
|
|
308
345
|
|
|
309
346
|
# Check if folder is empty and remove it
|
|
310
|
-
parent_path =
|
|
311
|
-
if
|
|
312
|
-
|
|
347
|
+
parent_path = BASE_PACKAGE_PATH / module.split("/")[0]
|
|
348
|
+
if parent_path.exists() and not any(parent_path.iterdir()):
|
|
349
|
+
parent_path.rmdir()
|
|
313
350
|
logger.info(f"Removed empty parent folder '{parent_path}'")
|
|
314
351
|
else:
|
|
315
352
|
logger.error(f"Module '{module}' not found.")
|
|
316
353
|
|
|
317
354
|
@staticmethod
|
|
318
355
|
def list_installed():
|
|
319
|
-
|
|
356
|
+
if not BASE_PACKAGE_PATH.exists():
|
|
357
|
+
return []
|
|
358
|
+
base_dirs = [entry for entry in BASE_PACKAGE_PATH.iterdir() if entry.is_dir()]
|
|
320
359
|
|
|
321
360
|
sub_dirs = []
|
|
322
361
|
for base_dir in base_dirs:
|
|
323
|
-
|
|
324
|
-
|
|
362
|
+
sub_dirs.extend(
|
|
363
|
+
[f"{base_dir.name}/{entry.name}" for entry in base_dir.iterdir() if entry.is_dir()]
|
|
364
|
+
)
|
|
325
365
|
|
|
326
366
|
return sub_dirs
|
|
327
367
|
|
|
@@ -335,19 +375,22 @@ class Import(Expression):
|
|
|
335
375
|
"""
|
|
336
376
|
if Import.exists(module):
|
|
337
377
|
# Determine if module is a local path
|
|
338
|
-
|
|
378
|
+
module_path_obj = Path(module)
|
|
379
|
+
is_local_path = module_path_obj.exists() and module_path_obj.is_dir()
|
|
339
380
|
|
|
340
381
|
# Use the appropriate path based on whether it's local or not
|
|
341
|
-
module_path =
|
|
382
|
+
module_path = (
|
|
383
|
+
module_path_obj if is_local_path else BASE_PACKAGE_PATH / module.replace(".", "/")
|
|
384
|
+
)
|
|
342
385
|
|
|
343
386
|
# Construct the git pull command based on whether submodules should be included
|
|
344
|
-
pull_cmd = [
|
|
387
|
+
pull_cmd = ["git", "-C", str(module_path)]
|
|
345
388
|
if submodules:
|
|
346
|
-
pull_cmd.extend([
|
|
389
|
+
pull_cmd.extend(["pull", "--recurse-submodules"])
|
|
347
390
|
subprocess.check_call(pull_cmd)
|
|
348
391
|
logger.success(f"Module '{module}' and its submodules updated.")
|
|
349
392
|
else:
|
|
350
|
-
pull_cmd.extend([
|
|
393
|
+
pull_cmd.extend(["pull"])
|
|
351
394
|
subprocess.check_call(pull_cmd)
|
|
352
395
|
logger.success(f"Module '{module}' updated.")
|
|
353
396
|
else:
|
symai/interfaces.py
CHANGED
|
@@ -5,7 +5,7 @@ from .backend.mixin import OPENAI_CHAT_MODELS, OPENAI_REASONING_MODELS
|
|
|
5
5
|
from .backend.settings import SYMAI_CONFIG
|
|
6
6
|
from .imports import Import
|
|
7
7
|
from .symbol import Expression
|
|
8
|
-
from .utils import
|
|
8
|
+
from .utils import UserMessage
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
class Interface(Expression):
|
|
@@ -13,19 +13,19 @@ class Interface(Expression):
|
|
|
13
13
|
super().__init__(*args, **kwargs)
|
|
14
14
|
self.logger = logging.getLogger(__name__)
|
|
15
15
|
|
|
16
|
-
def __new__(
|
|
16
|
+
def __new__(cls, module: str, *args, **kwargs):
|
|
17
17
|
module = str(module)
|
|
18
18
|
# if `/` in module, assume github repo; else assume local module
|
|
19
19
|
if "/" in module:
|
|
20
20
|
return Import(module)
|
|
21
21
|
module = module.lower()
|
|
22
22
|
module = module.replace("-", "_")
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
return Interface.load_module_class(
|
|
23
|
+
cls._module = module
|
|
24
|
+
cls.module_path = f"symai.extended.interfaces.{module}"
|
|
25
|
+
return Interface.load_module_class(cls.module_path, cls._module)(*args, **kwargs)
|
|
26
26
|
|
|
27
|
-
def __call__(self, *
|
|
28
|
-
|
|
27
|
+
def __call__(self, *_args, **_kwargs):
|
|
28
|
+
UserMessage(f"Interface {self._module} is not callable.", raise_with=NotImplementedError)
|
|
29
29
|
|
|
30
30
|
@staticmethod
|
|
31
31
|
def load_module_class(module_path, class_name):
|
|
@@ -33,39 +33,68 @@ class Interface(Expression):
|
|
|
33
33
|
return getattr(module_, class_name)
|
|
34
34
|
|
|
35
35
|
|
|
36
|
-
def
|
|
37
|
-
"""Maps configuration to interface."""
|
|
38
|
-
mapping = {}
|
|
36
|
+
def _add_symbolic_interface(mapping):
|
|
39
37
|
symbolic_api_key = SYMAI_CONFIG.get("SYMBOLIC_ENGINE_API_KEY")
|
|
40
38
|
if symbolic_api_key is not None:
|
|
41
39
|
mapping["symbolic"] = Interface("wolframalpha")
|
|
42
40
|
|
|
41
|
+
|
|
42
|
+
def _resolve_drawing_interface_name(drawing_engine_model):
|
|
43
|
+
if drawing_engine_model.startswith("flux"):
|
|
44
|
+
return "flux"
|
|
45
|
+
if drawing_engine_model.startswith("dall-e-"):
|
|
46
|
+
return "dall_e"
|
|
47
|
+
if drawing_engine_model.startswith("gpt-image-"):
|
|
48
|
+
return "gpt_image"
|
|
49
|
+
return None
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _add_drawing_interface(mapping):
|
|
43
53
|
drawing_engine_api_key = SYMAI_CONFIG.get("DRAWING_ENGINE_API_KEY")
|
|
44
|
-
if drawing_engine_api_key is
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
54
|
+
if drawing_engine_api_key is None:
|
|
55
|
+
return
|
|
56
|
+
drawing_engine_model = SYMAI_CONFIG.get("DRAWING_ENGINE_MODEL")
|
|
57
|
+
interface_name = _resolve_drawing_interface_name(drawing_engine_model)
|
|
58
|
+
if interface_name is not None:
|
|
59
|
+
mapping["drawing"] = Interface(interface_name)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _resolve_search_interface_name(search_engine_model):
|
|
63
|
+
if search_engine_model.startswith("google"):
|
|
64
|
+
return "serpapi"
|
|
65
|
+
if search_engine_model.startswith("sonar"):
|
|
66
|
+
return "perplexity"
|
|
67
|
+
if search_engine_model in OPENAI_REASONING_MODELS + OPENAI_CHAT_MODELS:
|
|
68
|
+
return "openai_search"
|
|
69
|
+
return None
|
|
52
70
|
|
|
71
|
+
|
|
72
|
+
def _add_search_interface(mapping):
|
|
53
73
|
search_engine_api_key = SYMAI_CONFIG.get("SEARCH_ENGINE_API_KEY")
|
|
54
|
-
if search_engine_api_key is
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
mapping["search"] = Interface("openai_search")
|
|
74
|
+
if search_engine_api_key is None:
|
|
75
|
+
return
|
|
76
|
+
search_engine_model = SYMAI_CONFIG.get("SEARCH_ENGINE_MODEL")
|
|
77
|
+
interface_name = _resolve_search_interface_name(search_engine_model)
|
|
78
|
+
if interface_name is not None:
|
|
79
|
+
mapping["search"] = Interface(interface_name)
|
|
80
|
+
|
|
62
81
|
|
|
82
|
+
def _add_tts_interface(mapping):
|
|
63
83
|
tts_engine_api_key = SYMAI_CONFIG.get("TEXT_TO_SPEECH_ENGINE_API_KEY")
|
|
64
|
-
if tts_engine_api_key is not None:
|
|
84
|
+
if tts_engine_api_key is not None: # TODO: add tests for this engine
|
|
65
85
|
mapping["tts"] = Interface("tts")
|
|
66
86
|
|
|
87
|
+
|
|
88
|
+
def cfg_to_interface():
|
|
89
|
+
"""Maps configuration to interface."""
|
|
90
|
+
mapping = {}
|
|
91
|
+
_add_symbolic_interface(mapping)
|
|
92
|
+
_add_drawing_interface(mapping)
|
|
93
|
+
_add_search_interface(mapping)
|
|
94
|
+
_add_tts_interface(mapping)
|
|
95
|
+
|
|
67
96
|
mapping["indexing"] = Interface("naive_vectordb")
|
|
68
|
-
mapping["scraper"] = Interface("
|
|
97
|
+
mapping["scraper"] = Interface("naive_scrape")
|
|
69
98
|
mapping["stt"] = Interface("whisper")
|
|
70
99
|
mapping["file"] = Interface("file")
|
|
71
100
|
|