stores 0.1.6__tar.gz → 0.1.7.dev1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {stores-0.1.6 → stores-0.1.7.dev1}/PKG-INFO +1 -1
- stores-0.1.7.dev1/local_generator.py +33 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/pyproject.toml +1 -1
- {stores-0.1.6 → stores-0.1.7.dev1}/stores/indexes/base_index.py +177 -44
- {stores-0.1.6 → stores-0.1.7.dev1}/stores/indexes/index.py +11 -1
- {stores-0.1.6 → stores-0.1.7.dev1}/stores/indexes/local_index.py +20 -4
- {stores-0.1.6 → stores-0.1.7.dev1}/stores/indexes/remote_index.py +7 -1
- {stores-0.1.6 → stores-0.1.7.dev1}/stores/indexes/venv_utils.py +143 -48
- {stores-0.1.6 → stores-0.1.7.dev1}/uv.lock +1 -1
- stores-0.1.6/run_complex.py +0 -123
- {stores-0.1.6 → stores-0.1.7.dev1}/.gitignore +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/.python-version +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/LICENSE +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/README.md +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/examples/README.md +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/examples/quickstarts/anthropic_api.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/examples/quickstarts/google_gemini_auto_call.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/examples/quickstarts/google_gemini_manual_call.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/examples/quickstarts/langchain_w_tool_calling.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/examples/quickstarts/langgraph_agent.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/examples/quickstarts/litellm_w_tool_calling.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/examples/quickstarts/llamaindex_agent.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/examples/quickstarts/openai_agent.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/examples/quickstarts/openai_chat_completions.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/examples/quickstarts/openai_responses.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/run_remote_tool.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/stores/__init__.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/stores/constants.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/stores/format.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/stores/indexes/__init__.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/stores/parse.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/stores/utils.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/tests/README.md +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/tests/mock_index/hello/__init__.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/tests/mock_index/tools.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/tests/mock_index/tools.toml +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/tests/mock_index_custom_class/foo.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/tests/mock_index_custom_class/tools.toml +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/tests/mock_index_function_error/foo.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/tests/mock_index_function_error/tools.toml +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/tests/mock_index_w_deps/mock_index/__init__.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/tests/mock_index_w_deps/pyproject.toml +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/tests/mock_index_w_deps/requirements.txt +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/tests/mock_index_w_deps/tools.toml +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/tests/test_format/conftest.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/tests/test_format/test_format.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/tests/test_indexes/conftest.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/tests/test_indexes/test_base_index.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/tests/test_indexes/test_index.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/tests/test_indexes/test_local_index.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/tests/test_indexes/test_remote_index.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/tests/test_indexes/test_venv_utils.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/tests/test_parse/conftest.py +0 -0
- {stores-0.1.6 → stores-0.1.7.dev1}/tests/test_parse/test_parse.py +0 -0
@@ -0,0 +1,33 @@
|
|
1
|
+
import asyncio
|
2
|
+
|
3
|
+
import stores
|
4
|
+
|
5
|
+
|
6
|
+
def func():
|
7
|
+
yield "Hello"
|
8
|
+
yield 51
|
9
|
+
yield "Good Bye"
|
10
|
+
|
11
|
+
|
12
|
+
async def func_d():
|
13
|
+
for i in range(3):
|
14
|
+
await asyncio.sleep(0.5)
|
15
|
+
yield f"sleep {i}"
|
16
|
+
|
17
|
+
|
18
|
+
index = stores.Index(
|
19
|
+
["silanthro/todoist"], include={"silanthro/todoist": ["todoist.func"]}
|
20
|
+
)
|
21
|
+
|
22
|
+
print(index.tools)
|
23
|
+
quit()
|
24
|
+
|
25
|
+
|
26
|
+
# for i in index.tools[1]():
|
27
|
+
# print(i)
|
28
|
+
|
29
|
+
print(index.execute("todoist.func_d"))
|
30
|
+
|
31
|
+
|
32
|
+
for i in index.stream_execute("todoist.func_d"):
|
33
|
+
print(i)
|
@@ -128,6 +128,29 @@ def _undo_non_string_literal(annotation: type, value: Any, literal_map: dict):
|
|
128
128
|
return value
|
129
129
|
|
130
130
|
|
131
|
+
def _preprocess_args(
|
132
|
+
original_signature: inspect.Signature, literal_maps: dict, args: list, kwargs: dict
|
133
|
+
):
|
134
|
+
# Inject default values within wrapper
|
135
|
+
bound_args = original_signature.bind(*args, **kwargs)
|
136
|
+
bound_args.apply_defaults()
|
137
|
+
for k, v in bound_args.arguments.items():
|
138
|
+
if (
|
139
|
+
v is None
|
140
|
+
and original_signature.parameters[k].default is not Parameter.empty
|
141
|
+
):
|
142
|
+
bound_args.arguments[k] = original_signature.parameters[k].default
|
143
|
+
_cast_bound_args(bound_args)
|
144
|
+
# Inject correct Literals
|
145
|
+
for k, v in bound_args.arguments.items():
|
146
|
+
if k in literal_maps:
|
147
|
+
param = original_signature.parameters[k]
|
148
|
+
bound_args.arguments[k] = _undo_non_string_literal(
|
149
|
+
param.annotation, v, literal_maps[k]
|
150
|
+
)
|
151
|
+
return bound_args
|
152
|
+
|
153
|
+
|
131
154
|
def wrap_tool(tool: Callable):
|
132
155
|
"""
|
133
156
|
Wrap tool to make it compatible with LLM libraries
|
@@ -189,42 +212,37 @@ def wrap_tool(tool: Callable):
|
|
189
212
|
new_args.append(new_arg)
|
190
213
|
new_sig = original_signature.replace(parameters=new_args)
|
191
214
|
|
192
|
-
if inspect.
|
215
|
+
if inspect.isasyncgenfunction(tool):
|
216
|
+
|
217
|
+
async def wrapper(*args, **kwargs):
|
218
|
+
bound_args = _preprocess_args(
|
219
|
+
original_signature, literal_maps, args, kwargs
|
220
|
+
)
|
221
|
+
async for value in tool(*bound_args.args, **bound_args.kwargs):
|
222
|
+
yield value
|
223
|
+
|
224
|
+
elif inspect.isgeneratorfunction(tool):
|
225
|
+
|
226
|
+
def wrapper(*args, **kwargs):
|
227
|
+
bound_args = _preprocess_args(
|
228
|
+
original_signature, literal_maps, args, kwargs
|
229
|
+
)
|
230
|
+
for value in tool(*bound_args.args, **bound_args.kwargs):
|
231
|
+
yield value
|
232
|
+
|
233
|
+
elif inspect.iscoroutinefunction(tool):
|
193
234
|
|
194
235
|
async def wrapper(*args, **kwargs):
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
_cast_bound_args(bound_args)
|
199
|
-
# Inject correct Literals
|
200
|
-
for k, v in bound_args.arguments.items():
|
201
|
-
if k in literal_maps:
|
202
|
-
param = original_signature.parameters[k]
|
203
|
-
bound_args.arguments[k] = _undo_non_string_literal(
|
204
|
-
param.annotation, v, literal_maps[k]
|
205
|
-
)
|
236
|
+
bound_args = _preprocess_args(
|
237
|
+
original_signature, literal_maps, args, kwargs
|
238
|
+
)
|
206
239
|
return await tool(*bound_args.args, **bound_args.kwargs)
|
207
240
|
else:
|
208
241
|
|
209
242
|
def wrapper(*args, **kwargs):
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
# Inject correct Literals
|
214
|
-
for k, v in bound_args.arguments.items():
|
215
|
-
if (
|
216
|
-
v is None
|
217
|
-
and original_signature.parameters[k].default is not Parameter.empty
|
218
|
-
):
|
219
|
-
bound_args.arguments[k] = original_signature.parameters[k].default
|
220
|
-
|
221
|
-
_cast_bound_args(bound_args)
|
222
|
-
for k, v in bound_args.arguments.items():
|
223
|
-
if k in literal_maps:
|
224
|
-
param = original_signature.parameters[k]
|
225
|
-
bound_args.arguments[k] = _undo_non_string_literal(
|
226
|
-
param.annotation, v, literal_maps[k]
|
227
|
-
)
|
243
|
+
bound_args = _preprocess_args(
|
244
|
+
original_signature, literal_maps, args, kwargs
|
245
|
+
)
|
228
246
|
return tool(*bound_args.args, **bound_args.kwargs)
|
229
247
|
|
230
248
|
wrapped = create_function(
|
@@ -249,14 +267,7 @@ class BaseIndex:
|
|
249
267
|
def tools_dict(self):
|
250
268
|
return {tool.__name__: tool for tool in self.tools}
|
251
269
|
|
252
|
-
def
|
253
|
-
loop = asyncio.new_event_loop()
|
254
|
-
asyncio.set_event_loop(loop)
|
255
|
-
return loop.run_until_complete(self.async_execute(toolname, kwargs))
|
256
|
-
|
257
|
-
async def async_execute(self, toolname: str, kwargs: dict | None = None):
|
258
|
-
kwargs = kwargs or {}
|
259
|
-
|
270
|
+
def _get_tool(self, toolname: str):
|
260
271
|
# Use regex since we need to match cases where we perform
|
261
272
|
# substitutions such as replace(".", "-")
|
262
273
|
pattern = re.compile(":?" + re.sub("-|\\.", "(-|\\.)", toolname) + "$")
|
@@ -272,11 +283,124 @@ class BaseIndex:
|
|
272
283
|
else:
|
273
284
|
toolname = matching_tools[0]
|
274
285
|
|
275
|
-
|
276
|
-
|
277
|
-
|
286
|
+
return self.tools_dict[toolname]
|
287
|
+
|
288
|
+
def execute(self, toolname: str, kwargs: dict | None = None, collect_results=False):
|
289
|
+
tool_fn = self._get_tool(toolname)
|
290
|
+
kwargs = kwargs or {}
|
291
|
+
if inspect.isasyncgenfunction(tool_fn):
|
292
|
+
# Handle async generator
|
293
|
+
|
294
|
+
async def collect():
|
295
|
+
results = []
|
296
|
+
async for value in tool_fn(**kwargs):
|
297
|
+
results.append(value)
|
298
|
+
if collect_results:
|
299
|
+
return results
|
300
|
+
else:
|
301
|
+
return results[-1]
|
302
|
+
|
303
|
+
loop = asyncio.new_event_loop()
|
304
|
+
asyncio.set_event_loop(loop)
|
305
|
+
return loop.run_until_complete(collect())
|
306
|
+
elif inspect.isgeneratorfunction(tool_fn):
|
307
|
+
# Handle sync generator
|
308
|
+
results = []
|
309
|
+
for value in tool_fn(**kwargs):
|
310
|
+
results.append(value)
|
311
|
+
if collect_results:
|
312
|
+
return results
|
313
|
+
else:
|
314
|
+
return results[-1]
|
315
|
+
elif inspect.iscoroutinefunction(tool_fn):
|
316
|
+
# Handle async
|
317
|
+
loop = asyncio.new_event_loop()
|
318
|
+
asyncio.set_event_loop(loop)
|
319
|
+
return loop.run_until_complete(tool_fn(**kwargs))
|
320
|
+
else:
|
321
|
+
# Handle sync
|
322
|
+
return tool_fn(**kwargs)
|
323
|
+
|
324
|
+
async def aexecute(
|
325
|
+
self, toolname: str, kwargs: dict | None = None, collect_results=False
|
326
|
+
):
|
327
|
+
tool_fn = self._get_tool(toolname)
|
328
|
+
kwargs = kwargs or {}
|
329
|
+
if inspect.isasyncgenfunction(tool_fn):
|
330
|
+
# Handle async generator
|
331
|
+
results = []
|
332
|
+
async for value in tool_fn(**kwargs):
|
333
|
+
results.append(value)
|
334
|
+
if collect_results:
|
335
|
+
return results
|
336
|
+
else:
|
337
|
+
return results[-1]
|
338
|
+
elif inspect.isgeneratorfunction(tool_fn):
|
339
|
+
# Handle sync generator
|
340
|
+
results = []
|
341
|
+
for value in tool_fn(**kwargs):
|
342
|
+
results.append(value)
|
343
|
+
if collect_results:
|
344
|
+
return results
|
345
|
+
else:
|
346
|
+
return results[-1]
|
347
|
+
elif inspect.iscoroutinefunction(tool_fn):
|
348
|
+
# Handle async
|
349
|
+
return await tool_fn(**kwargs)
|
350
|
+
else:
|
351
|
+
# Handle sync
|
352
|
+
return tool_fn(**kwargs)
|
353
|
+
|
354
|
+
def stream_execute(self, toolname: str, kwargs: dict | None = None):
|
355
|
+
tool_fn = self._get_tool(toolname)
|
356
|
+
kwargs = kwargs or {}
|
357
|
+
if inspect.isasyncgenfunction(tool_fn):
|
358
|
+
# Handle async generator
|
359
|
+
|
360
|
+
async def collect():
|
361
|
+
async for value in tool_fn(**kwargs):
|
362
|
+
yield value
|
363
|
+
|
364
|
+
loop = asyncio.new_event_loop()
|
365
|
+
asyncio.set_event_loop(loop)
|
366
|
+
agen = collect()
|
367
|
+
try:
|
368
|
+
while True:
|
369
|
+
yield loop.run_until_complete(agen.__anext__())
|
370
|
+
except StopAsyncIteration:
|
371
|
+
pass
|
372
|
+
finally:
|
373
|
+
loop.close()
|
374
|
+
elif inspect.isgeneratorfunction(tool_fn):
|
375
|
+
# Handle sync generator
|
376
|
+
for value in tool_fn(**kwargs):
|
377
|
+
yield value
|
378
|
+
elif inspect.iscoroutinefunction(tool_fn):
|
379
|
+
# Handle async
|
380
|
+
loop = asyncio.new_event_loop()
|
381
|
+
asyncio.set_event_loop(loop)
|
382
|
+
yield loop.run_until_complete(tool_fn(**kwargs))
|
383
|
+
else:
|
384
|
+
# Handle sync
|
385
|
+
yield tool_fn(**kwargs)
|
386
|
+
|
387
|
+
async def astream_execute(self, toolname: str, kwargs: dict | None = None):
|
388
|
+
tool_fn = self._get_tool(toolname)
|
389
|
+
kwargs = kwargs or {}
|
390
|
+
if inspect.isasyncgenfunction(tool_fn):
|
391
|
+
# Handle async generator
|
392
|
+
async for value in tool_fn(**kwargs):
|
393
|
+
yield value
|
394
|
+
elif inspect.isgeneratorfunction(tool_fn):
|
395
|
+
# Handle sync generator
|
396
|
+
for value in tool_fn(**kwargs):
|
397
|
+
yield value
|
398
|
+
elif inspect.iscoroutinefunction(tool_fn):
|
399
|
+
# Handle async
|
400
|
+
yield await tool_fn(**kwargs)
|
278
401
|
else:
|
279
|
-
|
402
|
+
# Handle sync
|
403
|
+
yield tool_fn(**kwargs)
|
280
404
|
|
281
405
|
def parse_and_execute(self, msg: str):
|
282
406
|
toolcall = llm_parse_json(msg, keys=["toolname", "kwargs"])
|
@@ -284,9 +408,18 @@ class BaseIndex:
|
|
284
408
|
|
285
409
|
async def async_parse_and_execute(self, msg: str):
|
286
410
|
toolcall = llm_parse_json(msg, keys=["toolname", "kwargs"])
|
287
|
-
return await self.
|
411
|
+
return await self.aexecute(toolcall.get("toolname"), toolcall.get("kwargs"))
|
412
|
+
|
413
|
+
def stream_parse_and_execute(self, msg: str):
|
414
|
+
toolcall = llm_parse_json(msg, keys=["toolname", "kwargs"])
|
415
|
+
return self.stream_execute(toolcall.get("toolname"), toolcall.get("kwargs"))
|
416
|
+
|
417
|
+
async def astream_parse_and_execute(self, msg: str):
|
418
|
+
toolcall = llm_parse_json(msg, keys=["toolname", "kwargs"])
|
419
|
+
async for value in self.astream_parse_and_execute(
|
288
420
|
toolcall.get("toolname"), toolcall.get("kwargs")
|
289
|
-
)
|
421
|
+
):
|
422
|
+
yield value
|
290
423
|
|
291
424
|
def format_tools(self, provider: ProviderFormat):
|
292
425
|
return format_tools(self.tools, provider)
|
@@ -17,11 +17,15 @@ class Index(BaseIndex):
|
|
17
17
|
self,
|
18
18
|
tools: list[Callable, os.PathLike] | None = None,
|
19
19
|
env_var: dict[str, dict] | None = None,
|
20
|
+
include: dict[str, list[str]] | None = None,
|
21
|
+
exclude: dict[str, list[str]] | None = None,
|
20
22
|
cache_dir: Optional[os.PathLike] = None,
|
21
23
|
reset_cache=False,
|
22
24
|
sys_executable: str | None = None,
|
23
25
|
):
|
24
26
|
self.env_var = env_var or {}
|
27
|
+
include = include or {}
|
28
|
+
exclude = exclude or {}
|
25
29
|
tools = tools or []
|
26
30
|
|
27
31
|
_tools = []
|
@@ -32,7 +36,11 @@ class Index(BaseIndex):
|
|
32
36
|
if Path(index_name).exists():
|
33
37
|
# Load LocalIndex
|
34
38
|
try:
|
35
|
-
loaded_index = LocalIndex(
|
39
|
+
loaded_index = LocalIndex(
|
40
|
+
index_name,
|
41
|
+
include=include.get(index_name),
|
42
|
+
exclude=exclude.get(index_name),
|
43
|
+
)
|
36
44
|
except Exception:
|
37
45
|
logger.warning(
|
38
46
|
f'Unable to load index "{index_name}"', exc_info=True
|
@@ -43,6 +51,8 @@ class Index(BaseIndex):
|
|
43
51
|
loaded_index = RemoteIndex(
|
44
52
|
index_name,
|
45
53
|
env_var=self.env_var.get(index_name),
|
54
|
+
include=include.get(index_name),
|
55
|
+
exclude=exclude.get(index_name),
|
46
56
|
cache_dir=cache_dir,
|
47
57
|
reset_cache=reset_cache,
|
48
58
|
sys_executable=sys_executable,
|
@@ -26,10 +26,14 @@ class LocalIndex(BaseIndex):
|
|
26
26
|
index_folder: os.PathLike,
|
27
27
|
create_venv: bool = False,
|
28
28
|
env_var: dict | None = None,
|
29
|
+
include: list[str] | None = None,
|
30
|
+
exclude: list[str] | None = None,
|
29
31
|
sys_executable: str | None = None,
|
30
32
|
):
|
31
33
|
self.index_folder = Path(index_folder)
|
32
34
|
self.env_var = env_var or {}
|
35
|
+
include = include or []
|
36
|
+
exclude = exclude or []
|
33
37
|
|
34
38
|
if not self.index_folder.exists():
|
35
39
|
raise ValueError(
|
@@ -50,21 +54,31 @@ class LocalIndex(BaseIndex):
|
|
50
54
|
)
|
51
55
|
install_venv_deps(self.index_folder)
|
52
56
|
# Initialize tools
|
53
|
-
tools = init_venv_tools(
|
57
|
+
tools = init_venv_tools(
|
58
|
+
self.index_folder,
|
59
|
+
env_var=self.env_var,
|
60
|
+
include=include,
|
61
|
+
exclude=exclude,
|
62
|
+
)
|
54
63
|
else:
|
55
64
|
if self.env_var:
|
56
65
|
raise ValueError(
|
57
66
|
"Environment variables will only be restricted if create_venv=True when initializing LocalIndex"
|
58
67
|
)
|
59
|
-
tools = self._init_tools()
|
68
|
+
tools = self._init_tools(include=include, exclude=exclude)
|
60
69
|
super().__init__(tools)
|
61
70
|
|
62
|
-
def _init_tools(
|
71
|
+
def _init_tools(
|
72
|
+
self, include: list[str] | None = None, exclude: list[str] | None = None
|
73
|
+
):
|
63
74
|
"""
|
64
75
|
Load local tools.toml file and import tool functions
|
65
76
|
|
66
77
|
NOTE: Can we just add index_folder to sys.path and import the functions?
|
67
78
|
"""
|
79
|
+
include = include or []
|
80
|
+
exclude = exclude or []
|
81
|
+
|
68
82
|
index_manifest = self.index_folder / TOOLS_CONFIG_FILENAME
|
69
83
|
if not index_manifest.exists():
|
70
84
|
raise ValueError(f"Unable to load index - {index_manifest} does not exist")
|
@@ -73,7 +87,9 @@ class LocalIndex(BaseIndex):
|
|
73
87
|
manifest = tomllib.load(file)["index"]
|
74
88
|
|
75
89
|
tools = []
|
76
|
-
for tool_id in manifest.get("tools", []):
|
90
|
+
for tool_id in include or manifest.get("tools", []):
|
91
|
+
if tool_id in exclude:
|
92
|
+
continue
|
77
93
|
module_name = ".".join(tool_id.split(".")[:-1])
|
78
94
|
tool_name = tool_id.split(".")[-1]
|
79
95
|
|
@@ -51,6 +51,8 @@ class RemoteIndex(BaseIndex):
|
|
51
51
|
self,
|
52
52
|
index_id: str,
|
53
53
|
env_var: dict | None = None,
|
54
|
+
include: list[str] | None = None,
|
55
|
+
exclude: list[str] | None = None,
|
54
56
|
cache_dir: Optional[PathLike] = None,
|
55
57
|
reset_cache=False,
|
56
58
|
sys_executable: str | None = None,
|
@@ -64,6 +66,8 @@ class RemoteIndex(BaseIndex):
|
|
64
66
|
shutil.rmtree(cache_dir)
|
65
67
|
self.index_folder = cache_dir / self.index_id
|
66
68
|
self.env_var = env_var or {}
|
69
|
+
include = include or []
|
70
|
+
exclude = exclude or []
|
67
71
|
if not self.index_folder.exists():
|
68
72
|
logger.info(f"Installing {index_id}...")
|
69
73
|
commit_like = None
|
@@ -99,5 +103,7 @@ class RemoteIndex(BaseIndex):
|
|
99
103
|
venv.create(self.venv, symlinks=True, with_pip=True, upgrade_deps=True)
|
100
104
|
install_venv_deps(self.index_folder)
|
101
105
|
# Initialize tools
|
102
|
-
tools = init_venv_tools(
|
106
|
+
tools = init_venv_tools(
|
107
|
+
self.index_folder, env_var=self.env_var, include=include, exclude=exclude
|
108
|
+
)
|
103
109
|
super().__init__(tools)
|
@@ -104,16 +104,25 @@ def install_venv_deps(index_folder: os.PathLike):
|
|
104
104
|
return message
|
105
105
|
|
106
106
|
|
107
|
-
def init_venv_tools(
|
107
|
+
def init_venv_tools(
|
108
|
+
index_folder: os.PathLike,
|
109
|
+
env_var: dict | None = None,
|
110
|
+
include: list[str] | None = None,
|
111
|
+
exclude: list[str] | None = None,
|
112
|
+
):
|
108
113
|
index_folder = Path(index_folder)
|
109
114
|
env_var = env_var or {}
|
115
|
+
include = include or []
|
116
|
+
exclude = exclude or []
|
110
117
|
|
111
118
|
index_manifest = index_folder / TOOLS_CONFIG_FILENAME
|
112
119
|
with open(index_manifest, "rb") as file:
|
113
120
|
manifest = tomllib.load(file)["index"]
|
114
121
|
|
115
122
|
tools = []
|
116
|
-
for tool_id in manifest.get("tools", []):
|
123
|
+
for tool_id in include or manifest.get("tools", []):
|
124
|
+
if tool_id in exclude:
|
125
|
+
continue
|
117
126
|
tool_sig = get_tool_signature(
|
118
127
|
tool_id=tool_id,
|
119
128
|
index_folder=index_folder,
|
@@ -215,7 +224,9 @@ try:
|
|
215
224
|
"tool_id": "{tool_id}",
|
216
225
|
"params": params,
|
217
226
|
"return": return_info,
|
218
|
-
"
|
227
|
+
"iscoroutinefunction": inspect.iscoroutinefunction({tool_name}),
|
228
|
+
"isgeneratorfunction": inspect.isgeneratorfunction({tool_name}),
|
229
|
+
"isasyncgenfunction": inspect.isasyncgenfunction({tool_name}),
|
219
230
|
"doc": inspect.getdoc({tool_name}),
|
220
231
|
}},
|
221
232
|
}},
|
@@ -293,25 +304,56 @@ def parse_tool_signature(
|
|
293
304
|
"""
|
294
305
|
env_var = env_var or {}
|
295
306
|
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
|
307
|
+
if signature_dict.get("isasyncgenfunction"):
|
308
|
+
|
309
|
+
async def func_handler(*args, **kwargs):
|
310
|
+
# TODO: Make this truly async
|
311
|
+
for value in run_remote_tool(
|
312
|
+
tool_id=signature_dict["tool_id"],
|
313
|
+
index_folder=index_folder,
|
314
|
+
args=args,
|
315
|
+
kwargs=kwargs,
|
316
|
+
venv=venv,
|
317
|
+
env_var=env_var,
|
318
|
+
stream=True,
|
319
|
+
):
|
320
|
+
yield value
|
321
|
+
elif signature_dict.get("isgeneratorfunction"):
|
322
|
+
|
323
|
+
def func_handler(*args, **kwargs):
|
324
|
+
for value in run_remote_tool(
|
325
|
+
tool_id=signature_dict["tool_id"],
|
326
|
+
index_folder=index_folder,
|
327
|
+
args=args,
|
328
|
+
kwargs=kwargs,
|
329
|
+
venv=venv,
|
330
|
+
env_var=env_var,
|
331
|
+
stream=True,
|
332
|
+
):
|
333
|
+
yield value
|
334
|
+
elif signature_dict.get("iscoroutinefunction"):
|
335
|
+
|
336
|
+
async def func_handler(*args, **kwargs):
|
337
|
+
# TODO: Make this truly async
|
338
|
+
return run_remote_tool(
|
339
|
+
tool_id=signature_dict["tool_id"],
|
340
|
+
index_folder=index_folder,
|
341
|
+
args=args,
|
342
|
+
kwargs=kwargs,
|
343
|
+
venv=venv,
|
344
|
+
env_var=env_var,
|
345
|
+
)
|
346
|
+
else:
|
305
347
|
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
348
|
+
def func_handler(*args, **kwargs):
|
349
|
+
return run_remote_tool(
|
350
|
+
tool_id=signature_dict["tool_id"],
|
351
|
+
index_folder=index_folder,
|
352
|
+
args=args,
|
353
|
+
kwargs=kwargs,
|
354
|
+
venv=venv,
|
355
|
+
env_var=env_var,
|
356
|
+
)
|
315
357
|
|
316
358
|
# Reconstruct signature from list of args
|
317
359
|
params = []
|
@@ -329,7 +371,7 @@ def parse_tool_signature(
|
|
329
371
|
signature = inspect.Signature(params, return_annotation=return_type)
|
330
372
|
func = create_function(
|
331
373
|
signature,
|
332
|
-
|
374
|
+
func_handler,
|
333
375
|
qualname=signature_dict["tool_id"],
|
334
376
|
doc=signature_dict.get("doc"),
|
335
377
|
)
|
@@ -345,6 +387,7 @@ def run_remote_tool(
|
|
345
387
|
kwargs: dict | None = None,
|
346
388
|
venv: str = VENV_NAME,
|
347
389
|
env_var: dict | None = None,
|
390
|
+
stream: bool = False,
|
348
391
|
):
|
349
392
|
args = args or []
|
350
393
|
kwargs = kwargs or {}
|
@@ -365,56 +408,108 @@ def run_remote_tool(
|
|
365
408
|
listener.listen(1)
|
366
409
|
_, port = listener.getsockname()
|
367
410
|
|
368
|
-
|
411
|
+
result_data = {}
|
412
|
+
|
413
|
+
def handle_connection(stream_mode: bool = False):
|
369
414
|
conn, _ = listener.accept()
|
370
415
|
with conn:
|
371
|
-
|
416
|
+
buffer = ""
|
372
417
|
while True:
|
373
|
-
chunk = conn.recv(4096)
|
418
|
+
chunk = conn.recv(4096).decode("utf-8")
|
374
419
|
if not chunk:
|
375
420
|
break
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
|
384
|
-
|
421
|
+
buffer += chunk
|
422
|
+
while "\n" in buffer:
|
423
|
+
line, buffer = buffer.split("\n", 1)
|
424
|
+
if not line.strip():
|
425
|
+
continue
|
426
|
+
msg = json.loads(line)
|
427
|
+
if msg.get("ok") and "stream" in msg:
|
428
|
+
if stream_mode:
|
429
|
+
yield msg["stream"]
|
430
|
+
else:
|
431
|
+
result_data.setdefault("stream", []).append(msg["stream"])
|
432
|
+
elif msg.get("ok") and "result" in msg:
|
433
|
+
result_data["result"] = msg["result"]
|
434
|
+
elif "error" in msg:
|
435
|
+
result_data["error"] = msg["error"]
|
436
|
+
elif msg.get("done"):
|
437
|
+
if stream_mode:
|
438
|
+
break
|
439
|
+
else:
|
440
|
+
return
|
441
|
+
|
442
|
+
if not stream:
|
443
|
+
thread = threading.Thread(
|
444
|
+
target=lambda: list(handle_connection(stream_mode=False))
|
445
|
+
)
|
446
|
+
thread.start()
|
385
447
|
|
386
448
|
runner = f"""
|
387
449
|
import asyncio, inspect, json, socket, sys, traceback
|
388
450
|
sys.path.insert(0, "{index_folder}")
|
451
|
+
|
452
|
+
def send(sock, payload):
|
453
|
+
sock.sendall((json.dumps(payload) + "\\n").encode("utf-8"))
|
454
|
+
|
455
|
+
sock = socket.create_connection(("localhost", {port}))
|
456
|
+
|
389
457
|
try:
|
390
458
|
from {module_name} import {tool_name}
|
391
459
|
params = json.load(sys.stdin)
|
392
460
|
args = params.get("args", [])
|
393
461
|
kwargs = params.get("kwargs", {{}})
|
394
|
-
|
462
|
+
|
463
|
+
func = {tool_name}
|
464
|
+
|
465
|
+
if inspect.isasyncgenfunction(func):
|
466
|
+
async def run():
|
467
|
+
async for value in func(*args, **kwargs):
|
468
|
+
send(sock, {{"ok": True, "stream": value}})
|
395
469
|
loop = asyncio.new_event_loop()
|
396
470
|
asyncio.set_event_loop(loop)
|
397
|
-
|
471
|
+
loop.run_until_complete(run())
|
472
|
+
elif inspect.isgeneratorfunction(func):
|
473
|
+
for value in func(*args, **kwargs):
|
474
|
+
send(sock, {{"ok": True, "stream": value}})
|
475
|
+
elif inspect.iscoroutinefunction(func):
|
476
|
+
loop = asyncio.new_event_loop()
|
477
|
+
asyncio.set_event_loop(loop)
|
478
|
+
result = loop.run_until_complete(func(*args, **kwargs))
|
479
|
+
send(sock, {{"ok": True, "result": result}})
|
398
480
|
else:
|
399
|
-
result =
|
400
|
-
|
481
|
+
result = func(*args, **kwargs)
|
482
|
+
send(sock, {{"ok": True, "result": result}})
|
483
|
+
send(sock, {{"done": True}})
|
401
484
|
except Exception as e:
|
402
485
|
err = traceback.format_exc()
|
403
|
-
|
404
|
-
sock
|
405
|
-
|
406
|
-
|
486
|
+
try:
|
487
|
+
send(sock, {{"ok": False, "error": err}})
|
488
|
+
except:
|
489
|
+
pass
|
490
|
+
finally:
|
491
|
+
try:
|
492
|
+
sock.close()
|
493
|
+
except:
|
494
|
+
pass
|
407
495
|
"""
|
408
|
-
|
496
|
+
subprocess.run(
|
409
497
|
[get_python_command(Path(index_folder) / venv), "-c", runner],
|
410
498
|
input=payload,
|
411
499
|
capture_output=True,
|
412
500
|
env=env_var or None,
|
413
501
|
)
|
414
502
|
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
503
|
+
if not stream:
|
504
|
+
thread.join()
|
505
|
+
|
506
|
+
if "error" in result_data:
|
507
|
+
raise RuntimeError(f"Subprocess failed with error:\n{result_data['error']}")
|
508
|
+
elif "result" in result_data:
|
509
|
+
return result_data["result"]
|
510
|
+
elif "stream" in result_data:
|
511
|
+
return result_data["stream"]
|
512
|
+
else:
|
513
|
+
raise RuntimeError("Subprocess completed without returning data.")
|
419
514
|
else:
|
420
|
-
|
515
|
+
return handle_connection(stream_mode=True)
|
stores-0.1.6/run_complex.py
DELETED
@@ -1,123 +0,0 @@
|
|
1
|
-
# import anthropic
|
2
|
-
import inspect
|
3
|
-
|
4
|
-
import dotenv
|
5
|
-
from langchain_core.tools import tool as callable_as_lc_tool
|
6
|
-
from langchain_google_genai import ChatGoogleGenerativeAI
|
7
|
-
|
8
|
-
# from openai import OpenAI
|
9
|
-
# from google import genai
|
10
|
-
# from google.genai import types
|
11
|
-
# from litellm import completion
|
12
|
-
# from stores.indexes import LocalIndex
|
13
|
-
import stores
|
14
|
-
|
15
|
-
dotenv.load_dotenv()
|
16
|
-
|
17
|
-
|
18
|
-
# def foo(bar: str = "test"):
|
19
|
-
# return bar
|
20
|
-
|
21
|
-
|
22
|
-
# index = stores.Index([foo])
|
23
|
-
# print(inspect.signature(index.tools[0]))
|
24
|
-
# print(index.tools[0]())
|
25
|
-
# quit()
|
26
|
-
|
27
|
-
|
28
|
-
index = stores.Index(
|
29
|
-
["silanthro/todoist"],
|
30
|
-
env_var={
|
31
|
-
"silanthro/todoist": {
|
32
|
-
"TODOIST_API_TOKEN": "6a0815ff164d7ebd6d45d433bdbd32fcb164e1ec",
|
33
|
-
}
|
34
|
-
},
|
35
|
-
)
|
36
|
-
|
37
|
-
sig = inspect.signature(index.tools[0])
|
38
|
-
print(sig)
|
39
|
-
|
40
|
-
wrap = callable_as_lc_tool()(index.tools[0])
|
41
|
-
sig_wrap = inspect.signature(wrap)
|
42
|
-
print(sig_wrap)
|
43
|
-
# quit()
|
44
|
-
|
45
|
-
for argname, arg in sig.parameters.items():
|
46
|
-
print(argname)
|
47
|
-
print(arg.annotation)
|
48
|
-
print([arg.default])
|
49
|
-
|
50
|
-
# print(sig)
|
51
|
-
# quit()
|
52
|
-
|
53
|
-
# result = index.execute(
|
54
|
-
# "todoist.get_tasks",
|
55
|
-
# {
|
56
|
-
# "project_id": None,
|
57
|
-
# "search_query": None,
|
58
|
-
# "due_date_filter": "today",
|
59
|
-
# "priority": None,
|
60
|
-
# "other_filters": None,
|
61
|
-
# "limit": None,
|
62
|
-
# },
|
63
|
-
# )
|
64
|
-
# print(result)
|
65
|
-
# quit()
|
66
|
-
|
67
|
-
|
68
|
-
task = "Find tasks of priority 1"
|
69
|
-
|
70
|
-
# print(index.tools)
|
71
|
-
|
72
|
-
# print(index.format_tools("openai-chat-completions"))
|
73
|
-
# quit()
|
74
|
-
|
75
|
-
# client = OpenAI()
|
76
|
-
# response = client.responses.create(
|
77
|
-
# model="gpt-4o-mini-2024-07-18",
|
78
|
-
# input=[{"role": "user", "content": task}],
|
79
|
-
# tools=index.format_tools("openai-responses"),
|
80
|
-
# )
|
81
|
-
# client = anthropic.Anthropic()
|
82
|
-
# response = client.messages.create(
|
83
|
-
# model="claude-3-5-sonnet-20241022",
|
84
|
-
# max_tokens=1024,
|
85
|
-
# messages=[{"role": "user", "content": task}],
|
86
|
-
# tools=index.format_tools("anthropic"),
|
87
|
-
# )
|
88
|
-
# response = completion(
|
89
|
-
# model="gemini/gemini-2.0-flash-001",
|
90
|
-
# messages=[
|
91
|
-
# {
|
92
|
-
# "role": "user",
|
93
|
-
# "content": task,
|
94
|
-
# }
|
95
|
-
# ],
|
96
|
-
# tools=index.format_tools("google-gemini"),
|
97
|
-
# )
|
98
|
-
# client = genai.Client(
|
99
|
-
# # api_key=os.environ["GEMINI_API_KEY"],
|
100
|
-
# )
|
101
|
-
# config = types.GenerateContentConfig(
|
102
|
-
# tools=index.tools,
|
103
|
-
# automatic_function_calling=types.AutomaticFunctionCallingConfig(
|
104
|
-
# disable=True # Gemini automatically executes tool calls. This script shows how to manually execute tool calls.
|
105
|
-
# ),
|
106
|
-
# )
|
107
|
-
|
108
|
-
# # Get the response from the model
|
109
|
-
# response = client.models.generate_content(
|
110
|
-
# model="gemini-2.0-flash",
|
111
|
-
# contents=task,
|
112
|
-
# config=config,
|
113
|
-
# )
|
114
|
-
# tool_call = response.candidates[0].content.parts[0].function_call
|
115
|
-
|
116
|
-
model = ChatGoogleGenerativeAI(model="gemini-2.0-flash-001")
|
117
|
-
model_with_tools = model.bind_tools(index.tools)
|
118
|
-
response = model_with_tools.invoke(task)
|
119
|
-
tool_call = response.tool_calls[0]
|
120
|
-
|
121
|
-
print(tool_call)
|
122
|
-
# Execute the tool call
|
123
|
-
# print(response)
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|