lionagi 0.2.11__py3-none-any.whl → 0.3.1__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- lionagi/core/action/function_calling.py +13 -6
- lionagi/core/action/tool.py +10 -9
- lionagi/core/action/tool_manager.py +18 -9
- lionagi/core/agent/README.md +1 -1
- lionagi/core/agent/base_agent.py +5 -2
- lionagi/core/agent/eval/README.md +1 -1
- lionagi/core/collections/README.md +1 -1
- lionagi/core/collections/_logger.py +16 -6
- lionagi/core/collections/abc/README.md +1 -1
- lionagi/core/collections/abc/component.py +35 -11
- lionagi/core/collections/abc/concepts.py +5 -3
- lionagi/core/collections/abc/exceptions.py +3 -1
- lionagi/core/collections/flow.py +16 -5
- lionagi/core/collections/model.py +34 -8
- lionagi/core/collections/pile.py +65 -28
- lionagi/core/collections/progression.py +1 -2
- lionagi/core/collections/util.py +11 -2
- lionagi/core/director/README.md +1 -1
- lionagi/core/engine/branch_engine.py +35 -10
- lionagi/core/engine/instruction_map_engine.py +14 -5
- lionagi/core/engine/sandbox_.py +3 -1
- lionagi/core/engine/script_engine.py +6 -2
- lionagi/core/executor/base_executor.py +10 -3
- lionagi/core/executor/graph_executor.py +12 -4
- lionagi/core/executor/neo4j_executor.py +18 -6
- lionagi/core/generic/edge.py +7 -2
- lionagi/core/generic/graph.py +23 -7
- lionagi/core/generic/node.py +14 -5
- lionagi/core/generic/tree_node.py +5 -1
- lionagi/core/mail/mail_manager.py +3 -1
- lionagi/core/mail/package.py +3 -1
- lionagi/core/message/action_request.py +9 -2
- lionagi/core/message/action_response.py +9 -3
- lionagi/core/message/instruction.py +8 -2
- lionagi/core/message/util.py +15 -5
- lionagi/core/report/base.py +12 -7
- lionagi/core/report/form.py +7 -4
- lionagi/core/report/report.py +10 -3
- lionagi/core/report/util.py +3 -1
- lionagi/core/rule/action.py +4 -1
- lionagi/core/rule/base.py +17 -6
- lionagi/core/rule/rulebook.py +8 -4
- lionagi/core/rule/string.py +3 -1
- lionagi/core/session/branch.py +15 -4
- lionagi/core/session/directive_mixin.py +11 -3
- lionagi/core/session/session.py +6 -2
- lionagi/core/unit/parallel_unit.py +9 -3
- lionagi/core/unit/template/action.py +1 -1
- lionagi/core/unit/template/predict.py +3 -1
- lionagi/core/unit/template/select.py +5 -3
- lionagi/core/unit/unit.py +38 -4
- lionagi/core/unit/unit_form.py +13 -15
- lionagi/core/unit/unit_mixin.py +45 -27
- lionagi/core/unit/util.py +7 -3
- lionagi/core/validator/validator.py +28 -15
- lionagi/core/work/work_edge.py +7 -3
- lionagi/core/work/work_task.py +11 -5
- lionagi/core/work/worker.py +20 -5
- lionagi/core/work/worker_engine.py +6 -2
- lionagi/core/work/worklog.py +3 -1
- lionagi/experimental/compressor/llm_compressor.py +20 -5
- lionagi/experimental/directive/README.md +1 -1
- lionagi/experimental/directive/parser/base_parser.py +41 -14
- lionagi/experimental/directive/parser/base_syntax.txt +23 -23
- lionagi/experimental/directive/template/base_template.py +14 -6
- lionagi/experimental/directive/tokenizer.py +3 -1
- lionagi/experimental/evaluator/README.md +1 -1
- lionagi/experimental/evaluator/ast_evaluator.py +6 -2
- lionagi/experimental/evaluator/base_evaluator.py +27 -16
- lionagi/integrations/bridge/autogen_/autogen_.py +7 -3
- lionagi/integrations/bridge/langchain_/documents.py +13 -10
- lionagi/integrations/bridge/llamaindex_/llama_pack.py +36 -12
- lionagi/integrations/bridge/llamaindex_/node_parser.py +8 -3
- lionagi/integrations/bridge/llamaindex_/reader.py +3 -1
- lionagi/integrations/bridge/llamaindex_/textnode.py +9 -3
- lionagi/integrations/bridge/pydantic_/pydantic_bridge.py +7 -1
- lionagi/integrations/bridge/transformers_/install_.py +3 -1
- lionagi/integrations/chunker/chunk.py +5 -2
- lionagi/integrations/loader/load.py +7 -3
- lionagi/integrations/loader/load_util.py +35 -16
- lionagi/integrations/provider/oai.py +13 -4
- lionagi/integrations/provider/openrouter.py +13 -4
- lionagi/integrations/provider/services.py +3 -1
- lionagi/integrations/provider/transformers.py +5 -3
- lionagi/integrations/storage/neo4j.py +23 -7
- lionagi/integrations/storage/storage_util.py +23 -7
- lionagi/integrations/storage/structure_excel.py +7 -2
- lionagi/integrations/storage/to_csv.py +8 -2
- lionagi/integrations/storage/to_excel.py +11 -3
- lionagi/libs/ln_api.py +41 -19
- lionagi/libs/ln_context.py +4 -4
- lionagi/libs/ln_convert.py +35 -14
- lionagi/libs/ln_dataframe.py +9 -3
- lionagi/libs/ln_func_call.py +53 -18
- lionagi/libs/ln_image.py +9 -5
- lionagi/libs/ln_knowledge_graph.py +21 -7
- lionagi/libs/ln_nested.py +57 -16
- lionagi/libs/ln_parse.py +45 -15
- lionagi/libs/ln_queue.py +8 -3
- lionagi/libs/ln_tokenize.py +19 -6
- lionagi/libs/ln_validate.py +14 -3
- lionagi/libs/sys_util.py +44 -12
- lionagi/lions/coder/coder.py +24 -8
- lionagi/lions/coder/util.py +6 -2
- lionagi/lions/researcher/data_source/google_.py +12 -4
- lionagi/lions/researcher/data_source/wiki_.py +3 -1
- lionagi/version.py +1 -1
- {lionagi-0.2.11.dist-info → lionagi-0.3.1.dist-info}/METADATA +6 -7
- lionagi-0.3.1.dist-info/RECORD +226 -0
- lionagi/tests/__init__.py +0 -0
- lionagi/tests/api/__init__.py +0 -0
- lionagi/tests/api/aws/__init__.py +0 -0
- lionagi/tests/api/aws/conftest.py +0 -25
- lionagi/tests/api/aws/test_aws_s3.py +0 -6
- lionagi/tests/integrations/__init__.py +0 -0
- lionagi/tests/libs/__init__.py +0 -0
- lionagi/tests/libs/test_api.py +0 -48
- lionagi/tests/libs/test_convert.py +0 -89
- lionagi/tests/libs/test_field_validators.py +0 -354
- lionagi/tests/libs/test_func_call.py +0 -701
- lionagi/tests/libs/test_nested.py +0 -382
- lionagi/tests/libs/test_parse.py +0 -171
- lionagi/tests/libs/test_queue.py +0 -68
- lionagi/tests/libs/test_sys_util.py +0 -222
- lionagi/tests/test_core/__init__.py +0 -0
- lionagi/tests/test_core/collections/__init__.py +0 -0
- lionagi/tests/test_core/collections/test_component.py +0 -208
- lionagi/tests/test_core/collections/test_exchange.py +0 -139
- lionagi/tests/test_core/collections/test_flow.py +0 -146
- lionagi/tests/test_core/collections/test_pile.py +0 -172
- lionagi/tests/test_core/collections/test_progression.py +0 -130
- lionagi/tests/test_core/generic/__init__.py +0 -0
- lionagi/tests/test_core/generic/test_edge.py +0 -69
- lionagi/tests/test_core/generic/test_graph.py +0 -97
- lionagi/tests/test_core/generic/test_node.py +0 -107
- lionagi/tests/test_core/generic/test_structure.py +0 -194
- lionagi/tests/test_core/generic/test_tree_node.py +0 -74
- lionagi/tests/test_core/graph/__init__.py +0 -0
- lionagi/tests/test_core/graph/test_graph.py +0 -71
- lionagi/tests/test_core/graph/test_tree.py +0 -76
- lionagi/tests/test_core/mail/__init__.py +0 -0
- lionagi/tests/test_core/mail/test_mail.py +0 -98
- lionagi/tests/test_core/test_branch.py +0 -116
- lionagi/tests/test_core/test_form.py +0 -47
- lionagi/tests/test_core/test_report.py +0 -106
- lionagi/tests/test_core/test_structure/__init__.py +0 -0
- lionagi/tests/test_core/test_structure/test_base_structure.py +0 -198
- lionagi/tests/test_core/test_structure/test_graph.py +0 -55
- lionagi/tests/test_core/test_structure/test_tree.py +0 -49
- lionagi/tests/test_core/test_validator.py +0 -112
- lionagi-0.2.11.dist-info/RECORD +0 -267
- {lionagi-0.2.11.dist-info → lionagi-0.3.1.dist-info}/LICENSE +0 -0
- {lionagi-0.2.11.dist-info → lionagi-0.3.1.dist-info}/WHEEL +0 -0
@@ -20,7 +20,9 @@ class LlamaPack:
|
|
20
20
|
return pack(*args, **kwargs)
|
21
21
|
|
22
22
|
@staticmethod
|
23
|
-
def stock_market_pack(
|
23
|
+
def stock_market_pack(
|
24
|
+
pack_path="./stock_market_data_pack", args=[], **kwargs
|
25
|
+
):
|
24
26
|
name_ = "StockMarketDataQueryEnginePack"
|
25
27
|
return LlamaPack.build(name_, pack_path, args, **kwargs)
|
26
28
|
|
@@ -32,7 +34,9 @@ class LlamaPack:
|
|
32
34
|
return LlamaPack.build(name_, pack_path, args, **kwargs)
|
33
35
|
|
34
36
|
@staticmethod
|
35
|
-
def rag_evaluator_pack(
|
37
|
+
def rag_evaluator_pack(
|
38
|
+
pack_path="./rag_evaluator_pack", args=[], **kwargs
|
39
|
+
):
|
36
40
|
name_ = "RagEvaluatorPack"
|
37
41
|
return LlamaPack.build(name_, pack_path, args, **kwargs)
|
38
42
|
|
@@ -49,17 +53,23 @@ class LlamaPack:
|
|
49
53
|
return LlamaPack.build(name_, pack_path, args, **kwargs)
|
50
54
|
|
51
55
|
@staticmethod
|
52
|
-
def resume_screener_pack(
|
56
|
+
def resume_screener_pack(
|
57
|
+
pack_path="./resume_screener_pack", args=[], **kwargs
|
58
|
+
):
|
53
59
|
name_ = "ResumeScreenerPack"
|
54
60
|
return LlamaPack.build(name_, pack_path, args, **kwargs)
|
55
61
|
|
56
62
|
@staticmethod
|
57
|
-
def ragatouille_retriever_pack(
|
63
|
+
def ragatouille_retriever_pack(
|
64
|
+
pack_path="./ragatouille_pack", args=[], **kwargs
|
65
|
+
):
|
58
66
|
name_ = "RAGatouilleRetrieverPack"
|
59
67
|
return LlamaPack.build(name_, pack_path, args, **kwargs)
|
60
68
|
|
61
69
|
@staticmethod
|
62
|
-
def chain_of_table_pack(
|
70
|
+
def chain_of_table_pack(
|
71
|
+
pack_path="./chain_of_table_pack", args=[], **kwargs
|
72
|
+
):
|
63
73
|
name_ = "ChainOfTablePack"
|
64
74
|
return LlamaPack.build(name_, pack_path, args, **kwargs)
|
65
75
|
|
@@ -152,7 +162,9 @@ class LlamaPack:
|
|
152
162
|
return LlamaPack.build(name_, pack_path, args, **kwargs)
|
153
163
|
|
154
164
|
@staticmethod
|
155
|
-
def llama_guard_moderator_pack(
|
165
|
+
def llama_guard_moderator_pack(
|
166
|
+
pack_path="./llamaguard_pack", args=[], **kwargs
|
167
|
+
):
|
156
168
|
name_ = "LlamaGuardModeratorPack"
|
157
169
|
return LlamaPack.build(name_, pack_path, args, **kwargs)
|
158
170
|
|
@@ -178,7 +190,9 @@ class LlamaPack:
|
|
178
190
|
return LlamaPack.build(name_, pack_path, args, **kwargs)
|
179
191
|
|
180
192
|
@staticmethod
|
181
|
-
def multi_tenancy_rag_pack(
|
193
|
+
def multi_tenancy_rag_pack(
|
194
|
+
pack_path="./multitenancy_rag_pack", args=[], **kwargs
|
195
|
+
):
|
182
196
|
name_ = "MultiTenancyRAGPack"
|
183
197
|
return LlamaPack.build(name_, pack_path, args, **kwargs)
|
184
198
|
|
@@ -188,12 +202,16 @@ class LlamaPack:
|
|
188
202
|
return LlamaPack.build(name_, pack_path, args, **kwargs)
|
189
203
|
|
190
204
|
@staticmethod
|
191
|
-
def snowflake_query_engine_pack(
|
205
|
+
def snowflake_query_engine_pack(
|
206
|
+
pack_path="./snowflake_pack", args=[], **kwargs
|
207
|
+
):
|
192
208
|
name_ = "SnowflakeQueryEnginePack"
|
193
209
|
return LlamaPack.build(name_, pack_path, args, **kwargs)
|
194
210
|
|
195
211
|
@staticmethod
|
196
|
-
def agent_search_retriever_pack(
|
212
|
+
def agent_search_retriever_pack(
|
213
|
+
pack_path="./agent_search_pack", args=[], **kwargs
|
214
|
+
):
|
197
215
|
name_ = "AgentSearchRetrieverPack"
|
198
216
|
return LlamaPack.build(name_, pack_path, args, **kwargs)
|
199
217
|
|
@@ -203,12 +221,16 @@ class LlamaPack:
|
|
203
221
|
return LlamaPack.build(name_, pack_path, args, **kwargs)
|
204
222
|
|
205
223
|
@staticmethod
|
206
|
-
def chroma_autoretrieval_pack(
|
224
|
+
def chroma_autoretrieval_pack(
|
225
|
+
pack_path="./chroma_pack", args=[], **kwargs
|
226
|
+
):
|
207
227
|
name_ = "ChromaAutoretrievalPack"
|
208
228
|
return LlamaPack.build(name_, pack_path, args, **kwargs)
|
209
229
|
|
210
230
|
@staticmethod
|
211
|
-
def arize_phoenix_query_engine_pack(
|
231
|
+
def arize_phoenix_query_engine_pack(
|
232
|
+
pack_path="./arize_pack", args=[], **kwargs
|
233
|
+
):
|
212
234
|
name_ = "ArizePhoenixQueryEnginePack"
|
213
235
|
return LlamaPack.build(name_, pack_path, args, **kwargs)
|
214
236
|
|
@@ -227,6 +249,8 @@ class LlamaPack:
|
|
227
249
|
return LlamaPack.build(name_, pack_path, args, **kwargs)
|
228
250
|
|
229
251
|
@staticmethod
|
230
|
-
def weaviate_retry_engine_pack(
|
252
|
+
def weaviate_retry_engine_pack(
|
253
|
+
pack_path="./weaviate_pack", args=[], **kwargs
|
254
|
+
):
|
231
255
|
name_ = "WeaviateRetryEnginePack"
|
232
256
|
return LlamaPack.build(name_, pack_path, args, **kwargs)
|
@@ -33,7 +33,9 @@ def get_llama_index_node_parser(node_parser: Any):
|
|
33
33
|
pip_name="llama-index",
|
34
34
|
)
|
35
35
|
|
36
|
-
if not isinstance(node_parser, str) and not issubclass(
|
36
|
+
if not isinstance(node_parser, str) and not issubclass(
|
37
|
+
node_parser, NodeParser
|
38
|
+
):
|
37
39
|
raise TypeError("node_parser must be a string or NodeParser.")
|
38
40
|
|
39
41
|
if isinstance(node_parser, str):
|
@@ -41,11 +43,14 @@ def get_llama_index_node_parser(node_parser: Any):
|
|
41
43
|
SysUtil.check_import("tree_sitter_languages")
|
42
44
|
|
43
45
|
try:
|
44
|
-
node_module = import_module(
|
46
|
+
node_module = import_module(
|
47
|
+
"llama_index.core", module_name="node_parser"
|
48
|
+
)
|
45
49
|
return getattr(node_module, node_parser)
|
46
50
|
except Exception as e:
|
47
51
|
raise AttributeError(
|
48
|
-
f"llama_index_core has no such attribute:"
|
52
|
+
f"llama_index_core has no such attribute:"
|
53
|
+
f" {node_parser}, Error: {e}"
|
49
54
|
) from e
|
50
55
|
|
51
56
|
elif isinstance(node_parser, NodeParser):
|
@@ -40,7 +40,9 @@ def get_llama_index_reader(reader: Any | str = None) -> Any:
|
|
40
40
|
]:
|
41
41
|
return SimpleDirectoryReader
|
42
42
|
|
43
|
-
if not isinstance(reader, str) and not issubclass(
|
43
|
+
if not isinstance(reader, str) and not issubclass(
|
44
|
+
reader, BasePydanticReader
|
45
|
+
):
|
44
46
|
raise TypeError(f"reader must be a string or BasePydanticReader.")
|
45
47
|
|
46
48
|
if isinstance(reader, str):
|
@@ -3,7 +3,9 @@ from typing import Any, TypeVar
|
|
3
3
|
from lionagi.libs.sys_util import SysUtil
|
4
4
|
|
5
5
|
|
6
|
-
def to_llama_index_node(
|
6
|
+
def to_llama_index_node(
|
7
|
+
lion_node, node_type: Any = None, **kwargs: Any
|
8
|
+
) -> Any:
|
7
9
|
"""
|
8
10
|
Converts a Lion node to a Llama Index node of a specified type.
|
9
11
|
|
@@ -44,10 +46,14 @@ def to_llama_index_node(lion_node, node_type: Any = None, **kwargs: Any) -> Any:
|
|
44
46
|
if isinstance(node_type, str) and hasattr(
|
45
47
|
llama_index.core.schema, node_type
|
46
48
|
):
|
47
|
-
return getattr(llama_index.core.schema, node_type).from_dict(
|
49
|
+
return getattr(llama_index.core.schema, node_type).from_dict(
|
50
|
+
_dict
|
51
|
+
)
|
48
52
|
elif issubclass(node_type, BaseNode):
|
49
53
|
return node_type.from_dict(_dict)
|
50
54
|
else:
|
51
|
-
raise AttributeError(
|
55
|
+
raise AttributeError(
|
56
|
+
f"Invalid llama-index node type: {node_type}"
|
57
|
+
)
|
52
58
|
except Exception as e:
|
53
59
|
raise AttributeError(f"Error: {e}")
|
@@ -34,4 +34,6 @@ def install_transformers():
|
|
34
34
|
"transformers is required. Would you like to install it now? (y/n): "
|
35
35
|
)
|
36
36
|
if in_ == "y":
|
37
|
-
SysUtil.install_import(
|
37
|
+
SysUtil.install_import(
|
38
|
+
package_name="transformers", import_name="pipeline"
|
39
|
+
)
|
@@ -1,4 +1,5 @@
|
|
1
|
-
from
|
1
|
+
from collections.abc import Callable
|
2
|
+
from typing import Union
|
2
3
|
|
3
4
|
from lionagi.core.collections import pile
|
4
5
|
from lionagi.core.generic import Node
|
@@ -56,7 +57,9 @@ def text_chunker(documents, args, kwargs):
|
|
56
57
|
func_call.lcall(chunks, lambda chunk: chunk.pop("ln_id"))
|
57
58
|
return [Node.from_obj({**chunk}) for chunk in chunks]
|
58
59
|
|
59
|
-
a = to_list(
|
60
|
+
a = to_list(
|
61
|
+
[chunk_node(doc) for doc in documents], flatten=True, dropna=True
|
62
|
+
)
|
60
63
|
return pile(a)
|
61
64
|
|
62
65
|
|
@@ -1,4 +1,4 @@
|
|
1
|
-
from
|
1
|
+
from collections.abc import Callable
|
2
2
|
|
3
3
|
from lionagi.core.collections import pile
|
4
4
|
from lionagi.core.generic import Node
|
@@ -141,7 +141,9 @@ def _plain_reader(reader, reader_args, reader_kwargs):
|
|
141
141
|
) from e
|
142
142
|
|
143
143
|
|
144
|
-
def _langchain_reader(
|
144
|
+
def _langchain_reader(
|
145
|
+
reader, reader_args, reader_kwargs, to_lion: bool | Callable
|
146
|
+
):
|
145
147
|
"""
|
146
148
|
Reads data using a Langchain reader.
|
147
149
|
|
@@ -157,7 +159,9 @@ def _langchain_reader(reader, reader_args, reader_kwargs, to_lion: bool | Callab
|
|
157
159
|
Example usage:
|
158
160
|
>>> nodes = _langchain_reader('langchain_reader', ['arg1'], {'key': 'value'}, True)
|
159
161
|
"""
|
160
|
-
nodes = LangchainBridge.langchain_loader(
|
162
|
+
nodes = LangchainBridge.langchain_loader(
|
163
|
+
reader, reader_args, reader_kwargs
|
164
|
+
)
|
161
165
|
if isinstance(to_lion, bool) and to_lion is True:
|
162
166
|
return pile([Node.from_langchain(i) for i in nodes])
|
163
167
|
|
@@ -24,7 +24,7 @@ class ChunkerType(str, Enum):
|
|
24
24
|
|
25
25
|
def dir_to_path(
|
26
26
|
dir: str, ext: str, recursive: bool = False, flatten: bool = True
|
27
|
-
) ->
|
27
|
+
) -> list[Path]:
|
28
28
|
"""
|
29
29
|
Generates a list of file paths from a directory with the given file extension.
|
30
30
|
|
@@ -53,16 +53,18 @@ def dir_to_path(
|
|
53
53
|
func_call.lcall(ext, _dir_to_path, flatten=True), flatten=flatten
|
54
54
|
)
|
55
55
|
except:
|
56
|
-
raise ValueError(
|
56
|
+
raise ValueError(
|
57
|
+
"Invalid directory or extension, please check the path"
|
58
|
+
)
|
57
59
|
|
58
60
|
|
59
61
|
def dir_to_nodes(
|
60
62
|
dir_: str,
|
61
|
-
ext:
|
63
|
+
ext: list[str] | str,
|
62
64
|
recursive: bool = False,
|
63
65
|
flatten: bool = True,
|
64
66
|
clean_text: bool = True,
|
65
|
-
) ->
|
67
|
+
) -> list[Node]:
|
66
68
|
"""
|
67
69
|
Converts directory contents into Node objects based on specified file extensions.
|
68
70
|
|
@@ -84,12 +86,16 @@ def dir_to_nodes(
|
|
84
86
|
# converting them into Node objects.
|
85
87
|
"""
|
86
88
|
|
87
|
-
path_list = dir_to_path(
|
89
|
+
path_list = dir_to_path(
|
90
|
+
dir=dir_, ext=ext, recursive=recursive, flatten=flatten
|
91
|
+
)
|
88
92
|
files_info = func_call.lcall(path_list, read_text, clean=clean_text)
|
89
|
-
return func_call.lcall(
|
93
|
+
return func_call.lcall(
|
94
|
+
files_info, lambda x: Node(content=x[0], metadata=x[1])
|
95
|
+
)
|
90
96
|
|
91
97
|
|
92
|
-
def read_text(filepath: str, clean: bool = True) ->
|
98
|
+
def read_text(filepath: str, clean: bool = True) -> tuple[str, dict]:
|
93
99
|
"""
|
94
100
|
Reads text from a file and optionally cleans it, returning the content and metadata.
|
95
101
|
|
@@ -115,9 +121,15 @@ def read_text(filepath: str, clean: bool = True) -> Tuple[str, dict]:
|
|
115
121
|
|
116
122
|
file = filepath
|
117
123
|
size = os.path.getsize(filepath)
|
118
|
-
creation_date = datetime.fromtimestamp(
|
119
|
-
|
120
|
-
|
124
|
+
creation_date = datetime.fromtimestamp(
|
125
|
+
os.path.getctime(filepath)
|
126
|
+
).date()
|
127
|
+
modified_date = datetime.fromtimestamp(
|
128
|
+
os.path.getmtime(filepath)
|
129
|
+
).date()
|
130
|
+
last_accessed_date = datetime.fromtimestamp(
|
131
|
+
os.path.getatime(filepath)
|
132
|
+
).date()
|
121
133
|
return {
|
122
134
|
"file": convert.to_str(file),
|
123
135
|
"size": size,
|
@@ -127,7 +139,7 @@ def read_text(filepath: str, clean: bool = True) -> Tuple[str, dict]:
|
|
127
139
|
}
|
128
140
|
|
129
141
|
try:
|
130
|
-
with open(filepath
|
142
|
+
with open(filepath) as f:
|
131
143
|
content = f.read()
|
132
144
|
if clean:
|
133
145
|
# Define characters to replace and their replacements
|
@@ -141,19 +153,22 @@ def read_text(filepath: str, clean: bool = True) -> Tuple[str, dict]:
|
|
141
153
|
|
142
154
|
|
143
155
|
def _file_to_chunks(
|
144
|
-
input:
|
156
|
+
input: dict[str, Any],
|
145
157
|
field: str = "content",
|
146
158
|
chunk_size: int = 1500,
|
147
159
|
overlap: float = 0.1,
|
148
160
|
threshold: int = 200,
|
149
|
-
) ->
|
161
|
+
) -> list[dict[str, Any]]:
|
150
162
|
try:
|
151
163
|
out = {key: value for key, value in input.items() if key != field} | {
|
152
164
|
"chunk_overlap": overlap,
|
153
165
|
"chunk_threshold": threshold,
|
154
166
|
}
|
155
167
|
chunks = TokenizeUtil.chunk_by_chars(
|
156
|
-
input[field],
|
168
|
+
input[field],
|
169
|
+
chunk_size=chunk_size,
|
170
|
+
overlap=overlap,
|
171
|
+
threshold=threshold,
|
157
172
|
)
|
158
173
|
logs = []
|
159
174
|
for i, chunk in enumerate(chunks):
|
@@ -168,7 +183,9 @@ def _file_to_chunks(
|
|
168
183
|
return logs
|
169
184
|
|
170
185
|
except Exception as e:
|
171
|
-
raise ValueError(
|
186
|
+
raise ValueError(
|
187
|
+
f"An error occurred while chunking the file. {e}"
|
188
|
+
) from e
|
172
189
|
|
173
190
|
|
174
191
|
# needs doing TODO
|
@@ -184,7 +201,9 @@ def file_to_chunks(
|
|
184
201
|
# verbose=True,
|
185
202
|
# timestamp=True,
|
186
203
|
# logger=None,
|
187
|
-
return convert.to_list(
|
204
|
+
return convert.to_list(
|
205
|
+
func_call.lcall(input, chunk_func, **kwargs), flatten=True
|
206
|
+
)
|
188
207
|
|
189
208
|
|
190
209
|
def _datanode_parser(nodes, parser):
|
@@ -76,7 +76,9 @@ class OpenAIService(BaseService):
|
|
76
76
|
self.active_endpoint = []
|
77
77
|
self.allowed_kwargs = allowed_kwargs
|
78
78
|
|
79
|
-
async def serve(
|
79
|
+
async def serve(
|
80
|
+
self, input_, endpoint="chat/completions", method="post", **kwargs
|
81
|
+
):
|
80
82
|
"""
|
81
83
|
Serves the input using the specified endpoint and method.
|
82
84
|
|
@@ -137,14 +139,18 @@ class OpenAIService(BaseService):
|
|
137
139
|
_content = []
|
138
140
|
for i in content:
|
139
141
|
if "text" in i:
|
140
|
-
_content.append(
|
142
|
+
_content.append(
|
143
|
+
{"type": "text", "text": str(i["text"])}
|
144
|
+
)
|
141
145
|
elif "image_url" in i:
|
142
146
|
_content.append(
|
143
147
|
{
|
144
148
|
"type": "image_url",
|
145
149
|
"image_url": {
|
146
150
|
"url": f"{i['image_url'].get('url')}",
|
147
|
-
"detail": i["image_url"].get(
|
151
|
+
"detail": i["image_url"].get(
|
152
|
+
"detail", "low"
|
153
|
+
),
|
148
154
|
},
|
149
155
|
}
|
150
156
|
)
|
@@ -158,7 +164,10 @@ class OpenAIService(BaseService):
|
|
158
164
|
)
|
159
165
|
try:
|
160
166
|
completion = await self.call_api(
|
161
|
-
payload,
|
167
|
+
payload,
|
168
|
+
"chat/completions",
|
169
|
+
"post",
|
170
|
+
required_tokens=required_tokens,
|
162
171
|
)
|
163
172
|
return payload, completion
|
164
173
|
except Exception as e:
|
@@ -50,7 +50,9 @@ class OpenRouterService(BaseService):
|
|
50
50
|
self.active_endpoint = []
|
51
51
|
self.allowed_kwargs = allowed_kwargs
|
52
52
|
|
53
|
-
async def serve(
|
53
|
+
async def serve(
|
54
|
+
self, input_, endpoint="chat/completions", method="post", **kwargs
|
55
|
+
):
|
54
56
|
"""
|
55
57
|
Serves the input using the specified endpoint and method.
|
56
58
|
|
@@ -111,14 +113,18 @@ class OpenRouterService(BaseService):
|
|
111
113
|
_content = []
|
112
114
|
for i in content:
|
113
115
|
if "text" in i:
|
114
|
-
_content.append(
|
116
|
+
_content.append(
|
117
|
+
{"type": "text", "text": str(i["text"])}
|
118
|
+
)
|
115
119
|
elif "image_url" in i:
|
116
120
|
_content.append(
|
117
121
|
{
|
118
122
|
"type": "image_url",
|
119
123
|
"image_url": {
|
120
124
|
"url": f"{i['image_url'].get('url')}",
|
121
|
-
"detail": i["image_url"].get(
|
125
|
+
"detail": i["image_url"].get(
|
126
|
+
"detail", "low"
|
127
|
+
),
|
122
128
|
},
|
123
129
|
}
|
124
130
|
)
|
@@ -132,7 +138,10 @@ class OpenRouterService(BaseService):
|
|
132
138
|
)
|
133
139
|
try:
|
134
140
|
completion = await self.call_api(
|
135
|
-
payload,
|
141
|
+
payload,
|
142
|
+
"chat/completions",
|
143
|
+
"post",
|
144
|
+
required_tokens=required_tokens,
|
136
145
|
)
|
137
146
|
return payload, completion
|
138
147
|
except Exception as e:
|
@@ -68,7 +68,9 @@ class Services:
|
|
68
68
|
asynchronous operations.
|
69
69
|
"""
|
70
70
|
|
71
|
-
from lionagi.integrations.provider.transformers import
|
71
|
+
from lionagi.integrations.provider.transformers import (
|
72
|
+
TransformersService,
|
73
|
+
)
|
72
74
|
|
73
75
|
return TransformersService(**kwargs)
|
74
76
|
|
@@ -46,8 +46,8 @@ class TransformersService(BaseService):
|
|
46
46
|
def __init__(
|
47
47
|
self,
|
48
48
|
task: str = None,
|
49
|
-
model:
|
50
|
-
config:
|
49
|
+
model: str | Any = None,
|
50
|
+
config: str | dict | Any = None,
|
51
51
|
device="cpu",
|
52
52
|
**kwargs,
|
53
53
|
):
|
@@ -83,7 +83,9 @@ class TransformersService(BaseService):
|
|
83
83
|
async def serve_chat(self, messages, **kwargs):
|
84
84
|
if self.task:
|
85
85
|
if self.task != "conversational":
|
86
|
-
raise ValueError(
|
86
|
+
raise ValueError(
|
87
|
+
f"Invalid transformers pipeline task: {self.task}."
|
88
|
+
)
|
87
89
|
|
88
90
|
payload = {"messages": messages}
|
89
91
|
config = {}
|
@@ -1,6 +1,9 @@
|
|
1
1
|
from neo4j import AsyncGraphDatabase
|
2
2
|
|
3
|
-
from lionagi.integrations.storage.storage_util import
|
3
|
+
from lionagi.integrations.storage.storage_util import (
|
4
|
+
output_edge_list,
|
5
|
+
output_node_list,
|
6
|
+
)
|
4
7
|
|
5
8
|
|
6
9
|
class Neo4j:
|
@@ -45,7 +48,9 @@ class Neo4j:
|
|
45
48
|
SET n.timestamp = $timestamp
|
46
49
|
SET n.name = $name
|
47
50
|
"""
|
48
|
-
await tx.run(
|
51
|
+
await tx.run(
|
52
|
+
query, ln_id=node["ln_id"], timestamp=node["timestamp"], name=name
|
53
|
+
)
|
49
54
|
# heads=node['head_nodes'],
|
50
55
|
# nodes=node['nodes'],
|
51
56
|
# edges=node['edges'])
|
@@ -247,7 +252,9 @@ class Neo4j:
|
|
247
252
|
MERGE (n:EdgeCondition:LionNode {className: $className})
|
248
253
|
SET n.code = $code
|
249
254
|
"""
|
250
|
-
await tx.run(
|
255
|
+
await tx.run(
|
256
|
+
query, className=condCls["class_name"], code=condCls["class"]
|
257
|
+
)
|
251
258
|
|
252
259
|
async def add_node(self, tx, node_dict, structure_name):
|
253
260
|
"""
|
@@ -278,7 +285,10 @@ class Neo4j:
|
|
278
285
|
elif node == "Tool":
|
279
286
|
[await self.add_tool_node(tx, i) for i in node_list]
|
280
287
|
elif node == "DirectiveSelection":
|
281
|
-
[
|
288
|
+
[
|
289
|
+
await self.add_directiveSelection_node(tx, i)
|
290
|
+
for i in node_list
|
291
|
+
]
|
282
292
|
elif node == "BaseAgent":
|
283
293
|
[await self.add_baseAgent_node(tx, i) for i in node_list]
|
284
294
|
else:
|
@@ -382,7 +392,9 @@ class Neo4j:
|
|
382
392
|
await self.check_structure_name_constraint(tx)
|
383
393
|
await tx.commit()
|
384
394
|
except Exception as e:
|
385
|
-
raise ValueError(
|
395
|
+
raise ValueError(
|
396
|
+
f"transaction rolled back due to exception: {e}"
|
397
|
+
)
|
386
398
|
finally:
|
387
399
|
await tx.close()
|
388
400
|
|
@@ -395,7 +407,9 @@ class Neo4j:
|
|
395
407
|
await self.add_head_edge(tx, structure)
|
396
408
|
await tx.commit()
|
397
409
|
except Exception as e:
|
398
|
-
raise ValueError(
|
410
|
+
raise ValueError(
|
411
|
+
f"transaction rolled back due to exception: {e}"
|
412
|
+
)
|
399
413
|
finally:
|
400
414
|
await tx.close()
|
401
415
|
|
@@ -566,7 +580,9 @@ class Neo4j:
|
|
566
580
|
else:
|
567
581
|
raise ValueError(f"Structure id {structure_id} is invalid")
|
568
582
|
|
569
|
-
async def get_heads(
|
583
|
+
async def get_heads(
|
584
|
+
self, structure_name: str = None, structure_id: str = None
|
585
|
+
):
|
570
586
|
"""
|
571
587
|
Asynchronously retrieves the head nodes associated with a given structure in the graph.
|
572
588
|
|
@@ -58,7 +58,9 @@ def output_node_list(structure):
|
|
58
58
|
elif isinstance(node, BaseAgent):
|
59
59
|
node_output["structure_id"] = node.structure.ln_id
|
60
60
|
node_output["output_parser"] = (
|
61
|
-
inspect.getsource(node.output_parser)
|
61
|
+
inspect.getsource(node.output_parser)
|
62
|
+
if node.output_parser
|
63
|
+
else None
|
62
64
|
)
|
63
65
|
else:
|
64
66
|
raise ValueError("Not supported node type detected")
|
@@ -94,7 +96,9 @@ def output_edge_list(structure):
|
|
94
96
|
}
|
95
97
|
if edge.condition:
|
96
98
|
cls_name = edge.condition.__class__.__qualname__
|
97
|
-
cond = json.dumps(
|
99
|
+
cond = json.dumps(
|
100
|
+
{"class": cls_name, "args": edge.condition.model_dump()}
|
101
|
+
)
|
98
102
|
cls = edge.string_condition()
|
99
103
|
if cls is not None and cls_name not in edge_cls_dict:
|
100
104
|
edge_cls_dict.update({cls_name: cls})
|
@@ -184,7 +188,9 @@ class ParseNode:
|
|
184
188
|
Returns:
|
185
189
|
System: An instantiated System node filled with properties from info_dict.
|
186
190
|
"""
|
187
|
-
info_dict["system"] = json.loads(info_dict.pop("content"))[
|
191
|
+
info_dict["system"] = json.loads(info_dict.pop("content"))[
|
192
|
+
"system_info"
|
193
|
+
]
|
188
194
|
node = System.from_obj(info_dict)
|
189
195
|
return node
|
190
196
|
|
@@ -199,7 +205,9 @@ class ParseNode:
|
|
199
205
|
Returns:
|
200
206
|
Instruction: An instantiated Instruction node filled with properties from info_dict.
|
201
207
|
"""
|
202
|
-
info_dict["instruction"] = json.loads(info_dict.pop("content"))[
|
208
|
+
info_dict["instruction"] = json.loads(info_dict.pop("content"))[
|
209
|
+
"instruction"
|
210
|
+
]
|
203
211
|
node = Instruction.from_obj(info_dict)
|
204
212
|
return node
|
205
213
|
|
@@ -218,10 +226,14 @@ class ParseNode:
|
|
218
226
|
node.directive = info_dict["directive"]
|
219
227
|
if "directive_kwargs" in info_dict:
|
220
228
|
if info_dict["directive_kwargs"]:
|
221
|
-
node.directive_kwargs = json.loads(
|
229
|
+
node.directive_kwargs = json.loads(
|
230
|
+
info_dict["directive_kwargs"]
|
231
|
+
)
|
222
232
|
elif "directiveKwargs" in info_dict:
|
223
233
|
if info_dict["directiveKwargs"]:
|
224
|
-
node.directive_kwargs = json.loads(
|
234
|
+
node.directive_kwargs = json.loads(
|
235
|
+
info_dict["directiveKwargs"]
|
236
|
+
)
|
225
237
|
return node
|
226
238
|
|
227
239
|
@staticmethod
|
@@ -239,7 +251,11 @@ class ParseNode:
|
|
239
251
|
ValueError: If unsafe code is detected in the function definition.
|
240
252
|
"""
|
241
253
|
func_code = info_dict["function"]
|
242
|
-
if
|
254
|
+
if (
|
255
|
+
"import os" in func_code
|
256
|
+
or "__" in func_code
|
257
|
+
or "import sys" in func_code
|
258
|
+
):
|
243
259
|
raise ValueError(
|
244
260
|
"Unsafe code detected in Tool function. Please double check or implement explicitly"
|
245
261
|
)
|
@@ -10,7 +10,9 @@ from lionagi.core.executor.graph_executor import GraphExecutor
|
|
10
10
|
from lionagi.integrations.storage.storage_util import ParseNode
|
11
11
|
|
12
12
|
|
13
|
-
def excel_reload(
|
13
|
+
def excel_reload(
|
14
|
+
structure_name=None, structure_id=None, dir="structure_storage"
|
15
|
+
):
|
14
16
|
"""
|
15
17
|
Loads a structure from an Excel file into a StructureExecutor instance.
|
16
18
|
|
@@ -58,7 +60,10 @@ class StructureExcel:
|
|
58
60
|
default_agent_executable: BaseExecutor = InstructionMapEngine()
|
59
61
|
|
60
62
|
def __init__(
|
61
|
-
self,
|
63
|
+
self,
|
64
|
+
structure_name=None,
|
65
|
+
structure_id=None,
|
66
|
+
file_path="structure_storage",
|
62
67
|
):
|
63
68
|
"""
|
64
69
|
Initializes the StructureExcel class with specified parameters.
|