lionagi 0.0.112__py3-none-any.whl → 0.0.113__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- lionagi/__init__.py +3 -3
- lionagi/bridge/__init__.py +7 -0
- lionagi/bridge/langchain.py +131 -0
- lionagi/bridge/llama_index.py +157 -0
- lionagi/configs/__init__.py +7 -0
- lionagi/configs/oai_configs.py +49 -0
- lionagi/configs/openrouter_config.py +49 -0
- lionagi/core/__init__.py +8 -2
- lionagi/core/instruction_sets.py +1 -3
- lionagi/core/messages.py +2 -2
- lionagi/core/sessions.py +174 -27
- lionagi/datastore/__init__.py +1 -0
- lionagi/loader/__init__.py +9 -4
- lionagi/loader/chunker.py +157 -0
- lionagi/loader/reader.py +124 -0
- lionagi/objs/__init__.py +7 -0
- lionagi/objs/messenger.py +163 -0
- lionagi/objs/tool_registry.py +247 -0
- lionagi/schema/__init__.py +11 -0
- lionagi/schema/base_schema.py +239 -0
- lionagi/schema/base_tool.py +9 -0
- lionagi/schema/data_logger.py +94 -0
- lionagi/services/__init__.py +14 -0
- lionagi/{service_/oai.py → services/base_api_service.py} +49 -82
- lionagi/{endpoint/base_endpoint.py → services/chatcompletion.py} +19 -22
- lionagi/services/oai.py +34 -0
- lionagi/services/openrouter.py +32 -0
- lionagi/{service_/service_utils.py → services/service_objs.py} +0 -1
- lionagi/structure/__init__.py +7 -0
- lionagi/structure/relationship.py +128 -0
- lionagi/structure/structure.py +160 -0
- lionagi/tests/test_flatten_util.py +426 -0
- lionagi/tools/__init__.py +0 -5
- lionagi/tools/coder.py +1 -0
- lionagi/tools/scorer.py +1 -0
- lionagi/tools/validator.py +1 -0
- lionagi/utils/__init__.py +46 -20
- lionagi/utils/api_util.py +86 -0
- lionagi/utils/call_util.py +347 -0
- lionagi/utils/flat_util.py +540 -0
- lionagi/utils/io_util.py +102 -0
- lionagi/utils/load_utils.py +190 -0
- lionagi/utils/sys_util.py +191 -0
- lionagi/utils/tool_util.py +92 -0
- lionagi/utils/type_util.py +81 -0
- lionagi/version.py +1 -1
- {lionagi-0.0.112.dist-info → lionagi-0.0.113.dist-info}/METADATA +37 -13
- lionagi-0.0.113.dist-info/RECORD +84 -0
- lionagi/endpoint/chat_completion.py +0 -20
- lionagi/endpoint/endpoint_utils.py +0 -0
- lionagi/llm_configs.py +0 -21
- lionagi/loader/load_utils.py +0 -161
- lionagi/schema.py +0 -275
- lionagi/service_/__init__.py +0 -6
- lionagi/service_/base_service.py +0 -48
- lionagi/service_/openrouter.py +0 -1
- lionagi/services.py +0 -1
- lionagi/tools/tool_utils.py +0 -75
- lionagi/utils/sys_utils.py +0 -799
- lionagi-0.0.112.dist-info/RECORD +0 -67
- /lionagi/{core/responses.py → datastore/chroma.py} +0 -0
- /lionagi/{endpoint/assistants.py → datastore/deeplake.py} +0 -0
- /lionagi/{endpoint/audio.py → datastore/elasticsearch.py} +0 -0
- /lionagi/{endpoint/embeddings.py → datastore/lantern.py} +0 -0
- /lionagi/{endpoint/files.py → datastore/pinecone.py} +0 -0
- /lionagi/{endpoint/fine_tuning.py → datastore/postgres.py} +0 -0
- /lionagi/{endpoint/images.py → datastore/qdrant.py} +0 -0
- /lionagi/{endpoint/messages.py → schema/base_condition.py} +0 -0
- /lionagi/{service_ → services}/anthropic.py +0 -0
- /lionagi/{service_ → services}/anyscale.py +0 -0
- /lionagi/{service_ → services}/azure.py +0 -0
- /lionagi/{service_ → services}/bedrock.py +0 -0
- /lionagi/{service_ → services}/everlyai.py +0 -0
- /lionagi/{service_ → services}/gemini.py +0 -0
- /lionagi/{service_ → services}/gpt4all.py +0 -0
- /lionagi/{service_ → services}/huggingface.py +0 -0
- /lionagi/{service_ → services}/litellm.py +0 -0
- /lionagi/{service_ → services}/localai.py +0 -0
- /lionagi/{service_ → services}/mistralai.py +0 -0
- /lionagi/{service_ → services}/ollama.py +0 -0
- /lionagi/{service_ → services}/openllm.py +0 -0
- /lionagi/{service_ → services}/perplexity.py +0 -0
- /lionagi/{service_ → services}/predibase.py +0 -0
- /lionagi/{service_ → services}/rungpt.py +0 -0
- /lionagi/{service_ → services}/vllm.py +0 -0
- /lionagi/{service_ → services}/xinference.py +0 -0
- /lionagi/{endpoint → tests}/__init__.py +0 -0
- /lionagi/{endpoint/models.py → tools/planner.py} +0 -0
- /lionagi/{endpoint/moderations.py → tools/prompter.py} +0 -0
- /lionagi/{endpoint/runs.py → tools/sandbox.py} +0 -0
- /lionagi/{endpoint/threads.py → tools/summarizer.py} +0 -0
- {lionagi-0.0.112.dist-info → lionagi-0.0.113.dist-info}/LICENSE +0 -0
- {lionagi-0.0.112.dist-info → lionagi-0.0.113.dist-info}/WHEEL +0 -0
- {lionagi-0.0.112.dist-info → lionagi-0.0.113.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,84 @@
|
|
1
|
+
lionagi/__init__.py,sha256=gAOIyJ-kItUn2ya8G3zW6pvuzmk60xk7zO_GN66UJAM,888
|
2
|
+
lionagi/version.py,sha256=WJPRYfehqmqr60a1h9m6a2fbtJa0dT03ixD6hahuBgE,24
|
3
|
+
lionagi/bridge/__init__.py,sha256=I19ztSDDwV1-e-GSRs45bJgjbg454e0vcFv2jwgwQY0,139
|
4
|
+
lionagi/bridge/langchain.py,sha256=nJdnv_zHpchQXUlh5ml6OZKGO9Rbaa7B-XbeKB3Ej7Y,4579
|
5
|
+
lionagi/bridge/llama_index.py,sha256=sapGIgS3RkDUyrPCdjZizoRdAmktZLHzUbYQBw6nnZw,5510
|
6
|
+
lionagi/configs/__init__.py,sha256=88KoeoB8dHhgZdQ8I4rbb7e8-FUd-_gi1dXaA95vJ3Y,141
|
7
|
+
lionagi/configs/oai_configs.py,sha256=C89jXb2Xb7qoOXIhtrMlIRVYJqjh7knpu6CAOloj9eo,1199
|
8
|
+
lionagi/configs/openrouter_config.py,sha256=qbEQQqQPpzMp3eP5U8kuvGTUYdMgemuuq-BdgOYgYI8,1256
|
9
|
+
lionagi/core/__init__.py,sha256=XAw2vF1wuQYKSAxHcBanfYN0k-5qLSggkl3yZgwjSRw,347
|
10
|
+
lionagi/core/conversations.py,sha256=Jh4Bvv0SBaW5ZG_D9hyWveQ-PXPDGfvp_V7UxbPsFoA,4084
|
11
|
+
lionagi/core/flows.py,sha256=rhPBmdN-JKo7Zcv9hiy6k0b-CjKBHMemkYITM7a9fNo,32
|
12
|
+
lionagi/core/instruction_sets.py,sha256=y-fDltfDtji4fCtHn8fJ_M9OtYCfjqeTds7bpzknvhU,51
|
13
|
+
lionagi/core/messages.py,sha256=XisuXcJBUoyYcd-6DLcV_NZnsKXzZcFf4Dk_fxTlXc8,6533
|
14
|
+
lionagi/core/sessions.py,sha256=sjg7bCORFhpaEgDBTExyhfgGxRAUwkOCft0MmP-BGoU,11161
|
15
|
+
lionagi/datastore/__init__.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
16
|
+
lionagi/datastore/chroma.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
17
|
+
lionagi/datastore/deeplake.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
18
|
+
lionagi/datastore/elasticsearch.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
19
|
+
lionagi/datastore/lantern.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
20
|
+
lionagi/datastore/pinecone.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
21
|
+
lionagi/datastore/postgres.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
22
|
+
lionagi/datastore/qdrant.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
23
|
+
lionagi/loader/__init__.py,sha256=0FNrggZnSq1ONLZY2f325SF19UfYHaqJeJXLvpu8KWQ,262
|
24
|
+
lionagi/loader/chunker.py,sha256=zGdfS-Gybxc6lm_vMIdt-f25kY2CwDFTLvY-arKxQ8c,6476
|
25
|
+
lionagi/loader/reader.py,sha256=8kxYlqhwmMpuNTWdYtsNc9g-1Mf264Y8XRu1GkevhTg,4527
|
26
|
+
lionagi/objs/__init__.py,sha256=4qxo9ZGKe37fjj_Gt2hgi4QTTIPmDf7mkyUK7euVmsU,137
|
27
|
+
lionagi/objs/messenger.py,sha256=k-ueq2LBLTTDgk8u-Vyja8WBFM7dWFInWO97HNyOXaI,6137
|
28
|
+
lionagi/objs/tool_registry.py,sha256=cnoaW1TvNhKKerQ4tTUOSHEADmsTdoUW2lT24LcHIx0,9357
|
29
|
+
lionagi/schema/__init__.py,sha256=cB7rSYeRXujDwp2f5IMK83-NzdqPfbMhGijPCPXgT_c,210
|
30
|
+
lionagi/schema/base_condition.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
31
|
+
lionagi/schema/base_schema.py,sha256=gx4LDxYnVO9vqp0JdevUSasqrat7Lu3rRURTiv7gGj0,8348
|
32
|
+
lionagi/schema/base_tool.py,sha256=sKbCrS0ZZINkr03Fj0lk0Kt0XPiBz9IXwA9A1aDgN88,192
|
33
|
+
lionagi/schema/data_logger.py,sha256=nmv0jJDkE-vzz_KyBl1_bHB8KVvhZ6sItOBQcvBkAvE,3417
|
34
|
+
lionagi/services/__init__.py,sha256=js4LnJEJv4kkxPPamkOgFkOsrc1yr2dK-z9CLTTsX04,313
|
35
|
+
lionagi/services/anthropic.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
36
|
+
lionagi/services/anyscale.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
37
|
+
lionagi/services/azure.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
38
|
+
lionagi/services/base_api_service.py,sha256=vGCnC1GnW8czpseOx8mcDeyiiAKc_dabUInxjytE-A0,9756
|
39
|
+
lionagi/services/bedrock.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
40
|
+
lionagi/services/chatcompletion.py,sha256=9jVkB4JUX-kzvAHsohA1ZO6ZeXdlN9OAe-cVvLCa9-0,1257
|
41
|
+
lionagi/services/everlyai.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
42
|
+
lionagi/services/gemini.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
43
|
+
lionagi/services/gpt4all.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
44
|
+
lionagi/services/huggingface.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
45
|
+
lionagi/services/litellm.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
46
|
+
lionagi/services/localai.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
47
|
+
lionagi/services/mistralai.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
48
|
+
lionagi/services/oai.py,sha256=NcKiqmQ8YpKEkoVv-6a-mvIZVBumWOVpstpVD0oTiUs,1122
|
49
|
+
lionagi/services/ollama.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
50
|
+
lionagi/services/openllm.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
51
|
+
lionagi/services/openrouter.py,sha256=vjY0ZydLY5XK4_T2cRiAl8_QNysWy26wcfyIP-1hlSw,1110
|
52
|
+
lionagi/services/perplexity.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
53
|
+
lionagi/services/predibase.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
54
|
+
lionagi/services/rungpt.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
55
|
+
lionagi/services/service_objs.py,sha256=w1Rs69eAGyEukTMMvwQHDmFkwTMf2pB9DdGQ-xCjTUw,9727
|
56
|
+
lionagi/services/vllm.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
57
|
+
lionagi/services/xinference.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
58
|
+
lionagi/structure/__init__.py,sha256=wMPekT2vbWwUkJ5aW5o-lzJC9Fzhta6RHDiFPTNUm_0,120
|
59
|
+
lionagi/structure/relationship.py,sha256=5urGnomfhBJUwPLpYpk5YCx-zO2yprJDaPQqMsgQM0A,4124
|
60
|
+
lionagi/structure/structure.py,sha256=KWVOTFWk_IBMtY57gut_GFUuO7dKWlIrMH7xZ_ys2c4,5952
|
61
|
+
lionagi/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
62
|
+
lionagi/tests/test_flatten_util.py,sha256=-sOAQriGtHK1U0FTd31IpdKJpgMQREpaXp-hf6FAojQ,17362
|
63
|
+
lionagi/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
64
|
+
lionagi/tools/coder.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
65
|
+
lionagi/tools/planner.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
66
|
+
lionagi/tools/prompter.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
67
|
+
lionagi/tools/sandbox.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
68
|
+
lionagi/tools/scorer.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
69
|
+
lionagi/tools/summarizer.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
70
|
+
lionagi/tools/validator.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
71
|
+
lionagi/utils/__init__.py,sha256=91eJpmYZLXi_xsFV7yb_uvlRmdhBQWBqa9Bmy18nJYY,1419
|
72
|
+
lionagi/utils/api_util.py,sha256=wNYillgmAY9UJ5zP6NLo37levC3JEJGiis7nbPBRYXw,2769
|
73
|
+
lionagi/utils/call_util.py,sha256=g4MRQChRGAz_1W35ThnEkyVR0W-neEk3ylgUq9IxbR8,12440
|
74
|
+
lionagi/utils/flat_util.py,sha256=I1HJXs5llQ6oOoJudT4B7bp9u9_TLEFT_yPcXoNhrNA,22899
|
75
|
+
lionagi/utils/io_util.py,sha256=Yzzo-GACV3QXdEFpdfVrpjxc8YJEHJof7hACiTQ59gU,3515
|
76
|
+
lionagi/utils/load_utils.py,sha256=nSDWSXRFHO-tJXGMW-7icOqkeL1baQENf9BDyJZU88o,6667
|
77
|
+
lionagi/utils/sys_util.py,sha256=nxv5mFagY0zBdMLhf9LD_SsK5Cd2rz9zMslr9xIwpfM,6305
|
78
|
+
lionagi/utils/tool_util.py,sha256=RgTML1ZZbmVGdBV2pMpbDCDaNawf_p4d8bqliaOS898,2914
|
79
|
+
lionagi/utils/type_util.py,sha256=eUfsvIr_VRykruE4TEUHssBME8Iug_4U4myY8xFV2bA,3135
|
80
|
+
lionagi-0.0.113.dist-info/LICENSE,sha256=TBnSyG8fs_tMRtK805GzA1cIyExleKyzoN_kuVxT9IY,11358
|
81
|
+
lionagi-0.0.113.dist-info/METADATA,sha256=-zs_6eEFEiCbvqycmpy9YxKVOOIUdzAlB6IrnrHJadk,18027
|
82
|
+
lionagi-0.0.113.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
|
83
|
+
lionagi-0.0.113.dist-info/top_level.txt,sha256=szvch_d2jE1Lu9ZIKsl26Ll6BGfYfbOgt5lm-UpFSo4,8
|
84
|
+
lionagi-0.0.113.dist-info/RECORD,,
|
@@ -1,20 +0,0 @@
|
|
1
|
-
from .base_endpoint import BaseEndpoint
|
2
|
-
|
3
|
-
|
4
|
-
class ChatCompletion(BaseEndpoint):
|
5
|
-
endpoint: str = "chat/completion"
|
6
|
-
|
7
|
-
@classmethod
|
8
|
-
def create_payload(scls, messages, llmconfig, schema, **kwargs):
|
9
|
-
config = {**llmconfig, **kwargs}
|
10
|
-
payload = {"messages": messages}
|
11
|
-
for key in schema['required']:
|
12
|
-
payload.update({key: config[key]})
|
13
|
-
|
14
|
-
for key in schema['optional']:
|
15
|
-
if bool(config[key]) is True and str(config[key]).lower() != "none":
|
16
|
-
payload.update({key: config[key]})
|
17
|
-
return payload
|
18
|
-
|
19
|
-
def process_response(self, session, payload, completion):
|
20
|
-
...
|
File without changes
|
lionagi/llm_configs.py
DELETED
@@ -1,21 +0,0 @@
|
|
1
|
-
oai_schema = {
|
2
|
-
"required" : ["model", "frequency_penalty", "n", "presence_penalty", "response_format", "temperature", "top_p"],
|
3
|
-
"optional": ["seed", "stop", "stream", "tools", "tool_choice", "user", "max_tokens"]
|
4
|
-
}
|
5
|
-
|
6
|
-
oai_llmconfig = {
|
7
|
-
"model": "gpt-4-1106-preview",
|
8
|
-
"frequency_penalty": 0,
|
9
|
-
"max_tokens": None,
|
10
|
-
"n": 1,
|
11
|
-
"presence_penalty": 0,
|
12
|
-
"response_format": {"type": "text"},
|
13
|
-
"seed": None,
|
14
|
-
"stop": None,
|
15
|
-
"stream": False,
|
16
|
-
"temperature": 0.7,
|
17
|
-
"top_p": 1,
|
18
|
-
"tools": None,
|
19
|
-
"tool_choice": "none",
|
20
|
-
"user": None
|
21
|
-
}
|
lionagi/loader/load_utils.py
DELETED
@@ -1,161 +0,0 @@
|
|
1
|
-
import math
|
2
|
-
from pathlib import Path
|
3
|
-
from typing import Any, Dict, List, Union, Callable, Optional
|
4
|
-
|
5
|
-
from ..utils.sys_utils import to_list, l_call
|
6
|
-
from ..schema import DataLogger
|
7
|
-
|
8
|
-
def _dir_to_path(dir, ext, recursive):
|
9
|
-
tem = '**/*' if recursive else '*'
|
10
|
-
return list(Path(dir).glob(tem + ext))
|
11
|
-
|
12
|
-
def _split_path(path: Path) -> tuple:
|
13
|
-
folder_name = path.parent.name
|
14
|
-
file_name = path.name
|
15
|
-
return (folder_name, file_name)
|
16
|
-
|
17
|
-
def dir_to_path(dir: str, ext, recursive: bool = False, flat: bool = True):
|
18
|
-
try:
|
19
|
-
return to_list(l_call(ext, _dir_to_path, flat=True,
|
20
|
-
recursive=recursive, dir=dir, ext=ext),
|
21
|
-
flat=flat)
|
22
|
-
except:
|
23
|
-
raise ValueError("Invalid directory or extension, please check the path")
|
24
|
-
|
25
|
-
def read_text(filepath: str, clean: bool = True) -> str:
|
26
|
-
with open(filepath, 'r') as f:
|
27
|
-
content = f.read()
|
28
|
-
if clean:
|
29
|
-
# Define characters to replace and their replacements
|
30
|
-
replacements = {'\\': ' ', '\n': ' ', '\t': ' ', ' ': ' ', '\'': ' '}
|
31
|
-
for old, new in replacements.items():
|
32
|
-
content = content.replace(old, new)
|
33
|
-
return content
|
34
|
-
|
35
|
-
def dir_to_files(dir: str,
|
36
|
-
ext: str,
|
37
|
-
recursive: bool = False,
|
38
|
-
reader: Callable = read_text,
|
39
|
-
clean: bool = True,
|
40
|
-
to_csv: bool = False,
|
41
|
-
project: str = 'project',
|
42
|
-
output_dir: str = 'data/logs/sources/',
|
43
|
-
filename: Optional[str] = None,
|
44
|
-
verbose: bool = True,
|
45
|
-
timestamp: bool = True,
|
46
|
-
logger: Optional[DataLogger] = None):
|
47
|
-
|
48
|
-
sources = dir_to_path(dir, ext, recursive)
|
49
|
-
|
50
|
-
def _to_dict(path_: Path) -> Dict[str, Union[str, Path]]:
|
51
|
-
folder, file = _split_path(path_)
|
52
|
-
content = reader(str(path_), clean=clean)
|
53
|
-
return {
|
54
|
-
'project': project,
|
55
|
-
'folder': folder,
|
56
|
-
'file': file,
|
57
|
-
"file_size": len(str(content)),
|
58
|
-
'content': content
|
59
|
-
} if content else None
|
60
|
-
|
61
|
-
logs = to_list(l_call(sources, _to_dict, flat=True), dropna=True)
|
62
|
-
|
63
|
-
if to_csv:
|
64
|
-
filename = filename or f"{project}_sources.csv"
|
65
|
-
logger = DataLogger(dir=output_dir, log=logs) if not logger else logger
|
66
|
-
logger.to_csv(dir=output_dir, filename=filename, verbose=verbose, timestamp=timestamp)
|
67
|
-
|
68
|
-
return logs
|
69
|
-
|
70
|
-
def _chunk_n1(input):
|
71
|
-
return [input]
|
72
|
-
|
73
|
-
def _chunk_n2(input, chunk_size, overlap_size, threshold):
|
74
|
-
chunks = []
|
75
|
-
chunks.append(input[:chunk_size + overlap_size])
|
76
|
-
if len(input) - chunk_size > threshold:
|
77
|
-
chunks.append(input[chunk_size - overlap_size:])
|
78
|
-
else:
|
79
|
-
return [input]
|
80
|
-
return chunks
|
81
|
-
|
82
|
-
def _chunk_n3(input, chunk_size, overlap_size, threshold, n_chunks):
|
83
|
-
chunks = []
|
84
|
-
chunks.append(input[:chunk_size + overlap_size])
|
85
|
-
for i in range(1, n_chunks - 1):
|
86
|
-
start_idx = chunk_size * i - overlap_size
|
87
|
-
end_idx = chunk_size * (i + 1) + overlap_size
|
88
|
-
chunks.append(input[start_idx:end_idx])
|
89
|
-
|
90
|
-
if len(input) - chunk_size * (n_chunks - 1) > threshold:
|
91
|
-
chunks.append(input[chunk_size * (n_chunks - 1) - overlap_size:])
|
92
|
-
else:
|
93
|
-
chunks[-1] += input[chunk_size * (n_chunks - 1) + overlap_size:]
|
94
|
-
|
95
|
-
return chunks
|
96
|
-
|
97
|
-
def chunk_text(input: str,
|
98
|
-
chunk_size: int,
|
99
|
-
overlap: float,
|
100
|
-
threshold: int) -> List[Union[str, None]]:
|
101
|
-
try:
|
102
|
-
if not isinstance(input, str): input = str(input)
|
103
|
-
|
104
|
-
n_chunks = math.ceil(len(input) / chunk_size)
|
105
|
-
overlap_size = int(chunk_size * overlap / 2)
|
106
|
-
|
107
|
-
if n_chunks == 1: return _chunk_n1(input)
|
108
|
-
|
109
|
-
elif n_chunks == 2:
|
110
|
-
return _chunk_n2(input=input, chunk_size=chunk_size, overlap_size=overlap_size,
|
111
|
-
threshold=threshold)
|
112
|
-
elif n_chunks > 2:
|
113
|
-
return _chunk_n3(input=input, chunk_size=chunk_size, overlap_size=overlap_size,
|
114
|
-
threshold=threshold, n_chunks=n_chunks)
|
115
|
-
|
116
|
-
except Exception as e:
|
117
|
-
raise ValueError(f"An error occurred while chunking the text. {e}")
|
118
|
-
|
119
|
-
def _file_to_chunks(input: Dict[str, Any],
|
120
|
-
field: str = 'content',
|
121
|
-
chunk_size: int = 1500,
|
122
|
-
overlap: float = 0.1,
|
123
|
-
threshold: int = 200) -> List[Dict[str, Any]]:
|
124
|
-
try:
|
125
|
-
out = {key: value for key, value in input.items() if key != field}
|
126
|
-
out.update({"chunk_overlap": overlap, "chunk_threshold": threshold})
|
127
|
-
|
128
|
-
chunks = chunk_text(input[field], chunk_size=chunk_size, overlap=overlap, threshold=threshold)
|
129
|
-
logs = []
|
130
|
-
for i, chunk in enumerate(chunks):
|
131
|
-
chunk_dict = out.copy()
|
132
|
-
chunk_dict.update({
|
133
|
-
'file_chunks': len(chunks),
|
134
|
-
'chunk_id': i + 1,
|
135
|
-
'chunk_size': len(chunk),
|
136
|
-
f'chunk_{field}': chunk
|
137
|
-
})
|
138
|
-
logs.append(chunk_dict)
|
139
|
-
|
140
|
-
return logs
|
141
|
-
|
142
|
-
except Exception as e:
|
143
|
-
raise ValueError(f"An error occurred while chunking the file. {e}")
|
144
|
-
|
145
|
-
def file_to_chunks(input,
|
146
|
-
project='project',
|
147
|
-
output_dir='data/logs/sources/',
|
148
|
-
chunk_func = _file_to_chunks,
|
149
|
-
to_csv=False,
|
150
|
-
filename=None,
|
151
|
-
verbose=True,
|
152
|
-
timestamp=True,
|
153
|
-
logger=None, **kwargs):
|
154
|
-
logs = to_list(l_call(input, chunk_func, **kwargs), flat=True)
|
155
|
-
|
156
|
-
if to_csv:
|
157
|
-
filename = filename if filename else f"{project}_sources.csv"
|
158
|
-
logger = logger or DataLogger(log=logs)
|
159
|
-
logger.to_csv(dir=output_dir, filename=filename, verbose=verbose, timestamp=timestamp)
|
160
|
-
|
161
|
-
return logs
|
lionagi/schema.py
DELETED
@@ -1,275 +0,0 @@
|
|
1
|
-
import json
|
2
|
-
import networkx as nx
|
3
|
-
from collections import deque
|
4
|
-
from typing import Any, Dict, Optional, Union, Callable
|
5
|
-
|
6
|
-
from pydantic import BaseModel, Field
|
7
|
-
from .utils.sys_utils import create_id, create_path, to_csv
|
8
|
-
|
9
|
-
|
10
|
-
# checked --------------------------------------------------------
|
11
|
-
class BaseNode(BaseModel):
|
12
|
-
id_: str = Field(default_factory=lambda: str(create_id()), alias="node_id")
|
13
|
-
content: Union[str, Dict[str, Any], None, Any] = None
|
14
|
-
metadata: Union[Dict[str, Any], None] = Field(default_factory=dict)
|
15
|
-
label: Optional[str] = None
|
16
|
-
|
17
|
-
@classmethod
|
18
|
-
def class_name(cls) -> str:
|
19
|
-
return cls.__name__
|
20
|
-
|
21
|
-
# to some structure
|
22
|
-
def to_json(self) -> str:
|
23
|
-
return json.dumps(self.model_dump(by_alias=True))
|
24
|
-
|
25
|
-
def to_dict(self) -> Dict[str, Any]:
|
26
|
-
return self.model_dump(by_alias=True)
|
27
|
-
|
28
|
-
# from some structure
|
29
|
-
@classmethod
|
30
|
-
def from_json(cls, json_str: str) -> "BaseNode":
|
31
|
-
data = json.loads(json_str)
|
32
|
-
return cls(**data)
|
33
|
-
|
34
|
-
@classmethod
|
35
|
-
def from_dict(cls, data: Dict[str, Any]) -> "BaseNode":
|
36
|
-
return cls(**data)
|
37
|
-
|
38
|
-
def add_meta(self, replace=True, **kwags) -> None:
|
39
|
-
if replace:
|
40
|
-
self.metadata.update(**kwags)
|
41
|
-
else:
|
42
|
-
for k, v in kwags.items():
|
43
|
-
if k in self.metadata.keys():
|
44
|
-
raise ValueError(f"Key already existed")
|
45
|
-
if k not in self.metadata.keys():
|
46
|
-
self.metadata.update({k:v})
|
47
|
-
|
48
|
-
def set_meta(self, metadata_: dict) -> None:
|
49
|
-
self.metadata = metadata_
|
50
|
-
|
51
|
-
def set_content(self, content: Union[str, Dict[str, Any], None, Any]):
|
52
|
-
self.content = content
|
53
|
-
|
54
|
-
def set_id(self, id: str):
|
55
|
-
self.id_ = id
|
56
|
-
|
57
|
-
def get_meta(self):
|
58
|
-
return self.metadata
|
59
|
-
|
60
|
-
def get_content(self):
|
61
|
-
return self.content
|
62
|
-
|
63
|
-
def get_id(self):
|
64
|
-
return self.id_
|
65
|
-
|
66
|
-
class ConditionalRelationship(BaseNode):
|
67
|
-
target_node_id: str
|
68
|
-
condition: Optional[Dict] = None
|
69
|
-
|
70
|
-
def condition_existed(self, condition_key):
|
71
|
-
if self.condition:
|
72
|
-
try:
|
73
|
-
self.condition.get(condition_key)
|
74
|
-
return True
|
75
|
-
except:
|
76
|
-
return False
|
77
|
-
return False
|
78
|
-
|
79
|
-
def get_condition(self, condition_key) -> bool:
|
80
|
-
if self.condition_existed(condition_key=condition_key):
|
81
|
-
a = self.condition.get(condition_key)
|
82
|
-
if a is not None and str(a).strip().lower() != 'none':
|
83
|
-
return a
|
84
|
-
else:
|
85
|
-
raise ValueError(f"Condition {condition_key} has no value")
|
86
|
-
|
87
|
-
|
88
|
-
class DataNode(BaseNode):
|
89
|
-
|
90
|
-
...
|
91
|
-
# def from_llama(self, data_:, **kwargs):
|
92
|
-
# ...
|
93
|
-
|
94
|
-
# def to_llama(self, **kwargs):
|
95
|
-
# # to llama_index textnode
|
96
|
-
# ...
|
97
|
-
|
98
|
-
# def from_langchain(self, data_, **kwargs):
|
99
|
-
# ...
|
100
|
-
|
101
|
-
# def to_langchain(self, **kwargs):
|
102
|
-
# ...
|
103
|
-
|
104
|
-
# def to_csv(self, **kwargs):
|
105
|
-
# ...
|
106
|
-
|
107
|
-
# def __call__(self, file_=None):
|
108
|
-
# ...
|
109
|
-
|
110
|
-
|
111
|
-
class MessageNode(BaseNode):
|
112
|
-
role: str
|
113
|
-
name: str
|
114
|
-
|
115
|
-
# def from_oai(self):
|
116
|
-
# ...
|
117
|
-
|
118
|
-
|
119
|
-
class File(DataNode):
|
120
|
-
...
|
121
|
-
|
122
|
-
# def from_path(self, path_, reader, clean=True, **kwargs):
|
123
|
-
# self.content = reader(path_=path_, clean=clean, **kwargs)
|
124
|
-
|
125
|
-
# def to_chunks(self, chunker, **kwargs):
|
126
|
-
# ...
|
127
|
-
|
128
|
-
|
129
|
-
class Chunk(DataNode):
|
130
|
-
...
|
131
|
-
# @classmethod
|
132
|
-
# def from_files(cls, files):
|
133
|
-
# ...
|
134
|
-
|
135
|
-
# @classmethod
|
136
|
-
# def to_files(cls):
|
137
|
-
# ...
|
138
|
-
|
139
|
-
|
140
|
-
class DataLogger:
|
141
|
-
"""
|
142
|
-
Logs data entries and outputs them to a CSV file.
|
143
|
-
|
144
|
-
This class provides a logging mechanism for data entries that can be saved to a
|
145
|
-
CSV file. It offers methods for appending new log entries, saving the log to a CSV file,
|
146
|
-
and setting the directory where the logs should be saved.
|
147
|
-
|
148
|
-
Attributes:
|
149
|
-
dir (str):
|
150
|
-
The directory where the log files are to be saved.
|
151
|
-
log (deque):
|
152
|
-
A deque that stores log entries.
|
153
|
-
|
154
|
-
Methods:
|
155
|
-
__call__(entry):
|
156
|
-
Appends a new entry to the log.
|
157
|
-
to_csv(dir: str, filename: str, verbose: bool, timestamp: bool, dir_exist_ok: bool, file_exist_ok: bool):
|
158
|
-
Converts the log to a CSV format and saves it to a file.
|
159
|
-
set_dir(dir: str):
|
160
|
-
Sets the directory for saving log files.
|
161
|
-
"""
|
162
|
-
|
163
|
-
def __init__(self, dir= None, log: list = None) -> None:
|
164
|
-
"""
|
165
|
-
Initializes a new instance of the DataLogger class.
|
166
|
-
|
167
|
-
Parameters:
|
168
|
-
dir (str, optional): The directory where the log files will be saved. Defaults to None.
|
169
|
-
|
170
|
-
log (list, optional): An initial list of log entries. Defaults to an empty deque.
|
171
|
-
"""
|
172
|
-
self.dir = dir
|
173
|
-
self.log = deque(log) if log else deque()
|
174
|
-
|
175
|
-
def __call__(self, entry):
|
176
|
-
"""
|
177
|
-
Appends a new entry to the log.
|
178
|
-
|
179
|
-
Parameters:
|
180
|
-
entry: The entry to append to the log. The entry can be of any datatype.
|
181
|
-
"""
|
182
|
-
self.log.append(entry)
|
183
|
-
|
184
|
-
def to_csv(self, filename: str, dir=None, verbose: bool = True, timestamp: bool = True, dir_exist_ok=True, file_exist_ok=False):
|
185
|
-
"""
|
186
|
-
Converts the log to a CSV format and saves it to a file.
|
187
|
-
|
188
|
-
Parameters:
|
189
|
-
dir (str): The directory where the CSV file will be saved.
|
190
|
-
|
191
|
-
filename (str): The name of the CSV file.
|
192
|
-
|
193
|
-
verbose (bool, optional): If True, prints a message after saving the log. Defaults to True.
|
194
|
-
|
195
|
-
timestamp (bool, optional): If True, appends a timestamp to the filename. Defaults to True.
|
196
|
-
|
197
|
-
dir_exist_ok (bool, optional): If True, overrides the existing directory if needed. Defaults to True.
|
198
|
-
|
199
|
-
file_exist_ok (bool, optional): If True, overrides the existing file if needed. Defaults to False.
|
200
|
-
|
201
|
-
Postconditions:
|
202
|
-
Saves the log entries to a CSV file and clears the `log` attribute.
|
203
|
-
|
204
|
-
Optionally prints a message with the number of log entries saved and the file path.
|
205
|
-
"""
|
206
|
-
dir = dir or self.dir
|
207
|
-
filepath = create_path(dir=dir, filename=filename, timestamp=timestamp, dir_exist_ok=dir_exist_ok)
|
208
|
-
to_csv(list(self.log), filepath, file_exist_ok=file_exist_ok)
|
209
|
-
n_logs = len(list(self.log))
|
210
|
-
self.log = deque()
|
211
|
-
if verbose:
|
212
|
-
print(f"{n_logs} logs saved to {filepath}")
|
213
|
-
|
214
|
-
def set_dir(self, dir: str):
|
215
|
-
"""
|
216
|
-
Sets the directory where log files will be saved.
|
217
|
-
|
218
|
-
Parameters:
|
219
|
-
dir (str): The directory to set for saving log files.
|
220
|
-
"""
|
221
|
-
self.dir = dir
|
222
|
-
|
223
|
-
|
224
|
-
class BaseTool(BaseNode):
|
225
|
-
name: str = None
|
226
|
-
func: Callable = None
|
227
|
-
content: Any = None
|
228
|
-
parser: Callable = None
|
229
|
-
|
230
|
-
def initialize(self):
|
231
|
-
...
|
232
|
-
|
233
|
-
def execute(self):
|
234
|
-
...
|
235
|
-
|
236
|
-
def shutdown(self):
|
237
|
-
...
|
238
|
-
|
239
|
-
def __enter__(self):
|
240
|
-
self.initialize()
|
241
|
-
return self
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
# checked ------------------------------------------------------
|
246
|
-
class Structure:
|
247
|
-
def __init__(self):
|
248
|
-
self.graph = nx.DiGraph()
|
249
|
-
|
250
|
-
def add_node(self, node: BaseNode) -> None:
|
251
|
-
self.graph.add_node(node.id_, content=node.content, metadata=node.metadata)
|
252
|
-
|
253
|
-
def add_relationship(self, source_node_id: str, target_node_id: str, relationship: ConditionalRelationship) -> None:
|
254
|
-
self.graph.add_edge(source_node_id, target_node_id,
|
255
|
-
label=relationship.label,
|
256
|
-
properties=relationship.properties,
|
257
|
-
condition=relationship.condition)
|
258
|
-
|
259
|
-
def remove_relationship(self, source_node_id: str, target_node_id: str) -> None:
|
260
|
-
if self.graph.has_edge(source_node_id, target_node_id):
|
261
|
-
self.graph.remove_edge(source_node_id, target_node_id)
|
262
|
-
|
263
|
-
def get_relationships(self, node_id: str) -> list:
|
264
|
-
return list(self.graph.edges(node_id, data=True))
|
265
|
-
|
266
|
-
def get_conditional_relationships(self, node_id: str) -> list:
|
267
|
-
return [(target_id, data) for _, target_id, data in self.graph.edges(node_id, data=True) if "condition" in data]
|
268
|
-
|
269
|
-
def to_json(self) -> str:
|
270
|
-
return json.dumps(nx.node_link_data(self.graph))
|
271
|
-
|
272
|
-
def from_json(self, data: str) -> None:
|
273
|
-
graph_data = json.loads(data)
|
274
|
-
self.graph = nx.node_link_graph(graph_data)
|
275
|
-
|
lionagi/service_/__init__.py
DELETED
lionagi/service_/base_service.py
DELETED
@@ -1,48 +0,0 @@
|
|
1
|
-
import re
|
2
|
-
from typing import Generator
|
3
|
-
from .service_utils import BaseService
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
class BaseAPIService(BaseService):
|
8
|
-
|
9
|
-
def __init__(self, api_key: str = None,
|
10
|
-
status_tracker = None,
|
11
|
-
queue = None, endpoint=None, schema=None,
|
12
|
-
ratelimiter=None, max_requests_per_minute=None, max_tokens_per_minute=None) -> None:
|
13
|
-
self.api_key = api_key
|
14
|
-
self.status_tracker = status_tracker
|
15
|
-
self.queue = queue
|
16
|
-
self.endpoint=endpoint
|
17
|
-
self.schema = schema
|
18
|
-
self.rate_limiter = ratelimiter(max_requests_per_minute, max_tokens_per_minute)
|
19
|
-
|
20
|
-
@staticmethod
|
21
|
-
def api_methods(http_session, method="post"):
|
22
|
-
if method not in ["post", "delete", "head", "options", "patch"]:
|
23
|
-
raise ValueError("Invalid request, method must be in ['post', 'delete', 'head', 'options', 'patch']")
|
24
|
-
elif method == "post":
|
25
|
-
return http_session.post
|
26
|
-
elif method == "delete":
|
27
|
-
return http_session.delete
|
28
|
-
elif method == "head":
|
29
|
-
return http_session.head
|
30
|
-
elif method == "options":
|
31
|
-
return http_session.options
|
32
|
-
elif method == "patch":
|
33
|
-
return http_session.patch
|
34
|
-
|
35
|
-
@staticmethod
|
36
|
-
def api_endpoint_from_url(request_url: str) -> str:
|
37
|
-
match = re.search(r"^https://[^/]+/v\d+/(.+)$", request_url)
|
38
|
-
if match:
|
39
|
-
return match.group(1)
|
40
|
-
else:
|
41
|
-
return ""
|
42
|
-
|
43
|
-
@staticmethod
|
44
|
-
def task_id_generator_function() -> Generator[int, None, None]:
|
45
|
-
task_id = 0
|
46
|
-
while True:
|
47
|
-
yield task_id
|
48
|
-
task_id += 1
|
lionagi/service_/openrouter.py
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
# TODO
|
lionagi/services.py
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
from .service_.oai import OpenAIService
|