lionagi 0.0.112__py3-none-any.whl → 0.0.113__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lionagi/__init__.py +3 -3
- lionagi/bridge/__init__.py +7 -0
- lionagi/bridge/langchain.py +131 -0
- lionagi/bridge/llama_index.py +157 -0
- lionagi/configs/__init__.py +7 -0
- lionagi/configs/oai_configs.py +49 -0
- lionagi/configs/openrouter_config.py +49 -0
- lionagi/core/__init__.py +8 -2
- lionagi/core/instruction_sets.py +1 -3
- lionagi/core/messages.py +2 -2
- lionagi/core/sessions.py +174 -27
- lionagi/datastore/__init__.py +1 -0
- lionagi/loader/__init__.py +9 -4
- lionagi/loader/chunker.py +157 -0
- lionagi/loader/reader.py +124 -0
- lionagi/objs/__init__.py +7 -0
- lionagi/objs/messenger.py +163 -0
- lionagi/objs/tool_registry.py +247 -0
- lionagi/schema/__init__.py +11 -0
- lionagi/schema/base_schema.py +239 -0
- lionagi/schema/base_tool.py +9 -0
- lionagi/schema/data_logger.py +94 -0
- lionagi/services/__init__.py +14 -0
- lionagi/{service_/oai.py → services/base_api_service.py} +49 -82
- lionagi/{endpoint/base_endpoint.py → services/chatcompletion.py} +19 -22
- lionagi/services/oai.py +34 -0
- lionagi/services/openrouter.py +32 -0
- lionagi/{service_/service_utils.py → services/service_objs.py} +0 -1
- lionagi/structure/__init__.py +7 -0
- lionagi/structure/relationship.py +128 -0
- lionagi/structure/structure.py +160 -0
- lionagi/tests/test_flatten_util.py +426 -0
- lionagi/tools/__init__.py +0 -5
- lionagi/tools/coder.py +1 -0
- lionagi/tools/scorer.py +1 -0
- lionagi/tools/validator.py +1 -0
- lionagi/utils/__init__.py +46 -20
- lionagi/utils/api_util.py +86 -0
- lionagi/utils/call_util.py +347 -0
- lionagi/utils/flat_util.py +540 -0
- lionagi/utils/io_util.py +102 -0
- lionagi/utils/load_utils.py +190 -0
- lionagi/utils/sys_util.py +191 -0
- lionagi/utils/tool_util.py +92 -0
- lionagi/utils/type_util.py +81 -0
- lionagi/version.py +1 -1
- {lionagi-0.0.112.dist-info → lionagi-0.0.113.dist-info}/METADATA +37 -13
- lionagi-0.0.113.dist-info/RECORD +84 -0
- lionagi/endpoint/chat_completion.py +0 -20
- lionagi/endpoint/endpoint_utils.py +0 -0
- lionagi/llm_configs.py +0 -21
- lionagi/loader/load_utils.py +0 -161
- lionagi/schema.py +0 -275
- lionagi/service_/__init__.py +0 -6
- lionagi/service_/base_service.py +0 -48
- lionagi/service_/openrouter.py +0 -1
- lionagi/services.py +0 -1
- lionagi/tools/tool_utils.py +0 -75
- lionagi/utils/sys_utils.py +0 -799
- lionagi-0.0.112.dist-info/RECORD +0 -67
- /lionagi/{core/responses.py → datastore/chroma.py} +0 -0
- /lionagi/{endpoint/assistants.py → datastore/deeplake.py} +0 -0
- /lionagi/{endpoint/audio.py → datastore/elasticsearch.py} +0 -0
- /lionagi/{endpoint/embeddings.py → datastore/lantern.py} +0 -0
- /lionagi/{endpoint/files.py → datastore/pinecone.py} +0 -0
- /lionagi/{endpoint/fine_tuning.py → datastore/postgres.py} +0 -0
- /lionagi/{endpoint/images.py → datastore/qdrant.py} +0 -0
- /lionagi/{endpoint/messages.py → schema/base_condition.py} +0 -0
- /lionagi/{service_ → services}/anthropic.py +0 -0
- /lionagi/{service_ → services}/anyscale.py +0 -0
- /lionagi/{service_ → services}/azure.py +0 -0
- /lionagi/{service_ → services}/bedrock.py +0 -0
- /lionagi/{service_ → services}/everlyai.py +0 -0
- /lionagi/{service_ → services}/gemini.py +0 -0
- /lionagi/{service_ → services}/gpt4all.py +0 -0
- /lionagi/{service_ → services}/huggingface.py +0 -0
- /lionagi/{service_ → services}/litellm.py +0 -0
- /lionagi/{service_ → services}/localai.py +0 -0
- /lionagi/{service_ → services}/mistralai.py +0 -0
- /lionagi/{service_ → services}/ollama.py +0 -0
- /lionagi/{service_ → services}/openllm.py +0 -0
- /lionagi/{service_ → services}/perplexity.py +0 -0
- /lionagi/{service_ → services}/predibase.py +0 -0
- /lionagi/{service_ → services}/rungpt.py +0 -0
- /lionagi/{service_ → services}/vllm.py +0 -0
- /lionagi/{service_ → services}/xinference.py +0 -0
- /lionagi/{endpoint → tests}/__init__.py +0 -0
- /lionagi/{endpoint/models.py → tools/planner.py} +0 -0
- /lionagi/{endpoint/moderations.py → tools/prompter.py} +0 -0
- /lionagi/{endpoint/runs.py → tools/sandbox.py} +0 -0
- /lionagi/{endpoint/threads.py → tools/summarizer.py} +0 -0
- {lionagi-0.0.112.dist-info → lionagi-0.0.113.dist-info}/LICENSE +0 -0
- {lionagi-0.0.112.dist-info → lionagi-0.0.113.dist-info}/WHEEL +0 -0
- {lionagi-0.0.112.dist-info → lionagi-0.0.113.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,84 @@
|
|
1
|
+
lionagi/__init__.py,sha256=gAOIyJ-kItUn2ya8G3zW6pvuzmk60xk7zO_GN66UJAM,888
|
2
|
+
lionagi/version.py,sha256=WJPRYfehqmqr60a1h9m6a2fbtJa0dT03ixD6hahuBgE,24
|
3
|
+
lionagi/bridge/__init__.py,sha256=I19ztSDDwV1-e-GSRs45bJgjbg454e0vcFv2jwgwQY0,139
|
4
|
+
lionagi/bridge/langchain.py,sha256=nJdnv_zHpchQXUlh5ml6OZKGO9Rbaa7B-XbeKB3Ej7Y,4579
|
5
|
+
lionagi/bridge/llama_index.py,sha256=sapGIgS3RkDUyrPCdjZizoRdAmktZLHzUbYQBw6nnZw,5510
|
6
|
+
lionagi/configs/__init__.py,sha256=88KoeoB8dHhgZdQ8I4rbb7e8-FUd-_gi1dXaA95vJ3Y,141
|
7
|
+
lionagi/configs/oai_configs.py,sha256=C89jXb2Xb7qoOXIhtrMlIRVYJqjh7knpu6CAOloj9eo,1199
|
8
|
+
lionagi/configs/openrouter_config.py,sha256=qbEQQqQPpzMp3eP5U8kuvGTUYdMgemuuq-BdgOYgYI8,1256
|
9
|
+
lionagi/core/__init__.py,sha256=XAw2vF1wuQYKSAxHcBanfYN0k-5qLSggkl3yZgwjSRw,347
|
10
|
+
lionagi/core/conversations.py,sha256=Jh4Bvv0SBaW5ZG_D9hyWveQ-PXPDGfvp_V7UxbPsFoA,4084
|
11
|
+
lionagi/core/flows.py,sha256=rhPBmdN-JKo7Zcv9hiy6k0b-CjKBHMemkYITM7a9fNo,32
|
12
|
+
lionagi/core/instruction_sets.py,sha256=y-fDltfDtji4fCtHn8fJ_M9OtYCfjqeTds7bpzknvhU,51
|
13
|
+
lionagi/core/messages.py,sha256=XisuXcJBUoyYcd-6DLcV_NZnsKXzZcFf4Dk_fxTlXc8,6533
|
14
|
+
lionagi/core/sessions.py,sha256=sjg7bCORFhpaEgDBTExyhfgGxRAUwkOCft0MmP-BGoU,11161
|
15
|
+
lionagi/datastore/__init__.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
16
|
+
lionagi/datastore/chroma.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
17
|
+
lionagi/datastore/deeplake.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
18
|
+
lionagi/datastore/elasticsearch.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
19
|
+
lionagi/datastore/lantern.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
20
|
+
lionagi/datastore/pinecone.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
21
|
+
lionagi/datastore/postgres.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
22
|
+
lionagi/datastore/qdrant.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
23
|
+
lionagi/loader/__init__.py,sha256=0FNrggZnSq1ONLZY2f325SF19UfYHaqJeJXLvpu8KWQ,262
|
24
|
+
lionagi/loader/chunker.py,sha256=zGdfS-Gybxc6lm_vMIdt-f25kY2CwDFTLvY-arKxQ8c,6476
|
25
|
+
lionagi/loader/reader.py,sha256=8kxYlqhwmMpuNTWdYtsNc9g-1Mf264Y8XRu1GkevhTg,4527
|
26
|
+
lionagi/objs/__init__.py,sha256=4qxo9ZGKe37fjj_Gt2hgi4QTTIPmDf7mkyUK7euVmsU,137
|
27
|
+
lionagi/objs/messenger.py,sha256=k-ueq2LBLTTDgk8u-Vyja8WBFM7dWFInWO97HNyOXaI,6137
|
28
|
+
lionagi/objs/tool_registry.py,sha256=cnoaW1TvNhKKerQ4tTUOSHEADmsTdoUW2lT24LcHIx0,9357
|
29
|
+
lionagi/schema/__init__.py,sha256=cB7rSYeRXujDwp2f5IMK83-NzdqPfbMhGijPCPXgT_c,210
|
30
|
+
lionagi/schema/base_condition.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
31
|
+
lionagi/schema/base_schema.py,sha256=gx4LDxYnVO9vqp0JdevUSasqrat7Lu3rRURTiv7gGj0,8348
|
32
|
+
lionagi/schema/base_tool.py,sha256=sKbCrS0ZZINkr03Fj0lk0Kt0XPiBz9IXwA9A1aDgN88,192
|
33
|
+
lionagi/schema/data_logger.py,sha256=nmv0jJDkE-vzz_KyBl1_bHB8KVvhZ6sItOBQcvBkAvE,3417
|
34
|
+
lionagi/services/__init__.py,sha256=js4LnJEJv4kkxPPamkOgFkOsrc1yr2dK-z9CLTTsX04,313
|
35
|
+
lionagi/services/anthropic.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
36
|
+
lionagi/services/anyscale.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
37
|
+
lionagi/services/azure.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
38
|
+
lionagi/services/base_api_service.py,sha256=vGCnC1GnW8czpseOx8mcDeyiiAKc_dabUInxjytE-A0,9756
|
39
|
+
lionagi/services/bedrock.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
40
|
+
lionagi/services/chatcompletion.py,sha256=9jVkB4JUX-kzvAHsohA1ZO6ZeXdlN9OAe-cVvLCa9-0,1257
|
41
|
+
lionagi/services/everlyai.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
42
|
+
lionagi/services/gemini.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
43
|
+
lionagi/services/gpt4all.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
44
|
+
lionagi/services/huggingface.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
45
|
+
lionagi/services/litellm.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
46
|
+
lionagi/services/localai.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
47
|
+
lionagi/services/mistralai.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
48
|
+
lionagi/services/oai.py,sha256=NcKiqmQ8YpKEkoVv-6a-mvIZVBumWOVpstpVD0oTiUs,1122
|
49
|
+
lionagi/services/ollama.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
50
|
+
lionagi/services/openllm.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
51
|
+
lionagi/services/openrouter.py,sha256=vjY0ZydLY5XK4_T2cRiAl8_QNysWy26wcfyIP-1hlSw,1110
|
52
|
+
lionagi/services/perplexity.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
53
|
+
lionagi/services/predibase.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
54
|
+
lionagi/services/rungpt.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
55
|
+
lionagi/services/service_objs.py,sha256=w1Rs69eAGyEukTMMvwQHDmFkwTMf2pB9DdGQ-xCjTUw,9727
|
56
|
+
lionagi/services/vllm.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
57
|
+
lionagi/services/xinference.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
58
|
+
lionagi/structure/__init__.py,sha256=wMPekT2vbWwUkJ5aW5o-lzJC9Fzhta6RHDiFPTNUm_0,120
|
59
|
+
lionagi/structure/relationship.py,sha256=5urGnomfhBJUwPLpYpk5YCx-zO2yprJDaPQqMsgQM0A,4124
|
60
|
+
lionagi/structure/structure.py,sha256=KWVOTFWk_IBMtY57gut_GFUuO7dKWlIrMH7xZ_ys2c4,5952
|
61
|
+
lionagi/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
62
|
+
lionagi/tests/test_flatten_util.py,sha256=-sOAQriGtHK1U0FTd31IpdKJpgMQREpaXp-hf6FAojQ,17362
|
63
|
+
lionagi/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
64
|
+
lionagi/tools/coder.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
65
|
+
lionagi/tools/planner.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
66
|
+
lionagi/tools/prompter.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
67
|
+
lionagi/tools/sandbox.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
68
|
+
lionagi/tools/scorer.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
69
|
+
lionagi/tools/summarizer.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
70
|
+
lionagi/tools/validator.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
|
71
|
+
lionagi/utils/__init__.py,sha256=91eJpmYZLXi_xsFV7yb_uvlRmdhBQWBqa9Bmy18nJYY,1419
|
72
|
+
lionagi/utils/api_util.py,sha256=wNYillgmAY9UJ5zP6NLo37levC3JEJGiis7nbPBRYXw,2769
|
73
|
+
lionagi/utils/call_util.py,sha256=g4MRQChRGAz_1W35ThnEkyVR0W-neEk3ylgUq9IxbR8,12440
|
74
|
+
lionagi/utils/flat_util.py,sha256=I1HJXs5llQ6oOoJudT4B7bp9u9_TLEFT_yPcXoNhrNA,22899
|
75
|
+
lionagi/utils/io_util.py,sha256=Yzzo-GACV3QXdEFpdfVrpjxc8YJEHJof7hACiTQ59gU,3515
|
76
|
+
lionagi/utils/load_utils.py,sha256=nSDWSXRFHO-tJXGMW-7icOqkeL1baQENf9BDyJZU88o,6667
|
77
|
+
lionagi/utils/sys_util.py,sha256=nxv5mFagY0zBdMLhf9LD_SsK5Cd2rz9zMslr9xIwpfM,6305
|
78
|
+
lionagi/utils/tool_util.py,sha256=RgTML1ZZbmVGdBV2pMpbDCDaNawf_p4d8bqliaOS898,2914
|
79
|
+
lionagi/utils/type_util.py,sha256=eUfsvIr_VRykruE4TEUHssBME8Iug_4U4myY8xFV2bA,3135
|
80
|
+
lionagi-0.0.113.dist-info/LICENSE,sha256=TBnSyG8fs_tMRtK805GzA1cIyExleKyzoN_kuVxT9IY,11358
|
81
|
+
lionagi-0.0.113.dist-info/METADATA,sha256=-zs_6eEFEiCbvqycmpy9YxKVOOIUdzAlB6IrnrHJadk,18027
|
82
|
+
lionagi-0.0.113.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
|
83
|
+
lionagi-0.0.113.dist-info/top_level.txt,sha256=szvch_d2jE1Lu9ZIKsl26Ll6BGfYfbOgt5lm-UpFSo4,8
|
84
|
+
lionagi-0.0.113.dist-info/RECORD,,
|
@@ -1,20 +0,0 @@
|
|
1
|
-
from .base_endpoint import BaseEndpoint
|
2
|
-
|
3
|
-
|
4
|
-
class ChatCompletion(BaseEndpoint):
|
5
|
-
endpoint: str = "chat/completion"
|
6
|
-
|
7
|
-
@classmethod
|
8
|
-
def create_payload(scls, messages, llmconfig, schema, **kwargs):
|
9
|
-
config = {**llmconfig, **kwargs}
|
10
|
-
payload = {"messages": messages}
|
11
|
-
for key in schema['required']:
|
12
|
-
payload.update({key: config[key]})
|
13
|
-
|
14
|
-
for key in schema['optional']:
|
15
|
-
if bool(config[key]) is True and str(config[key]).lower() != "none":
|
16
|
-
payload.update({key: config[key]})
|
17
|
-
return payload
|
18
|
-
|
19
|
-
def process_response(self, session, payload, completion):
|
20
|
-
...
|
File without changes
|
lionagi/llm_configs.py
DELETED
@@ -1,21 +0,0 @@
|
|
1
|
-
oai_schema = {
|
2
|
-
"required" : ["model", "frequency_penalty", "n", "presence_penalty", "response_format", "temperature", "top_p"],
|
3
|
-
"optional": ["seed", "stop", "stream", "tools", "tool_choice", "user", "max_tokens"]
|
4
|
-
}
|
5
|
-
|
6
|
-
oai_llmconfig = {
|
7
|
-
"model": "gpt-4-1106-preview",
|
8
|
-
"frequency_penalty": 0,
|
9
|
-
"max_tokens": None,
|
10
|
-
"n": 1,
|
11
|
-
"presence_penalty": 0,
|
12
|
-
"response_format": {"type": "text"},
|
13
|
-
"seed": None,
|
14
|
-
"stop": None,
|
15
|
-
"stream": False,
|
16
|
-
"temperature": 0.7,
|
17
|
-
"top_p": 1,
|
18
|
-
"tools": None,
|
19
|
-
"tool_choice": "none",
|
20
|
-
"user": None
|
21
|
-
}
|
lionagi/loader/load_utils.py
DELETED
@@ -1,161 +0,0 @@
|
|
1
|
-
import math
|
2
|
-
from pathlib import Path
|
3
|
-
from typing import Any, Dict, List, Union, Callable, Optional
|
4
|
-
|
5
|
-
from ..utils.sys_utils import to_list, l_call
|
6
|
-
from ..schema import DataLogger
|
7
|
-
|
8
|
-
def _dir_to_path(dir, ext, recursive):
|
9
|
-
tem = '**/*' if recursive else '*'
|
10
|
-
return list(Path(dir).glob(tem + ext))
|
11
|
-
|
12
|
-
def _split_path(path: Path) -> tuple:
|
13
|
-
folder_name = path.parent.name
|
14
|
-
file_name = path.name
|
15
|
-
return (folder_name, file_name)
|
16
|
-
|
17
|
-
def dir_to_path(dir: str, ext, recursive: bool = False, flat: bool = True):
|
18
|
-
try:
|
19
|
-
return to_list(l_call(ext, _dir_to_path, flat=True,
|
20
|
-
recursive=recursive, dir=dir, ext=ext),
|
21
|
-
flat=flat)
|
22
|
-
except:
|
23
|
-
raise ValueError("Invalid directory or extension, please check the path")
|
24
|
-
|
25
|
-
def read_text(filepath: str, clean: bool = True) -> str:
|
26
|
-
with open(filepath, 'r') as f:
|
27
|
-
content = f.read()
|
28
|
-
if clean:
|
29
|
-
# Define characters to replace and their replacements
|
30
|
-
replacements = {'\\': ' ', '\n': ' ', '\t': ' ', ' ': ' ', '\'': ' '}
|
31
|
-
for old, new in replacements.items():
|
32
|
-
content = content.replace(old, new)
|
33
|
-
return content
|
34
|
-
|
35
|
-
def dir_to_files(dir: str,
|
36
|
-
ext: str,
|
37
|
-
recursive: bool = False,
|
38
|
-
reader: Callable = read_text,
|
39
|
-
clean: bool = True,
|
40
|
-
to_csv: bool = False,
|
41
|
-
project: str = 'project',
|
42
|
-
output_dir: str = 'data/logs/sources/',
|
43
|
-
filename: Optional[str] = None,
|
44
|
-
verbose: bool = True,
|
45
|
-
timestamp: bool = True,
|
46
|
-
logger: Optional[DataLogger] = None):
|
47
|
-
|
48
|
-
sources = dir_to_path(dir, ext, recursive)
|
49
|
-
|
50
|
-
def _to_dict(path_: Path) -> Dict[str, Union[str, Path]]:
|
51
|
-
folder, file = _split_path(path_)
|
52
|
-
content = reader(str(path_), clean=clean)
|
53
|
-
return {
|
54
|
-
'project': project,
|
55
|
-
'folder': folder,
|
56
|
-
'file': file,
|
57
|
-
"file_size": len(str(content)),
|
58
|
-
'content': content
|
59
|
-
} if content else None
|
60
|
-
|
61
|
-
logs = to_list(l_call(sources, _to_dict, flat=True), dropna=True)
|
62
|
-
|
63
|
-
if to_csv:
|
64
|
-
filename = filename or f"{project}_sources.csv"
|
65
|
-
logger = DataLogger(dir=output_dir, log=logs) if not logger else logger
|
66
|
-
logger.to_csv(dir=output_dir, filename=filename, verbose=verbose, timestamp=timestamp)
|
67
|
-
|
68
|
-
return logs
|
69
|
-
|
70
|
-
def _chunk_n1(input):
|
71
|
-
return [input]
|
72
|
-
|
73
|
-
def _chunk_n2(input, chunk_size, overlap_size, threshold):
|
74
|
-
chunks = []
|
75
|
-
chunks.append(input[:chunk_size + overlap_size])
|
76
|
-
if len(input) - chunk_size > threshold:
|
77
|
-
chunks.append(input[chunk_size - overlap_size:])
|
78
|
-
else:
|
79
|
-
return [input]
|
80
|
-
return chunks
|
81
|
-
|
82
|
-
def _chunk_n3(input, chunk_size, overlap_size, threshold, n_chunks):
|
83
|
-
chunks = []
|
84
|
-
chunks.append(input[:chunk_size + overlap_size])
|
85
|
-
for i in range(1, n_chunks - 1):
|
86
|
-
start_idx = chunk_size * i - overlap_size
|
87
|
-
end_idx = chunk_size * (i + 1) + overlap_size
|
88
|
-
chunks.append(input[start_idx:end_idx])
|
89
|
-
|
90
|
-
if len(input) - chunk_size * (n_chunks - 1) > threshold:
|
91
|
-
chunks.append(input[chunk_size * (n_chunks - 1) - overlap_size:])
|
92
|
-
else:
|
93
|
-
chunks[-1] += input[chunk_size * (n_chunks - 1) + overlap_size:]
|
94
|
-
|
95
|
-
return chunks
|
96
|
-
|
97
|
-
def chunk_text(input: str,
|
98
|
-
chunk_size: int,
|
99
|
-
overlap: float,
|
100
|
-
threshold: int) -> List[Union[str, None]]:
|
101
|
-
try:
|
102
|
-
if not isinstance(input, str): input = str(input)
|
103
|
-
|
104
|
-
n_chunks = math.ceil(len(input) / chunk_size)
|
105
|
-
overlap_size = int(chunk_size * overlap / 2)
|
106
|
-
|
107
|
-
if n_chunks == 1: return _chunk_n1(input)
|
108
|
-
|
109
|
-
elif n_chunks == 2:
|
110
|
-
return _chunk_n2(input=input, chunk_size=chunk_size, overlap_size=overlap_size,
|
111
|
-
threshold=threshold)
|
112
|
-
elif n_chunks > 2:
|
113
|
-
return _chunk_n3(input=input, chunk_size=chunk_size, overlap_size=overlap_size,
|
114
|
-
threshold=threshold, n_chunks=n_chunks)
|
115
|
-
|
116
|
-
except Exception as e:
|
117
|
-
raise ValueError(f"An error occurred while chunking the text. {e}")
|
118
|
-
|
119
|
-
def _file_to_chunks(input: Dict[str, Any],
|
120
|
-
field: str = 'content',
|
121
|
-
chunk_size: int = 1500,
|
122
|
-
overlap: float = 0.1,
|
123
|
-
threshold: int = 200) -> List[Dict[str, Any]]:
|
124
|
-
try:
|
125
|
-
out = {key: value for key, value in input.items() if key != field}
|
126
|
-
out.update({"chunk_overlap": overlap, "chunk_threshold": threshold})
|
127
|
-
|
128
|
-
chunks = chunk_text(input[field], chunk_size=chunk_size, overlap=overlap, threshold=threshold)
|
129
|
-
logs = []
|
130
|
-
for i, chunk in enumerate(chunks):
|
131
|
-
chunk_dict = out.copy()
|
132
|
-
chunk_dict.update({
|
133
|
-
'file_chunks': len(chunks),
|
134
|
-
'chunk_id': i + 1,
|
135
|
-
'chunk_size': len(chunk),
|
136
|
-
f'chunk_{field}': chunk
|
137
|
-
})
|
138
|
-
logs.append(chunk_dict)
|
139
|
-
|
140
|
-
return logs
|
141
|
-
|
142
|
-
except Exception as e:
|
143
|
-
raise ValueError(f"An error occurred while chunking the file. {e}")
|
144
|
-
|
145
|
-
def file_to_chunks(input,
|
146
|
-
project='project',
|
147
|
-
output_dir='data/logs/sources/',
|
148
|
-
chunk_func = _file_to_chunks,
|
149
|
-
to_csv=False,
|
150
|
-
filename=None,
|
151
|
-
verbose=True,
|
152
|
-
timestamp=True,
|
153
|
-
logger=None, **kwargs):
|
154
|
-
logs = to_list(l_call(input, chunk_func, **kwargs), flat=True)
|
155
|
-
|
156
|
-
if to_csv:
|
157
|
-
filename = filename if filename else f"{project}_sources.csv"
|
158
|
-
logger = logger or DataLogger(log=logs)
|
159
|
-
logger.to_csv(dir=output_dir, filename=filename, verbose=verbose, timestamp=timestamp)
|
160
|
-
|
161
|
-
return logs
|
lionagi/schema.py
DELETED
@@ -1,275 +0,0 @@
|
|
1
|
-
import json
|
2
|
-
import networkx as nx
|
3
|
-
from collections import deque
|
4
|
-
from typing import Any, Dict, Optional, Union, Callable
|
5
|
-
|
6
|
-
from pydantic import BaseModel, Field
|
7
|
-
from .utils.sys_utils import create_id, create_path, to_csv
|
8
|
-
|
9
|
-
|
10
|
-
# checked --------------------------------------------------------
|
11
|
-
class BaseNode(BaseModel):
|
12
|
-
id_: str = Field(default_factory=lambda: str(create_id()), alias="node_id")
|
13
|
-
content: Union[str, Dict[str, Any], None, Any] = None
|
14
|
-
metadata: Union[Dict[str, Any], None] = Field(default_factory=dict)
|
15
|
-
label: Optional[str] = None
|
16
|
-
|
17
|
-
@classmethod
|
18
|
-
def class_name(cls) -> str:
|
19
|
-
return cls.__name__
|
20
|
-
|
21
|
-
# to some structure
|
22
|
-
def to_json(self) -> str:
|
23
|
-
return json.dumps(self.model_dump(by_alias=True))
|
24
|
-
|
25
|
-
def to_dict(self) -> Dict[str, Any]:
|
26
|
-
return self.model_dump(by_alias=True)
|
27
|
-
|
28
|
-
# from some structure
|
29
|
-
@classmethod
|
30
|
-
def from_json(cls, json_str: str) -> "BaseNode":
|
31
|
-
data = json.loads(json_str)
|
32
|
-
return cls(**data)
|
33
|
-
|
34
|
-
@classmethod
|
35
|
-
def from_dict(cls, data: Dict[str, Any]) -> "BaseNode":
|
36
|
-
return cls(**data)
|
37
|
-
|
38
|
-
def add_meta(self, replace=True, **kwags) -> None:
|
39
|
-
if replace:
|
40
|
-
self.metadata.update(**kwags)
|
41
|
-
else:
|
42
|
-
for k, v in kwags.items():
|
43
|
-
if k in self.metadata.keys():
|
44
|
-
raise ValueError(f"Key already existed")
|
45
|
-
if k not in self.metadata.keys():
|
46
|
-
self.metadata.update({k:v})
|
47
|
-
|
48
|
-
def set_meta(self, metadata_: dict) -> None:
|
49
|
-
self.metadata = metadata_
|
50
|
-
|
51
|
-
def set_content(self, content: Union[str, Dict[str, Any], None, Any]):
|
52
|
-
self.content = content
|
53
|
-
|
54
|
-
def set_id(self, id: str):
|
55
|
-
self.id_ = id
|
56
|
-
|
57
|
-
def get_meta(self):
|
58
|
-
return self.metadata
|
59
|
-
|
60
|
-
def get_content(self):
|
61
|
-
return self.content
|
62
|
-
|
63
|
-
def get_id(self):
|
64
|
-
return self.id_
|
65
|
-
|
66
|
-
class ConditionalRelationship(BaseNode):
|
67
|
-
target_node_id: str
|
68
|
-
condition: Optional[Dict] = None
|
69
|
-
|
70
|
-
def condition_existed(self, condition_key):
|
71
|
-
if self.condition:
|
72
|
-
try:
|
73
|
-
self.condition.get(condition_key)
|
74
|
-
return True
|
75
|
-
except:
|
76
|
-
return False
|
77
|
-
return False
|
78
|
-
|
79
|
-
def get_condition(self, condition_key) -> bool:
|
80
|
-
if self.condition_existed(condition_key=condition_key):
|
81
|
-
a = self.condition.get(condition_key)
|
82
|
-
if a is not None and str(a).strip().lower() != 'none':
|
83
|
-
return a
|
84
|
-
else:
|
85
|
-
raise ValueError(f"Condition {condition_key} has no value")
|
86
|
-
|
87
|
-
|
88
|
-
class DataNode(BaseNode):
|
89
|
-
|
90
|
-
...
|
91
|
-
# def from_llama(self, data_:, **kwargs):
|
92
|
-
# ...
|
93
|
-
|
94
|
-
# def to_llama(self, **kwargs):
|
95
|
-
# # to llama_index textnode
|
96
|
-
# ...
|
97
|
-
|
98
|
-
# def from_langchain(self, data_, **kwargs):
|
99
|
-
# ...
|
100
|
-
|
101
|
-
# def to_langchain(self, **kwargs):
|
102
|
-
# ...
|
103
|
-
|
104
|
-
# def to_csv(self, **kwargs):
|
105
|
-
# ...
|
106
|
-
|
107
|
-
# def __call__(self, file_=None):
|
108
|
-
# ...
|
109
|
-
|
110
|
-
|
111
|
-
class MessageNode(BaseNode):
|
112
|
-
role: str
|
113
|
-
name: str
|
114
|
-
|
115
|
-
# def from_oai(self):
|
116
|
-
# ...
|
117
|
-
|
118
|
-
|
119
|
-
class File(DataNode):
|
120
|
-
...
|
121
|
-
|
122
|
-
# def from_path(self, path_, reader, clean=True, **kwargs):
|
123
|
-
# self.content = reader(path_=path_, clean=clean, **kwargs)
|
124
|
-
|
125
|
-
# def to_chunks(self, chunker, **kwargs):
|
126
|
-
# ...
|
127
|
-
|
128
|
-
|
129
|
-
class Chunk(DataNode):
|
130
|
-
...
|
131
|
-
# @classmethod
|
132
|
-
# def from_files(cls, files):
|
133
|
-
# ...
|
134
|
-
|
135
|
-
# @classmethod
|
136
|
-
# def to_files(cls):
|
137
|
-
# ...
|
138
|
-
|
139
|
-
|
140
|
-
class DataLogger:
|
141
|
-
"""
|
142
|
-
Logs data entries and outputs them to a CSV file.
|
143
|
-
|
144
|
-
This class provides a logging mechanism for data entries that can be saved to a
|
145
|
-
CSV file. It offers methods for appending new log entries, saving the log to a CSV file,
|
146
|
-
and setting the directory where the logs should be saved.
|
147
|
-
|
148
|
-
Attributes:
|
149
|
-
dir (str):
|
150
|
-
The directory where the log files are to be saved.
|
151
|
-
log (deque):
|
152
|
-
A deque that stores log entries.
|
153
|
-
|
154
|
-
Methods:
|
155
|
-
__call__(entry):
|
156
|
-
Appends a new entry to the log.
|
157
|
-
to_csv(dir: str, filename: str, verbose: bool, timestamp: bool, dir_exist_ok: bool, file_exist_ok: bool):
|
158
|
-
Converts the log to a CSV format and saves it to a file.
|
159
|
-
set_dir(dir: str):
|
160
|
-
Sets the directory for saving log files.
|
161
|
-
"""
|
162
|
-
|
163
|
-
def __init__(self, dir= None, log: list = None) -> None:
|
164
|
-
"""
|
165
|
-
Initializes a new instance of the DataLogger class.
|
166
|
-
|
167
|
-
Parameters:
|
168
|
-
dir (str, optional): The directory where the log files will be saved. Defaults to None.
|
169
|
-
|
170
|
-
log (list, optional): An initial list of log entries. Defaults to an empty deque.
|
171
|
-
"""
|
172
|
-
self.dir = dir
|
173
|
-
self.log = deque(log) if log else deque()
|
174
|
-
|
175
|
-
def __call__(self, entry):
|
176
|
-
"""
|
177
|
-
Appends a new entry to the log.
|
178
|
-
|
179
|
-
Parameters:
|
180
|
-
entry: The entry to append to the log. The entry can be of any datatype.
|
181
|
-
"""
|
182
|
-
self.log.append(entry)
|
183
|
-
|
184
|
-
def to_csv(self, filename: str, dir=None, verbose: bool = True, timestamp: bool = True, dir_exist_ok=True, file_exist_ok=False):
|
185
|
-
"""
|
186
|
-
Converts the log to a CSV format and saves it to a file.
|
187
|
-
|
188
|
-
Parameters:
|
189
|
-
dir (str): The directory where the CSV file will be saved.
|
190
|
-
|
191
|
-
filename (str): The name of the CSV file.
|
192
|
-
|
193
|
-
verbose (bool, optional): If True, prints a message after saving the log. Defaults to True.
|
194
|
-
|
195
|
-
timestamp (bool, optional): If True, appends a timestamp to the filename. Defaults to True.
|
196
|
-
|
197
|
-
dir_exist_ok (bool, optional): If True, overrides the existing directory if needed. Defaults to True.
|
198
|
-
|
199
|
-
file_exist_ok (bool, optional): If True, overrides the existing file if needed. Defaults to False.
|
200
|
-
|
201
|
-
Postconditions:
|
202
|
-
Saves the log entries to a CSV file and clears the `log` attribute.
|
203
|
-
|
204
|
-
Optionally prints a message with the number of log entries saved and the file path.
|
205
|
-
"""
|
206
|
-
dir = dir or self.dir
|
207
|
-
filepath = create_path(dir=dir, filename=filename, timestamp=timestamp, dir_exist_ok=dir_exist_ok)
|
208
|
-
to_csv(list(self.log), filepath, file_exist_ok=file_exist_ok)
|
209
|
-
n_logs = len(list(self.log))
|
210
|
-
self.log = deque()
|
211
|
-
if verbose:
|
212
|
-
print(f"{n_logs} logs saved to {filepath}")
|
213
|
-
|
214
|
-
def set_dir(self, dir: str):
|
215
|
-
"""
|
216
|
-
Sets the directory where log files will be saved.
|
217
|
-
|
218
|
-
Parameters:
|
219
|
-
dir (str): The directory to set for saving log files.
|
220
|
-
"""
|
221
|
-
self.dir = dir
|
222
|
-
|
223
|
-
|
224
|
-
class BaseTool(BaseNode):
|
225
|
-
name: str = None
|
226
|
-
func: Callable = None
|
227
|
-
content: Any = None
|
228
|
-
parser: Callable = None
|
229
|
-
|
230
|
-
def initialize(self):
|
231
|
-
...
|
232
|
-
|
233
|
-
def execute(self):
|
234
|
-
...
|
235
|
-
|
236
|
-
def shutdown(self):
|
237
|
-
...
|
238
|
-
|
239
|
-
def __enter__(self):
|
240
|
-
self.initialize()
|
241
|
-
return self
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
# checked ------------------------------------------------------
|
246
|
-
class Structure:
|
247
|
-
def __init__(self):
|
248
|
-
self.graph = nx.DiGraph()
|
249
|
-
|
250
|
-
def add_node(self, node: BaseNode) -> None:
|
251
|
-
self.graph.add_node(node.id_, content=node.content, metadata=node.metadata)
|
252
|
-
|
253
|
-
def add_relationship(self, source_node_id: str, target_node_id: str, relationship: ConditionalRelationship) -> None:
|
254
|
-
self.graph.add_edge(source_node_id, target_node_id,
|
255
|
-
label=relationship.label,
|
256
|
-
properties=relationship.properties,
|
257
|
-
condition=relationship.condition)
|
258
|
-
|
259
|
-
def remove_relationship(self, source_node_id: str, target_node_id: str) -> None:
|
260
|
-
if self.graph.has_edge(source_node_id, target_node_id):
|
261
|
-
self.graph.remove_edge(source_node_id, target_node_id)
|
262
|
-
|
263
|
-
def get_relationships(self, node_id: str) -> list:
|
264
|
-
return list(self.graph.edges(node_id, data=True))
|
265
|
-
|
266
|
-
def get_conditional_relationships(self, node_id: str) -> list:
|
267
|
-
return [(target_id, data) for _, target_id, data in self.graph.edges(node_id, data=True) if "condition" in data]
|
268
|
-
|
269
|
-
def to_json(self) -> str:
|
270
|
-
return json.dumps(nx.node_link_data(self.graph))
|
271
|
-
|
272
|
-
def from_json(self, data: str) -> None:
|
273
|
-
graph_data = json.loads(data)
|
274
|
-
self.graph = nx.node_link_graph(graph_data)
|
275
|
-
|
lionagi/service_/__init__.py
DELETED
lionagi/service_/base_service.py
DELETED
@@ -1,48 +0,0 @@
|
|
1
|
-
import re
|
2
|
-
from typing import Generator
|
3
|
-
from .service_utils import BaseService
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
class BaseAPIService(BaseService):
|
8
|
-
|
9
|
-
def __init__(self, api_key: str = None,
|
10
|
-
status_tracker = None,
|
11
|
-
queue = None, endpoint=None, schema=None,
|
12
|
-
ratelimiter=None, max_requests_per_minute=None, max_tokens_per_minute=None) -> None:
|
13
|
-
self.api_key = api_key
|
14
|
-
self.status_tracker = status_tracker
|
15
|
-
self.queue = queue
|
16
|
-
self.endpoint=endpoint
|
17
|
-
self.schema = schema
|
18
|
-
self.rate_limiter = ratelimiter(max_requests_per_minute, max_tokens_per_minute)
|
19
|
-
|
20
|
-
@staticmethod
|
21
|
-
def api_methods(http_session, method="post"):
|
22
|
-
if method not in ["post", "delete", "head", "options", "patch"]:
|
23
|
-
raise ValueError("Invalid request, method must be in ['post', 'delete', 'head', 'options', 'patch']")
|
24
|
-
elif method == "post":
|
25
|
-
return http_session.post
|
26
|
-
elif method == "delete":
|
27
|
-
return http_session.delete
|
28
|
-
elif method == "head":
|
29
|
-
return http_session.head
|
30
|
-
elif method == "options":
|
31
|
-
return http_session.options
|
32
|
-
elif method == "patch":
|
33
|
-
return http_session.patch
|
34
|
-
|
35
|
-
@staticmethod
|
36
|
-
def api_endpoint_from_url(request_url: str) -> str:
|
37
|
-
match = re.search(r"^https://[^/]+/v\d+/(.+)$", request_url)
|
38
|
-
if match:
|
39
|
-
return match.group(1)
|
40
|
-
else:
|
41
|
-
return ""
|
42
|
-
|
43
|
-
@staticmethod
|
44
|
-
def task_id_generator_function() -> Generator[int, None, None]:
|
45
|
-
task_id = 0
|
46
|
-
while True:
|
47
|
-
yield task_id
|
48
|
-
task_id += 1
|
lionagi/service_/openrouter.py
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
# TODO
|
lionagi/services.py
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
from .service_.oai import OpenAIService
|