lionagi 0.0.112__py3-none-any.whl → 0.0.113__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (94) hide show
  1. lionagi/__init__.py +3 -3
  2. lionagi/bridge/__init__.py +7 -0
  3. lionagi/bridge/langchain.py +131 -0
  4. lionagi/bridge/llama_index.py +157 -0
  5. lionagi/configs/__init__.py +7 -0
  6. lionagi/configs/oai_configs.py +49 -0
  7. lionagi/configs/openrouter_config.py +49 -0
  8. lionagi/core/__init__.py +8 -2
  9. lionagi/core/instruction_sets.py +1 -3
  10. lionagi/core/messages.py +2 -2
  11. lionagi/core/sessions.py +174 -27
  12. lionagi/datastore/__init__.py +1 -0
  13. lionagi/loader/__init__.py +9 -4
  14. lionagi/loader/chunker.py +157 -0
  15. lionagi/loader/reader.py +124 -0
  16. lionagi/objs/__init__.py +7 -0
  17. lionagi/objs/messenger.py +163 -0
  18. lionagi/objs/tool_registry.py +247 -0
  19. lionagi/schema/__init__.py +11 -0
  20. lionagi/schema/base_schema.py +239 -0
  21. lionagi/schema/base_tool.py +9 -0
  22. lionagi/schema/data_logger.py +94 -0
  23. lionagi/services/__init__.py +14 -0
  24. lionagi/{service_/oai.py → services/base_api_service.py} +49 -82
  25. lionagi/{endpoint/base_endpoint.py → services/chatcompletion.py} +19 -22
  26. lionagi/services/oai.py +34 -0
  27. lionagi/services/openrouter.py +32 -0
  28. lionagi/{service_/service_utils.py → services/service_objs.py} +0 -1
  29. lionagi/structure/__init__.py +7 -0
  30. lionagi/structure/relationship.py +128 -0
  31. lionagi/structure/structure.py +160 -0
  32. lionagi/tests/test_flatten_util.py +426 -0
  33. lionagi/tools/__init__.py +0 -5
  34. lionagi/tools/coder.py +1 -0
  35. lionagi/tools/scorer.py +1 -0
  36. lionagi/tools/validator.py +1 -0
  37. lionagi/utils/__init__.py +46 -20
  38. lionagi/utils/api_util.py +86 -0
  39. lionagi/utils/call_util.py +347 -0
  40. lionagi/utils/flat_util.py +540 -0
  41. lionagi/utils/io_util.py +102 -0
  42. lionagi/utils/load_utils.py +190 -0
  43. lionagi/utils/sys_util.py +191 -0
  44. lionagi/utils/tool_util.py +92 -0
  45. lionagi/utils/type_util.py +81 -0
  46. lionagi/version.py +1 -1
  47. {lionagi-0.0.112.dist-info → lionagi-0.0.113.dist-info}/METADATA +37 -13
  48. lionagi-0.0.113.dist-info/RECORD +84 -0
  49. lionagi/endpoint/chat_completion.py +0 -20
  50. lionagi/endpoint/endpoint_utils.py +0 -0
  51. lionagi/llm_configs.py +0 -21
  52. lionagi/loader/load_utils.py +0 -161
  53. lionagi/schema.py +0 -275
  54. lionagi/service_/__init__.py +0 -6
  55. lionagi/service_/base_service.py +0 -48
  56. lionagi/service_/openrouter.py +0 -1
  57. lionagi/services.py +0 -1
  58. lionagi/tools/tool_utils.py +0 -75
  59. lionagi/utils/sys_utils.py +0 -799
  60. lionagi-0.0.112.dist-info/RECORD +0 -67
  61. /lionagi/{core/responses.py → datastore/chroma.py} +0 -0
  62. /lionagi/{endpoint/assistants.py → datastore/deeplake.py} +0 -0
  63. /lionagi/{endpoint/audio.py → datastore/elasticsearch.py} +0 -0
  64. /lionagi/{endpoint/embeddings.py → datastore/lantern.py} +0 -0
  65. /lionagi/{endpoint/files.py → datastore/pinecone.py} +0 -0
  66. /lionagi/{endpoint/fine_tuning.py → datastore/postgres.py} +0 -0
  67. /lionagi/{endpoint/images.py → datastore/qdrant.py} +0 -0
  68. /lionagi/{endpoint/messages.py → schema/base_condition.py} +0 -0
  69. /lionagi/{service_ → services}/anthropic.py +0 -0
  70. /lionagi/{service_ → services}/anyscale.py +0 -0
  71. /lionagi/{service_ → services}/azure.py +0 -0
  72. /lionagi/{service_ → services}/bedrock.py +0 -0
  73. /lionagi/{service_ → services}/everlyai.py +0 -0
  74. /lionagi/{service_ → services}/gemini.py +0 -0
  75. /lionagi/{service_ → services}/gpt4all.py +0 -0
  76. /lionagi/{service_ → services}/huggingface.py +0 -0
  77. /lionagi/{service_ → services}/litellm.py +0 -0
  78. /lionagi/{service_ → services}/localai.py +0 -0
  79. /lionagi/{service_ → services}/mistralai.py +0 -0
  80. /lionagi/{service_ → services}/ollama.py +0 -0
  81. /lionagi/{service_ → services}/openllm.py +0 -0
  82. /lionagi/{service_ → services}/perplexity.py +0 -0
  83. /lionagi/{service_ → services}/predibase.py +0 -0
  84. /lionagi/{service_ → services}/rungpt.py +0 -0
  85. /lionagi/{service_ → services}/vllm.py +0 -0
  86. /lionagi/{service_ → services}/xinference.py +0 -0
  87. /lionagi/{endpoint → tests}/__init__.py +0 -0
  88. /lionagi/{endpoint/models.py → tools/planner.py} +0 -0
  89. /lionagi/{endpoint/moderations.py → tools/prompter.py} +0 -0
  90. /lionagi/{endpoint/runs.py → tools/sandbox.py} +0 -0
  91. /lionagi/{endpoint/threads.py → tools/summarizer.py} +0 -0
  92. {lionagi-0.0.112.dist-info → lionagi-0.0.113.dist-info}/LICENSE +0 -0
  93. {lionagi-0.0.112.dist-info → lionagi-0.0.113.dist-info}/WHEEL +0 -0
  94. {lionagi-0.0.112.dist-info → lionagi-0.0.113.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,84 @@
1
+ lionagi/__init__.py,sha256=gAOIyJ-kItUn2ya8G3zW6pvuzmk60xk7zO_GN66UJAM,888
2
+ lionagi/version.py,sha256=WJPRYfehqmqr60a1h9m6a2fbtJa0dT03ixD6hahuBgE,24
3
+ lionagi/bridge/__init__.py,sha256=I19ztSDDwV1-e-GSRs45bJgjbg454e0vcFv2jwgwQY0,139
4
+ lionagi/bridge/langchain.py,sha256=nJdnv_zHpchQXUlh5ml6OZKGO9Rbaa7B-XbeKB3Ej7Y,4579
5
+ lionagi/bridge/llama_index.py,sha256=sapGIgS3RkDUyrPCdjZizoRdAmktZLHzUbYQBw6nnZw,5510
6
+ lionagi/configs/__init__.py,sha256=88KoeoB8dHhgZdQ8I4rbb7e8-FUd-_gi1dXaA95vJ3Y,141
7
+ lionagi/configs/oai_configs.py,sha256=C89jXb2Xb7qoOXIhtrMlIRVYJqjh7knpu6CAOloj9eo,1199
8
+ lionagi/configs/openrouter_config.py,sha256=qbEQQqQPpzMp3eP5U8kuvGTUYdMgemuuq-BdgOYgYI8,1256
9
+ lionagi/core/__init__.py,sha256=XAw2vF1wuQYKSAxHcBanfYN0k-5qLSggkl3yZgwjSRw,347
10
+ lionagi/core/conversations.py,sha256=Jh4Bvv0SBaW5ZG_D9hyWveQ-PXPDGfvp_V7UxbPsFoA,4084
11
+ lionagi/core/flows.py,sha256=rhPBmdN-JKo7Zcv9hiy6k0b-CjKBHMemkYITM7a9fNo,32
12
+ lionagi/core/instruction_sets.py,sha256=y-fDltfDtji4fCtHn8fJ_M9OtYCfjqeTds7bpzknvhU,51
13
+ lionagi/core/messages.py,sha256=XisuXcJBUoyYcd-6DLcV_NZnsKXzZcFf4Dk_fxTlXc8,6533
14
+ lionagi/core/sessions.py,sha256=sjg7bCORFhpaEgDBTExyhfgGxRAUwkOCft0MmP-BGoU,11161
15
+ lionagi/datastore/__init__.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
16
+ lionagi/datastore/chroma.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
17
+ lionagi/datastore/deeplake.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
18
+ lionagi/datastore/elasticsearch.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
19
+ lionagi/datastore/lantern.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
20
+ lionagi/datastore/pinecone.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
21
+ lionagi/datastore/postgres.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
22
+ lionagi/datastore/qdrant.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
23
+ lionagi/loader/__init__.py,sha256=0FNrggZnSq1ONLZY2f325SF19UfYHaqJeJXLvpu8KWQ,262
24
+ lionagi/loader/chunker.py,sha256=zGdfS-Gybxc6lm_vMIdt-f25kY2CwDFTLvY-arKxQ8c,6476
25
+ lionagi/loader/reader.py,sha256=8kxYlqhwmMpuNTWdYtsNc9g-1Mf264Y8XRu1GkevhTg,4527
26
+ lionagi/objs/__init__.py,sha256=4qxo9ZGKe37fjj_Gt2hgi4QTTIPmDf7mkyUK7euVmsU,137
27
+ lionagi/objs/messenger.py,sha256=k-ueq2LBLTTDgk8u-Vyja8WBFM7dWFInWO97HNyOXaI,6137
28
+ lionagi/objs/tool_registry.py,sha256=cnoaW1TvNhKKerQ4tTUOSHEADmsTdoUW2lT24LcHIx0,9357
29
+ lionagi/schema/__init__.py,sha256=cB7rSYeRXujDwp2f5IMK83-NzdqPfbMhGijPCPXgT_c,210
30
+ lionagi/schema/base_condition.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
31
+ lionagi/schema/base_schema.py,sha256=gx4LDxYnVO9vqp0JdevUSasqrat7Lu3rRURTiv7gGj0,8348
32
+ lionagi/schema/base_tool.py,sha256=sKbCrS0ZZINkr03Fj0lk0Kt0XPiBz9IXwA9A1aDgN88,192
33
+ lionagi/schema/data_logger.py,sha256=nmv0jJDkE-vzz_KyBl1_bHB8KVvhZ6sItOBQcvBkAvE,3417
34
+ lionagi/services/__init__.py,sha256=js4LnJEJv4kkxPPamkOgFkOsrc1yr2dK-z9CLTTsX04,313
35
+ lionagi/services/anthropic.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
36
+ lionagi/services/anyscale.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
37
+ lionagi/services/azure.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
38
+ lionagi/services/base_api_service.py,sha256=vGCnC1GnW8czpseOx8mcDeyiiAKc_dabUInxjytE-A0,9756
39
+ lionagi/services/bedrock.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
40
+ lionagi/services/chatcompletion.py,sha256=9jVkB4JUX-kzvAHsohA1ZO6ZeXdlN9OAe-cVvLCa9-0,1257
41
+ lionagi/services/everlyai.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
42
+ lionagi/services/gemini.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
43
+ lionagi/services/gpt4all.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
44
+ lionagi/services/huggingface.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
45
+ lionagi/services/litellm.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
46
+ lionagi/services/localai.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
47
+ lionagi/services/mistralai.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
48
+ lionagi/services/oai.py,sha256=NcKiqmQ8YpKEkoVv-6a-mvIZVBumWOVpstpVD0oTiUs,1122
49
+ lionagi/services/ollama.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
50
+ lionagi/services/openllm.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
51
+ lionagi/services/openrouter.py,sha256=vjY0ZydLY5XK4_T2cRiAl8_QNysWy26wcfyIP-1hlSw,1110
52
+ lionagi/services/perplexity.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
53
+ lionagi/services/predibase.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
54
+ lionagi/services/rungpt.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
55
+ lionagi/services/service_objs.py,sha256=w1Rs69eAGyEukTMMvwQHDmFkwTMf2pB9DdGQ-xCjTUw,9727
56
+ lionagi/services/vllm.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
57
+ lionagi/services/xinference.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
58
+ lionagi/structure/__init__.py,sha256=wMPekT2vbWwUkJ5aW5o-lzJC9Fzhta6RHDiFPTNUm_0,120
59
+ lionagi/structure/relationship.py,sha256=5urGnomfhBJUwPLpYpk5YCx-zO2yprJDaPQqMsgQM0A,4124
60
+ lionagi/structure/structure.py,sha256=KWVOTFWk_IBMtY57gut_GFUuO7dKWlIrMH7xZ_ys2c4,5952
61
+ lionagi/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
62
+ lionagi/tests/test_flatten_util.py,sha256=-sOAQriGtHK1U0FTd31IpdKJpgMQREpaXp-hf6FAojQ,17362
63
+ lionagi/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
64
+ lionagi/tools/coder.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
65
+ lionagi/tools/planner.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
66
+ lionagi/tools/prompter.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
67
+ lionagi/tools/sandbox.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
68
+ lionagi/tools/scorer.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
69
+ lionagi/tools/summarizer.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
70
+ lionagi/tools/validator.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
71
+ lionagi/utils/__init__.py,sha256=91eJpmYZLXi_xsFV7yb_uvlRmdhBQWBqa9Bmy18nJYY,1419
72
+ lionagi/utils/api_util.py,sha256=wNYillgmAY9UJ5zP6NLo37levC3JEJGiis7nbPBRYXw,2769
73
+ lionagi/utils/call_util.py,sha256=g4MRQChRGAz_1W35ThnEkyVR0W-neEk3ylgUq9IxbR8,12440
74
+ lionagi/utils/flat_util.py,sha256=I1HJXs5llQ6oOoJudT4B7bp9u9_TLEFT_yPcXoNhrNA,22899
75
+ lionagi/utils/io_util.py,sha256=Yzzo-GACV3QXdEFpdfVrpjxc8YJEHJof7hACiTQ59gU,3515
76
+ lionagi/utils/load_utils.py,sha256=nSDWSXRFHO-tJXGMW-7icOqkeL1baQENf9BDyJZU88o,6667
77
+ lionagi/utils/sys_util.py,sha256=nxv5mFagY0zBdMLhf9LD_SsK5Cd2rz9zMslr9xIwpfM,6305
78
+ lionagi/utils/tool_util.py,sha256=RgTML1ZZbmVGdBV2pMpbDCDaNawf_p4d8bqliaOS898,2914
79
+ lionagi/utils/type_util.py,sha256=eUfsvIr_VRykruE4TEUHssBME8Iug_4U4myY8xFV2bA,3135
80
+ lionagi-0.0.113.dist-info/LICENSE,sha256=TBnSyG8fs_tMRtK805GzA1cIyExleKyzoN_kuVxT9IY,11358
81
+ lionagi-0.0.113.dist-info/METADATA,sha256=-zs_6eEFEiCbvqycmpy9YxKVOOIUdzAlB6IrnrHJadk,18027
82
+ lionagi-0.0.113.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
83
+ lionagi-0.0.113.dist-info/top_level.txt,sha256=szvch_d2jE1Lu9ZIKsl26Ll6BGfYfbOgt5lm-UpFSo4,8
84
+ lionagi-0.0.113.dist-info/RECORD,,
@@ -1,20 +0,0 @@
1
- from .base_endpoint import BaseEndpoint
2
-
3
-
4
- class ChatCompletion(BaseEndpoint):
5
- endpoint: str = "chat/completion"
6
-
7
- @classmethod
8
- def create_payload(scls, messages, llmconfig, schema, **kwargs):
9
- config = {**llmconfig, **kwargs}
10
- payload = {"messages": messages}
11
- for key in schema['required']:
12
- payload.update({key: config[key]})
13
-
14
- for key in schema['optional']:
15
- if bool(config[key]) is True and str(config[key]).lower() != "none":
16
- payload.update({key: config[key]})
17
- return payload
18
-
19
- def process_response(self, session, payload, completion):
20
- ...
File without changes
lionagi/llm_configs.py DELETED
@@ -1,21 +0,0 @@
1
- oai_schema = {
2
- "required" : ["model", "frequency_penalty", "n", "presence_penalty", "response_format", "temperature", "top_p"],
3
- "optional": ["seed", "stop", "stream", "tools", "tool_choice", "user", "max_tokens"]
4
- }
5
-
6
- oai_llmconfig = {
7
- "model": "gpt-4-1106-preview",
8
- "frequency_penalty": 0,
9
- "max_tokens": None,
10
- "n": 1,
11
- "presence_penalty": 0,
12
- "response_format": {"type": "text"},
13
- "seed": None,
14
- "stop": None,
15
- "stream": False,
16
- "temperature": 0.7,
17
- "top_p": 1,
18
- "tools": None,
19
- "tool_choice": "none",
20
- "user": None
21
- }
@@ -1,161 +0,0 @@
1
- import math
2
- from pathlib import Path
3
- from typing import Any, Dict, List, Union, Callable, Optional
4
-
5
- from ..utils.sys_utils import to_list, l_call
6
- from ..schema import DataLogger
7
-
8
- def _dir_to_path(dir, ext, recursive):
9
- tem = '**/*' if recursive else '*'
10
- return list(Path(dir).glob(tem + ext))
11
-
12
- def _split_path(path: Path) -> tuple:
13
- folder_name = path.parent.name
14
- file_name = path.name
15
- return (folder_name, file_name)
16
-
17
- def dir_to_path(dir: str, ext, recursive: bool = False, flat: bool = True):
18
- try:
19
- return to_list(l_call(ext, _dir_to_path, flat=True,
20
- recursive=recursive, dir=dir, ext=ext),
21
- flat=flat)
22
- except:
23
- raise ValueError("Invalid directory or extension, please check the path")
24
-
25
- def read_text(filepath: str, clean: bool = True) -> str:
26
- with open(filepath, 'r') as f:
27
- content = f.read()
28
- if clean:
29
- # Define characters to replace and their replacements
30
- replacements = {'\\': ' ', '\n': ' ', '\t': ' ', ' ': ' ', '\'': ' '}
31
- for old, new in replacements.items():
32
- content = content.replace(old, new)
33
- return content
34
-
35
- def dir_to_files(dir: str,
36
- ext: str,
37
- recursive: bool = False,
38
- reader: Callable = read_text,
39
- clean: bool = True,
40
- to_csv: bool = False,
41
- project: str = 'project',
42
- output_dir: str = 'data/logs/sources/',
43
- filename: Optional[str] = None,
44
- verbose: bool = True,
45
- timestamp: bool = True,
46
- logger: Optional[DataLogger] = None):
47
-
48
- sources = dir_to_path(dir, ext, recursive)
49
-
50
- def _to_dict(path_: Path) -> Dict[str, Union[str, Path]]:
51
- folder, file = _split_path(path_)
52
- content = reader(str(path_), clean=clean)
53
- return {
54
- 'project': project,
55
- 'folder': folder,
56
- 'file': file,
57
- "file_size": len(str(content)),
58
- 'content': content
59
- } if content else None
60
-
61
- logs = to_list(l_call(sources, _to_dict, flat=True), dropna=True)
62
-
63
- if to_csv:
64
- filename = filename or f"{project}_sources.csv"
65
- logger = DataLogger(dir=output_dir, log=logs) if not logger else logger
66
- logger.to_csv(dir=output_dir, filename=filename, verbose=verbose, timestamp=timestamp)
67
-
68
- return logs
69
-
70
- def _chunk_n1(input):
71
- return [input]
72
-
73
- def _chunk_n2(input, chunk_size, overlap_size, threshold):
74
- chunks = []
75
- chunks.append(input[:chunk_size + overlap_size])
76
- if len(input) - chunk_size > threshold:
77
- chunks.append(input[chunk_size - overlap_size:])
78
- else:
79
- return [input]
80
- return chunks
81
-
82
- def _chunk_n3(input, chunk_size, overlap_size, threshold, n_chunks):
83
- chunks = []
84
- chunks.append(input[:chunk_size + overlap_size])
85
- for i in range(1, n_chunks - 1):
86
- start_idx = chunk_size * i - overlap_size
87
- end_idx = chunk_size * (i + 1) + overlap_size
88
- chunks.append(input[start_idx:end_idx])
89
-
90
- if len(input) - chunk_size * (n_chunks - 1) > threshold:
91
- chunks.append(input[chunk_size * (n_chunks - 1) - overlap_size:])
92
- else:
93
- chunks[-1] += input[chunk_size * (n_chunks - 1) + overlap_size:]
94
-
95
- return chunks
96
-
97
- def chunk_text(input: str,
98
- chunk_size: int,
99
- overlap: float,
100
- threshold: int) -> List[Union[str, None]]:
101
- try:
102
- if not isinstance(input, str): input = str(input)
103
-
104
- n_chunks = math.ceil(len(input) / chunk_size)
105
- overlap_size = int(chunk_size * overlap / 2)
106
-
107
- if n_chunks == 1: return _chunk_n1(input)
108
-
109
- elif n_chunks == 2:
110
- return _chunk_n2(input=input, chunk_size=chunk_size, overlap_size=overlap_size,
111
- threshold=threshold)
112
- elif n_chunks > 2:
113
- return _chunk_n3(input=input, chunk_size=chunk_size, overlap_size=overlap_size,
114
- threshold=threshold, n_chunks=n_chunks)
115
-
116
- except Exception as e:
117
- raise ValueError(f"An error occurred while chunking the text. {e}")
118
-
119
- def _file_to_chunks(input: Dict[str, Any],
120
- field: str = 'content',
121
- chunk_size: int = 1500,
122
- overlap: float = 0.1,
123
- threshold: int = 200) -> List[Dict[str, Any]]:
124
- try:
125
- out = {key: value for key, value in input.items() if key != field}
126
- out.update({"chunk_overlap": overlap, "chunk_threshold": threshold})
127
-
128
- chunks = chunk_text(input[field], chunk_size=chunk_size, overlap=overlap, threshold=threshold)
129
- logs = []
130
- for i, chunk in enumerate(chunks):
131
- chunk_dict = out.copy()
132
- chunk_dict.update({
133
- 'file_chunks': len(chunks),
134
- 'chunk_id': i + 1,
135
- 'chunk_size': len(chunk),
136
- f'chunk_{field}': chunk
137
- })
138
- logs.append(chunk_dict)
139
-
140
- return logs
141
-
142
- except Exception as e:
143
- raise ValueError(f"An error occurred while chunking the file. {e}")
144
-
145
- def file_to_chunks(input,
146
- project='project',
147
- output_dir='data/logs/sources/',
148
- chunk_func = _file_to_chunks,
149
- to_csv=False,
150
- filename=None,
151
- verbose=True,
152
- timestamp=True,
153
- logger=None, **kwargs):
154
- logs = to_list(l_call(input, chunk_func, **kwargs), flat=True)
155
-
156
- if to_csv:
157
- filename = filename if filename else f"{project}_sources.csv"
158
- logger = logger or DataLogger(log=logs)
159
- logger.to_csv(dir=output_dir, filename=filename, verbose=verbose, timestamp=timestamp)
160
-
161
- return logs
lionagi/schema.py DELETED
@@ -1,275 +0,0 @@
1
- import json
2
- import networkx as nx
3
- from collections import deque
4
- from typing import Any, Dict, Optional, Union, Callable
5
-
6
- from pydantic import BaseModel, Field
7
- from .utils.sys_utils import create_id, create_path, to_csv
8
-
9
-
10
- # checked --------------------------------------------------------
11
- class BaseNode(BaseModel):
12
- id_: str = Field(default_factory=lambda: str(create_id()), alias="node_id")
13
- content: Union[str, Dict[str, Any], None, Any] = None
14
- metadata: Union[Dict[str, Any], None] = Field(default_factory=dict)
15
- label: Optional[str] = None
16
-
17
- @classmethod
18
- def class_name(cls) -> str:
19
- return cls.__name__
20
-
21
- # to some structure
22
- def to_json(self) -> str:
23
- return json.dumps(self.model_dump(by_alias=True))
24
-
25
- def to_dict(self) -> Dict[str, Any]:
26
- return self.model_dump(by_alias=True)
27
-
28
- # from some structure
29
- @classmethod
30
- def from_json(cls, json_str: str) -> "BaseNode":
31
- data = json.loads(json_str)
32
- return cls(**data)
33
-
34
- @classmethod
35
- def from_dict(cls, data: Dict[str, Any]) -> "BaseNode":
36
- return cls(**data)
37
-
38
- def add_meta(self, replace=True, **kwags) -> None:
39
- if replace:
40
- self.metadata.update(**kwags)
41
- else:
42
- for k, v in kwags.items():
43
- if k in self.metadata.keys():
44
- raise ValueError(f"Key already existed")
45
- if k not in self.metadata.keys():
46
- self.metadata.update({k:v})
47
-
48
- def set_meta(self, metadata_: dict) -> None:
49
- self.metadata = metadata_
50
-
51
- def set_content(self, content: Union[str, Dict[str, Any], None, Any]):
52
- self.content = content
53
-
54
- def set_id(self, id: str):
55
- self.id_ = id
56
-
57
- def get_meta(self):
58
- return self.metadata
59
-
60
- def get_content(self):
61
- return self.content
62
-
63
- def get_id(self):
64
- return self.id_
65
-
66
- class ConditionalRelationship(BaseNode):
67
- target_node_id: str
68
- condition: Optional[Dict] = None
69
-
70
- def condition_existed(self, condition_key):
71
- if self.condition:
72
- try:
73
- self.condition.get(condition_key)
74
- return True
75
- except:
76
- return False
77
- return False
78
-
79
- def get_condition(self, condition_key) -> bool:
80
- if self.condition_existed(condition_key=condition_key):
81
- a = self.condition.get(condition_key)
82
- if a is not None and str(a).strip().lower() != 'none':
83
- return a
84
- else:
85
- raise ValueError(f"Condition {condition_key} has no value")
86
-
87
-
88
- class DataNode(BaseNode):
89
-
90
- ...
91
- # def from_llama(self, data_:, **kwargs):
92
- # ...
93
-
94
- # def to_llama(self, **kwargs):
95
- # # to llama_index textnode
96
- # ...
97
-
98
- # def from_langchain(self, data_, **kwargs):
99
- # ...
100
-
101
- # def to_langchain(self, **kwargs):
102
- # ...
103
-
104
- # def to_csv(self, **kwargs):
105
- # ...
106
-
107
- # def __call__(self, file_=None):
108
- # ...
109
-
110
-
111
- class MessageNode(BaseNode):
112
- role: str
113
- name: str
114
-
115
- # def from_oai(self):
116
- # ...
117
-
118
-
119
- class File(DataNode):
120
- ...
121
-
122
- # def from_path(self, path_, reader, clean=True, **kwargs):
123
- # self.content = reader(path_=path_, clean=clean, **kwargs)
124
-
125
- # def to_chunks(self, chunker, **kwargs):
126
- # ...
127
-
128
-
129
- class Chunk(DataNode):
130
- ...
131
- # @classmethod
132
- # def from_files(cls, files):
133
- # ...
134
-
135
- # @classmethod
136
- # def to_files(cls):
137
- # ...
138
-
139
-
140
- class DataLogger:
141
- """
142
- Logs data entries and outputs them to a CSV file.
143
-
144
- This class provides a logging mechanism for data entries that can be saved to a
145
- CSV file. It offers methods for appending new log entries, saving the log to a CSV file,
146
- and setting the directory where the logs should be saved.
147
-
148
- Attributes:
149
- dir (str):
150
- The directory where the log files are to be saved.
151
- log (deque):
152
- A deque that stores log entries.
153
-
154
- Methods:
155
- __call__(entry):
156
- Appends a new entry to the log.
157
- to_csv(dir: str, filename: str, verbose: bool, timestamp: bool, dir_exist_ok: bool, file_exist_ok: bool):
158
- Converts the log to a CSV format and saves it to a file.
159
- set_dir(dir: str):
160
- Sets the directory for saving log files.
161
- """
162
-
163
- def __init__(self, dir= None, log: list = None) -> None:
164
- """
165
- Initializes a new instance of the DataLogger class.
166
-
167
- Parameters:
168
- dir (str, optional): The directory where the log files will be saved. Defaults to None.
169
-
170
- log (list, optional): An initial list of log entries. Defaults to an empty deque.
171
- """
172
- self.dir = dir
173
- self.log = deque(log) if log else deque()
174
-
175
- def __call__(self, entry):
176
- """
177
- Appends a new entry to the log.
178
-
179
- Parameters:
180
- entry: The entry to append to the log. The entry can be of any datatype.
181
- """
182
- self.log.append(entry)
183
-
184
- def to_csv(self, filename: str, dir=None, verbose: bool = True, timestamp: bool = True, dir_exist_ok=True, file_exist_ok=False):
185
- """
186
- Converts the log to a CSV format and saves it to a file.
187
-
188
- Parameters:
189
- dir (str): The directory where the CSV file will be saved.
190
-
191
- filename (str): The name of the CSV file.
192
-
193
- verbose (bool, optional): If True, prints a message after saving the log. Defaults to True.
194
-
195
- timestamp (bool, optional): If True, appends a timestamp to the filename. Defaults to True.
196
-
197
- dir_exist_ok (bool, optional): If True, overrides the existing directory if needed. Defaults to True.
198
-
199
- file_exist_ok (bool, optional): If True, overrides the existing file if needed. Defaults to False.
200
-
201
- Postconditions:
202
- Saves the log entries to a CSV file and clears the `log` attribute.
203
-
204
- Optionally prints a message with the number of log entries saved and the file path.
205
- """
206
- dir = dir or self.dir
207
- filepath = create_path(dir=dir, filename=filename, timestamp=timestamp, dir_exist_ok=dir_exist_ok)
208
- to_csv(list(self.log), filepath, file_exist_ok=file_exist_ok)
209
- n_logs = len(list(self.log))
210
- self.log = deque()
211
- if verbose:
212
- print(f"{n_logs} logs saved to {filepath}")
213
-
214
- def set_dir(self, dir: str):
215
- """
216
- Sets the directory where log files will be saved.
217
-
218
- Parameters:
219
- dir (str): The directory to set for saving log files.
220
- """
221
- self.dir = dir
222
-
223
-
224
- class BaseTool(BaseNode):
225
- name: str = None
226
- func: Callable = None
227
- content: Any = None
228
- parser: Callable = None
229
-
230
- def initialize(self):
231
- ...
232
-
233
- def execute(self):
234
- ...
235
-
236
- def shutdown(self):
237
- ...
238
-
239
- def __enter__(self):
240
- self.initialize()
241
- return self
242
-
243
-
244
-
245
- # checked ------------------------------------------------------
246
- class Structure:
247
- def __init__(self):
248
- self.graph = nx.DiGraph()
249
-
250
- def add_node(self, node: BaseNode) -> None:
251
- self.graph.add_node(node.id_, content=node.content, metadata=node.metadata)
252
-
253
- def add_relationship(self, source_node_id: str, target_node_id: str, relationship: ConditionalRelationship) -> None:
254
- self.graph.add_edge(source_node_id, target_node_id,
255
- label=relationship.label,
256
- properties=relationship.properties,
257
- condition=relationship.condition)
258
-
259
- def remove_relationship(self, source_node_id: str, target_node_id: str) -> None:
260
- if self.graph.has_edge(source_node_id, target_node_id):
261
- self.graph.remove_edge(source_node_id, target_node_id)
262
-
263
- def get_relationships(self, node_id: str) -> list:
264
- return list(self.graph.edges(node_id, data=True))
265
-
266
- def get_conditional_relationships(self, node_id: str) -> list:
267
- return [(target_id, data) for _, target_id, data in self.graph.edges(node_id, data=True) if "condition" in data]
268
-
269
- def to_json(self) -> str:
270
- return json.dumps(nx.node_link_data(self.graph))
271
-
272
- def from_json(self, data: str) -> None:
273
- graph_data = json.loads(data)
274
- self.graph = nx.node_link_graph(graph_data)
275
-
@@ -1,6 +0,0 @@
1
- # from .service_utils import StatusTracker, AsyncQueue
2
-
3
- # __all__ = [
4
- # "AsyncQueue",
5
- # "StatusTracker"
6
- # ]
@@ -1,48 +0,0 @@
1
- import re
2
- from typing import Generator
3
- from .service_utils import BaseService
4
-
5
-
6
-
7
- class BaseAPIService(BaseService):
8
-
9
- def __init__(self, api_key: str = None,
10
- status_tracker = None,
11
- queue = None, endpoint=None, schema=None,
12
- ratelimiter=None, max_requests_per_minute=None, max_tokens_per_minute=None) -> None:
13
- self.api_key = api_key
14
- self.status_tracker = status_tracker
15
- self.queue = queue
16
- self.endpoint=endpoint
17
- self.schema = schema
18
- self.rate_limiter = ratelimiter(max_requests_per_minute, max_tokens_per_minute)
19
-
20
- @staticmethod
21
- def api_methods(http_session, method="post"):
22
- if method not in ["post", "delete", "head", "options", "patch"]:
23
- raise ValueError("Invalid request, method must be in ['post', 'delete', 'head', 'options', 'patch']")
24
- elif method == "post":
25
- return http_session.post
26
- elif method == "delete":
27
- return http_session.delete
28
- elif method == "head":
29
- return http_session.head
30
- elif method == "options":
31
- return http_session.options
32
- elif method == "patch":
33
- return http_session.patch
34
-
35
- @staticmethod
36
- def api_endpoint_from_url(request_url: str) -> str:
37
- match = re.search(r"^https://[^/]+/v\d+/(.+)$", request_url)
38
- if match:
39
- return match.group(1)
40
- else:
41
- return ""
42
-
43
- @staticmethod
44
- def task_id_generator_function() -> Generator[int, None, None]:
45
- task_id = 0
46
- while True:
47
- yield task_id
48
- task_id += 1
@@ -1 +0,0 @@
1
- # TODO
lionagi/services.py DELETED
@@ -1 +0,0 @@
1
- from .service_.oai import OpenAIService