lionagi 0.0.112__py3-none-any.whl → 0.0.113__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. lionagi/__init__.py +3 -3
  2. lionagi/bridge/__init__.py +7 -0
  3. lionagi/bridge/langchain.py +131 -0
  4. lionagi/bridge/llama_index.py +157 -0
  5. lionagi/configs/__init__.py +7 -0
  6. lionagi/configs/oai_configs.py +49 -0
  7. lionagi/configs/openrouter_config.py +49 -0
  8. lionagi/core/__init__.py +8 -2
  9. lionagi/core/instruction_sets.py +1 -3
  10. lionagi/core/messages.py +2 -2
  11. lionagi/core/sessions.py +174 -27
  12. lionagi/datastore/__init__.py +1 -0
  13. lionagi/loader/__init__.py +9 -4
  14. lionagi/loader/chunker.py +157 -0
  15. lionagi/loader/reader.py +124 -0
  16. lionagi/objs/__init__.py +7 -0
  17. lionagi/objs/messenger.py +163 -0
  18. lionagi/objs/tool_registry.py +247 -0
  19. lionagi/schema/__init__.py +11 -0
  20. lionagi/schema/base_schema.py +239 -0
  21. lionagi/schema/base_tool.py +9 -0
  22. lionagi/schema/data_logger.py +94 -0
  23. lionagi/services/__init__.py +14 -0
  24. lionagi/{service_/oai.py → services/base_api_service.py} +49 -82
  25. lionagi/{endpoint/base_endpoint.py → services/chatcompletion.py} +19 -22
  26. lionagi/services/oai.py +34 -0
  27. lionagi/services/openrouter.py +32 -0
  28. lionagi/{service_/service_utils.py → services/service_objs.py} +0 -1
  29. lionagi/structure/__init__.py +7 -0
  30. lionagi/structure/relationship.py +128 -0
  31. lionagi/structure/structure.py +160 -0
  32. lionagi/tests/test_flatten_util.py +426 -0
  33. lionagi/tools/__init__.py +0 -5
  34. lionagi/tools/coder.py +1 -0
  35. lionagi/tools/scorer.py +1 -0
  36. lionagi/tools/validator.py +1 -0
  37. lionagi/utils/__init__.py +46 -20
  38. lionagi/utils/api_util.py +86 -0
  39. lionagi/utils/call_util.py +347 -0
  40. lionagi/utils/flat_util.py +540 -0
  41. lionagi/utils/io_util.py +102 -0
  42. lionagi/utils/load_utils.py +190 -0
  43. lionagi/utils/sys_util.py +191 -0
  44. lionagi/utils/tool_util.py +92 -0
  45. lionagi/utils/type_util.py +81 -0
  46. lionagi/version.py +1 -1
  47. {lionagi-0.0.112.dist-info → lionagi-0.0.113.dist-info}/METADATA +37 -13
  48. lionagi-0.0.113.dist-info/RECORD +84 -0
  49. lionagi/endpoint/chat_completion.py +0 -20
  50. lionagi/endpoint/endpoint_utils.py +0 -0
  51. lionagi/llm_configs.py +0 -21
  52. lionagi/loader/load_utils.py +0 -161
  53. lionagi/schema.py +0 -275
  54. lionagi/service_/__init__.py +0 -6
  55. lionagi/service_/base_service.py +0 -48
  56. lionagi/service_/openrouter.py +0 -1
  57. lionagi/services.py +0 -1
  58. lionagi/tools/tool_utils.py +0 -75
  59. lionagi/utils/sys_utils.py +0 -799
  60. lionagi-0.0.112.dist-info/RECORD +0 -67
  61. /lionagi/{core/responses.py → datastore/chroma.py} +0 -0
  62. /lionagi/{endpoint/assistants.py → datastore/deeplake.py} +0 -0
  63. /lionagi/{endpoint/audio.py → datastore/elasticsearch.py} +0 -0
  64. /lionagi/{endpoint/embeddings.py → datastore/lantern.py} +0 -0
  65. /lionagi/{endpoint/files.py → datastore/pinecone.py} +0 -0
  66. /lionagi/{endpoint/fine_tuning.py → datastore/postgres.py} +0 -0
  67. /lionagi/{endpoint/images.py → datastore/qdrant.py} +0 -0
  68. /lionagi/{endpoint/messages.py → schema/base_condition.py} +0 -0
  69. /lionagi/{service_ → services}/anthropic.py +0 -0
  70. /lionagi/{service_ → services}/anyscale.py +0 -0
  71. /lionagi/{service_ → services}/azure.py +0 -0
  72. /lionagi/{service_ → services}/bedrock.py +0 -0
  73. /lionagi/{service_ → services}/everlyai.py +0 -0
  74. /lionagi/{service_ → services}/gemini.py +0 -0
  75. /lionagi/{service_ → services}/gpt4all.py +0 -0
  76. /lionagi/{service_ → services}/huggingface.py +0 -0
  77. /lionagi/{service_ → services}/litellm.py +0 -0
  78. /lionagi/{service_ → services}/localai.py +0 -0
  79. /lionagi/{service_ → services}/mistralai.py +0 -0
  80. /lionagi/{service_ → services}/ollama.py +0 -0
  81. /lionagi/{service_ → services}/openllm.py +0 -0
  82. /lionagi/{service_ → services}/perplexity.py +0 -0
  83. /lionagi/{service_ → services}/predibase.py +0 -0
  84. /lionagi/{service_ → services}/rungpt.py +0 -0
  85. /lionagi/{service_ → services}/vllm.py +0 -0
  86. /lionagi/{service_ → services}/xinference.py +0 -0
  87. /lionagi/{endpoint → tests}/__init__.py +0 -0
  88. /lionagi/{endpoint/models.py → tools/planner.py} +0 -0
  89. /lionagi/{endpoint/moderations.py → tools/prompter.py} +0 -0
  90. /lionagi/{endpoint/runs.py → tools/sandbox.py} +0 -0
  91. /lionagi/{endpoint/threads.py → tools/summarizer.py} +0 -0
  92. {lionagi-0.0.112.dist-info → lionagi-0.0.113.dist-info}/LICENSE +0 -0
  93. {lionagi-0.0.112.dist-info → lionagi-0.0.113.dist-info}/WHEEL +0 -0
  94. {lionagi-0.0.112.dist-info → lionagi-0.0.113.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,84 @@
1
+ lionagi/__init__.py,sha256=gAOIyJ-kItUn2ya8G3zW6pvuzmk60xk7zO_GN66UJAM,888
2
+ lionagi/version.py,sha256=WJPRYfehqmqr60a1h9m6a2fbtJa0dT03ixD6hahuBgE,24
3
+ lionagi/bridge/__init__.py,sha256=I19ztSDDwV1-e-GSRs45bJgjbg454e0vcFv2jwgwQY0,139
4
+ lionagi/bridge/langchain.py,sha256=nJdnv_zHpchQXUlh5ml6OZKGO9Rbaa7B-XbeKB3Ej7Y,4579
5
+ lionagi/bridge/llama_index.py,sha256=sapGIgS3RkDUyrPCdjZizoRdAmktZLHzUbYQBw6nnZw,5510
6
+ lionagi/configs/__init__.py,sha256=88KoeoB8dHhgZdQ8I4rbb7e8-FUd-_gi1dXaA95vJ3Y,141
7
+ lionagi/configs/oai_configs.py,sha256=C89jXb2Xb7qoOXIhtrMlIRVYJqjh7knpu6CAOloj9eo,1199
8
+ lionagi/configs/openrouter_config.py,sha256=qbEQQqQPpzMp3eP5U8kuvGTUYdMgemuuq-BdgOYgYI8,1256
9
+ lionagi/core/__init__.py,sha256=XAw2vF1wuQYKSAxHcBanfYN0k-5qLSggkl3yZgwjSRw,347
10
+ lionagi/core/conversations.py,sha256=Jh4Bvv0SBaW5ZG_D9hyWveQ-PXPDGfvp_V7UxbPsFoA,4084
11
+ lionagi/core/flows.py,sha256=rhPBmdN-JKo7Zcv9hiy6k0b-CjKBHMemkYITM7a9fNo,32
12
+ lionagi/core/instruction_sets.py,sha256=y-fDltfDtji4fCtHn8fJ_M9OtYCfjqeTds7bpzknvhU,51
13
+ lionagi/core/messages.py,sha256=XisuXcJBUoyYcd-6DLcV_NZnsKXzZcFf4Dk_fxTlXc8,6533
14
+ lionagi/core/sessions.py,sha256=sjg7bCORFhpaEgDBTExyhfgGxRAUwkOCft0MmP-BGoU,11161
15
+ lionagi/datastore/__init__.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
16
+ lionagi/datastore/chroma.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
17
+ lionagi/datastore/deeplake.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
18
+ lionagi/datastore/elasticsearch.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
19
+ lionagi/datastore/lantern.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
20
+ lionagi/datastore/pinecone.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
21
+ lionagi/datastore/postgres.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
22
+ lionagi/datastore/qdrant.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
23
+ lionagi/loader/__init__.py,sha256=0FNrggZnSq1ONLZY2f325SF19UfYHaqJeJXLvpu8KWQ,262
24
+ lionagi/loader/chunker.py,sha256=zGdfS-Gybxc6lm_vMIdt-f25kY2CwDFTLvY-arKxQ8c,6476
25
+ lionagi/loader/reader.py,sha256=8kxYlqhwmMpuNTWdYtsNc9g-1Mf264Y8XRu1GkevhTg,4527
26
+ lionagi/objs/__init__.py,sha256=4qxo9ZGKe37fjj_Gt2hgi4QTTIPmDf7mkyUK7euVmsU,137
27
+ lionagi/objs/messenger.py,sha256=k-ueq2LBLTTDgk8u-Vyja8WBFM7dWFInWO97HNyOXaI,6137
28
+ lionagi/objs/tool_registry.py,sha256=cnoaW1TvNhKKerQ4tTUOSHEADmsTdoUW2lT24LcHIx0,9357
29
+ lionagi/schema/__init__.py,sha256=cB7rSYeRXujDwp2f5IMK83-NzdqPfbMhGijPCPXgT_c,210
30
+ lionagi/schema/base_condition.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
31
+ lionagi/schema/base_schema.py,sha256=gx4LDxYnVO9vqp0JdevUSasqrat7Lu3rRURTiv7gGj0,8348
32
+ lionagi/schema/base_tool.py,sha256=sKbCrS0ZZINkr03Fj0lk0Kt0XPiBz9IXwA9A1aDgN88,192
33
+ lionagi/schema/data_logger.py,sha256=nmv0jJDkE-vzz_KyBl1_bHB8KVvhZ6sItOBQcvBkAvE,3417
34
+ lionagi/services/__init__.py,sha256=js4LnJEJv4kkxPPamkOgFkOsrc1yr2dK-z9CLTTsX04,313
35
+ lionagi/services/anthropic.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
36
+ lionagi/services/anyscale.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
37
+ lionagi/services/azure.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
38
+ lionagi/services/base_api_service.py,sha256=vGCnC1GnW8czpseOx8mcDeyiiAKc_dabUInxjytE-A0,9756
39
+ lionagi/services/bedrock.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
40
+ lionagi/services/chatcompletion.py,sha256=9jVkB4JUX-kzvAHsohA1ZO6ZeXdlN9OAe-cVvLCa9-0,1257
41
+ lionagi/services/everlyai.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
42
+ lionagi/services/gemini.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
43
+ lionagi/services/gpt4all.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
44
+ lionagi/services/huggingface.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
45
+ lionagi/services/litellm.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
46
+ lionagi/services/localai.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
47
+ lionagi/services/mistralai.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
48
+ lionagi/services/oai.py,sha256=NcKiqmQ8YpKEkoVv-6a-mvIZVBumWOVpstpVD0oTiUs,1122
49
+ lionagi/services/ollama.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
50
+ lionagi/services/openllm.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
51
+ lionagi/services/openrouter.py,sha256=vjY0ZydLY5XK4_T2cRiAl8_QNysWy26wcfyIP-1hlSw,1110
52
+ lionagi/services/perplexity.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
53
+ lionagi/services/predibase.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
54
+ lionagi/services/rungpt.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
55
+ lionagi/services/service_objs.py,sha256=w1Rs69eAGyEukTMMvwQHDmFkwTMf2pB9DdGQ-xCjTUw,9727
56
+ lionagi/services/vllm.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
57
+ lionagi/services/xinference.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
58
+ lionagi/structure/__init__.py,sha256=wMPekT2vbWwUkJ5aW5o-lzJC9Fzhta6RHDiFPTNUm_0,120
59
+ lionagi/structure/relationship.py,sha256=5urGnomfhBJUwPLpYpk5YCx-zO2yprJDaPQqMsgQM0A,4124
60
+ lionagi/structure/structure.py,sha256=KWVOTFWk_IBMtY57gut_GFUuO7dKWlIrMH7xZ_ys2c4,5952
61
+ lionagi/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
62
+ lionagi/tests/test_flatten_util.py,sha256=-sOAQriGtHK1U0FTd31IpdKJpgMQREpaXp-hf6FAojQ,17362
63
+ lionagi/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
64
+ lionagi/tools/coder.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
65
+ lionagi/tools/planner.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
66
+ lionagi/tools/prompter.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
67
+ lionagi/tools/sandbox.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
68
+ lionagi/tools/scorer.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
69
+ lionagi/tools/summarizer.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
70
+ lionagi/tools/validator.py,sha256=chG3GNX2BBDTWIuSVfZUJ_YF_ZVBSoel2d_AN0OChS0,6
71
+ lionagi/utils/__init__.py,sha256=91eJpmYZLXi_xsFV7yb_uvlRmdhBQWBqa9Bmy18nJYY,1419
72
+ lionagi/utils/api_util.py,sha256=wNYillgmAY9UJ5zP6NLo37levC3JEJGiis7nbPBRYXw,2769
73
+ lionagi/utils/call_util.py,sha256=g4MRQChRGAz_1W35ThnEkyVR0W-neEk3ylgUq9IxbR8,12440
74
+ lionagi/utils/flat_util.py,sha256=I1HJXs5llQ6oOoJudT4B7bp9u9_TLEFT_yPcXoNhrNA,22899
75
+ lionagi/utils/io_util.py,sha256=Yzzo-GACV3QXdEFpdfVrpjxc8YJEHJof7hACiTQ59gU,3515
76
+ lionagi/utils/load_utils.py,sha256=nSDWSXRFHO-tJXGMW-7icOqkeL1baQENf9BDyJZU88o,6667
77
+ lionagi/utils/sys_util.py,sha256=nxv5mFagY0zBdMLhf9LD_SsK5Cd2rz9zMslr9xIwpfM,6305
78
+ lionagi/utils/tool_util.py,sha256=RgTML1ZZbmVGdBV2pMpbDCDaNawf_p4d8bqliaOS898,2914
79
+ lionagi/utils/type_util.py,sha256=eUfsvIr_VRykruE4TEUHssBME8Iug_4U4myY8xFV2bA,3135
80
+ lionagi-0.0.113.dist-info/LICENSE,sha256=TBnSyG8fs_tMRtK805GzA1cIyExleKyzoN_kuVxT9IY,11358
81
+ lionagi-0.0.113.dist-info/METADATA,sha256=-zs_6eEFEiCbvqycmpy9YxKVOOIUdzAlB6IrnrHJadk,18027
82
+ lionagi-0.0.113.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
83
+ lionagi-0.0.113.dist-info/top_level.txt,sha256=szvch_d2jE1Lu9ZIKsl26Ll6BGfYfbOgt5lm-UpFSo4,8
84
+ lionagi-0.0.113.dist-info/RECORD,,
@@ -1,20 +0,0 @@
1
- from .base_endpoint import BaseEndpoint
2
-
3
-
4
- class ChatCompletion(BaseEndpoint):
5
- endpoint: str = "chat/completion"
6
-
7
- @classmethod
8
- def create_payload(scls, messages, llmconfig, schema, **kwargs):
9
- config = {**llmconfig, **kwargs}
10
- payload = {"messages": messages}
11
- for key in schema['required']:
12
- payload.update({key: config[key]})
13
-
14
- for key in schema['optional']:
15
- if bool(config[key]) is True and str(config[key]).lower() != "none":
16
- payload.update({key: config[key]})
17
- return payload
18
-
19
- def process_response(self, session, payload, completion):
20
- ...
File without changes
lionagi/llm_configs.py DELETED
@@ -1,21 +0,0 @@
1
- oai_schema = {
2
- "required" : ["model", "frequency_penalty", "n", "presence_penalty", "response_format", "temperature", "top_p"],
3
- "optional": ["seed", "stop", "stream", "tools", "tool_choice", "user", "max_tokens"]
4
- }
5
-
6
- oai_llmconfig = {
7
- "model": "gpt-4-1106-preview",
8
- "frequency_penalty": 0,
9
- "max_tokens": None,
10
- "n": 1,
11
- "presence_penalty": 0,
12
- "response_format": {"type": "text"},
13
- "seed": None,
14
- "stop": None,
15
- "stream": False,
16
- "temperature": 0.7,
17
- "top_p": 1,
18
- "tools": None,
19
- "tool_choice": "none",
20
- "user": None
21
- }
@@ -1,161 +0,0 @@
1
- import math
2
- from pathlib import Path
3
- from typing import Any, Dict, List, Union, Callable, Optional
4
-
5
- from ..utils.sys_utils import to_list, l_call
6
- from ..schema import DataLogger
7
-
8
- def _dir_to_path(dir, ext, recursive):
9
- tem = '**/*' if recursive else '*'
10
- return list(Path(dir).glob(tem + ext))
11
-
12
- def _split_path(path: Path) -> tuple:
13
- folder_name = path.parent.name
14
- file_name = path.name
15
- return (folder_name, file_name)
16
-
17
- def dir_to_path(dir: str, ext, recursive: bool = False, flat: bool = True):
18
- try:
19
- return to_list(l_call(ext, _dir_to_path, flat=True,
20
- recursive=recursive, dir=dir, ext=ext),
21
- flat=flat)
22
- except:
23
- raise ValueError("Invalid directory or extension, please check the path")
24
-
25
- def read_text(filepath: str, clean: bool = True) -> str:
26
- with open(filepath, 'r') as f:
27
- content = f.read()
28
- if clean:
29
- # Define characters to replace and their replacements
30
- replacements = {'\\': ' ', '\n': ' ', '\t': ' ', ' ': ' ', '\'': ' '}
31
- for old, new in replacements.items():
32
- content = content.replace(old, new)
33
- return content
34
-
35
- def dir_to_files(dir: str,
36
- ext: str,
37
- recursive: bool = False,
38
- reader: Callable = read_text,
39
- clean: bool = True,
40
- to_csv: bool = False,
41
- project: str = 'project',
42
- output_dir: str = 'data/logs/sources/',
43
- filename: Optional[str] = None,
44
- verbose: bool = True,
45
- timestamp: bool = True,
46
- logger: Optional[DataLogger] = None):
47
-
48
- sources = dir_to_path(dir, ext, recursive)
49
-
50
- def _to_dict(path_: Path) -> Dict[str, Union[str, Path]]:
51
- folder, file = _split_path(path_)
52
- content = reader(str(path_), clean=clean)
53
- return {
54
- 'project': project,
55
- 'folder': folder,
56
- 'file': file,
57
- "file_size": len(str(content)),
58
- 'content': content
59
- } if content else None
60
-
61
- logs = to_list(l_call(sources, _to_dict, flat=True), dropna=True)
62
-
63
- if to_csv:
64
- filename = filename or f"{project}_sources.csv"
65
- logger = DataLogger(dir=output_dir, log=logs) if not logger else logger
66
- logger.to_csv(dir=output_dir, filename=filename, verbose=verbose, timestamp=timestamp)
67
-
68
- return logs
69
-
70
- def _chunk_n1(input):
71
- return [input]
72
-
73
- def _chunk_n2(input, chunk_size, overlap_size, threshold):
74
- chunks = []
75
- chunks.append(input[:chunk_size + overlap_size])
76
- if len(input) - chunk_size > threshold:
77
- chunks.append(input[chunk_size - overlap_size:])
78
- else:
79
- return [input]
80
- return chunks
81
-
82
- def _chunk_n3(input, chunk_size, overlap_size, threshold, n_chunks):
83
- chunks = []
84
- chunks.append(input[:chunk_size + overlap_size])
85
- for i in range(1, n_chunks - 1):
86
- start_idx = chunk_size * i - overlap_size
87
- end_idx = chunk_size * (i + 1) + overlap_size
88
- chunks.append(input[start_idx:end_idx])
89
-
90
- if len(input) - chunk_size * (n_chunks - 1) > threshold:
91
- chunks.append(input[chunk_size * (n_chunks - 1) - overlap_size:])
92
- else:
93
- chunks[-1] += input[chunk_size * (n_chunks - 1) + overlap_size:]
94
-
95
- return chunks
96
-
97
- def chunk_text(input: str,
98
- chunk_size: int,
99
- overlap: float,
100
- threshold: int) -> List[Union[str, None]]:
101
- try:
102
- if not isinstance(input, str): input = str(input)
103
-
104
- n_chunks = math.ceil(len(input) / chunk_size)
105
- overlap_size = int(chunk_size * overlap / 2)
106
-
107
- if n_chunks == 1: return _chunk_n1(input)
108
-
109
- elif n_chunks == 2:
110
- return _chunk_n2(input=input, chunk_size=chunk_size, overlap_size=overlap_size,
111
- threshold=threshold)
112
- elif n_chunks > 2:
113
- return _chunk_n3(input=input, chunk_size=chunk_size, overlap_size=overlap_size,
114
- threshold=threshold, n_chunks=n_chunks)
115
-
116
- except Exception as e:
117
- raise ValueError(f"An error occurred while chunking the text. {e}")
118
-
119
- def _file_to_chunks(input: Dict[str, Any],
120
- field: str = 'content',
121
- chunk_size: int = 1500,
122
- overlap: float = 0.1,
123
- threshold: int = 200) -> List[Dict[str, Any]]:
124
- try:
125
- out = {key: value for key, value in input.items() if key != field}
126
- out.update({"chunk_overlap": overlap, "chunk_threshold": threshold})
127
-
128
- chunks = chunk_text(input[field], chunk_size=chunk_size, overlap=overlap, threshold=threshold)
129
- logs = []
130
- for i, chunk in enumerate(chunks):
131
- chunk_dict = out.copy()
132
- chunk_dict.update({
133
- 'file_chunks': len(chunks),
134
- 'chunk_id': i + 1,
135
- 'chunk_size': len(chunk),
136
- f'chunk_{field}': chunk
137
- })
138
- logs.append(chunk_dict)
139
-
140
- return logs
141
-
142
- except Exception as e:
143
- raise ValueError(f"An error occurred while chunking the file. {e}")
144
-
145
- def file_to_chunks(input,
146
- project='project',
147
- output_dir='data/logs/sources/',
148
- chunk_func = _file_to_chunks,
149
- to_csv=False,
150
- filename=None,
151
- verbose=True,
152
- timestamp=True,
153
- logger=None, **kwargs):
154
- logs = to_list(l_call(input, chunk_func, **kwargs), flat=True)
155
-
156
- if to_csv:
157
- filename = filename if filename else f"{project}_sources.csv"
158
- logger = logger or DataLogger(log=logs)
159
- logger.to_csv(dir=output_dir, filename=filename, verbose=verbose, timestamp=timestamp)
160
-
161
- return logs
lionagi/schema.py DELETED
@@ -1,275 +0,0 @@
1
- import json
2
- import networkx as nx
3
- from collections import deque
4
- from typing import Any, Dict, Optional, Union, Callable
5
-
6
- from pydantic import BaseModel, Field
7
- from .utils.sys_utils import create_id, create_path, to_csv
8
-
9
-
10
- # checked --------------------------------------------------------
11
- class BaseNode(BaseModel):
12
- id_: str = Field(default_factory=lambda: str(create_id()), alias="node_id")
13
- content: Union[str, Dict[str, Any], None, Any] = None
14
- metadata: Union[Dict[str, Any], None] = Field(default_factory=dict)
15
- label: Optional[str] = None
16
-
17
- @classmethod
18
- def class_name(cls) -> str:
19
- return cls.__name__
20
-
21
- # to some structure
22
- def to_json(self) -> str:
23
- return json.dumps(self.model_dump(by_alias=True))
24
-
25
- def to_dict(self) -> Dict[str, Any]:
26
- return self.model_dump(by_alias=True)
27
-
28
- # from some structure
29
- @classmethod
30
- def from_json(cls, json_str: str) -> "BaseNode":
31
- data = json.loads(json_str)
32
- return cls(**data)
33
-
34
- @classmethod
35
- def from_dict(cls, data: Dict[str, Any]) -> "BaseNode":
36
- return cls(**data)
37
-
38
- def add_meta(self, replace=True, **kwags) -> None:
39
- if replace:
40
- self.metadata.update(**kwags)
41
- else:
42
- for k, v in kwags.items():
43
- if k in self.metadata.keys():
44
- raise ValueError(f"Key already existed")
45
- if k not in self.metadata.keys():
46
- self.metadata.update({k:v})
47
-
48
- def set_meta(self, metadata_: dict) -> None:
49
- self.metadata = metadata_
50
-
51
- def set_content(self, content: Union[str, Dict[str, Any], None, Any]):
52
- self.content = content
53
-
54
- def set_id(self, id: str):
55
- self.id_ = id
56
-
57
- def get_meta(self):
58
- return self.metadata
59
-
60
- def get_content(self):
61
- return self.content
62
-
63
- def get_id(self):
64
- return self.id_
65
-
66
- class ConditionalRelationship(BaseNode):
67
- target_node_id: str
68
- condition: Optional[Dict] = None
69
-
70
- def condition_existed(self, condition_key):
71
- if self.condition:
72
- try:
73
- self.condition.get(condition_key)
74
- return True
75
- except:
76
- return False
77
- return False
78
-
79
- def get_condition(self, condition_key) -> bool:
80
- if self.condition_existed(condition_key=condition_key):
81
- a = self.condition.get(condition_key)
82
- if a is not None and str(a).strip().lower() != 'none':
83
- return a
84
- else:
85
- raise ValueError(f"Condition {condition_key} has no value")
86
-
87
-
88
- class DataNode(BaseNode):
89
-
90
- ...
91
- # def from_llama(self, data_:, **kwargs):
92
- # ...
93
-
94
- # def to_llama(self, **kwargs):
95
- # # to llama_index textnode
96
- # ...
97
-
98
- # def from_langchain(self, data_, **kwargs):
99
- # ...
100
-
101
- # def to_langchain(self, **kwargs):
102
- # ...
103
-
104
- # def to_csv(self, **kwargs):
105
- # ...
106
-
107
- # def __call__(self, file_=None):
108
- # ...
109
-
110
-
111
- class MessageNode(BaseNode):
112
- role: str
113
- name: str
114
-
115
- # def from_oai(self):
116
- # ...
117
-
118
-
119
- class File(DataNode):
120
- ...
121
-
122
- # def from_path(self, path_, reader, clean=True, **kwargs):
123
- # self.content = reader(path_=path_, clean=clean, **kwargs)
124
-
125
- # def to_chunks(self, chunker, **kwargs):
126
- # ...
127
-
128
-
129
- class Chunk(DataNode):
130
- ...
131
- # @classmethod
132
- # def from_files(cls, files):
133
- # ...
134
-
135
- # @classmethod
136
- # def to_files(cls):
137
- # ...
138
-
139
-
140
- class DataLogger:
141
- """
142
- Logs data entries and outputs them to a CSV file.
143
-
144
- This class provides a logging mechanism for data entries that can be saved to a
145
- CSV file. It offers methods for appending new log entries, saving the log to a CSV file,
146
- and setting the directory where the logs should be saved.
147
-
148
- Attributes:
149
- dir (str):
150
- The directory where the log files are to be saved.
151
- log (deque):
152
- A deque that stores log entries.
153
-
154
- Methods:
155
- __call__(entry):
156
- Appends a new entry to the log.
157
- to_csv(dir: str, filename: str, verbose: bool, timestamp: bool, dir_exist_ok: bool, file_exist_ok: bool):
158
- Converts the log to a CSV format and saves it to a file.
159
- set_dir(dir: str):
160
- Sets the directory for saving log files.
161
- """
162
-
163
- def __init__(self, dir= None, log: list = None) -> None:
164
- """
165
- Initializes a new instance of the DataLogger class.
166
-
167
- Parameters:
168
- dir (str, optional): The directory where the log files will be saved. Defaults to None.
169
-
170
- log (list, optional): An initial list of log entries. Defaults to an empty deque.
171
- """
172
- self.dir = dir
173
- self.log = deque(log) if log else deque()
174
-
175
- def __call__(self, entry):
176
- """
177
- Appends a new entry to the log.
178
-
179
- Parameters:
180
- entry: The entry to append to the log. The entry can be of any datatype.
181
- """
182
- self.log.append(entry)
183
-
184
- def to_csv(self, filename: str, dir=None, verbose: bool = True, timestamp: bool = True, dir_exist_ok=True, file_exist_ok=False):
185
- """
186
- Converts the log to a CSV format and saves it to a file.
187
-
188
- Parameters:
189
- dir (str): The directory where the CSV file will be saved.
190
-
191
- filename (str): The name of the CSV file.
192
-
193
- verbose (bool, optional): If True, prints a message after saving the log. Defaults to True.
194
-
195
- timestamp (bool, optional): If True, appends a timestamp to the filename. Defaults to True.
196
-
197
- dir_exist_ok (bool, optional): If True, overrides the existing directory if needed. Defaults to True.
198
-
199
- file_exist_ok (bool, optional): If True, overrides the existing file if needed. Defaults to False.
200
-
201
- Postconditions:
202
- Saves the log entries to a CSV file and clears the `log` attribute.
203
-
204
- Optionally prints a message with the number of log entries saved and the file path.
205
- """
206
- dir = dir or self.dir
207
- filepath = create_path(dir=dir, filename=filename, timestamp=timestamp, dir_exist_ok=dir_exist_ok)
208
- to_csv(list(self.log), filepath, file_exist_ok=file_exist_ok)
209
- n_logs = len(list(self.log))
210
- self.log = deque()
211
- if verbose:
212
- print(f"{n_logs} logs saved to {filepath}")
213
-
214
- def set_dir(self, dir: str):
215
- """
216
- Sets the directory where log files will be saved.
217
-
218
- Parameters:
219
- dir (str): The directory to set for saving log files.
220
- """
221
- self.dir = dir
222
-
223
-
224
- class BaseTool(BaseNode):
225
- name: str = None
226
- func: Callable = None
227
- content: Any = None
228
- parser: Callable = None
229
-
230
- def initialize(self):
231
- ...
232
-
233
- def execute(self):
234
- ...
235
-
236
- def shutdown(self):
237
- ...
238
-
239
- def __enter__(self):
240
- self.initialize()
241
- return self
242
-
243
-
244
-
245
- # checked ------------------------------------------------------
246
- class Structure:
247
- def __init__(self):
248
- self.graph = nx.DiGraph()
249
-
250
- def add_node(self, node: BaseNode) -> None:
251
- self.graph.add_node(node.id_, content=node.content, metadata=node.metadata)
252
-
253
- def add_relationship(self, source_node_id: str, target_node_id: str, relationship: ConditionalRelationship) -> None:
254
- self.graph.add_edge(source_node_id, target_node_id,
255
- label=relationship.label,
256
- properties=relationship.properties,
257
- condition=relationship.condition)
258
-
259
- def remove_relationship(self, source_node_id: str, target_node_id: str) -> None:
260
- if self.graph.has_edge(source_node_id, target_node_id):
261
- self.graph.remove_edge(source_node_id, target_node_id)
262
-
263
- def get_relationships(self, node_id: str) -> list:
264
- return list(self.graph.edges(node_id, data=True))
265
-
266
- def get_conditional_relationships(self, node_id: str) -> list:
267
- return [(target_id, data) for _, target_id, data in self.graph.edges(node_id, data=True) if "condition" in data]
268
-
269
- def to_json(self) -> str:
270
- return json.dumps(nx.node_link_data(self.graph))
271
-
272
- def from_json(self, data: str) -> None:
273
- graph_data = json.loads(data)
274
- self.graph = nx.node_link_graph(graph_data)
275
-
@@ -1,6 +0,0 @@
1
- # from .service_utils import StatusTracker, AsyncQueue
2
-
3
- # __all__ = [
4
- # "AsyncQueue",
5
- # "StatusTracker"
6
- # ]
@@ -1,48 +0,0 @@
1
- import re
2
- from typing import Generator
3
- from .service_utils import BaseService
4
-
5
-
6
-
7
- class BaseAPIService(BaseService):
8
-
9
- def __init__(self, api_key: str = None,
10
- status_tracker = None,
11
- queue = None, endpoint=None, schema=None,
12
- ratelimiter=None, max_requests_per_minute=None, max_tokens_per_minute=None) -> None:
13
- self.api_key = api_key
14
- self.status_tracker = status_tracker
15
- self.queue = queue
16
- self.endpoint=endpoint
17
- self.schema = schema
18
- self.rate_limiter = ratelimiter(max_requests_per_minute, max_tokens_per_minute)
19
-
20
- @staticmethod
21
- def api_methods(http_session, method="post"):
22
- if method not in ["post", "delete", "head", "options", "patch"]:
23
- raise ValueError("Invalid request, method must be in ['post', 'delete', 'head', 'options', 'patch']")
24
- elif method == "post":
25
- return http_session.post
26
- elif method == "delete":
27
- return http_session.delete
28
- elif method == "head":
29
- return http_session.head
30
- elif method == "options":
31
- return http_session.options
32
- elif method == "patch":
33
- return http_session.patch
34
-
35
- @staticmethod
36
- def api_endpoint_from_url(request_url: str) -> str:
37
- match = re.search(r"^https://[^/]+/v\d+/(.+)$", request_url)
38
- if match:
39
- return match.group(1)
40
- else:
41
- return ""
42
-
43
- @staticmethod
44
- def task_id_generator_function() -> Generator[int, None, None]:
45
- task_id = 0
46
- while True:
47
- yield task_id
48
- task_id += 1
@@ -1 +0,0 @@
1
- # TODO
lionagi/services.py DELETED
@@ -1 +0,0 @@
1
- from .service_.oai import OpenAIService