lionagi 0.1.2__py3-none-any.whl → 0.2.1__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (268) hide show
  1. lionagi/__init__.py +60 -5
  2. lionagi/core/__init__.py +0 -25
  3. lionagi/core/_setting/_setting.py +59 -0
  4. lionagi/core/action/__init__.py +14 -0
  5. lionagi/core/action/function_calling.py +136 -0
  6. lionagi/core/action/manual.py +1 -0
  7. lionagi/core/action/node.py +109 -0
  8. lionagi/core/action/tool.py +114 -0
  9. lionagi/core/action/tool_manager.py +356 -0
  10. lionagi/core/agent/base_agent.py +27 -13
  11. lionagi/core/agent/eval/evaluator.py +1 -0
  12. lionagi/core/agent/eval/vote.py +40 -0
  13. lionagi/core/agent/learn/learner.py +59 -0
  14. lionagi/core/agent/plan/unit_template.py +1 -0
  15. lionagi/core/collections/__init__.py +17 -0
  16. lionagi/core/{generic/data_logger.py → collections/_logger.py} +69 -55
  17. lionagi/core/collections/abc/__init__.py +53 -0
  18. lionagi/core/collections/abc/component.py +615 -0
  19. lionagi/core/collections/abc/concepts.py +297 -0
  20. lionagi/core/collections/abc/exceptions.py +150 -0
  21. lionagi/core/collections/abc/util.py +45 -0
  22. lionagi/core/collections/exchange.py +161 -0
  23. lionagi/core/collections/flow.py +426 -0
  24. lionagi/core/collections/model.py +419 -0
  25. lionagi/core/collections/pile.py +913 -0
  26. lionagi/core/collections/progression.py +236 -0
  27. lionagi/core/collections/util.py +64 -0
  28. lionagi/core/director/direct.py +314 -0
  29. lionagi/core/director/director.py +2 -0
  30. lionagi/core/{execute/branch_executor.py → engine/branch_engine.py} +134 -97
  31. lionagi/core/{execute/instruction_map_executor.py → engine/instruction_map_engine.py} +80 -55
  32. lionagi/{experimental/directive/evaluator → core/engine}/script_engine.py +17 -1
  33. lionagi/core/executor/base_executor.py +90 -0
  34. lionagi/core/{execute/structure_executor.py → executor/graph_executor.py} +62 -66
  35. lionagi/core/{execute → executor}/neo4j_executor.py +70 -67
  36. lionagi/core/generic/__init__.py +3 -33
  37. lionagi/core/generic/edge.py +29 -79
  38. lionagi/core/generic/edge_condition.py +16 -0
  39. lionagi/core/generic/graph.py +236 -0
  40. lionagi/core/generic/hyperedge.py +1 -0
  41. lionagi/core/generic/node.py +156 -221
  42. lionagi/core/generic/tree.py +48 -0
  43. lionagi/core/generic/tree_node.py +79 -0
  44. lionagi/core/mail/__init__.py +12 -0
  45. lionagi/core/mail/mail.py +25 -0
  46. lionagi/core/mail/mail_manager.py +139 -58
  47. lionagi/core/mail/package.py +45 -0
  48. lionagi/core/mail/start_mail.py +36 -0
  49. lionagi/core/message/__init__.py +19 -0
  50. lionagi/core/message/action_request.py +133 -0
  51. lionagi/core/message/action_response.py +135 -0
  52. lionagi/core/message/assistant_response.py +95 -0
  53. lionagi/core/message/instruction.py +234 -0
  54. lionagi/core/message/message.py +101 -0
  55. lionagi/core/message/system.py +86 -0
  56. lionagi/core/message/util.py +283 -0
  57. lionagi/core/report/__init__.py +4 -0
  58. lionagi/core/report/base.py +217 -0
  59. lionagi/core/report/form.py +231 -0
  60. lionagi/core/report/report.py +166 -0
  61. lionagi/core/report/util.py +28 -0
  62. lionagi/core/rule/_default.py +16 -0
  63. lionagi/core/rule/action.py +99 -0
  64. lionagi/core/rule/base.py +238 -0
  65. lionagi/core/rule/boolean.py +56 -0
  66. lionagi/core/rule/choice.py +47 -0
  67. lionagi/core/rule/mapping.py +96 -0
  68. lionagi/core/rule/number.py +71 -0
  69. lionagi/core/rule/rulebook.py +109 -0
  70. lionagi/core/rule/string.py +52 -0
  71. lionagi/core/rule/util.py +35 -0
  72. lionagi/core/session/branch.py +431 -0
  73. lionagi/core/session/directive_mixin.py +287 -0
  74. lionagi/core/session/session.py +229 -903
  75. lionagi/core/structure/__init__.py +1 -0
  76. lionagi/core/structure/chain.py +1 -0
  77. lionagi/core/structure/forest.py +1 -0
  78. lionagi/core/structure/graph.py +1 -0
  79. lionagi/core/structure/tree.py +1 -0
  80. lionagi/core/unit/__init__.py +5 -0
  81. lionagi/core/unit/parallel_unit.py +245 -0
  82. lionagi/core/unit/template/action.py +81 -0
  83. lionagi/core/unit/template/base.py +51 -0
  84. lionagi/core/unit/template/plan.py +84 -0
  85. lionagi/core/unit/template/predict.py +109 -0
  86. lionagi/core/unit/template/score.py +124 -0
  87. lionagi/core/unit/template/select.py +104 -0
  88. lionagi/core/unit/unit.py +362 -0
  89. lionagi/core/unit/unit_form.py +305 -0
  90. lionagi/core/unit/unit_mixin.py +1168 -0
  91. lionagi/core/unit/util.py +71 -0
  92. lionagi/core/validator/validator.py +364 -0
  93. lionagi/core/work/work.py +76 -0
  94. lionagi/core/work/work_function.py +101 -0
  95. lionagi/core/work/work_queue.py +103 -0
  96. lionagi/core/work/worker.py +258 -0
  97. lionagi/core/work/worklog.py +120 -0
  98. lionagi/experimental/compressor/base.py +46 -0
  99. lionagi/experimental/compressor/llm_compressor.py +247 -0
  100. lionagi/experimental/compressor/llm_summarizer.py +61 -0
  101. lionagi/experimental/compressor/util.py +70 -0
  102. lionagi/experimental/directive/__init__.py +19 -0
  103. lionagi/experimental/directive/parser/base_parser.py +69 -2
  104. lionagi/experimental/directive/{template_ → template}/base_template.py +17 -1
  105. lionagi/{libs/ln_tokenizer.py → experimental/directive/tokenizer.py} +16 -0
  106. lionagi/experimental/{directive/evaluator → evaluator}/ast_evaluator.py +16 -0
  107. lionagi/experimental/{directive/evaluator → evaluator}/base_evaluator.py +16 -0
  108. lionagi/experimental/knowledge/base.py +10 -0
  109. lionagi/experimental/memory/__init__.py +0 -0
  110. lionagi/experimental/strategies/__init__.py +0 -0
  111. lionagi/experimental/strategies/base.py +1 -0
  112. lionagi/integrations/bridge/langchain_/documents.py +4 -0
  113. lionagi/integrations/bridge/llamaindex_/index.py +30 -0
  114. lionagi/integrations/bridge/llamaindex_/llama_index_bridge.py +6 -0
  115. lionagi/integrations/chunker/chunk.py +161 -24
  116. lionagi/integrations/config/oai_configs.py +34 -3
  117. lionagi/integrations/config/openrouter_configs.py +14 -2
  118. lionagi/integrations/loader/load.py +122 -21
  119. lionagi/integrations/loader/load_util.py +6 -77
  120. lionagi/integrations/provider/_mapping.py +46 -0
  121. lionagi/integrations/provider/litellm.py +2 -1
  122. lionagi/integrations/provider/mlx_service.py +16 -9
  123. lionagi/integrations/provider/oai.py +91 -4
  124. lionagi/integrations/provider/ollama.py +6 -5
  125. lionagi/integrations/provider/openrouter.py +115 -8
  126. lionagi/integrations/provider/services.py +2 -2
  127. lionagi/integrations/provider/transformers.py +18 -22
  128. lionagi/integrations/storage/__init__.py +3 -3
  129. lionagi/integrations/storage/neo4j.py +52 -60
  130. lionagi/integrations/storage/storage_util.py +44 -46
  131. lionagi/integrations/storage/structure_excel.py +43 -26
  132. lionagi/integrations/storage/to_excel.py +11 -4
  133. lionagi/libs/__init__.py +22 -1
  134. lionagi/libs/ln_api.py +75 -20
  135. lionagi/libs/ln_context.py +37 -0
  136. lionagi/libs/ln_convert.py +21 -9
  137. lionagi/libs/ln_func_call.py +69 -28
  138. lionagi/libs/ln_image.py +107 -0
  139. lionagi/libs/ln_nested.py +26 -11
  140. lionagi/libs/ln_parse.py +82 -23
  141. lionagi/libs/ln_queue.py +16 -0
  142. lionagi/libs/ln_tokenize.py +164 -0
  143. lionagi/libs/ln_validate.py +16 -0
  144. lionagi/libs/special_tokens.py +172 -0
  145. lionagi/libs/sys_util.py +95 -24
  146. lionagi/lions/coder/code_form.py +13 -0
  147. lionagi/lions/coder/coder.py +50 -3
  148. lionagi/lions/coder/util.py +30 -25
  149. lionagi/tests/libs/test_func_call.py +23 -21
  150. lionagi/tests/libs/test_nested.py +36 -21
  151. lionagi/tests/libs/test_parse.py +1 -1
  152. lionagi/tests/test_core/collections/__init__.py +0 -0
  153. lionagi/tests/test_core/collections/test_component.py +206 -0
  154. lionagi/tests/test_core/collections/test_exchange.py +138 -0
  155. lionagi/tests/test_core/collections/test_flow.py +145 -0
  156. lionagi/tests/test_core/collections/test_pile.py +171 -0
  157. lionagi/tests/test_core/collections/test_progression.py +129 -0
  158. lionagi/tests/test_core/generic/test_edge.py +67 -0
  159. lionagi/tests/test_core/generic/test_graph.py +96 -0
  160. lionagi/tests/test_core/generic/test_node.py +106 -0
  161. lionagi/tests/test_core/generic/test_tree_node.py +73 -0
  162. lionagi/tests/test_core/test_branch.py +115 -294
  163. lionagi/tests/test_core/test_form.py +46 -0
  164. lionagi/tests/test_core/test_report.py +105 -0
  165. lionagi/tests/test_core/test_validator.py +111 -0
  166. lionagi/version.py +1 -1
  167. lionagi-0.2.1.dist-info/LICENSE +202 -0
  168. lionagi-0.2.1.dist-info/METADATA +272 -0
  169. lionagi-0.2.1.dist-info/RECORD +240 -0
  170. lionagi/core/branch/base.py +0 -653
  171. lionagi/core/branch/branch.py +0 -474
  172. lionagi/core/branch/flow_mixin.py +0 -96
  173. lionagi/core/branch/util.py +0 -323
  174. lionagi/core/direct/__init__.py +0 -19
  175. lionagi/core/direct/cot.py +0 -123
  176. lionagi/core/direct/plan.py +0 -164
  177. lionagi/core/direct/predict.py +0 -166
  178. lionagi/core/direct/react.py +0 -171
  179. lionagi/core/direct/score.py +0 -279
  180. lionagi/core/direct/select.py +0 -170
  181. lionagi/core/direct/sentiment.py +0 -1
  182. lionagi/core/direct/utils.py +0 -110
  183. lionagi/core/direct/vote.py +0 -64
  184. lionagi/core/execute/base_executor.py +0 -47
  185. lionagi/core/flow/baseflow.py +0 -23
  186. lionagi/core/flow/monoflow/ReAct.py +0 -240
  187. lionagi/core/flow/monoflow/__init__.py +0 -9
  188. lionagi/core/flow/monoflow/chat.py +0 -95
  189. lionagi/core/flow/monoflow/chat_mixin.py +0 -253
  190. lionagi/core/flow/monoflow/followup.py +0 -215
  191. lionagi/core/flow/polyflow/__init__.py +0 -1
  192. lionagi/core/flow/polyflow/chat.py +0 -251
  193. lionagi/core/form/action_form.py +0 -26
  194. lionagi/core/form/field_validator.py +0 -287
  195. lionagi/core/form/form.py +0 -302
  196. lionagi/core/form/mixin.py +0 -214
  197. lionagi/core/form/scored_form.py +0 -13
  198. lionagi/core/generic/action.py +0 -26
  199. lionagi/core/generic/component.py +0 -532
  200. lionagi/core/generic/condition.py +0 -46
  201. lionagi/core/generic/mail.py +0 -90
  202. lionagi/core/generic/mailbox.py +0 -36
  203. lionagi/core/generic/relation.py +0 -70
  204. lionagi/core/generic/signal.py +0 -22
  205. lionagi/core/generic/structure.py +0 -362
  206. lionagi/core/generic/transfer.py +0 -20
  207. lionagi/core/generic/work.py +0 -40
  208. lionagi/core/graph/graph.py +0 -126
  209. lionagi/core/graph/tree.py +0 -190
  210. lionagi/core/mail/schema.py +0 -63
  211. lionagi/core/messages/schema.py +0 -325
  212. lionagi/core/tool/__init__.py +0 -5
  213. lionagi/core/tool/tool.py +0 -28
  214. lionagi/core/tool/tool_manager.py +0 -283
  215. lionagi/experimental/report/form.py +0 -64
  216. lionagi/experimental/report/report.py +0 -138
  217. lionagi/experimental/report/util.py +0 -47
  218. lionagi/experimental/tool/function_calling.py +0 -43
  219. lionagi/experimental/tool/manual.py +0 -66
  220. lionagi/experimental/tool/schema.py +0 -59
  221. lionagi/experimental/tool/tool_manager.py +0 -138
  222. lionagi/experimental/tool/util.py +0 -16
  223. lionagi/experimental/validator/rule.py +0 -139
  224. lionagi/experimental/validator/validator.py +0 -56
  225. lionagi/experimental/work/__init__.py +0 -10
  226. lionagi/experimental/work/async_queue.py +0 -54
  227. lionagi/experimental/work/schema.py +0 -73
  228. lionagi/experimental/work/work_function.py +0 -67
  229. lionagi/experimental/work/worker.py +0 -56
  230. lionagi/experimental/work2/form.py +0 -371
  231. lionagi/experimental/work2/report.py +0 -289
  232. lionagi/experimental/work2/schema.py +0 -30
  233. lionagi/experimental/work2/tests.py +0 -72
  234. lionagi/experimental/work2/work_function.py +0 -89
  235. lionagi/experimental/work2/worker.py +0 -12
  236. lionagi/integrations/bridge/llamaindex_/get_index.py +0 -294
  237. lionagi/tests/test_core/generic/test_component.py +0 -89
  238. lionagi/tests/test_core/test_base_branch.py +0 -426
  239. lionagi/tests/test_core/test_chat_flow.py +0 -63
  240. lionagi/tests/test_core/test_mail_manager.py +0 -75
  241. lionagi/tests/test_core/test_prompts.py +0 -51
  242. lionagi/tests/test_core/test_session.py +0 -254
  243. lionagi/tests/test_core/test_session_base_util.py +0 -313
  244. lionagi/tests/test_core/test_tool_manager.py +0 -95
  245. lionagi-0.1.2.dist-info/LICENSE +0 -9
  246. lionagi-0.1.2.dist-info/METADATA +0 -174
  247. lionagi-0.1.2.dist-info/RECORD +0 -206
  248. /lionagi/core/{branch → _setting}/__init__.py +0 -0
  249. /lionagi/core/{execute → agent/eval}/__init__.py +0 -0
  250. /lionagi/core/{flow → agent/learn}/__init__.py +0 -0
  251. /lionagi/core/{form → agent/plan}/__init__.py +0 -0
  252. /lionagi/core/{branch/executable_branch.py → agent/plan/plan.py} +0 -0
  253. /lionagi/core/{graph → director}/__init__.py +0 -0
  254. /lionagi/core/{messages → engine}/__init__.py +0 -0
  255. /lionagi/{experimental/directive/evaluator → core/engine}/sandbox_.py +0 -0
  256. /lionagi/{experimental/directive/evaluator → core/executor}/__init__.py +0 -0
  257. /lionagi/{experimental/directive/template_ → core/rule}/__init__.py +0 -0
  258. /lionagi/{experimental/report → core/unit/template}/__init__.py +0 -0
  259. /lionagi/{experimental/tool → core/validator}/__init__.py +0 -0
  260. /lionagi/{experimental/validator → core/work}/__init__.py +0 -0
  261. /lionagi/experimental/{work2 → compressor}/__init__.py +0 -0
  262. /lionagi/{core/flow/mono_chat_mixin.py → experimental/directive/template/__init__.py} +0 -0
  263. /lionagi/experimental/directive/{schema.py → template/schema.py} +0 -0
  264. /lionagi/experimental/{work2/util.py → evaluator/__init__.py} +0 -0
  265. /lionagi/experimental/{work2/work.py → knowledge/__init__.py} +0 -0
  266. /lionagi/{tests/libs/test_async.py → experimental/knowledge/graph.py} +0 -0
  267. {lionagi-0.1.2.dist-info → lionagi-0.2.1.dist-info}/WHEEL +0 -0
  268. {lionagi-0.1.2.dist-info → lionagi-0.2.1.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,913 @@
1
+ """
2
+ Copyright 2024 HaiyangLi
3
+
4
+ Licensed under the Apache License, Version 2.0 (the "License");
5
+ you may not use this file except in compliance with the License.
6
+ You may obtain a copy of the License at
7
+
8
+ http://www.apache.org/licenses/LICENSE-2.0
9
+
10
+ Unless required by applicable law or agreed to in writing, software
11
+ distributed under the License is distributed on an "AS IS" BASIS,
12
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ See the License for the specific language governing permissions and
14
+ limitations under the License.
15
+ """
16
+
17
+ """
18
+ This module defines the Pile class, a versatile container for managing
19
+ collections of Element objects. It supports structured access and
20
+ manipulation, including retrieval, addition, and deletion of elements.
21
+ """
22
+
23
+ from collections.abc import Iterable
24
+ from typing import TypeVar, Type, Any, Generic
25
+
26
+ from pydantic import Field, field_validator
27
+
28
+ from lionagi.libs.ln_convert import is_same_dtype, to_df
29
+ from lionagi.libs.ln_func_call import bcall, alcall, CallDecorator as cd
30
+ from .abc import (
31
+ Element,
32
+ Record,
33
+ Component,
34
+ Ordering,
35
+ LionIDable,
36
+ get_lion_id,
37
+ LionValueError,
38
+ LionTypeError,
39
+ ItemNotFoundError,
40
+ ModelLimitExceededError,
41
+ )
42
+ from .model import iModel
43
+ from .util import to_list_type, _validate_order
44
+
45
+ T = TypeVar("T")
46
+
47
+
48
+ class Pile(Element, Record, Generic[T]):
49
+ """
50
+ Collection class for managing Element objects.
51
+
52
+ Facilitates ordered and type-validated storage and access, supporting
53
+ both index-based and key-based retrieval.
54
+
55
+ Attributes:
56
+ pile (dict[str, T]): Maps unique identifiers to items.
57
+ item_type (set[Type[Element]] | None): Allowed item types.
58
+ name (str | None): Optional name for the pile.
59
+ order (list[str]): Order of item identifiers.
60
+ use_obj (bool): If True, treat Record and Ordering as objects.
61
+ """
62
+
63
+ use_obj: bool = False
64
+ pile: dict[str, T] = Field(default_factory=dict)
65
+ item_type: set[Type[Element]] | None = Field(default=None)
66
+ name: str | None = None
67
+ order: list[str] = Field(default_factory=list)
68
+ index: Any = None
69
+ engines: dict[str, Any] = Field(default_factory=dict)
70
+ query_response: list = []
71
+ tools: dict = {}
72
+
73
+ def __init__(
74
+ self,
75
+ items=None,
76
+ item_type=None,
77
+ order=None,
78
+ use_obj=None,
79
+ ):
80
+ super().__init__()
81
+
82
+ self.use_obj = use_obj or False
83
+ self.pile = self._validate_pile(items or {})
84
+ self.item_type = self._validate_item_type(item_type)
85
+
86
+ order = order or list(self.pile.keys())
87
+ if not len(order) == len(self):
88
+ raise ValueError(
89
+ "The length of the order does not match the length of the pile"
90
+ )
91
+ self.order = order
92
+
93
+ def __getitem__(self, key) -> T | "Pile[T]":
94
+ """
95
+ Retrieve items from the pile using a key.
96
+
97
+ Supports multiple types of key access:
98
+ - By index or slice (list-like access)
99
+ - By LionID (dictionary-like access)
100
+ - By other complex types if item is of LionIDable
101
+
102
+ Args:
103
+ key: Key to retrieve items.
104
+
105
+ Returns:
106
+ The requested item(s). Single items returned directly,
107
+ multiple items returned in a new `Pile` instance.
108
+
109
+ Raises:
110
+ ItemNotFoundError: If requested item(s) not found.
111
+ LionTypeError: If provided key is invalid.
112
+ """
113
+ try:
114
+ if isinstance(key, (int, slice)):
115
+ # Handle list-like index or slice
116
+ _key = self.order[key]
117
+ _key = [_key] if isinstance(key, int) else _key
118
+ _out = [self.pile.get(i) for i in _key]
119
+ return _out[0] if len(_out) == 1 else pile(_out, self.item_type, _key)
120
+ except IndexError as e:
121
+ raise ItemNotFoundError(key) from e
122
+
123
+ keys = to_list_type(key)
124
+ for idx, item in enumerate(keys):
125
+ if isinstance(item, str):
126
+ keys[idx] = item
127
+ continue
128
+ if hasattr(item, "ln_id"):
129
+ keys[idx] = item.ln_id
130
+
131
+ if not all(keys):
132
+ raise LionTypeError("Invalid item type. Expected LionIDable object(s).")
133
+
134
+ try:
135
+ if len(keys) == 1:
136
+ return self.pile.get(keys[0])
137
+ return pile([self.pile.get(i) for i in keys], self.item_type, keys)
138
+ except KeyError as e:
139
+ raise ItemNotFoundError(key) from e
140
+
141
+ def __setitem__(self, key, item) -> None:
142
+ """
143
+ Set new values in the pile using various key types.
144
+
145
+ Handles single/multiple assignments, ensures type consistency.
146
+ Supports index/slice, LionID, and LionIDable key access.
147
+
148
+ Args:
149
+ key: Key to set items. Can be index, slice, LionID, LionIDable.
150
+ item: Item(s) to set. Can be single item or collection.
151
+
152
+ Raises:
153
+ ValueError: Length mismatch or multiple items to single key.
154
+ LionTypeError: Item type doesn't match allowed types.
155
+ """
156
+ item = self._validate_pile(item)
157
+
158
+ if isinstance(key, (int, slice)):
159
+ # Handle list-like index or slice
160
+ try:
161
+ _key = self.order[key]
162
+ except IndexError as e:
163
+ raise e
164
+
165
+ if isinstance(_key, str) and len(item) != 1:
166
+ raise ValueError("Cannot assign multiple items to a single item.")
167
+
168
+ if isinstance(_key, list) and len(item) != len(_key):
169
+ raise ValueError(
170
+ "The length of values does not match the length of the slice"
171
+ )
172
+
173
+ for k, v in item.items():
174
+ if self.item_type and type(v) not in self.item_type:
175
+ raise LionTypeError(f"Invalid item type. Expected {self.item_type}")
176
+
177
+ self.pile[k] = v
178
+ self.order[key] = k
179
+ self.pile.pop(_key)
180
+ return
181
+
182
+ if len(to_list_type(key)) != len(item):
183
+ raise ValueError("The length of keys does not match the length of values")
184
+
185
+ self.pile.update(item)
186
+ self.order.extend(item.keys())
187
+
188
+ def __contains__(self, item: Any) -> bool:
189
+ """
190
+ Check if item(s) are present in the pile.
191
+
192
+ Accepts individual items and collections. Returns `True` if all
193
+ provided items are found, `False` otherwise.
194
+
195
+ Args:
196
+ item: Item(s) to check. Can be single item or collection.
197
+
198
+ Returns:
199
+ `True` if all items are found, `False` otherwise.
200
+ """
201
+ item = to_list_type(item)
202
+ for i in item:
203
+ try:
204
+ a = i if isinstance(i, str) else get_lion_id(i)
205
+ if a not in self.pile:
206
+ return False
207
+ except Exception:
208
+ return False
209
+
210
+ return True
211
+
212
+ def pop(self, key: Any, default=...) -> T | "Pile[T]" | None:
213
+ """
214
+ Remove and return item(s) associated with given key.
215
+
216
+ Raises `ItemNotFoundError` if key not found and no default given.
217
+ Returns default if provided and key not found.
218
+
219
+ Args:
220
+ key: Key of item(s) to remove and return. Can be single key
221
+ or collection of keys.
222
+ default: Default value if key not found. If not specified
223
+ and key not found, raises `ItemNotFoundError`.
224
+
225
+ Returns:
226
+ Removed item(s) associated with key. Single items returned
227
+ directly, multiple items in new `Pile`. Returns default if
228
+ provided and key not found.
229
+
230
+ Raises:
231
+ ItemNotFoundError: If key not found and no default specified.
232
+ """
233
+ key = to_list_type(key)
234
+ items = []
235
+
236
+ for i in key:
237
+ if i not in self:
238
+ if default == ...:
239
+ raise ItemNotFoundError(i)
240
+ return default
241
+
242
+ for i in key:
243
+ _id = get_lion_id(i)
244
+ items.append(self.pile.pop(_id))
245
+ self.order.remove(_id)
246
+
247
+ return pile(items) if len(items) > 1 else items[0]
248
+
249
+ def get(self, key: Any, default=...) -> T | "Pile[T]" | None:
250
+ """
251
+ Retrieve item(s) associated with given key.
252
+
253
+ Raises `ItemNotFoundError` if key not found and no default given.
254
+ Returns default if provided and key not found.
255
+
256
+ Args:
257
+ key: Key of item(s) to retrieve. Can be single or collection.
258
+ default: Default value if key not found. If not specified
259
+ and key not found, raises `ItemNotFoundError`.
260
+
261
+ Returns:
262
+ Retrieved item(s) associated with key. Single items returned
263
+ directly, multiple items in new `Pile`. Returns default if
264
+ provided and key not found.
265
+
266
+ Raises:
267
+ ItemNotFoundError: If key not found and no default specified.
268
+ """
269
+ try:
270
+ return self[key]
271
+ except ItemNotFoundError as e:
272
+ if default == ...:
273
+ raise e
274
+ return default
275
+
276
+ def update(self, other: Any):
277
+ """
278
+ Update pile with another collection of items.
279
+
280
+ Accepts `Pile` or any iterable. Provided items added to current
281
+ pile, overwriting existing items with same keys.
282
+
283
+ Args:
284
+ other: Collection to update with. Can be any LionIDable
285
+ """
286
+ p = pile(other)
287
+ self[p] = p
288
+
289
+ def clear(self):
290
+ """Clear all items, resetting pile to empty state."""
291
+ self.pile.clear()
292
+ self.order.clear()
293
+
294
+ def include(self, item: Any) -> bool:
295
+ """
296
+ Include item(s) in pile if not already present.
297
+
298
+ Accepts individual items and collections. Adds items if not
299
+ present. Returns `True` if item(s) in pile after operation,
300
+ `False` otherwise.
301
+
302
+ Args:
303
+ item: Item(s) to include. Can be single item or collection.
304
+
305
+ Returns:
306
+ `True` if item(s) in pile after operation, `False` otherwise.
307
+ """
308
+ item = to_list_type(item)
309
+ if item not in self:
310
+ self[item] = item
311
+ return item in self
312
+
313
+ def exclude(self, item: Any) -> bool:
314
+ """
315
+ Exclude item(s) from pile if present.
316
+
317
+ Accepts individual items and collections. Removes items if
318
+ present. Returns `True` if item(s) not in pile after operation,
319
+ `False` otherwise.
320
+
321
+ Args:
322
+ item: Item(s) to exclude. Can be single item or collection.
323
+
324
+ Returns:
325
+ `True` if item(s) not in pile after operation, `False` else.
326
+ """
327
+ item = to_list_type(item)
328
+ for i in item:
329
+ if item in self:
330
+ self.pop(i)
331
+ return item not in self
332
+
333
+ def is_homogenous(self) -> bool:
334
+ """
335
+ Check if all items have the same data type.
336
+
337
+ Returns:
338
+ `True` if all items have the same type, `False` otherwise.
339
+ Empty pile or single-item pile considered homogenous.
340
+ """
341
+ return len(self.pile) < 2 or all(is_same_dtype(self.pile.values()))
342
+
343
+ def is_empty(self) -> bool:
344
+ """
345
+ Check if the pile is empty.
346
+
347
+ Returns:
348
+ bool: `True` if the pile is empty, `False` otherwise.
349
+ """
350
+ return not self.pile
351
+
352
+ def __iter__(self):
353
+ """Return an iterator over the items in the pile.
354
+
355
+ Yields:
356
+ The items in the pile in the order they were added.
357
+ """
358
+ return iter(self.values())
359
+
360
+ def __len__(self) -> int:
361
+ """Get the number of items in the pile.
362
+
363
+ Returns:
364
+ int: The number of items in the pile.
365
+ """
366
+ return len(self.pile)
367
+
368
+ def __add__(self, other: T) -> "Pile":
369
+ """Create a new pile by including item(s) using `+`.
370
+
371
+ Returns a new `Pile` with all items from the current pile plus
372
+ provided item(s). Raises `LionValueError` if item(s) can't be
373
+ included.
374
+
375
+ Args:
376
+ other: Item(s) to include. Can be single item or collection.
377
+
378
+ Returns:
379
+ New `Pile` with all items from current pile plus item(s).
380
+
381
+ Raises:
382
+ LionValueError: If item(s) can't be included.
383
+ """
384
+ _copy = self.model_copy(deep=True)
385
+ if _copy.include(other):
386
+ return _copy
387
+ raise LionValueError("Item cannot be included in the pile.")
388
+
389
+ def __sub__(self, other) -> "Pile":
390
+ """
391
+ Create a new pile by excluding item(s) using `-`.
392
+
393
+ Returns a new `Pile` with all items from the current pile except
394
+ provided item(s). Raises `ItemNotFoundError` if item(s) not found.
395
+
396
+ Args:
397
+ other: Item(s) to exclude. Can be single item or collection.
398
+
399
+ Returns:
400
+ New `Pile` with all items from current pile except item(s).
401
+
402
+ Raises:
403
+ ItemNotFoundError: If item(s) not found in pile.
404
+ """
405
+ _copy = self.model_copy(deep=True)
406
+ if other not in self:
407
+ raise ItemNotFoundError(other)
408
+
409
+ length = len(_copy)
410
+ if not _copy.exclude(other) or len(_copy) == length:
411
+ raise LionValueError("Item cannot be excluded from the pile.")
412
+ return _copy
413
+
414
+ def __iadd__(self, other: T) -> "Pile":
415
+ """
416
+ Include item(s) in the current pile in place using `+=`.
417
+
418
+ Modifies the current pile in-place by including item(s). Returns
419
+ the modified pile.
420
+
421
+ Args:
422
+ other: Item(s) to include. Can be single item or collection.
423
+ """
424
+
425
+ return self + other
426
+
427
+ def __isub__(self, other: LionIDable) -> "Pile":
428
+ """
429
+ Exclude item(s) from the current pile using `-=`.
430
+
431
+ Modifies the current pile in-place by excluding item(s). Returns
432
+ the modified pile.
433
+
434
+ Args:
435
+ other: Item(s) to exclude. Can be single item or collection.
436
+
437
+ Returns:
438
+ Modified pile after excluding item(s).
439
+ """
440
+ return self - other
441
+
442
+ def __radd__(self, other: T) -> "Pile":
443
+ return other + self
444
+
445
+ def size(self) -> int:
446
+ """Return the total size of the pile."""
447
+ return sum([len(i) for i in self])
448
+
449
+ def insert(self, index, item):
450
+ """
451
+ Insert item(s) at specific position.
452
+
453
+ Inserts item(s) at specified index. Index must be integer.
454
+ Raises `IndexError` if index out of range.
455
+
456
+ Args:
457
+ index: Index to insert item(s). Must be integer.
458
+ item: Item(s) to insert. Can be single item or collection.
459
+
460
+ Raises:
461
+ ValueError: If index not an integer.
462
+ IndexError: If index out of range.
463
+ """
464
+ if not isinstance(index, int):
465
+ raise ValueError("Index must be an integer for pile insertion.")
466
+ item = self._validate_pile(item)
467
+ for k, v in item.items():
468
+ self.order.insert(index, k)
469
+ self.pile[k] = v
470
+
471
+ def append(self, item: T):
472
+ """
473
+ Append item to end of pile.
474
+
475
+ Appends item to end of pile. If item is `Pile`, added as single
476
+ item, preserving structure. Only way to add `Pile` into another.
477
+ Other methods assume pile as container only.
478
+
479
+ Args:
480
+ item: Item to append. Can be any object, including `Pile`.
481
+ """
482
+ self.pile[item.ln_id] = item
483
+ self.order.append(item.ln_id)
484
+
485
+ def keys(self):
486
+ """Yield the keys of the items in the pile."""
487
+ return self.order
488
+
489
+ def values(self):
490
+ """Yield the values of the items in the pile."""
491
+ yield from (self.pile.get(i) for i in self.order)
492
+
493
+ def items(self):
494
+ """
495
+ Yield the items in the pile as (key, value) pairs.
496
+
497
+ Yields:
498
+ tuple: A tuple containing the key and value of each item in the pile.
499
+ """
500
+ yield from ((i, self.pile.get(i)) for i in self.order)
501
+
502
+ @field_validator("order", mode="before")
503
+ def _validate_order(cls, value):
504
+ return _validate_order(value)
505
+
506
+ def _validate_item_type(self, value):
507
+ """
508
+ Validate the item type for the pile.
509
+
510
+ Ensures that the provided item type is a subclass of Element or iModel.
511
+ Raises an error if the validation fails.
512
+
513
+ Args:
514
+ value: The item type to validate. Can be a single type or a list of types.
515
+
516
+ Returns:
517
+ set: A set of validated item types.
518
+
519
+ Raises:
520
+ LionTypeError: If an invalid item type is provided.
521
+ LionValueError: If duplicate item types are detected.
522
+ """
523
+ if value is None:
524
+ return None
525
+
526
+ value = to_list_type(value)
527
+
528
+ for i in value:
529
+ if not isinstance(i, (type(Element), type(iModel))):
530
+ raise LionTypeError(
531
+ "Invalid item type. Expected a subclass of Component."
532
+ )
533
+
534
+ if len(value) != len(set(value)):
535
+ raise LionValueError("Detected duplicated item types in item_type.")
536
+
537
+ if len(value) > 0:
538
+ return set(value)
539
+
540
+ def _validate_pile(
541
+ self,
542
+ value,
543
+ ):
544
+ if value == {}:
545
+ return value
546
+
547
+ if isinstance(value, Component):
548
+ return {value.ln_id: value}
549
+
550
+ if self.use_obj:
551
+ if not isinstance(value, list):
552
+ value = [value]
553
+ if isinstance(value[0], (Record, Ordering)):
554
+ return {getattr(i, "ln_id"): i for i in value}
555
+
556
+ value = to_list_type(value)
557
+ if getattr(self, "item_type", None) is not None:
558
+ for i in value:
559
+ if not type(i) in self.item_type:
560
+ raise LionTypeError(
561
+ f"Invalid item type in pile. Expected {self.item_type}"
562
+ )
563
+
564
+ if isinstance(value, list):
565
+ if len(value) == 1:
566
+ if isinstance(value[0], dict) and value[0] != {}:
567
+ k = list(value[0].keys())[0]
568
+ v = value[0][k]
569
+ return {k: v}
570
+
571
+ # [item]
572
+ k = getattr(value[0], "ln_id", None)
573
+ if k:
574
+ return {k: value[0]}
575
+
576
+ return {i.ln_id: i for i in value}
577
+
578
+ raise LionValueError("Invalid pile value")
579
+
580
+ def to_df(self):
581
+ """Return the pile as a DataFrame."""
582
+ dicts_ = []
583
+ for i in self.values():
584
+ _dict = i.to_dict()
585
+ if _dict.get("embedding", None):
586
+ _dict["embedding"] = str(_dict.get("embedding"))
587
+ dicts_.append(_dict)
588
+ return to_df(dicts_)
589
+
590
+ def create_index(self, index_type="llama_index", **kwargs):
591
+ """
592
+ Create an index for the pile.
593
+
594
+ Args:
595
+ index_type (str): The type of index to use. Default is "llama_index".
596
+ **kwargs: Additional keyword arguments for the index creation.
597
+
598
+ Returns:
599
+ The created index.
600
+
601
+ Raises:
602
+ ValueError: If an invalid index type is provided.
603
+ """
604
+ if index_type == "llama_index":
605
+ from lionagi.integrations.bridge import LlamaIndexBridge
606
+
607
+ index_nodes = None
608
+
609
+ try:
610
+ index_nodes = [i.to_llama_index_node() for i in self]
611
+ except AttributeError:
612
+ raise LionTypeError(
613
+ "Invalid item type. Expected a subclass of Component."
614
+ )
615
+
616
+ self.index = LlamaIndexBridge.index(index_nodes, **kwargs)
617
+ return self.index
618
+
619
+ raise ValueError("Invalid index type")
620
+
621
+ def create_query_engine(self, index_type="llama_index", engine_kwargs={}, **kwargs):
622
+ """
623
+ Create a query engine for the pile.
624
+
625
+ Args:
626
+ index_type (str): The type of index to use. Default is "llama_index".
627
+ engine_kwargs (dict): Additional keyword arguments for the engine.
628
+ **kwargs: Additional keyword arguments for the index creation.
629
+
630
+ Raises:
631
+ ValueError: If an invalid index type is provided.
632
+ """
633
+ if index_type == "llama_index":
634
+ if "node_postprocessor" in kwargs:
635
+ engine_kwargs["node_postprocessor"] = kwargs.pop("node_postprocessor")
636
+ if "llm" in kwargs:
637
+ engine_kwargs["llm"] = kwargs.pop("llm")
638
+ if not self.index:
639
+ self.create_index(index_type, **kwargs)
640
+ query_engine = self.index.as_query_engine(**engine_kwargs)
641
+ self.engines["query"] = query_engine
642
+ else:
643
+ raise ValueError("Invalid index type")
644
+
645
+ def create_chat_engine(self, index_type="llama_index", engine_kwargs={}, **kwargs):
646
+ """
647
+ Create a chat engine for the pile.
648
+
649
+ Args:
650
+ index_type (str): The type of index to use. Default is "llama_index".
651
+ engine_kwargs (dict): Additional keyword arguments for the engine.
652
+ **kwargs: Additional keyword arguments for the index creation.
653
+
654
+ Raises:
655
+ ValueError: If an invalid index type is provided.
656
+ """
657
+ if index_type == "llama_index":
658
+ if "node_postprocessor" in kwargs:
659
+ engine_kwargs["node_postprocessor"] = kwargs.pop("node_postprocessor")
660
+ if "llm" in kwargs:
661
+ engine_kwargs["llm"] = kwargs.pop("llm")
662
+ if not self.index:
663
+ self.create_index(index_type, **kwargs)
664
+ query_engine = self.index.as_chat_engine(**engine_kwargs)
665
+ self.engines["chat"] = query_engine
666
+ else:
667
+ raise ValueError("Invalid index type")
668
+
669
+ async def query_pile(self, query, engine_kwargs={}, **kwargs):
670
+ """
671
+ Query the pile using the created query engine.
672
+
673
+ Args:
674
+ query (str): The query to send.
675
+ engine_kwargs (dict): Additional keyword arguments for the engine.
676
+ **kwargs: Additional keyword arguments for the query.
677
+
678
+ Returns:
679
+ str: The response from the query engine.
680
+ """
681
+ if not self.engines.get("query", None):
682
+ self.create_query_engine(**engine_kwargs)
683
+ response = await self.engines["query"].aquery(query, **kwargs)
684
+ self.query_response.append(response)
685
+ return str(response)
686
+
687
+ async def chat_pile(self, query, engine_kwargs={}, **kwargs):
688
+ """
689
+ Chat with the pile using the created chat engine.
690
+
691
+ Args:
692
+ query (str): The query to send.
693
+ engine_kwargs (dict): Additional keyword arguments for the engine.
694
+ **kwargs: Additional keyword arguments for the query.
695
+
696
+ Returns:
697
+ str: The response from the chat engine.
698
+ """
699
+ if not self.engines.get("chat", None):
700
+ self.create_chat_engine(**engine_kwargs)
701
+ response = await self.engines["chat"].achat(query, **kwargs)
702
+ self.query_response.append(response)
703
+ return str(response)
704
+
705
+ async def embed_pile(
706
+ self, imodel=None, field="content", embed_kwargs={}, verbose=True, **kwargs
707
+ ):
708
+ """
709
+ Embed the items in the pile.
710
+
711
+ Args:
712
+ imodel: The embedding model to use.
713
+ field (str): The field to embed. Default is "content".
714
+ embed_kwargs (dict): Additional keyword arguments for the embedding.
715
+ verbose (bool): Whether to print verbose messages. Default is True.
716
+ **kwargs: Additional keyword arguments for the embedding.
717
+
718
+ Raises:
719
+ ModelLimitExceededError: If the model limit is exceeded.
720
+ """
721
+ from .model import iModel
722
+
723
+ imodel = imodel or iModel(endpoint="embeddings", **kwargs)
724
+
725
+ max_concurrency = kwargs.get("max_concurrency", None) or 100
726
+
727
+ @cd.max_concurrency(max_concurrency)
728
+ async def _embed_item(item):
729
+ try:
730
+ return await imodel.embed_node(item, field=field, **embed_kwargs)
731
+ except ModelLimitExceededError:
732
+ pass
733
+ return None
734
+
735
+ await alcall(list(self), _embed_item)
736
+
737
+ a = len([i for i in self if "embedding" in i._all_fields])
738
+ if len(self) > a and verbose:
739
+ print(
740
+ f"Successfully embedded {a}/{len(self)} items, Failed to embed {len(self) - a}/{len(self)} items"
741
+ )
742
+ return
743
+
744
+ print(f"Successfully embedded all {a}/{a} items")
745
+
746
+ def to_csv(self, file_name, **kwargs):
747
+ """
748
+ Save the pile to a CSV file.
749
+
750
+ Args:
751
+ file_name (str): The name of the CSV file.
752
+ **kwargs: Additional keyword arguments for the CSV writer.
753
+ """
754
+ self.to_df().to_csv(file_name, index=False, **kwargs)
755
+
756
+ @classmethod
757
+ def from_csv(cls, file_name, **kwargs):
758
+ """
759
+ Load a pile from a CSV file.
760
+
761
+ Args:
762
+ file_name (str): The name of the CSV file.
763
+ **kwargs: Additional keyword arguments for the CSV reader.
764
+
765
+ Returns:
766
+ Pile: The loaded pile.
767
+ """
768
+ from pandas import read_csv
769
+
770
+ df = read_csv(file_name, **kwargs)
771
+ items = Component.from_obj(df)
772
+ return cls(items)
773
+
774
+ @classmethod
775
+ def from_df(cls, df):
776
+ """
777
+ Load a pile from a DataFrame.
778
+
779
+ Args:
780
+ df (DataFrame): The DataFrame to load.
781
+
782
+ Returns:
783
+ Pile: The loaded pile.
784
+ """
785
+ items = Component.from_obj(df)
786
+ return cls(items)
787
+
788
+ def as_query_tool(
789
+ self,
790
+ index_type="llama_index",
791
+ query_type="query",
792
+ name=None,
793
+ guidance=None,
794
+ query_description=None,
795
+ **kwargs,
796
+ ):
797
+ """
798
+ Create a query tool for the pile.
799
+
800
+ Args:
801
+ index_type (str): The type of index to use. Default is "llama_index".
802
+ query_type (str): The type of query engine to use. Default is "query".
803
+ name (str): The name of the query tool. Default is "query".
804
+ guidance (str): The guidance for the query tool.
805
+ query_description (str): The description of the query parameter.
806
+ **kwargs: Additional keyword arguments for the query engine.
807
+
808
+ Returns:
809
+ Tool: The created query tool.
810
+ """
811
+ if not self.engines.get(query_type, None):
812
+ if query_type == "query":
813
+ self.create_query_engine(index_type=index_type, **kwargs)
814
+ elif query_type == "chat":
815
+ self.create_chat_engine(index_type=index_type, **kwargs)
816
+
817
+ from lionagi.core.action.tool_manager import func_to_tool
818
+
819
+ if not guidance:
820
+ if query_type == "query":
821
+ guidance = "Query a QA bot"
822
+ elif query_type == "chat":
823
+ guidance = "Chat with a QA bot"
824
+
825
+ if not query_description:
826
+ if query_type == "query":
827
+ query_description = "The query to send"
828
+ elif query_type == "chat":
829
+ query_description = "The message to send"
830
+
831
+ async def query(query: str):
832
+ if query_type == "query":
833
+ return await self.query_pile(query, **kwargs)
834
+
835
+ elif query_type == "chat":
836
+ return await self.chat_pile(query, **kwargs)
837
+
838
+ name = name or "query"
839
+ tool = func_to_tool(query)[0]
840
+ tool.schema_["function"]["name"] = name
841
+ tool.schema_["function"]["description"] = guidance
842
+ tool.schema_["function"]["parameters"]["properties"]["query"][
843
+ "description"
844
+ ] = query_description
845
+ self.tools[query_type] = tool
846
+ return self.tools[query_type]
847
+
848
+ def __list__(self):
849
+ """
850
+ Get a list of the items in the pile.
851
+
852
+ Returns:
853
+ list: The items in the pile.
854
+ """
855
+ return list(self.pile.values())
856
+
857
+ def __str__(self):
858
+ """
859
+ Get the string representation of the pile.
860
+
861
+ Returns:
862
+ str: The string representation of the pile.
863
+ """
864
+ return self.to_df().__str__()
865
+
866
+ def __repr__(self):
867
+ """
868
+ Get the representation of the pile.
869
+
870
+ Returns:
871
+ str: The representation of the pile.
872
+ """
873
+ return self.to_df().__repr__()
874
+
875
+
876
+ def pile(
877
+ items: Iterable[T] | None = None,
878
+ item_type: set[Type] | None = None,
879
+ order=None,
880
+ use_obj=None,
881
+ csv_file=None,
882
+ df=None,
883
+ **kwargs,
884
+ ) -> Pile[T]:
885
+ """
886
+ Create a new Pile instance.
887
+
888
+ This function provides various ways to create a Pile instance:
889
+ - Directly from items
890
+ - From a CSV file
891
+ - From a DataFrame
892
+
893
+ Args:
894
+ items (Iterable[T] | None): The items to include in the pile.
895
+ item_type (set[Type] | None): The allowed types of items in the pile.
896
+ order (list[str] | None): The order of items.
897
+ use_obj (bool | None): Whether to treat Record and Ordering as objects.
898
+ csv_file (str | None): The path to a CSV file to load items from.
899
+ df (DataFrame | None): A DataFrame to load items from.
900
+ **kwargs: Additional keyword arguments for loading from CSV or DataFrame.
901
+
902
+ Returns:
903
+ Pile[T]: A new Pile instance.
904
+
905
+ Raises:
906
+ ValueError: If invalid arguments are provided.
907
+ """
908
+ if csv_file:
909
+ return Pile.from_csv(csv_file, **kwargs)
910
+ if df:
911
+ return Pile.from_df(df)
912
+
913
+ return Pile(items, item_type, order, use_obj)