lionagi 0.0.312__py3-none-any.whl → 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (268) hide show
  1. lionagi/__init__.py +61 -3
  2. lionagi/core/__init__.py +0 -14
  3. lionagi/core/_setting/_setting.py +59 -0
  4. lionagi/core/action/__init__.py +14 -0
  5. lionagi/core/action/function_calling.py +136 -0
  6. lionagi/core/action/manual.py +1 -0
  7. lionagi/core/action/node.py +109 -0
  8. lionagi/core/action/tool.py +114 -0
  9. lionagi/core/action/tool_manager.py +356 -0
  10. lionagi/core/agent/__init__.py +0 -3
  11. lionagi/core/agent/base_agent.py +45 -36
  12. lionagi/core/agent/eval/evaluator.py +1 -0
  13. lionagi/core/agent/eval/vote.py +40 -0
  14. lionagi/core/agent/learn/learner.py +59 -0
  15. lionagi/core/agent/plan/unit_template.py +1 -0
  16. lionagi/core/collections/__init__.py +17 -0
  17. lionagi/core/collections/_logger.py +319 -0
  18. lionagi/core/collections/abc/__init__.py +53 -0
  19. lionagi/core/collections/abc/component.py +615 -0
  20. lionagi/core/collections/abc/concepts.py +297 -0
  21. lionagi/core/collections/abc/exceptions.py +150 -0
  22. lionagi/core/collections/abc/util.py +45 -0
  23. lionagi/core/collections/exchange.py +161 -0
  24. lionagi/core/collections/flow.py +426 -0
  25. lionagi/core/collections/model.py +419 -0
  26. lionagi/core/collections/pile.py +913 -0
  27. lionagi/core/collections/progression.py +236 -0
  28. lionagi/core/collections/util.py +64 -0
  29. lionagi/core/director/direct.py +314 -0
  30. lionagi/core/director/director.py +2 -0
  31. lionagi/core/engine/branch_engine.py +333 -0
  32. lionagi/core/engine/instruction_map_engine.py +204 -0
  33. lionagi/core/engine/sandbox_.py +14 -0
  34. lionagi/core/engine/script_engine.py +99 -0
  35. lionagi/core/executor/base_executor.py +90 -0
  36. lionagi/core/executor/graph_executor.py +330 -0
  37. lionagi/core/executor/neo4j_executor.py +384 -0
  38. lionagi/core/generic/__init__.py +7 -0
  39. lionagi/core/generic/edge.py +112 -0
  40. lionagi/core/generic/edge_condition.py +16 -0
  41. lionagi/core/generic/graph.py +236 -0
  42. lionagi/core/generic/hyperedge.py +1 -0
  43. lionagi/core/generic/node.py +220 -0
  44. lionagi/core/generic/tree.py +48 -0
  45. lionagi/core/generic/tree_node.py +79 -0
  46. lionagi/core/mail/__init__.py +7 -3
  47. lionagi/core/mail/mail.py +25 -0
  48. lionagi/core/mail/mail_manager.py +142 -58
  49. lionagi/core/mail/package.py +45 -0
  50. lionagi/core/mail/start_mail.py +36 -0
  51. lionagi/core/message/__init__.py +19 -0
  52. lionagi/core/message/action_request.py +133 -0
  53. lionagi/core/message/action_response.py +135 -0
  54. lionagi/core/message/assistant_response.py +95 -0
  55. lionagi/core/message/instruction.py +234 -0
  56. lionagi/core/message/message.py +101 -0
  57. lionagi/core/message/system.py +86 -0
  58. lionagi/core/message/util.py +283 -0
  59. lionagi/core/report/__init__.py +4 -0
  60. lionagi/core/report/base.py +217 -0
  61. lionagi/core/report/form.py +231 -0
  62. lionagi/core/report/report.py +166 -0
  63. lionagi/core/report/util.py +28 -0
  64. lionagi/core/rule/__init__.py +0 -0
  65. lionagi/core/rule/_default.py +16 -0
  66. lionagi/core/rule/action.py +99 -0
  67. lionagi/core/rule/base.py +238 -0
  68. lionagi/core/rule/boolean.py +56 -0
  69. lionagi/core/rule/choice.py +47 -0
  70. lionagi/core/rule/mapping.py +96 -0
  71. lionagi/core/rule/number.py +71 -0
  72. lionagi/core/rule/rulebook.py +109 -0
  73. lionagi/core/rule/string.py +52 -0
  74. lionagi/core/rule/util.py +35 -0
  75. lionagi/core/session/__init__.py +0 -3
  76. lionagi/core/session/branch.py +431 -0
  77. lionagi/core/session/directive_mixin.py +287 -0
  78. lionagi/core/session/session.py +230 -902
  79. lionagi/core/structure/__init__.py +1 -0
  80. lionagi/core/structure/chain.py +1 -0
  81. lionagi/core/structure/forest.py +1 -0
  82. lionagi/core/structure/graph.py +1 -0
  83. lionagi/core/structure/tree.py +1 -0
  84. lionagi/core/unit/__init__.py +5 -0
  85. lionagi/core/unit/parallel_unit.py +245 -0
  86. lionagi/core/unit/template/__init__.py +0 -0
  87. lionagi/core/unit/template/action.py +81 -0
  88. lionagi/core/unit/template/base.py +51 -0
  89. lionagi/core/unit/template/plan.py +84 -0
  90. lionagi/core/unit/template/predict.py +109 -0
  91. lionagi/core/unit/template/score.py +124 -0
  92. lionagi/core/unit/template/select.py +104 -0
  93. lionagi/core/unit/unit.py +362 -0
  94. lionagi/core/unit/unit_form.py +305 -0
  95. lionagi/core/unit/unit_mixin.py +1168 -0
  96. lionagi/core/unit/util.py +71 -0
  97. lionagi/core/validator/__init__.py +0 -0
  98. lionagi/core/validator/validator.py +364 -0
  99. lionagi/core/work/__init__.py +0 -0
  100. lionagi/core/work/work.py +76 -0
  101. lionagi/core/work/work_function.py +101 -0
  102. lionagi/core/work/work_queue.py +103 -0
  103. lionagi/core/work/worker.py +258 -0
  104. lionagi/core/work/worklog.py +120 -0
  105. lionagi/experimental/__init__.py +0 -0
  106. lionagi/experimental/compressor/__init__.py +0 -0
  107. lionagi/experimental/compressor/base.py +46 -0
  108. lionagi/experimental/compressor/llm_compressor.py +247 -0
  109. lionagi/experimental/compressor/llm_summarizer.py +61 -0
  110. lionagi/experimental/compressor/util.py +70 -0
  111. lionagi/experimental/directive/__init__.py +19 -0
  112. lionagi/experimental/directive/parser/__init__.py +0 -0
  113. lionagi/experimental/directive/parser/base_parser.py +282 -0
  114. lionagi/experimental/directive/template/__init__.py +0 -0
  115. lionagi/experimental/directive/template/base_template.py +79 -0
  116. lionagi/experimental/directive/template/schema.py +36 -0
  117. lionagi/experimental/directive/tokenizer.py +73 -0
  118. lionagi/experimental/evaluator/__init__.py +0 -0
  119. lionagi/experimental/evaluator/ast_evaluator.py +131 -0
  120. lionagi/experimental/evaluator/base_evaluator.py +218 -0
  121. lionagi/experimental/knowledge/__init__.py +0 -0
  122. lionagi/experimental/knowledge/base.py +10 -0
  123. lionagi/experimental/knowledge/graph.py +0 -0
  124. lionagi/experimental/memory/__init__.py +0 -0
  125. lionagi/experimental/strategies/__init__.py +0 -0
  126. lionagi/experimental/strategies/base.py +1 -0
  127. lionagi/integrations/bridge/autogen_/__init__.py +0 -0
  128. lionagi/integrations/bridge/autogen_/autogen_.py +124 -0
  129. lionagi/integrations/bridge/langchain_/documents.py +4 -0
  130. lionagi/integrations/bridge/llamaindex_/index.py +30 -0
  131. lionagi/integrations/bridge/llamaindex_/llama_index_bridge.py +6 -0
  132. lionagi/integrations/bridge/llamaindex_/llama_pack.py +227 -0
  133. lionagi/integrations/bridge/llamaindex_/node_parser.py +6 -9
  134. lionagi/integrations/bridge/pydantic_/pydantic_bridge.py +1 -0
  135. lionagi/integrations/bridge/transformers_/__init__.py +0 -0
  136. lionagi/integrations/bridge/transformers_/install_.py +36 -0
  137. lionagi/integrations/chunker/__init__.py +0 -0
  138. lionagi/integrations/chunker/chunk.py +312 -0
  139. lionagi/integrations/config/oai_configs.py +38 -7
  140. lionagi/integrations/config/ollama_configs.py +1 -1
  141. lionagi/integrations/config/openrouter_configs.py +14 -2
  142. lionagi/integrations/loader/__init__.py +0 -0
  143. lionagi/integrations/loader/load.py +253 -0
  144. lionagi/integrations/loader/load_util.py +195 -0
  145. lionagi/integrations/provider/_mapping.py +46 -0
  146. lionagi/integrations/provider/litellm.py +2 -1
  147. lionagi/integrations/provider/mlx_service.py +16 -9
  148. lionagi/integrations/provider/oai.py +91 -4
  149. lionagi/integrations/provider/ollama.py +7 -6
  150. lionagi/integrations/provider/openrouter.py +115 -8
  151. lionagi/integrations/provider/services.py +2 -2
  152. lionagi/integrations/provider/transformers.py +18 -22
  153. lionagi/integrations/storage/__init__.py +3 -0
  154. lionagi/integrations/storage/neo4j.py +665 -0
  155. lionagi/integrations/storage/storage_util.py +287 -0
  156. lionagi/integrations/storage/structure_excel.py +285 -0
  157. lionagi/integrations/storage/to_csv.py +63 -0
  158. lionagi/integrations/storage/to_excel.py +83 -0
  159. lionagi/libs/__init__.py +26 -1
  160. lionagi/libs/ln_api.py +78 -23
  161. lionagi/libs/ln_context.py +37 -0
  162. lionagi/libs/ln_convert.py +21 -9
  163. lionagi/libs/ln_func_call.py +69 -28
  164. lionagi/libs/ln_image.py +107 -0
  165. lionagi/libs/ln_knowledge_graph.py +405 -0
  166. lionagi/libs/ln_nested.py +26 -11
  167. lionagi/libs/ln_parse.py +110 -14
  168. lionagi/libs/ln_queue.py +117 -0
  169. lionagi/libs/ln_tokenize.py +164 -0
  170. lionagi/{core/prompt/field_validator.py → libs/ln_validate.py} +79 -14
  171. lionagi/libs/special_tokens.py +172 -0
  172. lionagi/libs/sys_util.py +107 -2
  173. lionagi/lions/__init__.py +0 -0
  174. lionagi/lions/coder/__init__.py +0 -0
  175. lionagi/lions/coder/add_feature.py +20 -0
  176. lionagi/lions/coder/base_prompts.py +22 -0
  177. lionagi/lions/coder/code_form.py +13 -0
  178. lionagi/lions/coder/coder.py +168 -0
  179. lionagi/lions/coder/util.py +96 -0
  180. lionagi/lions/researcher/__init__.py +0 -0
  181. lionagi/lions/researcher/data_source/__init__.py +0 -0
  182. lionagi/lions/researcher/data_source/finhub_.py +191 -0
  183. lionagi/lions/researcher/data_source/google_.py +199 -0
  184. lionagi/lions/researcher/data_source/wiki_.py +96 -0
  185. lionagi/lions/researcher/data_source/yfinance_.py +21 -0
  186. lionagi/tests/integrations/__init__.py +0 -0
  187. lionagi/tests/libs/__init__.py +0 -0
  188. lionagi/tests/libs/test_field_validators.py +353 -0
  189. lionagi/tests/{test_libs → libs}/test_func_call.py +23 -21
  190. lionagi/tests/{test_libs → libs}/test_nested.py +36 -21
  191. lionagi/tests/{test_libs → libs}/test_parse.py +1 -1
  192. lionagi/tests/libs/test_queue.py +67 -0
  193. lionagi/tests/test_core/collections/__init__.py +0 -0
  194. lionagi/tests/test_core/collections/test_component.py +206 -0
  195. lionagi/tests/test_core/collections/test_exchange.py +138 -0
  196. lionagi/tests/test_core/collections/test_flow.py +145 -0
  197. lionagi/tests/test_core/collections/test_pile.py +171 -0
  198. lionagi/tests/test_core/collections/test_progression.py +129 -0
  199. lionagi/tests/test_core/generic/__init__.py +0 -0
  200. lionagi/tests/test_core/generic/test_edge.py +67 -0
  201. lionagi/tests/test_core/generic/test_graph.py +96 -0
  202. lionagi/tests/test_core/generic/test_node.py +106 -0
  203. lionagi/tests/test_core/generic/test_tree_node.py +73 -0
  204. lionagi/tests/test_core/test_branch.py +115 -292
  205. lionagi/tests/test_core/test_form.py +46 -0
  206. lionagi/tests/test_core/test_report.py +105 -0
  207. lionagi/tests/test_core/test_validator.py +111 -0
  208. lionagi/version.py +1 -1
  209. {lionagi-0.0.312.dist-info → lionagi-0.2.1.dist-info}/LICENSE +12 -11
  210. {lionagi-0.0.312.dist-info → lionagi-0.2.1.dist-info}/METADATA +19 -118
  211. lionagi-0.2.1.dist-info/RECORD +240 -0
  212. lionagi/core/branch/__init__.py +0 -4
  213. lionagi/core/branch/base_branch.py +0 -654
  214. lionagi/core/branch/branch.py +0 -471
  215. lionagi/core/branch/branch_flow_mixin.py +0 -96
  216. lionagi/core/branch/executable_branch.py +0 -347
  217. lionagi/core/branch/util.py +0 -323
  218. lionagi/core/direct/__init__.py +0 -6
  219. lionagi/core/direct/predict.py +0 -161
  220. lionagi/core/direct/score.py +0 -278
  221. lionagi/core/direct/select.py +0 -169
  222. lionagi/core/direct/utils.py +0 -87
  223. lionagi/core/direct/vote.py +0 -64
  224. lionagi/core/flow/base/baseflow.py +0 -23
  225. lionagi/core/flow/monoflow/ReAct.py +0 -238
  226. lionagi/core/flow/monoflow/__init__.py +0 -9
  227. lionagi/core/flow/monoflow/chat.py +0 -95
  228. lionagi/core/flow/monoflow/chat_mixin.py +0 -263
  229. lionagi/core/flow/monoflow/followup.py +0 -214
  230. lionagi/core/flow/polyflow/__init__.py +0 -1
  231. lionagi/core/flow/polyflow/chat.py +0 -248
  232. lionagi/core/mail/schema.py +0 -56
  233. lionagi/core/messages/__init__.py +0 -3
  234. lionagi/core/messages/schema.py +0 -533
  235. lionagi/core/prompt/prompt_template.py +0 -316
  236. lionagi/core/schema/__init__.py +0 -22
  237. lionagi/core/schema/action_node.py +0 -29
  238. lionagi/core/schema/base_mixin.py +0 -296
  239. lionagi/core/schema/base_node.py +0 -199
  240. lionagi/core/schema/condition.py +0 -24
  241. lionagi/core/schema/data_logger.py +0 -354
  242. lionagi/core/schema/data_node.py +0 -93
  243. lionagi/core/schema/prompt_template.py +0 -67
  244. lionagi/core/schema/structure.py +0 -910
  245. lionagi/core/tool/__init__.py +0 -3
  246. lionagi/core/tool/tool_manager.py +0 -280
  247. lionagi/integrations/bridge/pydantic_/base_model.py +0 -7
  248. lionagi/tests/test_core/test_base_branch.py +0 -427
  249. lionagi/tests/test_core/test_chat_flow.py +0 -63
  250. lionagi/tests/test_core/test_mail_manager.py +0 -75
  251. lionagi/tests/test_core/test_prompts.py +0 -51
  252. lionagi/tests/test_core/test_session.py +0 -254
  253. lionagi/tests/test_core/test_session_base_util.py +0 -312
  254. lionagi/tests/test_core/test_tool_manager.py +0 -95
  255. lionagi-0.0.312.dist-info/RECORD +0 -111
  256. /lionagi/core/{branch/base → _setting}/__init__.py +0 -0
  257. /lionagi/core/{flow → agent/eval}/__init__.py +0 -0
  258. /lionagi/core/{flow/base → agent/learn}/__init__.py +0 -0
  259. /lionagi/core/{prompt → agent/plan}/__init__.py +0 -0
  260. /lionagi/core/{tool/manual.py → agent/plan/plan.py} +0 -0
  261. /lionagi/{tests/test_integrations → core/director}/__init__.py +0 -0
  262. /lionagi/{tests/test_libs → core/engine}/__init__.py +0 -0
  263. /lionagi/{tests/test_libs/test_async.py → core/executor/__init__.py} +0 -0
  264. /lionagi/tests/{test_libs → libs}/test_api.py +0 -0
  265. /lionagi/tests/{test_libs → libs}/test_convert.py +0 -0
  266. /lionagi/tests/{test_libs → libs}/test_sys_util.py +0 -0
  267. {lionagi-0.0.312.dist-info → lionagi-0.2.1.dist-info}/WHEEL +0 -0
  268. {lionagi-0.0.312.dist-info → lionagi-0.2.1.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,913 @@
1
+ """
2
+ Copyright 2024 HaiyangLi
3
+
4
+ Licensed under the Apache License, Version 2.0 (the "License");
5
+ you may not use this file except in compliance with the License.
6
+ You may obtain a copy of the License at
7
+
8
+ http://www.apache.org/licenses/LICENSE-2.0
9
+
10
+ Unless required by applicable law or agreed to in writing, software
11
+ distributed under the License is distributed on an "AS IS" BASIS,
12
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ See the License for the specific language governing permissions and
14
+ limitations under the License.
15
+ """
16
+
17
+ """
18
+ This module defines the Pile class, a versatile container for managing
19
+ collections of Element objects. It supports structured access and
20
+ manipulation, including retrieval, addition, and deletion of elements.
21
+ """
22
+
23
+ from collections.abc import Iterable
24
+ from typing import TypeVar, Type, Any, Generic
25
+
26
+ from pydantic import Field, field_validator
27
+
28
+ from lionagi.libs.ln_convert import is_same_dtype, to_df
29
+ from lionagi.libs.ln_func_call import bcall, alcall, CallDecorator as cd
30
+ from .abc import (
31
+ Element,
32
+ Record,
33
+ Component,
34
+ Ordering,
35
+ LionIDable,
36
+ get_lion_id,
37
+ LionValueError,
38
+ LionTypeError,
39
+ ItemNotFoundError,
40
+ ModelLimitExceededError,
41
+ )
42
+ from .model import iModel
43
+ from .util import to_list_type, _validate_order
44
+
45
+ T = TypeVar("T")
46
+
47
+
48
+ class Pile(Element, Record, Generic[T]):
49
+ """
50
+ Collection class for managing Element objects.
51
+
52
+ Facilitates ordered and type-validated storage and access, supporting
53
+ both index-based and key-based retrieval.
54
+
55
+ Attributes:
56
+ pile (dict[str, T]): Maps unique identifiers to items.
57
+ item_type (set[Type[Element]] | None): Allowed item types.
58
+ name (str | None): Optional name for the pile.
59
+ order (list[str]): Order of item identifiers.
60
+ use_obj (bool): If True, treat Record and Ordering as objects.
61
+ """
62
+
63
+ use_obj: bool = False
64
+ pile: dict[str, T] = Field(default_factory=dict)
65
+ item_type: set[Type[Element]] | None = Field(default=None)
66
+ name: str | None = None
67
+ order: list[str] = Field(default_factory=list)
68
+ index: Any = None
69
+ engines: dict[str, Any] = Field(default_factory=dict)
70
+ query_response: list = []
71
+ tools: dict = {}
72
+
73
+ def __init__(
74
+ self,
75
+ items=None,
76
+ item_type=None,
77
+ order=None,
78
+ use_obj=None,
79
+ ):
80
+ super().__init__()
81
+
82
+ self.use_obj = use_obj or False
83
+ self.pile = self._validate_pile(items or {})
84
+ self.item_type = self._validate_item_type(item_type)
85
+
86
+ order = order or list(self.pile.keys())
87
+ if not len(order) == len(self):
88
+ raise ValueError(
89
+ "The length of the order does not match the length of the pile"
90
+ )
91
+ self.order = order
92
+
93
+ def __getitem__(self, key) -> T | "Pile[T]":
94
+ """
95
+ Retrieve items from the pile using a key.
96
+
97
+ Supports multiple types of key access:
98
+ - By index or slice (list-like access)
99
+ - By LionID (dictionary-like access)
100
+ - By other complex types if item is of LionIDable
101
+
102
+ Args:
103
+ key: Key to retrieve items.
104
+
105
+ Returns:
106
+ The requested item(s). Single items returned directly,
107
+ multiple items returned in a new `Pile` instance.
108
+
109
+ Raises:
110
+ ItemNotFoundError: If requested item(s) not found.
111
+ LionTypeError: If provided key is invalid.
112
+ """
113
+ try:
114
+ if isinstance(key, (int, slice)):
115
+ # Handle list-like index or slice
116
+ _key = self.order[key]
117
+ _key = [_key] if isinstance(key, int) else _key
118
+ _out = [self.pile.get(i) for i in _key]
119
+ return _out[0] if len(_out) == 1 else pile(_out, self.item_type, _key)
120
+ except IndexError as e:
121
+ raise ItemNotFoundError(key) from e
122
+
123
+ keys = to_list_type(key)
124
+ for idx, item in enumerate(keys):
125
+ if isinstance(item, str):
126
+ keys[idx] = item
127
+ continue
128
+ if hasattr(item, "ln_id"):
129
+ keys[idx] = item.ln_id
130
+
131
+ if not all(keys):
132
+ raise LionTypeError("Invalid item type. Expected LionIDable object(s).")
133
+
134
+ try:
135
+ if len(keys) == 1:
136
+ return self.pile.get(keys[0])
137
+ return pile([self.pile.get(i) for i in keys], self.item_type, keys)
138
+ except KeyError as e:
139
+ raise ItemNotFoundError(key) from e
140
+
141
+ def __setitem__(self, key, item) -> None:
142
+ """
143
+ Set new values in the pile using various key types.
144
+
145
+ Handles single/multiple assignments, ensures type consistency.
146
+ Supports index/slice, LionID, and LionIDable key access.
147
+
148
+ Args:
149
+ key: Key to set items. Can be index, slice, LionID, LionIDable.
150
+ item: Item(s) to set. Can be single item or collection.
151
+
152
+ Raises:
153
+ ValueError: Length mismatch or multiple items to single key.
154
+ LionTypeError: Item type doesn't match allowed types.
155
+ """
156
+ item = self._validate_pile(item)
157
+
158
+ if isinstance(key, (int, slice)):
159
+ # Handle list-like index or slice
160
+ try:
161
+ _key = self.order[key]
162
+ except IndexError as e:
163
+ raise e
164
+
165
+ if isinstance(_key, str) and len(item) != 1:
166
+ raise ValueError("Cannot assign multiple items to a single item.")
167
+
168
+ if isinstance(_key, list) and len(item) != len(_key):
169
+ raise ValueError(
170
+ "The length of values does not match the length of the slice"
171
+ )
172
+
173
+ for k, v in item.items():
174
+ if self.item_type and type(v) not in self.item_type:
175
+ raise LionTypeError(f"Invalid item type. Expected {self.item_type}")
176
+
177
+ self.pile[k] = v
178
+ self.order[key] = k
179
+ self.pile.pop(_key)
180
+ return
181
+
182
+ if len(to_list_type(key)) != len(item):
183
+ raise ValueError("The length of keys does not match the length of values")
184
+
185
+ self.pile.update(item)
186
+ self.order.extend(item.keys())
187
+
188
+ def __contains__(self, item: Any) -> bool:
189
+ """
190
+ Check if item(s) are present in the pile.
191
+
192
+ Accepts individual items and collections. Returns `True` if all
193
+ provided items are found, `False` otherwise.
194
+
195
+ Args:
196
+ item: Item(s) to check. Can be single item or collection.
197
+
198
+ Returns:
199
+ `True` if all items are found, `False` otherwise.
200
+ """
201
+ item = to_list_type(item)
202
+ for i in item:
203
+ try:
204
+ a = i if isinstance(i, str) else get_lion_id(i)
205
+ if a not in self.pile:
206
+ return False
207
+ except Exception:
208
+ return False
209
+
210
+ return True
211
+
212
+ def pop(self, key: Any, default=...) -> T | "Pile[T]" | None:
213
+ """
214
+ Remove and return item(s) associated with given key.
215
+
216
+ Raises `ItemNotFoundError` if key not found and no default given.
217
+ Returns default if provided and key not found.
218
+
219
+ Args:
220
+ key: Key of item(s) to remove and return. Can be single key
221
+ or collection of keys.
222
+ default: Default value if key not found. If not specified
223
+ and key not found, raises `ItemNotFoundError`.
224
+
225
+ Returns:
226
+ Removed item(s) associated with key. Single items returned
227
+ directly, multiple items in new `Pile`. Returns default if
228
+ provided and key not found.
229
+
230
+ Raises:
231
+ ItemNotFoundError: If key not found and no default specified.
232
+ """
233
+ key = to_list_type(key)
234
+ items = []
235
+
236
+ for i in key:
237
+ if i not in self:
238
+ if default == ...:
239
+ raise ItemNotFoundError(i)
240
+ return default
241
+
242
+ for i in key:
243
+ _id = get_lion_id(i)
244
+ items.append(self.pile.pop(_id))
245
+ self.order.remove(_id)
246
+
247
+ return pile(items) if len(items) > 1 else items[0]
248
+
249
+ def get(self, key: Any, default=...) -> T | "Pile[T]" | None:
250
+ """
251
+ Retrieve item(s) associated with given key.
252
+
253
+ Raises `ItemNotFoundError` if key not found and no default given.
254
+ Returns default if provided and key not found.
255
+
256
+ Args:
257
+ key: Key of item(s) to retrieve. Can be single or collection.
258
+ default: Default value if key not found. If not specified
259
+ and key not found, raises `ItemNotFoundError`.
260
+
261
+ Returns:
262
+ Retrieved item(s) associated with key. Single items returned
263
+ directly, multiple items in new `Pile`. Returns default if
264
+ provided and key not found.
265
+
266
+ Raises:
267
+ ItemNotFoundError: If key not found and no default specified.
268
+ """
269
+ try:
270
+ return self[key]
271
+ except ItemNotFoundError as e:
272
+ if default == ...:
273
+ raise e
274
+ return default
275
+
276
+ def update(self, other: Any):
277
+ """
278
+ Update pile with another collection of items.
279
+
280
+ Accepts `Pile` or any iterable. Provided items added to current
281
+ pile, overwriting existing items with same keys.
282
+
283
+ Args:
284
+ other: Collection to update with. Can be any LionIDable
285
+ """
286
+ p = pile(other)
287
+ self[p] = p
288
+
289
+ def clear(self):
290
+ """Clear all items, resetting pile to empty state."""
291
+ self.pile.clear()
292
+ self.order.clear()
293
+
294
+ def include(self, item: Any) -> bool:
295
+ """
296
+ Include item(s) in pile if not already present.
297
+
298
+ Accepts individual items and collections. Adds items if not
299
+ present. Returns `True` if item(s) in pile after operation,
300
+ `False` otherwise.
301
+
302
+ Args:
303
+ item: Item(s) to include. Can be single item or collection.
304
+
305
+ Returns:
306
+ `True` if item(s) in pile after operation, `False` otherwise.
307
+ """
308
+ item = to_list_type(item)
309
+ if item not in self:
310
+ self[item] = item
311
+ return item in self
312
+
313
+ def exclude(self, item: Any) -> bool:
314
+ """
315
+ Exclude item(s) from pile if present.
316
+
317
+ Accepts individual items and collections. Removes items if
318
+ present. Returns `True` if item(s) not in pile after operation,
319
+ `False` otherwise.
320
+
321
+ Args:
322
+ item: Item(s) to exclude. Can be single item or collection.
323
+
324
+ Returns:
325
+ `True` if item(s) not in pile after operation, `False` else.
326
+ """
327
+ item = to_list_type(item)
328
+ for i in item:
329
+ if item in self:
330
+ self.pop(i)
331
+ return item not in self
332
+
333
+ def is_homogenous(self) -> bool:
334
+ """
335
+ Check if all items have the same data type.
336
+
337
+ Returns:
338
+ `True` if all items have the same type, `False` otherwise.
339
+ Empty pile or single-item pile considered homogenous.
340
+ """
341
+ return len(self.pile) < 2 or all(is_same_dtype(self.pile.values()))
342
+
343
+ def is_empty(self) -> bool:
344
+ """
345
+ Check if the pile is empty.
346
+
347
+ Returns:
348
+ bool: `True` if the pile is empty, `False` otherwise.
349
+ """
350
+ return not self.pile
351
+
352
+ def __iter__(self):
353
+ """Return an iterator over the items in the pile.
354
+
355
+ Yields:
356
+ The items in the pile in the order they were added.
357
+ """
358
+ return iter(self.values())
359
+
360
+ def __len__(self) -> int:
361
+ """Get the number of items in the pile.
362
+
363
+ Returns:
364
+ int: The number of items in the pile.
365
+ """
366
+ return len(self.pile)
367
+
368
+ def __add__(self, other: T) -> "Pile":
369
+ """Create a new pile by including item(s) using `+`.
370
+
371
+ Returns a new `Pile` with all items from the current pile plus
372
+ provided item(s). Raises `LionValueError` if item(s) can't be
373
+ included.
374
+
375
+ Args:
376
+ other: Item(s) to include. Can be single item or collection.
377
+
378
+ Returns:
379
+ New `Pile` with all items from current pile plus item(s).
380
+
381
+ Raises:
382
+ LionValueError: If item(s) can't be included.
383
+ """
384
+ _copy = self.model_copy(deep=True)
385
+ if _copy.include(other):
386
+ return _copy
387
+ raise LionValueError("Item cannot be included in the pile.")
388
+
389
+ def __sub__(self, other) -> "Pile":
390
+ """
391
+ Create a new pile by excluding item(s) using `-`.
392
+
393
+ Returns a new `Pile` with all items from the current pile except
394
+ provided item(s). Raises `ItemNotFoundError` if item(s) not found.
395
+
396
+ Args:
397
+ other: Item(s) to exclude. Can be single item or collection.
398
+
399
+ Returns:
400
+ New `Pile` with all items from current pile except item(s).
401
+
402
+ Raises:
403
+ ItemNotFoundError: If item(s) not found in pile.
404
+ """
405
+ _copy = self.model_copy(deep=True)
406
+ if other not in self:
407
+ raise ItemNotFoundError(other)
408
+
409
+ length = len(_copy)
410
+ if not _copy.exclude(other) or len(_copy) == length:
411
+ raise LionValueError("Item cannot be excluded from the pile.")
412
+ return _copy
413
+
414
+ def __iadd__(self, other: T) -> "Pile":
415
+ """
416
+ Include item(s) in the current pile in place using `+=`.
417
+
418
+ Modifies the current pile in-place by including item(s). Returns
419
+ the modified pile.
420
+
421
+ Args:
422
+ other: Item(s) to include. Can be single item or collection.
423
+ """
424
+
425
+ return self + other
426
+
427
+ def __isub__(self, other: LionIDable) -> "Pile":
428
+ """
429
+ Exclude item(s) from the current pile using `-=`.
430
+
431
+ Modifies the current pile in-place by excluding item(s). Returns
432
+ the modified pile.
433
+
434
+ Args:
435
+ other: Item(s) to exclude. Can be single item or collection.
436
+
437
+ Returns:
438
+ Modified pile after excluding item(s).
439
+ """
440
+ return self - other
441
+
442
+ def __radd__(self, other: T) -> "Pile":
443
+ return other + self
444
+
445
+ def size(self) -> int:
446
+ """Return the total size of the pile."""
447
+ return sum([len(i) for i in self])
448
+
449
+ def insert(self, index, item):
450
+ """
451
+ Insert item(s) at specific position.
452
+
453
+ Inserts item(s) at specified index. Index must be integer.
454
+ Raises `IndexError` if index out of range.
455
+
456
+ Args:
457
+ index: Index to insert item(s). Must be integer.
458
+ item: Item(s) to insert. Can be single item or collection.
459
+
460
+ Raises:
461
+ ValueError: If index not an integer.
462
+ IndexError: If index out of range.
463
+ """
464
+ if not isinstance(index, int):
465
+ raise ValueError("Index must be an integer for pile insertion.")
466
+ item = self._validate_pile(item)
467
+ for k, v in item.items():
468
+ self.order.insert(index, k)
469
+ self.pile[k] = v
470
+
471
+ def append(self, item: T):
472
+ """
473
+ Append item to end of pile.
474
+
475
+ Appends item to end of pile. If item is `Pile`, added as single
476
+ item, preserving structure. Only way to add `Pile` into another.
477
+ Other methods assume pile as container only.
478
+
479
+ Args:
480
+ item: Item to append. Can be any object, including `Pile`.
481
+ """
482
+ self.pile[item.ln_id] = item
483
+ self.order.append(item.ln_id)
484
+
485
+ def keys(self):
486
+ """Yield the keys of the items in the pile."""
487
+ return self.order
488
+
489
+ def values(self):
490
+ """Yield the values of the items in the pile."""
491
+ yield from (self.pile.get(i) for i in self.order)
492
+
493
+ def items(self):
494
+ """
495
+ Yield the items in the pile as (key, value) pairs.
496
+
497
+ Yields:
498
+ tuple: A tuple containing the key and value of each item in the pile.
499
+ """
500
+ yield from ((i, self.pile.get(i)) for i in self.order)
501
+
502
+ @field_validator("order", mode="before")
503
+ def _validate_order(cls, value):
504
+ return _validate_order(value)
505
+
506
+ def _validate_item_type(self, value):
507
+ """
508
+ Validate the item type for the pile.
509
+
510
+ Ensures that the provided item type is a subclass of Element or iModel.
511
+ Raises an error if the validation fails.
512
+
513
+ Args:
514
+ value: The item type to validate. Can be a single type or a list of types.
515
+
516
+ Returns:
517
+ set: A set of validated item types.
518
+
519
+ Raises:
520
+ LionTypeError: If an invalid item type is provided.
521
+ LionValueError: If duplicate item types are detected.
522
+ """
523
+ if value is None:
524
+ return None
525
+
526
+ value = to_list_type(value)
527
+
528
+ for i in value:
529
+ if not isinstance(i, (type(Element), type(iModel))):
530
+ raise LionTypeError(
531
+ "Invalid item type. Expected a subclass of Component."
532
+ )
533
+
534
+ if len(value) != len(set(value)):
535
+ raise LionValueError("Detected duplicated item types in item_type.")
536
+
537
+ if len(value) > 0:
538
+ return set(value)
539
+
540
+ def _validate_pile(
541
+ self,
542
+ value,
543
+ ):
544
+ if value == {}:
545
+ return value
546
+
547
+ if isinstance(value, Component):
548
+ return {value.ln_id: value}
549
+
550
+ if self.use_obj:
551
+ if not isinstance(value, list):
552
+ value = [value]
553
+ if isinstance(value[0], (Record, Ordering)):
554
+ return {getattr(i, "ln_id"): i for i in value}
555
+
556
+ value = to_list_type(value)
557
+ if getattr(self, "item_type", None) is not None:
558
+ for i in value:
559
+ if not type(i) in self.item_type:
560
+ raise LionTypeError(
561
+ f"Invalid item type in pile. Expected {self.item_type}"
562
+ )
563
+
564
+ if isinstance(value, list):
565
+ if len(value) == 1:
566
+ if isinstance(value[0], dict) and value[0] != {}:
567
+ k = list(value[0].keys())[0]
568
+ v = value[0][k]
569
+ return {k: v}
570
+
571
+ # [item]
572
+ k = getattr(value[0], "ln_id", None)
573
+ if k:
574
+ return {k: value[0]}
575
+
576
+ return {i.ln_id: i for i in value}
577
+
578
+ raise LionValueError("Invalid pile value")
579
+
580
+ def to_df(self):
581
+ """Return the pile as a DataFrame."""
582
+ dicts_ = []
583
+ for i in self.values():
584
+ _dict = i.to_dict()
585
+ if _dict.get("embedding", None):
586
+ _dict["embedding"] = str(_dict.get("embedding"))
587
+ dicts_.append(_dict)
588
+ return to_df(dicts_)
589
+
590
+ def create_index(self, index_type="llama_index", **kwargs):
591
+ """
592
+ Create an index for the pile.
593
+
594
+ Args:
595
+ index_type (str): The type of index to use. Default is "llama_index".
596
+ **kwargs: Additional keyword arguments for the index creation.
597
+
598
+ Returns:
599
+ The created index.
600
+
601
+ Raises:
602
+ ValueError: If an invalid index type is provided.
603
+ """
604
+ if index_type == "llama_index":
605
+ from lionagi.integrations.bridge import LlamaIndexBridge
606
+
607
+ index_nodes = None
608
+
609
+ try:
610
+ index_nodes = [i.to_llama_index_node() for i in self]
611
+ except AttributeError:
612
+ raise LionTypeError(
613
+ "Invalid item type. Expected a subclass of Component."
614
+ )
615
+
616
+ self.index = LlamaIndexBridge.index(index_nodes, **kwargs)
617
+ return self.index
618
+
619
+ raise ValueError("Invalid index type")
620
+
621
+ def create_query_engine(self, index_type="llama_index", engine_kwargs={}, **kwargs):
622
+ """
623
+ Create a query engine for the pile.
624
+
625
+ Args:
626
+ index_type (str): The type of index to use. Default is "llama_index".
627
+ engine_kwargs (dict): Additional keyword arguments for the engine.
628
+ **kwargs: Additional keyword arguments for the index creation.
629
+
630
+ Raises:
631
+ ValueError: If an invalid index type is provided.
632
+ """
633
+ if index_type == "llama_index":
634
+ if "node_postprocessor" in kwargs:
635
+ engine_kwargs["node_postprocessor"] = kwargs.pop("node_postprocessor")
636
+ if "llm" in kwargs:
637
+ engine_kwargs["llm"] = kwargs.pop("llm")
638
+ if not self.index:
639
+ self.create_index(index_type, **kwargs)
640
+ query_engine = self.index.as_query_engine(**engine_kwargs)
641
+ self.engines["query"] = query_engine
642
+ else:
643
+ raise ValueError("Invalid index type")
644
+
645
+ def create_chat_engine(self, index_type="llama_index", engine_kwargs={}, **kwargs):
646
+ """
647
+ Create a chat engine for the pile.
648
+
649
+ Args:
650
+ index_type (str): The type of index to use. Default is "llama_index".
651
+ engine_kwargs (dict): Additional keyword arguments for the engine.
652
+ **kwargs: Additional keyword arguments for the index creation.
653
+
654
+ Raises:
655
+ ValueError: If an invalid index type is provided.
656
+ """
657
+ if index_type == "llama_index":
658
+ if "node_postprocessor" in kwargs:
659
+ engine_kwargs["node_postprocessor"] = kwargs.pop("node_postprocessor")
660
+ if "llm" in kwargs:
661
+ engine_kwargs["llm"] = kwargs.pop("llm")
662
+ if not self.index:
663
+ self.create_index(index_type, **kwargs)
664
+ query_engine = self.index.as_chat_engine(**engine_kwargs)
665
+ self.engines["chat"] = query_engine
666
+ else:
667
+ raise ValueError("Invalid index type")
668
+
669
+ async def query_pile(self, query, engine_kwargs={}, **kwargs):
670
+ """
671
+ Query the pile using the created query engine.
672
+
673
+ Args:
674
+ query (str): The query to send.
675
+ engine_kwargs (dict): Additional keyword arguments for the engine.
676
+ **kwargs: Additional keyword arguments for the query.
677
+
678
+ Returns:
679
+ str: The response from the query engine.
680
+ """
681
+ if not self.engines.get("query", None):
682
+ self.create_query_engine(**engine_kwargs)
683
+ response = await self.engines["query"].aquery(query, **kwargs)
684
+ self.query_response.append(response)
685
+ return str(response)
686
+
687
+ async def chat_pile(self, query, engine_kwargs={}, **kwargs):
688
+ """
689
+ Chat with the pile using the created chat engine.
690
+
691
+ Args:
692
+ query (str): The query to send.
693
+ engine_kwargs (dict): Additional keyword arguments for the engine.
694
+ **kwargs: Additional keyword arguments for the query.
695
+
696
+ Returns:
697
+ str: The response from the chat engine.
698
+ """
699
+ if not self.engines.get("chat", None):
700
+ self.create_chat_engine(**engine_kwargs)
701
+ response = await self.engines["chat"].achat(query, **kwargs)
702
+ self.query_response.append(response)
703
+ return str(response)
704
+
705
+ async def embed_pile(
706
+ self, imodel=None, field="content", embed_kwargs={}, verbose=True, **kwargs
707
+ ):
708
+ """
709
+ Embed the items in the pile.
710
+
711
+ Args:
712
+ imodel: The embedding model to use.
713
+ field (str): The field to embed. Default is "content".
714
+ embed_kwargs (dict): Additional keyword arguments for the embedding.
715
+ verbose (bool): Whether to print verbose messages. Default is True.
716
+ **kwargs: Additional keyword arguments for the embedding.
717
+
718
+ Raises:
719
+ ModelLimitExceededError: If the model limit is exceeded.
720
+ """
721
+ from .model import iModel
722
+
723
+ imodel = imodel or iModel(endpoint="embeddings", **kwargs)
724
+
725
+ max_concurrency = kwargs.get("max_concurrency", None) or 100
726
+
727
+ @cd.max_concurrency(max_concurrency)
728
+ async def _embed_item(item):
729
+ try:
730
+ return await imodel.embed_node(item, field=field, **embed_kwargs)
731
+ except ModelLimitExceededError:
732
+ pass
733
+ return None
734
+
735
+ await alcall(list(self), _embed_item)
736
+
737
+ a = len([i for i in self if "embedding" in i._all_fields])
738
+ if len(self) > a and verbose:
739
+ print(
740
+ f"Successfully embedded {a}/{len(self)} items, Failed to embed {len(self) - a}/{len(self)} items"
741
+ )
742
+ return
743
+
744
+ print(f"Successfully embedded all {a}/{a} items")
745
+
746
+ def to_csv(self, file_name, **kwargs):
747
+ """
748
+ Save the pile to a CSV file.
749
+
750
+ Args:
751
+ file_name (str): The name of the CSV file.
752
+ **kwargs: Additional keyword arguments for the CSV writer.
753
+ """
754
+ self.to_df().to_csv(file_name, index=False, **kwargs)
755
+
756
+ @classmethod
757
+ def from_csv(cls, file_name, **kwargs):
758
+ """
759
+ Load a pile from a CSV file.
760
+
761
+ Args:
762
+ file_name (str): The name of the CSV file.
763
+ **kwargs: Additional keyword arguments for the CSV reader.
764
+
765
+ Returns:
766
+ Pile: The loaded pile.
767
+ """
768
+ from pandas import read_csv
769
+
770
+ df = read_csv(file_name, **kwargs)
771
+ items = Component.from_obj(df)
772
+ return cls(items)
773
+
774
+ @classmethod
775
+ def from_df(cls, df):
776
+ """
777
+ Load a pile from a DataFrame.
778
+
779
+ Args:
780
+ df (DataFrame): The DataFrame to load.
781
+
782
+ Returns:
783
+ Pile: The loaded pile.
784
+ """
785
+ items = Component.from_obj(df)
786
+ return cls(items)
787
+
788
+ def as_query_tool(
789
+ self,
790
+ index_type="llama_index",
791
+ query_type="query",
792
+ name=None,
793
+ guidance=None,
794
+ query_description=None,
795
+ **kwargs,
796
+ ):
797
+ """
798
+ Create a query tool for the pile.
799
+
800
+ Args:
801
+ index_type (str): The type of index to use. Default is "llama_index".
802
+ query_type (str): The type of query engine to use. Default is "query".
803
+ name (str): The name of the query tool. Default is "query".
804
+ guidance (str): The guidance for the query tool.
805
+ query_description (str): The description of the query parameter.
806
+ **kwargs: Additional keyword arguments for the query engine.
807
+
808
+ Returns:
809
+ Tool: The created query tool.
810
+ """
811
+ if not self.engines.get(query_type, None):
812
+ if query_type == "query":
813
+ self.create_query_engine(index_type=index_type, **kwargs)
814
+ elif query_type == "chat":
815
+ self.create_chat_engine(index_type=index_type, **kwargs)
816
+
817
+ from lionagi.core.action.tool_manager import func_to_tool
818
+
819
+ if not guidance:
820
+ if query_type == "query":
821
+ guidance = "Query a QA bot"
822
+ elif query_type == "chat":
823
+ guidance = "Chat with a QA bot"
824
+
825
+ if not query_description:
826
+ if query_type == "query":
827
+ query_description = "The query to send"
828
+ elif query_type == "chat":
829
+ query_description = "The message to send"
830
+
831
+ async def query(query: str):
832
+ if query_type == "query":
833
+ return await self.query_pile(query, **kwargs)
834
+
835
+ elif query_type == "chat":
836
+ return await self.chat_pile(query, **kwargs)
837
+
838
+ name = name or "query"
839
+ tool = func_to_tool(query)[0]
840
+ tool.schema_["function"]["name"] = name
841
+ tool.schema_["function"]["description"] = guidance
842
+ tool.schema_["function"]["parameters"]["properties"]["query"][
843
+ "description"
844
+ ] = query_description
845
+ self.tools[query_type] = tool
846
+ return self.tools[query_type]
847
+
848
+ def __list__(self):
849
+ """
850
+ Get a list of the items in the pile.
851
+
852
+ Returns:
853
+ list: The items in the pile.
854
+ """
855
+ return list(self.pile.values())
856
+
857
+ def __str__(self):
858
+ """
859
+ Get the string representation of the pile.
860
+
861
+ Returns:
862
+ str: The string representation of the pile.
863
+ """
864
+ return self.to_df().__str__()
865
+
866
+ def __repr__(self):
867
+ """
868
+ Get the representation of the pile.
869
+
870
+ Returns:
871
+ str: The representation of the pile.
872
+ """
873
+ return self.to_df().__repr__()
874
+
875
+
876
+ def pile(
877
+ items: Iterable[T] | None = None,
878
+ item_type: set[Type] | None = None,
879
+ order=None,
880
+ use_obj=None,
881
+ csv_file=None,
882
+ df=None,
883
+ **kwargs,
884
+ ) -> Pile[T]:
885
+ """
886
+ Create a new Pile instance.
887
+
888
+ This function provides various ways to create a Pile instance:
889
+ - Directly from items
890
+ - From a CSV file
891
+ - From a DataFrame
892
+
893
+ Args:
894
+ items (Iterable[T] | None): The items to include in the pile.
895
+ item_type (set[Type] | None): The allowed types of items in the pile.
896
+ order (list[str] | None): The order of items.
897
+ use_obj (bool | None): Whether to treat Record and Ordering as objects.
898
+ csv_file (str | None): The path to a CSV file to load items from.
899
+ df (DataFrame | None): A DataFrame to load items from.
900
+ **kwargs: Additional keyword arguments for loading from CSV or DataFrame.
901
+
902
+ Returns:
903
+ Pile[T]: A new Pile instance.
904
+
905
+ Raises:
906
+ ValueError: If invalid arguments are provided.
907
+ """
908
+ if csv_file:
909
+ return Pile.from_csv(csv_file, **kwargs)
910
+ if df:
911
+ return Pile.from_df(df)
912
+
913
+ return Pile(items, item_type, order, use_obj)