lionagi 0.0.111__py3-none-any.whl → 0.0.113__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- lionagi/__init__.py +7 -2
- lionagi/bridge/__init__.py +7 -0
- lionagi/bridge/langchain.py +131 -0
- lionagi/bridge/llama_index.py +157 -0
- lionagi/configs/__init__.py +7 -0
- lionagi/configs/oai_configs.py +49 -0
- lionagi/configs/openrouter_config.py +49 -0
- lionagi/core/__init__.py +15 -0
- lionagi/{session/conversation.py → core/conversations.py} +10 -17
- lionagi/core/flows.py +1 -0
- lionagi/core/instruction_sets.py +1 -0
- lionagi/{session/message.py → core/messages.py} +5 -5
- lionagi/core/sessions.py +262 -0
- lionagi/datastore/__init__.py +1 -0
- lionagi/datastore/chroma.py +1 -0
- lionagi/datastore/deeplake.py +1 -0
- lionagi/datastore/elasticsearch.py +1 -0
- lionagi/datastore/lantern.py +1 -0
- lionagi/datastore/pinecone.py +1 -0
- lionagi/datastore/postgres.py +1 -0
- lionagi/datastore/qdrant.py +1 -0
- lionagi/loader/__init__.py +12 -0
- lionagi/loader/chunker.py +157 -0
- lionagi/loader/reader.py +124 -0
- lionagi/objs/__init__.py +7 -0
- lionagi/objs/messenger.py +163 -0
- lionagi/objs/tool_registry.py +247 -0
- lionagi/schema/__init__.py +11 -0
- lionagi/schema/base_condition.py +1 -0
- lionagi/schema/base_schema.py +239 -0
- lionagi/schema/base_tool.py +9 -0
- lionagi/schema/data_logger.py +94 -0
- lionagi/services/__init__.py +14 -0
- lionagi/services/anthropic.py +1 -0
- lionagi/services/anyscale.py +0 -0
- lionagi/services/azure.py +1 -0
- lionagi/{api/oai_service.py → services/base_api_service.py} +74 -148
- lionagi/services/bedrock.py +0 -0
- lionagi/services/chatcompletion.py +48 -0
- lionagi/services/everlyai.py +0 -0
- lionagi/services/gemini.py +0 -0
- lionagi/services/gpt4all.py +0 -0
- lionagi/services/huggingface.py +0 -0
- lionagi/services/litellm.py +1 -0
- lionagi/services/localai.py +0 -0
- lionagi/services/mistralai.py +0 -0
- lionagi/services/oai.py +34 -0
- lionagi/services/ollama.py +1 -0
- lionagi/services/openllm.py +0 -0
- lionagi/services/openrouter.py +32 -0
- lionagi/services/perplexity.py +0 -0
- lionagi/services/predibase.py +0 -0
- lionagi/services/rungpt.py +0 -0
- lionagi/services/service_objs.py +282 -0
- lionagi/services/vllm.py +0 -0
- lionagi/services/xinference.py +0 -0
- lionagi/structure/__init__.py +7 -0
- lionagi/structure/relationship.py +128 -0
- lionagi/structure/structure.py +160 -0
- lionagi/tests/__init__.py +0 -0
- lionagi/tests/test_flatten_util.py +426 -0
- lionagi/tools/__init__.py +0 -0
- lionagi/tools/coder.py +1 -0
- lionagi/tools/planner.py +1 -0
- lionagi/tools/prompter.py +1 -0
- lionagi/tools/sandbox.py +1 -0
- lionagi/tools/scorer.py +1 -0
- lionagi/tools/summarizer.py +1 -0
- lionagi/tools/validator.py +1 -0
- lionagi/utils/__init__.py +46 -8
- lionagi/utils/api_util.py +63 -416
- lionagi/utils/call_util.py +347 -0
- lionagi/utils/flat_util.py +540 -0
- lionagi/utils/io_util.py +102 -0
- lionagi/utils/load_utils.py +190 -0
- lionagi/utils/sys_util.py +85 -660
- lionagi/utils/tool_util.py +82 -199
- lionagi/utils/type_util.py +81 -0
- lionagi/version.py +1 -1
- {lionagi-0.0.111.dist-info → lionagi-0.0.113.dist-info}/METADATA +44 -15
- lionagi-0.0.113.dist-info/RECORD +84 -0
- lionagi/api/__init__.py +0 -8
- lionagi/api/oai_config.py +0 -16
- lionagi/session/__init__.py +0 -7
- lionagi/session/session.py +0 -380
- lionagi/utils/doc_util.py +0 -331
- lionagi/utils/log_util.py +0 -86
- lionagi-0.0.111.dist-info/RECORD +0 -20
- {lionagi-0.0.111.dist-info → lionagi-0.0.113.dist-info}/LICENSE +0 -0
- {lionagi-0.0.111.dist-info → lionagi-0.0.113.dist-info}/WHEEL +0 -0
- {lionagi-0.0.111.dist-info → lionagi-0.0.113.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,540 @@
|
|
1
|
+
from typing import Dict, Iterable, List, Any, Callable, Generator, Tuple
|
2
|
+
|
3
|
+
|
4
|
+
def flatten_dict(d: Dict, parent_key: str = '', sep: str = '_') -> Dict:
|
5
|
+
"""
|
6
|
+
Flattens a nested dictionary by concatenating keys.
|
7
|
+
|
8
|
+
This function recursively flattens a nested dictionary by concatenating
|
9
|
+
the keys of nested dictionaries with a separator. The default separator
|
10
|
+
is an underscore (_).
|
11
|
+
|
12
|
+
Parameters:
|
13
|
+
d (dict): The dictionary to flatten.
|
14
|
+
|
15
|
+
parent_key (str, optional): The base key to use for the current level of recursion.
|
16
|
+
Defaults to an empty string, meaning no parent key, key cannot be a number
|
17
|
+
|
18
|
+
sep (str, optional): The separator to use when concatenating keys.
|
19
|
+
Defaults to an underscore (_).
|
20
|
+
|
21
|
+
Returns:
|
22
|
+
dict: A new dictionary with flattened keys and corresponding values.
|
23
|
+
"""
|
24
|
+
items = []
|
25
|
+
for k, v in d.items():
|
26
|
+
new_key = f"{parent_key}{sep}{k}" if parent_key else k
|
27
|
+
if isinstance(v, dict):
|
28
|
+
items.extend(flatten_dict(v, new_key, sep=sep).items())
|
29
|
+
else:
|
30
|
+
items.append((new_key, v))
|
31
|
+
return dict(items)
|
32
|
+
|
33
|
+
def flatten_list(l: List, dropna: bool = True) -> List:
|
34
|
+
"""
|
35
|
+
Flattens a nested list into a single list of values.
|
36
|
+
|
37
|
+
The function iterates over each element in the provided list. If an
|
38
|
+
element is a list itself, the function is called recursively to flatten
|
39
|
+
it. If the element is not a list, it is appended to the result list.
|
40
|
+
Optionally, None values can be dropped from the result list.
|
41
|
+
|
42
|
+
Parameters:
|
43
|
+
l (list): The list to flatten, which may contain nested lists.
|
44
|
+
|
45
|
+
dropna (bool, optional): Whether to exclude None values from the result list.
|
46
|
+
Defaults to True.
|
47
|
+
|
48
|
+
Returns:
|
49
|
+
list: A new flattened list with or without None values based on the dropna parameter.
|
50
|
+
"""
|
51
|
+
flat_list = []
|
52
|
+
for i in l:
|
53
|
+
if isinstance(i, list):
|
54
|
+
flat_list.extend(flatten_list(i, dropna))
|
55
|
+
elif i is not None or not dropna:
|
56
|
+
flat_list.append(i)
|
57
|
+
return flat_list
|
58
|
+
|
59
|
+
def change_separator(flat_dict, current_sep, new_sep):
|
60
|
+
"""
|
61
|
+
Changes the separator in the keys of a flat dictionary.
|
62
|
+
|
63
|
+
Parameters:
|
64
|
+
flat_dict (dict): The dictionary with keys containing the current separator.
|
65
|
+
current_sep (str): The current separator used in the dictionary keys.
|
66
|
+
new_sep (str): The new separator to replace the current separator in the dictionary keys.
|
67
|
+
|
68
|
+
Returns:
|
69
|
+
dict: A new dictionary with the separators in the keys replaced.
|
70
|
+
"""
|
71
|
+
return {
|
72
|
+
k.replace(current_sep, new_sep): v
|
73
|
+
for k, v in flat_dict.items()
|
74
|
+
}
|
75
|
+
|
76
|
+
def unflatten_dict(flat_dict: Dict, sep: str = '_') -> Dict:
|
77
|
+
"""
|
78
|
+
Unflattens a dictionary where keys are strings that represent nested dictionary paths separated by 'sep'.
|
79
|
+
|
80
|
+
Args:
|
81
|
+
flat_dict (dict): A dictionary with keys as strings that represent paths.
|
82
|
+
sep (str): The separator used in the keys of the flat dictionary.
|
83
|
+
|
84
|
+
Returns:
|
85
|
+
dict: A nested dictionary unflattened from the keys of the input dictionary.
|
86
|
+
|
87
|
+
Raises:
|
88
|
+
ValueError: If there are conflicting keys in the path.
|
89
|
+
|
90
|
+
Example:
|
91
|
+
>>> unflatten_dict({'a_0': 1, 'a_1': 2, 'b_x': 'X', 'b_y': 'Y'})
|
92
|
+
{'a': [1, 2], 'b': {'x': 'X', 'y': 'Y'}}
|
93
|
+
"""
|
94
|
+
result = {}
|
95
|
+
for flat_key, value in flat_dict.items():
|
96
|
+
parts = flat_key.split(sep)
|
97
|
+
d = result
|
98
|
+
for part in parts[:-1]:
|
99
|
+
part = int(part) if part.isdigit() else part
|
100
|
+
if part not in d:
|
101
|
+
if isinstance(d, list):
|
102
|
+
d.append({})
|
103
|
+
d = d[-1]
|
104
|
+
else:
|
105
|
+
d[part] = {}
|
106
|
+
d = d[part] if not isinstance(d, list) else d[-1]
|
107
|
+
last_part = parts[-1]
|
108
|
+
last_part = int(last_part) if last_part.isdigit() else last_part
|
109
|
+
if isinstance(d, list):
|
110
|
+
d.append(value)
|
111
|
+
elif last_part in d:
|
112
|
+
raise ValueError(f"Conflicting keys detected. Key part '{last_part}' is already present.")
|
113
|
+
else:
|
114
|
+
d[last_part] = value
|
115
|
+
|
116
|
+
# Convert dictionaries with contiguous integer keys starting from 0 to lists
|
117
|
+
def dict_to_list(d: dict) -> dict:
|
118
|
+
"""
|
119
|
+
Converts dictionaries with contiguous integer keys starting from 0 into lists.
|
120
|
+
|
121
|
+
Args:
|
122
|
+
d (dict): A dictionary that may have integer keys suitable for conversion to a list.
|
123
|
+
|
124
|
+
Returns:
|
125
|
+
dict or list: The input dictionary or a list if the dictionary keys match the criteria.
|
126
|
+
"""
|
127
|
+
if isinstance(d, dict) and all(isinstance(k, int) and k >= 0 for k in d.keys()):
|
128
|
+
keys = sorted(d.keys())
|
129
|
+
if keys == list(range(len(keys))):
|
130
|
+
return [dict_to_list(d[k]) for k in keys]
|
131
|
+
if isinstance(d, dict):
|
132
|
+
return {k: dict_to_list(v) for k, v in d.items()}
|
133
|
+
return d
|
134
|
+
|
135
|
+
result = dict_to_list(result)
|
136
|
+
return result
|
137
|
+
|
138
|
+
def is_flattenable(obj: Any) -> bool:
|
139
|
+
"""
|
140
|
+
Determines if the given object contains nested dictionaries or lists, making it suitable for flattening.
|
141
|
+
|
142
|
+
Args:
|
143
|
+
obj (object): The object to check for flattenable structures.
|
144
|
+
|
145
|
+
Returns:
|
146
|
+
bool: True if the object can be flattened (contains nested dicts or lists), False otherwise.
|
147
|
+
"""
|
148
|
+
if isinstance(obj, dict):
|
149
|
+
return any(isinstance(v, (dict, list)) for v in obj.values())
|
150
|
+
elif isinstance(obj, list):
|
151
|
+
return any(isinstance(i, (dict, list)) for i in obj)
|
152
|
+
return False
|
153
|
+
|
154
|
+
def flatten_with_custom_logic(obj, logic_func=None, parent_key='', sep='_'):
|
155
|
+
"""
|
156
|
+
Recursively flattens a nested dictionary or list and applies custom logic to the keys and values.
|
157
|
+
|
158
|
+
Parameters:
|
159
|
+
obj (dict | list): The dictionary or list to flatten.
|
160
|
+
logic_func (callable, optional): A function that takes four arguments (parent_key, key, value, sep)
|
161
|
+
and returns a tuple (new_key, new_value) after applying custom logic.
|
162
|
+
parent_key (str): The base key to use for creating new keys.
|
163
|
+
sep (str): The separator to use when joining nested keys.
|
164
|
+
|
165
|
+
Returns:
|
166
|
+
dict: A flattened dictionary with keys representing the nested paths and values from the original object.
|
167
|
+
|
168
|
+
Example Usage:
|
169
|
+
>>> sample_dict = {'a': 1, 'b': {'c': 2, 'd': {'e': 3}}}
|
170
|
+
>>> flatten_with_custom_logic(sample_dict)
|
171
|
+
{'a': 1, 'b_c': 2, 'b_d_e': 3}
|
172
|
+
|
173
|
+
>>> sample_list = [1, 2, [3, 4]]
|
174
|
+
>>> flatten_with_custom_logic(sample_list)
|
175
|
+
{'0': 1, '1': 2, '2_0': 3, '2_1': 4}
|
176
|
+
|
177
|
+
>>> def custom_logic(parent_key, key, value, sep='_'):
|
178
|
+
... new_key = f"{parent_key}{sep}{key}".upper() if parent_key else key.upper()
|
179
|
+
... return new_key, value * 2 if isinstance(value, int) else value
|
180
|
+
>>> flatten_with_custom_logic(sample_dict, custom_logic)
|
181
|
+
{'A': 2, 'B_C': 4, 'B_D_E': 6}
|
182
|
+
"""
|
183
|
+
items = {}
|
184
|
+
if isinstance(obj, dict):
|
185
|
+
for k, v in obj.items():
|
186
|
+
new_key = f"{parent_key}{sep}{k}" if parent_key else k
|
187
|
+
if isinstance(v, (dict, list)):
|
188
|
+
if v: # Check if the dictionary or list is not empty
|
189
|
+
items.update(flatten_with_custom_logic(v, logic_func, new_key, sep))
|
190
|
+
else: # Handle empty dictionaries and lists
|
191
|
+
items[new_key] = None
|
192
|
+
else:
|
193
|
+
if logic_func:
|
194
|
+
new_key, new_value = logic_func(parent_key, k, v, sep=sep)
|
195
|
+
else:
|
196
|
+
new_value = v
|
197
|
+
items[new_key] = new_value
|
198
|
+
elif isinstance(obj, list):
|
199
|
+
for i, item in enumerate(obj):
|
200
|
+
new_key = f"{parent_key}{sep}{str(i)}" if parent_key else str(i) # Cast index to string
|
201
|
+
if isinstance(item, (dict, list)):
|
202
|
+
if item: # Check if the dictionary or list is not empty
|
203
|
+
items.update(flatten_with_custom_logic(item, logic_func, new_key, sep))
|
204
|
+
else: # Handle empty lists
|
205
|
+
items[new_key] = None
|
206
|
+
else:
|
207
|
+
if logic_func:
|
208
|
+
new_key, new_value = logic_func(parent_key, str(i), item, sep=sep)
|
209
|
+
else:
|
210
|
+
new_value = item
|
211
|
+
items[new_key] = new_value
|
212
|
+
return items
|
213
|
+
|
214
|
+
def dynamic_flatten(obj, parent_key='', sep='_', max_depth=None, current_depth=0):
|
215
|
+
"""
|
216
|
+
Recursively flattens a nested dictionary or list, while allowing a maximum depth and custom separators.
|
217
|
+
|
218
|
+
Parameters:
|
219
|
+
obj (dict | list): The dictionary or list to flatten.
|
220
|
+
parent_key (str): The base key to use for creating new keys.
|
221
|
+
sep (str): The separator to use when joining nested keys.
|
222
|
+
max_depth (int, optional): The maximum depth to flatten.
|
223
|
+
current_depth (int): The current depth in the recursive call (used internally).
|
224
|
+
|
225
|
+
Returns:
|
226
|
+
dict: A flattened dictionary with keys representing the nested paths and values from the original object.
|
227
|
+
|
228
|
+
Raises:
|
229
|
+
TypeError: If the input object is neither a dictionary nor a list.
|
230
|
+
|
231
|
+
Example Usage:
|
232
|
+
>>> sample_dict = {'a': 1, 'b': {'c': 2, 'd': {'e': 3}}}
|
233
|
+
>>> dynamic_flatten(sample_dict)
|
234
|
+
{'a': 1, 'b_c': 2, 'b_d_e': 3}
|
235
|
+
|
236
|
+
>>> sample_list = [1, 2, [3, 4]]
|
237
|
+
>>> dynamic_flatten(sample_list)
|
238
|
+
{'0': 1, '1': 2, '2_0': 3, '2_1': 4}
|
239
|
+
|
240
|
+
>>> dynamic_flatten(sample_dict, max_depth=1)
|
241
|
+
{'a': 1, 'b_c': 2, 'b_d': {'e': 3}}
|
242
|
+
|
243
|
+
>>> dynamic_flatten(sample_dict, sep='.')
|
244
|
+
{'a': 1, 'b.c': 2, 'b.d.e': 3}
|
245
|
+
"""
|
246
|
+
items = []
|
247
|
+
if isinstance(obj, dict):
|
248
|
+
if not obj and parent_key:
|
249
|
+
items.append((parent_key, obj))
|
250
|
+
for k, v in obj.items():
|
251
|
+
new_key = f"{parent_key}{sep}{k}" if parent_key else k
|
252
|
+
if isinstance(v, (dict, list)) and (max_depth is None or current_depth < max_depth):
|
253
|
+
items.extend(dynamic_flatten(v, new_key, sep, max_depth, current_depth + 1).items())
|
254
|
+
else:
|
255
|
+
items.append((new_key, v))
|
256
|
+
elif isinstance(obj, list):
|
257
|
+
if not obj and parent_key:
|
258
|
+
items.append((parent_key, obj))
|
259
|
+
for i, item in enumerate(obj):
|
260
|
+
new_key = f"{parent_key}{sep}{i}" if parent_key else str(i)
|
261
|
+
if isinstance(item, (dict, list)) and (max_depth is None or current_depth < max_depth):
|
262
|
+
items.extend(dynamic_flatten(item, new_key, sep, max_depth, current_depth + 1).items())
|
263
|
+
else:
|
264
|
+
items.append((new_key, item))
|
265
|
+
else:
|
266
|
+
raise TypeError('Input object must be a dictionary or a list')
|
267
|
+
|
268
|
+
return dict(items)
|
269
|
+
|
270
|
+
# def dynamic_unflatten_dict(flat_dict, sep='_', custom_logic=None, max_depth=None):
|
271
|
+
# """
|
272
|
+
# Unflattens a dictionary with flat keys into a nested dictionary or list.
|
273
|
+
|
274
|
+
# :param flat_dict: A dictionary with flat keys that need to be nested.
|
275
|
+
# :param sep: The separator used in the flat keys (default is '_').
|
276
|
+
# :param custom_logic: A function that customizes the processing of keys.
|
277
|
+
# It takes a key part as input and returns the transformed key part.
|
278
|
+
# :param max_depth: The maximum depth to which the dictionary should be nested.
|
279
|
+
# If None, there is no maximum depth.
|
280
|
+
# :return: A nested dictionary or list based on the flat input dictionary.
|
281
|
+
|
282
|
+
# The function dynamically unflattens a dictionary with keys that represent nested paths.
|
283
|
+
# For example, the flat dictionary {'a_b_c': 1, 'a_b_d': 2} would be unflattened to
|
284
|
+
# {'a': {'b': {'c': 1, 'd': 2}}}. If the keys can be converted to integers and suggest list indices,
|
285
|
+
# the function produces lists instead of dictionaries. For example,
|
286
|
+
# {'0_a': 'foo', '1_b': 'bar'} becomes [{'a': 'foo'}, {'b': 'bar'}].
|
287
|
+
# """
|
288
|
+
|
289
|
+
# def handle_list_insert(sub_obj, part, value):
|
290
|
+
# # Ensure part index exists in the list, fill gaps with None
|
291
|
+
# while len(sub_obj) <= part:
|
292
|
+
# sub_obj.append(None)
|
293
|
+
|
294
|
+
# sub_obj[part] = value
|
295
|
+
|
296
|
+
# def insert(sub_obj, parts, value, max_depth, current_depth=0):
|
297
|
+
# for part in parts[:-1]:
|
298
|
+
# # Stop nesting further if max_depth is reached
|
299
|
+
# if max_depth is not None and current_depth >= max_depth:
|
300
|
+
# sub_obj[part] = {parts[-1]: value}
|
301
|
+
# return
|
302
|
+
# # Handle integer parts for list insertion
|
303
|
+
# if isinstance(part, int):
|
304
|
+
# # Ensure part index exists in the list or dict, fill gaps with None
|
305
|
+
# while len(sub_obj) <= part:
|
306
|
+
# sub_obj.append(None)
|
307
|
+
# if sub_obj[part] is None:
|
308
|
+
# if current_depth < max_depth - 1 if max_depth else True:
|
309
|
+
# sub_obj[part] = [] if isinstance(parts[parts.index(part) + 1], int) else {}
|
310
|
+
# sub_obj = sub_obj[part]
|
311
|
+
# else:
|
312
|
+
# # Handle string parts for dictionary insertion
|
313
|
+
# if part not in sub_obj:
|
314
|
+
# sub_obj[part] = {}
|
315
|
+
# sub_obj = sub_obj[part]
|
316
|
+
# current_depth += 1
|
317
|
+
# # Insert the value at the last part
|
318
|
+
# last_part = parts[-1]
|
319
|
+
# if isinstance(last_part, int) and isinstance(sub_obj, list):
|
320
|
+
# handle_list_insert(sub_obj, last_part, value, max_depth, current_depth)
|
321
|
+
# else:
|
322
|
+
# sub_obj[last_part] = value
|
323
|
+
|
324
|
+
# unflattened = {}
|
325
|
+
# for composite_key, value in flat_dict.items():
|
326
|
+
# parts = composite_key.split(sep)
|
327
|
+
# if custom_logic:
|
328
|
+
# parts = [custom_logic(part) for part in parts]
|
329
|
+
# else:
|
330
|
+
# parts = [int(part) if (isinstance(part, int) or part.isdigit()) else part for part in parts]
|
331
|
+
# insert(unflattened, parts, value, max_depth)
|
332
|
+
|
333
|
+
# # Convert top-level dictionary to a list if all keys are integers
|
334
|
+
# if isinstance(unflattened, dict) and all(isinstance(k, int) for k in unflattened.keys()):
|
335
|
+
# max_index = max(unflattened.keys(), default=-1)
|
336
|
+
# return [unflattened.get(i) for i in range(max_index + 1)]
|
337
|
+
# # Return an empty dictionary instead of converting to a list if unflattened is empty
|
338
|
+
# if (unflattened == []) or (not unflattened):
|
339
|
+
# return {}
|
340
|
+
# return unflattened
|
341
|
+
|
342
|
+
def _insert_with_dict_handling(container, indices, value):
|
343
|
+
"""
|
344
|
+
Helper function to insert a value into a nested container based on a list of indices.
|
345
|
+
|
346
|
+
:param container: The container (list or dict) to insert the value into.
|
347
|
+
:param indices: A list of indices indicating the path to the insertion point.
|
348
|
+
:param value: The value to be inserted.
|
349
|
+
"""
|
350
|
+
for i, index in enumerate(indices[:-1]):
|
351
|
+
# Check if index is an integer and ensure the list is long enough
|
352
|
+
if isinstance(index, int):
|
353
|
+
# Extend the list if necessary
|
354
|
+
while len(container) <= index:
|
355
|
+
container.append(None)
|
356
|
+
|
357
|
+
# Create a nested container if the current position is None
|
358
|
+
if container[index] is None:
|
359
|
+
next_index = indices[i + 1]
|
360
|
+
|
361
|
+
# Determine the type of the next container based on the next index
|
362
|
+
container[index] = {} if isinstance(next_index, str) else []
|
363
|
+
|
364
|
+
# Update the reference to point to the nested container
|
365
|
+
container = container[index]
|
366
|
+
|
367
|
+
# If index is a string, work with dictionaries
|
368
|
+
else:
|
369
|
+
if index not in container:
|
370
|
+
container[index] = {}
|
371
|
+
container = container[index]
|
372
|
+
|
373
|
+
# Insert value at the last index
|
374
|
+
last_index = indices[-1]
|
375
|
+
if isinstance(last_index, int):
|
376
|
+
|
377
|
+
# Negative indices are not allowed in this context
|
378
|
+
if last_index < 0:
|
379
|
+
raise ValueError("Negative index is not allowed for list unflattening.")
|
380
|
+
|
381
|
+
while len(container) <= last_index:
|
382
|
+
container.append(None)
|
383
|
+
container[last_index] = value
|
384
|
+
|
385
|
+
else:
|
386
|
+
if last_index in container and isinstance(container[last_index], list):
|
387
|
+
raise ValueError("Overlapping keys are not allowed.")
|
388
|
+
container[last_index] = value
|
389
|
+
|
390
|
+
def unflatten_to_list(flat_dict: Dict[str, Any], sep: str = '_') -> List:
|
391
|
+
"""
|
392
|
+
Unflattens a dictionary with keys as string paths into a nested list structure.
|
393
|
+
|
394
|
+
:param flat_dict: The flat dictionary to unflatten.
|
395
|
+
:param sep: The separator used in the flat dictionary keys to indicate nesting.
|
396
|
+
:return: A nested list that represents the unflattened structure.
|
397
|
+
"""
|
398
|
+
result_list = []
|
399
|
+
for flat_key, value in flat_dict.items():
|
400
|
+
indices = [int(p) if p.lstrip('-').isdigit() else p for p in flat_key.split(sep)]
|
401
|
+
_insert_with_dict_handling(result_list, indices, value)
|
402
|
+
return result_list
|
403
|
+
|
404
|
+
def flatten_iterable(iterable: Iterable, max_depth: int = None) -> Generator[Any, None, None]:
|
405
|
+
"""
|
406
|
+
Flattens a nested iterable up to a specified maximum depth.
|
407
|
+
|
408
|
+
Args:
|
409
|
+
iterable: An iterable to flatten.
|
410
|
+
max_depth: The maximum depth to flatten. If None, flattens completely.
|
411
|
+
|
412
|
+
Yields:
|
413
|
+
The flattened elements of the original iterable.
|
414
|
+
"""
|
415
|
+
def _flatten(input_iterable: Iterable, current_depth: int) -> Generator[Any, None, None]:
|
416
|
+
if isinstance(input_iterable, Iterable) and not isinstance(input_iterable, (str, bytes)):
|
417
|
+
if max_depth is not None and current_depth >= max_depth:
|
418
|
+
yield input_iterable
|
419
|
+
else:
|
420
|
+
for item in input_iterable:
|
421
|
+
yield from _flatten(item, current_depth + 1)
|
422
|
+
else:
|
423
|
+
yield input_iterable
|
424
|
+
|
425
|
+
yield from _flatten(iterable, 0)
|
426
|
+
|
427
|
+
def flatten_iterable_to_list(iterable: Iterable, max_depth: int = None) -> List[Any]:
|
428
|
+
"""
|
429
|
+
Converts a nested iterable into a flattened list up to a specified maximum depth.
|
430
|
+
|
431
|
+
Args:
|
432
|
+
iterable: An iterable to flatten.
|
433
|
+
max_depth: The maximum depth to flatten. If None, flattens completely.
|
434
|
+
|
435
|
+
Returns:
|
436
|
+
A list containing the flattened elements of the original iterable.
|
437
|
+
"""
|
438
|
+
return list(flatten_iterable(iterable, max_depth))
|
439
|
+
|
440
|
+
def unflatten_dict_with_custom_logic(
|
441
|
+
flat_dict: Dict[str, Any],
|
442
|
+
logic_func: Callable[[str, Any], Tuple[str, Any]],
|
443
|
+
sep: str = '_'
|
444
|
+
) -> Dict[str, Any]:
|
445
|
+
"""
|
446
|
+
Unflattens a dictionary with keys as string paths into a nested dictionary structure
|
447
|
+
while applying custom logic to each key and value.
|
448
|
+
|
449
|
+
Args:
|
450
|
+
flat_dict: The flat dictionary to unflatten.
|
451
|
+
logic_func: A function that takes a key and a value and returns a tuple of modified key and value.
|
452
|
+
sep: The separator used in the flat dictionary keys to indicate nesting.
|
453
|
+
|
454
|
+
Returns:
|
455
|
+
A nested dictionary that represents the unflattened structure with modified keys and values.
|
456
|
+
"""
|
457
|
+
reconstructed = {}
|
458
|
+
for flat_key, value in flat_dict.items():
|
459
|
+
parts = flat_key.split(sep)
|
460
|
+
d = reconstructed
|
461
|
+
for part in parts[:-1]:
|
462
|
+
modified_part, _ = logic_func(part, None) # Modify only the key
|
463
|
+
d = d.setdefault(modified_part, {})
|
464
|
+
last_part = parts[-1]
|
465
|
+
modified_last_part, modified_value = logic_func(last_part, value)
|
466
|
+
d[modified_last_part] = modified_value
|
467
|
+
return reconstructed
|
468
|
+
|
469
|
+
# def dynamic_unflatten(flat_dict, sep='_', custom_logic=None, max_depth=None):
|
470
|
+
# """
|
471
|
+
# Unflattens a dictionary with flat keys into a nested dictionary or list.
|
472
|
+
|
473
|
+
# :param flat_dict: A dictionary with flat keys that need to be nested.
|
474
|
+
# :param sep: The separator used in the flat keys (default is '_').
|
475
|
+
# :param custom_logic: A function that customizes the processing of keys.
|
476
|
+
# It takes a key part as input and returns the transformed key part.
|
477
|
+
# :param max_depth: The maximum depth to which the dictionary should be nested.
|
478
|
+
# If None, there is no maximum depth.
|
479
|
+
# :return: A nested dictionary or list based on the flat input dictionary.
|
480
|
+
|
481
|
+
# The function dynamically unflattens a dictionary with keys that represent nested paths.
|
482
|
+
# For example, the flat dictionary {'a_b_c': 1, 'a_b_d': 2} would be unflattened to
|
483
|
+
# {'a': {'b': {'c': 1, 'd': 2}}}. If the keys can be converted to integers and suggest list indices,
|
484
|
+
# the function produces lists instead of dictionaries. For example,
|
485
|
+
# {'0_a': 'foo', '1_b': 'bar'} becomes [{'a': 'foo'}, {'b': 'bar'}].
|
486
|
+
# """
|
487
|
+
|
488
|
+
# def handle_list_insert(sub_obj, part, value):
|
489
|
+
# # Ensure part index exists in the list, fill gaps with None
|
490
|
+
# while len(sub_obj) <= part:
|
491
|
+
# sub_obj.append(None)
|
492
|
+
|
493
|
+
# sub_obj[part] = value
|
494
|
+
|
495
|
+
# def insert(sub_obj, parts, value, max_depth, current_depth=0):
|
496
|
+
# for part in parts[:-1]:
|
497
|
+
# # Stop nesting further if max_depth is reached
|
498
|
+
# if max_depth is not None and current_depth >= max_depth:
|
499
|
+
# sub_obj[part] = {parts[-1]: value}
|
500
|
+
# return
|
501
|
+
# # Handle integer parts for list insertion
|
502
|
+
# if isinstance(part, int):
|
503
|
+
# # Ensure part index exists in the list or dict, fill gaps with None
|
504
|
+
# while len(sub_obj) <= part:
|
505
|
+
# sub_obj.append(None)
|
506
|
+
# if sub_obj[part] is None:
|
507
|
+
# if current_depth < max_depth - 1 if max_depth else True:
|
508
|
+
# sub_obj[part] = [] if isinstance(parts[parts.index(part) + 1], int) else {}
|
509
|
+
# sub_obj = sub_obj[part]
|
510
|
+
# else:
|
511
|
+
# # Handle string parts for dictionary insertion
|
512
|
+
# if part not in sub_obj:
|
513
|
+
# sub_obj[part] = {}
|
514
|
+
# sub_obj = sub_obj[part]
|
515
|
+
# current_depth += 1
|
516
|
+
# # Insert the value at the last part
|
517
|
+
# last_part = parts[-1]
|
518
|
+
# if isinstance(last_part, int) and isinstance(sub_obj, list):
|
519
|
+
# handle_list_insert(sub_obj, last_part, value, max_depth, current_depth)
|
520
|
+
# else:
|
521
|
+
# sub_obj[last_part] = value
|
522
|
+
|
523
|
+
# unflattened = {}
|
524
|
+
# for composite_key, value in flat_dict.items():
|
525
|
+
# parts = composite_key.split(sep)
|
526
|
+
# if custom_logic:
|
527
|
+
# parts = [custom_logic(part) for part in parts]
|
528
|
+
# else:
|
529
|
+
# parts = [int(part) if (isinstance(part, int) or part.isdigit()) else part for part in parts]
|
530
|
+
# insert(unflattened, parts, value, max_depth)
|
531
|
+
|
532
|
+
# # Convert top-level dictionary to a list if all keys are integers
|
533
|
+
# if isinstance(unflattened, dict) and all(isinstance(k, int) for k in unflattened.keys()):
|
534
|
+
# max_index = max(unflattened.keys(), default=-1)
|
535
|
+
# return [unflattened.get(i) for i in range(max_index + 1)]
|
536
|
+
# # Return an empty dictionary instead of converting to a list if unflattened is empty
|
537
|
+
# if (unflattened == []) or (not unflattened):
|
538
|
+
# return {}
|
539
|
+
# return unflattened
|
540
|
+
|
lionagi/utils/io_util.py
ADDED
@@ -0,0 +1,102 @@
|
|
1
|
+
import csv
|
2
|
+
import json
|
3
|
+
import os
|
4
|
+
import tempfile
|
5
|
+
from typing import Any, Dict, List
|
6
|
+
from .type_util import to_list
|
7
|
+
|
8
|
+
|
9
|
+
def to_temp(input: Any,
|
10
|
+
flatten_dict: bool = False,
|
11
|
+
flat: bool = False,
|
12
|
+
dropna: bool = False):
|
13
|
+
"""
|
14
|
+
Converts input to a list and writes it to a temporary file in JSON format, with flattening options.
|
15
|
+
|
16
|
+
This function serializes data to a temporary JSON file, useful for transient storage or testing.
|
17
|
+
It includes options to flatten the input if it contains dictionaries or lists.
|
18
|
+
|
19
|
+
Parameters:
|
20
|
+
input (Any): The data to be converted and written to a file.
|
21
|
+
|
22
|
+
flatten_dict (bool, optional): Flatten dictionaries in the input. Defaults to False.
|
23
|
+
|
24
|
+
flat (bool, optional): Flatten lists in the input. Defaults to False.
|
25
|
+
|
26
|
+
dropna (bool, optional): Exclude 'None' values during flattening. Defaults to False.
|
27
|
+
|
28
|
+
Raises:
|
29
|
+
TypeError: If the input is not JSON serializable.
|
30
|
+
|
31
|
+
Example:
|
32
|
+
>>> temp_file = to_temp({'a': 1, 'b': [2, 3]}, flatten_dict=True)
|
33
|
+
>>> temp_file.name # Doctest: +ELLIPSIS
|
34
|
+
'/var/folders/.../tmp...'
|
35
|
+
"""
|
36
|
+
input = to_list(input, flatten_dict, flat, dropna)
|
37
|
+
|
38
|
+
temp_file = tempfile.NamedTemporaryFile(mode='w', delete=False)
|
39
|
+
try:
|
40
|
+
json.dump(input, temp_file)
|
41
|
+
except TypeError as e:
|
42
|
+
temp_file.close() # Ensuring file closure before raising error
|
43
|
+
raise TypeError(f"Data provided is not JSON serializable: {e}")
|
44
|
+
temp_file.close()
|
45
|
+
return temp_file
|
46
|
+
|
47
|
+
def to_csv(input: List[Dict[str, Any]]=None,
|
48
|
+
filepath: str=None,
|
49
|
+
file_exist_ok: bool = False) -> None:
|
50
|
+
"""
|
51
|
+
Writes a list of dictionaries to a CSV file, with dictionary keys as headers.
|
52
|
+
|
53
|
+
This function writes a list of dictionaries to a CSV file. It checks if the file exists
|
54
|
+
and handles file creation based on the 'file_exist_ok' flag.
|
55
|
+
|
56
|
+
Parameters:
|
57
|
+
input (List[Dict[str, Any]]): Data to write to the CSV file.
|
58
|
+
|
59
|
+
filepath (str): Path of the output CSV file.
|
60
|
+
|
61
|
+
file_exist_ok (bool, optional): Create the file if it doesn't exist. Defaults to False.
|
62
|
+
|
63
|
+
Raises:
|
64
|
+
FileExistsError: If the file already exists and 'file_exist_ok' is False.
|
65
|
+
|
66
|
+
Example:
|
67
|
+
>>> data = [{'name': 'Alice', 'age': 30}, {'name': 'Bob', 'age': 25}]
|
68
|
+
>>> to_csv(data, 'people.csv')
|
69
|
+
"""
|
70
|
+
|
71
|
+
if not os.path.exists(os.path.dirname(filepath)) and os.path.dirname(filepath) != '':
|
72
|
+
if file_exist_ok:
|
73
|
+
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
74
|
+
else:
|
75
|
+
raise FileNotFoundError(f"The directory {os.path.dirname(filepath)} does not exist.")
|
76
|
+
|
77
|
+
with open(filepath, 'w', newline='') as csv_file:
|
78
|
+
writer = csv.DictWriter(csv_file, fieldnames=input[0].keys())
|
79
|
+
writer.writeheader()
|
80
|
+
writer.writerows(input)
|
81
|
+
|
82
|
+
|
83
|
+
def append_to_jsonl(data: Any, filepath: str) -> None:
|
84
|
+
"""
|
85
|
+
Appends data to a JSON lines (jsonl) file.
|
86
|
+
|
87
|
+
Serializes given data to a JSON-formatted string and appends it to a jsonl file.
|
88
|
+
Useful for logging or data collection where entries are added incrementally.
|
89
|
+
|
90
|
+
Parameters:
|
91
|
+
data (Any): Data to be serialized and appended.
|
92
|
+
|
93
|
+
filepath (str): Path to the jsonl file.
|
94
|
+
|
95
|
+
Example:
|
96
|
+
>>> append_to_jsonl({"key": "value"}, "data.jsonl")
|
97
|
+
# Appends {"key": "value"} to 'data.jsonl'
|
98
|
+
"""
|
99
|
+
json_string = json.dumps(data)
|
100
|
+
with open(filepath, "a") as f:
|
101
|
+
f.write(json_string + "\n")
|
102
|
+
|