lionagi 0.0.115__py3-none-any.whl → 0.0.204__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- lionagi/__init__.py +1 -2
- lionagi/_services/__init__.py +5 -0
- lionagi/_services/anthropic.py +79 -0
- lionagi/_services/base_service.py +414 -0
- lionagi/_services/oai.py +98 -0
- lionagi/_services/openrouter.py +44 -0
- lionagi/_services/services.py +91 -0
- lionagi/_services/transformers.py +46 -0
- lionagi/bridge/langchain.py +26 -16
- lionagi/bridge/llama_index.py +50 -20
- lionagi/configs/oai_configs.py +2 -14
- lionagi/configs/openrouter_configs.py +2 -2
- lionagi/core/__init__.py +7 -8
- lionagi/core/branch/branch.py +589 -0
- lionagi/core/branch/branch_manager.py +139 -0
- lionagi/core/branch/conversation.py +484 -0
- lionagi/core/core_util.py +59 -0
- lionagi/core/flow/flow.py +19 -0
- lionagi/core/flow/flow_util.py +62 -0
- lionagi/core/instruction_set/__init__.py +0 -5
- lionagi/core/instruction_set/instruction_set.py +343 -0
- lionagi/core/messages/messages.py +176 -0
- lionagi/core/sessions/__init__.py +0 -5
- lionagi/core/sessions/session.py +428 -0
- lionagi/loaders/chunker.py +51 -47
- lionagi/loaders/load_util.py +2 -2
- lionagi/loaders/reader.py +45 -39
- lionagi/models/imodel.py +53 -0
- lionagi/schema/async_queue.py +158 -0
- lionagi/schema/base_node.py +318 -147
- lionagi/schema/base_tool.py +31 -1
- lionagi/schema/data_logger.py +74 -38
- lionagi/schema/data_node.py +57 -6
- lionagi/structures/graph.py +132 -10
- lionagi/structures/relationship.py +58 -20
- lionagi/structures/structure.py +36 -25
- lionagi/tests/test_utils/test_api_util.py +219 -0
- lionagi/tests/test_utils/test_call_util.py +785 -0
- lionagi/tests/test_utils/test_encrypt_util.py +323 -0
- lionagi/tests/test_utils/test_io_util.py +238 -0
- lionagi/tests/test_utils/test_nested_util.py +338 -0
- lionagi/tests/test_utils/test_sys_util.py +358 -0
- lionagi/tools/tool_manager.py +186 -0
- lionagi/tools/tool_util.py +266 -3
- lionagi/utils/__init__.py +21 -61
- lionagi/utils/api_util.py +359 -71
- lionagi/utils/call_util.py +839 -264
- lionagi/utils/encrypt_util.py +283 -16
- lionagi/utils/io_util.py +178 -93
- lionagi/utils/nested_util.py +672 -0
- lionagi/utils/pd_util.py +57 -0
- lionagi/utils/sys_util.py +284 -156
- lionagi/utils/url_util.py +55 -0
- lionagi/version.py +1 -1
- {lionagi-0.0.115.dist-info → lionagi-0.0.204.dist-info}/METADATA +21 -17
- lionagi-0.0.204.dist-info/RECORD +106 -0
- lionagi/core/conversations/__init__.py +0 -5
- lionagi/core/conversations/conversation.py +0 -107
- lionagi/core/flows/__init__.py +0 -8
- lionagi/core/flows/flow.py +0 -8
- lionagi/core/flows/flow_util.py +0 -62
- lionagi/core/instruction_set/instruction_sets.py +0 -7
- lionagi/core/sessions/sessions.py +0 -185
- lionagi/endpoints/__init__.py +0 -5
- lionagi/endpoints/audio.py +0 -17
- lionagi/endpoints/chatcompletion.py +0 -54
- lionagi/messages/__init__.py +0 -11
- lionagi/messages/instruction.py +0 -15
- lionagi/messages/message.py +0 -110
- lionagi/messages/response.py +0 -33
- lionagi/messages/system.py +0 -12
- lionagi/objs/__init__.py +0 -11
- lionagi/objs/abc_objs.py +0 -39
- lionagi/objs/async_queue.py +0 -135
- lionagi/objs/messenger.py +0 -85
- lionagi/objs/tool_manager.py +0 -253
- lionagi/services/__init__.py +0 -11
- lionagi/services/base_api_service.py +0 -230
- lionagi/services/oai.py +0 -34
- lionagi/services/openrouter.py +0 -31
- lionagi/tests/test_api_util.py +0 -46
- lionagi/tests/test_call_util.py +0 -115
- lionagi/tests/test_convert_util.py +0 -202
- lionagi/tests/test_encrypt_util.py +0 -33
- lionagi/tests/test_flat_util.py +0 -426
- lionagi/tests/test_sys_util.py +0 -0
- lionagi/utils/convert_util.py +0 -229
- lionagi/utils/flat_util.py +0 -599
- lionagi-0.0.115.dist-info/RECORD +0 -110
- /lionagi/{services → _services}/anyscale.py +0 -0
- /lionagi/{services → _services}/azure.py +0 -0
- /lionagi/{services → _services}/bedrock.py +0 -0
- /lionagi/{services → _services}/everlyai.py +0 -0
- /lionagi/{services → _services}/gemini.py +0 -0
- /lionagi/{services → _services}/gpt4all.py +0 -0
- /lionagi/{services → _services}/huggingface.py +0 -0
- /lionagi/{services → _services}/litellm.py +0 -0
- /lionagi/{services → _services}/localai.py +0 -0
- /lionagi/{services → _services}/mistralai.py +0 -0
- /lionagi/{services → _services}/ollama.py +0 -0
- /lionagi/{services → _services}/openllm.py +0 -0
- /lionagi/{services → _services}/perplexity.py +0 -0
- /lionagi/{services → _services}/predibase.py +0 -0
- /lionagi/{services → _services}/rungpt.py +0 -0
- /lionagi/{services → _services}/vllm.py +0 -0
- /lionagi/{services → _services}/xinference.py +0 -0
- /lionagi/{endpoints/assistants.py → agents/__init__.py} +0 -0
- /lionagi/{tools → agents}/planner.py +0 -0
- /lionagi/{tools → agents}/prompter.py +0 -0
- /lionagi/{tools → agents}/scorer.py +0 -0
- /lionagi/{tools → agents}/summarizer.py +0 -0
- /lionagi/{tools → agents}/validator.py +0 -0
- /lionagi/{endpoints/embeddings.py → core/branch/__init__.py} +0 -0
- /lionagi/{services/anthropic.py → core/branch/cluster.py} +0 -0
- /lionagi/{endpoints/finetune.py → core/flow/__init__.py} +0 -0
- /lionagi/{endpoints/image.py → core/messages/__init__.py} +0 -0
- /lionagi/{endpoints/moderation.py → models/__init__.py} +0 -0
- /lionagi/{endpoints/vision.py → models/base_model.py} +0 -0
- /lionagi/{objs → schema}/status_tracker.py +0 -0
- /lionagi/tests/{test_io_util.py → test_utils/__init__.py} +0 -0
- {lionagi-0.0.115.dist-info → lionagi-0.0.204.dist-info}/LICENSE +0 -0
- {lionagi-0.0.115.dist-info → lionagi-0.0.204.dist-info}/WHEEL +0 -0
- {lionagi-0.0.115.dist-info → lionagi-0.0.204.dist-info}/top_level.txt +0 -0
lionagi/utils/flat_util.py
DELETED
@@ -1,599 +0,0 @@
|
|
1
|
-
# this module has no internal dependency
|
2
|
-
from typing import Dict, Iterable, List, Any, Callable, Generator, Tuple
|
3
|
-
|
4
|
-
|
5
|
-
def flatten_dict(d: Dict, parent_key: str = '', sep: str = '_') -> Dict:
|
6
|
-
"""
|
7
|
-
Flattens a nested dictionary by concatenating keys.
|
8
|
-
|
9
|
-
This function recursively flattens a nested dictionary by concatenating
|
10
|
-
the keys of nested dictionaries with a separator. The default separator
|
11
|
-
is an underscore (_).
|
12
|
-
|
13
|
-
Parameters:
|
14
|
-
d (dict): The dictionary to flatten.
|
15
|
-
|
16
|
-
parent_key (str, optional): The base key to use for the current level of recursion.
|
17
|
-
Defaults to an empty string, meaning no parent key, key cannot be a number
|
18
|
-
|
19
|
-
sep (str, optional): The separator to use when concatenating keys.
|
20
|
-
Defaults to an underscore (_).
|
21
|
-
|
22
|
-
Returns:
|
23
|
-
dict: A new dictionary with flattened keys and corresponding values.
|
24
|
-
"""
|
25
|
-
items = []
|
26
|
-
for k, v in d.items():
|
27
|
-
new_key = f"{parent_key}{sep}{k}" if parent_key else k
|
28
|
-
if isinstance(v, dict):
|
29
|
-
items.extend(flatten_dict(v, new_key, sep=sep).items())
|
30
|
-
else:
|
31
|
-
items.append((new_key, v))
|
32
|
-
return dict(items)
|
33
|
-
|
34
|
-
def flatten_list(l: List, dropna: bool = True) -> List:
|
35
|
-
"""
|
36
|
-
Flattens a nested list into a single list of values.
|
37
|
-
|
38
|
-
The function iterates over each element in the provided list. If an
|
39
|
-
element is a list itself, the function is called recursively to flatten
|
40
|
-
it. If the element is not a list, it is appended to the result list.
|
41
|
-
Optionally, None values can be dropped from the result list.
|
42
|
-
|
43
|
-
Parameters:
|
44
|
-
l (list): The list to flatten, which may contain nested lists.
|
45
|
-
|
46
|
-
dropna (bool, optional): Whether to exclude None values from the result list.
|
47
|
-
Defaults to True.
|
48
|
-
|
49
|
-
Returns:
|
50
|
-
list: A new flattened list with or without None values based on the dropna parameter.
|
51
|
-
"""
|
52
|
-
flat_list = []
|
53
|
-
for i in l:
|
54
|
-
if isinstance(i, list):
|
55
|
-
flat_list.extend(flatten_list(i, dropna))
|
56
|
-
elif i is not None or not dropna:
|
57
|
-
flat_list.append(i)
|
58
|
-
return flat_list
|
59
|
-
|
60
|
-
def change_separator(flat_dict, current_sep, new_sep):
|
61
|
-
"""
|
62
|
-
Changes the separator in the keys of a flat dictionary.
|
63
|
-
|
64
|
-
Parameters:
|
65
|
-
flat_dict (dict): The dictionary with keys containing the current separator.
|
66
|
-
|
67
|
-
current_sep (str): The current separator used in the dictionary keys.
|
68
|
-
|
69
|
-
new_sep (str): The new separator to replace the current separator in the dictionary keys.
|
70
|
-
|
71
|
-
Returns:
|
72
|
-
dict: A new dictionary with the separators in the keys replaced.
|
73
|
-
"""
|
74
|
-
return {
|
75
|
-
k.replace(current_sep, new_sep): v
|
76
|
-
for k, v in flat_dict.items()
|
77
|
-
}
|
78
|
-
|
79
|
-
def unflatten_dict(flat_dict: Dict, sep: str = '_') -> Dict:
|
80
|
-
"""
|
81
|
-
Unflattens a dictionary where keys are strings that represent nested dictionary paths separated by 'sep'.
|
82
|
-
|
83
|
-
Parameters:
|
84
|
-
flat_dict (dict): A dictionary with keys as strings that represent paths.
|
85
|
-
|
86
|
-
sep (str): The separator used in the keys of the flat dictionary.
|
87
|
-
|
88
|
-
Returns:
|
89
|
-
dict: A nested dictionary unflattened from the keys of the input dictionary.
|
90
|
-
|
91
|
-
Raises:
|
92
|
-
ValueError: If there are conflicting keys in the path.
|
93
|
-
|
94
|
-
Example:
|
95
|
-
>>> unflatten_dict({'a_0': 1, 'a_1': 2, 'b_x': 'X', 'b_y': 'Y'})
|
96
|
-
{'a': [1, 2], 'b': {'x': 'X', 'y': 'Y'}}
|
97
|
-
"""
|
98
|
-
result = {}
|
99
|
-
for flat_key, value in flat_dict.items():
|
100
|
-
parts = flat_key.split(sep)
|
101
|
-
d = result
|
102
|
-
for part in parts[:-1]:
|
103
|
-
part = int(part) if part.isdigit() else part
|
104
|
-
if part not in d:
|
105
|
-
if isinstance(d, list):
|
106
|
-
d.append({})
|
107
|
-
d = d[-1]
|
108
|
-
else:
|
109
|
-
d[part] = {}
|
110
|
-
d = d[part] if not isinstance(d, list) else d[-1]
|
111
|
-
last_part = parts[-1]
|
112
|
-
last_part = int(last_part) if last_part.isdigit() else last_part
|
113
|
-
if isinstance(d, list):
|
114
|
-
d.append(value)
|
115
|
-
elif last_part in d:
|
116
|
-
raise ValueError(f"Conflicting keys detected. Key part '{last_part}' is already present.")
|
117
|
-
else:
|
118
|
-
d[last_part] = value
|
119
|
-
|
120
|
-
# Convert dictionaries with contiguous integer keys starting from 0 to lists
|
121
|
-
def dict_to_list(d: dict) -> dict:
|
122
|
-
"""
|
123
|
-
Converts dictionaries with contiguous integer keys starting from 0 into lists.
|
124
|
-
|
125
|
-
Parameters:
|
126
|
-
d (dict): A dictionary that may have integer keys suitable for conversion to a list.
|
127
|
-
|
128
|
-
Returns:
|
129
|
-
dict or list: The input dictionary or a list if the dictionary keys match the criteria.
|
130
|
-
"""
|
131
|
-
if isinstance(d, dict) and all(isinstance(k, int) and k >= 0 for k in d.keys()):
|
132
|
-
keys = sorted(d.keys())
|
133
|
-
if keys == list(range(len(keys))):
|
134
|
-
return [dict_to_list(d[k]) for k in keys]
|
135
|
-
if isinstance(d, dict):
|
136
|
-
return {k: dict_to_list(v) for k, v in d.items()}
|
137
|
-
return d
|
138
|
-
|
139
|
-
result = dict_to_list(result)
|
140
|
-
return result
|
141
|
-
|
142
|
-
def is_flattenable(obj: Any) -> bool:
|
143
|
-
"""
|
144
|
-
Determines if the given object contains nested dictionaries or lists, making it suitable for flattening.
|
145
|
-
|
146
|
-
Parameters:
|
147
|
-
obj (object): The object to check for flattenable structures.
|
148
|
-
|
149
|
-
Returns:
|
150
|
-
bool: True if the object can be flattened (contains nested dicts or lists), False otherwise.
|
151
|
-
"""
|
152
|
-
if isinstance(obj, dict):
|
153
|
-
return any(isinstance(v, (dict, list)) for v in obj.values())
|
154
|
-
elif isinstance(obj, list):
|
155
|
-
return any(isinstance(i, (dict, list)) for i in obj)
|
156
|
-
return False
|
157
|
-
|
158
|
-
def flatten_with_custom_logic(obj, logic_func=None, parent_key='', sep='_'):
|
159
|
-
"""
|
160
|
-
Recursively flattens a nested dictionary or list and applies custom logic to the keys and values.
|
161
|
-
|
162
|
-
Parameters:
|
163
|
-
obj (dict | list): The dictionary or list to flatten.
|
164
|
-
|
165
|
-
logic_func (callable, optional): A function that takes four arguments (parent_key, key, value, sep)
|
166
|
-
and returns a tuple (new_key, new_value) after applying custom logic.
|
167
|
-
|
168
|
-
parent_key (str): The base key to use for creating new keys.
|
169
|
-
|
170
|
-
sep (str): The separator to use when joining nested keys.
|
171
|
-
|
172
|
-
Returns:
|
173
|
-
dict: A flattened dictionary with keys representing the nested paths and values from the original object.
|
174
|
-
|
175
|
-
Example Usage:
|
176
|
-
>>> sample_dict = {'a': 1, 'b': {'c': 2, 'd': {'e': 3}}}
|
177
|
-
>>> flatten_with_custom_logic(sample_dict)
|
178
|
-
{'a': 1, 'b_c': 2, 'b_d_e': 3}
|
179
|
-
|
180
|
-
>>> sample_list = [1, 2, [3, 4]]
|
181
|
-
>>> flatten_with_custom_logic(sample_list)
|
182
|
-
{'0': 1, '1': 2, '2_0': 3, '2_1': 4}
|
183
|
-
|
184
|
-
>>> def custom_logic(parent_key, key, value, sep='_'):
|
185
|
-
... new_key = f"{parent_key}{sep}{key}".upper() if parent_key else key.upper()
|
186
|
-
... return new_key, value * 2 if isinstance(value, int) else value
|
187
|
-
>>> flatten_with_custom_logic(sample_dict, custom_logic)
|
188
|
-
{'A': 2, 'B_C': 4, 'B_D_E': 6}
|
189
|
-
"""
|
190
|
-
items = {}
|
191
|
-
if isinstance(obj, dict):
|
192
|
-
for k, v in obj.items():
|
193
|
-
new_key = f"{parent_key}{sep}{k}" if parent_key else k
|
194
|
-
if isinstance(v, (dict, list)):
|
195
|
-
if v: # Check if the dictionary or list is not empty
|
196
|
-
items.update(flatten_with_custom_logic(v, logic_func, new_key, sep))
|
197
|
-
else: # Handle empty dictionaries and lists
|
198
|
-
items[new_key] = None
|
199
|
-
else:
|
200
|
-
if logic_func:
|
201
|
-
new_key, new_value = logic_func(parent_key, k, v, sep=sep)
|
202
|
-
else:
|
203
|
-
new_value = v
|
204
|
-
items[new_key] = new_value
|
205
|
-
elif isinstance(obj, list):
|
206
|
-
for i, item in enumerate(obj):
|
207
|
-
new_key = f"{parent_key}{sep}{str(i)}" if parent_key else str(i) # Cast index to string
|
208
|
-
if isinstance(item, (dict, list)):
|
209
|
-
if item: # Check if the dictionary or list is not empty
|
210
|
-
items.update(flatten_with_custom_logic(item, logic_func, new_key, sep))
|
211
|
-
else: # Handle empty lists
|
212
|
-
items[new_key] = None
|
213
|
-
else:
|
214
|
-
if logic_func:
|
215
|
-
new_key, new_value = logic_func(parent_key, str(i), item, sep=sep)
|
216
|
-
else:
|
217
|
-
new_value = item
|
218
|
-
items[new_key] = new_value
|
219
|
-
return items
|
220
|
-
|
221
|
-
def dynamic_flatten(obj, parent_key='', sep='_', max_depth=None, current_depth=0):
|
222
|
-
"""
|
223
|
-
Recursively flattens a nested dictionary or list, while allowing a maximum depth and custom separators.
|
224
|
-
|
225
|
-
Parameters:
|
226
|
-
obj (dict | list): The dictionary or list to flatten.
|
227
|
-
|
228
|
-
parent_key (str): The base key to use for creating new keys.
|
229
|
-
|
230
|
-
sep (str): The separator to use when joining nested keys.
|
231
|
-
|
232
|
-
max_depth (int, optional): The maximum depth to flatten.
|
233
|
-
|
234
|
-
current_depth (int): The current depth in the recursive call (used internally).
|
235
|
-
|
236
|
-
Returns:
|
237
|
-
dict: A flattened dictionary with keys representing the nested paths and values from the original object.
|
238
|
-
|
239
|
-
Raises:
|
240
|
-
TypeError: If the input object is neither a dictionary nor a list.
|
241
|
-
|
242
|
-
Example Usage:
|
243
|
-
>>> sample_dict = {'a': 1, 'b': {'c': 2, 'd': {'e': 3}}}
|
244
|
-
>>> dynamic_flatten(sample_dict)
|
245
|
-
{'a': 1, 'b_c': 2, 'b_d_e': 3}
|
246
|
-
|
247
|
-
>>> sample_list = [1, 2, [3, 4]]
|
248
|
-
>>> dynamic_flatten(sample_list)
|
249
|
-
{'0': 1, '1': 2, '2_0': 3, '2_1': 4}
|
250
|
-
|
251
|
-
>>> dynamic_flatten(sample_dict, max_depth=1)
|
252
|
-
{'a': 1, 'b_c': 2, 'b_d': {'e': 3}}
|
253
|
-
|
254
|
-
>>> dynamic_flatten(sample_dict, sep='.')
|
255
|
-
{'a': 1, 'b.c': 2, 'b.d.e': 3}
|
256
|
-
"""
|
257
|
-
items = []
|
258
|
-
if isinstance(obj, dict):
|
259
|
-
if not obj and parent_key:
|
260
|
-
items.append((parent_key, obj))
|
261
|
-
for k, v in obj.items():
|
262
|
-
new_key = f"{parent_key}{sep}{k}" if parent_key else k
|
263
|
-
if isinstance(v, (dict, list)) and (max_depth is None or current_depth < max_depth):
|
264
|
-
items.extend(dynamic_flatten(v, new_key, sep, max_depth, current_depth + 1).items())
|
265
|
-
else:
|
266
|
-
items.append((new_key, v))
|
267
|
-
elif isinstance(obj, list):
|
268
|
-
if not obj and parent_key:
|
269
|
-
items.append((parent_key, obj))
|
270
|
-
for i, item in enumerate(obj):
|
271
|
-
new_key = f"{parent_key}{sep}{i}" if parent_key else str(i)
|
272
|
-
if isinstance(item, (dict, list)) and (max_depth is None or current_depth < max_depth):
|
273
|
-
items.extend(dynamic_flatten(item, new_key, sep, max_depth, current_depth + 1).items())
|
274
|
-
else:
|
275
|
-
items.append((new_key, item))
|
276
|
-
else:
|
277
|
-
raise TypeError('Input object must be a dictionary or a list')
|
278
|
-
|
279
|
-
return dict(items)
|
280
|
-
|
281
|
-
# def dynamic_unflatten_dict(flat_dict, sep='_', custom_logic=None, max_depth=None):
|
282
|
-
# """
|
283
|
-
# Unflattens a dictionary with flat keys into a nested dictionary or list.
|
284
|
-
|
285
|
-
# :param flat_dict: A dictionary with flat keys that need to be nested.
|
286
|
-
# :param sep: The separator used in the flat keys (default is '_').
|
287
|
-
# :param custom_logic: A function that customizes the processing of keys.
|
288
|
-
# It takes a key part as input and returns the transformed key part.
|
289
|
-
# :param max_depth: The maximum depth to which the dictionary should be nested.
|
290
|
-
# If None, there is no maximum depth.
|
291
|
-
# :return: A nested dictionary or list based on the flat input dictionary.
|
292
|
-
|
293
|
-
# The function dynamically unflattens a dictionary with keys that represent nested paths.
|
294
|
-
# For example, the flat dictionary {'a_b_c': 1, 'a_b_d': 2} would be unflattened to
|
295
|
-
# {'a': {'b': {'c': 1, 'd': 2}}}. If the keys can be converted to integers and suggest list indices,
|
296
|
-
# the function produces lists instead of dictionaries. For example,
|
297
|
-
# {'0_a': 'foo', '1_b': 'bar'} becomes [{'a': 'foo'}, {'b': 'bar'}].
|
298
|
-
# """
|
299
|
-
|
300
|
-
# def handle_list_insert(sub_obj, part, value):
|
301
|
-
# # Ensure part index exists in the list, fill gaps with None
|
302
|
-
# while len(sub_obj) <= part:
|
303
|
-
# sub_obj.append(None)
|
304
|
-
|
305
|
-
# sub_obj[part] = value
|
306
|
-
|
307
|
-
# def insert(sub_obj, parts, value, max_depth, current_depth=0):
|
308
|
-
# for part in parts[:-1]:
|
309
|
-
# # Stop nesting further if max_depth is reached
|
310
|
-
# if max_depth is not None and current_depth >= max_depth:
|
311
|
-
# sub_obj[part] = {parts[-1]: value}
|
312
|
-
# return
|
313
|
-
# # Handle integer parts for list insertion
|
314
|
-
# if isinstance(part, int):
|
315
|
-
# # Ensure part index exists in the list or dict, fill gaps with None
|
316
|
-
# while len(sub_obj) <= part:
|
317
|
-
# sub_obj.append(None)
|
318
|
-
# if sub_obj[part] is None:
|
319
|
-
# if current_depth < max_depth - 1 if max_depth else True:
|
320
|
-
# sub_obj[part] = [] if isinstance(parts[parts.index(part) + 1], int) else {}
|
321
|
-
# sub_obj = sub_obj[part]
|
322
|
-
# else:
|
323
|
-
# # Handle string parts for dictionary insertion
|
324
|
-
# if part not in sub_obj:
|
325
|
-
# sub_obj[part] = {}
|
326
|
-
# sub_obj = sub_obj[part]
|
327
|
-
# current_depth += 1
|
328
|
-
# # Insert the value at the last part
|
329
|
-
# last_part = parts[-1]
|
330
|
-
# if isinstance(last_part, int) and isinstance(sub_obj, list):
|
331
|
-
# handle_list_insert(sub_obj, last_part, value, max_depth, current_depth)
|
332
|
-
# else:
|
333
|
-
# sub_obj[last_part] = value
|
334
|
-
|
335
|
-
# unflattened = {}
|
336
|
-
# for composite_key, value in flat_dict.items():
|
337
|
-
# parts = composite_key.split(sep)
|
338
|
-
# if custom_logic:
|
339
|
-
# parts = [custom_logic(part) for part in parts]
|
340
|
-
# else:
|
341
|
-
# parts = [int(part) if (isinstance(part, int) or part.isdigit()) else part for part in parts]
|
342
|
-
# insert(unflattened, parts, value, max_depth)
|
343
|
-
|
344
|
-
# # Convert top-level dictionary to a list if all keys are integers
|
345
|
-
# if isinstance(unflattened, dict) and all(isinstance(k, int) for k in unflattened.keys()):
|
346
|
-
# max_index = max(unflattened.keys(), default=-1)
|
347
|
-
# return [unflattened.get(i) for i in range(max_index + 1)]
|
348
|
-
# # Return an empty dictionary instead of converting to a list if unflattened is empty
|
349
|
-
# if (unflattened == []) or (not unflattened):
|
350
|
-
# return {}
|
351
|
-
# return unflattened
|
352
|
-
|
353
|
-
def _insert_with_dict_handling(container, indices, value):
|
354
|
-
"""
|
355
|
-
Helper function to insert a value into a nested container based on a list of indices.
|
356
|
-
|
357
|
-
Parameters:
|
358
|
-
container: The container (list or dict) to insert the value into.
|
359
|
-
|
360
|
-
indices: A list of indices indicating the path to the insertion point.
|
361
|
-
|
362
|
-
value: The value to be inserted.
|
363
|
-
"""
|
364
|
-
for i, index in enumerate(indices[:-1]):
|
365
|
-
# Check if index is an integer and ensure the list is long enough
|
366
|
-
if isinstance(index, int):
|
367
|
-
# Extend the list if necessary
|
368
|
-
while len(container) <= index:
|
369
|
-
container.append(None)
|
370
|
-
|
371
|
-
# Create a nested container if the current position is None
|
372
|
-
if container[index] is None:
|
373
|
-
next_index = indices[i + 1]
|
374
|
-
|
375
|
-
# Determine the type of the next container based on the next index
|
376
|
-
container[index] = {} if isinstance(next_index, str) else []
|
377
|
-
|
378
|
-
# Update the reference to point to the nested container
|
379
|
-
container = container[index]
|
380
|
-
|
381
|
-
# If index is a string, work with dictionaries
|
382
|
-
else:
|
383
|
-
if index not in container:
|
384
|
-
container[index] = {}
|
385
|
-
container = container[index]
|
386
|
-
|
387
|
-
# Insert value at the last index
|
388
|
-
last_index = indices[-1]
|
389
|
-
if isinstance(last_index, int):
|
390
|
-
|
391
|
-
# Negative indices are not allowed in this context
|
392
|
-
if last_index < 0:
|
393
|
-
raise ValueError("Negative index is not allowed for list unflattening.")
|
394
|
-
|
395
|
-
while len(container) <= last_index:
|
396
|
-
container.append(None)
|
397
|
-
container[last_index] = value
|
398
|
-
|
399
|
-
else:
|
400
|
-
if last_index in container and isinstance(container[last_index], list):
|
401
|
-
raise ValueError("Overlapping keys are not allowed.")
|
402
|
-
container[last_index] = value
|
403
|
-
|
404
|
-
def unflatten_to_list(flat_dict: Dict[str, Any], sep: str = '_') -> List:
|
405
|
-
"""
|
406
|
-
Unflattens a dictionary with keys as string paths into a nested list structure.
|
407
|
-
|
408
|
-
Parameters:
|
409
|
-
flat_dict: The flat dictionary to unflatten.
|
410
|
-
|
411
|
-
sep: The separator used in the flat dictionary keys to indicate nesting.
|
412
|
-
|
413
|
-
Returns:
|
414
|
-
A nested list that represents the unflattened structure.
|
415
|
-
"""
|
416
|
-
result_list = []
|
417
|
-
for flat_key, value in flat_dict.items():
|
418
|
-
indices = [int(p) if p.lstrip('-').isdigit() else p for p in flat_key.split(sep)]
|
419
|
-
_insert_with_dict_handling(result_list, indices, value)
|
420
|
-
return result_list
|
421
|
-
|
422
|
-
def flatten_iterable(iterable: Iterable, max_depth: int = None) -> Generator[Any, None, None]:
|
423
|
-
"""
|
424
|
-
Flattens a nested iterable up to a specified maximum depth.
|
425
|
-
|
426
|
-
Parameters:
|
427
|
-
iterable: An iterable to flatten.
|
428
|
-
|
429
|
-
max_depth: The maximum depth to flatten. If None, flattens completely.
|
430
|
-
|
431
|
-
Yields:
|
432
|
-
The flattened elements of the original iterable.
|
433
|
-
"""
|
434
|
-
def _flatten(input_iterable: Iterable, current_depth: int) -> Generator[Any, None, None]:
|
435
|
-
if isinstance(input_iterable, Iterable) and not isinstance(input_iterable, (str, bytes)):
|
436
|
-
if max_depth is not None and current_depth >= max_depth:
|
437
|
-
yield input_iterable
|
438
|
-
else:
|
439
|
-
for item in input_iterable:
|
440
|
-
yield from _flatten(item, current_depth + 1)
|
441
|
-
else:
|
442
|
-
yield input_iterable
|
443
|
-
|
444
|
-
yield from _flatten(iterable, 0)
|
445
|
-
|
446
|
-
def flatten_iterable_to_list(iterable: Iterable, max_depth: int = None) -> List[Any]:
|
447
|
-
"""
|
448
|
-
Converts a nested iterable into a flattened list up to a specified maximum depth.
|
449
|
-
|
450
|
-
Parameters:
|
451
|
-
iterable: An iterable to flatten.
|
452
|
-
|
453
|
-
max_depth: The maximum depth to flatten. If None, flattens completely.
|
454
|
-
|
455
|
-
Returns:
|
456
|
-
A list containing the flattened elements of the original iterable.
|
457
|
-
"""
|
458
|
-
return list(flatten_iterable(iterable, max_depth))
|
459
|
-
|
460
|
-
def unflatten_dict_with_custom_logic(
|
461
|
-
flat_dict: Dict[str, Any],
|
462
|
-
logic_func: Callable[[str, Any], Tuple[str, Any]],
|
463
|
-
sep: str = '_'
|
464
|
-
) -> Dict[str, Any]:
|
465
|
-
"""
|
466
|
-
Unflattens a dictionary with keys as string paths into a nested dictionary structure
|
467
|
-
while applying custom logic to each key and value.
|
468
|
-
|
469
|
-
Parameters:
|
470
|
-
flat_dict: The flat dictionary to unflatten.
|
471
|
-
|
472
|
-
logic_func: A function that takes a key and a value and returns a tuple of modified key and value.
|
473
|
-
|
474
|
-
sep: The separator used in the flat dictionary keys to indicate nesting.
|
475
|
-
|
476
|
-
Returns:
|
477
|
-
A nested dictionary that represents the unflattened structure with modified keys and values.
|
478
|
-
"""
|
479
|
-
reconstructed = {}
|
480
|
-
for flat_key, value in flat_dict.items():
|
481
|
-
parts = flat_key.split(sep)
|
482
|
-
d = reconstructed
|
483
|
-
for part in parts[:-1]:
|
484
|
-
modified_part, _ = logic_func(part, None) # Modify only the key
|
485
|
-
d = d.setdefault(modified_part, {})
|
486
|
-
last_part = parts[-1]
|
487
|
-
modified_last_part, modified_value = logic_func(last_part, value)
|
488
|
-
d[modified_last_part] = modified_value
|
489
|
-
return reconstructed
|
490
|
-
|
491
|
-
def to_list(input_: Any, flatten: bool = True, dropna: bool = False) -> List[Any]:
|
492
|
-
"""
|
493
|
-
Converts the input to a list, optionally flattening it and dropping None values.
|
494
|
-
|
495
|
-
Parameters:
|
496
|
-
input_ (Any): The input to convert to a list.
|
497
|
-
|
498
|
-
flatten (bool): Whether to flatten the input if it is a nested list. Defaults to True.
|
499
|
-
|
500
|
-
dropna (bool): Whether to drop None values from the list. Defaults to False.
|
501
|
-
|
502
|
-
Returns:
|
503
|
-
List[Any]: The input converted to a list.
|
504
|
-
|
505
|
-
Raises:
|
506
|
-
ValueError: If the input cannot be converted to a list.
|
507
|
-
"""
|
508
|
-
if isinstance(input_, list) and flatten:
|
509
|
-
input_ = flatten_list(input_)
|
510
|
-
if dropna:
|
511
|
-
input_ = [i for i in input_ if i is not None]
|
512
|
-
elif isinstance(input_, Iterable) and not isinstance(input_, (str, dict)):
|
513
|
-
try:
|
514
|
-
input_ = list(input_)
|
515
|
-
except:
|
516
|
-
raise ValueError("Input cannot be converted to a list.")
|
517
|
-
else:
|
518
|
-
input_ = [input_]
|
519
|
-
return input_
|
520
|
-
|
521
|
-
|
522
|
-
|
523
|
-
|
524
|
-
|
525
|
-
|
526
|
-
|
527
|
-
|
528
|
-
# def dynamic_unflatten(flat_dict, sep='_', custom_logic=None, max_depth=None):
|
529
|
-
# """
|
530
|
-
# Unflattens a dictionary with flat keys into a nested dictionary or list.
|
531
|
-
|
532
|
-
# :param flat_dict: A dictionary with flat keys that need to be nested.
|
533
|
-
# :param sep: The separator used in the flat keys (default is '_').
|
534
|
-
# :param custom_logic: A function that customizes the processing of keys.
|
535
|
-
# It takes a key part as input and returns the transformed key part.
|
536
|
-
# :param max_depth: The maximum depth to which the dictionary should be nested.
|
537
|
-
# If None, there is no maximum depth.
|
538
|
-
# :return: A nested dictionary or list based on the flat input dictionary.
|
539
|
-
|
540
|
-
# The function dynamically unflattens a dictionary with keys that represent nested paths.
|
541
|
-
# For example, the flat dictionary {'a_b_c': 1, 'a_b_d': 2} would be unflattened to
|
542
|
-
# {'a': {'b': {'c': 1, 'd': 2}}}. If the keys can be converted to integers and suggest list indices,
|
543
|
-
# the function produces lists instead of dictionaries. For example,
|
544
|
-
# {'0_a': 'foo', '1_b': 'bar'} becomes [{'a': 'foo'}, {'b': 'bar'}].
|
545
|
-
# """
|
546
|
-
|
547
|
-
# def handle_list_insert(sub_obj, part, value):
|
548
|
-
# # Ensure part index exists in the list, fill gaps with None
|
549
|
-
# while len(sub_obj) <= part:
|
550
|
-
# sub_obj.append(None)
|
551
|
-
|
552
|
-
# sub_obj[part] = value
|
553
|
-
|
554
|
-
# def insert(sub_obj, parts, value, max_depth, current_depth=0):
|
555
|
-
# for part in parts[:-1]:
|
556
|
-
# # Stop nesting further if max_depth is reached
|
557
|
-
# if max_depth is not None and current_depth >= max_depth:
|
558
|
-
# sub_obj[part] = {parts[-1]: value}
|
559
|
-
# return
|
560
|
-
# # Handle integer parts for list insertion
|
561
|
-
# if isinstance(part, int):
|
562
|
-
# # Ensure part index exists in the list or dict, fill gaps with None
|
563
|
-
# while len(sub_obj) <= part:
|
564
|
-
# sub_obj.append(None)
|
565
|
-
# if sub_obj[part] is None:
|
566
|
-
# if current_depth < max_depth - 1 if max_depth else True:
|
567
|
-
# sub_obj[part] = [] if isinstance(parts[parts.index(part) + 1], int) else {}
|
568
|
-
# sub_obj = sub_obj[part]
|
569
|
-
# else:
|
570
|
-
# # Handle string parts for dictionary insertion
|
571
|
-
# if part not in sub_obj:
|
572
|
-
# sub_obj[part] = {}
|
573
|
-
# sub_obj = sub_obj[part]
|
574
|
-
# current_depth += 1
|
575
|
-
# # Insert the value at the last part
|
576
|
-
# last_part = parts[-1]
|
577
|
-
# if isinstance(last_part, int) and isinstance(sub_obj, list):
|
578
|
-
# handle_list_insert(sub_obj, last_part, value, max_depth, current_depth)
|
579
|
-
# else:
|
580
|
-
# sub_obj[last_part] = value
|
581
|
-
|
582
|
-
# unflattened = {}
|
583
|
-
# for composite_key, value in flat_dict.items():
|
584
|
-
# parts = composite_key.split(sep)
|
585
|
-
# if custom_logic:
|
586
|
-
# parts = [custom_logic(part) for part in parts]
|
587
|
-
# else:
|
588
|
-
# parts = [int(part) if (isinstance(part, int) or part.isdigit()) else part for part in parts]
|
589
|
-
# insert(unflattened, parts, value, max_depth)
|
590
|
-
|
591
|
-
# # Convert top-level dictionary to a list if all keys are integers
|
592
|
-
# if isinstance(unflattened, dict) and all(isinstance(k, int) for k in unflattened.keys()):
|
593
|
-
# max_index = max(unflattened.keys(), default=-1)
|
594
|
-
# return [unflattened.get(i) for i in range(max_index + 1)]
|
595
|
-
# # Return an empty dictionary instead of converting to a list if unflattened is empty
|
596
|
-
# if (unflattened == []) or (not unflattened):
|
597
|
-
# return {}
|
598
|
-
# return unflattened
|
599
|
-
|