lionagi 0.0.112__py3-none-any.whl → 0.0.113__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. lionagi/__init__.py +3 -3
  2. lionagi/bridge/__init__.py +7 -0
  3. lionagi/bridge/langchain.py +131 -0
  4. lionagi/bridge/llama_index.py +157 -0
  5. lionagi/configs/__init__.py +7 -0
  6. lionagi/configs/oai_configs.py +49 -0
  7. lionagi/configs/openrouter_config.py +49 -0
  8. lionagi/core/__init__.py +8 -2
  9. lionagi/core/instruction_sets.py +1 -3
  10. lionagi/core/messages.py +2 -2
  11. lionagi/core/sessions.py +174 -27
  12. lionagi/datastore/__init__.py +1 -0
  13. lionagi/loader/__init__.py +9 -4
  14. lionagi/loader/chunker.py +157 -0
  15. lionagi/loader/reader.py +124 -0
  16. lionagi/objs/__init__.py +7 -0
  17. lionagi/objs/messenger.py +163 -0
  18. lionagi/objs/tool_registry.py +247 -0
  19. lionagi/schema/__init__.py +11 -0
  20. lionagi/schema/base_schema.py +239 -0
  21. lionagi/schema/base_tool.py +9 -0
  22. lionagi/schema/data_logger.py +94 -0
  23. lionagi/services/__init__.py +14 -0
  24. lionagi/{service_/oai.py → services/base_api_service.py} +49 -82
  25. lionagi/{endpoint/base_endpoint.py → services/chatcompletion.py} +19 -22
  26. lionagi/services/oai.py +34 -0
  27. lionagi/services/openrouter.py +32 -0
  28. lionagi/{service_/service_utils.py → services/service_objs.py} +0 -1
  29. lionagi/structure/__init__.py +7 -0
  30. lionagi/structure/relationship.py +128 -0
  31. lionagi/structure/structure.py +160 -0
  32. lionagi/tests/test_flatten_util.py +426 -0
  33. lionagi/tools/__init__.py +0 -5
  34. lionagi/tools/coder.py +1 -0
  35. lionagi/tools/scorer.py +1 -0
  36. lionagi/tools/validator.py +1 -0
  37. lionagi/utils/__init__.py +46 -20
  38. lionagi/utils/api_util.py +86 -0
  39. lionagi/utils/call_util.py +347 -0
  40. lionagi/utils/flat_util.py +540 -0
  41. lionagi/utils/io_util.py +102 -0
  42. lionagi/utils/load_utils.py +190 -0
  43. lionagi/utils/sys_util.py +191 -0
  44. lionagi/utils/tool_util.py +92 -0
  45. lionagi/utils/type_util.py +81 -0
  46. lionagi/version.py +1 -1
  47. {lionagi-0.0.112.dist-info → lionagi-0.0.113.dist-info}/METADATA +37 -13
  48. lionagi-0.0.113.dist-info/RECORD +84 -0
  49. lionagi/endpoint/chat_completion.py +0 -20
  50. lionagi/endpoint/endpoint_utils.py +0 -0
  51. lionagi/llm_configs.py +0 -21
  52. lionagi/loader/load_utils.py +0 -161
  53. lionagi/schema.py +0 -275
  54. lionagi/service_/__init__.py +0 -6
  55. lionagi/service_/base_service.py +0 -48
  56. lionagi/service_/openrouter.py +0 -1
  57. lionagi/services.py +0 -1
  58. lionagi/tools/tool_utils.py +0 -75
  59. lionagi/utils/sys_utils.py +0 -799
  60. lionagi-0.0.112.dist-info/RECORD +0 -67
  61. /lionagi/{core/responses.py → datastore/chroma.py} +0 -0
  62. /lionagi/{endpoint/assistants.py → datastore/deeplake.py} +0 -0
  63. /lionagi/{endpoint/audio.py → datastore/elasticsearch.py} +0 -0
  64. /lionagi/{endpoint/embeddings.py → datastore/lantern.py} +0 -0
  65. /lionagi/{endpoint/files.py → datastore/pinecone.py} +0 -0
  66. /lionagi/{endpoint/fine_tuning.py → datastore/postgres.py} +0 -0
  67. /lionagi/{endpoint/images.py → datastore/qdrant.py} +0 -0
  68. /lionagi/{endpoint/messages.py → schema/base_condition.py} +0 -0
  69. /lionagi/{service_ → services}/anthropic.py +0 -0
  70. /lionagi/{service_ → services}/anyscale.py +0 -0
  71. /lionagi/{service_ → services}/azure.py +0 -0
  72. /lionagi/{service_ → services}/bedrock.py +0 -0
  73. /lionagi/{service_ → services}/everlyai.py +0 -0
  74. /lionagi/{service_ → services}/gemini.py +0 -0
  75. /lionagi/{service_ → services}/gpt4all.py +0 -0
  76. /lionagi/{service_ → services}/huggingface.py +0 -0
  77. /lionagi/{service_ → services}/litellm.py +0 -0
  78. /lionagi/{service_ → services}/localai.py +0 -0
  79. /lionagi/{service_ → services}/mistralai.py +0 -0
  80. /lionagi/{service_ → services}/ollama.py +0 -0
  81. /lionagi/{service_ → services}/openllm.py +0 -0
  82. /lionagi/{service_ → services}/perplexity.py +0 -0
  83. /lionagi/{service_ → services}/predibase.py +0 -0
  84. /lionagi/{service_ → services}/rungpt.py +0 -0
  85. /lionagi/{service_ → services}/vllm.py +0 -0
  86. /lionagi/{service_ → services}/xinference.py +0 -0
  87. /lionagi/{endpoint → tests}/__init__.py +0 -0
  88. /lionagi/{endpoint/models.py → tools/planner.py} +0 -0
  89. /lionagi/{endpoint/moderations.py → tools/prompter.py} +0 -0
  90. /lionagi/{endpoint/runs.py → tools/sandbox.py} +0 -0
  91. /lionagi/{endpoint/threads.py → tools/summarizer.py} +0 -0
  92. {lionagi-0.0.112.dist-info → lionagi-0.0.113.dist-info}/LICENSE +0 -0
  93. {lionagi-0.0.112.dist-info → lionagi-0.0.113.dist-info}/WHEEL +0 -0
  94. {lionagi-0.0.112.dist-info → lionagi-0.0.113.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,540 @@
1
+ from typing import Dict, Iterable, List, Any, Callable, Generator, Tuple
2
+
3
+
4
+ def flatten_dict(d: Dict, parent_key: str = '', sep: str = '_') -> Dict:
5
+ """
6
+ Flattens a nested dictionary by concatenating keys.
7
+
8
+ This function recursively flattens a nested dictionary by concatenating
9
+ the keys of nested dictionaries with a separator. The default separator
10
+ is an underscore (_).
11
+
12
+ Parameters:
13
+ d (dict): The dictionary to flatten.
14
+
15
+ parent_key (str, optional): The base key to use for the current level of recursion.
16
+ Defaults to an empty string, meaning no parent key, key cannot be a number
17
+
18
+ sep (str, optional): The separator to use when concatenating keys.
19
+ Defaults to an underscore (_).
20
+
21
+ Returns:
22
+ dict: A new dictionary with flattened keys and corresponding values.
23
+ """
24
+ items = []
25
+ for k, v in d.items():
26
+ new_key = f"{parent_key}{sep}{k}" if parent_key else k
27
+ if isinstance(v, dict):
28
+ items.extend(flatten_dict(v, new_key, sep=sep).items())
29
+ else:
30
+ items.append((new_key, v))
31
+ return dict(items)
32
+
33
+ def flatten_list(l: List, dropna: bool = True) -> List:
34
+ """
35
+ Flattens a nested list into a single list of values.
36
+
37
+ The function iterates over each element in the provided list. If an
38
+ element is a list itself, the function is called recursively to flatten
39
+ it. If the element is not a list, it is appended to the result list.
40
+ Optionally, None values can be dropped from the result list.
41
+
42
+ Parameters:
43
+ l (list): The list to flatten, which may contain nested lists.
44
+
45
+ dropna (bool, optional): Whether to exclude None values from the result list.
46
+ Defaults to True.
47
+
48
+ Returns:
49
+ list: A new flattened list with or without None values based on the dropna parameter.
50
+ """
51
+ flat_list = []
52
+ for i in l:
53
+ if isinstance(i, list):
54
+ flat_list.extend(flatten_list(i, dropna))
55
+ elif i is not None or not dropna:
56
+ flat_list.append(i)
57
+ return flat_list
58
+
59
+ def change_separator(flat_dict, current_sep, new_sep):
60
+ """
61
+ Changes the separator in the keys of a flat dictionary.
62
+
63
+ Parameters:
64
+ flat_dict (dict): The dictionary with keys containing the current separator.
65
+ current_sep (str): The current separator used in the dictionary keys.
66
+ new_sep (str): The new separator to replace the current separator in the dictionary keys.
67
+
68
+ Returns:
69
+ dict: A new dictionary with the separators in the keys replaced.
70
+ """
71
+ return {
72
+ k.replace(current_sep, new_sep): v
73
+ for k, v in flat_dict.items()
74
+ }
75
+
76
+ def unflatten_dict(flat_dict: Dict, sep: str = '_') -> Dict:
77
+ """
78
+ Unflattens a dictionary where keys are strings that represent nested dictionary paths separated by 'sep'.
79
+
80
+ Args:
81
+ flat_dict (dict): A dictionary with keys as strings that represent paths.
82
+ sep (str): The separator used in the keys of the flat dictionary.
83
+
84
+ Returns:
85
+ dict: A nested dictionary unflattened from the keys of the input dictionary.
86
+
87
+ Raises:
88
+ ValueError: If there are conflicting keys in the path.
89
+
90
+ Example:
91
+ >>> unflatten_dict({'a_0': 1, 'a_1': 2, 'b_x': 'X', 'b_y': 'Y'})
92
+ {'a': [1, 2], 'b': {'x': 'X', 'y': 'Y'}}
93
+ """
94
+ result = {}
95
+ for flat_key, value in flat_dict.items():
96
+ parts = flat_key.split(sep)
97
+ d = result
98
+ for part in parts[:-1]:
99
+ part = int(part) if part.isdigit() else part
100
+ if part not in d:
101
+ if isinstance(d, list):
102
+ d.append({})
103
+ d = d[-1]
104
+ else:
105
+ d[part] = {}
106
+ d = d[part] if not isinstance(d, list) else d[-1]
107
+ last_part = parts[-1]
108
+ last_part = int(last_part) if last_part.isdigit() else last_part
109
+ if isinstance(d, list):
110
+ d.append(value)
111
+ elif last_part in d:
112
+ raise ValueError(f"Conflicting keys detected. Key part '{last_part}' is already present.")
113
+ else:
114
+ d[last_part] = value
115
+
116
+ # Convert dictionaries with contiguous integer keys starting from 0 to lists
117
+ def dict_to_list(d: dict) -> dict:
118
+ """
119
+ Converts dictionaries with contiguous integer keys starting from 0 into lists.
120
+
121
+ Args:
122
+ d (dict): A dictionary that may have integer keys suitable for conversion to a list.
123
+
124
+ Returns:
125
+ dict or list: The input dictionary or a list if the dictionary keys match the criteria.
126
+ """
127
+ if isinstance(d, dict) and all(isinstance(k, int) and k >= 0 for k in d.keys()):
128
+ keys = sorted(d.keys())
129
+ if keys == list(range(len(keys))):
130
+ return [dict_to_list(d[k]) for k in keys]
131
+ if isinstance(d, dict):
132
+ return {k: dict_to_list(v) for k, v in d.items()}
133
+ return d
134
+
135
+ result = dict_to_list(result)
136
+ return result
137
+
138
+ def is_flattenable(obj: Any) -> bool:
139
+ """
140
+ Determines if the given object contains nested dictionaries or lists, making it suitable for flattening.
141
+
142
+ Args:
143
+ obj (object): The object to check for flattenable structures.
144
+
145
+ Returns:
146
+ bool: True if the object can be flattened (contains nested dicts or lists), False otherwise.
147
+ """
148
+ if isinstance(obj, dict):
149
+ return any(isinstance(v, (dict, list)) for v in obj.values())
150
+ elif isinstance(obj, list):
151
+ return any(isinstance(i, (dict, list)) for i in obj)
152
+ return False
153
+
154
+ def flatten_with_custom_logic(obj, logic_func=None, parent_key='', sep='_'):
155
+ """
156
+ Recursively flattens a nested dictionary or list and applies custom logic to the keys and values.
157
+
158
+ Parameters:
159
+ obj (dict | list): The dictionary or list to flatten.
160
+ logic_func (callable, optional): A function that takes four arguments (parent_key, key, value, sep)
161
+ and returns a tuple (new_key, new_value) after applying custom logic.
162
+ parent_key (str): The base key to use for creating new keys.
163
+ sep (str): The separator to use when joining nested keys.
164
+
165
+ Returns:
166
+ dict: A flattened dictionary with keys representing the nested paths and values from the original object.
167
+
168
+ Example Usage:
169
+ >>> sample_dict = {'a': 1, 'b': {'c': 2, 'd': {'e': 3}}}
170
+ >>> flatten_with_custom_logic(sample_dict)
171
+ {'a': 1, 'b_c': 2, 'b_d_e': 3}
172
+
173
+ >>> sample_list = [1, 2, [3, 4]]
174
+ >>> flatten_with_custom_logic(sample_list)
175
+ {'0': 1, '1': 2, '2_0': 3, '2_1': 4}
176
+
177
+ >>> def custom_logic(parent_key, key, value, sep='_'):
178
+ ... new_key = f"{parent_key}{sep}{key}".upper() if parent_key else key.upper()
179
+ ... return new_key, value * 2 if isinstance(value, int) else value
180
+ >>> flatten_with_custom_logic(sample_dict, custom_logic)
181
+ {'A': 2, 'B_C': 4, 'B_D_E': 6}
182
+ """
183
+ items = {}
184
+ if isinstance(obj, dict):
185
+ for k, v in obj.items():
186
+ new_key = f"{parent_key}{sep}{k}" if parent_key else k
187
+ if isinstance(v, (dict, list)):
188
+ if v: # Check if the dictionary or list is not empty
189
+ items.update(flatten_with_custom_logic(v, logic_func, new_key, sep))
190
+ else: # Handle empty dictionaries and lists
191
+ items[new_key] = None
192
+ else:
193
+ if logic_func:
194
+ new_key, new_value = logic_func(parent_key, k, v, sep=sep)
195
+ else:
196
+ new_value = v
197
+ items[new_key] = new_value
198
+ elif isinstance(obj, list):
199
+ for i, item in enumerate(obj):
200
+ new_key = f"{parent_key}{sep}{str(i)}" if parent_key else str(i) # Cast index to string
201
+ if isinstance(item, (dict, list)):
202
+ if item: # Check if the dictionary or list is not empty
203
+ items.update(flatten_with_custom_logic(item, logic_func, new_key, sep))
204
+ else: # Handle empty lists
205
+ items[new_key] = None
206
+ else:
207
+ if logic_func:
208
+ new_key, new_value = logic_func(parent_key, str(i), item, sep=sep)
209
+ else:
210
+ new_value = item
211
+ items[new_key] = new_value
212
+ return items
213
+
214
+ def dynamic_flatten(obj, parent_key='', sep='_', max_depth=None, current_depth=0):
215
+ """
216
+ Recursively flattens a nested dictionary or list, while allowing a maximum depth and custom separators.
217
+
218
+ Parameters:
219
+ obj (dict | list): The dictionary or list to flatten.
220
+ parent_key (str): The base key to use for creating new keys.
221
+ sep (str): The separator to use when joining nested keys.
222
+ max_depth (int, optional): The maximum depth to flatten.
223
+ current_depth (int): The current depth in the recursive call (used internally).
224
+
225
+ Returns:
226
+ dict: A flattened dictionary with keys representing the nested paths and values from the original object.
227
+
228
+ Raises:
229
+ TypeError: If the input object is neither a dictionary nor a list.
230
+
231
+ Example Usage:
232
+ >>> sample_dict = {'a': 1, 'b': {'c': 2, 'd': {'e': 3}}}
233
+ >>> dynamic_flatten(sample_dict)
234
+ {'a': 1, 'b_c': 2, 'b_d_e': 3}
235
+
236
+ >>> sample_list = [1, 2, [3, 4]]
237
+ >>> dynamic_flatten(sample_list)
238
+ {'0': 1, '1': 2, '2_0': 3, '2_1': 4}
239
+
240
+ >>> dynamic_flatten(sample_dict, max_depth=1)
241
+ {'a': 1, 'b_c': 2, 'b_d': {'e': 3}}
242
+
243
+ >>> dynamic_flatten(sample_dict, sep='.')
244
+ {'a': 1, 'b.c': 2, 'b.d.e': 3}
245
+ """
246
+ items = []
247
+ if isinstance(obj, dict):
248
+ if not obj and parent_key:
249
+ items.append((parent_key, obj))
250
+ for k, v in obj.items():
251
+ new_key = f"{parent_key}{sep}{k}" if parent_key else k
252
+ if isinstance(v, (dict, list)) and (max_depth is None or current_depth < max_depth):
253
+ items.extend(dynamic_flatten(v, new_key, sep, max_depth, current_depth + 1).items())
254
+ else:
255
+ items.append((new_key, v))
256
+ elif isinstance(obj, list):
257
+ if not obj and parent_key:
258
+ items.append((parent_key, obj))
259
+ for i, item in enumerate(obj):
260
+ new_key = f"{parent_key}{sep}{i}" if parent_key else str(i)
261
+ if isinstance(item, (dict, list)) and (max_depth is None or current_depth < max_depth):
262
+ items.extend(dynamic_flatten(item, new_key, sep, max_depth, current_depth + 1).items())
263
+ else:
264
+ items.append((new_key, item))
265
+ else:
266
+ raise TypeError('Input object must be a dictionary or a list')
267
+
268
+ return dict(items)
269
+
270
+ # def dynamic_unflatten_dict(flat_dict, sep='_', custom_logic=None, max_depth=None):
271
+ # """
272
+ # Unflattens a dictionary with flat keys into a nested dictionary or list.
273
+
274
+ # :param flat_dict: A dictionary with flat keys that need to be nested.
275
+ # :param sep: The separator used in the flat keys (default is '_').
276
+ # :param custom_logic: A function that customizes the processing of keys.
277
+ # It takes a key part as input and returns the transformed key part.
278
+ # :param max_depth: The maximum depth to which the dictionary should be nested.
279
+ # If None, there is no maximum depth.
280
+ # :return: A nested dictionary or list based on the flat input dictionary.
281
+
282
+ # The function dynamically unflattens a dictionary with keys that represent nested paths.
283
+ # For example, the flat dictionary {'a_b_c': 1, 'a_b_d': 2} would be unflattened to
284
+ # {'a': {'b': {'c': 1, 'd': 2}}}. If the keys can be converted to integers and suggest list indices,
285
+ # the function produces lists instead of dictionaries. For example,
286
+ # {'0_a': 'foo', '1_b': 'bar'} becomes [{'a': 'foo'}, {'b': 'bar'}].
287
+ # """
288
+
289
+ # def handle_list_insert(sub_obj, part, value):
290
+ # # Ensure part index exists in the list, fill gaps with None
291
+ # while len(sub_obj) <= part:
292
+ # sub_obj.append(None)
293
+
294
+ # sub_obj[part] = value
295
+
296
+ # def insert(sub_obj, parts, value, max_depth, current_depth=0):
297
+ # for part in parts[:-1]:
298
+ # # Stop nesting further if max_depth is reached
299
+ # if max_depth is not None and current_depth >= max_depth:
300
+ # sub_obj[part] = {parts[-1]: value}
301
+ # return
302
+ # # Handle integer parts for list insertion
303
+ # if isinstance(part, int):
304
+ # # Ensure part index exists in the list or dict, fill gaps with None
305
+ # while len(sub_obj) <= part:
306
+ # sub_obj.append(None)
307
+ # if sub_obj[part] is None:
308
+ # if current_depth < max_depth - 1 if max_depth else True:
309
+ # sub_obj[part] = [] if isinstance(parts[parts.index(part) + 1], int) else {}
310
+ # sub_obj = sub_obj[part]
311
+ # else:
312
+ # # Handle string parts for dictionary insertion
313
+ # if part not in sub_obj:
314
+ # sub_obj[part] = {}
315
+ # sub_obj = sub_obj[part]
316
+ # current_depth += 1
317
+ # # Insert the value at the last part
318
+ # last_part = parts[-1]
319
+ # if isinstance(last_part, int) and isinstance(sub_obj, list):
320
+ # handle_list_insert(sub_obj, last_part, value, max_depth, current_depth)
321
+ # else:
322
+ # sub_obj[last_part] = value
323
+
324
+ # unflattened = {}
325
+ # for composite_key, value in flat_dict.items():
326
+ # parts = composite_key.split(sep)
327
+ # if custom_logic:
328
+ # parts = [custom_logic(part) for part in parts]
329
+ # else:
330
+ # parts = [int(part) if (isinstance(part, int) or part.isdigit()) else part for part in parts]
331
+ # insert(unflattened, parts, value, max_depth)
332
+
333
+ # # Convert top-level dictionary to a list if all keys are integers
334
+ # if isinstance(unflattened, dict) and all(isinstance(k, int) for k in unflattened.keys()):
335
+ # max_index = max(unflattened.keys(), default=-1)
336
+ # return [unflattened.get(i) for i in range(max_index + 1)]
337
+ # # Return an empty dictionary instead of converting to a list if unflattened is empty
338
+ # if (unflattened == []) or (not unflattened):
339
+ # return {}
340
+ # return unflattened
341
+
342
+ def _insert_with_dict_handling(container, indices, value):
343
+ """
344
+ Helper function to insert a value into a nested container based on a list of indices.
345
+
346
+ :param container: The container (list or dict) to insert the value into.
347
+ :param indices: A list of indices indicating the path to the insertion point.
348
+ :param value: The value to be inserted.
349
+ """
350
+ for i, index in enumerate(indices[:-1]):
351
+ # Check if index is an integer and ensure the list is long enough
352
+ if isinstance(index, int):
353
+ # Extend the list if necessary
354
+ while len(container) <= index:
355
+ container.append(None)
356
+
357
+ # Create a nested container if the current position is None
358
+ if container[index] is None:
359
+ next_index = indices[i + 1]
360
+
361
+ # Determine the type of the next container based on the next index
362
+ container[index] = {} if isinstance(next_index, str) else []
363
+
364
+ # Update the reference to point to the nested container
365
+ container = container[index]
366
+
367
+ # If index is a string, work with dictionaries
368
+ else:
369
+ if index not in container:
370
+ container[index] = {}
371
+ container = container[index]
372
+
373
+ # Insert value at the last index
374
+ last_index = indices[-1]
375
+ if isinstance(last_index, int):
376
+
377
+ # Negative indices are not allowed in this context
378
+ if last_index < 0:
379
+ raise ValueError("Negative index is not allowed for list unflattening.")
380
+
381
+ while len(container) <= last_index:
382
+ container.append(None)
383
+ container[last_index] = value
384
+
385
+ else:
386
+ if last_index in container and isinstance(container[last_index], list):
387
+ raise ValueError("Overlapping keys are not allowed.")
388
+ container[last_index] = value
389
+
390
+ def unflatten_to_list(flat_dict: Dict[str, Any], sep: str = '_') -> List:
391
+ """
392
+ Unflattens a dictionary with keys as string paths into a nested list structure.
393
+
394
+ :param flat_dict: The flat dictionary to unflatten.
395
+ :param sep: The separator used in the flat dictionary keys to indicate nesting.
396
+ :return: A nested list that represents the unflattened structure.
397
+ """
398
+ result_list = []
399
+ for flat_key, value in flat_dict.items():
400
+ indices = [int(p) if p.lstrip('-').isdigit() else p for p in flat_key.split(sep)]
401
+ _insert_with_dict_handling(result_list, indices, value)
402
+ return result_list
403
+
404
+ def flatten_iterable(iterable: Iterable, max_depth: int = None) -> Generator[Any, None, None]:
405
+ """
406
+ Flattens a nested iterable up to a specified maximum depth.
407
+
408
+ Args:
409
+ iterable: An iterable to flatten.
410
+ max_depth: The maximum depth to flatten. If None, flattens completely.
411
+
412
+ Yields:
413
+ The flattened elements of the original iterable.
414
+ """
415
+ def _flatten(input_iterable: Iterable, current_depth: int) -> Generator[Any, None, None]:
416
+ if isinstance(input_iterable, Iterable) and not isinstance(input_iterable, (str, bytes)):
417
+ if max_depth is not None and current_depth >= max_depth:
418
+ yield input_iterable
419
+ else:
420
+ for item in input_iterable:
421
+ yield from _flatten(item, current_depth + 1)
422
+ else:
423
+ yield input_iterable
424
+
425
+ yield from _flatten(iterable, 0)
426
+
427
+ def flatten_iterable_to_list(iterable: Iterable, max_depth: int = None) -> List[Any]:
428
+ """
429
+ Converts a nested iterable into a flattened list up to a specified maximum depth.
430
+
431
+ Args:
432
+ iterable: An iterable to flatten.
433
+ max_depth: The maximum depth to flatten. If None, flattens completely.
434
+
435
+ Returns:
436
+ A list containing the flattened elements of the original iterable.
437
+ """
438
+ return list(flatten_iterable(iterable, max_depth))
439
+
440
+ def unflatten_dict_with_custom_logic(
441
+ flat_dict: Dict[str, Any],
442
+ logic_func: Callable[[str, Any], Tuple[str, Any]],
443
+ sep: str = '_'
444
+ ) -> Dict[str, Any]:
445
+ """
446
+ Unflattens a dictionary with keys as string paths into a nested dictionary structure
447
+ while applying custom logic to each key and value.
448
+
449
+ Args:
450
+ flat_dict: The flat dictionary to unflatten.
451
+ logic_func: A function that takes a key and a value and returns a tuple of modified key and value.
452
+ sep: The separator used in the flat dictionary keys to indicate nesting.
453
+
454
+ Returns:
455
+ A nested dictionary that represents the unflattened structure with modified keys and values.
456
+ """
457
+ reconstructed = {}
458
+ for flat_key, value in flat_dict.items():
459
+ parts = flat_key.split(sep)
460
+ d = reconstructed
461
+ for part in parts[:-1]:
462
+ modified_part, _ = logic_func(part, None) # Modify only the key
463
+ d = d.setdefault(modified_part, {})
464
+ last_part = parts[-1]
465
+ modified_last_part, modified_value = logic_func(last_part, value)
466
+ d[modified_last_part] = modified_value
467
+ return reconstructed
468
+
469
+ # def dynamic_unflatten(flat_dict, sep='_', custom_logic=None, max_depth=None):
470
+ # """
471
+ # Unflattens a dictionary with flat keys into a nested dictionary or list.
472
+
473
+ # :param flat_dict: A dictionary with flat keys that need to be nested.
474
+ # :param sep: The separator used in the flat keys (default is '_').
475
+ # :param custom_logic: A function that customizes the processing of keys.
476
+ # It takes a key part as input and returns the transformed key part.
477
+ # :param max_depth: The maximum depth to which the dictionary should be nested.
478
+ # If None, there is no maximum depth.
479
+ # :return: A nested dictionary or list based on the flat input dictionary.
480
+
481
+ # The function dynamically unflattens a dictionary with keys that represent nested paths.
482
+ # For example, the flat dictionary {'a_b_c': 1, 'a_b_d': 2} would be unflattened to
483
+ # {'a': {'b': {'c': 1, 'd': 2}}}. If the keys can be converted to integers and suggest list indices,
484
+ # the function produces lists instead of dictionaries. For example,
485
+ # {'0_a': 'foo', '1_b': 'bar'} becomes [{'a': 'foo'}, {'b': 'bar'}].
486
+ # """
487
+
488
+ # def handle_list_insert(sub_obj, part, value):
489
+ # # Ensure part index exists in the list, fill gaps with None
490
+ # while len(sub_obj) <= part:
491
+ # sub_obj.append(None)
492
+
493
+ # sub_obj[part] = value
494
+
495
+ # def insert(sub_obj, parts, value, max_depth, current_depth=0):
496
+ # for part in parts[:-1]:
497
+ # # Stop nesting further if max_depth is reached
498
+ # if max_depth is not None and current_depth >= max_depth:
499
+ # sub_obj[part] = {parts[-1]: value}
500
+ # return
501
+ # # Handle integer parts for list insertion
502
+ # if isinstance(part, int):
503
+ # # Ensure part index exists in the list or dict, fill gaps with None
504
+ # while len(sub_obj) <= part:
505
+ # sub_obj.append(None)
506
+ # if sub_obj[part] is None:
507
+ # if current_depth < max_depth - 1 if max_depth else True:
508
+ # sub_obj[part] = [] if isinstance(parts[parts.index(part) + 1], int) else {}
509
+ # sub_obj = sub_obj[part]
510
+ # else:
511
+ # # Handle string parts for dictionary insertion
512
+ # if part not in sub_obj:
513
+ # sub_obj[part] = {}
514
+ # sub_obj = sub_obj[part]
515
+ # current_depth += 1
516
+ # # Insert the value at the last part
517
+ # last_part = parts[-1]
518
+ # if isinstance(last_part, int) and isinstance(sub_obj, list):
519
+ # handle_list_insert(sub_obj, last_part, value, max_depth, current_depth)
520
+ # else:
521
+ # sub_obj[last_part] = value
522
+
523
+ # unflattened = {}
524
+ # for composite_key, value in flat_dict.items():
525
+ # parts = composite_key.split(sep)
526
+ # if custom_logic:
527
+ # parts = [custom_logic(part) for part in parts]
528
+ # else:
529
+ # parts = [int(part) if (isinstance(part, int) or part.isdigit()) else part for part in parts]
530
+ # insert(unflattened, parts, value, max_depth)
531
+
532
+ # # Convert top-level dictionary to a list if all keys are integers
533
+ # if isinstance(unflattened, dict) and all(isinstance(k, int) for k in unflattened.keys()):
534
+ # max_index = max(unflattened.keys(), default=-1)
535
+ # return [unflattened.get(i) for i in range(max_index + 1)]
536
+ # # Return an empty dictionary instead of converting to a list if unflattened is empty
537
+ # if (unflattened == []) or (not unflattened):
538
+ # return {}
539
+ # return unflattened
540
+
@@ -0,0 +1,102 @@
1
+ import csv
2
+ import json
3
+ import os
4
+ import tempfile
5
+ from typing import Any, Dict, List
6
+ from .type_util import to_list
7
+
8
+
9
+ def to_temp(input: Any,
10
+ flatten_dict: bool = False,
11
+ flat: bool = False,
12
+ dropna: bool = False):
13
+ """
14
+ Converts input to a list and writes it to a temporary file in JSON format, with flattening options.
15
+
16
+ This function serializes data to a temporary JSON file, useful for transient storage or testing.
17
+ It includes options to flatten the input if it contains dictionaries or lists.
18
+
19
+ Parameters:
20
+ input (Any): The data to be converted and written to a file.
21
+
22
+ flatten_dict (bool, optional): Flatten dictionaries in the input. Defaults to False.
23
+
24
+ flat (bool, optional): Flatten lists in the input. Defaults to False.
25
+
26
+ dropna (bool, optional): Exclude 'None' values during flattening. Defaults to False.
27
+
28
+ Raises:
29
+ TypeError: If the input is not JSON serializable.
30
+
31
+ Example:
32
+ >>> temp_file = to_temp({'a': 1, 'b': [2, 3]}, flatten_dict=True)
33
+ >>> temp_file.name # Doctest: +ELLIPSIS
34
+ '/var/folders/.../tmp...'
35
+ """
36
+ input = to_list(input, flatten_dict, flat, dropna)
37
+
38
+ temp_file = tempfile.NamedTemporaryFile(mode='w', delete=False)
39
+ try:
40
+ json.dump(input, temp_file)
41
+ except TypeError as e:
42
+ temp_file.close() # Ensuring file closure before raising error
43
+ raise TypeError(f"Data provided is not JSON serializable: {e}")
44
+ temp_file.close()
45
+ return temp_file
46
+
47
+ def to_csv(input: List[Dict[str, Any]]=None,
48
+ filepath: str=None,
49
+ file_exist_ok: bool = False) -> None:
50
+ """
51
+ Writes a list of dictionaries to a CSV file, with dictionary keys as headers.
52
+
53
+ This function writes a list of dictionaries to a CSV file. It checks if the file exists
54
+ and handles file creation based on the 'file_exist_ok' flag.
55
+
56
+ Parameters:
57
+ input (List[Dict[str, Any]]): Data to write to the CSV file.
58
+
59
+ filepath (str): Path of the output CSV file.
60
+
61
+ file_exist_ok (bool, optional): Create the file if it doesn't exist. Defaults to False.
62
+
63
+ Raises:
64
+ FileExistsError: If the file already exists and 'file_exist_ok' is False.
65
+
66
+ Example:
67
+ >>> data = [{'name': 'Alice', 'age': 30}, {'name': 'Bob', 'age': 25}]
68
+ >>> to_csv(data, 'people.csv')
69
+ """
70
+
71
+ if not os.path.exists(os.path.dirname(filepath)) and os.path.dirname(filepath) != '':
72
+ if file_exist_ok:
73
+ os.makedirs(os.path.dirname(filepath), exist_ok=True)
74
+ else:
75
+ raise FileNotFoundError(f"The directory {os.path.dirname(filepath)} does not exist.")
76
+
77
+ with open(filepath, 'w', newline='') as csv_file:
78
+ writer = csv.DictWriter(csv_file, fieldnames=input[0].keys())
79
+ writer.writeheader()
80
+ writer.writerows(input)
81
+
82
+
83
+ def append_to_jsonl(data: Any, filepath: str) -> None:
84
+ """
85
+ Appends data to a JSON lines (jsonl) file.
86
+
87
+ Serializes given data to a JSON-formatted string and appends it to a jsonl file.
88
+ Useful for logging or data collection where entries are added incrementally.
89
+
90
+ Parameters:
91
+ data (Any): Data to be serialized and appended.
92
+
93
+ filepath (str): Path to the jsonl file.
94
+
95
+ Example:
96
+ >>> append_to_jsonl({"key": "value"}, "data.jsonl")
97
+ # Appends {"key": "value"} to 'data.jsonl'
98
+ """
99
+ json_string = json.dumps(data)
100
+ with open(filepath, "a") as f:
101
+ f.write(json_string + "\n")
102
+