lionagi 0.0.111__py3-none-any.whl → 0.0.113__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (91) hide show
  1. lionagi/__init__.py +7 -2
  2. lionagi/bridge/__init__.py +7 -0
  3. lionagi/bridge/langchain.py +131 -0
  4. lionagi/bridge/llama_index.py +157 -0
  5. lionagi/configs/__init__.py +7 -0
  6. lionagi/configs/oai_configs.py +49 -0
  7. lionagi/configs/openrouter_config.py +49 -0
  8. lionagi/core/__init__.py +15 -0
  9. lionagi/{session/conversation.py → core/conversations.py} +10 -17
  10. lionagi/core/flows.py +1 -0
  11. lionagi/core/instruction_sets.py +1 -0
  12. lionagi/{session/message.py → core/messages.py} +5 -5
  13. lionagi/core/sessions.py +262 -0
  14. lionagi/datastore/__init__.py +1 -0
  15. lionagi/datastore/chroma.py +1 -0
  16. lionagi/datastore/deeplake.py +1 -0
  17. lionagi/datastore/elasticsearch.py +1 -0
  18. lionagi/datastore/lantern.py +1 -0
  19. lionagi/datastore/pinecone.py +1 -0
  20. lionagi/datastore/postgres.py +1 -0
  21. lionagi/datastore/qdrant.py +1 -0
  22. lionagi/loader/__init__.py +12 -0
  23. lionagi/loader/chunker.py +157 -0
  24. lionagi/loader/reader.py +124 -0
  25. lionagi/objs/__init__.py +7 -0
  26. lionagi/objs/messenger.py +163 -0
  27. lionagi/objs/tool_registry.py +247 -0
  28. lionagi/schema/__init__.py +11 -0
  29. lionagi/schema/base_condition.py +1 -0
  30. lionagi/schema/base_schema.py +239 -0
  31. lionagi/schema/base_tool.py +9 -0
  32. lionagi/schema/data_logger.py +94 -0
  33. lionagi/services/__init__.py +14 -0
  34. lionagi/services/anthropic.py +1 -0
  35. lionagi/services/anyscale.py +0 -0
  36. lionagi/services/azure.py +1 -0
  37. lionagi/{api/oai_service.py → services/base_api_service.py} +74 -148
  38. lionagi/services/bedrock.py +0 -0
  39. lionagi/services/chatcompletion.py +48 -0
  40. lionagi/services/everlyai.py +0 -0
  41. lionagi/services/gemini.py +0 -0
  42. lionagi/services/gpt4all.py +0 -0
  43. lionagi/services/huggingface.py +0 -0
  44. lionagi/services/litellm.py +1 -0
  45. lionagi/services/localai.py +0 -0
  46. lionagi/services/mistralai.py +0 -0
  47. lionagi/services/oai.py +34 -0
  48. lionagi/services/ollama.py +1 -0
  49. lionagi/services/openllm.py +0 -0
  50. lionagi/services/openrouter.py +32 -0
  51. lionagi/services/perplexity.py +0 -0
  52. lionagi/services/predibase.py +0 -0
  53. lionagi/services/rungpt.py +0 -0
  54. lionagi/services/service_objs.py +282 -0
  55. lionagi/services/vllm.py +0 -0
  56. lionagi/services/xinference.py +0 -0
  57. lionagi/structure/__init__.py +7 -0
  58. lionagi/structure/relationship.py +128 -0
  59. lionagi/structure/structure.py +160 -0
  60. lionagi/tests/__init__.py +0 -0
  61. lionagi/tests/test_flatten_util.py +426 -0
  62. lionagi/tools/__init__.py +0 -0
  63. lionagi/tools/coder.py +1 -0
  64. lionagi/tools/planner.py +1 -0
  65. lionagi/tools/prompter.py +1 -0
  66. lionagi/tools/sandbox.py +1 -0
  67. lionagi/tools/scorer.py +1 -0
  68. lionagi/tools/summarizer.py +1 -0
  69. lionagi/tools/validator.py +1 -0
  70. lionagi/utils/__init__.py +46 -8
  71. lionagi/utils/api_util.py +63 -416
  72. lionagi/utils/call_util.py +347 -0
  73. lionagi/utils/flat_util.py +540 -0
  74. lionagi/utils/io_util.py +102 -0
  75. lionagi/utils/load_utils.py +190 -0
  76. lionagi/utils/sys_util.py +85 -660
  77. lionagi/utils/tool_util.py +82 -199
  78. lionagi/utils/type_util.py +81 -0
  79. lionagi/version.py +1 -1
  80. {lionagi-0.0.111.dist-info → lionagi-0.0.113.dist-info}/METADATA +44 -15
  81. lionagi-0.0.113.dist-info/RECORD +84 -0
  82. lionagi/api/__init__.py +0 -8
  83. lionagi/api/oai_config.py +0 -16
  84. lionagi/session/__init__.py +0 -7
  85. lionagi/session/session.py +0 -380
  86. lionagi/utils/doc_util.py +0 -331
  87. lionagi/utils/log_util.py +0 -86
  88. lionagi-0.0.111.dist-info/RECORD +0 -20
  89. {lionagi-0.0.111.dist-info → lionagi-0.0.113.dist-info}/LICENSE +0 -0
  90. {lionagi-0.0.111.dist-info → lionagi-0.0.113.dist-info}/WHEEL +0 -0
  91. {lionagi-0.0.111.dist-info → lionagi-0.0.113.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,540 @@
1
+ from typing import Dict, Iterable, List, Any, Callable, Generator, Tuple
2
+
3
+
4
+ def flatten_dict(d: Dict, parent_key: str = '', sep: str = '_') -> Dict:
5
+ """
6
+ Flattens a nested dictionary by concatenating keys.
7
+
8
+ This function recursively flattens a nested dictionary by concatenating
9
+ the keys of nested dictionaries with a separator. The default separator
10
+ is an underscore (_).
11
+
12
+ Parameters:
13
+ d (dict): The dictionary to flatten.
14
+
15
+ parent_key (str, optional): The base key to use for the current level of recursion.
16
+ Defaults to an empty string, meaning no parent key, key cannot be a number
17
+
18
+ sep (str, optional): The separator to use when concatenating keys.
19
+ Defaults to an underscore (_).
20
+
21
+ Returns:
22
+ dict: A new dictionary with flattened keys and corresponding values.
23
+ """
24
+ items = []
25
+ for k, v in d.items():
26
+ new_key = f"{parent_key}{sep}{k}" if parent_key else k
27
+ if isinstance(v, dict):
28
+ items.extend(flatten_dict(v, new_key, sep=sep).items())
29
+ else:
30
+ items.append((new_key, v))
31
+ return dict(items)
32
+
33
+ def flatten_list(l: List, dropna: bool = True) -> List:
34
+ """
35
+ Flattens a nested list into a single list of values.
36
+
37
+ The function iterates over each element in the provided list. If an
38
+ element is a list itself, the function is called recursively to flatten
39
+ it. If the element is not a list, it is appended to the result list.
40
+ Optionally, None values can be dropped from the result list.
41
+
42
+ Parameters:
43
+ l (list): The list to flatten, which may contain nested lists.
44
+
45
+ dropna (bool, optional): Whether to exclude None values from the result list.
46
+ Defaults to True.
47
+
48
+ Returns:
49
+ list: A new flattened list with or without None values based on the dropna parameter.
50
+ """
51
+ flat_list = []
52
+ for i in l:
53
+ if isinstance(i, list):
54
+ flat_list.extend(flatten_list(i, dropna))
55
+ elif i is not None or not dropna:
56
+ flat_list.append(i)
57
+ return flat_list
58
+
59
+ def change_separator(flat_dict, current_sep, new_sep):
60
+ """
61
+ Changes the separator in the keys of a flat dictionary.
62
+
63
+ Parameters:
64
+ flat_dict (dict): The dictionary with keys containing the current separator.
65
+ current_sep (str): The current separator used in the dictionary keys.
66
+ new_sep (str): The new separator to replace the current separator in the dictionary keys.
67
+
68
+ Returns:
69
+ dict: A new dictionary with the separators in the keys replaced.
70
+ """
71
+ return {
72
+ k.replace(current_sep, new_sep): v
73
+ for k, v in flat_dict.items()
74
+ }
75
+
76
+ def unflatten_dict(flat_dict: Dict, sep: str = '_') -> Dict:
77
+ """
78
+ Unflattens a dictionary where keys are strings that represent nested dictionary paths separated by 'sep'.
79
+
80
+ Args:
81
+ flat_dict (dict): A dictionary with keys as strings that represent paths.
82
+ sep (str): The separator used in the keys of the flat dictionary.
83
+
84
+ Returns:
85
+ dict: A nested dictionary unflattened from the keys of the input dictionary.
86
+
87
+ Raises:
88
+ ValueError: If there are conflicting keys in the path.
89
+
90
+ Example:
91
+ >>> unflatten_dict({'a_0': 1, 'a_1': 2, 'b_x': 'X', 'b_y': 'Y'})
92
+ {'a': [1, 2], 'b': {'x': 'X', 'y': 'Y'}}
93
+ """
94
+ result = {}
95
+ for flat_key, value in flat_dict.items():
96
+ parts = flat_key.split(sep)
97
+ d = result
98
+ for part in parts[:-1]:
99
+ part = int(part) if part.isdigit() else part
100
+ if part not in d:
101
+ if isinstance(d, list):
102
+ d.append({})
103
+ d = d[-1]
104
+ else:
105
+ d[part] = {}
106
+ d = d[part] if not isinstance(d, list) else d[-1]
107
+ last_part = parts[-1]
108
+ last_part = int(last_part) if last_part.isdigit() else last_part
109
+ if isinstance(d, list):
110
+ d.append(value)
111
+ elif last_part in d:
112
+ raise ValueError(f"Conflicting keys detected. Key part '{last_part}' is already present.")
113
+ else:
114
+ d[last_part] = value
115
+
116
+ # Convert dictionaries with contiguous integer keys starting from 0 to lists
117
+ def dict_to_list(d: dict) -> dict:
118
+ """
119
+ Converts dictionaries with contiguous integer keys starting from 0 into lists.
120
+
121
+ Args:
122
+ d (dict): A dictionary that may have integer keys suitable for conversion to a list.
123
+
124
+ Returns:
125
+ dict or list: The input dictionary or a list if the dictionary keys match the criteria.
126
+ """
127
+ if isinstance(d, dict) and all(isinstance(k, int) and k >= 0 for k in d.keys()):
128
+ keys = sorted(d.keys())
129
+ if keys == list(range(len(keys))):
130
+ return [dict_to_list(d[k]) for k in keys]
131
+ if isinstance(d, dict):
132
+ return {k: dict_to_list(v) for k, v in d.items()}
133
+ return d
134
+
135
+ result = dict_to_list(result)
136
+ return result
137
+
138
+ def is_flattenable(obj: Any) -> bool:
139
+ """
140
+ Determines if the given object contains nested dictionaries or lists, making it suitable for flattening.
141
+
142
+ Args:
143
+ obj (object): The object to check for flattenable structures.
144
+
145
+ Returns:
146
+ bool: True if the object can be flattened (contains nested dicts or lists), False otherwise.
147
+ """
148
+ if isinstance(obj, dict):
149
+ return any(isinstance(v, (dict, list)) for v in obj.values())
150
+ elif isinstance(obj, list):
151
+ return any(isinstance(i, (dict, list)) for i in obj)
152
+ return False
153
+
154
+ def flatten_with_custom_logic(obj, logic_func=None, parent_key='', sep='_'):
155
+ """
156
+ Recursively flattens a nested dictionary or list and applies custom logic to the keys and values.
157
+
158
+ Parameters:
159
+ obj (dict | list): The dictionary or list to flatten.
160
+ logic_func (callable, optional): A function that takes four arguments (parent_key, key, value, sep)
161
+ and returns a tuple (new_key, new_value) after applying custom logic.
162
+ parent_key (str): The base key to use for creating new keys.
163
+ sep (str): The separator to use when joining nested keys.
164
+
165
+ Returns:
166
+ dict: A flattened dictionary with keys representing the nested paths and values from the original object.
167
+
168
+ Example Usage:
169
+ >>> sample_dict = {'a': 1, 'b': {'c': 2, 'd': {'e': 3}}}
170
+ >>> flatten_with_custom_logic(sample_dict)
171
+ {'a': 1, 'b_c': 2, 'b_d_e': 3}
172
+
173
+ >>> sample_list = [1, 2, [3, 4]]
174
+ >>> flatten_with_custom_logic(sample_list)
175
+ {'0': 1, '1': 2, '2_0': 3, '2_1': 4}
176
+
177
+ >>> def custom_logic(parent_key, key, value, sep='_'):
178
+ ... new_key = f"{parent_key}{sep}{key}".upper() if parent_key else key.upper()
179
+ ... return new_key, value * 2 if isinstance(value, int) else value
180
+ >>> flatten_with_custom_logic(sample_dict, custom_logic)
181
+ {'A': 2, 'B_C': 4, 'B_D_E': 6}
182
+ """
183
+ items = {}
184
+ if isinstance(obj, dict):
185
+ for k, v in obj.items():
186
+ new_key = f"{parent_key}{sep}{k}" if parent_key else k
187
+ if isinstance(v, (dict, list)):
188
+ if v: # Check if the dictionary or list is not empty
189
+ items.update(flatten_with_custom_logic(v, logic_func, new_key, sep))
190
+ else: # Handle empty dictionaries and lists
191
+ items[new_key] = None
192
+ else:
193
+ if logic_func:
194
+ new_key, new_value = logic_func(parent_key, k, v, sep=sep)
195
+ else:
196
+ new_value = v
197
+ items[new_key] = new_value
198
+ elif isinstance(obj, list):
199
+ for i, item in enumerate(obj):
200
+ new_key = f"{parent_key}{sep}{str(i)}" if parent_key else str(i) # Cast index to string
201
+ if isinstance(item, (dict, list)):
202
+ if item: # Check if the dictionary or list is not empty
203
+ items.update(flatten_with_custom_logic(item, logic_func, new_key, sep))
204
+ else: # Handle empty lists
205
+ items[new_key] = None
206
+ else:
207
+ if logic_func:
208
+ new_key, new_value = logic_func(parent_key, str(i), item, sep=sep)
209
+ else:
210
+ new_value = item
211
+ items[new_key] = new_value
212
+ return items
213
+
214
+ def dynamic_flatten(obj, parent_key='', sep='_', max_depth=None, current_depth=0):
215
+ """
216
+ Recursively flattens a nested dictionary or list, while allowing a maximum depth and custom separators.
217
+
218
+ Parameters:
219
+ obj (dict | list): The dictionary or list to flatten.
220
+ parent_key (str): The base key to use for creating new keys.
221
+ sep (str): The separator to use when joining nested keys.
222
+ max_depth (int, optional): The maximum depth to flatten.
223
+ current_depth (int): The current depth in the recursive call (used internally).
224
+
225
+ Returns:
226
+ dict: A flattened dictionary with keys representing the nested paths and values from the original object.
227
+
228
+ Raises:
229
+ TypeError: If the input object is neither a dictionary nor a list.
230
+
231
+ Example Usage:
232
+ >>> sample_dict = {'a': 1, 'b': {'c': 2, 'd': {'e': 3}}}
233
+ >>> dynamic_flatten(sample_dict)
234
+ {'a': 1, 'b_c': 2, 'b_d_e': 3}
235
+
236
+ >>> sample_list = [1, 2, [3, 4]]
237
+ >>> dynamic_flatten(sample_list)
238
+ {'0': 1, '1': 2, '2_0': 3, '2_1': 4}
239
+
240
+ >>> dynamic_flatten(sample_dict, max_depth=1)
241
+ {'a': 1, 'b_c': 2, 'b_d': {'e': 3}}
242
+
243
+ >>> dynamic_flatten(sample_dict, sep='.')
244
+ {'a': 1, 'b.c': 2, 'b.d.e': 3}
245
+ """
246
+ items = []
247
+ if isinstance(obj, dict):
248
+ if not obj and parent_key:
249
+ items.append((parent_key, obj))
250
+ for k, v in obj.items():
251
+ new_key = f"{parent_key}{sep}{k}" if parent_key else k
252
+ if isinstance(v, (dict, list)) and (max_depth is None or current_depth < max_depth):
253
+ items.extend(dynamic_flatten(v, new_key, sep, max_depth, current_depth + 1).items())
254
+ else:
255
+ items.append((new_key, v))
256
+ elif isinstance(obj, list):
257
+ if not obj and parent_key:
258
+ items.append((parent_key, obj))
259
+ for i, item in enumerate(obj):
260
+ new_key = f"{parent_key}{sep}{i}" if parent_key else str(i)
261
+ if isinstance(item, (dict, list)) and (max_depth is None or current_depth < max_depth):
262
+ items.extend(dynamic_flatten(item, new_key, sep, max_depth, current_depth + 1).items())
263
+ else:
264
+ items.append((new_key, item))
265
+ else:
266
+ raise TypeError('Input object must be a dictionary or a list')
267
+
268
+ return dict(items)
269
+
270
+ # def dynamic_unflatten_dict(flat_dict, sep='_', custom_logic=None, max_depth=None):
271
+ # """
272
+ # Unflattens a dictionary with flat keys into a nested dictionary or list.
273
+
274
+ # :param flat_dict: A dictionary with flat keys that need to be nested.
275
+ # :param sep: The separator used in the flat keys (default is '_').
276
+ # :param custom_logic: A function that customizes the processing of keys.
277
+ # It takes a key part as input and returns the transformed key part.
278
+ # :param max_depth: The maximum depth to which the dictionary should be nested.
279
+ # If None, there is no maximum depth.
280
+ # :return: A nested dictionary or list based on the flat input dictionary.
281
+
282
+ # The function dynamically unflattens a dictionary with keys that represent nested paths.
283
+ # For example, the flat dictionary {'a_b_c': 1, 'a_b_d': 2} would be unflattened to
284
+ # {'a': {'b': {'c': 1, 'd': 2}}}. If the keys can be converted to integers and suggest list indices,
285
+ # the function produces lists instead of dictionaries. For example,
286
+ # {'0_a': 'foo', '1_b': 'bar'} becomes [{'a': 'foo'}, {'b': 'bar'}].
287
+ # """
288
+
289
+ # def handle_list_insert(sub_obj, part, value):
290
+ # # Ensure part index exists in the list, fill gaps with None
291
+ # while len(sub_obj) <= part:
292
+ # sub_obj.append(None)
293
+
294
+ # sub_obj[part] = value
295
+
296
+ # def insert(sub_obj, parts, value, max_depth, current_depth=0):
297
+ # for part in parts[:-1]:
298
+ # # Stop nesting further if max_depth is reached
299
+ # if max_depth is not None and current_depth >= max_depth:
300
+ # sub_obj[part] = {parts[-1]: value}
301
+ # return
302
+ # # Handle integer parts for list insertion
303
+ # if isinstance(part, int):
304
+ # # Ensure part index exists in the list or dict, fill gaps with None
305
+ # while len(sub_obj) <= part:
306
+ # sub_obj.append(None)
307
+ # if sub_obj[part] is None:
308
+ # if current_depth < max_depth - 1 if max_depth else True:
309
+ # sub_obj[part] = [] if isinstance(parts[parts.index(part) + 1], int) else {}
310
+ # sub_obj = sub_obj[part]
311
+ # else:
312
+ # # Handle string parts for dictionary insertion
313
+ # if part not in sub_obj:
314
+ # sub_obj[part] = {}
315
+ # sub_obj = sub_obj[part]
316
+ # current_depth += 1
317
+ # # Insert the value at the last part
318
+ # last_part = parts[-1]
319
+ # if isinstance(last_part, int) and isinstance(sub_obj, list):
320
+ # handle_list_insert(sub_obj, last_part, value, max_depth, current_depth)
321
+ # else:
322
+ # sub_obj[last_part] = value
323
+
324
+ # unflattened = {}
325
+ # for composite_key, value in flat_dict.items():
326
+ # parts = composite_key.split(sep)
327
+ # if custom_logic:
328
+ # parts = [custom_logic(part) for part in parts]
329
+ # else:
330
+ # parts = [int(part) if (isinstance(part, int) or part.isdigit()) else part for part in parts]
331
+ # insert(unflattened, parts, value, max_depth)
332
+
333
+ # # Convert top-level dictionary to a list if all keys are integers
334
+ # if isinstance(unflattened, dict) and all(isinstance(k, int) for k in unflattened.keys()):
335
+ # max_index = max(unflattened.keys(), default=-1)
336
+ # return [unflattened.get(i) for i in range(max_index + 1)]
337
+ # # Return an empty dictionary instead of converting to a list if unflattened is empty
338
+ # if (unflattened == []) or (not unflattened):
339
+ # return {}
340
+ # return unflattened
341
+
342
+ def _insert_with_dict_handling(container, indices, value):
343
+ """
344
+ Helper function to insert a value into a nested container based on a list of indices.
345
+
346
+ :param container: The container (list or dict) to insert the value into.
347
+ :param indices: A list of indices indicating the path to the insertion point.
348
+ :param value: The value to be inserted.
349
+ """
350
+ for i, index in enumerate(indices[:-1]):
351
+ # Check if index is an integer and ensure the list is long enough
352
+ if isinstance(index, int):
353
+ # Extend the list if necessary
354
+ while len(container) <= index:
355
+ container.append(None)
356
+
357
+ # Create a nested container if the current position is None
358
+ if container[index] is None:
359
+ next_index = indices[i + 1]
360
+
361
+ # Determine the type of the next container based on the next index
362
+ container[index] = {} if isinstance(next_index, str) else []
363
+
364
+ # Update the reference to point to the nested container
365
+ container = container[index]
366
+
367
+ # If index is a string, work with dictionaries
368
+ else:
369
+ if index not in container:
370
+ container[index] = {}
371
+ container = container[index]
372
+
373
+ # Insert value at the last index
374
+ last_index = indices[-1]
375
+ if isinstance(last_index, int):
376
+
377
+ # Negative indices are not allowed in this context
378
+ if last_index < 0:
379
+ raise ValueError("Negative index is not allowed for list unflattening.")
380
+
381
+ while len(container) <= last_index:
382
+ container.append(None)
383
+ container[last_index] = value
384
+
385
+ else:
386
+ if last_index in container and isinstance(container[last_index], list):
387
+ raise ValueError("Overlapping keys are not allowed.")
388
+ container[last_index] = value
389
+
390
+ def unflatten_to_list(flat_dict: Dict[str, Any], sep: str = '_') -> List:
391
+ """
392
+ Unflattens a dictionary with keys as string paths into a nested list structure.
393
+
394
+ :param flat_dict: The flat dictionary to unflatten.
395
+ :param sep: The separator used in the flat dictionary keys to indicate nesting.
396
+ :return: A nested list that represents the unflattened structure.
397
+ """
398
+ result_list = []
399
+ for flat_key, value in flat_dict.items():
400
+ indices = [int(p) if p.lstrip('-').isdigit() else p for p in flat_key.split(sep)]
401
+ _insert_with_dict_handling(result_list, indices, value)
402
+ return result_list
403
+
404
+ def flatten_iterable(iterable: Iterable, max_depth: int = None) -> Generator[Any, None, None]:
405
+ """
406
+ Flattens a nested iterable up to a specified maximum depth.
407
+
408
+ Args:
409
+ iterable: An iterable to flatten.
410
+ max_depth: The maximum depth to flatten. If None, flattens completely.
411
+
412
+ Yields:
413
+ The flattened elements of the original iterable.
414
+ """
415
+ def _flatten(input_iterable: Iterable, current_depth: int) -> Generator[Any, None, None]:
416
+ if isinstance(input_iterable, Iterable) and not isinstance(input_iterable, (str, bytes)):
417
+ if max_depth is not None and current_depth >= max_depth:
418
+ yield input_iterable
419
+ else:
420
+ for item in input_iterable:
421
+ yield from _flatten(item, current_depth + 1)
422
+ else:
423
+ yield input_iterable
424
+
425
+ yield from _flatten(iterable, 0)
426
+
427
+ def flatten_iterable_to_list(iterable: Iterable, max_depth: int = None) -> List[Any]:
428
+ """
429
+ Converts a nested iterable into a flattened list up to a specified maximum depth.
430
+
431
+ Args:
432
+ iterable: An iterable to flatten.
433
+ max_depth: The maximum depth to flatten. If None, flattens completely.
434
+
435
+ Returns:
436
+ A list containing the flattened elements of the original iterable.
437
+ """
438
+ return list(flatten_iterable(iterable, max_depth))
439
+
440
+ def unflatten_dict_with_custom_logic(
441
+ flat_dict: Dict[str, Any],
442
+ logic_func: Callable[[str, Any], Tuple[str, Any]],
443
+ sep: str = '_'
444
+ ) -> Dict[str, Any]:
445
+ """
446
+ Unflattens a dictionary with keys as string paths into a nested dictionary structure
447
+ while applying custom logic to each key and value.
448
+
449
+ Args:
450
+ flat_dict: The flat dictionary to unflatten.
451
+ logic_func: A function that takes a key and a value and returns a tuple of modified key and value.
452
+ sep: The separator used in the flat dictionary keys to indicate nesting.
453
+
454
+ Returns:
455
+ A nested dictionary that represents the unflattened structure with modified keys and values.
456
+ """
457
+ reconstructed = {}
458
+ for flat_key, value in flat_dict.items():
459
+ parts = flat_key.split(sep)
460
+ d = reconstructed
461
+ for part in parts[:-1]:
462
+ modified_part, _ = logic_func(part, None) # Modify only the key
463
+ d = d.setdefault(modified_part, {})
464
+ last_part = parts[-1]
465
+ modified_last_part, modified_value = logic_func(last_part, value)
466
+ d[modified_last_part] = modified_value
467
+ return reconstructed
468
+
469
+ # def dynamic_unflatten(flat_dict, sep='_', custom_logic=None, max_depth=None):
470
+ # """
471
+ # Unflattens a dictionary with flat keys into a nested dictionary or list.
472
+
473
+ # :param flat_dict: A dictionary with flat keys that need to be nested.
474
+ # :param sep: The separator used in the flat keys (default is '_').
475
+ # :param custom_logic: A function that customizes the processing of keys.
476
+ # It takes a key part as input and returns the transformed key part.
477
+ # :param max_depth: The maximum depth to which the dictionary should be nested.
478
+ # If None, there is no maximum depth.
479
+ # :return: A nested dictionary or list based on the flat input dictionary.
480
+
481
+ # The function dynamically unflattens a dictionary with keys that represent nested paths.
482
+ # For example, the flat dictionary {'a_b_c': 1, 'a_b_d': 2} would be unflattened to
483
+ # {'a': {'b': {'c': 1, 'd': 2}}}. If the keys can be converted to integers and suggest list indices,
484
+ # the function produces lists instead of dictionaries. For example,
485
+ # {'0_a': 'foo', '1_b': 'bar'} becomes [{'a': 'foo'}, {'b': 'bar'}].
486
+ # """
487
+
488
+ # def handle_list_insert(sub_obj, part, value):
489
+ # # Ensure part index exists in the list, fill gaps with None
490
+ # while len(sub_obj) <= part:
491
+ # sub_obj.append(None)
492
+
493
+ # sub_obj[part] = value
494
+
495
+ # def insert(sub_obj, parts, value, max_depth, current_depth=0):
496
+ # for part in parts[:-1]:
497
+ # # Stop nesting further if max_depth is reached
498
+ # if max_depth is not None and current_depth >= max_depth:
499
+ # sub_obj[part] = {parts[-1]: value}
500
+ # return
501
+ # # Handle integer parts for list insertion
502
+ # if isinstance(part, int):
503
+ # # Ensure part index exists in the list or dict, fill gaps with None
504
+ # while len(sub_obj) <= part:
505
+ # sub_obj.append(None)
506
+ # if sub_obj[part] is None:
507
+ # if current_depth < max_depth - 1 if max_depth else True:
508
+ # sub_obj[part] = [] if isinstance(parts[parts.index(part) + 1], int) else {}
509
+ # sub_obj = sub_obj[part]
510
+ # else:
511
+ # # Handle string parts for dictionary insertion
512
+ # if part not in sub_obj:
513
+ # sub_obj[part] = {}
514
+ # sub_obj = sub_obj[part]
515
+ # current_depth += 1
516
+ # # Insert the value at the last part
517
+ # last_part = parts[-1]
518
+ # if isinstance(last_part, int) and isinstance(sub_obj, list):
519
+ # handle_list_insert(sub_obj, last_part, value, max_depth, current_depth)
520
+ # else:
521
+ # sub_obj[last_part] = value
522
+
523
+ # unflattened = {}
524
+ # for composite_key, value in flat_dict.items():
525
+ # parts = composite_key.split(sep)
526
+ # if custom_logic:
527
+ # parts = [custom_logic(part) for part in parts]
528
+ # else:
529
+ # parts = [int(part) if (isinstance(part, int) or part.isdigit()) else part for part in parts]
530
+ # insert(unflattened, parts, value, max_depth)
531
+
532
+ # # Convert top-level dictionary to a list if all keys are integers
533
+ # if isinstance(unflattened, dict) and all(isinstance(k, int) for k in unflattened.keys()):
534
+ # max_index = max(unflattened.keys(), default=-1)
535
+ # return [unflattened.get(i) for i in range(max_index + 1)]
536
+ # # Return an empty dictionary instead of converting to a list if unflattened is empty
537
+ # if (unflattened == []) or (not unflattened):
538
+ # return {}
539
+ # return unflattened
540
+
@@ -0,0 +1,102 @@
1
+ import csv
2
+ import json
3
+ import os
4
+ import tempfile
5
+ from typing import Any, Dict, List
6
+ from .type_util import to_list
7
+
8
+
9
+ def to_temp(input: Any,
10
+ flatten_dict: bool = False,
11
+ flat: bool = False,
12
+ dropna: bool = False):
13
+ """
14
+ Converts input to a list and writes it to a temporary file in JSON format, with flattening options.
15
+
16
+ This function serializes data to a temporary JSON file, useful for transient storage or testing.
17
+ It includes options to flatten the input if it contains dictionaries or lists.
18
+
19
+ Parameters:
20
+ input (Any): The data to be converted and written to a file.
21
+
22
+ flatten_dict (bool, optional): Flatten dictionaries in the input. Defaults to False.
23
+
24
+ flat (bool, optional): Flatten lists in the input. Defaults to False.
25
+
26
+ dropna (bool, optional): Exclude 'None' values during flattening. Defaults to False.
27
+
28
+ Raises:
29
+ TypeError: If the input is not JSON serializable.
30
+
31
+ Example:
32
+ >>> temp_file = to_temp({'a': 1, 'b': [2, 3]}, flatten_dict=True)
33
+ >>> temp_file.name # Doctest: +ELLIPSIS
34
+ '/var/folders/.../tmp...'
35
+ """
36
+ input = to_list(input, flatten_dict, flat, dropna)
37
+
38
+ temp_file = tempfile.NamedTemporaryFile(mode='w', delete=False)
39
+ try:
40
+ json.dump(input, temp_file)
41
+ except TypeError as e:
42
+ temp_file.close() # Ensuring file closure before raising error
43
+ raise TypeError(f"Data provided is not JSON serializable: {e}")
44
+ temp_file.close()
45
+ return temp_file
46
+
47
+ def to_csv(input: List[Dict[str, Any]]=None,
48
+ filepath: str=None,
49
+ file_exist_ok: bool = False) -> None:
50
+ """
51
+ Writes a list of dictionaries to a CSV file, with dictionary keys as headers.
52
+
53
+ This function writes a list of dictionaries to a CSV file. It checks if the file exists
54
+ and handles file creation based on the 'file_exist_ok' flag.
55
+
56
+ Parameters:
57
+ input (List[Dict[str, Any]]): Data to write to the CSV file.
58
+
59
+ filepath (str): Path of the output CSV file.
60
+
61
+ file_exist_ok (bool, optional): Create the file if it doesn't exist. Defaults to False.
62
+
63
+ Raises:
64
+ FileExistsError: If the file already exists and 'file_exist_ok' is False.
65
+
66
+ Example:
67
+ >>> data = [{'name': 'Alice', 'age': 30}, {'name': 'Bob', 'age': 25}]
68
+ >>> to_csv(data, 'people.csv')
69
+ """
70
+
71
+ if not os.path.exists(os.path.dirname(filepath)) and os.path.dirname(filepath) != '':
72
+ if file_exist_ok:
73
+ os.makedirs(os.path.dirname(filepath), exist_ok=True)
74
+ else:
75
+ raise FileNotFoundError(f"The directory {os.path.dirname(filepath)} does not exist.")
76
+
77
+ with open(filepath, 'w', newline='') as csv_file:
78
+ writer = csv.DictWriter(csv_file, fieldnames=input[0].keys())
79
+ writer.writeheader()
80
+ writer.writerows(input)
81
+
82
+
83
+ def append_to_jsonl(data: Any, filepath: str) -> None:
84
+ """
85
+ Appends data to a JSON lines (jsonl) file.
86
+
87
+ Serializes given data to a JSON-formatted string and appends it to a jsonl file.
88
+ Useful for logging or data collection where entries are added incrementally.
89
+
90
+ Parameters:
91
+ data (Any): Data to be serialized and appended.
92
+
93
+ filepath (str): Path to the jsonl file.
94
+
95
+ Example:
96
+ >>> append_to_jsonl({"key": "value"}, "data.jsonl")
97
+ # Appends {"key": "value"} to 'data.jsonl'
98
+ """
99
+ json_string = json.dumps(data)
100
+ with open(filepath, "a") as f:
101
+ f.write(json_string + "\n")
102
+