lionagi 0.12.2__py3-none-any.whl → 0.12.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. lionagi/config.py +123 -0
  2. lionagi/fields/file.py +1 -1
  3. lionagi/fields/reason.py +1 -1
  4. lionagi/libs/file/concat.py +1 -6
  5. lionagi/libs/file/concat_files.py +1 -5
  6. lionagi/libs/file/save.py +1 -1
  7. lionagi/libs/package/imports.py +8 -177
  8. lionagi/libs/parse.py +30 -0
  9. lionagi/libs/schema/load_pydantic_model_from_schema.py +259 -0
  10. lionagi/libs/token_transform/perplexity.py +2 -4
  11. lionagi/libs/token_transform/synthlang_/resources/frameworks/framework_options.json +46 -46
  12. lionagi/libs/token_transform/synthlang_/translate_to_synthlang.py +1 -1
  13. lionagi/operations/chat/chat.py +2 -2
  14. lionagi/operations/communicate/communicate.py +20 -5
  15. lionagi/operations/parse/parse.py +131 -43
  16. lionagi/protocols/generic/log.py +1 -2
  17. lionagi/protocols/generic/pile.py +18 -4
  18. lionagi/protocols/messages/assistant_response.py +20 -1
  19. lionagi/protocols/messages/templates/README.md +6 -10
  20. lionagi/service/connections/__init__.py +15 -0
  21. lionagi/service/connections/api_calling.py +230 -0
  22. lionagi/service/connections/endpoint.py +410 -0
  23. lionagi/service/connections/endpoint_config.py +137 -0
  24. lionagi/service/connections/header_factory.py +56 -0
  25. lionagi/service/connections/match_endpoint.py +49 -0
  26. lionagi/service/connections/providers/__init__.py +3 -0
  27. lionagi/service/connections/providers/anthropic_.py +87 -0
  28. lionagi/service/connections/providers/exa_.py +33 -0
  29. lionagi/service/connections/providers/oai_.py +166 -0
  30. lionagi/service/connections/providers/ollama_.py +122 -0
  31. lionagi/service/connections/providers/perplexity_.py +29 -0
  32. lionagi/service/imodel.py +36 -144
  33. lionagi/service/manager.py +1 -7
  34. lionagi/service/{endpoints/rate_limited_processor.py → rate_limited_processor.py} +4 -2
  35. lionagi/service/resilience.py +545 -0
  36. lionagi/service/third_party/README.md +71 -0
  37. lionagi/service/third_party/__init__.py +0 -0
  38. lionagi/service/third_party/anthropic_models.py +159 -0
  39. lionagi/service/third_party/exa_models.py +165 -0
  40. lionagi/service/third_party/openai_models.py +18241 -0
  41. lionagi/service/third_party/pplx_models.py +156 -0
  42. lionagi/service/types.py +5 -4
  43. lionagi/session/branch.py +12 -7
  44. lionagi/tools/file/reader.py +1 -1
  45. lionagi/tools/memory/tools.py +497 -0
  46. lionagi/utils.py +921 -123
  47. lionagi/version.py +1 -1
  48. {lionagi-0.12.2.dist-info → lionagi-0.12.4.dist-info}/METADATA +33 -16
  49. {lionagi-0.12.2.dist-info → lionagi-0.12.4.dist-info}/RECORD +53 -63
  50. lionagi/libs/file/create_path.py +0 -80
  51. lionagi/libs/file/file_util.py +0 -358
  52. lionagi/libs/parse/__init__.py +0 -3
  53. lionagi/libs/parse/fuzzy_parse_json.py +0 -117
  54. lionagi/libs/parse/to_dict.py +0 -336
  55. lionagi/libs/parse/to_json.py +0 -61
  56. lionagi/libs/parse/to_num.py +0 -378
  57. lionagi/libs/parse/to_xml.py +0 -57
  58. lionagi/libs/parse/xml_parser.py +0 -148
  59. lionagi/libs/schema/breakdown_pydantic_annotation.py +0 -48
  60. lionagi/service/endpoints/__init__.py +0 -3
  61. lionagi/service/endpoints/base.py +0 -706
  62. lionagi/service/endpoints/chat_completion.py +0 -116
  63. lionagi/service/endpoints/match_endpoint.py +0 -72
  64. lionagi/service/providers/__init__.py +0 -3
  65. lionagi/service/providers/anthropic_/__init__.py +0 -3
  66. lionagi/service/providers/anthropic_/messages.py +0 -99
  67. lionagi/service/providers/exa_/models.py +0 -3
  68. lionagi/service/providers/exa_/search.py +0 -80
  69. lionagi/service/providers/exa_/types.py +0 -7
  70. lionagi/service/providers/groq_/__init__.py +0 -3
  71. lionagi/service/providers/groq_/chat_completions.py +0 -56
  72. lionagi/service/providers/ollama_/__init__.py +0 -3
  73. lionagi/service/providers/ollama_/chat_completions.py +0 -134
  74. lionagi/service/providers/openai_/__init__.py +0 -3
  75. lionagi/service/providers/openai_/chat_completions.py +0 -101
  76. lionagi/service/providers/openai_/spec.py +0 -14
  77. lionagi/service/providers/openrouter_/__init__.py +0 -3
  78. lionagi/service/providers/openrouter_/chat_completions.py +0 -62
  79. lionagi/service/providers/perplexity_/__init__.py +0 -3
  80. lionagi/service/providers/perplexity_/chat_completions.py +0 -44
  81. lionagi/service/providers/perplexity_/models.py +0 -5
  82. lionagi/service/providers/types.py +0 -17
  83. /lionagi/{service/providers/exa_/__init__.py → py.typed} +0 -0
  84. /lionagi/service/{endpoints/token_calculator.py → token_calculator.py} +0 -0
  85. {lionagi-0.12.2.dist-info → lionagi-0.12.4.dist-info}/WHEEL +0 -0
  86. {lionagi-0.12.2.dist-info → lionagi-0.12.4.dist-info}/licenses/LICENSE +0 -0
@@ -1,358 +0,0 @@
1
- # Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
2
- #
3
- # SPDX-License-Identifier: Apache-2.0
4
-
5
- from collections.abc import Callable
6
- from pathlib import Path
7
- from typing import Any, Literal
8
-
9
-
10
- class FileUtil:
11
-
12
- @staticmethod
13
- def chunk_by_chars(
14
- text: str,
15
- chunk_size: int = 2048,
16
- overlap: float = 0,
17
- threshold: int = 256,
18
- ) -> list[str]:
19
- from .chunk import chunk_by_chars
20
-
21
- return chunk_by_chars(
22
- text,
23
- chunk_size=chunk_size,
24
- overlap=overlap,
25
- threshold=threshold,
26
- )
27
-
28
- @staticmethod
29
- def chunk_by_tokens(
30
- text: str,
31
- tokenizer: Callable[[str], list[str]] = str.split,
32
- chunk_size: int = 1024,
33
- overlap: float = 0,
34
- threshold: int = 256,
35
- ) -> list[str]:
36
- from .chunk import chunk_by_tokens
37
-
38
- return chunk_by_tokens(
39
- text,
40
- tokenizer=tokenizer,
41
- chunk_size=chunk_size,
42
- overlap=overlap,
43
- threshold=threshold,
44
- )
45
-
46
- @staticmethod
47
- def chunk_content(
48
- content: str,
49
- chunk_by: Literal["chars", "tokens"] = "chars",
50
- tokenizer: Callable[[str], list[str]] = str.split,
51
- chunk_size: int = 1024,
52
- overlap: float = 0,
53
- threshold: int = 256,
54
- ) -> list[str]:
55
- from .chunk import chunk_content
56
-
57
- return chunk_content(
58
- content,
59
- chunk_by=chunk_by,
60
- tokenizer=tokenizer,
61
- chunk_size=chunk_size,
62
- overlap=overlap,
63
- threshold=threshold,
64
- )
65
-
66
- @staticmethod
67
- def concat_files(
68
- data_path: str | Path | list,
69
- file_types: list[str],
70
- output_dir: str | Path = None,
71
- output_filename: str = None,
72
- file_exist_ok: bool = True,
73
- recursive: bool = True,
74
- verbose: bool = True,
75
- threshold: int = 0,
76
- return_fps: bool = False,
77
- return_files: bool = False,
78
- **kwargs,
79
- ) -> (
80
- list[str] | str | tuple[list[str], list[Path]] | tuple[str, list[Path]]
81
- ):
82
- from .concat_files import concat_files
83
-
84
- return concat_files(
85
- data_path,
86
- file_types=file_types,
87
- output_dir=output_dir,
88
- output_filename=output_filename,
89
- file_exist_ok=file_exist_ok,
90
- recursive=recursive,
91
- verbose=verbose,
92
- threshold=threshold,
93
- return_fps=return_fps,
94
- return_files=return_files,
95
- **kwargs,
96
- )
97
-
98
- @staticmethod
99
- def concat(
100
- data_path: str | Path | list,
101
- file_types: list[str],
102
- output_dir: str | Path = None,
103
- output_filename: str = None,
104
- file_exist_ok: bool = True,
105
- recursive: bool = True,
106
- verbose: bool = True,
107
- threshold: int = 0,
108
- return_fps: bool = False,
109
- return_files: bool = False,
110
- exclude_patterns: list[str] = None,
111
- **kwargs,
112
- ) -> dict[str, Any]:
113
- from .concat import concat
114
-
115
- return concat(
116
- data_path,
117
- file_types=file_types,
118
- output_dir=output_dir,
119
- output_filename=output_filename,
120
- file_exist_ok=file_exist_ok,
121
- recursive=recursive,
122
- verbose=verbose,
123
- threshold=threshold,
124
- return_fps=return_fps,
125
- return_files=return_files,
126
- exclude_patterns=exclude_patterns,
127
- **kwargs,
128
- )
129
-
130
- @staticmethod
131
- def copy_file(src: Path | str, dest: Path | str) -> None:
132
- from .file_ops import copy_file
133
-
134
- copy_file(src, dest)
135
-
136
- @staticmethod
137
- def get_file_size(path: Path | str) -> int:
138
- from .file_ops import get_file_size
139
-
140
- return get_file_size(path)
141
-
142
- @staticmethod
143
- def list_files(
144
- dir_path: Path | str, extension: str | None = None
145
- ) -> list[Path]:
146
- from .file_ops import list_files
147
-
148
- return list_files(dir_path, extension=extension)
149
-
150
- @staticmethod
151
- def read_file(file_path: Path | str, encoding: str = "utf-8") -> str:
152
- from .file_ops import read_file
153
-
154
- return read_file(file_path, encoding=encoding)
155
-
156
- @staticmethod
157
- def read_image_to_base64(image_path: str | Path) -> str:
158
- import base64
159
-
160
- import cv2 # type: ignore[import]
161
-
162
- image_path = str(image_path)
163
- image = cv2.imread(image_path, cv2.COLOR_BGR2RGB)
164
-
165
- if image is None:
166
- raise ValueError(f"Could not read image from path: {image_path}")
167
-
168
- file_extension = "." + image_path.split(".")[-1]
169
-
170
- success, buffer = cv2.imencode(file_extension, image)
171
- if not success:
172
- raise ValueError(
173
- f"Could not encode image to {file_extension} format."
174
- )
175
- encoded_image = base64.b64encode(buffer).decode("utf-8")
176
- return encoded_image
177
-
178
- @staticmethod
179
- def pdf_to_images(
180
- pdf_path: str, output_folder: str, dpi: int = 300, fmt: str = "jpeg"
181
- ) -> list:
182
- """
183
- Convert a PDF file into images, one image per page.
184
-
185
- Args:
186
- pdf_path (str): Path to the input PDF file.
187
- output_folder (str): Directory to save the output images.
188
- dpi (int): Dots per inch (resolution) for conversion (default: 300).
189
- fmt (str): Image format (default: 'jpeg'). Use 'png' if preferred.
190
-
191
- Returns:
192
- list: A list of file paths for the saved images.
193
- """
194
- import os
195
-
196
- from lionagi.utils import check_import
197
-
198
- convert_from_path = check_import(
199
- "pdf2image", import_name="convert_from_path"
200
- )
201
-
202
- # Ensure the output folder exists
203
- os.makedirs(output_folder, exist_ok=True)
204
-
205
- # Convert PDF to a list of PIL Image objects
206
- images = convert_from_path(pdf_path, dpi=dpi)
207
-
208
- saved_paths = []
209
- for i, image in enumerate(images):
210
- # Construct the output file name
211
- image_file = os.path.join(output_folder, f"page_{i + 1}.{fmt}")
212
- image.save(image_file, fmt.upper())
213
- saved_paths.append(image_file)
214
-
215
- return saved_paths
216
-
217
- @staticmethod
218
- def dir_to_files(
219
- directory: str | Path,
220
- file_types: list[str] | None = None,
221
- max_workers: int | None = None,
222
- ignore_errors: bool = False,
223
- verbose: bool = False,
224
- recursive: bool = False,
225
- ) -> list[Path]:
226
- from .process import dir_to_files
227
-
228
- return dir_to_files(
229
- directory,
230
- file_types=file_types,
231
- max_workers=max_workers,
232
- ignore_errors=ignore_errors,
233
- verbose=verbose,
234
- recursive=recursive,
235
- )
236
-
237
- @staticmethod
238
- def file_to_chunks(
239
- file_path: str | Path,
240
- chunk_by: Literal["chars", "tokens"] = "chars",
241
- chunk_size: int = 1500,
242
- overlap: float = 0.1,
243
- threshold: int = 200,
244
- encoding: str = "utf-8",
245
- custom_metadata: dict[str, Any] | None = None,
246
- output_dir: str | Path | None = None,
247
- verbose: bool = False,
248
- timestamp: bool = True,
249
- random_hash_digits: int = 4,
250
- as_node: bool = False,
251
- ) -> list[dict[str, Any]]:
252
- from .file_ops import file_to_chunks
253
-
254
- return file_to_chunks(
255
- file_path,
256
- chunk_by=chunk_by,
257
- chunk_size=chunk_size,
258
- overlap=overlap,
259
- threshold=threshold,
260
- encoding=encoding,
261
- custom_metadata=custom_metadata,
262
- output_dir=output_dir,
263
- verbose=verbose,
264
- timestamp=timestamp,
265
- random_hash_digits=random_hash_digits,
266
- as_node=as_node,
267
- )
268
-
269
- @staticmethod
270
- def chunk(
271
- *,
272
- text: str | None = None,
273
- url_or_path: str | Path = None,
274
- file_types: list[str] | None = None, # only local files
275
- recursive: bool = False, # only local files
276
- tokenizer: Callable[[str], list[str]] = None,
277
- chunk_by: Literal["chars", "tokens"] = "chars",
278
- chunk_size: int = 1500,
279
- overlap: float = 0.1,
280
- threshold: int = 200,
281
- output_file: str | Path | None = None,
282
- metadata: dict[str, Any] | None = None,
283
- reader_tool: Callable = None,
284
- as_node: bool = False,
285
- ) -> list:
286
- from .process import chunk
287
-
288
- return chunk(
289
- text=text,
290
- url_or_path=url_or_path,
291
- file_types=file_types,
292
- recursive=recursive,
293
- tokenizer=tokenizer,
294
- chunk_by=chunk_by,
295
- chunk_size=chunk_size,
296
- overlap=overlap,
297
- threshold=threshold,
298
- output_file=output_file,
299
- metadata=metadata,
300
- reader_tool=reader_tool,
301
- as_node=as_node,
302
- )
303
-
304
- @staticmethod
305
- def save_to_file(
306
- text: str,
307
- directory: str | Path,
308
- filename: str,
309
- extension: str = "txt",
310
- timestamp: bool = True,
311
- dir_exist_ok: bool = True,
312
- file_exist_ok: bool = True,
313
- time_prefix: bool = False,
314
- timestamp_format: str = "%Y%m%d_%H%M%S",
315
- random_hash_digits: int = 4,
316
- verbose: bool = False,
317
- ) -> Path:
318
- from .save import save_to_file
319
-
320
- return save_to_file(
321
- text,
322
- directory=directory,
323
- filename=filename,
324
- extension=extension,
325
- timestamp=timestamp,
326
- dir_exist_ok=dir_exist_ok,
327
- file_exist_ok=file_exist_ok,
328
- time_prefix=time_prefix,
329
- timestamp_format=timestamp_format,
330
- random_hash_digits=random_hash_digits,
331
- verbose=verbose,
332
- )
333
-
334
- @staticmethod
335
- def create_path(
336
- directory: Path | str,
337
- filename: str,
338
- extension: str = None,
339
- timestamp: bool = False,
340
- dir_exist_ok: bool = True,
341
- file_exist_ok: bool = False,
342
- time_prefix: bool = False,
343
- timestamp_format: str | None = None,
344
- random_hash_digits: int = 0,
345
- ) -> Path:
346
- from .create_path import create_path
347
-
348
- return create_path(
349
- directory=directory,
350
- filename=filename,
351
- extension=extension,
352
- timestamp=timestamp,
353
- dir_exist_ok=dir_exist_ok,
354
- file_exist_ok=file_exist_ok,
355
- time_prefix=time_prefix,
356
- timestamp_format=timestamp_format,
357
- random_hash_digits=random_hash_digits,
358
- )
@@ -1,3 +0,0 @@
1
- # Copyright (c) 2023 - 2025, HaiyangLi <quantocean.li at gmail dot com>
2
- #
3
- # SPDX-License-Identifier: Apache-2.0
@@ -1,117 +0,0 @@
1
- import contextlib
2
- import json
3
- import re
4
- from typing import Any
5
-
6
-
7
- def fuzzy_parse_json(
8
- str_to_parse: str, /
9
- ) -> dict[str, Any] | list[dict[str, Any]]:
10
- """
11
- Attempt to parse a JSON string, trying a few minimal "fuzzy" fixes if needed.
12
-
13
- Steps:
14
- 1. Parse directly with json.loads.
15
- 2. Replace single quotes with double quotes, normalize spacing, and try again.
16
- 3. Attempt to fix unmatched brackets using fix_json_string.
17
- 4. If all fail, raise ValueError.
18
-
19
- Args:
20
- str_to_parse: The JSON string to parse
21
-
22
- Returns:
23
- Parsed JSON (dict or list of dicts)
24
-
25
- Raises:
26
- ValueError: If the string cannot be parsed as valid JSON
27
- TypeError: If the input is not a string
28
- """
29
- _check_valid_str(str_to_parse)
30
-
31
- # 1. Direct attempt
32
- with contextlib.suppress(Exception):
33
- return json.loads(str_to_parse)
34
-
35
- # 2. Try cleaning: replace single quotes with double and normalize
36
- cleaned = _clean_json_string(str_to_parse.replace("'", '"'))
37
- with contextlib.suppress(Exception):
38
- return json.loads(cleaned)
39
-
40
- # 3. Try fixing brackets
41
- fixed = fix_json_string(cleaned)
42
- with contextlib.suppress(Exception):
43
- return json.loads(fixed)
44
-
45
- # If all attempts fail
46
- raise ValueError("Invalid JSON string")
47
-
48
-
49
- def _check_valid_str(str_to_parse: str, /):
50
- if not isinstance(str_to_parse, str):
51
- raise TypeError("Input must be a string")
52
- if not str_to_parse.strip():
53
- raise ValueError("Input string is empty")
54
-
55
-
56
- def _clean_json_string(s: str) -> str:
57
- """Basic normalization: replace unescaped single quotes, trim spaces, ensure keys are quoted."""
58
- # Replace unescaped single quotes with double quotes
59
- # '(?<!\\)'" means a single quote not preceded by a backslash
60
- s = re.sub(r"(?<!\\)'", '"', s)
61
- # Collapse multiple whitespaces
62
- s = re.sub(r"\s+", " ", s)
63
- # Ensure keys are quoted
64
- # This attempts to find patterns like { key: value } and turn them into {"key": value}
65
- s = re.sub(r'([{,])\s*([^"\s]+)\s*:', r'\1"\2":', s)
66
- return s.strip()
67
-
68
-
69
- def fix_json_string(str_to_parse: str, /) -> str:
70
- """Try to fix JSON string by ensuring brackets are matched properly."""
71
- if not str_to_parse:
72
- raise ValueError("Input string is empty")
73
-
74
- brackets = {"{": "}", "[": "]"}
75
- open_brackets = []
76
- pos = 0
77
- length = len(str_to_parse)
78
-
79
- while pos < length:
80
- char = str_to_parse[pos]
81
-
82
- if char == "\\":
83
- pos += 2 # Skip escaped chars
84
- continue
85
-
86
- if char == '"':
87
- pos += 1
88
- # skip string content
89
- while pos < length:
90
- if str_to_parse[pos] == "\\":
91
- pos += 2
92
- continue
93
- if str_to_parse[pos] == '"':
94
- pos += 1
95
- break
96
- pos += 1
97
- continue
98
-
99
- if char in brackets:
100
- open_brackets.append(brackets[char])
101
- elif char in brackets.values():
102
- if not open_brackets:
103
- # Extra closing bracket
104
- # Better to raise error than guess
105
- raise ValueError("Extra closing bracket found.")
106
- if open_brackets[-1] != char:
107
- # Mismatched bracket
108
- raise ValueError("Mismatched brackets.")
109
- open_brackets.pop()
110
-
111
- pos += 1
112
-
113
- # Add missing closing brackets if any
114
- if open_brackets:
115
- str_to_parse += "".join(reversed(open_brackets))
116
-
117
- return str_to_parse