lionagi 0.9.14__py3-none-any.whl → 0.9.15__py3-none-any.whl

lionagi/libs/file/concat_files.py

@@ -0,0 +1,83 @@
+from pathlib import Path
+
+from lionagi.utils import create_path
+
+from .process import dir_to_files
+
+
+def concat_files(
+    data_path: str | Path | list,
+    file_types: list[str],
+    output_dir: str | Path = None,
+    output_filename: str = None,
+    file_exist_ok: bool = True,
+    recursive: bool = True,
+    verbose: bool = True,
+    threshold: int = 0,
+    return_fps: bool = False,
+    return_files: bool = False,
+    **kwargs,
+) -> list[str] | str | tuple[list[str], list[Path]] | tuple[str, list[Path]]:
+    """
+    data_path: str or Path or list of str or Path, the directory or file paths to concatenate.
+    file_types: list of str, the file types to concatenate. [e.g. ['.txt', '.md']]
+    output_dir: str or Path, the directory to save the concatenated file. If provided, will save the file.
+    output_filename: str, the filename to save the concatenated file.
+    file_exist_ok: bool, if True, overwrite the existing file. Default is True.
+    recursive: bool, if True, search files recursively. Default is True.
+    verbose: bool, if True, print the output path. Default is True.
+    threshold: int, the minimum number of chars for the file to be considered valid to concatenate.
+    kwargs: additional keyword arguments to pass to create_path.
+    """
+    persist_path = None
+    if output_dir:
+        if not output_filename:
+            output_filename = "concatenated_text.txt"
+            kwargs["timestamp"] = kwargs.get("timestamp", True)
+            kwargs["random_hash_digits"] = kwargs.get("random_hash_digits", 6)
+        output_filename = output_filename or "concatenated_text.txt"
+        persist_path = create_path(
+            output_dir, output_filename, file_exist_ok=file_exist_ok, **kwargs
+        )
+
+    texts = []
+    data_path = (
+        [str(data_path)] if not isinstance(data_path, list) else data_path
+    )
+    data_path = sorted(data_path)
+    data_path = [Path(dp) for dp in data_path if Path(dp).exists()]
+
+    for dp in data_path:
+        fps = dir_to_files(dp, recursive=recursive, file_types=file_types)
+
+        data_path = sorted([str(i) for i in fps])
+        data_path: list[Path] = [
+            Path(dp) for dp in data_path if Path(dp).exists()
+        ]
+
+        for fp in data_path:
+            text = fp.read_text(encoding="utf-8")
+            if len(text) >= threshold:
+                fp_text = (
+                    "\n----------------------------------------------------\n"
+                    f"{str(fp)}"
+                    "\n----------------------------------------------------\n"
+                )
+                text = fp_text + text
+                texts.append(text)
+
+    text = "\n".join(texts)
+    if persist_path:
+        persist_path.write_text(text, encoding="utf-8")
+    if verbose:
+        print(f"Concatenated {len(fps)} files to {persist_path}")
+        print(f"The file contains {len(text)} characters.")
+
+    if return_files:
+        if return_fps:
+            return texts, fps
+        return texts
+
+    if return_fps:
+        return text, fps
+    return text

lionagi/libs/file/process.py

@@ -164,10 +164,12 @@ def file_to_chunks(
 
 
 def chunk(
-    url_or_path: str | Path,
     *,
+    text: str | None = None,
+    url_or_path: str | Path = None,
     file_types: list[str] | None = None,  # only local files
     recursive: bool = False,  # only local files
+    tokenizer: Callable[[str], list[str]] = None,
     chunk_by: Literal["chars", "tokens"] = "chars",
     chunk_size: int = 1500,
     overlap: float = 0.1,
@@ -175,45 +177,52 @@ def chunk(
     output_file: str | Path | None = None,
     metadata: dict[str, Any] | None = None,
     reader_tool: Callable = None,
-):
-    if isinstance(url_or_path, str):
-        url_or_path = Path(url_or_path)
-
-    chunks = None
-    files = None
-    if url_or_path.exists():
-        if url_or_path.is_dir():
-            files = dir_to_files(
-                directory=url_or_path,
-                file_types=file_types,
-                recursive=recursive,
+    as_node: bool = False,
+) -> list:
+    texts = []
+    if not text:
+        if isinstance(url_or_path, str):
+            url_or_path = Path(url_or_path)
+
+        chunks = None
+        files = None
+        if url_or_path.exists():
+            if url_or_path.is_dir():
+                files = dir_to_files(
+                    directory=url_or_path,
+                    file_types=file_types,
+                    recursive=recursive,
+                )
+            elif url_or_path.is_file():
+                files = [url_or_path]
+        else:
+            files = (
+                [str(url_or_path)]
+                if not isinstance(url_or_path, list)
+                else url_or_path
             )
-        elif url_or_path.is_file():
-            files = [url_or_path]
-    else:
-        files = (
-            [str(url_or_path)]
-            if not isinstance(url_or_path, list)
-            else url_or_path
-        )
 
-    if reader_tool is None:
-        reader_tool = lambda x: x.read_text(encoding="utf-8")
+        if reader_tool is None:
+            reader_tool = lambda x: x.read_text(encoding="utf-8")
 
-    if reader_tool == "docling":
-        from lionagi.libs.package.imports import check_import
+        if reader_tool == "docling":
+            from lionagi.libs.package.imports import check_import
 
-        DocumentConverter = check_import(
-            "docling",
-            module_name="document_converter",
-            import_name="DocumentConverter",
-        )
-        converter = DocumentConverter()
-        reader_tool = lambda x: converter.convert(
-            x
-        ).document.export_to_markdown()
+            DocumentConverter = check_import(
+                "docling",
+                module_name="document_converter",
+                import_name="DocumentConverter",
+            )
+            converter = DocumentConverter()
+            reader_tool = lambda x: converter.convert(
+                x
+            ).document.export_to_markdown()
+
+        texts = lcall(files, reader_tool)
+
+    else:
+        texts = [text]
 
-    texts = lcall(files, reader_tool)
     chunks = lcall(
         texts,
         chunk_content,
@@ -224,6 +233,7 @@ def chunk(
         metadata=metadata,
         as_node=True,
         flatten=True,
+        tokenizer=tokenizer or str.split,
     )
     if threshold:
         chunks = [c for c in chunks if len(c.content) > threshold]
@@ -247,4 +257,7 @@ def chunk(
         else:
             raise ValueError(f"Unsupported output file format: {output_file}")
 
-    return chunks
+    if as_node:
+        return chunks
+
+    return [c.content for c in chunks]

lionagi/libs/token_transform/symbolic_compress_context.py

@@ -4,7 +4,7 @@ from typing import Literal
 
 from lionagi.service.imodel import iModel
 from lionagi.session.branch import Branch
-from lionagi.utils import alcall
+from lionagi.utils import alcall, get_bins
 
 from .base import TokenMapping, TokenMappingTemplate
 from .synthlang_.base import SynthlangFramework, SynthlangTemplate
@@ -19,10 +19,10 @@ async def symbolic_compress_context(
     url_or_path: str | Path = None,
     chunk_by="tokens",
     chunk_size: int = 1000,
-    chunk_tokenizer: Callable = str.split,
+    chunk_tokenizer: Callable = None,
     threshold=50,
     output_path: Path | str = None,
-    overlap=0.05,
+    overlap=0.025,
     system: str = None,
     chat_model: iModel = None,
     use_lion_system_message: bool = True,
@@ -42,8 +42,8 @@ async def symbolic_compress_context(
     compress_min_pplx=None,
     encode_token_map: TokenMappingTemplate | dict | TokenMapping = None,
     num_encodings: int = 3,
-    encode_output: bool = False,
-    num_output_encodings: int = None,
+    encode_output: bool = True,
+    num_output_encodings: int = 1,
     verbose: bool = True,
     branch: Branch = None,
     additional_text: str = "",
@@ -96,7 +96,7 @@ async def symbolic_compress_context(
 
     from lionagi.libs.file.process import chunk, chunk_content
 
-    texts = []
+    chunks = []
     if url_or_path:
         chunks = chunk(
             url_or_path=url_or_path,
@@ -105,26 +105,35 @@ async def symbolic_compress_context(
             overlap=overlap,
             threshold=threshold,
         )
-        texts = [i.content for i in chunks if i.content]
 
     elif text:
-        texts = chunk_content(
+        chunks = chunk_content(
             text=text,
             chunk_by=chunk_by,
             chunk_size=chunk_size,
             overlap=overlap,
             threshold=threshold,
-            tokenizer=chunk_tokenizer,
+            tokenizer=chunk_tokenizer or str.split,
         )
 
+    texts = [str(i).strip() for i in chunks if str(i).strip()]
+    bins = get_bins(texts, upper=chunk_size)
+    textss = []
+    for i in bins:
+        textss.append("\n".join([texts[j] for j in i]))
+
     results = await alcall(
-        texts,
+        textss,
         _inner,
         max_concurrent=max_concurrent,
         retry_default=None,
+        num_retries=2,
         throttle_period=throttle_period,
+        retry_delay=1,
+        backoff_factor=2,
         flatten=True,
         dropna=True,
+        unique_output=True,
     )
     text = "\n".join(results)
 

lionagi/libs/token_transform/synthlang_/resources/mapping/rust_chinese_mapping.toml

@@ -4,18 +4,18 @@ created_at = 1740603518.077121
 [metadata]
 title = "Rust-Chinese Symbolic Encoding Mapping"
 domain = "Symbolic Compression"
-version = "1.0"
-overview = "This resource provides a mapping of Rust-Chinese symbolic encoding for compressing rust codebase."
+version = "1.1"
+overview = "This resource provides a mapping of Rust-Chinese symbolic encoding for compressing rust codebase. When using, should keep the semantic structure of original rust codes, do not translate into chinese."
 category = "Token Transform"
 lion_class = "lionagi.libs.token_transform.base.TokenMapping"
 
 [content]
---- = "嗯"
-"```" = "引"
+--- = "横"
+"std::" = "标"
 "###" = "井"
 "..." = "点"
 "~~~" = "波"
-"|||" = "線"
+"|||" = "纵"
 "   " = "空"
 "===" = "等"
 "{{{" = "開"
@@ -34,4 +34,27 @@ lion_class = "lionagi.libs.token_transform.base.TokenMapping"
 "=>" = "⇒"
 "<=>" = "⇔"
 "::" = "的"
-"synthlang" = ""
+"assert_eq!" = "否"
+"assert_eq" = "是"
+"assert!" = "否"
+assert = "是"
+";//" = ";"
+"横横" = "横"
+"斜斜" = "斜"
+"//!" = " "
+"##// " = " "
+"#//" = " "
+"//点" = " "
+"空空" = " "
+Self = "自"
+self = "自"
+"enum " = "列"
+"00000000" = "亿"
+"0000" = "万"
+"万万" = "亿"
+";\n// " = "; "
+"pub fn " = "公"
+Clone = "隆"
+clone = "隆"
+derive = "衍"
+"println!" = "印"

lionagi/libs/token_transform/synthlang_/translate_to_synthlang.py

@@ -45,14 +45,18 @@ async def translate_to_synthlang(
             encode_token_map = TokenMapping.load_from_template(
                 encode_token_map
             )
+            additional_text += (
+                f"\nTransforming text with {encode_token_map.metadata['title']}"
+                f"\nOverview: {encode_token_map.metadata['overview']}"
+            )
             encode_token_map = encode_token_map.content
         if not isinstance(encode_token_map, dict):
             raise ValueError(
                 "encode_token_map must be a dict or TokenMappingTemplate"
             )
         for _ in range(num_encodings):
-            text = "\n".join(
-                [str(i).strip() for i in text.split("\n") if i.strip()]
+            text = " ".join(
+                [str(i).strip() for i in text.split() if i.strip()]
             )
             for k, v in encode_token_map.items():
                 text = text.replace(k, v)
@@ -60,7 +64,6 @@ async def translate_to_synthlang(
         additional_text += (
             f"\nthesaurus, lexicon, glossary:\n{encode_token_map}"
         )
-
     if not isinstance(framework_template, SynthlangFramework):
         framework_template = SynthlangFramework.load_from_template(
             framework_template
@@ -98,9 +101,10 @@ async def translate_to_synthlang(
 
     kwargs["guidance"] = (
         "Following SynthLang, translate the provided text into SynthLang syntax. "
-        "Reasonably shrink the token size by 50-80%. Return only the translated text"
-        "enclosed by ```synthlang...```. "
-        "\n\n" + kwargs.get("guidance", "")
+        "Reasonably reduce the token count by up to 80%. Return only the transformed"
+        " string enclosed by ```synthlang...```. \n\n"
+        "DO NOT include anything else, no comments, no explanations, no additional "
+        "text, just the transformed string." + kwargs.get("guidance", "")
     )
 
     out = await branch.chat(
@@ -114,8 +118,8 @@ async def translate_to_synthlang(
     if encode_output:
         if isinstance(num_output_encodings, int) and num_output_encodings > 0:
             for _ in range(num_output_encodings):
-                out = "\n".join(
-                    [str(i).strip() for i in out.split("\n") if i.strip()]
+                out = " ".join(
+                    [str(i).strip() for i in out.split(" ") if i.strip()]
                 )
                 for k, v in encode_token_map.items():
                     out = out.replace(k, v).strip()
@@ -123,7 +127,7 @@ async def translate_to_synthlang(
     if sys1:
         branch.msgs.add_message(system=sys1)
 
-    len_ = calculator.tokenize(out, return_tokens=False)
+    len_ = calculator.tokenize(out)
     if verbose:
         msg = "------------------------------------------\n"
         msg += f"Compression Method: SynthLang\n"

lionagi/version.py

@@ -1 +1 @@
-__version__ = "0.9.14"
+__version__ = "0.9.15"

{lionagi-0.9.14.dist-info → lionagi-0.9.15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lionagi
-Version: 0.9.14
+Version: 0.9.15
 Summary: An Intelligence Operating System.
 Author-email: HaiyangLi <quantocean.li@gmail.com>
 License:            Apache License

{lionagi-0.9.14.dist-info → lionagi-0.9.15.dist-info}/RECORD

@@ -4,14 +4,15 @@ lionagi/_errors.py,sha256=JlBTFJnRWtVYcRxKb7fWFiJHLbykl1E19mSJ8sXYVxg,455
 lionagi/_types.py,sha256=9g7iytvSj3UjZxD-jL06_fxuNfgZyWT3Qnp0XYp1wQU,63
 lionagi/settings.py,sha256=W52mM34E6jXF3GyqCFzVREKZrmnUqtZm_BVDsUiDI_s,1627
 lionagi/utils.py,sha256=K36D9AAGiMPR4eM9tYoiVgvH-NdPPSeMQPls09s7keQ,73223
-lionagi/version.py,sha256=XGLQjyj9cqDYEcMT9GOWB4Gw5INLWURmRG2KlvV7wDk,23
+lionagi/version.py,sha256=PPxUSD_g1KMh3wdAtX6CMb4gLw5y0_nXjKdNFQVNkvc,23
 lionagi/libs/__init__.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
 lionagi/libs/parse.py,sha256=JRS3bql0InHJqATnAatl-hQv4N--XXw4P77JHhTFnrc,1011
 lionagi/libs/file/__init__.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
 lionagi/libs/file/chunk.py,sha256=XeVMwM33JF0X1W6udz_nhlb3DCevA_EK6A50Hn_e5SY,9300
+lionagi/libs/file/concat_files.py,sha256=Kb4YhG-W9IpCYDSJUmvlZgpXXAvG71ocKL_ztHjNtUk,3044
 lionagi/libs/file/file_ops.py,sha256=HBiIh1EljIJ5VTIXuyvJM0ppSs0YYOPUWmgDMJT634U,3430
 lionagi/libs/file/params.py,sha256=SZ5DkoffWfxWudOAYCfCxpL8UIm-1UjeyTtploo-Lqs,5824
-lionagi/libs/file/process.py,sha256=bIe4AdQ7eT0NHSMn0_Ail_-ltlM21YWqUWiPGXEPuHU,8264
+lionagi/libs/file/process.py,sha256=NKC-rIkm83qv4rgz-otd89aziop0G_LPz7_kmBAowsQ,8686
 lionagi/libs/file/save.py,sha256=TCxVlKxFFnr3xZ-HAXPpTomQoyiVrp6nKRoj-bcQt4k,2863
 lionagi/libs/nested/__init__.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
 lionagi/libs/nested/flatten.py,sha256=sB4jxZRoaUbjak9RbIWVWNKz2hzkhQJPFffV_Ws1GA0,5479
@@ -38,11 +39,11 @@ lionagi/libs/token_transform/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NM
 lionagi/libs/token_transform/base.py,sha256=1rLtfKK9pG8jEykcPG7a3o58lMTXOazgWn9Ff6BX3Wo,1483
 lionagi/libs/token_transform/llmlingua.py,sha256=DkeLUlrb7rGx3nZ04aADU9HXXu5mZTf_DBwT0xhzIv4,7
 lionagi/libs/token_transform/perplexity.py,sha256=tcVRjPBX3nuVqsoTkowCf6RBXuybO--owH1lf2Ywj1s,14470
-lionagi/libs/token_transform/symbolic_compress_context.py,sha256=joWy9QDtYIl8TuvZu00s90FH6EYtLtmXi9wnTfCnB58,4190
+lionagi/libs/token_transform/symbolic_compress_context.py,sha256=YLrdxesFfsbuGucBjvr9XM4u5f0KQ62I_SK_XSmLB1I,4451
 lionagi/libs/token_transform/synthlang.py,sha256=W6e-_265UXqVosM9X0TLKW53rNHvWCKhsWbVAop49Ac,259
 lionagi/libs/token_transform/types.py,sha256=4HgAfNDlJ_Hu18kLt59GHr_76eF3xLpNCTbOXlAYVlA,491
 lionagi/libs/token_transform/synthlang_/base.py,sha256=diDFrm-1Zf3PerKjODo-uFQMEjxPeaGtbZ5N1lK59Kg,4114
-lionagi/libs/token_transform/synthlang_/translate_to_synthlang.py,sha256=cfhTSYDryN27Wu2iWDziToI1qidJ6sT08DKOZ51OqPQ,4659
+lionagi/libs/token_transform/synthlang_/translate_to_synthlang.py,sha256=lRBpeKGhyNlf8ngigjYAw56QbIAVZWLQUzRg2wpwMfQ,4956
 lionagi/libs/token_transform/synthlang_/resources/frameworks/abstract_algebra.toml,sha256=2TuOAo97g8mNhdPH96HP8vYZpnC8neiP-KlhVqbp1Us,970
 lionagi/libs/token_transform/synthlang_/resources/frameworks/category_theory.toml,sha256=Stg9W3h8o7VkQ9tdAfSZmR3LctFqcH6OhOPdaw9BlIg,1064
 lionagi/libs/token_transform/synthlang_/resources/frameworks/complex_analysis.toml,sha256=iE6FS7Cn5_uJRG5-StLuMM4XVAk95bxhbYWwlstw_tA,1044
@@ -52,7 +53,7 @@ lionagi/libs/token_transform/synthlang_/resources/frameworks/math_logic.toml,sha
 lionagi/libs/token_transform/synthlang_/resources/frameworks/reflective_patterns.toml,sha256=LxBIVLHNLfvVdXjLAzqivrYaHNix514DLNYsbA-VSQ4,5730
 lionagi/libs/token_transform/synthlang_/resources/frameworks/set_theory.toml,sha256=SZpBvUySZ3_0pIrRko24a3KfbPHd55LyNwzFHyznjs4,1457
 lionagi/libs/token_transform/synthlang_/resources/frameworks/topology_fundamentals.toml,sha256=nnhfbIJQ5pTGlX7lo1XzjyOevaZOHuusvBuhwWHzbLk,1008
-lionagi/libs/token_transform/synthlang_/resources/mapping/rust_chinese_mapping.toml,sha256=KTyNI9_xr_bK3ipMZmGzRf6LGbmhOJ6S2NAbyMcQ9Kc,753
+lionagi/libs/token_transform/synthlang_/resources/mapping/rust_chinese_mapping.toml,sha256=IU5qOB-a_ZJI_wMOa0OJu3foAp6tIdvegnVt-yw5URM,1216
 lionagi/libs/token_transform/synthlang_/resources/utility/base_synthlang_system_prompt.toml,sha256=8xhY14WdDRF6GIToqzRPM7EjM6-uO6-hQ9Muei1A2Iw,3458
 lionagi/libs/validate/__init__.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
 lionagi/libs/validate/common_field_validators.py,sha256=1BHznXnJYcLQrHqvHKUnP6aqCptuQ0qN7KJRCExcJBU,4778
@@ -231,7 +232,7 @@ lionagi/tools/file/writer.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,
 lionagi/tools/file/providers/__init__.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
 lionagi/tools/file/providers/docling_.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
 lionagi/tools/query/__init__.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
-lionagi-0.9.14.dist-info/METADATA,sha256=5W9E7xdnhy-LI4ReBsPHdczTCaEMUHFuhCJ9IoRImOE,18464
-lionagi-0.9.14.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-lionagi-0.9.14.dist-info/licenses/LICENSE,sha256=VXFWsdoN5AAknBCgFqQNgPWYx7OPp-PFEP961zGdOjc,11288
-lionagi-0.9.14.dist-info/RECORD,,
+lionagi-0.9.15.dist-info/METADATA,sha256=pmtvUHZCxF3WoXCoIpW_LF4cj4BeaBhY0GtBtPMYWG0,18464
+lionagi-0.9.15.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+lionagi-0.9.15.dist-info/licenses/LICENSE,sha256=VXFWsdoN5AAknBCgFqQNgPWYx7OPp-PFEP961zGdOjc,11288
+lionagi-0.9.15.dist-info/RECORD,,