lionagi 0.10.4__py3-none-any.whl → 0.10.6__py3-none-any.whl

lionagi/fields/research.py

@@ -16,8 +16,8 @@ class ResearchFinding(HashableModel):
     A single piece of information or insight from the research phase.
     """
 
-    summary: str = Field(
-        ...,
+    summary: str | None = Field(
+        None,
         description="Concise text describing the discovered fact, concept, or best practice.",
     )
     snippets: list[TextSnippet | CodeSnippet] = Field(
@@ -35,8 +35,8 @@ class PotentialRisk(HashableModel):
     Identifies a risk or challenge discovered during research.
     """
 
-    description: str = Field(
-        ...,
+    description: str | None = Field(
+        None,
         description="Short text describing the risk. E.g., 'Scalability concerns with chosen DB'.",
     )
     impact: str | None = Field(

lionagi/libs/file/concat.py

@@ -0,0 +1,121 @@
+from pathlib import Path
+from typing import Any
+
+from lionagi.utils import create_path, lcall
+
+from .process import dir_to_files
+
+
+def concat(
+    data_path: str | Path | list,
+    file_types: list[str],
+    output_dir: str | Path = None,
+    output_filename: str = None,
+    file_exist_ok: bool = True,
+    recursive: bool = True,
+    verbose: bool = True,
+    threshold: int = 0,
+    return_fps: bool = False,
+    return_files: bool = False,
+    exclude_patterns: list[str] = None,
+    **kwargs,
+) -> dict[str, Any]:
+    """
+    data_path: str or Path or list of str or Path, the directory or file paths to concatenate.
+    file_types: list of str, the file types to concatenate. [e.g. ['.txt', '.md']]
+    output_dir: str or Path, the directory to save the concatenated file. If provided, will save the file.
+    output_filename: str, the filename to save the concatenated file.
+    file_exist_ok: bool, if True, overwrite the existing file. Default is True.
+    recursive: bool, if True, search files recursively. Default is True.
+    verbose: bool, if True, print the output path. Default is True.
+    threshold: int, the minimum number of chars for the file to be considered valid to concatenate.
+    exclude_patterns: list of str, patterns to exclude files from concatenation (e.g. ['log', 'temp', '.venv']).
+    kwargs: additional keyword arguments to pass to create_path.
+    """
+    persist_path = None
+    if output_dir:
+        if not output_filename:
+            output_filename = "concatenated_text.txt"
+            kwargs["timestamp"] = kwargs.get("timestamp", True)
+            kwargs["random_hash_digits"] = kwargs.get("random_hash_digits", 6)
+        output_filename = output_filename or "concatenated_text.txt"
+        persist_path = create_path(
+            output_dir, output_filename, file_exist_ok=file_exist_ok, **kwargs
+        )
+
+    texts = []
+
+    def _check_existence(_p: str) -> Path | list[Path] | None:
+        if exclude_patterns:
+            _str_p = str(_p)
+            for pattern in exclude_patterns:
+                if pattern in _str_p:
+                    return None
+
+        if not Path(_p).exists():
+            # if the path doesn't exist, return None
+            if verbose:
+                print(f"Path {_p} does not exist, skipping...")
+            return None
+
+        p = Path(_p)
+        if p.is_dir():
+            return dir_to_files(
+                p,
+                recursive=recursive,
+                file_types=file_types,
+                ignore_errors=True,
+                max_workers=5,
+            )
+        if p.is_file():
+            return p
+
+    data_path: list[Path] = lcall(
+        data_path,
+        _check_existence,
+        sanitize_input=True,
+        unique_input=True,
+        flatten=True,
+        dropna=True,
+        unique_output=True,
+        flatten_tuple_set=True,
+    )
+
+    contents = {}
+    fps = []
+    for dp in data_path:
+        try:
+            text = dp.read_text(encoding="utf-8")
+
+        except Exception:
+            # if we cannot read the file, skip it
+            print(f"Could not read file: {dp}. Skipping...")
+            continue
+
+        if threshold > 0 and len(text) < threshold:
+            continue
+
+        fps.append(dp)
+        contents[str(dp)] = text
+
+    for k, text in sorted(contents.items(), key=lambda x: x[0]):
+        texts.extend(["---", k, "---\n", text])
+
+    text = "\n".join(texts)
+    if persist_path:
+        persist_path.write_text(text, encoding="utf-8")
+    if verbose:
+        print(
+            f"Concatenated {len(fps)} files to {persist_path}."
+            f" The file contains {len(text)} characters."
+        )
+
+    out = {"text": text}  # default output
+    if persist_path:
+        out["persist_path"] = persist_path
+    if return_files:
+        out["texts"] = texts
+    if return_fps:
+        out["fps"] = fps
+
+    return out

lionagi/libs/file/process.py

@@ -203,7 +203,7 @@ def chunk(
             )
 
         if reader_tool is None:
-            reader_tool = lambda x: x.read_text(encoding="utf-8")
+            reader_tool = lambda x: Path(x).read_text(encoding="utf-8")
 
         if reader_tool == "docling":
             from lionagi.libs.package.imports import check_import

lionagi/libs/token_transform/symbolic_compress_context.py

@@ -111,7 +111,7 @@ async def symbolic_compress_context(
 
     elif text:
         chunks = chunk_content(
-            text=text,
+            text,
             chunk_by=chunk_by,
             chunk_size=chunk_size,
             overlap=overlap,

lionagi/version.py

@@ -1 +1 @@
-__version__ = "0.10.4"
+__version__ = "0.10.6"

{lionagi-0.10.4.dist-info → lionagi-0.10.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lionagi
-Version: 0.10.4
+Version: 0.10.6
 Summary: An Intelligence Operating System.
 Author-email: HaiyangLi <quantocean.li@gmail.com>
 License:            Apache License
@@ -220,7 +220,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.10
 Requires-Dist: aiocache>=0.12.0
-Requires-Dist: aiohttp>=3.11.0
+Requires-Dist: aiohttp>=3.11.14
 Requires-Dist: jinja2>=3.0.0
 Requires-Dist: pandas>=2.0.0
 Requires-Dist: pillow>=10.0.0

{lionagi-0.10.4.dist-info → lionagi-0.10.6.dist-info}/RECORD

@@ -4,7 +4,7 @@ lionagi/_errors.py,sha256=JlBTFJnRWtVYcRxKb7fWFiJHLbykl1E19mSJ8sXYVxg,455
 lionagi/_types.py,sha256=iDdYewsP9rDrM7QY19_NDTcWUk7swp8vnGCrloHMtUM,53
 lionagi/settings.py,sha256=W52mM34E6jXF3GyqCFzVREKZrmnUqtZm_BVDsUiDI_s,1627
 lionagi/utils.py,sha256=uLTJKl7aTnFXV6ehA6zwiwEB7G2nQYKsO2pZ6mqFzUk,78908
-lionagi/version.py,sha256=fGZMaoPHZfTX9I4TDkr07gp-kj_1U_SD-gjQC_2flQs,23
+lionagi/version.py,sha256=7qmFu9Qmzy5OxKJPN-LQOkzV_2T4cJYrUSLTfq7F3kE,23
 lionagi/adapters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lionagi/adapters/adapter.py,sha256=aW7s1OKAdxHd8HBv2UcThn-r2Q08EyArssNyFobMLuA,3357
 lionagi/adapters/json_adapter.py,sha256=EJj0Jev46ZhU3ZMnlYwyzN2rLxjLCVrMDpHkEuggBvk,4561
@@ -22,15 +22,16 @@ lionagi/fields/code.py,sha256=TFym51obzaSfCmeRoHZJyBtjfDI4tvl9F-1sjFc9rMw,7713
 lionagi/fields/file.py,sha256=DhQ_HE0RvTNzkvBGQHRgbMYSokDkzE8GEu814i6jw5Q,7297
 lionagi/fields/instruct.py,sha256=sMbCxEv0HQLa31JkJDmdrWWEzIfeKbcmN2hYOehz3Q0,4773
 lionagi/fields/reason.py,sha256=3Ksz9_40dI-oQ9VtmpnYAmJdeDDIO-TwLDrf1ijbXGM,1438
-lionagi/fields/research.py,sha256=2x6SFDwMSwzl4uHHUBbhT8mgwa49MsEy1NaUIb84MHU,1405
+lionagi/fields/research.py,sha256=eEPKocx8eQy2E9FExRWVIo6MK_xvmwBAoRZciBY3RG0,1421
 lionagi/libs/__init__.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
 lionagi/libs/parse.py,sha256=JRS3bql0InHJqATnAatl-hQv4N--XXw4P77JHhTFnrc,1011
 lionagi/libs/file/__init__.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
 lionagi/libs/file/chunk.py,sha256=XeVMwM33JF0X1W6udz_nhlb3DCevA_EK6A50Hn_e5SY,9300
+lionagi/libs/file/concat.py,sha256=YSauXVBL3WWx5Mvpn208Cj7q9TLt_aq-0M9J1fm-ins,3864
 lionagi/libs/file/concat_files.py,sha256=FoI983oWFzp9VfFDP7kmbRb3t1CPe5F5LCtsux0ASAs,3089
 lionagi/libs/file/file_ops.py,sha256=HBiIh1EljIJ5VTIXuyvJM0ppSs0YYOPUWmgDMJT634U,3430
 lionagi/libs/file/params.py,sha256=SZ5DkoffWfxWudOAYCfCxpL8UIm-1UjeyTtploo-Lqs,5824
-lionagi/libs/file/process.py,sha256=NKC-rIkm83qv4rgz-otd89aziop0G_LPz7_kmBAowsQ,8686
+lionagi/libs/file/process.py,sha256=EsnEJcQUm4ReP7qkCeMvL4Qe6fLRcENVWZndh9TSUsc,8692
 lionagi/libs/file/save.py,sha256=TCxVlKxFFnr3xZ-HAXPpTomQoyiVrp6nKRoj-bcQt4k,2863
 lionagi/libs/nested/__init__.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
 lionagi/libs/nested/flatten.py,sha256=sB4jxZRoaUbjak9RbIWVWNKz2hzkhQJPFffV_Ws1GA0,5479
@@ -57,7 +58,7 @@ lionagi/libs/token_transform/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NM
 lionagi/libs/token_transform/base.py,sha256=LBnaDgi4HNgaJJGwIzWcQjVMdu49i_93rRvOvMU22Rw,1545
 lionagi/libs/token_transform/llmlingua.py,sha256=DkeLUlrb7rGx3nZ04aADU9HXXu5mZTf_DBwT0xhzIv4,7
 lionagi/libs/token_transform/perplexity.py,sha256=tcVRjPBX3nuVqsoTkowCf6RBXuybO--owH1lf2Ywj1s,14470
-lionagi/libs/token_transform/symbolic_compress_context.py,sha256=Nr4vSJSN6sUQgaA1QxHhidWH3pUG_5RnoYeLHjMsoLA,4603
+lionagi/libs/token_transform/symbolic_compress_context.py,sha256=J8UleoCRDBvbFvmss2JjOvJfuBT7LXUW7AsRB-oQq7c,4598
 lionagi/libs/token_transform/synthlang.py,sha256=W6e-_265UXqVosM9X0TLKW53rNHvWCKhsWbVAop49Ac,259
 lionagi/libs/token_transform/types.py,sha256=4HgAfNDlJ_Hu18kLt59GHr_76eF3xLpNCTbOXlAYVlA,491
 lionagi/libs/token_transform/synthlang_/base.py,sha256=GDle72c8EjFz_3hg_k-y0YmksBsn5TSRVTw5_cwfWTo,4109
@@ -222,7 +223,7 @@ lionagi/tools/file/writer.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,
 lionagi/tools/file/providers/__init__.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
 lionagi/tools/file/providers/docling_.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
 lionagi/tools/query/__init__.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
-lionagi-0.10.4.dist-info/METADATA,sha256=NQTeKO05k_YvV-OvrUO3ZwOemMxHwvzqdidzxPeJJNk,18464
-lionagi-0.10.4.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-lionagi-0.10.4.dist-info/licenses/LICENSE,sha256=VXFWsdoN5AAknBCgFqQNgPWYx7OPp-PFEP961zGdOjc,11288
-lionagi-0.10.4.dist-info/RECORD,,
+lionagi-0.10.6.dist-info/METADATA,sha256=5vxUHPlRrLHQDEKg7Ke_NT5wDxl1hj3Llx3coh2-yro,18465
+lionagi-0.10.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+lionagi-0.10.6.dist-info/licenses/LICENSE,sha256=VXFWsdoN5AAknBCgFqQNgPWYx7OPp-PFEP961zGdOjc,11288
+lionagi-0.10.6.dist-info/RECORD,,