lionagi 0.9.6__py3-none-any.whl → 0.9.8__py3-none-any.whl

lionagi/libs/file/chunk.py

@@ -218,6 +218,7 @@ def chunk_content(
     threshold: int = 256,
     metadata: dict[str, Any] = {},
     return_tokens: bool = False,
+    as_node: bool = False,
     **kwargs: Any,
 ) -> list[dict[str, Any]]:
     """
@@ -257,6 +258,22 @@ def chunk_content(
             threshold=threshold,
         )
 
+    if as_node:
+        from lionagi.protocols.graph.node import Node
+
+        return [
+            Node(
+                content=chunk,
+                metadata={
+                    "chunk_id": i + 1,
+                    "total_chunks": len(chunks),
+                    "chunk_size": len(chunk),
+                    **metadata,
+                },
+            )
+            for i, chunk in enumerate(chunks)
+        ]
+
     return [
         {
             "chunk_content": chunk,

lionagi/libs/file/process.py

@@ -6,7 +6,9 @@ import logging
 from collections.abc import Callable
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
-from typing import Any
+from typing import Any, Literal
+
+from lionagi.utils import lcall
 
 from .chunk import chunk_content
 from .save import save_chunks
@@ -87,7 +89,7 @@ def dir_to_files(
 
 def file_to_chunks(
     file_path: str | Path,
-    chunk_func: Callable[[str, int, float, int], list[str]],
+    chunk_by: Literal["chars", "tokens"] = "chars",
     chunk_size: int = 1500,
     overlap: float = 0.1,
     threshold: int = 200,
@@ -97,6 +99,7 @@ def file_to_chunks(
     verbose: bool = False,
     timestamp: bool = True,
     random_hash_digits: int = 4,
+    as_node: bool = False,
 ) -> list[dict[str, Any]]:
     """
     Process a file and split its content into chunks.
@@ -124,9 +127,10 @@ def file_to_chunks(
         ValueError: If there's an error processing the file.
     """
     try:
-        file_path = Path(file_path)
-        with open(file_path, encoding=encoding) as f:
-            content = f.read()
+        if isinstance(file_path, str):
+            file_path = Path(file_path)
+
+        text = file_path.read_text(encoding=encoding)
 
         metadata = {
             "file_path": str(file_path),
@@ -136,14 +140,111 @@ def file_to_chunks(
         }
 
         chunks = chunk_content(
-            content, chunk_func, chunk_size, overlap, threshold, metadata
+            text,
+            chunk_by=chunk_by,
+            chunk_size=chunk_size,
+            overlap=overlap,
+            threshold=threshold,
+            metadata=metadata,
+            as_node=as_node,
         )
 
         if output_dir:
             save_chunks(
-                chunks, output_dir, verbose, timestamp, random_hash_digits
+                chunks=chunks,
+                output_dir=output_dir,
+                verbose=verbose,
+                timestamp=timestamp,
+                random_hash_digits=random_hash_digits,
             )
 
         return chunks
     except Exception as e:
         raise ValueError(f"Error processing file {file_path}: {e}") from e
+
+
+def chunk(
+    url_or_path: str | Path,
+    *,
+    file_types: list[str] | None = None,  # only local files
+    recursive: bool = False,  # only local files
+    chunk_by: Literal["chars", "tokens"] = "chars",
+    chunk_size: int = 1500,
+    overlap: float = 0.1,
+    threshold: int = 200,
+    output_file: str | Path | None = None,
+    metadata: dict[str, Any] | None = None,
+    reader_tool: Callable = None,
+):
+    if isinstance(url_or_path, str):
+        url_or_path = Path(url_or_path)
+
+    chunks = None
+    files = None
+    if url_or_path.exists():
+        if url_or_path.is_dir():
+            files = dir_to_files(
+                directory=url_or_path,
+                file_types=file_types,
+                recursive=recursive,
+            )
+        elif url_or_path.is_file():
+            files = [url_or_path]
+    else:
+        files = (
+            [str(url_or_path)]
+            if not isinstance(url_or_path, list)
+            else url_or_path
+        )
+
+    if reader_tool is None:
+        reader_tool = lambda x: x.read_text(encoding="utf-8")
+
+    if reader_tool == "docling":
+        from lionagi.libs.package.imports import check_import
+
+        DocumentConverter = check_import(
+            "docling",
+            module_name="document_converter",
+            import_name="DocumentConverter",
+        )
+        converter = DocumentConverter()
+        reader_tool = lambda x: converter.convert(
+            x
+        ).document.export_to_markdown()
+
+    texts = lcall(files, reader_tool)
+    chunks = lcall(
+        texts,
+        chunk_content,
+        chunk_by=chunk_by,
+        chunk_size=chunk_size,
+        overlap=overlap,
+        threshold=threshold,
+        metadata=metadata,
+        as_node=True,
+        flatten=True,
+    )
+    if threshold:
+        chunks = [c for c in chunks if len(c.content) > threshold]
+
+    if output_file:
+        from lionagi.protocols.generic.pile import Pile
+
+        output_file = Path(output_file)
+        if output_file.suffix == ".csv":
+            p = Pile(chunks)
+            p.to_csv_file(output_file)
+
+        elif output_file.suffix == ".json":
+            p = Pile(chunks)
+            p.to_json_file(output_file, use_pd=True)
+
+        elif output_file.suffix in Pile.list_adapters():
+            p = Pile(chunks)
+            p.adapt_to(output_file.suffix, fp=output_file)
+
+        else:
+            raise ValueError(f"Unsupported output file format: {output_file}")
+
+    return chunks

lionagi/tools/types.py

@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: Apache-2.0
 
 
+from lionagi.libs.file.process import chunk
+
 from .file.reader import ReaderTool
 
-__all__ = ("ReaderTool",)
+__all__ = ("ReaderTool", "chunk")

lionagi/version.py

@@ -1 +1 @@
-__version__ = "0.9.6"
+__version__ = "0.9.8"

{lionagi-0.9.6.dist-info → lionagi-0.9.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lionagi
-Version: 0.9.6
+Version: 0.9.8
 Summary: An Intelligence Operating System.
 Author-email: HaiyangLi <quantocean.li@gmail.com>
 License:            Apache License
@@ -227,6 +227,13 @@ Requires-Dist: pillow>=10.0.0
 Requires-Dist: pydantic>=2.0.0
 Requires-Dist: python-dotenv>=1.0.1
 Requires-Dist: tiktoken>=0.8.0
+Provides-Extra: llms
+Requires-Dist: litellm>=1.59.5; extra == 'llms'
+Provides-Extra: ollama
+Requires-Dist: ollama>=0.4.7; extra == 'ollama'
+Requires-Dist: openai>=1.60.0; extra == 'ollama'
+Provides-Extra: tools
+Requires-Dist: docling>=2.15.1; extra == 'tools'
 Description-Content-Type: text/markdown
 
 ![PyPI - Version](https://img.shields.io/pypi/v/lionagi?labelColor=233476aa&color=231fc935)
@@ -358,6 +365,14 @@ analysis = await branch.communicate("Analyze these stats", imodel=sonnet)
 
 Seamlessly route to different models in the same workflow.
 
+### optional dependencies
+
+```
+pip install "lionagi[tools]"
+pip install "lionagi[llms]"
+pip install "lionagi[ollama]"
+```
+
 ## Community & Contributing
 
 We welcome issues, ideas, and pull requests:

{lionagi-0.9.6.dist-info → lionagi-0.9.8.dist-info}/RECORD

@@ -4,14 +4,14 @@ lionagi/_errors.py,sha256=JlBTFJnRWtVYcRxKb7fWFiJHLbykl1E19mSJ8sXYVxg,455
 lionagi/_types.py,sha256=9g7iytvSj3UjZxD-jL06_fxuNfgZyWT3Qnp0XYp1wQU,63
 lionagi/settings.py,sha256=W52mM34E6jXF3GyqCFzVREKZrmnUqtZm_BVDsUiDI_s,1627
 lionagi/utils.py,sha256=K36D9AAGiMPR4eM9tYoiVgvH-NdPPSeMQPls09s7keQ,73223
-lionagi/version.py,sha256=IgVHjr-TeioZYLJSkvpT80LLGi6U3ONzR1cfYfd5XNQ,22
+lionagi/version.py,sha256=GI3Cr15wC6XhdahtMoSOf62d2gYve5G_PiR_xx_RGKc,22
 lionagi/libs/__init__.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
 lionagi/libs/parse.py,sha256=JRS3bql0InHJqATnAatl-hQv4N--XXw4P77JHhTFnrc,1011
 lionagi/libs/file/__init__.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
-lionagi/libs/file/chunk.py,sha256=UiN92z52tlK2fUlnOYpAzTrjZV8sVDmPTnutnWkWH8Y,8835
+lionagi/libs/file/chunk.py,sha256=_C-b3M6Fe1O8Mb8MAromNpFqkFx7vfpPZvzqV5MUlus,9282
 lionagi/libs/file/file_ops.py,sha256=HBiIh1EljIJ5VTIXuyvJM0ppSs0YYOPUWmgDMJT634U,3430
 lionagi/libs/file/params.py,sha256=SZ5DkoffWfxWudOAYCfCxpL8UIm-1UjeyTtploo-Lqs,5824
-lionagi/libs/file/process.py,sha256=1nILukF_JXr03hVPQKmqOTwFYv9FhgZirLQuZG2u4KY,5486
+lionagi/libs/file/process.py,sha256=bIe4AdQ7eT0NHSMn0_Ail_-ltlM21YWqUWiPGXEPuHU,8264
 lionagi/libs/file/save.py,sha256=TCxVlKxFFnr3xZ-HAXPpTomQoyiVrp6nKRoj-bcQt4k,2863
 lionagi/libs/nested/__init__.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
 lionagi/libs/nested/flatten.py,sha256=sB4jxZRoaUbjak9RbIWVWNKz2hzkhQJPFffV_Ws1GA0,5479
@@ -195,7 +195,7 @@ lionagi/session/prompts.py,sha256=AhuHL19s0TijVZX3tMKUKMi6l88xeVdpkuEn2vJSRyU,32
 lionagi/session/session.py,sha256=8SuNMiJX6IAW6Ou8aDK0LsVG7zcD5yd22sakMyrd3pw,8987
 lionagi/tools/__init__.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
 lionagi/tools/base.py,sha256=R5T8hliDfJwXitNcgs2RPogE3yYserRjfRAVzAY2kM4,349
-lionagi/tools/types.py,sha256=f4TgF9LJ86P5dHIXNAHMSnLDnjto45M8Q_UJlyI-G3Y,177
+lionagi/tools/types.py,sha256=XtJLY0m-Yi_ZLWhm0KycayvqMCZd--HxfQ0x9vFUYDE,230
 lionagi/tools/browser/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lionagi/tools/browser/providers/__init__.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
 lionagi/tools/browser/providers/browser_use_.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
@@ -213,7 +213,7 @@ lionagi/tools/file/writer.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,
 lionagi/tools/file/providers/__init__.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
 lionagi/tools/file/providers/docling_.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
 lionagi/tools/query/__init__.py,sha256=5y5joOZzfFWERl75auAcNcKC3lImVJ5ZZGvvHZUFCJM,112
-lionagi-0.9.6.dist-info/METADATA,sha256=WGUAokWhfSlKKpws48WJRgBmwQJ1MTBAQd3o_EVdPdg,18053
-lionagi-0.9.6.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-lionagi-0.9.6.dist-info/licenses/LICENSE,sha256=VXFWsdoN5AAknBCgFqQNgPWYx7OPp-PFEP961zGdOjc,11288
-lionagi-0.9.6.dist-info/RECORD,,
+lionagi-0.9.8.dist-info/METADATA,sha256=Ywnd678RVNXSuGJYVwjOs9LAO8Ec6PXR8zY8nM0oemc,18436
+lionagi-0.9.8.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+lionagi-0.9.8.dist-info/licenses/LICENSE,sha256=VXFWsdoN5AAknBCgFqQNgPWYx7OPp-PFEP961zGdOjc,11288
+lionagi-0.9.8.dist-info/RECORD,,