lionagi 0.9.6__tar.gz → 0.9.8__tar.gz

{lionagi-0.9.6 → lionagi-0.9.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lionagi
-Version: 0.9.6
+Version: 0.9.8
 Summary: An Intelligence Operating System.
 Author-email: HaiyangLi <quantocean.li@gmail.com>
 License:            Apache License
@@ -227,6 +227,13 @@ Requires-Dist: pillow>=10.0.0
 Requires-Dist: pydantic>=2.0.0
 Requires-Dist: python-dotenv>=1.0.1
 Requires-Dist: tiktoken>=0.8.0
+Provides-Extra: llms
+Requires-Dist: litellm>=1.59.5; extra == 'llms'
+Provides-Extra: ollama
+Requires-Dist: ollama>=0.4.7; extra == 'ollama'
+Requires-Dist: openai>=1.60.0; extra == 'ollama'
+Provides-Extra: tools
+Requires-Dist: docling>=2.15.1; extra == 'tools'
 Description-Content-Type: text/markdown
 
 ![PyPI - Version](https://img.shields.io/pypi/v/lionagi?labelColor=233476aa&color=231fc935)
@@ -358,6 +365,14 @@ analysis = await branch.communicate("Analyze these stats", imodel=sonnet)
 
 Seamlessly route to different models in the same workflow.
 
+### optional dependencies
+
+```
+pip install "lionagi[tools]"
+pip install "lionagi[llms]"
+pip install "lionagi[ollama]"
+```
+
 ## Community & Contributing
 
 We welcome issues, ideas, and pull requests:

{lionagi-0.9.6 → lionagi-0.9.8}/README.md

@@ -127,6 +127,14 @@ analysis = await branch.communicate("Analyze these stats", imodel=sonnet)
 
 Seamlessly route to different models in the same workflow.
 
+### optional dependencies
+
+```
+pip install "lionagi[tools]"
+pip install "lionagi[llms]"
+pip install "lionagi[ollama]"
+```
+
 ## Community & Contributing
 
 We welcome issues, ideas, and pull requests:

{lionagi-0.9.6 → lionagi-0.9.8}/lionagi/libs/file/chunk.py

@@ -218,6 +218,7 @@ def chunk_content(
     threshold: int = 256,
     metadata: dict[str, Any] = {},
     return_tokens: bool = False,
+    as_node: bool = False,
     **kwargs: Any,
 ) -> list[dict[str, Any]]:
     """
@@ -257,6 +258,22 @@ def chunk_content(
             threshold=threshold,
         )
 
+    if as_node:
+        from lionagi.protocols.graph.node import Node
+
+        return [
+            Node(
+                content=chunk,
+                metadata={
+                    "chunk_id": i + 1,
+                    "total_chunks": len(chunks),
+                    "chunk_size": len(chunk),
+                    **metadata,
+                },
+            )
+            for i, chunk in enumerate(chunks)
+        ]
+
     return [
         {
             "chunk_content": chunk,

{lionagi-0.9.6 → lionagi-0.9.8}/lionagi/libs/file/process.py

@@ -6,7 +6,9 @@ import logging
 from collections.abc import Callable
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
-from typing import Any
+from typing import Any, Literal
+
+from lionagi.utils import lcall
 
 from .chunk import chunk_content
 from .save import save_chunks
@@ -87,7 +89,7 @@ def dir_to_files(
 
 def file_to_chunks(
     file_path: str | Path,
-    chunk_func: Callable[[str, int, float, int], list[str]],
+    chunk_by: Literal["chars", "tokens"] = "chars",
     chunk_size: int = 1500,
     overlap: float = 0.1,
     threshold: int = 200,
@@ -97,6 +99,7 @@ def file_to_chunks(
     verbose: bool = False,
     timestamp: bool = True,
     random_hash_digits: int = 4,
+    as_node: bool = False,
 ) -> list[dict[str, Any]]:
     """
     Process a file and split its content into chunks.
@@ -124,9 +127,10 @@ def file_to_chunks(
         ValueError: If there's an error processing the file.
     """
     try:
-        file_path = Path(file_path)
-        with open(file_path, encoding=encoding) as f:
-            content = f.read()
+        if isinstance(file_path, str):
+            file_path = Path(file_path)
+
+        text = file_path.read_text(encoding=encoding)
 
         metadata = {
             "file_path": str(file_path),
@@ -136,14 +140,111 @@ def file_to_chunks(
         }
 
         chunks = chunk_content(
-            content, chunk_func, chunk_size, overlap, threshold, metadata
+            text,
+            chunk_by=chunk_by,
+            chunk_size=chunk_size,
+            overlap=overlap,
+            threshold=threshold,
+            metadata=metadata,
+            as_node=as_node,
         )
 
         if output_dir:
             save_chunks(
-                chunks, output_dir, verbose, timestamp, random_hash_digits
+                chunks=chunks,
+                output_dir=output_dir,
+                verbose=verbose,
+                timestamp=timestamp,
+                random_hash_digits=random_hash_digits,
             )
 
         return chunks
     except Exception as e:
         raise ValueError(f"Error processing file {file_path}: {e}") from e
+
+
+def chunk(
+    url_or_path: str | Path,
+    *,
+    file_types: list[str] | None = None,  # only local files
+    recursive: bool = False,  # only local files
+    chunk_by: Literal["chars", "tokens"] = "chars",
+    chunk_size: int = 1500,
+    overlap: float = 0.1,
+    threshold: int = 200,
+    output_file: str | Path | None = None,
+    metadata: dict[str, Any] | None = None,
+    reader_tool: Callable = None,
+):
+    if isinstance(url_or_path, str):
+        url_or_path = Path(url_or_path)
+
+    chunks = None
+    files = None
+    if url_or_path.exists():
+        if url_or_path.is_dir():
+            files = dir_to_files(
+                directory=url_or_path,
+                file_types=file_types,
+                recursive=recursive,
+            )
+        elif url_or_path.is_file():
+            files = [url_or_path]
+    else:
+        files = (
+            [str(url_or_path)]
+            if not isinstance(url_or_path, list)
+            else url_or_path
+        )
+
+    if reader_tool is None:
+        reader_tool = lambda x: x.read_text(encoding="utf-8")
+
+    if reader_tool == "docling":
+        from lionagi.libs.package.imports import check_import
+
+        DocumentConverter = check_import(
+            "docling",
+            module_name="document_converter",
+            import_name="DocumentConverter",
+        )
+        converter = DocumentConverter()
+        reader_tool = lambda x: converter.convert(
+            x
+        ).document.export_to_markdown()
+
+    texts = lcall(files, reader_tool)
+    chunks = lcall(
+        texts,
+        chunk_content,
+        chunk_by=chunk_by,
+        chunk_size=chunk_size,
+        overlap=overlap,
+        threshold=threshold,
+        metadata=metadata,
+        as_node=True,
+        flatten=True,
+    )
+    if threshold:
+        chunks = [c for c in chunks if len(c.content) > threshold]
+
+    if output_file:
+        from lionagi.protocols.generic.pile import Pile
+
+        output_file = Path(output_file)
+        if output_file.suffix == ".csv":
+            p = Pile(chunks)
+            p.to_csv_file(output_file)
+
+        elif output_file.suffix == ".json":
+            p = Pile(chunks)
+            p.to_json_file(output_file, use_pd=True)
+
+        elif output_file.suffix in Pile.list_adapters():
+            p = Pile(chunks)
+            p.adapt_to(output_file.suffix, fp=output_file)
+
+        else:
+            raise ValueError(f"Unsupported output file format: {output_file}")
+
+    return chunks

{lionagi-0.9.6 → lionagi-0.9.8}/lionagi/tools/types.py

@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: Apache-2.0
 
 
+from lionagi.libs.file.process import chunk
+
 from .file.reader import ReaderTool
 
-__all__ = ("ReaderTool",)
+__all__ = ("ReaderTool", "chunk")

lionagi-0.9.8/lionagi/version.py

@@ -0,0 +1 @@
+__version__ = "0.9.8"

{lionagi-0.9.6 → lionagi-0.9.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lionagi"
-version = "0.9.6"
+version = "0.9.8"
 description = "An Intelligence Operating System."
 authors = [
     { name = "HaiyangLi", email = "quantocean.li@gmail.com" },
@@ -47,14 +47,16 @@ dev = [
     "ipykernel>=6.29.5",
     "ipywidgets>=8.1.5",
 ]
+
+[project.optional-dependencies]
 tools = [
     "docling>=2.15.1",
 ]
 llms = [
     "litellm>=1.59.5",
-    "openai>=1.60.0",
 ]
 ollama = [
+    "openai>=1.60.0",
     "ollama>=0.4.7",
 ]
 

{lionagi-0.9.6 → lionagi-0.9.8}/uv.lock

@@ -1208,7 +1208,7 @@ wheels = [
 
 [[package]]
 name = "lionagi"
-version = "0.9.6"
+version = "0.9.8"
 source = { editable = "." }
 dependencies = [
     { name = "aiocache" },
@@ -1221,6 +1221,18 @@ dependencies = [
     { name = "tiktoken" },
 ]
 
+[package.optional-dependencies]
+llms = [
+    { name = "litellm" },
+]
+ollama = [
+    { name = "ollama" },
+    { name = "openai" },
+]
+tools = [
+    { name = "docling" },
+]
+
 [package.dev-dependencies]
 dev = [
     { name = "black", extra = ["jupyter"] },
@@ -1235,22 +1247,16 @@ dev = [
     { name = "sphinx" },
     { name = "sphinx-autobuild" },
 ]
-llms = [
-    { name = "litellm" },
-    { name = "openai" },
-]
-ollama = [
-    { name = "ollama" },
-]
-tools = [
-    { name = "docling" },
-]
 
 [package.metadata]
 requires-dist = [
     { name = "aiocache", specifier = ">=0.12.0" },
     { name = "aiohttp", specifier = ">=3.11.0" },
+    { name = "docling", marker = "extra == 'tools'", specifier = ">=2.15.1" },
     { name = "jinja2", specifier = ">=3.1.0" },
+    { name = "litellm", marker = "extra == 'llms'", specifier = ">=1.59.5" },
+    { name = "ollama", marker = "extra == 'ollama'", specifier = ">=0.4.7" },
+    { name = "openai", marker = "extra == 'ollama'", specifier = ">=1.60.0" },
     { name = "pandas", specifier = ">=2.0.0" },
     { name = "pillow", specifier = ">=10.0.0" },
     { name = "pydantic", specifier = ">=2.0.0" },
@@ -1272,12 +1278,6 @@ dev = [
     { name = "sphinx", specifier = ">=8.1.3" },
     { name = "sphinx-autobuild", specifier = ">=2024.10.3" },
 ]
-llms = [
-    { name = "litellm", specifier = ">=1.59.5" },
-    { name = "openai", specifier = ">=1.60.0" },
-]
-ollama = [{ name = "ollama", specifier = ">=0.4.7" }]
-tools = [{ name = "docling", specifier = ">=2.15.1" }]
 
 [[package]]
 name = "litellm"

lionagi-0.9.6/lionagi/version.py

@@ -1 +0,0 @@
-__version__ = "0.9.6"