lionagi 0.9.7__tar.gz → 0.9.9__tar.gz

{lionagi-0.9.7 → lionagi-0.9.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lionagi
-Version: 0.9.7
+Version: 0.9.9
 Summary: An Intelligence Operating System.
 Author-email: HaiyangLi <quantocean.li@gmail.com>
 License:            Apache License

{lionagi-0.9.7 → lionagi-0.9.9}/lionagi/libs/file/chunk.py

@@ -218,6 +218,7 @@ def chunk_content(
     threshold: int = 256,
     metadata: dict[str, Any] = {},
     return_tokens: bool = False,
+    as_node: bool = False,
     **kwargs: Any,
 ) -> list[dict[str, Any]]:
     """
@@ -257,6 +258,22 @@ def chunk_content(
             threshold=threshold,
         )
 
+    if as_node:
+        from lionagi.protocols.graph.node import Node
+
+        return [
+            Node(
+                content=chunk,
+                metadata={
+                    "chunk_id": i + 1,
+                    "total_chunks": len(chunks),
+                    "chunk_size": len(chunk),
+                    **metadata,
+                },
+            )
+            for i, chunk in enumerate(chunks)
+        ]
+
     return [
         {
             "chunk_content": chunk,

{lionagi-0.9.7 → lionagi-0.9.9}/lionagi/libs/file/process.py

@@ -6,7 +6,9 @@ import logging
 from collections.abc import Callable
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
-from typing import Any
+from typing import Any, Literal
+
+from lionagi.utils import lcall
 
 from .chunk import chunk_content
 from .save import save_chunks
@@ -87,7 +89,7 @@ def dir_to_files(
 
 def file_to_chunks(
     file_path: str | Path,
-    chunk_func: Callable[[str, int, float, int], list[str]],
+    chunk_by: Literal["chars", "tokens"] = "chars",
     chunk_size: int = 1500,
     overlap: float = 0.1,
     threshold: int = 200,
@@ -97,6 +99,7 @@ def file_to_chunks(
     verbose: bool = False,
     timestamp: bool = True,
     random_hash_digits: int = 4,
+    as_node: bool = False,
 ) -> list[dict[str, Any]]:
     """
     Process a file and split its content into chunks.
@@ -124,9 +127,10 @@ def file_to_chunks(
         ValueError: If there's an error processing the file.
     """
     try:
-        file_path = Path(file_path)
-        with open(file_path, encoding=encoding) as f:
-            content = f.read()
+        if isinstance(file_path, str):
+            file_path = Path(file_path)
+
+        text = file_path.read_text(encoding=encoding)
 
         metadata = {
             "file_path": str(file_path),
@@ -136,14 +140,111 @@ def file_to_chunks(
         }
 
         chunks = chunk_content(
-            content, chunk_func, chunk_size, overlap, threshold, metadata
+            text,
+            chunk_by=chunk_by,
+            chunk_size=chunk_size,
+            overlap=overlap,
+            threshold=threshold,
+            metadata=metadata,
+            as_node=as_node,
         )
 
         if output_dir:
             save_chunks(
-                chunks, output_dir, verbose, timestamp, random_hash_digits
+                chunks=chunks,
+                output_dir=output_dir,
+                verbose=verbose,
+                timestamp=timestamp,
+                random_hash_digits=random_hash_digits,
             )
 
         return chunks
     except Exception as e:
         raise ValueError(f"Error processing file {file_path}: {e}") from e
+
+
+def chunk(
+    url_or_path: str | Path,
+    *,
+    file_types: list[str] | None = None,  # only local files
+    recursive: bool = False,  # only local files
+    chunk_by: Literal["chars", "tokens"] = "chars",
+    chunk_size: int = 1500,
+    overlap: float = 0.1,
+    threshold: int = 200,
+    output_file: str | Path | None = None,
+    metadata: dict[str, Any] | None = None,
+    reader_tool: Callable = None,
+):
+    if isinstance(url_or_path, str):
+        url_or_path = Path(url_or_path)
+
+    chunks = None
+    files = None
+    if url_or_path.exists():
+        if url_or_path.is_dir():
+            files = dir_to_files(
+                directory=url_or_path,
+                file_types=file_types,
+                recursive=recursive,
+            )
+        elif url_or_path.is_file():
+            files = [url_or_path]
+    else:
+        files = (
+            [str(url_or_path)]
+            if not isinstance(url_or_path, list)
+            else url_or_path
+        )
+
+    if reader_tool is None:
+        reader_tool = lambda x: x.read_text(encoding="utf-8")
+
+    if reader_tool == "docling":
+        from lionagi.libs.package.imports import check_import
+
+        DocumentConverter = check_import(
+            "docling",
+            module_name="document_converter",
+            import_name="DocumentConverter",
+        )
+        converter = DocumentConverter()
+        reader_tool = lambda x: converter.convert(
+            x
+        ).document.export_to_markdown()
+
+    texts = lcall(files, reader_tool)
+    chunks = lcall(
+        texts,
+        chunk_content,
+        chunk_by=chunk_by,
+        chunk_size=chunk_size,
+        overlap=overlap,
+        threshold=threshold,
+        metadata=metadata,
+        as_node=True,
+        flatten=True,
+    )
+    if threshold:
+        chunks = [c for c in chunks if len(c.content) > threshold]
+
+    if output_file:
+        from lionagi.protocols.generic.pile import Pile
+
+        output_file = Path(output_file)
+        if output_file.suffix == ".csv":
+            p = Pile(chunks)
+            p.to_csv_file(output_file)
+
+        elif output_file.suffix == ".json":
+            p = Pile(chunks)
+            p.to_json_file(output_file, use_pd=True)
+
+        elif output_file.suffix in Pile.list_adapters():
+            p = Pile(chunks)
+            p.adapt_to(output_file.suffix, fp=output_file)
+
+        else:
+            raise ValueError(f"Unsupported output file format: {output_file}")
+
+    return chunks

{lionagi-0.9.7 → lionagi-0.9.9}/lionagi/service/providers/anthropic_/messages.py

@@ -29,6 +29,7 @@ CHAT_COMPLETION_CONFIG = {
         "tools",
         "top_p",
         "top_k",
+        "cache_control",
     },
     "allowed_roles": ["user", "assistant"],
 }
@@ -49,6 +50,21 @@ class AnthropicChatCompletionEndPoint(ChatCompletionEndPoint):
         for k, v in kwargs.items():
             if k in self.acceptable_kwargs:
                 payload[k] = v
+        if "cache_control" in payload:
+            cache_control = payload.pop("cache_control")
+            if cache_control:
+                cache_control = "ephemeral"
+                last_message = payload["messages"][-1]["content"]
+                if isinstance(last_message, str):
+                    last_message = {
+                        "type": "text",
+                        "text": last_message,
+                        "cache_control": cache_control,
+                    }
+                elif isinstance(last_message, list) and isinstance(
+                    last_message[-1], dict
+                ):
+                    last_message[-1]["cache_control"] = cache_control
         if "api_key" in kwargs:
             headers["x-api-key"] = kwargs["api_key"]
         headers["anthropic-version"] = kwargs.pop(

{lionagi-0.9.7 → lionagi-0.9.9}/lionagi/service/providers/openai_/chat_completions.py

@@ -4,6 +4,8 @@
 
 from lionagi.service.endpoints.chat_completion import ChatCompletionEndPoint
 
+from .spec import reasoning_models, reasoning_not_support_params
+
 CHAT_COMPLETION_CONFIG = {
     "provider": "openai",
     "base_url": "https://api.openai.com/v1",
@@ -84,9 +86,9 @@ class OpenAIChatCompletionEndPoint(ChatCompletionEndPoint):
         if "api_key" in kwargs:
             headers["Authorization"] = f"Bearer {kwargs['api_key']}"
 
-        if payload.get("model") in ["o1", "o1-2024-12-17"]:
-            payload.pop("temperature", None)
-            payload.pop("top_p", None)
+        if payload.get("model") in reasoning_models:
+            for param in reasoning_not_support_params:
+                payload.pop(param, None)
             if payload["messages"][0].get("role") == "system":
                 payload["messages"][0]["role"] = "developer"
         else:

lionagi-0.9.9/lionagi/service/providers/openai_/spec.py

@@ -0,0 +1,14 @@
+reasoning_models = (
+    "o3-mini-2025-01-31",
+    "o3-mini",
+    "o1",
+    "o1-2024-12-17",
+)
+
+reasoning_not_support_params = (
+    "temperature",
+    "top_p",
+    "logit_bias",
+    "logprobs",
+    "top_logprobs",
+)

{lionagi-0.9.7 → lionagi-0.9.9}/lionagi/tools/types.py

@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: Apache-2.0
 
 
+from lionagi.libs.file.process import chunk
+
 from .file.reader import ReaderTool
 
-__all__ = ("ReaderTool",)
+__all__ = ("ReaderTool", "chunk")

lionagi-0.9.9/lionagi/version.py

@@ -0,0 +1 @@
+__version__ = "0.9.9"

{lionagi-0.9.7 → lionagi-0.9.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lionagi"
-version = "0.9.7"
+version = "0.9.9"
 description = "An Intelligence Operating System."
 authors = [
     { name = "HaiyangLi", email = "quantocean.li@gmail.com" },

{lionagi-0.9.7 → lionagi-0.9.9}/uv.lock

@@ -1208,7 +1208,7 @@ wheels = [
 
 [[package]]
 name = "lionagi"
-version = "0.9.7"
+version = "0.9.9"
 source = { editable = "." }
 dependencies = [
     { name = "aiocache" },

lionagi-0.9.7/lionagi/version.py

@@ -1 +0,0 @@
-__version__ = "0.9.7"