lionagi 0.0.314__py3-none-any.whl → 0.0.316__py3-none-any.whl

lionagi/__init__.py

@@ -8,6 +8,9 @@ from dotenv import load_dotenv
 
 from .core import direct, Branch, Session, func_to_tool
 from .integrations.provider.services import Services
+from .integrations.chunker.chunk import chunk
+from .integrations.loader.load import load
+
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)

lionagi/core/direct/__init__.py

@@ -3,5 +3,8 @@ from .select import select
 from .score import score
 from .react import react
 from .vote import vote
+from .plan import plan
+from .cot import chain_of_thoughts, chain_of_react
 
-__all__ = ["predict", "select", "score", "vote", "react"]
+
+__all__ = ["predict", "select", "score", "vote", "react", "plan", "chain_of_thoughts", "chain_of_react"]

lionagi/core/direct/cot.py

@@ -0,0 +1,88 @@
+from typing import Callable
+from lionagi.libs import convert
+from ..tool import func_to_tool
+from ..schema import Tool
+from .predict import predict
+from .plan import plan
+from .react import react
+
+from .utils import _process_tools
+
+
+async def chain_of_thoughts(
+    sentence=None,
+    branch=None,
+    instruction=None,
+    reason=False,
+    confidence_score=False,
+    num_steps=3,
+    directive_kwargs={},
+    return_branch=False,
+    **kwargs
+):
+    
+    out_, outs, answer, reasons, confidence_score = "", [], "", [], 0
+    if branch is not None:
+        out_ = await plan(sentence, branch=branch, instruction=instruction, num_steps=num_steps, **kwargs)
+    else:
+        out_, branch = await plan(sentence, instruction=instruction, branch=branch, num_steps=num_steps, return_branch=True, **kwargs)
+    
+    for i in range(len(out_.plan)):
+        _out = await predict(branch=branch, instruction=out_.plan[f"step_{i+1}"], reason=reason, confidence_score=confidence_score, **directive_kwargs)
+        answer += _out.answer
+        if reason:
+            reasons.append(_out.reason)
+        if confidence_score:
+            confidence_score += _out.confidence_score
+        outs.append(_out)
+    
+    setattr(out_, "chain_output", outs)
+    setattr(out_, "chain_answer", answer)
+    
+    if reason:
+        setattr(out_, "chain_reasons", reasons)
+    if confidence_score:
+        setattr(out_, "chain_confidence_score", confidence_score/len(outs))
+    
+    if return_branch:
+        return out_, branch
+    
+    return out_
+    
+
+async def chain_of_react(
+    sentence=None,
+    branch=None,
+    instruction=None,
+    num_steps=3,
+    tools=None,
+    directive_system=None,
+    directive_kwargs={},
+    return_branch=False,
+    **kwargs
+):
+    out_, outs, reasons, actions, action_responses = "", [], [], [], []
+    if branch is not None:
+        out_ = await plan(sentence, branch=branch, instruction=instruction, num_steps=num_steps, **kwargs)
+    else:
+        out_, branch = await plan(sentence, instruction=instruction, branch=branch, num_steps=num_steps, return_branch=True, **kwargs)
+    
+    _process_tools(tools, branch)
+    
+    for i in range(len(out_.plan)):
+        _out = await react(branch=branch, system=directive_system, instruction=out_.plan[f"step_{i+1}"], **directive_kwargs)
+        outs.append(_out)
+        reasons.append(_out.reason)
+        actions.append(_out.actions)
+        if _out.action_needed:
+            action_responses.append(_out.action_response)
+    
+    setattr(out_, "chain_output", convert.to_list(outs))
+    setattr(out_, "chain_reason", convert.to_list(reasons))
+    setattr(out_, "chain_actions", convert.to_list(actions))
+    setattr(out_, "chain_action_response", convert.to_list(action_responses))
+
+    if return_branch:
+        return out_, branch
+    
+    return out_

lionagi/core/direct/plan.py

@@ -0,0 +1,162 @@
+# plan.py
+
+from lionagi.libs import func_call, ParseUtil
+from lionagi.integrations.bridge.pydantic_.pydantic_bridge import Field
+from ..prompt.scored_template import ScoredTemplate
+from ..branch import Branch
+
+
+class PlanTemplate(ScoredTemplate):
+    template_name: str = "default_plan"
+    sentence: str | list | dict = Field(
+        default_factory=str,
+        description="the given sentence(s) or context to generate a plan for",
+    )
+    plan: dict | str= Field(
+        default_factory=dict, description="the generated step by step plan, return as a dictionary following {step_n: {plan: ..., reason: ...}} format")
+    signature: str = "sentence -> plan"
+
+    def __init__(
+        self,
+        sentence=None,
+        instruction=None,
+        confidence_score=False,
+        reason=False,
+        num_step=3,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.sentence = sentence
+        self.task = f"Generate a {num_step}_step plan based on the given context. Instruction: {instruction}."
+
+        if reason:
+            self.output_fields.append("reason")
+
+        if confidence_score:
+            self.output_fields.append("confidence_score")
+
+
+async def _plan(
+    sentence,
+    *,
+    instruction=None,
+    branch=None,
+    confidence_score=False,
+    reason=False,
+    retries=2,
+    delay=0.5,
+    backoff_factor=2,
+    default_value=None,
+    timeout=None,
+    branch_name=None,
+    system=None,
+    messages=None,
+    service=None,
+    sender=None,
+    llmconfig=None,
+    tools=None,
+    datalogger=None,
+    persist_path=None,
+    tool_manager=None,
+    return_branch=False,
+    **kwargs,
+):
+    if "temperature" not in kwargs:
+        kwargs["temperature"] = 0.1
+
+    instruction = instruction or ""
+
+    branch = branch or Branch(
+        name=branch_name,
+        system=system,
+        messages=messages,
+        service=service,
+        sender=sender,
+        llmconfig=llmconfig,
+        tools=tools,
+        datalogger=datalogger,
+        persist_path=persist_path,
+        tool_manager=tool_manager,
+    )
+
+    _template = PlanTemplate(
+        sentence=sentence,
+        instruction=instruction,
+        confidence_score=confidence_score,
+        reason=reason,
+    )
+
+    await func_call.rcall(
+        branch.chat,
+        prompt_template=_template,
+        retries=retries,
+        delay=delay,
+        backoff_factor=backoff_factor,
+        default=default_value,
+        timeout=timeout,
+        **kwargs,
+    )
+    
+    _template.plan = ParseUtil.fuzzy_parse_json(_template.plan)
+
+    return (_template, branch) if return_branch else _template
+
+
+async def plan(
+    sentence,
+    *,
+    instruction=None,
+    num_instances=1,
+    branch=None,
+    confidence_score=False,
+    reason=False,
+    retries=2,
+    delay=0.5,
+    backoff_factor=2,
+    default_value=None,
+    timeout=None,
+    branch_name=None,
+    system=None,
+    messages=None,
+    service=None,
+    sender=None,
+    llmconfig=None,
+    tools=None,
+    datalogger=None,
+    persist_path=None,
+    tool_manager=None,
+    return_branch=False,
+    **kwargs,
+):
+    async def _inner(i=0):
+        return await _plan(
+            sentence=sentence,
+            instruction=instruction,
+            branch=branch,
+            confidence_score=confidence_score,
+            reason=reason,
+            retries=retries,
+            delay=delay,
+            backoff_factor=backoff_factor,
+            default_value=default_value,
+            timeout=timeout,
+            branch_name=branch_name,
+            system=system,
+            messages=messages,
+            service=service,
+            sender=sender,
+            llmconfig=llmconfig,
+            tools=tools,
+            datalogger=datalogger,
+            persist_path=persist_path,
+            tool_manager=tool_manager,
+            return_branch=return_branch,
+            **kwargs,
+        )
+
+    if num_instances == 1:
+        return await _inner()
+
+    elif num_instances > 1:
+        return await func_call.alcall(range(num_instances), _inner)

lionagi/core/direct/predict.py

@@ -43,14 +43,15 @@ class PredictTemplate(ScoredTemplate):
         default_factory=int, description="the number of sentences to predict"
     )
     answer: str | list = Field(
-        default_factory=str, description="the predicted sentence(s)"
+        default_factory=str, description="the predicted sentence(s) or desired output"
     )
     signature: str = "sentence -> answer"
 
     def __init__(
         self,
         sentence=None,
-        num_sentences=None,
+        instruction=None,
+        num_sentences=1,
         confidence_score=False,
         reason=False,
         **kwargs,
@@ -67,9 +68,9 @@ class PredictTemplate(ScoredTemplate):
         """
         super().__init__(**kwargs)
 
-        self.sentence = sentence
+        self.sentence = sentence or ''
         self.num_sentences = num_sentences
-        self.task = f"predict the next {self.num_sentences} sentence(s)"
+        self.task = f"follow instruction to predict the next {self.num_sentences} sentence(s). Instruction: {instruction}."
 
         if reason:
             self.output_fields.append("reason")
@@ -82,6 +83,8 @@ async def predict(
     sentence=None,
     num_sentences=1,
     confidence_score=False,
+    instruction=None,
+    branch=None,
     reason=False,
     retries=2,
     delay=0.5,
@@ -128,7 +131,7 @@ async def predict(
     Returns:
         PredictTemplate: The predict template with the predicted sentence(s).
     """
-    branch = Branch(
+    branch = branch or Branch(
         name=branch_name,
         system=system,
         messages=messages,
@@ -142,6 +145,7 @@ async def predict(
     )
 
     predict_template = PredictTemplate(
+        instruction=instruction,
         sentence=sentence,
         num_sentences=num_sentences,
         confidence_score=confidence_score,

lionagi/core/direct/react.py

@@ -3,11 +3,12 @@ from lionagi.libs import func_call, convert, AsyncUtil
 from lionagi.integrations.bridge.pydantic_.pydantic_bridge import Field
 from ..prompt.action_template import ActionedTemplate
 from ..branch import Branch
+from .utils import _process_tools
 
 
 class ReactTemplate(ActionedTemplate):
     template_name: str = "default_react"
-    sentence: str | list | dict = Field(
+    sentence: str | list | dict | None= Field(
         default_factory=str,
         description="the given sentence(s) to reason and take actions on",
     )
@@ -29,7 +30,7 @@ class ReactTemplate(ActionedTemplate):
 
 
 async def _react(
-    sentence,
+    sentence=None,
     *,
     instruction=None,
     branch=None,
@@ -58,6 +59,9 @@ async def _react(
 
     instruction = instruction or ""
 
+    if branch and tools:
+        _process_tools(tools, branch)
+        
     branch = branch or Branch(
         name=branch_name,
         system=system,
@@ -109,7 +113,7 @@ async def _react(
 
 
 async def react(
-    sentence,
+    sentence=None,
     *,
     instruction=None,
     num_instances=1,
@@ -159,7 +163,7 @@ async def react(
             return_branch=return_branch,
             **kwargs,
         )
-        
+
     if num_instances == 1:
         return await _inner()
 

lionagi/core/direct/select.py

@@ -39,9 +39,7 @@ class SelectTemplate(ScoredTemplate):
     answer: Enum | str = Field(
         default_factory=str, description="selection from given choices"
     )
-    choices: list = Field(
-        default_factory=list, description="the given choices"
-    )
+    choices: list = Field(default_factory=list, description="the given choices")
 
     signature: str = "sentence -> answer"
 

lionagi/core/direct/sentiment.py

@@ -0,0 +1 @@
+# TODO: sentiment analysis

lionagi/core/direct/utils.py

@@ -1,3 +1,6 @@
+from typing import Callable
+from ..tool import func_to_tool
+from ..schema import Tool
 # import contextlib
 # from lionagi.libs import ParseUtil, StringMatch, convert, func_call
 
@@ -85,3 +88,20 @@
 #         return _out
 
 #     return out_ if len(out_) > 1 else out_[0]
+
+
+def _process_tools(tool_obj, branch):
+    if isinstance(tool_obj, Callable):
+        _process_tool(tool_obj, branch)
+    else:
+        for i in tool_obj:
+            _process_tool(i, branch)
+
+
+def _process_tool(tool_obj, branch):
+    if isinstance(tool_obj, Tool) and tool_obj.schema_["function"]["name"] not in branch.tool_manager.registry:
+        branch.register_tools(tool_obj)
+    if isinstance(tool_obj, Callable):
+        tool = func_to_tool(tool_obj)[0]
+        if tool.schema_["function"]["name"] not in branch.tool_manager.registry:
+            branch.register_tools(tool)

lionagi/integrations/bridge/llamaindex_/node_parser.py

@@ -29,19 +29,18 @@ def get_llama_index_node_parser(node_parser: Any):
     import llama_index.core.node_parser
 
     if not isinstance(node_parser, str) and not issubclass(node_parser, NodeParser):
-        raise TypeError(f"node_parser must be a string or NodeParser.")
+        raise TypeError("node_parser must be a string or NodeParser.")
 
     if isinstance(node_parser, str):
         if node_parser == "CodeSplitter":
             SysUtil.check_import("tree_sitter_languages")
 
         try:
-            parser = getattr(llama_index.core.node_parser, node_parser)
-            return parser
+            return getattr(llama_index.core.node_parser, node_parser)
         except Exception as e:
             raise AttributeError(
                 f"llama_index_core has no such attribute:" f" {node_parser}, Error: {e}"
-            )
+            ) from e
 
     elif isinstance(node_parser, NodeParser):
         return node_parser
@@ -75,10 +74,8 @@ def llama_index_parse_node(
         parser = get_llama_index_node_parser(node_parser)
         try:
             parser = parser(*parser_args, **parser_kwargs)
-        except:
+        except Exception:
             parser = parser.from_defaults(*parser_args, **parser_kwargs)
-        nodes = parser.get_nodes_from_documents(documents)
-        return nodes
-
+        return parser.get_nodes_from_documents(documents)
     except Exception as e:
-        raise ValueError(f"Failed to parse. Error: {e}")
+        raise ValueError(f"Failed to parse. Error: {e}") from e

lionagi/integrations/chunker/chunk.py

@@ -0,0 +1,175 @@
+from typing import Union, Callable
+
+from lionagi.libs import func_call
+from lionagi.core.schema import DataNode
+from ..bridge.langchain_.langchain_bridge import LangchainBridge
+from ..bridge.llamaindex_.llama_index_bridge import LlamaIndexBridge
+
+
+from ..loader.load_util import ChunkerType, file_to_chunks, _datanode_parser
+
+
+def datanodes_convert(documents, chunker_type):
+
+    for i in range(len(documents)):
+        if type(documents[i]) == DataNode:
+            if chunker_type == ChunkerType.LLAMAINDEX:
+                documents[i] = documents[i].to_llama_index()
+            elif chunker_type == ChunkerType.LANGCHAIN:
+                documents[i] = documents[i].to_langchain()
+    return documents
+
+
+def text_chunker(documents, args, kwargs):
+
+    def chunk_node(node):
+        chunks = file_to_chunks(node.to_dict(), *args, **kwargs)
+        func_call.lcall(chunks, lambda chunk: chunk.pop("node_id"))
+        return [DataNode.from_obj({**chunk}) for chunk in chunks]
+
+    return [chunk_node(doc) for doc in documents]
+
+
+def chunk(
+    documents,
+    chunker,
+    chunker_type=ChunkerType.PLAIN,
+    chunker_args=None,
+    chunker_kwargs=None,
+    chunking_kwargs=None,
+    documents_convert_func=None,
+    to_datanode: bool | Callable = True,
+):
+
+    if chunker_args is None:
+        chunker_args = []
+    if chunker_kwargs is None:
+        chunker_kwargs = {}
+    if chunking_kwargs is None:
+        chunking_kwargs = {}
+
+    if chunker_type == ChunkerType.PLAIN:
+        return chunk_funcs[ChunkerType.PLAIN](
+            documents, chunker, chunker_args, chunker_kwargs
+        )
+
+    elif chunker_type == ChunkerType.LANGCHAIN:
+        return chunk_funcs[ChunkerType.LANGCHAIN](
+            documents,
+            documents_convert_func,
+            chunker,
+            chunker_args,
+            chunker_kwargs,
+            to_datanode,
+        )
+
+    elif chunker_type == ChunkerType.LLAMAINDEX:
+        return chunk_funcs[ChunkerType.LLAMAINDEX](
+            documents,
+            documents_convert_func,
+            chunker,
+            chunker_args,
+            chunker_kwargs,
+            to_datanode,
+        )
+
+    elif chunker_type == ChunkerType.SELFDEFINED:
+        return chunk_funcs[ChunkerType.SELFDEFINED](
+            documents,
+            chunker,
+            chunker_args,
+            chunker_kwargs,
+            chunking_kwargs,
+            to_datanode,
+        )
+
+    else:
+        raise ValueError(
+            f"{chunker_type} is not supported. Please choose from {list(ChunkerType)}"
+        )
+
+
+def _self_defined_chunker(
+    documents,
+    chunker,
+    chunker_args,
+    chunker_kwargs,
+    chunking_kwargs,
+    to_datanode: bool | Callable,
+):
+    try:
+        splitter = chunker(*chunker_args, **chunker_kwargs)
+        nodes = splitter.split(documents, **chunking_kwargs)
+    except Exception as e:
+        raise ValueError(
+            f"Self defined chunker {chunker} is not valid. Error: {e}"
+        ) from e
+
+    if isinstance(to_datanode, bool) and to_datanode is True:
+        raise ValueError("Please define a valid parser to DataNode.")
+    elif isinstance(to_datanode, Callable):
+        nodes = _datanode_parser(nodes, to_datanode)
+    return nodes
+
+
+def _llama_index_chunker(
+    documents,
+    documents_convert_func,
+    chunker,
+    chunker_args,
+    chunker_kwargs,
+    to_datanode: bool | Callable,
+):
+    if documents_convert_func:
+        documents = documents_convert_func(documents, "llama_index")
+    nodes = LlamaIndexBridge.llama_index_parse_node(
+        documents, chunker, chunker_args, chunker_kwargs
+    )
+
+    if isinstance(to_datanode, bool) and to_datanode is True:
+        nodes = [DataNode.from_llama_index(i) for i in nodes]
+    elif isinstance(to_datanode, Callable):
+        nodes = _datanode_parser(nodes, to_datanode)
+    return nodes
+
+
+def _langchain_chunker(
+    documents,
+    documents_convert_func,
+    chunker,
+    chunker_args,
+    chunker_kwargs,
+    to_datanode: bool | Callable,
+):
+    if documents_convert_func:
+        documents = documents_convert_func(documents, "langchain")
+    nodes = LangchainBridge.langchain_text_splitter(
+        documents, chunker, chunker_args, chunker_kwargs
+    )
+    if isinstance(to_datanode, bool) and to_datanode is True:
+        if isinstance(documents, str):
+            nodes = [DataNode(content=i) for i in nodes]
+        else:
+            nodes = [DataNode.from_langchain(i) for i in nodes]
+    elif isinstance(to_datanode, Callable):
+        nodes = _datanode_parser(nodes, to_datanode)
+    return nodes
+
+
+def _plain_chunker(documents, chunker, chunker_args, chunker_kwargs):
+    try:
+        if chunker == "text_chunker":
+            chunker = text_chunker
+        return chunker(documents, chunker_args, chunker_kwargs)
+    except Exception as e:
+        raise ValueError(
+            f"Reader {chunker} is currently not supported. Error: {e}"
+        ) from e
+
+
+chunk_funcs = {
+    ChunkerType.PLAIN: _plain_chunker,
+    ChunkerType.LANGCHAIN: _langchain_chunker,
+    ChunkerType.LLAMAINDEX: _llama_index_chunker,
+    ChunkerType.SELFDEFINED: _self_defined_chunker,
+}

lionagi/integrations/loader/load.py

@@ -0,0 +1,152 @@
+from typing import Callable
+
+from lionagi.core.schema import DataNode
+from ..bridge.langchain_.langchain_bridge import LangchainBridge
+from ..bridge.llamaindex_.llama_index_bridge import LlamaIndexBridge
+
+from .load_util import dir_to_nodes, ReaderType, _datanode_parser
+
+
+def text_reader(args, kwargs):
+    """
+    Reads text files from a directory and converts them to DataNode instances.
+
+    Args:
+        args: Positional arguments for the dir_to_nodes function.
+        kwargs: Keyword arguments for the dir_to_nodes function.
+
+    Returns:
+        A list of DataNode instances.
+
+    Example usage:
+        >>> args = ['path/to/text/files']
+        >>> kwargs = {'file_extension': 'txt'}
+        >>> nodes = text_reader(args, kwargs)
+    """
+    return dir_to_nodes(*args, **kwargs)
+
+
+def load(
+    reader: str | Callable = "SimpleDirectoryReader",
+    input_dir=None,
+    input_files=None,
+    recursive: bool = False,
+    required_exts: list[str] = None,
+    reader_type=ReaderType.LLAMAINDEX,
+    reader_args=None,
+    reader_kwargs=None,
+    load_args=None,
+    load_kwargs=None,
+    to_datanode: bool | Callable = True,
+):
+
+    if reader_args is None:
+        reader_args = []
+    if reader_kwargs is None:
+        reader_kwargs = {}
+    if load_args is None:
+        load_args = []
+    if load_kwargs is None:
+        load_kwargs = {}
+
+    if reader_type == ReaderType.PLAIN:
+        return read_funcs[ReaderType.PLAIN](reader, reader_args, reader_kwargs)
+
+    if reader_type == ReaderType.LANGCHAIN:
+        return read_funcs[ReaderType.LANGCHAIN](
+            reader, reader_args, reader_kwargs, to_datanode
+        )
+
+    elif reader_type == ReaderType.LLAMAINDEX:
+        if input_dir is not None:
+            reader_kwargs["input_dir"] = input_dir
+        if input_files is not None:
+            reader_kwargs["input_files"] = input_files
+        if recursive:
+            reader_kwargs["recursive"] = True
+        if required_exts is not None:
+            reader_kwargs["required_exts"] = required_exts
+
+        return read_funcs[ReaderType.LLAMAINDEX](
+            reader, reader_args, reader_kwargs, load_args, load_kwargs, to_datanode
+        )
+
+    elif reader_type == ReaderType.SELFDEFINED:
+        return read_funcs[ReaderType.SELFDEFINED](
+            reader, reader_args, reader_kwargs, load_args, load_kwargs, to_datanode
+        )
+
+    else:
+        raise ValueError(
+            f"{reader_type} is not supported. Please choose from {list(ReaderType)}"
+        )
+
+
+def _plain_reader(reader, reader_args, reader_kwargs):
+    try:
+        if reader == "text_reader":
+            reader = text_reader
+        return reader(reader_args, reader_kwargs)
+    except Exception as e:
+        raise ValueError(
+            f"Reader {reader} is currently not supported. Error: {e}"
+        ) from e
+
+
+def _langchain_reader(reader, reader_args, reader_kwargs, to_datanode: bool | Callable):
+    nodes = LangchainBridge.langchain_loader(reader, reader_args, reader_kwargs)
+    if isinstance(to_datanode, bool) and to_datanode is True:
+        nodes = [DataNode.from_langchain(i) for i in nodes]
+
+    elif isinstance(to_datanode, Callable):
+        nodes = _datanode_parser(nodes, to_datanode)
+    return nodes
+
+
+def _llama_index_reader(
+    reader,
+    reader_args,
+    reader_kwargs,
+    load_args,
+    load_kwargs,
+    to_datanode: bool | Callable,
+):
+    nodes = LlamaIndexBridge.llama_index_read_data(
+        reader, reader_args, reader_kwargs, load_args, load_kwargs
+    )
+    if isinstance(to_datanode, bool) and to_datanode is True:
+        nodes = [DataNode.from_llama_index(i) for i in nodes]
+    elif isinstance(to_datanode, Callable):
+        nodes = _datanode_parser(nodes, to_datanode)
+    return nodes
+
+
+def _self_defined_reader(
+    reader,
+    reader_args,
+    reader_kwargs,
+    load_args,
+    load_kwargs,
+    to_datanode: bool | Callable,
+):
+    try:
+        loader = reader(*reader_args, **reader_kwargs)
+        nodes = loader.load(*load_args, **load_kwargs)
+    except Exception as e:
+        raise ValueError(
+            f"Self defined reader {reader} is not valid. Error: {e}"
+        ) from e
+
+    if isinstance(to_datanode, bool) and to_datanode is True:
+        raise ValueError("Please define a valid parser to DataNode.")
+    elif isinstance(to_datanode, Callable):
+        nodes = _datanode_parser(nodes, to_datanode)
+    return nodes
+
+
+read_funcs = {
+    ReaderType.PLAIN: _plain_reader,
+    ReaderType.LANGCHAIN: _langchain_reader,
+    ReaderType.LLAMAINDEX: _llama_index_reader,
+    ReaderType.SELFDEFINED: _self_defined_reader,
+}

lionagi/integrations/loader/load_util.py

@@ -0,0 +1,266 @@
+# use utils and schema
+import math
+from enum import Enum
+from pathlib import Path
+from typing import List, Union, Dict, Any, Tuple
+
+from lionagi.libs import convert, func_call
+from lionagi.core.schema import DataNode
+
+
+class ReaderType(str, Enum):
+    PLAIN = "plain"
+    LANGCHAIN = "langchain"
+    LLAMAINDEX = "llama_index"
+    SELFDEFINED = "self_defined"
+
+
+class ChunkerType(str, Enum):
+    PLAIN = "plain"  # default
+    LANGCHAIN = "langchain"  # using langchain functions
+    LLAMAINDEX = "llama_index"  # using llamaindex functions
+    SELFDEFINED = "self_defined"  # create custom functions
+
+
+def dir_to_path(
+    dir: str, ext: str, recursive: bool = False, flatten: bool = True
+) -> List[Path]:
+    """
+    Generates a list of file paths from a directory with the given file extension.
+
+    Parameters:
+        dir (str): The directory to search for files.
+
+        ext (str): The file extension to filter by.
+
+        recursive (bool): Whether to search subdirectories recursively. Defaults to False.
+
+        flatten (bool): Whether to flatten the list. Defaults to True.
+
+    Returns:
+        List[Path]: A list of Paths to the files.
+
+    Raises:
+        ValueError: If the directory or extension is invalid.
+    """
+
+    def _dir_to_path(ext):
+        tem = "**/*" if recursive else "*"
+        return list(Path(dir).glob(tem + ext))
+
+    try:
+        return convert.to_list(
+            func_call.lcall(ext, _dir_to_path, flatten=True), flatten=flatten
+        )
+    except:
+        raise ValueError("Invalid directory or extension, please check the path")
+
+
+def dir_to_nodes(
+    dir: str,
+    ext: Union[List[str], str],
+    recursive: bool = False,
+    flatten: bool = True,
+    clean_text: bool = True,
+) -> List[DataNode]:
+    """
+    Converts directory contents into DataNode objects based on specified file extensions.
+
+    This function first retrieves a list of file paths from the specified directory, matching the given file extension. It then reads the content of these files, optionally cleaning the text, and converts each file's content into a DataNode object.
+
+    Parameters:
+        dir (str): The directory path from which to read files.
+        ext: The file extension(s) to include. Can be a single string or a list/tuple of strings.
+        recursive (bool, optional): If True, the function searches for files recursively in subdirectories. Defaults to False.
+        flatten (bool, optional): If True, flattens the directory structure in the returned paths. Defaults to True.
+        clean_text (bool, optional): If True, cleans the text read from files. Defaults to True.
+
+    Returns:
+        list: A list of DataNode objects created from the files in the specified directory.
+
+    Example:
+        nodes = dir_to_nodes("/path/to/dir", ".txt", recursive=True)
+        # This would read all .txt files in /path/to/dir and its subdirectories,
+        # converting them into DataNode objects.
+    """
+
+    path_list = dir_to_path(dir, ext, recursive, flatten)
+    files_info = func_call.lcall(path_list, read_text, clean=clean_text)
+    return func_call.lcall(files_info, lambda x: DataNode(content=x[0], metadata=x[1]))
+
+
+def chunk_text(
+    input: str, chunk_size: int, overlap: float, threshold: int
+) -> List[Union[str, None]]:
+    """
+    Chunks the input text into smaller parts, with optional overlap and threshold for final chunk.
+
+    Parameters:
+        input (str): The input text to chunk.
+
+        chunk_size (int): The size of each chunk.
+
+        overlap (float): The amount of overlap between chunks.
+
+        threshold (int): The minimum size of the final chunk.
+
+    Returns:
+        List[Union[str, None]]: A list of text chunks.
+
+    Raises:
+        ValueError: If an error occurs during chunking.
+    """
+
+    def _chunk_n1():
+        return [input]
+
+    def _chunk_n2():
+        chunks = []
+        chunks.append(input[: chunk_size + overlap_size])
+
+        if len(input) - chunk_size > threshold:
+            chunks.append(input[chunk_size - overlap_size :])
+        else:
+            return _chunk_n1()
+
+        return chunks
+
+    def _chunk_n3():
+        chunks = []
+        chunks.append(input[: chunk_size + overlap_size])
+        for i in range(1, n_chunks - 1):
+            start_idx = chunk_size * i - overlap_size
+            end_idx = chunk_size * (i + 1) + overlap_size
+            chunks.append(input[start_idx:end_idx])
+
+        if len(input) - chunk_size * (n_chunks - 1) > threshold:
+            chunks.append(input[chunk_size * (n_chunks - 1) - overlap_size :])
+        else:
+            chunks[-1] += input[chunk_size * (n_chunks - 1) + overlap_size :]
+
+        return chunks
+
+    try:
+        if not isinstance(input, str):
+            input = convert.to_str(input)
+
+        n_chunks = math.ceil(len(input) / chunk_size)
+        overlap_size = int(overlap / 2)
+
+        if n_chunks == 1:
+            return _chunk_n1()
+
+        elif n_chunks == 2:
+            return _chunk_n2()
+
+        elif n_chunks > 2:
+            return _chunk_n3()
+
+    except Exception as e:
+        raise ValueError(f"An error occurred while chunking the text. {e}")
+
+
+def read_text(filepath: str, clean: bool = True) -> Tuple[str, dict]:
+    """
+    Reads text from a file and optionally cleans it, returning the content and metadata.
+
+    Parameters:
+        filepath (str): The path to the file to read.
+
+        clean (bool): Whether to clean the text by replacing certain characters. Defaults to True.
+
+    Returns:
+        Tuple[str, dict]: A tuple containing the content and metadata of the file.
+
+    Raises:
+        FileNotFoundError: If the file cannot be found.
+
+        PermissionError: If there are permissions issues.
+
+        OSError: For other OS-related errors.
+    """
+
+    def _get_metadata():
+        import os
+        from datetime import datetime
+
+        file = filepath
+        size = os.path.getsize(filepath)
+        creation_date = datetime.fromtimestamp(os.path.getctime(filepath)).date()
+        modified_date = datetime.fromtimestamp(os.path.getmtime(filepath)).date()
+        last_accessed_date = datetime.fromtimestamp(os.path.getatime(filepath)).date()
+        return {
+            "file": convert.to_str(file),
+            "size": size,
+            "creation_date": str(creation_date),
+            "modified_date": str(modified_date),
+            "last_accessed_date": str(last_accessed_date),
+        }
+
+    try:
+        with open(filepath, "r") as f:
+            content = f.read()
+            if clean:
+                # Define characters to replace and their replacements
+                replacements = {"\\": " ", "\n": " ", "\t": " ", "  ": " ", "'": " "}
+                for old, new in replacements.items():
+                    content = content.replace(old, new)
+            metadata = _get_metadata()
+            return content, metadata
+    except Exception as e:
+        raise e
+
+
+def _file_to_chunks(
+    input: Dict[str, Any],
+    field: str = "content",
+    chunk_size: int = 1500,
+    overlap: float = 0.1,
+    threshold: int = 200,
+) -> List[Dict[str, Any]]:
+    try:
+        out = {key: value for key, value in input.items() if key != field} | {
+            "chunk_overlap": overlap,
+            "chunk_threshold": threshold,
+        }
+        chunks = chunk_text(
+            input[field], chunk_size=chunk_size, overlap=overlap, threshold=threshold
+        )
+        logs = []
+        for i, chunk in enumerate(chunks):
+            chunk_dict = out | {
+                "file_chunks": len(chunks),
+                "chunk_id": i + 1,
+                "chunk_size": len(chunk),
+                f"chunk_{field}": chunk,
+            }
+            logs.append(chunk_dict)
+
+        return logs
+
+    except Exception as e:
+        raise ValueError(f"An error occurred while chunking the file. {e}") from e
+
+
+# needs doing TODO
+def file_to_chunks(
+    input,
+    #    project='project',
+    #    output_dir='data/logs/sources/',
+    chunk_func=_file_to_chunks,
+    **kwargs,
+):
+    #    out_to_csv=False,
+    #    filename=None,
+    #    verbose=True,
+    #    timestamp=True,
+    #    logger=None,
+    return convert.to_list(func_call.lcall(input, chunk_func, **kwargs), flatten=True)
+
+
+def _datanode_parser(nodes, parser):
+
+    try:
+        return parser(nodes)
+    except Exception as e:
+        raise ValueError(f"DataNode parser {parser} failed. Error:{e}") from e

lionagi/version.py

@@ -1 +1 @@
-__version__ = "0.0.314"
+__version__ = "0.0.316"

{lionagi-0.0.314.dist-info → lionagi-0.0.316.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lionagi
-Version: 0.0.314
+Version: 0.0.316
 Summary: Towards automated general intelligence.
 Author: HaiyangLi
 Author-email: Haiyang Li <ocean@lionagi.ai>

{lionagi-0.0.314.dist-info → lionagi-0.0.316.dist-info}/RECORD

@@ -1,5 +1,5 @@
-lionagi/__init__.py,sha256=xIFCHZZwgI_CCwZTmwkDpw05NkyZHDiD4M_bOmat3Gs,328
-lionagi/version.py,sha256=7Y-dGG_EKByDnmHmEWunw068fpetm8RhldmHmIBM-qU,24
+lionagi/__init__.py,sha256=i6Ci7FebU2s4EVVnBFj1Dsi5RvP80JqeSqW-iripRPg,418
+lionagi/version.py,sha256=Qy3TjUfD4-wZt6o0kQq4iMofgfH5V1ldDvANr5qAYN8,24
 lionagi/core/__init__.py,sha256=M5YXmJJiLcR5QB1VRmYvec14cHT6pKvxZOEs737BmP8,322
 lionagi/core/agent/__init__.py,sha256=IVcw9yn_QMBJGBou1Atck98Us9uwPGFs-gERTv0RWew,59
 lionagi/core/agent/base_agent.py,sha256=CRUpl7Zc5d2H9uCa17nMiFAnhKM_UH5Ujo1NHo3JAxg,3371
@@ -10,12 +10,15 @@ lionagi/core/branch/branch_flow_mixin.py,sha256=yXEfpxTaJ1aoDQQnCBYx5wShn9zt1ki8
 lionagi/core/branch/executable_branch.py,sha256=Yi0t4fDNMa5UaHo15sX-zBchr5auvXOtSc0RnSpG2a8,12151
 lionagi/core/branch/util.py,sha256=os7Qp7HpDfyyCvdkbBTyIQ3AYHfzUP0M684W4XMDHN4,11813
 lionagi/core/branch/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lionagi/core/direct/__init__.py,sha256=P17UfY3tLFgu0ncxMy4FRoVDlvOGUc7jzeowN41akBk,188
-lionagi/core/direct/predict.py,sha256=tkxvN9m_XOf3SW8xTi5yanXylV8xVNRn9a8DeGd9xgs,6355
-lionagi/core/direct/react.py,sha256=7u62B8sl_kLaNfmjkGyLn_u_cUdE69PhUCLra-6l_X0,4204
+lionagi/core/direct/__init__.py,sha256=nnA1v-pXDiypH9TvnhKW2igwHTYvnNFvu9l5PMupZwk,310
+lionagi/core/direct/cot.py,sha256=gfitm365i8m7Ov2YXWJyF0HbAceloLfUqEaj7Wkw45k,2819
+lionagi/core/direct/plan.py,sha256=1qK19Q63pGMcaByxNoxjQifg3WtpuyEtzcU8ptoZKfU,4022
+lionagi/core/direct/predict.py,sha256=xjO0o2OsibL_FEHg8aC70gUanqp514AJ6L0yQwQIwLs,6535
+lionagi/core/direct/react.py,sha256=rd2iXn6_VF5WBLOruQk9saUnafpo3qgu2Kc3pFvSpjU,4318
 lionagi/core/direct/score.py,sha256=QHO11WtAUfMEdfa1K-SRyn5uqf6_N0UmyCbEJsiqcQw,10328
-lionagi/core/direct/select.py,sha256=tXGIEC0NWTOcNI8LEfwWvxN0nGeKB0jwfjwefSal_lo,6141
-lionagi/core/direct/utils.py,sha256=yqu4qv9aaU4qzUD9QovtN2m21QySzdMLmcBp5recWC0,2333
+lionagi/core/direct/select.py,sha256=pPwesq29C3JZ5J3piwjBHqjOCsEM4uChPKMGBRxtSTE,6127
+lionagi/core/direct/sentiment.py,sha256=rNwBs-I2XICOwsXxFvfM1Tlc_afsVcRCNCXCxfxm_2k,27
+lionagi/core/direct/utils.py,sha256=Jszxx7QFIgC97YOwEhDStZC1g_9Zj-RhAxWaRqfeeJ8,3003
 lionagi/core/direct/vote.py,sha256=tjs-EYDGlGB3J6d_nSl1oIuJYHtxncjustBbU_pXDqQ,2449
 lionagi/core/flow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lionagi/core/flow/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -59,16 +62,21 @@ lionagi/integrations/bridge/langchain_/langchain_bridge.py,sha256=-lnJtyf4iJEwHK
 lionagi/integrations/bridge/llamaindex_/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lionagi/integrations/bridge/llamaindex_/index.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lionagi/integrations/bridge/llamaindex_/llama_index_bridge.py,sha256=SSnLSIu06xVsghTios01QKGrc2xCq3oC8fVNGO4QkF8,5064
-lionagi/integrations/bridge/llamaindex_/node_parser.py,sha256=cDbMqPT9cAXGVPCOPE5HrXxtTZ5ko2Wiw6m8hIaWoGA,3486
+lionagi/integrations/bridge/llamaindex_/node_parser.py,sha256=d8SPD6EMf9bZ6824jjeZOWmwm7BHBZQ0qGq1JnsKh9k,3458
 lionagi/integrations/bridge/llamaindex_/reader.py,sha256=VxdTk5h3a3_5RQzN15q75XGli52umhz9gLUrKk1Sg90,8235
 lionagi/integrations/bridge/llamaindex_/textnode.py,sha256=OszGitHZ36zbG4DCGWUnSV6EO7wChEH2VA5M50iBojs,2322
 lionagi/integrations/bridge/pydantic_/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lionagi/integrations/bridge/pydantic_/pydantic_bridge.py,sha256=TVh7sQX_LKERUvv1nxsA2JICY1S6ptPr3qFqzgHfGCY,87
+lionagi/integrations/chunker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lionagi/integrations/chunker/chunk.py,sha256=huLtaLVzs2Py6F9tFHdU0o67JncOiX4WfmmmFccbcgI,5107
 lionagi/integrations/config/__init__.py,sha256=zzQGZe3H5vofcNWSjjoqe_gqHpCO8Yl7FefmrUpLqnw,133
 lionagi/integrations/config/mlx_configs.py,sha256=xbostqjnk3aAN-qKyC54YBprHPA38C8YDevXMMEHXWY,44
 lionagi/integrations/config/oai_configs.py,sha256=aoKx91Nv5eQU2F8v8EsALXQCEEfy3sfCgUYjCYEGJPU,2754
 lionagi/integrations/config/ollama_configs.py,sha256=Np73p86bTJtxYwAj3lr5l8V9IMu7rHJPdyzHEqyzI2Q,17
 lionagi/integrations/config/openrouter_configs.py,sha256=Sz4IHrriXoB8RQ0Pj23Q13Ps4AnZ0BWrh5DhL18NLwQ,1379
+lionagi/integrations/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lionagi/integrations/loader/load.py,sha256=XYG93HzxKvnWIrPq-MotY6JTiuB_11rSADjOq_OdeiM,4582
+lionagi/integrations/loader/load_util.py,sha256=7VyoQ3z4kYscjqOGFYrhPq1U0yPIR9NHWCESfv2gmp4,8369
 lionagi/integrations/provider/__init__.py,sha256=MJhnq2tkBRcMH-3utc0G-Co20MmsxLBbp3fUwHrJGQ8,198
 lionagi/integrations/provider/litellm.py,sha256=l3sTtIPDeM_9soTLj9gpVfFWWDzFfIZ7rbVcuzeql2w,1181
 lionagi/integrations/provider/mistralai.py,sha256=G-StbfrnUcWZvl0eRby6CZYXxmJf6BRMFzDaix-brmU,7
@@ -106,8 +114,8 @@ lionagi/tests/test_libs/test_func_call.py,sha256=xvs19YBNxqh3RbWLjQXY19L06b1_uZY
 lionagi/tests/test_libs/test_nested.py,sha256=eEcE4BXJEkjoPZsd9-0rUxOJHjmu8W2hgVClUTwXEFY,13106
 lionagi/tests/test_libs/test_parse.py,sha256=aa74kfOoJwDU7L7-59EcgBGYc5-OtafPIP2oGTI3Zrk,6814
 lionagi/tests/test_libs/test_sys_util.py,sha256=Y-9jxLGxgbFNp78Z0PJyGUjRROMuRAG3Vo3i5LAH8Hs,7849
-lionagi-0.0.314.dist-info/LICENSE,sha256=vfczrx-xFNkybZ7Ef-lGUnA1Vorky6wL4kwb1Fd5o3I,1089
-lionagi-0.0.314.dist-info/METADATA,sha256=Uk-S68qitcwmCkrAXqkrOZIKT8FE3gotvmBcjKvPT4U,7934
-lionagi-0.0.314.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-lionagi-0.0.314.dist-info/top_level.txt,sha256=szvch_d2jE1Lu9ZIKsl26Ll6BGfYfbOgt5lm-UpFSo4,8
-lionagi-0.0.314.dist-info/RECORD,,
+lionagi-0.0.316.dist-info/LICENSE,sha256=vfczrx-xFNkybZ7Ef-lGUnA1Vorky6wL4kwb1Fd5o3I,1089
+lionagi-0.0.316.dist-info/METADATA,sha256=StvgG7rD-1_VdC1-ftaT_DKrO4VZP8QvRl29xPnnIAM,7934
+lionagi-0.0.316.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+lionagi-0.0.316.dist-info/top_level.txt,sha256=szvch_d2jE1Lu9ZIKsl26Ll6BGfYfbOgt5lm-UpFSo4,8
+lionagi-0.0.316.dist-info/RECORD,,