lionagi 0.1.2__py3-none-any.whl → 0.2.0__py3-none-any.whl

lionagi/experimental/compressor/util.py

@@ -0,0 +1,70 @@
+# import asyncio
+# from lionagi import alcall
+# from lionagi.libs.ln_convert import to_list
+# import numpy as np
+
+# def split_into_segments(text):
+#     segments = text.split(".")  # Splitting by period followed by a space
+#     return [segment.strip() for segment in segments if segment]
+
+# # Tokenize the segment
+# def tokenize(segment):
+#     tokens = segment.split()  # Simple space-based tokenization
+#     return tokens
+
+# async def calculate_perplexity(system_msg: str, imodel, tokens, initial_context=None, **kwargs):
+#     _tasks = []
+#     _context = initial_context or ""
+#     for i in range(len(tokens)):
+#         _context += " " + tokens[i]
+#         messages = [
+#             {"role": "system", "content": system_msg},
+#             {"role": "user", "content": _context},
+#         ]
+#         task = asyncio.create_task(
+#             imodel.call_chat_completion(
+#                 messages=messages, logprobs=True, max_tokens=1, **kwargs
+#             )
+#         )
+#         _tasks.append(task)
+
+#     results = await asyncio.gather(*_tasks)
+#     logprobs = [
+#         result[1]["choices"][0]["logprobs"]["content"] for result in results
+#     ]
+#     logprobs = to_list(logprobs, flatten=True, dropna=True)
+#     logprobs = [lprob_["logprob"] for lprob_ in logprobs]
+#     return np.exp(np.mean(logprobs))
+
+# async def rank_by_perplexity(
+#     text: str | list[str] = None,   # if list we assume they are already well split
+#     initial_text=None,
+
+#     segments,
+#     initial_text=None,
+#     cumulative=False,
+#     **kwargs
+# ):
+#     _segments = []
+#     _context = initial_text or ""
+#     _task = []
+
+#     if cumulative:
+#         for i in range(1, len(segments)):
+#             _context += " " + segments[i - 1]
+#             _segments.append(_context)
+#     else:
+#         _segments = segments
+
+#     for i in segments:
+#         _task.append(asyncio.create_task(
+#             calculate_perplexity(
+#                 self.system_msg, self.imodel, self.tokenize(i), **kwargs)
+#             )
+#         )
+#     segment_perplexities = await asyncio.gather(*_task)
+
+#     return {
+#         segment: perplexity
+#         for segment, perplexity in zip(segments, segment_perplexities)
+#     }

lionagi/experimental/directive/__init__.py

@@ -0,0 +1,19 @@
+# from ..form.predict import predict
+# from .select import select
+# from ..form.score import score
+# from ..form.react import react
+# from .vote import vote
+# from ..form.plan import plan
+# from .cot import chain_of_thoughts, chain_of_react
+
+
+# __all__ = [
+#     "predict",
+#     "select",
+#     "score",
+#     "vote",
+#     "react",
+#     "plan",
+#     "chain_of_thoughts",
+#     "chain_of_react",
+# ]

lionagi/experimental/directive/parser/base_parser.py

@@ -1,7 +1,23 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 from typing import List, Optional
 
-from lionagi.libs.ln_tokenizer import BaseToken
-from ..schema import IfNode, TryNode, ForNode
+from lionagi.experimental.directive.tokenizer import BaseToken
+from ..template.schema import IfNode, TryNode, ForNode
 
 
 class BaseDirectiveParser:
@@ -76,10 +92,19 @@ class BaseDirectiveParser:
         self.next_token()
 
     def skip_semicolon(self):
+        """Skips a semicolon token if it is the current token."""
         if self.current_token and self.current_token.value == ";":
             self.next_token()
 
     def parse_expression(self):
+        """Parses an expression until a semicolon is encountered.
+
+        Returns:
+            str: The parsed expression as a string.
+
+        Raises:
+            SyntaxError: If a semicolon is not found at the end of the expression.
+        """
         expr = ""
         while self.current_token and self.current_token.value != ";":
             expr += self.current_token.value + " "
@@ -91,6 +116,11 @@ class BaseDirectiveParser:
         return expr.strip()
 
     def parse_if_block(self):
+        """Parses a block of statements for an IF condition.
+
+        Returns:
+            list: The parsed block of statements as a list of strings.
+        """
         block = []
         # Parse the block until 'ELSE', 'ENDIF', ensuring not to include semicolons as part of the block
         while self.current_token and self.current_token.value not in ("ENDIF", "ELSE"):
@@ -103,6 +133,14 @@ class BaseDirectiveParser:
         return block
 
     def parse_if_statement(self):
+        """Parses an IF statement.
+
+        Returns:
+            IfNode: The parsed IF statement as an IfNode object.
+
+        Raises:
+            SyntaxError: If the IF statement is not properly formed.
+        """
         if self.current_token.type != "KEYWORD" or self.current_token.value != "IF":
             raise SyntaxError("Expected IF statement")
         self.next_token()  # Skip 'IF'
@@ -125,6 +163,14 @@ class BaseDirectiveParser:
         return IfNode(condition, true_block, false_block)
 
     def parse_for_statement(self):
+        """Parses a FOR statement.
+
+        Returns:
+            ForNode: The parsed FOR statement as a ForNode object.
+
+        Raises:
+            SyntaxError: If the FOR statement is not properly formed.
+        """
         if self.current_token.type != "KEYWORD" or self.current_token.value != "FOR":
             raise SyntaxError("Expected FOR statement")
         self.next_token()  # Skip 'FOR'
@@ -153,6 +199,11 @@ class BaseDirectiveParser:
         return ForNode(iterator, collection, true_block)
 
     def parse_for_block(self):
+        """Parses a block of statements for a FOR loop.
+
+        Returns:
+            list: The parsed block of statements as a list of strings.
+        """
         block = []
         # Skip initial 'DO' if present
         if self.current_token and self.current_token.value == "DO":
@@ -173,6 +224,14 @@ class BaseDirectiveParser:
         return block
 
     def parse_try_statement(self):
+        """Parses a TRY statement.
+
+        Returns:
+            TryNode: The parsed TRY statement as a TryNode object.
+
+        Raises:
+            SyntaxError: If the TRY statement is not properly formed.
+        """
         if self.current_token.type != "KEYWORD" or self.current_token.value != "TRY":
             raise SyntaxError("Expected TRY statement")
         self.next_token()  # Skip 'TRY'
@@ -196,6 +255,14 @@ class BaseDirectiveParser:
         return TryNode(try_block, except_block)
 
     def parse_try_block(self, stop_keyword):
+        """Parses a block of statements for a TRY or EXCEPT clause.
+
+        Args:
+            stop_keyword (str): The keyword that indicates the end of the block.
+
+        Returns:
+            list: The parsed block of statements as a list of strings.
+        """
         block = []
         while self.current_token and self.current_token.value != stop_keyword:
             if self.current_token.value == "DO":

lionagi/experimental/directive/{template_ → template}/base_template.py

@@ -1,10 +1,26 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 from typing import Any, Dict
 import re
 
 from ..evaluator.base_evaluator import BaseEvaluator
 
 
-class BaseDirectiveTemplate:
+class DirectiveTemplate:
     """Enhanced base template class for processing templates with conditionals and loops."""
 
     def __init__(self, template_str: str):

lionagi/{libs/ln_tokenizer.py → experimental/directive/tokenizer.py}

@@ -1,3 +1,19 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 import re
 
 

lionagi/experimental/{directive/evaluator → evaluator}/ast_evaluator.py

@@ -1,3 +1,19 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 import ast
 import operator
 

lionagi/experimental/{directive/evaluator → evaluator}/base_evaluator.py

@@ -1,3 +1,19 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 import ast
 import operator
 from typing import Any, Dict, Tuple, Callable

lionagi/experimental/knowledge/base.py

@@ -0,0 +1,10 @@
+from ..collections.abc import Component, Field
+from ..collections import Pile, pile
+from ..generic import Node
+
+
+class Knowledge(Component):
+
+    knowledge_base: Pile[Node] = Field(
+        default_factory=pile,
+    )

lionagi/experimental/strategies/base.py

@@ -0,0 +1 @@
+# TODO

lionagi/integrations/bridge/langchain_/documents.py

@@ -28,6 +28,10 @@ def to_langchain_document(datanode: T, **kwargs: Any) -> Any:
     SysUtil.change_dict_key(dnode, old_key="content", new_key="page_content")
     SysUtil.change_dict_key(dnode, old_key="lc_id", new_key="id_")
     dnode = {**dnode, **kwargs}
+    dnode = {k: v for k, v in dnode.items() if v is not None}
+    if "page_content" not in dnode:
+        dnode["page_content"] = ""
+
     return LangchainDocument(**dnode)
 
 

lionagi/integrations/bridge/llamaindex_/index.py

@@ -0,0 +1,30 @@
+class LlamaIndex:
+
+    @classmethod
+    def index(
+        cls,
+        nodes,
+        llm_obj=None,
+        llm_class=None,
+        llm_kwargs=None,
+        index_type=None,
+        **kwargs,
+    ):
+        from llama_index.core import Settings
+        from llama_index.llms.openai import OpenAI
+
+        if not llm_obj:
+            llm_class = llm_class or OpenAI
+            llm_kwargs = llm_kwargs or {}
+            if "model" not in llm_kwargs:
+                llm_kwargs["model"] = "gpt-4o"
+            llm_obj = llm_class(**llm_kwargs)
+
+        Settings.llm = llm_obj
+
+        if not index_type:
+            from llama_index.core import VectorStoreIndex
+
+            index_type = VectorStoreIndex
+
+        return index_type(nodes, **kwargs)

lionagi/integrations/bridge/llamaindex_/llama_index_bridge.py

@@ -100,3 +100,9 @@ class LlamaIndexBridge:
         from .reader import get_llama_index_reader
 
         return get_llama_index_reader(*args, **kwargs)
+
+    @staticmethod
+    def index(nodes, **kwargs):
+        from .index import LlamaIndex
+
+        return LlamaIndex.index(nodes, **kwargs)

lionagi/integrations/chunker/chunk.py

@@ -1,6 +1,8 @@
 from typing import Union, Callable
 
 from lionagi.libs import func_call
+from lionagi.libs.ln_convert import to_list
+from lionagi.core.collections import pile
 from lionagi.core.generic import Node
 from ..bridge.langchain_.langchain_bridge import LangchainBridge
 from ..bridge.llamaindex_.llama_index_bridge import LlamaIndexBridge
@@ -10,7 +12,20 @@ from ..loader.load_util import ChunkerType, file_to_chunks, _datanode_parser
 
 
 def datanodes_convert(documents, chunker_type):
+    """
+    Converts documents to the specified chunker type.
 
+    Args:
+        documents (list): List of documents to be converted.
+        chunker_type (ChunkerType): The type of chunker to convert the documents to.
+
+    Returns:
+        list: The converted documents.
+
+    Example usage:
+        >>> documents = [Node(...), Node(...)]
+        >>> converted_docs = datanodes_convert(documents, ChunkerType.LLAMAINDEX)
+    """
     for i in range(len(documents)):
         if type(documents[i]) == Node:
             if chunker_type == ChunkerType.LLAMAINDEX:
@@ -21,25 +36,71 @@ def datanodes_convert(documents, chunker_type):
 
 
 def text_chunker(documents, args, kwargs):
+    """
+    Chunks text documents into smaller pieces.
+
+    Args:
+        documents (list): List of documents to be chunked.
+        args (tuple): Positional arguments for the chunking function.
+        kwargs (dict): Keyword arguments for the chunking function.
+
+    Returns:
+        pile: A pile of chunked Node instances.
+
+    Example usage:
+        >>> documents = [Node(...), Node(...)]
+        >>> chunked_docs = text_chunker(documents, args, kwargs)
+    """
 
     def chunk_node(node):
         chunks = file_to_chunks(node.to_dict(), *args, **kwargs)
-        func_call.lcall(chunks, lambda chunk: chunk.pop("node_id"))
+        func_call.lcall(chunks, lambda chunk: chunk.pop("ln_id"))
         return [Node.from_obj({**chunk}) for chunk in chunks]
 
-    return [chunk_node(doc) for doc in documents]
+    a = to_list([chunk_node(doc) for doc in documents], flatten=True, dropna=True)
+    return pile(a)
 
 
 def chunk(
-    documents,
-    chunker,
+    docs,
+    field: str = "content",
+    chunk_size: int = 1500,
+    overlap: float = 0.1,
+    threshold: int = 200,
+    chunker="text_chunker",
     chunker_type=ChunkerType.PLAIN,
     chunker_args=None,
     chunker_kwargs=None,
     chunking_kwargs=None,
     documents_convert_func=None,
-    to_datanode: bool | Callable = True,
+    to_lion: bool | Callable = True,
 ):
+    """
+    Chunks documents using the specified chunker.
+
+    Args:
+        docs (list): List of documents to be chunked.
+        field (str, optional): The field to chunk. Defaults to "content".
+        chunk_size (int, optional): The size of each chunk. Defaults to 1500.
+        overlap (float, optional): The overlap between chunks. Defaults to 0.1.
+        threshold (int, optional): The threshold for chunking. Defaults to 200.
+        chunker (str, optional): The chunker function or its name. Defaults to "text_chunker".
+        chunker_type (ChunkerType, optional): The type of chunker to use. Defaults to ChunkerType.PLAIN.
+        chunker_args (list, optional): Positional arguments for the chunker function. Defaults to None.
+        chunker_kwargs (dict, optional): Keyword arguments for the chunker function. Defaults to None.
+        chunking_kwargs (dict, optional): Additional keyword arguments for chunking. Defaults to None.
+        documents_convert_func (Callable, optional): Function to convert documents. Defaults to None.
+        to_lion (bool | Callable, optional): Whether to convert the data to Node instances or a custom parser. Defaults to True.
+
+    Returns:
+        pile: A pile of chunked Node instances.
+
+    Raises:
+        ValueError: If the chunker_type is not supported.
+
+    Example usage:
+        >>> chunked_docs = chunk(docs, field='text', chunk_size=1000, overlap=0.2)
+    """
 
     if chunker_args is None:
         chunker_args = []
@@ -49,38 +110,42 @@ def chunk(
         chunking_kwargs = {}
 
     if chunker_type == ChunkerType.PLAIN:
+        chunker_kwargs["field"] = field
+        chunker_kwargs["chunk_size"] = chunk_size
+        chunker_kwargs["overlap"] = overlap
+        chunker_kwargs["threshold"] = threshold
         return chunk_funcs[ChunkerType.PLAIN](
-            documents, chunker, chunker_args, chunker_kwargs
+            docs, chunker, chunker_args, chunker_kwargs
         )
 
     elif chunker_type == ChunkerType.LANGCHAIN:
         return chunk_funcs[ChunkerType.LANGCHAIN](
-            documents,
+            docs,
             documents_convert_func,
             chunker,
             chunker_args,
             chunker_kwargs,
-            to_datanode,
+            to_lion,
         )
 
     elif chunker_type == ChunkerType.LLAMAINDEX:
         return chunk_funcs[ChunkerType.LLAMAINDEX](
-            documents,
+            docs,
             documents_convert_func,
             chunker,
             chunker_args,
             chunker_kwargs,
-            to_datanode,
+            to_lion,
         )
 
     elif chunker_type == ChunkerType.SELFDEFINED:
         return chunk_funcs[ChunkerType.SELFDEFINED](
-            documents,
+            docs,
             chunker,
             chunker_args,
             chunker_kwargs,
             chunking_kwargs,
-            to_datanode,
+            to_lion,
         )
 
     else:
@@ -95,8 +160,28 @@ def _self_defined_chunker(
     chunker_args,
     chunker_kwargs,
     chunking_kwargs,
-    to_datanode: bool | Callable,
+    to_lion: bool | Callable,
 ):
+    """
+    Chunks documents using a self-defined chunker.
+
+    Args:
+        documents (list): List of documents to be chunked.
+        chunker (str | Callable): The chunker function or its name.
+        chunker_args (list): Positional arguments for the chunker function.
+        chunker_kwargs (dict): Keyword arguments for the chunker function.
+        chunking_kwargs (dict): Additional keyword arguments for chunking.
+        to_lion (bool | Callable): Whether to convert the data to Node instances or a custom parser.
+
+    Returns:
+        pile: A pile of chunked Node instances or custom parsed nodes.
+
+    Raises:
+        ValueError: If the self-defined chunker is not valid.
+
+    Example usage:
+        >>> chunked_docs = _self_defined_chunker(docs, custom_chunker, ['arg1'], {'key': 'value'}, {}, custom_parser)
+    """
     try:
         splitter = chunker(*chunker_args, **chunker_kwargs)
         nodes = splitter.split(documents, **chunking_kwargs)
@@ -105,10 +190,10 @@ def _self_defined_chunker(
             f"Self defined chunker {chunker} is not valid. Error: {e}"
         ) from e
 
-    if isinstance(to_datanode, bool) and to_datanode is True:
+    if isinstance(to_lion, bool) and to_lion is True:
         raise ValueError("Please define a valid parser to Node.")
-    elif isinstance(to_datanode, Callable):
-        nodes = _datanode_parser(nodes, to_datanode)
+    elif isinstance(to_lion, Callable):
+        nodes = _datanode_parser(nodes, to_lion)
     return nodes
 
 
@@ -118,18 +203,35 @@ def _llama_index_chunker(
     chunker,
     chunker_args,
     chunker_kwargs,
-    to_datanode: bool | Callable,
+    to_lion: bool | Callable,
 ):
+    """
+    Chunks documents using a LlamaIndex chunker.
+
+    Args:
+        documents (list): List of documents to be chunked.
+        documents_convert_func (Callable): Function to convert documents.
+        chunker (str | Callable): The chunker function or its name.
+        chunker_args (list): Positional arguments for the chunker function.
+        chunker_kwargs (dict): Keyword arguments for the chunker function.
+        to_lion (bool | Callable): Whether to convert the data to Node instances or a custom parser.
+
+    Returns:
+        pile: A pile of chunked Node instances or custom parsed nodes.
+
+    Example usage:
+        >>> chunked_docs = _llama_index_chunker(docs, convert_func, llama_chunker, ['arg1'], {'key': 'value'}, True)
+    """
     if documents_convert_func:
         documents = documents_convert_func(documents, "llama_index")
     nodes = LlamaIndexBridge.llama_index_parse_node(
         documents, chunker, chunker_args, chunker_kwargs
     )
 
-    if isinstance(to_datanode, bool) and to_datanode is True:
+    if isinstance(to_lion, bool) and to_lion is True:
         nodes = [Node.from_llama_index(i) for i in nodes]
-    elif isinstance(to_datanode, Callable):
-        nodes = _datanode_parser(nodes, to_datanode)
+    elif isinstance(to_lion, Callable):
+        nodes = _datanode_parser(nodes, to_lion)
     return nodes
 
 
@@ -139,24 +241,59 @@ def _langchain_chunker(
     chunker,
     chunker_args,
     chunker_kwargs,
-    to_datanode: bool | Callable,
+    to_lion: bool | Callable,
 ):
+    """
+    Chunks documents using a Langchain chunker.
+
+    Args:
+        documents (list): List of documents to be chunked.
+        documents_convert_func (Callable): Function to convert documents.
+        chunker (str | Callable): The chunker function or its name.
+        chunker_args (list): Positional arguments for the chunker function.
+        chunker_kwargs (dict): Keyword arguments for the chunker function.
+        to_lion (bool | Callable): Whether to convert the data to Node instances or a custom parser.
+
+    Returns:
+        pile: A pile of chunked Node instances or custom parsed nodes.
+
+    Example usage:
+        >>> chunked_docs = _langchain_chunker(docs, convert_func, langchain_chunker, ['arg1'], {'key': 'value'}, True)
+    """
     if documents_convert_func:
         documents = documents_convert_func(documents, "langchain")
     nodes = LangchainBridge.langchain_text_splitter(
         documents, chunker, chunker_args, chunker_kwargs
     )
-    if isinstance(to_datanode, bool) and to_datanode is True:
+    if isinstance(to_lion, bool) and to_lion is True:
         if isinstance(documents, str):
             nodes = [Node(content=i) for i in nodes]
         else:
             nodes = [Node.from_langchain(i) for i in nodes]
-    elif isinstance(to_datanode, Callable):
-        nodes = _datanode_parser(nodes, to_datanode)
+    elif isinstance(to_lion, Callable):
+        nodes = _datanode_parser(nodes, to_lion)
     return nodes
 
 
 def _plain_chunker(documents, chunker, chunker_args, chunker_kwargs):
+    """
+    Chunks documents using a plain chunker.
+
+    Args:
+        documents (list): List of documents to be chunked.
+        chunker (str | Callable): The chunker function or its name.
+        chunker_args (list): Positional arguments for the chunker function.
+        chunker_kwargs (dict): Keyword arguments for the chunker function.
+
+    Returns:
+        pile: A pile of chunked Node instances.
+
+    Raises:
+        ValueError: If the chunker is not supported.
+
+    Example usage:
+        >>> chunked_docs = _plain_chunker(docs, 'text_chunker', ['arg1'], {'key': 'value'})
+    """
     try:
         if chunker == "text_chunker":
             chunker = text_chunker