lionagi 0.1.1__py3-none-any.whl → 0.2.0__py3-none-any.whl

lionagi/experimental/compressor/util.py

@@ -0,0 +1,70 @@
+# import asyncio
+# from lionagi import alcall
+# from lionagi.libs.ln_convert import to_list
+# import numpy as np
+
+# def split_into_segments(text):
+#     segments = text.split(".")  # Splitting by period followed by a space
+#     return [segment.strip() for segment in segments if segment]
+
+# # Tokenize the segment
+# def tokenize(segment):
+#     tokens = segment.split()  # Simple space-based tokenization
+#     return tokens
+
+# async def calculate_perplexity(system_msg: str, imodel, tokens, initial_context=None, **kwargs):
+#     _tasks = []
+#     _context = initial_context or ""
+#     for i in range(len(tokens)):
+#         _context += " " + tokens[i]
+#         messages = [
+#             {"role": "system", "content": system_msg},
+#             {"role": "user", "content": _context},
+#         ]
+#         task = asyncio.create_task(
+#             imodel.call_chat_completion(
+#                 messages=messages, logprobs=True, max_tokens=1, **kwargs
+#             )
+#         )
+#         _tasks.append(task)
+
+#     results = await asyncio.gather(*_tasks)
+#     logprobs = [
+#         result[1]["choices"][0]["logprobs"]["content"] for result in results
+#     ]
+#     logprobs = to_list(logprobs, flatten=True, dropna=True)
+#     logprobs = [lprob_["logprob"] for lprob_ in logprobs]
+#     return np.exp(np.mean(logprobs))
+
+# async def rank_by_perplexity(
+#     text: str | list[str] = None,   # if list we assume they are already well split
+#     initial_text=None,
+
+#     segments,
+#     initial_text=None,
+#     cumulative=False,
+#     **kwargs
+# ):
+#     _segments = []
+#     _context = initial_text or ""
+#     _task = []
+
+#     if cumulative:
+#         for i in range(1, len(segments)):
+#             _context += " " + segments[i - 1]
+#             _segments.append(_context)
+#     else:
+#         _segments = segments
+
+#     for i in segments:
+#         _task.append(asyncio.create_task(
+#             calculate_perplexity(
+#                 self.system_msg, self.imodel, self.tokenize(i), **kwargs)
+#             )
+#         )
+#     segment_perplexities = await asyncio.gather(*_task)
+
+#     return {
+#         segment: perplexity
+#         for segment, perplexity in zip(segments, segment_perplexities)
+#     }

lionagi/experimental/directive/__init__.py

@@ -0,0 +1,19 @@
+# from ..form.predict import predict
+# from .select import select
+# from ..form.score import score
+# from ..form.react import react
+# from .vote import vote
+# from ..form.plan import plan
+# from .cot import chain_of_thoughts, chain_of_react
+
+
+# __all__ = [
+#     "predict",
+#     "select",
+#     "score",
+#     "vote",
+#     "react",
+#     "plan",
+#     "chain_of_thoughts",
+#     "chain_of_react",
+# ]

lionagi/experimental/directive/parser/base_parser.py

@@ -1,7 +1,23 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 from typing import List, Optional
 
-from lionagi.libs.ln_tokenizer import BaseToken
-from ..schema import IfNode, TryNode, ForNode
+from lionagi.experimental.directive.tokenizer import BaseToken
+from ..template.schema import IfNode, TryNode, ForNode
 
 
 class BaseDirectiveParser:
@@ -76,10 +92,19 @@ class BaseDirectiveParser:
         self.next_token()
 
     def skip_semicolon(self):
+        """Skips a semicolon token if it is the current token."""
         if self.current_token and self.current_token.value == ";":
             self.next_token()
 
     def parse_expression(self):
+        """Parses an expression until a semicolon is encountered.
+
+        Returns:
+            str: The parsed expression as a string.
+
+        Raises:
+            SyntaxError: If a semicolon is not found at the end of the expression.
+        """
         expr = ""
         while self.current_token and self.current_token.value != ";":
             expr += self.current_token.value + " "
@@ -91,6 +116,11 @@ class BaseDirectiveParser:
         return expr.strip()
 
     def parse_if_block(self):
+        """Parses a block of statements for an IF condition.
+
+        Returns:
+            list: The parsed block of statements as a list of strings.
+        """
         block = []
         # Parse the block until 'ELSE', 'ENDIF', ensuring not to include semicolons as part of the block
         while self.current_token and self.current_token.value not in ("ENDIF", "ELSE"):
@@ -103,6 +133,14 @@ class BaseDirectiveParser:
         return block
 
     def parse_if_statement(self):
+        """Parses an IF statement.
+
+        Returns:
+            IfNode: The parsed IF statement as an IfNode object.
+
+        Raises:
+            SyntaxError: If the IF statement is not properly formed.
+        """
         if self.current_token.type != "KEYWORD" or self.current_token.value != "IF":
             raise SyntaxError("Expected IF statement")
         self.next_token()  # Skip 'IF'
@@ -125,6 +163,14 @@ class BaseDirectiveParser:
         return IfNode(condition, true_block, false_block)
 
     def parse_for_statement(self):
+        """Parses a FOR statement.
+
+        Returns:
+            ForNode: The parsed FOR statement as a ForNode object.
+
+        Raises:
+            SyntaxError: If the FOR statement is not properly formed.
+        """
         if self.current_token.type != "KEYWORD" or self.current_token.value != "FOR":
             raise SyntaxError("Expected FOR statement")
         self.next_token()  # Skip 'FOR'
@@ -153,6 +199,11 @@ class BaseDirectiveParser:
         return ForNode(iterator, collection, true_block)
 
     def parse_for_block(self):
+        """Parses a block of statements for a FOR loop.
+
+        Returns:
+            list: The parsed block of statements as a list of strings.
+        """
         block = []
         # Skip initial 'DO' if present
         if self.current_token and self.current_token.value == "DO":
@@ -173,6 +224,14 @@ class BaseDirectiveParser:
         return block
 
     def parse_try_statement(self):
+        """Parses a TRY statement.
+
+        Returns:
+            TryNode: The parsed TRY statement as a TryNode object.
+
+        Raises:
+            SyntaxError: If the TRY statement is not properly formed.
+        """
         if self.current_token.type != "KEYWORD" or self.current_token.value != "TRY":
             raise SyntaxError("Expected TRY statement")
         self.next_token()  # Skip 'TRY'
@@ -196,6 +255,14 @@ class BaseDirectiveParser:
         return TryNode(try_block, except_block)
 
     def parse_try_block(self, stop_keyword):
+        """Parses a block of statements for a TRY or EXCEPT clause.
+
+        Args:
+            stop_keyword (str): The keyword that indicates the end of the block.
+
+        Returns:
+            list: The parsed block of statements as a list of strings.
+        """
         block = []
         while self.current_token and self.current_token.value != stop_keyword:
             if self.current_token.value == "DO":

lionagi/experimental/directive/{template_ → template}/base_template.py

@@ -1,10 +1,26 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 from typing import Any, Dict
 import re
 
 from ..evaluator.base_evaluator import BaseEvaluator
 
 
-class BaseDirectiveTemplate:
+class DirectiveTemplate:
     """Enhanced base template class for processing templates with conditionals and loops."""
 
     def __init__(self, template_str: str):

lionagi/{libs/ln_tokenizer.py → experimental/directive/tokenizer.py}

@@ -1,3 +1,19 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 import re
 
 

lionagi/experimental/{directive/evaluator → evaluator}/ast_evaluator.py

@@ -1,3 +1,19 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 import ast
 import operator
 

lionagi/experimental/{directive/evaluator → evaluator}/base_evaluator.py

@@ -1,3 +1,19 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 import ast
 import operator
 from typing import Any, Dict, Tuple, Callable

lionagi/experimental/knowledge/base.py

@@ -0,0 +1,10 @@
+from ..collections.abc import Component, Field
+from ..collections import Pile, pile
+from ..generic import Node
+
+
+class Knowledge(Component):
+
+    knowledge_base: Pile[Node] = Field(
+        default_factory=pile,
+    )

lionagi/experimental/strategies/base.py

@@ -0,0 +1 @@
+# TODO

lionagi/integrations/bridge/langchain_/documents.py

@@ -28,6 +28,10 @@ def to_langchain_document(datanode: T, **kwargs: Any) -> Any:
     SysUtil.change_dict_key(dnode, old_key="content", new_key="page_content")
     SysUtil.change_dict_key(dnode, old_key="lc_id", new_key="id_")
     dnode = {**dnode, **kwargs}
+    dnode = {k: v for k, v in dnode.items() if v is not None}
+    if "page_content" not in dnode:
+        dnode["page_content"] = ""
+
     return LangchainDocument(**dnode)
 
 

lionagi/integrations/bridge/llamaindex_/index.py

@@ -0,0 +1,30 @@
+class LlamaIndex:
+
+    @classmethod
+    def index(
+        cls,
+        nodes,
+        llm_obj=None,
+        llm_class=None,
+        llm_kwargs=None,
+        index_type=None,
+        **kwargs,
+    ):
+        from llama_index.core import Settings
+        from llama_index.llms.openai import OpenAI
+
+        if not llm_obj:
+            llm_class = llm_class or OpenAI
+            llm_kwargs = llm_kwargs or {}
+            if "model" not in llm_kwargs:
+                llm_kwargs["model"] = "gpt-4o"
+            llm_obj = llm_class(**llm_kwargs)
+
+        Settings.llm = llm_obj
+
+        if not index_type:
+            from llama_index.core import VectorStoreIndex
+
+            index_type = VectorStoreIndex
+
+        return index_type(nodes, **kwargs)

lionagi/integrations/bridge/llamaindex_/llama_index_bridge.py

@@ -100,3 +100,9 @@ class LlamaIndexBridge:
         from .reader import get_llama_index_reader
 
         return get_llama_index_reader(*args, **kwargs)
+
+    @staticmethod
+    def index(nodes, **kwargs):
+        from .index import LlamaIndex
+
+        return LlamaIndex.index(nodes, **kwargs)