lionagi 0.1.2__py3-none-any.whl → 0.2.1__py3-none-any.whl

lionagi/experimental/compressor/llm_compressor.py

@@ -0,0 +1,247 @@
+import asyncio
+from lionagi import alcall
+from lionagi.libs.ln_convert import to_list
+import numpy as np
+from lionagi.core.collections import iModel
+from .base import TokenCompressor
+from lionagi.libs.ln_tokenize import TokenizeUtil
+from time import time
+
+# inspired by LLMLingua, MIT License, Copyright (c) Microsoft Corporation.
+# https://github.com/microsoft/LLMLingua
+
+
+class LLMCompressor(TokenCompressor):
+
+    def __init__(
+        self,
+        imodel: iModel = None,
+        system_msg=None,
+        tokenizer=None,  # must be a callable or object with a tokenize method
+        splitter=None,  # must be a callable or object with a split/chunk/segment method
+        target_ratio=0.2,
+        n_samples=5,  # the cumulative samples to take in each perplexity calculation
+        chunk_size=64,
+        max_tokens_per_sample=80,
+        min_compression_score=0,  # (0-1) the minimum score to consider for compression, 0 means all
+        split_overlap=0,
+        split_threshold=0,
+        verbose=True,
+    ):
+        imodel = imodel or iModel(model="gpt-3.5-turbo", temperature=0.3)
+        super().__init__(imodel=imodel, tokenizer=tokenizer, splitter=splitter)
+        self.system_msg = (
+            system_msg
+            or "Concisely summarize and compress the information for storage:"
+        )
+        self.target_ratio = target_ratio
+        self.n_samples = n_samples
+        self.chunk_size = chunk_size
+        self.max_tokens_per_sample = max_tokens_per_sample
+        self.min_compression_score = min_compression_score
+        self.verbose = verbose
+        self.split_overlap = split_overlap
+        self.split_threshold = split_threshold
+
+    def tokenize(self, text, encoding_name=None, return_byte=False, **kwargs):
+        """
+        by default you can use `encoding_name` to be one of,
+        ['gpt2', 'r50k_base', 'p50k_base', 'p50k_edit', 'cl100k_base', 'o200k_base']
+
+        or you can use `encoding_model` that tiktoken supports in their mapping such as "gpt-4o"
+        """
+        if not self.tokenizer:
+            return TokenizeUtil.tokenize(
+                text,
+                encoding_model=self.imodel.iModel_name,
+                encoding_name=encoding_name,
+                return_byte=return_byte,
+            )
+
+        if hasattr(self.tokenizer, "tokenize"):
+            return self.tokenizer.tokenize(text, **kwargs)
+
+        return self.tokenizer(text, **kwargs)
+
+    def split(
+        self,
+        text,
+        chunk_size=None,
+        overlap=None,
+        threshold=None,
+        by_chars=False,
+        return_tokens=False,
+        return_byte=False,
+        **kwargs,
+    ):
+        if not self.splitter:
+            splitter = (
+                TokenizeUtil.chunk_by_chars
+                if by_chars
+                else TokenizeUtil.chunk_by_tokens
+            )
+            return splitter(
+                text,
+                chunk_size or self.chunk_size,
+                overlap or self.split_overlap,
+                threshold or self.split_threshold,
+                return_tokens=return_tokens,
+                return_byte=return_byte,
+            )
+
+        a = [
+            getattr(self.splitter, i, None)
+            for i in ["split", "chunk", "segment"]
+            if i is not None
+        ][0]
+        a = getattr(self.splitter, a)
+        return a(text, **kwargs)
+
+    async def rank_by_pplex(
+        self, items: list, initial_text=None, cumulative=False, n_samples=None, **kwargs
+    ):
+        """
+        rank a list of items according to their perplexity
+        an item can be a single token or a list of tokens
+
+        kwargs: additional arguments to pass to the model
+        """
+
+        async def _get_item_perplexity(item):
+            item = item if isinstance(item, list) else [item]
+            item = (
+                item[: self.max_tokens_per_sample]
+                if len(item) > self.max_tokens_per_sample
+                else item
+            )
+            return await self.imodel.compute_perplexity(
+                initial_context=initial_text,
+                tokens=item,
+                n_samples=n_samples or self.n_samples,
+                system_msg=self.system_msg,
+                **kwargs,
+            )
+
+        if not isinstance(items, list):
+            items = self.tokenize(items)
+
+        if len(items) == 1:
+            return [items]  # no need to rank a single item
+
+        _segments = []
+        _context = initial_text or ""
+        _task = []
+
+        if cumulative:
+            for i in items:
+                if isinstance(i, list):
+                    _context += " " + " ".join(i).strip()
+                else:
+                    _context += " " + i.strip()
+
+                _segments.append(_context)
+        else:
+            _segments = items
+
+        for i in _segments:
+            _task.append(asyncio.create_task(_get_item_perplexity(i)))
+
+        results = await asyncio.gather(*_task)
+        results = [(item, pplex) for item, pplex in zip(items, results)]
+        return sorted(results, key=lambda x: x[1]["logprobs"], reverse=True)
+
+    async def compress(
+        self,
+        text,
+        target_ratio=None,
+        initial_text=None,
+        cumulative=False,
+        split_kwargs=None,
+        split_overlap=None,
+        split_threshold=None,
+        rank_by="perplexity",
+        min_compression_score=None,
+        verbose=True,
+        **kwargs,
+    ):
+        start = time()
+        if split_kwargs is None:
+            split_kwargs = {}
+            split_kwargs["chunk_size"] = self.max_tokens_per_sample
+            split_kwargs["overlap"] = split_overlap or 0
+            split_kwargs["threshold"] = split_threshold or 0
+
+        len_tokens = len(self.tokenize(text))
+
+        items = self.split(text, return_tokens=True, **split_kwargs)
+
+        if rank_by == "perplexity":
+            ranked_items = await self.rank_by_pplex(
+                items=items, initial_text=initial_text, cumulative=cumulative, **kwargs
+            )
+
+            prompt_tokens = sum([i[1]["num_prompt_tokens"] for i in ranked_items])
+
+            num_completion_tokens = sum(
+                [i[1]["num_completion_tokens"] for i in ranked_items]
+            )
+
+            price = (
+                prompt_tokens * 0.5 / 1000000 + num_completion_tokens * 1.5 / 1000000
+            )
+
+            selected_items = self.select_by_pplex(
+                ranked_items=ranked_items,
+                target_compression_ratio=target_ratio or self.target_ratio,
+                original_length=len_tokens,
+                min_pplex=min_compression_score or self.min_compression_score,
+            )
+
+            if verbose:
+                msg = ""
+                msg += f"Original Token number: {len_tokens}\n"
+
+                def _f(i):
+                    if isinstance(i, str):
+                        i = self.tokenize(i)
+
+                    if isinstance(i, list):
+                        return len(to_list(i, dropna=True, flatten=True))
+
+                len_ = sum([_f(i) for i in selected_items])
+                msg += f"Selected Token number: {len_}\n"
+                msg += f"Token Compression Ratio: {len_ / len_tokens:.03f}\n"
+                msg += f"Compression Time: {time() - start:.04f} seconds\n"
+                msg += f"Compression Model: {self.imodel.iModel_name}\n"
+                msg += f"Compression Method: {rank_by}\n"
+                msg += f"Compression Usage: ${price:.05f}\n"
+                print(msg)
+
+            a = "".join([i.strip() for i in selected_items]).strip()
+            a = a.replace("\n\n", "")
+            return a
+
+        raise ValueError(f"Ranking method {rank_by} is not supported")
+
+    def select_by_pplex(
+        self, ranked_items, target_compression_ratio, original_length, min_pplex=None
+    ):
+        min_pplex = min_pplex or 0
+
+        desired_length = int(original_length * target_compression_ratio)
+
+        items = []
+        current_length = 0
+
+        for item, info in ranked_items:
+            if info["perplexity"] > min_pplex:
+                item = self.tokenize(item) if isinstance(item, str) else item
+                item = item if isinstance(item, list) else [item]
+                item = to_list(item, dropna=True, flatten=True)
+                if current_length + len(item) > desired_length:
+                    break
+                else:
+                    current_length += len(item)
+                    items.append("".join(item))
+
+        return items

lionagi/experimental/compressor/llm_summarizer.py

@@ -0,0 +1,61 @@
+# from lionagi.core.collections import iModel
+# from .base import TokenCompressor
+
+
+# class LLMSummarizer(TokenCompressor):
+
+#     def __init__(
+#         self, imodel: iModel = None, system_msg=None, tokenizer=None, splitter=None,
+#         max_tokens=25, target_ratio=0.3
+#     ):
+#         imodel = imodel or iModel(model="gpt-3.5-turbo", max_tokens=max_tokens)
+#         super().__init__(imodel=imodel, tokenizer=tokenizer, splitter=splitter)
+#         self.system_msg = (
+#             system_msg
+#             or "Summarize the following sentence to be concise and informative:"
+#         )
+#         self.target_ratio = target_ratio
+
+#     async def summarize_sentence(self, sentence, **kwargs):
+#         messages = [
+#             {"role": "system", "content": self.system_msg},
+#             {"role": "user", "content": sentence},
+#         ]
+#         response = await self.imodel.call_chat_completion(messages, **kwargs)
+#         return response["choices"][0]["message"]["content"]
+
+#     def tokenize(self, text):
+#         tokenize_func = self.tokenizer or tokenize
+#         return tokenize_func(text)
+
+#     def split(self, text):
+#         split_func = self.splitter or split_into_segments
+#         return split_func(text)
+
+#     # Function to enforce maximum sentence length
+#     def enforce_max_sentence_length(self, sentence, max_words=25):
+#         words = self.tokenize(sentence)
+#         if len(words) > max_words:
+#             sentence = ' '.join(words[:max_words])
+#         return sentence
+
+#     async def summarize_text(self, text, max_length_per_sentence=25, target_ratio=None, **kwargs):
+#         sentences = self.split(text)
+#         summarized = await alcall(
+#             sentences, self.summarize_sentence, **kwargs
+#         )
+#         summarized = [
+#             self.enforce_max_sentence_length(sentence, max_length_per_sentence)
+#             for sentence in summarized
+#         ]
+
+#         original_length = len(self.tokenize(text))
+#         summarized_length = len(self.tokenize(' '.join(summarized)))
+#         current_ratio = summarized_length / original_length
+
+#         target_ratio = target_ratio or self.target_ratio
+#         if current_ratio > target_ratio:
+#             words_to_remove = int((current_ratio - target_ratio) * original_length)
+#             return ' '.join(summarized[:-words_to_remove])
+
+#         return ' '.join(summarized)

lionagi/experimental/compressor/util.py

@@ -0,0 +1,70 @@
+# import asyncio
+# from lionagi import alcall
+# from lionagi.libs.ln_convert import to_list
+# import numpy as np
+
+# def split_into_segments(text):
+#     segments = text.split(".")  # Splitting by period followed by a space
+#     return [segment.strip() for segment in segments if segment]
+
+# # Tokenize the segment
+# def tokenize(segment):
+#     tokens = segment.split()  # Simple space-based tokenization
+#     return tokens
+
+# async def calculate_perplexity(system_msg: str, imodel, tokens, initial_context=None, **kwargs):
+#     _tasks = []
+#     _context = initial_context or ""
+#     for i in range(len(tokens)):
+#         _context += " " + tokens[i]
+#         messages = [
+#             {"role": "system", "content": system_msg},
+#             {"role": "user", "content": _context},
+#         ]
+#         task = asyncio.create_task(
+#             imodel.call_chat_completion(
+#                 messages=messages, logprobs=True, max_tokens=1, **kwargs
+#             )
+#         )
+#         _tasks.append(task)
+
+#     results = await asyncio.gather(*_tasks)
+#     logprobs = [
+#         result[1]["choices"][0]["logprobs"]["content"] for result in results
+#     ]
+#     logprobs = to_list(logprobs, flatten=True, dropna=True)
+#     logprobs = [lprob_["logprob"] for lprob_ in logprobs]
+#     return np.exp(np.mean(logprobs))
+
+# async def rank_by_perplexity(
+#     text: str | list[str] = None,   # if list we assume they are already well split
+#     initial_text=None,
+
+#     segments,
+#     initial_text=None,
+#     cumulative=False,
+#     **kwargs
+# ):
+#     _segments = []
+#     _context = initial_text or ""
+#     _task = []
+
+#     if cumulative:
+#         for i in range(1, len(segments)):
+#             _context += " " + segments[i - 1]
+#             _segments.append(_context)
+#     else:
+#         _segments = segments
+
+#     for i in segments:
+#         _task.append(asyncio.create_task(
+#             calculate_perplexity(
+#                 self.system_msg, self.imodel, self.tokenize(i), **kwargs)
+#             )
+#         )
+#     segment_perplexities = await asyncio.gather(*_task)
+
+#     return {
+#         segment: perplexity
+#         for segment, perplexity in zip(segments, segment_perplexities)
+#     }

lionagi/experimental/directive/__init__.py

@@ -0,0 +1,19 @@
+# from ..form.predict import predict
+# from .select import select
+# from ..form.score import score
+# from ..form.react import react
+# from .vote import vote
+# from ..form.plan import plan
+# from .cot import chain_of_thoughts, chain_of_react
+
+
+# __all__ = [
+#     "predict",
+#     "select",
+#     "score",
+#     "vote",
+#     "react",
+#     "plan",
+#     "chain_of_thoughts",
+#     "chain_of_react",
+# ]

lionagi/experimental/directive/parser/base_parser.py

@@ -1,7 +1,23 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 from typing import List, Optional
 
-from lionagi.libs.ln_tokenizer import BaseToken
-from ..schema import IfNode, TryNode, ForNode
+from lionagi.experimental.directive.tokenizer import BaseToken
+from ..template.schema import IfNode, TryNode, ForNode
 
 
 class BaseDirectiveParser:
@@ -76,10 +92,19 @@ class BaseDirectiveParser:
         self.next_token()
 
     def skip_semicolon(self):
+        """Skips a semicolon token if it is the current token."""
         if self.current_token and self.current_token.value == ";":
             self.next_token()
 
     def parse_expression(self):
+        """Parses an expression until a semicolon is encountered.
+
+        Returns:
+            str: The parsed expression as a string.
+
+        Raises:
+            SyntaxError: If a semicolon is not found at the end of the expression.
+        """
         expr = ""
         while self.current_token and self.current_token.value != ";":
             expr += self.current_token.value + " "
@@ -91,6 +116,11 @@ class BaseDirectiveParser:
         return expr.strip()
 
     def parse_if_block(self):
+        """Parses a block of statements for an IF condition.
+
+        Returns:
+            list: The parsed block of statements as a list of strings.
+        """
         block = []
         # Parse the block until 'ELSE', 'ENDIF', ensuring not to include semicolons as part of the block
         while self.current_token and self.current_token.value not in ("ENDIF", "ELSE"):
@@ -103,6 +133,14 @@ class BaseDirectiveParser:
         return block
 
     def parse_if_statement(self):
+        """Parses an IF statement.
+
+        Returns:
+            IfNode: The parsed IF statement as an IfNode object.
+
+        Raises:
+            SyntaxError: If the IF statement is not properly formed.
+        """
         if self.current_token.type != "KEYWORD" or self.current_token.value != "IF":
             raise SyntaxError("Expected IF statement")
         self.next_token()  # Skip 'IF'
@@ -125,6 +163,14 @@ class BaseDirectiveParser:
         return IfNode(condition, true_block, false_block)
 
     def parse_for_statement(self):
+        """Parses a FOR statement.
+
+        Returns:
+            ForNode: The parsed FOR statement as a ForNode object.
+
+        Raises:
+            SyntaxError: If the FOR statement is not properly formed.
+        """
         if self.current_token.type != "KEYWORD" or self.current_token.value != "FOR":
             raise SyntaxError("Expected FOR statement")
         self.next_token()  # Skip 'FOR'
@@ -153,6 +199,11 @@ class BaseDirectiveParser:
         return ForNode(iterator, collection, true_block)
 
     def parse_for_block(self):
+        """Parses a block of statements for a FOR loop.
+
+        Returns:
+            list: The parsed block of statements as a list of strings.
+        """
         block = []
         # Skip initial 'DO' if present
         if self.current_token and self.current_token.value == "DO":
@@ -173,6 +224,14 @@ class BaseDirectiveParser:
         return block
 
     def parse_try_statement(self):
+        """Parses a TRY statement.
+
+        Returns:
+            TryNode: The parsed TRY statement as a TryNode object.
+
+        Raises:
+            SyntaxError: If the TRY statement is not properly formed.
+        """
         if self.current_token.type != "KEYWORD" or self.current_token.value != "TRY":
             raise SyntaxError("Expected TRY statement")
         self.next_token()  # Skip 'TRY'
@@ -196,6 +255,14 @@ class BaseDirectiveParser:
         return TryNode(try_block, except_block)
 
     def parse_try_block(self, stop_keyword):
+        """Parses a block of statements for a TRY or EXCEPT clause.
+
+        Args:
+            stop_keyword (str): The keyword that indicates the end of the block.
+
+        Returns:
+            list: The parsed block of statements as a list of strings.
+        """
         block = []
         while self.current_token and self.current_token.value != stop_keyword:
             if self.current_token.value == "DO":

lionagi/experimental/directive/{template_ → template}/base_template.py

@@ -1,10 +1,26 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 from typing import Any, Dict
 import re
 
 from ..evaluator.base_evaluator import BaseEvaluator
 
 
-class BaseDirectiveTemplate:
+class DirectiveTemplate:
     """Enhanced base template class for processing templates with conditionals and loops."""
 
     def __init__(self, template_str: str):

lionagi/{libs/ln_tokenizer.py → experimental/directive/tokenizer.py}

@@ -1,3 +1,19 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 import re
 
 

lionagi/experimental/{directive/evaluator → evaluator}/ast_evaluator.py

@@ -1,3 +1,19 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 import ast
 import operator
 

lionagi/experimental/{directive/evaluator → evaluator}/base_evaluator.py

@@ -1,3 +1,19 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
 import ast
 import operator
 from typing import Any, Dict, Tuple, Callable

lionagi/experimental/knowledge/base.py

@@ -0,0 +1,10 @@
+from ..collections.abc import Component, Field
+from ..collections import Pile, pile
+from ..generic import Node
+
+
+class Knowledge(Component):
+
+    knowledge_base: Pile[Node] = Field(
+        default_factory=pile,
+    )

lionagi/experimental/strategies/base.py

@@ -0,0 +1 @@
+# TODO

lionagi/integrations/bridge/langchain_/documents.py

@@ -28,6 +28,10 @@ def to_langchain_document(datanode: T, **kwargs: Any) -> Any:
     SysUtil.change_dict_key(dnode, old_key="content", new_key="page_content")
     SysUtil.change_dict_key(dnode, old_key="lc_id", new_key="id_")
     dnode = {**dnode, **kwargs}
+    dnode = {k: v for k, v in dnode.items() if v is not None}
+    if "page_content" not in dnode:
+        dnode["page_content"] = ""
+
     return LangchainDocument(**dnode)
 
 

lionagi/integrations/bridge/llamaindex_/index.py

@@ -0,0 +1,30 @@
+class LlamaIndex:
+
+    @classmethod
+    def index(
+        cls,
+        nodes,
+        llm_obj=None,
+        llm_class=None,
+        llm_kwargs=None,
+        index_type=None,
+        **kwargs,
+    ):
+        from llama_index.core import Settings
+        from llama_index.llms.openai import OpenAI
+
+        if not llm_obj:
+            llm_class = llm_class or OpenAI
+            llm_kwargs = llm_kwargs or {}
+            if "model" not in llm_kwargs:
+                llm_kwargs["model"] = "gpt-4o"
+            llm_obj = llm_class(**llm_kwargs)
+
+        Settings.llm = llm_obj
+
+        if not index_type:
+            from llama_index.core import VectorStoreIndex
+
+            index_type = VectorStoreIndex
+
+        return index_type(nodes, **kwargs)

lionagi/integrations/bridge/llamaindex_/llama_index_bridge.py

@@ -100,3 +100,9 @@ class LlamaIndexBridge:
         from .reader import get_llama_index_reader
 
         return get_llama_index_reader(*args, **kwargs)
+
+    @staticmethod
+    def index(nodes, **kwargs):
+        from .index import LlamaIndex
+
+        return LlamaIndex.index(nodes, **kwargs)