lionagi 0.0.312__py3-none-any.whl → 0.2.1__py3-none-any.whl

lionagi/experimental/compressor/llm_compressor.py

@@ -0,0 +1,247 @@
+import asyncio
+from lionagi import alcall
+from lionagi.libs.ln_convert import to_list
+import numpy as np
+from lionagi.core.collections import iModel
+from .base import TokenCompressor
+from lionagi.libs.ln_tokenize import TokenizeUtil
+from time import time
+
+# inspired by LLMLingua, MIT License, Copyright (c) Microsoft Corporation.
+# https://github.com/microsoft/LLMLingua
+
+
+class LLMCompressor(TokenCompressor):
+
+    def __init__(
+        self,
+        imodel: iModel = None,
+        system_msg=None,
+        tokenizer=None,  # must be a callable or object with a tokenize method
+        splitter=None,  # must be a callable or object with a split/chunk/segment method
+        target_ratio=0.2,
+        n_samples=5,  # the cumulative samples to take in each perplexity calculation
+        chunk_size=64,
+        max_tokens_per_sample=80,
+        min_compression_score=0,  # (0-1) the minimum score to consider for compression, 0 means all
+        split_overlap=0,
+        split_threshold=0,
+        verbose=True,
+    ):
+        imodel = imodel or iModel(model="gpt-3.5-turbo", temperature=0.3)
+        super().__init__(imodel=imodel, tokenizer=tokenizer, splitter=splitter)
+        self.system_msg = (
+            system_msg
+            or "Concisely summarize and compress the information for storage:"
+        )
+        self.target_ratio = target_ratio
+        self.n_samples = n_samples
+        self.chunk_size = chunk_size
+        self.max_tokens_per_sample = max_tokens_per_sample
+        self.min_compression_score = min_compression_score
+        self.verbose = verbose
+        self.split_overlap = split_overlap
+        self.split_threshold = split_threshold
+
+    def tokenize(self, text, encoding_name=None, return_byte=False, **kwargs):
+        """
+        by default you can use `encoding_name` to be one of,
+        ['gpt2', 'r50k_base', 'p50k_base', 'p50k_edit', 'cl100k_base', 'o200k_base']
+
+        or you can use `encoding_model` that tiktoken supports in their mapping such as "gpt-4o"
+        """
+        if not self.tokenizer:
+            return TokenizeUtil.tokenize(
+                text,
+                encoding_model=self.imodel.iModel_name,
+                encoding_name=encoding_name,
+                return_byte=return_byte,
+            )
+
+        if hasattr(self.tokenizer, "tokenize"):
+            return self.tokenizer.tokenize(text, **kwargs)
+
+        return self.tokenizer(text, **kwargs)
+
+    def split(
+        self,
+        text,
+        chunk_size=None,
+        overlap=None,
+        threshold=None,
+        by_chars=False,
+        return_tokens=False,
+        return_byte=False,
+        **kwargs,
+    ):
+        if not self.splitter:
+            splitter = (
+                TokenizeUtil.chunk_by_chars
+                if by_chars
+                else TokenizeUtil.chunk_by_tokens
+            )
+            return splitter(
+                text,
+                chunk_size or self.chunk_size,
+                overlap or self.split_overlap,
+                threshold or self.split_threshold,
+                return_tokens=return_tokens,
+                return_byte=return_byte,
+            )
+
+        a = [
+            getattr(self.splitter, i, None)
+            for i in ["split", "chunk", "segment"]
+            if i is not None
+        ][0]
+        a = getattr(self.splitter, a)
+        return a(text, **kwargs)
+
+    async def rank_by_pplex(
+        self, items: list, initial_text=None, cumulative=False, n_samples=None, **kwargs
+    ):
+        """
+        rank a list of items according to their perplexity
+        an item can be a single token or a list of tokens
+
+        kwargs: additional arguments to pass to the model
+        """
+
+        async def _get_item_perplexity(item):
+            item = item if isinstance(item, list) else [item]
+            item = (
+                item[: self.max_tokens_per_sample]
+                if len(item) > self.max_tokens_per_sample
+                else item
+            )
+            return await self.imodel.compute_perplexity(
+                initial_context=initial_text,
+                tokens=item,
+                n_samples=n_samples or self.n_samples,
+                system_msg=self.system_msg,
+                **kwargs,
+            )
+
+        if not isinstance(items, list):
+            items = self.tokenize(items)
+
+        if len(items) == 1:
+            return [items]  # no need to rank a single item
+
+        _segments = []
+        _context = initial_text or ""
+        _task = []
+
+        if cumulative:
+            for i in items:
+                if isinstance(i, list):
+                    _context += " " + " ".join(i).strip()
+                else:
+                    _context += " " + i.strip()
+
+                _segments.append(_context)
+        else:
+            _segments = items
+
+        for i in _segments:
+            _task.append(asyncio.create_task(_get_item_perplexity(i)))
+
+        results = await asyncio.gather(*_task)
+        results = [(item, pplex) for item, pplex in zip(items, results)]
+        return sorted(results, key=lambda x: x[1]["logprobs"], reverse=True)
+
+    async def compress(
+        self,
+        text,
+        target_ratio=None,
+        initial_text=None,
+        cumulative=False,
+        split_kwargs=None,
+        split_overlap=None,
+        split_threshold=None,
+        rank_by="perplexity",
+        min_compression_score=None,
+        verbose=True,
+        **kwargs,
+    ):
+        start = time()
+        if split_kwargs is None:
+            split_kwargs = {}
+            split_kwargs["chunk_size"] = self.max_tokens_per_sample
+            split_kwargs["overlap"] = split_overlap or 0
+            split_kwargs["threshold"] = split_threshold or 0
+
+        len_tokens = len(self.tokenize(text))
+
+        items = self.split(text, return_tokens=True, **split_kwargs)
+
+        if rank_by == "perplexity":
+            ranked_items = await self.rank_by_pplex(
+                items=items, initial_text=initial_text, cumulative=cumulative, **kwargs
+            )
+
+            prompt_tokens = sum([i[1]["num_prompt_tokens"] for i in ranked_items])
+
+            num_completion_tokens = sum(
+                [i[1]["num_completion_tokens"] for i in ranked_items]
+            )
+
+            price = (
+                prompt_tokens * 0.5 / 1000000 + num_completion_tokens * 1.5 / 1000000
+            )
+
+            selected_items = self.select_by_pplex(
+                ranked_items=ranked_items,
+                target_compression_ratio=target_ratio or self.target_ratio,
+                original_length=len_tokens,
+                min_pplex=min_compression_score or self.min_compression_score,
+            )
+
+            if verbose:
+                msg = ""
+                msg += f"Original Token number: {len_tokens}\n"
+
+                def _f(i):
+                    if isinstance(i, str):
+                        i = self.tokenize(i)
+
+                    if isinstance(i, list):
+                        return len(to_list(i, dropna=True, flatten=True))
+
+                len_ = sum([_f(i) for i in selected_items])
+                msg += f"Selected Token number: {len_}\n"
+                msg += f"Token Compression Ratio: {len_ / len_tokens:.03f}\n"
+                msg += f"Compression Time: {time() - start:.04f} seconds\n"
+                msg += f"Compression Model: {self.imodel.iModel_name}\n"
+                msg += f"Compression Method: {rank_by}\n"
+                msg += f"Compression Usage: ${price:.05f}\n"
+                print(msg)
+
+            a = "".join([i.strip() for i in selected_items]).strip()
+            a = a.replace("\n\n", "")
+            return a
+
+        raise ValueError(f"Ranking method {rank_by} is not supported")
+
+    def select_by_pplex(
+        self, ranked_items, target_compression_ratio, original_length, min_pplex=None
+    ):
+        min_pplex = min_pplex or 0
+
+        desired_length = int(original_length * target_compression_ratio)
+
+        items = []
+        current_length = 0
+
+        for item, info in ranked_items:
+            if info["perplexity"] > min_pplex:
+                item = self.tokenize(item) if isinstance(item, str) else item
+                item = item if isinstance(item, list) else [item]
+                item = to_list(item, dropna=True, flatten=True)
+                if current_length + len(item) > desired_length:
+                    break
+                else:
+                    current_length += len(item)
+                    items.append("".join(item))
+
+        return items

lionagi/experimental/compressor/llm_summarizer.py

@@ -0,0 +1,61 @@
+# from lionagi.core.collections import iModel
+# from .base import TokenCompressor
+
+
+# class LLMSummarizer(TokenCompressor):
+
+#     def __init__(
+#         self, imodel: iModel = None, system_msg=None, tokenizer=None, splitter=None,
+#         max_tokens=25, target_ratio=0.3
+#     ):
+#         imodel = imodel or iModel(model="gpt-3.5-turbo", max_tokens=max_tokens)
+#         super().__init__(imodel=imodel, tokenizer=tokenizer, splitter=splitter)
+#         self.system_msg = (
+#             system_msg
+#             or "Summarize the following sentence to be concise and informative:"
+#         )
+#         self.target_ratio = target_ratio
+
+#     async def summarize_sentence(self, sentence, **kwargs):
+#         messages = [
+#             {"role": "system", "content": self.system_msg},
+#             {"role": "user", "content": sentence},
+#         ]
+#         response = await self.imodel.call_chat_completion(messages, **kwargs)
+#         return response["choices"][0]["message"]["content"]
+
+#     def tokenize(self, text):
+#         tokenize_func = self.tokenizer or tokenize
+#         return tokenize_func(text)
+
+#     def split(self, text):
+#         split_func = self.splitter or split_into_segments
+#         return split_func(text)
+
+#     # Function to enforce maximum sentence length
+#     def enforce_max_sentence_length(self, sentence, max_words=25):
+#         words = self.tokenize(sentence)
+#         if len(words) > max_words:
+#             sentence = ' '.join(words[:max_words])
+#         return sentence
+
+#     async def summarize_text(self, text, max_length_per_sentence=25, target_ratio=None, **kwargs):
+#         sentences = self.split(text)
+#         summarized = await alcall(
+#             sentences, self.summarize_sentence, **kwargs
+#         )
+#         summarized = [
+#             self.enforce_max_sentence_length(sentence, max_length_per_sentence)
+#             for sentence in summarized
+#         ]
+
+#         original_length = len(self.tokenize(text))
+#         summarized_length = len(self.tokenize(' '.join(summarized)))
+#         current_ratio = summarized_length / original_length
+
+#         target_ratio = target_ratio or self.target_ratio
+#         if current_ratio > target_ratio:
+#             words_to_remove = int((current_ratio - target_ratio) * original_length)
+#             return ' '.join(summarized[:-words_to_remove])
+
+#         return ' '.join(summarized)

lionagi/experimental/compressor/util.py

@@ -0,0 +1,70 @@
+# import asyncio
+# from lionagi import alcall
+# from lionagi.libs.ln_convert import to_list
+# import numpy as np
+
+# def split_into_segments(text):
+#     segments = text.split(".")  # Splitting by period followed by a space
+#     return [segment.strip() for segment in segments if segment]
+
+# # Tokenize the segment
+# def tokenize(segment):
+#     tokens = segment.split()  # Simple space-based tokenization
+#     return tokens
+
+# async def calculate_perplexity(system_msg: str, imodel, tokens, initial_context=None, **kwargs):
+#     _tasks = []
+#     _context = initial_context or ""
+#     for i in range(len(tokens)):
+#         _context += " " + tokens[i]
+#         messages = [
+#             {"role": "system", "content": system_msg},
+#             {"role": "user", "content": _context},
+#         ]
+#         task = asyncio.create_task(
+#             imodel.call_chat_completion(
+#                 messages=messages, logprobs=True, max_tokens=1, **kwargs
+#             )
+#         )
+#         _tasks.append(task)
+
+#     results = await asyncio.gather(*_tasks)
+#     logprobs = [
+#         result[1]["choices"][0]["logprobs"]["content"] for result in results
+#     ]
+#     logprobs = to_list(logprobs, flatten=True, dropna=True)
+#     logprobs = [lprob_["logprob"] for lprob_ in logprobs]
+#     return np.exp(np.mean(logprobs))
+
+# async def rank_by_perplexity(
+#     text: str | list[str] = None,   # if list we assume they are already well split
+#     initial_text=None,
+
+#     segments,
+#     initial_text=None,
+#     cumulative=False,
+#     **kwargs
+# ):
+#     _segments = []
+#     _context = initial_text or ""
+#     _task = []
+
+#     if cumulative:
+#         for i in range(1, len(segments)):
+#             _context += " " + segments[i - 1]
+#             _segments.append(_context)
+#     else:
+#         _segments = segments
+
+#     for i in segments:
+#         _task.append(asyncio.create_task(
+#             calculate_perplexity(
+#                 self.system_msg, self.imodel, self.tokenize(i), **kwargs)
+#             )
+#         )
+#     segment_perplexities = await asyncio.gather(*_task)
+
+#     return {
+#         segment: perplexity
+#         for segment, perplexity in zip(segments, segment_perplexities)
+#     }

lionagi/experimental/directive/__init__.py

@@ -0,0 +1,19 @@
+# from ..form.predict import predict
+# from .select import select
+# from ..form.score import score
+# from ..form.react import react
+# from .vote import vote
+# from ..form.plan import plan
+# from .cot import chain_of_thoughts, chain_of_react
+
+
+# __all__ = [
+#     "predict",
+#     "select",
+#     "score",
+#     "vote",
+#     "react",
+#     "plan",
+#     "chain_of_thoughts",
+#     "chain_of_react",
+# ]

lionagi/experimental/directive/parser/base_parser.py

@@ -0,0 +1,282 @@
+"""
+Copyright 2024 HaiyangLi
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from typing import List, Optional
+
+from lionagi.experimental.directive.tokenizer import BaseToken
+from ..template.schema import IfNode, TryNode, ForNode
+
+
+class BaseDirectiveParser:
+    """A base parser with lookahead, error recovery, and backtracking support.
+
+    Attributes:
+        tokens (List[BaseToken]): A list of tokens to be parsed.
+        current_token_index (int): The index of the current token in the tokens list.
+        current_token (Optional[BaseToken]): The current token being processed.
+
+    Examples:
+        >>> tokenizer = BaseTokenizer("IF x > 10 THEN DO something ENDIF")
+        >>> tokens = tokenizer.get_tokens()
+        >>> parser = BaseParser(tokens)
+        >>> print(parser.current_token)
+        BaseToken(KEYWORD, IF)
+    """
+
+    def __init__(self, tokens: List[BaseToken]):
+        self.tokens = tokens
+        self.current_token_index = -1
+        self.current_token: Optional[BaseToken] = None
+        self.next_token()
+
+    def next_token(self) -> None:
+        """Advances to the next token in the list."""
+        self.current_token_index += 1
+        if self.current_token_index < len(self.tokens):
+            self.current_token = self.tokens[self.current_token_index]
+        else:
+            self.current_token = None
+
+    def peek_next_token(self, offset: int = 1) -> BaseToken | None:
+        """Peeks at the next token without consuming it.
+
+        Args:
+            offset (int): The number of tokens to look ahead.
+
+        Returns:
+            Optional[BaseToken]: The token at the specified lookahead offset, or None if end of list.
+        """
+        peek_index = self.current_token_index + offset
+        if peek_index < len(self.tokens):
+            return self.tokens[peek_index]
+        else:
+            return None
+
+    def skip_until(self, token_types: List[str]) -> None:
+        """Skips tokens until a token of the specified type is found.
+
+        Args:
+            token_types (List[str]): A list of token types to stop skipping.
+        """
+        while self.current_token and self.current_token.type not in token_types:
+            self.next_token()
+
+    def mark(self) -> int:
+        """Marks the current position in the token list for potential backtracking.
+
+        Returns:
+            int: The current token index.
+        """
+        return self.current_token_index
+
+    def reset_to_mark(self, mark: int) -> None:
+        """Resets the parser to a previously marked position.
+
+        Args:
+            mark (int): The token index to reset to.
+        """
+        self.current_token_index = mark - 1
+        self.next_token()
+
+    def skip_semicolon(self):
+        """Skips a semicolon token if it is the current token."""
+        if self.current_token and self.current_token.value == ";":
+            self.next_token()
+
+    def parse_expression(self):
+        """Parses an expression until a semicolon is encountered.
+
+        Returns:
+            str: The parsed expression as a string.
+
+        Raises:
+            SyntaxError: If a semicolon is not found at the end of the expression.
+        """
+        expr = ""
+        while self.current_token and self.current_token.value != ";":
+            expr += self.current_token.value + " "
+            self.next_token()
+        # Expecting a semicolon at the end of the condition
+        if self.current_token.value != ";":
+            raise SyntaxError("Expected ';' at the end of the condition")
+        self.next_token()  # Move past the semicolon to the next part of the statement
+        return expr.strip()
+
+    def parse_if_block(self):
+        """Parses a block of statements for an IF condition.
+
+        Returns:
+            list: The parsed block of statements as a list of strings.
+        """
+        block = []
+        # Parse the block until 'ELSE', 'ENDIF', ensuring not to include semicolons as part of the block
+        while self.current_token and self.current_token.value not in ("ENDIF", "ELSE"):
+            if self.current_token.value == "DO":
+                self.next_token()  # Move past 'DO' to get to the action
+            block.append(self.current_token.value)  # Add the action to the block
+            self.next_token()  # Move to the next token, which could be a semicolon or the next action
+            if self.current_token.value == ";":
+                self.next_token()  # Move past the semicolon
+        return block
+
+    def parse_if_statement(self):
+        """Parses an IF statement.
+
+        Returns:
+            IfNode: The parsed IF statement as an IfNode object.
+
+        Raises:
+            SyntaxError: If the IF statement is not properly formed.
+        """
+        if self.current_token.type != "KEYWORD" or self.current_token.value != "IF":
+            raise SyntaxError("Expected IF statement")
+        self.next_token()  # Skip 'IF'
+
+        condition = self.parse_expression()  # Now properly ends after the semicolon
+
+        true_block = []
+        if self.current_token.value == "DO":
+            true_block = self.parse_if_block()  # Parse true block after 'DO'
+
+        false_block = None
+        if self.current_token and self.current_token.value == "ELSE":
+            self.next_token()  # Skip 'ELSE', expect 'DO' next for the false block
+            self.skip_semicolon()
+            if self.current_token.value != "DO":
+                raise SyntaxError("Expected 'DO' after 'ELSE'")
+            self.next_token()  # Skip 'DO'
+            false_block = self.parse_if_block()  # Parse false block
+
+        return IfNode(condition, true_block, false_block)
+
+    def parse_for_statement(self):
+        """Parses a FOR statement.
+
+        Returns:
+            ForNode: The parsed FOR statement as a ForNode object.
+
+        Raises:
+            SyntaxError: If the FOR statement is not properly formed.
+        """
+        if self.current_token.type != "KEYWORD" or self.current_token.value != "FOR":
+            raise SyntaxError("Expected FOR statement")
+        self.next_token()  # Skip 'FOR'
+
+        # Parse the iterator variable
+        if self.current_token.type != "IDENTIFIER":
+            raise SyntaxError("Expected iterator variable after FOR")
+        iterator = self.current_token.value
+        self.next_token()  # Move past the iterator variable
+
+        # Expect and skip 'IN' keyword
+        if self.current_token.type != "KEYWORD" or self.current_token.value != "IN":
+            raise SyntaxError("Expected 'IN' after iterator variable")
+        self.next_token()  # Move past 'IN'
+
+        # Parse the collection
+        if self.current_token.type not in ["IDENTIFIER", "LITERAL"]:
+            raise SyntaxError("Expected collection after 'IN'")
+        collection = self.current_token.value
+        self.next_token()  # Move past the collection
+
+        # Now, parse the block of statements to execute
+        true_block = self.parse_for_block()
+
+        # Construct and return a ForNode
+        return ForNode(iterator, collection, true_block)
+
+    def parse_for_block(self):
+        """Parses a block of statements for a FOR loop.
+
+        Returns:
+            list: The parsed block of statements as a list of strings.
+        """
+        block = []
+        # Skip initial 'DO' if present
+        if self.current_token and self.current_token.value == "DO":
+            self.next_token()
+
+        while self.current_token and self.current_token.value not in ("ENDFOR",):
+            if self.current_token.value == ";":
+                # If a semicolon is encountered, skip it and move to the next token
+                self.next_token()
+                continue
+            # Add the current token to the block unless it's a 'DO' or ';'
+            if self.current_token.value != "DO":
+                block.append(self.current_token.value)
+            self.next_token()
+
+        # The loop exits when 'ENDFOR' is encountered; move past it for subsequent parsing
+        self.next_token()  # Skip 'ENDFOR'
+        return block
+
+    def parse_try_statement(self):
+        """Parses a TRY statement.
+
+        Returns:
+            TryNode: The parsed TRY statement as a TryNode object.
+
+        Raises:
+            SyntaxError: If the TRY statement is not properly formed.
+        """
+        if self.current_token.type != "KEYWORD" or self.current_token.value != "TRY":
+            raise SyntaxError("Expected TRY statement")
+        self.next_token()  # Skip 'TRY'
+
+        try_block = self.parse_try_block("EXCEPT")  # Parse the try block until 'EXCEPT'
+
+        # Now expecting 'EXCEPT' keyword
+        if not (self.current_token and self.current_token.value == "EXCEPT"):
+            raise SyntaxError("Expected 'EXCEPT' after try block")
+        self.next_token()  # Move past 'EXCEPT'
+
+        except_block = self.parse_try_block(
+            "ENDTRY"
+        )  # Parse the except block until 'ENDTRY'
+
+        # Ensure we are correctly positioned after 'ENDTRY'
+        if self.current_token and self.current_token.value != "ENDTRY":
+            raise SyntaxError("Expected 'ENDTRY' at the end of except block")
+        self.next_token()  # Move past 'ENDTRY' for subsequent parsing
+
+        return TryNode(try_block, except_block)
+
+    def parse_try_block(self, stop_keyword):
+        """Parses a block of statements for a TRY or EXCEPT clause.
+
+        Args:
+            stop_keyword (str): The keyword that indicates the end of the block.
+
+        Returns:
+            list: The parsed block of statements as a list of strings.
+        """
+        block = []
+        while self.current_token and self.current_token.value != stop_keyword:
+            if self.current_token.value == "DO":
+                self.next_token()  # Move past 'DO' to get to the action
+            elif self.current_token.value == ";":
+                self.next_token()  # Move past the semicolon
+                continue  # Skip adding ';' to the block
+            else:
+                block.append(self.current_token.value)  # Add the action to the block
+                self.next_token()
+
+        return block
+
+
+# "IF condition1 && condition2; DO action2; ELSE; DO action3; ENDIF;"
+# "FOR input_ IN collections; DO action(input_); ENDFOR;"
+# "TRY; DO action(); EXCEPT; DO action(input_); ENDTRY;"