promptolution 0.1.0__tar.gz

promptolution-0.1.0/PKG-INFO

@@ -0,0 +1,48 @@
+Metadata-Version: 2.1
+Name: promptolution
+Version: 0.1.0
+Summary: 
+Author: Tom Zehle, Moritz Schlager, Timo Heiß
+Requires-Python: >=3.11,<3.12
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Requires-Dist: langchain-anthropic (>=0.1.22,<0.2.0)
+Requires-Dist: langchain-community (>=0.2.12,<0.3.0)
+Requires-Dist: langchain-core (>=0.2.29,<0.3.0)
+Requires-Dist: langchain-openai (>=0.1.21,<0.2.0)
+Requires-Dist: matplotlib (>=3.9.2,<4.0.0)
+Requires-Dist: numpy (>=1.26.0,<2.0.0)
+Requires-Dist: pandas (>=2.2.2,<3.0.0)
+Requires-Dist: seaborn (>=0.13.2,<0.14.0)
+Requires-Dist: tqdm (>=4.66.5,<5.0.0)
+Description-Content-Type: text/markdown
+
+# Promptolution
+
+Project for seminar "AutoML in the age of large pre-trained language models" at LMU Munich , developed by [Timo Heiß](https://www.linkedin.com/in/timo-heiss/), [Moritz Schlager](https://www.linkedin.com/in/moritz-schlager/) and [Tom Zehle](https://www.linkedin.com/in/tom-zehle/).
+
+## Set Up
+
+After having cloned the repository, run
+
+```
+poetry install
+```
+
+to install the necessary dependencies.
+
+You might need to install [pipx](https://pipx.pypa.io/stable/installation/) and [poetry](https://python-poetry.org/docs/) first.
+
+## Usage
+
+Create API Keys for the models you want to use:
+- OpenAI: store token in openaitoken.txt
+- Anthropic: store token in anthropictoken.txt
+- DeepInfra (for Llama): store token in deepinfratoken.txt
+
+Run experiments based on config via:
+
+```
+poetry run python scripts/experiment_runs.py --experiment "configs/<my_experiment>.ini"
+```
+where `<my_experiment>.ini` is a config based on our templates.

promptolution-0.1.0/README.md

@@ -0,0 +1,29 @@
+# Promptolution
+
+Project for seminar "AutoML in the age of large pre-trained language models" at LMU Munich , developed by [Timo Heiß](https://www.linkedin.com/in/timo-heiss/), [Moritz Schlager](https://www.linkedin.com/in/moritz-schlager/) and [Tom Zehle](https://www.linkedin.com/in/tom-zehle/).
+
+## Set Up
+
+After having cloned the repository, run
+
+```
+poetry install
+```
+
+to install the necessary dependencies.
+
+You might need to install [pipx](https://pipx.pypa.io/stable/installation/) and [poetry](https://python-poetry.org/docs/) first.
+
+## Usage
+
+Create API Keys for the models you want to use:
+- OpenAI: store token in openaitoken.txt
+- Anthropic: store token in anthropictoken.txt
+- DeepInfra (for Llama): store token in deepinfratoken.txt
+
+Run experiments based on config via:
+
+```
+poetry run python scripts/experiment_runs.py --experiment "configs/<my_experiment>.ini"
+```
+where `<my_experiment>.ini` is a config based on our templates.

promptolution-0.1.0/promptolution/__init__.py

@@ -0,0 +1,6 @@
+from . import llms
+from . import optimizers
+from . import predictors
+from . import tasks
+from . import callbacks
+from . import config

promptolution-0.1.0/promptolution/callbacks.py

@@ -0,0 +1,120 @@
+import os
+
+import pandas as pd
+from tqdm import tqdm
+
+
+class Callback:
+    def on_step_end(self, optimizer):
+        pass
+
+    def on_epoch_end(self, epoch, logs=None):
+        pass
+
+    def on_train_end(self, logs=None):
+        pass
+
+
+class LoggerCallback(Callback):
+    """
+    Callback for logging optimization progress.
+
+    This callback logs information about each step, epoch, and the end of training.
+
+    Attributes:
+        logger: The logger object to use for logging.
+        step (int): The current step number.
+    """
+    def __init__(self, logger):
+        self.logger = logger
+        self.step = 0
+
+    def on_step_end(self, optimizer):
+        self.step += 1
+        self.logger.critical(f"✨Step {self.step} ended✨")
+        for i, (prompt, score) in enumerate(zip(optimizer.prompts, optimizer.scores)):
+            self.logger.critical(f"*** Prompt {i}: Score: {score}")
+            self.logger.critical(f"{prompt}")
+
+    def on_epoch_end(self, epoch, logs=None):
+        self.logger.critical(f"Epoch {epoch} - {logs}")
+
+    def on_train_end(self, logs=None):
+        self.logger.critical(f"Training ended - {logs}")
+
+
+class CSVCallback(Callback):
+    """
+    Callback for saving optimization progress to a CSV file.
+
+    This callback saves prompts and scores at each step to a CSV file.
+
+    Attributes:
+        path (str): The path to the CSV file.
+        step (int): The current step number.
+    """
+    def __init__(self, path):
+        # if dir does not exist
+        if not os.path.exists(os.path.dirname(path)):
+            os.makedirs(os.path.dirname(path))
+
+        # create file in path with header: "step,prompt,score"
+        with open(path, "w") as f:
+            f.write("step,prompt,score\n")
+        self.path = path
+        self.step = 0
+
+    def on_step_end(self, optimizer):
+        """
+        Save prompts and scores to csv
+        """
+        self.step += 1
+        df = pd.DataFrame(
+            {"step": [self.step] * len(optimizer.prompts), "prompt": optimizer.prompts, "score": optimizer.scores}
+        )
+        df.to_csv(self.path, mode="a", header=False, index=False)
+
+    def on_train_end(self, logs=None):
+        pass
+
+
+class BestPromptCallback(Callback):
+    """
+    Callback for tracking the best prompt during optimization.
+
+    This callback keeps track of the prompt with the highest score.
+
+    Attributes:
+        best_prompt (str): The prompt with the highest score so far.
+        best_score (float): The highest score achieved so far.
+    """
+    def __init__(self):
+        self.best_prompt = ""
+        self.best_score = -99999
+
+    def on_step_end(self, optimizer):
+        if optimizer.scores[0] > self.best_score:
+            self.best_score = optimizer.scores[0]
+            self.best_prompt = optimizer.prompts[0]
+
+    def get_best_prompt(self):
+        return self.best_prompt, self.best_score
+
+
+class ProgressBarCallback(Callback):
+    """
+    Callback for displaying a progress bar during optimization.
+
+    This callback uses tqdm to display a progress bar that updates at each step.
+
+    Attributes:
+        pbar (tqdm): The tqdm progress bar object.
+    """
+    def __init__(self, total_steps):
+        self.pbar = tqdm(total=total_steps)
+
+    def on_step_end(self, optimizer):
+        self.pbar.update(1)
+
+    def on_train_end(self, logs=None):
+        self.pbar.close()

promptolution-0.1.0/promptolution/config.py

@@ -0,0 +1,78 @@
+from configparser import ConfigParser
+from dataclasses import dataclass
+
+
+@dataclass
+class Config:
+    """
+    Configuration class for the promptolution library.
+
+    This class handles loading and parsing of configuration settings,
+    either from a config file or from keyword arguments.
+
+    Attributes:
+        task_name (str): Name of the task.
+        ds_path (str): Path to the dataset.
+        n_steps (int): Number of optimization steps.
+        optimizer (str): Name of the optimizer to use.
+        meta_prompt_path (str): Path to the meta prompt file.
+        meta_llms (str): Name of the meta language model.
+        downstream_llm (str): Name of the downstream language model.
+        evaluation_llm (str): Name of the evaluation language model.
+        init_pop_size (int): Initial population size. Defaults to 10.
+        logging_dir (str): Directory for logging. Defaults to "logs/run.csv".
+        experiment_name (str): Name of the experiment. Defaults to "experiment".
+        include_task_desc (bool): Whether to include task description. Defaults to False.
+        random_seed (int): Random seed for reproducibility. Defaults to 42.
+    """
+    task_name: str
+    ds_path: str
+    n_steps: int
+    optimizer: str
+    meta_prompt_path: str
+    meta_llms: str
+    downstream_llm: str
+    evaluation_llm: str
+    init_pop_size: int = 10
+    logging_dir: str = "logs/run.csv"
+    experiment_name: str = "experiment"
+    include_task_desc: bool = False
+    random_seed: int = 42
+
+    def __init__(self, config_path: str = None, **kwargs):
+        if config_path:
+            self.config_path = config_path
+            self.config = ConfigParser()
+            self.config.read(config_path)
+            self._parse_config()
+        else:
+            for key, value in kwargs.items():
+                setattr(self, key, value)
+
+    def _parse_config(self):
+        self.task_name = self.config["task"]["task_name"]
+        self.ds_path = self.config["task"]["ds_path"]
+        self.n_steps = int(self.config["task"]["steps"])
+        self.random_seed = int(self.config["task"]["random_seed"])
+        self.optimizer = self.config["optimizer"]["name"]
+        self.meta_prompt_path = self.config["optimizer"]["meta_prompt_path"]
+        self.meta_llm = self.config["meta_llm"]["name"]
+        self.downstream_llm = self.config["downstream_llm"]["name"]
+        self.evaluation_llm = self.config["evaluator_llm"]["name"]
+        self.init_pop_size = int(self.config["optimizer"]["init_pop_size"])
+        self.logging_dir = self.config["logging"]["dir"]
+        self.experiment_name = self.config["experiment"]["name"]
+
+        if "include_task_desc" in self.config["task"]:
+            self.include_task_desc = self.config["task"]["include_task_desc"] == "True"
+
+        if self.optimizer == "evopromptga":
+            self.selection_mode = self.config["optimizer"]["selection_mode"]
+        elif self.optimizer == "evopromptde":
+            self.selection_mode = self.config["optimizer"]["donor_random"]
+
+        if "local" in self.meta_llm:
+            self.meta_bs = int(self.config["meta_llm"]["batch_size"])
+
+        if "local" in self.downstream_llm:
+            self.downstream_bs = int(self.config["downstream_llm"]["batch_size"])

promptolution-0.1.0/promptolution/llms/__init__.py

@@ -0,0 +1,31 @@
+from .api_llm import APILLM
+from .base_llm import DummyLLM
+from .local_llm import LocalLLM
+
+
+def get_llm(model_id: str, *args, **kwargs):
+    """
+    Factory function to create and return a language model instance based on the provided model_id.
+
+    This function supports three types of language models:
+    1. DummyLLM: A mock LLM for testing purposes.
+    2. LocalLLM: For running models locally (identified by 'local' in the model_id).
+    3. APILLM: For API-based models (default if not matching other types).
+
+    Args:
+        model_id (str): Identifier for the model to use. Special cases:
+                        - "dummy" for DummyLLM
+                        - "local-{model_name}" for LocalLLM
+                        - Any other string for APILLM
+        *args: Variable length argument list passed to the LLM constructor.
+        **kwargs: Arbitrary keyword arguments passed to the LLM constructor.
+
+    Returns:
+        An instance of DummyLLM, LocalLLM, or APILLM based on the model_id.
+    """
+    if model_id == "dummy":
+        return DummyLLM(*args, **kwargs)
+    if "local" in model_id:
+        model_id = "-".join(model_id.split("-")[1:])
+        return LocalLLM(model_id, *args, **kwargs)
+    return APILLM(model_id, *args, **kwargs)

promptolution-0.1.0/promptolution/llms/api_llm.py

@@ -0,0 +1,149 @@
+import asyncio
+import requests
+import time
+import openai
+from logging import INFO, Logger
+
+from typing import List
+
+from langchain_anthropic import ChatAnthropic
+from langchain_community.chat_models.deepinfra import ChatDeepInfraException
+from langchain_core.messages import HumanMessage
+from langchain_openai import ChatOpenAI
+
+from promptolution.llms.deepinfra import ChatDeepInfra
+
+
+logger = Logger(__name__)
+logger.setLevel(INFO)
+
+
+async def invoke_model(prompt, model, semaphore):
+    """
+    Asynchronously invoke a language model with retry logic.
+
+    Args:
+        prompt (str): The input prompt for the model.
+        model: The language model to invoke.
+        semaphore (asyncio.Semaphore): Semaphore to limit concurrent calls.
+
+    Returns:
+        str: The model's response content.
+
+    Raises:
+        ChatDeepInfraException: If all retry attempts fail.
+    """
+    async with semaphore:
+        max_retries = 100
+        delay = 3
+        attempts = 0
+
+        while attempts < max_retries:
+            try:
+                response = await asyncio.to_thread(model.invoke, [HumanMessage(content=prompt)])
+                return response.content
+            except ChatDeepInfraException as e:
+                print(f"DeepInfra error: {e}. Attempt {attempts}/{max_retries}. Retrying in {delay} seconds...")
+                attempts += 1
+                time.sleep(delay)
+
+
+class APILLM:
+    """
+    A class to interface with various language models through their respective APIs.
+
+    This class supports Claude (Anthropic), GPT (OpenAI), and LLaMA (DeepInfra) models.
+    It handles API key management, model initialization, and provides methods for
+    both synchronous and asynchronous inference.
+
+    Attributes:
+        model: The initialized language model instance.
+
+    Methods:
+        get_response: Synchronously get responses for a list of prompts.
+        _get_response: Asynchronously get responses for a list of prompts.
+    """
+    def __init__(self, model_id: str):
+        """
+        Initialize the APILLM with a specific model.
+
+        Args:
+            model_id (str): Identifier for the model to use.
+
+        Raises:
+            ValueError: If an unknown model identifier is provided.
+        """
+        if "claude" in model_id:
+            ANTHROPIC_API_KEY = open("anthropictoken.txt", "r").read()
+            self.model = ChatAnthropic(model=model_id, api_key=ANTHROPIC_API_KEY)
+        elif "gpt" in model_id:
+            OPENAI_API_KEY = open("openaitoken.txt", "r").read()
+            self.model = ChatOpenAI(model=model_id, api_key=OPENAI_API_KEY)
+        elif "llama" in model_id:
+            DEEPINFRA_API_KEY = open("deepinfratoken.txt", "r").read()
+            self.model = ChatDeepInfra(model_name=model_id, deepinfra_api_token=DEEPINFRA_API_KEY)
+        else:
+            raise ValueError(f"Unknown model: {model_id}")
+
+    def get_response(self, prompts: List[str]) -> List[str]:
+        """
+        Synchronously get responses for a list of prompts.
+
+        This method includes retry logic for handling connection errors and rate limits.
+
+        Args:
+            prompts (list[str]): List of input prompts.
+
+        Returns:
+            list[str]: List of model responses.
+
+        Raises:
+            requests.exceptions.ConnectionError: If max retries are exceeded.
+        """
+        max_retries = 100
+        delay = 3
+        attempts = 0
+
+        while attempts < max_retries:
+            try:
+                responses = asyncio.run(self._get_response(prompts))
+                return responses
+            except requests.exceptions.ConnectionError as e:
+                attempts += 1
+                logger.critical(
+                    f"Connection error: {e}. Attempt {attempts}/{max_retries}. Retrying in {delay} seconds..."
+                )
+                time.sleep(delay)
+            except openai.RateLimitError as e:
+                attempts += 1
+                logger.critical(
+                    f"Rate limit error: {e}. Attempt {attempts}/{max_retries}. Retrying in {delay} seconds..."
+                )
+                time.sleep(delay)
+
+        # If the loop exits, it means max retries were reached
+        raise requests.exceptions.ConnectionError("Max retries exceeded. Connection could not be established.")
+
+    async def _get_response(
+        self, prompts: list[str], max_concurrent_calls=200
+    ) -> list[str]:  
+        """
+        Asynchronously get responses for a list of prompts.
+
+        This method uses a semaphore to limit the number of concurrent API calls.
+
+        Args:
+            prompts (list[str]): List of input prompts.
+            max_concurrent_calls (int): Maximum number of concurrent API calls allowed.
+
+        Returns:
+            list[str]: List of model responses.
+        """
+        semaphore = asyncio.Semaphore(max_concurrent_calls)  # Limit the number of concurrent calls
+        tasks = []
+
+        for prompt in prompts:
+            tasks.append(invoke_model(prompt, self.model, semaphore))
+
+        responses = await asyncio.gather(*tasks)
+        return responses

promptolution-0.1.0/promptolution/llms/base_llm.py

@@ -0,0 +1,74 @@
+from abc import ABC, abstractmethod
+from typing import List
+
+import numpy as np
+
+
+class BaseLLM(ABC):
+    """
+    Abstract base class for Language Models in the promptolution library.
+
+    This class defines the interface that all concrete LLM implementations should follow.
+
+    Methods:
+        get_response: An abstract method that should be implemented by subclasses
+                      to generate responses for given prompts.
+    """
+    def __init__(self, *args, **kwargs):
+        pass
+
+    @abstractmethod
+    def get_response(self, prompts: List[str]) -> List[str]:
+        """
+        Generate responses for the given prompts.
+
+        This method should be implemented by subclasses to define how
+        the LLM generates responses.
+
+        Args:
+            prompts (List[str]): A list of input prompts.
+
+        Returns:
+            List[str]: A list of generated responses corresponding to the input prompts.
+        """
+        pass
+
+
+class DummyLLM(BaseLLM):
+    """
+    A dummy implementation of the BaseLLM for testing purposes.
+
+    This class generates random responses for given prompts, simulating
+    the behavior of a language model without actually performing any
+    complex natural language processing.
+    """
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def get_response(self, prompts: str) -> str:
+        """
+        Generate random responses for the given prompts.
+
+        This method creates silly, random responses enclosed in <prompt> tags.
+        It's designed for testing and demonstration purposes.
+
+        Args:
+            prompts (str or List[str]): Input prompt(s). If a single string is provided,
+                                        it's converted to a list containing that string.
+
+        Returns:
+            List[str]: A list of randomly generated responses, one for each input prompt.
+        """
+        if isinstance(prompts, str):
+            prompts = [prompts]
+        results = []
+        for _ in prompts:
+            r = np.random.rand()
+            if r < 0.3:
+                results += [f"Joooo wazzuppp <prompt>hier gehts los {r} </prompt>"]
+            if 0.3 <= r < 0.6:
+                results += [f"was das hier? <prompt>peter lustig{r}</prompt>"]
+            else:
+                results += [f"hier ist ein <prompt>test{r}</prompt>"]
+
+        return results