user-simulator 0.1.0__py3-none-any.whl

user_sim/core/user_simulator.py

@@ -0,0 +1,302 @@
+from .data_extraction import DataExtraction
+from user_sim.utils.utilities import *
+from .data_gathering import *
+from langchain_core.prompts import PromptTemplate
+from langchain_core.output_parsers import StrOutputParser
+from langchain.chat_models import init_chat_model
+from user_sim.utils.token_cost_calculator import calculate_cost, max_input_tokens_allowed, max_output_tokens_allowed, invoke_llm
+from user_sim.utils import config
+
+import logging
+
+parser = StrOutputParser()
+logger = logging.getLogger('Info Logger')
+
+
+class UserChain:
+
+    def __init__(self, user_role, temp):
+        self.user_role = user_role
+        self.user_llm = None
+        self.model = None
+        self.model_provider = None
+        self.temperature = temp
+        self.init_user_module()
+        self.user_context = PromptTemplate(
+            input_variables=["reminder", "history"],
+            template=self.set_role_template()
+        )
+        self.chain = None
+
+    def init_user_module(self):
+        self.model = config.model
+        self.model_provider = config.model_provider
+
+        if self.model_provider is None:
+            params = {
+                "model": self.model,
+                "temperature": self.temperature
+            }
+        else:
+            params = {
+                "model": self.model,
+                "model_provider": self.model_provider,
+                "temperature": self.temperature
+            }
+
+        self.user_llm = init_chat_model(**params)
+
+    def set_role_template(self):
+        reminder = """{reminder}"""
+        history = """History of the conversation so far: {history}"""
+        role_prompt = self.user_role + reminder + history
+        return role_prompt
+
+    @staticmethod
+    def parse_history(conversation_history):
+        lines = []
+        for inp in conversation_history['interaction']:
+            for k, v in inp.items():
+                lines.append(f"{k}: {v}")
+        return "\n".join(lines)
+
+    def text_method(self, conversation_history, reminder):
+        history = self.parse_history(conversation_history)  # formats list to str
+        # input_params = {'history': history, 'reminder': reminder}
+        # invoke_llm(self.user_llm, self.user_context, input_params, self.model, module="user_simulator", parser=True)
+
+        if max_input_tokens_allowed(history+reminder, model_used=self.model):
+            logger.error(f"Token limit was surpassed")
+            return "exit"
+
+        if config.token_count_enabled:
+            self.user_llm.max_tokens = max_output_tokens_allowed(self.model)
+
+        self.chain = self.user_context | self.user_llm | parser
+
+        response = self.chain.invoke({'history': history, 'reminder': reminder})
+        if config.token_count_enabled:
+            calculate_cost(history + reminder, response, self.model, module="user_simulator")
+        return response
+
+    def invoke(self, conversation_history, reminder):
+
+        response = self.text_method(conversation_history, reminder)
+
+        return response
+
+class UserSimulator:
+
+    def __init__(self, user_profile, chatbot):
+
+        self.user_profile = user_profile
+        self.chatbot = chatbot
+        self.temp = user_profile.temperature
+        self.conversation_history = {'interaction': []}
+        self.ask_about = user_profile.ask_about.prompt()
+        self.data_gathering = ChatbotAssistant(user_profile.ask_about.phrases)
+        self.goal_style = user_profile.goal_style
+        self.test_name = user_profile.test_name
+        self.repeat_count = 0
+        self.loop_count = 0
+        self.interaction_count = 0
+        self.user_chain = UserChain(self.user_profile.role, self.temp)
+        self.my_context = self.InitialContext()
+        self.output_slots = self.__build_slot_dict()
+        self.error_report = []
+
+    def __build_slot_dict(self):
+        slot_dict = {}
+        output_list = self.user_profile.output
+        for output in output_list:
+            var_name = list(output.keys())[0]
+            slot_dict[var_name] = None
+        return slot_dict
+
+    class InitialContext:
+        def __init__(self):
+            self.original_context = []
+            self.context_list = []
+
+        def initiate_context(self, context):
+
+            default_context = config.default_context
+
+            if isinstance(context, list):
+                self.original_context = context.copy() + default_context.copy()
+                self.context_list = context.copy() + default_context.copy()
+            else:
+                self.original_context = [context] + default_context
+                self.context_list = [context] + default_context
+
+        def add_context(self, new_context):
+            if isinstance(new_context, list):
+                for cont in new_context:
+                    self.context_list.append(cont)
+            else:
+                self.context_list.append(new_context)
+                # TODO: add exception to force the user to initiate the context
+
+        def get_context(self):
+            return '. '.join(self.context_list)
+
+        def reset_context(self):
+            self.context_list = self.original_context.copy()
+
+    def repetition_track(self, response, reps=3):
+
+        self.my_context.reset_context()
+        logger.info(f'Context list: {self.my_context.context_list}')
+
+        if nlp_processor(response, self.chatbot.fallback, 0.6):
+
+            self.repeat_count += 1
+            self.loop_count += 1
+            logger.info(f"is fallback. Repeat_count: {self.repeat_count}. Loop count: {self.loop_count}")
+
+            if self.repeat_count >= reps:
+                self.repeat_count = 0
+                change_topic = """
+                               Since the assistant is not understanding what you're saying, change the 
+                               topic to other things to ask about without starting a new conversation
+                               """
+
+                self.my_context.add_context(change_topic)
+
+            else:
+                ask_repetition = """
+                                If the assistant asks you to repeat the question, repeat the last question the user 
+                                said but rephrase it.
+                                """
+
+                self.my_context.add_context(ask_repetition)
+        else:
+            self.repeat_count = 0
+            self.loop_count = 0
+
+    @staticmethod
+    def conversation_ending(response):
+        return nlp_processor(response, "src/testing/user_sim/end_conversation_patterns.yml", 0.5)
+
+    def get_history(self):
+
+        lines = []
+        for inp in self.conversation_history['interaction']:
+            for k, v in inp.items():
+                lines.append(f"{k}: {v}")
+        return "\n".join(lines)
+
+    def update_history(self, role, message):
+        self.conversation_history['interaction'].append({role: message})
+
+    def end_conversation(self, input_msg):
+
+
+        if config.total_cost >= config.limit_cost or config.total_individual_cost >= config.limit_individual_cost:
+            if config.total_cost >= config.limit_cost:
+                config.errors.append({2000: 'Exceeded global cost'})
+            elif config.total_individual_cost >= config.limit_individual_cost:
+                config.errors.append({2001: 'Exceeded conversation specific cost'})
+
+            logger.info('is end')
+            return True
+
+        if self.goal_style[0] == 'steps' or self.goal_style[0] == 'random steps':
+            if self.interaction_count >= self.goal_style[1]:
+                logger.info('is end')
+                return True
+
+        elif self.conversation_ending(input_msg) or self.loop_count >= 9:
+            config.errors.append({1000: 'Exceeded loop Limit'})
+            logger.warning('Loop count surpassed 9 interactions. Ending conversation.')
+            return True
+
+        elif 'all_answered' in self.goal_style[0] or 'default' in self.goal_style[0]:
+            if (self.data_gathering.gathering_register["verification"].all()
+                and self.all_data_collected()
+                    or self.goal_style[2] <= self.interaction_count):
+                logger.info(f'limit amount of interactions achieved: {self.goal_style[2]}. Ending conversation.')
+                return True
+            else:
+                return False
+
+        else:
+            return False
+
+    def all_data_collected(self):
+        output_list = self.user_profile.output
+        for output in output_list:
+            var_name = list(output.keys())[0]
+            var_dict = output.get(var_name)
+            if var_name in self.output_slots and self.output_slots[var_name] is not None:
+                continue
+            my_data_extract = DataExtraction(self.conversation_history,
+                                             var_name,
+                                             var_dict["type"],
+                                             var_dict["description"])
+            value = my_data_extract.get_data_extraction()
+            if value[var_name] is None:
+                return False
+            else:
+                self.output_slots[var_name] = value[var_name]
+        return True
+
+    def get_response(self, input_msg):
+
+        self.update_history("Assistant", input_msg)
+        self.data_gathering.add_message(self.conversation_history)
+
+        if self.end_conversation(input_msg):
+            return "exit"
+
+        self.repetition_track(input_msg)
+
+        self.my_context.add_context(self.user_profile.get_language())
+
+        # history = self.get_history()
+
+        user_response = self.user_chain.invoke(self.conversation_history, self.my_context.get_context())
+
+        self.update_history("User", user_response)
+
+        self.interaction_count += 1
+
+        return user_response
+
+    @staticmethod
+    def formatting(role, msg):
+        return [{"role": role, "content": msg}]
+
+    def get_interaction_styles_prompt(self):
+        interaction_style_prompt = []
+        for instance in self.user_profile.interaction_styles:
+            if instance.change_language_flag:
+                pass
+            else:
+                interaction_style_prompt.append(instance.get_prompt())
+        return ''.join(interaction_style_prompt)
+
+    def open_conversation(self, input_msg=None):
+
+        interaction_style_prompt = self.get_interaction_styles_prompt()
+        self.my_context.initiate_context([self.user_profile.context,
+                                          interaction_style_prompt,
+                                          self.ask_about])
+
+        language_context = self.user_profile.get_language()
+        self.my_context.add_context(language_context)
+
+        if input_msg:
+            self.update_history("Assistant", input_msg)
+            self.data_gathering.add_message(self.conversation_history)
+            if self.end_conversation(input_msg):
+                return "exit"
+            self.repetition_track(input_msg)
+
+        user_response = self.user_chain.invoke(self.conversation_history, self.my_context.get_context())
+
+        self.update_history("User", user_response)
+
+        self.data_gathering.add_message(self.conversation_history)
+        self.interaction_count += 1
+        return user_response

user_sim/handlers/asr_module.py

@@ -0,0 +1,128 @@
+import speech_recognition as sr
+from pydantic import BaseModel, ValidationError
+from typing import List, Union, Dict, Optional
+import time
+from user_sim.utils.utilities import read_yaml
+from user_sim.utils.token_cost_calculator import calculate_cost, max_input_tokens_allowed
+from openai import OpenAI
+import warnings
+import pygame
+import logging
+logger = logging.getLogger('Info Logger')
+
+
+pygame.mixer.init()
+warnings.filterwarnings("ignore", category=FutureWarning, module="whisper")
+warnings.filterwarnings("ignore", category=RuntimeWarning, module="pydub")
+client = OpenAI()
+audio_format = "mp3"
+
+
+def get_audio_duration(audio):
+    from pydub import AudioSegment
+    import io
+    wav_data = audio.get_wav_data()
+    audio_segment = AudioSegment.from_file(io.BytesIO(wav_data), format="wav")
+    duration_seconds = len(audio_segment) / 1000.0
+    return duration_seconds
+
+
+class SttModel(BaseModel):
+    energy_threshold: float = 50
+    pause_threshold: float = 1
+
+
+class TtsModel(BaseModel):
+    model: str = "tts-1"
+    voice: str = "alloy"
+    speed: float = 1.0
+
+
+class SpeechModel(BaseModel):
+    stt: Optional[SttModel] = SttModel
+    tts: Optional[TtsModel] = TtsModel
+
+
+class STTModule:
+
+    def __init__(self, config):
+
+        if config:
+            config_file = read_yaml(config)
+            try:
+                validated_data = SpeechModel(**config_file)
+            except ValidationError as e:
+                print(e.json())
+                raise
+
+        #STT
+            self.energy_th = validated_data.stt.energy_threshold
+            self.pause_th = validated_data.stt.pause_threshold
+
+        #TTS
+            self.model = validated_data.tts.model
+            self.voice = validated_data.tts.voice
+            self.speed = validated_data.tts.speed
+
+        else:
+        # STT
+            self.energy_th = 50
+            self.pause_th = 1
+
+        # TTS
+            self.model = "tts-1"
+            self.voice = "alloy"
+            self.speed = 1.0
+
+    def hear(self):
+        r = sr.Recognizer()
+        with sr.Microphone() as source:
+            logger.info("Listening...")
+            r.energy_threshold = self.energy_th  # lower values for quieter rooms
+            r.pause_threshold = self.pause_th
+            audio = r.listen(source)
+
+        try:
+            logger.info("Recognizing...")
+            start = time.time()
+            audio_length = get_audio_duration(audio)
+            if max_input_tokens_allowed(model_used="whisper", audio_length=audio_length):
+                logger.error(f"Token limit was surpassed")
+                return False, None
+
+            text = r.recognize_whisper(audio)
+            end = time.time()
+
+            calculate_cost(model="whisper", module="stt_module", audio_length=audio_length)
+            logger.info(f"Recognition time:  {end - start}")
+            return True, text
+        except sr.UnknownValueError:
+            logger.warning("Recognition model could not understand audio")
+            return True, "Repeat, please."
+        except sr.RequestError as e:
+            logger.warning("Could not request results from Speech Recognition service; {0}".format(e))
+            return False, None
+
+
+    def say(self, message):
+
+        if max_input_tokens_allowed(message, self.model):
+            return
+
+        with client.audio.speech.with_streaming_response.create(
+            model=self.model,
+            voice=self.voice,
+            speed=self.speed,
+            input=message,
+            response_format=audio_format
+        ) as response:
+            response.stream_to_file("data/audio_files/output." + audio_format)
+
+        calculate_cost(message, model=self.model, module="tts_module")
+        logger.info("Playing...")
+        audio_path = f"audio_files/output.{audio_format}"
+        with open(audio_path, 'rb') as audio_file:
+            pygame.mixer.music.load(audio_file)
+            pygame.mixer.music.play()
+            while pygame.mixer.music.get_busy():
+                pygame.time.Clock().tick(10)

user_sim/handlers/html_parser_module.py

@@ -0,0 +1,202 @@
+import requests
+import platform
+import shutil
+import logging
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.chrome.service import Service as ChromeService
+from selenium.webdriver.edge.service import Service as EdgeService
+from selenium.webdriver.firefox.service import Service as FirefoxService
+from webdriver_manager.chrome import ChromeDriverManager
+from webdriver_manager.microsoft import EdgeChromiumDriverManager
+from webdriver_manager.firefox import GeckoDriverManager
+from user_sim.handlers.image_recognition_module import image_description
+from user_sim.utils import config
+from user_sim.utils.register_management import save_register, load_register, hash_generate
+
+
+logger = logging.getLogger('Info Logger')
+
+wp_register_name = "webpage_register.json"
+
+
+def is_driver_installed(driver_name):
+    return shutil.which(driver_name) is not None
+
+
+def get_webdriver():
+    system = platform.system()
+
+    if system == "Windows":
+        print("Using Microsoft Edge on Windows")
+        if is_driver_installed("msedgedriver"):
+            return webdriver.Edge()
+        service = EdgeService(EdgeChromiumDriverManager().install())
+        return webdriver.Edge(service=service)
+
+    elif system == "Darwin":  # MacOS
+        print("Using Safari on Mac")
+        return webdriver.Safari()
+
+    elif system == "Linux":
+        print("Using Firefox on Linux")
+        if is_driver_installed("geckodriver"):
+                return webdriver.Firefox()
+        service = FirefoxService(GeckoDriverManager().install())
+        return webdriver.Firefox(service=service)
+
+    else:
+        print("Unknown system. Trying Chrome...")
+        if is_driver_installed("chromedriver"):
+            return webdriver.Chrome()
+        service = ChromeService(ChromeDriverManager().install())
+        return webdriver.Chrome(service=service)
+
+
+
+
+def is_dynamic_page(url):
+    headers = {"User-Agent": "Mozilla/5.0"}  # Simulate a real browser
+    response = requests.get(url, headers=headers)
+
+    if response.status_code != 200:
+        return False  # If the page does not respond, assume it's not JS
+
+    # Check content type
+    content_type = response.headers.get("Content-Type", "")
+    if "text/html" not in content_type:
+        logger.warning(f"Expected HTML, but got {content_type}")
+        return False
+
+    try:
+        soup = BeautifulSoup(response.text, "html.parser")
+    # Check if there is little visible content
+        text = soup.get_text(strip=True)
+
+        if len(text) < 200:  # Adjust threshold based on page type
+            return True  # Probably uses JavaScript
+
+    except Exception as e:
+        logger.error(f"Error parsing HTML: {e}")
+        # todo: add error code
+        return False
+
+    return False  # The page has enough static content
+
+
+def uses_ajax(url):
+    response = requests.get(url, headers={"User-Agent": "Mozilla/5.0"})
+
+    if response.status_code != 200:
+        return False
+
+    return "fetch" in response.text or "XMLHttpRequest" in response.text
+
+
+def detect_scraping_method(url):
+    if is_dynamic_page(url) or uses_ajax(url):
+        return "selenium"
+    return "requests"
+
+
+def describe_images_in_webpage(url, soup):
+    image_descriptions = []
+    for img_index, img in enumerate(soup.find_all("img")):
+        src = img.get("src")
+        if src:
+            full_url = urljoin(url, src)
+            description = image_description(full_url, detailed=False)
+            image_descriptions.append(f"Image description {img_index}: {description}")
+
+    image_text = " ".join(image_descriptions)
+    return image_text
+
+
+
+def webpage_reader(url):
+
+    if config.ignore_cache:
+        register = {}
+        logger.info("Cache will be ignored.")
+    else:
+        register = load_register(wp_register_name)
+
+    wp_hash = hash_generate(content=url)
+
+    def process_html(url):
+        method = detect_scraping_method(url)
+
+        if method == "selenium":
+            chrome_options = Options()
+            chrome_options.add_argument("--headless")
+            driver = get_webdriver()
+
+            driver.get(url)
+            page_source = driver.page_source
+            driver.quit()
+
+            try:
+                soup = BeautifulSoup(page_source, "lxml")
+            except Exception as e:
+                logger.error(f"Error parsing JavaScript-rendered HTML: {e}")
+                return None
+
+        else:
+            headers = {"User-Agent": "Mozilla/5.0"}
+            response = requests.get(url, headers=headers)
+
+            if response.status_code != 200:
+                logger.error(f"Error accessing the page: {response.status_code}")
+                return None
+
+            # Ensure proper encoding
+            response.encoding = response.apparent_encoding
+
+            # Check if the response is actually HTML
+            content_type = response.headers.get("Content-Type", "")
+            if "text/html" not in content_type:
+                logger.warning(f"Expected HTML, but got {content_type}")
+                return None
+
+            try:
+                soup = BeautifulSoup(response.text, "lxml")  # More robust parser
+            except Exception as e:
+                logger.error(f"Error parsing HTML: {e}")
+                return None
+
+        # Remove unnecessary elements
+        for script in soup(["script", "style", "header", "footer", "nav", "aside"]):
+            script.extract()
+
+        text = soup.get_text(separator=" ", strip=True)
+        images = describe_images_in_webpage(url, soup)
+        description = f"(Web page content: {text + images} >>)"
+        return description
+
+    if wp_hash in register:
+        if config.update_cache:
+            output_text = process_html(url)
+            if output_text is None:
+                logger.error("Cache couldn't be updated due to web page error.")
+                return f"(Web page content: web page couldn't be loaded.)"
+            register[wp_hash] = output_text
+            logger.info("Cache updated!")
+        output_text = register[wp_hash]
+        logger.info("Retrieved information from cache.")
+    else:
+        output_text = process_html(url)
+        if output_text:
+            register[wp_hash] = output_text
+        else:
+            output_text = f"(Web page content: web page couldn't be loaded.)"
+
+    if config.ignore_cache:
+        logger.info("PDF cache was ignored.")
+    else:
+        save_register(register, wp_register_name)
+        logger.info("PDF cache was saved!")
+
+    logger.info(output_text)
+    return output_text