user-simulator 0.1.1__py3-none-any.whl → 0.1.2__py3-none-any.whl

metamorphic/rules.py

@@ -0,0 +1,231 @@
+from pydantic import BaseModel, Field, model_validator
+from typing import Optional, List, Any
+from types import SimpleNamespace
+
+from . import get_filtered_tests, empty_filtered_tests
+from .rule_utils import *
+
+# Do not remove this import, it is used to dynamically import the functions
+from .rule_utils import filtered_tests, _conversation_length, extract_float, _only_talks_about
+from .rule_utils import _utterance_index, _chatbot_returns, _repeated_answers, _data_collected, _missing_slots, _responds_in_same_language
+from .rule_utils import _responds_in_same_language, semantic_content
+
+from metamorphic.tests import Test
+
+
+class Rule(BaseModel):
+    name: str
+    description: str
+    conversations: int = 1
+    active: Optional[bool] = True
+    when: Optional[str] = "True"
+    if_: Optional[str] = Field("True", alias="if")
+    then: str
+    yields: Optional[str] = None
+
+    @model_validator(mode='before')
+    @classmethod
+    def check_aliases(cls, values: Any) -> Any:
+        if 'oracle' in values:      # Handle 'oracle' or 'then' interchangeably
+            values['then'] = values.pop('oracle')
+        if 'conversations' in values and values['conversations'] == 'all':  # handle global rules ('all')
+            values['conversations'] = -1
+        return values
+
+    def test(self, tests: List[Test], verbose: bool = False) -> dict:
+        print(f" - Checking rule {self.name} [conversations: {self.conversations if self.conversations!=-1 else 'all'}]")
+        if self.conversations == 1:
+            return self.__property_test(tests, verbose)
+        elif self.conversations == -1: # global rules
+            return self.__global_test(tests, verbose)
+        else: # by now we assume just 2 conversations...
+            return self.__metamorphic_test(tests, verbose)
+
+    def __global_test(self, tests: List[Test], verbose: bool) -> dict:
+        results = {'pass': [], 'fail': [], 'not_applicable': []}
+        # filter the conversations, to select only those satisfying when and if
+        empty_filtered_tests()
+        filtered = get_filtered_tests()
+        for test in tests:
+            test_dict = test.to_dict()
+            conv = [SimpleNamespace(**test_dict)]
+            test_dict['conv'] = conv
+            test_dict.update(util_functions_to_dict())
+            if self.applies(test_dict) and self.if_eval(test_dict):
+                filtered.append(test)
+            else: # does not apply
+                results['not_applicable'].append(test.file_name)
+                if verbose:
+                    print(f"   - On file {test.file_name}")
+                    print(f"     -> Does not apply.")
+
+        try:
+            if self.then_eval(test_dict):
+                results['pass'].append(filtered)
+                if verbose:
+                    print(f"   - On files {', '.join([test.file_name for test in filtered])}")
+                    print(f"     -> Satisfied!")
+            else:
+                results['fail'].append(filtered)
+        except Exception:
+            results['not_applicable'].append(filtered)
+            if verbose:
+                print(f"   - On files {', '.join([test.file_name for test in filtered])}")
+                print(f"     -> Satisfied!")
+
+        return results
+
+    def __property_test(self, tests: List[Test], verbose: bool) -> dict:
+        results = {'pass': [], 'fail': [], 'not_applicable': []}
+        for test in tests:
+            test_dict = test.to_dict()
+            conv = [SimpleNamespace(**test_dict)]
+            test_dict['conv'] = conv
+            test_dict.update(util_functions_to_dict())
+            if verbose:
+                print(f"   - On file {test.file_name}")
+            if self.applies(test_dict):
+                if self.if_eval(test_dict):
+                    try:
+                        return_value = self.then_eval(test_dict)
+                    except Exception:
+                        self.__handle_not_applicable(verbose, results, test)
+                        continue
+                    if return_value == True:  # can be a boolean or another value to signal an error
+                        self.__handle_pass(verbose, results, test)
+                    else:
+                        self.__handle_fail(verbose, results, return_value, test_dict, test)
+                else:
+                    self.__handle_not_applicable(verbose, results, test)
+            else:
+                self.__handle_not_applicable(verbose, results, test)
+        return results
+
+
+    def __handle_not_applicable(self, verbose, results, *tests):
+        if len(tests)==1:
+            results['not_applicable'].append(tests[0].file_name)
+        else:
+            results['not_applicable'].append(tuple(test.file_name for test in tests))
+        if verbose: print(f"     -> Does not apply.")
+
+    def __handle_pass(self, verbose, results, *tests):
+        if len(tests)==1:
+            results['pass'].append(tests[0].file_name)
+        else:
+            results['pass'].append(tuple(test.file_name for test in tests))
+        if verbose:
+            print(f"     -> Satisfied!")
+
+    def __handle_fail(self, verbose, results, return_value, test_dict, *tests):
+        if len(tests)==1:
+            results['fail'].append(tests[0].file_name)
+        else:
+            results['fail'].append(tuple(test.file_name for test in tests))
+        if verbose:
+            message = ""
+            if self.yields is not None:
+                message = ". "+self.yield_eval(test_dict)
+            if return_value != False:   # return_value can be a boolean or something else
+                print(f"     -> NOT Satisfied!. Reason: {return_value}{message}.")
+            else:
+                print(f"     -> NOT Satisfied!. Reason: oracle violated{message}.")
+
+    def __metamorphic_test(self, tests: List[Test], verbose: bool) -> dict:
+        results = {'pass': [], 'fail': [], 'not_applicable': []}
+        for test1 in tests:
+            test_dict1 = test1.to_dict()
+            sns = SimpleNamespace(**test_dict1)
+            conv = [sns, sns]
+            test_dict = {'conv': conv, 'interaction': []}   # just add a dummy interaction variable
+            test_dict.update(util_functions_to_dict())
+            for test2 in tests:
+                if test1 == test2:
+                    continue
+                test_dict2 = test2.to_dict()
+                conv[1] = SimpleNamespace(**test_dict2)
+                if verbose:
+                    print(f"   - On files: {test1.file_name}, {test2.file_name}")
+                if self.applies(test_dict):
+                    if self.if_eval(test_dict):
+                        try:
+                            return_value = self.then_eval(test_dict)
+                        except Exception:
+                            self.__handle_fail(verbose, results, return_value, test_dict, test1, test2)
+                            continue
+                        if return_value == True:
+                            self.__handle_pass(verbose, results, test1, test2)
+                        else:
+                            self.__handle_fail(verbose, results, return_value, test_dict, test1, test2)
+                    else:
+                        self.__handle_not_applicable(verbose, results, test1, test2)
+                else:
+                    self.__handle_not_applicable(verbose, results, test1, test2)
+        return results
+
+    def applies(self, test_dict: dict):
+        try:
+            return eval(self.when, test_dict)
+        except Exception:
+            return False
+
+    def if_eval(self, test_dict: dict):
+        try:
+            return eval(self.if_, test_dict)
+        except Exception:
+            return False
+
+    def then_eval(self, test_dict: dict):
+        code = f"""        
+def _eval(**kwargs):
+    # unpack parameters
+    interaction = kwargs['interaction']
+    conv = kwargs['conv']
+{self.__unpack_dict(test_dict)}
+    
+    #wrappers for functions with implicit parameters
+{self.__wrapper_functions()}
+            
+    return {self.then}
+        """
+        local_namespace = {}
+        exec(code, globals(), local_namespace)
+        self._eval = local_namespace['_eval']
+        return self._eval(**test_dict)
+        #return eval(self.then, test_dict)
+
+    def yield_eval(self, test_dict: dict):
+        code = f"""        
+def _eval(**kwargs):
+    # unpack parameters
+    interaction = kwargs['interaction']
+    conv = kwargs['conv']
+{self.__unpack_dict(test_dict)}
+
+    #wrappers for functions with implicit parameters
+{self.__wrapper_functions()}
+
+    return {self.yields}
+            """
+        local_namespace = {}
+
+        exec(code, globals(), local_namespace)
+        self._eval = local_namespace['_eval']
+        try:
+            return str(self._eval(**test_dict))
+        except Exception:
+            return ""
+
+    def __wrapper_functions(self):
+        result = "\n".join(func for func in util_to_wrapper_dict().values())
+        return result
+
+    def __unpack_dict(self, dict):
+        reserved = ['conv', 'interaction', '__builtins__']
+        reserved += util_functions_to_dict().keys()
+        code = ""
+        for key, value in dict.items():
+            if key in reserved:
+                continue
+            code += "    " + key + f"= kwargs['{key}']\n"
+        return code

metamorphic/tests.py

@@ -0,0 +1,83 @@
+from pydantic import BaseModel, Field
+from typing import List, Optional
+
+
+class Test(BaseModel):
+    ask_about: list
+    conversation: list
+    data_output: Optional[list] = []
+    interaction: Optional[list] = []
+    language: Optional[str] = None
+    serial: str
+    file_name: Optional[str] = None
+    conversation_time: Optional[str] = None
+    errors: Optional[list] = []
+    assistant_times: Optional[list] = []
+
+    @staticmethod
+    def build_test(file, documents):
+        test_metadata = next(documents)  # unpack the 1st YAML document: meta_data
+        test = Test(**test_metadata)
+        test.file_name = file
+        times_doc = next(documents)
+        test.conversation_time = times_doc['conversation time']    # 2nd YAML document is time
+        if 'assistant response time' in times_doc:
+            test.assistant_times = times_doc['assistant response time']
+        test.interaction = next(documents)['interaction']         # 3rd YAML document is the conversation
+        return test
+
+    def to_dict(self):
+        variable_dict = self.__get_ask_about_dict()
+        variable_dict.update(self.__get_parameters_dict(self.conversation, 'conversation'))
+        variable_dict.update(self.__get_parameters_dict(self.data_output, 'data_output'))
+        variable_dict.update(self.__get_interactions_dict(self.interaction))
+        variable_dict.update({'data_output': self.data_output})
+        #print(f"Dict = {variable_dict}")
+        return variable_dict
+
+
+    def __get_interactions_dict(self, interactions_dict):
+        """
+        return a dictionary with, the interactions, the chatbot_phrases and the user_phrases
+        :param interactions_dict:
+        :return:
+        """
+        clean_dict = dict()
+        clean_dict.update({'interaction': interactions_dict})
+        chatbot_phrases = []
+        for phrase in interactions_dict:
+            if 'Assistant' in phrase:
+                chatbot_phrases.append(phrase['Assistant'])
+        clean_dict.update({'chatbot_phrases': chatbot_phrases})
+        user_phrases = []
+        for phrase in interactions_dict:
+            if 'User' in phrase:
+                user_phrases.append(phrase['User'])
+        clean_dict.update({'user_phrases': user_phrases})
+        return clean_dict
+
+
+    def __get_ask_about_dict(self):
+        clean_dict = dict()
+        for item in self.ask_about:
+            if isinstance(item, dict):
+                for key in item:
+                    clean_dict[key] = item[key]
+        return clean_dict
+
+    def __get_parameters_dict(self, attribute, name):
+        clean_dict = dict()
+        for item in attribute:
+            if isinstance(item, dict):
+                clean_dict.update(self.__flatten_dict(name, item))
+        return clean_dict
+
+    def __flatten_dict(self, name, map):
+        flatten_dict = dict()
+        for key in map:
+            if not isinstance(map[key], dict):
+                flatten_dict[name + '_' + key] = map[key]
+                flatten_dict[key] = map[key]
+            else:
+                flatten_dict.update(self.__flatten_dict(name + '_' + key, map[key]))
+        return flatten_dict

metamorphic/text_comparison_utils.py

@@ -0,0 +1,31 @@
+from difflib import SequenceMatcher
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+
+def sequence_similarity(text1, text2):
+    return SequenceMatcher(None, text1, text2).ratio()
+
+def exact_similarity(text1, text2):
+    if text1 == text2:
+        return 1.0
+    return 0.0
+
+def jaccard_similarity(text1, text2):
+    # Split the texts into sets of words
+    set1 = set(text1.lower().split())
+    set2 = set(text2.lower().split())
+
+    # Calculate the Jaccard similarity
+    intersection = set1.intersection(set2)
+    union = set1.union(set2)
+
+    return len(intersection) / len(union)
+
+def tf_idf_cosine_similarity(text1, text2):
+    # Convert the texts into TF-IDF vectors
+    vectorizer = TfidfVectorizer()
+    tfidf_matrix = vectorizer.fit_transform([text1, text2])
+
+    # Compute cosine similarity
+    cosine_sim = cosine_similarity(tfidf_matrix[0:1], tfidf_matrix[1:2])
+    return cosine_sim[0][0]