reflexive 0.1.7__2-py3-none-any.whl

reflexive/__init__.py

@@ -0,0 +1,9 @@
+from .common.parameters import *
+from .common.local import *
+from .common.util import *
+from .visual.display import *
+from .analyse.reflexive_expressions import *
+from .analyse.aws_nlp import *
+from .analyse.general import *
+from .aws_connect.comprehend import *
+from .aws_connect.s3 import *

reflexive/analyse/aws_nlp.py

@@ -0,0 +1,196 @@
+
+
+from reflexive.common.parameters import Parameters
+from reflexive.common.local import Local
+from reflexive.aws_connect.comprehend import Comprehend
+from reflexive.common.util import Util
+
+import json
+import logging
+import pandas as pd
+
+try:
+    import coloredlogs
+    coloredlogs.install(level='INFO')
+except:
+    print("Colored logs not available")
+
+class Nlp:
+    
+    logger = logging.getLogger(__name__)
+    
+    def __init__(self,parameters:Parameters,local:Local,comprehend:Comprehend):
+        self.__parameters = parameters.all_parameters()
+        self.logger.debug(f"Parameters: {self.__parameters}")
+        self.local_path = self.__parameters['local_path']
+        self.prefix = self.__parameters['prefix']
+        self.postfix = self.__parameters['postfix']
+        self.analysis_types = self.__parameters['analysis_types']
+        self.__local = local
+        self.__comprehend = comprehend
+
+
+
+    #### COMPREHEND ANALYSIS
+
+    def comprehend_analysis(self,df):
+        util = Util()
+        comprehend = self.__comprehend
+        self.analysis_types = self.__parameters['analysis_types']
+        #print(type(df.text))
+        # chunk the text for batch analysis
+        chunked_text = util.series_to_chunked_list(series=df.text)
+        print("Number of chunks:",len(chunked_text))
+        # start batch analysis
+        chunked_results = comprehend.get_multiple_batch_analysis(chunked_text)
+        print("Finished Analysis.")
+        # write to file
+        print("Writing data to file...")
+        with open(f"{self.local_path}{self.prefix}analysis_chunks{self.postfix}.json", "w") as fp:
+            json.dump(chunked_results,fp)            
+        print("DONE!")
+        # unchunk
+        final_results = {}
+        for key in chunked_results.keys():
+            final_results[key] = comprehend.unbatch_results(self.analysis_types[key],chunked_results[key])
+            print("Finished Unbatching",key," -  Writing data to file...")
+            filename = f"{self.local_path}{self.prefix}{key}{self.postfix}.json"
+            with open(filename, "w") as fp:
+                json.dump(final_results[key],fp)            
+        print("DONE!")
+        # Save final_results for reload if necessary
+        with open(f"{self.local_path}{self.prefix}final_results{self.postfix}.json", "w") as fp:
+            json.dump(final_results,fp) 
+        return final_results
+    
+    def check_results(self,results):
+        print("Checking for errors...")
+        for key in results.keys():
+            errors = results[key]['errors']
+            print(f"Errors for {key}: {errors}")
+        print()
+        print("Checking that we have results for all docs")
+        for key in results.keys():
+            num_results= len(results[key]['results'])
+            print(f"Number of results for {key}: {num_results}")
+        return errors
+    
+    def add_results_to_df(self,results,df):
+        for key in results.keys():
+            rs = results[key]['results']
+            newresults = {}
+            for oldkey in rs.keys():
+                newresults[int(oldkey)] = rs[oldkey] # Need to change keys to int to properly add to dataframe
+            df[key] = pd.Series(newresults)
+        return df
+    
+    def nlp_analytics(self,df):
+        temp_df = df.copy()
+        temp_df = self.keyphrase_analytics(temp_df)
+        temp_df = self.named_entity_analytics(temp_df)
+        temp_df = self.targeted_sentiment_analytics(temp_df)
+        temp_df = self.syntax_analytics(temp_df)
+        return temp_df
+    
+    
+    def keyphrase_analytics(self,df):
+        util = Util()
+        df["key_phrases"] = df.KeyPhraseResults.apply(self.parse_keyPhraseResults)
+        df["key_phrase_counts"] = df.key_phrases.apply(util.count_keys)
+        df["key_phrases_total"] = df.key_phrase_counts.apply(util.tuple_values_total)
+        if (len(df)>1):
+            df["key_phrases_scaled"] = util.scale_min_max(df[['key_phrases_total']])
+        else:
+            df["key_phrases_scaled"] = 1
+        # Normalise based on text_scaled
+        df['key_phrases_norm'] = util.normalise_scaled(df,'key_phrases_scaled')
+        return df
+    
+    def named_entity_analytics(self,df):
+        util = Util()
+        df["named_entities"] = df.TargetedSentimentResults.apply(self.parse_namedEntities)
+        df['named_entity_counts'] = df.named_entities.apply(util.count_entities)
+        df["named_entity_ratios"] = df.named_entity_counts.apply(util.ratios)
+        df["named_entities_total"] = df.named_entity_counts.apply(util.tuple_values_total)
+        if (len(df)>1):
+            df["named_entities_scaled"] = util.scale_min_max(df[['named_entities_total']])
+        else:
+            df["named_entities_scaled"] = 1
+        df['named_entities_norm'] = util.normalise_scaled(df,'named_entities_scaled')
+        return df
+    
+    def targeted_sentiment_analytics(self,df):
+        util = Util()
+        df["targeted_sentiment"] = df.TargetedSentimentResults.apply(self.parse_targetedSentimentResults)
+        df['targeted_sentiment_counts'] = df.targeted_sentiment.apply(util.count_entities)
+        df["targeted_sentiment_ratios"] = df.targeted_sentiment_counts.apply(util.ratios)
+        df["targeted_sentiment_total"] = df.targeted_sentiment_counts.apply(util.tuple_values_total)
+        if (len(df)>1):
+            df["targeted_sentiment_scaled"] = util.scale_min_max(df[['targeted_sentiment_total']])
+        else:
+            df["targeted_sentiment_scaled"] = 1
+        df['targeted_sentiment_norm'] = util.normalise_scaled(df,'targeted_sentiment_scaled')
+        return df
+    
+    def syntax_analytics(self,df):
+        util = Util()
+        df["pos_tags"] = df.SyntaxResults.apply(self.parse_syntaxResults)
+        df['pos_tag_counts'] = df.pos_tags.apply(util.count_labels)
+        df["pos_tag_ratios"] = df.pos_tag_counts.apply(util.ratios)
+        df["pos_tags_total"] = df.pos_tag_counts.apply(util.tuple_values_total)
+        if (len(df)>1):
+            df["pos_tags_scaled"] = util.scale_min_max(df[['pos_tags_total']])
+        else:
+            df["pos_tags_scaled"] = 1
+        df['pos_tags_norm'] = util.normalise_scaled(df,'pos_tags_scaled')
+        return df    
+    
+    
+    # Parse key_phrases results - include all above threshold
+    def parse_keyPhraseResults(self,keyPhraseResults,threshold=0.95,min_count=1):
+        util = Util()
+        phrases = {}
+        filtered = [str.lower(r['Text']) for r in keyPhraseResults if r['Score'] > threshold]
+        for phrase in filtered:
+            phrases[phrase] = phrases.get(phrase,0)+1
+
+        filtered_phrases = {k:v for k,v in phrases.items() if v >= min_count}
+        return util.sort_dict_by_value(filtered_phrases)
+
+    # Parse syntax results - include specific postags
+    def parse_syntaxResults(self,syntax_results,postags_keep = ['ADV','VERB','AUX','ADJ','NOUN','PRON','PROPN']):
+        sequence = list()
+        for token in syntax_results:
+            tag = token['PartOfSpeech']['Tag']
+            if tag in postags_keep:
+                sequence.append((str.lower(token['Text']),tag))
+        return sequence
+
+    # Parse targeted sentiment results - keep non-neutral above threshold
+
+    def parse_targetedSentimentResults(self,targetedSentiment_results,threshold = 0.4):
+        sents = dict()
+        for grp in targetedSentiment_results:
+            for mention in grp["Mentions"]:
+                if mention['Score'] >= threshold:
+                    if not "NEUTRAL" in mention['MentionSentiment']['Sentiment']:
+                        k = mention['MentionSentiment']['Sentiment']
+                        text = str.lower(mention['Text'])
+                        sents.setdefault(k,{text}).add(text)
+        for k,v in sents.items():
+            sents[k] = list(v) # change set to list
+        return sents
+
+    # Parse targeted sentiment results for named entities
+    def parse_namedEntities(self,targetedSentimentResults,threshold = 0.1):
+        ents = dict()
+        for grp in targetedSentimentResults:
+            for mention in grp["Mentions"]:
+                if mention['Score'] >= threshold:
+                    k = mention['Type']
+                    text = str.lower(mention['Text'])
+                    ents.setdefault(k,{text}).add(text)
+        for k,v in ents.items():
+            ents[k] = list(v) # change set to list
+        return ents       
+

reflexive/analyse/general.py

@@ -0,0 +1,128 @@
+
+import logging,coloredlogs
+import pandas as pd
+import json
+
+from reflexive.common.parameters import Parameters
+from reflexive.common.util import Util
+
+coloredlogs.install(level='INFO')
+
+class General:
+    
+    logger = logging.getLogger(__name__)
+    
+    def __init__(self,parameters:Parameters):
+        #print(parameters)
+        self.__parameters = parameters.all_parameters()
+        self.logger.debug(f"Parameters: {self.__parameters}")
+
+
+    def general_analytics(self,df):
+        util = Util()
+        custom_df = df.copy() 
+        custom_df["text_length"] = df.text.apply(lambda x: len(x))
+        if (len(custom_df)>1):
+            custom_df["text_scaled"] = util.scale_min_max(custom_df[['text_length']])
+        else:
+            custom_df["text_scaled"] = 1
+        return custom_df
+
+
+    def remove_IQR_outliers(self,df):
+        tempdf = df.copy()
+        # Calculate text length
+        tempdf["text_length"] = tempdf.text.apply(lambda t: len(t))
+        fence = Util.outlier_fence(tempdf.text_length)  
+        print(fence)
+        # Check change with removed outliers
+        checkdf = tempdf[tempdf.text_length<fence['UPPER']]
+        checkdf.reset_index(drop=True,inplace=True)
+        print("Original:",len(tempdf))
+        print(tempdf.describe())
+        print()
+        print("Outliers:",len(tempdf)-len(checkdf))
+        print()
+        print("No outliers:",len(checkdf))
+        print(checkdf.describe())
+        return checkdf
+
+    # Parse text for domain terms
+    def parse_domain_terms(self,text,domain_terms):
+        matched_terms = {}
+        for dtk,dtv in domain_terms.items():
+            matched_terms[dtk] = []
+            for term in dtv:
+                if term[0]=='_': #acronym - treat as whole word
+                    regex = r"\b{}\b".format(term[1:])
+                    matches = re.findall(regex,str.lower(text))
+                    if len(matches)>0:
+                        matched_terms[dtk].append((term[1:],len(matches)))
+                else:
+                    count = str.lower(text).count(term)
+                    if count > 0:
+                        matched_terms[dtk].append((term,count))
+        return matched_terms
+    
+    
+    def get_top_ngrams(self,text_series,min_val=3):
+        ngrams = {}
+        for text in text_series:
+            self.__ngrams345(text,ngrams)
+        #print("Generated 3,4,5 ngrams:", len(ngrams))
+        f_ngrams = self.filter_dict_by_value(ngrams,min_val)
+        return self.sort_dict_by_value(f_ngrams)
+
+    def get_top_ngrams_for_text(self,text,top_ngrams):
+        ngrams = self.__ngrams345(text,{})
+        return {key: ngrams[key] for key in top_ngrams.keys() if key in ngrams}
+
+    def ngram_counts(self,ref_top_ngrams):
+        return sum(ref_top_ngrams.values())
+    
+        # Given text and number of terms, create ngrams from the text
+    def __make_ngrams(self,text, n=1):
+        # Replace all none alphanumeric characters with spaces
+        s = re.sub(r'[^a-zA-Z0-9\s]', ' ', text.lower())
+        
+        tokens = [token for token in s.split(" ") if token != ""]
+        ngrams = zip(*[tokens[i:] for i in range(n)])
+        return [" ".join(ngram) for ngram in ngrams]
+
+    # Generate 3,4,5 -grams
+    def __ngrams345(self,text,ngrams):
+        ngrams3 = self.__make_ngrams(text,3)
+        for n in ngrams3:
+            ngrams[n] = ngrams.get(n,0)+1
+        ngrams4 = self.__make_ngrams(text,4)
+        for n in ngrams4:
+            ngrams[n] = ngrams.get(n,0)+1
+        ngrams5 = self.__make_ngrams(text,5)
+        for n in ngrams5:
+            ngrams[n] = ngrams.get(n,0)+1
+        return ngrams
+    
+
+    # Count domain terms
+    def count_domain_terms(self,terms):
+        counts = {}
+        for k,v in terms.items():
+            for term in v:
+                counts[k] = counts.setdefault(k,0) + term[1]
+        return counts
+
+
+    # Ratio between action POS and object POS
+    def action_object_ratio(self,pos_ratios,action_pos = ['VERB'],object_pos = ['NOUN','PROPN']):
+        ap = [s[1] for s in pos_ratios if s[0] in action_pos]
+        if ap:
+            aps = sum(ap)
+        else:
+            aps = 0
+        op = [s[1] for s in pos_ratios if s[0] in object_pos]
+        if op:
+            ops = sum(op)
+        else:
+            ops = 1 #avoid divide zero error - only happens with aps of 1
+            #print("aps",aps,"ops",ops)
+        return aps/ops

reflexive/analyse/reflexive_expressions.py

@@ -0,0 +1,124 @@
+#
+
+import logging,coloredlogs
+import pandas as pd
+import json
+
+from reflexive.common.parameters import Parameters
+from reflexive.common.local import Local
+from reflexive.common.util import Util
+from reflexive.aws_connect.s3 import S3
+from reflexive.aws_connect.comprehend import Comprehend
+
+
+coloredlogs.install(level='INFO')
+
+class ReflexiveExpressions:
+    
+    logger = logging.getLogger(__name__)
+    
+    def __init__(self,parameters:Parameters,aws_s3:S3,local:Local,comprehend:Comprehend):
+        #print(parameters)
+        self.__params = parameters
+        self.__parameters = parameters.all_parameters()
+        self.logger.debug(f"Parameters: {self.__parameters}")
+        self.prefix = self.__parameters['prefix']
+        self.postfix = self.__parameters['postfix']
+        self.local_path = self.__parameters['local_path']
+        self.__s3 = aws_s3
+        self.__local = local
+        self.__comprehend = comprehend
+
+        
+    ######## REFLEXIVE EXPRESSION ANALYSIS FUNCTIONS
+
+    def analyse_reflexive_expressions(self,df): #,s3_bucket_name,access_role_arn,entity_recogniser_arn):
+        #self.__bucket_name = s3_bucket_name
+        text = df.text.replace('\r\n','\n') # Comprehend treats \r\n as one character
+        # Upload reflections to S3 for analysis
+        self.__s3.upload_docs(text)
+        
+        # Save a copy of reflections locally for offline analysis
+        self.__local.save_docs(text)
+
+        # Submit the job
+        return self.__comprehend.submit_custom_entity_job("reflexive_expressions_analysis") #submitReflexiveExpressionsJob(access_role_arn, entity_recogniser_arn)
+    
+    def check_job_status(self):
+        return self.__comprehend.check_job_status()
+    
+    def get_job_details(self):
+        return self.__comprehend.get_job_details()
+
+    def download_and_extract(self):
+        local_output_dir = f"{self.local_path}{self.prefix}output{self.postfix}"
+        job_details = self.get_job_details()
+        s3Uri = job_details['OutputDataConfig']['S3Uri']
+        return self.__s3.results_download_save_extract(s3Uri,local_output_dir)
+
+    def extractAnalysisFromResults(self,results):
+        analysis_output = dict()
+        for result in results:
+            j = json.loads(result)
+            #print(j)
+            idx = j["File"].split('_')[-1].split('.')[0]
+            analysis_output[int(idx)] = j["Entities"]
+        return analysis_output
+    
+    def add_to_dataframe(self,df,results):
+        # Extract analysis from raw results
+        analysis_output = self.extractAnalysisFromResults(results)
+        # Add results to dataframe
+        results_df = df.copy()
+        results_df['reflexiveResults'] = pd.Series(analysis_output)
+        return results_df
+
+    def reflexive_analytics(self,df):
+        util = Util()
+        custom_df = df.copy() 
+        # custom_df["text_length"] = df.text.apply(lambda x: len(x))
+        # if (len(custom_df)>1):
+        #     custom_df["text_scaled"] = util.scale_min_max(custom_df[['text_length']])
+        # else:
+        #     custom_df["text_scaled"] = 1
+        custom_df["reflexive_results"] = df.reflexiveResults
+        # The expressions and their reflexive expression labels
+        custom_df["reflexive_expressions"] = df.reflexiveResults.apply(self.parse_reflexiveResults)
+        # The counts for each labels
+        custom_df["reflexive_counts"] = custom_df.reflexive_expressions.apply(util.count_labels)
+        # Ratios between reflexive expressions
+        custom_df["reflexive_ratio"] = custom_df.reflexive_counts.apply(util.ratios)
+        # Ratio vector
+        custom_df['ratio_vector'] = custom_df.reflexive_ratio.apply(self.make_ratio_vector)
+        # Get the diversity of reflexive types - out of 8 possible types
+        custom_df["reflexive_type_diversity"] = custom_df.reflexive_counts.apply(lambda x: len(x)/8)
+        # A total of all labels
+        custom_df["reflexive_total"] = custom_df.reflexive_counts.apply(util.tuple_values_total)
+        # MinMax scale the reflexive_total
+        if (len(custom_df)>1):
+            custom_df["reflexive_scaled"] = util.scale_min_max(custom_df[['reflexive_total']])
+        else:
+            custom_df["reflexive_scaled"] = 1
+        # Normalise based on text_scaled
+        custom_df['reflexive_norm'] = util.normalise_scaled(custom_df,'reflexive_scaled')
+        return custom_df
+
+
+    # Parse reflexive results - include all above threshold
+    def parse_reflexiveResults(self,reflexiveResults,threshold=0.5):
+        final_refs = list()
+        for ref in reflexiveResults:
+            if ref['Score'] > threshold:
+                final_refs.append((str.lower(ref['Text']),ref['Type']))
+        return final_refs
+    
+        # Function for creating a vector out of reflexive ratio - could be used for others
+    def make_ratio_vector(self,ratio_list,ref_codes = ['RR','ER','VR','AR','EP','AF','CN','EV']):
+        ratio_dict = dict(ratio_list)
+        vec = []
+        for rc in ref_codes:
+            if rc in ratio_dict.keys():
+                vec.append(ratio_dict[rc])
+            else:
+                vec.append(0)
+        return vec

reflexive/aws_connect/comprehend.py

@@ -0,0 +1,205 @@
+#
+
+
+import boto3
+import time
+import json
+import pandas as pd
+
+from reflexive.common.parameters import Parameters
+from reflexive.common.util import Util
+from reflexive.aws_connect.s3 import S3
+
+import logging
+try:
+    import coloredlogs
+    coloredlogs.install(level='INFO')
+except:
+    print("Colored logs not available")
+
+class Comprehend:
+    
+    logger = logging.getLogger(__name__)
+    
+    def __init__(self,parameters:Parameters):
+        #print(parameters)
+        self.__parameters = parameters.all_parameters()
+        self.logger.debug(f"Parameters: {self.__parameters}")
+        self.region = self.__parameters['region']
+        self.access_role_arn = self.__parameters['comprehend_access_role_arn']
+        self.entity_recogniser_arn = self.__parameters['reflexive_entity_arn']
+        self.local_path = self.__parameters['local_path']
+        self.prefix = self.__parameters['prefix']
+        self.postfix = self.__parameters['postfix']
+        self.bucket_name = self.__parameters["bucket_name"]
+        self.files_folder = f"{self.prefix}files{self.postfix}"
+        self.results_folder = f"{self.prefix}results{self.postfix}"
+        self.input_uri = f"s3://{self.bucket_name}/{self.files_folder}/{self.prefix}"
+        self.output_uri = f"s3://{self.bucket_name}/{self.results_folder}/"
+        self.analysis_types = self.__parameters['analysis_types']
+        # create client
+        try:
+            self.logger.debug(f"Region:{self.region}")
+            self.__comp_client = boto3.client(service_name='comprehend',region_name=self.region)
+        except Exception as err:
+            self.logger.error("Unable to create Comprehend client: ",err)
+            
+        
+    def client(self):
+        return self.__comp_client
+        
+        
+#### CUSTOM ENTITY
+
+    def submit_custom_entity_job(self,job_name): #access_role_arn,entity_recogniser_arn):
+        job_str = f"{self.prefix}{job_name}{self.postfix}"
+    
+        response = self.__comp_client.start_entities_detection_job(
+            InputDataConfig={
+                'S3Uri': self.input_uri,
+                'InputFormat': 'ONE_DOC_PER_FILE'
+            },
+            OutputDataConfig={
+                'S3Uri': self.output_uri
+            },
+            DataAccessRoleArn=self.access_role_arn,
+            JobName=job_str,
+            EntityRecognizerArn=self.entity_recogniser_arn,
+            LanguageCode='en'
+        )
+        self.job_id = response['JobId']
+        return response
+    
+            # Check job status
+    def check_job_status(self):
+        job_status = self.__comp_client.describe_entities_detection_job(
+            JobId=self.job_id
+        )
+        self.__job_properties = job_status['EntitiesDetectionJobProperties'] 
+        return self.__job_properties['JobStatus']
+
+    def get_job_details(self):
+        return self.__job_properties
+    
+        
+    # Use AWS comprehend to get bulk key phrases from single batch of chunked text
+    def get_single_batch_analysis(self,index,chunk):
+        comprehend = self.client()
+        results = {}
+        print("Analysing chunk",index)
+        print(" . key_phrase")
+        kpresult = comprehend.batch_detect_key_phrases(TextList=chunk,LanguageCode='en')
+        results['KeyPhraseResults'] = kpresult
+        #key_phrase_results.append(kpresult)
+        time.sleep(2)
+        print(" . sentiment")
+        senresult = comprehend.batch_detect_sentiment(TextList=chunk,LanguageCode='en')
+        results['SentimentResults'] = senresult
+        #sentiment_results.append(senresult)
+        time.sleep(2)
+        print(" . targeted_sentiment")
+        tsenresult = comprehend.batch_detect_targeted_sentiment(TextList=chunk,LanguageCode='en')
+        results['TargetedSentimentResults'] = tsenresult
+        #target_sent_results.append(tsenresult)
+        time.sleep(2)
+        print(" . syntax")
+        synresult = comprehend.batch_detect_syntax(TextList=chunk,LanguageCode='en')
+        results['SyntaxResults'] = synresult
+        #syntax_results.append(synresult)       
+        time.sleep(2)
+        return results
+
+
+    # Use AWS comprehend to get bulk key phrases from chunked text
+    def get_multiple_batch_analysis(self,chunked_text):
+        chunk_results = {}
+        for key in self.analysis_types.keys():
+            chunk_results[key] = []
+                
+        for idx,chunk in enumerate(chunked_text):
+            if len(chunked_text) > 4999:
+                print("WARNING: Text too long to analyse - index",idx,"skipped!")
+            else:
+                try:
+                    results = self.get_single_batch_analysis(index=idx,chunk=chunk)
+                except(Exception) as error:
+                    print("There was an error with index",idx,error)
+                finally:
+                    if results:
+                        for key in results.keys():
+                            chunk_results[key].append(results[key])
+
+        return chunk_results
+
+    # Take batched responses and concenate single lists of results, errors, and http responses
+    def unbatch_results(self,result_type,results,batch_size=25):
+        unbatched_results = {}
+        unbatched_errors = {}
+        batch_responses = {}
+        for idx,batch in enumerate(results):
+            #print("Response for batch:",idx)
+            batch_responses[idx] = batch['ResponseMetadata']
+            result_list = batch['ResultList']
+            error_list = batch['ErrorList']
+            for r in result_list:
+                ridx = idx*batch_size + r['Index']
+                rdata = r[result_type]
+                unbatched_results[ridx] = rdata
+            for e in error_list:
+                eidx = e['Index']
+                unbatched_errors[eidx] = 'ERROR' + e['ErrorCode'] + ': ' + e['ErrorMessage']
+        unbatched = {}
+        unbatched['results'] = unbatched_results
+        unbatched['errors'] = unbatched_errors
+        unbatched['responses'] = batch_responses
+        return unbatched
+
+
+
+    def check_long_text(self,df):
+        # Check for long reflections (too long for batch analysis)
+        long_df = df.copy()
+        long_df = long_df[long_df.text.str.len()>5000]
+        long_df['length'] = long_df.text.str.len()
+        return long_df
+    
+    
+    # def extract_result(self,result,batch,batch_params):
+    #     match batch:
+    #         case "KeyPhraseResults":
+    #             extracted = [r['Text'] for r in result if r['Score'] >= batch_params["min_score"]]
+    #         case "SentimentResults":
+    #             extracted = result
+    #         case "TargetedSentimentResults":
+    #             extracted = dict()
+    #             for r in result:
+    #                 for mention in r['Mentions']:
+    #                     if (mention['Score'] >= batch_params["min_score"]):
+    #                         text = mention['Text']
+    #                         key = f"{mention['Type']}_{mention['MentionSentiment']['Sentiment']}"
+    #                         if key in extracted.keys():
+    #                             extracted[key].add(text)
+    #                         else:
+    #                             extracted[key] = {text}
+    #         case "SyntaxResults":
+    #             tags = []
+    #             tokens = []
+    #             for r in result:
+    #                 pos = r['PartOfSpeech']
+    #                 tag = pos['Tag']
+    #                 if pos['Score'] < batch_params["max_score"]:
+    #                     tag = tag+"_?"
+    #                 tags.append(tag)
+    #                 tokens.append(r['Text'])
+
+    #             extracted = {'tokens':tokens,'tags':tags}
+    #         case other:
+    #             extracted = []
+    #     return extracted
+
+
+
+
+
+
+