reflexive 0.1.9__py3-none-any.whl → 1.0.13__py3-none-any.whl

reflexive/session.py

@@ -0,0 +1,265 @@
+
+import boto3
+import time
+import tarfile
+import json
+
+from reflexive import cfg
+import reflexive as rfx
+
+import logging
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger(__name__)
+
+class AWS:
+    
+    config = None
+    aws_session = None
+
+    def __init__(self,config:cfg.Config):
+        # on initialisation create a new session with provided profile (or with default profile)
+        #logger.error(config.get_parameters())
+        if config==None:
+            config = cfg.Config()
+        self.config = config
+        self.new_session()
+        
+    def get_parameters(self):
+        return self.__dict__
+        
+    def new_session(self):
+        logger.info("In new_session")
+        try:
+            self.aws_session = boto3.Session(profile_name=self.config.aws_profile)
+            self.config.aws_region = self.aws_session.region_name
+            self.config.aws_access_key = self.aws_session.get_credentials().access_key
+            logger.info("Created new AWS session in region %s for profile: %s",self.config.aws_region,self.config.aws_profile)
+            
+        except Exception as e:
+            logger.error("Unable to create an AWS session: %s",repr(e))
+            
+        try:
+            self.config.aws_account_number = self.aws_session.client('sts').get_caller_identity().get('Account')
+            logger.info("Retrieved account number from AWS")
+        except Exception as e:
+            logger.error("Unable to retrieve account number from AWS: %s",repr(e))
+        
+        return self.aws_session
+            
+            
+class S3:
+    
+    aws = None
+    config = None
+    __s3_client = None
+    
+    def __init__(self,aws:AWS):
+        self.aws = aws
+        self.config = self.aws.config
+
+        # create client
+        try:
+            logger.debug(f"Region:{self.aws.aws_session.region_name}")
+            self.__s3_client = aws.aws_session.client(service_name='s3')
+        except Exception as err:
+            logger.error("Unable to create S3 client: ",err)
+    
+    # Return the S3 client
+    def client(self):
+        return self.__s3_client
+    
+     # Function to upload reflections to S3
+    def upload_docs(self,text_series):
+
+        files_folder = f"{self.config.prefix}files{self.config.postfix}"
+
+        s3 = self.__s3_client
+        s3ap = self.config.s3_accesspoint_arn
+        logger.debug(f"ACCESS POINT: {s3ap}")
+
+        logger.info(f"Uploading {len(text_series)} reflections to S3 ({files_folder})...")
+        logger.debug(f"({s3ap}/{files_folder})")
+        for idx in text_series.index:
+            file_name = f"{self.config.prefix}{idx}.txt"
+            file_body = text_series.iloc[idx]
+            logger.info(f"Uploading {file_name}")
+            #print(file_body)
+            response = s3.put_object(Body=file_body,Bucket=s3ap,Key=f"{files_folder}/{file_name}")
+            if response['ResponseMetadata']['HTTPStatusCode'] != 200:
+                logger.error("------------------------------------------------------------")
+                logger.error(f"ERROR: There was a problem with {file_name}")
+                logger.error(response)
+                logger.error("------------------------------------------------------------")
+            else:
+                logger.info('Success')
+        logger.info("Finished uploading reflections to S3.")
+        return response
+    
+    # download and save results
+    def results_download_save_extract(self,s3Uri,local_file_path):
+        s3 = self.__s3_client
+        output_key = s3Uri.split(self.config.s3_bucket_name)[1]
+        # download from S3 to local path
+        with open(f"{local_file_path}.tar.gz",'wb') as output_data:
+            s3.download_fileobj(self.config.s3_bucket_name,output_key[1:],output_data)
+
+        # extract the files from tar archive
+        files = list()
+        with tarfile.open(f"{local_file_path}.tar.gz", "r:gz") as tf:
+            for member in tf.getmembers():
+                f = tf.extractfile(member)
+                if f is not None:
+                    content = f.read()
+                    files.append(content)
+        #print("Number of files:",len(files))
+        # extract results and save and return
+        raw_results = files[0].decode("utf-8").split('\n')
+        raw_results.pop() # pop last item off as empty entry due to final \n
+        json_results = json.dumps(raw_results)
+        with open(f"{local_file_path}.json","w") as fp:
+            fp.write(json_results)
+        return json_results
+
+
+class Comprehend:
+    
+    aws = None
+    config = None
+    __comp_client = None
+    
+    def __init__(self,aws:AWS):
+        self.aws = aws
+        self.config = self.aws.config
+    
+        # create client
+        try:
+            logger.debug(f"Region:{self.aws.aws_session.region_name}")
+            self.__comp_client = self.aws.aws_session.client(service_name='comprehend')
+        except Exception as err:
+            logger.error("Unable to create Comprehend client: ",err)
+                
+    def client(self):
+        return self.__comp_client
+        
+    # Use AWS comprehend to get bulk key phrases from single batch of chunked text
+    def get_single_batch_analysis(self,index,chunk):
+        comp_client = self.client()
+        results = {}
+        print("Analysing chunk",index)
+        print(" . key_phrase")
+        kpresult = comp_client.batch_detect_key_phrases(TextList=chunk,LanguageCode='en')
+        results['KeyPhraseResults'] = kpresult
+        #key_phrase_results.append(kpresult)
+        time.sleep(2)
+        print(" . sentiment")
+        senresult = comp_client.batch_detect_sentiment(TextList=chunk,LanguageCode='en')
+        results['SentimentResults'] = senresult
+        #sentiment_results.append(senresult)
+        time.sleep(2)
+        print(" . targeted_sentiment")
+        tsenresult = comp_client.batch_detect_targeted_sentiment(TextList=chunk,LanguageCode='en')
+        results['TargetedSentimentResults'] = tsenresult
+        #target_sent_results.append(tsenresult)
+        time.sleep(2)
+        print(" . syntax")
+        synresult = comp_client.batch_detect_syntax(TextList=chunk,LanguageCode='en')
+        results['SyntaxResults'] = synresult
+        #syntax_results.append(synresult)       
+        time.sleep(2)
+        return results
+
+
+    # Use AWS comprehend to get bulk key phrases from chunked text
+    def get_multiple_batch_analysis(self,chunked_text):
+        chunk_results = {}
+        for key in self.config.analysis_types.keys():
+            chunk_results[key] = []
+                
+        for idx,chunk in enumerate(chunked_text):
+            if len(chunked_text) > 4999:
+                print("WARNING: Text too long to analyse - index",idx,"skipped!")
+            else:
+                try:
+                    results = self.get_single_batch_analysis(index=idx,chunk=chunk)
+                except(Exception) as error:
+                    print("There was an error with index",idx,error)
+                finally:
+                    if results:
+                        for key in results.keys():
+                            chunk_results[key].append(results[key])
+
+        return chunk_results
+
+    # Take batched responses and concenate single lists of results, errors, and http responses
+    def unbatch_results(self,result_type,results,batch_size=25):
+        unbatched_results = {}
+        unbatched_errors = {}
+        batch_responses = {}
+        for idx,batch in enumerate(results):
+            #print("Response for batch:",idx)
+            batch_responses[idx] = batch['ResponseMetadata']
+            result_list = batch['ResultList']
+            error_list = batch['ErrorList']
+            for r in result_list:
+                ridx = idx*batch_size + r['Index']
+                rdata = r[result_type]
+                unbatched_results[ridx] = rdata
+            for e in error_list:
+                eidx = e['Index']
+                unbatched_errors[eidx] = 'ERROR' + e['ErrorCode'] + ': ' + e['ErrorMessage']
+        unbatched = {}
+        unbatched['results'] = unbatched_results
+        unbatched['errors'] = unbatched_errors
+        unbatched['responses'] = batch_responses
+        return unbatched
+
+    def check_long_text(self,df):
+        # Check for long reflections (too long for batch analysis)
+        long_df = df.copy()
+        long_df = long_df[long_df.text.str.len()>5000]
+        long_df['length'] = long_df.text.str.len()
+        return long_df
+    
+# #### CUSTOM ENTITY
+
+    def submit_custom_entity_job(self,job_name): #access_role_arn,entity_recogniser_arn):
+        job_str = f"{self.config.prefix}{job_name}{self.config.postfix}"
+        
+        response = self.__comp_client.start_entities_detection_job(
+            InputDataConfig={
+                'S3Uri': self.config.s3_input_uri,
+                'InputFormat': 'ONE_DOC_PER_FILE'
+            },
+            OutputDataConfig={
+                'S3Uri': self.config.s3_output_uri
+            },
+            DataAccessRoleArn=self.config.comprehend_access_role_arn,
+            JobName=job_str,
+            EntityRecognizerArn=self.config.reflexive_entity_arn,
+            LanguageCode='en'
+        )
+        self.job_id = response['JobId']
+        self.check_job_status() # force the creation of __job_properties
+        return response
+    
+    def get_current_job_id(self):
+        return self.job_id
+    
+            # Check job status
+    def check_job_status(self):
+        job_status = self.__comp_client.describe_entities_detection_job(
+            JobId=self.job_id
+        )
+        self.__job_properties = job_status['EntitiesDetectionJobProperties'] 
+        return self.__job_properties['JobStatus']
+
+    def get_job_details(self):
+        return self.__job_properties
+    
+        #checked
+    def download_and_extract(self,s3):
+        local_output_dir = f"{self.config.local_path}{self.config.prefix}output{self.config.postfix}"
+        job_details = self.get_job_details()
+        s3Uri = job_details['OutputDataConfig']['S3Uri']
+        return s3.results_download_save_extract(s3Uri,local_output_dir)
+       

reflexive/util.py

@@ -0,0 +1,125 @@
+import os
+
+import logging
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger(__name__)
+
+from sklearn.preprocessing import MinMaxScaler
+
+# File functions
+def get_data_path_name(config,name,ext):
+        return f"{config.local_path}{config.prefix}{name}{config.postfix}.{ext}"
+        
+def set_sub_dir(config,sub_dir=None):
+    # check dir sub_dir exists
+    if sub_dir:
+        local_dir = f"{config.local_path}{sub_dir}/"
+        logger.debug(f"local_dir: {local_dir}")
+        dirExists = os.path.exists(local_dir)
+        if not dirExists:
+            logger.info(f"Creating subdirectory: {local_dir}")
+            os.makedirs(local_dir)
+    else:
+        local_dir = local_path
+    return local_dir
+        
+
+
+# Function to write dictionaries to both json and csv
+def writeDictJsonCSV(dictionary,path_file):
+    with open(f"{path_file}.json",'w') as fp:
+        fp.write(json.dumps(dictionary))
+
+    ngram_df = pd.DataFrame.from_dict(dictionary,orient='index')   
+    ngram_df.to_csv(f"{path_file}.csv")
+    
+# Data functions
+def sort_dict_by_value(d):
+    return dict(sorted(d.items(), key=lambda x:x[1], reverse=True))
+
+def filter_dict_by_value(ngrams,min_val=3):
+    filtered_ngrams = {}
+    for k,v in ngrams.items():
+        if v >=min_val:
+            filtered_ngrams[k] = v
+    return filtered_ngrams
+
+# Input a series and output a list of lists with each maxn elements
+def series_to_chunked_list(series,maxn=25):
+    l = list(series)
+    return __chunk_list(l,maxn)
+
+# Chunk a list into a list of lists with maxn elements
+def __chunk_list(l,maxn=25):
+    return [l[i:i + maxn] for i in range(0, len(l), maxn)]
+
+# Count named entities
+def count_entities(entities):
+    counts = []
+    for k,v in entities.items():
+        counts.append((k,len(v))) 
+    return sorted(counts, key=lambda x: x[1], reverse=True)
+
+# Function for calculating proportions of features
+def ratios(elements):
+    etotal = sum([v[1] for v in elements])
+    if etotal==0:
+        return elements
+    else:
+        proportioned = []
+        for element in elements:
+            prop_val = round((element[1]/etotal),4)
+            proportioned.append((element[0],prop_val))
+        return proportioned
+
+
+
+# Count labels associated with strings
+def count_labels(string_labels):
+    counts = dict()
+    for rt in string_labels:
+        counts[rt[1]] = counts.setdefault(rt[1],0) + 1
+    return sorted(counts.items(), key=lambda x: x[1], reverse=True)
+
+def count_keys(key_count_dict):
+    counts = dict()
+    for k,v in key_count_dict.items():
+        counts[k] = counts.setdefault(k,0) + v 
+    return sorted(counts.items(), key=lambda x: x[1], reverse=True)
+    
+# Total the values in list of tuples
+def tuple_values_total(tuples):
+    tvs = [t[1] for t in tuples]
+    return sum(tvs)
+
+#### SCALING AND NORMALISING
+
+# Outliers
+
+def outlier_fence(series):
+    bounds = {}
+    stats = series.describe()
+    iqr = stats['75%'] - stats['25%']
+    bounds["IQR"]=iqr
+    upper = stats['75%']+1.5*iqr
+    bounds["UPPER"]=upper
+    lower = stats['25%']-1.5*iqr
+    bounds["LOWER"]=lower
+    return bounds
+
+# MinMax Scaling
+def scale_min_max(df_cols):
+    scaler = MinMaxScaler()
+    return scaler.fit_transform(df_cols)
+
+# Normalise domain term counts
+def normalise_domain_counts(domain_counts,text_size):
+    norms = {}
+    for k,v in domain_counts.items():
+        norms[k] = round(v*text_size,3)
+    return norms
+
+def normalise_scaled(df,feature,norm_feature = 'text_scaled'):
+    tempdf = df[[feature,norm_feature]].copy()
+    tempdf['norm_scaled'] = tempdf.apply(lambda r: round(r[feature]/(r[norm_feature]+0.01),4),axis=1)
+    return tempdf['norm_scaled']

reflexive/{visual/display.py → visualise.py}

@@ -1,30 +1,30 @@
-
-#from spacy import displacy
-
-#from reflexive.common.parameters import Parameters
-
-import logging
-try:
-    import coloredlogs
-    coloredlogs.install(level='INFO')
-except:
-    print("Colored logs not available")
+from reflexive import session
+from reflexive import cfg
 
 class Display:
+    aws:session.AWS = None
+    config:cfg.Config = None
     
-    logger = logging.getLogger(__name__)
+    defaults = {
+        "priority_tags": ["AR","EP","VR_EV_CN","ER_AF","RR","KP"],
+        "colours": {"VR_EV_CN": "#ff6644","ER_AF": "#dd44cc","AR": "#00cc00","EP": "#aacc33","RR": "#00aaff","KP":"#aaaacc"}}
     
-    def __init__(self): #,parameters:Parameters):
-        self.name="Display"
-        self.priority_tags = ["AR","EP","VR_EV_CN","ER_AF","RR","KP"]
-        self.colours = {"VR_EV_CN": "#ff6644","ER_AF": "#dd44cc","AR": "#00cc00","EP": "#aacc33","RR": "#00aaff","KP":"#aaaacc"}
-        self.options = {"ents": ["VR_EV_CN","ER_AF","AR","EP","RR","KP"], "colors": self.colours}
-        
-
-
+    def __init__(self,aws):
+        self.aws = aws
+        self.aws = aws
+        self.config = self.aws.config
+        self.set_default_parameters()
+    
+    def set_default_parameters(self):
+        priority_tags = self.defaults['priority_tags']
+        colours = self.defaults['colours']
+        options = {"ents": colours.keys(), "colors": colours.values}
+        self.config.set_display_parameters(priority_tags,colours,options)
+       
+     
     def add_reflexive_offsets(self,df):
         temp_df = df.copy()
-        temp_df['reflexive_offsets'] = temp_df.reflexiveResults.apply(self.collect_reflexive_offsets)
+        temp_df['reflexive_offsets'] = temp_df.ReflexiveResults.apply(self.collect_reflexive_offsets)
         return temp_df
         
     def add_keyphrase_offsets(self,df):
@@ -49,7 +49,7 @@ class Display:
         #pseudonym = record['pseudonym']
         #point_round = record['point_round']
         #title = f"{pseudonym} ({point_round}) - {timestamp}"
-        tags = self.priority_tags
+        tags = self.config.display_priority_tags
         text = record['text']
         reflexive_offsets = record['reflexive_offsets']
         keyphrase_offsets = record['keyphrase_offsets']

{reflexive-0.1.9.dist-info → reflexive-1.0.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: reflexive
-Version: 0.1.9
+Version: 1.0.13
 Summary: Supports AWS Reflexive Expressions Analysis
 Home-page: https://github.com/nlytx/reflexive
 Author: Andrew Gibson

reflexive-1.0.13.dist-info/RECORD

@@ -0,0 +1,12 @@
+reflexive/__init__.py,sha256=Ke3gGqrVgPP2IRgifljQL8Ep3qVuuOf4LgZUkxdJQ2k,119
+reflexive/analyse.py,sha256=UzWwgjAFNjeWFkCQ2o99g2vWajf17_OtSq4dFCvuPYU,17489
+reflexive/cfg.py,sha256=Ges35G234P2lvOQHgPZQae5hMSOGyBsmp1bY_yQEKkk,4303
+reflexive/session.py,sha256=MbqwTsYTgq_e_gw3mb1eRv6USs-zZ2cTCrvUNWuKfAQ,10067
+reflexive/util.py,sha256=WQ1oyzDi1i8wQ6IBwBPk6IFy07YKhg-Ug2FsOGVJRJQ,3649
+reflexive/visualise.py,sha256=weBNqd3uiCEg3bvLLNDPVxSkdjapm1jZrTw8cU3uZx8,4032
+reflexive-1.0.13.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+reflexive-1.0.13.dist-info/LICENSE.txt,sha256=WNHhf_5RCaeuKWyq_K39vmp9F28LxKsB4SpomwSZ2L0,11357
+reflexive-1.0.13.dist-info/METADATA,sha256=oT1mUVaMPVDIZ7oQdqe628lqmJROtjJhXfwuzDaRWmk,12038
+reflexive-1.0.13.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+reflexive-1.0.13.dist-info/top_level.txt,sha256=pOMr-QGleRBRCFBozgvM-UUUmOjD_-naJfu1522E2V8,10
+reflexive-1.0.13.dist-info/RECORD,,

reflexive/analyse/aws_nlp.py

@@ -1,196 +0,0 @@
-
-
-from reflexive.common.parameters import Parameters
-from reflexive.common.local import Local
-from reflexive.aws_connect.comprehend import Comprehend
-from reflexive.common.util import Util
-
-import json
-import logging
-import pandas as pd
-
-try:
-    import coloredlogs
-    coloredlogs.install(level='INFO')
-except:
-    print("Colored logs not available")
-
-class Nlp:
-    
-    logger = logging.getLogger(__name__)
-    
-    def __init__(self,parameters:Parameters,local:Local,comprehend:Comprehend):
-        self.__parameters = parameters.all_parameters()
-        self.logger.debug(f"Parameters: {self.__parameters}")
-        self.local_path = self.__parameters['local_path']
-        self.prefix = self.__parameters['prefix']
-        self.postfix = self.__parameters['postfix']
-        self.analysis_types = self.__parameters['analysis_types']
-        self.__local = local
-        self.__comprehend = comprehend
-
-
-
-    #### COMPREHEND ANALYSIS
-
-    def comprehend_analysis(self,df):
-        util = Util()
-        comprehend = self.__comprehend
-        self.analysis_types = self.__parameters['analysis_types']
-        #print(type(df.text))
-        # chunk the text for batch analysis
-        chunked_text = util.series_to_chunked_list(series=df.text)
-        print("Number of chunks:",len(chunked_text))
-        # start batch analysis
-        chunked_results = comprehend.get_multiple_batch_analysis(chunked_text)
-        print("Finished Analysis.")
-        # write to file
-        print("Writing data to file...")
-        with open(f"{self.local_path}{self.prefix}analysis_chunks{self.postfix}.json", "w") as fp:
-            json.dump(chunked_results,fp)            
-        print("DONE!")
-        # unchunk
-        final_results = {}
-        for key in chunked_results.keys():
-            final_results[key] = comprehend.unbatch_results(self.analysis_types[key],chunked_results[key])
-            print("Finished Unbatching",key," -  Writing data to file...")
-            filename = f"{self.local_path}{self.prefix}{key}{self.postfix}.json"
-            with open(filename, "w") as fp:
-                json.dump(final_results[key],fp)            
-        print("DONE!")
-        # Save final_results for reload if necessary
-        with open(f"{self.local_path}{self.prefix}final_results{self.postfix}.json", "w") as fp:
-            json.dump(final_results,fp) 
-        return final_results
-    
-    def check_results(self,results):
-        print("Checking for errors...")
-        for key in results.keys():
-            errors = results[key]['errors']
-            print(f"Errors for {key}: {errors}")
-        print()
-        print("Checking that we have results for all docs")
-        for key in results.keys():
-            num_results= len(results[key]['results'])
-            print(f"Number of results for {key}: {num_results}")
-        return errors
-    
-    def add_results_to_df(self,results,df):
-        for key in results.keys():
-            rs = results[key]['results']
-            newresults = {}
-            for oldkey in rs.keys():
-                newresults[int(oldkey)] = rs[oldkey] # Need to change keys to int to properly add to dataframe
-            df[key] = pd.Series(newresults)
-        return df
-    
-    def nlp_analytics(self,df):
-        temp_df = df.copy()
-        temp_df = self.keyphrase_analytics(temp_df)
-        temp_df = self.named_entity_analytics(temp_df)
-        temp_df = self.targeted_sentiment_analytics(temp_df)
-        temp_df = self.syntax_analytics(temp_df)
-        return temp_df
-    
-    
-    def keyphrase_analytics(self,df):
-        util = Util()
-        df["key_phrases"] = df.KeyPhraseResults.apply(self.parse_keyPhraseResults)
-        df["key_phrase_counts"] = df.key_phrases.apply(util.count_keys)
-        df["key_phrases_total"] = df.key_phrase_counts.apply(util.tuple_values_total)
-        if (len(df)>1):
-            df["key_phrases_scaled"] = util.scale_min_max(df[['key_phrases_total']])
-        else:
-            df["key_phrases_scaled"] = 1
-        # Normalise based on text_scaled
-        df['key_phrases_norm'] = util.normalise_scaled(df,'key_phrases_scaled')
-        return df
-    
-    def named_entity_analytics(self,df):
-        util = Util()
-        df["named_entities"] = df.TargetedSentimentResults.apply(self.parse_namedEntities)
-        df['named_entity_counts'] = df.named_entities.apply(util.count_entities)
-        df["named_entity_ratios"] = df.named_entity_counts.apply(util.ratios)
-        df["named_entities_total"] = df.named_entity_counts.apply(util.tuple_values_total)
-        if (len(df)>1):
-            df["named_entities_scaled"] = util.scale_min_max(df[['named_entities_total']])
-        else:
-            df["named_entities_scaled"] = 1
-        df['named_entities_norm'] = util.normalise_scaled(df,'named_entities_scaled')
-        return df
-    
-    def targeted_sentiment_analytics(self,df):
-        util = Util()
-        df["targeted_sentiment"] = df.TargetedSentimentResults.apply(self.parse_targetedSentimentResults)
-        df['targeted_sentiment_counts'] = df.targeted_sentiment.apply(util.count_entities)
-        df["targeted_sentiment_ratios"] = df.targeted_sentiment_counts.apply(util.ratios)
-        df["targeted_sentiment_total"] = df.targeted_sentiment_counts.apply(util.tuple_values_total)
-        if (len(df)>1):
-            df["targeted_sentiment_scaled"] = util.scale_min_max(df[['targeted_sentiment_total']])
-        else:
-            df["targeted_sentiment_scaled"] = 1
-        df['targeted_sentiment_norm'] = util.normalise_scaled(df,'targeted_sentiment_scaled')
-        return df
-    
-    def syntax_analytics(self,df):
-        util = Util()
-        df["pos_tags"] = df.SyntaxResults.apply(self.parse_syntaxResults)
-        df['pos_tag_counts'] = df.pos_tags.apply(util.count_labels)
-        df["pos_tag_ratios"] = df.pos_tag_counts.apply(util.ratios)
-        df["pos_tags_total"] = df.pos_tag_counts.apply(util.tuple_values_total)
-        if (len(df)>1):
-            df["pos_tags_scaled"] = util.scale_min_max(df[['pos_tags_total']])
-        else:
-            df["pos_tags_scaled"] = 1
-        df['pos_tags_norm'] = util.normalise_scaled(df,'pos_tags_scaled')
-        return df    
-    
-    
-    # Parse key_phrases results - include all above threshold
-    def parse_keyPhraseResults(self,keyPhraseResults,threshold=0.95,min_count=1):
-        util = Util()
-        phrases = {}
-        filtered = [str.lower(r['Text']) for r in keyPhraseResults if r['Score'] > threshold]
-        for phrase in filtered:
-            phrases[phrase] = phrases.get(phrase,0)+1
-
-        filtered_phrases = {k:v for k,v in phrases.items() if v >= min_count}
-        return util.sort_dict_by_value(filtered_phrases)
-
-    # Parse syntax results - include specific postags
-    def parse_syntaxResults(self,syntax_results,postags_keep = ['ADV','VERB','AUX','ADJ','NOUN','PRON','PROPN']):
-        sequence = list()
-        for token in syntax_results:
-            tag = token['PartOfSpeech']['Tag']
-            if tag in postags_keep:
-                sequence.append((str.lower(token['Text']),tag))
-        return sequence
-
-    # Parse targeted sentiment results - keep non-neutral above threshold
-
-    def parse_targetedSentimentResults(self,targetedSentiment_results,threshold = 0.4):
-        sents = dict()
-        for grp in targetedSentiment_results:
-            for mention in grp["Mentions"]:
-                if mention['Score'] >= threshold:
-                    if not "NEUTRAL" in mention['MentionSentiment']['Sentiment']:
-                        k = mention['MentionSentiment']['Sentiment']
-                        text = str.lower(mention['Text'])
-                        sents.setdefault(k,{text}).add(text)
-        for k,v in sents.items():
-            sents[k] = list(v) # change set to list
-        return sents
-
-    # Parse targeted sentiment results for named entities
-    def parse_namedEntities(self,targetedSentimentResults,threshold = 0.1):
-        ents = dict()
-        for grp in targetedSentimentResults:
-            for mention in grp["Mentions"]:
-                if mention['Score'] >= threshold:
-                    k = mention['Type']
-                    text = str.lower(mention['Text'])
-                    ents.setdefault(k,{text}).add(text)
-        for k,v in ents.items():
-            ents[k] = list(v) # change set to list
-        return ents       
-