reflexive 1.2.8__py3-none-any.whl → 2.0.0__py3-none-any.whl

reflexive/res.py

@@ -1,225 +0,0 @@
-import json
-from pandas import DataFrame
-from spacy import displacy
-from time import sleep
-from reflexive import Config
-from reflexive import AWS
-from reflexive import S3
-from reflexive import Comprehend
-from reflexive import Nlp
-from reflexive import Display
-from reflexive import RES_graph
-
-class Res_analyse:
-    
-    config:Config
-    aws:AWS
-    s3:S3
-    comprehend:Comprehend
-    nlp:Nlp
-    
-    def __init__(self,aws_profile="default") -> None:
-        return self._setup(aws_profile)
-    
-    def _setup(self,aws_profile) -> None:
-        self.config = Config(aws_profile)
-        self.config.set_parameters(name_prefix="RES")
-        self.aws = AWS(self.config)
-        return None
-    
-    def set_parameters(self,
-                       s3_access_point:str,
-                       s3_bucket_name:str,
-                       comprehend_service_role_name:str,
-                       reflexive_entity_name:str,
-                       reflexive_entity_version:str) -> str:
-        self.config.set_s3_parameters(s3_access_point,s3_bucket_name)
-        self.config.set_comprehend_parameters(comprehend_service_role_name)
-        self.config.set_comprehend_custom_entity_parameters(reflexive_entity_name,reflexive_entity_version)
-        params = self.config.get_parameters()
-        return json.dumps(params, indent=2)
-    
-    def setup_aws(self) -> None:
-        # Create a new S3 client
-        self.s3 = S3(self.aws)
-        # Create a new Comprehend client
-        self.comprehend = Comprehend(self.aws)
-        # Create an Nlp object to perform analysis on the text
-        self.nlp = Nlp(self.aws)
-        return None
-    
-    def get_basic_analytics(self,df:DataFrame) -> DataFrame:
-        
-        # Text length - this is needed for comprehend analytics
-        df = self.nlp.text_length(df)
-        #df = nlp.remove_IQR_outliers(df)
-        # Comprehend analysis
-        results = self.nlp.comprehend_analysis(self.comprehend,df)
-        #print(results)
-        errors = self.nlp.check_results(results)
-        #print(errors)
-        if errors=={}:
-            print("No errors, so adding results to dataframe")
-            df = self.nlp.add_results_to_df(results,df)
-            df = self.nlp.comprehend_analytics(df)
-        return df
-    
-    def get_reflexive_analytics(self,df:DataFrame) -> DataFrame:
-        # Reflexive expression analysis
-        response = self.nlp.analyse_reflexive_expressions(df,self.s3,self.comprehend)
-        #print(response)
-        job_id = self.comprehend.get_current_job_id()
-        print("Job ID:",job_id)
-        status = self.comprehend.check_job_status()
-        print("Status:",status)
-
-        # Get the details of the job
-        # details = comp.get_job_details()
-        # print("Job details:",details)
-
-        inc = 0
-        while status=="SUBMITTED" or status=="IN_PROGRESS":
-            print("Waiting 10 seconds...")
-            sleep(10)
-            status = self.comprehend.check_job_status()
-            print(f"Job status {inc}:",status)
-            inc += 1
-            
-        # Download from S3 and extract results 
-        print("Downloading and extracting results...")
-        results = self.comprehend.download_and_extract(self.s3)
-        print("RESULTS:")
-        print(results)
-        
-        # Extract output of analysis and add to df
-        return self.nlp.add_to_dataframe(df,results)
-
-class Res_display:
-    
-    res_analyse:Res_analyse
-    vis:Display
-    
-    def __init__(self,res:Res_analyse) -> None:
-        return self._setup(res)
-    
-    def _setup(self,res:Res_analyse) -> None:
-        self.res_analyse = res
-        self.vis = Display(res.aws)
-        return None  
-    
-    def show_text(self,df:DataFrame,inline=True) -> str:
-        df = self.vis.add_offsets(df)
-        disp_data = self.vis.create_displacy(df)
-        if inline:
-            displacy.render(disp_data,manual=True,style="ent", options=self.res_analyse.config.display_options)
-            html_out = "Set inline to false to produce HTML"
-        else:
-            html_out = displacy.render(disp_data,manual=True,style="ent", options=self.res_analyse.config.display_options,page=True,jupyter=False)
-        return html_out
-    
-    def get_interactions(self,df:DataFrame) -> DataFrame:
-        #Get RE sequence
-        df = self._add_res_sequence(df)
-        df = self._add_res_interactions(df)
-        df = self._add_res_weights(df)
-        df = self._add_res_adj_matrix(df)
-        return df
-    
-    def show_graph(self,df:DataFrame,scale=10,inline=True) -> str:   
-        for am in df.res_adj_matrix:
-            if scale > 1:
-                sm = self._scale_adj_matrix(am,scale)
-            else:
-                sm = am
-            g = RES_graph(sm)
-            g.show()
-        return ""
-    
-    def _scale_adj_matrix(self,adj_matrix,scale):
-        new_adj = []
-        for row in adj_matrix:
-            new_row = []
-            for c in row:
-                new_row.append(round(c*scale,1))
-            new_adj.append(new_row)
-        return new_adj
-    
-    def _add_res_sequence(self,df):
-        temp_df = df.copy()
-        temp_df['res_sequence'] = temp_df.reflexive_expressions.apply(self._get_res_sequence)
-        return temp_df
-    
-    def _add_res_interactions(self,df):
-        temp_df = df.copy()
-        temp_df['res_interactions'] = temp_df.res_sequence.apply(self._count_res_interactions)
-        return temp_df
-    
-    def _add_res_weights(self,df):
-        temp_df = df.copy()
-        temp_df['res_weights'] = temp_df.res_interactions.apply(self._calc_res_weights)
-        return temp_df
-    
-    def _add_res_adj_matrix(self,df):
-        temp_df = df.copy()
-        temp_df['res_adj_matrix'] = temp_df.res_weights.apply(self._create_adj_matrix)
-        return temp_df
-    
-    def _get_res_sequence(self,reflexive_expressions):
-        re_seq = [label for re,label in reflexive_expressions]
-        res_seq = []
-        # Need to substitute new RES labels for old RE labels
-        for re in re_seq:
-            if re=='ER' or re=='VR':
-                res_seq.append('NR')
-            elif re=='EV':
-                res_seq.append('EP')
-            elif re=='CN':
-                res_seq.append('AF')
-            else:
-                res_seq.append(re)
-        return res_seq
-    
-    def _empty_res_interactions(self) -> dict[tuple,int]:
-        RE_types = ['RR','NR','AR','AF','EP']
-        RE_interactions:dict[tuple,int] = dict()
-        for t1 in RE_types:
-            for t2 in RE_types:
-                entry = tuple(sorted((t1,t2)))
-                if entry not in RE_interactions.keys():
-                    RE_interactions[entry] = 0
-        return RE_interactions
-    
-    def _count_res_interactions(self,re_sequence:list[str]) -> dict[tuple,int]:
-        re_ints = self._empty_res_interactions()
-        limit = len(re_sequence)-1
-        for i,s in enumerate(re_sequence):
-            if i < limit:
-                rei = tuple(sorted((s,re_sequence[i+1])))
-                #print(i,rei)
-                re_ints[rei] += 1 
-        return re_ints
-    
-    def _calc_res_weights(self,interactions:dict[tuple,int])->dict[tuple,float]:
-        max_count = max(interactions.values())
-        weights = dict()
-        for edge,count in interactions.items():
-            weights[edge] = round(count/(max_count),2)
-        return weights
-            
-    
-    def _create_adj_matrix(self,weights:dict[tuple,float])->list[list[float]]:
-        re_types = ["RR","NR","AR","AF","EP"]
-        matrix = []
-        for r in re_types:
-            row = []
-            for c in re_types:
-                key = tuple(sorted((r,c)))
-                #print(key)
-                weight = weights.get(key,0)
-                row.append(weight)
-            matrix.append(row)
-        return matrix
-        
-        
-        
-        

reflexive/res_functions.py

@@ -1,62 +0,0 @@
-# import text into dataframe
-# accepts either an iterable of strings, or an iterable over text files
-# returns a pandas series iterable of type string
-
-# clean text and calculate length
-# accepts an iterable of strings in form of pandas series of type string
-# returns a pandas series iterable of type int
-
-
-# chunk text and keep original index ref
-# accepts an iterable of dataframe rows
-# returns an iterable of dataframe rows with added column 'text_chunks' - list of strings
-
-# upload docs to s3 and save local copy - side effects
-# accepts an iterable of iterable of chunks (with ids)
-# returns an an iterable of s3 responses? URLs to S3 file?
-
-# initiate custom entity job on comprehend
-# no parameters
-# returns job id for checking status, and downloading
-
-# check status
-# accepts job id
-# returns status
-
-# download results
-# accepts job id
-# returns iterable of results
-
-# unpack results and load into dataframe
-
-
-# extract reflexive expressions into dataframe
-
-
-# get reflexive sequences
-
-
-# get interactions
-
-
-# create count adj matrix
-
-
-# create weighted adj matrix
-
-
-# save dataframe to file
-
-
-# visualise expressions in text
-
-
-# visualise reflexive sequence
-
-
-# visualise res graph
-
-
-#
-# Network analysis functions
-#

reflexive/session.py

@@ -1,264 +0,0 @@
-
-import boto3
-import time
-import tarfile
-import json
-
-from reflexive import cfg
-
-import logging
-#logging.basicConfig(level=logging.DEBUG)
-logger = logging.getLogger(__name__)
-
-class AWS:
-    
-    config = None
-    aws_session = None
-
-    def __init__(self,config:cfg.Config):
-        # on initialisation create a new session with provided profile (or with default profile)
-        #logger.error(config.get_parameters())
-        if config is None:
-            config = cfg.Config()
-        self.config = config
-        self.new_session()
-        
-    def get_parameters(self):
-        return self.__dict__
-        
-    def new_session(self):
-        logger.info("In new_session")
-        try:
-            self.aws_session = boto3.Session(profile_name=self.config.aws_profile)
-            self.config.aws_region = self.aws_session.region_name
-            self.config.aws_access_key = self.aws_session.get_credentials().access_key
-            logger.info("Created new AWS session in region %s for profile: %s",self.config.aws_region,self.config.aws_profile)
-            
-        except Exception as e:
-            logger.error("Unable to create an AWS session: %s",repr(e))
-            
-        try:
-            self.config.aws_account_number = self.aws_session.client('sts').get_caller_identity().get('Account')
-            logger.info("Retrieved account number from AWS")
-        except Exception as e:
-            logger.error("Unable to retrieve account number from AWS: %s",repr(e))
-        
-        return self.aws_session
-            
-            
-class S3:
-    
-    aws = None
-    config = None
-    __s3_client = None
-    
-    def __init__(self,aws:AWS):
-        self.aws = aws
-        self.config = self.aws.config
-
-        # create client
-        try:
-            logger.debug(f"Region:{self.aws.aws_session.region_name}")
-            self.__s3_client = aws.aws_session.client(service_name='s3')
-        except Exception as err:
-            logger.error("Unable to create S3 client: ",err)
-    
-    # Return the S3 client
-    def client(self):
-        return self.__s3_client
-    
-     # Function to upload reflections to S3
-    def upload_docs(self,text_series):
-
-        files_folder = f"{self.config.prefix}files{self.config.postfix}"
-
-        s3 = self.__s3_client
-        s3ap = self.config.s3_accesspoint_arn
-        logger.debug(f"ACCESS POINT: {s3ap}")
-
-        logger.info(f"Uploading {len(text_series)} reflections to S3 ({files_folder})...")
-        logger.debug(f"({s3ap}/{files_folder})")
-        for idx in text_series.index:
-            file_name = f"{self.config.prefix}{idx}.txt"
-            file_body = text_series.iloc[idx]
-            logger.info(f"Uploading {file_name}")
-            #print(file_body)
-            response = s3.put_object(Body=file_body,Bucket=s3ap,Key=f"{files_folder}/{file_name}")
-            if response['ResponseMetadata']['HTTPStatusCode'] != 200:
-                logger.error("------------------------------------------------------------")
-                logger.error(f"ERROR: There was a problem with {file_name}")
-                logger.error(response)
-                logger.error("------------------------------------------------------------")
-            else:
-                logger.info('Success')
-        logger.info("Finished uploading reflections to S3.")
-        return response
-    
-    # download and save results
-    def results_download_save_extract(self,s3Uri,local_file_path):
-        s3 = self.__s3_client
-        output_key = s3Uri.split(self.config.s3_bucket_name)[1]
-        # download from S3 to local path
-        with open(f"{local_file_path}.tar.gz",'wb') as output_data:
-            s3.download_fileobj(self.config.s3_bucket_name,output_key[1:],output_data)
-
-        # extract the files from tar archive
-        files = list()
-        with tarfile.open(f"{local_file_path}.tar.gz", "r:gz") as tf:
-            for member in tf.getmembers():
-                f = tf.extractfile(member)
-                if f is not None:
-                    content = f.read()
-                    files.append(content)
-        #print("Number of files:",len(files))
-        # extract results and save and return
-        raw_results = files[0].decode("utf-8").split('\n')
-        raw_results.pop() # pop last item off as empty entry due to final \n
-        json_results = json.dumps(raw_results)
-        with open(f"{local_file_path}.json","w") as fp:
-            fp.write(json_results)
-        return json_results
-
-
-class Comprehend:
-    
-    aws = None
-    config = None
-    __comp_client = None
-    
-    def __init__(self,aws:AWS):
-        self.aws = aws
-        self.config = self.aws.config
-    
-        # create client
-        try:
-            logger.debug(f"Region:{self.aws.aws_session.region_name}")
-            self.__comp_client = self.aws.aws_session.client(service_name='comprehend')
-        except Exception as err:
-            logger.error("Unable to create Comprehend client: ",err)
-                
-    def client(self):
-        return self.__comp_client
-        
-    # Use AWS comprehend to get bulk key phrases from single batch of chunked text
-    def get_single_batch_analysis(self,index,chunk):
-        comp_client = self.client()
-        results = {}
-        print("Analysing chunk",index)
-        print(" . key_phrase")
-        kpresult = comp_client.batch_detect_key_phrases(TextList=chunk,LanguageCode='en')
-        results['KeyPhraseResults'] = kpresult
-        #key_phrase_results.append(kpresult)
-        time.sleep(2)
-        print(" . sentiment")
-        senresult = comp_client.batch_detect_sentiment(TextList=chunk,LanguageCode='en')
-        results['SentimentResults'] = senresult
-        #sentiment_results.append(senresult)
-        time.sleep(2)
-        print(" . targeted_sentiment")
-        tsenresult = comp_client.batch_detect_targeted_sentiment(TextList=chunk,LanguageCode='en')
-        results['TargetedSentimentResults'] = tsenresult
-        #target_sent_results.append(tsenresult)
-        time.sleep(2)
-        print(" . syntax")
-        synresult = comp_client.batch_detect_syntax(TextList=chunk,LanguageCode='en')
-        results['SyntaxResults'] = synresult
-        #syntax_results.append(synresult)       
-        time.sleep(2)
-        return results
-
-
-    # Use AWS comprehend to get bulk key phrases from chunked text
-    def get_multiple_batch_analysis(self,chunked_text):
-        chunk_results = {}
-        for key in self.config.analysis_types.keys():
-            chunk_results[key] = []
-                
-        for idx,chunk in enumerate(chunked_text):
-            if len(chunked_text) > 4999:
-                print("WARNING: Text too long to analyse - index",idx,"skipped!")
-            else:
-                try:
-                    results = self.get_single_batch_analysis(index=idx,chunk=chunk)
-                except(Exception) as error:
-                    print("There was an error with index",idx,error)
-                finally:
-                    if results:
-                        for key in results.keys():
-                            chunk_results[key].append(results[key])
-
-        return chunk_results
-
-    # Take batched responses and concenate single lists of results, errors, and http responses
-    def unbatch_results(self,result_type,results,batch_size=25):
-        unbatched_results = {}
-        unbatched_errors = {}
-        batch_responses = {}
-        for idx,batch in enumerate(results):
-            #print("Response for batch:",idx)
-            batch_responses[idx] = batch['ResponseMetadata']
-            result_list = batch['ResultList']
-            error_list = batch['ErrorList']
-            for r in result_list:
-                ridx = idx*batch_size + r['Index']
-                rdata = r[result_type]
-                unbatched_results[ridx] = rdata
-            for e in error_list:
-                eidx = e['Index']
-                unbatched_errors[eidx] = 'ERROR' + e['ErrorCode'] + ': ' + e['ErrorMessage']
-        unbatched = {}
-        unbatched['results'] = unbatched_results
-        unbatched['errors'] = unbatched_errors
-        unbatched['responses'] = batch_responses
-        return unbatched
-
-    def check_long_text(self,df):
-        # Check for long reflections (too long for batch analysis)
-        long_df = df.copy()
-        long_df = long_df[long_df.text.str.len()>5000]
-        long_df['length'] = long_df.text.str.len()
-        return long_df
-    
-# #### CUSTOM ENTITY
-
-    def submit_custom_entity_job(self,job_name): #access_role_arn,entity_recogniser_arn):
-        job_str = f"{self.config.prefix}{job_name}{self.config.postfix}"
-        
-        response = self.__comp_client.start_entities_detection_job(
-            InputDataConfig={
-                'S3Uri': self.config.s3_input_uri,
-                'InputFormat': 'ONE_DOC_PER_FILE'
-            },
-            OutputDataConfig={
-                'S3Uri': self.config.s3_output_uri
-            },
-            DataAccessRoleArn=self.config.comprehend_access_role_arn,
-            JobName=job_str,
-            EntityRecognizerArn=self.config.reflexive_entity_arn,
-            LanguageCode='en'
-        )
-        self.job_id = response['JobId']
-        self.check_job_status() # force the creation of __job_properties
-        return response
-    
-    def get_current_job_id(self):
-        return self.job_id
-    
-            # Check job status
-    def check_job_status(self):
-        job_status = self.__comp_client.describe_entities_detection_job(
-            JobId=self.job_id
-        )
-        self.__job_properties = job_status['EntitiesDetectionJobProperties'] 
-        return self.__job_properties['JobStatus']
-
-    def get_job_details(self):
-        return self.__job_properties
-    
-        #checked
-    def download_and_extract(self,s3):
-        local_output_dir = f"{self.config.local_path}{self.config.prefix}output{self.config.postfix}"
-        job_details = self.get_job_details()
-        s3Uri = job_details['OutputDataConfig']['S3Uri']
-        return s3.results_download_save_extract(s3Uri,local_output_dir)
-       

reflexive/util.py

@@ -1,127 +0,0 @@
-import os
-import json
-import pandas as pd
-from sklearn.preprocessing import MinMaxScaler
-import logging
-#logging.basicConfig(level=logging.DEBUG)
-logger = logging.getLogger(__name__)
-
-
-
-# File functions
-def get_data_path_name(config,name,ext):
-        return f"{config.local_path}{config.prefix}{name}{config.postfix}.{ext}"
-        
-def set_sub_dir(config,sub_dir=None):
-    # check dir sub_dir exists
-    if sub_dir:
-        local_dir = f"{config.local_path}{sub_dir}/"
-        logger.debug(f"local_dir: {local_dir}")
-        dirExists = os.path.exists(local_dir)
-        if not dirExists:
-            logger.info(f"Creating subdirectory: {local_dir}")
-            os.makedirs(local_dir)
-    else:
-        local_dir = config.local_path
-    return local_dir
-        
-
-
-# Function to write dictionaries to both json and csv
-def writeDictJsonCSV(dictionary,path_file):
-    with open(f"{path_file}.json",'w') as fp:
-        fp.write(json.dumps(dictionary))
-
-    ngram_df = pd.DataFrame.from_dict(dictionary,orient='index')   
-    ngram_df.to_csv(f"{path_file}.csv")
-    
-# Data functions
-def sort_dict_by_value(d):
-    return dict(sorted(d.items(), key=lambda x:x[1], reverse=True))
-
-def filter_dict_by_value(ngrams,min_val=3):
-    filtered_ngrams = {}
-    for k,v in ngrams.items():
-        if v >=min_val:
-            filtered_ngrams[k] = v
-    return filtered_ngrams
-
-# Input a series and output a list of lists with each maxn elements
-def series_to_chunked_list(series,maxn=25):
-    lst = list(series)
-    return __chunk_list(lst,maxn)
-
-# Chunk a list into a list of lists with maxn elements
-def __chunk_list(lst,maxn=25):
-    return [lst[i:i + maxn] for i in range(0, len(lst), maxn)]
-
-# Count named entities
-def count_entities(entities):
-    counts = []
-    for k,v in entities.items():
-        counts.append((k,len(v))) 
-    return sorted(counts, key=lambda x: x[1], reverse=True)
-
-# Function for calculating proportions of features
-def ratios(elements):
-    etotal = sum([v[1] for v in elements])
-    if etotal==0:
-        return elements
-    else:
-        proportioned = []
-        for element in elements:
-            prop_val = round((element[1]/etotal),4)
-            proportioned.append((element[0],prop_val))
-        return proportioned
-
-
-
-# Count labels associated with strings
-def count_labels(string_labels):
-    counts = dict()
-    for rt in string_labels:
-        counts[rt[1]] = counts.setdefault(rt[1],0) + 1
-    return sorted(counts.items(), key=lambda x: x[1], reverse=True)
-
-def count_keys(key_count_dict):
-    counts = dict()
-    for k,v in key_count_dict.items():
-        counts[k] = counts.setdefault(k,0) + v 
-    return sorted(counts.items(), key=lambda x: x[1], reverse=True)
-    
-# Total the values in list of tuples
-def tuple_values_total(tuples):
-    tvs = [t[1] for t in tuples]
-    return sum(tvs)
-
-#### SCALING AND NORMALISING
-
-# Outliers
-
-def outlier_fence(series):
-    bounds = {}
-    stats = series.describe()
-    iqr = stats['75%'] - stats['25%']
-    bounds["IQR"]=iqr
-    upper = stats['75%']+1.5*iqr
-    bounds["UPPER"]=upper
-    lower = stats['25%']-1.5*iqr
-    bounds["LOWER"]=lower
-    return bounds
-
-# MinMax Scaling
-def scale_min_max(df_cols):
-    scaler = MinMaxScaler()
-    return scaler.fit_transform(df_cols)
-
-# Normalise domain term counts
-def normalise_domain_counts(domain_counts,text_size):
-    norms = {}
-    for k,v in domain_counts.items():
-        norms[k] = round(v*text_size,3)
-    return norms
-
-def normalise_scaled(df,feature,norm_feature = 'text_scaled'):
-    tempdf = df[[feature,norm_feature]].copy()
-    tempdf['norm_scaled'] = tempdf.apply(lambda r: round(r[feature]/(r[norm_feature]+0.01),4),axis=1)
-    return tempdf['norm_scaled']