reflexive 1.2.8__py3-none-any.whl → 2.1.0__py3-none-any.whl

reflexive/__init__.py

@@ -1,19 +1,4 @@
-# Make classes available at top level
+from reflexive.service import (AWS_service)
+from reflexive.res_analysis import (RES_analyser,RES_visualiser)
 
-from reflexive.cfg import Config
-from reflexive.session import (
-    AWS,
-    S3,
-    Comprehend)
-
-# import reflexive.util
-from reflexive.analyse import Nlp
-from reflexive.visualise import (
-    Display,
-    RES_graph)
-
-from reflexive.res import (
-    Res_analyse,
-    Res_display)
-
-__all__ = ["Config","AWS","S3","Comprehend","Nlp","Display","RES_graph","Res_analyse","Res_display"]
+__all__ = ["AWS_service","RES_analyser","RES_visualiser"]

reflexive/analysis_functions.py

@@ -0,0 +1,399 @@
+from typing import Callable
+from pandas import (DataFrame,Series)
+from datetime import datetime
+from zoneinfo import ZoneInfo
+from time import sleep
+from functools import partial
+import tarfile
+import json
+import os
+import numpy as np
+from numpy.linalg import norm
+from itertools import chain
+from graph_tool.all import (
+    Graph,
+    similarity,
+    adjacency)
+
+### PIPELINE FUNCTIONS
+
+# Clean text using supplied function and calculate text length
+# Used by RES_analyser.preprocess_text()
+
+def _clean_text(df:DataFrame,text_cleaner:Callable[[str],str])->DataFrame:
+      return (df
+            .assign(text=lambda d: d.text.apply(text_cleaner))
+            .assign(text_length=lambda d: [len(row.text) for row in d.itertuples()]))
+
+# Upload text using supplied uploader function
+# Used by RES_analyser.upload_text_to_s3()
+      
+def _upload_text(df:DataFrame,uploader:Callable,res_analyser)->DataFrame:
+    upload = partial(uploader,aws_service=res_analyser.aws_service,config=res_analyser.config,logger=res_analyser.logger)
+    return df.assign(uploaded=lambda d: [upload(str(row.Index),row.text) for row in d.itertuples()])
+
+# Initiate comprehend analysis on S3 text documents
+
+def _analyse_text(analyser:Callable,res_analyser)->dict:
+    analyse = partial(analyser,
+                      aws_service=res_analyser.aws_service,
+                      config=res_analyser.config,
+                      logger = res_analyser.logger)
+    job_status = analyse()
+    return job_status['EntitiesDetectionJobProperties']
+
+# Add comprehend analysis results to dataframe
+def _analysis_to_dataframe(df:DataFrame,results:list)->DataFrame:
+    analysis = _extract_analysis(results=results)
+    df['res_results']=Series(analysis)
+    return df
+
+# Offsets to dataframe
+def _add_offsets(df:DataFrame,offset_cleaner,orphan_joiner)->DataFrame:
+      return (df
+            .assign(offsets=lambda d: d.res_results.apply(offset_cleaner))
+            .assign(offsets_clean=lambda d: [orphan_joiner(row.text,row.offsets) for row in d.itertuples()]))
+
+def _offset_cleaner(res_results):
+    offsets = _collect_offsets(res_results)
+    tuples = _offset_tuples(offsets)
+    return _sorted_offsets(tuples)
+
+def _orphan_joiner(text,offsets):
+    otuples = _orphaned_I(text,offsets)
+    offs =  _orphaned_word(text,otuples)
+    return _regroup(offs)
+
+def _collect_offsets(rrs):
+    new_rrs = {}
+    for rr in rrs:
+        if rr['Score']>0.6:
+            ent_type = rr['Type']
+            if ent_type in ['VR','ER']:
+                label = "NR"
+            elif ent_type in ['EP','EV']:
+                label = "EP"
+            elif ent_type in ['CN','AF']:
+                label = "AF"
+            else:
+                label = ent_type
+            new_rrs.setdefault(label,[]).append((rr['BeginOffset'],rr['EndOffset']))
+    return new_rrs
+
+
+
+#####
+
+def _add_res_sequence(df):
+    temp_df = df.copy()
+    temp_df['res_sequence'] = temp_df.offsets_clean.apply(_get_res_sequence)
+    return temp_df
+    
+def _add_res_interactions(df):
+    temp_df = df.copy()
+    temp_df['res_interactions'] = temp_df.res_sequence.apply(_count_res_interactions)
+    return temp_df
+
+def _add_res_weights(df):
+    temp_df = df.copy()
+    temp_df['res_weights'] = temp_df.res_interactions.apply(_calc_res_weights)
+    return temp_df
+
+def _add_semantic_weights(df,ranking_factors={}):
+    temp_df = df.copy()
+    ranks = partial(_calc_semantic_weights,factors=ranking_factors)
+    temp_df['semantic_weights'] = temp_df.res_weights.apply(ranks)
+    return temp_df
+
+def _add_res_adj_matrix(df):
+    temp_df = df.copy()
+    temp_df['res_adj_matrix'] = temp_df.semantic_weights.apply(_create_adj_matrix)
+    return temp_df
+
+def _get_res_sequence(offsets_clean):
+    return [label for label in offsets_clean.values()]
+
+
+def _empty_res_interactions() -> dict[tuple,int]:
+    RE_types = ['RR','NR','AR','AF','EP']
+    RE_interactions:dict[tuple,int] = dict()
+    for t1 in RE_types:
+        for t2 in RE_types:
+            entry = tuple(sorted((t1,t2)))
+            if entry not in RE_interactions.keys():
+                RE_interactions[entry] = 0
+    return RE_interactions
+
+def _count_res_interactions(re_sequence:list[str]) -> dict[tuple,int]:
+    re_ints = _empty_res_interactions()
+    limit = len(re_sequence)-1
+    for i,s in enumerate(re_sequence):
+        if i < limit:
+            rei = tuple(sorted((s,re_sequence[i+1])))
+            #print(i,rei)
+            re_ints[rei] += 1 
+    return re_ints
+
+def _calc_res_weights(interactions:dict[tuple,int])->dict[tuple,float]:
+    max_count = max(interactions.values())
+    weights = dict()
+    for edge,count in interactions.items():
+        weights[edge] = round(count/(max_count),2)
+    return weights
+
+
+
+def _calc_semantic_weights(weights:dict[tuple,float], factors:dict[tuple,float]={})->dict[tuple,float]:
+    if not factors:
+        return weights
+    else:
+        for edge,w in weights.items():
+            weights[edge] = factors[edge] * w
+        return weights
+        
+
+def _create_adj_matrix(weights:dict[tuple,float])->list[list[float]]:
+    re_types = ["RR","NR","AR","AF","EP"]
+    matrix = []
+    for r in re_types:
+        row = []
+        for c in re_types:
+            key = tuple(sorted((r,c)))
+            #print(key)
+            weight = weights.get(key,0)
+            row.append(weight)
+        matrix.append(row)
+    return matrix
+
+### GRAPH ANALYSIS
+
+def _jaccard_similarity(g1:Graph,g2:Graph)->float:
+    return similarity(g1, g2, 
+                      eweight1=g1.ep['e_weights'], eweight2=g2.ep['e_weights'], 
+                      #label1=g1.vp['v_labels'], label2=g2.vp['v_labels'], 
+                      norm=True, p=1.0, distance=False, asymmetric=False)
+    
+def _cosine_similarity(m1,m2)->float:
+    v1 = list(chain.from_iterable(m1))
+    v2 = list(chain.from_iterable(m2))
+    return np.dot(v1,v2)/(norm(v1)*norm(v2))
+
+    
+
+### PIPELINE SUPPORT FUNCTIONS
+
+# Clean return characters and strip whitespace
+# Used by preprocess_text()
+def _whitespace_cleaner(text:str)->str:
+      return text.strip().replace('\r\n','\n')
+
+# Upload text to S3
+def _s3_text_uploader(idx:str,text:str,aws_service,config:dict,logger)->bool:
+    try:
+        response = aws_service.s3_client.put_object(Body=text,
+                                        Bucket=aws_service.aws_params["s3_bucket_name"],
+                                        Key=f"{config["s3_source_dir"]}/{idx}.txt")
+    except Exception as e:
+        logger.error("There was an error when uploading text to s3 %s",repr(e))
+        return False
+    else:
+        if response['ResponseMetadata']['HTTPStatusCode']==200:
+            logger.debug(f"File {idx} uploaded successfully")
+            return True
+        else:
+            logger.error(f"File {idx} did not upload successfully to S3: {response}")
+            return False
+        
+# Analyse text with comprehend custom entity recognizer
+def _comprehend_cer_analyser(aws_service,config,logger)->dict:
+    try:
+        response = aws_service.comprehend_client.start_entities_detection_job(
+            InputDataConfig={
+                'S3Uri': _comprehend_input_uri(aws_service.aws_params["s3_bucket_name"],
+                                               config["s3_source_dir"]),
+                'InputFormat': 'ONE_DOC_PER_FILE'
+            },
+            OutputDataConfig={
+                'S3Uri': _comprehend_output_uri(aws_service.aws_params["s3_bucket_name"],
+                                                config["s3_target_dir"])
+            },
+            DataAccessRoleArn=_comprehend_access_role_arn(aws_service.aws_params["comprehend_service_role_name"],
+                                                          aws_service.aws_account_number),
+            JobName=f"res_analysis_{_date_string()}",
+            EntityRecognizerArn=_comprehend_cer_arn(aws_service.aws_session.region_name,
+                                                    aws_service.aws_account_number,
+                                                    aws_service.aws_params["reflexive_entity_name"],
+                                                    aws_service.aws_params["reflexive_entity_version"]),
+            LanguageCode='en'
+        )
+    except Exception as e:
+        logger.error("There was an error when analysing text with comprehend %s",repr(e))
+        return {"ERROR":repr(e)}
+    else:
+        return aws_service.comprehend_client.describe_entities_detection_job(JobId=response['JobId'])
+
+# Monitor a CER Analysis Job
+def _cer_job_progress(status:dict,aws_service,tz,output)->dict:
+    # Submitted
+    job_name = status['JobName']
+    job_id = status['JobId']
+    submit_time = status['SubmitTime'].astimezone(ZoneInfo(tz))
+    output(f"RES_ANALYSIS JOB {job_name} ({job_id}) submitted at: {submit_time}")
+    
+    # In progress
+    while status['JobStatus'] in ["SUBMITTED","IN_PROGRESS"]:
+        time = datetime.now().astimezone(ZoneInfo(tz))
+        job_status = status['JobStatus']
+        output(f"{time} [{job_id}] {job_name} status: {job_status}")
+        sleep(10)
+        properties = aws_service.comprehend_client.describe_entities_detection_job(JobId=job_id)
+        status=properties['EntitiesDetectionJobProperties']
+        
+    # Finished (complete or error)
+    job_status = status['JobStatus']
+    end_time = status['EndTime'].astimezone(ZoneInfo(tz))
+    time_taken = end_time - submit_time
+    output_url = status['OutputDataConfig']['S3Uri']
+    output(f"RES_ANALYSIS JOB {job_name} ({job_id}) finished with status: {job_status} at: {end_time}")
+    output(f"Analysis time: {str(time_taken)}")
+    output(f"Results available at: {output_url}")
+    return status    
+ 
+     
+# Download from S3 to local
+def _download_from_s3(res_analyser,status)->str:
+    local_file_path = f"{res_analyser.config['local_data_dir']}/{status['JobName']}.tar.gz"
+    bucket_name = res_analyser.aws_service.aws_params["s3_bucket_name"]
+    try:
+        output_key = status['OutputDataConfig']['S3Uri'].split(bucket_name)[1]
+        with open(f"{local_file_path}",'wb') as output_data:
+            res_analyser.aws_service.s3_client.download_fileobj(bucket_name,output_key[1:],output_data)  
+    except Exception as e:
+        res_analyser.logger.error("An error occured when downloading results from S3: %s",repr(e))
+        local_file_path = None
+    return local_file_path
+   
+# Extract results from tar.gz file and save as json  
+def _extract_save_results(res_analyser,local_file_path)->list:
+        # extract the tar archive
+        files = list()
+        with tarfile.open(f"{local_file_path}", "r:gz") as tf:
+            for member in tf.getmembers():
+                f = tf.extractfile(member)
+                if f is not None:
+                    content = f.read()
+                    files.append(content)
+        # extract results and save and return
+        raw_results = files[0].decode("utf-8").split('\n')
+        raw_results.pop() # pop last item off as empty entry due to final \n
+        #
+        #json_results = json.dumps(raw_results)
+        #res_analyser.logger.info("raw_results>> ",raw_results)
+        results = [json.loads(result) for result in raw_results]
+        with open(f"{local_file_path[:-7]}.json","w") as fp:
+            json.dump(results,fp)
+        return results
+
+# Get a dict of (index,entities) from cer analysis results
+def _extract_analysis(results):
+    file_ents =  ((result["File"],result["Entities"]) for result in results)
+    idx_ents = ((int(file.split('_')[-1].split('.')[0]),ents) for file,ents in file_ents)
+    return dict(idx_ents)
+
+
+   
+# Comprehend access role arn
+def _comprehend_access_role_arn(comprehend_service_role_name,aws_account_number):
+    return f"arn:aws:iam::{aws_account_number}:role/service-role/{comprehend_service_role_name}"
+
+# Comprehend input url
+def _comprehend_input_uri(s3_bucket_name,s3_files,prefix=""):
+    return f"s3://{s3_bucket_name}/{s3_files}/{prefix}" 
+
+# Comprehend output url
+def _comprehend_output_uri(s3_bucket_name,s3_results):
+    return f"s3://{s3_bucket_name}/{s3_results}/" 
+
+# Comprehend entity recognizer arn
+def _comprehend_cer_arn(region,account_number,cer_name,cer_version):
+    return f"arn:aws:comprehend:{region}:{account_number}:entity-recognizer/{cer_name}/version/{cer_version}"
+        
+## Offset functions
+
+def _offset_tuples(offsets):
+    for k,vs in offsets.items():
+        for b,e in vs:
+            yield (b,(e,k))
+
+def _sorted_offsets(offsets):
+    return sorted(offsets)
+
+def _orphaned_I(text,offsets):
+    for b,(e,t) in offsets:
+        if 'I' in text[(b-2):(b-1)].strip():
+            #print(text[(b-2):e],t)
+            yield (b-2, (e,t))
+        else:
+            yield (b, (e,t))
+            
+def _orphaned_word(text,offsets):
+    coffs = {}
+    p = (0,(-2,''))
+    for b,(e,t) in offsets:
+        #print(p[1][0])
+        if (p[1][0]+3)>=b:
+            #print("Prev:",p,f"|{df.text[0][p[0]:p[1][0]]}|")
+            #print("<--->",f"|{df.text[0][(p[1][0]+1):(b-1)]}|")
+            #print("This:",b,e,t,f"|{df.text[0][b:e]}|")
+            #print()
+            if len((text[p[0]:p[1][0]]).split(' '))<2:
+                #print(f"Removing {p[0]},{p[1][0]},{p[1][1]}")
+                coffs.pop(p[0])
+                #print(f"Replacing {b},{e},{t} with {p[0]},{e},{t}")
+                coffs[p[0]] = (e,t)
+                p=(p[0],(e,t))
+            else:
+                coffs[b] = (e,t)
+                p = (b,(e,t))
+        else:
+            coffs[b] = (e,t)
+            p = (b,(e,t))
+    return coffs.items()
+
+def _regroup(offsets):
+    grouped = (((b,e),k) for (b,(e,k)) in offsets)
+    return dict(grouped)
+
+
+   
+   
+### UTILITY FUNCTIONS  
+   
+# Create a reverse date string YYYYmmdd based on current local time   
+def _date_string()->str:
+    return datetime.today().strftime('%Y%m%d')
+
+# Get the current local working dir
+def _local_path(dir)->str:
+    return os.getcwd()+dir
+
+# Check if local directory exists
+def _dir_exists_local(dir:str)->bool:
+    return os.path.exists(_local_path(dir))
+
+# Return function to create directory
+def _create_dir(dir)->str:
+    os.makedirs(_local_path(dir))
+    return _local_path(dir)
+
+# Create local directory if required
+def _create_local_dir(dir,logger)->str:
+    if not _dir_exists_local(dir):
+        try:
+            path = _create_dir(dir)
+        except Exception as e:
+            logger.error("There was an error creating the local directory: %s",repr(e))
+        finally:
+            return path
+    else:
+        return _local_path(dir)

reflexive/display_functions.py

@@ -0,0 +1,165 @@
+from graph_tool.all import (
+    Graph,
+    VertexPropertyMap,
+    EdgePropertyMap,
+    GraphPropertyMap,
+    graph_draw,
+    ungroup_vector_property,
+    group_vector_property
+)
+import cairo
+from itertools import chain
+from spacy import displacy
+
+# Text Display functions
+
+def _create_displacy_ents(name,text,offsets):
+    return {"text": text,
+            "ents": [{"start":s,"end":e,"label":l}for (s,e),l in offsets.items()],
+            "title": name}
+    
+def _render_annotated_text(ents,inline=True):
+    #default to inline
+    page_opt = False
+    jupyter_opt = True
+    if not inline:
+        page_opt = True
+        jupyter_opt = False
+    
+    return displacy.render(ents,manual=True,style="ent", options=_get_text_display_options(),page=page_opt,jupyter=jupyter_opt)
+
+def _get_text_display_options():
+    colours = dict([(prop['lbl'],prop['clr']) for prop in _res_graph_properties().values()])
+    return {"ents": list(colours.keys()), "colors": colours}
+
+# RES properties for all graphs
+def _res_graph_properties()->dict:
+    return {0:{ "lbl":"RR",
+                "pos":(0.2,6.5),
+                "clr":"#00AEEF"},
+            1:{ "lbl":"NR",
+                "pos":(5,10),
+                "clr":"#ED1B23"},
+            2:{ "lbl":"AR",
+                "pos":(9.8,6.5),
+                "clr":"#00A64F"},
+            3:{ "lbl":"AF",
+                "pos":(7.9,1),
+                "clr":"#EC008C"},
+            4:{ "lbl":"EP",
+                "pos":(2.1,1),
+                "clr":"#FFF200"}}
+
+# Create a graph from an adjacency matrix
+def _create_graph(matrix,id)->Graph:
+    if matrix:
+        graph = _graph_from_edges(dict(_matrix_to_dict(matrix)))
+    else:
+        graph = _graph_no_edges()
+    prop_list = _res_graph_properties().values()
+    graph.vp["v_positions"] = graph.new_vp("vector<double>",vals=[prop['pos'] for prop in prop_list])
+    graph.vp["v_labels"] = graph.new_vp("string",vals=[prop['lbl'] for prop in prop_list])
+    graph.gp["id"] = graph.new_gp("string",val=id)
+    return graph
+        
+    # # Vertex properties common to all graphs    
+    # v_lbl = graph.new_vp("string",vals=_get_prop_values('lbl'))
+    # v_pos = graph.new_vp("vector<double>",vals=_get_prop_values('pos'))
+    # # Make propertyMaps internal to the graph
+    # graph.vp["v_colour"] = v_clr
+    # graph.vp["v_position"] = v_pos
+    # graph.vp["v_label"] = v_lbl
+    # graph.ep["e_weights"] = e_weight
+    
+def _graph_from_edges(edges:dict)->Graph:
+    graph = Graph(g=edges.keys(),directed=False)
+    graph.ep["e_weights"] = graph.new_ep("double",vals=edges.values())
+    graph.ep["e_widths"] = graph.new_ep("double",vals=_scale_weights(edges.values()))
+    graph.vp["v_colours"] = _get_vcolours_from_edges(graph)
+    return graph
+
+def _scale_weights(weights,factor=5):
+    return [round(w*factor,1) for w in weights]
+      
+def _graph_no_edges()->Graph:
+      graph = Graph(g=_empty_edge_dict(),directed=False)
+      graph.ep["e_weights"] = graph.new_ep("double")
+      graph.ep["e_widths"] = graph.new_ep("double")
+      graph.vp["v_colours"] = graph.new_vp("string",val="#cccccc")
+      return graph
+    
+def _get_vcolours_from_edges(graph:Graph)->VertexPropertyMap:
+    prop_list:dict[int,dict] = _res_graph_properties()
+    for i in _isolated_vertices(graph):
+        prop_list[i]['clr']= "#cccccc"
+    return graph.new_vp("string",[prop['clr'] for prop in prop_list.values()])   
+
+def _isolated_vertices(graph):
+    edgelist = chain.from_iterable([sorted((int(e.source()),int(e.target()))) for e in graph.edges()])
+    return set(range(5)) - set([e for e in set(edgelist)])
+
+#
+def _matrix_to_dict(matrix):
+    egen =  ((((tuple(sorted((r,c))),w)) for c,w in enumerate(row) if w>0) for r,row in enumerate(matrix) if sum(row)>0)
+    return dict(chain.from_iterable(egen))
+    # edges = {}
+    # for r,row in enumerate(matrix):
+    #     # if empty row, add to iso_vertices
+    #     # if sum(row) == 0:
+    #     #     self.iso_vertices.add(r)
+    #     # else:
+    #     if sum(row) > 0: # edge exists
+    #         for c,weight in enumerate(row):
+    #             if weight > 0:
+    #                 edge = tuple(sorted((r,c)))
+    #                 #print("r,c:",edge," - ",weight)
+    #                 edges[edge] = weight
+    # return edges
+
+#
+def _empty_edge_dict():
+    empty_edges = {}
+    for idx in range(5): #self.gt_props.keys():
+        empty_edges[idx] = []
+    return empty_edges
+
+#
+def _get_prop_values(key):
+    values_list =  self.gt_props.values()
+    return [p[key] for p in values_list]
+
+# flip coordinates for graph-tool
+def _flipY(vpositions):
+    x, y = ungroup_vector_property(vpositions, [0, 1])
+    y.fa *= -1
+    y.fa -= y.fa.min()
+    return group_vector_property([x, y])
+
+#
+def _draw_graph(graph:Graph,inline=True):
+
+    positions = _flipY(graph.vp["v_positions"])
+    labels = graph.vp["v_labels"]
+    colors = graph.vp["v_colours"]
+    widths = graph.ep["e_widths"]
+    graph_draw(graph, inline=inline,output_size=(300,300),fit_view=0.7,
+                    pos=positions, 
+                    vertex_text=labels,
+                    vertex_font_family="sans serif",
+                    vertex_font_size=18,
+                    vertex_font_weight=cairo.FONT_WEIGHT_BOLD,
+                    vertex_fill_color=colors,
+                    vertex_size = 50,
+                    vertex_halo=False,
+                    vertex_pen_width=1.2,
+                    vertex_color="#999999",
+                    edge_pen_width=widths)
+    
+# def get_vertex_labels(self):
+#     return self._get_prop_values('lbl')
+
+# def get_vertex_colours(self):
+#     return self._get_prop_values('clr')
+
+# def get_vertex_positions(self):
+#     return self._get_prop_values('pos')