reflexive 1.2.0__tar.gz → 1.2.2__tar.gz

{reflexive-1.2.0 → reflexive-1.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: reflexive
-Version: 1.2.0
+Version: 1.2.2
 Summary: Supports AWS Reflexive Expressions Systems (RES) Analysis
 Project-URL: Repository, https://github.com/nlytx/reflexive.git
 Author-email: Andrew Gibson <andrew@nlytx.io>

{reflexive-1.2.0 → reflexive-1.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "reflexive"
-version = "1.2.0"
+version = "1.2.2"
 description = "Supports AWS Reflexive Expressions Systems (RES) Analysis"
 authors = [{name = "Andrew Gibson", email = "andrew@nlytx.io"}]
 readme = "README.md"

reflexive-1.2.2/src/reflexive/__init__.py

@@ -0,0 +1,15 @@
+# Make classes available at top level
+
+from reflexive.cfg import Config
+from reflexive.session import (
+    AWS,
+    S3,
+    Comprehend)
+
+# import reflexive.util
+from reflexive.analyse import Nlp
+from reflexive.visualise import (
+    Display,
+    RES_graph)
+
+__all__ = ["Config","AWS","S3","Comprehend","Nlp","Display","RES_graph"]

{reflexive-1.2.0 → reflexive-1.2.2}/src/reflexive/analyse.py

@@ -6,7 +6,6 @@ import re
 from reflexive import util
 from reflexive import cfg
 from reflexive import session
-import reflexive as rfx
 
 import logging
 logging.basicConfig(level=logging.DEBUG)
@@ -290,7 +289,7 @@ class Nlp:
         for grp in targetedSentiment_results:
             for mention in grp["Mentions"]:
                 if mention['Score'] >= threshold:
-                    if not "NEUTRAL" in mention['MentionSentiment']['Sentiment']:
+                    if "NEUTRAL" not in mention['MentionSentiment']['Sentiment']:
                         k = mention['MentionSentiment']['Sentiment']
                         text = str.lower(mention['Text'])
                         sents.setdefault(k,{text}).add(text)

{reflexive-1.2.0 → reflexive-1.2.2}/src/reflexive/cfg.py

@@ -1,10 +1,8 @@
 import logging
 import os
 from datetime import datetime
-import json
-import reflexive as rfx
 
-logging.basicConfig(level=logging.DEBUG)
+#logging.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger(__name__)
 
 class Config:
@@ -74,7 +72,7 @@ class Config:
             if not os.path.exists(data_dir):
                 logger.warning("Path does not exist, creating directory")
                 os.makedirs(data_dir)
-                logger.info(f"Created %s",repr(data_dir))
+                logger.info("Created %s",repr(data_dir))
             self.local_path = local_path
         if not date_string:
             date_string = datetime.today().strftime('%Y%m%d')

{reflexive-1.2.0 → reflexive-1.2.2}/src/reflexive/session.py

@@ -5,7 +5,6 @@ import tarfile
 import json
 
 from reflexive import cfg
-import reflexive as rfx
 
 import logging
 logging.basicConfig(level=logging.DEBUG)
@@ -19,7 +18,7 @@ class AWS:
     def __init__(self,config:cfg.Config):
         # on initialisation create a new session with provided profile (or with default profile)
         #logger.error(config.get_parameters())
-        if config==None:
+        if config is None:
             config = cfg.Config()
         self.config = config
         self.new_session()

{reflexive-1.2.0 → reflexive-1.2.2}/src/reflexive/util.py

@@ -1,10 +1,12 @@
 import os
-
+import json
+import pandas as pd
+from sklearn.preprocessing import MinMaxScaler
 import logging
 logging.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger(__name__)
 
-from sklearn.preprocessing import MinMaxScaler
+
 
 # File functions
 def get_data_path_name(config,name,ext):
@@ -20,7 +22,7 @@ def set_sub_dir(config,sub_dir=None):
             logger.info(f"Creating subdirectory: {local_dir}")
             os.makedirs(local_dir)
     else:
-        local_dir = local_path
+        local_dir = config.local_path
     return local_dir
         
 
@@ -46,12 +48,12 @@ def filter_dict_by_value(ngrams,min_val=3):
 
 # Input a series and output a list of lists with each maxn elements
 def series_to_chunked_list(series,maxn=25):
-    l = list(series)
-    return __chunk_list(l,maxn)
+    lst = list(series)
+    return __chunk_list(lst,maxn)
 
 # Chunk a list into a list of lists with maxn elements
-def __chunk_list(l,maxn=25):
-    return [l[i:i + maxn] for i in range(0, len(l), maxn)]
+def __chunk_list(lst,maxn=25):
+    return [lst[i:i + maxn] for i in range(0, len(lst), maxn)]
 
 # Count named entities
 def count_entities(entities):

{reflexive-1.2.0 → reflexive-1.2.2}/src/reflexive/visualise.py

@@ -1,3 +1,13 @@
+from graph_tool.all import (
+    Graph,
+    graph_draw,
+    ungroup_vector_property,
+    group_vector_property
+)
+import cairo
+
+from spacy import displacy
+
 from reflexive import session
 from reflexive import cfg
 
@@ -183,4 +193,163 @@ class Display:
                     b = v[0]
                     e = v[1]
             #print("New offsets:",len(new_offsets[k]))
-        return new_offsets
+        return new_offsets
+    
+class RES_text:
+    
+    def __init__(self):
+        self._setup()
+        
+    def _setup(self):
+        return None
+    
+    def show(self):
+        #displacy.render(disp_data,manual=True,style="ent", options=cfg.display_options)
+        return None
+     
+class RES_graph:
+    
+    #
+    gt_props = {0:{ "lbl":"RR",
+                    "pos":(0.2,6.5),
+                    "clr":"#00AEEF"},
+                1:{ "lbl":"NR",
+                    "pos":(5,10),
+                    "clr":"#ED1B23"},
+                2:{ "lbl":"AR",
+                    "pos":(9.8,6.5),
+                    "clr":"#00A64F"},
+                3:{ "lbl":"AF",
+                    "pos":(7.9,1),
+                    "clr":"#EC008C"},
+                4:{ "lbl":"EP",
+                    "pos":(2.1,1),
+                    "clr":"#FFF200"}}
+                
+    
+    # re_props = {"RR":{
+    #                 "idx": 0,
+    #                 "pos":(0.2,6.5),
+    #                 "clr":"#00AEEF"},
+    #             "NR":{
+    #                 "idx": 1,
+    #                 "pos":(5,10),
+    #                 "clr":"#ED1B23"},
+    #             "AR":{
+    #                 "idx": 2,
+    #                 "pos":(9.8,6.5),
+    #                 "clr":"#00A64F"},
+    #             "AF":{
+    #                 "idx": 3,
+    #                 "pos":(7.9,1),
+    #                 "clr":"#EC008C"},
+    #             "EP":{
+    #                 "idx": 4,
+    #                 "pos":(2.1,1),
+    #                 "clr":"#FFF200"}}
+    
+    #
+    edges = dict()
+    #
+    iso_vertices = set()
+    #
+    v_label = None
+    #
+    v_color = None
+    #
+    v_pos = None
+
+    #
+    def __init__(self,matrix=None):
+        self._setup(matrix)
+    
+    #    
+    def _setup(self,matrix):
+        if matrix:
+            # Edges from matrix
+            self.edges = self._matrix_to_dict(matrix)
+            self.graph = Graph(g=self.edges.keys(),directed=False)
+            self.e_weight = self.graph.new_ep("double",vals=self.edges.values())
+            # Handle colour of isolated vertices
+            default_clrs = self._get_prop_values('clr')
+            actual_clrs = []
+            for i in range(5):
+                if i in self.iso_vertices:
+                    clr = "#cccccc"
+                else:
+                    clr = default_clrs[i]
+                actual_clrs.append(clr)
+            self.v_color = self.graph.new_vp("string",vals=actual_clrs)
+        else:
+            # No edges
+            self.graph = Graph(g=self._empty_edge_dict(),directed=False)
+            self.e_weight = self.graph.new_ep("double")
+            self.v_color = self.graph.new_vp("string",val="#cccccc")
+        # Vertex properties common to all graphs    
+        self.v_label = self.graph.new_vp("string",vals=self._get_prop_values('lbl'))
+        self.v_pos = self.graph.new_vp("vector<double>",vals=self._get_prop_values('pos'))
+        
+        
+
+    #
+    def _matrix_to_dict(self,matrix):
+        edges = {}
+        for r,row in enumerate(matrix):
+            # if empty row, add to iso_vertices
+            if sum(row) == 0:
+                self.iso_vertices.add(r)
+            else:
+                for c,weight in enumerate(row):
+                    if weight > 0:
+                        edge = tuple(sorted((r,c)))
+                        #print("r,c:",edge," - ",weight)
+                        edges[edge] = weight
+        return edges
+    
+    #
+    def _empty_edge_dict(self):
+        empty_edges = {}
+        for idx in self.gt_props.keys():
+            empty_edges[idx] = []
+        return empty_edges
+    
+    #
+    def _get_prop_values(self,key):
+        values_list =  self.gt_props.values()
+        return [p[key] for p in values_list]
+    
+    # flip coordinates for graph-tool
+    def _flipY(self,vpositions):
+        x, y = ungroup_vector_property(vpositions, [0, 1])
+        y.fa *= -1
+        y.fa -= y.fa.min()
+        return group_vector_property([x, y])
+    
+    #
+    def show(self,inline=True):
+        graph = self.graph
+        positions = self._flipY(self.v_pos)
+        labels = self.v_label
+        colors = self.v_color
+        weights = self.e_weight
+        graph_draw(graph, inline=inline,output_size=(300,300),fit_view=0.7,
+                        pos=positions, 
+                        vertex_text=labels,
+                        vertex_font_family="sans serif",
+                        vertex_font_size=18,
+                        vertex_font_weight=cairo.FONT_WEIGHT_BOLD,
+                        vertex_fill_color=colors,
+                        vertex_size = 50,
+                        vertex_halo=False,
+                        vertex_pen_width=1.2,
+                        vertex_color="#999999",
+                        edge_pen_width=weights)
+        
+    def get_vertex_labels(self):
+        return self._get_prop_values('lbl')
+    
+    def get_vertex_colours(self):
+        return self._get_prop_values('clr')
+    
+    def get_vertex_positions(self):
+        return self._get_prop_values('pos')

reflexive-1.2.2/tests/test_reflexive.py

@@ -0,0 +1,110 @@
+import os
+import sys
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+import logging
+logging.basicConfig(level=logging.INFO)
+
+import json
+import pandas as pd
+import time
+
+import reflexive as rfx
+
+# Create a config with a profile (local credentials) - leave blank for 'default' profile
+cfg = rfx.cfg.Config('r3-admin')
+# Set default parameters - specify name_prefix, local_path, and date_string for custom options
+parameters = cfg.set_parameters()
+#print("After set_parameters:",parameters)
+
+# Create a session with AWS using profile - some parameters require AWS session (like account number)
+aws = rfx.session.AWS(cfg)
+#print("aws parameters:",cfg.get_parameters())
+
+# Set the parameters for S3 - required for comprehend
+parameters = cfg.set_s3_parameters(s3_access_point="qut-gibsonap-reflexive",s3_bucket_name="au-edu-qld-qut-gibsonap-reflexive")
+#print("After set_s3_parameters:",parameters)
+
+# Set comprehend parameters
+parameters = cfg.set_comprehend_parameters(comprehend_service_role_name="AmazonComprehendServiceRole-gibsonap-reflexive")
+#print("After set_comprehend_parameters:",parameters)
+
+# Set custom entity recogniser parameters
+parameters = cfg.set_comprehend_custom_entity_parameters(reflexive_entity_name="ReflexiveExpressionRecogniser",reflexive_entity_version="v17")
+#print("After set_comprehend_custom_entity_parameters:",parameters)
+
+# Display all parameters stored in Config
+params_fmt = json.dumps(parameters, indent=2)
+#print(params_fmt)
+
+# Create a new S3 client
+s3 = rfx.session.S3(aws)
+s3_client = s3.client()
+
+# Create a new Comprehend client
+comp = rfx.session.Comprehend(aws)
+comp_client = comp.client()
+
+# Perform test analysis
+test_dict = {"text":["This is a test text. I hope that this will work for some basic analysis. Previously, this has not been as straight forward as I thought. Perhaps this time may be different. With any luck, this will not be a frustrating experience, but I hope that it will just work straight away. I can only hope!"]}
+test_series = pd.Series(test_dict['text'])
+df = pd.DataFrame.from_dict(test_dict)
+#print(df.describe())
+
+nlp = rfx.analyse.Nlp(aws)
+
+# Text length - this is needed for comprehend analytics
+df = nlp.text_length(df)
+df = nlp.remove_IQR_outliers(df)
+#print(df.describe())
+
+# Domain terms
+#domain_terms = {"hedge":['hope','perhaps','luck'],"emotion":['hope','frustrating']}
+#nlp.add_domain_terms(domain_terms)
+#df = nlp.match_domain_terms(df)
+#print(df['domain_counts'])
+
+# Ngrams
+#top_ngrams = nlp.get_top_ngrams(df['text'],2)
+#print(top_ngrams)
+#df = nlp.match_top_ngrams(df)
+#print(df)
+
+# Comprehend analysis
+# results = nlp.comprehend_analysis(comp,df)
+# print(results)
+# errors = nlp.check_results(results)
+# print(errors)
+# if errors=={}:
+#    print("No errors, so adding results to dataframe")
+#    df = nlp.add_results_to_df(results,df)
+# print(df)
+
+# df.to_pickle(f"{cfg.local_path}comp_df.pkl")
+df = pd.read_pickle(f"{cfg.local_path}comp_df.pkl")
+
+# Comprehend analytics
+df = nlp.comprehend_analytics(df)
+df.to_csv(f"{cfg.local_path}comp_analytics.csv")
+print("DONE!!")
+
+# Reflexive expression analysis
+response = nlp.analyse_reflexive_expressions(df,s3,comp)
+print(response)
+job_id = comp.get_current_job_id()
+time.sleep(5)
+status = comp.check_job_status()
+print(status)
+time.sleep(5)
+details = comp.get_job_details()
+print(details)
+
+while status=="IN_PROGRESS":
+    time.sleep(10)
+    status = comp.check_job_status()
+    print(status)
+
+result = comp.download_and_extract(s3)
+print(result)
+
+

reflexive-1.2.0/src/reflexive/__init__.py

@@ -1,5 +0,0 @@
-import reflexive.analyse
-import reflexive.cfg
-import reflexive.session
-import reflexive.util
-import reflexive.visualise