MSTR-Robotics-magerdaniel 0.3.9__tar.gz → 0.3.11__tar.gz

{mstr_robotics_magerdaniel-0.3.9 → mstr_robotics_magerdaniel-0.3.11/MSTR_Robotics_magerdaniel.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: MSTR_Robotics_magerdaniel
-Version: 0.3.9
+Version: 0.3.11
 Summary: MicroStrateg(P)ython
 Home-page: https://github.com/magerdaniel
 Author: Daniel Mager

{mstr_robotics_magerdaniel-0.3.9 → mstr_robotics_magerdaniel-0.3.11}/MSTR_Robotics_magerdaniel.egg-info/SOURCES.txt

@@ -12,10 +12,11 @@ mstr_robotics/_connectors.py
 mstr_robotics/_helper.py
 mstr_robotics/_lu_data.py
 mstr_robotics/_mod_prj_obj.py
-mstr_robotics/ai_openAi_faisst.py
 mstr_robotics/dossier.py
 mstr_robotics/mstr_classes.py
 mstr_robotics/mstr_pandas.py
+mstr_robotics/navigation.py
 mstr_robotics/prepare_AI_data.py
 mstr_robotics/read_out_prj_obj.py
-mstr_robotics/report.py
+mstr_robotics/report.py
+mstr_robotics/user_RAG.py

{mstr_robotics_magerdaniel-0.3.9/MSTR_Robotics_magerdaniel.egg-info → mstr_robotics_magerdaniel-0.3.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: MSTR_Robotics_magerdaniel
-Version: 0.3.9
+Version: 0.3.11
 Summary: MicroStrateg(P)ython
 Home-page: https://github.com/magerdaniel
 Author: Daniel Mager

{mstr_robotics_magerdaniel-0.3.9 → mstr_robotics_magerdaniel-0.3.11}/mstr_robotics/_connectors.py

@@ -193,6 +193,16 @@ class mstr_api():
         url = f'{conn.base_url}/api/v2/cubes/{cube_id}/instances?offset={offset_val}&limit={limit_val}'
         return conn.post(url)
 
+    def get_open_prp_stat(self,conn,report_id,instance_id):
+        url = f'{conn.base_url}/api/reports/{report_id}/instances/{instance_id}/status'
+        stat = conn.get(url)
+
+        try:
+            status=stat.json()["status"]
+        except:
+            print(stat)
+            status=1
+        return status
 
 
     def zzz_fetch_cube_elements(self,conn, cube_id, attribute_id, limit_val=100):

{mstr_robotics_magerdaniel-0.3.9 → mstr_robotics_magerdaniel-0.3.11}/mstr_robotics/_helper.py

@@ -128,6 +128,12 @@ class msic():
             dict_l.append(dict_d.copy())
         return dict_l
 
+    def get_vals_from_dict_l(self,dict_l, key="id"):
+        val_l = []
+        for d in dict_l:
+            val_l.append(d[key])
+        return val_l
+
     def add_prefix_to_dict_keys(self,dict, dpn_prefix, dict_cols= []):
         #adds a prefix to the keys of a dict
         # use case here is to distinct i.e. between the object_id and the depn_object_id

mstr_robotics_magerdaniel-0.3.11/mstr_robotics/navigation.py

@@ -0,0 +1,201 @@
+import pandas as pd
+from mstr_robotics.report import prompts,cube,rep
+from mstr_robotics.mstr_pandas import df_helper
+from mstr_robotics._helper import msic
+from mstr_robotics.prepare_AI_data import map_objects
+
+i_rep=rep()
+i_prompts=prompts()
+i_df_helper=df_helper()
+i_cube=cube()
+i_map_objects=map_objects()
+i_msic=msic()
+
+class answer_prompts():
+
+    def __init__(self,cube_obj_prp_rel_id
+                 ,cube_RAG_form_val_ans_id
+                 ,cube_att_form_def_id
+                 ,rep_dos_id):
+
+        self.cube_obj_prp_rel_id=cube_obj_prp_rel_id
+        self.cube_RAG_form_val_ans_id=cube_RAG_form_val_ans_id
+        self.rep_dos_id=rep_dos_id
+        self.cube_att_form_def_id=cube_att_form_def_id
+
+    def bld_exp_prp_met_ans(self,obj,operator,filter_val_l):
+        metric_exp_prp = {}
+        metric_exp_prp["met_id"] = obj["object_id"]
+        metric_exp_prp["data_type"] = "Real"
+        metric_exp_prp["operator"] = operator
+        metric_exp_prp["filter_val_l"] = filter_val_l
+        metric_exp_prp["level"] = "default"
+        prp_ans_d= i_prompts.bld_metric_exp_prp(metric_exp_prp=metric_exp_prp.copy())
+        return prp_ans_d
+
+    def bld_exp_prp_att_ans(self,conn,d_prp_ans_d,prp_ans_d_l, filt_obj_rag_df,operator,filter_val_l):
+
+        if operator in ("In", "notIn"):
+
+            filter_val_df = i_map_objects.bld_ai_prp_ans(conn, cube_id=self.cube_RAG_form_val_ans_id,
+                                                         rep_dos_id=self.rep_dos_id,
+                                                         key_word_l=filter_val_l)
+            filt_df = pd.merge(filt_obj_rag_df[["prompt_id", "prp_subType", "object_id", "obj_type"]],
+                               filter_val_df[
+                                   ["prompt_id", "prp_subType", "attribute_id", "attribute_name", "form_id",
+                                    "form_name", "form_dataType", "key"]],
+                               left_on=['object_id'], right_on=['attribute_id'], how='inner')
+            filt_df = i_df_helper.clean_double_col(df=filt_df)
+            # print(filt_df)
+            filter_val_l = []
+            for index, f_form_ele in filt_df.iterrows():
+                form_id = f_form_ele["form_id"]
+                form_dataType = f_form_ele["form_dataType"]
+                filter_val_l.append(f_form_ele["key"])
+
+            att_exp_ans_d = {"att_id": d_prp_ans_d["object_id"], "att_form_id": form_id,
+                             "form_data_type": form_dataType, "operator": operator,
+                             "filter_val_l": filter_val_l}
+
+            att_exp_ans_d = i_prompts.bld_att_exp_prp_l(prp_job_ans_d=att_exp_ans_d)
+            d_prp_ans_d["prp_ans_d"] = att_exp_ans_d
+            prp_ans_d_l.append(d_prp_ans_d.copy())
+            # print(d_prp_ans_d)
+
+        else:
+            col_id = i_cube.get_RAG_cube_col_mstr_id(conn=conn, cube_id=self.cube_att_form_def_id,
+                                                     col_name="attribute_id")
+            attr_elements = col_id + ":" + d_prp_ans_d["object_id"]
+            df = i_cube.quick_query_cube(conn, cube_id=self.cube_att_form_def_id, attribute_l=None,
+                                         metric_l=None,
+                                         attr_elements=attr_elements)
+
+            for index, form in df.iterrows():
+                if int(form["display_form_nr"]) > 0:
+                    att_exp_ans_d = {"att_id": form["attribute_id"], "att_form_id": form["form_id"],
+                                     "form_data_type": form["REST_form_type"], "operator": operator,
+                                     "filter_val_l": filter_val_l}
+
+                    att_exp_ans_d = i_prompts.bld_att_exp_prp(prp_job_ans_d=att_exp_ans_d)
+                    d_prp_ans_d["prp_ans_d"] = att_exp_ans_d
+                    prp_ans_d_l.append(d_prp_ans_d.copy())
+
+        return prp_ans_d_l
+
+    def bld_exp_prp_ans(self,conn,prp_ans_d_l,filt_obj_rag_df,operator,filter_val_l):
+        d_prp_ans_d = {}
+
+        for index, obj in filt_obj_rag_df.iterrows():
+            d_prp_ans_d["prompt_id"] = obj["prompt_id"]
+            d_prp_ans_d["prp_subType"] = obj["prp_subType"]
+            d_prp_ans_d["object_id"] = obj["object_id"]
+            d_prp_ans_d["obj_type"] = obj["obj_type"]
+            d_prp_ans_d["operator"] = operator
+
+            if obj["obj_type"] == "attribute":
+                prp_ans_d_l=self.bld_exp_prp_att_ans(conn,d_prp_ans_d,prp_ans_d_l, filt_obj_rag_df,operator,filter_val_l)
+
+                            # print(d_prp_ans_d)
+
+            if obj["obj_type"] == "metric":
+                d_prp_ans_d["prp_ans_d"] = self.bld_exp_prp_met_ans(obj,operator,filter_val_l )
+                prp_ans_d_l.append(d_prp_ans_d.copy())
+
+        return prp_ans_d_l
+
+    def bld_AI_filt_prp_ans(self,conn,vector_store,filter_d,*args,**kwargs):
+        prp_ans_d_l = []
+
+        for f in list(filter_d.keys()):
+            print(f)
+            filter_obj_name=vector_store.check_keyword( filt_obj_str=f)
+
+            filt_obj_rag_df = i_map_objects.bld_ai_prp_ans(conn, cube_id=self.cube_obj_prp_rel_id,
+                                                           rep_dos_id=self.rep_dos_id, key_word_l=[filter_obj_name])
+
+            filt_obj_rag_df = filt_obj_rag_df[filt_obj_rag_df['prp_subType'].isin(['prompt_expression'])]
+
+            operator = next(iter(filter_d[f]))
+            filter_val_l = filter_d[f][operator]
+
+            prp_ans_d_l=self.bld_exp_prp_ans(conn=conn, prp_ans_d_l=prp_ans_d_l
+                                 , filt_obj_rag_df=filt_obj_rag_df
+                                 , operator=operator, filter_val_l=filter_val_l)
+
+        return prp_ans_d_l
+
+    def bld_ai_obj_ans_prp(self,conn,json_t_d):
+        key_l = []
+        obj_l = []
+
+        for obj_type in json_t_d["template"].keys():
+            for obj in json_t_d["template"][obj_type]:
+                template_obj_d = {"obj_type": obj_type, "obj": obj}
+                obj_l.append(template_obj_d.copy())
+
+        for obj in obj_l:
+            key_l.append(obj["obj"])
+
+        template_obj_rag_df = i_map_objects.bld_ai_prp_ans(conn, cube_id=self.cube_obj_prp_rel_id,
+                                                           rep_dos_id=self.rep_dos_id, key_word_l=key_l)
+        template_obj_rag_df = template_obj_rag_df[template_obj_rag_df['prp_subType'].isin(['prompt_objects'])]
+        template_obj_rag_df = template_obj_rag_df[['prompt_id', 'prp_subType', "object_id", "obj_type"]]
+
+        template_obj_rag_df.rename(columns={'object_id': 'id', 'obj_type': 'type'}, inplace=True)
+        prp_ans_d_l = template_obj_rag_df.groupby(['prompt_id']).apply(
+            lambda x: x[['id', 'type']].to_dict(orient='records')).to_dict()
+        obj_prp_ans_l = []
+        for prp in prp_ans_d_l:
+            obj_prp_ans_d = {}
+            obj_prp_ans_d["id"] = prp
+            obj_prp_ans_d["type"] = "OBJECTS"
+            obj_prp_ans_d["answers"] = prp_ans_d_l[prp]
+            obj_prp_ans_l.append(obj_prp_ans_d)
+
+        return obj_prp_ans_l
+
+    def merge_exp_prp_ans_l(self,ai_prp_filt_ans_d):
+        prp_ans_l = []
+        prp_id_l = i_msic.get_key_form_dict_l(dict_l=ai_prp_filt_ans_d, key="prompt_id")
+        prp_ans_d_all = {}
+        operand_X = "AND"
+        for prp_id in prp_id_l:
+            prp_ans_d_all[prp_id] = []
+
+        for prp_ans in ai_prp_filt_ans_d:
+            for prp_id in prp_ans_d_all.keys():
+                if prp_ans["prompt_id"] == prp_id:
+                    prp_ans_d_all[prp_id].append(prp_ans["prp_ans_d"])
+
+        for prp_id in prp_ans_d_all:
+            # print(prp_ans_d_all[prp_id])
+            prp_ans_d_all[prp_id] = i_prompts.bld_exp_operands_d(p_ans_d_d_l=prp_ans_d_all[prp_id], operator="AND")
+
+        for prp_ans_id in prp_ans_d_all:
+            for prp in ai_prp_filt_ans_d:
+                if prp_ans_id == prp["prompt_id"]:
+                    if prp["prp_subType"] == "prompt_expression" and prp["obj_type"] == "attribute":
+                        prp_ans_l.append(i_prompts.bld_prp_exp_d(prompt_id=prp_ans_id,
+                                                       p_ans_d_j=prp_ans_d_all[prp_ans_id], operator="And"))
+
+                    if prp["prp_subType"] == "prompt_expression" and prp["obj_type"] == "metric":
+                        prp_ans_l.append(i_prompts.frame_metric_exp_prp(prompt_id=prp_ans_id, p_ans_d_j=prp_ans_d_all[prp_ans_id]))
+                    break
+        return prp_ans_l
+
+    def save_AI_rep(self, conn, report_id, prompt_answ, ai_rep_name, ai_rep_folder_id):
+        instance_id = i_rep.open_Instance(conn=conn, report_id=report_id)
+        first_draft = i_rep.set_inst_prompt_ans(conn=conn, report_id=report_id, instance_id=instance_id,
+                                                prompt_answ=prompt_answ)
+
+        rep_stat = i_rep.get_open_prp_stat(conn=conn, report_id=report_id, instance_id=instance_id)
+
+        if rep_stat == 2:
+            prompt_answ = i_rep.close_open_prp(conn=conn, report_id=report_id, instance_id=instance_id,
+                                         prompt_answ=prompt_answ)
+        rep_id = i_rep.save_rep_inst(conn=conn, instance_id=instance_id, rep_name=ai_rep_name
+                                          , save_mode="OVERWRITE"
+                                          , promptOption="static"
+                                          , setCurrentAsDefaultAnswer=True)
+
+        return rep_id

{mstr_robotics_magerdaniel-0.3.9 → mstr_robotics_magerdaniel-0.3.11}/mstr_robotics/prepare_AI_data.py

@@ -1,25 +1,17 @@
-import os
-import pandas as pd
-
 from mstr_robotics.read_out_prj_obj import i_prompts
 from mstr_robotics.report import cube,rep
 from mstr_robotics._helper import msic
 from mstr_robotics._connectors import mstr_api
 from mstr_robotics.mstr_classes import mstr_global
-from mstrio.project_objects import OlapCube
 from mstr_robotics.read_out_prj_obj import io_attributes
 
-io_att=io_attributes()
-from flashtext import KeywordProcessor
-from itertools import chain
-import json
+from mstrio.project_objects import OlapCube
 
+io_att=io_attributes()
 i_cube=cube()
 i_msic=msic()
 i_mstr_global=mstr_global()
 i_rep=rep()
-i_KeywordProcessor = KeywordProcessor()
-i_chain=chain()
 i_mstr_api=mstr_api()
 
 class read_out_cube_att():
@@ -32,7 +24,7 @@ class read_out_cube_att():
         return el_ans_d
 
 
-    def read_cube_att_elements(self,conn,cube_list_l ):
+    def zzz_read_cube_att_elements(self,conn,cube_list_l ):
         key_val_l=[]
 
         for cube_id in cube_list_l:
@@ -73,9 +65,6 @@ class read_out_cube_att():
 
         return rag_att_form_id_d
 
-    #def read_cube_att_form_exp_val_l(conn, cube_id,*args,**kwargs):
-
-
     def fetch_cube_elements(self, conn, cube_id,limit_val=10000,*args,**kwargs):
         rag_att_form_l = []
         offset_val = 0
@@ -111,40 +100,6 @@ class read_out_cube_att():
             all_cube_element_l.extend(cube_element_l)
         return all_cube_element_l
 
-    def zzz_read_cube_att_form_exp_val_l(self,conn,cube_list_l):
-        element_d_l = []
-        for cube_id in cube_list_l:
-            cube = OlapCube(connection=conn, id=cube_id["id"])
-            cube_df = cube.to_dataframe()
-            cube_def = i_cube.get_cube_def(conn=conn, cube_id=cube_id["id"])
-
-            for col in cube_df.columns:
-                #print(cube_id["id"])
-                #print(col)
-                col_att_form_l = col.split("@")
-                if len(col_att_form_l) > 1:
-                    #print("col added")
-                    att_name = col_att_form_l[0]
-                    form_name = col_att_form_l[1]
-                    att_form_d = self.read_cube_att_forms(conn=conn,cube_def=cube_def,att_name=att_name,form_name=form_name)
-                    distinct_val = cube_df[col].unique()
-                    for val in distinct_val:
-                        att_form_d["key"] = val
-                        #att_form_d["prp_ans"] = val
-                        element_d_l.append(att_form_d.copy())
-                else:
-                    #print("col not added")
-                    print("att_name :" +str(col))
-                    att_name = col
-                    att_form_d = self.read_cube_att_forms(conn=conn,cube_def=cube_def,att_name=att_name)
-                    distinct_val = cube_df[col].unique()
-                    for val in distinct_val:
-                        att_form_d["key"] = val
-                        element_d_l.append(att_form_d.copy())
-
-
-        return element_d_l
-
 class map_objects():
 
     def get_doss_rep_prp(self,conn, object_l):
@@ -183,85 +138,27 @@ class map_objects():
                 cube_att_l.append(cube_att_d.copy())
         return cube_att_l
 
-    def get_rag_rep_prp_df(self,conn,cube_id,rep_dos_id,key_word_l,mstr_rag_col_d):
-        rag_rep_prp_df=self.bld_ai_prp_ans(conn=conn,cube_id=cube_id,mstr_rag_col_d=mstr_rag_col_d, rep_dos_id=rep_dos_id, key_word_l=key_word_l)
-        return rag_rep_prp_df
+    def bld_ai_prp_ans(self,conn, cube_id,  rep_dos_id, key_word_l):
 
-    def bld_ai_prp_ans(self,conn, cube_id, mstr_rag_col_d, rep_dos_id, key_word_l):
-        obj_prp_ans_l = []
-        rag_files = []
-        cube_disp_col_l = ["prompt_id", "prp_subType", "rep_dos_id", "object_id", "attribute_id", "obj_type",
-                           "obj_prp_ans", "obj_prp_rel_dummy", "rep_prp_rel_dummy", "obj_prp_rel_dummy"]
-        cube_disp_col_l = ["attribute_id"]
-        key_val_l = []
         disp_col_ids_l = []
         attr_elements_l = []
         key_val_l = ["key"]  #
         # print(mstr_rag_col_d)
+        mstr_rag_col_d=i_cube.get_mtdi_cube_col_id(conn, cube_l=[cube_id])
         for col_name in mstr_rag_col_d[cube_id].keys():
-            cube_disp_col_l = mstr_rag_col_d[cube_id].keys()
+            #cube_disp_col_l = mstr_rag_col_d[cube_id].keys()
             # print(col_name)
             if col_name == 'rep_dos_id':
                 rep_dos_id_filt = mstr_rag_col_d[cube_id][col_name] + ":" + rep_dos_id
                 attr_elements_l.append(rep_dos_id_filt)
 
-            if col_name in cube_disp_col_l:
-                disp_col_ids_l.append(mstr_rag_col_d[cube_id][col_name])
-
             if col_name in key_val_l:
                 for key in key_word_l:
                     prp_ans_j = mstr_rag_col_d[cube_id][col_name] + ":" + str(key)
                     attr_elements_l.append(prp_ans_j)
 
+            disp_col_ids_l.append(mstr_rag_col_d[cube_id][col_name])
+
         df = i_cube.quick_query_cube(conn=conn, cube_id=cube_id, attribute_l=disp_col_ids_l, metric_l=None,
                               attr_elements=attr_elements_l)
         return df
-
-
-class zzz_bld_obj_ans():
-
-    def zzz_trans_att_exp_prp(self,attributes_l):
-        att_form_d_l = []
-        for att_form in attributes_l:
-            att_form_d = {}
-            att_form_d["attribute_id"] = att_form["attribute_id"]
-            att_form_d["attribute_name"] = att_form["attribute_name"]
-            att_form_d["form_id"] = att_form["form_id"]
-            att_form_d["form_dataType"] = att_form["form_dataType"]
-            att_form_d["key"] = att_form["form_name"]
-            att_form_d["display_form_nr"] = att_form["display_form_nr"]
-            att_form_d["browse_form_nr"] = att_form["browse_form_nr"]
-            att_form_d_l.append(att_form_d.copy())
-            att_form_d_l=i_msic.rem_dbl_dict_in_l(dict_l=att_form_d_l)
-        return att_form_d_l
-
-    def bld_att_obj_ans(self,attributes_l):
-        # Now 'data' contains the parsed JSON as a Python dictionary or list
-        att_d_l = []
-        att_l = []
-        key_word_l = []
-        # documents_l=[]
-        for m in attributes_l:
-            if m["attribute_id"] not in att_l:
-                att_l.append(m["attribute_id"])
-                att_d = {}
-                att_d["object_id"] = m["attribute_id"]
-                att_d["type"] = "attribute"
-                att_d["key"] = m["attribute_name"]
-                att_d["ans"]=json.dumps({"id":m["attribute_id"] , "type":"attribute"})
-
-                att_d_l.append(att_d.copy())
-        #att_d_l = i_msic.rem_dbl_dict_in_l(dict_l=att_d_l)
-        return att_d_l
-
-    def bld_met_obj_ans(self,metrics_l):
-        all_met_d_l = []
-        for m in metrics_l:
-            if "E0CCB9CF22104A489CBE78D974AFD19E" in m["metric_path_ids"]:
-                row_d = {}
-                row_d["object_id"] = m["metric_id"]
-                row_d["type"] = "metric"
-                row_d["key"] = m["metric_name"]
-                row_d["ans"] = json.dumps({"id": m["metric_id"], "type": "metric"})
-                all_met_d_l.append(row_d.copy())
-        return list(all_met_d_l)

{mstr_robotics_magerdaniel-0.3.9 → mstr_robotics_magerdaniel-0.3.11}/mstr_robotics/report.py

@@ -8,9 +8,11 @@ from mstrio.project_objects.report import Report
 from mstrio.api.cubes import cube_definition
 import pandas as pd
 from mstr_robotics.mstr_pandas import df_helper
+from mstr_robotics._helper import msic
 import json
 
 i_mstr_api=mstr_api()
+i_msic=msic()
 
 class rep:
 
@@ -28,6 +30,10 @@ class rep:
                                              ,instance_id=instance_id, closed = False)
         return open_prompts.json()
 
+    def get_open_prp_stat(self,conn, report_id, instance_id):
+        return i_mstr_api.get_open_prp_stat(conn=conn, report_id=report_id,
+                                      instance_id=instance_id)
+
     def set_inst_prompt_ans(self, conn, report_id, instance_id, prompt_answ):
         prompt_answ_url = f'{conn.base_url}/api/reports/{report_id}/instances/{instance_id}/prompts/answers'
         ret_prompt_ans = conn.put(prompt_answ_url, data=prompt_answ)
@@ -163,6 +169,7 @@ class cube():
     def get_cube_def(self,conn,cube_id):
         return cube_definition(connection=conn, id=cube_id).json()
 
+
     def get_mtdi_cube_col_id(self,conn, cube_l):
 
         mstr_rag_col_d = {}
@@ -177,7 +184,8 @@ class cube():
             mstr_rag_col_d[cube_id] = att_col_keys_d.copy()
         return mstr_rag_col_d
 
-    def get_RAG_cube_col_mstr_id(self,cube_id,mstr_rag_col_d, col_name):
+    def get_RAG_cube_col_mstr_id(self,conn,cube_id,col_name):
+        mstr_rag_col_d=self.get_mtdi_cube_col_id(conn, cube_l=[cube_id])
         col_id = mstr_rag_col_d[cube_id][col_name]
         return col_id
 
@@ -364,6 +372,35 @@ class prompts():
     def bld_obj_prp_json(self,object_id,object_type):
         return json.dumps({"id": object_id, "type": object_type})
 
+    def close_open_prp(self,conn, report_id, instance_id, prompt_answ):
+        # checks the answered prp
+        prompt_answ_d = json.loads(prompt_answ)
+        prp_ans_d_l = prompt_answ_d["prompts"]
+        prp_ans_id_l = i_msic.keep_cols_from_dict_l(list_l=prp_ans_d_l, keep_cols=["id"])
+
+        # checks the all prp in report / dashboard
+        rep_open_prp_d_l = rep().get_open_prompts(conn=conn, report_id=report_id, instance_id=instance_id)
+        rep_open_prp_id_l = i_msic.keep_cols_from_dict_l(list_l=rep_open_prp_d_l, keep_cols=["id"])
+
+        # checks the prp not answered jet
+        open_prp_l = [x for x in rep_open_prp_id_l if x not in prp_ans_id_l]
+        open_prp_l = i_msic.get_vals_from_dict_l(dict_l=open_prp_l)
+
+        for prp in rep_open_prp_d_l:
+            print(prp["id"])
+            if prp["id"] in open_prp_l:
+                print(prp)
+                open_prp_d = {}
+                open_prp_d["id"] = prp["id"]
+                open_prp_d["type"] = prp["type"]
+                if prp["type"] in ["OBJECTS", "ELEMENTS"]:
+                    open_prp_d["answers"] = []
+                else:
+                    open_prp_d["answers"] = {}
+                rep_open_prp_d_l.append(open_prp_d.copy())
+        prompt_answ = self.frame_prp(prp_ans=rep_open_prp_d_l)
+        return prompt_answ
+
     def zzz_loop_att_exp_prp(self,prp_job_ans_l):
         expr_JSON_l = []
         for att_exp_prp in prp_job_ans_l:
@@ -382,6 +419,7 @@ class prompts():
             expr_JSON_l.append(att_form_exp_j)
 
         return expr_JSON_l
+
     def zzz_get_form_type(self,baseFormType):
         form_type_d={"number":"Numeric",
                      "big_decimal":"BigDecimal"}

mstr_robotics_magerdaniel-0.3.11/mstr_robotics/user_RAG.py

@@ -0,0 +1,149 @@
+import json
+from openai import OpenAI
+from flashtext import KeywordProcessor
+from langchain.vectorstores import FAISS
+from langchain.embeddings import OpenAIEmbeddings
+
+class keyword_processor():
+
+    def __init__(self):
+        self.KeywordProcessor = KeywordProcessor()
+
+    def load_keyword_processor(self,key_l):
+        for val in key_l:
+            self.KeywordProcessor.add_keyword(val)
+        return self.KeywordProcessor
+
+    def extract_keywords(self,msg_t):
+        keywords_l = self.KeywordProcessor.extract_keywords(msg_t)
+        return keywords_l
+
+    def check_keyword(self,filt_obj_str):
+        keywords_l = self.KeywordProcessor.extract_keywords(filt_obj_str)
+        return keywords_l[0]
+
+class vectorDB_faisst():
+
+    def __init__(self,sKey):
+        self.i_embeddings = OpenAIEmbeddings(openai_api_key=sKey)
+
+    def load_vector_store(self,key_l):
+        # Flatten and ensure all items are strings
+        # Create FAISS vector store
+        self.vector_store = FAISS.from_texts(key_l, self.i_embeddings)
+
+        return self.vector_store
+
+    def check_keyword(self, filt_obj_str):
+        results = self.vector_store.similarity_search_with_score(query=filt_obj_str, top_k=1)
+        filter_obj_name = results[0][0].page_content
+        return filter_obj_name
+
+class mstr_openAI():
+
+    def __init__(self,sKey):
+        #self.sKey=sKey
+        self.i_embeddings = OpenAIEmbeddings(openai_api_key=sKey)
+
+
+    def check_trans_chatGPT(self,sKey,messages,*args,**kwargs):
+        chat_completion = self.call_open_AI(sKey=sKey,messages=messages,*args,**kwargs)
+        js = json.loads(chat_completion.json())
+        return js
+
+    def call_open_AI(self, sKey, messages, max_tokens=1000, temperature=0.3, model="gpt-4o-mini"):
+        # This is the default and can be omitted
+        client = OpenAI(api_key=sKey)
+
+        chat_completion = client.chat.completions.create(
+            messages=messages,
+            # model="gpt-3.5-turbo",
+            model=model,
+            max_tokens=max_tokens,
+            temperature=temperature
+        )
+        js = json.loads(chat_completion.json())
+        return js
+
+class chat_bot_RAG():
+
+    def filter_RAG_l(self,msg_t, key_word_l, msg_filter_t):
+        messages = [
+            {
+                "role": "system",
+                "content": f"You will be provided unstructured filter definition of the BI report request. the full message is {msg_t}"
+            },
+            {
+                "role": "system",
+                "content": "Always use as Filteroperator expressions like In, GreaterEqual,Equals,IsNull,IsNotNull,Greater,Between,BeginsWith,Like"
+            },
+            {
+                "role": "system",
+                "content": "When filtering a list, please ensure, that you use In od NotIn as operator"
+            },
+            {
+                "role": "system",
+                "content": "Ensure, that the out pu is a dictionary consiting of keywords"
+            },
+            {"role": "system",
+             "content": "top_n or bottom_n is part of the filter statement"
+             },
+            {
+                "role": "system",
+                "content": f"keywrods are: {key_word_l} "
+            },
+            {
+                "role": "system",
+                "content": " Please provide the filters as a valid Python dictionary without any code block formatting or syntax highlighting, where the kefilter objects"
+            },
+            {
+                "role": "system",
+                "content": "Set brackets to split multiple filter criteria"
+            },
+            {
+                "role": "system",
+                "content": "Ignore keywords, if they are not relevant for filtering"
+            },
+            {
+                "role": "system",
+                "content": f"keywrods are: {key_word_l} "
+            },
+            {
+                "role": "user",
+                "content": f"{msg_filter_t}"
+            }
+        ]
+        return messages
+
+    def split_AI_msg(self,msg_t, key_word_l):
+        messages = [
+            {
+                "role": "system",
+                "content": "You will be provided unstructured BI report request."
+            },
+            {
+                "role": "system",
+                "content": " Please provide the content as a valid Python dictionary without any code block formatting or syntax highlighting, where the keywords are splited in template and filter objects"
+            },
+            {
+                "role": "system",
+                "content": "Please use the key template and filter split"
+            },
+            {
+                "role": "system",
+                "content": "Set brackets to split multiple filter criteria"
+            },
+            {
+                "role": "system",
+                "content": "Please include in the JSON file, the filter critria as string"
+            },
+            {
+                "role": "system",
+                "content": f"keywrods are: {key_word_l} "
+            },
+            {
+                "role": "user",
+                "content": f"{msg_t}"
+            }
+        ]
+        return messages

{mstr_robotics_magerdaniel-0.3.9 → mstr_robotics_magerdaniel-0.3.11}/setup.py

@@ -7,7 +7,7 @@ long_description = (this_directory / "README.md").read_text()
 
 setup(
     name="MSTR_Robotics_magerdaniel",
-    version="0.3.9",
+    version="0.3.11",
     description="MicroStrateg(P)ython",
     author="Daniel Mager",
     author_email="danielmager@gmx.de",

mstr_robotics_magerdaniel-0.3.9/mstr_robotics/ai_openAi_faisst.py

@@ -1,250 +0,0 @@
-import os
-import pandas as pd
-from langchain.vectorstores import FAISS
-from langchain.embeddings import OpenAIEmbeddings
-from openai import OpenAI
-from flashtext import KeywordProcessor
-from itertools import chain
-from mstr_robotics.report import prompts,cube,rep
-from mstr_robotics.mstr_pandas import df_helper
-from mstr_robotics._helper import msic
-from mstr_robotics.prepare_AI_data import map_objects
-import json
-
-i_KeywordProcessor = KeywordProcessor()
-i_chain=chain()
-i_rep=rep()
-i_prompts=prompts()
-i_df_helper=df_helper()
-i_cube=cube()
-i_map_objects=map_objects()
-i_msic=msic()
-
-class vectorDB_faisst():
-
-    def __init__(self,sKey):
-        #self.sKey=sKey
-        os.environ["OPENAI_API_KEY"] = sKey
-        self.i_embeddings = OpenAIEmbeddings()
-
-    def load_vector_store(self,key_l):
-        # Flatten and ensure all items are strings
-        # Create FAISS vector store
-        vector_store = FAISS.from_texts(key_l, self.i_embeddings)
-
-        return vector_store
-
-    def load_keyword_processor(self,key_l):
-        for val in key_l:
-            i_KeywordProcessor.add_keyword(val)
-        return i_KeywordProcessor
-
-    def extract_keywords(self,msg_t):
-        keywords_l = i_KeywordProcessor.extract_keywords(msg_t)
-        return keywords_l
-
-class run_chat_request():
-
-    def __init__(self,cube_obj_prp_rel_id
-                 ,cube_RAG_form_val_ans_id
-                 ,cube_att_form_def_id
-                 ,rep_dos_id
-                 ,mstr_rag_col_d):
-
-        self.cube_obj_prp_rel_id=cube_obj_prp_rel_id
-        self.cube_RAG_form_val_ans_id=cube_RAG_form_val_ans_id
-        self.rep_dos_id=rep_dos_id
-        self.mstr_rag_col_d=mstr_rag_col_d
-        self.cube_att_form_def_id=cube_att_form_def_id
-
-
-    def bld_AI_filt_prp_ans(self,conn,vector_store,filter_d):
-        # print(json_f)
-        # v_andOr_s="And"
-        # att_filt_prp_id="1F21042E4FC08886A9D7E4920CA4EC59"
-        # met_filt_prp_id="14B57F1F46127BC6E6F746888BC3C7CB"
-        prp_ans_d_l = []
-
-        # ={}
-        last_p_ans_d_j = {}
-
-        for f in list(filter_d.keys()):
-            print(f)
-            results = vector_store.similarity_search_with_score(query=f, top_k=1)
-
-            filter_obj_name = results[0][0].page_content
-            #print(filter_obj_name)
-            #print(self.mstr_rag_col_d)
-            filt_obj_rag_df = i_map_objects.get_rag_rep_prp_df(conn, cube_id=self.cube_obj_prp_rel_id, rep_dos_id=self.rep_dos_id,
-                                                 key_word_l=[filter_obj_name],mstr_rag_col_d=self.mstr_rag_col_d)
-
-            filt_obj_rag_df = filt_obj_rag_df[filt_obj_rag_df['prp_subType'].isin(['prompt_expression'])]
-
-            operator = next(iter(filter_d[f]))
-            filter_val_l = filter_d[f][operator]
-            #print("DDDDDDDDDDDDDDDDD")
-            #print(filt_obj_rag_df)
-            d_prp_ans_d = {}
-            for index, obj in filt_obj_rag_df.iterrows():
-                d_prp_ans_d["prompt_id"] = obj["prompt_id"]
-                d_prp_ans_d["prp_subType"] = obj["prp_subType"]
-                d_prp_ans_d["object_id"] = obj["object_id"]
-                d_prp_ans_d["obj_type"] = obj["obj_type"]
-                d_prp_ans_d["operator"] = operator
-                if obj["obj_type"] == "attribute":
-                    if operator in ("In", "notIn"):
-
-                        filter_val_df = i_map_objects.get_rag_rep_prp_df(conn, cube_id=self.cube_RAG_form_val_ans_id,
-                                                           rep_dos_id=self.rep_dos_id, key_word_l=filter_val_l,mstr_rag_col_d=self.mstr_rag_col_d)
-                        filt_df = pd.merge(filt_obj_rag_df[["prompt_id", "prp_subType", "object_id", "obj_type"]],
-                                           filter_val_df[
-                                               ["prompt_id", "prp_subType", "attribute_id", "attribute_name", "form_id",
-                                                "form_name", "form_dataType", "key"]],
-                                           left_on=['object_id'], right_on=['attribute_id'], how='inner')
-                        filt_df = i_df_helper.clean_double_col(df=filt_df)
-                        #print(filt_df)
-                        filter_val_l = []
-                        for index, f_form_ele in filt_df.iterrows():
-                            form_id = f_form_ele["form_id"]
-                            form_dataType = f_form_ele["form_dataType"]
-                            filter_val_l.append(f_form_ele["key"])
-
-                        att_exp_ans_d = {"att_id": d_prp_ans_d["object_id"], "att_form_id": form_id,
-                                         "form_data_type": form_dataType, "operator": operator,
-                                         "filter_val_l": filter_val_l}
-
-                        att_exp_ans_d = i_prompts.bld_att_exp_prp_l(prp_job_ans_d=att_exp_ans_d)
-                        d_prp_ans_d["prp_ans_d"] = att_exp_ans_d
-                        prp_ans_d_l.append(d_prp_ans_d.copy())
-                        #print(d_prp_ans_d)
-
-                    else:
-                        col_id = i_cube.get_RAG_cube_col_mstr_id(cube_id=self.cube_att_form_def_id, col_name="attribute_id",mstr_rag_col_d=self.mstr_rag_col_d)
-                        attr_elements = col_id + ":" + obj["object_id"]
-                        df = i_cube.quick_query_cube(conn, cube_id=self.cube_att_form_def_id, attribute_l=None, metric_l=None,
-                                              attr_elements=attr_elements)
-
-                        for index, form in df.iterrows():
-                            if int(form["display_form_nr"]) > 0:
-                                att_exp_ans_d = {"att_id": form["attribute_id"], "att_form_id": form["form_id"],
-                                                 "form_data_type": form["REST_form_type"], "operator": operator,
-                                                 "filter_val_l": filter_val_l}
-
-                                att_exp_ans_d = i_prompts.bld_att_exp_prp(prp_job_ans_d=att_exp_ans_d)
-                                d_prp_ans_d["prp_ans_d"] = att_exp_ans_d
-                                prp_ans_d_l.append(d_prp_ans_d.copy())
-                                #print(d_prp_ans_d)
-
-                if obj["obj_type"] == "metric":
-                    metric_exp_prp = {}
-                    metric_exp_prp["met_id"] = obj["object_id"]
-                    metric_exp_prp["data_type"] = "Real"
-                    metric_exp_prp["operator"] = operator
-                    metric_exp_prp["filter_val_l"] = filter_val_l
-                    metric_exp_prp["level"] = "default"
-                    d_prp_ans_d["prp_ans_d"] = i_prompts.bld_metric_exp_prp(metric_exp_prp=metric_exp_prp.copy())
-                    prp_ans_d_l.append(d_prp_ans_d.copy())
-                    #print(print(d_prp_ans_d))
-
-            # prp_ans_d_l.append(last_p_ans_d_j.copy())
-        return prp_ans_d_l
-
-    def bld_ai_obj_ans_prp(self,conn,json_t_d):
-        key_l = []
-        col_id = i_cube.get_RAG_cube_col_mstr_id(cube_id=self.cube_obj_prp_rel_id, col_name="object_id",mstr_rag_col_d=self.mstr_rag_col_d)
-        obj_l = []
-        template_obj_d = {}
-
-        for obj_type in json_t_d["template"].keys():
-            for obj in json_t_d["template"][obj_type]:
-                template_obj_d = {"obj_type": obj_type, "obj": obj}
-                obj_l.append(template_obj_d.copy())
-
-        for obj in obj_l:
-            key_l.append(obj["obj"])
-
-        template_obj_rag_df = i_map_objects.get_rag_rep_prp_df(conn, cube_id=self.cube_obj_prp_rel_id, rep_dos_id=self.rep_dos_id,
-                                                 key_word_l=key_l,mstr_rag_col_d=self.mstr_rag_col_d)
-        template_obj_rag_df = template_obj_rag_df[template_obj_rag_df['prp_subType'].isin(['prompt_objects'])]
-        template_obj_rag_df = template_obj_rag_df[['prompt_id', 'prp_subType', "object_id", "obj_type"]]
-
-        template_obj_rag_df.rename(columns={'object_id': 'id', 'obj_type': 'type'}, inplace=True)
-        prp_ans_d_l = template_obj_rag_df.groupby(['prompt_id']).apply(
-            lambda x: x[['id', 'type']].to_dict(orient='records')).to_dict()
-        obj_prp_ans_l = []
-        for prp in prp_ans_d_l:
-            obj_prp_ans_d = {}
-            obj_prp_ans_d["id"] = prp
-            obj_prp_ans_d["type"] = "OBJECTS"
-            obj_prp_ans_d["answers"] = prp_ans_d_l[prp]
-            obj_prp_ans_l.append(obj_prp_ans_d)
-
-        return obj_prp_ans_l
-
-    def merge_exp_prp_ans_l(self,ai_prp_filt_ans_d):
-        prp_ans_l = []
-        prp_id_l = i_msic.get_key_form_dict_l(dict_l=ai_prp_filt_ans_d, key="prompt_id")
-        prp_ans_d_all = {}
-        operand_X = "AND"
-        for prp_id in prp_id_l:
-            prp_ans_d_all[prp_id] = []
-
-        for prp_ans in ai_prp_filt_ans_d:
-            for prp_id in prp_ans_d_all.keys():
-                if prp_ans["prompt_id"] == prp_id:
-                    prp_ans_d_all[prp_id].append(prp_ans["prp_ans_d"])
-
-        for prp_id in prp_ans_d_all:
-            # print(prp_ans_d_all[prp_id])
-            prp_ans_d_all[prp_id] = i_prompts.bld_exp_operands_d(p_ans_d_d_l=prp_ans_d_all[prp_id], operator="AND")
-
-        for prp_ans_id in prp_ans_d_all:
-            for prp in ai_prp_filt_ans_d:
-                if prp_ans_id == prp["prompt_id"]:
-                    if prp["prp_subType"] == "prompt_expression" and prp["obj_type"] == "attribute":
-                        prp_ans_l.append(i_prompts.bld_prp_exp_d(prompt_id=prp_ans_id,
-                                                       p_ans_d_j=prp_ans_d_all[prp_ans_id], operator="And"))
-
-                    if prp["prp_subType"] == "prompt_expression" and prp["obj_type"] == "metric":
-                        prp_ans_l.append(i_prompts.frame_metric_exp_prp(prompt_id=prp_ans_id, p_ans_d_j=prp_ans_d_all[prp_ans_id]))
-                    break
-        return prp_ans_l
-
-    def save_AI_rep(self, conn, rep_wizzard_id, prompt_answ, ai_rep_name, ai_rep_folder_id):
-        instance_id = i_rep.open_Instance(conn=conn, report_id=rep_wizzard_id)
-        first_draft = i_rep.set_inst_prompt_ans(conn=conn, report_id=rep_wizzard_id, instance_id=instance_id,
-                                                     prompt_answ=prompt_answ)
-        rep_id = i_rep.save_rep_inst(conn=conn, instance_id=instance_id, rep_name=ai_rep_name
-                                          , save_mode="OVERWRITE"
-                                          , promptOption="static"
-                                          , setCurrentAsDefaultAnswer=True)
-
-        return rep_id
-
-
-class openAI():
-
-    def __init__(self,sKey):
-        #self.sKey=sKey
-        os.environ["OPENAI_API_KEY"] = sKey
-        self.i_embeddings = OpenAIEmbeddings()
-
-    def check_trans_chatGPT(self,sKey,messages,*args,**kwargs):
-        chat_completion = self.call_open_AI(sKey=sKey,messages=messages,*args,**kwargs)
-        js = json.loads(chat_completion.json())
-        return js
-
-    def call_open_AI(self, sKey, messages, max_tokens=1000, temperature=0.3, model="gpt-4o-mini"):
-        # This is the default and can be omitted
-        client = OpenAI(api_key=sKey)
-
-        chat_completion = client.chat.completions.create(
-            messages=messages,
-            # model="gpt-3.5-turbo",
-            model=model,
-            max_tokens=max_tokens,
-            temperature=temperature
-        )
-        js = json.loads(chat_completion.json())
-        return js
-