virgo-modules 0.0.62__tar.gz → 0.0.63__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of virgo-modules might be problematic. Click here for more details.
- {virgo_modules-0.0.62 → virgo_modules-0.0.63}/PKG-INFO +1 -1
- {virgo_modules-0.0.62 → virgo_modules-0.0.63}/setup.py +1 -1
- virgo_modules-0.0.63/virgo_app/virgo_modules/src/aws_utils.py +35 -0
- {virgo_modules-0.0.62 → virgo_modules-0.0.63}/virgo_app/virgo_modules/src/re_utils.py +174 -13
- {virgo_modules-0.0.62 → virgo_modules-0.0.63}/virgo_app/virgo_modules/src/ticketer_source.py +13 -9
- {virgo_modules-0.0.62 → virgo_modules-0.0.63}/virgo_app/virgo_modules.egg-info/PKG-INFO +1 -1
- virgo_modules-0.0.62/virgo_app/virgo_modules/src/aws_utils.py +0 -38
- {virgo_modules-0.0.62 → virgo_modules-0.0.63}/LICENSE +0 -0
- {virgo_modules-0.0.62 → virgo_modules-0.0.63}/README.md +0 -0
- {virgo_modules-0.0.62 → virgo_modules-0.0.63}/setup.cfg +0 -0
- {virgo_modules-0.0.62 → virgo_modules-0.0.63}/virgo_app/virgo_modules/__init__.py +0 -0
- {virgo_modules-0.0.62 → virgo_modules-0.0.63}/virgo_app/virgo_modules/src/__init__.py +0 -0
- {virgo_modules-0.0.62 → virgo_modules-0.0.63}/virgo_app/virgo_modules/src/edge_utils.py +0 -0
- {virgo_modules-0.0.62 → virgo_modules-0.0.63}/virgo_app/virgo_modules/src/pull_artifacts.py +0 -0
- {virgo_modules-0.0.62 → virgo_modules-0.0.63}/virgo_app/virgo_modules.egg-info/SOURCES.txt +0 -0
- {virgo_modules-0.0.62 → virgo_modules-0.0.63}/virgo_app/virgo_modules.egg-info/dependency_links.txt +0 -0
- {virgo_modules-0.0.62 → virgo_modules-0.0.63}/virgo_app/virgo_modules.egg-info/requires.txt +0 -0
- {virgo_modules-0.0.62 → virgo_modules-0.0.63}/virgo_app/virgo_modules.egg-info/top_level.txt +0 -0
|
@@ -5,7 +5,7 @@ with open("virgo_app/README.md", "r") as f:
|
|
|
5
5
|
|
|
6
6
|
setup(
|
|
7
7
|
name="virgo_modules",
|
|
8
|
-
version="0.0.
|
|
8
|
+
version="0.0.63",
|
|
9
9
|
description="data processing and statistical modeling using stock market data",
|
|
10
10
|
package_dir={"": "virgo_app"},
|
|
11
11
|
packages=find_packages(where="virgo_app"),
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import yaml
|
|
2
|
+
import boto3
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from io import StringIO, BytesIO
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def upload_file_to_aws(bucket,key,input_path, aws_credentials):
|
|
9
|
+
|
|
10
|
+
session = boto3.Session(aws_access_key_id=aws_credentials['AWS_ACCESS_KEY_ID'],aws_secret_access_key=aws_credentials['AWS_SECRET_ACCESS_KEY'])
|
|
11
|
+
bucket = aws_credentials[bucket]
|
|
12
|
+
s3 = session.resource('s3')
|
|
13
|
+
s3.meta.client.upload_file(Filename=input_path , Bucket=bucket, Key=key)
|
|
14
|
+
|
|
15
|
+
def upload_pandas_to_s3(data_frame,bucket,key, aws_credentials):
|
|
16
|
+
|
|
17
|
+
csv_buffer = StringIO()
|
|
18
|
+
data_frame.to_csv(csv_buffer)
|
|
19
|
+
csv_buffer.seek(0)
|
|
20
|
+
|
|
21
|
+
s3 = boto3.client("s3",region_name=aws_credentials['AWS_DEFAULT_REGION'],aws_access_key_id=aws_credentials['AWS_ACCESS_KEY_ID'],aws_secret_access_key=aws_credentials['AWS_SECRET_ACCESS_KEY'])
|
|
22
|
+
bucket = aws_credentials[bucket]
|
|
23
|
+
s3.put_object(Bucket=bucket, Body=csv_buffer.getvalue(), Key= key)
|
|
24
|
+
|
|
25
|
+
def download_file_to_aws(bucket,key, aws_credentials):
|
|
26
|
+
|
|
27
|
+
s3c = boto3.client(
|
|
28
|
+
's3',
|
|
29
|
+
region_name = aws_credentials['AWS_DEFAULT_REGION'],
|
|
30
|
+
aws_access_key_id = aws_credentials['AWS_ACCESS_KEY_ID'],
|
|
31
|
+
aws_secret_access_key = aws_credentials['AWS_SECRET_ACCESS_KEY']
|
|
32
|
+
)
|
|
33
|
+
obj = s3c.get_object(Bucket= bucket , Key = key)
|
|
34
|
+
df = pd.read_csv(BytesIO(obj['Body'].read()), encoding='utf8')
|
|
35
|
+
return df
|
|
@@ -335,8 +335,17 @@ def ranking(data, weighted_features, top = 5, window = 5):
|
|
|
335
335
|
|
|
336
336
|
return top_up, top_low
|
|
337
337
|
|
|
338
|
-
def produce_dashboard(data, columns , ticket_list, show_plot = True, nrows = 150,save_name = False, save_path = False, save_aws = False):
|
|
339
|
-
|
|
338
|
+
def produce_dashboard(data, columns , ticket_list, show_plot = True, nrows = 150,save_name = False, save_path = False, save_aws = False, aws_credential = False):
|
|
339
|
+
"""
|
|
340
|
+
data: pandas df
|
|
341
|
+
columns: list
|
|
342
|
+
ticket_list: list asset list
|
|
343
|
+
nrows: int
|
|
344
|
+
show_plot: bool
|
|
345
|
+
save_path: str local path for saving e.g r'C:/path/to/the/file/'
|
|
346
|
+
save_aws: str remote key in s3 bucket path e.g. 'path/to/file/'
|
|
347
|
+
aws_credentials: dict
|
|
348
|
+
"""
|
|
340
349
|
top = len(ticket_list)
|
|
341
350
|
columns = ['history'] + columns
|
|
342
351
|
subtitles = list()
|
|
@@ -381,8 +390,10 @@ def produce_dashboard(data, columns , ticket_list, show_plot = True, nrows = 150
|
|
|
381
390
|
fig.write_json(save_path+save_name+'.json')
|
|
382
391
|
|
|
383
392
|
if save_name and save_path and save_aws:
|
|
384
|
-
upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = f'multi_dashboards/'+save_name+'.json',input_path = save_path+save_name+'.json')
|
|
385
|
-
|
|
393
|
+
# upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = f'multi_dashboards/'+save_name+'.json',input_path = save_path+save_name+'.json')
|
|
394
|
+
upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = save_aws + save_name + '.json', input_path = save_path + save_name + '.json', aws_credentials = aws_credential)
|
|
395
|
+
|
|
396
|
+
|
|
386
397
|
def rank_by_return(data, lag_days, top_n = 5):
|
|
387
398
|
|
|
388
399
|
data = data.sort_values(['Ticket','Date'], ascending=[False,False]).reset_index(drop = True)
|
|
@@ -573,15 +584,26 @@ def call_ml_objects(stock_code, client, call_models = False):
|
|
|
573
584
|
return objects
|
|
574
585
|
|
|
575
586
|
class produce_plotly_plots:
|
|
576
|
-
def __init__(self,ticket_name, data_frame,settings, save_path = False, save_aws = False, show_plot= True):
|
|
587
|
+
def __init__(self,ticket_name, data_frame,settings, save_path = False, save_aws = False, show_plot= True, aws_credentials = False):
|
|
588
|
+
"""
|
|
589
|
+
ticket_name: str asset name
|
|
590
|
+
data_frame: pandas df
|
|
591
|
+
settings: dict
|
|
592
|
+
show_plot: bool
|
|
593
|
+
save_path: str local path for saving e.g r'C:/path/to/the/file/'
|
|
594
|
+
save_aws: str remote key in s3 bucket path e.g. 'path/to/file/'
|
|
595
|
+
aws_credentials: dict
|
|
596
|
+
"""
|
|
597
|
+
|
|
577
598
|
self.ticket_name = ticket_name
|
|
578
599
|
self.data_frame = data_frame
|
|
579
600
|
self.settings = settings
|
|
580
601
|
self.save_path = save_path
|
|
581
602
|
self.save_aws = save_aws
|
|
582
603
|
self.show_plot = show_plot
|
|
583
|
-
|
|
584
|
-
|
|
604
|
+
self.aws_credentials = aws_credentials
|
|
605
|
+
|
|
606
|
+
def plot_asset_signals(self, feature_list,spread_column, date_intervals = False):
|
|
585
607
|
|
|
586
608
|
result_json_name = 'panel_signals.json'
|
|
587
609
|
df = self.data_frame
|
|
@@ -650,7 +672,8 @@ class produce_plotly_plots:
|
|
|
650
672
|
if self.show_plot:
|
|
651
673
|
fig.show()
|
|
652
674
|
if self.save_path and self.save_aws:
|
|
653
|
-
upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = f'market_plots/{self.ticket_name}/'+result_json_name ,input_path = self.save_path+result_json_name)
|
|
675
|
+
# upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = f'market_plots/{self.ticket_name}/'+result_json_name ,input_path = self.save_path+result_json_name)
|
|
676
|
+
upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = self.save_aws + result_json_name, input_path = self.save_path + result_json_name, aws_credentials = self.aws_credentials)
|
|
654
677
|
|
|
655
678
|
def explore_states_ts(self):
|
|
656
679
|
result_json_name = 'ts_hmm.json'
|
|
@@ -695,7 +718,8 @@ class produce_plotly_plots:
|
|
|
695
718
|
if self.show_plot:
|
|
696
719
|
fig.show()
|
|
697
720
|
if self.save_path and self.save_aws:
|
|
698
|
-
upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = f'market_plots/{self.ticket_name}/'+result_json_name ,input_path = self.save_path+result_json_name)
|
|
721
|
+
# upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = f'market_plots/{self.ticket_name}/'+result_json_name ,input_path = self.save_path+result_json_name)
|
|
722
|
+
upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = self.save_aws + result_json_name, input_path = self.save_path + result_json_name, aws_credentials = self.aws_credentials)
|
|
699
723
|
|
|
700
724
|
def plot_hmm_analysis(self,settings, hmm_model, date_intervals = False, model = False):
|
|
701
725
|
result_json_name = 'hmm_analysis.json'
|
|
@@ -810,9 +834,12 @@ class produce_plotly_plots:
|
|
|
810
834
|
json.dump(messages, outfile)
|
|
811
835
|
|
|
812
836
|
if self.save_path and self.save_aws:
|
|
813
|
-
upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = f'market_plots/{self.ticket_name}/'+result_json_name ,input_path = self.save_path+result_json_name)
|
|
814
|
-
upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = f'market_plots/{self.ticket_name}/'+'market_message.json',input_path = self.save_path+"market_message.json")
|
|
837
|
+
# upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = f'market_plots/{self.ticket_name}/'+result_json_name ,input_path = self.save_path+result_json_name)
|
|
838
|
+
# upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = f'market_plots/{self.ticket_name}/'+'market_message.json',input_path = self.save_path+"market_message.json")
|
|
815
839
|
|
|
840
|
+
upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = self.save_aws + result_json_name, input_path = self.save_path + result_json_name, aws_credentials = self.aws_credentials)
|
|
841
|
+
upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = self.save_aws + 'market_message.json', input_path = self.save_path + 'market_message.json', aws_credentials = self.aws_credentials)
|
|
842
|
+
|
|
816
843
|
def produce_forecasting_plot(self,predictions):
|
|
817
844
|
result_json_name = 'forecast_plot.json'
|
|
818
845
|
hmm_n_clust = self.settings['settings']['hmm']['n_clusters']
|
|
@@ -882,7 +909,8 @@ class produce_plotly_plots:
|
|
|
882
909
|
if self.show_plot:
|
|
883
910
|
fig.show()
|
|
884
911
|
if self.save_path and self.save_aws:
|
|
885
|
-
upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = f'market_plots/{self.ticket_name}/'+result_json_name ,input_path = self.save_path+result_json_name)
|
|
912
|
+
# upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = f'market_plots/{self.ticket_name}/'+result_json_name ,input_path = self.save_path+result_json_name)
|
|
913
|
+
upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = self.save_aws + result_json_name, input_path = self.save_path + result_json_name, aws_credentials = self.aws_credentials)
|
|
886
914
|
|
|
887
915
|
def plot_hmm_analysis_logger(data_frame,test_data_size, save_path = False, show_plot = True):
|
|
888
916
|
|
|
@@ -923,4 +951,137 @@ def plot_hmm_tsanalysis_logger(data_frame, test_data_size,save_path = False, sho
|
|
|
923
951
|
if save_path:
|
|
924
952
|
plt.savefig(save_path)
|
|
925
953
|
if not show_plot:
|
|
926
|
-
plt.close()
|
|
954
|
+
plt.close()
|
|
955
|
+
|
|
956
|
+
def extract_data_traintest(object_stock,features_to_search,configs, target_configs, window_analysis = False, drop_nan= True):
|
|
957
|
+
|
|
958
|
+
object_stock.get_data()
|
|
959
|
+
object_stock.volatility_analysis(**configs['volatility']['config_params'], plot = False, save_features = False)
|
|
960
|
+
target_params_up = target_configs['params_up']
|
|
961
|
+
target_params_down = target_configs['params_down']
|
|
962
|
+
|
|
963
|
+
for feature_name in features_to_search:
|
|
964
|
+
initial_columns = object_stock.df.columns
|
|
965
|
+
arguments_to_use = configs[feature_name]['config_params']
|
|
966
|
+
method_to_use = configs[feature_name]['method']
|
|
967
|
+
getattr(object_stock, method_to_use)(**arguments_to_use, plot = False, save_features = False)
|
|
968
|
+
object_stock.produce_order_features(feature_name)
|
|
969
|
+
# geting targets
|
|
970
|
+
object_stock.get_categorical_targets(**target_params_up)
|
|
971
|
+
object_stock.df = object_stock.df.drop(columns = ['target_down']).rename(columns = {'target_up':'target_up_save'})
|
|
972
|
+
object_stock.get_categorical_targets(**target_params_down)
|
|
973
|
+
object_stock.df = object_stock.df.drop(columns = ['target_up']).rename(columns = {'target_up_save':'target_up'})
|
|
974
|
+
|
|
975
|
+
if drop_nan:
|
|
976
|
+
object_stock.df = object_stock.df.dropna()
|
|
977
|
+
if window_analysis:
|
|
978
|
+
object_stock.df = object_stock.df.iloc[-window_analysis:,:]
|
|
979
|
+
|
|
980
|
+
return object_stock
|
|
981
|
+
|
|
982
|
+
def produce_simple_ts_from_model(stock_code, configs, n_days = 2000 , window_scope = '5y'):
|
|
983
|
+
|
|
984
|
+
## getting data
|
|
985
|
+
volat_args = {'lags': 3, 'trad_days': 15, 'window_log_return': 10}
|
|
986
|
+
|
|
987
|
+
object_stock = stock_eda_panel(stock_code , n_days, window_scope)
|
|
988
|
+
object_stock.get_data()
|
|
989
|
+
object_stock.volatility_analysis(**volat_args, plot = False, save_features = False)
|
|
990
|
+
features = list(configs.keys())
|
|
991
|
+
for feature_name in features:
|
|
992
|
+
arguments_to_use = configs[feature_name]['config_params']
|
|
993
|
+
method_to_use = configs[feature_name]['method']
|
|
994
|
+
getattr(object_stock, method_to_use)(**arguments_to_use, plot = False, save_features = False)
|
|
995
|
+
|
|
996
|
+
## ploting
|
|
997
|
+
df = object_stock.df
|
|
998
|
+
feature_rows = len(features)
|
|
999
|
+
rows_subplot = feature_rows + 1
|
|
1000
|
+
height_plot = rows_subplot * 400
|
|
1001
|
+
|
|
1002
|
+
fig = make_subplots(
|
|
1003
|
+
rows= rows_subplot, cols=1,
|
|
1004
|
+
vertical_spacing = 0.02, horizontal_spacing = 0.02, shared_xaxes=True,
|
|
1005
|
+
subplot_titles = [f'{stock_code} price history'] + features
|
|
1006
|
+
)
|
|
1007
|
+
|
|
1008
|
+
## initial plot:
|
|
1009
|
+
row_i = 1
|
|
1010
|
+
fig.add_trace(go.Scatter(x=df['Date'], y=df['Close'],showlegend= False, mode='lines', marker_color = 'blue'),col = 1, row = row_i)
|
|
1011
|
+
### signal plots
|
|
1012
|
+
for row_i, feature in enumerate(features,start=row_i+1):
|
|
1013
|
+
feature_2 = 'nan'
|
|
1014
|
+
signal_up_list = [f'signal_up_{feature}', f'signal_up_{feature_2}']
|
|
1015
|
+
signal_low_list = [f'signal_low_{feature}', f'signal_low_{feature_2}']
|
|
1016
|
+
norm_list = [f'norm_{feature}', f'z_{feature}', feature]
|
|
1017
|
+
# signal
|
|
1018
|
+
for norm_feat in norm_list:
|
|
1019
|
+
if norm_feat in df.columns:
|
|
1020
|
+
fig.add_trace(go.Scatter(x=df['Date'], y=df[norm_feat],showlegend= False, mode='lines', marker_color = 'grey'),col = 1, row = row_i)
|
|
1021
|
+
break
|
|
1022
|
+
for norm_feat in norm_list:
|
|
1023
|
+
if norm_feat in df.columns:
|
|
1024
|
+
fig.add_trace(go.Scatter(x=df['Date'], y=np.where(df[norm_feat] > 0, df[norm_feat], np.nan),showlegend= False, mode='markers', marker_color = 'green',opacity = 0.3),col = 1, row = row_i)
|
|
1025
|
+
fig.add_trace(go.Scatter(x=df['Date'], y=np.where(df[norm_feat] <= 0, df[norm_feat], np.nan),showlegend= False, mode='markers', marker_color = 'red',opacity = 0.3),col = 1, row = row_i)
|
|
1026
|
+
break
|
|
1027
|
+
for signal_up in signal_up_list:
|
|
1028
|
+
if signal_up in df.columns:
|
|
1029
|
+
fig.add_trace(go.Scatter(x=df['Date'], y=np.where(df[signal_up] == 1, df[norm_feat], np.nan),showlegend= False, mode='markers', marker_color = 'green'),col = 1, row = row_i)
|
|
1030
|
+
|
|
1031
|
+
for signal_low in signal_low_list:
|
|
1032
|
+
if signal_low in df.columns:
|
|
1033
|
+
fig.add_trace(go.Scatter(x=df['Date'], y=np.where(df[signal_low] == 1, df[norm_feat], np.nan),showlegend= False, mode='markers', marker_color = 'red'),col = 1, row = row_i)
|
|
1034
|
+
|
|
1035
|
+
fig.update_layout(height=height_plot, width=1600, title_text = f'asset plot and signals: {stock_code}')
|
|
1036
|
+
|
|
1037
|
+
del object_stock, df
|
|
1038
|
+
|
|
1039
|
+
return fig
|
|
1040
|
+
|
|
1041
|
+
def save_edge_model(data, save_path = False, save_aws = False, show_result = False, aws_credentials = False):
|
|
1042
|
+
"""
|
|
1043
|
+
data: pandas df
|
|
1044
|
+
model_name: str
|
|
1045
|
+
ticket_name: str name of the asset
|
|
1046
|
+
save_path: str local path for saving e.g r'C:/path/to/the/file/'
|
|
1047
|
+
save_aws: str remote key in s3 bucket path e.g. 'path/to/file/'
|
|
1048
|
+
show_results: bool
|
|
1049
|
+
aws_credentials: dict
|
|
1050
|
+
|
|
1051
|
+
return a print of the dictionary
|
|
1052
|
+
"""
|
|
1053
|
+
today = datetime.datetime.today().strftime('%Y-%m-%d')
|
|
1054
|
+
|
|
1055
|
+
curent_edge = data[['Date','proba_target_down','proba_target_up']].iloc[-1,:]
|
|
1056
|
+
curent_edge['Date'] = curent_edge['Date'].strftime('%Y-%m-%d')
|
|
1057
|
+
curent_edge = curent_edge.to_dict()
|
|
1058
|
+
curent_edge['ExecutionDate'] = today
|
|
1059
|
+
|
|
1060
|
+
if save_path:
|
|
1061
|
+
result_json_name = 'current_edge.json'
|
|
1062
|
+
with open(save_path+result_json_name, "w") as outfile:
|
|
1063
|
+
json.dump(curent_edge, outfile)
|
|
1064
|
+
|
|
1065
|
+
if save_path and save_aws:
|
|
1066
|
+
upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = save_aws + result_json_name, input_path = save_path+result_json_name, aws_credentials = aws_credentials)
|
|
1067
|
+
|
|
1068
|
+
if show_result:
|
|
1069
|
+
print(curent_edge)
|
|
1070
|
+
|
|
1071
|
+
def create_feature_edge(model, data,feature_name, threshold, target_variables):
|
|
1072
|
+
|
|
1073
|
+
label_prediction = ['proba_'+x for x in target_variables]
|
|
1074
|
+
predictions = model.predict_proba(data)
|
|
1075
|
+
predictions = pd.DataFrame(predictions, columns = label_prediction, index = data.index)
|
|
1076
|
+
|
|
1077
|
+
result_df = pd.concat([data, predictions], axis=1)
|
|
1078
|
+
|
|
1079
|
+
for pred_col in label_prediction:
|
|
1080
|
+
type_use = 'low'
|
|
1081
|
+
if 'down' in pred_col:
|
|
1082
|
+
type_use = 'up'
|
|
1083
|
+
|
|
1084
|
+
result_df[f'signal_{type_use}_{feature_name}'] = np.where(result_df[pred_col] >= threshold,1,0)
|
|
1085
|
+
result_df[f'acc_{type_use}_{feature_name}'] = np.where(result_df[f'signal_{type_use}_{feature_name}'] == result_df[pred_col.replace('proba_','')],1,0)
|
|
1086
|
+
|
|
1087
|
+
return result_df
|
{virgo_modules-0.0.62 → virgo_modules-0.0.63}/virgo_app/virgo_modules/src/ticketer_source.py
RENAMED
|
@@ -1687,20 +1687,22 @@ class hmm_feature_selector():
|
|
|
1687
1687
|
|
|
1688
1688
|
class signal_analyser_object:
|
|
1689
1689
|
|
|
1690
|
-
def __init__(self, data,symbol_name, show_plot = True, save_path = False, save_aws = False):
|
|
1690
|
+
def __init__(self, data,symbol_name, show_plot = True, save_path = False, save_aws = False, aws_credentials = False):
|
|
1691
1691
|
"""
|
|
1692
1692
|
data: pandas df
|
|
1693
1693
|
symbol_name: str name of the asset
|
|
1694
1694
|
show_plot: bool
|
|
1695
1695
|
save_path: str local path for saving e.g r'C:/path/to/the/file/'
|
|
1696
1696
|
save_aws: str remote key in s3 bucket path e.g. 'path/to/file/'
|
|
1697
|
+
aws_credentials: dict
|
|
1697
1698
|
"""
|
|
1698
1699
|
self.data = data.copy()
|
|
1699
1700
|
self.ticket_name = symbol_name
|
|
1700
1701
|
self.show_plot = show_plot
|
|
1701
1702
|
self.save_path = save_path
|
|
1702
1703
|
self.save_aws = save_aws
|
|
1703
|
-
|
|
1704
|
+
self.aws_credentials = aws_credentials
|
|
1705
|
+
|
|
1704
1706
|
def signal_analyser(self, test_size, feature_name, days_list, threshold = 0.05,verbose = False, signal_position = False):
|
|
1705
1707
|
data = self.data
|
|
1706
1708
|
self.feature_name = feature_name
|
|
@@ -1813,7 +1815,7 @@ class signal_analyser_object:
|
|
|
1813
1815
|
|
|
1814
1816
|
if self.save_path and self.save_aws:
|
|
1815
1817
|
# upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = f'market_plots/{self.ticket_name}/'+result_plot_name, input_path = self.save_path+result_plot_name)
|
|
1816
|
-
upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = self.save_aws + result_plot_name, input_path = self.save_path + result_plot_name)
|
|
1818
|
+
upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = self.save_aws + result_plot_name, input_path = self.save_path + result_plot_name, aws_credentials = self.aws_credentials)
|
|
1817
1819
|
if not self.show_plot:
|
|
1818
1820
|
plt.close()
|
|
1819
1821
|
del df
|
|
@@ -1909,8 +1911,8 @@ class signal_analyser_object:
|
|
|
1909
1911
|
# upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = f'market_plots/{self.ticket_name}/'+result_json_name ,input_path = self.save_path+result_json_name)
|
|
1910
1912
|
# upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = f'market_plots/{self.ticket_name}/'+result_plot_name,input_path = self.save_path+result_plot_name)
|
|
1911
1913
|
|
|
1912
|
-
upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = self.save_aws + result_json_name, input_path = self.save_path + result_json_name)
|
|
1913
|
-
upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = self.save_aws + result_plot_name, input_path = self.save_path + result_plot_name)
|
|
1914
|
+
upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = self.save_aws + result_json_name, input_path = self.save_path + result_json_name, aws_credentials = self.aws_credentials)
|
|
1915
|
+
upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = self.save_aws + result_plot_name, input_path = self.save_path + result_plot_name, aws_credentials = self.aws_credentials)
|
|
1914
1916
|
|
|
1915
1917
|
if not self.show_plot:
|
|
1916
1918
|
plt.close()
|
|
@@ -1948,7 +1950,7 @@ def iterate_signal_analyser(test_data_size,feature_name, days_list, arguments_to
|
|
|
1948
1950
|
return best_result
|
|
1949
1951
|
|
|
1950
1952
|
class analyse_index(stock_eda_panel):
|
|
1951
|
-
def __init__(self, index, asset, n_obs, lag, data_window = '5y', show_plot = True, save_path = False, save_aws = False):
|
|
1953
|
+
def __init__(self, index, asset, n_obs, lag, data_window = '5y', show_plot = True, save_path = False, save_aws = False, aws_credentials = False):
|
|
1952
1954
|
|
|
1953
1955
|
"""
|
|
1954
1956
|
data: pandas df
|
|
@@ -1960,6 +1962,7 @@ class analyse_index(stock_eda_panel):
|
|
|
1960
1962
|
show_plot: bool
|
|
1961
1963
|
save_path: str local path for saving e.g r'C:/path/to/the/file/'
|
|
1962
1964
|
save_aws: str remote key in s3 bucket path e.g. 'path/to/file/'
|
|
1965
|
+
aws_credentials: dict
|
|
1963
1966
|
"""
|
|
1964
1967
|
|
|
1965
1968
|
self.index = index
|
|
@@ -2043,7 +2046,7 @@ class analyse_index(stock_eda_panel):
|
|
|
2043
2046
|
|
|
2044
2047
|
if self.save_path and self.save_aws:
|
|
2045
2048
|
# upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = f'market_plots/{self.asset}/'+result_plot_name,input_path = self.save_path+result_plot_name)
|
|
2046
|
-
upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = self.save_aws + result_plot_name, input_path = self.save_path + result_plot_name)
|
|
2049
|
+
upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = self.save_aws + result_plot_name, input_path = self.save_path + result_plot_name, aws_credentials = self.aws_credentials)
|
|
2047
2050
|
if not self.show_plot:
|
|
2048
2051
|
plt.close()
|
|
2049
2052
|
|
|
@@ -2097,13 +2100,14 @@ class evaluate_markets(analyse_index):
|
|
|
2097
2100
|
|
|
2098
2101
|
self.best_result = best_result
|
|
2099
2102
|
|
|
2100
|
-
def get_relevant_beta(data_market, ticket_name, show_plot = True, save_path = False, save_aws = False):
|
|
2103
|
+
def get_relevant_beta(data_market, ticket_name, show_plot = True, save_path = False, save_aws = False, aws_credentials = False):
|
|
2101
2104
|
"""
|
|
2102
2105
|
data_market: pandas df
|
|
2103
2106
|
ticket_name: str name of the asset
|
|
2104
2107
|
show_plot: bool
|
|
2105
2108
|
save_path: str local path for saving e.g r'C:/path/to/the/file/'
|
|
2106
2109
|
save_aws: str remote key in s3 bucket path e.g. 'path/to/file/'
|
|
2110
|
+
aws_credentials: dict
|
|
2107
2111
|
"""
|
|
2108
2112
|
all_betas = data_market[data_market.asset == ticket_name].sort_values('general_r', ascending = False)
|
|
2109
2113
|
all_betas['gen_r2'] = all_betas.general_r ** 2
|
|
@@ -2118,5 +2122,5 @@ def get_relevant_beta(data_market, ticket_name, show_plot = True, save_path = F
|
|
|
2118
2122
|
|
|
2119
2123
|
if save_path and save_aws:
|
|
2120
2124
|
# upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = f'market_plots/{ticket_name}/'+result_plot_name,input_path = save_path+result_plot_name)
|
|
2121
|
-
upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = save_aws + result_plot_name, input_path = save_path + result_plot_name)
|
|
2125
|
+
upload_file_to_aws(bucket = 'VIRGO_BUCKET', key = save_aws + result_plot_name, input_path = save_path + result_plot_name, aws_credentials = aws_credentials)
|
|
2122
2126
|
return selection
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
import yaml
|
|
2
|
-
import boto3
|
|
3
|
-
from pathlib import Path
|
|
4
|
-
from io import StringIO, BytesIO
|
|
5
|
-
import pandas as pd
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
def upload_file_to_aws(bucket,key,input_path, secret_path = 'secrets.yaml'):
|
|
9
|
-
|
|
10
|
-
credentials = yaml.safe_load(Path(secret_path).read_text())
|
|
11
|
-
session = boto3.Session(aws_access_key_id=credentials['AWS_ACCESS_KEY_ID'],aws_secret_access_key=credentials['AWS_SECRET_ACCESS_KEY'])
|
|
12
|
-
bucket = credentials[bucket]
|
|
13
|
-
s3 = session.resource('s3')
|
|
14
|
-
s3.meta.client.upload_file(Filename=input_path , Bucket=bucket, Key=key)
|
|
15
|
-
|
|
16
|
-
def upload_pandas_to_s3(data_frame,bucket,key, secret_path = 'secrets.yaml'):
|
|
17
|
-
|
|
18
|
-
csv_buffer = StringIO()
|
|
19
|
-
data_frame.to_csv(csv_buffer)
|
|
20
|
-
csv_buffer.seek(0)
|
|
21
|
-
|
|
22
|
-
credentials = yaml.safe_load(Path(secret_path).read_text())
|
|
23
|
-
s3 = boto3.client("s3",region_name=credentials['AWS_DEFAULT_REGION'],aws_access_key_id=credentials['AWS_ACCESS_KEY_ID'],aws_secret_access_key=credentials['AWS_SECRET_ACCESS_KEY'])
|
|
24
|
-
bucket = credentials[bucket]
|
|
25
|
-
s3.put_object(Bucket=bucket, Body=csv_buffer.getvalue(), Key= key)
|
|
26
|
-
|
|
27
|
-
def download_file_to_aws(bucket,key, secret_path = 'secrets.yaml'):
|
|
28
|
-
|
|
29
|
-
credentials = yaml.safe_load(Path(secret_path).read_text())
|
|
30
|
-
s3c = boto3.client(
|
|
31
|
-
's3',
|
|
32
|
-
region_name = credentials['AWS_DEFAULT_REGION'],
|
|
33
|
-
aws_access_key_id = credentials['AWS_ACCESS_KEY_ID'],
|
|
34
|
-
aws_secret_access_key = credentials['AWS_SECRET_ACCESS_KEY']
|
|
35
|
-
)
|
|
36
|
-
obj = s3c.get_object(Bucket= bucket , Key = key)
|
|
37
|
-
df = pd.read_csv(BytesIO(obj['Body'].read()), encoding='utf8')
|
|
38
|
-
return df
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{virgo_modules-0.0.62 → virgo_modules-0.0.63}/virgo_app/virgo_modules.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
{virgo_modules-0.0.62 → virgo_modules-0.0.63}/virgo_app/virgo_modules.egg-info/top_level.txt
RENAMED
|
File without changes
|