saddle-ml 2.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. saddle/__init__.py +1 -0
  2. saddle/automl/__init__.py +0 -0
  3. saddle/automl/autogluon.py +52 -0
  4. saddle/comm/Agent.py +2054 -0
  5. saddle/comm/RAG.py +1770 -0
  6. saddle/comm/RealESRGANWrapper.py +105 -0
  7. saddle/comm/__init__.py +1 -0
  8. saddle/comm/automl_utils.py +58 -0
  9. saddle/comm/cache.py +21 -0
  10. saddle/comm/consul.py +507 -0
  11. saddle/comm/db.py +44 -0
  12. saddle/comm/eda_ops.py +24 -0
  13. saddle/comm/express_merge - /321/205/320/231/320/277/321/206/320/254/320/274.py" +347 -0
  14. saddle/comm/express_merge.py +365 -0
  15. saddle/comm/func_monitor.py +23 -0
  16. saddle/comm/genCustomReport.py +849 -0
  17. saddle/comm/gen_limereport.py +2287 -0
  18. saddle/comm/gen_optimize_data_cfg.py +1032 -0
  19. saddle/comm/gen_shapreport.py +2367 -0
  20. saddle/comm/imagechain.py +1596 -0
  21. saddle/comm/imagechainExtentTools.py +59 -0
  22. saddle/comm/kafka_flask.py +71 -0
  23. saddle/comm/kafka_paython_flask.py +23 -0
  24. saddle/comm/logic_dispatch.py +1063 -0
  25. saddle/comm/minio.py +291 -0
  26. saddle/comm/model_eval.py +139 -0
  27. saddle/comm/optimize.py +773 -0
  28. saddle/comm/optimize_case.py +416 -0
  29. saddle/comm/optimize_engine.py +2508 -0
  30. saddle/comm/redis_util.py +27 -0
  31. saddle/comm/sendmail.py +126 -0
  32. saddle/comm/tabledatapipeline.py +617 -0
  33. saddle/comm/tabledataproc.py +1565 -0
  34. saddle/comm/tablepipeline.py +681 -0
  35. saddle/comm/task_dispatch.py +5269 -0
  36. saddle/comm/time.py +215 -0
  37. saddle/comm/update_notify.py +55 -0
  38. saddle/comm/utils.py +402 -0
  39. saddle/ctr/__init__.py +0 -0
  40. saddle/ctr/dcn.py +167 -0
  41. saddle/ctr/fm.py +345 -0
  42. saddle/ctr/xdeepfm.py +221 -0
  43. saddle/cv/__init__.py +0 -0
  44. saddle/cv/deepdetect.py +384 -0
  45. saddle/cv/deepreg.py +2142 -0
  46. saddle/cv/deepsam.py +153 -0
  47. saddle/cv/deepseg.py +106 -0
  48. saddle/data_process/__init__.py +0 -0
  49. saddle/data_process/data_preprocess.py +848 -0
  50. saddle/data_process/feature_derive.py +64 -0
  51. saddle/data_process/feature_importance_explore.py +210 -0
  52. saddle/data_process/feature_selector.py +687 -0
  53. saddle/data_process/imblearn_packet.py +302 -0
  54. saddle/data_process/variable_bin_methods.py +622 -0
  55. saddle/data_process/variable_encode.py +281 -0
  56. saddle/dl/Bert_seq2seq/__init__.py +0 -0
  57. saddle/dl/Bert_seq2seq/load_data.py +85 -0
  58. saddle/dl/Bert_seq2seq/mask_demo.py +45 -0
  59. saddle/dl/Bert_seq2seq/model.py +596 -0
  60. saddle/dl/Bert_seq2seq/predict.py +82 -0
  61. saddle/dl/Bert_seq2seq/tokenizer.py +61 -0
  62. saddle/dl/Bert_seq2seq/train.py +85 -0
  63. saddle/dl/GPT/__init__.py +0 -0
  64. saddle/dl/GPT/generate_summary.py +86 -0
  65. saddle/dl/GPT/load_data.py +111 -0
  66. saddle/dl/GPT/train.py +106 -0
  67. saddle/dl/Seq2seq/__init__.py +0 -0
  68. saddle/dl/Seq2seq/load_data.py +39 -0
  69. saddle/dl/Seq2seq/model.py +98 -0
  70. saddle/dl/Seq2seq/predict.py +40 -0
  71. saddle/dl/Seq2seq/train_eval.py +90 -0
  72. saddle/dl/__init__.py +1 -0
  73. saddle/dl/deepreg.py +2143 -0
  74. saddle/dl/gpt_chat/__init__.py +0 -0
  75. saddle/dl/lstm_gru.py +261 -0
  76. saddle/dl/tensorflow_dcn.py +167 -0
  77. saddle/dl/tensorflow_fm.py +345 -0
  78. saddle/dl/tensorflow_test.py +13 -0
  79. saddle/dl/tensorflow_xdeepfm.py +221 -0
  80. saddle/dl/test_utils.py +32 -0
  81. saddle/dl/tf_utils.py +486 -0
  82. saddle/dl/transfomer/__init__.py +0 -0
  83. saddle/dl/transfomer/transformerData.py +28 -0
  84. saddle/dl/transfomer/transformerTS.py +104 -0
  85. saddle/feature_process/__init__.py +0 -0
  86. saddle/feature_process/data_preprocess.py +497 -0
  87. saddle/feature_process/data_preprocess2.py +351 -0
  88. saddle/feature_process/feature_derive.py +58 -0
  89. saddle/feature_process/feature_importance_explore.py +210 -0
  90. saddle/feature_process/feature_selector.py +687 -0
  91. saddle/feature_process/variable_bin_methods.py +622 -0
  92. saddle/feature_process/variable_encode.py +281 -0
  93. saddle/nlp/Bert_seq2seq/__init__.py +0 -0
  94. saddle/nlp/Bert_seq2seq/load_data.py +85 -0
  95. saddle/nlp/Bert_seq2seq/mask_demo.py +45 -0
  96. saddle/nlp/Bert_seq2seq/model.py +596 -0
  97. saddle/nlp/Bert_seq2seq/predict.py +82 -0
  98. saddle/nlp/Bert_seq2seq/tokenizer.py +61 -0
  99. saddle/nlp/Bert_seq2seq/train.py +85 -0
  100. saddle/nlp/GPT/__init__.py +0 -0
  101. saddle/nlp/GPT/generate_summary.py +86 -0
  102. saddle/nlp/GPT/load_data.py +111 -0
  103. saddle/nlp/GPT/train.py +106 -0
  104. saddle/nlp/__init__.py +0 -0
  105. saddle/nlp/gpt_chat/__init__.py +0 -0
  106. saddle/nlp/nlp_bert4keras.py +198 -0
  107. saddle/nlp/nlp_huggingface.py +179 -0
  108. saddle/nlp/nlp_process.py +478 -0
  109. saddle/nlp/summarizer/__init__.py +0 -0
  110. saddle/nlp/summarizer/bert_parent.py +52 -0
  111. saddle/nlp/summarizer/cluster_features.py +84 -0
  112. saddle/nlp/summarizer/sentence_handler.py +32 -0
  113. saddle/nlp/textRank.py +92 -0
  114. saddle/nlp/tf2_crf.py +284 -0
  115. saddle/nlp/torch_bert_crf.py +14 -0
  116. saddle/nlp/torch_bert_summarizer.py +99 -0
  117. saddle/risk_management/__init__.py +0 -0
  118. saddle/risk_management/credit_score.py +146 -0
  119. saddle/risk_management/test1.py +159 -0
  120. saddle/statistical_model/__init__.py +0 -0
  121. saddle/statistical_model/automl.py +38 -0
  122. saddle/statistical_model/model_train_predict.py +721 -0
  123. saddle/statistical_model/xgboost_train_eval.py +59 -0
  124. saddle/timeseries/__init__.py +0 -0
  125. saddle/timeseries/deep_time_series.py +128 -0
  126. saddle/timeseries/imblearn_packet.py +302 -0
  127. saddle/timeseries/lstnet.py +178 -0
  128. saddle/timeseries/prophet_method_predict.py +88 -0
  129. saddle/timeseries/tcn.py +479 -0
  130. saddle/timeseries/time_series.py +266 -0
  131. saddle/utility/__init__.py +1 -0
  132. saddle/utility/ai_time.py +215 -0
  133. saddle/utility/automl_utils.py +58 -0
  134. saddle/utility/cache.py +21 -0
  135. saddle/utility/consul.py +507 -0
  136. saddle/utility/db.py +38 -0
  137. saddle/utility/eda_ops.py +24 -0
  138. saddle/utility/func_monitor.py +23 -0
  139. saddle/utility/kafka_flask.py +71 -0
  140. saddle/utility/kafka_paython_flask.py +23 -0
  141. saddle/utility/logic_dispatch.py +1034 -0
  142. saddle/utility/minio.py +291 -0
  143. saddle/utility/model_eval.py +139 -0
  144. saddle/utility/redis_util.py +27 -0
  145. saddle/utility/sendmail.py +126 -0
  146. saddle/utility/task_dispatch.py +1477 -0
  147. saddle/utility/update_notify.py +55 -0
  148. saddle/utility/utilities_scott.py +1013 -0
  149. saddle/utility/utils.py +392 -0
  150. saddle_ml-2.0.0.dist-info/METADATA +49 -0
  151. saddle_ml-2.0.0.dist-info/RECORD +153 -0
  152. saddle_ml-2.0.0.dist-info/WHEEL +5 -0
  153. saddle_ml-2.0.0.dist-info/top_level.txt +1 -0
saddle/__init__.py ADDED
@@ -0,0 +1 @@
1
+ from . import ctr,dl,feature_process,nlp,risk_management,statistical_model,timeseries,comm
File without changes
@@ -0,0 +1,52 @@
1
+ #!/usr/bin/env python
2
+ # -*- coding: UTF-8 -*-
3
+ '''
4
+ @Project :saddle
5
+ @File :automl.py
6
+ @IDE :PyCharm
7
+ @Author :patrick
8
+ @Date :2022/6/1 23:00
9
+ '''
10
+
11
+ from autogluon.tabular import TabularPredictor
12
+ import logging
13
+ import pandas as pd
14
+ class AutogluonWrapper:
15
+ def __init__(self,presets="medium_quality_faster_train",model_type="regression"):# good_quality
16
+ self.presets = presets
17
+ self.model_type = model_type
18
+ def fit(self,X,y):
19
+ logging.info('X len [%s],y len[%s] ',len(X),len(y) )
20
+ X_df = pd.DataFrame( X,columns=[ 'X'+str(i) for i in range( 0, len(X[0]) ) ] )
21
+ y_df = pd.DataFrame(y, columns =['y'])
22
+ self.X_names = list(X_df.columns)
23
+ self.y_names = list(y_df.columns)
24
+ train_df = pd.concat([X_df,y_df],axis=1)
25
+ self.predictor = TabularPredictor(self.y_names[0],problem_type=self.model_type).fit( train_df,presets=self.presets )
26
+ def predict(self,X):
27
+ X_df = pd.DataFrame(X)
28
+ X_df.columns = self.X_names
29
+ if self.model_type=="regression":
30
+ return list(self.predictor.predict(X_df))
31
+ return "not implemented"
32
+
33
+ import pandas as pd
34
+ from autogluon.tabular import TabularDataset, TabularPredictor
35
+ def build_train_autogluon_classifier(train_x,train_y,feature_names,presets='best_quality'):
36
+ df_train = pd.concat([pd.DataFrame(train_x, columns=feature_names), pd.DataFrame(train_y, columns=['label'])], axis=1)
37
+ # df_test = pd.concat([pd.DataFrame(test_x, columns=feature_names), pd.DataFrame(test_y, columns=['label'])], axis=1)
38
+ # df_eval = pd.concat([pd.DataFrame(oot_x, columns=feature_names), pd.DataFrame(oot_y, columns=['label'])], axis=1)
39
+ # X_train = pd.concat([df_train, df_test])
40
+ # X_train.reset_index(drop=True, inplace=True)
41
+
42
+ from autogluon.tabular import TabularDataset, TabularPredictor
43
+ predictor = TabularPredictor( label='label', problem_type='binary', eval_metric='roc_auc').fit( df_train, presets=presets )
44
+
45
+ # predictor = TabularPredictor.load("AutogluonModels/ag-20221025_040314/")
46
+ # train_y_pred = predictor.predict_proba( df_train.drop(columns=['label']) );
47
+ # train_y_pred = train_y_pred[1]
48
+ # test_y_pred = predictor.predict_proba(df_test.drop(columns=['label']));
49
+ # test_y_pred = test_y_pred[1]
50
+ # oot_test_y_pre1 = predictor.predict_proba(df_eval.drop(columns=['label']));
51
+ # oot_test_y_pre1 = oot_test_y_pre1[1]
52
+ return predictor