mns-scheduler 1.1.8.4__py3-none-any.whl → 1.4.5.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. mns_scheduler/__init__.py +1 -3
  2. mns_scheduler/auto_da_ban/auto_da_ban_service.py +89 -0
  3. mns_scheduler/company_info/clean/__init__.py +1 -1
  4. mns_scheduler/company_info/clean/company_info_clean_api.py +62 -21
  5. mns_scheduler/company_info/common/company_common_query_service.py +45 -0
  6. mns_scheduler/company_info/constant/__init__.py +1 -1
  7. mns_scheduler/company_info/constant/company_constant_data.py +335 -224
  8. mns_scheduler/company_info/de_list_stock/__init__.py +1 -1
  9. mns_scheduler/company_info/de_list_stock/de_list_stock_service.py +1 -1
  10. mns_scheduler/company_info/em_stock_info/sync_em_stock_info_sync.py +134 -0
  11. mns_scheduler/company_info/sync/company_info_set_service.py +208 -0
  12. mns_scheduler/company_info/sync/sync_company_info_task.py +203 -0
  13. mns_scheduler/company_info/task/company_announce_info_task.py +77 -0
  14. mns_scheduler/company_info/task/company_base_info_task.py +64 -0
  15. mns_scheduler/company_info/task/company_business_info_task.py +177 -0
  16. mns_scheduler/company_info/task/company_hold_info_task.py +66 -0
  17. mns_scheduler/company_info/task/company_industry_info_task.py +167 -0
  18. mns_scheduler/company_info/task/company_total_task.py +69 -0
  19. mns_scheduler/concept/clean/kpl_concept_clean_api.py +1 -1
  20. mns_scheduler/concept/clean/ths_concept_clean_api.py +20 -4
  21. mns_scheduler/concept/ths/common/ths_concept_sync_common_api.py +23 -18
  22. mns_scheduler/concept/ths/common/ths_concept_update_common_api.py +4 -1
  23. mns_scheduler/concept/ths/detaill/ths_concept_detail_api.py +7 -7
  24. mns_scheduler/concept/ths/sync_new_index/sync_ths_concept_new_index_api.py +9 -4
  25. mns_scheduler/concept/ths/update_concept_info/sync_one_symbol_all_concepts_api.py +2 -2
  26. mns_scheduler/db/col_move_service.py +3 -3
  27. mns_scheduler/db/script/__init__.py +1 -1
  28. mns_scheduler/db/script/col_move_script.py +1 -1
  29. mns_scheduler/db/script/db_move/__init__.py +7 -0
  30. mns_scheduler/db/script/db_move/col_move_one_service.py +34 -0
  31. mns_scheduler/db/script/sync/__init__.py +1 -1
  32. mns_scheduler/db/script/sync/remote_data_sync_to_local.py +65 -4
  33. mns_scheduler/db/script/sync/sync_hui_ce_test_data.py +80 -0
  34. mns_scheduler/db/script/sync/sync_hui_ce_test_data_01.py +69 -0
  35. mns_scheduler/db/script/update/__init__.py +7 -0
  36. mns_scheduler/db/script/update/update_col_field.py +36 -0
  37. mns_scheduler/finance/__init__.py +1 -1
  38. mns_scheduler/finance/em/__init__.py +7 -0
  39. mns_scheduler/finance/{em_financial_asset_liability_sync_service_api.py → em/em_financial_asset_liability_sync_service_api.py} +2 -2
  40. mns_scheduler/finance/{em_financial_profit_sync_service_api.py → em/em_financial_profit_sync_service_api.py} +27 -26
  41. mns_scheduler/finance/{finance_common_api.py → em/finance_common_api.py} +3 -3
  42. mns_scheduler/finance/{sync_financial_report_service_api.py → sync_financial_report_service_task.py} +80 -27
  43. mns_scheduler/finance/xue_qiu/__init__.py +7 -0
  44. mns_scheduler/finance/xue_qiu/down_load_xueqiu_report_api.py +77 -0
  45. mns_scheduler/finance/xue_qiu/sync_xue_qiu_fiance_data.py +161 -0
  46. mns_scheduler/hk/__init__.py +1 -1
  47. mns_scheduler/hk/hk_company_info_sync_service_api.py +4 -4
  48. mns_scheduler/hk/hk_industry_info_sync_service_api.py +3 -5
  49. mns_scheduler/industry/__init__.py +7 -0
  50. mns_scheduler/industry/ths/__init__.py +7 -0
  51. mns_scheduler/industry/ths/ths_industry_index_service.py +58 -0
  52. mns_scheduler/industry/ths/ths_industry_sync_service.py +68 -0
  53. mns_scheduler/irm/__init__.py +1 -1
  54. mns_scheduler/irm/api/__init__.py +1 -1
  55. mns_scheduler/irm/api/sh_stock_sns_sse_info_api.py +8 -58
  56. mns_scheduler/irm/api/sz_stock_sns_sse_info_api.py +15 -26
  57. mns_scheduler/irm/stock_irm_cninfo_service.py +43 -31
  58. mns_scheduler/irm/stock_question_id_service.py +169 -0
  59. mns_scheduler/k_line/clean/daily/__init__.py +1 -1
  60. mns_scheduler/k_line/clean/daily/daily_k_line_clean_common_service.py +52 -6
  61. mns_scheduler/k_line/clean/daily/daily_k_line_service.py +7 -2
  62. mns_scheduler/k_line/clean/k_line_info_clean_impl.py +3 -2
  63. mns_scheduler/k_line/clean/k_line_info_clean_task.py +42 -15
  64. mns_scheduler/k_line/clean/week_month/__init__.py +1 -1
  65. mns_scheduler/k_line/clean/week_month/normal_week_month_k_line_service.py +124 -26
  66. mns_scheduler/k_line/clean/week_month/sub_new_week_month_k_line_service.py +2 -2
  67. mns_scheduler/k_line/common/__init__.py +7 -0
  68. mns_scheduler/k_line/common/k_line_common_api.py +188 -0
  69. mns_scheduler/k_line/hot_stocks/__init__.py +1 -1
  70. mns_scheduler/k_line/hot_stocks/recent_hot_stocks_clean_service.py +1 -1
  71. mns_scheduler/k_line/{sync → month_week_daily}/bfq_k_line_sync.py +14 -29
  72. mns_scheduler/k_line/{sync → month_week_daily}/daily_week_month_line_sync.py +11 -12
  73. mns_scheduler/k_line/sync_status/__init__.py +7 -0
  74. mns_scheduler/k_line/sync_status/k_line_sync_status_check.py +54 -0
  75. mns_scheduler/k_line/test/__init__.py +1 -1
  76. mns_scheduler/k_line/test/k_line_info_clean_his_data.py +14 -3
  77. mns_scheduler/k_line/year_quarter/__init__.py +7 -0
  78. mns_scheduler/k_line/year_quarter/year_quarter_line_sync.py +76 -0
  79. mns_scheduler/kpl/selection/symbol/sync_best_choose_symbol.py +1 -2
  80. mns_scheduler/kpl/selection/symbol/sync_kpl_concept_symbol_choose_reason_api.py +108 -0
  81. mns_scheduler/kpl/selection/total/sync_kpl_best_total_sync_api.py +5 -0
  82. mns_scheduler/lhb/__init__.py +1 -1
  83. mns_scheduler/lhb/stock_lhb_sync_service.py +1 -1
  84. mns_scheduler/open/__init__.py +1 -1
  85. mns_scheduler/open/sync_one_day_open_data_to_db_service.py +2 -3
  86. mns_scheduler/risk/__init__.py +1 -1
  87. mns_scheduler/risk/compliance/undisclosed_annual_report_api.py +8 -2
  88. mns_scheduler/risk/financial/annual_report_audit_check_api.py +13 -3
  89. mns_scheduler/risk/financial/net_assets_check_api.py +21 -18
  90. mns_scheduler/risk/financial/profit_income_check_api.py +7 -2
  91. mns_scheduler/risk/financial_report_risk_check_api.py +1 -1
  92. mns_scheduler/risk/major_violations/register_and_investigate_stock_sync_api.py +1 -1
  93. mns_scheduler/risk/self/wei_pan_stock_api.py +1 -1
  94. mns_scheduler/risk/test/__init__.py +1 -1
  95. mns_scheduler/risk/test/fix_blask_list.py +4 -6
  96. mns_scheduler/risk/transactions/transactions_check_api.py +22 -4
  97. mns_scheduler/self_choose/__init__.py +1 -1
  98. mns_scheduler/self_choose/ths_self_choose_service.py +99 -43
  99. mns_scheduler/trade/auto_login/trader_auto_service.py +7 -4
  100. mns_scheduler/trade/auto_sell_service_api.py +4 -4
  101. mns_scheduler/trade/balance/__init__.py +7 -0
  102. mns_scheduler/trade/balance/ths_account_balance_service.py +7 -0
  103. mns_scheduler/trade/sync_position_api.py +39 -6
  104. mns_scheduler/trade/task/trader_task_service.py +38 -10
  105. mns_scheduler/trade/tfp/__init__.py +7 -0
  106. mns_scheduler/trade/tfp/stock_tfp_info_sync.py +56 -0
  107. mns_scheduler/zb/stock_zb_pool_sync.py +1 -16
  108. mns_scheduler/zt/high_chg/sync_high_chg_pool_service.py +2 -2
  109. mns_scheduler/zt/high_chg/sync_high_chg_real_time_quotes_service.py +1 -1
  110. mns_scheduler/zt/script/__init__.py +1 -1
  111. mns_scheduler/zt/script/fix_error_deal_day.py +41 -0
  112. mns_scheduler/zt/script/kcx_high_chg_open_his_data_handle.py +2 -2
  113. mns_scheduler/zt/script/sync_high_chg_pool_his_data.py +2 -2
  114. mns_scheduler/zt/script/sync_now_higt_chg_zt.py +8 -7
  115. mns_scheduler/zt/zt_pool/em_zt_pool_sync_api.py +260 -83
  116. mns_scheduler/zt/zt_pool/ths_zt_pool_sync_api.py +33 -90
  117. mns_scheduler/zt/zt_pool/update_null_zt_reason_api.py +27 -13
  118. mns_scheduler/zz_task/compensation/__init__.py +0 -0
  119. mns_scheduler/zz_task/compensation/compensate_task.py +161 -0
  120. mns_scheduler/zz_task/compensation/compensate_task_one_day.py +142 -0
  121. mns_scheduler/zz_task/data_sync_task.py +197 -103
  122. {mns_scheduler-1.1.8.4.dist-info → mns_scheduler-1.4.5.7.dist-info}/METADATA +1 -1
  123. mns_scheduler-1.4.5.7.dist-info/RECORD +176 -0
  124. {mns_scheduler-1.1.8.4.dist-info → mns_scheduler-1.4.5.7.dist-info}/WHEEL +1 -1
  125. mns_scheduler/2014-2015-test/2014_2015_chg_statistics.py +0 -87
  126. mns_scheduler/big_deal/ths_big_deal_sync.py +0 -98
  127. mns_scheduler/company_info/base/sync_company_base_info_api.py +0 -439
  128. mns_scheduler/company_info/base/sync_company_hold_info_api.py +0 -40
  129. mns_scheduler/company_info/remark/company_remark_info_sync.py +0 -46
  130. mns_scheduler/db/real_time_task_check.py +0 -84
  131. mns_scheduler/debt/kzz_bond_info_sync.py +0 -33
  132. mns_scheduler-1.1.8.4.dist-info/RECORD +0 -142
  133. /mns_scheduler/{2014-2015-test → auto_da_ban}/__init__.py +0 -0
  134. /mns_scheduler/company_info/{base → common}/__init__.py +0 -0
  135. /mns_scheduler/{big_deal → company_info/em_stock_info}/__init__.py +0 -0
  136. /mns_scheduler/company_info/{remark → sync}/__init__.py +0 -0
  137. /mns_scheduler/{debt → company_info/task}/__init__.py +0 -0
  138. /mns_scheduler/k_line/{sync → month_week_daily}/__init__.py +0 -0
  139. {mns_scheduler-1.1.8.4.dist-info → mns_scheduler-1.4.5.7.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,64 @@
1
+ import sys
2
+ import os
3
+
4
+ file_path = os.path.abspath(__file__)
5
+ end = file_path.index('mns') + 17
6
+ project_path = file_path[0:end]
7
+ sys.path.append(project_path)
8
+ import mns_common.api.ths.company.ths_company_info_web as ths_company_info_web
9
+ from mns_common.db.MongodbUtil import MongodbUtil
10
+ import mns_common.utils.data_frame_util as data_frame_util
11
+ import mns_common.constant.db_name_constant as db_name_constant
12
+ from loguru import logger
13
+ from mns_scheduler.company_info.common.company_common_query_service import get_company_info
14
+ import time
15
+ from datetime import datetime
16
+
17
+ mongodb_util = MongodbUtil('27017')
18
+
19
+
20
+ def sync_company_base_info_task(symbol_list):
21
+ all_company_info_df = get_company_info()
22
+ if len(symbol_list) > 0:
23
+ all_company_info_df = all_company_info_df.loc[all_company_info_df['symbol'].isin(symbol_list)]
24
+ fail_list = []
25
+ for stock_one in all_company_info_df.itertuples():
26
+ try:
27
+ sync_one_symbol_base_info(stock_one.symbol)
28
+ time.sleep(0.5)
29
+ except BaseException as e:
30
+ time.sleep(3)
31
+ logger.error("同步公司基础信息发生异常:{},{}", stock_one.symbol, e)
32
+ fail_list.append(stock_one.symbol)
33
+ sync_number = 1
34
+ while len(fail_list) > 0 and sync_number < 10:
35
+ for symbol in fail_list:
36
+ try:
37
+ sync_one_symbol_base_info(symbol)
38
+ time.sleep(5)
39
+ except BaseException as e:
40
+ time.sleep(10)
41
+ logger.error("同步公司基础信息发生异常:{},{}", symbol, e)
42
+ sync_number = sync_number + 1
43
+
44
+
45
+ def sync_one_symbol_base_info(symbol):
46
+ company_remark_info = ths_company_info_web.get_company_info(symbol)
47
+ company_remark_info['_id'] = symbol
48
+ company_remark_info['symbol'] = symbol
49
+ company_remark_info['remark'] = ''
50
+
51
+ now_date = datetime.now()
52
+ sync_str_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
53
+
54
+ company_remark_info['sync_str_date'] = sync_str_date
55
+
56
+ exist_company_remark_df = mongodb_util.find_query_data(db_name_constant.COMPANY_BASE_INFO,
57
+ query={"symbol": symbol})
58
+ if data_frame_util.is_not_empty(exist_company_remark_df):
59
+ company_remark_info['remark'] = list(exist_company_remark_df['remark'])[0]
60
+ mongodb_util.save_mongo(company_remark_info, db_name_constant.COMPANY_BASE_INFO)
61
+
62
+
63
+ if __name__ == '__main__':
64
+ sync_company_base_info_task([])
@@ -0,0 +1,177 @@
1
+ import os
2
+ import sys
3
+
4
+ file_path = os.path.abspath(__file__)
5
+ end = file_path.index('mns') + 17
6
+ project_path = file_path[0:end]
7
+ sys.path.append(project_path)
8
+
9
+ import mns_common.component.common_service_fun_api as common_service_fun_api
10
+ import mns_common.api.ths.company.company_product_area_industry_index_query as company_product_area_industry_index_query
11
+ from loguru import logger
12
+ from mns_common.db.MongodbUtil import MongodbUtil
13
+ import mns_common.constant.db_name_constant as db_name_constant
14
+ import mns_common.utils.data_frame_util as data_frame_util
15
+ import pandas as pd
16
+ from datetime import datetime
17
+ from mns_scheduler.company_info.common.company_common_query_service import get_company_info
18
+
19
+ mongodb_util = MongodbUtil('27017')
20
+
21
+
22
+ def sync_company_business_task(symbol_list):
23
+ now_date = datetime.now()
24
+ now_year = now_date.year
25
+ now_month = now_date.month
26
+
27
+ if now_month in [1, 2, 3, 4]:
28
+ period_time_year = str(now_year - 1) + "-12-31"
29
+ sync_company_product_area_industry(symbol_list, period_time_year)
30
+
31
+ if now_month in [4, 5, 6]:
32
+ period_time_one = str(now_year) + "-03-31"
33
+ sync_company_product_area_industry(symbol_list, period_time_one)
34
+
35
+ elif now_month in [7, 8, 9]:
36
+ period_time_two = str(now_year) + "-06-30"
37
+ sync_company_product_area_industry(symbol_list, period_time_two)
38
+
39
+ elif now_month in [10, 11, 12]:
40
+ period_time_three = str(now_year) + "-09-30"
41
+ sync_company_product_area_industry(symbol_list, period_time_three)
42
+
43
+
44
+ def sync_company_product_area_industry(symbol_list, date):
45
+ all_company_info_df = get_company_info()
46
+
47
+ all_company_info_df = common_service_fun_api.classify_symbol(all_company_info_df)
48
+ if len(symbol_list) > 0:
49
+ all_company_info_df = all_company_info_df.loc[all_company_info_df['symbol'].isin(symbol_list)]
50
+ for stock_one in all_company_info_df.itertuples():
51
+ try:
52
+ symbol = stock_one.symbol
53
+
54
+ classification = stock_one.classification
55
+ if classification in ['H', 'K']:
56
+ market = '17'
57
+ elif classification in ['S', 'C']:
58
+ market = '33'
59
+ elif classification in ['X']:
60
+ market = '151'
61
+
62
+ query_exist = {'symbol': symbol, 'time': date}
63
+ exist_company_business_info_df = mongodb_util.find_query_data(db_name_constant.COMPANY_BUSINESS_INFO,
64
+ query_exist)
65
+ if data_frame_util.is_empty(exist_company_business_info_df):
66
+ exist_all = False
67
+ else:
68
+ exist_all = (
69
+ (exist_company_business_info_df.loc[
70
+ exist_company_business_info_df['analysis_type'] == 'area'].shape[0] > 0)
71
+ and (exist_company_business_info_df.loc[
72
+ exist_company_business_info_df['analysis_type'] == 'industry'].shape[0] > 0)
73
+ and (exist_company_business_info_df.loc[
74
+ exist_company_business_info_df['analysis_type'] == 'product'].shape[
75
+ 0] > 0))
76
+ if exist_all:
77
+ continue
78
+ company_product_area_industry_list = company_product_area_industry_index_query.company_product_area_industry(
79
+ symbol, market, date)
80
+ for company_one in company_product_area_industry_list:
81
+ try:
82
+ analysis_type = company_one['analysis_type']
83
+ time_operate_index_item_list = company_one['time_operate_index_item_list']
84
+ time_operate_index_item_df = pd.DataFrame(time_operate_index_item_list)
85
+ if data_frame_util.is_empty(time_operate_index_item_df):
86
+ continue
87
+ time_operate_index_item_df['symbol'] = symbol
88
+ time_operate_index_item_df['analysis_type'] = analysis_type
89
+
90
+ time_operate_index_item_df['_id'] = symbol + '_' + time_operate_index_item_df[
91
+ 'time'] + '_' + analysis_type
92
+ handle_industry_area_product(time_operate_index_item_df, symbol)
93
+ except BaseException as e:
94
+ logger.error("同步经营数据异常:{},{}", symbol, e)
95
+ except BaseException as e:
96
+ logger.error("同步经营数据异常:{},{}", stock_one.symbol, e)
97
+
98
+
99
+ def handle_industry_area_product(time_operate_index_item_df, symbol):
100
+ if data_frame_util.is_empty(time_operate_index_item_df):
101
+ return None
102
+
103
+ for business_one in time_operate_index_item_df.itertuples():
104
+ time = business_one.time
105
+ analysis_type = business_one.analysis_type
106
+
107
+ product_index_item_list = business_one.product_index_item_list
108
+ for product_one in product_index_item_list:
109
+ try:
110
+ # 初始化数据
111
+ income_amount = 0
112
+ income_percent = 0
113
+ cost_amount = 0
114
+ cost_percent = 0
115
+ gross_profit_amount = 0
116
+ gross_profit_percent = 0
117
+ gross_profit_rate_amount = 0
118
+ gross_profit_rate_percent = 0
119
+
120
+ product_name = product_one['product_name']
121
+ index_analysis_list = product_one['index_analysis_list']
122
+ for index_one in index_analysis_list:
123
+ try:
124
+ index_id = index_one['index_id']
125
+ if index_id == 'income':
126
+ income_amount = index_one['index_value']
127
+ income_percent = index_one['account']
128
+ elif index_id == 'cost':
129
+ cost_amount = index_one['index_value']
130
+ cost_percent = index_one['account']
131
+ elif index_id == 'gross_profit':
132
+ gross_profit_amount = index_one['index_value']
133
+ gross_profit_percent = index_one['account']
134
+
135
+ elif index_id == 'gross_profit_rate':
136
+ gross_profit_rate_amount = index_one['index_value']
137
+ gross_profit_rate_percent = index_one['account']
138
+ except BaseException as e:
139
+ logger.error("同步经营数据异常:{},{}", symbol, e)
140
+
141
+ id_key = symbol + '_' + time + '_' + analysis_type + '_' + product_name
142
+ result_dict = {
143
+ '_id': id_key,
144
+ 'symbol': symbol,
145
+ 'time': time,
146
+ 'analysis_type': analysis_type,
147
+ 'product_name': product_name,
148
+
149
+ 'income_amount': income_amount,
150
+ 'income_percent': income_percent,
151
+
152
+ 'cost_amount': cost_amount,
153
+ 'cost_percent': cost_percent,
154
+
155
+ 'gross_profit_amount': gross_profit_amount,
156
+ 'gross_profit_percent': gross_profit_percent,
157
+
158
+ 'gross_profit_rate_amount': gross_profit_rate_amount,
159
+ 'gross_profit_rate_percent': gross_profit_rate_percent,
160
+ }
161
+ result_dict_df = pd.DataFrame(result_dict, index=[1])
162
+ mongodb_util.save_mongo(result_dict_df, db_name_constant.COMPANY_BUSINESS_INFO)
163
+ except BaseException as e:
164
+ logger.error("同步经营数据异常:{},{}", symbol, e)
165
+
166
+
167
+ if __name__ == '__main__':
168
+ sync_company_product_area_industry('300211', '2024-12-31')
169
+ # sync_company_product_area_industry('002323')
170
+ # sync_company_product_area_industry('300901')
171
+ # sync_company_product_area_industry('603225')
172
+ # sync_company_product_area_industry('688039')
173
+ # sync_company_product_area_industry('600849')
174
+ # sync_company_product_area_industry('000508')
175
+ # sync_company_product_area_industry('810011')
176
+
177
+ sync_company_product_area_industry([], None)
@@ -0,0 +1,66 @@
1
+ import sys
2
+ import os
3
+
4
+ file_path = os.path.abspath(__file__)
5
+ end = file_path.index('mns') + 16
6
+ project_path = file_path[0:end]
7
+ sys.path.append(project_path)
8
+ import mns_common.api.ths.company.ths_company_info_api as ths_company_info_api
9
+ import mns_common.component.cookie.cookie_info_service as cookie_info_service
10
+ import mns_common.utils.data_frame_util as data_frame_util
11
+ from mns_common.db.MongodbUtil import MongodbUtil
12
+ import mns_common.constant.db_name_constant as db_name_constant
13
+ from datetime import datetime
14
+ from loguru import logger
15
+ from mns_scheduler.company_info.common.company_common_query_service import get_company_info
16
+ import time
17
+
18
+ mongodb_util = MongodbUtil('27017')
19
+
20
+
21
+ # 同步公司控股子公司信息任务
22
+ def sync_one_company_hold_info(symbol):
23
+ try:
24
+ ths_cookie = cookie_info_service.get_ths_cookie()
25
+ company_hold_info_df = ths_company_info_api.get_company_hold_info(symbol, ths_cookie)
26
+
27
+ now_date = datetime.now()
28
+ sync_str_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
29
+
30
+ if data_frame_util.is_not_empty(company_hold_info_df):
31
+ query_exist = {'symbol': symbol}
32
+ exist_company_holding_info_df = mongodb_util.find_query_data(db_name_constant.COMPANY_HOLDING_INFO,
33
+ query_exist)
34
+ if data_frame_util.is_not_empty(exist_company_holding_info_df):
35
+ # 作废不在参股控股的子公司
36
+ invalid_symbol_df = exist_company_holding_info_df.loc[
37
+ ~exist_company_holding_info_df['symbol'].isin(list(company_hold_info_df['symbol']))]
38
+ invalid_symbol_df['valid'] = False
39
+
40
+ company_hold_info_df['valid'] = True
41
+ company_hold_info_df['_id'] = company_hold_info_df['symbol'] + '_' + company_hold_info_df[
42
+ 'holding_company']
43
+ company_hold_info_df['sync_str_date'] = sync_str_date
44
+ mongodb_util.save_mongo(company_hold_info_df, db_name_constant.COMPANY_HOLDING_INFO)
45
+ else:
46
+ logger.warning("同步控股子公司为空:{}", symbol)
47
+
48
+ except BaseException as e:
49
+ logger.error("同步公司控股子公司信息:{},{}", symbol, e)
50
+
51
+
52
+ def sync_all_company_hold_info_task(symbol_list):
53
+ all_company_info_df = get_company_info()
54
+ if len(symbol_list) > 0:
55
+ all_company_info_df = all_company_info_df.loc[all_company_info_df['symbol'].isin(symbol_list)]
56
+ for stock_one in all_company_info_df.itertuples():
57
+ try:
58
+ sync_one_company_hold_info(stock_one.symbol)
59
+ time.sleep(1)
60
+ except BaseException as e:
61
+ logger.error("同步控股子公司信息异常:{},{}", stock_one.symbol, e)
62
+ time.sleep(5)
63
+
64
+
65
+ if __name__ == '__main__':
66
+ sync_all_company_hold_info_task([])
@@ -0,0 +1,167 @@
1
+ import sys
2
+ import os
3
+
4
+ file_path = os.path.abspath(__file__)
5
+ end = file_path.index('mns') + 16
6
+ project_path = file_path[0:end]
7
+ sys.path.append(project_path)
8
+ import mns_common.api.ths.company.ths_company_info_web as ths_company_info_web
9
+ from mns_common.db.MongodbUtil import MongodbUtil
10
+ import mns_common.constant.db_name_constant as db_name_constant
11
+ from loguru import logger
12
+ from mns_scheduler.company_info.common.company_common_query_service import get_company_info
13
+ import time
14
+ import mns_common.utils.data_frame_util as data_frame_util
15
+ import mns_common.component.common_service_fun_api as common_service_fun_api
16
+
17
+ mongodb_util = MongodbUtil('27017')
18
+
19
+
20
+ def sync_company_industry_info_task(symbol_list):
21
+ all_company_info_df = get_company_info()
22
+ if len(symbol_list) > 0:
23
+ all_company_info_df = all_company_info_df.loc[all_company_info_df['symbol'].isin(symbol_list)]
24
+
25
+ all_company_info_df = common_service_fun_api.classify_symbol(all_company_info_df)
26
+ fail_list = []
27
+ for stock_one in all_company_info_df.itertuples():
28
+ try:
29
+ tag = sync_one_company_industry_info(stock_one.symbol, stock_one.classification)
30
+ if bool(1 - tag):
31
+ fail_list.append(stock_one.symbol)
32
+ time.sleep(0.5)
33
+ except BaseException as e:
34
+ time.sleep(1)
35
+ logger.error("同步公司行业信息发生异常:{},{}", stock_one.symbol, e)
36
+ fail_list.append(stock_one.symbol)
37
+ sync_number = 1
38
+ while len(fail_list) > 0 and sync_number < 10:
39
+ for symbol in fail_list:
40
+ try:
41
+ company_info_one_df = all_company_info_df.loc[all_company_info_df['symbol'] == symbol]
42
+ classification = list(company_info_one_df['classification'])[0]
43
+ tag = sync_one_company_industry_info(symbol, classification)
44
+ if tag and symbol in fail_list:
45
+ fail_list.remove(symbol)
46
+ time.sleep(2)
47
+ except BaseException as e:
48
+ time.sleep(3)
49
+ logger.error("同步公司行业信息发生异常:{},{}", symbol, e)
50
+ sync_number = sync_number + 1
51
+
52
+
53
+ def sync_one_company_industry_info(symbol, classification):
54
+ if classification in ['H', 'K']:
55
+ market_id = '17'
56
+ elif classification in ['S', 'C']:
57
+ market_id = '33'
58
+ elif classification in ['X']:
59
+ market_id = '151'
60
+
61
+ company_industry_info = ths_company_info_web.get_company_info_detail(symbol, market_id)
62
+ if data_frame_util.is_empty(company_industry_info):
63
+ return False
64
+
65
+ company_industry_info['first_industry_code'] = company_industry_info['hycode'].apply(
66
+ lambda x: x[1:3] + '0000')
67
+ company_industry_info['second_industry_code'] = company_industry_info['hy2code'].apply(
68
+ lambda x: x[1:5] + '00')
69
+ company_industry_info['third_industry_code'] = company_industry_info['hy3code'].apply(
70
+ lambda x: x[1:7])
71
+
72
+ company_industry_info['first_sw_industry'] = company_industry_info['hy']
73
+ company_industry_info['second_sw_industry'] = company_industry_info['hy2']
74
+ company_industry_info['third_sw_industry'] = company_industry_info['hy3']
75
+ del company_industry_info['hy']
76
+ del company_industry_info['hy2']
77
+ del company_industry_info['hy3']
78
+ del company_industry_info['hycode']
79
+ del company_industry_info['hy2code']
80
+ del company_industry_info['hy3code']
81
+
82
+ company_industry_info['_id'] = symbol
83
+ company_industry_info['symbol'] = symbol
84
+ mongodb_util.save_mongo(company_industry_info, db_name_constant.COMPANY_INDUSTRY_INFO)
85
+ # 保存股票申万行业原始信息
86
+ save_sw_data(company_industry_info)
87
+ return True
88
+
89
+
90
+ # 保存申万行业分类
91
+ def save_sw_data(company_industry_info):
92
+ first_sw_info = company_industry_info[[
93
+ 'first_sw_industry',
94
+ 'first_industry_code'
95
+ ]].copy()
96
+
97
+ first_sw_info.loc[:, "industry_code"] = first_sw_info['first_industry_code']
98
+ first_sw_info.loc[:, "_id"] = first_sw_info['first_industry_code']
99
+ first_sw_info.loc[:, "second_sw_industry"] = 0
100
+ first_sw_info.loc[:, "third_sw_industry"] = 0
101
+ first_sw_info.loc[:, "second_industry_code"] = 0
102
+ first_sw_info.loc[:, "third_industry_code"] = 0
103
+ first_sw_info = first_sw_info[[
104
+ "_id",
105
+ "industry_code",
106
+ 'first_industry_code',
107
+ 'first_sw_industry',
108
+ 'second_industry_code',
109
+ 'second_sw_industry',
110
+ 'third_industry_code',
111
+ 'third_sw_industry'
112
+ ]]
113
+ mongodb_util.save_mongo(first_sw_info, 'sw_industry')
114
+
115
+ second_sw_info = company_industry_info[[
116
+ 'first_industry_code',
117
+ 'first_sw_industry',
118
+ 'second_sw_industry',
119
+ 'second_industry_code',
120
+ ]].copy()
121
+
122
+ second_sw_info.loc[:, "industry_code"] = second_sw_info['second_industry_code']
123
+ second_sw_info.loc[:, "_id"] = second_sw_info['industry_code']
124
+
125
+ second_sw_info.loc[:, "third_sw_industry"] = 0
126
+ second_sw_info.loc[:, "third_sw_industry"] = 0
127
+ second_sw_info.loc[:, "third_industry_code"] = 0
128
+ second_sw_info = second_sw_info[[
129
+ "_id",
130
+ "industry_code",
131
+ 'first_industry_code',
132
+ 'first_sw_industry',
133
+ 'second_industry_code',
134
+ 'second_sw_industry',
135
+ 'third_industry_code',
136
+ 'third_sw_industry'
137
+ ]]
138
+ mongodb_util.save_mongo(second_sw_info, 'sw_industry')
139
+
140
+ third_sw_info = company_industry_info[[
141
+ 'first_industry_code',
142
+ 'first_sw_industry',
143
+ 'second_industry_code',
144
+ 'second_sw_industry',
145
+ 'third_industry_code',
146
+ 'third_sw_industry'
147
+ ]].copy()
148
+
149
+ third_sw_info.loc[:, "industry_code"] = third_sw_info['third_industry_code']
150
+
151
+ third_sw_info.loc[:, "_id"] = third_sw_info['industry_code']
152
+
153
+ third_sw_info = third_sw_info[[
154
+ "_id",
155
+ "industry_code",
156
+ 'first_industry_code',
157
+ 'first_sw_industry',
158
+ 'second_industry_code',
159
+ 'second_sw_industry',
160
+ 'third_industry_code',
161
+ 'third_sw_industry'
162
+ ]]
163
+ mongodb_util.save_mongo(third_sw_info, 'sw_industry')
164
+
165
+
166
+ if __name__ == '__main__':
167
+ sync_company_industry_info_task(['688795'])
@@ -0,0 +1,69 @@
1
+ import sys
2
+ import os
3
+
4
+ file_path = os.path.abspath(__file__)
5
+ end = file_path.index('mns') + 16
6
+ project_path = file_path[0:end]
7
+ sys.path.append(project_path)
8
+ import mns_scheduler.company_info.task.company_industry_info_task as company_industry_info_task
9
+ import mns_scheduler.company_info.task.company_business_info_task as company_business_info_task
10
+ import mns_scheduler.company_info.task.company_base_info_task as company_base_info_task
11
+ import mns_scheduler.company_info.task.company_hold_info_task as company_hold_info_task
12
+ import mns_scheduler.company_info.task.company_announce_info_task as company_announce_info_task
13
+ from mns_scheduler.company_info.common.company_common_query_service import get_company_info
14
+ from datetime import datetime
15
+ import mns_common.utils.data_frame_util as data_frame_util
16
+ from loguru import logger
17
+
18
+
19
+ def sync_all_company_task():
20
+ # 同步公司行业信息
21
+ company_industry_info_task.sync_company_industry_info_task([])
22
+ logger.info("[全量]同步公司[行业]信息完成")
23
+ # 同步公司业务信息
24
+ company_business_info_task.sync_company_business_task([])
25
+ logger.info("[全量]同步公司[业务]信息完成")
26
+ # 同步公司基本信息
27
+ company_base_info_task.sync_company_base_info_task([])
28
+ logger.info("[全量]同步公司[基本]信息完成")
29
+ # 同步公司控股信息
30
+ company_hold_info_task.sync_all_company_hold_info_task([])
31
+ logger.info("[全量]同步公司[控股]信息完成")
32
+ # 同步公司公告信息
33
+ company_announce_info_task.sync_company_announce_task([])
34
+ logger.info("[全量]同步公司[公告]信息完成")
35
+
36
+
37
+ def sync_new_stock_company_task():
38
+ all_company_info_df = get_company_info()
39
+
40
+ now_date = datetime.now()
41
+ str_day = now_date.strftime('%Y-%m-%d')
42
+ # 去掉横线并转换为整数
43
+ now_day = int(str_day.replace('-', ''))
44
+
45
+ new_company_info_df = all_company_info_df.loc[
46
+ (all_company_info_df['list_date'] == 19890604) | (all_company_info_df['list_date'] >= now_day)]
47
+ if data_frame_util.is_empty(new_company_info_df):
48
+ return
49
+ symbol_list = list(new_company_info_df['symbol'])
50
+ # 同步公司行业信息
51
+ company_industry_info_task.sync_company_industry_info_task(symbol_list)
52
+ logger.info("[新股]同步公司[行业]信息完成")
53
+ # 同步公司业务信息
54
+ company_business_info_task.sync_company_business_task(symbol_list)
55
+ logger.info("[新股]同步公司[业务]信息完成")
56
+ # 同步公司基本信息
57
+ company_base_info_task.sync_company_base_info_task(symbol_list)
58
+ logger.info("[新股]同步公司[基本]信息完成")
59
+ # 同步公司控股信息
60
+ company_hold_info_task.sync_all_company_hold_info_task(symbol_list)
61
+ logger.info("[新股]同步公司[控股]信息完成")
62
+ # 同步公司公告信息
63
+ company_announce_info_task.sync_company_announce_task(symbol_list)
64
+ logger.info("[新股]同步公司[公告]信息完成")
65
+
66
+
67
+ if __name__ == '__main__':
68
+ sync_new_stock_company_task()
69
+ sync_all_company_task()
@@ -2,7 +2,7 @@ import sys
2
2
  import os
3
3
 
4
4
  file_path = os.path.abspath(__file__)
5
- end = file_path.index('mns') + 16
5
+ end = file_path.index('mns') + 17
6
6
  project_path = file_path[0:end]
7
7
  sys.path.append(project_path)
8
8
  from mns_common.db.MongodbUtil import MongodbUtil
@@ -7,12 +7,13 @@ project_path = file_path[0:end]
7
7
  sys.path.append(project_path)
8
8
  from mns_common.db.MongodbUtil import MongodbUtil
9
9
  import pandas as pd
10
- import mns_scheduler.concept.ths.detaill.ths_concept_detail_api as ths_concept_detail_api
10
+ import mns_common.api.ths.concept.web.ths_concept_index_web as ths_concept_index_web
11
11
  import time
12
12
  from loguru import logger
13
13
  import mns_common.component.company.company_common_service_api as company_common_service_api
14
14
  import mns_common.constant.db_name_constant as db_name_constant
15
15
  import mns_common.component.concept.ths_concept_common_service_api as ths_concept_common_service_api
16
+ import mns_common.api.ths.concept.app.ths_concept_index_app as ths_concept_index_app
16
17
 
17
18
  mongodb_util = MongodbUtil('27017')
18
19
 
@@ -76,9 +77,22 @@ def update_one_concept_relevance_industry(concept_code):
76
77
  ths_stock_concept_detail_df['first_relevance_industry'] = first_relevance_industry
77
78
  ths_stock_concept_detail_df['second_relevance_industry'] = second_relevance_industry
78
79
 
79
- mongodb_util.save_mongo(ths_concept_list_one_df, db_name_constant.THS_CONCEPT_LIST)
80
+ new_values_list = {"$set": {"first_relevance_industry": first_relevance_industry,
81
+ "second_relevance_industry": second_relevance_industry,
82
+ "first_relevance_industry_number": first_relevance_industry_number,
83
+ "second_relevance_industry_number": second_relevance_industry_number,
84
+ "concept_count": concept_count
85
+ }}
80
86
 
81
- mongodb_util.save_mongo(ths_stock_concept_detail_df, db_name_constant.THS_STOCK_CONCEPT_DETAIL)
87
+ update_query = {"symbol": concept_code}
88
+ mongodb_util.update_one_query(update_query, new_values_list, db_name_constant.THS_CONCEPT_LIST)
89
+
90
+ new_values_detail = {"$set": {"first_relevance_industry": first_relevance_industry,
91
+ "second_relevance_industry": second_relevance_industry
92
+
93
+ }}
94
+ update_query_detail = {"concept_code": concept_code}
95
+ mongodb_util.update_one_query(update_query_detail, new_values_detail, db_name_constant.THS_STOCK_CONCEPT_DETAIL)
82
96
 
83
97
 
84
98
  # 更新空名字
@@ -93,7 +107,7 @@ def update_null_name():
93
107
  exist_url = concept_one.url
94
108
 
95
109
  if name == '':
96
- concept_name = ths_concept_detail_api.get_ths_concept_detail(concept_code, None)
110
+ concept_name = ths_concept_index_web.get_concept_name(concept_code)
97
111
  query_concept = {"symbol": concept_code}
98
112
  new_values = {'$set': {"name": concept_name}}
99
113
  mongodb_util.update_one_query(query_concept, new_values, 'ths_concept_list')
@@ -114,6 +128,8 @@ def update_null_name():
114
128
 
115
129
 
116
130
  if __name__ == '__main__':
131
+ # update_one_concept_relevance_industry(886095)
132
+ update_null_name()
117
133
  logger.info("开始")
118
134
  update_ths_concept_info()
119
135
  logger.info("结束")