mns-scheduler 1.2.3.6__py3-none-any.whl → 1.2.3.8__py3-none-any.whl

mns_scheduler/company_info/base/sync_company_base_info_api.py

@@ -162,12 +162,11 @@ def single_thread_sync_company_info(east_money_stock_info,
                 lambda x: x[1:5] + '00')
             company_info_type['third_industry_code'] = company_info_type['hy3code'].apply(
                 lambda x: x[1:7])
-
-            company_info_type['main_business_list'] = company_info_type['main_business_list']
-            company_info_type['most_profitable_business'] = company_info_type['most_profitable_business']
-            company_info_type['most_profitable_business_rate'] = company_info_type['most_profitable_business_rate']
-            company_info_type['most_profitable_business_profit'] = company_info_type['most_profitable_business_profit']
-
+            # company_info_type['main_business_list'] = company_info_type['main_business_list']
+            # company_info_type['most_profitable_business'] = company_info_type['most_profitable_business']
+            # company_info_type['most_profitable_business_rate'] = company_info_type['most_profitable_business_rate']
+            # company_info_type['most_profitable_business_profit'] = company_info_type['most_profitable_business_profit']
+            #
             company_info_type['first_sw_industry'] = company_info_type['hy']
             company_info_type['second_sw_industry'] = company_info_type['hy2']
             company_info_type['third_sw_industry'] = company_info_type['hy3']
@@ -438,5 +437,5 @@ if __name__ == '__main__':
     # query = {"total_operate_income": 0}
     # un_report_company_info = mongodb_util.find_query_data(db_name_constant.COMPANY_INFO, query)
     # symbol_list = list(un_report_company_info['symbol'])
-    sync_company_base_info(['301269','688206','301095'])
+    sync_company_base_info(None)
     # group_by_industry()

mns_scheduler/company_info/clean/company_info_clean_api.py

@@ -13,6 +13,7 @@ import mns_scheduler.company_info.constant.company_constant_data as company_cons
 import mns_common.constant.db_name_constant as db_name_constant
 import mns_scheduler.concept.ths.detaill.ths_concept_detail_api as ths_concept_detail_api
 import mns_scheduler.company_info.base.sync_company_base_info_api as company_info_sync_api
+import mns_common.utils.data_frame_util as data_frame_util
 
 mongodb_util = MongodbUtil('27017')
 import mns_common.component.common_service_fun_api as common_service_fun_api
@@ -20,21 +21,26 @@ import mns_common.component.company.company_common_service_api as company_common
 
 
 # 修改行业信息
-def fix_company_industry(symbol):
+def clean_company_info(symbol):
     if symbol is not None:
         query = {"symbol": symbol}
         company_info = mongodb_util.find_query_data('company_info_base', query)
     else:
         company_info = mongodb_util.find_all_data('company_info_base')
 
-    # company_info = company_info.set_index(['second_sw_industry'], drop=False)
-    # del company_info['industry']
+    company_info = company_constant_data.fix_second_industry(company_info)
+
+    company_info = company_info.set_index(['second_sw_industry'], drop=False)
+
+    # 修改行业名称
+    del company_info['industry']
     # fix industry  name
-    # company_info = company_constant_data.remove_industry_name_special(company_info)
-    # industry_final_fix_df = industry_final_fix_df.set_index(['second_sw_industry'], drop=True)
-    # company_info = pd.merge(company_info, industry_final_fix_df, how='outer',
-    #                         left_index=True, right_index=True)
-    # 将申万第三行业做为行业
+    industry_final_fix_df = company_constant_data.get_fix_industry_name_df()
+    industry_final_fix_df = industry_final_fix_df.set_index(['second_sw_industry'], drop=True)
+    company_info = pd.merge(company_info, industry_final_fix_df, how='outer',
+                            left_index=True, right_index=True)
+
+    # 将申万第三行业做为行业 拆分过大的二级行业 主要有通用设备 和专业设备
     company_info = company_constant_data.fix_industry_use_sw_third(company_info.copy())
     company_info['industry'] = company_info['industry'].fillna('综合')
     company_info = company_constant_data.filed_sort(company_info)
@@ -65,7 +71,7 @@ def fix_company_industry(symbol):
     except BaseException as e:
         logger.error("出现异常:{},{}", symbol, e)
         query = {'concept_code': 885598}
-        ths_stock_concept_detail = mongodb_util.find_query_data('ths_stock_concept_detail', query)
+        ths_stock_concept_detail = mongodb_util.find_query_data(db_name_constant.THS_STOCK_CONCEPT_DETAIL, query)
         sub_stock_symbol_list = list(ths_stock_concept_detail['symbol'])
     company_info.loc[:, 'sub_stock'] = False
     company_info.loc[company_info['symbol'].isin(sub_stock_symbol_list), 'sub_stock'] = True
@@ -73,6 +79,20 @@ def fix_company_industry(symbol):
     try:
         company_info.dropna(subset=['symbol'], axis=0, inplace=True)
         company_info.dropna(subset=['_id'], axis=0, inplace=True)
+        ths_stock_industry_detail_df = mongodb_util.find_all_data(db_name_constant.THS_STOCK_INDUSTRY_DETAIL)
+        if data_frame_util.is_not_empty(ths_stock_industry_detail_df):
+            ths_stock_industry_detail_df = ths_stock_industry_detail_df[
+                ['symbol', 'ths_industry_name', 'ths_industry_code']]
+            ths_stock_industry_detail_df = ths_stock_industry_detail_df.set_index(['symbol'], drop=True)
+            company_info = company_info.set_index(['_id'], drop=False)
+            company_info = pd.merge(company_info, ths_stock_industry_detail_df, how='outer',
+                                    left_index=True, right_index=True)
+            company_info['ths_industry_code'] = company_info['ths_industry_code'].fillna('0')
+            company_info['ths_industry_name'] = company_info['ths_industry_name'].fillna('异常')
+
+        else:
+            company_info['ths_industry_code'] = '0'
+            company_info['ths_industry_name'] = '异常'
         mongodb_util.save_mongo(company_info, db_name_constant.COMPANY_INFO)
         # 保存历史数据
         save_company_info_his(company_info)
@@ -102,7 +122,7 @@ def new_company_info_update():
     for company_one in new_stock.itertuples():
         try:
             company_info_sync_api.sync_company_base_info([company_one.symbol])
-            fix_company_industry(company_one.symbol)
+            clean_company_info(company_one.symbol)
 
         except BaseException as e:
             logger.error("出现异常:{}", e)
@@ -110,4 +130,4 @@ def new_company_info_update():
 
 
 if __name__ == '__main__':
-    fix_company_industry(None)
+    clean_company_info(None)

mns_scheduler/company_info/constant/company_constant_data.py

@@ -59,32 +59,66 @@ def get_fix_symbol_industry():
                                  'new_industry'])
 
 
-# 修改行业分类的股票
-def get_fix_industry_info():
-    return pd.DataFrame([['450700', '旅游零售Ⅱ', '461000', '旅游及景区'],
-                         []],
-                        columns=['original_second_industry_code',
-                                 'original_second_industry_name',
-                                 'new_third_second_code',  # 二级级行业代码
-                                 'new_third_second_name'])
+# 修改二级行业分类的股票
+def get_fix_second_industry_info():
+    return pd.DataFrame([
+        ['110200', '渔业', "农林牧渔", "110000", '农业综合Ⅱ', '110900'],
+
+        ['110300', '林业', "农林牧渔", "110000", '农业综合Ⅱ', '110900'],
+
+        ['450700', '旅游零售Ⅱ', '社会服务', '460000', '旅游及景区', '461000'],
+
+        ['220700', '化工新材料Ⅱ', '基础化工', '220000', '非金属材料Ⅱ', '220900'],
+
+        ['330700', '其他家电Ⅱ', '家用电器', '330000', '小家电', '330300'],
+
+        ['250100', '建筑材料', '建筑材料', '610000', '装修建材', '610300'],
+
+        ['460600', '体育Ⅱ', '综合', '510000', '综合Ⅱ', '510100'],
 
+        ['770300', '医疗美容', '美容护理', '770000', '化妆品', '770200'],
 
-# def remove_industry_name_special(industry_df):
-#     industry_df['industry'] = industry_df['industry'].str.replace('Ⅱ', '', regex=False)
-#     return industry_df
+    ],
+        columns=['original_second_industry_code',
+                 'original_second_industry_name',
+                 'first_sw_industry',
+                 'first_industry_code',
+                 'second_sw_industry',  # 二级级行业代码
+                 'second_industry_code'])
 
 
-# 修改行业名称 暂时作废
-def get_industry_final_fix_df():
+def fix_second_industry(company_info):
+    fix_second_industry_df = get_fix_second_industry_info()
+    del fix_second_industry_df['original_second_industry_name']
+    fix_second_company_df = company_info.loc[
+        company_info['second_industry_code'].isin(fix_second_industry_df['original_second_industry_code'])]
+
+    no_fix_second_company_df = company_info.loc[~(
+        company_info['second_industry_code'].isin(fix_second_industry_df['original_second_industry_code']))]
+
+    fix_second_company_df = fix_second_company_df.set_index(['second_industry_code'], drop=True)
+    del fix_second_company_df['second_sw_industry']
+    del fix_second_company_df['first_industry_code']
+    del fix_second_company_df['first_sw_industry']
+    fix_second_industry_df = fix_second_industry_df.set_index(['original_second_industry_code'], drop=True)
+
+    fix_second_company_df = pd.merge(fix_second_company_df, fix_second_industry_df, how='outer',
+                                     left_index=True, right_index=True)
+
+    return pd.concat([no_fix_second_company_df, fix_second_company_df])
+
+
+# 修改行业名
+def get_fix_industry_name_df():
     return pd.DataFrame([
 
-        # 交通运输
+        # 交通运输 1
         ['物流', '物流'],
         ['铁路公路', '铁路公路'],
         ['航运港口', '航运港口'],
         ['航空机场', '航空机场'],
 
-        # 传媒
+        # 传媒 2
         ['数字媒体', '数字媒体'],
         ['电视广播Ⅱ', '电视广播'],
         ['游戏Ⅱ', '游戏'],
@@ -92,22 +126,21 @@ def get_industry_final_fix_df():
         ['影视院线', '影视院线'],
         ['广告营销', '广告营销'],
 
-        # 公用事业
+        # 公用事业 3
         ['燃气Ⅱ', '燃气'],
         ['电力', '电力'],
 
-        # 农林牧渔
-        ['养殖业', '养殖业'],  # merge 农林牧渔
-        ['农产品加工', '农产品加工'],  # merge 农林牧渔
-        ['饲料', '饲料'],  # merge  to 农林牧渔
-        ['渔业', '渔业'],  # merge  to 农林牧渔
-        # merge 农林牧渔
+        # 农林牧渔 4
+        ['养殖业', '养殖业'],
+        ['农产品加工', '农产品加工'],
+        ['饲料', '饲料'],
+        ['渔业', '渔业'],  # merge  to 农业综合
         ['动物保健Ⅱ', '动物保健'],
         ['种植业', '种植业'],
-        ['林业Ⅱ', '林业'],  # merge  农林牧渔
-        ['农业综合Ⅱ', '农业综合'],  # merge  农林牧渔
+        # ['林业Ⅱ', '林业'],  # merge  农业综合
+        ['农业综合Ⅱ', '农业综合'],  #
 
-        # 医药生物
+        # 医药生物 5
         ['化学制药', '化学制药'],
         ['生物制品', '生物制品'],
         ['中药Ⅱ', '中药'],
@@ -115,46 +148,85 @@ def get_industry_final_fix_df():
         ['医疗服务', '医疗服务'],
         ['医药商业', '医药商业'],
 
-        # 商贸零售
+        # 商贸零售 6
         ['一般零售', '一般零售'],
         ['互联网电商', '互联网电商'],
         ['贸易Ⅱ', '贸易'],  #
-        ['专业连锁Ⅱ', '专业连锁'],  # 专业连锁 综合Ⅱ
-        ['旅游零售Ⅱ', '旅游零售'],  # m
+        ['专业连锁Ⅱ', '零售专业连锁'],  # 专业连锁 综合Ⅱ
+        # ['旅游零售Ⅱ', '旅游零售'],  # merge旅游及景区
 
-        # 国防
-        ['军工电子Ⅱ', '军工电子'],  # todo merge 国防
-        ['地面兵装Ⅱ', '地面兵装'],  # todo merge 国防
+        # 国防 7
+        ['军工电子Ⅱ', '军工电子'],
+        ['地面兵装Ⅱ', '地面兵装'],
         ['航天装备Ⅱ', '航天装备'],
-        ['航空装备Ⅱ', '航空装备'],  # todo merge 国防
+        ['航空装备Ⅱ', '航空装备'],
         ['航海装备Ⅱ', '航海装备'],
 
-        # 基础化工
+        # 基础化工 8
         ['化学制品', '化学制品'],
         ['化学原料', '化学原料'],
         ['化学纤维', '化学纤维'],
         ['农化制品', '化肥农药'],
         ['塑料', '塑料'],
-        ['橡胶', '橡胶'],  # todo   exclude
+        ['橡胶', '橡胶'],
         ['非金属材料Ⅱ', '非金属材料'],
 
-        # 家用电器
+        # 家用电器 9
         ['白色家电', '白色家电'],  # merge 家用电器
         ['照明设备Ⅱ', '照明设备'],  # merge 家用电器
-        ['其他家电Ⅱ', '其他家电'],  # merge  家用电器
+        # ['其他家电Ⅱ', '其他家电'],  # merge  小家电
         ['家电零部件Ⅱ', '家电零部件'],  # merge 家用电器
         ['小家电', '小家电'],  # merge '家用电器'
         ['黑色家电', '黑色家电'],  # merge 家用电器
         ['厨卫电器', '厨卫电器'],  # merge   家用电器
 
-        # 汽车
+        # 建筑材料  10
+        ['装修建材', '装修建材'],  #
+        ['建筑建材', '装修建材'],  #
+        ['玻璃玻纤', '玻璃玻纤'],
+        ['水泥', '水泥'],
+
+        # 建筑装饰 11
+        ['基础建设', '基础建设'],
+        ['房屋建设Ⅱ', '房屋建设'],
+        ['工程咨询服务Ⅱ', '工程咨询服务'],
+        ['专业工程', '建筑专业工程'],
+        ['装修装饰Ⅱ', '装修装饰'],
+
+        # 房地产 12
+        ['房地产开发', '房地产'],
+        ['房地产服务', '房地产'],
+
+        # 有色金属 13
+        ['能源金属', '能源金属'],
+        ['小金属', '小金属'],
+        ['贵金属', '贵金属'],  #
+        ['金属新材料', '金属新材料'],
+        ['工业金属', '工业金属'],  # 铅锌  铝  铜
+
+        # 机械设备 14
+        ['自动化设备', '自动化设备'],
+        ['轨交设备Ⅱ', '轨交设备'],
+        ['通用设备', '通用设备'],
+        ['专用设备', '专用设备'],
+        ['工程机械', '工程机械'],
+
+        # 汽车 15
         ['汽车零部件', '汽车零部件'],
         ['汽车服务', '汽车服务'],
-        ['乘用车', '汽车整车'],  # = merge 汽车整车
-        ['商用车', '汽车整车'],
+        ['乘用车', '乘用车'],  # = merge 汽车整车
+        ['商用车', '商用车'],
         ['摩托车及其他', '摩托车及其他'],
 
-        # 电力设备
+        # 煤炭 16
+        ['焦炭Ⅱ', '焦炭'],  # merge  to 煤炭
+        ['煤炭开采', '煤炭开采'],  # merge 煤炭
+
+        # 环保 17
+        ['环境治理', '环境治理'],  # merge, 环保
+        ['环保设备Ⅱ', '环保设备'],  # merge '环保'
+
+        # 电力设备 18
         ['电网设备', '电网设备'],
         ['电池', '电池'],
         ['电机Ⅱ', '电机'],  #
@@ -162,20 +234,7 @@ def get_industry_final_fix_df():
         ['风电设备', '风电设备'],
         ['其他电源设备Ⅱ', '其他电源设备'],
 
-        # 房地产
-        ['房地产开发', '房地产'],
-        ['房地产服务', '房地产'],
-
-        # 计算机 联动板块
-        ['计算机设备', '计算机设备'],
-        ['IT服务Ⅱ', 'IT服务'],
-        ['软件开发', '软件开发'],
-
-        # 环保
-        ['环境治理', '环保'],  # merge, 环保
-        ['环保设备Ⅱ', '环保'],  # merge '环保'
-
-        # 电子
+        # 电子 19
         ['半导体', '半导体'],
         ['电子化学品Ⅱ', '电子化学品'],
         ['光学光电子', '光学光电子'],
@@ -183,99 +242,69 @@ def get_industry_final_fix_df():
         ['元件', '元件'],
         ['其他电子Ⅱ', '其他电子'],  #
 
-        # 金融 互联网金融 参股券商
-        ['证券Ⅱ', '证券'],
-        ['国有大型银行Ⅱ', '银行'],  # merge  to 银行
-        ['城商行Ⅱ', '银行'],  #
-        ['农商行Ⅱ', '银行'],  # merge   银行
-        ['股份制银行Ⅱ', '银行'],  # merge  to 银行
-        ['保险Ⅱ', '保险'],
-        ['多元金融', '多元金融'],
-
-        # 通信
-        ['通信服务', '通信服务'],
-        ['通信设备', '通信设备'],
-
-        # 酒 喝死你
-        ['白酒Ⅱ', '酒类'],  # merge '酒类'
-        ['非白酒', '酒类'],  # merge '酒类'
-
-        # 轻工制造
-        ['造纸', '造纸'],
-        ['包装印刷', '包装印刷'],  #
-        ['文娱用品', '文娱用品'],  #
-        ['家居用品', '家居用品'],
-
-        # 纺织服装
-        ['服装家纺', '纺织服装'],  # merge 纺织服装
-        ['纺织制造', '纺织服装'],  # merge 纺织服装
-        ['饰品', '饰品'],  # todo exclude
-
-        # 美容护理
-        ['化妆品', '美容护理'],  # merge  to 美容护理
-        ['医疗美容', '美容护理'],  # merge  美容护理
-        ['个护用品', '个护用品'],
-
-        # 食品饮料
-        ['饮料乳品', '饮料乳品'],  # merge 食品饮料
-        ['食品加工', '食品加工'],  # merge 食品饮料
-        ['调味发酵品Ⅱ', '调味发酵品'],  # merge  to  食品饮料
-        ['休闲食品', '休闲食品'],  # merge '食品饮料'
-
-        # 石油石化
+        # 石油石化 20
         ['炼化及贸易', '石油行业'],  # merge 石油行业
         ['油服工程', '采掘行业'],  # merge '采掘行业'
         ['油气开采Ⅱ', '石油行业'],  # merge  to 石油行业
 
-        #  钢铁
-        ['普钢', '钢铁'],  # merge 钢铁
-        ['特钢Ⅱ', '钢铁'],  # merge '钢铁'
-        ['冶钢原料', '钢铁'],  # merge to 钢铁
+        # 社会服务 21
+        ['专业服务', '社会专业服务'],
+        ['旅游及景区', '旅游及景区'],
+        ['酒店餐饮', '旅游酒店'],
+        ['教育', '教育'],
+        # ['体育Ⅱ', '体育'],
 
-        # 有色金属
-        ['能源金属', '能源金属'],  # merge
-        ['小金属', '小金属'],
-        ['贵金属', '贵金属'],  #
-        ['金属新材料', '金属新材料'],
-        # 铅锌  铝  铜
-        ['工业金属', '工业金属'],
+        # 纺织服装 22
+        ['服装家纺', '纺织服装'],  # merge 纺织服装
+        ['纺织制造', '纺织服装'],  # merge 纺织服装
+        ['饰品', '饰品'],
 
-        # 建筑装饰
-        ['基础建设', '工程建设'],
-        ['房屋建设Ⅱ', '工程建设'],  # merge 工程建设
-        ['工程咨询服务Ⅱ', '工程咨询服务'],
-        ['专业工程', '建筑工程'],
-        ['装修装饰Ⅱ', '装修装饰'],
-        ['工程机械', '工程机械'],
+        # 综合 23
+        ['综合Ⅱ', '综合'],
 
-        # 建筑材料
-        ['装修建材', '装修建材'],  #
-        ['建筑建材', '装修建材'],  #
-        ['玻璃玻纤', '玻璃玻纤'],
-        ['水泥', '水泥'],
+        # 美容护理 24
+        ['化妆品', '化妆美容'],  # merge  to 化妆美容
+        # ['医疗美容', '化妆美容'],  # merge  化妆美容
+        ['个护用品', '个护用品'],
 
-        # 旅游酒店
-        ['酒店餐饮', '旅游酒店'],  # merge  to 旅游酒店
+        # 计算机 25
+        ['计算机设备', '计算机设备'],
+        ['IT服务Ⅱ', 'IT服务'],
+        ['软件开发', '软件开发'],
 
-        ['旅游及景区', '旅游酒店'],  # merge 旅游酒店
+        # 轻工制造 26
+        ['造纸', '造纸'],
+        ['包装印刷', '包装印刷'],  #
+        ['文娱用品', '文娱用品'],  #
+        ['家居用品', '家居用品'],
 
-        # 煤炭
-        ['焦炭Ⅱ', '煤炭'],  # merge  to 煤炭
-        ['煤炭开采', '煤炭'],  # merge 煤炭
+        # 通信 27
+        ['通信服务', '通信服务'],
+        ['通信设备', '通信设备'],
 
-        ['教育', '教育'],
+        #  钢铁 28
+        ['普钢', '普钢'],  #
+        ['特钢Ⅱ', '特钢'],
+        ['冶钢原料', '冶钢原料'],
 
-        ['自动化设备', '自动化设备'],
-        ['轨交设备Ⅱ', '轨交设备'],
+        # 银行 29
+        ['国有大型银行Ⅱ', '银行'],  # merge  to 银行
+        ['城商行Ⅱ', '银行'],  #
+        ['农商行Ⅱ', '银行'],  # merge   银行
+        ['股份制银行Ⅱ', '银行'],  # merge  to 银行
 
-        # 机械设备  仪器仪表 农用机械 制冷空调设备  印刷包装机械 机床工具
-        # 楼宇设备 磨具磨料  纺织服装设备  能源及重型设备 金属制品
-        ['专用设备', '专用设备'],  # todo exclude
-        ['通用设备', '通用设备'],  # todo exclude
-        ['专业服务', '专业服务'],  # todo exclude 无法具体分类
+        # 非银金融  30
+        ['证券Ⅱ', '证券'],
+        ['保险Ⅱ', '保险'],
+        ['多元金融', '多元金融'],
 
-        ['体育Ⅱ', '综合'],  # merge  综合Ⅱ
-        ['综合Ⅱ', '综合']  # todo exclude
+        # 食品饮料 31
+        ['饮料乳品', '饮料乳品'],  # merge 食品饮料
+        ['食品加工', '食品加工'],  # merge 食品饮料
+        ['调味发酵品Ⅱ', '调味发酵品'],  # merge  to  食品饮料
+        ['休闲食品', '休闲食品'],  # merge '食品饮料'
+        ['白酒Ⅱ', '白酒'],
+        ['非白酒', '非白酒']
 
     ], columns=['second_sw_industry', 'industry'])
 
@@ -287,24 +316,65 @@ def fix_industry_use_sw_third(company_info_df):
     company_info.loc[company_info.third_industry_code == '240303', 'industry'] = '铅锌'
     company_info.loc[company_info.third_industry_code == '240301', 'industry'] = '铝'
     company_info.loc[company_info.third_industry_code == '240302', 'industry'] = '铜'
+
+    company_info.loc[company_info.third_industry_code == '240303', 'second_sw_industry'] = '铅锌'
+    company_info.loc[company_info.third_industry_code == '240301', 'second_sw_industry'] = '铝'
+    company_info.loc[company_info.third_industry_code == '240302', 'second_sw_industry'] = '铜'
+
+    company_info.loc[company_info.third_industry_code == '240303', 'second_industry_code'] = '240303'
+    company_info.loc[company_info.third_industry_code == '240301', 'second_industry_code'] = '240301'
+    company_info.loc[company_info.third_industry_code == '240302', 'second_industry_code'] = '240302'
+
     # 细分专业设备
     company_info.loc[company_info.third_industry_code == '640203', 'industry'] = '能源及重型设备'
     company_info.loc[company_info.third_industry_code == '640204', 'industry'] = '楼宇设备'
     company_info.loc[company_info.third_industry_code == '640206', 'industry'] = '纺织服装设备'
     company_info.loc[company_info.third_industry_code == '640207', 'industry'] = '农用机械'
     company_info.loc[company_info.third_industry_code == '640208', 'industry'] = '印刷包装机械'
+
+    company_info.loc[company_info.third_industry_code == '640203', 'second_sw_industry'] = '能源及重型设备'
+    company_info.loc[company_info.third_industry_code == '640204', 'second_sw_industry'] = '楼宇设备'
+    company_info.loc[company_info.third_industry_code == '640206', 'second_sw_industry'] = '纺织服装设备'
+    company_info.loc[company_info.third_industry_code == '640207', 'second_sw_industry'] = '农用机械'
+    company_info.loc[company_info.third_industry_code == '640208', 'second_sw_industry'] = '印刷包装机械'
+
+    company_info.loc[company_info.third_industry_code == '640203', 'second_industry_code'] = '640203'
+    company_info.loc[company_info.third_industry_code == '640204', 'second_industry_code'] = '640204'
+    company_info.loc[company_info.third_industry_code == '640206', 'second_industry_code'] = '640206'
+    company_info.loc[company_info.third_industry_code == '640207', 'second_industry_code'] = '640207'
+    company_info.loc[company_info.third_industry_code == '640208', 'second_industry_code'] = '640208'
+
     # todo 细分
     company_info.loc[company_info.third_industry_code == '640209', 'industry'] = '专用设备'
-    company_info.loc[company_info.third_industry_code == '630402', 'industry'] = '专用设备'
+    company_info.loc[company_info.third_industry_code == '640209', 'second_sw_industry'] = '专用设备'
+    company_info.loc[company_info.third_industry_code == '640209', 'second_industry_code'] = '640208'
+
+    company_info.loc[company_info.third_industry_code == '260205', 'industry'] = '专用设备'
+    company_info.loc[company_info.third_industry_code == '260205', 'second_sw_industry'] = '专用设备'
+    company_info.loc[company_info.third_industry_code == '260205', 'second_industry_code'] = '260205'
+
     # 细分通用设备
     company_info.loc[company_info.third_industry_code == '640101', 'industry'] = '机床工具'
     company_info.loc[company_info.third_industry_code == '640103', 'industry'] = '磨具磨料'
     company_info.loc[company_info.third_industry_code == '640105', 'industry'] = '制冷空调设备'
     company_info.loc[company_info.third_industry_code == '640106', 'industry'] = '通用设备'
     company_info.loc[company_info.third_industry_code == '640107', 'industry'] = '仪器仪表'
-    company_info.loc[company_info.third_industry_code == '640301', 'industry'] = '仪器仪表'
-    # todo 细分
     company_info.loc[company_info.third_industry_code == '640108', 'industry'] = '金属制品'
+
+    company_info.loc[company_info.third_industry_code == '640101', 'second_sw_industry'] = '机床工具'
+    company_info.loc[company_info.third_industry_code == '640103', 'second_sw_industry'] = '磨具磨料'
+    company_info.loc[company_info.third_industry_code == '640105', 'second_sw_industry'] = '制冷空调设备'
+    company_info.loc[company_info.third_industry_code == '640106', 'second_sw_industry'] = '通用设备'
+    company_info.loc[company_info.third_industry_code == '640107', 'second_sw_industry'] = '仪器仪表'
+    company_info.loc[company_info.third_industry_code == '640108', 'second_sw_industry'] = '金属制品'
+
+    company_info.loc[company_info.third_industry_code == '640101', 'second_industry_code'] = '640101'
+    company_info.loc[company_info.third_industry_code == '640103', 'second_industry_code'] = '640103'
+    company_info.loc[company_info.third_industry_code == '640105', 'second_industry_code'] = '640105'
+    company_info.loc[company_info.third_industry_code == '640106', 'second_industry_code'] = '640106'
+    company_info.loc[company_info.third_industry_code == '640107', 'second_industry_code'] = '640107'
+    company_info.loc[company_info.third_industry_code == '640108', 'second_industry_code'] = '640108'
+
     return company_info
 
 
@@ -351,14 +421,6 @@ def fix_symbol_industry(company_info, symbol):
     return company_info
 
 
-def fix_one_symbol():
-    symbol = '300483'
-    company_info = mongodb_util.find_query_data('company_info', query={'_id': symbol})
-    company_info = fix_symbol_industry(company_info, symbol)
-    company_info['industry'] = company_info['second_sw_industry']
-    mongodb_util.save_mongo(company_info, 'company_info')
-
-
 def filed_sort(company_info):
     return company_info[[
         "_id",
@@ -399,6 +461,7 @@ def filed_sort(company_info):
         "ROE",
         "classification",
         "base_business",
+        "intro",
         "address",
         "market_id",
         "symbol",
@@ -423,6 +486,5 @@ def filed_sort(company_info):
 
 
 if __name__ == '__main__':
-    fix_one_symbol()
-    industry_df_test = get_industry_final_fix_df()
+    industry_df_test = get_fix_industry_name_df()
     print(industry_df_test)

mns_scheduler/concept/ths/common/ths_concept_sync_common_api.py

@@ -196,7 +196,7 @@ def update_company_info(new_concept_symbol_df):
     if new_concept_symbol_df.shape[0] > 0:
         symbol_list = list(new_concept_symbol_df['symbol'])
         company_info_sync_api.sync_company_base_info(symbol_list)
-        company_info_clean_api.fix_company_industry(None)
+        company_info_clean_api.clean_company_info(None)
         # 公司缓存信息清除
         company_common_service_api.company_info_industry_cache_clear()
 

mns_scheduler/industry/__init__.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)

mns_scheduler/industry/ths/__init__.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)

mns_scheduler/industry/ths/ths_industry_index_service.py

@@ -0,0 +1,58 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)
+import mns_common.api.ths.concept.app.ths_concept_index_app as ths_concept_index_app
+from datetime import datetime
+import mns_common.component.trade_date.trade_date_common_service_api as trade_date_common_service_api
+import mns_common.utils.date_handle_util as date_handle_util
+import mns_common.utils.data_frame_util as data_frame_util
+import pandas as pd
+import mns_common.component.common_service_fun_api as common_service_fun_api
+
+
+# 通过api 获取ths行业和指数
+def get_ths_index_by_api(query_type):
+    now_date = datetime.now()
+    hour = now_date.hour
+    minute = now_date.minute
+    now_str_day = now_date.strftime('%Y-%m-%d')
+
+    is_trade_day = trade_date_common_service_api.is_trade_day(now_str_day)
+
+    if bool(1 - is_trade_day):
+        last_trade_day = trade_date_common_service_api.get_before_trade_date(now_str_day, 1)
+        begin_time = date_handle_util.no_slash_date(last_trade_day) + '093000'
+        end_time = date_handle_util.no_slash_date(last_trade_day) + '150000'
+    else:
+        if hour < 9 or (hour == 9 and minute <= 25):
+            last_trade_day = trade_date_common_service_api.get_before_trade_date(now_str_day, 2)
+            begin_time = date_handle_util.no_slash_date(last_trade_day) + '093000'
+            end_time = date_handle_util.no_slash_date(last_trade_day) + '150000'
+        else:
+            begin_time = date_handle_util.no_slash_date(now_str_day) + '093000'
+            if hour == 9:
+                hour = '0' + str(hour)
+                end_time = date_handle_util.no_slash_date(now_str_day) + str(hour) + str(minute) + '00'
+            elif (hour == 11 and minute >= 30) or (hour == 12):
+                end_time = date_handle_util.no_slash_date(now_str_day) + '113000'
+            elif hour >= 15:
+                end_time = date_handle_util.no_slash_date(now_str_day) + '150000'
+            else:
+                end_time = date_handle_util.no_slash_date(now_str_day) + str(hour) + str(minute) + '00'
+
+    df = ths_concept_index_app.get_ths_concept_his_info(begin_time, end_time, 500, query_type)
+    if data_frame_util.is_empty(df):
+        return pd.DataFrame()
+    df['turnover'] = round(df['turnover'] / common_service_fun_api.HUNDRED_MILLION, 1)
+    df['net_inflow_of_main_force'] = round(df['net_inflow_of_main_force'] / common_service_fun_api.TEN_THOUSAND, 1)
+    df.fillna('', inplace=True)
+    return df
+
+
+if __name__ == '__main__':
+    df_industry = get_ths_index_by_api(1)
+    print(df_industry)

mns_scheduler/industry/ths/ths_industry_sync_service.py

@@ -0,0 +1,68 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)
+import mns_scheduler.industry.ths.ths_industry_index_service as ths_industry_index_service
+from mns_common.db.MongodbUtil import MongodbUtil
+import time
+from datetime import datetime
+from loguru import logger
+import mns_common.utils.data_frame_util as data_frame_util
+import mns_common.api.ths.concept.app.ths_concept_detail_app as ths_concept_detail_app
+
+mongodb_util = MongodbUtil('27017')
+import mns_common.constant.db_name_constant as db_name_constant
+
+
+# 同步同花顺行业指数
+def sync_ths_industry_index():
+    ths_industry_index_df = ths_industry_index_service.get_ths_index_by_api(1)
+    if data_frame_util.is_empty(ths_industry_index_df):
+        return None
+    ths_industry_index_df['_id'] = ths_industry_index_df['block_code']
+    ths_industry_index_df = ths_industry_index_df[[
+        '_id',
+        'turnover',
+        'block_market',
+        'block_code',
+        'block_name',
+        'net_inflow_of_main_force',
+        'chg'
+    ]]
+    now_date = datetime.now()
+    str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
+    ths_industry_index_df['str_now_date'] = str_now_date
+    mongodb_util.save_mongo(ths_industry_index_df, db_name_constant.THS_INDUSTRY_LIST)
+
+
+def sync_ths_industry_detail():
+    ths_industry_list_df = mongodb_util.find_all_data(db_name_constant.THS_INDUSTRY_LIST)
+    for industry_one in ths_industry_list_df.itertuples():
+        try:
+            time.sleep(1)
+            now_date = datetime.now()
+            str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
+
+            ths_industry_symbol_detail_df = ths_concept_detail_app.get_ths_concept_detail_by_app(
+                industry_one.block_code)
+            ths_industry_symbol_detail_df = ths_industry_symbol_detail_df.rename(
+                columns={"concept_code": 'ths_industry_code',
+                         "concept_name": 'ths_industry_name',
+                         })
+            if data_frame_util.is_empty(ths_industry_symbol_detail_df):
+                continue
+            ths_industry_symbol_detail_df['str_now_date'] = str_now_date
+            ths_industry_symbol_detail_df['_id'] = ths_industry_symbol_detail_df['symbol']
+            mongodb_util.save_mongo(ths_industry_symbol_detail_df, db_name_constant.THS_STOCK_INDUSTRY_DETAIL)
+            logger.info("同步ths行业股票详情:{}", industry_one.block_name)
+
+        except BaseException as e:
+            logger.error("同步ths行业股票详情异常:{}", e)
+
+
+if __name__ == '__main__':
+    sync_ths_industry_index()
+    sync_ths_industry_detail()

mns_scheduler/zz_task/data_sync_task.py

@@ -56,6 +56,7 @@ import mns_scheduler.hk.hk_industry_info_sync_service_api as hk_industry_info_sy
 import mns_scheduler.hk.hk_company_info_sync_service_api as hk_company_info_sync_service_api
 import mns_scheduler.zt.zt_pool.update_null_zt_reason_api as update_null_zt_reason_api
 import mns_scheduler.trade.tfp.stock_tfp_info_sync as stock_tfp_info_sync
+import mns_scheduler.industry.ths.ths_industry_sync_service as ths_industry_sync_service
 
 
 # 同步交易日期任务完成
@@ -288,7 +289,7 @@ def sync_new_concept_index_task():
 # 清洗公司基本信息
 def update_company_base_info():
     company_info_sync_api.sync_company_base_info(None)
-    company_info_clean_api.fix_company_industry(None)
+    company_info_clean_api.clean_company_info(None)
     # 退市股票同步
     de_list_stock_service.sync_de_list_stock()
     logger.info('同步公司基本信息任务完成')
@@ -404,6 +405,13 @@ def sync_stock_tfp():
     stock_tfp_info_sync.sync_stock_tfp(str_day)
 
 
+def sync_ths_industry_info():
+    logger.info("同步同花顺行业信息开始")
+    ths_industry_sync_service.sync_ths_industry_index()
+    ths_industry_sync_service.sync_ths_industry_detail()
+    logger.info("同步同花顺行业信息完成")
+
+
 # # 定义BlockingScheduler
 blockingScheduler = BlockingScheduler()
 # sync_trade_date 同步交易日期
@@ -495,7 +503,7 @@ blockingScheduler.add_job(trader_client_auto_login, 'cron', hour='08,12', minute
 # 同步公司备注信息
 blockingScheduler.add_job(sync_company_remark_info, 'cron', hour='22', minute='30')
 # 自选股操作
-blockingScheduler.add_job(self_choose_stock_task, 'cron', hour='17,22', minute='30')
+blockingScheduler.add_job(self_choose_stock_task, 'cron', hour='08,17,21', minute='30')
 # 同步财务报表任务
 blockingScheduler.add_job(sync_financial_report_task, 'cron', hour='17', minute='30')
 
@@ -504,6 +512,9 @@ blockingScheduler.add_job(sync_hk_company_industry_info, 'cron', hour='08,15', m
 # 更新空的涨停原因股票
 blockingScheduler.add_job(update_null_zt_reason, 'cron', hour='16,17,18,19,20,21,22,23', minute='19')
 
+# 同步同花顺行业信息
+blockingScheduler.add_job(sync_ths_industry_info, 'cron', hour='17,22', minute='38')
+
 # 同步新公告信息 感觉没有必要同步 直接连接过去查看
 # blockingScheduler.add_job(sync_company_announce, 'cron', hour='07,18,23', minute='33')
 

{mns_scheduler-1.2.3.6.dist-info → mns_scheduler-1.2.3.8.dist-info}/METADATA

@@ -1,4 +1,4 @@
 Metadata-Version: 2.1
 Name: mns-scheduler
-Version: 1.2.3.6
+Version: 1.2.3.8
 

{mns_scheduler-1.2.3.6.dist-info → mns_scheduler-1.2.3.8.dist-info}/RECORD

@@ -7,12 +7,12 @@ mns_scheduler/company_info/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlI
 mns_scheduler/company_info/announce/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
 mns_scheduler/company_info/announce/company_announce_sync_service.py,sha256=O0x0dqqnCmzzFenw4fShzWPAi_wegtvAZ9EB0ubrvnM,2808
 mns_scheduler/company_info/base/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
-mns_scheduler/company_info/base/sync_company_base_info_api.py,sha256=aaOqOs9X-0K7Wkqo0uOeuT4BERKx6HkmaSZmGuD3DBc,20553
+mns_scheduler/company_info/base/sync_company_base_info_api.py,sha256=D6puyEfh0JtS7grQ4OQ1AK2c4eDXlhOHFAOIEfNUMUM,20548
 mns_scheduler/company_info/base/sync_company_hold_info_api.py,sha256=W3Nj9st45efx8cy_42PRTcOXijWKnkO1-ZFRyyfR3S0,1587
 mns_scheduler/company_info/clean/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
-mns_scheduler/company_info/clean/company_info_clean_api.py,sha256=HvWZl70zgDxD74wk1z0jZqj3DIUBQGs0wqeT3g6yWi8,4876
+mns_scheduler/company_info/clean/company_info_clean_api.py,sha256=pDTOowP7byNK2i3XQJhKwLx6YFUq2Az30ckeqe5EEQE,6095
 mns_scheduler/company_info/constant/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
-mns_scheduler/company_info/constant/company_constant_data.py,sha256=-v3HDHO5pWrCgfHk-NjDPVrQb2yhjw2PByzyOcHLLwU,18361
+mns_scheduler/company_info/constant/company_constant_data.py,sha256=az9oWY6yffa9dSvkwh6P9SZpVgub0GRaafEWrAvnURU,22341
 mns_scheduler/company_info/de_list_stock/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
 mns_scheduler/company_info/de_list_stock/de_list_stock_service.py,sha256=GCp6hlvO-SuH1oIpEsYZwEnGUOa6fXb2D7CqAUYXKQA,1993
 mns_scheduler/company_info/remark/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
@@ -23,7 +23,7 @@ mns_scheduler/concept/clean/kpl_concept_clean_api.py,sha256=xxIIgrXLI6xLf10t4unJ
 mns_scheduler/concept/clean/ths_concept_clean_api.py,sha256=ICp1g2jP13Px1ett1En05vPzUaB25Usp7r2_L-seRgU,7077
 mns_scheduler/concept/ths/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
 mns_scheduler/concept/ths/common/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
-mns_scheduler/concept/ths/common/ths_concept_sync_common_api.py,sha256=lXzAWRJMU3jsXf1mhFaIOyEsRBWCndXDf41Dc40FDdU,10320
+mns_scheduler/concept/ths/common/ths_concept_sync_common_api.py,sha256=uQ-cOBZ7mpYCvYwSvoJXadOqlHnoLvwUaEbm1ccr6Zs,10318
 mns_scheduler/concept/ths/common/ths_concept_update_common_api.py,sha256=4BQT3A9t-nDIyCpILgRZF7ZOgK1oabp-gJl5nyWvKWc,4418
 mns_scheduler/concept/ths/detaill/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 mns_scheduler/concept/ths/detaill/ths_concept_detail_api.py,sha256=WLXmy6dhKF5GJePDoj-E80KLLMwJHeDYlq8bEhGS4W4,10883
@@ -55,6 +55,10 @@ mns_scheduler/finance/sync_financial_report_service_api.py,sha256=NTdL2bIpo3-pGC
 mns_scheduler/hk/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
 mns_scheduler/hk/hk_company_info_sync_service_api.py,sha256=yZuJa3hDxudkfjurG4Z0LpA8X5JT6UAKESkVhKRNqx4,3746
 mns_scheduler/hk/hk_industry_info_sync_service_api.py,sha256=HG-_KMQL3irL5pFbrnnv6srFT1Bo04QWwS_Kh3Qg0OU,2403
+mns_scheduler/industry/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
+mns_scheduler/industry/ths/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
+mns_scheduler/industry/ths/ths_industry_index_service.py,sha256=FcP4J0p3HP2tqD19dNmQfcShTl0hY7Ke7DJWat1MZFE,2686
+mns_scheduler/industry/ths/ths_industry_sync_service.py,sha256=gAzvEG6rHdi0AXmo5vFxeJE81H1bqhE2kMLxZLW0nMQ,2762
 mns_scheduler/irm/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
 mns_scheduler/irm/stock_irm_cninfo_service.py,sha256=mTwD7ZYKk9XlA3KD1SQWz2A8-OKAH2mdUl2A9Pzwrz8,9560
 mns_scheduler/irm/api/__init__.py,sha256=wEg73KlZo-dU0yKGwpA1C2y6LZm4IBb94tNda1tqLeg,163
@@ -142,8 +146,8 @@ mns_scheduler/zt/zt_pool/em_zt_pool_sync_api.py,sha256=F3IzCFgVPRxyC9j90aLLF1SO3
 mns_scheduler/zt/zt_pool/ths_zt_pool_sync_api.py,sha256=Sy39T-yFwLSIIoSZqQzS-6-W1RlaFWvYpksEXKQVFdI,10456
 mns_scheduler/zt/zt_pool/update_null_zt_reason_api.py,sha256=1uoiR2Uw46kDfjkvNg2US5rd_4OIkYO3872gIJOufUY,2135
 mns_scheduler/zz_task/__init__.py,sha256=QWBdZwBCvQw8aS4hnL9_pg3U3ZiNLUXzlImyy9WhUcI,163
-mns_scheduler/zz_task/data_sync_task.py,sha256=Kpv5LS_Vw6W90Ak_R2uOtCcYXRMu9LRKqviVA3OqHtI,21201
-mns_scheduler-1.2.3.6.dist-info/METADATA,sha256=xGoalNKW0yEnEMLzqY3favOvn-BhWPYzjNKno_sXlxM,64
-mns_scheduler-1.2.3.6.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-mns_scheduler-1.2.3.6.dist-info/top_level.txt,sha256=PXQDFBGR1pWmsUbH5yiLAh71P5HZODTRED0zJ8CCgOc,14
-mns_scheduler-1.2.3.6.dist-info/RECORD,,
+mns_scheduler/zz_task/data_sync_task.py,sha256=RIk2ymL9xFQB5OkKWL2NAbTaDzQVaAZ1PkkhgQb9pxI,21669
+mns_scheduler-1.2.3.8.dist-info/METADATA,sha256=SMEQ6g-kjUXCZ0Ii30oesbFqQnOeGw6Dc4dXO9MIABA,64
+mns_scheduler-1.2.3.8.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+mns_scheduler-1.2.3.8.dist-info/top_level.txt,sha256=PXQDFBGR1pWmsUbH5yiLAh71P5HZODTRED0zJ8CCgOc,14
+mns_scheduler-1.2.3.8.dist-info/RECORD,,