mns-scheduler 1.4.5.6__tar.gz → 1.4.7.4__tar.gz

{mns_scheduler-1.4.5.6 → mns_scheduler-1.4.7.4}/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.1
 Name: mns_scheduler
-Version: 1.4.5.6
+Version: 1.4.7.4

mns_scheduler-1.4.7.4/mns_scheduler/company_info/em_stock_info/clean_em_us_hk_stock_info.py

@@ -0,0 +1,30 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)
+
+from mns_common.db.MongodbUtil import MongodbUtil
+from loguru import logger
+from datetime import datetime
+import mns_common.constant.extra_income_db_name as extra_income_db_name
+
+mongodb_util = MongodbUtil('27017')
+
+
+def clean_us_stock_concept_industry():
+    em_us_stock_info_df = mongodb_util.find_all_data(extra_income_db_name.EM_US_STOCK_INFO)
+    return em_us_stock_info_df
+
+
+if __name__ == '__main__':
+    em_us_stock_info_test_df = clean_us_stock_concept_industry()
+    em_us_stock_info_test_industry_df = em_us_stock_info_test_df.loc[em_us_stock_info_test_df['industry'] != '-']
+
+    em_us_stock_info_test_no_industry_df = em_us_stock_info_test_df.loc[em_us_stock_info_test_df['industry'] == '-']
+    em_us_stock_info_test_no_industry_df = em_us_stock_info_test_no_industry_df.sort_values(by=['flow_mv'],
+                                                                                            ascending=False)
+
+    print(em_us_stock_info_test_industry_df)

{mns_scheduler-1.4.5.6 → mns_scheduler-1.4.7.4}/mns_scheduler/company_info/em_stock_info/sync_em_stock_info_sync.py

@@ -23,8 +23,8 @@ mongodb_util = MongodbUtil('27017')
 
 def sync_all_em_stock_info():
     logger.info("同步东方财富a,etf,kzz,us,hk,信息开始")
-    # 同步东方财富A股股票信息
-    sync_stock_info()
+    # # 同步东方财富A股股票信息
+    # sync_stock_info()
     # 同步东方财富A股可转债信息
     sync_kzz_info()
     # 同步东方财富A股ETF信息
@@ -48,6 +48,7 @@ def sync_kzz_info():
         em_kzz_info = east_money_debt_api.get_kzz_real_time_quotes(30, 6)
         em_kzz_info['_id'] = em_kzz_info['symbol']
         em_kzz_info['sync_time'] = str_now_date
+        em_kzz_info = em_kzz_info.fillna(0)
         mongodb_util.save_mongo(em_kzz_info, extra_income_db_name.EM_KZZ_INFO)
     except BaseException as e:
         logger.error("同步东方财富可转债信息异常:{}", e)
@@ -62,6 +63,7 @@ def sync_etf_info():
         em_etf_info = east_money_etf_api.get_etf_real_time_quotes(30, 6)
         em_etf_info['_id'] = em_etf_info['symbol']
         em_etf_info['sync_time'] = str_now_date
+        em_etf_info = em_etf_info.fillna(0)
         mongodb_util.save_mongo(em_etf_info, extra_income_db_name.EM_ETF_INFO)
     except BaseException as e:
         logger.error("同步东方财富ETF信息异常:{}", e)
@@ -73,10 +75,11 @@ def sync_stock_info():
     str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
     logger.error("同步东方财富股票信息")
     try:
-        em_etf_info = east_money_stock_a_v2_api.get_stock_real_time_quotes(60)
-        em_etf_info['_id'] = em_etf_info['symbol']
-        em_etf_info['sync_time'] = str_now_date
-        mongodb_util.save_mongo(em_etf_info, extra_income_db_name.EM_A_STOCK_INFO)
+        em_stock_info = east_money_stock_a_v2_api.get_stock_real_time_quotes(60)
+        em_stock_info['_id'] = em_stock_info['symbol']
+        em_stock_info['sync_time'] = str_now_date
+        em_stock_info = em_stock_info.fillna(0)
+        mongodb_util.save_mongo(em_stock_info, extra_income_db_name.EM_A_STOCK_INFO)
     except BaseException as e:
         logger.error("同步东方财富ETF信息异常:{}", e)
 
@@ -93,6 +96,7 @@ def sync_hk_stock_info():
         em_hk_info = east_money_stock_hk_api.get_hk_real_time_quotes(30, em_cookie)
         em_hk_info['_id'] = em_hk_info['symbol']
         em_hk_info['sync_time'] = str_now_date
+        em_hk_info = em_hk_info.fillna(0)
         mongodb_util.save_mongo(em_hk_info, extra_income_db_name.EM_HK_STOCK_INFO)
     except BaseException as e:
         logger.error("同步东方财富ETF信息异常:{}", e)
@@ -109,6 +113,7 @@ def sync_hk_ggt_stock_info():
         em_hk_gtt_info = east_money_stock_hk_gtt_api.get_ggt_real_time_quotes(em_cookie, 30, 6)
         em_hk_gtt_info['_id'] = em_hk_gtt_info['symbol']
         em_hk_gtt_info['sync_time'] = str_now_date
+        em_hk_gtt_info = em_hk_gtt_info.fillna(0)
         mongodb_util.save_mongo(em_hk_gtt_info, extra_income_db_name.EM_HK_GGT_STOCK_INFO)
     except BaseException as e:
         logger.error("同步东方财富ETF信息异常:{}", e)
@@ -120,14 +125,16 @@ def sync_us_stock_info():
     str_now_date = now_date.strftime('%Y-%m-%d %H:%M:%S')
     logger.error("同步东方财富美股信息")
     em_cookie = cookie_info_service.get_em_cookie()
-    us_stock_info = east_money_stock_us_api.get_us_stock_real_time_quotes(em_cookie, None)
+    us_stock_info = east_money_stock_us_api.get_us_real_time_quotes(30, em_cookie)
     us_stock_info['_id'] = us_stock_info['symbol']
     us_stock_info['sync_time'] = str_now_date
-    mongodb_util.save_mongo(us_stock_info, extra_income_db_name.EM_US_STOCK_INFO)
+
+    us_stock_info = us_stock_info.fillna(0)
+    mongodb_util.save_mongo(us_stock_info, extra_income_db_name.US_STOCK_INFO_EM)
 
 
 if __name__ == '__main__':
-    sync_stock_info()
+    sync_all_em_stock_info()
     # em_cookie = cookie_info_service.get_em_cookie()
     # em_us_stock_info = east_money_stock_us_api.get_us_stock_real_time_quotes(em_cookie, None)
     # em_us_stock_info['_id'] = em_us_stock_info['symbol']

{mns_scheduler-1.4.5.6 → mns_scheduler-1.4.7.4}/mns_scheduler/k_line/clean/k_line_info_clean_task.py

@@ -116,6 +116,7 @@ def multi_threaded_k_line_sync(str_day):
     # 将list_date列中的所有NaN值设置为19890604
     real_time_quotes_now['list_date'].fillna(19890604, inplace=True)
     real_time_quotes_now['list_date'] = real_time_quotes_now['list_date'].replace(99990909, 19890604)
+    real_time_quotes_now['list_date'] = real_time_quotes_now['list_date'].replace(0, 19890604)
 
     # 将日期数值转换为日期时间格式
     real_time_quotes_now['list_date_01'] = pd.to_datetime(real_time_quotes_now['list_date'], format='%Y%m%d')

mns_scheduler-1.4.7.4/mns_scheduler/kpl/theme/__init__.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)

mns_scheduler-1.4.7.4/mns_scheduler/kpl/theme/kpl_theme_sync_service.py

@@ -0,0 +1,256 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)
+
+import mns_common.api.kpl.theme.kpl_theme_api as kpl_theme_api
+from mns_common.db.MongodbUtil import MongodbUtil
+from datetime import datetime
+import pandas as pd
+import mns_common.constant.db_name_constant as db_name_constant
+from loguru import logger
+import mns_common.component.cookie.cookie_info_service as cookie_info_service
+import mns_common.utils.data_frame_util as data_frame_util
+import mns_common.api.msg.push_msg_api as push_msg_api
+
+mongodb_util = MongodbUtil('27017')
+
+
+# 获取最大id
+def query_max_theme_id():
+    kpl_theme_max_one_df = mongodb_util.descend_query({}, db_name_constant.KPL_THEME_LIST, '_id', 1)
+    if data_frame_util.is_empty(kpl_theme_max_one_df):
+        return 381
+    else:
+        return int(list(kpl_theme_max_one_df['theme_id'])[0])
+
+
+def sync_new_kpl_theme_info():
+    exist_max_theme_id = query_max_theme_id() + 1
+    max_sync_theme_id = exist_max_theme_id + 10
+
+    while exist_max_theme_id <= max_sync_theme_id:
+        try:
+            kpl_token = cookie_info_service.get_kpl_cookie()
+
+            json_data = kpl_theme_api.kpl_theme_index(exist_max_theme_id, kpl_token)
+            tables = json_data.get('Table')
+            if tables is not None and len(tables) > 0:
+                theme_name = json_data.get('Name')
+
+                title = '新增开盘啦题材:' + theme_name
+                push_msg_api.push_msg_to_wechat(title, title)
+
+                # 保存到主线列表
+                sync_kpl_theme_list(json_data, exist_max_theme_id)
+                # 保存到主线详细列表
+                sync_kpl_theme_detail(json_data, exist_max_theme_id)
+                exist_max_theme_id = exist_max_theme_id + 1
+        except BaseException as e:
+            logger.error("更新新题材信息异常:{},{}", str(exist_max_theme_id), e)
+
+
+def update_all_kpl_theme_info():
+    theme_id = 0
+    max_theme_id = query_max_theme_id() + 1
+    while theme_id < max_theme_id:
+        try:
+            kpl_token = cookie_info_service.get_kpl_cookie()
+            json_data = kpl_theme_api.kpl_theme_index(theme_id, kpl_token)
+            # 保存到主线列表
+            sync_kpl_theme_list(json_data, theme_id)
+            # 保存到主线详细列表
+            sync_kpl_theme_detail(json_data, theme_id)
+            theme_id = theme_id + 1
+        except BaseException as e:
+            logger.error("出现异常:{}", e)
+            theme_id = theme_id + 1
+
+
+def sync_kpl_theme_list(json_data, theme_id):
+    tables = json_data.get('Table')
+    if tables is None or len(tables) == 0:
+        return None
+
+    theme_name = json_data.get('Name')
+
+    stock_list_df = pd.DataFrame(json_data.get('StockList'))
+
+    stock_list_df.drop_duplicates('StockID', keep='last', inplace=True)
+
+    logger.info("同步题材:{},{}", str(theme_id), theme_name)
+    brief_intro = json_data.get('BriefIntro')
+    introduction = json_data.get('Introduction')
+    kpl_create_time = covert_time(json_data.get('CreateTime'))
+    kpl_update_time = covert_time(json_data.get('UpdateTime'))
+    now_date = datetime.now()
+    sync_time = now_date.strftime('%Y-%m-%d %H:%M:%S')
+
+    theme_dict = {
+        '_id': int(theme_id),
+        'theme_id': str(theme_id),
+        'theme_name': theme_name,
+        'brief_intro': brief_intro,
+        'introduction': introduction,
+        'kpl_create_time': kpl_create_time,
+        'kpl_update_time': kpl_update_time,
+        'theme_remark': '',
+        'sync_time': sync_time,
+        'grade': 1,
+        'stock_count': stock_list_df.shape[0],
+        'valid': True,
+        'theme_classify':2,
+    }
+
+    query = {'_id': str(theme_id)}
+
+    kpl_theme_one_df = mongodb_util.find_query_data(db_name_constant.KPL_THEME_LIST, query)
+
+    if data_frame_util.is_empty(kpl_theme_one_df):
+        ths_concept_one_df = pd.DataFrame(theme_dict, index=[1])
+        mongodb_util.save_mongo(ths_concept_one_df, db_name_constant.KPL_THEME_LIST)
+
+    else:
+        new_values = {'$set': {
+            'brief_intro': brief_intro,
+            'introduction': introduction,
+            'kpl_update_time': kpl_update_time,
+            'theme_name': theme_name}}
+
+        mongodb_util.update_many(query, new_values, db_name_constant.KPL_THEME_LIST)
+
+
+def sync_kpl_theme_detail(json_data, theme_id):
+    tables = json_data.get('Table')
+    theme_name = json_data.get('Name')
+    if tables is None or len(tables) == 0:
+        return None
+    for table in tables:
+        save_theme_l1(table, theme_name, theme_id)
+
+    return tables
+
+
+def save_theme_l1(table, theme_name, theme_id):
+    theme_l1 = table.get('Level1')
+
+    first_shelve_time = covert_time(theme_l1.get('FirstShelveTime'))
+    theme_l1_id = theme_l1.get('ID')
+    is_new = theme_l1.get('IsNew')
+    theme_l1_name = theme_l1.get('Name')
+    update_cache_time = covert_time(theme_l1.get('UpdateCacheTime'))
+    zs_code = theme_l1.get('ZSCode')
+    now_date = datetime.now()
+    sync_time = now_date.strftime('%Y-%m-%d %H:%M:%S')
+    theme_dict = {
+        '_id': str(theme_l1_id),
+        'theme_l1_id': str(theme_l1_id),
+        'theme_l1_name': str(theme_l1_name),
+        'theme_id': str(theme_id),
+        'theme_name': str(theme_name),
+        'first_shelve_time': first_shelve_time,
+        'update_cache_time': update_cache_time,
+        'zs_code': zs_code,
+        'is_new': is_new,
+        'sync_time': sync_time,
+    }
+
+    ths_concept_one_df = pd.DataFrame(theme_dict, index=[1])
+    mongodb_util.save_mongo(ths_concept_one_df, db_name_constant.KPL_THEME_LIST_L1)
+    stock_list = theme_l1.get('Stocks')
+    if len(stock_list) != 0:
+        save_theme_stocks(stock_list, theme_name, theme_id, theme_l1_name, theme_l1_id, '无', 0)
+    else:
+        theme_l2_list = table.get('Level2')
+        save_theme_l2(theme_l2_list, theme_name, theme_id, theme_l1_name, theme_l1_id)
+
+
+def save_theme_l2(theme_l2_list, theme_name, theme_id, theme_l1_name, theme_l1_id):
+    for theme_l2_one in theme_l2_list:
+
+        first_shelve_time = covert_time(theme_l2_one.get('FirstShelveTime'))
+        theme_l2_id = theme_l2_one.get('ID')
+        is_new = theme_l2_one.get('IsNew')
+        theme_l2_name = theme_l2_one.get('Name')
+        update_cache_time = covert_time(theme_l2_one.get('UpdateCacheTime'))
+        zs_code = theme_l2_one.get('ZSCode')
+        now_date = datetime.now()
+        sync_time = now_date.strftime('%Y-%m-%d %H:%M:%S')
+        theme_dict = {
+            '_id': str(theme_l2_id),
+            'theme_l2_id': str(theme_l2_id),
+            'theme_l2_name': str(theme_l2_name),
+
+            'theme_l1_id': str(theme_l1_id),
+            'theme_l1_name': str(theme_l1_name),
+            'theme_id': str(theme_id),
+            'theme_name': str(theme_name),
+            'first_shelve_time': first_shelve_time,
+            'update_cache_time': update_cache_time,
+            'zs_code': zs_code,
+            'is_new': is_new,
+            'sync_time': sync_time,
+        }
+
+        ths_concept_one_df = pd.DataFrame(theme_dict, index=[1])
+        mongodb_util.save_mongo(ths_concept_one_df, db_name_constant.KPL_THEME_LIST_L2)
+        stock_list = theme_l2_one.get('Stocks')
+        if len(stock_list) != 0:
+            save_theme_stocks(stock_list, theme_name, theme_id, theme_l1_name, theme_l1_id, theme_l2_name, theme_l2_id)
+
+
+def save_theme_stocks(stock_list, theme_name, theme_id, theme_l1_name, theme_l1_id, theme_l2_name, theme_l2_id):
+    stock_list_df = pd.DataFrame(stock_list)
+    stock_list_df = stock_list_df.rename(columns={
+        "StockID": "symbol",
+        "IsZz": "is_zz",
+        "IsHot": "is_hot",
+        "Reason": "reason",
+        "FirstShelveTime": "first_shelve_time",
+        "UpdateCacheTime": "update_cache_time",
+        "IsNew": "is_new",
+        "name": "prod_name",
+        "Hot": "hot",
+    })
+    stock_list_df["first_shelve_time"] = stock_list_df["first_shelve_time"].apply(covert_time)
+    stock_list_df["update_cache_time"] = stock_list_df["update_cache_time"].apply(covert_time)
+
+    stock_list_df['theme_id'] = str(theme_id)
+    stock_list_df['theme_name'] = theme_name
+
+    stock_list_df['theme_l1_id'] = str(theme_l1_id)
+    stock_list_df['theme_l1_name'] = theme_l1_name
+
+    stock_list_df['theme_l2_name'] = theme_l2_name
+    stock_list_df['theme_l2_id'] = str(theme_l2_id)
+
+    stock_list_df['_id'] = (stock_list_df['symbol']
+                            + '_'
+                            + stock_list_df['theme_id']
+                            + '_'
+                            + stock_list_df['theme_l1_id']
+                            + '_'
+                            + stock_list_df['theme_l2_id'])
+    now_date = datetime.now()
+    sync_time = now_date.strftime('%Y-%m-%d %H:%M:%S')
+    stock_list_df['sync_time'] = sync_time
+    mongodb_util.save_mongo(stock_list_df, db_name_constant.KPL_THEME_DETAILS)
+
+
+def covert_time(timestamp_str):
+    timestamp = int(timestamp_str)
+    if timestamp == 0:
+        return "1989-06-04"
+    else:
+        # 3. 转换为本地时区的datetime对象
+        dt_local = datetime.fromtimestamp(timestamp)
+        # 4. 格式化为「年月日 时分秒」的字符串（格式可自定义）
+        datetime_str = dt_local.strftime('%Y-%m-%d %H:%M:%S')
+        return datetime_str
+
+
+if __name__ == '__main__':
+    update_all_kpl_theme_info()

{mns_scheduler-1.4.5.6 → mns_scheduler-1.4.7.4}/mns_scheduler/self_choose/ths_self_choose_service.py

@@ -137,14 +137,30 @@ def add_continue_boards_zt_stocks(ths_cookie):
 def add_main_line_leader(ths_cookie):
     main_line_leader_df = mongodb_util.find_query_data('main_line_leader', {})
     if data_frame_util.is_not_empty(main_line_leader_df):
+        main_line_leader_df = count_main_line_number(main_line_leader_df)
+
         main_line_leader_df = main_line_leader_df.sort_values(
-            by=['main_line_number', 'main_line_leader_grade', 'chg'],
-            ascending=[False, False, False]
+            by=['main_line_number', 'main_line_leader_grade'],
+            ascending=[False, False,]
         )
         for stock_one in main_line_leader_df.itertuples():
             ths_self_choose_api.add_stock_to_account(stock_one.symbol, ths_cookie)
 
 
+# 统计主线的数量
+def count_main_line_number(main_line_detail_df):
+    # 统计主线的数量
+    main_line_detail_df['main_line_number'] = main_line_detail_df.groupby('main_line')['main_line'].transform(
+        'count')
+    # 相同主线数量 数量加1
+    main_line_detail_df['main_line_number'] = (
+                                                      main_line_detail_df.groupby('main_line_number')['main_line']
+                                                      .transform('rank', method='dense') - 1
+                                              ) * 0.1 + main_line_detail_df['main_line_number']
+
+    return main_line_detail_df
+
+
 # 自选股操作 删除当天自选股 增加新的连板股票  添加固定选择自选
 def self_choose_stock_handle():
     ths_cookie = cookie_info_service.get_ths_cookie()

{mns_scheduler-1.4.5.6 → mns_scheduler-1.4.7.4}/mns_scheduler/zt/script/sync_now_higt_chg_zt.py

@@ -9,7 +9,7 @@ import mns_common.component.em.em_stock_info_api as em_stock_info_api
 import mns_common.component.common_service_fun_api as common_service_fun_api
 import mns_scheduler.k_line.month_week_daily.daily_week_month_line_sync as daily_week_month_line_sync
 from datetime import datetime
-import mns_scheduler.zt.zt_pool.em_zt_pool_sync_api as em_zt_pool_sync_api
+import mns_scheduler.zt.zt_pool.em.em_zt_pool_sync_api as em_zt_pool_sync_api
 import mns_scheduler.zt.high_chg.sync_high_chg_pool_service as sync_high_chg_pool_service
 import mns_scheduler.zt.open_data.kcx_high_chg_open_data_sync as kcx_high_chg_open_data_sync
 import mns_scheduler.zt.high_chg.sync_high_chg_real_time_quotes_service as sync_high_chg_real_time_quotes_service

mns_scheduler-1.4.7.4/mns_scheduler/zt/zt_pool/em/__init__.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)