mns-scheduler 1.1.0.9__tar.gz → 1.1.1.1__tar.gz

{mns-scheduler-1.1.0.9 → mns-scheduler-1.1.1.1}/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.1
 Name: mns-scheduler
-Version: 1.1.0.9
+Version: 1.1.1.1

{mns-scheduler-1.1.0.9 → mns-scheduler-1.1.1.1}/mns_scheduler/k_line/clean/k_line_info_clean_task.py

@@ -70,7 +70,7 @@ def sync_k_line_info(str_day, symbol_list):
                     k_line_result = k_line_info_clean_impl.calculate_k_line_info(str_day, symbol, diff_days)
                     save_k_line_data(symbol, str_day, k_line_result)
                     if result_k_line_list_df is None:
-                        result_k_line_list_df = result
+                        result_k_line_list_df = k_line_result
                     else:
                         result_k_line_list_df = pd.concat([result_k_line_list_df, k_line_result])
 

mns-scheduler-1.1.1.1/mns_scheduler/k_line/sync/bfq_k_line_sync.py

@@ -0,0 +1,92 @@
+import os
+import sys
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)
+
+from loguru import logger
+import mns_common.api.em.east_money_stock_api as east_money_stock_api
+from mns_common.db.MongodbUtil import MongodbUtil
+import mns_common.utils.date_handle_util as date_handle_util
+import mns_common.component.common_service_fun_api as common_service_fun_api
+import mns_common.api.akshare.k_line_api as k_line_api
+import mns_common.component.company.company_common_service_new_api as company_common_service_new_api
+
+mongodb_util = MongodbUtil('27017')
+
+
+def sync_bfq_k_line_data(period='daily',
+                         hq='hfq',
+                         hq_col='stock_hfq_daily',
+                         end_date='22220101',
+                         symbol=None):
+    stock_hfq_df = k_line_api.stock_zh_a_hist(symbol=symbol, period=period,
+                                              start_date=date_handle_util.no_slash_date('1990-12-19'),
+                                              end_date=date_handle_util.no_slash_date(end_date),
+                                              adjust=hq)
+
+    stock_hfq_df.rename(columns={"日期": "date", "开盘": "open",
+                                 "收盘": "close", "最高": "high",
+                                 "最低": "low", "成交量": "volume",
+                                 "成交额": "amount", "振幅": "pct_chg",
+                                 "涨跌幅": "chg", "涨跌额": "change",
+                                 "换手率": "exchange"}, inplace=True)
+
+    stock_hfq_df['symbol'] = symbol
+    stock_hfq_df['_id'] = stock_hfq_df['symbol'] + '-' + stock_hfq_df['date']
+    stock_hfq_df['last_price'] = round(((stock_hfq_df['close']) / (1 + stock_hfq_df['chg'] / 100)), 2)
+    stock_hfq_df['max_chg'] = round(
+        ((stock_hfq_df['high'] - stock_hfq_df['last_price']) / stock_hfq_df['last_price']) * 100, 2)
+    stock_hfq_df['amount_level'] = round((stock_hfq_df['amount'] / common_service_fun_api.HUNDRED_MILLION), 2)
+    stock_hfq_df['flow_mv'] = round(stock_hfq_df['amount'] * 100 / stock_hfq_df['exchange'], 2)
+    stock_hfq_df['flow_mv_sp'] = round(stock_hfq_df['flow_mv'] / common_service_fun_api.HUNDRED_MILLION, 2)
+
+    classification = common_service_fun_api.classify_symbol_one(symbol)
+    stock_hfq_df['classification'] = classification
+    stock_hfq_df = stock_hfq_df.sort_values(by=['date'], ascending=False)
+    insert_data(stock_hfq_df, hq_col, symbol)
+    logger.info(period + 'k线同步-' + hq + '-' + symbol)
+    return stock_hfq_df
+
+
+def sync_all_bfq_k_line(period='daily',
+                        hq='hfq',
+                        hq_col='stock_hfq_daily',
+                        end_date='22220101',
+                        symbol=None):
+    real_time_quotes_now_es = east_money_stock_api.get_real_time_quotes_all_stocks()
+
+    symbol_list = list(real_time_quotes_now_es['symbol'])
+    # 退市公司
+    de_list_company = company_common_service_new_api.get_de_list_company()
+    symbol_list.extend(de_list_company)
+    symbol_list = set(symbol_list)
+    if symbol is not None:
+        symbol_list = [symbol]
+    for symbol in symbol_list:
+        try:
+            sync_bfq_k_line_data(period,
+                                 hq,
+                                 hq_col,
+                                 end_date,
+                                 symbol)
+        except BaseException as e:
+            logger.warning("同步不复权k线:{},{}", symbol, e)
+
+
+def insert_data(stock_hfq_df, hq_col, symbol):
+    query = {'symbol': symbol}
+    tag = mongodb_util.remove_data(query, hq_col)
+    success = tag.acknowledged
+    if success:
+        mongodb_util.insert_mongo(stock_hfq_df, hq_col)
+
+
+if __name__ == '__main__':
+    sync_all_bfq_k_line('daily',
+                        '',
+                        'stock_bfq_daily',
+                        None,
+                        None)

{mns-scheduler-1.1.0.9 → mns-scheduler-1.1.1.1}/mns_scheduler/zt/script/kcx_high_chg_open_his_data_handle.py

@@ -52,14 +52,6 @@ realtime_quotes_now_zt_new_kc_open_field = ['_id',
                                             'max_real_main_inflow_multiple', 'list_date',
                                             'real_super_main_inflow_multiple', 'real_flow_mv',
                                             'real_disk_diff_amount_exchange', 'no_open_data']
-# kcx 高涨幅开盘数据同步
-def realtime_quotes_now_zt_new_kc_open_sync():
-    realtime_quotes_now_zt_new_kc_open = mongodb_util.find_all_data('realtime_quotes_now_zt_new_kc_open')
-    realtime_quotes_now_zt_new_kc_open = realtime_quotes_now_zt_new_kc_open[realtime_quotes_now_zt_new_kc_open_field]
-    realtime_quotes_now_zt_new_kc_open = realtime_quotes_now_zt_new_kc_open.sort_values(by=['str_day'], ascending=False)
-
-    mongodb_util.insert_mongo(realtime_quotes_now_zt_new_kc_open, 'realtime_quotes_now_zt_new_kc_open_copy')
-
 
 #     query = {'$and': [{"_id": {'$lte': str_end}}, {"_id": {'$gte': '2022-04-25'}}]}
 def sync_all_high_chg_data(str_end):
@@ -92,7 +84,7 @@ def fix_miss_data(str_end):
 
 
 if __name__ == '__main__':
-    kcx_high_chg_open_data_sync.sync_all_kc_zt_data('2024-06-04', None)
+    kcx_high_chg_open_data_sync.sync_all_kc_zt_data('2024-06-19', None)
     # sync_all_kc_zt_data('2023-08-16')
     # sync_all_kc_zt_data('2023-07-07')
     # realtime_quotes_now_zt_new_kc_open_sync()

{mns-scheduler-1.1.0.9 → mns-scheduler-1.1.1.1}/mns_scheduler/zt/script/sync_high_chg_pool_his_data.py

@@ -36,4 +36,4 @@ def sync_his_zt_pool_data(begin_day, end_day):
 
 
 if __name__ == '__main__':
-    sync_high_chg_pool_service.sync_stock_high_chg_pool_list('2024-06-07', ['605058'])
+    sync_high_chg_pool_service.sync_stock_high_chg_pool_list('2024-06-19', None)

mns-scheduler-1.1.0.9/mns_scheduler/zt/zt_pool/zt_pool_sync_api.py → mns-scheduler-1.1.1.1/mns_scheduler/zt/zt_pool/em_zt_pool_sync_api.py

@@ -17,6 +17,10 @@ import mns_common.utils.data_frame_util as data_frame_util
 import mns_common.component.common_service_fun_api as common_service_fun_api
 import mns_common.component.trade_date.trade_date_common_service_api as trade_date_common_service_api
 
+'''
+东方财富涨停池
+'''
+
 mongodb_util = MongodbUtil('27017')
 
 ZT_FIELD = ['_id', 'symbol', 'name', 'now_price', 'chg', 'first_closure_time',

mns-scheduler-1.1.1.1/mns_scheduler/zt/zt_pool/ths_zt_pool_sync_api.py

@@ -0,0 +1,231 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)
+import mns_common.api.ths.zt.ths_stock_zt_pool_api as ths_stock_zt_pool_api
+from datetime import datetime
+import mns_common.utils.data_frame_util as data_frame_util
+import pandas as pd
+import mns_common.api.em.east_money_stock_api as east_money_stock_api
+import mns_common.component.company.company_common_service_new_api as company_common_service_new_api
+import mns_common.component.common_service_fun_api as common_service_fun_api
+from loguru import logger
+import mns_common.api.akshare.k_line_api as k_line_api
+import mns_common.utils.date_handle_util as date_handle_util
+from mns_common.db.MongodbUtil import MongodbUtil
+import mns_common.constant.db_name_constant as db_name_constant
+
+mongodb_util = MongodbUtil('27017')
+
+
+def ths_zt_pool(str_day, real_time_quotes_all_stocks):
+    '''
+    获取请求头
+    :param str_day: 日期
+    :param real_time_quotes_all_stocks: 实时行情
+    :return:
+    '''
+    now_date = datetime.now()
+    now_day_str_day = now_date.strftime('%Y-%m-%d')
+    ths_zt_pool_df = ths_stock_zt_pool_api.get_zt_reason(str_day)
+    if data_frame_util.is_empty(ths_zt_pool_df):
+        return pd.DataFrame()
+    if str_day == now_day_str_day:
+        ths_zt_pool_df = merge_now_day_zt_info(ths_zt_pool_df, real_time_quotes_all_stocks)
+    else:
+        ths_zt_pool_df = merge_his_day_zt_info(ths_zt_pool_df, str_day)
+
+    return ths_zt_pool_df
+
+
+def merge_now_day_zt_info(ths_zt_pool_df, real_time_quotes_all_stocks):
+    if real_time_quotes_all_stocks is None:
+        real_time_quotes_all_stocks = east_money_stock_api.get_real_time_quotes_all_stocks()
+    real_time_quotes_all_stocks = real_time_quotes_all_stocks[['symbol', 'amount',
+                                                               'exchange', 'flow_mv', 'total_mv']]
+    real_time_quotes_all_stocks_zt = real_time_quotes_all_stocks.loc[
+        real_time_quotes_all_stocks['symbol'].isin(ths_zt_pool_df['symbol'])]
+
+    query_field = {
+        "ths_concept_name": 1,
+        "ths_concept_code": 1,
+        "ths_concept_sync_day": 1,
+        "company_type": 1,
+        "concept_create_day": 1,
+        "first_sw_industry": 1,
+        "third_sw_industry": 1,
+        "industry": 1,
+        "list_date": 1,
+    }
+    query_field_key = str(query_field)
+    query_key = str({'symbol': {"$in": list(ths_zt_pool_df['symbol'])}})
+    company_df_zt = company_common_service_new_api.get_company_info_by_field(query_key, query_field_key)
+
+    company_df_zt = company_df_zt.set_index(['_id'], drop=True)
+    real_time_quotes_all_stocks_zt = real_time_quotes_all_stocks_zt.set_index(['symbol'], drop=True)
+    ths_zt_pool_df = ths_zt_pool_df.set_index(['symbol'], drop=False)
+
+    ths_zt_pool_df = pd.merge(ths_zt_pool_df, company_df_zt, how='outer',
+                              left_index=True, right_index=True)
+
+    ths_zt_pool_df = pd.merge(ths_zt_pool_df, real_time_quotes_all_stocks_zt, how='outer',
+                              left_index=True, right_index=True)
+    ths_zt_pool_df['amount_level'] = round(ths_zt_pool_df['amount'] / common_service_fun_api.HUNDRED_MILLION, 2)
+    ths_zt_pool_df = common_service_fun_api.classify_symbol(ths_zt_pool_df)
+    ths_zt_pool_df = common_service_fun_api.total_mv_classification(ths_zt_pool_df)
+    ths_zt_pool_df.fillna('', inplace=True)
+    if 'zt_flag' in ths_zt_pool_df.columns:
+        del ths_zt_pool_df['zt_flag']
+    if 'zt_tag' in ths_zt_pool_df.columns:
+        del ths_zt_pool_df['zt_tag']
+
+    return ths_zt_pool_df
+
+
+# 历史数据merge
+def merge_his_day_zt_info(ths_zt_pool_df, str_day):
+    '''
+       获取请求头
+       :param ths_zt_pool_df: 涨停df
+       :param str_day: 日期
+       :return:
+       '''
+
+    query_field = {
+        "ths_concept_name": 1,
+        "ths_concept_code": 1,
+        "ths_concept_sync_day": 1,
+        "company_type": 1,
+        "concept_create_day": 1,
+        "first_sw_industry": 1,
+        "third_sw_industry": 1,
+        "industry": 1,
+        "list_date": 1,
+    }
+    query_field_key = str(query_field)
+    query_key = str({'symbol': {"$in": list(ths_zt_pool_df['symbol'])}})
+    company_df_zt = company_common_service_new_api.get_company_info_by_field(query_key, query_field_key)
+
+    bfq_k_line_df = get_bfq_daily_line(ths_zt_pool_df, str_day)
+    bfq_k_line_df['total_mv'] = bfq_k_line_df['flow_mv']
+
+    company_df_zt = company_df_zt.set_index(['_id'], drop=True)
+    bfq_k_line_df = bfq_k_line_df.set_index(['symbol'], drop=True)
+    ths_zt_pool_df = ths_zt_pool_df.set_index(['symbol'], drop=False)
+
+    if 'chg' in ths_zt_pool_df.columns:
+        del ths_zt_pool_df['chg']
+    if 'now_price' in ths_zt_pool_df.columns:
+        del ths_zt_pool_df['now_price']
+
+    ths_zt_pool_df = pd.merge(ths_zt_pool_df, company_df_zt, how='outer',
+                              left_index=True, right_index=True)
+
+    ths_zt_pool_df = pd.merge(ths_zt_pool_df, bfq_k_line_df, how='outer',
+                              left_index=True, right_index=True)
+    ths_zt_pool_df = common_service_fun_api.classify_symbol(ths_zt_pool_df)
+    ths_zt_pool_df = common_service_fun_api.total_mv_classification(ths_zt_pool_df)
+    ths_zt_pool_df.fillna('', inplace=True)
+    if 'zt_flag' in ths_zt_pool_df.columns:
+        del ths_zt_pool_df['zt_flag']
+    if 'zt_tag' in ths_zt_pool_df.columns:
+        del ths_zt_pool_df['zt_tag']
+
+    return ths_zt_pool_df
+
+
+# 获取不复权k线信息
+def get_bfq_daily_line(ths_zt_pool_df, str_day):
+    query_k_line = {'symbol': {"$in": list(ths_zt_pool_df['symbol'])}, 'date': date_handle_util.no_slash_date(str_day)}
+    bfq_daily_line_df = mongodb_util.find_query_data('stock_bfq_daily', query_k_line)
+    if data_frame_util.is_not_empty(bfq_daily_line_df):
+        bfq_daily_line_df = bfq_daily_line_df[['amount', 'chg', 'close', 'exchange',
+                                               'symbol', 'amount_level',
+                                               'flow_mv', 'flow_mv_sp'
+                                               ]]
+        bfq_daily_line_df = bfq_daily_line_df.rename(columns={"close": 'now_price'})
+        return bfq_daily_line_df
+
+    bfq_k_line_df = pd.DataFrame()
+    for zt_one in ths_zt_pool_df.itertuples():
+        try:
+            symbol = zt_one.symbol
+            bfq_daily_line_df = k_line_api.stock_zh_a_hist(symbol,
+                                                           'daily',
+                                                           date_handle_util.no_slash_date(str_day),
+                                                           date_handle_util.no_slash_date(str_day),
+                                                           '')
+            if data_frame_util.is_empty(bfq_daily_line_df):
+                continue
+            bfq_daily_line_df = bfq_daily_line_df[['amount', 'chg', 'close', 'exchange',
+                                                   'symbol', 'amount_level',
+                                                   'flow_mv', 'flow_mv_sp'
+                                                   ]]
+            bfq_daily_line_df = bfq_daily_line_df.rename(columns={"close": 'now_price'})
+            bfq_k_line_df = pd.concat([bfq_k_line_df, bfq_daily_line_df])
+        except BaseException as e:
+            logger.warning("同步不复权k线异常:{},{}", symbol, e)
+
+    return bfq_k_line_df
+
+
+def save_ths_zt_pool(ths_zt_pool_df):
+    ths_zt_pool_df = ths_zt_pool_df[[
+        "symbol",
+        "name",
+        "chg",
+        "connected_boards_numbers",
+        "statistics",
+        "statistics_detail",
+        "first_closure_time",
+        "last_closure_time",
+        "zt_detail",
+        "zt_reason",
+        "closure_volume",
+        "closure_funds",
+        "closure_funds_per_amount",
+        "closure_funds_per_flow_mv",
+        "frying_plates_numbers",
+        "zt_type",
+        "market_code",
+        "str_day",
+        "industry",
+        "first_sw_industry",
+        "third_sw_industry",
+        "ths_concept_name",
+        "ths_concept_code",
+        "ths_concept_sync_day",
+        "list_date",
+        "company_type",
+        "amount",
+        "now_price",
+        "exchange",
+        "amount_level",
+        "flow_mv",
+        "flow_mv_sp",
+        "total_mv",
+        "classification",
+        "total_mv_sp",
+        "flow_mv_level"
+    ]]
+    ths_zt_pool_df['_id'] = ths_zt_pool_df['symbol'] + '_' + ths_zt_pool_df['str_day']
+    ths_zt_pool_df = ths_zt_pool_df.sort_values(by=['connected_boards_numbers'], ascending=False)
+    mongodb_util.save_mongo(ths_zt_pool_df, db_name_constant.THS_ZT_POOL)
+
+
+if __name__ == '__main__':
+    while True:
+        query = {'_id': {"$lte": '2019-10-29'}}
+        trade_date_list = mongodb_util.find_query_data('trade_date_list', query)
+        trade_date_list = trade_date_list.sort_values(by=['trade_date'], ascending=False)
+        for trade_one in trade_date_list.itertuples():
+            try:
+                trade_date = trade_one.trade_date
+                zt_df = ths_zt_pool(trade_date, None)
+                save_ths_zt_pool(zt_df)
+                logger.info("同步到日期:{}", trade_date)
+            except BaseException as e:
+                logger.warning("同步涨停到日期:{},{}", trade_one.trade_date, e)

{mns-scheduler-1.1.0.9 → mns-scheduler-1.1.1.1}/mns_scheduler/zz_task/data_sync_task.py

@@ -23,7 +23,7 @@ import mns_scheduler.big_deal.ths_big_deal_sync as ths_big_deal_sync_api
 import mns_scheduler.zt.open_data.kcx_high_chg_open_data_sync as kcx_high_chg_open_data_sync
 import mns_scheduler.zt.export.export_kcx_high_chg_open_data_to_excel as export_kcx_high_chg_open_data_to_excel
 import mns_scheduler.zt.connected_boards.zt_five_boards_sync_api as zt_five_boards_sync_api
-import mns_scheduler.zt.zt_pool.zt_pool_sync_api as zt_pool_sync_api
+import mns_scheduler.zt.zt_pool.em_zt_pool_sync_api as em_zt_pool_sync_api
 import mns_scheduler.k_line.clean.k_line_info_clean_task as k_line_info_clean_service
 import mns_scheduler.concept.clean.ths_concept_clean_api as ths_concept_choose_api
 import mns_common.api.em.east_money_stock_gdfx_free_top_10_api as east_money_stock_gdfx_free_top_10_api
@@ -47,6 +47,7 @@ import mns_scheduler.risk.self.wei_pan_stock_api as wei_pan_stock_api
 import mns_scheduler.risk.transactions.transactions_check_api as transactions_check_api
 import mns_scheduler.concept.ths.sync_new_index.sync_ths_concept_new_index_api as sync_ths_concept_new_index_api
 import mns_scheduler.company_info.clean.company_info_clean_api as company_info_clean_api
+import mns_scheduler.zt.zt_pool.ths_zt_pool_sync_api as ths_zt_pool_sync_api
 
 
 # 同步交易日期任务完成
@@ -148,8 +149,10 @@ def sync_stock_zt_pool():
     str_day = now_date.strftime('%Y-%m-%d')
     if trade_date_common_service_api.is_trade_day(str_day):
         logger.info('同步当天涨停池股开始')
-        stock_zt_pool = zt_pool_sync_api.save_zt_info(str_day)
-        zt_five_boards_sync_api.update_five_connected_boards_task(stock_zt_pool)
+        em_stock_zt_pool = em_zt_pool_sync_api.save_zt_info(str_day)
+        zt_five_boards_sync_api.update_five_connected_boards_task(em_stock_zt_pool)
+        logger.info('同步ths股票涨停池')
+        ths_zt_pool_sync_api.ths_zt_pool(str_day, None)
         logger.info('同步当天涨停池股票完成')
 
 

{mns-scheduler-1.1.0.9 → mns-scheduler-1.1.1.1}/mns_scheduler.egg-info/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.1
 Name: mns-scheduler
-Version: 1.1.0.9
+Version: 1.1.1.1

{mns-scheduler-1.1.0.9 → mns-scheduler-1.1.1.1}/mns_scheduler.egg-info/SOURCES.txt

@@ -70,6 +70,7 @@ mns_scheduler/k_line/clean/week_month/sub_new_week_month_k_line_service.py
 mns_scheduler/k_line/hot_stocks/__init__.py
 mns_scheduler/k_line/hot_stocks/recent_hot_stocks_clean_service.py
 mns_scheduler/k_line/sync/__init__.py
+mns_scheduler/k_line/sync/bfq_k_line_sync.py
 mns_scheduler/k_line/sync/daily_week_month_line_sync.py
 mns_scheduler/k_line/test/__init__.py
 mns_scheduler/k_line/test/k_line_info_clean_his_data.py
@@ -126,7 +127,8 @@ mns_scheduler/zt/script/__init__.py
 mns_scheduler/zt/script/kcx_high_chg_open_his_data_handle.py
 mns_scheduler/zt/script/sync_high_chg_pool_his_data.py
 mns_scheduler/zt/zt_pool/__init__.py
-mns_scheduler/zt/zt_pool/zt_pool_sync_api.py
+mns_scheduler/zt/zt_pool/em_zt_pool_sync_api.py
+mns_scheduler/zt/zt_pool/ths_zt_pool_sync_api.py
 mns_scheduler/zz_task/__init__.py
 mns_scheduler/zz_task/data_sync_task.py
 mns_scheduler/zz_task/sync_realtime_quotes_task.py

{mns-scheduler-1.1.0.9 → mns-scheduler-1.1.1.1}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='mns-scheduler',
-    version='1.1.0.9',
+    version='1.1.1.1',
     packages=find_packages(),
     install_requires=[],  # 如果有依赖项，可以在这里列出
 )