mns-scheduler 1.3.1.2__py3-none-any.whl → 1.3.1.6__py3-none-any.whl

mns_scheduler/baidu/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/baidu/baidu_yun_pan_handle_service.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import sys
@@ -11,7 +11,7 @@ import json
 import pandas as pd
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 from bypy import ByPy

mns_scheduler/company_info/announce/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/company_info/announce/company_announce_sync_service.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import mns_common.component.company.company_common_service_new_api as company_common_service_new_api

mns_scheduler/company_info/base/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/company_info/base/sync_company_hold_info_api.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import mns_common.api.ths.company.ths_company_info_api as ths_company_info_api

mns_scheduler/company_info/clean/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/company_info/clean/company_info_clean_api.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 from mns_common.db.MongodbUtil import MongodbUtil

mns_scheduler/company_info/constant/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/company_info/de_list_stock/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/company_info/de_list_stock/de_list_stock_service.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import pandas as pd

mns_scheduler/company_info/em_stock_info/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/company_info/em_stock_info/sync_em_stock_info_sync.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import datetime
@@ -13,10 +13,11 @@ import mns_common.api.em.real_time.east_money_stock_a_v2_api as east_money_stock
 import mns_common.api.em.real_time.east_money_debt_api as east_money_debt_api
 import mns_common.api.em.real_time.east_money_etf_api as east_money_etf_api
 import mns_common.api.em.real_time.east_money_stock_hk_api as east_money_stock_hk_api
-import mns_common.api.em.real_time.east_money_stock_us_api as east_money_stock_us_api
 import mns_common.constant.db_name_constant as db_name_constant
 import mns_common.component.cookie.cookie_info_service as cookie_info_service
 
+import mns_common.api.em.real_time.east_money_stock_us_api as east_money_stock_us_api
+
 mongodb_util = MongodbUtil('27017')
 
 
@@ -57,6 +58,7 @@ def sync_all_em_stock_info():
         logger.error("同步东方财富港股信息异常:{}", e)
 
     try:
+        em_cookie = cookie_info_service.get_em_cookie()
         em_us_stock_info = east_money_stock_us_api.get_us_stock_real_time_quotes(em_cookie, proxies)
         em_us_stock_info['_id'] = em_us_stock_info['symbol']
         mongodb_util.save_mongo(em_us_stock_info, db_name_constant.EM_US_STOCK_INFO)
@@ -66,4 +68,7 @@ def sync_all_em_stock_info():
 
 
 if __name__ == '__main__':
-    sync_all_em_stock_info()
+    em_cookie = cookie_info_service.get_em_cookie()
+    em_us_stock_info = east_money_stock_us_api.get_us_stock_real_time_quotes(em_cookie, None)
+    em_us_stock_info['_id'] = em_us_stock_info['symbol']
+    mongodb_util.save_mongo(em_us_stock_info, db_name_constant.EM_US_STOCK_INFO)

mns_scheduler/company_info/remark/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/company_info/remark/company_remark_info_sync.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import mns_common.api.ths.concept.web.ths_company_info_web as ths_company_info_web

mns_scheduler/concept/clean/kpl_concept_clean_api.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 from mns_common.db.MongodbUtil import MongodbUtil

mns_scheduler/db/script/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/db/script/col_move_script.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import pymongo

mns_scheduler/db/script/db_move/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/db/script/db_move/col_move_one_service.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 from loguru import logger

mns_scheduler/db/script/sync/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/db/script/sync/remote_data_sync_to_local.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import mns_common.utils.data_frame_util as data_frame_util

mns_scheduler/db/script/sync/sync_hui_ce_test_data.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 

mns_scheduler/db/script/sync/sync_hui_ce_test_data_01.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 

mns_scheduler/db/script/update/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/db/script/update/update_col_field.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 from mns_common.db.MongodbUtil import MongodbUtil

mns_scheduler/debt/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/debt/kzz_bond_info_sync.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import mns_common.api.em.real_time.east_money_debt_api as east_money_debt_api

mns_scheduler/extraIncome/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/extraIncome/a_stock/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/extraIncome/a_stock/one_minute/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/extraIncome/a_stock/one_minute/common/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/extraIncome/a_stock/one_minute/common/symbol_handle_util.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import mns_common.constant.extra_income_db_name as extra_income_db_name

mns_scheduler/extraIncome/a_stock/one_minute/etf/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/extraIncome/a_stock/one_minute/etf/etf_one_minute_sync_task.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 

mns_scheduler/extraIncome/a_stock/one_minute/index/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/extraIncome/a_stock/one_minute/index/main_index_sync_task.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 

mns_scheduler/extraIncome/a_stock/one_minute/kzz/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/extraIncome/a_stock/one_minute/kzz/kzz_one_minute_sync_task.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 

mns_scheduler/extraIncome/a_stock/one_minute/one_minute_sync_task.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import mns_scheduler.extraIncome.a_stock.one_minute.index.main_index_sync_task as main_index_sync_task

mns_scheduler/extraIncome/a_stock/one_minute/stock/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/extraIncome/a_stock/one_minute/stock/stock_one_minute_sync_task.py

@@ -2,7 +2,7 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import mns_common.component.common_service_fun_api as common_service_fun_api

mns_scheduler/extraIncome/a_stock/one_minute/upload/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/extraIncome/a_stock/one_minute/upload/etf_upload_to_baidu_task.py

@@ -0,0 +1,95 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)
+from mns_common.db.MongodbUtil import MongodbUtil
+from loguru import logger
+import mns_scheduler.baidu.baidu_yun_pan_handle_service as baidu_yun_pan_handle_service
+from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
+import mns_common.utils.data_frame_util as data_frame_util
+from datetime import datetime
+import mns_common.constant.extra_income_db_name as extra_income_db_name
+import mns_common.component.em.em_stock_info_api as em_stock_info_api
+
+mongodb_util = MongodbUtil('27017')
+mongodbUtilV2_27019 = MongodbUtilV2('27019', extra_income_db_name.EXTRA_INCOME)
+
+
+def upload_etf_to_baidu():
+    a_stock_path = '/A股/ETF/'
+
+    now_date_time = datetime.now()
+    now_year = now_date_time.year
+    month = now_date_time.month
+    a_stock_path = a_stock_path + str(now_year) + '/' + str(month)
+    # 创建路径
+    baidu_yun_pan_handle_service.mkdir_baidu_new_folder(a_stock_path)
+    em_a_etf_info_df = em_stock_info_api.get_etf_info()
+
+    em_a_etf_info_df = classify_symbol(em_a_etf_info_df)
+
+    em_a_etf_info_df['symbol'] = em_a_etf_info_df.apply(
+        lambda row: row['symbol'] + '.SZ' if row['classification'] in ['S', 'C']
+        else row['symbol'] + '.BJ' if row['classification'] in ['X']
+        else row['symbol'] + '.SH',
+        axis=1
+    )
+
+    file_folder_df = baidu_yun_pan_handle_service.get_file_folder(a_stock_path)
+    if data_frame_util.is_not_empty(file_folder_df):
+        # 去除文件名中的 .csv 后缀
+        file_folder_df['name'] = file_folder_df['name'].str.replace(r'\.csv$', '', regex=True)
+        em_a_etf_info_df = em_a_etf_info_df.loc[~(em_a_etf_info_df['symbol'].isin(file_folder_df['name']))]
+    fail_list = []
+    for stock_one in em_a_etf_info_df.itertuples():
+        symbol = stock_one.symbol
+        name = stock_one.name
+        try:
+
+            col_name = extra_income_db_name.ONE_MINUTE_K_LINE_BFQ_ETF + '_' + str(now_year)
+
+            if month < 10:
+                month_str = '0' + str(month)
+            else:
+                month_str = str(month)
+            begin_time = str(now_year) + '-' + month_str + '-01 09:00:00'
+            query = {'symbol': symbol, 'time': {"$gte": begin_time}}
+            one_minute_k_line_bfq_df = mongodbUtilV2_27019.find_query_data(col_name, query)
+            if data_frame_util.is_not_empty(one_minute_k_line_bfq_df):
+                one_minute_k_line_bfq_df = one_minute_k_line_bfq_df.sort_values(by=['time'], ascending=True)
+                del one_minute_k_line_bfq_df['_id']
+                del one_minute_k_line_bfq_df['symbol']
+                result = baidu_yun_pan_handle_service.upload_to_baidu(symbol, a_stock_path, one_minute_k_line_bfq_df)
+                if result != 0:
+                    fail_list.append(symbol)
+        except BaseException as e:
+            fail_list.append(symbol)
+            logger.error("上传数据异常:{}", e)
+
+
+def classify_symbol(etf_real_time_quotes):
+    etf_real_time_quotes['classification'] = etf_real_time_quotes['market'].apply(
+        lambda market: classify_symbol_one(market))
+    return etf_real_time_quotes
+
+
+# 单个股票分类
+def classify_symbol_one(market):
+    if market == 0:
+        return 'S'
+    else:
+        return 'H'
+
+
+if __name__ == '__main__':
+    upload_etf_to_baidu()

mns_scheduler/extraIncome/a_stock/one_minute/upload/index_upload_to_baidu_task.py

@@ -0,0 +1,87 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)
+
+from mns_common.db.MongodbUtil import MongodbUtil
+from loguru import logger
+import mns_scheduler.baidu.baidu_yun_pan_handle_service as baidu_yun_pan_handle_service
+from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
+import mns_common.utils.data_frame_util as data_frame_util
+from datetime import datetime
+import mns_common.constant.extra_income_db_name as extra_income_db_name
+
+mongodb_util = MongodbUtil('27017')
+mongodbUtilV2_27019 = MongodbUtilV2('27019', extra_income_db_name.EXTRA_INCOME)
+
+main_index_list = [
+    "000001.SH",
+    "000016.SH",
+    "000010.SH",
+    "000009.SH",
+    "000015.SH",
+    "399001.SZ",
+    "399004.SZ",
+    "399005.SZ",
+    "399006.SZ",
+    "000300.SH",
+    "000905.SH",
+    "000688.SH",
+    "000903.SH",
+    "000906.SH",
+    "000852.SH",
+    "000932.SH",
+    "000933.SH",
+    "980017.SZ",
+    "399808.SZ",
+    "399997.SZ",
+]
+
+
+def upload_index_to_baidu():
+    a_stock_path = '/A股/主要指数/'
+
+    now_date_time = datetime.now()
+    now_year = now_date_time.year
+    month = now_date_time.month
+    a_stock_path = a_stock_path + str(now_year) + '/' + str(month)
+    # 创建路径
+    baidu_yun_pan_handle_service.mkdir_baidu_new_folder(a_stock_path)
+
+
+
+
+
+    fail_list = []
+    for symbol in main_index_list:
+
+        try:
+
+            col_name = extra_income_db_name.ONE_MINUTE_K_LINE_BFQ_MAIN_INDEX + '_' + str(now_year)
+
+            if month < 10:
+                month_str = '0' + str(month)
+            else:
+                month_str = str(month)
+            begin_time = str(now_year) + '-' + month_str + '-01 09:00:00'
+            query = {'symbol': symbol, 'time': {"$gte": begin_time}}
+            one_minute_k_line_bfq_df = mongodbUtilV2_27019.find_query_data(col_name, query)
+            if data_frame_util.is_not_empty(one_minute_k_line_bfq_df):
+                one_minute_k_line_bfq_df = one_minute_k_line_bfq_df.sort_values(by=['time'], ascending=True)
+                del one_minute_k_line_bfq_df['_id']
+                del one_minute_k_line_bfq_df['symbol']
+                result = baidu_yun_pan_handle_service.upload_to_baidu(symbol, a_stock_path, one_minute_k_line_bfq_df)
+                if result != 0:
+                    fail_list.append(symbol)
+        except BaseException as e:
+            fail_list.append(symbol)
+            logger.error("上传数据异常:{}", e)
+
+
+
+
+if __name__ == '__main__':
+    upload_index_to_baidu()