mns-scheduler 1.1.8.4__py3-none-any.whl → 1.4.3.2__py3-none-any.whl

mns_scheduler/db/script/sync/sync_hui_ce_test_data_01.py

@@ -0,0 +1,69 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)
+
+import mns_common.utils.data_frame_util as data_frame_util
+from mns_scheduler.db.script.sync.remote_mongo_util import RemoteMongodbUtil
+from mns_scheduler.db.script.sync.local_mongo_util import LocalMongodbUtil
+from loguru import logger
+import numpy as np
+
+remote_mongodb_util = RemoteMongodbUtil('27017')
+local_mongodb_util = LocalMongodbUtil('27017')
+
+
+def sync_real_time_data(str_day, min_number, max_number):
+    db_name = 'realtime_quotes_now_' + str_day
+    while min_number <= max_number:
+        query = {'number': min_number}
+        data_df = remote_mongodb_util.find_query_data(db_name, query)
+        local_mongodb_util.insert_mongo(data_df, db_name)
+        min_number = min_number + 1
+        logger.info(min_number)
+
+
+def sync_k_line(str_day):
+    query = {"str_day": str_day}
+    db_name = 'k_line_info'
+    data_df = remote_mongodb_util.find_query_data(db_name, query)
+    try:
+        local_mongodb_util.save_mongo(data_df, db_name)
+    except BaseException as e:
+        pass
+
+
+if __name__ == '__main__':
+    sync_k_line('2025-08-27')
+    # sync_k_line('2025-07-24')
+    # sync_k_line('2025-07-25')
+    # sync_k_line(str_day_01)
+    # sync_k_line('2024-11-13')
+    # sync_k_line('2024-11-12')
+    # sync_k_line('2024-11-15') 232
+    # sync_real_time_data(str_day_01, 1, 2000)
+    # sync_real_time_data('2024-11-15', 10, 1010)
+    #
+    # sync_k_line('2024-12-20')
+    # sync_k_line('2024-12-19')
+    # sync_k_line('2024-12-18')
+    # sync_k_line('2024-12-17')
+    # sync_k_line('2024-12-16')
+    #
+    # sync_k_line('2024-12-13')
+    # sync_k_line('2024-12-12')
+    # sync_k_line('2024-12-11')
+    # sync_k_line('2024-12-10')
+    # sync_k_line('2024-12-09')
+    #
+    # sync_k_line('2024-12-06')
+    # sync_k_line('2024-12-05')
+    # sync_k_line('2024-12-04')
+    # sync_k_line('2024-12-03')
+    # sync_k_line('2024-12-02')
+
+    # sync_real_time_data('2024-12-20', 1000, 2000)
+    # sync_real_time_data('2024-11-13', 10, 1010)

mns_scheduler/db/script/update/__init__.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)

mns_scheduler/db/script/update/update_col_field.py

@@ -0,0 +1,36 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 17
+project_path = file_path[0:end]
+sys.path.append(project_path)
+from mns_common.db.MongodbUtil import MongodbUtil
+from loguru import logger
+
+mongodb_util_27017 = MongodbUtil('27017')
+mongodb_util_27019 = MongodbUtil('27019')
+
+
+def update_col_data():
+    query = {"$and": [{'trade_date': {'$lte': '2024-05-23'}},
+                      {'trade_date': {'$gte': '2022-07-02'}}]}
+    trade_date_list_df = mongodb_util_27017.find_query_data('trade_date_list', query)
+    trade_date_list_df = trade_date_list_df.sort_values(by=['trade_date'], ascending=True)
+    for trade_one in trade_date_list_df.itertuples():
+        col_name = 'realtime_quotes_now_' + trade_one.trade_date
+        new_values = {'$unset': {
+            'classification': '',
+            'medium_order_net_inflow': '',
+            'small_order_net_inflow': '',
+            'str_day': '',
+            'list_date': '',
+            'amount_level': '',
+            'name': '', 'industry': '', 'concept': ''}}
+
+        mongodb_util_27019.update_many({}, new_values, col_name)
+        logger.info("完成集合数据更新:{}", col_name)
+
+
+if __name__ == '__main__':
+    update_col_data()

mns_scheduler/finance/__init__.py

@@ -2,6 +2,6 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)

mns_scheduler/finance/{em_financial_asset_liability_sync_service_api.py → em/em_financial_asset_liability_sync_service_api.py}

@@ -2,12 +2,12 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 import akshare as ak
 from mns_common.db.MongodbUtil import MongodbUtil
-import mns_scheduler.finance.finance_common_api as finance_common_api
+import mns_scheduler.finance.em.finance_common_api as finance_common_api
 import mns_common.constant.db_name_constant as db_name_constant
 
 mongodb_util = MongodbUtil('27017')

mns_scheduler/finance/{em_financial_profit_sync_service_api.py → em/em_financial_profit_sync_service_api.py}

@@ -1,8 +1,10 @@
 import sys
 import os
 
+import pandas as pd
+
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 
@@ -12,7 +14,7 @@ import mns_common.utils.data_frame_util as data_frame_util
 
 mongodb_util = MongodbUtil('27017')
 import mns_common.constant.db_name_constant as db_name_constant
-import mns_scheduler.finance.finance_common_api as finance_common_api
+import mns_scheduler.finance.em.finance_common_api as finance_common_api
 from loguru import logger
 
 
@@ -228,7 +230,7 @@ def get_em_profit_api(symbol):
         stock_profit_sheet_by_report_em = ak.stock_profit_sheet_by_report_em(sec_code)
     except Exception as e:
         logger.error("同步利润表异常:{},{}", symbol, e)
-    if data_frame_util.is_empty(stock_profit_sheet_by_report_em):
+    if data_frame_util.is_empty(stock_profit_sheet_by_report_em.copy()):
         return None
     stock_profit_sheet_by_report_em = check_columns(stock_profit_sheet_by_report_em)
     stock_profit_sheet_by_report_em = stock_profit_sheet_by_report_em[[
@@ -293,57 +295,56 @@ def get_em_profit_api(symbol):
     else:
         new_profit_df = stock_profit_sheet_by_report_em
     if data_frame_util.is_empty(new_profit_df):
-        return None
-    new_profit_df.fillna(0, inplace=True)
+        return pd.DataFrame()
+    new_profit_df = new_profit_df.fillna(0)
     return new_profit_df
 
 
 def check_columns(profit_df):
     if 'TOTAL_OPERATE_INCOME' not in profit_df.columns:
-        profit_df['TOTAL_OPERATE_INCOME'] = 0
+        profit_df = profit_df.assign(TOTAL_OPERATE_INCOME=0)
 
     if 'FAIRVALUE_CHANGE_INCOME' not in profit_df.columns:
-        profit_df['FAIRVALUE_CHANGE_INCOME'] = 0
+        profit_df = profit_df.assign(FAIRVALUE_CHANGE_INCOME=0)
 
     if 'INTEREST_INCOME' not in profit_df.columns:
-        profit_df['INTEREST_INCOME'] = 0
+        profit_df = profit_df.assign(INTEREST_INCOME=0)
     if 'TOTAL_OPERATE_COST' not in profit_df.columns:
-        profit_df['TOTAL_OPERATE_COST'] = 0
+        profit_df = profit_df.assign(TOTAL_OPERATE_COST=0)
     if 'OPERATE_COST' not in profit_df.columns:
-        profit_df['OPERATE_COST'] = 0
+        profit_df = profit_df.assign(OPERATE_COST=0)
     if 'INTEREST_EXPENSE' not in profit_df.columns:
-        profit_df['INTEREST_EXPENSE'] = 0
+        profit_df = profit_df.assign(INTEREST_EXPENSE=0)
     if 'FEE_COMMISSION_EXPENSE' not in profit_df.columns:
-        profit_df['FEE_COMMISSION_EXPENSE'] = 0
+        profit_df = profit_df.assign(FEE_COMMISSION_EXPENSE=0)
     if 'RESEARCH_EXPENSE' not in profit_df.columns:
-        profit_df['RESEARCH_EXPENSE'] = 0
+        profit_df = profit_df.assign(RESEARCH_EXPENSE=0)
     if 'SALE_EXPENSE' not in profit_df.columns:
-        profit_df['SALE_EXPENSE'] = 0
+        profit_df = profit_df.assign(SALE_EXPENSE=0)
     if 'MANAGE_EXPENSE' not in profit_df.columns:
-        profit_df['MANAGE_EXPENSE'] = 0
+        profit_df = profit_df.assign(MANAGE_EXPENSE=0)
     if 'FINANCE_EXPENSE' not in profit_df.columns:
-        profit_df['FINANCE_EXPENSE'] = 0
+        profit_df = profit_df.assign(FINANCE_EXPENSE=0)
     if 'FE_INTEREST_EXPENSE' not in profit_df.columns:
-        profit_df['FE_INTEREST_EXPENSE'] = 0
+        profit_df = profit_df.assign(FE_INTEREST_EXPENSE=0)
     if 'FE_INTEREST_INCOME' not in profit_df.columns:
-        profit_df['FE_INTEREST_INCOME'] = 0
+        profit_df = profit_df.assign(FE_INTEREST_INCOME=0)
     if 'CREDIT_IMPAIRMENT_INCOME' not in profit_df.columns:
-        profit_df['CREDIT_IMPAIRMENT_INCOME'] = 0
+        profit_df = profit_df.assign(CREDIT_IMPAIRMENT_INCOME=0)
     if 'ACCOUNTS_RECE' not in profit_df.columns:
-        profit_df['ACCOUNTS_RECE'] = 0
-
+        profit_df = profit_df.assign(ACCOUNTS_RECE=0)
     if 'LOAN_ADVANCE' not in profit_df.columns:
-        profit_df['LOAN_ADVANCE'] = 0
+        profit_df = profit_df.assign(LOAN_ADVANCE=0)
     if 'MONETARYFUNDS' not in profit_df.columns:
-        profit_df['MONETARYFUNDS'] = 0
+        profit_df = profit_df.assign(MONETARYFUNDS=0)
     return profit_df
 
 
-import mns_common.api.em.east_money_stock_api as east_money_stock_api
+import mns_common.component.em.em_stock_info_api as em_stock_info_api
 
 if __name__ == '__main__':
-    get_em_profit_api('600519')
-    em_df = east_money_stock_api.get_real_time_quotes_all_stocks()
+    get_em_profit_api('688302')
+    em_df = em_stock_info_api.get_a_stock_info()
     for em_one in em_df.itertuples():
         try:
             get_em_profit_api(em_one.symbol)

mns_scheduler/finance/{finance_common_api.py → em/finance_common_api.py}

@@ -2,14 +2,14 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 
 import mns_common.component.common_service_fun_api as common_service_fun_api
 from mns_common.db.MongodbUtil import MongodbUtil
 import mns_common.constant.db_name_constant as db_name_constant
-import mns_common.api.em.east_money_stock_api as east_money_stock_api
+import mns_common.component.em.em_stock_info_api as em_stock_info_api
 import mns_common.utils.data_frame_util as data_frame_util
 
 mongodb_util = MongodbUtil('27017')
@@ -39,7 +39,7 @@ def find_asset_liability_report(period_time):
 
 # 查出未报告的股票
 def find_un_report_symbol(period_time, report_name):
-    real_time_quotes_df = east_money_stock_api.get_real_time_quotes_all_stocks()
+    real_time_quotes_df = em_stock_info_api.get_a_stock_info()
     real_time_quotes_df = real_time_quotes_df.loc[~(real_time_quotes_df['name'].str.contains('退'))]
     real_time_quotes_df.dropna(subset=['list_date'], axis=0, inplace=True)
 

mns_scheduler/finance/{sync_financial_report_service_api.py → sync_financial_report_service_task.py}

@@ -2,21 +2,22 @@ import sys
 import os
 
 file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
+end = file_path.index('mns') + 17
 project_path = file_path[0:end]
 sys.path.append(project_path)
 from datetime import datetime
 
 import mns_common.constant.db_name_constant as db_name_constant
-import mns_scheduler.finance.em_financial_profit_sync_service_api as em_financial_profit_sync_service_api
+import mns_scheduler.finance.em.em_financial_profit_sync_service_api as em_financial_profit_sync_service_api
 from mns_common.db.MongodbUtil import MongodbUtil
 from loguru import logger
 import \
-    mns_scheduler.finance.em_financial_asset_liability_sync_service_api as em_financial_asset_liability_sync_service_api
+    mns_scheduler.finance.em.em_financial_asset_liability_sync_service_api as em_financial_asset_liability_sync_service_api
 import mns_scheduler.risk.financial_report_risk_check_api as financial_report_risk_check_api
 import mns_common.utils.data_frame_util as data_frame_util
-import mns_scheduler.finance.finance_common_api as finance_common_api
+import mns_scheduler.finance.em.finance_common_api as finance_common_api
 import mns_scheduler.risk.compliance.undisclosed_annual_report_api as undisclosed_annual_report_api
+import mns_scheduler.finance.xue_qiu.sync_xue_qiu_fiance_data as sync_xue_qiu_fiance_data
 
 mongodb_util = MongodbUtil('27017')
 
@@ -26,39 +27,90 @@ mongodb_util = MongodbUtil('27017')
 # 3、上市公司季报披露时间:
 #    1季报:每年4月1日-- -4月30日。
 #    2季报(中报) :每年7月1日--8月30日。
-#    3季报:每年10月1日--10月31日4季报(年报) :每年1月1日--4月30日
+#    3季报:每年10月1日--10月31日
+#    4季报(年报) :每年1月1日--4月30日
 
 def sync_financial_report():
     now_date = datetime.now()
     now_year = now_date.year
     now_month = now_date.month
     sync_time = now_date.strftime('%Y-%m-%d %H:%M:%S')
-    # 年报
-    if 1 <= now_month <= 5:
+
+    if now_month in [1, 2, 3, 4]:
+        sync_xue_qiu_fiance_data.sync_xue_qiu_very_period_report(str(now_year - 1) + '年报', None)
+
+        # 年报
         period = 4
         period_time = str(now_year - 1) + "-12-31 00:00:00"
-        sync_profit_report(period_time, sync_time, period)
-        sync_asset_liability_report(period_time, sync_time, period)
+        sync_em_profit_report(period_time, sync_time, period)
+        sync_em_asset_liability_report(period_time, sync_time, period)
+
+        # 一季报
+        if now_month == 4:
+            sync_xue_qiu_fiance_data.sync_xue_qiu_very_period_report(str(now_year) + '一季报', None)
+
+            period = 1
+            period_time = str(now_year) + "-03-31 00:00:00"
+            sync_em_profit_report(period_time, sync_time, period)
+            sync_em_asset_liability_report(period_time, sync_time, period)
+
 
-    # 一季报
-    elif now_month == 5:
-        period = 1
-        period_time = str(now_year) + "-03-31 00:00:00"
-        sync_profit_report(period_time, sync_time, period)
-        sync_asset_liability_report(period_time, sync_time, period)
 
     # 二季报
-    elif 7 <= now_month <= 8:
+    elif now_month in [7, 8]:
+        sync_xue_qiu_fiance_data.sync_xue_qiu_very_period_report(str(now_year) + '中报', None)
+
         period = 2
         period_time = str(now_year) + "-06-30 00:00:00"
-        sync_profit_report(period_time, sync_time, period)
-        sync_asset_liability_report(period_time, sync_time, period)
+        sync_em_profit_report(period_time, sync_time, period)
+        sync_em_asset_liability_report(period_time, sync_time, period)
+
+
+
     # 三季报
     elif now_month == 10:
+        sync_xue_qiu_fiance_data.sync_xue_qiu_very_period_report(str(now_year) + '三季报', None)
+
         period = 3
         period_time = str(now_year) + "-09-30 00:00:00"
-        sync_profit_report(period_time, sync_time, period)
-        sync_asset_liability_report(period_time, sync_time, period)
+        sync_em_profit_report(period_time, sync_time, period)
+        sync_em_asset_liability_report(period_time, sync_time, period)
+
+    elif now_month == 5:
+
+        sync_xue_qiu_fiance_data.sync_xue_qiu_very_period_report(str(now_year - 1) + '年报', None)
+        sync_xue_qiu_fiance_data.sync_xue_qiu_very_period_report(str(now_year) + '一季报', None)
+
+        # 补偿年报和一季度,出不了报告的
+        miss_period_04 = 4
+        miss_period_time_04 = str(now_year - 1) + "-12-31 00:00:00"
+
+        sync_em_profit_report(miss_period_time_04, sync_time, miss_period_04)
+        sync_em_asset_liability_report(miss_period_time_04, sync_time, miss_period_04)
+
+        period_01 = 1
+        period_time_01 = str(now_year) + "-03-31 00:00:00"
+        sync_em_profit_report(period_time_01, sync_time, period_01)
+        sync_em_asset_liability_report(period_time_01, sync_time, period_01)
+
+    elif now_month == 9:
+        sync_xue_qiu_fiance_data.sync_xue_qiu_very_period_report(str(now_year) + '中报', None)
+
+        # 补偿二季度
+        period_02 = 2
+        period_time_02 = str(now_year) + "-06-30 00:00:00"
+        sync_em_profit_report(period_time_02, sync_time, period_02)
+        sync_em_asset_liability_report(period_time_02, sync_time, period_02)
+
+    elif now_month in [11, 12]:
+        sync_xue_qiu_fiance_data.sync_xue_qiu_very_period_report(str(now_year) + '三季报', None)
+
+        # 补偿三季度
+        period_03 = 3
+        period_time_03 = str(now_year) + "-09-30 00:00:00"
+        sync_em_profit_report(period_time_03, sync_time, period_03)
+        sync_em_asset_liability_report(period_time_03, sync_time, period_03)
+
     # 未出报告check
     undisclosed_annual_report_api.un_disclosed_report_check(sync_time, now_year, period, period_time)
     # 新股或者未出报告的
@@ -66,7 +118,7 @@ def sync_financial_report():
 
 
 # 同步资产表
-def sync_asset_liability_report(period_time, sync_time, period):
+def sync_em_asset_liability_report(period_time, sync_time, period):
     un_report_asset_df = finance_common_api.find_un_report_symbol(period_time,
                                                                   db_name_constant.EM_STOCK_ASSET_LIABILITY)
     for un_report_asset_one in un_report_asset_df.itertuples():
@@ -91,8 +143,8 @@ def sync_asset_liability_report(period_time, sync_time, period):
             logger.error("同步资产表异常:{},{},{}", symbol, period_time, e)
 
 
-# 同步利润表
-def sync_profit_report(period_time, sync_time, period):
+# 同步em利润表
+def sync_em_profit_report(period_time, sync_time, period):
     un_report_profit_df = finance_common_api.find_un_report_symbol(period_time, db_name_constant.EM_STOCK_PROFIT)
     for un_report_profit_one in un_report_profit_df.itertuples():
         try:
@@ -142,8 +194,9 @@ def sync_miss_report(sync_time):
 
 
 if __name__ == '__main__':
-    now_date = datetime.now()
-    now_year = now_date.year
-    now_month = now_date.month
-    sync_time_test = now_date.strftime('%Y-%m-%d %H:%M:%S')
+    now_date_test = datetime.now()
+    now_year_test = now_date_test.year
+    now_month_test = now_date_test.month
+    sync_time_test = now_date_test.strftime('%Y-%m-%d %H:%M:%S')
     sync_miss_report(sync_time_test)
+    sync_financial_report()

mns_scheduler/finance/xue_qiu/down_load_xueqiu_report_api.py

@@ -0,0 +1,77 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)
+import requests
+import pandas as pd
+from loguru import logger
+
+
+# report_type income 利润表
+# cash_flow 现金流量
+# balance 资产负债
+
+
+def get_xue_qiu_report(symbol, report_type, cookie, count, period_type):
+    # 请求 URL
+    url = (f"https://stock.xueqiu.com/v5/stock/finance/cn/{report_type}.json?symbol={symbol}&type={period_type}"
+           f"&is_detail=true&count={count}")
+
+    # 请求头（关键是 cookies）
+    headers = {
+        "accept": "application/json, text/plain, */*",
+        "accept-language": "zh-CN,zh;q=0.9",
+        "origin": "https://xueqiu.com",
+        "referer": f"https://xueqiu.com/snowman/S/{symbol}/detail",
+        "sec-fetch-mode": "cors",
+        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
+        # ⚠ 替换为你浏览器访问雪球后的 Cookie（需含 xq_a_token）
+        "cookie": cookie,
+    }
+
+    # 发送请求
+    response = requests.get(url, headers=headers)
+
+    # 检查结果
+    if response.status_code == 200:
+        data = response.json()
+
+        # 提取财报数据列表
+        raw_list = data['data']['list']
+        processed_list = []
+
+        for entry in raw_list:
+            flat_entry = {}
+            for key, value in entry.items():
+                if isinstance(value, list) and len(value) == 2:
+                    # 处理列表字段：提取两个值，空则填 0
+                    v0 = value[0] if value[0] is not None else 0
+                    v1 = value[1] if value[1] is not None else 0
+                    flat_entry[key] = v0
+                    flat_entry[key + "_chg"] = round(v1 * 100, 2)
+                elif value is None:
+                    # 整个字段为空，设置为 0 和 0（变化量）
+                    flat_entry[key] = 0
+                else:
+                    flat_entry[key] = value
+            processed_list.append(flat_entry)
+
+        # 转换为 DataFrame
+        df = pd.DataFrame(processed_list)
+        return df
+
+    else:
+        logger.error("请求失败，状态码:{}", response.status_code)
+        return pd.DataFrame()
+
+
+import mns_common.component.cookie.cookie_info_service as cookie_info_service
+
+if __name__ == '__main__':
+    cash_flow_df = get_xue_qiu_report('SZ301662', 'cash_flow', cookie_info_service.get_xue_qiu_cookie(), 1, 'all')
+    income_df = get_xue_qiu_report('SZ301662', 'income', cookie_info_service.get_xue_qiu_cookie(), 1, 'all')
+    balance_df = get_xue_qiu_report('SZ301662', 'balance', cookie_info_service.get_xue_qiu_cookie(), 1, 'all')
+    pass