mns-scheduler 1.2.9.7__tar.gz → 1.2.9.8__tar.gz

{mns-scheduler-1.2.9.7 → mns-scheduler-1.2.9.8}/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.1
 Name: mns-scheduler
-Version: 1.2.9.7
+Version: 1.2.9.8

mns-scheduler-1.2.9.8/mns_scheduler/baidu/baidu_yun_pan_handle_service.py

@@ -0,0 +1,122 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)
+import sys
+import os
+import json
+import pandas as pd
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)
+from bypy import ByPy
+from loguru import logger
+import subprocess
+import re
+
+
+def upload_to_baidu(file_name, folder_name, data_df):
+    upload_path_temp = fr'D:\upload_temp\{file_name}.csv'
+    bp = ByPy()
+
+    data_df.to_csv(upload_path_temp, index=False, encoding='gbk')
+
+    # 上传临时文件到百度云
+    remote_path = f'/{folder_name}/{file_name}.csv'
+    result = bp.upload(upload_path_temp, remote_path)
+    if result == 0:
+        logger.info("上传成功:{}", file_name)
+    else:
+        logger.error("上传失败:{}", file_name)
+    del_local_file(upload_path_temp)
+    return result
+
+
+def del_local_file(local_file_path):
+    try:
+        os.remove(local_file_path)
+    except Exception as e:
+        print(f"删除文件时出错: {e}")
+
+
+def mkdir_baidu_new_folder(remote_path):
+    bp = ByPy()
+    try:
+        # 调用 mkdir 方法创建文件夹
+        result = bp.mkdir(remote_path)
+
+        if result == 0:
+            logger.info("成功创建文件夹:{}", remote_path)
+        else:
+            logger.error("创建文件夹失败:{}", result)
+
+    except Exception as e:
+        logger.error("创建文件夹失败:{}", e)
+
+
+def del_baidu_old_folder(remote_path):
+    bp = ByPy()
+    try:
+        # 调用 mkdir 方法创建文件夹
+        result = bp.delete(remote_path)
+
+        if result == 0:
+            logger.info("成功删除文件夹:{}", remote_path)
+        else:
+            logger.error("删除文件夹失败:{}", result)
+
+    except Exception as e:
+        logger.error("删除文件夹失败:{}", e)
+
+
+def get_file_folder(path):
+    result = subprocess.run(
+        ['bypy', 'list', path],
+        capture_output=True,
+        text=True,
+        check=True,
+        encoding='utf-8'
+    )
+
+    # 假设 result 是 subprocess.run 的返回结果
+    stdout = result.stdout
+
+    # 正则表达式匹配文件行
+    pattern = re.compile(
+        r'^[FD]\s+(\S+)\s+(\d+)\s+(\d{4}-\d{2}-\d{2},\s\d{2}:\d{2}:\d{2})\s+([a-zA-Z0-9]{32})$',
+        re.IGNORECASE
+    )
+
+    data = []
+    for line in stdout.split('\n'):
+        line = line.strip()
+        if not line:
+            continue
+        match = pattern.match(line)
+        if match:
+            filename, size, mod_time, hash_val = match.groups()
+            # 处理时间格式：替换逗号为空格
+            mod_time = mod_time.replace(', ', ' ')
+            data.append({
+                'name': filename,
+                'size': int(size),
+                'update_time': mod_time,
+                'hash_value': hash_val
+            })
+
+    # 创建 DataFrame 并转换时间类型
+    df = pd.DataFrame(data)
+    if not df.empty:
+        df['update_time'] = pd.to_datetime(df['update_time'], format='%Y-%m-%d %H:%M:%S')
+
+    return df
+
+
+if __name__ == '__main__':
+    folder_name1 = '/A股/1分钟/2025/4'
+    get_file_folder(folder_name1)

mns-scheduler-1.2.9.8/mns_scheduler/extraIncome/one_minute/upload/upload_to_baidu_task.py

@@ -0,0 +1,83 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)
+from mns_common.db.MongodbUtil import MongodbUtil
+from loguru import logger
+import mns_scheduler.baidu.baidu_yun_pan_handle_service as baidu_yun_pan_handle_service
+from mns_common.db.v2.MongodbUtilV2 import MongodbUtilV2
+import mns_common.component.common_service_fun_api as common_service_fun_api
+import mns_common.utils.data_frame_util as data_frame_util
+from datetime import datetime
+import mns_common.constant.extra_income_db_name as extra_income_db_name
+
+mongodb_util = MongodbUtil('27017')
+mongodbUtilV2_27019 = MongodbUtilV2('27019', extra_income_db_name.EXTRA_INCOME)
+
+
+def upload_stock_to_baidu():
+    a_stock_path = '/A股/1分钟/'
+
+    now_date_time = datetime.now()
+    now_year = now_date_time.year
+    month = now_date_time.month
+    a_stock_path = a_stock_path + str(now_year) + '/' + str(month)
+    # 创建路径
+    baidu_yun_pan_handle_service.mkdir_baidu_new_folder(a_stock_path)
+    em_a_stock_info_df = mongodb_util.find_all_data('em_a_stock_info')
+
+    em_a_stock_info_df = common_service_fun_api.classify_symbol(em_a_stock_info_df)
+
+    em_a_stock_info_df['symbol'] = em_a_stock_info_df.apply(
+        lambda row: row['symbol'] + '.SZ' if row['classification'] in ['S', 'C']
+        else row['symbol'] + '.BJ' if row['classification'] in ['X']
+        else row['symbol'] + '.SH',
+        axis=1
+    )
+
+    file_folder_df = baidu_yun_pan_handle_service.get_file_folder(a_stock_path)
+    if data_frame_util.is_not_empty(file_folder_df):
+        # 去除文件名中的 .csv 后缀
+        file_folder_df['name'] = file_folder_df['name'].str.replace(r'\.csv$', '', regex=True)
+        em_a_stock_info_df = em_a_stock_info_df.loc[~(em_a_stock_info_df['symbol'].isin(file_folder_df['name']))]
+    fail_list = []
+    for stock_one in em_a_stock_info_df.itertuples():
+        try:
+            classification = stock_one.classification
+            if classification == 'X':
+                col_name = extra_income_db_name.ONE_MINUTE_K_LINE_BFQ_BJ
+            elif classification == 'S':
+                col_name = extra_income_db_name.ONE_MINUTE_K_LINE_BFQ_S
+
+            elif classification == 'H':
+                col_name = extra_income_db_name.ONE_MINUTE_K_LINE_BFQ_H
+            elif classification == 'K':
+                col_name = extra_income_db_name.ONE_MINUTE_K_LINE_BFQ_K
+            elif classification == 'C':
+                col_name = extra_income_db_name.ONE_MINUTE_K_LINE_BFQ_C
+            col_name = col_name + '_' + str(now_year)
+            symbol = stock_one.symbol
+            if month < 10:
+                month_str = '0' + str(month)
+            else:
+                month_str = str(month)
+            begin_time = str(now_year) + '-' + month_str + '-01 09:00:00'
+            query = {'symbol': symbol, 'time': {"$gte": begin_time}}
+            one_minute_k_line_bfq_df = mongodbUtilV2_27019.find_query_data(col_name, query)
+            if data_frame_util.is_not_empty(one_minute_k_line_bfq_df):
+                one_minute_k_line_bfq_df = one_minute_k_line_bfq_df.sort_values(by=['time'], ascending=True)
+                del one_minute_k_line_bfq_df['_id']
+                del one_minute_k_line_bfq_df['symbol']
+                result = baidu_yun_pan_handle_service.upload_to_baidu(symbol, a_stock_path, one_minute_k_line_bfq_df)
+                if result != 0:
+                    fail_list.append(symbol)
+        except BaseException as e:
+            fail_list.append(symbol)
+            logger.error("上传数据异常:{}", e)
+
+
+if __name__ == '__main__':
+    upload_stock_to_baidu()

mns-scheduler-1.2.9.8/mns_scheduler/us/__init__.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)

{mns-scheduler-1.2.9.7 → mns-scheduler-1.2.9.8}/mns_scheduler/us/baidu_yun_pan_handle_service.py

@@ -1,131 +1,91 @@
-import sys
-import os
-
-file_path = os.path.abspath(__file__)
-end = file_path.index('mns') + 16
-project_path = file_path[0:end]
-sys.path.append(project_path)
-from bypy import ByPy
-from mns_common.db.MongodbUtil import MongodbUtil
-import tempfile
-from loguru import logger
-import akshare as ak
-
-mongodb_util = MongodbUtil('27017')
-
-import subprocess
-
-
-def get_file_list(path):
-    """
-    获取百度网盘指定路径下的文件列表
-    :param path: 百度网盘中的路径，例如 '/我的资源'
-    :return: 文件列表
-    """
-    try:
-        # 调用 bypy list 命令
-        result = subprocess.run(['bypy', 'list', path], capture_output=True, text=True, check=True)
-
-        # 输出结果
-        if result.returncode == 0:
-            file_list = result.stdout.splitlines()  # 按行分割结果
-            return file_list
-        else:
-            logger.error("获取文件路径异常:{}", result.stderr)
-            return []
-    except subprocess.CalledProcessError as e:
-        logger.error("获取文件路径异常:{}", e)
-        return []
-
-
-def upload_to_baidu(file_name, folder_name, data_df):
-    bp = ByPy()
-    file_name = file_name + '.csv'
-    with tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False) as temp_file:
-        data_df.to_csv(temp_file, index=False)
-        temp_file_path = temp_file.name  # 获取临时文件的路径
-
-    # 上传临时文件到百度云
-    remote_path = f'/{folder_name}/{file_name}'
-    result = bp.upload(temp_file_path, remote_path)
-    if result == 0:
-        logger.info("上传成功:{}", file_name)
-    else:
-        logger.error("上传失败:{}", file_name)
-    return result
-
-
-def mkdir_baidu_new_folder(remote_path):
-    bp = ByPy()
-    try:
-        # 调用 mkdir 方法创建文件夹
-        result = bp.mkdir(remote_path)
-
-        if result == 0:
-            logger.info("成功创建文件夹:{}", remote_path)
-        else:
-            logger.error("创建文件夹失败:{}", result)
-
-    except Exception as e:
-        logger.error("创建文件夹失败:{}", e)
-
-
-def del_baidu_old_folder(remote_path):
-    bp = ByPy()
-    try:
-        # 调用 mkdir 方法创建文件夹
-        result = bp.delete(remote_path)
-
-        if result == 0:
-            logger.info("成功删除文件夹:{}", remote_path)
-        else:
-            logger.error("删除文件夹失败:{}", result)
-
-    except Exception as e:
-        logger.error("删除文件夹失败:{}", e)
-
-
-if __name__ == '__main__':
-    folder_name1 = '/美股/不复权日线'
-    mkdir_baidu_new_folder(folder_name1)
-    # get_file_list(folder_name1)
-    stock_us_spot_em_df = ak.stock_us_spot_em()
-    stock_us_spot_em_df = stock_us_spot_em_df.rename(columns={
-        "序号": "index",
-        "代码": "symbol",
-        "名称": "name",
-        "涨跌额": "change_price",
-        "涨跌幅": "chg",
-        "开盘价": "open",
-        "最高价": "high",
-        "最低价": "low",
-        "最新价": "now_price",
-        "昨收价": "last_price",
-        "总市值": "total_mv",
-        "市盈率": "pe",
-        "成交量": "volume",
-        "成交额": "amount",
-        "振幅": "pct_chg",
-        "换手率": "exchange"
-    })
-    stock_us_spot_em_df = stock_us_spot_em_df.sort_values(by=['amount'], ascending=False)
-    stock_us_spot_em_df = stock_us_spot_em_df.fillna(0)
-    stock_us_spot_em_df = stock_us_spot_em_df.loc[stock_us_spot_em_df['total_mv'] != 0]
-    for stock_one in stock_us_spot_em_df.itertuples():
-        try:
-            symbol = stock_one.symbol
-            name = stock_one.name
-            query = {'symbol': symbol, 'amount': {"$gt": 0}}
-            us_stock_bfq_daily_df_one = mongodb_util.find_query_data('us_stock_bfq_daily', query)
-            del us_stock_bfq_daily_df_one['_id']
-            del us_stock_bfq_daily_df_one['name']
-            file_name_one = name + '_' + symbol
-            upload_to_baidu(file_name_one, folder_name1, us_stock_bfq_daily_df_one)
-
-        except BaseException as e:
-            logger.error("同步数据发生异常:{}", e)
-
-# data_df = mongodb_util.find_query_data('us_stock_bfq_daily', query={'name': file_name1})
-# upload_to_baidu(file_name1, folder_name1, data_df)
-
-
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)
+from bypy import ByPy
+from mns_common.db.MongodbUtil import MongodbUtil
+import tempfile
+from loguru import logger
+import akshare as ak
+
+mongodb_util = MongodbUtil('27017')
+
+import subprocess
+
+
+def get_file_list(path):
+    """
+    获取百度网盘指定路径下的文件列表
+    :param path: 百度网盘中的路径，例如 '/我的资源'
+    :return: 文件列表
+    """
+    try:
+        # 调用 bypy list 命令
+        result = subprocess.run(['bypy', 'list', path], capture_output=True, text=True, check=True, encoding='utf-8')
+
+        # 输出结果
+        if result.returncode == 0:
+            file_list = result.stdout.splitlines()  # 按行分割结果
+            return file_list
+        else:
+            logger.error("获取文件路径异常:{}", result.stderr)
+            return []
+    except subprocess.CalledProcessError as e:
+        logger.error("获取文件路径异常:{}", e)
+        return []
+
+
+def upload_to_baidu(file_name, folder_name, data_df):
+    bp = ByPy()
+    file_name = file_name + '.csv'
+    with tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False) as temp_file:
+        data_df.to_csv(temp_file, index=False)
+        temp_file_path = temp_file.name  # 获取临时文件的路径
+
+    # 上传临时文件到百度云
+    remote_path = f'/{folder_name}/{file_name}'
+    result = bp.upload(temp_file_path, remote_path)
+    if result == 0:
+        logger.info("上传成功:{}", file_name)
+    else:
+        logger.error("上传失败:{}", file_name)
+    return result
+
+
+def mkdir_baidu_new_folder(remote_path):
+    bp = ByPy()
+    try:
+        # 调用 mkdir 方法创建文件夹
+        result = bp.mkdir(remote_path)
+
+        if result == 0:
+            logger.info("成功创建文件夹:{}", remote_path)
+        else:
+            logger.error("创建文件夹失败:{}", result)
+
+    except Exception as e:
+        logger.error("创建文件夹失败:{}", e)
+
+
+def del_baidu_old_folder(remote_path):
+    bp = ByPy()
+    try:
+        # 调用 mkdir 方法创建文件夹
+        result = bp.delete(remote_path)
+
+        if result == 0:
+            logger.info("成功删除文件夹:{}", remote_path)
+        else:
+            logger.error("删除文件夹失败:{}", result)
+
+    except Exception as e:
+        logger.error("删除文件夹失败:{}", e)
+
+
+if __name__ == '__main__':
+    folder_name1 = '/美股/不复权日线'
+    mkdir_baidu_new_folder(folder_name1)
+    list_df = get_file_list('/bypy/美股')

mns-scheduler-1.2.9.8/mns_scheduler/zt/script/__init__.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+file_path = os.path.abspath(__file__)
+end = file_path.index('mns') + 16
+project_path = file_path[0:end]
+sys.path.append(project_path)

{mns-scheduler-1.2.9.7 → mns-scheduler-1.2.9.8}/mns_scheduler.egg-info/PKG-INFO

@@ -1,3 +1,3 @@
 Metadata-Version: 2.1
 Name: mns-scheduler
-Version: 1.2.9.7
+Version: 1.2.9.8

{mns-scheduler-1.2.9.7 → mns-scheduler-1.2.9.8}/mns_scheduler.egg-info/SOURCES.txt

@@ -5,6 +5,8 @@ mns_scheduler.egg-info/PKG-INFO
 mns_scheduler.egg-info/SOURCES.txt
 mns_scheduler.egg-info/dependency_links.txt
 mns_scheduler.egg-info/top_level.txt
+mns_scheduler/baidu/__init__.py
+mns_scheduler/baidu/baidu_yun_pan_handle_service.py
 mns_scheduler/big_deal/__init__.py
 mns_scheduler/big_deal/ths_big_deal_sync.py
 mns_scheduler/company_info/__init__.py
@@ -71,6 +73,8 @@ mns_scheduler/extraIncome/one_minute/kzz/__init__.py
 mns_scheduler/extraIncome/one_minute/kzz/kzz_one_minute_sync_task.py
 mns_scheduler/extraIncome/one_minute/stock/__init__.py
 mns_scheduler/extraIncome/one_minute/stock/stock_one_minute_sync_task.py
+mns_scheduler/extraIncome/one_minute/upload/__init__.py
+mns_scheduler/extraIncome/one_minute/upload/upload_to_baidu_task.py
 mns_scheduler/extraIncome/temp/__init__.py
 mns_scheduler/extraIncome/temp/tu_share_data_etf_sync.py
 mns_scheduler/extraIncome/temp/tu_share_data_kzz_sync.py

{mns-scheduler-1.2.9.7 → mns-scheduler-1.2.9.8}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='mns-scheduler',
-    version='1.2.9.7',
+    version='1.2.9.8',
     packages=find_packages(),
     install_requires=[],  # 如果有依赖项，可以在这里列出
 )