mdbq 3.6.24__py3-none-any.whl → 3.7.1__py3-none-any.whl

mdbq/bdup/bdup.py

@@ -2,6 +2,7 @@
 import os
 import platform
 import subprocess
+from concurrent.futures import ThreadPoolExecutor
 from bypy import ByPy
 
 
@@ -12,13 +13,15 @@ class BaiDu:
     def __init__(self):
         self.local_path = None
         self.remote_path = None
-        self.skip = ['.DS_Store', '.localized', 'desktop.ini', '$RECYCLE.BIN', 'Icon']
-        self.delete_remote_files = ['.DS_Store', '.localized', 'desktop.ini', '$RECYCLE.BIN', 'Icon']
+        self.skip:list = []
+        self.delete_remote_files:list = []
         self.bp = ByPy()
+        self.count = 0
+        self.total = 0
 
     def upload_dir(self, local_path, remote_path):
         """
-        上传整个文件夹，执行完后删除指定文件
+        上传整个文件夹，执行完后删除指定文件, 指定 self.delete_remote_files
         如果通过调用命令行终端运行, 《云端路径!!》必须使用linux格式，不要使用反斜杆,否则在windows系统里面会上传失败
         """
         self.local_path = local_path
@@ -37,38 +40,59 @@ class BaiDu:
                 print(e)
         self.delete_files()  # 最好是在内部执行删除, 避免路径异常
 
-    def upload_file(self, local_path, remote_path):
+    def upload_file(self, local_path, remote_path, processes=False):
         """
-        上传文件夹，按单个文件上传，可以跳过指定文件
+        上传文件夹，按单个文件上传，可以跳过指定文件/文件夹, 指定 self.skip
         《云端路径!!》必须使用linux格式
         """
+        if not isinstance(self.skip, list):
+            raise TypeError('skip must be a list')
+        self.skip += ['.DS_Store', '.localized', 'desktop.ini', '$RECYCLE.BIN', 'Icon']
         self.local_path = local_path
         self.remote_path = remote_path.replace('\\', '/')
         if not os.path.exists(self.local_path):
             print(f'{self.local_path}: 本地目录不存在，没有什么可传的')
             return
 
-        files = os.listdir(self.local_path)
-        for file in files:
-            if os.path.isfile(os.path.join(self.local_path, file)):
-                if file in self.skip:
-                    continue
-                lc_path = os.path.join(self.local_path, file)
+        local_files = os.listdir(self.local_path)
+
+        local_file_list = []
+        for file in local_files:
+            if file in self.skip:  # 跳过指定文件/文件夹
+                continue
+            local_p = os.path.join(self.local_path, file)
+            if os.path.isfile(local_p):
                 rt_path = os.path.join(self.remote_path, file).replace('\\', '/')
-                print(f'上传: {file}')
-                self.bp.upload(localpath=lc_path, remotepath=rt_path)  # 上传文件到百度云
-            elif os.path.isdir(os.path.join(self.local_path, file)):
-                for root, dirs, files in os.walk(os.path.join(self.local_path, file), topdown=False):
+                self.total += 1
+                local_file_list.append({local_p: rt_path})
+            elif os.path.isdir(local_p):
+                for root, dirs, files in os.walk(local_p, topdown=False):
                     for name in files:
-                        if name in self.skip:
+                        if name in self.skip:  # 从子文件夹内跳过指定文件
                             continue
                         lc_path = os.path.join(root, name)
                         rt_path = lc_path.replace(self.local_path, self.remote_path).replace('\\', '/')
-                        print(f'上传: {name}')
-                        self.bp.upload(localpath=lc_path, remotepath=rt_path)  # 上传文件到百度云
+                        self.total += 1
+                        local_file_list.append({lc_path: rt_path})
+        if processes:
+            # 不指定 max_workers 参数，默认值是 os.cpu_count() * 5
+            with ThreadPoolExecutor() as executor:
+                    executor.map(self.up_one_file, local_file_list)
+        else:
+            for item in local_file_list:
+                self.up_one_file(file_dict=item)
+
+    def up_one_file(self, file_dict:dict):
+        if not isinstance(file_dict, dict):
+            raise TypeError('file_dict must be a dict')
+        for k, v in file_dict.items():
+            self.count += 1
+            print(f'上传: {self.count}/{self.total}  {k}')
+            self.bp.upload(localpath=k, remotepath=v)  # 上传文件到百度云
 
     def delete_files(self):
         """ 移除云端文件，位于 self.remote_path 文件夹下的子文件 """
+        self.delete_remote_files += ['.DS_Store', '.localized', 'desktop.ini', '$RECYCLE.BIN', 'Icon']
         for delete_file in self.delete_remote_files:
             self.bp.remove(remotepath=f'{self.remote_path.replace('\\', '/')}/{delete_file}')  # 移除文件
 
@@ -83,9 +107,5 @@ class BaiDu:
 
 
 if __name__ == '__main__':
-    pass
-    import datetime
-    upload_path = f'windows/{str(datetime.date.today().strftime("%Y-%m"))}/{str(datetime.date.today())}'
-    b = BaiDu()
-    # b.upload_dir(local_path='/Users/xigua/Downloads', remote_path=upload_path)
-    b.download_dir(local_path='/Users/xigua/Downloads', remote_path=upload_path)
+    bp = ByPy()
+    bp.list()

mdbq/config/default.py

@@ -62,7 +62,10 @@ def get_hostname(platform, hostname, sql, local):
     return return_host(conf_data)
 
 
-def get_engine():
+def get_engine_bak():
+    """
+    要删除
+    """
     if not os.path.isdir(support_path):
         print(f'缺少配置文件，无法读取配置文件： {file}')
         return

mdbq/config/products.py

@@ -6,7 +6,6 @@ import getpass
 import socket
 import pandas as pd
 from mdbq.mysql import mysql
-# from mdbq.config import myconfig
 from mdbq.config import default
 from numpy.ma.core import product
 
@@ -14,18 +13,7 @@ from numpy.ma.core import product
 天猫货品年份基准对照
 用于聚合数据，通过此数据表进一步可确定商品上架年月
 """
-m_engine, company_engine, ts_engine, (username, password, host, port) = default.get_engine()
-# username, password, host, port, service_database = None, None, None, None, None,
-# if socket.gethostname() in ['xigua_lx', 'xigua1', 'MacBookPro']:
-#     conf = myconfig.main()
-#     data = conf['Windows']['xigua_lx']['mysql']['local']
-#     username, password, host, port = data['username'], data['password'], data['host'], data['port']
-#     service_database = {'xigua_lx': 'mysql'}
-# elif socket.gethostname() in ['company', 'Mac2.local']:
-#     conf = myconfig.main()
-#     data = conf['Windows']['company']['mysql']['local']
-#     username, password, host, port = data['username'], data['password'], data['host'], data['port']
-#     service_database = {'company': 'mysql'}
+m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname='xigua_lx', sql='mysql', local='remoto', config_file=None)
 if not username:
     print(f'找不到主机1：')
 

{mdbq-3.6.24.dist-info → mdbq-3.7.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 3.6.24
+Version: 3.7.1
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.6.24.dist-info → mdbq-3.7.1.dist-info}/RECORD

@@ -6,11 +6,11 @@ mdbq/aggregation/datashow_bak.py,sha256=1AYSIDkdUx-4as1Ax2rPj0cExM9d-qFMrFYLAaPH
 mdbq/aggregation/optimize_data.py,sha256=foZGLDGJRhM2qOr2mTvB3InDFId7r4KBXrJfB3-xq1k,2639
 mdbq/aggregation/query_data.py,sha256=AiG0W9Rum_hKCTPNB-nuA5ehpnVW12byLA7uwzOmO6Q,168639
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
-mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
+mdbq/bdup/bdup.py,sha256=hJs815hGFwm_X5bP2i9XugG2w2ZY_F0n3-Q0hVpIPPw,4892
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
-mdbq/config/default.py,sha256=p0iCODXGxpcHGevDcu54KZbOG1iBUuOCtmIdwdCrPpQ,4939
+mdbq/config/default.py,sha256=3IGc4aVuoL6sZh7xkM0GUXwHJD3-HCfYTnb1Q5ZL1UM,4976
 mdbq/config/myconfig.py,sha256=Akt-7KqSUBdoHQa4_Mw6YC4pA75219d-21iDO30iaD8,894
-mdbq/config/products.py,sha256=UAPcS0CbKmHribxgwtqiAsVICkuGLrizLDT6zbf4kFY,6353
+mdbq/config/products.py,sha256=FbBIqmyaiq9h03FIeE9W2bwbLm2_5pr6xyzPV-u7Ges,5689
 mdbq/config/set_support.py,sha256=6EJC1PUCvP8HnRsljOrlBbK-65lZjh8_8mCCOVoPkFM,691
 mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
 mdbq/dataframe/converter.py,sha256=lETYhT7KXlWzWwqguqhk6vI6kj4rnOBEW1lhqKy2Abc,5035
@@ -36,7 +36,7 @@ mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=oyFwE-8c6uErSGYNIO0z2ng93mH0zstRLD86MWqF6M8,25636
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
 mdbq/spider/aikucun.py,sha256=1gAEwCUmhCSpOSRPD2EEcbH3bFGn4sUQnQUAsJb5-qM,19391
-mdbq-3.6.24.dist-info/METADATA,sha256=BTBOMOtj2D-viT3P9SQ68RkCfIpQOuRic2XsUf2pjWA,244
-mdbq-3.6.24.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-3.6.24.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.6.24.dist-info/RECORD,,
+mdbq-3.7.1.dist-info/METADATA,sha256=uHL1bLpTvZHQVsfp6WHV5Ceog2VTGOfZhcbj0-WW16I,243
+mdbq-3.7.1.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-3.7.1.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.7.1.dist-info/RECORD,,

{mdbq-3.6.24.dist-info → mdbq-3.7.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.1.0)
+Generator: bdist_wheel (0.44.0)
 Root-Is-Purelib: true
 Tag: py3-none-any