mdbq 4.0.28__py3-none-any.whl → 4.0.30__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '4.0.28'
+VERSION = '4.0.30'

mdbq/mysql/deduplicator.py

@@ -63,7 +63,9 @@ class MySQLDeduplicator:
             skip_system_dbs: bool = True,
             max_retries: int = 3,
             retry_waiting_time: int = 5,
-            pool_size: int = 10,
+            pool_size: int = 20,
+            mincached: int = 5,
+            maxcached: int = 10,
             primary_key: str = 'id',
             date_range: Optional[List[str]] = None,
             recent_month: Optional[int] = None,
@@ -83,6 +85,8 @@ class MySQLDeduplicator:
         :param exclude_databases: 排除的数据库名列表
         :param exclude_tables: 排除的表名字典 {数据库名: [表名, ...]}
         :param duplicate_keep_mode: 'keep_one'（默认，重复组保留一条），'remove_all'（全部删除重复组）
+        :param mincached: 连接池空闲连接数
+        :param maxcached: 连接池最大缓存连接数
         """
         # 连接池状态标志
         self._closed = False
@@ -100,7 +104,9 @@ class MySQLDeduplicator:
             password=password,
             charset=charset,
             maxconnections=pool_size,
-            cursorclass=pymysql.cursors.DictCursor
+            cursorclass=pymysql.cursors.DictCursor,
+            mincached=mincached,
+            maxcached=maxcached,
         )
 
         # 并发模式要将 pool_size 加大
@@ -508,7 +514,7 @@ class MySQLDeduplicator:
                         dup_count_row = cursor.fetchone()
                         dup_count = dup_count_row['cnt'] if dup_count_row and 'cnt' in dup_count_row else 0
                         if dup_count == 0:
-                            logger.debug('没有重复数据', {"库": database, "表": table, "数据量": total_count, "数据日期": date_val})
+                            logger.info('没有重复数据', {"库": database, "表": table, "数据量": total_count, "数据日期": date_val})
                             cursor.execute(drop_temp_sql)
                             conn.commit()
                             return (0, 0)
@@ -1357,7 +1363,7 @@ def main():
         section='mysql',
         keys=['host', 'port', 'username', 'password'],
     )
-    # host = 'localhost'
+    host = 'localhost'
     
     deduplicator = MySQLDeduplicator(
         username=username,
@@ -1369,11 +1375,13 @@ def main():
         skip_system_dbs=True,
         max_retries=3,
         retry_waiting_time=5,
-        pool_size=30,
-        # recent_month=1,
+        pool_size=20,
+        mincached=5,
+        maxcached=10,
+        recent_month=1,
         # date_range=['2025-06-09', '2025-06-10'],
         exclude_columns=['更新时间'],
-        # exclude_databases=['测试库4'],
+        exclude_databases=['cookie文件', '日志', '视频数据', '云电影'],
         # exclude_tables={
         #     '推广数据2': [
         #         '地域报表_城市_2025_04', 
@@ -1387,7 +1395,7 @@ def main():
     )
 
     # 全库去重(单线程)
-    # deduplicator.deduplicate_all(dry_run=False, parallel=True, reorder_id=True)
+    deduplicator.deduplicate_all(dry_run=False, parallel=True, reorder_id=True)
 
     # # 指定数据库去重(多线程)
     # deduplicator.deduplicate_database('数据引擎2', dry_run=False, parallel=True, reorder_id=True)

mdbq/other/download_sku_picture.py

@@ -119,6 +119,12 @@ class LoadAccount:
             os.environ["webdriver.chrome.driver"] = chrome_path
             # option.binary_location = chrome_path  # Macos 设置此参数报错
             service = Service(chromedriver_path)
+        elif platform.system().lower() == 'linux':
+            # ubuntu
+            chrome_path = '/usr/bin/google-chrome'
+            chromedriver_path = '/usr/local/bin/chromedriver'
+            # option.binary_location = chrome_path  # macOS 设置此参数有效
+            service = Service(chromedriver_path)
         else:
             chrome_path = '/usr/local/chrome/Google Chrome for Testing.app'
             chromedriver_path = '/usr/local/chrome/chromedriver'

mdbq/spider/aikucun.py

@@ -125,6 +125,12 @@ class AikuCun:
             os.environ["webdriver.chrome.driver"] = chrome_path
             # option.binary_location = chrome_path  # Macos 设置此参数报错
             service = Service(chromedriver_path)
+        elif platform.system().lower() == 'linux':
+            # ubuntu
+            chrome_path = '/usr/bin/google-chrome'
+            chromedriver_path = '/usr/local/bin/chromedriver'
+            # option.binary_location = chrome_path  # macOS 设置此参数有效
+            service = Service(chromedriver_path)
         else:
             chrome_path = '/usr/local/chrome/Google Chrome for Testing.app'
             chromedriver_path = '/usr/local/chrome/chromedriver'

{mdbq-4.0.28.dist-info → mdbq-4.0.30.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.28
+Version: 4.0.30
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.0.28.dist-info → mdbq-4.0.30.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=xMsXHu_A2cflhY5yKH_XTsHjVS0UvcjRyCC0GhRxiV0,18
+mdbq/__version__.py,sha256=dvApMa3Ja0z33lQcWZtoQiZp-2Lxso9cz0yfXlPVDUQ,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/query_data.py,sha256=UWwVcpOo9Ro6u9VtXcr8T8Hdh8Atem4yjO4wCsvcOTw,166882
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
@@ -8,13 +8,13 @@ mdbq/myconf/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/myconf/myconf.py,sha256=GR250mf2KKImRUamPM2TEi9no_65tR4uKXn7eHNCAmg,31205
 mdbq/myconf/myconf_bak.py,sha256=39tLUBVlWQZzQfrwk7YoLEfipo11fpwWjaLBHcUt2qM,33341
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/deduplicator.py,sha256=0cyGZIHqWM1ELCPrepLtb51jludzvKMIeWFV7jPuIp4,73225
+mdbq/mysql/deduplicator.py,sha256=d8eDUvoNAE1OoquK45iCGc0mk1sP9orRgRQjRQIG_7w,73540
 mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
 mdbq/mysql/s_query.py,sha256=nByjtk5G18zgpomldb_5xDUAwBPXyXAcV1zvq5vkM_4,45005
 mdbq/mysql/unique_.py,sha256=XHDBiY-zF_Hmt28Jf-xh3E-_sIWoJCxdi2dOAfnAyQ0,21127
 mdbq/mysql/uploader.py,sha256=jW4aHm3kvt3EJ4tgnMl2fRPLY06VWaBv8JI5LSSMVzE,81151
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
-mdbq/other/download_sku_picture.py,sha256=MfHBBffM8oWiso2jbrrIET7CQCkypSov0SYOoSN-PqA,44798
+mdbq/other/download_sku_picture.py,sha256=EBRgO4L4k1AoCLehtCY_DvjUlYUNj_9CcePnsSKjx5Q,45114
 mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
 mdbq/other/pov_city.py,sha256=AEOmCOzOwyjHi9LLZWPKi6DUuSC-_M163664I52u9qw,21050
 mdbq/other/ua_sj.py,sha256=JuVYzc_5QZ9s_oQSrTHVKkQv4S_7-CWx4oIKOARn_9U,22178
@@ -24,8 +24,8 @@ mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=vpBuNc22uj9Vr-_Dh25_wpwWM1e-072EAAIBdB_IpL0,23494
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=juOqpr_dHeE1RyjCu67VcpzoJAWMO7FKv0i8KiH8WUo,21552
-mdbq-4.0.28.dist-info/METADATA,sha256=ALYXGbIVSK6h34xxXxs_DSGkNrMTRZzch1nv3ge1IGg,364
-mdbq-4.0.28.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-4.0.28.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.0.28.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=XptHjGzbout9IYzWAOQUpMMV5qEgLTU8pL1ZGt8oNEA,21868
+mdbq-4.0.30.dist-info/METADATA,sha256=NQZCJJKOfy3kdneuarm2UbrvbG3C5S4gDCDPxOS60ls,364
+mdbq-4.0.30.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-4.0.30.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.0.30.dist-info/RECORD,,