mdbq 4.0.11__py3-none-any.whl → 4.0.13__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '4.0.11'
+VERSION = '4.0.13'

mdbq/aggregation/query_data.py

@@ -3,7 +3,7 @@ import re
 # from mdbq.mysql import mysql
 from mdbq.mysql import uploader
 from mdbq.mysql import s_query
-from mdbq.config import config
+from mdbq.conf import conf
 from mdbq.log import mylogger
 import datetime
 from dateutil.relativedelta import relativedelta
@@ -18,10 +18,13 @@ from collections.abc import Mapping, Sequence
 import inspect
 
 dir_path = os.path.expanduser("~")
-config_file = os.path.join(dir_path, 'spd.txt')
-content = config.read_config(file_path=config_file)
-username, password, host, port = content['username'], content['password'], content['host'], content['port']
-host = 'localhost'
+parser = conf.ConfigParser()
+host, port, username, password = parser.get_section_values(
+    file_path=os.path.join(dir_path, 'spd.txt'),
+    section='mysql',
+    keys=['host', 'port', 'username', 'password'],
+)
+# host = 'localhost'
 uld = uploader.MySQLUploader(username=username, password=password, host=host, port=int(port), pool_size=10)
 
 logger = mylogger.MyLogger(
@@ -3685,7 +3688,7 @@ def main(months=3):
         password=password, 
         host=host, 
         port=port,
-        maxconnections=20,
+        pool_size=20,
         )
     query1(download_manager=download_manager, months=months)
     query2(download_manager=download_manager, months=months)

mdbq/mysql/deduplicator.py

@@ -6,7 +6,7 @@ import warnings
 import pymysql
 import os
 from mdbq.log import mylogger
-from mdbq.config import config
+from mdbq.conf import conf
 from typing import List, Dict, Optional, Any, Tuple
 from dbutils.pooled_db import PooledDB
 import threading
@@ -1351,8 +1351,12 @@ class MySQLDeduplicator:
 def main():
     logger.info('去重任务开始')
     dir_path = os.path.expanduser("~")
-    my_cont = config.read_config(file_path=os.path.join(dir_path, 'spd.txt'))
-    username, password, host, port = my_cont['username'], my_cont['password'], my_cont['host'], int(my_cont['port'])
+    parser = conf.ConfigParser()
+    host, port, username, password = parser.get_section_values(
+        file_path=os.path.join(dir_path, 'spd.txt'),
+        section='mysql',
+        keys=['host', 'port', 'username', 'password'],
+    )
     # host = 'localhost'
     
     deduplicator = MySQLDeduplicator(

mdbq/mysql/s_query.py

@@ -7,7 +7,7 @@ from decimal import Decimal
 from contextlib import closing
 from mdbq.log import mylogger
 import os
-from mdbq.config import config
+from mdbq.conf import conf
 from typing import Optional, Dict, List, Set, Tuple, Union, Any, Literal
 from dbutils.pooled_db import PooledDB
 import time
@@ -949,8 +949,12 @@ class QueryDatas:
 
 def main():
     dir_path = os.path.expanduser("~")
-    my_cont = config.read_config(file_path=os.path.join(dir_path, 'spd.txt'))
-    username, password, host, port = my_cont['username'], my_cont['password'], my_cont['host'], int(my_cont['port'])
+    parser = conf.ConfigParser()
+    host, port, username, password = parser.get_section_values(
+        file_path=os.path.join(dir_path, 'spd.txt'),
+        section='mysql',
+        keys=['host', 'port', 'username', 'password'],
+    )
     host = 'localhost'
     
     qd = QueryDatas(username=username, password=password, host=host, port=port)

mdbq/mysql/unique_.py

@@ -2,7 +2,7 @@ import re
 import pymysql
 from typing import List, Dict, Any, Tuple
 from mdbq.log import mylogger
-from mdbq.config import config
+from mdbq.conf import conf
 from dbutils.pooled_db import PooledDB
 import os
 
@@ -274,8 +274,12 @@ class UniqueManager:
 
 def main():
     dir_path = os.path.expanduser("~")
-    my_cont = config.read_config(file_path=os.path.join(dir_path, 'spd.txt'))
-    username, password, host, port = my_cont['username'], my_cont['password'], my_cont['host'], int(my_cont['port'])
+    parser = conf.ConfigParser()
+    host, port, username, password = parser.get_section_values(
+        file_path=os.path.join(dir_path, 'spd.txt'),
+        section='mysql',
+        keys=['host', 'port', 'username', 'password'],
+    )
     # host = 'localhost'
 
     my_databases = [

mdbq/mysql/uploader.py

@@ -8,7 +8,7 @@ import pymysql
 import pandas as pd
 import os
 from mdbq.log import mylogger
-from mdbq.config import config
+from mdbq.conf import conf
 from typing import Union, List, Dict, Optional, Any, Tuple, Set
 from dbutils.pooled_db import PooledDB
 import json
@@ -1737,8 +1737,12 @@ class MySQLUploader:
 
 def main():
     dir_path = os.path.expanduser("~")
-    my_cont = config.read_config(file_path=os.path.join(dir_path, 'spd.txt'))
-    username, password, host, port = my_cont['username'], my_cont['password'], my_cont['host'], int(my_cont['port'])
+    parser = conf.ConfigParser()
+    host, port, username, password = parser.get_section_values(
+        file_path=os.path.join(dir_path, 'spd.txt'),
+        section='mysql',
+        keys=['host', 'port', 'username', 'password'],
+    )
     host = 'localhost'
 
     uploader = MySQLUploader(

mdbq/other/download_sku_picture.py

@@ -17,8 +17,7 @@ from selenium.webdriver.support.wait import WebDriverWait
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.chrome.service import Service
-from mdbq.config import set_support
-from mdbq.config import config
+from mdbq.conf import conf
 from mdbq.mysql import mysql
 from mdbq.mysql import s_query
 from mdbq.other import ua_sj
@@ -49,8 +48,12 @@ if not os.path.exists(upload_path):  # 数据中心根目录
 
 dir_path = os.path.expanduser("~")
 config_file = os.path.join(dir_path, 'spd.txt')
-content = config.read_config(file_path=config_file)
-username, password, host, port = content['username'], content['password'], content['host'], content['port']
+parser = conf.ConfigParser()
+host, port, username, password = parser.get_section_values(
+    file_path=config_file,
+    section='mysql',
+    keys=['host', 'port', 'username', 'password'],
+)
 m_engine = mysql.MysqlUpload(username=username, password=password, host=host, port=port, charset='utf8mb4')
 
 if not username:
@@ -62,7 +65,7 @@ class LoadAccount:
 
     def __init__(self):
         self.url = 'https://login.taobao.com/'  # 默认登录淘宝
-        self.cookie_path = os.path.join(set_support.SetSupport(dirname='support').dirname, 'cookies')
+        self.cookie_path = None
 
     def __call__(self, *args, **kwargs):
         self.check_cookie()  # 检测cookie有效期, 但不阻断任务

mdbq/spider/aikucun.py

@@ -15,15 +15,18 @@ from selenium.webdriver.chrome.service import Service
 import pymysql
 from mdbq.mysql import uploader
 from mdbq.mysql import s_query
-from mdbq.config import config
+from mdbq.conf import conf
 from mdbq.other import ua_sj
 from mdbq.other import otk
 from mdbq.log import mylogger
 
 dir_path = os.path.expanduser("~")
-config_file = os.path.join(dir_path, 'spd.txt')
-content = config.read_config(file_path=config_file)
-username, password, host, port = content['username'], content['password'], content['host'], content['port']
+parser = conf.ConfigParser()
+host, port, username, password = parser.get_section_values(
+    file_path=os.path.join(dir_path, 'spd.txt'),
+    section='mysql',
+    keys=['host', 'port', 'username', 'password'],
+)
 
 # 实例化一个数据查询类，用来获取 cookies 表数据
 logger = mylogger.MyLogger(

{mdbq-4.0.11.dist-info → mdbq-4.0.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.11
+Version: 4.0.13
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.0.11.dist-info → mdbq-4.0.13.dist-info}/RECORD

@@ -1,19 +1,17 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=PQJs_Lgx6OvamcsXbLCVuBAvLc7j2xwJDZEWigwyUy8,18
+mdbq/__version__.py,sha256=3xO8DhCz_Dag4SoPDVXTcch1mlh-rJ-HFhoFON-nJiQ,18
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/query_data.py,sha256=SM8cS9lBKmhLBQdwJz-sRu9bl7w1HS0MEq10s6Tqf_0,166777
-mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
-mdbq/config/config.py,sha256=eaTfrfXQ65xLqjr5I8-HkZd_jEY1JkGinEgv3TSLeoQ,3170
+mdbq/aggregation/query_data.py,sha256=1J6ujDa2QVCp5-9sfkHSwxqv3vV4sZ6w0CZGp-xuQUM,166777
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=9w_o5mYB3FooIxobq_lSa6oCYTKIhPxDFox-jeLtUHI,21714
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/deduplicator.py,sha256=fS1dSs92vN15tuqmAKrUVdKk6z9dwW_Fe9WHMBYsy2U,73172
+mdbq/mysql/deduplicator.py,sha256=h1tpWnfPwDgOyhblippsl8saPIi4f4e1bZvtML6pRt0,73209
 mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
-mdbq/mysql/s_query.py,sha256=RPC-KZVuqPlCSmpmtUmYAOJdxJT01i0DvlIbmum4MxM,42882
-mdbq/mysql/unique_.py,sha256=Wgqq_PjAAD757JTa10wjYaJgssZ_C_ypU6DW56jbuyw,21074
-mdbq/mysql/uploader.py,sha256=wX2gHhVQJwGErnjUbLnsljkZ8Yd3YK-HS3P7q8DizAA,81053
+mdbq/mysql/s_query.py,sha256=pcqUSJXla9xCITiBpcsPcPeduUcZt_RQ9r0x4f22vqc,42919
+mdbq/mysql/unique_.py,sha256=d9bEdwhlxc6oZs3IR-9vd__1FOzGxLWwGUSCHoppcZg,21111
+mdbq/mysql/uploader.py,sha256=cJ_Bs_AYlLPcMh1TevkGfzFSACUDgsM8lQXE799a6Q8,81090
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
-mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
+mdbq/other/download_sku_picture.py,sha256=Pf_w7j6dBbTKS00Vt4qceG1v5KaEHvz3FBFeG3BQbGE,44752
 mdbq/other/otk.py,sha256=iclBIFbQbhlqzUbcMMoePXBpcP1eZ06ZtjnhcA_EbmE,7241
 mdbq/other/pov_city.py,sha256=AEOmCOzOwyjHi9LLZWPKi6DUuSC-_M163664I52u9qw,21050
 mdbq/other/ua_sj.py,sha256=JuVYzc_5QZ9s_oQSrTHVKkQv4S_7-CWx4oIKOARn_9U,22178
@@ -23,8 +21,8 @@ mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=vpBuNc22uj9Vr-_Dh25_wpwWM1e-072EAAIBdB_IpL0,23494
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=7oquQ2RIJr6B1xblQMfnmHzteOlvHA7dIcPRaAPfHBc,21546
-mdbq-4.0.11.dist-info/METADATA,sha256=zZh35aA-suJ3B_v39Mw8V_O2GSdOLdylfNPl_E99uqQ,364
-mdbq-4.0.11.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-4.0.11.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.0.11.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=Ld46z_29n1XgDwyESY_kCHHjEF3M3LLJxiv0mFhT4sw,21546
+mdbq-4.0.13.dist-info/METADATA,sha256=0XFWPQ6rPoRx09hOwChOidIbM_RvquGwcCNlzT9dHKo,364
+mdbq-4.0.13.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-4.0.13.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.0.13.dist-info/RECORD,,

mdbq/config/__init__.py

@@ -1,4 +0,0 @@
-
-
-
-# 配置文件

mdbq/config/config.py

@@ -1,95 +0,0 @@
-import os.path
-import re
-
-
-def read_config(file_path):
-    """读取配置文件，返回字典"""
-    if not os.path.isfile(file_path):
-        print(f'配置文件不存在: {file_path}')
-        return
-    config = {}
-    with open(file_path, 'r', encoding='utf-8') as file:
-        for line in file:
-            stripped_line = line.strip()
-            # 跳过空行和注释行（以 # 或 // 开头）
-            if not stripped_line or stripped_line.startswith(('#', '//')):
-                continue
-            # 处理行内注释（# 或 // 前有空格）
-            comment_match = re.search(r'\s+[#//]', line)
-            if comment_match:
-                line = line[:comment_match.start()].strip()
-            else:
-                line = line.strip()
-            # 解析键值对
-            if '=' in line:
-                key, value = line.split('=', 1)
-                config[key.strip()] = value.strip()
-    return config
-
-
-def write_config(file_path, rewrite):
-    """
-    更新配置文件中的键值对，保留注释和其他内容，修复等号空格问题
-    示例：
-    write_config('spd.txt', {'is_spider': True})
-    """
-    # 读取所有行到内存
-    try:
-        with open(file_path, 'r', encoding='utf-8') as file:
-            lines = file.readlines()
-    except FileNotFoundError:
-        with open(file_path, 'w', encoding='utf-8') as file:
-            lines = []
-
-    new_lines = []
-    found_keys = set()
-
-    for line in lines:
-        stripped = line.strip()
-        if not stripped or stripped.startswith(('#', '//')):
-            new_lines.append(line)
-            continue
-
-        # 使用 partition 保留等号格式
-        key_part, sep, value_part = line.partition('=')
-        if not sep:  # 没有等号的行直接保留
-            new_lines.append(line)
-            continue
-
-        key = key_part.strip()
-        if key in rewrite:
-            # 处理值部分和注释
-            comment_match = re.search(r'\s+([#//].*)$', value_part)
-            if comment_match:
-                comment = comment_match.group(0)
-                raw_value = value_part[:comment_match.start()].rstrip()
-            else:
-                comment = ''
-                raw_value = value_part.strip()
-
-            # 保留原值前导空格
-            leading_space = re.match(r'^(\s*)', value_part).group(1)
-            new_value = f"{leading_space}{rewrite[key]}{comment}"
-
-            # 构建新行（保留原等号格式）
-            new_line = f"{key_part}{sep}{new_value}\n"
-            new_lines.append(new_line)
-            found_keys.add(key)
-        else:
-            new_lines.append(line)
-
-    # 添加新键值对
-    for key in rewrite:
-        if key not in found_keys:
-            new_lines.append(f"{key} = {rewrite[key]}\n")
-
-    # 写入文件
-    with open(file_path, 'w', encoding='utf-8') as file:
-        file.writelines(new_lines)
-
-
-if __name__ == '__main__':
-    res = read_config('/Users/xigua/数据中心2/spider/spd.txt')
-    print(res)
-    # write_config('spd.txt', {'is_spider': False})
-