mdbq 4.0.11__tar.gz → 4.0.13__tar.gz

{mdbq-4.0.11 → mdbq-4.0.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.11
+Version: 4.0.13
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

mdbq-4.0.13/mdbq/__version__.py

@@ -0,0 +1 @@
+VERSION = '4.0.13'

{mdbq-4.0.11 → mdbq-4.0.13}/mdbq/aggregation/query_data.py

@@ -3,7 +3,7 @@ import re
 # from mdbq.mysql import mysql
 from mdbq.mysql import uploader
 from mdbq.mysql import s_query
-from mdbq.config import config
+from mdbq.conf import conf
 from mdbq.log import mylogger
 import datetime
 from dateutil.relativedelta import relativedelta
@@ -18,10 +18,13 @@ from collections.abc import Mapping, Sequence
 import inspect
 
 dir_path = os.path.expanduser("~")
-config_file = os.path.join(dir_path, 'spd.txt')
-content = config.read_config(file_path=config_file)
-username, password, host, port = content['username'], content['password'], content['host'], content['port']
-host = 'localhost'
+parser = conf.ConfigParser()
+host, port, username, password = parser.get_section_values(
+    file_path=os.path.join(dir_path, 'spd.txt'),
+    section='mysql',
+    keys=['host', 'port', 'username', 'password'],
+)
+# host = 'localhost'
 uld = uploader.MySQLUploader(username=username, password=password, host=host, port=int(port), pool_size=10)
 
 logger = mylogger.MyLogger(
@@ -3685,7 +3688,7 @@ def main(months=3):
         password=password, 
         host=host, 
         port=port,
-        maxconnections=20,
+        pool_size=20,
         )
     query1(download_manager=download_manager, months=months)
     query2(download_manager=download_manager, months=months)

{mdbq-4.0.11 → mdbq-4.0.13}/mdbq/mysql/deduplicator.py

@@ -6,7 +6,7 @@ import warnings
 import pymysql
 import os
 from mdbq.log import mylogger
-from mdbq.config import config
+from mdbq.conf import conf
 from typing import List, Dict, Optional, Any, Tuple
 from dbutils.pooled_db import PooledDB
 import threading
@@ -1351,8 +1351,12 @@ class MySQLDeduplicator:
 def main():
     logger.info('去重任务开始')
     dir_path = os.path.expanduser("~")
-    my_cont = config.read_config(file_path=os.path.join(dir_path, 'spd.txt'))
-    username, password, host, port = my_cont['username'], my_cont['password'], my_cont['host'], int(my_cont['port'])
+    parser = conf.ConfigParser()
+    host, port, username, password = parser.get_section_values(
+        file_path=os.path.join(dir_path, 'spd.txt'),
+        section='mysql',
+        keys=['host', 'port', 'username', 'password'],
+    )
     # host = 'localhost'
     
     deduplicator = MySQLDeduplicator(

{mdbq-4.0.11 → mdbq-4.0.13}/mdbq/mysql/s_query.py

@@ -7,7 +7,7 @@ from decimal import Decimal
 from contextlib import closing
 from mdbq.log import mylogger
 import os
-from mdbq.config import config
+from mdbq.conf import conf
 from typing import Optional, Dict, List, Set, Tuple, Union, Any, Literal
 from dbutils.pooled_db import PooledDB
 import time
@@ -949,8 +949,12 @@ class QueryDatas:
 
 def main():
     dir_path = os.path.expanduser("~")
-    my_cont = config.read_config(file_path=os.path.join(dir_path, 'spd.txt'))
-    username, password, host, port = my_cont['username'], my_cont['password'], my_cont['host'], int(my_cont['port'])
+    parser = conf.ConfigParser()
+    host, port, username, password = parser.get_section_values(
+        file_path=os.path.join(dir_path, 'spd.txt'),
+        section='mysql',
+        keys=['host', 'port', 'username', 'password'],
+    )
     host = 'localhost'
     
     qd = QueryDatas(username=username, password=password, host=host, port=port)

{mdbq-4.0.11 → mdbq-4.0.13}/mdbq/mysql/unique_.py

@@ -2,7 +2,7 @@ import re
 import pymysql
 from typing import List, Dict, Any, Tuple
 from mdbq.log import mylogger
-from mdbq.config import config
+from mdbq.conf import conf
 from dbutils.pooled_db import PooledDB
 import os
 
@@ -274,8 +274,12 @@ class UniqueManager:
 
 def main():
     dir_path = os.path.expanduser("~")
-    my_cont = config.read_config(file_path=os.path.join(dir_path, 'spd.txt'))
-    username, password, host, port = my_cont['username'], my_cont['password'], my_cont['host'], int(my_cont['port'])
+    parser = conf.ConfigParser()
+    host, port, username, password = parser.get_section_values(
+        file_path=os.path.join(dir_path, 'spd.txt'),
+        section='mysql',
+        keys=['host', 'port', 'username', 'password'],
+    )
     # host = 'localhost'
 
     my_databases = [

{mdbq-4.0.11 → mdbq-4.0.13}/mdbq/mysql/uploader.py

@@ -8,7 +8,7 @@ import pymysql
 import pandas as pd
 import os
 from mdbq.log import mylogger
-from mdbq.config import config
+from mdbq.conf import conf
 from typing import Union, List, Dict, Optional, Any, Tuple, Set
 from dbutils.pooled_db import PooledDB
 import json
@@ -1737,8 +1737,12 @@ class MySQLUploader:
 
 def main():
     dir_path = os.path.expanduser("~")
-    my_cont = config.read_config(file_path=os.path.join(dir_path, 'spd.txt'))
-    username, password, host, port = my_cont['username'], my_cont['password'], my_cont['host'], int(my_cont['port'])
+    parser = conf.ConfigParser()
+    host, port, username, password = parser.get_section_values(
+        file_path=os.path.join(dir_path, 'spd.txt'),
+        section='mysql',
+        keys=['host', 'port', 'username', 'password'],
+    )
     host = 'localhost'
 
     uploader = MySQLUploader(

{mdbq-4.0.11 → mdbq-4.0.13}/mdbq/other/download_sku_picture.py

@@ -17,8 +17,7 @@ from selenium.webdriver.support.wait import WebDriverWait
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.chrome.service import Service
-from mdbq.config import set_support
-from mdbq.config import config
+from mdbq.conf import conf
 from mdbq.mysql import mysql
 from mdbq.mysql import s_query
 from mdbq.other import ua_sj
@@ -49,8 +48,12 @@ if not os.path.exists(upload_path):  # 数据中心根目录
 
 dir_path = os.path.expanduser("~")
 config_file = os.path.join(dir_path, 'spd.txt')
-content = config.read_config(file_path=config_file)
-username, password, host, port = content['username'], content['password'], content['host'], content['port']
+parser = conf.ConfigParser()
+host, port, username, password = parser.get_section_values(
+    file_path=config_file,
+    section='mysql',
+    keys=['host', 'port', 'username', 'password'],
+)
 m_engine = mysql.MysqlUpload(username=username, password=password, host=host, port=port, charset='utf8mb4')
 
 if not username:
@@ -62,7 +65,7 @@ class LoadAccount:
 
     def __init__(self):
         self.url = 'https://login.taobao.com/'  # 默认登录淘宝
-        self.cookie_path = os.path.join(set_support.SetSupport(dirname='support').dirname, 'cookies')
+        self.cookie_path = None
 
     def __call__(self, *args, **kwargs):
         self.check_cookie()  # 检测cookie有效期, 但不阻断任务

{mdbq-4.0.11 → mdbq-4.0.13}/mdbq/spider/aikucun.py

@@ -15,15 +15,18 @@ from selenium.webdriver.chrome.service import Service
 import pymysql
 from mdbq.mysql import uploader
 from mdbq.mysql import s_query
-from mdbq.config import config
+from mdbq.conf import conf
 from mdbq.other import ua_sj
 from mdbq.other import otk
 from mdbq.log import mylogger
 
 dir_path = os.path.expanduser("~")
-config_file = os.path.join(dir_path, 'spd.txt')
-content = config.read_config(file_path=config_file)
-username, password, host, port = content['username'], content['password'], content['host'], content['port']
+parser = conf.ConfigParser()
+host, port, username, password = parser.get_section_values(
+    file_path=os.path.join(dir_path, 'spd.txt'),
+    section='mysql',
+    keys=['host', 'port', 'username', 'password'],
+)
 
 # 实例化一个数据查询类，用来获取 cookies 表数据
 logger = mylogger.MyLogger(

{mdbq-4.0.11 → mdbq-4.0.13}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.11
+Version: 4.0.13
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.0.11 → mdbq-4.0.13}/mdbq.egg-info/SOURCES.txt

@@ -8,8 +8,6 @@ mdbq.egg-info/dependency_links.txt
 mdbq.egg-info/top_level.txt
 mdbq/aggregation/__init__.py
 mdbq/aggregation/query_data.py
-mdbq/config/__init__.py
-mdbq/config/config.py
 mdbq/log/__init__.py
 mdbq/log/mylogger.py
 mdbq/mysql/__init__.py

mdbq-4.0.11/mdbq/__version__.py

@@ -1 +0,0 @@
-VERSION = '4.0.11'

mdbq-4.0.11/mdbq/config/config.py

@@ -1,95 +0,0 @@
-import os.path
-import re
-
-
-def read_config(file_path):
-    """读取配置文件，返回字典"""
-    if not os.path.isfile(file_path):
-        print(f'配置文件不存在: {file_path}')
-        return
-    config = {}
-    with open(file_path, 'r', encoding='utf-8') as file:
-        for line in file:
-            stripped_line = line.strip()
-            # 跳过空行和注释行（以 # 或 // 开头）
-            if not stripped_line or stripped_line.startswith(('#', '//')):
-                continue
-            # 处理行内注释（# 或 // 前有空格）
-            comment_match = re.search(r'\s+[#//]', line)
-            if comment_match:
-                line = line[:comment_match.start()].strip()
-            else:
-                line = line.strip()
-            # 解析键值对
-            if '=' in line:
-                key, value = line.split('=', 1)
-                config[key.strip()] = value.strip()
-    return config
-
-
-def write_config(file_path, rewrite):
-    """
-    更新配置文件中的键值对，保留注释和其他内容，修复等号空格问题
-    示例：
-    write_config('spd.txt', {'is_spider': True})
-    """
-    # 读取所有行到内存
-    try:
-        with open(file_path, 'r', encoding='utf-8') as file:
-            lines = file.readlines()
-    except FileNotFoundError:
-        with open(file_path, 'w', encoding='utf-8') as file:
-            lines = []
-
-    new_lines = []
-    found_keys = set()
-
-    for line in lines:
-        stripped = line.strip()
-        if not stripped or stripped.startswith(('#', '//')):
-            new_lines.append(line)
-            continue
-
-        # 使用 partition 保留等号格式
-        key_part, sep, value_part = line.partition('=')
-        if not sep:  # 没有等号的行直接保留
-            new_lines.append(line)
-            continue
-
-        key = key_part.strip()
-        if key in rewrite:
-            # 处理值部分和注释
-            comment_match = re.search(r'\s+([#//].*)$', value_part)
-            if comment_match:
-                comment = comment_match.group(0)
-                raw_value = value_part[:comment_match.start()].rstrip()
-            else:
-                comment = ''
-                raw_value = value_part.strip()
-
-            # 保留原值前导空格
-            leading_space = re.match(r'^(\s*)', value_part).group(1)
-            new_value = f"{leading_space}{rewrite[key]}{comment}"
-
-            # 构建新行（保留原等号格式）
-            new_line = f"{key_part}{sep}{new_value}\n"
-            new_lines.append(new_line)
-            found_keys.add(key)
-        else:
-            new_lines.append(line)
-
-    # 添加新键值对
-    for key in rewrite:
-        if key not in found_keys:
-            new_lines.append(f"{key} = {rewrite[key]}\n")
-
-    # 写入文件
-    with open(file_path, 'w', encoding='utf-8') as file:
-        file.writelines(new_lines)
-
-
-if __name__ == '__main__':
-    res = read_config('/Users/xigua/数据中心2/spider/spd.txt')
-    print(res)
-    # write_config('spd.txt', {'is_spider': False})
-

mdbq-4.0.11/mdbq/other/__init__.py

@@ -1,4 +0,0 @@
-
-
-
-# 配置文件