mdbq 3.7.15__py3-none-any.whl → 3.7.17__py3-none-any.whl

mdbq/aggregation/optimize_data.py

@@ -1,7 +1,6 @@
 # -*- coding: UTF-8 –*-
 from mdbq.mysql import mysql
 from mdbq.config import default
-import socket
 import subprocess
 import psutil
 import time
@@ -10,7 +9,8 @@ import logging
 """
 对指定数据库所有冗余数据进行清理
 """
-m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname='xigua_lx', sql='mysql', local='remoto', config_file=None)
+targe_host, hostname, local =  default.return_default_host()
+m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname=hostname, sql='mysql', local=local, config_file=None)
 if not username:
     logger.info(f'找不到主机：')
 

mdbq/aggregation/query_data.py

@@ -1,6 +1,5 @@
 # -*- coding: UTF-8 –*-
 import re
-import socket
 from mdbq.mysql import mysql
 from mdbq.mysql import s_query
 from mdbq.aggregation import optimize_data
@@ -26,7 +25,8 @@ import sys
 
 """
 dirname = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'support')
-m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname='xigua_lx', sql='mysql', local='remoto', config_file=None)
+targe_host, hostname, local =  default.return_default_host()
+m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname=hostname, sql='mysql', local=local, config_file=None)
 logger = logging.getLogger(__name__)
 
 

mdbq/config/default.py

@@ -2,22 +2,29 @@
 import os
 import sys
 import json
-# import platform
-# import getpass
 import socket
 import logging
 from mdbq.mysql import mysql
 
-# if platform.system() == 'Windows':
-#     support_path = r'C:\数据中心2\support'
-# elif platform.system() == 'Darwin':
-#     support_path = f'/Users/{getpass.getuser()}/数据中心2/support'
-# else:
-#     support_path = '数据中心2/support'  # 没有用, 可以删
 support_path = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'support')
 logger = logging.getLogger(__name__)
 
 
+def return_default_host():
+    targe_host = socket.gethostname()
+    hostname = 'xigua_lx'  # 正常情况下 spider_tg 只会在 xigua_lx 或者 xigua1 运行，且都指向主库
+    local = 'remoto'
+
+    if targe_host == 'xigua_lx':
+        local = 'local'  # 直接指向局域网地址
+    elif targe_host in ['localhost.localdomain', 'xigua1']:
+        targe_host = 'xigua1'  # 修正 Linux 系统用户名
+        local = 'local'  # 直接指向局域网地址
+    elif targe_host == 'MacBookPro':
+        local = 'local'  # 直接指向局域网地址
+    return targe_host, hostname, local
+
+
 def get_mysql_engine(platform='Windows', hostname='xigua_lx', sql='mysql', local='remoto', config_file=None):
     if not config_file:
         config_file = os.path.join(support_path, 'my_config.txt')
@@ -36,76 +43,7 @@ def get_mysql_engine(platform='Windows', hostname='xigua_lx', sql='mysql', local
     return _engine, username, password, host, port
 
 
-def return_host(conf_data):
-    """
-    从配置文件数据中获取: username, password, host, port
-    :param conf_data:
-    :return:
-    """
-    return conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
-
-
-def return_one_engine(conf_data):
-    """
-    返回一个 engine
-    """
-    username, password, host, port = return_host(conf_data)
-    return mysql.MysqlUpload(username=username, password=password, host=host, port=port, charset='utf8mb4')
-
-
-def get_hostname(platform, hostname, sql, local):
-    """
-    返回一个主机的: username, password, host, port
-    """
-    config_file = os.path.join(support_path, 'my_config.txt')
-    with open(config_file, 'r', encoding='utf-8') as f:
-        conf = json.load(f)
-    conf_data = conf[platform][hostname][sql][local]
-    return return_host(conf_data)
-
-
-def get_engine_bak():
-    """
-    要删除
-    """
-    if not os.path.isdir(support_path):
-        print(f'缺少配置文件，无法读取配置文件： {file}')
-        return
-    config_file = os.path.join(support_path, 'my_config.txt')
-
-    with open(config_file, 'r', encoding='utf-8') as f:
-        conf = json.load(f)
-
-    if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
-        conf_data = conf['Windows']['xigua_lx']['mysql']['remoto']
-        m_engine = return_one_engine(conf_data=conf_data)
-        conf_data = conf['Windows']['company']['mysql']['local']
-        company_engine = return_one_engine(conf_data=conf_data)
-        username, password, host, port = return_host(conf_data)  # 顺序不能乱
-        conf_data = conf['Windows']['xigua_ts']['mysql']['remoto']
-        ts_engine = return_one_engine(conf_data=conf_data)
-    elif socket.gethostname() == 'xigua_lx' or socket.gethostname() == 'xigua1' or socket.gethostname() == 'MacBookPro':
-        conf_data = conf['Windows']['xigua_lx']['mysql']['local']
-        m_engine = return_one_engine(conf_data=conf_data)
-        username, password, host, port = return_host(conf_data)  # 顺序不能乱
-        conf_data = conf['Windows']['company']['mysql']['remoto']
-        company_engine = return_one_engine(conf_data=conf_data)
-        conf_data = conf['Windows']['xigua_ts']['mysql']['remoto']
-        ts_engine = return_one_engine(conf_data=conf_data)
-
-    else:
-        conf_data = conf['Windows']['xigua_lx']['mysql']['remoto']
-        m_engine = return_one_engine(conf_data=conf_data)
-        username, password, host, port = return_host(conf_data)  # 顺序不能乱
-        conf_data = conf['Windows']['company']['mysql']['remoto']
-        company_engine = return_one_engine(conf_data=conf_data)
-        conf_data = conf['Windows']['xigua_ts']['mysql']['remoto']
-        ts_engine = return_one_engine(conf_data=conf_data)
-
-    return m_engine, company_engine, ts_engine, (username, password, host, port)
-
-
-def write_back(datas):
+def write_back_bak(datas):
     """ 将数据写回本地 """
     if not os.path.isdir(support_path):
         print(f'缺少配置文件，无法读取配置文件： {file}')
@@ -118,16 +56,3 @@ def write_back(datas):
 
 if __name__ == '__main__':
     pass
-    username, password, host, port = get_hostname(
-        platform='Windows',
-        hostname='xigua_lx',
-        sql='mysql',
-        local='remoto'
-    )
-    res = get_mysql_engine(
-        platform='Windows',
-        hostname='xigua_lx',
-        sql='mysql',
-        local='remoto'
-    )
-    print(res)

mdbq/config/myconfig.py

@@ -21,7 +21,7 @@ def main():
 
 
 def write_back(datas):
-    """ 将数据写回本地 """
+    """ 将数据写回本地   all-datas.py 调用 """
     # support_path = set_support.SetSupport(dirname='support').dirname
     file = os.path.join(support_path, 'my_config.txt')
     with open(file, 'w+', encoding='utf-8') as f:

mdbq/config/products.py

@@ -3,7 +3,6 @@ import json
 import os
 import platform
 import getpass
-import socket
 import pandas as pd
 from mdbq.mysql import mysql
 from mdbq.config import default
@@ -13,7 +12,8 @@ from numpy.ma.core import product
 天猫货品年份基准对照
 用于聚合数据，通过此数据表进一步可确定商品上架年月
 """
-m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname='xigua_lx', sql='mysql', local='remoto', config_file=None)
+targe_host, hostname, local =  default.return_default_host()
+m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname=hostname, sql='mysql', local=local, config_file=None)
 if not username:
     print(f'找不到主机1：')
 

mdbq/config/set_support.py

@@ -13,14 +13,6 @@ support 文件夹包含很多配置类文件，是程序必不可少的依赖
 class SetSupport:
     def __init__(self, dirname):
         self.dirname = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), dirname)
-        # if platform.system() == 'Windows':
-        #     self.dirname = f'C:\\数据中心2\\support'
-        # elif platform.system() == 'Darwin':
-        #     self.dirname = f'/Users/{getpass.getuser()}/数据中心2/support'
-        # else:
-        #     self.dirname = '数据中心2/support'  # 没有用, 可以删
-        # if not os.path.isdir(self.dirname):
-        #     os.mkdir(self.dirname)
 
 
 if __name__ == '__main__':

mdbq/log/spider_logging.py

@@ -10,15 +10,6 @@ def setup_logging(reMoveOldHandler=True):
     """
     reMoveOldHandler: 替换根日志记录器的所有现有处理器
     """
-    # if platform.system() == 'Windows':
-    #     from mdbq.pbix import refresh_all
-    #     D_PATH = os.path.join(f'C:\\Users\\{getpass.getuser()}\\Downloads')
-    # elif platform.system() == 'Linux':
-    #     D_PATH = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'Downloads')
-    #     if not os.path.exists(D_PATH):
-    #         os.makedirs(D_PATH)
-    # else:
-    #     D_PATH = os.path.join(f'/Users/{getpass.getuser()}/Downloads')
     D_PATH = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])))
 
     if not os.path.isdir(os.path.join(D_PATH, 'logfile')):

mdbq/other/download_sku_picture.py

@@ -4,7 +4,6 @@ import getpass
 import json
 import os
 import platform
-import socket
 import random
 from dateutil.relativedelta import relativedelta
 import re
@@ -50,7 +49,8 @@ upload_path = os.path.join(D_PATH, '数据上传中心')  # 此目录位于下
 if not os.path.exists(upload_path):  # 数据中心根目录
     os.makedirs(upload_path)
 
-m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname='xigua_lx', sql='mysql', local='remoto', config_file=None)
+targe_host, hostname, local =  default.return_default_host()
+m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname=hostname, sql='mysql', local=local, config_file=None)
 if not username:
     print(f'找不到主机：')
 

mdbq/redis/getredis.py

@@ -1,9 +1,8 @@
 # -*- coding: UTF-8 –*-
-import os.path
+import os
 import sys
 import random
 import redis
-import socket
 from mdbq.mysql import s_query
 from mdbq.config import default
 import pandas as pd
@@ -13,12 +12,11 @@ import datetime
 import threading
 import logging
 from logging.handlers import RotatingFileHandler
-import getpass
-import platform
 from decimal import Decimal
 import orjson
 
-m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname='xigua_lx', sql='mysql', local='remoto', config_file=None)
+targe_host, hostname, local =  default.return_default_host()
+m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname=hostname, sql='mysql', local=local, config_file=None)
 
 # 获取当前模块的日志记录器
 logger = logging.getLogger(__name__)
@@ -26,6 +24,8 @@ logger = logging.getLogger(__name__)
 # 创建一个文件处理器，用于将日志写入文件
 dir_path = os.path.realpath(os.path.dirname(sys.argv[0]))
 log_file = os.path.join(dir_path, 'logfile', 'redis.log')
+if not os.path.isdir(os.path.join(dir_path, 'logfile')):
+    os.mkdir(os.path.join(dir_path, 'logfile'))
 file_handler = RotatingFileHandler(log_file, maxBytes=3 * 1024 * 1024, backupCount=10, encoding='utf-8')  # 保留10个备份文件
 file_handler.setLevel(logging.INFO)  # 设置文件处理器的日志级别
 
@@ -633,5 +633,4 @@ if __name__ == '__main__':
     # )
     # logger.info(df)
     #
-
-    logger.info(socket.gethostname())
+    pass

mdbq/spider/aikucun.py

@@ -22,7 +22,6 @@ from mdbq.other import ua_sj
 from mdbq.mysql import mysql
 from mdbq.mysql import s_query
 from mdbq.config import default
-import socket
 
 warnings.filterwarnings('ignore')
 
@@ -38,7 +37,8 @@ else:
     D_PATH = str(pathlib.Path(f'/Users/{getpass.getuser()}/Downloads'))
 upload_path = os.path.join(D_PATH, '数据上传中心', '爱库存')  # 此目录位于下载文件夹
 
-m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname='xigua_lx', sql='mysql', local='remoto', config_file=None)
+targe_host, hostname, local =  default.return_default_host()
+m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname=hostname, sql='mysql', local=local, config_file=None)
 print(username, password, host, port)
 # 实例化一个数据查询类，用来获取 cookies 表数据
 download = s_query.QueryDatas(username=username, password=password, host=host, port=port)

{mdbq-3.7.15.dist-info → mdbq-3.7.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 3.7.15
+Version: 3.7.17
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.7.15.dist-info → mdbq-3.7.17.dist-info}/RECORD

@@ -1,28 +1,27 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/datashow_bak.py,sha256=1AYSIDkdUx-4as1Ax2rPj0cExM9d-qFMrFYLAaPHNuk,54962
-mdbq/aggregation/optimize_data.py,sha256=ZR6iXO25EMWyiy-oy_T8H8K9xA8q1ecC8gmUbOqJOOc,910
-mdbq/aggregation/query_data.py,sha256=jGKnHm9vRlKCZeIdP15lgpASKaQgoQjPZyd2IsTJY6k,174426
+mdbq/aggregation/optimize_data.py,sha256=wB7prQdZAHyjzXH9V8g8X_JBMdvCCUITN1hVwK72Tdg,952
+mdbq/aggregation/query_data.py,sha256=I2esDWtp_dg49bqvD-qIZIpF9Z2kyo8My_oWuo6RcC4,174468
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=hJs815hGFwm_X5bP2i9XugG2w2ZY_F0n3-Q0hVpIPPw,4892
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
-mdbq/config/default.py,sha256=LEDVIBR6aEG5iTKgNzNHOPEujVxp6MF-1i7TwqMM2BI,5092
-mdbq/config/myconfig.py,sha256=IEJOYoSiDivplE6N-1UWuVKUj6zLa4EOsuj3aONb85g,1000
-mdbq/config/products.py,sha256=FbBIqmyaiq9h03FIeE9W2bwbLm2_5pr6xyzPV-u7Ges,5689
-mdbq/config/set_support.py,sha256=eM2scqDzGNR2kkbYmguRB2ucrqeX0KMh5OMIaGsLem4,877
+mdbq/config/default.py,sha256=pOBmlxmGzmK8XYstv_qwE4vDwJmytEmVIYE0bDnCXOs,2143
+mdbq/config/myconfig.py,sha256=5gg3B8fnDx2_a5I3kfPLXx7xePqZwfb3ohj6iMVhplc,1022
+mdbq/config/products.py,sha256=02sZAhFyF5dpapXj9h5iOKMXvC4ZNZjHTnQZgbzc9NA,5731
+mdbq/config/set_support.py,sha256=7C7NFy7Em_uC7lig54qQlIlKG_AJeMCskxzK87anGkM,462
 mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
 mdbq/dataframe/converter.py,sha256=lETYhT7KXlWzWwqguqhk6vI6kj4rnOBEW1lhqKy2Abc,5035
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
-mdbq/log/spider_logging.py,sha256=cTI9BHePz9Q4QThc_OBxTLqX5tF9iPwIQ-dASpf6zCA,2142
+mdbq/log/spider_logging.py,sha256=59xe4Ckb7m-sBt3GYk8DC_hQg7-jnjBRq1o718r1Ry8,1676
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/mysql.py,sha256=bsv-khT7fyoYxEJMbJPks-V1tYvwX-mNHsoTNXfWiKk,95884
 mdbq/mysql/s_query.py,sha256=pj5ioJfUT81Su9S-km9G49gF5F2MmXXfw_oAIUzhN28,8794
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
-mdbq/other/download_sku_picture.py,sha256=uMC6H7SVEFj3cuS8IJNPOY3v4oJVtGOE-vb2XJ7Q0tM,44718
+mdbq/other/download_sku_picture.py,sha256=Da_5hsw3yeDm2lXE-YLMbIxk2-36K5-o7D3Sm9APj9M,44760
 mdbq/other/porxy.py,sha256=UHfgEyXugogvXgsG68a7QouUCKaohTKKkI4RN-kYSdQ,4961
 mdbq/other/pov_city.py,sha256=AEOmCOzOwyjHi9LLZWPKi6DUuSC-_M163664I52u9qw,21050
 mdbq/other/ua_sj.py,sha256=JuVYzc_5QZ9s_oQSrTHVKkQv4S_7-CWx4oIKOARn_9U,22178
@@ -31,10 +30,10 @@ mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,239
 mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,7192
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
-mdbq/redis/getredis.py,sha256=vEEoAfev4RxVso1cc_776bUoaiApsgfeKTHztm0bF5s,25417
+mdbq/redis/getredis.py,sha256=MLVWENvFtLFpxUlkUMBvc5bXPXeG-QIl8f_A200TIWE,25497
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=HK6x0MnpUN1jUSWnLQ_UEH2QSYcLFBGhhUBD3FVwrzY,21840
-mdbq-3.7.15.dist-info/METADATA,sha256=9wgxH4T1ZRrLdj4JqhvAuGLnK3vJ6VWZ8tSd2GCTW3g,244
-mdbq-3.7.15.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-3.7.15.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.7.15.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=o_QwFWbD6O2F56k6bwnpVV55EcdFCyes05ON7iu9TrA,21882
+mdbq-3.7.17.dist-info/METADATA,sha256=QcNNflNFDMslBNGYbl047wqJGPBu7Ohd_WiowCL7xCk,244
+mdbq-3.7.17.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-3.7.17.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.7.17.dist-info/RECORD,,

{mdbq-3.7.15.dist-info → mdbq-3.7.17.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.1.0)
+Generator: bdist_wheel (0.44.0)
 Root-Is-Purelib: true
 Tag: py3-none-any