mdbq 3.7.16__py3-none-any.whl → 3.7.18__py3-none-any.whl

mdbq/aggregation/optimize_data.py

@@ -1,7 +1,6 @@
 # -*- coding: UTF-8 –*-
 from mdbq.mysql import mysql
 from mdbq.config import default
-import socket
 import subprocess
 import psutil
 import time
@@ -10,16 +9,7 @@ import logging
 """
 对指定数据库所有冗余数据进行清理
 """
-hostname = 'xigua_lx'  # 正常情况下 spider_tg 只会在 xigua_lx 或者 xigua1 运行，且都指向主库
-local = 'remoto'
-targe_host = socket.gethostname()
-if targe_host == 'xigua_lx':
-    local = 'local'  # 直接指向局域网地址
-elif targe_host in ['localhost.localdomain', 'xigua1']:
-    targe_host = 'xigua1'  # 修正 Linux 系统用户名
-    local = 'local'  # 直接指向局域网地址
-elif targe_host == 'MacBookPro':
-    local = 'local'  # 直接指向局域网地址
+targe_host, hostname, local =  default.return_default_host()
 m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname=hostname, sql='mysql', local=local, config_file=None)
 if not username:
     logger.info(f'找不到主机：')

mdbq/aggregation/query_data.py

@@ -1,6 +1,5 @@
 # -*- coding: UTF-8 –*-
 import re
-import socket
 from mdbq.mysql import mysql
 from mdbq.mysql import s_query
 from mdbq.aggregation import optimize_data
@@ -21,23 +20,12 @@ import concurrent.futures
 import traceback
 import logging
 import sys
-import socket
 
 """
 
 """
 dirname = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'support')
-hostname = 'xigua_lx'  # 正常情况下 spider_tg 只会在 xigua_lx 或者 xigua1 运行，且都指向主库
-local = 'remoto'
-targe_host = socket.gethostname()
-
-if targe_host == 'xigua_lx':
-    local = 'local'  # 直接指向局域网地址
-elif targe_host in ['localhost.localdomain', 'xigua1']:
-    targe_host = 'xigua1'  # 修正 Linux 系统用户名
-    local = 'local'  # 直接指向局域网地址
-elif targe_host == 'MacBookPro':
-    local = 'local'  # 直接指向局域网地址
+targe_host, hostname, local =  default.return_default_host()
 m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname=hostname, sql='mysql', local=local, config_file=None)
 logger = logging.getLogger(__name__)
 
@@ -1466,9 +1454,13 @@ class MysqlDatasQuery:
             end_date=end_date,
             projection=projection,
         )
-        df.drop_duplicates(
-            subset=['日期', '店铺名称', '商品id', '商品访客数'], keep='last',
-            inplace=True, ignore_index=True)
+        # df.drop_duplicates(
+        #     subset=['日期', '店铺名称', '商品id', '商品访客数'], keep='last',
+        #     inplace=True, ignore_index=True)
+        # 保留最新日期的数据
+        idx = df.groupby(['日期', '店铺名称', '商品id'])['更新时间'].idxmax()
+        df = df.loc[idx]
+
         df_set['商品id'] = df_set['商品id'].astype('int64')
         df_set = df_set[['商品id', '上市年份']]
         df['商品id'] = df['商品id'].astype('int64')
@@ -3864,13 +3856,17 @@ def main(days=150, months=3):
     )
 
 
+def test():
+    sdq = MysqlDatasQuery()  # 实例化数据处理类
+    sdq.months = 1  # 设置数据周期， 1 表示近 2 个月
+    sdq.update_service = True  # 调试时加，true: 将数据写入 mysql 服务器
+    sdq.spph(db_name='聚合数据', table_name='天猫_商品排行')
+
 if __name__ == '__main__':
     main(
         days=150,  # 清理聚合数据的日期长度
         months=3  # 生成聚合数据的长度
     )
 
-    # sdq = MysqlDatasQuery()  # 实例化数据处理类
-    # sdq.months = 20  # 设置数据周期， 1 表示近 2 个月
-    # sdq.tg_by_day(db_name='聚合数据', table_name='多店推广场景_按日聚合')
+
 

mdbq/config/default.py

@@ -10,6 +10,21 @@ support_path = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'sup
 logger = logging.getLogger(__name__)
 
 
+def return_default_host():
+    targe_host = socket.gethostname()
+    hostname = 'xigua_lx'  # 正常情况下 spider_tg 只会在 xigua_lx 或者 xigua1 运行，且都指向主库
+    local = 'remoto'
+
+    if targe_host == 'xigua_lx':
+        local = 'local'  # 直接指向局域网地址
+    elif targe_host in ['localhost.localdomain', 'xigua1']:
+        targe_host = 'xigua1'  # 修正 Linux 系统用户名
+        local = 'local'  # 直接指向局域网地址
+    elif targe_host == 'MacBookPro':
+        local = 'local'  # 直接指向局域网地址
+    return targe_host, hostname, local
+
+
 def get_mysql_engine(platform='Windows', hostname='xigua_lx', sql='mysql', local='remoto', config_file=None):
     if not config_file:
         config_file = os.path.join(support_path, 'my_config.txt')
@@ -28,76 +43,7 @@ def get_mysql_engine(platform='Windows', hostname='xigua_lx', sql='mysql', local
     return _engine, username, password, host, port
 
 
-def return_host(conf_data):
-    """
-    从配置文件数据中获取: username, password, host, port
-    :param conf_data:
-    :return:
-    """
-    return conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
-
-
-def return_one_engine(conf_data):
-    """
-    返回一个 engine
-    """
-    username, password, host, port = return_host(conf_data)
-    return mysql.MysqlUpload(username=username, password=password, host=host, port=port, charset='utf8mb4')
-
-
-def get_hostname(platform, hostname, sql, local):
-    """
-    返回一个主机的: username, password, host, port
-    """
-    config_file = os.path.join(support_path, 'my_config.txt')
-    with open(config_file, 'r', encoding='utf-8') as f:
-        conf = json.load(f)
-    conf_data = conf[platform][hostname][sql][local]
-    return return_host(conf_data)
-
-
-def get_engine_bak():
-    """
-    要删除
-    """
-    if not os.path.isdir(support_path):
-        print(f'缺少配置文件，无法读取配置文件： {file}')
-        return
-    config_file = os.path.join(support_path, 'my_config.txt')
-
-    with open(config_file, 'r', encoding='utf-8') as f:
-        conf = json.load(f)
-
-    if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
-        conf_data = conf['Windows']['xigua_lx']['mysql']['remoto']
-        m_engine = return_one_engine(conf_data=conf_data)
-        conf_data = conf['Windows']['company']['mysql']['local']
-        company_engine = return_one_engine(conf_data=conf_data)
-        username, password, host, port = return_host(conf_data)  # 顺序不能乱
-        conf_data = conf['Windows']['xigua_ts']['mysql']['remoto']
-        ts_engine = return_one_engine(conf_data=conf_data)
-    elif socket.gethostname() == 'xigua_lx' or socket.gethostname() == 'xigua1' or socket.gethostname() == 'MacBookPro':
-        conf_data = conf['Windows']['xigua_lx']['mysql']['local']
-        m_engine = return_one_engine(conf_data=conf_data)
-        username, password, host, port = return_host(conf_data)  # 顺序不能乱
-        conf_data = conf['Windows']['company']['mysql']['remoto']
-        company_engine = return_one_engine(conf_data=conf_data)
-        conf_data = conf['Windows']['xigua_ts']['mysql']['remoto']
-        ts_engine = return_one_engine(conf_data=conf_data)
-
-    else:
-        conf_data = conf['Windows']['xigua_lx']['mysql']['remoto']
-        m_engine = return_one_engine(conf_data=conf_data)
-        username, password, host, port = return_host(conf_data)  # 顺序不能乱
-        conf_data = conf['Windows']['company']['mysql']['remoto']
-        company_engine = return_one_engine(conf_data=conf_data)
-        conf_data = conf['Windows']['xigua_ts']['mysql']['remoto']
-        ts_engine = return_one_engine(conf_data=conf_data)
-
-    return m_engine, company_engine, ts_engine, (username, password, host, port)
-
-
-def write_back(datas):
+def write_back_bak(datas):
     """ 将数据写回本地 """
     if not os.path.isdir(support_path):
         print(f'缺少配置文件，无法读取配置文件： {file}')
@@ -110,16 +56,3 @@ def write_back(datas):
 
 if __name__ == '__main__':
     pass
-    username, password, host, port = get_hostname(
-        platform='Windows',
-        hostname='xigua_lx',
-        sql='mysql',
-        local='remoto'
-    )
-    res = get_mysql_engine(
-        platform='Windows',
-        hostname='xigua_lx',
-        sql='mysql',
-        local='remoto'
-    )
-    print(res)

mdbq/config/myconfig.py

@@ -21,7 +21,7 @@ def main():
 
 
 def write_back(datas):
-    """ 将数据写回本地 """
+    """ 将数据写回本地   all-datas.py 调用 """
     # support_path = set_support.SetSupport(dirname='support').dirname
     file = os.path.join(support_path, 'my_config.txt')
     with open(file, 'w+', encoding='utf-8') as f:

mdbq/config/products.py

@@ -3,7 +3,6 @@ import json
 import os
 import platform
 import getpass
-import socket
 import pandas as pd
 from mdbq.mysql import mysql
 from mdbq.config import default
@@ -13,17 +12,7 @@ from numpy.ma.core import product
 天猫货品年份基准对照
 用于聚合数据，通过此数据表进一步可确定商品上架年月
 """
-hostname = 'xigua_lx'  # 正常情况下 spider_tg 只会在 xigua_lx 或者 xigua1 运行，且都指向主库
-local = 'remoto'
-targe_host = socket.gethostname()
-
-if targe_host == 'xigua_lx':
-    local = 'local'  # 直接指向局域网地址
-elif targe_host in ['localhost.localdomain', 'xigua1']:
-    targe_host = 'xigua1'  # 修正 Linux 系统用户名
-    local = 'local'  # 直接指向局域网地址
-elif targe_host == 'MacBookPro':
-    local = 'local'  # 直接指向局域网地址
+targe_host, hostname, local =  default.return_default_host()
 m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname=hostname, sql='mysql', local=local, config_file=None)
 if not username:
     print(f'找不到主机1：')

mdbq/config/set_support.py

@@ -13,14 +13,6 @@ support 文件夹包含很多配置类文件，是程序必不可少的依赖
 class SetSupport:
     def __init__(self, dirname):
         self.dirname = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), dirname)
-        # if platform.system() == 'Windows':
-        #     self.dirname = f'C:\\数据中心2\\support'
-        # elif platform.system() == 'Darwin':
-        #     self.dirname = f'/Users/{getpass.getuser()}/数据中心2/support'
-        # else:
-        #     self.dirname = '数据中心2/support'  # 没有用, 可以删
-        # if not os.path.isdir(self.dirname):
-        #     os.mkdir(self.dirname)
 
 
 if __name__ == '__main__':

mdbq/other/download_sku_picture.py

@@ -4,7 +4,6 @@ import getpass
 import json
 import os
 import platform
-import socket
 import random
 from dateutil.relativedelta import relativedelta
 import re
@@ -50,18 +49,7 @@ upload_path = os.path.join(D_PATH, '数据上传中心')  # 此目录位于下
 if not os.path.exists(upload_path):  # 数据中心根目录
     os.makedirs(upload_path)
 
-hostname = 'xigua_lx'  # 正常情况下 spider_tg 只会在 xigua_lx 或者 xigua1 运行，且都指向主库
-local = 'remoto'
-targe_host = socket.gethostname()
-
-if targe_host == 'xigua_lx':
-    local = 'local'  # 直接指向局域网地址
-elif targe_host in ['localhost.localdomain', 'xigua1']:
-    targe_host = 'xigua1'  # 修正 Linux 系统用户名
-    local = 'local'  # 直接指向局域网地址
-elif targe_host == 'MacBookPro':
-    local = 'local'  # 直接指向局域网地址
-
+targe_host, hostname, local =  default.return_default_host()
 m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname=hostname, sql='mysql', local=local, config_file=None)
 if not username:
     print(f'找不到主机：')

mdbq/redis/getredis.py

@@ -1,9 +1,8 @@
 # -*- coding: UTF-8 –*-
-import os.path
+import os
 import sys
 import random
 import redis
-import socket
 from mdbq.mysql import s_query
 from mdbq.config import default
 import pandas as pd
@@ -13,22 +12,10 @@ import datetime
 import threading
 import logging
 from logging.handlers import RotatingFileHandler
-import getpass
-import platform
 from decimal import Decimal
 import orjson
 
-hostname = 'xigua_lx'  # 正常情况下 spider_tg 只会在 xigua_lx 或者 xigua1 运行，且都指向主库
-local = 'remoto'
-targe_host = socket.gethostname()
-
-if targe_host == 'xigua_lx':
-    local = 'local'  # 直接指向局域网地址
-elif targe_host in ['localhost.localdomain', 'xigua1']:
-    targe_host = 'xigua1'  # 修正 Linux 系统用户名
-    local = 'local'  # 直接指向局域网地址
-elif targe_host == 'MacBookPro':
-    local = 'local'  # 直接指向局域网地址
+targe_host, hostname, local =  default.return_default_host()
 m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname=hostname, sql='mysql', local=local, config_file=None)
 
 # 获取当前模块的日志记录器
@@ -37,6 +24,8 @@ logger = logging.getLogger(__name__)
 # 创建一个文件处理器，用于将日志写入文件
 dir_path = os.path.realpath(os.path.dirname(sys.argv[0]))
 log_file = os.path.join(dir_path, 'logfile', 'redis.log')
+if not os.path.isdir(os.path.join(dir_path, 'logfile')):
+    os.mkdir(os.path.join(dir_path, 'logfile'))
 file_handler = RotatingFileHandler(log_file, maxBytes=3 * 1024 * 1024, backupCount=10, encoding='utf-8')  # 保留10个备份文件
 file_handler.setLevel(logging.INFO)  # 设置文件处理器的日志级别
 
@@ -644,5 +633,4 @@ if __name__ == '__main__':
     # )
     # logger.info(df)
     #
-
-    logger.info(socket.gethostname())
+    pass

mdbq/spider/aikucun.py

@@ -22,7 +22,6 @@ from mdbq.other import ua_sj
 from mdbq.mysql import mysql
 from mdbq.mysql import s_query
 from mdbq.config import default
-import socket
 
 warnings.filterwarnings('ignore')
 
@@ -38,17 +37,7 @@ else:
     D_PATH = str(pathlib.Path(f'/Users/{getpass.getuser()}/Downloads'))
 upload_path = os.path.join(D_PATH, '数据上传中心', '爱库存')  # 此目录位于下载文件夹
 
-hostname = 'xigua_lx'  # 正常情况下 spider_tg 只会在 xigua_lx 或者 xigua1 运行，且都指向主库
-local = 'remoto'
-targe_host = socket.gethostname()
-
-if targe_host == 'xigua_lx':
-    local = 'local'  # 直接指向局域网地址
-elif targe_host in ['localhost.localdomain', 'xigua1']:
-    targe_host = 'xigua1'  # 修正 Linux 系统用户名
-    local = 'local'  # 直接指向局域网地址
-elif targe_host == 'MacBookPro':
-    local = 'local'  # 直接指向局域网地址
+targe_host, hostname, local =  default.return_default_host()
 m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname=hostname, sql='mysql', local=local, config_file=None)
 print(username, password, host, port)
 # 实例化一个数据查询类，用来获取 cookies 表数据

{mdbq-3.7.16.dist-info → mdbq-3.7.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 3.7.16
+Version: 3.7.18
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.7.16.dist-info → mdbq-3.7.18.dist-info}/RECORD

@@ -1,15 +1,15 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/optimize_data.py,sha256=_bI9gIMiqO6mIiwcMEfMqTpclrHDF3G6dpo7uzSlZ4Q,1407
-mdbq/aggregation/query_data.py,sha256=4y5sGSEpwCodZPEv6oC9aBJOAkYaRtVh4IQh-6MAf8s,174940
+mdbq/aggregation/optimize_data.py,sha256=wB7prQdZAHyjzXH9V8g8X_JBMdvCCUITN1hVwK72Tdg,952
+mdbq/aggregation/query_data.py,sha256=uD9fHJrcgmHFBnEdhwvQ0Icthz8SzwaJedX3SFfVXfs,174712
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=hJs815hGFwm_X5bP2i9XugG2w2ZY_F0n3-Q0hVpIPPw,4892
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
-mdbq/config/default.py,sha256=-mSFxj5tluuPZw2pGRJkHESoX-x0WAHHO3sQzh8_DPE,4775
-mdbq/config/myconfig.py,sha256=IEJOYoSiDivplE6N-1UWuVKUj6zLa4EOsuj3aONb85g,1000
-mdbq/config/products.py,sha256=93YnQEqms2gacLaaPtnkEpjAe9-uTRSTkAcP_OC2wWk,6188
-mdbq/config/set_support.py,sha256=eM2scqDzGNR2kkbYmguRB2ucrqeX0KMh5OMIaGsLem4,877
+mdbq/config/default.py,sha256=pOBmlxmGzmK8XYstv_qwE4vDwJmytEmVIYE0bDnCXOs,2143
+mdbq/config/myconfig.py,sha256=5gg3B8fnDx2_a5I3kfPLXx7xePqZwfb3ohj6iMVhplc,1022
+mdbq/config/products.py,sha256=02sZAhFyF5dpapXj9h5iOKMXvC4ZNZjHTnQZgbzc9NA,5731
+mdbq/config/set_support.py,sha256=7C7NFy7Em_uC7lig54qQlIlKG_AJeMCskxzK87anGkM,462
 mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
 mdbq/dataframe/converter.py,sha256=lETYhT7KXlWzWwqguqhk6vI6kj4rnOBEW1lhqKy2Abc,5035
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
@@ -21,7 +21,7 @@ mdbq/mysql/mysql.py,sha256=bsv-khT7fyoYxEJMbJPks-V1tYvwX-mNHsoTNXfWiKk,95884
 mdbq/mysql/s_query.py,sha256=pj5ioJfUT81Su9S-km9G49gF5F2MmXXfw_oAIUzhN28,8794
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
-mdbq/other/download_sku_picture.py,sha256=zmd02td0Wz2cv44rrCUS3zv0GiInQFP6hRfHzSAcavY,45207
+mdbq/other/download_sku_picture.py,sha256=Da_5hsw3yeDm2lXE-YLMbIxk2-36K5-o7D3Sm9APj9M,44760
 mdbq/other/porxy.py,sha256=UHfgEyXugogvXgsG68a7QouUCKaohTKKkI4RN-kYSdQ,4961
 mdbq/other/pov_city.py,sha256=AEOmCOzOwyjHi9LLZWPKi6DUuSC-_M163664I52u9qw,21050
 mdbq/other/ua_sj.py,sha256=JuVYzc_5QZ9s_oQSrTHVKkQv4S_7-CWx4oIKOARn_9U,22178
@@ -30,10 +30,10 @@ mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,239
 mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,7192
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
-mdbq/redis/getredis.py,sha256=IoS40CaOfcWHe74iFVvlygvy-duNcO-CoYCbyc_7qdQ,25916
+mdbq/redis/getredis.py,sha256=MLVWENvFtLFpxUlkUMBvc5bXPXeG-QIl8f_A200TIWE,25497
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=yDeuMLClM2Dp8pFqfRypAvILb4kT4sB-CSR9HF8_Nao,22328
-mdbq-3.7.16.dist-info/METADATA,sha256=C4hGq3pI11My5xNZOCOaapmirsLkXiF8YbtPnb_ta78,244
-mdbq-3.7.16.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-3.7.16.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.7.16.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=o_QwFWbD6O2F56k6bwnpVV55EcdFCyes05ON7iu9TrA,21882
+mdbq-3.7.18.dist-info/METADATA,sha256=lMvpo0kmnnG8XQQmln8HqSV8L6k1ZzcDYslHmuyqUTo,244
+mdbq-3.7.18.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-3.7.18.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.7.18.dist-info/RECORD,,