mdbq 3.7.16__tar.gz → 3.7.18__tar.gz

{mdbq-3.7.16 → mdbq-3.7.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 3.7.16
+Version: 3.7.18
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.7.16 → mdbq-3.7.18}/mdbq/aggregation/optimize_data.py

@@ -1,7 +1,6 @@
 # -*- coding: UTF-8 –*-
 from mdbq.mysql import mysql
 from mdbq.config import default
-import socket
 import subprocess
 import psutil
 import time
@@ -10,16 +9,7 @@ import logging
 """
 对指定数据库所有冗余数据进行清理
 """
-hostname = 'xigua_lx'  # 正常情况下 spider_tg 只会在 xigua_lx 或者 xigua1 运行，且都指向主库
-local = 'remoto'
-targe_host = socket.gethostname()
-if targe_host == 'xigua_lx':
-    local = 'local'  # 直接指向局域网地址
-elif targe_host in ['localhost.localdomain', 'xigua1']:
-    targe_host = 'xigua1'  # 修正 Linux 系统用户名
-    local = 'local'  # 直接指向局域网地址
-elif targe_host == 'MacBookPro':
-    local = 'local'  # 直接指向局域网地址
+targe_host, hostname, local =  default.return_default_host()
 m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname=hostname, sql='mysql', local=local, config_file=None)
 if not username:
     logger.info(f'找不到主机：')

{mdbq-3.7.16 → mdbq-3.7.18}/mdbq/aggregation/query_data.py

@@ -1,6 +1,5 @@
 # -*- coding: UTF-8 –*-
 import re
-import socket
 from mdbq.mysql import mysql
 from mdbq.mysql import s_query
 from mdbq.aggregation import optimize_data
@@ -21,23 +20,12 @@ import concurrent.futures
 import traceback
 import logging
 import sys
-import socket
 
 """
 
 """
 dirname = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'support')
-hostname = 'xigua_lx'  # 正常情况下 spider_tg 只会在 xigua_lx 或者 xigua1 运行，且都指向主库
-local = 'remoto'
-targe_host = socket.gethostname()
-
-if targe_host == 'xigua_lx':
-    local = 'local'  # 直接指向局域网地址
-elif targe_host in ['localhost.localdomain', 'xigua1']:
-    targe_host = 'xigua1'  # 修正 Linux 系统用户名
-    local = 'local'  # 直接指向局域网地址
-elif targe_host == 'MacBookPro':
-    local = 'local'  # 直接指向局域网地址
+targe_host, hostname, local =  default.return_default_host()
 m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname=hostname, sql='mysql', local=local, config_file=None)
 logger = logging.getLogger(__name__)
 
@@ -1466,9 +1454,13 @@ class MysqlDatasQuery:
             end_date=end_date,
             projection=projection,
         )
-        df.drop_duplicates(
-            subset=['日期', '店铺名称', '商品id', '商品访客数'], keep='last',
-            inplace=True, ignore_index=True)
+        # df.drop_duplicates(
+        #     subset=['日期', '店铺名称', '商品id', '商品访客数'], keep='last',
+        #     inplace=True, ignore_index=True)
+        # 保留最新日期的数据
+        idx = df.groupby(['日期', '店铺名称', '商品id'])['更新时间'].idxmax()
+        df = df.loc[idx]
+
         df_set['商品id'] = df_set['商品id'].astype('int64')
         df_set = df_set[['商品id', '上市年份']]
         df['商品id'] = df['商品id'].astype('int64')
@@ -3864,13 +3856,17 @@ def main(days=150, months=3):
     )
 
 
+def test():
+    sdq = MysqlDatasQuery()  # 实例化数据处理类
+    sdq.months = 1  # 设置数据周期， 1 表示近 2 个月
+    sdq.update_service = True  # 调试时加，true: 将数据写入 mysql 服务器
+    sdq.spph(db_name='聚合数据', table_name='天猫_商品排行')
+
 if __name__ == '__main__':
     main(
         days=150,  # 清理聚合数据的日期长度
         months=3  # 生成聚合数据的长度
     )
 
-    # sdq = MysqlDatasQuery()  # 实例化数据处理类
-    # sdq.months = 20  # 设置数据周期， 1 表示近 2 个月
-    # sdq.tg_by_day(db_name='聚合数据', table_name='多店推广场景_按日聚合')
+
 

mdbq-3.7.18/mdbq/config/default.py

@@ -0,0 +1,58 @@
+# -*- coding: UTF-8 –*-
+import os
+import sys
+import json
+import socket
+import logging
+from mdbq.mysql import mysql
+
+support_path = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'support')
+logger = logging.getLogger(__name__)
+
+
+def return_default_host():
+    targe_host = socket.gethostname()
+    hostname = 'xigua_lx'  # 正常情况下 spider_tg 只会在 xigua_lx 或者 xigua1 运行，且都指向主库
+    local = 'remoto'
+
+    if targe_host == 'xigua_lx':
+        local = 'local'  # 直接指向局域网地址
+    elif targe_host in ['localhost.localdomain', 'xigua1']:
+        targe_host = 'xigua1'  # 修正 Linux 系统用户名
+        local = 'local'  # 直接指向局域网地址
+    elif targe_host == 'MacBookPro':
+        local = 'local'  # 直接指向局域网地址
+    return targe_host, hostname, local
+
+
+def get_mysql_engine(platform='Windows', hostname='xigua_lx', sql='mysql', local='remoto', config_file=None):
+    if not config_file:
+        config_file = os.path.join(support_path, 'my_config.txt')
+    if not os.path.isfile(config_file):
+        print(f'缺少配置文件，无法读取配置文件： {config_file}')
+        return None
+
+    if socket.gethostname() == 'xigua_lx':
+        local = 'local'
+
+    with open(config_file, 'r', encoding='utf-8') as f:
+        conf = json.load(f)
+    conf_data = conf[platform][hostname][sql][local]
+    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
+    _engine = mysql.MysqlUpload(username=username, password=password, host=host, port=port, charset='utf8mb4')
+    return _engine, username, password, host, port
+
+
+def write_back_bak(datas):
+    """ 将数据写回本地 """
+    if not os.path.isdir(support_path):
+        print(f'缺少配置文件，无法读取配置文件： {file}')
+        return
+    file = os.path.join(support_path, 'my_config.txt')
+    with open(file, 'w+', encoding='utf-8') as f:
+        json.dump(datas, f, ensure_ascii=False, sort_keys=False, indent=4)
+
+
+
+if __name__ == '__main__':
+    pass

{mdbq-3.7.16 → mdbq-3.7.18}/mdbq/config/myconfig.py

@@ -21,7 +21,7 @@ def main():
 
 
 def write_back(datas):
-    """ 将数据写回本地 """
+    """ 将数据写回本地   all-datas.py 调用 """
     # support_path = set_support.SetSupport(dirname='support').dirname
     file = os.path.join(support_path, 'my_config.txt')
     with open(file, 'w+', encoding='utf-8') as f:

{mdbq-3.7.16 → mdbq-3.7.18}/mdbq/config/products.py

@@ -3,7 +3,6 @@ import json
 import os
 import platform
 import getpass
-import socket
 import pandas as pd
 from mdbq.mysql import mysql
 from mdbq.config import default
@@ -13,17 +12,7 @@ from numpy.ma.core import product
 天猫货品年份基准对照
 用于聚合数据，通过此数据表进一步可确定商品上架年月
 """
-hostname = 'xigua_lx'  # 正常情况下 spider_tg 只会在 xigua_lx 或者 xigua1 运行，且都指向主库
-local = 'remoto'
-targe_host = socket.gethostname()
-
-if targe_host == 'xigua_lx':
-    local = 'local'  # 直接指向局域网地址
-elif targe_host in ['localhost.localdomain', 'xigua1']:
-    targe_host = 'xigua1'  # 修正 Linux 系统用户名
-    local = 'local'  # 直接指向局域网地址
-elif targe_host == 'MacBookPro':
-    local = 'local'  # 直接指向局域网地址
+targe_host, hostname, local =  default.return_default_host()
 m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname=hostname, sql='mysql', local=local, config_file=None)
 if not username:
     print(f'找不到主机1：')

{mdbq-3.7.16 → mdbq-3.7.18}/mdbq/config/set_support.py

@@ -13,14 +13,6 @@ support 文件夹包含很多配置类文件，是程序必不可少的依赖
 class SetSupport:
     def __init__(self, dirname):
         self.dirname = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), dirname)
-        # if platform.system() == 'Windows':
-        #     self.dirname = f'C:\\数据中心2\\support'
-        # elif platform.system() == 'Darwin':
-        #     self.dirname = f'/Users/{getpass.getuser()}/数据中心2/support'
-        # else:
-        #     self.dirname = '数据中心2/support'  # 没有用, 可以删
-        # if not os.path.isdir(self.dirname):
-        #     os.mkdir(self.dirname)
 
 
 if __name__ == '__main__':

{mdbq-3.7.16 → mdbq-3.7.18}/mdbq/other/download_sku_picture.py

@@ -4,7 +4,6 @@ import getpass
 import json
 import os
 import platform
-import socket
 import random
 from dateutil.relativedelta import relativedelta
 import re
@@ -50,18 +49,7 @@ upload_path = os.path.join(D_PATH, '数据上传中心')  # 此目录位于下
 if not os.path.exists(upload_path):  # 数据中心根目录
     os.makedirs(upload_path)
 
-hostname = 'xigua_lx'  # 正常情况下 spider_tg 只会在 xigua_lx 或者 xigua1 运行，且都指向主库
-local = 'remoto'
-targe_host = socket.gethostname()
-
-if targe_host == 'xigua_lx':
-    local = 'local'  # 直接指向局域网地址
-elif targe_host in ['localhost.localdomain', 'xigua1']:
-    targe_host = 'xigua1'  # 修正 Linux 系统用户名
-    local = 'local'  # 直接指向局域网地址
-elif targe_host == 'MacBookPro':
-    local = 'local'  # 直接指向局域网地址
-
+targe_host, hostname, local =  default.return_default_host()
 m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname=hostname, sql='mysql', local=local, config_file=None)
 if not username:
     print(f'找不到主机：')

{mdbq-3.7.16 → mdbq-3.7.18}/mdbq/redis/getredis.py

@@ -1,9 +1,8 @@
 # -*- coding: UTF-8 –*-
-import os.path
+import os
 import sys
 import random
 import redis
-import socket
 from mdbq.mysql import s_query
 from mdbq.config import default
 import pandas as pd
@@ -13,22 +12,10 @@ import datetime
 import threading
 import logging
 from logging.handlers import RotatingFileHandler
-import getpass
-import platform
 from decimal import Decimal
 import orjson
 
-hostname = 'xigua_lx'  # 正常情况下 spider_tg 只会在 xigua_lx 或者 xigua1 运行，且都指向主库
-local = 'remoto'
-targe_host = socket.gethostname()
-
-if targe_host == 'xigua_lx':
-    local = 'local'  # 直接指向局域网地址
-elif targe_host in ['localhost.localdomain', 'xigua1']:
-    targe_host = 'xigua1'  # 修正 Linux 系统用户名
-    local = 'local'  # 直接指向局域网地址
-elif targe_host == 'MacBookPro':
-    local = 'local'  # 直接指向局域网地址
+targe_host, hostname, local =  default.return_default_host()
 m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname=hostname, sql='mysql', local=local, config_file=None)
 
 # 获取当前模块的日志记录器
@@ -37,6 +24,8 @@ logger = logging.getLogger(__name__)
 # 创建一个文件处理器，用于将日志写入文件
 dir_path = os.path.realpath(os.path.dirname(sys.argv[0]))
 log_file = os.path.join(dir_path, 'logfile', 'redis.log')
+if not os.path.isdir(os.path.join(dir_path, 'logfile')):
+    os.mkdir(os.path.join(dir_path, 'logfile'))
 file_handler = RotatingFileHandler(log_file, maxBytes=3 * 1024 * 1024, backupCount=10, encoding='utf-8')  # 保留10个备份文件
 file_handler.setLevel(logging.INFO)  # 设置文件处理器的日志级别
 
@@ -644,5 +633,4 @@ if __name__ == '__main__':
     # )
     # logger.info(df)
     #
-
-    logger.info(socket.gethostname())
+    pass

{mdbq-3.7.16 → mdbq-3.7.18}/mdbq/spider/aikucun.py

@@ -22,7 +22,6 @@ from mdbq.other import ua_sj
 from mdbq.mysql import mysql
 from mdbq.mysql import s_query
 from mdbq.config import default
-import socket
 
 warnings.filterwarnings('ignore')
 
@@ -38,17 +37,7 @@ else:
     D_PATH = str(pathlib.Path(f'/Users/{getpass.getuser()}/Downloads'))
 upload_path = os.path.join(D_PATH, '数据上传中心', '爱库存')  # 此目录位于下载文件夹
 
-hostname = 'xigua_lx'  # 正常情况下 spider_tg 只会在 xigua_lx 或者 xigua1 运行，且都指向主库
-local = 'remoto'
-targe_host = socket.gethostname()
-
-if targe_host == 'xigua_lx':
-    local = 'local'  # 直接指向局域网地址
-elif targe_host in ['localhost.localdomain', 'xigua1']:
-    targe_host = 'xigua1'  # 修正 Linux 系统用户名
-    local = 'local'  # 直接指向局域网地址
-elif targe_host == 'MacBookPro':
-    local = 'local'  # 直接指向局域网地址
+targe_host, hostname, local =  default.return_default_host()
 m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname=hostname, sql='mysql', local=local, config_file=None)
 print(username, password, host, port)
 # 实例化一个数据查询类，用来获取 cookies 表数据

{mdbq-3.7.16 → mdbq-3.7.18}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 3.7.16
+Version: 3.7.18
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.7.16 → mdbq-3.7.18}/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='mdbq',
-      version='3.7.16',
+      version='3.7.18',
       author='xigua, ',
       author_email="2587125111@qq.com",
       url='https://pypi.org/project/mdbq',

mdbq-3.7.16/mdbq/config/default.py

@@ -1,125 +0,0 @@
-# -*- coding: UTF-8 –*-
-import os
-import sys
-import json
-import socket
-import logging
-from mdbq.mysql import mysql
-
-support_path = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'support')
-logger = logging.getLogger(__name__)
-
-
-def get_mysql_engine(platform='Windows', hostname='xigua_lx', sql='mysql', local='remoto', config_file=None):
-    if not config_file:
-        config_file = os.path.join(support_path, 'my_config.txt')
-    if not os.path.isfile(config_file):
-        print(f'缺少配置文件，无法读取配置文件： {config_file}')
-        return None
-
-    if socket.gethostname() == 'xigua_lx':
-        local = 'local'
-
-    with open(config_file, 'r', encoding='utf-8') as f:
-        conf = json.load(f)
-    conf_data = conf[platform][hostname][sql][local]
-    username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
-    _engine = mysql.MysqlUpload(username=username, password=password, host=host, port=port, charset='utf8mb4')
-    return _engine, username, password, host, port
-
-
-def return_host(conf_data):
-    """
-    从配置文件数据中获取: username, password, host, port
-    :param conf_data:
-    :return:
-    """
-    return conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
-
-
-def return_one_engine(conf_data):
-    """
-    返回一个 engine
-    """
-    username, password, host, port = return_host(conf_data)
-    return mysql.MysqlUpload(username=username, password=password, host=host, port=port, charset='utf8mb4')
-
-
-def get_hostname(platform, hostname, sql, local):
-    """
-    返回一个主机的: username, password, host, port
-    """
-    config_file = os.path.join(support_path, 'my_config.txt')
-    with open(config_file, 'r', encoding='utf-8') as f:
-        conf = json.load(f)
-    conf_data = conf[platform][hostname][sql][local]
-    return return_host(conf_data)
-
-
-def get_engine_bak():
-    """
-    要删除
-    """
-    if not os.path.isdir(support_path):
-        print(f'缺少配置文件，无法读取配置文件： {file}')
-        return
-    config_file = os.path.join(support_path, 'my_config.txt')
-
-    with open(config_file, 'r', encoding='utf-8') as f:
-        conf = json.load(f)
-
-    if socket.gethostname() == 'company' or socket.gethostname() == 'Mac2.local':
-        conf_data = conf['Windows']['xigua_lx']['mysql']['remoto']
-        m_engine = return_one_engine(conf_data=conf_data)
-        conf_data = conf['Windows']['company']['mysql']['local']
-        company_engine = return_one_engine(conf_data=conf_data)
-        username, password, host, port = return_host(conf_data)  # 顺序不能乱
-        conf_data = conf['Windows']['xigua_ts']['mysql']['remoto']
-        ts_engine = return_one_engine(conf_data=conf_data)
-    elif socket.gethostname() == 'xigua_lx' or socket.gethostname() == 'xigua1' or socket.gethostname() == 'MacBookPro':
-        conf_data = conf['Windows']['xigua_lx']['mysql']['local']
-        m_engine = return_one_engine(conf_data=conf_data)
-        username, password, host, port = return_host(conf_data)  # 顺序不能乱
-        conf_data = conf['Windows']['company']['mysql']['remoto']
-        company_engine = return_one_engine(conf_data=conf_data)
-        conf_data = conf['Windows']['xigua_ts']['mysql']['remoto']
-        ts_engine = return_one_engine(conf_data=conf_data)
-
-    else:
-        conf_data = conf['Windows']['xigua_lx']['mysql']['remoto']
-        m_engine = return_one_engine(conf_data=conf_data)
-        username, password, host, port = return_host(conf_data)  # 顺序不能乱
-        conf_data = conf['Windows']['company']['mysql']['remoto']
-        company_engine = return_one_engine(conf_data=conf_data)
-        conf_data = conf['Windows']['xigua_ts']['mysql']['remoto']
-        ts_engine = return_one_engine(conf_data=conf_data)
-
-    return m_engine, company_engine, ts_engine, (username, password, host, port)
-
-
-def write_back(datas):
-    """ 将数据写回本地 """
-    if not os.path.isdir(support_path):
-        print(f'缺少配置文件，无法读取配置文件： {file}')
-        return
-    file = os.path.join(support_path, 'my_config.txt')
-    with open(file, 'w+', encoding='utf-8') as f:
-        json.dump(datas, f, ensure_ascii=False, sort_keys=False, indent=4)
-
-
-
-if __name__ == '__main__':
-    pass
-    username, password, host, port = get_hostname(
-        platform='Windows',
-        hostname='xigua_lx',
-        sql='mysql',
-        local='remoto'
-    )
-    res = get_mysql_engine(
-        platform='Windows',
-        hostname='xigua_lx',
-        sql='mysql',
-        local='remoto'
-    )
-    print(res)