mdbq 2.7.1__py3-none-any.whl → 2.7.3__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -1290,33 +1290,35 @@ def file_dir(one_file=True, target_service='company'):
 
 
 def test():
-    path = os.path.relpath(r'C:\Users\Administrator\Downloads\JD商品明细sku')
+    path = os.path.relpath(r'/Users/xigua/数据中心/原始文件3/达摩盘/dmp人群报表')
     for root, dirs, files in os.walk(path, topdown=False):
         for name in files:
             if name.endswith('.csv') and 'baidu' not in name and '~' not in name:
-                print(name)
+                # print(name)
                 # df = pd.read_excel(os.path.join(root, name), header=0)
                 df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
-                cols = df.columns.tolist()
-                if '店铺名称' not in cols:
-                    df.insert(loc=1, column='店铺名称', value='京东箱包旗舰店')
-                if '曝光量' in cols:
-                    df.rename(columns={
-                        '曝光量': '搜索曝光量',
-                        '点击次数': '搜索点击次数',
-                        '点击率': '搜索点击率',
-                    }, inplace=True)
-                if '取消金额' in cols:
-                    df.rename(columns={
-                        '取消金额': '取消及售后退款金额',
-                        '取消商品件数': '取消及售后退款件数',
-                        '取消单量': '取消及售后退款单量',
-                    }, inplace=True)
-                if '取消及售后退款金额' not in cols:
-                    df['取消及售后退款金额'] = '0.0'
-                    df['取消及售后退款件数'] = 0
-                    df['取消及售后退款单量'] = 0
+                print(name)
+                if len(df) == 0:
+                    print(name)
+                    os.remove(os.path.join(root, name))
+                    continue
+                df = df[df['日期'] != '']
+                # cols = df.columns.tolist()
+                # if '千次展现花费' not in cols:
+                #     df.insert(loc=14, column='千次展现花费', value=0.0)
+                    # df.rename(columns={'总费用': '花费', '直接购物车数': '直接加购数', '总购物车数': '总加购数'}, inplace=True)
+                # if '类目ID' not in cols:
+                #     df['类目ID'] = 'null'
+                # if '类目名称' not in cols:
+                #     df['类目名称'] = 'null'
+                # if '店铺名称' not in cols:
+                #     df.insert(loc=1, column='店铺名称', value='京东箱包旗舰店')
+                # if '全站roi' in cols:
+                #     df.rename(columns={'全站roi': '全站投产比'}, inplace=True)
+                # new_name = f'{os.path.splitext(name)[0]}.csv'
                 df.to_csv(os.path.join(root, name), encoding='utf-8_sig', index=False, header=True)
+                # breakpoint()
+                # os.remove(os.path.join(root, name))
                 # new_name = f'{os.path.splitext(name)[0]}.xlsx'
                 # df.to_excel(os.path.join(root, name),
                 #             index=False, header=True, engine='openpyxl', freeze_panes=(1, 0))
@@ -1331,23 +1333,22 @@ if __name__ == '__main__':
 
     # # 上传 1 个文件到数据库
     # one_file_to_mysql(
-    #     file='/Users/xigua/Downloads/万里马箱包推广1_营销概况_qwqw全站营销_2024-08-18_2024-09-01.csv',
-    #     db_name='京东数据3',
-    #     table_name='推广数据_全站营销',
-    #     target_service='company',
+    #     file=r'C:\同步空间\BaiduSyncdisk\原始文件2\属性设置\电商定价.csv',
+    #     db_name='属性设置3',
+    #     table_name='电商定价',
+    #     target_service='home_lx',
     #     database='mysql'
     # )
-
+    #
     # 上传一个目录到指定数据库
     db_name = '京东数据3'
-    table_name = '京东商智_spu_商品明细'
+    table_name = '京东商智_店铺来源'
     upload_dir(
-        path=os.path.relpath(r'C:\同步空间\BaiduSyncdisk\原始文件3\京东报表\京东商智_spu_商品明细'),
+        path=os.path.relpath(r'/Users/xigua/数据中心/原始文件3/京东报表/店铺来源_三级来asdasdas源'),
         db_name=db_name,
         collection_name=table_name,
         dbs={'mysql': True, 'mongodb': False},
-        target_service='home_lx',
+        target_service='company',
     )
 
-
     # test()

mdbq/aggregation/mysql_types.py

@@ -1,28 +1,20 @@
 # -*- coding:utf-8 -*-
 import warnings
 import pandas as pd
-import numpy as np
-import chardet
-import zipfile
-
-from numpy import dtype
-from pandas.tseries.holiday import next_monday
-from pyzipper import PyZipFile
 import os
 import platform
 import json
 import pymysql
+import socket
 from mdbq.mongo import mongo
 from mdbq.mysql import mysql
 from mdbq.mysql import s_query
-from mdbq.config import get_myconf
+from mdbq.config import myconfig
 from mdbq.config import set_support
 from mdbq.dataframe import converter
 import datetime
 import time
 import re
-import shutil
-import getpass
 
 from sqlalchemy.dialects.postgresql.pg_catalog import pg_get_serial_sequence
 
@@ -54,7 +46,7 @@ class DataTypes:
             self.path = set_support.SetSupport(dirname='support').dirname
         self.service_name = service_name
         if not self.service_name:
-            self.service_name = 'home_lx'
+            self.service_name = 'xigua_lx'
         self.json_file = os.path.join(self.path, f'mysql_types_{self.service_name}.json')
         if not os.path.isdir(self.path):
             os.makedirs(self.path)
@@ -154,88 +146,89 @@ class DataTypes:
             return {}, cl, None, None  # 返回这些结果的目的是等添加完列再写 json 文件才能读到 types 信息
 
 
-def mysql_all_dtypes(db_name=None, table_name=None, service_database={'home_lx': 'mysql'}, path=None):
+def mysql_all_dtypes(db_name=None, table_name=None, path=None):
     """
-    更新笔记本 mysql 中所有数据库的 dtypes 信息到本地 json
+    更新 mysql 中所有数据库的 dtypes 信息到本地 json
     """
-    for service_name, database in service_database.items():
-        username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
-        config = {
-            'host': host,
-            'port': port,
-            'user': username,
-            'password': password,
-            'charset': 'utf8mb4',  # utf8mb4 支持存储四字节的UTF-8字符集
-            'cursorclass': pymysql.cursors.DictCursor,
-        }
-
+    username, password, host, port, service_name = None, None, None, None, None
+    conf = myconfig.main()
+    if socket.gethostname().lower() in ['xigua_lx', 'xigua1', 'macbook pro']:
+        data = conf['Windows']['xigua_lx']['mysql']['local']
+        username, password, host, port = data['username'], data['password'], data['host'], data['port']
+        service_name = 'xigua_lx'  # 影响 mysql_types_xigua_lx.json 文件名
+    elif socket.gethostname().lower() in ['company', 'Mac2.local']:
+        data = conf['Windows']['company']['mysql']['local']
+        username, password, host, port = data['username'], data['password'], data['host'], data['port']
+        service_name = 'company'  # 影响 mysql_types_company.json 文件名
+    if not username or not service_name:
+        return
+
+    config = {
+        'host': host,
+        'port': int(port),
+        'user': username,
+        'password': password,
+        'charset': 'utf8mb4',  # utf8mb4 支持存储四字节的UTF-8字符集
+        'cursorclass': pymysql.cursors.DictCursor,
+    }
+    connection = pymysql.connect(**config)  # 连接数据库
+    with connection.cursor() as cursor:
+        sql = "SHOW DATABASES;"
+        cursor.execute(sql)
+        db_name_lists = cursor.fetchall()
+        db_name_lists = [item['Database'] for item in db_name_lists]
+        connection.close()
+
+    sys_lists = ['information_schema', 'mysql', 'performance_schema', 'sakila', 'sys']
+    db_name_lists = [item for item in db_name_lists if item not in sys_lists]
+
+    results = []  # 返回结果示例: [{'云电影': '电影更新'}, {'生意经2': 'e3_零售明细统计'}]
+    for db_ in db_name_lists:
+        config.update({'database': db_})  # 添加更新 config 字段
         connection = pymysql.connect(**config)  # 连接数据库
-        with connection.cursor() as cursor:
-            sql = "SHOW DATABASES;"
-            cursor.execute(sql)
-            db_name_lists = cursor.fetchall()
-            db_name_lists = [item['Database'] for item in db_name_lists]
+        try:
+            with connection.cursor() as cursor:
+                sql = f"SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '{db_}';"
+                sql = "SHOW TABLES;"
+                cursor.execute(sql)
+                res_tables = cursor.fetchall()
+                for res_table in res_tables:
+                    for k, v in res_table.items():
+                        results.append({db_: v})
+        except:
+            pass
+        finally:
             connection.close()
-
-        sys_lists = ['information_schema', 'mysql', 'performance_schema', 'sakila', 'sys']
-        db_name_lists = [item for item in db_name_lists if item not in sys_lists]
-
-        # db_name_lists = [
-        #     '京东数据2',
-        #     '推广数据2',
-        #     '市场数据2',
-        #     '生意参谋2',
-        #     '生意经2',
-        #     '属性设置2',
-        #     '聚合数据',
-        # ]
-        results = []  # 返回结果示例: [{'云电影': '电影更新'}, {'生意经2': 'e3_零售明细统计'}]
-        for db_ in db_name_lists:
-            config.update({'database': db_})  # 添加更新 config 字段
-            connection = pymysql.connect(**config)  # 连接数据库
-            try:
-                with connection.cursor() as cursor:
-                    sql = f"SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '{db_}';"
-                    sql = "SHOW TABLES;"
-                    cursor.execute(sql)
-                    res_tables = cursor.fetchall()
-                    for res_table in res_tables:
-                        for k, v in res_table.items():
-                            results.append({db_: v})
-            except:
-                pass
-            finally:
-                connection.close()
-            time.sleep(0.5)
-
-        d = DataTypes(path=path, service_name=service_name)
-        for result in results:
-            for db_n, table_n in result.items():
-                # print(db_n, table_n, db_name, table_name)
-                if db_name and table_name:  # 下载一个指定的数据表
-                    if db_name != db_n or table_name != table_n:
-                        continue
-                elif db_name:  # 下载一个数据库的所有数据表
-                    if db_name != db_n:
-                        continue
-                # 如果 db_name 和 table_name 都不指定，则下载所有数据库的所有数据表
-                print(f'获取列信息 数据库: < {db_n} >, 数据表: < {table_n} >')
-                sq = s_query.QueryDatas(username=username, password=password, host=host, port=port)
-                # 获取数据表的指定列, 返回列表
-                # [{'视频bv号': 'BV1Dm4y1S7BU', '下载进度': 1}, {'视频bv号': 'BV1ov411c7US', '下载进度': 1}]
-                name_type = sq.dtypes_to_list(db_name=db_n, table_name=table_n)
-                if name_type:
-                    dtypes = {item['COLUMN_NAME']: item['COLUMN_TYPE'] for item in name_type}
-                    dtypes = {'mysql': {db_n: {table_n: dtypes}}}
-                    d.get_mysql_types(
-                        dtypes=dtypes,
-                        cl='mysql',
-                        db_name=db_n,
-                        table_name=table_n,
-                        is_file_dtype=True  # True表示旧文件有限
-                    )
-                else:
-                    print(f'数据库回传数据(name_type)为空')
+        time.sleep(0.5)
+
+    d = DataTypes(path=path, service_name=service_name)
+    for result in results:
+        for db_n, table_n in result.items():
+            # print(db_n, table_n, db_name, table_name)
+            if db_name and table_name:  # 下载一个指定的数据表
+                if db_name != db_n or table_name != table_n:
+                    continue
+            elif db_name:  # 下载一个数据库的所有数据表
+                if db_name != db_n:
+                    continue
+            # 如果 db_name 和 table_name 都不指定，则下载所有数据库的所有数据表
+            print(f'获取列信息 数据库: < {db_n} >, 数据表: < {table_n} >')
+            sq = s_query.QueryDatas(username=username, password=password, host=host, port=port)
+            # 获取数据表的指定列, 返回列表
+            # [{'视频bv号': 'BV1Dm4y1S7BU', '下载进度': 1}, {'视频bv号': 'BV1ov411c7US', '下载进度': 1}]
+            name_type = sq.dtypes_to_list(db_name=db_n, table_name=table_n)
+            if name_type:
+                dtypes = {item['COLUMN_NAME']: item['COLUMN_TYPE'] for item in name_type}
+                dtypes = {'mysql': {db_n: {table_n: dtypes}}}
+                d.get_mysql_types(
+                    dtypes=dtypes,
+                    cl='mysql',
+                    db_name=db_n,
+                    table_name=table_n,
+                    is_file_dtype=True  # True表示旧文件有限
+                )
+            else:
+                print(f'数据库回传数据(name_type)为空')
         # print(d.datas)
         d.as_json_file()
 
@@ -243,5 +236,5 @@ def mysql_all_dtypes(db_name=None, table_name=None, service_database={'home_lx':
 if __name__ == '__main__':
     # 更新 mysql 中所有数据库的 dtypes 信息到本地 json
     mysql_all_dtypes(
-        path='/Users/xigua/Downloads', service_name='company',
+        path='/Users/xigua/Downloads',
     )

mdbq/aggregation/optimize_data.py

@@ -1,7 +1,7 @@
 # -*- coding: UTF-8 –*-
 from mdbq.mongo import mongo
 from mdbq.mysql import mysql
-from mdbq.config import get_myconf
+from mdbq.config import myconfig
 import socket
 import subprocess
 import psutil
@@ -10,6 +10,20 @@ import platform
 """
 对指定数据库所有冗余数据进行清理
 """
+username, password, host, port, service_database = None, None, None, None, None,
+if socket.gethostname().lower() in ['xigua_lx', 'xigua1', 'macbook pro']:
+    conf = myconfig.main()
+    data = conf['Windows']['xigua_lx']['mysql']['local']
+    username, password, host, port = data['username'], data['password'], data['host'], data['port']
+    service_database = {'xigua_lx': 'mysql'}
+elif socket.gethostname().lower() in ['company', 'mac2.local']:
+    conf = myconfig.main()
+    data = conf['Windows']['company']['mysql']['local']
+    username, password, host, port = data['username'], data['password'], data['host'], data['port']
+    service_database = {'company': 'mysql'}
+if not username:
+    print(f'找不到主机：')
+
 
 
 def restart_mongodb():
@@ -57,60 +71,25 @@ def restart_mongodb():
         subprocess.call(command, shell=True)
 
 
-def op_data(db_name_lists, service_databases=[{'home_lx': 'mysql', 'home_lx': 'mongodb'}], days: int = 63, is_mongo=True, is_mysql=True):
+def op_data(db_name_lists, days: int = 63, is_mongo=True, is_mysql=True):
     """ """
-    # for service_database in service_databases:
-    #     for service_name, database in service_database.items():
-    #         username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
-    #         s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
-    #         s.db_name_lists = [
-    #             '聚合数据',
-    #         ]
-    #         s.days = days
-    #         s.optimize_list()
-    for service_database in service_databases:
-        for service_name, database in service_database.items():
-            if socket.gethostname() == 'xigua_lx' or socket.gethostname() == 'xigua1' or socket.gethostname() == 'Mac2.local':
-                # mongodb
-                if is_mongo and database == 'mongodb':
-                    username, password, host, port = get_myconf.select_config_values(
-                        target_service=service_name,
-                        database=database,
-                    )
-                    m = mongo.OptimizeDatas(username=username, password=password, host=host, port=port)
-                    m.db_name_lists = db_name_lists
-                    m.days = days
-                    m.optimize_list()
-                    if m.client:
-                        m.client.close()
-                        print(f'已关闭 mongodb 连接')
-
-                    if socket.gethostname() == 'xigua_lx':
-                        restart_mongodb()  # mongodb 太占内存了, 重启服务， 释放内存
 
-                # Mysql
-                if is_mysql and database == 'mysql':
-                    username, password, host, port = get_myconf.select_config_values(
-                        target_service=service_name,
-                        database=database,
-                    )
-                    s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
-                    s.db_name_lists = db_name_lists
-                    s.days = days
-                    s.optimize_list()
+    if socket.gethostname() == 'xigua_lx' or socket.gethostname() == 'xigua1' or socket.gethostname() == 'mac2.local':
+        # Mysql
+        if is_mysql:
+            s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
+            s.db_name_lists = db_name_lists
+            s.days = days
+            s.optimize_list()
 
-            elif socket.gethostname() == 'company':
-                # Mysql
-                if is_mysql and database == 'mysql':
-                    username, password, host, port = get_myconf.select_config_values(
-                        target_service=service_name,
-                        database=database,
-                    )
-                    s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
-                    s.db_name_lists = db_name_lists
-                    s.days = days
-                    s.optimize_list()
+    elif socket.gethostname() == 'company':
+        # Mysql
+        if is_mysql:
+            s = mysql.OptimizeDatas(username=username, password=password, host=host, port=port)
+            s.db_name_lists = db_name_lists
+            s.days = days
+            s.optimize_list()
 
 
 if __name__ == '__main__':
-    op_data(db_name_lists=['聚合数据'], service_databases=[{'company': 'mysql'}], days=3650, is_mongo=True, is_mysql=True)
+    op_data(db_name_lists=['聚合数据'], days=10, is_mongo=True, is_mysql=True)