mdbq 1.4.1__tar.gz → 1.4.2__tar.gz

{mdbq-1.4.1 → mdbq-1.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.4.1
+Version: 1.4.2
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.4.1 → mdbq-1.4.2}/mdbq/aggregation/mysql_types.py

@@ -154,90 +154,91 @@ class DataTypes:
             return {}, cl, None, None  # 返回这些结果的目的是等添加完列再写 json 文件才能读到 types 信息
 
 
-def mysql_all_dtypes(db_name=None, table_name=None, path=None, system_name=None):
+def mysql_all_dtypes(db_name=None, table_name=None, service_databases=[{'home_lx': 'mysql'}], path=None, system_name=None):
     """
     更新笔记本 mysql 中所有数据库的 dtypes 信息到本地 json
     """
-
-    username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mysql')
-    config = {
-        'host': host,
-        'port': port,
-        'user': username,
-        'password': password,
-        'charset': 'utf8mb4',  # utf8mb4 支持存储四字节的UTF-8字符集
-        'cursorclass': pymysql.cursors.DictCursor,
-    }
-
-    connection = pymysql.connect(**config)  # 连接数据库
-    with connection.cursor() as cursor:
-        sql = "SHOW DATABASES;"
-        cursor.execute(sql)
-        db_name_lists = cursor.fetchall()
-        db_name_lists = [item['Database'] for item in db_name_lists]
-        connection.close()
-
-    sys_lists = ['information_schema', 'mysql', 'performance_schema', 'sakila', 'sys']
-    db_name_lists = [item for item in db_name_lists if item not in sys_lists]
-
-    # db_name_lists = [
-    #     '京东数据2',
-    #     '推广数据2',
-    #     '市场数据2',
-    #     '生意参谋2',
-    #     '生意经2',
-    #     '属性设置2',
-    #     '聚合数据',
-    # ]
-    results = []  # 返回结果示例: [{'云电影': '电影更新'}, {'生意经2': 'e3_零售明细统计'}]
-    for db_ in db_name_lists:
-        config.update({'database': db_})  # 添加更新 config 字段
-        connection = pymysql.connect(**config)  # 连接数据库
-        try:
+    for service_database in service_databases:
+        for service_name, database in service_database.items():
+            username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
+            config = {
+                'host': host,
+                'port': port,
+                'user': username,
+                'password': password,
+                'charset': 'utf8mb4',  # utf8mb4 支持存储四字节的UTF-8字符集
+                'cursorclass': pymysql.cursors.DictCursor,
+            }
+
+            connection = pymysql.connect(**config)  # 连接数据库
             with connection.cursor() as cursor:
-                sql = f"SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '{db_}';"
-                sql = "SHOW TABLES;"
+                sql = "SHOW DATABASES;"
                 cursor.execute(sql)
-                res_tables = cursor.fetchall()
-                for res_table in res_tables:
-                    for k, v in res_table.items():
-                        results.append({db_: v})
-        except:
-            pass
-        finally:
-            connection.close()
-        time.sleep(0.5)
-
-    d = DataTypes(path=path, system_name=system_name)
-    for result in results:
-        for db_n, table_n in result.items():
-            # print(db_n, table_n, db_name, table_name)
-            if db_name and table_name:  # 下载一个指定的数据表
-                if db_name != db_n or table_name != table_n:
-                    continue
-            elif db_name:  # 下载一个数据库的所有数据表
-                if db_name != db_n:
-                    continue
-            # 如果 db_name 和 table_name 都不指定，则下载所有数据库的所有数据表
-            print(f'获取列信息 数据库: < {db_n} >, 数据表: < {table_n} >')
-            sq = s_query.QueryDatas(username=username, password=password, host=host, port=port)
-            # 获取数据表的指定列, 返回列表
-            # [{'视频bv号': 'BV1Dm4y1S7BU', '下载进度': 1}, {'视频bv号': 'BV1ov411c7US', '下载进度': 1}]
-            name_type = sq.dtypes_to_list(db_name=db_n, table_name=table_n)
-            if name_type:
-                dtypes = {item['COLUMN_NAME']: item['COLUMN_TYPE'] for item in name_type}
-                dtypes = {'mysql': {db_n: {table_n: dtypes}}}
-                d.get_mysql_types(
-                    dtypes=dtypes, 
-                    cl='mysql', 
-                    db_name=db_n,
-                    table_name=table_n,
-                    is_file_dtype=True  # True表示旧文件有限
-                )
-            else:
-                print(f'数据库回传数据(name_type)为空')
-    # print(d.datas)
-    d.as_json_file()
+                db_name_lists = cursor.fetchall()
+                db_name_lists = [item['Database'] for item in db_name_lists]
+                connection.close()
+
+            sys_lists = ['information_schema', 'mysql', 'performance_schema', 'sakila', 'sys']
+            db_name_lists = [item for item in db_name_lists if item not in sys_lists]
+
+            # db_name_lists = [
+            #     '京东数据2',
+            #     '推广数据2',
+            #     '市场数据2',
+            #     '生意参谋2',
+            #     '生意经2',
+            #     '属性设置2',
+            #     '聚合数据',
+            # ]
+            results = []  # 返回结果示例: [{'云电影': '电影更新'}, {'生意经2': 'e3_零售明细统计'}]
+            for db_ in db_name_lists:
+                config.update({'database': db_})  # 添加更新 config 字段
+                connection = pymysql.connect(**config)  # 连接数据库
+                try:
+                    with connection.cursor() as cursor:
+                        sql = f"SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '{db_}';"
+                        sql = "SHOW TABLES;"
+                        cursor.execute(sql)
+                        res_tables = cursor.fetchall()
+                        for res_table in res_tables:
+                            for k, v in res_table.items():
+                                results.append({db_: v})
+                except:
+                    pass
+                finally:
+                    connection.close()
+                time.sleep(0.5)
+
+            d = DataTypes(path=path, system_name=system_name)
+            for result in results:
+                for db_n, table_n in result.items():
+                    # print(db_n, table_n, db_name, table_name)
+                    if db_name and table_name:  # 下载一个指定的数据表
+                        if db_name != db_n or table_name != table_n:
+                            continue
+                    elif db_name:  # 下载一个数据库的所有数据表
+                        if db_name != db_n:
+                            continue
+                    # 如果 db_name 和 table_name 都不指定，则下载所有数据库的所有数据表
+                    print(f'获取列信息 数据库: < {db_n} >, 数据表: < {table_n} >')
+                    sq = s_query.QueryDatas(username=username, password=password, host=host, port=port)
+                    # 获取数据表的指定列, 返回列表
+                    # [{'视频bv号': 'BV1Dm4y1S7BU', '下载进度': 1}, {'视频bv号': 'BV1ov411c7US', '下载进度': 1}]
+                    name_type = sq.dtypes_to_list(db_name=db_n, table_name=table_n)
+                    if name_type:
+                        dtypes = {item['COLUMN_NAME']: item['COLUMN_TYPE'] for item in name_type}
+                        dtypes = {'mysql': {db_n: {table_n: dtypes}}}
+                        d.get_mysql_types(
+                            dtypes=dtypes,
+                            cl='mysql',
+                            db_name=db_n,
+                            table_name=table_n,
+                            is_file_dtype=True  # True表示旧文件有限
+                        )
+                    else:
+                        print(f'数据库回传数据(name_type)为空')
+            # print(d.datas)
+            d.as_json_file()
 
 
 if __name__ == '__main__':

{mdbq-1.4.1 → mdbq-1.4.2}/mdbq/aggregation/query_data.py

@@ -494,7 +494,45 @@ class GroupBy:
                 print(f'{filename}: sort_values排序参数错误！')
         df.to_excel(os.path.join(path, filename + '.xlsx'), index=index, header=header, engine=engine, freeze_panes=freeze_panes)
         
-        
+
+def data_aggregation_one(service_databases=[{}], months=1, system_name=None,):
+    """
+    # 单独处理某一个聚合数据库，修改添加 data_dict 的值
+    """
+    for service_database in service_databases:
+        for service_name, database in service_database.items():
+            sdq = MysqlDatasQuery(target_service=service_name)  # 实例化数据处理类
+            sdq.months = months  # 设置数据周期， 1 表示近 2 个月
+            g = GroupBy()  # 实例化数据聚合类
+            # 实例化数据库连接
+            username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
+            m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
+
+            # 从数据库中获取数据, 返回包含 df 数据的字典
+            # 单独处理某一个聚合数据库，在这里修改添加 data_dict 的值
+            data_dict = [
+                {
+                    '数据库名': '聚合数据',
+                    '集合名': '天猫_店铺来源_日数据_旧版',
+                    '唯一主键': ['日期', '一级来源', '二级来源', '三级来源'],
+                    '数据主体': sdq.dplyd(),
+                },
+            ]
+            for items in data_dict:  # 遍历返回结果
+                db_name, table_name, unique_key_list, df = items['数据库名'], items['集合名'], items['唯一主键'], items['数据主体']
+                df = g.groupby(df=df, table_name=table_name, is_maximize=True)  # 2. 聚合数据
+                # g.as_csv(df=df, filename=table_name + '.csv')  # 导出 csv
+                m.df_to_mysql(
+                    df=df,
+                    db_name=db_name,
+                    table_name=table_name,
+                    drop_dup=False,
+                    icm_update=unique_key_list,
+                    system_name=system_name,
+                    service_databases=service_databases,
+                )  # 3. 回传数据库
+
+
 def data_aggregation(service_databases=[{}], months=1, system_name=None,):
     """
     1. 从数据库中读取数据
@@ -561,6 +599,7 @@ def data_aggregation(service_databases=[{}], months=1, system_name=None,):
                     drop_dup=False,
                     icm_update=unique_key_list,
                     system_name=system_name,
+                    service_databases=service_databases,
                 )  # 3. 回传数据库
             res = g.performance(bb_tg=True)   # 盈亏表，依赖其他表，单独做
             m.df_to_mysql(
@@ -570,6 +609,7 @@ def data_aggregation(service_databases=[{}], months=1, system_name=None,):
                 drop_dup=False,
                 icm_update=['日期', '商品id'],  # 设置唯一主键
                 system_name = system_name,
+                service_databases=service_databases,
             )
             res = g.performance(bb_tg=False)  # 盈亏表，依赖其他表，单独做
             m.df_to_mysql(
@@ -579,6 +619,7 @@ def data_aggregation(service_databases=[{}], months=1, system_name=None,):
                 drop_dup=False,
                 icm_update=['日期', '商品id'],  # 设置唯一主键
                 system_name=system_name,
+                service_databases=service_databases,
             )
 
     # 这里要注释掉，不然 copysh.py 可能有问题，这里主要修改配置文件，后续触发 home_lx 的 optimize_datas.py(有s)程序进行全局清理
@@ -586,5 +627,6 @@ def data_aggregation(service_databases=[{}], months=1, system_name=None,):
 
 
 if __name__ == '__main__':
-    data_aggregation(service_databases=[{'company': 'mysql'}], months=1, system_name='company')
+    # data_aggregation(service_databases=[{'company': 'mysql'}], months=1, system_name='company')
+    data_aggregation_one(service_databases=[{'company': 'mysql'}], months=10, system_name='company')
     # optimize_data.op_data(service_databases=[{'company': 'mysql'}], days=3650)  # 立即启动对聚合数据的清理工作

{mdbq-1.4.1 → mdbq-1.4.2}/mdbq/mysql/mysql.py

@@ -57,7 +57,7 @@ class MysqlUpload:
         }
         self.filename = None
 
-    def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], icm_up=[], df_sql=False, drop_dup=False, drop_duplicates=False, filename=None, count=None, json_path=None, system_name=None):
+    def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], icm_up=[], service_databases=[{'home_lx': 'mysql'}], df_sql=False, drop_dup=False, drop_duplicates=False, filename=None, count=None, json_path=None, system_name=None):
         """
         将 df 写入数据库
         db_name: 数据库名称
@@ -167,11 +167,11 @@ class MysqlUpload:
             # 返回这些结果的目的是等添加完列再写 json 文件才能读到 types 信息
             # ⚠️ mysql_all_dtypes 函数默认只读取 home_lx 的数据库信息，不会读取其他系统
             if cl and db_n and tb_n:
-                mysql_types.mysql_all_dtypes(db_name=db_name, table_name=table_name, system_name=system_name)  # 更新一个表的 dtypes
+                mysql_types.mysql_all_dtypes(service_databases=service_databases, db_name=db_name, table_name=table_name, system_name=system_name)  # 更新一个表的 dtypes
             elif cl and db_n:
-                mysql_types.mysql_all_dtypes(db_name=db_name, system_name=system_name)  # 更新一个数据库的 dtypes
+                mysql_types.mysql_all_dtypes(service_databases=service_databases, db_name=db_name, system_name=system_name)  # 更新一个数据库的 dtypes
             elif cl:
-                mysql_types.mysql_all_dtypes(system_name=system_name)  # 更新所有数据库所有数据表的 dtypes 信息到本地 json
+                mysql_types.mysql_all_dtypes(service_databases=service_databases, system_name=system_name)  # 更新所有数据库所有数据表的 dtypes 信息到本地 json
 
             # 4. 移除指定日期范围内的数据，仅限于聚合数据使用，其他情况不要设置
             if drop_duplicates and '日期' in df.columns.tolist():

{mdbq-1.4.1 → mdbq-1.4.2}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.4.1
+Version: 1.4.2
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.4.1 → mdbq-1.4.2}/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='mdbq',
-      version='1.4.1',
+      version='1.4.2',
       author='xigua, ',
       author_email="2587125111@qq.com",
       url='https://pypi.org/project/mdbsql',