mdbq 0.0.6__tar.gz → 0.0.8__tar.gz

{mdbq-0.0.6 → mdbq-0.0.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 0.0.6
+Version: 0.0.8
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-0.0.6 → mdbq-0.0.8}/mdbq/aggregation/aggregation.py

@@ -118,11 +118,13 @@ class DataTypes:
         """
         # dtypes = df.dtypes.apply(str).to_dict()  # 将 dataframe 数据类型转为字典形式
         df.replace([np.inf, -np.inf], 0, inplace=True)  # 清理一些非法值
-
-        cols = df.columns.tolist()
         df.replace(to_replace=['\\N', '-', '--', '', 'nan'], value=0, regex=False, inplace=True)  # 替换掉特殊字符
         df.replace(to_replace=[','], value='', regex=True, inplace=True)
+        df.replace(to_replace=['="'], value='', regex=True, inplace=True)  # ="和"不可以放在一起清洗, 因为有: id=86785565
+        df.replace(to_replace=['"'], value='', regex=True, inplace=True)
+        cols = df.columns.tolist()
         for col in cols:
+            # df[col] = df[col].apply(lambda x: re.sub('[="]', '', str(x)) if '="' in str(x) else x)
             # 百分比在某些数据库中不兼容, 转换百分比为小数
             df[col] = df[col].apply(lambda x: float(float((str(x).rstrip("%"))) / 100) if str(x).endswith('%') and '~' not in str(x) else x)
             # 尝试转换合适的数据类型
@@ -976,9 +978,9 @@ def update_dtypte():
 
 def upload():
     """ 上传一个文件夹到数据库 """
-    path = '/Users/xigua/数据中心/原始文件2/生意参谋/客户_客户概况_画像'
-    db_name = '生意参谋数据2'
-    collection_name = '客户_客户概况_画像'
+    path = '/Users/xigua/数据中心/原始文件2/生意经/宝贝指标'
+    db_name = '生意经2'
+    collection_name = '宝贝指标'
 
     username, password, host, port = get_myconf.select_config_values(
         target_service='home_lx',
@@ -1018,10 +1020,7 @@ def upload():
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                     if len(df) == 0:
                         continue
-                    for col in df.columns.tolist():
-                        df[col] = df[col].apply(lambda x: re.sub('[="]', '', str(x)) if '="' in str(x) else x)
-                    # df.replace(to_replace=['--'], value='', regex=False, inplace=True)
-                    df = dt.convert_df_cols(df=df)
+                    df = dt.convert_df_cols(df=df)  # 清理列名和 df 中的非法字符
                     try:
                         df = df.astype(dtypes)
                     except Exception as e:
@@ -1044,8 +1043,8 @@ def upload():
 
 
 if __name__ == '__main__':
-    username, password, host, port = get_myconf.select_config_values(target_service='aliyun', database='mongodb')
-    print(username, password, host, port)
+    # username, password, host, port = get_myconf.select_config_values(target_service='aliyun', database='mongodb')
+    # print(username, password, host, port)
 
     # main()
-    # upload()
+    upload()

mdbq-0.0.8/mdbq/aggregation/query_data.py

@@ -0,0 +1,321 @@
+# -*- coding: UTF-8 –*-
+from mdbq.mongo import mongo
+from mdbq.mysql import s_query
+from mdbq.config import get_myconf
+import datetime
+from dateutil.relativedelta import relativedelta
+import pandas as pd
+import numpy as np
+import platform
+import getpass
+import json
+import os
+
+
+class MongoDatasQuery:
+    """
+    从 数据库 中下载数据
+    self.output: 数据库默认导出目录
+    self.is_maximize: 是否最大转化数据
+    """
+    def __init__(self, target_service):
+        # target_service 从哪个服务器下载数据
+        self.months = 0  # 下载几个月数据, 0 表示当月, 1 是上月 1 号至今
+        # 实例化一个下载类
+        username, password, host, port = get_myconf.select_config_values(target_service=target_service, database='mongodb')
+        self.download = mongo.DownMongo(username=username, password=password, host=host, port=port, save_path=None)
+
+    def tg_wxt(self):
+        self.download.start_date, self.download.end_date = self.months_data(num=self.months)
+        projection = {
+            '日期': 1,
+            '场景名字': 1,
+            '主体id': 1,
+            '花费': 1,
+            '展现量': 1,
+            '点击量': 1,
+            '总购物车数': 1,
+            '总成交笔数': 1,
+            '总成交金额': 1,
+            '自然流量曝光量': 1,
+            '直接成交笔数': 1,
+            '直接成交金额': 1,
+        }
+        df = self.download.data_to_df(
+            db_name='天猫数据2',
+            collection_name='推广数据_宝贝主体报表',
+            projection=projection,
+        )
+        return df
+
+    @staticmethod
+    def days_data(days, end_date=None):
+        """ 读取近 days 天的数据 """
+        if not end_date:
+            end_date = datetime.datetime.now()
+        start_date = end_date - datetime.timedelta(days=days)
+        return pd.to_datetime(start_date), pd.to_datetime(end_date)
+
+    @staticmethod
+    def months_data(num=0, end_date=None):
+        """ 读取近 num 个月的数据, 0 表示读取当月的数据 """
+        if not end_date:
+            end_date = datetime.datetime.now()
+        start_date = end_date - relativedelta(months=num)  # n 月以前的今天
+        start_date = f'{start_date.year}-{start_date.month}-01'  # 替换为 n 月以前的第一天
+        return pd.to_datetime(start_date), pd.to_datetime(end_date)
+
+
+class MysqlDatasQuery:
+    """
+    从数据库中下载数据
+    """
+    def __init__(self, target_service):
+        # target_service 从哪个服务器下载数据
+        self.months = 0  # 下载几个月数据, 0 表示当月, 1 是上月 1 号至今
+        # 实例化一个下载类
+        username, password, host, port = get_myconf.select_config_values(target_service=target_service, database='mysql')
+        self.download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
+
+    def tg_wxt(self):
+        start_date, end_date = self.months_data(num=self.months)
+        projection = {
+            '日期': 1,
+            '场景名字': 1,
+            '主体id': 1,
+            '花费': 1,
+            '展现量': 1,
+            '点击量': 1,
+            '总购物车数': 1,
+            '总成交笔数': 1,
+            '总成交金额': 1,
+            '自然流量曝光量': 1,
+            '直接成交笔数': 1,
+            '直接成交金额': 1,
+        }
+        df = self.download.data_to_df(
+            db_name='天猫数据2',
+            tabel_name='推广数据_宝贝主体报表',
+            start_date=start_date,
+            end_date=end_date,
+            projection=projection,
+        )
+        return df
+
+    def syj(self):
+        start_date, end_date = self.months_data(num=self.months)
+        projection = {
+            '日期': 1,
+            '宝贝id': 1,
+            '商家编码': 1,
+            '行业类目': 1,
+            '销售额': 1,
+            '销售量': 1,
+            '订单数': 1,
+            '退货量': 1,
+            '退款额': 1,
+            '退货量_发货后_': 1,
+        }
+        df = self.download.data_to_df(
+            db_name='生意经2',
+            tabel_name='宝贝指标',
+            start_date=start_date,
+            end_date=end_date,
+            projection=projection,
+        )
+        return df
+
+
+    @staticmethod
+    def months_data(num=0, end_date=None):
+        """ 读取近 num 个月的数据, 0 表示读取当月的数据 """
+        if not end_date:
+            end_date = datetime.datetime.now()
+        start_date = end_date - relativedelta(months=num)  # n 月以前的今天
+        start_date = f'{start_date.year}-{start_date.month}-01'  # 替换为 n 月以前的第一天
+        return pd.to_datetime(start_date), pd.to_datetime(end_date)
+
+
+class GroupBy:
+    """ 数据聚合和导出 """
+    def __init__(self):
+        # self.output: 数据库默认导出目录
+        if platform.system() == 'Darwin':
+            self.output = os.path.join('/Users', getpass.getuser(), '数据中心/数据库导出')
+        elif platform.system() == 'Windows':
+            self.output = os.path.join('C:\\同步空间\\BaiduSyncdisk\\数据库导出')
+        else:
+            self.output = os.path.join('数据中心/数据库导出')
+
+    def groupby(self, df, tabel_name, is_maximize=True):
+        """
+        self.is_maximize: 是否最大转化数据
+        """
+
+        if '宝贝主体报表' in tabel_name:
+            df.rename(columns={
+                '场景名字': '营销场景',
+                '主体id': '商品id',
+                '总购物车数': '加购量',
+                '总成交笔数': '成交笔数',
+                '总成交金额': '成交金额'
+            }, inplace=True)
+            df = df.astype({
+                '花费': float,
+                '展现量': int,
+                '点击量': int,
+                '加购量': int,
+                '成交笔数': int,
+                '成交金额': float,
+                '自然流量曝光量': int,
+                '直接成交笔数': int,
+                '直接成交金额': float,
+            }, errors='raise')
+            df.fillna(0, inplace=True)
+            if is_maximize:
+                df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
+                    **{'加购量': ('加购量', np.max),
+                       '成交笔数': ('成交笔数', np.max),
+                       '成交金额': ('成交金额', np.max),
+                       '自然流量曝光量': ('自然流量曝光量', np.max),
+                       '直接成交笔数': ('直接成交笔数', np.max),
+                       '直接成交金额': ('直接成交金额', np.max)
+                       }
+                )
+            else:
+                df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
+                    **{'加购量': ('加购量', np.min),
+                       '成交笔数': ('成交笔数', np.min),
+                       '成交金额': ('成交金额', np.min),
+                       '自然流量曝光量': ('自然流量曝光量', np.min),
+                       '直接成交笔数': ('直接成交笔数', np.max),
+                       '直接成交金额': ('直接成交金额', np.max)
+                       }
+                )
+            df.insert(loc=1, column='推广渠道', value='万相台无界版')  # df中插入新列
+            return df
+        if '宝贝指标' in tabel_name:
+            df.fillna(0, inplace=True)
+            df = df[(df['销售额'] != 0) | (df['退款额'] != 0)]
+            df = df.groupby(['日期', '宝贝id', '商家编码', '行业类目'], as_index=False).agg(
+                **{'销售额': ('销售额', np.min),
+                   '销售量': ('销售量', np.min),
+                   '订单数': ('订单数', np.min),
+                   '退货量': ('退货量', np.max),
+                   '退款额': ('退款额', np.max),
+                   '退货量_发货后_': ('退货量_发货后_', np.max),
+                   }
+            )
+            df['件均价'] = df.apply(lambda x: x['销售额'] / x['销售量'] if x['销售量'] > 0 else 0, axis=1).round(
+                0)  # 两列运算, 避免除以0
+            df['价格带'] = df['件均价'].apply(
+                lambda x: '2000+' if x >= 2000
+                else '1000+' if x >= 1000
+                else '500+' if x >= 500
+                else '300+' if x >= 300
+                else '300以下'
+            )
+            return df
+        else:
+            print(f'<{tabel_name}>: Groupby 类尚未配置，数据为空')
+            return pd.DataFrame({})
+    
+    def as_csv(self, df, filename, path=None, encoding='utf-8_sig',
+               index=False, header=True, st_ascend=None, ascend=None, freq=None):
+        """
+        path: 默认导出目录 self.output, 这个函数的 path 作为子文件夹，可以不传，
+        st_ascend: 排序参数 ['column1', 'column2']
+        ascend: 升降序 [True, False]
+        freq: 将创建子文件夹并按月分类存储,  freq='Y', 或 freq='M'
+        """
+        if len(df) == 0:
+            return
+        if not path:
+            path = self.output
+        else:
+            path = os.path.join(self.output, path)
+        if not os.path.exists(path):
+            os.makedirs(path)
+        if st_ascend and ascend:
+            try:
+                df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
+            except:
+                print(f'{filename}: sort_values排序参数错误！')
+        if freq:
+            if '日期' not in df.columns.tolist():
+                return print(f'{filename}: 数据缺少日期列，无法按日期分组')
+            groups = df.groupby(pd.Grouper(key='日期', freq=freq))
+            for name1, df in groups:
+                if freq == 'M':
+                    sheet_name = name1.strftime('%Y-%m')
+                elif freq == 'Y':
+                    sheet_name = name1.strftime('%Y年')
+                else:
+                    sheet_name = '_未分类'
+                new_path = os.path.join(path, filename)
+                if not os.path.exists(new_path):
+                    os.makedirs(new_path)
+                new_path = os.path.join(new_path, f'{filename}{sheet_name}.csv')
+                if st_ascend and ascend:  # 这里需要重新排序一次，原因未知
+                    try:
+                        df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
+                    except:
+                        print(f'{filename}: sort_values排序参数错误！')
+
+                df.to_csv(new_path, encoding=encoding, index=index, header=header)
+        else:
+            df.to_csv(os.path.join(path, filename + '.csv'), encoding=encoding, index=index, header=header)
+
+    def as_json(self, df, filename, path=None, orient='records', force_ascii=False, st_ascend=None, ascend=None):
+        if len(df) == 0:
+            return
+        if not path:
+            path = self.output
+        else:
+            path = os.path.join(self.output, path)
+        if not os.path.exists(path):
+            os.makedirs(path)
+        if st_ascend and ascend:
+            try:
+                df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
+            except:
+                print(f'{filename}: sort_values排序参数错误！')
+        df.to_json(os.path.join(path, filename + '.json'),
+                   orient=orient, force_ascii=force_ascii)
+
+    def as_excel(self, df, filename, path=None, index=False, header=True, engine='openpyxl',
+                 freeze_panes=(1, 0), st_ascend=None, ascend=None):
+        if len(df) == 0:
+            return
+        if not path:
+            path = self.output
+        else:
+            path = os.path.join(self.output, path)
+        if not os.path.exists(path):
+            os.makedirs(path)
+        if st_ascend and ascend:
+            try:
+                df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
+            except:
+                print(f'{filename}: sort_values排序参数错误！')
+        df.to_excel(os.path.join(path, filename + '.xlsx'),
+                    index=index, header=header, engine=engine, freeze_panes=freeze_panes)
+        
+        
+def main():
+    sdq = MysqlDatasQuery(target_service='home_lx')
+    sdq.months = 0
+
+    # df = sdq.tg_wxt()  # 从数据库中获取数据并转为 df
+    # g = GroupBy()  # 数据聚合
+    # df = g.groupby(df=df, tabel_name='推广数据_宝贝主体报表', is_maximize=True)
+    # g.as_csv(df=df, filename='推广数据_宝贝主体报表')  # 数据导出
+
+    df = sdq.syj()
+    g = GroupBy()
+    df = g.groupby(df=df, tabel_name='宝贝指标', is_maximize=True)
+    g.as_csv(df=df, filename='宝贝指标')
+
+
+if __name__ == '__main__':
+    main()

mdbq-0.0.8/mdbq/mysql/s_query.py

@@ -0,0 +1,120 @@
+# -*- coding:utf-8 -*-
+import datetime
+import platform
+import re
+import time
+from functools import wraps
+import warnings
+import pymysql
+import numpy as np
+import pandas as pd
+from sqlalchemy import create_engine
+import os
+import calendar
+from mdbq.config import get_myconf
+
+warnings.filterwarnings('ignore')
+
+
+class QueryDatas:
+    def __init__(self, username: str, password: str, host: str, port: int, charset: str = 'utf8mb4'):
+        self.username = username
+        self.password = password
+        self.host = host
+        self.port = port
+        self.config = {
+            'host': self.host,
+            'port': self.port,
+            'user': self.username,
+            'password': self.password,
+            'charset': charset,  # utf8mb4 支持存储四字节的UTF-8字符集
+            'cursorclass': pymysql.cursors.DictCursor,
+        }
+
+    def data_to_df(self, db_name, tabel_name, start_date, end_date, projection: dict=[]):
+
+        start_date = pd.to_datetime(start_date).strftime('%Y-%m-%d')
+        end_date = pd.to_datetime(end_date).strftime('%Y-%m-%d')
+        df = pd.DataFrame()  # 初始化df
+
+        if self.check_infos(db_name, tabel_name) == False:
+            return df
+
+        self.config.update({'database': db_name})
+        connection = pymysql.connect(**self.config)  # 重新连接数据库
+        try:
+            with connection.cursor() as cursor:
+                # 3. 获取数据表的所有列信息
+                sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
+                cursor.execute(sql, (db_name, {tabel_name}))
+                columns = cursor.fetchall()
+                cols_exist = [col['COLUMN_NAME'] for col in columns]  # 数据表的所有列, 返回 list
+
+                # 4. 构建 SQL 查询语句
+                if projection:  # 获取指定列
+                    columns_in = []
+                    for key, value in projection.items():
+                        if value == 1 and key in cols_exist:
+                            columns_in.append(key)  # 提取值为 1 的键并清理不在数据表的键
+                    columns_in = ', '.join(columns_in)
+                    if '日期' in cols_exist:  # 不论是否指定, 只要数据表有日期，则执行
+                        sql = (f"SELECT {columns_in} FROM {db_name}.{tabel_name} "
+                               f"WHERE {'日期'} BETWEEN '{start_date}' AND '{end_date}'")
+                    else:  # 数据表没有日期列时，返回指定列的所有数据
+                        sql = f"SELECT {columns_in} FROM {db_name}.{tabel_name}"
+                else:  # 没有指定获取列时
+                    if '日期' in cols_exist:  # 但数据表有日期，仍然执行
+                        columns_in = ', '.join(cols_exist)
+                        sql = (f"SELECT {columns_in} FROM {db_name}.{tabel_name} "
+                               f"WHERE {'日期'} BETWEEN '{start_date}' AND '{end_date}'")
+                    else:  # 没有指定获取列，且数据表也没有日期列，则返回全部列的全部数据
+                        sql = f"SELECT * FROM {db_name}.{tabel_name}"
+                cursor.execute(sql)
+                rows = cursor.fetchall()  # 获取查询结果
+                columns = [desc[0] for desc in cursor.description]
+                df = pd.DataFrame(rows, columns=columns)  # 转为 df
+        except Exception as e:
+            print(f'{e}')
+            return df
+        finally:
+            connection.close()
+
+        if len(df) == 0:
+            print(f'database: {db_name}, table: {tabel_name} 查询的数据为空')
+        return df
+
+    def check_infos(self, db_name, tabel_name) -> bool:
+        """ 检查数据库、数据表是否存在 """
+        connection = pymysql.connect(**self.config)  # 连接数据库
+        try:
+            with connection.cursor() as cursor:
+                # 1. 检查数据库是否存在
+                cursor.execute(f"SHOW DATABASES LIKE '{db_name}'")  # 检查数据库是否存在
+                database_exists = cursor.fetchone()
+                if not database_exists:
+                    print(f"Database <{db_name}>: 数据库不存在")
+                    return False
+        finally:
+            connection.close()  # 这里要断开连接
+
+        self.config.update({'database': db_name})  # 添加更新 config 字段
+        connection = pymysql.connect(**self.config)  # 重新连接数据库
+        try:
+            with connection.cursor() as cursor:
+                # 2. 查询表是否存在
+                sql = f"SHOW TABLES LIKE '{tabel_name}'"
+                cursor.execute(sql)
+                if not cursor.fetchone():
+                    print(f'{db_name} -> <{tabel_name}>: 表不存在')
+                    return False
+                return True
+        except Exception as e:
+            print(e)
+            return False
+        finally:
+            connection.close()  # 断开连接
+
+
+if __name__ == '__main__':
+    username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
+    print(username, password, host, port)

mdbq-0.0.8/mdbq/mysql/year_month_day.py

@@ -0,0 +1,38 @@
+# -*- coding:utf-8 -*-
+import warnings
+import pandas as pd
+import calendar
+
+warnings.filterwarnings('ignore')
+
+
+def year_month_day(start_date, end_date):
+    """
+    使用date_range函数和DataFrame来获取从start_date至end_date之间的所有年月日
+    calendar.monthrange： 获取当月第一个工作日的星期值(0,6) 以及当月天数
+    返回值: [{'起始日期': '2025-05-01', '结束日期': '2025-05-31'}, {'起始日期': '2025-06-01', '结束日期': '2025-06-30'}]
+    """
+    # 替换年月日中的日, 以便即使传入当月日期也有返回值
+    try:
+        start_date = f'{pd.to_datetime(start_date).year}-{pd.to_datetime(start_date).month}-01'
+    except Exception as e:
+        print(e)
+        return []
+    # 使用pandas的date_range创建一个日期范围，频率为'MS'代表每月开始
+    date_range = pd.date_range(start=start_date, end=end_date, freq='MS')
+    # 转换格式
+    year_months = date_range.strftime('%Y-%m').drop_duplicates().sort_values()
+
+    results = []
+    for year_month in year_months:
+        year = re.findall(r'(\d{4})', year_month)[0]
+        month = re.findall(r'\d{4}-(\d{2})', year_month)[0]
+        s, d = calendar.monthrange(int(year), int(month))
+        results.append({'起始日期': f'{year_month}-01', '结束日期': f'{year_month}-{d}'})
+
+    return results  # start_date至end_date之间的所有年月日
+
+
+if __name__ == '__main__':
+    results = year_month_day(start_date='2025-05-01', end_date='2025-08-01')
+    print(results)

{mdbq-0.0.6 → mdbq-0.0.8}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 0.0.6
+Version: 0.0.8
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-0.0.6 → mdbq-0.0.8}/mdbq.egg-info/SOURCES.txt

@@ -25,6 +25,7 @@ mdbq/mongo/mongo.py
 mdbq/mysql/__init__.py
 mdbq/mysql/mysql.py
 mdbq/mysql/s_query.py
+mdbq/mysql/year_month_day.py
 mdbq/other/__init__.py
 mdbq/other/porxy.py
 mdbq/other/pov_city.py

{mdbq-0.0.6 → mdbq-0.0.8}/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='mdbq',
-      version='0.0.6',
+      version='0.0.8',
       author='xigua, ',
       author_email="2587125111@qq.com",
       url='https://pypi.org/project/mdbsql',

mdbq-0.0.6/mdbq/aggregation/query_data.py

@@ -1,307 +0,0 @@
-# -*- coding: UTF-8 –*-
-from mdbq.mongo import mongo
-from mdbq.mysql import s_query
-from mdbq.config import get_myconf
-import datetime
-from dateutil.relativedelta import relativedelta
-import pandas as pd
-import numpy as np
-import platform
-import getpass
-import json
-import os
-
-
-class MongoDatasQuery:
-    """
-    从 数据库 中下载数据
-    self.output: 数据库默认导出目录
-    self.is_maximize: 是否最大转化数据
-    """
-    def __init__(self, target_service):
-        # target_service 从哪个服务器下载数据
-        self.is_maximize = True
-        if platform.system() == 'Darwin':
-            self.output = os.path.join('/Users', getpass.getuser(), '数据中心/数据库导出')
-        elif platform.system() == 'Windows':
-            self.output = os.path.join('C:\\同步空间\\BaiduSyncdisk\\数据库导出')
-        else:
-            self.output = os.path.join('数据中心/数据库导出')
-
-        # 实例化一个下载类
-        username, password, host, port = get_myconf.select_config_values(target_service=target_service, database='mongodb')
-        self.download = mongo.DownMongo(username=username, password=password, host=host, port=port, save_path=None)
-
-    def tg_wxt(self):
-        self.download.start_date, self.download.end_date = self.months_data(num=1)
-        projection = {
-            '日期': 1,
-            '场景名字': 1,
-            '主体id': 1,
-            '花费': 1,
-            '展现量': 1,
-            '点击量': 1,
-            '总购物车数': 1,
-            '总成交笔数': 1,
-            '总成交金额': 1,
-            '自然流量曝光量': 1,
-            '直接成交笔数': 1,
-            '直接成交金额': 1,
-        }
-        df = self.download.data_to_df(db_name='天猫数据2', collection_name='推广数据_宝贝主体报表', projection=projection)
-        df.rename(columns={
-            '场景名字': '营销场景',
-            '主体id': '商品id',
-            '总购物车数': '加购量',
-            '总成交笔数': '成交笔数',
-            '总成交金额': '成交金额'
-        }, inplace=True)
-        df = df.astype({
-            '花费': float,
-            '展现量': int,
-            '点击量': int,
-            '加购量': int,
-            '成交笔数': int,
-            '成交金额': float,
-            '自然流量曝光量': int,
-            '直接成交笔数': int,
-            '直接成交金额': float,
-        }, errors='raise')
-        df.fillna(0, inplace=True)
-        if self.is_maximize:
-            df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
-                **{'加购量': ('加购量', np.max),
-                   '成交笔数': ('成交笔数', np.max),
-                   '成交金额': ('成交金额', np.max),
-                   '自然流量曝光量': ('自然流量曝光量', np.max),
-                   '直接成交笔数': ('直接成交笔数', np.max),
-                   '直接成交金额': ('直接成交金额', np.max)
-                   }
-            )
-        else:
-            df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
-                **{'加购量': ('加购量', np.min),
-                   '成交笔数': ('成交笔数', np.min),
-                   '成交金额': ('成交金额', np.min),
-                   '自然流量曝光量': ('自然流量曝光量', np.min),
-                   '直接成交笔数': ('直接成交笔数', np.max),
-                   '直接成交金额': ('直接成交金额', np.max)
-                   }
-            )
-        df.insert(loc=1, column='推广渠道', value='万相台无界版')  # df中插入新列
-        # print(df)
-        return df
-
-    @staticmethod
-    def days_data(days, end_date=None):
-        """ 读取近 days 天的数据 """
-        if not end_date:
-            end_date = datetime.datetime.now()
-        start_date = end_date - datetime.timedelta(days=days)
-        return pd.to_datetime(start_date), pd.to_datetime(end_date)
-
-    @staticmethod
-    def months_data(num=0, end_date=None):
-        """ 读取近 num 个月的数据, 0 表示读取当月的数据 """
-        if not end_date:
-            end_date = datetime.datetime.now()
-        start_date = end_date - relativedelta(months=num)  # n 月以前的今天
-        start_date = f'{start_date.year}-{start_date.month}-01'  # 替换为 n 月以前的第一天
-        return pd.to_datetime(start_date), pd.to_datetime(end_date)
-
-    def as_csv(self, df, filename, path=None, encoding='utf-8_sig',
-               index=False, header=True, st_ascend=None, ascend=None, freq=None):
-        """
-        path: 子文件夹，可以不传，默认导出目录 self.output
-        st_ascend: 排序参数
-        ascend: 升降序
-        freq: 将创建子文件夹并按月分类存储,  freq='Y'，或 freq='M'
-        """
-        if not path:
-            path = self.output
-        else:
-            path = os.path.join(self.output, path)
-        if not os.path.exists(path):
-            os.makedirs(path)
-        if st_ascend and ascend:
-            try:
-                df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
-            except:
-                print(f'{filename}: sort_values排序参数错误！')
-        if freq:
-            if '日期' not in df.columns.tolist():
-                return print(f'{filename}: 数据缺少日期列，无法按日期分组')
-            groups = df.groupby(pd.Grouper(key='日期', freq=freq))
-            for name1, df in groups:
-                if freq == 'M':
-                    sheet_name = name1.strftime('%Y-%m')
-                elif freq == 'Y':
-                    sheet_name = name1.strftime('%Y年')
-                else:
-                    sheet_name = '_未分类'
-                new_path = os.path.join(path, filename)
-                if not os.path.exists(new_path):
-                    os.makedirs(new_path)
-                new_path = os.path.join(new_path, f'{filename}{sheet_name}.csv')
-                if st_ascend and ascend:  # 这里需要重新排序一次，原因未知
-                    try:
-                        df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
-                    except:
-                        print(f'{filename}: sort_values排序参数错误！')
-
-                df.to_csv(new_path, encoding=encoding, index=index, header=header)
-        else:
-            df.to_csv(os.path.join(path, filename + '.csv'), encoding=encoding, index=index, header=header)
-
-    def as_json(self, df, filename, path=None, orient='records', force_ascii=False, st_ascend=None, ascend=None):
-        if not path:
-            path = self.output
-        else:
-            path = os.path.join(self.output, path)
-        if st_ascend and ascend:
-            try:
-                df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
-            except:
-                print(f'{filename}: sort_values排序参数错误！')
-        df.to_json(os.path.join(path, filename + '.json'),
-                   orient=orient, force_ascii=force_ascii)
-
-    def as_excel(self, df, filename, path=None, index=False, header=True, engine='openpyxl',
-                 freeze_panes=(1, 0), st_ascend=None, ascend=None):
-        if not path:
-            path = self.output
-        else:
-            path = os.path.join(self.output, path)
-        if st_ascend and ascend:
-            try:
-                df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
-            except:
-                print(f'{filename}: sort_values排序参数错误！')
-        df.to_excel(os.path.join(path, filename + '.xlsx'),
-                    index=index, header=header, engine=engine, freeze_panes=freeze_panes)
-
-
-class MysqlDatasQuery:
-    """
-    从 数据库 中下载数据
-    self.output: 数据库默认导出目录
-    self.is_maximize: 是否最大转化数据
-    """
-    def __init__(self, target_service):
-        # target_service 从哪个服务器下载数据
-        self.is_maximize = True
-        if platform.system() == 'Darwin':
-            self.output = os.path.join('/Users', getpass.getuser(), '数据中心/数据库导出')
-        elif platform.system() == 'Windows':
-            self.output = os.path.join('C:\\同步空间\\BaiduSyncdisk\\数据库导出')
-        else:
-            self.output = os.path.join('数据中心/数据库导出')
-        self.months = 1  # 下载几个月数据, 0 表示当月, 1 是上月 1 号至今
-
-        # 实例化一个下载类
-        username, password, host, port = get_myconf.select_config_values(target_service=target_service, database='mysql')
-        self.download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
-
-    def tg_wxt(self):
-        start_date, end_date = self.months_data(num=self.months)
-        df = self.download.data_to_df(db_name='天猫数据2', tabel_name='推广数据_宝贝主体报表', start_date=start_date, end_date=end_date)
-        return df
-
-    @staticmethod
-    def days_data(days, end_date=None):
-        """ 读取近 days 天的数据 """
-        if not end_date:
-            end_date = datetime.datetime.now()
-        start_date = end_date - datetime.timedelta(days=days)
-        return pd.to_datetime(start_date), pd.to_datetime(end_date)
-
-    @staticmethod
-    def months_data(num=0, end_date=None):
-        """ 读取近 num 个月的数据, 0 表示读取当月的数据 """
-        if not end_date:
-            end_date = datetime.datetime.now()
-        start_date = end_date - relativedelta(months=num)  # n 月以前的今天
-        start_date = f'{start_date.year}-{start_date.month}-01'  # 替换为 n 月以前的第一天
-        return pd.to_datetime(start_date), pd.to_datetime(end_date)
-
-    def as_csv(self, df, filename, path=None, encoding='utf-8_sig',
-               index=False, header=True, st_ascend=None, ascend=None, freq=None):
-        """
-        path: 子文件夹，可以不传，默认导出目录 self.output
-        st_ascend: 排序参数
-        ascend: 升降序
-        freq: 将创建子文件夹并按月分类存储,  freq='Y'，或 freq='M'
-        """
-        if not path:
-            path = self.output
-        else:
-            path = os.path.join(self.output, path)
-        if not os.path.exists(path):
-            os.makedirs(path)
-        if st_ascend and ascend:
-            try:
-                df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
-            except:
-                print(f'{filename}: sort_values排序参数错误！')
-        if freq:
-            if '日期' not in df.columns.tolist():
-                return print(f'{filename}: 数据缺少日期列，无法按日期分组')
-            groups = df.groupby(pd.Grouper(key='日期', freq=freq))
-            for name1, df in groups:
-                if freq == 'M':
-                    sheet_name = name1.strftime('%Y-%m')
-                elif freq == 'Y':
-                    sheet_name = name1.strftime('%Y年')
-                else:
-                    sheet_name = '_未分类'
-                new_path = os.path.join(path, filename)
-                if not os.path.exists(new_path):
-                    os.makedirs(new_path)
-                new_path = os.path.join(new_path, f'{filename}{sheet_name}.csv')
-                if st_ascend and ascend:  # 这里需要重新排序一次，原因未知
-                    try:
-                        df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
-                    except:
-                        print(f'{filename}: sort_values排序参数错误！')
-
-                df.to_csv(new_path, encoding=encoding, index=index, header=header)
-        else:
-            df.to_csv(os.path.join(path, filename + '.csv'), encoding=encoding, index=index, header=header)
-
-    def as_json(self, df, filename, path=None, orient='records', force_ascii=False, st_ascend=None, ascend=None):
-        if not path:
-            path = self.output
-        else:
-            path = os.path.join(self.output, path)
-        if st_ascend and ascend:
-            try:
-                df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
-            except:
-                print(f'{filename}: sort_values排序参数错误！')
-        df.to_json(os.path.join(path, filename + '.json'),
-                   orient=orient, force_ascii=force_ascii)
-
-    def as_excel(self, df, filename, path=None, index=False, header=True, engine='openpyxl',
-                 freeze_panes=(1, 0), st_ascend=None, ascend=None):
-        if not path:
-            path = self.output
-        else:
-            path = os.path.join(self.output, path)
-        if st_ascend and ascend:
-            try:
-                df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
-            except:
-                print(f'{filename}: sort_values排序参数错误！')
-        df.to_excel(os.path.join(path, filename + '.xlsx'),
-                    index=index, header=header, engine=engine, freeze_panes=freeze_panes)
-
-
-def main():
-    sdq = MysqlDatasQuery(target_service='company')
-    sdq.months = 0
-    df = sdq.tg_wxt()
-    print(df)
-
-
-if __name__ == '__main__':
-    main()

mdbq-0.0.6/mdbq/mysql/s_query.py

@@ -1,165 +0,0 @@
-# -*- coding:utf-8 -*-
-import datetime
-import platform
-import re
-import time
-from functools import wraps
-import warnings
-import pymysql
-import numpy as np
-import pandas as pd
-from sqlalchemy import create_engine
-import os
-import calendar
-from mdbq.config import get_myconf
-
-warnings.filterwarnings('ignore')
-
-
-class QueryDatas:
-    def __init__(self, username: str, password: str, host: str, port: int, charset: str = 'utf8mb4'):
-        self.username = username
-        self.password = password
-        self.host = host
-        self.port = port
-        self.config = {
-            'host': self.host,
-            'port': self.port,
-            'user': self.username,
-            'password': self.password,
-            'charset': charset,  # utf8mb4 支持存储四字节的UTF-8字符集
-            'cursorclass': pymysql.cursors.DictCursor,
-        }
-
-    def data_to_df(self, db_name, tabel_name, start_date, end_date, projection=[]):
-        start_date = pd.to_datetime(start_date).strftime('%Y-%m-%d')
-        end_date = pd.to_datetime(end_date).strftime('%Y-%m-%d')
-        df = pd.DataFrame()
-
-        connection = pymysql.connect(**self.config)  # 连接数据库
-        try:
-            with connection.cursor() as cursor:
-                cursor.execute(f"SHOW DATABASES LIKE '{db_name}'")  # 检查数据库是否存在
-                database_exists = cursor.fetchone()
-                if not database_exists:
-                    print(f"Database <{db_name}>: 数据库不存在")
-        finally:
-            connection.close()  # 这里要断开连接
-            time.sleep(0.2)
-
-        self.config.update({'database': db_name})  # 添加更新 config 字段
-        connection = pymysql.connect(**self.config)  # 重新连接数据库
-        try:
-            with connection.cursor() as cursor:
-                # 1. 查询表是否存在
-                sql = f"SHOW TABLES LIKE '{tabel_name}'"
-                cursor.execute(sql)
-                if not cursor.fetchone():
-                    print(f'{db_name} -> <{tabel_name}>: 表不存在')
-                    return df
-
-                # 查询列
-                for col in projection:
-                    sql = ('SELECT 1 FROM information_schema.columns WHERE table_schema = %s AND table_name = %s AND '
-                           'column_name = %s')
-                    cursor.execute(sql, (db_name, {tabel_name}, col))
-                    if cursor.fetchone() is None:  # 移除不存在的列
-                        projection.remove(col)
-        except Exception as e:
-            print(e)
-            return df
-        finally:
-            connection.close()  # 断开连接
-
-        # before_time = time.time()
-        # 读取数据
-        self.config.update({'database': db_name})
-        connection = pymysql.connect(**self.config)  # 重新连接数据库
-        try:
-            with connection.cursor() as cursor:
-                if not projection:  # 如果未指定，则查询所有列，获取 cols_exist
-                    sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
-                    cursor.execute(sql, (db_name, {tabel_name}))
-                    columns = cursor.fetchall()
-                    cols_exist = [col['COLUMN_NAME'] for col in columns]
-
-                if '日期' in projection or '日期' in cols_exist:  # 指定含日期的 projection 或者未指定 projection 但表中有日期列
-                    sql = f"SELECT * FROM {db_name}.{tabel_name} WHERE {'日期'} BETWEEN '%s' AND '%s'" % (start_date, end_date)
-                elif projection:  # 指定未含日期的 projection
-                    sql = f"SELECT '%s' FROM {db_name}.{tabel_name}" % (', '.join(projection))
-                else:  # 未指定 projection 且表中无日期
-                    sql = f"SELECT * FROM {db_name}.{tabel_name}"
-                cursor.execute(sql)
-                rows = cursor.fetchall()  # 获取查询结果
-                columns = [desc[0] for desc in cursor.description]
-                df = pd.DataFrame(rows, columns=columns)
-        except Exception as e:
-            print(f'{e} {db_name} -> <{tabel_name}>: 表不存在')
-            return df
-        finally:
-            connection.close()
-
-        if len(df) == 0:
-            print(f'database: {db_name}, table: {tabel_name} 查询的数据为空')
-        # else:
-        #     now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
-        #     cost_time = int(time.time() - before_time)
-        #     if cost_time < 1:
-        #         cost_time = round(time.time() - before_time, 2)
-        #     print(f'{now}mysql ({self.host}) 表: {tabel_name} 获取数据长度: {len(df)}, 用时: {cost_time} 秒')
-        return df
-
-
-def year_month_day(start_date, end_date):
-    """
-    使用date_range函数和DataFrame来获取从start_date至end_date之间的所有年月日
-    calendar.monthrange： 获取当月第一个工作日的星期值(0,6) 以及当月天数
-    """
-    # 替换年月日中的日, 以便即使传入当月日期也有返回值
-    try:
-        start_date = f'{pd.to_datetime(start_date).year}-{pd.to_datetime(start_date).month}-01'
-    except Exception as e:
-        print(e)
-        return []
-    # 使用pandas的date_range创建一个日期范围，频率为'MS'代表每月开始
-    date_range = pd.date_range(start=start_date, end=end_date, freq='MS')
-    # 转换格式
-    year_months = date_range.strftime('%Y-%m').drop_duplicates().sort_values()
-
-    results = []
-    for year_month in year_months:
-        year = re.findall(r'(\d{4})', year_month)[0]
-        month = re.findall(r'\d{4}-(\d{2})', year_month)[0]
-        s, d = calendar.monthrange(int(year), int(month))
-        results.append({'起始日期': f'{year_month}-01', '结束日期': f'{year_month}-{d}'})
-
-    return results  # start_date至end_date之间的所有年月日
-
-
-def download_datas(tabel_name, save_path, start_date):
-    username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
-    print(username, password, host, port)
-    m = MysqlUpload(username=username, password=password, host=host, port=port)
-    m.port = port
-    results = year_month_day(start_date=start_date, end_date='today')
-    # print(results)
-    for result in results:
-        start_date = result['起始日期']
-        end_date = result['结束日期']
-        # print(start_date, end_date)
-        df = m.data_to_df(db_name='市场数据2', tabel_name=tabel_name, start_date=start_date, end_date=end_date)
-        if len(df) == 0:
-            continue
-        path = os.path.join(save_path, f'{tabel_name}_{str(start_date)}_{str(end_date)}.csv')
-        df['日期'] = df['日期'].apply(lambda x: re.sub(' .*', '', str(x)))
-        df.to_csv(path, index=False, encoding='utf-8_sig', header=True)
-
-
-if __name__ == '__main__':
-    # username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
-    # print(username, password, host, port)
-
-    username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
-    qd = QueryDatas(username=username, password=password, host=host, port=port)
-    df = qd.data_to_df(db_name='市场数据2', tabel_name='市场排行_店铺', start_date='2024-08-13', end_date='2024-08-31')
-    print(df)