mdbq 0.0.6__py3-none-any.whl → 0.0.8__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -118,11 +118,13 @@ class DataTypes:
         """
         # dtypes = df.dtypes.apply(str).to_dict()  # 将 dataframe 数据类型转为字典形式
         df.replace([np.inf, -np.inf], 0, inplace=True)  # 清理一些非法值
-
-        cols = df.columns.tolist()
         df.replace(to_replace=['\\N', '-', '--', '', 'nan'], value=0, regex=False, inplace=True)  # 替换掉特殊字符
         df.replace(to_replace=[','], value='', regex=True, inplace=True)
+        df.replace(to_replace=['="'], value='', regex=True, inplace=True)  # ="和"不可以放在一起清洗, 因为有: id=86785565
+        df.replace(to_replace=['"'], value='', regex=True, inplace=True)
+        cols = df.columns.tolist()
         for col in cols:
+            # df[col] = df[col].apply(lambda x: re.sub('[="]', '', str(x)) if '="' in str(x) else x)
             # 百分比在某些数据库中不兼容, 转换百分比为小数
             df[col] = df[col].apply(lambda x: float(float((str(x).rstrip("%"))) / 100) if str(x).endswith('%') and '~' not in str(x) else x)
             # 尝试转换合适的数据类型
@@ -976,9 +978,9 @@ def update_dtypte():
 
 def upload():
     """ 上传一个文件夹到数据库 """
-    path = '/Users/xigua/数据中心/原始文件2/生意参谋/客户_客户概况_画像'
-    db_name = '生意参谋数据2'
-    collection_name = '客户_客户概况_画像'
+    path = '/Users/xigua/数据中心/原始文件2/生意经/宝贝指标'
+    db_name = '生意经2'
+    collection_name = '宝贝指标'
 
     username, password, host, port = get_myconf.select_config_values(
         target_service='home_lx',
@@ -1018,10 +1020,7 @@ def upload():
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
                     if len(df) == 0:
                         continue
-                    for col in df.columns.tolist():
-                        df[col] = df[col].apply(lambda x: re.sub('[="]', '', str(x)) if '="' in str(x) else x)
-                    # df.replace(to_replace=['--'], value='', regex=False, inplace=True)
-                    df = dt.convert_df_cols(df=df)
+                    df = dt.convert_df_cols(df=df)  # 清理列名和 df 中的非法字符
                     try:
                         df = df.astype(dtypes)
                     except Exception as e:
@@ -1044,8 +1043,8 @@ def upload():
 
 
 if __name__ == '__main__':
-    username, password, host, port = get_myconf.select_config_values(target_service='aliyun', database='mongodb')
-    print(username, password, host, port)
+    # username, password, host, port = get_myconf.select_config_values(target_service='aliyun', database='mongodb')
+    # print(username, password, host, port)
 
     # main()
-    # upload()
+    upload()

mdbq/aggregation/query_data.py

@@ -20,20 +20,13 @@ class MongoDatasQuery:
     """
     def __init__(self, target_service):
         # target_service 从哪个服务器下载数据
-        self.is_maximize = True
-        if platform.system() == 'Darwin':
-            self.output = os.path.join('/Users', getpass.getuser(), '数据中心/数据库导出')
-        elif platform.system() == 'Windows':
-            self.output = os.path.join('C:\\同步空间\\BaiduSyncdisk\\数据库导出')
-        else:
-            self.output = os.path.join('数据中心/数据库导出')
-
+        self.months = 0  # 下载几个月数据, 0 表示当月, 1 是上月 1 号至今
         # 实例化一个下载类
         username, password, host, port = get_myconf.select_config_values(target_service=target_service, database='mongodb')
         self.download = mongo.DownMongo(username=username, password=password, host=host, port=port, save_path=None)
 
     def tg_wxt(self):
-        self.download.start_date, self.download.end_date = self.months_data(num=1)
+        self.download.start_date, self.download.end_date = self.months_data(num=self.months)
         projection = {
             '日期': 1,
             '场景名字': 1,
@@ -48,48 +41,11 @@ class MongoDatasQuery:
             '直接成交笔数': 1,
             '直接成交金额': 1,
         }
-        df = self.download.data_to_df(db_name='天猫数据2', collection_name='推广数据_宝贝主体报表', projection=projection)
-        df.rename(columns={
-            '场景名字': '营销场景',
-            '主体id': '商品id',
-            '总购物车数': '加购量',
-            '总成交笔数': '成交笔数',
-            '总成交金额': '成交金额'
-        }, inplace=True)
-        df = df.astype({
-            '花费': float,
-            '展现量': int,
-            '点击量': int,
-            '加购量': int,
-            '成交笔数': int,
-            '成交金额': float,
-            '自然流量曝光量': int,
-            '直接成交笔数': int,
-            '直接成交金额': float,
-        }, errors='raise')
-        df.fillna(0, inplace=True)
-        if self.is_maximize:
-            df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
-                **{'加购量': ('加购量', np.max),
-                   '成交笔数': ('成交笔数', np.max),
-                   '成交金额': ('成交金额', np.max),
-                   '自然流量曝光量': ('自然流量曝光量', np.max),
-                   '直接成交笔数': ('直接成交笔数', np.max),
-                   '直接成交金额': ('直接成交金额', np.max)
-                   }
-            )
-        else:
-            df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
-                **{'加购量': ('加购量', np.min),
-                   '成交笔数': ('成交笔数', np.min),
-                   '成交金额': ('成交金额', np.min),
-                   '自然流量曝光量': ('自然流量曝光量', np.min),
-                   '直接成交笔数': ('直接成交笔数', np.max),
-                   '直接成交金额': ('直接成交金额', np.max)
-                   }
-            )
-        df.insert(loc=1, column='推广渠道', value='万相台无界版')  # df中插入新列
-        # print(df)
+        df = self.download.data_to_df(
+            db_name='天猫数据2',
+            collection_name='推广数据_宝贝主体报表',
+            projection=projection,
+        )
         return df
 
     @staticmethod
@@ -109,111 +65,66 @@ class MongoDatasQuery:
         start_date = f'{start_date.year}-{start_date.month}-01'  # 替换为 n 月以前的第一天
         return pd.to_datetime(start_date), pd.to_datetime(end_date)
 
-    def as_csv(self, df, filename, path=None, encoding='utf-8_sig',
-               index=False, header=True, st_ascend=None, ascend=None, freq=None):
-        """
-        path: 子文件夹，可以不传，默认导出目录 self.output
-        st_ascend: 排序参数
-        ascend: 升降序
-        freq: 将创建子文件夹并按月分类存储,  freq='Y'，或 freq='M'
-        """
-        if not path:
-            path = self.output
-        else:
-            path = os.path.join(self.output, path)
-        if not os.path.exists(path):
-            os.makedirs(path)
-        if st_ascend and ascend:
-            try:
-                df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
-            except:
-                print(f'{filename}: sort_values排序参数错误！')
-        if freq:
-            if '日期' not in df.columns.tolist():
-                return print(f'{filename}: 数据缺少日期列，无法按日期分组')
-            groups = df.groupby(pd.Grouper(key='日期', freq=freq))
-            for name1, df in groups:
-                if freq == 'M':
-                    sheet_name = name1.strftime('%Y-%m')
-                elif freq == 'Y':
-                    sheet_name = name1.strftime('%Y年')
-                else:
-                    sheet_name = '_未分类'
-                new_path = os.path.join(path, filename)
-                if not os.path.exists(new_path):
-                    os.makedirs(new_path)
-                new_path = os.path.join(new_path, f'{filename}{sheet_name}.csv')
-                if st_ascend and ascend:  # 这里需要重新排序一次，原因未知
-                    try:
-                        df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
-                    except:
-                        print(f'{filename}: sort_values排序参数错误！')
-
-                df.to_csv(new_path, encoding=encoding, index=index, header=header)
-        else:
-            df.to_csv(os.path.join(path, filename + '.csv'), encoding=encoding, index=index, header=header)
-
-    def as_json(self, df, filename, path=None, orient='records', force_ascii=False, st_ascend=None, ascend=None):
-        if not path:
-            path = self.output
-        else:
-            path = os.path.join(self.output, path)
-        if st_ascend and ascend:
-            try:
-                df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
-            except:
-                print(f'{filename}: sort_values排序参数错误！')
-        df.to_json(os.path.join(path, filename + '.json'),
-                   orient=orient, force_ascii=force_ascii)
-
-    def as_excel(self, df, filename, path=None, index=False, header=True, engine='openpyxl',
-                 freeze_panes=(1, 0), st_ascend=None, ascend=None):
-        if not path:
-            path = self.output
-        else:
-            path = os.path.join(self.output, path)
-        if st_ascend and ascend:
-            try:
-                df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
-            except:
-                print(f'{filename}: sort_values排序参数错误！')
-        df.to_excel(os.path.join(path, filename + '.xlsx'),
-                    index=index, header=header, engine=engine, freeze_panes=freeze_panes)
-
 
 class MysqlDatasQuery:
     """
-    从 数据库 中下载数据
-    self.output: 数据库默认导出目录
-    self.is_maximize: 是否最大转化数据
+    从数据库中下载数据
     """
     def __init__(self, target_service):
         # target_service 从哪个服务器下载数据
-        self.is_maximize = True
-        if platform.system() == 'Darwin':
-            self.output = os.path.join('/Users', getpass.getuser(), '数据中心/数据库导出')
-        elif platform.system() == 'Windows':
-            self.output = os.path.join('C:\\同步空间\\BaiduSyncdisk\\数据库导出')
-        else:
-            self.output = os.path.join('数据中心/数据库导出')
-        self.months = 1  # 下载几个月数据, 0 表示当月, 1 是上月 1 号至今
-
+        self.months = 0  # 下载几个月数据, 0 表示当月, 1 是上月 1 号至今
         # 实例化一个下载类
         username, password, host, port = get_myconf.select_config_values(target_service=target_service, database='mysql')
         self.download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
 
     def tg_wxt(self):
         start_date, end_date = self.months_data(num=self.months)
-        df = self.download.data_to_df(db_name='天猫数据2', tabel_name='推广数据_宝贝主体报表', start_date=start_date, end_date=end_date)
+        projection = {
+            '日期': 1,
+            '场景名字': 1,
+            '主体id': 1,
+            '花费': 1,
+            '展现量': 1,
+            '点击量': 1,
+            '总购物车数': 1,
+            '总成交笔数': 1,
+            '总成交金额': 1,
+            '自然流量曝光量': 1,
+            '直接成交笔数': 1,
+            '直接成交金额': 1,
+        }
+        df = self.download.data_to_df(
+            db_name='天猫数据2',
+            tabel_name='推广数据_宝贝主体报表',
+            start_date=start_date,
+            end_date=end_date,
+            projection=projection,
+        )
+        return df
+
+    def syj(self):
+        start_date, end_date = self.months_data(num=self.months)
+        projection = {
+            '日期': 1,
+            '宝贝id': 1,
+            '商家编码': 1,
+            '行业类目': 1,
+            '销售额': 1,
+            '销售量': 1,
+            '订单数': 1,
+            '退货量': 1,
+            '退款额': 1,
+            '退货量_发货后_': 1,
+        }
+        df = self.download.data_to_df(
+            db_name='生意经2',
+            tabel_name='宝贝指标',
+            start_date=start_date,
+            end_date=end_date,
+            projection=projection,
+        )
         return df
 
-    @staticmethod
-    def days_data(days, end_date=None):
-        """ 读取近 days 天的数据 """
-        if not end_date:
-            end_date = datetime.datetime.now()
-        start_date = end_date - datetime.timedelta(days=days)
-        return pd.to_datetime(start_date), pd.to_datetime(end_date)
 
     @staticmethod
     def months_data(num=0, end_date=None):
@@ -224,14 +135,101 @@ class MysqlDatasQuery:
         start_date = f'{start_date.year}-{start_date.month}-01'  # 替换为 n 月以前的第一天
         return pd.to_datetime(start_date), pd.to_datetime(end_date)
 
+
+class GroupBy:
+    """ 数据聚合和导出 """
+    def __init__(self):
+        # self.output: 数据库默认导出目录
+        if platform.system() == 'Darwin':
+            self.output = os.path.join('/Users', getpass.getuser(), '数据中心/数据库导出')
+        elif platform.system() == 'Windows':
+            self.output = os.path.join('C:\\同步空间\\BaiduSyncdisk\\数据库导出')
+        else:
+            self.output = os.path.join('数据中心/数据库导出')
+
+    def groupby(self, df, tabel_name, is_maximize=True):
+        """
+        self.is_maximize: 是否最大转化数据
+        """
+
+        if '宝贝主体报表' in tabel_name:
+            df.rename(columns={
+                '场景名字': '营销场景',
+                '主体id': '商品id',
+                '总购物车数': '加购量',
+                '总成交笔数': '成交笔数',
+                '总成交金额': '成交金额'
+            }, inplace=True)
+            df = df.astype({
+                '花费': float,
+                '展现量': int,
+                '点击量': int,
+                '加购量': int,
+                '成交笔数': int,
+                '成交金额': float,
+                '自然流量曝光量': int,
+                '直接成交笔数': int,
+                '直接成交金额': float,
+            }, errors='raise')
+            df.fillna(0, inplace=True)
+            if is_maximize:
+                df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
+                    **{'加购量': ('加购量', np.max),
+                       '成交笔数': ('成交笔数', np.max),
+                       '成交金额': ('成交金额', np.max),
+                       '自然流量曝光量': ('自然流量曝光量', np.max),
+                       '直接成交笔数': ('直接成交笔数', np.max),
+                       '直接成交金额': ('直接成交金额', np.max)
+                       }
+                )
+            else:
+                df = df.groupby(['日期', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
+                    **{'加购量': ('加购量', np.min),
+                       '成交笔数': ('成交笔数', np.min),
+                       '成交金额': ('成交金额', np.min),
+                       '自然流量曝光量': ('自然流量曝光量', np.min),
+                       '直接成交笔数': ('直接成交笔数', np.max),
+                       '直接成交金额': ('直接成交金额', np.max)
+                       }
+                )
+            df.insert(loc=1, column='推广渠道', value='万相台无界版')  # df中插入新列
+            return df
+        if '宝贝指标' in tabel_name:
+            df.fillna(0, inplace=True)
+            df = df[(df['销售额'] != 0) | (df['退款额'] != 0)]
+            df = df.groupby(['日期', '宝贝id', '商家编码', '行业类目'], as_index=False).agg(
+                **{'销售额': ('销售额', np.min),
+                   '销售量': ('销售量', np.min),
+                   '订单数': ('订单数', np.min),
+                   '退货量': ('退货量', np.max),
+                   '退款额': ('退款额', np.max),
+                   '退货量_发货后_': ('退货量_发货后_', np.max),
+                   }
+            )
+            df['件均价'] = df.apply(lambda x: x['销售额'] / x['销售量'] if x['销售量'] > 0 else 0, axis=1).round(
+                0)  # 两列运算, 避免除以0
+            df['价格带'] = df['件均价'].apply(
+                lambda x: '2000+' if x >= 2000
+                else '1000+' if x >= 1000
+                else '500+' if x >= 500
+                else '300+' if x >= 300
+                else '300以下'
+            )
+            return df
+        else:
+            print(f'<{tabel_name}>: Groupby 类尚未配置，数据为空')
+            return pd.DataFrame({})
+    
     def as_csv(self, df, filename, path=None, encoding='utf-8_sig',
                index=False, header=True, st_ascend=None, ascend=None, freq=None):
         """
-        path: 子文件夹，可以不传，默认导出目录 self.output
-        st_ascend: 排序参数
-        ascend: 升降序
-        freq: 将创建子文件夹并按月分类存储,  freq='Y'，或 freq='M'
+        path: 默认导出目录 self.output, 这个函数的 path 作为子文件夹，可以不传，
+        st_ascend: 排序参数 ['column1', 'column2']
+        ascend: 升降序 [True, False]
+        freq: 将创建子文件夹并按月分类存储,  freq='Y', 或 freq='M'
         """
+        if len(df) == 0:
+            return
         if not path:
             path = self.output
         else:
@@ -269,10 +267,14 @@ class MysqlDatasQuery:
             df.to_csv(os.path.join(path, filename + '.csv'), encoding=encoding, index=index, header=header)
 
     def as_json(self, df, filename, path=None, orient='records', force_ascii=False, st_ascend=None, ascend=None):
+        if len(df) == 0:
+            return
         if not path:
             path = self.output
         else:
             path = os.path.join(self.output, path)
+        if not os.path.exists(path):
+            os.makedirs(path)
         if st_ascend and ascend:
             try:
                 df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
@@ -283,10 +285,14 @@ class MysqlDatasQuery:
 
     def as_excel(self, df, filename, path=None, index=False, header=True, engine='openpyxl',
                  freeze_panes=(1, 0), st_ascend=None, ascend=None):
+        if len(df) == 0:
+            return
         if not path:
             path = self.output
         else:
             path = os.path.join(self.output, path)
+        if not os.path.exists(path):
+            os.makedirs(path)
         if st_ascend and ascend:
             try:
                 df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
@@ -294,13 +300,21 @@ class MysqlDatasQuery:
                 print(f'{filename}: sort_values排序参数错误！')
         df.to_excel(os.path.join(path, filename + '.xlsx'),
                     index=index, header=header, engine=engine, freeze_panes=freeze_panes)
-
-
+        
+        
 def main():
-    sdq = MysqlDatasQuery(target_service='company')
+    sdq = MysqlDatasQuery(target_service='home_lx')
     sdq.months = 0
-    df = sdq.tg_wxt()
-    print(df)
+
+    # df = sdq.tg_wxt()  # 从数据库中获取数据并转为 df
+    # g = GroupBy()  # 数据聚合
+    # df = g.groupby(df=df, tabel_name='推广数据_宝贝主体报表', is_maximize=True)
+    # g.as_csv(df=df, filename='推广数据_宝贝主体报表')  # 数据导出
+
+    df = sdq.syj()
+    g = GroupBy()
+    df = g.groupby(df=df, tabel_name='宝贝指标', is_maximize=True)
+    g.as_csv(df=df, filename='宝贝指标')
 
 
 if __name__ == '__main__':

mdbq/mysql/s_query.py

@@ -31,135 +31,90 @@ class QueryDatas:
             'cursorclass': pymysql.cursors.DictCursor,
         }
 
-    def data_to_df(self, db_name, tabel_name, start_date, end_date, projection=[]):
+    def data_to_df(self, db_name, tabel_name, start_date, end_date, projection: dict=[]):
+
         start_date = pd.to_datetime(start_date).strftime('%Y-%m-%d')
         end_date = pd.to_datetime(end_date).strftime('%Y-%m-%d')
-        df = pd.DataFrame()
+        df = pd.DataFrame()  # 初始化df
+
+        if self.check_infos(db_name, tabel_name) == False:
+            return df
 
+        self.config.update({'database': db_name})
+        connection = pymysql.connect(**self.config)  # 重新连接数据库
+        try:
+            with connection.cursor() as cursor:
+                # 3. 获取数据表的所有列信息
+                sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
+                cursor.execute(sql, (db_name, {tabel_name}))
+                columns = cursor.fetchall()
+                cols_exist = [col['COLUMN_NAME'] for col in columns]  # 数据表的所有列, 返回 list
+
+                # 4. 构建 SQL 查询语句
+                if projection:  # 获取指定列
+                    columns_in = []
+                    for key, value in projection.items():
+                        if value == 1 and key in cols_exist:
+                            columns_in.append(key)  # 提取值为 1 的键并清理不在数据表的键
+                    columns_in = ', '.join(columns_in)
+                    if '日期' in cols_exist:  # 不论是否指定, 只要数据表有日期，则执行
+                        sql = (f"SELECT {columns_in} FROM {db_name}.{tabel_name} "
+                               f"WHERE {'日期'} BETWEEN '{start_date}' AND '{end_date}'")
+                    else:  # 数据表没有日期列时，返回指定列的所有数据
+                        sql = f"SELECT {columns_in} FROM {db_name}.{tabel_name}"
+                else:  # 没有指定获取列时
+                    if '日期' in cols_exist:  # 但数据表有日期，仍然执行
+                        columns_in = ', '.join(cols_exist)
+                        sql = (f"SELECT {columns_in} FROM {db_name}.{tabel_name} "
+                               f"WHERE {'日期'} BETWEEN '{start_date}' AND '{end_date}'")
+                    else:  # 没有指定获取列，且数据表也没有日期列，则返回全部列的全部数据
+                        sql = f"SELECT * FROM {db_name}.{tabel_name}"
+                cursor.execute(sql)
+                rows = cursor.fetchall()  # 获取查询结果
+                columns = [desc[0] for desc in cursor.description]
+                df = pd.DataFrame(rows, columns=columns)  # 转为 df
+        except Exception as e:
+            print(f'{e}')
+            return df
+        finally:
+            connection.close()
+
+        if len(df) == 0:
+            print(f'database: {db_name}, table: {tabel_name} 查询的数据为空')
+        return df
+
+    def check_infos(self, db_name, tabel_name) -> bool:
+        """ 检查数据库、数据表是否存在 """
         connection = pymysql.connect(**self.config)  # 连接数据库
         try:
             with connection.cursor() as cursor:
+                # 1. 检查数据库是否存在
                 cursor.execute(f"SHOW DATABASES LIKE '{db_name}'")  # 检查数据库是否存在
                 database_exists = cursor.fetchone()
                 if not database_exists:
                     print(f"Database <{db_name}>: 数据库不存在")
+                    return False
         finally:
             connection.close()  # 这里要断开连接
-            time.sleep(0.2)
 
         self.config.update({'database': db_name})  # 添加更新 config 字段
         connection = pymysql.connect(**self.config)  # 重新连接数据库
         try:
             with connection.cursor() as cursor:
-                # 1. 查询表是否存在
+                # 2. 查询表是否存在
                 sql = f"SHOW TABLES LIKE '{tabel_name}'"
                 cursor.execute(sql)
                 if not cursor.fetchone():
                     print(f'{db_name} -> <{tabel_name}>: 表不存在')
-                    return df
-
-                # 查询列
-                for col in projection:
-                    sql = ('SELECT 1 FROM information_schema.columns WHERE table_schema = %s AND table_name = %s AND '
-                           'column_name = %s')
-                    cursor.execute(sql, (db_name, {tabel_name}, col))
-                    if cursor.fetchone() is None:  # 移除不存在的列
-                        projection.remove(col)
+                    return False
+                return True
         except Exception as e:
             print(e)
-            return df
+            return False
         finally:
             connection.close()  # 断开连接
 
-        # before_time = time.time()
-        # 读取数据
-        self.config.update({'database': db_name})
-        connection = pymysql.connect(**self.config)  # 重新连接数据库
-        try:
-            with connection.cursor() as cursor:
-                if not projection:  # 如果未指定，则查询所有列，获取 cols_exist
-                    sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
-                    cursor.execute(sql, (db_name, {tabel_name}))
-                    columns = cursor.fetchall()
-                    cols_exist = [col['COLUMN_NAME'] for col in columns]
-
-                if '日期' in projection or '日期' in cols_exist:  # 指定含日期的 projection 或者未指定 projection 但表中有日期列
-                    sql = f"SELECT * FROM {db_name}.{tabel_name} WHERE {'日期'} BETWEEN '%s' AND '%s'" % (start_date, end_date)
-                elif projection:  # 指定未含日期的 projection
-                    sql = f"SELECT '%s' FROM {db_name}.{tabel_name}" % (', '.join(projection))
-                else:  # 未指定 projection 且表中无日期
-                    sql = f"SELECT * FROM {db_name}.{tabel_name}"
-                cursor.execute(sql)
-                rows = cursor.fetchall()  # 获取查询结果
-                columns = [desc[0] for desc in cursor.description]
-                df = pd.DataFrame(rows, columns=columns)
-        except Exception as e:
-            print(f'{e} {db_name} -> <{tabel_name}>: 表不存在')
-            return df
-        finally:
-            connection.close()
-
-        if len(df) == 0:
-            print(f'database: {db_name}, table: {tabel_name} 查询的数据为空')
-        # else:
-        #     now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S ")
-        #     cost_time = int(time.time() - before_time)
-        #     if cost_time < 1:
-        #         cost_time = round(time.time() - before_time, 2)
-        #     print(f'{now}mysql ({self.host}) 表: {tabel_name} 获取数据长度: {len(df)}, 用时: {cost_time} 秒')
-        return df
-
-
-def year_month_day(start_date, end_date):
-    """
-    使用date_range函数和DataFrame来获取从start_date至end_date之间的所有年月日
-    calendar.monthrange： 获取当月第一个工作日的星期值(0,6) 以及当月天数
-    """
-    # 替换年月日中的日, 以便即使传入当月日期也有返回值
-    try:
-        start_date = f'{pd.to_datetime(start_date).year}-{pd.to_datetime(start_date).month}-01'
-    except Exception as e:
-        print(e)
-        return []
-    # 使用pandas的date_range创建一个日期范围，频率为'MS'代表每月开始
-    date_range = pd.date_range(start=start_date, end=end_date, freq='MS')
-    # 转换格式
-    year_months = date_range.strftime('%Y-%m').drop_duplicates().sort_values()
-
-    results = []
-    for year_month in year_months:
-        year = re.findall(r'(\d{4})', year_month)[0]
-        month = re.findall(r'\d{4}-(\d{2})', year_month)[0]
-        s, d = calendar.monthrange(int(year), int(month))
-        results.append({'起始日期': f'{year_month}-01', '结束日期': f'{year_month}-{d}'})
-
-    return results  # start_date至end_date之间的所有年月日
-
-
-def download_datas(tabel_name, save_path, start_date):
-    username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
-    print(username, password, host, port)
-    m = MysqlUpload(username=username, password=password, host=host, port=port)
-    m.port = port
-    results = year_month_day(start_date=start_date, end_date='today')
-    # print(results)
-    for result in results:
-        start_date = result['起始日期']
-        end_date = result['结束日期']
-        # print(start_date, end_date)
-        df = m.data_to_df(db_name='市场数据2', tabel_name=tabel_name, start_date=start_date, end_date=end_date)
-        if len(df) == 0:
-            continue
-        path = os.path.join(save_path, f'{tabel_name}_{str(start_date)}_{str(end_date)}.csv')
-        df['日期'] = df['日期'].apply(lambda x: re.sub(' .*', '', str(x)))
-        df.to_csv(path, index=False, encoding='utf-8_sig', header=True)
-
 
 if __name__ == '__main__':
-    # username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
-    # print(username, password, host, port)
-
     username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
-    qd = QueryDatas(username=username, password=password, host=host, port=port)
-    df = qd.data_to_df(db_name='市场数据2', tabel_name='市场排行_店铺', start_date='2024-08-13', end_date='2024-08-31')
-    print(df)
+    print(username, password, host, port)

mdbq/mysql/year_month_day.py

@@ -0,0 +1,38 @@
+# -*- coding:utf-8 -*-
+import warnings
+import pandas as pd
+import calendar
+
+warnings.filterwarnings('ignore')
+
+
+def year_month_day(start_date, end_date):
+    """
+    使用date_range函数和DataFrame来获取从start_date至end_date之间的所有年月日
+    calendar.monthrange： 获取当月第一个工作日的星期值(0,6) 以及当月天数
+    返回值: [{'起始日期': '2025-05-01', '结束日期': '2025-05-31'}, {'起始日期': '2025-06-01', '结束日期': '2025-06-30'}]
+    """
+    # 替换年月日中的日, 以便即使传入当月日期也有返回值
+    try:
+        start_date = f'{pd.to_datetime(start_date).year}-{pd.to_datetime(start_date).month}-01'
+    except Exception as e:
+        print(e)
+        return []
+    # 使用pandas的date_range创建一个日期范围，频率为'MS'代表每月开始
+    date_range = pd.date_range(start=start_date, end=end_date, freq='MS')
+    # 转换格式
+    year_months = date_range.strftime('%Y-%m').drop_duplicates().sort_values()
+
+    results = []
+    for year_month in year_months:
+        year = re.findall(r'(\d{4})', year_month)[0]
+        month = re.findall(r'\d{4}-(\d{2})', year_month)[0]
+        s, d = calendar.monthrange(int(year), int(month))
+        results.append({'起始日期': f'{year_month}-01', '结束日期': f'{year_month}-{d}'})
+
+    return results  # start_date至end_date之间的所有年月日
+
+
+if __name__ == '__main__':
+    results = year_month_day(start_date='2025-05-01', end_date='2025-08-01')
+    print(results)

{mdbq-0.0.6.dist-info → mdbq-0.0.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 0.0.6
+Version: 0.0.8
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-0.0.6.dist-info → mdbq-0.0.8.dist-info}/RECORD

@@ -1,8 +1,8 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=RHQa2rs4fimRvJzluujErg6I8fn7s9q1-kwC2bPZohE,59439
-mdbq/aggregation/query_data.py,sha256=avbMc36kCuyTjLOXMzIEIKSc0x227c1t-Ydf0vdkViM,13756
+mdbq/aggregation/aggregation.py,sha256=7Id0cZ5QR7rPR9srbNRv78lGBwQ7pQre65qYIhu89Ms,59498
+mdbq/aggregation/query_data.py,sha256=9E4dbeQAq7r1srlszP27X3jFiKtMmFc6lP965GHjQms,13368
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
@@ -18,7 +18,8 @@ mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=hF93-kP2lxK4WY1KCdBBszLQ_I7W0mQQxZ7t4qU2w3A,32930
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/mysql.py,sha256=H9onFYKSYRjdXghK_29Aj7vgvUgDHexJjIECrdxLbE0,29925
-mdbq/mysql/s_query.py,sha256=P0QNwJL3ytyN75c8Qny1xfxrOUI4ks-FuRghNsyMWic,7409
+mdbq/mysql/s_query.py,sha256=bRnW8Cpy4fSsbMhzGCvjiK2kin9uamVumJC3nLAyjMg,5213
+mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/porxy.py,sha256=UHfgEyXugogvXgsG68a7QouUCKaohTKKkI4RN-kYSdQ,4961
 mdbq/other/pov_city.py,sha256=AEOmCOzOwyjHi9LLZWPKi6DUuSC-_M163664I52u9qw,21050
@@ -27,7 +28,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=wulHs4rivf4Mi0Pii2QR5Nk9-TBcvSwnCB_WH9QULKE,5939
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-0.0.6.dist-info/METADATA,sha256=G_RkhGDJMmXpNdrECY4LBT0f6HF9sHpfU-LYxNUzWPk,245
-mdbq-0.0.6.dist-info/WHEEL,sha256=y4mX-SOX4fYIkonsAGA5N0Oy-8_gI4FXw5HNI1xqvWg,91
-mdbq-0.0.6.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-0.0.6.dist-info/RECORD,,
+mdbq-0.0.8.dist-info/METADATA,sha256=dVu6kA6RtVWnfsNTWXnOv_d1TmKhM4Gw6PrypX9MF0E,245
+mdbq-0.0.8.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-0.0.8.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-0.0.8.dist-info/RECORD,,

{mdbq-0.0.6.dist-info → mdbq-0.0.8.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.2.0)
+Generator: setuptools (70.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any