mdbq 0.1.5__py3-none-any.whl → 0.1.7__py3-none-any.whl

mdbq/aggregation/query_data.py

@@ -1,5 +1,6 @@
 # -*- coding: UTF-8 –*-
 from mdbq.mongo import mongo
+from mdbq.mysql import mysql
 from mdbq.mysql import s_query
 from mdbq.config import get_myconf
 import datetime
@@ -117,7 +118,7 @@ class MysqlDatasQuery:
             '订单数': 1,
             '退货量': 1,
             '退款额': 1,
-            '退货量_发货后_': 1,
+            '退货量_发货后': 1,
         }
         df = self.download.data_to_df(
             db_name='生意经2',
@@ -128,6 +129,26 @@ class MysqlDatasQuery:
         )
         return df
 
+    def idbm(self):
+        """ 用生意经日数据制作商品 id 和编码对照表 """
+        data_values = self.download.columns_to_list(
+            db_name='生意经2',
+            tabel_name='宝贝指标',
+            columns_name=['宝贝id', '商家编码', '行业类目'],
+        )
+        df = pd.DataFrame(data=data_values)
+        return df
+
+    def sp_picture(self):
+        """ 用生意经日数据制作商品 id 和编码对照表 """
+        data_values = self.download.columns_to_list(
+            db_name='属性设置2',
+            tabel_name='商品素材导出',
+            columns_name=['日期', '商品id', '商品白底图', '方版场景图'],
+        )
+        df = pd.DataFrame(data=data_values)
+        return df
+
     def dplyd(self):
         start_date, end_date = self.months_data(num=self.months)
         projection = {
@@ -150,6 +171,16 @@ class MysqlDatasQuery:
         )
         return df
 
+    def sp_cost(self):
+        """ 电商定价 """
+        data_values = self.download.columns_to_list(
+            db_name='属性设置2',
+            tabel_name='电商定价',
+            columns_name=['日期', '款号', '年份季节', '吊牌价', '商家平台', '成本价', '天猫页面价', '天猫中促价'],
+        )
+        df = pd.DataFrame(data=data_values)
+        return df
+
     @staticmethod
     def months_data(num=0, end_date=None):
         """ 读取近 num 个月的数据, 0 表示读取当月的数据 """
@@ -229,7 +260,7 @@ class GroupBy:
                    '订单数': ('订单数', np.min),
                    '退货量': ('退货量', np.max),
                    '退款额': ('退款额', np.max),
-                   '退货量_发货后_': ('退货量_发货后_', np.max),
+                   '退货量_发货后': ('退货量_发货后', np.max),
                    }
             )
             df['件均价'] = df.apply(lambda x: x['销售额'] / x['销售量'] if x['销售量'] > 0 else 0, axis=1).round(
@@ -244,6 +275,41 @@ class GroupBy:
             return df
         elif '店铺来源_日数据' in tabel_name:
             return df
+        elif '商品id编码表' in tabel_name:
+            df.drop_duplicates(subset='宝贝id', keep='last', inplace=True, ignore_index=True)
+            # df['行业类目'] = df['行业类目'].apply(lambda x: re.sub(' ', '', x))
+            try:
+                df[['一级类目', '二级类目', '三级类目']] = df['行业类目'].str.split(' -> ', expand=True).loc[:, 0:2]
+            except:
+                try:
+                    df[['一级类目', '二级类目']] = df['行业类目'].str.split(' -> ', expand=True).loc[:, 0:1]
+                except:
+                    df['一级类目'] = df['行业类目']
+            df.drop('行业类目', axis=1, inplace=True)
+            df.sort_values('宝贝id', ascending=False, inplace=True)
+            df = df[(df['宝贝id'] != '973') & (df['宝贝id'] != '973')]
+            return df
+        elif '商品id图片对照表' in tabel_name:
+            df['商品id'] = df['商品id'].astype('int64')
+            df['日期'] = df['日期'].astype('datetime64[ns]')
+            df = df[(df['商品白底图'] != '0') | (df['方版场景图'] != '0')]
+            # 白底图优先
+            df['商品图片'] = df[['商品白底图', '方版场景图']].apply(
+                lambda x: x['商品白底图'] if x['商品白底图'] !='0' else x['方版场景图'], axis=1)
+            # # 方版场景图优先
+            # df['商品图片'] = df[['商品白底图', '方版场景图']].apply(
+            #     lambda x: x['方版场景图'] if x['方版场景图'] != '0' else x['商品白底图'], axis=1)
+            df.sort_values(by=['商品id', '日期'], ascending=[False, True], ignore_index=True, inplace=True)
+            df.drop_duplicates(subset=['商品id'], keep='last', inplace=True, ignore_index=True)
+            df = df[['商品id', '商品图片', '日期']]
+            df['商品图片'] = df['商品图片'].apply(lambda x: x if 'http' in x else None)  # 检查是否是 http 链接
+            df.dropna(how='all', subset=['商品图片'], axis=0, inplace=True)  # 删除指定列含有空值的行
+            df.sort_values(by='商品id', ascending=False, ignore_index=True, inplace=True)  # ascending=False 降序排列
+            return df
+        elif '商品成本' in tabel_name:
+            df.sort_values(by=['款号', '日期'], ascending=[False, True], ignore_index=True, inplace=True)
+            df.drop_duplicates(subset=['款号'], keep='last', inplace=True, ignore_index=True)
+            return df
         else:
             print(f'<{tabel_name}>: Groupby 类尚未配置，数据为空')
             return pd.DataFrame({})
@@ -330,25 +396,57 @@ class GroupBy:
                     index=index, header=header, engine=engine, freeze_panes=freeze_panes)
         
         
-def data_output():
-    sdq = MysqlDatasQuery(target_service='home_lx')
-    sdq.months = 0
-
-    # df = sdq.tg_wxt()  # 从数据库中获取数据并转为 df
-    # g = GroupBy()  # 数据聚合
-    # df = g.groupby(df=df, tabel_name='推广数据_宝贝主体报表', is_maximize=True)
-    # g.as_csv(df=df, filename='推广数据_宝贝主体报表')  # 数据导出
-
-    # df = sdq.syj()
-    # g = GroupBy()
-    # df = g.groupby(df=df, tabel_name='宝贝指标', is_maximize=True)
-    # g.as_csv(df=df, filename='宝贝指标')
+def data_aggregation():
+    """
+    1. 从数据库中读取数据
+    2. 数据聚合清洗
+    3. 统一回传数据库: <聚合数据>  （不再导出为文件）
+    """
+    sdq = MysqlDatasQuery(target_service='company')  # 实例化数据处理类
+    sdq.months = 0  # 设置数据周期
+    g = GroupBy()  # 实例化数据聚合类
+    # 实例化数据库连接
+    username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
+    m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
 
-    df = sdq.dplyd()
-    g = GroupBy()
-    df = g.groupby(df=df, tabel_name='店铺来源_日数据', is_maximize=True)
-    g.as_csv(df=df, filename='店铺来源_日数据')
+    data_dict = [
+        {
+            '数据库名': '聚合数据',
+            '集合名': '推广数据_宝贝主体报表',
+            '数据主体': sdq.tg_wxt(),
+        },
+        {
+            '数据库名': '聚合数据',
+            '集合名': '天猫生意经_宝贝指标',
+            '数据主体': sdq.syj(),
+        },
+        {
+            '数据库名': '聚合数据',
+            '集合名': '天猫_店铺来源_日数据',
+            '数据主体': sdq.dplyd(),
+        },
+        {
+            '数据库名': '聚合数据',
+            '集合名': '商品id编码表',
+            '数据主体': sdq.idbm(),
+        },
+        {
+            '数据库名': '聚合数据',
+            '集合名': '商品id图片对照表',
+            '数据主体': sdq.sp_picture(),
+        },
+        {
+            '数据库名': '聚合数据',
+            '集合名': '商品成本',
+            '数据主体': sdq.sp_cost(),
+        },
+    ]
+    for items in data_dict:
+        db_name, tabel_name, df = items['数据库名'], items['集合名'], items['数据主体']
+        df = g.groupby(df=df, tabel_name=tabel_name, is_maximize=True)  # 2. 聚合数据
+        # g.as_csv(df=df, filename=tabel_name + '.csv')
+        m.df_to_mysql(df=df, db_name=db_name, tabel_name=tabel_name)  # 3. 回传数据库
 
 
 if __name__ == '__main__':
-    main()
+    data_aggregation()

mdbq/company/copysh.py

@@ -314,4 +314,8 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    # main()
+    dp = aggregation.DatabaseUpdate(path='')
+    # dp.new_unzip(is_move=True)
+    # dp.cleaning(is_move=True)  # 公司台式机需要移除
+    dp.upload_df(service_databases=[{'company': 'mysql'}])

mdbq/mysql/s_query.py

@@ -86,6 +86,33 @@ class QueryDatas:
             print(f'database: {db_name}, table: {tabel_name} 查询的数据为空')
         return df
 
+    def columns_to_list(self, db_name, tabel_name,  columns_name) -> list:
+        """
+        获取数据表的指定列, 返回列表
+        [{'视频bv号': 'BV1Dm4y1S7BU', '下载进度': 1}, {'视频bv号': 'BV1ov411c7US', '下载进度': 1}]
+        """
+        if self.check_infos(db_name, tabel_name) == False:  # 检查传入的数据库和数据表是否存在
+            return []
+
+        self.config.update({'database': db_name})
+        connection = pymysql.connect(**self.config)  # 重新连接数据库
+        with connection.cursor() as cursor:
+            # 3. 获取数据表的所有列信息
+            sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
+            cursor.execute(sql, (db_name, {tabel_name}))
+            columns = cursor.fetchall()
+            cols_exist = [col['COLUMN_NAME'] for col in columns]  # 数据表的所有列, 返回 list
+            columns_name = [item for item in columns_name if item in cols_exist]
+            if len(columns_name) == 0:
+                return []
+            columns_in = ', '.join(columns_name)
+            sql = (f"SELECT {columns_in} FROM {db_name}.{tabel_name} ")
+            cursor.execute(sql)
+            column_values = cursor.fetchall()  # 返回指定列，结果是[dict, dict, dict, ...]
+            # column_values = [item[column_name] for item in column_values]  # 提取字典的值, 组成列表
+        connection.close()
+        return column_values
+
     def check_infos(self, db_name, tabel_name) -> bool:
         """ 检查数据库、数据表是否存在 """
         connection = pymysql.connect(**self.config)  # 连接数据库
@@ -119,5 +146,9 @@ class QueryDatas:
 
 
 if __name__ == '__main__':
-    username, password, host, port = get_myconf.select_config_values(target_service='company', database='mysql')
+    username, password, host, port = get_myconf.select_config_values(target_service='home_lx', database='mysql')
     print(username, password, host, port)
+
+    q = QueryDatas(username, password, host, port)
+    res = q.columns_to_list(db_name='视频数据', tabel_name='bilibili视频', columns_name=['视频bv号', '下载进度'])
+    print(res)

{mdbq-0.1.5.dist-info → mdbq-0.1.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 0.1.5
+Version: 0.1.7
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-0.1.5.dist-info → mdbq-0.1.7.dist-info}/RECORD

@@ -2,13 +2,13 @@ mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
 mdbq/aggregation/aggregation.py,sha256=n2MoKwN1ltJD3juu59zkhc7PGUMDTkn0zCcZGs8RnXI,56775
-mdbq/aggregation/query_data.py,sha256=RaZ0NJwTeNhSgpyj9eUjEVc6Kp83etr6luyzSf9U5WE,14465
+mdbq/aggregation/query_data.py,sha256=w_C29vHZJpmj_TMjpNgb_a-7eZ9pO0PixPkWFyLr50I,19476
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
 mdbq/clean/data_clean.py,sha256=33OmeQFl9AW21P5EOay52W_S8DF96H5oHwCg4fSuBxA,85359
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
-mdbq/company/copysh.py,sha256=m7N3DKdbBBvzwlV438cky53L8Akp4jXQUZCJRIUxk0E,15613
+mdbq/company/copysh.py,sha256=4cfFnxuYhfVAdROX59I-pzood3T3ebPPHDYxyFs8NHk,15819
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/get_myconf.py,sha256=9v3xebfcS1tptxpvk3_tGxfXjAehGVCveYe4iRUzLQQ,6372
 mdbq/config/update_conf.py,sha256=YjGjjRchu5BcrmLJkoLjHEF2TbGOmsgCWX4LroXOYWQ,3455
@@ -20,7 +20,7 @@ mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=q0B4wXDSTtXg_vMN7MPh6zdxl6tT68tM74LmdVNQQek,31892
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
 mdbq/mysql/mysql.py,sha256=w_FDE4ulOnHdfIjjXB1Ff_cnXyx2PuBAQWGjTYyVm3U,30629
-mdbq/mysql/s_query.py,sha256=pzxRGBRP3Ku4oVLqfpMd1VWWct3YlxxnVavtKa9kgSM,5302
+mdbq/mysql/s_query.py,sha256=mNrdyMeiQ5wQVD_9IDTrk6jrMtszQftYQg3AwNWhROc,6997
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/porxy.py,sha256=UHfgEyXugogvXgsG68a7QouUCKaohTKKkI4RN-kYSdQ,4961
@@ -30,7 +30,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=wulHs4rivf4Mi0Pii2QR5Nk9-TBcvSwnCB_WH9QULKE,5939
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-0.1.5.dist-info/METADATA,sha256=X-iiiuZTrT9STDW8bThvKswQ_cLjqv_rqdIVZ4Of4oI,245
-mdbq-0.1.5.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-0.1.5.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-0.1.5.dist-info/RECORD,,
+mdbq-0.1.7.dist-info/METADATA,sha256=6IJUwkrTSwOhf6lTMUtOXI5JpiDGHVTVo3C06u7yUrg,245
+mdbq-0.1.7.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-0.1.7.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-0.1.7.dist-info/RECORD,,