mdbq 2.1.1__py3-none-any.whl → 2.1.2__py3-none-any.whl

mdbq/aggregation/query_data.py

@@ -421,6 +421,27 @@ class MysqlDatasQuery:
             projection=projection,
         )
         return df
+    def spu_sales(self):
+        start_date, end_date = self.months_data(num=self.months)
+        projection = {
+            '日期': 1,
+            '商品id': 1,
+            '货号': 1,
+            '成交单量': 1,
+            '成交金额': 1,
+            '访客数': 1,
+            '成交客户数': 1,
+            '加购商品件数': 1,
+            '加购人数': 1,
+        }
+        df = self.download.data_to_df(
+            db_name='京东数据2',
+            table_name='spu_商品明细',
+            start_date=start_date,
+            end_date=end_date,
+            projection=projection,
+        )
+        return df
 
     @staticmethod
     def months_data(num=0, end_date=None):
@@ -1190,6 +1211,21 @@ class GroupBy:
                 }
             )
             return df
+        elif '京东_spu_商品明细' in table_name:
+            df = df[df['商品id'] != '合计']
+            df = df.groupby(['日期', '商品id', '货号', '访客数', '成交客户数', '加购商品件数', '加购人数'],
+                            as_index=False).agg(
+                **{
+                    '成交单量': ('成交单量', np.max),
+                    '成交金额': ('成交金额', np.max),
+                   }
+            )
+            self.data_jdtg.update(
+                {
+                    table_name: df,
+                }
+            )
+            return df
         elif '京东_关键词报表' in table_name:
             df_lin = df[['计划id', '推广计划']]
             df_lin.drop_duplicates(subset=['计划id'], keep='last', inplace=True, ignore_index=True)
@@ -1580,6 +1616,12 @@ def data_aggregation(service_databases=[{}], months=1):
                     '唯一主键': ['日期', '商品id', '成交单量'],
                     '数据主体': sdq.sku_sales(),
                 },
+                {
+                    '数据库名': '聚合数据',
+                    '集合名': '京东_spu_商品明细',
+                    '唯一主键': ['日期', '商品id', '成交单量'],
+                    '数据主体': sdq.spu_sales(),
+                },
                 {
                     '数据库名': '聚合数据',
                     '集合名': '天猫_人群报表',

mdbq/mysql/mysql.py

@@ -68,8 +68,8 @@ class MysqlUpload:
 
         return wrapper
 
-    @try_except
-    def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], service_database={'home_lx': 'mysql'}, move_insert=False, df_sql=False, drop_duplicates=False, filename=None, count=None, json_path=None):
+    # @try_except
+    def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], service_database={'home_lx': 'mysql'}, move_insert=False, df_sql=False, drop_duplicates=False, filename=None, count=None, json_path=None, reset_id=False):
         """
         将 df 写入数据库
         db_name: 数据库名称
@@ -167,6 +167,17 @@ class MysqlUpload:
                     index=False,
                     chunksize=1000
                 )
+                try:
+                    cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
+                    result = cursor.fetchone()
+                    if result:
+                        cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
+                    cursor.execute(
+                        f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                    cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+                except Exception as e:
+                    print(f'{e}')
+                    connection.rollback()
                 connection.close()
                 return
 
@@ -205,6 +216,19 @@ class MysqlUpload:
                     index=False,
                     chunksize=1000
                 )
+                # 6. 重置自增列
+                if reset_id:
+                    try:
+                        cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
+                        result = cursor.fetchone()
+                        if result:
+                            cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
+                        cursor.execute(
+                            f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                        cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+                    except Exception as e:
+                        print(f'{e}')
+                        connection.rollback()
                 connection.close()
                 return
 
@@ -293,6 +317,19 @@ class MysqlUpload:
                     # print(values)
                     print(f'mysql -> df_to_mysql 报错: {e}, {self.filename}')
                     # breakpoint()
+
+            # 6. 重置自增列
+            try:
+                cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
+                result = cursor.fetchone()
+                if result:
+                    cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
+                cursor.execute(
+                    f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+            except Exception as e:
+                print(f'{e}')
+                connection.rollback()
         connection.commit()  # 提交事务
         connection.close()
 

mdbq/other/sku_picture.py

@@ -11,6 +11,7 @@ import time
 import warnings
 import pandas as pd
 from lxml import etree
+from rich.pretty import pretty_repr
 from selenium import webdriver
 from selenium.webdriver.support.wait import WebDriverWait
 from selenium.webdriver.common.by import By
@@ -582,6 +583,33 @@ class DownloadPicture():
             i += 1
             time.sleep(0.5)
 
+    def download_from_df(self, col_name='商品图片'):
+        if not os.path.exists(self.save_path):
+            os.mkdir(self.save_path)
+        dict_data = self.df.to_dict('records')
+        num = len(dict_data)
+        i = 1
+        for data in dict_data:
+            url = data[col_name]
+            self.filename = f'{data['店铺名称']}_{data['商品id']}_{data['商家编码']}.jpg'
+            if os.path.isfile(os.path.join(self.save_path, self.filename)):
+                self.finish_download.append(data['商品id'])
+                i += 1
+                continue
+            if 'https' not in url:
+                i += 1
+                continue
+
+            print(f'正在下载: {i}/{num},   {data['商品id']}')
+            self.headers.update({'User-Agent': ua_sj.get_ua()})
+            res = requests.get(url, headers=self.headers)  # 下载图片到内存
+            # 保存图片到本地文件夹
+            with open(os.path.join(self.save_path, self.filename), 'wb') as f:
+                f.write(res.content)
+            self.finish_download.append(data['商品id'])
+            i += 1
+            time.sleep(0.5)
+
     @staticmethod
     def months_data(num=0, end_date=None):
         """ 读取近 num 个月的数据, 0 表示读取当月的数据 """
@@ -783,7 +811,7 @@ def main3():
     p.insert_data()
 
 
-def get_sp_id(service_name='company', database='mysql', db_name='属性设置2', table_name='商品素材下载记录', col_name='sku图片链接'):
+def download_sku(service_name='company', database='mysql', db_name='属性设置2', table_name='商品素材下载记录', col_name='sku图片链接'):
     """ 从数据库中获取商品id信息 """
     # 实例化一个下载类
     username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
@@ -885,9 +913,125 @@ def get_sp_id(service_name='company', database='mysql', db_name='属性设置2',
     )  # 3. 回传数据库
 
 
+def download_spu(service_name='company', database='mysql', db_name='属性设置2', table_name='商品spu素材下载记录', col_name='商品图片'):
+    """
+
+    """
+
+    # 1. 从商品素材导出中获取数据
+    username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
+    download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
+    projection = {
+        '店铺名称': 1,
+        '商品id': 1,
+        '商品标题': 1,
+        '商品状态': 1,
+        '商品白底图': 1,
+        '方版场景图': 1,
+        '日期':1,
+    }
+    df = download.data_to_df(
+        db_name='属性设置2',
+        table_name='商品素材导出',
+        start_date='2019-01-01',
+        end_date='2099-12-31',
+        projection=projection,
+    )
+    df['商品id'] = df['商品id'].astype('int64')
+    df['日期'] = df['日期'].astype('datetime64[ns]')
+    df = df[(df['商品白底图'] != '0') | (df['方版场景图'] != '0')]
+    # 白底图优先
+    df['商品图片'] = df[['商品白底图', '方版场景图']].apply(
+        lambda x: x['商品白底图'] if x['商品白底图'] != '0' else x['方版场景图'], axis=1)
+    # # 方版场景图优先
+    # df['商品图片'] = df[['商品白底图', '方版场景图']].apply(
+    #     lambda x: x['方版场景图'] if x['方版场景图'] != '0' else x['商品白底图'], axis=1)
+    df.sort_values(by=['商品id', '日期'], ascending=[False, True], ignore_index=True, inplace=True)
+    df.drop_duplicates(subset=['商品id'], keep='last', inplace=True, ignore_index=True)
+    # df = df[['商品id', '商品图片', '日期']]
+    df['商品图片'] = df['商品图片'].apply(lambda x: x if 'http' in x else None)  # 检查是否是 http 链接
+    df.dropna(how='all', subset=['商品图片'], axis=0, inplace=True)  # 删除指定列含有空值的行
+    df['商品链接'] = df['商品id'].apply(
+        lambda x: f'https://detail.tmall.com/item.htm?id={str(x)}' if x and '.com' not in str(x) else x)
+    df.sort_values(by='商品id', ascending=False, ignore_index=True, inplace=True)  # ascending=False 降序排列
+    df['商品id'] = df['商品id'].astype(str)
+    # df = df.head(2)
+
+    # 2. 从商品id编码表 中获取数据
+    projection = {
+        '宝贝id': 1,
+        '商家编码': 1,
+    }
+    df_spbm = download.data_to_df(
+        db_name='聚合数据',
+        table_name='商品id编码表',
+        start_date='2019-01-01',
+        end_date='2099-12-31',
+        projection=projection,
+    )
+    df_spbm.drop_duplicates(subset=['宝贝id'], keep='last', inplace=True, ignore_index=True)
+    # 合并两个表
+    df = pd.merge(df, df_spbm, left_on=['商品id'], right_on=['宝贝id'], how='left')
+    df.pop('宝贝id')
+    df['获取与下载'] = '已获取'
+    df['时间'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+    # df.to_csv(os.path.join(D_PATH, f'{col_name}.csv'), index=False, header=True, encoding='utf-8_sig')
+    if '方版场景图' in df.columns.tolist():
+        df['方版场景图'] = df['方版场景图'].astype(str)
+
+    # 3. 更新数据库
+    username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
+    m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
+    m.df_to_mysql(
+        df=df,
+        db_name=db_name,
+        table_name=table_name,
+        move_insert=True,  # 先删除，再插入
+        df_sql=False,
+        drop_duplicates=False,
+        icm_update=[],
+        service_database={service_name: database},
+    )  # 3. 回传数据库
+
+    # 4. 从数据库中提取未下载的数据
+    projection = {}
+    df_before = download.data_to_df(
+        db_name=db_name,
+        table_name=table_name,
+        start_date='2019-01-01',
+        end_date='2099-12-31',
+        projection=projection,
+    )
+    df = df_before[df_before['获取与下载'] != '已下载']
+
+    if len(df) > 0:
+        #  5. 实例化一个下载器类，并下载数据
+        d = DownloadPicture(service_name=service_name)
+        d.save_path = os.path.join(D_PATH, '商品id_商家编码_图片')  # 下载图片到本地时的存储位置
+        d.filename = f'{db_name}_{table_name}.xlsx'
+        d.df = df
+        d.download_from_df(col_name=col_name)
+        df['获取与下载'] = df.apply(lambda x: '已下载' if x['商品id'] in d.finish_download else x['获取与下载'], axis=1)
+
+        # 6. 回传数据库
+        username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
+        m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
+        m.df_to_mysql(
+            df=df,
+            db_name=db_name,
+            table_name=table_name,
+            move_insert=True,  # 先删除，再插入
+            df_sql=False,
+            drop_duplicates=False,
+            icm_update=[],
+            service_database={service_name: database},
+            reset_id=True,
+        )  # 3. 回传数据库
+
 
 if __name__ == '__main__':
     # main(service_name='home_lx', database='mysql')
     # main2(service_name='home_lx', database='mysql')
     # main3()
-    get_sp_id(service_name='company', database='mysql', db_name='属性设置2', table_name='商品素材下载记录')
+    # download_sku(service_name='company', database='mysql', db_name='属性设置2', table_name='商品素材下载记录')
+    download_spu(service_name='company', database='mysql', db_name='属性设置2', table_name='商品spu素材下载记录', col_name='商品图片')

{mdbq-2.1.1.dist-info → mdbq-2.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.1.1
+Version: 2.1.2
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.1.1.dist-info → mdbq-2.1.2.dist-info}/RECORD

@@ -5,7 +5,7 @@ mdbq/aggregation/aggregation.py,sha256=us2FJjLCYlyttATHc0xYRo1ZvcC4E7lB_JI034jI6
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
 mdbq/aggregation/mysql_types.py,sha256=DQYROALDiwjJzjhaJfIIdnsrNs11i5BORlj_v6bp67Y,11062
 mdbq/aggregation/optimize_data.py,sha256=Wis40oL04M7E1pkvgNPjyVFAUe-zgjimjIVAikxYY8Y,4418
-mdbq/aggregation/query_data.py,sha256=-oW4QMZESaK2e_MCKli3iK46jZGGdaQHAr-LduZ_Wo0,80524
+mdbq/aggregation/query_data.py,sha256=AsXH0LEnmuf1k35UvzALeJwmZwBWaenh8wfEv-u2s8U,82089
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
@@ -25,19 +25,19 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=LJJja2S5OWc-3lOHDmsVFJieFM3U69pbyWBEYQVn7P4,44541
+mdbq/mysql/mysql.py,sha256=zSGiKCMexc_6OSNMHHCM6SkY1sCKxtRuDbY352Xk9gI,46603
 mdbq/mysql/s_query.py,sha256=fIQvQKPyV7rvSUuxVWXv9S5FmCnIM4GHKconE1Zn5BA,8378
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/porxy.py,sha256=UHfgEyXugogvXgsG68a7QouUCKaohTKKkI4RN-kYSdQ,4961
 mdbq/other/pov_city.py,sha256=AEOmCOzOwyjHi9LLZWPKi6DUuSC-_M163664I52u9qw,21050
-mdbq/other/sku_picture.py,sha256=lYzm2L4SlfWq09ddr_41JjP5UcpksZpnRQYOoXNXA-k,41279
+mdbq/other/sku_picture.py,sha256=pGPQrAQluP1VRLA1UdSmdlG3JKpw8-zqy8b4r3cpzEE,47733
 mdbq/other/ua_sj.py,sha256=JuVYzc_5QZ9s_oQSrTHVKkQv4S_7-CWx4oIKOARn_9U,22178
 mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=0uAnBKCd5cx5FLTkawN1GV9yi87rfyMgYal5LABtumQ,7186
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-2.1.1.dist-info/METADATA,sha256=TGqdUDMFAQNf7fe5e3DPcL7MCfwm7UgIuAYvZoDGhdA,245
-mdbq-2.1.1.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-2.1.1.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.1.1.dist-info/RECORD,,
+mdbq-2.1.2.dist-info/METADATA,sha256=a6hTSyyVaqUNjVuLygswCreJKwgkx_cy8E7umhTVQjM,245
+mdbq-2.1.2.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-2.1.2.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.1.2.dist-info/RECORD,,