mdbq 2.1.1__tar.gz → 2.1.3__tar.gz

{mdbq-2.1.1 → mdbq-2.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.1.1
+Version: 2.1.3
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.1.1 → mdbq-2.1.3}/mdbq/aggregation/aggregation.py

@@ -610,6 +610,7 @@ class DatabaseUpdate:
                     except Exception as e:
                         print(e)
                         print(name, '报错')
+                        os.remove(os.path.join(root, name))
                         check_remove_file = True
                         continue
                 elif name.endswith('.xlsx') and '搜索分析-排名定位-商品排名' in name:

{mdbq-2.1.1 → mdbq-2.1.3}/mdbq/aggregation/query_data.py

@@ -421,6 +421,27 @@ class MysqlDatasQuery:
             projection=projection,
         )
         return df
+    def spu_sales(self):
+        start_date, end_date = self.months_data(num=self.months)
+        projection = {
+            '日期': 1,
+            '商品id': 1,
+            '货号': 1,
+            '成交单量': 1,
+            '成交金额': 1,
+            '访客数': 1,
+            '成交客户数': 1,
+            '加购商品件数': 1,
+            '加购人数': 1,
+        }
+        df = self.download.data_to_df(
+            db_name='京东数据2',
+            table_name='spu_商品明细',
+            start_date=start_date,
+            end_date=end_date,
+            projection=projection,
+        )
+        return df
 
     @staticmethod
     def months_data(num=0, end_date=None):
@@ -1190,6 +1211,21 @@ class GroupBy:
                 }
             )
             return df
+        elif '京东_spu_商品明细' in table_name:
+            df = df[df['商品id'] != '合计']
+            df = df.groupby(['日期', '商品id', '货号', '访客数', '成交客户数', '加购商品件数', '加购人数'],
+                            as_index=False).agg(
+                **{
+                    '成交单量': ('成交单量', np.max),
+                    '成交金额': ('成交金额', np.max),
+                   }
+            )
+            self.data_jdtg.update(
+                {
+                    table_name: df,
+                }
+            )
+            return df
         elif '京东_关键词报表' in table_name:
             df_lin = df[['计划id', '推广计划']]
             df_lin.drop_duplicates(subset=['计划id'], keep='last', inplace=True, ignore_index=True)
@@ -1580,6 +1616,12 @@ def data_aggregation(service_databases=[{}], months=1):
                     '唯一主键': ['日期', '商品id', '成交单量'],
                     '数据主体': sdq.sku_sales(),
                 },
+                {
+                    '数据库名': '聚合数据',
+                    '集合名': '京东_spu_商品明细',
+                    '唯一主键': ['日期', '商品id', '成交单量'],
+                    '数据主体': sdq.spu_sales(),
+                },
                 {
                     '数据库名': '聚合数据',
                     '集合名': '天猫_人群报表',

{mdbq-2.1.1 → mdbq-2.1.3}/mdbq/mysql/mysql.py

@@ -68,8 +68,8 @@ class MysqlUpload:
 
         return wrapper
 
-    @try_except
-    def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], service_database={'home_lx': 'mysql'}, move_insert=False, df_sql=False, drop_duplicates=False, filename=None, count=None, json_path=None):
+    # @try_except
+    def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], service_database={'home_lx': 'mysql'}, move_insert=False, df_sql=False, drop_duplicates=False, filename=None, count=None, json_path=None, reset_id=False):
         """
         将 df 写入数据库
         db_name: 数据库名称
@@ -167,6 +167,17 @@ class MysqlUpload:
                     index=False,
                     chunksize=1000
                 )
+                try:
+                    cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
+                    result = cursor.fetchone()
+                    if result:
+                        cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
+                    cursor.execute(
+                        f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                    cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+                except Exception as e:
+                    print(f'{e}')
+                    connection.rollback()
                 connection.close()
                 return
 
@@ -205,6 +216,19 @@ class MysqlUpload:
                     index=False,
                     chunksize=1000
                 )
+                # 6. 重置自增列
+                if reset_id:
+                    try:
+                        cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
+                        result = cursor.fetchone()
+                        if result:
+                            cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
+                        cursor.execute(
+                            f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                        cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+                    except Exception as e:
+                        print(f'{e}')
+                        connection.rollback()
                 connection.close()
                 return
 
@@ -293,6 +317,19 @@ class MysqlUpload:
                     # print(values)
                     print(f'mysql -> df_to_mysql 报错: {e}, {self.filename}')
                     # breakpoint()
+
+            # 6. 重置自增列
+            try:
+                cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
+                result = cursor.fetchone()
+                if result:
+                    cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
+                cursor.execute(
+                    f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+            except Exception as e:
+                print(f'{e}')
+                connection.rollback()
         connection.commit()  # 提交事务
         connection.close()
 

{mdbq-2.1.1 → mdbq-2.1.3}/mdbq/other/sku_picture.py

@@ -11,6 +11,7 @@ import time
 import warnings
 import pandas as pd
 from lxml import etree
+from rich.pretty import pretty_repr
 from selenium import webdriver
 from selenium.webdriver.support.wait import WebDriverWait
 from selenium.webdriver.common.by import By
@@ -501,7 +502,7 @@ class DownloadPicture():
         self.download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
         self.df = pd.DataFrame()
         self.headers = {'User-Agent': ua_sj.get_ua()}
-        self.save_path = '/Users/xigua/Downloads/sku图片链接'
+        self.save_path = os.path.join(D_PATH, 'sku图片链接')
         self.filename = ''
         if not os.path.exists(self.save_path):
             os.mkdir(self.save_path)
@@ -582,6 +583,34 @@ class DownloadPicture():
             i += 1
             time.sleep(0.5)
 
+    def download_from_df(self, col_name='商品图片'):
+        if not os.path.exists(self.save_path):
+            os.mkdir(self.save_path)
+        dict_data = self.df.to_dict('records')
+        num = len(dict_data)
+        i = 1
+        for data in dict_data:
+            url = data[col_name]
+            # self.filename = f'{data['店铺名称']}_{data['商品id']}_{data['商家编码']}.jpg'
+            self.filename = f'{data['商品id']}_{data['商家编码']}.jpg'
+            if os.path.isfile(os.path.join(self.save_path, self.filename)):
+                self.finish_download.append(data['商品id'])
+                i += 1
+                continue
+            if 'https' not in url:
+                i += 1
+                continue
+
+            print(f'正在下载: {i}/{num},   {data['商品id']}')
+            self.headers.update({'User-Agent': ua_sj.get_ua()})
+            res = requests.get(url, headers=self.headers)  # 下载图片到内存
+            # 保存图片到本地文件夹
+            with open(os.path.join(self.save_path, self.filename), 'wb') as f:
+                f.write(res.content)
+            self.finish_download.append(data['商品id'])
+            i += 1
+            time.sleep(0.5)
+
     @staticmethod
     def months_data(num=0, end_date=None):
         """ 读取近 num 个月的数据, 0 表示读取当月的数据 """
@@ -783,7 +812,7 @@ def main3():
     p.insert_data()
 
 
-def get_sp_id(service_name='company', database='mysql', db_name='属性设置2', table_name='商品素材下载记录', col_name='sku图片链接'):
+def download_sku(service_name='company', database='mysql', db_name='属性设置2', table_name='商品素材下载记录', col_name='sku图片链接'):
     """ 从数据库中获取商品id信息 """
     # 实例化一个下载类
     username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
@@ -885,9 +914,125 @@ def get_sp_id(service_name='company', database='mysql', db_name='属性设置2',
     )  # 3. 回传数据库
 
 
+def download_spu(service_name='company', database='mysql', db_name='属性设置2', table_name='商品spu素材下载记录', col_name='商品图片'):
+    """
+
+    """
+
+    # 1. 从商品素材导出中获取数据
+    username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
+    download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
+    projection = {
+        '店铺名称': 1,
+        '商品id': 1,
+        '商品标题': 1,
+        '商品状态': 1,
+        '商品白底图': 1,
+        '方版场景图': 1,
+        '日期':1,
+    }
+    df = download.data_to_df(
+        db_name='属性设置2',
+        table_name='商品素材导出',
+        start_date='2019-01-01',
+        end_date='2099-12-31',
+        projection=projection,
+    )
+    df['商品id'] = df['商品id'].astype('int64')
+    df['日期'] = df['日期'].astype('datetime64[ns]')
+    df = df[(df['商品白底图'] != '0') | (df['方版场景图'] != '0')]
+    # 白底图优先
+    df['商品图片'] = df[['商品白底图', '方版场景图']].apply(
+        lambda x: x['商品白底图'] if x['商品白底图'] != '0' else x['方版场景图'], axis=1)
+    # # 方版场景图优先
+    # df['商品图片'] = df[['商品白底图', '方版场景图']].apply(
+    #     lambda x: x['方版场景图'] if x['方版场景图'] != '0' else x['商品白底图'], axis=1)
+    df.sort_values(by=['商品id', '日期'], ascending=[False, True], ignore_index=True, inplace=True)
+    df.drop_duplicates(subset=['商品id'], keep='last', inplace=True, ignore_index=True)
+    # df = df[['商品id', '商品图片', '日期']]
+    df['商品图片'] = df['商品图片'].apply(lambda x: x if 'http' in x else None)  # 检查是否是 http 链接
+    df.dropna(how='all', subset=['商品图片'], axis=0, inplace=True)  # 删除指定列含有空值的行
+    df['商品链接'] = df['商品id'].apply(
+        lambda x: f'https://detail.tmall.com/item.htm?id={str(x)}' if x and '.com' not in str(x) else x)
+    df.sort_values(by='商品id', ascending=False, ignore_index=True, inplace=True)  # ascending=False 降序排列
+    df['商品id'] = df['商品id'].astype(str)
+    # df = df.head(2)
+
+    # 2. 从商品id编码表 中获取数据
+    projection = {
+        '宝贝id': 1,
+        '商家编码': 1,
+    }
+    df_spbm = download.data_to_df(
+        db_name='聚合数据',
+        table_name='商品id编码表',
+        start_date='2019-01-01',
+        end_date='2099-12-31',
+        projection=projection,
+    )
+    df_spbm.drop_duplicates(subset=['宝贝id'], keep='last', inplace=True, ignore_index=True)
+    # 合并两个表
+    df = pd.merge(df, df_spbm, left_on=['商品id'], right_on=['宝贝id'], how='left')
+    df.pop('宝贝id')
+    df['获取与下载'] = '已获取'
+    df['时间'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+    # df.to_csv(os.path.join(D_PATH, f'{col_name}.csv'), index=False, header=True, encoding='utf-8_sig')
+    if '方版场景图' in df.columns.tolist():
+        df['方版场景图'] = df['方版场景图'].astype(str)
+
+    # 3. 更新数据库
+    username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
+    m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
+    m.df_to_mysql(
+        df=df,
+        db_name=db_name,
+        table_name=table_name,
+        move_insert=True,  # 先删除，再插入
+        df_sql=False,
+        drop_duplicates=False,
+        icm_update=[],
+        service_database={service_name: database},
+    )  # 3. 回传数据库
+
+    # 4. 从数据库中提取未下载的数据
+    projection = {}
+    df_before = download.data_to_df(
+        db_name=db_name,
+        table_name=table_name,
+        start_date='2019-01-01',
+        end_date='2099-12-31',
+        projection=projection,
+    )
+    df = df_before[df_before['获取与下载'] != '已下载']
+
+    if len(df) > 0:
+        #  5. 实例化一个下载器类，并下载数据
+        d = DownloadPicture(service_name=service_name)
+        d.save_path = os.path.join(D_PATH, '商品id_商家编码_图片')  # 下载图片到本地时的存储位置
+        d.filename = f'{db_name}_{table_name}.xlsx'
+        d.df = df
+        d.download_from_df(col_name=col_name)
+        df['获取与下载'] = df.apply(lambda x: '已下载' if x['商品id'] in d.finish_download else x['获取与下载'], axis=1)
+
+        # 6. 回传数据库
+        username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
+        m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
+        m.df_to_mysql(
+            df=df,
+            db_name=db_name,
+            table_name=table_name,
+            move_insert=True,  # 先删除，再插入
+            df_sql=False,
+            drop_duplicates=False,
+            icm_update=[],
+            service_database={service_name: database},
+            reset_id=True,
+        )  # 3. 回传数据库
+
 
 if __name__ == '__main__':
     # main(service_name='home_lx', database='mysql')
     # main2(service_name='home_lx', database='mysql')
     # main3()
-    get_sp_id(service_name='company', database='mysql', db_name='属性设置2', table_name='商品素材下载记录')
+    # download_sku(service_name='company', database='mysql', db_name='属性设置2', table_name='商品素材下载记录')
+    download_spu(service_name='company', database='mysql', db_name='属性设置2', table_name='商品spu素材下载记录', col_name='商品图片')

{mdbq-2.1.1 → mdbq-2.1.3}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.1.1
+Version: 2.1.3
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.1.1 → mdbq-2.1.3}/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='mdbq',
-      version='2.1.1',
+      version='2.1.3',
       author='xigua, ',
       author_email="2587125111@qq.com",
       url='https://pypi.org/project/mdbsql',