mdbq 2.1.0__tar.gz → 2.1.2__tar.gz

{mdbq-2.1.0 → mdbq-2.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.1.0
+Version: 2.1.2
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.1.0 → mdbq-2.1.2}/mdbq/aggregation/query_data.py

@@ -421,6 +421,27 @@ class MysqlDatasQuery:
             projection=projection,
         )
         return df
+    def spu_sales(self):
+        start_date, end_date = self.months_data(num=self.months)
+        projection = {
+            '日期': 1,
+            '商品id': 1,
+            '货号': 1,
+            '成交单量': 1,
+            '成交金额': 1,
+            '访客数': 1,
+            '成交客户数': 1,
+            '加购商品件数': 1,
+            '加购人数': 1,
+        }
+        df = self.download.data_to_df(
+            db_name='京东数据2',
+            table_name='spu_商品明细',
+            start_date=start_date,
+            end_date=end_date,
+            projection=projection,
+        )
+        return df
 
     @staticmethod
     def months_data(num=0, end_date=None):
@@ -1190,6 +1211,21 @@ class GroupBy:
                 }
             )
             return df
+        elif '京东_spu_商品明细' in table_name:
+            df = df[df['商品id'] != '合计']
+            df = df.groupby(['日期', '商品id', '货号', '访客数', '成交客户数', '加购商品件数', '加购人数'],
+                            as_index=False).agg(
+                **{
+                    '成交单量': ('成交单量', np.max),
+                    '成交金额': ('成交金额', np.max),
+                   }
+            )
+            self.data_jdtg.update(
+                {
+                    table_name: df,
+                }
+            )
+            return df
         elif '京东_关键词报表' in table_name:
             df_lin = df[['计划id', '推广计划']]
             df_lin.drop_duplicates(subset=['计划id'], keep='last', inplace=True, ignore_index=True)
@@ -1580,6 +1616,12 @@ def data_aggregation(service_databases=[{}], months=1):
                     '唯一主键': ['日期', '商品id', '成交单量'],
                     '数据主体': sdq.sku_sales(),
                 },
+                {
+                    '数据库名': '聚合数据',
+                    '集合名': '京东_spu_商品明细',
+                    '唯一主键': ['日期', '商品id', '成交单量'],
+                    '数据主体': sdq.spu_sales(),
+                },
                 {
                     '数据库名': '聚合数据',
                     '集合名': '天猫_人群报表',

{mdbq-2.1.0 → mdbq-2.1.2}/mdbq/mysql/mysql.py

@@ -68,8 +68,8 @@ class MysqlUpload:
 
         return wrapper
 
-    @try_except
-    def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], service_database={'home_lx': 'mysql'}, move_insert=False, df_sql=False, drop_duplicates=False, filename=None, count=None, json_path=None):
+    # @try_except
+    def df_to_mysql(self, df, table_name, db_name='远程数据源', icm_update=[], service_database={'home_lx': 'mysql'}, move_insert=False, df_sql=False, drop_duplicates=False, filename=None, count=None, json_path=None, reset_id=False):
         """
         将 df 写入数据库
         db_name: 数据库名称
@@ -167,6 +167,17 @@ class MysqlUpload:
                     index=False,
                     chunksize=1000
                 )
+                try:
+                    cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
+                    result = cursor.fetchone()
+                    if result:
+                        cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
+                    cursor.execute(
+                        f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                    cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+                except Exception as e:
+                    print(f'{e}')
+                    connection.rollback()
                 connection.close()
                 return
 
@@ -205,6 +216,19 @@ class MysqlUpload:
                     index=False,
                     chunksize=1000
                 )
+                # 6. 重置自增列
+                if reset_id:
+                    try:
+                        cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
+                        result = cursor.fetchone()
+                        if result:
+                            cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
+                        cursor.execute(
+                            f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                        cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+                    except Exception as e:
+                        print(f'{e}')
+                        connection.rollback()
                 connection.close()
                 return
 
@@ -293,6 +317,19 @@ class MysqlUpload:
                     # print(values)
                     print(f'mysql -> df_to_mysql 报错: {e}, {self.filename}')
                     # breakpoint()
+
+            # 6. 重置自增列
+            try:
+                cursor.execute(f"SHOW COLUMNS FROM {table_name} LIKE 'id'")
+                result = cursor.fetchone()
+                if result:
+                    cursor.execute(f"ALTER TABLE {table_name} DROP COLUMN id;")  # 删除 id  列
+                cursor.execute(
+                    f"ALTER TABLE {table_name} ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
+                cursor.execute(f"ALTER TABLE {table_name} AUTO_INCREMENT = 1")  # 设置自增从 1 开始
+            except Exception as e:
+                print(f'{e}')
+                connection.rollback()
         connection.commit()  # 提交事务
         connection.close()
 

{mdbq-2.1.0 → mdbq-2.1.2}/mdbq/other/sku_picture.py

@@ -11,6 +11,7 @@ import time
 import warnings
 import pandas as pd
 from lxml import etree
+from rich.pretty import pretty_repr
 from selenium import webdriver
 from selenium.webdriver.support.wait import WebDriverWait
 from selenium.webdriver.common.by import By
@@ -31,11 +32,13 @@ from openpyxl.utils import get_column_letter
 warnings.filterwarnings('ignore')
 
 if platform.system() == 'Windows':
-    Share_Path = os.path.join(r'\\192.168.1.198\时尚事业部\01.运营部\天猫报表')  # 共享文件根目录
-    # Share_Path = os.path.join(r'\\192.168.1.198\时尚事业部\01.运营部\天猫报表')  # 共享文件根目录
+    D_PATH = os.path.normpath(f'C:\\Users\\{getpass.getuser()}\\Downloads')
+    Share_Path = os.path.normpath(r'\\192.168.1.198\时尚事业部\01.运营部\天猫报表')  # 共享文件根目录
 elif platform.system() == 'Darwin':
-    Share_Path = os.path.join('/Volumes/时尚事业部/01.运营部/天猫报表')  # 共享文件根目录
+    D_PATH = os.path.normpath(f'/Users/{getpass.getuser()}/Downloads')
+    Share_Path = os.path.normpath('/Volumes/时尚事业部/01.运营部/天猫报表')  # 共享文件根目录
 else:
+    D_PATH = 'Downloads'
     Share_Path = ''
 
 
@@ -85,9 +88,25 @@ class LoadAccount:
         option.add_experimental_option('prefs', prefs)
         option.add_experimental_option('excludeSwitches', ['enable-automation'])  # 实验性参数, 左上角小字
         if platform.system() == 'Windows':
-            service = Service(os.path.join(f'C:\\Users\\{getpass.getuser()}\\chromedriver.exe'))
+            # 设置 chrome 和 chromedriver 启动路径
+            chrome_path = os.path.join(f'C:\\Users\\{getpass.getuser()}', 'chrome\\chrome_win64\\chrome.exe')
+            chromedriver_path = os.path.join(f'C:\\Users\\{getpass.getuser()}', 'chrome\\chromedriver.exe')
+            # os.environ["webdriver.chrome.driver"] = chrome_path
+            option.binary_location = chrome_path  # windows 设置此参数有效
+            service = Service(chromedriver_path)
+            # service = Service(str(pathlib.Path(f'C:\\Users\\{getpass.getuser()}\\chromedriver.exe')))  # 旧路径
+        elif platform.system() == 'Darwin':
+            chrome_path = '/usr/local/chrome/Google Chrome for Testing.app'
+            chromedriver_path = '/usr/local/chrome/chromedriver'
+            os.environ["webdriver.chrome.driver"] = chrome_path
+            # option.binary_location = chrome_path  # Macos 设置此参数报错
+            service = Service(chromedriver_path)
         else:
-            service = Service('/usr/local/bin/chromedriver')
+            chrome_path = '/usr/local/chrome/Google Chrome for Testing.app'
+            chromedriver_path = '/usr/local/chrome/chromedriver'
+            os.environ["webdriver.chrome.driver"] = chrome_path
+            # option.binary_location = chrome_path  # macos 设置此参数报错
+            service = Service(chromedriver_path)
         _driver = webdriver.Chrome(options=option, service=service,  )  # 创建Chrome驱动程序实例
         _driver.maximize_window()  # 窗口最大化 方便后续加载数据
 
@@ -355,8 +374,9 @@ class SkuPicture:
         self.urls = []
         self.datas = []  # 从单品页面获取数据，存储这部分数据，作为中转
         self.df = pd.DataFrame()
+        self.xlsx_save_path = os.path.join(D_PATH, '商品id_编码_图片_临时文件.xlsx')
 
-    def each_page(self):
+    def each_page(self, as_local_file=True):
         wait = WebDriverWait(self.driver, timeout=15)
         num = len(self.urls)
         i = 0
@@ -432,13 +452,13 @@ class SkuPicture:
                         'sku编码': sku_spbm,
                         '商家编码': data['商家编码'],
                         '推荐卖点': desc,
-                        '是否新增': data['是否新增'],
+                        '获取与下载': '已获取',
                         '类目': leimu,
                         '更新时间': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                     }
                 )
             except Exception as e:
-                # print(e)
+                print('报错信息: ', e, '-'*10, data)
                 pass
             i += 1
             # if i > 3:
@@ -448,15 +468,16 @@ class SkuPicture:
         results = []
         for data in self.datas:
             try:
-                df = pd.DataFrame.from_dict(data, orient='columns')
-                results.append(df)
+                _df = pd.DataFrame.from_dict(data, orient='columns')
+                results.append(_df)
             except:
                 pass
 
-        if results:
-            self.df = pd.concat(results)
-            self.df.to_csv('C:\\Users\\Administrator\\Downloads\\商品id_编码_图片_临时文件.csv', encoding='utf-8_sig', index=False, header=True)
-            self.df = self.df[df['sku图片链接'] != '0']
+        self.df = pd.concat(results)  # 更新 df
+        self.df = self.df[self.df['sku图片链接'] != '0']
+        if results and as_local_file:
+            self.df.to_excel(self.xlsx_save_path, index=False, header=True, engine='openpyxl',
+                        freeze_panes=(1, 0))
 
     def read_df(self):
         path = os.path.join(self.path, self.filename)
@@ -486,6 +507,7 @@ class DownloadPicture():
         if not os.path.exists(self.save_path):
             os.mkdir(self.save_path)
         self.local_file = ''
+        self.finish_download = []
 
     def get_df_from_service(self):
         start_date, end_date = self.months_data(num=self.months)
@@ -535,16 +557,43 @@ class DownloadPicture():
             i += 1
             time.sleep(0.5)
 
-    def download_data_from_local(self):
+    def download_data_from_local(self, col_name='sku图片链接'):
         if not os.path.exists(self.save_path):
             os.mkdir(self.save_path)
         dict_data = self.df.to_dict('records')
         num = len(dict_data)
         i = 0
         for data in dict_data:
-            url = data['商品图片']
-            self.filename = f'{data['商品id']}_{data['商家编码']}.jpg'
+            url = data[col_name]
+            self.filename = f'{data['商品id']}_{data['商家编码']}_{data['sku编码']}.jpg'
+            if os.path.isfile(os.path.join(self.save_path, self.filename)):
+                i += 1
+                continue
+            if 'https' not in url:
+                i += 1
+                continue
+
+            print(f'正在下载: {i}/{num},   {data['商品id']}')
+            self.headers.update({'User-Agent': ua_sj.get_ua()})
+            res = requests.get(url, headers=self.headers)  # 下载图片到内存
+            # 保存图片到本地文件夹
+            with open(os.path.join(self.save_path, self.filename), 'wb') as f:
+                f.write(res.content)
+            self.finish_download.append(data['sku编码'])
+            i += 1
+            time.sleep(0.5)
+
+    def download_from_df(self, col_name='商品图片'):
+        if not os.path.exists(self.save_path):
+            os.mkdir(self.save_path)
+        dict_data = self.df.to_dict('records')
+        num = len(dict_data)
+        i = 1
+        for data in dict_data:
+            url = data[col_name]
+            self.filename = f'{data['店铺名称']}_{data['商品id']}_{data['商家编码']}.jpg'
             if os.path.isfile(os.path.join(self.save_path, self.filename)):
+                self.finish_download.append(data['商品id'])
                 i += 1
                 continue
             if 'https' not in url:
@@ -557,6 +606,7 @@ class DownloadPicture():
             # 保存图片到本地文件夹
             with open(os.path.join(self.save_path, self.filename), 'wb') as f:
                 f.write(res.content)
+            self.finish_download.append(data['商品id'])
             i += 1
             time.sleep(0.5)
 
@@ -761,12 +811,227 @@ def main3():
     p.insert_data()
 
 
-def main4():
-    """ 从 文件中读取图片链接并下载到本地 """
+def download_sku(service_name='company', database='mysql', db_name='属性设置2', table_name='商品素材下载记录', col_name='sku图片链接'):
+    """ 从数据库中获取商品id信息 """
+    # 实例化一个下载类
+    username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
+    download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
+    projection = {
+        '宝贝id': 1,
+        '商家编码': 1,
+    }
+    df = download.data_to_df(
+        db_name='生意经2',
+        table_name='宝贝指标',
+        start_date='2019-01-01',
+        end_date='2099-12-31',
+        projection=projection,
+    )
+    df.rename(columns={'宝贝id': '商品id'}, inplace=True)
+    df.drop_duplicates(subset='商品id', keep='last', inplace=True, ignore_index=True)
+    df = df.head(2)
+
+    projection = {
+        '商品id': 1,
+        '商家编码': 1,
+    }
+    df_new = download.data_to_df(
+        db_name='属性设置2',
+        table_name='商品素材下载记录',
+        start_date='2019-01-01',
+        end_date='2099-12-31',
+        projection=projection,
+    )
+    df_new.drop_duplicates(subset='商品id', keep='last', inplace=True, ignore_index=True)
+    # 使用merge获取交集
+    df = pd.merge(df, df_new, left_on=['商品id'], right_on=['商品id'], how='left')
+    df.rename(columns={'商家编码_x': '商家编码'}, inplace=True)
+    df.pop('商家编码_y')
+    urls = df.to_dict('records')
+
+    _driver = LoadAccount()  # 账号域不同, 要重新实例化
+    tb_driver2 = _driver.load_account(shop_name='万里马官方旗舰店')
+    if tb_driver2:
+        s = SkuPicture(driver=tb_driver2)
+        s.urls = urls
+        s.each_page(as_local_file=True)  # 根据 urls 获取每个商品数据并更新 df
+        tb_driver2.quit()
+
+        # 回传数据库
+        username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
+        m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
+        m.df_to_mysql(
+            df=s.df,
+            db_name=db_name,
+            table_name=table_name,
+            move_insert=True,  # 先删除，再插入
+            df_sql=False,
+            drop_duplicates=False,
+            icm_update=[],
+            service_database={service_name: database},
+        )  # 3. 回传数据库
+
+    # 从数据库中读取数据，并下载素材到本地
+
+    # 留空，必须留空
+    projection = {
+        # '商品id': 1,
+        # '商家编码': 1,
+        # 'sku编码': 1,
+        # col_name: 1,
+        # '获取与下载': 1,
+    }
+    df = download.data_to_df(
+        db_name=db_name,
+        table_name=table_name,
+        start_date='2019-01-01',
+        end_date='2099-12-31',
+        projection=projection,
+    )
+    df = df[df['获取与下载'] != '已下载']
+
+    # 实例化一个下载器类
+    d = DownloadPicture(service_name=service_name)
+    d.save_path = os.path.join(D_PATH, col_name)  # 下载图片到本地时的存储位置
+    d.filename = f'{db_name}_{table_name}.xlsx'
+    d.df = df
+    d.download_data_from_local(col_name=col_name)
+    df['获取与下载'] = df.apply(lambda x: '已下载' if x['sku编码'] in d.finish_download else x['获取与下载'], axis=1)
+
+    # 回传数据库
+    username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
+    m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
+    m.df_to_mysql(
+        df=df,
+        db_name=db_name,
+        table_name=table_name,
+        move_insert=True,  # 先删除，再插入
+        df_sql=False,
+        drop_duplicates=False,
+        icm_update=[],
+        service_database={service_name: database},
+    )  # 3. 回传数据库
+
+
+def download_spu(service_name='company', database='mysql', db_name='属性设置2', table_name='商品spu素材下载记录', col_name='商品图片'):
+    """
+
+    """
 
+    # 1. 从商品素材导出中获取数据
+    username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
+    download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
+    projection = {
+        '店铺名称': 1,
+        '商品id': 1,
+        '商品标题': 1,
+        '商品状态': 1,
+        '商品白底图': 1,
+        '方版场景图': 1,
+        '日期':1,
+    }
+    df = download.data_to_df(
+        db_name='属性设置2',
+        table_name='商品素材导出',
+        start_date='2019-01-01',
+        end_date='2099-12-31',
+        projection=projection,
+    )
+    df['商品id'] = df['商品id'].astype('int64')
+    df['日期'] = df['日期'].astype('datetime64[ns]')
+    df = df[(df['商品白底图'] != '0') | (df['方版场景图'] != '0')]
+    # 白底图优先
+    df['商品图片'] = df[['商品白底图', '方版场景图']].apply(
+        lambda x: x['商品白底图'] if x['商品白底图'] != '0' else x['方版场景图'], axis=1)
+    # # 方版场景图优先
+    # df['商品图片'] = df[['商品白底图', '方版场景图']].apply(
+    #     lambda x: x['方版场景图'] if x['方版场景图'] != '0' else x['商品白底图'], axis=1)
+    df.sort_values(by=['商品id', '日期'], ascending=[False, True], ignore_index=True, inplace=True)
+    df.drop_duplicates(subset=['商品id'], keep='last', inplace=True, ignore_index=True)
+    # df = df[['商品id', '商品图片', '日期']]
+    df['商品图片'] = df['商品图片'].apply(lambda x: x if 'http' in x else None)  # 检查是否是 http 链接
+    df.dropna(how='all', subset=['商品图片'], axis=0, inplace=True)  # 删除指定列含有空值的行
+    df['商品链接'] = df['商品id'].apply(
+        lambda x: f'https://detail.tmall.com/item.htm?id={str(x)}' if x and '.com' not in str(x) else x)
+    df.sort_values(by='商品id', ascending=False, ignore_index=True, inplace=True)  # ascending=False 降序排列
+    df['商品id'] = df['商品id'].astype(str)
+    # df = df.head(2)
+
+    # 2. 从商品id编码表 中获取数据
+    projection = {
+        '宝贝id': 1,
+        '商家编码': 1,
+    }
+    df_spbm = download.data_to_df(
+        db_name='聚合数据',
+        table_name='商品id编码表',
+        start_date='2019-01-01',
+        end_date='2099-12-31',
+        projection=projection,
+    )
+    df_spbm.drop_duplicates(subset=['宝贝id'], keep='last', inplace=True, ignore_index=True)
+    # 合并两个表
+    df = pd.merge(df, df_spbm, left_on=['商品id'], right_on=['宝贝id'], how='left')
+    df.pop('宝贝id')
+    df['获取与下载'] = '已获取'
+    df['时间'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+    # df.to_csv(os.path.join(D_PATH, f'{col_name}.csv'), index=False, header=True, encoding='utf-8_sig')
+    if '方版场景图' in df.columns.tolist():
+        df['方版场景图'] = df['方版场景图'].astype(str)
+
+    # 3. 更新数据库
+    username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
+    m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
+    m.df_to_mysql(
+        df=df,
+        db_name=db_name,
+        table_name=table_name,
+        move_insert=True,  # 先删除，再插入
+        df_sql=False,
+        drop_duplicates=False,
+        icm_update=[],
+        service_database={service_name: database},
+    )  # 3. 回传数据库
+
+    # 4. 从数据库中提取未下载的数据
+    projection = {}
+    df_before = download.data_to_df(
+        db_name=db_name,
+        table_name=table_name,
+        start_date='2019-01-01',
+        end_date='2099-12-31',
+        projection=projection,
+    )
+    df = df_before[df_before['获取与下载'] != '已下载']
+
+    if len(df) > 0:
+        #  5. 实例化一个下载器类，并下载数据
+        d = DownloadPicture(service_name=service_name)
+        d.save_path = os.path.join(D_PATH, '商品id_商家编码_图片')  # 下载图片到本地时的存储位置
+        d.filename = f'{db_name}_{table_name}.xlsx'
+        d.df = df
+        d.download_from_df(col_name=col_name)
+        df['获取与下载'] = df.apply(lambda x: '已下载' if x['商品id'] in d.finish_download else x['获取与下载'], axis=1)
+
+        # 6. 回传数据库
+        username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
+        m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
+        m.df_to_mysql(
+            df=df,
+            db_name=db_name,
+            table_name=table_name,
+            move_insert=True,  # 先删除，再插入
+            df_sql=False,
+            drop_duplicates=False,
+            icm_update=[],
+            service_database={service_name: database},
+            reset_id=True,
+        )  # 3. 回传数据库
 
 
 if __name__ == '__main__':
     # main(service_name='home_lx', database='mysql')
-    main2(service_name='home_lx', database='mysql')
+    # main2(service_name='home_lx', database='mysql')
     # main3()
+    # download_sku(service_name='company', database='mysql', db_name='属性设置2', table_name='商品素材下载记录')
+    download_spu(service_name='company', database='mysql', db_name='属性设置2', table_name='商品spu素材下载记录', col_name='商品图片')

{mdbq-2.1.0 → mdbq-2.1.2}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.1.0
+Version: 2.1.2
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.1.0 → mdbq-2.1.2}/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='mdbq',
-      version='2.1.0',
+      version='2.1.2',
       author='xigua, ',
       author_email="2587125111@qq.com",
       url='https://pypi.org/project/mdbsql',