mdbq 2.0.9__py3-none-any.whl → 2.1.1__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -1084,6 +1084,7 @@ class DatabaseUpdate:
             if '预算' in col:
                 df.rename(columns={col: '预算占比'}, inplace=True)
         df = df[['商品id', '商家编码', '预算占比']]
+        df['日期'] = datetime.datetime.now().strftime('%Y-%m-%d')
         for service_database in service_databases:
             for service_name, database in service_database.items():
                 username, password, host, port = get_myconf.select_config_values(

mdbq/company/home_sh.py

@@ -23,11 +23,14 @@ from mdbq.mysql import mysql
 if platform.system() == 'Windows':
     from mdbq.pbix import refresh_all
 warnings.filterwarnings('ignore')
+"""
+除公司台式机外，其他主机执行下载更新任务
+"""
 
 
 class TbFiles:
     """
-    用于在公司台式机中 定时同步pandas数据源文件到共享
+    用于定时同步pandas数据源文件到共享
     """
     def __init__(self):
 
@@ -108,7 +111,7 @@ class TbFiles:
             f.write(f'data_path = {self.data_path}\n\n')
             f.write('# 共享目录\n')
             f.write(f'share_path = {self.share_path}\n\n')
-            f.write('# 公司台式机中，用于触发下载百度云文件，更新至本机数据库\n')
+            f.write('# 用于触发下载百度云文件，更新至本机数据库\n')
             f.write(f'home_record = False\n\n')
         print('目录初始化!')
 
@@ -287,7 +290,7 @@ class UpdateMysql:
         self.d_path = None
 
     def check_date(self):
-        """ 检查公司台式机 .copysh_conf 文件中的 ch_record 值，决定是否执行更新"""
+        """ 检查文件中的 home_record 值，决定是否执行更新"""
         config = configparser.ConfigParser()  # 初始化configparser类
         try:
             config.read(self.my_conf, 'UTF-8')

mdbq/other/sku_picture.py

@@ -31,11 +31,13 @@ from openpyxl.utils import get_column_letter
 warnings.filterwarnings('ignore')
 
 if platform.system() == 'Windows':
-    Share_Path = os.path.join(r'\\192.168.1.198\时尚事业部\01.运营部\天猫报表')  # 共享文件根目录
-    # Share_Path = os.path.join(r'\\192.168.1.198\时尚事业部\01.运营部\天猫报表')  # 共享文件根目录
+    D_PATH = os.path.normpath(f'C:\\Users\\{getpass.getuser()}\\Downloads')
+    Share_Path = os.path.normpath(r'\\192.168.1.198\时尚事业部\01.运营部\天猫报表')  # 共享文件根目录
 elif platform.system() == 'Darwin':
-    Share_Path = os.path.join('/Volumes/时尚事业部/01.运营部/天猫报表')  # 共享文件根目录
+    D_PATH = os.path.normpath(f'/Users/{getpass.getuser()}/Downloads')
+    Share_Path = os.path.normpath('/Volumes/时尚事业部/01.运营部/天猫报表')  # 共享文件根目录
 else:
+    D_PATH = 'Downloads'
     Share_Path = ''
 
 
@@ -85,9 +87,25 @@ class LoadAccount:
         option.add_experimental_option('prefs', prefs)
         option.add_experimental_option('excludeSwitches', ['enable-automation'])  # 实验性参数, 左上角小字
         if platform.system() == 'Windows':
-            service = Service(os.path.join(f'C:\\Users\\{getpass.getuser()}\\chromedriver.exe'))
+            # 设置 chrome 和 chromedriver 启动路径
+            chrome_path = os.path.join(f'C:\\Users\\{getpass.getuser()}', 'chrome\\chrome_win64\\chrome.exe')
+            chromedriver_path = os.path.join(f'C:\\Users\\{getpass.getuser()}', 'chrome\\chromedriver.exe')
+            # os.environ["webdriver.chrome.driver"] = chrome_path
+            option.binary_location = chrome_path  # windows 设置此参数有效
+            service = Service(chromedriver_path)
+            # service = Service(str(pathlib.Path(f'C:\\Users\\{getpass.getuser()}\\chromedriver.exe')))  # 旧路径
+        elif platform.system() == 'Darwin':
+            chrome_path = '/usr/local/chrome/Google Chrome for Testing.app'
+            chromedriver_path = '/usr/local/chrome/chromedriver'
+            os.environ["webdriver.chrome.driver"] = chrome_path
+            # option.binary_location = chrome_path  # Macos 设置此参数报错
+            service = Service(chromedriver_path)
         else:
-            service = Service('/usr/local/bin/chromedriver')
+            chrome_path = '/usr/local/chrome/Google Chrome for Testing.app'
+            chromedriver_path = '/usr/local/chrome/chromedriver'
+            os.environ["webdriver.chrome.driver"] = chrome_path
+            # option.binary_location = chrome_path  # macos 设置此参数报错
+            service = Service(chromedriver_path)
         _driver = webdriver.Chrome(options=option, service=service,  )  # 创建Chrome驱动程序实例
         _driver.maximize_window()  # 窗口最大化 方便后续加载数据
 
@@ -355,8 +373,9 @@ class SkuPicture:
         self.urls = []
         self.datas = []  # 从单品页面获取数据，存储这部分数据，作为中转
         self.df = pd.DataFrame()
+        self.xlsx_save_path = os.path.join(D_PATH, '商品id_编码_图片_临时文件.xlsx')
 
-    def each_page(self):
+    def each_page(self, as_local_file=True):
         wait = WebDriverWait(self.driver, timeout=15)
         num = len(self.urls)
         i = 0
@@ -432,13 +451,13 @@ class SkuPicture:
                         'sku编码': sku_spbm,
                         '商家编码': data['商家编码'],
                         '推荐卖点': desc,
-                        '是否新增': data['是否新增'],
+                        '获取与下载': '已获取',
                         '类目': leimu,
                         '更新时间': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                     }
                 )
             except Exception as e:
-                # print(e)
+                print('报错信息: ', e, '-'*10, data)
                 pass
             i += 1
             # if i > 3:
@@ -448,15 +467,16 @@ class SkuPicture:
         results = []
         for data in self.datas:
             try:
-                df = pd.DataFrame.from_dict(data, orient='columns')
-                results.append(df)
+                _df = pd.DataFrame.from_dict(data, orient='columns')
+                results.append(_df)
             except:
                 pass
 
-        if results:
-            self.df = pd.concat(results)
-            self.df.to_csv('C:\\Users\\Administrator\\Downloads\\商品id_编码_图片_临时文件.csv', encoding='utf-8_sig', index=False, header=True)
-            self.df = self.df[df['sku图片链接'] != '0']
+        self.df = pd.concat(results)  # 更新 df
+        self.df = self.df[self.df['sku图片链接'] != '0']
+        if results and as_local_file:
+            self.df.to_excel(self.xlsx_save_path, index=False, header=True, engine='openpyxl',
+                        freeze_panes=(1, 0))
 
     def read_df(self):
         path = os.path.join(self.path, self.filename)
@@ -486,6 +506,7 @@ class DownloadPicture():
         if not os.path.exists(self.save_path):
             os.mkdir(self.save_path)
         self.local_file = ''
+        self.finish_download = []
 
     def get_df_from_service(self):
         start_date, end_date = self.months_data(num=self.months)
@@ -535,15 +556,15 @@ class DownloadPicture():
             i += 1
             time.sleep(0.5)
 
-    def download_data_from_local(self):
+    def download_data_from_local(self, col_name='sku图片链接'):
         if not os.path.exists(self.save_path):
             os.mkdir(self.save_path)
         dict_data = self.df.to_dict('records')
         num = len(dict_data)
         i = 0
         for data in dict_data:
-            url = data['商品图片']
-            self.filename = f'{data['商品id']}_{data['商家编码']}.jpg'
+            url = data[col_name]
+            self.filename = f'{data['商品id']}_{data['商家编码']}_{data['sku编码']}.jpg'
             if os.path.isfile(os.path.join(self.save_path, self.filename)):
                 i += 1
                 continue
@@ -557,6 +578,7 @@ class DownloadPicture():
             # 保存图片到本地文件夹
             with open(os.path.join(self.save_path, self.filename), 'wb') as f:
                 f.write(res.content)
+            self.finish_download.append(data['sku编码'])
             i += 1
             time.sleep(0.5)
 
@@ -761,12 +783,111 @@ def main3():
     p.insert_data()
 
 
-def main4():
-    """ 从 文件中读取图片链接并下载到本地 """
+def get_sp_id(service_name='company', database='mysql', db_name='属性设置2', table_name='商品素材下载记录', col_name='sku图片链接'):
+    """ 从数据库中获取商品id信息 """
+    # 实例化一个下载类
+    username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
+    download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
+    projection = {
+        '宝贝id': 1,
+        '商家编码': 1,
+    }
+    df = download.data_to_df(
+        db_name='生意经2',
+        table_name='宝贝指标',
+        start_date='2019-01-01',
+        end_date='2099-12-31',
+        projection=projection,
+    )
+    df.rename(columns={'宝贝id': '商品id'}, inplace=True)
+    df.drop_duplicates(subset='商品id', keep='last', inplace=True, ignore_index=True)
+    df = df.head(2)
+
+    projection = {
+        '商品id': 1,
+        '商家编码': 1,
+    }
+    df_new = download.data_to_df(
+        db_name='属性设置2',
+        table_name='商品素材下载记录',
+        start_date='2019-01-01',
+        end_date='2099-12-31',
+        projection=projection,
+    )
+    df_new.drop_duplicates(subset='商品id', keep='last', inplace=True, ignore_index=True)
+    # 使用merge获取交集
+    df = pd.merge(df, df_new, left_on=['商品id'], right_on=['商品id'], how='left')
+    df.rename(columns={'商家编码_x': '商家编码'}, inplace=True)
+    df.pop('商家编码_y')
+    urls = df.to_dict('records')
+
+    _driver = LoadAccount()  # 账号域不同, 要重新实例化
+    tb_driver2 = _driver.load_account(shop_name='万里马官方旗舰店')
+    if tb_driver2:
+        s = SkuPicture(driver=tb_driver2)
+        s.urls = urls
+        s.each_page(as_local_file=True)  # 根据 urls 获取每个商品数据并更新 df
+        tb_driver2.quit()
+
+        # 回传数据库
+        username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
+        m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
+        m.df_to_mysql(
+            df=s.df,
+            db_name=db_name,
+            table_name=table_name,
+            move_insert=True,  # 先删除，再插入
+            df_sql=False,
+            drop_duplicates=False,
+            icm_update=[],
+            service_database={service_name: database},
+        )  # 3. 回传数据库
+
+    # 从数据库中读取数据，并下载素材到本地
+
+    # 留空，必须留空
+    projection = {
+        # '商品id': 1,
+        # '商家编码': 1,
+        # 'sku编码': 1,
+        # col_name: 1,
+        # '获取与下载': 1,
+    }
+    df = download.data_to_df(
+        db_name=db_name,
+        table_name=table_name,
+        start_date='2019-01-01',
+        end_date='2099-12-31',
+        projection=projection,
+    )
+    df = df[df['获取与下载'] != '已下载']
+
+    # 实例化一个下载器类
+    d = DownloadPicture(service_name=service_name)
+    d.save_path = os.path.join(D_PATH, col_name)  # 下载图片到本地时的存储位置
+    d.filename = f'{db_name}_{table_name}.xlsx'
+    d.df = df
+    d.download_data_from_local(col_name=col_name)
+    df['获取与下载'] = df.apply(lambda x: '已下载' if x['sku编码'] in d.finish_download else x['获取与下载'], axis=1)
+
+    # 回传数据库
+    username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)
+    m = mysql.MysqlUpload(username=username, password=password, host=host, port=port)
+    m.df_to_mysql(
+        df=df,
+        db_name=db_name,
+        table_name=table_name,
+        move_insert=True,  # 先删除，再插入
+        df_sql=False,
+        drop_duplicates=False,
+        icm_update=[],
+        service_database={service_name: database},
+    )  # 3. 回传数据库
 
 
 
 if __name__ == '__main__':
     # main(service_name='home_lx', database='mysql')
-    main2(service_name='home_lx', database='mysql')
+    # main2(service_name='home_lx', database='mysql')
     # main3()
+    get_sp_id(service_name='company', database='mysql', db_name='属性设置2', table_name='商品素材下载记录')

{mdbq-2.0.9.dist-info → mdbq-2.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.0.9
+Version: 2.1.1
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.0.9.dist-info → mdbq-2.1.1.dist-info}/RECORD

@@ -1,7 +1,7 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=3C30v7gZaSoUqwUQH5xeEBZDUA-hIo-xxFdKQgUCi7o,75742
+mdbq/aggregation/aggregation.py,sha256=us2FJjLCYlyttATHc0xYRo1ZvcC4E7lB_JI034jI6h4,75810
 mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
 mdbq/aggregation/mysql_types.py,sha256=DQYROALDiwjJzjhaJfIIdnsrNs11i5BORlj_v6bp67Y,11062
 mdbq/aggregation/optimize_data.py,sha256=Wis40oL04M7E1pkvgNPjyVFAUe-zgjimjIVAikxYY8Y,4418
@@ -12,7 +12,7 @@ mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
 mdbq/clean/data_clean.py,sha256=ZJOTT0sNWP85x4pJl39UZBwN4Bnf8Ao-iVOJNiiRPEE,103996
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
 mdbq/company/copysh.py,sha256=GaFl7lZ5pENys0W2N8SOjUdA0uz5v_u-aDZMLZADito,17778
-mdbq/company/home_sh.py,sha256=Sd2rWX4FjPSBiSiLK5jzTuoahZoVeHhjJxfTX4AXXxQ,18019
+mdbq/company/home_sh.py,sha256=42CZ2tZIXHLl2mOl2gk2fZnjH2IHh1VJ1s3qHABjonY,18021
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/get_myconf.py,sha256=cmNvsyoNa0RbZ9FOTjSd3jyyGwkxjUo0phvdHbGlrms,6010
 mdbq/config/products.py,sha256=L1uhzdbqTprQg_rekKt0ucgpeIuMvi3H2v48_GZWPuY,5803
@@ -31,13 +31,13 @@ mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/porxy.py,sha256=UHfgEyXugogvXgsG68a7QouUCKaohTKKkI4RN-kYSdQ,4961
 mdbq/other/pov_city.py,sha256=AEOmCOzOwyjHi9LLZWPKi6DUuSC-_M163664I52u9qw,21050
-mdbq/other/sku_picture.py,sha256=Et8gpfAEqe7J9Z1TQSSPNuZ1OytesUztnFHMoxCfNv0,35866
+mdbq/other/sku_picture.py,sha256=lYzm2L4SlfWq09ddr_41JjP5UcpksZpnRQYOoXNXA-k,41279
 mdbq/other/ua_sj.py,sha256=JuVYzc_5QZ9s_oQSrTHVKkQv4S_7-CWx4oIKOARn_9U,22178
 mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=0uAnBKCd5cx5FLTkawN1GV9yi87rfyMgYal5LABtumQ,7186
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-2.0.9.dist-info/METADATA,sha256=dtpUOBBxdWBbRVn04ZFKE_SN24tDjeEFUZzUChIf30I,245
-mdbq-2.0.9.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-2.0.9.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.0.9.dist-info/RECORD,,
+mdbq-2.1.1.dist-info/METADATA,sha256=TGqdUDMFAQNf7fe5e3DPcL7MCfwm7UgIuAYvZoDGhdA,245
+mdbq-2.1.1.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-2.1.1.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.1.1.dist-info/RECORD,,