mdbq 2.2.4__py3-none-any.whl → 2.2.6__py3-none-any.whl

mdbq/company/copysh.py

@@ -371,12 +371,8 @@ def main():
             # 更新货品年份基准表， 属性设置 2 - 货品年份基准
             p = products.Products()
             p.to_mysql(service_databases=[
-                # {
-                #     'home_lx': 'mysql'
-                # },
-                {
-                    'company': 'mysql'
-                }
+                # {'home_lx': 'mysql'},
+                {'company': 'mysql'}
             ]
             )
 
@@ -405,3 +401,12 @@ if __name__ == '__main__':
     main()
     # # 聚合数据，并清理聚合数据
     # query_data.data_aggregation(service_databases=[{'company': 'mysql'}], months=1)
+
+    # sku_picture.download_spu(
+    #     service_name='company',
+    #     database='mysql',
+    #     db_name='属性设置2',
+    #     table_name='商品spu素材下载记录',
+    #     col_name='商品图片',
+    #     save_path=os.path.join(f'\\\\192.168.1.198\\时尚事业部\\01.运营部\\天猫报表\\其他文件', '商品id_商家编码_图片'),
+    # )

mdbq/other/sku_picture.py

@@ -511,6 +511,7 @@ class DownloadPicture():
         self.filename = ''
         self.local_file = ''
         self.finish_download = []
+        self.finish_d2 = [{'完成时间': [], '完成id': []}]
 
     def get_df_from_service(self):
         start_date, end_date = self.months_data(num=self.months)
@@ -598,8 +599,16 @@ class DownloadPicture():
             url = data[col_name]
             # self.filename = f'{data['店铺名称']}_{data['商品id']}_{data['商家编码']}.jpg'
             self.filename = f'{data['商品id']}_{data['商家编码']}.jpg'
+            # 清除特殊符号，避免无法创建文件
+            self.filename = re.sub(r'[\\/\u4e00-\u9fa5‘’“”【】\[\]{}、，,～~!！]', '_', self.filename)
             if os.path.isfile(os.path.join(self.save_path, self.filename)):
-                self.finish_download.append(data['商品id'])
+                # self.finish_download.append(data['商品id'])
+                self.finish_d2.append(
+                    [
+                        {'完成时间': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')},
+                        {'完成id': data['商品id']}
+                    ]
+                )
                 i += 1
                 continue
             if 'https' not in url:
@@ -609,10 +618,19 @@ class DownloadPicture():
             print(f'正在下载: {i}/{num},   {data['商品id']}')
             self.headers.update({'User-Agent': ua_sj.get_ua()})
             res = requests.get(url, headers=self.headers)  # 下载图片到内存
-            # 保存图片到本地文件夹
-            with open(os.path.join(self.save_path, self.filename), 'wb') as f:
-                f.write(res.content)
-            self.finish_download.append(data['商品id'])
+            try:
+                # 保存图片到本地文件夹
+                with open(os.path.join(self.save_path, self.filename), 'wb') as f:
+                    f.write(res.content)
+                # self.finish_download.append(data['商品id'])
+                self.finish_d2.append(
+                    [
+                        {'完成时间': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')},
+                        {'完成id': data['商品id']}
+                    ]
+                )
+            except Exception as e:
+                print(f'{self.filename}:   {e}')
             i += 1
             time.sleep(0.5)
 
@@ -921,7 +939,14 @@ def download_sku(service_name='company', database='mysql', db_name='属性设置
 
 def download_spu(service_name='company', database='mysql', db_name='属性设置2', table_name='商品spu素材下载记录', col_name='商品图片', save_path=os.path.join(D_PATH, '商品id_商家编码_图片')):
     """
-
+    1. 从商品素材导出中获取数据
+    2. 从商品id编码表 中获取数据
+        商品id编码表 是合并生意经的数据，主要获取商品编码，
+        对于未上架或者刚上架的货品，生意经未有数据，所以合并两表后，可能缺失商品编码信息
+    3. 更新数据库，不存在则创建
+    4. 从数据库中提取未下载的数据记录
+    5. 实例化一个下载器类，并下载数据
+    6. 回传并更新数据库
     """
 
     # 1. 从商品素材导出中获取数据
@@ -964,6 +989,8 @@ def download_spu(service_name='company', database='mysql', db_name='属性设置
     # df = df.head(2)
 
     # 2. 从商品id编码表 中获取数据
+    # 商品id编码表 是合并生意经的数据，主要获取商品编码，
+    # 对于未上架或者刚上架的货品，生意经未有数据，所以合并两表后，可能缺失商品编码信息
     projection = {
         '宝贝id': 1,
         '商家编码': 1,
@@ -1017,7 +1044,11 @@ def download_spu(service_name='company', database='mysql', db_name='属性设置
         d.filename = f'{db_name}_{table_name}.xlsx'
         d.df = df
         d.download_from_df(col_name=col_name, save_path=save_path)
-        df['获取与下载'] = df.apply(lambda x: '已下载' if x['商品id'] in d.finish_download else x['获取与下载'], axis=1)
+        # finish_id = [item['完成id'] for item in d.finish_d2]
+        # df['获取与下载'] = df.apply(lambda x: '已下载' if x['商品id'] in finish_id else x['获取与下载'], axis=1)
+        for item in d.finish_d2:
+            df['获取与下载'] = df.apply(lambda x: '已下载' if x['商品id'] == item['完成id'] else x['获取与下载'], axis=1)
+            df['时间'] = df.apply(lambda x: item['完成时间'] if x['商品id'] == item['完成id'] else x['时间'], axis=1)
 
         # 6. 回传数据库
         username, password, host, port = get_myconf.select_config_values(target_service=service_name, database=database)

{mdbq-2.2.4.dist-info → mdbq-2.2.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 2.2.4
+Version: 2.2.6
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-2.2.4.dist-info → mdbq-2.2.6.dist-info}/RECORD

@@ -11,7 +11,7 @@ mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
 mdbq/clean/data_clean.py,sha256=ZJOTT0sNWP85x4pJl39UZBwN4Bnf8Ao-iVOJNiiRPEE,103996
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
-mdbq/company/copysh.py,sha256=LYMONSIX2EoPyDQqFODRfIZ2TJtNQBKEeWXCbawTraI,19048
+mdbq/company/copysh.py,sha256=xjlfb-RwW0oLyyHQZFUtmXvCpWI7Y_nhwa0tqW14wZk,19345
 mdbq/company/home_sh.py,sha256=42CZ2tZIXHLl2mOl2gk2fZnjH2IHh1VJ1s3qHABjonY,18021
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/get_myconf.py,sha256=cmNvsyoNa0RbZ9FOTjSd3jyyGwkxjUo0phvdHbGlrms,6010
@@ -31,14 +31,14 @@ mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/porxy.py,sha256=UHfgEyXugogvXgsG68a7QouUCKaohTKKkI4RN-kYSdQ,4961
 mdbq/other/pov_city.py,sha256=AEOmCOzOwyjHi9LLZWPKi6DUuSC-_M163664I52u9qw,21050
-mdbq/other/sku_picture.py,sha256=vHHnajaS6Nz2sEduENihEUY_ZMhyBuQO7WuzrXSDSXM,48148
+mdbq/other/sku_picture.py,sha256=Qb2ANG-c79A6WctB4lziAwrDNUMyc6JIhKzNemqGff4,50068
 mdbq/other/ua_sj.py,sha256=JuVYzc_5QZ9s_oQSrTHVKkQv4S_7-CWx4oIKOARn_9U,22178
 mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=viOlLCmz9zg61Q2nzjgl8dChfQxnxRd1A_jmQMb2oDM,5918
 mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,7192
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-2.2.4.dist-info/METADATA,sha256=KiryBvuQemT-aGcf786FzHmwJJ3eXp7tAzIdHH6I4BM,245
-mdbq-2.2.4.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-2.2.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-2.2.4.dist-info/RECORD,,
+mdbq-2.2.6.dist-info/METADATA,sha256=dmwcBZiI-zVqfBkU3i6-YNS6rpfzNX9FJwoYCj3i9qQ,245
+mdbq-2.2.6.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-2.2.6.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-2.2.6.dist-info/RECORD,,