mdbq 1.8.8__py3-none-any.whl → 1.9.0__py3-none-any.whl

mdbq/aggregation/aggregation.py

@@ -326,6 +326,7 @@ class DatabaseUpdate:
                     df['省份'] = pov
                     df['省+市'] = df[['省份', '城市']].apply(lambda x: f'{x["省份"]}-{x["城市"]}', axis=1)
                     df.replace('NAN', 0, inplace=True)
+                    df['笔单价'] = df.apply(lambda x: 0 if x['销售量'] == 0 else 0 if x['销售量'] == '0' else x['笔单价'], axis=1)
                 elif name.endswith('csv') and 'order' in name:
                     # 生意经，订单数据，仅限月数据
                     pattern = re.findall(r'(.*)(\d{4})(\d{2})(\d{2})-(\d{4})(\d{2})(\d{2})', name)
@@ -378,6 +379,11 @@ class DatabaseUpdate:
                     df.replace(to_replace=['--'], value='', regex=False, inplace=True)
                     df = df[df['人群规模'] != '']
                     if len(df) == 0:
+                        if is_move:
+                            try:
+                                os.remove(os.path.join(root, name))  # 是否移除原文件
+                            except Exception as e:
+                                print(f'{name},  {e}')
                         continue
                 elif name.endswith('.csv') and '客户_客户概况_画像' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
@@ -1109,10 +1115,10 @@ if __name__ == '__main__':
     #     database='mysql'
     # )
 
-    db_name = '推广数据2'
-    table_name = '营销场景报表'
+    db_name = '生意经2'
+    table_name = '省份城市分析'
     upload_dir(
-        path='/Users/xigua/数据中心/原始文件2/推广报表/营销场景报表',
+        path='/Users/xigua/数据中心/原始文件2/生意经/地域分布',
         db_name=db_name,
         collection_name=table_name,
         dbs={'mysql': True, 'mongodb': False},

mdbq/aggregation/query_data.py

@@ -1483,7 +1483,7 @@ def main():
 
 
 if __name__ == '__main__':
-    data_aggregation(service_databases=[{'company': 'mysql'}], months=1)  # 正常的聚合所有数据
+    data_aggregation(service_databases=[{'company': 'mysql'}], months=24)  # 正常的聚合所有数据
     # data_aggregation_one(service_databases=[{'company': 'mysql'}], months=1)  # 单独聚合某一个数据库，具体库进函数编辑
     # optimize_data.op_data(service_databases=[{'company': 'mysql'}], days=3650)  # 立即启动对聚合数据的清理工作
 

mdbq/company/copysh.py

@@ -256,8 +256,8 @@ class TbFiles:
         r = refresh_all.RefreshAll()
         for file in files:
             if file.endswith('.xlsx'):
-                now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
-                print(f'正在刷新 excel: {file}')
+                # now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+                # print(f'{now}正在刷新 excel: {file}')
                 r.refresh_excel2(excel_file=os.path.join(excel_path, file))
             time.sleep(10)
 

mdbq/other/sku_picture.py

@@ -434,6 +434,7 @@ class SkuPicture:
                         '推荐卖点': desc,
                         '是否新增': data['是否新增'],
                         '类目': leimu,
+                        '更新时间': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                     }
                 )
             except Exception as e:
@@ -454,7 +455,8 @@ class SkuPicture:
 
         if results:
             self.df = pd.concat(results)
-            self.df = self.df[['sku图片链接'] != '0']
+            self.df.to_csv('C:\\Users\\Administrator\\Downloads\\商品id_编码_图片_临时文件.csv', encoding='utf-8_sig', index=False, header=True)
+            self.df = self.df[df['sku图片链接'] != '0']
 
     def read_df(self):
         path = os.path.join(self.path, self.filename)
@@ -483,6 +485,7 @@ class DownloadPicture():
         self.filename = ''
         if not os.path.exists(self.save_path):
             os.mkdir(self.save_path)
+        self.local_file = ''
 
     def get_df_from_service(self):
         start_date, end_date = self.months_data(num=self.months)
@@ -501,7 +504,14 @@ class DownloadPicture():
             projection=projection,
         )
 
+    def get_df_from_local(self):
+        if not os.path.isfile(self.local_file):
+            return
+        self.df = pd.read_excel(self.local_file, header=0, engine='openpyxl')
+
     def download_data(self):
+        if not os.path.exists(self.save_path):
+            os.mkdir(self.save_path)
         dict_data = self.df.to_dict('records')
         num = len(dict_data)
         i = 0
@@ -525,6 +535,31 @@ class DownloadPicture():
             i += 1
             time.sleep(0.5)
 
+    def download_data_from_local(self):
+        if not os.path.exists(self.save_path):
+            os.mkdir(self.save_path)
+        dict_data = self.df.to_dict('records')
+        num = len(dict_data)
+        i = 0
+        for data in dict_data:
+            url = data['商品图片']
+            self.filename = f'{data['商品id']}_{data['商家编码']}.jpg'
+            if os.path.isfile(os.path.join(self.save_path, self.filename)):
+                i += 1
+                continue
+            if 'https' not in url:
+                i += 1
+                continue
+
+            print(f'正在下载: {i}/{num},   {data['商品id']}')
+            self.headers.update({'User-Agent': ua_sj.get_ua()})
+            res = requests.get(url, headers=self.headers)  # 下载图片到内存
+            # 保存图片到本地文件夹
+            with open(os.path.join(self.save_path, self.filename), 'wb') as f:
+                f.write(res.content)
+            i += 1
+            time.sleep(0.5)
+
     @staticmethod
     def months_data(num=0, end_date=None):
         """ 读取近 num 个月的数据, 0 表示读取当月的数据 """
@@ -708,20 +743,30 @@ def main(service_name, database):
 def main2(service_name, database):
     """ 从数据库读取数据，并下载图片到本地 """
     d = DownloadPicture(service_name=service_name)
-    d.save_path = '/Users/xigua/Downloads/sku图片链接'  # 下载图片到本地时的存储位置
-    d.get_df_from_service()
-    d.download_data()
+    # d.save_path = '/Users/xigua/Downloads/sku图片链接'  # 下载图片到本地时的存储位置
+    # d.get_df_from_service()  # 从数据库读取数据
+    # d.download_data()
+
+    d.save_path = '/Users/xigua/Downloads/商品id_商家编码_图片'  # 下载图片到本地时的存储位置
+    d.local_file = '/Users/xigua/Downloads/商品id图片对照表.xlsx'
+    d.get_df_from_local()
+    d.download_data_from_local()
 
 
 def main3():
+    """  """
     p = InsertPicture()
     p.filename = 'test.xlsx'
     # p.header = 1
     p.insert_data()
 
 
+def main4():
+    """ 从 文件中读取图片链接并下载到本地 """
+
+
 
 if __name__ == '__main__':
-    main(service_name='home_lx', database='mysql')
-    # main2(service_name='home_lx', database='mysql')
+    # main(service_name='home_lx', database='mysql')
+    main2(service_name='home_lx', database='mysql')
     # main3()

{mdbq-1.8.8.dist-info → mdbq-1.9.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.8.8
+Version: 1.9.0
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.8.8.dist-info → mdbq-1.9.0.dist-info}/RECORD

@@ -1,17 +1,17 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
 mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
 mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
-mdbq/aggregation/aggregation.py,sha256=zEdc7mktE2Mxfubuq4i95fvGJ93tLga8uPHcGWEyaQ8,64630
+mdbq/aggregation/aggregation.py,sha256=LEOpmH_Kyf08N4z9pdqMznRicLp9ON28UJZyZMZS1Uo,65029
 mdbq/aggregation/df_types.py,sha256=oQJS2IBU3_IO6GMgbssHuC2yCjNnbta0QPGrFOwNLnU,7591
 mdbq/aggregation/mysql_types.py,sha256=DQYROALDiwjJzjhaJfIIdnsrNs11i5BORlj_v6bp67Y,11062
 mdbq/aggregation/optimize_data.py,sha256=u2Kl_MFtZueXJ57ycy4H2OhXD431RctUYJYCl637uT0,4176
-mdbq/aggregation/query_data.py,sha256=rK7CiaLgVdUARcYLU9tVZBweIQFZ75eiaT4f1toeBs8,70446
+mdbq/aggregation/query_data.py,sha256=u4f5Pdy_foD225FzNigOJ20FCiu0wedP2gVvlai-mUw,70447
 mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
 mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
 mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
 mdbq/clean/data_clean.py,sha256=T0WYOKFwNZTNk3temKOw1K2H54kxu9QBJjlTbkMtxNk,94217
 mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
-mdbq/company/copysh.py,sha256=tEWittsWE_ocp-ilUvbDQncIVaVgtjjja5M0WHjecE4,17753
+mdbq/company/copysh.py,sha256=4PGjvmPzvrmstOaAwHQGFXIGCWqqNXZEOYf1QdUvMlI,17762
 mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/config/get_myconf.py,sha256=-CFEW0dQh4OIwVgwK-cL0eVp1LN3PjJgN89d4P5TB9I,6011
 mdbq/config/products.py,sha256=vIK8DJ-F3XXwvNPK-4OJq2tZITNlL6Sub8QBdoOng8U,5676
@@ -30,13 +30,13 @@ mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/porxy.py,sha256=UHfgEyXugogvXgsG68a7QouUCKaohTKKkI4RN-kYSdQ,4961
 mdbq/other/pov_city.py,sha256=AEOmCOzOwyjHi9LLZWPKi6DUuSC-_M163664I52u9qw,21050
-mdbq/other/sku_picture.py,sha256=LSydTC94j-ZKMAYyf9dBt8oWa9c3gjKXV7W9G6RTc6I,33958
+mdbq/other/sku_picture.py,sha256=Et8gpfAEqe7J9Z1TQSSPNuZ1OytesUztnFHMoxCfNv0,35866
 mdbq/other/ua_sj.py,sha256=JuVYzc_5QZ9s_oQSrTHVKkQv4S_7-CWx4oIKOARn_9U,22178
 mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=0uAnBKCd5cx5FLTkawN1GV9yi87rfyMgYal5LABtumQ,7186
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-1.8.8.dist-info/METADATA,sha256=nrg6a1-7bb9W7AM1-8u6qvwRp2MvoqtQtdn8nctfnhQ,245
-mdbq-1.8.8.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-1.8.8.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-1.8.8.dist-info/RECORD,,
+mdbq-1.9.0.dist-info/METADATA,sha256=2SY_Iv7xOesLzV_0mOfXlEPeYjbVSshWuoLOXj-mAoA,245
+mdbq-1.9.0.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-1.9.0.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-1.9.0.dist-info/RECORD,,