mdbq 1.8.8__tar.gz → 1.9.0__tar.gz

{mdbq-1.8.8 → mdbq-1.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.8.8
+Version: 1.9.0
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.8.8 → mdbq-1.9.0}/mdbq/aggregation/aggregation.py

@@ -326,6 +326,7 @@ class DatabaseUpdate:
                     df['省份'] = pov
                     df['省+市'] = df[['省份', '城市']].apply(lambda x: f'{x["省份"]}-{x["城市"]}', axis=1)
                     df.replace('NAN', 0, inplace=True)
+                    df['笔单价'] = df.apply(lambda x: 0 if x['销售量'] == 0 else 0 if x['销售量'] == '0' else x['笔单价'], axis=1)
                 elif name.endswith('csv') and 'order' in name:
                     # 生意经，订单数据，仅限月数据
                     pattern = re.findall(r'(.*)(\d{4})(\d{2})(\d{2})-(\d{4})(\d{2})(\d{2})', name)
@@ -378,6 +379,11 @@ class DatabaseUpdate:
                     df.replace(to_replace=['--'], value='', regex=False, inplace=True)
                     df = df[df['人群规模'] != '']
                     if len(df) == 0:
+                        if is_move:
+                            try:
+                                os.remove(os.path.join(root, name))  # 是否移除原文件
+                            except Exception as e:
+                                print(f'{name},  {e}')
                         continue
                 elif name.endswith('.csv') and '客户_客户概况_画像' in name:
                     df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
@@ -1109,10 +1115,10 @@ if __name__ == '__main__':
     #     database='mysql'
     # )
 
-    db_name = '推广数据2'
-    table_name = '营销场景报表'
+    db_name = '生意经2'
+    table_name = '省份城市分析'
     upload_dir(
-        path='/Users/xigua/数据中心/原始文件2/推广报表/营销场景报表',
+        path='/Users/xigua/数据中心/原始文件2/生意经/地域分布',
         db_name=db_name,
         collection_name=table_name,
         dbs={'mysql': True, 'mongodb': False},

{mdbq-1.8.8 → mdbq-1.9.0}/mdbq/aggregation/query_data.py

@@ -1483,7 +1483,7 @@ def main():
 
 
 if __name__ == '__main__':
-    data_aggregation(service_databases=[{'company': 'mysql'}], months=1)  # 正常的聚合所有数据
+    data_aggregation(service_databases=[{'company': 'mysql'}], months=24)  # 正常的聚合所有数据
     # data_aggregation_one(service_databases=[{'company': 'mysql'}], months=1)  # 单独聚合某一个数据库，具体库进函数编辑
     # optimize_data.op_data(service_databases=[{'company': 'mysql'}], days=3650)  # 立即启动对聚合数据的清理工作
 

{mdbq-1.8.8 → mdbq-1.9.0}/mdbq/company/copysh.py

@@ -256,8 +256,8 @@ class TbFiles:
         r = refresh_all.RefreshAll()
         for file in files:
             if file.endswith('.xlsx'):
-                now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
-                print(f'正在刷新 excel: {file}')
+                # now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+                # print(f'{now}正在刷新 excel: {file}')
                 r.refresh_excel2(excel_file=os.path.join(excel_path, file))
             time.sleep(10)
 

{mdbq-1.8.8 → mdbq-1.9.0}/mdbq/other/sku_picture.py

@@ -434,6 +434,7 @@ class SkuPicture:
                         '推荐卖点': desc,
                         '是否新增': data['是否新增'],
                         '类目': leimu,
+                        '更新时间': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                     }
                 )
             except Exception as e:
@@ -454,7 +455,8 @@ class SkuPicture:
 
         if results:
             self.df = pd.concat(results)
-            self.df = self.df[['sku图片链接'] != '0']
+            self.df.to_csv('C:\\Users\\Administrator\\Downloads\\商品id_编码_图片_临时文件.csv', encoding='utf-8_sig', index=False, header=True)
+            self.df = self.df[df['sku图片链接'] != '0']
 
     def read_df(self):
         path = os.path.join(self.path, self.filename)
@@ -483,6 +485,7 @@ class DownloadPicture():
         self.filename = ''
         if not os.path.exists(self.save_path):
             os.mkdir(self.save_path)
+        self.local_file = ''
 
     def get_df_from_service(self):
         start_date, end_date = self.months_data(num=self.months)
@@ -501,7 +504,14 @@ class DownloadPicture():
             projection=projection,
         )
 
+    def get_df_from_local(self):
+        if not os.path.isfile(self.local_file):
+            return
+        self.df = pd.read_excel(self.local_file, header=0, engine='openpyxl')
+
     def download_data(self):
+        if not os.path.exists(self.save_path):
+            os.mkdir(self.save_path)
         dict_data = self.df.to_dict('records')
         num = len(dict_data)
         i = 0
@@ -525,6 +535,31 @@ class DownloadPicture():
             i += 1
             time.sleep(0.5)
 
+    def download_data_from_local(self):
+        if not os.path.exists(self.save_path):
+            os.mkdir(self.save_path)
+        dict_data = self.df.to_dict('records')
+        num = len(dict_data)
+        i = 0
+        for data in dict_data:
+            url = data['商品图片']
+            self.filename = f'{data['商品id']}_{data['商家编码']}.jpg'
+            if os.path.isfile(os.path.join(self.save_path, self.filename)):
+                i += 1
+                continue
+            if 'https' not in url:
+                i += 1
+                continue
+
+            print(f'正在下载: {i}/{num},   {data['商品id']}')
+            self.headers.update({'User-Agent': ua_sj.get_ua()})
+            res = requests.get(url, headers=self.headers)  # 下载图片到内存
+            # 保存图片到本地文件夹
+            with open(os.path.join(self.save_path, self.filename), 'wb') as f:
+                f.write(res.content)
+            i += 1
+            time.sleep(0.5)
+
     @staticmethod
     def months_data(num=0, end_date=None):
         """ 读取近 num 个月的数据, 0 表示读取当月的数据 """
@@ -708,20 +743,30 @@ def main(service_name, database):
 def main2(service_name, database):
     """ 从数据库读取数据，并下载图片到本地 """
     d = DownloadPicture(service_name=service_name)
-    d.save_path = '/Users/xigua/Downloads/sku图片链接'  # 下载图片到本地时的存储位置
-    d.get_df_from_service()
-    d.download_data()
+    # d.save_path = '/Users/xigua/Downloads/sku图片链接'  # 下载图片到本地时的存储位置
+    # d.get_df_from_service()  # 从数据库读取数据
+    # d.download_data()
+
+    d.save_path = '/Users/xigua/Downloads/商品id_商家编码_图片'  # 下载图片到本地时的存储位置
+    d.local_file = '/Users/xigua/Downloads/商品id图片对照表.xlsx'
+    d.get_df_from_local()
+    d.download_data_from_local()
 
 
 def main3():
+    """  """
     p = InsertPicture()
     p.filename = 'test.xlsx'
     # p.header = 1
     p.insert_data()
 
 
+def main4():
+    """ 从 文件中读取图片链接并下载到本地 """
+
+
 
 if __name__ == '__main__':
-    main(service_name='home_lx', database='mysql')
-    # main2(service_name='home_lx', database='mysql')
+    # main(service_name='home_lx', database='mysql')
+    main2(service_name='home_lx', database='mysql')
     # main3()

{mdbq-1.8.8 → mdbq-1.9.0}/mdbq.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.8.8
+Version: 1.9.0
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.8.8 → mdbq-1.9.0}/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='mdbq',
-      version='1.8.8',
+      version='1.9.0',
       author='xigua, ',
       author_email="2587125111@qq.com",
       url='https://pypi.org/project/mdbsql',