mdbq 3.1.2__py3-none-any.whl → 3.1.4__py3-none-any.whl

mdbq/mysql/mysql.py

@@ -127,7 +127,7 @@ class MysqlUpload:
         return wrapper
 
     @try_except
-    def dict_to_mysql(self, db_name, table_name, dict_data, icm_update=None, main_key=None, unique_main_key=None, index_length=100, set_typ=None):
+    def dict_to_mysql(self, db_name, table_name, dict_data, icm_update=None, main_key=None, unique_main_key=None, index_length=100, set_typ=None, allow_not_null=False):
         """
         插入字典数据
         dict_data： 字典
@@ -136,6 +136,7 @@ class MysqlUpload:
         index_length: 索引长度
         icm_update: 增量更正，指定后 main_key 只用于检查/创建列，不能更新数据
         set_typ: {}
+        allow_not_null: 创建允许插入空值的列，正常情况下不允许空值
         """
         if not main_key:
             main_key = []
@@ -186,7 +187,11 @@ class MysqlUpload:
             if col_not_exist:  # 数据表中不存在的列
                 for col in col_not_exist:
                     #  创建列，需转义
-                    sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]} NOT NULL;"
+                    if allow_not_null:
+                        sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]};"
+                    else:
+                        sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]} NOT NULL;"
+                    # sql = f"ALTER TABLE `{table_name}` ADD COLUMN `{col}` {dtypes[col]} NOT NULL;"
                     cursor.execute(sql)
                     print(f"添加列: {col}({dtypes[col]})")  # 添加列并指定数据类型
 
@@ -324,20 +329,21 @@ class MysqlUpload:
                 v = str(float(v.rstrip("%")) / 100)
 
             result1 = re.findall(r'编码|_?id|货号|款号|文件大小', k, re.IGNORECASE)
-            result2 = re.findall(r'占比$|投产$|产出$|同比$|环比$|roi$|率$', k, re.IGNORECASE)
+            result2 = re.findall(r'占比$|投产$|产出$|roi$|率$', k, re.IGNORECASE)
+            result3 = re.findall(r'同比$|环比$', k, re.IGNORECASE)
+            result4 = re.findall(r'花费$|消耗$|金额$', k, re.IGNORECASE)
+
             date_type = is_valid_date(v)  # 判断日期时间
             int_num = is_integer(v)  # 判断整数
             count_int, count_float = count_decimal_places(v)  # 判断小数，返回小数位数
             if result1:  # 京东sku/spu商品信息
-                __res_dict.update({k: 'mediumtext'})
+                __res_dict.update({k: 'varchar(100)'})
             elif k == '日期':
                 __res_dict.update({k: 'DATE'})
             elif k == '更新时间':
                 __res_dict.update({k: 'TIMESTAMP'})
-            elif str(v) == '':
-                __res_dict.update({k: 'mediumtext'})
             elif result2:  # 小数
-                __res_dict.update({k: 'decimal(12,4)'})
+                __res_dict.update({k: 'decimal(10,4)'})
             elif date_type == 1:  # 纯日期
                 __res_dict.update({k: 'DATE'})
             elif date_type == 2:  # 日期+时间
@@ -356,7 +362,7 @@ class MysqlUpload:
                 else:
                     __res_dict.update({k: 'decimal(10,2)'})
             else:
-                __res_dict.update({k: 'mediumtext'})
+                __res_dict.update({k: 'varchar(255)'})
             new_dict_data.update({k: v})
         return __res_dict, new_dict_data
 

mdbq/mysql/s_query.py

@@ -35,6 +35,20 @@ class QueryDatas:
             'cursorclass': pymysql.cursors.DictCursor,
         }
 
+    def check_condition(self, db_name, table_name, condition):
+        """ 按指定条件查询数据库，并返回 """
+        if self.check_infos(db_name, table_name) == False:
+            return
+
+        self.config.update({'database': db_name})
+        connection = pymysql.connect(**self.config)  # 重新连接数据库
+        with connection.cursor() as cursor:
+            sql = f"SELECT 更新时间 FROM {table_name} WHERE {condition}"
+            # print(sql)
+            cursor.execute(sql)
+            columns = cursor.fetchall()
+            return columns
+
     def data_to_df(self, db_name, table_name, start_date, end_date, projection: dict=[]):
 
         start_date = pd.to_datetime(start_date).strftime('%Y-%m-%d')

mdbq/other/download_sku_picture.py

@@ -958,8 +958,6 @@ def download_spu(username, password, host, port, col_name='白底图', save_path
     从 属性设置 3 ->  商品sku属性 数据库中提取图片地址，下载图片
     col_name： 从那一列提取图片下载地址 ，， 白底图
     """
-
-    # 1. 从商品素材导出中获取数据
     download = s_query.QueryDatas(username=username, password=password, host=host, port=port)
     projection = {
         '日期': 1,
@@ -975,6 +973,7 @@ def download_spu(username, password, host, port, col_name='白底图', save_path
         end_date='2099-12-31',
         projection=projection,
     )
+    df = df[df['白底图'] != '0']
     df['商品id'] = df['商品id'].astype('int64')
     df['日期'] = df['日期'].astype('datetime64[ns]')
     df.sort_values(by=['商品id', '日期'], ascending=[False, True], ignore_index=True, inplace=True)
@@ -990,10 +989,6 @@ def download_spu(username, password, host, port, col_name='白底图', save_path
 
 
 if __name__ == '__main__':
-    # main(service_name='home_lx', database='mysql')
-    # main2(service_name='home_lx', database='mysql')
-    # main3()
-    # download_sku(service_name='company', database='mysql', db_name='属性设置2', table_name='商品素材下载记录')
     download_spu(
         username=username,
         password=password,

mdbq/spider/aikucun.py

@@ -405,7 +405,7 @@ class AikuCunNew:
 
 if __name__ == '__main__':
     pass
-    # get_cookie_aikucun()
+    get_cookie_aikucun()
     akucun()
 
     # a = AikuCunNew(shop_name='aikucun')

{mdbq-3.1.2.dist-info → mdbq-3.1.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 3.1.2
+Version: 3.1.4
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-3.1.2.dist-info → mdbq-3.1.4.dist-info}/RECORD

@@ -27,12 +27,12 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=u2aKSn72SGCkRIo9UvV7pibjYadIBz0V4fp8uK3gnIE,60972
+mdbq/mysql/mysql.py,sha256=AUaHL51CuPrAXrrUKFYm_Gu1yWMvJbs0luZtg0b4q2c,61401
 mdbq/mysql/recheck_mysql.py,sha256=rgTpvDMWYTyEn7UQdlig-pdXDluTgiU8JG6lkMh8DV0,8665
-mdbq/mysql/s_query.py,sha256=fnXncwSmA7CB0ELn1a-YxYZDrYkC2Bcgnj2J4dcQ8X8,8481
+mdbq/mysql/s_query.py,sha256=30WilUkm6lsiI5HfGphnnBSVbAXiNsgn2mEPd7AhKJs,9055
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
-mdbq/other/download_sku_picture.py,sha256=YsVYTveZsNo-pZDL2Lw6nILXaKSV36V_i-P2X391gWE,46062
+mdbq/other/download_sku_picture.py,sha256=Daef1a_cCsh__2g1E8E0KRmZJI-3ERCxCY6CBLs9fuo,45805
 mdbq/other/porxy.py,sha256=UHfgEyXugogvXgsG68a7QouUCKaohTKKkI4RN-kYSdQ,4961
 mdbq/other/pov_city.py,sha256=AEOmCOzOwyjHi9LLZWPKi6DUuSC-_M163664I52u9qw,21050
 mdbq/other/sku_picture.py,sha256=JwSXYlzamVqcKCD2tRH2VqYVZNr8fM6f--kcGlTVRnM,50026
@@ -44,8 +44,8 @@ mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,
 mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/req_post/req_tb.py,sha256=qg7pet73IgKGmCwxaeUyImJIoeK_pBQT9BBKD7fkBNg,36160
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq/spider/aikucun.py,sha256=01qJo_Di5Kmi2lG5_HKb0OI283b1-Pgqh-nnA0pX4TY,19038
-mdbq-3.1.2.dist-info/METADATA,sha256=Rzl3rAPJpEtvqP42LB7gy8DobZqRAtijVHGpc3HHYz8,243
-mdbq-3.1.2.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-mdbq-3.1.2.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-3.1.2.dist-info/RECORD,,
+mdbq/spider/aikucun.py,sha256=48isoL6nEi_uniV-ja2HwYAI7O8D1i1goO4SzBwDGSU,19036
+mdbq-3.1.4.dist-info/METADATA,sha256=8Xz3qFCp1N7uxkpQXEBU5CkWHNwWrOiqnYFC5CNzjOM,243
+mdbq-3.1.4.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+mdbq-3.1.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-3.1.4.dist-info/RECORD,,