mdbq 1.5.4__py3-none-any.whl → 1.5.5__py3-none-any.whl

mdbq/dataframe/converter.py

@@ -37,6 +37,7 @@ class DataFrameConverter(object):
         df.replace(to_replace=['="'], value='', regex=True, inplace=True)  # ="和"不可以放在一起清洗, 因为有: id=86785565
         df.replace(to_replace=['"'], value='', regex=True, inplace=True)
         cols = df.columns.tolist()
+        df.reset_index(inplace=True, drop=True)  # 重置索引，避免下面的 df.loc[0, col] 会出错
 
         for col in cols:
             # 百分比在某些数据库中不兼容, 转换百分比为小数

mdbq/mysql/mysql.py

@@ -180,85 +180,87 @@ class MysqlUpload:
             for data in datas:
                 # data 是传进来待处理的数据, 不是数据库数据
                 # data 示例: {'日期': Timestamp('2024-08-27 00:00:00'), '推广费余额': 33299, '品销宝余额': 2930.73, '短信剩余': 67471}
-                try:
-                    cols = ', '.join(f"`{item}`" for item in data.keys())  # 列名需要转义
-                    # data.update({item: f"{data[item]}" for item in data.keys()})  # 全部值转字符, 不是必须的
-                    values = ', '.join([f'"{item}"' for item in data.values()])  # 值要加引号
-                    condition = []
-                    for k, v in data.items():
-                        condition += [f'`{k}` = "{v}"']
-                    condition = ' AND '.join(condition)  # 构建查询条件
-                    # print(condition)
-
-                    if drop_duplicates:  # 查重插入
-                        sql = "SELECT %s FROM %s WHERE %s" % (cols, table_name, condition)
-                        # sql = f"SELECT {cols} FROM `{table_name}` WHERE `创建时间` = '2014-09-19 14:32:33'"
-                        # print(sql)
-                        cursor.execute(sql)
-                        result = cursor.fetchall()  # 获取查询结果, 有结果返回 list 表示数据已存在(不重复插入)，没有则返回空 tuple
-                        if not result:  # 数据不存在则插入
-                            sql = f"INSERT INTO `{table_name}` ({cols}) VALUES (%s);" % (values)
-                            cursor.execute(sql)
-                        # else:
-                        #     print(f'重复数据不插入: {condition[:50]}...')
-                    elif icm_update:  # 增量更新, 专门用于聚合数据，其他库不要调用
-                        """ 使用增量更新: 需确保 icm_update['主键'] 传进来的列必须是数据表中唯一主键，值不会发生变化且不会重复，否则可能产生覆盖情况 """
-                        sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
-                        cursor.execute(sql, (db_name, {table_name}))
-                        columns = cursor.fetchall()
-                        cols_exist = [col['COLUMN_NAME'] for col in columns]  # 数据表的所有列, 返回 list
-                        update_col = [item for item in cols_exist if item not in icm_update and item != 'id']  # 除了主键外的其他列
-
-                        # unique_keys 示例: `日期`, `推广费余额`
-                        unique_keys = ', '.join(f"`{item}`" for item in update_col)  # 列名需要转义
-                        condition = []
-                        for up_col in icm_update:
-                            condition += [f'`{up_col}` = "{data[up_col]}"']
-                        condition = ' AND '.join(condition)  # condition值示例: `品销宝余额` = '2930.73' AND `短信剩余` = '67471'
-                        sql = f"SELECT {unique_keys} FROM `{table_name}` WHERE {condition}"
-                        # print(sql)
-                        # sql = f"SELECT {unique_keys} FROM `{table_name}` WHERE `创建时间` = '2014-09-19 14:32:33'"
+                # try:
+                cols = ', '.join(f"`{item}`" for item in data.keys())  # 列名需要转义
+                # data.update({item: f"{data[item]}" for item in data.keys()})  # 全部值转字符, 不是必须的
+                values = ', '.join([f'"{item}"' for item in data.values()])  # 值要加引号
+                condition = []
+                for k, v in data.items():
+                    condition += [f'`{k}` = "{v}"']
+                condition = ' AND '.join(condition)  # 构建查询条件
+                # print(condition)
+
+                if drop_duplicates:  # 查重插入
+                    sql = "SELECT %s FROM %s WHERE %s" % (cols, table_name, condition)
+                    # sql = f"SELECT {cols} FROM `{table_name}` WHERE `创建时间` = '2014-09-19 14:32:33'"
+                    # print(sql)
+                    cursor.execute(sql)
+                    result = cursor.fetchall()  # 获取查询结果, 有结果返回 list 表示数据已存在(不重复插入)，没有则返回空 tuple
+                    if not result:  # 数据不存在则插入
+                        sql = f"INSERT INTO `{table_name}` ({cols}) VALUES (%s);" % (values)
                         cursor.execute(sql)
-                        results = cursor.fetchall()  # results 是数据库取出的数据
-                        if results:  # 有数据返回，再进行增量检查
-                            for result in results:  # results 是数据库数据, data 是传进来的数据
-                                not_change_col = []
-                                change_values = []
-                                for col in update_col:
-                                    # 因为 mysql 里面有 decimal 数据类型，要移除末尾的 0 再做比较（df 默认将 5.00 小数截断为 5.0）
-                                    df_value = str(data[col])
-                                    mysql_value = str(result[col])
-                                    if '.' in df_value:
-                                        df_value = re.sub(r'0+$', '', df_value)
-                                        df_value = re.sub(r'\.$', '', df_value)
-                                    if '.' in mysql_value:
-                                        mysql_value = re.sub(r'0+$', '', mysql_value)
-                                        mysql_value = re.sub(r'\.$', '', mysql_value)
-                                    if df_value != mysql_value:  # 传进来的数据和数据库比较, 有变化
-                                        # print(f'{data['日期']}{data['商品id']}{col} 列的值有变化，{str(data[col])}  !=  {str(result[col])}')
-                                        change_values += [f"`{col}` = '{str(data[col])}'"]
-                                        not_change_col += [item for item in update_col if item != col]
-                                # change_values 是 df 传进来且和数据库对比后，发生了变化的数据，值示例： [`品销宝余额` = '9999.0', `短信剩余` = '888']
-                                if change_values:  # change_values 有数据返回，表示值需要更新
+                    # else:
+                    #     print(f'重复数据不插入: {condition[:50]}...')
+                elif icm_update:  # 增量更新, 专门用于聚合数据，其他库不要调用
+                    """ 使用增量更新: 需确保 icm_update['主键'] 传进来的列必须是数据表中唯一主键，值不会发生变化且不会重复，否则可能产生覆盖情况 """
+                    sql = 'SELECT COLUMN_NAME FROM information_schema.columns WHERE table_schema = %s AND table_name = %s'
+                    cursor.execute(sql, (db_name, {table_name}))
+                    columns = cursor.fetchall()
+                    cols_exist = [col['COLUMN_NAME'] for col in columns]  # 数据表的所有列, 返回 list
+                    update_col = [item for item in cols_exist if item not in icm_update and item != 'id']  # 除了主键外的其他列
+
+                    # unique_keys 示例: `日期`, `推广费余额`
+                    unique_keys = ', '.join(f"`{item}`" for item in update_col)  # 列名需要转义
+                    condition = []
+                    for up_col in icm_update:
+                        condition += [f'`{up_col}` = "{data[up_col]}"']
+                    condition = ' AND '.join(condition)  # condition值示例: `品销宝余额` = '2930.73' AND `短信剩余` = '67471'
+                    sql = f"SELECT {unique_keys} FROM `{table_name}` WHERE {condition}"
+                    # print(sql)
+                    # sql = f"SELECT {unique_keys} FROM `{table_name}` WHERE `创建时间` = '2014-09-19 14:32:33'"
+                    cursor.execute(sql)
+                    results = cursor.fetchall()  # results 是数据库取出的数据
+                    if results:  # 有数据返回，再进行增量检查
+                        for result in results:  # results 是数据库数据, data 是传进来的数据
+                            change_col = []  # 发生变化的列名
+                            change_values = []  # 发生变化的数据
+                            for col in update_col:
+                                # 因为 mysql 里面有 decimal 数据类型，要移除末尾的 0 再做比较（df 默认将 5.00 小数截断为 5.0）
+                                df_value = str(data[col])
+                                mysql_value = str(result[col])
+                                if '.' in df_value:
+                                    df_value = re.sub(r'0+$', '', df_value)
+                                    df_value = re.sub(r'\.$', '', df_value)
+                                if '.' in mysql_value:
+                                    mysql_value = re.sub(r'0+$', '', mysql_value)
+                                    mysql_value = re.sub(r'\.$', '', mysql_value)
+                                if df_value != mysql_value:  # 传进来的数据和数据库比较, 有变化
+                                    # print(f'{data['日期']}{data['商品id']}{col} 列的值有变化，{str(data[col])}  !=  {str(result[col])}')
+                                    change_values += [f"`{col}` = \"{str(data[col])}\""]
+                                    change_col.append(col)
+                            not_change_col = [item for item in update_col if item not in change_col]
+                            # change_values 是 df 传进来且和数据库对比后，发生了变化的数据，值示例： [`品销宝余额` = '9999.0', `短信剩余` = '888']
+                            if change_values:  # change_values 有数据返回，表示值需要更新
+                                if not_change_col:
                                     not_change_values = [f'`{col}` = "{str(data[col])}"' for col in not_change_col]
                                     not_change_values = ' AND '.join(not_change_values)  # 示例: `短信剩余` = '888' AND `test1` = '93'
                                     # print(change_values, not_change_values)
                                     condition += f' AND {not_change_values}'  # 重新构建完整的查询条件，将未发生变化的列加进查询条件
-                                    change_values = ', '.join(f"{item}" for item in change_values)  # 注意这里 item 外面没有反引号
-                                    sql = f"UPDATE {table_name} SET {change_values} WHERE {condition}"
-                                    # print(sql)
-                                    cursor.execute(sql)
-                        else:  # 没有数据返回，则直接插入数据
-                            sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
-                            cursor.execute(sql)
-                    else:
-                        sql = f"INSERT INTO `{table_name}` ({cols}) VALUES (%s);" % (values)
+                                change_values = ', '.join(f"{item}" for item in change_values)  # 注意这里 item 外面没有反引号
+                                sql = "UPDATE `%s` SET %s WHERE %s" % (table_name, change_values, condition)
+                                # print(sql)
+                                cursor.execute(sql)
+                    else:  # 没有数据返回，则直接插入数据
+                        sql = f"INSERT INTO `{table_name}` ({cols}) VALUES ({values});"
                         cursor.execute(sql)
-                except Exception as e:
-                    # print(data)
-                    # print(values)
-                    print(f'mysql -> df_to_mysql 报错: {e}, {self.filename}')
-                    # breakpoint()
+                else:
+                    sql = f"INSERT INTO `{table_name}` ({cols}) VALUES (%s);" % (values)
+                    cursor.execute(sql)
+                # except Exception as e:
+                #     # print(data)
+                #     # print(values)
+                #     print(f'mysql -> df_to_mysql 报错: {e}, {self.filename}')
+                #     # breakpoint()
         connection.commit()  # 提交事务
         connection.close()
 

{mdbq-1.5.4.dist-info → mdbq-1.5.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: mdbq
-Version: 1.5.4
+Version: 1.5.5
 Home-page: https://pypi.org/project/mdbsql
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-1.5.4.dist-info → mdbq-1.5.5.dist-info}/RECORD

@@ -18,13 +18,13 @@ mdbq/config/products.py,sha256=vIK8DJ-F3XXwvNPK-4OJq2tZITNlL6Sub8QBdoOng8U,5676
 mdbq/config/set_support.py,sha256=xkZCX6y9Bq1ppBpJAofld4B2YtchA7fl0eT3dx3CrSI,777
 mdbq/config/update_conf.py,sha256=taL3ZqKgiVWwUrDFuaYhim9a72Hm4BHRhhDscJTziR8,4535
 mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
-mdbq/dataframe/converter.py,sha256=w0-gGJnIajGIhOgYGkCvc0JMcmxIHNpgPf_bgWUSOG4,3699
+mdbq/dataframe/converter.py,sha256=acNdbayG5924l8UFEoHuPn-6rAX0szzT7rm2UdYs9fc,3806
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
 mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
 mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
 mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
-mdbq/mysql/mysql.py,sha256=odWLYE4deH0AYO_EeCyf3vpw3-jEgmQW0cu572sGrdQ,43137
+mdbq/mysql/mysql.py,sha256=o2QRV3JZ_jva8u3ScFiMGcgnInSGFYfADm1-nEKMr4M,43051
 mdbq/mysql/s_query.py,sha256=a33aYhW6gAnspIZfQ7l23ePln9-MD1f_ukypr5M0jd8,8018
 mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -35,7 +35,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
 mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
 mdbq/pbix/refresh_all.py,sha256=tgy762608HMaXWynbOURIf2UVMuSPybzrDXQnOOcnZU,6102
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-1.5.4.dist-info/METADATA,sha256=hInTl3JNbUASWDNTi5AiSKMBL38BeugG0xcqN0QilFA,245
-mdbq-1.5.4.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
-mdbq-1.5.4.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-1.5.4.dist-info/RECORD,,
+mdbq-1.5.5.dist-info/METADATA,sha256=sojaLXm4UyJOVGaIdmUSed7v7vgOLZWovptcEW_oH5s,245
+mdbq-1.5.5.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
+mdbq-1.5.5.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-1.5.5.dist-info/RECORD,,