cppackage 0.2.9__py3-none-any.whl → 0.3.1__py3-none-any.whl

CPpackage/db/sql_model.py

@@ -1,8 +1,6 @@
 import pymysql
 import time
 from pymysql import Error
-import pandas as pd
-# 条件导入：支持直接运行和包模式
 try:
     from .config import get_db_config
 except (ImportError, ValueError):
@@ -161,6 +159,13 @@ def find_duplicate_records(table_name, database, unique_index_fields, port=None)
     """
     
     try:
+        # Lazy import pandas here to avoid heavy imports during module import
+        try:
+            import pandas as pd
+        except Exception as e:
+            print("无法导入 pandas:", e)
+            return None
+
         with _get_connection(database, port) as conn:
             df_duplicates = pd.read_sql(sql, conn)
             if df_duplicates.empty:
@@ -168,8 +173,8 @@ def find_duplicate_records(table_name, database, unique_index_fields, port=None)
             else:
                 print(f"发现 {len(df_duplicates)} 条重复记录（{len(df_duplicates.drop_duplicates(subset=unique_index_fields))} 组重复组合）")
                 # 可选：保存重复记录到CSV
-                df_duplicates.to_csv(f"duplicate_records_{table_name}.csv", index=False, encoding="utf-8-sig")
-                print("重复记录已保存到 duplicate_records_{table_name}.csv")
+                # df_duplicates.to_csv(f"duplicate_records_{table_name}.csv", index=False, encoding="utf-8-sig")
+                # print("重复记录已保存到 duplicate_records_{table_name}.csv")
             return df_duplicates
     except pymysql.MySQLError as e:
         print("数据库查询失败:", e)
@@ -178,7 +183,7 @@ def find_duplicate_records(table_name, database, unique_index_fields, port=None)
         print("未知错误:", e)
         return None
 # ===================== 删除重复记录函数（修复参数+调用逻辑） =====================
-def delete_duplicate_records(table_name, database, unique_index_fields, port=None, keep_strategy="min_id"):
+def delete_duplicate_records(table_name, database, unique_index_fields, port=None, keep_strategy="max_id"):
     """
     删除重复记录，仅保留每组唯一值的一条记录
     参数:
@@ -190,10 +195,6 @@ def delete_duplicate_records(table_name, database, unique_index_fields, port=Non
     返回:
         int: 删除的记录数；None: 出错；0: 无重复
     """
-    # 1. 先调用修复后的find_duplicate_records查询重复数据
-    duplicate_df = find_duplicate_records(table_name, database, unique_index_fields, port)
-    if duplicate_df is None or duplicate_df.empty:
-        return 0
     
     # 2. 构建删除SQL（核心逻辑）
     group_by_str = ",".join([f"`{field}`" for field in unique_index_fields])
@@ -243,43 +244,51 @@ def delete_duplicate_records(table_name, database, unique_index_fields, port=Non
         return None
 # ===================== DataFrame 入库 =====================
 
-def update_datas(df, table_name, database):
-
-    conn = None
-    for i in range(2):
-        try:
-            conn = _get_connection(database)
-            cursor = conn.cursor()
-            cols = list(df.columns)
-            col_str = ",".join([f"`{c}`" for c in cols])
-            value_tpl = "(" + ",".join(["%s"] * len(cols)) + ")"
-            values_str = ",".join([value_tpl] * len(df))
-            update_clause = ",".join(
-                [f"`{c}`=VALUES(`{c}`)" for c in cols if c != "id"]
-            )
-            sql = f"""
-            INSERT INTO `{table_name}` ({col_str})
-            VALUES {values_str}
-            ON DUPLICATE KEY UPDATE {update_clause}
-            """
-            print(sql)
-            data = [tuple(row) for row in df.values]
-            flat_data = [v for row in data for v in row]
-            cursor.execute(sql, flat_data)
-            conn.commit()
-            print(f"成功插入/更新 {cursor.rowcount} 条记录")
-            break
-        except Exception as e:
-            if conn:
-                conn.rollback()
-            print("入库失败:", e)
-            if '1054, "Unknown column' in str(e):
-                # 表结构校验,df为抓取数据表
-                check_and_sync_columns(df,table_name,database)
-            else:
-                raise Exception(e)
-        finally:
-            if conn:
-                conn.close()
+def update_datas(df, table_name, database, batch_size=20):
 
+    if df.empty:
+        print("数据为空，无需入库")
+        return
+    cols = list(df.columns)
+    col_str = ",".join([f"`{c}`" for c in cols])
+    value_tpl = "(" + ",".join(["%s"] * len(cols)) + ")"
+    update_clause = ",".join(
+        [f"`{c}`=VALUES(`{c}`)" for c in cols if c != "id"]
+    )
+    total = len(df)
+    success_count = 0
+    for start in range(0, total, batch_size):
+        batch_df = df.iloc[start:start + batch_size]
+        values_str = ",".join([value_tpl] * len(batch_df))
+        sql = f"""
+        INSERT INTO `{table_name}` ({col_str})
+        VALUES {values_str}
+        ON DUPLICATE KEY UPDATE {update_clause}
+        """
+        data = [tuple(row) for row in batch_df.values]
+        flat_data = [v for row in data for v in row]
+        conn = None
+        for attempt in range(2):
+            try:
+                conn = _get_connection(database)
+                cursor = conn.cursor()
+                cursor.execute(sql, flat_data)
+                conn.commit()
+                success_count += cursor.rowcount
+                break
+            except Exception as e:
+                if conn:
+                    conn.rollback()
+                print(f"批次入库失败(start={start}):", e)
+                if '1054, "Unknown column' in str(e):
+                    check_and_sync_columns(df, table_name, database)
+                    break
+                elif attempt == 0:
+                    time.sleep(1)
+                else:
+                    raise Exception(e)
+            finally:
+                if conn:
+                    conn.close()
+    print(f"成功插入/更新 {success_count} 条记录")
 

CPpackage/db/test.py

@@ -4,21 +4,22 @@ import os
            
 if __name__ == "__main__":
     # 配置你的参数
-    TABLE_NAME = "pinlei_hgjk"  # 替换为实际表名
-    DATABASE_NAME = "shengyicanmou_copy"  # 替换为实际数据库名
+    TABLE_NAME = "pinlei_spph"  
+    DATABASE_NAME = "shengyicanmou"  
     # 你的唯一索引字段列表（必须是列表类型！）
-    UNIQUE_FIELDS = ['begindate', 'enddate', 'value_l', 'sellerId', 'statDate', 'date_effect', 'store_id']
+    UNIQUE_FIELDS = ["itemId", "store_id", "begindate", "value_l", "sellerId", "cateLevel2Id", "cateId", "cateLevel1Id", "mainProductId", "itemTag", "date_effect"]
     PORT = 3306  # 可选，默认从配置获取
     
     # 第一步：查询重复记录
     dup_df = find_duplicate_records(TABLE_NAME, DATABASE_NAME, UNIQUE_FIELDS, PORT)
     
-    # 第二步：确认有重复后删除
+    # # 第二步：确认有重复后删除
     if dup_df is not None and not dup_df.empty:
+
         delete_count = delete_duplicate_records(
             table_name=TABLE_NAME,
             database=DATABASE_NAME,
             unique_index_fields=UNIQUE_FIELDS,  # 必须传列表！
             port=PORT,
-            keep_strategy="min_id"
+            keep_strategy="max_id"
         )

{cppackage-0.2.9.dist-info → cppackage-0.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.1
 Name: cppackage
-Version: 0.2.9
+Version: 0.3.1
 Summary: 超品集团自用的Python包
 Home-page: https://github.com/example/CPpackage
 Author: team-数智组
@@ -20,16 +20,6 @@ License-File: LICENSE
 Requires-Dist: pymysql
 Requires-Dist: pandas
 Requires-Dist: numpy
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: license-file
-Dynamic: requires-dist
-Dynamic: requires-python
-Dynamic: summary
 
 # CPpackage
 

cppackage-0.3.1.dist-info/RECORD

@@ -0,0 +1,13 @@
+CPpackage/__init__.py,sha256=KpVaXkIzkNDl-dfSX8Rr7z9XFlipxx1nZ_GzS4Qls4I,173
+CPpackage/core/__init__.py,sha256=92mF0310uQ5ujlgd-LCMs5kZVzfGRdcl67GHIRWhHcA,504
+CPpackage/db/__init__.py,sha256=WhrLqsjq97KKHF7YV-HcZf9pRtHuqKhRjXDNPCBVbjY,107
+CPpackage/db/config.py,sha256=A9HFCXqlrvssHLD1Ys9HpjDckavalFj31X1Np_aDNp4,729
+CPpackage/db/sql_model.py,sha256=QnWEBb-ZqioZf62q-RQPgG8uJqLY9KgcyGW3zFkH4YY,10198
+CPpackage/db/test.py,sha256=dEeoV5FBuWEt5MFujrt3fd2KSQ690zyw6mANgeiVRCg,1015
+CPpackage/utils/__init__.py,sha256=sAespT0aoLr3O0WF27DwHLYF60VZgPA1tiT0JJ8s8zI,264
+cppackage-0.3.1.dist-info/LICENSE,sha256=6Pj597LnvlRtq3Vgjkfp6zgBeBh7wl8jKE-nCFg4vZI,1065
+cppackage-0.3.1.dist-info/METADATA,sha256=UZsPKX3T4zA7FbzYpb6Ppy0UvjzFjgq5zXMmZmFjqnE,2287
+cppackage-0.3.1.dist-info/WHEEL,sha256=hPN0AlP2dZM_3ZJZWP4WooepkmU9wzjGgCLCeFjkHLA,92
+cppackage-0.3.1.dist-info/entry_points.txt,sha256=SAzF2klkNyeNCqXuiNkDirN999l7v8CGCrHR0StD_oo,50
+cppackage-0.3.1.dist-info/top_level.txt,sha256=FQrfWDbJistWQIrHW_Aoxoy_UWSzjOByWVcVS5ig9Tk,10
+cppackage-0.3.1.dist-info/RECORD,,

{cppackage-0.2.9.dist-info → cppackage-0.3.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.10.2)
+Generator: bdist_wheel (0.46.3)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

cppackage-0.2.9.dist-info/RECORD

@@ -1,13 +0,0 @@
-CPpackage/__init__.py,sha256=KpVaXkIzkNDl-dfSX8Rr7z9XFlipxx1nZ_GzS4Qls4I,173
-CPpackage/core/__init__.py,sha256=92mF0310uQ5ujlgd-LCMs5kZVzfGRdcl67GHIRWhHcA,504
-CPpackage/db/__init__.py,sha256=WhrLqsjq97KKHF7YV-HcZf9pRtHuqKhRjXDNPCBVbjY,107
-CPpackage/db/config.py,sha256=A9HFCXqlrvssHLD1Ys9HpjDckavalFj31X1Np_aDNp4,729
-CPpackage/db/sql_model.py,sha256=tF31cF1SADulPGQ0TWcqhKOE393hdL4TMe6Xycv44UA,9942
-CPpackage/db/test.py,sha256=XzPTllM5S4s7M4NHJ_i4h_ZEEalnDTWhGd7d__nUS10,1011
-CPpackage/utils/__init__.py,sha256=sAespT0aoLr3O0WF27DwHLYF60VZgPA1tiT0JJ8s8zI,264
-cppackage-0.2.9.dist-info/licenses/LICENSE,sha256=6Pj597LnvlRtq3Vgjkfp6zgBeBh7wl8jKE-nCFg4vZI,1065
-cppackage-0.2.9.dist-info/METADATA,sha256=TaYeSok8htlPG0vIGSHkffQemPNolIT_koOk5B3mqpw,2516
-cppackage-0.2.9.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-cppackage-0.2.9.dist-info/entry_points.txt,sha256=SAzF2klkNyeNCqXuiNkDirN999l7v8CGCrHR0StD_oo,50
-cppackage-0.2.9.dist-info/top_level.txt,sha256=FQrfWDbJistWQIrHW_Aoxoy_UWSzjOByWVcVS5ig9Tk,10
-cppackage-0.2.9.dist-info/RECORD,,