cppackage 0.2.8__tar.gz → 0.2.9__tar.gz

{cppackage-0.2.8 → cppackage-0.2.9}/CPpackage/db/sql_model.py

@@ -1,7 +1,7 @@
 import pymysql
 import time
 from pymysql import Error
-
+import pandas as pd
 # 条件导入：支持直接运行和包模式
 try:
     from .config import get_db_config
@@ -109,7 +109,6 @@ def check_and_sync_columns(df ,table_name, database, max_new=5):
     db_cols = set(get_table_columns(table_name, database))
     # 新增字段
     new_cols = df_cols - db_cols
-
     # 缺失字段（减少的字段，不处理）
     missing_cols = db_cols - df_cols
     print("新增字段:", new_cols if new_cols else "无")
@@ -126,8 +125,122 @@ def check_and_sync_columns(df ,table_name, database, max_new=5):
     else:
         raise Exception(f"新增字段超过{max_new}个: {new_cols}")
     return True
-
-
+# ===================== 通用查询重复记录函数（适配自定义索引） =====================
+def find_duplicate_records(table_name, database, unique_index_fields, port=None):
+    """
+    查询指定表中基于自定义唯一索引字段组合重复的记录
+    参数:
+        table_name: 表名（必选）
+        database: 数据库名（必选）
+        unique_index_fields: 唯一索引字段列表（必选，如 ['begindate', 'enddate']）
+        port: 数据库端口（可选）
+    返回:
+        DataFrame: 重复记录详情；None: 出错；空DataFrame: 无重复
+    """
+    # 核心校验：确保unique_index_fields是可迭代的非空列表
+    if not isinstance(unique_index_fields, list) or len(unique_index_fields) == 0:
+        print("错误：unique_index_fields 必须是非空列表！")
+        return None
+    
+    fields_str = ",".join([f"`{field}`" for field in unique_index_fields])
+    group_by_str = ",".join([f"`{field}`" for field in unique_index_fields])
+    
+    # 核心SQL：查询所有重复记录
+    sql = f"""
+    SELECT t.* 
+    FROM `{table_name}` t
+    INNER JOIN (
+        SELECT {group_by_str}, COUNT(*) AS duplicate_count
+        FROM `{table_name}`
+        GROUP BY {group_by_str}
+        HAVING COUNT(*) > 1
+    ) dup ON {
+        " AND ".join([f"t.`{field}` = dup.`{field}`" for field in unique_index_fields])
+    }
+    ORDER BY {group_by_str}
+    """
+    
+    try:
+        with _get_connection(database, port) as conn:
+            df_duplicates = pd.read_sql(sql, conn)
+            if df_duplicates.empty:
+                print(f"表 {table_name} 中无重复记录")
+            else:
+                print(f"发现 {len(df_duplicates)} 条重复记录（{len(df_duplicates.drop_duplicates(subset=unique_index_fields))} 组重复组合）")
+                # 可选：保存重复记录到CSV
+                df_duplicates.to_csv(f"duplicate_records_{table_name}.csv", index=False, encoding="utf-8-sig")
+                print("重复记录已保存到 duplicate_records_{table_name}.csv")
+            return df_duplicates
+    except pymysql.MySQLError as e:
+        print("数据库查询失败:", e)
+        return None
+    except Exception as e:
+        print("未知错误:", e)
+        return None
+# ===================== 删除重复记录函数（修复参数+调用逻辑） =====================
+def delete_duplicate_records(table_name, database, unique_index_fields, port=None, keep_strategy="min_id"):
+    """
+    删除重复记录，仅保留每组唯一值的一条记录
+    参数:
+        table_name: 表名（必选）
+        database: 数据库名（必选）
+        unique_index_fields: 唯一索引字段列表（必选）
+        port: 数据库端口（可选）
+        keep_strategy: 保留策略（min_id/max_id）
+    返回:
+        int: 删除的记录数；None: 出错；0: 无重复
+    """
+    # 1. 先调用修复后的find_duplicate_records查询重复数据
+    duplicate_df = find_duplicate_records(table_name, database, unique_index_fields, port)
+    if duplicate_df is None or duplicate_df.empty:
+        return 0
+    
+    # 2. 构建删除SQL（核心逻辑）
+    group_by_str = ",".join([f"`{field}`" for field in unique_index_fields])
+    if keep_strategy == "min_id":
+        keep_condition = "MIN(`id`)"
+    elif keep_strategy == "max_id":
+        keep_condition = "MAX(`id`)"
+    else:
+        print(f"不支持的策略：{keep_strategy}，默认使用min_id")
+        keep_condition = "MIN(`id`)"
+    
+    delete_sql = f"""
+    DELETE t FROM `{table_name}` t
+    INNER JOIN (
+        SELECT {group_by_str}, {keep_condition} AS keep_id
+        FROM `{table_name}`
+        GROUP BY {group_by_str}
+        HAVING COUNT(*) > 1
+    ) dup ON {
+        " AND ".join([f"t.`{field}` = dup.`{field}`" for field in unique_index_fields])
+    }
+    WHERE t.`id` != dup.keep_id
+    """
+    
+    try:
+        with _get_connection(database, port) as conn:
+            with conn.cursor() as cursor:
+                # 安全确认
+                print(f"\n⚠️  即将删除表 {table_name} 的重复记录，保留策略：{keep_strategy}")
+                print("确认删除请按任意键，取消请关闭程序...")
+                input()
+                
+                # 执行删除
+                affected_rows = cursor.execute(delete_sql)
+                conn.commit()
+                print(f"✅ 成功删除 {affected_rows} 条重复记录")
+                return affected_rows
+    except pymysql.MySQLError as e:
+        print("❌ 删除失败（数据库错误）:", e)
+        if 'conn' in locals():
+            conn.rollback()
+        return None
+    except Exception as e:
+        print("❌ 删除失败（未知错误）:", e)
+        if 'conn' in locals():
+            conn.rollback()
+        return None
 # ===================== DataFrame 入库 =====================
 
 def update_datas(df, table_name, database):
@@ -135,7 +248,6 @@ def update_datas(df, table_name, database):
     conn = None
     for i in range(2):
         try:
-            # 表结构校验,df为抓取数据表，db_cols为数据库现有字段
             conn = _get_connection(database)
             cursor = conn.cursor()
             cols = list(df.columns)
@@ -150,6 +262,7 @@ def update_datas(df, table_name, database):
             VALUES {values_str}
             ON DUPLICATE KEY UPDATE {update_clause}
             """
+            print(sql)
             data = [tuple(row) for row in df.values]
             flat_data = [v for row in data for v in row]
             cursor.execute(sql, flat_data)
@@ -161,6 +274,7 @@ def update_datas(df, table_name, database):
                 conn.rollback()
             print("入库失败:", e)
             if '1054, "Unknown column' in str(e):
+                # 表结构校验,df为抓取数据表
                 check_and_sync_columns(df,table_name,database)
             else:
                 raise Exception(e)

cppackage-0.2.9/CPpackage/db/test.py

@@ -0,0 +1,24 @@
+from CPpackage.db.sql_model import find_duplicate_records,delete_duplicate_records
+import os
+
+           
+if __name__ == "__main__":
+    # 配置你的参数
+    TABLE_NAME = "pinlei_hgjk"  # 替换为实际表名
+    DATABASE_NAME = "shengyicanmou_copy"  # 替换为实际数据库名
+    # 你的唯一索引字段列表（必须是列表类型！）
+    UNIQUE_FIELDS = ['begindate', 'enddate', 'value_l', 'sellerId', 'statDate', 'date_effect', 'store_id']
+    PORT = 3306  # 可选，默认从配置获取
+    
+    # 第一步：查询重复记录
+    dup_df = find_duplicate_records(TABLE_NAME, DATABASE_NAME, UNIQUE_FIELDS, PORT)
+    
+    # 第二步：确认有重复后删除
+    if dup_df is not None and not dup_df.empty:
+        delete_count = delete_duplicate_records(
+            table_name=TABLE_NAME,
+            database=DATABASE_NAME,
+            unique_index_fields=UNIQUE_FIELDS,  # 必须传列表！
+            port=PORT,
+            keep_strategy="min_id"
+        )

{cppackage-0.2.8 → cppackage-0.2.9}/CPpackage.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cppackage
-Version: 0.2.8
+Version: 0.2.9
 Summary: 超品集团自用的Python包
 Home-page: https://github.com/example/CPpackage
 Author: team-数智组

{cppackage-0.2.8 → cppackage-0.2.9}/CPpackage.egg-info/SOURCES.txt

@@ -14,6 +14,7 @@ CPpackage/core/__init__.py
 CPpackage/db/__init__.py
 CPpackage/db/config.py
 CPpackage/db/sql_model.py
+CPpackage/db/test.py
 CPpackage/utils/__init__.py
 cppackage.egg-info/PKG-INFO
 cppackage.egg-info/SOURCES.txt

{cppackage-0.2.8 → cppackage-0.2.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cppackage
-Version: 0.2.8
+Version: 0.2.9
 Summary: 超品集团自用的Python包
 Home-page: https://github.com/example/CPpackage
 Author: team-数智组

{cppackage-0.2.8 → cppackage-0.2.9}/setup.py

@@ -10,7 +10,7 @@ with open('readme.md', 'r', encoding='utf-8') as f:
 # 包的基本信息
 setup(
     name='cppackage',
-    version='0.2.8',
+    version='0.2.9',
     description='超品集团自用的Python包',
     long_description=long_description,
     long_description_content_type='text/markdown',