cppackage 0.2.8__py3-none-any.whl → 0.3.0__py3-none-any.whl

CPpackage/db/sql_model.py

@@ -1,8 +1,6 @@
 import pymysql
 import time
 from pymysql import Error
-
-# 条件导入：支持直接运行和包模式
 try:
     from .config import get_db_config
 except (ImportError, ValueError):
@@ -109,7 +107,6 @@ def check_and_sync_columns(df ,table_name, database, max_new=5):
     db_cols = set(get_table_columns(table_name, database))
     # 新增字段
     new_cols = df_cols - db_cols
-
     # 缺失字段（减少的字段，不处理）
     missing_cols = db_cols - df_cols
     print("新增字段:", new_cols if new_cols else "无")
@@ -126,8 +123,125 @@ def check_and_sync_columns(df ,table_name, database, max_new=5):
     else:
         raise Exception(f"新增字段超过{max_new}个: {new_cols}")
     return True
+# ===================== 通用查询重复记录函数（适配自定义索引） =====================
+def find_duplicate_records(table_name, database, unique_index_fields, port=None):
+    """
+    查询指定表中基于自定义唯一索引字段组合重复的记录
+    参数:
+        table_name: 表名（必选）
+        database: 数据库名（必选）
+        unique_index_fields: 唯一索引字段列表（必选，如 ['begindate', 'enddate']）
+        port: 数据库端口（可选）
+    返回:
+        DataFrame: 重复记录详情；None: 出错；空DataFrame: 无重复
+    """
+    # 核心校验：确保unique_index_fields是可迭代的非空列表
+    if not isinstance(unique_index_fields, list) or len(unique_index_fields) == 0:
+        print("错误：unique_index_fields 必须是非空列表！")
+        return None
+    
+    fields_str = ",".join([f"`{field}`" for field in unique_index_fields])
+    group_by_str = ",".join([f"`{field}`" for field in unique_index_fields])
+    
+    # 核心SQL：查询所有重复记录
+    sql = f"""
+    SELECT t.* 
+    FROM `{table_name}` t
+    INNER JOIN (
+        SELECT {group_by_str}, COUNT(*) AS duplicate_count
+        FROM `{table_name}`
+        GROUP BY {group_by_str}
+        HAVING COUNT(*) > 1
+    ) dup ON {
+        " AND ".join([f"t.`{field}` = dup.`{field}`" for field in unique_index_fields])
+    }
+    ORDER BY {group_by_str}
+    """
+    
+    try:
+        # Lazy import pandas here to avoid heavy imports during module import
+        try:
+            import pandas as pd
+        except Exception as e:
+            print("无法导入 pandas:", e)
+            return None
 
-
+        with _get_connection(database, port) as conn:
+            df_duplicates = pd.read_sql(sql, conn)
+            if df_duplicates.empty:
+                print(f"表 {table_name} 中无重复记录")
+            else:
+                print(f"发现 {len(df_duplicates)} 条重复记录（{len(df_duplicates.drop_duplicates(subset=unique_index_fields))} 组重复组合）")
+                # 可选：保存重复记录到CSV
+                # df_duplicates.to_csv(f"duplicate_records_{table_name}.csv", index=False, encoding="utf-8-sig")
+                # print("重复记录已保存到 duplicate_records_{table_name}.csv")
+            return df_duplicates
+    except pymysql.MySQLError as e:
+        print("数据库查询失败:", e)
+        return None
+    except Exception as e:
+        print("未知错误:", e)
+        return None
+# ===================== 删除重复记录函数（修复参数+调用逻辑） =====================
+def delete_duplicate_records(table_name, database, unique_index_fields, port=None, keep_strategy="max_id"):
+    """
+    删除重复记录，仅保留每组唯一值的一条记录
+    参数:
+        table_name: 表名（必选）
+        database: 数据库名（必选）
+        unique_index_fields: 唯一索引字段列表（必选）
+        port: 数据库端口（可选）
+        keep_strategy: 保留策略（min_id/max_id）
+    返回:
+        int: 删除的记录数；None: 出错；0: 无重复
+    """
+    
+    # 2. 构建删除SQL（核心逻辑）
+    group_by_str = ",".join([f"`{field}`" for field in unique_index_fields])
+    if keep_strategy == "min_id":
+        keep_condition = "MIN(`id`)"
+    elif keep_strategy == "max_id":
+        keep_condition = "MAX(`id`)"
+    else:
+        print(f"不支持的策略：{keep_strategy}，默认使用min_id")
+        keep_condition = "MIN(`id`)"
+    
+    delete_sql = f"""
+    DELETE t FROM `{table_name}` t
+    INNER JOIN (
+        SELECT {group_by_str}, {keep_condition} AS keep_id
+        FROM `{table_name}`
+        GROUP BY {group_by_str}
+        HAVING COUNT(*) > 1
+    ) dup ON {
+        " AND ".join([f"t.`{field}` = dup.`{field}`" for field in unique_index_fields])
+    }
+    WHERE t.`id` != dup.keep_id
+    """
+    
+    try:
+        with _get_connection(database, port) as conn:
+            with conn.cursor() as cursor:
+                # 安全确认
+                print(f"\n⚠️  即将删除表 {table_name} 的重复记录，保留策略：{keep_strategy}")
+                print("确认删除请按任意键，取消请关闭程序...")
+                input()
+                
+                # 执行删除
+                affected_rows = cursor.execute(delete_sql)
+                conn.commit()
+                print(f"✅ 成功删除 {affected_rows} 条重复记录")
+                return affected_rows
+    except pymysql.MySQLError as e:
+        print("❌ 删除失败（数据库错误）:", e)
+        if 'conn' in locals():
+            conn.rollback()
+        return None
+    except Exception as e:
+        print("❌ 删除失败（未知错误）:", e)
+        if 'conn' in locals():
+            conn.rollback()
+        return None
 # ===================== DataFrame 入库 =====================
 
 def update_datas(df, table_name, database):
@@ -135,7 +249,9 @@ def update_datas(df, table_name, database):
     conn = None
     for i in range(2):
         try:
-            # 表结构校验,df为抓取数据表，db_cols为数据库现有字段
+            if df.empty:
+                print("数据为空，无需入库")
+                return
             conn = _get_connection(database)
             cursor = conn.cursor()
             cols = list(df.columns)
@@ -161,11 +277,10 @@ def update_datas(df, table_name, database):
                 conn.rollback()
             print("入库失败:", e)
             if '1054, "Unknown column' in str(e):
+                # 表结构校验,df为抓取数据表
                 check_and_sync_columns(df,table_name,database)
             else:
                 raise Exception(e)
         finally:
             if conn:
                 conn.close()
-
-

CPpackage/db/test.py

@@ -0,0 +1,25 @@
+from CPpackage.db.sql_model import find_duplicate_records,delete_duplicate_records
+import os
+
+           
+if __name__ == "__main__":
+    # 配置你的参数
+    TABLE_NAME = "liuliang_plly"  
+    DATABASE_NAME = "shengyicanmou"  
+    # 你的唯一索引字段列表（必须是列表类型！）
+    UNIQUE_FIELDS = ['begindate', 'value_l', 'pageName', 'itemId', 'name_l1', 'name_l2', 'date_effect', 'store_id']
+    PORT = 3306  # 可选，默认从配置获取
+    
+    # 第一步：查询重复记录
+    # dup_df = find_duplicate_records(TABLE_NAME, DATABASE_NAME, UNIQUE_FIELDS, PORT)
+    
+    # # 第二步：确认有重复后删除
+    # if dup_df is not None and not dup_df.empty:
+
+    delete_count = delete_duplicate_records(
+        table_name=TABLE_NAME,
+        database=DATABASE_NAME,
+        unique_index_fields=UNIQUE_FIELDS,  # 必须传列表！
+        port=PORT,
+        keep_strategy="max_id"
+    )

{cppackage-0.2.8.dist-info → cppackage-0.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.1
 Name: cppackage
-Version: 0.2.8
+Version: 0.3.0
 Summary: 超品集团自用的Python包
 Home-page: https://github.com/example/CPpackage
 Author: team-数智组
@@ -20,16 +20,6 @@ License-File: LICENSE
 Requires-Dist: pymysql
 Requires-Dist: pandas
 Requires-Dist: numpy
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: license-file
-Dynamic: requires-dist
-Dynamic: requires-python
-Dynamic: summary
 
 # CPpackage
 

cppackage-0.3.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+CPpackage/__init__.py,sha256=KpVaXkIzkNDl-dfSX8Rr7z9XFlipxx1nZ_GzS4Qls4I,173
+CPpackage/core/__init__.py,sha256=92mF0310uQ5ujlgd-LCMs5kZVzfGRdcl67GHIRWhHcA,504
+CPpackage/db/__init__.py,sha256=WhrLqsjq97KKHF7YV-HcZf9pRtHuqKhRjXDNPCBVbjY,107
+CPpackage/db/config.py,sha256=A9HFCXqlrvssHLD1Ys9HpjDckavalFj31X1Np_aDNp4,729
+CPpackage/db/sql_model.py,sha256=uoqyNTM7tDVh9rt-2AQ8r6CyXf1DtsbBcjLd9tMq4oY,9948
+CPpackage/db/test.py,sha256=qi6dM_ITxFb87CugaQuUdeZi1tJHSX8euoXiZyX242s,945
+CPpackage/utils/__init__.py,sha256=sAespT0aoLr3O0WF27DwHLYF60VZgPA1tiT0JJ8s8zI,264
+cppackage-0.3.0.dist-info/LICENSE,sha256=6Pj597LnvlRtq3Vgjkfp6zgBeBh7wl8jKE-nCFg4vZI,1065
+cppackage-0.3.0.dist-info/METADATA,sha256=kOyAMCsz-eOUeAo6SgkimCJ1kN8_gzD6-Qg_PEZhFQg,2287
+cppackage-0.3.0.dist-info/WHEEL,sha256=hPN0AlP2dZM_3ZJZWP4WooepkmU9wzjGgCLCeFjkHLA,92
+cppackage-0.3.0.dist-info/entry_points.txt,sha256=SAzF2klkNyeNCqXuiNkDirN999l7v8CGCrHR0StD_oo,50
+cppackage-0.3.0.dist-info/top_level.txt,sha256=FQrfWDbJistWQIrHW_Aoxoy_UWSzjOByWVcVS5ig9Tk,10
+cppackage-0.3.0.dist-info/RECORD,,

{cppackage-0.2.8.dist-info → cppackage-0.3.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.10.2)
+Generator: bdist_wheel (0.46.3)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

cppackage-0.2.8.dist-info/RECORD

@@ -1,12 +0,0 @@
-CPpackage/__init__.py,sha256=KpVaXkIzkNDl-dfSX8Rr7z9XFlipxx1nZ_GzS4Qls4I,173
-CPpackage/core/__init__.py,sha256=92mF0310uQ5ujlgd-LCMs5kZVzfGRdcl67GHIRWhHcA,504
-CPpackage/db/__init__.py,sha256=WhrLqsjq97KKHF7YV-HcZf9pRtHuqKhRjXDNPCBVbjY,107
-CPpackage/db/config.py,sha256=A9HFCXqlrvssHLD1Ys9HpjDckavalFj31X1Np_aDNp4,729
-CPpackage/db/sql_model.py,sha256=81L6WFmwkTJGjwJVRwEDxonU9evwETh_rfCcKkk9Jgc,5096
-CPpackage/utils/__init__.py,sha256=sAespT0aoLr3O0WF27DwHLYF60VZgPA1tiT0JJ8s8zI,264
-cppackage-0.2.8.dist-info/licenses/LICENSE,sha256=6Pj597LnvlRtq3Vgjkfp6zgBeBh7wl8jKE-nCFg4vZI,1065
-cppackage-0.2.8.dist-info/METADATA,sha256=m314EcXCtOKmcIMiWy3gGmGzIHyx77sTBjZIQeH535k,2516
-cppackage-0.2.8.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-cppackage-0.2.8.dist-info/entry_points.txt,sha256=SAzF2klkNyeNCqXuiNkDirN999l7v8CGCrHR0StD_oo,50
-cppackage-0.2.8.dist-info/top_level.txt,sha256=FQrfWDbJistWQIrHW_Aoxoy_UWSzjOByWVcVS5ig9Tk,10
-cppackage-0.2.8.dist-info/RECORD,,