cppackage 0.3.0__py3-none-any.whl → 0.3.2__py3-none-any.whl

CPpackage/db/sql_model.py

@@ -1,6 +1,5 @@
 import pymysql
 import time
-from pymysql import Error
 try:
     from .config import get_db_config
 except (ImportError, ValueError):
@@ -13,12 +12,8 @@ except (ImportError, ValueError):
 
 # ===================== 数据库连接 =====================
 
-def _get_connection(database=None, port=None):
-    cfg = get_db_config()
-
-    db = database if database else cfg.get('database')
-    prt = port if port else cfg.get('port', 3306)
-
+def _create_new_connection(cfg, db, prt):
+    """创建新的数据库连接"""
     return pymysql.connect(
         host=cfg.get('host'),
         user=cfg.get('user'),
@@ -30,6 +25,33 @@ def _get_connection(database=None, port=None):
     )
 
 
+def _get_connection(database=None, port=None):
+    """
+    获取数据库连接，包含健康检查机制
+    - 使用 ping(reconnect=True) 检测连接有效性
+    - 如果连接失效，自动重新创建连接
+    """
+    cfg = get_db_config()
+
+    db = database if database else cfg.get('database')
+    prt = port if port else cfg.get('port', 3306)
+
+    conn = _create_new_connection(cfg, db, prt)
+
+    # 连接健康检查：发送 ping 确保连接有效
+    try:
+        conn.ping(reconnect=False)
+    except Exception:
+        # 如果 ping 失败，重新创建连接
+        try:
+            conn.close()
+        except Exception:
+            pass
+        conn = _create_new_connection(cfg, db, prt)
+
+    return conn
+
+
 # ===================== 查询操作 =====================
 
 def sel_data(sql, params=None, port=None, database=None):
@@ -139,8 +161,7 @@ def find_duplicate_records(table_name, database, unique_index_fields, port=None)
     if not isinstance(unique_index_fields, list) or len(unique_index_fields) == 0:
         print("错误：unique_index_fields 必须是非空列表！")
         return None
-    
-    fields_str = ",".join([f"`{field}`" for field in unique_index_fields])
+
     group_by_str = ",".join([f"`{field}`" for field in unique_index_fields])
     
     # 核心SQL：查询所有重复记录
@@ -244,43 +265,110 @@ def delete_duplicate_records(table_name, database, unique_index_fields, port=Non
         return None
 # ===================== DataFrame 入库 =====================
 
-def update_datas(df, table_name, database):
+def _is_connection_error(error_msg):
+    """
+    判断是否是数据库连接相关错误
+    返回 True 表示是连接错误，可以通过重试解决
+    """
+    connection_error_codes = [
+        '(0, \'\')',     # pymysql InterfaceError: 连接已关闭
+        '2003',          # Can't connect to MySQL server
+        '2006',          # MySQL server has gone away
+        '2013',          # Lost connection to MySQL server during query
+        '2014',          # Commands out of sync
+        '1045',          # Access denied
+        '2055',          # Lost connection to MySQL server at 'xxx'
+    ]
+    return any(code in error_msg for code in connection_error_codes)
+
+
+def update_datas(df, table_name, database, batch_size=20):
+    """
+    将 DataFrame 数据批量插入/更新到数据库
+    包含完善的重试机制：
+    - 连接错误：最多重试3次，指数退避
+    - 字段缺失错误：自动同步字段后重试
+    - 其他错误：立即抛出
+    """
+    if df.empty:
+        print("数据为空，无需入库")
+        return
+
+    cols = list(df.columns)
+    col_str = ",".join([f"`{c}`" for c in cols])
+    value_tpl = "(" + ",".join(["%s"] * len(cols)) + ")"
+    update_clause = ",".join(
+        [f"`{c}`=VALUES(`{c}`)" for c in cols if c != "id"]
+    )
+
+    total = len(df)
+    success_count = 0
+    max_retries = 3  # 最大重试次数
+
+    for start in range(0, total, batch_size):
+        batch_df = df.iloc[start:start + batch_size]
+        values_str = ",".join([value_tpl] * len(batch_df))
+        sql = f"""
+        INSERT INTO `{table_name}` ({col_str})
+        VALUES {values_str}
+        ON DUPLICATE KEY UPDATE {update_clause}
+        """
+        data = [tuple(row) for row in batch_df.values]
+        flat_data = [v for row in data for v in row]
+
+        for attempt in range(max_retries):
+            conn = None
+            try:
+                conn = _get_connection(database)
+                # 连接健康检查
+                conn.ping(reconnect=False)
+
+                cursor = conn.cursor()
+                cursor.execute(sql, flat_data)
+                conn.commit()
+                success_count += cursor.rowcount
+                break  # 成功，跳出重试循环
+
+            except Exception as e:
+                error_str = str(e)
+                if conn:
+                    try:
+                        conn.rollback()
+                    except Exception:
+                        pass
+
+                print(f"批次入库失败(start={start}, attempt={attempt+1}/{max_retries}): {e}")
+
+                # 判断是否是连接相关错误
+                if _is_connection_error(error_str):
+                    if attempt < max_retries - 1:
+                        # 指数退避：2秒、4秒、8秒
+                        wait_time = 2 * (2 ** attempt)
+                        print(f"连接错误，等待{wait_time}秒后重试...")
+                        time.sleep(wait_time)
+                        continue
+                    else:
+                        # 最后一次重试仍然失败，抛出异常
+                        raise Exception(f"数据库连接错误，已重试{max_retries}次: {e}")
+
+                elif '1054, "Unknown column' in error_str:
+                    # 字段缺失错误，同步字段后重试
+                    check_and_sync_columns(df, table_name, database)
+                    # 字段同步成功后，继续当前批次的重试
+                    if attempt < max_retries - 1:
+                        continue
+                    else:
+                        raise Exception(f"字段同步后仍然失败: {e}")
+                else:
+                    # 其他非连接错误，直接抛出
+                    raise
+
+            finally:
+                if conn:
+                    try:
+                        conn.close()
+                    except Exception:
+                        pass
+
+    print(f"成功插入/更新 {success_count} 条记录")
 
-    conn = None
-    for i in range(2):
-        try:
-            if df.empty:
-                print("数据为空，无需入库")
-                return
-            conn = _get_connection(database)
-            cursor = conn.cursor()
-            cols = list(df.columns)
-            col_str = ",".join([f"`{c}`" for c in cols])
-            value_tpl = "(" + ",".join(["%s"] * len(cols)) + ")"
-            values_str = ",".join([value_tpl] * len(df))
-            update_clause = ",".join(
-                [f"`{c}`=VALUES(`{c}`)" for c in cols if c != "id"]
-            )
-            sql = f"""
-            INSERT INTO `{table_name}` ({col_str})
-            VALUES {values_str}
-            ON DUPLICATE KEY UPDATE {update_clause}
-            """
-            data = [tuple(row) for row in df.values]
-            flat_data = [v for row in data for v in row]
-            cursor.execute(sql, flat_data)
-            conn.commit()
-            print(f"成功插入/更新 {cursor.rowcount} 条记录")
-            break
-        except Exception as e:
-            if conn:
-                conn.rollback()
-            print("入库失败:", e)
-            if '1054, "Unknown column' in str(e):
-                # 表结构校验,df为抓取数据表
-                check_and_sync_columns(df,table_name,database)
-            else:
-                raise Exception(e)
-        finally:
-            if conn:
-                conn.close()

CPpackage/db/test.py

@@ -4,22 +4,22 @@ import os
            
 if __name__ == "__main__":
     # 配置你的参数
-    TABLE_NAME = "liuliang_plly"  
+    TABLE_NAME = "pinlei_sp360_xsfx"  
     DATABASE_NAME = "shengyicanmou"  
     # 你的唯一索引字段列表（必须是列表类型！）
-    UNIQUE_FIELDS = ['begindate', 'value_l', 'pageName', 'itemId', 'name_l1', 'name_l2', 'date_effect', 'store_id']
+    UNIQUE_FIELDS = ["begindate", "value_l", "itemId", "store_id", "date_effect"]
     PORT = 3306  # 可选，默认从配置获取
     
     # 第一步：查询重复记录
-    # dup_df = find_duplicate_records(TABLE_NAME, DATABASE_NAME, UNIQUE_FIELDS, PORT)
+    dup_df = find_duplicate_records(TABLE_NAME, DATABASE_NAME, UNIQUE_FIELDS, PORT)
     
     # # 第二步：确认有重复后删除
-    # if dup_df is not None and not dup_df.empty:
+    if dup_df is not None and not dup_df.empty:
 
-    delete_count = delete_duplicate_records(
-        table_name=TABLE_NAME,
-        database=DATABASE_NAME,
-        unique_index_fields=UNIQUE_FIELDS,  # 必须传列表！
-        port=PORT,
-        keep_strategy="max_id"
-    )
+        delete_count = delete_duplicate_records(
+            table_name=TABLE_NAME,
+            database=DATABASE_NAME,
+            unique_index_fields=UNIQUE_FIELDS,  # 必须传列表！
+            port=PORT,
+            keep_strategy="max_id"
+        )

{cppackage-0.3.0.dist-info → cppackage-0.3.2.dist-info}/METADATA

@@ -1,10 +1,12 @@
 Metadata-Version: 2.1
 Name: cppackage
-Version: 0.3.0
+Version: 0.3.2
 Summary: 超品集团自用的Python包
 Home-page: https://github.com/example/CPpackage
 Author: team-数智组
 Author-email: m110135@163.com
+License: UNKNOWN
+Platform: UNKNOWN
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
@@ -16,7 +18,6 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-License-File: LICENSE
 Requires-Dist: pymysql
 Requires-Dist: pandas
 Requires-Dist: numpy
@@ -70,3 +71,5 @@ CPpackage/
 ## 许可证
 
 本项目采用MIT许可证。详情请参阅LICENSE文件。
+
+

cppackage-0.3.2.dist-info/RECORD

@@ -0,0 +1,13 @@
+CPpackage/__init__.py,sha256=KpVaXkIzkNDl-dfSX8Rr7z9XFlipxx1nZ_GzS4Qls4I,173
+CPpackage/core/__init__.py,sha256=92mF0310uQ5ujlgd-LCMs5kZVzfGRdcl67GHIRWhHcA,504
+CPpackage/db/__init__.py,sha256=WhrLqsjq97KKHF7YV-HcZf9pRtHuqKhRjXDNPCBVbjY,107
+CPpackage/db/config.py,sha256=A9HFCXqlrvssHLD1Ys9HpjDckavalFj31X1Np_aDNp4,729
+CPpackage/db/sql_model.py,sha256=suwQMXf_hpfYfZR07SadIa2E_y_mbEKKy-6BuZCwWE4,13002
+CPpackage/db/test.py,sha256=1EfTt53H9HLIGK6U2SAF6lIDgz2LItJ9tNHpxojz9ro,939
+CPpackage/utils/__init__.py,sha256=sAespT0aoLr3O0WF27DwHLYF60VZgPA1tiT0JJ8s8zI,264
+cppackage-0.3.2.dist-info/LICENSE,sha256=6Pj597LnvlRtq3Vgjkfp6zgBeBh7wl8jKE-nCFg4vZI,1065
+cppackage-0.3.2.dist-info/METADATA,sha256=KFc2Lr9sQj2vjZaiVRAHKd65oMROmyAUBxSbiQDv5Ug,2305
+cppackage-0.3.2.dist-info/WHEEL,sha256=tZoeGjtWxWRfdplE7E3d45VPlLNQnvbKiYnx7gwAy8A,92
+cppackage-0.3.2.dist-info/entry_points.txt,sha256=6MmsHuaQY5w0GxhHAx3lQfHTaocmueEf1OGsoM-hMrQ,51
+cppackage-0.3.2.dist-info/top_level.txt,sha256=FQrfWDbJistWQIrHW_Aoxoy_UWSzjOByWVcVS5ig9Tk,10
+cppackage-0.3.2.dist-info/RECORD,,

{cppackage-0.3.0.dist-info → cppackage-0.3.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.46.3)
+Generator: bdist_wheel (0.45.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{cppackage-0.3.0.dist-info → cppackage-0.3.2.dist-info}/entry_points.txt

@@ -1,2 +1,3 @@
 [console_scripts]
 cppackage = CPpackage.core:main
+

cppackage-0.3.0.dist-info/RECORD

@@ -1,13 +0,0 @@
-CPpackage/__init__.py,sha256=KpVaXkIzkNDl-dfSX8Rr7z9XFlipxx1nZ_GzS4Qls4I,173
-CPpackage/core/__init__.py,sha256=92mF0310uQ5ujlgd-LCMs5kZVzfGRdcl67GHIRWhHcA,504
-CPpackage/db/__init__.py,sha256=WhrLqsjq97KKHF7YV-HcZf9pRtHuqKhRjXDNPCBVbjY,107
-CPpackage/db/config.py,sha256=A9HFCXqlrvssHLD1Ys9HpjDckavalFj31X1Np_aDNp4,729
-CPpackage/db/sql_model.py,sha256=uoqyNTM7tDVh9rt-2AQ8r6CyXf1DtsbBcjLd9tMq4oY,9948
-CPpackage/db/test.py,sha256=qi6dM_ITxFb87CugaQuUdeZi1tJHSX8euoXiZyX242s,945
-CPpackage/utils/__init__.py,sha256=sAespT0aoLr3O0WF27DwHLYF60VZgPA1tiT0JJ8s8zI,264
-cppackage-0.3.0.dist-info/LICENSE,sha256=6Pj597LnvlRtq3Vgjkfp6zgBeBh7wl8jKE-nCFg4vZI,1065
-cppackage-0.3.0.dist-info/METADATA,sha256=kOyAMCsz-eOUeAo6SgkimCJ1kN8_gzD6-Qg_PEZhFQg,2287
-cppackage-0.3.0.dist-info/WHEEL,sha256=hPN0AlP2dZM_3ZJZWP4WooepkmU9wzjGgCLCeFjkHLA,92
-cppackage-0.3.0.dist-info/entry_points.txt,sha256=SAzF2klkNyeNCqXuiNkDirN999l7v8CGCrHR0StD_oo,50
-cppackage-0.3.0.dist-info/top_level.txt,sha256=FQrfWDbJistWQIrHW_Aoxoy_UWSzjOByWVcVS5ig9Tk,10
-cppackage-0.3.0.dist-info/RECORD,,