cppackage 0.2.2__tar.gz → 0.2.4__tar.gz

{cppackage-0.2.2 → cppackage-0.2.4}/CPpackage/db/config.py

@@ -5,7 +5,7 @@ _db_config = {
     'user': os.getenv('CPPACKAGE_DB_USER'),
     'password': os.getenv('CPPACKAGE_DB_PASSWORD'),
     'database': os.getenv('CPPACKAGE_DB_NAME'),
-    'port': int(os.getenv('CPPACKAGE_DB_PORT')),
+    'port': int(os.getenv('CPPACKAGE_DB_PORT'), 3306),
 }
 
 def set_db_config(host=None, user=None, password=None, database=None, port=None):

{cppackage-0.2.2 → cppackage-0.2.4}/CPpackage/db/sql_model.py

@@ -101,12 +101,10 @@ def add_columns(table_name, database, columns):
             conn.commit()
 
  # 表结构校验
-def check_and_sync_columns(df, table_name, database, max_new=5):
+def check_and_sync_columns(df ,table_name, database, max_new=5):
 
     df_cols = set(df.columns)
-
-    db_cols = set(get_table_columns(table_name, database))
-
+    db_cols= get_table_columns(table_name, database)
     new_cols = df_cols - db_cols
     missing_cols = db_cols - df_cols
     print("新增字段:", new_cols if new_cols else "无")
@@ -121,13 +119,6 @@ def check_and_sync_columns(df, table_name, database, max_new=5):
 
         print("开始同步字段...")
         add_columns(table_name, database, new_cols)
-
-        # 二次校验
-        new_db_cols = set(get_table_columns(table_name, database))
-
-        if not df_cols.issubset(new_db_cols):
-            raise Exception("字段同步失败")
-
         print("字段同步成功")
 
     return True
@@ -138,34 +129,37 @@ def check_and_sync_columns(df, table_name, database, max_new=5):
 def update_datas(df, table_name, database):
 
     conn = None
-
-    try:
-        # 表结构校验
-        conn = _get_connection(database)
-        cursor = conn.cursor()
-        cols = list(df.columns)
-        col_str = ",".join([f"`{c}`" for c in cols])
-        value_tpl = "(" + ",".join(["%s"] * len(cols)) + ")"
-        values_str = ",".join([value_tpl] * len(df))
-        update_clause = ",".join(
-            [f"`{c}`=VALUES(`{c}`)" for c in cols if c != "id"]
-        )
-        sql = f"""
-        INSERT INTO `{table_name}` ({col_str})
-        VALUES {values_str}
-        ON DUPLICATE KEY UPDATE {update_clause}
-        """
-        data = [tuple(row) for row in df.values]
-        flat_data = [v for row in data for v in row]
-        cursor.execute(sql, flat_data)
-        conn.commit()
-        print(f"成功插入/更新 {cursor.rowcount} 条记录")
-    except Exception as e:
-        if conn:
-            conn.rollback()
-        print("入库失败:", e)
-    finally:
-        if conn:
-            conn.close()
+    for i in range(2):
+        try:
+            # 表结构校验,df为抓取数据表，db_cols为数据库现有字段
+            conn = _get_connection(database)
+            cursor = conn.cursor()
+            cols = list(df.columns)
+            col_str = ",".join([f"`{c}`" for c in cols])
+            value_tpl = "(" + ",".join(["%s"] * len(cols)) + ")"
+            values_str = ",".join([value_tpl] * len(df))
+            update_clause = ",".join(
+                [f"`{c}`=VALUES(`{c}`)" for c in cols if c != "id"]
+            )
+            sql = f"""
+            INSERT INTO `{table_name}` ({col_str})
+            VALUES {values_str}
+            ON DUPLICATE KEY UPDATE {update_clause}
+            """
+            data = [tuple(row) for row in df.values]
+            flat_data = [v for row in data for v in row]
+            cursor.execute(sql, flat_data)
+            conn.commit()
+            print(f"成功插入/更新 {cursor.rowcount} 条记录")
+            break
+        except Exception as e:
+            if conn:
+                conn.rollback()
+            print("入库失败:", e)
+            if '1054, "Unknown column' in str(e):
+                check_and_sync_columns(df,table_name,database)
+        finally:
+            if conn:
+                conn.close()
 
 

{cppackage-0.2.2 → cppackage-0.2.4}/CPpackage.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cppackage
-Version: 0.2.2
+Version: 0.2.4
 Summary: 超品集团自用的Python包
 Home-page: https://github.com/example/CPpackage
 Author: team-数智组
@@ -50,6 +50,8 @@ pip install cppackage
 from CPpackage.db.config import set_db_config
 from CPpackage.db.sql_model import sel_data, del_data , incremental_update, get_table_columns,add_columns,check_and_sync_columns,update_datas
 
+# 
+
 # 配置数据库连接（也可通过环境变量CPPACKAGE_DB_*配置）
 set_db_config(host='your_host', user='your_user', password='your_password', database='your_database', port=3306)
 

{cppackage-0.2.2 → cppackage-0.2.4}/CPpackage.egg-info/SOURCES.txt

@@ -14,7 +14,6 @@ CPpackage/core/__init__.py
 CPpackage/db/__init__.py
 CPpackage/db/config.py
 CPpackage/db/sql_model.py
-CPpackage/db/sql_modelcopy.py
 CPpackage/utils/__init__.py
 cppackage.egg-info/PKG-INFO
 cppackage.egg-info/SOURCES.txt

{cppackage-0.2.2 → cppackage-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cppackage
-Version: 0.2.2
+Version: 0.2.4
 Summary: 超品集团自用的Python包
 Home-page: https://github.com/example/CPpackage
 Author: team-数智组
@@ -50,6 +50,8 @@ pip install cppackage
 from CPpackage.db.config import set_db_config
 from CPpackage.db.sql_model import sel_data, del_data , incremental_update, get_table_columns,add_columns,check_and_sync_columns,update_datas
 
+# 
+
 # 配置数据库连接（也可通过环境变量CPPACKAGE_DB_*配置）
 set_db_config(host='your_host', user='your_user', password='your_password', database='your_database', port=3306)
 

{cppackage-0.2.2 → cppackage-0.2.4}/readme.md

@@ -17,6 +17,8 @@ pip install cppackage
 from CPpackage.db.config import set_db_config
 from CPpackage.db.sql_model import sel_data, del_data , incremental_update, get_table_columns,add_columns,check_and_sync_columns,update_datas
 
+# 
+
 # 配置数据库连接（也可通过环境变量CPPACKAGE_DB_*配置）
 set_db_config(host='your_host', user='your_user', password='your_password', database='your_database', port=3306)
 

{cppackage-0.2.2 → cppackage-0.2.4}/setup.py

@@ -10,7 +10,7 @@ with open('readme.md', 'r', encoding='utf-8') as f:
 # 包的基本信息
 setup(
     name='cppackage',
-    version='0.2.2',
+    version='0.2.4',
     description='超品集团自用的Python包',
     long_description=long_description,
     long_description_content_type='text/markdown',

cppackage-0.2.2/CPpackage/db/sql_modelcopy.py

@@ -1,105 +0,0 @@
-import pymysql
-import time
-from pymysql import Error
-from .config import get_db_config
-
-def _get_connection(database=None, port=None):
-    cfg = get_db_config()
-    db = database if database is not None else cfg.get('database')
-    prt = port if port is not None else cfg.get('port', 3306)
-    return pymysql.connect(
-        host=cfg.get('host'),
-        user=cfg.get('user'),
-        password=cfg.get('password'),
-        database=db,
-        port=prt,
-    )
-
-def sel_data(sql, port=None, database=None):
-    try:
-        conn = _get_connection(database=database, port=port)
-        cursor = conn.cursor()
-        cursor.execute(sql)
-        result = cursor.fetchall()
-        cursor.close()
-        conn.close()
-        return result
-    except pymysql.MySQLError as e:
-        print(f"An error occurred: {e}")
-
-def update_data(sql, values, port=None, database=None):
-    conn = _get_connection(database=database, port=port)
-    cursor = conn.cursor()
-    cursor.execute(sql, values)
-    conn.commit()
-    cursor.close()
-    conn.close()
-
-def updata(data, databases, table):
-    s = ""
-    st1 = ""
-    for i in data.columns.values:
-        s = s + "`" + i + "`,"
-        st1 = st1 + "%s,"
-    s = s[:-1]
-    st1 = st1[:-1]
-    all_data = []
-    for _, k in data.iterrows():
-        all_data.append(tuple(k.to_list()))
-    sql = 'INSERT INTO %s (%s) VALUES (%s)' % (table, s, st1)
-    update_datas(sql, tuple(all_data), database=databases)
-    time.sleep(2)
-
-def deldata(all_del_list, id_name, table_name, database):
-    str11 = ""
-    for te in all_del_list:
-        if "str" in str(type(te)):
-            str11 = str11 + "'" + str(te) + "',"
-        else:
-            str11 = str11 + str(te) + ","
-    str11 = str11[:-1]
-    delsql = 'DELETE from `%s` where %s in (%s)' % (table_name, id_name, str11)
-    del_data(delsql, database=database)
-
-def deldata_time(id_name, table_name, id, database, time_name, start_time, end_time):
-    delsql = 'DELETE from `%s` where %s = %s and `%s`>= "%s" and  `%s`<= "%s"' % (
-        table_name, id_name, id, time_name, start_time, time_name, end_time
-    )
-    del_data(delsql, database=database)
-
-def update_datas(sql, values, port=None, database=None):
-    with _get_connection(database=database, port=port) as conn:
-        with conn.cursor() as cursor:
-            cursor.executemany(sql, values)
-            conn.commit()
-
-def del_data(sql, port=None, database=None):
-    try:
-        conn = _get_connection(database=database, port=port)
-        cursor = conn.cursor()
-        cursor.execute(sql)
-        conn.commit()
-        cursor.close()
-        conn.close()
-    except pymysql.MySQLError as e:
-        print(f"An error occurred: {e}")
-
-def update_test(df, table_name, databases):
-    try:
-        conn = _get_connection(database=databases)
-        cursor = conn.cursor()
-        columns = ', '.join(df.columns)
-        values_template = ', '.join([f'({", ".join(["%s"] * len(df.columns))})'] * len(df))
-        update_clause = ', '.join([f'{col} = VALUES({col})' for col in df.columns if col != 'id'])
-        sql = f"INSERT INTO {table_name} ({columns}) VALUES {values_template} " \
-                  f"ON DUPLICATE KEY UPDATE {update_clause}"
-        values = [tuple(row) for row in df.values]
-        flat_values = [item for sublist in values for item in sublist]
-        cursor.execute(sql, flat_values)
-        conn.commit()
-        print(f"成功插入/更新 {cursor.rowcount} 条记录")
-    except Error as e:
-        print(f"数据库错误: {e}")
-    finally:
-        if conn:
-            conn.close()