mdbq 4.0.57__py3-none-any.whl → 4.0.59__py3-none-any.whl

mdbq/__version__.py

@@ -1 +1 @@
-VERSION = '4.0.57'
+VERSION = '4.0.59'

mdbq/mysql/uploader.py

@@ -341,9 +341,17 @@ class MySQLUploader:
         # 始终做特殊字符清理
         cleaned = re.sub(r'[^\w\u4e00-\u9fff$]', '_', identifier)
         cleaned = re.sub(r'_+', '_', cleaned).strip('_')
+        # 如果清理后为空字符串，使用默认标识符
         if not cleaned:
-            logger.error('无法清理异常标识符', {'原始标识符': identifier})
-            raise ValueError(f"无法清理异常标识符: `{identifier}`")
+            logger.warning('标识符清理后为空，使用默认标识符', {'原始标识符': identifier})
+            # 使用原始标识符的哈希值作为后缀，保持可追溯性
+            import hashlib
+            hash_suffix = hashlib.md5(identifier.encode('utf-8')).hexdigest()[:8]
+            cleaned = f'unknown_col_{hash_suffix}'
+        
+        # 确保标识符不以数字开头（MySQL要求）
+        if cleaned and cleaned[0].isdigit():
+            cleaned = f'col_{cleaned}'
         mysql_keywords = {
             'select', 'insert', 'update', 'delete', 'from', 'where', 'and', 'or',
             'not', 'like', 'in', 'is', 'null', 'true', 'false', 'between'
@@ -448,9 +456,33 @@ class MySQLUploader:
             
         # 处理主键
         if primary_keys and len(primary_keys) > 0:
-            # 如果指定了主键，直接使用指定的主键
-            safe_primary_keys = [_index_col_sql(pk) for pk in primary_keys]
-            primary_key_sql = f"PRIMARY KEY ({','.join(safe_primary_keys)})"
+            # 验证主键列是否存在于set_typ中
+            valid_primary_keys = []
+            for pk in primary_keys:
+                normalized_pk = self._normalize_col(pk)
+                if normalized_pk in set_typ:
+                    valid_primary_keys.append(pk)
+                else:
+                    logger.warning('主键列不存在于表结构中，跳过', {
+                        '库': db_name,
+                        '表': table_name,
+                        '列': pk,
+                        '规范化后': normalized_pk,
+                        '可用列': list(set_typ.keys())
+                    })
+            
+            if valid_primary_keys:
+                # 如果指定了主键，直接使用指定的主键
+                safe_primary_keys = [_index_col_sql(pk) for pk in valid_primary_keys]
+                primary_key_sql = f"PRIMARY KEY ({','.join(safe_primary_keys)})"
+            else:
+                # 如果没有有效的主键，使用id作为主键
+                logger.warning('所有主键列都不存在于表结构中，使用默认id主键', {
+                    '库': db_name,
+                    '表': table_name,
+                    '原始主键': primary_keys
+                })
+                primary_key_sql = f"PRIMARY KEY (`id`)"
         else:
             # 如果没有指定主键，使用id作为主键
             primary_key_sql = f"PRIMARY KEY (`id`)"
@@ -462,9 +494,18 @@ class MySQLUploader:
             index_defs.append(f"INDEX `idx_{self._normalize_col(date_column)}` ({safe_date_col})")
         if indexes:
             for idx_col in indexes:
-                if idx_col in set_typ:
+                normalized_idx_col = self._normalize_col(idx_col)
+                if normalized_idx_col in set_typ:
                     safe_idx_col = _index_col_sql(idx_col)
-                    index_defs.append(f"INDEX `idx_{self._normalize_col(idx_col)}` ({safe_idx_col})")
+                    index_defs.append(f"INDEX `idx_{normalized_idx_col}` ({safe_idx_col})")
+                else:
+                    logger.warning('索引列不存在于表结构中，跳过', {
+                        '库': db_name,
+                        '表': table_name,
+                        '列': idx_col,
+                        '规范化后': normalized_idx_col,
+                        '可用列': list(set_typ.keys())
+                    })
                     
         # UNIQUE KEY定义
         unique_defs = []
@@ -483,12 +524,48 @@ class MySQLUploader:
                             '主键': primary_keys
                         })
                         continue
-                safe_unique_cols = [_index_col_sql(col) for col in unique_cols]
-                unique_name = f"uniq_{'_'.join([self._normalize_col(c) for c in unique_cols])}"
+                
+                # 验证唯一约束的列是否存在于set_typ中
+                valid_unique_cols = []
+                for col in unique_cols:
+                    normalized_col = self._normalize_col(col)
+                    if normalized_col in set_typ:
+                        valid_unique_cols.append(col)
+                    else:
+                        logger.warning('唯一约束列不存在于表结构中，跳过', {
+                            '库': db_name,
+                            '表': table_name,
+                            '列': col,
+                            '规范化后': normalized_col,
+                            '可用列': list(set_typ.keys())
+                        })
+                
+                if not valid_unique_cols:
+                    logger.warning('唯一约束的所有列都不存在于表结构中，跳过整个约束', {
+                        '库': db_name,
+                        '表': table_name,
+                        '原始约束': unique_cols
+                    })
+                    continue
+                
+                safe_unique_cols = [_index_col_sql(col) for col in valid_unique_cols]
+                unique_name = f"uniq_{'_'.join([self._normalize_col(c) for c in valid_unique_cols])}"
                 unique_defs.append(f"UNIQUE KEY `{unique_name}` ({','.join(safe_unique_cols)})")
                 
         index_defs = list(set(index_defs))
         all_defs = column_defs + [primary_key_sql] + index_defs + unique_defs
+        
+        # 添加调试日志
+        logger.debug('建表SQL生成', {
+            '库': db_name,
+            '表': table_name,
+            '列定义': column_defs,
+            '主键': primary_key_sql,
+            '索引': index_defs,
+            '唯一约束': unique_defs,
+            'set_typ键': list(set_typ.keys())
+        })
+        
         sql = f"""
         CREATE TABLE IF NOT EXISTS `{db_name}`.`{table_name}` (
             {','.join(all_defs)}

{mdbq-4.0.57.dist-info → mdbq-4.0.59.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: mdbq
-Version: 4.0.57
+Version: 4.0.59
 Home-page: https://pypi.org/project/mdbq
 Author: xigua, 
 Author-email: 2587125111@qq.com

{mdbq-4.0.57.dist-info → mdbq-4.0.59.dist-info}/RECORD

@@ -1,5 +1,5 @@
 mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
-mdbq/__version__.py,sha256=h4Mph1EXalN5pS5TeCj37jNZiUp31v2ZO-LykNJBpY0,18
+mdbq/__version__.py,sha256=621mRjqeg4oOiqnQ029WztdhGqT609a4KpKuDnjD0zY,18
 mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
 mdbq/log/mylogger.py,sha256=kPe3wsQNaB1slfX-Z7VMqzZoMoqPfc7ylYXZDBeFzzI,24945
 mdbq/myconf/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
@@ -9,7 +9,7 @@ mdbq/mysql/deduplicator.py,sha256=AB3gL7ZwhcmzGHSu4UY4M6YZVPFZ2wlAN3BCcwAhegQ,73
 mdbq/mysql/mysql.py,sha256=pDg771xBugCMSTWeskIFTi3pFLgaqgyG3smzf-86Wn8,56772
 mdbq/mysql/s_query.py,sha256=rSlAIUNfTrI25vYjoq9OpEbzsbrzc0m8mxb_0G7h878,45496
 mdbq/mysql/unique_.py,sha256=MaztT-WIyEQUs-OOYY4pFulgHVcXR1BfCy3QUz0XM_U,21127
-mdbq/mysql/uploader.py,sha256=dJRAtj-IebPgOHvKKeu-CqoeqfSaRqusSBgt_hXgNdw,88926
+mdbq/mysql/uploader.py,sha256=V7XPuYncg3Fvj6YhIiqwD6c4s3tLCKBYHwySYm69Av4,92334
 mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
 mdbq/other/download_sku_picture.py,sha256=X66sVdvVgzoNzmgVJyPtd7bjEvctEKtLPblEPF65EWc,46940
 mdbq/other/error_handler.py,sha256=4p5haAXSY-P78stp4Xwo_MwAngWYqyKj5ogWIuYXMeY,12631
@@ -22,7 +22,7 @@ mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
 mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
 mdbq/redis/getredis.py,sha256=vpBuNc22uj9Vr-_Dh25_wpwWM1e-072EAAIBdB_IpL0,23494
 mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
-mdbq-4.0.57.dist-info/METADATA,sha256=TfW6yMLFlhUzAu4IqtGdSijN5UwnmJHvj3absBB8b5Y,364
-mdbq-4.0.57.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-mdbq-4.0.57.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
-mdbq-4.0.57.dist-info/RECORD,,
+mdbq-4.0.59.dist-info/METADATA,sha256=HuBoWGvOsn7xMnETkobaMBb3kgBFgAY_UE48Wk42lak,364
+mdbq-4.0.59.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
+mdbq-4.0.59.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
+mdbq-4.0.59.dist-info/RECORD,,