mdbq 3.6.15__py3-none-any.whl → 3.6.17__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. mdbq/mysql/mysql.py +24 -28
  2. mdbq/mysql/s_query.py +2 -2
  3. mdbq/redis/getredis.py +6 -6
  4. {mdbq-3.6.15.dist-info → mdbq-3.6.17.dist-info}/METADATA +1 -1
  5. {mdbq-3.6.15.dist-info → mdbq-3.6.17.dist-info}/RECORD +7 -10
  6. mdbq/mysql/mysql_bak.py +0 -1808
  7. mdbq/mysql/recheck_mysql.py +0 -201
  8. mdbq/redis/getredis_/344/274/230/345/214/226hash.py +0 -710
  9. {mdbq-3.6.15.dist-info → mdbq-3.6.17.dist-info}/WHEEL +0 -0
  10. {mdbq-3.6.15.dist-info → mdbq-3.6.17.dist-info}/top_level.txt +0 -0
mdbq/mysql/mysql.py CHANGED
@@ -121,14 +121,14 @@ class MysqlUpload:
121
121
  try:
122
122
  return func(*args, **kwargs)
123
123
  except Exception as e:
124
- logger.info(f'{func.__name__}, {e}') # 将异常信息返回
124
+ logger.error(f'{func.__name__}, {e}') # 将异常信息返回
125
125
  with open(error_file, 'a') as f:
126
126
  now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
127
127
  f.write(f'\n{now} \n')
128
128
  f.write(f'函数注释内容(用于定位函数): {func.__doc__} \n')
129
129
  # f.write(f'报错的文件:\n{e.__traceback__.tb_frame.f_globals["__file__"]}\n') # 发生异常所在的文件
130
130
  traceback.print_exc(file=open(error_file, 'a')) # 返回完整的堆栈信息
131
- logger.info(f'更多信息请查看日志文件: {error_file}')
131
+ logger.error(f'更多信息请查看日志文件: {error_file}')
132
132
 
133
133
  return wrapper
134
134
 
@@ -139,10 +139,10 @@ class MysqlUpload:
139
139
  connection = pymysql.connect(**_config) # 连接数据库
140
140
  return connection
141
141
  except Exception as e:
142
- logger.info(f'连接失败，正在重试: {attempts}/{max_try} {e}')
142
+ logger.error(f'连接失败，正在重试: {attempts}/{max_try} {e}')
143
143
  attempts += 1
144
144
  time.sleep(30)
145
- logger.info(f'{_db_name}: 连接失败，重试次数超限，当前设定次数: {max_try}')
145
+ logger.error(f'{_db_name}: 连接失败，重试次数超限，当前设定次数: {max_try}')
146
146
  return None
147
147
 
148
148
  def cover_doc_dtypes(self, dict_data):
@@ -343,7 +343,7 @@ class MysqlUpload:
343
343
  else:
344
344
  logger.info(f'{table_name} 存在复合主键: 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
345
345
  except Exception as e:
346
- logger.info(f'333 {table_name} {e}')
346
+ logger.error(f'333 {table_name} {e}')
347
347
  connection.rollback()
348
348
  connection.commit()
349
349
 
@@ -386,7 +386,7 @@ class MysqlUpload:
386
386
  else:
387
387
  logger.info(f'参数不正确，cut_data应为 year 或 month ')
388
388
  except Exception as e:
389
- logger.info(f'{table_name} 将数据按年/月保存(cut_data)，但在转换日期时报错 -> {e}')
389
+ logger.error(f'{table_name} 将数据按年/月保存(cut_data)，但在转换日期时报错 -> {e}')
390
390
 
391
391
  # connection = pymysql.connect(**self.config) # 连接数据库
392
392
  connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=10)
@@ -597,7 +597,7 @@ class MysqlUpload:
597
597
  else:
598
598
  logger.info(f'参数不正确，cut_data应为 year 或 month ')
599
599
  except Exception as e:
600
- logger.info(f'{table_name} 将数据按年/月保存(cut_data)，但在转换日期时报错 -> {e}')
600
+ logger.error(f'{table_name} 将数据按年/月保存(cut_data)，但在转换日期时报错 -> {e}')
601
601
 
602
602
  # connection = pymysql.connect(**self.config) # 连接数据库
603
603
  connection = self.keep_connect(_db_name=db_name, _config=self.config, max_try=10)
@@ -959,7 +959,7 @@ class MysqlUpload:
959
959
  else:
960
960
  logger.info(f'参数不正确，cut_data应为 year 或 month ')
961
961
  except Exception as e:
962
- logger.info(f'{table_name} 将数据按年/月保存(cut_data)，但在转换日期时报错 -> {e}')
962
+ logger.error(f'{table_name} 将数据按年/月保存(cut_data)，但在转换日期时报错 -> {e}')
963
963
  # 清理 dataframe 非法值，并转换获取数据类型
964
964
  dtypes, df = self.convert_df_dtypes(df)
965
965
  if set_typ:
@@ -1070,7 +1070,7 @@ class MysqlUpload:
1070
1070
  else:
1071
1071
  logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
1072
1072
  except Exception as e:
1073
- logger.info(f'333 {table_name} {e}')
1073
+ logger.error(f'333 {table_name} {e}')
1074
1074
  connection.rollback()
1075
1075
  connection.commit() # 提交事务
1076
1076
  connection.close()
@@ -1126,7 +1126,7 @@ class MysqlUpload:
1126
1126
  else:
1127
1127
  logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
1128
1128
  except Exception as e:
1129
- logger.info(f'333 {table_name} {e}')
1129
+ logger.error(f'333 {table_name} {e}')
1130
1130
  connection.rollback()
1131
1131
  connection.close()
1132
1132
  return
@@ -1216,10 +1216,6 @@ class MysqlUpload:
1216
1216
  cursor.execute(sql)
1217
1217
  except Exception as e:
1218
1218
  pass
1219
- # logger.info(data)
1220
- # logger.info(values)
1221
- # logger.info(f'mysql -> df_to_mysql 报错: {e}, {self.filename}')
1222
- # breakpoint()
1223
1219
 
1224
1220
  if reset_id:
1225
1221
  # 6. 重置自增列
@@ -1249,7 +1245,7 @@ class MysqlUpload:
1249
1245
  else:
1250
1246
  logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
1251
1247
  except Exception as e:
1252
- logger.info(f'333 {table_name} {e}')
1248
+ logger.error(f'333 {table_name} {e}')
1253
1249
  connection.rollback()
1254
1250
  connection.commit() # 提交事务
1255
1251
  connection.close()
@@ -1343,7 +1339,7 @@ class MysqlUpload:
1343
1339
  columns = [desc[0] for desc in cursor.description]
1344
1340
  df = pd.DataFrame(rows, columns=columns) # 转为 df
1345
1341
  except Exception as e:
1346
- logger.info(f'{e} {db_name} -> {table_name} 表不存在')
1342
+ logger.error(f'{e} {db_name} -> {table_name} 表不存在')
1347
1343
  return df
1348
1344
  finally:
1349
1345
  connection.close()
@@ -1437,14 +1433,14 @@ class OptimizeDatas:
1437
1433
  try:
1438
1434
  return func(*args, **kwargs)
1439
1435
  except Exception as e:
1440
- logger.info(f'{func.__name__}, {e}') # 将异常信息返回
1436
+ logger.error(f'{func.__name__}, {e}') # 将异常信息返回
1441
1437
  with open(error_file, 'a') as f:
1442
1438
  now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
1443
1439
  f.write(f'\n{now} \n')
1444
1440
  f.write(f'函数注释内容(用于定位函数): {func.__doc__} \n')
1445
1441
  # f.write(f'报错的文件:\n{e.__traceback__.tb_frame.f_globals["__file__"]}\n') # 发生异常所在的文件
1446
1442
  traceback.print_exc(file=open(error_file, 'a')) # 返回完整的堆栈信息
1447
- logger.info(f'更多信息请查看日志文件: {error_file}')
1443
+ logger.error(f'更多信息请查看日志文件: {error_file}')
1448
1444
 
1449
1445
  return wrapper
1450
1446
 
@@ -1455,10 +1451,10 @@ class OptimizeDatas:
1455
1451
  connection = pymysql.connect(**_config) # 连接数据库
1456
1452
  return connection
1457
1453
  except Exception as e:
1458
- logger.info(f'连接失败，正在重试: {attempts}/{max_try} {e}')
1454
+ logger.error(f'连接失败，正在重试: {attempts}/{max_try} {e}')
1459
1455
  attempts += 1
1460
1456
  time.sleep(30)
1461
- logger.info(f'{_db_name}: 连接失败，重试次数超限，当前设定次数: {max_try}')
1457
+ logger.error(f'{_db_name}: 连接失败，重试次数超限，当前设定次数: {max_try}')
1462
1458
  return None
1463
1459
 
1464
1460
  def optimize_list(self):
@@ -1572,7 +1568,7 @@ class OptimizeDatas:
1572
1568
  else:
1573
1569
  logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
1574
1570
  except Exception as e:
1575
- logger.info(f'333 {table_name} {e}')
1571
+ logger.error(f'333 {table_name} {e}')
1576
1572
  self.connection.rollback()
1577
1573
  self.connection.close()
1578
1574
  logger.info(f'mysql({self.host}: {self.port}) {self.db_name} 数据库优化完成!')
@@ -1596,7 +1592,7 @@ class OptimizeDatas:
1596
1592
  continue
1597
1593
  all_datas.append(data) # 数据没有重复
1598
1594
  except Exception as e:
1599
- logger.info(f'{table_name} 函数: mysql - > OptimizeDatas -> delete_duplicate -> {e}')
1595
+ logger.error(f'{table_name} 函数: mysql - > OptimizeDatas -> delete_duplicate -> {e}')
1600
1596
  del all_datas
1601
1597
 
1602
1598
  if not duplicate_id: # 如果没有重复数据，则跳过该数据表
@@ -1608,10 +1604,10 @@ class OptimizeDatas:
1608
1604
  # 移除冗余数据
1609
1605
  sql = f"DELETE FROM `{table_name}` WHERE id IN ({placeholders})"
1610
1606
  cursor.execute(sql, duplicate_id)
1611
- logger.info(f"{table_name} -> {date.strftime('%Y-%m-%d')} before: {len(datas)}, remove: {cursor.rowcount}")
1607
+ logger.debug(f"{table_name} -> {date.strftime('%Y-%m-%d')} before: {len(datas)}, remove: {cursor.rowcount}")
1612
1608
  self.connection.commit() # 提交事务
1613
1609
  except Exception as e:
1614
- logger.info(f'{self.db_name}/{table_name}, {e}')
1610
+ logger.error(f'{self.db_name}/{table_name}, {e}')
1615
1611
  self.connection.rollback() # 异常则回滚
1616
1612
 
1617
1613
  def delete_duplicate2(self, table_name, except_key=['更新时间']):
@@ -1649,7 +1645,7 @@ class OptimizeDatas:
1649
1645
  f"remove: {cursor.rowcount}")
1650
1646
  self.connection.commit() # 提交事务
1651
1647
  except Exception as e:
1652
- logger.info(f'{self.db_name}/{table_name}, {e}')
1648
+ logger.error(f'{self.db_name}/{table_name}, {e}')
1653
1649
  self.connection.rollback() # 异常则回滚
1654
1650
 
1655
1651
  def database_list(self):
@@ -1678,7 +1674,7 @@ class OptimizeDatas:
1678
1674
  logger.info(f'{db_name}: 数据表不存在!')
1679
1675
  return
1680
1676
  except Exception as e:
1681
- logger.info(f'002 {e}')
1677
+ logger.error(f'002 {e}')
1682
1678
  return
1683
1679
  finally:
1684
1680
  connection.close() # 断开连接
@@ -1709,7 +1705,7 @@ class OptimizeDatas:
1709
1705
  cursor.execute(sql)
1710
1706
  results = cursor.fetchall()
1711
1707
  except Exception as e:
1712
- logger.info(f'001 {e}')
1708
+ logger.error(f'001 {e}')
1713
1709
  finally:
1714
1710
  connection.close()
1715
1711
  return results
@@ -1763,7 +1759,7 @@ def year_month_day_bak(start_date, end_date):
1763
1759
  try:
1764
1760
  start_date = f'{pd.to_datetime(start_date).year}-{pd.to_datetime(start_date).month}-01'
1765
1761
  except Exception as e:
1766
- logger.info(e)
1762
+ logger.error(e)
1767
1763
  return []
1768
1764
  # 使用pandas的date_range创建一个日期范围，频率为'MS'代表每月开始
1769
1765
  date_range = pd.date_range(start=start_date, end=end_date, freq='MS')
mdbq/mysql/s_query.py CHANGED
@@ -117,7 +117,7 @@ class QueryDatas:
117
117
  df[decimal_cols] = df[decimal_cols].astype(float)
118
118
 
119
119
  except Exception as e:
120
- logger.info(f"Database operation failed: {str(e)}")
120
+ logger.error(f"Database operation failed: {str(e)}")
121
121
  finally:
122
122
  if connection:
123
123
  connection.close()
@@ -195,7 +195,7 @@ class QueryDatas:
195
195
  return False
196
196
  return True
197
197
  except Exception as e:
198
- logger.info(e)
198
+ logger.error(e)
199
199
  return False
200
200
  finally:
201
201
  connection.close() # 断开连接
mdbq/redis/getredis.py CHANGED
@@ -39,7 +39,7 @@ else:
39
39
  username, password, host, port = conf_data['username'], conf_data['password'], conf_data['host'], conf_data['port']
40
40
  redis_password = conf['Windows']['company']['redis']['local']['password'] # redis 使用本地数据，全部机子相同
41
41
 
42
- logging.basicConfig(level=logging.INFO, format='%(asctime)s | %(levelname)s | %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
42
+ # logging.basicConfig(level=logging.INFO, format='%(asctime)s | %(levelname)s | %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
43
43
 
44
44
  # 获取当前模块的日志记录器
45
45
  logger = logging.getLogger(__name__)
@@ -48,7 +48,7 @@ logger = logging.getLogger(__name__)
48
48
  if not os.path.isdir(os.path.join(D_PATH, 'logfile')):
49
49
  os.makedirs(os.path.join(D_PATH, 'logfile'))
50
50
  log_file = os.path.join(D_PATH, 'logfile', 'redis.log')
51
- file_handler = RotatingFileHandler(log_file, maxBytes=3 * 1024 * 1024, backupCount=10) # 保留10个备份文件
51
+ file_handler = RotatingFileHandler(log_file, maxBytes=3 * 1024 * 1024, backupCount=10, encoding='utf-8') # 保留10个备份文件
52
52
  file_handler.setLevel(logging.INFO) # 设置文件处理器的日志级别
53
53
 
54
54
  # 创建一个日志格式器，并设置给文件处理器
@@ -123,7 +123,7 @@ class RedisData(object):
123
123
  ttl = self.redis_engine.ttl(cache_key)
124
124
  cache_data = self._fetch_redis_data(cache_key)
125
125
  except Exception as e:
126
- logger.info(f"Redis 连接异常: {e}，直接访问 MySQL")
126
+ logger.error(f"Redis 连接异常: {e}，直接访问 MySQL")
127
127
  return self.get_from_mysql(db_name, table_name, set_year, start_date, end_date)
128
128
 
129
129
  # 缓存失效处理逻辑
@@ -175,7 +175,7 @@ class RedisData(object):
175
175
  return combined_data
176
176
 
177
177
  except Exception as e:
178
- logger.info(f"缓存更新失败: {cache_key} - {str(e)}")
178
+ logger.error(f"缓存更新失败: {cache_key} - {str(e)}")
179
179
  return pd.DataFrame()
180
180
 
181
181
  # Helper Methods ------------------------------------------------
@@ -197,7 +197,7 @@ class RedisData(object):
197
197
  projection={}
198
198
  )
199
199
  except Exception as e:
200
- logger.info(f"MySQL 查询异常 {db_name}.{table_name}: {e}")
200
+ logger.error(f"MySQL 查询异常 {db_name}.{table_name}: {e}")
201
201
  return pd.DataFrame()
202
202
 
203
203
  def _fetch_redis_data(self, cache_key: str) -> pd.DataFrame:
@@ -210,7 +210,7 @@ class RedisData(object):
210
210
  df = pd.DataFrame(json.loads(data.decode("utf-8")))
211
211
  return self._convert_date_columns(df)
212
212
  except Exception as e:
213
- logger.info(f"Redis 数据解析失败 {cache_key}: {e}")
213
+ logger.error(f"Redis 数据解析失败 {cache_key}: {e}")
214
214
  return pd.DataFrame()
215
215
 
216
216
  def _convert_date_columns(self, df: pd.DataFrame) -> pd.DataFrame:
{mdbq-3.6.15.dist-info → mdbq-3.6.17.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: mdbq
3
- Version: 3.6.15
3
+ Version: 3.6.17
4
4
  Home-page: https://pypi.org/project/mdbq
5
5
  Author: xigua,
6
6
  Author-email: 2587125111@qq.com
{mdbq-3.6.15.dist-info → mdbq-3.6.17.dist-info}/RECORD RENAMED
@@ -18,10 +18,8 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
18
18
  mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
19
19
  mdbq/mongo/mongo.py,sha256=M9DUeUCMPDngkwn9-ui0uTiFrvfNU1kLs22s5SmoNm0,31899
20
20
  mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
21
- mdbq/mysql/mysql.py,sha256=ZB552VAMn7tz-Z0MPZKwY2LL2mAms0ATb5y68OBL9Tg,98712
22
- mdbq/mysql/mysql_bak.py,sha256=_jFo2_OC1BNm5wEmoYiBG_TcuNNA2xUWKNhMBfgDiAM,99699
23
- mdbq/mysql/recheck_mysql.py,sha256=ppBTfBLgkRWirMVZ31e_ZPULiGPJU7K3PP9G6QBZ3QI,8605
24
- mdbq/mysql/s_query.py,sha256=CL2Ayo2sL11RbLnh9nE-GXA-NpA815-rrlFo24TipKY,8792
21
+ mdbq/mysql/mysql.py,sha256=v9nr-tb2L1VN0mKUVqEkfcWP1qXQRPVHIOHs3q89Cos,98531
22
+ mdbq/mysql/s_query.py,sha256=pj5ioJfUT81Su9S-km9G49gF5F2MmXXfw_oAIUzhN28,8794
25
23
  mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
26
24
  mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
27
25
  mdbq/other/download_sku_picture.py,sha256=GdphR7Q3psXXVuZoyJ4u_6OWn_rWlcbT0iJ-1zPT6O0,45368
@@ -34,11 +32,10 @@ mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,239
34
32
  mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
35
33
  mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,7192
36
34
  mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
37
- mdbq/redis/getredis.py,sha256=4rYk9lMRvvlpY7cV3VNQcSnbDWlZIsZZ-tSq--YqfSQ,26638
38
- mdbq/redis/getredis_优化hash.py,sha256=q7omKJCPw_6Zr_r6WwTv4RGSXzZzpLPkIaqJ22svJhE,29104
35
+ mdbq/redis/getredis.py,sha256=XUuUEsvRzbknloSxC6poA_-AE29eSGmUAqcbK2RjZYQ,26662
39
36
  mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
40
37
  mdbq/spider/aikucun.py,sha256=v7VO5gtEXR6_4Q6ujbTyu1FHu7TXHcwSQ6hIO249YH0,22208
41
- mdbq-3.6.15.dist-info/METADATA,sha256=Zb_mvyFxjvFYudvISR7D9HMXnBj34_ZvWJXHDo9_pOY,244
42
- mdbq-3.6.15.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
43
- mdbq-3.6.15.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
44
- mdbq-3.6.15.dist-info/RECORD,,
38
+ mdbq-3.6.17.dist-info/METADATA,sha256=DMsQwJrQfNY8_7rOCmElf9lnitv8qHLqsUZa2bGkb74,244
39
+ mdbq-3.6.17.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
40
+ mdbq-3.6.17.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
41
+ mdbq-3.6.17.dist-info/RECORD,,