mdbq 3.8.10__py3-none-any.whl → 3.8.12__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
mdbq/__version__.py CHANGED
@@ -1 +1 @@
1
- VERSION = '3.8.10'
1
+ VERSION = '3.8.12'
@@ -147,12 +147,12 @@ class MysqlDatasQuery:
147
147
  '直接成交笔数': 'int',
148
148
  '直接成交金额': 'decimal(12,2)',
149
149
  }
150
- self.pf_datas.append(
151
- {
152
- '集合名称': table_name,
153
- '数据主体': df[['日期', '店铺名称', '商品id', '花费', '成交金额', '直接成交金额']]
154
- }
155
- ) # 制作其他聚合表
150
+ # self.pf_datas.append(
151
+ # {
152
+ # '集合名称': table_name,
153
+ # '数据主体': df[['日期', '店铺名称', '商品id', '花费', '成交金额', '直接成交金额']]
154
+ # }
155
+ # ) # 制作其他聚合表
156
156
  self.pf_datas.append(
157
157
  {
158
158
  '集合名称': '天猫汇总表调用',
@@ -633,12 +633,12 @@ class MysqlDatasQuery:
633
633
  '件均价': 'mediumint',
634
634
  '价格带': 'varchar(100)',
635
635
  }
636
- self.pf_datas.append(
637
- {
638
- '集合名称': table_name,
639
- '数据主体': df[['日期', '店铺名称', '宝贝id', '销售额', '销售量', '退款额_发货后', '退货量_发货后']]
640
- }
641
- ) # 制作其他聚合表
636
+ # self.pf_datas.append(
637
+ # {
638
+ # '集合名称': table_name,
639
+ # '数据主体': df[['日期', '店铺名称', '宝贝id', '销售额', '销售量', '退款额_发货后', '退货量_发货后']]
640
+ # }
641
+ # ) # 制作其他聚合表
642
642
  if not self.update_service:
643
643
  return
644
644
  min_date = df['日期'].min()
@@ -1333,12 +1333,12 @@ class MysqlDatasQuery:
1333
1333
  '二级类目': 'varchar(100)',
1334
1334
  '三级类目': 'varchar(100)',
1335
1335
  }
1336
- self.pf_datas.append(
1337
- {
1338
- '集合名称': table_name,
1339
- '数据主体': df[['宝贝id', '商家编码']]
1340
- }
1341
- ) # 制作其他聚合表
1336
+ # self.pf_datas.append(
1337
+ # {
1338
+ # '集合名称': table_name,
1339
+ # '数据主体': df[['宝贝id', '商家编码']]
1340
+ # }
1341
+ # ) # 制作其他聚合表
1342
1342
  if not self.update_service:
1343
1343
  return
1344
1344
  now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
@@ -1390,12 +1390,12 @@ class MysqlDatasQuery:
1390
1390
  '三级类目': 'varchar(100)',
1391
1391
  '更新时间': 'timestamp'
1392
1392
  }
1393
- self.pf_datas.append(
1394
- {
1395
- '集合名称': table_name,
1396
- '数据主体': df[['宝贝id', '商家编码']]
1397
- }
1398
- ) # 制作其他聚合表
1393
+ # self.pf_datas.append(
1394
+ # {
1395
+ # '集合名称': table_name,
1396
+ # '数据主体': df[['宝贝id', '商家编码']]
1397
+ # }
1398
+ # ) # 制作其他聚合表
1399
1399
  if not self.update_service:
1400
1400
  return
1401
1401
  now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
@@ -1447,12 +1447,12 @@ class MysqlDatasQuery:
1447
1447
  '日期': 'date',
1448
1448
  '商品链接': 'varchar(255)',
1449
1449
  }
1450
- self.pf_datas.append(
1451
- {
1452
- '集合名称': table_name,
1453
- '数据主体': df[['商品id', '商品图片']]
1454
- }
1455
- ) # 制作其他聚合表
1450
+ # self.pf_datas.append(
1451
+ # {
1452
+ # '集合名称': table_name,
1453
+ # '数据主体': df[['商品id', '商品图片']]
1454
+ # }
1455
+ # ) # 制作其他聚合表
1456
1456
  if not self.update_service:
1457
1457
  return
1458
1458
  now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
@@ -1505,13 +1505,13 @@ class MysqlDatasQuery:
1505
1505
  'sku地址': 'varchar(255)',
1506
1506
  '更新时间': 'timestamp'
1507
1507
  }
1508
- # 制作其他聚合表
1509
- self.pf_datas.append(
1510
- {
1511
- '集合名称': table_name,
1512
- '数据主体': df[['商品id', '商品图片']]
1513
- }
1514
- )
1508
+ # # 制作其他聚合表
1509
+ # self.pf_datas.append(
1510
+ # {
1511
+ # '集合名称': table_name,
1512
+ # '数据主体': df[['商品id', '商品图片']]
1513
+ # }
1514
+ # )
1515
1515
  if not self.update_service: # 调试加,是否继续执行下面的入库操作
1516
1516
  return
1517
1517
  now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
@@ -1852,12 +1852,12 @@ class MysqlDatasQuery:
1852
1852
  '天猫页面价': 'decimal(10,2)',
1853
1853
  '天猫中促价': 'decimal(10,2)',
1854
1854
  }
1855
- self.pf_datas.append(
1856
- {
1857
- '集合名称': table_name,
1858
- '数据主体': df[['款号', '成本价']]
1859
- }
1860
- ) # 制作其他聚合表
1855
+ # self.pf_datas.append(
1856
+ # {
1857
+ # '集合名称': table_name,
1858
+ # '数据主体': df[['款号', '成本价']]
1859
+ # }
1860
+ # ) # 制作其他聚合表
1861
1861
  if not self.update_service:
1862
1862
  return
1863
1863
  min_date = pd.to_datetime(df['日期'].min()).strftime('%Y-%m-%d')
@@ -1951,12 +1951,12 @@ class MysqlDatasQuery:
1951
1951
  p = df.pop('spu_id')
1952
1952
  df.insert(loc=3, column='spu_id', value=p)
1953
1953
 
1954
- self.pf_datas_jd.append(
1955
- {
1956
- '集合名称': table_name,
1957
- '数据主体': df[['日期', '产品线', '触发sku_id', '跟单sku_id', '花费']]
1958
- }
1959
- ) # 制作其他聚合表
1954
+ # self.pf_datas_jd.append(
1955
+ # {
1956
+ # '集合名称': table_name,
1957
+ # '数据主体': df[['日期', '产品线', '触发sku_id', '跟单sku_id', '花费']]
1958
+ # }
1959
+ # ) # 制作其他聚合表
1960
1960
  if not self.update_service:
1961
1961
  return
1962
1962
  set_typ = {
@@ -2204,12 +2204,12 @@ class MysqlDatasQuery:
2204
2204
  idx = df.groupby(['日期', '店铺名称', '商品id', '货号', '访客数', '成交客户数', '加购商品件数', '加购人数'])['更新时间'].idxmax()
2205
2205
  df = df.loc[idx]
2206
2206
  df = df[['日期', '店铺名称', '商品id', '货号', '访客数', '成交客户数', '加购商品件数', '加购人数', '成交单量', '成交金额']]
2207
- self.pf_datas_jd.append(
2208
- {
2209
- '集合名称': table_name,
2210
- '数据主体': df
2211
- }
2212
- ) # 制作其他聚合表
2207
+ # self.pf_datas_jd.append(
2208
+ # {
2209
+ # '集合名称': table_name,
2210
+ # '数据主体': df
2211
+ # }
2212
+ # ) # 制作其他聚合表
2213
2213
  if not self.update_service:
2214
2214
  return
2215
2215
  set_typ = {
@@ -3758,15 +3758,6 @@ class MysqlDatasQuery:
3758
3758
  '直接成交金额': 'float64',
3759
3759
  '自然流量曝光量': 'int64',
3760
3760
  }, errors='raise')
3761
- # tg = tg.groupby(['日期', '推广渠道', '营销场景', '商品id', '花费', '展现量', '点击量'], as_index=False).agg(
3762
- # **{'加购量': ('加购量', np.max),
3763
- # '成交笔数': ('成交笔数', np.max),
3764
- # '成交金额': ('成交金额', np.max),
3765
- # '自然流量曝光量': ('自然流量曝光量', np.max),
3766
- # '直接成交笔数': ('直接成交笔数', np.max),
3767
- # '直接成交金额': ('直接成交金额', np.max)
3768
- # }
3769
- # )
3770
3761
 
3771
3762
  df = pd.concat([tg, zb, pxb], axis=0, ignore_index=True)
3772
3763
  df.fillna(0, inplace=True) # concat 之后要填充空值
@@ -3994,25 +3985,29 @@ def query1(months=1, less_dict=[]):
3994
3985
  sdq.months = months # 设置数据周期, 1 表示近 2 个月
3995
3986
  sdq.update_service = True # 调试时加,true: 将数据写入 mysql 服务器
3996
3987
 
3997
- sdq._ald_wxt(db_name='聚合数据', table_name='奥莱店_主体报表')
3998
- sdq._sj_wxt(db_name='聚合数据', table_name='圣积天猫店_主体报表')
3999
- sdq._tb_wxt(db_name='聚合数据', table_name='淘宝_主体报表')
3988
+ # 依赖表 -- >>
4000
3989
  sdq.tg_wxt(db_name='聚合数据', table_name='天猫_主体报表')
3990
+ sdq.tg_cjzb(db_name='聚合数据', table_name='天猫_超级直播')
3991
+ sdq.pxb_zh(db_name='聚合数据', table_name='天猫_品销宝账户报表')
3992
+ # 依赖表 << --
3993
+
4001
3994
  sdq.syj(db_name='聚合数据', table_name='生意经_宝贝指标')
4002
3995
  sdq.idbm(db_name='聚合数据', table_name='商品id编码表')
4003
3996
  sdq.sp_picture(db_name='聚合数据', table_name='商品id图片对照表')
4004
3997
  sdq.sp_cost(db_name='聚合数据', table_name='商品成本')
4005
3998
  sdq.jdjzt(db_name='聚合数据', table_name='京东_京准通')
4006
- sdq.jdqzyx(db_name='聚合数据', table_name='京东_京准通_全站营销')
4007
3999
  sdq.sku_sales(db_name='聚合数据', table_name='京东_sku_商品明细')
4000
+
4001
+ sdq._ald_wxt(db_name='聚合数据', table_name='奥莱店_主体报表')
4002
+ sdq._sj_wxt(db_name='聚合数据', table_name='圣积天猫店_主体报表')
4003
+ sdq._tb_wxt(db_name='聚合数据', table_name='淘宝_主体报表')
4004
+ sdq.jdqzyx(db_name='聚合数据', table_name='京东_京准通_全站营销')
4008
4005
  sdq.spu_sales(db_name='聚合数据', table_name='京东_spu_商品明细')
4009
- sdq.tg_cjzb(db_name='聚合数据', table_name='天猫_超级直播')
4010
- sdq.pxb_zh(db_name='聚合数据', table_name='天猫_品销宝账户报表')
4011
4006
  sdq.zb_ccfx(db_name='聚合数据', table_name='生意参谋_直播场次分析')
4012
4007
  sdq.tg_by_day(db_name='聚合数据', table_name='多店推广场景_按日聚合')
4013
- sdq.performance(bb_tg=True, db_name='聚合数据', table_name='_全店商品销售') # _全店商品销售
4014
- sdq.performance(bb_tg=False, db_name='聚合数据', table_name='_推广商品销售') # _推广商品销售
4015
- sdq.performance_jd(jd_tg=False, db_name='聚合数据', table_name='_京东_推广商品销售') # _推广商品销售
4008
+ # sdq.performance(bb_tg=True, db_name='聚合数据', table_name='_全店商品销售') # _全店商品销售
4009
+ # sdq.performance(bb_tg=False, db_name='聚合数据', table_name='_推广商品销售') # _推广商品销售
4010
+ # sdq.performance_jd(jd_tg=False, db_name='聚合数据', table_name='_京东_推广商品销售') # _推广商品销售
4016
4011
  sdq.performance_concat(bb_tg=False, db_name='聚合数据', table_name='天猫_推广汇总') # _推广商品销售
4017
4012
 
4018
4013
 
@@ -6,7 +6,7 @@ import sys
6
6
  import getpass
7
7
 
8
8
 
9
- def setup_logging(reMoveOldHandler=True):
9
+ def setup_logging(reMoveOldHandler=True, filename='spider_tg.log'):
10
10
  """
11
11
  reMoveOldHandler: 替换根日志记录器的所有现有处理器
12
12
  """
@@ -14,7 +14,7 @@ def setup_logging(reMoveOldHandler=True):
14
14
  if not os.path.isdir(os.path.join(dir_path, 'logfile')):
15
15
  os.makedirs(os.path.join(dir_path, 'logfile'))
16
16
 
17
- log_file = os.path.join(dir_path, 'logfile', 'spider_tg.log')
17
+ log_file = os.path.join(dir_path, 'logfile', filename)
18
18
  file_handler = RotatingFileHandler(
19
19
  filename=log_file,
20
20
  maxBytes=3*1024*1024, # 3MB
mdbq/mysql/mysql.py CHANGED
@@ -246,32 +246,6 @@ class MysqlUpload:
246
246
 
247
247
  if reset_id:
248
248
  pass
249
- # # 6. 重置自增列
250
- # try:
251
- # # 查询所有复合主键
252
- # sql = (
253
- # f"SELECT `COLUMN_NAME` AS `PrimaryKey` FROM `information_schema`.`COLUMNS` "
254
- # f"WHERE `TABLE_SCHEMA` = '{db_name}'AND `TABLE_NAME` = '{table_name}' AND `COLUMN_KEY` = 'PRI';"
255
- # )
256
- # cursor.execute(sql)
257
- # result = cursor.fetchall() # 复合主键数
258
- # if len(result) <= 1: # 如果存在复合主键,则不能直接删除 id 键,其他主键可能不是唯一,会报错
259
- # column_name = 'id'
260
- # sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
261
- # f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
262
- # cursor.execute(sql, (db_name, table_name, column_name))
263
- # result = cursor.fetchone()
264
- # if result:
265
- # sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}" # 删除 id 列
266
- # cursor.execute(sql)
267
- # cursor.execute(
268
- # f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
269
- # cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1") # 设置自增从 1 开始
270
- # else:
271
- # logger.info(f'{table_name} 存在复合主键: 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
272
- # except Exception as e:
273
- # logger.error(f'333 {table_name} {e}')
274
- # connection.rollback()
275
249
  connection.commit()
276
250
 
277
251
  @try_except
@@ -919,35 +893,6 @@ class MysqlUpload:
919
893
  )
920
894
  if reset_id:
921
895
  pass
922
- # # 6. 重置自增列
923
- # try:
924
- # # 查询所有复合主键
925
- # sql = (
926
- # f"SELECT `COLUMN_NAME` AS `PrimaryKey` FROM `information_schema`.`COLUMNS` "
927
- # f"WHERE `TABLE_SCHEMA` = '{db_name}'AND `TABLE_NAME` = '{table_name}' AND `COLUMN_KEY` = 'PRI';"
928
- # )
929
- # cursor.execute(sql)
930
- # result = cursor.fetchall() # 复合主键数
931
- # if len(result) <= 1: # 如果存在复合主键,则不能直接删除 id 键,其他主键可能不是唯一,会报错
932
- # column_name = 'id'
933
- # sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
934
- # f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
935
- # # cursor.execute(f"SHOW COLUMNS FROM `{table_name}` LIKE 'id'")
936
- # cursor.execute(sql, (db_name, table_name, column_name))
937
- # result = cursor.fetchone()
938
- # if result:
939
- # # cursor.execute(f"ALTER TABLE `{table_name}` DROP COLUMN id;") # 删除 id 列
940
- # sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}" # 删除 id 列
941
- # cursor.execute(sql)
942
- # cursor.execute(
943
- # f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
944
- # cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1") # 设置自增从 1 开始
945
- # # logger.info(f'重置自增id')
946
- # else:
947
- # logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
948
- # except Exception as e:
949
- # logger.error(f'333 {table_name} {e}')
950
- # connection.rollback()
951
896
  connection.commit() # 提交事务
952
897
  connection.close()
953
898
  return
@@ -975,36 +920,6 @@ class MysqlUpload:
975
920
  index=False,
976
921
  chunksize=1000
977
922
  )
978
- # # 6. 重置自增列
979
- # try:
980
- # # 查询所有复合主键
981
- # sql = (
982
- # f"SELECT `COLUMN_NAME` AS `PrimaryKey` FROM `information_schema`.`COLUMNS` "
983
- # f"WHERE `TABLE_SCHEMA` = '{db_name}'AND `TABLE_NAME` = '{table_name}' AND `COLUMN_KEY` = 'PRI';"
984
- # )
985
- # cursor.execute(sql)
986
- # result = cursor.fetchall() # 复合主键数
987
- # if len(result) <= 1: # 如果存在复合主键,则不能直接删除 id 键,其他主键可能不是唯一,会报错
988
- # column_name = 'id'
989
- # sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
990
- # f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
991
- # # cursor.execute(f"SHOW COLUMNS FROM `{table_name}` LIKE 'id'")
992
- # cursor.execute(sql, (db_name, table_name, column_name))
993
- # result = cursor.fetchone()
994
- # if result:
995
- # # cursor.execute(f"ALTER TABLE `{table_name}` DROP COLUMN id;") # 删除 id 列
996
- # sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}" # 删除 id 列
997
- # cursor.execute(sql)
998
- # cursor.execute(
999
- # f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
1000
- # cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1") # 设置自增从 1 开始
1001
- # # logger.info(f'重置自增id')
1002
- # else:
1003
- # logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
1004
- # except Exception as e:
1005
- # logger.error(f'333 {table_name} {e}')
1006
- # connection.rollback()
1007
- # connection.close()
1008
923
  return
1009
924
 
1010
925
  datas = df.to_dict(orient='records')
@@ -1095,35 +1010,6 @@ class MysqlUpload:
1095
1010
 
1096
1011
  if reset_id:
1097
1012
  pass
1098
- # # 6. 重置自增列
1099
- # try:
1100
- # # 查询所有复合主键
1101
- # sql = (
1102
- # f"SELECT `COLUMN_NAME` AS `PrimaryKey` FROM `information_schema`.`COLUMNS` "
1103
- # f"WHERE `TABLE_SCHEMA` = '{db_name}'AND `TABLE_NAME` = '{table_name}' AND `COLUMN_KEY` = 'PRI';"
1104
- # )
1105
- # cursor.execute(sql)
1106
- # result = cursor.fetchall() # 复合主键数
1107
- # if len(result) <= 1: # 如果存在复合主键,则不能直接删除 id 键,其他主键可能不是唯一,会报错
1108
- # column_name = 'id'
1109
- # sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
1110
- # f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
1111
- # # cursor.execute(f"SHOW COLUMNS FROM `{table_name}` LIKE 'id'")
1112
- # cursor.execute(sql, (db_name, table_name, column_name))
1113
- # result = cursor.fetchone()
1114
- # if result:
1115
- # # cursor.execute(f"ALTER TABLE `{table_name}` DROP COLUMN id;") # 删除 id 列
1116
- # sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}" # 删除 id 列
1117
- # cursor.execute(sql)
1118
- # cursor.execute(
1119
- # f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
1120
- # cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1") # 设置自增从 1 开始
1121
- # # logger.info(f'重置自增id')
1122
- # else:
1123
- # logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
1124
- # except Exception as e:
1125
- # logger.error(f'333 {table_name} {e}')
1126
- # connection.rollback()
1127
1013
  connection.commit() # 提交事务
1128
1014
  connection.close()
1129
1015
 
@@ -1405,33 +1291,6 @@ class OptimizeDatas:
1405
1291
  self.end_date = end_date_before
1406
1292
  else: # 不存在日期列的情况
1407
1293
  self.delete_duplicate2(table_name=table_name, except_key=except_key)
1408
-
1409
- # # 6. 重置自增列
1410
- # try:
1411
- # # 查询所有复合主键
1412
- # sql = (
1413
- # f"SELECT `COLUMN_NAME` AS `PrimaryKey` FROM `information_schema`.`COLUMNS` "
1414
- # f"WHERE `TABLE_SCHEMA` = '{self.db_name}'AND `TABLE_NAME` = '{table_name}' AND `COLUMN_KEY` = 'PRI';"
1415
- # )
1416
- # cursor.execute(sql)
1417
- # result = cursor.fetchall() # 复合主键数
1418
- # if len(result) <= 1: # 如果存在复合主键,则不能直接删除 id 键,其他主键可能不是唯一,会报错
1419
- # column_name = 'id'
1420
- # sql = (f'SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS '
1421
- # f'WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND COLUMN_NAME = %s')
1422
- # cursor.execute(sql, (self.db_name, table_name, column_name))
1423
- # result = cursor.fetchone()
1424
- # if result:
1425
- # sql = f"ALTER TABLE `{table_name}` DROP COLUMN {column_name}" # 删除 id 列
1426
- # cursor.execute(sql)
1427
- # cursor.execute(
1428
- # f"ALTER TABLE `{table_name}` ADD column id INT AUTO_INCREMENT PRIMARY KEY FIRST;")
1429
- # cursor.execute(f"ALTER TABLE `{table_name}` AUTO_INCREMENT = 1") # 设置自增从 1 开始
1430
- # else:
1431
- # logger.info(f'{table_name} 存在复合主键: {[item['PrimaryKey'] for item in result]}, 无法重置自增id')
1432
- # except Exception as e:
1433
- # logger.error(f'333 {table_name} {e}')
1434
- # self.connection.rollback()
1435
1294
  self.connection.close()
1436
1295
  logger.info(f'mysql({self.host}: {self.port}) {self.db_name} 数据库优化完成!')
1437
1296
 
mdbq/other/otk.py CHANGED
@@ -4,6 +4,31 @@ import datetime
4
4
  from dateutil.relativedelta import relativedelta
5
5
  import pandas as pd
6
6
  import numpy as np
7
+ from mdbq.other import ua_sj
8
+ import requests
9
+
10
+
11
+ def get_public_ip():
12
+ services = [
13
+ 'https://checkip.amazonaws.com',
14
+ 'https://ipinfo.io/ip',
15
+ 'https://icanhazip.com',
16
+ 'https://ifconfig.me/ip',
17
+ 'https://ipecho.net/plain',
18
+ 'https://myexternalip.com/raw',
19
+ 'https://ipapi.co/ip'
20
+ ]
21
+ for url in services:
22
+ try:
23
+ response = requests.get(url, timeout=5, headers={'User-Agent': ua_sj.get_ua()})
24
+ if response.status_code == 200:
25
+ response = re.findall(r'\d+\.\d+\.\d+\.\d+', response.text.strip())
26
+ if response:
27
+ return response[0]
28
+ except:
29
+ continue
30
+ print("无法获取外网 IP")
31
+ return '120.236.0.0/8'
7
32
 
8
33
 
9
34
  def first_and_last_day(date):
@@ -148,4 +173,4 @@ def is_integer(int_str):
148
173
 
149
174
 
150
175
  if __name__ == '__main__':
151
- pass
176
+ pass
mdbq/spider/aikucun.py CHANGED
@@ -262,7 +262,7 @@ class AikuCun:
262
262
  print(f'正在获取数据({num}/{len(date_list)}): {item_type}榜单 {date}')
263
263
  # print(res.json())
264
264
  if not res.json()['success']:
265
- print('没有获取到数据, requests 请求不成功, success 返回值应为 True')
265
+ print('没有获取到数据, 请求不成功, 如果连续请求失败 > 5, 则需重新获取cookie后继续')
266
266
  num += 1
267
267
  self.error_count += 1
268
268
  time.sleep(1)
@@ -484,6 +484,6 @@ def main(start_date, end_date, item_type=['spu']):
484
484
  if __name__ == '__main__':
485
485
  main(
486
486
  start_date='2025-03-01',
487
- end_date='2025-03-12',
487
+ # end_date='2025-03-26', # 不传则默认到今天
488
488
  item_type=['spu', 'sku']
489
489
  )
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: mdbq
3
- Version: 3.8.10
3
+ Version: 3.8.12
4
4
  Home-page: https://pypi.org/project/mdbq
5
5
  Author: xigua,
6
6
  Author-email: 2587125111@qq.com
@@ -1,17 +1,17 @@
1
1
  mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
2
- mdbq/__version__.py,sha256=30BOEMWMMdvugdYm1n90xiBvxiQzusLf7XtVO4-Zjr8,18
2
+ mdbq/__version__.py,sha256=Z4fNVSgyq1SukYbN9PNs-2zCbMVcI9eO75ErgL1qq4w,18
3
3
  mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
4
- mdbq/aggregation/query_data.py,sha256=-4HWC1HZmgqUAuvcRiHZU4FLtI70nRq_Hp3eXVZTyH8,185843
4
+ mdbq/aggregation/query_data.py,sha256=9doVoOBEaLyHlT0fVEXpWtwLLV4NhpQs17kHQQ0p3ys,185443
5
5
  mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
6
6
  mdbq/config/config.py,sha256=eaTfrfXQ65xLqjr5I8-HkZd_jEY1JkGinEgv3TSLeoQ,3170
7
7
  mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
8
- mdbq/log/spider_logging.py,sha256=KX9TTUn9naZNBACCEFhyTktnWhr5JaSNQLppLGyrm9Y,1645
8
+ mdbq/log/spider_logging.py,sha256=-ozWWEGm3HVv604ozs_OOvVwumjokmUPwbaodesUrPY,1664
9
9
  mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
10
- mdbq/mysql/mysql.py,sha256=2cPuqX4zq2b6ghFWxTylr52DPZGE2WNrCdFV0RcF6LY,89048
10
+ mdbq/mysql/mysql.py,sha256=bYwzqB3oMUbTRlIWUdZCXso9oll1vHbcNvvBzT3AsKA,78967
11
11
  mdbq/mysql/s_query.py,sha256=X055aLRAgxVvueXx4NbfNjp6MyBI02_XBb1pTKw09L0,8660
12
12
  mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
13
13
  mdbq/other/download_sku_picture.py,sha256=YU8DxKMXbdeE1OOKEA848WVp62jYHw5O4tXTjUdq9H0,44832
14
- mdbq/other/otk.py,sha256=amIFeLDNUJpSi0U6hXbnqXeGTbYL-8-5U5yAATzSM3Y,5947
14
+ mdbq/other/otk.py,sha256=i8Eomxk7i3tU41Pv_dbFCo8ghZ1dQsMJVRp3ZWVtCY8,6710
15
15
  mdbq/other/pov_city.py,sha256=AEOmCOzOwyjHi9LLZWPKi6DUuSC-_M163664I52u9qw,21050
16
16
  mdbq/other/ua_sj.py,sha256=JuVYzc_5QZ9s_oQSrTHVKkQv4S_7-CWx4oIKOARn_9U,22178
17
17
  mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
@@ -20,8 +20,8 @@ mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
20
20
  mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
21
21
  mdbq/redis/getredis.py,sha256=Uk8-cOWT0JU1qRyIVqdbYokSLvkDIAfcokmYj1ebw8k,24104
22
22
  mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
23
- mdbq/spider/aikucun.py,sha256=YLRTDgOKPGDyNB-z5dPOJhBoTzM6Rmbjy1Qng_KyJQc,19906
24
- mdbq-3.8.10.dist-info/METADATA,sha256=bYHqcx9saoyDq-BQSXUtUQWTcDNICdZeF5jaKP6Dlyc,364
25
- mdbq-3.8.10.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
26
- mdbq-3.8.10.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
27
- mdbq-3.8.10.dist-info/RECORD,,
23
+ mdbq/spider/aikucun.py,sha256=m7ZIvrc9pqoGCYEH3FtgKTwqhX7QB6qzgc2twDzhX4w,19962
24
+ mdbq-3.8.12.dist-info/METADATA,sha256=IkddKvTvsNeBEFDqgpNfJOcJj0kkOy1TFt6wKoNiQYw,364
25
+ mdbq-3.8.12.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
26
+ mdbq-3.8.12.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
27
+ mdbq-3.8.12.dist-info/RECORD,,
File without changes