mdbq 3.2.9__py3-none-any.whl → 3.2.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mdbq/aggregation/aggregation.py +41 -0
- mdbq/aggregation/query_data.py +41 -41
- {mdbq-3.2.9.dist-info → mdbq-3.2.10.dist-info}/METADATA +1 -1
- {mdbq-3.2.9.dist-info → mdbq-3.2.10.dist-info}/RECORD +6 -6
- {mdbq-3.2.9.dist-info → mdbq-3.2.10.dist-info}/WHEEL +0 -0
- {mdbq-3.2.9.dist-info → mdbq-3.2.10.dist-info}/top_level.txt +0 -0
mdbq/aggregation/aggregation.py
CHANGED
|
@@ -1174,12 +1174,53 @@ def upload_dir(path, db_name, collection_name, json_path=None):
|
|
|
1174
1174
|
if '更新时间' not in df.columns.tolist():
|
|
1175
1175
|
df['更新时间'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
|
1176
1176
|
|
|
1177
|
+
# set_typ = {
|
|
1178
|
+
# '日期': 'date',
|
|
1179
|
+
# '店铺名称': 'varchar(100)',
|
|
1180
|
+
# 'spu_id': 'varchar(100)',
|
|
1181
|
+
# '图片': 'varchar(255)',
|
|
1182
|
+
# '序号': 'smallint',
|
|
1183
|
+
# '商品名称': 'varchar(255)',
|
|
1184
|
+
# '商品款号': 'varchar(255)',
|
|
1185
|
+
# '一级类目名称': 'varchar(255)',
|
|
1186
|
+
# '二级类目名称': 'varchar(255)',
|
|
1187
|
+
# '三级类目名称': 'varchar(255)',
|
|
1188
|
+
# '数据更新时间': 'timestamp',
|
|
1189
|
+
# '更新时间': 'timestamp',
|
|
1190
|
+
# }
|
|
1191
|
+
# new_dict = {
|
|
1192
|
+
# '日期': '',
|
|
1193
|
+
# '店铺名称': '',
|
|
1194
|
+
# '序号': '',
|
|
1195
|
+
# '商品名称': '',
|
|
1196
|
+
# 'spu_id': '',
|
|
1197
|
+
# '商品款号': '',
|
|
1198
|
+
# '一级类目名称': '',
|
|
1199
|
+
# '二级类目名称': '',
|
|
1200
|
+
# '三级类目名称': '',
|
|
1201
|
+
# '访客量': '',
|
|
1202
|
+
# '浏览量': '',
|
|
1203
|
+
# '下单gmv': '',
|
|
1204
|
+
# '成交gmv': '',
|
|
1205
|
+
# '支付人数_成交': '',
|
|
1206
|
+
# }
|
|
1207
|
+
# for dict_data in df.to_dict(orient='records'):
|
|
1208
|
+
# new_dict.update(dict_data)
|
|
1209
|
+
# m.dict_to_mysql(
|
|
1210
|
+
# db_name=db_name,
|
|
1211
|
+
# table_name=collection_name,
|
|
1212
|
+
# dict_data=new_dict,
|
|
1213
|
+
# # icm_update=['日期', '店铺名称', 'spu_id', '商品款号'],
|
|
1214
|
+
# unique_main_key=None,
|
|
1215
|
+
# set_typ=set_typ,
|
|
1216
|
+
# )
|
|
1177
1217
|
m.df_to_mysql(df=df, db_name=db_name, table_name=collection_name,
|
|
1178
1218
|
move_insert=False, # 先删除，再插入
|
|
1179
1219
|
df_sql = True,
|
|
1180
1220
|
drop_duplicates=False,
|
|
1181
1221
|
count=f'{i}/{count}',
|
|
1182
1222
|
filename=name,
|
|
1223
|
+
set_typ={},
|
|
1183
1224
|
)
|
|
1184
1225
|
# nas.df_to_mysql(df=df, db_name=db_name, table_name=collection_name, drop_duplicates=True,)
|
|
1185
1226
|
|
mdbq/aggregation/query_data.py
CHANGED
|
@@ -2288,46 +2288,46 @@ class MysqlDatasQuery:
|
|
|
2288
2288
|
max_date = df['日期'].max()
|
|
2289
2289
|
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
2290
2290
|
print(f'{now} 正在更新: mysql ({host}:{port}) {db_name}/{table_name} -> {min_date}~{max_date}')
|
|
2291
|
-
new_dict = {
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
}
|
|
2307
|
-
for dict_data in df.to_dict(orient='records'):
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2291
|
+
# new_dict = {
|
|
2292
|
+
# '日期': '',
|
|
2293
|
+
# '店铺名称': '',
|
|
2294
|
+
# '序号': '',
|
|
2295
|
+
# '商品名称': '',
|
|
2296
|
+
# 'spu_id': '',
|
|
2297
|
+
# '商品款号': '',
|
|
2298
|
+
# '一级类目名称': '',
|
|
2299
|
+
# '二级类目名称': '',
|
|
2300
|
+
# '三级类目名称': '',
|
|
2301
|
+
# '访客量': '',
|
|
2302
|
+
# '浏览量': '',
|
|
2303
|
+
# '下单gmv': '',
|
|
2304
|
+
# '成交gmv': '',
|
|
2305
|
+
# '支付人数_成交': '',
|
|
2306
|
+
# }
|
|
2307
|
+
# for dict_data in df.to_dict(orient='records'):
|
|
2308
|
+
# new_dict.update(dict_data)
|
|
2309
|
+
# m_engine.dict_to_mysql(
|
|
2310
|
+
# db_name=db_name,
|
|
2311
|
+
# table_name=table_name,
|
|
2312
|
+
# dict_data=new_dict,
|
|
2313
|
+
# icm_update=['日期', '店铺名称', 'spu_id', '商品款号'],
|
|
2314
|
+
# unique_main_key=None,
|
|
2315
|
+
# set_typ=set_typ,
|
|
2316
|
+
# )
|
|
2317
|
+
m_engine.df_to_mysql(
|
|
2318
|
+
df=df,
|
|
2319
|
+
db_name=db_name,
|
|
2320
|
+
table_name=table_name,
|
|
2321
|
+
icm_update=[], # 增量更新, 在聚合数据中使用，其他不要用
|
|
2322
|
+
move_insert=True, # 先删除，再插入
|
|
2323
|
+
df_sql=False, # 值为 True 时使用 df.to_sql 函数上传整个表, 不会排重
|
|
2324
|
+
drop_duplicates=False, # 值为 True 时检查重复数据再插入，反之直接上传，会比较慢
|
|
2325
|
+
count=None,
|
|
2326
|
+
filename=None, # 用来追踪处理进度
|
|
2327
|
+
reset_id=False, # 是否重置自增列
|
|
2328
|
+
set_typ=set_typ,
|
|
2329
|
+
|
|
2330
|
+
)
|
|
2331
2331
|
return True
|
|
2332
2332
|
|
|
2333
2333
|
def deeplink(self, db_name='聚合数据', table_name='达摩盘_deeplink人群洞察'):
|
|
@@ -3318,7 +3318,7 @@ if __name__ == '__main__':
|
|
|
3318
3318
|
# 3. 清理聚合数据
|
|
3319
3319
|
optimize_data.op_data(
|
|
3320
3320
|
db_name_lists=['聚合数据'],
|
|
3321
|
-
days=
|
|
3321
|
+
days=100, # 清理聚合数据的日期长度
|
|
3322
3322
|
is_mongo=False,
|
|
3323
3323
|
is_mysql=True,
|
|
3324
3324
|
)
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
|
|
2
2
|
mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
|
|
3
3
|
mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
|
|
4
|
-
mdbq/aggregation/aggregation.py,sha256=
|
|
4
|
+
mdbq/aggregation/aggregation.py,sha256=kdWeVjvUoWOZhidez0FyMtutIrPwnjLCY7USaQVNxRk,76336
|
|
5
5
|
mdbq/aggregation/df_types.py,sha256=U9i3q2eRPTDY8qAPTw7irzu-Tlg4CIySW9uYro81wdk,8125
|
|
6
6
|
mdbq/aggregation/mysql_types.py,sha256=YTGyrF9vcRgfkQbpT-e-JdJ7c7VF1dDHgyx9YZRES8w,10934
|
|
7
7
|
mdbq/aggregation/optimize_data.py,sha256=RXIv7cACCgYyehAxMjUYi_S7rVyjIwXKWMaM3nduGtA,3068
|
|
8
|
-
mdbq/aggregation/query_data.py,sha256=
|
|
8
|
+
mdbq/aggregation/query_data.py,sha256=GbmvkRYEv_xg90vHp2FszjFZuMqO3ZPSEp6lZrnOrIE,148227
|
|
9
9
|
mdbq/aggregation/query_data_bak.py,sha256=r1FU0C4zjXln7oVSrRkElh4Ehl-9mYhGcq57jLbViUA,104071
|
|
10
10
|
mdbq/aggregation/query_data_bak20241124.py,sha256=oY95ZK3qt3Wx9pdZKZ5cvDh45Yi5yGj1kl8G6riumHA,144513
|
|
11
11
|
mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
|
|
@@ -46,7 +46,7 @@ mdbq/req_post/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
|
|
|
46
46
|
mdbq/req_post/req_tb.py,sha256=qg7pet73IgKGmCwxaeUyImJIoeK_pBQT9BBKD7fkBNg,36160
|
|
47
47
|
mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
|
|
48
48
|
mdbq/spider/aikucun.py,sha256=nIKKZOZbemKqcrikcrMmtksLgJjjzeU0I99teBgU1jE,22439
|
|
49
|
-
mdbq-3.2.
|
|
50
|
-
mdbq-3.2.
|
|
51
|
-
mdbq-3.2.
|
|
52
|
-
mdbq-3.2.
|
|
49
|
+
mdbq-3.2.10.dist-info/METADATA,sha256=l7LN00jP2XEWyB9qTPGKZIbV0Aucaa57dyB50mgImJU,244
|
|
50
|
+
mdbq-3.2.10.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
|
|
51
|
+
mdbq-3.2.10.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
|
|
52
|
+
mdbq-3.2.10.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|