mdbq 1.7.0__py3-none-any.whl → 1.7.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mdbq/aggregation/aggregation.py +20 -5
- mdbq/aggregation/query_data.py +102 -102
- mdbq/clean/data_clean.py +17 -0
- {mdbq-1.7.0.dist-info → mdbq-1.7.2.dist-info}/METADATA +1 -1
- {mdbq-1.7.0.dist-info → mdbq-1.7.2.dist-info}/RECORD +7 -7
- {mdbq-1.7.0.dist-info → mdbq-1.7.2.dist-info}/WHEEL +0 -0
- {mdbq-1.7.0.dist-info → mdbq-1.7.2.dist-info}/top_level.txt +0 -0
mdbq/aggregation/aggregation.py
CHANGED
|
@@ -530,6 +530,17 @@ class DatabaseUpdate:
|
|
|
530
530
|
df.drop("'当前时间'", axis=1, inplace=True)
|
|
531
531
|
df.rename(columns={'全站ROI': '全站roi'}, inplace=True)
|
|
532
532
|
df.insert(loc=1, column='产品线', value='全站营销')
|
|
533
|
+
elif name.endswith('.csv') and '关键词点击成交报表_pbix同步_勿删改' in name:
|
|
534
|
+
df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
|
|
535
|
+
for col in df.columns.tolist():
|
|
536
|
+
if '（' in col:
|
|
537
|
+
new_col = re.sub('[（）]', '_', col)
|
|
538
|
+
new_col = new_col.strip('_')
|
|
539
|
+
df.rename(columns={col: new_col}, inplace=True)
|
|
540
|
+
df['日期'] = df['日期'].apply(lambda x: f'{str(x)[:4]}-{str(x)[4:6]}-{str(x)[6:8]}')
|
|
541
|
+
df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
|
|
542
|
+
# min_clm = str(df['日期'].min()).split(' ')[0]
|
|
543
|
+
# max_clm = str(df['日期'].max()).split(' ')[0]
|
|
533
544
|
|
|
534
545
|
# 商品素材，必须保持放在最后处理
|
|
535
546
|
elif name.endswith('xlsx'):
|
|
@@ -960,7 +971,9 @@ def upload_dir(path, db_name, collection_name, dbs={'mysql': True, 'mongodb': Tr
|
|
|
960
971
|
d.df_to_mongo(df=df, db_name=db_name, collection_name=collection_name)
|
|
961
972
|
if dbs['mysql']: # drop_duplicates: 值为 True 时检查重复数据再插入
|
|
962
973
|
m.df_to_mysql(df=df, db_name=db_name, table_name=collection_name,
|
|
963
|
-
|
|
974
|
+
df_sql = True,
|
|
975
|
+
drop_duplicates=False,
|
|
976
|
+
filename=name, count=f'{i}/{count}')
|
|
964
977
|
# nas.df_to_mysql(df=df, db_name=db_name, table_name=collection_name, drop_duplicates=True,)
|
|
965
978
|
except Exception as e:
|
|
966
979
|
print(name, e)
|
|
@@ -1068,11 +1081,13 @@ if __name__ == '__main__':
|
|
|
1068
1081
|
# target_service='company',
|
|
1069
1082
|
# database='mysql'
|
|
1070
1083
|
# )
|
|
1071
|
-
|
|
1072
|
-
|
|
1084
|
+
|
|
1085
|
+
db_name = '京东数据2'
|
|
1086
|
+
table_name = '推广数据_关键词报表'
|
|
1073
1087
|
upload_dir(
|
|
1074
|
-
path='/Users/xigua
|
|
1088
|
+
path='/Users/xigua/数据中心/原始文件2/京东报表/JD推广_关键词报表',
|
|
1075
1089
|
db_name=db_name,
|
|
1076
1090
|
collection_name=table_name,
|
|
1077
|
-
dbs={'mysql':
|
|
1091
|
+
dbs={'mysql': True, 'mongodb': False},
|
|
1078
1092
|
)
|
|
1093
|
+
|
mdbq/aggregation/query_data.py
CHANGED
|
@@ -1099,72 +1099,72 @@ def data_aggregation(service_databases=[{}], months=1):
|
|
|
1099
1099
|
'唯一主键': ['日期', '推广渠道', '营销场景', '商品id', '花费'],
|
|
1100
1100
|
'数据主体': sdq.tg_wxt(),
|
|
1101
1101
|
},
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1102
|
+
{
|
|
1103
|
+
'数据库名': '聚合数据',
|
|
1104
|
+
'集合名': '天猫生意经_宝贝指标',
|
|
1105
|
+
'唯一主键': ['日期', '宝贝id'],
|
|
1106
|
+
'数据主体': sdq.syj(),
|
|
1107
|
+
},
|
|
1108
|
+
{
|
|
1109
|
+
'数据库名': '聚合数据',
|
|
1110
|
+
'集合名': '天猫_店铺来源_日数据',
|
|
1111
|
+
'唯一主键': ['日期', '一级来源', '二级来源', '三级来源'],
|
|
1112
|
+
'数据主体': sdq.dplyd(),
|
|
1113
|
+
},
|
|
1114
|
+
{
|
|
1115
|
+
'数据库名': '聚合数据',
|
|
1116
|
+
'集合名': '天猫_店铺来源_日数据_旧版',
|
|
1117
|
+
'唯一主键': ['日期', '一级来源', '二级来源', '三级来源'],
|
|
1118
|
+
'数据主体': sdq.dplyd_old(),
|
|
1119
|
+
},
|
|
1120
|
+
{
|
|
1121
|
+
'数据库名': '聚合数据',
|
|
1122
|
+
'集合名': '商品id编码表',
|
|
1123
|
+
'唯一主键': ['宝贝id'],
|
|
1124
|
+
'数据主体': sdq.idbm(),
|
|
1125
|
+
},
|
|
1126
|
+
{
|
|
1127
|
+
'数据库名': '聚合数据',
|
|
1128
|
+
'集合名': '商品id图片对照表',
|
|
1129
|
+
'唯一主键': ['商品id'],
|
|
1130
|
+
'数据主体': sdq.sp_picture(),
|
|
1131
|
+
},
|
|
1132
|
+
{
|
|
1133
|
+
'数据库名': '聚合数据',
|
|
1134
|
+
'集合名': '商品成本',
|
|
1135
|
+
'唯一主键': ['款号'],
|
|
1136
|
+
'数据主体': sdq.sp_cost(),
|
|
1137
|
+
},
|
|
1138
|
+
{
|
|
1139
|
+
'数据库名': '聚合数据',
|
|
1140
|
+
'集合名': '京东_京准通',
|
|
1141
|
+
'唯一主键': ['日期', '产品线', '触发sku id', '跟单sku id', '花费', ],
|
|
1142
|
+
'数据主体': sdq.jdjzt(),
|
|
1143
|
+
},
|
|
1144
|
+
{
|
|
1145
|
+
'数据库名': '聚合数据',
|
|
1146
|
+
'集合名': '京东_京准通_全站营销',
|
|
1147
|
+
'唯一主键': ['日期', '产品线', '花费'],
|
|
1148
|
+
'数据主体': sdq.jdqzyx(),
|
|
1149
|
+
},
|
|
1150
|
+
{
|
|
1151
|
+
'数据库名': '聚合数据',
|
|
1152
|
+
'集合名': '京东_sku_商品明细',
|
|
1153
|
+
'唯一主键': ['日期', '商品id', '成交单量'],
|
|
1154
|
+
'数据主体': sdq.sku_sales(),
|
|
1155
|
+
},
|
|
1156
|
+
{
|
|
1157
|
+
'数据库名': '聚合数据',
|
|
1158
|
+
'集合名': '天猫_人群报表',
|
|
1159
|
+
'唯一主键': ['日期', '推广渠道', '营销场景', '商品id', '花费', '人群名字'],
|
|
1160
|
+
'数据主体': sdq.tg_rqbb(),
|
|
1161
|
+
},
|
|
1162
|
+
{
|
|
1163
|
+
'数据库名': '聚合数据',
|
|
1164
|
+
'集合名': '天猫_关键词报表',
|
|
1165
|
+
'唯一主键': ['日期', '推广渠道', '营销场景', '商品id', '花费', '词类型', '词名字/词包名字',],
|
|
1166
|
+
'数据主体': sdq.tg_gjc(),
|
|
1167
|
+
},
|
|
1168
1168
|
{
|
|
1169
1169
|
'数据库名': '聚合数据',
|
|
1170
1170
|
'集合名': '天猫_超级直播',
|
|
@@ -1176,15 +1176,15 @@ def data_aggregation(service_databases=[{}], months=1):
|
|
|
1176
1176
|
db_name, table_name, unique_key_list, df = items['数据库名'], items['集合名'], items['唯一主键'], items['数据主体']
|
|
1177
1177
|
df = g.groupby(df=df, table_name=table_name, is_maximize=True) # 2. 聚合数据
|
|
1178
1178
|
if len(g.sp_index_datas) != 0:
|
|
1179
|
-
#
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1179
|
+
# 由推广主体报表，写入一个商品索引表，索引规则：从上月 1 号至今花费从高到低排序
|
|
1180
|
+
m.df_to_mysql(
|
|
1181
|
+
df=g.sp_index_datas,
|
|
1182
|
+
db_name='属性设置2',
|
|
1183
|
+
table_name='商品索引表',
|
|
1184
|
+
drop_duplicates=False,
|
|
1185
|
+
icm_update=['商品id'],
|
|
1186
|
+
service_database=service_database,
|
|
1187
|
+
)
|
|
1188
1188
|
g.sp_index_datas = pd.DataFrame() # 重置，不然下个循环会继续刷入数据库
|
|
1189
1189
|
# g.as_csv(df=df, filename=table_name + '.csv') # 导出 csv
|
|
1190
1190
|
m.df_to_mysql(
|
|
@@ -1195,24 +1195,24 @@ def data_aggregation(service_databases=[{}], months=1):
|
|
|
1195
1195
|
icm_update=unique_key_list,
|
|
1196
1196
|
service_database=service_database,
|
|
1197
1197
|
) # 3. 回传数据库
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1198
|
+
res = g.performance(bb_tg=True) # 盈亏表，依赖其他表，单独做
|
|
1199
|
+
m.df_to_mysql(
|
|
1200
|
+
df=res,
|
|
1201
|
+
db_name='聚合数据',
|
|
1202
|
+
table_name='_全店商品销售',
|
|
1203
|
+
drop_duplicates=False,
|
|
1204
|
+
icm_update=['日期', '商品id'], # 设置唯一主键
|
|
1205
|
+
service_database=service_database,
|
|
1206
|
+
)
|
|
1207
|
+
res = g.performance(bb_tg=False) # 盈亏表，依赖其他表，单独做
|
|
1208
|
+
m.df_to_mysql(
|
|
1209
|
+
df=res,
|
|
1210
|
+
db_name='聚合数据',
|
|
1211
|
+
table_name='_推广商品销售',
|
|
1212
|
+
drop_duplicates=False,
|
|
1213
|
+
icm_update=['日期', '商品id'], # 设置唯一主键
|
|
1214
|
+
service_database=service_database,
|
|
1215
|
+
)
|
|
1216
1216
|
|
|
1217
1217
|
res = g.performance_concat(bb_tg=False) # 推广主体合并直播表，依赖其他表，单独做
|
|
1218
1218
|
m.df_to_mysql(
|
|
@@ -1225,15 +1225,15 @@ def data_aggregation(service_databases=[{}], months=1):
|
|
|
1225
1225
|
)
|
|
1226
1226
|
|
|
1227
1227
|
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1228
|
+
res = g.performance_jd(jd_tg=False) # 盈亏表，依赖其他表，单独做
|
|
1229
|
+
m.df_to_mysql(
|
|
1230
|
+
df=res,
|
|
1231
|
+
db_name='聚合数据',
|
|
1232
|
+
table_name='_京东_推广商品销售',
|
|
1233
|
+
drop_duplicates=False,
|
|
1234
|
+
icm_update=['日期', '跟单sku id', '货号', '花费'], # 设置唯一主键
|
|
1235
|
+
service_database=service_database,
|
|
1236
|
+
)
|
|
1237
1237
|
|
|
1238
1238
|
|
|
1239
1239
|
# 这里要注释掉，不然 copysh.py 可能有问题，这里主要修改配置文件，后续触发 home_lx 的 optimize_datas.py(有s)程序进行全局清理
|
mdbq/clean/data_clean.py
CHANGED
|
@@ -683,6 +683,20 @@ class DataClean:
|
|
|
683
683
|
m.df_to_mysql(df=df, db_name='天猫数据1', tabel_name='万相台_人群洞察')
|
|
684
684
|
|
|
685
685
|
# ----------------------- 京东数据处理分界线 -----------------------
|
|
686
|
+
elif name.endswith('.csv') and '关键词点击成交报表_pbix同步_勿删改' in name:
|
|
687
|
+
df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=0, na_filter=False)
|
|
688
|
+
for col in df.columns.tolist():
|
|
689
|
+
if '（' in col:
|
|
690
|
+
new_col = re.sub('[（）]', '_', col)
|
|
691
|
+
new_col = new_col.strip('_')
|
|
692
|
+
df.rename(columns={col: new_col}, inplace=True)
|
|
693
|
+
df['日期'] = df['日期'].apply(lambda x: f'{str(x)[:4]}-{str(x)[4:6]}-{str(x)[6:8]}')
|
|
694
|
+
df['日期'] = pd.to_datetime(df['日期'], format='%Y-%m-%d', errors='ignore')
|
|
695
|
+
min_clm = str(df['日期'].min()).split(' ')[0]
|
|
696
|
+
max_clm = str(df['日期'].max()).split(' ')[0]
|
|
697
|
+
new_name = f'京东推广关键词点击成交报表_{min_clm}_{max_clm}.csv'
|
|
698
|
+
self.save_to_csv(df, root, new_name)
|
|
699
|
+
os.remove(os.path.join(root, name))
|
|
686
700
|
elif name.endswith('.csv') and '营销概况_全站营销' in name:
|
|
687
701
|
df = pd.read_csv(os.path.join(root, name), encoding='utf-8_sig', header=1, na_filter=False)
|
|
688
702
|
df = df[(df['日期'] != '日期') & (df['日期'] != '汇总') & (df['日期'] != '0') & (df['花费'] != '0') & (df['花费'] != '0.00')]
|
|
@@ -1199,6 +1213,9 @@ class DataClean:
|
|
|
1199
1213
|
elif name.endswith('.csv') and '营销概况_全站营销' in name:
|
|
1200
1214
|
t_path = str(pathlib.Path(self.source_path, '京东报表/JD推广_全站营销报表'))
|
|
1201
1215
|
bib(t_path, _as_month=True)
|
|
1216
|
+
elif name.endswith('.csv') and '京东推广关键词点击成交报表' in name:
|
|
1217
|
+
t_path = str(pathlib.Path(self.source_path, '京东报表/JD推广_关键词报表'))
|
|
1218
|
+
bib(t_path, _as_month=True)
|
|
1202
1219
|
# 京东分界线 ------- 结束标记
|
|
1203
1220
|
|
|
1204
1221
|
def attribute(self, path=None, _str='商品素材导出', ):
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
mdbq/__init__.py,sha256=Il5Q9ATdX8yXqVxtP_nYqUhExzxPC_qk_WXQ_4h0exg,16
|
|
2
2
|
mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
|
|
3
3
|
mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
|
|
4
|
-
mdbq/aggregation/aggregation.py,sha256=
|
|
4
|
+
mdbq/aggregation/aggregation.py,sha256=cydwGivXyoLzvDrbhn8BR-rBRLnY8VjJK9R16pVkR4s,62979
|
|
5
5
|
mdbq/aggregation/df_types.py,sha256=oQJS2IBU3_IO6GMgbssHuC2yCjNnbta0QPGrFOwNLnU,7591
|
|
6
6
|
mdbq/aggregation/mysql_types.py,sha256=DQYROALDiwjJzjhaJfIIdnsrNs11i5BORlj_v6bp67Y,11062
|
|
7
7
|
mdbq/aggregation/optimize_data.py,sha256=u2Kl_MFtZueXJ57ycy4H2OhXD431RctUYJYCl637uT0,4176
|
|
8
|
-
mdbq/aggregation/query_data.py,sha256=
|
|
8
|
+
mdbq/aggregation/query_data.py,sha256=N7y9bzmoK3hnurpA2hbYSJ6IMznj3D7NzmrlQo5gGg0,59148
|
|
9
9
|
mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
|
|
10
10
|
mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
|
|
11
11
|
mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
|
|
12
|
-
mdbq/clean/data_clean.py,sha256=
|
|
12
|
+
mdbq/clean/data_clean.py,sha256=dVycc_wyXH96hJvAXw-lVAZV-3eCEGD-o04oHxAecHs,92545
|
|
13
13
|
mdbq/company/__init__.py,sha256=qz8F_GsP_pMB5PblgJAUAMjasuZbOEp3qQOCB39E8f0,21
|
|
14
14
|
mdbq/company/copysh.py,sha256=WCZ92vCJAy6_ZFeOxWL-U9gArIpyga4xts-s1wKsspY,17268
|
|
15
15
|
mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
|
|
@@ -35,7 +35,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
|
|
|
35
35
|
mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
|
|
36
36
|
mdbq/pbix/refresh_all.py,sha256=tgy762608HMaXWynbOURIf2UVMuSPybzrDXQnOOcnZU,6102
|
|
37
37
|
mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
|
|
38
|
-
mdbq-1.7.
|
|
39
|
-
mdbq-1.7.
|
|
40
|
-
mdbq-1.7.
|
|
41
|
-
mdbq-1.7.
|
|
38
|
+
mdbq-1.7.2.dist-info/METADATA,sha256=oY3Kodabng2WfHmpryLIpCIry7d6w4Bt-F5KwwIzNfs,245
|
|
39
|
+
mdbq-1.7.2.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
|
|
40
|
+
mdbq-1.7.2.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
|
|
41
|
+
mdbq-1.7.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|