mdbq 1.2.7__py3-none-any.whl → 1.2.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mdbq/aggregation/query_data.py +11 -6
- mdbq/mysql/mysql.py +11 -5
- {mdbq-1.2.7.dist-info → mdbq-1.2.8.dist-info}/METADATA +1 -1
- {mdbq-1.2.7.dist-info → mdbq-1.2.8.dist-info}/RECORD +6 -6
- {mdbq-1.2.7.dist-info → mdbq-1.2.8.dist-info}/WHEEL +0 -0
- {mdbq-1.2.7.dist-info → mdbq-1.2.8.dist-info}/top_level.txt +0 -0
mdbq/aggregation/query_data.py
CHANGED
|
@@ -388,19 +388,22 @@ class GroupBy:
|
|
|
388
388
|
self.data_tgyj['商品id图片对照表'],
|
|
389
389
|
self.data_tgyj['商品成本']) # 这里不要加逗号
|
|
390
390
|
pic['商品id'] = pic['商品id'].astype(str)
|
|
391
|
-
df = pd.merge(idbm, pic, how='left', left_on='宝贝id', right_on='商品id')
|
|
391
|
+
df = pd.merge(idbm, pic, how='left', left_on='宝贝id', right_on='商品id') # id 编码表合并图片表
|
|
392
392
|
df = df[['宝贝id', '商家编码', '商品图片']]
|
|
393
|
-
df = pd.merge(df, cost, how='left', left_on='商家编码', right_on='款号')
|
|
393
|
+
df = pd.merge(df, cost, how='left', left_on='商家编码', right_on='款号') # df 合并商品成本表
|
|
394
394
|
df = df[['宝贝id', '商家编码', '商品图片', '成本价']]
|
|
395
|
-
df = pd.merge(tg, df, how='left', left_on='商品id', right_on='宝贝id')
|
|
395
|
+
df = pd.merge(tg, df, how='left', left_on='商品id', right_on='宝贝id') # 推广表合并 df
|
|
396
396
|
df.drop(labels='宝贝id', axis=1, inplace=True)
|
|
397
397
|
if bb_tg is True:
|
|
398
398
|
# 生意经合并推广表，完整的数据表，包含全店所有推广、销售数据
|
|
399
399
|
df = pd.merge(syj, df, how='left', left_on=['日期', '宝贝id'], right_on=['日期', '商品id'])
|
|
400
|
+
df.drop(labels='商品id', axis=1, inplace=True) # 因为生意经中的宝贝 id 列才是完整的
|
|
401
|
+
df.rename(columns={'宝贝id': '商品id'}, inplace=True)
|
|
402
|
+
# df.to_csv('/Users/xigua/Downloads/test.csv', encoding='utf-8_sig', index=False, header=True)
|
|
400
403
|
else:
|
|
401
404
|
# 推广表合并生意经 , 以推广数据为基准，销售数据不齐全
|
|
402
405
|
df = pd.merge(df, syj, how='left', left_on=['日期', '商品id'], right_on=['日期', '宝贝id'])
|
|
403
|
-
|
|
406
|
+
df.drop(labels='宝贝id', axis=1, inplace=True)
|
|
404
407
|
df.drop_duplicates(subset=['日期', '商品id', '花费', '销售额'], keep='last', inplace=True, ignore_index=True)
|
|
405
408
|
df['成本价'] = df['成本价'].astype('float64')
|
|
406
409
|
df['商品成本'] = df.apply(lambda x: (x['成本价'] + x['销售额']/x['销售量'] * 0.11 + 6) * x['销售量'] if x['销售量'] > 0 else 0, axis=1)
|
|
@@ -425,6 +428,8 @@ class GroupBy:
|
|
|
425
428
|
path = os.path.join(self.output, path)
|
|
426
429
|
if not os.path.exists(path):
|
|
427
430
|
os.makedirs(path)
|
|
431
|
+
if filename.endswith('.csv'):
|
|
432
|
+
filename = filename[:-4]
|
|
428
433
|
if st_ascend and ascend:
|
|
429
434
|
try:
|
|
430
435
|
df.sort_values(st_ascend, ascending=ascend, ignore_index=True, inplace=True)
|
|
@@ -510,7 +515,7 @@ def data_aggregation(service_databases=[{}]):
|
|
|
510
515
|
{
|
|
511
516
|
'数据库名': '聚合数据',
|
|
512
517
|
'集合名': '宝贝主体报表',
|
|
513
|
-
'唯一主键': ['日期', '推广渠道', '营销场景', '商品id'],
|
|
518
|
+
'唯一主键': ['日期', '推广渠道', '营销场景', '商品id', '花费'],
|
|
514
519
|
'数据主体': sdq.tg_wxt(),
|
|
515
520
|
},
|
|
516
521
|
{
|
|
@@ -547,7 +552,7 @@ def data_aggregation(service_databases=[{}]):
|
|
|
547
552
|
for items in data_dict: # 遍历返回结果
|
|
548
553
|
db_name, table_name, unique_key_list, df = items['数据库名'], items['集合名'], items['唯一主键'], items['数据主体']
|
|
549
554
|
df = g.groupby(df=df, table_name=table_name, is_maximize=True) # 2. 聚合数据
|
|
550
|
-
|
|
555
|
+
g.as_csv(df=df, filename=table_name + '.csv') # 导出 csv
|
|
551
556
|
m.df_to_mysql(
|
|
552
557
|
df=df,
|
|
553
558
|
db_name=db_name,
|
mdbq/mysql/mysql.py
CHANGED
|
@@ -195,7 +195,6 @@ class MysqlUpload:
|
|
|
195
195
|
datas = df.to_dict(orient='records')
|
|
196
196
|
for data in datas:
|
|
197
197
|
# data 是传进来待处理的数据, 不是数据库数据
|
|
198
|
-
# print(data)
|
|
199
198
|
# data 示例: {'日期': Timestamp('2024-08-27 00:00:00'), '推广费余额': 33299, '品销宝余额': 2930.73, '短信剩余': 67471}
|
|
200
199
|
try:
|
|
201
200
|
cols = ', '.join(f"`{item}`" for item in data.keys()) # 列名需要转义
|
|
@@ -236,17 +235,24 @@ class MysqlUpload:
|
|
|
236
235
|
# sql = f"SELECT {unique_keys} FROM `{table_name}` WHERE `创建时间` = '2014-09-19 14:32:33'"
|
|
237
236
|
cursor.execute(sql)
|
|
238
237
|
results = cursor.fetchall() # results 是数据库取出的数据
|
|
239
|
-
if results: #
|
|
238
|
+
if results: # 有数据返回，再进行增量检查
|
|
240
239
|
for result in results: # results 是数据库数据, data 是传进来的数据
|
|
241
240
|
not_change_col = []
|
|
242
241
|
change_values = []
|
|
243
242
|
for col in update_col:
|
|
244
243
|
# 因为 mysql 里面有 decimal 数据类型，要移除末尾的 0 再做比较（df 默认将 5.00 小数截断为 5.0）
|
|
245
|
-
|
|
246
|
-
|
|
244
|
+
df_value = str(data[col])
|
|
245
|
+
mysql_value = str(result[col])
|
|
246
|
+
if '.' in df_value:
|
|
247
|
+
df_value = re.sub('0+$', '', df_value)
|
|
248
|
+
df_value = re.sub('\.$', '', df_value)
|
|
249
|
+
if '.' in mysql_value:
|
|
250
|
+
mysql_value = re.sub('0+$', '', mysql_value)
|
|
251
|
+
mysql_value = re.sub('\.$', '', mysql_value)
|
|
252
|
+
if df_value != mysql_value: # 传进来的数据和数据库比较, 有变化
|
|
253
|
+
# print(f'{data['日期']}{data['商品id']}{col} 列的值有变化，{str(data[col])} != {str(result[col])}')
|
|
247
254
|
change_values += [f"`{col}` = '{str(data[col])}'"]
|
|
248
255
|
not_change_col += [item for item in update_col if item != col]
|
|
249
|
-
|
|
250
256
|
# change_values 是 df 传进来且和数据库对比后，发生了变化的数据，值示例： [`品销宝余额` = '9999.0', `短信剩余` = '888']
|
|
251
257
|
if change_values: # change_values 有数据返回，表示值需要更新
|
|
252
258
|
not_change_values = [f"`{col}` = '{str(data[col])}'" for col in not_change_col]
|
|
@@ -5,7 +5,7 @@ mdbq/aggregation/aggregation.py,sha256=ukOtdTJNXoCM0M1Nhrax4J5rJoWLSVYCw55TnrNSt
|
|
|
5
5
|
mdbq/aggregation/df_types.py,sha256=rHLIgv82PJSFmDvXkZyOJAffXkFyyMyFO23w9tUt8EQ,7525
|
|
6
6
|
mdbq/aggregation/mysql_types.py,sha256=_XIqpaX_qmqolFlGywMYfvBn32u8MbPCaX6n7rQOVRQ,10634
|
|
7
7
|
mdbq/aggregation/optimize_data.py,sha256=jLAWtxPUuhpo4XTVrhKtT4xK3grs7r73ePQfLhxlu1I,779
|
|
8
|
-
mdbq/aggregation/query_data.py,sha256=
|
|
8
|
+
mdbq/aggregation/query_data.py,sha256=kNX9htViFN0EnpF7D_eOQtTWy8BIa5-yJmJiqY7f8ds,27083
|
|
9
9
|
mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
|
|
10
10
|
mdbq/bdup/bdup.py,sha256=LAV0TgnQpc-LB-YuJthxb0U42_VkPidzQzAagan46lU,4234
|
|
11
11
|
mdbq/clean/__init__.py,sha256=A1d6x3L27j4NtLgiFV5TANwEkLuaDfPHDQNrPBbNWtU,41
|
|
@@ -24,7 +24,7 @@ mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
|
|
|
24
24
|
mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
|
|
25
25
|
mdbq/mongo/mongo.py,sha256=v9qvrp6p1ZRWuPpbSilqveiE0FEcZF7U5xUPI0RN4xs,31880
|
|
26
26
|
mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
|
|
27
|
-
mdbq/mysql/mysql.py,sha256=
|
|
27
|
+
mdbq/mysql/mysql.py,sha256=4Omt9Su0Cv-oRDGBKUi4_62wbs8OfDE5ssoHIWn3Kys,44328
|
|
28
28
|
mdbq/mysql/s_query.py,sha256=a33aYhW6gAnspIZfQ7l23ePln9-MD1f_ukypr5M0jd8,8018
|
|
29
29
|
mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
|
|
30
30
|
mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
|
|
@@ -35,7 +35,7 @@ mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
|
|
|
35
35
|
mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
|
|
36
36
|
mdbq/pbix/refresh_all.py,sha256=tgy762608HMaXWynbOURIf2UVMuSPybzrDXQnOOcnZU,6102
|
|
37
37
|
mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
|
|
38
|
-
mdbq-1.2.
|
|
39
|
-
mdbq-1.2.
|
|
40
|
-
mdbq-1.2.
|
|
41
|
-
mdbq-1.2.
|
|
38
|
+
mdbq-1.2.8.dist-info/METADATA,sha256=m6-ftUmS0npMhvz1brisQdXGN4Kc7jTAXBueFeE4HkE,245
|
|
39
|
+
mdbq-1.2.8.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
|
|
40
|
+
mdbq-1.2.8.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
|
|
41
|
+
mdbq-1.2.8.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|