mdbq 3.7.8__py3-none-any.whl → 3.7.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mdbq/aggregation/query_data.py +4 -9
- mdbq/config/default.py +8 -6
- mdbq/config/set_support.py +12 -9
- mdbq/log/spider_logging.py +11 -9
- mdbq/mysql/mysql.py +0 -16
- mdbq/other/download_sku_picture.py +1 -1
- mdbq/redis/getredis.py +4 -1
- mdbq/spider/aikucun.py +2 -1
- {mdbq-3.7.8.dist-info → mdbq-3.7.10.dist-info}/METADATA +1 -1
- {mdbq-3.7.8.dist-info → mdbq-3.7.10.dist-info}/RECORD +12 -13
- mdbq/other/sku_picture_bak.py +0 -1081
- {mdbq-3.7.8.dist-info → mdbq-3.7.10.dist-info}/WHEEL +0 -0
- {mdbq-3.7.8.dist-info → mdbq-3.7.10.dist-info}/top_level.txt +0 -0
mdbq/aggregation/query_data.py
CHANGED
@@ -5,7 +5,6 @@ from mdbq.mysql import mysql
|
|
5
5
|
from mdbq.mysql import s_query
|
6
6
|
from mdbq.aggregation import optimize_data
|
7
7
|
from mdbq.config import products
|
8
|
-
from mdbq.config import set_support
|
9
8
|
from mdbq.config import default
|
10
9
|
import datetime
|
11
10
|
from dateutil.relativedelta import relativedelta
|
@@ -21,11 +20,12 @@ import calendar
|
|
21
20
|
import concurrent.futures
|
22
21
|
import traceback
|
23
22
|
import logging
|
23
|
+
import sys
|
24
24
|
|
25
25
|
"""
|
26
26
|
|
27
27
|
"""
|
28
|
-
|
28
|
+
dirname = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'support')
|
29
29
|
m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname='xigua_lx', sql='mysql', local='remoto', config_file=None)
|
30
30
|
logger = logging.getLogger(__name__)
|
31
31
|
|
@@ -42,7 +42,8 @@ class MysqlDatasQuery:
|
|
42
42
|
self.update_service = True # 调试时加,true: 将数据写入 mysql 服务器
|
43
43
|
self.pf_datas = []
|
44
44
|
self.pf_datas_jd = [] # 京东聚合销售表
|
45
|
-
self.output = set_support.SetSupport(dirname='support')
|
45
|
+
# self.output = set_support.SetSupport(dirname='support')
|
46
|
+
self.output = dirname
|
46
47
|
|
47
48
|
@staticmethod
|
48
49
|
def try_except(func): # 在类内部定义一个异常处理方法
|
@@ -52,12 +53,6 @@ class MysqlDatasQuery:
|
|
52
53
|
return func(*args, **kwargs)
|
53
54
|
except Exception as e:
|
54
55
|
logger.info(f'函数: {func.__name__}, 报错信息: {e}') # 将异常信息返回
|
55
|
-
with open(error_file, 'a', encoding='utf-8') as f:
|
56
|
-
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
57
|
-
f.write(f'\n{now}\n')
|
58
|
-
# f.write(f'报错的文件:\n{e.__traceback__.tb_frame.f_globals["__file__"]}\n') # 发生异常所在的文件
|
59
|
-
traceback.print_exc(file=open(error_file, 'a')) # 返回完整的堆栈信息
|
60
|
-
logger.info(f'更多信息请查看日志文件: {error_file}')
|
61
56
|
|
62
57
|
return wrapper
|
63
58
|
|
mdbq/config/default.py
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
# -*- coding: UTF-8 –*-
|
2
2
|
import os
|
3
|
+
import sys
|
3
4
|
import json
|
4
5
|
import platform
|
5
6
|
import getpass
|
@@ -7,12 +8,13 @@ import socket
|
|
7
8
|
import logging
|
8
9
|
from mdbq.mysql import mysql
|
9
10
|
|
10
|
-
if platform.system() == 'Windows':
|
11
|
-
|
12
|
-
elif platform.system() == 'Darwin':
|
13
|
-
|
14
|
-
else:
|
15
|
-
|
11
|
+
# if platform.system() == 'Windows':
|
12
|
+
# support_path = r'C:\数据中心2\support'
|
13
|
+
# elif platform.system() == 'Darwin':
|
14
|
+
# support_path = f'/Users/{getpass.getuser()}/数据中心2/support'
|
15
|
+
# else:
|
16
|
+
# support_path = '数据中心2/support' # 没有用, 可以删
|
17
|
+
support_path = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'support')
|
16
18
|
logger = logging.getLogger(__name__)
|
17
19
|
|
18
20
|
|
mdbq/config/set_support.py
CHANGED
@@ -2,22 +2,25 @@
|
|
2
2
|
import platform
|
3
3
|
import getpass
|
4
4
|
import os
|
5
|
+
import sys
|
5
6
|
|
6
7
|
"""
|
7
8
|
专门用来设置 support 文件夹路径
|
8
9
|
support 文件夹包含很多配置类文件,是程序必不可少的依赖
|
9
10
|
"""
|
11
|
+
|
12
|
+
|
10
13
|
class SetSupport:
|
11
14
|
def __init__(self, dirname):
|
12
|
-
self.dirname = dirname
|
13
|
-
if platform.system() == 'Windows':
|
14
|
-
|
15
|
-
elif platform.system() == 'Darwin':
|
16
|
-
|
17
|
-
else:
|
18
|
-
|
19
|
-
if not os.path.isdir(self.dirname):
|
20
|
-
|
15
|
+
self.dirname = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), dirname)
|
16
|
+
# if platform.system() == 'Windows':
|
17
|
+
# self.dirname = f'C:\\数据中心2\\support'
|
18
|
+
# elif platform.system() == 'Darwin':
|
19
|
+
# self.dirname = f'/Users/{getpass.getuser()}/数据中心2/support'
|
20
|
+
# else:
|
21
|
+
# self.dirname = '数据中心2/support' # 没有用, 可以删
|
22
|
+
# if not os.path.isdir(self.dirname):
|
23
|
+
# os.mkdir(self.dirname)
|
21
24
|
|
22
25
|
|
23
26
|
if __name__ == '__main__':
|
mdbq/log/spider_logging.py
CHANGED
@@ -2,6 +2,7 @@ import logging
|
|
2
2
|
from logging.handlers import RotatingFileHandler
|
3
3
|
import platform
|
4
4
|
import os
|
5
|
+
import sys
|
5
6
|
import getpass
|
6
7
|
|
7
8
|
|
@@ -9,15 +10,16 @@ def setup_logging(reMoveOldHandler=True):
|
|
9
10
|
"""
|
10
11
|
reMoveOldHandler: 替换根日志记录器的所有现有处理器
|
11
12
|
"""
|
12
|
-
if platform.system() == 'Windows':
|
13
|
-
|
14
|
-
|
15
|
-
elif platform.system() == 'Linux':
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
else:
|
20
|
-
|
13
|
+
# if platform.system() == 'Windows':
|
14
|
+
# from mdbq.pbix import refresh_all
|
15
|
+
# D_PATH = os.path.join(f'C:\\Users\\{getpass.getuser()}\\Downloads')
|
16
|
+
# elif platform.system() == 'Linux':
|
17
|
+
# D_PATH = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'Downloads')
|
18
|
+
# if not os.path.exists(D_PATH):
|
19
|
+
# os.makedirs(D_PATH)
|
20
|
+
# else:
|
21
|
+
# D_PATH = os.path.join(f'/Users/{getpass.getuser()}/Downloads')
|
22
|
+
D_PATH = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])))
|
21
23
|
|
22
24
|
if not os.path.isdir(os.path.join(D_PATH, 'logfile')):
|
23
25
|
os.makedirs(os.path.join(D_PATH, 'logfile'))
|
mdbq/mysql/mysql.py
CHANGED
@@ -13,7 +13,6 @@ import pandas as pd
|
|
13
13
|
from sqlalchemy import create_engine
|
14
14
|
import os
|
15
15
|
import calendar
|
16
|
-
from mdbq.config import set_support
|
17
16
|
from mdbq.config import myconfig
|
18
17
|
import traceback
|
19
18
|
import logging
|
@@ -29,7 +28,6 @@ warnings.filterwarnings('ignore')
|
|
29
28
|
3. 小数必须使用 decimal, 禁止 float 和 double, 因为计算精度差异,后续需要聚合数据时会引发很多问题
|
30
29
|
|
31
30
|
"""
|
32
|
-
error_file = os.path.join(set_support.SetSupport(dirname='support').dirname, 'error.log')
|
33
31
|
logger = logging.getLogger(__name__)
|
34
32
|
|
35
33
|
|
@@ -122,13 +120,6 @@ class MysqlUpload:
|
|
122
120
|
return func(*args, **kwargs)
|
123
121
|
except Exception as e:
|
124
122
|
logger.error(f'{func.__name__}, {e}') # 将异常信息返回
|
125
|
-
with open(error_file, 'a', encoding='utf-8') as f:
|
126
|
-
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
127
|
-
f.write(f'\n{now} \n')
|
128
|
-
f.write(f'函数注释内容(用于定位函数): {func.__doc__} \n')
|
129
|
-
# f.write(f'报错的文件:\n{e.__traceback__.tb_frame.f_globals["__file__"]}\n') # 发生异常所在的文件
|
130
|
-
traceback.print_exc(file=open(error_file, 'a')) # 返回完整的堆栈信息
|
131
|
-
logger.error(f'更多信息请查看日志文件: {error_file}')
|
132
123
|
|
133
124
|
return wrapper
|
134
125
|
|
@@ -1434,13 +1425,6 @@ class OptimizeDatas:
|
|
1434
1425
|
return func(*args, **kwargs)
|
1435
1426
|
except Exception as e:
|
1436
1427
|
logger.error(f'{func.__name__}, {e}') # 将异常信息返回
|
1437
|
-
with open(error_file, 'a') as f:
|
1438
|
-
now = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
1439
|
-
f.write(f'\n{now} \n')
|
1440
|
-
f.write(f'函数注释内容(用于定位函数): {func.__doc__} \n')
|
1441
|
-
# f.write(f'报错的文件:\n{e.__traceback__.tb_frame.f_globals["__file__"]}\n') # 发生异常所在的文件
|
1442
|
-
traceback.print_exc(file=open(error_file, 'a')) # 返回完整的堆栈信息
|
1443
|
-
logger.error(f'更多信息请查看日志文件: {error_file}')
|
1444
1428
|
|
1445
1429
|
return wrapper
|
1446
1430
|
|
@@ -44,7 +44,7 @@ elif platform.system() == 'Darwin':
|
|
44
44
|
D_PATH = os.path.normpath(f'/Users/{getpass.getuser()}/Downloads')
|
45
45
|
Share_Path = os.path.normpath('/Volumes/时尚事业部/01.运营部/天猫报表') # 共享文件根目录
|
46
46
|
else:
|
47
|
-
D_PATH = 'Downloads'
|
47
|
+
D_PATH = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'Downloads')
|
48
48
|
Share_Path = ''
|
49
49
|
upload_path = os.path.join(D_PATH, '数据上传中心') # 此目录位于下载文件夹,将统一上传百度云备份
|
50
50
|
if not os.path.exists(upload_path): # 数据中心根目录
|
mdbq/redis/getredis.py
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
# -*- coding: UTF-8 –*-
|
2
2
|
import os.path
|
3
|
+
import sys
|
3
4
|
import random
|
4
5
|
import redis
|
5
6
|
import socket
|
@@ -20,7 +21,9 @@ import orjson
|
|
20
21
|
if platform.system() == 'Windows':
|
21
22
|
D_PATH = os.path.join(f'C:\\Users\\{getpass.getuser()}\\Downloads')
|
22
23
|
else:
|
23
|
-
D_PATH = os.path.join(
|
24
|
+
D_PATH = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'Downloads')
|
25
|
+
if not os.path.exists(D_PATH):
|
26
|
+
os.makedir(D_PATH)
|
24
27
|
|
25
28
|
m_engine, username, password, host, port = default.get_mysql_engine(platform='Windows', hostname='xigua_lx', sql='mysql', local='remoto', config_file=None)
|
26
29
|
|
mdbq/spider/aikucun.py
CHANGED
@@ -3,6 +3,7 @@ import datetime
|
|
3
3
|
import getpass
|
4
4
|
import json
|
5
5
|
import os
|
6
|
+
import sys
|
6
7
|
import pathlib
|
7
8
|
import platform
|
8
9
|
import re
|
@@ -30,7 +31,7 @@ if platform.system() == 'Windows':
|
|
30
31
|
# windows版本
|
31
32
|
D_PATH = str(pathlib.Path(f'C:\\Users\\{getpass.getuser()}\\Downloads'))
|
32
33
|
elif platform.system() == 'Linux':
|
33
|
-
D_PATH = 'Downloads'
|
34
|
+
D_PATH = os.path.join(os.path.realpath(os.path.dirname(sys.argv[0])), 'Downloads')
|
34
35
|
if not os.path.exists(D_PATH):
|
35
36
|
os.makedirs(D_PATH)
|
36
37
|
else:
|
@@ -3,39 +3,38 @@ mdbq/__version__.py,sha256=y9Mp_8x0BCZSHsdLT_q5tX9wZwd5QgqrSIENLrb6vXA,62
|
|
3
3
|
mdbq/aggregation/__init__.py,sha256=EeDqX2Aml6SPx8363J-v1lz0EcZtgwIBYyCJV6CcEDU,40
|
4
4
|
mdbq/aggregation/datashow_bak.py,sha256=1AYSIDkdUx-4as1Ax2rPj0cExM9d-qFMrFYLAaPHNuk,54962
|
5
5
|
mdbq/aggregation/optimize_data.py,sha256=ZR6iXO25EMWyiy-oy_T8H8K9xA8q1ecC8gmUbOqJOOc,910
|
6
|
-
mdbq/aggregation/query_data.py,sha256=
|
6
|
+
mdbq/aggregation/query_data.py,sha256=jGKnHm9vRlKCZeIdP15lgpASKaQgoQjPZyd2IsTJY6k,174426
|
7
7
|
mdbq/bdup/__init__.py,sha256=AkhsGk81SkG1c8FqDH5tRq-8MZmFobVbN60DTyukYTY,28
|
8
8
|
mdbq/bdup/bdup.py,sha256=hJs815hGFwm_X5bP2i9XugG2w2ZY_F0n3-Q0hVpIPPw,4892
|
9
9
|
mdbq/config/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
|
10
|
-
mdbq/config/default.py,sha256=
|
10
|
+
mdbq/config/default.py,sha256=tbfjR5lMkvNmwdivtnBxnKVqH5PwFpPM9jaZ9JIOkv4,5088
|
11
11
|
mdbq/config/myconfig.py,sha256=Akt-7KqSUBdoHQa4_Mw6YC4pA75219d-21iDO30iaD8,894
|
12
12
|
mdbq/config/products.py,sha256=FbBIqmyaiq9h03FIeE9W2bwbLm2_5pr6xyzPV-u7Ges,5689
|
13
|
-
mdbq/config/set_support.py,sha256=
|
13
|
+
mdbq/config/set_support.py,sha256=eM2scqDzGNR2kkbYmguRB2ucrqeX0KMh5OMIaGsLem4,877
|
14
14
|
mdbq/dataframe/__init__.py,sha256=2HtCN8AdRj53teXDqzysC1h8aPL-mMFy561ESmhehGQ,22
|
15
15
|
mdbq/dataframe/converter.py,sha256=lETYhT7KXlWzWwqguqhk6vI6kj4rnOBEW1lhqKy2Abc,5035
|
16
16
|
mdbq/log/__init__.py,sha256=Mpbrav0s0ifLL7lVDAuePEi1hJKiSHhxcv1byBKDl5E,15
|
17
17
|
mdbq/log/mylogger.py,sha256=oaT7Bp-Hb9jZt52seP3ISUuxVcI19s4UiqTeouScBO0,3258
|
18
|
-
mdbq/log/spider_logging.py,sha256=
|
18
|
+
mdbq/log/spider_logging.py,sha256=cTI9BHePz9Q4QThc_OBxTLqX5tF9iPwIQ-dASpf6zCA,2142
|
19
19
|
mdbq/mongo/__init__.py,sha256=SILt7xMtQIQl_m-ik9WLtJSXIVf424iYgCfE_tnQFbw,13
|
20
20
|
mdbq/mysql/__init__.py,sha256=A_DPJyAoEvTSFojiI2e94zP0FKtCkkwKP1kYUCSyQzo,11
|
21
|
-
mdbq/mysql/mysql.py,sha256=
|
21
|
+
mdbq/mysql/mysql.py,sha256=SY_1UxoB_fQoPTsQeBbThHDsHFAK5Jr7csbzE8z3GPg,97303
|
22
22
|
mdbq/mysql/s_query.py,sha256=pj5ioJfUT81Su9S-km9G49gF5F2MmXXfw_oAIUzhN28,8794
|
23
23
|
mdbq/mysql/year_month_day.py,sha256=VgewoE2pJxK7ErjfviL_SMTN77ki8GVbTUcao3vFUCE,1523
|
24
24
|
mdbq/other/__init__.py,sha256=jso1oHcy6cJEfa7udS_9uO5X6kZLoPBF8l3wCYmr5dM,18
|
25
|
-
mdbq/other/download_sku_picture.py,sha256=
|
25
|
+
mdbq/other/download_sku_picture.py,sha256=uMC6H7SVEFj3cuS8IJNPOY3v4oJVtGOE-vb2XJ7Q0tM,44718
|
26
26
|
mdbq/other/porxy.py,sha256=UHfgEyXugogvXgsG68a7QouUCKaohTKKkI4RN-kYSdQ,4961
|
27
27
|
mdbq/other/pov_city.py,sha256=AEOmCOzOwyjHi9LLZWPKi6DUuSC-_M163664I52u9qw,21050
|
28
|
-
mdbq/other/sku_picture_bak.py,sha256=JwSXYlzamVqcKCD2tRH2VqYVZNr8fM6f--kcGlTVRnM,50026
|
29
28
|
mdbq/other/ua_sj.py,sha256=JuVYzc_5QZ9s_oQSrTHVKkQv4S_7-CWx4oIKOARn_9U,22178
|
30
29
|
mdbq/pbix/__init__.py,sha256=Trtfaynu9RjoTyLLYBN2xdRxTvm_zhCniUkVTAYwcjo,24
|
31
30
|
mdbq/pbix/pbix_refresh.py,sha256=JUjKW3bNEyoMVfVfo77UhguvS5AWkixvVhDbw4_MHco,2396
|
32
31
|
mdbq/pbix/refresh_all.py,sha256=OBT9EewSZ0aRS9vL_FflVn74d4l2G00wzHiikCC4TC0,5926
|
33
32
|
mdbq/pbix/refresh_all_old.py,sha256=_pq3WSQ728GPtEG5pfsZI2uTJhU8D6ra-htIk1JXYzw,7192
|
34
33
|
mdbq/redis/__init__.py,sha256=YtgBlVSMDphtpwYX248wGge1x-Ex_mMufz4-8W0XRmA,12
|
35
|
-
mdbq/redis/getredis.py,sha256=
|
34
|
+
mdbq/redis/getredis.py,sha256=8ONSVHEeEY6C68OxAMAR4jB-CtA3mZyd-v6vs94zlnU,25732
|
36
35
|
mdbq/spider/__init__.py,sha256=RBMFXGy_jd1HXZhngB2T2XTvJqki8P_Fr-pBcwijnew,18
|
37
|
-
mdbq/spider/aikucun.py,sha256=
|
38
|
-
mdbq-3.7.
|
39
|
-
mdbq-3.7.
|
40
|
-
mdbq-3.7.
|
41
|
-
mdbq-3.7.
|
36
|
+
mdbq/spider/aikucun.py,sha256=HK6x0MnpUN1jUSWnLQ_UEH2QSYcLFBGhhUBD3FVwrzY,21840
|
37
|
+
mdbq-3.7.10.dist-info/METADATA,sha256=hP3J0cL11HuzjbHeMJ_UklVo6CdFmgpak3pw9XTWwPY,244
|
38
|
+
mdbq-3.7.10.dist-info/WHEEL,sha256=cpQTJ5IWu9CdaPViMhC9YzF8gZuS5-vlfoFihTBC86A,91
|
39
|
+
mdbq-3.7.10.dist-info/top_level.txt,sha256=2FQ-uLnCSB-OwFiWntzmwosW3X2Xqsg0ewh1axsaylA,5
|
40
|
+
mdbq-3.7.10.dist-info/RECORD,,
|