re-common 10.0.37__py3-none-any.whl → 10.0.39__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- re_common/baselibrary/__init__.py +4 -4
- re_common/baselibrary/baseabs/__init__.py +6 -6
- re_common/baselibrary/baseabs/baseabs.py +26 -26
- re_common/baselibrary/database/mbuilder.py +132 -132
- re_common/baselibrary/database/moudle.py +93 -93
- re_common/baselibrary/database/msqlite3.py +194 -194
- re_common/baselibrary/database/mysql.py +169 -169
- re_common/baselibrary/database/sql_factory.py +26 -26
- re_common/baselibrary/mthread/MThreadingRun.py +486 -486
- re_common/baselibrary/mthread/MThreadingRunEvent.py +349 -349
- re_common/baselibrary/mthread/__init__.py +2 -2
- re_common/baselibrary/mthread/mythreading.py +695 -695
- re_common/baselibrary/pakge_other/socks.py +404 -404
- re_common/baselibrary/readconfig/config_factory.py +18 -18
- re_common/baselibrary/readconfig/ini_config.py +317 -317
- re_common/baselibrary/readconfig/toml_config.py +49 -49
- re_common/baselibrary/temporary/envdata.py +36 -36
- re_common/baselibrary/tools/all_requests/aiohttp_request.py +118 -118
- re_common/baselibrary/tools/all_requests/httpx_requet.py +102 -102
- re_common/baselibrary/tools/all_requests/mrequest.py +412 -412
- re_common/baselibrary/tools/all_requests/requests_request.py +81 -81
- re_common/baselibrary/tools/batch_compre/bijiao_batch.py +31 -31
- re_common/baselibrary/tools/contrast_db3.py +123 -123
- re_common/baselibrary/tools/copy_file.py +39 -39
- re_common/baselibrary/tools/db3_2_sizedb3.py +102 -102
- re_common/baselibrary/tools/foreachgz.py +39 -39
- re_common/baselibrary/tools/get_attr.py +10 -10
- re_common/baselibrary/tools/image_to_pdf.py +61 -61
- re_common/baselibrary/tools/java_code_deal.py +139 -139
- re_common/baselibrary/tools/javacode.py +79 -79
- re_common/baselibrary/tools/mdb_db3.py +48 -48
- re_common/baselibrary/tools/merge_file.py +171 -171
- re_common/baselibrary/tools/merge_gz_file.py +165 -165
- re_common/baselibrary/tools/mhdfstools/down_hdfs_files.py +42 -42
- re_common/baselibrary/tools/mhdfstools/hdfst.py +42 -42
- re_common/baselibrary/tools/mhdfstools/up_hdfs_files.py +38 -38
- re_common/baselibrary/tools/mongo_tools.py +50 -50
- re_common/baselibrary/tools/move_file.py +170 -170
- re_common/baselibrary/tools/move_mongo/mongo_table_to_file.py +63 -63
- re_common/baselibrary/tools/move_mongo/move_mongo_table.py +354 -354
- re_common/baselibrary/tools/move_mongo/use_mttf.py +18 -18
- re_common/baselibrary/tools/move_mongo/use_mv.py +93 -93
- re_common/baselibrary/tools/mpandas/mpandasreadexcel.py +125 -125
- re_common/baselibrary/tools/mpandas/pandas_visualization.py +7 -7
- re_common/baselibrary/tools/myparsel.py +104 -104
- re_common/baselibrary/tools/rename_dir_file.py +37 -37
- re_common/baselibrary/tools/sequoiadb_utils.py +398 -398
- re_common/baselibrary/tools/split_line_to_many.py +25 -25
- re_common/baselibrary/tools/stringtodicts.py +33 -33
- re_common/baselibrary/tools/workwechant_bot.py +84 -84
- re_common/baselibrary/utils/baseaiohttp.py +296 -296
- re_common/baselibrary/utils/baseaiomysql.py +87 -87
- re_common/baselibrary/utils/baseallstep.py +191 -191
- re_common/baselibrary/utils/baseavro.py +19 -19
- re_common/baselibrary/utils/baseboto3.py +291 -291
- re_common/baselibrary/utils/basecsv.py +32 -32
- re_common/baselibrary/utils/basedict.py +133 -133
- re_common/baselibrary/utils/basedir.py +241 -241
- re_common/baselibrary/utils/baseencode.py +351 -351
- re_common/baselibrary/utils/baseencoding.py +28 -28
- re_common/baselibrary/utils/baseesdsl.py +86 -86
- re_common/baselibrary/utils/baseexcel.py +264 -264
- re_common/baselibrary/utils/baseexcept.py +109 -109
- re_common/baselibrary/utils/basefile.py +654 -654
- re_common/baselibrary/utils/baseftp.py +214 -214
- re_common/baselibrary/utils/basegzip.py +60 -60
- re_common/baselibrary/utils/basehdfs.py +135 -135
- re_common/baselibrary/utils/basehttpx.py +268 -268
- re_common/baselibrary/utils/baseip.py +87 -87
- re_common/baselibrary/utils/basejson.py +2 -2
- re_common/baselibrary/utils/baselist.py +32 -32
- re_common/baselibrary/utils/basemotor.py +190 -190
- re_common/baselibrary/utils/basemssql.py +98 -98
- re_common/baselibrary/utils/baseodbc.py +113 -113
- re_common/baselibrary/utils/basepandas.py +302 -302
- re_common/baselibrary/utils/basepeewee.py +11 -11
- re_common/baselibrary/utils/basepika.py +180 -180
- re_common/baselibrary/utils/basepydash.py +143 -143
- re_common/baselibrary/utils/basepymongo.py +230 -230
- re_common/baselibrary/utils/basequeue.py +22 -22
- re_common/baselibrary/utils/baserar.py +57 -57
- re_common/baselibrary/utils/baserequest.py +279 -279
- re_common/baselibrary/utils/baseset.py +8 -8
- re_common/baselibrary/utils/basesmb.py +403 -403
- re_common/baselibrary/utils/basestring.py +382 -382
- re_common/baselibrary/utils/basetime.py +320 -320
- re_common/baselibrary/utils/baseurl.py +121 -121
- re_common/baselibrary/utils/basezip.py +57 -57
- re_common/baselibrary/utils/core/__init__.py +7 -7
- re_common/baselibrary/utils/core/bottomutils.py +18 -18
- re_common/baselibrary/utils/core/mdeprecated.py +327 -327
- re_common/baselibrary/utils/core/mlamada.py +16 -16
- re_common/baselibrary/utils/core/msginfo.py +25 -25
- re_common/baselibrary/utils/core/requests_core.py +103 -103
- re_common/baselibrary/utils/fateadm.py +429 -429
- re_common/baselibrary/utils/importfun.py +123 -123
- re_common/baselibrary/utils/mfaker.py +57 -57
- re_common/baselibrary/utils/my_abc/__init__.py +3 -3
- re_common/baselibrary/utils/my_abc/better_abc.py +32 -32
- re_common/baselibrary/utils/mylogger.py +414 -414
- re_common/baselibrary/utils/myredisclient.py +861 -861
- re_common/baselibrary/utils/pipupgrade.py +21 -21
- re_common/baselibrary/utils/ringlist.py +85 -85
- re_common/baselibrary/utils/version_compare.py +36 -36
- re_common/baselibrary/utils/ydmhttp.py +126 -126
- re_common/facade/lazy_import.py +11 -11
- re_common/facade/loggerfacade.py +25 -25
- re_common/facade/mysqlfacade.py +467 -467
- re_common/facade/now.py +31 -31
- re_common/facade/sqlite3facade.py +257 -257
- re_common/facade/use/mq_use_facade.py +83 -83
- re_common/facade/use/proxy_use_facade.py +19 -19
- re_common/libtest/base_dict_test.py +19 -19
- re_common/libtest/baseavro_test.py +13 -13
- re_common/libtest/basefile_test.py +14 -14
- re_common/libtest/basemssql_test.py +77 -77
- re_common/libtest/baseodbc_test.py +7 -7
- re_common/libtest/basepandas_test.py +38 -38
- re_common/libtest/get_attr_test/get_attr_test_settings.py +14 -14
- re_common/libtest/get_attr_test/settings.py +54 -54
- re_common/libtest/idencode_test.py +53 -53
- re_common/libtest/iniconfig_test.py +35 -35
- re_common/libtest/ip_test.py +34 -34
- re_common/libtest/merge_file_test.py +20 -20
- re_common/libtest/mfaker_test.py +8 -8
- re_common/libtest/mm3_test.py +31 -31
- re_common/libtest/mylogger_test.py +88 -88
- re_common/libtest/myparsel_test.py +27 -27
- re_common/libtest/mysql_test.py +151 -151
- re_common/libtest/pymongo_test.py +21 -21
- re_common/libtest/split_test.py +11 -11
- re_common/libtest/sqlite3_merge_test.py +5 -5
- re_common/libtest/sqlite3_test.py +34 -34
- re_common/libtest/tomlconfig_test.py +30 -30
- re_common/libtest/use_tools_test/__init__.py +2 -2
- re_common/libtest/user/__init__.py +4 -4
- re_common/studio/__init__.py +4 -4
- re_common/studio/assignment_expressions.py +36 -36
- re_common/studio/mydash/test1.py +18 -18
- re_common/studio/pydashstudio/first.py +9 -9
- re_common/studio/streamlitstudio/first_app.py +65 -65
- re_common/studio/streamlitstudio/uber_pickups.py +23 -23
- re_common/studio/test.py +18 -18
- re_common/v2/baselibrary/business_utils/BusinessStringUtil.py +219 -219
- re_common/v2/baselibrary/business_utils/baseencodeid.py +100 -100
- re_common/v2/baselibrary/business_utils/full_doi_path.py +116 -116
- re_common/v2/baselibrary/business_utils/rel_tools.py +6 -6
- re_common/v2/baselibrary/decorators/utils.py +59 -59
- re_common/v2/baselibrary/helpers/search_packge/NearestNeighbors_test.py +105 -105
- re_common/v2/baselibrary/helpers/search_packge/fit_text_match.py +253 -253
- re_common/v2/baselibrary/helpers/search_packge/scikit_learn_text_matcher.py +260 -260
- re_common/v2/baselibrary/helpers/search_packge/test.py +1 -1
- re_common/v2/baselibrary/s3object/baseboto3.py +230 -230
- re_common/v2/baselibrary/tools/WeChatRobot.py +95 -95
- re_common/v2/baselibrary/tools/ac_ahocorasick.py +75 -75
- re_common/v2/baselibrary/tools/concurrency.py +35 -35
- re_common/v2/baselibrary/tools/data_processer/base.py +53 -53
- re_common/v2/baselibrary/tools/data_processer/data_processer.py +508 -508
- re_common/v2/baselibrary/tools/data_processer/data_reader.py +187 -187
- re_common/v2/baselibrary/tools/data_processer/data_writer.py +38 -38
- re_common/v2/baselibrary/tools/dict_tools.py +44 -44
- re_common/v2/baselibrary/tools/dolphinscheduler.py +187 -187
- re_common/v2/baselibrary/tools/hdfs_base_processor.py +204 -204
- re_common/v2/baselibrary/tools/hdfs_bulk_processor.py +67 -67
- re_common/v2/baselibrary/tools/hdfs_data_processer.py +338 -338
- re_common/v2/baselibrary/tools/hdfs_line_processor.py +74 -74
- re_common/v2/baselibrary/tools/list_tools.py +69 -69
- re_common/v2/baselibrary/tools/resume_tracker.py +94 -94
- re_common/v2/baselibrary/tools/search_hash_tools.py +54 -54
- re_common/v2/baselibrary/tools/text_matcher.py +326 -326
- re_common/v2/baselibrary/tools/unionfind_tools.py +60 -60
- re_common/v2/baselibrary/utils/BusinessStringUtil.py +196 -196
- re_common/v2/baselibrary/utils/api_net_utils.py +270 -270
- re_common/v2/baselibrary/utils/author_smi.py +361 -361
- re_common/v2/baselibrary/utils/base_string_similarity.py +158 -158
- re_common/v2/baselibrary/utils/basedict.py +37 -37
- re_common/v2/baselibrary/utils/basehdfs.py +163 -163
- re_common/v2/baselibrary/utils/basepika.py +180 -180
- re_common/v2/baselibrary/utils/basetime.py +77 -77
- re_common/v2/baselibrary/utils/db.py +156 -156
- re_common/v2/baselibrary/utils/json_cls.py +16 -16
- re_common/v2/baselibrary/utils/mq.py +83 -83
- re_common/v2/baselibrary/utils/n_ary_expression_tree.py +243 -243
- re_common/v2/baselibrary/utils/string_bool.py +186 -186
- re_common/v2/baselibrary/utils/string_clear.py +246 -246
- re_common/v2/baselibrary/utils/string_smi.py +18 -18
- re_common/v2/baselibrary/utils/stringutils.py +271 -278
- re_common/vip/base_step_process.py +11 -11
- re_common/vip/baseencodeid.py +90 -90
- re_common/vip/changetaskname.py +28 -28
- re_common/vip/core_var.py +24 -24
- re_common/vip/mmh3Hash.py +89 -89
- re_common/vip/proxy/allproxys.py +127 -127
- re_common/vip/proxy/allproxys_thread.py +159 -159
- re_common/vip/proxy/cnki_proxy.py +153 -153
- re_common/vip/proxy/kuaidaili.py +87 -87
- re_common/vip/proxy/proxy_all.py +113 -113
- re_common/vip/proxy/update_kuaidaili_0.py +42 -42
- re_common/vip/proxy/wanfang_proxy.py +152 -152
- re_common/vip/proxy/wp_proxy_all.py +181 -181
- re_common/vip/read_rawid_to_txt.py +91 -91
- re_common/vip/title/__init__.py +5 -5
- re_common/vip/title/transform/TransformBookTitleToZt.py +125 -125
- re_common/vip/title/transform/TransformConferenceTitleToZt.py +139 -139
- re_common/vip/title/transform/TransformCstadTitleToZt.py +195 -195
- re_common/vip/title/transform/TransformJournalTitleToZt.py +203 -203
- re_common/vip/title/transform/TransformPatentTitleToZt.py +132 -132
- re_common/vip/title/transform/TransformRegulationTitleToZt.py +114 -114
- re_common/vip/title/transform/TransformStandardTitleToZt.py +135 -135
- re_common/vip/title/transform/TransformThesisTitleToZt.py +135 -135
- re_common/vip/title/transform/__init__.py +10 -10
- {re_common-10.0.37.dist-info → re_common-10.0.39.dist-info}/LICENSE +201 -201
- {re_common-10.0.37.dist-info → re_common-10.0.39.dist-info}/METADATA +16 -16
- re_common-10.0.39.dist-info/RECORD +248 -0
- {re_common-10.0.37.dist-info → re_common-10.0.39.dist-info}/WHEEL +1 -1
- re_common-10.0.37.dist-info/RECORD +0 -248
- {re_common-10.0.37.dist-info → re_common-10.0.39.dist-info}/top_level.txt +0 -0
re_common/facade/now.py
CHANGED
|
@@ -1,31 +1,31 @@
|
|
|
1
|
-
# 这个库会加载所有的模块,也可以知道本基础库使用了哪些模块
|
|
2
|
-
from re_common.baselibrary.utils.baserequest import BaseRequest
|
|
3
|
-
|
|
4
|
-
from .loggerfacade import MLogger
|
|
5
|
-
|
|
6
|
-
__all__ = ['get_streamlogger', 'get_filelogger', 'get_timerotatingfilelogger', 'BaseRequest']
|
|
7
|
-
|
|
8
|
-
"""
|
|
9
|
-
提供logger的使用外观 外观模式
|
|
10
|
-
"""
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
def get_streamlogger():
|
|
14
|
-
"""
|
|
15
|
-
提供一个输出流的logger
|
|
16
|
-
:return:
|
|
17
|
-
"""
|
|
18
|
-
return MLogger().streamlogger
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
def get_filelogger(filedir):
|
|
22
|
-
"""
|
|
23
|
-
文件流的logger
|
|
24
|
-
:param filedir:
|
|
25
|
-
:return:
|
|
26
|
-
"""
|
|
27
|
-
return MLogger(filedir).filelogger
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
def get_timerotatingfilelogger(filedir, fileter=None):
|
|
31
|
-
return MLogger(filedir, fileter).timerotatingfilelogger
|
|
1
|
+
# 这个库会加载所有的模块,也可以知道本基础库使用了哪些模块
|
|
2
|
+
from re_common.baselibrary.utils.baserequest import BaseRequest
|
|
3
|
+
|
|
4
|
+
from .loggerfacade import MLogger
|
|
5
|
+
|
|
6
|
+
__all__ = ['get_streamlogger', 'get_filelogger', 'get_timerotatingfilelogger', 'BaseRequest']
|
|
7
|
+
|
|
8
|
+
"""
|
|
9
|
+
提供logger的使用外观 外观模式
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def get_streamlogger():
|
|
14
|
+
"""
|
|
15
|
+
提供一个输出流的logger
|
|
16
|
+
:return:
|
|
17
|
+
"""
|
|
18
|
+
return MLogger().streamlogger
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def get_filelogger(filedir):
|
|
22
|
+
"""
|
|
23
|
+
文件流的logger
|
|
24
|
+
:param filedir:
|
|
25
|
+
:return:
|
|
26
|
+
"""
|
|
27
|
+
return MLogger(filedir).filelogger
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def get_timerotatingfilelogger(filedir, fileter=None):
|
|
31
|
+
return MLogger(filedir, fileter).timerotatingfilelogger
|
|
@@ -1,257 +1,257 @@
|
|
|
1
|
-
# db3数据的连接返回一个连接对象
|
|
2
|
-
import sys
|
|
3
|
-
import traceback
|
|
4
|
-
|
|
5
|
-
from re_common.baselibrary import MLogger
|
|
6
|
-
from re_common.baselibrary.baseabs import BaseAbs
|
|
7
|
-
from re_common.baselibrary.database.mbuilder import Sqlite3Builder
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
class Sqlite3Utiles(object):
|
|
11
|
-
def __init__(self, logger=None):
|
|
12
|
-
self.sqllite3 = None
|
|
13
|
-
self.conn = None
|
|
14
|
-
self.cursor = None
|
|
15
|
-
|
|
16
|
-
self._logger = logger
|
|
17
|
-
|
|
18
|
-
@property
|
|
19
|
-
def logger(self):
|
|
20
|
-
if self._logger is None:
|
|
21
|
-
return MLogger().streamlogger
|
|
22
|
-
return self._logger
|
|
23
|
-
|
|
24
|
-
@logger.setter
|
|
25
|
-
def logger(self, value):
|
|
26
|
-
assert isinstance(value, MLogger)
|
|
27
|
-
self._logger = value
|
|
28
|
-
|
|
29
|
-
def get_new_cursor(self):
|
|
30
|
-
"""
|
|
31
|
-
获取一个新的游标
|
|
32
|
-
:return:
|
|
33
|
-
"""
|
|
34
|
-
# 检查db的存在及是否断掉 不知道为什么 不可以加括号 但编辑器会警告
|
|
35
|
-
return self.conn.cursor()
|
|
36
|
-
|
|
37
|
-
def Sqlite3DBConnectFromFilePath(self, dbfilepath, encoding="gbk", timeout=60):
|
|
38
|
-
"""
|
|
39
|
-
通过直接文件连接 我使用Hadoop导下来的db3是gbk编码的
|
|
40
|
-
如果为utf-8编码请改为utf-8
|
|
41
|
-
:param sec: section
|
|
42
|
-
:return:
|
|
43
|
-
"""
|
|
44
|
-
sqllite3 = BaseAbs.get_sql_factory().sqlite_factory()
|
|
45
|
-
# 传入的是配置文件和section选项 dbpath为配置文件路径
|
|
46
|
-
sqlite3_moudle = Sqlite3Builder().build_file_path(dbfilepath).build_timeout(timeout).get_moudle()
|
|
47
|
-
sqllite3.link(sqlite3_moudle)
|
|
48
|
-
# 设置txt的编码格式 hadoop 上的db3为gbk 默认为utf8
|
|
49
|
-
sqllite3.set_encoding(encoding=encoding)
|
|
50
|
-
# 返回一个连接
|
|
51
|
-
self.sqllite3 = sqllite3
|
|
52
|
-
self.conn = self.sqllite3.db
|
|
53
|
-
return self
|
|
54
|
-
|
|
55
|
-
def Sqlite3DBConnectFromConfig(self, cobnfigfilepath, sec, encoding="gbk"):
|
|
56
|
-
"""
|
|
57
|
-
通过配置文件连接
|
|
58
|
-
:param sec: section
|
|
59
|
-
:return:
|
|
60
|
-
"""
|
|
61
|
-
sqllite3 = BaseAbs.get_sql_factory().sqlite_factory()
|
|
62
|
-
# 传入的是配置文件和section选项 dbpath为配置文件路径
|
|
63
|
-
sqlite3_moudle = Sqlite3Builder(cobnfigfilepath, sec).build_all().get_moudle()
|
|
64
|
-
sqllite3.link(sqlite3_moudle)
|
|
65
|
-
# 设置txt的编码格式 hadoop 上的db3为gbk 默认为utf8
|
|
66
|
-
sqllite3.set_encoding(encoding=encoding)
|
|
67
|
-
# 返回一个连接
|
|
68
|
-
self.sqllite3 = sqllite3
|
|
69
|
-
self.conn = self.sqllite3.db
|
|
70
|
-
return self
|
|
71
|
-
|
|
72
|
-
def ExeSqlliteList(self, sqlList, errExit=True):
|
|
73
|
-
"""
|
|
74
|
-
该函数和上面一样执行一个sql列表且不返回结果
|
|
75
|
-
属于插入和更新类函数 但该函数针对db3
|
|
76
|
-
:param sqllitename:
|
|
77
|
-
:param sqlList:
|
|
78
|
-
:return:
|
|
79
|
-
"""
|
|
80
|
-
dbMsg = None
|
|
81
|
-
cur = self.get_new_cursor()
|
|
82
|
-
if self.conn:
|
|
83
|
-
count = 0
|
|
84
|
-
for sql in sqlList:
|
|
85
|
-
count += 1
|
|
86
|
-
try:
|
|
87
|
-
self.logger.info("{} 执行sql数量:{}".format(sql, str(count)))
|
|
88
|
-
cur.execute(sql)
|
|
89
|
-
except:
|
|
90
|
-
self.logger.error('*errSql:' + sql)
|
|
91
|
-
dbMsg = '*InsertError:' + traceback.format_exc()
|
|
92
|
-
if errExit:
|
|
93
|
-
self.logger.error(dbMsg)
|
|
94
|
-
sys.exit()
|
|
95
|
-
if dbMsg:
|
|
96
|
-
self.logger.error(dbMsg)
|
|
97
|
-
continue
|
|
98
|
-
self.conn.commit()
|
|
99
|
-
cur.close()
|
|
100
|
-
|
|
101
|
-
def ExeSqlliteMany(self, sql, itermany, errExit=True):
|
|
102
|
-
dbMsg = None
|
|
103
|
-
cur = self.get_new_cursor()
|
|
104
|
-
if self.conn:
|
|
105
|
-
try:
|
|
106
|
-
self.logger.info("{}\n{}".format(sql, str(itermany)))
|
|
107
|
-
cur.executemany(sql, itermany)
|
|
108
|
-
except:
|
|
109
|
-
self.logger.error('*errSql:' + sql)
|
|
110
|
-
dbMsg = '*InsertError:' + traceback.format_exc()
|
|
111
|
-
if errExit:
|
|
112
|
-
self.logger.error(dbMsg)
|
|
113
|
-
sys.exit()
|
|
114
|
-
if dbMsg:
|
|
115
|
-
self.logger.error(dbMsg)
|
|
116
|
-
self.conn.commit()
|
|
117
|
-
cur.close()
|
|
118
|
-
|
|
119
|
-
def ExeSqlliteSql(self, sql):
|
|
120
|
-
"""
|
|
121
|
-
该函数和上面一样执行一个sql列表且不返回结果
|
|
122
|
-
属于插入和更新类函数 但该函数针对db3
|
|
123
|
-
:param sqllitename:
|
|
124
|
-
:param sqlList:
|
|
125
|
-
:return:
|
|
126
|
-
"""
|
|
127
|
-
dbMsg = None
|
|
128
|
-
cur = self.get_new_cursor()
|
|
129
|
-
if self.conn:
|
|
130
|
-
try:
|
|
131
|
-
self.logger.info(sql)
|
|
132
|
-
cur.execute(sql)
|
|
133
|
-
self.conn.commit()
|
|
134
|
-
except:
|
|
135
|
-
self.logger.error('*errSql:' + sql)
|
|
136
|
-
dbMsg = '*InsertError:' + traceback.format_exc()
|
|
137
|
-
if dbMsg:
|
|
138
|
-
self.logger.error(dbMsg)
|
|
139
|
-
return False
|
|
140
|
-
else:
|
|
141
|
-
return False
|
|
142
|
-
cur.close()
|
|
143
|
-
return True
|
|
144
|
-
|
|
145
|
-
def SelectFromSqlliteFetchall(self, sql):
|
|
146
|
-
"""
|
|
147
|
-
该函数和上面一样执行一个sql列表且不返回结果
|
|
148
|
-
属于插入和更新类函数 但该函数针对db3
|
|
149
|
-
:param sqllitename:
|
|
150
|
-
:param sqlList:
|
|
151
|
-
:return:
|
|
152
|
-
"""
|
|
153
|
-
cur = self.get_new_cursor()
|
|
154
|
-
if self.conn:
|
|
155
|
-
try:
|
|
156
|
-
self.logger.info(sql)
|
|
157
|
-
cur.execute(sql)
|
|
158
|
-
rows = cur.fetchall()
|
|
159
|
-
return rows
|
|
160
|
-
except:
|
|
161
|
-
self.logger.error('*errSql:' + sql)
|
|
162
|
-
dbMsg = '*InsertError:' + traceback.format_exc()
|
|
163
|
-
if dbMsg:
|
|
164
|
-
self.logger.error(dbMsg)
|
|
165
|
-
return False
|
|
166
|
-
else:
|
|
167
|
-
return False
|
|
168
|
-
cur.close()
|
|
169
|
-
|
|
170
|
-
def SelectFromSqlliteFetchall_dicts(self, sql):
|
|
171
|
-
"""
|
|
172
|
-
该函数和上面一样执行一个sql列表且不返回结果
|
|
173
|
-
属于插入和更新类函数 但该函数针对db3
|
|
174
|
-
:param sqllitename:
|
|
175
|
-
:param sqlList:
|
|
176
|
-
:return:
|
|
177
|
-
"""
|
|
178
|
-
|
|
179
|
-
def dict_factory(cursor, row):
|
|
180
|
-
d = {}
|
|
181
|
-
for idx, col in enumerate(cursor.description):
|
|
182
|
-
d[col[0]] = row[idx]
|
|
183
|
-
return d
|
|
184
|
-
|
|
185
|
-
self.conn.row_factory = dict_factory
|
|
186
|
-
cur = self.get_new_cursor()
|
|
187
|
-
if self.conn:
|
|
188
|
-
try:
|
|
189
|
-
self.logger.info(sql)
|
|
190
|
-
cur.execute(sql)
|
|
191
|
-
rows = cur.fetchall()
|
|
192
|
-
return rows
|
|
193
|
-
except:
|
|
194
|
-
self.logger.error('*errSql:' + sql)
|
|
195
|
-
dbMsg = '*InsertError:' + traceback.format_exc()
|
|
196
|
-
if dbMsg:
|
|
197
|
-
self.logger.error(dbMsg)
|
|
198
|
-
return False
|
|
199
|
-
else:
|
|
200
|
-
return False
|
|
201
|
-
cur.close()
|
|
202
|
-
|
|
203
|
-
def SelectFromSqlliteFetchOne(self, sql):
|
|
204
|
-
"""
|
|
205
|
-
该函数和上面一样执行一个sql列表且不返回结果
|
|
206
|
-
属于插入和更新类函数 但该函数针对db3
|
|
207
|
-
:param sqllitename:
|
|
208
|
-
:param sqlList:
|
|
209
|
-
:return:
|
|
210
|
-
"""
|
|
211
|
-
dbMsg = None
|
|
212
|
-
cur = self.get_new_cursor()
|
|
213
|
-
if self.conn:
|
|
214
|
-
try:
|
|
215
|
-
self.logger.info(sql)
|
|
216
|
-
cur.execute(sql)
|
|
217
|
-
while True:
|
|
218
|
-
row = cur.fetchone()
|
|
219
|
-
if row is None:
|
|
220
|
-
return None
|
|
221
|
-
else:
|
|
222
|
-
yield row
|
|
223
|
-
except:
|
|
224
|
-
self.logger.error('*errSql:' + sql)
|
|
225
|
-
dbMsg = '*InsertError:' + traceback.format_exc()
|
|
226
|
-
if dbMsg:
|
|
227
|
-
self.logger.error(dbMsg)
|
|
228
|
-
return False
|
|
229
|
-
else:
|
|
230
|
-
return False
|
|
231
|
-
cur.close()
|
|
232
|
-
|
|
233
|
-
def ExeVACUUM(self):
|
|
234
|
-
"""
|
|
235
|
-
清理空间
|
|
236
|
-
:return:
|
|
237
|
-
"""
|
|
238
|
-
dbMsg = None
|
|
239
|
-
if self.conn:
|
|
240
|
-
try:
|
|
241
|
-
self.conn.execute("VACUUM")
|
|
242
|
-
except:
|
|
243
|
-
dbMsg = '*VACUUMError:' + traceback.format_exc()
|
|
244
|
-
if dbMsg:
|
|
245
|
-
self.logger.error(dbMsg)
|
|
246
|
-
return False
|
|
247
|
-
else:
|
|
248
|
-
return False
|
|
249
|
-
return True
|
|
250
|
-
|
|
251
|
-
def sqliteEscape(self, keyWord):
|
|
252
|
-
keyWord = keyWord.replace("'", "''")
|
|
253
|
-
keyWord = keyWord.replace("\\", "\\\\")
|
|
254
|
-
return keyWord
|
|
255
|
-
|
|
256
|
-
def close(self):
|
|
257
|
-
self.conn.close()
|
|
1
|
+
# db3数据的连接返回一个连接对象
|
|
2
|
+
import sys
|
|
3
|
+
import traceback
|
|
4
|
+
|
|
5
|
+
from re_common.baselibrary import MLogger
|
|
6
|
+
from re_common.baselibrary.baseabs import BaseAbs
|
|
7
|
+
from re_common.baselibrary.database.mbuilder import Sqlite3Builder
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Sqlite3Utiles(object):
|
|
11
|
+
def __init__(self, logger=None):
|
|
12
|
+
self.sqllite3 = None
|
|
13
|
+
self.conn = None
|
|
14
|
+
self.cursor = None
|
|
15
|
+
|
|
16
|
+
self._logger = logger
|
|
17
|
+
|
|
18
|
+
@property
|
|
19
|
+
def logger(self):
|
|
20
|
+
if self._logger is None:
|
|
21
|
+
return MLogger().streamlogger
|
|
22
|
+
return self._logger
|
|
23
|
+
|
|
24
|
+
@logger.setter
|
|
25
|
+
def logger(self, value):
|
|
26
|
+
assert isinstance(value, MLogger)
|
|
27
|
+
self._logger = value
|
|
28
|
+
|
|
29
|
+
def get_new_cursor(self):
|
|
30
|
+
"""
|
|
31
|
+
获取一个新的游标
|
|
32
|
+
:return:
|
|
33
|
+
"""
|
|
34
|
+
# 检查db的存在及是否断掉 不知道为什么 不可以加括号 但编辑器会警告
|
|
35
|
+
return self.conn.cursor()
|
|
36
|
+
|
|
37
|
+
def Sqlite3DBConnectFromFilePath(self, dbfilepath, encoding="gbk", timeout=60):
|
|
38
|
+
"""
|
|
39
|
+
通过直接文件连接 我使用Hadoop导下来的db3是gbk编码的
|
|
40
|
+
如果为utf-8编码请改为utf-8
|
|
41
|
+
:param sec: section
|
|
42
|
+
:return:
|
|
43
|
+
"""
|
|
44
|
+
sqllite3 = BaseAbs.get_sql_factory().sqlite_factory()
|
|
45
|
+
# 传入的是配置文件和section选项 dbpath为配置文件路径
|
|
46
|
+
sqlite3_moudle = Sqlite3Builder().build_file_path(dbfilepath).build_timeout(timeout).get_moudle()
|
|
47
|
+
sqllite3.link(sqlite3_moudle)
|
|
48
|
+
# 设置txt的编码格式 hadoop 上的db3为gbk 默认为utf8
|
|
49
|
+
sqllite3.set_encoding(encoding=encoding)
|
|
50
|
+
# 返回一个连接
|
|
51
|
+
self.sqllite3 = sqllite3
|
|
52
|
+
self.conn = self.sqllite3.db
|
|
53
|
+
return self
|
|
54
|
+
|
|
55
|
+
def Sqlite3DBConnectFromConfig(self, cobnfigfilepath, sec, encoding="gbk"):
|
|
56
|
+
"""
|
|
57
|
+
通过配置文件连接
|
|
58
|
+
:param sec: section
|
|
59
|
+
:return:
|
|
60
|
+
"""
|
|
61
|
+
sqllite3 = BaseAbs.get_sql_factory().sqlite_factory()
|
|
62
|
+
# 传入的是配置文件和section选项 dbpath为配置文件路径
|
|
63
|
+
sqlite3_moudle = Sqlite3Builder(cobnfigfilepath, sec).build_all().get_moudle()
|
|
64
|
+
sqllite3.link(sqlite3_moudle)
|
|
65
|
+
# 设置txt的编码格式 hadoop 上的db3为gbk 默认为utf8
|
|
66
|
+
sqllite3.set_encoding(encoding=encoding)
|
|
67
|
+
# 返回一个连接
|
|
68
|
+
self.sqllite3 = sqllite3
|
|
69
|
+
self.conn = self.sqllite3.db
|
|
70
|
+
return self
|
|
71
|
+
|
|
72
|
+
def ExeSqlliteList(self, sqlList, errExit=True):
|
|
73
|
+
"""
|
|
74
|
+
该函数和上面一样执行一个sql列表且不返回结果
|
|
75
|
+
属于插入和更新类函数 但该函数针对db3
|
|
76
|
+
:param sqllitename:
|
|
77
|
+
:param sqlList:
|
|
78
|
+
:return:
|
|
79
|
+
"""
|
|
80
|
+
dbMsg = None
|
|
81
|
+
cur = self.get_new_cursor()
|
|
82
|
+
if self.conn:
|
|
83
|
+
count = 0
|
|
84
|
+
for sql in sqlList:
|
|
85
|
+
count += 1
|
|
86
|
+
try:
|
|
87
|
+
self.logger.info("{} 执行sql数量:{}".format(sql, str(count)))
|
|
88
|
+
cur.execute(sql)
|
|
89
|
+
except:
|
|
90
|
+
self.logger.error('*errSql:' + sql)
|
|
91
|
+
dbMsg = '*InsertError:' + traceback.format_exc()
|
|
92
|
+
if errExit:
|
|
93
|
+
self.logger.error(dbMsg)
|
|
94
|
+
sys.exit()
|
|
95
|
+
if dbMsg:
|
|
96
|
+
self.logger.error(dbMsg)
|
|
97
|
+
continue
|
|
98
|
+
self.conn.commit()
|
|
99
|
+
cur.close()
|
|
100
|
+
|
|
101
|
+
def ExeSqlliteMany(self, sql, itermany, errExit=True):
|
|
102
|
+
dbMsg = None
|
|
103
|
+
cur = self.get_new_cursor()
|
|
104
|
+
if self.conn:
|
|
105
|
+
try:
|
|
106
|
+
self.logger.info("{}\n{}".format(sql, str(itermany)))
|
|
107
|
+
cur.executemany(sql, itermany)
|
|
108
|
+
except:
|
|
109
|
+
self.logger.error('*errSql:' + sql)
|
|
110
|
+
dbMsg = '*InsertError:' + traceback.format_exc()
|
|
111
|
+
if errExit:
|
|
112
|
+
self.logger.error(dbMsg)
|
|
113
|
+
sys.exit()
|
|
114
|
+
if dbMsg:
|
|
115
|
+
self.logger.error(dbMsg)
|
|
116
|
+
self.conn.commit()
|
|
117
|
+
cur.close()
|
|
118
|
+
|
|
119
|
+
def ExeSqlliteSql(self, sql):
|
|
120
|
+
"""
|
|
121
|
+
该函数和上面一样执行一个sql列表且不返回结果
|
|
122
|
+
属于插入和更新类函数 但该函数针对db3
|
|
123
|
+
:param sqllitename:
|
|
124
|
+
:param sqlList:
|
|
125
|
+
:return:
|
|
126
|
+
"""
|
|
127
|
+
dbMsg = None
|
|
128
|
+
cur = self.get_new_cursor()
|
|
129
|
+
if self.conn:
|
|
130
|
+
try:
|
|
131
|
+
self.logger.info(sql)
|
|
132
|
+
cur.execute(sql)
|
|
133
|
+
self.conn.commit()
|
|
134
|
+
except:
|
|
135
|
+
self.logger.error('*errSql:' + sql)
|
|
136
|
+
dbMsg = '*InsertError:' + traceback.format_exc()
|
|
137
|
+
if dbMsg:
|
|
138
|
+
self.logger.error(dbMsg)
|
|
139
|
+
return False
|
|
140
|
+
else:
|
|
141
|
+
return False
|
|
142
|
+
cur.close()
|
|
143
|
+
return True
|
|
144
|
+
|
|
145
|
+
def SelectFromSqlliteFetchall(self, sql):
|
|
146
|
+
"""
|
|
147
|
+
该函数和上面一样执行一个sql列表且不返回结果
|
|
148
|
+
属于插入和更新类函数 但该函数针对db3
|
|
149
|
+
:param sqllitename:
|
|
150
|
+
:param sqlList:
|
|
151
|
+
:return:
|
|
152
|
+
"""
|
|
153
|
+
cur = self.get_new_cursor()
|
|
154
|
+
if self.conn:
|
|
155
|
+
try:
|
|
156
|
+
self.logger.info(sql)
|
|
157
|
+
cur.execute(sql)
|
|
158
|
+
rows = cur.fetchall()
|
|
159
|
+
return rows
|
|
160
|
+
except:
|
|
161
|
+
self.logger.error('*errSql:' + sql)
|
|
162
|
+
dbMsg = '*InsertError:' + traceback.format_exc()
|
|
163
|
+
if dbMsg:
|
|
164
|
+
self.logger.error(dbMsg)
|
|
165
|
+
return False
|
|
166
|
+
else:
|
|
167
|
+
return False
|
|
168
|
+
cur.close()
|
|
169
|
+
|
|
170
|
+
def SelectFromSqlliteFetchall_dicts(self, sql):
|
|
171
|
+
"""
|
|
172
|
+
该函数和上面一样执行一个sql列表且不返回结果
|
|
173
|
+
属于插入和更新类函数 但该函数针对db3
|
|
174
|
+
:param sqllitename:
|
|
175
|
+
:param sqlList:
|
|
176
|
+
:return:
|
|
177
|
+
"""
|
|
178
|
+
|
|
179
|
+
def dict_factory(cursor, row):
|
|
180
|
+
d = {}
|
|
181
|
+
for idx, col in enumerate(cursor.description):
|
|
182
|
+
d[col[0]] = row[idx]
|
|
183
|
+
return d
|
|
184
|
+
|
|
185
|
+
self.conn.row_factory = dict_factory
|
|
186
|
+
cur = self.get_new_cursor()
|
|
187
|
+
if self.conn:
|
|
188
|
+
try:
|
|
189
|
+
self.logger.info(sql)
|
|
190
|
+
cur.execute(sql)
|
|
191
|
+
rows = cur.fetchall()
|
|
192
|
+
return rows
|
|
193
|
+
except:
|
|
194
|
+
self.logger.error('*errSql:' + sql)
|
|
195
|
+
dbMsg = '*InsertError:' + traceback.format_exc()
|
|
196
|
+
if dbMsg:
|
|
197
|
+
self.logger.error(dbMsg)
|
|
198
|
+
return False
|
|
199
|
+
else:
|
|
200
|
+
return False
|
|
201
|
+
cur.close()
|
|
202
|
+
|
|
203
|
+
def SelectFromSqlliteFetchOne(self, sql):
|
|
204
|
+
"""
|
|
205
|
+
该函数和上面一样执行一个sql列表且不返回结果
|
|
206
|
+
属于插入和更新类函数 但该函数针对db3
|
|
207
|
+
:param sqllitename:
|
|
208
|
+
:param sqlList:
|
|
209
|
+
:return:
|
|
210
|
+
"""
|
|
211
|
+
dbMsg = None
|
|
212
|
+
cur = self.get_new_cursor()
|
|
213
|
+
if self.conn:
|
|
214
|
+
try:
|
|
215
|
+
self.logger.info(sql)
|
|
216
|
+
cur.execute(sql)
|
|
217
|
+
while True:
|
|
218
|
+
row = cur.fetchone()
|
|
219
|
+
if row is None:
|
|
220
|
+
return None
|
|
221
|
+
else:
|
|
222
|
+
yield row
|
|
223
|
+
except:
|
|
224
|
+
self.logger.error('*errSql:' + sql)
|
|
225
|
+
dbMsg = '*InsertError:' + traceback.format_exc()
|
|
226
|
+
if dbMsg:
|
|
227
|
+
self.logger.error(dbMsg)
|
|
228
|
+
return False
|
|
229
|
+
else:
|
|
230
|
+
return False
|
|
231
|
+
cur.close()
|
|
232
|
+
|
|
233
|
+
def ExeVACUUM(self):
|
|
234
|
+
"""
|
|
235
|
+
清理空间
|
|
236
|
+
:return:
|
|
237
|
+
"""
|
|
238
|
+
dbMsg = None
|
|
239
|
+
if self.conn:
|
|
240
|
+
try:
|
|
241
|
+
self.conn.execute("VACUUM")
|
|
242
|
+
except:
|
|
243
|
+
dbMsg = '*VACUUMError:' + traceback.format_exc()
|
|
244
|
+
if dbMsg:
|
|
245
|
+
self.logger.error(dbMsg)
|
|
246
|
+
return False
|
|
247
|
+
else:
|
|
248
|
+
return False
|
|
249
|
+
return True
|
|
250
|
+
|
|
251
|
+
def sqliteEscape(self, keyWord):
|
|
252
|
+
keyWord = keyWord.replace("'", "''")
|
|
253
|
+
keyWord = keyWord.replace("\\", "\\\\")
|
|
254
|
+
return keyWord
|
|
255
|
+
|
|
256
|
+
def close(self):
|
|
257
|
+
self.conn.close()
|