re-common 10.0.37__py3-none-any.whl → 10.0.39__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (217) hide show
  1. re_common/baselibrary/__init__.py +4 -4
  2. re_common/baselibrary/baseabs/__init__.py +6 -6
  3. re_common/baselibrary/baseabs/baseabs.py +26 -26
  4. re_common/baselibrary/database/mbuilder.py +132 -132
  5. re_common/baselibrary/database/moudle.py +93 -93
  6. re_common/baselibrary/database/msqlite3.py +194 -194
  7. re_common/baselibrary/database/mysql.py +169 -169
  8. re_common/baselibrary/database/sql_factory.py +26 -26
  9. re_common/baselibrary/mthread/MThreadingRun.py +486 -486
  10. re_common/baselibrary/mthread/MThreadingRunEvent.py +349 -349
  11. re_common/baselibrary/mthread/__init__.py +2 -2
  12. re_common/baselibrary/mthread/mythreading.py +695 -695
  13. re_common/baselibrary/pakge_other/socks.py +404 -404
  14. re_common/baselibrary/readconfig/config_factory.py +18 -18
  15. re_common/baselibrary/readconfig/ini_config.py +317 -317
  16. re_common/baselibrary/readconfig/toml_config.py +49 -49
  17. re_common/baselibrary/temporary/envdata.py +36 -36
  18. re_common/baselibrary/tools/all_requests/aiohttp_request.py +118 -118
  19. re_common/baselibrary/tools/all_requests/httpx_requet.py +102 -102
  20. re_common/baselibrary/tools/all_requests/mrequest.py +412 -412
  21. re_common/baselibrary/tools/all_requests/requests_request.py +81 -81
  22. re_common/baselibrary/tools/batch_compre/bijiao_batch.py +31 -31
  23. re_common/baselibrary/tools/contrast_db3.py +123 -123
  24. re_common/baselibrary/tools/copy_file.py +39 -39
  25. re_common/baselibrary/tools/db3_2_sizedb3.py +102 -102
  26. re_common/baselibrary/tools/foreachgz.py +39 -39
  27. re_common/baselibrary/tools/get_attr.py +10 -10
  28. re_common/baselibrary/tools/image_to_pdf.py +61 -61
  29. re_common/baselibrary/tools/java_code_deal.py +139 -139
  30. re_common/baselibrary/tools/javacode.py +79 -79
  31. re_common/baselibrary/tools/mdb_db3.py +48 -48
  32. re_common/baselibrary/tools/merge_file.py +171 -171
  33. re_common/baselibrary/tools/merge_gz_file.py +165 -165
  34. re_common/baselibrary/tools/mhdfstools/down_hdfs_files.py +42 -42
  35. re_common/baselibrary/tools/mhdfstools/hdfst.py +42 -42
  36. re_common/baselibrary/tools/mhdfstools/up_hdfs_files.py +38 -38
  37. re_common/baselibrary/tools/mongo_tools.py +50 -50
  38. re_common/baselibrary/tools/move_file.py +170 -170
  39. re_common/baselibrary/tools/move_mongo/mongo_table_to_file.py +63 -63
  40. re_common/baselibrary/tools/move_mongo/move_mongo_table.py +354 -354
  41. re_common/baselibrary/tools/move_mongo/use_mttf.py +18 -18
  42. re_common/baselibrary/tools/move_mongo/use_mv.py +93 -93
  43. re_common/baselibrary/tools/mpandas/mpandasreadexcel.py +125 -125
  44. re_common/baselibrary/tools/mpandas/pandas_visualization.py +7 -7
  45. re_common/baselibrary/tools/myparsel.py +104 -104
  46. re_common/baselibrary/tools/rename_dir_file.py +37 -37
  47. re_common/baselibrary/tools/sequoiadb_utils.py +398 -398
  48. re_common/baselibrary/tools/split_line_to_many.py +25 -25
  49. re_common/baselibrary/tools/stringtodicts.py +33 -33
  50. re_common/baselibrary/tools/workwechant_bot.py +84 -84
  51. re_common/baselibrary/utils/baseaiohttp.py +296 -296
  52. re_common/baselibrary/utils/baseaiomysql.py +87 -87
  53. re_common/baselibrary/utils/baseallstep.py +191 -191
  54. re_common/baselibrary/utils/baseavro.py +19 -19
  55. re_common/baselibrary/utils/baseboto3.py +291 -291
  56. re_common/baselibrary/utils/basecsv.py +32 -32
  57. re_common/baselibrary/utils/basedict.py +133 -133
  58. re_common/baselibrary/utils/basedir.py +241 -241
  59. re_common/baselibrary/utils/baseencode.py +351 -351
  60. re_common/baselibrary/utils/baseencoding.py +28 -28
  61. re_common/baselibrary/utils/baseesdsl.py +86 -86
  62. re_common/baselibrary/utils/baseexcel.py +264 -264
  63. re_common/baselibrary/utils/baseexcept.py +109 -109
  64. re_common/baselibrary/utils/basefile.py +654 -654
  65. re_common/baselibrary/utils/baseftp.py +214 -214
  66. re_common/baselibrary/utils/basegzip.py +60 -60
  67. re_common/baselibrary/utils/basehdfs.py +135 -135
  68. re_common/baselibrary/utils/basehttpx.py +268 -268
  69. re_common/baselibrary/utils/baseip.py +87 -87
  70. re_common/baselibrary/utils/basejson.py +2 -2
  71. re_common/baselibrary/utils/baselist.py +32 -32
  72. re_common/baselibrary/utils/basemotor.py +190 -190
  73. re_common/baselibrary/utils/basemssql.py +98 -98
  74. re_common/baselibrary/utils/baseodbc.py +113 -113
  75. re_common/baselibrary/utils/basepandas.py +302 -302
  76. re_common/baselibrary/utils/basepeewee.py +11 -11
  77. re_common/baselibrary/utils/basepika.py +180 -180
  78. re_common/baselibrary/utils/basepydash.py +143 -143
  79. re_common/baselibrary/utils/basepymongo.py +230 -230
  80. re_common/baselibrary/utils/basequeue.py +22 -22
  81. re_common/baselibrary/utils/baserar.py +57 -57
  82. re_common/baselibrary/utils/baserequest.py +279 -279
  83. re_common/baselibrary/utils/baseset.py +8 -8
  84. re_common/baselibrary/utils/basesmb.py +403 -403
  85. re_common/baselibrary/utils/basestring.py +382 -382
  86. re_common/baselibrary/utils/basetime.py +320 -320
  87. re_common/baselibrary/utils/baseurl.py +121 -121
  88. re_common/baselibrary/utils/basezip.py +57 -57
  89. re_common/baselibrary/utils/core/__init__.py +7 -7
  90. re_common/baselibrary/utils/core/bottomutils.py +18 -18
  91. re_common/baselibrary/utils/core/mdeprecated.py +327 -327
  92. re_common/baselibrary/utils/core/mlamada.py +16 -16
  93. re_common/baselibrary/utils/core/msginfo.py +25 -25
  94. re_common/baselibrary/utils/core/requests_core.py +103 -103
  95. re_common/baselibrary/utils/fateadm.py +429 -429
  96. re_common/baselibrary/utils/importfun.py +123 -123
  97. re_common/baselibrary/utils/mfaker.py +57 -57
  98. re_common/baselibrary/utils/my_abc/__init__.py +3 -3
  99. re_common/baselibrary/utils/my_abc/better_abc.py +32 -32
  100. re_common/baselibrary/utils/mylogger.py +414 -414
  101. re_common/baselibrary/utils/myredisclient.py +861 -861
  102. re_common/baselibrary/utils/pipupgrade.py +21 -21
  103. re_common/baselibrary/utils/ringlist.py +85 -85
  104. re_common/baselibrary/utils/version_compare.py +36 -36
  105. re_common/baselibrary/utils/ydmhttp.py +126 -126
  106. re_common/facade/lazy_import.py +11 -11
  107. re_common/facade/loggerfacade.py +25 -25
  108. re_common/facade/mysqlfacade.py +467 -467
  109. re_common/facade/now.py +31 -31
  110. re_common/facade/sqlite3facade.py +257 -257
  111. re_common/facade/use/mq_use_facade.py +83 -83
  112. re_common/facade/use/proxy_use_facade.py +19 -19
  113. re_common/libtest/base_dict_test.py +19 -19
  114. re_common/libtest/baseavro_test.py +13 -13
  115. re_common/libtest/basefile_test.py +14 -14
  116. re_common/libtest/basemssql_test.py +77 -77
  117. re_common/libtest/baseodbc_test.py +7 -7
  118. re_common/libtest/basepandas_test.py +38 -38
  119. re_common/libtest/get_attr_test/get_attr_test_settings.py +14 -14
  120. re_common/libtest/get_attr_test/settings.py +54 -54
  121. re_common/libtest/idencode_test.py +53 -53
  122. re_common/libtest/iniconfig_test.py +35 -35
  123. re_common/libtest/ip_test.py +34 -34
  124. re_common/libtest/merge_file_test.py +20 -20
  125. re_common/libtest/mfaker_test.py +8 -8
  126. re_common/libtest/mm3_test.py +31 -31
  127. re_common/libtest/mylogger_test.py +88 -88
  128. re_common/libtest/myparsel_test.py +27 -27
  129. re_common/libtest/mysql_test.py +151 -151
  130. re_common/libtest/pymongo_test.py +21 -21
  131. re_common/libtest/split_test.py +11 -11
  132. re_common/libtest/sqlite3_merge_test.py +5 -5
  133. re_common/libtest/sqlite3_test.py +34 -34
  134. re_common/libtest/tomlconfig_test.py +30 -30
  135. re_common/libtest/use_tools_test/__init__.py +2 -2
  136. re_common/libtest/user/__init__.py +4 -4
  137. re_common/studio/__init__.py +4 -4
  138. re_common/studio/assignment_expressions.py +36 -36
  139. re_common/studio/mydash/test1.py +18 -18
  140. re_common/studio/pydashstudio/first.py +9 -9
  141. re_common/studio/streamlitstudio/first_app.py +65 -65
  142. re_common/studio/streamlitstudio/uber_pickups.py +23 -23
  143. re_common/studio/test.py +18 -18
  144. re_common/v2/baselibrary/business_utils/BusinessStringUtil.py +219 -219
  145. re_common/v2/baselibrary/business_utils/baseencodeid.py +100 -100
  146. re_common/v2/baselibrary/business_utils/full_doi_path.py +116 -116
  147. re_common/v2/baselibrary/business_utils/rel_tools.py +6 -6
  148. re_common/v2/baselibrary/decorators/utils.py +59 -59
  149. re_common/v2/baselibrary/helpers/search_packge/NearestNeighbors_test.py +105 -105
  150. re_common/v2/baselibrary/helpers/search_packge/fit_text_match.py +253 -253
  151. re_common/v2/baselibrary/helpers/search_packge/scikit_learn_text_matcher.py +260 -260
  152. re_common/v2/baselibrary/helpers/search_packge/test.py +1 -1
  153. re_common/v2/baselibrary/s3object/baseboto3.py +230 -230
  154. re_common/v2/baselibrary/tools/WeChatRobot.py +95 -95
  155. re_common/v2/baselibrary/tools/ac_ahocorasick.py +75 -75
  156. re_common/v2/baselibrary/tools/concurrency.py +35 -35
  157. re_common/v2/baselibrary/tools/data_processer/base.py +53 -53
  158. re_common/v2/baselibrary/tools/data_processer/data_processer.py +508 -508
  159. re_common/v2/baselibrary/tools/data_processer/data_reader.py +187 -187
  160. re_common/v2/baselibrary/tools/data_processer/data_writer.py +38 -38
  161. re_common/v2/baselibrary/tools/dict_tools.py +44 -44
  162. re_common/v2/baselibrary/tools/dolphinscheduler.py +187 -187
  163. re_common/v2/baselibrary/tools/hdfs_base_processor.py +204 -204
  164. re_common/v2/baselibrary/tools/hdfs_bulk_processor.py +67 -67
  165. re_common/v2/baselibrary/tools/hdfs_data_processer.py +338 -338
  166. re_common/v2/baselibrary/tools/hdfs_line_processor.py +74 -74
  167. re_common/v2/baselibrary/tools/list_tools.py +69 -69
  168. re_common/v2/baselibrary/tools/resume_tracker.py +94 -94
  169. re_common/v2/baselibrary/tools/search_hash_tools.py +54 -54
  170. re_common/v2/baselibrary/tools/text_matcher.py +326 -326
  171. re_common/v2/baselibrary/tools/unionfind_tools.py +60 -60
  172. re_common/v2/baselibrary/utils/BusinessStringUtil.py +196 -196
  173. re_common/v2/baselibrary/utils/api_net_utils.py +270 -270
  174. re_common/v2/baselibrary/utils/author_smi.py +361 -361
  175. re_common/v2/baselibrary/utils/base_string_similarity.py +158 -158
  176. re_common/v2/baselibrary/utils/basedict.py +37 -37
  177. re_common/v2/baselibrary/utils/basehdfs.py +163 -163
  178. re_common/v2/baselibrary/utils/basepika.py +180 -180
  179. re_common/v2/baselibrary/utils/basetime.py +77 -77
  180. re_common/v2/baselibrary/utils/db.py +156 -156
  181. re_common/v2/baselibrary/utils/json_cls.py +16 -16
  182. re_common/v2/baselibrary/utils/mq.py +83 -83
  183. re_common/v2/baselibrary/utils/n_ary_expression_tree.py +243 -243
  184. re_common/v2/baselibrary/utils/string_bool.py +186 -186
  185. re_common/v2/baselibrary/utils/string_clear.py +246 -246
  186. re_common/v2/baselibrary/utils/string_smi.py +18 -18
  187. re_common/v2/baselibrary/utils/stringutils.py +271 -278
  188. re_common/vip/base_step_process.py +11 -11
  189. re_common/vip/baseencodeid.py +90 -90
  190. re_common/vip/changetaskname.py +28 -28
  191. re_common/vip/core_var.py +24 -24
  192. re_common/vip/mmh3Hash.py +89 -89
  193. re_common/vip/proxy/allproxys.py +127 -127
  194. re_common/vip/proxy/allproxys_thread.py +159 -159
  195. re_common/vip/proxy/cnki_proxy.py +153 -153
  196. re_common/vip/proxy/kuaidaili.py +87 -87
  197. re_common/vip/proxy/proxy_all.py +113 -113
  198. re_common/vip/proxy/update_kuaidaili_0.py +42 -42
  199. re_common/vip/proxy/wanfang_proxy.py +152 -152
  200. re_common/vip/proxy/wp_proxy_all.py +181 -181
  201. re_common/vip/read_rawid_to_txt.py +91 -91
  202. re_common/vip/title/__init__.py +5 -5
  203. re_common/vip/title/transform/TransformBookTitleToZt.py +125 -125
  204. re_common/vip/title/transform/TransformConferenceTitleToZt.py +139 -139
  205. re_common/vip/title/transform/TransformCstadTitleToZt.py +195 -195
  206. re_common/vip/title/transform/TransformJournalTitleToZt.py +203 -203
  207. re_common/vip/title/transform/TransformPatentTitleToZt.py +132 -132
  208. re_common/vip/title/transform/TransformRegulationTitleToZt.py +114 -114
  209. re_common/vip/title/transform/TransformStandardTitleToZt.py +135 -135
  210. re_common/vip/title/transform/TransformThesisTitleToZt.py +135 -135
  211. re_common/vip/title/transform/__init__.py +10 -10
  212. {re_common-10.0.37.dist-info → re_common-10.0.39.dist-info}/LICENSE +201 -201
  213. {re_common-10.0.37.dist-info → re_common-10.0.39.dist-info}/METADATA +16 -16
  214. re_common-10.0.39.dist-info/RECORD +248 -0
  215. {re_common-10.0.37.dist-info → re_common-10.0.39.dist-info}/WHEEL +1 -1
  216. re_common-10.0.37.dist-info/RECORD +0 -248
  217. {re_common-10.0.37.dist-info → re_common-10.0.39.dist-info}/top_level.txt +0 -0
re_common/facade/now.py CHANGED
@@ -1,31 +1,31 @@
1
- # 这个库会加载所有的模块,也可以知道本基础库使用了哪些模块
2
- from re_common.baselibrary.utils.baserequest import BaseRequest
3
-
4
- from .loggerfacade import MLogger
5
-
6
- __all__ = ['get_streamlogger', 'get_filelogger', 'get_timerotatingfilelogger', 'BaseRequest']
7
-
8
- """
9
- 提供logger的使用外观 外观模式
10
- """
11
-
12
-
13
- def get_streamlogger():
14
- """
15
- 提供一个输出流的logger
16
- :return:
17
- """
18
- return MLogger().streamlogger
19
-
20
-
21
- def get_filelogger(filedir):
22
- """
23
- 文件流的logger
24
- :param filedir:
25
- :return:
26
- """
27
- return MLogger(filedir).filelogger
28
-
29
-
30
- def get_timerotatingfilelogger(filedir, fileter=None):
31
- return MLogger(filedir, fileter).timerotatingfilelogger
1
+ # 这个库会加载所有的模块,也可以知道本基础库使用了哪些模块
2
+ from re_common.baselibrary.utils.baserequest import BaseRequest
3
+
4
+ from .loggerfacade import MLogger
5
+
6
+ __all__ = ['get_streamlogger', 'get_filelogger', 'get_timerotatingfilelogger', 'BaseRequest']
7
+
8
+ """
9
+ 提供logger的使用外观 外观模式
10
+ """
11
+
12
+
13
+ def get_streamlogger():
14
+ """
15
+ 提供一个输出流的logger
16
+ :return:
17
+ """
18
+ return MLogger().streamlogger
19
+
20
+
21
+ def get_filelogger(filedir):
22
+ """
23
+ 文件流的logger
24
+ :param filedir:
25
+ :return:
26
+ """
27
+ return MLogger(filedir).filelogger
28
+
29
+
30
+ def get_timerotatingfilelogger(filedir, fileter=None):
31
+ return MLogger(filedir, fileter).timerotatingfilelogger
@@ -1,257 +1,257 @@
1
- # db3数据的连接返回一个连接对象
2
- import sys
3
- import traceback
4
-
5
- from re_common.baselibrary import MLogger
6
- from re_common.baselibrary.baseabs import BaseAbs
7
- from re_common.baselibrary.database.mbuilder import Sqlite3Builder
8
-
9
-
10
- class Sqlite3Utiles(object):
11
- def __init__(self, logger=None):
12
- self.sqllite3 = None
13
- self.conn = None
14
- self.cursor = None
15
-
16
- self._logger = logger
17
-
18
- @property
19
- def logger(self):
20
- if self._logger is None:
21
- return MLogger().streamlogger
22
- return self._logger
23
-
24
- @logger.setter
25
- def logger(self, value):
26
- assert isinstance(value, MLogger)
27
- self._logger = value
28
-
29
- def get_new_cursor(self):
30
- """
31
- 获取一个新的游标
32
- :return:
33
- """
34
- # 检查db的存在及是否断掉 不知道为什么 不可以加括号 但编辑器会警告
35
- return self.conn.cursor()
36
-
37
- def Sqlite3DBConnectFromFilePath(self, dbfilepath, encoding="gbk", timeout=60):
38
- """
39
- 通过直接文件连接 我使用Hadoop导下来的db3是gbk编码的
40
- 如果为utf-8编码请改为utf-8
41
- :param sec: section
42
- :return:
43
- """
44
- sqllite3 = BaseAbs.get_sql_factory().sqlite_factory()
45
- # 传入的是配置文件和section选项 dbpath为配置文件路径
46
- sqlite3_moudle = Sqlite3Builder().build_file_path(dbfilepath).build_timeout(timeout).get_moudle()
47
- sqllite3.link(sqlite3_moudle)
48
- # 设置txt的编码格式 hadoop 上的db3为gbk 默认为utf8
49
- sqllite3.set_encoding(encoding=encoding)
50
- # 返回一个连接
51
- self.sqllite3 = sqllite3
52
- self.conn = self.sqllite3.db
53
- return self
54
-
55
- def Sqlite3DBConnectFromConfig(self, cobnfigfilepath, sec, encoding="gbk"):
56
- """
57
- 通过配置文件连接
58
- :param sec: section
59
- :return:
60
- """
61
- sqllite3 = BaseAbs.get_sql_factory().sqlite_factory()
62
- # 传入的是配置文件和section选项 dbpath为配置文件路径
63
- sqlite3_moudle = Sqlite3Builder(cobnfigfilepath, sec).build_all().get_moudle()
64
- sqllite3.link(sqlite3_moudle)
65
- # 设置txt的编码格式 hadoop 上的db3为gbk 默认为utf8
66
- sqllite3.set_encoding(encoding=encoding)
67
- # 返回一个连接
68
- self.sqllite3 = sqllite3
69
- self.conn = self.sqllite3.db
70
- return self
71
-
72
- def ExeSqlliteList(self, sqlList, errExit=True):
73
- """
74
- 该函数和上面一样执行一个sql列表且不返回结果
75
- 属于插入和更新类函数 但该函数针对db3
76
- :param sqllitename:
77
- :param sqlList:
78
- :return:
79
- """
80
- dbMsg = None
81
- cur = self.get_new_cursor()
82
- if self.conn:
83
- count = 0
84
- for sql in sqlList:
85
- count += 1
86
- try:
87
- self.logger.info("{} 执行sql数量:{}".format(sql, str(count)))
88
- cur.execute(sql)
89
- except:
90
- self.logger.error('*errSql:' + sql)
91
- dbMsg = '*InsertError:' + traceback.format_exc()
92
- if errExit:
93
- self.logger.error(dbMsg)
94
- sys.exit()
95
- if dbMsg:
96
- self.logger.error(dbMsg)
97
- continue
98
- self.conn.commit()
99
- cur.close()
100
-
101
- def ExeSqlliteMany(self, sql, itermany, errExit=True):
102
- dbMsg = None
103
- cur = self.get_new_cursor()
104
- if self.conn:
105
- try:
106
- self.logger.info("{}\n{}".format(sql, str(itermany)))
107
- cur.executemany(sql, itermany)
108
- except:
109
- self.logger.error('*errSql:' + sql)
110
- dbMsg = '*InsertError:' + traceback.format_exc()
111
- if errExit:
112
- self.logger.error(dbMsg)
113
- sys.exit()
114
- if dbMsg:
115
- self.logger.error(dbMsg)
116
- self.conn.commit()
117
- cur.close()
118
-
119
- def ExeSqlliteSql(self, sql):
120
- """
121
- 该函数和上面一样执行一个sql列表且不返回结果
122
- 属于插入和更新类函数 但该函数针对db3
123
- :param sqllitename:
124
- :param sqlList:
125
- :return:
126
- """
127
- dbMsg = None
128
- cur = self.get_new_cursor()
129
- if self.conn:
130
- try:
131
- self.logger.info(sql)
132
- cur.execute(sql)
133
- self.conn.commit()
134
- except:
135
- self.logger.error('*errSql:' + sql)
136
- dbMsg = '*InsertError:' + traceback.format_exc()
137
- if dbMsg:
138
- self.logger.error(dbMsg)
139
- return False
140
- else:
141
- return False
142
- cur.close()
143
- return True
144
-
145
- def SelectFromSqlliteFetchall(self, sql):
146
- """
147
- 该函数和上面一样执行一个sql列表且不返回结果
148
- 属于插入和更新类函数 但该函数针对db3
149
- :param sqllitename:
150
- :param sqlList:
151
- :return:
152
- """
153
- cur = self.get_new_cursor()
154
- if self.conn:
155
- try:
156
- self.logger.info(sql)
157
- cur.execute(sql)
158
- rows = cur.fetchall()
159
- return rows
160
- except:
161
- self.logger.error('*errSql:' + sql)
162
- dbMsg = '*InsertError:' + traceback.format_exc()
163
- if dbMsg:
164
- self.logger.error(dbMsg)
165
- return False
166
- else:
167
- return False
168
- cur.close()
169
-
170
- def SelectFromSqlliteFetchall_dicts(self, sql):
171
- """
172
- 该函数和上面一样执行一个sql列表且不返回结果
173
- 属于插入和更新类函数 但该函数针对db3
174
- :param sqllitename:
175
- :param sqlList:
176
- :return:
177
- """
178
-
179
- def dict_factory(cursor, row):
180
- d = {}
181
- for idx, col in enumerate(cursor.description):
182
- d[col[0]] = row[idx]
183
- return d
184
-
185
- self.conn.row_factory = dict_factory
186
- cur = self.get_new_cursor()
187
- if self.conn:
188
- try:
189
- self.logger.info(sql)
190
- cur.execute(sql)
191
- rows = cur.fetchall()
192
- return rows
193
- except:
194
- self.logger.error('*errSql:' + sql)
195
- dbMsg = '*InsertError:' + traceback.format_exc()
196
- if dbMsg:
197
- self.logger.error(dbMsg)
198
- return False
199
- else:
200
- return False
201
- cur.close()
202
-
203
- def SelectFromSqlliteFetchOne(self, sql):
204
- """
205
- 该函数和上面一样执行一个sql列表且不返回结果
206
- 属于插入和更新类函数 但该函数针对db3
207
- :param sqllitename:
208
- :param sqlList:
209
- :return:
210
- """
211
- dbMsg = None
212
- cur = self.get_new_cursor()
213
- if self.conn:
214
- try:
215
- self.logger.info(sql)
216
- cur.execute(sql)
217
- while True:
218
- row = cur.fetchone()
219
- if row is None:
220
- return None
221
- else:
222
- yield row
223
- except:
224
- self.logger.error('*errSql:' + sql)
225
- dbMsg = '*InsertError:' + traceback.format_exc()
226
- if dbMsg:
227
- self.logger.error(dbMsg)
228
- return False
229
- else:
230
- return False
231
- cur.close()
232
-
233
- def ExeVACUUM(self):
234
- """
235
- 清理空间
236
- :return:
237
- """
238
- dbMsg = None
239
- if self.conn:
240
- try:
241
- self.conn.execute("VACUUM")
242
- except:
243
- dbMsg = '*VACUUMError:' + traceback.format_exc()
244
- if dbMsg:
245
- self.logger.error(dbMsg)
246
- return False
247
- else:
248
- return False
249
- return True
250
-
251
- def sqliteEscape(self, keyWord):
252
- keyWord = keyWord.replace("'", "''")
253
- keyWord = keyWord.replace("\\", "\\\\")
254
- return keyWord
255
-
256
- def close(self):
257
- self.conn.close()
1
+ # db3数据的连接返回一个连接对象
2
+ import sys
3
+ import traceback
4
+
5
+ from re_common.baselibrary import MLogger
6
+ from re_common.baselibrary.baseabs import BaseAbs
7
+ from re_common.baselibrary.database.mbuilder import Sqlite3Builder
8
+
9
+
10
+ class Sqlite3Utiles(object):
11
+ def __init__(self, logger=None):
12
+ self.sqllite3 = None
13
+ self.conn = None
14
+ self.cursor = None
15
+
16
+ self._logger = logger
17
+
18
+ @property
19
+ def logger(self):
20
+ if self._logger is None:
21
+ return MLogger().streamlogger
22
+ return self._logger
23
+
24
+ @logger.setter
25
+ def logger(self, value):
26
+ assert isinstance(value, MLogger)
27
+ self._logger = value
28
+
29
+ def get_new_cursor(self):
30
+ """
31
+ 获取一个新的游标
32
+ :return:
33
+ """
34
+ # 检查db的存在及是否断掉 不知道为什么 不可以加括号 但编辑器会警告
35
+ return self.conn.cursor()
36
+
37
+ def Sqlite3DBConnectFromFilePath(self, dbfilepath, encoding="gbk", timeout=60):
38
+ """
39
+ 通过直接文件连接 我使用Hadoop导下来的db3是gbk编码的
40
+ 如果为utf-8编码请改为utf-8
41
+ :param sec: section
42
+ :return:
43
+ """
44
+ sqllite3 = BaseAbs.get_sql_factory().sqlite_factory()
45
+ # 传入的是配置文件和section选项 dbpath为配置文件路径
46
+ sqlite3_moudle = Sqlite3Builder().build_file_path(dbfilepath).build_timeout(timeout).get_moudle()
47
+ sqllite3.link(sqlite3_moudle)
48
+ # 设置txt的编码格式 hadoop 上的db3为gbk 默认为utf8
49
+ sqllite3.set_encoding(encoding=encoding)
50
+ # 返回一个连接
51
+ self.sqllite3 = sqllite3
52
+ self.conn = self.sqllite3.db
53
+ return self
54
+
55
+ def Sqlite3DBConnectFromConfig(self, cobnfigfilepath, sec, encoding="gbk"):
56
+ """
57
+ 通过配置文件连接
58
+ :param sec: section
59
+ :return:
60
+ """
61
+ sqllite3 = BaseAbs.get_sql_factory().sqlite_factory()
62
+ # 传入的是配置文件和section选项 dbpath为配置文件路径
63
+ sqlite3_moudle = Sqlite3Builder(cobnfigfilepath, sec).build_all().get_moudle()
64
+ sqllite3.link(sqlite3_moudle)
65
+ # 设置txt的编码格式 hadoop 上的db3为gbk 默认为utf8
66
+ sqllite3.set_encoding(encoding=encoding)
67
+ # 返回一个连接
68
+ self.sqllite3 = sqllite3
69
+ self.conn = self.sqllite3.db
70
+ return self
71
+
72
+ def ExeSqlliteList(self, sqlList, errExit=True):
73
+ """
74
+ 该函数和上面一样执行一个sql列表且不返回结果
75
+ 属于插入和更新类函数 但该函数针对db3
76
+ :param sqllitename:
77
+ :param sqlList:
78
+ :return:
79
+ """
80
+ dbMsg = None
81
+ cur = self.get_new_cursor()
82
+ if self.conn:
83
+ count = 0
84
+ for sql in sqlList:
85
+ count += 1
86
+ try:
87
+ self.logger.info("{} 执行sql数量:{}".format(sql, str(count)))
88
+ cur.execute(sql)
89
+ except:
90
+ self.logger.error('*errSql:' + sql)
91
+ dbMsg = '*InsertError:' + traceback.format_exc()
92
+ if errExit:
93
+ self.logger.error(dbMsg)
94
+ sys.exit()
95
+ if dbMsg:
96
+ self.logger.error(dbMsg)
97
+ continue
98
+ self.conn.commit()
99
+ cur.close()
100
+
101
+ def ExeSqlliteMany(self, sql, itermany, errExit=True):
102
+ dbMsg = None
103
+ cur = self.get_new_cursor()
104
+ if self.conn:
105
+ try:
106
+ self.logger.info("{}\n{}".format(sql, str(itermany)))
107
+ cur.executemany(sql, itermany)
108
+ except:
109
+ self.logger.error('*errSql:' + sql)
110
+ dbMsg = '*InsertError:' + traceback.format_exc()
111
+ if errExit:
112
+ self.logger.error(dbMsg)
113
+ sys.exit()
114
+ if dbMsg:
115
+ self.logger.error(dbMsg)
116
+ self.conn.commit()
117
+ cur.close()
118
+
119
+ def ExeSqlliteSql(self, sql):
120
+ """
121
+ 该函数和上面一样执行一个sql列表且不返回结果
122
+ 属于插入和更新类函数 但该函数针对db3
123
+ :param sqllitename:
124
+ :param sqlList:
125
+ :return:
126
+ """
127
+ dbMsg = None
128
+ cur = self.get_new_cursor()
129
+ if self.conn:
130
+ try:
131
+ self.logger.info(sql)
132
+ cur.execute(sql)
133
+ self.conn.commit()
134
+ except:
135
+ self.logger.error('*errSql:' + sql)
136
+ dbMsg = '*InsertError:' + traceback.format_exc()
137
+ if dbMsg:
138
+ self.logger.error(dbMsg)
139
+ return False
140
+ else:
141
+ return False
142
+ cur.close()
143
+ return True
144
+
145
+ def SelectFromSqlliteFetchall(self, sql):
146
+ """
147
+ 该函数和上面一样执行一个sql列表且不返回结果
148
+ 属于插入和更新类函数 但该函数针对db3
149
+ :param sqllitename:
150
+ :param sqlList:
151
+ :return:
152
+ """
153
+ cur = self.get_new_cursor()
154
+ if self.conn:
155
+ try:
156
+ self.logger.info(sql)
157
+ cur.execute(sql)
158
+ rows = cur.fetchall()
159
+ return rows
160
+ except:
161
+ self.logger.error('*errSql:' + sql)
162
+ dbMsg = '*InsertError:' + traceback.format_exc()
163
+ if dbMsg:
164
+ self.logger.error(dbMsg)
165
+ return False
166
+ else:
167
+ return False
168
+ cur.close()
169
+
170
+ def SelectFromSqlliteFetchall_dicts(self, sql):
171
+ """
172
+ 该函数和上面一样执行一个sql列表且不返回结果
173
+ 属于插入和更新类函数 但该函数针对db3
174
+ :param sqllitename:
175
+ :param sqlList:
176
+ :return:
177
+ """
178
+
179
+ def dict_factory(cursor, row):
180
+ d = {}
181
+ for idx, col in enumerate(cursor.description):
182
+ d[col[0]] = row[idx]
183
+ return d
184
+
185
+ self.conn.row_factory = dict_factory
186
+ cur = self.get_new_cursor()
187
+ if self.conn:
188
+ try:
189
+ self.logger.info(sql)
190
+ cur.execute(sql)
191
+ rows = cur.fetchall()
192
+ return rows
193
+ except:
194
+ self.logger.error('*errSql:' + sql)
195
+ dbMsg = '*InsertError:' + traceback.format_exc()
196
+ if dbMsg:
197
+ self.logger.error(dbMsg)
198
+ return False
199
+ else:
200
+ return False
201
+ cur.close()
202
+
203
+ def SelectFromSqlliteFetchOne(self, sql):
204
+ """
205
+ 该函数和上面一样执行一个sql列表且不返回结果
206
+ 属于插入和更新类函数 但该函数针对db3
207
+ :param sqllitename:
208
+ :param sqlList:
209
+ :return:
210
+ """
211
+ dbMsg = None
212
+ cur = self.get_new_cursor()
213
+ if self.conn:
214
+ try:
215
+ self.logger.info(sql)
216
+ cur.execute(sql)
217
+ while True:
218
+ row = cur.fetchone()
219
+ if row is None:
220
+ return None
221
+ else:
222
+ yield row
223
+ except:
224
+ self.logger.error('*errSql:' + sql)
225
+ dbMsg = '*InsertError:' + traceback.format_exc()
226
+ if dbMsg:
227
+ self.logger.error(dbMsg)
228
+ return False
229
+ else:
230
+ return False
231
+ cur.close()
232
+
233
+ def ExeVACUUM(self):
234
+ """
235
+ 清理空间
236
+ :return:
237
+ """
238
+ dbMsg = None
239
+ if self.conn:
240
+ try:
241
+ self.conn.execute("VACUUM")
242
+ except:
243
+ dbMsg = '*VACUUMError:' + traceback.format_exc()
244
+ if dbMsg:
245
+ self.logger.error(dbMsg)
246
+ return False
247
+ else:
248
+ return False
249
+ return True
250
+
251
+ def sqliteEscape(self, keyWord):
252
+ keyWord = keyWord.replace("'", "''")
253
+ keyWord = keyWord.replace("\\", "\\\\")
254
+ return keyWord
255
+
256
+ def close(self):
257
+ self.conn.close()