re-common 10.0.37__py3-none-any.whl → 10.0.39__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (217) hide show
  1. re_common/baselibrary/__init__.py +4 -4
  2. re_common/baselibrary/baseabs/__init__.py +6 -6
  3. re_common/baselibrary/baseabs/baseabs.py +26 -26
  4. re_common/baselibrary/database/mbuilder.py +132 -132
  5. re_common/baselibrary/database/moudle.py +93 -93
  6. re_common/baselibrary/database/msqlite3.py +194 -194
  7. re_common/baselibrary/database/mysql.py +169 -169
  8. re_common/baselibrary/database/sql_factory.py +26 -26
  9. re_common/baselibrary/mthread/MThreadingRun.py +486 -486
  10. re_common/baselibrary/mthread/MThreadingRunEvent.py +349 -349
  11. re_common/baselibrary/mthread/__init__.py +2 -2
  12. re_common/baselibrary/mthread/mythreading.py +695 -695
  13. re_common/baselibrary/pakge_other/socks.py +404 -404
  14. re_common/baselibrary/readconfig/config_factory.py +18 -18
  15. re_common/baselibrary/readconfig/ini_config.py +317 -317
  16. re_common/baselibrary/readconfig/toml_config.py +49 -49
  17. re_common/baselibrary/temporary/envdata.py +36 -36
  18. re_common/baselibrary/tools/all_requests/aiohttp_request.py +118 -118
  19. re_common/baselibrary/tools/all_requests/httpx_requet.py +102 -102
  20. re_common/baselibrary/tools/all_requests/mrequest.py +412 -412
  21. re_common/baselibrary/tools/all_requests/requests_request.py +81 -81
  22. re_common/baselibrary/tools/batch_compre/bijiao_batch.py +31 -31
  23. re_common/baselibrary/tools/contrast_db3.py +123 -123
  24. re_common/baselibrary/tools/copy_file.py +39 -39
  25. re_common/baselibrary/tools/db3_2_sizedb3.py +102 -102
  26. re_common/baselibrary/tools/foreachgz.py +39 -39
  27. re_common/baselibrary/tools/get_attr.py +10 -10
  28. re_common/baselibrary/tools/image_to_pdf.py +61 -61
  29. re_common/baselibrary/tools/java_code_deal.py +139 -139
  30. re_common/baselibrary/tools/javacode.py +79 -79
  31. re_common/baselibrary/tools/mdb_db3.py +48 -48
  32. re_common/baselibrary/tools/merge_file.py +171 -171
  33. re_common/baselibrary/tools/merge_gz_file.py +165 -165
  34. re_common/baselibrary/tools/mhdfstools/down_hdfs_files.py +42 -42
  35. re_common/baselibrary/tools/mhdfstools/hdfst.py +42 -42
  36. re_common/baselibrary/tools/mhdfstools/up_hdfs_files.py +38 -38
  37. re_common/baselibrary/tools/mongo_tools.py +50 -50
  38. re_common/baselibrary/tools/move_file.py +170 -170
  39. re_common/baselibrary/tools/move_mongo/mongo_table_to_file.py +63 -63
  40. re_common/baselibrary/tools/move_mongo/move_mongo_table.py +354 -354
  41. re_common/baselibrary/tools/move_mongo/use_mttf.py +18 -18
  42. re_common/baselibrary/tools/move_mongo/use_mv.py +93 -93
  43. re_common/baselibrary/tools/mpandas/mpandasreadexcel.py +125 -125
  44. re_common/baselibrary/tools/mpandas/pandas_visualization.py +7 -7
  45. re_common/baselibrary/tools/myparsel.py +104 -104
  46. re_common/baselibrary/tools/rename_dir_file.py +37 -37
  47. re_common/baselibrary/tools/sequoiadb_utils.py +398 -398
  48. re_common/baselibrary/tools/split_line_to_many.py +25 -25
  49. re_common/baselibrary/tools/stringtodicts.py +33 -33
  50. re_common/baselibrary/tools/workwechant_bot.py +84 -84
  51. re_common/baselibrary/utils/baseaiohttp.py +296 -296
  52. re_common/baselibrary/utils/baseaiomysql.py +87 -87
  53. re_common/baselibrary/utils/baseallstep.py +191 -191
  54. re_common/baselibrary/utils/baseavro.py +19 -19
  55. re_common/baselibrary/utils/baseboto3.py +291 -291
  56. re_common/baselibrary/utils/basecsv.py +32 -32
  57. re_common/baselibrary/utils/basedict.py +133 -133
  58. re_common/baselibrary/utils/basedir.py +241 -241
  59. re_common/baselibrary/utils/baseencode.py +351 -351
  60. re_common/baselibrary/utils/baseencoding.py +28 -28
  61. re_common/baselibrary/utils/baseesdsl.py +86 -86
  62. re_common/baselibrary/utils/baseexcel.py +264 -264
  63. re_common/baselibrary/utils/baseexcept.py +109 -109
  64. re_common/baselibrary/utils/basefile.py +654 -654
  65. re_common/baselibrary/utils/baseftp.py +214 -214
  66. re_common/baselibrary/utils/basegzip.py +60 -60
  67. re_common/baselibrary/utils/basehdfs.py +135 -135
  68. re_common/baselibrary/utils/basehttpx.py +268 -268
  69. re_common/baselibrary/utils/baseip.py +87 -87
  70. re_common/baselibrary/utils/basejson.py +2 -2
  71. re_common/baselibrary/utils/baselist.py +32 -32
  72. re_common/baselibrary/utils/basemotor.py +190 -190
  73. re_common/baselibrary/utils/basemssql.py +98 -98
  74. re_common/baselibrary/utils/baseodbc.py +113 -113
  75. re_common/baselibrary/utils/basepandas.py +302 -302
  76. re_common/baselibrary/utils/basepeewee.py +11 -11
  77. re_common/baselibrary/utils/basepika.py +180 -180
  78. re_common/baselibrary/utils/basepydash.py +143 -143
  79. re_common/baselibrary/utils/basepymongo.py +230 -230
  80. re_common/baselibrary/utils/basequeue.py +22 -22
  81. re_common/baselibrary/utils/baserar.py +57 -57
  82. re_common/baselibrary/utils/baserequest.py +279 -279
  83. re_common/baselibrary/utils/baseset.py +8 -8
  84. re_common/baselibrary/utils/basesmb.py +403 -403
  85. re_common/baselibrary/utils/basestring.py +382 -382
  86. re_common/baselibrary/utils/basetime.py +320 -320
  87. re_common/baselibrary/utils/baseurl.py +121 -121
  88. re_common/baselibrary/utils/basezip.py +57 -57
  89. re_common/baselibrary/utils/core/__init__.py +7 -7
  90. re_common/baselibrary/utils/core/bottomutils.py +18 -18
  91. re_common/baselibrary/utils/core/mdeprecated.py +327 -327
  92. re_common/baselibrary/utils/core/mlamada.py +16 -16
  93. re_common/baselibrary/utils/core/msginfo.py +25 -25
  94. re_common/baselibrary/utils/core/requests_core.py +103 -103
  95. re_common/baselibrary/utils/fateadm.py +429 -429
  96. re_common/baselibrary/utils/importfun.py +123 -123
  97. re_common/baselibrary/utils/mfaker.py +57 -57
  98. re_common/baselibrary/utils/my_abc/__init__.py +3 -3
  99. re_common/baselibrary/utils/my_abc/better_abc.py +32 -32
  100. re_common/baselibrary/utils/mylogger.py +414 -414
  101. re_common/baselibrary/utils/myredisclient.py +861 -861
  102. re_common/baselibrary/utils/pipupgrade.py +21 -21
  103. re_common/baselibrary/utils/ringlist.py +85 -85
  104. re_common/baselibrary/utils/version_compare.py +36 -36
  105. re_common/baselibrary/utils/ydmhttp.py +126 -126
  106. re_common/facade/lazy_import.py +11 -11
  107. re_common/facade/loggerfacade.py +25 -25
  108. re_common/facade/mysqlfacade.py +467 -467
  109. re_common/facade/now.py +31 -31
  110. re_common/facade/sqlite3facade.py +257 -257
  111. re_common/facade/use/mq_use_facade.py +83 -83
  112. re_common/facade/use/proxy_use_facade.py +19 -19
  113. re_common/libtest/base_dict_test.py +19 -19
  114. re_common/libtest/baseavro_test.py +13 -13
  115. re_common/libtest/basefile_test.py +14 -14
  116. re_common/libtest/basemssql_test.py +77 -77
  117. re_common/libtest/baseodbc_test.py +7 -7
  118. re_common/libtest/basepandas_test.py +38 -38
  119. re_common/libtest/get_attr_test/get_attr_test_settings.py +14 -14
  120. re_common/libtest/get_attr_test/settings.py +54 -54
  121. re_common/libtest/idencode_test.py +53 -53
  122. re_common/libtest/iniconfig_test.py +35 -35
  123. re_common/libtest/ip_test.py +34 -34
  124. re_common/libtest/merge_file_test.py +20 -20
  125. re_common/libtest/mfaker_test.py +8 -8
  126. re_common/libtest/mm3_test.py +31 -31
  127. re_common/libtest/mylogger_test.py +88 -88
  128. re_common/libtest/myparsel_test.py +27 -27
  129. re_common/libtest/mysql_test.py +151 -151
  130. re_common/libtest/pymongo_test.py +21 -21
  131. re_common/libtest/split_test.py +11 -11
  132. re_common/libtest/sqlite3_merge_test.py +5 -5
  133. re_common/libtest/sqlite3_test.py +34 -34
  134. re_common/libtest/tomlconfig_test.py +30 -30
  135. re_common/libtest/use_tools_test/__init__.py +2 -2
  136. re_common/libtest/user/__init__.py +4 -4
  137. re_common/studio/__init__.py +4 -4
  138. re_common/studio/assignment_expressions.py +36 -36
  139. re_common/studio/mydash/test1.py +18 -18
  140. re_common/studio/pydashstudio/first.py +9 -9
  141. re_common/studio/streamlitstudio/first_app.py +65 -65
  142. re_common/studio/streamlitstudio/uber_pickups.py +23 -23
  143. re_common/studio/test.py +18 -18
  144. re_common/v2/baselibrary/business_utils/BusinessStringUtil.py +219 -219
  145. re_common/v2/baselibrary/business_utils/baseencodeid.py +100 -100
  146. re_common/v2/baselibrary/business_utils/full_doi_path.py +116 -116
  147. re_common/v2/baselibrary/business_utils/rel_tools.py +6 -6
  148. re_common/v2/baselibrary/decorators/utils.py +59 -59
  149. re_common/v2/baselibrary/helpers/search_packge/NearestNeighbors_test.py +105 -105
  150. re_common/v2/baselibrary/helpers/search_packge/fit_text_match.py +253 -253
  151. re_common/v2/baselibrary/helpers/search_packge/scikit_learn_text_matcher.py +260 -260
  152. re_common/v2/baselibrary/helpers/search_packge/test.py +1 -1
  153. re_common/v2/baselibrary/s3object/baseboto3.py +230 -230
  154. re_common/v2/baselibrary/tools/WeChatRobot.py +95 -95
  155. re_common/v2/baselibrary/tools/ac_ahocorasick.py +75 -75
  156. re_common/v2/baselibrary/tools/concurrency.py +35 -35
  157. re_common/v2/baselibrary/tools/data_processer/base.py +53 -53
  158. re_common/v2/baselibrary/tools/data_processer/data_processer.py +508 -508
  159. re_common/v2/baselibrary/tools/data_processer/data_reader.py +187 -187
  160. re_common/v2/baselibrary/tools/data_processer/data_writer.py +38 -38
  161. re_common/v2/baselibrary/tools/dict_tools.py +44 -44
  162. re_common/v2/baselibrary/tools/dolphinscheduler.py +187 -187
  163. re_common/v2/baselibrary/tools/hdfs_base_processor.py +204 -204
  164. re_common/v2/baselibrary/tools/hdfs_bulk_processor.py +67 -67
  165. re_common/v2/baselibrary/tools/hdfs_data_processer.py +338 -338
  166. re_common/v2/baselibrary/tools/hdfs_line_processor.py +74 -74
  167. re_common/v2/baselibrary/tools/list_tools.py +69 -69
  168. re_common/v2/baselibrary/tools/resume_tracker.py +94 -94
  169. re_common/v2/baselibrary/tools/search_hash_tools.py +54 -54
  170. re_common/v2/baselibrary/tools/text_matcher.py +326 -326
  171. re_common/v2/baselibrary/tools/unionfind_tools.py +60 -60
  172. re_common/v2/baselibrary/utils/BusinessStringUtil.py +196 -196
  173. re_common/v2/baselibrary/utils/api_net_utils.py +270 -270
  174. re_common/v2/baselibrary/utils/author_smi.py +361 -361
  175. re_common/v2/baselibrary/utils/base_string_similarity.py +158 -158
  176. re_common/v2/baselibrary/utils/basedict.py +37 -37
  177. re_common/v2/baselibrary/utils/basehdfs.py +163 -163
  178. re_common/v2/baselibrary/utils/basepika.py +180 -180
  179. re_common/v2/baselibrary/utils/basetime.py +77 -77
  180. re_common/v2/baselibrary/utils/db.py +156 -156
  181. re_common/v2/baselibrary/utils/json_cls.py +16 -16
  182. re_common/v2/baselibrary/utils/mq.py +83 -83
  183. re_common/v2/baselibrary/utils/n_ary_expression_tree.py +243 -243
  184. re_common/v2/baselibrary/utils/string_bool.py +186 -186
  185. re_common/v2/baselibrary/utils/string_clear.py +246 -246
  186. re_common/v2/baselibrary/utils/string_smi.py +18 -18
  187. re_common/v2/baselibrary/utils/stringutils.py +271 -278
  188. re_common/vip/base_step_process.py +11 -11
  189. re_common/vip/baseencodeid.py +90 -90
  190. re_common/vip/changetaskname.py +28 -28
  191. re_common/vip/core_var.py +24 -24
  192. re_common/vip/mmh3Hash.py +89 -89
  193. re_common/vip/proxy/allproxys.py +127 -127
  194. re_common/vip/proxy/allproxys_thread.py +159 -159
  195. re_common/vip/proxy/cnki_proxy.py +153 -153
  196. re_common/vip/proxy/kuaidaili.py +87 -87
  197. re_common/vip/proxy/proxy_all.py +113 -113
  198. re_common/vip/proxy/update_kuaidaili_0.py +42 -42
  199. re_common/vip/proxy/wanfang_proxy.py +152 -152
  200. re_common/vip/proxy/wp_proxy_all.py +181 -181
  201. re_common/vip/read_rawid_to_txt.py +91 -91
  202. re_common/vip/title/__init__.py +5 -5
  203. re_common/vip/title/transform/TransformBookTitleToZt.py +125 -125
  204. re_common/vip/title/transform/TransformConferenceTitleToZt.py +139 -139
  205. re_common/vip/title/transform/TransformCstadTitleToZt.py +195 -195
  206. re_common/vip/title/transform/TransformJournalTitleToZt.py +203 -203
  207. re_common/vip/title/transform/TransformPatentTitleToZt.py +132 -132
  208. re_common/vip/title/transform/TransformRegulationTitleToZt.py +114 -114
  209. re_common/vip/title/transform/TransformStandardTitleToZt.py +135 -135
  210. re_common/vip/title/transform/TransformThesisTitleToZt.py +135 -135
  211. re_common/vip/title/transform/__init__.py +10 -10
  212. {re_common-10.0.37.dist-info → re_common-10.0.39.dist-info}/LICENSE +201 -201
  213. {re_common-10.0.37.dist-info → re_common-10.0.39.dist-info}/METADATA +16 -16
  214. re_common-10.0.39.dist-info/RECORD +248 -0
  215. {re_common-10.0.37.dist-info → re_common-10.0.39.dist-info}/WHEEL +1 -1
  216. re_common-10.0.37.dist-info/RECORD +0 -248
  217. {re_common-10.0.37.dist-info → re_common-10.0.39.dist-info}/top_level.txt +0 -0
@@ -1,398 +1,398 @@
1
- from typing import Union
2
-
3
- from pysequoiadb import client
4
- from pysequoiadb.collection import INSERT_FLG_DEFAULT
5
- from pysequoiadb.lob import LOB_READ, LOB_WRITE
6
-
7
-
8
- class SequoiadbUtils(object):
9
-
10
- def __init__(self):
11
- self.db_data = None
12
- self.db = None
13
- self.cs = None
14
- self.cl = None
15
- self.list_hosts = None
16
-
17
- def set_db_data(self, db_data: dict):
18
- self.db_data = db_data
19
- return self
20
-
21
- def set_list_hosts(self, list_hosts):
22
- self.list_hosts = list_hosts
23
- return self
24
-
25
- def build(self):
26
- self.link_db()
27
- self.link_col()
28
- return self
29
-
30
- def set_cl(self, cl):
31
- self.cl = cl
32
-
33
- def link_db(self):
34
- """
35
- 初始化 host, port, user, password ,连接
36
- """
37
- host = self.db_data.get("host", "")
38
- port = self.db_data.get("port", "")
39
- user = self.db_data.get("user", "")
40
- password = self.db_data.get("password", "")
41
- try:
42
- self.db = client(host=host, service=port, user=user, psw=password)
43
- if self.list_hosts:
44
- # self.list_hosts = [
45
- # {'host': '192.168.31.21', 'service': '11810'},
46
- # {'host': '192.168.31.22', 'service': '11810'},
47
- # {'host': '192.168.31.23', 'service': '11810'},
48
- # {'host': '192.168.31.24', 'service': '11810'},
49
- # {'host': '192.168.31.25', 'service': '11810'},
50
- # {'host': '192.168.31.26', 'service': '11810'},
51
- # {'host': '192.168.31.27', 'service': '11810'},
52
- # {'host': '192.168.31.219', 'service': '11810'},
53
- # {'host': '192.168.31.208', 'service': '11810'}
54
- # ]
55
- self.db.connect_to_hosts(self.list_hosts, user=user, psw=password)
56
- return self.db
57
- except Exception as e:
58
- raise e
59
-
60
- def link_col(self):
61
- collection_space_name = self.db_data.get("collection_space_name", "")
62
- collection_name = self.db_data.get("collection_name", "")
63
- is_create = self.db_data.get("is_create", "false")
64
- try:
65
- self.cl = self.db.get_collection("{}.{}".format(collection_space_name, collection_name))
66
- return self.cl
67
- except:
68
- print('集合空间或者集合不存在')
69
- if is_create == "true":
70
- try:
71
- self.cs = self.db.create_collection_space(collection_space_name)
72
- print('创建集合空间 ', collection_space_name)
73
- except:
74
- self.cs = self.db.get_collection_space(collection_space_name)
75
- try:
76
- self.cl = self.cs.create_collection(collection_name)
77
- print('创建集合空间 ', collection_name)
78
- return self.cl
79
- except:
80
- self.cl = self.cs.get_collection(collection_name)
81
-
82
- def insert(self, record: dict, cl=None):
83
- """
84
- 插入数据
85
- :return A ObjectId object of record inserted. eg: { '_id': ObjectId('5d5149ade3071dce3692e93b') }
86
- """
87
- if cl is None:
88
- cl = self.cl
89
- return cl.insert(record)
90
-
91
- def replace_one(self, doc, cl=None, **kwargs):
92
- """
93
- 更新数据
94
- rule: 更新数据
95
- kwargs:
96
- condition: 匹配规则
97
- hint: https://doc.sequoiadb.com/cn/SequoiaDB-cat_id-1432190835-edition_id-300
98
- flags: 默认为0 UPDATE_FLG_KEEP_SHARDINGKEY 更新规则中的分片键在执行时没有被过滤更新或插入。
99
- 如果不给flags,不保留更新规则中的分区键字段,只更新非分区键字段。否则 保留更新规则中的分区键字段。
100
- 指定 flags 参数:保留更新规则中的分区键字段。因为目前不支持更新分区键,所以会报错。
101
- """
102
- if cl is None:
103
- cl = self.cl
104
- rule = {"$replace": doc}
105
- return cl.update(rule, **kwargs)
106
-
107
- def bulk_insert(self, flag: int, records: Union[list, tuple], cl=None):
108
- """
109
- 插入多条数据
110
- :param flag:
111
- INSERT_FLG_DEFAULT : 当设置INSERT_FLG_DEFAULT时,当记录命中索引时,数据库将停止插入键重复错误。,插入正常也不会返回id
112
- INSERT_FLG_CONTONDUP : 如果记录命中索引键重复错误,数据库将跳过它
113
- INSERT_FLG_RETURN_OID : 返回记录中“_id”字段的值。,如果有重复会报错
114
- INSERT_FLG_REPLACEONDUP : 如果记录命中索引键重复错误,数据库将用插入新记录,然后继续插入。
115
- :param records:
116
- :return:
117
- 当flag不等于INSERT_FLG_RETURN_OID时,将返回一个空字典, eg: { }.
118
- eg: { '_id': [ObjectId('5d514a25c764c60acb58de38'), ObjectId('5d514a25c764c60acb58de39')]}
119
- """
120
- if cl is None:
121
- cl = self.cl
122
- return cl.bulk_insert(flag, records)
123
-
124
- def insert_with_flag(self, record, flag=INSERT_FLG_DEFAULT, cl=None):
125
- """
126
- 插入数据,并可以选择flags
127
- 和 bulk_insert 一样的标志,这样可以实现replace
128
- :param record:
129
- :param flag:
130
- :return:
131
- """
132
- if cl is None:
133
- cl = self.cl
134
- return cl.insert_with_flag(record, flag)
135
-
136
- def delete(self, cl=None, **kwargs):
137
- """
138
- 删除数据
139
- condition: 匹配规则后,删除文档,如果没有提供,删除所有文档
140
- 在 https://doc.sequoiadb.com/cn/sequoiadb-cat_id-1432190843-edition_id-300对hint
141
- hint: 指定访问计划(v3.0后意义不大)
142
- """
143
- if cl is None:
144
- cl = self.cl
145
- return cl.delete(**kwargs)
146
-
147
- def update(self, rule: dict, cl=None, **kwargs):
148
- """
149
- 更新数据
150
- rule: 更新数据
151
- kwargs:
152
- condition: 匹配规则
153
- hint: https://doc.sequoiadb.com/cn/SequoiaDB-cat_id-1432190835-edition_id-300
154
- flags: 默认为0 UPDATE_FLG_KEEP_SHARDINGKEY 更新规则中的分片键在执行时没有被过滤更新或插入。
155
- 如果不给flags,不保留更新规则中的分区键字段,只更新非分区键字段。否则 保留更新规则中的分区键字段。
156
- 指定 flags 参数:保留更新规则中的分区键字段。因为目前不支持更新分区键,所以会报错。
157
- """
158
- if cl is None:
159
- cl = self.cl
160
- return cl.update(rule, **kwargs)
161
-
162
- def upsert(self, rule, cl=None, **kwargs):
163
- """
164
- 该函数用于更新集合记录。upsert 方法跟 update 方法都是对记录进行更新,不同的是当使用 cond 参数在集合中匹配不到记录时,update 不做任何操作,而 upsert 方法会做一次插入操作
165
- https://doc.sequoiadb.com/cn/sequoiadb-cat_id-1432190848-edition_id-500
166
- :param rule: 更新规则
167
- :param kwargs: condition: 条件
168
- hint: 指定索引
169
- setOnInsert: 在做插入操作时向插入的记录中追加字段。
170
- flags:UPDATE_FLG_KEEP_SHARDINGKEY 默认不给
171
- :return:
172
- """
173
- if cl is None:
174
- cl = self.cl
175
- cl.upsert(rule, **kwargs)
176
-
177
- def query(
178
- self,
179
- condition: dict = None,
180
- selector: dict = None,
181
- order_by: dict = None,
182
- hint: dict = None,
183
- num_to_skip: int = 0,
184
- num_to_return: int = -1,
185
- flags: int = 0,
186
- cl=None):
187
- """
188
- https://doc.sequoiadb.com/cn/sequoiadb-cat_id-1552489127-edition_id-500
189
- 查询数据
190
- condition:为空时,查询所有记录;不为空时,查询符合条件记录。如:{"age":{"$gt":30}}。关于匹配条件的使用,可参考匹配符
191
- selector:为空时,返回记录的所有字段;如果指定的字段名记录中不存在,则按用户设定的内容原样返回。如:{"name":"","age":"","addr":""}。字段值为空字符串即可,数据库只关心字段名。关于选择条件的使用,可参考选择符。
192
- order_by:指定结果集按指定字段名排序的情况。字段名的值为1或者-1,如:{"name":1,"age":-1}。1代表升序;-1代表降序。 如果不设定 sort 则表示不对结果集做排序。
193
- hint:指定查询使用索引的情况。字段名可以为任意不重复的字符串,数据库只关心字段值。
194
- num_to_skip:自定义从结果集哪条记录开始返回。默认值为0,表示从第一条记录开始返回。
195
- num_to_return:自定义返回结果集的记录条数。默认值为-1,表示返回从skipNum位置开始到结果集结束位置的所有记录。
196
- flags: QUERY_FLG_WITH_RETURNDATA:强制使用指定的提示进行查询,如果数据库没有提示分配的索引,则查询失败
197
- query_flg_parallelall:启用并行子查询,每个子查询将完成扫描数据的不同部分
198
- QUERY_FLG_FORCE_HINT:一般情况下,直到游标从数据库获取数据,查询才会返回数据,当添加此标志时,在查询响应中返回数据,将会更加高性能
199
- QUERY_PREPARE_MORE:启用查询时准备更多数据
200
- QUERY_FLG_FOR_UPDATE:当事务被打开且事务隔离级别为“RC”时,事务锁将不会在事务提交或回滚之前释放。
201
- """
202
- if condition is None:
203
- condition = {}
204
- if selector is None:
205
- selector = {}
206
- if order_by is None:
207
- order_by = {}
208
- if hint is None:
209
- hint = {}
210
- if cl is None:
211
- cl = self.cl
212
- return cl.query(condition=condition,
213
- selector=selector,
214
- order_by=order_by,
215
- hint=hint,
216
- num_to_skip=num_to_skip,
217
- num_to_return=num_to_return,
218
- flags=flags)
219
- # data_list = list()
220
- # while 1:
221
- # try:
222
- # result = results.next()
223
- # data_list.append(result)
224
- # except:
225
- # break
226
- # return str(data_list)
227
-
228
- def query_and_update(self, update, cl=None, **kwargs):
229
- """
230
-
231
- :param update: 更新的规则
232
- :param kwargs: 查询条件,和上面查询一样多一个参数
233
- return_new bool 当为True时,返回更新后的文档,而不是原始文档
234
- :return: 返回查询的结果
235
- """
236
- if cl is None:
237
- cl = self.cl
238
- return cl.query_and_update(update, **kwargs)
239
-
240
- def query_and_remove(self, cl=None, **kwargs):
241
- """
242
- 查询删除
243
- 条件和查询完全一样
244
- :param kwargs:
245
- :return: 返回查询的结果
246
- """
247
- if cl is None:
248
- cl = self.cl
249
- return cl.query_and_remove(**kwargs)
250
-
251
- def save(self, doc, cl=None):
252
- """
253
- 存在更新,不存在插入
254
- :param doc:
255
- :return:
256
- """
257
- if cl is None:
258
- cl = self.cl
259
- return cl.save(doc)
260
-
261
- def create_lob(self, oid=None, cl=None):
262
- """
263
- 相当于一个空间
264
- :param oid: bson.ObjectId 指定要创建的lob的oid,如果为None,则自动生成oid
265
- :return: 一个objectid
266
- """
267
- if cl is None:
268
- cl = self.cl
269
- return cl.create_lob(oid=oid)
270
-
271
- def create_lob_id(self, timestamp=None, cl=None):
272
- """
273
- 创建lob id
274
- :param timestamp: 用于于生成lob id,如果为None则由服务器生成时间戳。
275
- 格式:YYYY-MM-DD-HH.mm.ss。如:“2019-07-23 18.04.07”
276
- :return: lob的一个ObjectId对象。
277
- """
278
- if cl is None:
279
- cl = self.cl
280
- return cl.create_lob_id(timestamp=timestamp)
281
-
282
- def open_lob(self, oid, mode=LOB_READ, cl=None):
283
- """
284
- 打开指定的lob进行读写
285
- :param oid: str/bson.ObjectId 指定的oid
286
- :param mode:
287
- int The open mode:
288
- lob.LOB_READ for reading.
289
- lob.LOB_WRITE for writing.
290
- lob.LOB_SHARE_READ for share reading.
291
- lob.LOB_SHARE_READ | lob.LOB_WRITE for both reading and writing.
292
-
293
- :return: a lob object
294
- """
295
- if cl is None:
296
- cl = self.cl
297
- return cl.open_lob(oid, mode)
298
-
299
- def get_lob(self, oid, cl=None):
300
- """
301
- 获取指定的lob
302
- :param oid: str/bson.ObjectId The specified oid
303
- :return: a lob object
304
- """
305
- if cl is None:
306
- cl = self.cl
307
- return cl.get_lob(oid)
308
-
309
- def remove_lob(self, oid, cl=None):
310
- """
311
- 删除lob
312
- :param oid: str/bson.ObjectId The oid of the lob to be remove.
313
- :return:
314
- """
315
- if cl is None:
316
- cl = self.cl
317
- cl.remove_lob(oid)
318
-
319
- def truncate_lob(self, oid, length, cl=None):
320
- """截断lob
321
-
322
- Parameters:
323
- Name Type Info:
324
- oid str/bson.ObjectId The oid of the lob to be truncated.
325
- length int/long The truncate length
326
- Exceptions:
327
- pysequoiadb.error.SDBBaseError
328
- """
329
- if cl is None:
330
- cl = self.cl
331
- cl.truncate_lob(oid, length)
332
-
333
- def list_lobs(self, cl=None, **kwargs):
334
- """
335
- lob 列表
336
- :param kwargs:
337
- Name Type Info:
338
- - condition dict The matching rule, return all the lob if not provided.
339
- - selected dict The selective rule, return the whole infomation if not provided.
340
- - order_by dict The ordered rule, result set is unordered if not provided.
341
- - hint dict Specified options. eg. {"ListPieces": 1} means get the detail piece info of lobs.
342
- - num_to_skip long Skip the first numToSkip lob, default is 0.
343
- - num_to_Return long Only return numToReturn lob, default is -1 for returning all results.
344
-
345
- :return: a cursor object of query
346
- """
347
- if cl is None:
348
- cl = self.cl
349
- return cl.list_lobs(**kwargs)
350
-
351
- def lob_insert(self, record: str, cl=None):
352
- """
353
- 插入lob数据,输入字符串
354
- """
355
- if cl is None:
356
- cl = self.cl
357
- try:
358
- lob_oid = cl.create_lob_id()
359
- lob_obj = cl.create_lob(lob_oid)
360
- lob_obj.write(record, len(record))
361
- lob_obj.close()
362
- return str(lob_oid)
363
- except Exception as e:
364
- raise e
365
-
366
- def replace_lob(self, record: str, lob_oid: str, cl=None):
367
- """
368
- 插入lob数据,输入字符串
369
- """
370
- if cl is None:
371
- cl = self.cl
372
- try:
373
- lob_obj = cl.open_lob(lob_oid, mode=LOB_WRITE)
374
- lob_obj.write(record, len(record))
375
- lob_obj.close()
376
- return str(lob_oid)
377
- except Exception as e:
378
- raise e
379
-
380
- def lob_insert_for_file(self, filepath: str):
381
- pass
382
-
383
- def lob_query(self, oid, cl=None):
384
- """
385
- 查询lob数据
386
- """
387
- if cl is None:
388
- cl = self.cl
389
- lob_file = cl.open_lob(oid)
390
- return str(lob_file.read(lob_file.get_size()), encoding='utf-8')
391
-
392
- def close(self):
393
- self.db.disconnect()
394
-
395
-
396
- def get_col(sdb_util: SequoiadbUtils, collection_space_name, collection_name):
397
- cl = sdb_util.db.get_collection("{}.{}".format(collection_space_name, collection_name))
398
- return cl
1
+ from typing import Union
2
+
3
+ from pysequoiadb import client
4
+ from pysequoiadb.collection import INSERT_FLG_DEFAULT
5
+ from pysequoiadb.lob import LOB_READ, LOB_WRITE
6
+
7
+
8
+ class SequoiadbUtils(object):
9
+
10
+ def __init__(self):
11
+ self.db_data = None
12
+ self.db = None
13
+ self.cs = None
14
+ self.cl = None
15
+ self.list_hosts = None
16
+
17
+ def set_db_data(self, db_data: dict):
18
+ self.db_data = db_data
19
+ return self
20
+
21
+ def set_list_hosts(self, list_hosts):
22
+ self.list_hosts = list_hosts
23
+ return self
24
+
25
+ def build(self):
26
+ self.link_db()
27
+ self.link_col()
28
+ return self
29
+
30
+ def set_cl(self, cl):
31
+ self.cl = cl
32
+
33
+ def link_db(self):
34
+ """
35
+ 初始化 host, port, user, password ,连接
36
+ """
37
+ host = self.db_data.get("host", "")
38
+ port = self.db_data.get("port", "")
39
+ user = self.db_data.get("user", "")
40
+ password = self.db_data.get("password", "")
41
+ try:
42
+ self.db = client(host=host, service=port, user=user, psw=password)
43
+ if self.list_hosts:
44
+ # self.list_hosts = [
45
+ # {'host': '192.168.31.21', 'service': '11810'},
46
+ # {'host': '192.168.31.22', 'service': '11810'},
47
+ # {'host': '192.168.31.23', 'service': '11810'},
48
+ # {'host': '192.168.31.24', 'service': '11810'},
49
+ # {'host': '192.168.31.25', 'service': '11810'},
50
+ # {'host': '192.168.31.26', 'service': '11810'},
51
+ # {'host': '192.168.31.27', 'service': '11810'},
52
+ # {'host': '192.168.31.219', 'service': '11810'},
53
+ # {'host': '192.168.31.208', 'service': '11810'}
54
+ # ]
55
+ self.db.connect_to_hosts(self.list_hosts, user=user, psw=password)
56
+ return self.db
57
+ except Exception as e:
58
+ raise e
59
+
60
+ def link_col(self):
61
+ collection_space_name = self.db_data.get("collection_space_name", "")
62
+ collection_name = self.db_data.get("collection_name", "")
63
+ is_create = self.db_data.get("is_create", "false")
64
+ try:
65
+ self.cl = self.db.get_collection("{}.{}".format(collection_space_name, collection_name))
66
+ return self.cl
67
+ except:
68
+ print('集合空间或者集合不存在')
69
+ if is_create == "true":
70
+ try:
71
+ self.cs = self.db.create_collection_space(collection_space_name)
72
+ print('创建集合空间 ', collection_space_name)
73
+ except:
74
+ self.cs = self.db.get_collection_space(collection_space_name)
75
+ try:
76
+ self.cl = self.cs.create_collection(collection_name)
77
+ print('创建集合空间 ', collection_name)
78
+ return self.cl
79
+ except:
80
+ self.cl = self.cs.get_collection(collection_name)
81
+
82
+ def insert(self, record: dict, cl=None):
83
+ """
84
+ 插入数据
85
+ :return A ObjectId object of record inserted. eg: { '_id': ObjectId('5d5149ade3071dce3692e93b') }
86
+ """
87
+ if cl is None:
88
+ cl = self.cl
89
+ return cl.insert(record)
90
+
91
+ def replace_one(self, doc, cl=None, **kwargs):
92
+ """
93
+ 更新数据
94
+ rule: 更新数据
95
+ kwargs:
96
+ condition: 匹配规则
97
+ hint: https://doc.sequoiadb.com/cn/SequoiaDB-cat_id-1432190835-edition_id-300
98
+ flags: 默认为0 UPDATE_FLG_KEEP_SHARDINGKEY 更新规则中的分片键在执行时没有被过滤更新或插入。
99
+ 如果不给flags,不保留更新规则中的分区键字段,只更新非分区键字段。否则 保留更新规则中的分区键字段。
100
+ 指定 flags 参数:保留更新规则中的分区键字段。因为目前不支持更新分区键,所以会报错。
101
+ """
102
+ if cl is None:
103
+ cl = self.cl
104
+ rule = {"$replace": doc}
105
+ return cl.update(rule, **kwargs)
106
+
107
+ def bulk_insert(self, flag: int, records: Union[list, tuple], cl=None):
108
+ """
109
+ 插入多条数据
110
+ :param flag:
111
+ INSERT_FLG_DEFAULT : 当设置INSERT_FLG_DEFAULT时,当记录命中索引时,数据库将停止插入键重复错误。,插入正常也不会返回id
112
+ INSERT_FLG_CONTONDUP : 如果记录命中索引键重复错误,数据库将跳过它
113
+ INSERT_FLG_RETURN_OID : 返回记录中“_id”字段的值。,如果有重复会报错
114
+ INSERT_FLG_REPLACEONDUP : 如果记录命中索引键重复错误,数据库将用插入新记录,然后继续插入。
115
+ :param records:
116
+ :return:
117
+ 当flag不等于INSERT_FLG_RETURN_OID时,将返回一个空字典, eg: { }.
118
+ eg: { '_id': [ObjectId('5d514a25c764c60acb58de38'), ObjectId('5d514a25c764c60acb58de39')]}
119
+ """
120
+ if cl is None:
121
+ cl = self.cl
122
+ return cl.bulk_insert(flag, records)
123
+
124
+ def insert_with_flag(self, record, flag=INSERT_FLG_DEFAULT, cl=None):
125
+ """
126
+ 插入数据,并可以选择flags
127
+ 和 bulk_insert 一样的标志,这样可以实现replace
128
+ :param record:
129
+ :param flag:
130
+ :return:
131
+ """
132
+ if cl is None:
133
+ cl = self.cl
134
+ return cl.insert_with_flag(record, flag)
135
+
136
+ def delete(self, cl=None, **kwargs):
137
+ """
138
+ 删除数据
139
+ condition: 匹配规则后,删除文档,如果没有提供,删除所有文档
140
+ 在 https://doc.sequoiadb.com/cn/sequoiadb-cat_id-1432190843-edition_id-300对hint
141
+ hint: 指定访问计划(v3.0后意义不大)
142
+ """
143
+ if cl is None:
144
+ cl = self.cl
145
+ return cl.delete(**kwargs)
146
+
147
+ def update(self, rule: dict, cl=None, **kwargs):
148
+ """
149
+ 更新数据
150
+ rule: 更新数据
151
+ kwargs:
152
+ condition: 匹配规则
153
+ hint: https://doc.sequoiadb.com/cn/SequoiaDB-cat_id-1432190835-edition_id-300
154
+ flags: 默认为0 UPDATE_FLG_KEEP_SHARDINGKEY 更新规则中的分片键在执行时没有被过滤更新或插入。
155
+ 如果不给flags,不保留更新规则中的分区键字段,只更新非分区键字段。否则 保留更新规则中的分区键字段。
156
+ 指定 flags 参数:保留更新规则中的分区键字段。因为目前不支持更新分区键,所以会报错。
157
+ """
158
+ if cl is None:
159
+ cl = self.cl
160
+ return cl.update(rule, **kwargs)
161
+
162
+ def upsert(self, rule, cl=None, **kwargs):
163
+ """
164
+ 该函数用于更新集合记录。upsert 方法跟 update 方法都是对记录进行更新,不同的是当使用 cond 参数在集合中匹配不到记录时,update 不做任何操作,而 upsert 方法会做一次插入操作
165
+ https://doc.sequoiadb.com/cn/sequoiadb-cat_id-1432190848-edition_id-500
166
+ :param rule: 更新规则
167
+ :param kwargs: condition: 条件
168
+ hint: 指定索引
169
+ setOnInsert: 在做插入操作时向插入的记录中追加字段。
170
+ flags:UPDATE_FLG_KEEP_SHARDINGKEY 默认不给
171
+ :return:
172
+ """
173
+ if cl is None:
174
+ cl = self.cl
175
+ cl.upsert(rule, **kwargs)
176
+
177
+ def query(
178
+ self,
179
+ condition: dict = None,
180
+ selector: dict = None,
181
+ order_by: dict = None,
182
+ hint: dict = None,
183
+ num_to_skip: int = 0,
184
+ num_to_return: int = -1,
185
+ flags: int = 0,
186
+ cl=None):
187
+ """
188
+ https://doc.sequoiadb.com/cn/sequoiadb-cat_id-1552489127-edition_id-500
189
+ 查询数据
190
+ condition:为空时,查询所有记录;不为空时,查询符合条件记录。如:{"age":{"$gt":30}}。关于匹配条件的使用,可参考匹配符
191
+ selector:为空时,返回记录的所有字段;如果指定的字段名记录中不存在,则按用户设定的内容原样返回。如:{"name":"","age":"","addr":""}。字段值为空字符串即可,数据库只关心字段名。关于选择条件的使用,可参考选择符。
192
+ order_by:指定结果集按指定字段名排序的情况。字段名的值为1或者-1,如:{"name":1,"age":-1}。1代表升序;-1代表降序。 如果不设定 sort 则表示不对结果集做排序。
193
+ hint:指定查询使用索引的情况。字段名可以为任意不重复的字符串,数据库只关心字段值。
194
+ num_to_skip:自定义从结果集哪条记录开始返回。默认值为0,表示从第一条记录开始返回。
195
+ num_to_return:自定义返回结果集的记录条数。默认值为-1,表示返回从skipNum位置开始到结果集结束位置的所有记录。
196
+ flags: QUERY_FLG_WITH_RETURNDATA:强制使用指定的提示进行查询,如果数据库没有提示分配的索引,则查询失败
197
+ query_flg_parallelall:启用并行子查询,每个子查询将完成扫描数据的不同部分
198
+ QUERY_FLG_FORCE_HINT:一般情况下,直到游标从数据库获取数据,查询才会返回数据,当添加此标志时,在查询响应中返回数据,将会更加高性能
199
+ QUERY_PREPARE_MORE:启用查询时准备更多数据
200
+ QUERY_FLG_FOR_UPDATE:当事务被打开且事务隔离级别为“RC”时,事务锁将不会在事务提交或回滚之前释放。
201
+ """
202
+ if condition is None:
203
+ condition = {}
204
+ if selector is None:
205
+ selector = {}
206
+ if order_by is None:
207
+ order_by = {}
208
+ if hint is None:
209
+ hint = {}
210
+ if cl is None:
211
+ cl = self.cl
212
+ return cl.query(condition=condition,
213
+ selector=selector,
214
+ order_by=order_by,
215
+ hint=hint,
216
+ num_to_skip=num_to_skip,
217
+ num_to_return=num_to_return,
218
+ flags=flags)
219
+ # data_list = list()
220
+ # while 1:
221
+ # try:
222
+ # result = results.next()
223
+ # data_list.append(result)
224
+ # except:
225
+ # break
226
+ # return str(data_list)
227
+
228
+ def query_and_update(self, update, cl=None, **kwargs):
229
+ """
230
+
231
+ :param update: 更新的规则
232
+ :param kwargs: 查询条件,和上面查询一样多一个参数
233
+ return_new bool 当为True时,返回更新后的文档,而不是原始文档
234
+ :return: 返回查询的结果
235
+ """
236
+ if cl is None:
237
+ cl = self.cl
238
+ return cl.query_and_update(update, **kwargs)
239
+
240
+ def query_and_remove(self, cl=None, **kwargs):
241
+ """
242
+ 查询删除
243
+ 条件和查询完全一样
244
+ :param kwargs:
245
+ :return: 返回查询的结果
246
+ """
247
+ if cl is None:
248
+ cl = self.cl
249
+ return cl.query_and_remove(**kwargs)
250
+
251
+ def save(self, doc, cl=None):
252
+ """
253
+ 存在更新,不存在插入
254
+ :param doc:
255
+ :return:
256
+ """
257
+ if cl is None:
258
+ cl = self.cl
259
+ return cl.save(doc)
260
+
261
+ def create_lob(self, oid=None, cl=None):
262
+ """
263
+ 相当于一个空间
264
+ :param oid: bson.ObjectId 指定要创建的lob的oid,如果为None,则自动生成oid
265
+ :return: 一个objectid
266
+ """
267
+ if cl is None:
268
+ cl = self.cl
269
+ return cl.create_lob(oid=oid)
270
+
271
+ def create_lob_id(self, timestamp=None, cl=None):
272
+ """
273
+ 创建lob id
274
+ :param timestamp: 用于于生成lob id,如果为None则由服务器生成时间戳。
275
+ 格式:YYYY-MM-DD-HH.mm.ss。如:“2019-07-23 18.04.07”
276
+ :return: lob的一个ObjectId对象。
277
+ """
278
+ if cl is None:
279
+ cl = self.cl
280
+ return cl.create_lob_id(timestamp=timestamp)
281
+
282
+ def open_lob(self, oid, mode=LOB_READ, cl=None):
283
+ """
284
+ 打开指定的lob进行读写
285
+ :param oid: str/bson.ObjectId 指定的oid
286
+ :param mode:
287
+ int The open mode:
288
+ lob.LOB_READ for reading.
289
+ lob.LOB_WRITE for writing.
290
+ lob.LOB_SHARE_READ for share reading.
291
+ lob.LOB_SHARE_READ | lob.LOB_WRITE for both reading and writing.
292
+
293
+ :return: a lob object
294
+ """
295
+ if cl is None:
296
+ cl = self.cl
297
+ return cl.open_lob(oid, mode)
298
+
299
+ def get_lob(self, oid, cl=None):
300
+ """
301
+ 获取指定的lob
302
+ :param oid: str/bson.ObjectId The specified oid
303
+ :return: a lob object
304
+ """
305
+ if cl is None:
306
+ cl = self.cl
307
+ return cl.get_lob(oid)
308
+
309
+ def remove_lob(self, oid, cl=None):
310
+ """
311
+ 删除lob
312
+ :param oid: str/bson.ObjectId The oid of the lob to be remove.
313
+ :return:
314
+ """
315
+ if cl is None:
316
+ cl = self.cl
317
+ cl.remove_lob(oid)
318
+
319
+ def truncate_lob(self, oid, length, cl=None):
320
+ """截断lob
321
+
322
+ Parameters:
323
+ Name Type Info:
324
+ oid str/bson.ObjectId The oid of the lob to be truncated.
325
+ length int/long The truncate length
326
+ Exceptions:
327
+ pysequoiadb.error.SDBBaseError
328
+ """
329
+ if cl is None:
330
+ cl = self.cl
331
+ cl.truncate_lob(oid, length)
332
+
333
+ def list_lobs(self, cl=None, **kwargs):
334
+ """
335
+ lob 列表
336
+ :param kwargs:
337
+ Name Type Info:
338
+ - condition dict The matching rule, return all the lob if not provided.
339
+ - selected dict The selective rule, return the whole infomation if not provided.
340
+ - order_by dict The ordered rule, result set is unordered if not provided.
341
+ - hint dict Specified options. eg. {"ListPieces": 1} means get the detail piece info of lobs.
342
+ - num_to_skip long Skip the first numToSkip lob, default is 0.
343
+ - num_to_Return long Only return numToReturn lob, default is -1 for returning all results.
344
+
345
+ :return: a cursor object of query
346
+ """
347
+ if cl is None:
348
+ cl = self.cl
349
+ return cl.list_lobs(**kwargs)
350
+
351
+ def lob_insert(self, record: str, cl=None):
352
+ """
353
+ 插入lob数据,输入字符串
354
+ """
355
+ if cl is None:
356
+ cl = self.cl
357
+ try:
358
+ lob_oid = cl.create_lob_id()
359
+ lob_obj = cl.create_lob(lob_oid)
360
+ lob_obj.write(record, len(record))
361
+ lob_obj.close()
362
+ return str(lob_oid)
363
+ except Exception as e:
364
+ raise e
365
+
366
+ def replace_lob(self, record: str, lob_oid: str, cl=None):
367
+ """
368
+ 插入lob数据,输入字符串
369
+ """
370
+ if cl is None:
371
+ cl = self.cl
372
+ try:
373
+ lob_obj = cl.open_lob(lob_oid, mode=LOB_WRITE)
374
+ lob_obj.write(record, len(record))
375
+ lob_obj.close()
376
+ return str(lob_oid)
377
+ except Exception as e:
378
+ raise e
379
+
380
+ def lob_insert_for_file(self, filepath: str):
381
+ pass
382
+
383
+ def lob_query(self, oid, cl=None):
384
+ """
385
+ 查询lob数据
386
+ """
387
+ if cl is None:
388
+ cl = self.cl
389
+ lob_file = cl.open_lob(oid)
390
+ return str(lob_file.read(lob_file.get_size()), encoding='utf-8')
391
+
392
+ def close(self):
393
+ self.db.disconnect()
394
+
395
+
396
+ def get_col(sdb_util: SequoiadbUtils, collection_space_name, collection_name):
397
+ cl = sdb_util.db.get_collection("{}.{}".format(collection_space_name, collection_name))
398
+ return cl