re-common 10.0.39__py3-none-any.whl → 10.0.40__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- re_common/baselibrary/__init__.py +4 -4
- re_common/baselibrary/baseabs/__init__.py +6 -6
- re_common/baselibrary/baseabs/baseabs.py +26 -26
- re_common/baselibrary/database/mbuilder.py +132 -132
- re_common/baselibrary/database/moudle.py +93 -93
- re_common/baselibrary/database/msqlite3.py +194 -194
- re_common/baselibrary/database/mysql.py +169 -169
- re_common/baselibrary/database/sql_factory.py +26 -26
- re_common/baselibrary/mthread/MThreadingRun.py +486 -486
- re_common/baselibrary/mthread/MThreadingRunEvent.py +349 -349
- re_common/baselibrary/mthread/__init__.py +2 -2
- re_common/baselibrary/mthread/mythreading.py +695 -695
- re_common/baselibrary/pakge_other/socks.py +404 -404
- re_common/baselibrary/readconfig/config_factory.py +18 -18
- re_common/baselibrary/readconfig/ini_config.py +317 -317
- re_common/baselibrary/readconfig/toml_config.py +49 -49
- re_common/baselibrary/temporary/envdata.py +36 -36
- re_common/baselibrary/tools/all_requests/aiohttp_request.py +118 -118
- re_common/baselibrary/tools/all_requests/httpx_requet.py +102 -102
- re_common/baselibrary/tools/all_requests/mrequest.py +412 -412
- re_common/baselibrary/tools/all_requests/requests_request.py +81 -81
- re_common/baselibrary/tools/batch_compre/bijiao_batch.py +31 -31
- re_common/baselibrary/tools/contrast_db3.py +123 -123
- re_common/baselibrary/tools/copy_file.py +39 -39
- re_common/baselibrary/tools/db3_2_sizedb3.py +102 -102
- re_common/baselibrary/tools/foreachgz.py +39 -39
- re_common/baselibrary/tools/get_attr.py +10 -10
- re_common/baselibrary/tools/image_to_pdf.py +61 -61
- re_common/baselibrary/tools/java_code_deal.py +139 -139
- re_common/baselibrary/tools/javacode.py +79 -79
- re_common/baselibrary/tools/mdb_db3.py +48 -48
- re_common/baselibrary/tools/merge_file.py +171 -171
- re_common/baselibrary/tools/merge_gz_file.py +165 -165
- re_common/baselibrary/tools/mhdfstools/down_hdfs_files.py +42 -42
- re_common/baselibrary/tools/mhdfstools/hdfst.py +42 -42
- re_common/baselibrary/tools/mhdfstools/up_hdfs_files.py +38 -38
- re_common/baselibrary/tools/mongo_tools.py +50 -50
- re_common/baselibrary/tools/move_file.py +170 -170
- re_common/baselibrary/tools/move_mongo/mongo_table_to_file.py +63 -63
- re_common/baselibrary/tools/move_mongo/move_mongo_table.py +354 -354
- re_common/baselibrary/tools/move_mongo/use_mttf.py +18 -18
- re_common/baselibrary/tools/move_mongo/use_mv.py +93 -93
- re_common/baselibrary/tools/mpandas/mpandasreadexcel.py +125 -125
- re_common/baselibrary/tools/mpandas/pandas_visualization.py +7 -7
- re_common/baselibrary/tools/myparsel.py +104 -104
- re_common/baselibrary/tools/rename_dir_file.py +37 -37
- re_common/baselibrary/tools/sequoiadb_utils.py +398 -398
- re_common/baselibrary/tools/split_line_to_many.py +25 -25
- re_common/baselibrary/tools/stringtodicts.py +33 -33
- re_common/baselibrary/tools/workwechant_bot.py +84 -84
- re_common/baselibrary/utils/baseaiohttp.py +296 -296
- re_common/baselibrary/utils/baseaiomysql.py +87 -87
- re_common/baselibrary/utils/baseallstep.py +191 -191
- re_common/baselibrary/utils/baseavro.py +19 -19
- re_common/baselibrary/utils/baseboto3.py +291 -291
- re_common/baselibrary/utils/basecsv.py +32 -32
- re_common/baselibrary/utils/basedict.py +133 -133
- re_common/baselibrary/utils/basedir.py +241 -241
- re_common/baselibrary/utils/baseencode.py +351 -351
- re_common/baselibrary/utils/baseencoding.py +28 -28
- re_common/baselibrary/utils/baseesdsl.py +86 -86
- re_common/baselibrary/utils/baseexcel.py +264 -264
- re_common/baselibrary/utils/baseexcept.py +109 -109
- re_common/baselibrary/utils/basefile.py +654 -654
- re_common/baselibrary/utils/baseftp.py +214 -214
- re_common/baselibrary/utils/basegzip.py +60 -60
- re_common/baselibrary/utils/basehdfs.py +135 -135
- re_common/baselibrary/utils/basehttpx.py +268 -268
- re_common/baselibrary/utils/baseip.py +87 -87
- re_common/baselibrary/utils/basejson.py +2 -2
- re_common/baselibrary/utils/baselist.py +32 -32
- re_common/baselibrary/utils/basemotor.py +190 -190
- re_common/baselibrary/utils/basemssql.py +98 -98
- re_common/baselibrary/utils/baseodbc.py +113 -113
- re_common/baselibrary/utils/basepandas.py +302 -302
- re_common/baselibrary/utils/basepeewee.py +11 -11
- re_common/baselibrary/utils/basepika.py +180 -180
- re_common/baselibrary/utils/basepydash.py +143 -143
- re_common/baselibrary/utils/basepymongo.py +230 -230
- re_common/baselibrary/utils/basequeue.py +22 -22
- re_common/baselibrary/utils/baserar.py +57 -57
- re_common/baselibrary/utils/baserequest.py +279 -279
- re_common/baselibrary/utils/baseset.py +8 -8
- re_common/baselibrary/utils/basesmb.py +403 -403
- re_common/baselibrary/utils/basestring.py +382 -382
- re_common/baselibrary/utils/basetime.py +320 -320
- re_common/baselibrary/utils/baseurl.py +121 -121
- re_common/baselibrary/utils/basezip.py +57 -57
- re_common/baselibrary/utils/core/__init__.py +7 -7
- re_common/baselibrary/utils/core/bottomutils.py +18 -18
- re_common/baselibrary/utils/core/mdeprecated.py +327 -327
- re_common/baselibrary/utils/core/mlamada.py +16 -16
- re_common/baselibrary/utils/core/msginfo.py +25 -25
- re_common/baselibrary/utils/core/requests_core.py +103 -103
- re_common/baselibrary/utils/fateadm.py +429 -429
- re_common/baselibrary/utils/importfun.py +123 -123
- re_common/baselibrary/utils/mfaker.py +57 -57
- re_common/baselibrary/utils/my_abc/__init__.py +3 -3
- re_common/baselibrary/utils/my_abc/better_abc.py +32 -32
- re_common/baselibrary/utils/mylogger.py +414 -414
- re_common/baselibrary/utils/myredisclient.py +861 -861
- re_common/baselibrary/utils/pipupgrade.py +21 -21
- re_common/baselibrary/utils/ringlist.py +85 -85
- re_common/baselibrary/utils/version_compare.py +36 -36
- re_common/baselibrary/utils/ydmhttp.py +126 -126
- re_common/facade/lazy_import.py +11 -11
- re_common/facade/loggerfacade.py +25 -25
- re_common/facade/mysqlfacade.py +467 -467
- re_common/facade/now.py +31 -31
- re_common/facade/sqlite3facade.py +257 -257
- re_common/facade/use/mq_use_facade.py +83 -83
- re_common/facade/use/proxy_use_facade.py +19 -19
- re_common/libtest/base_dict_test.py +19 -19
- re_common/libtest/baseavro_test.py +13 -13
- re_common/libtest/basefile_test.py +14 -14
- re_common/libtest/basemssql_test.py +77 -77
- re_common/libtest/baseodbc_test.py +7 -7
- re_common/libtest/basepandas_test.py +38 -38
- re_common/libtest/get_attr_test/get_attr_test_settings.py +14 -14
- re_common/libtest/get_attr_test/settings.py +54 -54
- re_common/libtest/idencode_test.py +53 -53
- re_common/libtest/iniconfig_test.py +35 -35
- re_common/libtest/ip_test.py +34 -34
- re_common/libtest/merge_file_test.py +20 -20
- re_common/libtest/mfaker_test.py +8 -8
- re_common/libtest/mm3_test.py +31 -31
- re_common/libtest/mylogger_test.py +88 -88
- re_common/libtest/myparsel_test.py +27 -27
- re_common/libtest/mysql_test.py +151 -151
- re_common/libtest/pymongo_test.py +21 -21
- re_common/libtest/split_test.py +11 -11
- re_common/libtest/sqlite3_merge_test.py +5 -5
- re_common/libtest/sqlite3_test.py +34 -34
- re_common/libtest/tomlconfig_test.py +30 -30
- re_common/libtest/use_tools_test/__init__.py +2 -2
- re_common/libtest/user/__init__.py +4 -4
- re_common/studio/__init__.py +4 -4
- re_common/studio/assignment_expressions.py +36 -36
- re_common/studio/mydash/test1.py +18 -18
- re_common/studio/pydashstudio/first.py +9 -9
- re_common/studio/streamlitstudio/first_app.py +65 -65
- re_common/studio/streamlitstudio/uber_pickups.py +23 -23
- re_common/studio/test.py +18 -18
- re_common/v2/baselibrary/business_utils/BusinessStringUtil.py +219 -219
- re_common/v2/baselibrary/business_utils/baseencodeid.py +100 -100
- re_common/v2/baselibrary/business_utils/full_doi_path.py +116 -116
- re_common/v2/baselibrary/business_utils/rel_tools.py +6 -6
- re_common/v2/baselibrary/decorators/utils.py +59 -59
- re_common/v2/baselibrary/helpers/search_packge/NearestNeighbors_test.py +105 -105
- re_common/v2/baselibrary/helpers/search_packge/fit_text_match.py +253 -253
- re_common/v2/baselibrary/helpers/search_packge/scikit_learn_text_matcher.py +260 -260
- re_common/v2/baselibrary/helpers/search_packge/test.py +1 -1
- re_common/v2/baselibrary/s3object/baseboto3.py +230 -230
- re_common/v2/baselibrary/tools/WeChatRobot.py +95 -95
- re_common/v2/baselibrary/tools/ac_ahocorasick.py +75 -75
- re_common/v2/baselibrary/tools/concurrency.py +35 -35
- re_common/v2/baselibrary/tools/data_processer/base.py +53 -53
- re_common/v2/baselibrary/tools/data_processer/data_processer.py +508 -508
- re_common/v2/baselibrary/tools/data_processer/data_reader.py +187 -187
- re_common/v2/baselibrary/tools/data_processer/data_writer.py +38 -38
- re_common/v2/baselibrary/tools/dict_tools.py +44 -44
- re_common/v2/baselibrary/tools/dolphinscheduler.py +187 -187
- re_common/v2/baselibrary/tools/hdfs_base_processor.py +204 -204
- re_common/v2/baselibrary/tools/hdfs_bulk_processor.py +67 -67
- re_common/v2/baselibrary/tools/hdfs_data_processer.py +338 -338
- re_common/v2/baselibrary/tools/hdfs_line_processor.py +74 -74
- re_common/v2/baselibrary/tools/list_tools.py +69 -69
- re_common/v2/baselibrary/tools/resume_tracker.py +94 -94
- re_common/v2/baselibrary/tools/search_hash_tools.py +54 -54
- re_common/v2/baselibrary/tools/text_matcher.py +326 -326
- re_common/v2/baselibrary/tools/unionfind_tools.py +60 -60
- re_common/v2/baselibrary/utils/BusinessStringUtil.py +196 -196
- re_common/v2/baselibrary/utils/api_net_utils.py +270 -270
- re_common/v2/baselibrary/utils/author_smi.py +361 -361
- re_common/v2/baselibrary/utils/base_string_similarity.py +158 -158
- re_common/v2/baselibrary/utils/basedict.py +37 -37
- re_common/v2/baselibrary/utils/basehdfs.py +163 -163
- re_common/v2/baselibrary/utils/basepika.py +180 -180
- re_common/v2/baselibrary/utils/basetime.py +77 -77
- re_common/v2/baselibrary/utils/db.py +156 -156
- re_common/v2/baselibrary/utils/elasticsearch.py +46 -0
- re_common/v2/baselibrary/utils/json_cls.py +16 -16
- re_common/v2/baselibrary/utils/mq.py +83 -83
- re_common/v2/baselibrary/utils/n_ary_expression_tree.py +243 -243
- re_common/v2/baselibrary/utils/string_bool.py +186 -186
- re_common/v2/baselibrary/utils/string_clear.py +246 -246
- re_common/v2/baselibrary/utils/string_smi.py +18 -18
- re_common/v2/baselibrary/utils/stringutils.py +271 -271
- re_common/vip/base_step_process.py +11 -11
- re_common/vip/baseencodeid.py +90 -90
- re_common/vip/changetaskname.py +28 -28
- re_common/vip/core_var.py +24 -24
- re_common/vip/mmh3Hash.py +89 -89
- re_common/vip/proxy/allproxys.py +127 -127
- re_common/vip/proxy/allproxys_thread.py +159 -159
- re_common/vip/proxy/cnki_proxy.py +153 -153
- re_common/vip/proxy/kuaidaili.py +87 -87
- re_common/vip/proxy/proxy_all.py +113 -113
- re_common/vip/proxy/update_kuaidaili_0.py +42 -42
- re_common/vip/proxy/wanfang_proxy.py +152 -152
- re_common/vip/proxy/wp_proxy_all.py +181 -181
- re_common/vip/read_rawid_to_txt.py +91 -91
- re_common/vip/title/__init__.py +5 -5
- re_common/vip/title/transform/TransformBookTitleToZt.py +125 -125
- re_common/vip/title/transform/TransformConferenceTitleToZt.py +139 -139
- re_common/vip/title/transform/TransformCstadTitleToZt.py +195 -195
- re_common/vip/title/transform/TransformJournalTitleToZt.py +203 -203
- re_common/vip/title/transform/TransformPatentTitleToZt.py +132 -132
- re_common/vip/title/transform/TransformRegulationTitleToZt.py +114 -114
- re_common/vip/title/transform/TransformStandardTitleToZt.py +135 -135
- re_common/vip/title/transform/TransformThesisTitleToZt.py +135 -135
- re_common/vip/title/transform/__init__.py +10 -10
- {re_common-10.0.39.dist-info → re_common-10.0.40.dist-info}/LICENSE +201 -201
- {re_common-10.0.39.dist-info → re_common-10.0.40.dist-info}/METADATA +24 -16
- re_common-10.0.40.dist-info/RECORD +249 -0
- {re_common-10.0.39.dist-info → re_common-10.0.40.dist-info}/WHEEL +1 -1
- re_common-10.0.39.dist-info/RECORD +0 -248
- {re_common-10.0.39.dist-info → re_common-10.0.40.dist-info}/top_level.txt +0 -0
|
@@ -1,28 +1,28 @@
|
|
|
1
|
-
from re_common.baselibrary.tools.myparsel import MParsel
|
|
2
|
-
|
|
3
|
-
if __name__ == '__main__':
|
|
4
|
-
htmlText = r'''
|
|
5
|
-
<body>
|
|
6
|
-
<div class="book">111</div>
|
|
7
|
-
<div class="journal_name">222</div>
|
|
8
|
-
<div class="title">333</div>
|
|
9
|
-
<a class ="link">link1</a>
|
|
10
|
-
<a class ="link">link2</a>
|
|
11
|
-
<a class ="link">link3</a>
|
|
12
|
-
</body>
|
|
13
|
-
'''
|
|
14
|
-
mc = MParsel()
|
|
15
|
-
css_selector = {
|
|
16
|
-
'book':'div[class="book"]::text',
|
|
17
|
-
'journal_name':'div[class="journal_name"]::text',
|
|
18
|
-
'title':'div[class="title"]::text',
|
|
19
|
-
'link' :'a[class*="link"]::text'
|
|
20
|
-
|
|
21
|
-
}
|
|
22
|
-
xpath_selector = {
|
|
23
|
-
'book': "//div[@class='book']/text()",
|
|
24
|
-
}
|
|
25
|
-
bools,new_dict = mc.css_parsel_html(htmlText,css_selector=css_selector)
|
|
26
|
-
print(new_dict)
|
|
27
|
-
bools, new_dict = mc.xpath_parsel_html(htmlText, xpath_selector=xpath_selector)
|
|
1
|
+
from re_common.baselibrary.tools.myparsel import MParsel
|
|
2
|
+
|
|
3
|
+
if __name__ == '__main__':
|
|
4
|
+
htmlText = r'''
|
|
5
|
+
<body>
|
|
6
|
+
<div class="book">111</div>
|
|
7
|
+
<div class="journal_name">222</div>
|
|
8
|
+
<div class="title">333</div>
|
|
9
|
+
<a class ="link">link1</a>
|
|
10
|
+
<a class ="link">link2</a>
|
|
11
|
+
<a class ="link">link3</a>
|
|
12
|
+
</body>
|
|
13
|
+
'''
|
|
14
|
+
mc = MParsel()
|
|
15
|
+
css_selector = {
|
|
16
|
+
'book':'div[class="book"]::text',
|
|
17
|
+
'journal_name':'div[class="journal_name"]::text',
|
|
18
|
+
'title':'div[class="title"]::text',
|
|
19
|
+
'link' :'a[class*="link"]::text'
|
|
20
|
+
|
|
21
|
+
}
|
|
22
|
+
xpath_selector = {
|
|
23
|
+
'book': "//div[@class='book']/text()",
|
|
24
|
+
}
|
|
25
|
+
bools,new_dict = mc.css_parsel_html(htmlText,css_selector=css_selector)
|
|
26
|
+
print(new_dict)
|
|
27
|
+
bools, new_dict = mc.xpath_parsel_html(htmlText, xpath_selector=xpath_selector)
|
|
28
28
|
print(new_dict)
|
re_common/libtest/mysql_test.py
CHANGED
|
@@ -1,151 +1,151 @@
|
|
|
1
|
-
from re_common.baselibrary.utils.basefile import BaseFile
|
|
2
|
-
|
|
3
|
-
from re_common.facade.mysqlfacade import MysqlUtiles
|
|
4
|
-
|
|
5
|
-
from re_common.baselibrary.utils.basedir import BaseDir
|
|
6
|
-
from re_common.facade.loggerfacade import get_streamlogger
|
|
7
|
-
|
|
8
|
-
curPath = BaseDir.get_file_dir_absolute(__file__)
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
def test_mysql_create():
|
|
12
|
-
"""
|
|
13
|
-
创建表 pytest mysql_test.py::test_mysql_create -s
|
|
14
|
-
:return:
|
|
15
|
-
"""
|
|
16
|
-
mysqls = MysqlUtiles(BaseFile.get_new_path(curPath, "mysql.ini"), "db", get_streamlogger())
|
|
17
|
-
sql = """
|
|
18
|
-
CREATE TABLE `test1` (
|
|
19
|
-
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT '自增id',
|
|
20
|
-
`username` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '用户名',
|
|
21
|
-
`passwd` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '密码',
|
|
22
|
-
`only` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '唯一性约束',
|
|
23
|
-
PRIMARY KEY (`id`),
|
|
24
|
-
UNIQUE KEY `only` (`only`)
|
|
25
|
-
) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
|
26
|
-
"""
|
|
27
|
-
bool, msg = mysqls.ExeSqlToDB(sql)
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
def test_mysql_insert():
|
|
31
|
-
"""
|
|
32
|
-
插入表 更新删除都可以使用ExeSqlToDB这个函数
|
|
33
|
-
pytest mysql_test.py::test_mysql_insert -s
|
|
34
|
-
:return:
|
|
35
|
-
"""
|
|
36
|
-
mysqls = MysqlUtiles(BaseFile.get_new_path(curPath, "mysql.ini"), "db", get_streamlogger())
|
|
37
|
-
sql = "INSERT INTO mysqlltest.test1 (username, passwd, `only`) VALUES('test1', 'test1', 'test1');"
|
|
38
|
-
bool, msg = mysqls.ExeSqlToDB(sql, errExit=False)
|
|
39
|
-
print(bool)
|
|
40
|
-
print(msg)
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
def test_mysql_insert_list():
|
|
44
|
-
"""
|
|
45
|
-
插入表 pytest mysql_test.py::test_mysql_insert_list -s
|
|
46
|
-
:return:
|
|
47
|
-
"""
|
|
48
|
-
listsql = []
|
|
49
|
-
mysqls = MysqlUtiles(BaseFile.get_new_path(curPath, "mysql.ini"), "db", get_streamlogger())
|
|
50
|
-
sql = "INSERT INTO mysqlltest.test1 (username, passwd, `only`) VALUES('%s', '%s', '%s');"
|
|
51
|
-
for i in range(100):
|
|
52
|
-
temp = sql % (i, i, i)
|
|
53
|
-
listsql.append(temp)
|
|
54
|
-
bool, successcount, errcount = mysqls.ExeSqlListToDB(listsql, errExit=False)
|
|
55
|
-
print(bool)
|
|
56
|
-
print(successcount)
|
|
57
|
-
print(errcount)
|
|
58
|
-
bool, successcount, errcount = mysqls.ExeSqlListToDB(listsql, errExit=True)
|
|
59
|
-
print(bool)
|
|
60
|
-
print(successcount)
|
|
61
|
-
print(errcount)
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
def test_mysql_select():
|
|
65
|
-
"""
|
|
66
|
-
查询数据 pytest mysql_test.py::test_mysql_select -s
|
|
67
|
-
:return:
|
|
68
|
-
"""
|
|
69
|
-
listsql = []
|
|
70
|
-
mysqls = MysqlUtiles(BaseFile.get_new_path(curPath, "mysql.ini"), "db", get_streamlogger())
|
|
71
|
-
sql = "SELECT * FROM test1"
|
|
72
|
-
bool, rows = mysqls.SelectFromDB(sql)
|
|
73
|
-
print(bool)
|
|
74
|
-
print(rows)
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
def test_mysql_select_dicts():
|
|
78
|
-
"""
|
|
79
|
-
查询数据 返回dicts 形式的数据
|
|
80
|
-
pytest mysql_test.py::test_mysql_select_dicts -s
|
|
81
|
-
:return:
|
|
82
|
-
"""
|
|
83
|
-
listsql = []
|
|
84
|
-
mysqls = MysqlUtiles(BaseFile.get_new_path(curPath, "mysql.ini"), "db", get_streamlogger(), cursorsnum=1)
|
|
85
|
-
sql = "SELECT * FROM test1"
|
|
86
|
-
bool, rows = mysqls.SelectFromDB(sql)
|
|
87
|
-
print(bool)
|
|
88
|
-
print(rows)
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
def test_mysql_select_one():
|
|
92
|
-
"""
|
|
93
|
-
查询数据只返回一条数据
|
|
94
|
-
pytest mysql_test.py::test_mysql_select_one -s
|
|
95
|
-
:return:
|
|
96
|
-
"""
|
|
97
|
-
listsql = []
|
|
98
|
-
mysqls = MysqlUtiles(BaseFile.get_new_path(curPath, "mysql.ini"), "db", get_streamlogger())
|
|
99
|
-
sql = "SELECT * FROM test1"
|
|
100
|
-
bool, rows = mysqls.SelectFromDBFetchOne_noyield(sql)
|
|
101
|
-
print(bool)
|
|
102
|
-
print(rows)
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
def test_mysql_select_one_yeild():
|
|
106
|
-
"""
|
|
107
|
-
查询数据 以yeild循环返回
|
|
108
|
-
pytest mysql_test.py::test_mysql_select_one_yeild -s
|
|
109
|
-
:return:
|
|
110
|
-
"""
|
|
111
|
-
listsql = []
|
|
112
|
-
mysqls = MysqlUtiles(BaseFile.get_new_path(curPath, "mysql.ini"), "db", get_streamlogger())
|
|
113
|
-
sql = "SELECT * FROM test1"
|
|
114
|
-
for one in mysqls.SelectFromDBFetchOne(sql):
|
|
115
|
-
print(one)
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
def test_mysql_exesqlmany():
|
|
119
|
-
"""
|
|
120
|
-
同时插入多条数据
|
|
121
|
-
pytest mysql_test.py::test_mysql_exesqlmany -s
|
|
122
|
-
:return:
|
|
123
|
-
"""
|
|
124
|
-
listpara = []
|
|
125
|
-
mysqls = MysqlUtiles(BaseFile.get_new_path(curPath, "mysql.ini"), "db", get_streamlogger())
|
|
126
|
-
sql = "INSERT INTO mysqlltest.test1 (username, passwd, `only`) VALUES(%s, %s, %s);"
|
|
127
|
-
for i in range(101, 201):
|
|
128
|
-
listpara.append((i, i, i))
|
|
129
|
-
bool, rows = mysqls.ExeSqlMany(sql, listpara)
|
|
130
|
-
print(bool)
|
|
131
|
-
print(rows)
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
test_mysql_exesqlmany()
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
def test_dict_mysql():
|
|
138
|
-
strings = """
|
|
139
|
-
host = 192.168.31.209
|
|
140
|
-
user = root
|
|
141
|
-
passwd = vipdatacenter
|
|
142
|
-
db = data_gather_record
|
|
143
|
-
port = 3306
|
|
144
|
-
chartset = utf8
|
|
145
|
-
"""
|
|
146
|
-
|
|
147
|
-
dicts_change = {"key为原来的": "values为现在的"}
|
|
148
|
-
|
|
149
|
-
from re_common.baselibrary.tools.stringtodicts import StringToDicts
|
|
150
|
-
dicts = StringToDicts().string_to_dicts_by_equal(strings)
|
|
151
|
-
mysqlutils = MysqlUtiles("", "", builder="MysqlBuilderForDicts", dicts=dicts)
|
|
1
|
+
from re_common.baselibrary.utils.basefile import BaseFile
|
|
2
|
+
|
|
3
|
+
from re_common.facade.mysqlfacade import MysqlUtiles
|
|
4
|
+
|
|
5
|
+
from re_common.baselibrary.utils.basedir import BaseDir
|
|
6
|
+
from re_common.facade.loggerfacade import get_streamlogger
|
|
7
|
+
|
|
8
|
+
curPath = BaseDir.get_file_dir_absolute(__file__)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def test_mysql_create():
|
|
12
|
+
"""
|
|
13
|
+
创建表 pytest mysql_test.py::test_mysql_create -s
|
|
14
|
+
:return:
|
|
15
|
+
"""
|
|
16
|
+
mysqls = MysqlUtiles(BaseFile.get_new_path(curPath, "mysql.ini"), "db", get_streamlogger())
|
|
17
|
+
sql = """
|
|
18
|
+
CREATE TABLE `test1` (
|
|
19
|
+
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT '自增id',
|
|
20
|
+
`username` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '用户名',
|
|
21
|
+
`passwd` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '密码',
|
|
22
|
+
`only` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '唯一性约束',
|
|
23
|
+
PRIMARY KEY (`id`),
|
|
24
|
+
UNIQUE KEY `only` (`only`)
|
|
25
|
+
) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
|
26
|
+
"""
|
|
27
|
+
bool, msg = mysqls.ExeSqlToDB(sql)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_mysql_insert():
|
|
31
|
+
"""
|
|
32
|
+
插入表 更新删除都可以使用ExeSqlToDB这个函数
|
|
33
|
+
pytest mysql_test.py::test_mysql_insert -s
|
|
34
|
+
:return:
|
|
35
|
+
"""
|
|
36
|
+
mysqls = MysqlUtiles(BaseFile.get_new_path(curPath, "mysql.ini"), "db", get_streamlogger())
|
|
37
|
+
sql = "INSERT INTO mysqlltest.test1 (username, passwd, `only`) VALUES('test1', 'test1', 'test1');"
|
|
38
|
+
bool, msg = mysqls.ExeSqlToDB(sql, errExit=False)
|
|
39
|
+
print(bool)
|
|
40
|
+
print(msg)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def test_mysql_insert_list():
|
|
44
|
+
"""
|
|
45
|
+
插入表 pytest mysql_test.py::test_mysql_insert_list -s
|
|
46
|
+
:return:
|
|
47
|
+
"""
|
|
48
|
+
listsql = []
|
|
49
|
+
mysqls = MysqlUtiles(BaseFile.get_new_path(curPath, "mysql.ini"), "db", get_streamlogger())
|
|
50
|
+
sql = "INSERT INTO mysqlltest.test1 (username, passwd, `only`) VALUES('%s', '%s', '%s');"
|
|
51
|
+
for i in range(100):
|
|
52
|
+
temp = sql % (i, i, i)
|
|
53
|
+
listsql.append(temp)
|
|
54
|
+
bool, successcount, errcount = mysqls.ExeSqlListToDB(listsql, errExit=False)
|
|
55
|
+
print(bool)
|
|
56
|
+
print(successcount)
|
|
57
|
+
print(errcount)
|
|
58
|
+
bool, successcount, errcount = mysqls.ExeSqlListToDB(listsql, errExit=True)
|
|
59
|
+
print(bool)
|
|
60
|
+
print(successcount)
|
|
61
|
+
print(errcount)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def test_mysql_select():
|
|
65
|
+
"""
|
|
66
|
+
查询数据 pytest mysql_test.py::test_mysql_select -s
|
|
67
|
+
:return:
|
|
68
|
+
"""
|
|
69
|
+
listsql = []
|
|
70
|
+
mysqls = MysqlUtiles(BaseFile.get_new_path(curPath, "mysql.ini"), "db", get_streamlogger())
|
|
71
|
+
sql = "SELECT * FROM test1"
|
|
72
|
+
bool, rows = mysqls.SelectFromDB(sql)
|
|
73
|
+
print(bool)
|
|
74
|
+
print(rows)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def test_mysql_select_dicts():
|
|
78
|
+
"""
|
|
79
|
+
查询数据 返回dicts 形式的数据
|
|
80
|
+
pytest mysql_test.py::test_mysql_select_dicts -s
|
|
81
|
+
:return:
|
|
82
|
+
"""
|
|
83
|
+
listsql = []
|
|
84
|
+
mysqls = MysqlUtiles(BaseFile.get_new_path(curPath, "mysql.ini"), "db", get_streamlogger(), cursorsnum=1)
|
|
85
|
+
sql = "SELECT * FROM test1"
|
|
86
|
+
bool, rows = mysqls.SelectFromDB(sql)
|
|
87
|
+
print(bool)
|
|
88
|
+
print(rows)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def test_mysql_select_one():
|
|
92
|
+
"""
|
|
93
|
+
查询数据只返回一条数据
|
|
94
|
+
pytest mysql_test.py::test_mysql_select_one -s
|
|
95
|
+
:return:
|
|
96
|
+
"""
|
|
97
|
+
listsql = []
|
|
98
|
+
mysqls = MysqlUtiles(BaseFile.get_new_path(curPath, "mysql.ini"), "db", get_streamlogger())
|
|
99
|
+
sql = "SELECT * FROM test1"
|
|
100
|
+
bool, rows = mysqls.SelectFromDBFetchOne_noyield(sql)
|
|
101
|
+
print(bool)
|
|
102
|
+
print(rows)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def test_mysql_select_one_yeild():
|
|
106
|
+
"""
|
|
107
|
+
查询数据 以yeild循环返回
|
|
108
|
+
pytest mysql_test.py::test_mysql_select_one_yeild -s
|
|
109
|
+
:return:
|
|
110
|
+
"""
|
|
111
|
+
listsql = []
|
|
112
|
+
mysqls = MysqlUtiles(BaseFile.get_new_path(curPath, "mysql.ini"), "db", get_streamlogger())
|
|
113
|
+
sql = "SELECT * FROM test1"
|
|
114
|
+
for one in mysqls.SelectFromDBFetchOne(sql):
|
|
115
|
+
print(one)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def test_mysql_exesqlmany():
|
|
119
|
+
"""
|
|
120
|
+
同时插入多条数据
|
|
121
|
+
pytest mysql_test.py::test_mysql_exesqlmany -s
|
|
122
|
+
:return:
|
|
123
|
+
"""
|
|
124
|
+
listpara = []
|
|
125
|
+
mysqls = MysqlUtiles(BaseFile.get_new_path(curPath, "mysql.ini"), "db", get_streamlogger())
|
|
126
|
+
sql = "INSERT INTO mysqlltest.test1 (username, passwd, `only`) VALUES(%s, %s, %s);"
|
|
127
|
+
for i in range(101, 201):
|
|
128
|
+
listpara.append((i, i, i))
|
|
129
|
+
bool, rows = mysqls.ExeSqlMany(sql, listpara)
|
|
130
|
+
print(bool)
|
|
131
|
+
print(rows)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
test_mysql_exesqlmany()
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def test_dict_mysql():
|
|
138
|
+
strings = """
|
|
139
|
+
host = 192.168.31.209
|
|
140
|
+
user = root
|
|
141
|
+
passwd = vipdatacenter
|
|
142
|
+
db = data_gather_record
|
|
143
|
+
port = 3306
|
|
144
|
+
chartset = utf8
|
|
145
|
+
"""
|
|
146
|
+
|
|
147
|
+
dicts_change = {"key为原来的": "values为现在的"}
|
|
148
|
+
|
|
149
|
+
from re_common.baselibrary.tools.stringtodicts import StringToDicts
|
|
150
|
+
dicts = StringToDicts().string_to_dicts_by_equal(strings)
|
|
151
|
+
mysqlutils = MysqlUtiles("", "", builder="MysqlBuilderForDicts", dicts=dicts)
|
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
import time
|
|
2
|
-
|
|
3
|
-
import pymongo
|
|
4
|
-
|
|
5
|
-
from re_common.baselibrary.utils.basepymongo import BasePyMongo
|
|
6
|
-
|
|
7
|
-
basemongo = BasePyMongo("mongodb://cjrw:vipdatacenter@192.168.31.166:27017/")
|
|
8
|
-
basemongo.use_db("collection")
|
|
9
|
-
basemongo.create_col("jss_w_f_panternt_page")
|
|
10
|
-
starttime = time.time()
|
|
11
|
-
print(basemongo.delete_col())
|
|
12
|
-
|
|
13
|
-
print(time.time() - starttime)
|
|
14
|
-
# dicts = {
|
|
15
|
-
# "cid": "string",
|
|
16
|
-
# "pagenum": "string",
|
|
17
|
-
# "down_date": "string",
|
|
18
|
-
# "html": "string",
|
|
19
|
-
# "types": "string"
|
|
20
|
-
# }
|
|
21
|
-
# basemongo.insert_one(dicts)
|
|
1
|
+
import time
|
|
2
|
+
|
|
3
|
+
import pymongo
|
|
4
|
+
|
|
5
|
+
from re_common.baselibrary.utils.basepymongo import BasePyMongo
|
|
6
|
+
|
|
7
|
+
basemongo = BasePyMongo("mongodb://cjrw:vipdatacenter@192.168.31.166:27017/")
|
|
8
|
+
basemongo.use_db("collection")
|
|
9
|
+
basemongo.create_col("jss_w_f_panternt_page")
|
|
10
|
+
starttime = time.time()
|
|
11
|
+
print(basemongo.delete_col())
|
|
12
|
+
|
|
13
|
+
print(time.time() - starttime)
|
|
14
|
+
# dicts = {
|
|
15
|
+
# "cid": "string",
|
|
16
|
+
# "pagenum": "string",
|
|
17
|
+
# "down_date": "string",
|
|
18
|
+
# "html": "string",
|
|
19
|
+
# "types": "string"
|
|
20
|
+
# }
|
|
21
|
+
# basemongo.insert_one(dicts)
|
re_common/libtest/split_test.py
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
from re_common.baselibrary.tools.split_line_to_many import Split_2_lines
|
|
2
|
-
|
|
3
|
-
def splite_test():
|
|
4
|
-
infilepath = r'E:\download\cnkistandard\download\db3\cnki_bz_append.big_json'
|
|
5
|
-
outfilepath = r'E:\download\cnkistandard\download\db3\cnki_bz_append_new.big_json'
|
|
6
|
-
spli = Split_2_lines()
|
|
7
|
-
spli.split_line(infilepath,
|
|
8
|
-
"\"}",
|
|
9
|
-
outfilepath)
|
|
10
|
-
|
|
11
|
-
if __name__ == '__main__':
|
|
1
|
+
from re_common.baselibrary.tools.split_line_to_many import Split_2_lines
|
|
2
|
+
|
|
3
|
+
def splite_test():
|
|
4
|
+
infilepath = r'E:\download\cnkistandard\download\db3\cnki_bz_append.big_json'
|
|
5
|
+
outfilepath = r'E:\download\cnkistandard\download\db3\cnki_bz_append_new.big_json'
|
|
6
|
+
spli = Split_2_lines()
|
|
7
|
+
spli.split_line(infilepath,
|
|
8
|
+
"\"}",
|
|
9
|
+
outfilepath)
|
|
10
|
+
|
|
11
|
+
if __name__ == '__main__':
|
|
12
12
|
splite_test()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
from re_common.baselibrary.database.msqlite3 import Sqlite3
|
|
2
|
-
from re_common.baselibrary.utils.basedir import BaseDir
|
|
3
|
-
|
|
4
|
-
onepath = r"F:\db32\test\base_obj_meta_a_qkwx.cnki_qk.20200527_000.db3"
|
|
5
|
-
for file in BaseDir.get_dir_all_files(r"F:\db32\cnki"):
|
|
1
|
+
from re_common.baselibrary.database.msqlite3 import Sqlite3
|
|
2
|
+
from re_common.baselibrary.utils.basedir import BaseDir
|
|
3
|
+
|
|
4
|
+
onepath = r"F:\db32\test\base_obj_meta_a_qkwx.cnki_qk.20200527_000.db3"
|
|
5
|
+
for file in BaseDir.get_dir_all_files(r"F:\db32\cnki"):
|
|
6
6
|
Sqlite3.sqlite3_merge(onepath, file, tablename="base_obj_meta_a")
|
|
@@ -1,34 +1,34 @@
|
|
|
1
|
-
onepath = r"F:\db3\ids\base_obj_meta_a.wf_qk.20201101_00000.db3"
|
|
2
|
-
|
|
3
|
-
import sqlite3
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
def dataMerge(inpath, attachpath):
|
|
7
|
-
"""
|
|
8
|
-
合并两个db3
|
|
9
|
-
:param inpath:
|
|
10
|
-
:param attachpath:
|
|
11
|
-
:return:
|
|
12
|
-
"""
|
|
13
|
-
conn = sqlite3.connect(inpath)
|
|
14
|
-
conn.text_factory = str
|
|
15
|
-
cur = conn.cursor()
|
|
16
|
-
attach = 'attach database "' + attachpath + '" as w;'
|
|
17
|
-
sql1 = 'insert into base_obj_meta_a select * from w.base_obj_meta_a;'
|
|
18
|
-
cur.execute(attach)
|
|
19
|
-
cur.execute(sql1)
|
|
20
|
-
conn.commit()
|
|
21
|
-
cur.close()
|
|
22
|
-
conn.close()
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
for i in range(0, 10):
|
|
26
|
-
attachpath = r"F:\db3\ids\base_obj_meta_a.wf_qk.20201101_00%s.db3" % (str(i))
|
|
27
|
-
dataMerge(onepath, attachpath)
|
|
28
|
-
print(attachpath)
|
|
29
|
-
|
|
30
|
-
# attachpath = r"\\192.168.31.177\down_data\cnkiwanfang\zt_wf_qk_20200323\db3\zt_wf_qk_20200323_0002.db3"
|
|
31
|
-
# dataMerge(onepath, attachpath)
|
|
32
|
-
# print(attachpath)
|
|
33
|
-
|
|
34
|
-
|
|
1
|
+
onepath = r"F:\db3\ids\base_obj_meta_a.wf_qk.20201101_00000.db3"
|
|
2
|
+
|
|
3
|
+
import sqlite3
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def dataMerge(inpath, attachpath):
|
|
7
|
+
"""
|
|
8
|
+
合并两个db3
|
|
9
|
+
:param inpath:
|
|
10
|
+
:param attachpath:
|
|
11
|
+
:return:
|
|
12
|
+
"""
|
|
13
|
+
conn = sqlite3.connect(inpath)
|
|
14
|
+
conn.text_factory = str
|
|
15
|
+
cur = conn.cursor()
|
|
16
|
+
attach = 'attach database "' + attachpath + '" as w;'
|
|
17
|
+
sql1 = 'insert into base_obj_meta_a select * from w.base_obj_meta_a;'
|
|
18
|
+
cur.execute(attach)
|
|
19
|
+
cur.execute(sql1)
|
|
20
|
+
conn.commit()
|
|
21
|
+
cur.close()
|
|
22
|
+
conn.close()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
for i in range(0, 10):
|
|
26
|
+
attachpath = r"F:\db3\ids\base_obj_meta_a.wf_qk.20201101_00%s.db3" % (str(i))
|
|
27
|
+
dataMerge(onepath, attachpath)
|
|
28
|
+
print(attachpath)
|
|
29
|
+
|
|
30
|
+
# attachpath = r"\\192.168.31.177\down_data\cnkiwanfang\zt_wf_qk_20200323\db3\zt_wf_qk_20200323_0002.db3"
|
|
31
|
+
# dataMerge(onepath, attachpath)
|
|
32
|
+
# print(attachpath)
|
|
33
|
+
|
|
34
|
+
|
|
@@ -1,30 +1,30 @@
|
|
|
1
|
-
|
|
2
|
-
def test_BaseAbs_config():
|
|
3
|
-
"""
|
|
4
|
-
命令行下 在当前目录使用该命令测试
|
|
5
|
-
pytest tomlconfig_test.py::test_BaseAbs_config -s
|
|
6
|
-
:return:
|
|
7
|
-
"""
|
|
8
|
-
from re_common.baselibrary.baseabs import BaseAbs
|
|
9
|
-
dicts = BaseAbs.get_config_factory().toml_factory().set_config_path(
|
|
10
|
-
"./config_down.toml").read_file_remove_bom().get_dicts(encoding="utf-8")
|
|
11
|
-
|
|
12
|
-
print(type(dicts))
|
|
13
|
-
print(dicts)
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def test_Tomlconfig_config():
|
|
17
|
-
"""
|
|
18
|
-
第二种简便用法
|
|
19
|
-
命令行下 在当前目录使用该命令测试
|
|
20
|
-
pytest tomlconfig_test.py::test_Tomlconfig_config -s
|
|
21
|
-
:return:
|
|
22
|
-
"""
|
|
23
|
-
from re_common.baselibrary.readconfig.toml_config import TomlConfig
|
|
24
|
-
dicts = TomlConfig("./config_down.toml").read_file_remove_bom().get_dicts()
|
|
25
|
-
print(type(dicts))
|
|
26
|
-
print(dicts)
|
|
27
|
-
|
|
28
|
-
# 已经导如facade 可以通过 facade 直接导入
|
|
29
|
-
# from facade import IniConfig
|
|
30
|
-
# 建议外部使用导入时 全部通过facade 导入
|
|
1
|
+
|
|
2
|
+
def test_BaseAbs_config():
|
|
3
|
+
"""
|
|
4
|
+
命令行下 在当前目录使用该命令测试
|
|
5
|
+
pytest tomlconfig_test.py::test_BaseAbs_config -s
|
|
6
|
+
:return:
|
|
7
|
+
"""
|
|
8
|
+
from re_common.baselibrary.baseabs import BaseAbs
|
|
9
|
+
dicts = BaseAbs.get_config_factory().toml_factory().set_config_path(
|
|
10
|
+
"./config_down.toml").read_file_remove_bom().get_dicts(encoding="utf-8")
|
|
11
|
+
|
|
12
|
+
print(type(dicts))
|
|
13
|
+
print(dicts)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def test_Tomlconfig_config():
|
|
17
|
+
"""
|
|
18
|
+
第二种简便用法
|
|
19
|
+
命令行下 在当前目录使用该命令测试
|
|
20
|
+
pytest tomlconfig_test.py::test_Tomlconfig_config -s
|
|
21
|
+
:return:
|
|
22
|
+
"""
|
|
23
|
+
from re_common.baselibrary.readconfig.toml_config import TomlConfig
|
|
24
|
+
dicts = TomlConfig("./config_down.toml").read_file_remove_bom().get_dicts()
|
|
25
|
+
print(type(dicts))
|
|
26
|
+
print(dicts)
|
|
27
|
+
|
|
28
|
+
# 已经导如facade 可以通过 facade 直接导入
|
|
29
|
+
# from facade import IniConfig
|
|
30
|
+
# 建议外部使用导入时 全部通过facade 导入
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
"""
|
|
2
|
-
对baselibrary下工具的使用
|
|
1
|
+
"""
|
|
2
|
+
对baselibrary下工具的使用
|
|
3
3
|
"""
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
# -*- coding: utf-8 -*-
|
|
2
|
-
# @Time : 2020/8/11 14:24
|
|
3
|
-
# @Author : suhong
|
|
4
|
-
# @File : __init__.py.py
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# @Time : 2020/8/11 14:24
|
|
3
|
+
# @Author : suhong
|
|
4
|
+
# @File : __init__.py.py
|
|
5
5
|
# @Software: PyCharm
|
re_common/studio/__init__.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
'''
|
|
2
|
-
改包下主要是一些python版本的特性介绍，比如某个特性是在哪个版本引入和废弃的
|
|
3
|
-
以及一些使用的样例和说明，方便查询和使用，以免一些因为版本问题引起的错误而
|
|
4
|
-
不知道，还有就是在某些版本发布的不常用语法举例。
|
|
1
|
+
'''
|
|
2
|
+
改包下主要是一些python版本的特性介绍，比如某个特性是在哪个版本引入和废弃的
|
|
3
|
+
以及一些使用的样例和说明，方便查询和使用，以免一些因为版本问题引起的错误而
|
|
4
|
+
不知道，还有就是在某些版本发布的不常用语法举例。
|
|
5
5
|
'''
|