re-common 0.2.54__py3-none-any.whl → 2.0.1__py3-none-any.whl

re_common/baselibrary/tools/move_mongo/move_mongo_table.py

@@ -1,354 +0,0 @@
-import asyncio
-import copy
-import datetime
-import json
-import time
-
-from pymongo.errors import DuplicateKeyError
-import traceback
-
-from re_common.baselibrary.mthread.MThreadingRun import MThreadingRun2
-from re_common.baselibrary.mthread.mythreading import ThreadVal, ThreadInfo
-from re_common.baselibrary.utils.basefile import BaseFile
-from re_common.baselibrary.utils.basemotor import BaseMotor
-from re_common.baselibrary.utils.basepymongo import BasePyMongo
-from re_common.facade.sqlite3facade import Sqlite3Utiles
-from re_common.facade.use.mq_use_facade import UseMq
-
-
-class Configs(object):
-
-    def __init__(self):
-        self.db3_path = r"F:\fun2\test_images.db3"
-        self.db3_encoding = "utf-8"
-        self.mgdb_conn = "mongodb://192.168.31.30:32417/"
-        self.mgdb_conn_motor = "mongodb://192.168.31.30:32417/htmljson.wanfang_ref?authSource=htmljson"
-        self.mgdb_db = "htmljson"
-        self.mgdb_col = "wanfang_ref"
-
-        self.mgdb_conn2_motor = "mongodb://cjrw:vipdatacenter@192.168.31.243:32920,192.168.31.206:32920,192.168.31.208:32920/?authSource=htmljson"
-        self.mgdb_db2 = "htmljson"
-        self.mgdb_col2 = "wanfang_ref"
-
-        self.mq_name = "mongodb.move.send"
-        self.mq_name_work = "mongodb.move.worker"
-
-        self.error_dir = r"F:\fun2\log"
-
-
-class MoveMongodbColl(object):
-    def __init__(self, conf):
-        self.conf = conf
-        self.first_id = ""
-        self.id_list = []
-        self.recv_list = []
-
-    def init_conn_mongodb(self):
-        self.basemongo = BasePyMongo(self.conf.mgdb_conn)
-        self.basemongo.use_db(self.conf.mgdb_db)
-        self.basemongo.create_col(self.conf.mgdb_col)
-
-        self.bs = BaseMotor()
-        self.bs.AsyncIOMotorClient(
-            self.conf.mgdb_conn_motor,
-            self.conf.mgdb_db)
-        self.bs.get_col(self.conf.mgdb_col)
-
-        self.bs2 = BaseMotor()
-        self.bs2.AsyncIOMotorClient(
-            self.conf.mgdb_conn2_motor,
-            self.conf.mgdb_db2)
-        self.bs2.get_col(self.conf.mgdb_col2)
-
-    def create_db3_table(self):
-        """
-        创建表
-        :return:
-        """
-        sql1 = "PRAGMA foreign_keys = false;"
-        sql2 = 'DROP TABLE IF EXISTS "cxids";'
-        sql3 = 'CREATE TABLE "cxids" ("ids" TEXT NOT NULL,"stat" integer NOT NULL DEFAULT 0,PRIMARY KEY ("ids"));'
-        sql4 = 'PRAGMA foreign_keys = true;'
-        self.db3.ExeSqlliteList([sql1, sql2, sql3, sql4])
-
-    def init_db3(self):
-        self.db3 = Sqlite3Utiles().Sqlite3DBConnectFromFilePath(self.conf.db3_path, encoding=self.conf.db3_encoding)
-
-    def init_mq(self):
-        self.use_send = UseMq(self.conf.mq_name)
-        self.use_work = UseMq(self.conf.mq_name_work)
-
-    def send_list(self):
-        while True:
-            if self.use_send.get_server_mq_num(10000):
-                for i in self.id_list:
-                    dict_info = {
-                        '_id': i
-                    }
-                    info_str = json.dumps(dict_info)
-                    print(info_str)
-                    self.use_send.easy_send_mq(info_str)
-                self.id_list.clear()
-                break
-            else:
-                time.sleep(1)
-
-    def send_db3(self):
-        while True:
-            sql = 'select * from cxids where stat=0 limit 20000'
-            rows = self.db3.SelectFromSqlliteFetchall(sql)
-            if len(rows) == 0:
-                print('查询结束 0 状态结束 查询-1状态 time sleep 60s')
-                time.sleep(60)
-                sql = 'select * from cxids where stat=-1 limit 20000'
-                rows = self.db3.SelectFromSqlliteFetchall(sql)
-                if len(rows) == 0:
-                    print('查询结束 -1 状态结束 结束发送')
-                    break
-            for row in rows:
-                _id = row[0]
-                self.id_list.append(_id)
-                if len(self.id_list) >= 10000:
-                    sql = "update cxids set stat = -1 where ids in {}".format(tuple(self.id_list))
-                    self.db3.ExeSqlliteSql(sql)
-                    self.send_list()
-
-            if len(self.id_list) > 1:
-                sql = "update cxids set stat = -1 where ids in {}".format(tuple(self.id_list))
-                self.db3.ExeSqlliteSql(sql)
-                self.send_list()
-
-            if len(self.id_list) == 1:
-                sql = "update cxids set stat = -1 where ids='{}'".format(self.id_list[0])
-                self.db3.ExeSqlliteSql(sql)
-                self.send_list()
-
-    def callback2(self, ch, method, properties, body):
-        json_data = json.loads(body)
-        _id = json_data['_id']
-        self.recv_list.append(_id)
-        if len(self.recv_list) >= 500:
-            sql = "update cxids set stat = 1 where ids in {}".format(tuple(self.recv_list))
-            if self.db3.ExeSqlliteSql(sql):
-                self.recv_list.clear()
-        else:
-            print('[{}]未更新stat条数{}'.format(datetime.datetime.now(), len(self.recv_list)))
-
-    def recv(self, results=None, *args, **kwargs):
-        self.use_work.callback2 = self.callback2
-        self.use_work.get_mq()
-
-    def get_first_mongo_id(self):
-        for i in self.basemongo.find({"_id": {"$gt": self.first_id}}, {"_id": 1}).sort([("_id", 1)]).limit(1):
-            self.first_id = i["_id"]
-            print("first_id is:" + self.first_id)
-
-    def init_data_db3(self):
-        c = 0
-        c1 = -1
-        while True:
-            lists = []
-            for i in self.basemongo.find({"_id": {"$gte": self.first_id}}, {"_id": 1}).sort([("_id", 1)]).limit(
-                    1000000):
-                c = c + 1
-                self.first_id = i["_id"]
-                lists.append((i["_id"], 0))
-                if c % 10000 == 1:
-                    print(len(lists))
-
-            sql = "insert or ignore into cxids(`ids`,`stat`) values (?,?)"
-            self.db3.ExeSqlliteMany(sql, lists)
-            print(c)
-            if c1 == c:
-                break
-            if len(lists) == 1:
-                break
-            c1 = c
-            lists.clear()
-
-    def one_init(self):
-        """
-        第一步 初始化id数据到db3目录
-        :return:
-        """
-        self.init_conn_mongodb()
-        self.init_db3()
-        self.create_db3_table()
-        self.get_first_mongo_id()
-        self.init_data_db3()
-
-    def two_send(self):
-        """
-        分布式的send方法
-        :return:
-        """
-        self.init_db3()
-        self.init_mq()
-        self.send_db3()
-
-    def two_recv(self):
-        self.init_db3()
-        self.init_mq()
-        self.recv()
-
-
-class MoveMongodbThreadRun(MThreadingRun2):
-    def __init__(self, num, conf):
-        super(MoveMongodbThreadRun, self).__init__(num)
-        self.thread_pool.work_queue.set_size(10)
-        self.loop = asyncio.new_event_loop()
-        asyncio.set_event_loop(self.loop)
-
-        self.mvmc = MoveMongodbColl(conf)
-        self.mvmc.init_mq()
-        self.mvmc.init_conn_mongodb()
-        self.mvmc.use_send.callback2 = self.callback2
-        self.lists = []
-        self.info_list = []
-        self.is_many_move = False
-        # 配置二维数组的每组数据数量，如果为批量转移 建议设置为10000
-        self.num_list = 100
-        self.num_info_list = 10
-
-    def callback2(self, ch, method, properties, body):
-        json_data = json.loads(body.decode())
-        # self.add_job(self.func, dicts)
-        self.lists.append((json_data))
-        # 异步需要 10 * 100 的二维list
-        if self.is_many_move:
-            work_size = self.thread_pool.work_queue.get_size()
-            if work_size >= 3:
-                time.sleep(10)
-        if len(self.lists) >= self.num_list:
-            self.info_list.append(copy.deepcopy(self.lists))
-            self.lists.clear()
-            print(len(self.info_list))
-
-        if len(self.info_list) >= self.num_info_list:
-            self.add_job(self.func, copy.deepcopy(self.info_list))
-            self.info_list.clear()
-
-    def set_task(self, threadval: ThreadVal, *args, **kwargs):
-        self.mvmc.use_send.get_mq()
-
-    def deal_results(self, threadval: ThreadVal, *args, **kwargs):
-        result_queue = threadval.get_result_queue()
-        while True:
-            while not result_queue.is_empty():
-                result = result_queue.get()
-                t_1, t_2 = result
-                if t_1 == "err":
-                    file_path = BaseFile.get_new_filename(self.mvmc.conf.error_dir, "err_parse_2.txt")
-                    BaseFile.single_add_file(file_path, t_2 + '\n')
-                if t_1 == "err_Exception":
-                    file_path = BaseFile.get_new_filename(self.mvmc.conf.error_dir, "err_Exception_2.txt")
-                    BaseFile.single_add_file(file_path, t_2 + '\n')
-                if t_1 == 'right':
-                    self.send_update_info(t_2)
-                self.thread_pool.result_queue.task_done()
-            time.sleep(1)
-
-    def send_update_info(self, _id):
-        dict_info = {
-            "_id": _id,
-        }
-        info_str = json.dumps(dict_info)
-        while True:
-            if self.mvmc.use_work.send_mq(info_str, num=10000):
-                break
-            time.sleep(1)
-
-    def setProxy(self, threadval: ThreadVal, proxysList=None):
-        time.sleep(60)
-
-    def is_break(self):
-        return False
-
-    def thread_pool_hook(self, threadinfo: ThreadInfo):
-        # 设置代理线程不重启，默认会重启
-        if threadinfo.get_thread_name() == self.etn.proxythreadname:
-            threadinfo.set_is_restart(False)
-        # if threadinfo.get_thread_name() == self.etn.taskthreadname:
-        #     threadinfo.set_is_restart(False)
-        return {}
-
-    def doc_hook(self, item):
-        return item
-
-    async def par_html(self, result_queue, lists):
-        for info in lists:
-            _id = info["_id"]
-            try:
-                try:
-                    # 不存在就插入
-
-                    item = await self.mvmc.bs.select_one({"_id": _id})
-                    item_result = self.doc_hook(item)
-                    await self.mvmc.bs2.insert_one(item_result)
-                    print("{}插入".format(_id))
-                    result_queue.put(("right", _id))
-                except DuplicateKeyError as e:
-                    print("{}存在".format(_id))
-                    result_queue.put(("right", _id))
-                except Exception as e:
-                    traceback.print_exc()
-                    result_queue.put(("err_Exception", _id + ":" + traceback.format_exc()))
-            except Exception as e:
-                traceback.print_exc()
-                result_queue.put(("err_Exception", _id + ":" + traceback.format_exc()))
-
-    async def par_html_many(self, result_queue, lists):
-        insert_list = []
-        for i in range(0, len(lists), 500):
-            lists_item = lists[i:i + 500]
-            try:
-                try:
-                    def deal_dicts(dd):
-                        return dd["_id"]
-                    lists_item = list(map(deal_dicts, lists_item))
-                    # 不存在就插入
-                    docs = await self.mvmc.bs.select({"_id": {"$in": lists_item}})
-                    for item in docs:
-                        item_result = self.doc_hook(item)
-                        insert_list.append(item_result)
-                        print("获取mongo num:" + str(len(insert_list)))
-                        if len(insert_list) >= 100:
-                            start_time = time.time()
-                            result = await self.mvmc.bs2.insert_many(insert_list)
-                            print("百条数据插入时间:" + str(time.time() - start_time))
-                            for _id in result.inserted_ids:
-                                print("{}插入".format(_id))
-                                result_queue.put(("right", _id))
-                            insert_list.clear()
-                except DuplicateKeyError as e:
-                    print("有数据存在，无法插入")
-
-                except Exception as e:
-                    traceback.print_exc()
-                    result_queue.put(("err_Exception", traceback.format_exc()))
-            except Exception as e:
-                traceback.print_exc()
-                result_queue.put(("err_Exception", traceback.format_exc()))
-
-        if len(insert_list) > 0:
-            try:
-                result = await self.mvmc.bs2.insert_many(insert_list)
-                for _id in result.inserted_ids:
-                    print("{}插入".format(_id))
-                    result_queue.put(("right", _id))
-                insert_list.clear()
-            except DuplicateKeyError as e:
-                print("存在某个key 批量插入失败")
-            except Exception as e:
-                traceback.print_exc()
-                result_queue.put(("err_Exception", traceback.format_exc()))
-
-    def fun(self, threadval, *args, **kwargs):
-        result_queue = threadval.get_result_queue()
-        func_list = []
-        for lists in args[0]:
-            if self.is_many_move:
-                func_list.append(self.par_html_many(result_queue, lists))
-            else:
-                func_list.append(self.par_html(result_queue, lists))
-        self.loop.run_until_complete(asyncio.wait(func_list))

re_common/baselibrary/tools/move_mongo/use_mttf.py

@@ -1,18 +0,0 @@
-from re_common.baselibrary.tools.move_mongo.mongo_table_to_file import Configs, MongoToFile
-
-conf = Configs()
-
-
-def hook_doc(doc):
-    doc_info = doc["step_info"]
-    id_ = doc_info["id"]
-    url = f"https://wiki.mbalib.com/wiki/{id_}"
-    doc_info["url"] = url
-    return doc
-
-
-mtf = MongoToFile(conf)
-mtf.hook_doc = hook_doc
-mtf.init_conn_mongodb()
-mtf.open_file()
-mtf.asyncio_run()

re_common/baselibrary/tools/move_mongo/use_mv.py

@@ -1,93 +0,0 @@
-import click
-
-###########################################
-# 同项目调用基础包
-import datetime
-import gzip
-import json
-import os
-import sys
-import time
-
-filepath = os.path.abspath(__file__)
-pathlist = filepath.split(os.sep)
-pathlist = pathlist[:-5]
-TopPath = os.sep.join(pathlist)
-sys.path.insert(0, TopPath)
-print(TopPath)
-############################################
-
-
-from re_common.baselibrary.tools.move_mongo.move_mongo_table import MoveMongodbColl, MoveMongodbThreadRun
-
-
-class Configs(object):
-
-    def __init__(self):
-        self.db3_path = r"D:\config\mvmg\db3\test_images.db3"
-        self.db3_encoding = "utf-8"
-        self.mgdb_conn = "mongodb://192.168.31.30:32417/"
-        self.mgdb_conn_motor = "mongodb://192.168.31.30:32417/htmljson.wanfang_ref?authSource=htmljson"
-        self.mgdb_db = "htmljson"
-        self.mgdb_col = "cx_journal_detail"
-
-        self.mgdb_conn2_motor = "mongodb://cjrw:vipdatacenter@192.168.31.243:32920,192.168.31.206:32920,192.168.31.208:32920/?authSource=htmljson"
-        self.mgdb_db2 = "htmljson"
-        self.mgdb_col2 = "cx_journal_detail"
-
-        self.mq_name = "mongodb.move.send"
-        self.mq_name_work = "mongodb.move.worker"
-
-        self.error_dir = r"D:\config\mvmg\log"
-
-
-conf = Configs()
-
-
-def one_init():
-    # 第一步 初始化数据到db3文件
-    MoveMongodbColl(conf).one_init()
-
-
-# 第二步 开始转移 使用send_work_recv结构 共开三个进程
-
-def send():
-    MoveMongodbColl(conf).two_send()
-
-
-def works():
-    MoveMongodbThreadRun(1, conf).run()
-
-
-def works_many():
-    mmtr = MoveMongodbThreadRun(1, conf)
-    mmtr.is_many_move = True
-    mmtr.num_list = 1000
-    mmtr.num_info_list = 10
-    mmtr.run()
-
-
-def recv():
-    MoveMongodbColl(conf).two_recv()
-
-
-@click.command()
-@click.option('--name',
-              help='func name')
-def main(name):
-    if name == "send":
-        send()
-    elif name == "one_init":
-        one_init()
-    elif name == "works":
-        works()
-
-    elif name == "works_many":
-        works_many()
-
-    elif name == "recv":
-        recv()
-
-
-if __name__ == '__main__':
-    main()

re_common/baselibrary/tools/mpandas/mpandasreadexcel.py

@@ -1,125 +0,0 @@
-###############################################################
-# import pandas as pd
-#
-# # io = r'C:\Users\xuzhu\Desktop\OutK.xlsx'
-# io = r'.\raw.xlsx'
-# data = pd.read_excel(io, sheet_name=0)
-#
-# # print(data.head())
-# # print(data.tail())
-# # print(data.groupby(["GCH","years"]).groups)
-# onedata = data.groupby('GCH').apply(lambda t: t[t.years == t.years.min()])
-# print("write excel")
-# with pd.ExcelWriter('temp1.xlsx') as writer:
-#     onedata.to_excel(writer, sheet_name='Sheet1')
-
-##########################################################
-
-# import pandas as pd
-#
-# # io = r'C:\Users\xuzhu\Desktop\OutK.xlsx'
-# io = r'.\temp1.xlsx'
-# data = pd.read_excel(io, sheet_name=0)
-#
-# # print(data.head())
-# # print(data.tail())
-# # print(data.groupby(["GCH","years"]).groups)
-# onedata = data.groupby('GCH').apply(lambda t: t[t.num == t.num.min()])
-# print("write excel")
-# with pd.ExcelWriter('temp2.xlsx') as writer:
-#     onedata.to_excel(writer, sheet_name='Sheet1')
-
-############################################################
-
-# import pandas as pd
-#
-# # io = r'C:\Users\xuzhu\Desktop\OutK.xlsx'
-# io = r'.\temp2.xlsx'
-# data = pd.read_excel(io, sheet_name=0)
-#
-# def bug_rule(x):
-#     gch = x.GCH
-#     gch1 = gch[:-1]
-#     return gch1
-#
-# data["gch5"] = data.apply(lambda x: bug_rule(x), axis=1)
-# print("write excel")
-# with pd.ExcelWriter('temp3.xlsx') as writer:
-#     data.to_excel(writer, sheet_name='Sheet1')
-
-
-###################################################
-#
-#
-# import sys
-#
-# import pandas as pd
-#
-# # io = r'C:\Users\xuzhu\Desktop\OutK.xlsx'
-# io = r'.\temp3.xlsx'
-# io2 = r'.\raw2.xlsx'
-# data = pd.read_excel(io, sheet_name=0)
-# print(data.dtypes)
-# print("****************")
-# data2 = pd.read_excel(io2, sheet_name=1)
-# print(data2.dtypes)
-# data3 = pd.merge(data, data2, on='gch5')
-#
-# with pd.ExcelWriter('temp4.xlsx') as writer:
-#     data3.to_excel(writer, sheet_name='Sheet1')
-
-##########################################################
-
-
-# import pandas as pd
-#
-# # io = r'C:\Users\xuzhu\Desktop\OutK.xlsx'
-# io = r'.\temp4.xlsx'
-# data = pd.read_excel(io, sheet_name=0)
-#
-# # print(data.head())
-# # print(data.tail())
-# # print(data.groupby(["GCH","years"]).groups)
-# onedata = data.groupby('gch5').apply(lambda t: t[t.years == t.years.min()])
-# print("write excel")
-# with pd.ExcelWriter('temp5.xlsx') as writer:
-#     onedata.to_excel(writer, sheet_name='Sheet1')
-
-##########################################################
-
-# import pandas as pd
-#
-# # io = r'C:\Users\xuzhu\Desktop\OutK.xlsx'
-# io = r'.\temp5.xlsx'
-# data = pd.read_excel(io, sheet_name=0)
-#
-# # print(data.head())
-# # print(data.tail())
-# # print(data.groupby(["GCH","years"]).groups)
-# onedata = data.groupby('gch5').apply(lambda t: t[t.num == t.num.min()])
-# print("write excel")
-# with pd.ExcelWriter('temp6.xlsx') as writer:
-#     onedata.to_excel(writer, sheet_name='Sheet1')
-
-#########################################################
-
-import pandas as pd
-
-
-io = r'.\temp4.xlsx'
-io2 = r'.\raw2.xlsx'
-data = pd.read_excel(io, sheet_name=0)
-data2 = pd.read_excel(io2, sheet_name=1)
-
-print(type(data["刊名"]))
-print(data["刊名"].values.tolist())
-print(type(data2["刊名"]))
-print(data2["刊名"].values.tolist())
-
-print(set(data2["刊名"].values.tolist()) - set(data["刊名"].values.tolist()))
-
-# onedata = data.groupby('gch5').apply(lambda t: t[t.num == t.num.min()])
-# print("write excel")
-# with pd.ExcelWriter('temp6.xlsx') as writer:
-#     onedata.to_excel(writer, sheet_name='Sheet1')
-

re_common/baselibrary/tools/mpandas/pandas_visualization.py

@@ -1,8 +0,0 @@
-import numpy as np
-import pandas as pd
-
-ts = pd.Series(np.random.randn(1000),
-               index=pd.date_range('1/1/2000', periods=1000))
-
-ts = ts.cumsum()
-ts.plot()