re-common 10.0.39__py3-none-any.whl → 10.0.41__py3-none-any.whl

re_common/facade/use/mq_use_facade.py

@@ -1,83 +1,83 @@
-import logging
-import traceback
-
-from re_common.baselibrary.utils.basepika import BasePika
-from retry import retry
-
-logging_logger = logging.getLogger(__name__)
-
-
-class UseMq(object):
-
-    def __init__(self, queue, qos=1):
-        self.queue = queue
-        self.qos = qos
-        self.basepika = BasePika()
-        self.basepika.set_default()
-        self.basepika.connect()
-        self.basepika.create_channel()
-        self.basepika.queue_declare(queue=queue, durable=True)
-        self.basepika.basic_qos(qos)
-        self.properties = self.basepika.get_properties()
-
-    def re_conn(self):
-        """
-        重新连接
-        :return:
-        """
-        self.basepika.connect()
-        self.basepika.create_channel()
-        self.basepika.queue_declare(queue=self.queue, durable=True)
-        self.basepika.basic_qos(self.qos)
-
-    @retry(delay=5, backoff=2, max_delay=60 * 3, logger=logging_logger)
-    def get_mq(self):
-        try:
-            if self.basepika.channel.is_closed:
-                logging_logger.info("重连中......")
-                self.re_conn()
-                logging_logger.info("重连完成......")
-            self.basepika.set_get_msg_callback(routing_key=self.queue, callback=self.callback, auto_ack=False)
-            self.basepika.start_get_msg()
-        except:
-            traceback.print_exc()
-            logging_logger.info("重连中......")
-            self.re_conn()
-
-    def callback(self, ch, method, properties, body):
-        # print(type(body))
-        # print(" [x] Received %r" % body)
-        # body = body.decode()
-        self.callback2(ch, method, properties, body)
-        if self.basepika.auto_ack is False:
-            self.basepika.basic_ack(ch, method)
-
-    def callback2(self, ch, method, properties, body):
-        pass
-
-    @retry(delay=5, backoff=2, max_delay=60 * 3, logger=logging_logger)
-    def send_mq(self, body, num=100):
-        try:
-            if self.basepika.get_queue_size(self.queue) < num:
-                self.basepika.easy_send_msg(routing_key=self.queue,
-                                            body=body,
-                                            properties=self.properties)
-                return True
-            else:
-                return False
-        except:
-            traceback.print_exc()
-            logging_logger.info("重连中......")
-            self.re_conn()
-            return False
-
-    def get_server_mq_num(self, num=100):
-        if self.basepika.get_queue_size(self.queue) < num:
-            return True
-        else:
-            return False
-
-    def easy_send_mq(self, body):
-        self.basepika.easy_send_msg(routing_key=self.queue,
-                                    body=body,
-                                    properties=self.properties)
+import logging
+import traceback
+
+from re_common.baselibrary.utils.basepika import BasePika
+from retry import retry
+
+logging_logger = logging.getLogger(__name__)
+
+
+class UseMq(object):
+
+    def __init__(self, queue, qos=1):
+        self.queue = queue
+        self.qos = qos
+        self.basepika = BasePika()
+        self.basepika.set_default()
+        self.basepika.connect()
+        self.basepika.create_channel()
+        self.basepika.queue_declare(queue=queue, durable=True)
+        self.basepika.basic_qos(qos)
+        self.properties = self.basepika.get_properties()
+
+    def re_conn(self):
+        """
+        重新连接
+        :return:
+        """
+        self.basepika.connect()
+        self.basepika.create_channel()
+        self.basepika.queue_declare(queue=self.queue, durable=True)
+        self.basepika.basic_qos(self.qos)
+
+    @retry(delay=5, backoff=2, max_delay=60 * 3, logger=logging_logger)
+    def get_mq(self):
+        try:
+            if self.basepika.channel.is_closed:
+                logging_logger.info("重连中......")
+                self.re_conn()
+                logging_logger.info("重连完成......")
+            self.basepika.set_get_msg_callback(routing_key=self.queue, callback=self.callback, auto_ack=False)
+            self.basepika.start_get_msg()
+        except:
+            traceback.print_exc()
+            logging_logger.info("重连中......")
+            self.re_conn()
+
+    def callback(self, ch, method, properties, body):
+        # print(type(body))
+        # print(" [x] Received %r" % body)
+        # body = body.decode()
+        self.callback2(ch, method, properties, body)
+        if self.basepika.auto_ack is False:
+            self.basepika.basic_ack(ch, method)
+
+    def callback2(self, ch, method, properties, body):
+        pass
+
+    @retry(delay=5, backoff=2, max_delay=60 * 3, logger=logging_logger)
+    def send_mq(self, body, num=100):
+        try:
+            if self.basepika.get_queue_size(self.queue) < num:
+                self.basepika.easy_send_msg(routing_key=self.queue,
+                                            body=body,
+                                            properties=self.properties)
+                return True
+            else:
+                return False
+        except:
+            traceback.print_exc()
+            logging_logger.info("重连中......")
+            self.re_conn()
+            return False
+
+    def get_server_mq_num(self, num=100):
+        if self.basepika.get_queue_size(self.queue) < num:
+            return True
+        else:
+            return False
+
+    def easy_send_mq(self, body):
+        self.basepika.easy_send_msg(routing_key=self.queue,
+                                    body=body,
+                                    properties=self.properties)

re_common/facade/use/proxy_use_facade.py

@@ -1,20 +1,20 @@
-import time
-
-
-def set_school_list_proxy(self):
-    """
-    使用该函数条件, self为类的对象
-    代理在 proxyset set里
-    使用的是bshttpx这个变量
-    """
-    try:
-        proxy = self.proxyset.pop()
-        self.logger.info("proxy is:{},proxy size is: {}".format(proxy, len(self.proxyset)))
-        return self.bshttpx.creat_sn(proxy=proxy,
-                                     headers=self.headers,
-                                     verify=False)
-    except KeyError as e:
-        time.sleep(15)
-        if str(e) == "'pop from an empty set'":
-            self.proxyset = set(self.school_proxy_list)
+import time
+
+
+def set_school_list_proxy(self):
+    """
+    使用该函数条件, self为类的对象
+    代理在 proxyset set里
+    使用的是bshttpx这个变量
+    """
+    try:
+        proxy = self.proxyset.pop()
+        self.logger.info("proxy is:{},proxy size is: {}".format(proxy, len(self.proxyset)))
+        return self.bshttpx.creat_sn(proxy=proxy,
+                                     headers=self.headers,
+                                     verify=False)
+    except KeyError as e:
+        time.sleep(15)
+        if str(e) == "'pop from an empty set'":
+            self.proxyset = set(self.school_proxy_list)
             return self.set_school_list_proxy()

re_common/libtest/base_dict_test.py

@@ -1,19 +1,19 @@
-from re_common.baselibrary import BaseDicts
-from re_common.baselibrary.utils.mfaker import MFaker
-
-
-def test_basedict_sortkeys():
-    fake = MFaker()
-    dicts = fake.create_data(MFaker.m_pydict, **fake.py_para())
-    print(dicts)
-    dicts2 = BaseDicts.sortkeys(dicts)
-    print(dicts2)
-    print(dicts)
-
-
-def test_basedict_sortvalues():
-    dicts = {'data': 'FDZqqOGNMyGJlNRoCsJd', 'participant': 'petersonadrienne@bennett.com',
-             'often': 'KnJSSDeSTPboiwjSdGwR', 'friend': '1639', 'above': '8144', 'in': '1614'}
-    dicts2 = BaseDicts.sortvalues(dicts)
-    print(dicts2)
-
+from re_common.baselibrary import BaseDicts
+from re_common.baselibrary.utils.mfaker import MFaker
+
+
+def test_basedict_sortkeys():
+    fake = MFaker()
+    dicts = fake.create_data(MFaker.m_pydict, **fake.py_para())
+    print(dicts)
+    dicts2 = BaseDicts.sortkeys(dicts)
+    print(dicts2)
+    print(dicts)
+
+
+def test_basedict_sortvalues():
+    dicts = {'data': 'FDZqqOGNMyGJlNRoCsJd', 'participant': 'petersonadrienne@bennett.com',
+             'often': 'KnJSSDeSTPboiwjSdGwR', 'friend': '1639', 'above': '8144', 'in': '1614'}
+    dicts2 = BaseDicts.sortvalues(dicts)
+    print(dicts2)
+

re_common/libtest/baseavro_test.py

@@ -1,13 +1,13 @@
-from re_common.baselibrary.utils.baseavro import BaseAvro
-
-id_set = set()
-for line in BaseAvro().read_line_yeild(r"F:\fun2\avro"):
-    id_set.add(line["key"])
-
-print(len(id_set))
-lines = ""
-
-with open(r"F:\fun2\avro1.txt", 'w', encoding="utf-8") as f:
-    for id in id_set:
-        lines = id + "\n"
-        f.write(lines)
+from re_common.baselibrary.utils.baseavro import BaseAvro
+
+id_set = set()
+for line in BaseAvro().read_line_yeild(r"F:\fun2\avro"):
+    id_set.add(line["key"])
+
+print(len(id_set))
+lines = ""
+
+with open(r"F:\fun2\avro1.txt", 'w', encoding="utf-8") as f:
+    for id in id_set:
+        lines = id + "\n"
+        f.write(lines)

re_common/libtest/basefile_test.py

@@ -1,14 +1,14 @@
-from re_common.baselibrary.utils.basedir import BaseDir
-from re_common.baselibrary.utils.basefile import BaseFile
-
-# files_line_list = BaseFile.read_end_line(r"F:\db3\mysql_date\test\part-00000", 3)
-# for file_line in files_line_list:
-#     # file_line = str(file_line, encoding="utf-8")
-#     # file_line = file_line.decode(encoding="utf-8")
-#     print(file_line)
-
-for file in BaseDir.get_dir_all_files(r"F:\db3\mysql_date\data_dir"):
-    files_line_list = BaseFile.read_end_line(r"F:\db3\mysql_date\test\part-00000", 11000)
-    strs = "\n".join(files_line_list)
-    BaseFile.single_add_file(r"F:\db3\mysql_date\end\part-00000", strs + "\n")
-
+from re_common.baselibrary.utils.basedir import BaseDir
+from re_common.baselibrary.utils.basefile import BaseFile
+
+# files_line_list = BaseFile.read_end_line(r"F:\db3\mysql_date\test\part-00000", 3)
+# for file_line in files_line_list:
+#     # file_line = str(file_line, encoding="utf-8")
+#     # file_line = file_line.decode(encoding="utf-8")
+#     print(file_line)
+
+for file in BaseDir.get_dir_all_files(r"F:\db3\mysql_date\data_dir"):
+    files_line_list = BaseFile.read_end_line(r"F:\db3\mysql_date\test\part-00000", 11000)
+    strs = "\n".join(files_line_list)
+    BaseFile.single_add_file(r"F:\db3\mysql_date\end\part-00000", strs + "\n")
+

re_common/libtest/basemssql_test.py

@@ -1,77 +1,77 @@
-###########################################
-# 同项目调用基础包
-import datetime
-import gzip
-import json
-import os
-import sys
-import time
-
-filepath = os.path.abspath(__file__)
-pathlist = filepath.split(os.sep)
-pathlist = pathlist[:-3]
-TopPath = os.sep.join(pathlist)
-sys.path.insert(0, TopPath)
-print(TopPath)
-############################################
-
-from re_common.baselibrary.utils.basemssql import BaseMsSql
-from re_common.baselibrary.utils.basefile import BaseFile
-from re_common.baselibrary.utils.basetime import BaseTime
-bt = BaseTime()
-
-host = "127.0.0.1"
-user = "sa"
-pwd = "xujiang1994323"
-db = "patData"
-charset = "utf8"
-
-basemssql = BaseMsSql(host, user, pwd, db, charset, as_dict=True)
-basemssql.conn()
-basemssql.exec_select_query("select * from [dbo].[New_pattzb]")
-outPathFile = r"F:\db3\patnetjson_big\jss_patent.big_json.gz"
-i = 0
-size = 10000
-count = 2000000
-outPathFile = BaseFile.get_new_filename(outPathFile, sign=".")
-start = time.time()
-end = False
-while True:
-    with gzip.open(outPathFile, 'wb') as f:
-        while True:
-            a = basemssql.cur.fetchmany(size=size)
-            if not a:
-                print(i)
-                print(int(time.time() - start))
-                print("break")
-                end = True
-                break
-            for row in a:
-                dicts = {}
-                for k,v in row.items():
-                    # print(k)
-                    try:
-                        v = v.encode('latin-1').decode('gbk')
-                    except:
-                        pass
-                    if isinstance(v, datetime.datetime):
-                        v = bt.datetime_to_string(v, "%Y-%m-%d %H:%M:%S")
-                    dicts[k] = v
-                    # print(type(v))
-                # print(dicts)
-                line = json.dumps(dicts,ensure_ascii=False)+"\n"
-                # print(line)
-                lines = line.encode(encoding="utf8")
-                f.write(lines)
-            i = i + size
-            print(i)
-            print(int(time.time() - start))
-            if i >= count:
-                outPathFile = BaseFile.get_new_filename(outPathFile, sign=".")
-                i=0
-                break
-    if end:
-        break
-
-
-
+###########################################
+# 同项目调用基础包
+import datetime
+import gzip
+import json
+import os
+import sys
+import time
+
+filepath = os.path.abspath(__file__)
+pathlist = filepath.split(os.sep)
+pathlist = pathlist[:-3]
+TopPath = os.sep.join(pathlist)
+sys.path.insert(0, TopPath)
+print(TopPath)
+############################################
+
+from re_common.baselibrary.utils.basemssql import BaseMsSql
+from re_common.baselibrary.utils.basefile import BaseFile
+from re_common.baselibrary.utils.basetime import BaseTime
+bt = BaseTime()
+
+host = "127.0.0.1"
+user = "sa"
+pwd = "xujiang1994323"
+db = "patData"
+charset = "utf8"
+
+basemssql = BaseMsSql(host, user, pwd, db, charset, as_dict=True)
+basemssql.conn()
+basemssql.exec_select_query("select * from [dbo].[New_pattzb]")
+outPathFile = r"F:\db3\patnetjson_big\jss_patent.big_json.gz"
+i = 0
+size = 10000
+count = 2000000
+outPathFile = BaseFile.get_new_filename(outPathFile, sign=".")
+start = time.time()
+end = False
+while True:
+    with gzip.open(outPathFile, 'wb') as f:
+        while True:
+            a = basemssql.cur.fetchmany(size=size)
+            if not a:
+                print(i)
+                print(int(time.time() - start))
+                print("break")
+                end = True
+                break
+            for row in a:
+                dicts = {}
+                for k,v in row.items():
+                    # print(k)
+                    try:
+                        v = v.encode('latin-1').decode('gbk')
+                    except:
+                        pass
+                    if isinstance(v, datetime.datetime):
+                        v = bt.datetime_to_string(v, "%Y-%m-%d %H:%M:%S")
+                    dicts[k] = v
+                    # print(type(v))
+                # print(dicts)
+                line = json.dumps(dicts,ensure_ascii=False)+"\n"
+                # print(line)
+                lines = line.encode(encoding="utf8")
+                f.write(lines)
+            i = i + size
+            print(i)
+            print(int(time.time() - start))
+            if i >= count:
+                outPathFile = BaseFile.get_new_filename(outPathFile, sign=".")
+                i=0
+                break
+    if end:
+        break
+
+
+

re_common/libtest/baseodbc_test.py

@@ -1,8 +1,8 @@
-from re_common.baselibrary.utils.baseodbc import BaseODBC
-
-baseodbc = BaseODBC(r"C:\Users\xuzhu\Desktop\DB_20200701_GB.mdb")
-# baseodbc = BaseODBC(r"D:\download\cnki_qk\download\get_journal\mdb\cnki期刊信息_20200315.mdb")
-baseodbc.get_cur()
-sql = "select * from `CN`"
-for row in baseodbc.select(sql):
+from re_common.baselibrary.utils.baseodbc import BaseODBC
+
+baseodbc = BaseODBC(r"C:\Users\xuzhu\Desktop\DB_20200701_GB.mdb")
+# baseodbc = BaseODBC(r"D:\download\cnki_qk\download\get_journal\mdb\cnki期刊信息_20200315.mdb")
+baseodbc.get_cur()
+sql = "select * from `CN`"
+for row in baseodbc.select(sql):
     print(row)

re_common/libtest/basepandas_test.py

@@ -1,38 +1,38 @@
-import time
-
-from pandas._libs.tslibs.timestamps import Timestamp
-
-
-def test_dataform_to_numpy():
-    import pandas as pd
-    import numpy as np
-    dicts = {'A': 1.,
-             'B': pd.Timestamp('20130102'),
-             'C': pd.Series(1, index=list(range(4)), dtype='float32'),
-             'D': np.array([3] * 4, dtype='int32'),
-             'E': pd.Categorical(["test", "train", "test", "train"]),
-             'F': 'foo'}
-    from re_common.baselibrary.utils.basepandas import BasePandas
-    bp = BasePandas()
-    startTime = time.time()
-    df = bp.dicts_to_dataform(dicts)
-    # 数据类型一致时速度会很快
-    print(bp.dataform_to_numpy(df))
-    endTime = time.time()
-    print(endTime - startTime)
-    dicts = {'A': [1.0, Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo'],
-             'B': [Timestamp('2013-01-02 00:00:00'), Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo'],
-             'C': [1.0, Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo'],
-             'D': [3, Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo'],
-             'E': ['test', Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo'],
-             'F': ['foo', Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo']}
-
-
-    startTime = time.time()
-    df = bp.dicts_to_dataform(dicts)
-    # 数据类型一致时速度会很快
-    print(bp.dataform_to_numpy(df))
-    endTime = time.time()
-    print(endTime - startTime)
-
-test_dataform_to_numpy()
+import time
+
+from pandas._libs.tslibs.timestamps import Timestamp
+
+
+def test_dataform_to_numpy():
+    import pandas as pd
+    import numpy as np
+    dicts = {'A': 1.,
+             'B': pd.Timestamp('20130102'),
+             'C': pd.Series(1, index=list(range(4)), dtype='float32'),
+             'D': np.array([3] * 4, dtype='int32'),
+             'E': pd.Categorical(["test", "train", "test", "train"]),
+             'F': 'foo'}
+    from re_common.baselibrary.utils.basepandas import BasePandas
+    bp = BasePandas()
+    startTime = time.time()
+    df = bp.dicts_to_dataform(dicts)
+    # 数据类型一致时速度会很快
+    print(bp.dataform_to_numpy(df))
+    endTime = time.time()
+    print(endTime - startTime)
+    dicts = {'A': [1.0, Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo'],
+             'B': [Timestamp('2013-01-02 00:00:00'), Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo'],
+             'C': [1.0, Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo'],
+             'D': [3, Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo'],
+             'E': ['test', Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo'],
+             'F': ['foo', Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo']}
+
+
+    startTime = time.time()
+    df = bp.dicts_to_dataform(dicts)
+    # 数据类型一致时速度会很快
+    print(bp.dataform_to_numpy(df))
+    endTime = time.time()
+    print(endTime - startTime)
+
+test_dataform_to_numpy()

re_common/libtest/get_attr_test/get_attr_test_settings.py

@@ -1,14 +1,14 @@
-import os
-import sys
-
-filepath = os.path.abspath(__file__)
-pathlist = filepath.split(os.sep)
-pathlist = pathlist[:-4]
-TopPath = os.sep.join(pathlist)
-sys.path.insert(0, TopPath)
-print(TopPath)
-
-from re_common.libtest.get_attr_test import settings
-from re_common.baselibrary.tools.get_attr import get_attrs
-
-print(get_attrs(settings))
+import os
+import sys
+
+filepath = os.path.abspath(__file__)
+pathlist = filepath.split(os.sep)
+pathlist = pathlist[:-4]
+TopPath = os.sep.join(pathlist)
+sys.path.insert(0, TopPath)
+print(TopPath)
+
+from re_common.libtest.get_attr_test import settings
+from re_common.baselibrary.tools.get_attr import get_attrs
+
+print(get_attrs(settings))