re-common 2.0.1__py3-none-any.whl → 10.0.0__py3-none-any.whl

re_common/facade/sqlite3facade.py

@@ -0,0 +1,257 @@
+# db3数据的连接返回一个连接对象
+import sys
+import traceback
+
+from re_common.baselibrary import MLogger
+from re_common.baselibrary.baseabs import BaseAbs
+from re_common.baselibrary.database.mbuilder import Sqlite3Builder
+
+
+class Sqlite3Utiles(object):
+    def __init__(self, logger=None):
+        self.sqllite3 = None
+        self.conn = None
+        self.cursor = None
+
+        self._logger = logger
+
+    @property
+    def logger(self):
+        if self._logger is None:
+            return MLogger().streamlogger
+        return self._logger
+
+    @logger.setter
+    def logger(self, value):
+        assert isinstance(value, MLogger)
+        self._logger = value
+
+    def get_new_cursor(self):
+        """
+        获取一个新的游标
+        :return:
+        """
+        # 检查db的存在及是否断掉 不知道为什么 不可以加括号 但编辑器会警告
+        return self.conn.cursor()
+
+    def Sqlite3DBConnectFromFilePath(self, dbfilepath, encoding="gbk", timeout=60):
+        """
+        通过直接文件连接 我使用Hadoop导下来的db3是gbk编码的
+        如果为utf-8编码请改为utf-8
+        :param sec:  section
+        :return:
+        """
+        sqllite3 = BaseAbs.get_sql_factory().sqlite_factory()
+        # 传入的是配置文件和section选项 dbpath为配置文件路径
+        sqlite3_moudle = Sqlite3Builder().build_file_path(dbfilepath).build_timeout(timeout).get_moudle()
+        sqllite3.link(sqlite3_moudle)
+        # 设置txt的编码格式  hadoop 上的db3为gbk 默认为utf8
+        sqllite3.set_encoding(encoding=encoding)
+        # 返回一个连接
+        self.sqllite3 = sqllite3
+        self.conn = self.sqllite3.db
+        return self
+
+    def Sqlite3DBConnectFromConfig(self, cobnfigfilepath, sec, encoding="gbk"):
+        """
+        通过配置文件连接
+        :param sec:  section
+        :return:
+        """
+        sqllite3 = BaseAbs.get_sql_factory().sqlite_factory()
+        # 传入的是配置文件和section选项 dbpath为配置文件路径
+        sqlite3_moudle = Sqlite3Builder(cobnfigfilepath, sec).build_all().get_moudle()
+        sqllite3.link(sqlite3_moudle)
+        # 设置txt的编码格式  hadoop 上的db3为gbk 默认为utf8
+        sqllite3.set_encoding(encoding=encoding)
+        # 返回一个连接
+        self.sqllite3 = sqllite3
+        self.conn = self.sqllite3.db
+        return self
+
+    def ExeSqlliteList(self, sqlList, errExit=True):
+        """
+        该函数和上面一样执行一个sql列表且不返回结果
+        属于插入和更新类函数 但该函数针对db3
+        :param sqllitename:
+        :param sqlList:
+        :return:
+        """
+        dbMsg = None
+        cur = self.get_new_cursor()
+        if self.conn:
+            count = 0
+            for sql in sqlList:
+                count += 1
+                try:
+                    self.logger.info("{} 执行sql数量:{}".format(sql, str(count)))
+                    cur.execute(sql)
+                except:
+                    self.logger.error('*errSql:' + sql)
+                    dbMsg = '*InsertError:' + traceback.format_exc()
+                    if errExit:
+                        self.logger.error(dbMsg)
+                        sys.exit()
+                if dbMsg:
+                    self.logger.error(dbMsg)
+                    continue
+        self.conn.commit()
+        cur.close()
+
+    def ExeSqlliteMany(self, sql, itermany, errExit=True):
+        dbMsg = None
+        cur = self.get_new_cursor()
+        if self.conn:
+            try:
+                self.logger.info("{}\n{}".format(sql, str(itermany)))
+                cur.executemany(sql, itermany)
+            except:
+                self.logger.error('*errSql:' + sql)
+                dbMsg = '*InsertError:' + traceback.format_exc()
+                if errExit:
+                    self.logger.error(dbMsg)
+                    sys.exit()
+            if dbMsg:
+                self.logger.error(dbMsg)
+        self.conn.commit()
+        cur.close()
+
+    def ExeSqlliteSql(self, sql):
+        """
+        该函数和上面一样执行一个sql列表且不返回结果
+        属于插入和更新类函数 但该函数针对db3
+        :param sqllitename:
+        :param sqlList:
+        :return:
+        """
+        dbMsg = None
+        cur = self.get_new_cursor()
+        if self.conn:
+            try:
+                self.logger.info(sql)
+                cur.execute(sql)
+                self.conn.commit()
+            except:
+                self.logger.error('*errSql:' + sql)
+                dbMsg = '*InsertError:' + traceback.format_exc()
+            if dbMsg:
+                self.logger.error(dbMsg)
+                return False
+        else:
+            return False
+        cur.close()
+        return True
+
+    def SelectFromSqlliteFetchall(self, sql):
+        """
+        该函数和上面一样执行一个sql列表且不返回结果
+        属于插入和更新类函数 但该函数针对db3
+        :param sqllitename:
+        :param sqlList:
+        :return:
+        """
+        cur = self.get_new_cursor()
+        if self.conn:
+            try:
+                self.logger.info(sql)
+                cur.execute(sql)
+                rows = cur.fetchall()
+                return rows
+            except:
+                self.logger.error('*errSql:' + sql)
+                dbMsg = '*InsertError:' + traceback.format_exc()
+            if dbMsg:
+                self.logger.error(dbMsg)
+                return False
+        else:
+            return False
+        cur.close()
+
+    def SelectFromSqlliteFetchall_dicts(self, sql):
+        """
+        该函数和上面一样执行一个sql列表且不返回结果
+        属于插入和更新类函数 但该函数针对db3
+        :param sqllitename:
+        :param sqlList:
+        :return:
+        """
+
+        def dict_factory(cursor, row):
+            d = {}
+            for idx, col in enumerate(cursor.description):
+                d[col[0]] = row[idx]
+            return d
+
+        self.conn.row_factory = dict_factory
+        cur = self.get_new_cursor()
+        if self.conn:
+            try:
+                self.logger.info(sql)
+                cur.execute(sql)
+                rows = cur.fetchall()
+                return rows
+            except:
+                self.logger.error('*errSql:' + sql)
+                dbMsg = '*InsertError:' + traceback.format_exc()
+            if dbMsg:
+                self.logger.error(dbMsg)
+                return False
+        else:
+            return False
+        cur.close()
+
+    def SelectFromSqlliteFetchOne(self, sql):
+        """
+        该函数和上面一样执行一个sql列表且不返回结果
+        属于插入和更新类函数 但该函数针对db3
+        :param sqllitename:
+        :param sqlList:
+        :return:
+        """
+        dbMsg = None
+        cur = self.get_new_cursor()
+        if self.conn:
+            try:
+                self.logger.info(sql)
+                cur.execute(sql)
+                while True:
+                    row = cur.fetchone()
+                    if row is None:
+                        return None
+                    else:
+                        yield row
+            except:
+                self.logger.error('*errSql:' + sql)
+                dbMsg = '*InsertError:' + traceback.format_exc()
+            if dbMsg:
+                self.logger.error(dbMsg)
+                return False
+        else:
+            return False
+        cur.close()
+
+    def ExeVACUUM(self):
+        """
+        清理空间
+        :return:
+        """
+        dbMsg = None
+        if self.conn:
+            try:
+                self.conn.execute("VACUUM")
+            except:
+                dbMsg = '*VACUUMError:' + traceback.format_exc()
+            if dbMsg:
+                self.logger.error(dbMsg)
+                return False
+        else:
+            return False
+        return True
+
+    def sqliteEscape(self, keyWord):
+        keyWord = keyWord.replace("'", "''")
+        keyWord = keyWord.replace("\\", "\\\\")
+        return keyWord
+
+    def close(self):
+        self.conn.close()

re_common/facade/use/mq_use_facade.py

@@ -0,0 +1,83 @@
+import logging
+import traceback
+
+from re_common.baselibrary.utils.basepika import BasePika
+from retry import retry
+
+logging_logger = logging.getLogger(__name__)
+
+
+class UseMq(object):
+
+    def __init__(self, queue, qos=1):
+        self.queue = queue
+        self.qos = qos
+        self.basepika = BasePika()
+        self.basepika.set_default()
+        self.basepika.connect()
+        self.basepika.create_channel()
+        self.basepika.queue_declare(queue=queue, durable=True)
+        self.basepika.basic_qos(qos)
+        self.properties = self.basepika.get_properties()
+
+    def re_conn(self):
+        """
+        重新连接
+        :return:
+        """
+        self.basepika.connect()
+        self.basepika.create_channel()
+        self.basepika.queue_declare(queue=self.queue, durable=True)
+        self.basepika.basic_qos(self.qos)
+
+    @retry(delay=5, backoff=2, max_delay=60 * 3, logger=logging_logger)
+    def get_mq(self):
+        try:
+            if self.basepika.channel.is_closed:
+                logging_logger.info("重连中......")
+                self.re_conn()
+                logging_logger.info("重连完成......")
+            self.basepika.set_get_msg_callback(routing_key=self.queue, callback=self.callback, auto_ack=False)
+            self.basepika.start_get_msg()
+        except:
+            traceback.print_exc()
+            logging_logger.info("重连中......")
+            self.re_conn()
+
+    def callback(self, ch, method, properties, body):
+        # print(type(body))
+        # print(" [x] Received %r" % body)
+        # body = body.decode()
+        self.callback2(ch, method, properties, body)
+        if self.basepika.auto_ack is False:
+            self.basepika.basic_ack(ch, method)
+
+    def callback2(self, ch, method, properties, body):
+        pass
+
+    @retry(delay=5, backoff=2, max_delay=60 * 3, logger=logging_logger)
+    def send_mq(self, body, num=100):
+        try:
+            if self.basepika.get_queue_size(self.queue) < num:
+                self.basepika.easy_send_msg(routing_key=self.queue,
+                                            body=body,
+                                            properties=self.properties)
+                return True
+            else:
+                return False
+        except:
+            traceback.print_exc()
+            logging_logger.info("重连中......")
+            self.re_conn()
+            return False
+
+    def get_server_mq_num(self, num=100):
+        if self.basepika.get_queue_size(self.queue) < num:
+            return True
+        else:
+            return False
+
+    def easy_send_mq(self, body):
+        self.basepika.easy_send_msg(routing_key=self.queue,
+                                    body=body,
+                                    properties=self.properties)

re_common/facade/use/proxy_use_facade.py

@@ -0,0 +1,20 @@
+import time
+
+
+def set_school_list_proxy(self):
+    """
+    使用该函数条件, self为类的对象
+    代理在 proxyset set里
+    使用的是bshttpx这个变量
+    """
+    try:
+        proxy = self.proxyset.pop()
+        self.logger.info("proxy is:{},proxy size is: {}".format(proxy, len(self.proxyset)))
+        return self.bshttpx.creat_sn(proxy=proxy,
+                                     headers=self.headers,
+                                     verify=False)
+    except KeyError as e:
+        time.sleep(15)
+        if str(e) == "'pop from an empty set'":
+            self.proxyset = set(self.school_proxy_list)
+            return self.set_school_list_proxy()

re_common/libtest/base_dict_test.py

@@ -0,0 +1,19 @@
+from re_common.baselibrary import BaseDicts
+from re_common.baselibrary.utils.mfaker import MFaker
+
+
+def test_basedict_sortkeys():
+    fake = MFaker()
+    dicts = fake.create_data(MFaker.m_pydict, **fake.py_para())
+    print(dicts)
+    dicts2 = BaseDicts.sortkeys(dicts)
+    print(dicts2)
+    print(dicts)
+
+
+def test_basedict_sortvalues():
+    dicts = {'data': 'FDZqqOGNMyGJlNRoCsJd', 'participant': 'petersonadrienne@bennett.com',
+             'often': 'KnJSSDeSTPboiwjSdGwR', 'friend': '1639', 'above': '8144', 'in': '1614'}
+    dicts2 = BaseDicts.sortvalues(dicts)
+    print(dicts2)
+

re_common/libtest/baseavro_test.py

@@ -0,0 +1,13 @@
+from re_common.baselibrary.utils.baseavro import BaseAvro
+
+id_set = set()
+for line in BaseAvro().read_line_yeild(r"F:\fun2\avro"):
+    id_set.add(line["key"])
+
+print(len(id_set))
+lines = ""
+
+with open(r"F:\fun2\avro1.txt", 'w', encoding="utf-8") as f:
+    for id in id_set:
+        lines = id + "\n"
+        f.write(lines)

re_common/libtest/basefile_test.py

@@ -0,0 +1,14 @@
+from re_common.baselibrary.utils.basedir import BaseDir
+from re_common.baselibrary.utils.basefile import BaseFile
+
+# files_line_list = BaseFile.read_end_line(r"F:\db3\mysql_date\test\part-00000", 3)
+# for file_line in files_line_list:
+#     # file_line = str(file_line, encoding="utf-8")
+#     # file_line = file_line.decode(encoding="utf-8")
+#     print(file_line)
+
+for file in BaseDir.get_dir_all_files(r"F:\db3\mysql_date\data_dir"):
+    files_line_list = BaseFile.read_end_line(r"F:\db3\mysql_date\test\part-00000", 11000)
+    strs = "\n".join(files_line_list)
+    BaseFile.single_add_file(r"F:\db3\mysql_date\end\part-00000", strs + "\n")
+

re_common/libtest/basemssql_test.py

@@ -0,0 +1,77 @@
+###########################################
+# 同项目调用基础包
+import datetime
+import gzip
+import json
+import os
+import sys
+import time
+
+filepath = os.path.abspath(__file__)
+pathlist = filepath.split(os.sep)
+pathlist = pathlist[:-3]
+TopPath = os.sep.join(pathlist)
+sys.path.insert(0, TopPath)
+print(TopPath)
+############################################
+
+from re_common.baselibrary.utils.basemssql import BaseMsSql
+from re_common.baselibrary.utils.basefile import BaseFile
+from re_common.baselibrary.utils.basetime import BaseTime
+bt = BaseTime()
+
+host = "127.0.0.1"
+user = "sa"
+pwd = "xujiang1994323"
+db = "patData"
+charset = "utf8"
+
+basemssql = BaseMsSql(host, user, pwd, db, charset, as_dict=True)
+basemssql.conn()
+basemssql.exec_select_query("select * from [dbo].[New_pattzb]")
+outPathFile = r"F:\db3\patnetjson_big\jss_patent.big_json.gz"
+i = 0
+size = 10000
+count = 2000000
+outPathFile = BaseFile.get_new_filename(outPathFile, sign=".")
+start = time.time()
+end = False
+while True:
+    with gzip.open(outPathFile, 'wb') as f:
+        while True:
+            a = basemssql.cur.fetchmany(size=size)
+            if not a:
+                print(i)
+                print(int(time.time() - start))
+                print("break")
+                end = True
+                break
+            for row in a:
+                dicts = {}
+                for k,v in row.items():
+                    # print(k)
+                    try:
+                        v = v.encode('latin-1').decode('gbk')
+                    except:
+                        pass
+                    if isinstance(v, datetime.datetime):
+                        v = bt.datetime_to_string(v, "%Y-%m-%d %H:%M:%S")
+                    dicts[k] = v
+                    # print(type(v))
+                # print(dicts)
+                line = json.dumps(dicts,ensure_ascii=False)+"\n"
+                # print(line)
+                lines = line.encode(encoding="utf8")
+                f.write(lines)
+            i = i + size
+            print(i)
+            print(int(time.time() - start))
+            if i >= count:
+                outPathFile = BaseFile.get_new_filename(outPathFile, sign=".")
+                i=0
+                break
+    if end:
+        break
+
+
+

re_common/libtest/baseodbc_test.py

@@ -0,0 +1,8 @@
+from re_common.baselibrary.utils.baseodbc import BaseODBC
+
+baseodbc = BaseODBC(r"C:\Users\xuzhu\Desktop\DB_20200701_GB.mdb")
+# baseodbc = BaseODBC(r"D:\download\cnki_qk\download\get_journal\mdb\cnki期刊信息_20200315.mdb")
+baseodbc.get_cur()
+sql = "select * from `CN`"
+for row in baseodbc.select(sql):
+    print(row)

re_common/libtest/basepandas_test.py

@@ -0,0 +1,38 @@
+import time
+
+from pandas._libs.tslibs.timestamps import Timestamp
+
+
+def test_dataform_to_numpy():
+    import pandas as pd
+    import numpy as np
+    dicts = {'A': 1.,
+             'B': pd.Timestamp('20130102'),
+             'C': pd.Series(1, index=list(range(4)), dtype='float32'),
+             'D': np.array([3] * 4, dtype='int32'),
+             'E': pd.Categorical(["test", "train", "test", "train"]),
+             'F': 'foo'}
+    from re_common.baselibrary.utils.basepandas import BasePandas
+    bp = BasePandas()
+    startTime = time.time()
+    df = bp.dicts_to_dataform(dicts)
+    # 数据类型一致时速度会很快
+    print(bp.dataform_to_numpy(df))
+    endTime = time.time()
+    print(endTime - startTime)
+    dicts = {'A': [1.0, Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo'],
+             'B': [Timestamp('2013-01-02 00:00:00'), Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo'],
+             'C': [1.0, Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo'],
+             'D': [3, Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo'],
+             'E': ['test', Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo'],
+             'F': ['foo', Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo']}
+
+
+    startTime = time.time()
+    df = bp.dicts_to_dataform(dicts)
+    # 数据类型一致时速度会很快
+    print(bp.dataform_to_numpy(df))
+    endTime = time.time()
+    print(endTime - startTime)
+
+test_dataform_to_numpy()

re_common/libtest/get_attr_test/get_attr_test_settings.py

@@ -0,0 +1,14 @@
+import os
+import sys
+
+filepath = os.path.abspath(__file__)
+pathlist = filepath.split(os.sep)
+pathlist = pathlist[:-4]
+TopPath = os.sep.join(pathlist)
+sys.path.insert(0, TopPath)
+print(TopPath)
+
+from re_common.libtest.get_attr_test import settings
+from re_common.baselibrary.tools.get_attr import get_attrs
+
+print(get_attrs(settings))

re_common/libtest/get_attr_test/settings.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+
+
+
+"""
+CREATE YOUR DEFAULT_CONFIG !
+
+Some configuration:
+        CONCURRENT_REQUESTS     并发数量
+        RETRIES                 重试次数
+        DOWNLOAD_DELAY          下载延时
+        RETRY_DELAY             重试延时
+        DOWNLOAD_TIMEOUT        超时限制
+        USER_AGENT              用户代理
+        LOG_FILE                日志路径
+        LOG_LEVEL               日志等级
+"""
+
+
+CONCURRENT_REQUESTS = 20
+
+
+
+
+
+MIDDLEWARE = [
+'middlewares.middleware',
+            ]
+
+
+# PIPELINES = []
+
+
+
+
+DEFAULT_REQUEST_CONFIG = {
+    "RETRIES": 0,
+    "DOWNLOAD_DELAY": 0,
+    "RETRY_DELAY": 0,
+    "DOWNLOAD_TIMEOUT": 10,
+}
+
+
+# '''生成日志文件'''
+# LOG_FILE = './asyncpy.log'
+# LOG_LEVEL = 'DEBUG'
+
+
+
+# CLOSESPIDER_TIMEOUT = 10
+
+
+
+
+USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3100.0 Safari/537.36"

re_common/libtest/idencode_test.py

@@ -0,0 +1,54 @@
+
+
+from re_common.vip.baseencodeid import BaseLngid
+
+
+def encode_id():
+    """"
+    由 sub_db_id 和 rawid 得到 lngid。
+    case_insensitive: 标识源网站的 rawid 是否区分大小写
+    """
+
+    sub_db_id = "1"
+    rawid = "A44cbe4375b431741"
+    b = BaseLngid()
+    print("********区分大小写************")
+    lngid = b.GetLngid(sub_db_id,
+                       rawid,
+                       case_insensitive=True)
+    print(lngid)
+    print("********不区分大小写************")
+    lngid = b.GetLngid(sub_db_id,
+                       rawid,
+                       case_insensitive=False)
+    print(lngid)
+
+def decode_id():
+    """
+    limited_id 是由 lngid去除sub_db_id后得到的字符串
+    limited_id超过20长度时，为不可逆的
+    :return:
+    """
+    limited_id_big = "F65F41FFCF049A3A21516C5CDFE40A22"
+    limited_id_small = "HLXMH9XNH5XMJC04HHXMJDG2J919Y"
+    limited_id_err = "123456" * 20
+    b = BaseLngid()
+    print("********区分大小写************")
+    rawid = b.GetRawid(limited_id_big,
+                       case_insensitive=True)
+    print(rawid)
+    print("********不区分大小写************")
+    rawid = b.GetRawid(limited_id_small,
+                       case_insensitive=False)
+    print(rawid)
+    print("*******limited_id超过20长度时*******")
+    rawid = b.GetRawid(limited_id_err,
+                       case_insensitive=False)
+
+
+
+
+
+if __name__ == '__main__':
+    # encode_id()
+    decode_id()