re-common 2.0.0__py3-none-any.whl → 2.0.1__py3-none-any.whl

re_common/facade/sqlite3facade.py

@@ -1,257 +0,0 @@
-# db3数据的连接返回一个连接对象
-import sys
-import traceback
-
-from re_common.baselibrary import MLogger
-from re_common.baselibrary.baseabs import BaseAbs
-from re_common.baselibrary.database.mbuilder import Sqlite3Builder
-
-
-class Sqlite3Utiles(object):
-    def __init__(self, logger=None):
-        self.sqllite3 = None
-        self.conn = None
-        self.cursor = None
-
-        self._logger = logger
-
-    @property
-    def logger(self):
-        if self._logger is None:
-            return MLogger().streamlogger
-        return self._logger
-
-    @logger.setter
-    def logger(self, value):
-        assert isinstance(value, MLogger)
-        self._logger = value
-
-    def get_new_cursor(self):
-        """
-        获取一个新的游标
-        :return:
-        """
-        # 检查db的存在及是否断掉 不知道为什么 不可以加括号 但编辑器会警告
-        return self.conn.cursor()
-
-    def Sqlite3DBConnectFromFilePath(self, dbfilepath, encoding="gbk", timeout=60):
-        """
-        通过直接文件连接 我使用Hadoop导下来的db3是gbk编码的
-        如果为utf-8编码请改为utf-8
-        :param sec:  section
-        :return:
-        """
-        sqllite3 = BaseAbs.get_sql_factory().sqlite_factory()
-        # 传入的是配置文件和section选项 dbpath为配置文件路径
-        sqlite3_moudle = Sqlite3Builder().build_file_path(dbfilepath).build_timeout(timeout).get_moudle()
-        sqllite3.link(sqlite3_moudle)
-        # 设置txt的编码格式  hadoop 上的db3为gbk 默认为utf8
-        sqllite3.set_encoding(encoding=encoding)
-        # 返回一个连接
-        self.sqllite3 = sqllite3
-        self.conn = self.sqllite3.db
-        return self
-
-    def Sqlite3DBConnectFromConfig(self, cobnfigfilepath, sec, encoding="gbk"):
-        """
-        通过配置文件连接
-        :param sec:  section
-        :return:
-        """
-        sqllite3 = BaseAbs.get_sql_factory().sqlite_factory()
-        # 传入的是配置文件和section选项 dbpath为配置文件路径
-        sqlite3_moudle = Sqlite3Builder(cobnfigfilepath, sec).build_all().get_moudle()
-        sqllite3.link(sqlite3_moudle)
-        # 设置txt的编码格式  hadoop 上的db3为gbk 默认为utf8
-        sqllite3.set_encoding(encoding=encoding)
-        # 返回一个连接
-        self.sqllite3 = sqllite3
-        self.conn = self.sqllite3.db
-        return self
-
-    def ExeSqlliteList(self, sqlList, errExit=True):
-        """
-        该函数和上面一样执行一个sql列表且不返回结果
-        属于插入和更新类函数 但该函数针对db3
-        :param sqllitename:
-        :param sqlList:
-        :return:
-        """
-        dbMsg = None
-        cur = self.get_new_cursor()
-        if self.conn:
-            count = 0
-            for sql in sqlList:
-                count += 1
-                try:
-                    self.logger.info("{} 执行sql数量:{}".format(sql, str(count)))
-                    cur.execute(sql)
-                except:
-                    self.logger.error('*errSql:' + sql)
-                    dbMsg = '*InsertError:' + traceback.format_exc()
-                    if errExit:
-                        self.logger.error(dbMsg)
-                        sys.exit()
-                if dbMsg:
-                    self.logger.error(dbMsg)
-                    continue
-        self.conn.commit()
-        cur.close()
-
-    def ExeSqlliteMany(self, sql, itermany, errExit=True):
-        dbMsg = None
-        cur = self.get_new_cursor()
-        if self.conn:
-            try:
-                self.logger.info("{}\n{}".format(sql, str(itermany)))
-                cur.executemany(sql, itermany)
-            except:
-                self.logger.error('*errSql:' + sql)
-                dbMsg = '*InsertError:' + traceback.format_exc()
-                if errExit:
-                    self.logger.error(dbMsg)
-                    sys.exit()
-            if dbMsg:
-                self.logger.error(dbMsg)
-        self.conn.commit()
-        cur.close()
-
-    def ExeSqlliteSql(self, sql):
-        """
-        该函数和上面一样执行一个sql列表且不返回结果
-        属于插入和更新类函数 但该函数针对db3
-        :param sqllitename:
-        :param sqlList:
-        :return:
-        """
-        dbMsg = None
-        cur = self.get_new_cursor()
-        if self.conn:
-            try:
-                self.logger.info(sql)
-                cur.execute(sql)
-                self.conn.commit()
-            except:
-                self.logger.error('*errSql:' + sql)
-                dbMsg = '*InsertError:' + traceback.format_exc()
-            if dbMsg:
-                self.logger.error(dbMsg)
-                return False
-        else:
-            return False
-        cur.close()
-        return True
-
-    def SelectFromSqlliteFetchall(self, sql):
-        """
-        该函数和上面一样执行一个sql列表且不返回结果
-        属于插入和更新类函数 但该函数针对db3
-        :param sqllitename:
-        :param sqlList:
-        :return:
-        """
-        cur = self.get_new_cursor()
-        if self.conn:
-            try:
-                self.logger.info(sql)
-                cur.execute(sql)
-                rows = cur.fetchall()
-                return rows
-            except:
-                self.logger.error('*errSql:' + sql)
-                dbMsg = '*InsertError:' + traceback.format_exc()
-            if dbMsg:
-                self.logger.error(dbMsg)
-                return False
-        else:
-            return False
-        cur.close()
-
-    def SelectFromSqlliteFetchall_dicts(self, sql):
-        """
-        该函数和上面一样执行一个sql列表且不返回结果
-        属于插入和更新类函数 但该函数针对db3
-        :param sqllitename:
-        :param sqlList:
-        :return:
-        """
-
-        def dict_factory(cursor, row):
-            d = {}
-            for idx, col in enumerate(cursor.description):
-                d[col[0]] = row[idx]
-            return d
-
-        self.conn.row_factory = dict_factory
-        cur = self.get_new_cursor()
-        if self.conn:
-            try:
-                self.logger.info(sql)
-                cur.execute(sql)
-                rows = cur.fetchall()
-                return rows
-            except:
-                self.logger.error('*errSql:' + sql)
-                dbMsg = '*InsertError:' + traceback.format_exc()
-            if dbMsg:
-                self.logger.error(dbMsg)
-                return False
-        else:
-            return False
-        cur.close()
-
-    def SelectFromSqlliteFetchOne(self, sql):
-        """
-        该函数和上面一样执行一个sql列表且不返回结果
-        属于插入和更新类函数 但该函数针对db3
-        :param sqllitename:
-        :param sqlList:
-        :return:
-        """
-        dbMsg = None
-        cur = self.get_new_cursor()
-        if self.conn:
-            try:
-                self.logger.info(sql)
-                cur.execute(sql)
-                while True:
-                    row = cur.fetchone()
-                    if row is None:
-                        return None
-                    else:
-                        yield row
-            except:
-                self.logger.error('*errSql:' + sql)
-                dbMsg = '*InsertError:' + traceback.format_exc()
-            if dbMsg:
-                self.logger.error(dbMsg)
-                return False
-        else:
-            return False
-        cur.close()
-
-    def ExeVACUUM(self):
-        """
-        清理空间
-        :return:
-        """
-        dbMsg = None
-        if self.conn:
-            try:
-                self.conn.execute("VACUUM")
-            except:
-                dbMsg = '*VACUUMError:' + traceback.format_exc()
-            if dbMsg:
-                self.logger.error(dbMsg)
-                return False
-        else:
-            return False
-        return True
-
-    def sqliteEscape(self, keyWord):
-        keyWord = keyWord.replace("'", "''")
-        keyWord = keyWord.replace("\\", "\\\\")
-        return keyWord
-
-    def close(self):
-        self.conn.close()

re_common/facade/use/mq_use_facade.py

@@ -1,83 +0,0 @@
-import logging
-import traceback
-
-from re_common.baselibrary.utils.basepika import BasePika
-from retry import retry
-
-logging_logger = logging.getLogger(__name__)
-
-
-class UseMq(object):
-
-    def __init__(self, queue, qos=1):
-        self.queue = queue
-        self.qos = qos
-        self.basepika = BasePika()
-        self.basepika.set_default()
-        self.basepika.connect()
-        self.basepika.create_channel()
-        self.basepika.queue_declare(queue=queue, durable=True)
-        self.basepika.basic_qos(qos)
-        self.properties = self.basepika.get_properties()
-
-    def re_conn(self):
-        """
-        重新连接
-        :return:
-        """
-        self.basepika.connect()
-        self.basepika.create_channel()
-        self.basepika.queue_declare(queue=self.queue, durable=True)
-        self.basepika.basic_qos(self.qos)
-
-    @retry(delay=5, backoff=2, max_delay=60 * 3, logger=logging_logger)
-    def get_mq(self):
-        try:
-            if self.basepika.channel.is_closed:
-                logging_logger.info("重连中......")
-                self.re_conn()
-                logging_logger.info("重连完成......")
-            self.basepika.set_get_msg_callback(routing_key=self.queue, callback=self.callback, auto_ack=False)
-            self.basepika.start_get_msg()
-        except:
-            traceback.print_exc()
-            logging_logger.info("重连中......")
-            self.re_conn()
-
-    def callback(self, ch, method, properties, body):
-        # print(type(body))
-        # print(" [x] Received %r" % body)
-        # body = body.decode()
-        self.callback2(ch, method, properties, body)
-        if self.basepika.auto_ack is False:
-            self.basepika.basic_ack(ch, method)
-
-    def callback2(self, ch, method, properties, body):
-        pass
-
-    @retry(delay=5, backoff=2, max_delay=60 * 3, logger=logging_logger)
-    def send_mq(self, body, num=100):
-        try:
-            if self.basepika.get_queue_size(self.queue) < num:
-                self.basepika.easy_send_msg(routing_key=self.queue,
-                                            body=body,
-                                            properties=self.properties)
-                return True
-            else:
-                return False
-        except:
-            traceback.print_exc()
-            logging_logger.info("重连中......")
-            self.re_conn()
-            return False
-
-    def get_server_mq_num(self, num=100):
-        if self.basepika.get_queue_size(self.queue) < num:
-            return True
-        else:
-            return False
-
-    def easy_send_mq(self, body):
-        self.basepika.easy_send_msg(routing_key=self.queue,
-                                    body=body,
-                                    properties=self.properties)

re_common/facade/use/proxy_use_facade.py

@@ -1,20 +0,0 @@
-import time
-
-
-def set_school_list_proxy(self):
-    """
-    使用该函数条件, self为类的对象
-    代理在 proxyset set里
-    使用的是bshttpx这个变量
-    """
-    try:
-        proxy = self.proxyset.pop()
-        self.logger.info("proxy is:{},proxy size is: {}".format(proxy, len(self.proxyset)))
-        return self.bshttpx.creat_sn(proxy=proxy,
-                                     headers=self.headers,
-                                     verify=False)
-    except KeyError as e:
-        time.sleep(15)
-        if str(e) == "'pop from an empty set'":
-            self.proxyset = set(self.school_proxy_list)
-            return self.set_school_list_proxy()

re_common/libtest/base_dict_test.py

@@ -1,19 +0,0 @@
-from re_common.baselibrary import BaseDicts
-from re_common.baselibrary.utils.mfaker import MFaker
-
-
-def test_basedict_sortkeys():
-    fake = MFaker()
-    dicts = fake.create_data(MFaker.m_pydict, **fake.py_para())
-    print(dicts)
-    dicts2 = BaseDicts.sortkeys(dicts)
-    print(dicts2)
-    print(dicts)
-
-
-def test_basedict_sortvalues():
-    dicts = {'data': 'FDZqqOGNMyGJlNRoCsJd', 'participant': 'petersonadrienne@bennett.com',
-             'often': 'KnJSSDeSTPboiwjSdGwR', 'friend': '1639', 'above': '8144', 'in': '1614'}
-    dicts2 = BaseDicts.sortvalues(dicts)
-    print(dicts2)
-

re_common/libtest/baseavro_test.py

@@ -1,13 +0,0 @@
-from re_common.baselibrary.utils.baseavro import BaseAvro
-
-id_set = set()
-for line in BaseAvro().read_line_yeild(r"F:\fun2\avro"):
-    id_set.add(line["key"])
-
-print(len(id_set))
-lines = ""
-
-with open(r"F:\fun2\avro1.txt", 'w', encoding="utf-8") as f:
-    for id in id_set:
-        lines = id + "\n"
-        f.write(lines)

re_common/libtest/basefile_test.py

@@ -1,14 +0,0 @@
-from re_common.baselibrary.utils.basedir import BaseDir
-from re_common.baselibrary.utils.basefile import BaseFile
-
-# files_line_list = BaseFile.read_end_line(r"F:\db3\mysql_date\test\part-00000", 3)
-# for file_line in files_line_list:
-#     # file_line = str(file_line, encoding="utf-8")
-#     # file_line = file_line.decode(encoding="utf-8")
-#     print(file_line)
-
-for file in BaseDir.get_dir_all_files(r"F:\db3\mysql_date\data_dir"):
-    files_line_list = BaseFile.read_end_line(r"F:\db3\mysql_date\test\part-00000", 11000)
-    strs = "\n".join(files_line_list)
-    BaseFile.single_add_file(r"F:\db3\mysql_date\end\part-00000", strs + "\n")
-

re_common/libtest/basemssql_test.py

@@ -1,77 +0,0 @@
-###########################################
-# 同项目调用基础包
-import datetime
-import gzip
-import json
-import os
-import sys
-import time
-
-filepath = os.path.abspath(__file__)
-pathlist = filepath.split(os.sep)
-pathlist = pathlist[:-3]
-TopPath = os.sep.join(pathlist)
-sys.path.insert(0, TopPath)
-print(TopPath)
-############################################
-
-from re_common.baselibrary.utils.basemssql import BaseMsSql
-from re_common.baselibrary.utils.basefile import BaseFile
-from re_common.baselibrary.utils.basetime import BaseTime
-bt = BaseTime()
-
-host = "127.0.0.1"
-user = "sa"
-pwd = "xujiang1994323"
-db = "patData"
-charset = "utf8"
-
-basemssql = BaseMsSql(host, user, pwd, db, charset, as_dict=True)
-basemssql.conn()
-basemssql.exec_select_query("select * from [dbo].[New_pattzb]")
-outPathFile = r"F:\db3\patnetjson_big\jss_patent.big_json.gz"
-i = 0
-size = 10000
-count = 2000000
-outPathFile = BaseFile.get_new_filename(outPathFile, sign=".")
-start = time.time()
-end = False
-while True:
-    with gzip.open(outPathFile, 'wb') as f:
-        while True:
-            a = basemssql.cur.fetchmany(size=size)
-            if not a:
-                print(i)
-                print(int(time.time() - start))
-                print("break")
-                end = True
-                break
-            for row in a:
-                dicts = {}
-                for k,v in row.items():
-                    # print(k)
-                    try:
-                        v = v.encode('latin-1').decode('gbk')
-                    except:
-                        pass
-                    if isinstance(v, datetime.datetime):
-                        v = bt.datetime_to_string(v, "%Y-%m-%d %H:%M:%S")
-                    dicts[k] = v
-                    # print(type(v))
-                # print(dicts)
-                line = json.dumps(dicts,ensure_ascii=False)+"\n"
-                # print(line)
-                lines = line.encode(encoding="utf8")
-                f.write(lines)
-            i = i + size
-            print(i)
-            print(int(time.time() - start))
-            if i >= count:
-                outPathFile = BaseFile.get_new_filename(outPathFile, sign=".")
-                i=0
-                break
-    if end:
-        break
-
-
-

re_common/libtest/baseodbc_test.py

@@ -1,8 +0,0 @@
-from re_common.baselibrary.utils.baseodbc import BaseODBC
-
-baseodbc = BaseODBC(r"C:\Users\xuzhu\Desktop\DB_20200701_GB.mdb")
-# baseodbc = BaseODBC(r"D:\download\cnki_qk\download\get_journal\mdb\cnki期刊信息_20200315.mdb")
-baseodbc.get_cur()
-sql = "select * from `CN`"
-for row in baseodbc.select(sql):
-    print(row)

re_common/libtest/basepandas_test.py

@@ -1,38 +0,0 @@
-import time
-
-from pandas._libs.tslibs.timestamps import Timestamp
-
-
-def test_dataform_to_numpy():
-    import pandas as pd
-    import numpy as np
-    dicts = {'A': 1.,
-             'B': pd.Timestamp('20130102'),
-             'C': pd.Series(1, index=list(range(4)), dtype='float32'),
-             'D': np.array([3] * 4, dtype='int32'),
-             'E': pd.Categorical(["test", "train", "test", "train"]),
-             'F': 'foo'}
-    from re_common.baselibrary.utils.basepandas import BasePandas
-    bp = BasePandas()
-    startTime = time.time()
-    df = bp.dicts_to_dataform(dicts)
-    # 数据类型一致时速度会很快
-    print(bp.dataform_to_numpy(df))
-    endTime = time.time()
-    print(endTime - startTime)
-    dicts = {'A': [1.0, Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo'],
-             'B': [Timestamp('2013-01-02 00:00:00'), Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo'],
-             'C': [1.0, Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo'],
-             'D': [3, Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo'],
-             'E': ['test', Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo'],
-             'F': ['foo', Timestamp('2013-01-02 00:00:00'), 1.0, 3, 'test', 'foo']}
-
-
-    startTime = time.time()
-    df = bp.dicts_to_dataform(dicts)
-    # 数据类型一致时速度会很快
-    print(bp.dataform_to_numpy(df))
-    endTime = time.time()
-    print(endTime - startTime)
-
-test_dataform_to_numpy()

re_common/libtest/get_attr_test/get_attr_test_settings.py

@@ -1,14 +0,0 @@
-import os
-import sys
-
-filepath = os.path.abspath(__file__)
-pathlist = filepath.split(os.sep)
-pathlist = pathlist[:-4]
-TopPath = os.sep.join(pathlist)
-sys.path.insert(0, TopPath)
-print(TopPath)
-
-from re_common.libtest.get_attr_test import settings
-from re_common.baselibrary.tools.get_attr import get_attrs
-
-print(get_attrs(settings))

re_common/libtest/get_attr_test/settings.py

@@ -1,55 +0,0 @@
-# -*- coding: utf-8 -*-
-
-
-
-"""
-CREATE YOUR DEFAULT_CONFIG !
-
-Some configuration:
-        CONCURRENT_REQUESTS     并发数量
-        RETRIES                 重试次数
-        DOWNLOAD_DELAY          下载延时
-        RETRY_DELAY             重试延时
-        DOWNLOAD_TIMEOUT        超时限制
-        USER_AGENT              用户代理
-        LOG_FILE                日志路径
-        LOG_LEVEL               日志等级
-"""
-
-
-CONCURRENT_REQUESTS = 20
-
-
-
-
-
-MIDDLEWARE = [
-'middlewares.middleware',
-            ]
-
-
-# PIPELINES = []
-
-
-
-
-DEFAULT_REQUEST_CONFIG = {
-    "RETRIES": 0,
-    "DOWNLOAD_DELAY": 0,
-    "RETRY_DELAY": 0,
-    "DOWNLOAD_TIMEOUT": 10,
-}
-
-
-# '''生成日志文件'''
-# LOG_FILE = './asyncpy.log'
-# LOG_LEVEL = 'DEBUG'
-
-
-
-# CLOSESPIDER_TIMEOUT = 10
-
-
-
-
-USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3100.0 Safari/537.36"

re_common/libtest/idencode_test.py

@@ -1,54 +0,0 @@
-
-
-from re_common.vip.baseencodeid import BaseLngid
-
-
-def encode_id():
-    """"
-    由 sub_db_id 和 rawid 得到 lngid。
-    case_insensitive: 标识源网站的 rawid 是否区分大小写
-    """
-
-    sub_db_id = "1"
-    rawid = "A44cbe4375b431741"
-    b = BaseLngid()
-    print("********区分大小写************")
-    lngid = b.GetLngid(sub_db_id,
-                       rawid,
-                       case_insensitive=True)
-    print(lngid)
-    print("********不区分大小写************")
-    lngid = b.GetLngid(sub_db_id,
-                       rawid,
-                       case_insensitive=False)
-    print(lngid)
-
-def decode_id():
-    """
-    limited_id 是由 lngid去除sub_db_id后得到的字符串
-    limited_id超过20长度时，为不可逆的
-    :return:
-    """
-    limited_id_big = "F65F41FFCF049A3A21516C5CDFE40A22"
-    limited_id_small = "HLXMH9XNH5XMJC04HHXMJDG2J919Y"
-    limited_id_err = "123456" * 20
-    b = BaseLngid()
-    print("********区分大小写************")
-    rawid = b.GetRawid(limited_id_big,
-                       case_insensitive=True)
-    print(rawid)
-    print("********不区分大小写************")
-    rawid = b.GetRawid(limited_id_small,
-                       case_insensitive=False)
-    print(rawid)
-    print("*******limited_id超过20长度时*******")
-    rawid = b.GetRawid(limited_id_err,
-                       case_insensitive=False)
-
-
-
-
-
-if __name__ == '__main__':
-    # encode_id()
-    decode_id()