nlpertools 1.0.4__py3-none-any.whl → 1.0.6.dev0__py3-none-any.whl

nlpertools/utils/log_util.py

@@ -0,0 +1,20 @@
+#!/usr/bin/python3.8
+# -*- coding: utf-8 -*-
+# @Author  : youshu.Ji
+"""
+# 该项目暂时没有日志输出
+import codecs
+import logging.config
+
+import nlpertools
+import yaml
+
+nlpertools.j_mkdir("logs")
+
+with codecs.open('log_config.yml', 'r', 'utf-8') as stream:
+    config = yaml.load(stream, Loader=yaml.FullLoader)
+
+# logging.basicConfig(level=logging.INFO)
+logging.config.dictConfig(config)
+logger = logging.getLogger()
+"""

nlpertools/utils/package.py

@@ -0,0 +1,89 @@
+#!/usr/bin/python3.8
+# -*- coding: utf-8 -*-
+# @Author  : youshu.Ji
+import importlib
+from importlib import import_module
+
+
+def try_import(name, package):
+    try:
+        return import_module(name, package=package)
+    except:
+        pass
+        # print("import {} failed".format(name))
+    finally:
+        pass
+
+
+def lazy_import(importer_name, to_import):
+    """
+    Example from net
+    author: unknown
+    this function is not used
+    """
+    """Return the importing module and a callable for lazy importing.
+
+    The module named by importer_name represents the module performing the
+    import to help facilitate resolving relative imports.
+
+    to_import is an iterable of the modules to be potentially imported (absolute
+    or relative). The `as` form of importing is also supported,
+    e.g. `pkg.mod as spam`.
+
+    This function returns a tuple of two items. The first is the importer
+    module for easy reference within itself. The second item is a callable to be
+    set to `__getattr__`.
+    """
+    module = importlib.import_module(importer_name)
+    import_mapping = {}
+    for name in to_import:
+        importing, _, binding = name.partition(' as ')
+        if not binding:
+            _, _, binding = importing.rpartition('.')
+        import_mapping[binding] = importing
+
+    def __getattr__(name):
+        if name not in import_mapping:
+            message = f'module {importer_name!r} has no attribute {name!r}'
+            raise AttributeError(message)
+        importing = import_mapping[name]
+        # imortlib.import_module() implicitly sets submodules on this module as
+        # appropriate for direct imports.
+        imported = importlib.import_module(importing,
+                                           module.__spec__.parent)
+        setattr(module, name, imported)
+        return imported
+
+    return module, __getattr__
+
+
+# jieba = try_import("jieba", None)
+# sns = try_import("seaborn", None)
+# torch = try_import("torch", None)
+# nn = try_import("torch.nn", None)
+# BertTokenizer = try_import("transformers", "BertTokenizer")
+# BertForMaskedLM = try_import("transformers", "BertForMaskedLM")
+# Elasticsearch = try_import("elasticsearch", "Elasticsearch")
+# pd = try_import("pandas", None)
+# xgb = try_import("xgboost", None)
+
+aioredis = try_import("aioredis", None)
+pymysql = try_import("pymysql", None)
+zhconv = try_import("zhconv", None)
+KafkaProducer = try_import("kafka", "KafkaProducer")
+KafkaConsumer = try_import("kafka", "KafkaConsumer")
+np = try_import("numpy", None)
+plt = try_import("matplotlib", "pyplot")
+WordNetLemmatizer = try_import("nltk.stem", "WordNetLemmatizer")
+metrics = try_import("sklearn", "metrics")
+requests = try_import("requests", None)
+pq = try_import("pyquery", None)
+CountVectorizer = try_import("sklearn.feature_extraction.text", "CountVectorizer")
+precision_recall_fscore_support = try_import("sklearn.metrics", "precision_recall_fscore_support")
+tqdm = try_import("tqdm", "tqdm")
+# TODO 自动导出langid和win32evtlogutil输出有bug
+langid = try_import("langid", None)
+win32evtlogutil = try_import("win32evtlogutil", None)
+TfidfTransformer = try_import("sklearn.feature_extraction.text", "TfidfTransformer")
+yaml = try_import("yaml", None)
+omegaconf = try_import("omegaconf", None)

nlpertools/utils/package_v1.py

@@ -0,0 +1,94 @@
+#!/usr/bin/python3.8
+# -*- coding: utf-8 -*-
+# @Author  : youshu.Ji
+import importlib
+from importlib import import_module
+from importlib.util import LazyLoader
+from .lazy import lazy_module
+
+EXCLUDE_LAZYIMPORT = {"torch", "torch.nn", "numpy"}
+
+
+def try_import(name, package):
+    try:
+        if package:
+            # print("import {} success".format(name))
+            return lazy_module("{}.{}".format(package, name))
+        else:
+            if name in EXCLUDE_LAZYIMPORT:
+                return import_module(name, package=package)
+            return lazy_module(name)
+        # return import_module(name, package=package)
+    except:
+        pass
+        print("import {} failed".format(name))
+    finally:
+        pass
+
+
+def lazy_import(importer_name, to_import):
+    """Return the importing module and a callable for lazy importing.
+
+    The module named by importer_name represents the module performing the
+    import to help facilitate resolving relative imports.
+
+    to_import is an iterable of the modules to be potentially imported (absolute
+    or relative). The `as` form of importing is also supported,
+    e.g. `pkg.mod as spam`.
+
+    This function returns a tuple of two items. The first is the importer
+    module for easy reference within itself. The second item is a callable to be
+    set to `__getattr__`.
+    """
+    module = importlib.import_module(importer_name)
+    import_mapping = {}
+    for name in to_import:
+        importing, _, binding = name.partition(' as ')
+        if not binding:
+            _, _, binding = importing.rpartition('.')
+        import_mapping[binding] = importing
+
+    def __getattr__(name):
+        if name not in import_mapping:
+            message = f'module {importer_name!r} has no attribute {name!r}'
+            raise AttributeError(message)
+        importing = import_mapping[name]
+        # imortlib.import_module() implicitly sets submodules on this module as
+        # appropriate for direct imports.
+        imported = importlib.import_module(importing,
+                                           module.__spec__.parent)
+        setattr(module, name, imported)
+        return imported
+
+    return module, __getattr__
+
+
+aioredis = try_import("aioredis", None)
+happybase = try_import("happybase", None)
+pd = try_import("pandas", None)
+pymysql = try_import("pymysql", None)
+Elasticsearch = try_import("elasticsearch", "Elasticsearch")
+KafkaProducer = try_import("kafka", "KafkaProducer")
+MongoClient = try_import("pymongo", "MongoClient")
+helpers = try_import("elasticsearch", "helpers")
+KafkaConsumer = try_import("kafka", "KafkaConsumer")
+np = try_import("numpy", None)
+sns = try_import("seaborn", None)
+torch = try_import("torch", None)
+nn = try_import("torch.nn", None)
+xgb = try_import("xgboost", None)
+plt = try_import("matplotlib", "pyplot")
+WordNetLemmatizer = try_import("nltk.stem", "WordNetLemmatizer")
+metrics = try_import("sklearn", "metrics")
+BertTokenizer = try_import("transformers", "BertTokenizer")
+BertForMaskedLM = try_import("transformers", "BertForMaskedLM")
+requests = try_import("requests", None)
+psutil = try_import("psutil", None)
+pq = try_import("pyquery", None)
+CountVectorizer = try_import("sklearn.feature_extraction.text", "CountVectorizer")
+precision_recall_fscore_support = try_import("sklearn.metrics", "precision_recall_fscore_support")
+tqdm = try_import("tqdm", "tqdm")
+langid = try_import("langid", None)
+# win32evtlogutil?
+TfidfTransformer = try_import("sklearn.feature_extraction.text", "TfidfTransformer")
+yaml = try_import("yaml", None)

nlpertools/utils/package_v2.py

@@ -0,0 +1,117 @@
+# !/usr/bin/python3.8
+# -*- coding: utf-8 -*-
+# @Author  : youshu.Ji
+import importlib
+from importlib import import_module
+import os
+
+
+def try_import(name, package):
+    try:
+        return import_module(name, package=package)
+    except:
+        pass
+        # print("import {} failed".format(name))
+    finally:
+        pass
+
+
+aioredis = None
+happybase = None
+pd = None
+pymysql = None
+Elasticsearch = None
+KafkaProducer = None
+MongoClient = None
+helpers = None
+KafkaConsumer = None
+np = None
+sns = None
+torch = None
+nn = None
+xgb = None
+plt = None
+WordNetLemmatizer = None
+metrics = None
+BertTokenizer = None
+BertForMaskedLM = None
+requests = None
+psutil = None
+pq = None
+CountVectorizer = None
+precision_recall_fscore_support = None
+tqdm = None
+langid = None
+win32evtlogutil = None
+TfidfTransformer = None
+yaml = None
+
+import_dict = {
+    "aioredis": ("aioredis", None),
+    "happybase": ("happybase", None),
+    "pd": ("pandas", None),
+    "pymysql": ("pymysql", None),
+    "Elasticsearch": ("elasticsearch", "Elasticsearch"),
+    "KafkaProducer": ("kafka", "KafkaProducer"),
+    "MongoClient": ("pymongo", "MongoClient"),
+    "helpers": ("elasticsearch", "helpers"),
+    "KafkaConsumer": ("kafka", "KafkaConsumer"),
+    "np": ("numpy", None),
+    "sns": ("seaborn", None),
+    "torch": ("torch", None),
+    "nn": ("torch.nn", None),
+    "xgb": ("xgboost", None),
+    "plt": ("matplotlib", "pyplot"),
+    "WordNetLemmatizer": ("nltk.stem", "WordNetLemmatizer"),
+    "metrics": ("sklearn", "metrics"),
+    "BertTokenizer": ("transformers", "BertTokenizer"),
+    "BertForMaskedLM": ("transformers", "BertForMaskedLM"),
+    "requests": ("requests", None),
+    "psutil": ("psutil", None),
+    "pq": ("pyquery", None),
+    "CountVectorizer": ("sklearn.feature_extraction.text", "CountVectorizer"),
+    "precision_recall_fscore_support": ("sklearn.metrics", "precision_recall_fscore_support"),
+    "tqdm": ("tqdm", "tqdm"),
+    "langid": ("langid", None),
+    "win32evtlogutil": ("win32evtlogutil", None),
+    "TfidfTransformer": ("sklearn.feature_extraction.text", "TfidfTransformer"),
+    "yaml": ("yaml", None)
+}
+if "nlpertools_helper" in os.environ.keys():
+    # TODO 该方法未经过测试
+    import_list = os.environ["nlpertools_helper"]
+
+    for k in import_list:
+        name, package = import_dict[k]
+        globals()[k] = try_import(name, package)
+else:
+    aioredis = try_import("aioredis", None)
+    happybase = try_import("happybase", None)
+    pd = try_import("pandas", None)
+    pymysql = try_import("pymysql", None)
+    Elasticsearch = try_import("elasticsearch", "Elasticsearch")
+    KafkaProducer = try_import("kafka", "KafkaProducer")
+    MongoClient = try_import("pymongo", "MongoClient")
+    helpers = try_import("elasticsearch", "helpers")
+    KafkaConsumer = try_import("kafka", "KafkaConsumer")
+    np = try_import("numpy", None)
+    sns = try_import("seaborn", None)
+    torch = try_import("torch", None)
+    nn = try_import("torch.nn", None)
+    xgb = try_import("xgboost", None)
+    plt = try_import("matplotlib", "pyplot")
+    WordNetLemmatizer = try_import("nltk.stem", "WordNetLemmatizer")
+    metrics = try_import("sklearn", "metrics")
+    BertTokenizer = try_import("transformers", "BertTokenizer")
+    BertForMaskedLM = try_import("transformers", "BertForMaskedLM")
+    requests = try_import("requests", None)
+    psutil = try_import("psutil", None)
+    pq = try_import("pyquery", None)
+    CountVectorizer = try_import("sklearn.feature_extraction.text", "CountVectorizer")
+    precision_recall_fscore_support = try_import("sklearn.metrics", "precision_recall_fscore_support")
+    tqdm = try_import("tqdm", "tqdm")
+    # TODO 自动导出langid和win32evtlogutil输出有bug
+    langid = try_import("langid", None)
+    win32evtlogutil = try_import("win32evtlogutil", None)
+    TfidfTransformer = try_import("sklearn.feature_extraction.text", "TfidfTransformer")
+    yaml = try_import("yaml", None)

nlpertools/utils_for_nlpertools.py

@@ -0,0 +1,93 @@
+import os
+import shutil
+from importlib import import_module
+
+from .io.dir import j_mkdir
+from .io.file import readtxt_list_all_strip, writetxt_w_list
+
+
+def try_import(name, package):
+    try:
+        return import_module(name, package=package)
+    except:
+        print("import {} failed".format(name))
+    finally:
+        pass
+
+
+def convert_import_to_try_import(from_path, to_path):
+    j_mkdir(to_path)
+    for root, dirs, files in os.walk(from_path):
+        for sub_dir in dirs:
+            j_mkdir(os.path.join(root.replace(from_path, to_path), sub_dir))
+        for file in files:
+            src = os.path.join(root, file)
+            dst = os.path.join(root.replace(from_path, to_path), file)
+            excluded_file = ["wrapper.py", "kmp.py", "__init__.py"]
+            if file.endswith(".py") and file != "utils_for_nlpertools.py" and file not in excluded_file:
+                raw_code = readtxt_list_all_strip(src)
+                start_idx, end_idx = 0, 0
+
+                for idx, each_line in enumerate(raw_code[:30]):
+                    each_line = each_line.lstrip("# ")
+                    if start_idx == 0 and (each_line.startswith("from") or each_line.startswith("import")):
+                        try:
+                            exec(each_line)
+                        except:
+                            start_idx = idx
+                    if start_idx != 0 and not each_line:
+                        end_idx = idx
+                        break
+                # print(file, start_idx, end_idx)
+                if start_idx != 0 and end_idx != 0:
+                    new_code = raw_code[:start_idx] + convert_import_string_to_import_list(
+                        "\n".join(raw_code[start_idx:end_idx])) + raw_code[end_idx:]
+                else:
+                    new_code = raw_code
+                writetxt_w_list(new_code, dst)
+            else:
+                shutil.copy(src=src, dst=dst)
+    print("convert over")
+
+
+def get_import_info(text):
+    pass
+
+
+def convert_import_string_to_import_list(text):
+    """
+    该方法将 import 转变为 try import
+    """
+    models_to_import = []
+    import_list = text.split("\n")
+    for each in import_list:
+        print(each)
+        name, package, as_name = None, None, None
+        elements = each.split(" ")
+        for pre, cur in zip(elements, elements[1:]):
+            if cur.endswith(","):
+                cur = cur.rstrip(",")
+            # 为了实现from import 和 import统一，首先把package和name的含义反过来，后面再掉换
+            if pre == "import":
+                package = cur
+            if pre == "from":
+                name = cur
+            if pre == "as":
+                as_name = cur
+            if pre[-1] == ",":
+                # 针对 from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
+                # 将将前面部分和当前的组成新字段
+                prefix = each.split("import")[0]
+                import_list.append("{}import {}".format(prefix, cur))
+        if not as_name:
+            as_name = package.split(".")[-1]
+        if not name:
+            name, package = package, name
+        models_to_import.append((name, package, as_name))
+    # 打印
+    all_import_info = ["", "from utils_for_nlpertools import try_import", ""]
+    for name, package, as_name in models_to_import:
+        import_info = '{} = try_import("{}", {})'.format(as_name, name, '"{}"'.format(package) if package else package)
+        all_import_info.append(import_info)
+        print(import_info)
+    return all_import_info

nlpertools/vector_index_demo.py

@@ -0,0 +1,108 @@
+#!/usr/bin/python3.8
+# -*- coding: utf-8 -*-
+# @Author  : youshu.Ji
+import math
+
+import faiss
+import gensim
+import numpy as np
+import pandas as pd
+
+
+def build_index_use(vectors):
+    d = len(vectors[0])
+    nlist = 100
+    quantizer = faiss.IndexFlatL2(d)
+    index = faiss.IndexIVFFlat(quantizer, d, nlist, faiss.METRIC_L2)
+    index.train(vectors)
+    index.add(vectors)
+    return index
+
+
+def build_index(vectors, distances="L2", nprobe=10):
+    """ 建立 faiss 索引.
+
+    Args:
+        vectors(numpy.array): 向量矩阵，shape=(向量数, 向量维度)
+        distance(str): 度量距离，支持 L2、COS 和 INNER_PRODUCT.
+        nprobe(int): 向量搜索时需要搜索的聚类数.
+
+    Return: 返回 faiss 索引对象.
+
+    """
+    metric_type = None
+    if distances == "L2":
+        metric_type = faiss.METRIC_L2
+    elif distances in ("COS", "INNER_PRODUCT"):
+        metric_type = faiss.METRIC_INNER_PRODUCT
+    else:
+        raise NotImplementedError
+
+    index_pipes = []
+
+    if distances == "COS":
+        index_pipes.append("L2norm")
+
+    K = 4 * math.sqrt(vectors.shape[0])
+    use_ivf = False
+    if vectors.shape[0] >= 30 * K:
+        index_pipes.append(f"IVF{K}")
+        use_ivf = True
+
+    index_pipes.append("Flat")
+
+    index = faiss.index_factory(vectors.shape[1], ",".join(index_pipes),
+                                metric_type)
+
+    vectors = vectors.astype(np.float32)
+    if not index.is_trained:
+        index.train(vectors)
+
+    index.add(vectors)
+
+    # IVF 使用 reconstruct 时必须执行此函数
+    if use_ivf:
+        ivf_index = faiss.extract_index_ivf(index)
+        ivf_index.make_direct_map()
+        ivf_index.nprobe = nprobe
+
+    return index
+
+
+def read_index_from_file(filename):
+    """ 从向量文件中读取 faiss 向量对象. """
+    return faiss.read_index(filename)
+
+
+def write_index_to_file(index, filename):
+    """ 将 faiss 向量对象写入文件. """
+    faiss.write_index(index, filename)
+
+
+word2vec_path = "glove_vector_path"
+wv_from_text = gensim.models.KeyedVectors.load_word2vec_format(word2vec_path, binary=False, no_header=True)
+vectors = wv_from_text.vectors
+
+name_example = ["{}.jpg".format((i % 9) + 1) for i in range(len(vectors))]
+df = pd.DataFrame({
+    "name": name_example,
+    # "vector": str(vectors[0]),
+    # "text": list(wv_from_text.key_to_index.keys()),
+})
+test_index = build_index_use(vectors)
+write_index_to_file(test_index, "test.index")
+
+df.to_csv("test.csv", index=False)
+
+import gensim
+import hnswlib
+
+word2vec_path = "glove_vector_path"
+wv_from_text = gensim.models.KeyedVectors.load_word2vec_format(word2vec_path, binary=False, no_header=True)
+vectors = wv_from_text.vectors
+
+labels = [idx for idx, i in enumerate(vectors)]
+index = hnswlib.Index(space="l2", dim=len(vectors[0]))
+index.init_index(max_elements=len(vectors), ef_construction=200, M=16)
+index.add_items(vectors, labels)
+index.save_index("hnswlib.index")

nlpertools/wrapper.py

@@ -0,0 +1,161 @@
+#!/usr/bin/python3.8
+# -*- coding: utf-8 -*-
+# @Author  : youshu.Ji
+# 定义装饰器
+import logging
+import time
+from functools import wraps
+import asyncio
+
+def fn_async_timer(function):
+    """
+    针对异步函数的装饰器
+    """
+    @wraps(function)
+    async def function_timer(*args, **kwargs):
+        t0 = time.time()
+        result = await function(*args, **kwargs)
+        t1 = time.time()
+        print('[finished {func_name} in {time:.2f}s]'.format(func_name=function.__name__, time=t1 - t0))
+        return result
+
+    return function_timer
+
+
+def fn_timer(async_func=False, analyse=False):
+    """
+    >>> @fn_timer()
+    >>> def example():
+    >>>     time.sleep(2)
+    :param analyse:
+    :return:
+    """
+
+    def wrapper(func):
+        async def func_time_async(*args, **kwargs):
+            t0 = time.time()
+            result = await asyncio.create_task(func(*args, **kwargs))
+            t1 = time.time()
+            print('[finished {func_name} in {time:.2f}s]'.format(func_name=func.__name__, time=t1 - t0))
+            return result
+
+        def func_time(*args, **kwargs):
+            t0 = time.time()
+            result = func(*args, **kwargs)
+            t1 = time.time()
+            print('[finished {func_name} in {time:.2f}s]'.format(func_name=func.__name__, time=t1 - t0))
+            return result
+
+        def func_time_analyse(*args, **kwargs):
+            from pyinstrument import Profiler
+
+            profiler = Profiler()
+            profiler.start()
+
+            result = func(*args, **kwargs)
+
+            profiler.stop()
+            profiler.print()
+            return result
+
+        if async_func is True:
+            return func_time_async
+        else:
+            if analyse:
+                return func_time_analyse
+            else:
+                return func_time
+
+    return wrapper
+
+
+def fn_timeout_checker(wait_time, callback):
+    """
+    超时判断的装饰器
+    两个包，使用gevent出现bug
+    """
+    # from gevent import Timeout
+    # from gevent.monkey import patch_all
+
+    # patch_all() # thread=False加了这个参数，配合flask app的threaded=True,会报错，目前还没有理解阻塞，线程之间的关系。不加即thread=True时没问题
+
+    from eventlet import Timeout
+    from eventlet import monkey_patch
+
+    monkey_patch(time=True)
+
+    def wrapper(func):
+        def inner(*args, **kwargs):
+            finish_flag = False
+            with Timeout(wait_time, False):
+                res = func(*args, **kwargs)
+                finish_flag = True
+            if not finish_flag:
+                res = callback()
+            return res
+
+        return inner
+
+    return wrapper
+
+
+def fn_try(parameter):
+    """
+    该函数把try...catch...封装成装饰器，
+    接收一个字典参数，并把其中的msg字段改为具体报错信息
+    :param parameter: {"msg": "", etc.}
+    :return: parameter: {"msg": 内容填充为具体的报错信息, etc.}
+    """
+
+    def wrapper(function):
+        def inner(*args, **kwargs):
+            try:
+                result = function(*args, **kwargs)
+                return result
+            except Exception as e:
+                msg = "报错！"
+                print('[func_name: {func_name} {msg}]'.format(func_name=function.__name__, msg=msg))
+                parameter["msg"] = parameter["msg"].format(str(e))
+                return parameter
+            finally:
+                pass
+
+        return inner
+
+    return wrapper
+
+
+def try_log(function):
+    @wraps(function)
+    def inner(*args, **kwargs):
+        try:
+            result = function(*args, **kwargs)
+            return result
+        except Exception as e:
+            logging.error(*args)
+            logging.error(e.__traceback__.tb_frame.f_globals["__file__"])
+            logging.error(e.__traceback__.tb_lineno)
+            logging.error(repr(e))
+
+    return inner
+
+
+def example(function):
+    @wraps(function)
+    def function_example(*args, **kwargs):
+        print("此方法仅仅用于提示该方法怎么写")
+        result = function(*args, **kwargs)
+        return result
+
+    return function_example
+
+
+def singleton(cls):
+    instances = {}
+
+    def _singleton(*args, **kwargs):
+        if cls not in instances:
+            instances[cls] = cls(*args, **kwargs)
+        return instances[cls]
+
+    return _singleton