nlpertools 1.0.5__py3-none-any.whl → 1.0.6.dev0__py3-none-any.whl

nlpertools/vector_index_demo.py

@@ -0,0 +1,108 @@
+#!/usr/bin/python3.8
+# -*- coding: utf-8 -*-
+# @Author  : youshu.Ji
+import math
+
+import faiss
+import gensim
+import numpy as np
+import pandas as pd
+
+
+def build_index_use(vectors):
+    d = len(vectors[0])
+    nlist = 100
+    quantizer = faiss.IndexFlatL2(d)
+    index = faiss.IndexIVFFlat(quantizer, d, nlist, faiss.METRIC_L2)
+    index.train(vectors)
+    index.add(vectors)
+    return index
+
+
+def build_index(vectors, distances="L2", nprobe=10):
+    """ 建立 faiss 索引.
+
+    Args:
+        vectors(numpy.array): 向量矩阵，shape=(向量数, 向量维度)
+        distance(str): 度量距离，支持 L2、COS 和 INNER_PRODUCT.
+        nprobe(int): 向量搜索时需要搜索的聚类数.
+
+    Return: 返回 faiss 索引对象.
+
+    """
+    metric_type = None
+    if distances == "L2":
+        metric_type = faiss.METRIC_L2
+    elif distances in ("COS", "INNER_PRODUCT"):
+        metric_type = faiss.METRIC_INNER_PRODUCT
+    else:
+        raise NotImplementedError
+
+    index_pipes = []
+
+    if distances == "COS":
+        index_pipes.append("L2norm")
+
+    K = 4 * math.sqrt(vectors.shape[0])
+    use_ivf = False
+    if vectors.shape[0] >= 30 * K:
+        index_pipes.append(f"IVF{K}")
+        use_ivf = True
+
+    index_pipes.append("Flat")
+
+    index = faiss.index_factory(vectors.shape[1], ",".join(index_pipes),
+                                metric_type)
+
+    vectors = vectors.astype(np.float32)
+    if not index.is_trained:
+        index.train(vectors)
+
+    index.add(vectors)
+
+    # IVF 使用 reconstruct 时必须执行此函数
+    if use_ivf:
+        ivf_index = faiss.extract_index_ivf(index)
+        ivf_index.make_direct_map()
+        ivf_index.nprobe = nprobe
+
+    return index
+
+
+def read_index_from_file(filename):
+    """ 从向量文件中读取 faiss 向量对象. """
+    return faiss.read_index(filename)
+
+
+def write_index_to_file(index, filename):
+    """ 将 faiss 向量对象写入文件. """
+    faiss.write_index(index, filename)
+
+
+word2vec_path = "glove_vector_path"
+wv_from_text = gensim.models.KeyedVectors.load_word2vec_format(word2vec_path, binary=False, no_header=True)
+vectors = wv_from_text.vectors
+
+name_example = ["{}.jpg".format((i % 9) + 1) for i in range(len(vectors))]
+df = pd.DataFrame({
+    "name": name_example,
+    # "vector": str(vectors[0]),
+    # "text": list(wv_from_text.key_to_index.keys()),
+})
+test_index = build_index_use(vectors)
+write_index_to_file(test_index, "test.index")
+
+df.to_csv("test.csv", index=False)
+
+import gensim
+import hnswlib
+
+word2vec_path = "glove_vector_path"
+wv_from_text = gensim.models.KeyedVectors.load_word2vec_format(word2vec_path, binary=False, no_header=True)
+vectors = wv_from_text.vectors
+
+labels = [idx for idx, i in enumerate(vectors)]
+index = hnswlib.Index(space="l2", dim=len(vectors[0]))
+index.init_index(max_elements=len(vectors), ef_construction=200, M=16)
+index.add_items(vectors, labels)
+index.save_index("hnswlib.index")

nlpertools/wrapper.py

@@ -1,96 +1,161 @@
-#!/usr/bin/python3.8
-# -*- coding: utf-8 -*-
-# @Author  : youshu.Ji
-# 定义装饰器
-from functools import wraps
-
-
-def fn_timer(function):
-    @wraps(function)
-    def function_timer(*args, **kwargs):
-        t0 = time.time()
-        result = function(*args, **kwargs)
-        t1 = time.time()
-        print('[finished {func_name} in {time:.2f}s]'.format(func_name=function.__name__, time=t1 - t0))
-        return result
-
-    return function_timer
-
-
-def fn_timeout_checker(wait_time, callback):
-    """
-    超时判断的装饰器
-    两个包，使用gevent出现bug
-    """
-    # from gevent import Timeout
-    # from gevent.monkey import patch_all
-
-    # patch_all() # thread=False加了这个参数，配合flask app的threaded=True,会报错，目前还没有理解阻塞，线程之间的关系。不加即thread=True时没问题
-
-    from eventlet import Timeout
-    from eventlet import monkey_patch
-
-    monkey_patch(time=True)
-
-    def wrapper(func):
-        def inner(*args, **kwargs):
-            finish_flag = False
-            with Timeout(wait_time, False):
-                res = func(*args, **kwargs)
-                finish_flag = True
-            if not finish_flag:
-                res = callback()
-            return res
-
-        return inner
-
-    return wrapper
-
-
-def fn_try(parameter):
-    """
-    该函数把try...catch...封装成装饰器，
-    接收一个字典参数，并把其中的msg字段改为具体报错信息
-    :param parameter: {"msg": "", etc.}
-    :return: parameter: {"msg": 内容填充为具体的报错信息, etc.}
-    """
-
-    def wrapper(function):
-        def inner(*args, **kwargs):
-            try:
-                result = function(*args, **kwargs)
-                return result
-            except Exception as e:
-                msg = "报错！"
-                print('[func_name: {func_name} {msg}]'.format(func_name=function.__name__, msg=msg))
-                parameter["msg"] = parameter["msg"].format(str(e))
-                return parameter
-            finally:
-                pass
-
-        return inner
-
-    return wrapper
-
-
-def example(function):
-    @wraps(function)
-    def function_example(*args, **kwargs):
-        print("此方法仅仅用于提示该方法怎么写")
-        result = function(*args, **kwargs)
-        return result
-
-    return function_example
-
-
-def singleton(cls):
-    instances = {}
-
-    def _singleton(*args, **kwargs):
-        if cls not in instances:
-            instances[cls] = cls(*args, **kwargs)
-        return instances[cls]
-
-    return _singleton
-
-
+#!/usr/bin/python3.8
+# -*- coding: utf-8 -*-
+# @Author  : youshu.Ji
+# 定义装饰器
+import logging
+import time
+from functools import wraps
+import asyncio
+
+def fn_async_timer(function):
+    """
+    针对异步函数的装饰器
+    """
+    @wraps(function)
+    async def function_timer(*args, **kwargs):
+        t0 = time.time()
+        result = await function(*args, **kwargs)
+        t1 = time.time()
+        print('[finished {func_name} in {time:.2f}s]'.format(func_name=function.__name__, time=t1 - t0))
+        return result
+
+    return function_timer
+
+
+def fn_timer(async_func=False, analyse=False):
+    """
+    >>> @fn_timer()
+    >>> def example():
+    >>>     time.sleep(2)
+    :param analyse:
+    :return:
+    """
+
+    def wrapper(func):
+        async def func_time_async(*args, **kwargs):
+            t0 = time.time()
+            result = await asyncio.create_task(func(*args, **kwargs))
+            t1 = time.time()
+            print('[finished {func_name} in {time:.2f}s]'.format(func_name=func.__name__, time=t1 - t0))
+            return result
+
+        def func_time(*args, **kwargs):
+            t0 = time.time()
+            result = func(*args, **kwargs)
+            t1 = time.time()
+            print('[finished {func_name} in {time:.2f}s]'.format(func_name=func.__name__, time=t1 - t0))
+            return result
+
+        def func_time_analyse(*args, **kwargs):
+            from pyinstrument import Profiler
+
+            profiler = Profiler()
+            profiler.start()
+
+            result = func(*args, **kwargs)
+
+            profiler.stop()
+            profiler.print()
+            return result
+
+        if async_func is True:
+            return func_time_async
+        else:
+            if analyse:
+                return func_time_analyse
+            else:
+                return func_time
+
+    return wrapper
+
+
+def fn_timeout_checker(wait_time, callback):
+    """
+    超时判断的装饰器
+    两个包，使用gevent出现bug
+    """
+    # from gevent import Timeout
+    # from gevent.monkey import patch_all
+
+    # patch_all() # thread=False加了这个参数，配合flask app的threaded=True,会报错，目前还没有理解阻塞，线程之间的关系。不加即thread=True时没问题
+
+    from eventlet import Timeout
+    from eventlet import monkey_patch
+
+    monkey_patch(time=True)
+
+    def wrapper(func):
+        def inner(*args, **kwargs):
+            finish_flag = False
+            with Timeout(wait_time, False):
+                res = func(*args, **kwargs)
+                finish_flag = True
+            if not finish_flag:
+                res = callback()
+            return res
+
+        return inner
+
+    return wrapper
+
+
+def fn_try(parameter):
+    """
+    该函数把try...catch...封装成装饰器，
+    接收一个字典参数，并把其中的msg字段改为具体报错信息
+    :param parameter: {"msg": "", etc.}
+    :return: parameter: {"msg": 内容填充为具体的报错信息, etc.}
+    """
+
+    def wrapper(function):
+        def inner(*args, **kwargs):
+            try:
+                result = function(*args, **kwargs)
+                return result
+            except Exception as e:
+                msg = "报错！"
+                print('[func_name: {func_name} {msg}]'.format(func_name=function.__name__, msg=msg))
+                parameter["msg"] = parameter["msg"].format(str(e))
+                return parameter
+            finally:
+                pass
+
+        return inner
+
+    return wrapper
+
+
+def try_log(function):
+    @wraps(function)
+    def inner(*args, **kwargs):
+        try:
+            result = function(*args, **kwargs)
+            return result
+        except Exception as e:
+            logging.error(*args)
+            logging.error(e.__traceback__.tb_frame.f_globals["__file__"])
+            logging.error(e.__traceback__.tb_lineno)
+            logging.error(repr(e))
+
+    return inner
+
+
+def example(function):
+    @wraps(function)
+    def function_example(*args, **kwargs):
+        print("此方法仅仅用于提示该方法怎么写")
+        result = function(*args, **kwargs)
+        return result
+
+    return function_example
+
+
+def singleton(cls):
+    instances = {}
+
+    def _singleton(*args, **kwargs):
+        if cls not in instances:
+            instances[cls] = cls(*args, **kwargs)
+        return instances[cls]
+
+    return _singleton