nlpertools 1.0.5__py3-none-any.whl → 1.0.6.dev0__py3-none-any.whl

nlpertools/utils/log_util.py

@@ -0,0 +1,20 @@
+#!/usr/bin/python3.8
+# -*- coding: utf-8 -*-
+# @Author  : youshu.Ji
+"""
+# 该项目暂时没有日志输出
+import codecs
+import logging.config
+
+import nlpertools
+import yaml
+
+nlpertools.j_mkdir("logs")
+
+with codecs.open('log_config.yml', 'r', 'utf-8') as stream:
+    config = yaml.load(stream, Loader=yaml.FullLoader)
+
+# logging.basicConfig(level=logging.INFO)
+logging.config.dictConfig(config)
+logger = logging.getLogger()
+"""

nlpertools/utils/package.py

@@ -1,76 +1,89 @@
-#!/usr/bin/python3.8
-# -*- coding: utf-8 -*-
-# @Author  : youshu.Ji
-from importlib import import_module
-
-
-def try_import(name, package):
-    try:
-        return import_module(name, package=package)
-    except:
-        pass
-        # print("import {} failed".format(name))
-    finally:
-        pass
-
-
-# import aioredis
-# import happybase
-# import pandas as pd
-# import pymysql
-# from elasticsearch import Elasticsearch, helpers
-# from kafka import KafkaProducer, KafkaConsumer
-# from pymongo import MongoClient
-# from elasticsearch import helpers
-# from kafka import KafkaConsumer
-aioredis = try_import("aioredis", None)
-happybase = try_import("happybase", None)
-pd = try_import("pandas", None)
-pymysql = try_import("pymysql", None)
-Elasticsearch = try_import("elasticsearch", "Elasticsearch")
-KafkaProducer = try_import("kafka", "KafkaProducer")
-MongoClient = try_import("pymongo", "MongoClient")
-helpers = try_import("elasticsearch", "helpers")
-KafkaConsumer = try_import("kafka", "KafkaConsumer")
-# import numpy as np
-# import seaborn as sns
-# import torch
-# import torch.nn as nn
-# import xgboost as xgb
-# from matplotlib import pyplot as plt
-# from nltk.stem import WordNetLemmatizer
-# from sklearn import metrics
-# from transformers import BertTokenizer, BertForMaskedLM
-# from transformers import BertForMaskedLM
-np = try_import("numpy", None)
-sns = try_import("seaborn", None)
-torch = try_import("torch", None)
-nn = try_import("torch.nn", None)
-xgb = try_import("xgboost", None)
-plt = try_import("matplotlib", "pyplot")
-WordNetLemmatizer = try_import("nltk.stem", "WordNetLemmatizer")
-metrics = try_import("sklearn", "metrics")
-BertTokenizer = try_import("transformers", "BertTokenizer")
-BertForMaskedLM = try_import("transformers", "BertForMaskedLM")
-# import requests
-requests = try_import("requests", None)
-# import numpy as np
-# import psutil
-# import pyquery as pq
-# import requests
-# import torch
-# from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
-# from sklearn.metrics import precision_recall_fscore_support
-# from tqdm import tqdm
-# from win32evtlogutil import langid
-# from sklearn.feature_extraction.text import TfidfTransformer
-psutil = try_import("psutil", None)
-pq = try_import("pyquery", None)
-CountVectorizer = try_import("sklearn.feature_extraction.text", "CountVectorizer")
-precision_recall_fscore_support = try_import("sklearn.metrics", "precision_recall_fscore_support")
-tqdm = try_import("tqdm", "tqdm")
-langid = try_import("win32evtlogutil", "langid")
-TfidfTransformer = try_import("sklearn.feature_extraction.text", "TfidfTransformer")
-# import pandas as pd
-# import yaml
-yaml = try_import("yaml", None)
+#!/usr/bin/python3.8
+# -*- coding: utf-8 -*-
+# @Author  : youshu.Ji
+import importlib
+from importlib import import_module
+
+
+def try_import(name, package):
+    try:
+        return import_module(name, package=package)
+    except:
+        pass
+        # print("import {} failed".format(name))
+    finally:
+        pass
+
+
+def lazy_import(importer_name, to_import):
+    """
+    Example from net
+    author: unknown
+    this function is not used
+    """
+    """Return the importing module and a callable for lazy importing.
+
+    The module named by importer_name represents the module performing the
+    import to help facilitate resolving relative imports.
+
+    to_import is an iterable of the modules to be potentially imported (absolute
+    or relative). The `as` form of importing is also supported,
+    e.g. `pkg.mod as spam`.
+
+    This function returns a tuple of two items. The first is the importer
+    module for easy reference within itself. The second item is a callable to be
+    set to `__getattr__`.
+    """
+    module = importlib.import_module(importer_name)
+    import_mapping = {}
+    for name in to_import:
+        importing, _, binding = name.partition(' as ')
+        if not binding:
+            _, _, binding = importing.rpartition('.')
+        import_mapping[binding] = importing
+
+    def __getattr__(name):
+        if name not in import_mapping:
+            message = f'module {importer_name!r} has no attribute {name!r}'
+            raise AttributeError(message)
+        importing = import_mapping[name]
+        # imortlib.import_module() implicitly sets submodules on this module as
+        # appropriate for direct imports.
+        imported = importlib.import_module(importing,
+                                           module.__spec__.parent)
+        setattr(module, name, imported)
+        return imported
+
+    return module, __getattr__
+
+
+# jieba = try_import("jieba", None)
+# sns = try_import("seaborn", None)
+# torch = try_import("torch", None)
+# nn = try_import("torch.nn", None)
+# BertTokenizer = try_import("transformers", "BertTokenizer")
+# BertForMaskedLM = try_import("transformers", "BertForMaskedLM")
+# Elasticsearch = try_import("elasticsearch", "Elasticsearch")
+# pd = try_import("pandas", None)
+# xgb = try_import("xgboost", None)
+
+aioredis = try_import("aioredis", None)
+pymysql = try_import("pymysql", None)
+zhconv = try_import("zhconv", None)
+KafkaProducer = try_import("kafka", "KafkaProducer")
+KafkaConsumer = try_import("kafka", "KafkaConsumer")
+np = try_import("numpy", None)
+plt = try_import("matplotlib", "pyplot")
+WordNetLemmatizer = try_import("nltk.stem", "WordNetLemmatizer")
+metrics = try_import("sklearn", "metrics")
+requests = try_import("requests", None)
+pq = try_import("pyquery", None)
+CountVectorizer = try_import("sklearn.feature_extraction.text", "CountVectorizer")
+precision_recall_fscore_support = try_import("sklearn.metrics", "precision_recall_fscore_support")
+tqdm = try_import("tqdm", "tqdm")
+# TODO 自动导出langid和win32evtlogutil输出有bug
+langid = try_import("langid", None)
+win32evtlogutil = try_import("win32evtlogutil", None)
+TfidfTransformer = try_import("sklearn.feature_extraction.text", "TfidfTransformer")
+yaml = try_import("yaml", None)
+omegaconf = try_import("omegaconf", None)

nlpertools/utils/package_v1.py

@@ -0,0 +1,94 @@
+#!/usr/bin/python3.8
+# -*- coding: utf-8 -*-
+# @Author  : youshu.Ji
+import importlib
+from importlib import import_module
+from importlib.util import LazyLoader
+from .lazy import lazy_module
+
+EXCLUDE_LAZYIMPORT = {"torch", "torch.nn", "numpy"}
+
+
+def try_import(name, package):
+    try:
+        if package:
+            # print("import {} success".format(name))
+            return lazy_module("{}.{}".format(package, name))
+        else:
+            if name in EXCLUDE_LAZYIMPORT:
+                return import_module(name, package=package)
+            return lazy_module(name)
+        # return import_module(name, package=package)
+    except:
+        pass
+        print("import {} failed".format(name))
+    finally:
+        pass
+
+
+def lazy_import(importer_name, to_import):
+    """Return the importing module and a callable for lazy importing.
+
+    The module named by importer_name represents the module performing the
+    import to help facilitate resolving relative imports.
+
+    to_import is an iterable of the modules to be potentially imported (absolute
+    or relative). The `as` form of importing is also supported,
+    e.g. `pkg.mod as spam`.
+
+    This function returns a tuple of two items. The first is the importer
+    module for easy reference within itself. The second item is a callable to be
+    set to `__getattr__`.
+    """
+    module = importlib.import_module(importer_name)
+    import_mapping = {}
+    for name in to_import:
+        importing, _, binding = name.partition(' as ')
+        if not binding:
+            _, _, binding = importing.rpartition('.')
+        import_mapping[binding] = importing
+
+    def __getattr__(name):
+        if name not in import_mapping:
+            message = f'module {importer_name!r} has no attribute {name!r}'
+            raise AttributeError(message)
+        importing = import_mapping[name]
+        # imortlib.import_module() implicitly sets submodules on this module as
+        # appropriate for direct imports.
+        imported = importlib.import_module(importing,
+                                           module.__spec__.parent)
+        setattr(module, name, imported)
+        return imported
+
+    return module, __getattr__
+
+
+aioredis = try_import("aioredis", None)
+happybase = try_import("happybase", None)
+pd = try_import("pandas", None)
+pymysql = try_import("pymysql", None)
+Elasticsearch = try_import("elasticsearch", "Elasticsearch")
+KafkaProducer = try_import("kafka", "KafkaProducer")
+MongoClient = try_import("pymongo", "MongoClient")
+helpers = try_import("elasticsearch", "helpers")
+KafkaConsumer = try_import("kafka", "KafkaConsumer")
+np = try_import("numpy", None)
+sns = try_import("seaborn", None)
+torch = try_import("torch", None)
+nn = try_import("torch.nn", None)
+xgb = try_import("xgboost", None)
+plt = try_import("matplotlib", "pyplot")
+WordNetLemmatizer = try_import("nltk.stem", "WordNetLemmatizer")
+metrics = try_import("sklearn", "metrics")
+BertTokenizer = try_import("transformers", "BertTokenizer")
+BertForMaskedLM = try_import("transformers", "BertForMaskedLM")
+requests = try_import("requests", None)
+psutil = try_import("psutil", None)
+pq = try_import("pyquery", None)
+CountVectorizer = try_import("sklearn.feature_extraction.text", "CountVectorizer")
+precision_recall_fscore_support = try_import("sklearn.metrics", "precision_recall_fscore_support")
+tqdm = try_import("tqdm", "tqdm")
+langid = try_import("langid", None)
+# win32evtlogutil?
+TfidfTransformer = try_import("sklearn.feature_extraction.text", "TfidfTransformer")
+yaml = try_import("yaml", None)

nlpertools/utils/package_v2.py

@@ -0,0 +1,117 @@
+# !/usr/bin/python3.8
+# -*- coding: utf-8 -*-
+# @Author  : youshu.Ji
+import importlib
+from importlib import import_module
+import os
+
+
+def try_import(name, package):
+    try:
+        return import_module(name, package=package)
+    except:
+        pass
+        # print("import {} failed".format(name))
+    finally:
+        pass
+
+
+aioredis = None
+happybase = None
+pd = None
+pymysql = None
+Elasticsearch = None
+KafkaProducer = None
+MongoClient = None
+helpers = None
+KafkaConsumer = None
+np = None
+sns = None
+torch = None
+nn = None
+xgb = None
+plt = None
+WordNetLemmatizer = None
+metrics = None
+BertTokenizer = None
+BertForMaskedLM = None
+requests = None
+psutil = None
+pq = None
+CountVectorizer = None
+precision_recall_fscore_support = None
+tqdm = None
+langid = None
+win32evtlogutil = None
+TfidfTransformer = None
+yaml = None
+
+import_dict = {
+    "aioredis": ("aioredis", None),
+    "happybase": ("happybase", None),
+    "pd": ("pandas", None),
+    "pymysql": ("pymysql", None),
+    "Elasticsearch": ("elasticsearch", "Elasticsearch"),
+    "KafkaProducer": ("kafka", "KafkaProducer"),
+    "MongoClient": ("pymongo", "MongoClient"),
+    "helpers": ("elasticsearch", "helpers"),
+    "KafkaConsumer": ("kafka", "KafkaConsumer"),
+    "np": ("numpy", None),
+    "sns": ("seaborn", None),
+    "torch": ("torch", None),
+    "nn": ("torch.nn", None),
+    "xgb": ("xgboost", None),
+    "plt": ("matplotlib", "pyplot"),
+    "WordNetLemmatizer": ("nltk.stem", "WordNetLemmatizer"),
+    "metrics": ("sklearn", "metrics"),
+    "BertTokenizer": ("transformers", "BertTokenizer"),
+    "BertForMaskedLM": ("transformers", "BertForMaskedLM"),
+    "requests": ("requests", None),
+    "psutil": ("psutil", None),
+    "pq": ("pyquery", None),
+    "CountVectorizer": ("sklearn.feature_extraction.text", "CountVectorizer"),
+    "precision_recall_fscore_support": ("sklearn.metrics", "precision_recall_fscore_support"),
+    "tqdm": ("tqdm", "tqdm"),
+    "langid": ("langid", None),
+    "win32evtlogutil": ("win32evtlogutil", None),
+    "TfidfTransformer": ("sklearn.feature_extraction.text", "TfidfTransformer"),
+    "yaml": ("yaml", None)
+}
+if "nlpertools_helper" in os.environ.keys():
+    # TODO 该方法未经过测试
+    import_list = os.environ["nlpertools_helper"]
+
+    for k in import_list:
+        name, package = import_dict[k]
+        globals()[k] = try_import(name, package)
+else:
+    aioredis = try_import("aioredis", None)
+    happybase = try_import("happybase", None)
+    pd = try_import("pandas", None)
+    pymysql = try_import("pymysql", None)
+    Elasticsearch = try_import("elasticsearch", "Elasticsearch")
+    KafkaProducer = try_import("kafka", "KafkaProducer")
+    MongoClient = try_import("pymongo", "MongoClient")
+    helpers = try_import("elasticsearch", "helpers")
+    KafkaConsumer = try_import("kafka", "KafkaConsumer")
+    np = try_import("numpy", None)
+    sns = try_import("seaborn", None)
+    torch = try_import("torch", None)
+    nn = try_import("torch.nn", None)
+    xgb = try_import("xgboost", None)
+    plt = try_import("matplotlib", "pyplot")
+    WordNetLemmatizer = try_import("nltk.stem", "WordNetLemmatizer")
+    metrics = try_import("sklearn", "metrics")
+    BertTokenizer = try_import("transformers", "BertTokenizer")
+    BertForMaskedLM = try_import("transformers", "BertForMaskedLM")
+    requests = try_import("requests", None)
+    psutil = try_import("psutil", None)
+    pq = try_import("pyquery", None)
+    CountVectorizer = try_import("sklearn.feature_extraction.text", "CountVectorizer")
+    precision_recall_fscore_support = try_import("sklearn.metrics", "precision_recall_fscore_support")
+    tqdm = try_import("tqdm", "tqdm")
+    # TODO 自动导出langid和win32evtlogutil输出有bug
+    langid = try_import("langid", None)
+    win32evtlogutil = try_import("win32evtlogutil", None)
+    TfidfTransformer = try_import("sklearn.feature_extraction.text", "TfidfTransformer")
+    yaml = try_import("yaml", None)

nlpertools/utils_for_nlpertools.py

@@ -1,93 +1,93 @@
-import os
-import shutil
-from importlib import import_module
-
-from .io.dir import j_mkdir
-from .io.file import readtxt_list_all_strip, writetxt_w_list
-
-
-def try_import(name, package):
-    try:
-        return import_module(name, package=package)
-    except:
-        print("import {} failed".format(name))
-    finally:
-        pass
-
-
-def convert_import_to_try_import(from_path, to_path):
-    j_mkdir(to_path)
-    for root, dirs, files in os.walk(from_path):
-        for sub_dir in dirs:
-            j_mkdir(os.path.join(root.replace(from_path, to_path), sub_dir))
-        for file in files:
-            src = os.path.join(root, file)
-            dst = os.path.join(root.replace(from_path, to_path), file)
-            excluded_file = ["wrapper.py", "kmp.py", "__init__.py"]
-            if file.endswith(".py") and file != "utils_for_nlpertools.py" and file not in excluded_file:
-                raw_code = readtxt_list_all_strip(src)
-                start_idx, end_idx = 0, 0
-
-                for idx, each_line in enumerate(raw_code[:30]):
-                    each_line = each_line.lstrip("# ")
-                    if start_idx == 0 and (each_line.startswith("from") or each_line.startswith("import")):
-                        try:
-                            exec(each_line)
-                        except:
-                            start_idx = idx
-                    if start_idx != 0 and not each_line:
-                        end_idx = idx
-                        break
-                # print(file, start_idx, end_idx)
-                if start_idx != 0 and end_idx != 0:
-                    new_code = raw_code[:start_idx] + convert_import_string_to_import_list(
-                        "\n".join(raw_code[start_idx:end_idx])) + raw_code[end_idx:]
-                else:
-                    new_code = raw_code
-                writetxt_w_list(new_code, dst)
-            else:
-                shutil.copy(src=src, dst=dst)
-    print("convert over")
-
-
-def get_import_info(text):
-    pass
-
-
-def convert_import_string_to_import_list(text):
-    """
-    该方法将 import 转变为 try import
-    """
-    models_to_import = []
-    import_list = text.split("\n")
-    for each in import_list:
-        print(each)
-        name, package, as_name = None, None, None
-        elements = each.split(" ")
-        for pre, cur in zip(elements, elements[1:]):
-            if cur.endswith(","):
-                cur = cur.rstrip(",")
-            # 为了实现from import 和 import统一，首先把package和name的含义反过来，后面再掉换
-            if pre == "import":
-                package = cur
-            if pre == "from":
-                name = cur
-            if pre == "as":
-                as_name = cur
-            if pre[-1] == ",":
-                # 针对 from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
-                # 将将前面部分和当前的组成新字段
-                prefix = each.split("import")[0]
-                import_list.append("{}import {}".format(prefix, cur))
-        if not as_name:
-            as_name = package.split(".")[-1]
-        if not name:
-            name, package = package, name
-        models_to_import.append((name, package, as_name))
-    # 打印
-    all_import_info = ["", "from utils_for_nlpertools import try_import", ""]
-    for name, package, as_name in models_to_import:
-        import_info = '{} = try_import("{}", {})'.format(as_name, name, '"{}"'.format(package) if package else package)
-        all_import_info.append(import_info)
-        print(import_info)
-    return all_import_info
+import os
+import shutil
+from importlib import import_module
+
+from .io.dir import j_mkdir
+from .io.file import readtxt_list_all_strip, writetxt_w_list
+
+
+def try_import(name, package):
+    try:
+        return import_module(name, package=package)
+    except:
+        print("import {} failed".format(name))
+    finally:
+        pass
+
+
+def convert_import_to_try_import(from_path, to_path):
+    j_mkdir(to_path)
+    for root, dirs, files in os.walk(from_path):
+        for sub_dir in dirs:
+            j_mkdir(os.path.join(root.replace(from_path, to_path), sub_dir))
+        for file in files:
+            src = os.path.join(root, file)
+            dst = os.path.join(root.replace(from_path, to_path), file)
+            excluded_file = ["wrapper.py", "kmp.py", "__init__.py"]
+            if file.endswith(".py") and file != "utils_for_nlpertools.py" and file not in excluded_file:
+                raw_code = readtxt_list_all_strip(src)
+                start_idx, end_idx = 0, 0
+
+                for idx, each_line in enumerate(raw_code[:30]):
+                    each_line = each_line.lstrip("# ")
+                    if start_idx == 0 and (each_line.startswith("from") or each_line.startswith("import")):
+                        try:
+                            exec(each_line)
+                        except:
+                            start_idx = idx
+                    if start_idx != 0 and not each_line:
+                        end_idx = idx
+                        break
+                # print(file, start_idx, end_idx)
+                if start_idx != 0 and end_idx != 0:
+                    new_code = raw_code[:start_idx] + convert_import_string_to_import_list(
+                        "\n".join(raw_code[start_idx:end_idx])) + raw_code[end_idx:]
+                else:
+                    new_code = raw_code
+                writetxt_w_list(new_code, dst)
+            else:
+                shutil.copy(src=src, dst=dst)
+    print("convert over")
+
+
+def get_import_info(text):
+    pass
+
+
+def convert_import_string_to_import_list(text):
+    """
+    该方法将 import 转变为 try import
+    """
+    models_to_import = []
+    import_list = text.split("\n")
+    for each in import_list:
+        print(each)
+        name, package, as_name = None, None, None
+        elements = each.split(" ")
+        for pre, cur in zip(elements, elements[1:]):
+            if cur.endswith(","):
+                cur = cur.rstrip(",")
+            # 为了实现from import 和 import统一，首先把package和name的含义反过来，后面再掉换
+            if pre == "import":
+                package = cur
+            if pre == "from":
+                name = cur
+            if pre == "as":
+                as_name = cur
+            if pre[-1] == ",":
+                # 针对 from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
+                # 将将前面部分和当前的组成新字段
+                prefix = each.split("import")[0]
+                import_list.append("{}import {}".format(prefix, cur))
+        if not as_name:
+            as_name = package.split(".")[-1]
+        if not name:
+            name, package = package, name
+        models_to_import.append((name, package, as_name))
+    # 打印
+    all_import_info = ["", "from utils_for_nlpertools import try_import", ""]
+    for name, package, as_name in models_to_import:
+        import_info = '{} = try_import("{}", {})'.format(as_name, name, '"{}"'.format(package) if package else package)
+        all_import_info.append(import_info)
+        print(import_info)
+    return all_import_info