nlpertools 1.0.5__py3-none-any.whl → 1.0.6.dev0__py3-none-any.whl

nlpertools/__init__.py

@@ -1,20 +1,24 @@
-#!/usr/bin/python3.8
-# -*- coding: utf-8 -*-
-# @Author  : youshu.Ji
-from .algo.kmp import *
-from .data_structure.base_structure import *
-from .dataprocess import *
-from .io.dir import *
-from .io.file import *
-from .ml import *
-from .ml import *
-from .openApi import *
-from .other import *
-from .plugin import *
-from .reminder import *
-from .utils_for_nlpertools import *
-from .wrapper import *
-
-DB_CONFIG_FILE = "./default_db_config.yml"
-
-__version__ = '1.0.5'
+#!/usr/bin/python3.8
+# -*- coding: utf-8 -*-
+# @Author  : youshu.Ji
+from .algo.kmp import *
+from .data_structure.base_structure import *
+from .dataprocess import *
+from .io.dir import *
+from .io.file import *
+from .ml import *
+from .open_api import *
+from .other import *
+from .pic import *
+from .plugin import *
+from .reminder import *
+from .utils_for_nlpertools import *
+from .wrapper import *
+from .monitor import *
+
+import os
+
+
+DB_CONFIG_FILE = os.path.join(os.path.dirname(__file__),"default_db_config.yml")
+
+__version__ = '1.0.5'

nlpertools/algo/ac.py

@@ -0,0 +1,18 @@
+#!/usr/bin/python3.8
+# -*- coding: utf-8 -*-
+# @Author  : youshu.Ji
+from ..io.file import readtxt_list_all_strip
+
+
+def find_sentence_covered_vocab(vocab, sentences):
+    """
+    找到词典中
+    此为参照写法，具体用的时候复制出去用避免重复构建
+    """
+
+    from ahocorasick import Automaton
+    atm = Automaton()
+    for word in vocab:
+        atm.add_word(word, word)
+    atm.make_automaton()
+

nlpertools/algo/bit_ops.py

@@ -0,0 +1,28 @@
+#!/usr/bin/python3.8
+# -*- coding: utf-8 -*-
+# @Author  : youshu.Ji
+def foo(num):
+    return num & -num
+
+
+def foo2(num):
+    """
+    raw: 0 1 2 3 4 5 6 7 8 9
+    res: 0 0 0 2 0 4 4 6 0 8
+    """
+    return num & (num - 1)
+
+
+def _lowbit(index: int) -> int:
+    """
+    raw: 0 1 2 3 4 5 6 7 8 9
+    res: 0 1 2 1 4 1 2 1 8 1
+    """
+    return index & -index
+
+if __name__ == '__main__':
+    for i in range(10):
+        print(i, end=" ")
+    print()
+    for i in range(10):
+        print(foo2(i), end=" ")

nlpertools/algo/kmp.py

@@ -1,55 +1,94 @@
-#!/usr/bin/python3.8
-# -*- coding: utf-8 -*-
-# @Author  : youshu.Ji
-
-def build(pattern_string):
-    """
-    构建模式串的PMT
-    [zhihu](https://www.zhihu.com/question/21923021/answer/281346746)
-
-    """
-    # 构建pattern需要回溯的位置，
-    backtrace_points = [0] * len(pattern_string)
-    main_pointer, pattern_pointer = 0, -1
-    backtrace_points[0] = -1
-    while main_pointer < len(pattern_string) - 1:
-        if pattern_pointer == -1 or pattern_string[pattern_pointer] == pattern_string[main_pointer]:
-            main_pointer += 1
-            pattern_pointer += 1
-            backtrace_points[main_pointer] = pattern_pointer
-        else:
-            pattern_pointer = backtrace_points[pattern_pointer]
-    return backtrace_points
-
-
-def find(main_string, pattern_string):
-    """
-    模式匹配
-    一边构建字串的回溯点，一边判断模式是否匹配
-    """
-    if len(main_string) < len(pattern_string):
-        return False
-    main_string = " " + main_string
-    backtrace_points = [0] * (len(main_string) + 1)
-    main_pointer, pattern_pointer = 0, -1
-    backtrace_points[0] = -1
-    while main_pointer < len(main_string):
-        if pattern_pointer == -1 or pattern_string[pattern_pointer] == main_string[main_pointer]:
-            if pattern_pointer == len(pattern_string) - 1:
-                return True
-            main_pointer += 1
-            pattern_pointer += 1
-            backtrace_points[main_pointer] = pattern_pointer
-        else:
-            pattern_pointer = backtrace_points[pattern_pointer]
-    return False
-
-
-if __name__ == '__main__':
-    test_main_string = "aa"
-    test_pattern_string = "aa"
-
-    res = build(test_pattern_string)
-    print(res)
-    res = find(test_main_string, test_pattern_string)
-    print(res)
+#!/usr/bin/python3.8
+# -*- coding: utf-8 -*-
+# @Author  : youshu.Ji
+
+def build(pattern_string):
+    """
+    构建模式串的PMT
+    [zhihu](https://www.zhihu.com/question/21923021/answer/281346746)
+
+    """
+    # 构建pattern需要回溯的位置，
+    backtrace_points = [0] * len(pattern_string)
+    main_pointer, pattern_pointer = 0, -1
+    backtrace_points[0] = -1
+    while main_pointer < len(pattern_string) - 1:
+        if pattern_pointer == -1 or pattern_string[pattern_pointer] == pattern_string[main_pointer]:
+            main_pointer += 1
+            pattern_pointer += 1
+            backtrace_points[main_pointer] = pattern_pointer
+        else:
+            pattern_pointer = backtrace_points[pattern_pointer]
+    return backtrace_points
+
+
+def build_2(needle: str):
+    # 这写的比第一种简洁
+    # 查找方法也是自己，唯一就是判断结束条件，不是用-1了
+    m = len(needle)
+    if m == 0:
+        return 0
+
+    pmt = [0] * m
+    pattern_pointer = 0
+    for main_pointer in range(1, m):
+        while pattern_pointer > 0 and needle[main_pointer] != needle[pattern_pointer]:
+            pattern_pointer = pmt[pattern_pointer - 1]
+        if needle[main_pointer] == needle[pattern_pointer]:
+            pattern_pointer += 1
+        pmt[main_pointer] = pattern_pointer
+    return pmt
+
+
+def find_after_build(main_string, pattern_string):
+    backtracker = build(pattern_string)
+    # print(backtracker)
+    main_pointer, pattern_pointer = -1, -1
+    while main_pointer <= len(main_string) - 1:
+        if pattern_pointer == -1 or pattern_string[pattern_pointer] == main_string[main_pointer]:
+            # 这是返回首次匹配时main的位置
+            if pattern_pointer == len(pattern_string) - 1:
+                return main_pointer - len(pattern_string) + 1
+            pattern_pointer += 1
+            main_pointer += 1
+        else:
+            pattern_pointer = backtracker[pattern_pointer]
+    return -1
+
+
+def find(main_string, pattern_string):
+    """
+    模式匹配
+    一边构建字串的回溯点，一边判断模式是否匹配
+    """
+    if len(main_string) < len(pattern_string):
+        return False
+    main_string = " " + main_string
+    backtrace_points = [0] * (len(main_string) + 1)
+    main_pointer, pattern_pointer = 0, -1
+    backtrace_points[0] = -1
+    while main_pointer < len(main_string):
+        if pattern_pointer == -1 or pattern_string[pattern_pointer] == main_string[main_pointer]:
+            if pattern_pointer == len(pattern_string) - 1:
+                return True
+            main_pointer += 1
+            pattern_pointer += 1
+            backtrace_points[main_pointer] = pattern_pointer
+        else:
+            pattern_pointer = backtrace_points[pattern_pointer]
+    return False
+
+
+if __name__ == '__main__':
+    test_main_string = "abababc"
+    test_pattern_string = "abababc"
+
+    res = build(test_pattern_string)
+    print(res)
+    res = build_2(test_pattern_string)
+    print(res)
+    # res = find(test_main_string, test_pattern_string)
+    # print(res)
+    #
+    # res = find_after_build(test_main_string, test_pattern_string)
+    # print(res)

nlpertools/algo/num_ops.py

@@ -0,0 +1,12 @@
+#!/usr/bin/python3.8
+# -*- coding: utf-8 -*-
+# @Author  : youshu.Ji
+def gcd(a, b):
+    """
+    math.gcd()等包对于gcd的实现源码中看不到
+    实现方法；辗转相除法
+    """
+    a, b = b, a % b
+    if b == 0:
+        return a
+    return gcd(a, b)

nlpertools/algo/template.py

@@ -0,0 +1,116 @@
+#!/usr/bin/python3.8
+# -*- coding: utf-8 -*-
+# @Author  : youshu.Ji
+from collections import defaultdict
+
+
+# from sortedcontainers import SortedDict, SortedList
+
+# 树状数组只能维护前缀“操作和”(前缀和，前缀积，前缀最大最小)，而线段树可以维护区间操作和。
+
+# 线段树
+class SegmentTree:
+    """
+    https://www.zhihu.com/question/346961479/answer/2274087021
+    性质：线段树的每一个树节点其实都存储了一个「区间（段）的信息」
+    通过add添加
+    """
+
+    pass
+
+
+# 树状数组（二进制下标树） 模板
+class BIT:
+    """
+    TODO 以前在logseq写过笔记，整理到web上
+    代码来自https://leetcode.cn/problems/number-of-recent-calls/solutions/1472043/by-ac_oier-evqe/下的评论
+    """
+
+    def __init__(self, n: int):
+        self.size = n
+        self.tree = defaultdict(int)
+
+    @staticmethod
+    def _lowbit(index: int) -> int:
+        # TODO 同样整理到web
+        return index & -index
+
+    def add(self, index: int, delta: int) -> None:
+        """
+        delta为index位置加的值
+        """
+        while index <= self.size:
+            self.tree[index] += delta
+            index += self._lowbit(index)
+
+    def query(self, index: int) -> int:
+        if index > self.size:
+            index = self.size
+        res = 0
+        while index > 0:
+            res += self.tree[index]
+            index -= self._lowbit(index)
+        return res
+
+    def sumRange(self, left: int, right: int) -> int:
+        return self.query(right) - self.query(left - 1)
+
+
+class BITUsageDemo:
+    """
+    查找区间值的出现次数
+    """
+
+    def __init__(self):
+        self.bit = BIT(10)
+
+    def add(self, x: int):
+        self.bit.add(x, 1)
+
+    def query(self, x, y):
+        return self.bit.sumRange(x, y)
+
+
+class Trie:
+    def __init__(self):
+        self.children = [None] * 26
+        self.isEnd = False
+
+    def insert(self, word: str) -> None:
+        node = self
+        for ch in word:
+            ch = ord(ch) - ord("a")
+            if not node.children[ch]:
+                node.children[ch] = Trie()
+            node = node.children[ch]
+        node.isEnd = True
+
+    def search_prefix(self, prefix: str):
+        node = self
+        for ch in prefix:
+            ch = ord(ch) - ord("a")
+            if not node.children[ch]:
+                return None
+            node = node.children[ch]
+
+        return node
+
+    def search(self, word: str) -> bool:
+        node = self.search_prefix(word)
+        return node is not None and node.isEnd
+
+    def starts_with(self, prefix: str) -> bool:
+        return self.search_prefix(prefix) is not None
+
+
+if __name__ == "__main__":
+    bit_usage_demo = BITUsageDemo()
+    bit_usage_demo.add(1)
+    bit_usage_demo.add(2)
+    res = bit_usage_demo.query(1, 2)
+    print(res)
+    # 前缀树
+    # obj = Trie()
+    # obj.insert(word)
+    # param_2 = obj.search(word)
+    # param_3 = obj.startsWith(prefix)

nlpertools/algo/union.py

@@ -0,0 +1,13 @@
+#!/usr/bin/python3.8
+# -*- coding: utf-8 -*-
+# @Author  : youshu.Ji
+class UnionFind():
+    def __init__(self, num):
+        self.parent = list(range(num))
+
+    def union(self):
+        pass
+
+    def find(self, index):
+        if 1:
+            pass