fast-langdetect 0.1.0__tar.gz

fast_langdetect-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 LLM Kira
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

fast_langdetect-0.1.0/PKG-INFO

@@ -0,0 +1,77 @@
+Metadata-Version: 2.1
+Name: fast-langdetect
+Version: 0.1.0
+Summary: Quickly detect text language and segment language
+Author-Email: sudoskys <coldlando@hotmail.com>
+License: MIT
+Requires-Python: <3.12,>=3.8
+Requires-Dist: fasttext-wheel>=0.9.2
+Requires-Dist: requests>=2.31.0
+Requires-Dist: robust-downloader>=0.0.2
+Description-Content-Type: text/markdown
+
+# fast-langdetect
+
+[![PyPI version](https://badge.fury.io/py/fast-langdetect.svg)](https://badge.fury.io/py/fast-langdetect)
+[![Downloads](https://pepy.tech/badge/fast-langdetect)](https://pepy.tech/project/fast-langdetect)
+[![Downloads](https://pepy.tech/badge/fast-langdetect/month)](https://pepy.tech/project/fast-langdetect/month)
+
+Python 3.8-3.11 support only.
+
+80x faster and 95% accurate language identification with Fasttext
+
+This library is a wrapper for the language detection model trained on fasttext by Facebook. For more information, please
+visit: https://fasttext.cc/docs/en/language-identification.html
+
+This repository is patched
+from [zafercavdar/fasttext-langdetect](https://github.com/zafercavdar/fasttext-langdetect#benchmark), adding
+multi-language segmentation and better packaging
+support.
+
+Facilitates more accurate TTS implementation.
+
+## Installation
+
+```bash
+pip install fast-langdetect
+```
+
+## Usage
+
+**For more accurate language detection, please use `detect(text,low_memory=False)` to load the big model.**
+
+**Model will be downloaded in `/tmp/fasttext-langdetect` directory when you first use it.**
+
+```python
+from fast_langdetect import detect_langs
+
+print(detect_langs("Hello, world!"))
+# [en:0.9999961853027344]
+
+print(detect_langs("Привет, мир!"))
+# [ru:0.9999961853027344]
+
+
+print(detect_langs("你好，世界！"))
+# [zh:0.9999961853027344]
+
+```
+
+## Advanced usage
+
+```python
+from fast_langdetect import detect, parse_sentence, detect_multilingual
+
+print(detect("Hello, world!"))
+# {'lang': 'en', 'score': 0.1520957201719284}
+
+print(detect_multilingual("Hello, world!你好世界!Привет, мир!"))
+# [{'lang': 'ru', 'score': 0.39008623361587524}, {'lang': 'zh', 'score': 0.18235979974269867}, {'lang': 'ja', 'score': 0.08473210036754608}, {'lang': 'sr', 'score': 0.057975586503744125}, {'lang': 'en', 'score': 0.05422825738787651}]
+
+print(parse_sentence("你好世界！Hello, world！Привет, мир！"))
+# [{'text': '你好世界！Hello, world！', 'lang': 'ZH', 'length': 18}, {'text': 'Привет, мир！', 'lang': 'UK', 'length': 12}, {'text': '', 'lang': 'EN', 'length': 0}]
+```
+
+## Accuracy
+
+References to the [benchmark](https://github.com/zafercavdar/fasttext-langdetect#benchmark)

fast_langdetect-0.1.0/README.md

@@ -0,0 +1,65 @@
+# fast-langdetect
+
+[![PyPI version](https://badge.fury.io/py/fast-langdetect.svg)](https://badge.fury.io/py/fast-langdetect)
+[![Downloads](https://pepy.tech/badge/fast-langdetect)](https://pepy.tech/project/fast-langdetect)
+[![Downloads](https://pepy.tech/badge/fast-langdetect/month)](https://pepy.tech/project/fast-langdetect/month)
+
+Python 3.8-3.11 support only.
+
+80x faster and 95% accurate language identification with Fasttext
+
+This library is a wrapper for the language detection model trained on fasttext by Facebook. For more information, please
+visit: https://fasttext.cc/docs/en/language-identification.html
+
+This repository is patched
+from [zafercavdar/fasttext-langdetect](https://github.com/zafercavdar/fasttext-langdetect#benchmark), adding
+multi-language segmentation and better packaging
+support.
+
+Facilitates more accurate TTS implementation.
+
+## Installation
+
+```bash
+pip install fast-langdetect
+```
+
+## Usage
+
+**For more accurate language detection, please use `detect(text,low_memory=False)` to load the big model.**
+
+**Model will be downloaded in `/tmp/fasttext-langdetect` directory when you first use it.**
+
+```python
+from fast_langdetect import detect_langs
+
+print(detect_langs("Hello, world!"))
+# [en:0.9999961853027344]
+
+print(detect_langs("Привет, мир!"))
+# [ru:0.9999961853027344]
+
+
+print(detect_langs("你好，世界！"))
+# [zh:0.9999961853027344]
+
+```
+
+## Advanced usage
+
+```python
+from fast_langdetect import detect, parse_sentence, detect_multilingual
+
+print(detect("Hello, world!"))
+# {'lang': 'en', 'score': 0.1520957201719284}
+
+print(detect_multilingual("Hello, world!你好世界!Привет, мир!"))
+# [{'lang': 'ru', 'score': 0.39008623361587524}, {'lang': 'zh', 'score': 0.18235979974269867}, {'lang': 'ja', 'score': 0.08473210036754608}, {'lang': 'sr', 'score': 0.057975586503744125}, {'lang': 'en', 'score': 0.05422825738787651}]
+
+print(parse_sentence("你好世界！Hello, world！Привет, мир！"))
+# [{'text': '你好世界！Hello, world！', 'lang': 'ZH', 'length': 18}, {'text': 'Привет, мир！', 'lang': 'UK', 'length': 12}, {'text': '', 'lang': 'EN', 'length': 0}]
+```
+
+## Accuracy
+
+References to the [benchmark](https://github.com/zafercavdar/fasttext-langdetect#benchmark)

fast_langdetect-0.1.0/pyproject.toml

@@ -0,0 +1,31 @@
+[project]
+name = "fast-langdetect"
+version = "0.1.0"
+description = "Quickly detect text language and segment language"
+authors = [
+    { name = "sudoskys", email = "coldlando@hotmail.com" },
+]
+dependencies = [
+    "fasttext-wheel>=0.9.2",
+    "requests>=2.31.0",
+    "robust-downloader>=0.0.2",
+]
+requires-python = ">=3.8,<3.12"
+readme = "README.md"
+
+[project.license]
+text = "MIT"
+
+[build-system]
+requires = [
+    "pdm-backend",
+]
+build-backend = "pdm.backend"
+
+[tool.pdm]
+package-type = "library"
+
+[tool.pdm.dev-dependencies]
+dev = [
+    "pytest>=7.4.4",
+]

fast_langdetect-0.1.0/src/fast_langdetect/__init__.py

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+
+from .ft_detect import detect, detect_langs, detect_multilingual  # noqa: F401
+from .split import parse_sentence  # noqa: F401

fast_langdetect-0.1.0/src/fast_langdetect/ft_detect/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Zafer Çavdar
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

fast_langdetect-0.1.0/src/fast_langdetect/ft_detect/__init__.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/1/17 下午4:00
+# @Author  : sudoskys
+# @File    : __init__.py
+from .infer import detect
+from .infer import detect_multilingual  # noqa: F401
+
+
+def is_japanese(string):
+    for ch in string:
+        if 0x3040 < ord(ch) < 0x30FF:
+            return True
+    return False
+
+
+def detect_langs(sentence, *, low_memory: bool = True):
+    """
+    Detect language
+    :param sentence: str sentence
+    :param low_memory: bool (default: True) whether to use low memory mode
+    :return: ZH, EN, JA, KO, FR, DE, ES, .... (two uppercase letters)
+    """
+    lang_code = detect(sentence, low_memory=low_memory).get("lang").upper()
+    if lang_code == "JA" and not is_japanese(sentence):
+        lang_code = "ZH"
+    return lang_code

fast_langdetect-0.1.0/src/fast_langdetect/ft_detect/infer.py

@@ -0,0 +1,107 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/1/17 下午8:30
+# @Author  : sudoskys
+# @File    : infer.py
+# @Software: PyCharm
+import logging
+import os
+from pathlib import Path
+from typing import Dict, Union, List
+
+import fasttext
+from robust_downloader import download
+
+logger = logging.getLogger(__name__)
+MODELS = {"low_mem": None, "high_mem": None}
+FTLANG_CACHE = os.getenv("FTLANG_CACHE", "/tmp/fasttext-langdetect")
+
+try:
+    # silences warnings as the package does not properly use the python 'warnings' package
+    # see https://github.com/facebookresearch/fastText/issues/1056
+    fasttext.FastText.eprint = lambda *args, **kwargs: None
+except Exception:
+    pass
+
+
+class DetectError(Exception):
+    pass
+
+
+def get_model_map(low_memory=False):
+    """
+    Getting model map
+    :param low_memory:
+    :return:
+    """
+    if low_memory:
+        return "low_mem", FTLANG_CACHE, "lid.176.ftz", "https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.ftz"
+    else:
+        return "high_mem", FTLANG_CACHE, "lid.176.bin", "https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin"
+
+
+def get_model_loaded(
+        low_memory: bool = False,
+        download_proxy: str = None
+):
+    """
+    Getting model loaded
+    :param low_memory:
+    :param download_proxy:
+    :return:
+    """
+    mode, cache, name, url = get_model_map(low_memory)
+    loaded = MODELS.get(mode, None)
+    if loaded:
+        return loaded
+    model_path = os.path.join(cache, name)
+    if Path(model_path).exists():
+        if Path(model_path).is_dir():
+            raise Exception(f"{model_path} is a directory")
+        try:
+            loaded_model = fasttext.load_model(model_path)
+            MODELS[mode] = loaded_model
+        except Exception as e:
+            logger.error(f"Error loading model {model_path}: {e}")
+            download(url=url, folder=cache, filename=name, proxy=download_proxy)
+            raise e
+        else:
+            return loaded_model
+
+    download(url=url, folder=cache, filename=name, proxy=download_proxy, retry_max=3, timeout=20)
+    loaded_model = fasttext.load_model(model_path)
+    MODELS[mode] = loaded_model
+    return loaded_model
+
+
+def detect(text: str, *,
+           low_memory: bool = True,
+           model_download_proxy: str = None
+           ) -> Dict[str, Union[str, float]]:
+    model = get_model_loaded(low_memory=low_memory, download_proxy=model_download_proxy)
+    labels, scores = model.predict(text)
+    label = labels[0].replace("__label__", '')
+    score = min(float(scores[0]), 1.0)
+    return {
+        "lang": label,
+        "score": score,
+    }
+
+
+def detect_multilingual(text: str, *,
+                        low_memory: bool = True,
+                        model_download_proxy: str = None,
+                        k: int = 5,
+                        threshold: float = 0.0,
+                        on_unicode_error: str = "strict"
+                        ) -> List[dict]:
+    model = get_model_loaded(low_memory=low_memory, download_proxy=model_download_proxy)
+    labels, scores = model.predict(text=text, k=k, threshold=threshold, on_unicode_error=on_unicode_error)
+    detect_result = []
+    for label, score in zip(labels, scores):
+        label = label.replace("__label__", '')
+        score = min(float(score), 1.0)
+        detect_result.append({
+            "lang": label,
+            "score": score,
+        })
+    return sorted(detect_result, key=lambda i: i['score'], reverse=True)

fast_langdetect-0.1.0/src/fast_langdetect/split/__init__.py

@@ -0,0 +1,66 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/1/17 下午4:23
+# @Author  : sudoskys
+# @File    : __init__.py.py
+from typing import List
+
+from .cut import CutSentence
+from ..ft_detect import detect_langs
+
+CUT = CutSentence()
+
+
+def _merge_cell(result: List[dict]) -> List[dict]:
+    _merged = []
+    _cache = []
+    last_lang = None
+    for _result in result:
+        if _result["lang"] == last_lang:
+            _cache.append(_result["text"])
+        else:
+            if _cache:
+                _length = sum([len(_c) for _c in _cache])
+                _merged.append({"text": "".join(_cache), "lang": last_lang, "length": _length})
+            _cache = [_result["text"]]
+            last_lang = _result["lang"]
+    if _cache:
+        _length = sum([len(_c) for _c in _cache])
+        _merged.append({"text": "".join(_cache), "lang": last_lang, "length": _length})
+    return _merged
+
+
+def parse_sentence(sentence: str,
+                   merge_same: bool = True,
+                   cell_limit: int = 150,
+                   filter_space: bool = True,
+                   low_memory: bool = True
+                   ) -> list:
+    """
+    Parse sentence
+    :param sentence:
+    :param merge_same:
+    :param cell_limit:
+    :param filter_space:
+    :param low_memory:
+    :return:
+    """
+
+    cut_list = CUT.chinese_sentence_cut(sentence)
+    _cut_list = []
+    for _cut in cut_list:
+        if len(_cut) > cell_limit:
+            _text_list = [_cut[i:i + cell_limit] for i in range(0, len(_cut), cell_limit)]
+            _cut_list.extend(_text_list)
+        else:
+            _cut_list.append(_cut)
+    _result = []
+    for _cut in _cut_list:
+        _lang = detect_langs(_cut, low_memory=low_memory)
+        if not filter_space:
+            _result.append({"text": _cut, "lang": _lang, "length": len(_cut)})
+        else:
+            if _lang:
+                _result.append({"text": _cut, "lang": _lang, "length": len(_cut)})
+    if merge_same:
+        _result = _merge_cell(_result)
+    return _result

fast_langdetect-0.1.0/src/fast_langdetect/split/cut.py

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/1/17 下午8:27
+# @Author  : sudoskys
+# @File    : cut.py
+# @Software: PyCharm
+import re
+
+
+class CutSentence(object):
+    @staticmethod
+    def english_sentence_cut(text) -> list:
+        list_ = list()
+        for s_str in text.split('.'):
+            if '?' in s_str:
+                list_.extend(s_str.split('?'))
+            elif '!' in s_str:
+                list_.extend(s_str.split('!'))
+            else:
+                list_.append(s_str)
+        return list_
+
+    @staticmethod
+    def chinese_sentence_cut(text) -> list:
+        """
+        中文断句
+        """
+        text = re.sub('([^\n])([.!?。！？]+)(?!\d|[a-zA-Z]|[^\s\u4e00-\u9fa5\u3040-\u309f\u30a0-\u30ff\uac00-\ud7a3])',
+                      r'\1\2\n', text)  # 根据句子末尾的标点符号进行切分
+        text = re.sub('([。：:！？\?])([^’”])', r'\1\n\2', text)  # 根据中文断句符号进行切分
+        text = re.sub('(\.{6})([^’”])', r'\1\n\2', text)  # 普通断句符号且后面没有引号
+        text = re.sub('(\…{2})([^’”])', r'\1\n\2', text)  # 英文省略号且后面没有引号
+        text = re.sub('([.。！？\?\.{6}\…{2}][’”])([^’”])', r'\1\n\2', text)  # 中文省略号且后面没有引号
+        text = re.sub('(\. )([^a-zA-Z\d])', r'\1\n\2', text)  # 根据英文断句号+空格进行切分
+        text = re.sub('\n\n+', '\n', text)  # 删除多余的换行符
+        return text.split("\n")  # 断句号+引号且后面没有引号
+
+    def cut_chinese_sentence(self, text):
+        """
+        中文断句
+        """
+        p = re.compile("“.*?”")
+        listr = []
+        index = 0
+        for i in p.finditer(text):
+            temp = ''
+            start = i.start()
+            end = i.end()
+            for j in range(index, start):
+                temp += text[j]
+            if temp != '':
+                temp_list = self.chinese_sentence_cut(temp)
+                listr += temp_list
+            temp = ''
+            for k in range(start, end):
+                temp += text[k]
+            if temp != ' ':
+                listr.append(temp)
+            index = end
+        return listr

fast_langdetect-0.1.0/tests/__init__.py

@@ -0,0 +1,3 @@
+
+
+

fast_langdetect-0.1.0/tests/test_detect.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# @Time    : 2024/1/17 下午5:28
+# @Author  : sudoskys
+# @File    : test_detect.py
+# @Software: PyCharm
+
+
+def test_muti_detect():
+    from fast_langdetect.ft_detect import detect_multilingual
+    result = detect_multilingual("hello world", low_memory=True)
+    assert result[0].get("lang") == "en", "ft_detect error"
+
+
+def test_detect_totally():
+    from fast_langdetect import detect_langs
+    assert detect_langs("hello world") == "EN", "ft_detect error"
+    assert detect_langs("你好世界") == "ZH", "ft_detect error"
+    assert detect_langs("こんにちは世界") == "JA", "ft_detect error"
+
+
+def test_parse():
+    from fast_langdetect import parse_sentence
+    assert parse_sentence("hello world") == [{"text": "hello world", "lang": "EN", "length": 11}], "ft_detect error"