pythainlp 5.0.0.dev0__tar.gz → 5.0.0.dev2__tar.gz

{pythainlp-5.0.0.dev0 → pythainlp-5.0.0.dev2}/CONTRIBUTING.md

@@ -135,10 +135,10 @@ Thanks to all [contributors](https://github.com/PyThaiNLP/pythainlp/graphs/contr
 - Wannaphong Phatthiyaphaibun <wannaphong@yahoo.com> - foundation, distribution and maintenance
 - Korakot Chaovavanich - initial tokenization and soundex codes
 - Charin Polpanumas - classification and benchmarking
-- Peeradej Tanruangporn - documentation
 - Arthit Suriyawongkul - refactoring, packaging, distribution, and maintenance
-- Chakri Lowphansirikul - documentation
+- Lalita Lowphansirikul - documentation
 - Pattarawat Chormai - benchmarking
+- Peerat Limkonchotiwat
 - Thanathip Suntorntip - nlpO3 maintenance, Rust Developer
 - Can Udomcharoenchaikit - documentation and codes
 
@@ -147,6 +147,9 @@ Thanks to all [contributors](https://github.com/PyThaiNLP/pythainlp/graphs/contr
 - Wannaphong Phatthiyaphaibun
 
 
+### Past
+- Peeradej Tanruangporn - documentation
+
 ## References
 
 - **[Maximum Matching]** -- Manabu Sassano. Deterministic Word Segmentation Using Maximum Matching with Fully Lexicalized Rules. Retrieved from http://www.aclweb.org/anthology/E14-4016

{pythainlp-5.0.0.dev0/pythainlp.egg-info → pythainlp-5.0.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pythainlp
-Version: 5.0.0.dev0
+Version: 5.0.0.dev2
 Summary: Thai Natural Language Processing library
 Home-page: https://github.com/PyThaiNLP/pythainlp
 Author: PyThaiNLP

{pythainlp-5.0.0.dev0 → pythainlp-5.0.0.dev2}/README.md

@@ -29,7 +29,7 @@ PyThaiNLP เป็นไลบารีภาษาไพทอนสำหร
 
 ## Getting Started
 
-- PyThaiNLP 2 requires Python 3.6+. Python 2.7 users can use PyThaiNLP 1.6. See [2.0 change log](https://github.com/PyThaiNLP/pythainlp/issues/118) | [Upgrading from 1.7](https://pythainlp.github.io/docs/2.0/notes/pythainlp-1_7-2_0.html) | [Upgrading ThaiNER from 1.7](https://github.com/PyThaiNLP/pythainlp/wiki/Upgrade-ThaiNER-from-PyThaiNLP-1.7-to-PyThaiNLP-2.0)
+- PyThaiNLP 2 requires Python 3.7+. Python 2.7 users can use PyThaiNLP 1.6. See [2.0 change log](https://github.com/PyThaiNLP/pythainlp/issues/118) | [Upgrading from 1.7](https://pythainlp.github.io/docs/2.0/notes/pythainlp-1_7-2_0.html) | [Upgrading ThaiNER from 1.7](https://github.com/PyThaiNLP/pythainlp/wiki/Upgrade-ThaiNER-from-PyThaiNLP-1.7-to-PyThaiNLP-2.0)
 - [PyThaiNLP Get Started notebook](https://pythainlp.github.io/tutorials/notebooks/pythainlp_get_started.html) | [API document](https://pythainlp.github.io/docs) | [Tutorials](https://pythainlp.github.io/tutorials)
 - [Official website](https://pythainlp.github.io/) | [PyPI](https://pypi.org/project/pythainlp/) | [Facebook page](https://www.facebook.com/pythainlp/)
 - [Who uses PyThaiNLP?](https://github.com/PyThaiNLP/pythainlp/blob/dev/INTHEWILD.md)
@@ -142,24 +142,60 @@ You can read [INTHEWILD.md](https://github.com/PyThaiNLP/pythainlp/blob/dev/INTH
 
 If you use `PyThaiNLP` in your project or publication, please cite the library as follows:
 
-```
 Wannaphong Phatthiyaphaibun, Korakot Chaovavanich, Charin Polpanumas, Arthit Suriyawongkul, Lalita Lowphansirikul, & Pattarawat Chormai. (2016, Jun 27). PyThaiNLP: Thai Natural Language Processing in Python. Zenodo. http://doi.org/10.5281/zenodo.3519354
-```
 
 or by BibTeX entry:
 
 ``` bib
 @misc{pythainlp,
-    author       = {Wannaphong Phatthiyaphaibun and Korakot Chaovavanich and Charin Polpanumas and Arthit Suriyawongkul and Lalita Lowphansirikul and Pattarawat Chormai},
-    title        = {{PyThaiNLP: Thai Natural Language Processing in Python}},
-    month        = Jun,
-    year         = 2016,
-    doi          = {10.5281/zenodo.3519354},
-    publisher    = {Zenodo},
-    url          = {http://doi.org/10.5281/zenodo.3519354}
+    title = "{P}y{T}hai{NLP}: {T}hai Natural Language Processing in {P}ython",
+    author = "Phatthiyaphaibun, Wannaphong  and
+      Chaovavanich, Korakot  and
+      Polpanumas, Charin  and
+      Suriyawongkul, Arthit  and
+      Lowphansirikul, Lalita  and
+      Chormai, Pattarawat",
+    month = jun,
+    year = "2016",
+    doi = {10.5281/zenodo.3519354},
+    publisher = {Zenodo},
+    url = {http://doi.org/10.5281/zenodo.3519354}
 }
 ```
 
+Our [NLP-OSS 2023](https://nlposs.github.io/2023/) paper:
+
+Wannaphong Phatthiyaphaibun, Korakot Chaovavanich, Charin Polpanumas, Arthit Suriyawongkul, Lalita Lowphansirikul, Pattarawat Chormai, Peerat Limkonchotiwat, Thanathip Suntorntip, and Can Udomcharoenchaikit. 2023. [PyThaiNLP: Thai Natural Language Processing in Python.](https://aclanthology.org/2023.nlposs-1.4) In Proceedings of the 3rd Workshop for Natural Language Processing Open Source Software (NLP-OSS 2023), pages 25–36, Singapore, Singapore. Empirical Methods in Natural Language Processing.
+
+and its BibTeX entry:
+
+```bib
+@inproceedings{phatthiyaphaibun-etal-2023-pythainlp,
+    title = "{P}y{T}hai{NLP}: {T}hai Natural Language Processing in {P}ython",
+    author = "Phatthiyaphaibun, Wannaphong  and
+      Chaovavanich, Korakot  and
+      Polpanumas, Charin  and
+      Suriyawongkul, Arthit  and
+      Lowphansirikul, Lalita  and
+      Chormai, Pattarawat  and
+      Limkonchotiwat, Peerat  and
+      Suntorntip, Thanathip  and
+      Udomcharoenchaikit, Can",
+    editor = "Tan, Liling  and
+      Milajevs, Dmitrijs  and
+      Chauhan, Geeticka  and
+      Gwinnup, Jeremy  and
+      Rippeth, Elijah",
+    booktitle = "Proceedings of the 3rd Workshop for Natural Language Processing Open Source Software (NLP-OSS 2023)",
+    month = dec,
+    year = "2023",
+    address = "Singapore, Singapore",
+    publisher = "Empirical Methods in Natural Language Processing",
+    url = "https://aclanthology.org/2023.nlposs-1.4",
+    pages = "25--36",
+    abstract = "We present PyThaiNLP, a free and open-source natural language processing (NLP) library for Thai language implemented in Python. It provides a wide range of software, models, and datasets for Thai language. We first provide a brief historical context of tools for Thai language prior to the development of PyThaiNLP. We then outline the functionalities it provided as well as datasets and pre-trained language models. We later summarize its development milestones and discuss our experience during its development. We conclude by demonstrating how industrial and research communities utilize PyThaiNLP in their work. The library is freely available at https://github.com/pythainlp/pythainlp.",
+}
+```
 
 ## Sponsors
 

{pythainlp-5.0.0.dev0 → pythainlp-5.0.0.dev2}/README_TH.md

@@ -123,13 +123,11 @@ thainlp help
 
 ## การอ้างอิง
 
-ถ้าคุณใช้ `PyThaiNLP` ในโปรเจคหรืองานวิจัยของคุณ คุณสามารถอ้างอิงได้ตามนี้
+หากคุณใช้ซอฟต์แวร์ `PyThaiNLP` ในโครงงานหรืองานวิจัยของคุณ คุณสามารถอ้างอิงได้ตามนี้
 
-```
 Wannaphong Phatthiyaphaibun, Korakot Chaovavanich, Charin Polpanumas, Arthit Suriyawongkul, Lalita Lowphansirikul, & Pattarawat Chormai. (2016, Jun 27). PyThaiNLP: Thai Natural Language Processing in Python. Zenodo. http://doi.org/10.5281/zenodo.3519354
-```
 
-หรือ BibTeX entry:
+โดยสามารถใช้ BibTeX นี้:
 
 ``` bib
 @misc{pythainlp,
@@ -143,6 +141,40 @@ Wannaphong Phatthiyaphaibun, Korakot Chaovavanich, Charin Polpanumas, Arthit Sur
 }
 ```
 
+บทความของเราในงานประชุมวิชาการ [NLP-OSS 2023](https://nlposs.github.io/2023/):
+
+Wannaphong Phatthiyaphaibun, Korakot Chaovavanich, Charin Polpanumas, Arthit Suriyawongkul, Lalita Lowphansirikul, Pattarawat Chormai, Peerat Limkonchotiwat, Thanathip Suntorntip, and Can Udomcharoenchaikit. 2023. [PyThaiNLP: Thai Natural Language Processing in Python.](https://aclanthology.org/2023.nlposs-1.4) In Proceedings of the 3rd Workshop for Natural Language Processing Open Source Software (NLP-OSS 2023), pages 25–36, Singapore, Singapore. Empirical Methods in Natural Language Processing.
+
+โดยสามารถใช้ BibTeX นี้:
+
+```bib
+@inproceedings{phatthiyaphaibun-etal-2023-pythainlp,
+    title = "{P}y{T}hai{NLP}: {T}hai Natural Language Processing in Python",
+    author = "Phatthiyaphaibun, Wannaphong  and
+      Chaovavanich, Korakot  and
+      Polpanumas, Charin  and
+      Suriyawongkul, Arthit  and
+      Lowphansirikul, Lalita  and
+      Chormai, Pattarawat  and
+      Limkonchotiwat, Peerat  and
+      Suntorntip, Thanathip  and
+      Udomcharoenchaikit, Can",
+    editor = "Tan, Liling  and
+      Milajevs, Dmitrijs  and
+      Chauhan, Geeticka  and
+      Gwinnup, Jeremy  and
+      Rippeth, Elijah",
+    booktitle = "Proceedings of the 3rd Workshop for Natural Language Processing Open Source Software (NLP-OSS 2023)",
+    month = dec,
+    year = "2023",
+    address = "Singapore, Singapore",
+    publisher = "Empirical Methods in Natural Language Processing",
+    url = "https://aclanthology.org/2023.nlposs-1.4",
+    pages = "25--36",
+    abstract = "We present PyThaiNLP, a free and open-source natural language processing (NLP) library for Thai language implemented in Python. It provides a wide range of software, models, and datasets for Thai language. We first provide a brief historical context of tools for Thai language prior to the development of PyThaiNLP. We then outline the functionalities it provided as well as datasets and pre-trained language models. We later summarize its development milestones and discuss our experience during its development. We conclude by demonstrating how industrial and research communities utilize PyThaiNLP in their work. The library is freely available at https://github.com/pythainlp/pythainlp.",
+}
+```
+
 ## ร่วมสนับสนุน PyThaiNLP
 
 - กรุณา fork แล้วพัฒนาต่อ จากนั้นสร้าง pull request กลับมา :)
@@ -157,10 +189,10 @@ Wannaphong Phatthiyaphaibun, Korakot Chaovavanich, Charin Polpanumas, Arthit Sur
 
 | | สัญญาอนุญาต |
 |:---|:----|
-| PyThaiNLP Source Code and Notebooks | [Apache Software License 2.0](https://github.com/PyThaiNLP/pythainlp/blob/dev/LICENSE) |
-| Corpora, datasets, and documentations created by PyThaiNLP | [Creative Commons Zero 1.0 Universal Public Domain Dedication License (CC0)](https://creativecommons.org/publicdomain/zero/1.0/)|
+| ต้นรหัสซอร์สโค้ดและโน๊ตบุ๊กของ PyThaiNLP | [Apache Software License 2.0](https://github.com/PyThaiNLP/pythainlp/blob/dev/LICENSE) |
+| ฐานข้อมูลภาษา ชุดข้อมูล และเอกสารที่สร้างโดยโครงการ PyThaiNLP | [Creative Commons Zero 1.0 Universal Public Domain Dedication License (CC0)](https://creativecommons.org/publicdomain/zero/1.0/)|
 | Language models created by PyThaiNLP | [Creative Commons Attribution 4.0 International Public License (CC-by)](https://creativecommons.org/licenses/by/4.0/)  |
-| Other corpora and models that may included with PyThaiNLP | See [Corpus License](https://github.com/PyThaiNLP/pythainlp/blob/dev/pythainlp/corpus/corpus_license.md) |
+| สำหรับฐานข้อมูลภาษาและโมเดลอื่นที่อาจมาพร้อมกับซอฟต์แวร์ PyThaiNLP | ดู [Corpus License](https://github.com/PyThaiNLP/pythainlp/blob/dev/pythainlp/corpus/corpus_license.md) |
 
 
 ## บัตรโมเดล

pythainlp-5.0.0.dev2/pyproject.toml

@@ -0,0 +1,16 @@
+[tool.ruff]
+line-length = 79
+indent-width = 4
+target-version = "py38"
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+skip-magic-trailing-comma = false
+line-ending = "auto"
+docstring-code-format = true
+
+[tool.ruff.lint.mccabe]
+# Flag errors (`C901`) whenever the complexity level exceeds 5. Default is 10.
+# We should aim to gradually reduce this to 10.
+max-complexity = 40

{pythainlp-5.0.0.dev0 → pythainlp-5.0.0.dev2}/pythainlp/__init__.py

@@ -1,23 +1,7 @@
 # -*- coding: utf-8 -*-
-# PyThaiNLP: Thai Natural Language Processing in Python
-#
-# Copyright (C) 2016-2023 PyThaiNLP Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# URL: <https://pythainlp.github.io/>
-# For license information, see LICENSE
-__version__ = "5.0.0dev0"
+# SPDX-FileCopyrightText: Copyright 2016-2024 PyThaiNLP Project
+# SPDX-License-Identifier: Apache-2.0
+__version__ = "5.0.0dev2"
 
 thai_consonants = "กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรลวศษสหฬอฮ"  # 44 chars
 

{pythainlp-5.0.0.dev0 → pythainlp-5.0.0.dev2}/pythainlp/__main__.py

@@ -1,17 +1,6 @@
 # -*- coding: utf-8 -*-
-# Copyright (C) 2016-2023 PyThaiNLP Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# SPDX-FileCopyrightText: Copyright 2016-2024 PyThaiNLP Project
+# SPDX-License-Identifier: Apache-2.0
 import argparse
 import sys
 

pythainlp-5.0.0.dev2/pythainlp/ancient/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: Copyright 2016-2024 PyThaiNLP Project
+# SPDX-License-Identifier: Apache-2.0
+"""
+Ancient versions of the Thai language
+"""
+
+__all__ = ["aksonhan_to_current"]
+
+from pythainlp.ancient.aksonhan import aksonhan_to_current

{pythainlp-5.0.0.dev0 → pythainlp-5.0.0.dev2}/pythainlp/ancient/aksonhan.py

@@ -1,34 +1,23 @@
 # -*- coding: utf-8 -*-
-# Copyright (C) 2016-2023 PyThaiNLP Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# SPDX-FileCopyrightText: Copyright 2016-2024 PyThaiNLP Project
+# SPDX-License-Identifier: Apache-2.0
 from pythainlp.util import Trie
-from pythainlp import thai_consonants,thai_tonemarks
+from pythainlp import thai_consonants, thai_tonemarks
 from pythainlp.tokenize import Tokenizer
 from pythainlp.corpus import thai_orst_words
 
 
 _dict_aksonhan = {}
 for i in list(thai_consonants):
-    if i=="ร":
+    if i == "ร":
         continue
     for j in list(thai_tonemarks):
-        _dict_aksonhan[i+j+i] = "ั"+j+i
-        _dict_aksonhan[i+i+j+i] = i+"ั"+j+i
-    _dict_aksonhan[i+i] = "ั"+i
+        _dict_aksonhan[i + j + i] = "ั" + j + i
+        _dict_aksonhan[i + i + j + i] = i + "ั" + j + i
+    _dict_aksonhan[i + i] = "ั" + i
 _set_aksonhan = set(_dict_aksonhan.keys())
-_trie = Trie(list(_dict_aksonhan.keys())+list(thai_consonants))
-_tokenizer = Tokenizer(custom_dict=_trie,engine="mm")
+_trie = Trie(list(_dict_aksonhan.keys()) + list(thai_consonants))
+_tokenizer = Tokenizer(custom_dict=_trie, engine="mm")
 _dict_thai = set(thai_orst_words())  # call Thai words
 
 
@@ -63,8 +52,9 @@ def aksonhan_to_current(word: str) -> str:
         return word
     elif word in _set_aksonhan:
         return _dict_aksonhan[word]
-    elif word in _dict_thai: # word in Thai words
+    elif word in _dict_thai:  # word in Thai words
         return word
+
     _seg = _tokenizer.word_tokenize(word)
     _w = []
     for i in _seg:
@@ -72,4 +62,4 @@ def aksonhan_to_current(word: str) -> str:
             _w.append(_dict_aksonhan[i])
         else:
             _w.append(i)
-    return ''.join(_w)
+    return "".join(_w)

pythainlp-5.0.0.dev2/pythainlp/augment/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: Copyright 2016-2024 PyThaiNLP Project
+# SPDX-License-Identifier: Apache-2.0
+"""
+Thai text augment
+"""
+
+__all__ = ["WordNetAug"]
+
+from pythainlp.augment.wordnet import WordNetAug

pythainlp-5.0.0.dev2/pythainlp/augment/lm/__init__.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: Copyright 2016-2024 PyThaiNLP Project
+# SPDX-License-Identifier: Apache-2.0
+"""
+Language Models
+"""
+
+__all__ = [
+    "FastTextAug",
+    "Thai2transformersAug",
+    "ThaiTextAugmenter",
+]
+
+from pythainlp.augment.lm.fasttext import FastTextAug
+from pythainlp.augment.lm.phayathaibert import ThaiTextAugmenter
+from pythainlp.augment.lm.wangchanberta import Thai2transformersAug

{pythainlp-5.0.0.dev0 → pythainlp-5.0.0.dev2}/pythainlp/augment/lm/fasttext.py

@@ -1,17 +1,6 @@
 # -*- coding: utf-8 -*-
-# Copyright (C) 2016-2023 PyThaiNLP Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# SPDX-FileCopyrightText: Copyright 2016-2024 PyThaiNLP Project
+# SPDX-License-Identifier: Apache-2.0
 import itertools
 from typing import List, Tuple
 from gensim.models.fasttext import FastText as FastText_gensim

pythainlp-5.0.0.dev2/pythainlp/augment/lm/phayathaibert.py

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: Copyright 2016-2024 PyThaiNLP Project
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import List
+import random
+import re
+
+from pythainlp.phayathaibert.core import ThaiTextProcessor
+
+
+_MODEL_NAME = "clicknext/phayathaibert"
+
+
+class ThaiTextAugmenter:
+    def __init__(self) -> None:
+        from transformers import (
+            AutoTokenizer,
+            AutoModelForMaskedLM,
+            pipeline,
+        )
+
+        self.tokenizer = AutoTokenizer.from_pretrained(_MODEL_NAME)
+        self.model_for_masked_lm = AutoModelForMaskedLM.from_pretrained(
+            _MODEL_NAME
+        )
+        self.model = pipeline(
+            "fill-mask",
+            tokenizer=self.tokenizer,
+            model=self.model_for_masked_lm,
+        )
+        self.processor = ThaiTextProcessor()
+
+    def generate(
+        self,
+        sample_text: str,
+        word_rank: int,
+        max_length: int = 3,
+        sample: bool = False,
+    ) -> str:
+        sample_txt = sample_text
+        final_text = ""
+
+        for j in range(max_length):
+            input = self.processor.preprocess(sample_txt)
+            if sample:
+                random_word_idx = random.randint(0, 4)
+                output = self.model(input)[random_word_idx]["sequence"]
+            else:
+                output = self.model(input)[word_rank]["sequence"]
+            sample_txt = output + "<mask>"
+            final_text = sample_txt
+
+        gen_txt = re.sub("<mask>", "", final_text)
+
+        return gen_txt
+
+    def augment(
+        self, text: str, num_augs: int = 3, sample: bool = False
+    ) -> List[str]:
+        """
+        Text augmentation from PhayaThaiBERT
+
+        :param str text: Thai text
+        :param int num_augs: an amount of augmentation text needed as an output
+        :param bool sample: whether to sample the text as an output or not, \
+                            true if more word diversity is needed
+
+        :return: list of text augment
+        :rtype: List[str]
+
+        :Example:
+        ::
+
+            from pythainlp.augment.lm import ThaiTextAugmenter
+
+            aug = ThaiTextAugmenter()
+            aug.augment("ช้างมีทั้งหมด 50 ตัว บน", num_args=5)
+
+            # output = ['ช้างมีทั้งหมด 50 ตัว บนโลกใบนี้ครับ.',
+                'ช้างมีทั้งหมด 50 ตัว บนพื้นดินครับ...',
+                'ช้างมีทั้งหมด 50 ตัว บนท้องฟ้าครับ...',
+                'ช้างมีทั้งหมด 50 ตัว บนดวงจันทร์.‼',
+                'ช้างมีทั้งหมด 50 ตัว บนเขาค่ะ😁']
+        """
+        MAX_NUM_AUGS = 5
+        augment_list = []
+
+        if "<mask>" not in text:
+            text = text + "<mask>"
+
+        if num_augs <= MAX_NUM_AUGS:
+            for rank in range(num_augs):
+                gen_text = self.generate(text, rank, sample=sample)
+                processed_text = re.sub(
+                    "<_>", " ", self.processor.preprocess(gen_text)
+                )
+                augment_list.append(processed_text)
+        else:
+            raise ValueError(
+                f"augmentation of more than {num_augs} is exceeded \
+                    the default limit: {MAX_NUM_AUGS}"
+            )
+
+        return augment_list

{pythainlp-5.0.0.dev0 → pythainlp-5.0.0.dev2}/pythainlp/augment/lm/wangchanberta.py

@@ -1,18 +1,9 @@
 # -*- coding: utf-8 -*-
-# Copyright (C) 2016-2023 PyThaiNLP Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# SPDX-FileCopyrightText: Copyright 2016-2024 PyThaiNLP Project
+# SPDX-License-Identifier: Apache-2.0
+
 from typing import List
+
 from transformers import (
     CamembertTokenizer,
     pipeline,
@@ -62,9 +53,9 @@ class Thai2transformersAug:
 
     def augment(self, sentence: str, num_replace_tokens: int = 3) -> List[str]:
         """
-        Text Augment from wangchanberta
+        Text augmentation from WangchanBERTa
 
-        :param str sentence: thai sentence
+        :param str sentence: Thai sentence
         :param int num_replace_tokens: number replace tokens
 
         :return: list of text augment
@@ -75,7 +66,7 @@ class Thai2transformersAug:
 
             from pythainlp.augment.lm import Thai2transformersAug
 
-            aug=Thai2transformersAug()
+            aug = Thai2transformersAug()
 
             aug.augment("ช้างมีทั้งหมด 50 ตัว บน")
             # output: ['ช้างมีทั้งหมด 50 ตัว บนโลกใบนี้',

pythainlp-5.0.0.dev2/pythainlp/augment/word2vec/__init__.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+# SPDX-FileCopyrightText: Copyright 2016-2024 PyThaiNLP Project
+# SPDX-License-Identifier: Apache-2.0
+"""
+Word2Vec
+"""
+
+__all__ = ["Word2VecAug", "Thai2fitAug", "LTW2VAug"]
+
+from pythainlp.augment.word2vec.core import Word2VecAug
+from pythainlp.augment.word2vec.thai2fit import Thai2fitAug
+from pythainlp.augment.word2vec.ltw2v import LTW2VAug

{pythainlp-5.0.0.dev0 → pythainlp-5.0.0.dev2}/pythainlp/augment/word2vec/bpemb_wv.py

@@ -1,17 +1,6 @@
 # -*- coding: utf-8 -*-
-# Copyright (C) 2016-2023 PyThaiNLP Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# SPDX-FileCopyrightText: Copyright 2016-2024 PyThaiNLP Project
+# SPDX-License-Identifier: Apache-2.0
 from typing import List, Tuple
 from pythainlp.augment.word2vec.core import Word2VecAug
 

{pythainlp-5.0.0.dev0 → pythainlp-5.0.0.dev2}/pythainlp/augment/word2vec/core.py

@@ -1,17 +1,6 @@
 # -*- coding: utf-8 -*-
-# Copyright (C) 2016-2023 PyThaiNLP Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# SPDX-FileCopyrightText: Copyright 2016-2024 PyThaiNLP Project
+# SPDX-License-Identifier: Apache-2.0
 from typing import List, Tuple
 import itertools
 

{pythainlp-5.0.0.dev0 → pythainlp-5.0.0.dev2}/pythainlp/augment/word2vec/ltw2v.py

@@ -1,17 +1,6 @@
 # -*- coding: utf-8 -*-
-# Copyright (C) 2016-2023 PyThaiNLP Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# SPDX-FileCopyrightText: Copyright 2016-2024 PyThaiNLP Project
+# SPDX-License-Identifier: Apache-2.0
 from typing import List, Tuple
 from pythainlp.augment.word2vec.core import Word2VecAug
 from pythainlp.corpus import get_corpus_path

{pythainlp-5.0.0.dev0 → pythainlp-5.0.0.dev2}/pythainlp/augment/word2vec/thai2fit.py

@@ -1,17 +1,6 @@
 # -*- coding: utf-8 -*-
-# Copyright (C) 2016-2023 PyThaiNLP Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# SPDX-FileCopyrightText: Copyright 2016-2024 PyThaiNLP Project
+# SPDX-License-Identifier: Apache-2.0
 from typing import List, Tuple
 from pythainlp.augment.word2vec.core import Word2VecAug
 from pythainlp.corpus import get_corpus_path

{pythainlp-5.0.0.dev0 → pythainlp-5.0.0.dev2}/pythainlp/augment/wordnet.py

@@ -1,17 +1,6 @@
 # -*- coding: utf-8 -*-
-# Copyright (C) 2016-2023 PyThaiNLP Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# SPDX-FileCopyrightText: Copyright 2016-2024 PyThaiNLP Project
+# SPDX-License-Identifier: Apache-2.0
 """
 Thank https://dev.to/ton_ami/text-data-augmentation-synonym-replacement-4h8l
 """
@@ -23,6 +12,7 @@ __all__ = [
 from collections import OrderedDict
 import itertools
 from typing import List
+
 from nltk.corpus import wordnet as wn
 from pythainlp.corpus import wordnet
 from pythainlp.tokenize import word_tokenize