BackTranslation 0.3.1__tar.gz → 0.5.0__tar.gz

{BackTranslation-0.3.1 → backtranslation-0.5.0}/BackTranslation/__init__.py

@@ -1,2 +1,2 @@
-from BackTranslation.translation import BackTranslation
+from BackTranslation.translation import BackTranslation
 from BackTranslation.translation_Baidu import BackTranslation_Baidu

{BackTranslation-0.3.1 → backtranslation-0.5.0}/BackTranslation/languages.py

@@ -1,109 +1,110 @@
-LANG_CODES = {
-    'afrikaans': 'af',
-    'albanian': 'sq',
-    'amharic': 'am',
-    'arabic': 'ar',
-    'armenian': 'hy',
-    'azerbaijani': 'az',
-    'basque': 'eu',
-    'belarusian': 'be',
-    'bengali': 'bn',
-    'bosnian': 'bs',
-    'bulgarian': 'bg',
-    'catalan': 'ca',
-    'cebuano': 'ceb',
-    'chichewa': 'ny',
-    'chinese (simplified)': 'zh-cn',
-    'chinese (traditional)': 'zh-tw',
-    'corsican': 'co',
-    'croatian': 'hr',
-    'czech': 'cs',
-    'danish': 'da',
-    'dutch': 'nl',
-    'english': 'en',
-    'esperanto': 'eo',
-    'estonian': 'et',
-    'filipino': 'tl',
-    'finnish': 'fi',
-    'french': 'fr',
-    'frisian': 'fy',
-    'galician': 'gl',
-    'georgian': 'ka',
-    'german': 'de',
-    'greek': 'el',
-    'gujarati': 'gu',
-    'haitian creole': 'ht',
-    'hausa': 'ha',
-    'hawaiian': 'haw',
-    'hebrew': 'he',
-    'hindi': 'hi',
-    'hmong': 'hmn',
-    'hungarian': 'hu',
-    'icelandic': 'is',
-    'igbo': 'ig',
-    'indonesian': 'id',
-    'irish': 'ga',
-    'italian': 'it',
-    'japanese': 'ja',
-    'javanese': 'jw',
-    'kannada': 'kn',
-    'kazakh': 'kk',
-    'khmer': 'km',
-    'korean': 'ko',
-    'kurdish (kurmanji)': 'ku',
-    'kyrgyz': 'ky',
-    'lao': 'lo',
-    'latin': 'la',
-    'latvian': 'lv',
-    'lithuanian': 'lt',
-    'luxembourgish': 'lb',
-    'macedonian': 'mk',
-    'malagasy': 'mg',
-    'malay': 'ms',
-    'malayalam': 'ml',
-    'maltese': 'mt',
-    'maori': 'mi',
-    'marathi': 'mr',
-    'mongolian': 'mn',
-    'myanmar (burmese)': 'my',
-    'nepali': 'ne',
-    'norwegian': 'no',
-    'odia': 'or',
-    'pashto': 'ps',
-    'persian': 'fa',
-    'polish': 'pl',
-    'portuguese': 'pt',
-    'punjabi': 'pa',
-    'romanian': 'ro',
-    'russian': 'ru',
-    'samoan': 'sm',
-    'scots gaelic': 'gd',
-    'serbian': 'sr',
-    'sesotho': 'st',
-    'shona': 'sn',
-    'sindhi': 'sd',
-    'sinhala': 'si',
-    'slovak': 'sk',
-    'slovenian': 'sl',
-    'somali': 'so',
-    'spanish': 'es',
-    'sundanese': 'su',
-    'swahili': 'sw',
-    'swedish': 'sv',
-    'tajik': 'tg',
-    'tamil': 'ta',
-    'telugu': 'te',
-    'thai': 'th',
-    'turkish': 'tr',
-    'ukrainian': 'uk',
-    'urdu': 'ur',
-    'uyghur': 'ug',
-    'uzbek': 'uz',
-    'vietnamese': 'vi',
-    'welsh': 'cy',
-    'xhosa': 'xh',
-    'yiddish': 'yi',
-    'yoruba': 'yo',
-    'zulu': 'zu'}
-
-LANGUAGES = {value:key for key, value in LANG_CODES.items()}
+LANG_CODES = {
+    'afrikaans': 'af',
+    'albanian': 'sq',
+    'amharic': 'am',
+    'arabic': 'ar',
+    'armenian': 'hy',
+    'azerbaijani': 'az',
+    'basque': 'eu',
+    'belarusian': 'be',
+    'bengali': 'bn',
+    'bosnian': 'bs',
+    'bulgarian': 'bg',
+    'catalan': 'ca',
+    'cebuano': 'ceb',
+    'chichewa': 'ny',
+    'chinese (simplified)': 'zh-cn',
+    'chinese (traditional)': 'zh-tw',
+    'chinese (cantonese)': 'yue',
+    'corsican': 'co',
+    'croatian': 'hr',
+    'czech': 'cs',
+    'danish': 'da',
+    'dutch': 'nl',
+    'english': 'en',
+    'esperanto': 'eo',
+    'estonian': 'et',
+    'filipino': 'tl',
+    'finnish': 'fi',
+    'french': 'fr',
+    'frisian': 'fy',
+    'galician': 'gl',
+    'georgian': 'ka',
+    'german': 'de',
+    'greek': 'el',
+    'gujarati': 'gu',
+    'haitian creole': 'ht',
+    'hausa': 'ha',
+    'hawaiian': 'haw',
+    'hebrew': 'he',
+    'hindi': 'hi',
+    'hmong': 'hmn',
+    'hungarian': 'hu',
+    'icelandic': 'is',
+    'igbo': 'ig',
+    'indonesian': 'id',
+    'irish': 'ga',
+    'italian': 'it',
+    'japanese': 'ja',
+    'javanese': 'jw',
+    'kannada': 'kn',
+    'kazakh': 'kk',
+    'khmer': 'km',
+    'korean': 'ko',
+    'kurdish (kurmanji)': 'ku',
+    'kyrgyz': 'ky',
+    'lao': 'lo',
+    'latin': 'la',
+    'latvian': 'lv',
+    'lithuanian': 'lt',
+    'luxembourgish': 'lb',
+    'macedonian': 'mk',
+    'malagasy': 'mg',
+    'malay': 'ms',
+    'malayalam': 'ml',
+    'maltese': 'mt',
+    'maori': 'mi',
+    'marathi': 'mr',
+    'mongolian': 'mn',
+    'myanmar (burmese)': 'my',
+    'nepali': 'ne',
+    'norwegian': 'no',
+    'odia': 'or',
+    'pashto': 'ps',
+    'persian': 'fa',
+    'polish': 'pl',
+    'portuguese': 'pt',
+    'punjabi': 'pa',
+    'romanian': 'ro',
+    'russian': 'ru',
+    'samoan': 'sm',
+    'scots gaelic': 'gd',
+    'serbian': 'sr',
+    'sesotho': 'st',
+    'shona': 'sn',
+    'sindhi': 'sd',
+    'sinhala': 'si',
+    'slovak': 'sk',
+    'slovenian': 'sl',
+    'somali': 'so',
+    'spanish': 'es',
+    'sundanese': 'su',
+    'swahili': 'sw',
+    'swedish': 'sv',
+    'tajik': 'tg',
+    'tamil': 'ta',
+    'telugu': 'te',
+    'thai': 'th',
+    'turkish': 'tr',
+    'ukrainian': 'uk',
+    'urdu': 'ur',
+    'uyghur': 'ug',
+    'uzbek': 'uz',
+    'vietnamese': 'vi',
+    'welsh': 'cy',
+    'xhosa': 'xh',
+    'yiddish': 'yi',
+    'yoruba': 'yo',
+    'zulu': 'zu'}
+
+LANGUAGES = {value:key for key, value in LANG_CODES.items()}

{BackTranslation-0.3.1 → backtranslation-0.5.0}/BackTranslation/translated.py

@@ -1,22 +1,23 @@
-class Translated(object):
-    """
-    Save the translated result
-    """
-    def __init__(self, src_lang, tmp_lang, text, trans_text, back_text):
-        self.source_text = text
-        self.src = src_lang
-        self.tmp = tmp_lang
-        self.tran_text = trans_text
-        self.result_text = back_text
-
-
-    def __str__(self):
-        return self.__unicode__()
-
-
-
-    def __unicode__(self):
-        if len(self.result_text) > 30 :
-            show_text = ' '.join(self.result_text.split(' ')[:30])
-            show_text += '...'
-        return u'Trabslated(src={src}, tmp={tmp}, result_text={text})'.format(src=self.src, tmp=self.tmp, text = self.result_text)
+class Translated(object):
+    """
+    Save the translated result
+    """
+    def __init__(self, src_lang, tmp_lang, text, trans_text, back_text, dst_lang=None, mode='back_translation'):
+        self.source_text = text
+        self.src = src_lang
+        self.tmp = tmp_lang
+        self.dst = dst_lang if dst_lang is not None else tmp_lang
+        self.mode = mode
+        self.tran_text = trans_text
+        self.result_text = back_text
+
+    def __str__(self):
+        return self.__unicode__()
+
+    def __unicode__(self):
+        if len(self.result_text.split(' ')) > 30:
+            show_text = ' '.join(self.result_text.split(' ')[:30])
+            show_text += '...'
+        else:
+            show_text = self.result_text
+        return u'Translated(src={src}, tmp={tmp}, result_text={text})'.format(src=self.src, tmp=self.tmp, text=show_text)

{BackTranslation-0.3.1 → backtranslation-0.5.0}/BackTranslation/translation.py

@@ -1,116 +1,154 @@
-from googletrans import Translator
-import nltk
-from nltk.tokenize import sent_tokenize
-import time
-from BackTranslation.languages import LANGUAGES, LANG_CODES
-from BackTranslation.translated import Translated
-import typing
-import httpcore
-
-try:
-    nltk.data.find('tokenizers/punkt')
-except LookupError:
-    nltk.download('punkt')
-
-
-class BackTranslation(object):
-    """
-    Translate the words to other language and translate back
-    in order to generate more samples for NLP.
-    
-    This package built based on googletrans package.
-    Thus, you have to create only one instance to translate your words.
-    Otherwise, an error will occur from googletrans.
-
-    """
-
-    def __init__(self, url=['translate.google.com'], proxies: typing.Dict[str, httpcore.SyncHTTPTransport] = None):
-        self.translator = Translator(service_urls=url, proxies=proxies)
-        self.Languages = LANGUAGES
-        self.langCodes = LANG_CODES
-        self.MAX_LENGTH = 5000
-
-    def translate(self, text, src=None, tmp=None, sleeping=0):
-
-        if not src:
-            src = self.translator.detect(text).lang
-
-        if src.lower() not in self.Languages:
-            raise ValueError("'{}': INVALID source language.".format(src))
-
-        # if tmp is null, set a default language for tmp.
-        if not tmp:
-            if src == 'en':
-                tmp = 'zh-cn'
-            else:
-                tmp = 'en'
-
-        if tmp not in self.Languages:
-            raise ValueError("'{}': INVALID transited language.".format(tmp))
-
-        if src == tmp:
-            raise ValueError("Transited language ({tmp}) should different from srouce language ({src}).".format(
-                tmp=self.langCodes[tmp], src=self.langCodes[src]))
-
-        # check the length of text
-        if len(text) > self.MAX_LENGTH:
-            original_sentences = self._split_segement(sent_tokenize(text))
-
-            t_text = self.translator.translate(original_sentences, src=src, dest=tmp)
-            tran_text = ' '.join([t.text for t in t_text])
-            time.sleep(sleeping)
-            r_text = self.translator.translate([t.text for t in t_text], src=tmp, dest=src)
-            back_text = ' '.join([r.text for r in r_text])
-            back_text.rstrip()
-        else:
-            mid_text = self.translator.translate(text, src=src, dest=tmp)
-            tran_text = mid_text.text
-            time.sleep(sleeping)  # Sleep between translation
-            result_text = self.translator.translate(tran_text, src=tmp, dest=src)
-            back_text = result_text.text
-        result = Translated(src_lang=src, tmp_lang=tmp, text=text, trans_text=tran_text, back_text=back_text)
-        return result
-
-    def _split_segement(self, sentences):
-        """
-        Split the long sentences into multiple sentences whose lengths are less than MAX_LENGTH.
-
-        :param sentences: the list of tokenized sentences from source text
-        :return: the list of sentences with proper length
-        :rtype: list
-        """
-        sentences_list = []
-        block = ""
-        for sentence in sentences:
-            if len((block.rstrip() + ' ' + sentence).encode('utf-8')) > self.MAX_LENGTH:
-                sentences_list.append(block.rstrip())
-                block = sentence
-            else:
-                block = block + sentence + ' '
-        sentences_list.append(block.rstrip())
-        return sentences_list
-
-    def searchLanguage(self, language):
-        """
-        Search a matched language code by language name.
-
-        :param language: the full name of language in english
-        :return: return matched language name and their code
-        :rtype: dict
-
-        Usage:
-            >>> trans = BackTranslation()
-            >>> trans.searchLanguage('frenc')
-            # {'french': 'fr'}
-            >>> trans.searchLanguage('Chinese')
-            # {'chinese (simplified)': 'zh-cn', 'chinese (traditional)': 'zh-tw'}
-        """
-        language = language.lower()
-        if language in self.langCodes:
-            return {language: self.langCodes[language]}
-        else:
-            language_list = [key for key in self.langCodes.keys() if language in key]
-            if language_list != []:
-                return {lan: self.langCodes[lan] for lan in language_list}
-            else:
-                raise ValueError("{}: No existing language.".format(language))
+from googletrans import Translator
+import nltk
+from nltk.tokenize import sent_tokenize
+import time
+from BackTranslation.languages import LANGUAGES, LANG_CODES
+from BackTranslation.translated import Translated
+import typing
+import httpcore
+
+try:
+    nltk.data.find('tokenizers/punkt')
+except LookupError:
+    nltk.download('punkt')
+
+
+class BackTranslation(object):
+    """
+    Translate the words to other language and translate back
+    in order to generate more samples for NLP.
+    
+    This package built based on googletrans package.
+    Thus, you have to create only one instance to translate your words.
+    Otherwise, an error will occur from googletrans.
+
+    """
+
+    def __init__(self, url=['translate.google.com'], proxies: typing.Dict[str, httpcore.SyncHTTPTransport] = None):
+        self.translator = Translator(service_urls=url, proxies=proxies)
+        self.Languages = LANGUAGES
+        self.langCodes = LANG_CODES
+        self.MAX_LENGTH = 5000
+
+    def translate(self, text, src=None, tmp=None, sleeping=0):
+        src = self._resolve_src(text, src)
+        self._validate_language(src, 'source')
+
+        # if tmp is null, set a default language for tmp.
+        if not tmp:
+            if src == 'en':
+                tmp = 'zh-cn'
+            else:
+                tmp = 'en'
+
+        self._validate_language(tmp, 'transited')
+        self._validate_different(src, tmp, 'Transited')
+
+        # check the length of text
+        if len(text) > self.MAX_LENGTH:
+            original_sentences = self._split_segement(sent_tokenize(text))
+
+            t_text = self._translate_with_google(original_sentences, src, tmp, 'Translation')
+            tran_text = ' '.join([t.text for t in t_text])
+            time.sleep(sleeping)
+            r_text = self._translate_with_google([t.text for t in t_text], tmp, src, 'Back-translation')
+            back_text = ' '.join([r.text for r in r_text])
+            back_text.rstrip()
+        else:
+            mid_text = self._translate_with_google(text, src, tmp, 'Translation')
+            tran_text = mid_text.text
+            time.sleep(sleeping)  # Sleep between translation
+            result_text = self._translate_with_google(tran_text, tmp, src, 'Back-translation')
+            back_text = result_text.text
+        result = Translated(src_lang=src, tmp_lang=tmp, text=text, trans_text=tran_text, back_text=back_text,
+                            dst_lang=tmp, mode='back_translation')
+        return result
+
+    def direct_translate(self, text, src=None, dst=None, sleeping=0):
+        if not dst:
+            raise ValueError("destination language is required")
+
+        src = self._resolve_src(text, src)
+
+        self._validate_language(src, 'source')
+        self._validate_language(dst, 'destination')
+        self._validate_different(src, dst, 'Destination')
+
+        if len(text) > self.MAX_LENGTH:
+            original_sentences = self._split_segement(sent_tokenize(text))
+            translated_text = self._translate_with_google(original_sentences, src, dst, 'Translation')
+            result_text = ' '.join([t.text for t in translated_text])
+        else:
+            translated_text = self._translate_with_google(text, src, dst, 'Translation')
+            result_text = translated_text.text
+
+        result = Translated(src_lang=src, tmp_lang=dst, text=text, trans_text=result_text, back_text=result_text,
+                            dst_lang=dst, mode='direct')
+        return result
+
+    def _resolve_src(self, text, src):
+        if not src:
+            return self.translator.detect(text).lang
+        return src
+
+    def _validate_language(self, language, language_type):
+        if language.lower() not in self.Languages:
+            raise ValueError("'{}': INVALID {} language.".format(language, language_type))
+
+    def _validate_different(self, src, dst, dst_label):
+        if src == dst:
+            raise ValueError("{} language ({dst}) should different from source language ({src}).".format(
+                dst_label, dst=dst, src=src))
+
+    def _translate_with_google(self, text, src, dst, action):
+        try:
+            return self.translator.translate(text, src=src, dest=dst)
+        except httpcore.ConnectTimeout:
+            raise httpcore.ConnectTimeout(
+                "Connection timed out. If you are blocked by Google, try using the 'proxies' parameter or a different 'url'.")
+        except Exception as e:
+            raise Exception("{} failed (try increasing the 'sleeping' parameter): {}".format(action, e))
+
+    def _split_segement(self, sentences):
+        """
+        Split the long sentences into multiple sentences whose lengths are less than MAX_LENGTH.
+
+        :param sentences: the list of tokenized sentences from source text
+        :return: the list of sentences with proper length
+        :rtype: list
+        """
+        sentences_list = []
+        block = ""
+        for sentence in sentences:
+            if len((block.rstrip() + ' ' + sentence).encode('utf-8')) > self.MAX_LENGTH:
+                sentences_list.append(block.rstrip())
+                block = sentence
+            else:
+                block = block + sentence + ' '
+        sentences_list.append(block.rstrip())
+        return sentences_list
+
+    def searchLanguage(self, language):
+        """
+        Search a matched language code by language name.
+
+        :param language: the full name of language in english
+        :return: return matched language name and their code
+        :rtype: dict
+
+        Usage:
+            >>> trans = BackTranslation()
+            >>> trans.searchLanguage('frenc')
+            # {'french': 'fr'}
+            >>> trans.searchLanguage('Chinese')
+            # {'chinese (simplified)': 'zh-cn', 'chinese (traditional)': 'zh-tw'}
+        """
+        language = language.lower()
+        if language in self.langCodes:
+            return {language: self.langCodes[language]}
+        else:
+            language_list = [key for key in self.langCodes.keys() if language in key]
+            if language_list != []:
+                return {lan: self.langCodes[lan] for lan in language_list}
+            else:
+                raise ValueError("{}: No existing language.".format(language))