rara-tools 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

rara_tools/parsers/tools/russian_transliterator.py

@@ -0,0 +1,248 @@
+import regex as re
+
+
+class Transliterate():
+    """
+    Transliterate class to transliterate text from Cyrillic to Latin
+
+    Ruleset: http://www.eki.ee/books/ekk09/index.php?id=37&p=2&p1=6
+    'Vene keele tähestikust eesti tähestikku' 2005.
+
+    Usage:
+        Initialize the class:
+            foo = Transliterate()
+        Call with a list of sentences you'd like to transliterate:
+            foo(['Дженни играет с мяц.', 'Яне говорит на телефоне.', 'Рассекречены планы выпуска дешевого iPhone Читать далее'])
+            # This Returns -- ['Дженни играет с мяц.', 'Яне говорит на телефоне.', 'Рассекречены планы выпуска дешевого iPhone Читать далее']
+    """
+
+    def __call__(self, sentences):
+        return self._transliterate(sentences)
+
+    def _transliterate(self, sentences):
+        """The main transliteration processor
+
+        Arguments:
+            sentences {list of strings} -- A list of string sentences to tranliterate
+
+        Returns:
+            [list of strings] -- A transliterated version of the output sentences
+        """
+        new_sentences = []
+        for sentence in sentences:
+            converted_sentence = []
+            sentence = sentence.split(' ')
+            for word in sentence:
+                word = self._check_doublechar_rules(word)
+                converted_word = ''
+                for char_index, char in enumerate(word):
+                    transchar = ''
+                    if char in self._translit_table:
+                        transchar = self._translit_table[char]
+                    elif char in self._chars_with_rules:
+                        transchar = self._check_char_rules(
+                            word, char, char_index)
+                    else:
+                        transchar = char
+
+                    converted_word += transchar
+                converted_sentence.append(converted_word)
+            new_sentences.append(' '.join(converted_sentence))
+
+        return new_sentences
+
+    def _check_doublechar_rules(self, word):
+        """Checks for doublechar rules, currently only for the cyr characters ij
+
+        Arguments:
+            word {string} -- The word in which to check the rule in
+
+        Returns:
+            [string] -- The modified version of the input word relative to the rules, if no rules were found, returns the unchanged input word.
+        """
+        word_to_return = word
+        if len(word) > 3:
+            for match in re.finditer("ий", word.lower()):
+                if match.span()[1] - 1 == len(word) - 1:
+                    new_char = self._check_if_upper_and_return(
+                        word[match.span()[0]], 'I')
+                    word_to_return = list(word_to_return[:match.span()[0] + 1])
+                    word_to_return[-1] = new_char
+                    word_to_return = ''.join(word_to_return)
+
+        return word_to_return
+
+    def _check_char_rules(self, word, char, char_index):
+        """Checks for specific rules for the given transliteration
+
+        Arguments:
+            word {string} -- The word in which the character originated from.
+            char {string} -- The character about to be transliterated.
+            char_index {int} -- The index of the character in the word.
+        """
+        if char in 'Ее':
+            return self._rules_for_e(word, char, char_index)
+        elif char in 'Ёё':
+            return self._rules_for_jo(word, char, char_index)
+        elif char in 'ИиЙй':
+            return self._rules_for_i(word, char, char_index)
+        elif char in 'Хх':
+            return self._rules_for_h(word, char, char_index)
+        elif char in 'Ьь':
+            return self._rules_for_soft(word, char, char_index)
+        elif char in 'Сс':
+            return self._rules_for_s(word, char, char_index)
+
+    def _rules_for_s(self, word, char, char_index):
+        """Checks rules for the cyr s character
+            Returns: [string] -- Returns 'Ss/ss/S/s' based on the rules
+        """
+        char_to_return = ''
+        if char_index > 0:
+            if word[char_index-1] in self._vocals_rus:
+                if len(word) == 2:
+                    char_to_return = self._check_if_upper_and_return(
+                        char, 'Ss')
+                # To avoid keyerror use len(word[char_index+1:])
+                elif len(word[char_index+1:]) > 0 and word[char_index+1] in self._vocals_rus:
+                    char_to_return = self._check_if_upper_and_return(
+                        char, 'Ss')
+                elif word[-1] == char:
+                    char_to_return = self._check_if_upper_and_return(
+                        char, 'Ss')
+
+        if char_to_return == '':
+            char_to_return = self._chars_with_rules[char]
+
+        return char_to_return
+
+    def _rules_for_soft(self, word, char, char_index):
+        """Checks rules for the cyr softening character
+            Returns: [string] -- Returns 'J/j//' based on the rules
+        """
+        if len(word[char_index+1:]) > 0 and word[char_index+1] not in 'еЕёЁюЮяЯ' and word[char_index+1] in self._vocals_rus:
+            char_to_return = self._check_if_upper_and_return(char, 'J')
+        else:
+            return self._chars_with_rules[char]
+
+        return char_to_return
+
+    def _rules_for_e(self, word, char, char_index):
+        """Checks rules for the cyr character 'e'
+            Returns: [string] -- Returns 'Je/je/E/e' based on the rules
+        """
+        if char_index == 0:
+            char_to_return = self._check_if_upper_and_return(char, 'Je')
+        elif word[char_index - 1] in self._vocals_rus or word[char_index - 1] in 'ЬьЪъ':
+            char_to_return = self._check_if_upper_and_return(char, 'Je')
+        else:
+            char_to_return = self._chars_with_rules[char]
+
+        return char_to_return
+
+    def _rules_for_jo(self, word, char, char_index):
+        """Checks rules for the cyr character 'jo'
+            Returns: [string] -- Returns 'O/o/Jo/jo' based on the rules
+        """
+        char_to_return = ''
+        if char_index != 0:
+            if word[char_index-1] in 'ЖжЧчШшЩщ':
+                char_to_return = self._check_if_upper_and_return(char, 'O')
+
+        if char_to_return == '':
+            char_to_return = self._chars_with_rules[char]
+
+        return char_to_return
+
+    def _rules_for_i(self, word, char, char_index):
+        """Checks rules for the cyr character 'i'
+        Returns: [string] -- Returns 'J/j/I/i' based on the rules
+        """
+        char_to_return = ''
+        if len(word) > 1:
+            if char_index == 0 and word[char_index + 1] in self._vocals_rus:
+                char_to_return = self._check_if_upper_and_return(char, 'J')
+
+        if char_to_return == '':
+            char_to_return = self._chars_with_rules[char]
+
+        return char_to_return
+
+    def _rules_for_h(self, word, char, char_index):
+        """Checks rules for the cyr character 'h'
+        Returns: [string] -- Returns 'Hh/hh/H/h' based on the rules
+        """
+        char_to_return = ''
+        if char_index > 0:
+            if word[char_index-1] in self._vocals_rus:
+                if len(word) == 2:
+                    char_to_return = self._check_if_upper_and_return(
+                        char, 'Hh')
+                # To avoid keyerror use len(word[char_index+1:])
+                elif len(word[char_index+1:]) > 0 and word[char_index+1] in self._vocals_rus:
+                    char_to_return = self._check_if_upper_and_return(
+                        char, 'Hh')
+                elif word[-1] == char:
+                    char_to_return = self._check_if_upper_and_return(
+                        char, 'Hh')
+
+        if char_to_return == '':
+            char_to_return = self._chars_with_rules[char]
+
+        return char_to_return
+
+    @staticmethod
+    def _check_if_upper_and_return(char_in_index, char):
+        """Checks whether the character is supposed to be lower or upper case
+
+        Arguments:
+            char_in_index {string} -- [The value of the char index in the word]
+            char {string} -- [The character to return, should be first letter uppercase, ex: 'Je']
+
+        Returns:
+            [string] -- [Upper or lower case version of the character]
+        """
+        if char_in_index.isupper():
+            return char
+        else:
+            return char.lower()
+
+    # Ugly constants too big for init method
+    _vocals_est = ['i', 'ü', 'u', 'e', 'ö', 'õ', 'o', 'ä', 'a']
+    _vocals_rus = ['а', 'э', 'ы', 'у', 'о', 'я', 'е', 'ё', 'ю', 'и']
+    _translit_table = {
+        'А': 'A', 'а': 'a',
+        'Б': 'B', 'б': 'b',
+        'В': 'V', 'в': 'v',
+        'Г': 'G', 'г': 'g',
+        'Д': 'D', 'д': 'd',
+        'Ж': 'Ž', 'ж':  'ž',
+        'З': 'Z', 'з': 'z',
+        'К': 'K', 'к': 'k',
+        'Л': 'L', 'л': 'l',
+        'М': 'M', 'м': 'm',
+        'Н': 'N', 'н': 'n',
+        'О': 'O', 'о': 'o',
+        'П': 'P', 'п': 'p',
+        'Р': 'R', 'р': 'r',
+        'Т': 'T', 'т': 't',
+        'У': 'U', 'у': 'u',
+        'Ф': 'F', 'ф': 'f',
+        'Ц': 'Ts', 'ц':  'ts',
+        'Ч': 'Tš', 'ч':  'tš',
+        'Ш': 'Š', 'ш':  'š',
+        'Щ': 'Štš', 'щ': 'štš',
+        'Ы': 'õ', 'ы': 'õ',
+        'Ъ': "", 'ъ': "",
+        'Э': 'E', 'э': 'e',
+        'Ю': 'Ju', 'ю': 'ju',
+        'Я': 'Ja', 'я': 'ja'}
+    _chars_with_rules = {
+        'Е': 'E', 'е': 'e',
+        'Ё': 'Jo', 'ё': 'jo',
+        'И': 'I', 'и': 'i',
+        'Й': 'I', 'й': 'i',
+        'Х': 'H', 'х':  'h',
+        'Ь': "", 'ь': "",
+        'С': 'S', 'с': 's',
+    }

{rara_tools-0.3.0.dist-info → rara_tools-0.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rara-tools
-Version: 0.3.0
+Version: 0.4.0
 Summary: Tools to support Kata's work.
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
@@ -13,9 +13,13 @@ License-File: LICENSE.md
 Requires-Dist: elasticsearch==8.*
 Requires-Dist: elasticsearch_dsl==8.*
 Requires-Dist: minio==7.*
+Requires-Dist: estnltk==1.7.3
+Requires-Dist: nltk
+Requires-Dist: jsonlines
 Requires-Dist: requests
 Requires-Dist: iso639-lang
 Requires-Dist: pymarc
+Requires-Dist: regex
 Requires-Dist: glom
 Provides-Extra: testing
 Requires-Dist: pytest>=8.0; extra == "testing"

rara_tools-0.4.0.dist-info/RECORD

@@ -0,0 +1,37 @@
+rara_tools/converters.py,sha256=_1ZRH4ACLOolI1G5b_aSssN68rWOvan-q2dTq7D7-j4,2794
+rara_tools/decorators.py,sha256=MjOyvZ5nTkwxwx2JLFEGpKKBysvecFw6EN6UDrSvZLU,2187
+rara_tools/digar_schema_converter.py,sha256=k95U2iRlEA3sh772-v6snhHW6fju6qSTMnvWJ6DpzZk,14254
+rara_tools/elastic.py,sha256=MgPHxZ3UbSTIL8_sT9gU5V4PLKJjo3aQ8CGyhXjRz6M,13065
+rara_tools/exceptions.py,sha256=YQyaueUbXeTkJYFDEuN6iWTXMI3eCv5l7PxGp87vg5I,550
+rara_tools/s3.py,sha256=9ziDXsLjBtFAvsjTPxFddhfvkpA8773rzPJqO7y1N5Q,6415
+rara_tools/task_reporter.py,sha256=WCcZts9dAUokPc4vbrG3-lNAFLnWaMgE3b3iaUB7mr8,3256
+rara_tools/utils.py,sha256=9vSbmuWYU5ydr4lXBKlUKa0xzDccFsaJv4T-XwgUfuY,2578
+rara_tools/constants/__init__.py,sha256=r78laM9vyRDAvzDhPvzDlhaX6qPwUUBBtwf1WosrW3o,27
+rara_tools/constants/digitizer.py,sha256=MND0dUQySBAOVWzuUBxQGZWv_Ckdz2jCp25F2_oHGi8,496
+rara_tools/constants/general.py,sha256=aVUQTMss89atAkTDZKJXNdnsBHPX-RSrlBOtt-wdPGU,195
+rara_tools/constants/normalizers.py,sha256=GmWY89kYfX7_YJ8sdy1vb8ABJc_ABdw_zVVOxd9UZgY,171
+rara_tools/constants/parsers.py,sha256=stXOyA1dEOgxdCUT4Mp4pvvGLmdE7DAjTe8Jq71tcS4,5453
+rara_tools/normalizers/__init__.py,sha256=_NqpS5w710DhaURytHq9JpEt8HgYpSPfRDcOtOymJgE,193
+rara_tools/normalizers/authorities.py,sha256=IDtcm0yNZNhv1f-WcdqWFSRzZk_CoKuBFsk6hEPddWM,4513
+rara_tools/normalizers/base.py,sha256=taOboGURQF_ACPVWHX_wMsaDEo8gYdAkiOw0yT0zzR8,10910
+rara_tools/normalizers/bibs.py,sha256=4DTS6k37z8qR5B3n7aiCXsT5Z49rLTvQ60lKKr5dyLs,2352
+rara_tools/normalizers/viaf.py,sha256=9uTyEadSaoFedUbUfY_iWPJtgrt04jP71i_6MLPM08I,6919
+rara_tools/parsers/marc_parsers/base_parser.py,sha256=wzCccZaiN4p2iUms3PAOfXihNgEeg1cGRzRx26ytJeA,1661
+rara_tools/parsers/marc_parsers/ems_parser.py,sha256=70WdxnbxZmOqla7EAUapCiCWZl0zRPwghKy25sSKFiY,1801
+rara_tools/parsers/marc_parsers/location_parser.py,sha256=18HExO2BLs1ZJodjNF8YOkB5CJ7bd6_zmyIIzALL7aI,1658
+rara_tools/parsers/marc_parsers/organization_parser.py,sha256=faqQEYsut_ZF3kX1QycTnbRIqC7W8sULxmG75ICfya8,1629
+rara_tools/parsers/marc_parsers/person_parser.py,sha256=iMycHSlgfvgB0axE_rneB5sImVlc920FcBnTsUsmVW4,1582
+rara_tools/parsers/marc_parsers/title_parser.py,sha256=0FnX1kl9InELlSqMGECjswEbhP-sKl55TuhV05RhWSw,14
+rara_tools/parsers/marc_records/base_record.py,sha256=oDp4yjPMEmSD3F_dWIdx7IRtZfKwD7ydMFUW9YXAhSQ,4322
+rara_tools/parsers/marc_records/ems_record.py,sha256=B2YZLEeDd-GmmYqxhczbMsSEB7-x6ZLjB8OeDnzOxww,9376
+rara_tools/parsers/marc_records/organization_record.py,sha256=HmDqAqAL_Tw7ppEsS5HfogrfNuQMNChCkrdPu6K-SUE,9141
+rara_tools/parsers/marc_records/person_record.py,sha256=BZrXqd7hCOqm-c-sjmsOfaAI4L7lLSjIUWtxHqPjhTs,7863
+rara_tools/parsers/marc_records/title_record.py,sha256=0FnX1kl9InELlSqMGECjswEbhP-sKl55TuhV05RhWSw,14
+rara_tools/parsers/tools/entity_normalizers.py,sha256=afOMqJoL4aeq0cfsohIuxkxzvqNdZ_ba7U32eyogbzk,8722
+rara_tools/parsers/tools/marc_converter.py,sha256=PUbggzJ_wHfke_bHTF2LOZyzX1t0wRM8qIFL36Dl3AI,414
+rara_tools/parsers/tools/russian_transliterator.py,sha256=5ZU66iTqAhr7pmfVqXPAI_cidF43VqqmuN4d7H4_JuA,9770
+rara_tools-0.4.0.dist-info/licenses/LICENSE.md,sha256=hkZVnIZll7e_KNEQzeY94Y9tlzVL8iVZBTMBvDykksU,35142
+rara_tools-0.4.0.dist-info/METADATA,sha256=93dBS6GW3Q9CoA3vyXvsjqrZauntruPKuT8vF_jHSm4,4054
+rara_tools-0.4.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+rara_tools-0.4.0.dist-info/top_level.txt,sha256=JwfB5b8BAtW5OFKRln2AQ_WElTRyIBM4nO0FKN1cupY,11
+rara_tools-0.4.0.dist-info/RECORD,,

rara_tools-0.3.0.dist-info/RECORD

@@ -1,17 +0,0 @@
-rara_tools/converters.py,sha256=_1ZRH4ACLOolI1G5b_aSssN68rWOvan-q2dTq7D7-j4,2794
-rara_tools/decorators.py,sha256=MjOyvZ5nTkwxwx2JLFEGpKKBysvecFw6EN6UDrSvZLU,2187
-rara_tools/digar_schema_converter.py,sha256=k95U2iRlEA3sh772-v6snhHW6fju6qSTMnvWJ6DpzZk,14254
-rara_tools/elastic.py,sha256=MgPHxZ3UbSTIL8_sT9gU5V4PLKJjo3aQ8CGyhXjRz6M,13065
-rara_tools/exceptions.py,sha256=YQyaueUbXeTkJYFDEuN6iWTXMI3eCv5l7PxGp87vg5I,550
-rara_tools/s3.py,sha256=9ziDXsLjBtFAvsjTPxFddhfvkpA8773rzPJqO7y1N5Q,6415
-rara_tools/task_reporter.py,sha256=WCcZts9dAUokPc4vbrG3-lNAFLnWaMgE3b3iaUB7mr8,3256
-rara_tools/utils.py,sha256=9vSbmuWYU5ydr4lXBKlUKa0xzDccFsaJv4T-XwgUfuY,2578
-rara_tools/constants/__init__.py,sha256=r78laM9vyRDAvzDhPvzDlhaX6qPwUUBBtwf1WosrW3o,27
-rara_tools/constants/digitizer.py,sha256=MND0dUQySBAOVWzuUBxQGZWv_Ckdz2jCp25F2_oHGi8,496
-rara_tools/constants/general.py,sha256=aVUQTMss89atAkTDZKJXNdnsBHPX-RSrlBOtt-wdPGU,195
-rara_tools/constants/normalizers.py,sha256=GmWY89kYfX7_YJ8sdy1vb8ABJc_ABdw_zVVOxd9UZgY,171
-rara_tools-0.3.0.dist-info/licenses/LICENSE.md,sha256=hkZVnIZll7e_KNEQzeY94Y9tlzVL8iVZBTMBvDykksU,35142
-rara_tools-0.3.0.dist-info/METADATA,sha256=At1_wVByRjodCwBJDJYyhHbJ1lWf7O3Hv2d7b-QjlzI,3958
-rara_tools-0.3.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-rara_tools-0.3.0.dist-info/top_level.txt,sha256=JwfB5b8BAtW5OFKRln2AQ_WElTRyIBM4nO0FKN1cupY,11
-rara_tools-0.3.0.dist-info/RECORD,,