yakutmorph 0.0.5__py3-none-any.whl

tests/test_tokenizers.py

@@ -0,0 +1,62 @@
+import unittest
+from yakutmorph.tokenizers import YakutTokenizer
+
+
+class TestYakutTokenizer(unittest.TestCase):
+
+    def setUp(self):
+        self.tokenizer = YakutTokenizer()
+
+    def test_phone(self):
+        text = 'Сиһилии билсиҥ: https://zhataytekh.ru саайтка, +7(4114)426407, +7(4113)427303 төлөпүөнүнэн.'
+        expected = [
+            ('Сиһилии', 'title'), ('билсиҥ', 'lowercase'), (':', 'colon'), ('https://zhataytekh.ru', 'uri'),
+            ('саайтка', 'lowercase'), (',', 'comma'), ('+7(4114)426407', 'phone'), (',', 'comma'),
+            ('+7(4113)427303', 'phone'), ('төлөпүөнүнэн', 'lowercase'), ('.', 'period')
+        ]
+        self.assertEqual(self.tokenizer.tokenize(text), expected)
+
+    def test_date(self):
+        text = 'дата 01/04/2024'
+        expected = [
+            ('дата', 'lowercase'), ('01/04/2024', 'date')
+        ]
+        self.assertEqual(self.tokenizer.tokenize(text), expected)
+
+    def test_number(self):
+        text = '1949 сыллаахха оруобуна ахсынньы 10 күнүгэр аан бастаан көрдөрүллүбүт эбит.'
+        expected = [
+            ('1949', 'number'), ('сыллаахха', 'lowercase'), ('оруобуна', 'lowercase'), ('ахсынньы', 'lowercase'),
+            ('10', 'number'), ('күнүгэр', 'lowercase'), ('аан', 'lowercase'), ('бастаан', 'lowercase'),
+            ('көрдөрүллүбүт', 'lowercase'), ('эбит', 'lowercase'), ('.', 'period')
+        ]
+        self.assertEqual(self.tokenizer.tokenize(text), expected)
+
+    def test_url(self):
+        text = 'Хаартыска https://www.trud.ru саайтан.'
+        expected = [
+            ('Хаартыска', 'title'), ('https://www.trud.ru', 'uri'), ('саайтан', 'lowercase'), ('.', 'period')
+        ]
+        self.assertEqual(self.tokenizer.tokenize(text), expected)
+
+    def test_email(self):
+        text = 'Кыттааччыларгыт тустарынан иһитиннэриини sleptsova191_@mail.ru ыытыҥ.'
+        expected = [
+            ('Кыттааччыларгыт', 'title'), ('тустарынан', 'lowercase'), ('иһитиннэриини', 'lowercase'),
+            ('sleptsova191_@mail.ru', 'email'), ('ыытыҥ', 'lowercase'), ('.', 'period')
+        ]
+        self.assertEqual(self.tokenizer.tokenize(text), expected)
+
+    def test_roman(self):
+        text = 'Ити саха хомустара бары XIX үйэ бүтүүтэ – XX үйэ бастакы аҥаарыгар оҥоһуллубуттар.'
+        expected = [
+            ('Ити', 'title'), ('саха', 'lowercase'), ('хомустара', 'lowercase'), ('бары', 'lowercase'),
+            ('XIX', 'roman'), ('үйэ', 'lowercase'), ('бүтүүтэ', 'lowercase'), ('–', 'dash'), ('XX', 'roman'),
+            ('үйэ', 'lowercase'), ('бастакы', 'lowercase'), ('аҥаарыгар', 'lowercase'),
+            ('оҥоһуллубуттар', 'lowercase'), ('.', 'period')
+        ]
+        self.assertEqual(self.tokenizer.tokenize(text), expected)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_transducers.py

@@ -0,0 +1,22 @@
+import unittest
+from yakutmorph.transducers import YakutTransducer
+
+
+class TestYakutTransducer(unittest.TestCase):
+
+    def setUp(self):
+        self.transducer = YakutTransducer()
+
+    def test_analyse(self):
+        surface_form = 'дьоннор'
+        analysis_form = ['дьон^N+PL']
+        self.assertEqual(self.transducer.analyse(surface_form), analysis_form)
+
+    def test_generate(self):
+        analysis_form = 'дьон^N+PL'
+        surface_form = ['дьоннор']
+        self.assertEqual(self.transducer.generate(analysis_form), surface_form)
+
+
+if __name__ == '__main__':
+    unittest.main()

yakutmorph/data/mappings.yaml

@@ -0,0 +1,87 @@
+morph_mapping:
+    ^VRBLZ: ^V
+    ^NMLZ: ^N
+    ^ADJZ: ^Adj
+    ^ADVZ: ^Adv
+    +CONV_COND: +PART
+    +PART_PRES: +PART
+    +PART_PAST: +PART
+    +PART_FUT: +PART
+    ^POSS: ^Pron
+
+token_type_mapping:
+    uri: ^PN
+    email: ^PN
+    phone: ^PN
+    date: ^Adv
+    number: ^Num
+    roman: ^Num
+    hyphen: <HYP>
+    comma: <COMMA>
+    colon: <COMMA>
+    period: <STOP>
+    ellipsis: <STOP>
+    question: <STOP>
+    exclamation: <STOP>
+    semicolon: <STOP>
+    quotation: <SYM>
+    bracket: <SYM>
+    dash: <SYM>
+    symbol: <SYM>
+    unknown: ^UNK
+
+feature_morphs:
+    - ^Adv
+    - ^ADVZ
+    - ^Adj
+    - ^ADJZ
+    - ^aux
+    - ^CONJ
+    - ^DEF
+    - ^IMIT
+    - ^INJ
+    - ^N
+    - ^NMLZ
+    - ^Num
+    - ^PN
+    - ^MOD
+    - ^ORD
+    - ^Part
+    - ^POST
+    - ^POSS
+    - ^Pron
+    - ^UNK
+    - ^V
+    - ^VRBLZ
+    - +1SG
+    - +2SG
+    - +1PL
+    - +2PL
+    - +3SG
+    - +3PL
+    - +ABL
+    - +ACC
+    - +COMIT
+    - +COMP
+    - +CONV
+    - +DAT
+    - +INST
+    - +LOC
+    - +CONV_COND
+    - +PART_PAST
+    - +PART_PRES
+    - +PART_FUT
+    - +PL
+    - +POSS.1SG
+    - +POSS.2SG
+    - +POSS.3SG
+    - +POSS.1PL
+    - +POSS.2PL
+    - +POSS.3PL
+    - +PRTV
+    - +PST.1SG
+    - +PST.2SG
+    - +PST.3SG
+    - +PST.1PL
+    - +PST.2PL
+    - +PST.3PL