ts-tokenizer 0.2.2__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ts_tokenizer-0.2.2/ts_tokenizer.egg-info → ts_tokenizer-0.2.4}/PKG-INFO +38 -35
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/README.md +37 -34
- ts_tokenizer-0.2.4/tests/test_abbr_apostrophe_suffix.py +260 -0
- ts_tokenizer-0.2.4/tests/test_abbreviation_lookup.py +13 -0
- ts_tokenizer-0.2.4/tests/test_currency_suffix.py +17 -0
- ts_tokenizer-0.2.4/tests/test_symbolic_marks.py +43 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer/__init__.py +1 -1
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer/char_fix.py +1 -1
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer/data/abbr_list.txt +2 -1
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer/data/cand_abbr_list.txt +1 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer/data/currency_symbols.txt +1 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer/data.py +8 -1
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer/date_check.py +10 -0
- ts_tokenizer-0.2.4/ts_tokenizer/emoticon_check.py +85 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer/token_handler.py +368 -31
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4/ts_tokenizer.egg-info}/PKG-INFO +38 -35
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer.egg-info/SOURCES.txt +4 -0
- ts_tokenizer-0.2.2/ts_tokenizer/emoticon_check.py +0 -36
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/LICENSE +0 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/MANIFEST.in +0 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/pyproject.toml +0 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/setup.cfg +0 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/setup.py +0 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/tests/test_opening_quote_tokenization.py +0 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer/cli.py +0 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer/data/TS_Corpus_Turkish_Word_List.txt +0 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer/data/corection_mark.txt +0 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer/data/domains.txt +0 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer/data/emoticons.txt +0 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer/data/eng_word_list.txt +0 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer/data/exceptions.txt +0 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer/data/smileys.txt +0 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer/inner_punc.py +0 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer/punctuation_process.py +0 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer/smiley_check.py +0 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer/tokenizer.py +0 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer/unit_test.py +0 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer.egg-info/dependency_links.txt +0 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer.egg-info/entry_points.txt +0 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer.egg-info/requires.txt +0 -0
- {ts_tokenizer-0.2.2 → ts_tokenizer-0.2.4}/ts_tokenizer.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ts_tokenizer
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.4
|
|
4
4
|
Summary: TS Tokenizer is a hybrid (lexicon-based and rule-based) tokenizer designed specifically for tokenizing Turkish texts.
|
|
5
5
|
Home-page: https://github.com/tanerim/ts_tokenizer
|
|
6
6
|
Author: Taner Sezer
|
|
@@ -466,6 +466,8 @@ from ts_tokenizer import TokenPreProcess
|
|
|
466
466
|
|----|----------------------------|------------------------------|----------------|--------------------|
|
|
467
467
|
| 01 | is_mention | @ts-tokenizer | Yes | Mention |
|
|
468
468
|
| 02 | is_hashtag | #ts-tokenizer | Yes | Hashtag |
|
|
469
|
+
| -- | is_mention_suffix | @SerkanÇayoğlu'ndan | Yes | Mention_Suffix |
|
|
470
|
+
| -- | is_hashtag_suffix | #Şanlıurfa'da | Yes | Hashtag_Suffix |
|
|
469
471
|
| 03 | is_in_quotes | "ts-tokenizer" | No | ----- |
|
|
470
472
|
| 04 | is_numbered_title | (1) | Yes | Numbered_Title |
|
|
471
473
|
| 05 | is_in_paranthesis | (bilgisayar) | No | ----- |
|
|
@@ -481,40 +483,41 @@ from ts_tokenizer import TokenPreProcess
|
|
|
481
483
|
| 15 | is_email_punc | tanersezerr@gmail.com. | No | ----- |
|
|
482
484
|
| 16 | is_full_url | https://tscorpus.com | Yes | Full_URL |
|
|
483
485
|
| 17 | is_web_url | www.tscorpus.com | Yes | Web_URL |
|
|
484
|
-
|
|
|
485
|
-
|
|
|
486
|
-
|
|
|
487
|
-
|
|
|
488
|
-
|
|
|
489
|
-
|
|
|
490
|
-
|
|
|
491
|
-
|
|
|
492
|
-
|
|
|
493
|
-
|
|
|
494
|
-
|
|
|
495
|
-
|
|
|
496
|
-
|
|
|
497
|
-
|
|
|
498
|
-
|
|
|
499
|
-
|
|
|
500
|
-
|
|
|
501
|
-
|
|
|
502
|
-
|
|
|
503
|
-
|
|
|
504
|
-
|
|
|
505
|
-
|
|
|
506
|
-
|
|
|
507
|
-
|
|
|
508
|
-
| 42 |
|
|
509
|
-
| 43 |
|
|
510
|
-
| 44 |
|
|
511
|
-
| 45 |
|
|
512
|
-
| 46 |
|
|
513
|
-
| 47 |
|
|
514
|
-
| 48 |
|
|
515
|
-
| 49 |
|
|
516
|
-
| 50 |
|
|
517
|
-
| 51 |
|
|
486
|
+
| 18 | is_full_url | www.example.com'un | Yes | URL_Suffix |
|
|
487
|
+
| 19 | is_formula | F(2,37)=6.42 | Yes | Formula |
|
|
488
|
+
| 20 | is_copyright | ©tscorpus | Yes | Copyright |
|
|
489
|
+
| 21 | is_registered | tscorpus® | Yes | Registered |
|
|
490
|
+
| 22 | is_trademark | tscorpus™ | Yes | Trademark |
|
|
491
|
+
| 23 | is_currency | 100$ | Yes | Currency |
|
|
492
|
+
| 24 | is_num_char_sequence | 380A | No | ----- |
|
|
493
|
+
| 25 | is_abbr | TBMM | Yes | Abbr |
|
|
494
|
+
| 26 | is_in_lexicon | bilgisayar | Yes | Valid_Word |
|
|
495
|
+
| 27 | is_in_exceptions | e-mail | Yes | Exception |
|
|
496
|
+
| 28 | is_in_eng_words | computer | Yes | English_Word |
|
|
497
|
+
| 29 | is_smiley | :) | Yes | Smiley |
|
|
498
|
+
| 30 | is_multiple_smiley | :):) | No | ----- |
|
|
499
|
+
| 31 | is_emoticon | 🍻 | Yes | Emoticon |
|
|
500
|
+
| 32 | is_multiple_emoticon | 🍻🍻 | No | ----- |
|
|
501
|
+
| 33 | is_multiple_smiley_in | hey:):) | No | ----- |
|
|
502
|
+
| 34 | is_number | 175.01 | Yes | Number |
|
|
503
|
+
| 35 | is_apostrophed | Türkiye'nin | Yes | Apostrophed |
|
|
504
|
+
| 36 | is_single_punc | ! | Yes | Punc |
|
|
505
|
+
| 37 | is_multi_punc | !! | No | ----- |
|
|
506
|
+
| 38 | is_single_hyphenated | sabah-akşam | Yes | Single_Hyphenated |
|
|
507
|
+
| 39 | is_multi_hyphenated | çay-su-kahve | Yes | Multi-Hyphenated |
|
|
508
|
+
| 40 | is_single_underscored | Gel_Git | Yes | Single_Underscored |
|
|
509
|
+
| 41 | is_multi_underscored | Yarı_Yapılandırılmış_Mülakat | Yes | Multi_Underscored |
|
|
510
|
+
| 42 | is_one_char_fixable | bilgisa¬yar | Yes | One_Char_Fixed |
|
|
511
|
+
| 43 | is_three_or_more | heyyyyy | No | ----- |
|
|
512
|
+
| 44 | is_fsp | bilgisayar. | No | ----- |
|
|
513
|
+
| 45 | is_isp | .bilgisayar | No | ----- |
|
|
514
|
+
| 46 | is_fmp | bilgisayar.. | No | ----- |
|
|
515
|
+
| 47 | is_imp | ..bilgisayar | No | ----- |
|
|
516
|
+
| 48 | is_msp | --bilgisayar-- | No | ----- |
|
|
517
|
+
| 49 | is_mssp | -bilgisayar- | No | ----- |
|
|
518
|
+
| 50 | is_midsp | okul,öğrenci | No | ----- |
|
|
519
|
+
| 51 | is_midmp | okul,öğrenci, öğretmen | No | ----- |
|
|
520
|
+
| 52 | is_non_latin | 한국드 | No | Non_Latin |
|
|
518
521
|
|
|
519
522
|
----------------------
|
|
520
523
|
|
|
@@ -426,6 +426,8 @@ from ts_tokenizer import TokenPreProcess
|
|
|
426
426
|
|----|----------------------------|------------------------------|----------------|--------------------|
|
|
427
427
|
| 01 | is_mention | @ts-tokenizer | Yes | Mention |
|
|
428
428
|
| 02 | is_hashtag | #ts-tokenizer | Yes | Hashtag |
|
|
429
|
+
| -- | is_mention_suffix | @SerkanÇayoğlu'ndan | Yes | Mention_Suffix |
|
|
430
|
+
| -- | is_hashtag_suffix | #Şanlıurfa'da | Yes | Hashtag_Suffix |
|
|
429
431
|
| 03 | is_in_quotes | "ts-tokenizer" | No | ----- |
|
|
430
432
|
| 04 | is_numbered_title | (1) | Yes | Numbered_Title |
|
|
431
433
|
| 05 | is_in_paranthesis | (bilgisayar) | No | ----- |
|
|
@@ -441,40 +443,41 @@ from ts_tokenizer import TokenPreProcess
|
|
|
441
443
|
| 15 | is_email_punc | tanersezerr@gmail.com. | No | ----- |
|
|
442
444
|
| 16 | is_full_url | https://tscorpus.com | Yes | Full_URL |
|
|
443
445
|
| 17 | is_web_url | www.tscorpus.com | Yes | Web_URL |
|
|
444
|
-
|
|
|
445
|
-
|
|
|
446
|
-
|
|
|
447
|
-
|
|
|
448
|
-
|
|
|
449
|
-
|
|
|
450
|
-
|
|
|
451
|
-
|
|
|
452
|
-
|
|
|
453
|
-
|
|
|
454
|
-
|
|
|
455
|
-
|
|
|
456
|
-
|
|
|
457
|
-
|
|
|
458
|
-
|
|
|
459
|
-
|
|
|
460
|
-
|
|
|
461
|
-
|
|
|
462
|
-
|
|
|
463
|
-
|
|
|
464
|
-
|
|
|
465
|
-
|
|
|
466
|
-
|
|
|
467
|
-
|
|
|
468
|
-
| 42 |
|
|
469
|
-
| 43 |
|
|
470
|
-
| 44 |
|
|
471
|
-
| 45 |
|
|
472
|
-
| 46 |
|
|
473
|
-
| 47 |
|
|
474
|
-
| 48 |
|
|
475
|
-
| 49 |
|
|
476
|
-
| 50 |
|
|
477
|
-
| 51 |
|
|
446
|
+
| 18 | is_full_url | www.example.com'un | Yes | URL_Suffix |
|
|
447
|
+
| 19 | is_formula | F(2,37)=6.42 | Yes | Formula |
|
|
448
|
+
| 20 | is_copyright | ©tscorpus | Yes | Copyright |
|
|
449
|
+
| 21 | is_registered | tscorpus® | Yes | Registered |
|
|
450
|
+
| 22 | is_trademark | tscorpus™ | Yes | Trademark |
|
|
451
|
+
| 23 | is_currency | 100$ | Yes | Currency |
|
|
452
|
+
| 24 | is_num_char_sequence | 380A | No | ----- |
|
|
453
|
+
| 25 | is_abbr | TBMM | Yes | Abbr |
|
|
454
|
+
| 26 | is_in_lexicon | bilgisayar | Yes | Valid_Word |
|
|
455
|
+
| 27 | is_in_exceptions | e-mail | Yes | Exception |
|
|
456
|
+
| 28 | is_in_eng_words | computer | Yes | English_Word |
|
|
457
|
+
| 29 | is_smiley | :) | Yes | Smiley |
|
|
458
|
+
| 30 | is_multiple_smiley | :):) | No | ----- |
|
|
459
|
+
| 31 | is_emoticon | 🍻 | Yes | Emoticon |
|
|
460
|
+
| 32 | is_multiple_emoticon | 🍻🍻 | No | ----- |
|
|
461
|
+
| 33 | is_multiple_smiley_in | hey:):) | No | ----- |
|
|
462
|
+
| 34 | is_number | 175.01 | Yes | Number |
|
|
463
|
+
| 35 | is_apostrophed | Türkiye'nin | Yes | Apostrophed |
|
|
464
|
+
| 36 | is_single_punc | ! | Yes | Punc |
|
|
465
|
+
| 37 | is_multi_punc | !! | No | ----- |
|
|
466
|
+
| 38 | is_single_hyphenated | sabah-akşam | Yes | Single_Hyphenated |
|
|
467
|
+
| 39 | is_multi_hyphenated | çay-su-kahve | Yes | Multi-Hyphenated |
|
|
468
|
+
| 40 | is_single_underscored | Gel_Git | Yes | Single_Underscored |
|
|
469
|
+
| 41 | is_multi_underscored | Yarı_Yapılandırılmış_Mülakat | Yes | Multi_Underscored |
|
|
470
|
+
| 42 | is_one_char_fixable | bilgisa¬yar | Yes | One_Char_Fixed |
|
|
471
|
+
| 43 | is_three_or_more | heyyyyy | No | ----- |
|
|
472
|
+
| 44 | is_fsp | bilgisayar. | No | ----- |
|
|
473
|
+
| 45 | is_isp | .bilgisayar | No | ----- |
|
|
474
|
+
| 46 | is_fmp | bilgisayar.. | No | ----- |
|
|
475
|
+
| 47 | is_imp | ..bilgisayar | No | ----- |
|
|
476
|
+
| 48 | is_msp | --bilgisayar-- | No | ----- |
|
|
477
|
+
| 49 | is_mssp | -bilgisayar- | No | ----- |
|
|
478
|
+
| 50 | is_midsp | okul,öğrenci | No | ----- |
|
|
479
|
+
| 51 | is_midmp | okul,öğrenci, öğretmen | No | ----- |
|
|
480
|
+
| 52 | is_non_latin | 한국드 | No | Non_Latin |
|
|
478
481
|
|
|
479
482
|
----------------------
|
|
480
483
|
|
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
from ts_tokenizer.tokenizer import tokenize
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def tagged_lines(text):
|
|
5
|
+
return [tuple(line.split("\t")) for line in tokenize(text, "tagged").splitlines()]
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def test_candidate_abbr_with_apostrophe_suffix():
|
|
9
|
+
assert tagged_lines("md.'ye") == [
|
|
10
|
+
("md.", "Abbr"),
|
|
11
|
+
("'", "Punc"),
|
|
12
|
+
("ye", "Valid_Word"),
|
|
13
|
+
]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def test_sentence_fragment_with_ordinal_and_abbr_suffix():
|
|
17
|
+
assert tagged_lines("2. md.'ye") == [
|
|
18
|
+
("2.", "Ordinal_Number"),
|
|
19
|
+
("md.", "Abbr"),
|
|
20
|
+
("'", "Punc"),
|
|
21
|
+
("ye", "Valid_Word"),
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def test_existing_abbr_with_apostrophe_suffix():
|
|
26
|
+
assert tagged_lines("Prof.'a") == [
|
|
27
|
+
("Prof.", "Abbr"),
|
|
28
|
+
("'", "Punc"),
|
|
29
|
+
("a", "OOV"),
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def test_ordinal_numbers_before_commas_are_preserved():
|
|
34
|
+
assert tagged_lines("17., 18., 19. yüzyıl.") == [
|
|
35
|
+
("17.", "Ordinal_Number"),
|
|
36
|
+
(",", "Punc"),
|
|
37
|
+
("18.", "Ordinal_Number"),
|
|
38
|
+
(",", "Punc"),
|
|
39
|
+
("19.", "Ordinal_Number"),
|
|
40
|
+
("yüzyıl", "Valid_Word"),
|
|
41
|
+
(".", "Punc"),
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def test_trailing_comma_after_plain_number_is_tokenized():
|
|
46
|
+
assert tagged_lines("Tarih 1453, yer") == [
|
|
47
|
+
("Tarih", "Valid_Word"),
|
|
48
|
+
("1453", "Number"),
|
|
49
|
+
(",", "Punc"),
|
|
50
|
+
("yer", "Valid_Word"),
|
|
51
|
+
]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def test_decimal_number_with_comma_is_preserved():
|
|
55
|
+
assert tagged_lines("1,5") == [("1,5", "Number")]
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def test_signed_numbers_are_preserved_as_numbers():
|
|
59
|
+
assert tagged_lines("+7 +777 -8 -8.5 +7.5") == [
|
|
60
|
+
("+7", "Number"),
|
|
61
|
+
("+777", "Number"),
|
|
62
|
+
("-8", "Number"),
|
|
63
|
+
("-8.5", "Number"),
|
|
64
|
+
("+7.5", "Number"),
|
|
65
|
+
]
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def test_hashtag_with_apostrophe_suffix_is_preserved():
|
|
69
|
+
assert tagged_lines("#Şanlıurfa'da #starbucks'a") == [
|
|
70
|
+
("#Şanlıurfa'da", "Hashtag_Suffix"),
|
|
71
|
+
("#starbucks'a", "Hashtag_Suffix"),
|
|
72
|
+
]
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def test_consecutive_hashtags_are_split_without_breaking_single_hashtags():
|
|
76
|
+
assert tagged_lines("#deprem#yardım #deprem#yardım#cadir #deprem") == [
|
|
77
|
+
("#deprem", "Hashtag"),
|
|
78
|
+
("#yardım", "Hashtag"),
|
|
79
|
+
("#deprem", "Hashtag"),
|
|
80
|
+
("#yardım", "Hashtag"),
|
|
81
|
+
("#cadir", "Hashtag"),
|
|
82
|
+
("#deprem", "Hashtag"),
|
|
83
|
+
]
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def test_mention_with_apostrophe_suffix_is_preserved():
|
|
87
|
+
assert tagged_lines("@SerkanÇayoğlu'ndan") == [
|
|
88
|
+
("@SerkanÇayoğlu'ndan", "Mention_Suffix"),
|
|
89
|
+
]
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def test_single_hyphenated_with_apostrophe_suffix_is_split():
|
|
93
|
+
assert tagged_lines("tom-jerry'nin TOM-jerry'nin") == [
|
|
94
|
+
("tom-jerry", "Single_Hyphenated"),
|
|
95
|
+
("'", "Punc"),
|
|
96
|
+
("nin", "OOV"),
|
|
97
|
+
("TOM-jerry", "Single_Hyphenated"),
|
|
98
|
+
("'", "Punc"),
|
|
99
|
+
("nin", "OOV"),
|
|
100
|
+
]
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def test_numeric_hyphenated_with_apostrophe_suffix_is_preserved():
|
|
104
|
+
assert tagged_lines("COVID-19'un TOM-12'nin") == [
|
|
105
|
+
("COVID-19'un", "Apostrophed"),
|
|
106
|
+
("TOM-12'nin", "Apostrophed"),
|
|
107
|
+
]
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def test_zwj_emoji_sequence_is_preserved_as_single_emoticon():
|
|
111
|
+
assert tagged_lines("👩🏽💻") == [("👩🏽💻", "Emoticon")]
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def test_variation_selector_stays_with_emoji_between_words():
|
|
115
|
+
assert tagged_lines("kelime❤️kelime") == [
|
|
116
|
+
("kelime", "Valid_Word"),
|
|
117
|
+
("❤️", "Emoticon"),
|
|
118
|
+
("kelime", "Valid_Word"),
|
|
119
|
+
]
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def test_formula_tokens_are_preserved():
|
|
123
|
+
assert tagged_lines(
|
|
124
|
+
"p<0.05 g>0.05 p≤.001 F(2,37)=6.42 χ²(3)=12.6 R²=.87 N=1,024 x̄=4.25 ±1.96 α=.05 β₁=-0.37 a=b+c x=3"
|
|
125
|
+
) == [
|
|
126
|
+
("p<0.05", "Formula"),
|
|
127
|
+
("g>0.05", "Formula"),
|
|
128
|
+
("p≤.001", "Formula"),
|
|
129
|
+
("F(2,37)=6.42", "Formula"),
|
|
130
|
+
("χ²(3)=12.6", "Formula"),
|
|
131
|
+
("R²=.87", "Formula"),
|
|
132
|
+
("N=1,024", "Formula"),
|
|
133
|
+
("x̄=4.25", "Formula"),
|
|
134
|
+
("±1.96", "Formula"),
|
|
135
|
+
("α=.05", "Formula"),
|
|
136
|
+
("β₁=-0.37", "Formula"),
|
|
137
|
+
("a=b+c", "Formula"),
|
|
138
|
+
("x=3", "Formula"),
|
|
139
|
+
]
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def test_non_formula_operator_sequences_remain_split():
|
|
143
|
+
assert tagged_lines("kelime=değer bugun=yarin merhaba+dunya") == [
|
|
144
|
+
("kelime", "Valid_Word"),
|
|
145
|
+
("=", "Punc"),
|
|
146
|
+
("değer", "Valid_Word"),
|
|
147
|
+
("bugun", "OOV"),
|
|
148
|
+
("=", "Punc"),
|
|
149
|
+
("yarin", "Valid_Word"),
|
|
150
|
+
("merhaba", "Valid_Word"),
|
|
151
|
+
("+", "Punc"),
|
|
152
|
+
("dunya", "OOV"),
|
|
153
|
+
]
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def test_date_range_suffix_is_preserved():
|
|
157
|
+
assert tagged_lines("01.01.2024-01.01.2025'te") == [
|
|
158
|
+
("01.01.2024-01.01.2025'te", "Date_Range_Suffix"),
|
|
159
|
+
]
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def test_iso_like_dates_are_detected_as_dates():
|
|
163
|
+
assert tagged_lines("2026-08-18 2026.08.18 2026/08/18") == [
|
|
164
|
+
("2026-08-18", "Date"),
|
|
165
|
+
("2026.08.18", "Date"),
|
|
166
|
+
("2026/08/18", "Date"),
|
|
167
|
+
]
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def test_markdown_link_without_space_uses_existing_bracket_and_url_rules():
|
|
171
|
+
assert tagged_lines("[TS Corpus](https://tscorpus.com)") == [
|
|
172
|
+
("[", "Punc"),
|
|
173
|
+
("TS", "Abbr"),
|
|
174
|
+
("Corpus", "English_Word"),
|
|
175
|
+
("]", "Punc"),
|
|
176
|
+
("(", "Punc"),
|
|
177
|
+
("https://tscorpus.com", "Full_URL"),
|
|
178
|
+
(")", "Punc"),
|
|
179
|
+
]
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def test_markdown_link_without_space_supports_email_targets():
|
|
183
|
+
assert tagged_lines("[tanersezerr@gmail.com](tanersezerr@gmail.com)") == [
|
|
184
|
+
("[", "Punc"),
|
|
185
|
+
("tanersezerr@gmail.com", "Email"),
|
|
186
|
+
("]", "Punc"),
|
|
187
|
+
("(", "Punc"),
|
|
188
|
+
("tanersezerr@gmail.com", "Email"),
|
|
189
|
+
(")", "Punc"),
|
|
190
|
+
]
|
|
191
|
+
assert tagged_lines("[tanersezerr@gmail.com](mailto:tanersezerr@gmail.com)") == [
|
|
192
|
+
("[", "Punc"),
|
|
193
|
+
("tanersezerr@gmail.com", "Email"),
|
|
194
|
+
("]", "Punc"),
|
|
195
|
+
("(", "Punc"),
|
|
196
|
+
("mailto:tanersezerr@gmail.com", "Email"),
|
|
197
|
+
(")", "Punc"),
|
|
198
|
+
]
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def test_ip_addresses_are_preserved_without_breaking_trailing_punctuation():
|
|
202
|
+
assert tagged_lines("192.168.1.20 2001:db8::8a2e:370:7334 192.168.1.20.") == [
|
|
203
|
+
("192.168.1.20", "IP_Address"),
|
|
204
|
+
("2001:db8::8a2e:370:7334", "IP_Address"),
|
|
205
|
+
("192.168.1.20", "IP_Address"),
|
|
206
|
+
(".", "Punc"),
|
|
207
|
+
]
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def test_doi_and_hyphenated_isbn_are_preserved():
|
|
211
|
+
assert tagged_lines("ISBN 978-975-0000-00-0 DOI:10.1234/abcd.2026.001 DOI: 10.1234/abcd.2026.001 9789750000000") == [
|
|
212
|
+
("ISBN", "Abbr"),
|
|
213
|
+
("978-975-0000-00-0", "ISBN"),
|
|
214
|
+
("DOI:10.1234/abcd.2026.001", "DOI"),
|
|
215
|
+
("DOI", "Abbr"),
|
|
216
|
+
(":", "Punc"),
|
|
217
|
+
("10.1234/abcd.2026.001", "DOI"),
|
|
218
|
+
("9789750000000", "Number"),
|
|
219
|
+
]
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def test_three_or_more_initial_punctuation_before_word_is_split_consistently():
|
|
223
|
+
assert tagged_lines("???merhaba---") == [
|
|
224
|
+
("?", "Punc"),
|
|
225
|
+
("?", "Punc"),
|
|
226
|
+
("?", "Punc"),
|
|
227
|
+
("merhaba", "Valid_Word"),
|
|
228
|
+
("-", "Punc"),
|
|
229
|
+
("-", "Punc"),
|
|
230
|
+
("-", "Punc"),
|
|
231
|
+
]
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def test_three_or_more_symmetric_punctuation_around_word_is_split_consistently():
|
|
235
|
+
assert tagged_lines("***merhaba***") == [
|
|
236
|
+
("*", "Punc"),
|
|
237
|
+
("*", "Punc"),
|
|
238
|
+
("*", "Punc"),
|
|
239
|
+
("merhaba", "Valid_Word"),
|
|
240
|
+
("*", "Punc"),
|
|
241
|
+
("*", "Punc"),
|
|
242
|
+
("*", "Punc"),
|
|
243
|
+
]
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def test_roman_numeral_in_parentheses_is_numbered_title():
|
|
247
|
+
assert tagged_lines("Ardından (1), (2), (III) ve (iii) numaralı bölümlere geçti.") == [
|
|
248
|
+
("Ardından", "Valid_Word"),
|
|
249
|
+
("(1)", "Numbered_Title"),
|
|
250
|
+
(",", "Punc"),
|
|
251
|
+
("(2)", "Numbered_Title"),
|
|
252
|
+
(",", "Punc"),
|
|
253
|
+
("(III)", "Numbered_Title"),
|
|
254
|
+
("ve", "Valid_Word"),
|
|
255
|
+
("(iii)", "Numbered_Title"),
|
|
256
|
+
("numaralı", "Valid_Word"),
|
|
257
|
+
("bölümlere", "Valid_Word"),
|
|
258
|
+
("geçti", "Valid_Word"),
|
|
259
|
+
(".", "Punc"),
|
|
260
|
+
]
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
from ts_tokenizer.tokenizer import tokenize
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def tagged_lines(text):
|
|
5
|
+
return [tuple(line.split("\t")) for line in tokenize(text, "tagged").splitlines()]
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def test_uppercase_abbreviation_from_abbr_list():
|
|
9
|
+
assert tagged_lines("YHT") == [("YHT", "Abbr")]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def test_existing_abbreviation_behavior_is_preserved():
|
|
13
|
+
assert tagged_lines("TBMM'ye") == [("TBMM'ye", "Apostrophed")]
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from ts_tokenizer.tokenizer import tokenize
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def tagged_lines(text):
|
|
5
|
+
return [tuple(line.split("\t")) for line in tokenize(text, "tagged").splitlines()]
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def test_currency_suffix_forms_are_preserved():
|
|
9
|
+
assert tagged_lines("₺350'lik") == [("₺350'lik", "Currency")]
|
|
10
|
+
assert tagged_lines("350₺'lik") == [("350₺'lik", "Currency")]
|
|
11
|
+
assert tagged_lines("$100'luk") == [("$100'luk", "Currency")]
|
|
12
|
+
assert tagged_lines("100$'lık") == [("100$'lık", "Currency")]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def test_plain_currency_forms_remain_unchanged():
|
|
16
|
+
assert tagged_lines("350₺") == [("350₺", "Currency")]
|
|
17
|
+
assert tagged_lines("₺350") == [("₺350", "Currency")]
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
from ts_tokenizer.tokenizer import tokenize
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def tagged_lines(text):
|
|
5
|
+
return [tuple(line.split("\t")) for line in tokenize(text, "tagged").splitlines()]
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def test_registered_mark_allows_underscored_words():
|
|
9
|
+
assert tagged_lines("yeni_firma®") == [
|
|
10
|
+
("yeni_firma®", "Registered"),
|
|
11
|
+
]
|
|
12
|
+
assert tagged_lines("eski_Firma®") == [
|
|
13
|
+
("eski_Firma®", "Registered"),
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def test_existing_symbolic_mark_behavior_is_preserved():
|
|
18
|
+
assert tagged_lines("tscorpus®") == [("tscorpus®", "Registered")]
|
|
19
|
+
assert tagged_lines("abc™") == [("abc™", "Trademark")]
|
|
20
|
+
assert tagged_lines("©tscorpus") == [("©tscorpus", "Copyright")]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_symbolic_marks_with_trailing_punctuation_are_preserved():
|
|
24
|
+
assert tagged_lines("™kuruldu.") == [
|
|
25
|
+
("™kuruldu", "Trademark"),
|
|
26
|
+
(".", "Punc"),
|
|
27
|
+
]
|
|
28
|
+
assert tagged_lines("kapandı©.") == [
|
|
29
|
+
("kapandı©", "Copyright"),
|
|
30
|
+
(".", "Punc"),
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def test_mixed_sentence_with_symbolic_marks():
|
|
35
|
+
assert tagged_lines("yeni_firma® ™kuruldu. eskiFirma® kapandı©. ®yepyeni") == [
|
|
36
|
+
("yeni_firma®", "Registered"),
|
|
37
|
+
("™kuruldu", "Trademark"),
|
|
38
|
+
(".", "Punc"),
|
|
39
|
+
("eskiFirma®", "Registered"),
|
|
40
|
+
("kapandı©", "Copyright"),
|
|
41
|
+
(".", "Punc"),
|
|
42
|
+
("®yepyeni", "Registered"),
|
|
43
|
+
]
|
|
@@ -20,7 +20,7 @@ REPLACEMENTS_CHAR = [
|
|
|
20
20
|
# Fix Unicoed Chars
|
|
21
21
|
("і", "i"),
|
|
22
22
|
# Fix Dashes - Need a better solution
|
|
23
|
-
("-", "-"), ("–", "-"), ("⁄", "/"), ("-", "-"), ("—-", "-"), ("﴾", "("), ("﴿", ")"), ("-", "-"), ("","-"), ("−", "-"),
|
|
23
|
+
("-", "-"), ("–", "-"), ("⁄", "/"), ("-", "-"), ("—-", "-"), ("﴾", "("), ("﴿", ")"), ("-", "-"), ("","-"), ("−", "-"), ("—", "-"),
|
|
24
24
|
# punctuation_process.py line 178 - fails if not replaced
|
|
25
25
|
# needs a better solution
|
|
26
26
|
("°°", "°"), ("_\uFE0F", "_"),
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import os
|
|
2
2
|
from typing import FrozenSet
|
|
3
3
|
|
|
4
|
+
from .char_fix import CharFix
|
|
5
|
+
|
|
4
6
|
base_dir = os.path.dirname(os.path.abspath(__file__))
|
|
5
7
|
data_dir = os.path.join(base_dir, 'data')
|
|
6
8
|
|
|
@@ -8,7 +10,8 @@ data_dir = os.path.join(base_dir, 'data')
|
|
|
8
10
|
|
|
9
11
|
emoticons = set(line.strip() for line in open(os.path.join(data_dir, 'emoticons.txt')))
|
|
10
12
|
smileys = set(line.strip() for line in open(os.path.join(data_dir, 'smileys.txt')))
|
|
11
|
-
abbrs = set(line.strip() for line in open(os.path.join(data_dir, 'abbr_list.txt')))
|
|
13
|
+
abbrs = set(CharFix.tr_lowercase(line.strip()) for line in open(os.path.join(data_dir, 'abbr_list.txt')))
|
|
14
|
+
candidate_abbrs = set(line.strip() for line in open(os.path.join(data_dir, 'cand_abbr_list.txt')))
|
|
12
15
|
word_list = set(line.strip() for line in open(os.path.join(data_dir, 'TS_Corpus_Turkish_Word_List.txt')))
|
|
13
16
|
exception_words = set(line.strip() for line in open(os.path.join(data_dir, 'exceptions.txt')))
|
|
14
17
|
eng_words = set(line.strip() for line in open(os.path.join(data_dir, 'eng_word_list.txt')))
|
|
@@ -30,6 +33,10 @@ class LocalData:
|
|
|
30
33
|
def abbrs():
|
|
31
34
|
return abbrs
|
|
32
35
|
|
|
36
|
+
@staticmethod
|
|
37
|
+
def candidate_abbrs():
|
|
38
|
+
return candidate_abbrs
|
|
39
|
+
|
|
33
40
|
@staticmethod
|
|
34
41
|
def word_list():
|
|
35
42
|
return word_list
|
|
@@ -23,6 +23,15 @@ class DateValidator:
|
|
|
23
23
|
month, day, year = map(int, [month, day, year])
|
|
24
24
|
return cls._is_valid_day_month_year(day, month, year)
|
|
25
25
|
|
|
26
|
+
@classmethod
|
|
27
|
+
def is_valid_date_yyyy_mm_dd(cls, date_str: str) -> bool:
|
|
28
|
+
match = re.match(r"^(\d{4})([-/.:])(0?[1-9]|1[0-2])\2(0?[1-9]|[1-2][0-9]|3[0-1])$", date_str)
|
|
29
|
+
if not match:
|
|
30
|
+
return False
|
|
31
|
+
year, _, month, day = match.groups()
|
|
32
|
+
day, month, year = map(int, [day, month, year])
|
|
33
|
+
return cls._is_valid_day_month_year(day, month, year)
|
|
34
|
+
|
|
26
35
|
@classmethod
|
|
27
36
|
def _is_valid_day_month_year(cls, day: int, month: int, year: int) -> bool:
|
|
28
37
|
if month in [1, 3, 5, 7, 8, 10, 12]:
|
|
@@ -44,6 +53,7 @@ class DateCheck:
|
|
|
44
53
|
def is_date(date_str: str) -> bool:
|
|
45
54
|
# Check if the entire string is a date
|
|
46
55
|
return (
|
|
56
|
+
DateValidator.is_valid_date_yyyy_mm_dd(date_str.strip()) or
|
|
47
57
|
DateValidator.is_valid_date_mm_dd_yyyy(date_str.strip()) or
|
|
48
58
|
DateValidator.is_valid_date_dd_mm_yyyy(date_str.strip())
|
|
49
59
|
)
|