ts-tokenizer 0.2.4__tar.gz → 0.2.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. {ts_tokenizer-0.2.4/ts_tokenizer.egg-info → ts_tokenizer-0.2.6}/PKG-INFO +57 -57
  2. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/README.md +56 -56
  3. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer/__init__.py +1 -1
  4. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6/ts_tokenizer.egg-info}/PKG-INFO +57 -57
  5. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/LICENSE +0 -0
  6. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/MANIFEST.in +0 -0
  7. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/pyproject.toml +0 -0
  8. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/setup.cfg +0 -0
  9. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/setup.py +0 -0
  10. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/tests/test_abbr_apostrophe_suffix.py +0 -0
  11. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/tests/test_abbreviation_lookup.py +0 -0
  12. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/tests/test_currency_suffix.py +0 -0
  13. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/tests/test_opening_quote_tokenization.py +0 -0
  14. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/tests/test_symbolic_marks.py +0 -0
  15. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer/char_fix.py +0 -0
  16. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer/cli.py +0 -0
  17. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer/data/TS_Corpus_Turkish_Word_List.txt +0 -0
  18. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer/data/abbr_list.txt +0 -0
  19. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer/data/cand_abbr_list.txt +0 -0
  20. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer/data/corection_mark.txt +0 -0
  21. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer/data/currency_symbols.txt +0 -0
  22. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer/data/domains.txt +0 -0
  23. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer/data/emoticons.txt +0 -0
  24. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer/data/eng_word_list.txt +0 -0
  25. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer/data/exceptions.txt +0 -0
  26. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer/data/smileys.txt +0 -0
  27. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer/data.py +0 -0
  28. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer/date_check.py +0 -0
  29. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer/emoticon_check.py +0 -0
  30. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer/inner_punc.py +0 -0
  31. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer/punctuation_process.py +0 -0
  32. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer/smiley_check.py +0 -0
  33. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer/token_handler.py +0 -0
  34. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer/tokenizer.py +0 -0
  35. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer/unit_test.py +0 -0
  36. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer.egg-info/SOURCES.txt +0 -0
  37. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer.egg-info/dependency_links.txt +0 -0
  38. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer.egg-info/entry_points.txt +0 -0
  39. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer.egg-info/requires.txt +0 -0
  40. {ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer.egg-info/top_level.txt +0 -0
{ts_tokenizer-0.2.4/ts_tokenizer.egg-info → ts_tokenizer-0.2.6}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ts_tokenizer
3
- Version: 0.2.4
3
+ Version: 0.2.6
4
4
  Summary: TS Tokenizer is a hybrid (lexicon-based and rule-based) tokenizer designed specifically for tokenizing Turkish texts.
5
5
  Home-page: https://github.com/tanerim/ts_tokenizer
6
6
  Author: Taner Sezer
@@ -462,62 +462,62 @@ from ts_tokenizer import TokenPreProcess
462
462
  ### Below are the list of tags generated by tokenizer to process tokens:
463
463
 
464
464
 
465
- | # | Function | Sample | Used As Output | Tag |
466
- |----|----------------------------|------------------------------|----------------|--------------------|
467
- | 01 | is_mention | @ts-tokenizer | Yes | Mention |
468
- | 02 | is_hashtag | #ts-tokenizer | Yes | Hashtag |
469
- | -- | is_mention_suffix | @SerkanÇayoğlu'ndan | Yes | Mention_Suffix |
470
- | -- | is_hashtag_suffix | #Şanlıurfa'da | Yes | Hashtag_Suffix |
471
- | 03 | is_in_quotes | "ts-tokenizer" | No | ----- |
472
- | 04 | is_numbered_title | (1) | Yes | Numbered_Title |
473
- | 05 | is_in_paranthesis | (bilgisayar) | No | ----- |
474
- | 06 | is_date_range | 01.01.2024-01.01.2025 | Yes | Date_Range |
475
- | 07 | is_complex_punc | -yeniden,sonradan.. | No | ----- |
476
- | 08 | is_date | 22.02.2016 | Yes | Date |
477
- | 09 | is_hour | 14.05 | Yes | Hour |
478
- | 10 | is_percentage_numbers | %75 | Yes | Percentage_Numbers |
479
- | 11 | is_percentage_numbers_chars | %75'lik | Yes | Percentage_Numbers |
480
- | 12 | is_roman_number | XI | Yes | Roman_Number |
481
- | 13 | is_bullet_list | •Giriş | Yes | Bullet_List |
482
- | 14 | is_email | tanersezerr@gmail.com | Yes | Email |
483
- | 15 | is_email_punc | tanersezerr@gmail.com. | No | ----- |
484
- | 16 | is_full_url | https://tscorpus.com | Yes | Full_URL |
485
- | 17 | is_web_url | www.tscorpus.com | Yes | Web_URL |
486
- | 18 | is_full_url | www.example.com'un | Yes | URL_Suffix |
487
- | 19 | is_formula | F(2,37)=6.42 | Yes | Formula |
488
- | 20 | is_copyright | ©tscorpus | Yes | Copyright |
489
- | 21 | is_registered | tscorpus® | Yes | Registered |
490
- | 22 | is_trademark | tscorpus™ | Yes | Trademark |
491
- | 23 | is_currency | 100$ | Yes | Currency |
492
- | 24 | is_num_char_sequence | 380A | No | ----- |
493
- | 25 | is_abbr | TBMM | Yes | Abbr |
494
- | 26 | is_in_lexicon | bilgisayar | Yes | Valid_Word |
495
- | 27 | is_in_exceptions | e-mail | Yes | Exception |
496
- | 28 | is_in_eng_words | computer | Yes | English_Word |
497
- | 29 | is_smiley | :) | Yes | Smiley |
498
- | 30 | is_multiple_smiley | :):) | No | ----- |
499
- | 31 | is_emoticon | 🍻 | Yes | Emoticon |
500
- | 32 | is_multiple_emoticon | 🍻🍻 | No | ----- |
501
- | 33 | is_multiple_smiley_in | hey:):) | No | ----- |
502
- | 34 | is_number | 175.01 | Yes | Number |
503
- | 35 | is_apostrophed | Türkiye'nin | Yes | Apostrophed |
504
- | 36 | is_single_punc | ! | Yes | Punc |
505
- | 37 | is_multi_punc | !! | No | ----- |
506
- | 38 | is_single_hyphenated | sabah-akşam | Yes | Single_Hyphenated |
507
- | 39 | is_multi_hyphenated | çay-su-kahve | Yes | Multi-Hyphenated |
508
- | 40 | is_single_underscored | Gel_Git | Yes | Single_Underscored |
509
- | 41 | is_multi_underscored | Yarı_Yapılandırılmış_Mülakat | Yes | Multi_Underscored |
510
- | 42 | is_one_char_fixable | bilgisa¬yar | Yes | One_Char_Fixed |
511
- | 43 | is_three_or_more | heyyyyy | No | ----- |
512
- | 44 | is_fsp | bilgisayar. | No | ----- |
513
- | 45 | is_isp | .bilgisayar | No | ----- |
514
- | 46 | is_fmp | bilgisayar.. | No | ----- |
515
- | 47 | is_imp | ..bilgisayar | No | ----- |
516
- | 48 | is_msp | --bilgisayar-- | No | ----- |
517
- | 49 | is_mssp | -bilgisayar- | No | ----- |
518
- | 50 | is_midsp | okul,öğrenci | No | ----- |
519
- | 51 | is_midmp | okul,öğrenci, öğretmen | No | ----- |
520
- | 52 | is_non_latin | 한국드 | No | Non_Latin |
465
+ | # | Function | Sample | Used As Output | Tag |
466
+ |----|-----------------------------|------------------------------|----------------|--------------------|
467
+ | 01 | is_mention | @ts-tokenizer | Yes | Mention |
468
+ | 02 | is_hashtag | #ts-tokenizer | Yes | Hashtag |
469
+ | 03 | is_mention_suffix | @SerkanÇayoğlu'ndan | Yes | Mention_Suffix |
470
+ | 04 | is_hashtag_suffix | #Şanlıurfa'da | Yes | Hashtag_Suffix |
471
+ | 05 | is_in_quotes | "ts-tokenizer" | No | ----- |
472
+ | 06 | is_numbered_title | (1) | Yes | Numbered_Title |
473
+ | 07 | is_in_paranthesis | (bilgisayar) | No | ----- |
474
+ | 08 | is_date_range | 01.01.2024-01.01.2025 | Yes | Date_Range |
475
+ | 09 | is_complex_punc | -yeniden,sonradan.. | No | ----- |
476
+ | 10 | is_date | 22.02.2016 | Yes | Date |
477
+ | 11 | is_hour | 14.05 | Yes | Hour |
478
+ | 12 | is_percentage_numbers | %75 | Yes | Percentage_Numbers |
479
+ | 13 | is_percentage_numbers_chars | %75'lik | Yes | Percentage_Numbers |
480
+ | 14 | is_roman_number | XI | Yes | Roman_Number |
481
+ | 15 | is_bullet_list | •Giriş | Yes | Bullet_List |
482
+ | 16 | is_email | tanersezerr@gmail.com | Yes | Email |
483
+ | 17 | is_email_punc | tanersezerr@gmail.com. | No | ----- |
484
+ | 18 | is_full_url | https://tscorpus.com | Yes | Full_URL |
485
+ | 19 | is_web_url | www.tscorpus.com | Yes | Web_URL |
486
+ | 20 | is_full_url | www.example.com'un | Yes | URL_Suffix |
487
+ | 21 | is_copyright | ©tscorpus | Yes | Copyright |
488
+ | 22 | is_registered | tscorpus® | Yes | Registered |
489
+ | 23 | is_trademark | tscorpus™ | Yes | Trademark |
490
+ | 24 | is_currency | 100$ | Yes | Currency |
491
+ | 25 | is_num_char_sequence | 380A | No | ----- |
492
+ | 26 | is_abbr | TBMM | Yes | Abbr |
493
+ | 27 | is_in_lexicon | bilgisayar | Yes | Valid_Word |
494
+ | 28 | is_in_exceptions | e-mail | Yes | Exception |
495
+ | 29 | is_in_eng_words | computer | Yes | English_Word |
496
+ | 30 | is_smiley | :) | Yes | Smiley |
497
+ | 31 | is_multiple_smiley | :):) | No | ----- |
498
+ | 32 | is_emoticon | 🍻 | Yes | Emoticon |
499
+ | 33 | is_multiple_emoticon | 🍻🍻 | No | ----- |
500
+ | 34 | is_multiple_smiley_in | hey:):) | No | ----- |
501
+ | 35 | is_number | 175.01 | Yes | Number |
502
+ | 36 | is_apostrophed | Türkiye'nin | Yes | Apostrophed |
503
+ | 37 | is_single_punc | ! | Yes | Punc |
504
+ | 38 | is_multi_punc | !! | No | ----- |
505
+ | 39 | is_single_hyphenated | sabah-akşam | Yes | Single_Hyphenated |
506
+ | 40 | is_multi_hyphenated | çay-su-kahve | Yes | Multi-Hyphenated |
507
+ | 41 | is_single_underscored | Gel_Git | Yes | Single_Underscored |
508
+ | 42 | is_multi_underscored | Yarı_Yapılandırılmış_Mülakat | Yes | Multi_Underscored |
509
+ | 43 | is_one_char_fixable | bilgisa¬yar | Yes | One_Char_Fixed |
510
+ | 44 | is_formula | F(2,37)=6.42 | Yes | Formula |
511
+ | 45 | is_three_or_more | heyyyyy | No | ----- |
512
+ | 46 | is_fsp | bilgisayar. | No | ----- |
513
+ | 47 | is_isp | .bilgisayar | No | ----- |
514
+ | 48 | is_fmp | bilgisayar.. | No | ----- |
515
+ | 49 | is_imp | ..bilgisayar | No | ----- |
516
+ | 50 | is_msp | --bilgisayar-- | No | ----- |
517
+ | 51 | is_mssp | -bilgisayar- | No | ----- |
518
+ | 52 | is_midsp | okul,öğrenci | No | ----- |
519
+ | 53 | is_midmp | okul,öğrenci, öğretmen | No | ----- |
520
+ | 54 | is_non_latin | 한국드 | No | Non_Latin |
521
521
 
522
522
  ----------------------
523
523
 
{ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/README.md RENAMED
@@ -422,62 +422,62 @@ from ts_tokenizer import TokenPreProcess
422
422
  ### Below are the list of tags generated by tokenizer to process tokens:
423
423
 
424
424
 
425
- | # | Function | Sample | Used As Output | Tag |
426
- |----|----------------------------|------------------------------|----------------|--------------------|
427
- | 01 | is_mention | @ts-tokenizer | Yes | Mention |
428
- | 02 | is_hashtag | #ts-tokenizer | Yes | Hashtag |
429
- | -- | is_mention_suffix | @SerkanÇayoğlu'ndan | Yes | Mention_Suffix |
430
- | -- | is_hashtag_suffix | #Şanlıurfa'da | Yes | Hashtag_Suffix |
431
- | 03 | is_in_quotes | "ts-tokenizer" | No | ----- |
432
- | 04 | is_numbered_title | (1) | Yes | Numbered_Title |
433
- | 05 | is_in_paranthesis | (bilgisayar) | No | ----- |
434
- | 06 | is_date_range | 01.01.2024-01.01.2025 | Yes | Date_Range |
435
- | 07 | is_complex_punc | -yeniden,sonradan.. | No | ----- |
436
- | 08 | is_date | 22.02.2016 | Yes | Date |
437
- | 09 | is_hour | 14.05 | Yes | Hour |
438
- | 10 | is_percentage_numbers | %75 | Yes | Percentage_Numbers |
439
- | 11 | is_percentage_numbers_chars | %75'lik | Yes | Percentage_Numbers |
440
- | 12 | is_roman_number | XI | Yes | Roman_Number |
441
- | 13 | is_bullet_list | •Giriş | Yes | Bullet_List |
442
- | 14 | is_email | tanersezerr@gmail.com | Yes | Email |
443
- | 15 | is_email_punc | tanersezerr@gmail.com. | No | ----- |
444
- | 16 | is_full_url | https://tscorpus.com | Yes | Full_URL |
445
- | 17 | is_web_url | www.tscorpus.com | Yes | Web_URL |
446
- | 18 | is_full_url | www.example.com'un | Yes | URL_Suffix |
447
- | 19 | is_formula | F(2,37)=6.42 | Yes | Formula |
448
- | 20 | is_copyright | ©tscorpus | Yes | Copyright |
449
- | 21 | is_registered | tscorpus® | Yes | Registered |
450
- | 22 | is_trademark | tscorpus™ | Yes | Trademark |
451
- | 23 | is_currency | 100$ | Yes | Currency |
452
- | 24 | is_num_char_sequence | 380A | No | ----- |
453
- | 25 | is_abbr | TBMM | Yes | Abbr |
454
- | 26 | is_in_lexicon | bilgisayar | Yes | Valid_Word |
455
- | 27 | is_in_exceptions | e-mail | Yes | Exception |
456
- | 28 | is_in_eng_words | computer | Yes | English_Word |
457
- | 29 | is_smiley | :) | Yes | Smiley |
458
- | 30 | is_multiple_smiley | :):) | No | ----- |
459
- | 31 | is_emoticon | 🍻 | Yes | Emoticon |
460
- | 32 | is_multiple_emoticon | 🍻🍻 | No | ----- |
461
- | 33 | is_multiple_smiley_in | hey:):) | No | ----- |
462
- | 34 | is_number | 175.01 | Yes | Number |
463
- | 35 | is_apostrophed | Türkiye'nin | Yes | Apostrophed |
464
- | 36 | is_single_punc | ! | Yes | Punc |
465
- | 37 | is_multi_punc | !! | No | ----- |
466
- | 38 | is_single_hyphenated | sabah-akşam | Yes | Single_Hyphenated |
467
- | 39 | is_multi_hyphenated | çay-su-kahve | Yes | Multi-Hyphenated |
468
- | 40 | is_single_underscored | Gel_Git | Yes | Single_Underscored |
469
- | 41 | is_multi_underscored | Yarı_Yapılandırılmış_Mülakat | Yes | Multi_Underscored |
470
- | 42 | is_one_char_fixable | bilgisa¬yar | Yes | One_Char_Fixed |
471
- | 43 | is_three_or_more | heyyyyy | No | ----- |
472
- | 44 | is_fsp | bilgisayar. | No | ----- |
473
- | 45 | is_isp | .bilgisayar | No | ----- |
474
- | 46 | is_fmp | bilgisayar.. | No | ----- |
475
- | 47 | is_imp | ..bilgisayar | No | ----- |
476
- | 48 | is_msp | --bilgisayar-- | No | ----- |
477
- | 49 | is_mssp | -bilgisayar- | No | ----- |
478
- | 50 | is_midsp | okul,öğrenci | No | ----- |
479
- | 51 | is_midmp | okul,öğrenci, öğretmen | No | ----- |
480
- | 52 | is_non_latin | 한국드 | No | Non_Latin |
425
+ | # | Function | Sample | Used As Output | Tag |
426
+ |----|-----------------------------|------------------------------|----------------|--------------------|
427
+ | 01 | is_mention | @ts-tokenizer | Yes | Mention |
428
+ | 02 | is_hashtag | #ts-tokenizer | Yes | Hashtag |
429
+ | 03 | is_mention_suffix | @SerkanÇayoğlu'ndan | Yes | Mention_Suffix |
430
+ | 04 | is_hashtag_suffix | #Şanlıurfa'da | Yes | Hashtag_Suffix |
431
+ | 05 | is_in_quotes | "ts-tokenizer" | No | ----- |
432
+ | 06 | is_numbered_title | (1) | Yes | Numbered_Title |
433
+ | 07 | is_in_paranthesis | (bilgisayar) | No | ----- |
434
+ | 08 | is_date_range | 01.01.2024-01.01.2025 | Yes | Date_Range |
435
+ | 09 | is_complex_punc | -yeniden,sonradan.. | No | ----- |
436
+ | 10 | is_date | 22.02.2016 | Yes | Date |
437
+ | 11 | is_hour | 14.05 | Yes | Hour |
438
+ | 12 | is_percentage_numbers | %75 | Yes | Percentage_Numbers |
439
+ | 13 | is_percentage_numbers_chars | %75'lik | Yes | Percentage_Numbers |
440
+ | 14 | is_roman_number | XI | Yes | Roman_Number |
441
+ | 15 | is_bullet_list | •Giriş | Yes | Bullet_List |
442
+ | 16 | is_email | tanersezerr@gmail.com | Yes | Email |
443
+ | 17 | is_email_punc | tanersezerr@gmail.com. | No | ----- |
444
+ | 18 | is_full_url | https://tscorpus.com | Yes | Full_URL |
445
+ | 19 | is_web_url | www.tscorpus.com | Yes | Web_URL |
446
+ | 20 | is_full_url | www.example.com'un | Yes | URL_Suffix |
447
+ | 21 | is_copyright | ©tscorpus | Yes | Copyright |
448
+ | 22 | is_registered | tscorpus® | Yes | Registered |
449
+ | 23 | is_trademark | tscorpus™ | Yes | Trademark |
450
+ | 24 | is_currency | 100$ | Yes | Currency |
451
+ | 25 | is_num_char_sequence | 380A | No | ----- |
452
+ | 26 | is_abbr | TBMM | Yes | Abbr |
453
+ | 27 | is_in_lexicon | bilgisayar | Yes | Valid_Word |
454
+ | 28 | is_in_exceptions | e-mail | Yes | Exception |
455
+ | 29 | is_in_eng_words | computer | Yes | English_Word |
456
+ | 30 | is_smiley | :) | Yes | Smiley |
457
+ | 31 | is_multiple_smiley | :):) | No | ----- |
458
+ | 32 | is_emoticon | 🍻 | Yes | Emoticon |
459
+ | 33 | is_multiple_emoticon | 🍻🍻 | No | ----- |
460
+ | 34 | is_multiple_smiley_in | hey:):) | No | ----- |
461
+ | 35 | is_number | 175.01 | Yes | Number |
462
+ | 36 | is_apostrophed | Türkiye'nin | Yes | Apostrophed |
463
+ | 37 | is_single_punc | ! | Yes | Punc |
464
+ | 38 | is_multi_punc | !! | No | ----- |
465
+ | 39 | is_single_hyphenated | sabah-akşam | Yes | Single_Hyphenated |
466
+ | 40 | is_multi_hyphenated | çay-su-kahve | Yes | Multi-Hyphenated |
467
+ | 41 | is_single_underscored | Gel_Git | Yes | Single_Underscored |
468
+ | 42 | is_multi_underscored | Yarı_Yapılandırılmış_Mülakat | Yes | Multi_Underscored |
469
+ | 43 | is_one_char_fixable | bilgisa¬yar | Yes | One_Char_Fixed |
470
+ | 44 | is_formula | F(2,37)=6.42 | Yes | Formula |
471
+ | 45 | is_three_or_more | heyyyyy | No | ----- |
472
+ | 46 | is_fsp | bilgisayar. | No | ----- |
473
+ | 47 | is_isp | .bilgisayar | No | ----- |
474
+ | 48 | is_fmp | bilgisayar.. | No | ----- |
475
+ | 49 | is_imp | ..bilgisayar | No | ----- |
476
+ | 50 | is_msp | --bilgisayar-- | No | ----- |
477
+ | 51 | is_mssp | -bilgisayar- | No | ----- |
478
+ | 52 | is_midsp | okul,öğrenci | No | ----- |
479
+ | 53 | is_midmp | okul,öğrenci, öğretmen | No | ----- |
480
+ | 54 | is_non_latin | 한국드 | No | Non_Latin |
481
481
 
482
482
  ----------------------
483
483
 
{ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/ts_tokenizer/__init__.py RENAMED
@@ -1,6 +1,6 @@
1
1
  from importlib import import_module
2
2
 
3
- __version__ = "0.2.4"
3
+ __version__ = "0.2.6"
4
4
 
5
5
  _EXPORT_MAP = {
6
6
  "tokenize": (".tokenizer", "tokenize"),
{ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6/ts_tokenizer.egg-info}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ts_tokenizer
3
- Version: 0.2.4
3
+ Version: 0.2.6
4
4
  Summary: TS Tokenizer is a hybrid (lexicon-based and rule-based) tokenizer designed specifically for tokenizing Turkish texts.
5
5
  Home-page: https://github.com/tanerim/ts_tokenizer
6
6
  Author: Taner Sezer
@@ -462,62 +462,62 @@ from ts_tokenizer import TokenPreProcess
462
462
  ### Below are the list of tags generated by tokenizer to process tokens:
463
463
 
464
464
 
465
- | # | Function | Sample | Used As Output | Tag |
466
- |----|----------------------------|------------------------------|----------------|--------------------|
467
- | 01 | is_mention | @ts-tokenizer | Yes | Mention |
468
- | 02 | is_hashtag | #ts-tokenizer | Yes | Hashtag |
469
- | -- | is_mention_suffix | @SerkanÇayoğlu'ndan | Yes | Mention_Suffix |
470
- | -- | is_hashtag_suffix | #Şanlıurfa'da | Yes | Hashtag_Suffix |
471
- | 03 | is_in_quotes | "ts-tokenizer" | No | ----- |
472
- | 04 | is_numbered_title | (1) | Yes | Numbered_Title |
473
- | 05 | is_in_paranthesis | (bilgisayar) | No | ----- |
474
- | 06 | is_date_range | 01.01.2024-01.01.2025 | Yes | Date_Range |
475
- | 07 | is_complex_punc | -yeniden,sonradan.. | No | ----- |
476
- | 08 | is_date | 22.02.2016 | Yes | Date |
477
- | 09 | is_hour | 14.05 | Yes | Hour |
478
- | 10 | is_percentage_numbers | %75 | Yes | Percentage_Numbers |
479
- | 11 | is_percentage_numbers_chars | %75'lik | Yes | Percentage_Numbers |
480
- | 12 | is_roman_number | XI | Yes | Roman_Number |
481
- | 13 | is_bullet_list | •Giriş | Yes | Bullet_List |
482
- | 14 | is_email | tanersezerr@gmail.com | Yes | Email |
483
- | 15 | is_email_punc | tanersezerr@gmail.com. | No | ----- |
484
- | 16 | is_full_url | https://tscorpus.com | Yes | Full_URL |
485
- | 17 | is_web_url | www.tscorpus.com | Yes | Web_URL |
486
- | 18 | is_full_url | www.example.com'un | Yes | URL_Suffix |
487
- | 19 | is_formula | F(2,37)=6.42 | Yes | Formula |
488
- | 20 | is_copyright | ©tscorpus | Yes | Copyright |
489
- | 21 | is_registered | tscorpus® | Yes | Registered |
490
- | 22 | is_trademark | tscorpus™ | Yes | Trademark |
491
- | 23 | is_currency | 100$ | Yes | Currency |
492
- | 24 | is_num_char_sequence | 380A | No | ----- |
493
- | 25 | is_abbr | TBMM | Yes | Abbr |
494
- | 26 | is_in_lexicon | bilgisayar | Yes | Valid_Word |
495
- | 27 | is_in_exceptions | e-mail | Yes | Exception |
496
- | 28 | is_in_eng_words | computer | Yes | English_Word |
497
- | 29 | is_smiley | :) | Yes | Smiley |
498
- | 30 | is_multiple_smiley | :):) | No | ----- |
499
- | 31 | is_emoticon | 🍻 | Yes | Emoticon |
500
- | 32 | is_multiple_emoticon | 🍻🍻 | No | ----- |
501
- | 33 | is_multiple_smiley_in | hey:):) | No | ----- |
502
- | 34 | is_number | 175.01 | Yes | Number |
503
- | 35 | is_apostrophed | Türkiye'nin | Yes | Apostrophed |
504
- | 36 | is_single_punc | ! | Yes | Punc |
505
- | 37 | is_multi_punc | !! | No | ----- |
506
- | 38 | is_single_hyphenated | sabah-akşam | Yes | Single_Hyphenated |
507
- | 39 | is_multi_hyphenated | çay-su-kahve | Yes | Multi-Hyphenated |
508
- | 40 | is_single_underscored | Gel_Git | Yes | Single_Underscored |
509
- | 41 | is_multi_underscored | Yarı_Yapılandırılmış_Mülakat | Yes | Multi_Underscored |
510
- | 42 | is_one_char_fixable | bilgisa¬yar | Yes | One_Char_Fixed |
511
- | 43 | is_three_or_more | heyyyyy | No | ----- |
512
- | 44 | is_fsp | bilgisayar. | No | ----- |
513
- | 45 | is_isp | .bilgisayar | No | ----- |
514
- | 46 | is_fmp | bilgisayar.. | No | ----- |
515
- | 47 | is_imp | ..bilgisayar | No | ----- |
516
- | 48 | is_msp | --bilgisayar-- | No | ----- |
517
- | 49 | is_mssp | -bilgisayar- | No | ----- |
518
- | 50 | is_midsp | okul,öğrenci | No | ----- |
519
- | 51 | is_midmp | okul,öğrenci, öğretmen | No | ----- |
520
- | 52 | is_non_latin | 한국드 | No | Non_Latin |
465
+ | # | Function | Sample | Used As Output | Tag |
466
+ |----|-----------------------------|------------------------------|----------------|--------------------|
467
+ | 01 | is_mention | @ts-tokenizer | Yes | Mention |
468
+ | 02 | is_hashtag | #ts-tokenizer | Yes | Hashtag |
469
+ | 03 | is_mention_suffix | @SerkanÇayoğlu'ndan | Yes | Mention_Suffix |
470
+ | 04 | is_hashtag_suffix | #Şanlıurfa'da | Yes | Hashtag_Suffix |
471
+ | 05 | is_in_quotes | "ts-tokenizer" | No | ----- |
472
+ | 06 | is_numbered_title | (1) | Yes | Numbered_Title |
473
+ | 07 | is_in_paranthesis | (bilgisayar) | No | ----- |
474
+ | 08 | is_date_range | 01.01.2024-01.01.2025 | Yes | Date_Range |
475
+ | 09 | is_complex_punc | -yeniden,sonradan.. | No | ----- |
476
+ | 10 | is_date | 22.02.2016 | Yes | Date |
477
+ | 11 | is_hour | 14.05 | Yes | Hour |
478
+ | 12 | is_percentage_numbers | %75 | Yes | Percentage_Numbers |
479
+ | 13 | is_percentage_numbers_chars | %75'lik | Yes | Percentage_Numbers |
480
+ | 14 | is_roman_number | XI | Yes | Roman_Number |
481
+ | 15 | is_bullet_list | •Giriş | Yes | Bullet_List |
482
+ | 16 | is_email | tanersezerr@gmail.com | Yes | Email |
483
+ | 17 | is_email_punc | tanersezerr@gmail.com. | No | ----- |
484
+ | 18 | is_full_url | https://tscorpus.com | Yes | Full_URL |
485
+ | 19 | is_web_url | www.tscorpus.com | Yes | Web_URL |
486
+ | 20 | is_full_url | www.example.com'un | Yes | URL_Suffix |
487
+ | 21 | is_copyright | ©tscorpus | Yes | Copyright |
488
+ | 22 | is_registered | tscorpus® | Yes | Registered |
489
+ | 23 | is_trademark | tscorpus™ | Yes | Trademark |
490
+ | 24 | is_currency | 100$ | Yes | Currency |
491
+ | 25 | is_num_char_sequence | 380A | No | ----- |
492
+ | 26 | is_abbr | TBMM | Yes | Abbr |
493
+ | 27 | is_in_lexicon | bilgisayar | Yes | Valid_Word |
494
+ | 28 | is_in_exceptions | e-mail | Yes | Exception |
495
+ | 29 | is_in_eng_words | computer | Yes | English_Word |
496
+ | 30 | is_smiley | :) | Yes | Smiley |
497
+ | 31 | is_multiple_smiley | :):) | No | ----- |
498
+ | 32 | is_emoticon | 🍻 | Yes | Emoticon |
499
+ | 33 | is_multiple_emoticon | 🍻🍻 | No | ----- |
500
+ | 34 | is_multiple_smiley_in | hey:):) | No | ----- |
501
+ | 35 | is_number | 175.01 | Yes | Number |
502
+ | 36 | is_apostrophed | Türkiye'nin | Yes | Apostrophed |
503
+ | 37 | is_single_punc | ! | Yes | Punc |
504
+ | 38 | is_multi_punc | !! | No | ----- |
505
+ | 39 | is_single_hyphenated | sabah-akşam | Yes | Single_Hyphenated |
506
+ | 40 | is_multi_hyphenated | çay-su-kahve | Yes | Multi-Hyphenated |
507
+ | 41 | is_single_underscored | Gel_Git | Yes | Single_Underscored |
508
+ | 42 | is_multi_underscored | Yarı_Yapılandırılmış_Mülakat | Yes | Multi_Underscored |
509
+ | 43 | is_one_char_fixable | bilgisa¬yar | Yes | One_Char_Fixed |
510
+ | 44 | is_formula | F(2,37)=6.42 | Yes | Formula |
511
+ | 45 | is_three_or_more | heyyyyy | No | ----- |
512
+ | 46 | is_fsp | bilgisayar. | No | ----- |
513
+ | 47 | is_isp | .bilgisayar | No | ----- |
514
+ | 48 | is_fmp | bilgisayar.. | No | ----- |
515
+ | 49 | is_imp | ..bilgisayar | No | ----- |
516
+ | 50 | is_msp | --bilgisayar-- | No | ----- |
517
+ | 51 | is_mssp | -bilgisayar- | No | ----- |
518
+ | 52 | is_midsp | okul,öğrenci | No | ----- |
519
+ | 53 | is_midmp | okul,öğrenci, öğretmen | No | ----- |
520
+ | 54 | is_non_latin | 한국드 | No | Non_Latin |
521
521
 
522
522
  ----------------------
523
523
 
{ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/LICENSE RENAMED
File without changes
{ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/MANIFEST.in RENAMED
File without changes
{ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/setup.cfg RENAMED
File without changes
{ts_tokenizer-0.2.4 → ts_tokenizer-0.2.6}/setup.py RENAMED
File without changes