sonatoki 0.6.0__tar.gz → 0.6.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. {sonatoki-0.6.0 → sonatoki-0.6.2}/PKG-INFO +1 -1
  2. {sonatoki-0.6.0 → sonatoki-0.6.2}/pyproject.toml +1 -1
  3. {sonatoki-0.6.0 → sonatoki-0.6.2}/src/sonatoki/constants.py +2 -2
  4. {sonatoki-0.6.0 → sonatoki-0.6.2}/tests/test_ilo.py +2 -1
  5. {sonatoki-0.6.0 → sonatoki-0.6.2}/LICENSE +0 -0
  6. {sonatoki-0.6.0 → sonatoki-0.6.2}/README.md +0 -0
  7. {sonatoki-0.6.0 → sonatoki-0.6.2}/src/sonatoki/Cleaners.py +0 -0
  8. {sonatoki-0.6.0 → sonatoki-0.6.2}/src/sonatoki/Configs.py +0 -0
  9. {sonatoki-0.6.0 → sonatoki-0.6.2}/src/sonatoki/Filters.py +0 -0
  10. {sonatoki-0.6.0 → sonatoki-0.6.2}/src/sonatoki/Preprocessors.py +0 -0
  11. {sonatoki-0.6.0 → sonatoki-0.6.2}/src/sonatoki/Scorers.py +0 -0
  12. {sonatoki-0.6.0 → sonatoki-0.6.2}/src/sonatoki/Tokenizers.py +0 -0
  13. {sonatoki-0.6.0 → sonatoki-0.6.2}/src/sonatoki/__init__.py +0 -0
  14. {sonatoki-0.6.0 → sonatoki-0.6.2}/src/sonatoki/__main__.py +0 -0
  15. {sonatoki-0.6.0 → sonatoki-0.6.2}/src/sonatoki/alphabetic.txt +0 -0
  16. {sonatoki-0.6.0 → sonatoki-0.6.2}/src/sonatoki/ilo.py +0 -0
  17. {sonatoki-0.6.0 → sonatoki-0.6.2}/src/sonatoki/linku.json +0 -0
  18. {sonatoki-0.6.0 → sonatoki-0.6.2}/src/sonatoki/py.typed +0 -0
  19. {sonatoki-0.6.0 → sonatoki-0.6.2}/src/sonatoki/sandbox.json +0 -0
  20. {sonatoki-0.6.0 → sonatoki-0.6.2}/src/sonatoki/syllabic.txt +0 -0
  21. {sonatoki-0.6.0 → sonatoki-0.6.2}/src/sonatoki/utils.py +0 -0
  22. {sonatoki-0.6.0 → sonatoki-0.6.2}/tests/__init__.py +0 -0
  23. {sonatoki-0.6.0 → sonatoki-0.6.2}/tests/test_cleaners.py +0 -0
  24. {sonatoki-0.6.0 → sonatoki-0.6.2}/tests/test_filters.py +0 -0
  25. {sonatoki-0.6.0 → sonatoki-0.6.2}/tests/test_preprocessors.py +0 -0
  26. {sonatoki-0.6.0 → sonatoki-0.6.2}/tests/test_properties.py +0 -0
  27. {sonatoki-0.6.0 → sonatoki-0.6.2}/tests/test_scorers.py +0 -0
  28. {sonatoki-0.6.0 → sonatoki-0.6.2}/tests/test_tokenize.py +0 -0
  29. {sonatoki-0.6.0 → sonatoki-0.6.2}/tests/test_utils.py +0 -0
  30. {sonatoki-0.6.0 → sonatoki-0.6.2}/tests/tokenize_cases/tokenize_sentences_tok.yml +0 -0
  31. {sonatoki-0.6.0 → sonatoki-0.6.2}/tests/tokenize_cases/tokenize_words_tok.yml +0 -0
{sonatoki-0.6.0 → sonatoki-0.6.2}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: sonatoki
3
- Version: 0.6.0
3
+ Version: 0.6.2
4
4
  Summary: ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?
5
5
  Author-Email: "jan Kekan San (@gregdan3)" <gregory.danielson3@gmail.com>
6
6
  License: AGPL-3.0-or-later
{sonatoki-0.6.0 → sonatoki-0.6.2}/pyproject.toml RENAMED
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "sonatoki"
3
- version = "0.6.0"
3
+ version = "0.6.2"
4
4
  description = "ilo li moku e toki li pana e sona ni: ni li toki ala toki pona?"
5
5
  authors = [
6
6
  { name = "jan Kekan San (@gregdan3)", email = "gregory.danielson3@gmail.com" },
{sonatoki-0.6.0 → sonatoki-0.6.2}/src/sonatoki/constants.py RENAMED
@@ -498,10 +498,10 @@ ALL_PUNCT = "".join(sorted(list(set(POSIX_PUNCT + UNICODE_PUNCT))))
498
498
  ALL_PUNCT_RANGES_STR = "".join(find_unicode_ranges(ALL_PUNCT))
499
499
  # combined bc the result could be simpler
500
500
 
501
- SENTENCE_PUNCT = """.?!:;()[-]·…"""
501
+ SENTENCE_PUNCT = """.?!:;()[-]·•…"""
502
502
  # NOTE: quotes were previously included, but in TP they are *not* reliably sentence boundaries
503
503
 
504
- INTRA_WORD_PUNCT = """-'"""
504
+ INTRA_WORD_PUNCT = """-'’"""
505
505
 
506
506
 
507
507
  LINKU = Path(__file__).resolve().parent / Path("linku.json")
{sonatoki-0.6.0 → sonatoki-0.6.2}/tests/test_ilo.py RENAMED
@@ -192,10 +192,11 @@ FALSE_NEGATIVES = [
192
192
  "poan",
193
193
  "mtue",
194
194
  "mi nasa B^)", # emoticon
195
+ "lete li ike x.x", # this is an emoticon but passes because 'x' is in Filters.Miscellaneous
195
196
  ]
196
197
 
197
198
  FALSE_POSITIVES = [
198
- "lete li ike x.x", # this is an emoticon but passes because 'x' is in Filters.Miscellaneous
199
+ "Knowing a little toki pona",
199
200
  ]
200
201
 
201
202
 
{sonatoki-0.6.0 → sonatoki-0.6.2}/LICENSE RENAMED
File without changes
{sonatoki-0.6.0 → sonatoki-0.6.2}/README.md RENAMED
File without changes
{sonatoki-0.6.0 → sonatoki-0.6.2}/src/sonatoki/ilo.py RENAMED
File without changes
{sonatoki-0.6.0 → sonatoki-0.6.2}/src/sonatoki/py.typed RENAMED
File without changes
{sonatoki-0.6.0 → sonatoki-0.6.2}/src/sonatoki/utils.py RENAMED
File without changes
{sonatoki-0.6.0 → sonatoki-0.6.2}/tests/__init__.py RENAMED
File without changes
{sonatoki-0.6.0 → sonatoki-0.6.2}/tests/test_filters.py RENAMED
File without changes
{sonatoki-0.6.0 → sonatoki-0.6.2}/tests/test_scorers.py RENAMED
File without changes
{sonatoki-0.6.0 → sonatoki-0.6.2}/tests/test_utils.py RENAMED
File without changes