token-utils 0.2.2__tar.gz → 0.2.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. {token_utils-0.2.2 → token_utils-0.2.3}/PKG-INFO +1 -1
  2. token_utils-0.2.3/src/token_utils/__about__.py +2 -0
  3. {token_utils-0.2.2 → token_utils-0.2.3}/src/token_utils/py_tokenize.py +14 -0
  4. token_utils-0.2.2/src/token_utils/__about__.py +0 -2
  5. {token_utils-0.2.2 → token_utils-0.2.3}/.flake8 +0 -0
  6. {token_utils-0.2.2 → token_utils-0.2.3}/.gitignore +0 -0
  7. {token_utils-0.2.2 → token_utils-0.2.3}/.idea/.gitignore +0 -0
  8. {token_utils-0.2.2 → token_utils-0.2.3}/.idea/inspectionProfiles/profiles_settings.xml +0 -0
  9. {token_utils-0.2.2 → token_utils-0.2.3}/.idea/modules.xml +0 -0
  10. {token_utils-0.2.2 → token_utils-0.2.3}/.idea/token-utils.iml +0 -0
  11. {token_utils-0.2.2 → token_utils-0.2.3}/.idea/vcs.xml +0 -0
  12. {token_utils-0.2.2 → token_utils-0.2.3}/.idea/workspace.xml +0 -0
  13. {token_utils-0.2.2 → token_utils-0.2.3}/.nojekyll +0 -0
  14. {token_utils-0.2.2 → token_utils-0.2.3}/LICENSE +0 -0
  15. {token_utils-0.2.2 → token_utils-0.2.3}/README.md +0 -0
  16. {token_utils-0.2.2 → token_utils-0.2.3}/ae.bat +0 -0
  17. {token_utils-0.2.2 → token_utils-0.2.3}/docs/Makefile +0 -0
  18. {token_utils-0.2.2 → token_utils-0.2.3}/docs/doctrees/about_tokens.doctree +0 -0
  19. {token_utils-0.2.2 → token_utils-0.2.3}/docs/doctrees/api.doctree +0 -0
  20. {token_utils-0.2.2 → token_utils-0.2.3}/docs/doctrees/environment.pickle +0 -0
  21. {token_utils-0.2.2 → token_utils-0.2.3}/docs/doctrees/history.doctree +0 -0
  22. {token_utils-0.2.2 → token_utils-0.2.3}/docs/doctrees/index.doctree +0 -0
  23. {token_utils-0.2.2 → token_utils-0.2.3}/docs/doctrees/nbsphinx/tokenize_notebook.ipynb +0 -0
  24. {token_utils-0.2.2 → token_utils-0.2.3}/docs/doctrees/tokenize.doctree +0 -0
  25. {token_utils-0.2.2 → token_utils-0.2.3}/docs/doctrees/tokenize_notebook.doctree +0 -0
  26. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/.buildinfo +0 -0
  27. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/.buildinfo.bak +0 -0
  28. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/.nojekyll +0 -0
  29. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_modules/ideas/token_utils.html +0 -0
  30. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_modules/index.html +0 -0
  31. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_modules/token_utils/_token_utils.html +0 -0
  32. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_modules/token_utils/token_class.html +0 -0
  33. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_modules/token_utils/utils.html +0 -0
  34. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_sources/about_tokens.rst.txt +0 -0
  35. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_sources/api.rst.txt +0 -0
  36. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_sources/history.rst.txt +0 -0
  37. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_sources/index.rst.txt +0 -0
  38. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_sources/tokenize.rst.txt +0 -0
  39. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_sources/tokenize_notebook.ipynb.txt +0 -0
  40. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/_sphinx_javascript_frameworks_compat.js +0 -0
  41. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/basic.css +0 -0
  42. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/css/badge_only.css +0 -0
  43. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/css/fonts/Roboto-Slab-Bold.woff +0 -0
  44. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/css/fonts/Roboto-Slab-Bold.woff2 +0 -0
  45. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/css/fonts/Roboto-Slab-Regular.woff +0 -0
  46. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/css/fonts/Roboto-Slab-Regular.woff2 +0 -0
  47. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/css/fonts/fontawesome-webfont.eot +0 -0
  48. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/css/fonts/fontawesome-webfont.svg +0 -0
  49. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/css/fonts/fontawesome-webfont.ttf +0 -0
  50. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/css/fonts/fontawesome-webfont.woff +0 -0
  51. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/css/fonts/fontawesome-webfont.woff2 +0 -0
  52. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/css/fonts/lato-bold-italic.woff +0 -0
  53. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/css/fonts/lato-bold-italic.woff2 +0 -0
  54. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/css/fonts/lato-bold.woff +0 -0
  55. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/css/fonts/lato-bold.woff2 +0 -0
  56. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/css/fonts/lato-normal-italic.woff +0 -0
  57. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/css/fonts/lato-normal-italic.woff2 +0 -0
  58. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/css/fonts/lato-normal.woff +0 -0
  59. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/css/fonts/lato-normal.woff2 +0 -0
  60. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/css/theme.css +0 -0
  61. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/custom.css +0 -0
  62. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/doctools.js +0 -0
  63. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/documentation_options.js +0 -0
  64. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/file.png +0 -0
  65. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/Inconsolata-Bold.ttf +0 -0
  66. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/Inconsolata-Regular.ttf +0 -0
  67. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/Inconsolata.ttf +0 -0
  68. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/Lato/lato-bold.eot +0 -0
  69. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/Lato/lato-bold.ttf +0 -0
  70. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/Lato/lato-bold.woff +0 -0
  71. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/Lato/lato-bold.woff2 +0 -0
  72. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/Lato/lato-bolditalic.eot +0 -0
  73. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/Lato/lato-bolditalic.ttf +0 -0
  74. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/Lato/lato-bolditalic.woff +0 -0
  75. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/Lato/lato-bolditalic.woff2 +0 -0
  76. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/Lato/lato-italic.eot +0 -0
  77. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/Lato/lato-italic.ttf +0 -0
  78. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/Lato/lato-italic.woff +0 -0
  79. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/Lato/lato-italic.woff2 +0 -0
  80. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/Lato/lato-regular.eot +0 -0
  81. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/Lato/lato-regular.ttf +0 -0
  82. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/Lato/lato-regular.woff +0 -0
  83. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/Lato/lato-regular.woff2 +0 -0
  84. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/Lato-Bold.ttf +0 -0
  85. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/Lato-Regular.ttf +0 -0
  86. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/RobotoSlab/roboto-slab-v7-bold.eot +0 -0
  87. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/RobotoSlab/roboto-slab-v7-bold.ttf +0 -0
  88. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff +0 -0
  89. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/RobotoSlab/roboto-slab-v7-bold.woff2 +0 -0
  90. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/RobotoSlab/roboto-slab-v7-regular.eot +0 -0
  91. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/RobotoSlab/roboto-slab-v7-regular.ttf +0 -0
  92. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/RobotoSlab/roboto-slab-v7-regular.woff +0 -0
  93. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/RobotoSlab/roboto-slab-v7-regular.woff2 +0 -0
  94. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/RobotoSlab-Bold.ttf +0 -0
  95. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/RobotoSlab-Regular.ttf +0 -0
  96. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/fontawesome-webfont.eot +0 -0
  97. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/fontawesome-webfont.svg +0 -0
  98. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/fontawesome-webfont.ttf +0 -0
  99. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/fontawesome-webfont.woff +0 -0
  100. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/fonts/fontawesome-webfont.woff2 +0 -0
  101. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/jquery-3.2.1.js +0 -0
  102. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/jquery.js +0 -0
  103. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/js/badge_only.js +0 -0
  104. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/js/modernizr.min.js +0 -0
  105. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/js/theme.js +0 -0
  106. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/js/versions.js +0 -0
  107. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/language_data.js +0 -0
  108. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/minus.png +0 -0
  109. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/nbsphinx-broken-thumbnail.svg +0 -0
  110. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/nbsphinx-code-cells.css +0 -0
  111. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/nbsphinx-gallery.css +0 -0
  112. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/nbsphinx-no-thumbnail.svg +0 -0
  113. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/plus.png +0 -0
  114. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/pygments.css +0 -0
  115. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/searchtools.js +0 -0
  116. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/sphinx_highlight.js +0 -0
  117. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/underscore-1.3.1.js +0 -0
  118. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/_static/underscore.js +0 -0
  119. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/about_tokens.html +0 -0
  120. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/api.html +0 -0
  121. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/genindex.html +0 -0
  122. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/history.html +0 -0
  123. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/index.html +0 -0
  124. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/objects.inv +0 -0
  125. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/py-modindex.html +0 -0
  126. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/search.html +0 -0
  127. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/searchindex.js +0 -0
  128. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/tokenize.html +0 -0
  129. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/tokenize_notebook.html +0 -0
  130. {token_utils-0.2.2 → token_utils-0.2.3}/docs/html/tokenize_notebook.ipynb +0 -0
  131. {token_utils-0.2.2 → token_utils-0.2.3}/docs/make.bat +0 -0
  132. {token_utils-0.2.2 → token_utils-0.2.3}/docs/source/_static/custom.css +0 -0
  133. {token_utils-0.2.2 → token_utils-0.2.3}/docs/source/about_tokens.rst +0 -0
  134. {token_utils-0.2.2 → token_utils-0.2.3}/docs/source/api.rst +0 -0
  135. {token_utils-0.2.2 → token_utils-0.2.3}/docs/source/conf.py +0 -0
  136. {token_utils-0.2.2 → token_utils-0.2.3}/docs/source/history.rst +0 -0
  137. {token_utils-0.2.2 → token_utils-0.2.3}/docs/source/index.rst +0 -0
  138. {token_utils-0.2.2 → token_utils-0.2.3}/docs/source/tokenize_notebook.ipynb +0 -0
  139. {token_utils-0.2.2 → token_utils-0.2.3}/pypi_upload.bat +0 -0
  140. {token_utils-0.2.2 → token_utils-0.2.3}/pyproject.toml +0 -0
  141. {token_utils-0.2.2 → token_utils-0.2.3}/src/token_utils/__init__.py +0 -0
  142. {token_utils-0.2.2 → token_utils-0.2.3}/src/token_utils/py_token.py +0 -0
  143. {token_utils-0.2.2 → token_utils-0.2.3}/src/token_utils/token_class.py +0 -0
  144. {token_utils-0.2.2 → token_utils-0.2.3}/src/token_utils/utils.py +0 -0
  145. {token_utils-0.2.2 → token_utils-0.2.3}/tests/__init__.py +0 -0
  146. {token_utils-0.2.2 → token_utils-0.2.3}/tests/test_token_utils.py +0 -0
{token_utils-0.2.2 → token_utils-0.2.3}/PKG-INFO RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: token-utils
3
- Version: 0.2.2
3
+ Version: 0.2.3
4
4
  Summary: Convenient tools to deal with tokens.
5
5
  Project-URL: Documentation, https://aroberge.github.io/token-utils/docs/html/
6
6
  Project-URL: Issues, https://github.com/aroberge/token-utils/issues
token_utils-0.2.3/src/token_utils/__about__.py ADDED
@@ -0,0 +1,2 @@
1
+ version = "0.2.3"
2
+ name = "token_utils"
{token_utils-0.2.2 → token_utils-0.2.3}/src/token_utils/py_tokenize.py RENAMED
@@ -565,7 +565,21 @@ def _tokenize(readline, encoding):
565
565
  )
566
566
  pos += 1
567
567
 
568
+ # TODO: see if we can remove this!!!
569
+ # Add an implicit NEWLINE if the input doesn't end in one
568
570
  if not unterminated_triple:
571
+ if (
572
+ last_line
573
+ and last_line[-1] not in "\r\n"
574
+ and not last_line.strip().startswith("#")
575
+ ):
576
+ yield TokenInfo(
577
+ NEWLINE,
578
+ "",
579
+ (lnum - 1, len(last_line)),
580
+ (lnum - 1, len(last_line) + 1),
581
+ last_line,
582
+ )
569
583
  yield TokenInfo(ENDMARKER, "", (lnum, 0), (lnum, 0), "")
570
584
 
571
585
 
token_utils-0.2.2/src/token_utils/__about__.py DELETED
@@ -1,2 +0,0 @@
1
- version = "0.2.2"
2
- name = "token_utils"
{token_utils-0.2.2 → token_utils-0.2.3}/.flake8 RENAMED
File without changes
{token_utils-0.2.2 → token_utils-0.2.3}/.gitignore RENAMED
File without changes
{token_utils-0.2.2 → token_utils-0.2.3}/.idea/vcs.xml RENAMED
File without changes
{token_utils-0.2.2 → token_utils-0.2.3}/.nojekyll RENAMED
File without changes
{token_utils-0.2.2 → token_utils-0.2.3}/LICENSE RENAMED
File without changes
{token_utils-0.2.2 → token_utils-0.2.3}/README.md RENAMED
File without changes
{token_utils-0.2.2 → token_utils-0.2.3}/ae.bat RENAMED
File without changes
{token_utils-0.2.2 → token_utils-0.2.3}/docs/Makefile RENAMED
File without changes
{token_utils-0.2.2 → token_utils-0.2.3}/docs/make.bat RENAMED
File without changes
{token_utils-0.2.2 → token_utils-0.2.3}/pypi_upload.bat RENAMED
File without changes
{token_utils-0.2.2 → token_utils-0.2.3}/pyproject.toml RENAMED
File without changes