token-utils 0.2.1__tar.gz → 0.2.2__tar.gz

{token_utils-0.2.1 → token_utils-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: token-utils
-Version: 0.2.1
+Version: 0.2.2
 Summary: Convenient tools to deal with tokens.
 Project-URL: Documentation, https://aroberge.github.io/token-utils/docs/html/
 Project-URL: Issues, https://github.com/aroberge/token-utils/issues

token_utils-0.2.2/src/token_utils/__about__.py

@@ -0,0 +1,2 @@
+version = "0.2.2"
+name = "token_utils"

{token_utils-0.2.1 → token_utils-0.2.2}/src/token_utils/py_tokenize.py

@@ -565,20 +565,7 @@ def _tokenize(readline, encoding):
                         )
                 pos += 1
 
-    # Add an implicit NEWLINE if the input doesn't end in one
     if not unterminated_triple:
-        if (
-            last_line
-            and last_line[-1] not in "\r\n"
-            and not last_line.strip().startswith("#")
-        ):
-            yield TokenInfo(
-                NEWLINE,
-                last_line,
-                (lnum - 1, len(last_line)),
-                (lnum - 1, len(last_line) + 1),
-                last_line,
-            )
         yield TokenInfo(ENDMARKER, "", (lnum, 0), (lnum, 0), "")
 
 

{token_utils-0.2.1 → token_utils-0.2.2}/src/token_utils/utils.py

@@ -23,6 +23,7 @@ def fix_empty_line(source, prev_token, last_token):
 
     we correct the last token content if needed.
     """
+    print(f"{last_token=}")
     if prev_token is None:  # should not happen
         return last_token
     if prev_token.line.endswith((" ", "\t")):  # fix not needed
@@ -36,6 +37,7 @@ def fix_empty_line(source, prev_token, last_token):
             break
     last_token.string = source[-nb:]
     assert nb > 0
+    print(f"Fix applied {last_token=}")
     return last_token
 
 

{token_utils-0.2.1 → token_utils-0.2.2}/tests/test_token_utils.py

@@ -91,6 +91,23 @@ def test_bad_dedent():
     """
     check(source)
 
+def test_problematic_newline():
+    """When modifying Python's tokenize, I accidently added a
+    string attribute to "NEWLINE" token. This test is to
+    ensure that this does not happen again!
+    """
+    source = "2n"
+    assert token_utils.untokenize(token_utils.tokenize(source)) == source
+    #
+    # test inserting non-token
+    tokens = token_utils.tokenize(source)
+    new_tokens = []
+    for tok in tokens:
+        new_tokens.append(tok)
+        if tok.is_number():
+            new_tokens.append("*")
+    assert token_utils.untokenize(new_tokens) == "2*n"
+
 
 source1 = "a = b"
 source2 = "a = b # comment\n"

token_utils-0.2.1/src/token_utils/__about__.py

@@ -1,2 +0,0 @@
-version = "0.2.1"
-name = "token_utils"