token-utils 0.2.1__tar.gz → 0.2.3__tar.gz

{token_utils-0.2.1 → token_utils-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: token-utils
-Version: 0.2.1
+Version: 0.2.3
 Summary: Convenient tools to deal with tokens.
 Project-URL: Documentation, https://aroberge.github.io/token-utils/docs/html/
 Project-URL: Issues, https://github.com/aroberge/token-utils/issues

token_utils-0.2.3/src/token_utils/__about__.py

@@ -0,0 +1,2 @@
+version = "0.2.3"
+name = "token_utils"

{token_utils-0.2.1 → token_utils-0.2.3}/src/token_utils/py_tokenize.py

@@ -565,6 +565,7 @@ def _tokenize(readline, encoding):
                         )
                 pos += 1
 
+    # TODO: see if we can remove this!!!
     # Add an implicit NEWLINE if the input doesn't end in one
     if not unterminated_triple:
         if (
@@ -574,7 +575,7 @@ def _tokenize(readline, encoding):
         ):
             yield TokenInfo(
                 NEWLINE,
-                last_line,
+                "",
                 (lnum - 1, len(last_line)),
                 (lnum - 1, len(last_line) + 1),
                 last_line,

{token_utils-0.2.1 → token_utils-0.2.3}/src/token_utils/utils.py

@@ -23,6 +23,7 @@ def fix_empty_line(source, prev_token, last_token):
 
     we correct the last token content if needed.
     """
+    print(f"{last_token=}")
     if prev_token is None:  # should not happen
         return last_token
     if prev_token.line.endswith((" ", "\t")):  # fix not needed
@@ -36,6 +37,7 @@ def fix_empty_line(source, prev_token, last_token):
             break
     last_token.string = source[-nb:]
     assert nb > 0
+    print(f"Fix applied {last_token=}")
     return last_token
 
 

{token_utils-0.2.1 → token_utils-0.2.3}/tests/test_token_utils.py

@@ -91,6 +91,23 @@ def test_bad_dedent():
     """
     check(source)
 
+def test_problematic_newline():
+    """When modifying Python's tokenize, I accidently added a
+    string attribute to "NEWLINE" token. This test is to
+    ensure that this does not happen again!
+    """
+    source = "2n"
+    assert token_utils.untokenize(token_utils.tokenize(source)) == source
+    #
+    # test inserting non-token
+    tokens = token_utils.tokenize(source)
+    new_tokens = []
+    for tok in tokens:
+        new_tokens.append(tok)
+        if tok.is_number():
+            new_tokens.append("*")
+    assert token_utils.untokenize(new_tokens) == "2*n"
+
 
 source1 = "a = b"
 source2 = "a = b # comment\n"

token_utils-0.2.1/src/token_utils/__about__.py

@@ -1,2 +0,0 @@
-version = "0.2.1"
-name = "token_utils"