token-utils 0.1.7__tar.gz → 0.2.0__tar.gz

token_utils-0.2.0/.flake8

@@ -0,0 +1,14 @@
+[flake8]
+max-line-length = 88
+# see https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes
+ignore = E203, E266, E402, E501, W503
+exclude =
+    # No need to traverse our git directory
+    .git/*,
+    # There's no value in checking cache directories
+    __pycache__,
+    # documentation, to be ignored
+    docs/*,
+    images/*,
+    # This contains our tests directory
+    tests/*,

token_utils-0.2.0/.gitignore

@@ -0,0 +1,133 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+venv*/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+
+temp*.*

token_utils-0.2.0/.idea/.gitignore

@@ -0,0 +1,5 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/

token_utils-0.2.0/.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

token_utils-0.2.0/.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/token-utils.iml" filepath="$PROJECT_DIR$/.idea/token-utils.iml" />
+    </modules>
+  </component>
+</project>

token_utils-0.2.0/.idea/token-utils.iml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module external.system.id="pyproject.toml" type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/src" isTestSource="false" />
+      <excludeFolder url="file://$MODULE_DIR$/venv-token3.10" />
+    </content>
+    <orderEntry type="jdk" jdkName="~\github\token-utils\venv-token3.10" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

token_utils-0.2.0/.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

token_utils-0.2.0/.idea/workspace.xml

@@ -0,0 +1,67 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="AutoImportSettings">
+    <option name="autoReloadType" value="SELECTIVE" />
+  </component>
+  <component name="ChangeListManager">
+    <list default="true" id="3545b7d4-d817-410c-bc69-d7ce96a4dbcc" name="Changes" comment="">
+      <change beforePath="$PROJECT_DIR$/src/token_utils/py_tokenize.py" beforeDir="false" afterPath="$PROJECT_DIR$/src/token_utils/py_tokenize.py" afterDir="false" />
+    </list>
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="FileTemplateManagerImpl">
+    <option name="RECENT_TEMPLATES">
+      <list>
+        <option value="Python Script" />
+      </list>
+    </option>
+  </component>
+  <component name="Git.Settings">
+    <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
+  </component>
+  <component name="ProjectColorInfo">{
+  &quot;associatedIndex&quot;: 4,
+  &quot;fromUser&quot;: false
+}</component>
+  <component name="ProjectId" id="3JjJsQEtY2V4TIefkRlMyE5hIqZ" />
+  <component name="ProjectViewState">
+    <option name="hideEmptyMiddlePackages" value="true" />
+    <option name="showLibraryContents" value="true" />
+  </component>
+  <component name="PropertiesComponent"><![CDATA[{
+  "keyToString": {
+    "ModuleVcsDetector.initialDetectionPerformed": "true",
+    "RunOnceActivity.ShowReadmeOnStart": "true",
+    "RunOnceActivity.git.unshallow": "true",
+    "RunOnceActivity.typescript.service.memoryLimit.init": "true",
+    "codeWithMe.voiceChat.enabledByDefault": "false",
+    "git-widget-placeholder": "master",
+    "last_opened_file_path": "C:/Users/Andre/github/token-utils",
+    "nodejs_package_manager_path": "npm",
+    "python.packages.toolwindow.project.migrated.to.right.top.v2": "true"
+  }
+}]]></component>
+  <component name="SharedIndexes">
+    <attachedChunks>
+      <set>
+        <option value="bundled-python-sdk-73a66231627c-5bd6d13a2a75-com.jetbrains.pycharm.pro.sharedIndexes.bundled-PY-262.10968.92" />
+      </set>
+    </attachedChunks>
+  </component>
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="3545b7d4-d817-410c-bc69-d7ce96a4dbcc" name="Changes" comment="" />
+      <created>1790168749536</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1790168749536</updated>
+    </task>
+    <servers />
+  </component>
+  <component name="TypeScriptGeneratedFilesManager">
+    <option name="version" value="3" />
+  </component>
+</project>

token_utils-0.2.0/LICENSE

@@ -0,0 +1,25 @@
+MIT License
+
+Copyright (c) 2020 André Roberge
+
+Note: the untokenize function was modified and adapted from 
+https://github.com/myint/untokenize,
+Copyright (C) 2013-2018 Steven Myint, MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

token_utils-0.2.0/PKG-INFO

@@ -0,0 +1,79 @@
+Metadata-Version: 2.4
+Name: token-utils
+Version: 0.2.0
+Summary: Convenient tools to deal with tokens.
+Project-URL: Documentation, https://aroberge.github.io/token-utils/docs/html/
+Project-URL: Issues, https://github.com/aroberge/token-utils/issues
+Project-URL: Source, https://github.com/aroberge/token-utils
+Author-email: André Roberge <andre.roberge@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Requires-Python: >=3.6
+Description-Content-Type: text/markdown
+
+# token-utils
+
+Installation: `pip install token-utils`.
+
+The purpose of token-utils is to simplify manipulations of tokens normally
+obtaind from Python's [tokenize module](https://docs.python.org/3/library/tokenize.html).
+One of token-utils' features is that, unlike Python's version, the following
+is always guaranteed:
+
+```python
+from token_utils import tokenize, untokenize
+
+source = "Arbitrary Python code here"
+
+assert source == untokenize(tokenize(source))
+```
+
+To get an idea of the simplicity of using token-utils,
+consider [this example from Python's standard library](https://docs.python.org/3/library/tokenize.html#examples) which substitute Decimals for floats in a string of statements:
+
+```python
+from tokenize import tokenize, untokenize, NUMBER, STRING, NAME, OP
+from io import BytesIO
+
+def decistmt(s):
+    result = []
+    g = tokenize(BytesIO(s.encode('utf-8')).readline)  # tokenize the string
+    for toknum, tokval, _, _, _ in g:
+        if toknum == NUMBER and '.' in tokval:  # replace NUMBER tokens
+            result.extend([
+                (NAME, 'Decimal'),
+                (OP, '('),
+                (STRING, repr(tokval)),
+                (OP, ')')
+            ])
+        else:
+            result.append((toknum, tokval))
+    return untokenize(result).decode('utf-8')
+```
+
+Here's how you could achieve the same result with token-utils:
+
+```python
+from token_utils import tokenize, untokenize
+
+def decistmt(source):
+    tokens = tokenize(source)
+    for token in tokens:
+        if token.is_float():
+            token.string = f"Decimal('{token.string}')"
+    return untokenize(tokens)
+```
+
+See [the documentation](https://aroberge.github.io/token-utils/docs/html/) for more information.

token_utils-0.2.0/README.md

@@ -0,0 +1,54 @@
+# token-utils
+
+Installation: `pip install token-utils`.
+
+The purpose of token-utils is to simplify manipulations of tokens normally
+obtaind from Python's [tokenize module](https://docs.python.org/3/library/tokenize.html).
+One of token-utils' features is that, unlike Python's version, the following
+is always guaranteed:
+
+```python
+from token_utils import tokenize, untokenize
+
+source = "Arbitrary Python code here"
+
+assert source == untokenize(tokenize(source))
+```
+
+To get an idea of the simplicity of using token-utils,
+consider [this example from Python's standard library](https://docs.python.org/3/library/tokenize.html#examples) which substitute Decimals for floats in a string of statements:
+
+```python
+from tokenize import tokenize, untokenize, NUMBER, STRING, NAME, OP
+from io import BytesIO
+
+def decistmt(s):
+    result = []
+    g = tokenize(BytesIO(s.encode('utf-8')).readline)  # tokenize the string
+    for toknum, tokval, _, _, _ in g:
+        if toknum == NUMBER and '.' in tokval:  # replace NUMBER tokens
+            result.extend([
+                (NAME, 'Decimal'),
+                (OP, '('),
+                (STRING, repr(tokval)),
+                (OP, ')')
+            ])
+        else:
+            result.append((toknum, tokval))
+    return untokenize(result).decode('utf-8')
+```
+
+Here's how you could achieve the same result with token-utils:
+
+```python
+from token_utils import tokenize, untokenize
+
+def decistmt(source):
+    tokens = tokenize(source)
+    for token in tokens:
+        if token.is_float():
+            token.string = f"Decimal('{token.string}')"
+    return untokenize(tokens)
+```
+
+See [the documentation](https://aroberge.github.io/token-utils/docs/html/) for more information.

token_utils-0.2.0/ae.bat

@@ -0,0 +1,50 @@
+echo off
+REM Default is Python 3.9
+
+if "%1"=="3.6" goto py_36
+if "%1"=="3.7" goto py_37
+if "%1"=="3.8" goto py_38
+if "%1"=="3.9" goto py_39
+if "%1"=="3.10" goto py_310
+if "%1"=="3.11" goto py_311
+if "%1"=="3.12" goto py_312
+if "%1"=="3.13" goto py_313
+if "%1"=="3.14" goto py_314
+
+:py_39
+venv-token3.9\scripts\activate
+goto end
+
+:py_36
+venv-token3.6\scripts\activate
+goto end
+
+:py_37
+venv-token3.7\scripts\activate
+goto end 
+
+:py_38
+venv-token3.8\scripts\activate
+goto end
+
+:py_310
+venv-token3.10\scripts\activate
+goto end
+
+:py_311
+venv-token3.11\scripts\activate
+goto end
+
+:py_312
+venv-token3.12\scripts\activate
+goto end
+
+:py_313
+venv-token3.13\scripts\activate
+goto end
+
+:py_314
+venv-token3.14\scripts\activate
+goto end
+
+:end

token_utils-0.2.0/docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = token-utils
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)