textmate-grammar-python 0.1.2__tar.gz → 0.3.0__tar.gz

textmate_grammar_python-0.3.0/PKG-INFO

@@ -0,0 +1,105 @@
+Metadata-Version: 2.1
+Name: textmate-grammar-python
+Version: 0.3.0
+Summary: A lexer and tokenizer for grammar files as defined by TextMate and used in VSCode, implemented in Python.
+Home-page: https://github.com/watermarkhu/textmate-grammar-python
+License: MIT
+Keywords: textmate,tokenization
+Author: Mark Shui Hu
+Author-email: watermarkhu@gmail.com
+Requires-Python: >=3.11,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
+Requires-Dist: onigurumacffi (>=1.3.0,<2.0.0)
+Project-URL: Documentation, https://textmate-grammar-python.readthedocs.io
+Project-URL: Repository, https://github.com/watermarkhu/textmate-grammar-python
+Description-Content-Type: text/markdown
+
+[![PyPI - Version](https://img.shields.io/pypi/v/textmate-grammar-python.svg)](https://pypi.python.org/pypi/textmate-grammar-python)
+[![PyPI - License](https://img.shields.io/pypi/l/textmate-grammar-python.svg)](https://github.com/watermarkhu/textmate-grammar-python/tree/main?tab=MIT-1-ov-file)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![Checked with mypy](https://img.shields.io/badge/mypy-checked-blue)](http://mypy-lang.org/)
+[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit)
+[![Python versions](https://img.shields.io/pypi/pyversions/textmate-grammar-python.svg)](https://pypi.python.org/pypi/textmate-grammar-python)
+[![CI/CD](https://github.com/watermarkhu/textmate-grammar-python/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/watermarkhu/textmate-grammar-python/blob/main/.github/workflows/ci.yml)
+[![readthedocs](https://readthedocs.org/projects/textmate-grammar-python/badge/?version=latest)](https://textmate-grammar-python.readthedocs.io)
+
+# textmate-grammar-python
+
+A lexer and tokenizer for grammar files as defined by TextMate and used in VSCode, implemented in Python. 
+
+Textmate grammars are made for [vscode-texmate](https://github.com/microsoft/vscode-textmate), allowing for syntax highlighting in VSCode after tokenization. This presents textmate-grammar-python with a large list of potentially supported languages. 
+
+```mermaid
+flowchart TD
+    A[grammar file] 
+    Z[code]
+    B("`vscode-textmate **js**`")
+    C("`textmate-grammar-**python**`")
+    D[tokens]
+
+    click C "https://github.com/microsoft/vscode-textmate"
+    
+    Z --> B
+    Z --> C
+    A -.-> B --> D
+    A -.-> C --> D
+```
+
+## Usage
+Install the module with:
+```bash
+pip install textmate-grammar-python
+```
+
+Before tokenization is possible, a `LanguageParser` needs to be initialized using a loaded grammar. 
+
+```python
+from textmate_grammar.language import LanguageParser
+from textmate_grammar.grammars import matlab
+parser = LanguageParser(matlab.GRAMMAR)
+```
+
+After this, one can either choose to call [`parser.parsing_string`](https://textmate-grammar-python.readthedocs.io/en/latest/apidocs/textmate_grammar/textmate_grammar.language.html#textmate_grammar.language.LanguageParser.parse_string) to parse a input string directly, or call [`parser.parse_file`](https://textmate-grammar-python.readthedocs.io/en/latest/apidocs/textmate_grammar/textmate_grammar.language.html#textmate_grammar.language.LanguageParser.parse_file) with the path to the appropiate source file as the first argument, such as in the example [`example.py`](https://github.com/watermarkhu/textmate-grammar-python/blob/main/example.py). 
+
+The parsed `element` object can be displayed directly by calling the [`print`](https://textmate-grammar-python.readthedocs.io/en/latest/apidocs/textmate_grammar/textmate_grammar.elements.html#textmate_grammar.elements.ContentElement.print) method. By default the element is printed as an element tree in a dictionary format. 
+
+```python
+>>> element = parser.parse_string("value = num2str(10);")
+>>> element.print()
+
+{'token': 'source.matlab',
+ 'children': [{'token': 'meta.assignment.variable.single.matlab', 
+               'children': [{'token': 'variable.other.readwrite.matlab', 'content': 'value'}]},
+              {'token': 'keyword.operator.assignment.matlab', 'content': '='},
+              {'token': 'meta.function-call.parens.matlab',
+               'begin': [{'token': 'entity.name.function.matlab', 'content': 'num2str'},
+                         {'token': 'punctuation.section.parens.begin.matlab', 'content': '('}],
+               'end': [{'token': 'punctuation.section.parens.end.matlab', 'content': ')'}],
+               'children': [{'token': 'constant.numeric.decimal.matlab', 'content': '10'}]},
+              {'token': 'punctuation.terminator.semicolon.matlab', 'content': ';'}]}
+
+```
+Alternatively, with the keyword argument `flatten` the element is displayed as a list per unique token. Here the first item in the list is the starting position (line, column) of the unique tokenized element. 
+
+```python
+>>> element.print(flatten=True)
+
+[[(0, 0), 'value', ['source.matlab', 'meta.assignment.variable.single.matlab', 'variable.other.readwrite.matlab']],
+ [(0, 5), ' ', ['source.matlab']],
+ [(0, 6), '=', ['source.matlab', 'keyword.operator.assignment.matlab']],
+ [(0, 7), ' ', ['source.matlab']],
+ [(0, 8), 'num2str', ['source.matlab', 'meta.function-call.parens.matlab', 'entity.name.function.matlab']],
+ [(0, 15), '(', ['source.matlab', 'meta.function-call.parens.matlab', 'punctuation.section.parens.begin.matlab']],
+ [(0, 16), '10', ['source.matlab', 'meta.function-call.parens.matlab', 'constant.numeric.decimal.matlab']],
+ [(0, 18), ')', ['source.matlab', 'meta.function-call.parens.matlab', 'punctuation.section.parens.end.matlab']],
+ [(0, 19), ';', ['source.matlab', 'punctuation.terminator.semicolon.matlab']]]
+```
+
+## Information
+
+- For further information, please checkout the [documentation](https://textmate-grammar-python.readthedocs.io/en/latest/). 
+- To setup an environment for development, see [CONTRIBUTING.md](https://github.com/watermarkhu/textmate-grammar-python/blob/main/CONTRIBUTING.md)

textmate_grammar_python-0.3.0/README.md

@@ -0,0 +1,85 @@
+[![PyPI - Version](https://img.shields.io/pypi/v/textmate-grammar-python.svg)](https://pypi.python.org/pypi/textmate-grammar-python)
+[![PyPI - License](https://img.shields.io/pypi/l/textmate-grammar-python.svg)](https://github.com/watermarkhu/textmate-grammar-python/tree/main?tab=MIT-1-ov-file)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![Checked with mypy](https://img.shields.io/badge/mypy-checked-blue)](http://mypy-lang.org/)
+[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit)](https://github.com/pre-commit/pre-commit)
+[![Python versions](https://img.shields.io/pypi/pyversions/textmate-grammar-python.svg)](https://pypi.python.org/pypi/textmate-grammar-python)
+[![CI/CD](https://github.com/watermarkhu/textmate-grammar-python/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/watermarkhu/textmate-grammar-python/blob/main/.github/workflows/ci.yml)
+[![readthedocs](https://readthedocs.org/projects/textmate-grammar-python/badge/?version=latest)](https://textmate-grammar-python.readthedocs.io)
+
+# textmate-grammar-python
+
+A lexer and tokenizer for grammar files as defined by TextMate and used in VSCode, implemented in Python. 
+
+Textmate grammars are made for [vscode-texmate](https://github.com/microsoft/vscode-textmate), allowing for syntax highlighting in VSCode after tokenization. This presents textmate-grammar-python with a large list of potentially supported languages. 
+
+```mermaid
+flowchart TD
+    A[grammar file] 
+    Z[code]
+    B("`vscode-textmate **js**`")
+    C("`textmate-grammar-**python**`")
+    D[tokens]
+
+    click C "https://github.com/microsoft/vscode-textmate"
+    
+    Z --> B
+    Z --> C
+    A -.-> B --> D
+    A -.-> C --> D
+```
+
+## Usage
+Install the module with:
+```bash
+pip install textmate-grammar-python
+```
+
+Before tokenization is possible, a `LanguageParser` needs to be initialized using a loaded grammar. 
+
+```python
+from textmate_grammar.language import LanguageParser
+from textmate_grammar.grammars import matlab
+parser = LanguageParser(matlab.GRAMMAR)
+```
+
+After this, one can either choose to call [`parser.parsing_string`](https://textmate-grammar-python.readthedocs.io/en/latest/apidocs/textmate_grammar/textmate_grammar.language.html#textmate_grammar.language.LanguageParser.parse_string) to parse a input string directly, or call [`parser.parse_file`](https://textmate-grammar-python.readthedocs.io/en/latest/apidocs/textmate_grammar/textmate_grammar.language.html#textmate_grammar.language.LanguageParser.parse_file) with the path to the appropiate source file as the first argument, such as in the example [`example.py`](https://github.com/watermarkhu/textmate-grammar-python/blob/main/example.py). 
+
+The parsed `element` object can be displayed directly by calling the [`print`](https://textmate-grammar-python.readthedocs.io/en/latest/apidocs/textmate_grammar/textmate_grammar.elements.html#textmate_grammar.elements.ContentElement.print) method. By default the element is printed as an element tree in a dictionary format. 
+
+```python
+>>> element = parser.parse_string("value = num2str(10);")
+>>> element.print()
+
+{'token': 'source.matlab',
+ 'children': [{'token': 'meta.assignment.variable.single.matlab', 
+               'children': [{'token': 'variable.other.readwrite.matlab', 'content': 'value'}]},
+              {'token': 'keyword.operator.assignment.matlab', 'content': '='},
+              {'token': 'meta.function-call.parens.matlab',
+               'begin': [{'token': 'entity.name.function.matlab', 'content': 'num2str'},
+                         {'token': 'punctuation.section.parens.begin.matlab', 'content': '('}],
+               'end': [{'token': 'punctuation.section.parens.end.matlab', 'content': ')'}],
+               'children': [{'token': 'constant.numeric.decimal.matlab', 'content': '10'}]},
+              {'token': 'punctuation.terminator.semicolon.matlab', 'content': ';'}]}
+
+```
+Alternatively, with the keyword argument `flatten` the element is displayed as a list per unique token. Here the first item in the list is the starting position (line, column) of the unique tokenized element. 
+
+```python
+>>> element.print(flatten=True)
+
+[[(0, 0), 'value', ['source.matlab', 'meta.assignment.variable.single.matlab', 'variable.other.readwrite.matlab']],
+ [(0, 5), ' ', ['source.matlab']],
+ [(0, 6), '=', ['source.matlab', 'keyword.operator.assignment.matlab']],
+ [(0, 7), ' ', ['source.matlab']],
+ [(0, 8), 'num2str', ['source.matlab', 'meta.function-call.parens.matlab', 'entity.name.function.matlab']],
+ [(0, 15), '(', ['source.matlab', 'meta.function-call.parens.matlab', 'punctuation.section.parens.begin.matlab']],
+ [(0, 16), '10', ['source.matlab', 'meta.function-call.parens.matlab', 'constant.numeric.decimal.matlab']],
+ [(0, 18), ')', ['source.matlab', 'meta.function-call.parens.matlab', 'punctuation.section.parens.end.matlab']],
+ [(0, 19), ';', ['source.matlab', 'punctuation.terminator.semicolon.matlab']]]
+```
+
+## Information
+
+- For further information, please checkout the [documentation](https://textmate-grammar-python.readthedocs.io/en/latest/). 
+- To setup an environment for development, see [CONTRIBUTING.md](https://github.com/watermarkhu/textmate-grammar-python/blob/main/CONTRIBUTING.md)

textmate_grammar_python-0.3.0/pyproject.toml

@@ -0,0 +1,95 @@
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry]
+name = "textmate-grammar-python"
+version = "0.3.0"
+description = "A lexer and tokenizer for grammar files as defined by TextMate and used in VSCode, implemented in Python."
+authors = ["Mark Shui Hu <watermarkhu@gmail.com>"]
+license = "MIT"
+readme = "README.md"
+repository = "https://github.com/watermarkhu/textmate-grammar-python"
+documentation = "https://textmate-grammar-python.readthedocs.io"
+keywords = ["textmate", "tokenization"]
+packages = [{include = "textmate_grammar", from = "src"}]
+
+[tool.poetry.dependencies]
+python = "^3.11"
+onigurumacffi = "^1.3.0"
+PyYAML = "^6.0.1"
+
+[tool.poetry.group.test.dependencies]
+pytest = ">=7.3.1,<9.0.0"
+pytest-icdiff = ">=0.8,<0.10"
+tox = "^4.11.4"
+
+[tool.poetry.group.dev.dependencies]
+pre-commit = "^3.6.0"
+mypy = "^1.8.0"
+ruff = "^0.3.0"
+types-pyyaml = "^6.0.12.12"
+poetry-bumpversion = "^0.3.2"
+
+[tool.poetry.group.doc.dependencies]
+sphinx = "^7.2.6"
+sphinx-autodoc2 = "^0.5.0"
+myst-parser = "^2.0.0"
+furo = "^2024.1.29"
+sphinxcontrib-mermaid = "^0.9.2"
+
+[tool.poetry_bumpversion.file."src/textmate_grammar/__init__.py"]
+
+[tool.poetry_bumpversion.file."docs/conf.py"]
+search = 'version = "{current_version}"'
+replace = 'version = "{new_version}"'
+
+[tool.ruff]
+include = ["pyproject.toml", "src/textmate_grammar/**/*.py"]
+exclude = ["src/textmate_grammar/grammars/"]
+line-length = 100
+indent-width = 4
+
+[tool.ruff.lint]
+select = ["E", "F", "UP", "B", "SIM", "I"]
+ignore = ["E111", "E114", "E117", "E501", "F401"]
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+skip-magic-trailing-comma = false
+line-ending = "auto"
+docstring-code-format = true
+docstring-code-line-length = "dynamic"
+
+[tool.mypy]
+ignore_missing_imports = true
+exclude = ["test/*", "docs/*", "example.py"]
+
+[tool.tox]
+legacy_tox_ini = """
+    [tox]
+    isolated_build = true
+
+    [testenv]
+    skip_install = true
+    allowlist_externals = poetry
+    commands =
+        poetry run pytest test/unit
+
+    [testenv:mypy]
+    skip_install = true
+    allowlist_externals = mypy
+    commands = 
+        mypy .
+
+    [testenv:regression]
+    skip_install = true
+    allowlist_externals = poetry, bash, sudo
+    platform = linux
+    change_dir = {tox_root}/test/regression
+    commands_pre =
+        bash install.sh
+    commands =
+        poetry run pytest .
+"""

textmate_grammar_python-0.3.0/src/textmate_grammar/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.0"