textmate-grammar-python 0.5.3__tar.gz → 0.6.1__tar.gz

{textmate_grammar_python-0.5.3 → textmate_grammar_python-0.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: textmate-grammar-python
-Version: 0.5.3
+Version: 0.6.1
 Summary: A lexer and tokenizer for grammar files as defined by TextMate and used in VSCode, implemented in Python.
 Home-page: https://github.com/watermarkhu/textmate-grammar-python
 License: MIT
@@ -15,6 +15,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
+Requires-Dist: charset-normalizer (>=3.3.2,<4.0.0)
 Requires-Dist: onigurumacffi (>=1.3.0,<2.0.0)
 Project-URL: Documentation, https://textmate-grammar-python.readthedocs.io
 Project-URL: Repository, https://github.com/watermarkhu/textmate-grammar-python

{textmate_grammar_python-0.5.3 → textmate_grammar_python-0.6.1}/pyproject.toml

@@ -1,14 +1,13 @@
 [project]
 name = "textmate-grammar-python"
 
-
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "textmate-grammar-python"
-version = "0.5.3"
+version = "0.6.1"
 description = "A lexer and tokenizer for grammar files as defined by TextMate and used in VSCode, implemented in Python."
 authors = ["Mark Shui Hu <watermarkhu@gmail.com>"]
 license = "MIT"
@@ -25,6 +24,7 @@ managed = true
 python = "^3.9"
 onigurumacffi = "^1.3.0"
 PyYAML = "^6.0.1"
+charset-normalizer = "^3.3.2"
 
 [tool.poetry.group.test.dependencies]
 pytest = ">=7.3.1,<9.0.0"
@@ -33,15 +33,16 @@ tox = "^4.11.4"
 
 [tool.poetry.group.dev.dependencies]
 pre-commit = "^3.6.0"
-mypy = "^1.8.0"
-ruff = "^0.3.0"
+mypy = "^1.10.0"
+ruff = "^0.5.0"
 types-pyyaml = "^6.0.12.12"
+poetry-plugin-export = "^1.8.0"
 poetry-bumpversion = "^0.3.2"
 
 [tool.poetry.group.doc.dependencies]
 sphinx = "^7.2.6"
 sphinx-autodoc2 = "^0.5.0"
-myst-parser = "^2.0.0"
+myst-parser = "^3.0.0"
 furo = "^2024.1.29"
 sphinxcontrib-mermaid = "^0.9.2"
 

textmate_grammar_python-0.6.1/src/textmate_grammar/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.6.1"

{textmate_grammar_python-0.5.3 → textmate_grammar_python-0.6.1}/src/textmate_grammar/handler.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from pathlib import Path
 from typing import Callable
 
+import charset_normalizer as charset
 from onigurumacffi import _Match as Match
 from onigurumacffi import _Pattern as Pattern
 from onigurumacffi import compile
@@ -44,6 +45,7 @@ class ContentHandler:
         :ivar line_lengths: A list of lengths of each line in the source code.
         :ivar anchor: The current position in the source code.
         """
+        # Proprocess the content, replace all newline characters with \n
         prepared_content = pre_processor(content.replace("\r\n", "\n").replace("\r", "\n"))
 
         self.content = prepared_content
@@ -58,9 +60,8 @@ class ContentHandler:
         if not file_path.exists():
             raise FileNotFound(str(file_path))
 
-        # Open file and replace Windows/Mac line endings
-        with open(file_path) as file:
-            content = file.read()
+        # Open file with best guess encoding from charset_normalizer
+        content = str(charset.from_path(file_path).best())
 
         return cls(content, **kwargs)
 

textmate_grammar_python-0.5.3/src/textmate_grammar/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.5.3"