textmate-grammar-python 0.5.2__tar.gz → 0.6.0__tar.gz

{textmate_grammar_python-0.5.2 → textmate_grammar_python-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: textmate-grammar-python
-Version: 0.5.2
+Version: 0.6.0
 Summary: A lexer and tokenizer for grammar files as defined by TextMate and used in VSCode, implemented in Python.
 Home-page: https://github.com/watermarkhu/textmate-grammar-python
 License: MIT
@@ -15,6 +15,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: PyYAML (>=6.0.1,<7.0.0)
+Requires-Dist: charset-normalizer (>=3.3.2,<4.0.0)
 Requires-Dist: onigurumacffi (>=1.3.0,<2.0.0)
 Project-URL: Documentation, https://textmate-grammar-python.readthedocs.io
 Project-URL: Repository, https://github.com/watermarkhu/textmate-grammar-python

{textmate_grammar_python-0.5.2 → textmate_grammar_python-0.6.0}/pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "textmate-grammar-python"
-version = "0.5.2"
+version = "0.6.0"
 description = "A lexer and tokenizer for grammar files as defined by TextMate and used in VSCode, implemented in Python."
 authors = ["Mark Shui Hu <watermarkhu@gmail.com>"]
 license = "MIT"
@@ -25,6 +25,7 @@ managed = true
 python = "^3.9"
 onigurumacffi = "^1.3.0"
 PyYAML = "^6.0.1"
+charset-normalizer = "^3.3.2"
 
 [tool.poetry.group.test.dependencies]
 pytest = ">=7.3.1,<9.0.0"

textmate_grammar_python-0.6.0/src/textmate_grammar/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.6.0"

{textmate_grammar_python-0.5.2 → textmate_grammar_python-0.6.0}/src/textmate_grammar/handler.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from pathlib import Path
 from typing import Callable
 
+import charset_normalizer as charset
 from onigurumacffi import _Match as Match
 from onigurumacffi import _Pattern as Pattern
 from onigurumacffi import compile
@@ -44,6 +45,7 @@ class ContentHandler:
         :ivar line_lengths: A list of lengths of each line in the source code.
         :ivar anchor: The current position in the source code.
         """
+        # Proprocess the content, replace all newline characters with \n
         prepared_content = pre_processor(content.replace("\r\n", "\n").replace("\r", "\n"))
 
         self.content = prepared_content
@@ -58,9 +60,8 @@ class ContentHandler:
         if not file_path.exists():
             raise FileNotFound(str(file_path))
 
-        # Open file and replace Windows/Mac line endings
-        with open(file_path) as file:
-            content = file.read()
+        # Open file with best guess encoding from charset_normalizer
+        content = str(charset.from_path(file_path).best())
 
         return cls(content, **kwargs)
 

textmate_grammar_python-0.5.2/src/textmate_grammar/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.5.2"