textmate-grammar-python 0.4.0__tar.gz → 0.5.1__tar.gz

{textmate_grammar_python-0.4.0 → textmate_grammar_python-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: textmate-grammar-python
-Version: 0.4.0
+Version: 0.5.1
 Summary: A lexer and tokenizer for grammar files as defined by TextMate and used in VSCode, implemented in Python.
 Home-page: https://github.com/watermarkhu/textmate-grammar-python
 License: MIT
@@ -44,9 +44,8 @@ pip install textmate-grammar-python
 Before tokenization is possible, a `LanguageParser` needs to be initialized using a loaded grammar. 
 
 ```python
-from textmate_grammar.language import LanguageParser
-from textmate_grammar.grammars import matlab
-parser = LanguageParser(matlab.GRAMMAR)
+from textmate_grammar.parsers.matlab import MatlabParser
+parser = MatlabParser()
 ```
 
 After this, one can either choose to call [`parser.parsing_string`](https://textmate-grammar-python.readthedocs.io/en/latest/apidocs/textmate_grammar/textmate_grammar.language.html#textmate_grammar.language.LanguageParser.parse_string) to parse a input string directly, or call [`parser.parse_file`](https://textmate-grammar-python.readthedocs.io/en/latest/apidocs/textmate_grammar/textmate_grammar.language.html#textmate_grammar.language.LanguageParser.parse_file) with the path to the appropiate source file as the first argument, such as in the example [`example.py`](https://github.com/watermarkhu/textmate-grammar-python/blob/main/example.py). 

{textmate_grammar_python-0.4.0 → textmate_grammar_python-0.5.1}/README.md

@@ -22,9 +22,8 @@ pip install textmate-grammar-python
 Before tokenization is possible, a `LanguageParser` needs to be initialized using a loaded grammar. 
 
 ```python
-from textmate_grammar.language import LanguageParser
-from textmate_grammar.grammars import matlab
-parser = LanguageParser(matlab.GRAMMAR)
+from textmate_grammar.parsers.matlab import MatlabParser
+parser = MatlabParser()
 ```
 
 After this, one can either choose to call [`parser.parsing_string`](https://textmate-grammar-python.readthedocs.io/en/latest/apidocs/textmate_grammar/textmate_grammar.language.html#textmate_grammar.language.LanguageParser.parse_string) to parse a input string directly, or call [`parser.parse_file`](https://textmate-grammar-python.readthedocs.io/en/latest/apidocs/textmate_grammar/textmate_grammar.language.html#textmate_grammar.language.LanguageParser.parse_file) with the path to the appropiate source file as the first argument, such as in the example [`example.py`](https://github.com/watermarkhu/textmate-grammar-python/blob/main/example.py). 

{textmate_grammar_python-0.4.0 → textmate_grammar_python-0.5.1}/pyproject.toml

@@ -1,10 +1,14 @@
+[project]
+name = "textmate-grammar-python"
+
+
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "textmate-grammar-python"
-version = "0.4.0"
+version = "0.5.1"
 description = "A lexer and tokenizer for grammar files as defined by TextMate and used in VSCode, implemented in Python."
 authors = ["Mark Shui Hu <watermarkhu@gmail.com>"]
 license = "MIT"
@@ -14,6 +18,9 @@ documentation = "https://textmate-grammar-python.readthedocs.io"
 keywords = ["textmate", "tokenization"]
 packages = [{include = "textmate_grammar", from = "src"}]
 
+[tool.rye]
+managed = true
+
 [tool.poetry.dependencies]
 python = "^3.9"
 onigurumacffi = "^1.3.0"

textmate_grammar_python-0.5.1/src/textmate_grammar/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.5.1"

{textmate_grammar_python-0.4.0 → textmate_grammar_python-0.5.1}/src/textmate_grammar/elements.py

@@ -6,7 +6,7 @@ from itertools import groupby
 from pprint import pprint
 from typing import TYPE_CHECKING, Generator
 
-from .utils.handler import POS, ContentHandler, Match, Pattern
+from .handler import POS, ContentHandler, Match, Pattern
 from .utils.logger import LOGGER
 
 if TYPE_CHECKING:

{textmate_grammar_python-0.4.0/src/textmate_grammar/utils → textmate_grammar_python-0.5.1/src/textmate_grammar}/handler.py

@@ -1,17 +1,22 @@
 from __future__ import annotations
 
 from pathlib import Path
+from typing import Callable
 
 from onigurumacffi import _Match as Match
 from onigurumacffi import _Pattern as Pattern
 from onigurumacffi import compile
 
-from .exceptions import FileNotFound, ImpossibleSpan
-from .logger import LOGGER
+from .utils.exceptions import FileNotFound, ImpossibleSpan
+from .utils.logger import LOGGER
 
 POS = tuple[int, int]
 
 
+def _dummy_pre_processor(input: str) -> str:
+    return input
+
+
 class ContentHandler:
     """The handler object targetted for parsing.
 
@@ -23,25 +28,31 @@ class ContentHandler:
 
     notLookForwardEOL = compile(r"(?<!\(\?=[^\(]*)\$")
 
-    def __init__(self, source: str) -> None:
+    def __init__(
+        self, content: str, pre_processor: Callable[[str], str] = _dummy_pre_processor
+    ) -> None:
         """
         Initialize a new instance of the Handler class.
 
-        :param source: The source code to be processed.
-        :type source: str
+        :param content: The source code to be processed.
+        :type content: str
+        :param pre_processor: A pre-processor to use on the input string of the parser
+        :type pre_processor: BasePreProcessor
 
-        :ivar source: The source code to be processed.
+        :ivar content: The source code to be processed.
         :ivar lines: A list of lines in the source code, with a newline character at the end of each line.
         :ivar line_lengths: A list of lengths of each line in the source code.
         :ivar anchor: The current position in the source code.
         """
-        self.source = source
-        self.lines = [line + "\n" for line in source.split("\n")]
+        prepared_content = pre_processor(content.replace("\r\n", "\n").replace("\r", "\n"))
+
+        self.content = prepared_content
+        self.lines = [line + "\n" for line in prepared_content.split("\n")]
         self.line_lengths = [len(line) for line in self.lines]
         self.anchor: int = 0
 
     @classmethod
-    def from_path(cls, file_path: Path):
+    def from_path(cls, file_path: Path, **kwargs) -> ContentHandler:
         """Loads a file from a path"""
 
         if not file_path.exists():
@@ -50,10 +61,8 @@ class ContentHandler:
         # Open file and replace Windows/Mac line endings
         with open(file_path) as file:
             content = file.read()
-        content = content.replace("\r\n", "\n")
-        content = content.replace("\r", "\n")
 
-        return cls(content)
+        return cls(content, **kwargs)
 
     def _check_pos(self, pos: POS):
         if pos[0] > len(self.lines) or pos[1] > self.line_lengths[pos[0]]:

{textmate_grammar_python-0.4.0 → textmate_grammar_python-0.5.1}/src/textmate_grammar/parser.py

@@ -6,12 +6,12 @@ from typing import TYPE_CHECKING
 import onigurumacffi as re
 
 from .elements import Capture, ContentBlockElement, ContentElement
+from .handler import POS, ContentHandler, Pattern
 from .utils.exceptions import IncludedParserNotFound
-from .utils.handler import POS, ContentHandler, Pattern
 from .utils.logger import LOGGER, track_depth
 
 if TYPE_CHECKING:
-    from .language import LanguageParser
+    from .parsers.base import LanguageParser
 
 
 class GrammarParser(ABC):
@@ -42,7 +42,7 @@ class GrammarParser(ABC):
     def __init__(
         self,
         grammar: dict,
-        language: LanguageParser | None = None,
+        language_parser: LanguageParser | None = None,
         key: str = "",
         is_capture: bool = False,
         **kwargs,
@@ -59,7 +59,7 @@ class GrammarParser(ABC):
         :return: None
         """
         self.grammar = grammar
-        self.language = language
+        self.language_parser = language_parser
         self.key = key
         self.token = grammar.get("name", "")
         self.is_capture = is_capture
@@ -83,21 +83,21 @@ class GrammarParser(ABC):
         if key in grammar:
             for group_id, pattern in grammar[key].items():
                 captures[int(group_id)] = self.initialize(
-                    pattern, language=self.language, is_capture=True
+                    pattern, language_parser=self.language_parser, is_capture=True
                 )
         return captures
 
     def _find_include(self, key: str, **kwargs) -> GrammarParser:
         """Find the included grammars and during repository initialization"""
-        if not self.language:
+        if not self.language_parser:
             raise IncludedParserNotFound(key)
 
         if key in ["$self", "$base"]:  # TODO there is a difference between these
-            return self.language
+            return self.language_parser
         elif key[0] == "#":
-            return self.language.repository.get(key[1:], None)
+            return self.language_parser.repository.get(key[1:], None)
         else:
-            return self.language._find_include_scopes(key)
+            return self.language_parser._find_include_scopes(key)
 
     @abstractmethod
     def _parse(
@@ -146,8 +146,8 @@ class GrammarParser(ABC):
             - elements: A list of Capture or ContentElement objects representing the parsed content.
             - span: A tuple containing the starting and ending positions of the parsed content, or None if parsing failed.
         """
-        if not self.initialized and self.language is not None:
-            self.language._initialize_repository()
+        if not self.initialized and self.language_parser is not None:
+            self.language_parser._initialize_repository()
         parsed, elements, span = self._parse(handler, starting, boundary=boundary, **kwargs)
         return parsed, elements, span
 
@@ -325,7 +325,7 @@ class ParserHasPatterns(GrammarParser, ABC):
     def __init__(self, grammar: dict, **kwargs) -> None:
         super().__init__(grammar, **kwargs)
         self.patterns = [
-            self.initialize(pattern, language=self.language)
+            self.initialize(pattern, language_parser=self.language_parser)
             for pattern in grammar.get("patterns", [])
         ]
 
@@ -347,7 +347,7 @@ class ParserHasPatterns(GrammarParser, ABC):
             self.patterns[parser_index : parser_index + 1] = parser.patterns
 
         # Injection grammars
-        for exception_scopes, injection_pattern in self.language.injections:
+        for exception_scopes, injection_pattern in self.language_parser.injections:
             if self.token:
                 if self.token.split(".")[0] not in exception_scopes:
                     self.patterns.append(injection_pattern)
@@ -442,7 +442,7 @@ class PatternsParser(ParserHasPatterns):
                         current,
                         kwargs.get("depth", 0),
                     )
-                elif self != self.language:
+                elif self != self.language_parser:
                     break
                 else:
                     remainder = handler.read_line(current)

textmate_grammar_python-0.4.0/src/textmate_grammar/language.py → textmate_grammar_python-0.5.1/src/textmate_grammar/parsers/base.py

@@ -2,12 +2,12 @@ from __future__ import annotations
 
 from pathlib import Path
 
-from .elements import Capture, ContentElement
-from .parser import GrammarParser, PatternsParser
-from .utils.cache import TextmateCache, init_cache
-from .utils.exceptions import IncompatibleFileType
-from .utils.handler import POS, ContentHandler
-from .utils.logger import LOGGER
+from ..elements import Capture, ContentElement
+from ..handler import POS, ContentHandler
+from ..parser import GrammarParser, PatternsParser
+from ..utils.cache import TextmateCache, init_cache
+from ..utils.exceptions import IncompatibleFileType
+from ..utils.logger import LOGGER
 
 LANGUAGE_PARSERS = {}
 
@@ -34,6 +34,9 @@ class LanguageParser(PatternsParser):
         Initialize a Language object.
 
         :param grammar: The grammar definition for the language.
+        :type grammar: dict
+        :param pre_processor: A pre-processor to use on the input string of the parser
+        :type pre_processor: BasePreProcessor
         :param kwargs: Additional keyword arguments.
 
         :ivar name: The name of the language.
@@ -44,7 +47,10 @@ class LanguageParser(PatternsParser):
         :ivar injections: The list of injection rules for the language.
         :ivar _cache: The cache object for the language.
         """
-        super().__init__(grammar, key=grammar.get("name", "myLanguage"), language=self, **kwargs)
+
+        super().__init__(
+            grammar, key=grammar.get("name", "myLanguage"), language_parser=self, **kwargs
+        )
 
         self.name = grammar.get("name", "")
         self.uuid = grammar.get("uuid", "")
@@ -58,7 +64,7 @@ class LanguageParser(PatternsParser):
         for repo in _gen_repositories(grammar):
             for key, parser_grammar in repo.items():
                 self.repository[key] = GrammarParser.initialize(
-                    parser_grammar, key=key, language=self
+                    parser_grammar, key=key, language_parser=self
                 )
 
         # Update language parser store
@@ -67,6 +73,14 @@ class LanguageParser(PatternsParser):
 
         self._initialize_repository()
 
+    def pre_process(self, input: str) -> str:
+        """
+        Pre-processes the input string before parsing.
+
+        This method can be overloaded in language specific parsers with custom pre-processing logic.
+        """
+        return input
+
     def __repr__(self) -> str:
         return f"{self.__class__.__name__}:{self.key}"
 
@@ -88,7 +102,7 @@ class LanguageParser(PatternsParser):
             injected_parser = GrammarParser.initialize(
                 injected_grammar,
                 key=f"{target_string}.injection",
-                language=target_language,
+                language_parser=target_language,
             )
             injected_parser._initialize_repository()
 
@@ -115,8 +129,8 @@ class LanguageParser(PatternsParser):
         if self._cache.cache_valid(filePath):
             element = self._cache.load(filePath)
         else:
-            handler = ContentHandler.from_path(filePath)
-            if handler.source == "":
+            handler = ContentHandler.from_path(filePath, pre_processor=self.pre_process, **kwargs)
+            if handler.content == "":
                 return None
 
             # Configure logger
@@ -135,7 +149,8 @@ class LanguageParser(PatternsParser):
         :param kwargs: Additional keyword arguments.
         :return: The result of parsing the input string.
         """
-        handler = ContentHandler(input)
+        handler = ContentHandler(input, pre_processor=self.pre_process, **kwargs)
+
         # Configure logger
         LOGGER.configure(self, height=len(handler.lines), width=max(handler.line_lengths))
 

textmate_grammar_python-0.5.1/src/textmate_grammar/parsers/markdown/__init__.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+import yaml
+
+from ..base import LanguageParser
+
+
+class MarkdownParser(LanguageParser):
+    def __init__(self, **kwargs):
+        tmLanguageFile = (
+            Path(__file__).parents[3] / "syntaxes" / "markdown" / "markdown.tmLanguage.base.yaml"
+        )
+        tmLanguageYAML = Path(__file__).parent / "grammar.yaml"
+
+        if tmLanguageFile.exists():
+            shutil.copyfile(tmLanguageFile, tmLanguageYAML)
+
+        with open(tmLanguageYAML) as file:
+            try:
+                grammar = yaml.load(file.read(), Loader=yaml.CLoader)
+            except ImportError:
+                grammar = yaml.load(file.read(), Loader=yaml.Loader)
+
+        super().__init__(grammar, **kwargs)

textmate_grammar_python-0.5.1/src/textmate_grammar/parsers/matlab/__init__.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import plistlib
+import re
+from pathlib import Path
+
+import yaml
+
+from ..base import LanguageParser
+
+
+class MatlabParser(LanguageParser):
+    """
+    Represents a grammar for the MATLAB language.
+    """
+
+    def __init__(self, remove_line_continuations: bool = False, **kwargs):
+        """
+        Initializes a new instance of the MatlabGrammar class.
+
+        Args:
+            remove_line_continuations (bool, optional): Whether to remove line continuations. Defaults to False.
+        """
+
+        self._rlc = remove_line_continuations
+
+        tmLanguageFile = (
+            Path(__file__).parents[3]
+            / "syntaxes"
+            / "matlab"
+            / "Matlab.tmbundle"
+            / "Syntaxes"
+            / "MATLAB.tmLanguage"
+        )
+        tmLanguageYAML = Path(__file__).parent / "grammar.yaml"
+
+        if tmLanguageFile.exists():
+            with open(tmLanguageFile, "rb") as tmFile:
+                grammar = plistlib.load(tmFile, fmt=plistlib.FMT_XML)
+            with open(tmLanguageYAML, "w") as ymlFile:
+                ymlFile.write(yaml.dump(grammar, indent=2))
+        else:
+            with open(tmLanguageYAML) as ymlFile:
+                try:
+                    grammar = yaml.load(ymlFile.read(), Loader=yaml.CLoader)
+                except ImportError:
+                    grammar = yaml.load(ymlFile.read(), Loader=yaml.Loader)
+
+        super().__init__(grammar, **kwargs)
+
+    def pre_process(self, input: str) -> str:
+        """
+        Pre-processes the input text.
+        """
+        if self._rlc:
+            input = self._remove_line_continuations(input)
+        return input
+
+    def _remove_line_continuations(self, input: str) -> str:
+        """
+        Removes line continuations from the input text.
+        """
+        output = ""
+        for split in input.split("..."):
+            matching = re.search(r"\n[\t\f\v ]*", split)
+            if matching:
+                output += split[matching.span()[1] :]
+            else:
+                output += split
+        return output

textmate_grammar_python-0.4.0/src/textmate_grammar/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.4.0"

textmate_grammar_python-0.4.0/src/textmate_grammar/grammars/markdown/__init__.py

@@ -1,21 +0,0 @@
-from __future__ import annotations
-
-import shutil
-from pathlib import Path
-
-import yaml
-
-tmLanguageFile = (
-    Path(__file__).parents[3] / "syntaxes" / "markdown" / "markdown.tmLanguage.base.yaml"
-)
-tmLanguageYAML = Path(__file__).parent / "grammar.yaml"
-
-
-if tmLanguageFile.exists():
-    shutil.copyfile(tmLanguageFile, tmLanguageYAML)
-
-with open(tmLanguageYAML) as file:
-    try:
-        GRAMMAR = yaml.load(file.read(), Loader=yaml.CLoader)
-    except ImportError:
-        GRAMMAR = yaml.load(file.read(), Loader=yaml.Loader)

textmate_grammar_python-0.4.0/src/textmate_grammar/grammars/matlab/__init__.py

@@ -1,29 +0,0 @@
-from __future__ import annotations
-
-import plistlib
-from pathlib import Path
-
-import yaml
-
-tmLanguageFile = (
-    Path(__file__).parents[3]
-    / "syntaxes"
-    / "matlab"
-    / "Matlab.tmbundle"
-    / "Syntaxes"
-    / "MATLAB.tmLanguage"
-)
-tmLanguageYAML = Path(__file__).parent / "grammar.yaml"
-
-
-if tmLanguageFile.exists():
-    with open(tmLanguageFile, "rb") as file:
-        GRAMMAR = plistlib.load(file, fmt=plistlib.FMT_XML)
-    with open(tmLanguageYAML, "w") as f:
-        f.write(yaml.dump(GRAMMAR, indent=2))
-else:
-    with open(tmLanguageYAML) as file:
-        try:
-            GRAMMAR = yaml.load(file.read(), Loader=yaml.CLoader)
-        except ImportError:
-            GRAMMAR = yaml.load(file.read(), Loader=yaml.Loader)