textmate-grammar-python 0.2.0__py3-none-any.whl → 0.3.0__py3-none-any.whl

textmate_grammar/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.0"

textmate_grammar/elements.py

@@ -6,8 +6,8 @@ from itertools import groupby
 from pprint import pprint
 from typing import TYPE_CHECKING, Generator
 
-from .handler import POS, ContentHandler, Match, Pattern
-from .logger import LOGGER
+from .utils.handler import POS, ContentHandler, Match, Pattern
+from .utils.logger import LOGGER
 
 if TYPE_CHECKING:
     from .parser import GrammarParser
@@ -33,7 +33,20 @@ class Capture:
         boundary: tuple[int, int],
         key: str = "",
         **kwargs,
-    ) -> None:
+    ):
+        """
+        Initialize a new instance of the Element class.
+
+        :param handler: The content handler for the element.
+        :param pattern: The pattern used for matching.
+        :param matching: The match object.
+        :param parsers: A dictionary of grammar parsers.
+        :param starting: The starting position of the element.
+        :param boundary: The boundary position of the element.
+        :param key: The key for the element. Defaults to "".
+        :param **kwargs: Additional keyword arguments.
+        :returns: None
+        """
         self.handler = handler
         self.pattern = pattern
         self.matching = matching
@@ -57,7 +70,13 @@ class Capture:
         return f"@capture<{self.key}>"
 
     def dispatch(self) -> list[Capture | ContentElement]:
-        """Dispatches the remaining parse of the capture group."""
+        """Dispatches the remaining parse of the capture group.
+
+        This method iterates over the defined parsers for the capture group and dispatches the remaining parse
+        based on the captured elements. It returns a list of captured elements or captures.
+
+        :return: A list of Capture or ContentElement objects representing the parsed elements.
+        """
         elements = []
         for group_id, parser in self.parsers.items():
             if group_id > self.pattern.number_of_captures():
@@ -100,22 +119,29 @@ class Capture:
         return elements
 
 
-def dispatch_list(
+def _dispatch_list(
     pending_elements: list[Capture | ContentElement], parent: ContentElement | None = None
 ) -> list[ContentElement]:
     """Dispatches all captured parsers in the list."""
     elements = []
     for item in pending_elements:
         if isinstance(item, Capture):
-            captured_elements: list[ContentElement] = dispatch_list(item.dispatch())
+            captured_elements: list[ContentElement] = _dispatch_list(item.dispatch())
             elements.extend(captured_elements)
         elif item != parent:
             elements.append(item)
     return elements
 
 
+def _str_to_list(input: str | list[str]) -> list[str]:
+    if isinstance(input, str):
+        return [input] if input else []
+    else:
+        return input
+
+
 class ContentElement:
-    """The base grammar element object."""
+    """The parsed grammar element."""
 
     def __init__(
         self,
@@ -125,15 +151,23 @@ class ContentElement:
         characters: dict[POS, str],
         children: list[Capture | ContentElement] | None = None,
     ) -> None:
+        """
+        Initialize a new instance of the Element class.
+
+        :param token: The token associated with the element.
+        :param grammar: The grammar associated with the element.
+        :param content: The content associated with the element.
+        :param characters: The characters associated with the element.
+        :param children: The children associated with the element. Defaults to None.
+        """
         if children is None:
             children = []
         self.token = token
         self.grammar = grammar
         self.content = content
         self.characters = characters
-        self._children_pending = children
-        self._children_dispached: list[ContentElement] = []
-        self._dispatched_children: bool = False
+        self._children_captures = children
+        self._dispatched: bool = False
 
     @property
     def _subelements(self) -> list[ContentElement]:
@@ -141,52 +175,54 @@ class ContentElement:
 
     @property
     def children(self) -> list[ContentElement]:
-        "Children elements"
-        if self._children_pending:
-            if not self._dispatched_children:
-                self._children_dispached = dispatch_list(self._children_pending, parent=self)
-                self._dispatched_children = True
-            return self._children_dispached
-        else:
-            return []
+        """
+        Returns a list of children elements.
+
+        If the elements have not been dispatched yet, this method will dispatch them before returning.
+
+        :return: A list of ContentElement objects representing the children elements.
+        """
+        if not self._dispatched:
+            self._dispatch()
+        return self._children
+
+    def _dispatch(self, nested: bool = False):
+        """
+        Dispatches the content element and its children.
+
+        :param nested: Indicates whether the dispatch is nested within another dispatch.
+        :type nested: bool
+        :return: None
+        """
+        if self._dispatched:
+            return
+        self._dispatched = True
+        self._children: list[ContentElement] = _dispatch_list(self._children_captures, parent=self)
+        self._children_captures = []
+        if nested:
+            for child in self._children:
+                child._dispatch(True)
 
     def __eq__(self, other):
         if not isinstance(other, ContentElement):
             return False
         return bool(self.grammar == other.grammar and self.characters == other.characters)
 
-    def to_dict(self, verbosity: int = -1, all_content: bool = False, **kwargs) -> dict:
-        "Converts the object to dictionary."
-        out_dict = {"token": self.token}
-        if all_content or not self.children:
-            out_dict["content"] = self.content
-        if self.children:
-            out_dict["children"] = (
-                self._list_property_to_dict(
-                    "children", verbosity=verbosity - 1, all_content=all_content
-                )
-                if verbosity
-                else self.children
-            )
-        return out_dict
-
-    def find(
+    def _find(
         self,
         tokens: str | list[str],
+        start_tokens: str | list[str] = "",
+        hide_tokens: str | list[str] = "",
         stop_tokens: str | list[str] = "",
-        verbosity: int = -1,
-        stack: list[str] | None = None,
+        depth: int = -1,
         attribute: str = "_subelements",
+        stack: list[str] | None = None,
     ) -> Generator[tuple[ContentElement, list[str]], None, None]:
-        """Find the next subelement that match the input token(s).
+        tokens = _str_to_list(tokens)
+        start_tokens = _str_to_list(start_tokens)
+        hide_tokens = _str_to_list(hide_tokens)
+        stop_tokens = _str_to_list(stop_tokens)
 
-        The find method will return a generator that globs though the element-tree, searching for the next
-        subelement that matches the given token.
-        """
-        if isinstance(tokens, str):
-            tokens = [tokens]
-        if isinstance(stop_tokens, str):
-            stop_tokens = [stop_tokens] if stop_tokens else []
         if not set(tokens).isdisjoint(set(stop_tokens)):
             raise ValueError("Input tokens and stop_tokens must be disjoint")
 
@@ -194,8 +230,10 @@ class ContentElement:
             stack = []
         stack += [self.token]
 
-        if verbosity:
-            verbosity -= 1
+        start_found = not start_tokens
+
+        if depth:
+            depth -= 1
             children: list[ContentElement] = getattr(self, attribute, self._subelements)
             for child in children:
                 if stop_tokens and (
@@ -204,29 +242,126 @@ class ContentElement:
                 ):
                     return None
 
-                if child.token in tokens or tokens == ["*"]:
+                if not start_found and child.token in start_tokens:
+                    start_found = True
+                    start_tokens = []
+
+                if (
+                    start_found
+                    and (child.token in tokens or tokens == ["*"])
+                    and child.token not in hide_tokens
+                ):
                     yield child, [e for e in stack]
-                if verbosity:
-                    nested_generator = child.find(
-                        tokens, verbosity=verbosity - 1, stack=[e for e in stack]
+                if depth:
+                    nested_generator = child._find(
+                        tokens,
+                        start_tokens=start_tokens,
+                        hide_tokens=hide_tokens,
+                        stop_tokens=stop_tokens,
+                        depth=depth - 1,
+                        stack=[e for e in stack],
                     )
                     yield from nested_generator
         return None
 
+    def find(
+        self,
+        tokens: str | list[str],
+        start_tokens: str | list[str] = "",
+        hide_tokens: str | list[str] = "",
+        stop_tokens: str | list[str] = "",
+        depth: int = -1,
+        attribute: str = "_subelements",
+    ) -> Generator[tuple[ContentElement, list[str]], None, None]:
+        """
+        Find content elements based on the given criteria.
+
+        The find method will return a generator that globs though the element-tree, searching for the next
+        subelement that matches the given token.
+
+        :param tokens: The tokens to search for. Can be a single token or a list of tokens.
+        :param start_tokens: The tokens that mark the start of the search. Can be a single token or a list of tokens.
+        :param hide_tokens: The tokens to hide from the search results. Can be a single token or a list of tokens.
+        :param stop_tokens: The tokens that mark the end of the search. Can be a single token or a list of tokens.
+        :param depth: The maximum depth to search. Defaults to -1 (unlimited depth).
+        :param attribute: The attribute name to access the subelements. Defaults to "_subelements".
+
+        :yield: A tuple containing the found content element and the stack of tokens encountered.
+
+        :raises ValueError: If the input tokens and stop_tokens are not disjoint.
+
+        :return: None if no matching content elements are found.
+        """
+        return self._find(
+            tokens,
+            start_tokens=start_tokens,
+            hide_tokens=hide_tokens,
+            stop_tokens=stop_tokens,
+            depth=depth,
+            attribute=attribute,
+        )
+
     def findall(
         self,
         tokens: str | list[str],
+        start_tokens: str | list[str] = "",
+        hide_tokens: str | list[str] = "",
         stop_tokens: str | list[str] = "",
-        verbosity: int = -1,
+        depth: int = -1,
         attribute: str = "_subelements",
     ) -> list[tuple[ContentElement, list[str]]]:
-        """Returns subelements that match the input token(s)."""
+        """
+        Find all occurrences of the specified tokens within the content element.
+
+        :param tokens: The tokens to search for.
+        :param start_tokens: The tokens that must appear before the found tokens. Defaults to "".
+        :param hide_tokens: The tokens that should be hidden from the search. Defaults to "".
+        :param stop_tokens: The tokens that, if found, should stop the search. Defaults to "".
+        :param depth: The maximum depth to search. Defaults to -1 (unlimited depth).
+        :param attribute: The attribute to search within. Defaults to "_subelements".
+
+        :return: A list of tuples containing the content element and the found tokens.
+        """
         return list(
-            self.find(tokens, stop_tokens=stop_tokens, verbosity=verbosity, attribute=attribute)
+            self._find(
+                tokens,
+                start_tokens=start_tokens,
+                hide_tokens=hide_tokens,
+                stop_tokens=stop_tokens,
+                depth=depth,
+                attribute=attribute,
+            )
         )
 
+    def to_dict(self, depth: int = -1, all_content: bool = False, **kwargs) -> dict:
+        """
+        Converts the object to a dictionary.
+
+        :param depth: The depth of the conversion. Defaults to -1.
+        :param all_content: Whether to include all content or only the top-level content. Defaults to False.
+
+        :return: The converted dictionary representation of the object.
+        """
+        out_dict = {"token": self.token}
+        if all_content or not self.children:
+            out_dict["content"] = self.content
+        if self.children:
+            out_dict["children"] = (
+                self._list_property_to_dict("children", depth=depth - 1, all_content=all_content)
+                if depth
+                else self.children
+            )
+        return out_dict
+
     def flatten(self) -> list[tuple[tuple[int, int], str, list[str]]]:
-        """Converts the object to a flattened array of tokens per index."""
+        """
+        Converts the object to a flattened array of tokens per index, similarly to vscode-textmate.
+
+        :return: A list of tuples representing the flattened tokens. Each tuple contains:
+             - A tuple representing the starting and ending index of the token.
+             - The content of the token.
+             - A list of keys associated with the token.
+        """
         token_dict = self._token_by_index(defaultdict(list))
         tokens = []
         for (_, key), group in groupby(sorted(token_dict.items()), lambda x: (x[0][0], x[1])):
@@ -242,11 +377,20 @@ class ContentElement:
     def print(
         self,
         flatten: bool = False,
-        verbosity: int = -1,
+        depth: int = -1,
         all_content: bool = False,
         **kwargs,
     ) -> None:
-        """Prints the current object recursively by converting to dictionary."""
+        """
+        Prints the current object recursively by converting it to a dictionary or a flattened array.
+
+        :param flatten: If True, flattens the object before printing. Defaults to False.
+        :param depth: The maximum depth to print. Defaults to -1 (unlimited depth).
+        :param all_content: If True, includes all content in the printout. Defaults to False.
+        :param **kwargs: Additional keyword arguments to be passed to the pprint function.
+
+        :return: None
+        """
         if flatten:
             pprint(
                 self.flatten(**kwargs),
@@ -256,14 +400,23 @@ class ContentElement:
             )
         else:
             pprint(
-                self.to_dict(verbosity=verbosity, all_content=all_content, **kwargs),
+                self.to_dict(depth=depth, all_content=all_content, **kwargs),
                 sort_dicts=False,
                 width=kwargs.pop("width", 150),
                 **kwargs,
             )
 
     def _token_by_index(self, token_dict: TOKEN_DICT | None = None) -> TOKEN_DICT:
-        """Recursively tokenize every index between start and close."""
+        """Recursively tokenize every index between start and close.
+
+        This method recursively tokenizes every index between the start and close positions of the element.
+        It populates a dictionary, `token_dict`, with the tokens corresponding to each index.
+
+        :param token_dict: A dictionary to store the tokens. If None, a new dictionary is created.
+        :type token_dict: dict | None
+        :return: A dictionary containing the tokens for each index.
+        :rtype: dict
+        """
         if token_dict is None:
             token_dict = defaultdict(list)
         for pos in self.characters:
@@ -291,21 +444,27 @@ class ContentBlockElement(ContentElement):
 
     def __init__(
         self,
+        *args,
         begin: list[Capture | ContentElement] | None = None,
         end: list[Capture | ContentElement] | None = None,
         **kwargs,
     ) -> None:
+        """
+        Initialize a new instance of the Element class.
+
+        :param begin: A list of Capture or ContentElement objects representing the beginning captures of the element. Defaults to None.
+        :param end: A list of Capture or ContentElement objects representing the ending captures of the element. Defaults to None.
+        :param **kwargs: Additional keyword arguments to be passed to the parent class constructor.
+
+        :return: None
+        """
         if end is None:
             end = []
         if begin is None:
             begin = []
-        super().__init__(**kwargs)
-        self._begin_pending = begin
-        self._end_pending = end
-        self._begin_dispached: list[ContentElement] = []
-        self._end_dispached: list[ContentElement] = []
-        self._dispatched_begin: bool = False
-        self._dispatched_end: bool = False
+        super().__init__(*args, **kwargs)
+        self._begin_captures = begin
+        self._end_captures = end
 
     @property
     def _subelements(self) -> list[ContentElement]:
@@ -313,39 +472,63 @@ class ContentBlockElement(ContentElement):
 
     @property
     def begin(self) -> list[ContentElement]:
-        "Begin elements"
-        if self._begin_pending:
-            if not self._dispatched_begin:
-                self._begin_dispached = dispatch_list(self._begin_pending, parent=self)
-                self._dispatched_begin = True
-            return self._begin_dispached
-        else:
-            return []
+        """
+        Returns the list of begin elements.
+
+        If the elements have not been dispatched yet, this method will dispatch them before returning.
+
+        :return: The list of begin elements.
+        """
+        if not self._dispatched:
+            self._dispatch()
+        return self._begin
 
     @property
     def end(self) -> list[ContentElement]:
-        "End elements"
-        if self._end_pending:
-            if not self._dispatched_end:
-                self._end_dispached = dispatch_list(self._end_pending, parent=self)
-                self._dispatched_end = True
-            return self._end_dispached
-        else:
-            return []
+        """
+        Returns the end elements.
 
-    def to_dict(self, verbosity: int = -1, all_content: bool = False, **kwargs) -> dict:
-        out_dict = super().to_dict(verbosity=verbosity, all_content=all_content, **kwargs)
+        If the elements have not been dispatched yet, this method will dispatch them before returning.
+
+        :return: A list of end elements.
+        """
+        if not self._dispatched:
+            self._dispatch()
+        return self._end
+
+    def _dispatch(self, nested: bool = False):
+        if self._dispatched:
+            return
+        super()._dispatch(nested)
+        self._begin: list[ContentElement] = _dispatch_list(self._begin_captures, parent=self)
+        self._end: list[ContentElement] = _dispatch_list(self._end_captures, parent=self)
+        self._begin_captures, self._end_captures = [], []
+        if nested:
+            for item in self._begin:
+                item._dispatch(True)
+            for item in self._end:
+                item._dispatch(True)
+
+    def to_dict(self, depth: int = -1, all_content: bool = False, **kwargs) -> dict:
+        """
+        Converts the element to a dictionary representation.
+
+        :param depth: The depth of the conversion. Defaults to -1.
+        :param all_content: Whether to include all content. Defaults to False.
+        :param **kwargs: Additional keyword arguments.
+
+        :return: The dictionary representation of the element.
+        """
+        out_dict = super().to_dict(depth=depth, all_content=all_content, **kwargs)
         if self.begin:
             out_dict["begin"] = (
-                self._list_property_to_dict("begin", verbosity=verbosity - 1, **kwargs)
-                if verbosity
+                self._list_property_to_dict("begin", depth=depth - 1, **kwargs)
+                if depth
                 else self.begin
             )
         if self.end:
             out_dict["end"] = (
-                self._list_property_to_dict("end", verbosity=verbosity - 1, **kwargs)
-                if verbosity
-                else self.end
+                self._list_property_to_dict("end", depth=depth - 1, **kwargs) if depth else self.end
             )
 
         ordered_keys = [

textmate_grammar/language.py

@@ -1,10 +1,11 @@
 from pathlib import Path
 
 from .elements import Capture, ContentElement
-from .exceptions import IncompatibleFileType
-from .handler import POS, ContentHandler
-from .logger import LOGGER
 from .parser import GrammarParser, PatternsParser
+from .utils.cache import TextmateCache, init_cache
+from .utils.exceptions import IncompatibleFileType
+from .utils.handler import POS, ContentHandler
+from .utils.logger import LOGGER
 
 LANGUAGE_PARSERS = {}
 
@@ -27,6 +28,20 @@ class LanguageParser(PatternsParser):
     """The parser of a language grammar."""
 
     def __init__(self, grammar: dict, **kwargs):
+        """
+        Initialize a Language object.
+
+        :param grammar: The grammar definition for the language.
+        :param kwargs: Additional keyword arguments.
+
+        :ivar name: The name of the language.
+        :ivar uuid: The UUID of the language.
+        :ivar file_types: The file types associated with the language.
+        :ivar token: The scope name of the language.
+        :ivar repository: The repository of grammar rules for the language.
+        :ivar injections: The list of injection rules for the language.
+        :ivar _cache: The cache object for the language.
+        """
         super().__init__(grammar, key=grammar.get("name", "myLanguage"), language=self, **kwargs)
 
         self.name = grammar.get("name", "")
@@ -35,9 +50,10 @@ class LanguageParser(PatternsParser):
         self.token = grammar.get("scopeName", "myScope")
         self.repository = {}
         self.injections: list[dict] = []
+        self._cache: TextmateCache = init_cache()
 
-        # Initalize grammars in repository
-        for repo in gen_repositories(grammar):
+        # Initialize grammars in repository
+        for repo in _gen_repositories(grammar):
             for key, parser_grammar in repo.items():
                 self.repository[key] = GrammarParser.initialize(
                     parser_grammar, key=key, language=self
@@ -80,35 +96,62 @@ class LanguageParser(PatternsParser):
 
         super()._initialize_repository()
 
-    def parse_file(self, filePath: str | Path, **kwargs) -> Capture | ContentElement | None:
-        """Parses an entire file with the current grammar"""
-        if type(filePath) != Path:
-            filePath = Path(filePath)
+    def parse_file(self, filePath: str | Path, **kwargs) -> ContentElement | None:
+        """
+        Parses an entire file with the current grammar.
+
+        :param filePath: The path to the file to be parsed.
+        :param kwargs: Additional keyword arguments to be passed to the parser.
+        :return: The parsed element if successful, None otherwise.
+        """
+        if not isinstance(filePath, Path):
+            filePath = Path(filePath).resolve()
 
         if filePath.suffix.split(".")[-1] not in self.file_types:
             raise IncompatibleFileType(extensions=self.file_types)
 
-        handler = ContentHandler.from_path(filePath)
-        if handler.source == "":
-            return None
-
+        if self._cache.cache_valid(filePath):
+            element = self._cache.load(filePath)
+        else:
+            handler = ContentHandler.from_path(filePath)
+            if handler.source == "":
+                return None
+
+            # Configure logger
+            LOGGER.configure(self, height=len(handler.lines), width=max(handler.line_lengths))
+            element = self._parse_language(handler, **kwargs)  # type: ignore
+
+            if element is not None:
+                self._cache.save(filePath, element)
+        return element
+
+    def parse_string(self, input: str, **kwargs) -> ContentElement | None:
+        """
+        Parses an input string.
+
+        :param input: The input string to be parsed.
+        :param kwargs: Additional keyword arguments.
+        :return: The result of parsing the input string.
+        """
+        handler = ContentHandler(input)
         # Configure logger
         LOGGER.configure(self, height=len(handler.lines), width=max(handler.line_lengths))
 
-        return self._parse_language(handler, **kwargs)
+        element = self._parse_language(handler, **kwargs)
 
-    def parse_string(self, input: str, **kwargs):
-        """Parses an input string"""
-        handler = ContentHandler(input)
-        # Configure logger
-        LOGGER.configure(self, height=len(handler.lines), width=max(handler.line_lengths))
-        return self._parse_language(handler, **kwargs)
+        return element
 
-    def _parse_language(self, handler: ContentHandler, **kwargs) -> Capture | ContentElement | None:
+    def _parse_language(self, handler: ContentHandler, **kwargs) -> ContentElement | None:
         """Parses the current stream with the language scope."""
 
         parsed, elements, _ = self.parse(handler, (0, 0), **kwargs)
-        return elements[0] if parsed else None
+
+        if parsed:
+            element = elements[0]
+            element._dispatch(nested=True)  # type: ignore
+        else:
+            element = None
+        return element  # type: ignore
 
     def _parse(
         self, handler: ContentHandler, starting: POS, **kwargs
@@ -117,16 +160,16 @@ class LanguageParser(PatternsParser):
         return super()._parse(handler, starting, find_one=False, **kwargs)
 
 
-def gen_repositories(grammar, key="repository"):
+def _gen_repositories(grammar, key="repository"):
     """Recursively gets all repositories from a grammar dictionary"""
     if hasattr(grammar, "items"):
         for k, v in grammar.items():
             if k == key:
                 yield v
             if isinstance(v, dict):
-                for result in gen_repositories(v, key):
+                for result in _gen_repositories(v, key):
                     yield result
             elif isinstance(v, list):
                 for d in v:
-                    for result in gen_repositories(d, key):
+                    for result in _gen_repositories(d, key):
                         yield result