dbrownell-parserlib 0.4.0__tar.gz → 0.6.0__tar.gz

{dbrownell_parserlib-0.4.0 → dbrownell_parserlib-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: dbrownell-parserlib
-Version: 0.4.0
+Version: 0.6.0
 Summary: Functionality useful when creating parsers.
 Author: Dave Brownell
 Author-email: Dave Brownell <github@DavidBrownell.com>
@@ -10,6 +10,10 @@ Classifier: Operating System :: Microsoft :: Windows
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3.14
+Requires-Dist: antlr-denter>=1.3.1
+Requires-Dist: antlr4-python3-runtime>=4.13.2
+Requires-Dist: dbrownell-common>=0.16.8
+Requires-Dist: plum>=2.9.0
 Requires-Python: >=3.14
 Project-URL: Homepage, https://github.com/davidbrownell/dbrownell_ParserLib
 Project-URL: Documentation, https://github.com/davidbrownell/dbrownell_ParserLib

{dbrownell_parserlib-0.4.0 → dbrownell_parserlib-0.6.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dbrownell-parserlib"
-version = "0.4.0"
+version = "0.6.0"
 #          ^^^^^
 # Wheel names will be generated according to this value. Do not manually modify this value; instead
 # update it according to committed changes by running this command from the root of the repository:
@@ -13,7 +13,12 @@ authors = [
     { name = "Dave Brownell", email = "github@DavidBrownell.com" }
 ]
 requires-python = ">= 3.14"
-dependencies = []
+dependencies = [
+    "antlr-denter>=1.3.1",
+    "antlr4-python3-runtime>=4.13.2",
+    "dbrownell-common>=0.16.8",
+    "plum>=2.9.0",
+]
 classifiers = [
     "Operating System :: MacOS",
     "Operating System :: Microsoft :: Windows",
@@ -90,5 +95,8 @@ max-args = 10
 max-branches = 20
 max-returns = 20
 
+[tool.ty.rules]
+useless-overload-body = "ignore"  # This rule is not compatible with plum overloads
+
 [tool.uv.build-backend]
 module-name = "dbrownell_ParserLib"

{dbrownell_parserlib-0.4.0 → dbrownell_parserlib-0.6.0}/src/dbrownell_ParserLib/__init__.py

@@ -2,12 +2,18 @@
 
 from importlib.metadata import version
 
+from dbrownell_ParserLib.antlr import AntlrParser, AntlrParserException, BuildAntlrGrammar, CreateAntlrParser
 from dbrownell_ParserLib.errors import Error, PythonError
 from dbrownell_ParserLib.location import Location
 from dbrownell_ParserLib.region import Region
 
+
 # ----------------------------------------------------------------------
 __all__ = [
+    "AntlrParser",
+    "AntlrParserException",
+    "BuildAntlrGrammar",
+    "CreateAntlrParser",
     "Error",
     "Location",
     "PythonError",

dbrownell_parserlib-0.6.0/src/dbrownell_ParserLib/antlr/__init__.py

@@ -0,0 +1,12 @@
+# noqa: D104
+from dbrownell_ParserLib.antlr.build_antlr_grammar import BuildAntlrGrammar
+from dbrownell_ParserLib.antlr.antlr_parser import AntlrParser, AntlrParserException, CreateAntlrParser
+
+
+# ----------------------------------------------------------------------
+__all__ = [
+    "AntlrParser",
+    "AntlrParserException",
+    "BuildAntlrGrammar",
+    "CreateAntlrParser",
+]

dbrownell_parserlib-0.6.0/src/dbrownell_ParserLib/antlr/antlr_parser.py

@@ -0,0 +1,503 @@
+"""Contains functionality used when parsing with ANTLR parsers."""
+
+import os
+import threading
+
+from abc import ABC, abstractmethod
+from collections.abc import Callable  # noqa: TC003
+from dataclasses import dataclass, field
+from datetime import datetime, UTC
+from pathlib import Path
+from typing import cast, Protocol
+
+import antlr4
+
+from dbrownell_Common.ContextlibEx import ExitStack
+from dbrownell_Common import ExecuteTasks
+from dbrownell_Common.Streams.DoneManager import DoneManager  # noqa: TC002
+from plum import dispatch, overload
+
+from dbrownell_ParserLib.errors import Error, PythonError
+from dbrownell_ParserLib.region import Location, Region
+
+
+# ----------------------------------------------------------------------
+# |
+# |  Public Types
+# |
+# ----------------------------------------------------------------------
+class AntlrParserException(Exception):  # noqa: N818
+    """Exception raised by the ANTLR parser or a visitor processing the AST."""
+
+    # ----------------------------------------------------------------------
+    def __init__(self, error: Error) -> None:
+        super().__init__(str(error))
+
+        self.error = error
+
+
+# ----------------------------------------------------------------------
+class AntlrParser(ABC):
+    """Abstract base class for parsers created with `CreateAntlrParser`."""
+
+    # ----------------------------------------------------------------------
+    @overload
+    @abstractmethod
+    def __call__(
+        self,
+        dm: DoneManager,
+        workspaces: dict[
+            Path,  # workspace root
+            dict[
+                Path,  # Relative path to a translation unit
+                Callable[
+                    [], tuple[str, datetime]
+                ],  # Function that returns the text and last modified time of the translation unit
+            ],
+        ],
+        supported_file_extensions: set[str] | None,
+        *,
+        single_threaded: bool = False,
+        quiet: bool = False,
+    ) -> dict[
+        Path,  # workspace root
+        dict[
+            Path,  # Relative path to a translation unit
+            Error | antlr4.ParseTreeVisitor,
+        ],
+    ]:
+        """Parse files from different workspaces; includes (if any) are allowed from each of the workspaces."""
+
+    # ----------------------------------------------------------------------
+    @overload
+    @abstractmethod
+    def __call__(  # noqa: F811
+        self,
+        dm: DoneManager,
+        filenames: list[Path],
+        supported_file_extensions: set[str] | None,
+        *,
+        single_threaded: bool = False,
+        quiet: bool = False,
+    ) -> dict[Path, Error | antlr4.ParseTreeVisitor]:
+        """Parse a list of files that reside in the same directory and its descendants; includes (if any) are allowed from the same root directory."""
+
+    # ----------------------------------------------------------------------
+    @overload
+    @abstractmethod
+    def __call__(  # noqa: F811
+        self,
+        dm: DoneManager,
+        filename: Path,
+        supported_file_extensions: set[str] | None,
+        *,
+        single_threaded: bool = False,
+        quiet: bool = False,
+    ) -> dict[Path, Error | antlr4.ParseTreeVisitor]:
+        """Parse a single file; includes (if any) are allowed from the same directory as the file."""
+
+    # ----------------------------------------------------------------------
+    @dispatch
+    @abstractmethod
+    def __call__(self, *args, **kwargs):  # noqa: ANN204, F811
+        """Parse one or more translation units across one or more workspaces."""
+
+
+# ----------------------------------------------------------------------
+class CreateVisitorFuncType(Protocol):
+    """Signature for the function used to create the visitor for use with `CreateAntlrParser`."""
+
+    def __call__(  # noqa: D102
+        self,
+        filename: Path,
+        on_progress_func: Callable[[int], None],
+        *,
+        is_included_file: bool,
+    ) -> antlr4.ParseTreeVisitor: ...
+
+
+# ----------------------------------------------------------------------
+# |
+# |  Public Functions
+# |
+# ----------------------------------------------------------------------
+def CreateAntlrParser(
+    antlr_lexer: type[antlr4.Lexer],
+    antlr_parser: type[antlr4.Parser],
+    create_visitor_func: CreateVisitorFuncType,
+    get_ast_func: Callable[[antlr4.Parser], antlr4.ParserRuleContext],
+    custom_lexer_init_func: Callable[[antlr4.Lexer], None] | None = None,
+    custom_parser_init_func: Callable[[antlr4.Parser], None] | None = None,
+) -> AntlrParser:
+    """Create an `AntlrParser` that uses the given code generated by ANTLR."""
+
+    # ----------------------------------------------------------------------
+    def CreateLexer(input_stream: antlr4.InputStream) -> antlr4.Lexer:
+        lexer = antlr_lexer(input_stream)
+
+        if custom_lexer_init_func is not None:
+            custom_lexer_init_func(lexer)
+
+        return lexer
+
+    # ----------------------------------------------------------------------
+    def CreateParser(token_stream: antlr4.CommonTokenStream) -> antlr4.Parser:
+        parser = antlr_parser(token_stream)
+
+        if custom_parser_init_func is not None:
+            custom_parser_init_func(parser)
+
+        return parser
+
+    # ----------------------------------------------------------------------
+
+    return _AntlrParser(
+        CreateLexer,
+        CreateParser,
+        create_visitor_func,
+        get_ast_func,
+    )
+
+
+# ----------------------------------------------------------------------
+# |
+# |  Private Types
+# |
+# ----------------------------------------------------------------------
+class _AntlrParser(AntlrParser):
+    # ----------------------------------------------------------------------
+    # |  Public Methods
+    def __init__(
+        self,
+        create_lexer_func: Callable[[antlr4.InputStream], antlr4.Lexer],
+        create_parser_func: Callable[[antlr4.CommonTokenStream], antlr4.Parser],
+        create_visitor_func: CreateVisitorFuncType,
+        get_ast_func: Callable[[antlr4.Parser], antlr4.ParserRuleContext],
+    ) -> None:
+        self._create_lexer_func = create_lexer_func
+        self._create_parser_func = create_parser_func
+        self._create_visitor_func = create_visitor_func
+        self._get_ast_func = get_ast_func
+
+        self._execute_state: _AntlrParser._ExecuteState | None = None
+
+    # ----------------------------------------------------------------------
+    @overload
+    def __call__(  # noqa: C901
+        self,
+        dm: DoneManager,
+        workspaces: dict[Path, dict[Path, Callable[[], tuple[str, datetime]]]],
+        supported_file_extensions: set[str] | None,
+        *,
+        single_threaded: bool = False,
+        quiet: bool = False,
+    ) -> dict[Path, dict[Path, Error | antlr4.ParseTreeVisitor]]:
+        _ = supported_file_extensions  # Currently unused
+
+        # Create the workspace infos for all of the workspaces. At the same time, ensure that
+        # there aren't any roots that are nested within other roots.
+        all_roots: set[Path] = set()
+
+        # ----------------------------------------------------------------------
+        def CheckRoot(path: Path) -> None:
+            for root in all_roots:
+                if path == root:
+                    msg = f"The path '{path.as_posix()}' is duplicated as a root."
+                    raise ValueError(msg)
+
+                if path.is_relative_to(root) or root.is_relative_to(path):
+                    if path.is_relative_to(root):
+                        nested = path
+                        container = root
+                    elif root.is_relative_to(path):
+                        nested = root
+                        container = path
+                    else:
+                        assert False, (path, root)  # noqa: B011, PT015  # pragma: no cover
+
+                    msg = (
+                        f"The path '{nested.as_posix()}' is nested within the root '{container.as_posix()}'."
+                    )
+                    raise ValueError(msg)
+
+            all_roots.add(path)
+
+        # ----------------------------------------------------------------------
+
+        workspace_infos: dict[Path, _AntlrParser._WorkspaceInfo] = {}
+
+        for workspace_name in workspaces:
+            workspace_path = workspace_name.resolve()
+
+            CheckRoot(workspace_path)
+            workspace_infos[workspace_path] = _AntlrParser._WorkspaceInfo()
+
+        # ----------------------------------------------------------------------
+        def OnExit() -> None:
+            self._execute_state = None
+
+        # ----------------------------------------------------------------------
+
+        with ExitStack(OnExit):
+            with ExecuteTasks.YieldQueueExecutor(
+                dm,
+                "Parsing...",
+                quiet=quiet,
+                max_num_threads=1 if single_threaded else None,
+            ) as enqueue_func:
+                self._execute_state = _AntlrParser._ExecuteState(enqueue_func, workspace_infos)
+
+                if len(workspace_infos) == 1:
+                    create_description_func = lambda w, t: t  # noqa: ARG005, E731
+                else:
+                    create_description_func = lambda w, t: (w / t).as_posix()  # noqa: E731
+
+                for (workspace_path, workspace_info), translation_units in zip(
+                    workspace_infos.items(), workspaces.values(), strict=True
+                ):
+                    with workspace_info.results_lock:
+                        for translation_unit, content_func in translation_units.items():
+                            # ----------------------------------------------------------------------
+                            def ThisExecuteTask(
+                                on_simple_status_func: Callable[[str], None],  # noqa: ARG001
+                                workspace_path: Path = workspace_path,
+                                translation_unit: Path = translation_unit,
+                                content_func: Callable[[], tuple[str, datetime]] = content_func,
+                            ) -> tuple[int, ExecuteTasks.YieldQueueExecutorTypes.ExecuteFuncType]:
+                                return self._ExecuteTask(
+                                    workspace_path,
+                                    translation_unit,
+                                    content_func,
+                                    is_included_file=False,
+                                )
+
+                            # ----------------------------------------------------------------------
+
+                            workspace_info.results[translation_unit] = None
+
+                            enqueue_func(
+                                create_description_func(workspace_path, translation_unit),
+                                ThisExecuteTask,
+                            )
+
+            # Capture the results
+            results: dict[Path, dict[Path, Error | antlr4.ParseTreeVisitor]] = {}
+            errors: list[Error] = []
+
+            for workspace_path, workspace_info in workspace_infos.items():
+                these_results: dict[Path, Error | antlr4.ParseTreeVisitor] = {}
+
+                for translation_unit, translation_unit_result in workspace_info.results.items():
+                    if translation_unit_result is None:
+                        continue
+
+                    these_results[translation_unit] = translation_unit_result
+
+                    if isinstance(translation_unit_result, Error):
+                        errors.append(translation_unit_result)
+
+                results[workspace_path] = these_results
+
+            dm.result = len(errors)
+
+            return results
+
+    # ----------------------------------------------------------------------
+    @overload
+    def __call__(  # noqa: F811
+        self,
+        dm: DoneManager,
+        filenames: list[Path],
+        supported_file_extensions: set[str] | None,
+        *,
+        single_threaded: bool = False,
+        quiet: bool = False,
+    ) -> dict[Path, Error | antlr4.ParseTreeVisitor]:
+        if not filenames:
+            msg = "Invalid filenames"
+            raise ValueError(msg)
+
+        workspace_root: Path | None = None
+        translation_units: dict[Path, Callable[[], tuple[str, datetime]]] = {}
+
+        if len(filenames) == 1:
+            workspace_root = filenames[0].parent
+            translation_units[Path(filenames[0].name)] = lambda filename=filenames[0]: _LoadFile(filename)
+        else:
+            potential_root = os.path.commonpath(filenames)
+            if not potential_root:
+                msg = "Filenames must share a common root"
+                raise ValueError(msg)
+
+            workspace_root = Path(potential_root)
+
+            for filename in filenames:
+                translation_units[filename.relative_to(workspace_root)] = lambda filename=filename: _LoadFile(
+                    filename
+                )
+
+        assert workspace_root is not None
+        assert translation_units
+
+        result = self(
+            dm,
+            {workspace_root: translation_units},
+            supported_file_extensions,
+            single_threaded=single_threaded,
+            quiet=quiet,
+        )
+
+        assert len(result) == 1
+        return next(iter(result.values()))
+
+    # ----------------------------------------------------------------------
+    @overload
+    def __call__(  # noqa: F811
+        self,
+        dm: DoneManager,
+        filename: Path,
+        supported_file_extensions: set[str] | None,
+        *,
+        single_threaded: bool = False,
+        quiet: bool = False,
+    ) -> dict[Path, Error | antlr4.ParseTreeVisitor]:
+        return self(
+            dm,
+            [filename],
+            supported_file_extensions,
+            single_threaded=single_threaded,
+            quiet=quiet,
+        )
+
+    # ----------------------------------------------------------------------
+    @dispatch
+    def __call__(self, *args, **kwargs):  # noqa: ANN204, F811
+        pass  # pragma: no cover
+
+    # ----------------------------------------------------------------------
+    # |  Private Types
+    @dataclass(frozen=True)
+    class _WorkspaceInfo:
+        results: dict[Path, None | Error | antlr4.ParseTreeVisitor] = field(init=False, default_factory=dict)
+        results_lock: threading.Lock = field(init=False, default_factory=threading.Lock)
+
+    # ----------------------------------------------------------------------
+    @dataclass(frozen=True)
+    class _ExecuteState:
+        enqueue_func: ExecuteTasks.YieldQueueExecutorTypes.EnqueueFuncType
+        workspace_infos: dict[Path, _AntlrParser._WorkspaceInfo]
+
+    # ----------------------------------------------------------------------
+    # |  Private Methods
+    def _ExecuteTask(
+        self,
+        workspace_path: Path,
+        translation_unit: Path,
+        content_func: Callable[[], tuple[str, datetime]],
+        *,
+        is_included_file: bool,
+    ) -> tuple[int, ExecuteTasks.YieldQueueExecutorTypes.ExecuteFuncType]:
+        content, last_modified_time = content_func()
+
+        _ = last_modified_time  # Currently unused
+
+        # ----------------------------------------------------------------------
+        def Execute(
+            status: ExecuteTasks.Status,
+        ) -> str | None:
+            result: None | Error | antlr4.ParseTreeVisitor = None
+
+            # ----------------------------------------------------------------------
+            def OnExit() -> None:
+                assert result is not None
+                assert self._execute_state is not None
+
+                workspace_info = self._execute_state.workspace_infos[workspace_path]
+
+                with workspace_info.results_lock:
+                    assert workspace_info.results[translation_unit] is None
+                    workspace_info.results[translation_unit] = result
+
+            # ----------------------------------------------------------------------
+
+            with ExitStack(OnExit):
+                try:
+                    fullpath = workspace_path / translation_unit
+                    error_listener = _ErrorListener(fullpath)
+
+                    antlr_stream = antlr4.InputStream(content)
+
+                    lexer = self._create_lexer_func(antlr_stream)
+
+                    lexer.removeErrorListeners()
+                    lexer.addErrorListener(error_listener)
+
+                    tokens = antlr4.CommonTokenStream(lexer)
+                    tokens.fill()
+
+                    parser = self._create_parser_func(tokens)
+
+                    parser.removeErrorListeners()
+                    parser.addErrorListener(error_listener)
+
+                    visitor = self._create_visitor_func(
+                        fullpath,
+                        lambda line: cast(None, status.OnProgress(line, None)),
+                        is_included_file=is_included_file,
+                    )
+
+                    ast = self._get_ast_func(parser)
+                    ast.accept(visitor)
+
+                    result = visitor
+
+                except AntlrParserException as ex:
+                    result = ex.error
+                except Exception as ex:
+                    result = PythonError.Create(ex)
+
+        # ----------------------------------------------------------------------
+
+        return len(content.split("\n")), Execute
+
+
+# ----------------------------------------------------------------------
+class _ErrorListener(antlr4.DiagnosticErrorListener):
+    # ----------------------------------------------------------------------
+    def __init__(self, source: Path, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+
+        self._source = source
+
+    # ----------------------------------------------------------------------
+    def syntaxError(
+        self,
+        recognizer: antlr4.Parser,  # noqa: ARG002
+        offendingSymbol: antlr4.Token,  # noqa: ARG002, N803
+        line: int,
+        column: int,
+        msg: str,
+        e: antlr4.RecognitionException | None,
+    ) -> None:
+        location = Location(line, column + 1)
+        error = Error(msg, Region(self._source, location, location))
+        ex = AntlrParserException(error)
+
+        if e is None:
+            raise ex
+
+        raise ex from e
+
+
+# ----------------------------------------------------------------------
+# |
+# |  Private Functions
+# |
+# ----------------------------------------------------------------------
+def _LoadFile(filename: Path) -> tuple[str, datetime]:
+    """Load the contents of a file and return it along with its last modified time."""
+
+    text = filename.read_text(encoding="utf-8")
+    last_modified_time = datetime.fromtimestamp(filename.stat().st_mtime, tz=UTC)
+
+    return text, last_modified_time

dbrownell_parserlib-0.6.0/src/dbrownell_ParserLib/antlr/build_antlr_grammar/__init__.py

@@ -0,0 +1,45 @@
+# noqa: D104
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from dbrownell_Common import SubprocessEx
+
+if TYPE_CHECKING:
+    from dbrownell_Common.Streams.DoneManager import DoneManager
+
+
+# ----------------------------------------------------------------------
+def BuildAntlrGrammar(
+    dm: DoneManager,
+    antlr_grammar_filename: Path,
+    output_dir: Path,
+    *,
+    create_init_file: bool = True,
+    create_gitignore_file: bool = True,
+) -> None:
+    """Build the Antlr grammar; note that java must be available on the command line."""
+
+    with dm.Nested(f"Generating '{antlr_grammar_filename.name}'...") as generate_dm:
+        jar_filename = Path(__file__).parent / "antlr-4.13.2-complete.jar"
+        assert jar_filename.is_file(), jar_filename
+
+        command_line = f'java -jar "{jar_filename}" -Dlanguage=Python3 -o "{output_dir}" -no-listener -visitor "{antlr_grammar_filename}"'
+
+        generate_dm.WriteVerbose(f"Command line: {command_line}\n\n")
+
+        with generate_dm.YieldStream() as stream:
+            generate_dm.result = SubprocessEx.Stream(command_line, stream)
+            if generate_dm.result != 0:
+                return
+
+        if create_init_file:
+            init_filename = output_dir / "__init__.py"
+
+            if not init_filename.is_file():
+                init_filename.touch()
+
+        if create_gitignore_file:
+            gitignore_filename = output_dir / ".gitignore"
+
+            if not gitignore_filename.is_file():
+                gitignore_filename.write_text("*\n")