coverage 7.6.10__cp312-cp312-musllinux_1_2_aarch64.whl → 7.12.0__cp312-cp312-musllinux_1_2_aarch64.whl

coverage/sysmon.py

@@ -1,12 +1,11 @@
 # Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
-# For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt
+# For details: https://github.com/coveragepy/coveragepy/blob/main/NOTICE.txt
 
 """Callback functions and support for sys.monitoring data collection."""
 
-# TODO: https://github.com/python/cpython/issues/111963#issuecomment-2386584080
-
 from __future__ import annotations
 
+import collections
 import functools
 import inspect
 import os
@@ -14,46 +13,49 @@ import os.path
 import sys
 import threading
 import traceback
-
 from dataclasses import dataclass
-from types import CodeType, FrameType
-from typing import (
-    Any,
-    Callable,
-    TYPE_CHECKING,
-    cast,
-)
+from types import CodeType
+from typing import Any, Callable, NewType, Optional, cast
 
+from coverage import env
+from coverage.bytecode import TBranchTrails, always_jumps, branch_trails
 from coverage.debug import short_filename, short_stack
+from coverage.exceptions import NoSource, NotPython
 from coverage.misc import isolate_module
+from coverage.parser import PythonParser
 from coverage.types import (
     AnyCallable,
-    TArc,
     TFileDisposition,
     TLineNo,
+    TOffset,
+    Tracer,
     TShouldStartContextFn,
     TShouldTraceFn,
     TTraceData,
     TTraceFileData,
-    Tracer,
     TWarnFn,
 )
 
+# Only needed for some of the commented-out logging:
+# from coverage.debug import ppformat
+
 os = isolate_module(os)
 
 # pylint: disable=unused-argument
 
-LOG = False
+# $set_env.py: COVERAGE_SYSMON_LOG - Log sys.monitoring activity
+LOG = bool(int(os.getenv("COVERAGE_SYSMON_LOG", 0)))
+
+# $set_env.py: COVERAGE_SYSMON_STATS - Collect sys.monitoring stats
+COLLECT_STATS = bool(int(os.getenv("COVERAGE_SYSMON_STATS", 0)))
 
 # This module will be imported in all versions of Python, but only used in 3.12+
 # It will be type-checked for 3.12, but not for earlier versions.
 sys_monitoring = getattr(sys, "monitoring", None)
 
-if TYPE_CHECKING:
-    assert sys_monitoring is not None
-    # I want to say this but it's not allowed:
-    #   MonitorReturn = Literal[sys.monitoring.DISABLE] | None
-    MonitorReturn = Any
+DISABLE_TYPE = NewType("DISABLE_TYPE", object)
+MonitorReturn = Optional[DISABLE_TYPE]
+DISABLE = cast(MonitorReturn, getattr(sys_monitoring, "DISABLE", None))
 
 
 if LOG:  # pragma: debugging
@@ -76,7 +78,10 @@ if LOG:  # pragma: debugging
     assert sys_monitoring is not None
 
     short_stack = functools.partial(
-        short_stack, full=True, short_filenames=True, frame_ids=True,
+        short_stack,
+        full=True,
+        short_filenames=True,
+        frame_ids=True,
     )
     seen_threads: set[int] = set()
 
@@ -98,8 +103,11 @@ if LOG:  # pragma: debugging
             # f"{root}-{pid}.out",
             # f"{root}-{pid}-{tslug}.out",
         ]:
-            with open(filename, "a") as f:
-                print(f"{pid}:{tslug}: {msg}", file=f, flush=True)
+            with open(filename, "a", encoding="utf-8") as f:
+                try:
+                    print(f"{pid}:{tslug}: {msg}", file=f, flush=True)
+                except UnicodeError:
+                    print(f"{pid}:{tslug}: {ascii(msg)}", file=f, flush=True)
 
     def arg_repr(arg: Any) -> str:
         """Make a customized repr for logged values."""
@@ -130,7 +138,8 @@ if LOG:  # pragma: debugging
                     return ret
                 except Exception as exc:
                     log(f"!!{exc.__class__.__name__}: {exc}")
-                    log("".join(traceback.format_exception(exc))) # pylint: disable=[no-value-for-parameter]
+                    if 1:
+                        log("".join(traceback.format_exception(exc)))
                     try:
                         assert sys_monitoring is not None
                         sys_monitoring.set_events(sys.monitoring.COVERAGE_ID, 0)
@@ -163,11 +172,22 @@ class CodeInfo:
 
     tracing: bool
     file_data: TTraceFileData | None
-    # TODO: what is byte_to_line for?
-    byte_to_line: dict[int, int] | None
+    byte_to_line: dict[TOffset, TLineNo] | None
 
+    # Keys are start instruction offsets for branches.
+    # Values are dicts:
+    #   {
+    #       (from_line, to_line): {offset, offset, ...},
+    #       (from_line, to_line): {offset, offset, ...},
+    #   }
+    branch_trails: TBranchTrails
 
-def bytes_to_lines(code: CodeType) -> dict[int, int]:
+    # Always-jumps are bytecode offsets that do no work but move
+    # to another offset.
+    always_jumps: dict[TOffset, TOffset]
+
+
+def bytes_to_lines(code: CodeType) -> dict[TOffset, TLineNo]:
     """Make a dict mapping byte code offsets to line numbers."""
     b2l = {}
     for bstart, bend, lineno in code.co_lines():
@@ -204,17 +224,20 @@ class SysMonitor(Tracer):
         # A list of code_objects, just to keep them alive so that id's are
         # useful as identity.
         self.code_objects: list[CodeType] = []
-        self.last_lines: dict[FrameType, int] = {}
-        # Map id(code_object) -> code_object
-        self.local_event_codes: dict[int, CodeType] = {}
+
+        # Map filename:__name__ -> set(id(code_object))
+        self.filename_code_ids: dict[str, set[int]] = collections.defaultdict(set)
+
         self.sysmon_on = False
         self.lock = threading.Lock()
 
-        self.stats = {
-            "starts": 0,
-        }
+        self.stats: dict[str, int] | None = None
+        if COLLECT_STATS:
+            self.stats = dict.fromkeys(
+                "starts start_tracing returns line_lines line_arcs branches branch_trails".split(),
+                0,
+            )
 
-        self.stopped = False
         self._activity = False
 
     def __repr__(self) -> str:
@@ -225,44 +248,59 @@ class SysMonitor(Tracer):
     @panopticon()
     def start(self) -> None:
         """Start this Tracer."""
-        self.stopped = False
-
-        assert sys_monitoring is not None
-        sys_monitoring.use_tool_id(self.myid, "coverage.py")
-        register = functools.partial(sys_monitoring.register_callback, self.myid)
-        events = sys_monitoring.events
-        if self.trace_arcs:
-            sys_monitoring.set_events(
-                self.myid,
-                events.PY_START | events.PY_UNWIND,
-            )
-            register(events.PY_START, self.sysmon_py_start)
-            register(events.PY_RESUME, self.sysmon_py_resume_arcs)
-            register(events.PY_RETURN, self.sysmon_py_return_arcs)
-            register(events.PY_UNWIND, self.sysmon_py_unwind_arcs)
-            register(events.LINE, self.sysmon_line_arcs)
-        else:
+        with self.lock:
+            assert sys_monitoring is not None
+            sys_monitoring.use_tool_id(self.myid, "coverage.py")
+            register = functools.partial(sys_monitoring.register_callback, self.myid)
+            events = sys.monitoring.events
+
             sys_monitoring.set_events(self.myid, events.PY_START)
             register(events.PY_START, self.sysmon_py_start)
-            register(events.LINE, self.sysmon_line_lines)
-        sys_monitoring.restart_events()
-        self.sysmon_on = True
+            if self.trace_arcs:
+                register(events.PY_RETURN, self.sysmon_py_return)
+                register(events.LINE, self.sysmon_line_arcs)
+                if env.PYBEHAVIOR.branch_right_left:
+                    register(
+                        events.BRANCH_RIGHT,  # type:ignore[attr-defined]
+                        self.sysmon_branch_either,
+                    )
+                    register(
+                        events.BRANCH_LEFT,  # type:ignore[attr-defined]
+                        self.sysmon_branch_either,
+                    )
+            else:
+                register(events.LINE, self.sysmon_line_lines)
+            sys_monitoring.restart_events()
+            self.sysmon_on = True
 
     @panopticon()
     def stop(self) -> None:
         """Stop this Tracer."""
-        if not self.sysmon_on:
-            # In forking situations, we might try to stop when we are not
-            # started.  Do nothing in that case.
-            return
-        assert sys_monitoring is not None
-        sys_monitoring.set_events(self.myid, 0)
         with self.lock:
+            if not self.sysmon_on:
+                # In forking situations, we might try to stop when we are not
+                # started.  Do nothing in that case.
+                return
+            assert sys_monitoring is not None
+            sys_monitoring.set_events(self.myid, 0)
             self.sysmon_on = False
-            for code in self.local_event_codes.values():
-                sys_monitoring.set_local_events(self.myid, code, 0)
-            self.local_event_codes = {}
-        sys_monitoring.free_tool_id(self.myid)
+            sys_monitoring.free_tool_id(self.myid)
+
+        if LOG:  # pragma: debugging
+            items = sorted(
+                self.filename_code_ids.items(),
+                key=lambda item: len(item[1]),
+                reverse=True,
+            )
+            code_objs = sum(len(code_ids) for _, code_ids in items)
+            dupes = code_objs - len(items)
+            if dupes:
+                log(f"==== Duplicate code objects: {dupes} duplicates, {code_objs} total")
+                for filename, code_ids in items:
+                    if len(code_ids) > 1:
+                        log(f"{len(code_ids):>5} objects: {filename}")
+            else:
+                log("==== Duplicate code objects: none")
 
     @panopticon()
     def post_fork(self) -> None:
@@ -279,30 +317,16 @@ class SysMonitor(Tracer):
 
     def get_stats(self) -> dict[str, int] | None:
         """Return a dictionary of statistics, or None."""
-        return None
-
-    # The number of frames in callers_frame takes @panopticon into account.
-    if LOG:
-
-        def callers_frame(self) -> FrameType:
-            """Get the frame of the Python code we're monitoring."""
-            return (
-                inspect.currentframe().f_back.f_back.f_back  # type: ignore[union-attr,return-value]
-            )
-
-    else:
-
-        def callers_frame(self) -> FrameType:
-            """Get the frame of the Python code we're monitoring."""
-            return inspect.currentframe().f_back.f_back  # type: ignore[union-attr,return-value]
+        return self.stats
 
     @panopticon("code", "@")
-    def sysmon_py_start(self, code: CodeType, instruction_offset: int) -> MonitorReturn:
+    def sysmon_py_start(self, code: CodeType, instruction_offset: TOffset) -> MonitorReturn:
         """Handle sys.monitoring.events.PY_START events."""
-        # Entering a new frame.  Decide if we should trace in this file.
         self._activity = True
-        self.stats["starts"] += 1
+        if self.stats is not None:
+            self.stats["starts"] += 1
 
+        # Entering a new frame.  Decide if we should trace in this file.
         code_info = self.code_infos.get(id(code))
         tracing_code: bool | None = None
         file_data: TTraceFileData | None = None
@@ -314,10 +338,12 @@ class SysMonitor(Tracer):
             filename = code.co_filename
             disp = self.should_trace_cache.get(filename)
             if disp is None:
-                frame = inspect.currentframe().f_back  # type: ignore[union-attr]
-                if LOG:
-                    # @panopticon adds a frame.
-                    frame = frame.f_back  # type: ignore[union-attr]
+                frame = inspect.currentframe()
+                if frame is not None:
+                    frame = inspect.currentframe().f_back  # type: ignore[union-attr]
+                    if LOG:  # pragma: debugging
+                        # @panopticon adds a frame.
+                        frame = frame.f_back  # type: ignore[union-attr]
                 disp = self.should_trace(filename, frame)  # type: ignore[arg-type]
                 self.should_trace_cache[filename] = disp
 
@@ -337,102 +363,151 @@ class SysMonitor(Tracer):
                 file_data = None
                 b2l = None
 
-            self.code_infos[id(code)] = CodeInfo(
+            code_info = CodeInfo(
                 tracing=tracing_code,
                 file_data=file_data,
                 byte_to_line=b2l,
+                branch_trails={},
+                always_jumps={},
             )
+            self.code_infos[id(code)] = code_info
             self.code_objects.append(code)
 
             if tracing_code:
+                if self.stats is not None:
+                    self.stats["start_tracing"] += 1
                 events = sys.monitoring.events
                 with self.lock:
                     if self.sysmon_on:
                         assert sys_monitoring is not None
-                        sys_monitoring.set_local_events(
-                            self.myid,
-                            code,
-                            events.PY_RETURN
-                            #
-                            | events.PY_RESUME
-                            # | events.PY_YIELD
-                            | events.LINE,
-                            # | events.BRANCH
-                            # | events.JUMP
-                        )
-                        self.local_event_codes[id(code)] = code
-
-        if tracing_code and self.trace_arcs:
-            frame = self.callers_frame()
-            self.last_lines[frame] = -code.co_firstlineno
-            return None
-        else:
-            return sys.monitoring.DISABLE
-
-    @panopticon("code", "@")
-    def sysmon_py_resume_arcs(
-        self, code: CodeType, instruction_offset: int,
-    ) -> MonitorReturn:
-        """Handle sys.monitoring.events.PY_RESUME events for branch coverage."""
-        frame = self.callers_frame()
-        self.last_lines[frame] = frame.f_lineno
+                        local_events = events.PY_RETURN | events.PY_RESUME | events.LINE
+                        if self.trace_arcs:
+                            assert env.PYBEHAVIOR.branch_right_left
+                            local_events |= (
+                                events.BRANCH_RIGHT  # type:ignore[attr-defined]
+                                | events.BRANCH_LEFT  # type:ignore[attr-defined]
+                            )
+                        sys_monitoring.set_local_events(self.myid, code, local_events)
+
+                        if LOG:  # pragma: debugging
+                            if code.co_filename not in {"<string>"}:
+                                self.filename_code_ids[f"{code.co_filename}:{code.co_name}"].add(
+                                    id(code)
+                                )
+
+        return DISABLE
 
     @panopticon("code", "@", None)
-    def sysmon_py_return_arcs(
-        self, code: CodeType, instruction_offset: int, retval: object,
+    def sysmon_py_return(
+        self,
+        code: CodeType,
+        instruction_offset: TOffset,
+        retval: object,
     ) -> MonitorReturn:
         """Handle sys.monitoring.events.PY_RETURN events for branch coverage."""
-        frame = self.callers_frame()
-        code_info = self.code_infos.get(id(code))
-        if code_info is not None and code_info.file_data is not None:
-            last_line = self.last_lines.get(frame)
-            if last_line is not None:
-                arc = (last_line, -code.co_firstlineno)
-                # log(f"adding {arc=}")
-                cast(set[TArc], code_info.file_data).add(arc)
-
-        # Leaving this function, no need for the frame any more.
-        self.last_lines.pop(frame, None)
-
-    @panopticon("code", "@", "exc")
-    def sysmon_py_unwind_arcs(
-        self, code: CodeType, instruction_offset: int, exception: BaseException,
-    ) -> MonitorReturn:
-        """Handle sys.monitoring.events.PY_UNWIND events for branch coverage."""
-        frame = self.callers_frame()
-        # Leaving this function.
-        last_line = self.last_lines.pop(frame, None)
-        if isinstance(exception, GeneratorExit):
-            # We don't want to count generator exits as arcs.
-            return
+        if self.stats is not None:
+            self.stats["returns"] += 1
         code_info = self.code_infos.get(id(code))
-        if code_info is not None and code_info.file_data is not None:
-            if last_line is not None:
-                arc = (last_line, -code.co_firstlineno)
-                # log(f"adding {arc=}")
-                cast(set[TArc], code_info.file_data).add(arc)
-
+        # code_info is not None and code_info.file_data is not None, since we
+        # wouldn't have enabled this event if they were.
+        last_line = code_info.byte_to_line.get(instruction_offset)  # type: ignore
+        if last_line is not None:
+            arc = (last_line, -code.co_firstlineno)
+            code_info.file_data.add(arc)  # type: ignore
+            # log(f"adding {arc=}")
+        return DISABLE
 
     @panopticon("code", "line")
-    def sysmon_line_lines(self, code: CodeType, line_number: int) -> MonitorReturn:
+    def sysmon_line_lines(self, code: CodeType, line_number: TLineNo) -> MonitorReturn:
         """Handle sys.monitoring.events.LINE events for line coverage."""
-        code_info = self.code_infos[id(code)]
-        if code_info.file_data is not None:
-            cast(set[TLineNo], code_info.file_data).add(line_number)
-            # log(f"adding {line_number=}")
-        return sys.monitoring.DISABLE
+        if self.stats is not None:
+            self.stats["line_lines"] += 1
+        code_info = self.code_infos.get(id(code))
+        # It should be true that code_info is not None and code_info.file_data
+        # is not None, since we wouldn't have enabled this event if they were.
+        # But somehow code_info can be None here, so we have to check.
+        if code_info is not None and code_info.file_data is not None:
+            code_info.file_data.add(line_number)  # type: ignore
+        # log(f"adding {line_number=}")
+        return DISABLE
 
     @panopticon("code", "line")
-    def sysmon_line_arcs(self, code: CodeType, line_number: int) -> MonitorReturn:
+    def sysmon_line_arcs(self, code: CodeType, line_number: TLineNo) -> MonitorReturn:
         """Handle sys.monitoring.events.LINE events for branch coverage."""
+        if self.stats is not None:
+            self.stats["line_arcs"] += 1
         code_info = self.code_infos[id(code)]
-        ret = None
-        if code_info.file_data is not None:
-            frame = self.callers_frame()
-            last_line = self.last_lines.get(frame)
-            if last_line is not None:
-                arc = (last_line, line_number)
-                cast(set[TArc], code_info.file_data).add(arc)
-            # log(f"adding {arc=}")
-            self.last_lines[frame] = line_number
-        return ret
+        # code_info is not None and code_info.file_data is not None, since we
+        # wouldn't have enabled this event if they were.
+        arc = (line_number, line_number)
+        code_info.file_data.add(arc)  # type: ignore
+        # log(f"adding {arc=}")
+        return DISABLE
+
+    @panopticon("code", "@", "@")
+    def sysmon_branch_either(
+        self, code: CodeType, instruction_offset: TOffset, destination_offset: TOffset
+    ) -> MonitorReturn:
+        """Handle BRANCH_RIGHT and BRANCH_LEFT events."""
+        if self.stats is not None:
+            self.stats["branches"] += 1
+        code_info = self.code_infos[id(code)]
+        # code_info is not None and code_info.file_data is not None, since we
+        # wouldn't have enabled this event if they were.
+        if not code_info.branch_trails:
+            if self.stats is not None:
+                self.stats["branch_trails"] += 1
+            multiline_map = get_multiline_map(code.co_filename)
+            code_info.branch_trails = branch_trails(code, multiline_map=multiline_map)
+            code_info.always_jumps = always_jumps(code)
+            # log(f"branch_trails for {code}:\n{ppformat(code_info.branch_trails)}")
+        added_arc = False
+        dest_info = code_info.branch_trails.get(instruction_offset)
+
+        # Re-map the destination offset through always-jumps to deal with NOP etc.
+        dests = {destination_offset}
+        while (dest := code_info.always_jumps.get(destination_offset)) is not None:
+            destination_offset = dest
+            dests.add(destination_offset)
+
+        # log(f"dest_info = {ppformat(dest_info)}")
+        if dest_info is not None:
+            for arc, offsets in dest_info.items():
+                if arc is None:
+                    continue
+                if dests & offsets:
+                    code_info.file_data.add(arc)  # type: ignore
+                    # log(f"adding {arc=}")
+                    added_arc = True
+                    break
+
+        if not added_arc:
+            # This could be an exception jumping from line to line.
+            assert code_info.byte_to_line is not None
+            l1 = code_info.byte_to_line.get(instruction_offset)
+            if l1 is not None:
+                l2 = code_info.byte_to_line.get(destination_offset)
+                if l2 is not None and l1 != l2:
+                    arc = (l1, l2)
+                    code_info.file_data.add(arc)  # type: ignore
+                    # log(f"adding unforeseen {arc=}")
+
+        return DISABLE
+
+
+@functools.lru_cache(maxsize=5)
+def get_multiline_map(filename: str) -> dict[TLineNo, TLineNo]:
+    """Get a PythonParser for the given filename, cached."""
+    try:
+        parser = PythonParser(filename=filename)
+        parser.parse_source()
+    except NotPython:
+        # The file was not Python. This can happen when the code object refers
+        # to an original non-Python source file, like a Jinja template.
+        # In that case, just return an empty map, which might lead to slightly
+        # wrong branch coverage, but we don't have any better option.
+        return {}
+    except NoSource:
+        # This can happen if open() in python.py fails.
+        return {}
+    return parser.multiline_map

coverage/templite.py

@@ -1,5 +1,5 @@
 # Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
-# For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt
+# For details: https://github.com/coveragepy/coveragepy/blob/main/NOTICE.txt
 
 """A simple Python template renderer, for a nano-subset of Django syntax.
 
@@ -13,19 +13,18 @@ http://aosabook.org/en/500L/a-template-engine.html
 from __future__ import annotations
 
 import re
-
-from typing import (
-    Any, Callable, NoReturn, cast,
-)
+from typing import Any, Callable, NoReturn, cast
 
 
 class TempliteSyntaxError(ValueError):
     """Raised when a template has a syntax error."""
+
     pass
 
 
 class TempliteValueError(ValueError):
     """Raised when an expression won't evaluate in a template."""
+
     pass
 
 
@@ -53,7 +52,7 @@ class CodeBuilder:
         self.code.append(section)
         return section
 
-    INDENT_STEP = 4      # PEP8 says so!
+    INDENT_STEP = 4  # PEP8 says so!
 
     def indent(self) -> None:
         """Increase the current indent for following lines."""
@@ -90,6 +89,10 @@ class Templite:
 
         {% if var %}...{% endif %}
 
+    if-else::
+
+        {% if var %}...{% else %}...{% endif %}
+
     Comments are within curly-hash markers::
 
         {# This will be ignored #}
@@ -117,6 +120,7 @@ class Templite:
         })
 
     """
+
     def __init__(self, text: str, *contexts: dict[str, Any]) -> None:
         """Construct a Templite with the given `text`.
 
@@ -163,7 +167,7 @@ class Templite:
         for token in tokens:
             if token.startswith("{"):
                 start, end = 2, -2
-                squash = (token[-3] == "-")
+                squash = (token[-3] == "-")  # fmt: skip
                 if squash:
                     end = -3
 
@@ -187,6 +191,14 @@ class Templite:
                         ops_stack.append("if")
                         code.add_line("if %s:" % self._expr_code(words[1]))
                         code.indent()
+                    elif words[0] == "else":
+                        if len(words) != 1:
+                            self._syntax_error("Don't understand else", token)
+                        if not ops_stack or ops_stack[-1] != "if":
+                            self._syntax_error("Mismatched else", token)
+                        code.dedent()
+                        code.add_line("else:")
+                        code.indent()
                     elif words[0] == "for":
                         # A loop: iterate over expression result.
                         if len(words) != 4 or words[2] != "in":