pykdebugparser 1.4.0__tar.gz → 1.6.0__tar.gz

{pykdebugparser-1.4.0 → pykdebugparser-1.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pykdebugparser
-Version: 1.4.0
+Version: 1.6.0
 Summary: Python parser for kdebug events
 Author: Matan Perelman
 Author-email: matan1008@gmail.com
@@ -39,7 +39,6 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: bytestruct>=0.1.0
-Requires-Dist: pygments
 Requires-Dist: click
 Requires-Dist: termcolor
 Provides-Extra: testing

pykdebugparser-1.6.0/pykdebugparser/highlight.py

@@ -0,0 +1,66 @@
+"""Syntax highlighting of trace lines, such as `open("/etc/hosts", O_RDONLY), fd: 3`.
+
+Lines are colored with the terminal's 16 basic colors, so they follow its theme:
+
+- the call or event name at the start, bold;
+- strings (paths, symbols, ...), green;
+- numbers, cyan;
+- constants (flags, enums, signals: `O_RDONLY`, `KERN_SUCCESS`, `SIGKILL`), yellow;
+- an error (`errno: ENOENT(2)`, a `KERN_*` failure), red;
+- the labels of results and fields (`fd:`, `count:`), dim.
+
+A trace is mostly the same calls with the same arguments, so the most recent lines are cached.
+"""
+import re
+from functools import lru_cache
+
+_RESET = '\x1b[0m'
+_BOLD = '\x1b[1m'
+_DIM = '\x1b[2m'
+_RED = '\x1b[31m'
+_GREEN = '\x1b[32m'
+_YELLOW = '\x1b[33m'
+_CYAN = '\x1b[36m'
+
+_TOKEN = re.compile(
+    r'(?P<string>"[^"]*")'
+    r'|(?P<number>-?0[xX][0-9a-fA-F]+|-?\d+)'
+    r'|(?P<label>[A-Za-z_][\w ]*?:)(?= |$)'
+    r'|(?P<name>[A-Za-z_]\w*)'
+)
+# A constant is all capitals, digits and underscores, with at least one capital.
+_CONSTANT = re.compile(r'[A-Z][A-Z0-9_]*')
+# `KERN_SUCCESS` is a result, not an error.
+_SUCCESS = {'KERN_SUCCESS'}
+
+
+@lru_cache(maxsize=1 << 14)
+def highlight(text: str) -> str:
+    """`text` with terminal colors (SGR escape sequences); the text itself is unchanged."""
+    out = []
+    pos = 0
+    error = False  # inside the value of an `errno:` result
+    first = True
+    for match in _TOKEN.finditer(text):
+        if match.start() > pos:
+            out.append(text[pos:match.start()])
+        pos = match.end()
+        kind = match.lastgroup
+        value = match.group()
+        if kind == 'string':
+            out.append(_GREEN + value + _RESET)
+        elif kind == 'number':
+            out.append((_RED if error else _CYAN) + value + _RESET)
+        elif kind == 'label':
+            error = value == 'errno:'
+            out.append((_RED if error else _DIM) + value + _RESET)
+        elif first:
+            out.append(_BOLD + value + _RESET)
+        elif _CONSTANT.fullmatch(value):
+            failed = error or (value.startswith('KERN_') and value not in _SUCCESS)
+            out.append((_RED if failed else _YELLOW) + value + _RESET)
+        else:
+            out.append(value)
+        first = False
+    out.append(text[pos:])
+    return ''.join(out)

{pykdebugparser-1.4.0 → pykdebugparser-1.6.0}/pykdebugparser/pykdebugparser.py

@@ -1,25 +1,53 @@
+import dataclasses
 from datetime import datetime
+from enum import Enum
 import io
 import math
 
-from pygments import highlight, lexers, formatters
 from termcolor import colored
 
 from pykdebugparser.callstacks_parser import CallstacksParser
+from pykdebugparser.highlight import highlight
 from pykdebugparser.kd_buf_parser import KdBufParser
 from pykdebugparser.kevent import DgbFuncQual
 from pykdebugparser.trace_codes import default_trace_codes
+from pykdebugparser.trace_handlers import UnknownEnumValue
+from pykdebugparser.trace_handlers.trace import TRACE_LOST_EVENTS
 from pykdebugparser.traces_parser import TracesParser
 from pykdebugparser.os_log_event import OsLogEvent
 
-c_lexer = lexers.CLexer()
-color_formatter = formatters.TerminalTrueColorFormatter(style='stata-dark')
 
 DBG_TRACE = 7
 DBG_FSYSTEM = 3
 DBG_BSD = 4
 
 
+def _fields(trace) -> dict:
+    """A trace's parsed values (its dataclass fields but the raw kevents), as JSON-compatible values."""
+    return {field.name: _jsonable(getattr(trace, field.name)) for field in dataclasses.fields(trace)
+            if field.name != 'ktraces'}
+
+
+def _jsonable(value):
+    if isinstance(value, (bool, str)) or value is None:
+        return value
+    if isinstance(value, UnknownEnumValue):
+        return int(value)
+    if isinstance(value, Enum):
+        return value.name
+    if isinstance(value, (int, float)):
+        return value
+    if isinstance(value, (bytes, bytearray)):
+        return value.hex()
+    if isinstance(value, (list, tuple)):
+        return [_jsonable(item) for item in value]
+    if isinstance(value, dict):
+        return {str(key): _jsonable(item) for key, item in value.items()}
+    if dataclasses.is_dataclass(value) and not isinstance(value, type):
+        return _fields(value)
+    return str(value)
+
+
 class PyKdebugParser:
     def __init__(self):
         self.filter_tid = None
@@ -93,11 +121,39 @@ class PyKdebugParser:
         if self.filter_process is not None:
             trace_generator = filter(self._filter_process_callback, trace_generator)
         if add_trace_class:
-            trace_generator = filter(lambda t: t.ktraces[0].eventid >> 24 != DBG_TRACE, trace_generator)
+            # Lost events stay: they tell the reader the trace has a gap.
+            trace_generator = filter(lambda t: t.ktraces[0].eventid >> 24 != DBG_TRACE
+                                     or t.ktraces[0].eventid == TRACE_LOST_EVENTS, trace_generator)
         if add_fs_class:
             trace_generator = filter(lambda t: t.ktraces[0].eventid >> 24 != DBG_FSYSTEM, trace_generator)
         return trace_generator
 
+    def dict_traces(self, kdebug: io.IOBase, trace_codes=None):
+        """The traces as JSON-compatible dicts (see `trace_dict`)."""
+        trace_codes_map = default_trace_codes() if trace_codes is None else trace_codes
+        return map(lambda t: self.trace_dict(t, trace_codes_map), self.traces(kdebug, trace_codes_map))
+
+    def trace_dict(self, trace, trace_codes=None) -> dict:
+        """
+        A trace as JSON-compatible data: when (`timestamp` in mach ticks, and `time` in ISO 8601 when the device's time
+        information is set), where (`tid`, `pid`, `process`), what (`event`, the trace code name of its first kevent,
+        and `fields`, the parsed values) and `text`, as `formatted_traces` prints it.
+        """
+        trace_codes_map = default_trace_codes() if trace_codes is None else trace_codes
+        first = trace.ktraces[0]
+        pid = self.threads_pids.get(first.tid)
+        posix_time = self._posix_time(first.timestamp)
+        return {
+            'timestamp': first.timestamp,
+            'time': None if posix_time is None else datetime.fromtimestamp(posix_time, tz=self.timezone).isoformat(),
+            'tid': first.tid,
+            'pid': pid,
+            'process': self.pids_names.get(pid, '') if pid is not None else '',
+            'event': trace_codes_map.get(first.eventid, hex(first.eventid)),
+            'fields': _fields(trace),
+            'text': str(trace),
+        }
+
     def formatted_traces(self, kdebug: io.IOBase, trace_codes=None):
         return map(lambda t: self._format_trace(t), self.traces(kdebug, trace_codes))
 
@@ -127,13 +183,19 @@ class PyKdebugParser:
         process_name = self.pids_names.get(pid, '')
         return self.filter_process == str(pid) or self.filter_process == process_name
 
-    def _format_timestamp(self, timestamp):
+    def _posix_time(self, timestamp):
+        """A kevent timestamp (mach ticks) as POSIX time, or None without the device's time information."""
         if None in (self.mach_absolute_time, self.numer, self.denom, self.usecs_since_epoch, self.timezone):
-            return str(timestamp) + ' '
+            return None
         offset_usec = (
                 ((timestamp - self.mach_absolute_time) * self.numer) / (self.denom * 1000)
         )
-        posix_time = (self.usecs_since_epoch + offset_usec) / 1000000
+        return (self.usecs_since_epoch + offset_usec) / 1000000
+
+    def _format_timestamp(self, timestamp):
+        posix_time = self._posix_time(timestamp)
+        if posix_time is None:
+            return str(timestamp) + ' '
         # Split to seconds and microseconds exactly like `datetime.fromtimestamp` does (round half to even), so
         # the date and time part can be formatted only once per second.
         fraction, seconds = math.modf(posix_time)
@@ -187,7 +249,7 @@ class PyKdebugParser:
             formatted_data += f'{self._format_process(tid):<34}'
         event_rep = str(trace)
         if self.color:
-            event_rep = highlight(event_rep, c_lexer, color_formatter).strip()
+            event_rep = highlight(event_rep)
 
         return formatted_data + event_rep
 

{pykdebugparser-1.4.0 → pykdebugparser-1.6.0}/pykdebugparser/trace_handlers/bsd.py

@@ -378,11 +378,18 @@ def serialize_stat_flags(flags: int) -> List[StatFlags]:
     return stat_flags
 
 
+# bsd/sys/errno.h: codes the kernel uses internally, which a syscall end event can carry.
+KERNEL_ERRNOS = {-1: 'ERESTART', -2: 'EJUSTRETURN'}
+
+
 def serialize_result(end_event, success_name='', fmt=lambda x: x) -> str:
     error_code = end_event.values[0]
     res = end_event.values[1]
+    signed = ctypes.c_int64(error_code).value
     if error_code in errno.errorcode:
         err = f'errno: {errno.errorcode[error_code]}({error_code})'
+    elif signed in KERNEL_ERRNOS:
+        err = f'errno: {KERNEL_ERRNOS[signed]}({signed})'
     else:
         err = f'errno: {error_code}'
     success = f'{success_name}: {fmt(res)}' if success_name else ''
@@ -2823,6 +2830,55 @@ class BscGeteuid:
         return f'geteuid(), uid: {self.uid}'
 
 
+# Darwin's signal numbers, which differ from Linux's.
+DARWIN_SIGNALS = (None, 'SIGHUP', 'SIGINT', 'SIGQUIT', 'SIGILL', 'SIGTRAP', 'SIGABRT', 'SIGEMT', 'SIGFPE', 'SIGKILL',
+                  'SIGBUS', 'SIGSEGV', 'SIGSYS', 'SIGPIPE', 'SIGALRM', 'SIGTERM', 'SIGURG', 'SIGSTOP', 'SIGTSTP',
+                  'SIGCONT', 'SIGCHLD', 'SIGTTIN', 'SIGTTOU', 'SIGIO', 'SIGXCPU', 'SIGXFSZ', 'SIGVTALRM', 'SIGPROF',
+                  'SIGWINCH', 'SIGINFO', 'SIGUSR1', 'SIGUSR2')
+
+
+def format_wait_status(status: int) -> str:
+    """Describe a wait status (`W_EXITCODE(code, signal)`) as its exit code or terminating signal."""
+    signal = status & 0x7f
+    if not signal:
+        return f'exit code: {(status >> 8) & 0xff}'
+    name = DARWIN_SIGNALS[signal] if signal < len(DARWIN_SIGNALS) else str(signal)
+    return f'signal: {name}' + (', core dumped' if status & 0x80 else '')
+
+
+@dataclass
+class ProcExit:
+    """A process exiting (`proc_exit`). An exec also ends the process image it replaces: xnu gives the new image a new
+    proc and exits the old one, with the same pid and status 0."""
+    ktraces: List
+    pid: int
+    status: int
+
+    def __str__(self):
+        return f'proc_exit, pid: {self.pid}, {format_wait_status(self.status)}'
+
+
+@dataclass
+class BscExit:
+    ktraces: List
+    rval: int
+
+    def __str__(self):
+        return f'exit({self.rval})'
+
+
+@dataclass
+class BscBsdthreadTerminate:
+    ktraces: List
+    stackaddr: int
+    freesize: int
+    port: int
+    sem: int
+
+    def __str__(self):
+        return f'bsdthread_terminate({hex(self.stackaddr)}, {hex(self.freesize)}, {self.port}, {self.sem})'
+
+
 @dataclass
 class BscWait4:
     ktraces: List
@@ -2970,6 +3026,35 @@ class BscSync:
         return 'sync()'
 
 
+@dataclass
+class BscPthreadKill:
+    ktraces: List
+    thread_port: int
+    sig: int
+    result: str
+
+    def __str__(self):
+        rep = f'pthread_kill({self.thread_port}, {self.sig})'
+        if self.result:
+            rep += f', {self.result}'
+        return rep
+
+
+@dataclass
+class BscSigreturn:
+    ktraces: List
+    uctx: int
+    infostyle: int
+    token: int
+    result: str
+
+    def __str__(self):
+        rep = f'sigreturn({hex(self.uctx)}, {self.infostyle}, {hex(self.token)})'
+        if self.result:
+            rep += f', {self.result}'
+        return rep
+
+
 @dataclass
 class BscKill:
     ktraces: List
@@ -5384,6 +5469,20 @@ def handle_geteuid(parser, events):
     return BscGeteuid(events, events[-1].values[1])
 
 
+def handle_proc_exit(parser, events):
+    args = events[0].values
+    return ProcExit(events, args[0], ctypes.c_int32(args[1]).value)
+
+
+def handle_exit(parser, events):
+    return BscExit(events, ctypes.c_int32(events[0].values[0]).value)
+
+
+def handle_bsdthread_terminate(parser, events):
+    args = events[0].values
+    return BscBsdthreadTerminate(events, args[0], args[1], args[2] & 0xffffffff, args[3] & 0xffffffff)
+
+
 def handle_wait4(parser, events, no_cancel=False):
     args = events[0].values
     return BscWait4(events, args[0], args[1], args[2], args[3], serialize_result(events[-1], 'pid'), no_cancel)
@@ -5438,6 +5537,16 @@ def handle_sync(parser, events):
     return BscSync(events)
 
 
+def handle_pthread_kill(parser, events):
+    args = events[0].values
+    return BscPthreadKill(events, args[0] & 0xffffffff, args[1], serialize_result(events[-1]))
+
+
+def handle_sigreturn(parser, events):
+    args = events[0].values
+    return BscSigreturn(events, args[0], args[1], args[2], serialize_result(events[-1]))
+
+
 def handle_kill(parser, events):
     return BscKill(events, events[0].values[0], events[0].values[1], serialize_result(events[-1]))
 
@@ -7133,6 +7242,8 @@ handlers = {
     'BSC_fchflags': handle_fchflags,
     'BSC_sync': handle_sync,
     'BSC_kill': handle_kill,
+    'BSC_pthread_kill': handle_pthread_kill,
+    'BSC_sigreturn': handle_sigreturn,
     'BSC_getppid': handle_getppid,
     'BSC_sys_dup': handle_sys_dup,
     'BSC_pipe': handle_pipe,
@@ -7472,6 +7583,9 @@ handlers = {
     'BSC_sys_crossarch_trap': handle_sys_crossarch_trap,
     'BSC_map_with_linking_np': handle_map_with_linking_np,
     'BSC_wait4': handle_wait4,
+    'proc_exit': handle_proc_exit,
+    'BSC_exit': handle_exit,
+    'BSC_bsdthread_terminate': handle_bsdthread_terminate,
     'BSC_mknodat': handle_mknodat,
     'BSC_mkfifoat': handle_mkfifoat,
     'BSC_freadlink': handle_freadlink,

{pykdebugparser-1.4.0 → pykdebugparser-1.6.0}/pykdebugparser/trace_handlers/dyld.py

@@ -18,6 +18,10 @@ def to_rtld_flags(flags: int):
     return [r for r in RtldFlag if r.value & flags]
 
 
+def _image(path: str) -> str:
+    return f', image: "{path}"' if path else ''
+
+
 @dataclass
 class DyldUuidMapA:
     ktraces: List
@@ -34,9 +38,11 @@ class DyldUuidMapB:
     ktraces: List
     fid_objno: int
     fid_generation: int
+    path: str = ''
 
     def __str__(self):
-        return f'DYLD_uuid_map_b, fid_objno: {self.fid_objno}, fid_generation: {hex(self.fid_generation)}'
+        rep = f'DYLD_uuid_map_b, fid_objno: {self.fid_objno}, fid_generation: {hex(self.fid_generation)}'
+        return rep + f', path: "{self.path}"' if self.path else rep
 
 
 @dataclass
@@ -55,9 +61,11 @@ class DyldUuidUnmapB:
     ktraces: List
     fid_objno: int
     fid_generation: int
+    path: str = ''
 
     def __str__(self):
-        return f'DYLD_uuid_unmap_b, fid_objno: {self.fid_objno}, fid_generation: {hex(self.fid_generation)}'
+        rep = f'DYLD_uuid_unmap_b, fid_objno: {self.fid_objno}, fid_generation: {hex(self.fid_generation)}'
+        return rep + f', path: "{self.path}"' if self.path else rep
 
 
 @dataclass
@@ -77,9 +85,11 @@ class DyldUuidSharedCacheB:
     ktraces: List
     fid_objno: int
     fid_generation: int
+    path: str = ''
 
     def __str__(self):
-        return f'DYLD_uuid_shared_cache_b, fid_objno: {self.fid_objno}, fid_generation: {hex(self.fid_generation)}'
+        rep = f'DYLD_uuid_shared_cache_b, fid_objno: {self.fid_objno}, fid_generation: {hex(self.fid_generation)}'
+        return rep + f', path: "{self.path}"' if self.path else rep
 
 
 @dataclass
@@ -106,9 +116,53 @@ class DyldFuncForAddImage:
     ktraces: List
     addr: int
     func: int
+    image: str = ''
+
+    def __str__(self):
+        return f'DBG_DYLD_TIMING_FUNC_FOR_ADD_IMAGE, addr: {hex(self.addr)}{_image(self.image)}, func: {hex(self.func)}'
+
+
+@dataclass
+class DyldFuncForRemoveImage:
+    ktraces: List
+    addr: int
+    func: int
+    image: str = ''
+
+    def __str__(self):
+        return (f'DBG_DYLD_TIMING_FUNC_FOR_REMOVE_IMAGE, addr: {hex(self.addr)}{_image(self.image)}, '
+                f'func: {hex(self.func)}')
+
+
+@dataclass
+class DyldStaticInitializer:
+    ktraces: List
+    mh: int
+    func: int
+    image: str = ''
+
+    def __str__(self):
+        return f'DBG_DYLD_TIMING_STATIC_INITIALIZER, mh: {hex(self.mh)}{_image(self.image)}, func: {hex(self.func)}'
+
+
+@dataclass
+class DyldObjcInit:
+    ktraces: List
+    mh: int
+    image: str = ''
+
+    def __str__(self):
+        return f'DBG_DYLD_TIMING_OBJC_INIT, mh: {hex(self.mh)}{_image(self.image)}'
+
+
+@dataclass
+class DyldTiming:
+    """A dyld phase traced without arguments (applying fixups, mapping ObjC images, ...)."""
+    ktraces: List
+    name: str
 
     def __str__(self):
-        return f'DBG_DYLD_TIMING_FUNC_FOR_ADD_IMAGE, addr: {hex(self.addr)}, func: {hex(self.func)}'
+        return self.name
 
 
 @dataclass
@@ -171,34 +225,55 @@ class Dladdr:
         return f'dladdr({hex(self.addr)}), ret: {self.ret}'
 
 
-def handle_uuid_map_a(parser, events):
+# dyld traces each image (un)load as an `_a` event with its UUID and load address, followed on the same thread by a
+# `_b` event with the id of its path string. The parser keeps the loaded images per process, so the events that only
+# carry a mach header (initializers, ObjC init, ...) can name their image.
+
+def _image_path(parser, events, mh):
+    return parser.dyld_images.get((parser.threads_pids.get(events[0].tid, -1), mh), '')
+
+
+def _map_a(parser, events, cls):
     args = events[0].values
-    return DyldUuidMapA(events, UUID(bytes=events[0].data[:16]), args[2], args[3])
+    parser.dyld_pending_load_addr[events[0].tid] = args[2]
+    return cls(events, UUID(bytes=events[0].data[:16]), args[2], args[3])
+
+
+def _map_b(parser, events, cls, loaded):
+    args = events[0].values
+    path = parser.global_strings.get(args[1], '') if args[1] else ''
+    load_addr = parser.dyld_pending_load_addr.pop(events[0].tid, None)
+    if load_addr is not None:
+        key = (parser.threads_pids.get(events[0].tid, -1), load_addr)
+        if loaded and path:
+            parser.dyld_images[key] = path
+        elif not loaded:
+            parser.dyld_images.pop(key, None)
+    return cls(events, args[0] & 0xffffffff, args[0] >> 32, path)
+
+
+def handle_uuid_map_a(parser, events):
+    return _map_a(parser, events, DyldUuidMapA)
 
 
 def handle_uuid_map_b(parser, events):
-    arg = events[0].values[0]
-    return DyldUuidMapB(events, arg & 0xffffffff, arg >> 32)
+    return _map_b(parser, events, DyldUuidMapB, loaded=True)
 
 
 def handle_uuid_unmap_a(parser, events):
-    args = events[0].values
-    return DyldUuidUnmapA(events, UUID(bytes=events[0].data[:16]), args[2], args[3])
+    return _map_a(parser, events, DyldUuidUnmapA)
 
 
 def handle_uuid_unmap_b(parser, events):
-    arg = events[0].values[0]
-    return DyldUuidUnmapB(events, arg & 0xffffffff, arg >> 32)
+    return _map_b(parser, events, DyldUuidUnmapB, loaded=False)
 
 
 def handle_uuid_shared_cache_a(parser, events):
-    args = events[0].values
-    return DyldUuidSharedCacheA(events, UUID(bytes=events[0].data[:16]), args[2], args[3])
+    return _map_a(parser, events, DyldUuidSharedCacheA)
 
 
 def handle_uuid_shared_cache_b(parser, events):
-    arg = events[0].values[0]
-    return DyldUuidSharedCacheB(events, arg & 0xffffffff, arg >> 32)
+    return _map_b(parser, events, DyldUuidSharedCacheB, loaded=True)
 
 
 def handle_timing_launch_executable(parser, events):
@@ -217,7 +292,26 @@ def handle_timing_map_image(parser, events):
 
 def handle_timing_func_for_add_image(parser, events):
     args = events[0].values
-    return DyldFuncForAddImage(events, args[1], args[2])
+    return DyldFuncForAddImage(events, args[1], args[2], _image_path(parser, events, args[1]))
+
+
+def handle_timing_func_for_remove_image(parser, events):
+    args = events[0].values
+    return DyldFuncForRemoveImage(events, args[1], args[2], _image_path(parser, events, args[1]))
+
+
+def handle_timing_static_initializer(parser, events):
+    args = events[0].values
+    return DyldStaticInitializer(events, args[1], args[2], _image_path(parser, events, args[1]))
+
+
+def handle_timing_objc_init(parser, events):
+    args = events[0].values
+    return DyldObjcInit(events, args[1], _image_path(parser, events, args[1]))
+
+
+def handle_timing(parser, events):
+    return DyldTiming(events, parser.trace_codes[events[0].eventid])
 
 
 def handle_timing_bootstrap_start(parser, events):
@@ -257,6 +351,14 @@ handlers = {
     'DBG_DYLD_TIMING_LAUNCH_EXECUTABLE': handle_timing_launch_executable,
     'DBG_DYLD_TIMING_MAP_IMAGE': handle_timing_map_image,
     'DBG_DYLD_TIMING_FUNC_FOR_ADD_IMAGE': handle_timing_func_for_add_image,
+    'DBG_DYLD_TIMING_FUNC_FOR_REMOVE_IMAGE': handle_timing_func_for_remove_image,
+    'DBG_DYLD_TIMING_STATIC_INITIALIZER': handle_timing_static_initializer,
+    'DBG_DYLD_TIMING_OBJC_INIT': handle_timing_objc_init,
+    'DBG_DYLD_TIMING_OBJC_MAP': handle_timing,
+    'DBG_DYLD_TIMING_APPLY_FIXUPS': handle_timing,
+    'DBG_DYLD_TIMING_ATTACH_CODESIGNATURE': handle_timing,
+    'DBG_DYLD_TIMING_BUILD_CLOSURE': handle_timing,
+    'DBG_DYLD_REMOTE_IMAGE_NOTIFIER': handle_timing,
     'DBG_DYLD_TIMING_BOOTSTRAP_START': handle_timing_bootstrap_start,
     'DBG_DYLD_TIMING_DLOPEN': handle_timing_dlopen,
     'DBG_DYLD_TIMING_DLOPEN_PREFLIGHT': handle_timing_dlopen_preflight,

pykdebugparser-1.6.0/pykdebugparser/trace_handlers/fsystem.py

@@ -0,0 +1,64 @@
+from dataclasses import dataclass
+from typing import List, Optional
+
+from pykdebugparser.trace_codes import default_trace_codes
+
+DBG_DKRW = 0x302
+# bsd/sys/kdebug.h: the code of a disk I/O event is these flags (`P_RdMetaAsync`, `P_WrDataDone`, ...).
+DKIO_DONE = 0x01
+
+
+@dataclass
+class VfsLookup:
+    ktraces: List
+    path: str
+    vnode_id: int
+
+    def __str__(self):
+        return f'lookup("{self.path}"), vnode id: {self.vnode_id}'
+
+
+@dataclass
+class DiskIo:
+    """A disk I/O, from its issue (device, block, size) to its completion (bytes not transferred, error)."""
+    ktraces: List
+    name: str
+    dev: Optional[int]
+    blkno: Optional[int]
+    size: Optional[int]
+    resid: int
+    error: int
+
+    def __str__(self):
+        rep = self.name
+        if self.dev is not None:
+            rep += f', dev: {self.dev >> 24},{self.dev & 0xffffff}, blkno: {hex(self.blkno)}, size: {self.size}'
+        return rep + f', resid: {self.resid}, error: {self.error}'
+
+
+def handle_vfs_lookup(parser, events):
+    node = parser.parse_vnode(events)
+    return VfsLookup(events, node.path, node.vnode_id)
+
+
+def handle_disk_io(parser, events):
+    # xnu traces the issue as (buf, dev, blkno, size) and the completion, often on another thread, as
+    # (buf, vnode, resid, error); the buf address pairs them. Print one line when the I/O completes.
+    event = events[0]
+    buf = event.values[0]
+    if not (event.eventid >> 2) & DKIO_DONE:
+        parser.pending_disk_io[buf] = event
+        return None
+    start = parser.pending_disk_io.pop(buf, None)
+    resid, error = event.values[2], event.values[3]
+    if start is None:
+        return DiskIo(events, parser.trace_codes[event.eventid].removesuffix('Done'), None, None, None, resid, error)
+    args = start.values
+    return DiskIo([start, event], parser.trace_codes[start.eventid], args[1] & 0xffffffff, args[2], args[3], resid,
+                  error)
+
+
+handlers = {
+    'VFS_LOOKUP': handle_vfs_lookup,
+}
+handlers.update({name: handle_disk_io for eventid, name in default_trace_codes().items() if eventid >> 16 == DBG_DKRW})

{pykdebugparser-1.4.0 → pykdebugparser-1.6.0}/pykdebugparser/trace_handlers/mach.py

@@ -2,10 +2,14 @@ import ctypes
 from dataclasses import dataclass
 from enum import Enum
 from functools import partial
-from typing import List
+from typing import List, Optional
 
+from pykdebugparser.trace_codes import default_trace_codes
 from pykdebugparser.trace_handlers import parse_enum
 
+# bsd/sys/kdebug.h: (DBG_MACH << 8) | DBG_MACH_EXCP_SYNC_ARM
+DBG_MACH_EXCP_SYNC_ARM = 0x103
+
 
 class AsynchronousSystemTrapsReason(Enum):
     AST_NONE = 0x00
@@ -59,10 +63,13 @@ class ExceptionSyndromeRegisterClass(Enum):
     ESR_EC_TRAP_SIMD_FP = 0x07
     ESR_EC_PTRAUTH_INSTR_TRAP = 0x09
     ESR_EC_MCRR_MRRC_CP14_TRAP = 0x0c
+    ESR_EC_BTI_FAIL = 0x0d
     ESR_EC_ILLEGAL_INSTR_SET = 0x0e
     ESR_EC_SVC_32 = 0x11
     ESR_EC_SVC_64 = 0x15
+    ESR_EC_HVC_64 = 0x16
     ESR_EC_MSR_TRAP = 0x18
+    ESR_EC_PAC_FAIL = 0x1c
     ESR_EC_IABORT_EL0 = 0x20
     ESR_EC_IABORT_EL1 = 0x21
     ESR_EC_PC_ALIGN = 0x22
@@ -281,6 +288,25 @@ class KernelUncategorizedExcArm:
         return f'KernelUncategorizedExcArm, class: {esr_class}, far: {hex(self.far)}, pc: {hex(self.pc)}'
 
 
+@dataclass
+class ArmException:
+    """An arm64 synchronous exception (`sleh.c`): its ESR, fault address and PC, for the classes without their own
+    trace class (breakpoints, `brk` traps, pointer authentication failures, alignment faults, ...)."""
+    ktraces: List
+    name: str
+    esr: int
+    far: int
+    pc: int
+
+    def __str__(self):
+        esr_class = self.esr >> 26
+        try:
+            esr_class = ExceptionSyndromeRegisterClass(esr_class).name
+        except ValueError:
+            pass
+        return f'{self.name}, class: {esr_class}, far: {hex(self.far)}, pc: {hex(self.pc)}'
+
+
 @dataclass
 class KernelDataAbortSameElExcArm:
     ktraces: List
@@ -594,6 +620,58 @@ class SemaphoreTimedwait:
         return f'semaphore_timedwait({hex(self.wait_name)}, {self.sec}, {self.nsec}), result: {self.result.name}'
 
 
+@dataclass
+class TaskForPid:
+    ktraces: List
+    target_tport: int
+    pid: int
+    t: int
+    result: KernReturn
+
+    def __str__(self):
+        return f'task_for_pid({self.target_tport}, {self.pid}, {hex(self.t)}), result: {self.result.name}'
+
+
+@dataclass
+class TaskNameForPid:
+    ktraces: List
+    target_tport: int
+    pid: int
+    tn: int
+    result: KernReturn
+
+    def __str__(self):
+        return f'task_name_for_pid({self.target_tport}, {self.pid}, {hex(self.tn)}), result: {self.result.name}'
+
+
+@dataclass
+class PidForTask:
+    ktraces: List
+    task: int
+    pid_addr: int
+    result: KernReturn
+
+    def __str__(self):
+        return f'pid_for_task({self.task}, {hex(self.pid_addr)}), result: {self.result.name}'
+
+
+@dataclass
+class TaskSuspension:
+    """A task or thread suspended (by `task_suspend`, a debugger, ...) or resumed (osfmk `DBG_MACH_SUSPENSION`)."""
+    ktraces: List
+    name: str
+    target: str
+    target_id: int
+    user_stop_count: Optional[int] = None
+    pidsuspended: Optional[int] = None
+
+    def __str__(self):
+        rep = f'{self.name}, {self.target}: {self.target_id}'
+        if self.user_stop_count is not None:
+            rep += f', user_stop_count: {self.user_stop_count}, pidsuspended: {bool(self.pidsuspended)}'
+        return rep
+
+
 @dataclass
 class MachPortGetAttributes:
     ktraces: List
@@ -1017,6 +1095,11 @@ def handle_user_instr_abort_lower_el_exc_arm(parser, events):
     return UserInstrAbortLowerElExcArm(events, *events[0].values[:3])
 
 
+def handle_arm_exception(parser, events):
+    args = events[0].values
+    return ArmException(events, parser.trace_codes[events[0].eventid], args[0], args[1], args[2])
+
+
 def handle_user_data_abort_lower_el_exc_arm(parser, events):
     args = events[0].values
     return UserDataAbortLowerElExcArm(events, args[0], args[1], args[2])
@@ -1057,6 +1140,41 @@ def handle_msc_mach_vm_map_trap(parser, events):
     return MachVmMap(events, *events[0].values)
 
 
+def handle_msc_task_for_pid(parser, events):
+    args = events[0].values
+    return TaskForPid(events, args[0] & 0xffffffff, ctypes.c_int32(args[1]).value, args[2],
+                      parse_enum(KernReturn, events[-1].values[0]))
+
+
+def handle_msc_task_name_for_pid(parser, events):
+    args = events[0].values
+    return TaskNameForPid(events, args[0] & 0xffffffff, ctypes.c_int32(args[1]).value, args[2],
+                          parse_enum(KernReturn, events[-1].values[0]))
+
+
+def handle_msc_pid_for_task(parser, events):
+    args = events[0].values
+    return PidForTask(events, args[0] & 0xffffffff, args[1], parse_enum(KernReturn, events[-1].values[0]))
+
+
+def handle_task_suspend(parser, events):
+    args = events[0].values
+    return TaskSuspension(events, 'task_suspend', 'pid', args[0], args[1], args[2])
+
+
+def handle_task_resume(parser, events):
+    return TaskSuspension(events, 'task_resume', 'pid', events[0].values[0])
+
+
+def handle_thread_suspend(parser, events):
+    args = events[0].values
+    return TaskSuspension(events, 'thread_suspend', 'tid', args[0], args[1], args[2])
+
+
+def handle_thread_resume(parser, events):
+    return TaskSuspension(events, 'thread_resume', 'tid', events[0].values[0])
+
+
 def handle_msc_mach_port_allocate_trap(parser, events):
     args = events[0].values
     return MachPortAllocate(events, args[0], parse_enum(MachPortRight, args[1]), args[2])
@@ -1306,6 +1424,13 @@ handlers = {
     'MSC_semaphore_signal_trap': handle_msc_semaphore_signal_trap,
     'MSC_semaphore_wait_trap': handle_msc_semaphore_wait_trap,
     'MSC_semaphore_timedwait_trap': handle_msc_semaphore_timedwait_trap,
+    'MSC_task_for_pid': handle_msc_task_for_pid,
+    'MSC_task_name_for_pid': handle_msc_task_name_for_pid,
+    'MSC_pid_for_task': handle_msc_pid_for_task,
+    'MACH_task_suspend': handle_task_suspend,
+    'MACH_task_resume': handle_task_resume,
+    'MACH_thread_suspend': handle_thread_suspend,
+    'MACH_thread_resume': handle_thread_resume,
     'MSC_mach_port_get_attributes_trap': handle_msc_mach_port_get_attributes_trap,
     'MSC_mach_port_guard_trap': handle_msc_mach_port_guard_trap,
     'MSC_mach_port_unguard_trap': handle_msc_mach_port_unguard_trap,
@@ -1341,3 +1466,6 @@ handlers = {
     'SCHED_CLUTCH_CPU_THREAD_SELECT': handle_sched_clutch_cpu_thread_select,
     'SCHED_CLUTCH_TG_BUCKET_PRI': handle_sched_clutch_tg_bucket_pri,
 }
+# Every other arm64 synchronous exception class shares the same arguments.
+handlers.update({name: handle_arm_exception for eventid, name in default_trace_codes().items()
+                 if eventid >> 16 == DBG_MACH_EXCP_SYNC_ARM and name.endswith('_Exc_ARM') and name not in handlers})

{pykdebugparser-1.4.0 → pykdebugparser-1.6.0}/pykdebugparser/trace_handlers/trace.py

@@ -189,6 +189,23 @@ def handle_trace_string_threadname_prev(parser, events):
     return event
 
 
+# bsd/sys/kdebug.h
+TRACE_LOST_EVENTS = 0x7020008
+
+
+@dataclass
+class TraceLostEvents:
+    """The kernel's trace buffer filled up and dropped events before this point."""
+    ktraces: List
+
+    def __str__(self):
+        return 'TRACE_LOST_EVENTS: the kernel dropped trace events here'
+
+
+def handle_trace_lost_events(parser, events):
+    return TraceLostEvents(events)
+
+
 handlers = {
     'TRACE_DATA_NEWTHREAD': handle_trace_data_newthread,
     'TRACE_DATA_EXEC': handle_trace_data_exec,
@@ -200,4 +217,5 @@ handlers = {
     'TRACE_STRING_PROC_EXIT': handle_trace_string_proc_exit,
     'TRACE_STRING_THREADNAME': handle_trace_string_threadname,
     'TRACE_STRING_THREADNAME_PREV': handle_trace_string_threadname_prev,
+    'TRACE_LOST_EVENTS': handle_trace_lost_events,
 }

{pykdebugparser-1.4.0 → pykdebugparser-1.6.0}/pykdebugparser/traces_parser.py

@@ -29,6 +29,9 @@ class RawTrace:
         return rep
 
 
+NO_RETURN_CALLS = ('BSC_exit', 'BSC_bsdthread_terminate')
+
+
 class TracesParser:
     def __init__(self, trace_codes_map, threads_pids, pids_names):
         self.trace_codes = trace_codes_map
@@ -46,6 +49,13 @@ class TracesParser:
         }
         self.last_data_newthread = None
         self.last_data_exec = None
+        # Calls that never return, parsed from their start event.
+        self.no_return_eventids = {eventid for eventid, name in trace_codes_map.items() if name in NO_RETURN_CALLS}
+        # Disk I/Os issued and not completed yet, by their buf address.
+        self.pending_disk_io = {}
+        # Images dyld loaded, by (pid, load address), and the load address of each thread's last `DYLD_uuid_*_a` event.
+        self.dyld_images = {}
+        self.dyld_pending_load_addr = {}
         # Event ids to show as `RawTrace` when they have no handler.
         self.raw_eventids = set()
         self.handlers = {}
@@ -109,6 +119,9 @@ class TracesParser:
         return list(self.vnode_generator([e for e in events if self.trace_codes.get(e.eventid) == 'VFS_LOOKUP']))
 
     def _feed_start_event(self, event, state):
+        if event.eventid in self.no_return_eventids:
+            # The thread or process ends inside the call, so its end is never traced.
+            return self._feed_single_event(event, state)
         tid_state = state.get(event.tid)
         if tid_state is None:
             # New tid

{pykdebugparser-1.4.0 → pykdebugparser-1.6.0}/pykdebugparser.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pykdebugparser
-Version: 1.4.0
+Version: 1.6.0
 Summary: Python parser for kdebug events
 Author: Matan Perelman
 Author-email: matan1008@gmail.com
@@ -39,7 +39,6 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: bytestruct>=0.1.0
-Requires-Dist: pygments
 Requires-Dist: click
 Requires-Dist: termcolor
 Provides-Extra: testing

{pykdebugparser-1.4.0 → pykdebugparser-1.6.0}/pykdebugparser.egg-info/SOURCES.txt

@@ -5,6 +5,7 @@ setup.cfg
 pykdebugparser/__init__.py
 pykdebugparser/__main__.py
 pykdebugparser/callstacks_parser.py
+pykdebugparser/highlight.py
 pykdebugparser/kd_buf_parser.py
 pykdebugparser/kevent.py
 pykdebugparser/os_log_event.py
@@ -26,6 +27,7 @@ pykdebugparser/trace_handlers/mach.py
 pykdebugparser/trace_handlers/perf.py
 pykdebugparser/trace_handlers/trace.py
 pykdebugparser/trace_handlers/turnstile.py
+tests/test_highlight.py
 tests/test_kd_buf_parser.py
 tests/test_kevent.py
 tests/test_os_log_event.py

{pykdebugparser-1.4.0 → pykdebugparser-1.6.0}/pykdebugparser.egg-info/requires.txt

@@ -1,5 +1,4 @@
 bytestruct>=0.1.0
-pygments
 click
 termcolor
 

{pykdebugparser-1.4.0 → pykdebugparser-1.6.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pykdebugparser"
-version = "1.4.0"
+version = "1.6.0"
 description = "Python parser for kdebug events"
 readme.file = "README.md"
 readme.content-type = "text/markdown"
@@ -25,7 +25,6 @@ classifiers = [
 ]
 dependencies = [
     "bytestruct>=0.1.0",
-    "pygments",
     "click",
     "termcolor",
 ]

pykdebugparser-1.6.0/tests/test_highlight.py

@@ -0,0 +1,52 @@
+import re
+
+import pytest
+
+from pykdebugparser.highlight import highlight
+
+_SGR = re.compile(r'\x1b\[[0-9;]*m')
+
+BOLD, DIM, RESET = '\x1b[1m', '\x1b[2m', '\x1b[0m'
+RED, GREEN, YELLOW, CYAN = '\x1b[31m', '\x1b[32m', '\x1b[33m', '\x1b[36m'
+
+
+def _colored(color, text):
+    return color + text + RESET
+
+
+@pytest.mark.parametrize('line, expected', [
+    ('open("/etc/hosts", O_RDONLY | O_CLOEXEC), fd: 3',
+     f'{_colored(BOLD, "open")}({_colored(GREEN, chr(34) + "/etc/hosts" + chr(34))}, {_colored(YELLOW, "O_RDONLY")} | '
+     f'{_colored(YELLOW, "O_CLOEXEC")}), {_colored(DIM, "fd:")} {_colored(CYAN, "3")}'),
+    ('stat64("/x"), errno: ENOENT(2)',
+     f'{_colored(BOLD, "stat64")}({_colored(GREEN, chr(34) + "/x" + chr(34))}), {_colored(RED, "errno:")} '
+     f'{_colored(RED, "ENOENT")}({_colored(RED, "2")})'),
+    ('task_for_pid(515, 97997, 0x16bd572fc), result: KERN_SUCCESS',
+     f'{_colored(BOLD, "task_for_pid")}({_colored(CYAN, "515")}, {_colored(CYAN, "97997")}, '
+     f'{_colored(CYAN, "0x16bd572fc")}), {_colored(DIM, "result:")} {_colored(YELLOW, "KERN_SUCCESS")}'),
+    ('mach_port_mod_refs(1, 0x0, MACH_PORT_RIGHT_SEND, 1), result: KERN_INVALID_RIGHT',
+     f'{_colored(BOLD, "mach_port_mod_refs")}({_colored(CYAN, "1")}, {_colored(CYAN, "0x0")}, '
+     f'{_colored(YELLOW, "MACH_PORT_RIGHT_SEND")}, {_colored(CYAN, "1")}), {_colored(DIM, "result:")} '
+     f'{_colored(RED, "KERN_INVALID_RIGHT")}'),
+    ('lookup("private/var/db"), vnode id: 42',
+     f'{_colored(BOLD, "lookup")}({_colored(GREEN, chr(34) + "private/var/db" + chr(34))}), '
+     f'{_colored(DIM, "vnode id:")} {_colored(CYAN, "42")}'),
+    ('sigreturn(0x1, 30, 0x2), errno: EJUSTRETURN(-2)',
+     f'{_colored(BOLD, "sigreturn")}({_colored(CYAN, "0x1")}, {_colored(CYAN, "30")}, {_colored(CYAN, "0x2")}), '
+     f'{_colored(RED, "errno:")} {_colored(RED, "EJUSTRETURN")}({_colored(RED, "-2")})'),
+    ('DBG_DYLD_TIMING_OBJC_MAP', _colored(BOLD, 'DBG_DYLD_TIMING_OBJC_MAP')),
+])
+def test_highlight(line, expected):
+    assert highlight(line) == expected
+
+
+@pytest.mark.parametrize('line', [
+    'open("/etc/hosts", O_RDONLY), fd: 3',
+    "ioctl(8, 0xc020699d /* _IOC(IOC_IN | IOC_OUT, 'i', 157, 32) */, 0x16d3cbea0)",
+    'TRACE_LOST_EVENTS: the kernel dropped trace events here',
+    'stat64("/a b/{c}: d", 0x1), count: 0x10',
+    'unterminated "string',
+    '',
+])
+def test_highlight_keeps_the_text(line):
+    assert _SGR.sub('', highlight(line)) == line

{pykdebugparser-1.4.0 → pykdebugparser-1.6.0}/tests/test_pykdebugparser.py

@@ -1,4 +1,5 @@
 from datetime import datetime, timedelta, timezone
+import json
 from io import BytesIO
 import struct
 
@@ -164,3 +165,82 @@ def test_traces_filter_eventid_without_handler():
     assert len(traces) == 2
     assert traces[0].endswith('MACH_SCHED_THREAD_SELECT(0x0, 0x0, 0x0, 0x0), end: (0x1d3, 0x0, 0x0, 0x4)')
     assert traces[1].endswith('MACH_PSET_AVG_EXEC_TIME(0x0, 0x3, 0x0, 0x0)')
+
+
+def test_traces_keep_lost_events_when_filtered():
+    # The parser adds the TRACE class to filtered traces for its own bookkeeping and hides it again, except for lost
+    # events, which tell the reader the trace has a gap.
+    events = [
+        _kd_buf(1000, 0x7020008, (1, 0, 0, 0)),  # TRACE_LOST_EVENTS
+        _kd_buf(1001, 0x1400140, (0, 3, 0, 0)),  # MACH_PSET_AVG_EXEC_TIME
+    ]
+    events_buf = RAW_VERSION2_BYTES + b'\x00' * 0x11c + b''.join(events)
+    parser = _traces_parser()
+    parser.filter_class = [1]
+    traces = list(parser.formatted_traces(BytesIO(events_buf)))
+    assert len(traces) == 1
+    assert traces[0].endswith('TRACE_LOST_EVENTS: the kernel dropped trace events here')
+
+
+def test_dict_traces():
+    events = [
+        _kd_buf(1000, 0x40c0014 | 1, (0x16d2c2b80, 0, 0, 0)),  # open start
+        _kd_buf(2000, 0x40c0014 | 2, (2, 0, 0, 0)),  # open end: ENOENT
+    ]
+    events_buf = RAW_VERSION2_BYTES + b'\x00' * 0x11c + b''.join(events)
+    parser = _traces_parser()
+    (unknown_process,) = parser.dict_traces(BytesIO(events_buf))
+    assert (unknown_process['pid'], unknown_process['process']) == (None, '')
+    # The thread's process, once the trace has seen it.
+    (parsed,) = parser.traces(BytesIO(events_buf))
+    parser.threads_pids[100] = 42
+    parser.pids_names[42] = 'probe'
+    trace = parser.trace_dict(parsed)
+    assert trace == {
+        'timestamp': 1000,
+        'time': '1970-01-01T00:00:00.000001+00:00',
+        'tid': 100,
+        'pid': 42,
+        'process': 'probe',
+        'event': 'BSC_open',
+        'fields': trace['fields'],
+        'text': trace['text'],
+    }
+    assert trace['text'].startswith('open(') and trace['text'].endswith('errno: ENOENT(2)')
+    assert json.loads(json.dumps(trace)) == trace  # JSON-compatible: enums by name, flags as lists of names
+
+
+def test_trace_dict_values():
+    from enum import Enum
+    from dataclasses import dataclass
+    from uuid import UUID
+    from pykdebugparser.trace_handlers import UnknownEnumValue
+
+    class Color(Enum):
+        RED = 1
+
+    @dataclass
+    class Inner:
+        ktraces: list
+        value: int
+
+    @dataclass
+    class Trace:
+        ktraces: list
+        color: Color
+        unknown: UnknownEnumValue
+        flags: list
+        uuid: UUID
+        data: bytes
+        inner: Inner
+
+    event = Kevent(timestamp=5, data=b'\x00' * 32, values=(0, 0, 0, 0), tid=7, debugid=0x40c0015, eventid=0x40c0014,
+                   func_qualifier=1)
+    trace = Trace([event], Color.RED, UnknownEnumValue(9), [Color.RED], UUID(int=1), b'\x01\x02', Inner([event], 3))
+    parser = PyKdebugParser()
+    assert parser.trace_dict(trace)['fields'] == {
+        'color': 'RED', 'unknown': 9, 'flags': ['RED'], 'uuid': '00000000-0000-0000-0000-000000000001',
+        'data': '0102', 'inner': {'value': 3},
+    }
+    # Without the device's time information there is no wall-clock time.
+    assert parser.trace_dict(trace)['time'] is None

pykdebugparser-1.4.0/pykdebugparser/trace_handlers/fsystem.py

@@ -1,22 +0,0 @@
-from dataclasses import dataclass
-from typing import List
-
-
-@dataclass
-class VfsLookup:
-    ktraces: List
-    path: str
-    vnode_id: int
-
-    def __str__(self):
-        return f'lookup("{self.path}"), vnode id: {self.vnode_id}'
-
-
-def handle_vfs_lookup(parser, events):
-    node = parser.parse_vnode(events)
-    return VfsLookup(events, node.path, node.vnode_id)
-
-
-handlers = {
-    'VFS_LOOKUP': handle_vfs_lookup,
-}