pykdebugparser 1.4.0__py2.py3-none-any.whl → 1.5.0__py2.py3-none-any.whl

pykdebugparser/highlight.py

@@ -0,0 +1,134 @@
+"""Syntax highlighting of trace lines, as pygments' C lexer and `stata-dark` true color formatter do it.
+
+pygments lexes each line through a general state machine, which formats about 19k lines/s: too slow for a live
+trace. Trace lines use a small part of C's syntax, so `highlight` tokenizes them with one regular expression that
+follows the same rules, and produces the same escape sequences. A line with anything outside that part (a label at its
+start, `;` or `{`, comments, character literals, string escapes, floating-point numbers, ...) goes to pygments, so the
+result is always pygments' own.
+"""
+import re
+from functools import lru_cache
+
+from pygments import highlight as pygments_highlight
+from pygments import lexers
+from pygments.formatters import TerminalTrueColorFormatter
+from pygments.token import Keyword, Name, Number, Operator, Punctuation, String, Whitespace
+
+c_lexer = lexers.CLexer()
+color_formatter = TerminalTrueColorFormatter(style='stata-dark')
+
+# pygments.lexers.c_cpp.CFamilyLexer's 'keywords', 'types' and builtins, the names it does not lex as `Name`.
+_KEYWORDS = {
+    'asm', 'auto', 'break', 'const', 'constexpr', 'continue', 'countof', 'default', 'defer', 'do', 'else', 'enum',
+    'extern', 'for', 'goto', 'if', 'register', 'restricted', 'return', 'sizeof', 'static', 'switch', 'typedef',
+    'typeof', 'typeof_unqual', 'volatile', 'while', 'thread_local', 'alignas', 'alignof', 'static_assert', '_Pragma',
+    'fortran',
+}
+_RESERVED = {'inline', '_inline', '__inline', 'naked', 'restrict', 'thread', '__int8', '__int16', '__int32',
+             '__int64', '__wchar_t', '__m128i', '__m128d', '__m128', '__m64'} | {
+    '__' + name for name in ('asm', 'based', 'except', 'stdcall', 'cdecl', 'fastcall', 'declspec', 'finally', 'try',
+                             'leave', 'w64', 'unaligned', 'raise', 'noop', 'identifier', 'forceinline', 'assume',
+                             'null')}
+_TYPES = {'bool', 'int', 'long', 'float', 'short', 'double', 'char', 'unsigned', 'signed', 'void', '_BitInt',
+          '__int128'}
+_BUILTINS = {'true', 'false', 'NULL', 'nullptr'}
+# Names that start another lexer state, or a string with an affix (`L"..."`): left to pygments.
+_STATEFUL = {'struct', 'union', 'case', 'L', 'u', 'U', 'u8'}
+
+_INT_SUFFIX = r'(?:[uU][lL]{0,2}|[lL]{1,2}[uU]?)?'
+_DEC = r'\d(?:\'?\d)*'
+_HEX = r'[0-9a-fA-F](?:\'?[0-9a-fA-F])*'
+# The alternatives are tried in the lexer's order at each token start. `unsupported` marks what pygments lexes
+# differently from this tokenizer (comments and floating-point numbers); characters no alternative matches (`'`, `\\`,
+# `#`, `$`, a string with an escape, ...) also send the line to pygments.
+_TOKEN = re.compile(
+    r'(?P<unsupported>/[/*]'
+    r'|0[xX](?:' + _HEX + r'\.' + _HEX + r'|\.' + _HEX + r'|' + _HEX + r')[pP]'
+    r'|-?(?:' + _DEC + r'\.' + _DEC + r'|\.' + _DEC + r'|' + _DEC + r')[eE][+-]?' + _DEC +
+    r'|-?(?:' + _DEC + r'\.|\.' + _DEC + r'|' + _DEC + r'[fFlL])'
+    r')'
+    r'|(?P<string>")(?P<content>[^\\"\n]*)(?P<close>")'
+    r'|(?P<hex>-?0[xX]' + _HEX + _INT_SUFFIX + r')'
+    r'|(?P<bin>-?0[bB][01](?:\'?[01])*' + _INT_SUFFIX + r')'
+    r'|(?P<oct>-?0(?:\'?[0-7])+' + _INT_SUFFIX + r')'
+    r'|(?P<int>-?' + _DEC + _INT_SUFFIX + r')'
+    r'|(?P<op>[~!%^&*+=|?:<>/-])'
+    r'|(?P<punct>[()\[\],.])'
+    r'|(?P<name>[a-zA-Z_]\w*)'
+    r'|(?P<ws>[^\S\n]+)'
+)
+# Whole-line rules of the lexer: a label at the start of a line, and functions (their rules need a `;` or `{`).
+_UNSUPPORTED = re.compile(r'^\s*[a-zA-Z_$]\w*\s*:(?!:)|[;{]')
+
+
+def _style(ttype):
+    # The formatter uses the style of the token type, or of its closest parent that has one.
+    while str(ttype) not in color_formatter.style_string and ttype.parent is not None:
+        ttype = ttype.parent
+    return color_formatter.style_string.get(str(ttype), ('', ''))
+
+
+_GROUPS = {'hex': _style(Number.Hex), 'bin': _style(Number.Bin), 'oct': _style(Number.Oct),
+           'int': _style(Number.Integer), 'op': _style(Operator), 'punct': _style(Punctuation)}
+_NAME_STYLES = {}
+for _names, _ttype in ((_KEYWORDS, Keyword), (_RESERVED, Keyword.Reserved), (_TYPES, Keyword.Type),
+                       (_BUILTINS, Name.Builtin)):
+    for _name in _names:
+        _NAME_STYLES[_name] = _style(_ttype)
+_NAME = _style(Name)
+_STRING = _style(String)
+# Whitespace only gets a foreground color, which a space does not show: it can stay in the style before it.
+_FG_RESET = '\x1b[39m'
+assert _style(Whitespace)[1] == _FG_RESET
+
+
+def _pygments(text: str) -> str:
+    return pygments_highlight(text, c_lexer, color_formatter).strip()
+
+
+# Traces repeat (the same calls with the same arguments): in a busy `--bsc` trace, 77% of the lines were among the
+# 16k most recent ones.
+@lru_cache(maxsize=1 << 14)
+def highlight(text: str) -> str:
+    """`text` colored as `pygments.highlight(text, CLexer(), TerminalTrueColorFormatter('stata-dark'))` shows it.
+
+    pygments starts and ends the escape sequence of every token, even between tokens of the same color, and colors
+    spaces. Here a sequence starts only where the color changes, which renders the same in about half the bytes (the
+    terminal is the bottleneck of a busy trace).
+    """
+    if _UNSUPPORTED.search(text):
+        return _pygments(text)
+    out = []
+    style = None  # the (on, off) in effect
+    pos = 0
+    for match in _TOKEN.finditer(text):
+        if match.start() != pos:
+            return _pygments(text)
+        pos = match.end()
+        kind = match.lastgroup
+        if kind == 'ws':
+            out.append(match.group())
+            continue
+        if kind == 'unsupported':
+            return _pygments(text)
+        if kind == 'close':
+            token_style = _STRING
+        elif kind == 'name':
+            name = match.group()
+            if name in _STATEFUL:
+                return _pygments(text)
+            token_style = _NAME_STYLES.get(name, _NAME)
+        else:
+            token_style = _GROUPS[kind]
+        if style is None or token_style[0] != style[0]:
+            # Ending a foreground color is not needed when the next one replaces it.
+            if style is not None and style[1] != _FG_RESET:
+                out.append(style[1])
+            out.append(token_style[0])
+            style = token_style
+        out.append(match.group())
+    if pos != len(text):
+        return _pygments(text)
+    if style is not None:
+        out.append(style[1])
+    return ''.join(out).strip()

pykdebugparser/pykdebugparser.py

@@ -2,18 +2,17 @@ from datetime import datetime
 import io
 import math
 
-from pygments import highlight, lexers, formatters
 from termcolor import colored
 
 from pykdebugparser.callstacks_parser import CallstacksParser
+from pykdebugparser.highlight import highlight
 from pykdebugparser.kd_buf_parser import KdBufParser
 from pykdebugparser.kevent import DgbFuncQual
 from pykdebugparser.trace_codes import default_trace_codes
+from pykdebugparser.trace_handlers.trace import TRACE_LOST_EVENTS
 from pykdebugparser.traces_parser import TracesParser
 from pykdebugparser.os_log_event import OsLogEvent
 
-c_lexer = lexers.CLexer()
-color_formatter = formatters.TerminalTrueColorFormatter(style='stata-dark')
 
 DBG_TRACE = 7
 DBG_FSYSTEM = 3
@@ -93,7 +92,9 @@ class PyKdebugParser:
         if self.filter_process is not None:
             trace_generator = filter(self._filter_process_callback, trace_generator)
         if add_trace_class:
-            trace_generator = filter(lambda t: t.ktraces[0].eventid >> 24 != DBG_TRACE, trace_generator)
+            # Lost events stay: they tell the reader the trace has a gap.
+            trace_generator = filter(lambda t: t.ktraces[0].eventid >> 24 != DBG_TRACE
+                                     or t.ktraces[0].eventid == TRACE_LOST_EVENTS, trace_generator)
         if add_fs_class:
             trace_generator = filter(lambda t: t.ktraces[0].eventid >> 24 != DBG_FSYSTEM, trace_generator)
         return trace_generator
@@ -187,7 +188,7 @@ class PyKdebugParser:
             formatted_data += f'{self._format_process(tid):<34}'
         event_rep = str(trace)
         if self.color:
-            event_rep = highlight(event_rep, c_lexer, color_formatter).strip()
+            event_rep = highlight(event_rep)
 
         return formatted_data + event_rep
 

pykdebugparser/trace_handlers/bsd.py

@@ -378,11 +378,18 @@ def serialize_stat_flags(flags: int) -> List[StatFlags]:
     return stat_flags
 
 
+# bsd/sys/errno.h: codes the kernel uses internally, which a syscall end event can carry.
+KERNEL_ERRNOS = {-1: 'ERESTART', -2: 'EJUSTRETURN'}
+
+
 def serialize_result(end_event, success_name='', fmt=lambda x: x) -> str:
     error_code = end_event.values[0]
     res = end_event.values[1]
+    signed = ctypes.c_int64(error_code).value
     if error_code in errno.errorcode:
         err = f'errno: {errno.errorcode[error_code]}({error_code})'
+    elif signed in KERNEL_ERRNOS:
+        err = f'errno: {KERNEL_ERRNOS[signed]}({signed})'
     else:
         err = f'errno: {error_code}'
     success = f'{success_name}: {fmt(res)}' if success_name else ''
@@ -2823,6 +2830,55 @@ class BscGeteuid:
         return f'geteuid(), uid: {self.uid}'
 
 
+# Darwin's signal numbers, which differ from Linux's.
+DARWIN_SIGNALS = (None, 'SIGHUP', 'SIGINT', 'SIGQUIT', 'SIGILL', 'SIGTRAP', 'SIGABRT', 'SIGEMT', 'SIGFPE', 'SIGKILL',
+                  'SIGBUS', 'SIGSEGV', 'SIGSYS', 'SIGPIPE', 'SIGALRM', 'SIGTERM', 'SIGURG', 'SIGSTOP', 'SIGTSTP',
+                  'SIGCONT', 'SIGCHLD', 'SIGTTIN', 'SIGTTOU', 'SIGIO', 'SIGXCPU', 'SIGXFSZ', 'SIGVTALRM', 'SIGPROF',
+                  'SIGWINCH', 'SIGINFO', 'SIGUSR1', 'SIGUSR2')
+
+
+def format_wait_status(status: int) -> str:
+    """Describe a wait status (`W_EXITCODE(code, signal)`) as its exit code or terminating signal."""
+    signal = status & 0x7f
+    if not signal:
+        return f'exit code: {(status >> 8) & 0xff}'
+    name = DARWIN_SIGNALS[signal] if signal < len(DARWIN_SIGNALS) else str(signal)
+    return f'signal: {name}' + (', core dumped' if status & 0x80 else '')
+
+
+@dataclass
+class ProcExit:
+    """A process exiting (`proc_exit`). An exec also ends the process image it replaces: xnu gives the new image a new
+    proc and exits the old one, with the same pid and status 0."""
+    ktraces: List
+    pid: int
+    status: int
+
+    def __str__(self):
+        return f'proc_exit, pid: {self.pid}, {format_wait_status(self.status)}'
+
+
+@dataclass
+class BscExit:
+    ktraces: List
+    rval: int
+
+    def __str__(self):
+        return f'exit({self.rval})'
+
+
+@dataclass
+class BscBsdthreadTerminate:
+    ktraces: List
+    stackaddr: int
+    freesize: int
+    port: int
+    sem: int
+
+    def __str__(self):
+        return f'bsdthread_terminate({hex(self.stackaddr)}, {hex(self.freesize)}, {self.port}, {self.sem})'
+
+
 @dataclass
 class BscWait4:
     ktraces: List
@@ -2970,6 +3026,35 @@ class BscSync:
         return 'sync()'
 
 
+@dataclass
+class BscPthreadKill:
+    ktraces: List
+    thread_port: int
+    sig: int
+    result: str
+
+    def __str__(self):
+        rep = f'pthread_kill({self.thread_port}, {self.sig})'
+        if self.result:
+            rep += f', {self.result}'
+        return rep
+
+
+@dataclass
+class BscSigreturn:
+    ktraces: List
+    uctx: int
+    infostyle: int
+    token: int
+    result: str
+
+    def __str__(self):
+        rep = f'sigreturn({hex(self.uctx)}, {self.infostyle}, {hex(self.token)})'
+        if self.result:
+            rep += f', {self.result}'
+        return rep
+
+
 @dataclass
 class BscKill:
     ktraces: List
@@ -5384,6 +5469,20 @@ def handle_geteuid(parser, events):
     return BscGeteuid(events, events[-1].values[1])
 
 
+def handle_proc_exit(parser, events):
+    args = events[0].values
+    return ProcExit(events, args[0], ctypes.c_int32(args[1]).value)
+
+
+def handle_exit(parser, events):
+    return BscExit(events, ctypes.c_int32(events[0].values[0]).value)
+
+
+def handle_bsdthread_terminate(parser, events):
+    args = events[0].values
+    return BscBsdthreadTerminate(events, args[0], args[1], args[2] & 0xffffffff, args[3] & 0xffffffff)
+
+
 def handle_wait4(parser, events, no_cancel=False):
     args = events[0].values
     return BscWait4(events, args[0], args[1], args[2], args[3], serialize_result(events[-1], 'pid'), no_cancel)
@@ -5438,6 +5537,16 @@ def handle_sync(parser, events):
     return BscSync(events)
 
 
+def handle_pthread_kill(parser, events):
+    args = events[0].values
+    return BscPthreadKill(events, args[0] & 0xffffffff, args[1], serialize_result(events[-1]))
+
+
+def handle_sigreturn(parser, events):
+    args = events[0].values
+    return BscSigreturn(events, args[0], args[1], args[2], serialize_result(events[-1]))
+
+
 def handle_kill(parser, events):
     return BscKill(events, events[0].values[0], events[0].values[1], serialize_result(events[-1]))
 
@@ -7133,6 +7242,8 @@ handlers = {
     'BSC_fchflags': handle_fchflags,
     'BSC_sync': handle_sync,
     'BSC_kill': handle_kill,
+    'BSC_pthread_kill': handle_pthread_kill,
+    'BSC_sigreturn': handle_sigreturn,
     'BSC_getppid': handle_getppid,
     'BSC_sys_dup': handle_sys_dup,
     'BSC_pipe': handle_pipe,
@@ -7472,6 +7583,9 @@ handlers = {
     'BSC_sys_crossarch_trap': handle_sys_crossarch_trap,
     'BSC_map_with_linking_np': handle_map_with_linking_np,
     'BSC_wait4': handle_wait4,
+    'proc_exit': handle_proc_exit,
+    'BSC_exit': handle_exit,
+    'BSC_bsdthread_terminate': handle_bsdthread_terminate,
     'BSC_mknodat': handle_mknodat,
     'BSC_mkfifoat': handle_mkfifoat,
     'BSC_freadlink': handle_freadlink,

pykdebugparser/trace_handlers/dyld.py

@@ -18,6 +18,10 @@ def to_rtld_flags(flags: int):
     return [r for r in RtldFlag if r.value & flags]
 
 
+def _image(path: str) -> str:
+    return f', image: "{path}"' if path else ''
+
+
 @dataclass
 class DyldUuidMapA:
     ktraces: List
@@ -34,9 +38,11 @@ class DyldUuidMapB:
     ktraces: List
     fid_objno: int
     fid_generation: int
+    path: str = ''
 
     def __str__(self):
-        return f'DYLD_uuid_map_b, fid_objno: {self.fid_objno}, fid_generation: {hex(self.fid_generation)}'
+        rep = f'DYLD_uuid_map_b, fid_objno: {self.fid_objno}, fid_generation: {hex(self.fid_generation)}'
+        return rep + f', path: "{self.path}"' if self.path else rep
 
 
 @dataclass
@@ -55,9 +61,11 @@ class DyldUuidUnmapB:
     ktraces: List
     fid_objno: int
     fid_generation: int
+    path: str = ''
 
     def __str__(self):
-        return f'DYLD_uuid_unmap_b, fid_objno: {self.fid_objno}, fid_generation: {hex(self.fid_generation)}'
+        rep = f'DYLD_uuid_unmap_b, fid_objno: {self.fid_objno}, fid_generation: {hex(self.fid_generation)}'
+        return rep + f', path: "{self.path}"' if self.path else rep
 
 
 @dataclass
@@ -77,9 +85,11 @@ class DyldUuidSharedCacheB:
     ktraces: List
     fid_objno: int
     fid_generation: int
+    path: str = ''
 
     def __str__(self):
-        return f'DYLD_uuid_shared_cache_b, fid_objno: {self.fid_objno}, fid_generation: {hex(self.fid_generation)}'
+        rep = f'DYLD_uuid_shared_cache_b, fid_objno: {self.fid_objno}, fid_generation: {hex(self.fid_generation)}'
+        return rep + f', path: "{self.path}"' if self.path else rep
 
 
 @dataclass
@@ -106,9 +116,53 @@ class DyldFuncForAddImage:
     ktraces: List
     addr: int
     func: int
+    image: str = ''
+
+    def __str__(self):
+        return f'DBG_DYLD_TIMING_FUNC_FOR_ADD_IMAGE, addr: {hex(self.addr)}{_image(self.image)}, func: {hex(self.func)}'
+
+
+@dataclass
+class DyldFuncForRemoveImage:
+    ktraces: List
+    addr: int
+    func: int
+    image: str = ''
+
+    def __str__(self):
+        return (f'DBG_DYLD_TIMING_FUNC_FOR_REMOVE_IMAGE, addr: {hex(self.addr)}{_image(self.image)}, '
+                f'func: {hex(self.func)}')
+
+
+@dataclass
+class DyldStaticInitializer:
+    ktraces: List
+    mh: int
+    func: int
+    image: str = ''
+
+    def __str__(self):
+        return f'DBG_DYLD_TIMING_STATIC_INITIALIZER, mh: {hex(self.mh)}{_image(self.image)}, func: {hex(self.func)}'
+
+
+@dataclass
+class DyldObjcInit:
+    ktraces: List
+    mh: int
+    image: str = ''
+
+    def __str__(self):
+        return f'DBG_DYLD_TIMING_OBJC_INIT, mh: {hex(self.mh)}{_image(self.image)}'
+
+
+@dataclass
+class DyldTiming:
+    """A dyld phase traced without arguments (applying fixups, mapping ObjC images, ...)."""
+    ktraces: List
+    name: str
 
     def __str__(self):
-        return f'DBG_DYLD_TIMING_FUNC_FOR_ADD_IMAGE, addr: {hex(self.addr)}, func: {hex(self.func)}'
+        return self.name
 
 
 @dataclass
@@ -171,34 +225,55 @@ class Dladdr:
         return f'dladdr({hex(self.addr)}), ret: {self.ret}'
 
 
-def handle_uuid_map_a(parser, events):
+# dyld traces each image (un)load as an `_a` event with its UUID and load address, followed on the same thread by a
+# `_b` event with the id of its path string. The parser keeps the loaded images per process, so the events that only
+# carry a mach header (initializers, ObjC init, ...) can name their image.
+
+def _image_path(parser, events, mh):
+    return parser.dyld_images.get((parser.threads_pids.get(events[0].tid, -1), mh), '')
+
+
+def _map_a(parser, events, cls):
     args = events[0].values
-    return DyldUuidMapA(events, UUID(bytes=events[0].data[:16]), args[2], args[3])
+    parser.dyld_pending_load_addr[events[0].tid] = args[2]
+    return cls(events, UUID(bytes=events[0].data[:16]), args[2], args[3])
+
+
+def _map_b(parser, events, cls, loaded):
+    args = events[0].values
+    path = parser.global_strings.get(args[1], '') if args[1] else ''
+    load_addr = parser.dyld_pending_load_addr.pop(events[0].tid, None)
+    if load_addr is not None:
+        key = (parser.threads_pids.get(events[0].tid, -1), load_addr)
+        if loaded and path:
+            parser.dyld_images[key] = path
+        elif not loaded:
+            parser.dyld_images.pop(key, None)
+    return cls(events, args[0] & 0xffffffff, args[0] >> 32, path)
+
+
+def handle_uuid_map_a(parser, events):
+    return _map_a(parser, events, DyldUuidMapA)
 
 
 def handle_uuid_map_b(parser, events):
-    arg = events[0].values[0]
-    return DyldUuidMapB(events, arg & 0xffffffff, arg >> 32)
+    return _map_b(parser, events, DyldUuidMapB, loaded=True)
 
 
 def handle_uuid_unmap_a(parser, events):
-    args = events[0].values
-    return DyldUuidUnmapA(events, UUID(bytes=events[0].data[:16]), args[2], args[3])
+    return _map_a(parser, events, DyldUuidUnmapA)
 
 
 def handle_uuid_unmap_b(parser, events):
-    arg = events[0].values[0]
-    return DyldUuidUnmapB(events, arg & 0xffffffff, arg >> 32)
+    return _map_b(parser, events, DyldUuidUnmapB, loaded=False)
 
 
 def handle_uuid_shared_cache_a(parser, events):
-    args = events[0].values
-    return DyldUuidSharedCacheA(events, UUID(bytes=events[0].data[:16]), args[2], args[3])
+    return _map_a(parser, events, DyldUuidSharedCacheA)
 
 
 def handle_uuid_shared_cache_b(parser, events):
-    arg = events[0].values[0]
-    return DyldUuidSharedCacheB(events, arg & 0xffffffff, arg >> 32)
+    return _map_b(parser, events, DyldUuidSharedCacheB, loaded=True)
 
 
 def handle_timing_launch_executable(parser, events):
@@ -217,7 +292,26 @@ def handle_timing_map_image(parser, events):
 
 def handle_timing_func_for_add_image(parser, events):
     args = events[0].values
-    return DyldFuncForAddImage(events, args[1], args[2])
+    return DyldFuncForAddImage(events, args[1], args[2], _image_path(parser, events, args[1]))
+
+
+def handle_timing_func_for_remove_image(parser, events):
+    args = events[0].values
+    return DyldFuncForRemoveImage(events, args[1], args[2], _image_path(parser, events, args[1]))
+
+
+def handle_timing_static_initializer(parser, events):
+    args = events[0].values
+    return DyldStaticInitializer(events, args[1], args[2], _image_path(parser, events, args[1]))
+
+
+def handle_timing_objc_init(parser, events):
+    args = events[0].values
+    return DyldObjcInit(events, args[1], _image_path(parser, events, args[1]))
+
+
+def handle_timing(parser, events):
+    return DyldTiming(events, parser.trace_codes[events[0].eventid])
 
 
 def handle_timing_bootstrap_start(parser, events):
@@ -257,6 +351,14 @@ handlers = {
     'DBG_DYLD_TIMING_LAUNCH_EXECUTABLE': handle_timing_launch_executable,
     'DBG_DYLD_TIMING_MAP_IMAGE': handle_timing_map_image,
     'DBG_DYLD_TIMING_FUNC_FOR_ADD_IMAGE': handle_timing_func_for_add_image,
+    'DBG_DYLD_TIMING_FUNC_FOR_REMOVE_IMAGE': handle_timing_func_for_remove_image,
+    'DBG_DYLD_TIMING_STATIC_INITIALIZER': handle_timing_static_initializer,
+    'DBG_DYLD_TIMING_OBJC_INIT': handle_timing_objc_init,
+    'DBG_DYLD_TIMING_OBJC_MAP': handle_timing,
+    'DBG_DYLD_TIMING_APPLY_FIXUPS': handle_timing,
+    'DBG_DYLD_TIMING_ATTACH_CODESIGNATURE': handle_timing,
+    'DBG_DYLD_TIMING_BUILD_CLOSURE': handle_timing,
+    'DBG_DYLD_REMOTE_IMAGE_NOTIFIER': handle_timing,
     'DBG_DYLD_TIMING_BOOTSTRAP_START': handle_timing_bootstrap_start,
     'DBG_DYLD_TIMING_DLOPEN': handle_timing_dlopen,
     'DBG_DYLD_TIMING_DLOPEN_PREFLIGHT': handle_timing_dlopen_preflight,

pykdebugparser/trace_handlers/fsystem.py

@@ -1,5 +1,11 @@
 from dataclasses import dataclass
-from typing import List
+from typing import List, Optional
+
+from pykdebugparser.trace_codes import default_trace_codes
+
+DBG_DKRW = 0x302
+# bsd/sys/kdebug.h: the code of a disk I/O event is these flags (`P_RdMetaAsync`, `P_WrDataDone`, ...).
+DKIO_DONE = 0x01
 
 
 @dataclass
@@ -12,11 +18,47 @@ class VfsLookup:
         return f'lookup("{self.path}"), vnode id: {self.vnode_id}'
 
 
+@dataclass
+class DiskIo:
+    """A disk I/O, from its issue (device, block, size) to its completion (bytes not transferred, error)."""
+    ktraces: List
+    name: str
+    dev: Optional[int]
+    blkno: Optional[int]
+    size: Optional[int]
+    resid: int
+    error: int
+
+    def __str__(self):
+        rep = self.name
+        if self.dev is not None:
+            rep += f', dev: {self.dev >> 24},{self.dev & 0xffffff}, blkno: {hex(self.blkno)}, size: {self.size}'
+        return rep + f', resid: {self.resid}, error: {self.error}'
+
+
 def handle_vfs_lookup(parser, events):
     node = parser.parse_vnode(events)
     return VfsLookup(events, node.path, node.vnode_id)
 
 
+def handle_disk_io(parser, events):
+    # xnu traces the issue as (buf, dev, blkno, size) and the completion, often on another thread, as
+    # (buf, vnode, resid, error); the buf address pairs them. Print one line when the I/O completes.
+    event = events[0]
+    buf = event.values[0]
+    if not (event.eventid >> 2) & DKIO_DONE:
+        parser.pending_disk_io[buf] = event
+        return None
+    start = parser.pending_disk_io.pop(buf, None)
+    resid, error = event.values[2], event.values[3]
+    if start is None:
+        return DiskIo(events, parser.trace_codes[event.eventid].removesuffix('Done'), None, None, None, resid, error)
+    args = start.values
+    return DiskIo([start, event], parser.trace_codes[start.eventid], args[1] & 0xffffffff, args[2], args[3], resid,
+                  error)
+
+
 handlers = {
     'VFS_LOOKUP': handle_vfs_lookup,
 }
+handlers.update({name: handle_disk_io for eventid, name in default_trace_codes().items() if eventid >> 16 == DBG_DKRW})

pykdebugparser/trace_handlers/mach.py

@@ -2,10 +2,14 @@ import ctypes
 from dataclasses import dataclass
 from enum import Enum
 from functools import partial
-from typing import List
+from typing import List, Optional
 
+from pykdebugparser.trace_codes import default_trace_codes
 from pykdebugparser.trace_handlers import parse_enum
 
+# bsd/sys/kdebug.h: (DBG_MACH << 8) | DBG_MACH_EXCP_SYNC_ARM
+DBG_MACH_EXCP_SYNC_ARM = 0x103
+
 
 class AsynchronousSystemTrapsReason(Enum):
     AST_NONE = 0x00
@@ -59,10 +63,13 @@ class ExceptionSyndromeRegisterClass(Enum):
     ESR_EC_TRAP_SIMD_FP = 0x07
     ESR_EC_PTRAUTH_INSTR_TRAP = 0x09
     ESR_EC_MCRR_MRRC_CP14_TRAP = 0x0c
+    ESR_EC_BTI_FAIL = 0x0d
     ESR_EC_ILLEGAL_INSTR_SET = 0x0e
     ESR_EC_SVC_32 = 0x11
     ESR_EC_SVC_64 = 0x15
+    ESR_EC_HVC_64 = 0x16
     ESR_EC_MSR_TRAP = 0x18
+    ESR_EC_PAC_FAIL = 0x1c
     ESR_EC_IABORT_EL0 = 0x20
     ESR_EC_IABORT_EL1 = 0x21
     ESR_EC_PC_ALIGN = 0x22
@@ -281,6 +288,25 @@ class KernelUncategorizedExcArm:
         return f'KernelUncategorizedExcArm, class: {esr_class}, far: {hex(self.far)}, pc: {hex(self.pc)}'
 
 
+@dataclass
+class ArmException:
+    """An arm64 synchronous exception (`sleh.c`): its ESR, fault address and PC, for the classes without their own
+    trace class (breakpoints, `brk` traps, pointer authentication failures, alignment faults, ...)."""
+    ktraces: List
+    name: str
+    esr: int
+    far: int
+    pc: int
+
+    def __str__(self):
+        esr_class = self.esr >> 26
+        try:
+            esr_class = ExceptionSyndromeRegisterClass(esr_class).name
+        except ValueError:
+            pass
+        return f'{self.name}, class: {esr_class}, far: {hex(self.far)}, pc: {hex(self.pc)}'
+
+
 @dataclass
 class KernelDataAbortSameElExcArm:
     ktraces: List
@@ -594,6 +620,58 @@ class SemaphoreTimedwait:
         return f'semaphore_timedwait({hex(self.wait_name)}, {self.sec}, {self.nsec}), result: {self.result.name}'
 
 
+@dataclass
+class TaskForPid:
+    ktraces: List
+    target_tport: int
+    pid: int
+    t: int
+    result: KernReturn
+
+    def __str__(self):
+        return f'task_for_pid({self.target_tport}, {self.pid}, {hex(self.t)}), result: {self.result.name}'
+
+
+@dataclass
+class TaskNameForPid:
+    ktraces: List
+    target_tport: int
+    pid: int
+    tn: int
+    result: KernReturn
+
+    def __str__(self):
+        return f'task_name_for_pid({self.target_tport}, {self.pid}, {hex(self.tn)}), result: {self.result.name}'
+
+
+@dataclass
+class PidForTask:
+    ktraces: List
+    task: int
+    pid_addr: int
+    result: KernReturn
+
+    def __str__(self):
+        return f'pid_for_task({self.task}, {hex(self.pid_addr)}), result: {self.result.name}'
+
+
+@dataclass
+class TaskSuspension:
+    """A task or thread suspended (by `task_suspend`, a debugger, ...) or resumed (osfmk `DBG_MACH_SUSPENSION`)."""
+    ktraces: List
+    name: str
+    target: str
+    target_id: int
+    user_stop_count: Optional[int] = None
+    pidsuspended: Optional[int] = None
+
+    def __str__(self):
+        rep = f'{self.name}, {self.target}: {self.target_id}'
+        if self.user_stop_count is not None:
+            rep += f', user_stop_count: {self.user_stop_count}, pidsuspended: {bool(self.pidsuspended)}'
+        return rep
+
+
 @dataclass
 class MachPortGetAttributes:
     ktraces: List
@@ -1017,6 +1095,11 @@ def handle_user_instr_abort_lower_el_exc_arm(parser, events):
     return UserInstrAbortLowerElExcArm(events, *events[0].values[:3])
 
 
+def handle_arm_exception(parser, events):
+    args = events[0].values
+    return ArmException(events, parser.trace_codes[events[0].eventid], args[0], args[1], args[2])
+
+
 def handle_user_data_abort_lower_el_exc_arm(parser, events):
     args = events[0].values
     return UserDataAbortLowerElExcArm(events, args[0], args[1], args[2])
@@ -1057,6 +1140,41 @@ def handle_msc_mach_vm_map_trap(parser, events):
     return MachVmMap(events, *events[0].values)
 
 
+def handle_msc_task_for_pid(parser, events):
+    args = events[0].values
+    return TaskForPid(events, args[0] & 0xffffffff, ctypes.c_int32(args[1]).value, args[2],
+                      parse_enum(KernReturn, events[-1].values[0]))
+
+
+def handle_msc_task_name_for_pid(parser, events):
+    args = events[0].values
+    return TaskNameForPid(events, args[0] & 0xffffffff, ctypes.c_int32(args[1]).value, args[2],
+                          parse_enum(KernReturn, events[-1].values[0]))
+
+
+def handle_msc_pid_for_task(parser, events):
+    args = events[0].values
+    return PidForTask(events, args[0] & 0xffffffff, args[1], parse_enum(KernReturn, events[-1].values[0]))
+
+
+def handle_task_suspend(parser, events):
+    args = events[0].values
+    return TaskSuspension(events, 'task_suspend', 'pid', args[0], args[1], args[2])
+
+
+def handle_task_resume(parser, events):
+    return TaskSuspension(events, 'task_resume', 'pid', events[0].values[0])
+
+
+def handle_thread_suspend(parser, events):
+    args = events[0].values
+    return TaskSuspension(events, 'thread_suspend', 'tid', args[0], args[1], args[2])
+
+
+def handle_thread_resume(parser, events):
+    return TaskSuspension(events, 'thread_resume', 'tid', events[0].values[0])
+
+
 def handle_msc_mach_port_allocate_trap(parser, events):
     args = events[0].values
     return MachPortAllocate(events, args[0], parse_enum(MachPortRight, args[1]), args[2])
@@ -1306,6 +1424,13 @@ handlers = {
     'MSC_semaphore_signal_trap': handle_msc_semaphore_signal_trap,
     'MSC_semaphore_wait_trap': handle_msc_semaphore_wait_trap,
     'MSC_semaphore_timedwait_trap': handle_msc_semaphore_timedwait_trap,
+    'MSC_task_for_pid': handle_msc_task_for_pid,
+    'MSC_task_name_for_pid': handle_msc_task_name_for_pid,
+    'MSC_pid_for_task': handle_msc_pid_for_task,
+    'MACH_task_suspend': handle_task_suspend,
+    'MACH_task_resume': handle_task_resume,
+    'MACH_thread_suspend': handle_thread_suspend,
+    'MACH_thread_resume': handle_thread_resume,
     'MSC_mach_port_get_attributes_trap': handle_msc_mach_port_get_attributes_trap,
     'MSC_mach_port_guard_trap': handle_msc_mach_port_guard_trap,
     'MSC_mach_port_unguard_trap': handle_msc_mach_port_unguard_trap,
@@ -1341,3 +1466,6 @@ handlers = {
     'SCHED_CLUTCH_CPU_THREAD_SELECT': handle_sched_clutch_cpu_thread_select,
     'SCHED_CLUTCH_TG_BUCKET_PRI': handle_sched_clutch_tg_bucket_pri,
 }
+# Every other arm64 synchronous exception class shares the same arguments.
+handlers.update({name: handle_arm_exception for eventid, name in default_trace_codes().items()
+                 if eventid >> 16 == DBG_MACH_EXCP_SYNC_ARM and name.endswith('_Exc_ARM') and name not in handlers})

pykdebugparser/trace_handlers/trace.py

@@ -189,6 +189,23 @@ def handle_trace_string_threadname_prev(parser, events):
     return event
 
 
+# bsd/sys/kdebug.h
+TRACE_LOST_EVENTS = 0x7020008
+
+
+@dataclass
+class TraceLostEvents:
+    """The kernel's trace buffer filled up and dropped events before this point."""
+    ktraces: List
+
+    def __str__(self):
+        return 'TRACE_LOST_EVENTS: the kernel dropped trace events here'
+
+
+def handle_trace_lost_events(parser, events):
+    return TraceLostEvents(events)
+
+
 handlers = {
     'TRACE_DATA_NEWTHREAD': handle_trace_data_newthread,
     'TRACE_DATA_EXEC': handle_trace_data_exec,
@@ -200,4 +217,5 @@ handlers = {
     'TRACE_STRING_PROC_EXIT': handle_trace_string_proc_exit,
     'TRACE_STRING_THREADNAME': handle_trace_string_threadname,
     'TRACE_STRING_THREADNAME_PREV': handle_trace_string_threadname_prev,
+    'TRACE_LOST_EVENTS': handle_trace_lost_events,
 }

pykdebugparser/traces_parser.py

@@ -29,6 +29,9 @@ class RawTrace:
         return rep
 
 
+NO_RETURN_CALLS = ('BSC_exit', 'BSC_bsdthread_terminate')
+
+
 class TracesParser:
     def __init__(self, trace_codes_map, threads_pids, pids_names):
         self.trace_codes = trace_codes_map
@@ -46,6 +49,13 @@ class TracesParser:
         }
         self.last_data_newthread = None
         self.last_data_exec = None
+        # Calls that never return, parsed from their start event.
+        self.no_return_eventids = {eventid for eventid, name in trace_codes_map.items() if name in NO_RETURN_CALLS}
+        # Disk I/Os issued and not completed yet, by their buf address.
+        self.pending_disk_io = {}
+        # Images dyld loaded, by (pid, load address), and the load address of each thread's last `DYLD_uuid_*_a` event.
+        self.dyld_images = {}
+        self.dyld_pending_load_addr = {}
         # Event ids to show as `RawTrace` when they have no handler.
         self.raw_eventids = set()
         self.handlers = {}
@@ -109,6 +119,9 @@ class TracesParser:
         return list(self.vnode_generator([e for e in events if self.trace_codes.get(e.eventid) == 'VFS_LOOKUP']))
 
     def _feed_start_event(self, event, state):
+        if event.eventid in self.no_return_eventids:
+            # The thread or process ends inside the call, so its end is never traced.
+            return self._feed_single_event(event, state)
         tid_state = state.get(event.tid)
         if tid_state is None:
             # New tid

{pykdebugparser-1.4.0.dist-info → pykdebugparser-1.5.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pykdebugparser
-Version: 1.4.0
+Version: 1.5.0
 Summary: Python parser for kdebug events
 Author: Matan Perelman
 Author-email: matan1008@gmail.com

pykdebugparser-1.5.0.dist-info/RECORD

@@ -0,0 +1,25 @@
+pykdebugparser/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pykdebugparser/__main__.py,sha256=WfgLv5SGr0hQlUeM68ai1g0AqtYDgDFqYlbvvm5rua4,3711
+pykdebugparser/callstacks_parser.py,sha256=i2nfhx0olZZ65UV_ZV3Tw9E_6SWtre8CAbfRBSig4-E,1654
+pykdebugparser/highlight.py,sha256=i-mjfyak0VTo8P_7EeZLIUsUH08Qs-1yAjHBJVWW448,6439
+pykdebugparser/kd_buf_parser.py,sha256=CpifFeBnQ_MZSokjO44lzhXO-KNnZvO-teaPyQqSfio,10991
+pykdebugparser/kevent.py,sha256=DudovIuES4QGxosYeEFJm3AW_FWGbJJBH1kcLnL7UFM,1779
+pykdebugparser/os_log_event.py,sha256=OQUlHpS44oUNwma_bxvnK0SKh_RFeEUdH1RmyrM4nLc,12356
+pykdebugparser/pykdebugparser.py,sha256=RipUcCh6dChgtdh_owvlprjLRMF1dvgic6ZyCqC5I8I,10320
+pykdebugparser/trace.codes,sha256=S6ZN5l9FSGrlSon8nKoVaqRfLdCPl3HL93XSjuoCUnM,103995
+pykdebugparser/trace_codes.py,sha256=4ImGc_jNpY-12cGXdwy6NMCEYcI24oYn1NBOaKNdWNo,1984
+pykdebugparser/traces_parser.py,sha256=LyHVTi4P2OeOLNG2Tahu1x7roQXkC-Jx8LS-E4mBcAQ,5799
+pykdebugparser/trace_handlers/__init__.py,sha256=r1dzmNTxiEtgENq9WFRVQX0g9f9dRxgAgVxjDXSc7s4,679
+pykdebugparser/trace_handlers/bsd.py,sha256=XJjKxDzHSD2VkwEPOqizdTA6ZydFXq7piEVJKCAa0ZY,189267
+pykdebugparser/trace_handlers/dyld.py,sha256=LDgWzOl90qXzcc_q8OvttV2gJ4JNodXFwMemSiV8_ps,10154
+pykdebugparser/trace_handlers/fsystem.py,sha256=AE9L_00-11VyZAYd6dzkmTCP6EVFmXzyae2VikrsiJo,2082
+pykdebugparser/trace_handlers/mach.py,sha256=_2YEgUfKYxaB7NfA2jCVxUbdKDrU21KJKhxtc5yP1M8,39172
+pykdebugparser/trace_handlers/perf.py,sha256=CBffheZFlHa9IOVUZ-UcjNtKmjamv-un44bRJYYo9dw,5070
+pykdebugparser/trace_handlers/trace.py,sha256=s3Kr0wDvWZL2JZVKhxcvA1uvqsdePlEWiXfqSI76znw,5797
+pykdebugparser/trace_handlers/turnstile.py,sha256=8zZfba2Obq7a6qdhLIkItmZxRzzOe1PTTeHnrLuJUx0,8233
+pykdebugparser-1.5.0.dist-info/licenses/LICENSE,sha256=KjZXYP_CGYzDTtVfcxPT6wlGRoQKHPSJu0vh55_2QSo,1071
+pykdebugparser-1.5.0.dist-info/METADATA,sha256=biG_5LGv2RvIABkToh09hXR3eQ8Pm6BUlsI59SZPvdA,18995
+pykdebugparser-1.5.0.dist-info/WHEEL,sha256=4YBfCYNH4wlLpv3pzq1hbEuIlXA4WJabKLFurZ7eTL0,109
+pykdebugparser-1.5.0.dist-info/entry_points.txt,sha256=NrVIOgPR6v2MbpTZ3aKedP1jQ93nUQus6sgP4wAD0_E,63
+pykdebugparser-1.5.0.dist-info/top_level.txt,sha256=FcHHXR9VKmqufFw0a6UP4c8NyS-u4ORY-yXQ6ZzP_28,15
+pykdebugparser-1.5.0.dist-info/RECORD,,

pykdebugparser-1.4.0.dist-info/RECORD

@@ -1,24 +0,0 @@
-pykdebugparser/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pykdebugparser/__main__.py,sha256=WfgLv5SGr0hQlUeM68ai1g0AqtYDgDFqYlbvvm5rua4,3711
-pykdebugparser/callstacks_parser.py,sha256=i2nfhx0olZZ65UV_ZV3Tw9E_6SWtre8CAbfRBSig4-E,1654
-pykdebugparser/kd_buf_parser.py,sha256=CpifFeBnQ_MZSokjO44lzhXO-KNnZvO-teaPyQqSfio,10991
-pykdebugparser/kevent.py,sha256=DudovIuES4QGxosYeEFJm3AW_FWGbJJBH1kcLnL7UFM,1779
-pykdebugparser/os_log_event.py,sha256=OQUlHpS44oUNwma_bxvnK0SKh_RFeEUdH1RmyrM4nLc,12356
-pykdebugparser/pykdebugparser.py,sha256=87c7HT6NXl820A7bv4g2iMD3kWYKpq6ya49eT8p46nE,10238
-pykdebugparser/trace.codes,sha256=S6ZN5l9FSGrlSon8nKoVaqRfLdCPl3HL93XSjuoCUnM,103995
-pykdebugparser/trace_codes.py,sha256=4ImGc_jNpY-12cGXdwy6NMCEYcI24oYn1NBOaKNdWNo,1984
-pykdebugparser/traces_parser.py,sha256=2nerqDgGLkaHWZaqHDE28gZvlRrbNcYANaBbcYwSlLk,5064
-pykdebugparser/trace_handlers/__init__.py,sha256=r1dzmNTxiEtgENq9WFRVQX0g9f9dRxgAgVxjDXSc7s4,679
-pykdebugparser/trace_handlers/bsd.py,sha256=2dg4wFdfiZ79jARUcB1VCfCPw2z8ni11eWFsMQtQk-E,185848
-pykdebugparser/trace_handlers/dyld.py,sha256=ydyFhEkXKJzgo-1X5bH7UqiiLAenUa-KpBDfgSXMuAM,6839
-pykdebugparser/trace_handlers/fsystem.py,sha256=yOILOEuQDdjPDKq6HhzsjI27z36fkT6TqgCBddTvENg,417
-pykdebugparser/trace_handlers/mach.py,sha256=JL1v4rCCdGULsnVm3CYOLegL3JqNt3u7X-lJw1xzBRQ,35023
-pykdebugparser/trace_handlers/perf.py,sha256=CBffheZFlHa9IOVUZ-UcjNtKmjamv-un44bRJYYo9dw,5070
-pykdebugparser/trace_handlers/trace.py,sha256=Lu8s9XT1jb5avtrVXF6SQ6FUj93xYc0TwJd7yoIjV9U,5377
-pykdebugparser/trace_handlers/turnstile.py,sha256=8zZfba2Obq7a6qdhLIkItmZxRzzOe1PTTeHnrLuJUx0,8233
-pykdebugparser-1.4.0.dist-info/licenses/LICENSE,sha256=KjZXYP_CGYzDTtVfcxPT6wlGRoQKHPSJu0vh55_2QSo,1071
-pykdebugparser-1.4.0.dist-info/METADATA,sha256=RcvGghbwZc-Xd7KHUzdlOdrnNH9i58vS0oHcqBWYedo,18995
-pykdebugparser-1.4.0.dist-info/WHEEL,sha256=4YBfCYNH4wlLpv3pzq1hbEuIlXA4WJabKLFurZ7eTL0,109
-pykdebugparser-1.4.0.dist-info/entry_points.txt,sha256=NrVIOgPR6v2MbpTZ3aKedP1jQ93nUQus6sgP4wAD0_E,63
-pykdebugparser-1.4.0.dist-info/top_level.txt,sha256=FcHHXR9VKmqufFw0a6UP4c8NyS-u4ORY-yXQ6ZzP_28,15
-pykdebugparser-1.4.0.dist-info/RECORD,,