pykdebugparser 1.5.0__tar.gz → 1.6.1__tar.gz

{pykdebugparser-1.5.0 → pykdebugparser-1.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pykdebugparser
-Version: 1.5.0
+Version: 1.6.1
 Summary: Python parser for kdebug events
 Author: Matan Perelman
 Author-email: matan1008@gmail.com
@@ -39,7 +39,6 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: bytestruct>=0.1.0
-Requires-Dist: pygments
 Requires-Dist: click
 Requires-Dist: termcolor
 Provides-Extra: testing

pykdebugparser-1.6.1/pykdebugparser/highlight.py

@@ -0,0 +1,66 @@
+"""Syntax highlighting of trace lines, such as `open("/etc/hosts", O_RDONLY), fd: 3`.
+
+Lines are colored with the terminal's 16 basic colors, so they follow its theme:
+
+- the call or event name at the start, bold;
+- strings (paths, symbols, ...), green;
+- numbers, cyan;
+- constants (flags, enums, signals: `O_RDONLY`, `KERN_SUCCESS`, `SIGKILL`), yellow;
+- an error (`errno: ENOENT(2)`, a `KERN_*` failure), red;
+- the labels of results and fields (`fd:`, `count:`), dim.
+
+A trace is mostly the same calls with the same arguments, so the most recent lines are cached.
+"""
+import re
+from functools import lru_cache
+
+_RESET = '\x1b[0m'
+_BOLD = '\x1b[1m'
+_DIM = '\x1b[2m'
+_RED = '\x1b[31m'
+_GREEN = '\x1b[32m'
+_YELLOW = '\x1b[33m'
+_CYAN = '\x1b[36m'
+
+_TOKEN = re.compile(
+    r'(?P<string>"[^"]*")'
+    r'|(?P<number>-?0[xX][0-9a-fA-F]+|-?\d+)'
+    r'|(?P<label>[A-Za-z_][\w ]*?:)(?= |$)'
+    r'|(?P<name>[A-Za-z_]\w*)'
+)
+# A constant is all capitals, digits and underscores, with at least one capital.
+_CONSTANT = re.compile(r'[A-Z][A-Z0-9_]*')
+# `KERN_SUCCESS` is a result, not an error.
+_SUCCESS = {'KERN_SUCCESS'}
+
+
+@lru_cache(maxsize=1 << 14)
+def highlight(text: str) -> str:
+    """`text` with terminal colors (SGR escape sequences); the text itself is unchanged."""
+    out = []
+    pos = 0
+    error = False  # inside the value of an `errno:` result
+    first = True
+    for match in _TOKEN.finditer(text):
+        if match.start() > pos:
+            out.append(text[pos:match.start()])
+        pos = match.end()
+        kind = match.lastgroup
+        value = match.group()
+        if kind == 'string':
+            out.append(_GREEN + value + _RESET)
+        elif kind == 'number':
+            out.append((_RED if error else _CYAN) + value + _RESET)
+        elif kind == 'label':
+            error = value == 'errno:'
+            out.append((_RED if error else _DIM) + value + _RESET)
+        elif first:
+            out.append(_BOLD + value + _RESET)
+        elif _CONSTANT.fullmatch(value):
+            failed = error or (value.startswith('KERN_') and value not in _SUCCESS)
+            out.append((_RED if failed else _YELLOW) + value + _RESET)
+        else:
+            out.append(value)
+        first = False
+    out.append(text[pos:])
+    return ''.join(out)

{pykdebugparser-1.5.0 → pykdebugparser-1.6.1}/pykdebugparser/pykdebugparser.py

@@ -1,4 +1,6 @@
+import dataclasses
 from datetime import datetime
+from enum import Enum
 import io
 import math
 
@@ -9,6 +11,7 @@ from pykdebugparser.highlight import highlight
 from pykdebugparser.kd_buf_parser import KdBufParser
 from pykdebugparser.kevent import DgbFuncQual
 from pykdebugparser.trace_codes import default_trace_codes
+from pykdebugparser.trace_handlers import UnknownEnumValue
 from pykdebugparser.trace_handlers.trace import TRACE_LOST_EVENTS
 from pykdebugparser.traces_parser import TracesParser
 from pykdebugparser.os_log_event import OsLogEvent
@@ -19,6 +22,32 @@ DBG_FSYSTEM = 3
 DBG_BSD = 4
 
 
+def _fields(trace) -> dict:
+    """A trace's parsed values (its dataclass fields but the raw kevents), as JSON-compatible values."""
+    return {field.name: _jsonable(getattr(trace, field.name)) for field in dataclasses.fields(trace)
+            if field.name != 'ktraces'}
+
+
+def _jsonable(value):
+    if isinstance(value, (bool, str)) or value is None:
+        return value
+    if isinstance(value, UnknownEnumValue):
+        return int(value)
+    if isinstance(value, Enum):
+        return value.name
+    if isinstance(value, (int, float)):
+        return value
+    if isinstance(value, (bytes, bytearray)):
+        return value.hex()
+    if isinstance(value, (list, tuple)):
+        return [_jsonable(item) for item in value]
+    if isinstance(value, dict):
+        return {str(key): _jsonable(item) for key, item in value.items()}
+    if dataclasses.is_dataclass(value) and not isinstance(value, type):
+        return _fields(value)
+    return str(value)
+
+
 class PyKdebugParser:
     def __init__(self):
         self.filter_tid = None
@@ -99,6 +128,32 @@ class PyKdebugParser:
             trace_generator = filter(lambda t: t.ktraces[0].eventid >> 24 != DBG_FSYSTEM, trace_generator)
         return trace_generator
 
+    def dict_traces(self, kdebug: io.IOBase, trace_codes=None):
+        """The traces as JSON-compatible dicts (see `trace_dict`)."""
+        trace_codes_map = default_trace_codes() if trace_codes is None else trace_codes
+        return map(lambda t: self.trace_dict(t, trace_codes_map), self.traces(kdebug, trace_codes_map))
+
+    def trace_dict(self, trace, trace_codes=None) -> dict:
+        """
+        A trace as JSON-compatible data: when (`timestamp` in mach ticks, and `time` in ISO 8601 when the device's time
+        information is set), where (`tid`, `pid`, `process`), what (`event`, the trace code name of its first kevent,
+        and `fields`, the parsed values) and `text`, as `formatted_traces` prints it.
+        """
+        trace_codes_map = default_trace_codes() if trace_codes is None else trace_codes
+        first = trace.ktraces[0]
+        pid = self.threads_pids.get(first.tid)
+        posix_time = self._posix_time(first.timestamp)
+        return {
+            'timestamp': first.timestamp,
+            'time': None if posix_time is None else datetime.fromtimestamp(posix_time, tz=self.timezone).isoformat(),
+            'tid': first.tid,
+            'pid': pid,
+            'process': self.pids_names.get(pid, '') if pid is not None else '',
+            'event': trace_codes_map.get(first.eventid, hex(first.eventid)),
+            'fields': _fields(trace),
+            'text': str(trace),
+        }
+
     def formatted_traces(self, kdebug: io.IOBase, trace_codes=None):
         return map(lambda t: self._format_trace(t), self.traces(kdebug, trace_codes))
 
@@ -128,13 +183,19 @@ class PyKdebugParser:
         process_name = self.pids_names.get(pid, '')
         return self.filter_process == str(pid) or self.filter_process == process_name
 
-    def _format_timestamp(self, timestamp):
+    def _posix_time(self, timestamp):
+        """A kevent timestamp (mach ticks) as POSIX time, or None without the device's time information."""
         if None in (self.mach_absolute_time, self.numer, self.denom, self.usecs_since_epoch, self.timezone):
-            return str(timestamp) + ' '
+            return None
         offset_usec = (
                 ((timestamp - self.mach_absolute_time) * self.numer) / (self.denom * 1000)
         )
-        posix_time = (self.usecs_since_epoch + offset_usec) / 1000000
+        return (self.usecs_since_epoch + offset_usec) / 1000000
+
+    def _format_timestamp(self, timestamp):
+        posix_time = self._posix_time(timestamp)
+        if posix_time is None:
+            return str(timestamp) + ' '
         # Split to seconds and microseconds exactly like `datetime.fromtimestamp` does (round half to even), so
         # the date and time part can be formatted only once per second.
         fraction, seconds = math.modf(posix_time)

{pykdebugparser-1.5.0 → pykdebugparser-1.6.1}/pykdebugparser/trace_handlers/bsd.py

@@ -2837,13 +2837,22 @@ DARWIN_SIGNALS = (None, 'SIGHUP', 'SIGINT', 'SIGQUIT', 'SIGILL', 'SIGTRAP', 'SIG
                   'SIGWINCH', 'SIGINFO', 'SIGUSR1', 'SIGUSR2')
 
 
+def pid(value: int) -> int:
+    """A `pid_t` argument: a C int, where -1 means any and a negative pid a process group (`kill(-pgid, sig)`)."""
+    return ctypes.c_int32(value).value
+
+
+def signal_name(sig: int) -> str:
+    """A Darwin signal by name (`SIGTERM`), or its number (0 only checks that the target exists)."""
+    return DARWIN_SIGNALS[sig] if 0 < sig < len(DARWIN_SIGNALS) else str(sig)
+
+
 def format_wait_status(status: int) -> str:
     """Describe a wait status (`W_EXITCODE(code, signal)`) as its exit code or terminating signal."""
     signal = status & 0x7f
     if not signal:
         return f'exit code: {(status >> 8) & 0xff}'
-    name = DARWIN_SIGNALS[signal] if signal < len(DARWIN_SIGNALS) else str(signal)
-    return f'signal: {name}' + (', core dumped' if status & 0x80 else '')
+    return f'signal: {signal_name(signal)}' + (', core dumped' if status & 0x80 else '')
 
 
 @dataclass
@@ -3034,7 +3043,7 @@ class BscPthreadKill:
     result: str
 
     def __str__(self):
-        rep = f'pthread_kill({self.thread_port}, {self.sig})'
+        rep = f'pthread_kill({self.thread_port}, {signal_name(self.sig)})'
         if self.result:
             rep += f', {self.result}'
         return rep
@@ -3063,7 +3072,7 @@ class BscKill:
     result: str
 
     def __str__(self):
-        rep = f'kill({self.pid}, {self.sig})'
+        rep = f'kill({self.pid}, {signal_name(self.sig)})'
         if self.result:
             rep += f', {self.result}'
         return rep
@@ -3991,7 +4000,7 @@ class BscObsKillpg:
     result: str
 
     def __str__(self):
-        rep = f'killpg({self.pgrp}, {self.sig})'
+        rep = f'killpg({self.pgrp}, {signal_name(self.sig)})'
         if self.result:
             rep += f', {self.result}'
         return rep
@@ -5485,7 +5494,7 @@ def handle_bsdthread_terminate(parser, events):
 
 def handle_wait4(parser, events, no_cancel=False):
     args = events[0].values
-    return BscWait4(events, args[0], args[1], args[2], args[3], serialize_result(events[-1], 'pid'), no_cancel)
+    return BscWait4(events, pid(args[0]), args[1], args[2], args[3], serialize_result(events[-1], 'pid'), no_cancel)
 
 
 def handle_recvmsg(parser, events, no_cancel=False):
@@ -5548,7 +5557,7 @@ def handle_sigreturn(parser, events):
 
 
 def handle_kill(parser, events):
-    return BscKill(events, events[0].values[0], events[0].values[1], serialize_result(events[-1]))
+    return BscKill(events, pid(events[0].values[0]), events[0].values[1], serialize_result(events[-1]))
 
 
 def handle_getppid(parser, events):
@@ -5688,7 +5697,7 @@ def handle_getpgrp(parser, events):
 
 
 def handle_setpgid(parser, events):
-    return BscSetpgid(events, events[0].values[0], events[0].values[1], serialize_result(events[-1]))
+    return BscSetpgid(events, pid(events[0].values[0]), pid(events[0].values[1]), serialize_result(events[-1]))
 
 
 def handle_setitimer(parser, events):
@@ -5885,7 +5894,7 @@ def handle_gethostuuid(parser, events):
 
 
 def handle_obs_killpg(parser, events):
-    return BscObsKillpg(events, events[0].values[0], events[0].values[1], serialize_result(events[-1]))
+    return BscObsKillpg(events, pid(events[0].values[0]), events[0].values[1], serialize_result(events[-1]))
 
 
 def handle_setsid(parser, events):
@@ -5893,7 +5902,7 @@ def handle_setsid(parser, events):
 
 
 def handle_getpgid(parser, events):
-    return BscGetpgid(events, events[0].values[0], serialize_result(events[-1], 'gid'))
+    return BscGetpgid(events, pid(events[0].values[0]), serialize_result(events[-1], 'gid'))
 
 
 def handle_setprivexec(parser, events):
@@ -5952,12 +5961,12 @@ def handle_mount(parser, events):
 
 def handle_csops(parser, events):
     args = events[0].values
-    return BscCsops(events, args[0], parse_enum(CsopsOps, args[1]), args[2], args[3], serialize_result(events[-1]))
+    return BscCsops(events, pid(args[0]), parse_enum(CsopsOps, args[1]), args[2], args[3], serialize_result(events[-1]))
 
 
 def handle_csops_audittoken(parser, events):
     args = events[0].values
-    return BscCsopsAudittoken(events, args[0], parse_enum(CsopsOps, args[1]), args[2], args[3], serialize_result(events[-1]))
+    return BscCsopsAudittoken(events, pid(args[0]), parse_enum(CsopsOps, args[1]), args[2], args[3], serialize_result(events[-1]))
 
 
 def handle_waitid(parser, events, no_cancel=False):
@@ -6353,7 +6362,7 @@ def handle_psynch_cvwait(parser, events):
 
 def handle_getsid(parser, events):
     args = events[0].values
-    return BscGetsid(events, args[0], serialize_result(events[-1], 'sid'))
+    return BscGetsid(events, pid(args[0]), serialize_result(events[-1], 'sid'))
 
 
 def handle_psynch_cvclrprepost(parser, events):
@@ -6399,7 +6408,7 @@ def handle_semwait_signal(parser, events, no_cancel=False):
 
 def handle_proc_info(parser, events):
     args = events[0].values
-    return BscProcInfo(events, parse_enum(ProcInfoCall, args[0]), args[1], args[2], args[3], serialize_result(events[-1]))
+    return BscProcInfo(events, parse_enum(ProcInfoCall, args[0]), pid(args[1]), args[2], args[3], serialize_result(events[-1]))
 
 
 def handle_sendfile(parser, events):
@@ -6548,22 +6557,22 @@ def handle_audit_session_port(parser, events):
 
 def handle_pid_suspend(parser, events):
     args = events[0].values
-    return BscPidSuspend(events, args[0], serialize_result(events[-1]))
+    return BscPidSuspend(events, pid(args[0]), serialize_result(events[-1]))
 
 
 def handle_pid_resume(parser, events):
     args = events[0].values
-    return BscPidResume(events, args[0], serialize_result(events[-1]))
+    return BscPidResume(events, pid(args[0]), serialize_result(events[-1]))
 
 
 def handle_pid_hibernate(parser, events):
     args = events[0].values
-    return BscPidHibernate(events, args[0], serialize_result(events[-1]))
+    return BscPidHibernate(events, pid(args[0]), serialize_result(events[-1]))
 
 
 def handle_pid_shutdown_sockets(parser, events):
     args = events[0].values
-    return BscPidShutdownSockets(events, args[0], args[1], serialize_result(events[-1]))
+    return BscPidShutdownSockets(events, pid(args[0]), args[1], serialize_result(events[-1]))
 
 
 def handle_shared_region_map_and_slide_np(parser, events):
@@ -6578,7 +6587,7 @@ def handle_kas_info(parser, events):
 
 def handle_memorystatus_control(parser, events):
     args = events[0].values
-    return BscMemorystatusControl(events, args[0], args[1], args[2], args[3], serialize_result(events[-1]))
+    return BscMemorystatusControl(events, args[0], pid(args[1]), args[2], args[3], serialize_result(events[-1]))
 
 
 def handle_guarded_open_np(parser, events):
@@ -6609,7 +6618,7 @@ def handle_usrctl(parser, events):
 
 def handle_proc_rlimit_control(parser, events):
     args = events[0].values
-    return BscProcRlimitControl(events, args[0], args[1], args[2], serialize_result(events[-1]))
+    return BscProcRlimitControl(events, pid(args[0]), args[1], args[2], serialize_result(events[-1]))
 
 
 def handle_connectx(parser, events):
@@ -6663,7 +6672,7 @@ def handle_sfi_ctl(parser, events):
 
 def handle_sfi_pidctl(parser, events):
     args = events[0].values
-    return BscSfiPidctl(events, args[0], args[1], args[2], args[3], serialize_result(events[-1]))
+    return BscSfiPidctl(events, args[0], pid(args[1]), args[2], args[3], serialize_result(events[-1]))
 
 
 def handle_coalition(parser, events):
@@ -6778,7 +6787,7 @@ def handle_getattrlistat(parser, events):
 
 def handle_proc_trace_log(parser, events):
     args = events[0].values
-    return BscProcTraceLog(events, args[0], args[1], serialize_result(events[-1]))
+    return BscProcTraceLog(events, pid(args[0]), args[1], serialize_result(events[-1]))
 
 
 def handle_bsdthread_ctl(parser, events):
@@ -6979,7 +6988,7 @@ def handle_fs_snapshot(parser, events):
 
 def handle_terminate_with_payload(parser, events):
     args = events[0].values
-    return BscTerminateWithPayload(events, args[0], args[1], args[2], args[3], serialize_result(events[-1]))
+    return BscTerminateWithPayload(events, pid(args[0]), args[1], args[2], args[3], serialize_result(events[-1]))
 
 
 def handle_abort_with_payload(parser, events):
@@ -7065,12 +7074,12 @@ def handle_pivot_root(parser, events):
 
 def handle_task_inspect_for_pid(parser, events):
     args = events[0].values
-    return BscTaskInspectForPid(events, args[0], args[1], args[2], serialize_result(events[-1]))
+    return BscTaskInspectForPid(events, args[0], pid(args[1]), args[2], serialize_result(events[-1]))
 
 
 def handle_task_read_for_pid(parser, events):
     args = events[0].values
-    return BscTaskReadForPid(events, args[0], args[1], args[2], serialize_result(events[-1]))
+    return BscTaskReadForPid(events, args[0], pid(args[1]), args[2], serialize_result(events[-1]))
 
 
 def handle_sys_preadv(parser, events, no_cancel=False):
@@ -7092,7 +7101,7 @@ def handle_ulock_wait2(parser, events):
 
 def handle_proc_info_extended_id(parser, events):
     args = events[0].values
-    return BscProcInfoExtendedId(events, args[0], args[1], args[2], args[3], serialize_result(events[-1]))
+    return BscProcInfoExtendedId(events, args[0], pid(args[1]), args[2], args[3], serialize_result(events[-1]))
 
 
 def handle_openat_dprotected_np(parser, events):

{pykdebugparser-1.5.0 → pykdebugparser-1.6.1}/pykdebugparser.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pykdebugparser
-Version: 1.5.0
+Version: 1.6.1
 Summary: Python parser for kdebug events
 Author: Matan Perelman
 Author-email: matan1008@gmail.com
@@ -39,7 +39,6 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: bytestruct>=0.1.0
-Requires-Dist: pygments
 Requires-Dist: click
 Requires-Dist: termcolor
 Provides-Extra: testing

{pykdebugparser-1.5.0 → pykdebugparser-1.6.1}/pykdebugparser.egg-info/requires.txt

@@ -1,5 +1,4 @@
 bytestruct>=0.1.0
-pygments
 click
 termcolor
 

{pykdebugparser-1.5.0 → pykdebugparser-1.6.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pykdebugparser"
-version = "1.5.0"
+version = "1.6.1"
 description = "Python parser for kdebug events"
 readme.file = "README.md"
 readme.content-type = "text/markdown"
@@ -25,7 +25,6 @@ classifiers = [
 ]
 dependencies = [
     "bytestruct>=0.1.0",
-    "pygments",
     "click",
     "termcolor",
 ]

pykdebugparser-1.6.1/tests/test_highlight.py

@@ -0,0 +1,52 @@
+import re
+
+import pytest
+
+from pykdebugparser.highlight import highlight
+
+_SGR = re.compile(r'\x1b\[[0-9;]*m')
+
+BOLD, DIM, RESET = '\x1b[1m', '\x1b[2m', '\x1b[0m'
+RED, GREEN, YELLOW, CYAN = '\x1b[31m', '\x1b[32m', '\x1b[33m', '\x1b[36m'
+
+
+def _colored(color, text):
+    return color + text + RESET
+
+
+@pytest.mark.parametrize('line, expected', [
+    ('open("/etc/hosts", O_RDONLY | O_CLOEXEC), fd: 3',
+     f'{_colored(BOLD, "open")}({_colored(GREEN, chr(34) + "/etc/hosts" + chr(34))}, {_colored(YELLOW, "O_RDONLY")} | '
+     f'{_colored(YELLOW, "O_CLOEXEC")}), {_colored(DIM, "fd:")} {_colored(CYAN, "3")}'),
+    ('stat64("/x"), errno: ENOENT(2)',
+     f'{_colored(BOLD, "stat64")}({_colored(GREEN, chr(34) + "/x" + chr(34))}), {_colored(RED, "errno:")} '
+     f'{_colored(RED, "ENOENT")}({_colored(RED, "2")})'),
+    ('task_for_pid(515, 97997, 0x16bd572fc), result: KERN_SUCCESS',
+     f'{_colored(BOLD, "task_for_pid")}({_colored(CYAN, "515")}, {_colored(CYAN, "97997")}, '
+     f'{_colored(CYAN, "0x16bd572fc")}), {_colored(DIM, "result:")} {_colored(YELLOW, "KERN_SUCCESS")}'),
+    ('mach_port_mod_refs(1, 0x0, MACH_PORT_RIGHT_SEND, 1), result: KERN_INVALID_RIGHT',
+     f'{_colored(BOLD, "mach_port_mod_refs")}({_colored(CYAN, "1")}, {_colored(CYAN, "0x0")}, '
+     f'{_colored(YELLOW, "MACH_PORT_RIGHT_SEND")}, {_colored(CYAN, "1")}), {_colored(DIM, "result:")} '
+     f'{_colored(RED, "KERN_INVALID_RIGHT")}'),
+    ('lookup("private/var/db"), vnode id: 42',
+     f'{_colored(BOLD, "lookup")}({_colored(GREEN, chr(34) + "private/var/db" + chr(34))}), '
+     f'{_colored(DIM, "vnode id:")} {_colored(CYAN, "42")}'),
+    ('sigreturn(0x1, 30, 0x2), errno: EJUSTRETURN(-2)',
+     f'{_colored(BOLD, "sigreturn")}({_colored(CYAN, "0x1")}, {_colored(CYAN, "30")}, {_colored(CYAN, "0x2")}), '
+     f'{_colored(RED, "errno:")} {_colored(RED, "EJUSTRETURN")}({_colored(RED, "-2")})'),
+    ('DBG_DYLD_TIMING_OBJC_MAP', _colored(BOLD, 'DBG_DYLD_TIMING_OBJC_MAP')),
+])
+def test_highlight(line, expected):
+    assert highlight(line) == expected
+
+
+@pytest.mark.parametrize('line', [
+    'open("/etc/hosts", O_RDONLY), fd: 3',
+    "ioctl(8, 0xc020699d /* _IOC(IOC_IN | IOC_OUT, 'i', 157, 32) */, 0x16d3cbea0)",
+    'TRACE_LOST_EVENTS: the kernel dropped trace events here',
+    'stat64("/a b/{c}: d", 0x1), count: 0x10',
+    'unterminated "string',
+    '',
+])
+def test_highlight_keeps_the_text(line):
+    assert _SGR.sub('', highlight(line)) == line

{pykdebugparser-1.5.0 → pykdebugparser-1.6.1}/tests/test_pykdebugparser.py

@@ -1,4 +1,5 @@
 from datetime import datetime, timedelta, timezone
+import json
 from io import BytesIO
 import struct
 
@@ -179,3 +180,67 @@ def test_traces_keep_lost_events_when_filtered():
     traces = list(parser.formatted_traces(BytesIO(events_buf)))
     assert len(traces) == 1
     assert traces[0].endswith('TRACE_LOST_EVENTS: the kernel dropped trace events here')
+
+
+def test_dict_traces():
+    events = [
+        _kd_buf(1000, 0x40c0014 | 1, (0x16d2c2b80, 0, 0, 0)),  # open start
+        _kd_buf(2000, 0x40c0014 | 2, (2, 0, 0, 0)),  # open end: ENOENT
+    ]
+    events_buf = RAW_VERSION2_BYTES + b'\x00' * 0x11c + b''.join(events)
+    parser = _traces_parser()
+    (unknown_process,) = parser.dict_traces(BytesIO(events_buf))
+    assert (unknown_process['pid'], unknown_process['process']) == (None, '')
+    # The thread's process, once the trace has seen it.
+    (parsed,) = parser.traces(BytesIO(events_buf))
+    parser.threads_pids[100] = 42
+    parser.pids_names[42] = 'probe'
+    trace = parser.trace_dict(parsed)
+    assert trace == {
+        'timestamp': 1000,
+        'time': '1970-01-01T00:00:00.000001+00:00',
+        'tid': 100,
+        'pid': 42,
+        'process': 'probe',
+        'event': 'BSC_open',
+        'fields': trace['fields'],
+        'text': trace['text'],
+    }
+    assert trace['text'].startswith('open(') and trace['text'].endswith('errno: ENOENT(2)')
+    assert json.loads(json.dumps(trace)) == trace  # JSON-compatible: enums by name, flags as lists of names
+
+
+def test_trace_dict_values():
+    from enum import Enum
+    from dataclasses import dataclass
+    from uuid import UUID
+    from pykdebugparser.trace_handlers import UnknownEnumValue
+
+    class Color(Enum):
+        RED = 1
+
+    @dataclass
+    class Inner:
+        ktraces: list
+        value: int
+
+    @dataclass
+    class Trace:
+        ktraces: list
+        color: Color
+        unknown: UnknownEnumValue
+        flags: list
+        uuid: UUID
+        data: bytes
+        inner: Inner
+
+    event = Kevent(timestamp=5, data=b'\x00' * 32, values=(0, 0, 0, 0), tid=7, debugid=0x40c0015, eventid=0x40c0014,
+                   func_qualifier=1)
+    trace = Trace([event], Color.RED, UnknownEnumValue(9), [Color.RED], UUID(int=1), b'\x01\x02', Inner([event], 3))
+    parser = PyKdebugParser()
+    assert parser.trace_dict(trace)['fields'] == {
+        'color': 'RED', 'unknown': 9, 'flags': ['RED'], 'uuid': '00000000-0000-0000-0000-000000000001',
+        'data': '0102', 'inner': {'value': 3},
+    }
+    # Without the device's time information there is no wall-clock time.
+    assert parser.trace_dict(trace)['time'] is None

pykdebugparser-1.5.0/pykdebugparser/highlight.py

@@ -1,134 +0,0 @@
-"""Syntax highlighting of trace lines, as pygments' C lexer and `stata-dark` true color formatter do it.
-
-pygments lexes each line through a general state machine, which formats about 19k lines/s: too slow for a live
-trace. Trace lines use a small part of C's syntax, so `highlight` tokenizes them with one regular expression that
-follows the same rules, and produces the same escape sequences. A line with anything outside that part (a label at its
-start, `;` or `{`, comments, character literals, string escapes, floating-point numbers, ...) goes to pygments, so the
-result is always pygments' own.
-"""
-import re
-from functools import lru_cache
-
-from pygments import highlight as pygments_highlight
-from pygments import lexers
-from pygments.formatters import TerminalTrueColorFormatter
-from pygments.token import Keyword, Name, Number, Operator, Punctuation, String, Whitespace
-
-c_lexer = lexers.CLexer()
-color_formatter = TerminalTrueColorFormatter(style='stata-dark')
-
-# pygments.lexers.c_cpp.CFamilyLexer's 'keywords', 'types' and builtins, the names it does not lex as `Name`.
-_KEYWORDS = {
-    'asm', 'auto', 'break', 'const', 'constexpr', 'continue', 'countof', 'default', 'defer', 'do', 'else', 'enum',
-    'extern', 'for', 'goto', 'if', 'register', 'restricted', 'return', 'sizeof', 'static', 'switch', 'typedef',
-    'typeof', 'typeof_unqual', 'volatile', 'while', 'thread_local', 'alignas', 'alignof', 'static_assert', '_Pragma',
-    'fortran',
-}
-_RESERVED = {'inline', '_inline', '__inline', 'naked', 'restrict', 'thread', '__int8', '__int16', '__int32',
-             '__int64', '__wchar_t', '__m128i', '__m128d', '__m128', '__m64'} | {
-    '__' + name for name in ('asm', 'based', 'except', 'stdcall', 'cdecl', 'fastcall', 'declspec', 'finally', 'try',
-                             'leave', 'w64', 'unaligned', 'raise', 'noop', 'identifier', 'forceinline', 'assume',
-                             'null')}
-_TYPES = {'bool', 'int', 'long', 'float', 'short', 'double', 'char', 'unsigned', 'signed', 'void', '_BitInt',
-          '__int128'}
-_BUILTINS = {'true', 'false', 'NULL', 'nullptr'}
-# Names that start another lexer state, or a string with an affix (`L"..."`): left to pygments.
-_STATEFUL = {'struct', 'union', 'case', 'L', 'u', 'U', 'u8'}
-
-_INT_SUFFIX = r'(?:[uU][lL]{0,2}|[lL]{1,2}[uU]?)?'
-_DEC = r'\d(?:\'?\d)*'
-_HEX = r'[0-9a-fA-F](?:\'?[0-9a-fA-F])*'
-# The alternatives are tried in the lexer's order at each token start. `unsupported` marks what pygments lexes
-# differently from this tokenizer (comments and floating-point numbers); characters no alternative matches (`'`, `\\`,
-# `#`, `$`, a string with an escape, ...) also send the line to pygments.
-_TOKEN = re.compile(
-    r'(?P<unsupported>/[/*]'
-    r'|0[xX](?:' + _HEX + r'\.' + _HEX + r'|\.' + _HEX + r'|' + _HEX + r')[pP]'
-    r'|-?(?:' + _DEC + r'\.' + _DEC + r'|\.' + _DEC + r'|' + _DEC + r')[eE][+-]?' + _DEC +
-    r'|-?(?:' + _DEC + r'\.|\.' + _DEC + r'|' + _DEC + r'[fFlL])'
-    r')'
-    r'|(?P<string>")(?P<content>[^\\"\n]*)(?P<close>")'
-    r'|(?P<hex>-?0[xX]' + _HEX + _INT_SUFFIX + r')'
-    r'|(?P<bin>-?0[bB][01](?:\'?[01])*' + _INT_SUFFIX + r')'
-    r'|(?P<oct>-?0(?:\'?[0-7])+' + _INT_SUFFIX + r')'
-    r'|(?P<int>-?' + _DEC + _INT_SUFFIX + r')'
-    r'|(?P<op>[~!%^&*+=|?:<>/-])'
-    r'|(?P<punct>[()\[\],.])'
-    r'|(?P<name>[a-zA-Z_]\w*)'
-    r'|(?P<ws>[^\S\n]+)'
-)
-# Whole-line rules of the lexer: a label at the start of a line, and functions (their rules need a `;` or `{`).
-_UNSUPPORTED = re.compile(r'^\s*[a-zA-Z_$]\w*\s*:(?!:)|[;{]')
-
-
-def _style(ttype):
-    # The formatter uses the style of the token type, or of its closest parent that has one.
-    while str(ttype) not in color_formatter.style_string and ttype.parent is not None:
-        ttype = ttype.parent
-    return color_formatter.style_string.get(str(ttype), ('', ''))
-
-
-_GROUPS = {'hex': _style(Number.Hex), 'bin': _style(Number.Bin), 'oct': _style(Number.Oct),
-           'int': _style(Number.Integer), 'op': _style(Operator), 'punct': _style(Punctuation)}
-_NAME_STYLES = {}
-for _names, _ttype in ((_KEYWORDS, Keyword), (_RESERVED, Keyword.Reserved), (_TYPES, Keyword.Type),
-                       (_BUILTINS, Name.Builtin)):
-    for _name in _names:
-        _NAME_STYLES[_name] = _style(_ttype)
-_NAME = _style(Name)
-_STRING = _style(String)
-# Whitespace only gets a foreground color, which a space does not show: it can stay in the style before it.
-_FG_RESET = '\x1b[39m'
-assert _style(Whitespace)[1] == _FG_RESET
-
-
-def _pygments(text: str) -> str:
-    return pygments_highlight(text, c_lexer, color_formatter).strip()
-
-
-# Traces repeat (the same calls with the same arguments): in a busy `--bsc` trace, 77% of the lines were among the
-# 16k most recent ones.
-@lru_cache(maxsize=1 << 14)
-def highlight(text: str) -> str:
-    """`text` colored as `pygments.highlight(text, CLexer(), TerminalTrueColorFormatter('stata-dark'))` shows it.
-
-    pygments starts and ends the escape sequence of every token, even between tokens of the same color, and colors
-    spaces. Here a sequence starts only where the color changes, which renders the same in about half the bytes (the
-    terminal is the bottleneck of a busy trace).
-    """
-    if _UNSUPPORTED.search(text):
-        return _pygments(text)
-    out = []
-    style = None  # the (on, off) in effect
-    pos = 0
-    for match in _TOKEN.finditer(text):
-        if match.start() != pos:
-            return _pygments(text)
-        pos = match.end()
-        kind = match.lastgroup
-        if kind == 'ws':
-            out.append(match.group())
-            continue
-        if kind == 'unsupported':
-            return _pygments(text)
-        if kind == 'close':
-            token_style = _STRING
-        elif kind == 'name':
-            name = match.group()
-            if name in _STATEFUL:
-                return _pygments(text)
-            token_style = _NAME_STYLES.get(name, _NAME)
-        else:
-            token_style = _GROUPS[kind]
-        if style is None or token_style[0] != style[0]:
-            # Ending a foreground color is not needed when the next one replaces it.
-            if style is not None and style[1] != _FG_RESET:
-                out.append(style[1])
-            out.append(token_style[0])
-            style = token_style
-        out.append(match.group())
-    if pos != len(text):
-        return _pygments(text)
-    if style is not None:
-        out.append(style[1])
-    return ''.join(out).strip()

pykdebugparser-1.5.0/tests/test_highlight.py

@@ -1,80 +0,0 @@
-import re
-
-import pytest
-from pygments import highlight as pygments_highlight
-
-from pykdebugparser import highlight
-
-
-@pytest.mark.parametrize('line', [
-    # The tokenizer's own lines.
-    'read(7, 0x11bf1c000, 25558), count: 25558',
-    'open("/private/var/x", O_RDONLY | O_CLOEXEC), fd: 3',
-    'open_nocancel("/System/Library/Foundation.framework//en.lproj", O_RDONLY), errno: ENOENT(2)',
-    'lookup(".tmp.OgDV0NXsrZ"), vnode id: 8638794530369352533',
-    'workq_kernreturn(256, 0x16d2c2b80, 1, 0), return: 0',
-    'sigreturn(0x16bd56ec8, 30, 0x17fcd6aa5d512e6c), errno: EJUSTRETURN(-2)',
-    'pread(12, 0xad80793d2, 4096, 0x31e000), count: 4096',
-    'getxattr("", NULL, 0x0, 0, int, true), size: 0x10',
-    'thread_suspend, tid: 55996, user_stop_count: 0, pidsuspended: True',
-    'mmap(0x0, 0755, 0b101, 10u, 3LL), count: 0x8e0000',
-    'DBG_DYLD_TIMING_OBJC_MAP',
-    # Lines left to pygments.
-    "ioctl(8, 0xc020699d /* _IOC(IOC_IN | IOC_OUT, 'i', 157, 32) */, 0x16d3cbea0)",
-    'TRACE_LOST_EVENTS: the kernel dropped trace events here',
-    'stat64("/private/var/Snapshots/com.apple.AppStore - {DEFAULT GROUP}", 0x16db399d0)',
-    'open("a\\"b"), fd: 3',
-    'struct x, case 1, L"wide", u8"x"',
-    'duration: 1.5, 2e10, .5, 3f, 7l',
-    'mixed $dollar # hash',
-])
-def test_highlight_renders_like_pygments(line):
-    assert _render(highlight.highlight(line)) == _render(
-        pygments_highlight(line, highlight.c_lexer, highlight.color_formatter).strip())
-
-
-def test_highlight_uses_fewer_escape_sequences():
-    line = 'read(7, 0x11bf1c000, 25558), count: 25558'
-    assert len(highlight.highlight(line)) < len(pygments_highlight(line, highlight.c_lexer,
-                                                                   highlight.color_formatter).strip()) / 2
-
-
-_SGR = re.compile(r'\x1b\[([0-9;]*)m')
-
-
-def _render(text):
-    """Each character with the terminal attributes it shows (a space shows only its background)."""
-    fg = bg = None
-    bold = italic = False
-    cells = []
-    pos = 0
-    for match in _SGR.finditer(text):
-        cells += [(char, fg, bg, bold, italic) for char in text[pos:match.start()]]
-        pos = match.end()
-        codes = [int(code) for code in match.group(1).split(';') if code] or [0]
-        i = 0
-        while i < len(codes):
-            code = codes[i]
-            if code == 0:
-                fg = bg = None
-                bold = italic = False
-            elif code == 1:
-                bold = True
-            elif code == 3:
-                italic = True
-            elif code == 39:
-                fg = None
-            elif code == 49:
-                bg = None
-            elif code in (38, 48) and codes[i + 1] == 2:
-                if code == 38:
-                    fg = tuple(codes[i + 2:i + 5])
-                else:
-                    bg = tuple(codes[i + 2:i + 5])
-                i += 4
-            else:
-                raise ValueError(f'unexpected SGR code {code}')
-            i += 1
-    cells += [(char, fg, bg, bold, italic) for char in text[pos:]]
-    return [(char, None, bg, False, False) if char == ' ' else (char, fg, bg, bold, italic)
-            for char, fg, bg, bold, italic in cells]