pykdebugparser 1.5.0__py2.py3-none-any.whl → 1.6.0__py2.py3-none-any.whl

pykdebugparser/highlight.py

@@ -1,134 +1,66 @@
-"""Syntax highlighting of trace lines, as pygments' C lexer and `stata-dark` true color formatter do it.
+"""Syntax highlighting of trace lines, such as `open("/etc/hosts", O_RDONLY), fd: 3`.
 
-pygments lexes each line through a general state machine, which formats about 19k lines/s: too slow for a live
-trace. Trace lines use a small part of C's syntax, so `highlight` tokenizes them with one regular expression that
-follows the same rules, and produces the same escape sequences. A line with anything outside that part (a label at its
-start, `;` or `{`, comments, character literals, string escapes, floating-point numbers, ...) goes to pygments, so the
-result is always pygments' own.
+Lines are colored with the terminal's 16 basic colors, so they follow its theme:
+
+- the call or event name at the start, bold;
+- strings (paths, symbols, ...), green;
+- numbers, cyan;
+- constants (flags, enums, signals: `O_RDONLY`, `KERN_SUCCESS`, `SIGKILL`), yellow;
+- an error (`errno: ENOENT(2)`, a `KERN_*` failure), red;
+- the labels of results and fields (`fd:`, `count:`), dim.
+
+A trace is mostly the same calls with the same arguments, so the most recent lines are cached.
 """
 import re
 from functools import lru_cache
 
-from pygments import highlight as pygments_highlight
-from pygments import lexers
-from pygments.formatters import TerminalTrueColorFormatter
-from pygments.token import Keyword, Name, Number, Operator, Punctuation, String, Whitespace
-
-c_lexer = lexers.CLexer()
-color_formatter = TerminalTrueColorFormatter(style='stata-dark')
+_RESET = '\x1b[0m'
+_BOLD = '\x1b[1m'
+_DIM = '\x1b[2m'
+_RED = '\x1b[31m'
+_GREEN = '\x1b[32m'
+_YELLOW = '\x1b[33m'
+_CYAN = '\x1b[36m'
 
-# pygments.lexers.c_cpp.CFamilyLexer's 'keywords', 'types' and builtins, the names it does not lex as `Name`.
-_KEYWORDS = {
-    'asm', 'auto', 'break', 'const', 'constexpr', 'continue', 'countof', 'default', 'defer', 'do', 'else', 'enum',
-    'extern', 'for', 'goto', 'if', 'register', 'restricted', 'return', 'sizeof', 'static', 'switch', 'typedef',
-    'typeof', 'typeof_unqual', 'volatile', 'while', 'thread_local', 'alignas', 'alignof', 'static_assert', '_Pragma',
-    'fortran',
-}
-_RESERVED = {'inline', '_inline', '__inline', 'naked', 'restrict', 'thread', '__int8', '__int16', '__int32',
-             '__int64', '__wchar_t', '__m128i', '__m128d', '__m128', '__m64'} | {
-    '__' + name for name in ('asm', 'based', 'except', 'stdcall', 'cdecl', 'fastcall', 'declspec', 'finally', 'try',
-                             'leave', 'w64', 'unaligned', 'raise', 'noop', 'identifier', 'forceinline', 'assume',
-                             'null')}
-_TYPES = {'bool', 'int', 'long', 'float', 'short', 'double', 'char', 'unsigned', 'signed', 'void', '_BitInt',
-          '__int128'}
-_BUILTINS = {'true', 'false', 'NULL', 'nullptr'}
-# Names that start another lexer state, or a string with an affix (`L"..."`): left to pygments.
-_STATEFUL = {'struct', 'union', 'case', 'L', 'u', 'U', 'u8'}
-
-_INT_SUFFIX = r'(?:[uU][lL]{0,2}|[lL]{1,2}[uU]?)?'
-_DEC = r'\d(?:\'?\d)*'
-_HEX = r'[0-9a-fA-F](?:\'?[0-9a-fA-F])*'
-# The alternatives are tried in the lexer's order at each token start. `unsupported` marks what pygments lexes
-# differently from this tokenizer (comments and floating-point numbers); characters no alternative matches (`'`, `\\`,
-# `#`, `$`, a string with an escape, ...) also send the line to pygments.
 _TOKEN = re.compile(
-    r'(?P<unsupported>/[/*]'
-    r'|0[xX](?:' + _HEX + r'\.' + _HEX + r'|\.' + _HEX + r'|' + _HEX + r')[pP]'
-    r'|-?(?:' + _DEC + r'\.' + _DEC + r'|\.' + _DEC + r'|' + _DEC + r')[eE][+-]?' + _DEC +
-    r'|-?(?:' + _DEC + r'\.|\.' + _DEC + r'|' + _DEC + r'[fFlL])'
-    r')'
-    r'|(?P<string>")(?P<content>[^\\"\n]*)(?P<close>")'
-    r'|(?P<hex>-?0[xX]' + _HEX + _INT_SUFFIX + r')'
-    r'|(?P<bin>-?0[bB][01](?:\'?[01])*' + _INT_SUFFIX + r')'
-    r'|(?P<oct>-?0(?:\'?[0-7])+' + _INT_SUFFIX + r')'
-    r'|(?P<int>-?' + _DEC + _INT_SUFFIX + r')'
-    r'|(?P<op>[~!%^&*+=|?:<>/-])'
-    r'|(?P<punct>[()\[\],.])'
-    r'|(?P<name>[a-zA-Z_]\w*)'
-    r'|(?P<ws>[^\S\n]+)'
+    r'(?P<string>"[^"]*")'
+    r'|(?P<number>-?0[xX][0-9a-fA-F]+|-?\d+)'
+    r'|(?P<label>[A-Za-z_][\w ]*?:)(?= |$)'
+    r'|(?P<name>[A-Za-z_]\w*)'
 )
-# Whole-line rules of the lexer: a label at the start of a line, and functions (their rules need a `;` or `{`).
-_UNSUPPORTED = re.compile(r'^\s*[a-zA-Z_$]\w*\s*:(?!:)|[;{]')
-
-
-def _style(ttype):
-    # The formatter uses the style of the token type, or of its closest parent that has one.
-    while str(ttype) not in color_formatter.style_string and ttype.parent is not None:
-        ttype = ttype.parent
-    return color_formatter.style_string.get(str(ttype), ('', ''))
+# A constant is all capitals, digits and underscores, with at least one capital.
+_CONSTANT = re.compile(r'[A-Z][A-Z0-9_]*')
+# `KERN_SUCCESS` is a result, not an error.
+_SUCCESS = {'KERN_SUCCESS'}
 
 
-_GROUPS = {'hex': _style(Number.Hex), 'bin': _style(Number.Bin), 'oct': _style(Number.Oct),
-           'int': _style(Number.Integer), 'op': _style(Operator), 'punct': _style(Punctuation)}
-_NAME_STYLES = {}
-for _names, _ttype in ((_KEYWORDS, Keyword), (_RESERVED, Keyword.Reserved), (_TYPES, Keyword.Type),
-                       (_BUILTINS, Name.Builtin)):
-    for _name in _names:
-        _NAME_STYLES[_name] = _style(_ttype)
-_NAME = _style(Name)
-_STRING = _style(String)
-# Whitespace only gets a foreground color, which a space does not show: it can stay in the style before it.
-_FG_RESET = '\x1b[39m'
-assert _style(Whitespace)[1] == _FG_RESET
-
-
-def _pygments(text: str) -> str:
-    return pygments_highlight(text, c_lexer, color_formatter).strip()
-
-
-# Traces repeat (the same calls with the same arguments): in a busy `--bsc` trace, 77% of the lines were among the
-# 16k most recent ones.
 @lru_cache(maxsize=1 << 14)
 def highlight(text: str) -> str:
-    """`text` colored as `pygments.highlight(text, CLexer(), TerminalTrueColorFormatter('stata-dark'))` shows it.
-
-    pygments starts and ends the escape sequence of every token, even between tokens of the same color, and colors
-    spaces. Here a sequence starts only where the color changes, which renders the same in about half the bytes (the
-    terminal is the bottleneck of a busy trace).
-    """
-    if _UNSUPPORTED.search(text):
-        return _pygments(text)
+    """`text` with terminal colors (SGR escape sequences); the text itself is unchanged."""
     out = []
-    style = None  # the (on, off) in effect
     pos = 0
+    error = False  # inside the value of an `errno:` result
+    first = True
     for match in _TOKEN.finditer(text):
-        if match.start() != pos:
-            return _pygments(text)
+        if match.start() > pos:
+            out.append(text[pos:match.start()])
         pos = match.end()
         kind = match.lastgroup
-        if kind == 'ws':
-            out.append(match.group())
-            continue
-        if kind == 'unsupported':
-            return _pygments(text)
-        if kind == 'close':
-            token_style = _STRING
-        elif kind == 'name':
-            name = match.group()
-            if name in _STATEFUL:
-                return _pygments(text)
-            token_style = _NAME_STYLES.get(name, _NAME)
+        value = match.group()
+        if kind == 'string':
+            out.append(_GREEN + value + _RESET)
+        elif kind == 'number':
+            out.append((_RED if error else _CYAN) + value + _RESET)
+        elif kind == 'label':
+            error = value == 'errno:'
+            out.append((_RED if error else _DIM) + value + _RESET)
+        elif first:
+            out.append(_BOLD + value + _RESET)
+        elif _CONSTANT.fullmatch(value):
+            failed = error or (value.startswith('KERN_') and value not in _SUCCESS)
+            out.append((_RED if failed else _YELLOW) + value + _RESET)
         else:
-            token_style = _GROUPS[kind]
-        if style is None or token_style[0] != style[0]:
-            # Ending a foreground color is not needed when the next one replaces it.
-            if style is not None and style[1] != _FG_RESET:
-                out.append(style[1])
-            out.append(token_style[0])
-            style = token_style
-        out.append(match.group())
-    if pos != len(text):
-        return _pygments(text)
-    if style is not None:
-        out.append(style[1])
-    return ''.join(out).strip()
+            out.append(value)
+        first = False
+    out.append(text[pos:])
+    return ''.join(out)

pykdebugparser/pykdebugparser.py

@@ -1,4 +1,6 @@
+import dataclasses
 from datetime import datetime
+from enum import Enum
 import io
 import math
 
@@ -9,6 +11,7 @@ from pykdebugparser.highlight import highlight
 from pykdebugparser.kd_buf_parser import KdBufParser
 from pykdebugparser.kevent import DgbFuncQual
 from pykdebugparser.trace_codes import default_trace_codes
+from pykdebugparser.trace_handlers import UnknownEnumValue
 from pykdebugparser.trace_handlers.trace import TRACE_LOST_EVENTS
 from pykdebugparser.traces_parser import TracesParser
 from pykdebugparser.os_log_event import OsLogEvent
@@ -19,6 +22,32 @@ DBG_FSYSTEM = 3
 DBG_BSD = 4
 
 
+def _fields(trace) -> dict:
+    """A trace's parsed values (its dataclass fields but the raw kevents), as JSON-compatible values."""
+    return {field.name: _jsonable(getattr(trace, field.name)) for field in dataclasses.fields(trace)
+            if field.name != 'ktraces'}
+
+
+def _jsonable(value):
+    if isinstance(value, (bool, str)) or value is None:
+        return value
+    if isinstance(value, UnknownEnumValue):
+        return int(value)
+    if isinstance(value, Enum):
+        return value.name
+    if isinstance(value, (int, float)):
+        return value
+    if isinstance(value, (bytes, bytearray)):
+        return value.hex()
+    if isinstance(value, (list, tuple)):
+        return [_jsonable(item) for item in value]
+    if isinstance(value, dict):
+        return {str(key): _jsonable(item) for key, item in value.items()}
+    if dataclasses.is_dataclass(value) and not isinstance(value, type):
+        return _fields(value)
+    return str(value)
+
+
 class PyKdebugParser:
     def __init__(self):
         self.filter_tid = None
@@ -99,6 +128,32 @@ class PyKdebugParser:
             trace_generator = filter(lambda t: t.ktraces[0].eventid >> 24 != DBG_FSYSTEM, trace_generator)
         return trace_generator
 
+    def dict_traces(self, kdebug: io.IOBase, trace_codes=None):
+        """The traces as JSON-compatible dicts (see `trace_dict`)."""
+        trace_codes_map = default_trace_codes() if trace_codes is None else trace_codes
+        return map(lambda t: self.trace_dict(t, trace_codes_map), self.traces(kdebug, trace_codes_map))
+
+    def trace_dict(self, trace, trace_codes=None) -> dict:
+        """
+        A trace as JSON-compatible data: when (`timestamp` in mach ticks, and `time` in ISO 8601 when the device's time
+        information is set), where (`tid`, `pid`, `process`), what (`event`, the trace code name of its first kevent,
+        and `fields`, the parsed values) and `text`, as `formatted_traces` prints it.
+        """
+        trace_codes_map = default_trace_codes() if trace_codes is None else trace_codes
+        first = trace.ktraces[0]
+        pid = self.threads_pids.get(first.tid)
+        posix_time = self._posix_time(first.timestamp)
+        return {
+            'timestamp': first.timestamp,
+            'time': None if posix_time is None else datetime.fromtimestamp(posix_time, tz=self.timezone).isoformat(),
+            'tid': first.tid,
+            'pid': pid,
+            'process': self.pids_names.get(pid, '') if pid is not None else '',
+            'event': trace_codes_map.get(first.eventid, hex(first.eventid)),
+            'fields': _fields(trace),
+            'text': str(trace),
+        }
+
     def formatted_traces(self, kdebug: io.IOBase, trace_codes=None):
         return map(lambda t: self._format_trace(t), self.traces(kdebug, trace_codes))
 
@@ -128,13 +183,19 @@ class PyKdebugParser:
         process_name = self.pids_names.get(pid, '')
         return self.filter_process == str(pid) or self.filter_process == process_name
 
-    def _format_timestamp(self, timestamp):
+    def _posix_time(self, timestamp):
+        """A kevent timestamp (mach ticks) as POSIX time, or None without the device's time information."""
         if None in (self.mach_absolute_time, self.numer, self.denom, self.usecs_since_epoch, self.timezone):
-            return str(timestamp) + ' '
+            return None
         offset_usec = (
                 ((timestamp - self.mach_absolute_time) * self.numer) / (self.denom * 1000)
         )
-        posix_time = (self.usecs_since_epoch + offset_usec) / 1000000
+        return (self.usecs_since_epoch + offset_usec) / 1000000
+
+    def _format_timestamp(self, timestamp):
+        posix_time = self._posix_time(timestamp)
+        if posix_time is None:
+            return str(timestamp) + ' '
         # Split to seconds and microseconds exactly like `datetime.fromtimestamp` does (round half to even), so
         # the date and time part can be formatted only once per second.
         fraction, seconds = math.modf(posix_time)

{pykdebugparser-1.5.0.dist-info → pykdebugparser-1.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pykdebugparser
-Version: 1.5.0
+Version: 1.6.0
 Summary: Python parser for kdebug events
 Author: Matan Perelman
 Author-email: matan1008@gmail.com
@@ -39,7 +39,6 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: bytestruct>=0.1.0
-Requires-Dist: pygments
 Requires-Dist: click
 Requires-Dist: termcolor
 Provides-Extra: testing

{pykdebugparser-1.5.0.dist-info → pykdebugparser-1.6.0.dist-info}/RECORD

@@ -1,11 +1,11 @@
 pykdebugparser/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 pykdebugparser/__main__.py,sha256=WfgLv5SGr0hQlUeM68ai1g0AqtYDgDFqYlbvvm5rua4,3711
 pykdebugparser/callstacks_parser.py,sha256=i2nfhx0olZZ65UV_ZV3Tw9E_6SWtre8CAbfRBSig4-E,1654
-pykdebugparser/highlight.py,sha256=i-mjfyak0VTo8P_7EeZLIUsUH08Qs-1yAjHBJVWW448,6439
+pykdebugparser/highlight.py,sha256=LqIf0kn8va0aE6tVGJyzu0MRnsR8o4OEJsYaUPXjDjM,2270
 pykdebugparser/kd_buf_parser.py,sha256=CpifFeBnQ_MZSokjO44lzhXO-KNnZvO-teaPyQqSfio,10991
 pykdebugparser/kevent.py,sha256=DudovIuES4QGxosYeEFJm3AW_FWGbJJBH1kcLnL7UFM,1779
 pykdebugparser/os_log_event.py,sha256=OQUlHpS44oUNwma_bxvnK0SKh_RFeEUdH1RmyrM4nLc,12356
-pykdebugparser/pykdebugparser.py,sha256=RipUcCh6dChgtdh_owvlprjLRMF1dvgic6ZyCqC5I8I,10320
+pykdebugparser/pykdebugparser.py,sha256=sx77jkvXEyEpmoW3XjANjTUZbm3BaOqRJjbQhIZxVxU,13052
 pykdebugparser/trace.codes,sha256=S6ZN5l9FSGrlSon8nKoVaqRfLdCPl3HL93XSjuoCUnM,103995
 pykdebugparser/trace_codes.py,sha256=4ImGc_jNpY-12cGXdwy6NMCEYcI24oYn1NBOaKNdWNo,1984
 pykdebugparser/traces_parser.py,sha256=LyHVTi4P2OeOLNG2Tahu1x7roQXkC-Jx8LS-E4mBcAQ,5799
@@ -17,9 +17,9 @@ pykdebugparser/trace_handlers/mach.py,sha256=_2YEgUfKYxaB7NfA2jCVxUbdKDrU21KJKhx
 pykdebugparser/trace_handlers/perf.py,sha256=CBffheZFlHa9IOVUZ-UcjNtKmjamv-un44bRJYYo9dw,5070
 pykdebugparser/trace_handlers/trace.py,sha256=s3Kr0wDvWZL2JZVKhxcvA1uvqsdePlEWiXfqSI76znw,5797
 pykdebugparser/trace_handlers/turnstile.py,sha256=8zZfba2Obq7a6qdhLIkItmZxRzzOe1PTTeHnrLuJUx0,8233
-pykdebugparser-1.5.0.dist-info/licenses/LICENSE,sha256=KjZXYP_CGYzDTtVfcxPT6wlGRoQKHPSJu0vh55_2QSo,1071
-pykdebugparser-1.5.0.dist-info/METADATA,sha256=biG_5LGv2RvIABkToh09hXR3eQ8Pm6BUlsI59SZPvdA,18995
-pykdebugparser-1.5.0.dist-info/WHEEL,sha256=4YBfCYNH4wlLpv3pzq1hbEuIlXA4WJabKLFurZ7eTL0,109
-pykdebugparser-1.5.0.dist-info/entry_points.txt,sha256=NrVIOgPR6v2MbpTZ3aKedP1jQ93nUQus6sgP4wAD0_E,63
-pykdebugparser-1.5.0.dist-info/top_level.txt,sha256=FcHHXR9VKmqufFw0a6UP4c8NyS-u4ORY-yXQ6ZzP_28,15
-pykdebugparser-1.5.0.dist-info/RECORD,,
+pykdebugparser-1.6.0.dist-info/licenses/LICENSE,sha256=KjZXYP_CGYzDTtVfcxPT6wlGRoQKHPSJu0vh55_2QSo,1071
+pykdebugparser-1.6.0.dist-info/METADATA,sha256=U2COUKhE9WYKZlpGQulRUvrspyB7k0qMgKMmKg5S3l0,18971
+pykdebugparser-1.6.0.dist-info/WHEEL,sha256=4YBfCYNH4wlLpv3pzq1hbEuIlXA4WJabKLFurZ7eTL0,109
+pykdebugparser-1.6.0.dist-info/entry_points.txt,sha256=NrVIOgPR6v2MbpTZ3aKedP1jQ93nUQus6sgP4wAD0_E,63
+pykdebugparser-1.6.0.dist-info/top_level.txt,sha256=FcHHXR9VKmqufFw0a6UP4c8NyS-u4ORY-yXQ6ZzP_28,15
+pykdebugparser-1.6.0.dist-info/RECORD,,