Chronoscope 0.0.4.dev1__tar.gz → 0.0.5.dev0__tar.gz

{chronoscope-0.0.4.dev1 → chronoscope-0.0.5.dev0/Chronoscope.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: Chronoscope
-Version: 0.0.4.dev1
+Version: 0.0.5.dev0
 Summary: A cross-platform matplotlib-based observability tool
 Home-page: https://github.com/just-now/chronoscope
 Author: Anatoliy Bilenko

{chronoscope-0.0.4.dev1/Chronoscope.egg-info → chronoscope-0.0.5.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: Chronoscope
-Version: 0.0.4.dev1
+Version: 0.0.5.dev0
 Summary: A cross-platform matplotlib-based observability tool
 Home-page: https://github.com/just-now/chronoscope
 Author: Anatoliy Bilenko

{chronoscope-0.0.4.dev1 → chronoscope-0.0.5.dev0}/README.md

@@ -1,5 +1,6 @@
 ![pytest/systest workflow](https://github.com/just-now/chronoscope/actions/workflows/python-package.yml/badge.svg)
 ![systest workflow](https://github.com/just-now/chronoscope/actions/workflows/makefile.yml/badge.svg)
+[![Open Raft demo in Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/just-now/chronoscope/blob/feat/chronoscope-event-relation/examples/raft_demo.ipynb)
 
 # chronoscope
 Chronoscope, a cross-platform matplotlib-based observability tool

{chronoscope-0.0.4.dev1 → chronoscope-0.0.5.dev0}/chronoscope/__init__.py

@@ -19,7 +19,7 @@ import argparse as arg
 import errno
 import sys
 
-__version__ = '0.0.4.dev1'
+__version__ = '0.0.5.dev0'
 __author__ = 'Anatoliy Bilenko <anatoliy.bilenko@gmail.com>'
 __license__ = 'LGPLv3'
 
@@ -47,6 +47,8 @@ def parse_args():
     parser.add_argument("-t", "--trace", type=str, help="User's traces")
     parser.add_argument("-D", "--depth", type=int, default=50,
                         help="limits output to given level of ticks")
+    parser.add_argument("-r", "--reverse", action='store_true',
+                        help="follow state-machine relations to parents")
     parser.add_argument("command", type=str,
                         choices=["create", "chart", "tree",
                                  "hist", "queue", "vcd"],
@@ -73,7 +75,8 @@ def main() -> int:
                 db.close()
             case "chart":
                 db.open(args.db, db_options, verbose=args.verbose)
-                chart.plot(args.tick_id, args.fig_size, args.depth)
+                chart.plot(args.tick_id, args.fig_size, args.depth,
+                           args.reverse)
             case "tree":
                 db.open(args.db, db_options, verbose=args.verbose)
                 tree.plot(args.tick_id, args.depth)

{chronoscope-0.0.4.dev1 → chronoscope-0.0.5.dev0}/chronoscope/chart.py

@@ -12,34 +12,30 @@ import chronoscope.utils as utils
 import matplotlib.cm as cm              # type: ignore
 import matplotlib.pyplot as pt          # type: ignore
 import matplotlib.ticker as ticker      # type: ignore
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 import sys
 
 
 Y_LINE_SPACING = 2
 X_TICKS_MAX = 5
 
-def plot_timeline(timeline, y_pos: int) -> list[dict]:
-    ret = []
+def plot_timeline(timeline, y_pos: int):
     y_pos_scaled = -Y_LINE_SPACING * y_pos
 
     for current, current_tick in enumerate(timeline[:-1]):
         next_tick = timeline[current + 1]
         start_time, end_time = current_tick["time"], next_tick["time"]
-        event_label = current_tick["event"]
+        event_label = current_tick["name"]
 
         pt.hlines(y_pos_scaled, start_time, end_time, lw=4,
                   colors=cm.tab10(current % 7))  # type: ignore[attr-defined]
-        pt.text(start_time, y_pos_scaled, event_label, rotation=45)
-        ret += [{"x": start_time, "y": y_pos_scaled, "eid": current_tick["eid"]}]
+        pt.text(start_time, y_pos_scaled, event_label, rotation=90)
 
     if len(timeline[:]) == 1:
         pt.hlines(y_pos_scaled, timeline[0]["time"], timeline[0]["time"])
 
     pt.text(timeline[-1]["time"], y_pos_scaled,
-            timeline[-1]["event"], rotation=45)
-    ret += [{"x": timeline[-1]["time"], "y": y_pos_scaled, "eid": timeline[-1]["eid"]}]
-    return ret
+            timeline[-1]["name"], rotation=90)
 
 @dataclass
 class timeline_visitor:
@@ -47,18 +43,56 @@ class timeline_visitor:
     y_pos: int
     x_min: int
     x_max: int
-    ret: list[dict]
+    # sm_id -> y_pos, for drawing event_relation arrows
+    sm_to_y: dict = field(default_factory=dict)
+    # (from_time, from_y, to_time, to_y) pairs for thin arrows
+    arrows: list = field(default_factory=list)
 
     def __call__(self, timeline: list[dict], origin: int, parent: None | int):
-        self.ret += plot_timeline(timeline, self.y_pos)
-        self.y_pos += 1
+        if not timeline:
+            self.sm_to_y[origin] = self.y_pos
+            self.y_pos += 1
+            self.y_labels.append(f"?{utils.format_event_id(origin)} [empty]")
+            return
+        plot_timeline(timeline, self.y_pos)
+        self.sm_to_y[origin] = self.y_pos
         duration = round((timeline[-1]["time"] - timeline[0]["time"]) / 1e6, 3)
-        type, id = timeline[0]["type"], timeline[0]["id"]
-        self.y_labels.append(f"{type}{utils.unpack(id)} [{duration}ms]")
+        sm_type = timeline[0].get("sm_type", "?")
+        label = f"{sm_type}{utils.format_event_id(origin)} [{duration}ms]"
+        self.y_labels.append(label)
 
         times = [tick["time"] for tick in timeline]
         self.x_min = min(self.x_min, min(times))
         self.x_max = max(self.x_max, max(times))
+        self.y_pos += 1
+
+    def collect_arrows(self):
+        """Query event_relation for (from_sm, from_time) -> (to_sm, to_time).
+
+        The sender's state machine and time come from the send event referenced
+        by each receive relation's from_event_id."""
+        sql = """
+        SELECT s.state_machine_id AS from_sm, s.time AS from_time,
+               r.to_sm_id AS to_sm,   r.to_time AS to_time
+        FROM event_relation r
+        JOIN event s ON s.id = r.from_event_id
+        WHERE r.from_event_id IS NOT NULL
+        """
+        for from_sm, from_time, to_sm, to_time in db.db.execute_sql(sql).fetchall():
+            if from_sm in self.sm_to_y and to_sm in self.sm_to_y:
+                self.arrows.append((
+                    from_time, -Y_LINE_SPACING * self.sm_to_y[from_sm],
+                    to_time,   -Y_LINE_SPACING * self.sm_to_y[to_sm],
+                ))
+
+def plot_arrows(arrows: list):
+    for from_time, from_y, to_time, to_y in arrows:
+        pt.annotate("",
+                    xy=(to_time, to_y), xytext=(from_time, from_y),
+                    arrowprops=dict(arrowstyle="-|>", color="navy",
+                                    lw=0.6, alpha=0.7,
+                                    shrinkA=2, shrinkB=2),
+                    zorder=10)
 
 class chart_annotation:
     def __init__(self, fig):
@@ -92,7 +126,7 @@ class chart_annotation:
         cursor = chr(self.cur_mark)
         if self.cur_mark % 2 == 0:
             cursor = chr(self.cur_mark - 1) + cursor + ": "
-            cursor += utils.str_ns_diff(int(abs(self.cur[-1] - self.cur[-2])))
+            cursor += utils.str_us_diff(int(abs(self.cur[-1] - self.cur[-2])))
         self.cur_mark += 1
 
         self.ann.append(ax.axvline(x=x, color="lightgray"))
@@ -101,24 +135,15 @@ class chart_annotation:
         self.ann_mode = not self.ann_mode
         event.canvas.draw()
 
-def plot(origin: int, figsize=(16, 4), depth_max=50):
+def plot(origin: int, figsize=(16, 4), depth_max=50, reverse=False):
     fig = pt.figure(figsize=figsize)
     pt.style.use("bmh")
     pt.rcParams["font.size"] = 8
     pt.subplots_adjust(top=0.75)
 
-    v = timeline_visitor([], 0, utils.MAX_INT, utils.MIN_INT, [])
-    db.iterate(origin, None, db.tick, v, 0, depth_max)
-
-    ev_relations = db.iterate_ev_relations([x["eid"] for x in v.ret])
-    ev_eid_to_xy = {}
-    for erel in v.ret:
-        ev_eid_to_xy[erel["eid"]] = (erel["x"], erel["y"])
-
-    for (orig, dest) in ev_relations:
-        (x0, y0) = ev_eid_to_xy[orig]
-        (x1, y1) = ev_eid_to_xy[dest]
-        pt.arrow(x0, y0, x1 - x0, y1 - y0, color='red')
+    v = timeline_visitor([], 0, utils.MAX_INT, utils.MIN_INT)
+    db.iterate(origin, None, v, 0, depth_max, reverse)
+    v.collect_arrows()
 
     end = -Y_LINE_SPACING * v.y_pos
     y_range = [float(x) for x in range(0, end, -Y_LINE_SPACING)]
@@ -131,14 +156,18 @@ def plot(origin: int, figsize=(16, 4), depth_max=50):
     pt.gca().xaxis.set_major_formatter(ticker.FuncFormatter(
         lambda value, pos: utils.str_ns(value, compact=True)))
 
+    plot_arrows(v.arrows)
     pt.yticks(y_range, v.y_labels)
     pt.xlabel("Time")
     pt.autoscale(enable=True, axis="x", tight=True)
     pt.margins(0.1)
-    _ = chart_annotation(fig)
+    # Keep the callback owner alive after non-blocking show() returns in
+    # notebook backends such as ipympl. Matplotlib stores weak references to
+    # bound-method callbacks.
+    setattr(fig, "_chronoscope_annotation", chart_annotation(fig))
 
     pt.grid(True)
-    title = f"Request {utils.unpack(origin)}\n"
+    title = f"Request {utils.format_event_id(origin)}\n"
     title += f"[{utils.str_ns(x_range[0])}...{utils.str_ns(x_range[-1])}]"
     pt.suptitle(title)
     pt.show()

chronoscope-0.0.5.dev0/chronoscope/db.py

@@ -0,0 +1,188 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Chronoscope.
+#
+# SPDX-FileCopyrightText: 2024 Anatoliy Bilenko <anatoliy.bilenko@gmail.com>
+#
+# SPDX-License-Identifier: LGPL-3.0-only
+#
+
+import chronoscope.parser as pr
+from typing import Callable, cast
+import subprocess as sp
+import builtins as b
+import peewee as p
+import os
+
+db = p.SqliteDatabase(None)
+
+
+class state_machine(p.Model):
+    id = p.IntegerField()
+    name = p.TextField()
+    type = p.TextField(null=True)
+
+    class Meta:
+        database = db
+        primary_key = p.CompositeKey("id")
+
+
+class event(p.Model):
+    id = p.IntegerField()
+    state_machine_id = p.IntegerField()
+    time = p.IntegerField()
+    name = p.TextField()
+
+    class Meta:
+        database = db
+        primary_key = p.CompositeKey("id")
+
+
+class event_relation(p.Model):
+    from_event_id = p.IntegerField(null=True)
+    to_event_id = p.IntegerField()
+    from_sm_id = p.IntegerField(null=True)
+    from_time = p.IntegerField(null=True)
+    to_sm_id = p.IntegerField()
+    to_time = p.IntegerField()
+    relation = p.TextField()
+
+    class Meta:
+        database = db
+        primary_key = p.CompositeKey("from_event_id", "to_event_id", "relation")
+
+
+class state_machine_relation(p.Model):
+    from_sm_id = p.IntegerField()
+    to_sm_id = p.IntegerField()
+    relation = p.TextField()
+
+    class Meta:
+        database = db
+        primary_key = p.CompositeKey("from_sm_id", "to_sm_id", "relation")
+
+
+class state_machine_attribute(p.Model):
+    state_machine_id = p.IntegerField()
+    key = p.TextField()
+    value = p.TextField()
+
+    class Meta:
+        database = db
+        primary_key = p.CompositeKey("state_machine_id", "key")
+
+
+class event_attribute(p.Model):
+    event_id = p.IntegerField()
+    key = p.TextField()
+    value = p.TextField()
+
+    class Meta:
+        database = db
+        primary_key = p.CompositeKey("event_id", "key")
+
+
+TABLES = [state_machine, event, event_relation,
+          state_machine_relation, state_machine_attribute, event_attribute]
+VERBOSE = False
+
+
+def open(path: str, opts: None | dict[str, int | str] = None, create=False,
+         verbose=False):
+    global VERBOSE
+    VERBOSE = verbose
+
+    if create and os.path.exists(path):
+        raise FileExistsError(f"`{path}' exists!")
+    if not create and not os.path.exists(path):
+        raise FileNotFoundError(f"`{path}' not found!")
+
+    db.init(path, opts)
+    db.connect()
+    if create:
+        with db:
+            db.create_tables(TABLES)
+
+
+def close():
+    db.close()
+
+
+def mkidx():
+    db.execute_sql("CREATE INDEX event_sm_idx on event(state_machine_id);")
+    db.execute_sql("CREATE INDEX event_time_idx on event(time);")
+    db.execute_sql("CREATE INDEX event_relation_from_idx on event_relation(from_event_id);")
+    db.execute_sql("CREATE INDEX event_relation_to_idx on event_relation(to_event_id);")
+    db.execute_sql("CREATE INDEX sm_relation_from_idx on state_machine_relation(from_sm_id);")
+    db.execute_sql("CREATE INDEX sm_relation_to_idx on state_machine_relation(to_sm_id);")
+
+
+def line_nr(file: str) -> int:
+    result = sp.run(['wc', file], stdout=sp.PIPE, text=True)
+    return int(result.stdout.split()[0])
+
+
+def load(pr: pr.parser, trace_path: str, fd_chunk_size=900, db_chunk_size=100):
+    if not os.path.exists(trace_path):
+        raise FileNotFoundError("`{trace_path}' not found!")
+
+    with b.open(trace_path) as fd:
+        for fd_chunk in p.chunked(fd, fd_chunk_size):
+            records = pr.parse(fd_chunk)
+            with db.atomic():
+                for table in TABLES:
+                    t_name: str = table._meta.name  # type: ignore
+                    if t_name not in records:
+                        continue
+                    for db_chunk in p.chunked(records[t_name], db_chunk_size):
+                        table.insert_many(db_chunk).on_conflict_ignore().execute()
+
+
+def iterate(origin: int, parent: None | int,
+            visit: Callable, depth: int, depth_max: int, reverse=False):
+    if depth_max < depth:
+        return
+
+    # pull events of this state machine, enriched with sm_type
+    timeline = (event
+                .select(event, state_machine.type.alias("sm_type"))
+                .join(state_machine,
+                      on=(event.state_machine_id == state_machine.id))
+                .where(event.state_machine_id == origin)
+                .dicts())
+    visit(list(timeline), origin, parent)
+
+    relation_column = (state_machine_relation.to_sm_id if reverse
+                       else state_machine_relation.from_sm_id)
+    related_column = (state_machine_relation.from_sm_id if reverse
+                      else state_machine_relation.to_sm_id)
+    relations = state_machine_relation.select().where(
+        relation_column == origin)
+    for relation in relations.dicts():
+        relation_data = cast(dict[str, int], relation)
+        related = relation_data[related_column.name]
+        if VERBOSE:
+            print(f"@[{depth}] {hex(origin)} ... {hex(related)}")
+        iterate(related, origin, visit, depth + 1, depth_max, reverse)
+
+
+def spans(event_begin: str, event_end: str, sm_type: str) -> list:
+    sql = f"""
+    SELECT (e2.time - e1.time) FROM event e1
+    JOIN event e2 ON e2.state_machine_id = e1.state_machine_id
+    JOIN state_machine sm ON sm.id = e1.state_machine_id
+    WHERE e1.name="{event_begin}" AND e2.name="{event_end}"
+    AND sm.type="{sm_type}";
+    """
+    return db.execute_sql(sql).fetchall()
+
+
+def queues(event_begin: str, event_end: str, sm_type: str) -> list:
+    sql = f"""
+    SELECT (time/1000)*1000 as timer,
+    COUNT(CASE WHEN name = "{event_begin}" THEN 1 END) as cc1,
+    COUNT(CASE WHEN name = "{event_end}" THEN 1 END) as cc2
+    FROM event JOIN state_machine sm ON sm.id = event.state_machine_id
+    WHERE sm.type="{sm_type}" GROUP BY timer;
+    """
+    return db.execute_sql(sql).fetchall()

chronoscope-0.0.5.dev0/chronoscope/parser.py

@@ -0,0 +1,175 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Chronoscope.
+#
+# SPDX-FileCopyrightText: 2024 Anatoliy Bilenko <anatoliy.bilenko@gmail.com>
+#
+# SPDX-License-Identifier: LGPL-3.0-only
+#
+
+import chronoscope.utils as u
+from typing import Callable
+import yaml
+import sys
+
+
+class parser:
+    def __init__(self, conf_path: str, verbose=False):
+        # type -> (dest_table, parser_func)
+        self.parsers: dict[str, tuple[str, Callable]] = {}
+        # the parser knows about table names
+        self.tables = ["state_machine", "event", "event_relation",
+                       "state_machine_relation", "event_attribute"]
+        self.verbose = verbose
+        self.load_config(conf_path)
+
+    def load_config(self, conf_path: str):
+        with open(conf_path) as fd:
+            conf = yaml.safe_load(fd)
+            conf_tables = [t for t in conf.keys() if t[0] != '.']
+            if not all(t in self.tables for t in conf_tables):
+                raise SyntaxError(f"a few of {conf_tables} aren't known!")
+
+            for table in conf_tables:
+                for trace in conf[table]:
+                    self.register_parser(table, trace["type"],
+                                         self.make_parser(table, trace["pos"]))
+
+    def make_parser(self, table: str, kwargs: dict[str, int]) -> Callable:
+        match table:
+            case "state_machine":
+                return self.make_state_machine_parser(**kwargs)
+            case "event":
+                return self.make_event_parser(**kwargs)
+            case "event_relation":
+                return self.make_event_rel_parser(**kwargs)
+            case "state_machine_relation":
+                return self.make_sm_rel_parser(**kwargs)
+            case "event_attribute":
+                return self.make_event_attribute_parser(**kwargs)
+        raise NotImplementedError()
+
+    def make_event_parser(self, type: int, time: int, event: int,
+                           sm_id: int) -> Callable:
+        def parse(line: list[str], parse_type: str):
+            if type >= len(line) or parse_type != line[type]:
+                return None
+            eid = None
+            for token in line:
+                if token.startswith("eid="):
+                    eid = int(token.split("=", 1)[1], 16)
+                    break
+            if eid is None:
+                return None
+            sm = int(line[sm_id], 0)
+            return {
+                "state_machine": {"id": sm, "name": parse_type, "type": parse_type},
+                "event": {"id": eid, "state_machine_id": sm,
+                          "time": u.ns(line[time]), "name": line[event]},
+            }
+        return parse
+
+    def make_event_rel_parser(self, type: int, sm_id: int,
+                              peid: int, eid: int, time: int) -> Callable:
+        def parse(line: list[str], parse_type: str):
+            if type >= len(line) or parse_type != line[type]:
+                return None
+            raw_peid = line[peid].split("=", 1)[1]
+            from_eid = None if raw_peid == "None" else int(raw_peid, 16)
+            to_eid = int(line[eid].split("=", 1)[1], 16)
+            sm = int(line[sm_id], 0)
+            ts = u.ns(line[time])
+            return {
+                "event_relation": {
+                    "from_event_id": from_eid,
+                    "to_event_id": to_eid,
+                    "from_sm_id": sm if from_eid is not None else None,
+                    "from_time": ts if from_eid is not None else None,
+                    "to_sm_id": sm,
+                    "to_time": ts,
+                    "relation": parse_type,
+                },
+            }
+        return parse
+
+    def make_state_machine_parser(self, type: int, time: int,
+                                  sm_id: int, name: int, state: int) -> Callable:
+        def parse(line: list[str], parse_type: str):
+            if type >= len(line) or parse_type != line[type]:
+                return None
+            sm = int(line[sm_id].split("=", 1)[1], 0)
+            raw_name = line[name].split("=", 1)[1]
+            raw_state = line[state].split("=", 1)[1]
+            eid = None
+            for token in line:
+                if token.startswith("eid="):
+                    eid = int(token.split("=", 1)[1], 16)
+                    break
+            if eid is None:
+                return None
+            return {
+                "state_machine": {"id": sm, "name": raw_name, "type": raw_name},
+                "event": {"id": eid, "state_machine_id": sm,
+                          "time": u.ns(line[time]), "name": raw_state},
+            }
+        return parse
+
+    def make_sm_rel_parser(self, type: int,
+                           from_sm_id: int, to_sm_id: int,
+                           relation: int) -> Callable:
+        def parse(line: list[str], parse_type: str):
+            if type >= len(line) or parse_type != line[type]:
+                return None
+            from_sm = int(line[from_sm_id].split("=", 1)[1], 0)
+            to_sm = int(line[to_sm_id].split("=", 1)[1], 0)
+            rel = line[relation].split("=", 1)[1]
+            record = {
+                "state_machine_relation": {
+                    "from_sm_id": from_sm,
+                    "to_sm_id": to_sm,
+                    "relation": rel,
+                }
+            }
+            if rel == "top-to-raft":
+                record["state_machine"] = {"id": from_sm, "name": "top", "type": "top"}
+            return record
+        return parse
+
+    def make_event_attribute_parser(self, type: int,
+                                    eid: int, attribute: int) -> Callable:
+        def parse(line: list[str], parse_type: str):
+            if type >= len(line) or parse_type != line[type]:
+                return None
+            event_id = int(line[eid].split("=", 1)[1], 16)
+            key, value = line[attribute].split("=", 1)
+            return {
+                "event_attribute": {
+                    "event_id": event_id,
+                    "key": key,
+                    "value": value,
+                },
+            }
+        return parse
+
+    def register_parser(self, dest_table: str, type: str, parse: Callable):
+        self.parsers[type] = (dest_table, parse)
+
+    def parse(self,
+              fd_chunk: list[str]) -> dict[str, list[dict[str, str | int]]]:
+        records: dict[str, list] = {t: [] for t in self.tables}
+        for line in fd_chunk:
+            for p_name, (dest_table, parser) in self.parsers.items():
+                try:
+                    if record := parser(line.split(), p_name):
+                        self._merge_record(records, record)
+                except Exception as e:
+                    if self.verbose:
+                        print(f"{e}: {line=}", file=sys.stderr)
+        return records
+
+    def _merge_record(self, records: dict, record: dict):
+        for table, payload in record.items():
+            if isinstance(payload, list):
+                records[table].extend(payload)
+            else:
+                records[table].append(payload)

chronoscope-0.0.5.dev0/chronoscope/tree.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of Chronoscope.
+#
+# SPDX-FileCopyrightText: 2024 Anatoliy Bilenko <anatoliy.bilenko@gmail.com>
+#
+# SPDX-License-Identifier: LGPL-3.0-only
+#
+
+import chronoscope.db as db
+import chronoscope.utils as utils
+from graphviz import Graph  # type: ignore
+from typing import cast
+
+# https://graphviz.readthedocs.io/en/stable/examples.html#structs-py
+FMT_NODE = """<
+<table border="0" cellborder="1" cellspacing="0">
+<tr><td>{}</td></tr><tr><td>{}</td></tr></table>>
+"""
+
+class attr_visitor:
+    def __init__(self, graph: Graph):
+        self.graph = graph
+
+    def __call__(self, timeline: list[dict], current: int,
+                 parent: None | int):
+        machine = cast(dict[str, object],
+                       (db.state_machine
+                        .select()
+                        .where(db.state_machine.id == current)
+                        .dicts()
+                        .get()))
+        attributes = (db.state_machine_attribute
+                      .select()
+                      .where(db.state_machine_attribute.state_machine_id == current)
+                      .dicts())
+        rows = [f"name={machine['name']}", f"type={machine['type']}"]
+        for attribute in attributes:
+            attribute_data = cast(dict[str, object], attribute)
+            rows.append(f"{attribute_data['key']}={attribute_data['value']}")
+        contents = "<br/>".join(rows)
+        label = FMT_NODE.format(utils.format_event_id(current), contents)
+        self.graph.node(str(current), label)
+
+        if parent is not None:
+            self.graph.edge(str(parent), str(current))
+
+def plot(origin: int, depth_max=50):
+    g = Graph(strict=True, format="png", node_attr={"shape": "plaintext"})
+    db.iterate(origin, None, attr_visitor(g), 0, depth_max)
+    net_pid, boot_counter, id = utils.unpack_event_id(origin)
+    g.render(f"tree_{net_pid}_{boot_counter}_{id}", cleanup=True)

{chronoscope-0.0.4.dev1 → chronoscope-0.0.5.dev0}/chronoscope/utils.py

@@ -17,6 +17,9 @@ MAX_INT = sys.maxsize
 MIN_INT = -sys.maxsize - 1
 BITS_PER_PID = 16
 DB_INT_SIZE = 64
+NET_PID_BITS = 4
+BOOT_COUNTER_BITS = 12
+EVENT_COUNTER_BITS = 48
 
 
 def pack_unpack_init(p_bits_per_pid: int, p_db_int_size: int):
@@ -35,6 +38,17 @@ def unpack(id_pid: int) -> tuple[int, int]:
     id = ((1 << (DB_INT_SIZE - BITS_PER_PID)) - 1) & id_pid
     return pid, id
 
+def unpack_event_id(event_id: int) -> tuple[int, int, int]:
+    net_pid = event_id >> (BOOT_COUNTER_BITS + EVENT_COUNTER_BITS)
+    boot_mask = (1 << BOOT_COUNTER_BITS) - 1
+    boot_counter = (event_id >> EVENT_COUNTER_BITS) & boot_mask
+    counter = event_id & ((1 << EVENT_COUNTER_BITS) - 1)
+    return net_pid, boot_counter, counter
+
+def format_event_id(event_id: int) -> str:
+    net_pid, boot_counter, counter = unpack_event_id(event_id)
+    return f"({net_pid} {boot_counter} {counter})"
+
 def ns(time: str) -> int:
     if len(time) != NS_TIME_LEN:
         raise ValueError("Not a nanosecond time format")
@@ -47,5 +61,5 @@ def str_ns(unix_time_ns: int, compact=False) -> str:
         return dt.strftime(FMT_MS_COMPACT)
     return dt.strftime(FMT_MS)
 
-def str_ns_diff(unix_time_ns: int) -> str:
-    return str(unix_time_ns) + "ns"
+def str_us_diff(unix_time_ns: int) -> str:
+    return str(unix_time_ns // 1_000) + "us"

{chronoscope-0.0.4.dev1 → chronoscope-0.0.5.dev0}/chronoscope/vcd.py

@@ -21,14 +21,18 @@ class timeline_visitor:
         self.timetable = timetable
 
     def __call__(self, timeline: list[dict], origin: int, parent: None | int):
+        if not timeline:
+            return
+
         t0 = timeline[0]
-        var = "{}_{}_{}".format(t0["type"], *u.unpack(t0["id"]))
+        var = "{}_{}_{}_{}".format(
+            t0["sm_type"], *u.unpack_event_id(origin))
 
         ctr = self.writer.register_var("c", var, "string")
         self.gtkw.trace(f"c.{var}")
 
         self.counters[var] = ctr
-        self.timetable += [{"var": var, "time": t["time"], "event": t["event"]}
+        self.timetable += [{"var": var, "time": t["time"], "event": t["name"]}
                            for t in timeline]
 
 
@@ -42,9 +46,11 @@ def plot(origin: int, depth_max=50):
             counters: dict = {}
             timetable: list[dict] = []
             v = timeline_visitor(gtkw, writer, counters, timetable)
-            db.iterate(origin, None, db.tick, v, 0, depth_max)
+            db.iterate(origin, None, v, 0, depth_max)
 
             timetable.sort(key=lambda t: t["time"])
+            if not timetable:
+                return
             t0 = timetable[0]["time"]
             for t in timetable:
                 writer.change(counters[t["var"]], t["time"] - t0, t["event"])

chronoscope-0.0.4.dev1/chronoscope/db.py

@@ -1,125 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# This file is part of Chronoscope.
-#
-# SPDX-FileCopyrightText: 2024 Anatoliy Bilenko <anatoliy.bilenko@gmail.com>
-#
-# SPDX-License-Identifier: LGPL-3.0-only
-#
-
-import chronoscope.parser as pr
-from typing import Callable
-import subprocess as sp
-import builtins as b
-import peewee as p
-import os
-
-db = p.SqliteDatabase(None)
-
-class T(p.Model):
-    class Meta:
-        database = db
-        primary_key = False
-
-class tick(T):
-    id = p.IntegerField()
-    eid = p.IntegerField()
-    time = p.IntegerField()
-    event = p.TextField()
-    type = p.TextField()
-
-class attr(T):
-    id = p.IntegerField()
-    name = p.TextField()
-    val = p.TextField()
-
-class relation(p.Model):
-    orig = p.IntegerField()
-    dest = p.IntegerField()
-    type = p.TextField()
-
-    class Meta:
-        database = db
-        primary_key = p.CompositeKey("orig", "dest")
-
-
-TABLES = [tick, attr, relation]
-VERBOSE = False
-
-def open(path: str, opts: None | dict[str, int | str] = None, create=False,
-         verbose=False):
-    global VERBOSE
-    VERBOSE = verbose
-
-    if create and os.path.exists(path):
-        raise FileExistsError(f"`{path}' exists!")
-    if not create and not os.path.exists(path):
-        raise FileNotFoundError(f"`{path}' not found!")
-
-    db.init(path, opts)
-    db.connect()
-    if create:
-        with db:
-            db.create_tables(TABLES)
-
-def close():
-    db.close()
-
-def mkidx():
-    db.execute_sql("CREATE INDEX tick_idx on tick(id);")
-    db.execute_sql("CREATE INDEX relation_idx on relation(orig,dest);")
-    db.execute_sql("CREATE INDEX attr_idx on attr(id);")
-
-def line_nr(file: str) -> int:
-    result = sp.run(['wc', file], stdout=sp.PIPE, text=True)
-    return int(result.stdout.split()[0])
-
-def load(pr: pr.parser, trace_path: str, fd_chunk_size=900, db_chunk_size=100):
-    if not os.path.exists(trace_path):
-        raise FileNotFoundError("`{trace_path}' not found!")
-
-    with b.open(trace_path) as fd:
-        for fd_chunk in p.chunked(fd, fd_chunk_size):
-            records = pr.parse(fd_chunk)
-            with db.atomic():
-                for table in TABLES:
-                    t_name: str = table._meta.name  # type: ignore
-                    for db_chunk in p.chunked(records[t_name], db_chunk_size):
-                        table.insert_many(db_chunk).execute()
-
-def iterate(origin: int, parent: None | int, samples: type[tick] | type[attr],
-            visit: Callable, depth: int, depth_max: int):
-    if depth_max < depth:
-        return
-
-    # pull origin
-    if timeline := samples.select().where((samples.id == origin)):
-        visit(timeline.dicts(), origin, parent)
-
-    # pull children
-    orig_to_children = relation.select().where((relation.orig == origin))
-    for child in orig_to_children.dicts():
-        if VERBOSE:
-            print(f"@[{depth}] {hex(child['orig'])} ... {hex(child['dest'])}")
-        iterate(child["dest"], origin, samples, visit, depth + 1, depth_max)
-
-def iterate_ev_relations(events: list[int]) -> list[(int, int)]:
-    relations = relation.select().where((relation.orig.in_(events))).dicts()
-    return [(rel["orig"], rel["dest"]) for rel in relations]
-
-def spans(event_begin: str, event_end: str, tick_type: str) -> list:
-    sql = f"""
-    SELECT (tk.time - tick.time) FROM tick JOIN tick tk ON tk.id=tick.id
-    WHERE tick.event="{event_begin}" AND tk.event="{event_end}"
-    AND tick.type="{tick_type}";
-    """
-    return db.execute_sql(sql).fetchall()
-
-def queues(event_begin: str, event_end: str, tick_type: str) -> list:
-    sql = f"""
-    SELECT (time/1000)*1000 as timer,
-    COUNT(CASE WHEN event = "{event_begin}" THEN 1 END) as cc1,
-    COUNT(CASE WHEN event = "{event_end}" THEN 1 END) as cc2
-    FROM tick WHERE type="{tick_type}" GROUP BY timer;
-    """
-    return db.execute_sql(sql).fetchall()

chronoscope-0.0.4.dev1/chronoscope/parser.py

@@ -1,91 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# This file is part of Chronoscope.
-#
-# SPDX-FileCopyrightText: 2024 Anatoliy Bilenko <anatoliy.bilenko@gmail.com>
-#
-# SPDX-License-Identifier: LGPL-3.0-only
-#
-
-import chronoscope.utils as u
-from typing import Callable
-import yaml
-import sys
-
-class parser:
-    def __init__(self, conf_path: str, verbose=False):
-        # type -> (dest_table, parser_func)
-        self.parsers: dict[str, tuple[str, Callable]] = {}
-        # the parser knows about table names
-        self.tables = ["tick", "attr", "relation"]
-        self.verbose = verbose
-        self.load_config(conf_path)
-
-    def load_config(self, conf_path: str):
-        with open(conf_path) as fd:
-            conf = yaml.safe_load(fd)
-            conf_tables = [t for t in conf.keys() if t[0] != '.']
-            if not all(t in self.tables for t in conf_tables):
-                raise SyntaxError(f"a few of {conf_tables} aren't known!")
-
-            for table in conf_tables:
-                for trace in conf[table]:
-                    self.register_parser(table, trace["type"],
-                                         self.make_parser(table, trace["pos"]))
-
-    def make_parser(self, table: str, kwargs: dict[str, int]) -> Callable:
-        match table:
-            case "tick":
-                return self.make_req_parser(**kwargs)
-            case "relation":
-                return self.make_rel_parser(**kwargs)
-            case "attr":
-                return self.make_attr_parser(**kwargs)
-        raise NotImplementedError()
-
-    def make_req_parser(self, type: int, time: int, event: int,
-                        pid: int, id: int, eid: int) -> Callable:
-        def parse(line: list[str], parse_type: str):
-            return {
-                "time": u.ns(line[time]),
-                "type": line[type], "event": line[event],
-                "eid": u.pack(int(line[eid]), int(line[pid])),
-                "id": u.pack(int(line[id]), int(line[pid]))
-            } if type < len(line) and parse_type == line[type] else None
-        return parse
-
-    def make_rel_parser(self, orig_id: int, dest_id: int,
-                        orig_pid: int, dest_pid: int, type: int) -> Callable:
-        def parse(line: list[str], parse_type: str):
-            return {
-                "orig": u.pack(int(line[orig_id]), int(line[orig_pid])),
-                "dest": u.pack(int(line[dest_id]), int(line[dest_pid])),
-                "type": line[type]
-            } if type < len(line) and parse_type == line[type] else None
-        return parse
-
-    def make_attr_parser(self, id: int, pid: int,
-                         name: int, value: int, type: int) -> Callable:
-        def parse(line: list[str], parse_type: str):
-            return {
-                "id": u.pack(int(line[id]), int(line[pid])),
-                "val": line[value],
-                "name": line[name],
-            } if type < len(line) and parse_type == line[type] else None
-        return parse
-
-    def register_parser(self, dest_table: str, type: str, parse: Callable):
-        self.parsers[type] = (dest_table, parse)
-
-    def parse(self,
-              fd_chunk: list[str]) -> dict[str, list[dict[str, str | int]]]:
-        records: dict[str, list] = {t: [] for t in self.tables}
-        for line in fd_chunk:
-            for p_name, (dest_table, parser) in self.parsers.items():
-                try:
-                    if record := parser(line.split(), p_name):
-                        records[dest_table].append(record)
-                except Exception as e:
-                    if self.verbose:
-                        print(f"{e}: {line=}", file=sys.stderr)
-        return records

chronoscope-0.0.4.dev1/chronoscope/tree.py

@@ -1,35 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# This file is part of Chronoscope.
-#
-# SPDX-FileCopyrightText: 2024 Anatoliy Bilenko <anatoliy.bilenko@gmail.com>
-#
-# SPDX-License-Identifier: LGPL-3.0-only
-#
-
-import chronoscope.db as db
-import chronoscope.utils as utils
-from graphviz import Graph  # type: ignore
-
-# https://graphviz.readthedocs.io/en/stable/examples.html#structs-py
-FMT_NODE = """<
-<table border="0" cellborder="1" cellspacing="0">
-<tr><td>{}</td></tr><tr><td>{}</td></tr></table>>
-"""
-
-class attr_visitor:
-    def __init__(self, graph: Graph):
-        self.graph = graph
-
-    def __call__(self, node_attrs: list[dict], current: int, parent: None | int):
-        contents = "<br/>".join([f"{na['name']}={na['val']}" for na in node_attrs])
-        self.graph.node(str(current), FMT_NODE.format(utils.unpack(current), contents))
-
-        if parent:
-            self.graph.edge(str(parent), str(current))
-
-def plot(origin: int, depth_max=50):
-    g = Graph(strict=True, format="png", node_attr={"shape": "plaintext"})
-    db.iterate(origin, None, db.attr, attr_visitor(g), 0, depth_max)
-    pid, id = utils.unpack(origin)
-    g.render(f"tree_{pid}_{id}", cleanup=True)