pg-lock-tracer 0.6.2__py3-none-any.whl → 0.7.1__py3-none-any.whl

pg_lock_tracer/__init__.py

@@ -1 +1 @@
-__version__ = "0.6.2"
+__version__ = "0.7.1"

pg_lock_tracer/bpf/pg_spinlock_delay_tracer.c

@@ -0,0 +1,57 @@
+#include <uapi/linux/ptrace.h>
+
+#define MAX_STR_LEN 128
+
+typedef struct SpinDelayStatus_t {
+  int spins;
+  int delays;
+  int cur_delay;
+  const char *file;
+  int line;
+  const char *func;
+} SpinDelayStatus;
+
+typedef struct SpinDelayEvent_t {
+  u32 pid;
+  u64 timestamp;
+  int spins;
+  int delays;
+  int cur_delay;
+  int line;
+  char file[MAX_STR_LEN];
+  char func[MAX_STR_LEN];
+} SpinDelayEvent;
+
+BPF_PERF_OUTPUT(lockevents);
+
+int spin_delay(struct pt_regs *ctx) {
+  SpinDelayEvent event = {};
+  SpinDelayStatus status = {};
+  SpinDelayStatus *status_ptr = (SpinDelayStatus *)PT_REGS_PARM1(ctx);
+
+  event.pid = bpf_get_current_pid_tgid();
+  event.timestamp = bpf_ktime_get_ns();
+
+  if (!status_ptr) {
+    lockevents.perf_submit(ctx, &event, sizeof(SpinDelayEvent));
+    return 0;
+  }
+
+  bpf_probe_read_user(&status, sizeof(status), status_ptr);
+
+  event.spins = status.spins;
+  event.delays = status.delays;
+  event.cur_delay = status.cur_delay;
+  event.line = status.line;
+
+  if (status.file) {
+    bpf_probe_read_user_str(&event.file, sizeof(event.file), status.file);
+  }
+
+  if (status.func) {
+    bpf_probe_read_user_str(&event.func, sizeof(event.func), status.func);
+  }
+
+  lockevents.perf_submit(ctx, &event, sizeof(SpinDelayEvent));
+  return 0;
+}

pg_lock_tracer/oid_resolver.py

@@ -34,7 +34,7 @@ class OIDResolver:
                 host=hostname,
                 port=port,
             )
-
+            self.connection.set_session(autocommit=True)
             self.cur = self.connection.cursor()
 
             # Warmup cache

pg_lock_tracer/pg_lock_tracer.py

@@ -470,7 +470,8 @@ class PGLockTraceOutputHuman(PGLockTraceOutput):
                 )
                 line = line.decode("utf-8")
                 # Get line with: 'gdb info line *(symbol+0x1111)'
-                print(f"\t{line}")
+                line = f"\t{line}"
+                self.handle_output_line(line)
 
 
 class PGLockTraceOutputJSON(PGLockTraceOutput):

pg_lock_tracer/pg_spinlock_delay_tracer.py

@@ -0,0 +1,159 @@
+#!/usr/bin/env python3
+#
+# PostgreSQL spinlock delay tracer.
+#
+###############################################
+
+import sys
+import argparse
+
+from bcc import BPF
+
+from pg_lock_tracer import __version__
+from pg_lock_tracer.helper import BPFHelper
+
+EXAMPLES = """examples:
+# Trace spin delays of the given PostgreSQL binary
+pg_spinlock_delay_tracer -x /home/jan/postgresql-sandbox/bin/REL_14_9_DEBUG/bin/postgres
+
+# Trace spin delays of the PID 1234
+pg_spinlock_delay_tracer -p 1234 -x /home/jan/postgresql-sandbox/bin/REL_14_9_DEBUG/bin/postgres
+
+# Trace spin delays of the PID 1234 and 5678
+pg_spinlock_delay_tracer -p 1234 -p 5678 -x /home/jan/postgresql-sandbox/bin/REL_14_9_DEBUG/bin/postgres
+
+# Trace spin delays of the PID 1234 and be verbose
+pg_spinlock_delay_tracer -p 1234 -x /home/jan/postgresql-sandbox/bin/REL_14_9_DEBUG/bin/postgres -v
+"""
+
+parser = argparse.ArgumentParser(
+    description="",
+    formatter_class=argparse.RawDescriptionHelpFormatter,
+    epilog=EXAMPLES,
+)
+parser.add_argument(
+    "-V",
+    "--version",
+    action="version",
+    version=f"{parser.prog} ({__version__})",
+)
+parser.add_argument("-v", "--verbose", action="store_true", help="Be verbose")
+parser.add_argument(
+    "-p",
+    "--pid",
+    type=int,
+    nargs="+",
+    dest="pids",
+    metavar="PID",
+    help="the pid(s) to trace",
+)
+parser.add_argument(
+    "-x",
+    "--exe",
+    type=str,
+    required=True,
+    dest="path",
+    metavar="PATH",
+    help="path to binary",
+)
+parser.add_argument(
+    "-d",
+    "--dry-run",
+    action="store_true",
+    help="compile and load the BPF program but exit afterward",
+)
+
+
+class PGSpinDelayTracer:
+    def __init__(self, prog_args):
+        self.bpf_instance = None
+        self.args = prog_args
+
+        # Belong the processes to the binary?
+        BPFHelper.check_pid_exe(self.args.pids, self.args.path)
+
+    @staticmethod
+    def _decode_field(value):
+        return value.decode("utf-8", "replace").rstrip("\x00")
+
+    def print_lock_event(self, _cpu, data, _size):
+        """
+        Print a new spin delay event.
+        """
+        event = self.bpf_instance["lockevents"].event(data)
+
+        if self.args.pids and event.pid not in self.args.pids:
+            return
+
+        file_name = self._decode_field(event.file) or "(unknown)"
+        func_name = self._decode_field(event.func) or "(unknown)"
+
+        print(
+            f"{event.timestamp} [Pid {event.pid}] SpinDelay "
+            f"spins={event.spins} delays={event.delays} "
+            f"cur_delay={event.cur_delay} at {func_name}, {file_name}:{event.line}"
+        )
+
+    def init(self):
+        """
+        Init the PostgreSQL spin delay tracer
+        """
+        bpf_program = BPFHelper.read_bpf_program("pg_spinlock_delay_tracer.c")
+
+        if self.args.verbose:
+            print(bpf_program)
+
+        # Disable warnings like
+        # 'warning: '__HAVE_BUILTIN_BSWAP32__' macro redefined [-Wmacro-redefined]'
+        bpf_cflags = ["-Wno-macro-redefined"] if not self.args.verbose else []
+
+        print("===> Compiling BPF program")
+        self.bpf_instance = BPF(text=bpf_program, cflags=bpf_cflags)
+
+        print("===> Attaching BPF probes")
+        self.attach_probes()
+
+        # Open the event queue
+        self.bpf_instance["lockevents"].open_perf_buffer(
+            self.print_lock_event, page_cnt=BPFHelper.page_cnt
+        )
+
+    def attach_probes(self):
+        """
+        Attach BPF probes
+        """
+        BPFHelper.register_ebpf_probe(
+            self.args.path,
+            self.bpf_instance,
+            "^perform_spin_delay$",
+            "spin_delay",
+            self.args.verbose,
+        )
+
+    def run(self):
+        """
+        Run the BPF program and read results
+        """
+        print("===> Ready to trace")
+        while True:
+            try:
+                self.bpf_instance.perf_buffer_poll()
+            except KeyboardInterrupt:
+                sys.exit(0)
+
+
+def main():
+    """
+    Entry point for the BPF based PostgreSQL spin delay tracer.
+    """
+    args = parser.parse_args()
+
+    pg_spin_delay_tracer = PGSpinDelayTracer(args)
+    pg_spin_delay_tracer.init()
+
+    if not args.dry_run:
+        pg_spin_delay_tracer.run()
+
+
+if __name__ == "__main__":
+    main()

{pg_lock_tracer-0.6.2.dist-info → pg_lock_tracer-0.7.1.dist-info}/METADATA

@@ -1,24 +1,25 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: pg_lock_tracer
-Version: 0.6.2
-Summary: A BPF based lock tracer for the PostgreSQL database
-Home-page: https://github.com/jnidzwetzki/pg-lock-tracer
-Author: Jan Nidzwetzki
+Version: 0.7.1
+Summary: An eBPF based lock tracer for PostgreSQL
+Author-email: Jan Nidzwetzki <jnidzwetzki@gmx.de>
+License-Expression: Apache-2.0
+Project-URL: Homepage, https://github.com/jnidzwetzki/pg-lock-tracer
 Project-URL: Bug Tracker, https://github.com/jnidzwetzki/pg-lock-tracer/issues
-Keywords: postgresql bpf lock locktracer
+Keywords: postgresql,postgres,ebpf,locktracer
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Programming Language :: Python
 Classifier: Topic :: Software Development :: Debuggers
-Classifier: License :: OSI Approved :: Apache Software License
-Requires-Python: >=3.6
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: graphviz
 Requires-Dist: igraph
 Requires-Dist: prettytable
 Requires-Dist: psycopg2
+Dynamic: license-file
 
 # Lock tracing tools for PostgreSQL
 [![Make a PR](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](http://makeapullrequest.com)
@@ -33,9 +34,10 @@ This project provides tools that allow you to gain deep insights into PostgreSQL
 * `pg_lock_tracer` - is a PostgreSQL table level lock tracer.
 * `pg_lw_lock_tracer` - is a tracer for PostgreSQL lightweight locks (LWLocks).
 * `pg_row_lock_tracer` - is a tracer for PostgreSQL row locks.
+* `pg_spinlock_delay_tracer` - is a tracer for PostgreSQL spinlock delays.
 * `animate_lock_graph` - creates animated locks graphs based on the `pg_lock_tracer` output.
 
-__Note:__ These tools employ the [eBPF](https://ebpf.io/) (_Extended Berkeley Packet Filter_) technology. At the moment, PostgreSQL 12, 13, 14, 15, and 16 are supported (see additional information below).
+__Note:__ These tools employ the [eBPF](https://ebpf.io/) (_Extended Berkeley Packet Filter_) technology. At the moment, PostgreSQL 14, 15, 16, 17, and 18 are supported (see additional information below).
 
 # pg_lock_tracer
 `pg_lock_tracer` observes the locking activity of a running PostgreSQL process (using _eBPF_ and _UProbes_). In contrast to the information that is present in the table `pg_locks` (which provides information about which locks are _currently_ requested), `pg_lock_tracer` gives you a continuous view of the locking activity and collects statistics and timings.
@@ -880,6 +882,53 @@ Lock results:
 +---------+-------+--------------+-----------------+------------+------------+------------------+---------------+
 ```
 
+# pg_spinlock_delay_tracer
+`pg_spinlock_delay_tracer` allows tracing spinlock delays in a PostgreSQL process. Spin locks are used in PostgreSQL to protect short critical sections in the code. If another process already holds a spinlock, the requesting process will repeatedly check (i.e., "spin") until the lock becomes available. If the lock is held for a longer period, PostgreSQL performs a ["spin delay"](https://github.com/postgres/postgres/blob/0c8e082fba8d36434552d3d7800abda54acafd57/src/backend/storage/lmgr/s_lock.c#L106) and yields the CPU for a short time to avoid busy-waiting. Such delays are reported via the `pg_spinlock_delay_tracer`. For each delay, it prints the content of the `SpinDelayStatus` structure, which contains information about the number of spins, delays, and the current delay time. Additionally, the function name and source code location where the spin delay occurred are reported.
+
+## Usage Examples
+```
+# Trace the spinlock delays of the given PostgreSQL binary
+pg_spinlock_delay_tracer -x /home/jan/postgresql-sandbox/bin/REL_17_1_DEBUG/bin/postgres
+
+# Trace the spinlock delays of the PID 1234
+pg_spinlock_delay_tracer -p 1234 -x /home/jan/postgresql-sandbox/bin/REL_17_1_DEBUG/bin/postgres
+```
+
+## Example output
+To reproduce a spinlock delay, follow these steps to simulate a delay at the WAL insert position. First, start two different sessions connected to the same database. Then, create two tables:
+
+```sql
+CREATE TABLE mydata1 (id INT);
+CREATE TABLE mydata2 (id INT);
+```
+
+Next, open a debugger and set a breakpoint in the function `ReserveXLogInsertLocation` after the spin lock `SpinLockAcquire(&Insert->insertpos_lck);` is acquired. Afterward, start the `pg_spinlock_delay_tracer` and perform two inserts in the two different sessions:
+
+```
+# Session 1
+INSERT INTO mydata1 VALUES(1);
+# Session 2
+INSERT INTO mydata2 VALUES(2);
+```
+
+Note: Two different tables are used to prevent both sessions from trying to lock the same buffer and waiting for each other on a different lock.
+
+The debugger should stop in the first session at the breakpoint. Furthermore, the pg_spinlock_delay_tracer should report spinlock delays in the second session, as the first session holds the spinlock for a longer period.
+
+```
+pg_spinlock_delay_tracer -x /home/jan/postgresql-sandbox/bin/REL_17_1_DEBUG/bin/postgres
+[...]
+13180680737869452 [Pid 1864403] SpinDelay spins=996 delays=939 cur_delay=566086 at ReserveXLogInsertLocation, xlog.c:1132
+13180680737874986 [Pid 1864403] SpinDelay spins=997 delays=939 cur_delay=566086 at ReserveXLogInsertLocation, xlog.c:1132
+13180680737880522 [Pid 1864403] SpinDelay spins=998 delays=939 cur_delay=566086 at ReserveXLogInsertLocation, xlog.c:1132
+13180680737886009 [Pid 1864403] SpinDelay spins=999 delays=939 cur_delay=566086 at ReserveXLogInsertLocation, xlog.c:1132
+13180681304189362 [Pid 1864403] SpinDelay spins=0 delays=940 cur_delay=661655 at ReserveXLogInsertLocation, xlog.c:1132
+13180681304227806 [Pid 1864403] SpinDelay spins=1 delays=940 cur_delay=661655 at ReserveXLogInsertLocation, xlog.c:1132
+13180681304241759 [Pid 1864403] SpinDelay spins=2 delays=940 cur_delay=661655 at ReserveXLogInsertLocation, xlog.c:1132
+13180681304255150 [Pid 1864403] SpinDelay spins=3 delays=940 cur_delay=661655 at ReserveXLogInsertLocation, xlog.c:1132
+[...]
+```
+
 # Additional Information
 
 ## Installation
@@ -917,8 +966,8 @@ pip install git+https://github.com/jnidzwetzki/pg-lock-tracer
 ```
 
 ## PostgreSQL Build
-The software is tested with PostgreSQL versions 12, 13, 14, and 15. In order to be able to attach the _uprobes_ to the functions, they should not to be optimized away (e.g., inlined) during the compilation of PostgreSQL. Otherwise errors like `Unable to locate function XXX` will occur when `pg_lock_tracer` is started.
+The software is tested with PostgreSQL versions 14, 15, 16, 17, and 18. In order to be able to attach the _uprobes_ to the functions, they should not to be optimized away (e.g., inlined) during the compilation of PostgreSQL. Otherwise errors like `Unable to locate function XXX` will occur when `pg_lock_tracer` is started.
 
-It is recommended to compile PostgreSQL with following CFLAGS: `CFLAGS="-ggdb -Og -g3 -fno-omit-frame-pointer"`. 
+It is recommended to compile PostgreSQL with the following CFLAGS: `CFLAGS="-ggdb -Og -g3 -fno-omit-frame-pointer"`. 
 
 `pg_lw_lock_trace` uses [USDT probes](https://www.postgresql.org/docs/current/dynamic-trace.html). Therefore, PostgreSQL has to be compiled with `--enable-dtrace` to use this script. 

pg_lock_tracer-0.7.1.dist-info/RECORD

@@ -0,0 +1,19 @@
+pg_lock_tracer/__init__.py,sha256=2KJZDSMOG7KS82AxYOrZ4ZihYxX0wjfUjDsIZh3L024,22
+pg_lock_tracer/animate_lock_graph.py,sha256=J8b57ySVq_6QeL6_IJyycYzQK0CBkOcdGdicdJOa3Mc,12296
+pg_lock_tracer/helper.py,sha256=1RnApDu4u3tzyy2fh3aIfNTPToZTUOVZvzpYvNFGvvA,5270
+pg_lock_tracer/oid_resolver.py,sha256=ryNmoHphrHCcnv8RauhikPUvTseBSuCOOk_sI950GiM,3591
+pg_lock_tracer/pg_lock_tracer.py,sha256=Y6Fs1PFKJdd7MCzq2AeoQtqGuspiMi8Pl9yF7sCZHgM,30168
+pg_lock_tracer/pg_lw_lock_tracer.py,sha256=Jgsr9ArWQigj-ndeAKEObVIYyNIaTKNGIkDrtA8sj9w,13204
+pg_lock_tracer/pg_row_lock_tracer.py,sha256=duQalO2vf-QX4WRFd6K0WRunFlZbTHZeVkeWkr35zbo,10405
+pg_lock_tracer/pg_spinlock_delay_tracer.py,sha256=Y7jmWvQBUfsOGnC73VSzEKh0piB9Pb_2t9b_bpdfliw,4227
+pg_lock_tracer/bpf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pg_lock_tracer/bpf/pg_lock_tracer.c,sha256=EM9gFZtkeBROIM692s-DrD-xlW0khFRNLXPN5_nBFBg,14003
+pg_lock_tracer/bpf/pg_lw_lock_tracer.c,sha256=BxEpEOCo7hLJSOjcsKkng3kZhYJ80A_6k-9nxaJ6e2A,4916
+pg_lock_tracer/bpf/pg_row_lock_tracer.c,sha256=U2Uqtv68qi72bJCVNHdpO68QTVJzhUFRATHX4hCWzDY,3344
+pg_lock_tracer/bpf/pg_spinlock_delay_tracer.c,sha256=2jnGd4cd-98L4F4Xmq20OgzUR-EtcJ6AS-pdZ3uCfcU,1254
+pg_lock_tracer-0.7.1.dist-info/licenses/LICENSE,sha256=opL3U54Vdmg02lhP1XzHATC9Fyy393DcZBi-HqotSlQ,11348
+pg_lock_tracer-0.7.1.dist-info/METADATA,sha256=4tSjM_f3jM0uDqrhvGAfN6rzF6B6Ii-fcQvvAeH-7_0,76948
+pg_lock_tracer-0.7.1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+pg_lock_tracer-0.7.1.dist-info/entry_points.txt,sha256=2INTMo_Wcg6yPd559KVI44CWNifXRG7j_WVJNN6xZCg,320
+pg_lock_tracer-0.7.1.dist-info/top_level.txt,sha256=nRoLeqWT0GSR4-JDcVRuI5ltF69rs7ftfkvImNbiR1Q,15
+pg_lock_tracer-0.7.1.dist-info/RECORD,,

{pg_lock_tracer-0.6.2.dist-info → pg_lock_tracer-0.7.1.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.42.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{pg_lock_tracer-0.6.2.dist-info → pg_lock_tracer-0.7.1.dist-info}/entry_points.txt

@@ -3,3 +3,4 @@ animate_lock_graph = pg_lock_tracer.animate_lock_graph:main
 pg_lock_tracer = pg_lock_tracer.pg_lock_tracer:main
 pg_lw_lock_tracer = pg_lock_tracer.pg_lw_lock_tracer:main
 pg_row_lock_tracer = pg_lock_tracer.pg_row_lock_tracer:main
+pg_spinlock_delay_tracer = pg_lock_tracer.pg_spinlock_delay_tracer:main

pg_lock_tracer-0.6.2.dist-info/RECORD

@@ -1,17 +0,0 @@
-pg_lock_tracer/__init__.py,sha256=jFlbxEJFS0G44LE-yXXVSwXACA1J_NyYDk5E20_2zpc,22
-pg_lock_tracer/animate_lock_graph.py,sha256=J8b57ySVq_6QeL6_IJyycYzQK0CBkOcdGdicdJOa3Mc,12296
-pg_lock_tracer/helper.py,sha256=1RnApDu4u3tzyy2fh3aIfNTPToZTUOVZvzpYvNFGvvA,5270
-pg_lock_tracer/oid_resolver.py,sha256=B9kAqNzrFPP3UdstpUXkKR_No9EKLD_5nsd9ERH8vKI,3535
-pg_lock_tracer/pg_lock_tracer.py,sha256=VnhM_XMYJu6-3D8ileMiWAaFezNRpkYll_x_DL1f6jE,30122
-pg_lock_tracer/pg_lw_lock_tracer.py,sha256=Jgsr9ArWQigj-ndeAKEObVIYyNIaTKNGIkDrtA8sj9w,13204
-pg_lock_tracer/pg_row_lock_tracer.py,sha256=duQalO2vf-QX4WRFd6K0WRunFlZbTHZeVkeWkr35zbo,10405
-pg_lock_tracer/bpf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-pg_lock_tracer/bpf/pg_lock_tracer.c,sha256=EM9gFZtkeBROIM692s-DrD-xlW0khFRNLXPN5_nBFBg,14003
-pg_lock_tracer/bpf/pg_lw_lock_tracer.c,sha256=BxEpEOCo7hLJSOjcsKkng3kZhYJ80A_6k-9nxaJ6e2A,4916
-pg_lock_tracer/bpf/pg_row_lock_tracer.c,sha256=U2Uqtv68qi72bJCVNHdpO68QTVJzhUFRATHX4hCWzDY,3344
-pg_lock_tracer-0.6.2.dist-info/LICENSE,sha256=opL3U54Vdmg02lhP1XzHATC9Fyy393DcZBi-HqotSlQ,11348
-pg_lock_tracer-0.6.2.dist-info/METADATA,sha256=ek8e1a6N2wJf_IqFI5OL8YCGRgDyNdqOHDnbHyXf_5Y,73514
-pg_lock_tracer-0.6.2.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-pg_lock_tracer-0.6.2.dist-info/entry_points.txt,sha256=kCKw8Zbn3YXKdu8mBeVzIzOmGoLjD-5D40CZg8-_W9c,248
-pg_lock_tracer-0.6.2.dist-info/top_level.txt,sha256=nRoLeqWT0GSR4-JDcVRuI5ltF69rs7ftfkvImNbiR1Q,15
-pg_lock_tracer-0.6.2.dist-info/RECORD,,