testgres-exts.pglab 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1131 @@
1
+ from __future__ import annotations
2
+
3
+ import hashlib
4
+ import inspect
5
+ import ipaddress
6
+ import os
7
+ import pwd
8
+ import signal
9
+ import shutil
10
+ import stat
11
+ import subprocess
12
+ from dataclasses import dataclass
13
+ from pathlib import Path
14
+ from typing import Optional
15
+
16
+ from testgres.api import get_new_node
17
+ from testgres.backup import NodeBackup
18
+ from testgres.node import PostgresNode
19
+ from testgres.operations.local_ops import LocalOperations
20
+ import tempfile
21
+
22
+
23
+ __version__ = "0.1.0"
24
+
25
+
26
+ class PgLabError(RuntimeError):
27
+ pass
28
+
29
+
30
+ def _check_testgres_api():
31
+ missing = []
32
+
33
+ replicate = inspect.signature(PostgresNode.replicate).parameters
34
+ for name in ("target_os_ops", "target_host", "target_port"):
35
+ if name not in replicate:
36
+ missing.append("PostgresNode.replicate({})".format(name))
37
+
38
+ backup_init = inspect.signature(NodeBackup.__init__).parameters
39
+ if "os_ops" not in backup_init:
40
+ missing.append("NodeBackup.__init__(os_ops)")
41
+
42
+ if missing:
43
+ raise PgLabError(
44
+ "installed Testgres lacks the target execution-context API: {}".format(
45
+ ", ".join(missing)
46
+ )
47
+ )
48
+
49
+
50
+ def _run(cmd, check=True, capture=False):
51
+ kwargs = {"check": check, "text": True}
52
+ if capture:
53
+ kwargs.update(stdout=subprocess.PIPE, stderr=subprocess.PIPE)
54
+ return subprocess.run(list(cmd), **kwargs)
55
+
56
+
57
+ def _which(name):
58
+ path = shutil.which(name)
59
+ if not path:
60
+ raise PgLabError("required command not found: {}".format(name))
61
+ return path
62
+
63
+
64
+ def _ifname(prefix, name):
65
+ # Linux IFNAMSIZ is 16 including the terminating NUL.
66
+ digest = hashlib.sha1(name.encode("utf-8")).hexdigest()[:6]
67
+ return (prefix + digest)[:15]
68
+
69
+
70
+ @dataclass
71
+ class IOLimits:
72
+ read_bps: Optional[int] = None
73
+ write_bps: Optional[int] = None
74
+ read_iops: Optional[int] = None
75
+ write_iops: Optional[int] = None
76
+
77
+ def as_io_max(self, major, minor):
78
+ parts = []
79
+ if self.read_bps is not None:
80
+ parts.append("rbps={}".format(self.read_bps))
81
+ if self.write_bps is not None:
82
+ parts.append("wbps={}".format(self.write_bps))
83
+ if self.read_iops is not None:
84
+ parts.append("riops={}".format(self.read_iops))
85
+ if self.write_iops is not None:
86
+ parts.append("wiops={}".format(self.write_iops))
87
+ if not parts:
88
+ return None
89
+ return "{}:{} {}".format(major, minor, " ".join(parts))
90
+
91
+
92
+ class CgroupIOBackend:
93
+ version = None
94
+
95
+ def setup_root(self):
96
+ raise NotImplementedError
97
+
98
+ def create_node(self, name):
99
+ raise NotImplementedError
100
+
101
+ def apply_limits(self, cgroup, device, limits):
102
+ raise NotImplementedError
103
+
104
+ def membership_file(self, cgroup):
105
+ raise NotImplementedError
106
+
107
+ def kill_node(self, cgroup):
108
+ pass
109
+
110
+ def remove_node(self, cgroup):
111
+ cgroup.rmdir()
112
+
113
+ def cleanup_root(self):
114
+ try:
115
+ self.root.rmdir()
116
+ except OSError:
117
+ pass
118
+
119
+
120
+ class CgroupV2IO(CgroupIOBackend):
121
+ version = 2
122
+
123
+ def __init__(self, root=Path("/sys/fs/cgroup/testgres-pglab")):
124
+ self.mount = Path("/sys/fs/cgroup")
125
+ self.root = Path(root)
126
+
127
+ def setup_root(self):
128
+ controllers = (self.mount / "cgroup.controllers").read_text().split()
129
+ if "io" not in controllers:
130
+ raise PgLabError("cgroup v2 io controller is unavailable")
131
+ subtree = self.mount / "cgroup.subtree_control"
132
+ enabled = subtree.read_text().split()
133
+ if "io" not in enabled:
134
+ try:
135
+ subtree.write_text("+io\n")
136
+ except OSError as exc:
137
+ raise PgLabError("cannot enable cgroup v2 io controller: {}".format(exc))
138
+ self.root.mkdir(exist_ok=True)
139
+ child_subtree = self.root / "cgroup.subtree_control"
140
+ available = (self.root / "cgroup.controllers").read_text().split()
141
+ if "io" in available and "io" not in child_subtree.read_text().split():
142
+ child_subtree.write_text("+io\n")
143
+
144
+ def create_node(self, name):
145
+ path = self.root / name
146
+ path.mkdir(exist_ok=True)
147
+ return path
148
+
149
+ def apply_limits(self, cgroup, device, limits):
150
+ st = os.stat(device)
151
+ line = limits.as_io_max(os.major(st.st_rdev), os.minor(st.st_rdev))
152
+ if line:
153
+ (cgroup / "io.max").write_text(line + "\n")
154
+
155
+ def membership_file(self, cgroup):
156
+ return cgroup / "cgroup.procs"
157
+
158
+ def kill_node(self, cgroup):
159
+ try:
160
+ (cgroup / "cgroup.kill").write_text("1\n")
161
+ except OSError:
162
+ pass
163
+
164
+
165
+ class CgroupV1Blkio(CgroupIOBackend):
166
+ version = 1
167
+
168
+ def __init__(self, mount=Path("/sys/fs/cgroup/blkio")):
169
+ self.mount = Path(mount)
170
+ self.root = self.mount / "testgres-pglab"
171
+
172
+ def setup_root(self):
173
+ if not self.mount.is_dir():
174
+ raise PgLabError("cgroup v1 blkio controller is unavailable")
175
+ self.root.mkdir(exist_ok=True)
176
+
177
+ def create_node(self, name):
178
+ path = self.root / name
179
+ path.mkdir(exist_ok=True)
180
+ return path
181
+
182
+ def _write_limit(self, cgroup, filename, major, minor, value):
183
+ if value is not None:
184
+ (cgroup / filename).write_text("{}:{} {}\n".format(major, minor, value))
185
+
186
+ def apply_limits(self, cgroup, device, limits):
187
+ st = os.stat(device)
188
+ major, minor = os.major(st.st_rdev), os.minor(st.st_rdev)
189
+ self._write_limit(cgroup, "blkio.throttle.read_bps_device", major, minor, limits.read_bps)
190
+ self._write_limit(cgroup, "blkio.throttle.write_bps_device", major, minor, limits.write_bps)
191
+ self._write_limit(cgroup, "blkio.throttle.read_iops_device", major, minor, limits.read_iops)
192
+ self._write_limit(cgroup, "blkio.throttle.write_iops_device", major, minor, limits.write_iops)
193
+
194
+ def membership_file(self, cgroup):
195
+ return cgroup / "tasks"
196
+
197
+
198
+ def detect_cgroup_io_backend():
199
+ # Pure/unified v2 has cgroup.controllers at the top-level mount.
200
+ if Path("/sys/fs/cgroup/cgroup.controllers").is_file():
201
+ return CgroupV2IO()
202
+ # Ubuntu 20.04 commonly uses hybrid mode: cgroup2 is mounted under
203
+ # /sys/fs/cgroup/unified, while blkio remains on the v1 hierarchy.
204
+ if Path("/sys/fs/cgroup/blkio").is_dir():
205
+ return CgroupV1Blkio()
206
+ raise PgLabError("no supported cgroup I/O controller found (need v2 io or v1 blkio)")
207
+
208
+
209
+ class NamespaceOperations(LocalOperations):
210
+ """Local Testgres operations whose *commands* execute in a netns.
211
+
212
+ Filesystem operations intentionally remain ordinary local operations: a Linux
213
+ network namespace shares the host filesystem. PostgreSQL utilities launched
214
+ by Testgres (initdb, pg_ctl, psql, pg_basebackup, pg_isready, ...) are wrapped
215
+ in::
216
+
217
+ ip netns exec <namespace> ... runuser -u <os_user> -- <command>
218
+
219
+ The wrapper enters the node cgroup before dropping privileges, so postgres
220
+ and all of its descendants inherit the cgroup.
221
+ """
222
+
223
+ def __init__(self, namespace, os_user, db_user, cgroup=None, cgroup_membership_file=None, pg_bin=None):
224
+ super().__init__()
225
+ self.namespace = namespace
226
+ self.os_user = os_user
227
+ self.db_user = db_user
228
+ self.pg_bin = str(pg_bin) if pg_bin is not None else None
229
+ self.cgroup = Path(cgroup) if cgroup is not None else None
230
+ self.cgroup_membership_file = (
231
+ Path(cgroup_membership_file) if cgroup_membership_file is not None else None
232
+ )
233
+
234
+ # Testgres uses os_ops.username as the default database username in a
235
+ # number of places. Keep it distinct from the Unix account we use in
236
+ # runuser.
237
+ self._username = db_user
238
+
239
+ def environ(self, name):
240
+ # target_os_ops may be asked to resolve PostgreSQL binaries while a
241
+ # replica is being cloned. Expose the PgLab-specific bin directory to
242
+ # Testgres without requiring the caller to mutate the process-wide
243
+ # PG_BIN environment variable.
244
+ if name == "PG_BIN" and self.pg_bin is not None:
245
+ return self.pg_bin
246
+ return super().environ(name)
247
+
248
+ def mkdtemp(self, prefix=None):
249
+ assert prefix is None or type(prefix) is str
250
+
251
+ base = "/tmp/testgres-pglab-tmp"
252
+ os.makedirs(base, mode=0o755, exist_ok=True)
253
+
254
+ path = tempfile.mkdtemp(
255
+ prefix=prefix,
256
+ dir=base,
257
+ )
258
+
259
+ pw = pwd.getpwnam(self.os_user)
260
+ os.chown(path, pw.pw_uid, pw.pw_gid)
261
+ os.chmod(path, 0o700)
262
+
263
+ return path
264
+
265
+ def mkstemp(self, prefix=None):
266
+ assert prefix is None or type(prefix) is str
267
+
268
+ base = "/tmp/testgres-pglab-tmp"
269
+ os.makedirs(base, mode=0o755, exist_ok=True)
270
+
271
+ fd, path = tempfile.mkstemp(
272
+ prefix=prefix,
273
+ dir=base,
274
+ )
275
+ os.close(fd)
276
+
277
+ pw = pwd.getpwnam(self.os_user)
278
+ os.chown(path, pw.pw_uid, pw.pw_gid)
279
+ os.chmod(path, 0o600)
280
+
281
+ return path
282
+
283
+ def create_clone(self):
284
+ return self.__class__(
285
+ namespace=self.namespace,
286
+ os_user=self.os_user,
287
+ db_user=self.db_user,
288
+ cgroup=str(self.cgroup) if self.cgroup is not None else None,
289
+ cgroup_membership_file=(
290
+ str(self.cgroup_membership_file)
291
+ if self.cgroup_membership_file is not None else None
292
+ ),
293
+ pg_bin=self.pg_bin,
294
+ )
295
+
296
+ def _command_needs_server_cgroup(self, payload):
297
+ """Return True only for commands that may start the postgres server.
298
+
299
+ Client utilities such as psql, pg_basebackup and pg_isready should not
300
+ be charged to the server's I/O cgroup. pg_ctl inherits the cgroup and
301
+ the postmaster it starts keeps that membership.
302
+ """
303
+ if not payload:
304
+ return False
305
+ exe = os.path.basename(str(payload[0]))
306
+ if exe == "postgres":
307
+ return True
308
+ if exe == "pg_ctl":
309
+ # Testgres normally invokes pg_ctl with an explicit action. Only
310
+ # start/restart can create a postmaster that needs to inherit the
311
+ # node cgroup.
312
+ args = [str(x) for x in payload[1:]]
313
+ return "start" in args or "restart" in args
314
+ return False
315
+
316
+ def _with_pg_path(self, argv):
317
+ if self.pg_bin is None:
318
+ return list(argv)
319
+ path = self.pg_bin + os.pathsep + os.environ.get("PATH", "")
320
+ return ["env", "PATH=" + path] + list(argv)
321
+
322
+ def _wrap_command(self, cmd, shell):
323
+ if shell:
324
+ if isinstance(cmd, str):
325
+ command = cmd
326
+ else:
327
+ command = self.join_command_arguments(list(cmd))
328
+ payload = ["sh", "-c", command]
329
+ else:
330
+ if isinstance(cmd, str):
331
+ payload = ["sh", "-c", cmd]
332
+ else:
333
+ payload = list(cmd)
334
+
335
+ prefix = ["ip", "netns", "exec", self.namespace]
336
+ run_as_user = ["runuser", "-u", self.os_user, "--"] + self._with_pg_path(payload)
337
+
338
+ # Most Testgres utilities are clients. They belong to the controller,
339
+ # not to the PostgreSQL server cgroup.
340
+ if self.cgroup is None or not self._command_needs_server_cgroup(payload):
341
+ return prefix + run_as_user, False
342
+
343
+ membership = self.cgroup_membership_file
344
+ if membership is None:
345
+ raise PgLabError("cgroup membership file was not configured")
346
+ if not membership.exists():
347
+ raise PgLabError(
348
+ "node cgroup disappeared before postgres start: {}".format(membership)
349
+ )
350
+
351
+ # Join the cgroup in the host namespace first, then enter the network
352
+ # namespace. Some systems do not expose the host cgroup filesystem from
353
+ # an ip-netns helper mount namespace, while cgroup membership itself is
354
+ # inherited perfectly well across ip netns exec/runuser/pg_ctl.
355
+ script = (
356
+ 'echo $$ > "$1"; '
357
+ 'ns="$2"; user="$3"; shift 3; '
358
+ 'exec ip netns exec "$ns" runuser -u "$user" -- "$@"'
359
+ )
360
+ wrapped = [
361
+ "sh", "-c", script, "pglab-cgroup", str(membership),
362
+ self.namespace, self.os_user,
363
+ ] + self._with_pg_path(payload)
364
+ return wrapped, False
365
+
366
+ def exec_command(self, cmd, *args, **kwargs):
367
+ shell = kwargs.pop("shell", False)
368
+ wrapped, wrapped_shell = self._wrap_command(cmd, shell)
369
+ return super().exec_command(wrapped, *args, shell=wrapped_shell, **kwargs)
370
+
371
+ def popen(self, cmd, *args, **kwargs):
372
+ shell = kwargs.pop("shell", False)
373
+ wrapped, wrapped_shell = self._wrap_command(cmd, shell)
374
+ return super().popen(wrapped, *args, shell=wrapped_shell, **kwargs)
375
+
376
+
377
+ class LabNode:
378
+ """A Linux-isolated node backed by a real testgres.PostgresNode."""
379
+
380
+ def __init__(self, lab, name, ip, namespace, veth_host, cgroup,
381
+ pg_node, os_ops, role):
382
+ self.lab = lab
383
+ self.name = name
384
+ self.ip = ip
385
+ self.namespace = namespace
386
+ self.veth_host = veth_host
387
+ self.veth_ns = "eth0"
388
+ self.cgroup = cgroup
389
+ self.pg = pg_node
390
+ self.os_ops = os_ops
391
+ self.role = role
392
+
393
+ def __getattr__(self, name):
394
+ # Keep the normal Testgres API pleasant to use:
395
+ # node.execute(), node.safe_psql(), node.append_conf(), ...
396
+ return getattr(self.pg, name)
397
+
398
+ def init(self, initdb_params=None, **kwargs):
399
+ params = list(initdb_params or [])
400
+ if "-U" not in params and "--username" not in params:
401
+ params += ["-U", self.lab.db_superuser]
402
+ self.lab._ensure_node_ownership(self)
403
+ self.pg.init(initdb_params=params, **kwargs)
404
+
405
+ # Network namespaces isolate TCP/IP but share the host filesystem.
406
+ # Without this, nodes using the same PostgreSQL port collide on
407
+ # /tmp/.s.PGSQL.<port> even though their TCP sockets are isolated.
408
+ self.pg.append_conf(unix_socket_directories="")
409
+
410
+ self.lab._ensure_node_ownership(self)
411
+ return self
412
+
413
+ def start(self, *args, **kwargs):
414
+ # Testgres may lazily create logs_dir during start(). Force creation
415
+ # while the controller is root, then hand the whole node tree to the
416
+ # configured OS user before pg_ctl is run through runuser.
417
+ _ = self.pg.logs_dir
418
+ self.lab._ensure_node_ownership(self)
419
+ self.pg.start(*args, **kwargs)
420
+ return self
421
+
422
+ def stop(self, *args, **kwargs):
423
+ self.pg.stop(*args, **kwargs)
424
+ return self
425
+
426
+ def network_down(self):
427
+ self.lab.network_down(self.name)
428
+ return self
429
+
430
+ def network_up(self):
431
+ self.lab.network_up(self.name)
432
+ return self
433
+
434
+ def netem(self, **kwargs):
435
+ self.lab.netem(self.name, **kwargs)
436
+ return self
437
+
438
+ def clear_netem(self):
439
+ self.lab.clear_netem(self.name)
440
+ return self
441
+
442
+ def freeze(self):
443
+ self.lab.freeze(self.name)
444
+ return self
445
+
446
+ def unfreeze(self):
447
+ self.lab.unfreeze(self.name)
448
+ return self
449
+
450
+
451
+ class PgLab:
452
+ """Testgres + Linux network namespaces + cgroup I/O fault injection.
453
+
454
+ PostgreSQL lifecycle is delegated to Testgres. PgLab owns only Linux
455
+ isolation (netns/veth/bridge/cgroup) and fault injection.
456
+ """
457
+
458
+ def __init__(
459
+ self,
460
+ pg_bin=None,
461
+ base_dir="/tmp/testgres-pglab",
462
+ subnet="10.203.0.0/24",
463
+ bridge_name="pglab0",
464
+ bridge_ip=None,
465
+ os_user="postgres",
466
+ db_superuser=None,
467
+ keep_data=False,
468
+ ):
469
+ self.pg_bin = str(pg_bin) if pg_bin is not None else None
470
+ self.base_dir = Path(base_dir)
471
+ self.subnet = ipaddress.ip_network(subnet, strict=True)
472
+ self.bridge_name = bridge_name
473
+ self.os_user = os_user
474
+ self.db_superuser = db_superuser or os_user
475
+ self.keep_data = keep_data
476
+ self.nodes = {} # type: Dict[str, LabNode]
477
+ self._created_bridge = False
478
+ self._closed = False
479
+ self._firewall_backend = None
480
+ self._firewall_source_added = False
481
+ self._iptables_rule_added = False
482
+
483
+ _check_testgres_api()
484
+
485
+ hosts = self.subnet.hosts()
486
+ default_bridge_ip = str(next(hosts))
487
+ self.bridge_ip = bridge_ip or default_bridge_ip
488
+ if ipaddress.ip_address(self.bridge_ip) not in self.subnet:
489
+ raise PgLabError("bridge IP is outside lab subnet: {}".format(self.bridge_ip))
490
+
491
+ self._preflight()
492
+ self.base_dir.mkdir(parents=True, exist_ok=True)
493
+ self._setup_cgroups()
494
+ self._ensure_bridge()
495
+ self._setup_firewall()
496
+
497
+ def _preflight(self):
498
+ if os.name != "posix" or not Path("/proc/self/ns/net").exists():
499
+ raise PgLabError("pglab requires Linux network namespaces")
500
+ if os.geteuid() != 0:
501
+ raise PgLabError("pglab must currently be run as root")
502
+ _which("ip")
503
+ _which("runuser")
504
+ _which("findmnt")
505
+ try:
506
+ pwd.getpwnam(self.os_user)
507
+ except KeyError as exc:
508
+ raise PgLabError("OS user does not exist: {}".format(self.os_user)) from exc
509
+ # cgroup mode is detected separately. Ubuntu 20.04 commonly uses
510
+ # hybrid mode with blkio on cgroup v1.
511
+ if self.pg_bin is not None:
512
+ required = ("initdb", "pg_ctl", "postgres", "psql", "pg_basebackup")
513
+ for exe in required:
514
+ p = Path(self.pg_bin) / exe
515
+ if not p.is_file() or not os.access(str(p), os.X_OK):
516
+ raise PgLabError("PostgreSQL executable missing/not executable: {}".format(p))
517
+
518
+ def _setup_cgroups(self):
519
+ self.cgroup_backend = detect_cgroup_io_backend()
520
+ self.cgroup_backend.setup_root()
521
+ self.cgroup_root = self.cgroup_backend.root
522
+
523
+ def _ensure_bridge(self):
524
+ exists = subprocess.run(
525
+ ["ip", "link", "show", "dev", self.bridge_name],
526
+ stdout=subprocess.DEVNULL,
527
+ stderr=subprocess.DEVNULL,
528
+ ).returncode == 0
529
+ if not exists:
530
+ _run(["ip", "link", "add", self.bridge_name, "type", "bridge"])
531
+ self._created_bridge = True
532
+
533
+ # The host gets an address in the lab subnet. This is important because
534
+ # testgres' Python DB driver connects from the controller namespace to
535
+ # node.host; unlike psql, execute()/connect() do not go through os_ops.
536
+ addr = "{}/{}".format(self.bridge_ip, self.subnet.prefixlen)
537
+ _run(["ip", "addr", "replace", addr, "dev", self.bridge_name])
538
+ _run(["ip", "link", "set", self.bridge_name, "up"])
539
+
540
+ def _setup_firewall(self):
541
+ """Allow traffic between PgLab nodes without changing persistent rules.
542
+
543
+ firewalld and iptables-nft can be independent forwarding layers. When
544
+ firewalld is active we add the lab subnet to the runtime trusted zone,
545
+ and independently add a temporary iptables FORWARD allow rule.
546
+ """
547
+ source = str(self.subnet)
548
+ firewall_cmd = shutil.which("firewall-cmd")
549
+
550
+ if firewall_cmd is not None:
551
+ state = subprocess.run(
552
+ [firewall_cmd, "--state"],
553
+ stdout=subprocess.DEVNULL,
554
+ stderr=subprocess.DEVNULL,
555
+ )
556
+ if state.returncode == 0:
557
+ zone_proc = subprocess.run(
558
+ [firewall_cmd, "--get-zone-of-source={}".format(source)],
559
+ stdout=subprocess.PIPE,
560
+ stderr=subprocess.PIPE,
561
+ text=True,
562
+ )
563
+ zone = zone_proc.stdout.strip()
564
+ if zone and zone != "trusted":
565
+ raise PgLabError(
566
+ "lab subnet {} already belongs to firewalld zone {!r}; "
567
+ "refusing to move an existing firewall configuration".format(
568
+ source, zone
569
+ )
570
+ )
571
+ if zone != "trusted":
572
+ proc = subprocess.run(
573
+ [
574
+ firewall_cmd,
575
+ "--zone=trusted",
576
+ "--add-source={}".format(source),
577
+ ],
578
+ stdout=subprocess.DEVNULL,
579
+ stderr=subprocess.PIPE,
580
+ text=True,
581
+ )
582
+ if proc.returncode != 0:
583
+ raise PgLabError(
584
+ "failed to add lab subnet {} to firewalld trusted zone: {}".format(
585
+ source, proc.stderr.strip()
586
+ )
587
+ )
588
+ self._firewall_source_added = True
589
+ self._firewall_backend = "firewalld"
590
+
591
+ # firewalld and iptables-nft are not alternatives on many developer
592
+ # hosts. Docker/libvirt may leave an iptables FORWARD base chain with
593
+ # policy DROP at nft priority 0, while firewalld evaluates forwarding
594
+ # again in a later base chain (commonly priority +10). Both layers
595
+ # therefore need a temporary allow for the lab subnet.
596
+ iptables = shutil.which("iptables")
597
+ if iptables is None:
598
+ return
599
+
600
+ rule = [
601
+ "FORWARD", "-s", source, "-d", source, "-j", "ACCEPT",
602
+ ]
603
+ exists = subprocess.run(
604
+ [iptables, "-C"] + rule,
605
+ stdout=subprocess.DEVNULL,
606
+ stderr=subprocess.DEVNULL,
607
+ ).returncode == 0
608
+ if not exists:
609
+ _run([iptables, "-I"] + rule)
610
+ self._iptables_rule_added = True
611
+
612
+ if self._firewall_backend is None:
613
+ self._firewall_backend = "iptables"
614
+
615
+ def _cleanup_firewall(self):
616
+ source = str(self.subnet)
617
+
618
+ # The two firewall layers are independent and can both be active.
619
+ # Remove only rules that this PgLab instance actually added.
620
+ if self._firewall_source_added:
621
+ firewall_cmd = shutil.which("firewall-cmd")
622
+ if firewall_cmd is not None:
623
+ subprocess.run(
624
+ [
625
+ firewall_cmd,
626
+ "--zone=trusted",
627
+ "--remove-source={}".format(source),
628
+ ],
629
+ stdout=subprocess.DEVNULL,
630
+ stderr=subprocess.DEVNULL,
631
+ )
632
+ self._firewall_source_added = False
633
+
634
+ if self._iptables_rule_added:
635
+ iptables = shutil.which("iptables")
636
+ if iptables is not None:
637
+ subprocess.run(
638
+ [
639
+ iptables, "-D", "FORWARD",
640
+ "-s", source, "-d", source,
641
+ "-j", "ACCEPT",
642
+ ],
643
+ stdout=subprocess.DEVNULL,
644
+ stderr=subprocess.DEVNULL,
645
+ )
646
+ self._iptables_rule_added = False
647
+
648
+ @staticmethod
649
+ def _sysfs_block_path(device):
650
+ st = os.stat(device)
651
+ if not stat.S_ISBLK(st.st_mode):
652
+ raise PgLabError("not a block device: {}".format(device))
653
+ return Path("/sys/dev/block/{}:{}".format(os.major(st.st_rdev), os.minor(st.st_rdev)))
654
+
655
+ @staticmethod
656
+ def _device_from_sysfs(sysdev):
657
+ # sysdev may be a /sys/dev/block/M:m symlink or a path reached through
658
+ # a device-mapper/md 'slaves' directory. The kernel exposes DEVNAME
659
+ # in uevent for both disks and partitions.
660
+ uevent = Path(sysdev) / "uevent"
661
+ if not uevent.is_file():
662
+ raise PgLabError("cannot resolve block device from sysfs: {}".format(sysdev))
663
+ values = {}
664
+ for line in uevent.read_text().splitlines():
665
+ if "=" in line:
666
+ k, v = line.split("=", 1)
667
+ values[k] = v
668
+ devname = values.get("DEVNAME")
669
+ if not devname:
670
+ raise PgLabError("DEVNAME is missing in {}".format(uevent))
671
+ return "/dev/{}".format(devname)
672
+
673
+ def _leaf_block_devices(self, device):
674
+ """Return physical queue devices suitable for blkio throttling.
675
+
676
+ findmnt may return a partition or a stacked device such as LVM/dm-crypt.
677
+ cgroup-v1 blkio throttling can reject such a major:minor with ENODEV.
678
+ Walk sysfs 'slaves' recursively and, for partitions, use the parent disk.
679
+ """
680
+ start = self._sysfs_block_path(device).resolve()
681
+ found = {}
682
+
683
+ def visit(sysdev):
684
+ sysdev = Path(sysdev).resolve()
685
+ slaves = sysdev / "slaves"
686
+ if slaves.is_dir():
687
+ children = list(slaves.iterdir())
688
+ if children:
689
+ for child in children:
690
+ visit(child)
691
+ return
692
+
693
+ # A partition has its own major:minor, but blkio v1 throttling is
694
+ # most portable when applied to the whole queue-bearing disk.
695
+ if (sysdev / "partition").is_file():
696
+ parent = sysdev.parent
697
+ if (parent / "dev").is_file():
698
+ sysdev = parent
699
+
700
+ dev = self._device_from_sysfs(sysdev)
701
+ st = os.stat(dev)
702
+ key = (os.major(st.st_rdev), os.minor(st.st_rdev))
703
+ found[key] = dev
704
+
705
+ visit(start)
706
+ return [found[k] for k in sorted(found)]
707
+
708
+ def _backing_block_devices(self, path):
709
+ res = _run(["findmnt", "-n", "-o", "SOURCE", "-T", str(path)], capture=True)
710
+ source = res.stdout.strip()
711
+ if not source.startswith("/dev/"):
712
+ raise PgLabError(
713
+ "cannot infer block device for {}: {!r}; pass io_device".format(path, source)
714
+ )
715
+ devices = self._leaf_block_devices(source)
716
+ if not devices:
717
+ raise PgLabError("cannot resolve backing block device for {}".format(source))
718
+ return devices
719
+
720
+ def _apply_io_limits(self, cgroup, devices, limits):
721
+ if isinstance(devices, (str, Path)):
722
+ devices = [str(devices)]
723
+ for device in devices:
724
+ st = os.stat(device)
725
+ if not stat.S_ISBLK(st.st_mode):
726
+ raise PgLabError("not a block device: {}".format(device))
727
+ self.cgroup_backend.apply_limits(cgroup, device, limits)
728
+
729
+ def _ensure_node_ownership(self, node):
730
+ pw = pwd.getpwnam(self.os_user)
731
+ base = Path(node.pg.base_dir)
732
+ base.mkdir(parents=True, exist_ok=True)
733
+ for root, dirs, files in os.walk(str(base)):
734
+ os.chown(root, pw.pw_uid, pw.pw_gid)
735
+ for d in dirs:
736
+ os.chown(os.path.join(root, d), pw.pw_uid, pw.pw_gid)
737
+ for f in files:
738
+ try:
739
+ os.chown(os.path.join(root, f), pw.pw_uid, pw.pw_gid)
740
+ except FileNotFoundError:
741
+ pass
742
+
743
+ def create_node(self, name, ip, role="standalone", io=None, io_device=None, port=5432):
744
+ if name in self.nodes:
745
+ raise PgLabError("node already exists: {}".format(name))
746
+ if role not in ("primary", "replica", "standalone"):
747
+ raise PgLabError("invalid role: {}".format(role))
748
+
749
+ addr_obj = ipaddress.ip_address(ip)
750
+ if addr_obj not in self.subnet:
751
+ raise PgLabError("{} is outside lab subnet {}".format(ip, self.subnet))
752
+ if ip == self.bridge_ip:
753
+ raise PgLabError("{} is reserved for the pglab bridge".format(ip))
754
+ if any(n.ip == ip for n in self.nodes.values()):
755
+ raise PgLabError("IP already in use: {}".format(ip))
756
+
757
+ ns = "pglab-{}".format(name)
758
+ vh = _ifname("ph", name)
759
+ peer = _ifname("pn", name)
760
+ cgroup = self.cgroup_root / name
761
+ node_base = self.base_dir / name
762
+
763
+ created_ns = False
764
+ created_veth = False
765
+ created_cgroup = False
766
+ try:
767
+ _run(["ip", "netns", "add", ns])
768
+ created_ns = True
769
+ _run(["ip", "link", "add", vh, "type", "veth", "peer", "name", peer])
770
+ created_veth = True
771
+ _run(["ip", "link", "set", peer, "netns", ns])
772
+ _run(["ip", "link", "set", vh, "master", self.bridge_name])
773
+ _run(["ip", "link", "set", vh, "up"])
774
+ _run(["ip", "-n", ns, "link", "set", "lo", "up"])
775
+ _run(["ip", "-n", ns, "link", "set", peer, "name", "eth0"])
776
+ _run(["ip", "-n", ns, "addr", "add",
777
+ "{}/{}".format(ip, self.subnet.prefixlen), "dev", "eth0"])
778
+ _run(["ip", "-n", ns, "link", "set", "eth0", "up"])
779
+
780
+ node_base.mkdir(parents=True, exist_ok=True)
781
+ pw = pwd.getpwnam(self.os_user)
782
+ os.chown(str(node_base), pw.pw_uid, pw.pw_gid)
783
+
784
+ cgroup = self.cgroup_backend.create_node(name)
785
+ created_cgroup = True
786
+ if io is not None:
787
+ devices = [io_device] if io_device else self._backing_block_devices(node_base)
788
+ self._apply_io_limits(cgroup, devices, io)
789
+
790
+ os_ops = NamespaceOperations(
791
+ namespace=ns,
792
+ os_user=self.os_user,
793
+ db_user=self.db_superuser,
794
+ cgroup=cgroup,
795
+ cgroup_membership_file=self.cgroup_backend.membership_file(cgroup),
796
+ pg_bin=self.pg_bin,
797
+ )
798
+
799
+ kwargs = {
800
+ "name": name,
801
+ "base_dir": str(node_base),
802
+ "port": int(port),
803
+ "host": ip,
804
+ "os_ops": os_ops,
805
+ }
806
+ if self.pg_bin is not None:
807
+ kwargs["bin_dir"] = self.pg_bin
808
+ pg_node = get_new_node(**kwargs)
809
+
810
+ node = LabNode(
811
+ lab=self,
812
+ name=name,
813
+ ip=ip,
814
+ namespace=ns,
815
+ veth_host=vh,
816
+ cgroup=cgroup,
817
+ pg_node=pg_node,
818
+ os_ops=os_ops,
819
+ role=role,
820
+ )
821
+ self.nodes[name] = node
822
+ return node
823
+ except Exception:
824
+ if created_veth:
825
+ subprocess.run(["ip", "link", "del", vh], stdout=subprocess.DEVNULL,
826
+ stderr=subprocess.DEVNULL)
827
+ if created_ns:
828
+ subprocess.run(["ip", "netns", "del", ns], stdout=subprocess.DEVNULL,
829
+ stderr=subprocess.DEVNULL)
830
+ if created_cgroup:
831
+ try:
832
+ cgroup.rmdir()
833
+ except OSError:
834
+ pass
835
+ if not self.keep_data:
836
+ shutil.rmtree(str(node_base), ignore_errors=True)
837
+ raise
838
+
839
+ def create_primary(self, name, ip, **kwargs):
840
+ return self.create_node(name, ip, role="primary", **kwargs)
841
+
842
+ def create_replica(self, primary_name, name, ip, **kwargs):
843
+ """Create a physical replica in its own network namespace.
844
+
845
+ Current Testgres can run pg_basebackup directly in ``target_os_ops``.
846
+ PgLab creates the namespace/cgroup first, then asks Testgres to create
847
+ the replica through that execution context. The base backup therefore
848
+ originates from the replica namespace instead of being copied into it
849
+ afterwards.
850
+ """
851
+ primary = self.nodes[primary_name]
852
+ replica_shell = self.create_node(name, ip, role="replica", **kwargs)
853
+
854
+ target_ops = replica_shell.os_ops
855
+ target_port = replica_shell.pg.port
856
+ target_base = replica_shell.pg.base_dir
857
+
858
+ # create_node() constructs a placeholder PostgresNode so PgLab can
859
+ # allocate the namespace, cgroup, address, port and storage location
860
+ # before replication. Reuse that base directory for the target-side
861
+ # base backup; this keeps replica data under PgLab.base_dir (important
862
+ # when PGLAB_TEST_ROOT selects a block-backed filesystem for I/O tests).
863
+ try:
864
+ spawned = primary.pg.replicate(
865
+ name=name,
866
+ target_os_ops=target_ops,
867
+ target_host=ip,
868
+ target_port=target_port,
869
+ username=self.db_superuser,
870
+ base_dir=target_base,
871
+ )
872
+ except Exception:
873
+ self.remove_node(name)
874
+ raise
875
+
876
+ replica_shell.pg = spawned
877
+ self._ensure_node_ownership(replica_shell)
878
+ return replica_shell
879
+
880
+ def network_down(self, name):
881
+ node = self.nodes[name]
882
+ _run(["ip", "-n", node.namespace, "link", "set", node.veth_ns, "down"])
883
+
884
+ def network_up(self, name):
885
+ node = self.nodes[name]
886
+ _run(["ip", "-n", node.namespace, "link", "set", node.veth_ns, "up"])
887
+
888
+ @staticmethod
889
+ def _percent(value, name):
890
+ value = float(value)
891
+ if value < 0.0 or value > 100.0:
892
+ raise PgLabError("{} must be between 0 and 100".format(name))
893
+ return value
894
+
895
+ def netem(
896
+ self,
897
+ name,
898
+ delay_ms=None,
899
+ jitter_ms=None,
900
+ loss_pct=None,
901
+ duplicate_pct=None,
902
+ corrupt_pct=None,
903
+ reorder_pct=None,
904
+ rate_kbit=None,
905
+ ):
906
+ """Replace the node's root qdisc with a tc-netem fault profile.
907
+
908
+ The profile is applied to traffic leaving the node's eth0 inside its
909
+ network namespace. Calling netem() again replaces the previous profile.
910
+
911
+ Examples:
912
+ lab.netem("r1", delay_ms=100)
913
+ lab.netem("r1", delay_ms=100, jitter_ms=20, loss_pct=5)
914
+ lab.netem("r1", rate_kbit=1024)
915
+ lab.clear_netem("r1")
916
+ """
917
+ node = self.nodes[name]
918
+ _which("tc")
919
+
920
+ args = [
921
+ "ip", "netns", "exec", node.namespace,
922
+ "tc", "qdisc", "replace", "dev", node.veth_ns,
923
+ "root", "netem",
924
+ ]
925
+
926
+ have_fault = False
927
+
928
+ if delay_ms is not None:
929
+ delay_ms = float(delay_ms)
930
+ if delay_ms < 0:
931
+ raise PgLabError("delay_ms must be >= 0")
932
+ args += ["delay", "{}ms".format(delay_ms)]
933
+ have_fault = True
934
+
935
+ if jitter_ms is not None:
936
+ jitter_ms = float(jitter_ms)
937
+ if jitter_ms < 0:
938
+ raise PgLabError("jitter_ms must be >= 0")
939
+ args += ["{}ms".format(jitter_ms)]
940
+ elif jitter_ms is not None:
941
+ raise PgLabError("jitter_ms requires delay_ms")
942
+
943
+ if loss_pct is not None:
944
+ loss_pct = self._percent(loss_pct, "loss_pct")
945
+ args += ["loss", "{}%".format(loss_pct)]
946
+ have_fault = True
947
+
948
+ if duplicate_pct is not None:
949
+ duplicate_pct = self._percent(duplicate_pct, "duplicate_pct")
950
+ args += ["duplicate", "{}%".format(duplicate_pct)]
951
+ have_fault = True
952
+
953
+ if corrupt_pct is not None:
954
+ corrupt_pct = self._percent(corrupt_pct, "corrupt_pct")
955
+ args += ["corrupt", "{}%".format(corrupt_pct)]
956
+ have_fault = True
957
+
958
+ if reorder_pct is not None:
959
+ reorder_pct = self._percent(reorder_pct, "reorder_pct")
960
+ if delay_ms is None:
961
+ raise PgLabError("reorder_pct requires delay_ms")
962
+ args += ["reorder", "{}%".format(reorder_pct)]
963
+ have_fault = True
964
+
965
+ if rate_kbit is not None:
966
+ rate_kbit = int(rate_kbit)
967
+ if rate_kbit <= 0:
968
+ raise PgLabError("rate_kbit must be > 0")
969
+ args += ["rate", "{}kbit".format(rate_kbit)]
970
+ have_fault = True
971
+
972
+ if not have_fault:
973
+ raise PgLabError("netem requires at least one fault parameter")
974
+
975
+ _run(args)
976
+
977
+ def clear_netem(self, name):
978
+ """Remove the root qdisc fault profile from a node, if present."""
979
+ node = self.nodes[name]
980
+ if shutil.which("tc") is None:
981
+ return
982
+ subprocess.run(
983
+ [
984
+ "ip", "netns", "exec", node.namespace,
985
+ "tc", "qdisc", "del", "dev", node.veth_ns, "root",
986
+ ],
987
+ stdout=subprocess.DEVNULL,
988
+ stderr=subprocess.DEVNULL,
989
+ check=False,
990
+ )
991
+
992
+ def _node_cgroup_pids(self, node):
993
+ membership = Path(node.os_ops.cgroup_membership_file)
994
+ try:
995
+ lines = membership.read_text().splitlines()
996
+ except FileNotFoundError:
997
+ return []
998
+ pids = []
999
+ for line in lines:
1000
+ line = line.strip()
1001
+ if line.isdigit():
1002
+ pids.append(int(line))
1003
+ return sorted(set(pids))
1004
+
1005
+ def freeze(self, name):
1006
+ """SIGSTOP every task currently in the node's PostgreSQL cgroup.
1007
+
1008
+ Unlike a network partition this leaves sockets and processes alive, but
1009
+ the PostgreSQL node stops getting CPU time. This models a wedged VM,
1010
+ scheduler stall, or process group that has stopped responding.
1011
+ """
1012
+ node = self.nodes[name]
1013
+ pids = self._node_cgroup_pids(node)
1014
+ if not pids:
1015
+ raise PgLabError("node cgroup has no processes to freeze: {}".format(name))
1016
+
1017
+ stopped = []
1018
+ try:
1019
+ for pid in pids:
1020
+ try:
1021
+ os.kill(pid, signal.SIGSTOP)
1022
+ stopped.append(pid)
1023
+ except ProcessLookupError:
1024
+ pass
1025
+ except Exception:
1026
+ # Do not leave a partially frozen node after a failed injection.
1027
+ for pid in stopped:
1028
+ try:
1029
+ os.kill(pid, signal.SIGCONT)
1030
+ except ProcessLookupError:
1031
+ pass
1032
+ raise
1033
+
1034
+ def unfreeze(self, name):
1035
+ """SIGCONT all tasks currently present in the node cgroup."""
1036
+ node = self.nodes[name]
1037
+ for pid in self._node_cgroup_pids(node):
1038
+ try:
1039
+ os.kill(pid, signal.SIGCONT)
1040
+ except ProcessLookupError:
1041
+ pass
1042
+
1043
+ def partition(self, left, right):
1044
+ lnode = self.nodes[left]
1045
+ rnode = self.nodes[right]
1046
+ _run(["ip", "-n", lnode.namespace, "route", "replace", "blackhole",
1047
+ "{}/32".format(rnode.ip)])
1048
+ _run(["ip", "-n", rnode.namespace, "route", "replace", "blackhole",
1049
+ "{}/32".format(lnode.ip)])
1050
+
1051
+ def heal_partition(self, left, right):
1052
+ lnode = self.nodes[left]
1053
+ rnode = self.nodes[right]
1054
+ subprocess.run(["ip", "-n", lnode.namespace, "route", "del", "blackhole",
1055
+ "{}/32".format(rnode.ip)], check=False)
1056
+ subprocess.run(["ip", "-n", rnode.namespace, "route", "del", "blackhole",
1057
+ "{}/32".format(lnode.ip)], check=False)
1058
+
1059
+ def set_io_limits(self, name, limits, io_device=None):
1060
+ node = self.nodes[name]
1061
+ devices = [io_device] if io_device else self._backing_block_devices(Path(node.pg.base_dir))
1062
+ self._apply_io_limits(node.cgroup, devices, limits)
1063
+
1064
+ def remove_node(self, name):
1065
+ node = self.nodes.get(name)
1066
+ if node is None:
1067
+ return
1068
+
1069
+ # A frozen PostgreSQL cannot cooperate with pg_ctl stop/cleanup.
1070
+ # Restore process scheduling and remove traffic shaping first.
1071
+ try:
1072
+ self.unfreeze(name)
1073
+ except Exception:
1074
+ pass
1075
+ try:
1076
+ self.clear_netem(name)
1077
+ except Exception:
1078
+ pass
1079
+
1080
+ # PgLab removes the whole node directory itself below. Using
1081
+ # PostgresNode.cleanup() here would perform another stop attempt first.
1082
+ # That is noisy for two intentional PgLab states: an uninitialized node
1083
+ # and a postmaster that a fault-injection test already killed. stop()
1084
+ # itself always stops Testgres' log reader in its finally block, so call
1085
+ # it directly, tolerate the expected failure, and release Testgres-owned
1086
+ # resources before removing the isolation objects.
1087
+ try:
1088
+ node.pg.stop()
1089
+ except Exception:
1090
+ pass
1091
+ try:
1092
+ node.pg.release_resources()
1093
+ except Exception:
1094
+ pass
1095
+
1096
+ subprocess.run(["ip", "netns", "del", node.namespace],
1097
+ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
1098
+ subprocess.run(["ip", "link", "del", node.veth_host],
1099
+ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
1100
+ self.cgroup_backend.kill_node(node.cgroup)
1101
+ try:
1102
+ self.cgroup_backend.remove_node(node.cgroup)
1103
+ except OSError:
1104
+ pass
1105
+ if not self.keep_data:
1106
+ shutil.rmtree(str(self.base_dir / name), ignore_errors=True)
1107
+ del self.nodes[name]
1108
+
1109
+ def close(self):
1110
+ if self._closed:
1111
+ return
1112
+ for name in list(self.nodes.keys()):
1113
+ self.remove_node(name)
1114
+ self._cleanup_firewall()
1115
+ if self._created_bridge:
1116
+ subprocess.run(["ip", "link", "del", self.bridge_name],
1117
+ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
1118
+ self.cgroup_backend.cleanup_root()
1119
+ if not self.keep_data:
1120
+ try:
1121
+ self.base_dir.rmdir()
1122
+ except OSError:
1123
+ pass
1124
+ self._closed = True
1125
+
1126
+ def __enter__(self):
1127
+ return self
1128
+
1129
+ def __exit__(self, exc_type, exc, tb):
1130
+ self.close()
1131
+ return False
@@ -0,0 +1,217 @@
1
+ Metadata-Version: 2.4
2
+ Name: testgres_exts.pglab
3
+ Version: 0.1.0
4
+ Summary: Linux netns/cgroup/netem fault injection for PostgreSQL tests using Testgres
5
+ License-Expression: PostgreSQL
6
+ Classifier: Development Status :: 3 - Alpha
7
+ Classifier: Intended Audience :: Developers
8
+ Classifier: Operating System :: POSIX :: Linux
9
+ Classifier: Programming Language :: Python :: 3 :: Only
10
+ Classifier: Programming Language :: Python :: 3.8
11
+ Classifier: Programming Language :: Python :: 3.9
12
+ Classifier: Programming Language :: Python :: 3.10
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: Programming Language :: Python :: 3.13
16
+ Classifier: Programming Language :: Python :: 3.14
17
+ Classifier: Topic :: Software Development :: Testing
18
+ Classifier: Topic :: Database
19
+ Requires-Python: >=3.8
20
+ Description-Content-Type: text/markdown
21
+ License-File: LICENSE
22
+ Requires-Dist: testgres<1.18,>=1.17
23
+ Provides-Extra: test
24
+ Requires-Dist: pytest<10,>=8; extra == "test"
25
+ Dynamic: license-file
26
+
27
+ [![CI Status](https://img.shields.io/github/actions/workflow/status/postgrespro/testgres_exts.pglab/.github/workflows/ci.yml?label=CI)](https://github.com/postgrespro/testgres_exts.pglab/actions/workflows/ci.yml)
28
+ [![PyPI package version](https://badge.fury.io/py/testgres_exts_pglab.svg)](https://badge.fury.io/py/testgres_exts.pglab)
29
+ [![PyPI python versions](https://img.shields.io/pypi/pyversions/testgres_exts_pglab)](https://pypi.org/project/testgres-exts.pglab)
30
+
31
+ # PgLab
32
+
33
+ PgLab is a Linux fault-injection layer for PostgreSQL tests built on top of
34
+ [Testgres](https://github.com/postgrespro/testgres).
35
+
36
+ Testgres owns PostgreSQL lifecycle, SQL access, backups and physical replication.
37
+ PgLab owns Linux isolation and faults:
38
+
39
+ - network namespaces, veth pairs and a bridge;
40
+ - cgroup v1/v2 I/O throttling;
41
+ - network partitions;
42
+ - `tc netem` delay, jitter, loss, duplication, corruption, reordering and rate limits;
43
+ - freeze/unfreeze of a PostgreSQL process group;
44
+ - cleanup of namespaces, cgroups, qdiscs and temporary firewall rules.
45
+
46
+ PgLab currently requires Linux and root privileges because it creates network
47
+ namespaces, interfaces and cgroups. PostgreSQL itself runs as a configurable
48
+ unprivileged Unix account.
49
+
50
+ ## Why this needed changes in Testgres
51
+
52
+ Each PgLab node has a different `OsOperations` execution context: PostgreSQL
53
+ commands for a replica must run inside that replica's network namespace, and
54
+ server-starting commands must also enter its cgroup. A replica therefore cannot
55
+ simply inherit the primary node's `os_ops`, host and port-management context.
56
+
57
+ The Testgres target execution-context API lets PgLab keep the proper separation
58
+ of responsibilities: Testgres still owns `pg_basebackup`, physical replication
59
+ and `PostgresNode` lifecycle; PgLab only supplies the target execution context
60
+ and Linux fault injection.
61
+
62
+ See [docs/why-testgres-execution-context.md](docs/why-testgres-execution-context.md)
63
+ for the concrete primary/replica flow and the reason behind the upstream
64
+ Testgres changes.
65
+
66
+ ## Testgres compatibility
67
+
68
+ PgLab uses the target execution-context API in current Testgres:
69
+
70
+ ```python
71
+ primary.replicate(
72
+ target_os_ops=replica_ops,
73
+ target_host=replica_ip,
74
+ target_port=5432,
75
+ )
76
+ ```
77
+
78
+ The compatible Testgres tree must provide:
79
+
80
+ - `PostgresNode.replicate(target_os_ops=..., target_host=..., target_port=...)`;
81
+ - `NodeBackup(..., os_ops=...)`, so `pg_basebackup` can run in the target context;
82
+ - target execution-context support in node cloning.
83
+
84
+ The current development target is Testgres 1.17.0 with that API. Until those
85
+ changes are present in a released Testgres build, install Testgres from the
86
+ compatible checkout and install PgLab editable beside it.
87
+
88
+ ## Development install
89
+
90
+ With a compatible Testgres checkout in `../testgres`:
91
+
92
+ ```bash
93
+ ./setup.sh --testgres-dir ../testgres
94
+ ```
95
+
96
+ Or manually:
97
+
98
+ ```bash
99
+ python3 -m venv .venv
100
+ .venv/bin/python -m pip install -U pip setuptools wheel
101
+ .venv/bin/python -m pip install -e ../testgres
102
+ .venv/bin/python -m pip install -e '.[test]'
103
+ .venv/bin/python tools/verify_install.py
104
+ ```
105
+
106
+ When the required Testgres API is available from PyPI, a normal editable install
107
+ is enough:
108
+
109
+ ```bash
110
+ python3 -m venv .venv
111
+ .venv/bin/python -m pip install -e '.[test]'
112
+ ```
113
+
114
+ No `setup.py` is required. Packaging metadata lives in `pyproject.toml`.
115
+
116
+ ## PostgreSQL environment
117
+
118
+ Point PgLab at the PostgreSQL build you want to test:
119
+
120
+ ```bash
121
+ export PG_BIN=/home/user/build/postgres/bin
122
+ export PGLAB_OS_USER=user
123
+ export PGLAB_DB_USER=postgres
124
+ export PGLAB_TEST_ROOT=/var/tmp
125
+ ```
126
+
127
+ `PGLAB_TEST_ROOT` should live on a block-backed filesystem for cgroup I/O tests.
128
+ A tmpfs or overlay filesystem is fine for networking-only tests but cannot be
129
+ mapped reliably to a block device for `io.max`/blkio throttling.
130
+
131
+ Check the host:
132
+
133
+ ```bash
134
+ ./scripts/doctor.sh
135
+ ```
136
+
137
+ ## Example
138
+
139
+ ```python
140
+ from pglab import PgLab
141
+
142
+ with PgLab(
143
+ pg_bin="/home/user/build/postgres/bin",
144
+ base_dir="/var/tmp/pglab-demo",
145
+ subnet="10.203.0.0/24",
146
+ os_user="user",
147
+ db_superuser="postgres",
148
+ ) as lab:
149
+ primary = lab.create_primary("p1", "10.203.0.10", port=5432)
150
+ primary.init().start()
151
+
152
+ replica = lab.create_replica("p1", "r1", "10.203.0.11", port=5432)
153
+ replica.start()
154
+ replica.catchup()
155
+
156
+ replica.netem(delay_ms=100, loss_pct=1)
157
+ replica.clear_netem()
158
+
159
+ lab.partition("p1", "r1")
160
+ lab.heal_partition("p1", "r1")
161
+
162
+ replica.freeze()
163
+ replica.unfreeze()
164
+ ```
165
+
166
+ The same TCP port may be used by several nodes because each PostgreSQL server
167
+ runs in its own network namespace. PgLab disables Unix-domain PostgreSQL
168
+ sockets to avoid collisions in the shared mount namespace.
169
+
170
+ ## Tests
171
+
172
+ Smoke test:
173
+
174
+ ```bash
175
+ ./scripts/run-smoke.sh
176
+ ```
177
+
178
+ Full integration suite:
179
+
180
+ ```bash
181
+ ./scripts/run-tests.sh
182
+ ```
183
+
184
+ One file or test can be selected normally:
185
+
186
+ ```bash
187
+ ./scripts/run-tests.sh tests/test_pglab_netem_effects.py
188
+ ```
189
+
190
+ The integration suite covers primary/replica lifecycle, same-port replicas,
191
+ partitions and recovery, cgroup I/O controls, cleanup after failures, netem
192
+ latency/loss/rate effects, freeze/unfreeze, replication lag and an unrecoverable
193
+ WAL-gap scenario.
194
+
195
+ ## Project layout
196
+
197
+ ```text
198
+ pglab.py PgLab implementation
199
+ pyproject.toml Python package metadata
200
+ example.py small usage example
201
+ smoke.py end-to-end smoke test
202
+ tests/ integration and compatibility tests
203
+ scripts/doctor.sh environment diagnostics
204
+ scripts/run-tests.sh privileged test runner
205
+ scripts/run-smoke.sh privileged smoke runner
206
+ scripts/install-system-deps.sh host package helper
207
+ tools/verify_install.py Testgres API compatibility check
208
+ docs/why-testgres-execution-context.md
209
+ rationale for the Testgres target-context API
210
+ ```
211
+
212
+ PgLab is still experimental. The public API and packaging may change while the
213
+ corresponding Testgres execution-context API is being reviewed upstream.
214
+
215
+ ## License
216
+
217
+ PgLab is distributed under the PostgreSQL License. See [`LICENSE`](LICENSE).
@@ -0,0 +1,6 @@
1
+ testgres_exts/pglab/__init__.py,sha256=l1Jc7Zp3ofi7L9usl75HG-Z4XsRDwHh9u4CukuSzFT4,41358
2
+ testgres_exts_pglab-0.1.0.dist-info/licenses/LICENSE,sha256=NvRQMtgLIOzjx5s-K4oconnRwszGJbT0SpKsrr7Y6h8,1017
3
+ testgres_exts_pglab-0.1.0.dist-info/METADATA,sha256=he9hSov0Hf2yLWj0jzEXrYNuOH2zjwZunp6ngxGXwZA,7099
4
+ testgres_exts_pglab-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
5
+ testgres_exts_pglab-0.1.0.dist-info/top_level.txt,sha256=2CRTXcq9OpQHuExOmK7BIMAlzaTkYOQGAEm_V02ekFA,14
6
+ testgres_exts_pglab-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.0.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,18 @@
1
+ Copyright (c) 2026, PgLab contributors
2
+
3
+ Permission to use, copy, modify, and distribute this software and its
4
+ documentation for any purpose, without fee, and without a written agreement is
5
+ hereby granted, provided that the above copyright notice and this paragraph and
6
+ the following two paragraphs appear in all copies.
7
+
8
+ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE TO ANY PARTY FOR
9
+ DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST
10
+ PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF
11
+ THE AUTHORS OR COPYRIGHT HOLDERS HAVE BEEN ADVISED OF THE POSSIBILITY OF SUCH
12
+ DAMAGE.
13
+
14
+ THE AUTHORS AND COPYRIGHT HOLDERS SPECIFICALLY DISCLAIM ANY WARRANTIES,
15
+ INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
16
+ FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS
17
+ IS" BASIS, AND THE AUTHORS AND COPYRIGHT HOLDERS HAVE NO OBLIGATIONS TO PROVIDE
18
+ MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
@@ -0,0 +1 @@
1
+ testgres_exts